Update man pages to include newer options.

Signed-off-by: Sachidananda Urs <sac@gluster.com> Signed-off-by: Anand Avati <avati@gluster.com> BUG: 2444 (keep man pages upto date with new codebase.) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2444
author: Sachidananda <sac@gluster.com> 2011-06-16 07:56:33 +0000
committer: Anand Avati <avati@gluster.com> 2011-06-16 09:15:22 -0700
commit: dad63fa8ac00397e337f328b0bb7b641e2680932 (patch)
tree: 7d09c3dfc765d6719b1607cb2470beb3b48f93ee /doc
parent: 1d4378e0e78309682c13d0fc0c97c3c3bb6674b7 (diff)
3 files changed, 22 insertions, 3 deletions
diff --git a/doc/glusterd.8 b/doc/glusterd.8
index 43c1570f66a..720db738b7b 100644
--- a/doc/glusterd.8
+++ b/doc/glusterd.8
@@ -33,10 +33,10 @@ The glusterd daemon is used for elastic volume management. The daemon must be ru
 .PP
 .TP
 
-\fB\-l=<LOGFILE>, \fB\-\-log\-file=<LOGFILE>\fR
+\fB\-l <LOGFILE>, \fB\-\-log\-file=<LOGFILE>\fR
 File to use for logging.
 .TP
-\fB\-L=<LOGLEVEL>, \fB\-\-log\-level=<LOGLEVEL>\fR
+\fB\-L <LOGLEVEL>, \fB\-\-log\-level=<LOGLEVEL>\fR
 Logging severity.  Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is INFO).
 .TP
 \fB\-\-debug\fR
diff --git a/doc/glusterfs.8 b/doc/glusterfs.8
index 7e4b2b49d35..6033b0af55a 100644
--- a/doc/glusterfs.8
+++ b/doc/glusterfs.8
@@ -102,7 +102,7 @@ Enable/Disable the direct-I/O mode in fuse module (the default is enable).
 \fB\-?, \fB\-\-help\fR
 Display this help.
 .TP
-\fB\-\-usage\fReew
+\fB\-\-usage\fR
 Display a short usage message.
 .TP
 \fB\-V, \fB\-\-version\fR
@@ -111,6 +111,11 @@ Print the program version.
 .PP
 .SH FILES
 /etc/glusterfs/*.vol, /etc/glusterd/vols/*/*.vol
+.SH EXAMPLES
+mount a volume named foo on server bar with log level DEBUG on mount point
+/mnt/foo
+
+# glusterfs --log-level=DEBUG --volfile-id=foo --volfile-server=bar /mnt/foo
 
 .SH SEE ALSO
 .nf
diff --git a/doc/glusterfsd.8 b/doc/glusterfsd.8
index dbf3e3fda1f..2374918b211 100644
--- a/doc/glusterfsd.8
+++ b/doc/glusterfsd.8
@@ -73,6 +73,15 @@ Makes the filesystem read-only
 \fB\-p, \fB\-\-pid\-file=PIDFILE\fR
 File to use as pid file
 .TP
+\fB\-S SOCKFILE
+Socket file to used for inter-process communication
+.TP
+\fB\-\-brick\-name DIRECTORY
+Directory to be used as export directory for GlusterFS
+.TP
+\fB\-\-brick\-port PORT
+Brick Port to be registered with Gluster portmapper
+.TP
 \fB\-\-volfile\-id=KEY\fR
 KEY of the volume file to be fetched from server
 .TP
@@ -119,6 +128,11 @@ Print program version
 .SH FILES
 /etc/glusterfs/*.vol
 
+.SH EXAMPLES
+Start a GlusterFS server on localhost with volume name foo
+
+glusterfsd -s localhost --volfile-id foo.server.media-disk-1 -p /etc/glusterd/vols/foo/run/server-media-disk-1.pid -S /tmp/<uniqueid>.socket --brick-name /media/disk-1 -l /var/log/glusterfs/bricks/media-disk-1.log --brick-port 24009 --xlator-option foo-server.listen-port=24009
+
 .SH SEE ALSO
 .nf
 \fBfusermount\fR(1), \fBmount.glusterfs\fR(8), \fBgluster\fR(8)
diff --git a/.gitignore b/.gitignore
index c5371b264..ff253c1da 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,26 +8,41 @@ install-sh
 ltmain.sh
 Makefile.in
 missing
+py-compile
 *.sw?
 *~
-*lo
-*la
-*o
+*.lo
+*.la
+*.o
+*.tar.gz
+*.rpm
 .libs
+.deps
 Makefile
 stamp-h1
 
 # Generated files
-extras/init.d/glusterfs-server.plist
-extras/init.d/glusterfsd-Debian
-extras/init.d/glusterfsd-Redhat
-extras/init.d/glusterfsd-SuSE
-glusterfs.spec
-libtool
-xlators/mount/fuse/utils/mount.glusterfs
-xlators/mount/fuse/utils/mount_glusterfs
+api/examples/__init__.py*
+api/examples/setup.py
 argp-standalone/libargp.a
+contrib/uuid/uuid_types.h
+extras/init.d/glusterd-Debian
+extras/init.d/glusterd-Redhat
+extras/init.d/glusterd-SuSE
+extras/init.d/glusterd.plist
+extras/ocf/glusterd
+extras/ocf/volume
+extras/who-wrote-glusterfs/gitdm
+glusterfs-api.pc
+glusterfs.spec
 glusterfsd/src/glusterfsd
+libgfchangelog.pc
 libglusterfs/src/spec.lex.c
 libglusterfs/src/y.tab.c
 libglusterfs/src/y.tab.h
+libtool
+run-tests.sh
+ufo/.tox
+ufo/test/unit/.coverage
+xlators/mount/fuse/utils/mount.glusterfs
+xlators/mount/fuse/utils/mount_glusterfs
diff --git a/.mailmap b/.mailmap
new file mode 100644
index 000000000..6bcd95dea
--- /dev/null
+++ b/.mailmap
@@ -0,0 +1,31 @@
+# .mailmap, see 'git short-log --help' for details
+#
+# Listing of contributors that filed patches with different email addresses.
+# Format: <name> <main-email> <alias> [<alias> ...]
+#
+
+Amar Tumballi <amarts@redhat.com> <amar@gluster.com> <amar@del.gluster.com>
+Anand Avati <avati@redhat.com> <avati@gluster.com> <avati@dev.gluster.com> <avati@amp.gluster.com> <avati@blackhole.gluster.com>
+Anush Shetty <ashetty@redhat.com> <anush@gluster.com>
+Csaba Henk <csaba@redhat.com> <csaba@gluster.com> <csaba@lowlife.hu> <csaba@zresearch.com>
+Harshavardhana <fharshav@redhat.com> <harsha@gluster.com> <harsha@zresearch.com> <harsha@dev.gluster.com> <harsha@harshavardhana.net>
+Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com>
+Kaushal M <kaushal@redhat.com> <kaushal@gluster.com>
+Kaushik BV <kbudiger@redhat.com> <kaushikbv@gluster.com>
+Krishna Srinivas <ksriniva@redhat.com> <krishna@gluster.com> <krishna@zresearch.com> <krishna@guest-laptop>
+Krishnan Parthasarathi <kparthas@redhat.com> <kp@gluster.com>
+Louis Zuckerman <louiszuckerman@gmail.com> <me@louiszuckerman.com>
+M S Vishwanath Bhat <vbhat@redhat.com> <msvbhat@gmail.com> <vishwanath@gluster.com>
+Pavan Sondur <pavan@gluster.com> <pavan@dev.gluster.com>
+Pete Zaitcev <zaitcev@kotori.zaitcev.us> <zaitcev@yahoo.com>
+Pranith Kumar K <pkarampu@redhat.com> <pranithk@gluster.com>
+Raghavendra Bhat <raghavendra@redhat.com> <raghavendrabhat@gluster.com>
+Raghavendra G <rgowdapp@redhat.com> <raghavendra@gluster.com> <raghavendra@zresearch.com>
+Rahul C S <rahulcs@redhat.com> <rahulcssjce@gmail.com>
+Rajesh Amaravathi <rajesh@redhat.com> <rajesh@gluster.com> <rajesh.amaravathi@gmail.com>
+Shehjar Tikoo <shehjart@gluster.com> <shehjart@zresearch.com>
+Venky Shankar <vshankar@redhat.com> <venky@gluster.com>
+Vijay Bellur <vbellur@redhat.com> <vijay@gluster.com> <vijay@dev.gluster.com>
+Vijaykumar Koppad <vkoppad@redhat.com> <vijaykumar.koppad@gmail.com>
+Vikas Gorur <vikas@gluster.com> <vikas@zresearch.com>
+shishir gowda <sgowda@redhat.com> <shishirng@gluster.com>
diff --git a/CONTRIBUTING b/CONTRIBUTING
index 8b3baa7e5..7bccd88d7 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -22,4 +22,4 @@
             are public and that a record of the contribution (including all
             personal information I submit with it, including my sign-off) is
             maintained indefinitely and may be redistributed consistent with
-            this project or the open source license(s) involved.
-\ No newline at end of file
+            this project or the open source license(s) involved.
diff --git a/COPYING b/COPYING
deleted file mode 100644
index 5737cfa27..000000000
--- a/COPYING
+++ /dev/null
@@ -1,674 +0,0 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
-
-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
diff --git a/COPYING-GPLV2 b/COPYING-GPLV2
new file mode 100644
index 000000000..d159169d1
--- /dev/null
+++ b/COPYING-GPLV2
@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/COPYING-LGPLV3 b/COPYING-LGPLV3
new file mode 100644
index 000000000..65c5ca88a
--- /dev/null
+++ b/COPYING-LGPLV3
@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/Makefile.am b/Makefile.am
index 85e0a13f1..598ebb410 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,8 +1,18 @@
-EXTRA_DIST = autogen.sh COPYING INSTALL README AUTHORS THANKS NEWS glusterfs.spec
+EXTRA_DIST = autogen.sh \
+	COPYING-GPLV2 COPYING-LGPLV3 \
+	INSTALL README AUTHORS THANKS NEWS \
+	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in \
+	error-codes.json gf-error-codes.h.template \
+	gen-headers.py run-tests.sh \
+	$(shell find $(top_srcdir)/tests -type f -print)
 
-SUBDIRS = argp-standalone libglusterfs rpc xlators glusterfsd $(FUSERMOUNT_SUBDIR) doc extras cli
+SUBDIRS = argp-standalone libglusterfs rpc api xlators glusterfsd \
+	$(FUSERMOUNT_SUBDIR) doc extras cli @SYNCDAEMON_SUBDIR@
 
-CLEANFILES = 
+pkgconfigdir = @pkgconfigdir@
+pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc
+
+CLEANFILES =
 
 gitclean: distclean
 	find . -name Makefile.in -exec rm -f {} \;
@@ -11,4 +21,9 @@ gitclean: distclean
 	rm -fr autom4te.cache
 	rm -f missing aclocal.m4 config.h.in config.guess config.sub ltmain.sh install-sh configure depcomp
 	rm -fr argp-standalone/autom4te.cache
-	rm -f argp-standalone/aclocal.m4 argp-standalone/config.h.in argp-standalone/configure argp-standalone/depcomp argp-standalone/install-sh argp-standalone/missing
+	rm -f argp-standalone/aclocal.m4 argp-standalone/config.h.in
+	rm -f argp-standalone/configure argp-standalone/depcomp
+	rm -f argp-standalone/install-sh argp-standalone/missing
+
+dist-hook:
+	(cd $(srcdir) && git diff && echo ===== git log ==== && git log) > $(distdir)/ChangeLog
diff --git a/api/Makefile.am b/api/Makefile.am
new file mode 100644
index 000000000..f0ad1ee97
--- /dev/null
+++ b/api/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = src examples
diff --git a/api/examples/Makefile.am b/api/examples/Makefile.am
new file mode 100644
index 000000000..05f40ff53
--- /dev/null
+++ b/api/examples/Makefile.am
@@ -0,0 +1,6 @@
+EXTRA_PROGRAMS = glfsxmp
+glfsxmp_SOURCES = glfsxmp.c
+glfsxmp_CFLAGS = $(GLFS_CFLAGS) -Wall
+glfsxmp_LDADD = $(GLFS_LIBS) -lrt
+
+EXTRA_DIST = gfapi.py
diff --git a/api/examples/README b/api/examples/README
new file mode 100644
index 000000000..4d2b521f7
--- /dev/null
+++ b/api/examples/README
@@ -0,0 +1,36 @@
+This is an example application which uses libgfapi. It is
+a complete autotools based build system which demonstrates the
+required changes in configure.ac, Makefile.am etc to successfuly
+detect for and build an application against libgfapi.
+
+There are two approaches to building a libgfapi based application:
+
+1. In the presence of pkg-config in your build system.
+This is the recommended approach which is also used in this example.
+For this approach to work, you need to build glusterfs by passing
+--pkgconfigdir=/usr/lib64/pkgconfig (or the appropriate directory)
+in your distro. This already happens if you build RPMs with the
+glusterfs.spec provided in glusterfs.git. You will also need to
+install glusterfs-api RPM.
+
+2. In the absence of pkg-config in your build system.
+Make sure your LDFLAGS includes -L/path/to/lib where libgfapi.so is
+installed and -I/path/to/include/glusterfs where the 'api' directory
+containing the headers are available.
+
+glfsxmp.c
+=========
+
+glfsxmp.c is an example application which uses libgfapi
+
+Compilation Steps For glfsxmp.c
+===============================
+
+1. $./autogen.sh
+2. $./configure
+
+Note: Before running ./configure , as mentioned above, you need to
+      take care of #1 or #2 i.e. pkg-config path or LDFLAGS and
+      -I/<path> with correct values.
+
+3. $make glfsxmp
diff --git a/api/examples/autogen.sh b/api/examples/autogen.sh
new file mode 100755
index 000000000..1fee6be11
--- /dev/null
+++ b/api/examples/autogen.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+aclocal
+autoconf
+automake --foreign
diff --git a/api/examples/configure.ac b/api/examples/configure.ac
new file mode 100644
index 000000000..b80177a4e
--- /dev/null
+++ b/api/examples/configure.ac
@@ -0,0 +1,12 @@
+
+AC_INIT([glfs-test],[0.1],[gluster-devel@nongu.org])
+
+AM_INIT_AUTOMAKE
+
+AC_CONFIG_FILES([Makefile])
+
+AC_PROG_CC
+
+PKG_CHECK_MODULES([GLFS], [glusterfs-api >= 3])
+
+AC_OUTPUT
diff --git a/api/examples/gfapi.py b/api/examples/gfapi.py
new file mode 100755
index 000000000..3ac67f4d5
--- /dev/null
+++ b/api/examples/gfapi.py
@@ -0,0 +1,422 @@
+#!/usr/bin/python
+
+from ctypes import *
+from ctypes.util import find_library
+import os
+import sys
+import time
+import types
+
+# Looks like ctypes is having trouble with dependencies, so just force them to
+# load with RTLD_GLOBAL until I figure that out.
+glfs = CDLL(find_library("glusterfs"),RTLD_GLOBAL)
+xdr = CDLL(find_library("gfxdr"),RTLD_GLOBAL)
+api = CDLL(find_library("gfapi"),RTLD_GLOBAL)
+
+# Wow, the Linux kernel folks really play nasty games with this structure.  If
+# you look at the man page for stat(2) and then at this definition you'll note
+# two discrepancies.  First, we seem to have st_nlink and st_mode reversed.  In
+# fact that's exactly how they're defined *for 64-bit systems*; for 32-bit
+# they're in the man-page order.  Even uglier, the man page makes no mention of
+# the *nsec fields, but they are very much present and if they're not included
+# then we get memory corruption because libgfapi has a structure definition
+# that's longer than ours and they overwrite some random bit of memory after
+# the space we allocated.  Yes, that's all very disgusting, and I'm still not
+# sure this will really work on 32-bit because all of the field types are so
+# obfuscated behind macros and feature checks.
+class Stat (Structure):
+        _fields_ = [
+                ("st_dev",              c_ulong),
+                ("st_ino",              c_ulong),
+                ("st_nlink",            c_ulong),
+                ("st_mode",             c_uint),
+                ("st_uid",              c_uint),
+                ("st_gid",              c_uint),
+                ("st_rdev",             c_ulong),
+                ("st_size",             c_ulong),
+                ("st_blksize",          c_ulong),
+                ("st_blocks",           c_ulong),
+                ("st_atime",            c_ulong),
+                ("st_atimensec",        c_ulong),
+                ("st_mtime",            c_ulong),
+                ("st_mtimensec",        c_ulong),
+                ("st_ctime",            c_ulong),
+                ("st_ctimensec",        c_ulong),
+        ]
+api.glfs_creat.restype = c_void_p
+api.glfs_open.restype = c_void_p
+api.glfs_lstat.restype = c_int
+api.glfs_lstat.argtypes = [c_void_p, c_char_p, POINTER(Stat)]
+
+class Dirent (Structure):
+        _fields_ = [
+                ("d_ino",       c_ulong),
+                ("d_off",       c_ulong),
+                ("d_reclen",    c_ushort),
+                ("d_type",      c_char),
+                ("d_name",      c_char * 256),
+        ]
+api.glfs_opendir.restype = c_void_p
+api.glfs_readdir_r.restype = c_int
+api.glfs_readdir_r.argtypes = [c_void_p, POINTER(Dirent),
+                               POINTER(POINTER(Dirent))]
+
+# There's a bit of ctypes glitchiness around __del__ functions and module-level
+# variables.  If we unload the module while we still have references to File or
+# Volume objects, the module-level variables might have disappeared by the time
+# __del__ gets called.  Therefore the objects hold references which they
+# release when __del__ is done.  We only actually use the object-local values
+# in __del__; for clarity, we just use the simpler module-level form elsewhere.
+
+class File(object):
+
+        def __init__ (self, fd):
+                # Add a reference so the module-level variable "api" doesn't
+                # get yanked out from under us (see comment above File def'n).
+                self._api = api
+                self.fd = fd
+
+        def __del__ (self):
+                self._api.glfs_close(self.fd)
+                self._api = None
+
+        # File operations, in alphabetical order.
+
+        def fsync (self):
+                return api.glfs_fsync(self.fd)
+
+        def read (self, buflen, flags=0):
+                rbuf = create_string_buffer(buflen)
+                rc = api.glfs_read(self.fd,rbuf,buflen,flags)
+                if rc > 0:
+                        return rbuf.value[:rc]
+                else:
+                        return rc
+
+        def read_buffer (self, buf, flags=0):
+                return api.glfs_read(self.fd,buf,len(buf),flags)
+
+        def write (self, data, flags=0):
+                return api.glfs_write(self.fd,data,len(data),flags)
+
+        def fallocate (self, mode, offset, len):
+            return api.glfs_fallocate(self.fd, mode, offset, len)
+
+        def discard (self, offset, len):
+            return api.glfs_discard(self.fd, offset, len)
+
+
+class Dir(object):
+
+        def __init__ (self, fd):
+                # Add a reference so the module-level variable "api" doesn't
+                # get yanked out from under us (see comment above File def'n).
+                self._api = api
+                self.fd = fd
+                self.cursor = POINTER(Dirent)()
+
+        def __del__ (self):
+                self._api.glfs_closedir(self.fd)
+                self._api = None
+
+        def next (self):
+                entry = Dirent()
+                entry.d_reclen = 256
+                rc = api.glfs_readdir_r(self.fd,byref(entry),byref(self.cursor))
+                if (rc < 0) or (not self.cursor) or (not self.cursor.contents):
+                        return rc
+                return entry
+
+class Volume(object):
+
+        # Housekeeping functions.
+
+        def __init__ (self, host, volid, proto="tcp", port=24007):
+                # Add a reference so the module-level variable "api" doesn't
+                # get yanked out from under us (see comment above File def'n).
+                self._api = api
+                self.fs = api.glfs_new(volid)
+                api.glfs_set_volfile_server(self.fs,proto,host,port)
+
+        def __del__ (self):
+                self._api.glfs_fini(self.fs)
+                self._api = None
+
+        def set_logging (self, path, level):
+                api.glfs_set_logging(self.fs,path,level)
+
+        def mount (self):
+                api.glfs_init(self.fs)
+
+        # File operations, in alphabetical order.
+
+        def creat (self, path, flags, mode):
+                fd = api.glfs_creat(self.fs,path,flags,mode)
+                if not fd:
+                        return fd
+                return File(fd)
+
+        def getxattr (self, path, key, maxlen):
+                buf = create_string_buffer(maxlen)
+                rc = api.glfs_getxattr(self.fs,path,key,buf,maxlen)
+                if rc < 0:
+                        return rc
+                return buf.value[:rc]
+
+        def listxattr (self, path):
+                buf = create_string_buffer(512)
+                rc = api.glfs_listxattr(self.fs,path,buf,512)
+                if rc < 0:
+                        return rc
+                xattrs = []
+                # Parsing character by character is ugly, but it seems like the
+                # easiest way to deal with the "strings separated by NUL in one
+                # buffer" format.
+                i = 0
+                while i < rc:
+                        new_xa = buf.raw[i]
+                        i += 1
+                        while i < rc:
+                                next_char = buf.raw[i]
+                                i += 1
+                                if next_char == '\0':
+                                        xattrs.append(new_xa)
+                                        break
+                                new_xa += next_char
+                xattrs.sort()
+                return xattrs
+
+        def lstat (self, path):
+                x = Stat()
+                rc = api.glfs_lstat(self.fs,path,byref(x))
+                if rc >= 0:
+                        return x
+                else:
+                        return rc
+
+        def mkdir (self, path):
+                return api.glfs_mkdir(self.fs,path)
+
+        def open (self, path, flags):
+                fd = api.glfs_open(self.fs,path,flags)
+                if not fd:
+                        return fd
+                return File(fd)
+
+        def opendir (self, path):
+                fd = api.glfs_opendir(self.fs,path)
+                if not fd:
+                        return fd
+                return Dir(fd)
+
+        def rename (self, opath, npath):
+                return api.glfs_rename(self.fs,opath,npath)
+
+        def rmdir (self, path):
+                return api.glfs_rmdir(self.fs,path)
+
+        def setxattr (self, path, key, value, vlen):
+                return api.glfs_setxattr(self.fs,path,key,value,vlen,0)
+
+        def unlink (self, path):
+                return api.glfs_unlink(self.fs,path)
+
+if __name__ == "__main__":
+        def test_create_write (vol, path, data):
+                mypath = path + ".io"
+                fd = vol.creat(mypath,os.O_WRONLY|os.O_EXCL,0644)
+                if not fd:
+                        return False, "creat error"
+                rc = fd.write(data)
+                if rc != len(data):
+                        return False, "wrote %d/%d bytes" % (rc, len(data))
+                return True, "wrote %d bytes" % rc
+
+        # TBD: this test fails if we do create, open, write, read
+        def test_open_read (vol, path, data):
+                mypath = path + ".io"
+                fd = vol.open(mypath,os.O_RDONLY)
+                if not fd:
+                        return False, "open error"
+                dlen = len(data) * 2
+                buf = fd.read(dlen)
+                if type(buf) == types.IntType:
+                        return False, "read error %d" % buf
+                if len(buf) != len(data):
+                        return False, "read %d/%d bytes" % (len(buf), len(data))
+                return True, "read '%s'" % buf
+
+        def test_lstat (vol, path, data):
+                mypath = path + ".io"
+                sb = vol.lstat(mypath)
+                if type(sb) == types.IntType:
+                        return False, "lstat error %d" % sb
+                if sb.st_size != len(data):
+                        return False, "lstat size is %d, expected %d" % (
+                                sb.st_size, len(data))
+                return True, "lstat got correct size %d" % sb.st_size
+
+        def test_rename (vol, path, data):
+                opath = path + ".io"
+                npath = path + ".tmp"
+                rc = vol.rename(opath,npath)
+                if rc < 0:
+                        return False, "rename error %d" % rc
+                ofd = vol.open(opath,os.O_RDWR)
+                if isinstance(ofd,File):
+                        return False, "old path working after rename"
+                nfd = vol.open(npath,os.O_RDWR)
+                if isinstance(nfd,File):
+                        return False, "new path not working after rename"
+                return True, "rename worked"
+
+        def test_unlink (vol, path, data):
+                mypath = path + ".tmp"
+                rc = vol.unlink(mypath)
+                if rc < 0:
+                        return False, "unlink error %d" % fd
+                fd = vol.open(mypath,os.O_RDWR)
+                if isinstance(fd,File):
+                        return False, "path still usable after unlink"
+                return True, "unlink worked"
+
+        def test_mkdir (vol, path, data):
+                mypath = path + ".dir"
+                rc = vol.mkdir(mypath)
+                if rc < 0:
+                        return False, "mkdir error %d" % rc
+                return True, "mkdir worked"
+
+        def test_create_in_dir (vol, path, data):
+                mypath = path + ".dir/probe"
+                fd = vol.creat(mypath,os.O_RDWR,0644)
+                if not isinstance(fd,File):
+                        return False, "create (in dir) error"
+                return True, "create (in dir) worked"
+
+        def test_dir_listing (vol, path, data):
+                mypath = path + ".dir"
+                fd = vol.opendir(mypath)
+                if not isinstance(fd,Dir):
+                        return False, "opendir error %d" % fd
+                files = []
+                while True:
+                        ent = fd.next()
+                        if not isinstance(ent,Dirent):
+                                break
+                        name = ent.d_name[:ent.d_reclen]
+                        files.append(name)
+                if files != [".", "..", "probe"]:
+                        return False, "wrong directory contents"
+                return True, "directory listing worked"
+
+        def test_unlink_in_dir (vol, path, data):
+                mypath = path + ".dir/probe"
+                rc = vol.unlink(mypath)
+                if rc < 0:
+                        return False, "unlink (in dir) error %d" % rc
+                return True, "unlink (in dir) worked"
+
+        def test_rmdir (vol, path, data):
+                mypath = path + ".dir"
+                rc = vol.rmdir(mypath)
+                if rc < 0:
+                        return False, "rmdir error %d" % rc
+                sb = vol.lstat(mypath)
+                if not isinstance(sb,Stat):
+                        return False, "dir still there after rmdir"
+                return True, "rmdir worked"
+
+        def test_setxattr (vol, path, data):
+                mypath = path + ".xa"
+                fd = vol.creat(mypath,os.O_RDWR|os.O_EXCL,0644)
+                if not fd:
+                        return False, "creat (xattr test) error"
+                key1, key2 = "hello", "goodbye"
+                if vol.setxattr(mypath,"trusted.key1",key1,len(key1)) < 0:
+                        return False, "setxattr (key1) error"
+                if vol.setxattr(mypath,"trusted.key2",key2,len(key2)) < 0:
+                        return False, "setxattr (key2) error"
+                return True, "setxattr worked"
+
+        def test_getxattr (vol, path, data):
+                mypath = path + ".xa"
+                buf = vol.getxattr(mypath,"trusted.key1",32)
+                if type(buf) == types.IntType:
+                        return False, "getxattr error"
+                if buf != "hello":
+                        return False, "wrong getxattr value %s" % buf
+                return True, "getxattr worked"
+
+        def test_listxattr (vol, path, data):
+                mypath = path + ".xa"
+                xattrs = vol.listxattr(mypath)
+                if type(xattrs) == types.IntType:
+                        return False, "listxattr error"
+                if xattrs != ["trusted.key1","trusted.key2"]:
+                        return False, "wrong listxattr value %s" % repr(xattrs)
+                return True, "listxattr worked"
+
+        def test_fallocate (vol, path, data):
+                mypath = path + ".io"
+                fd = vol.creat(mypath,os.O_WRONLY|os.O_EXCL,0644)
+                if not fd:
+                        return False, "creat error"
+		rc = fd.fallocate(0, 0, 1024*1024)
+                if rc != 0:
+                        return False, "fallocate error"
+		rc = fd.discard(4096, 4096)
+		if rc != 0:
+			return False, "discard error"
+                return True, "fallocate/discard worked"
+
+        test_list = (
+                test_create_write,
+                test_open_read,
+                test_lstat,
+                test_rename,
+                test_unlink,
+                test_mkdir,
+                test_create_in_dir,
+                test_dir_listing,
+                test_unlink_in_dir,
+                test_rmdir,
+                test_setxattr,
+                test_getxattr,
+                test_listxattr,
+                test_fallocate,
+        )
+
+        ok_to_fail = (
+                # TBD: this fails opening the new file, even though the file
+                # did get renamed.  Looks like a gfapi bug, not ours.
+                (test_rename, "new path not working after rename"),
+                # TBD: similar, call returns error even though it worked
+                (test_rmdir, "dir still there after rmdir"),
+        )
+
+        volid, path = sys.argv[1:3]
+        data = "fubar"
+        vol = Volume("localhost",volid)
+        vol.set_logging("/dev/null",7)
+        #vol.set_logging("/dev/stderr",7)
+        vol.mount()
+
+        failures = 0
+        expected = 0
+        for t in test_list:
+                rc, msg = t(vol,path,data)
+                if rc:
+                        print "PASS: %s" % msg
+                else:
+                        print "FAIL: %s" % msg
+                        failures += 1
+                        for otf in ok_to_fail:
+                                if (t == otf[0]) and (msg == otf[1]):
+                                        print "  (skipping known failure)"
+                                        expected += 1
+                                        break # from the *inner* for loop
+                        else:
+                                break # from the *outer* for loop
+
+        print "%d failures (%d expected)" % (failures, expected)
diff --git a/api/examples/glfsxmp.c b/api/examples/glfsxmp.c
new file mode 100644
index 000000000..600d72fb5
--- /dev/null
+++ b/api/examples/glfsxmp.c
@@ -0,0 +1,1598 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include "api/glfs.h"
+#include "api/glfs-handles.h"
+#include <string.h>
+#include <time.h>
+
+
+int
+test_dirops (glfs_t *fs)
+{
+        glfs_fd_t *fd = NULL;
+        char buf[512];
+        struct dirent *entry = NULL;
+
+        fd = glfs_opendir (fs, "/");
+        if (!fd) {
+                fprintf (stderr, "/: %s\n", strerror (errno));
+                return -1;
+        }
+
+        fprintf (stderr, "Entries:\n");
+        while (glfs_readdir_r (fd, (struct dirent *)buf, &entry), entry) {
+                fprintf (stderr, "%s: %lu\n", entry->d_name, glfs_telldir (fd));
+        }
+
+        glfs_closedir (fd);
+        return 0;
+}
+
+
+int
+test_xattr (glfs_t *fs)
+{
+        char *filename = "/filename2";
+        char buf[512];
+        char *ptr;
+        int ret;
+
+        ret = glfs_setxattr (fs, filename, "user.testkey", "testval", 8, 0);
+        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
+                 strerror (errno));
+
+        ret = glfs_setxattr (fs, filename, "user.testkey2", "testval", 8, 0);
+        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
+                 strerror (errno));
+
+        ret = glfs_listxattr (fs, filename, buf, 512);
+        fprintf (stderr, "listxattr(%s): %d (%s)\n", filename, ret,
+                 strerror (errno));
+        if (ret < 0)
+                return -1;
+
+        for (ptr = buf; ptr < buf + ret; ptr++) {
+                printf ("key=%s\n", ptr);
+                ptr += strlen (ptr);
+        }
+
+        return 0;
+}
+
+
+int
+test_chdir (glfs_t *fs)
+{
+        int ret = -1;
+        char *topdir = "/topdir";
+        char *linkdir = "/linkdir";
+        char *subdir = "./subdir";
+        char *respath = NULL;
+        char pathbuf[4096];
+
+        ret = glfs_mkdir (fs, topdir, 0755);
+        if (ret) {
+                fprintf (stderr, "mkdir(%s): %s\n", topdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_getcwd (fs, pathbuf, 4096);
+        fprintf (stdout, "getcwd() = %s\n", respath);
+
+        ret = glfs_symlink (fs, topdir, linkdir);
+        if (ret) {
+                fprintf (stderr, "symlink(%s, %s): %s\n", topdir, linkdir, strerror (errno));
+                return -1;
+        }
+
+        ret = glfs_chdir (fs, linkdir);
+        if (ret) {
+                fprintf (stderr, "chdir(%s): %s\n", linkdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_getcwd (fs, pathbuf, 4096);
+        fprintf (stdout, "getcwd() = %s\n", respath);
+
+        respath = glfs_realpath (fs, subdir, pathbuf);
+        if (respath) {
+                fprintf (stderr, "realpath(%s) worked unexpectedly: %s\n", subdir, respath);
+                return -1;
+        }
+
+        ret = glfs_mkdir (fs, subdir, 0755);
+        if (ret) {
+                fprintf (stderr, "mkdir(%s): %s\n", subdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_realpath (fs, subdir, pathbuf);
+        if (!respath) {
+                fprintf (stderr, "realpath(%s): %s\n", subdir, strerror (errno));
+        } else {
+                fprintf (stdout, "realpath(%s) = %s\n", subdir, respath);
+        }
+
+        ret = glfs_chdir (fs, subdir);
+        if (ret) {
+                fprintf (stderr, "chdir(%s): %s\n", subdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_getcwd (fs, pathbuf, 4096);
+        fprintf (stdout, "getcwd() = %s\n", respath);
+
+        respath = glfs_realpath (fs, "/linkdir/subdir", pathbuf);
+        if (!respath) {
+                fprintf (stderr, "realpath(/linkdir/subdir): %s\n", strerror (errno));
+        } else {
+                fprintf (stdout, "realpath(/linkdir/subdir) = %s\n", respath);
+        }
+
+        return 0;
+}
+
+#ifdef DEBUG
+static void
+peek_stat (struct stat *sb)
+{
+        printf ("Dumping stat information:\n");
+        printf ("File type:                ");
+
+        switch (sb->st_mode & S_IFMT) {
+                case S_IFBLK:  printf ("block device\n");            break;
+                case S_IFCHR:  printf ("character device\n");        break;
+                case S_IFDIR:  printf ("directory\n");               break;
+                case S_IFIFO:  printf ("FIFO/pipe\n");               break;
+                case S_IFLNK:  printf ("symlink\n");                 break;
+                case S_IFREG:  printf ("regular file\n");            break;
+                case S_IFSOCK: printf ("socket\n");                  break;
+                default:       printf ("unknown?\n");                break;
+        }
+
+        printf ("I-node number:            %ld\n", (long) sb->st_ino);
+
+        printf ("Mode:                     %lo (octal)\n",
+                (unsigned long) sb->st_mode);
+
+        printf ("Link count:               %ld\n", (long) sb->st_nlink);
+        printf ("Ownership:                UID=%ld   GID=%ld\n",
+                (long) sb->st_uid, (long) sb->st_gid);
+
+        printf ("Preferred I/O block size: %ld bytes\n",
+                (long) sb->st_blksize);
+        printf ("File size:                %lld bytes\n",
+                (long long) sb->st_size);
+        printf ("Blocks allocated:         %lld\n",
+                (long long) sb->st_blocks);
+
+        printf ("Last status change:       %s", ctime(&sb->st_ctime));
+        printf ("Last file access:         %s", ctime(&sb->st_atime));
+        printf ("Last file modification:   %s", ctime(&sb->st_mtime));
+
+        return;
+}
+
+static void
+peek_handle (unsigned char *glid)
+{
+        int i;
+
+        for (i = 0; i < GFAPI_HANDLE_LENGTH; i++)
+        {
+                printf (":%02x:", glid[i]);
+        }
+        printf ("\n");
+}
+#else /* DEBUG */
+static void
+peek_stat (struct stat *sb)
+{
+        return;
+}
+
+static void
+peek_handle (unsigned char *id)
+{
+        return;
+}
+#endif /* DEBUG */
+
+glfs_t    *fs = NULL;
+char      *full_parent_name = "/testdir", *parent_name = "testdir";
+
+void
+test_h_unlink (void)
+{
+        char               *my_dir = "unlinkdir";
+        char               *my_file = "file.txt";
+        char               *my_subdir = "dir1";
+        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
+                           *subdir = NULL, *subleaf = NULL;
+        struct stat         sb;
+        int                 ret;
+
+        printf ("glfs_h_unlink tests: In Progress\n");
+
+        /* Prepare tests */
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
+        if (dir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        subdir = glfs_h_mkdir (fs, dir, my_subdir, 0644, &sb);
+        if (subdir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_subdir, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        subleaf = glfs_h_creat (fs, subdir, my_file, O_CREAT, 0644, &sb);
+        if (subleaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, subdir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink non empty directory */
+        ret = glfs_h_unlink (fs, dir, my_subdir);
+        if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
+                         my_subdir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink regular file */
+        ret = glfs_h_unlink (fs, subdir, my_file);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_file, subdir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink directory */
+        ret = glfs_h_unlink (fs, dir, my_subdir);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_subdir, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink regular file */
+        ret = glfs_h_unlink (fs, dir, my_file);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_file, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink non-existant regular file */
+        ret = glfs_h_unlink (fs, dir, my_file);
+        if ((ret && errno != ENOENT) || (ret == 0)) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking non-existant %s: invalid errno ,%d, %s\n",
+                         my_file, ret, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink non-existant directory */
+        ret = glfs_h_unlink (fs, dir, my_subdir);
+        if ((ret && errno != ENOENT) || (ret == 0)) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking non-existant %s:  invalid errno ,%d, %s\n",
+                         my_subdir, ret, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink directory */
+        ret = glfs_h_unlink (fs, parent, my_dir);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        printf ("glfs_h_unlink tests: PASSED\n");
+
+out:
+        if (dir)
+                glfs_h_close (dir);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (subdir)
+                glfs_h_close (subdir);
+        if (subleaf)
+                glfs_h_close (subleaf);
+        if (parent)
+                glfs_h_close (parent);
+
+        return;
+}
+
+void
+test_h_getsetattrs (void)
+{
+        char               *my_dir = "attrdir";
+        char               *my_file = "attrfile.txt";
+        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+        struct stat         sb, retsb;
+        int                 ret, valid;
+        struct timespec     timestamp;
+
+        printf("glfs_h_getattrs and setattrs tests: In Progress\n");
+
+        /* Prepare tests */
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
+        if (dir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        ret = glfs_h_getattrs (fs, dir, &retsb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&retsb);
+        /* TODO: Compare stat information */
+
+        retsb.st_mode = 00666;
+        retsb.st_uid = 1000;
+        retsb.st_gid = 1001;
+        ret = clock_gettime (CLOCK_REALTIME, &timestamp);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+        retsb.st_atim = timestamp;
+        retsb.st_mtim = timestamp;
+        valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
+        GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
+        peek_stat (&retsb);
+
+        ret = glfs_h_setattrs (fs, dir, &retsb, valid);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_setattrs: error %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        memset(&retsb, 0, sizeof (struct stat));
+        ret = glfs_h_stat (fs, dir, &retsb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_stat: error %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&retsb);
+
+        printf ("glfs_h_getattrs and setattrs tests: PASSED\n");
+out:
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (dir)
+                glfs_h_close (dir);
+
+        return;
+}
+
+void
+test_h_truncate (void)
+{
+        char               *my_dir = "truncatedir";
+        char               *my_file = "file.txt";
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
+        struct stat         sb;
+        glfs_fd_t          *fd = NULL;
+        char                buf[32];
+        off_t               offset = 0;
+        int                 ret = 0;
+
+        printf("glfs_h_truncate tests: In Progress\n");
+
+        /* Prepare tests */
+        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, root, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        fd = glfs_h_open (fs, leaf, O_RDWR);
+        if (fd == NULL) {
+                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
+                         my_file, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        memcpy (buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+        ret = glfs_write (fd, buf, 32, 0);
+
+        /* run tests */
+        /* truncate lower */
+        offset = 30;
+        ret = glfs_h_truncate (fs, leaf, offset);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        ret = glfs_h_getattrs (fs, leaf, &sb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
+                         my_file, leaf, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        if (sb.st_size != offset) {
+                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        /* truncate higher */
+        offset = 32;
+        ret = glfs_h_truncate (fs, leaf, offset);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        ret = glfs_h_getattrs (fs, leaf, &sb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
+                         my_file, leaf, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        if (sb.st_size != offset) {
+                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        /* truncate equal */
+        offset = 30;
+        ret = glfs_h_truncate (fs, leaf, offset);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        ret = glfs_h_getattrs (fs, leaf, &sb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
+                         my_file, leaf, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        if (sb.st_size != offset) {
+                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        printf ("glfs_h_truncate tests: PASSED\n");
+out:
+        if (fd)
+                glfs_close (fd);
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+
+        return;
+}
+
+void
+test_h_links (void)
+{
+        char               *my_dir = "linkdir";
+        char               *my_file = "file.txt";
+        char               *my_symlnk = "slnk.txt";
+        char               *my_lnk = "lnk.txt";
+        char               *linksrc_dir = "dir1";
+        char               *linktgt_dir = "dir2";
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+        struct glfs_object *ln1 = NULL;
+        struct stat         sb;
+        int                 ret;
+        char               *buf = NULL;
+
+        printf("glfs_h_link(s) tests: In Progress\n");
+
+        /* Prepare tests */
+        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, root, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirsrc = glfs_h_mkdir (fs, parent, linksrc_dir, 0644, &sb);
+        if (dirsrc == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         linksrc_dir, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirtgt = glfs_h_mkdir (fs, parent, linktgt_dir, 0644, &sb);
+        if (dirtgt == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         linktgt_dir, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+        if (dleaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dirsrc, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* run tests */
+        /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
+        ln1 = glfs_h_symlink (fs, parent, my_symlnk, "./file.txt", &sb);
+        if (ln1 == NULL) {
+                fprintf (stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
+                         my_symlnk, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        buf = calloc (1024, sizeof(char));
+        if (buf == NULL) {
+                fprintf (stderr, "Error allocating memory\n");
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+
+        ret = glfs_h_readlink (fs, ln1, buf, 1024);
+        if (ret <= 0) {
+                fprintf (stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
+                         my_symlnk, ln1, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        if (!(strncmp (buf, my_symlnk, strlen (my_symlnk)))) {
+                fprintf (stderr, "glfs_h_readlink: error mismatch in link name: actual %s: retrieved %s\n",
+                         my_symlnk, buf);
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+
+        /* link: /testdir/linkdir/file.txt to ./lnk.txt */
+        ret = glfs_h_link (fs, leaf, parent, my_lnk);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                         my_lnk, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        /* TODO: Should write content to a file and read from the link */
+
+        /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
+        ret = glfs_h_link (fs, dleaf, dirtgt, my_lnk);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                         my_lnk, dirtgt, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        /* TODO: Should write content to a file and read from the link */
+
+        printf ("glfs_h_link(s) tests: PASSED\n");
+
+out:
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (dirsrc)
+                glfs_h_close (dirsrc);
+        if (dirtgt)
+                glfs_h_close (dirtgt);
+        if (dleaf)
+                glfs_h_close (dleaf);
+        if (ln1)
+                glfs_h_close (ln1);
+        if (buf)
+                free (buf);
+
+        return;
+}
+
+void
+test_h_rename (void)
+{
+        char               *my_dir = "renamedir";
+        char               *my_file = "file.txt";
+        char               *src_dir = "dir1";
+        char               *tgt_dir = "dir2";
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+        struct stat         sb;
+        int                 ret;
+
+        printf("glfs_h_rename tests: In Progress\n");
+
+        /* Prepare tests */
+        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, root, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirsrc = glfs_h_mkdir (fs, parent, src_dir, 0644, &sb);
+        if (dirsrc == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         src_dir, parent, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirtgt = glfs_h_mkdir (fs, parent, tgt_dir, 0644, &sb);
+        if (dirtgt == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         tgt_dir, parent, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+        if (dleaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dirsrc, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* run tests */
+        /* Rename file.txt -> file1.txt */
+        ret = glfs_h_rename (fs, parent, "file.txt", parent, "file1.txt");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "file.txt", "file1.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir1/file.txt -> file.txt */
+        ret = glfs_h_rename (fs, dirsrc, "file.txt", parent, "file.txt");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
+                         src_dir, "file.txt", "file.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename file1.txt -> file.txt (exists) */
+        ret = glfs_h_rename (fs, parent, "file1.txt", parent, "file.txt");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "file.txt", "file.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir1 -> dir3 */
+        ret = glfs_h_rename (fs, parent, "dir1", parent, "dir3");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "dir1", "dir3", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir2 ->dir3 (exists) */
+        ret = glfs_h_rename (fs, parent, "dir2", parent, "dir3");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "dir2", "dir3", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename file.txt -> dir3 (fail) */
+        ret = glfs_h_rename (fs, parent, "file.txt", parent, "dir3");
+        if (ret == 0) {
+                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                         "file.txt", "dir3", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir3 -> file.txt (fail) */
+        ret = glfs_h_rename (fs, parent, "dir3", parent, "file.txt");
+        if (ret == 0) {
+                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                         "dir3", "file.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        printf ("glfs_h_rename tests: PASSED\n");
+
+out:
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (dirsrc)
+                glfs_h_close (dirsrc);
+        if (dirtgt)
+                glfs_h_close (dirtgt);
+        if (dleaf)
+                glfs_h_close (dleaf);
+
+        return;
+}
+
+void
+assimilatetime (struct timespec *ts, struct timespec ts_st,
+                struct timespec ts_ed)
+{
+        if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
+                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
+                ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
+        } else {
+                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
+                ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
+        }
+
+        if (ts->tv_nsec > 1000000000) {
+                ts->tv_nsec = ts->tv_nsec - 1000000000;
+                ts->tv_sec += 1;
+        }
+
+        return;
+}
+
+#define MAX_FILES_CREATE 10
+#define MAXPATHNAME      512
+void
+test_h_performance (void)
+{
+        char               *my_dir = "perftest",
+                           *full_dir_path="/testdir/perftest";
+        char               *my_file = "file_", my_file_name[MAXPATHNAME];
+        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+        struct stat         sb;
+        int                 ret, i;
+        struct glfs_fd     *fd;
+        struct timespec     c_ts = {0, 0}, c_ts_st, c_ts_ed;
+        struct timespec     o_ts = {0, 0}, o_ts_st, o_ts_ed;
+
+        printf("glfs_h_performance tests: In Progress\n");
+
+        /* Prepare tests */
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_performance tests: FAILED\n");
+                goto out;
+        }
+
+        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
+        if (dir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_performance tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* create performance */
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        for (i = 0; i < MAX_FILES_CREATE; i++) {
+                sprintf (my_file_name, "%s%d", my_file, i);
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                leaf = glfs_h_lookupat (fs, dir, my_file_name, &sb);
+                if (leaf != NULL) {
+                        fprintf (stderr, "glfs_h_lookup: exists %s\n",
+                                 my_file_name);
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                leaf = glfs_h_creat (fs, dir, my_file_name, O_CREAT, 0644, &sb);
+                if (leaf == NULL) {
+                        fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                                 my_file, dir, strerror (errno));
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
+                glfs_h_close (leaf); leaf = NULL;
+        }
+
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
+
+        printf ("Creation performance (handle based):\n\t# empty files:%d\n",
+                MAX_FILES_CREATE);
+        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                o_ts.tv_sec, o_ts.tv_nsec);
+        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                c_ts.tv_sec, c_ts.tv_nsec);
+
+        /* create using path */
+        c_ts.tv_sec = o_ts.tv_sec = 0;
+        c_ts.tv_nsec = o_ts.tv_nsec = 0;
+
+        sprintf (my_file_name, "%s1", full_dir_path);
+        ret = glfs_mkdir (fs, my_file_name, 0644);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_performance tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        for (i = 0; i < MAX_FILES_CREATE; i++) {
+                sprintf (my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                ret = glfs_stat (fs, my_file_name, &sb);
+                if (ret == 0) {
+                        fprintf (stderr, "glfs_stat: exists %s\n",
+                                 my_file_name);
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                fd = glfs_creat (fs, my_file_name, O_CREAT, 0644);
+                if (fd == NULL) {
+                        fprintf (stderr, "glfs_creat: error creating %s: from (%p),%s\n",
+                                 my_file, dir, strerror (errno));
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
+                glfs_close (fd);
+        }
+
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
+
+        printf ("Creation performance (path based):\n\t# empty files:%d\n",
+                MAX_FILES_CREATE);
+        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                o_ts.tv_sec, o_ts.tv_nsec);
+        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                c_ts.tv_sec, c_ts.tv_nsec);
+out:
+        return;
+}
+
+int
+test_handleops (int argc, char *argv[])
+{
+        int                 ret = 0;
+        glfs_fd_t          *fd = NULL;
+        struct stat         sb = {0, };
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                           *tmp = NULL;
+        char                readbuf[32], writebuf[32];
+        unsigned char       leaf_handle[GFAPI_HANDLE_LENGTH];
+
+        char *full_leaf_name = "/testdir/testfile.txt",
+             *leaf_name = "testfile.txt",
+             *relative_leaf_name = "testdir/testfile.txt";
+        char *leaf_name1 = "testfile1.txt";
+        char *full_newparent_name = "/testdir/dir1",
+             *newparent_name = "dir1";
+        char *full_newnod_name = "/testdir/nod1",
+             *newnod_name = "nod1";
+
+        /* Initialize test area */
+        ret = glfs_mkdir (fs, full_parent_name, 0644);
+        if (ret != 0 && errno != EEXIST) {
+                fprintf (stderr, "%s: (%p) %s\n", full_parent_name, fd,
+                        strerror (errno));
+                printf ("Test initialization failed on volume %s\n", argv[1]);
+                goto out;
+        }
+        else if (ret != 0) {
+                printf ("Found test directory %s to be existing\n",
+                        full_parent_name);
+                printf ("Cleanup test directory and restart tests\n");
+                goto out;
+        }
+
+        fd = glfs_creat (fs, full_leaf_name, O_CREAT, 0644);
+        if (fd == NULL) {
+                fprintf (stderr, "%s: (%p) %s\n", full_leaf_name, fd,
+                        strerror (errno));
+                printf ("Test initialization failed on volume %s\n", argv[1]);
+                goto out;
+        }
+        glfs_close (fd);
+
+        printf ("Initialized the test area, within volume %s\n", argv[1]);
+
+        /* Handle based APIs test area */
+
+        /* glfs_lookupat test */
+        printf ("glfs_h_lookupat tests: In Progress\n");
+        /* start at root of the volume */
+        root = glfs_h_lookupat (fs, NULL, "/", &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         "/", NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* lookup a parent within root */
+        parent = glfs_h_lookupat (fs, root, parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         parent_name, root, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* lookup a leaf/child within the parent */
+        leaf = glfs_h_lookupat (fs, parent, leaf_name, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         leaf_name, parent, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* reset */
+        glfs_h_close (root); root = NULL;
+        glfs_h_close (leaf); leaf = NULL;
+        glfs_h_close (parent); parent = NULL;
+
+        /* check absolute paths */
+        root = glfs_h_lookupat (fs, NULL, "/", &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         "/", NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_lookupat (fs, NULL, full_leaf_name, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_leaf_name, parent, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* reset */
+        glfs_h_close (leaf); leaf = NULL;
+
+        /* check multiple component paths */
+        leaf = glfs_h_lookupat (fs, root, relative_leaf_name, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         relative_leaf_name, parent, strerror (errno));
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* reset */
+        glfs_h_close (root); root = NULL;
+        glfs_h_close (parent); parent = NULL;
+
+        /* check symlinks in path */
+
+        /* TODO: -ve test cases */
+        /* parent invalid
+        * path invalid
+        * path does not exist after some components
+        * no parent, but relative path
+        * parent and full path? -ve?
+        */
+
+        printf ("glfs_h_lookupat tests: PASSED\n");
+
+        /* glfs_openat test */
+        printf ("glfs_h_open tests: In Progress\n");
+        fd = glfs_h_open (fs, leaf, O_RDWR);
+        if (fd == NULL) {
+                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
+                         full_leaf_name, strerror (errno));
+                printf ("glfs_h_open tests: FAILED\n");
+                goto out;
+        }
+
+        /* test read/write based on fd */
+        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+        ret = glfs_write (fd, writebuf, 32, 0);
+
+        glfs_lseek (fd, 0, SEEK_SET);
+
+        ret = glfs_read (fd, readbuf, 32, 0);
+        if (memcmp (readbuf, writebuf, 32)) {
+                printf ("Failed to read what I wrote: %s %s\n", readbuf,
+                        writebuf);
+                glfs_close (fd);
+                printf ("glfs_h_open tests: FAILED\n");
+                goto out;
+        }
+
+        glfs_h_close (leaf); leaf = NULL;
+        glfs_close (fd);
+
+        printf ("glfs_h_open tests: PASSED\n");
+
+        /* Create tests */
+        printf ("glfs_h_creat tests: In Progress\n");
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
+                         leaf_name1, parent, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644,
+                            &sb);
+        if (leaf != NULL || errno != EEXIST) {
+                fprintf (stderr, "glfs_h_creat: existing file, leaf = (%p), errno = %s\n",
+                        leaf, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                if (leaf != NULL) {
+                        glfs_h_close (leaf); leaf = NULL;
+                }
+        }
+
+        tmp = glfs_h_creat (fs, root, parent_name, O_CREAT, 0644, &sb);
+        if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
+                fprintf (stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
+                        leaf, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                if (tmp != NULL) {
+                        glfs_h_close (tmp); tmp = NULL;
+                }
+        }
+
+        /* TODO: Other combinations and -ve cases as applicable */
+        printf ("glfs_h_creat tests: PASSED\n");
+
+        /* extract handle and create from handle test */
+        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: In Progress\n");
+        /* TODO: Change the lookup to creat below for a GIFD recovery falure,
+         * that needs to be fixed */
+        leaf = glfs_h_lookupat (fs, parent, leaf_name1, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         leaf_name1, parent, strerror (errno));
+                printf ("glfs_h_extract_handle tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        ret = glfs_h_extract_handle (leaf, leaf_handle,
+                                             GFAPI_HANDLE_LENGTH);
+        if (ret < 0) {
+                fprintf (stderr, "glfs_h_extract_handle: error extracting handle of %s: %s\n",
+                         full_leaf_name, strerror (errno));
+                printf ("glfs_h_extract_handle tests: FAILED\n");
+                goto out;
+        }
+        peek_handle (leaf_handle);
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_create_from_handle (fs, leaf_handle, GFAPI_HANDLE_LENGTH,
+                                          &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
+                         leaf_name1, leaf_handle, strerror (errno));
+                printf ("glfs_h_create_from_handle tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        fd = glfs_h_open (fs, leaf, O_RDWR);
+        if (fd == NULL) {
+                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
+                         full_leaf_name, strerror (errno));
+                printf ("glfs_h_create_from_handle tests: FAILED\n");
+                goto out;
+        }
+
+        /* test read/write based on fd */
+        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+        ret = glfs_write (fd, writebuf, 32, 0);
+
+        glfs_lseek (fd, 0, SEEK_SET);
+
+        ret = glfs_read (fd, readbuf, 32, 0);
+        if (memcmp (readbuf, writebuf, 32)) {
+                printf ("Failed to read what I wrote: %s %s\n", writebuf,
+                        writebuf);
+                printf ("glfs_h_create_from_handle tests: FAILED\n");
+                glfs_close (fd);
+                goto out;
+        }
+
+        glfs_close (fd);
+        glfs_h_close (leaf); leaf = NULL;
+        glfs_h_close (parent); parent = NULL;
+
+        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
+
+        /* Mkdir tests */
+        printf ("glfs_h_mkdir tests: In Progress\n");
+
+        ret = glfs_rmdir (fs, full_newparent_name);
+        if (ret && errno != ENOENT) {
+                fprintf (stderr, "glfs_rmdir: Failed for %s: %s\n",
+                         full_newparent_name, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                goto out;
+        }
+
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                         newparent_name, parent, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
+        if (leaf != NULL || errno != EEXIST) {
+                fprintf (stderr, "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
+                         leaf, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                if (leaf != NULL) {
+                        glfs_h_close (leaf); leaf = NULL;
+                }
+        }
+
+        glfs_h_close (parent); parent = NULL;
+
+        printf ("glfs_h_mkdir tests: PASSED\n");
+
+        /* Mknod tests */
+        printf ("glfs_h_mknod tests: In Progress\n");
+        ret = glfs_unlink (fs, full_newnod_name);
+        if (ret && errno != ENOENT) {
+                fprintf (stderr, "glfs_unlink: Failed for %s: %s\n",
+                         full_newnod_name, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                goto out;
+        }
+
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_mknod (fs, parent, newnod_name, S_IFIFO, 0, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                         newnod_name, parent, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* TODO: creat op on a FIFO node hangs, need to check and fix
+        tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
+        if (tmp != NULL || errno != EINVAL) {
+                fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno = %s\n",
+                        tmp, strerror (errno));
+                printf ("glfs_h_creat/mknod tests: FAILED\n");
+                if (tmp != NULL) {
+                        glfs_h_close(tmp); tmp = NULL;
+                }
+        } */
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_mknod (fs, parent, newnod_name, 0644, 0, &sb);
+        if (leaf != NULL || errno != EEXIST) {
+                fprintf (stderr, "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n",
+                         leaf, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                if (leaf != NULL) {
+                        glfs_h_close (leaf); leaf = NULL;
+                }
+        }
+
+        glfs_h_close (parent); parent = NULL;
+
+        printf ("glfs_h_mknod tests: PASSED\n");
+
+        /* unlink tests */
+        test_h_unlink ();
+
+        /* TODO: opendir tests */
+
+        /* getattr tests */
+        test_h_getsetattrs ();
+
+        /* TODO: setattr tests */
+
+        /* truncate tests */
+        test_h_truncate();
+
+        /* link tests */
+        test_h_links ();
+
+        /* rename tests */
+        test_h_rename ();
+
+        /* performance tests */
+        test_h_performance ();
+
+        /* END: New APIs test area */
+
+out:
+        /* Cleanup glfs handles */
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+
+        return ret;
+}
+
+int
+main (int argc, char *argv[])
+{
+        glfs_t    *fs2 = NULL;
+        int        ret = 0;
+        glfs_fd_t *fd = NULL;
+        glfs_fd_t *fd2 = NULL;
+        struct stat sb = {0, };
+        char       readbuf[32];
+        char       writebuf[32];
+
+        char      *filename = "/filename2";
+
+        if (argc != 3) {
+                printf ("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
+                return -1;
+        }
+
+        fs = glfs_new (argv[1]);
+        if (!fs) {
+                fprintf (stderr, "glfs_new: returned NULL\n");
+                return 1;
+        }
+
+//      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
+
+        ret = glfs_set_volfile_server (fs, "tcp", argv[2], 24007);
+
+//      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
+
+        ret = glfs_set_logging (fs, "/dev/stderr", 7);
+
+        ret = glfs_init (fs);
+
+        fprintf (stderr, "glfs_init: returned %d\n", ret);
+
+        sleep (2);
+
+        fs2 = glfs_new (argv[1]);
+        if (!fs2) {
+                fprintf (stderr, "glfs_new: returned NULL\n");
+                return 1;
+        }
+
+
+//      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
+
+        ret = glfs_set_volfile_server (fs2, "tcp", argv[2], 24007);
+
+        ret = glfs_set_logging (fs2, "/dev/stderr", 7);
+
+        ret = glfs_init (fs2);
+
+        fprintf (stderr, "glfs_init: returned %d\n", ret);
+
+        ret = glfs_lstat (fs, filename, &sb);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+
+        fd = glfs_creat (fs, filename, O_RDWR, 0644);
+        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+
+        fd2 = glfs_open (fs2, filename, O_RDWR);
+        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+
+        sprintf (writebuf, "hi there\n");
+        ret = glfs_write (fd, writebuf, 32, 0);
+
+        glfs_lseek (fd2, 0, SEEK_SET);
+
+        ret = glfs_read (fd2, readbuf, 32, 0);
+
+        printf ("read %d, %s", ret, readbuf);
+
+        glfs_close (fd);
+        glfs_close (fd2);
+
+        filename = "/filename3";
+        ret = glfs_mknod (fs, filename, S_IFIFO, 0);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+
+        ret = glfs_lstat (fs, filename, &sb);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+
+
+        ret = glfs_rename (fs, filename, "/filename4");
+        fprintf (stderr, "rename(%s): (%d) %s\n", filename, ret,
+                 strerror (errno));
+
+        ret = glfs_unlink (fs, "/filename4");
+        fprintf (stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
+                 strerror (errno));
+
+        filename = "/dirname2";
+        ret = glfs_mkdir (fs, filename, 0);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+
+        ret = glfs_lstat (fs, filename, &sb);
+        fprintf (stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror (errno));
+
+        ret = glfs_rmdir (fs, filename);
+        fprintf (stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror (errno));
+
+        test_dirops (fs);
+
+        test_xattr (fs);
+
+        test_chdir (fs);
+
+        test_handleops (argc, argv);
+        // done
+
+        glfs_fini (fs);
+        glfs_fini (fs2);
+
+        return ret;
+}
diff --git a/api/examples/setup.py.in b/api/examples/setup.py.in
new file mode 100644
index 000000000..44b738094
--- /dev/null
+++ b/api/examples/setup.py.in
@@ -0,0 +1,29 @@
+from distutils.core import setup
+
+# generate a __init__.py for the package namespace
+fo = open('__init__.py', 'w')
+fo.write('__version__ = "@PACKAGE_VERSION@"\n')
+fo.close()
+
+DESC = """GlusterFS is a clustered file-system capable of scaling to
+several petabytes. It aggregates various storage bricks over Infiniband
+RDMA or TCP/IP interconnect into one large parallel network file system.
+GlusterFS is one of the most sophisticated file systems in terms of
+features and extensibility.  It borrows a powerful concept called
+Translators from GNU Hurd kernel. Much of the code in GlusterFS is in
+user space and easily manageable.
+
+This package contains the Python interface to the libgfapi library."""
+
+setup(
+    name='glusterfs-api',
+    version='@PACKAGE_VERSION@',
+    description='Python client library for the GlusterFS libgfapi',
+    long_description=DESC,
+    author='Gluster Community',
+    author_email='gluster-devel@nongnu.org',
+    license='LGPLv3',
+    url='http://gluster.org/',
+    package_dir={'gluster':''},
+    packages=['gluster']
+)
diff --git a/api/src/Makefile.am b/api/src/Makefile.am
new file mode 100644
index 000000000..7c5df3e20
--- /dev/null
+++ b/api/src/Makefile.am
@@ -0,0 +1,36 @@
+lib_LTLIBRARIES = libgfapi.la
+noinst_HEADERS = glfs-mem-types.h glfs-internal.h
+libgfapi_HEADERS = glfs.h glfs-handles.h
+libgfapidir = $(includedir)/glusterfs/api
+
+libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c \
+	glfs-handleops.c
+libgfapi_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(GF_LDADD)
+
+libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
+	-I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src
+
+
+xlator_LTLIBRARIES = api.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
+# workaround for broken parallel install support in automake with LTLIBRARIES
+# http://debbugs.gnu.org/cgi/bugreport.cgi?bug=7328
+install_xlatorLTLIBRARIES = install-xlatorLTLIBRARIES
+$(install_xlatorLTLIBRARIES): install-libLTLIBRARIES
+
+api_la_SOURCES = glfs-master.c
+api_la_DEPENDENCIES = libgfapi.la
+api_la_LDFLAGS = -module -avoid-version
+api_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/api/src/libgfapi.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
new file mode 100644
index 000000000..10bb7d38b
--- /dev/null
+++ b/api/src/glfs-fops.c
@@ -0,0 +1,3252 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+#include "syncop.h"
+#include "glfs.h"
+#include <limits.h>
+
+#ifdef NAME_MAX
+#define GF_NAME_MAX NAME_MAX
+#else
+#define GF_NAME_MAX 255
+#endif
+
+#define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
+
+int
+glfs_loc_link (loc_t *loc, struct iatt *iatt)
+{
+	int ret = -1;
+	inode_t *linked_inode = NULL;
+
+	if (!loc->inode) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	linked_inode = inode_link (loc->inode, loc->parent, loc->name, iatt);
+	if (linked_inode) {
+		inode_lookup (linked_inode);
+		inode_unref (linked_inode);
+		ret = 0;
+	} else {
+		ret = -1;
+		errno = ENOMEM;
+	}
+
+	return ret;
+}
+
+
+void
+glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat)
+{
+	iatt_to_stat (iatt, stat);
+	stat->st_dev = fs->dev_id;
+}
+
+
+int
+glfs_loc_unlink (loc_t *loc)
+{
+	inode_unlink (loc->inode, loc->parent, loc->name);
+
+	return 0;
+}
+
+
+struct glfs_fd *
+glfs_open (struct glfs *fs, const char *path, int flags)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (IA_ISDIR (iatt.ia_type)) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	if (!IA_ISREG (iatt.ia_type)) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	if (glfd->fd) {
+		/* Retry. Safe to touch glfd->fd as we
+		   still have not glfs_fd_bind() yet.
+		*/
+		fd_unref (glfd->fd);
+		glfd->fd = NULL;
+	}
+
+	glfd->fd = fd_create (loc.inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_open (subvol, &loc, flags, glfd->fd);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else if (glfd) {
+                glfd->fd->flags = flags;
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+
+int
+glfs_close (struct glfs_fd *glfd)
+{
+	xlator_t  *subvol = NULL;
+	int        ret = -1;
+	fd_t      *fd = NULL;
+	struct glfs *fs = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+        if (!subvol) {
+                ret = -1;
+                errno = EIO;
+                goto out;
+        }
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_flush (subvol, fd);
+out:
+	fs = glfd->fs;
+	glfs_fd_destroy (glfd);
+
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_lstat (struct glfs *fs, const char *path, struct stat *stat)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0 && stat)
+		glfs_iatt_to_stat (fs, &iatt, stat);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_stat (struct glfs *fs, const char *path, struct stat *stat)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0 && stat)
+		glfs_iatt_to_stat (fs, &iatt, stat);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fstat (struct glfs_fd *glfd, struct stat *stat)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	struct iatt      iatt = {0, };
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fstat (subvol, fd, &iatt);
+
+	if (ret == 0 && stat)
+		glfs_iatt_to_stat (glfd->fs, &iatt, stat);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+struct glfs_fd *
+glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	/* This must be glfs_resolve() and NOT glfs_lresolve().
+	   That is because open("name", O_CREAT) where "name"
+	   is a danging symlink must create the dangling
+	   destinataion.
+	*/
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	if (loc.inode) {
+		if (flags & O_EXCL) {
+			ret = -1;
+			errno = EEXIST;
+			goto out;
+		}
+
+		if (IA_ISDIR (iatt.ia_type)) {
+			ret = -1;
+			errno = EISDIR;
+			goto out;
+		}
+
+		if (!IA_ISREG (iatt.ia_type)) {
+			ret = -1;
+			errno = EINVAL;
+			goto out;
+		}
+	}
+
+	if (ret == -1 && errno == ENOENT) {
+		loc.inode = inode_new (loc.parent->table);
+		if (!loc.inode) {
+			ret = -1;
+			errno = ENOMEM;
+			goto out;
+		}
+	}
+
+	if (glfd->fd) {
+		/* Retry. Safe to touch glfd->fd as we
+		   still have not glfs_fd_bind() yet.
+		*/
+		fd_unref (glfd->fd);
+		glfd->fd = NULL;
+	}
+
+	glfd->fd = fd_create (loc.inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	if (ret == 0) {
+		ret = syncop_open (subvol, &loc, flags, glfd->fd);
+	} else {
+		ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
+				     xattr_req, &iatt);
+	}
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else if (glfd) {
+                glfd->fd->flags = flags;
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+
+off_t
+glfs_lseek (struct glfs_fd *glfd, off_t offset, int whence)
+{
+	struct stat sb = {0, };
+	int         ret = -1;
+
+	__glfs_entry_fd (glfd);
+
+	switch (whence) {
+	case SEEK_SET:
+		glfd->offset = offset;
+		break;
+	case SEEK_CUR:
+		glfd->offset += offset;
+		break;
+	case SEEK_END:
+		ret = glfs_fstat (glfd, &sb);
+		if (ret) {
+			/* seek cannot fail :O */
+			break;
+		}
+		glfd->offset = sb.st_size + offset;
+		break;
+	}
+
+	return glfd->offset;
+}
+
+
+//////////////
+
+ssize_t
+glfs_preadv (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+	     off_t offset, int flags)
+{
+	xlator_t       *subvol = NULL;
+	ssize_t         ret = -1;
+	ssize_t         size = -1;
+	struct iovec   *iov = NULL;
+	int             cnt = 0;
+	struct iobref  *iobref = NULL;
+	fd_t           *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	size = iov_length (iovec, iovcnt);
+
+	ret = syncop_readv (subvol, fd, size, offset, 0, &iov, &cnt, &iobref);
+	if (ret <= 0)
+		goto out;
+
+	size = iov_copy (iovec, iovcnt, iov, cnt); /* FIXME!!! */
+
+	glfd->offset = (offset + size);
+
+	ret = size;
+out:
+        if (iov)
+                GF_FREE (iov);
+        if (iobref)
+                iobref_unref (iobref);
+
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_read (struct glfs_fd *glfd, void *buf, size_t count, int flags)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = buf;
+	iov.iov_len = count;
+
+	ret = glfs_preadv (glfd, &iov, 1, glfd->offset, flags);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_pread (struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+	    int flags)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = buf;
+	iov.iov_len = count;
+
+	ret = glfs_preadv (glfd, &iov, 1, offset, flags);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_readv (struct glfs_fd *glfd, const struct iovec *iov, int count,
+	    int flags)
+{
+	ssize_t      ret = 0;
+
+	ret = glfs_preadv (glfd, iov, count, glfd->offset, flags);
+
+	return ret;
+}
+
+
+struct glfs_io {
+	struct glfs_fd      *glfd;
+	int                  op;
+	off_t                offset;
+	struct iovec        *iov;
+	int                  count;
+	int                  flags;
+	glfs_io_cbk          fn;
+	void                *data;
+};
+
+
+static int
+glfs_io_async_cbk (int ret, call_frame_t *frame, void *data)
+{
+	struct glfs_io  *gio = data;
+
+	gio->fn (gio->glfd, ret, gio->data);
+
+	GF_FREE (gio->iov);
+	GF_FREE (gio);
+
+	return 0;
+}
+
+
+static int
+glfs_io_async_task (void *data)
+{
+	struct glfs_io *gio = data;
+	ssize_t         ret = 0;
+
+	switch (gio->op) {
+	case GF_FOP_WRITE:
+		ret = glfs_pwritev (gio->glfd, gio->iov, gio->count,
+				    gio->offset, gio->flags);
+		break;
+	case GF_FOP_FTRUNCATE:
+		ret = glfs_ftruncate (gio->glfd, gio->offset);
+		break;
+	case GF_FOP_FSYNC:
+		if (gio->flags)
+			ret = glfs_fdatasync (gio->glfd);
+		else
+			ret = glfs_fsync (gio->glfd);
+		break;
+	case GF_FOP_DISCARD:
+		ret = glfs_discard (gio->glfd, gio->offset, gio->count);
+		break;
+        case GF_FOP_ZEROFILL:
+                ret = glfs_zerofill(gio->glfd, gio->offset, gio->count);
+                break;
+	}
+
+	return (int) ret;
+}
+
+
+int
+glfs_preadv_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int op_ret, int op_errno, struct iovec *iovec,
+		       int count, struct iatt *stbuf, struct iobref *iobref,
+		       dict_t *xdata)
+{
+	struct glfs_io *gio = NULL;
+	xlator_t       *subvol = NULL;
+	struct glfs    *fs = NULL;
+	struct glfs_fd *glfd = NULL;
+
+
+	gio = frame->local;
+	frame->local = NULL;
+	subvol = cookie;
+	glfd = gio->glfd;
+	fs = glfd->fs;
+
+	if (op_ret <= 0)
+		goto out;
+
+	op_ret = iov_copy (gio->iov, gio->count, iovec, count);
+
+	glfd->offset = gio->offset + op_ret;
+out:
+	errno = op_errno;
+	gio->fn (gio->glfd, op_ret, gio->data);
+
+	GF_FREE (gio->iov);
+	GF_FREE (gio);
+	STACK_DESTROY (frame->root);
+	glfs_subvol_done (fs, subvol);
+
+	return 0;
+}
+
+
+int
+glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec, int count,
+		   off_t offset, int flags, glfs_io_cbk fn, void *data)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+	call_frame_t   *frame = NULL;
+	xlator_t       *subvol = NULL;
+	glfs_t         *fs = NULL;
+	fd_t           *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	fs = glfd->fs;
+
+	frame = syncop_create_frame (THIS);
+	if (!frame) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	gio->iov = iov_dup (iovec, count);
+	if (!gio->iov) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	gio->op     = GF_FOP_READ;
+	gio->glfd   = glfd;
+	gio->count  = count;
+	gio->offset = offset;
+	gio->flags  = flags;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	frame->local = gio;
+
+	STACK_WIND_COOKIE (frame, glfs_preadv_async_cbk, subvol, subvol,
+			   subvol->fops->readv, fd, iov_length (iovec, count),
+			   offset, flags, NULL);
+
+out:
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+		STACK_DESTROY (frame->root);
+		glfs_subvol_done (fs, subvol);
+	}
+
+	if (fd)
+		fd_unref (fd);
+
+	return ret;
+}
+
+
+int
+glfs_read_async (struct glfs_fd *glfd, void *buf, size_t count, int flags,
+		 glfs_io_cbk fn, void *data)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = buf;
+	iov.iov_len = count;
+
+	ret = glfs_preadv_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+
+	return ret;
+}
+
+
+int
+glfs_pread_async (struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+		  int flags, glfs_io_cbk fn, void *data)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = buf;
+	iov.iov_len = count;
+
+	ret = glfs_preadv_async (glfd, &iov, 1, offset, flags, fn, data);
+
+	return ret;
+}
+
+
+int
+glfs_readv_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
+		  int flags, glfs_io_cbk fn, void *data)
+{
+	ssize_t      ret = 0;
+
+	ret = glfs_preadv_async (glfd, iov, count, glfd->offset, flags,
+				 fn, data);
+	return ret;
+}
+
+///// writev /////
+
+ssize_t
+glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+	      off_t offset, int flags)
+{
+	xlator_t       *subvol = NULL;
+	int             ret = -1;
+	size_t          size = -1;
+	struct iobref  *iobref = NULL;
+	struct iobuf   *iobuf = NULL;
+	struct iovec    iov = {0, };
+	fd_t           *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	size = iov_length (iovec, iovcnt);
+
+	iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
+	if (!iobuf) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	iobref = iobref_new ();
+	if (!iobref) {
+		iobuf_unref (iobuf);
+		errno = ENOMEM;
+		ret = -1;
+		goto out;
+	}
+
+	ret = iobref_add (iobref, iobuf);
+	if (ret) {
+		iobuf_unref (iobuf);
+		iobref_unref (iobref);
+		errno = ENOMEM;
+		ret = -1;
+		goto out;
+	}
+
+	iov_unload (iobuf_ptr (iobuf), iovec, iovcnt);  /* FIXME!!! */
+
+	iov.iov_base = iobuf_ptr (iobuf);
+	iov.iov_len = size;
+
+	ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags);
+
+	iobuf_unref (iobuf);
+	iobref_unref (iobref);
+
+	if (ret <= 0)
+		goto out;
+
+	glfd->offset = (offset + size);
+
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_write (struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = (void *) buf;
+	iov.iov_len = count;
+
+	ret = glfs_pwritev (glfd, &iov, 1, glfd->offset, flags);
+
+	return ret;
+}
+
+
+
+ssize_t
+glfs_writev (struct glfs_fd *glfd, const struct iovec *iov, int count,
+	     int flags)
+{
+	ssize_t      ret = 0;
+
+	ret = glfs_pwritev (glfd, iov, count, glfd->offset, flags);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_pwrite (struct glfs_fd *glfd, const void *buf, size_t count, off_t offset,
+	     int flags)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = (void *) buf;
+	iov.iov_len = count;
+
+	ret = glfs_pwritev (glfd, &iov, 1, offset, flags);
+
+	return ret;
+}
+
+
+int
+glfs_pwritev_async (struct glfs_fd *glfd, const struct iovec *iovec, int count,
+		    off_t offset, int flags, glfs_io_cbk fn, void *data)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->iov = iov_dup (iovec, count);
+	if (!gio->iov) {
+		GF_FREE (gio);
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->op     = GF_FOP_WRITE;
+	gio->glfd   = glfd;
+	gio->count  = count;
+	gio->offset = offset;
+	gio->flags  = flags;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+			    glfs_io_async_task, glfs_io_async_cbk,
+			    NULL, gio);
+
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+	}
+
+	return ret;
+}
+
+
+int
+glfs_write_async (struct glfs_fd *glfd, const void *buf, size_t count, int flags,
+		  glfs_io_cbk fn, void *data)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = (void *) buf;
+	iov.iov_len = count;
+
+	ret = glfs_pwritev_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+
+	return ret;
+}
+
+
+int
+glfs_pwrite_async (struct glfs_fd *glfd, const void *buf, int count,
+		   off_t offset, int flags, glfs_io_cbk fn, void *data)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = (void *) buf;
+	iov.iov_len = count;
+
+	ret = glfs_pwritev_async (glfd, &iov, 1, offset, flags, fn, data);
+
+	return ret;
+}
+
+
+int
+glfs_writev_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
+		   int flags, glfs_io_cbk fn, void *data)
+{
+	ssize_t      ret = 0;
+
+	ret = glfs_pwritev_async (glfd, iov, count, glfd->offset, flags,
+				  fn, data);
+	return ret;
+}
+
+
+int
+glfs_fsync (struct glfs_fd *glfd)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fsync (subvol, fd, 0);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+static int
+glfs_fsync_async_common (struct glfs_fd *glfd, glfs_io_cbk fn, void *data,
+			 int dataonly)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->op     = GF_FOP_FSYNC;
+	gio->glfd   = glfd;
+	gio->flags  = dataonly;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+			    glfs_io_async_task, glfs_io_async_cbk,
+			    NULL, gio);
+
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+	}
+
+	return ret;
+
+}
+
+
+int
+glfs_fsync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+{
+	return glfs_fsync_async_common (glfd, fn, data, 0);
+}
+
+
+int
+glfs_fdatasync (struct glfs_fd *glfd)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fsync (subvol, fd, 1);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fdatasync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+{
+	return glfs_fsync_async_common (glfd, fn, data, 1);
+}
+
+
+int
+glfs_ftruncate (struct glfs_fd *glfd, off_t offset)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_ftruncate (subvol, fd, offset);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_ftruncate_async (struct glfs_fd *glfd, off_t offset,
+		      glfs_io_cbk fn, void *data)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->op     = GF_FOP_FTRUNCATE;
+	gio->glfd   = glfd;
+	gio->offset = offset;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+			    glfs_io_async_task, glfs_io_async_cbk,
+			    NULL, gio);
+
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+	}
+
+	return ret;
+}
+
+
+int
+glfs_access (struct glfs *fs, const char *path, int mode)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_access (subvol, &loc, mode);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_symlink (struct glfs *fs, const char *data, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (loc.inode) {
+		errno = EEXIST;
+		ret = -1;
+		goto out;
+	}
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	/* ret == -1 && errno == ENOENT */
+	loc.inode = inode_new (loc.parent->table);
+	if (!loc.inode) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_readlink (struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+	char            *linkval = NULL;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (iatt.ia_type != IA_IFLNK) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
+	if (ret > 0) {
+		memcpy (buf, linkval, ret);
+		GF_FREE (linkval);
+	}
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_mknod (struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (loc.inode) {
+		errno = EEXIST;
+		ret = -1;
+		goto out;
+	}
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	/* ret == -1 && errno == ENOENT */
+	loc.inode = inode_new (loc.parent->table);
+	if (!loc.inode) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_mkdir (struct glfs *fs, const char *path, mode_t mode)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (loc.inode) {
+		errno = EEXIST;
+		ret = -1;
+		goto out;
+	}
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	/* ret == -1 && errno == ENOENT */
+	loc.inode = inode_new (loc.parent->table);
+	if (!loc.inode) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_unlink (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (iatt.ia_type == IA_IFDIR) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	ret = syncop_unlink (subvol, &loc);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_rmdir (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (iatt.ia_type != IA_IFDIR) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	ret = syncop_rmdir (subvol, &loc);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_rename (struct glfs *fs, const char *oldpath, const char *newpath)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            oldloc = {0, };
+	loc_t            newloc = {0, };
+	struct iatt      oldiatt = {0, };
+	struct iatt      newiatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+
+	if (ret)
+		goto out;
+retrynew:
+	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+
+	if (ret && errno != ENOENT && newloc.parent)
+		goto out;
+
+	if (newiatt.ia_type != IA_INVAL) {
+                if ((oldiatt.ia_type == IA_IFDIR) !=
+                    (newiatt.ia_type == IA_IFDIR)) {
+                       /* Either both old and new must be dirs,
+                        * or both must be non-dirs. Else, fail.
+                        */
+                       ret = -1;
+                       errno = EISDIR;
+                       goto out;
+                }
+        }
+
+	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+	ret = syncop_rename (subvol, &oldloc, &newloc);
+
+	if (ret == -1 && errno == ESTALE) {
+		if (reval < DEFAULT_REVAL_COUNT) {
+			reval++;
+			loc_wipe (&oldloc);
+			loc_wipe (&newloc);
+			goto retry;
+		}
+	}
+
+	if (ret == 0)
+		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
+			      newloc.parent, newloc.name, oldloc.inode,
+			      &oldiatt);
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_link (struct glfs *fs, const char *oldpath, const char *newpath)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            oldloc = {0, };
+	loc_t            newloc = {0, };
+	struct iatt      oldiatt = {0, };
+	struct iatt      newiatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+
+	if (ret)
+		goto out;
+retrynew:
+	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+
+	if (ret == 0) {
+		ret = -1;
+		errno = EEXIST;
+		goto out;
+	}
+
+	if (oldiatt.ia_type == IA_IFDIR) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+        /* Filling the inode of the hard link to be same as that of the
+           original file
+        */
+	if (newloc.inode) {
+		inode_unref (newloc.inode);
+		newloc.inode = NULL;
+	}
+        newloc.inode = inode_ref (oldloc.inode);
+
+	ret = syncop_link (subvol, &oldloc, &newloc);
+
+	if (ret == -1 && errno == ESTALE) {
+		loc_wipe (&oldloc);
+		loc_wipe (&newloc);
+		if (reval--)
+			goto retry;
+	}
+
+	if (ret == 0)
+		ret = glfs_loc_link (&newloc, &oldiatt);
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+struct glfs_fd *
+glfs_opendir (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	INIT_LIST_HEAD (&glfd->entries);
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (!IA_ISDIR (iatt.ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	if (glfd->fd) {
+		/* Retry. Safe to touch glfd->fd as we
+		   still have not glfs_fd_bind() yet.
+		*/
+		fd_unref (glfd->fd);
+		glfd->fd = NULL;
+	}
+
+	glfd->fd = fd_create (loc.inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_opendir (subvol, &loc, glfd->fd);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else {
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+
+int
+glfs_closedir (struct glfs_fd *glfd)
+{
+	__glfs_entry_fd (glfd);
+
+	gf_dirent_free (list_entry (&glfd->entries, gf_dirent_t, list));
+
+	glfs_fd_destroy (glfd);
+
+	return 0;
+}
+
+
+long
+glfs_telldir (struct glfs_fd *fd)
+{
+	return fd->offset;
+}
+
+
+void
+glfs_seekdir (struct glfs_fd *fd, long offset)
+{
+	gf_dirent_t *entry = NULL;
+	gf_dirent_t *tmp = NULL;
+
+	if (fd->offset == offset)
+		return;
+
+	fd->offset = offset;
+	fd->next = NULL;
+
+	list_for_each_entry_safe (entry, tmp, &fd->entries, list) {
+		if (entry->d_off != offset)
+			continue;
+
+		if (&tmp->list != &fd->entries) {
+			/* found! */
+			fd->next = tmp;
+			return;
+		}
+	}
+	/* could not find entry at requested offset in the cache.
+	   next readdir_r() will result in glfd_entry_refresh()
+	*/
+}
+
+int
+glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
+		      glfs_io_cbk fn, void *data)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->op     = GF_FOP_DISCARD;
+	gio->glfd   = glfd;
+	gio->offset = offset;
+	gio->count  = len;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+			    glfs_io_async_task, glfs_io_async_cbk,
+			    NULL, gio);
+
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+	}
+
+	return ret;
+}
+
+int
+glfs_zerofill_async (struct glfs_fd *glfd, off_t offset, size_t len,
+                      glfs_io_cbk fn, void *data)
+{
+        struct glfs_io *gio  = NULL;
+        int             ret  = 0;
+
+        gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+        if (!gio) {
+                errno = ENOMEM;
+                return -1;
+        }
+
+        gio->op     = GF_FOP_ZEROFILL;
+        gio->glfd   = glfd;
+        gio->offset = offset;
+        gio->count  = len;
+        gio->fn     = fn;
+        gio->data   = data;
+
+        ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+                            glfs_io_async_task, glfs_io_async_cbk,
+                            NULL, gio);
+
+        if (ret) {
+                GF_FREE (gio->iov);
+                GF_FREE (gio);
+        }
+
+        return ret;
+}
+
+
+void
+gf_dirent_to_dirent (gf_dirent_t *gf_dirent, struct dirent *dirent)
+{
+	dirent->d_ino = gf_dirent->d_ino;
+
+#ifdef _DIRENT_HAVE_D_OFF
+	dirent->d_off = gf_dirent->d_off;
+#endif
+
+#ifdef _DIRENT_HAVE_D_TYPE
+	dirent->d_type = gf_dirent->d_type;
+#endif
+
+#ifdef _DIRENT_HAVE_D_NAMLEN
+	dirent->d_namlen = strlen (gf_dirent->d_name);
+#endif
+
+	strncpy (dirent->d_name, gf_dirent->d_name, GF_NAME_MAX + 1);
+}
+
+
+int
+glfd_entry_refresh (struct glfs_fd *glfd, int plus)
+{
+	xlator_t        *subvol = NULL;
+	gf_dirent_t      entries;
+	gf_dirent_t      old;
+	int              ret = -1;
+	fd_t            *fd = NULL;
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	if (fd->inode->ia_type != IA_IFDIR) {
+		ret = -1;
+		errno = EBADF;
+		goto out;
+	}
+
+	INIT_LIST_HEAD (&entries.list);
+	INIT_LIST_HEAD (&old.list);
+
+	if (plus)
+		ret = syncop_readdirp (subvol, fd, 131072, glfd->offset,
+				       NULL, &entries);
+	else
+		ret = syncop_readdir (subvol, fd, 131072, glfd->offset,
+				      &entries);
+	if (ret >= 0) {
+		if (plus)
+			gf_link_inodes_from_dirent (THIS, fd->inode, &entries);
+
+		list_splice_init (&glfd->entries, &old.list);
+		list_splice_init (&entries.list, &glfd->entries);
+
+		/* spurious errno is dangerous for glfd_entry_next() */
+		errno = 0;
+	}
+
+	if (ret > 0)
+		glfd->next = list_entry (glfd->entries.next, gf_dirent_t, list);
+
+	gf_dirent_free (&old);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+gf_dirent_t *
+glfd_entry_next (struct glfs_fd *glfd, int plus)
+{
+	gf_dirent_t     *entry = NULL;
+	int              ret = -1;
+
+	if (!glfd->offset || !glfd->next) {
+		ret = glfd_entry_refresh (glfd, plus);
+		if (ret < 0)
+			return NULL;
+	}
+
+	entry = glfd->next;
+	if (!entry)
+		return NULL;
+
+	if (&entry->next->list == &glfd->entries)
+		glfd->next = NULL;
+	else
+		glfd->next = entry->next;
+
+	glfd->offset = entry->d_off;
+
+	return entry;
+}
+
+
+static struct dirent *
+glfs_readdirbuf_get (struct glfs_fd *glfd)
+{
+        struct dirent *buf = NULL;
+
+        LOCK (&glfd->fd->lock);
+        {
+                buf = glfd->readdirbuf;
+                if (buf) {
+                        memset (buf, 0, READDIRBUF_SIZE);
+                        goto unlock;
+                }
+
+                buf = GF_CALLOC (1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
+                if (!buf) {
+                        errno = ENOMEM;
+                        goto unlock;
+                }
+
+                glfd->readdirbuf = buf;
+        }
+unlock:
+        UNLOCK (&glfd->fd->lock);
+
+        return buf;
+}
+
+
+int
+glfs_readdirplus_r (struct glfs_fd *glfd, struct stat *stat, struct dirent *ext,
+		    struct dirent **res)
+{
+	int              ret = 0;
+	gf_dirent_t     *entry = NULL;
+	struct dirent   *buf = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	errno = 0;
+
+	if (ext)
+		buf = ext;
+	else
+		buf = glfs_readdirbuf_get (glfd);
+
+	if (!buf) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	entry = glfd_entry_next (glfd, !!stat);
+	if (errno)
+		ret = -1;
+
+	if (res) {
+		if (entry)
+			*res = buf;
+		else
+			*res = NULL;
+	}
+
+	if (entry) {
+		gf_dirent_to_dirent (entry, buf);
+		if (stat)
+			glfs_iatt_to_stat (glfd->fs, &entry->d_stat, stat);
+	}
+
+	return ret;
+}
+
+
+int
+glfs_readdir_r (struct glfs_fd *glfd, struct dirent *buf, struct dirent **res)
+{
+	return glfs_readdirplus_r (glfd, 0, buf, res);
+}
+
+
+struct dirent *
+glfs_readdirplus (struct glfs_fd *glfd, struct stat *stat)
+{
+        struct dirent *res = NULL;
+        int ret = -1;
+
+        ret = glfs_readdirplus_r (glfd, stat, NULL, &res);
+        if (ret)
+                return NULL;
+
+        return res;
+}
+
+
+
+struct dirent *
+glfs_readdir (struct glfs_fd *glfd)
+{
+        return glfs_readdirplus (glfd, NULL);
+}
+
+
+int
+glfs_statvfs (struct glfs *fs, const char *path, struct statvfs *buf)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_statfs (subvol, &loc, buf);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_setattr (struct glfs *fs, const char *path, struct iatt *iatt,
+	      int valid, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      riatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &riatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &riatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_setattr (subvol, &loc, iatt, valid, 0, 0);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fsetattr (subvol, fd, iatt, valid, 0, 0);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_chmod (struct glfs *fs, const char *path, mode_t mode)
+{
+	int              ret = -1;
+	struct iatt      iatt = {0, };
+	int              valid = 0;
+
+	iatt.ia_prot = ia_prot_from_st_mode (mode);
+	valid = GF_SET_ATTR_MODE;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+
+	return ret;
+}
+
+
+int
+glfs_fchmod (struct glfs_fd *glfd, mode_t mode)
+{
+	int              ret = -1;
+	struct iatt      iatt = {0, };
+	int              valid = 0;
+
+	iatt.ia_prot = ia_prot_from_st_mode (mode);
+	valid = GF_SET_ATTR_MODE;
+
+	ret = glfs_fsetattr (glfd, &iatt, valid);
+
+	return ret;
+}
+
+
+int
+glfs_chown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_uid = uid;
+	iatt.ia_gid = gid;
+	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+
+	return ret;
+}
+
+
+int
+glfs_lchown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_uid = uid;
+	iatt.ia_gid = gid;
+	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+
+	return ret;
+}
+
+
+int
+glfs_fchown (struct glfs_fd *glfd, uid_t uid, gid_t gid)
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_uid = uid;
+	iatt.ia_gid = gid;
+	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+
+	ret = glfs_fsetattr (glfd, &iatt, valid);
+
+	return ret;
+}
+
+
+int
+glfs_utimens (struct glfs *fs, const char *path, struct timespec times[2])
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_atime = times[0].tv_sec;
+	iatt.ia_atime_nsec = times[0].tv_nsec;
+	iatt.ia_mtime = times[1].tv_sec;
+	iatt.ia_mtime_nsec = times[1].tv_nsec;
+
+	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+
+	return ret;
+}
+
+
+int
+glfs_lutimens (struct glfs *fs, const char *path, struct timespec times[2])
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_atime = times[0].tv_sec;
+	iatt.ia_atime_nsec = times[0].tv_nsec;
+	iatt.ia_mtime = times[1].tv_sec;
+	iatt.ia_mtime_nsec = times[1].tv_nsec;
+
+	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+
+	return ret;
+}
+
+
+int
+glfs_futimens (struct glfs_fd *glfd, struct timespec times[2])
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_atime = times[0].tv_sec;
+	iatt.ia_atime_nsec = times[0].tv_nsec;
+	iatt.ia_mtime = times[1].tv_sec;
+	iatt.ia_mtime_nsec = times[1].tv_nsec;
+
+	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+
+	ret = glfs_fsetattr (glfd, &iatt, valid);
+
+	return ret;
+}
+
+
+int
+glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
+		       const char *name)
+{
+	data_t *data = NULL;
+	int     ret = -1;
+
+	data = dict_get (xattr, (char *)name);
+	if (!data) {
+		errno = ENODATA;
+		ret = -1;
+		goto out;
+	}
+
+	ret = data->len;
+	if (!value || !size)
+		goto out;
+
+	if (size < ret) {
+		ret = -1;
+		errno = ERANGE;
+		goto out;
+	}
+
+	memcpy (value, data->data, ret);
+out:
+	if (xattr)
+		dict_unref (xattr);
+	return ret;
+}
+
+
+ssize_t
+glfs_getxattr_common (struct glfs *fs, const char *path, const char *name,
+		      void *value, size_t size, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	dict_t          *xattr = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_getxattr (subvol, &loc, &xattr, name);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = glfs_getxattr_process (value, size, xattr, name);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_getxattr (struct glfs *fs, const char *path, const char *name,
+	       void *value, size_t size)
+{
+	return glfs_getxattr_common (fs, path, name, value, size, 1);
+}
+
+
+ssize_t
+glfs_lgetxattr (struct glfs *fs, const char *path, const char *name,
+		void *value, size_t size)
+{
+	return glfs_getxattr_common (fs, path, name, value, size, 0);
+}
+
+
+ssize_t
+glfs_fgetxattr (struct glfs_fd *glfd, const char *name, void *value,
+		size_t size)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	dict_t          *xattr = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fgetxattr (subvol, fd, &xattr, name);
+	if (ret)
+		goto out;
+
+	ret = glfs_getxattr_process (value, size, xattr, name);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_listxattr_process (void *value, size_t size, dict_t *xattr)
+{
+	int     ret = -1;
+
+	ret = dict_keys_join (NULL, 0, xattr, NULL);
+
+	if (!value || !size)
+		goto out;
+
+	if (size < ret) {
+		ret = -1;
+		errno = ERANGE;
+		goto out;
+	}
+
+	dict_keys_join (value, size, xattr, NULL);
+out:
+	if (xattr)
+		dict_unref (xattr);
+	return ret;
+}
+
+
+ssize_t
+glfs_listxattr_common (struct glfs *fs, const char *path, void *value,
+		       size_t size, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	dict_t          *xattr = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_getxattr (subvol, &loc, &xattr, NULL);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = glfs_listxattr_process (value, size, xattr);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_listxattr (struct glfs *fs, const char *path, void *value, size_t size)
+{
+	return glfs_listxattr_common (fs, path, value, size, 1);
+}
+
+
+ssize_t
+glfs_llistxattr (struct glfs *fs, const char *path, void *value, size_t size)
+{
+	return glfs_listxattr_common (fs, path, value, size, 0);
+}
+
+
+ssize_t
+glfs_flistxattr (struct glfs_fd *glfd, void *value, size_t size)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	dict_t          *xattr = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fgetxattr (subvol, fd, &xattr, NULL);
+	if (ret)
+		goto out;
+
+	ret = glfs_listxattr_process (value, size, xattr);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+dict_t *
+dict_for_key_value (const char *name, const char *value, size_t size)
+{
+	dict_t *xattr = NULL;
+	int     ret = 0;
+
+	xattr = dict_new ();
+	if (!xattr)
+		return NULL;
+
+	ret = dict_set_static_bin (xattr, (char *)name, (void *)value, size);
+	if (ret) {
+		dict_destroy (xattr);
+		xattr = NULL;
+	}
+
+	return xattr;
+}
+
+
+int
+glfs_setxattr_common (struct glfs *fs, const char *path, const char *name,
+		      const void *value, size_t size, int flags, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	dict_t          *xattr = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	xattr = dict_for_key_value (name, value, size);
+	if (!xattr) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_setxattr (subvol, &loc, xattr, flags);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+out:
+	loc_wipe (&loc);
+	if (xattr)
+		dict_unref (xattr);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_setxattr (struct glfs *fs, const char *path, const char *name,
+	       const void *value, size_t size, int flags)
+{
+	return glfs_setxattr_common (fs, path, name, value, size, flags, 1);
+}
+
+
+int
+glfs_lsetxattr (struct glfs *fs, const char *path, const char *name,
+		const void *value, size_t size, int flags)
+{
+	return glfs_setxattr_common (fs, path, name, value, size, flags, 0);
+}
+
+
+int
+glfs_fsetxattr (struct glfs_fd *glfd, const char *name, const void *value,
+		size_t size, int flags)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	dict_t          *xattr = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	xattr = dict_for_key_value (name, value, size);
+	if (!xattr) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_fsetxattr (subvol, fd, xattr, flags);
+out:
+	if (xattr)
+		dict_unref (xattr);
+
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_removexattr_common (struct glfs *fs, const char *path, const char *name,
+			 int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_removexattr (subvol, &loc, name);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_removexattr (struct glfs *fs, const char *path, const char *name)
+{
+	return glfs_removexattr_common (fs, path, name, 1);
+}
+
+
+int
+glfs_lremovexattr (struct glfs *fs, const char *path, const char *name)
+{
+	return glfs_removexattr_common (fs, path, name, 0);
+}
+
+
+int
+glfs_fremovexattr (struct glfs_fd *glfd, const char *name)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fremovexattr (subvol, fd, name);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t len)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t		*fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fallocate (subvol, fd, keep_size, offset, len);
+out:
+	if (fd)
+		fd_unref(fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t		*fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_discard (subvol, fd, offset, len);
+out:
+	if (fd)
+		fd_unref(fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_zerofill (struct glfs_fd *glfd, off_t offset, size_t len)
+{
+        int               ret             = -1;
+        xlator_t         *subvol          = NULL;
+        fd_t             *fd              = NULL;
+
+        __glfs_entry_fd (glfd);
+
+        subvol = glfs_active_subvol (glfd->fs);
+        if (!subvol) {
+                errno = EIO;
+                goto out;
+        }
+
+        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+        if (!fd) {
+                errno = EBADFD;
+                goto out;
+        }
+
+        ret = syncop_zerofill (subvol, fd, offset, len);
+out:
+        if (fd)
+                fd_unref(fd);
+
+        glfs_subvol_done (glfd->fs, subvol);
+
+        return ret;
+}
+
+int
+glfs_chdir (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (!IA_ISDIR (iatt.ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	glfs_cwd_set (fs, loc.inode);
+
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fchdir (struct glfs_fd *glfd)
+{
+	int       ret = -1;
+	inode_t  *inode = NULL;
+	xlator_t *subvol = NULL;
+	fd_t     *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	inode = fd->inode;
+
+	if (!IA_ISDIR (inode->ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	glfs_cwd_set (glfd->fs, inode);
+	ret = 0;
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+char *
+glfs_realpath (struct glfs *fs, const char *path, char *resolved_path)
+{
+	int              ret = -1;
+	char            *retpath = NULL;
+	char            *allocpath = NULL;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	if (resolved_path)
+		retpath = resolved_path;
+	else
+		retpath = allocpath = malloc (PATH_MAX + 1);
+
+	if (!retpath) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (loc.path) {
+		strncpy (retpath, loc.path, PATH_MAX);
+		retpath[PATH_MAX] = 0;
+	}
+
+out:
+	loc_wipe (&loc);
+
+	if (ret == -1) {
+		if (allocpath)
+			free (allocpath);
+		retpath = NULL;
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return retpath;
+}
+
+
+char *
+glfs_getcwd (struct glfs *fs, char *buf, size_t n)
+{
+	int              ret = -1;
+	inode_t         *inode = NULL;
+	char            *path = NULL;
+
+	__glfs_entry_fs (fs);
+
+	if (!buf || n < 2) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	inode = glfs_cwd_get (fs);
+
+	if (!inode) {
+		strncpy (buf, "/", n);
+		ret = 0;
+		goto out;
+	}
+
+	ret = inode_path (inode, 0, &path);
+	if (n <= ret) {
+		ret = -1;
+		errno = ERANGE;
+		goto out;
+	}
+
+	strncpy (buf, path, n);
+	ret = 0;
+out:
+	GF_FREE (path);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (ret < 0)
+		return NULL;
+
+	return buf;
+}
+
+
+static void
+gf_flock_to_flock (struct gf_flock *gf_flock, struct flock *flock)
+{
+	flock->l_type   = gf_flock->l_type;
+	flock->l_whence = gf_flock->l_whence;
+	flock->l_start  = gf_flock->l_start;
+	flock->l_len    = gf_flock->l_len;
+	flock->l_pid    = gf_flock->l_pid;
+}
+
+
+static void
+gf_flock_from_flock (struct gf_flock *gf_flock, struct flock *flock)
+{
+	gf_flock->l_type   = flock->l_type;
+	gf_flock->l_whence = flock->l_whence;
+	gf_flock->l_start  = flock->l_start;
+	gf_flock->l_len    = flock->l_len;
+	gf_flock->l_pid    = flock->l_pid;
+}
+
+
+int
+glfs_posix_lock (struct glfs_fd *glfd, int cmd, struct flock *flock)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	struct gf_flock  gf_flock = {0, };
+	struct gf_flock  saved_flock = {0, };
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	gf_flock_from_flock (&gf_flock, flock);
+	gf_flock_from_flock (&saved_flock, flock);
+	ret = syncop_lk (subvol, fd, cmd, &gf_flock);
+	gf_flock_to_flock (&gf_flock, flock);
+
+	if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
+		fd_lk_insert_and_merge (fd, cmd, &saved_flock);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+struct glfs_fd *
+glfs_dup (struct glfs_fd *glfd)
+{
+	xlator_t  *subvol = NULL;
+	fd_t      *fd = NULL;
+	glfs_fd_t *dupfd = NULL;
+	struct glfs *fs = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	fs = glfd->fs;
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (fs, subvol, glfd);
+	if (!fd) {
+		errno = EBADFD;
+		goto out;
+	}
+
+	dupfd = glfs_fd_new (fs);
+	if (!dupfd) {
+		errno = ENOMEM;
+		goto out;
+	}
+
+	dupfd->fd = fd_ref (fd);
+out:
+	if (fd)
+		fd_unref (fd);
+	if (dupfd)
+		glfs_fd_bind (dupfd);
+
+	glfs_subvol_done (fs, subvol);
+
+	return dupfd;
+}
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
new file mode 100644
index 000000000..9c707a619
--- /dev/null
+++ b/api/src/glfs-handleops.c
@@ -0,0 +1,1278 @@
+/*
+ *  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ *  This file is part of GlusterFS.
+ *
+ *  This file is licensed to you under your choice of the GNU Lesser
+ *  General Public License, version 3 or any later version (LGPLv3 or
+ *  later), or the GNU General Public License, version 2 (GPLv2), in all
+ *  cases as published by the Free Software Foundation.
+ */
+
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+#include "syncop.h"
+#include "glfs.h"
+#include "glfs-handles.h"
+
+static void
+glfs_iatt_from_stat (struct stat *stat, int valid, struct iatt *iatt,
+		     int *glvalid)
+{
+	/* validate in args */
+	if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
+		errno = EINVAL;
+		return;
+	}
+
+	*glvalid = 0;
+
+	if (valid & GFAPI_SET_ATTR_MODE) {
+		iatt->ia_prot = ia_prot_from_st_mode (stat->st_mode);
+		*glvalid |= GF_SET_ATTR_MODE;
+	}
+
+	if (valid & GFAPI_SET_ATTR_UID) {
+		iatt->ia_uid = stat->st_uid;
+		*glvalid |= GF_SET_ATTR_UID;
+	}
+
+	if (valid & GFAPI_SET_ATTR_GID) {
+		iatt->ia_gid = stat->st_gid;
+		*glvalid |= GF_SET_ATTR_GID;
+	}
+
+	if (valid & GFAPI_SET_ATTR_ATIME) {
+		iatt->ia_atime = stat->st_atime;
+		iatt->ia_atime_nsec = ST_ATIM_NSEC (stat);
+		*glvalid |= GF_SET_ATTR_ATIME;
+	}
+
+	if (valid & GFAPI_SET_ATTR_MTIME) {
+		iatt->ia_mtime = stat->st_mtime;
+		iatt->ia_mtime_nsec = ST_MTIM_NSEC (stat);
+		*glvalid |= GF_SET_ATTR_MTIME;
+	}
+
+	return;
+}
+
+struct glfs_object *
+glfs_h_lookupat (struct glfs *fs, struct glfs_object *parent,
+		 const char *path, struct stat *stat)
+{
+	int                      ret = 0;
+	xlator_t                *subvol = NULL;
+	inode_t                 *inode = NULL;
+	struct iatt              iatt = {0, };
+	struct glfs_object      *object = NULL;
+	loc_t                    loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	if (parent) {
+		inode = glfs_resolve_inode (fs, subvol, parent);
+		if (!inode) {
+			errno = ESTALE;
+			goto out;
+		}
+	}
+
+	/* fop/op */
+	ret = glfs_resolve_at (fs, subvol, inode, path, &loc, &iatt,
+			       0 /*TODO: links? */, 0);
+
+	/* populate out args */
+	if (!ret) {
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_stat (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* populate loc */
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_stat (subvol, &loc, &iatt);
+
+	/* populate out args */
+	if (!ret && stat) {
+		glfs_iatt_to_stat (fs, &iatt, stat);
+	}
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_getattrs (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+{
+	int                      ret = 0;
+	xlator_t                *subvol = NULL;
+	inode_t                 *inode = NULL;
+	struct iatt              iatt = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* fop/op */
+	ret = glfs_resolve_base (fs, subvol, inode, &iatt);
+
+	/* populate out args */
+	if (!ret && stat) {
+		glfs_iatt_to_stat (fs, &iatt, stat);
+	}
+
+out:
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_setattrs (struct glfs *fs, struct glfs_object *object, struct stat *stat,
+		 int valid)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              glvalid = 0;
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* map valid masks from in args */
+	glfs_iatt_from_stat (stat, valid, &iatt, &glvalid);
+
+	/* populate loc */
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_setattr (subvol, &loc, &iatt, glvalid, 0, 0);
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+struct glfs_fd *
+glfs_h_open (struct glfs *fs, struct glfs_object *object, int flags)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* check types to open */
+	if (IA_ISDIR (inode->ia_type)) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	if (!IA_ISREG (inode->ia_type)) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd) {
+		errno = ENOMEM;
+		goto out;
+	}
+
+	glfd->fd = fd_create (inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	/* populate loc */
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_open (subvol, &loc, flags, glfd->fd);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else {
+		glfd->fd->flags = flags;
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+struct glfs_object *
+glfs_h_creat (struct glfs *fs, struct glfs_object *parent, const char *path,
+	      int flags, mode_t mode, struct stat *stat)
+{
+	int                 ret = -1;
+	struct glfs_fd     *glfd = NULL;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	glfd->fd = fd_create (loc.inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	/* fop/op */
+	ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
+			     xattr_req, &iatt);
+
+	/* populate out args */
+	if (ret == 0) {
+		/* TODO: If the inode existed in the cache (say file already
+		   exists), then the glfs_loc_link will not update the
+		   loc.inode, as a result we will have a 0000 GFID that we
+		   would copy out to the object, this needs to be fixed.
+		*/
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	if (glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+struct glfs_object *
+glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent, const char *path,
+	      mode_t mode, struct stat *stat)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+
+	/* fop/op */
+	ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
+
+	/* populate out args */
+	if ( ret == 0 )  {
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+struct glfs_object *
+glfs_h_mknod (struct glfs *fs, struct glfs_object *parent, const char *path,
+	      mode_t mode, dev_t dev, struct stat *stat)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+
+	/* fop/op */
+	ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
+
+	/* populate out args */
+	if (ret == 0) {
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_unlink (struct glfs *fs, struct glfs_object *parent, const char *path)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if ( !subvol ) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	ret = glfs_resolve_at (fs, subvol, inode, path, &loc, NULL, 0 , 0);
+	if (ret != 0) {
+		goto out;
+	}
+
+	if (!IA_ISDIR(loc.inode->ia_type)) {
+		ret = syncop_unlink (subvol, &loc);
+		if (ret != 0) {
+			goto out;
+		}
+	} else {
+		ret = syncop_rmdir (subvol, &loc);
+		if (ret != 0) {
+			goto out;
+		}
+	}
+
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+struct glfs_fd *
+glfs_h_opendir (struct glfs *fs, struct glfs_object *object)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	if (!IA_ISDIR (inode->ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	INIT_LIST_HEAD (&glfd->entries);
+
+	glfd->fd = fd_create (inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_opendir (subvol, &loc, glfd->fd);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else {
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+ssize_t
+glfs_h_extract_handle (struct glfs_object *object, unsigned char *handle,
+		       int len)
+{
+	ssize_t ret = -1;
+
+	/* validate in args */
+	if (object == NULL) {
+		errno = EINVAL;
+		goto out;
+	}
+
+	if (!handle || !len) {
+		ret = GFAPI_HANDLE_LENGTH;
+		goto out;
+	}
+
+	if (len < GFAPI_HANDLE_LENGTH)
+	{
+		errno = ERANGE;
+		goto out;
+	}
+
+	memcpy (handle, object->gfid, GFAPI_HANDLE_LENGTH);
+
+	ret = GFAPI_HANDLE_LENGTH;
+
+out:
+	return ret;
+}
+
+struct glfs_object *
+glfs_h_create_from_handle (struct glfs *fs, unsigned char *handle, int len,
+			   struct stat *stat)
+{
+	loc_t               loc = {0, };
+	int                 ret = -1;
+	struct iatt         iatt = {0, };
+	inode_t            *newinode = NULL;
+	xlator_t           *subvol = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	memcpy (loc.gfid, handle, GFAPI_HANDLE_LENGTH);
+
+	newinode = inode_find (subvol->itable, loc.gfid);
+	if (newinode)
+		loc.inode = newinode;
+	else {
+		loc.inode = inode_new (subvol->itable);
+		if (!loc.inode) {
+			errno = ENOMEM;
+			goto out;
+		}
+	}
+
+	ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
+	if (ret) {
+		gf_log (subvol->name, GF_LOG_WARNING,
+			"inode refresh of %s failed: %s",
+			uuid_utoa (loc.gfid), strerror (errno));
+		goto out;
+	}
+
+	newinode = inode_link (loc.inode, 0, 0, &iatt);
+	if (newinode)
+		inode_lookup (newinode);
+	else {
+		gf_log (subvol->name, GF_LOG_WARNING,
+			"inode linking of %s failed: %s",
+			uuid_utoa (loc.gfid), strerror (errno));
+		errno = EINVAL;
+		goto out;
+	}
+
+	/* populate stat */
+	if (stat)
+		glfs_iatt_to_stat (fs, &iatt, stat);
+
+	object = GF_CALLOC (1, sizeof(struct glfs_object),
+			    glfs_mt_glfs_object_t);
+	if (object == NULL) {
+		errno = ENOMEM;
+		ret = -1;
+		goto out;
+	}
+
+	/* populate the return object */
+	object->inode = newinode;
+	uuid_copy (object->gfid, object->inode->gfid);
+
+out:
+	/* TODO: Check where the inode ref is being held? */
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_close (struct glfs_object *object)
+{
+	/* Release the held reference */
+	inode_unref (object->inode);
+	GF_FREE (object);
+
+	return 0;
+}
+
+int
+glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset)
+{
+	loc_t               loc = {0, };
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_truncate (subvol, &loc, (off_t)offset);
+
+	/* populate out args */
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+struct glfs_object *
+glfs_h_symlink (struct glfs *fs, struct glfs_object *parent, const char *name,
+		const char *data, struct stat *stat)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (name == NULL) ||
+		(data == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, name);
+
+	/* fop/op */
+	ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
+
+	/* populate out args */
+	if (ret == 0) {
+		/* TODO: If the inode existed in the cache (say file already
+		 * exists), then the glfs_loc_link will not update the
+		 * loc.inode, as a result we will have a 0000 GFID that we
+		 * would copy out to the object, this needs to be fixed.
+		 */
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
+		 size_t bufsiz)
+{
+	loc_t               loc = {0, };
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	char               *linkval = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL) || (buf == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
+
+	/* populate out args */
+	if (ret > 0)
+		memcpy (buf, linkval, ret);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (linkval)
+		GF_FREE (linkval);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_link (struct glfs *fs, struct glfs_object *linksrc,
+	     struct glfs_object *parent, const char *name)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	inode_t            *pinode = NULL;
+	loc_t               oldloc = {0, };
+	loc_t               newloc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (linksrc == NULL) || (parent == NULL) ||
+		(name == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, linksrc);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	if (inode->ia_type == IA_IFDIR) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, oldloc, out);
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	pinode = glfs_resolve_inode (fs, subvol, parent);
+	if (!pinode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* setup newloc based on parent */
+	newloc.parent = inode_ref (pinode);
+	newloc.name = name;
+	ret = glfs_loc_touchup (&newloc);
+	if (ret != 0) {
+		errno = EINVAL;
+		goto out;
+	}
+
+	/* Filling the inode of the hard link to be same as that of the
+	 * original file
+	 */
+	newloc.inode = inode_ref (inode);
+
+	/* fop/op */
+	ret = syncop_link (subvol, &oldloc, &newloc);
+
+	if (ret == 0)
+		/* TODO: No iatt to pass as there has been no lookup */
+		ret = glfs_loc_link (&newloc, NULL);
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (pinode)
+		inode_unref (pinode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_rename (struct glfs *fs, struct glfs_object *olddir, const char *oldname,
+	       struct glfs_object *newdir, const char *newname)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *oldpinode = NULL;
+	inode_t            *newpinode = NULL;
+	loc_t               oldloc = {0, };
+	loc_t               newloc = {0, };
+	struct iatt         oldiatt = {0, };
+	struct iatt         newiatt = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (olddir == NULL) || (oldname == NULL) ||
+		(newdir == NULL) || (newname == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if ( !subvol ) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	oldpinode = glfs_resolve_inode (fs, subvol, olddir);
+	if (!oldpinode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	ret = glfs_resolve_at (fs, subvol, oldpinode, oldname, &oldloc,
+			       &oldiatt, 0 , 0);
+	if (ret != 0) {
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	newpinode = glfs_resolve_inode (fs, subvol, newdir);
+	if (!newpinode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	ret = glfs_resolve_at (fs, subvol, newpinode, newname, &newloc,
+			       &newiatt, 0, 0);
+
+	if (ret && errno != ENOENT && newloc.parent)
+		goto out;
+
+	if (newiatt.ia_type != IA_INVAL) {
+		if ((oldiatt.ia_type == IA_IFDIR) !=
+			(newiatt.ia_type == IA_IFDIR)) {
+			/* Either both old and new must be dirs,
+			 * or both must be non-dirs. Else, fail.
+			 */
+			ret = -1;
+			errno = EISDIR;
+			goto out;
+		}
+	}
+
+	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+	ret = syncop_rename (subvol, &oldloc, &newloc);
+
+	if (ret == 0)
+		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
+			      newloc.parent, newloc.name, oldloc.inode,
+			      &oldiatt);
+
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	if (oldpinode)
+		inode_unref (oldpinode);
+
+	if (newpinode)
+		inode_unref (newpinode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
new file mode 100644
index 000000000..437f2cbc8
--- /dev/null
+++ b/api/src/glfs-handles.h
@@ -0,0 +1,143 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_HANDLES_H
+#define _GLFS_HANDLES_H
+
+#include "glfs.h"
+
+/* GLFS OBJECT BASED OPERATIONS
+ *
+ * The following APIs are introduced to provide an API framework that can work
+ * with gluster objects (files and directories), instead of absolute paths.
+ *
+ * The following API set can be related to the POSIX *at interfaces (like
+ * openat (2)). The intention of these APIs is to be able to operate based
+ * on parent object and looking up or creating child objects within, OR to be
+ * used on the actual object thus looked up or created, and retrieve information
+ * regarding the same.
+ *
+ * The APIs also provide for generating an opaque invariant handle to the
+ * object, that can later be used to lookup the object, instead of the regular
+ * glfs_h_* variants. The APIs that provide this behaviour are,
+ * glfs_h_extract_handle and glfs_h_create_from_handle.
+ *
+ * The object handles can be transitioned to fd based operations as supported
+ * by glfs.h calls, using the glfs_h_open call. This provides a way to move
+ * from objects to fd's akin to moving from path to fd for required operations.
+ *
+ * NOTE: The opaque invariant handle is the GFID of the object in reality, but
+ * maintained as an opaque data value, for potential internal changes to the
+ * same without impacting the caller.
+ *
+ * NOTE: Currently looking up an object can create multiple object handles to
+ * the same, i.e distinct glfs_object *. Hence each such looked up or received
+ * handle from other calls, would need to be closed. In the future, for a given
+ * object these pointers would be the same, and an ease of use API to forget all
+ * instances of this bject would be provided (instead of a per lookup close).
+ * This should not change the APIs in their current form.
+ *
+ */
+
+/* Values for valid falgs to be used when using XXXsetattr, to set multiple
+ attribute values passed via the related stat structure.
+ */
+#define GFAPI_SET_ATTR_MODE  0x1
+#define GFAPI_SET_ATTR_UID   0x2
+#define GFAPI_SET_ATTR_GID   0x4
+#define GFAPI_SET_ATTR_SIZE  0x8
+#define GFAPI_SET_ATTR_ATIME 0x10
+#define GFAPI_SET_ATTR_MTIME 0x20
+
+/* Handle length for object handles returned from glfs_h_extract_handle or
+ * glfs_h_create_from_handle */
+#define GFAPI_HANDLE_LENGTH 16
+
+__BEGIN_DECLS
+
+/*
+ * Notes:
+ *
+ * The file object handle. One per looked up, created file/directory
+ *
+ * This had been introduced to facilitate gfid/inode based gfapi
+ * - a requirement introduced by nfs-ganesha
+ */
+struct glfs_object;
+typedef struct glfs_object glfs_object_t;
+
+/* Handle based operations */
+/* Operations that generate handles */
+struct glfs_object *glfs_h_lookupat (struct glfs *fs,
+				     struct glfs_object *parent,
+				     const char *path, struct stat *stat);
+
+struct glfs_object *glfs_h_creat (struct glfs *fs, struct glfs_object *parent,
+				  const char *path, int flags, mode_t mode,
+				  struct stat *sb);
+
+struct glfs_object *glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent,
+				  const char *path, mode_t flags,
+				  struct stat *sb);
+
+struct glfs_object *glfs_h_mknod (struct glfs *fs, struct glfs_object *parent,
+				  const char *path, mode_t mode, dev_t dev,
+				  struct stat *sb);
+
+struct glfs_object *glfs_h_symlink (struct glfs *fs, struct glfs_object *parent,
+				    const char *name, const char *data,
+				    struct stat *stat);
+
+/* Operations on the actual objects */
+int glfs_h_unlink (struct glfs *fs, struct glfs_object *parent,
+		   const char *path);
+
+int glfs_h_close (struct glfs_object *object);
+
+int glfs_caller_specific_init (void *uid_caller_key, void *gid_caller_key,
+			       void *future);
+
+int glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset);
+
+int glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat);
+
+int glfs_h_getattrs (struct glfs *fs, struct glfs_object *object,
+		     struct stat *stat);
+
+int glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
+		     struct stat *sb, int valid);
+
+int glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
+		     size_t bufsiz);
+
+int glfs_h_link (struct glfs *fs, struct glfs_object *linktgt,
+		 struct glfs_object *parent, const char *name);
+
+int glfs_h_rename (struct glfs *fs, struct glfs_object *olddir,
+		   const char *oldname, struct glfs_object *newdir,
+		   const char *newname);
+
+/* Operations enabling opaque invariant handle to object transitions */
+ssize_t glfs_h_extract_handle (struct glfs_object *object,
+			       unsigned char *handle, int len);
+
+struct glfs_object *glfs_h_create_from_handle (struct glfs *fs,
+					       unsigned char *handle, int len,
+					       struct stat *stat);
+
+/* Operations enabling object handles to fd transitions */
+struct glfs_fd *glfs_h_opendir (struct glfs *fs, struct glfs_object *object);
+
+struct glfs_fd *glfs_h_open (struct glfs *fs, struct glfs_object *object,
+			     int flags);
+
+__END_DECLS
+
+#endif /* !_GLFS_HANDLES_H */
+\ No newline at end of file
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
new file mode 100644
index 000000000..ec1d5579d
--- /dev/null
+++ b/api/src/glfs-internal.h
@@ -0,0 +1,200 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _GLFS_INTERNAL_H
+#define _GLFS_INTERNAL_H
+
+#include "xlator.h"
+
+#define GLFS_SYMLINK_MAX_FOLLOW 2048
+
+#define DEFAULT_REVAL_COUNT 1
+
+#define ESTALE_RETRY(ret,errno,reval,loc,label) do {	\
+	if (ret == -1 && errno == ESTALE) {	        \
+		if (reval < DEFAULT_REVAL_COUNT) {	\
+			reval++;			\
+			loc_wipe (loc);			\
+			goto label;			\
+		}					\
+	}						\
+	} while (0)
+
+#define GLFS_LOC_FILL_INODE(oinode, loc, label) do {   \
+	loc.inode = inode_ref (oinode);                \
+	uuid_copy (loc.gfid, oinode->gfid);            \
+	ret = glfs_loc_touchup (&loc);                 \
+	if (ret != 0) {                                \
+		errno = EINVAL;                        \
+		goto label;                            \
+	}                                              \
+	} while (0)
+
+#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path) do {   \
+	loc.inode = inode_new (pinode->table);                            \
+	if (!loc.inode) {                                                 \
+		ret = -1;                                                 \
+		errno = ENOMEM;                                           \
+		goto label;                                               \
+	}                                                                 \
+	loc.parent = inode_ref (pinode);                                  \
+	loc.name = path;                                                  \
+	ret = glfs_loc_touchup (&loc);                                    \
+	if (ret != 0) {                                                   \
+		errno = EINVAL;                                           \
+		goto label;                                               \
+	}                                                                 \
+	} while (0)
+
+struct glfs;
+
+typedef int (*glfs_init_cbk) (struct glfs *fs, int ret);
+
+struct glfs {
+	char               *volname;
+
+	glusterfs_ctx_t    *ctx;
+
+	pthread_t           poller;
+
+	glfs_init_cbk       init_cbk;
+	pthread_mutex_t     mutex;
+	pthread_cond_t      cond;
+	int                 init;
+	int                 ret;
+	int                 err;
+
+	xlator_t           *active_subvol;
+	xlator_t           *next_subvol;
+	xlator_t           *old_subvol;
+
+	char               *oldvolfile;
+	ssize_t             oldvollen;
+
+	inode_t            *cwd;
+
+	uint32_t            dev_id; /* Used to fill st_dev in struct stat */
+
+	struct list_head    openfds;
+
+	gf_boolean_t        migration_in_progress;
+};
+
+struct glfs_fd {
+	struct list_head   openfds;
+	struct glfs       *fs;
+	off_t              offset;
+	fd_t              *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
+	struct list_head   entries;
+	gf_dirent_t       *next;
+	struct dirent     *readdirbuf;
+};
+
+/* glfs object handle introduced for the alternate gfapi implementation based
+   on glfs handles/gfid/inode
+*/
+struct glfs_object {
+        inode_t         *inode;
+        uuid_t          gfid;
+};
+
+#define DEFAULT_EVENT_POOL_SIZE           16384
+#define GF_MEMPOOL_COUNT_OF_DICT_T        4096
+#define GF_MEMPOOL_COUNT_OF_DATA_T        (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T   (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+
+int glfs_mgmt_init (struct glfs *fs);
+void glfs_init_done (struct glfs *fs, int ret);
+int glfs_process_volfp (struct glfs *fs, FILE *fp);
+int glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+		  struct iatt *iatt, int reval);
+int glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+		   struct iatt *iatt, int reval);
+fd_t *glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+fd_t *__glfs_migrate_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+int glfs_first_lookup (xlator_t *subvol);
+
+static inline void
+__glfs_entry_fs (struct glfs *fs)
+{
+	THIS = fs->ctx->master;
+}
+
+
+static inline void
+__glfs_entry_fd (struct glfs_fd *fd)
+{
+	THIS = fd->fd->inode->table->xl->ctx->master;
+}
+
+
+/*
+  By default all lock attempts from user context must
+  use glfs_lock() and glfs_unlock(). This allows
+  for a safe implementation of graph migration where
+  we can give up the mutex during syncop calls so
+  that bottom up calls (particularly CHILD_UP notify)
+  can do a mutex_lock() on @glfs without deadlocking
+  the filesystem
+*/
+static inline int
+glfs_lock (struct glfs *fs)
+{
+	pthread_mutex_lock (&fs->mutex);
+
+	while (!fs->init)
+		pthread_cond_wait (&fs->cond, &fs->mutex);
+
+	while (fs->migration_in_progress)
+		pthread_cond_wait (&fs->cond, &fs->mutex);
+
+	return 0;
+}
+
+
+static inline void
+glfs_unlock (struct glfs *fs)
+{
+	pthread_mutex_unlock (&fs->mutex);
+}
+
+
+void glfs_fd_destroy (struct glfs_fd *glfd);
+
+struct glfs_fd *glfs_fd_new (struct glfs *fs);
+void glfs_fd_bind (struct glfs_fd *glfd);
+
+xlator_t * glfs_active_subvol (struct glfs *fs);
+xlator_t * __glfs_active_subvol (struct glfs *fs);
+void glfs_subvol_done (struct glfs *fs, xlator_t *subvol);
+
+inode_t * glfs_refresh_inode (xlator_t *subvol, inode_t *inode);
+
+inode_t *glfs_cwd_get (struct glfs *fs);
+int glfs_cwd_set (struct glfs *fs, inode_t *inode);
+inode_t *glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
+			     struct glfs_object *object);
+int glfs_create_object (loc_t *loc, struct glfs_object **retobject);
+int __glfs_cwd_set (struct glfs *fs, inode_t *inode);
+
+int glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
+		       struct iatt *iatt);
+int glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
+                     const char *origpath, loc_t *loc, struct iatt *iatt,
+                     int follow, int reval);
+int glfs_loc_touchup (loc_t *loc);
+void glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat);
+int glfs_loc_link (loc_t *loc, struct iatt *iatt);
+int glfs_loc_unlink (loc_t *loc);
+
+#endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
new file mode 100644
index 000000000..c02534c18
--- /dev/null
+++ b/api/src/glfs-master.c
@@ -0,0 +1,154 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <limits.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "glusterfs.h"
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+
+
+int
+glfs_graph_setup (struct glfs *fs, glusterfs_graph_t *graph)
+{
+	xlator_t      *new_subvol = NULL;
+	xlator_t      *old_subvol = NULL;
+	inode_table_t *itable = NULL;
+	int            ret = -1;
+
+	new_subvol = graph->top;
+
+	/* This is called in a bottom-up context, it should specifically
+	   NOT be glfs_lock()
+	*/
+	pthread_mutex_lock (&fs->mutex);
+	{
+		if (new_subvol->switched ||
+		    new_subvol == fs->active_subvol ||
+		    new_subvol == fs->next_subvol) {
+			/* Spurious CHILD_UP event on old graph */
+			ret = 0;
+			goto unlock;
+		}
+
+		if (!new_subvol->itable) {
+			itable = inode_table_new (131072, new_subvol);
+			if (!itable) {
+				errno = ENOMEM;
+				ret = -1;
+				goto unlock;
+			}
+
+			new_subvol->itable = itable;
+		}
+
+		old_subvol = fs->next_subvol;
+		fs->next_subvol = new_subvol;
+		fs->next_subvol->winds++; /* first ref */
+		ret = 0;
+	}
+unlock:
+	pthread_mutex_unlock (&fs->mutex);
+
+	if (old_subvol)
+		/* wasn't picked up so far, skip */
+		glfs_subvol_done (fs, old_subvol);
+
+	return ret;
+}
+
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+	glusterfs_graph_t   *graph = NULL;
+	struct glfs	    *fs = NULL;
+
+	graph = data;
+	fs = this->private;
+
+	switch (event) {
+	case GF_EVENT_GRAPH_NEW:
+		gf_log (this->name, GF_LOG_INFO, "New graph %s (%d) coming up",
+			uuid_utoa ((unsigned char *)graph->graph_uuid),
+			graph->id);
+		break;
+	case GF_EVENT_CHILD_UP:
+		glfs_graph_setup (fs, graph);
+		glfs_init_done (fs, 0);
+		break;
+	case GF_EVENT_CHILD_DOWN:
+		glfs_graph_setup (fs, graph);
+		glfs_init_done (fs, 1);
+		break;
+	case GF_EVENT_CHILD_CONNECTING:
+		break;
+	default:
+		gf_log (this->name, GF_LOG_DEBUG,
+			"got notify event %d", event);
+		break;
+	}
+
+	return 0;
+}
+
+
+int
+mem_acct_init (xlator_t *this)
+{
+	int ret = -1;
+
+	if (!this)
+		return ret;
+
+	ret = xlator_mem_acct_init (this, glfs_mt_end + 1);
+	if (ret) {
+		gf_log (this->name, GF_LOG_ERROR, "Failed to initialise "
+                        "memory accounting");
+		return ret;
+	}
+
+	return 0;
+}
+
+
+int
+init (xlator_t *this)
+{
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+
+}
+
+
+struct xlator_dumpops dumpops;
+
+
+struct xlator_fops fops;
+
+
+struct xlator_cbks cbks;
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
new file mode 100644
index 000000000..3301b3da5
--- /dev/null
+++ b/api/src/glfs-mem-types.h
@@ -0,0 +1,32 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_MEM_TYPES_H
+#define _GLFS_MEM_TYPES_H
+
+#include "mem-types.h"
+
+#define GF_MEM_TYPE_START (gf_common_mt_end + 1)
+
+enum glfs_mem_types_ {
+        glfs_mt_glfs_t = GF_MEM_TYPE_START,
+        glfs_mt_call_pool_t,
+        glfs_mt_xlator_t,
+	glfs_mt_glfs_fd_t,
+	glfs_mt_glfs_io_t,
+	glfs_mt_volfile_t,
+	glfs_mt_xlator_cmdline_option_t,
+	glfs_mt_glfs_object_t,
+	glfs_mt_readdirbuf_t,
+	glfs_mt_end
+
+};
+#endif
+
diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
new file mode 100644
index 000000000..6843e9cb3
--- /dev/null
+++ b/api/src/glfs-mgmt.c
@@ -0,0 +1,543 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <pthread.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif /* _CONFIG_H */
+
+#include "glusterfs.h"
+#include "stack.h"
+#include "dict.h"
+#include "event.h"
+#include "defaults.h"
+
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "glusterfs3.h"
+#include "portmap-xdr.h"
+#include "xdr-generic.h"
+
+#include "syncop.h"
+#include "xlator.h"
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+
+
+int glfs_volfile_fetch (struct glfs *fs);
+
+int
+glfs_process_volfp (struct glfs *fs, FILE *fp)
+{
+	glusterfs_graph_t  *graph = NULL;
+	int		    ret = -1;
+	xlator_t	   *trav = NULL;
+	glusterfs_ctx_t	   *ctx = NULL;
+
+	ctx = fs->ctx;
+	graph = glusterfs_graph_construct (fp);
+	if (!graph) {
+		gf_log ("glfs", GF_LOG_ERROR, "failed to construct the graph");
+		goto out;
+	}
+
+	for (trav = graph->first; trav; trav = trav->next) {
+		if (strcmp (trav->type, "mount/fuse") == 0) {
+			gf_log ("glfs", GF_LOG_ERROR,
+				"fuse xlator cannot be specified "
+				"in volume file");
+			goto out;
+		}
+	}
+
+	ret = glusterfs_graph_prepare (graph, ctx);
+	if (ret) {
+		glusterfs_graph_destroy (graph);
+		goto out;
+	}
+
+	ret = glusterfs_graph_activate (graph, ctx);
+
+	if (ret) {
+		glusterfs_graph_destroy (graph);
+		goto out;
+	}
+
+	ret = 0;
+out:
+	if (fp)
+		fclose (fp);
+
+	if (!ctx->active) {
+		ret = -1;
+	}
+
+	return ret;
+}
+
+
+int
+mgmt_cbk_spec (struct rpc_clnt *rpc, void *mydata, void *data)
+{
+	struct glfs *fs = NULL;
+	xlator_t    *this = NULL;
+
+	this = mydata;
+	fs = this->private;
+
+	glfs_volfile_fetch (fs);
+
+	return 0;
+}
+
+
+int
+mgmt_cbk_event (struct rpc_clnt *rpc, void *mydata, void *data)
+{
+	return 0;
+}
+
+
+rpcclnt_cb_actor_t mgmt_cbk_actors[] = {
+	[GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
+	[GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
+				 mgmt_cbk_event},
+};
+
+
+struct rpcclnt_cb_program mgmt_cbk_prog = {
+	.progname  = "GlusterFS Callback",
+	.prognum   = GLUSTER_CBK_PROGRAM,
+	.progver   = GLUSTER_CBK_VERSION,
+	.actors	   = mgmt_cbk_actors,
+	.numactors = GF_CBK_MAXVALUE,
+};
+
+char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+	[GF_HNDSK_NULL]		= "NULL",
+	[GF_HNDSK_SETVOLUME]	= "SETVOLUME",
+	[GF_HNDSK_GETSPEC]	= "GETSPEC",
+	[GF_HNDSK_PING]		= "PING",
+	[GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+};
+
+rpc_clnt_prog_t clnt_handshake_prog = {
+	.progname  = "GlusterFS Handshake",
+	.prognum   = GLUSTER_HNDSK_PROGRAM,
+	.progver   = GLUSTER_HNDSK_VERSION,
+	.procnames = clnt_handshake_procs,
+};
+
+
+int
+mgmt_submit_request (void *req, call_frame_t *frame,
+		     glusterfs_ctx_t *ctx,
+		     rpc_clnt_prog_t *prog, int procnum,
+		     fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+	int			ret	    = -1;
+	int			count	   = 0;
+	struct iovec		iov	    = {0, };
+	struct iobuf		*iobuf = NULL;
+	struct iobref		*iobref = NULL;
+	ssize_t			xdr_size = 0;
+
+	iobref = iobref_new ();
+	if (!iobref) {
+		goto out;
+	}
+
+	if (req) {
+		xdr_size = xdr_sizeof (xdrproc, req);
+
+		iobuf = iobuf_get2 (ctx->iobuf_pool, xdr_size);
+		if (!iobuf) {
+			goto out;
+		};
+
+		iobref_add (iobref, iobuf);
+
+		iov.iov_base = iobuf->ptr;
+		iov.iov_len  = iobuf_pagesize (iobuf);
+
+		/* Create the xdr payload */
+		ret = xdr_serialize_generic (iov, req, xdrproc);
+		if (ret == -1) {
+			gf_log (THIS->name, GF_LOG_WARNING,
+				"failed to create XDR payload");
+			goto out;
+		}
+		iov.iov_len = ret;
+		count = 1;
+	}
+
+	/* Send the msg */
+	ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
+			       &iov, count,
+			       NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+
+out:
+	if (iobref)
+		iobref_unref (iobref);
+
+	if (iobuf)
+		iobuf_unref (iobuf);
+	return ret;
+}
+
+
+static int
+glusterfs_oldvolfile_update (struct glfs *fs, char *volfile, ssize_t size)
+{
+	int ret = -1;
+
+	fs->oldvollen = size;
+	if (!fs->oldvolfile) {
+		fs->oldvolfile = GF_CALLOC (1, size+1, glfs_mt_volfile_t);
+	} else {
+		fs->oldvolfile = GF_REALLOC (fs->oldvolfile, size+1);
+	}
+
+	if (!fs->oldvolfile) {
+		fs->oldvollen = 0;
+	} else {
+		memcpy (fs->oldvolfile, volfile, size);
+		fs->oldvollen = size;
+		ret = 0;
+	}
+
+	return ret;
+}
+
+
+int
+mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
+		  void *myframe)
+{
+	gf_getspec_rsp		 rsp   = {0,};
+	call_frame_t		*frame = NULL;
+	glusterfs_ctx_t		*ctx = NULL;
+	int			 ret   = 0;
+	ssize_t			 size = 0;
+	FILE			*tmpfp = NULL;
+	int                      need_retry = 0;
+	struct glfs		*fs = NULL;
+
+	frame = myframe;
+	ctx = frame->this->ctx;
+	fs = ((xlator_t *)ctx->master)->private;
+
+	if (-1 == req->rpc_status) {
+		ret = -1;
+		need_retry = 1;
+		goto out;
+	}
+
+	ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+	if (ret < 0) {
+		gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+		ret   = -1;
+		goto out;
+	}
+
+	if (-1 == rsp.op_ret) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"failed to get the 'volume file' from server");
+		ret = -1;
+                errno = rsp.op_errno;
+		goto out;
+	}
+
+	ret = 0;
+	size = rsp.op_ret;
+
+	if ((size == fs->oldvollen) &&
+	    (memcmp (fs->oldvolfile, rsp.spec, size) == 0)) {
+		gf_log (frame->this->name, GF_LOG_INFO,
+			"No change in volfile, continuing");
+		goto out;
+	}
+
+	tmpfp = tmpfile ();
+	if (!tmpfp) {
+		ret = -1;
+		goto out;
+	}
+
+	fwrite (rsp.spec, size, 1, tmpfp);
+	fflush (tmpfp);
+	if (ferror (tmpfp)) {
+		ret = -1;
+		goto out;
+	}
+
+	/*  Check if only options have changed. No need to reload the
+	*  volfile if topology hasn't changed.
+	*  glusterfs_volfile_reconfigure returns 3 possible return states
+	*  return 0	     =======> reconfiguration of options has succeeded
+	*  return 1	     =======> the graph has to be reconstructed and all the xlators should be inited
+	*  return -1(or -ve) =======> Some Internal Error occurred during the operation
+	*/
+
+	ret = glusterfs_volfile_reconfigure (fs->oldvollen, tmpfp, fs->ctx,
+                                             fs->oldvolfile);
+	if (ret == 0) {
+		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+			"No need to re-load volfile, reconfigure done");
+		ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
+		goto out;
+	}
+
+	if (ret < 0) {
+		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+			"Reconfigure failed !!");
+		goto out;
+	}
+
+	ret = glfs_process_volfp (fs, tmpfp);
+	/* tmpfp closed */
+	tmpfp = NULL;
+	if (ret)
+		goto out;
+
+	ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
+out:
+	STACK_DESTROY (frame->root);
+
+	if (rsp.spec)
+		free (rsp.spec);
+
+        // Stop if server is running at an unsupported op-version
+        if (ENOTSUP == ret) {
+                gf_log ("mgmt", GF_LOG_ERROR, "Server is operating at an "
+                        "op-version which is not supported");
+                errno = ENOTSUP;
+                glfs_init_done (fs, -1);
+        }
+
+	if (ret && ctx && !ctx->active) {
+		/* Do it only for the first time */
+		/* Failed to get the volume file, something wrong,
+		   restart the process */
+		gf_log ("glfs-mgmt", GF_LOG_ERROR,
+			"failed to fetch volume file (key:%s)",
+			ctx->cmd_args.volfile_id);
+		if (!need_retry) {
+                        if (!errno)
+                                errno = EINVAL;
+			glfs_init_done (fs, -1);
+                }
+	}
+
+	if (tmpfp)
+		fclose (tmpfp);
+
+	return 0;
+}
+
+
+int
+glfs_volfile_fetch (struct glfs *fs)
+{
+	cmd_args_t	 *cmd_args = NULL;
+	gf_getspec_req	  req = {0, };
+	int		  ret = 0;
+	call_frame_t	 *frame = NULL;
+	glusterfs_ctx_t	 *ctx = NULL;
+        dict_t           *dict = NULL;
+
+	ctx = fs->ctx;
+	cmd_args = &ctx->cmd_args;
+
+	frame = create_frame (THIS, ctx->pool);
+
+	req.key = cmd_args->volfile_id;
+	req.flags = 0;
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        // Set the supported min and max op-versions, so glusterd can make a
+        // decision
+        ret = dict_set_int32 (dict, "min-op-version", GD_OP_VERSION_MIN);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, "max-op-version", GD_OP_VERSION_MAX);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
+                                           &req.xdata.xdata_len);
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to serialize dictionary");
+                goto out;
+        }
+
+	ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
+				   GF_HNDSK_GETSPEC, mgmt_getspec_cbk,
+				   (xdrproc_t)xdr_gf_getspec_req);
+out:
+        return ret;
+}
+
+
+static int
+mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+		 void *data)
+{
+	xlator_t	*this = NULL;
+	cmd_args_t	*cmd_args = NULL;
+	glusterfs_ctx_t *ctx = NULL;
+	struct glfs	 *fs = NULL;
+	int		 ret = 0;
+
+	this = mydata;
+	ctx = this->ctx;
+	fs = ((xlator_t *)ctx->master)->private;
+	cmd_args = &ctx->cmd_args;
+
+	switch (event) {
+	case RPC_CLNT_DISCONNECT:
+		if (!ctx->active) {
+			cmd_args->max_connect_attempts--;
+			gf_log ("glfs-mgmt", GF_LOG_ERROR,
+				"failed to connect with remote-host: %s",
+				strerror (errno));
+			gf_log ("glfs-mgmt", GF_LOG_INFO,
+				"%d connect attempts left",
+				cmd_args->max_connect_attempts);
+			if (0 >= cmd_args->max_connect_attempts) {
+                                errno = ENOTCONN;
+				glfs_init_done (fs, -1);
+                        }
+		}
+		break;
+	case RPC_CLNT_CONNECT:
+		rpc_clnt_set_connected (&((struct rpc_clnt*)ctx->mgmt)->conn);
+
+		ret = glfs_volfile_fetch (fs);
+		if (ret && ctx && (ctx->active == NULL)) {
+			/* Do it only for the first time */
+			/* Exit the process.. there are some wrong options */
+			gf_log ("glfs-mgmt", GF_LOG_ERROR,
+				"failed to fetch volume file (key:%s)",
+				ctx->cmd_args.volfile_id);
+                        errno = EINVAL;
+			glfs_init_done (fs, -1);
+		}
+
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+
+int
+glusterfs_mgmt_notify (int32_t op, void *data, ...)
+{
+	int ret = 0;
+
+	switch (op)
+	{
+		case GF_EN_DEFRAG_STATUS:
+			break;
+
+		default:
+			break;
+	}
+
+	return ret;
+}
+
+
+int
+glfs_mgmt_init (struct glfs *fs)
+{
+	cmd_args_t		*cmd_args = NULL;
+	struct rpc_clnt		*rpc = NULL;
+	dict_t			*options = NULL;
+	int			ret = -1;
+	int			port = GF_DEFAULT_BASE_PORT;
+	char			*host = NULL;
+	glusterfs_ctx_t		*ctx = NULL;
+
+	ctx = fs->ctx;
+	cmd_args = &ctx->cmd_args;
+
+	if (ctx->mgmt)
+		return 0;
+
+	if (cmd_args->volfile_server_port)
+		port = cmd_args->volfile_server_port;
+
+	host = "localhost";
+	if (cmd_args->volfile_server)
+		host = cmd_args->volfile_server;
+
+	ret = rpc_transport_inet_options_build (&options, host, port);
+	if (ret)
+		goto out;
+
+	rpc = rpc_clnt_new (options, THIS->ctx, THIS->name, 8);
+	if (!rpc) {
+		ret = -1;
+		gf_log (THIS->name, GF_LOG_WARNING,
+			"failed to create rpc clnt");
+		goto out;
+	}
+
+	ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
+	if (ret) {
+		gf_log (THIS->name, GF_LOG_WARNING,
+			"failed to register notify function");
+		goto out;
+	}
+
+	ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog, THIS);
+	if (ret) {
+		gf_log (THIS->name, GF_LOG_WARNING,
+			"failed to register callback function");
+		goto out;
+	}
+
+	ctx->notify = glusterfs_mgmt_notify;
+
+	/* This value should be set before doing the 'rpc_clnt_start()' as
+	   the notify function uses this variable */
+	ctx->mgmt = rpc;
+
+	ret = rpc_clnt_start (rpc);
+out:
+	return ret;
+}
+
diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c
new file mode 100644
index 000000000..4ca2eb6fc
--- /dev/null
+++ b/api/src/glfs-resolve.c
@@ -0,0 +1,969 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <limits.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "stack.h"
+#include "event.h"
+#include "glfs-mem-types.h"
+#include "common-utils.h"
+#include "syncop.h"
+#include "call-stub.h"
+
+#include "glfs-internal.h"
+
+#define graphid_str(subvol) (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
+
+
+int
+glfs_first_lookup_safe (xlator_t *subvol)
+{
+	loc_t  loc = {0, };
+	int    ret = -1;
+
+	loc.inode = subvol->itable->root;
+	memset (loc.gfid, 0, 16);
+	loc.gfid[15] = 1;
+	loc.path = "/";
+	loc.name = "";
+
+	ret = syncop_lookup (subvol, &loc, 0, 0, 0, 0);
+
+	gf_log (subvol->name, GF_LOG_DEBUG, "first lookup complete %d", ret);
+
+	return ret;
+}
+
+
+int
+__glfs_first_lookup (struct glfs *fs, xlator_t *subvol)
+{
+	int ret = -1;
+
+	fs->migration_in_progress = 1;
+	pthread_mutex_unlock (&fs->mutex);
+	{
+		ret = glfs_first_lookup_safe (subvol);
+	}
+	pthread_mutex_lock (&fs->mutex);
+	fs->migration_in_progress = 0;
+	pthread_cond_broadcast (&fs->cond);
+
+	return ret;
+}
+
+
+inode_t *
+glfs_refresh_inode_safe (xlator_t *subvol, inode_t *oldinode)
+{
+	loc_t        loc = {0, };
+	int          ret = -1;
+	struct iatt  iatt = {0, };
+	inode_t     *newinode = NULL;
+
+
+	if (!oldinode)
+		return NULL;
+
+	if (oldinode->table->xl == subvol)
+		return inode_ref (oldinode);
+
+	newinode = inode_find (subvol->itable, oldinode->gfid);
+	if (newinode)
+		return newinode;
+
+	uuid_copy (loc.gfid, oldinode->gfid);
+	loc.inode = inode_new (subvol->itable);
+	if (!loc.inode)
+		return NULL;
+
+	ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
+
+	if (ret) {
+		gf_log (subvol->name, GF_LOG_WARNING,
+			"inode refresh of %s failed: %s",
+			uuid_utoa (oldinode->gfid), strerror (errno));
+		loc_wipe (&loc);
+		return NULL;
+	}
+
+	newinode = inode_link (loc.inode, 0, 0, &iatt);
+	if (newinode)
+		inode_lookup (newinode);
+
+	loc_wipe (&loc);
+
+	return newinode;
+}
+
+
+inode_t *
+__glfs_refresh_inode (struct glfs *fs, xlator_t *subvol, inode_t *inode)
+{
+	inode_t *newinode = NULL;
+
+	fs->migration_in_progress = 1;
+	pthread_mutex_unlock (&fs->mutex);
+	{
+		newinode = glfs_refresh_inode_safe (subvol, inode);
+	}
+	pthread_mutex_lock (&fs->mutex);
+	fs->migration_in_progress = 0;
+	pthread_cond_broadcast (&fs->cond);
+
+	return newinode;
+}
+
+int
+glfs_loc_touchup (loc_t *loc)
+{
+	char *path = NULL;
+	int   ret = -1;
+	char *bn = NULL;
+
+	if (loc->parent)
+		ret = inode_path (loc->parent, loc->name, &path);
+	else
+		ret = inode_path (loc->inode, 0, &path);
+
+	loc->path = path;
+
+	if (ret < 0 || !path) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	bn = strrchr (path, '/');
+	if (bn)
+		bn++;
+	loc->name = bn;
+	ret = 0;
+out:
+	return ret;
+}
+
+
+int
+glfs_resolve_symlink (struct glfs *fs, xlator_t *subvol, inode_t *inode,
+		      char **lpath)
+{
+	loc_t  loc = {0, };
+	char  *path = NULL;
+	char  *rpath = NULL;
+	int    ret = -1;
+
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
+	ret = inode_path (inode, NULL, &rpath);
+	if (ret < 0)
+		goto out;
+	loc.path = rpath;
+
+	ret = syncop_readlink (subvol, &loc, &path, 4096);
+
+	if (ret < 0)
+		goto out;
+
+	if (lpath)
+		*lpath = path;
+out:
+	loc_wipe (&loc);
+	return ret;
+}
+
+
+int
+glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
+		   struct iatt *iatt)
+{
+	loc_t       loc = {0, };
+	int         ret = -1;
+	char       *path = NULL;
+
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
+
+	ret = inode_path (loc.inode, NULL, &path);
+	loc.path = path;
+	if (ret < 0)
+		goto out;
+
+	ret = syncop_lookup (subvol, &loc, NULL, iatt, NULL, NULL);
+out:
+	loc_wipe (&loc);
+
+	return ret;
+}
+
+
+inode_t *
+glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
+			const char *component, struct iatt *iatt,
+			int force_lookup)
+{
+	loc_t        loc = {0, };
+	inode_t     *inode = NULL;
+	int          reval = 0;
+	int          ret = -1;
+	int          glret = -1;
+	struct iatt  ciatt = {0, };
+	uuid_t       gfid;
+	dict_t      *xattr_req = NULL;
+
+	loc.name = component;
+
+	loc.parent = inode_ref (parent);
+	uuid_copy (loc.pargfid, parent->gfid);
+
+
+	if (strcmp (component, ".") == 0)
+		loc.inode = inode_ref (parent);
+	else if (strcmp (component, "..") == 0)
+		loc.inode = inode_parent (parent, 0, 0);
+	else
+		loc.inode = inode_grep (parent->table, parent, component);
+
+	if (loc.inode) {
+		uuid_copy (loc.gfid, loc.inode->gfid);
+		reval = 1;
+
+		if (!force_lookup) {
+			inode = inode_ref (loc.inode);
+			ciatt.ia_type = inode->ia_type;
+			goto found;
+		}
+	} else {
+		uuid_generate (gfid);
+		loc.inode = inode_new (parent->table);
+	}
+
+	if (!loc.inode)
+		goto out;
+
+	glret = glfs_loc_touchup (&loc);
+	if (glret < 0) {
+		ret = -1;
+		goto out;
+	}
+
+	ret = syncop_lookup (subvol, &loc, NULL, &ciatt, NULL, NULL);
+	if (ret && reval) {
+		inode_unref (loc.inode);
+		loc.inode = inode_new (parent->table);
+		if (!loc.inode) {
+			errno = ENOMEM;
+			goto out;
+		}
+
+		xattr_req = dict_new ();
+		if (!xattr_req) {
+			errno = ENOMEM;
+			goto out;
+		}
+
+		uuid_generate (gfid);
+
+		ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+		if (ret) {
+			errno = ENOMEM;
+			goto out;
+		}
+
+		ret = syncop_lookup (subvol, &loc, xattr_req, &ciatt,
+				     NULL, NULL);
+	}
+	if (ret)
+		goto out;
+
+	inode = inode_link (loc.inode, loc.parent, component, &ciatt);
+found:
+	if (inode)
+		inode_lookup (inode);
+	if (iatt)
+		*iatt = ciatt;
+out:
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	loc_wipe (&loc);
+
+	return inode;
+}
+
+
+int
+glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
+		 const char *origpath, loc_t *loc, struct iatt *iatt,
+		 int follow, int reval)
+{
+	inode_t    *inode = NULL;
+	inode_t    *parent = NULL;
+	char       *saveptr = NULL;
+	char       *path = NULL;
+	char       *component = NULL;
+	char       *next_component = NULL;
+	int         ret = -1;
+	struct iatt ciatt = {0, };
+
+	path = gf_strdup (origpath);
+	if (!path) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	parent = NULL;
+	if (at && path[0] != '/') {
+		/* A relative resolution of a path which starts with '/'
+		   is equal to an absolute path resolution.
+		*/
+		inode = inode_ref (at);
+	} else {
+		inode = inode_ref (subvol->itable->root);
+
+		if (strcmp (path, "/") == 0)
+			glfs_resolve_base (fs, subvol, inode, &ciatt);
+	}
+
+	for (component = strtok_r (path, "/", &saveptr);
+	     component; component = next_component) {
+
+		next_component = strtok_r (NULL, "/", &saveptr);
+
+		if (parent)
+			inode_unref (parent);
+
+		parent = inode;
+
+		inode = glfs_resolve_component (fs, subvol, parent,
+						component, &ciatt,
+						/* force hard lookup on the last
+						   component, as the caller
+						   wants proper iatt filled
+						*/
+						(reval || (!next_component &&
+						iatt)));
+		if (!inode)
+			break;
+
+		if (IA_ISLNK (ciatt.ia_type) && (next_component || follow)) {
+			/* If the component is not the last piece,
+			   then following it is necessary even if
+			   not requested by the caller
+			*/
+			char *lpath = NULL;
+			loc_t sym_loc = {0,};
+
+			if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
+				errno = ELOOP;
+				ret = -1;
+				if (inode) {
+					inode_unref (inode);
+					inode = NULL;
+				}
+				break;
+			}
+
+			ret = glfs_resolve_symlink (fs, subvol, inode, &lpath);
+			inode_unref (inode);
+			inode = NULL;
+			if (ret < 0)
+				break;
+
+			ret = glfs_resolve_at (fs, subvol, parent, lpath,
+					       &sym_loc,
+					       /* followed iatt becomes the
+						  component iatt
+					       */
+					       &ciatt,
+					       /* always recurisvely follow while
+						  following symlink
+					       */
+					       follow + 1, reval);
+			if (ret == 0)
+				inode = inode_ref (sym_loc.inode);
+			loc_wipe (&sym_loc);
+			GF_FREE (lpath);
+		}
+
+		if (!next_component)
+			break;
+
+		if (!IA_ISDIR (ciatt.ia_type)) {
+			/* next_component exists and this component is
+			   not a directory
+			*/
+			inode_unref (inode);
+			inode = NULL;
+			ret = -1;
+			errno = ENOTDIR;
+			break;
+		}
+	}
+
+	if (parent && next_component)
+		/* resolution failed mid-way */
+		goto out;
+
+	/* At this point, all components up to the last parent directory
+	   have been resolved successfully (@parent). Resolution of basename
+	   might have failed (@inode) if at all.
+	*/
+
+	loc->parent = parent;
+	if (parent) {
+		uuid_copy (loc->pargfid, parent->gfid);
+		loc->name = component;
+	}
+
+	loc->inode = inode;
+	if (inode) {
+		uuid_copy (loc->gfid, inode->gfid);
+		if (iatt)
+			*iatt = ciatt;
+		ret = 0;
+	}
+
+	glfs_loc_touchup (loc);
+out:
+	GF_FREE (path);
+
+	/* do NOT loc_wipe here as only last component might be missing */
+
+	return ret;
+}
+
+
+int
+glfs_resolve_path (struct glfs *fs, xlator_t *subvol, const char *origpath,
+		   loc_t *loc, struct iatt *iatt, int follow, int reval)
+{
+	int ret = -1;
+	inode_t *cwd = NULL;
+
+	if (origpath[0] == '/')
+		return glfs_resolve_at (fs, subvol, NULL, origpath, loc, iatt,
+					follow, reval);
+
+	cwd = glfs_cwd_get (fs);
+
+	ret = glfs_resolve_at (fs, subvol, cwd, origpath, loc, iatt,
+			       follow, reval);
+	if (cwd)
+		inode_unref (cwd);
+
+	return ret;
+}
+
+
+int
+glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
+	      loc_t *loc, struct iatt *iatt, int reval)
+{
+	int ret = -1;
+
+	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 1, reval);
+
+	return ret;
+}
+
+
+int
+glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
+	       loc_t *loc, struct iatt *iatt, int reval)
+{
+	int ret = -1;
+
+	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 0, reval);
+
+	return ret;
+}
+
+
+int
+glfs_migrate_fd_locks_safe (struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
+			    xlator_t *newsubvol, fd_t *newfd)
+{
+	dict_t *lockinfo = NULL;
+	int ret = 0;
+	char uuid1[64];
+
+	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
+		return 0;
+
+	newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
+
+	ret = syncop_fgetxattr (oldsubvol, oldfd, &lockinfo,
+				GF_XATTR_LOCKINFO_KEY);
+	if (ret < 0) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"fgetxattr (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (oldfd->inode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (oldsubvol), oldsubvol->graph->id);
+		goto out;
+	}
+
+	if (!dict_get (lockinfo, GF_XATTR_LOCKINFO_KEY)) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"missing lokinfo key (%s) on graph %s (%d)",
+			uuid_utoa_r (oldfd->inode->gfid, uuid1),
+			graphid_str (oldsubvol), oldsubvol->graph->id);
+		goto out;
+	}
+
+	ret = syncop_fsetxattr (newsubvol, newfd, lockinfo, 0);
+	if (ret < 0) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"fsetxattr (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (newfd->inode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+out:
+	if (lockinfo)
+		dict_unref (lockinfo);
+	return ret;
+}
+
+
+fd_t *
+glfs_migrate_fd_safe (struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
+{
+	fd_t *newfd = NULL;
+	inode_t *oldinode = NULL;
+	inode_t *newinode = NULL;
+	xlator_t *oldsubvol = NULL;
+	int ret = -1;
+	loc_t loc = {0, };
+	char uuid1[64];
+
+
+	oldinode = oldfd->inode;
+	oldsubvol = oldinode->table->xl;
+
+	if (oldsubvol == newsubvol)
+		return fd_ref (oldfd);
+
+	if (!oldsubvol->switched) {
+		ret = syncop_fsync (oldsubvol, oldfd, 0);
+		if (ret) {
+			gf_log (fs->volname, GF_LOG_WARNING,
+				"fsync() failed (%s) on %s graph %s (%d)",
+				strerror (errno),
+				uuid_utoa_r (oldfd->inode->gfid, uuid1),
+				graphid_str (oldsubvol), oldsubvol->graph->id);
+		}
+	}
+
+	newinode = glfs_refresh_inode_safe (newsubvol, oldinode);
+	if (!newinode) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"inode (%s) refresh failed (%s) on graph %s (%d)",
+			uuid_utoa_r (oldinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+	newfd = fd_create (newinode, getpid());
+	if (!newfd) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"fd_create (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (newinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+	loc.inode = inode_ref (newinode);
+
+        ret = inode_path (oldfd->inode, NULL, (char **)&loc.path);
+        if (ret < 0) {
+                gf_log (fs->volname, GF_LOG_INFO, "inode_path failed");
+                goto out;
+        }
+
+        uuid_copy (loc.gfid, oldinode->gfid);
+
+
+	if (IA_ISDIR (oldinode->ia_type))
+		ret = syncop_opendir (newsubvol, &loc, newfd);
+	else
+		ret = syncop_open (newsubvol, &loc,
+				   oldfd->flags & ~(O_TRUNC|O_EXCL|O_CREAT),
+				   newfd);
+	loc_wipe (&loc);
+
+	if (ret) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"syncop_open%s (%s) failed (%s) on graph %s (%d)",
+			IA_ISDIR (oldinode->ia_type) ? "dir" : "",
+			uuid_utoa_r (newinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+	ret = glfs_migrate_fd_locks_safe (fs, oldsubvol, oldfd, newsubvol,
+					  newfd);
+
+	if (ret) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"lock migration (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (newinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+        newfd->flags = oldfd->flags;
+	fd_bind (newfd);
+out:
+	if (newinode)
+		inode_unref (newinode);
+
+	if (ret) {
+		fd_unref (newfd);
+		newfd = NULL;
+	}
+
+	return newfd;
+}
+
+
+fd_t *
+__glfs_migrate_fd (struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
+{
+	fd_t *oldfd = NULL;
+	fd_t *newfd = NULL;
+
+	oldfd = glfd->fd;
+
+	fs->migration_in_progress = 1;
+	pthread_mutex_unlock (&fs->mutex);
+	{
+		newfd = glfs_migrate_fd_safe (fs, newsubvol, oldfd);
+	}
+	pthread_mutex_lock (&fs->mutex);
+	fs->migration_in_progress = 0;
+	pthread_cond_broadcast (&fs->cond);
+
+	return newfd;
+}
+
+
+fd_t *
+__glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+{
+	fd_t *fd = NULL;
+
+	if (glfd->fd->inode->table->xl == subvol)
+		return fd_ref (glfd->fd);
+
+	fd = __glfs_migrate_fd (fs, subvol, glfd);
+	if (!fd)
+		return NULL;
+
+	if (subvol == fs->active_subvol) {
+		fd_unref (glfd->fd);
+		glfd->fd = fd_ref (fd);
+	}
+
+	return fd;
+}
+
+
+fd_t *
+glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+{
+	fd_t *fd = NULL;
+
+	glfs_lock (fs);
+	{
+		fd = __glfs_resolve_fd (fs, subvol, glfd);
+	}
+	glfs_unlock (fs);
+
+	return fd;
+}
+
+
+void
+__glfs_migrate_openfds (struct glfs *fs, xlator_t *subvol)
+{
+	struct glfs_fd *glfd = NULL;
+	fd_t *fd = NULL;
+
+	list_for_each_entry (glfd, &fs->openfds, openfds) {
+		if (uuid_is_null (glfd->fd->inode->gfid)) {
+			gf_log (fs->volname, GF_LOG_INFO,
+				"skipping openfd %p/%p in graph %s (%d)",
+				glfd, glfd->fd,	graphid_str(subvol),
+				subvol->graph->id);
+			/* create in progress, defer */
+			continue;
+		}
+
+		fd = __glfs_migrate_fd (fs, subvol, glfd);
+		if (fd) {
+			fd_unref (glfd->fd);
+			glfd->fd = fd;
+		}
+	}
+}
+
+
+xlator_t *
+__glfs_active_subvol (struct glfs *fs)
+{
+	xlator_t      *new_subvol = NULL;
+	int            ret = -1;
+	inode_t       *new_cwd = NULL;
+
+	if (!fs->next_subvol)
+		return fs->active_subvol;
+
+	new_subvol = fs->next_subvol;
+
+	ret = __glfs_first_lookup (fs, new_subvol);
+	if (ret) {
+		gf_log (fs->volname, GF_LOG_INFO,
+			"first lookup on graph %s (%d) failed (%s)",
+			graphid_str (new_subvol), new_subvol->graph->id,
+			strerror (errno));
+		return NULL;
+	}
+
+	if (fs->cwd) {
+		new_cwd = __glfs_refresh_inode (fs, new_subvol, fs->cwd);
+
+		if (!new_cwd) {
+			char buf1[64];
+			gf_log (fs->volname, GF_LOG_INFO,
+				"cwd refresh of %s graph %s (%d) failed (%s)",
+				uuid_utoa_r (fs->cwd->gfid, buf1),
+				graphid_str (new_subvol),
+				new_subvol->graph->id, strerror (errno));
+			return NULL;
+		}
+	}
+
+	__glfs_migrate_openfds (fs, new_subvol);
+
+	/* switching @active_subvol and @cwd
+	   should be atomic
+	*/
+	fs->old_subvol = fs->active_subvol;
+	fs->active_subvol = fs->next_subvol;
+	fs->next_subvol = NULL;
+
+	if (new_cwd) {
+		__glfs_cwd_set (fs, new_cwd);
+		inode_unref (new_cwd);
+	}
+
+	gf_log (fs->volname, GF_LOG_INFO, "switched to graph %s (%d)",
+		graphid_str (new_subvol), new_subvol->graph->id);
+
+	return new_subvol;
+}
+
+xlator_t *
+glfs_active_subvol (struct glfs *fs)
+{
+	xlator_t      *subvol = NULL;
+	xlator_t      *old_subvol = NULL;
+
+	glfs_lock (fs);
+	{
+		subvol = __glfs_active_subvol (fs);
+
+		if (subvol)
+			subvol->winds++;
+
+		if (fs->old_subvol) {
+			old_subvol = fs->old_subvol;
+			fs->old_subvol = NULL;
+			old_subvol->switched = 1;
+		}
+	}
+	glfs_unlock (fs);
+
+	if (old_subvol)
+		glfs_subvol_done (fs, old_subvol);
+
+	return subvol;
+}
+
+
+void
+glfs_subvol_done (struct glfs *fs, xlator_t *subvol)
+{
+	int ref = 0;
+	xlator_t *active_subvol = NULL;
+
+	if (!subvol)
+		return;
+
+	glfs_lock (fs);
+	{
+		ref = (--subvol->winds);
+		active_subvol = fs->active_subvol;
+	}
+	glfs_unlock (fs);
+
+	if (ref == 0) {
+		assert (subvol != active_subvol);
+		xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
+	}
+}
+
+
+int
+__glfs_cwd_set (struct glfs *fs, inode_t *inode)
+{
+	if (inode->table->xl != fs->active_subvol) {
+		inode = __glfs_refresh_inode (fs, fs->active_subvol, inode);
+		if (!inode)
+			return -1;
+	} else {
+		inode_ref (inode);
+	}
+
+	if (fs->cwd)
+		inode_unref (fs->cwd);
+
+	fs->cwd = inode;
+
+	return 0;
+}
+
+
+int
+glfs_cwd_set (struct glfs *fs, inode_t *inode)
+{
+	int ret = 0;
+
+	glfs_lock (fs);
+	{
+		ret = __glfs_cwd_set (fs, inode);
+	}
+	glfs_unlock (fs);
+
+	return ret;
+}
+
+
+inode_t *
+__glfs_cwd_get (struct glfs *fs)
+{
+	inode_t *cwd = NULL;
+
+	if (!fs->cwd)
+		return NULL;
+
+	if (fs->cwd->table->xl == fs->active_subvol) {
+		cwd = inode_ref (fs->cwd);
+		return cwd;
+	}
+
+	cwd = __glfs_refresh_inode (fs, fs->active_subvol, fs->cwd);
+
+	return cwd;
+}
+
+inode_t *
+glfs_cwd_get (struct glfs *fs)
+{
+	inode_t *cwd = NULL;
+
+	glfs_lock (fs);
+	{
+		cwd = __glfs_cwd_get (fs);
+	}
+	glfs_unlock (fs);
+
+	return cwd;
+}
+
+inode_t *
+__glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
+		    struct glfs_object *object)
+{
+	inode_t *inode = NULL;
+
+	if (object->inode->table->xl == subvol)
+		return inode_ref (object->inode);
+
+	inode = __glfs_refresh_inode (fs, fs->active_subvol,
+					object->inode);
+	if (!inode)
+		return NULL;
+
+	if (subvol == fs->active_subvol) {
+		inode_unref (object->inode);
+		object->inode = inode_ref (inode);
+	}
+
+	return inode;
+}
+
+inode_t *
+glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
+		    struct glfs_object *object)
+{
+	inode_t *inode = NULL;
+
+	glfs_lock (fs);
+	{
+		inode = __glfs_resolve_inode(fs, subvol, object);
+	}
+	glfs_unlock (fs);
+
+	return inode;
+}
+
+int
+glfs_create_object (loc_t *loc, struct glfs_object **retobject)
+{
+	struct glfs_object *object = NULL;
+
+	object = GF_CALLOC (1, sizeof(struct glfs_object),
+			    glfs_mt_glfs_object_t);
+	if (object == NULL) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	object->inode = loc->inode;
+	uuid_copy (object->gfid, object->inode->gfid);
+
+	/* we hold the reference */
+	loc->inode = NULL;
+
+	*retobject = object;
+
+	return 0;
+}
diff --git a/api/src/glfs.c b/api/src/glfs.c
new file mode 100644
index 000000000..29ed47c0c
--- /dev/null
+++ b/api/src/glfs.c
@@ -0,0 +1,673 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+/*
+  TODO:
+  - merge locks in glfs_posix_lock for lock self-healing
+  - set proper pid/lk_owner to call frames (currently buried in syncop)
+  - fix logging.c/h to store logfp and loglevel in glusterfs_ctx_t and
+    reach it via THIS.
+  - update syncop functions to accept/return xdata. ???
+  - protocol/client to reconnect immediately after portmap disconnect.
+  - handle SEEK_END failure in _lseek()
+  - handle umask (per filesystem?)
+  - make itables LRU based
+  - 0-copy for readv/writev
+  - reconcile the open/creat mess
+*/
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <limits.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "stack.h"
+#include "event.h"
+#include "glfs-mem-types.h"
+#include "common-utils.h"
+#include "syncop.h"
+#include "call-stub.h"
+
+#include "glfs.h"
+#include "glfs-internal.h"
+#include "hashfn.h"
+#include "rpc-clnt.h"
+
+
+static gf_boolean_t
+vol_assigned (cmd_args_t *args)
+{
+	return args->volfile || args->volfile_server;
+}
+
+
+static int
+glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
+{
+	call_pool_t   *pool = NULL;
+	int	       ret = -1;
+
+	xlator_mem_acct_init (THIS, glfs_mt_end + 1);
+
+	ctx->process_uuid = generate_glusterfs_ctx_id ();
+	if (!ctx->process_uuid) {
+		goto err;
+	}
+
+	ctx->page_size	= 128 * GF_UNIT_KB;
+
+	ctx->iobuf_pool = iobuf_pool_new ();
+	if (!ctx->iobuf_pool) {
+		goto err;
+	}
+
+	ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE);
+	if (!ctx->event_pool) {
+		goto err;
+	}
+
+	ctx->env = syncenv_new (0, 0, 0);
+	if (!ctx->env) {
+		goto err;
+	}
+
+	pool = GF_CALLOC (1, sizeof (call_pool_t),
+			  glfs_mt_call_pool_t);
+	if (!pool) {
+		goto err;
+	}
+
+	/* frame_mem_pool size 112 * 4k */
+	pool->frame_mem_pool = mem_pool_new (call_frame_t, 4096);
+	if (!pool->frame_mem_pool) {
+		goto err;
+	}
+	/* stack_mem_pool size 256 * 1024 */
+	pool->stack_mem_pool = mem_pool_new (call_stack_t, 1024);
+	if (!pool->stack_mem_pool) {
+		goto err;
+	}
+
+	ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
+	if (!ctx->stub_mem_pool) {
+		goto err;
+	}
+
+	ctx->dict_pool = mem_pool_new (dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+	if (!ctx->dict_pool)
+		goto err;
+
+	ctx->dict_pair_pool = mem_pool_new (data_pair_t,
+					    GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+	if (!ctx->dict_pair_pool)
+		goto err;
+
+	ctx->dict_data_pool = mem_pool_new (data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+	if (!ctx->dict_data_pool)
+		goto err;
+
+	INIT_LIST_HEAD (&pool->all_frames);
+	INIT_LIST_HEAD (&ctx->cmd_args.xlator_options);
+	LOCK_INIT (&pool->lock);
+	ctx->pool = pool;
+
+	pthread_mutex_init (&(ctx->lock), NULL);
+
+	ret = 0;
+err:
+	if (ret && pool) {
+		if (pool->frame_mem_pool)
+			mem_pool_destroy (pool->frame_mem_pool);
+		if (pool->stack_mem_pool)
+			mem_pool_destroy (pool->stack_mem_pool);
+		GF_FREE (pool);
+	}
+
+	if (ret && ctx) {
+		if (ctx->stub_mem_pool)
+			mem_pool_destroy (ctx->stub_mem_pool);
+		if (ctx->dict_pool)
+			mem_pool_destroy (ctx->dict_pool);
+		if (ctx->dict_data_pool)
+			mem_pool_destroy (ctx->dict_data_pool);
+		if (ctx->dict_pair_pool)
+			mem_pool_destroy (ctx->dict_pair_pool);
+	}
+
+	return ret;
+}
+
+
+static int
+create_master (struct glfs *fs)
+{
+	int		 ret = 0;
+	xlator_t	*master = NULL;
+
+	master = GF_CALLOC (1, sizeof (*master),
+			    glfs_mt_xlator_t);
+	if (!master)
+		goto err;
+
+	master->name = gf_strdup ("gfapi");
+	if (!master->name)
+		goto err;
+
+	if (xlator_set_type (master, "mount/api") == -1) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"master xlator for %s initialization failed",
+			fs->volname);
+		goto err;
+	}
+
+	master->ctx	 = fs->ctx;
+	master->private	 = fs;
+	master->options	 = get_new_dict ();
+	if (!master->options)
+		goto err;
+
+
+	ret = xlator_init (master);
+	if (ret) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"failed to initialize gfapi translator");
+		goto err;
+	}
+
+	fs->ctx->master = master;
+	THIS = master;
+
+	return 0;
+
+err:
+	if (master) {
+		xlator_destroy (master);
+	}
+
+	return -1;
+}
+
+
+static FILE *
+get_volfp (struct glfs *fs)
+{
+	int	     ret = 0;
+	cmd_args_t  *cmd_args = NULL;
+	FILE	    *specfp = NULL;
+	struct stat  statbuf;
+
+	cmd_args = &fs->ctx->cmd_args;
+
+	ret = lstat (cmd_args->volfile, &statbuf);
+	if (ret == -1) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"%s: %s", cmd_args->volfile, strerror (errno));
+		return NULL;
+	}
+
+	if ((specfp = fopen (cmd_args->volfile, "r")) == NULL) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"volume file %s: %s",
+			cmd_args->volfile,
+			strerror (errno));
+		return NULL;
+	}
+
+	gf_log ("glfs", GF_LOG_DEBUG,
+		"loading volume file %s", cmd_args->volfile);
+
+	return specfp;
+}
+
+
+int
+glfs_volumes_init (struct glfs *fs)
+{
+	FILE		   *fp = NULL;
+	cmd_args_t	   *cmd_args = NULL;
+	int		    ret = 0;
+
+	cmd_args = &fs->ctx->cmd_args;
+
+	if (!vol_assigned (cmd_args))
+		return -1;
+
+	if (cmd_args->volfile_server) {
+		ret = glfs_mgmt_init (fs);
+		goto out;
+	}
+
+	fp = get_volfp (fs);
+
+	if (!fp) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"Cannot reach volume specification file");
+		ret = -1;
+		goto out;
+	}
+
+	ret = glfs_process_volfp (fs, fp);
+	if (ret)
+		goto out;
+
+out:
+	return ret;
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+
+
+int
+glfs_set_xlator_option (struct glfs *fs, const char *xlator, const char *key,
+			const char *value)
+{
+	xlator_cmdline_option_t *option = NULL;
+
+	option = GF_CALLOC (1, sizeof (*option),
+			    glfs_mt_xlator_cmdline_option_t);
+	if (!option)
+		goto enomem;
+
+	INIT_LIST_HEAD (&option->cmd_args);
+
+	option->volume = gf_strdup (xlator);
+	if (!option->volume)
+		goto enomem;
+	option->key = gf_strdup (key);
+	if (!option->key)
+		goto enomem;
+	option->value = gf_strdup (value);
+	if (!option->value)
+		goto enomem;
+
+	list_add (&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
+
+	return 0;
+enomem:
+	errno = ENOMEM;
+
+	if (!option)
+		return -1;
+
+	GF_FREE (option->volume);
+	GF_FREE (option->key);
+	GF_FREE (option->value);
+	GF_FREE (option);
+
+	return -1;
+}
+
+int glfs_setfsuid (uid_t fsuid)
+{
+	return syncopctx_setfsuid (&fsuid);
+}
+
+int glfs_setfsgid (gid_t fsgid)
+{
+	return syncopctx_setfsgid (&fsgid);
+}
+
+int glfs_setfsgroups (size_t size, const gid_t *list)
+{
+	return syncopctx_setfsgroups(size, list);
+}
+
+struct glfs *
+glfs_from_glfd (struct glfs_fd *glfd)
+{
+	return glfd->fs;
+}
+
+
+struct glfs_fd *
+glfs_fd_new (struct glfs *fs)
+{
+	struct glfs_fd  *glfd = NULL;
+
+	glfd = GF_CALLOC (1, sizeof (*glfd), glfs_mt_glfs_fd_t);
+	if (!glfd)
+		return NULL;
+
+	glfd->fs = fs;
+
+	INIT_LIST_HEAD (&glfd->openfds);
+
+	return glfd;
+}
+
+
+void
+glfs_fd_bind (struct glfs_fd *glfd)
+{
+	struct glfs *fs = NULL;
+
+	fs = glfd->fs;
+
+	glfs_lock (fs);
+	{
+		list_add_tail (&glfd->openfds, &fs->openfds);
+	}
+	glfs_unlock (fs);
+}
+
+void
+glfs_fd_destroy (struct glfs_fd *glfd)
+{
+	if (!glfd)
+		return;
+
+	glfs_lock (glfd->fs);
+	{
+		list_del_init (&glfd->openfds);
+	}
+	glfs_unlock (glfd->fs);
+
+	if (glfd->fd)
+		fd_unref (glfd->fd);
+
+	GF_FREE (glfd->readdirbuf);
+
+	GF_FREE (glfd);
+}
+
+
+static void *
+glfs_poller (void *data)
+{
+	struct glfs  *fs = NULL;
+
+	fs = data;
+
+	event_dispatch (fs->ctx->event_pool);
+
+	return NULL;
+}
+
+
+struct glfs *
+glfs_new (const char *volname)
+{
+	struct glfs     *fs = NULL;
+	int              ret = -1;
+	glusterfs_ctx_t *ctx = NULL;
+
+	ctx = glusterfs_ctx_new ();
+	if (!ctx) {
+		return NULL;
+	}
+
+#ifdef DEBUG
+        gf_mem_acct_enable_set (ctx);
+#endif
+
+	/* first globals init, for gf_mem_acct_enable_set () */
+	ret = glusterfs_globals_init (ctx);
+	if (ret)
+		return NULL;
+
+	THIS->ctx = ctx;
+
+	/* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
+	ret = glusterfs_ctx_defaults_init (ctx);
+	if (ret)
+		return NULL;
+
+	fs = GF_CALLOC (1, sizeof (*fs), glfs_mt_glfs_t);
+	if (!fs)
+		return NULL;
+	fs->ctx = ctx;
+
+	glfs_set_logging (fs, "/dev/null", 0);
+
+	fs->ctx->cmd_args.volfile_id = gf_strdup (volname);
+
+	fs->volname = gf_strdup (volname);
+
+	pthread_mutex_init (&fs->mutex, NULL);
+	pthread_cond_init (&fs->cond, NULL);
+
+	INIT_LIST_HEAD (&fs->openfds);
+
+	return fs;
+}
+
+
+int
+glfs_set_volfile (struct glfs *fs, const char *volfile)
+{
+	cmd_args_t  *cmd_args = NULL;
+
+	cmd_args = &fs->ctx->cmd_args;
+
+	if (vol_assigned (cmd_args))
+		return -1;
+
+	cmd_args->volfile = gf_strdup (volfile);
+
+	return 0;
+}
+
+
+int
+glfs_set_volfile_server (struct glfs *fs, const char *transport,
+			 const char *host, int port)
+{
+	cmd_args_t  *cmd_args = NULL;
+
+	cmd_args = &fs->ctx->cmd_args;
+
+	if (vol_assigned (cmd_args))
+		return -1;
+
+	cmd_args->volfile_server = gf_strdup (host);
+	cmd_args->volfile_server_transport = gf_strdup (transport);
+	cmd_args->volfile_server_port = port;
+	cmd_args->max_connect_attempts = 2;
+
+	return 0;
+}
+
+
+int
+glfs_set_logging (struct glfs *fs, const char *logfile, int loglevel)
+{
+	int  ret = 0;
+        char *tmplog = NULL;
+
+        if (!logfile) {
+                ret = gf_set_log_file_path (&fs->ctx->cmd_args);
+                if (ret)
+                        goto out;
+                tmplog = fs->ctx->cmd_args.log_file;
+        } else {
+                tmplog = (char *)logfile;
+        }
+
+        ret = gf_log_init (fs->ctx, tmplog, NULL);
+        if (ret)
+                goto out;
+
+	if (loglevel >= 0)
+		gf_log_set_loglevel (loglevel);
+
+out:
+	return ret;
+}
+
+
+int
+glfs_init_wait (struct glfs *fs)
+{
+	int   ret = -1;
+
+	/* Always a top-down call, use glfs_lock() */
+	glfs_lock (fs);
+	{
+		while (!fs->init)
+			pthread_cond_wait (&fs->cond,
+					   &fs->mutex);
+		ret = fs->ret;
+		errno = fs->err;
+	}
+	glfs_unlock (fs);
+
+	return ret;
+}
+
+
+void
+glfs_init_done (struct glfs *fs, int ret)
+{
+	glfs_init_cbk init_cbk;
+
+	if (!fs) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"fs is NULL");
+		goto out;
+	}
+
+	init_cbk = fs->init_cbk;
+
+	/* Always a bottom-up call, use mutex_lock() */
+	pthread_mutex_lock (&fs->mutex);
+	{
+		fs->init = 1;
+		fs->ret = ret;
+		fs->err = errno;
+
+		if (!init_cbk)
+			pthread_cond_broadcast (&fs->cond);
+	}
+	pthread_mutex_unlock (&fs->mutex);
+
+	if (init_cbk)
+		init_cbk (fs, ret);
+out:
+	return;
+}
+
+
+int
+glfs_init_common (struct glfs *fs)
+{
+	int  ret = -1;
+
+	ret = create_master (fs);
+	if (ret)
+		return ret;
+
+	ret = gf_thread_create (&fs->poller, NULL, glfs_poller, fs);
+	if (ret)
+		return ret;
+
+	ret = glfs_volumes_init (fs);
+	if (ret)
+		return ret;
+
+	fs->dev_id = gf_dm_hashfn (fs->volname, strlen (fs->volname));
+	return ret;
+}
+
+
+int
+glfs_init_async (struct glfs *fs, glfs_init_cbk cbk)
+{
+	int  ret = -1;
+
+	fs->init_cbk = cbk;
+
+	ret = glfs_init_common (fs);
+
+	return ret;
+}
+
+
+int
+glfs_init (struct glfs *fs)
+{
+	int  ret = -1;
+
+	ret = glfs_init_common (fs);
+	if (ret)
+		return ret;
+
+	ret = glfs_init_wait (fs);
+
+	return ret;
+}
+
+
+int
+glfs_fini (struct glfs *fs)
+{
+        int             ret = -1;
+        int             countdown = 100;
+        xlator_t        *subvol = NULL;
+        glusterfs_ctx_t *ctx = NULL;
+        call_pool_t     *call_pool = NULL;
+
+        ctx = fs->ctx;
+
+        if (ctx->mgmt) {
+                rpc_clnt_disable (ctx->mgmt);
+                ctx->mgmt = NULL;
+        }
+
+        __glfs_entry_fs (fs);
+
+        call_pool = fs->ctx->pool;
+
+        while (countdown--) {
+                /* give some time for background frames to finish */
+                if (!call_pool->cnt)
+                        break;
+                usleep (100000);
+        }
+        /* leaked frames may exist, we ignore */
+
+        /*We deem glfs_fini as successful if there are no pending frames in the call
+         *pool*/
+        ret = (call_pool->cnt == 0)? 0: -1;
+
+        subvol = glfs_active_subvol (fs);
+        if (subvol) {
+                /* PARENT_DOWN within glfs_subvol_done() is issued only
+                   on graph switch (new graph should activiate and
+                   decrement the extra @winds count taken in glfs_graph_setup()
+
+                   Since we are explicitly destroying, PARENT_DOWN is necessary
+                */
+                xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
+                /* TBD: wait for CHILD_DOWN before exiting, in case of
+                   asynchronous cleanup like graceful socket disconnection
+                   in the future.
+                */
+        }
+
+        glfs_subvol_done (fs, subvol);
+
+        if (ctx->log.logfile)
+                fclose (ctx->log.logfile);
+
+        return ret;
+}
diff --git a/api/src/glfs.h b/api/src/glfs.h
new file mode 100644
index 000000000..18fda496e
--- /dev/null
+++ b/api/src/glfs.h
@@ -0,0 +1,581 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _GLFS_H
+#define _GLFS_H
+
+/*
+  Enforce the following flags as libgfapi is built
+  with them, and we want programs linking against them to also
+  be built with these flags. This is necessary as it affects
+  some of the structures defined in libc headers (like struct stat)
+  and those definitions need to be consistently compiled in
+  both the library and the application.
+*/
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifndef __USE_FILE_OFFSET64
+#define __USE_FILE_OFFSET64
+#endif
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/uio.h>
+#include <unistd.h>
+#include <sys/cdefs.h>
+#include <dirent.h>
+#include <sys/statvfs.h>
+
+__BEGIN_DECLS
+
+/* The filesystem object. One object per 'virtual mount' */
+struct glfs;
+typedef struct glfs glfs_t;
+
+
+/*
+  SYNOPSIS
+
+  glfs_new: Create a new 'virtual mount' object.
+
+  DESCRIPTION
+
+  This is most likely the very first function you will use. This function
+  will create a new glfs_t (virtual mount) object in memory.
+
+  On this newly created glfs_t, you need to be either set a volfile path
+  (glfs_set_volfile) or a volfile server (glfs_set_volfile_server).
+
+  The glfs_t object needs to be initialized with glfs_init() before you
+  can start issuing file operations on it.
+
+  PARAMETERS
+
+  @volname: Name of the volume. This identifies the server-side volume and
+            the fetched volfile (equivalent of --volfile-id command line
+	    parameter to glusterfsd). When used with glfs_set_volfile() the
+	    @volname has no effect (except for appearing in log messages).
+
+  RETURN VALUES
+
+  NULL   : Out of memory condition.
+  Others : Pointer to the newly created glfs_t virtual mount object.
+
+*/
+
+glfs_t *glfs_new (const char *volname);
+
+
+/*
+  SYNOPSIS
+
+  glfs_set_volfile: Specify the path to the volume specification file.
+
+  DESCRIPTION
+
+  If you are using a static volume specification file (without dynamic
+  volume management abilities from the CLI), then specify the path to
+  the volume specification file.
+
+  This is incompatible with glfs_set_volfile_server().
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @volfile: Path to the locally available volume specification file.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int glfs_set_volfile (glfs_t *fs, const char *volfile);
+
+
+/*
+  SYNOPSIS
+
+  glfs_set_volfile_server: Specify the address of management server.
+
+  DESCRIPTION
+
+  This function specifies the address of the management server (glusterd)
+  to connect, and establish the volume configuration. The @volname
+  parameter passed to glfs_new() is the volume which will be virtually
+  mounted as the glfs_t object. All operations performed by the CLI at
+  the management server will automatically be reflected in the 'virtual
+  mount' object as it maintains a connection to glusterd and polls on
+  configuration change notifications.
+
+  This is incompatible with glfs_set_volfile().
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @transport: String specifying the transport used to connect to the
+              management daemon. Specifying NULL will result in the usage
+	      of the default (tcp) transport type. Permitted values
+	      are those what you specify as transport-type in a volume
+	      specification file (e.g "tcp", "rdma", "unix".)
+
+  @host: String specifying the address of where to find the management
+         daemon. Depending on the transport type this would either be
+	 an FQDN (e.g: "storage01.company.com"), ASCII encoded IP
+	 address "192.168.22.1", or a UNIX domain socket path (e.g
+	 "/tmp/glusterd.socket".)
+
+  @port: The TCP port number where gluster management daemon is listening.
+         Specifying 0 uses the default port number GF_DEFAULT_BASE_PORT.
+	 This parameter is unused if you are using a UNIX domain socket.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int glfs_set_volfile_server (glfs_t *fs, const char *transport,
+			     const char *host, int port);
+
+
+/*
+  SYNOPSIS
+
+  glfs_set_logging: Specify logging parameters.
+
+  DESCRIPTION
+
+  This function specifies logging parameters for the virtual mount.
+  Default log file is /dev/null.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the logging parameters.
+
+  @logfile: The logfile to be used for logging. Will be created if it does not
+            already exist (provided system permissions allow). If NULL, a new
+            logfile will be created in default log directory associated with
+            the glusterfs installation.
+
+  @loglevel: Numerical value specifying the degree of verbosity. Higher the
+             value, more verbose the logging.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel);
+
+
+/*
+  SYNOPSIS
+
+  glfs_init: Initialize the 'virtual mount'
+
+  DESCRIPTION
+
+  This function initializes the glfs_t object. This consists of many steps:
+  - Spawn a poll-loop thread.
+  - Establish connection to management daemon and receive volume specification.
+  - Construct translator graph and initialize graph.
+  - Wait for initialization (connecting to all bricks) to complete.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be initialized.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int glfs_init (glfs_t *fs);
+
+
+/*
+  SYNOPSIS
+
+  glfs_fini: Cleanup and destroy the 'virtual mount'
+
+  DESCRIPTION
+
+  This function attempts to gracefully destroy glfs_t object. An attempt is
+  made to wait for all background processing to complete before returning.
+
+  glfs_fini() must be called after all operations on glfs_t is finished.
+
+  IMPORTANT
+
+  IT IS NECESSARY TO CALL glfs_fini() ON ALL THE INITIALIZED glfs_t
+  OBJECTS BEFORE TERMINATING THE PROGRAM. THERE MAY BE CACHED AND
+  UNWRITTEN / INCOMPLETE OPERATIONS STILL IN PROGRESS EVEN THOUGH THE
+  API CALLS HAVE RETURNED. glfs_fini() WILL WAIT FOR BACKGROUND OPERATIONS
+  TO COMPLETE BEFORE RETURNING, THEREBY MAKING IT SAFE FOR THE PROGRAM TO
+  EXIT.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be destroyed.
+
+  RETURN VALUES
+
+   0 : Success.
+*/
+
+int glfs_fini (glfs_t *fs);
+
+/*
+ * FILE OPERATION
+ *
+ * What follows are filesystem operations performed on the
+ * 'virtual mount'. The calls here are kept as close to
+ * the POSIX system calls as possible.
+ *
+ * Notes:
+ *
+ * - All paths specified, even if absolute, are relative to the
+ *   root of the virtual mount and not the system root (/).
+ *
+ */
+
+/* The file descriptor object. One per open file/directory. */
+
+struct glfs_fd;
+typedef struct glfs_fd glfs_fd_t;
+
+/*
+ * PER THREAD IDENTITY MODIFIERS
+ *
+ * The following operations enable to set a per thread identity context
+ * for the glfs APIs to perform operations as. The calls here are kept as close
+ * to POSIX equivalents as possible.
+ *
+ * NOTES:
+ *
+ *  - setgroups is a per thread setting, hence this is named as fsgroups to be
+ *    close in naming to the fs(u/g)id APIs
+ *  - Typical mode of operation is to set the IDs as required, with the
+ *    supplementary groups being optionally set, make the glfs call and post the
+ *    glfs operation set them back to eu/gid or uid/gid as appropriate to the
+ *    caller
+ *  - The groups once set, need to be unset by setting the size to 0 (in which
+ *    case the list argument is a do not care)
+ *  - Once a process for a thread of operation choses to set the IDs, all glfs
+ *    calls made from that thread would default to the IDs set for the thread.
+ *    As a result use these APIs with care and ensure that the set IDs are
+ *    reverted to global process defaults as required.
+ *
+ */
+int glfs_setfsuid (uid_t fsuid);
+int glfs_setfsgid (gid_t fsgid);
+int glfs_setfsgroups (size_t size, const gid_t *list);
+
+/*
+  SYNOPSIS
+
+  glfs_open: Open a file.
+
+  DESCRIPTION
+
+  This function opens a file on a virtual mount.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be initialized.
+
+  @path: Path of the file within the virtual mount.
+
+  @flags: Open flags. See open(2). O_CREAT is not supported.
+          Use glfs_creat() for creating files.
+
+  RETURN VALUES
+
+  NULL   : Failure. @errno will be set with the type of failure.
+  Others : Pointer to the opened glfs_fd_t.
+
+ */
+
+glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags);
+
+
+/*
+  SYNOPSIS
+
+  glfs_creat: Create a file.
+
+  DESCRIPTION
+
+  This function opens a file on a virtual mount.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be initialized.
+
+  @path: Path of the file within the virtual mount.
+
+  @mode: Permission of the file to be created.
+
+  @flags: Create flags. See open(2). O_EXCL is supported.
+
+  RETURN VALUES
+
+  NULL   : Failure. @errno will be set with the type of failure.
+  Others : Pointer to the opened glfs_fd_t.
+
+ */
+
+glfs_fd_t *glfs_creat (glfs_t *fs, const char *path, int flags,
+		       mode_t mode);
+
+int glfs_close (glfs_fd_t *fd);
+
+glfs_t *glfs_from_glfd (glfs_fd_t *fd);
+
+int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
+			    const char *value);
+
+/*
+
+  glfs_io_cbk
+
+  The following is the function type definition of the callback
+  function pointer which has to be provided by the caller to the
+  *_async() versions of the IO calls.
+
+  The callback function is called on completion of the requested
+  IO, and the appropriate return value is returned in @ret.
+
+  In case of an error in completing the IO, @ret will be -1 and
+  @errno will be set with the appropriate error.
+
+  @ret will be same as the return value of the non _async() variant
+  of the particular call
+
+  @data is the same context pointer provided by the caller at the
+  time of issuing the async IO call. This can be used by the
+  caller to differentiate different instances of the async requests
+  in a common callback function.
+*/
+
+typedef void (*glfs_io_cbk) (glfs_fd_t *fd, ssize_t ret, void *data);
+
+// glfs_{read,write}[_async]
+
+ssize_t glfs_read (glfs_fd_t *fd, void *buf, size_t count, int flags);
+ssize_t glfs_write (glfs_fd_t *fd, const void *buf, size_t count, int flags);
+int glfs_read_async (glfs_fd_t *fd, void *buf, size_t count, int flags,
+		     glfs_io_cbk fn, void *data);
+int glfs_write_async (glfs_fd_t *fd, const void *buf, size_t count, int flags,
+		      glfs_io_cbk fn, void *data);
+
+// glfs_{read,write}v[_async]
+
+ssize_t glfs_readv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+		    int flags);
+ssize_t glfs_writev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+		     int flags);
+int glfs_readv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
+		      int flags, glfs_io_cbk fn, void *data);
+int glfs_writev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
+		       int flags, glfs_io_cbk fn, void *data);
+
+// glfs_p{read,write}[_async]
+
+ssize_t glfs_pread (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
+		    int flags);
+ssize_t glfs_pwrite (glfs_fd_t *fd, const void *buf, size_t count,
+		     off_t offset, int flags);
+int glfs_pread_async (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
+		      int flags, glfs_io_cbk fn, void *data);
+int glfs_pwrite_async (glfs_fd_t *fd, const void *buf, int count, off_t offset,
+		       int flags, glfs_io_cbk fn, void *data);
+
+// glfs_p{read,write}v[_async]
+
+ssize_t glfs_preadv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+		     off_t offset, int flags);
+ssize_t glfs_pwritev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+		      off_t offset, int flags);
+int glfs_preadv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
+		       off_t offset, int flags, glfs_io_cbk fn, void *data);
+int glfs_pwritev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
+			off_t offset, int flags, glfs_io_cbk fn, void *data);
+
+
+off_t glfs_lseek (glfs_fd_t *fd, off_t offset, int whence);
+
+int glfs_truncate (glfs_t *fs, const char *path, off_t length);
+
+int glfs_ftruncate (glfs_fd_t *fd, off_t length);
+int glfs_ftruncate_async (glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
+			  void *data);
+
+int glfs_lstat (glfs_t *fs, const char *path, struct stat *buf);
+int glfs_stat (glfs_t *fs, const char *path, struct stat *buf);
+int glfs_fstat (glfs_fd_t *fd, struct stat *buf);
+
+int glfs_fsync (glfs_fd_t *fd);
+int glfs_fsync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data);
+
+int glfs_fdatasync (glfs_fd_t *fd);
+int glfs_fdatasync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data);
+
+int glfs_access (glfs_t *fs, const char *path, int mode);
+
+int glfs_symlink (glfs_t *fs, const char *oldpath, const char *newpath);
+
+int glfs_readlink (glfs_t *fs, const char *path, char *buf, size_t bufsiz);
+
+int glfs_mknod (glfs_t *fs, const char *path, mode_t mode, dev_t dev);
+
+int glfs_mkdir (glfs_t *fs, const char *path, mode_t mode);
+
+int glfs_unlink (glfs_t *fs, const char *path);
+
+int glfs_rmdir (glfs_t *fs, const char *path);
+
+int glfs_rename (glfs_t *fs, const char *oldpath, const char *newpath);
+
+int glfs_link (glfs_t *fs, const char *oldpath, const char *newpath);
+
+glfs_fd_t *glfs_opendir (glfs_t *fs, const char *path);
+
+/*
+ * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
+ * but the interface has ambiguity about the size of @dirent to be allocated
+ * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
+ * all known systems which are tested againt glusterfs/gfapi, but may be
+ * insufficient in the future.
+ */
+
+int glfs_readdir_r (glfs_fd_t *fd, struct dirent *dirent,
+		    struct dirent **result);
+
+int glfs_readdirplus_r (glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
+			struct dirent **result);
+
+/*
+ * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
+ * when called on the same directory handle. However they ARE thread safe
+ * AND re-entrant when called on different directory handles (which may be
+ * referring to the same directory too.)
+ */
+
+struct dirent *glfs_readdir (glfs_fd_t *fd);
+
+struct dirent *glfs_readdirplus (glfs_fd_t *fd, struct stat *stat);
+
+long glfs_telldir (glfs_fd_t *fd);
+
+void glfs_seekdir (glfs_fd_t *fd, long offset);
+
+int glfs_closedir (glfs_fd_t *fd);
+
+int glfs_statvfs (glfs_t *fs, const char *path, struct statvfs *buf);
+
+int glfs_chmod (glfs_t *fs, const char *path, mode_t mode);
+
+int glfs_fchmod (glfs_fd_t *fd, mode_t mode);
+
+int glfs_chown (glfs_t *fs, const char *path, uid_t uid, gid_t gid);
+
+int glfs_lchown (glfs_t *fs, const char *path, uid_t uid, gid_t gid);
+
+int glfs_fchown (glfs_fd_t *fd, uid_t uid, gid_t gid);
+
+int glfs_utimens (glfs_t *fs, const char *path, struct timespec times[2]);
+
+int glfs_lutimens (glfs_t *fs, const char *path, struct timespec times[2]);
+
+int glfs_futimens (glfs_fd_t *fd, struct timespec times[2]);
+
+ssize_t glfs_getxattr (glfs_t *fs, const char *path, const char *name,
+		       void *value, size_t size);
+
+ssize_t glfs_lgetxattr (glfs_t *fs, const char *path, const char *name,
+			void *value, size_t size);
+
+ssize_t glfs_fgetxattr (glfs_fd_t *fd, const char *name,
+			void *value, size_t size);
+
+ssize_t glfs_listxattr (glfs_t *fs, const char *path, void *value, size_t size);
+
+ssize_t glfs_llistxattr (glfs_t *fs, const char *path, void *value,
+			 size_t size);
+
+ssize_t glfs_flistxattr (glfs_fd_t *fd, void *value, size_t size);
+
+int glfs_setxattr (glfs_t *fs, const char *path, const char *name,
+		   const void *value, size_t size, int flags);
+
+int glfs_lsetxattr (glfs_t *fs, const char *path, const char *name,
+		    const void *value, size_t size, int flags);
+
+int glfs_fsetxattr (glfs_fd_t *fd, const char *name,
+		    const void *value, size_t size, int flags);
+
+int glfs_removexattr (glfs_t *fs, const char *path, const char *name);
+
+int glfs_lremovexattr (glfs_t *fs, const char *path, const char *name);
+
+int glfs_fremovexattr (glfs_fd_t *fd, const char *name);
+
+int glfs_fallocate(glfs_fd_t *fd, int keep_size, off_t offset, size_t len);
+
+int glfs_discard(glfs_fd_t *fd, off_t offset, size_t len);
+
+
+int glfs_discard_async (glfs_fd_t *fd, off_t length, size_t lent,
+			glfs_io_cbk fn, void *data);
+
+int glfs_zerofill(glfs_fd_t *fd, off_t offset, size_t len);
+
+int glfs_zerofill_async (glfs_fd_t *fd, off_t length, size_t len,
+                        glfs_io_cbk fn, void *data);
+
+char *glfs_getcwd (glfs_t *fs, char *buf, size_t size);
+
+int glfs_chdir (glfs_t *fs, const char *path);
+
+int glfs_fchdir (glfs_fd_t *fd);
+
+char *glfs_realpath (glfs_t *fs, const char *path, char *resolved_path);
+
+/*
+ * @cmd and @flock are as specified in man fcntl(2).
+ */
+int glfs_posix_lock (glfs_fd_t *fd, int cmd, struct flock *flock);
+
+glfs_fd_t *glfs_dup (glfs_fd_t *fd);
+
+__END_DECLS
+
+#endif /* !_GLFS_H */
diff --git a/argp-standalone/configure.ac b/argp-standalone/configure.ac
index 65ebc4518..2ecd2a801 100644
--- a/argp-standalone/configure.ac
+++ b/argp-standalone/configure.ac
@@ -8,7 +8,7 @@ AC_CONFIG_SRCDIR([argp-ba.c])
 AC_CONFIG_AUX_DIR([.])
 
 AM_INIT_AUTOMAKE
-AM_CONFIG_HEADER(config.h)
+AC_CONFIG_HEADERS(config.h)
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES(yes)])
 
@@ -22,7 +22,7 @@ AC_GNU_SOURCE
 AC_PROG_CC
 AC_PROG_MAKE_SET
 AC_PROG_RANLIB
-AM_PROG_CC_STDC
+AC_PROG_CC
 
 if test "x$am_cv_prog_cc_stdc" = xno ; then
   AC_ERROR([the C compiler doesn't handle ANSI-C])
diff --git a/autogen.sh b/autogen.sh
index e20408bf2..f937e6be0 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,8 +1,105 @@
 #!/bin/sh
 
-aclocal
-autoheader
-(libtoolize --automake --copy --force || glibtoolize --automake --copy --force)
-autoconf
-automake --add-missing --copy --foreign
+echo
+echo ... GlusterFS autogen ...
+echo
+
+## Check all dependencies are present
+MISSING=""
+
+# Check for aclocal
+env aclocal --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  ACLOCAL=aclocal
+else
+  MISSING="$MISSING aclocal"
+fi
+
+# Check for autoconf
+env autoconf --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  AUTOCONF=autoconf
+else
+  MISSING="$MISSING autoconf"
+fi
+
+# Check for autoheader
+env autoheader --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  AUTOHEADER=autoheader
+else
+  MISSING="$MISSING autoheader"
+fi
+
+# Check for automake
+env automake --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  AUTOMAKE=automake
+else
+  MISSING="$MISSING automake"
+fi
+
+# Check for libtoolize or glibtoolize
+env libtoolize --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  # libtoolize was found, so use it
+  TOOL=libtoolize
+else
+  # libtoolize wasn't found, so check for glibtoolize
+  env glibtoolize --version > /dev/null 2>&1
+  if [ $? -eq 0 ]; then
+    TOOL=glibtoolize
+  else
+    MISSING="$MISSING libtoolize/glibtoolize"
+  fi
+fi
+
+# Check for tar
+env tar --version > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+  MISSING="$MISSING tar"
+fi
+
+## If dependencies are missing, warn the user and abort
+if [ "x$MISSING" != "x" ]; then
+  echo "Aborting."
+  echo
+  echo "The following build tools are missing:"
+  echo
+  for pkg in $MISSING; do
+    echo "  * $pkg"
+  done
+  echo
+  echo "Please install them and try again."
+  echo
+  exit 1
+fi
+
+## generate gf-error-codes.h from error-codes.json
+echo "Generate gf-error-codes.h ..."
+if ./gen-headers.py; then
+    if ! mv -fv gf-error-codes.h libglusterfs/src/gf-error-codes.h; then
+	exit 1
+    fi
+else
+    exit 1
+fi
+
+## Do the autogeneration
+echo Running ${ACLOCAL}...
+$ACLOCAL -I ./contrib/aclocal
+echo Running ${AUTOHEADER}...
+$AUTOHEADER
+echo Running ${TOOL}...
+$TOOL --automake --copy --force
+echo Running ${AUTOCONF}...
+$AUTOCONF
+echo Running ${AUTOMAKE}...
+$AUTOMAKE --add-missing --copy --foreign
+
+# Run autogen in the argp-standalone sub-directory
 cd argp-standalone;./autogen.sh
+
+# Instruct user on next steps
+echo
+echo "Please proceed with configuring, compiling, and installing."
diff --git a/booster/Makefile.am b/booster/Makefile.am
deleted file mode 100644
index e1c45f305..000000000
--- a/booster/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS=src
-\ No newline at end of file
diff --git a/booster/src/Makefile.am b/booster/src/Makefile.am
deleted file mode 100644
index d7d83abf5..000000000
--- a/booster/src/Makefile.am
+++ /dev/null
@@ -1,21 +0,0 @@
-ldpreload_LTLIBRARIES = libglusterfs-booster.la
-ldpreloaddir = $(libdir)/glusterfs
-noinst_HEADERS = booster_fstab.h booster-fd.h
-libglusterfs_booster_la_SOURCES = booster.c booster_stat.c booster_fstab.c booster-fd.c
-libglusterfs_booster_la_CFLAGS = -I$(top_srcdir)/libglusterfsclient/src/ -D_GNU_SOURCE -D$(GF_HOST_OS) -fPIC -Wall \
-	-pthread $(GF_BOOSTER_CFLAGS) -shared -nostartfiles 
-libglusterfs_booster_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE  \
-	-I$(top_srcdir)/libglusterfsclient/src \
-	-I$(top_srcdir)/libglusterfs/src -DDATADIR=\"$(localstatedir)\" \
-	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(ARGP_STANDALONE_CPPFLAGS)
-
-libglusterfs_booster_la_LDFLAGS = -module -avoidversion
-libglusterfs_booster_la_LIBADD =  $(top_builddir)/libglusterfs/src/libglusterfs.la $(top_builddir)/libglusterfsclient/src/libglusterfsclient.la
-
-CLEANFILES =
-
-uninstall-local:
-	rm -f $(DESTDIR)$(ldpreloaddir)/glusterfs-booster.so
-
-install-data-hook:
-	ln -sf libglusterfs-booster.so $(DESTDIR)$(ldpreloaddir)/glusterfs-booster.so
diff --git a/booster/src/booster-fd.c b/booster/src/booster-fd.c
deleted file mode 100644
index fa5b0cde2..000000000
--- a/booster/src/booster-fd.c
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-
-#include "booster-fd.h"
-#include <logging.h>
-#include <mem-pool.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <common-utils.h>
-#include <string.h>
-
-#include <assert.h>
-
-extern fd_t *
-fd_ref (fd_t *fd);
-
-extern void
-fd_unref (fd_t *fd);
-/*
-   Allocate in memory chunks of power of 2 starting from 1024B
-   Assumes fdtable->lock is held
-   */
-static inline uint
-gf_roundup_power_of_two (uint nr)
-{
-        uint result = 1;
-
-        if (nr < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Negative number passed");
-                return -1;
-        }
-        
-        while (result <= nr)
-                result *= 2;
-
-        return result;
-}
-
-#define BOOSTER_NFDBITS (sizeof (unsigned long))
-
-#define BOOSTER_FDMASK(d)             (1UL << ((d) % BOOSTER_NFDBITS))
-#define BOOSTER_FDELT(d)              (d / BOOSTER_NFDBITS)
-#define BOOSTER_FD_SET(set, d)        (set->fd_bits[BOOSTER_FDELT(d)] |= BOOSTER_FDMASK(d))
-#define BOOSTER_FD_CLR(set, d)        (set->fd_bits[BOOSTER_FDELT(d)] &= ~BOOSTER_FDMASK(d))
-#define BOOSTER_FD_ISSET(set, d)      (set->fd_bits[BOOSTER_FDELT(d)] & BOOSTER_FDMASK(d))
-
-inline int
-booster_get_close_on_exec (booster_fdtable_t *fdtable, int fd)
-{
-        return BOOSTER_FD_ISSET(fdtable->close_on_exec, fd);
-}
-
-inline void
-booster_set_close_on_exec (booster_fdtable_t *fdtable, int fd)
-{
-        BOOSTER_FD_SET(fdtable->close_on_exec, fd);
-}
-
-int
-booster_fdtable_expand (booster_fdtable_t *fdtable, uint nr)
-{
-        fd_t    **oldfds = NULL, **tmp = NULL;
-        uint    oldmax_fds = -1;
-        uint    cpy = 0;
-        int32_t ret = -1, bytes = 0;
-        booster_fd_set_t *oldclose_on_exec = NULL;
-
-        if (fdtable == NULL || nr < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Invalid argument");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        nr /= (1024 / sizeof (fd_t *));
-        nr = gf_roundup_power_of_two (nr + 1);
-        nr *= (1024 / sizeof (fd_t *));
-
-        oldfds = fdtable->fds;
-        oldmax_fds = fdtable->max_fds;
-        oldclose_on_exec = fdtable->close_on_exec;
-
-        fdtable->fds = CALLOC (nr, sizeof (fd_t *));
-        if (fdtable->fds == NULL) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Memory allocation failed");
-                fdtable->fds = oldfds;
-                oldfds = NULL;
-                ret = -1;
-                goto out;
-        }
-
-        fdtable->max_fds = nr;
-
-        if (oldfds) {
-                cpy = oldmax_fds * sizeof (fd_t *);
-                memcpy (fdtable->fds, oldfds, cpy);
-        }
-
-        /* nr will be either less than 8 or a multiple of 8 */
-        bytes = nr/8;
-        bytes = bytes ? bytes : 1;
-        fdtable->close_on_exec = CALLOC (bytes, 1);
-        if (fdtable->close_on_exec == NULL) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Memory allocation "
-                        "failed");
-                tmp = fdtable->fds;
-                fdtable->fds = oldfds;
-                oldfds = tmp;
-                ret = -1;
-                goto out;
-        }
-
-        if (oldclose_on_exec != NULL) {
-                bytes = oldmax_fds/8;
-                cpy = bytes ? bytes : 1;
-                memcpy (fdtable->close_on_exec, oldclose_on_exec, cpy);
-        }
-        gf_log ("booster-fd", GF_LOG_TRACE, "FD-table expanded: Old: %d,New: %d"
-                , oldmax_fds, nr);
-        ret = 0;
-
-out:
-        FREE (oldfds);
-        FREE (oldclose_on_exec);
-
-        return ret;
-}
-
-booster_fdtable_t *
-booster_fdtable_alloc (void)
-{
-        booster_fdtable_t *fdtable = NULL;
-        int32_t            ret = -1;
-
-        fdtable = CALLOC (1, sizeof (*fdtable));
-        GF_VALIDATE_OR_GOTO ("booster-fd", fdtable, out);
-
-        LOCK_INIT (&fdtable->lock);
-
-        LOCK (&fdtable->lock);
-        {
-                ret = booster_fdtable_expand (fdtable, 0);
-        }
-        UNLOCK (&fdtable->lock);
-
-        if (ret == -1) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "FD-table allocation "
-                        "failed");
-                FREE (fdtable);
-                fdtable = NULL;
-        }
-
-out:
-        return fdtable;
-}
-
-fd_t **
-__booster_fdtable_get_all_fds (booster_fdtable_t *fdtable, uint *count)
-{
-        fd_t **fds = NULL;
-
-        if (count == NULL)
-                goto out;
-
-        fds = fdtable->fds;
-        fdtable->fds = calloc (fdtable->max_fds, sizeof (fd_t *));
-        *count = fdtable->max_fds;
-
-out:
-        return fds;
-}
-
-fd_t **
-booster_fdtable_get_all_fds (booster_fdtable_t *fdtable, uint *count)
-{
-        fd_t **fds = NULL;
-        if (!fdtable)
-                return NULL;
-
-        LOCK (&fdtable->lock);
-        {
-                fds = __booster_fdtable_get_all_fds (fdtable, count);
-        }
-        UNLOCK (&fdtable->lock);
-
-        return fds;
-}
-
-void
-booster_fdtable_destroy (booster_fdtable_t *fdtable)
-{
-        fd_t                    *fd = NULL;
-        fd_t                    **fds = NULL;
-        uint                    fd_count = 0;
-        int                     i = 0;
-
-        if (!fdtable)
-                return;
-
-        LOCK (&fdtable->lock);
-        {
-                fds = __booster_fdtable_get_all_fds (fdtable, &fd_count);
-                FREE (fdtable->fds);
-        }
-        UNLOCK (&fdtable->lock);
-
-        if (!fds)
-                goto free_table;
-        
-        for (i = 0; i < fd_count; i++) {
-                fd = fds[i];
-                if (fd != NULL)
-                        fd_unref (fd);
-        }
-        FREE (fds);
-free_table:
-        LOCK_DESTROY (&fdtable->lock);
-        FREE (fdtable);
-}
-
-int
-booster_fd_unused_get (booster_fdtable_t *fdtable, fd_t *fdptr, int fd)
-{
-        int ret = -1;
-        int error = 0;
-
-        if (fdtable == NULL || fdptr == NULL || fd < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "invalid argument");
-                errno = EINVAL;
-                return -1;
-        }
-
-        gf_log ("booster-fd", GF_LOG_TRACE, "Requested fd: %d", fd);
-        LOCK (&fdtable->lock);
-        {
-                while (fdtable->max_fds < fd) {
-                        error = 0;
-                        error = booster_fdtable_expand (fdtable,
-                                                        fdtable->max_fds + 1);
-                        if (error) {
-                                gf_log ("booster-fd", GF_LOG_ERROR,
-                                        "Cannot expand fdtable:%s",
-                                        strerror (error));
-                                goto err;
-                        }
-                }
-
-                if (!fdtable->fds[fd]) {
-                        fdtable->fds[fd] = fdptr;
-                        fd_ref (fdptr);
-                        ret = fd;
-                } else
-                        gf_log ("booster-fd", GF_LOG_ERROR, "Cannot allocate fd"
-                                " %d (slot not empty in fdtable)", fd);
-        }
-err:
-        UNLOCK (&fdtable->lock);
-
-        return ret;
-}
-
-void
-booster_fd_put (booster_fdtable_t *fdtable, int fd)
-{
-        fd_t *fdptr = NULL;
-        if (fdtable == NULL || fd < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "invalid argument");
-                return;
-        }
-
-        gf_log ("booster-fd", GF_LOG_TRACE, "FD put: %d", fd);
-        if (!(fd < fdtable->max_fds)) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "FD not in booster fd"
-                        " table");
-                return;
-        }
-
-        LOCK (&fdtable->lock);
-        {
-                fdptr = fdtable->fds[fd];
-                fdtable->fds[fd] = NULL;
-        }
-        UNLOCK (&fdtable->lock);
-
-        if (fdptr)
-                fd_unref (fdptr);
-}
-
-fd_t *
-booster_fdptr_get (booster_fdtable_t *fdtable, int fd)
-{
-        fd_t *fdptr = NULL;
-
-        if (fdtable == NULL || fd < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "invalid argument");
-                errno = EINVAL;
-                return NULL;
-        }
-
-        gf_log ("booster-fd", GF_LOG_TRACE, "FD ptr request: %d", fd);
-        if (!(fd < fdtable->max_fds)) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "FD not in booster fd"
-                        " table");
-                errno = EINVAL;
-                return NULL;
-        }
-
-        LOCK (&fdtable->lock);
-        {
-                fdptr = fdtable->fds[fd];
-                if (fdptr)
-                        fd_ref (fdptr);
-        }
-        UNLOCK (&fdtable->lock);
-
-        return fdptr;
-}
-
-void
-booster_fdptr_put (fd_t *booster_fd)
-{
-        if (booster_fd)
-                fd_unref (booster_fd);
-}
diff --git a/booster/src/booster-fd.h b/booster/src/booster-fd.h
deleted file mode 100644
index 595a112bd..000000000
--- a/booster/src/booster-fd.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _BOOSTER_FD_H
-#define _BOOSTER_FD_H
-
-#include <libglusterfsclient.h>
-#include <locking.h>
-#include <list.h>
-
-/* This struct must be updated if the fd_t in fd.h changes.
- * We cannot include those headers here because unistd.h, included
- * by glusterfs headers, conflicts with the syscall prototypes we
- * define for booster.
- */
-struct _fd {
-        pid_t             pid;
-	int32_t           flags;
-        int32_t           refcount;
-        struct list_head  inode_list;
-        struct _inode    *inode;
-        struct _dict     *ctx;
-        gf_lock_t         lock; /* used ONLY for manipulating
-                                   'struct _fd_ctx' array (_ctx).*/
-	struct _fd_ctx   *_ctx;
-};
-typedef struct _fd fd_t;
-
-struct _booster_fd_set {
-        unsigned long fd_bits[0];
-};
-typedef struct _booster_fd_set booster_fd_set_t;
-
-struct _booster_fdtable {
-        booster_fd_set_t *close_on_exec;
-        int               refcount;
-        unsigned int      max_fds;
-        gf_lock_t         lock;
-        fd_t            **fds;
-};
-typedef struct _booster_fdtable booster_fdtable_t;
-
-void
-booster_set_close_on_exec (booster_fdtable_t *fdtable, int fd);
-
-int
-booster_get_close_on_exec (booster_fdtable_t *fdtable, int fd);
-
-extern int
-booster_fd_unused_get (booster_fdtable_t *fdtable, fd_t *fdptr, int fd);
-
-extern void
-booster_fd_put (booster_fdtable_t *fdtable, int fd);
-
-extern fd_t *
-booster_fdptr_get (booster_fdtable_t *fdtable, int fd);
-
-extern void
-booster_fdptr_put (fd_t *fdptr);
-
-extern void
-booster_fdtable_destroy (booster_fdtable_t *fdtable);
-
-booster_fdtable_t *
-booster_fdtable_alloc (void);
-
-#endif /* #ifndef _BOOSTER_FD_H */
diff --git a/booster/src/booster.c b/booster/src/booster.c
deleted file mode 100644
index c34ec1146..000000000
--- a/booster/src/booster.c
+++ /dev/null
@@ -1,3172 +0,0 @@
-/*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <dlfcn.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <stdlib.h>
-#include <inttypes.h>
-#include <libglusterfsclient.h>
-#include <list.h>
-#include <pthread.h>
-#include <sys/xattr.h>
-#include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <ctype.h>
-#include <logging.h>
-#include <utime.h>
-#include <dirent.h>
-#include <sys/statfs.h>
-#include <sys/statvfs.h>
-#include <fcntl.h>
-#include "booster-fd.h"
-
-#ifndef GF_UNIT_KB
-#define GF_UNIT_KB 1024
-#endif
-
-static pthread_mutex_t cwdlock   = PTHREAD_MUTEX_INITIALIZER;
-
-/* attr constructor registers this function with libc's
- * _init function as a function that must be called before
- * the main() of the program.
- */
-static void booster_lib_init (void) __attribute__((constructor));
-
-extern fd_t *
-fd_ref (fd_t *fd);
-
-extern void
-fd_unref (fd_t *fd);
-
-extern int pipe (int filedes[2]);
-/* We define these flags so that we can remove fcntl.h from the include path.
- * fcntl.h has certain defines and other lines of code that redirect the
- * application's open and open64 calls to the syscalls defined by
- * libc, for us, thats not a Good Thing (TM).
- */
-#ifndef GF_O_CREAT
-#define GF_O_CREAT      0x40
-#endif
-
-#ifndef GF_O_TRUNC
-#define GF_O_TRUNC      0x200
-#endif
-
-#ifndef GF_O_RDWR
-#define GF_O_RDWR       0x2
-#endif
-
-#ifndef GF_O_WRONLY
-#define GF_O_WRONLY     0x1
-#endif
-
-#ifndef UNIX_PATH_MAX
-#define UNIX_PATH_MAX 108
-#endif
-
-typedef enum {
-        BOOSTER_OPEN,
-        BOOSTER_CREAT
-} booster_op_t;
-
-struct _inode;
-struct _dict;
-
-ssize_t
-write (int fd, const void *buf, size_t count);
-
-/* open, open64, creat */
-static int (*real_open) (const char *pathname, int flags, ...);
-static int (*real_open64) (const char *pathname, int flags, ...);
-static int (*real_creat) (const char *pathname, mode_t mode);
-static int (*real_creat64) (const char *pathname, mode_t mode);
-
-/* read, readv, pread, pread64 */
-static ssize_t (*real_read) (int fd, void *buf, size_t count);
-static ssize_t (*real_readv) (int fd, const struct iovec *vector, int count);
-static ssize_t (*real_pread) (int fd, void *buf, size_t count,
-                              unsigned long offset);
-static ssize_t (*real_pread64) (int fd, void *buf, size_t count,
-                                uint64_t offset);
-
-/* write, writev, pwrite, pwrite64 */
-static ssize_t (*real_write) (int fd, const void *buf, size_t count);
-static ssize_t (*real_writev) (int fd, const struct iovec *vector, int count);
-static ssize_t (*real_pwrite) (int fd, const void *buf, size_t count,
-                               unsigned long offset);
-static ssize_t (*real_pwrite64) (int fd, const void *buf, size_t count,
-                                 uint64_t offset);
-
-/* lseek, llseek, lseek64 */
-static off_t (*real_lseek) (int fildes, unsigned long offset, int whence);
-static off_t (*real_lseek64) (int fildes, uint64_t offset, int whence);
-
-/* close */
-static int (*real_close) (int fd);
-
-/* dup dup2 */
-static int (*real_dup) (int fd);
-static int (*real_dup2) (int oldfd, int newfd);
-
-static pid_t (*real_fork) (void);
-static int (*real_mkdir) (const char *pathname, mode_t mode);
-static int (*real_rmdir) (const char *pathname);
-static int (*real_chmod) (const char *pathname, mode_t mode);
-static int (*real_chown) (const char *pathname, uid_t owner, gid_t group);
-static int (*real_fchmod) (int fd, mode_t mode);
-static int (*real_fchown) (int fd, uid_t, gid_t gid);
-static int (*real_fsync) (int fd);
-static int (*real_ftruncate) (int fd, off_t length);
-static int (*real_ftruncate64) (int fd, loff_t length);
-static int (*real_link) (const char *oldpath, const char *newname);
-static int (*real_rename) (const char *oldpath, const char *newpath);
-static int (*real_utimes) (const char *path, const struct timeval times[2]);
-static int (*real_utime) (const char *path, const struct utimbuf *buf);
-static int (*real_mknod) (const char *path, mode_t mode, dev_t dev);
-static int (*real_mkfifo) (const char *path, mode_t mode);
-static int (*real_unlink) (const char *path);
-static int (*real_symlink) (const char *oldpath, const char *newpath);
-static int (*real_readlink) (const char *path, char *buf, size_t bufsize);
-static char * (*real_realpath) (const char *path, char *resolved);
-static DIR * (*real_opendir) (const char *path);
-static struct dirent * (*real_readdir) (DIR *dir);
-static struct dirent64 * (*real_readdir64) (DIR *dir);
-static int (*real_readdir_r) (DIR *dir, struct dirent *entry,
-                              struct dirent **result);
-static int (*real_readdir64_r) (DIR *dir, struct dirent64 *entry,
-                                struct dirent64 **result);
-static int (*real_closedir) (DIR *dh);
-static int (*real___xstat) (int ver, const char *path, struct stat *buf);
-static int (*real___xstat64) (int ver, const char *path, struct stat64 *buf);
-static int (*real_stat) (const char *path, struct stat *buf);
-static int (*real_stat64) (const char *path, struct stat64 *buf);
-static int (*real___fxstat) (int ver, int fd, struct stat *buf);
-static int (*real___fxstat64) (int ver, int fd, struct stat64 *buf);
-static int (*real_fstat) (int fd, struct stat *buf);
-static int (*real_fstat64) (int fd , struct stat64 *buf);
-static int (*real___lxstat) (int ver, const char *path, struct stat *buf);
-static int (*real___lxstat64) (int ver, const char *path, struct stat64 *buf);
-static int (*real_lstat) (const char *path, struct stat *buf);
-static int (*real_lstat64) (const char *path, struct stat64 *buf);
-static int (*real_statfs) (const char *path, struct statfs *buf);
-static int (*real_statfs64) (const char *path, struct statfs64 *buf);
-static int (*real_statvfs) (const char *path, struct statvfs *buf);
-static int (*real_statvfs64) (const char *path, struct statvfs64 *buf);
-static ssize_t (*real_getxattr) (const char *path, const char *name,
-                                 void *value, size_t size);
-static ssize_t (*real_lgetxattr) (const char *path, const char *name,
-                                  void *value, size_t size);
-static int (*real_remove) (const char* path);
-static int (*real_lchown) (const char *path, uid_t owner, gid_t group);
-static void (*real_rewinddir) (DIR *dirp);
-static void (*real_seekdir) (DIR *dirp, off_t offset);
-static off_t (*real_telldir) (DIR *dirp);
-
-static ssize_t (*real_sendfile) (int out_fd, int in_fd, off_t *offset,
-                                 size_t count);
-static ssize_t (*real_sendfile64) (int out_fd, int in_fd, off_t *offset,
-                                   size_t count);
-static int (*real_fcntl) (int fd, int cmd, ...);
-static int (*real_chdir) (const char *path);
-static int (*real_fchdir) (int fd);  
-static char * (*real_getcwd) (char *buf, size_t size);
-static int (*real_truncate) (const char *path, off_t length);
-static int (*real_truncate64) (const char *path, loff_t length);
-static int (*real_setxattr) (const char *path, const char *name,
-                             const void *value, size_t size, int flags);
-static int (*real_lsetxattr) (const char *path, const char *name,
-                              const void *value, size_t size, int flags);
-static int (*real_fsetxattr) (int filedes, const char *name,
-                              const void *value, size_t size, int flags);
-
-
-#define RESOLVE(sym) do {                                       \
-                if (!real_##sym)                                \
-                        real_##sym = dlsym (RTLD_NEXT, #sym);   \
-        } while (0)
-
-/*TODO: set proper value */
-#define MOUNT_HASH_SIZE 256
-
-struct booster_mount {
-        dev_t st_dev;
-        glusterfs_handle_t handle;
-        struct list_head device_list;
-};
-typedef struct booster_mount booster_mount_t;
-
-static booster_fdtable_t *booster_fdtable = NULL;
-
-extern int booster_configure (char *confpath);
-/* This is dup'ed every time VMP open/creat wants a new fd.
- * This is needed so we occupy an entry in the process' file
- * table.
- */
-int process_piped_fd = -1;
-
-static int
-booster_get_process_fd ()
-{
-        return real_dup (process_piped_fd);
-}
-
-/* The following two define which file contains
- * the FSTAB configuration for VMP-based usage.
- */
-#define DEFAULT_BOOSTER_CONF    CONFDIR"/booster.conf"
-#define BOOSTER_CONF_ENV_VAR    "GLUSTERFS_BOOSTER_FSTAB"
-
-
-/* The following define which log file is used when
- * using the old mount point bypass approach.
- */
-#define BOOSTER_DEFAULT_LOG     CONFDIR"/booster.log"
-#define BOOSTER_LOG_ENV_VAR     "GLUSTERFS_BOOSTER_LOG"
-
-void
-do_open (int fd, const char *pathname, int flags, mode_t mode, booster_op_t op)
-{
-        char                   *specfile = NULL;
-        char                   *mount_point = NULL; 
-        int32_t                 size = 0;
-        int32_t                 ret = -1;
-        FILE                   *specfp = NULL;
-        glusterfs_file_t        fh = NULL;
-        char                   *logfile = NULL;
-        glusterfs_init_params_t iparams = {
-                .loglevel = "error",
-                .lookup_timeout = 600,
-                .stat_timeout = 600,
-        };
-      
-        gf_log ("booster", GF_LOG_DEBUG, "Opening using MPB: %s", pathname);
-        size = fgetxattr (fd, "user.glusterfs-booster-volfile", NULL, 0);
-        if (size == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-volfile not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-		
-        specfile = calloc (1, size);
-        if (!specfile) {
-                gf_log ("booster", GF_LOG_ERROR, "Memory allocation failed");
-                goto out;
-        }
-
-        ret = fgetxattr (fd, "user.glusterfs-booster-volfile", specfile,
-                         size);
-        if (ret == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-volfile not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-    
-        specfp = tmpfile ();
-        if (!specfp) {
-                gf_log ("booster", GF_LOG_ERROR, "Temp file creation failed"
-                        ": %s", strerror (errno));
-                goto out;
-        }
-
-        ret = fwrite (specfile, size, 1, specfp);
-        if (ret != 1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to write volfile: %s",
-                        strerror (errno));
-                goto out;
-        }
-		
-        fseek (specfp, 0L, SEEK_SET);
-
-        size = fgetxattr (fd, "user.glusterfs-booster-mount", NULL, 0);
-        if (size == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-mount not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-        
-        mount_point = calloc (size, sizeof (char));
-        if (!mount_point) {
-                gf_log ("booster", GF_LOG_ERROR, "Memory allocation failed");
-                goto out;
-        }
-	
-        ret = fgetxattr (fd, "user.glusterfs-booster-mount", mount_point, size);
-        if (ret == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-mount not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        logfile = getenv (BOOSTER_LOG_ENV_VAR);
-        if (logfile) {
-                if (strlen (logfile) > 0)
-                        iparams.logfile = strdup (logfile);
-                else
-                        iparams.logfile = strdup (BOOSTER_DEFAULT_LOG);
-        } else {
-                iparams.logfile = strdup (BOOSTER_DEFAULT_LOG);
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Using log-file: %s", iparams.logfile);
-        iparams.specfp = specfp;
-
-        ret = glusterfs_mount (mount_point, &iparams);
-        if (ret == -1) {
-                if (errno != EEXIST) {
-                        gf_log ("booster", GF_LOG_ERROR, "Mount failed over"
-                                " glusterfs");
-                        goto out;
-                } else
-                        gf_log ("booster", GF_LOG_ERROR, "Already mounted");
-        }
-
-        switch (op) {
-        case BOOSTER_OPEN:
-                gf_log ("booster", GF_LOG_TRACE, "Booster open call");
-                fh = glusterfs_open (pathname, flags, mode);
-                break;
-
-        case BOOSTER_CREAT:
-                gf_log ("booster", GF_LOG_TRACE, "Booster create call");
-                fh = glusterfs_creat (pathname, mode);
-                break;
-        }
-
-        if (!fh) {
-                gf_log ("booster", GF_LOG_ERROR, "Error performing operation");
-                goto out;
-        }
-
-        if (booster_fd_unused_get (booster_fdtable, fh, fd) == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to get unused FD");
-                goto out;
-        }
-        fh = NULL;
-
-out:
-        if (specfile) {
-                free (specfile);
-        }
-
-        if (specfp) {
-                fclose (specfp);
-        }
-
-        if (mount_point) {
-                free (mount_point);
-        }
-
-        if (fh) {
-                glusterfs_close (fh);
-        }
-
-        return;
-}
-
-int
-vmp_open (const char *pathname, int flags, ...)
-{
-        mode_t                  mode = 0;
-        int                     fd = -1;
-        glusterfs_file_t        fh = NULL;
-        va_list                 ap;
-
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-
-                fh = glusterfs_open (pathname, flags, mode);
-        }
-        else
-                fh = glusterfs_open (pathname, flags);
-
-        if (!fh) {
-                gf_log ("booster", GF_LOG_ERROR, "VMP open failed");
-                goto out;
-        }
-
-        fd = booster_get_process_fd ();
-        if (fd == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to create open fd");
-                goto fh_close_out;
-        }
-
-        if (booster_fd_unused_get (booster_fdtable, fh, fd) == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to map fd into table");
-                goto realfd_close_out;
-        }
-
-        return fd;
-
-realfd_close_out:
-        real_close (fd);
-        fd = -1;
-
-fh_close_out:
-        glusterfs_close (fh);
-
-out:
-        return fd;
-}
-
-#define BOOSTER_USE_OPEN64          1
-#define BOOSTER_DONT_USE_OPEN64     0
-
-int
-booster_open (const char *pathname, int use64, int flags, ...)
-{
-        int     ret = -1;
-        mode_t  mode = 0;
-        va_list ap;
-        int     (*my_open) (const char *pathname, int flags, ...);
-
-        if (!pathname) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Open: %s", pathname);
-        /* First try opening through the virtual mount point.
-         * The difference lies in the fact that:
-         * 1. We depend on libglusterfsclient library to perform
-         * the translation from the path to handle.
-         * 2. We do not go to the file system for the fd, instead
-         * we use booster_get_process_fd (), which returns a dup'ed
-         * fd of a pipe created in booster_init.
-         */
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-                ret = vmp_open (pathname, flags, mode);
-        }
-        else
-                ret = vmp_open (pathname, flags);
-
-        /* We receive an ENODEV if the VMP does not exist. If we
-         * receive an error other than ENODEV, it means, there
-         * actually was an error performing vmp_open. This must
-         * be returned to the user.
-         */
-        if ((ret < 0) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Error in opening file over "
-                        " VMP: %s", strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "File opened");
-                goto out;
-        }
-
-        if (use64) {
-                gf_log ("booster", GF_LOG_TRACE, "Using 64-bit open");
-		my_open = real_open64;
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Using 32-bit open");
-		my_open = real_open;
-        }
-
-        /* It is possible the RESOLVE macro is not able
-         * to resolve the symbol of a function, in that case
-         * we dont want to seg-fault on calling a NULL functor.
-         */
-        if (my_open == NULL) {
-                gf_log ("booster", GF_LOG_ERROR, "open not resolved");
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-	if (flags & GF_O_CREAT) {
-		va_start (ap, flags);
-		mode = va_arg (ap, mode_t);
-		va_end (ap);
-
-                ret = my_open (pathname, flags, mode);
-	} else
-                ret = my_open (pathname, flags);
-
-        if (ret != -1) {
-                do_open (ret, pathname, flags, mode, BOOSTER_OPEN);
-        }
-
-out:
-        return ret;
-}
-
-/* This is done to over-write existing definitions of open and open64 inside
- * libc with our own copies. __REDIRECT is provided by libc.
- *
- * XXX: This will not work anywhere other than libc based systems.
- */
-int __REDIRECT (booster_false_open, (__const char *__file, int __oflag, ...),
-                open) __nonnull ((1));
-int __REDIRECT (booster_false_open64, (__const char *__file, int __oflag, ...),
-                open64) __nonnull ((1));
-int
-booster_false_open (const char *pathname, int flags, ...)
-{
-        int     ret;
-        mode_t  mode = 0;
-        va_list ap;
-
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-
-                ret = booster_open (pathname, BOOSTER_DONT_USE_OPEN64, flags,
-                                    mode);
-        }
-        else
-                ret = booster_open (pathname, BOOSTER_DONT_USE_OPEN64, flags);
-
-        return ret;
-}
-
-int
-booster_false_open64 (const char *pathname, int flags, ...)
-{
-        int     ret;
-        mode_t  mode = 0;
-        va_list ap;
-
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-
-                ret = booster_open (pathname, BOOSTER_USE_OPEN64, flags, mode);
-        }
-        else
-                ret = booster_open (pathname, BOOSTER_USE_OPEN64, flags);
-
-        return ret;
-}
-
-int
-vmp_creat (const char *pathname, mode_t mode)
-{
-        int                     fd = -1;
-        glusterfs_file_t        fh = NULL;
-
-        fh = glusterfs_creat (pathname, mode);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_ERROR, "Create failed: %s: %s",
-                        pathname, strerror (errno));
-                goto out;
-        }
-
-        fd = booster_get_process_fd ();
-        if (fd == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to create fd");
-                goto close_out;
-        }
-
-        if ((booster_fd_unused_get (booster_fdtable, fh, fd)) == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to map unused fd");
-                goto real_close_out;
-        }
-
-        return fd;
-
-real_close_out:
-        real_close (fd);
-        fd = -1;
-
-close_out:
-        glusterfs_close (fh);
-
-out:
-        return -1;
-}
-
-int __REDIRECT (booster_false_creat, (const char *pathname, mode_t mode),
-                creat) __nonnull ((1));
-int __REDIRECT (booster_false_creat64, (const char *pathname, mode_t mode),
-                creat64) __nonnull ((1));
-
-int
-booster_false_creat (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-        if (!pathname) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Create: %s", pathname);
-        ret = vmp_creat (pathname, mode);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "VMP create failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "File created");
-                goto out;
-        }
-
-        if (real_creat == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_creat (pathname, mode);
-
-        if (ret != -1) {
-                do_open (ret, pathname, GF_O_WRONLY | GF_O_TRUNC, mode,
-                         BOOSTER_CREAT);
-        } else
-                gf_log ("booster", GF_LOG_ERROR, "real create failed: %s",
-                        strerror (errno));
-
-out:
-        return ret;
-}
-
-
-int
-booster_false_creat64 (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-        if (!pathname) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Create: %s", pathname);
-        ret = vmp_creat (pathname, mode);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "VMP create failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "File created");
-                goto out;
-        }
-
-        if (real_creat64 == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_creat64 (pathname, mode);
-
-        if (ret != -1) {
-                do_open (ret, pathname, GF_O_WRONLY | GF_O_TRUNC, mode,
-                         BOOSTER_CREAT);
-        } else
-                gf_log ("booster", GF_LOG_ERROR, "real create failed: %s",
-                        strerror (errno));
-
-out:
-        return ret;
-}
-
-
-/* pread */
-
-ssize_t
-pread (int fd, void *buf, size_t count, unsigned long offset)
-{
-        ssize_t ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pread: fd %d, count %lu, offset %lu"
-                ,fd, (long unsigned)count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) { 
-                gf_log ("booster", GF_LOG_TRACE, "Not booster fd");
-                if (real_pread == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pread (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pread (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-pread64 (int fd, void *buf, size_t count, uint64_t offset)
-{
-        ssize_t ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pread64: fd %d, count %lu, offset %"
-                PRIu64, fd, (long unsigned)count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) { 
-                gf_log ("booster", GF_LOG_TRACE, "Not booster fd");
-                if (real_pread64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pread64 (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pread (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-read (int fd, void *buf, size_t count)
-{
-        int ret;
-        glusterfs_file_t glfs_fd;
-
-        gf_log ("booster", GF_LOG_TRACE, "read: fd %d, count %lu", fd,
-                (long unsigned)count);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not booster fd");
-                if (real_read == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_read (fd, buf, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_read (glfs_fd, buf, count);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-readv (int fd, const struct iovec *vector, int count)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "readv: fd %d, iovecs %d", fd, count);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_readv == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_readv (fd, vector, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-		ret = glusterfs_readv (glfs_fd, vector, count);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-write (int fd, const void *buf, size_t count)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "write: fd %d, count %"GF_PRI_SIZET,
-                fd, count);
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_write == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_write (fd, buf, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_write (glfs_fd, buf, count);
-                booster_fdptr_put (glfs_fd);
-        }
- 
-        return ret;
-}
-
-ssize_t
-writev (int fd, const struct iovec *vector, int count)
-{
-        int ret = 0;
-        glusterfs_file_t glfs_fd = 0; 
-
-        gf_log ("booster", GF_LOG_TRACE, "writev: fd %d, iovecs %d", fd, count);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_writev == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_writev (fd, vector, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_writev (glfs_fd, vector, count);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-pwrite (int fd, const void *buf, size_t count, unsigned long offset)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pwrite: fd %d, count %"GF_PRI_SIZET
-                ", offset %lu", fd, count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_pwrite == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pwrite (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pwrite (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-pwrite64 (int fd, const void *buf, size_t count, uint64_t offset)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pwrite64: fd %d, count %"GF_PRI_SIZET
-                ", offset %"PRIu64, fd, count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_pwrite64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pwrite64 (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pwrite (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-int
-close (int fd)
-{
-        int ret = -1;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "close: fd %d", fd);
-	glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-    
-	if (glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-		booster_fd_put (booster_fdtable, fd);
-		ret = glusterfs_close (glfs_fd);
-		booster_fdptr_put (glfs_fd);
-	}
-
-        ret = real_close (fd);
-
-        return ret;
-}
-
-#ifndef _LSEEK_DECLARED
-#define _LSEEK_DECLARED
-off_t
-lseek (int filedes, unsigned long offset, int whence)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "lseek: fd %d, offset %ld",
-                filedes, offset);
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, filedes);
-        if (glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_lseek (glfs_fd, offset, whence);
-                booster_fdptr_put (glfs_fd);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_lseek == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_lseek (filedes, offset, whence);
-        }
-
-        return ret;
-}
-#endif
-
-off_t
-lseek64 (int filedes, uint64_t offset, int whence)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-
-        gf_log ("booster", GF_LOG_TRACE, "lseek: fd %d, offset %"PRIu64,
-                filedes, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, filedes);
-        if (glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_lseek (glfs_fd, offset, whence);
-                booster_fdptr_put (glfs_fd);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_lseek64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_lseek64 (filedes, offset, whence);
-        }
-
-        return ret;
-}
-
-int 
-dup (int oldfd)
-{
-        int ret = -1, new_fd = -1;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "dup: fd %d", oldfd);
-        glfs_fd = booster_fdptr_get (booster_fdtable, oldfd);
-        new_fd = real_dup (oldfd);
-
-        if (new_fd >=0 && glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = booster_fd_unused_get (booster_fdtable, glfs_fd,
-                                             new_fd);
-                fd_ref ((fd_t *)glfs_fd);
-                if (ret == -1) {
-                        gf_log ("booster", GF_LOG_ERROR,"Failed to map new fd");
-                        real_close (new_fd);
-                } 
-        }
-
-        if (glfs_fd) {
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return new_fd;
-}
-
-
-int 
-dup2 (int oldfd, int newfd)
-{
-        int ret = -1;
-        glusterfs_file_t old_glfs_fd = NULL, new_glfs_fd = NULL;
-
-        if (oldfd == newfd) {
-                return newfd;
-        }
-
-        old_glfs_fd = booster_fdptr_get (booster_fdtable, oldfd);
-        new_glfs_fd = booster_fdptr_get (booster_fdtable, newfd);
- 
-        ret = real_dup2 (oldfd, newfd); 
-        if (ret >= 0) {
-                if (new_glfs_fd) {
-                        glusterfs_close (new_glfs_fd);
-                        booster_fdptr_put (new_glfs_fd);
-                        booster_fd_put (booster_fdtable, newfd);
-                        new_glfs_fd = 0;
-                }
-
-                if (old_glfs_fd) {
-                        ret = booster_fd_unused_get (booster_fdtable,
-                                                     old_glfs_fd, newfd);
-                        fd_ref ((fd_t *)old_glfs_fd);
-                        if (ret == -1) {
-                                real_close (newfd);
-                        }
-                }
-        } 
-
-        if (old_glfs_fd) {
-                booster_fdptr_put (old_glfs_fd);
-        }
-
-        if (new_glfs_fd) {
-                booster_fdptr_put (new_glfs_fd);
-        }
-
-        return ret;
-}
-
-int
-mkdir (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "mkdir: path %s", pathname);
-        ret = glusterfs_mkdir (pathname, mode);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "mkdir failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "directory created");
-                return ret;
-        }
-
-        if (real_mkdir == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-        } else
-                ret = real_mkdir (pathname, mode);
-
-        return ret;
-}
-
-int
-rmdir (const char *pathname)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "rmdir: path %s", pathname);
-        ret = glusterfs_rmdir (pathname);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "rmdir failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "directory removed");
-                return ret;
-        }
-
-        if (real_rmdir == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_rmdir (pathname);
-
-        return ret;
-}
-
-int
-chmod (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "chmod: path %s", pathname);
-        ret = glusterfs_chmod (pathname, mode);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "chmod failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "chmod succeeded");
-                return ret;
-        }
-
-        if (real_chmod == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_chmod (pathname, mode);
-
-        return ret;
-}
-
-int
-chown (const char *pathname, uid_t owner, gid_t group)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "chown: path: %s", pathname);
-        ret = glusterfs_chown (pathname, owner, group);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "chown failed: %s\n",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "chown succeeded");
-                return ret;
-        }
-
-        if (real_chown == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_chown (pathname, owner, group);
-
-        return ret;
-}
-
-int
-fchown (int fd, uid_t owner, gid_t group)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fchown: fd %d, uid %d, gid %d", fd,
-                owner, group);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fchown == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_fchown (fd, owner, group);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fchown (fh, owner, group);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-
-#define MOUNT_TABLE_HASH_SIZE 256
-
-
-static void booster_cleanup (void);
-static int 
-booster_init (void)
-{
-        char    *booster_conf_path = NULL;
-        int     ret = -1;
-        int     pipefd[2];
-
-        booster_fdtable = booster_fdtable_alloc ();
-        if (!booster_fdtable) {
-                fprintf (stderr, "cannot allocate fdtable: %s\n",
-                         strerror (errno));
-		goto err;
-        }
- 
-        if (pipe (pipefd) == -1) {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "Pipe creation failed:%s"
-                        , strerror (errno));
-                goto err;
-        }
-
-        process_piped_fd = pipefd[0];
-        real_close (pipefd[1]);
-        /* libglusterfsclient based VMPs should be inited only
-         * after the file tables are inited so that if the socket
-         * calls use the fd based syscalls, the fd tables are
-         * correctly initialized to return a NULL handle, on which the
-         * socket calls will fall-back to the real API.
-         */
-        booster_conf_path = getenv (BOOSTER_CONF_ENV_VAR);
-        if (booster_conf_path != NULL) {
-                if (strlen (booster_conf_path) > 0)
-                        ret = booster_configure (booster_conf_path);
-                else {
-                        gf_log ("booster", GF_LOG_ERROR, "%s not defined, "
-                                "using default path: %s", BOOSTER_CONF_ENV_VAR,
-                                DEFAULT_BOOSTER_CONF);
-                        ret = booster_configure (DEFAULT_BOOSTER_CONF);
-                }
-        } else {
-                gf_log ("booster", GF_LOG_ERROR, "%s not defined, using default"
-                        " path: %s", BOOSTER_CONF_ENV_VAR,DEFAULT_BOOSTER_CONF);
-                ret = booster_configure (DEFAULT_BOOSTER_CONF);
-        }
-
-        atexit (booster_cleanup);
-        if (ret == 0)
-                gf_log ("booster", GF_LOG_DEBUG, "booster is inited");
-	return 0;
-
-err:
-        /* Sure we return an error value here
-         * but who cares about booster.
-         */
-	return -1; 
-}
-
-
-static void
-booster_cleanup (void)
-{
-        /* Ideally, we should be de-initing the fd-table
-         * here but the problem is that I've seen file accesses through booster
-         * continuing while the atexit registered function is called. That means
-         * , we cannot dealloc the fd-table since then there could be a crash
-         * while trying to determine whether a given fd is for libc or for
-         * libglusterfsclient.
-         * We should be satisfied with having cleaned up glusterfs contexts.
-         */
-        glusterfs_umount_all ();
-	glusterfs_reset ();
-}
-
-int
-fchmod (int fd, mode_t mode)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fchmod: fd %d, mode: 0x%x", fd, mode);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fchmod == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_fchmod (fd, mode);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fchmod (fh, mode);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int
-fsync (int fd)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fsync: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fsync == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_fsync (fd);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fsync (fh);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int __REDIRECT (booster_false_ftruncate, (int fd, off_t length),
-                ftruncate);
-int __REDIRECT (booster_false_ftruncate64, (int fd, loff_t length),
-                ftruncate64);
-
-int
-booster_false_ftruncate (int fd, off_t length)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "ftruncate: fd %d, length: %"PRIu64,fd
-                , length);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_ftruncate == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_ftruncate (fd, length);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_ftruncate (fh, length);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int
-booster_false_ftruncate64 (int fd, loff_t length)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "ftruncate: fd %d, length: %"PRIu64,fd
-                , length);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_ftruncate == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_ftruncate64 (fd, length);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_ftruncate (fh, length);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int
-link (const char *old, const char *new)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "link: old: %s, new: %s", old, new);
-        ret = glusterfs_link (old, new);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Link failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "link call succeeded");
-                return ret;
-        }
-
-        if (real_link == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_link (old, new);
-
-        return ret;
-}
-
-int
-rename (const char *old, const char *new)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "link: old: %s, new: %s", old, new);
-        ret = glusterfs_rename (old, new);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Rename failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "Rename succeeded");
-                return ret;
-        }
-
-        if (real_rename == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_rename (old, new);
-
-        return ret;
-}
-
-int
-utimes (const char *path, const struct timeval times[2])
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "utimes: path %s", path);
-        ret = glusterfs_utimes (path, times);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "utimes failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "utimes succeeded");
-                return ret;
-        }
-
-        if (real_utimes == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_utimes (path, times);
-
-        return ret;
-}
-
-int
-utime (const char *path, const struct utimbuf *buf)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "utime: path %s", path);
-        ret = glusterfs_utime (path, buf);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "utime failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "utime succeeded");
-                return ret;
-        }
-
-        if (real_utime == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_utime (path, buf);
-
-        return ret;
-}
-
-int
-mknod (const char *path, mode_t mode, dev_t dev)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "mknod: path %s", path);
-        ret = glusterfs_mknod (path, mode, dev);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "mknod failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "mknod succeeded");
-                return ret;
-        }
-
-        if (real_mknod) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_mknod (path, mode, dev);
-
-        return ret;
-}
-
-int
-mkfifo (const  char *path, mode_t mode)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "mkfifo: path %s", path);
-        ret = glusterfs_mkfifo (path, mode);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "mkfifo failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "mkfifo succeeded");
-                return ret;
-        }
-
-        if (real_mkfifo == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_mkfifo (path, mode);
-
-        return ret;
-}
-
-int
-unlink (const char *path)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "unlink: path %s", path);
-        ret = glusterfs_unlink (path);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "unlink failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "unlink succeeded");
-                return ret;
-        }
-
-        if (real_unlink == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_unlink (path);
-
-        return ret;
-}
-
-int
-symlink (const char *oldpath, const char *newpath)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "symlink: old: %s, new: %s",
-                oldpath, newpath);
-        ret = glusterfs_symlink (oldpath, newpath);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "symlink failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "symlink succeeded");
-                return ret;
-        }
-
-        if (real_symlink == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_symlink (oldpath, newpath);
-
-        return ret;
-}
-
-int
-readlink (const char *path, char *buf, size_t bufsize)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "readlink: path %s", path);
-        ret = glusterfs_readlink (path, buf, bufsize);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "readlink failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "readlink succeeded");
-                return ret;
-        }
-
-        if (real_readlink == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_readlink (path, buf, bufsize);
-
-        return ret;
-}
-
-char *
-realpath (const char *path, char *resolved_path)
-{
-        char    *res = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "realpath: path %s", path);
-        res = glusterfs_realpath (path, resolved_path);
-        if ((res == NULL) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "realpath failed: %s",
-                        strerror (errno));
-                return res;
-        }
-
-        if (res != NULL) {
-                gf_log ("booster", GF_LOG_TRACE, "realpath succeeded");
-                return res;
-        }
-
-        if (real_realpath == NULL) {
-                errno = ENOSYS;
-                res = NULL;
-        } else
-                res = real_realpath (path, resolved_path);
-
-        return res;
-}
-
-#define BOOSTER_GL_DIR          1
-#define BOOSTER_POSIX_DIR       2
-
-struct booster_dir_handle {
-        int type;
-        void *dirh;
-};
-
-DIR *
-opendir (const char *path)
-{
-        glusterfs_dir_t                 gdir = NULL;
-        struct booster_dir_handle       *bh = NULL;
-        DIR                             *pdir = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "opendir: path: %s", path);
-        bh = calloc (1, sizeof (struct booster_dir_handle));
-        if (!bh) {
-                gf_log ("booster", GF_LOG_ERROR, "memory allocation failed");
-                errno = ENOMEM;
-                goto out;
-        }
-
-        gdir = glusterfs_opendir (path);
-        if (gdir) {
-                gf_log ("booster", GF_LOG_TRACE, "Gluster dir opened");
-                bh->type = BOOSTER_GL_DIR;
-                bh->dirh = (void *)gdir;
-                goto out;
-        } else if ((!gdir) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Opendir failed");
-                goto free_out;
-        }
-
-        if (real_opendir == NULL) {
-                errno = ENOSYS;
-                goto free_out;
-        }
-
-        pdir = real_opendir (path);
-
-        if (pdir) {
-                bh->type = BOOSTER_POSIX_DIR;
-                bh->dirh = (void *)pdir;
-                goto out;
-        }
-
-free_out:
-        if (bh) {
-                free (bh);
-                bh = NULL;
-        }
-out:
-        return (DIR *)bh;
-}
-
-int __REDIRECT (booster_false_readdir_r, (DIR *dir, struct dirent *entry,
-                struct dirent **result), readdir_r) __nonnull ((1));
-int __REDIRECT (booster_false_readdir64_r, (DIR *dir, struct dirent64 *entry,
-                struct dirent64 **result), readdir64_r) __nonnull ((1));
-
-int
-booster_false_readdir_r (DIR *dir, struct dirent *entry, struct dirent **result)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        int                              ret = 0;  
-
-        if (!bh) {
-                ret = errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on gluster");
-                ret = glusterfs_readdir_r ((glusterfs_dir_t)bh->dirh, entry,
-                                           result);
-                
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on posix");
-                if (real_readdir_r == NULL) {
-                        ret = errno = ENOSYS;
-                        goto out;
-                }
-
-                ret = real_readdir_r ((DIR *)bh->dirh, entry, result);
-        } else {
-                ret = errno = EINVAL;
-        }
-
-out:
-        return  ret;
-}
-
-int
-booster_false_readdir64_r (DIR *dir, struct dirent64 *entry,
-                           struct dirent64 **result)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        int                              ret = 0;  
-
-        if (!bh) {
-                ret = errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on gluster");
-                ret = glusterfs_readdir_r ((glusterfs_dir_t)bh->dirh,
-                                           (struct dirent *)entry,
-                                           (struct dirent **)result);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on posix");
-                if (real_readdir64_r == NULL) {
-                        ret = errno = ENOSYS;
-                        goto out;
-                }
-
-                ret = real_readdir64_r ((DIR *)bh->dirh, entry, result);
-        } else {
-                ret = errno = EINVAL;
-        }
-
-out:
-        return  ret;
-}
-
-struct dirent *
-__REDIRECT (booster_false_readdir, (DIR *dir), readdir) __nonnull ((1));
-
-struct dirent64 *
-__REDIRECT (booster_false_readdir64, (DIR *dir), readdir64) __nonnull ((1));
-
-struct dirent *
-booster_false_readdir (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        struct dirent                   *dirp = NULL;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on gluster");
-                dirp = glusterfs_readdir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on posix");
-                if (real_readdir == NULL) {
-                        errno = ENOSYS;
-                        dirp = NULL;
-                        goto out;
-                }
-
-                dirp = real_readdir ((DIR *)bh->dirh);
-        } else {
-                dirp = NULL;
-                errno = EINVAL;
-        }
-
-out:
-        return  dirp;
-}
-
-struct dirent64 *
-booster_false_readdir64 (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        struct dirent64                 *dirp = NULL;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on gluster");
-                dirp = glusterfs_readdir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on posix");
-                if (real_readdir == NULL) {
-                        errno = ENOSYS;
-                        dirp = NULL;
-                        goto out;
-                }
-
-                dirp = real_readdir64 ((DIR *)bh->dirh);
-        } else {
-                dirp = NULL;
-                errno = EINVAL;
-        }
-
-out:
-        return  dirp;
-}
-
-int
-closedir (DIR *dh)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dh;
-        int                             ret = -1;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "closedir on gluster");
-                ret = glusterfs_closedir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "closedir on posix");
-                if (real_closedir == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_closedir ((DIR *)bh->dirh);
-        } else {
-                errno = EBADF;
-        }
-
-        if (ret == 0) {
-                free (bh);
-                bh = NULL;
-        }
-out:
-        return ret;
-}
-
-/* The real stat functions reside in booster_stat.c to
- * prevent clash with the statX prototype and functions
- * declared from sys/stat.h
- */
-int
-booster_xstat (int ver, const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "xstat: path: %s", path);
-        ret = glusterfs_stat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "xstat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "xstat succeeded");
-                goto out;
-        }
-
-        if (real___xstat == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real___xstat (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_xstat64 (int ver, const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "xstat64: path: %s", path);
-        ret = glusterfs_stat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "xstat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "xstat64 succeeded");
-                goto out;
-        }
-
-        if (real___xstat64 == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real___xstat64 (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_stat (const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "stat: path: %s", path);
-        ret = glusterfs_stat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "stat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "stat succeeded");
-                goto out;
-        }
-
-        if (real_stat != NULL)
-                ret = real_stat (path, sbuf);
-        else if (real___xstat != NULL)
-                ret = real___xstat (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-
-out:
-        return ret;
-}
-
-int
-booster_stat64 (const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "stat64: %s", path);
-        ret = glusterfs_stat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "stat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "stat64 succeeded");
-                goto out;
-        }
-
-        if (real_stat64 != NULL)
-                ret = real_stat64 (path, sbuf);
-        else if (real___xstat64 != NULL)
-                ret = real___xstat64 (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fxstat (int ver, int fd, void *buf)
-{
-        struct stat             *sbuf = (struct stat *)buf;
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fxstat: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real___fxstat == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = real___fxstat (ver, fd, sbuf);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fxstat64 (int ver, int fd, void *buf)
-{
-        int                     ret = -1;
-        struct stat64           *sbuf = (struct stat64 *)buf;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fxstat64: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real___fxstat64 == NULL) {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-                ret = real___fxstat64 (ver, fd, sbuf);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, (struct stat *)sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fstat (int fd, void *buf)
-{
-        struct stat             *sbuf = (struct stat *)buf;
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fstat: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fstat != NULL)
-                        ret = real_fstat (fd, sbuf);
-                else if (real___fxstat != NULL)
-                        ret = real___fxstat (0, fd, sbuf);
-                else {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fstat64 (int fd, void *buf)
-{
-        int                     ret = -1;
-        struct stat64           *sbuf = (struct stat64 *)buf;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fstat64: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fstat64 != NULL)
-                        ret = real_fstat64 (fd, sbuf);
-                else if (real___fxstat64 != NULL)
-                        /* Not sure how portable the use of 0 for
-                         * version number is but it works over glibc.
-                         * We need this because, I've
-                         * observed that all the above real* functors can be
-                         * NULL. In that case, this is our last and only option.
-                         */
-                        ret = real___fxstat64 (0, fd, sbuf);
-                else {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, (struct stat *)sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_lxstat (int ver, const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lxstat: path %s", path);
-        ret = glusterfs_lstat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lxstat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lxstat succeeded");
-                goto out;
-        }
-
-        if (real___lxstat == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real___lxstat (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_lxstat64 (int ver, const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "lxstat64: path %s", path);
-        ret = glusterfs_lstat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lxstat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lxstat64 succeeded");
-                goto out;
-        }
-
-        if (real___lxstat64 == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real___lxstat64 (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_lstat (const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lstat: path %s", path);
-        ret = glusterfs_lstat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lstat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lstat succeeded");
-                goto out;
-        }
-
-        if (real_lstat != NULL)
-                ret = real_lstat (path, sbuf);
-        else if (real___lxstat != NULL)
-                ret = real___lxstat (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-
-out:
-        return ret;
-}
-
-int
-booster_lstat64 (const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "lstat64: path %s", path);
-        ret = glusterfs_lstat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lstat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lstat64 succeeded");
-                goto out;
-        }
-
-        if (real_lstat64 != NULL)
-                ret = real_lstat64 (path, sbuf);
-        else if (real___lxstat64 != NULL)
-                ret = real___lxstat64 (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_statfs (const char *pathname, struct statfs *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "statfs: path %s", pathname);
-        ret = glusterfs_statfs (pathname, buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statfs failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statfs succeeded");
-                goto out;
-        }
-
-        if (real_statfs == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statfs (pathname, buf);
-
-out:
-        return ret;
-}
-
-int
-booster_statfs64 (const char *pathname, struct statfs64 *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "stat64: path %s", pathname);
-        ret = glusterfs_statfs (pathname, (struct statfs *)buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statfs64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statfs64 succeeded");
-                goto out;
-        }
-
-        if (real_statfs64 == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statfs64 (pathname, buf);
-
-out:
-        return ret;
-}
-
-int
-booster_statvfs (const char *pathname, struct statvfs *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "statvfs: path %s", pathname);
-        ret = glusterfs_statvfs (pathname, buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statvfs failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statvfs succeeded");
-                goto out;
-        }
-
-        if (real_statvfs == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statvfs (pathname, buf);
-
-out:
-        return ret;
-}
-
-int
-booster_statvfs64 (const char *pathname, struct statvfs64 *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "statvfs64: path %s", pathname);
-        ret = glusterfs_statvfs (pathname, (struct statvfs *)buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statvfs64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statvfs64 succeeded");
-                goto out;
-        }
-
-        if (real_statvfs64 == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statvfs64 (pathname, buf);
-
-out:
-        return ret;
-}
-
-ssize_t
-getxattr (const char *path, const char *name, void *value, size_t size)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "getxattr: path %s, name %s", path,
-                name);
-        ret = glusterfs_getxattr (path, name, value, size);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "getxattr failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "getxattr succeeded");
-                return ret;
-        }
-
-        if (real_getxattr == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_getxattr (path, name, value, size);
-out:
-        return ret;
-}
-
-
-ssize_t
-lgetxattr (const char *path, const char *name, void *value, size_t size)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lgetxattr: path %s, name %s", path,
-                name);
-        ret = glusterfs_lgetxattr (path, name, value, size);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lgetxattr failed: %s",
-                        strerror (errno));
-
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lgetxattr succeeded");
-                return ret;
-        }
-
-        if (real_lgetxattr == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_lgetxattr (path, name, value, size);
-out:
-        return ret;
-}
-
-int
-remove (const char *path)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "remove: %s", path);
-        ret = glusterfs_remove (path);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "remove failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "remove succeeded");
-                goto out;
-        }
-
-        if (real_remove == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_remove (path);
-
-out:
-        return ret;
-}
-
-int
-lchown (const char *path, uid_t owner, gid_t group)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lchown: path %s", path);
-        ret = glusterfs_lchown (path, owner, group);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lchown failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_ERROR, "lchown succeeded");
-                goto out;
-        }
-
-        if (real_lchown == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_lchown (path, owner, group);
-
-out:
-        return ret;
-}
-
-void
-booster_rewinddir (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "rewinddir on glusterfs");
-                glusterfs_rewinddir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                if (real_rewinddir == NULL) {
-                        errno = ENOSYS;
-                        goto out;
-                }
-                gf_log ("booster", GF_LOG_TRACE, "rewinddir on posix");
-                real_rewinddir ((DIR *)bh->dirh);
-        } else
-                errno = EINVAL;
-out:
-        return;
-}
-
-void
-booster_seekdir (DIR *dir, off_t offset)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "seekdir on glusterfs");
-                glusterfs_seekdir ((glusterfs_dir_t)bh->dirh, offset);
-         } else if (bh->type == BOOSTER_POSIX_DIR) {
-                if (real_seekdir == NULL) {
-                        errno = ENOSYS;
-                        goto out;
-                }
-
-                gf_log ("booster", GF_LOG_TRACE, "seekdir on posix");
-                real_seekdir ((DIR *)bh->dirh, offset);
-        } else
-                errno = EINVAL;
-out:
-        return;
-}
-
-off_t
-booster_telldir (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        off_t	offset = -1;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "telldir on glusterfs");
-                offset = glusterfs_telldir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                if (real_telldir == NULL) {
-                        errno = ENOSYS;
-                        goto out;
-                }
-
-                gf_log ("booster", GF_LOG_TRACE, "telldir on posix");
-                offset = real_telldir ((DIR *)bh->dirh);
-        } else
-                errno = EINVAL;
-out:
-        return offset;
-}
-
-
-pid_t 
-fork (void)
-{
-	pid_t pid = 0;
-	char child = 0;
-
-	glusterfs_log_lock ();
-	{
-		pid = real_fork ();
-	}
-	glusterfs_log_unlock ();
-
-	child = (pid == 0);
-	if (child) {
-		booster_cleanup ();
-		booster_init ();
-	}
-
-	return pid;
-}
-
-ssize_t
-sendfile (int out_fd, int in_fd, off_t *offset, size_t count)
-{
-        glusterfs_file_t            in_fh = NULL;
-        ssize_t                     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "sendfile: in fd %d, out fd %d, offset"
-                " %"PRIu64", count %"GF_PRI_SIZET, in_fd, out_fd, *offset,
-                count);
-        /*
-         * handle sendfile in booster only if in_fd corresponds to a glusterfs
-         * file handle 
-         */
-        in_fh = booster_fdptr_get (booster_fdtable, in_fd);
-        if (!in_fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_sendfile == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else {
-                        ret = real_sendfile (out_fd, in_fd, offset, count);
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_sendfile (out_fd, in_fh, offset, count);
-                booster_fdptr_put (in_fh);
-        }
-        
-        return ret;
-}
-
-ssize_t
-sendfile64 (int out_fd, int in_fd, off_t *offset, size_t count)
-{
-        glusterfs_file_t            in_fh = NULL;
-        ssize_t                     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "sendfile64: in fd %d, out fd %d,"
-                " offset %"PRIu64", count %"GF_PRI_SIZET, in_fd, out_fd,
-                *offset, count);
-        /*
-         * handle sendfile in booster only if in_fd corresponds to a glusterfs
-         * file handle 
-         */
-        in_fh = booster_fdptr_get (booster_fdtable, in_fd);
-        if (!in_fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_sendfile64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else {
-                        ret = real_sendfile64 (out_fd, in_fd, offset, count);
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_sendfile (out_fd, in_fh, offset, count);
-                booster_fdptr_put (in_fh);
-        }
-        
-        return ret;
-}
-
-
-int
-fcntl (int fd, int cmd, ...)
-{
-        va_list           ap;
-        int               ret = -1;
-        long              arg = 0;
-        struct flock     *lock = NULL;
-        glusterfs_file_t  glfs_fd = 0; 
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        gf_log ("booster", GF_LOG_TRACE, "fcntl: fd %d, cmd %d", fd, cmd);
-	switch (cmd) {
-	case F_DUPFD:
-                ret = dup (fd);
-                break;
-                /* 
-                 * FIXME: Consider this case when implementing F_DUPFD, F_GETFD
-                 *        etc flags in libglusterfsclient. Commenting it out for
-                 *        timebeing since it is defined only in linux kernel 
-                 *        versions >= 2.6.24.
-                 */
-                /* case F_DUPFD_CLOEXEC: */
-	case F_GETFD:
-                if (glfs_fd != NULL) {
-                        ret = booster_get_close_on_exec (booster_fdtable, fd)
-                                ? FD_CLOEXEC : 0;
-                } else {
-                        if (real_fcntl == NULL) {
-                                ret = -1;
-                                errno = ENOSYS;
-                        } else {
-                                ret = real_fcntl (fd, cmd);
-                        }
-                }
-                break;
-
-	case F_GETFL:
-	case F_GETOWN:
-	case F_GETSIG:
-	case F_GETLEASE:
-                if (glfs_fd) {
-                        gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                        ret = glusterfs_fcntl (glfs_fd, cmd);
-                } else {
-                        if (!real_fcntl) {
-                                errno = ENOSYS;
-                                goto out;
-                        }
-
-                        gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                        ret = real_fcntl (fd, cmd);
-                }
-		break;
-
-	case F_SETFD:
-                if (glfs_fd != NULL) {
-                        booster_set_close_on_exec (booster_fdtable, fd);
-                        ret = 0;
-                } else {
-                        if (real_fcntl == NULL) {
-                                ret = -1;
-                                errno = ENOSYS;
-                        } else {
-                                ret = real_fcntl (fd, cmd);
-                        }
-                }
-                break;
-
-	case F_SETFL:
-	case F_SETOWN:
-	case F_SETSIG:
-	case F_SETLEASE:
-	case F_NOTIFY:
-                va_start (ap, cmd);
-                arg = va_arg (ap, long);
-                va_end (ap);
-
-                if (glfs_fd) {
-                        gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                        ret = glusterfs_fcntl (glfs_fd, cmd, arg);
-                } else {
-                        if (!real_fcntl) {
-                                errno = ENOSYS;
-                                goto out;
-                        }
-
-                        gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                        ret = real_fcntl (fd, cmd, arg);
-                }
-		break;
-
-	case F_GETLK:
-	case F_SETLK:
-	case F_SETLKW:
-#if F_GETLK != F_GETLK64 
-        case F_GETLK64:
-#endif
-#if F_SETLK != F_SETLK64
-        case F_SETLK64:
-#endif
-#if F_SETLKW != F_SETLKW64
-        case F_SETLKW64:
-#endif
-                va_start (ap, cmd);
-                lock = va_arg (ap, struct flock *);
-                va_end (ap);
-
-                if (lock == NULL) {
-                        errno = EINVAL;
-                        goto out;
-                }
-
-                if (glfs_fd) {
-                        gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                        ret = glusterfs_fcntl (glfs_fd, cmd, lock);
-                } else {
-                        if (!real_fcntl) {
-                                errno = ENOSYS;
-                                goto out;
-                        }
-
-                        gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                        ret = real_fcntl (fd, cmd, lock);
-                }
-		break;
-
-	default:
-                errno = EINVAL;
-		break;
-	}
-
-out:
-        if (glfs_fd) {
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-int
-chdir (const char *path)
-{
-        int     ret = -1;
-        char    cwd[PATH_MAX];
-        char   *res = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "chdir: path %s", path);
-
-        pthread_mutex_lock (&cwdlock);
-        {
-                res = glusterfs_getcwd (cwd, PATH_MAX);
-                if (res == NULL) {
-                        gf_log ("booster", GF_LOG_ERROR, "getcwd failed: %s",
-                                strerror (errno));
-                        goto unlock;
-                }
-
-                ret = glusterfs_chdir (path);
-                if ((ret == -1) && (errno != ENODEV)) {
-                        gf_log ("booster", GF_LOG_ERROR, "chdir failed: %s",
-                                strerror (errno));
-                        goto unlock;
-                }
-
-                if (ret == 0) {
-                        gf_log ("booster", GF_LOG_TRACE, "chdir succeeded");
-                        goto unlock;
-                }
-
-                if (real_chdir == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                        goto unlock;
-                }
-
-                ret = real_chdir (path);
-                if (ret == -1) {
-                        glusterfs_chdir (cwd);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&cwdlock);
-
-        return ret;
-}
-
-
-int
-fchdir (int fd)
-{
-        int              ret     = -1;
-        glusterfs_file_t glfs_fd = 0;
-        char             cwd[PATH_MAX]; 
-        char            *res     = NULL;
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_write == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else {
-                        ret = real_fchdir (fd);
-                        if (ret == 0) {
-                                res = real_getcwd (cwd, PATH_MAX);
-                                if (res == NULL) {
-                                        gf_log ("booster", GF_LOG_ERROR,
-                                                "getcwd failed (%s)",
-                                                strerror (errno));
-                                        ret = -1;
-                                } else {
-                                        glusterfs_chdir (cwd);
-                                }
-                        }
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fchdir (glfs_fd);
-                booster_fdptr_put (glfs_fd);
-        }
- 
-        return ret;
-}
-
-
-char *
-getcwd (char *buf, size_t size)
-{
-        char *res = NULL;
-
-        res = glusterfs_getcwd (buf, size);
-        if ((res == NULL) && (errno == ENODEV)) {
-                res = real_getcwd (buf, size);
-        }
-
-        return res;
-}
-
-
-int __REDIRECT (booster_false_truncate, (const char *path, off_t length),
-                truncate) __nonnull ((1));
-int __REDIRECT (booster_false_truncate64, (const char *path, loff_t length),
-                truncate64) __nonnull ((1));;
-
-int
-booster_false_truncate (const char *path, off_t length)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "truncate: path (%s) length (%"PRIu64
-                ")", path, length);
-
-        ret = glusterfs_truncate (path, length);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "truncate failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "truncate succeeded");
-                goto out;
-        }
-
-        if (real_truncate != NULL)
-                ret = real_truncate (path, length);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-booster_false_truncate64 (const char *path, loff_t length)
-{
-        int             ret = -1;
-  
-        gf_log ("booster", GF_LOG_TRACE, "truncate64: path (%s) length "
-                "(%"PRIu64")", path, length);
-
-        ret = glusterfs_truncate (path, length);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "truncate64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "truncate64 succeeded");
-                goto out;
-        }
-
-        if (real_truncate64 != NULL)
-                ret = real_truncate64 (path, length);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-setxattr (const char *path, const char *name, const void *value, size_t size,
-          int flags)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "setxattr: path: %s", path);
-        ret = glusterfs_setxattr (path, name, value, size, flags);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "setxattr failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "setxattr succeeded");
-                goto out;
-        }
-
-        if (real_setxattr != NULL)
-                ret = real_setxattr (path, name, value, size, flags);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-lsetxattr (const char *path, const char *name, const void *value, size_t size,
-           int flags)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lsetxattr: path: %s", path);
-        ret = glusterfs_lsetxattr (path, name, value, size, flags);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lsetxattr failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lsetxattr succeeded");
-                goto out;
-        }
-
-        if (real_lsetxattr != NULL)
-                ret = real_lsetxattr (path, name, value, size, flags);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-fsetxattr (int fd, const char *name, const void *value, size_t size, int flags)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fsetxattr: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fsetxattr != NULL)
-                        ret = real_fsetxattr (fd, name, value, size, flags);
-                else {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fsetxattr (fh, name, value, size, flags);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-
-void
-booster_lib_init (void)
-{
-
-        RESOLVE (open);
-        RESOLVE (open64);
-        RESOLVE (creat);
-        RESOLVE (creat64);
-
-        RESOLVE (read);
-        RESOLVE (readv);
-        RESOLVE (pread);
-        RESOLVE (pread64);
-
-        RESOLVE (write);
-        RESOLVE (writev);
-        RESOLVE (pwrite);
-        RESOLVE (pwrite64);
-
-        RESOLVE (lseek);
-        RESOLVE (lseek64);
-
-        RESOLVE (close);
-
-        RESOLVE (dup);
-        RESOLVE (dup2);
-
-	RESOLVE (fork); 
-        RESOLVE (mkdir);
-        RESOLVE (rmdir);
-        RESOLVE (chmod);
-        RESOLVE (chown);
-        RESOLVE (fchmod);
-        RESOLVE (fchown);
-        RESOLVE (fsync);
-        RESOLVE (ftruncate);
-        RESOLVE (ftruncate64);
-        RESOLVE (link);
-        RESOLVE (rename);
-        RESOLVE (utimes);
-        RESOLVE (utime);
-        RESOLVE (mknod);
-        RESOLVE (mkfifo);
-        RESOLVE (unlink);
-        RESOLVE (symlink);
-        RESOLVE (readlink);
-        RESOLVE (realpath);
-        RESOLVE (opendir);
-        RESOLVE (readdir);
-        RESOLVE (readdir64);
-        RESOLVE (closedir);
-        RESOLVE (__xstat);
-        RESOLVE (__xstat64);
-        RESOLVE (stat);
-        RESOLVE (stat64);
-        RESOLVE (__fxstat);
-        RESOLVE (__fxstat64);
-        RESOLVE (fstat);
-        RESOLVE (fstat64);
-        RESOLVE (__lxstat);
-        RESOLVE (__lxstat64);
-        RESOLVE (lstat);
-        RESOLVE (lstat64);
-        RESOLVE (statfs);
-        RESOLVE (statfs64);
-        RESOLVE (statvfs);
-        RESOLVE (statvfs64);
-        RESOLVE (getxattr);
-        RESOLVE (lgetxattr);
-        RESOLVE (remove);
-        RESOLVE (lchown);
-	RESOLVE (rewinddir);
-	RESOLVE (seekdir);
-	RESOLVE (telldir);
-        RESOLVE (sendfile);
-        RESOLVE (sendfile64);
-        RESOLVE (readdir_r);
-        RESOLVE (readdir64_r);
-        RESOLVE (fcntl);
-        RESOLVE (chdir);
-        RESOLVE (fchdir);
-        RESOLVE (getcwd);
-        RESOLVE (truncate);
-        RESOLVE (truncate64);
-        RESOLVE (setxattr);
-        RESOLVE (lsetxattr);
-        RESOLVE (fsetxattr);
-
-        /* This must be called after resolving real functions
-         * above so that the socket based IO calls in libglusterfsclient
-         * can fall back to a non-NULL real_XXX function pointer.
-         * Calling booster_init before resolving the names above
-         * results in seg-faults because the function symbols above are NULL.
-         */
-	booster_init ();
-}
-
diff --git a/booster/src/booster_fstab.c b/booster/src/booster_fstab.c
deleted file mode 100644
index 202249cad..000000000
--- a/booster/src/booster_fstab.c
+++ /dev/null
@@ -1,452 +0,0 @@
-/* Utilities for reading/writing fstab, mtab, etc.
-   Copyright (C) 1995-2000, 2001, 2002, 2003, 2006
-   Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, write to the Free
-   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-   02111-1307 USA.  */
-
-#include <alloca.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/types.h>
-#include "booster_fstab.h"
-#include <stdlib.h>
-#include <libglusterfsclient.h>
-#include <errno.h>
-
-/* The default timeout for inode and stat cache. */
-#define BOOSTER_DEFAULT_ATTR_TIMEO      5 /* In Secs */
-
-/* Prepare to begin reading and/or writing mount table entries from the
-   beginning of FILE.  MODE is as for `fopen'.  */
-glusterfs_fstab_t *
-glusterfs_fstab_init (const char *file, const char *mode)
-{
-        glusterfs_fstab_t *handle = NULL;
-        handle = calloc (1, sizeof (glusterfs_fstab_t));
-        if (!handle) {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "Memory allocation"
-                        " failed");
-                goto out;
-        }
-
-        gf_log ("booster-fstab", GF_LOG_DEBUG, "FSTAB file: %s", file);
-        FILE *result = fopen (file,mode);
-        if (result != NULL) {
-                handle->fp = result;
-        } else {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "FSTAB file open failed:"
-                        " %s", strerror (errno));
-                free (handle);
-                handle = NULL;
-        }
-
-out:
-
-        return handle;
-}
-
-int
-glusterfs_fstab_close (glusterfs_fstab_t *h)
-{
-        if (!h)
-                return -1;
-
-        if (h->fp)
-                fclose (h->fp);
-
-        return 0;
-}
-
-/* Since the values in a line are separated by spaces, a name cannot
-   contain a space.  Therefore some programs encode spaces in names
-   by the strings "\040".  We undo the encoding when reading an entry.
-   The decoding happens in place.  */
-static char *
-decode_name (char *buf)
-{
-        char *rp = buf;
-        char *wp = buf;
-
-        do
-                if (rp[0] == '\\' && rp[1] == '0' && rp[2] == '4'
-                                && rp[3] == '0')
-                {
-                        /* \040 is a SPACE.  */
-                        *wp++ = ' ';
-                        rp += 3;
-                }
-                else if (rp[0] == '\\' && rp[1] == '0' && rp[2] == '1'
-                                && rp[3] == '1')
-                {
-                        /* \011 is a TAB.  */
-                        *wp++ = '\t';
-                        rp += 3;
-                }
-                else if (rp[0] == '\\' && rp[1] == '0' && rp[2] == '1'
-                                && rp[3] == '2')
-                {
-                        /* \012 is a NEWLINE.  */
-                        *wp++ = '\n';
-                        rp += 3;
-                }
-                else if (rp[0] == '\\' && rp[1] == '\\')
-                {
-                        /* We have to escape \\ to be able to represent all
-                         * characters.  */
-                        *wp++ = '\\';
-                        rp += 1;
-                }
-                else if (rp[0] == '\\' && rp[1] == '1' && rp[2] == '3'
-                                && rp[3] == '4')
-                {
-                        /* \134 is also \\.  */
-                        *wp++ = '\\';
-                        rp += 3;
-                }
-                else
-                        *wp++ = *rp;
-        while (*rp++ != '\0');
-
-        return buf;
-}
-
-
-/* Read one mount table entry from STREAM.  Returns a pointer to storage
-   reused on the next call, or null for EOF or error (use feof/ferror to
-   check).  */
-struct glusterfs_mntent *
-__glusterfs_fstab_getent (FILE *stream, struct glusterfs_mntent *mp,
-                          char *buffer, int bufsiz)
-{
-        char *cp;
-        char *head;
-
-        do
-        {
-                char *end_ptr;
-
-                if (fgets (buffer, bufsiz, stream) == NULL)
-                {
-                        return NULL;
-                }
-
-                end_ptr = strchr (buffer, '\n');
-                if (end_ptr != NULL)	/* chop newline */
-                        *end_ptr = '\0';
-                else
-                {
-                        /* Not the whole line was read.  Do it now but forget
-                         * it.  */
-                        char tmp[1024];
-                        while (fgets (tmp, sizeof tmp, stream) != NULL)
-                                if (strchr (tmp, '\n') != NULL)
-                                        break;
-                }
-
-                head = buffer + strspn (buffer, " \t");
-                /* skip empty lines and comment lines:  */
-        }
-        while (head[0] == '\0' || head[0] == '#');
-
-        cp = strsep (&head, " \t");
-        mp->mnt_fsname = cp != NULL ? decode_name (cp) : (char *) "";
-        if (head)
-                head += strspn (head, " \t");
-        cp = strsep (&head, " \t");
-        mp->mnt_dir = cp != NULL ? decode_name (cp) : (char *) "";
-        if (head)
-                head += strspn (head, " \t");
-        cp = strsep (&head, " \t");
-        mp->mnt_type = cp != NULL ? decode_name (cp) : (char *) "";
-        if (head)
-                head += strspn (head, " \t");
-        cp = strsep (&head, " \t");
-        mp->mnt_opts = cp != NULL ? decode_name (cp) : (char *) "";
-        switch (head ? sscanf (head, " %d %d ", &mp->mnt_freq,
-                               &mp->mnt_passno) : 0)
-        {
-                case 0:
-                        mp->mnt_freq = 0;
-                case 1:
-                        mp->mnt_passno = 0;
-                case 2:
-                        break;
-        }
-
-        return mp;
-}
-
-struct glusterfs_mntent *
-glusterfs_fstab_getent (glusterfs_fstab_t *h)
-{
-        if (!h)
-                return NULL;
-
-        if (!h->fp)
-                return NULL;
-
-        return __glusterfs_fstab_getent (h->fp, &h->tmpent, h->buf,
-                                         GF_MNTENT_BUFSIZE);
-}
-
-/* We have to use an encoding for names if they contain spaces or tabs.
-   To be able to represent all characters we also have to escape the
-   backslash itself.  This "function" must be a macro since we use
-   `alloca'.  */
-#define encode_name(name)                                               \
-        do {                                                            \
-                const char *rp = name;		                        \
-                                                                        \
-                while (*rp != '\0')     	                        \
-                        if (*rp == ' ' || *rp == '\t' || *rp == '\\')   \
-                                break;                                  \
-                        else	                                        \
-                                ++rp;                                   \
-                                                                        \
-                if (*rp != '\0')                                        \
-                {                                               \
-                /* In the worst case the length of the string   \
-                 * can increase to four times the current       \
-                 * length.  */				        \
-                        char *wp;				\
-                                                                \
-                        rp = name;				\
-                        name = wp = (char *) alloca (strlen (name) * 4 + 1);                                                                 \
-                                                                \
-                        do {                            \
-                                if (*rp == ' ')		\
-                                {       		\
-                                        *wp++ = '\\';   \
-                                        *wp++ = '0';	\
-                                        *wp++ = '4';	\
-                                        *wp++ = '0';    \
-                                }			\
-                                else if (*rp == '\t')	\
-                                {			\
-                                        *wp++ = '\\';	\
-                                        *wp++ = '0';	\
-                                        *wp++ = '1';	\
-                                        *wp++ = '1';	\
-                                }			\
-                                else if (*rp == '\n')	\
-                                {	                \
-                                        *wp++ = '\\';	\
-                                        *wp++ = '0';	\
-                                        *wp++ = '1';	\
-                                        *wp++ = '2';	\
-                                }	                \
-                                else if (*rp == '\\')	\
-                                {                       \
-                                        *wp++ = '\\';	\
-                                        *wp++ = '\\';	\
-                                }                       \
-                                else	                \
-                                        *wp++ = *rp;	\
-                        } while (*rp++ != '\0');	\
-                }                                       \
-        } while (0)                                     \
-
-
-int
-glusterfs_fstab_addent (glusterfs_fstab_t *h,
-                const struct glusterfs_mntent *mnt)
-{
-        struct glusterfs_mntent mntcopy = *mnt;
-        if (!h)
-                return -1;
-
-        if (!h->fp)
-                return -1;
-
-        if (fseek (h->fp, 0, SEEK_END))
-                return -1;
-
-        /* Encode spaces and tabs in the names.  */
-        encode_name (mntcopy.mnt_fsname);
-        encode_name (mntcopy.mnt_dir);
-        encode_name (mntcopy.mnt_type);
-        encode_name (mntcopy.mnt_opts);
-
-        return (fprintf (h->fp, "%s %s %s %s %d %d\n",
-                                mntcopy.mnt_fsname,
-                                mntcopy.mnt_dir,
-                                mntcopy.mnt_type,
-                                mntcopy.mnt_opts,
-                                mntcopy.mnt_freq,
-                                mntcopy.mnt_passno)
-                        < 0 ? 1 : 0);
-}
-
-
-/* Search MNT->mnt_opts for an option matching OPT.
-   Returns the address of the substring, or null if none found.  */
-char *
-glusterfs_fstab_hasoption (const struct glusterfs_mntent *mnt, const char *opt)
-{
-        const size_t optlen = strlen (opt);
-        char *rest = mnt->mnt_opts, *p;
-
-        while ((p = strstr (rest, opt)) != NULL)
-        {
-                if ((p == rest || p[-1] == ',')
-                                && (p[optlen] == '\0' || p[optlen] == '=' || p[optlen] == ','))
-                        return p;
-
-                rest = strchr (p, ',');
-                if (rest == NULL)
-                        break;
-                ++rest;
-        }
-
-        return NULL;
-}
-
-void
-clean_init_params (glusterfs_init_params_t *ipars)
-{
-        if (!ipars)
-                return;
-
-        if (ipars->volume_name)
-                free (ipars->volume_name);
-
-        if (ipars->specfile)
-                free (ipars->specfile);
-
-        if (ipars->logfile)
-                free (ipars->logfile);
-
-        if (ipars->loglevel)
-                free (ipars->loglevel);
-
-        return;
-}
-
-char *
-get_option_value (char *opt)
-{
-        char *val = NULL;
-        char *saveptr = NULL;
-        char *copy_opt = NULL;
-        char *retval = NULL;
-
-        copy_opt = strdup (opt);
-
-        /* Get the = before the value of the option. */
-        val = index (copy_opt, '=');
-        if (val) {
-                /* Move to start of option */
-                ++val;
-
-                /* Now, to create a '\0' delimited string out of the
-                 * options string, first get the position where the
-                 * next option starts, that would be the next ','.
-                 */
-                saveptr = index (val, ',');
-                if (saveptr)
-                        *saveptr = '\0';
-                retval = strdup (val);
-        }
-
-        free (copy_opt);
-
-        return retval;
-}
-
-void
-booster_mount (struct glusterfs_mntent *ent)
-{
-        char                    *opt = NULL;
-        glusterfs_init_params_t ipars;
-        time_t                  timeout = BOOSTER_DEFAULT_ATTR_TIMEO;
-        char                    *timeostr = NULL;
-        char                    *endptr = NULL;
-
-        if (!ent)
-                return;
-
-        gf_log ("booster-fstab", GF_LOG_DEBUG, "Mount entry: volfile: %s,"
-                " VMP: %s, Type: %s, Options: %s", ent->mnt_fsname,
-                ent->mnt_dir, ent->mnt_type, ent->mnt_opts);
-        if ((strcmp (ent->mnt_type, "glusterfs") != 0)) {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "Type is not glusterfs");
-                return;
-        }
-
-        memset (&ipars, 0, sizeof (glusterfs_init_params_t));
-        if (ent->mnt_fsname)
-                ipars.specfile = strdup (ent->mnt_fsname);
-
-        opt = glusterfs_fstab_hasoption (ent, "subvolume");
-        if (opt)
-                ipars.volume_name = get_option_value (opt);
-
-        opt = glusterfs_fstab_hasoption (ent, "log-file");
-        if (!opt)
-                opt = glusterfs_fstab_hasoption (ent, "logfile");
-
-        if (opt)
-                ipars.logfile = get_option_value (opt);
-
-        opt = glusterfs_fstab_hasoption (ent, "log-level");
-        if (!opt)
-                opt = glusterfs_fstab_hasoption (ent, "loglevel");
-
-        if (opt)
-                ipars.loglevel = get_option_value (opt);
-
-        /* Attribute cache timeout */
-        opt = glusterfs_fstab_hasoption (ent, "attr_timeout");
-        if (opt) {
-                 timeostr = get_option_value (opt);
-                 if (timeostr)
-                         timeout = strtol (timeostr, &endptr, 10);
-        }
-
-        ipars.lookup_timeout = timeout;
-        ipars.stat_timeout = timeout;
-
-        if ((glusterfs_mount (ent->mnt_dir, &ipars)) == -1)
-                gf_log ("booster-fstab", GF_LOG_ERROR, "VMP mounting failed");
-
-        clean_init_params (&ipars);
-}
-
-int
-booster_configure (char *confpath)
-{
-        int                     ret = -1;
-        glusterfs_fstab_t       *handle = NULL;
-        struct glusterfs_mntent *ent = NULL;
-
-        if (!confpath)
-                goto out;
-
-        handle = glusterfs_fstab_init (confpath, "r");
-        if (!handle)
-                goto out;
-
-        while ((ent = glusterfs_fstab_getent (handle)) != NULL)
-                booster_mount (ent);
-
-        glusterfs_fstab_close (handle);
-        ret = 0;
-out:
-        return ret;
-}
-
-
diff --git a/booster/src/booster_fstab.h b/booster/src/booster_fstab.h
deleted file mode 100644
index 9bab04c5a..000000000
--- a/booster/src/booster_fstab.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/* Utilities for reading/writing fstab, mtab, etc.
-   Copyright (C) 1995, 1996, 1997, 1998, 1999 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, write to the Free
-   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-   02111-1307 USA.  */
-
-#ifndef	GLUSTERFS_FSTAB_MNTENT_H
-#define	GLUSTERFS_FSTAB_MNTENT_H	1
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
- 
-#include "compat.h"
-
-/* General filesystem types.  */
-#define GF_MNTTYPE_IGNORE	"ignore"	/* Ignore this entry.  */
-#define GF_MNTTYPE_NFS	"nfs"		/* Network file system.  */
-#define GF_MNTTYPE_SWAP	"swap"		/* Swap device.  */
-
-
-/* Generic mount options.  */
-#define GF_MNTOPT_DEFAULTS	"defaults"	/* Use all default options.  */
-#define GF_MNTOPT_RO	        "ro"		/* Read only.  */
-#define GF_MNTOPT_RW	        "rw"		/* Read/write.  */
-#define GF_MNTOPT_SUID	        "suid"		/* Set uid allowed.  */
-#define GF_MNTOPT_NOSUID	"nosuid"	/* No set uid allowed.  */
-#define GF_MNTOPT_NOAUTO	"noauto"	/* Do not auto mount.  */
-
-
-/* Structure describing a mount table entry.  */
-struct glusterfs_mntent
-{
-        char *mnt_fsname;		/* Device or server for filesystem.  */
-        char *mnt_dir;		/* Directory mounted on.  */
-        char *mnt_type;		/* Type of filesystem: ufs, nfs, etc.  */
-        char *mnt_opts;		/* Comma-separated options for fs.  */
-        int mnt_freq;		/* Dump frequency (in days).  */
-        int mnt_passno;		/* Pass number for `fsck'.  */
-};
-
-#define GF_MNTENT_BUFSIZE       1024
-typedef struct glusterfs_fstab_handle {
-        FILE *fp;
-        char buf[GF_MNTENT_BUFSIZE];
-        struct glusterfs_mntent tmpent;
-}glusterfs_fstab_t;
-
-
-/* Prepare to begin reading and/or writing mount table entries from the
-   beginning of FILE.  MODE is as for `fopen'.  */
-extern glusterfs_fstab_t *glusterfs_fstab_init (const char *file,
-                const char *mode);
-
-extern struct glusterfs_mntent *glusterfs_fstab_getent (glusterfs_fstab_t *h);
-
-/* Write the mount table entry described by MNT to STREAM.
-   Return zero on success, nonzero on failure.  */
-extern int glusterfs_fstab_addent (glusterfs_fstab_t *h,
-                const struct glusterfs_mntent *mnt);
-
-/* Close a stream opened with `glusterfs_fstab_init'.  */
-extern int glusterfs_fstab_close (glusterfs_fstab_t *h);
-
-/* Search MNT->mnt_opts for an option matching OPT.
-   Returns the address of the substring, or null if none found.  */
-extern char *glusterfs_fstab_hasoption (const struct glusterfs_mntent *mnt,
-                const char *opt);
-
-#endif
diff --git a/booster/src/booster_stat.c b/booster/src/booster_stat.c
deleted file mode 100644
index 8f76cfe37..000000000
--- a/booster/src/booster_stat.c
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#include <sys/types.h>
-
-extern int
-booster_stat (const char *path, void *buf);
-
-extern int
-booster_stat64 (const char *path, void *buf);
-
-extern int
-booster_xstat (int ver, const char *path, void *buf);
-
-extern int
-booster_xstat64 (int ver, const char *path, void *buf);
-
-extern int
-booster_fxstat (int ver, int fd, void *buf);
-extern int
-booster_fxstat64 (int ver, int fd, void *buf);
-extern int
-booster_fstat (int fd, void *buf);
-extern int
-booster_fstat64 (int fd, void *buf);
-
-extern int
-booster_lstat (const char *path, void *buf);
-extern int
-booster_lstat64 (const char *path, void *buf);
-extern int
-booster_lxstat (int ver, const char *path, void *buf);
-extern int
-booster_lxstat64 (int ver, const char *path, void *buf);
-
-
-extern int
-booster_statfs (const char *path, void *buf);
-extern int
-booster_statfs64 (const char *path, void *buf);
-
-extern int
-booster_statvfs (const char *path, void *buf);
-
-extern int
-booster_statvfs64 (const char *path, void *buf);
-
-extern void *
-booster_readdir (void *dir);
-
-extern void
-booster_rewinddir (void *dir);
-
-extern void
-booster_seekdir (void *dir, off_t offset);
-
-extern off_t
-booster_telldir (void *dir);
-
-int
-stat (const char *path, void *buf)
-{
-        return booster_stat (path, buf);
-}
-
-int
-stat64 (const char *path, void *buf)
-{
-        return booster_stat64 (path, buf);
-}
-
-int
-__xstat (int ver, const char *path, void *buf)
-{
-        return booster_xstat (ver, path, buf);
-}
-
-int
-__xstat64 (int ver, const char *path, void *buf)
-{
-        return booster_xstat64 (ver, path, buf);
-}
-
-int
-__fxstat (int ver, int fd, void *buf)
-{
-        return booster_fxstat (ver, fd, buf);
-}
-
-int
-__fxstat64 (int ver, int fd, void *buf)
-{
-        return booster_fxstat64 (ver, fd, buf);
-}
-
-int
-fstat (int fd, void *buf)
-{
-        return booster_fstat (fd, buf);
-}
-
-int
-fstat64 (int fd, void *buf)
-{
-        return booster_fstat64 (fd, buf);
-}
-
-int
-lstat (const char *path, void *buf)
-{
-        return booster_lstat (path, buf);
-}
-
-int
-lstat64 (const char *path, void *buf)
-{
-        return booster_lstat64 (path, buf);
-}
-
-int
-__lxstat (int ver, const char *path, void *buf)
-{
-        return booster_lxstat (ver, path, buf);
-}
-
-int
-__lxstat64 (int ver, const char *path, void *buf)
-{
-        return booster_lxstat64 (ver, path, buf);
-}
-
-int
-statfs (const char *pathname, void *buf)
-{
-        return booster_statfs (pathname, buf);
-}
-
-int
-statfs64 (const char *pathname, void *buf)
-{
-        return booster_statfs64 (pathname, buf);
-}
-
-int
-statvfs (const char *pathname, void *buf)
-{
-        return booster_statvfs (pathname, buf);
-}
-
-int
-statvfs64 (const char *pathname, void *buf)
-{
-        return booster_statvfs64 (pathname, buf);
-}
-
-void
-rewinddir (void *dir)
-{
-	return booster_rewinddir (dir);
-}
-
-void
-seekdir (void *dir, off_t offset)
-{
-	return booster_seekdir (dir, offset);
-}
-
-off_t
-telldir (void *dir)
-{
-	return booster_telldir (dir);
-}
diff --git a/cli/src/Makefile.am b/cli/src/Makefile.am
index b76a9efd3..216d1bb55 100644
--- a/cli/src/Makefile.am
+++ b/cli/src/Makefile.am
@@ -2,23 +2,26 @@ sbin_PROGRAMS = gluster
 
 gluster_SOURCES = cli.c registry.c input.c cli-cmd.c cli-rl.c \
 	 cli-cmd-volume.c cli-cmd-peer.c cli-rpc-ops.c cli-cmd-parser.c\
-	 cli-cmd-system.c cli-cmd-misc.c
+	 cli-cmd-system.c cli-cmd-misc.c cli-xml-output.c cli-cmd-snapshot.c
 
 gluster_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD)\
 		$(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+		$(GF_GLUSTERFS_LIBS) $(XML_LIBS)
 
-gluster_LDFLAGS = $(GF_LDFLAGS) $(GF_GLUSTERFS_LDFLAGS)
+gluster_LDFLAGS = $(GF_LDFLAGS)
 noinst_HEADERS = cli.h cli-mem-types.h cli-cmd.h
 
-AM_CFLAGS = -fPIC -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src\
 	-I$(top_srcdir)/rpc/xdr/src\
 	-DDATADIR=\"$(localstatedir)\" \
-	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_GLUSTERFS_CFLAGS)\
+	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
 	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
-	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) -DSBIN_DIR=\"$(sbindir)\"
+	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) -DSBIN_DIR=\"$(sbindir)\"\
+	$(XML_CPPFLAGS)
 
+AM_CFLAGS = -Wall $(GF_GLUSTERFS_CFLAGS)
 
 CLEANFILES =
 
diff --git a/cli/src/cli-cmd-misc.c b/cli/src/cli-cmd-misc.c
index 40f419cde..566d7c978 100644
--- a/cli/src/cli-cmd-misc.c
+++ b/cli/src/cli-cmd-misc.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -41,6 +31,8 @@ extern struct cli_cmd volume_cmds[];
 extern struct cli_cmd cli_probe_cmds[];
 extern struct cli_cmd cli_log_cmds[];
 extern struct cli_cmd cli_system_cmds[];
+extern struct cli_cmd cli_bd_cmds[];
+extern struct cli_cmd snapshot_cmds[];
 struct cli_cmd cli_misc_cmds[];
 
 int
@@ -55,7 +47,8 @@ cli_cmd_display_help (struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount)
 {
         struct cli_cmd        *cmd[] = {volume_cmds, cli_probe_cmds,
-                                       cli_misc_cmds, NULL};
+                                       cli_misc_cmds, snapshot_cmds,
+                                       NULL};
         struct cli_cmd        *cmd_ind = NULL;
         int                   i = 0;
 
@@ -80,6 +73,10 @@ struct cli_cmd cli_misc_cmds[] = {
            cli_cmd_display_help,
            "display command options"},
 
+        { "exit",
+          cli_cmd_quit_cbk,
+          "exit"},
+
         { NULL, NULL, NULL }
 };
 
diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index b169b77c6..5ab208b8f 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -36,6 +26,38 @@
 #include "protocol-common.h"
 #include "cli1-xdr.h"
 
+#define MAX_SNAP_DESCRIPTION_LEN 1024
+
+struct snap_config_opt_vals_ snap_confopt_vals[] = {
+        {.op_name        = "snap-max-hard-limit",
+         .question       = "Changing snapshot-max-hard-limit "
+                           "will lead to deletion of snapshots "
+                           "if they exceed the new limit.\n"
+                           "Do you want to continue?"
+        },
+        {.op_name        = "snap-max-soft-limit",
+         .question       = "Changing snapshot-max-soft-limit "
+                           "will lead to deletion of snapshots "
+                           "if they exceed the new limit.\n"
+                           "Do you want to continue?"
+        },
+        {.op_name        = "both",
+        .question        = "Changing snapshot-max-hard-limit & "
+                           "snapshot-max-soft-limit will lead to "
+                           "deletion of snapshots if they exceed "
+                           "the new limit.\nDo you want to continue?"
+        },
+        {.op_name        = NULL,
+        }
+};
+
+enum cli_snap_config_set_types {
+        GF_SNAP_CONFIG_SET_HARD = 0,
+        GF_SNAP_CONFIG_SET_SOFT = 1,
+        GF_SNAP_CONFIG_SET_BOTH = 2,
+};
+typedef enum cli_snap_config_set_types cli_snap_config_set_types;
+
 static const char *
 id_sel (void *wcon)
 {
@@ -74,21 +96,20 @@ cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
         brick_list_len++;
         while (brick_index < wordcount) {
                 if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_out ("Wrong brick type: %s, use <HOSTNAME>:"
+                        cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
                                  "<export-dir-abs-path>", words[brick_index]);
                         ret = -1;
                         goto out;
                 } else {
                         delimiter = strrchr (words[brick_index], ':');
-                        ret = cli_canonicalize_path (delimiter + 1);
+                        ret = gf_canonicalize_path (delimiter + 1);
                         if (ret)
                                 goto out;
                 }
 
                 if ((brick_list_len + strlen (words[brick_index]) + 1) > sizeof (brick_list)) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "total brick list is larger than a request "
-                                "can take (brick_count %d)", *brick_count);
+                        cli_err ("Total brick list is larger than a request. "
+                                 "Can take (brick_count %d)", *brick_count);
                         ret = -1;
                         goto out;
                 }
@@ -108,16 +129,18 @@ cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
                 }
 
                 if (!(strcmp (host_name, "localhost") &&
-                      strcmp (host_name, "127.0.0.1"))) {
-                        cli_out ("Please provide a valid hostname/ip other "
-                                 "than localhost or 127.0.0.1");
+                      strcmp (host_name, "127.0.0.1") &&
+                      strncmp (host_name, "0.", 2))) {
+                        cli_err ("Please provide a valid hostname/ip other "
+                                 "than localhost, 127.0.0.1 or loopback "
+                                 "address (0.0.0.0 to 0.255.255.255).");
                         ret = -1;
                         GF_FREE (tmp_host);
                         goto out;
                 }
-                if (!valid_internet_address (host_name)) {
-                        cli_out ("internet address '%s' does not comform to "
-			         "standards", host_name);
+                if (!valid_internet_address (host_name, _gf_false)) {
+                        cli_err ("internet address '%s' does not conform to "
+                                 "standards", host_name);
                 }
                 GF_FREE (tmp_host);
                 tmp_list = gf_strdup (brick_list + 1);
@@ -131,7 +154,7 @@ cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
                         strtok_r (tmp_list, " ", &tmpptr);
                         if (!(strcmp (tmp_list, words[brick_index]))) {
                                 ret = -1;
-                                cli_out ("Found duplicate"
+                                cli_err ("Found duplicate"
                                          " exports %s",words[brick_index]);
                                 goto out;
                        }
@@ -149,8 +172,7 @@ cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
         if (!*bricks)
                 ret = -1;
 out:
-        if (free_list_ptr)
-                GF_FREE (free_list_ptr);
+        GF_FREE (free_list_ptr);
         return ret;
 }
 
@@ -170,10 +192,18 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
         char    *bricks = NULL;
         int32_t brick_count = 0;
         char    *opwords[] = { "replica", "stripe", "transport", NULL };
+
+        char    *invalid_volnames[] = {"volume", "type", "subvolumes", "option",
+                                       "end-volume", "all", "volume_not_in_ring",
+                                       "description", "force",
+                                       "snap-max-hard-limit",
+                                       "snap-max-soft-limit", NULL};
         char    *w = NULL;
         int      op_count = 0;
         int32_t  replica_count = 1;
         int32_t  stripe_count = 1;
+        gf_boolean_t is_force = _gf_false;
+        int wc = wordcount;
 
         GF_ASSERT (words);
         GF_ASSERT (options);
@@ -195,9 +225,12 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                 if (volname[0] == '-')
                         goto out;
 
-                if (!strcmp (volname, "all")) {
-                        cli_out ("\"all\" cannot be the name of a volume.");
-                        goto out;
+                for (i = 0; invalid_volnames[i]; i++) {
+                        if (!strcmp (volname, invalid_volnames[i])) {
+                                cli_err ("\"%s\" cannot be the name of a volume.",
+                                         volname);
+                                goto out;
+                        }
                 }
 
                 if (strchr (volname, '/'))
@@ -228,7 +261,7 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                         switch (type) {
                         case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
                         case GF_CLUSTER_TYPE_REPLICATE:
-                                cli_out ("replica option given twice");
+                                cli_err ("replica option given twice");
                                 goto out;
                         case GF_CLUSTER_TYPE_NONE:
                                 type = GF_CLUSTER_TYPE_REPLICATE;
@@ -244,7 +277,8 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                         }
                         replica_count = strtol (words[index+1], NULL, 0);
                         if (replica_count < 2) {
-                                cli_out ("replica count should be greater than 1");
+                                cli_err ("replica count should be greater"
+                                         " than 1");
                                 ret = -1;
                                 goto out;
                         }
@@ -258,7 +292,7 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                         switch (type) {
                         case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
                         case GF_CLUSTER_TYPE_STRIPE:
-                                cli_out ("stripe option given twice");
+                                cli_err ("stripe option given twice");
                                 goto out;
                         case GF_CLUSTER_TYPE_NONE:
                                 type = GF_CLUSTER_TYPE_STRIPE;
@@ -273,7 +307,8 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                         }
                         stripe_count = strtol (words[index+1], NULL, 0);
                         if (stripe_count < 2) {
-                                cli_out ("stripe count should be greater than 1");
+                                cli_err ("stripe count should be greater"
+                                         " than 1");
                                 ret = -1;
                                 goto out;
                         }
@@ -285,7 +320,7 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
 
                 } else if ((strcmp (w, "transport")) == 0) {
                         if (trans_type) {
-                                cli_out ("'transport' option given more"
+                                cli_err ("'transport' option given more"
                                          " than one time");
                                 goto out;
                         }
@@ -303,7 +338,7 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                                 goto out;
                         }
                         index += 2;
-                } else {
+                }              else {
                         GF_ASSERT (!"opword mismatch");
                         ret = -1;
                         goto out;
@@ -326,7 +361,12 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
 
         brick_index = index;
 
-        ret = cli_cmd_bricks_parse (words, wordcount, brick_index, &bricks,
+        if (strcmp (words[wordcount - 1], "force") == 0) {
+                is_force = _gf_true;
+                wc = wordcount - 1;
+        }
+
+        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
                                     &brick_count);
         if (ret)
                 goto out;
@@ -334,20 +374,20 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
         /* If brick-count is not valid when replica or stripe is
            given, exit here */
         if (!brick_count) {
-                cli_out ("No bricks specified");
+                cli_err ("No bricks specified");
                 ret = -1;
                 goto out;
         }
 
         if (brick_count % sub_count) {
                 if (type == GF_CLUSTER_TYPE_STRIPE)
-                        cli_out ("number of bricks is not a multiple of "
+                        cli_err ("number of bricks is not a multiple of "
                                  "stripe count");
                 else if (type == GF_CLUSTER_TYPE_REPLICATE)
-                        cli_out ("number of bricks is not a multiple of "
+                        cli_err ("number of bricks is not a multiple of "
                                  "replica count");
                 else
-                        cli_out ("number of bricks given doesn't match "
+                        cli_err ("number of bricks given doesn't match "
                                  "required count");
 
                 ret = -1;
@@ -366,7 +406,7 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
         ret = dict_set_dynstr (dict, "transport", trans_type);
         if (ret)
                 goto out;
-
+        trans_type = NULL;
 
         ret = dict_set_dynstr (dict, "bricks", bricks);
         if (ret)
@@ -376,6 +416,10 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
         if (ret)
                 goto out;
 
+        ret = dict_set_int32 (dict, "force", is_force);
+        if (ret)
+                goto out;
+
         *options = dict;
 
 out:
@@ -384,6 +428,9 @@ out:
                 if (dict)
                         dict_destroy (dict);
         }
+
+        GF_FREE (trans_type);
+
         return ret;
 }
 
@@ -497,7 +544,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
                         goto out;
 
                 if (!strcmp (volname, "all")) {
-                        cli_out ("\"all\" cannot be the name of a volume.");
+                        cli_err ("\"all\" cannot be the name of a volume.");
                         goto out;
                 }
 
@@ -522,16 +569,26 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
                 goto out;
         }
 
-        if ((strcmp (w, "enable")) == 0 && wordcount == 4) {
-                type = GF_QUOTA_OPTION_TYPE_ENABLE;
-                ret = 0;
-                goto set_type;
+        if (strcmp (w, "enable") == 0) {
+                if (wordcount == 4) {
+                        type = GF_QUOTA_OPTION_TYPE_ENABLE;
+                        ret = 0;
+                        goto set_type;
+                } else {
+                        ret = -1;
+                        goto out;
+                }
         }
 
-        if (strcmp (w, "disable") == 0 && wordcount == 4) {
-                type = GF_QUOTA_OPTION_TYPE_DISABLE;
-                ret = 0;
-                goto set_type;
+        if (strcmp (w, "disable") == 0) {
+                if (wordcount == 4) {
+                        type = GF_QUOTA_OPTION_TYPE_DISABLE;
+                        ret = 0;
+                        goto set_type;
+                } else {
+                        ret = -1;
+                        goto out;
+                }
         }
 
         if (strcmp (w, "limit-usage") == 0) {
@@ -543,7 +600,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
                 type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
 
                 if (words[4][0] != '/') {
-                        cli_out ("Please enter absolute path");
+                        cli_err ("Please enter absolute path");
 
                         return -2;
                 }
@@ -552,14 +609,14 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
                         goto out;
 
                 if (!words[5]) {
-                        cli_out ("Please enter the limit value to be set");
+                        cli_err ("Please enter the limit value to be set");
 
                         return -2;
                 }
 
                 ret = gf_string2bytesize (words[5], &value);
                 if (ret != 0) {
-                        cli_out ("Please enter a correct value");
+                        cli_err ("Please enter a correct value");
                         return -1;
                 }
 
@@ -578,7 +635,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
                 type = GF_QUOTA_OPTION_TYPE_REMOVE;
 
                 if (words[4][0] != '/') {
-                        cli_out ("Please enter absolute path");
+                        cli_err ("Please enter absolute path");
 
                         return -2;
                 }
@@ -634,54 +691,117 @@ out:
         return ret;
 }
 
-int32_t
-cli_cmd_valid_ip_list (char *iplist)
+static inline gf_boolean_t
+cli_is_key_spl (char *key)
 {
-        int     ret = 0;
-        char    *duplist = NULL;
-        char    *addr = NULL;
-        char    *saveptr = NULL;
-
-        GF_ASSERT (iplist);
-        duplist = gf_strdup (iplist);
+        return (strcmp (key, "group") == 0);
+}
 
-        if (!duplist) {
-                ret = -1;
+#define GLUSTERD_DEFAULT_WORKDIR "/var/lib/glusterd"
+static int
+cli_add_key_group (dict_t *dict, char *key, char *value, char **op_errstr)
+{
+        int             ret = -1;
+        int             opt_count = 0;
+        char            iter_key[1024] = {0,};
+        char            iter_val[1024] = {0,};
+        char            *saveptr = NULL;
+        char            *tok_key = NULL;
+        char            *tok_val = NULL;
+        char            *dkey = NULL;
+        char            *dval = NULL;
+        char            *tagpath = NULL;
+        char            *buf = NULL;
+        char            line[PATH_MAX + 256] = {0,};
+        char            errstr[2048] = "";
+        FILE            *fp = NULL;
+
+        ret = gf_asprintf (&tagpath, "%s/groups/%s",
+                           GLUSTERD_DEFAULT_WORKDIR, value);
+        if (ret == -1) {
+                tagpath = NULL;
                 goto out;
         }
 
-        addr = strtok_r (duplist, ",", &saveptr);
-        if (!addr) {
+        fp = fopen (tagpath, "r");
+        if (!fp) {
                 ret = -1;
+                snprintf(errstr, sizeof(errstr), "Unable to open file '%s'."
+                         " Error: %s", tagpath, strerror (errno));
+                if (op_errstr)
+                        *op_errstr = gf_strdup(errstr);
                 goto out;
         }
-        while (addr) {
-                if (!valid_internet_address (addr) &&
-                    !valid_wildcard_internet_address (addr)) {
-                        cli_out ("Invalid ip or wildcard : %s", addr);
-                        ret= -1;
+
+        opt_count = 0;
+        buf = line;
+        while (fscanf (fp, "%s", buf) != EOF) {
+
+                opt_count++;
+                tok_key = strtok_r (line, "=", &saveptr);
+                tok_val = strtok_r (NULL, "=", &saveptr);
+                if (!tok_key || !tok_val) {
+                        ret = -1;
+                        snprintf(errstr, sizeof(errstr), "'%s' file format "
+                                 "not valid.", tagpath);
+                        if (op_errstr)
+                                *op_errstr = gf_strdup(errstr);
                         goto out;
                 }
-                addr = strtok_r (NULL, ",", &saveptr);
+
+                snprintf (iter_key, sizeof (iter_key), "key%d", opt_count);
+                dkey = gf_strdup (tok_key);
+                ret = dict_set_dynstr (dict, iter_key, dkey);
+                if (ret)
+                        goto out;
+                dkey = NULL;
+
+                snprintf (iter_val, sizeof (iter_val), "value%d", opt_count);
+                dval = gf_strdup (tok_val);
+                ret = dict_set_dynstr (dict, iter_val, dval);
+                if (ret)
+                        goto out;
+                dval = NULL;
+
+        }
+
+        if (!opt_count) {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr), "'%s' file format "
+                         "not valid.", tagpath);
+                if (op_errstr)
+                        *op_errstr = gf_strdup(errstr);
+                goto out;
         }
+        ret = dict_set_int32 (dict, "count", opt_count);
 out:
-        if (duplist)
-                GF_FREE (duplist);
-        gf_log ("cli", GF_LOG_INFO, "Returning %d", ret);
+
+        GF_FREE (tagpath);
+
+        if (ret) {
+                GF_FREE (dkey);
+                GF_FREE (dval);
+        }
+
+        if (fp)
+                fclose (fp);
+
         return ret;
 }
+#undef GLUSTERD_DEFAULT_WORKDIR
 
 int32_t
-cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options,
+                          char **op_errstr)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        int     count = 0;
-        char    *key = NULL;
-        char    *value = NULL;
-        int     i = 0;
-        char    str[50] = {0,};
+        dict_t                  *dict = NULL;
+        char                    *volname = NULL;
+        int                     ret = -1;
+        int                     count = 0;
+        char                    *key = NULL;
+        char                    *value = NULL;
+        int                     i = 0;
+        char                    str[50] = {0,};
 
         GF_ASSERT (words);
         GF_ASSERT (options);
@@ -703,42 +823,65 @@ cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options)
         if (ret)
                 goto out;
 
-        if (wordcount == 3) {
-                if (!strcmp (volname, "help")) {
-                        ret = dict_set_str (dict, "help", volname);
-                        if (ret)
-                                goto out;
-                } else if (!strcmp (volname, "help-xml")) {
-                        ret = dict_set_str (dict, "help-xml", volname);
-                        if (ret)
-                                goto out;
-                } else {
+        if ((!strcmp (volname, "help") || !strcmp (volname, "help-xml"))
+            && wordcount == 3 ) {
+                ret = dict_set_str (dict, volname, volname);
+                if (ret)
+                        goto out;
+
+        } else if (wordcount < 5) {
+                ret = -1;
+                goto out;
+
+        } else if (wordcount == 5  && cli_is_key_spl ((char *)words[3])) {
+                key = (char *) words[3];
+                value = (char *) words[4];
+                if ( !key || !value) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = gf_strip_whitespace (value, strlen (value));
+                if (ret == -1)
+                        goto out;
+
+                if (strlen (value) == 0) {
                         ret = -1;
                         goto out;
                 }
-        }
 
+                ret = cli_add_key_group (dict, key, value, op_errstr);
+                if (ret == 0)
+                        *options = dict;
+                goto out;
+        }
 
         for (i = 3; i < wordcount; i+=2) {
 
-		key = (char *) words[i];
-		value = (char *) words[i+1];
+                key = (char *) words[i];
+                value = (char *) words[i+1];
 
-		if ( !key || !value) {
-			ret = -1;
-			goto out;
-	        }
+                if ( !key || !value) {
+                        ret = -1;
+                        goto out;
+                }
 
                 count++;
-                if (!strncmp ("auth.allow", key, sizeof (key)) ||
-                    !strncmp ("auth.reject", key, sizeof (key))) {
-                        ret = cli_cmd_valid_ip_list (value);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "invalid ips given");
-                                goto out;
-                        }
+
+                ret = gf_strip_whitespace (value, strlen (value));
+                if (ret == -1)
+                        goto out;
+
+                if (strlen (value) == 0) {
+                        ret = -1;
+                        goto out;
+                }
+
+                if (cli_is_key_spl (key)) {
+                        ret = -1;
+                        goto out;
                 }
+
                 sprintf (str, "key%d", count);
                 ret = dict_set_str (dict, str, key);
                 if (ret)
@@ -759,10 +902,8 @@ cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options)
         *options = dict;
 
 out:
-        if (ret) {
-                if (dict)
-                        dict_destroy (dict);
-        }
+        if (ret)
+                dict_destroy (dict);
 
         return ret;
 }
@@ -781,6 +922,8 @@ cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
         int     count = 1;
         char    *w = NULL;
         int     index;
+        gf_boolean_t is_force = _gf_false;
+        int wc = wordcount;
 
         GF_ASSERT (words);
         GF_ASSERT (options);
@@ -825,7 +968,7 @@ cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
                 }
                 count = strtol (words[4], NULL, 0);
                 if (!count || (count < 2)) {
-                        cli_out ("replica count should be greater than 1");
+                        cli_err ("replica count should be greater than 1");
                         ret = -1;
                         goto out;
                 }
@@ -841,7 +984,7 @@ cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
                 }
                 count = strtol (words[4], NULL, 0);
                 if (!count || (count < 2)) {
-                        cli_out ("stripe count should be greater than 1");
+                        cli_err ("stripe count should be greater than 1");
                         ret = -1;
                         goto out;
                 }
@@ -858,7 +1001,13 @@ cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
         brick_index = index;
 
 parse_bricks:
-        ret = cli_cmd_bricks_parse (words, wordcount, brick_index, &bricks,
+
+        if (strcmp (words[wordcount - 1], "force") == 0) {
+                is_force = _gf_true;
+                wc = wordcount - 1;
+        }
+
+        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
                                     &brick_count);
         if (ret)
                 goto out;
@@ -872,6 +1021,10 @@ parse_bricks:
         if (ret)
                 goto out;
 
+        ret = dict_set_int32 (dict, "force", is_force);
+        if (ret)
+                goto out;
+
         *options = dict;
 
 out:
@@ -900,7 +1053,7 @@ cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
         char    *tmp_brick = NULL;
         char    *tmp_brick1 = NULL;
         char    *type_opword[] = { "replica", NULL };
-        char    *opwords[] = { "start", "commit", "pause", "abort", "status",
+        char    *opwords[] = { "start", "commit", "stop", "status",
                                "force", NULL };
         char    *w = NULL;
         int32_t  command = GF_OP_CMD_NONE;
@@ -933,7 +1086,7 @@ cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
                 }
                 count = strtol (words[4], NULL, 0);
                 if (count < 1) {
-                        cli_out ("replica count should be greater than 0 in "
+                        cli_err ("replica count should be greater than 0 in "
                                  "case of remove-brick");
                         ret = -1;
                         goto out;
@@ -962,10 +1115,8 @@ cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
                         command = GF_OP_CMD_COMMIT;
                         if (question)
                                 *question = 1;
-                } else if (!strcmp ("pause", w)) {
-                        command = GF_OP_CMD_PAUSE;
-                } else if (!strcmp ("abort", w)) {
-                        command = GF_OP_CMD_ABORT;
+                } else if (!strcmp ("stop", w)) {
+                        command = GF_OP_CMD_STOP;
                 } else if (!strcmp ("status", w)) {
                         command = GF_OP_CMD_STATUS;
                 } else if (!strcmp ("force", w)) {
@@ -1011,13 +1162,13 @@ cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
 
         while (brick_index < wordcount) {
                 if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
+                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
                                  "<export-dir-abs-path>", words[brick_index]);
                         ret = -1;
                         goto out;
                 } else {
                         delimiter = strrchr(words[brick_index], ':');
-                        ret = cli_canonicalize_path (delimiter + 1);
+                        ret = gf_canonicalize_path (delimiter + 1);
                         if (ret)
                                 goto out;
                 }
@@ -1029,7 +1180,7 @@ cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
                         if (!(strcmp (tmp_brick, tmp_brick1))) {
                                 gf_log("",GF_LOG_ERROR, "Duplicate bricks"
                                        " found %s", words[brick_index]);
-                                cli_out("Duplicate bricks found %s",
+                                cli_err("Duplicate bricks found %s",
                                         words[brick_index]);
                                 ret = -1;
                                 goto out;
@@ -1056,10 +1207,8 @@ out:
                         dict_destroy (dict);
         }
 
-        if (tmp_brick)
-                GF_FREE (tmp_brick);
-        if (tmp_brick1)
-                GF_FREE (tmp_brick1);
+        GF_FREE (tmp_brick);
+        GF_FREE (tmp_brick1);
 
         return ret;
 }
@@ -1078,6 +1227,7 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         char    *opwords[] = { "start", "commit", "pause", "abort", "status",
                                 NULL };
         char    *w = NULL;
+        gf_boolean_t is_force = _gf_false;
 
         GF_ASSERT (words);
         GF_ASSERT (options);
@@ -1105,13 +1255,13 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         }
 
         if (validate_brick_name ((char *)words[3])) {
-                cli_out ("wrong brick type: %s, use "
+                cli_err ("wrong brick type: %s, use "
                          "<HOSTNAME>:<export-dir-abs-path>", words[3]);
                 ret = -1;
                 goto out;
         } else {
                 delimiter = strrchr ((char *)words[3], ':');
-                ret = cli_canonicalize_path (delimiter + 1);
+                ret = gf_canonicalize_path (delimiter + 1);
                 if (ret)
                         goto out;
         }
@@ -1126,13 +1276,13 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         }
 
         if (validate_brick_name ((char *)words[4])) {
-                cli_out ("wrong brick type: %s, use "
+                cli_err ("wrong brick type: %s, use "
                          "<HOSTNAME>:<export-dir-abs-path>", words[4]);
                 ret = -1;
                 goto out;
         } else {
                 delimiter = strrchr ((char *)words[4], ':');
-                ret = cli_canonicalize_path (delimiter + 1);
+                ret = gf_canonicalize_path (delimiter + 1);
                 if (ret)
                         goto out;
         }
@@ -1175,12 +1325,17 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         }
 
         if (wordcount == (op_index + 1)) {
-                if (replace_op != GF_REPLACE_OP_COMMIT) {
+                if ((replace_op != GF_REPLACE_OP_COMMIT) &&
+                    (replace_op != GF_REPLACE_OP_START)) {
                         ret = -1;
                         goto out;
                 }
                 if (!strcmp ("force", words[op_index])) {
-                        replace_op = GF_REPLACE_OP_COMMIT_FORCE;
+                        if (replace_op == GF_REPLACE_OP_COMMIT)
+                                replace_op = GF_REPLACE_OP_COMMIT_FORCE;
+
+                        else if (replace_op == GF_REPLACE_OP_START)
+                                is_force = _gf_true;
                 }
         }
 
@@ -1194,6 +1349,9 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         if (ret)
                 goto out;
 
+        ret = dict_set_int32 (dict, "force", is_force);
+        if (ret)
+                goto out;
 
         *options = dict;
 
@@ -1235,12 +1393,12 @@ cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options)
                 delimiter = strchr (words[4], ':');
                 if (!delimiter || delimiter == words[4]
                     || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
+                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
                                  "<export-dir-abs-path>", words[4]);
                         ret = -1;
                         goto out;
                 } else {
-                        ret = cli_canonicalize_path (delimiter + 1);
+                        ret = gf_canonicalize_path (delimiter + 1);
                         if (ret)
                                 goto out;
                 }
@@ -1290,8 +1448,8 @@ cli_cmd_log_level_parse (const char **words, int worcount, dict_t **options)
 
         ret = glusterd_check_log_level(words[5]);
         if (ret == -1) {
-                cli_out("Invalid log level [%s] specified", words[5]);
-                cli_out("Valid values for loglevel: (DEBUG|WARNING|ERROR"
+                cli_err("Invalid log level [%s] specified", words[5]);
+                cli_err("Valid values for loglevel: (DEBUG|WARNING|ERROR"
                         "|CRITICAL|NONE|TRACE)");
                 goto out;
         }
@@ -1351,12 +1509,12 @@ cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options)
                 delimiter = strchr (words[4], ':');
                 if (!delimiter || delimiter == words[4]
                     || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
+                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
                                  "<export-dir-abs-path>", words[4]);
                         ret = -1;
                         goto out;
                 } else {
-                        ret = cli_canonicalize_path (delimiter + 1);
+                        ret = gf_canonicalize_path (delimiter + 1);
                         if (ret)
                                 goto out;
                 }
@@ -1402,12 +1560,12 @@ cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options)
                 delimiter = strchr (words[4], ':');
                 if (!delimiter || delimiter == words[4]
                     || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
+                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
                                  "<export-dir-abs-path>", words[4]);
                         ret = -1;
                         goto out;
                 } else {
-                        ret = cli_canonicalize_path (delimiter + 1);
+                        ret = gf_canonicalize_path (delimiter + 1);
                         if (ret)
                                 goto out;
                 }
@@ -1438,22 +1596,161 @@ gsyncd_glob_check (const char *w)
         return !!strpbrk (w, "*?[");
 }
 
+static int
+config_parse (const char **words, int wordcount, dict_t *dict,
+              unsigned cmdi, unsigned glob)
+{
+        int32_t            ret     = -1;
+        int32_t            i       = -1;
+        char               *append_str = NULL;
+        size_t             append_len = 0;
+        char               *subop = NULL;
+
+        switch ((wordcount - 1) - cmdi) {
+        case 0:
+                subop = gf_strdup ("get-all");
+                break;
+        case 1:
+                if (words[cmdi + 1][0] == '!') {
+                        (words[cmdi + 1])++;
+                        if (gf_asprintf (&subop, "del%s",
+                                         glob ? "-glob" : "") == -1)
+                                subop = NULL;
+                } else
+                        subop = gf_strdup ("get");
+
+                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
+                if (ret < 0)
+                        goto out;
+                break;
+        default:
+                if (gf_asprintf (&subop, "set%s", glob ? "-glob" : "") == -1)
+                        subop = NULL;
+
+                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
+                if (ret < 0)
+                        goto out;
+
+                /* join the varargs by spaces to get the op_value */
+
+                for (i = cmdi + 2; i < wordcount; i++)
+                        append_len += (strlen (words[i]) + 1);
+                /* trailing strcat will add two bytes, make space for that */
+                append_len++;
+
+                append_str = GF_CALLOC (1, append_len, cli_mt_append_str);
+                if (!append_str) {
+                        ret = -1;
+                        goto out;
+                }
+
+                for (i = cmdi + 2; i < wordcount; i++) {
+                        strcat (append_str, words[i]);
+                        strcat (append_str, " ");
+                }
+                append_str[append_len - 2] = '\0';
+                /* "checkpoint now" is special: we resolve that "now" */
+                if (strcmp (words[cmdi + 1], "checkpoint") == 0 &&
+                    strcmp (append_str, "now") == 0) {
+                        struct timeval tv = {0,};
+
+                        ret = gettimeofday (&tv, NULL);
+                        if (ret == -1)
+                                goto out; /* FIXME: free append_str? */
+
+                        GF_FREE (append_str);
+                        append_str = GF_CALLOC (1, 300, cli_mt_append_str);
+                        if (!append_str) {
+                                ret = -1;
+                                goto out;
+                        }
+                        strcpy (append_str, "as of ");
+                        gf_time_fmt (append_str + strlen ("as of "),
+                                     300 - strlen ("as of "),
+                                     tv.tv_sec, gf_timefmt_FT);
+                }
+
+                ret = dict_set_dynstr (dict, "op_value", append_str);
+        }
+
+        ret = -1;
+        if (subop) {
+                ret = dict_set_dynstr (dict, "subop", subop);
+                if (!ret)
+                      subop = NULL;
+        }
+
+out:
+        if (ret && append_str)
+                GF_FREE (append_str);
+
+        GF_FREE (subop);
+
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int32_t
+force_push_pem_parse (const char **words, int wordcount,
+                      dict_t *dict, unsigned *cmdi)
+{
+        int32_t            ret     = 0;
+
+        if (!strcmp ((char *)words[wordcount-1], "force")) {
+                if ((strcmp ((char *)words[wordcount-2], "start")) &&
+                    (strcmp ((char *)words[wordcount-2], "stop")) &&
+                    (strcmp ((char *)words[wordcount-2], "create")) &&
+                    (strcmp ((char *)words[wordcount-2], "push-pem"))) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_uint32 (dict, "force",
+                                       _gf_true);
+                if (ret)
+                        goto out;
+                (*cmdi)++;
+
+                if (!strcmp ((char *)words[wordcount-2], "push-pem")) {
+                        if (strcmp ((char *)words[wordcount-3], "create")) {
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = dict_set_int32 (dict, "push_pem", 1);
+                        if (ret)
+                                goto out;
+                        (*cmdi)++;
+                }
+        } else if (!strcmp ((char *)words[wordcount-1], "push-pem")) {
+                if (strcmp ((char *)words[wordcount-2], "create")) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_int32 (dict, "push_pem", 1);
+                if (ret)
+                        goto out;
+                (*cmdi)++;
+        }
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
 int32_t
 cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
 {
         int32_t            ret     = -1;
         dict_t             *dict   = NULL;
         gf1_cli_gsync_set  type    = GF_GSYNC_OPTION_TYPE_NONE;
-        char               *append_str = NULL;
-        size_t             append_len = 0;
-        char               *subop = NULL;
         int                i       = 0;
         unsigned           masteri = 0;
         unsigned           slavei  = 0;
         unsigned           glob    = 0;
         unsigned           cmdi    = 0;
-        char               *opwords[] = { "status", "start", "stop", "config",
-                                          "log-rotate", NULL };
+        char               *opwords[] = { "create", "status", "start", "stop",
+                                          "config", "force", "delete",
+                                          "push-pem", "detail", NULL };
         char               *w = NULL;
 
         GF_ASSERT (words);
@@ -1465,10 +1762,11 @@ cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
 
         /* new syntax:
          *
-         * volume geo-replication [$m [$s]] status
+         * volume geo-replication $m $s create [push-pem] [force]
+         * volume geo-replication [$m [$s]] status [detail]
          * volume geo-replication [$m] $s config [[!]$opt [$val]]
-         * volume geo-replication $m $s start|stop
-         * volume geo-replication $m [$s] log-rotate
+         * volume geo-replication $m $s start|stop [force]
+         * volume geo-replication $m $s delete
          */
 
         if (wordcount < 3)
@@ -1499,6 +1797,13 @@ cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
                 if (slavei == 3)
                         masteri = 2;
         } else if (i <= 3) {
+                if (!strcmp ((char *)words[wordcount-1], "detail")) {
+                        /* For status detail it is mandatory to provide
+                         * both master and slave */
+                        ret = -1;
+                        goto out;
+                }
+
                 /* no $s, can only be status cmd
                  * (with either a single $m before it or nothing)
                  * -- these conditions imply that i <= 3 after
@@ -1525,7 +1830,12 @@ cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
         if (!w)
                 goto out;
 
-        if (strcmp (w, "status") == 0) {
+        if (strcmp (w, "create") == 0) {
+                type = GF_GSYNC_OPTION_TYPE_CREATE;
+
+                if (!masteri || !slavei)
+                        goto out;
+        } else if (strcmp (w, "status") == 0) {
                 type = GF_GSYNC_OPTION_TYPE_STATUS;
 
                 if (slavei && !masteri)
@@ -1545,14 +1855,33 @@ cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
 
                 if (!masteri || !slavei)
                         goto out;
-        } else if (strcmp(w, "log-rotate") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_ROTATE;
+        } else if (strcmp (w, "delete") == 0) {
+                type = GF_GSYNC_OPTION_TYPE_DELETE;
 
-                if (slavei && !masteri)
+                if (!masteri || !slavei)
                         goto out;
         } else
                 GF_ASSERT (!"opword mismatch");
 
+        ret = force_push_pem_parse (words, wordcount, dict, &cmdi);
+        if (ret)
+                goto out;
+
+        if (!strcmp ((char *)words[wordcount-1], "detail")) {
+                if (strcmp ((char *)words[wordcount-2], "status")) {
+                        ret = -1;
+                        goto out;
+                }
+                if (!slavei || !masteri) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_uint32 (dict, "status-detail", _gf_true);
+                if (ret)
+                        goto out;
+                cmdi++;
+        }
+
         if (type != GF_GSYNC_OPTION_TYPE_CONFIG &&
             (cmdi < wordcount - 1 || glob))
                 goto out;
@@ -1561,72 +1890,27 @@ cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
 
         ret = 0;
 
-        if (masteri)
+        if (masteri) {
                 ret = dict_set_str (dict, "master", (char *)words[masteri]);
+                if (!ret)
+                        ret = dict_set_str (dict, "volname",
+                                            (char *)words[masteri]);
+        }
         if (!ret && slavei)
                 ret = dict_set_str (dict, "slave", (char *)words[slavei]);
         if (!ret)
                 ret = dict_set_int32 (dict, "type", type);
-        if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG) {
-                switch ((wordcount - 1) - cmdi) {
-                case 0:
-                        subop = gf_strdup ("get-all");
-                        break;
-                case 1:
-                        if (words[cmdi + 1][0] == '!') {
-                                (words[cmdi + 1])++;
-                                if (gf_asprintf (&subop, "del%s", glob ? "-glob" : "") == -1)
-                                        subop = NULL;
-                        } else
-                                subop = gf_strdup ("get");
-
-                        ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                        if (ret < 0)
-                                goto out;
-                        break;
-                default:
-                        if (gf_asprintf (&subop, "set%s", glob ? "-glob" : "") == -1)
-                                subop = NULL;
-
-                        ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                        if (ret < 0)
-                                goto out;
-
-                        /* join the varargs by spaces to get the op_value */
-
-                        for (i = cmdi + 2; i < wordcount; i++)
-                                append_len += (strlen (words[i]) + 1);
-                        /* trailing strcat will add two bytes, make space for that */
-                        append_len++;
-
-                        append_str = GF_CALLOC (1, append_len, cli_mt_append_str);
-                        if (!append_str) {
-                                ret = -1;
-                                goto out;
-                        }
-
-                        for (i = cmdi + 2; i < wordcount; i++) {
-                                strcat (append_str, words[i]);
-                                strcat (append_str, " ");
-                        }
-                        append_str[append_len - 2] = '\0';
-
-                        ret = dict_set_dynstr (dict, "op_value", append_str);
-                }
-
-                if (!subop || dict_set_dynstr (dict, "subop", subop) != 0)
-                        ret = -1;
-        }
+        if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG)
+                ret = config_parse (words, wordcount, dict, cmdi, glob);
 
 out:
         if (ret) {
                 if (dict)
                         dict_destroy (dict);
-                if (append_str)
-                        GF_FREE (append_str);
         } else
                 *options = dict;
 
+
         return ret;
 }
 
@@ -1648,7 +1932,7 @@ cli_cmd_volume_profile_parse (const char **words, int wordcount,
         if (!dict)
                 goto out;
 
-        if (wordcount != 4)
+        if (wordcount < 4 || wordcount >5)
                 goto out;
 
         volname = (char *)words[2];
@@ -1670,7 +1954,19 @@ cli_cmd_volume_profile_parse (const char **words, int wordcount,
                 op = GF_CLI_STATS_INFO;
         } else
                 GF_ASSERT (!"opword mismatch");
+
         ret = dict_set_int32 (dict, "op", (int32_t)op);
+        if (ret)
+                goto out;
+
+        if (wordcount == 5) {
+                if (!strcmp (words[4], "nfs")) {
+                        ret = dict_set_int32 (dict, "nfs", _gf_true);
+                        if (ret)
+                                goto out;
+                }
+        }
+
         *options = dict;
 out:
         if (ret && dict)
@@ -1692,12 +1988,13 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
         int32_t  list_cnt       = -1;
         int      index          = 0;
         int      perf           = 0;
-        int32_t  blk_size       = 0;
-        int32_t  count          = 0;
+        uint32_t  blk_size      = 0;
+        uint32_t  count         = 0;
+        gf_boolean_t nfs        = _gf_false;
         char    *delimiter      = NULL;
         char    *opwords[]      = { "open", "read", "write", "opendir",
                                     "readdir", "read-perf", "write-perf",
-                                    NULL };
+                                    "clear", NULL };
         char    *w = NULL;
 
         GF_ASSERT (words);
@@ -1742,13 +2039,30 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
         } else if (strcmp (w, "write-perf") == 0) {
                 top_op = GF_CLI_TOP_WRITE_PERF;
                 perf = 1;
+        } else if (strcmp (w, "clear") == 0) {
+                ret = dict_set_int32 (dict, "clear-stats", 1);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Could not set clear-stats in dict");
+                        goto out;
+                }
         } else
                 GF_ASSERT (!"opword mismatch");
         ret = dict_set_int32 (dict, "top-op", (int32_t)top_op);
         if (ret)
                 goto out;
 
-        for (index = 4; index < wordcount; index+=2) {
+        if ((wordcount > 4) && !strcmp (words[4], "nfs")) {
+                nfs = _gf_true;
+                ret = dict_set_int32 (dict, "nfs", nfs);
+                if (ret)
+                        goto out;
+                index = 5;
+        } else {
+                index = 4;
+        }
+
+        for (; index < wordcount; index+=2) {
 
                 key = (char *) words[index];
                 value = (char *) words[index+1];
@@ -1759,14 +2073,14 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
                 }
                 if (!strcmp (key, "brick")) {
                         delimiter = strchr (value, ':');
-                        if (!delimiter || delimiter == value 
+                        if (!delimiter || delimiter == value
                             || *(delimiter+1) != '/') {
-                                cli_out ("wrong brick type: %s, use <HOSTNAME>:"
+                                cli_err ("wrong brick type: %s, use <HOSTNAME>:"
                                          "<export-dir-abs-path>", value);
                                 ret = -1;
                                 goto out;
                         } else {
-                                ret = cli_canonicalize_path (delimiter + 1);
+                                ret = gf_canonicalize_path (delimiter + 1);
                                 if (ret)
                                         goto out;
                         }
@@ -1777,39 +2091,40 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
                         if (!ret)
                                 list_cnt = atoi (value);
                         if (ret || (list_cnt < 0) || (list_cnt > 100)) {
-                                cli_out ("list-cnt should be between 0 to 100");
+                                cli_err ("list-cnt should be between 0 to 100");
                                 ret = -1;
                                 goto out;
                         }
-                } else if (perf && !strcmp (key, "bs")) {
+                } else if (perf && !nfs && !strcmp (key, "bs")) {
                         ret = gf_is_str_int (value);
                         if (!ret)
                                 blk_size = atoi (value);
                         if (ret || (blk_size <= 0)) {
                                 if (blk_size < 0)
-                                        cli_out ("block size is an invalid number");
+                                        cli_err ("block size is an invalid"
+                                                 " number");
                                 else
-                                        cli_out ("block size should be an integer "
-                                         "greater than zero");
+                                        cli_err ("block size should be an "
+                                                 "integer greater than zero");
                                 ret = -1;
                                 goto out;
                         }
-                        ret = dict_set_int32 (dict, "blk-size", blk_size);
-                } else if (perf && !strcmp (key, "count")) {
+                        ret = dict_set_uint32 (dict, "blk-size", blk_size);
+                } else if (perf && !nfs && !strcmp (key, "count")) {
                         ret = gf_is_str_int (value);
                         if (!ret)
                                 count = atoi(value);
                         if (ret || (count <= 0)) {
                                 if (count < 0)
-                                        cli_out ("count is an invalid number");
-                                else 
-                                        cli_out ("count should be an integer "
+                                        cli_err ("count is an invalid number");
+                                else
+                                        cli_err ("count should be an integer "
                                                  "greater than zero");
 
                                 ret = -1;
                                 goto out;
                         }
-                        ret = dict_set_int32 (dict, "blk-cnt", count);
+                        ret = dict_set_uint32 (dict, "blk-cnt", count);
                 } else {
                         ret = -1;
                         goto out;
@@ -1829,9 +2144,17 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
         }
 
         if ((blk_size > 0) ^ (count > 0)) {
+                cli_err ("Need to give both 'bs' and 'count'");
+                ret = -1;
+                goto out;
+        } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
+                cli_err ("'bs * count' value %"PRIu64" is greater than "
+                         "maximum allowed value of 10GB",
+                         ((uint64_t)blk_size * count));
                 ret = -1;
                 goto out;
         }
+
         *options = dict;
 out:
         if (ret && dict)
@@ -1839,39 +2162,43 @@ out:
         return ret;
 }
 
-gf_boolean_t
-cli_cmd_validate_statusop (const char *arg)
+uint32_t
+cli_cmd_get_statusop (const char *arg)
 {
-        char    *opwords[] = {"misc-details", "mem", "clients", "fd", "inode",
-                              "callpool", NULL};
-        char    *w = NULL;
+        int        i         = 0;
+        uint32_t   ret       = GF_CLI_STATUS_NONE;
+        char      *w         = NULL;
+        char      *opwords[] = {"detail", "mem", "clients", "fd",
+                                "inode", "callpool", "tasks", NULL};
+        struct {
+                char      *opname;
+                uint32_t   opcode;
+        } optable[] = {
+                { "detail",   GF_CLI_STATUS_DETAIL   },
+                { "mem",      GF_CLI_STATUS_MEM      },
+                { "clients",  GF_CLI_STATUS_CLIENTS  },
+                { "fd",       GF_CLI_STATUS_FD       },
+                { "inode",    GF_CLI_STATUS_INODE    },
+                { "callpool", GF_CLI_STATUS_CALLPOOL },
+                { "tasks",    GF_CLI_STATUS_TASKS    },
+                { NULL }
+        };
 
         w = str_getunamb (arg, opwords);
         if (!w) {
-                gf_log ("cli", GF_LOG_ERROR, "Unknown status op  %s",
-                        arg);
-                return _gf_false;
+                gf_log ("cli", GF_LOG_DEBUG,
+                        "Not a status op  %s", arg);
+                goto out;
         }
-        return _gf_true;
-}
 
-int
-cli_cmd_get_statusop (const char *arg)
-{
-        int ret = GF_CLI_STATUS_INVAL;
-        if (!strcmp (arg, "misc-details"))
-                ret = GF_CLI_STATUS_DETAIL;
-        else if (!strcmp (arg, "mem"))
-                ret = GF_CLI_STATUS_MEM;
-        else if (!strcmp (arg, "clients"))
-                ret = GF_CLI_STATUS_CLIENTS;
-        else if (!strcmp (arg, "inode"))
-                ret = GF_CLI_STATUS_INODE;
-        else if (!strcmp (arg, "fd"))
-                ret = GF_CLI_STATUS_FD;
-        else if (!strcmp (arg, "callpool"))
-                ret = GF_CLI_STATUS_CALLPOOL;
+        for (i = 0; optable[i].opname; i++) {
+                if (!strcmp (w, optable[i].opname)) {
+                        ret = optable[i].opcode;
+                        break;
+                }
+        }
 
+ out:
         return ret;
 }
 
@@ -1879,9 +2206,9 @@ int
 cli_cmd_volume_status_parse (const char **words, int wordcount,
                              dict_t **options)
 {
-        dict_t *dict            = NULL;
-        int     ret             = -1;
-        int     cmd             = 0;
+        dict_t    *dict            = NULL;
+        int        ret             = -1;
+        uint32_t   cmd             = 0;
 
         GF_ASSERT (options);
 
@@ -1898,61 +2225,105 @@ cli_cmd_volume_status_parse (const char **words, int wordcount,
 
         case 3:
                 if (!strcmp (words[2], "all")) {
-
                         cmd = GF_CLI_STATUS_ALL;
                         ret = 0;
+
                 } else {
                         cmd = GF_CLI_STATUS_VOL;
                         ret = dict_set_str (dict, "volname", (char *)words[2]);
                 }
+
                 break;
 
         case 4:
+                cmd = cli_cmd_get_statusop (words[3]);
+
                 if (!strcmp (words[2], "all")) {
-                        cli_out ("Cannot specify brick/status-type for \"all\"");
-                        ret = -1;
-                        goto out;
+                        if (cmd == GF_CLI_STATUS_NONE) {
+                                cli_err ("%s is not a valid status option",
+                                         words[3]);
+                                ret = -1;
+                                goto out;
+                        }
+                        cmd |= GF_CLI_STATUS_ALL;
+                        ret  = 0;
+
                 } else {
-                        cmd = GF_CLI_STATUS_VOL;
-                        ret = dict_set_str (dict, "volname", (char *)words[2]);
+                        ret = dict_set_str (dict, "volname",
+                                            (char *)words[2]);
                         if (ret)
                                 goto out;
-                }
 
-                if (cli_cmd_validate_statusop (words[3])) {
-                        ret = cli_cmd_get_statusop (words[3]);
-                        if (GF_CLI_STATUS_INVAL == ret)
-                                goto out;
-                        cmd |= ret;
-                        ret = 0;
-                } else {
-                        cmd = GF_CLI_STATUS_BRICK;
-                        ret = dict_set_str (dict, "brick", (char *)words[3]);
+                        if (cmd == GF_CLI_STATUS_NONE) {
+                                if (!strcmp (words[3], "nfs")) {
+                                        cmd |= GF_CLI_STATUS_NFS;
+                                } else if (!strcmp (words[3], "shd")) {
+                                        cmd |= GF_CLI_STATUS_SHD;
+                                } else {
+                                        cmd = GF_CLI_STATUS_BRICK;
+                                        ret = dict_set_str (dict, "brick",
+                                                            (char *)words[3]);
+                                }
+
+                        } else {
+                                cmd |= GF_CLI_STATUS_VOL;
+                                ret  = 0;
+                        }
                 }
+
                 break;
 
         case 5:
-                if (!cli_cmd_validate_statusop (words[4])) {
+                if (!strcmp (words[2], "all")) {
+                        cli_err ("Cannot specify brick/nfs for \"all\"");
                         ret = -1;
                         goto out;
                 }
 
-                cmd = GF_CLI_STATUS_BRICK;
-                ret = cli_cmd_get_statusop (words[4]);
-                if (GF_CLI_STATUS_INVAL == ret)
+                cmd = cli_cmd_get_statusop (words[4]);
+                if (cmd == GF_CLI_STATUS_NONE) {
+                        cli_err ("%s is not a valid status option",
+                                 words[4]);
+                        ret = -1;
                         goto out;
-                cmd |= ret;
+                }
 
-                if (!strcmp (words[2], "all")) {
-                        cli_out ("Cannot specify brick/status-type for \"all\"");
-                        ret = -1;
+
+                ret = dict_set_str (dict, "volname", (char *)words[2]);
+                if (ret)
                         goto out;
+
+                if (!strcmp (words[3], "nfs")) {
+                        if (cmd == GF_CLI_STATUS_FD ||
+                            cmd == GF_CLI_STATUS_DETAIL ||
+                            cmd == GF_CLI_STATUS_TASKS) {
+                                cli_err ("Detail/FD/Tasks status not available"
+                                         " for NFS Servers");
+                                ret = -1;
+                                goto out;
+                        }
+                        cmd |= GF_CLI_STATUS_NFS;
+                } else if (!strcmp (words[3], "shd")){
+                        if (cmd == GF_CLI_STATUS_FD ||
+                            cmd == GF_CLI_STATUS_CLIENTS ||
+                            cmd == GF_CLI_STATUS_DETAIL ||
+                            cmd == GF_CLI_STATUS_TASKS) {
+                                cli_err ("Detail/FD/Clients/Tasks status not "
+                                         "available for Self-heal Daemons");
+                                ret = -1;
+                                goto out;
+                        }
+                        cmd |= GF_CLI_STATUS_SHD;
                 } else {
-                        ret = dict_set_str (dict, "volname", (char *)words[2]);
-                        if (ret)
+                        if (cmd == GF_CLI_STATUS_TASKS) {
+                                cli_err ("Tasks status not available for "
+                                         "bricks");
+                                ret = -1;
                                 goto out;
+                        }
+                        cmd |= GF_CLI_STATUS_BRICK;
+                        ret = dict_set_str (dict, "brick", (char *)words[3]);
                 }
-                ret = dict_set_str (dict, "brick", (char *)words[3]);
                 break;
 
         default:
@@ -1976,18 +2347,20 @@ cli_cmd_volume_status_parse (const char **words, int wordcount,
 }
 
 gf_boolean_t
-cli_cmd_validate_dumpoption (const char *option)
+cli_cmd_validate_dumpoption (const char *arg, char **option)
 {
         char    *opwords[] = {"all", "nfs", "mem", "iobuf", "callpool", "priv",
-			      "fd", "inode", NULL};
+                              "fd", "inode", "history", "inodectx", "fdctx",
+                              NULL};
         char    *w = NULL;
 
-        w = str_getunamb (option, opwords);
+        w = str_getunamb (arg, opwords);
         if (!w) {
                 gf_log ("cli", GF_LOG_DEBUG, "Unknown statedump option  %s",
-                        option);
+                        arg);
                 return _gf_false;
         }
+        *option = w;
         return _gf_true;
 }
 
@@ -1999,14 +2372,15 @@ cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
         int     i = 0;
         dict_t  *dict = NULL;
         int     option_cnt = 0;
+        char    *option = NULL;
         char    option_str[100] = {0,};
 
         for (i = 3; i < wordcount; i++, option_cnt++) {
-                if (!cli_cmd_validate_dumpoption (words[i])) {
+                if (!cli_cmd_validate_dumpoption (words[i], &option)) {
                         ret = -1;
                         goto out;
                 }
-                strncat (option_str, words[i], sizeof (words [i]));
+                strncat (option_str, option, strlen (option));
                 strncat (option_str, " ", 1);
         }
 
@@ -2014,7 +2388,7 @@ cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
         if (!dict)
                 goto out;
 
-        ret = dict_set_str (dict, "options", gf_strdup (option_str));
+        ret = dict_set_dynstr (dict, "options", gf_strdup (option_str));
         if (ret)
                 goto out;
 
@@ -2030,3 +2404,1279 @@ out:
                 gf_log ("cli", GF_LOG_ERROR, "Error parsing dumpoptions");
         return ret;
 }
+
+int
+cli_cmd_volume_clrlks_opts_parse (const char **words, int wordcount,
+                                  dict_t **options)
+{
+        int     ret = -1;
+        int     i = 0;
+        dict_t  *dict = NULL;
+        char    *kind_opts[4] = {"blocked", "granted", "all", NULL};
+        char    *types[4] = {"inode", "entry", "posix", NULL};
+        char    *free_ptr = NULL;
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        if (strcmp (words[4], "kind"))
+                goto out;
+
+        for (i = 0; kind_opts[i]; i++) {
+               if (!strcmp (words[5], kind_opts[i])) {
+                       free_ptr = gf_strdup (words[5]);
+                       ret = dict_set_dynstr (dict, "kind", free_ptr);
+                       if (ret)
+                               goto out;
+                       free_ptr = NULL;
+                       break;
+               }
+        }
+        if (i == 3)
+                goto out;
+
+        ret = -1;
+        for (i = 0; types[i]; i++) {
+               if (!strcmp (words[6], types[i])) {
+                       free_ptr = gf_strdup (words[6]);
+                       ret = dict_set_dynstr (dict, "type", free_ptr);
+                       if (ret)
+                               goto out;
+                       free_ptr = NULL;
+                       break;
+               }
+        }
+        if (i == 3)
+                goto out;
+
+        if (wordcount == 8) {
+                free_ptr = gf_strdup (words[7]);
+                ret = dict_set_dynstr (dict, "opts", free_ptr);
+                if (ret)
+                        goto out;
+                free_ptr = NULL;
+        }
+
+        ret = 0;
+        *options = dict;
+out:
+       if (ret) {
+               GF_FREE (free_ptr);
+               dict_unref (dict);
+       }
+
+       return ret;
+}
+
+static int
+extract_hostname_path_from_token (const char *tmp_words, char **hostname,
+                                  char **path)
+{
+        int ret = 0;
+        char *delimiter = NULL;
+        char *tmp_host = NULL;
+        char *host_name = NULL;
+        char *words = NULL;
+
+        *hostname = NULL;
+        *path = NULL;
+
+        words = GF_CALLOC (1, strlen (tmp_words) + 1, gf_common_mt_char);
+        if (!words){
+                ret = -1;
+                goto out;
+        }
+
+        strncpy (words, tmp_words, strlen (tmp_words) + 1);
+
+        if (validate_brick_name (words)) {
+                cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
+                        "<export-dir-abs-path>", words);
+                ret = -1;
+                goto out;
+        } else {
+                delimiter = strrchr (words, ':');
+                ret = gf_canonicalize_path (delimiter + 1);
+                if (ret) {
+                        goto out;
+                } else {
+                        *path = GF_CALLOC (1, strlen (delimiter+1) +1,
+                                           gf_common_mt_char);
+                        if (!*path) {
+                           ret = -1;
+                                goto out;
+
+                        }
+                        strncpy (*path, delimiter +1,
+                                 strlen(delimiter + 1) + 1);
+                }
+        }
+
+        tmp_host = gf_strdup (words);
+        if (!tmp_host) {
+                gf_log ("cli", GF_LOG_ERROR, "Out of memory");
+                ret = -1;
+                goto out;
+        }
+        get_host_name (tmp_host, &host_name);
+        if (!host_name) {
+                ret = -1;
+                gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
+                        "memory");
+                goto out;
+        }
+        if (!(strcmp (host_name, "localhost") &&
+            strcmp (host_name, "127.0.0.1") &&
+            strncmp (host_name, "0.", 2))) {
+                cli_err ("Please provide a valid hostname/ip other "
+                         "than localhost, 127.0.0.1 or loopback "
+                         "address (0.0.0.0 to 0.255.255.255).");
+                ret = -1;
+                goto out;
+        }
+        if (!valid_internet_address (host_name, _gf_false)) {
+                cli_err ("internet address '%s' does not conform to "
+                          "standards", host_name);
+                ret = -1;
+                goto out;
+        }
+
+        *hostname = GF_CALLOC (1, strlen (host_name) + 1,
+                                       gf_common_mt_char);
+        if (!*hostname) {
+                ret = -1;
+                goto out;
+        }
+        strncpy (*hostname, host_name, strlen (host_name) + 1);
+        ret = 0;
+
+out:
+        GF_FREE (words);
+        GF_FREE (tmp_host);
+        return ret;
+}
+
+
+int
+cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
+                                   dict_t **options)
+{
+        int     ret = 0;
+        dict_t  *dict = NULL;
+        char    *hostname = NULL;
+        char    *path = NULL;
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_str (dict, "volname", (char *) words[2]);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "failed to set volname");
+                goto out;
+        }
+
+        if (wordcount == 3) {
+                ret = dict_set_int32 (dict, "heal-op", GF_AFR_OP_HEAL_INDEX);
+                goto done;
+        }
+
+        if (wordcount == 4) {
+                if (!strcmp (words[3], "full")) {
+                        ret = dict_set_int32 (dict, "heal-op",
+                                              GF_AFR_OP_HEAL_FULL);
+                        goto done;
+                } else if (!strcmp (words[3], "statistics")) {
+                        ret = dict_set_int32 (dict, "heal-op",
+                                              GF_AFR_OP_STATISTICS);
+                        goto done;
+
+                } else if (!strcmp (words[3], "info")) {
+                        ret = dict_set_int32 (dict, "heal-op",
+                                              GF_AFR_OP_INDEX_SUMMARY);
+                        goto done;
+                } else {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        if (wordcount == 5) {
+                if (strcmp (words[3], "info") &&
+                    strcmp (words[3], "statistics")) {
+                        ret = -1;
+                        goto out;
+                }
+
+                if (!strcmp (words[3], "info")) {
+                        if (!strcmp (words[4], "healed")) {
+                                ret = dict_set_int32 (dict, "heal-op",
+                                                      GF_AFR_OP_HEALED_FILES);
+                                goto done;
+                        }
+                        if (!strcmp (words[4], "heal-failed")) {
+                                ret = dict_set_int32 (dict, "heal-op",
+                                                   GF_AFR_OP_HEAL_FAILED_FILES);
+                                goto done;
+                        }
+                        if (!strcmp (words[4], "split-brain")) {
+                                ret = dict_set_int32 (dict, "heal-op",
+                                                   GF_AFR_OP_SPLIT_BRAIN_FILES);
+                                goto done;
+                        }
+                }
+
+                if (!strcmp (words[3], "statistics")) {
+                        if (!strcmp (words[4], "heal-count")) {
+                                ret = dict_set_int32 (dict, "heal-op",
+                                               GF_AFR_OP_STATISTICS_HEAL_COUNT);
+                                goto done;
+                        }
+                }
+                ret = -1;
+                goto out;
+        }
+        if (wordcount == 7) {
+                if (!strcmp (words[3], "statistics")
+                    && !strcmp (words[4], "heal-count")
+                    && !strcmp (words[5], "replica")) {
+
+                        ret = dict_set_int32 (dict, "heal-op",
+                                   GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+                        if (ret)
+                                goto out;
+                        ret = extract_hostname_path_from_token (words[6],
+                                                              &hostname, &path);
+                        if (ret)
+                                goto out;
+                        ret = dict_set_dynstr (dict, "per-replica-cmd-hostname",
+                                               hostname);
+                        if (ret)
+                                goto out;
+                        ret = dict_set_dynstr (dict, "per-replica-cmd-path",
+                                               path);
+                        if (ret)
+                                goto out;
+                        else
+                                goto done;
+
+                }
+        }
+        ret = -1;
+        goto out;
+done:
+        *options = dict;
+out:
+        if (ret && dict) {
+                dict_unref (dict);
+                *options = NULL;
+        }
+
+        return ret;
+}
+
+int
+cli_cmd_volume_defrag_parse (const char **words, int wordcount,
+                             dict_t **options)
+{
+        dict_t                 *dict = NULL;
+        int                      ret = -1;
+        char                *option  = NULL;
+        char                *volname = NULL;
+        char                *command = NULL;
+        gf_cli_defrag_type       cmd = 0;
+
+        GF_ASSERT (words);
+        GF_ASSERT (options);
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        if (!((wordcount == 4) || (wordcount == 5)))
+                goto out;
+
+        if (wordcount == 4) {
+                if (strcmp (words[3], "start") && strcmp (words[3], "stop") &&
+                    strcmp (words[3], "status"))
+                            goto out;
+        } else {
+                if (strcmp (words[3], "fix-layout") &&
+                    strcmp (words[3], "start"))
+                        goto out;
+        }
+
+        volname = (char *) words[2];
+
+        if (wordcount == 4) {
+                command = (char *) words[3];
+        }
+        if (wordcount == 5) {
+               if ((strcmp (words[3], "fix-layout") ||
+                    strcmp (words[4], "start")) &&
+                    (strcmp (words[3], "start") ||
+                    strcmp (words[4], "force"))) {
+                        ret = -1;
+                        goto out;
+                }
+                command = (char *) words[3];
+                option = (char *) words[4];
+        }
+
+        if (strcmp (command, "start") == 0) {
+                cmd = GF_DEFRAG_CMD_START;
+                if (option && strcmp (option, "force") == 0) {
+                                cmd = GF_DEFRAG_CMD_START_FORCE;
+                        }
+                goto done;
+        }
+
+        if (strcmp (command, "fix-layout") == 0) {
+                cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
+                goto done;
+        }
+        if (strcmp (command, "stop") == 0) {
+                cmd = GF_DEFRAG_CMD_STOP;
+                goto done;
+        }
+        if (strcmp (command, "status") == 0) {
+                cmd = GF_DEFRAG_CMD_STATUS;
+        }
+
+done:
+        ret = dict_set_str (dict, "volname", volname);
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, "rebalance-command", (int32_t) cmd);
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
+                goto out;
+        }
+
+        *options = dict;
+
+out:
+        if (ret && dict)
+                dict_destroy (dict);
+
+        return ret;
+}
+
+int32_t
+cli_snap_create_desc_parse (dict_t *dict, const char **words,
+                            size_t wordcount, int32_t desc_opt_loc)
+{
+        int32_t        ret      = -1;
+        char          *desc     = NULL;
+        int32_t        desc_len = 0;
+
+        desc = GF_CALLOC (MAX_SNAP_DESCRIPTION_LEN + 1, sizeof(char),
+                          gf_common_mt_char);
+        if (!desc) {
+                ret = -1;
+                goto out;
+        }
+
+
+        if (strlen (words[desc_opt_loc]) >= MAX_SNAP_DESCRIPTION_LEN) {
+                cli_out ("snapshot create: description truncated: "
+                         "Description provided is longer than 1024 characters");
+                desc_len = MAX_SNAP_DESCRIPTION_LEN;
+        } else {
+                desc_len = strlen (words[desc_opt_loc]);
+        }
+
+        strncpy (desc, words[desc_opt_loc], desc_len);
+        desc[desc_len] = '\0';
+        /* Calculating the size of the description as given by the user */
+
+        ret = dict_set_dynstr (dict, "description", desc);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap "
+                        "description");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret && desc)
+                GF_FREE (desc);
+
+        return ret;
+}
+
+/* Function to check whether the Volume name is repeated */
+int
+cli_check_if_volname_repeated (const char **words, unsigned int start_index,
+                           uint64_t cur_index) {
+        uint64_t        i       =       -1;
+        int             ret     =        0;
+
+        GF_ASSERT (words);
+
+        for (i = start_index ; i < cur_index ; i++) {
+                if (strcmp (words[i], words[cur_index]) == 0) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+out :
+        return ret;
+}
+
+/* snapshot create <snapname> <vol-name(s)> [description <description>]
+ *                                           [force]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_create_parse (dict_t *dict, const char **words, int wordcount) {
+        uint64_t        i               =        0;
+        int             ret             =       -1;
+        uint64_t        volcount        =        0;
+        char            key[PATH_MAX]   =        "";
+        char            *snapname       =       NULL;
+        unsigned int    cmdi            =        2;
+        /* cmdi is command index, here cmdi is "2" (gluster snapshot create)*/
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount <= cmdi + 1) {
+                cli_err ("Invalid Syntax.");
+                gf_log ("cli", GF_LOG_ERROR,
+                        "Too less words for snap create command");
+                goto out;
+        }
+
+        if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+                cli_err ("snapshot create: failed: snapname cannot exceed "
+                         "255 characters.");
+                gf_log ("cli", GF_LOG_ERROR, "Snapname too long");
+
+                goto out;
+        }
+
+        snapname = (char *) words[cmdi];
+        for (i = 0 ; i < strlen (snapname); i++) {
+                /* Following volume name convention */
+                if (!isalnum (snapname[i]) && (snapname[i] != '_'
+                                           && (snapname[i] != '-'))) {
+                        /* TODO : Is this message enough?? */
+                        cli_err ("Snapname can contain only alphanumeric, "
+                                 "\"-\" and \"_\" characters");
+                        goto out;
+                }
+        }
+
+        ret = dict_set_str (dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not save snap "
+                        "name");
+                goto out;
+        }
+
+        /* Filling volume name in the dictionary */
+        for (i = cmdi + 1 ; i < wordcount
+                            && (strcmp (words[i], "description")) != 0
+                            && (strcmp (words[i], "force") != 0); i++) {
+                volcount++;
+                /* volume index starts from 1 */
+                ret = snprintf (key, sizeof (key),"volname%ld", volcount);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_set_str (dict, key, (char *)words[i]);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not "
+                                "save volume name");
+                        goto out;
+                }
+
+                if (i >= cmdi + 2) {
+                        ret = -1;
+                        cli_err("Creating multiple volume snapshot is not "
+                                "supported as of now");
+                        goto out;
+                }
+                /* TODO : remove this above condition check once
+                 * multiple volume snapshot is supported */
+        }
+
+        if (volcount == 0) {
+                ret = -1;
+                cli_err ("Please provide the volume name");
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, "volcount", volcount);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not save volcount");
+                goto out;
+        }
+
+        /* Verify how we got out of "for" loop,
+         * if it is by reaching wordcount limit then goto "out",
+         * because we need not parse for "description" and "force"
+         * after this.
+         */
+        if (i == wordcount) {
+                goto out;
+        }
+
+        if ((strcmp (words[i], "description")) == 0) {
+                ++i;
+                if (i > (wordcount - 1)) {
+                        ret = -1;
+                        cli_err ("Please provide a description");
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Description not provided");
+                        goto out;
+                }
+
+                ret = cli_snap_create_desc_parse(dict, words, wordcount, i);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not save snap "
+                                "description");
+                        goto out;
+                }
+
+                if ( i == (wordcount - 1))
+                        goto out;
+                i++;
+                /* point the index to next word.
+                 * As description might be follwed by force option.
+                 * Before that, check if wordcount limit is reached
+                 */
+        }
+
+        if ((strcmp (words[i], "force") != 0)) {
+                ret = -1;
+                cli_err ("Invalid Syntax.");
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+        ret = dict_set_int8 (dict, "snap-force", 1);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not save "
+                        "snap force option");
+                goto out;
+        }
+
+        /* Check if the command has anything after "force" keyword */
+        if (++i < wordcount) {
+                ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = 0;
+
+out :
+        return ret;
+}
+
+/* snapshot list [volname]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_list_parse (dict_t *dict, const char **words, int wordcount) {
+        int             ret     =       -1;
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount < 2 || wordcount > 3) {
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        if (wordcount == 2) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "volname", (char *)words[2]);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR,
+                        "Failed to save volname in dictionary");
+                goto out;
+        }
+out :
+        return ret;
+}
+
+/* snapshot info [(snapname |  volume <volname>)]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_info_parse (dict_t *dict, const char **words, int wordcount)
+{
+
+        int             ret             =       -1;
+        int32_t         cmd             =       GF_SNAP_INFO_TYPE_ALL;
+        unsigned int    cmdi            =        2;
+        /* cmdi is command index, here cmdi is "2" (gluster snapshot info)*/
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount > 4 || wordcount < cmdi) {
+                gf_log ("", GF_LOG_ERROR, "Invalid syntax");
+                goto out;
+        }
+
+        if (wordcount == cmdi) {
+                ret = 0;
+                goto out;
+        }
+
+        /* If 3rd word is not "volume", then it must
+         * be snapname.
+         */
+        if (strcmp (words[cmdi], "volume") != 0) {
+                ret = dict_set_str (dict, "snapname",
+                                   (char *)words[cmdi]);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Unable to save "
+                                "snapname %s", words[cmdi]);
+                        goto out;
+                }
+
+                /* Once snap name is parsed, if we encounter any other
+                 * word then fail it. Invalid Syntax.
+                 * example : snapshot info <snapname> word
+                 */
+                if ((cmdi + 1) != wordcount) {
+                        ret = -1;
+                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                        goto out;
+                }
+
+                cmd = GF_SNAP_INFO_TYPE_SNAP;
+                ret = 0;
+                goto out;
+                /* No need to continue the parsing once we
+                 * get the snapname
+                 */
+        }
+
+        /* If 3rd word is "volume", then check if next word
+         * is present. As, "snapshot info volume" is an
+         * invalid command.
+         */
+        if ((cmdi + 1) == wordcount) {
+                ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "volname", (char *)words[wordcount - 1]);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Count not save "
+                        "volume name %s", words[wordcount - 1]);
+                goto out;
+        }
+        cmd = GF_SNAP_INFO_TYPE_VOL;
+out :
+        if (ret == 0) {
+                ret = dict_set_int32 (dict, "cmd", cmd);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
+                                "type of snapshot info");
+                }
+        }
+        return ret;
+}
+
+
+
+/* snapshot restore <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_restore_parse (dict_t *dict, const char **words, int wordcount)
+{
+
+        int             ret             =       -1;
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount != 3) {
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "snapname", (char *)words[2]);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
+                        words[2]);
+                goto out;
+        }
+out :
+        return ret;
+}
+
+/* snapshot delete <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ *                 1 if user cancel the operation
+ */
+int
+cli_snap_delete_parse (dict_t *dict, const char **words, int wordcount,
+                       struct cli_state *state) {
+
+        int             ret             =       -1;
+        const char      *question       =       NULL;
+        gf_answer_t     answer          =       GF_ANSWER_NO;
+
+        question = "Deleting snap will erase all the information about "
+                   "the snap. Do you still want to continue?";
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount != 3) {
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "snapname", (char *)words[2]);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to save snapname %s",
+                        words[2]);
+                goto out;
+        }
+
+        answer = cli_cmd_get_confirmation (state, question);
+        if (GF_ANSWER_NO == answer) {
+                ret = 1;
+                gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
+                        "snapshot delete operation");
+                goto out;
+        }
+out :
+        return ret;
+}
+
+/* snapshot status [(snapname | volume <volname>)]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_status_parse (dict_t *dict, const char **words, int wordcount)
+{
+
+        int             ret  =        -1;
+        int32_t         cmd  =       GF_SNAP_STATUS_TYPE_ALL;
+        unsigned int    cmdi =        2;
+        /* cmdi is command index, here cmdi is "2" (gluster snapshot status)*/
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount > 4 || wordcount < cmdi) {
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        if (wordcount == cmdi) {
+                ret = 0;
+                goto out;
+        }
+
+        /* if 3rd word is not "volume", then it must be "snapname"
+        */
+        if (strcmp (words[cmdi], "volume") != 0) {
+                ret = dict_set_str (dict, "snapname",
+                                   (char *)words[cmdi]);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Count not save "
+                                "snap name %s", words[cmdi]);
+                        goto out;
+                }
+
+                if ((cmdi + 1) != wordcount) {
+                        ret = -1;
+                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                        goto out;
+                }
+
+                ret = 0;
+                cmd = GF_SNAP_STATUS_TYPE_SNAP;
+                goto out;
+        }
+
+        /* If 3rd word is "volume", then check if next word is present.
+         * As, "snapshot info volume" is an invalid command
+         */
+        if ((cmdi + 1) == wordcount) {
+                ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "volname", (char *)words [wordcount - 1]);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Count not save "
+                        "volume name %s", words[wordcount - 1]);
+                goto out;
+        }
+        cmd = GF_SNAP_STATUS_TYPE_VOL;
+
+out :
+        if (ret == 0) {
+                ret = dict_set_int32 (dict, "cmd", cmd);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not save cmd "
+                                "of snapshot status");
+                }
+        }
+        return ret;
+}
+
+
+int32_t
+cli_snap_config_limit_parse (const char **words, dict_t *dict,
+                             unsigned int wordcount, unsigned int index,
+                             char *key)
+{
+        int             ret             =       -1;
+        int             limit           =       0;
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+        GF_ASSERT (key);
+
+        if (index >= wordcount) {
+                ret = -1;
+                cli_err ("Please provide a value for %s.",key);
+                gf_log ("cli", GF_LOG_ERROR, "Value not provided for %s", key);
+                goto out;
+        }
+
+        limit = strtol (words[index], NULL, 0);
+        if (limit <= 0) {
+                ret = -1;
+                cli_err ("%s should be greater than 0.", key);
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, key, limit);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not set "
+                        "%s in dictionary", key);
+                goto out;
+        }
+
+out :
+        return ret;
+}
+
+/* function cli_snap_config_parse
+ * Config Syntax : gluster snapshot config [volname]
+ *                                         [snap-max-hard-limit <count>]
+ *                                         [snap-max-soft-limit <count>]
+ *
+   return value: <0  on failure
+                  1  if user cancels the operation
+                  0  on success
+
+  NOTE : snap-max-soft-limit can only be set for system.
+*/
+int32_t
+cli_snap_config_parse (const char **words, int wordcount, dict_t *dict,
+                       struct cli_state *state)
+{
+        int                            ret                 = -1;
+        gf_answer_t                    answer              = GF_ANSWER_NO;
+        gf_boolean_t                   vol_presence        = _gf_false;
+        struct snap_config_opt_vals_  *conf_vals           = NULL;
+        int8_t                         hard_limit          = 0;
+        int8_t                         soft_limit          = 0;
+        int8_t                         config_type         = -1;
+        const char                    *question            = NULL;
+        unsigned int                    cmdi               = 2;
+        /* cmdi is command index, here cmdi is "2" (gluster snapshot config)*/
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+        GF_ASSERT (state);
+
+        if ((wordcount < 2) || (wordcount > 7)) {
+                gf_log ("cli", GF_LOG_ERROR,
+                        "Invalid wordcount(%d)", wordcount);
+                goto out;
+        }
+
+        if (wordcount == 2) {
+                config_type = GF_SNAP_CONFIG_DISPLAY;
+                ret = 0;
+                goto set;
+        }
+
+        /* Check whether the 3rd word is volname */
+        if (strcmp (words[cmdi], "snap-max-hard-limit") != 0
+             && strcmp (words[cmdi], "snap-max-soft-limit") != 0) {
+                ret = dict_set_str (dict, "volname", (char *)words[cmdi]);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to set volname");
+                        goto out;
+                }
+                cmdi++;
+                vol_presence = _gf_true;
+
+                if (cmdi == wordcount) {
+                        config_type = GF_SNAP_CONFIG_DISPLAY;
+                        ret = 0;
+                        goto set;
+                }
+        }
+
+        config_type = GF_SNAP_CONFIG_TYPE_SET;
+
+        if (strcmp (words[cmdi], "snap-max-hard-limit") == 0) {
+                ret = cli_snap_config_limit_parse (words, dict, wordcount,
+                                                ++cmdi, "snap-max-hard-limit");
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse snap "
+                                "config hard limit");
+                        goto out;
+                }
+                hard_limit = 1;
+
+                if (++cmdi == wordcount) {
+                        ret = 0;
+                        goto set;
+                }
+        }
+
+        if (strcmp (words[cmdi], "snap-max-soft-limit") == 0) {
+                if (vol_presence == 1) {
+                        ret = -1;
+                        cli_err ("Soft limit cannot be set to individual "
+                                  "volumes.");
+                        gf_log ("cli", GF_LOG_ERROR, "Soft limit cannot be "
+                                "set to volumes");
+                        goto out;
+                }
+
+                ret = cli_snap_config_limit_parse (words, dict, wordcount,
+                                                ++cmdi, "snap-max-soft-limit");
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse snap "
+                                "config soft limit");
+                        goto out;
+                }
+
+                if (++cmdi != wordcount) {
+                        ret = -1;
+                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                        goto out;
+                }
+                soft_limit = 1;
+        } else {
+                ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+        ret = 0; /* Success */
+
+set:
+        ret = dict_set_int32 (dict, "config-command", config_type);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to set "
+                        "config-command");
+                goto out;
+        }
+
+        if (config_type == GF_SNAP_CONFIG_TYPE_SET) {
+                conf_vals = snap_confopt_vals;
+                if (hard_limit && soft_limit) {
+                        question = conf_vals[GF_SNAP_CONFIG_SET_BOTH].question;
+                } else if (soft_limit) {
+                        question = conf_vals[GF_SNAP_CONFIG_SET_SOFT].question;
+                } else if (hard_limit) {
+                        question = conf_vals[GF_SNAP_CONFIG_SET_HARD].question;
+                }
+
+                answer = cli_cmd_get_confirmation (state, question);
+                if (GF_ANSWER_NO == answer) {
+                        ret = 1;
+                        gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
+                        "snapshot config operation");
+                }
+        }
+
+out:
+        return ret;
+}
+
+int
+validate_snapname (const char *snapname, char **opwords) {
+        int     ret     =       -1;
+        int     i       =       0;
+
+        GF_ASSERT (snapname);
+        GF_ASSERT (opwords);
+
+        for (i = 0 ; opwords[i] != NULL; i++) {
+                if (strcmp (opwords[i], snapname) == 0) {
+                        cli_out ("\"%s\" cannot be a snapname", snapname);
+                        goto out;
+                }
+        }
+        ret = 0;
+out :
+        return ret;
+}
+
+int32_t
+cli_cmd_snapshot_parse (const char **words, int wordcount, dict_t **options,
+                        struct cli_state *state)
+{
+        int32_t            ret        = -1;
+        dict_t             *dict      = NULL;
+        gf1_cli_snapshot   type       = GF_SNAP_OPTION_TYPE_NONE;
+        char               *w         = NULL;
+        char               *opwords[] = {"create", "delete", "restore", "start",
+                                         "stop", "list", "status", "config",
+                                         "info", NULL};
+        char               *invalid_snapnames[] = {"description", "force",
+                                                  "volume", NULL};
+
+        GF_ASSERT (words);
+        GF_ASSERT (options);
+        GF_ASSERT (state);
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        /* Lowest wordcount possible */
+        if (wordcount < 2) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Invalid command: Not enough arguments");
+                goto out;
+        }
+
+        w = str_getunamb (words[1], opwords);
+        if (!w) {
+                /* Checks if the operation is a valid operation */
+                gf_log ("", GF_LOG_ERROR, "Opword Mismatch");
+                goto out;
+        }
+
+        if (!strcmp (w, "create")) {
+                type = GF_SNAP_OPTION_TYPE_CREATE;
+        } else if (!strcmp (w, "list")) {
+                type = GF_SNAP_OPTION_TYPE_LIST;
+        } else if (!strcmp (w, "info")) {
+                type = GF_SNAP_OPTION_TYPE_INFO;
+        } else if (!strcmp (w, "delete")) {
+                type = GF_SNAP_OPTION_TYPE_DELETE;
+        } else if (!strcmp (w, "config")) {
+                type = GF_SNAP_OPTION_TYPE_CONFIG;
+        } else if (!strcmp (w, "restore")) {
+                type = GF_SNAP_OPTION_TYPE_RESTORE;
+        } else if (!strcmp (w, "status")) {
+                type = GF_SNAP_OPTION_TYPE_STATUS;
+        }
+
+        if (type != GF_SNAP_OPTION_TYPE_CONFIG) {
+                ret = dict_set_int32 (dict, "hold_snap_locks", _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Unable to set hold-snap-locks value "
+                                "as _gf_true");
+                        goto out;
+                }
+        }
+
+        /* Check which op is intended */
+        switch (type) {
+               case GF_SNAP_OPTION_TYPE_CREATE:
+               {
+                        /* Syntax :
+                         * gluster snapshot create <snapname> <vol-name(s)>
+                         *                         [description <description>]
+                         *                         [force]
+                         */
+
+                        /* In cases where the snapname is not given then
+                         * parsing fails & snapname cannot be "description",
+                         * "force" and "volume", that check is made here
+                         */
+                       if (wordcount == 2){
+                                ret = -1;
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Invalid Syntax");
+                                goto out;
+                       }
+
+                       ret = validate_snapname (words[2], invalid_snapnames);
+                       if (ret) {
+                               goto out;
+                       }
+
+                       ret = cli_snap_create_parse (dict, words, wordcount);
+                       if (ret) {
+                               gf_log ("cli", GF_LOG_ERROR,
+                                       "create command parsing failed.");
+                               goto out;
+                       }
+                       break;
+               }
+                case GF_SNAP_OPTION_TYPE_INFO:
+                {
+                        /* Syntax :
+                         * gluster snapshot info [(snapname] | [vol <volname>)]
+                         */
+                        ret = cli_snap_info_parse (dict, words, wordcount);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
+                                        "snapshot info command");
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_OPTION_TYPE_LIST:
+                {
+                        /* Syntax :
+                         * gluster snaphsot list [volname]
+                         */
+
+                        ret = cli_snap_list_parse (dict, words, wordcount);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
+                                        "snapshot list command");
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_OPTION_TYPE_DELETE:
+                {
+                        /* Syntax :
+                         * gluster snapshot delete <snapname>
+                         */
+                        ret = cli_snap_delete_parse (dict, words, wordcount,
+                                                     state);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
+                                        "snapshot delete command");
+                                goto out;
+                        }
+                        break;
+                }
+
+               case GF_SNAP_OPTION_TYPE_CONFIG:
+               {
+                /* snapshot config [volname]  [snap-max-hard-limit <count>]
+                 *                            [snap-max-soft-limit <percent>] */
+                       ret = cli_snap_config_parse (words, wordcount, dict,
+                                                    state);
+                       if (ret) {
+                                if (ret < 0)
+                                        gf_log ("cli", GF_LOG_ERROR,
+                                              "config command parsing failed.");
+                               goto out;
+                       }
+
+                       ret = dict_set_int32 (dict, "type",
+                                             GF_SNAP_OPTION_TYPE_CONFIG);
+                       if (ret) {
+                               gf_log ("cli", GF_LOG_ERROR, "Unable to set "
+                                       "config type");
+                               ret = -1;
+                               goto out;
+                       }
+                       break;
+               }
+                case GF_SNAP_OPTION_TYPE_STATUS:
+                {
+                        /* Syntax :
+                         * gluster snapshot status [(snapname |
+                         *                         volume <volname>)]
+                         */
+                        ret = cli_snap_status_parse (dict, words, wordcount);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
+                                        "snapshot status command");
+                                goto out;
+                        }
+                        break;
+                }
+
+               case GF_SNAP_OPTION_TYPE_RESTORE:
+               {
+                       /* Syntax:
+                        * snapshot restore <snapname>
+                        */
+                       ret = cli_snap_restore_parse (dict, words, wordcount);
+                       if (ret) {
+                               gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
+                                       "restore command");
+                               goto out;
+                       }
+                       break;
+               }
+               default:
+                       gf_log ("", GF_LOG_ERROR, "Opword Mismatch");
+                       goto out;
+                       break;
+        }
+
+        ret = dict_set_int32 (dict, "type", type);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Failed to set type.");
+                goto out;
+        }
+        /* If you got so far, input is valid */
+        ret = 0;
+out:
+        if (ret) {
+                if (dict)
+                        dict_destroy (dict);
+        } else
+                *options = dict;
+
+        return ret;
+}
diff --git a/cli/src/cli-cmd-peer.c b/cli/src/cli-cmd-peer.c
index cad520dfc..551312411 100644
--- a/cli/src/cli-cmd-peer.c
+++ b/cli/src/cli-cmd-peer.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -51,6 +41,7 @@ cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t                  *dict = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
         if (!(wordcount == 3)) {
                 cli_usage_out (word->pattern);
@@ -72,12 +63,14 @@ cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (ret)
                 goto out;
 
-        ret = valid_internet_address ((char *) words[2]);
+        ret = valid_internet_address ((char *) words[2], _gf_false);
         if (ret == 1) {
                 ret = 0;
         } else {
+                cli_out ("%s is an invalid address", words[2]);
                 cli_usage_out (word->pattern);
                 parse_error = 1;
+                ret = -1;
                 goto out;
         }
 /*        if (words[3]) {
@@ -86,6 +79,9 @@ cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
                         goto out;
         }
 */
+
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
@@ -96,6 +92,9 @@ out:
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Peer probe failed");
         }
+
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -111,6 +110,7 @@ cli_cmd_peer_deprobe_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                  flags = 0;
         int                  sent = 0;
         int                  parse_error = 0;
+        cli_local_t          *local = NULL;
 
         if ((wordcount < 3) || (wordcount > 4)) {
                 cli_usage_out (word->pattern);
@@ -150,6 +150,8 @@ cli_cmd_peer_deprobe_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (ret)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
@@ -161,6 +163,8 @@ out:
                         cli_out ("Peer detach failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -187,7 +191,7 @@ cli_cmd_peer_status_cbk (struct cli_state *state, struct cli_cmd_word *word,
                 goto out;
 
         if (proc->fn) {
-                ret = proc->fn (frame, THIS, (char *)words[1] );
+                ret = proc->fn (frame, THIS, (void *)GF_CLI_LIST_PEERS);
         }
 
 out:
@@ -196,6 +200,48 @@ out:
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Peer status failed");
         }
+
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
+
+int
+cli_cmd_pool_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
+{
+        int                     ret = -1;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        int                     sent = 0;
+        int                     parse_error = 0;
+
+        if (wordcount != 2) {
+                cli_usage_out (word->pattern);
+                parse_error = 1;
+                goto out;
+        }
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto out;
+
+        if (proc->fn) {
+                ret = proc->fn (frame, THIS,
+                                (void *)GF_CLI_LIST_POOL_NODES);
+        }
+
+out:
+        if (ret) {
+                cli_cmd_sent_status_get (&sent);
+                if ((sent == 0) && (parse_error == 0))
+                        cli_err ("pool list: command execution failed");
+        }
+
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -216,6 +262,10 @@ struct cli_cmd cli_probe_cmds[] = {
            cli_cmd_peer_help_cbk,
            "Help command for peer "},
 
+        { "pool list",
+          cli_cmd_pool_list_cbk,
+          "list all the nodes in the pool (including localhost)"},
+
         { NULL, NULL, NULL }
 };
 
diff --git a/cli/src/cli-cmd-snapshot.c b/cli/src/cli-cmd-snapshot.c
new file mode 100644
index 000000000..de492d683
--- /dev/null
+++ b/cli/src/cli-cmd-snapshot.c
@@ -0,0 +1,146 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <pthread.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "cli.h"
+#include "cli-cmd.h"
+
+extern rpc_clnt_prog_t *cli_rpc_prog;
+
+int
+cli_cmd_snapshot_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+                           const char **words, int wordcount);
+
+int
+cli_cmd_snapshot_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
+{
+        int                      ret       = 0;
+        int                      parse_err = 0;
+        dict_t                  *options   = NULL;
+        rpc_clnt_procedure_t    *proc      = NULL;
+        call_frame_t            *frame     = NULL;
+        cli_local_t             *local     = NULL;
+
+        proc = &cli_rpc_prog->proctable [GLUSTER_CLI_SNAP];
+        if (proc == NULL) {
+               ret = -1;
+                goto out;
+        }
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (frame == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        /* Parses the command entered by the user */
+        ret = cli_cmd_snapshot_parse (words, wordcount, &options, state);
+        if (ret) {
+                if (ret < 0) {
+                        cli_usage_out (word->pattern);
+                        parse_err = 1;
+                }
+                else {
+                        /* User might have cancelled the snapshot operation */
+                        ret = 0;
+                }
+                goto out;
+        }
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
+        if (proc->fn)
+                ret = proc->fn (frame, THIS, options);
+
+out:
+        if (ret && parse_err == 0)
+                cli_out ("Snapshot command failed");
+
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
+
+struct cli_cmd snapshot_cmds[] = {
+        { "snapshot help",
+          cli_cmd_snapshot_help_cbk,
+          "display help for snapshot commands"
+        },
+        { "snapshot create <snapname> <volname(s)> [description <description>] [force]",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Create."
+        },
+        { "snapshot restore <snapname>",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Restore."
+        },
+        { "snapshot status [(snapname | volume <volname>)]",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Status."
+        },
+        { "snapshot info [(snapname | volume <volname>)]",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Info."
+        },
+        { "snapshot list [volname]",
+          cli_cmd_snapshot_cbk,
+          "Snapshot List."
+        },
+        {"snapshot config [volname] [snap-max-hard-limit <count>] [snap-max-soft-limit <percent>]",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Config."
+        },
+        {"snapshot delete <snapname>",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Delete."
+        },
+        { NULL, NULL, NULL }
+};
+
+int
+cli_cmd_snapshot_help_cbk (struct cli_state *state,
+                           struct cli_cmd_word *in_word,
+                           const char **words,
+                           int wordcount)
+{
+        struct cli_cmd        *cmd = NULL;
+
+        for (cmd = snapshot_cmds; cmd->pattern; cmd++)
+                if (_gf_false == cmd->disable)
+                        cli_out ("%s - %s", cmd->pattern, cmd->desc);
+
+        return 0;
+}
+
+int
+cli_cmd_snapshot_register (struct cli_state *state)
+{
+        int  ret = 0;
+        struct cli_cmd *cmd = NULL;
+
+        for (cmd = snapshot_cmds; cmd->pattern; cmd++) {
+
+                ret = cli_cmd_register (&state->tree, cmd);
+                if (ret)
+                        goto out;
+        }
+out:
+        return ret;
+}
diff --git a/cli/src/cli-cmd-system.c b/cli/src/cli-cmd-system.c
index 25938b897..8cfa5e70c 100644
--- a/cli/src/cli-cmd-system.c
+++ b/cli/src/cli-cmd-system.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -41,6 +31,12 @@ extern rpc_clnt_prog_t *cli_rpc_prog;
 int cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
                              const char **words, int wordcount);
 
+int cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount);
+
+int cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount);
+
 int
 cli_cmd_getspec_cbk (struct cli_state *state, struct cli_cmd_word *word,
                      const char **words, int wordcount)
@@ -288,6 +284,114 @@ cli_cmd_umount_cbk (struct cli_state *state, struct cli_cmd_word *word,
         return ret;
 }
 
+int
+cli_cmd_uuid_get_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
+{
+        int                     ret = -1;
+        int                     sent = 0;
+        int                     parse_error = 0;
+        dict_t                  *dict  = NULL;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        cli_local_t             *local = NULL;
+        xlator_t                *this  = NULL;
+
+        this = THIS;
+        if (wordcount != 3) {
+                cli_usage_out (word->pattern);
+                parse_error = 1;
+                goto out;
+        }
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        CLI_LOCAL_INIT (local, words, frame, dict);
+        if (proc->fn)
+                ret = proc->fn (frame, this, dict);
+
+out:
+        if (ret) {
+                cli_cmd_sent_status_get (&sent);
+                if ((sent == 0) && (parse_error == 0))
+                        cli_out ("uuid get failed");
+        }
+
+        if (dict)
+                dict_unref (dict);
+
+        CLI_STACK_DESTROY (frame);
+        return ret;
+}
+
+int
+cli_cmd_uuid_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
+{
+        int                     ret = -1;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        int                     sent = 0;
+        int                     parse_error = 0;
+        gf_answer_t             answer = GF_ANSWER_NO;
+        char                    *question = NULL;
+        cli_local_t             *local = NULL;
+        dict_t                  *dict  = NULL;
+        xlator_t                *this  = NULL;
+
+        question = "Resetting uuid changes the uuid of local glusterd. "
+                   "Do you want to continue?";
+
+        if (wordcount != 3) {
+                cli_usage_out (word->pattern);
+                parse_error = 1;
+                goto out;
+        }
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
+
+        this = THIS;
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+        CLI_LOCAL_INIT (local, words, frame, dict);
+        answer = cli_cmd_get_confirmation (state, question);
+
+        if (GF_ANSWER_NO == answer) {
+                ret = 0;
+                goto out;
+        }
+
+        //send NULL as argument since no dictionary is sent to glusterd
+        if (proc->fn) {
+                ret = proc->fn (frame, this, dict);
+        }
+
+out:
+        if (ret) {
+                cli_cmd_sent_status_get (&sent);
+                if ((sent == 0) && (parse_error == 0))
+                        cli_out ("uuid reset failed");
+        }
+
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
+
 struct cli_cmd cli_system_cmds[] = {
         { "system:: getspec <VOLID>",
           cli_cmd_getspec_cbk,
@@ -313,14 +417,163 @@ struct cli_cmd cli_system_cmds[] = {
           cli_cmd_umount_cbk,
           "request an umount"},
 
+        { "system:: uuid get",
+          cli_cmd_uuid_get_cbk,
+          "get uuid of glusterd"},
+
+        { "system:: uuid reset",
+          cli_cmd_uuid_reset_cbk,
+          "reset the uuid of glusterd"},
+
         { "system:: help",
            cli_cmd_system_help_cbk,
            "display help for system commands"},
 
+        { "system:: copy file [<filename>]",
+           cli_cmd_copy_file_cbk,
+           "Copy file from current node's $working_dir to "
+           "$working_dir of all cluster nodes"},
+
+        { "system:: execute <command> <args>",
+           cli_cmd_sys_exec_cbk,
+           "Execute the command on all the nodes "
+           "in the cluster and display their output."},
+
         { NULL, NULL, NULL }
 };
 
 int
+cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
+{
+        char                   cmd_arg_name[PATH_MAX] = "";
+        char                  *command                = NULL;
+        char                  *saveptr                = NULL;
+        char                  *tmp                    = NULL;
+        int                    ret                    = -1;
+        int                    i                      = -1;
+        int                    cmd_args_count         = 0;
+        int                    in_cmd_args_count      = 0;
+        rpc_clnt_procedure_t  *proc                   = NULL;
+        call_frame_t          *frame                  = NULL;
+        dict_t                *dict                   = NULL;
+        cli_local_t           *local                  = NULL;
+
+        if (wordcount < 3) {
+                cli_usage_out (word->pattern);
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        command = strtok_r ((char *)words[2], " ", &saveptr);
+        do {
+                tmp = strtok_r (NULL, " ", &saveptr);
+                if (tmp) {
+                        in_cmd_args_count++;
+                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
+                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
+                                  "cmd_arg_%d", in_cmd_args_count);
+                        ret = dict_set_str (dict, cmd_arg_name, tmp);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR, "Unable to set "
+                                        "%s in dict", cmd_arg_name);
+                                goto out;
+                        }
+                }
+        } while (tmp);
+
+        cmd_args_count = wordcount - 3;
+
+        ret = dict_set_str (dict, "command", command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to set command in dict");
+                goto out;
+        }
+
+        for (i=1; i <= cmd_args_count; i++) {
+                in_cmd_args_count++;
+                memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
+                snprintf (cmd_arg_name, sizeof(cmd_arg_name),
+                          "cmd_arg_%d", in_cmd_args_count);
+                ret = dict_set_str (dict, cmd_arg_name,
+                                    (char *)words[2+i]);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to set %s in dict",
+                               cmd_arg_name);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "cmd_args_count", in_cmd_args_count);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to set cmd_args_count in dict");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "volname", "N/A");
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
+                goto out;
+        }
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
+        if (proc && proc->fn) {
+                frame = create_frame (THIS, THIS->ctx->pool);
+                if (!frame)
+                        goto out;
+                CLI_LOCAL_INIT (local, words, frame, dict);
+                ret = proc->fn (frame, THIS, (void*)dict);
+        }
+out:
+        return ret;
+}
+
+int
+cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
+{
+        int                    ret      = -1;
+        rpc_clnt_procedure_t  *proc     = NULL;
+        call_frame_t          *frame    = NULL;
+        char                  *filename = "";
+        dict_t                *dict     = NULL;
+        cli_local_t           *local    = NULL;
+
+        if (wordcount != 4) {
+                cli_usage_out (word->pattern);
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        filename = (char*)words[3];
+        ret = dict_set_str (dict, "source", filename);
+        if (ret)
+                 gf_log ("", GF_LOG_ERROR, "Unable to set filename in dict");
+
+        ret = dict_set_str (dict, "volname", "N/A");
+        if (ret)
+                 gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
+        if (proc && proc->fn) {
+                frame = create_frame (THIS, THIS->ctx->pool);
+                if (!frame)
+                        goto out;
+                CLI_LOCAL_INIT (local, words, frame, dict);
+                ret = proc->fn (frame, THIS, (void*)dict);
+        }
+out:
+        return ret;
+}
+
+int
 cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
                          const char **words, int wordcount)
 {
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 521e3eb84..100be0b73 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -105,8 +95,7 @@ out:
                         cli_out ("Getting Volume information failed!");
         }
 
-        if (frame)
-                FRAME_DESTROY (frame);
+        CLI_STACK_DESTROY (frame);
 
         return ret;
 
@@ -122,6 +111,12 @@ cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     sent = 0;
         int                     parse_error = 0;
         dict_t                  *dict = NULL;
+        cli_local_t             *local = NULL;
+        gf_answer_t             answer = GF_ANSWER_NO;
+        const char              *question = "Sync volume may make data "
+                                            "inaccessible while the sync "
+                                            "is in progress. Do you want "
+                                            "to continue?";
 
         if ((wordcount < 3) || (wordcount > 4)) {
                 cli_usage_out (word->pattern);
@@ -156,12 +151,22 @@ cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
                 goto out;
         }
 
+        if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
+                answer = cli_cmd_get_confirmation (state, question);
+                if (GF_ANSWER_NO == answer) {
+                        ret = 0;
+                        goto out;
+                }
+        }
+
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
@@ -173,8 +178,8 @@ out:
                         cli_out ("Volume sync failed");
         }
 
-        if (dict)
-                dict_unref (dict);
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -315,13 +320,11 @@ found_bad_brick_order:
 out:
         ai_list_tmp2 = NULL;
         i = 0;
-        if (brick_list_dup)
-                GF_FREE (brick_list_dup);
+        GF_FREE (brick_list_dup);
         list_for_each_entry (ai_list_tmp1, &ai_list->list, list) {
                 if (ai_list_tmp1->info)
                         freeaddrinfo (ai_list_tmp1->info);
-                if (ai_list_tmp2)
-                        free (ai_list_tmp2);
+                free (ai_list_tmp2);
                 ai_list_tmp2 = ai_list_tmp1;
         }
         free (ai_list_tmp2);
@@ -342,7 +345,7 @@ cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int32_t                 brick_count = 0;
         int32_t                 sub_count = 0;
         int32_t                 type = GF_CLUSTER_TYPE_NONE;
-
+        cli_local_t             *local = NULL;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
 
@@ -388,19 +391,31 @@ cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
                         goto out;
                 }
         }
+
+        if (state->mode & GLUSTER_MODE_SCRIPT) {
+                ret = dict_set_int32 (options, "force", _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
+                                "option");
+                        goto out;
+                }
+        }
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume create failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -417,6 +432,8 @@ cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
         const char              *question = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
+        dict_t                  *dict = NULL;
 
         question = "Deleting volume will erase all information about the volume. "
                    "Do you want to continue?";
@@ -426,6 +443,10 @@ cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
         if (wordcount != 3) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
@@ -441,8 +462,17 @@ cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
 
         volname = (char *)words[2];
 
+        ret = dict_set_str (dict, "volname", volname);
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, "dict set failed");
+                goto out;
+        }
+
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
-                ret = proc->fn (frame, THIS, volname);
+                ret = proc->fn (frame, THIS, dict);
         }
 
 out:
@@ -452,6 +482,8 @@ out:
                         cli_out ("Volume delete failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -466,6 +498,7 @@ cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     parse_error = 0;
         dict_t                  *dict = NULL;
         int                     flags = 0;
+        cli_local_t             *local = NULL;
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
@@ -516,19 +549,21 @@ cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
 
 out:
-        if (dict)
-                dict_unref (dict);
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume start failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -537,7 +572,7 @@ cli_cmd_get_confirmation (struct cli_state *state, const char *question)
 {
         char                    answer[5] = {'\0', };
         char                    flush = '\0';
-	int			len = 0;
+        size_t			len;
 
         if (state->mode & GLUSTER_MODE_SCRIPT)
                 return GF_ANSWER_YES;
@@ -551,7 +586,7 @@ cli_cmd_get_confirmation (struct cli_state *state, const char *question)
 
 	len = strlen (answer);
 
-	if (answer [len - 1] == '\n'){
+	if (len && answer [len - 1] == '\n'){
 		answer [--len] = '\0';
 	} else {
 		do{
@@ -587,6 +622,7 @@ cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     parse_error = 0;
         dict_t                  *dict = NULL;
         char                    *volname = NULL;
+        cli_local_t             *local = NULL;
 
         const char *question = "Stopping volume will make its data inaccessible. "
                                "Do you want to continue?";
@@ -636,6 +672,8 @@ cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
@@ -646,8 +684,9 @@ out:
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume stop on '%s' failed", volname);
         }
-        if (dict)
-                dict_unref (dict);
+
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -704,6 +743,8 @@ out:
                         cli_out ("Volume rename on '%s' failed", (char *)words[2]);
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -717,7 +758,7 @@ cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t               *dict = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
-        int                     index = 0;
+        cli_local_t          *local = NULL;
 #ifdef GF_SOLARIS_HOST_OS
         cli_out ("Command not supported on Solaris");
         goto out;
@@ -727,86 +768,30 @@ cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+        ret = cli_cmd_volume_defrag_parse (words, wordcount, &dict);
 
-        if (!((wordcount == 4) || (wordcount == 5) || (wordcount == 6))) {
+        if (ret) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
-                goto out;
-        }
-
-        if (wordcount == 4) {
-                index = 3;
-        } else {
-                if (strcmp (words[3], "fix-layout") &&
-                    strcmp (words[3], "migrate-data")) {
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-                index = 4;
-        }
-
-	if (strcmp (words[index], "start") && strcmp (words[index], "stop") &&
-            strcmp (words[index], "status")) {
-	        cli_usage_out (word->pattern);
-		parse_error = 1;
-		goto out;
-	}
-
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        if (wordcount == 4) {
-                ret = dict_set_str (dict, "command", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-        if (wordcount == 5) {
-                ret = dict_set_str (dict, "start-type", (char *)words[3]);
-                if (ret)
-                        goto out;
-                ret = dict_set_str (dict, "command", (char *)words[4]);
-                if (ret)
-                        goto out;
-        }
-
-        /* 'force' option is valid only for the 'migrate-data' key */
-        if (wordcount == 6) {
-                if (strcmp (words[3], "migrate-data") ||
-                    strcmp (words[4], "start") ||
-                    strcmp (words[5], "force")) {
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-                ret = dict_set_str (dict, "start-type", "migrate-data-force");
-                if (ret)
-                        goto out;
-                ret = dict_set_str (dict, "command", (char *)words[4]);
-                if (ret)
-                        goto out;
         }
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
 
 out:
-        if (dict)
-                dict_destroy (dict);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume rebalance failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -816,11 +801,11 @@ cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
 {
         int                     sent = 0;
         int                     parse_error = 0;
-
         int                     ret = -1;
         rpc_clnt_procedure_t    *proc = NULL;
         call_frame_t            *frame = NULL;
         dict_t                  *options = NULL;
+        cli_local_t             *local = NULL;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
 
@@ -829,27 +814,27 @@ cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
                 goto out;
 
         ret = cli_cmd_volume_reset_parse (words, wordcount, &options);
-
         if (ret) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
                 goto out;
         }
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume reset failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -865,6 +850,7 @@ cli_cmd_volume_profile_cbk (struct cli_state *state, struct cli_cmd_word *word,
         rpc_clnt_procedure_t    *proc    = NULL;
         call_frame_t            *frame   = NULL;
         dict_t                  *options = NULL;
+        cli_local_t             *local = NULL;
 
         ret = cli_cmd_volume_profile_parse (words, wordcount, &options);
 
@@ -880,20 +866,21 @@ cli_cmd_volume_profile_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume profile failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -909,6 +896,8 @@ cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
         rpc_clnt_procedure_t    *proc = NULL;
         call_frame_t            *frame = NULL;
         dict_t                  *options = NULL;
+        cli_local_t             *local = NULL;
+        char                    *op_errstr = NULL;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
 
@@ -916,28 +905,33 @@ cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
-        ret = cli_cmd_volume_set_parse (words, wordcount, &options);
-
+        ret = cli_cmd_volume_set_parse (words, wordcount, &options, &op_errstr);
         if (ret) {
-                cli_usage_out (word->pattern);
+                if (op_errstr) {
+                    cli_err ("%s", op_errstr);
+                    GF_FREE (op_errstr);
+                } else
+                    cli_usage_out (word->pattern);
+
                 parse_error = 1;
                 goto out;
         }
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume set failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -953,35 +947,63 @@ cli_cmd_volume_add_brick_cbk (struct cli_state *state,
         dict_t                  *options = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        gf_answer_t             answer = GF_ANSWER_NO;
+        cli_local_t             *local = NULL;
+
+        const char *question = "Changing the 'stripe count' of the volume is "
+                "not a supported feature. In some cases it may result in data "
+                "loss on the volume. Also there may be issues with regular "
+                "filesystem operations on the volume after the change. Do you "
+                "really want to continue with 'stripe' count option ? ";
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
                 goto out;
 
         ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options);
-
         if (ret) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
                 goto out;
         }
 
+        /* TODO: there are challenges in supporting changing of
+           stripe-count, untill it is properly supported give warning to user */
+        if (dict_get (options, "stripe-count")) {
+                answer = cli_cmd_get_confirmation (state, question);
+
+                if (GF_ANSWER_NO == answer) {
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        if (state->mode & GLUSTER_MODE_SCRIPT) {
+                ret = dict_set_int32 (options, "force", _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
+                                "option");
+                        goto out;
+                }
+        }
+
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume add-brick failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -997,6 +1019,7 @@ cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
         call_frame_t            *frame     = NULL;
         dict_t                  *options   = NULL;
         gf_answer_t              answer    = GF_ANSWER_NO;
+        cli_local_t             *local     = NULL;
         const char *question = "Disabling quota will delete all the quota "
                                "configuration. Do you want to continue?";
 
@@ -1013,6 +1036,7 @@ cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
         }
 
         ret = cli_cmd_quota_parse (words, wordcount, &options);
+
         if (ret < 0) {
                 cli_usage_out (word->pattern);
                 parse_err = 1;
@@ -1024,16 +1048,17 @@ cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
                         goto out;
         }
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn)
                 ret = proc->fn (frame, THIS, options);
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret && parse_err == 0)
                 cli_out ("Quota command failed");
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -1051,6 +1076,7 @@ cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
         int                     sent = 0;
         int                     parse_error = 0;
         int                     need_question = 0;
+        cli_local_t             *local = NULL;
 
         const char *question = "Removing brick(s) can result in data loss. "
                                "Do you want to Continue?";
@@ -1061,7 +1087,6 @@ cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
 
         ret = cli_cmd_volume_remove_brick_parse (words, wordcount, &options,
                                                  &need_question);
-
         if (ret) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
@@ -1079,6 +1104,8 @@ cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
@@ -1090,8 +1117,8 @@ out:
                         cli_out ("Volume remove-brick failed");
         }
 
-        if (options)
-                dict_unref (options);
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -1108,6 +1135,7 @@ cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
         dict_t                  *options = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
 #ifdef GF_SOLARIS_HOST_OS
         cli_out ("Command not supported on Solaris");
@@ -1127,20 +1155,30 @@ cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
                 goto out;
         }
 
+        if (state->mode & GLUSTER_MODE_SCRIPT) {
+                ret = dict_set_int32 (options, "force", _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force"
+                                "option");
+                        goto out;
+                }
+        }
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume replace-brick failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -1165,6 +1203,7 @@ cli_cmd_volume_top_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t                  *options = NULL;
         int                     sent     = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
         ret = cli_cmd_volume_top_parse (words, wordcount, &options);
 
@@ -1180,20 +1219,21 @@ cli_cmd_volume_top_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume top failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -1209,6 +1249,7 @@ cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t                  *options = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
         if (!((wordcount == 4) || (wordcount == 5))) {
                 cli_usage_out (word->pattern);
@@ -1226,19 +1267,19 @@ cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (ret)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_destroy (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume log rotate failed");
         }
+        CLI_STACK_DESTROY (frame);
 
         return ret;
 }
@@ -1312,6 +1353,7 @@ cli_cmd_volume_gsync_set_cbk (struct cli_state *state, struct cli_cmd_word *word
         dict_t                  *options = NULL;
         rpc_clnt_procedure_t    *proc    = NULL;
         call_frame_t            *frame   = NULL;
+        cli_local_t             *local   = NULL;
 
         proc = &cli_rpc_prog->proctable [GLUSTER_CLI_GSYNC_SET];
         if (proc == NULL) {
@@ -1332,16 +1374,17 @@ cli_cmd_volume_gsync_set_cbk (struct cli_state *state, struct cli_cmd_word *word
                 goto out;
         }
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn)
                 ret = proc->fn (frame, THIS, options);
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret && parse_err == 0)
                 cli_out (GEOREP" command failed");
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -1355,6 +1398,7 @@ cli_cmd_volume_status_cbk (struct cli_state *state,
         call_frame_t         *frame       = NULL;
         dict_t               *dict        = NULL;
         uint32_t              cmd         = 0;
+        cli_local_t          *local       = NULL;
 
         ret = cli_cmd_volume_status_parse (words, wordcount, &dict);
 
@@ -1382,11 +1426,13 @@ cli_cmd_volume_status_cbk (struct cli_state *state,
         if (!frame)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         ret = proc->fn (frame, THIS, dict);
 
- out:
-        if (dict)
-                dict_unref (dict);
+out:
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -1394,16 +1440,14 @@ cli_cmd_volume_status_cbk (struct cli_state *state,
 int
 cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status)
 {
-        uint64_t                   free            = -1;
-        uint64_t                   total           = -1;
+        uint64_t                   free            = 0;
+        uint64_t                   total           = 0;
         char                       key[1024]       = {0};
         int                        ret             = 0;
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.free", i);
         ret = dict_get_uint64 (dict, key, &free);
-        if (ret)
-                goto out;
 
         status->free = gf_uint64_2human_readable (free);
         if (!status->free)
@@ -1412,64 +1456,65 @@ cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status)
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.total", i);
         ret = dict_get_uint64 (dict, key, &total);
-        if (ret)
-                goto out;
 
         status->total = gf_uint64_2human_readable (total);
         if (!status->total)
                 goto out;
 
+#ifdef GF_LINUX_HOST_OS
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.device", i);
         ret = dict_get_str (dict, key, &(status->device));
         if (ret)
-                goto out;
+                status->device = NULL;
+#endif
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.block_size", i);
         ret = dict_get_uint64 (dict, key, &(status->block_size));
-        if (ret)
-                goto out;
+        if (ret) {
+                ret = 0;
+                status->block_size = 0;
+        }
 
+#ifdef GF_LINUX_HOST_OS
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.mnt_options", i);
         ret = dict_get_str (dict, key, &(status->mount_options));
         if (ret)
-                goto out;
+                status->mount_options = NULL;
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.fs_name", i);
         ret = dict_get_str (dict, key, &(status->fs_name));
-        if (ret)
-                goto out;
-
-        if (IS_EXT_FS(status->fs_name) ||
-            !strcmp (status->fs_name, "xfs")) {
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.inode_size", i);
-                ret = dict_get_str (dict, key, &(status->inode_size));
-                if (ret)
-                        status->inode_size = NULL;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.total_inodes", i);
-                ret = dict_get_uint64 (dict, key, &(status->total_inodes));
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.free_inodes", i);
-                ret = dict_get_uint64 (dict, key, &(status->free_inodes));
-                if (ret)
-                        goto out;
+        if (ret) {
+                ret = 0;
+                status->fs_name = NULL;
+        }
 
-        } else {
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.inode_size", i);
+        ret = dict_get_str (dict, key, &(status->inode_size));
+        if (ret)
                 status->inode_size = NULL;
+#endif /* GF_LINUX_HOST_OS */
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.total_inodes", i);
+        ret = dict_get_uint64 (dict, key,
+                        &(status->total_inodes));
+        if (ret)
                 status->total_inodes = 0;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.free_inodes", i);
+        ret = dict_get_uint64 (dict, key, &(status->free_inodes));
+        if (ret) {
+                ret = 0;
                 status->free_inodes = 0;
         }
 
+
  out:
         return ret;
 }
@@ -1478,11 +1523,23 @@ void
 cli_print_detailed_status (cli_volume_status_t *status)
 {
         cli_out ("%-20s : %-20s", "Brick", status->brick);
-        cli_out ("%-20s : %-20d", "Port", status->port);
+        if (status->online)
+                cli_out ("%-20s : %-20d", "Port", status->port);
+        else
+                cli_out ("%-20s : %-20s", "Port", "N/A");
         cli_out ("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
         cli_out ("%-20s : %-20s", "Pid", status->pid_str);
-        cli_out ("%-20s : %-20s", "File System", status->fs_name);
-        cli_out ("%-20s : %-20s", "Device", status->device);
+
+#ifdef GF_LINUX_HOST_OS
+        if (status->fs_name)
+                cli_out ("%-20s : %-20s", "File System", status->fs_name);
+        else
+                cli_out ("%-20s : %-20s", "File System", "N/A");
+
+        if (status->device)
+                cli_out ("%-20s : %-20s", "Device", status->device);
+        else
+                cli_out ("%-20s : %-20s", "Device", "N/A");
 
         if (status->mount_options) {
                 cli_out ("%-20s : %-20s", "Mount Options",
@@ -1491,15 +1548,23 @@ cli_print_detailed_status (cli_volume_status_t *status)
                 cli_out ("%-20s : %-20s", "Mount Options", "N/A");
         }
 
-        cli_out ("%-20s : %-20s", "Disk Space Free", status->free);
-        cli_out ("%-20s : %-20s", "Total Disk Space", status->total);
-
         if (status->inode_size) {
                 cli_out ("%-20s : %-20s", "Inode Size",
                          status->inode_size);
         } else {
                 cli_out ("%-20s : %-20s", "Inode Size", "N/A");
         }
+#endif
+        if (status->free)
+                cli_out ("%-20s : %-20s", "Disk Space Free", status->free);
+        else
+                cli_out ("%-20s : %-20s", "Disk Space Free", "N/A");
+
+        if (status->total)
+                cli_out ("%-20s : %-20s", "Total Disk Space", status->total);
+        else
+                cli_out ("%-20s : %-20s", "Total Disk Space", "N/A");
+
 
         if (status->total_inodes) {
                 cli_out ("%-20s : %-20ld", "Inode Count",
@@ -1520,30 +1585,38 @@ int
 cli_print_brick_status (cli_volume_status_t *status)
 {
         int  fieldlen = CLI_VOL_STATUS_BRICK_LEN;
-        char buf[80] = {0,};
         int  bricklen = 0;
-        int  i = 0;
         char *p = NULL;
         int  num_tabs = 0;
 
-        bricklen = strlen (status->brick);
         p = status->brick;
+        bricklen = strlen (p);
         while (bricklen > 0) {
                 if (bricklen > fieldlen) {
-                        i++;
-                        strncpy (buf, p, fieldlen);
-                        buf[strlen(buf) + 1] = '\0';
-                        cli_out ("%s", buf);
-                        p = status->brick + i * fieldlen;
+                        cli_out ("%.*s", fieldlen, p);
+                        p += fieldlen;
                         bricklen -= fieldlen;
                 } else {
                         num_tabs = (fieldlen - bricklen) / CLI_TAB_LENGTH + 1;
                         printf ("%s", p);
                         while (num_tabs-- != 0)
                                 printf ("\t");
-                        cli_out ("%d\t%c\t%s",
-                                 status->port, status->online?'Y':'N',
-                                 status->pid_str);
+                        if (status->port) {
+                                if (status->online)
+                                        cli_out ("%d\t%c\t%s",
+                                                 status->port,
+                                                 status->online?'Y':'N',
+                                                 status->pid_str);
+                                else
+                                        cli_out ("%s\t%c\t%s",
+                                                 "N/A",
+                                                 status->online?'Y':'N',
+                                                 status->pid_str);
+                        }
+                        else
+                                cli_out ("%s\t%c\t%s",
+                                         "N/A", status->online?'Y':'N',
+                                         status->pid_str);
                         bricklen = 0;
                 }
         }
@@ -1560,32 +1633,34 @@ cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
         call_frame_t            *frame = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
-        dict_t                  *dict = NULL;
+        dict_t                  *options = NULL;
+        xlator_t                *this = NULL;
+        cli_local_t             *local = NULL;
 
-        frame = create_frame (THIS, THIS->ctx->pool);
+        this = THIS;
+        frame = create_frame (this, this->ctx->pool);
         if (!frame)
                 goto out;
 
-        if (wordcount != 3) {
+        if (wordcount < 3) {
                cli_usage_out (word->pattern);
-                parse_error = 1;
+               parse_error = 1;
                goto out;
         }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_str (dict, "volname", (char *) words[2]);
+        ret = cli_cmd_volume_heal_options_parse (words, wordcount, &options);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set volname");
+                cli_usage_out (word->pattern);
+                parse_error = 1;
                 goto out;
         }
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+                ret = proc->fn (frame, THIS, options);
         }
 
 out:
@@ -1595,8 +1670,7 @@ out:
                         cli_out ("Volume heal failed");
         }
 
-        if (dict)
-                dict_unref (dict);
+        CLI_STACK_DESTROY (frame);
 
         return ret;
 }
@@ -1611,6 +1685,7 @@ cli_cmd_volume_statedump_cbk (struct cli_state *state, struct cli_cmd_word *word
         dict_t                          *options = NULL;
         int                             sent = 0;
         int                             parse_error = 0;
+        cli_local_t                     *local = NULL;
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
@@ -1639,6 +1714,9 @@ cli_cmd_volume_statedump_cbk (struct cli_state *state, struct cli_cmd_word *word
                 goto out;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
@@ -1650,16 +1728,113 @@ out:
                         cli_out ("Volume statedump failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
+
+int
+cli_cmd_volume_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
+{
+        int                     ret = -1;
+        call_frame_t            *frame = NULL;
+        rpc_clnt_procedure_t    *proc = NULL;
+        int                     sent = 0;
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto out;
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_VOLUME];
+        if (proc->fn) {
+                ret = proc->fn (frame, THIS, NULL);
+        }
+
+out:
+        if (ret) {
+                cli_cmd_sent_status_get (&sent);
+                if (sent == 0)
+                        cli_out ("Volume list failed");
+        }
+
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
+int
+cli_cmd_volume_clearlocks_cbk (struct cli_state *state,
+                               struct cli_cmd_word *word,
+                               const char **words, int wordcount)
+{
+        int                             ret = -1;
+        rpc_clnt_procedure_t            *proc = NULL;
+        call_frame_t                    *frame = NULL;
+        dict_t                          *options = NULL;
+        int                             sent = 0;
+        int                             parse_error = 0;
+        cli_local_t                     *local = NULL;
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto out;
+
+        if (wordcount < 7 || wordcount > 8) {
+                cli_usage_out (word->pattern);
+                parse_error = 1;
+                goto out;
+        }
+
+       ret = cli_cmd_volume_clrlks_opts_parse (words, wordcount, &options);
+       if (ret) {
+               parse_error = 1;
+               gf_log ("cli", GF_LOG_ERROR, "Error parsing "
+                       "clear-locks options");
+               cli_out ("Error parsing options");
+               cli_usage_out (word->pattern);
+       }
+
+        ret = dict_set_str (options, "volname", (char *)words[2]);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (options, "path", (char *)words[3]);
+        if (ret)
+                goto out;
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
+        if (proc->fn) {
+                ret = proc->fn (frame, THIS, options);
+        }
+
+out:
+        if (ret) {
+                cli_cmd_sent_status_get (&sent);
+                if ((sent == 0) && (parse_error = 0))
+                        cli_out ("Volume clear-locks failed");
+        }
+
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
 
 struct cli_cmd volume_cmds[] = {
         { "volume info [all|<VOLNAME>]",
           cli_cmd_volume_info_cbk,
           "list information of all volumes"},
 
-        { "volume create <NEW-VOLNAME> [stripe <COUNT>] [replica <COUNT>] [transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> ...",
+        { "volume create <NEW-VOLNAME> [stripe <COUNT>] [replica <COUNT>] "
+          "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK>"
+#ifdef HAVE_BD_XLATOR
+          "?<vg_name>"
+#endif
+          "... [force]",
+
           cli_cmd_volume_create_cbk,
           "create a new volume of specified type with mentioned bricks"},
 
@@ -1679,19 +1854,19 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_volume_rename_cbk,
           "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
 
-        { "volume add-brick <VOLNAME> [<stripe|replica> <COUNT>] <NEW-BRICK> ...",
+        { "volume add-brick <VOLNAME> [<stripe|replica> <COUNT>] <NEW-BRICK> ... [force]",
           cli_cmd_volume_add_brick_cbk,
           "add brick to volume <VOLNAME>"},
 
-        { "volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ... {start|pause|abort|status|commit|force}",
+        { "volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ... [start|stop|status|commit|force]",
           cli_cmd_volume_remove_brick_cbk,
           "remove brick from volume <VOLNAME>"},
 
-        { "volume rebalance <VOLNAME> [fix-layout|migrate-data] {start|stop|status} [force]",
+        { "volume rebalance <VOLNAME> [fix-layout] {start|stop|status} [force]",
           cli_cmd_volume_defrag_cbk,
           "rebalance operations"},
 
-        { "volume replace-brick <VOLNAME> <BRICK> <NEW-BRICK> {start|pause|abort|status|commit [force]}",
+        { "volume replace-brick <VOLNAME> <BRICK> <NEW-BRICK> {start [force]|pause|abort|status|commit [force]}",
           cli_cmd_volume_replace_brick_cbk,
           "replace-brick operations"},
 
@@ -1720,13 +1895,14 @@ struct cli_cmd volume_cmds[] = {
          "reset all the reconfigured options"},
 
 #if (SYNCDAEMON_COMPILE)
-        {"volume "GEOREP" [<VOLNAME>] [<SLAVE-URL>] {start|stop|config|status|log-rotate} [options...]",
+        {"volume "GEOREP" [<VOLNAME>] [<SLAVE-URL>] {create [push-pem] [force]"
+         "|start [force]|stop [force]|config|status [detail]|delete} [options...]",
          cli_cmd_volume_gsync_set_cbk,
          "Geo-sync operations",
          cli_cmd_check_gsync_exists_cbk},
 #endif
 
-         { "volume profile <VOLNAME> {start|info|stop}",
+         { "volume profile <VOLNAME> {start|stop|info [nfs]}",
            cli_cmd_volume_profile_cbk,
            "volume profile operations"},
 
@@ -1734,26 +1910,35 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_quota_cbk,
           "quota translator specific operations"},
 
-         { "volume top <VOLNAME> {[open|read|write|opendir|readdir] "
-           "|[read-perf|write-perf bs <size> count <count>]} "
-           " [brick <brick>] [list-cnt <count>]",
+         { "volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick <brick>] [list-cnt <value>] |\n"
+           "volume top <VOLNAME> {read-perf|write-perf} [bs <size> count <count>] [brick <brick>] [list-cnt <value>]",
            cli_cmd_volume_top_cbk,
            "volume top operations"},
 
-        { "volume status [all|{<VOLNAME> [<BRICKNAME>] "
-          "[misc-details|clients|mem|inode|fd|callpool]}]",
+        { "volume status [all | <VOLNAME> [nfs|shd|<BRICK>]]"
+          " [detail|clients|mem|inode|fd|callpool|tasks]",
           cli_cmd_volume_status_cbk,
-         "display status of specified volume"},
+          "display status of all or specified volume(s)/brick"},
 
-        { "volume heal <VOLNAME>",
+        { "volume heal <VOLNAME> [{full | statistics {heal-count {replica <hostname:brickname>}} |info {healed | heal-failed | split-brain}}]",
           cli_cmd_volume_heal_cbk,
-          "Start healing of volume specified by <VOLNAME>"},
+          "self-heal commands on volume specified by <VOLNAME>"},
 
         {"volume statedump <VOLNAME> [nfs] [all|mem|iobuf|callpool|priv|fd|"
-         "inode]...",
+         "inode|history]...",
          cli_cmd_volume_statedump_cbk,
          "perform statedump on bricks"},
 
+        {"volume list",
+         cli_cmd_volume_list_cbk,
+         "list all volumes in cluster"},
+
+        {"volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}"
+          "{inode [range]|entry [basename]|posix [range]}",
+          cli_cmd_volume_clearlocks_cbk,
+          "Clear locks held on path"
+        },
+
         { NULL, NULL, NULL }
 };
 
diff --git a/cli/src/cli-cmd.c b/cli/src/cli-cmd.c
index a7adfa981..b81f75b5b 100644
--- a/cli/src/cli-cmd.c
+++ b/cli/src/cli-cmd.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -60,6 +50,9 @@ cli_cmd_needs_connection (struct cli_cmd_word *word)
         if (!strcasecmp ("getwd", word->word))
                 return 1;
 
+        if (!strcasecmp ("exit", word->word))
+                return 0;
+
         return CLI_DEFAULT_CONN_TIMEOUT;
 }
 
@@ -208,8 +201,7 @@ cli_cmd_process_line (struct cli_state *state, const char *text)
 
         ret = cli_cmd_process (state, count, tokens);
 out:
-        if (copy)
-                free (copy);
+        free (copy);
 
         if (tokens)
                 cli_cmd_tokens_destroy (tokens);
@@ -239,6 +231,9 @@ cli_cmds_register (struct cli_state *state)
         if (ret)
                 goto out;
 
+        ret = cli_cmd_snapshot_register (state);
+        if (ret)
+                goto out;
 out:
         return ret;
 }
@@ -368,8 +363,11 @@ cli_cmd_submit (void *req, call_frame_t *frame,
         int             ret = -1;
         unsigned        timeout = 0;
 
-        timeout = (GLUSTER_CLI_PROFILE_VOLUME == procnum) ?
-                   CLI_TOP_CMD_TIMEOUT : CLI_DEFAULT_CMD_TIMEOUT;
+        if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
+            (GLUSTER_CLI_HEAL_VOLUME == procnum))
+                timeout = CLI_TEN_MINUTES_TIMEOUT;
+        else
+                timeout = CLI_DEFAULT_CMD_TIMEOUT;
 
         cli_cmd_lock ();
         cmd_sent = 0;
diff --git a/cli/src/cli-cmd.h b/cli/src/cli-cmd.h
index 82e0ff113..041729276 100644
--- a/cli/src/cli-cmd.h
+++ b/cli/src/cli-cmd.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLI_CMD_H__
 #define __CLI_CMD_H__
 
@@ -30,6 +20,32 @@
 #include "cli.h"
 #include "list.h"
 
+#define CLI_LOCAL_INIT(local, words, frame, dictionary) \
+        do {                                                 \
+                local = cli_local_get ();                    \
+                                                             \
+                if (local) {                                 \
+                        local->words = words;                \
+                        if (dictionary)                      \
+                                local->dict = dictionary;    \
+                        if (frame)                           \
+                                frame->local = local;        \
+                }                                            \
+        } while (0)
+
+#define CLI_STACK_DESTROY(_frame)                                       \
+        do {                                                            \
+                if (_frame) {                                           \
+                        if (_frame->local) {                            \
+                                gf_log ("cli", GF_LOG_DEBUG, "frame->local " \
+                                        "is not NULL (%p)", _frame->local); \
+                                cli_local_wipe (_frame->local);         \
+                                _frame->local = NULL;                   \
+                        }                                               \
+                        STACK_DESTROY (_frame->root);                   \
+                }                                                       \
+        } while (0);
+
 typedef enum {
         GF_ANSWER_YES = 1,
         GF_ANSWER_NO  = 2
@@ -77,6 +93,8 @@ int cli_cmd_probe_register (struct cli_state *state);
 
 int cli_cmd_system_register (struct cli_state *state);
 
+int cli_cmd_snapshot_register (struct cli_state *state);
+
 int cli_cmd_misc_register (struct cli_state *state);
 
 struct cli_cmd_word *cli_cmd_nextword (struct cli_cmd_word *word,
@@ -102,4 +120,5 @@ cli_cmd_submit (void *req, call_frame_t *frame,
 gf_answer_t
 cli_cmd_get_confirmation (struct cli_state *state, const char *question);
 int cli_cmd_sent_status_get (int *status);
+
 #endif /* __CLI_CMD_H__ */
diff --git a/cli/src/cli-mem-types.h b/cli/src/cli-mem-types.h
index 3c49d2183..09fcb639b 100644
--- a/cli/src/cli-mem-types.h
+++ b/cli/src/cli-mem-types.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLI_MEM_TYPES_H__
 #define __CLI_MEM_TYPES_H__
 
diff --git a/cli/src/cli-rl.c b/cli/src/cli-rl.c
index 9a37a77de..ade1c8ebb 100644
--- a/cli/src/cli-rl.c
+++ b/cli/src/cli-rl.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -48,7 +38,6 @@ cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
 {
         int tmp_rl_point = rl_point;
         int            n = rl_end;
-        int            i = 0;
         int            ret = 0;
 
         if (rl_end >= 0 ) {
@@ -56,12 +45,7 @@ cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
                 rl_redisplay ();
         }
 
-        printf ("\r");
-
-        for (i = 0; i <= strlen (state->prompt); i++)
-                printf (" ");
-
-        printf ("\r");
+        printf ("\r%*s\r", (int)strlen (state->prompt), "");
 
         ret = vprintf (fmt, ap);
 
@@ -77,6 +61,34 @@ cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
         return ret;
 }
 
+int
+cli_rl_err (struct cli_state *state, const char *fmt, va_list ap)
+{
+        int tmp_rl_point = rl_point;
+        int            n = rl_end;
+        int            ret = 0;
+
+        if (rl_end >= 0 ) {
+                rl_kill_text (0, rl_end);
+                rl_redisplay ();
+        }
+
+        fprintf (stderr, "\r%*s\r", (int)strlen (state->prompt), "");
+
+        ret = vfprintf (stderr, fmt, ap);
+
+        fprintf (stderr, "\n");
+        fflush(stderr);
+
+        if (n) {
+                rl_do_undo ();
+                rl_point = tmp_rl_point;
+                rl_reset_line_state ();
+        }
+
+        return ret;
+}
+
 
 void
 cli_rl_process_line (char *line)
@@ -204,8 +216,7 @@ cli_rl_tokenize (const char *text)
         }
 
 out:
-        if (copy)
-                free (copy);
+        free (copy);
 
         if (i < count) {
                 cli_cmd_tokens_destroy (tokens);
@@ -352,9 +363,10 @@ cli_rl_input (void *_data)
         for (;;) {
                 line = readline (state->prompt);
                 if (!line)
-                        break;
+                        exit(0);  //break;
 
-                cli_rl_process_line (line);
+                if (*line)
+                        cli_rl_process_line (line);
 
                 free (line);
         }
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 78b2980ad..bfeb854ad 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -1,33 +1,18 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#ifndef GSYNC_CONF
-#define GSYNC_CONF GEOREP"/gsyncd.conf"
-#endif
-#define DEFAULT_LOG_FILE_DIRECTORY      DATADIR "/log/glusterfs"
-
 /* Widths of various columns in top read/write-perf output
  * Total width of top read/write-perf should be 80 chars
  * including one space between column
@@ -37,6 +22,8 @@
 #define VOL_TOP_PERF_SPEED_WIDTH        4
 #define VOL_TOP_PERF_TIME_WIDTH         26
 
+#define INDENT_MAIN_HEAD "%-25s %s "
+
 #include "cli.h"
 #include "compat-errno.h"
 #include "cli-cmd.h"
@@ -59,25 +46,39 @@ extern rpc_clnt_prog_t *cli_rpc_prog;
 extern int              cli_op_ret;
 extern int              connected;
 
-char *cli_volume_type[] = {"Distribute",
-                           "Stripe",
-                           "Replicate",
-                           "Striped-Replicate",
-                           "Distributed-Stripe",
-                           "Distributed-Replicate",
-                           "Distributed-Striped-Replicate",
-};
-
-
-char *cli_volume_status[] = {"Created",
-                             "Started",
-                             "Stopped"
+char *cli_vol_type_str[] = {"Distribute",
+                            "Stripe",
+                            "Replicate",
+                            "Striped-Replicate",
+                            "Distributed-Stripe",
+                            "Distributed-Replicate",
+                            "Distributed-Striped-Replicate",
+                           };
+
+char *cli_vol_status_str[] = {"Created",
+                              "Started",
+                              "Stopped",
+                             };
+
+char *cli_vol_task_status_str[] = {"not started",
+                                   "in progress",
+                                   "stopped",
+                                   "completed",
+                                   "failed",
+                                   "fix-layout in progress",
+                                   "fix-layout stopped",
+                                   "fix-layout completed",
+                                   "fix-layout failed",
 };
 
 int32_t
-gf_cli3_1_get_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
                       void *data);
 
+int
+cli_to_glusterd (gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
+                 xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
+                 rpc_clnt_prog_t *prog, struct iobref *iobref);
 
 rpc_clnt_prog_t cli_handshake_prog = {
         .progname  = "cli handshake",
@@ -91,76 +92,51 @@ rpc_clnt_prog_t cli_pmap_prog = {
         .progver    = GLUSTER_PMAP_VERSION,
 };
 
-
 int
-gf_cli3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_probe_cbk (struct rpc_req *req, struct iovec *iov,
                         int count, void *myframe)
 {
-        gf1_cli_probe_rsp    rsp   = {0,};
+        gf_cli_rsp            rsp   = {0,};
         int                   ret   = -1;
+        char                  msg[1024] = {0,};
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_probe_rsp);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 //rsp.op_ret   = -1;
                 //rsp.op_errno = EINVAL;
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to probe");
-         if (!rsp.op_ret) {
-                switch (rsp.op_errno) {
-                        case GF_PROBE_SUCCESS:
-                                cli_out ("Probe successful");
-                                break;
-                        case GF_PROBE_LOCALHOST:
-                                cli_out ("Probe on localhost not needed");
-                                break;
-                        case GF_PROBE_FRIEND:
-                                cli_out ("Probe on host %s port %d already"
-                                         " in peer list", rsp.hostname, rsp.port);
-                                break;
-                        default:
-                                cli_out ("Probe returned with unknown errno %d",
-                                        rsp.op_errno);
-                                break;
-                }
-         }
 
-        if (rsp.op_ret) {
-                switch (rsp.op_errno) {
-                        case GF_PROBE_ANOTHER_CLUSTER:
-                                cli_out ("%s is already part of "
-                                         "another cluster", rsp.hostname);
-                                break;
-                        case GF_PROBE_VOLUME_CONFLICT:
-                                cli_out ("Atleast one volume on %s conflicts "
-                                         "with existing volumes in the "
-                                         "cluster", rsp.hostname);
-                                break;
-                        case GF_PROBE_UNKNOWN_PEER:
-                                cli_out ("%s responded with 'unknown peer' error, "
-                                         "this could happen if %s doesn't have"
-                                         " localhost in its peer database",
-                                         rsp.hostname, rsp.hostname);
-                                break;
-                        case GF_PROBE_ADD_FAILED:
-                                cli_out ("Failed to add peer information "
-                                         "on %s" , rsp.hostname);
-                                break;
+        if (rsp.op_errstr && (strlen (rsp.op_errstr) > 0)) {
+                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+                if (rsp.op_ret)
+                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
+        }
 
-                        default:
-                                cli_out ("Probe unsuccessful\nProbe returned "
-                                         "with unknown errno %d", rsp.op_errno);
-                                break;
-                }
-                gf_log ("glusterd",GF_LOG_ERROR,"Probe failed with op_ret %d"
-                        " and op_errno %d", rsp.op_ret, rsp.op_errno);
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_str (NULL,
+                                          (rsp.op_ret)? NULL : msg,
+                                          rsp.op_ret, rsp.op_errno,
+                                          (rsp.op_ret)? msg : NULL);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
         }
+
+        if (!rsp.op_ret)
+                cli_out ("peer probe: success. %s", msg);
+        else
+                cli_err ("peer probe: failed: %s", msg);
+
         ret = rsp.op_ret;
 
 out:
@@ -169,55 +145,52 @@ out:
 }
 
 int
-gf_cli3_1_deprobe_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_deprobe_cbk (struct rpc_req *req, struct iovec *iov,
                        int count, void *myframe)
 {
-        gf1_cli_deprobe_rsp    rsp   = {0,};
+        gf_cli_rsp            rsp   = {0,};
         int                   ret   = -1;
+        char              msg[1024] = {0,};
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_deprobe_rsp);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 //rsp.op_ret   = -1;
                 //rsp.op_errno = EINVAL;
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to deprobe");
+
         if (rsp.op_ret) {
-                switch (rsp.op_errno) {
-                        case GF_DEPROBE_LOCALHOST:
-                                cli_out ("%s is localhost",
-                                         rsp.hostname);
-                                break;
-                        case GF_DEPROBE_NOT_FRIEND:
-                                cli_out ("%s is not part of cluster",
-                                         rsp.hostname);
-                                break;
-                        case GF_DEPROBE_BRICK_EXIST:
-                                cli_out ("Brick(s) with the peer %s exist in "
-                                         "cluster", rsp.hostname);
-                                break;
-                        case GF_DEPROBE_FRIEND_DOWN:
-                                cli_out ("One of the peers is probably down."
-                                         " Check with 'peer status'.");
-                                break;
-                        default:
-                                cli_out ("Detach unsuccessful\nDetach returned "
-                                         "with unknown errno %d",
-                                         rsp.op_errno);
-                                break;
+                if (strlen (rsp.op_errstr) > 0) {
+                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+                        gf_log ("cli", GF_LOG_ERROR, "%s", rsp.op_errstr);
                 }
-                gf_log ("glusterd",GF_LOG_ERROR,"Detach failed with op_ret %d"
-                        " and op_errno %d", rsp.op_ret, rsp.op_errno);
         } else {
-                cli_out ("Detach successful");
+                snprintf (msg, sizeof (msg), "success");
         }
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_str (NULL,
+                                          (rsp.op_ret)? NULL : msg,
+                                          rsp.op_ret, rsp.op_errno,
+                                          (rsp.op_ret)? msg : NULL);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (!rsp.op_ret)
+                cli_out ("peer detach: %s", msg);
+        else
+                cli_err ("peer detach: failed: %s", msg);
 
         ret = rsp.op_ret;
 
@@ -227,35 +200,143 @@ out:
 }
 
 int
-gf_cli3_1_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+gf_cli_output_peer_status (dict_t *dict, int count)
 {
-        gf1_cli_peer_list_rsp      rsp   = {0,};
         int                        ret   = -1;
-        dict_t                     *dict = NULL;
         char                       *uuid_buf = NULL;
         char                       *hostname_buf = NULL;
         int32_t                    i = 1;
         char                       key[256] = {0,};
         char                       *state = NULL;
-        int32_t                    port = 0;
         int32_t                    connected = 0;
         char                       *connected_str = NULL;
 
+        cli_out ("Number of Peers: %d", count);
+        i = 1;
+        while ( i <= count) {
+                snprintf (key, 256, "friend%d.uuid", i);
+                ret = dict_get_str (dict, key, &uuid_buf);
+                if (ret)
+                        goto out;
+
+                snprintf (key, 256, "friend%d.hostname", i);
+                ret = dict_get_str (dict, key, &hostname_buf);
+                if (ret)
+                        goto out;
+
+                snprintf (key, 256, "friend%d.connected", i);
+                ret = dict_get_int32 (dict, key, &connected);
+                if (ret)
+                        goto out;
+                if (connected)
+                        connected_str = "Connected";
+                else
+                        connected_str = "Disconnected";
+
+
+                snprintf (key, 256, "friend%d.state", i);
+                ret = dict_get_str (dict, key, &state);
+                if (ret)
+                        goto out;
+
+                cli_out ("\nHostname: %s\nUuid: %s\nState: %s (%s)",
+                         hostname_buf, uuid_buf, state, connected_str);
+                i++;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+gf_cli_output_pool_list (dict_t *dict, int count)
+{
+        int                        ret   = -1;
+        char                       *uuid_buf = NULL;
+        char                       *hostname_buf = NULL;
+        int32_t                    i = 1;
+        char                       key[256] = {0,};
+        int32_t                    connected = 0;
+        char                       *connected_str = NULL;
+
+        if (count >= 1)
+                cli_out ("UUID\t\t\t\t\tHostname\tState");
+
+        while ( i <= count) {
+                snprintf (key, 256, "friend%d.uuid", i);
+                ret = dict_get_str (dict, key, &uuid_buf);
+                if (ret)
+                        goto out;
+
+                snprintf (key, 256, "friend%d.hostname", i);
+                ret = dict_get_str (dict, key, &hostname_buf);
+                if (ret)
+                        goto out;
+
+                snprintf (key, 256, "friend%d.connected", i);
+                ret = dict_get_int32 (dict, key, &connected);
+                if (ret)
+                        goto out;
+                if (connected)
+                        connected_str = "Connected";
+                else
+                        connected_str = "Disconnected";
+
+                cli_out ("%s\t%-9s\t%s ", uuid_buf, hostname_buf,
+                         connected_str);
+                i++;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/* function pointer for gf_cli_output_{pool_list,peer_status} */
+typedef int (*cli_friend_output_fn) (dict_t*, int);
+
+int
+gf_cli_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
+                             int count, void *myframe)
+{
+        gf1_cli_peer_list_rsp      rsp   = {0,};
+        int                        ret   = -1;
+        dict_t                    *dict = NULL;
+        char                       msg[1024] = {0,};
+        char                      *cmd = NULL;
+        cli_friend_output_fn       friend_output_fn;
+        call_frame_t              *frame = NULL;
+        unsigned long              flags = 0;
+
+        frame = myframe;
+        flags = (long)frame->local;
+
+        if (flags == GF_CLI_LIST_POOL_NODES) {
+                cmd = "pool list";
+                friend_output_fn = &gf_cli_output_pool_list;
+        } else {
+                cmd = "peer status";
+                friend_output_fn = &gf_cli_output_peer_status;
+        }
+
+        /* 'free' the flags set by gf_cli_list_friends */
+        frame->local = NULL;
+
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 //rsp.op_ret   = -1;
                 //rsp.op_errno = EINVAL;
                 goto out;
         }
 
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to list: %d",
+        gf_log ("cli", GF_LOG_DEBUG, "Received resp to list: %d",
                 rsp.op_ret);
 
         ret = rsp.op_ret;
@@ -263,7 +344,19 @@ gf_cli3_1_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
         if (!rsp.op_ret) {
 
                 if (!rsp.friends.friends_len) {
-                        cli_out ("No peers present");
+                        snprintf (msg, sizeof (msg),
+                                  "%s: No peers present", cmd);
+                        if (global_state->mode & GLUSTER_MODE_XML) {
+                                ret = cli_xml_output_peer_status (dict,
+                                                                  rsp.op_ret,
+                                                                  rsp.op_errno,
+                                                                  msg);
+                                if (ret)
+                                        gf_log ("cli", GF_LOG_ERROR,
+                                                "Error outputting to xml");
+                                goto out;
+                        }
+                        cli_err ("%s", msg);
                         ret = 0;
                         goto out;
                 }
@@ -285,58 +378,34 @@ gf_cli3_1_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
                         goto out;
                 }
 
-                ret = dict_get_int32 (dict, "count", &count);
+                if (global_state->mode & GLUSTER_MODE_XML) {
+                        ret = cli_xml_output_peer_status (dict, rsp.op_ret,
+                                                          rsp.op_errno, msg);
+                        if (ret)
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                        goto out;
+                }
 
+                ret = dict_get_int32 (dict, "count", &count);
                 if (ret) {
                         goto out;
                 }
 
-                cli_out ("Number of Peers: %d", count);
-
-                while ( i <= count) {
-                        snprintf (key, 256, "friend%d.uuid", i);
-                        ret = dict_get_str (dict, key, &uuid_buf);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "friend%d.hostname", i);
-                        ret = dict_get_str (dict, key, &hostname_buf);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "friend%d.connected", i);
-                        ret = dict_get_int32 (dict, key, &connected);
-                        if (ret)
-                                goto out;
-                        if (connected)
-                                connected_str = "Connected";
-                        else
-                                connected_str = "Disconnected";
-
-                        snprintf (key, 256, "friend%d.port", i);
-                        ret = dict_get_int32 (dict, key, &port);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "friend%d.state", i);
-                        ret = dict_get_str (dict, key, &state);
-                        if (ret)
-                                goto out;
-
-                        if (!port) {
-                                cli_out ("\nHostname: %s\nUuid: %s\nState: %s "
-                                         "(%s)",
-                                         hostname_buf, uuid_buf, state,
-                                         connected_str);
-                        } else {
-                                cli_out ("\nHostname: %s\nPort: %d\nUuid: %s\n"
-                                         "State: %s (%s)", hostname_buf, port,
-                                         uuid_buf, state, connected_str);
-                        }
-                        i++;
+                ret = friend_output_fn (dict, count);
+                if (ret) {
+                        goto out;
                 }
         } else {
-                ret = -1;
+                if (global_state->mode & GLUSTER_MODE_XML) {
+                        ret = cli_xml_output_peer_status (dict, rsp.op_ret,
+                                                          rsp.op_errno, NULL);
+                        if (ret)
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                } else {
+                        ret = -1;
+                }
                 goto out;
         }
 
@@ -346,7 +415,7 @@ gf_cli3_1_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         cli_cmd_broadcast_response (ret);
         if (ret)
-                cli_out ("Peer status unsuccessful");
+                cli_err ("%s: failed", cmd);
 
         if (dict)
                 dict_destroy (dict);
@@ -380,14 +449,37 @@ cli_out_options ( char *substr, char *optstr, char *valstr)
         cli_out ("%s: %s",ptr2 , valstr);
 }
 
+static int
+_gf_cli_output_volinfo_opts (dict_t *d, char *k,
+                             data_t *v, void *tmp)
+{
+        int     ret   = 0;
+        char   *key   = NULL;
+        char   *ptr   = NULL;
+        data_t *value = NULL;
+
+        key = tmp;
+
+        ptr = strstr (k, "option.");
+        if (ptr) {
+                value = v;
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+                cli_out_options (key, k, v->data);
+        }
+out:
+        return ret;
+}
+
 
 int
-gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
 {
         int                        ret                  = -1;
         int                        opt_count            = 0;
-        int                        k                    = 0;
         int32_t                    i                    = 0;
         int32_t                    j                    = 1;
         int32_t                    status               = 0;
@@ -398,24 +490,27 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
         int32_t                    replica_count        = 0;
         int32_t                    vol_type             = 0;
         int32_t                    transport            = 0;
-        char                      *ptr                  = NULL;
         char                      *volume_id_str        = NULL;
         char                      *brick                = NULL;
         char                      *volname              = NULL;
         dict_t                    *dict                 = NULL;
-        data_pair_t               *pairs                = NULL;
-        data_t                    *value                = NULL;
         cli_local_t               *local                = NULL;
         char                       key[1024]            = {0};
         char                       err_str[2048]        = {0};
         gf_cli_rsp                 rsp                  = {0};
+        char                      *caps                 = NULL;
+        int                        k __attribute__((unused)) = 0;
+        // snap_volume variable helps in showing whether a volume is a normal
+        //volume or a volume for the snapshot
+        int32_t                    snap_volume          = 0;
 
         if (-1 == req->rpc_status)
                 goto out;
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -428,7 +523,9 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
         if (!rsp.dict.dict_len) {
-                cli_out ("No volumes present");
+                if (global_state->mode & GLUSTER_MODE_XML)
+                        goto xml_output;
+                cli_err ("No volumes present");
                 ret = 0;
                 goto out;
         }
@@ -471,10 +568,45 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                   "Volume %s does not exist",
                                   local->get_vol.volname);
                         ret = -1;
-                        goto out;
+                        if (!(global_state->mode & GLUSTER_MODE_XML))
+                                goto out;
                 }
         }
 
+xml_output:
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                /* For GET_NEXT_VOLUME output is already begun in
+                 * and will also end in gf_cli_get_next_volume()
+                 */
+                if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
+                        ret = cli_xml_output_vol_info_begin
+                                (local, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                                goto out;
+                        }
+                }
+
+                if (dict) {
+                        ret = cli_xml_output_vol_info (local, dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                                goto out;
+                        }
+                }
+
+                if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
+                        ret = cli_xml_output_vol_info_end (local);
+                        if (ret)
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                }
+                goto out;
+        }
+
         while ( i < count) {
                 cli_out (" ");
                 snprintf (key, 256, "volume%d.name", i);
@@ -492,6 +624,11 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 if (ret)
                         goto out;
 
+                snprintf (key, sizeof (key), "volume%d.snap_volume", i);
+                ret = dict_get_int32 (dict, key, &snap_volume);
+                if (ret)
+                        goto out;
+
                 snprintf (key, 256, "volume%d.brick_count", i);
                 ret = dict_get_int32 (dict, key, &brick_count);
                 if (ret)
@@ -529,9 +666,47 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
                         vol_type = type + 3;
 
                 cli_out ("Volume Name: %s", volname);
-                cli_out ("Type: %s", cli_volume_type[vol_type]);
+                cli_out ("Type: %s", cli_vol_type_str[vol_type]);
                 cli_out ("Volume ID: %s", volume_id_str);
-                cli_out ("Status: %s", cli_volume_status[status]);
+                cli_out ("Status: %s", cli_vol_status_str[status]);
+                if (snap_volume)
+                        cli_out ("Snap Volume: %s", "yes");
+                else
+                        cli_out ("Snap Volume: %s", "no");
+
+#ifdef HAVE_BD_XLATOR
+                k = 0;
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.xlator%d", i, k);
+                ret = dict_get_str (dict, key, &caps);
+                if (ret)
+                        goto next;
+                do {
+                        j = 0;
+                        cli_out ("Xlator %d: %s", k + 1, caps);
+                        do {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key),
+                                          "volume%d.xlator%d.caps%d",
+                                          i, k, j++);
+                                ret = dict_get_str (dict, key, &caps);
+                                if (ret)
+                                        break;
+                                cli_out ("Capability %d: %s", j, caps);
+                        } while (1);
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key),
+                                  "volume%d.xlator%d", i, ++k);
+                        ret = dict_get_str (dict, key, &caps);
+                        if (ret)
+                                break;
+                } while (1);
+
+next:
+#else
+                caps = 0; /* Avoid compiler warnings when BD not enabled */
+#endif
 
                 if (type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) {
                         cli_out ("Number of Bricks: %d x %d x %d = %d",
@@ -539,10 +714,8 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                  stripe_count,
                                  replica_count,
                                  brick_count);
-
                 } else if (type == GF_CLUSTER_TYPE_NONE) {
                         cli_out ("Number of Bricks: %d", brick_count);
-
                 } else {
                         /* For both replicate and stripe, dist_count is
                            good enough */
@@ -570,15 +743,15 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 goto out;
 
                         cli_out ("Brick%d: %s", j, brick);
+#ifdef HAVE_BD_XLATOR
+                        snprintf (key, 256, "volume%d.vg%d", i, j);
+                        ret = dict_get_str (dict, key, &caps);
+                        if (!ret)
+                                cli_out ("Brick%d VG: %s", j, caps);
+#endif
                         j++;
                 }
 
-                pairs = dict->members_list;
-                if (!pairs) {
-                        ret = -1;
-                        goto out;
-                }
-
                 snprintf (key, 256, "volume%d.opt_count",i);
                 ret = dict_get_int32 (dict, key, &opt_count);
                 if (ret)
@@ -588,26 +761,12 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
                         goto out;
 
                 cli_out ("Options Reconfigured:");
-                k = 0;
 
-                while (k < opt_count) {
-
-                        snprintf (key, 256, "volume%d.option.",i);
-                        while (pairs) {
-                                ptr = strstr (pairs->key, "option.");
-                                if (ptr) {
-                                        value = pairs->value;
-                                        if (!value) {
-                                                ret = -1;
-                                                goto out;
-                                        }
-                                        cli_out_options (key, pairs->key,
-                                                         value->data);
-                                }
-                                pairs = pairs->next;
-                        }
-                        k++;
-                }
+                snprintf (key, 256, "volume%d.option.",i);
+
+                ret = dict_foreach (dict, _gf_cli_output_volinfo_opts, key);
+                if (ret)
+                        goto out;
 
                 i++;
         }
@@ -617,23 +776,21 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         cli_cmd_broadcast_response (ret);
         if (ret)
-                cli_out ("%s", err_str);
+                cli_err ("%s", err_str);
 
         if (dict)
                 dict_destroy (dict);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
 
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.op_errstr);
 
         gf_log ("cli", GF_LOG_INFO, "Returning: %d", ret);
         return ret;
 }
 
 int
-gf_cli3_1_create_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_create_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp              rsp   = {0,};
@@ -641,49 +798,70 @@ gf_cli3_1_create_volume_cbk (struct rpc_req *req, struct iovec *iov,
         cli_local_t             *local = NULL;
         char                    *volname = NULL;
         dict_t                  *dict = NULL;
+        dict_t                  *rsp_dict = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
         local = ((call_frame_t *) (myframe))->local;
-        ((call_frame_t *) (myframe))->local = NULL;
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
+        gf_log ("cli", GF_LOG_INFO, "Received resp to create volume");
+
         dict = local->dict;
 
         ret = dict_get_str (dict, "volname", &volname);
+        if (ret)
+                goto out;
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                if (rsp.op_ret == 0) {
+                        rsp_dict = dict_new ();
+                        ret = dict_unserialize (rsp.dict.dict_val,
+                                                rsp.dict.dict_len,
+                                                &rsp_dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Failed rsp_dict unserialization");
+                                goto out;
+                        }
+                }
+
+                ret = cli_xml_output_vol_create (rsp_dict, rsp.op_ret,
+                                                 rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to create volume");
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("volume create: %s: failed: %s", volname,
+                         rsp.op_errstr);
+        else if (rsp.op_ret)
+                cli_err ("volume create: %s: failed", volname);
         else
-                cli_out ("Creation of volume %s has been %s", volname,
-                                (rsp.op_ret) ? "unsuccessful":
-                                "successful. Please start the volume to "
-                                "access data.");
+                cli_out ("volume create: %s: success: "
+                         "please start the volume to access data", volname);
+
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (dict)
-                dict_unref (dict);
-        if (local)
-                cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.dict.dict_val);
+        free (rsp.op_errstr);
         return ret;
 }
 
 int
-gf_cli3_1_delete_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_delete_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp              rsp   = {0,};
@@ -692,36 +870,197 @@ gf_cli3_1_delete_volume_cbk (struct rpc_req *req, struct iovec *iov,
         char                    *volname = NULL;
         call_frame_t            *frame = NULL;
         dict_t                  *dict = NULL;
+        dict_t                  *rsp_dict = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (local)
                 dict = local->dict;
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (frame->this->name, GF_LOG_ERROR,
                         "dict get failed");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to delete volume");
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                if (rsp.op_ret == 0) {
+                        rsp_dict = dict_new ();
+                        ret = dict_unserialize (rsp.dict.dict_val,
+                                                rsp.dict.dict_len,
+                                                &rsp_dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Failed rsp_dict unserialization");
+                                goto out;
+                        }
+                }
+
+                ret = cli_xml_output_generic_volume ("volDelete", rsp_dict,
+                                                     rsp.op_ret, rsp.op_errno,
+                                                     rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("volume delete: %s: failed: %s", volname,
+                         rsp.op_errstr);
+        else if (rsp.op_ret)
+                cli_err ("volume delete: %s: failed", volname);
         else
-                cli_out ("Deleting volume %s has been %s", volname,
+                cli_out ("volume delete: %s: success", volname);
+
+        ret = rsp.op_ret;
+
+out:
+        cli_cmd_broadcast_response (ret);
+        free (rsp.dict.dict_val);
+
+        gf_log ("", GF_LOG_INFO, "Returning with %d", ret);
+        return ret;
+}
+
+int
+gf_cli3_1_uuid_get_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        char                    *uuid_str = NULL;
+        gf_cli_rsp              rsp   = {0,};
+        int                     ret   = -1;
+        cli_local_t             *local = NULL;
+        call_frame_t            *frame = NULL;
+        dict_t                  *dict = NULL;
+
+        if (-1 == req->rpc_status)
+                goto out;
+
+        frame = myframe;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        local = frame->local;
+        frame->local = NULL;
+
+        gf_log ("cli", GF_LOG_INFO, "Received resp to uuid get");
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
+                                &dict);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to unserialize "
+                        "response for uuid get");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "uuid", &uuid_str);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to get uuid "
+                        "from dictionary");
+                goto out;
+        }
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_dict ("uuidGenerate", dict, rsp.op_ret,
+                                           rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (rsp.op_ret) {
+                if (strcmp (rsp.op_errstr, "") == 0)
+                        cli_err ("Get uuid was unsuccessful");
+                else
+                        cli_err ("%s", rsp.op_errstr);
+
+        } else {
+                cli_out ("UUID: %s", uuid_str);
+
+        }
+        ret = rsp.op_ret;
+
+out:
+        cli_cmd_broadcast_response (ret);
+        cli_local_wipe (local);
+        if (rsp.dict.dict_val)
+                free (rsp.dict.dict_val);
+        if (dict)
+                dict_unref (dict);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int
+gf_cli3_1_uuid_reset_cbk (struct rpc_req *req, struct iovec *iov,
+                             int count, void *myframe)
+{
+        gf_cli_rsp              rsp   = {0,};
+        int                     ret   = -1;
+        cli_local_t             *local = NULL;
+        call_frame_t            *frame = NULL;
+        dict_t                  *dict = NULL;
+
+        if (-1 == req->rpc_status) {
+                goto out;
+        }
+
+        frame = myframe;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        local = frame->local;
+        frame->local = NULL;
+
+        gf_log ("cli", GF_LOG_INFO, "Received resp to uuid reset");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_dict ("uuidReset", dict, rsp.op_ret,
+                                           rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
+                cli_err ("%s", rsp.op_errstr);
+        else
+                cli_out ("resetting the peer uuid has been %s",
                          (rsp.op_ret) ? "unsuccessful": "successful");
         ret = rsp.op_ret;
 
@@ -738,7 +1077,7 @@ out:
 }
 
 int
-gf_cli3_1_start_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_start_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp              rsp   = {0,};
@@ -747,58 +1086,76 @@ gf_cli3_1_start_volume_cbk (struct rpc_req *req, struct iovec *iov,
         char                    *volname = NULL;
         call_frame_t            *frame = NULL;
         dict_t                  *dict = NULL;
+        dict_t                  *rsp_dict = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        frame = myframe;
-
-        if (frame) {
+        if (frame)
                 local = frame->local;
-                frame->local = NULL;
-        }
 
         if (local)
                 dict = local->dict;
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict get failed");
+                gf_log (frame->this->name, GF_LOG_ERROR, "dict get failed");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to start volume");
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                if (rsp.op_ret == 0) {
+                        rsp_dict = dict_new ();
+                        ret = dict_unserialize (rsp.dict.dict_val,
+                                                rsp.dict.dict_len,
+                                                &rsp_dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Failed rsp_dict unserialization");
+                                goto out;
+                        }
+                }
+
+                ret = cli_xml_output_generic_volume ("volStart", rsp_dict,
+                                                     rsp.op_ret, rsp.op_errno,
+                                                     rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("volume start: %s: failed: %s", volname,
+                         rsp.op_errstr);
+        else if (rsp.op_ret)
+                cli_err ("volume start: %s: failed", volname);
         else
-                cli_out ("Starting volume %s has been %s", volname,
-                        (rsp.op_ret) ? "unsuccessful": "successful");
+                cli_out ("volume start: %s: success", volname);
 
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (local)
-                cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-        if (dict)
-                dict_unref (dict);
+        free (rsp.dict.dict_val);
+        free (rsp.op_errstr);
         return ret;
 }
 
 int
-gf_cli3_1_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp            rsp   = {0,};
@@ -807,19 +1164,21 @@ gf_cli3_1_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
         char                  *volname = NULL;
         call_frame_t          *frame = NULL;
         dict_t                *dict = NULL;
+        dict_t                *rsp_dict = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        frame = myframe;
-
         if (frame)
                 local = frame->local;
 
@@ -827,7 +1186,7 @@ gf_cli3_1_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 dict = local->dict;
                 ret = dict_get_str (dict, "volname", &volname);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (frame->this->name, GF_LOG_ERROR,
                                 "Unable to get volname from dict");
                         goto out;
                 }
@@ -835,26 +1194,47 @@ gf_cli3_1_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to stop volume");
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                if (rsp.op_ret == 0) {
+                        rsp_dict = dict_new ();
+                        ret = dict_unserialize (rsp.dict.dict_val,
+                                                rsp.dict.dict_len,
+                                                &rsp_dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Failed rsp_dict unserialization");
+                                goto out;
+                        }
+                }
+
+                ret = cli_xml_output_generic_volume ("volStop", rsp_dict,
+                                                     rsp.op_ret, rsp.op_errno,
+                                                     rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("volume stop: %s: failed: %s", volname, rsp.op_errstr);
+        else if (rsp.op_ret)
+                cli_err ("volume stop: %s: failed", volname);
         else
-                cli_out ("Stopping volume %s has been %s", volname,
-                        (rsp.op_ret) ? "unsuccessful": "successful");
+                cli_out ("volume stop: %s: success", volname);
+
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (local)
-                cli_local_wipe (local);
+        free (rsp.op_errstr);
+        free (rsp.dict.dict_val);
+
         return ret;
 }
 
 int
-gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp               rsp     = {0,};
@@ -869,37 +1249,48 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
         uint64_t                 files   = 0;
         uint64_t                 size    = 0;
         uint64_t                 lookup  = 0;
+        char                     msg[1024] = {0,};
+        gf_defrag_status_t       status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
+        int32_t                  counter = 0;
+        char                    *node_name = NULL;
+        char                     key[256] = {0,};
+        int32_t                  i = 1;
+        uint64_t                 failures = 0;
+        uint64_t                 skipped = 0;
+        double                   elapsed = 0;
+        char                    *size_str = NULL;
+        char                    *task_id_str = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp,
                               (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        frame = myframe;
-
         if (frame)
                 local = frame->local;
 
-        if (local) {
+        if (local)
                 local_dict = local->dict;
-        }
 
         ret = dict_get_str (local_dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (frame->this->name, GF_LOG_ERROR,
                         "Failed to get volname");
                 goto out;
         }
 
         ret = dict_get_int32 (local_dict, "rebalance-command", (int32_t*)&cmd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (frame->this->name, GF_LOG_ERROR,
                         "Failed to get command");
                 goto out;
         }
@@ -919,128 +1310,176 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 }
         }
 
-        ret = dict_get_uint64 (dict, "files", &files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get file count");
-
-        ret = dict_get_uint64 (dict, "size", &size);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get size of xfer");
-
-        ret = dict_get_uint64 (dict, "lookups", &lookup);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get lookedup file count");
+        if (!((cmd == GF_DEFRAG_CMD_STOP) || (cmd == GF_DEFRAG_CMD_STATUS)) &&
+             !(global_state->mode & GLUSTER_MODE_XML)) {
+                /* All other possibilites are about starting a rebalance */
+                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
+                if (rsp.op_ret && strcmp (rsp.op_errstr, "")) {
+                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+                } else {
+                        if (!rsp.op_ret) {
+                                snprintf (msg, sizeof (msg),
+                                          "Starting rebalance on volume %s has "
+                                          "been successful.\nID: %s", volname,
+                                          task_id_str);
+                        } else {
+                                snprintf (msg, sizeof (msg),
+                                          "Starting rebalance on volume %s has "
+                                          "been unsuccessful.", volname);
+                        }
+                }
+                goto done;
+        }
 
         if (cmd == GF_DEFRAG_CMD_STOP) {
                 if (rsp.op_ret == -1) {
                         if (strcmp (rsp.op_errstr, ""))
-                                cli_out ("%s", rsp.op_errstr);
+                                snprintf (msg, sizeof (msg),
+                                          "%s", rsp.op_errstr);
                         else
-                                cli_out ("rebalance volume %s stop failed",
-                                         volname);
+                                snprintf (msg, sizeof (msg),
+                                          "rebalance volume %s stop failed",
+                                          volname);
+                        goto done;
                 } else {
-                        cli_out ("stopped rebalance process of volume %s \n"
-                                 "(after rebalancing %"PRId64" files totaling "
-                                 "%"PRId64" bytes)", volname, files, size);
+                        snprintf (msg, sizeof (msg),
+                                  "rebalance process may be in the middle of a "
+                                  "file migration.\nThe process will be fully "
+                                  "stopped once the migration of the file is "
+                                  "complete.\nPlease check rebalance process "
+                                  "for completion before doing any further "
+                                  "brick related tasks on the volume.");
                 }
-                goto done;
         }
         if (cmd == GF_DEFRAG_CMD_STATUS) {
                 if (rsp.op_ret == -1) {
                         if (strcmp (rsp.op_errstr, ""))
-                                cli_out ("%s", rsp.op_errstr);
+                                snprintf (msg, sizeof (msg),
+                                          "%s", rsp.op_errstr);
                         else
-                                cli_out ("failed to get the status of "
-                                         "rebalance process");
+                                snprintf (msg, sizeof (msg),
+                                          "Failed to get the status of "
+                                          "rebalance process");
                         goto done;
                 }
+        }
 
-                switch (rsp.op_errno) {
-                case GF_DEFRAG_STATUS_NOT_STARTED:
-                        status = "not started";
-                        break;
-                case GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED:
-                        status = "step 1: layout fix in progress";
-                        break;
-                case GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED:
-                        status = "step 2: data migration in progress";
-                        break;
-                case GF_DEFRAG_STATUS_STOPPED:
-                        status = "stopped";
-                        break;
-                case GF_DEFRAG_STATUS_COMPLETE:
-                        status = "completed";
-                        break;
-                case GF_DEFRAG_STATUS_FAILED:
-                        status = "failed";
-                        break;
-                case GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE:
-                        status = "step 1: layout fix complete";
-                        break;
-                case GF_DEFRAG_STATUS_MIGRATE_DATA_COMPLETE:
-                        status = "step 2: data migration complete";
-                        break;
-                case GF_DEFRAG_STATUS_PAUSED:
-                        status = "paused";
-                        break;
-                }
-                if (files && (rsp.op_errno == 1)) {
-                        cli_out ("rebalance %s: fixed layout %"PRId64,
-                                 status, files);
-                        goto done;
-                }
-                if (files && (rsp.op_errno == 6)) {
-                        cli_out ("rebalance %s: fixed layout %"PRId64,
-                                 status, files);
-                        goto done;
-                }
-                if (files) {
-                        cli_out ("rebalance %s: rebalanced %"PRId64
-                                 " files of size %"PRId64" (total files"
-                                 " scanned %"PRId64")", status,
-                                 files, size, lookup);
-                        goto done;
-                }
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_rebalance (cmd, dict, rsp.op_ret,
+                                                    rsp.op_errno,
+                                                    rsp.op_errstr);
+                goto out;
+        }
 
-                cli_out ("rebalance %s", status);
-                goto done;
+        ret = dict_get_int32 (dict, "count", &counter);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR, "count not set");
+                goto out;
         }
 
-        /* All other possibility is about starting a volume */
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-        else
-                cli_out ("starting rebalance on volume %s has been %s",
-                         volname, (rsp.op_ret) ? "unsuccessful":
-                         "successful");
+        cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s", "Node",
+                 "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                 "status", "run time in secs");
+        cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s", "---------",
+                 "-----------", "-----------", "-----------", "-----------",
+                 "-----------", "------------", "--------------");
+        do {
+                snprintf (key, 256, "node-name-%d", i);
+                ret = dict_get_str (dict, key, &node_name);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get node-name");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "files-%d", i);
+                ret = dict_get_uint64 (dict, key, &files);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get file count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "size-%d", i);
+                ret = dict_get_uint64 (dict, key, &size);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get size of xfer");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "lookups-%d", i);
+                ret = dict_get_uint64 (dict, key, &lookup);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get lookedup file count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", i);
+                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get status");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &failures);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get failures count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "skipped-%d", i);
+                ret = dict_get_uint64 (dict, key, &skipped);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get skipped count");
+                memset (key, 0, 256);
+                snprintf (key, 256, "run-time-%d", i);
+                ret = dict_get_double (dict, key, &elapsed);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get run-time");
+
+                status = cli_vol_task_status_str[status_rcd];
+                size_str = gf_uint64_2human_readable(size);
+                cli_out ("%40s %16"PRIu64 " %13s" " %13"PRIu64 " %13"PRIu64
+                         " %13"PRIu64 " %20s %18.2f", node_name, files,
+                         size_str, lookup, failures, skipped, status, elapsed);
+                GF_FREE(size_str);
+
+                i++;
+        } while (i <= counter);
+
 
 done:
+        if (global_state->mode & GLUSTER_MODE_XML)
+                cli_xml_output_str ("volRebalance", msg,
+                                    rsp.op_ret, rsp.op_errno,
+                                    rsp.op_errstr);
+        else {
+                if (rsp.op_ret)
+                        cli_err ("volume rebalance: %s: failed: %s", volname,
+                                 msg);
+                else
+                        cli_out ("volume rebalance: %s: success: %s", volname,
+                                 msg);
+        }
         ret = rsp.op_ret;
 
 out:
-        if (rsp.op_errstr)
-                free (rsp.op_errstr); //malloced by xdr
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val); //malloced by xdr
+        free (rsp.op_errstr); //malloced by xdr
+        free (rsp.dict.dict_val); //malloced by xdr
         if (dict)
                 dict_unref (dict);
-        if (local_dict)
-                dict_unref (local_dict);
-        if (local)
-                cli_local_wipe (local);
         cli_cmd_broadcast_response (ret);
         return ret;
 }
 
 int
-gf_cli3_1_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp              rsp   = {0,};
         int                     ret   = -1;
+        char                    msg[1024] = {0,};
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1048,14 +1487,29 @@ gf_cli3_1_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to probe");
-        cli_out ("Rename volume %s", (rsp.op_ret) ? "unsuccessful":
-                                        "successful");
+        snprintf (msg, sizeof (msg), "Rename volume %s",
+                  (rsp.op_ret) ? "unsuccessful": "successful");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_str ("volRename", msg, rsp.op_ret,
+                                          rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (rsp.op_ret)
+                cli_err ("volume rename: failed");
+        else
+                cli_out ("volume rename: success");
 
         ret = rsp.op_ret;
 
@@ -1065,11 +1519,12 @@ out:
 }
 
 int
-gf_cli3_1_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp           rsp   = {0,};
         int                  ret   = -1;
+        char                 msg[1024] = {0,};
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1077,33 +1532,98 @@ gf_cli3_1_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to reset");
 
-        if (rsp.op_ret &&  strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+        if (strcmp (rsp.op_errstr, ""))
+                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
         else
-                cli_out ("reset volume %s", (rsp.op_ret) ? "unsuccessful":
-                                "successful");
+                snprintf (msg, sizeof (msg), "reset volume %s",
+                          (rsp.op_ret) ? "unsuccessful": "successful");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_str ("volReset", msg, rsp.op_ret,
+                                          rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (rsp.op_ret)
+                cli_err ("volume reset: failed: %s", msg);
+        else
+                cli_out ("volume reset: success: %s", msg);
 
         ret = rsp.op_ret;
 
 out:
-                cli_cmd_broadcast_response (ret);
+        cli_cmd_broadcast_response (ret);
         return ret;
 }
 
+char *
+is_server_debug_xlator (void *myframe)
+{
+        call_frame_t         *frame        = NULL;
+        cli_local_t          *local        = NULL;
+        char                 **words       = NULL;
+        char                 *key          = NULL;
+        char                 *value        = NULL;
+        char                 *debug_xlator = NULL;
+
+        frame = myframe;
+        local = frame->local;
+        words = (char **)local->words;
+
+        while (*words != NULL) {
+                if (strstr (*words, "trace") == NULL &&
+                    strstr (*words, "error-gen") == NULL) {
+                        words++;
+                        continue;
+                }
+
+                key = *words;
+                words++;
+                value = *words;
+                if (value == NULL)
+                        break;
+                if (strstr (value, "client")) {
+                        words++;
+                        continue;
+                } else {
+                        if (!(strstr (value, "posix") || strstr (value, "acl")
+                              || strstr (value, "locks") ||
+                              strstr (value, "io-threads") ||
+                              strstr (value, "marker") ||
+                              strstr (value, "index"))) {
+                                words++;
+                                continue;
+                        } else {
+                                debug_xlator = gf_strdup (key);
+                                break;
+                        }
+                }
+        }
+
+        return debug_xlator;
+}
+
 int
-gf_cli3_1_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp           rsp   = {0,};
         int                  ret   = -1;
         dict_t               *dict = NULL;
         char                 *help_str = NULL;
+        char                 msg[1024] = {0,};
+        char                 *debug_xlator = _gf_false;
+        char                 tmp_str[512] = {0,};
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1111,15 +1631,13 @@ gf_cli3_1_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to set");
 
-        if (rsp.op_ret &&  strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-
         dict = dict_new ();
 
         if (!dict) {
@@ -1129,28 +1647,64 @@ gf_cli3_1_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-        if (ret)
+        /* For brick processes graph change does not happen on the fly.
+         * The proces has to be restarted. So this is a check from the
+         * volume set option such that if debug xlators such as trace/errorgen
+         * are provided in the set command, warn the user.
+         */
+        debug_xlator = is_server_debug_xlator (myframe);
+
+        if (dict_get_str (dict, "help-str", &help_str) && !msg[0])
+                snprintf (msg, sizeof (msg), "Set volume %s",
+                          (rsp.op_ret) ? "unsuccessful": "successful");
+        if (rsp.op_ret == 0 && debug_xlator) {
+                snprintf (tmp_str, sizeof (tmp_str), "\n%s translator has been "
+                          "added to the server volume file. Please restart the"
+                          " volume for enabling the translator", debug_xlator);
+        }
+
+        if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {
+                ret = cli_xml_output_str ("volSet", msg, rsp.op_ret,
+                                          rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
                 goto out;
+        }
 
-        if (dict_get_str (dict, "help-str", &help_str))
-                cli_out ("Set volume %s", (rsp.op_ret) ? "unsuccessful":
-                                                         "successful");
-        else
-                cli_out ("%s", help_str);
+        if (rsp.op_ret) {
+                if (strcmp (rsp.op_errstr, ""))
+                        cli_err ("volume set: failed: %s", rsp.op_errstr);
+                else
+                        cli_err ("volume set: failed");
+        } else {
+                if (help_str == NULL) {
+                        if (debug_xlator == NULL)
+                                cli_out ("volume set: success");
+                        else
+                                cli_out ("volume set: success%s", tmp_str);
+                }else {
+                        cli_out ("%s", help_str);
+                }
+        }
 
         ret = rsp.op_ret;
 
 out:
+        if (dict)
+                dict_unref (dict);
+        GF_FREE (debug_xlator);
         cli_cmd_broadcast_response (ret);
         return ret;
 }
 
 int
-gf_cli3_1_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp                  rsp   = {0,};
         int                         ret   = -1;
+        char                        msg[1024] = {0,};
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1158,7 +1712,8 @@ gf_cli3_1_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -1166,18 +1721,30 @@ gf_cli3_1_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
         gf_log ("cli", GF_LOG_INFO, "Received resp to add brick");
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
         else
-                cli_out ("Add Brick %s", (rsp.op_ret) ? "unsuccessful":
-                                                        "successful");
+                snprintf (msg, sizeof (msg), "Add Brick %s",
+                          (rsp.op_ret) ? "unsuccessful": "successful");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_str ("volAddBrick", msg, rsp.op_ret,
+                                          rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (rsp.op_ret)
+                cli_err ("volume add-brick: failed: %s", rsp.op_errstr);
+        else
+                cli_out ("volume add-brick: success");
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.dict.dict_val);
+        free (rsp.op_errstr);
         return ret;
 }
 
@@ -1190,53 +1757,72 @@ gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
         int                      ret     = -1;
         uint64_t                 files   = 0;
         uint64_t                 size    = 0;
+        uint64_t                 lookup  = 0;
         dict_t                  *dict    = NULL;
+        char                     msg[1024] = {0,};
+        char                     key[256] = {0,};
+        int32_t                  i       = 1;
+        int32_t                  counter = 0;
+        char                    *node_name = 0;
+        gf_defrag_status_t       status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
+        uint64_t                 failures = 0;
+        uint64_t                 skipped = 0;
+        double                   elapsed = 0;
+        char                    *size_str = NULL;
+        int32_t                  command = 0;
+        gf1_op_commands          cmd = GF_OP_CMD_NONE;
+        cli_local_t             *local = NULL;
+        call_frame_t            *frame = NULL;
+        char                    *cmd_str = "unknown";
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp,
                               (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        ret = rsp.op_ret;
-        if (rsp.op_ret == -1) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_out ("%s", rsp.op_errstr);
-                else
-                        cli_out ("failed to get the status of "
-                                 "remove-brick process");
+        if (frame)
+                local = frame->local;
+        ret = dict_get_int32 (local->dict, "command", &command);
+        if (ret)
                 goto out;
-        }
+        cmd = command;
 
-        switch (rsp.op_errno) {
-        case GF_DEFRAG_STATUS_NOT_STARTED:
-                status = "not started";
-                break;
-        case GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED:
-        case GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED:
-        case GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE:
-                status = "in progress";
-                break;
-        case GF_DEFRAG_STATUS_STOPPED:
-                status = "stopped";
-                break;
-        case GF_DEFRAG_STATUS_COMPLETE:
-        case GF_DEFRAG_STATUS_MIGRATE_DATA_COMPLETE:
-                status = "completed";
+        switch (cmd) {
+        case GF_OP_CMD_STOP:
+                cmd_str = "stop";
                 break;
-        case GF_DEFRAG_STATUS_FAILED:
-                status = "failed";
+        case GF_OP_CMD_STATUS:
+                cmd_str = "status";
                 break;
-        case GF_DEFRAG_STATUS_PAUSED:
-                status = "paused";
+        default:
                 break;
         }
 
+        ret = rsp.op_ret;
+        if (rsp.op_ret == -1) {
+                if (strcmp (rsp.op_errstr, ""))
+                        snprintf (msg, sizeof (msg), "volume remove-brick %s: "
+                                  "failed: %s", cmd_str, rsp.op_errstr);
+                else
+                        snprintf (msg, sizeof (msg), "volume remove-brick %s: "
+                                  "failed", cmd_str);
+
+                if (global_state->mode & GLUSTER_MODE_XML)
+                        goto xml_output;
+
+                cli_err ("%s", msg);
+                goto out;
+        }
+
         if (rsp.dict.dict_len) {
                 /* Unserialize the dictionary */
                 dict  = dict_new ();
@@ -1245,45 +1831,146 @@ gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
                                         rsp.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
+                        strncpy (msg, "failed to unserialize req-buffer to "
+                                 "dictionary", sizeof (msg));
+
+                        if (global_state->mode & GLUSTER_MODE_XML) {
+                                rsp.op_ret = -1;
+                                goto xml_output;
+                        }
+
+                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
                         goto out;
                 }
         }
 
-        ret = dict_get_uint64 (dict, "files", &files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get file count");
-
-        ret = dict_get_uint64 (dict, "size", &size);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get size of xfer");
-
-        if (files && (rsp.op_errno == 1)) {
-                cli_out ("remove-brick %s: fixed layout %"PRId64,
-                         status,files);
-                goto out;
-        }
-        if (files && (rsp.op_errno == 6)) {
-                cli_out ("remove-brick %s: fixed layout %"PRId64,
-                         status, files);
+xml_output:
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                if (strcmp (rsp.op_errstr, "")) {
+                        ret = cli_xml_output_vol_remove_brick (_gf_true, dict,
+                                                               rsp.op_ret,
+                                                               rsp.op_errno,
+                                                               rsp.op_errstr);
+                } else {
+                        ret = cli_xml_output_vol_remove_brick (_gf_true, dict,
+                                                               rsp.op_ret,
+                                                               rsp.op_errno,
+                                                               msg);
+                }
                 goto out;
         }
-        if (files) {
-                cli_out ("remove-brick %s: decommissioned %"PRId64
-                         " files of size %"PRId64, status,
-                         files, size);
+
+        ret = dict_get_int32 (dict, "count", &counter);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR, "count not set");
                 goto out;
         }
 
-        cli_out ("remove-brick %s", status);
+
+        cli_out ("%40s %16s %13s %13s %13s %13s %14s %s", "Node",
+                 "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                 "status", "run-time in secs");
+        cli_out ("%40s %16s %13s %13s %13s %13s %14s %16s", "---------",
+                 "-----------", "-----------", "-----------", "-----------",
+                  "-----------","------------", "--------------");
+
+        do {
+                snprintf (key, 256, "node-name-%d", i);
+                ret = dict_get_str (dict, key, &node_name);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get node-name");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "files-%d", i);
+                ret = dict_get_uint64 (dict, key, &files);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get file count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "size-%d", i);
+                ret = dict_get_uint64 (dict, key, &size);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get size of xfer");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "lookups-%d", i);
+                ret = dict_get_uint64 (dict, key, &lookup);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get lookedup file count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", i);
+                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get status");
+
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &failures);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "Failed to get failure on files");
+
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &skipped);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "Failed to get skipped files");
+                memset (key, 0, 256);
+                snprintf (key, 256, "run-time-%d", i);
+                ret = dict_get_double (dict, key, &elapsed);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "Failed to get run-time");
+
+                switch (status_rcd) {
+                case GF_DEFRAG_STATUS_NOT_STARTED:
+                        status = "not started";
+                        break;
+                case GF_DEFRAG_STATUS_STARTED:
+                        status = "in progress";
+                        break;
+                case GF_DEFRAG_STATUS_STOPPED:
+                        status = "stopped";
+                        break;
+                case GF_DEFRAG_STATUS_COMPLETE:
+                        status = "completed";
+                        break;
+                case GF_DEFRAG_STATUS_FAILED:
+                        status = "failed";
+                        break;
+                default:
+                        break;
+                }
+
+                size_str = gf_uint64_2human_readable(size);
+                
+		if (strcmp (status, "not started")) {
+			cli_out ("%40s %16"PRIu64 " %13s" " %13"PRIu64 " %13"
+				PRIu64 " %13"PRIu64 " %14s %16.2f", node_name,
+                                files, size_str, lookup, failures, skipped,
+                                status, elapsed);
+		}
+                GF_FREE(size_str);
+
+                i++;
+        } while (i <= counter);
+
+        if ((cmd == GF_OP_CMD_STOP) && (rsp.op_ret == 0)) {
+                cli_out ("'remove-brick' process may be in the middle of a "
+                         "file migration.\nThe process will be fully stopped "
+                         "once the migration of the file is complete.\nPlease "
+                         "check remove-brick process for completion before "
+                         "doing any further brick related tasks on the "
+                         "volume.");
+        }
 
 out:
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val); //malloced by xdr
+        free (rsp.dict.dict_val); //malloced by xdr
         if (dict)
                 dict_unref (dict);
         cli_cmd_broadcast_response (ret);
@@ -1292,45 +1979,117 @@ out:
 
 
 int
-gf_cli3_1_remove_brick_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_remove_brick_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp                      rsp   = {0,};
         int                             ret   = -1;
+        char                            msg[1024] = {0,};
+        gf1_op_commands                 cmd = GF_OP_CMD_NONE;
+        char                           *cmd_str = "unknown";
+        cli_local_t                    *local = NULL;
+        call_frame_t                   *frame = NULL;
+        char                           *task_id_str = NULL;
+        dict_t                         *rsp_dict = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+        local = frame->local;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
+        ret = dict_get_int32 (local->dict, "command", (int32_t *)&cmd);
+        if (ret) {
+                 gf_log ("", GF_LOG_ERROR, "failed to get command");
+                 goto out;
+        }
+
+        if (rsp.dict.dict_len) {
+                rsp_dict = dict_new ();
+                if (!rsp_dict) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Failed to unserialize rsp_dict");
+                        goto out;
+                }
+        }
+
+        switch (cmd) {
+        case GF_OP_CMD_START:
+                cmd_str = "start";
+
+                ret = dict_get_str (rsp_dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "remove-brick-id is not present in dict");
+                }
+                break;
+        case GF_OP_CMD_COMMIT:
+                cmd_str = "commit";
+                break;
+        case GF_OP_CMD_COMMIT_FORCE:
+                cmd_str = "commit force";
+                break;
+        default:
+                cmd_str = "unknown";
+                break;
+        }
+
         gf_log ("cli", GF_LOG_INFO, "Received resp to remove brick");
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
         else
-                cli_out ("Remove Brick %s", (rsp.op_ret) ? "unsuccessful":
-                                                           "successful");
+                snprintf (msg, sizeof (msg), "Remove Brick %s %s", cmd_str,
+                          (rsp.op_ret) ? "unsuccessful": "successful");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_remove_brick (_gf_false, rsp_dict,
+                                                       rsp.op_ret, rsp.op_errno,
+                                                       msg);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (rsp.op_ret) {
+                cli_err ("volume remove-brick %s: failed: %s", cmd_str,
+                         msg);
+        } else {
+                cli_out ("volume remove-brick %s: success", cmd_str);
+                if (GF_OP_CMD_START == cmd && task_id_str != NULL)
+                        cli_out ("ID: %s", task_id_str);
+        }
 
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.dict.dict_val);
+        free (rsp.op_errstr);
+
         return ret;
 }
 
 
 
 int
-gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp                       rsp              = {0,};
@@ -1344,6 +2103,8 @@ gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
         gf1_cli_replace_op               replace_op       = 0;
         char                            *rb_operation_str = NULL;
         dict_t                          *rsp_dict         = NULL;
+        char                             msg[1024]        = {0,};
+        char                            *task_id_str      = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1353,7 +2114,8 @@ gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -1368,58 +2130,77 @@ gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                rsp.dict.dict_len,
+                                &rsp_dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                        "failed to "
+                                        "unserialize rsp buffer to dictionary");
+                        goto out;
+                }
+        }
+
         switch (replace_op) {
         case GF_REPLACE_OP_START:
-                if (rsp.op_ret)
-                        rb_operation_str = "replace-brick failed to start";
-                else
-                        rb_operation_str = "replace-brick started successfully";
+                if (rsp.op_ret) {
+                        rb_operation_str = gf_strdup ("replace-brick failed to"
+                                                      " start");
+                } else {
+                        ret = dict_get_str (rsp_dict, GF_REPLACE_BRICK_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
+                                        "\"replace-brick-id\" from dict");
+                                goto out;
+                        }
+                        ret = gf_asprintf (&rb_operation_str,
+                                           "replace-brick started successfully"
+                                           "\nID: %s", task_id_str);
+                        if (ret < 0)
+                                goto out;
+                }
                 break;
 
         case GF_REPLACE_OP_STATUS:
 
-                if (rsp.op_ret || ret)
-                        rb_operation_str = "replace-brick status unknown";
-                else {
-                        if (rsp.dict.dict_len) {
-                                /* Unserialize the dictionary */
-                                rsp_dict  = dict_new ();
-
-                                ret = dict_unserialize (rsp.dict.dict_val,
-                                                        rsp.dict.dict_len,
-                                                        &rsp_dict);
-                                if (ret < 0) {
-                                        gf_log ("glusterd", GF_LOG_ERROR,
-                                                "failed to "
-                                                "unserialize req-buffer to dictionary");
-                                        goto out;
-                                }
-                        }
+                if (rsp.op_ret || ret) {
+                        rb_operation_str = gf_strdup ("replace-brick status "
+                                                      "unknown");
+                } else {
                         ret = dict_get_str (rsp_dict, "status-reply",
                                             &status_reply);
                         if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR, "failed to"
+                                gf_log (frame->this->name, GF_LOG_ERROR, "failed to"
                                         "get status");
                                 goto out;
                         }
 
-                        rb_operation_str = status_reply;
+                        rb_operation_str = gf_strdup (status_reply);
                 }
 
                 break;
 
         case GF_REPLACE_OP_PAUSE:
                 if (rsp.op_ret)
-                        rb_operation_str = "replace-brick pause failed";
+                        rb_operation_str = gf_strdup ("replace-brick pause "
+                                                      "failed");
                 else
-                        rb_operation_str = "replace-brick paused successfully";
+                        rb_operation_str = gf_strdup ("replace-brick paused "
+                                                      "successfully");
                 break;
 
         case GF_REPLACE_OP_ABORT:
                 if (rsp.op_ret)
-                        rb_operation_str = "replace-brick abort failed";
+                        rb_operation_str = gf_strdup ("replace-brick abort "
+                                                      "failed");
                 else
-                        rb_operation_str = "replace-brick aborted successfully";
+                        rb_operation_str = gf_strdup ("replace-brick aborted "
+                                                      "successfully");
                 break;
 
         case GF_REPLACE_OP_COMMIT:
@@ -1440,9 +2221,11 @@ gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
 
 
                 if (rsp.op_ret || ret)
-                        rb_operation_str = "replace-brick commit failed";
+                        rb_operation_str = gf_strdup ("replace-brick commit "
+                                                      "failed");
                 else
-                        rb_operation_str = "replace-brick commit successful";
+                        rb_operation_str = gf_strdup ("replace-brick commit "
+                                                      "successful");
 
                 break;
 
@@ -1453,24 +2236,43 @@ gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
         if (rsp.op_ret && (strcmp (rsp.op_errstr, ""))) {
-                rb_operation_str = rsp.op_errstr;
+                rb_operation_str = gf_strdup (rsp.op_errstr);
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to replace brick");
-        cli_out ("%s",
-                 rb_operation_str ? rb_operation_str : "Unknown operation");
+        snprintf (msg, sizeof (msg), "%s",
+                  rb_operation_str ? rb_operation_str : "Unknown operation");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_replace_brick (replace_op, rsp_dict,
+                                                        rsp.op_ret,
+                                                        rsp.op_errno, msg);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
 
+        if (rsp.op_ret)
+                cli_err ("volume replace-brick: failed: %s", msg);
+        else
+                cli_out ("volume replace-brick: success: %s", msg);
         ret = rsp.op_ret;
 
 out:
+        if (frame)
+                frame->local = NULL;
+
         if (local) {
                 dict_unref (local->dict);
                 cli_local_wipe (local);
         }
 
+        if (rb_operation_str)
+                GF_FREE (rb_operation_str);
+
         cli_cmd_broadcast_response (ret);
-        if (rsp.dict.dict_val)
-               free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
         if (rsp_dict)
                 dict_unref (rsp_dict);
 
@@ -1479,11 +2281,12 @@ out:
 
 
 static int
-gf_cli3_1_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
 {
         gf_cli_rsp             rsp   = {0,};
         int                    ret   = -1;
+        char                   msg[1024] = {0,};
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1491,34 +2294,48 @@ gf_cli3_1_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_DEBUG, "Received resp to log rotate");
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
         else
-                cli_out ("log rotate %s", (rsp.op_ret) ? "unsuccessful":
-                                                         "successful");
+                snprintf (msg, sizeof (msg), "log rotate %s",
+                          (rsp.op_ret) ? "unsuccessful": "successful");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_str ("volLogRotate", msg, rsp.op_ret,
+                                          rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
 
+        if (rsp.op_ret)
+                cli_err ("volume log-rotate: failed: %s", msg);
+        else
+                cli_out ("volume log-rotate: success");
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
 
         return ret;
 }
 
 static int
-gf_cli3_1_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
                            int count, void *myframe)
 {
         gf_cli_rsp                     rsp   = {0,};
         int                            ret   = -1;
+        char                           msg[1024] = {0,};
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1526,17 +2343,33 @@ gf_cli3_1_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_DEBUG, "Received resp to sync");
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                snprintf (msg, sizeof (msg), "volume sync: failed: %s",
+                          rsp.op_errstr);
         else
-                cli_out ("volume sync: %s",
-                         (rsp.op_ret) ? "unsuccessful": "successful");
+                snprintf (msg, sizeof (msg), "volume sync: %s",
+                          (rsp.op_ret) ? "failed": "success");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_str ("volSync", msg, rsp.op_ret,
+                                          rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (rsp.op_ret)
+                cli_err ("%s", msg);
+        else
+                cli_out ("%s", msg);
         ret = rsp.op_ret;
 
 out:
@@ -1545,12 +2378,12 @@ out:
 }
 
 int32_t
-gf_cli3_1_print_limit_list (char *volname, char *limit_list,
+gf_cli_print_limit_list (char *volname, char *limit_list,
                             char *op_errstr)
 {
         int64_t  size            = 0;
         int64_t  limit_value     = 0;
-        int32_t  i, j, k;
+        int32_t  i, j;
         int32_t  len = 0, ret    = -1;
         char     *size_str       = NULL;
         char     path [PATH_MAX] = {0, };
@@ -1558,6 +2391,7 @@ gf_cli3_1_print_limit_list (char *volname, char *limit_list,
         char     value [1024]    = {0, };
         char     mountdir []     = "/tmp/mntXXXXXX";
         char     abspath [PATH_MAX] = {0, };
+        char     *colon_ptr      = NULL;
         runner_t runner          = {0,};
 
         GF_VALIDATE_OR_GOTO ("cli", volname, out);
@@ -1568,7 +2402,7 @@ gf_cli3_1_print_limit_list (char *volname, char *limit_list,
 
         len = strlen (limit_list);
         if (len == 0) {
-                cli_out ("%s", op_errstr?op_errstr:"quota limit not set ");
+                cli_err ("%s", op_errstr?op_errstr:"quota limit not set ");
                 goto out;
         }
 
@@ -1594,7 +2428,7 @@ gf_cli3_1_print_limit_list (char *volname, char *limit_list,
 
         len = strlen (limit_list);
         if (len == 0) {
-                cli_out ("quota limit not set ");
+                cli_err ("quota limit not set ");
                 goto unmount;
         }
 
@@ -1605,19 +2439,16 @@ gf_cli3_1_print_limit_list (char *volname, char *limit_list,
                  "-----------------------");
         while (i < len) {
                 j = 0;
-                k = 0;
-
-                while (limit_list [i] != ':') {
-                        path [k++] = limit_list [i++];
-                }
-                path [k] = '\0';
-
-                i++;  //skip ':'
 
                 while (limit_list [i] != ',' && limit_list [i] != '\0') {
-                        value [j++] = limit_list[i++];
+                        path [j++] = limit_list[i++];
                 }
-                value [j] = '\0';
+                path [j] = '\0';
+                //here path[] contains both path and limit value
+
+                colon_ptr = strrchr (path, ':');
+                *colon_ptr = '\0';
+                strcpy (value, ++colon_ptr);
 
                 snprintf (abspath, sizeof (abspath), "%s/%s", mountdir, path);
 
@@ -1660,7 +2491,7 @@ out:
 }
 
 int
-gf_cli3_1_quota_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_quota_cbk (struct rpc_req *req, struct iovec *iov,
                      int count, void *myframe)
 {
         gf_cli_rsp         rsp        = {0,};
@@ -1669,20 +2500,29 @@ gf_cli3_1_quota_cbk (struct rpc_req *req, struct iovec *iov,
         char              *volname    = NULL;
         char              *limit_list = NULL;
         int32_t            type       = 0;
+        char               msg[1024]  = {0,};
+        call_frame_t      *frame      = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         if (rsp.op_ret &&
             strcmp (rsp.op_errstr, "") == 0) {
-                cli_out ("command unsuccessful %s", rsp.op_errstr);
+                snprintf (msg, sizeof (msg), "command unsuccessful %s",
+                          rsp.op_errstr);
+
+                if (global_state->mode & GLUSTER_MODE_XML)
+                        goto xml_output;
                 goto out;
         }
 
@@ -1703,64 +2543,103 @@ gf_cli3_1_quota_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
+                gf_log (frame->this->name, GF_LOG_TRACE,
                         "failed to get volname");
 
         ret = dict_get_str (dict, "limit_list", &limit_list);
         if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
+                gf_log (frame->this->name, GF_LOG_TRACE,
                         "failed to get limit_list");
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
+                gf_log (frame->this->name, GF_LOG_TRACE,
                         "failed to get type");
 
         if (type == GF_QUOTA_OPTION_TYPE_LIST) {
+                if (global_state->mode & GLUSTER_MODE_XML) {
+                        ret = cli_xml_output_vol_quota_limit_list
+                                (volname, limit_list, rsp.op_ret,
+                                 rsp.op_errno, rsp.op_errstr);
+                        if (ret)
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                        goto out;
+
+                }
+
                 if (limit_list) {
-                        gf_cli3_1_print_limit_list (volname,
+                        gf_cli_print_limit_list (volname,
                                                     limit_list,
                                                     rsp.op_errstr);
                 } else {
                         gf_log ("cli", GF_LOG_INFO, "Received resp to quota "
                                 "command ");
                         if (rsp.op_errstr)
-                                cli_out ("%s", rsp.op_errstr);
+                                snprintf (msg, sizeof (msg), "%s",
+                                          rsp.op_errstr);
                 }
         } else {
                 gf_log ("cli", GF_LOG_INFO, "Received resp to quota command ");
                 if (rsp.op_errstr)
-                        cli_out ("%s", rsp.op_errstr);
+                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
                 else
-                        cli_out ("%s", "successful");
+                        snprintf (msg, sizeof (msg), "successful");
         }
 
-out:
-        ret = rsp.op_ret;
+xml_output:
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_str ("volQuota", msg, rsp.op_ret,
+                                          rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (strlen (msg) > 0) {
+                if (rsp.op_ret)
+                        cli_err ("%s", msg);
+                else
+                        cli_out ("%s", msg);
+        }
 
+        ret = rsp.op_ret;
+out:
         cli_cmd_broadcast_response (ret);
+        if (dict)
+                dict_unref (dict);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
 
         return ret;
 }
 
 int
-gf_cli3_1_getspec_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_getspec_cbk (struct rpc_req *req, struct iovec *iov,
                        int count, void *myframe)
 {
         gf_getspec_rsp          rsp   = {0,};
         int                     ret   = -1;
         char                   *spec  = NULL;
+        call_frame_t           *frame = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-        if (ret < 0 || rsp.op_ret == -1) {
-                gf_log ("", GF_LOG_ERROR, "error");
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        if (rsp.op_ret == -1) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "getspec failed");
                 goto out;
         }
 
@@ -1784,20 +2663,30 @@ out:
 }
 
 int
-gf_cli3_1_pmap_b2p_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_pmap_b2p_cbk (struct rpc_req *req, struct iovec *iov,
                         int count, void *myframe)
 {
         pmap_port_by_brick_rsp rsp = {0,};
         int                     ret   = -1;
         char                   *spec  = NULL;
+        call_frame_t           *frame = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
-        if (ret < 0 || rsp.op_ret == -1) {
-                gf_log ("", GF_LOG_ERROR, "error");
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        if (rsp.op_ret == -1) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "pump_b2p failed");
                 goto out;
         }
 
@@ -1815,13 +2704,12 @@ out:
 
 
 int32_t
-gf_cli3_1_probe (call_frame_t *frame, xlator_t *this,
+gf_cli_probe (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
-        gf1_cli_probe_req  req      = {0,};
+        gf_cli_req         req      = {{0,},};
         int                ret      = 0;
         dict_t            *dict     = NULL;
-        char              *hostname = NULL;
         int                port     = 0;
 
         if (!frame || !this ||  !data) {
@@ -1830,35 +2718,32 @@ gf_cli3_1_probe (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret)
-                goto out;
 
         ret = dict_get_int32 (dict, "port", &port);
-        if (ret)
-                port = CLI_GLUSTERD_PORT;
-
-        req.hostname = hostname;
-        req.port     = port;
+        if (ret) {
+                ret = dict_set_int32 (dict, "port", CLI_GLUSTERD_PORT);
+                if (ret)
+                        goto out;
+        }
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_PROBE, NULL,
-                              this, gf_cli3_1_probe_cbk,
-                              (xdrproc_t)xdr_gf1_cli_probe_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_probe_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_PROBE, this, cli_rpc_prog, NULL);
 
 out:
+        GF_FREE (req.dict.dict_val);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
         return ret;
 }
 
 int32_t
-gf_cli3_1_deprobe (call_frame_t *frame, xlator_t *this,
+gf_cli_deprobe (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
-        gf1_cli_deprobe_req  req      = {0,};
+        gf_cli_req           req      = {{0,},};
         int                  ret      = 0;
         dict_t              *dict     = NULL;
-        char                *hostname = NULL;
         int                  port     = 0;
         int                  flags    = 0;
 
@@ -1868,57 +2753,70 @@ gf_cli3_1_deprobe (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret)
-                goto out;
-
         ret = dict_get_int32 (dict, "port", &port);
-        if (ret)
-                port = CLI_GLUSTERD_PORT;
+        if (ret) {
+                ret = dict_set_int32 (dict, "port", CLI_GLUSTERD_PORT);
+                if (ret)
+                        goto out;
+        }
 
         ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret)
-                flags = 0;
+        if (ret) {
+                ret = dict_set_int32 (dict, "flags", 0);
+                if (ret)
+                        goto out;
+        }
 
-        req.hostname = hostname;
-        req.port     = port;
-        req.flags    = flags;
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_DEPROBE, NULL,
-                              this, gf_cli3_1_deprobe_cbk,
-                              (xdrproc_t)xdr_gf1_cli_deprobe_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_deprobe_cbk,
+                               (xdrproc_t)xdr_gf_cli_req, dict,
+                              GLUSTER_CLI_DEPROBE, this, cli_rpc_prog, NULL);
 
 out:
+        GF_FREE (req.dict.dict_val);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
         return ret;
 }
 
 int32_t
-gf_cli3_1_list_friends (call_frame_t *frame, xlator_t *this,
-                        void *data)
+gf_cli_list_friends (call_frame_t *frame, xlator_t *this,
+                     void *data)
 {
         gf1_cli_peer_list_req   req = {0,};
         int                     ret = 0;
+        unsigned long           flags = 0;
 
         if (!frame || !this) {
                 ret = -1;
                 goto out;
         }
 
-        req.flags = GF_CLI_LIST_ALL;
+        GF_ASSERT (frame->local == NULL);
 
+        flags = (long)data;
+        req.flags = flags;
+        frame->local = (void*)flags;
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_LIST_FRIENDS, NULL,
-                              this, gf_cli3_1_list_friends_cbk,
+                              this, gf_cli_list_friends_cbk,
                               (xdrproc_t) xdr_gf1_cli_peer_list_req);
 
 out:
+        if (ret) {
+                /*
+                 * If everything goes fine, gf_cli_list_friends_cbk()
+                 * [invoked through cli_cmd_submit()]resets the
+                 * frame->local to NULL. In case cli_cmd_submit()
+                 * fails in between, RESET frame->local here.
+                 */
+                frame->local = NULL;
+        }
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-gf_cli3_1_get_next_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_get_next_volume (call_frame_t *frame, xlator_t *this,
                            void *data)
 {
 
@@ -1932,31 +2830,51 @@ gf_cli3_1_get_next_volume (call_frame_t *frame, xlator_t *this,
         }
 
         ctx = data;
+        local = frame->local;
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_info_begin (local, 0, 0, "");
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
+                        goto out;
+                }
+        }
 
-        ret = gf_cli3_1_get_volume (frame, this, data);
+        ret = gf_cli_get_volume (frame, this, data);
 
-        local = frame->local;
 
         if (!local || !local->get_vol.volname) {
-                cli_out ("No volumes present");
+                if ((global_state->mode & GLUSTER_MODE_XML))
+                        goto end_xml;
+
+                cli_err ("No volumes present");
                 goto out;
         }
 
+
         ctx->volname = local->get_vol.volname;
 
         while (ctx->volname) {
-                ret = gf_cli3_1_get_volume (frame, this, ctx);
+                ret = gf_cli_get_volume (frame, this, ctx);
                 if (ret)
                         goto out;
                 ctx->volname = local->get_vol.volname;
         }
 
+end_xml:
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_info_end (local);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
+        }
+
 out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-gf_cli3_1_get_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
         gf_cli_req                      req = {{0,}};
@@ -1985,90 +2903,108 @@ gf_cli3_1_get_volume (call_frame_t *frame, xlator_t *this,
         flags = ctx->flags;
         ret = dict_set_int32 (dict, "flags", flags);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set flags");
+                gf_log (frame->this->name, GF_LOG_ERROR, "failed to set flags");
                 goto out;
         }
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_GET_VOLUME, NULL,
-                              this, gf_cli3_1_get_volume_cbk,
+                              this, gf_cli_get_volume_cbk,
                               (xdrproc_t) xdr_gf_cli_req);
 
 out:
         if (dict)
                 dict_unref (dict);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-
 int32_t
-gf_cli3_1_create_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+gf_cli3_1_uuid_get (call_frame_t *frame, xlator_t *this,
+                      void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-        cli_local_t             *local = NULL;
+        gf_cli_req                      req = {{0,}};
+        int                             ret = 0;
+        dict_t                          *dict = NULL;
 
-        if (!frame || !this ||  !data) {
+        if (!frame || !this || !data) {
                 ret = -1;
                 goto out;
         }
 
-        dict = dict_ref ((dict_t *)data);
+        dict = data;
+        ret = cli_to_glusterd (&req, frame, gf_cli3_1_uuid_get_cbk,
+                               (xdrproc_t)xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_UUID_GET, this, cli_rpc_prog,
+                               NULL);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
+int32_t
+gf_cli3_1_uuid_reset (call_frame_t *frame, xlator_t *this,
+                      void *data)
+{
+        gf_cli_req                      req = {{0,}};
+        int                             ret = 0;
+        dict_t                          *dict = NULL;
+
+        if (!frame || !this || !data) {
+                ret = -1;
                 goto out;
         }
 
-        local = cli_local_get ();
+        dict = data;
+        ret = cli_to_glusterd (&req, frame, gf_cli3_1_uuid_reset_cbk,
+                               (xdrproc_t)xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_UUID_RESET, this, cli_rpc_prog,
+                               NULL);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+gf_cli_create_volume (call_frame_t *frame, xlator_t *this,
+                         void *data)
+{
+        gf_cli_req              req = {{0,}};
+        int                     ret = 0;
+        dict_t                  *dict = NULL;
 
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
+        if (!frame || !this ||  !data) {
+                ret = -1;
+                goto out;
         }
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_CREATE_VOLUME, NULL,
-                              this, gf_cli3_1_create_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
-
+        dict = data;
 
+        ret = cli_to_glusterd (&req, frame, gf_cli_create_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_CREATE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (dict)
-                dict_unref (dict);
-
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_delete_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_delete_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req = {{0,}};
         int                     ret = 0;
-        cli_local_t             *local = NULL;
         dict_t                  *dict = NULL;
 
         if (!frame || !this ||  !data) {
@@ -2076,50 +3012,26 @@ gf_cli3_1_delete_volume (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        local = cli_local_get ();
-
-        dict = dict_new ();
-        ret = dict_set_str (dict, "volname", data);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "dict set failed");
-                goto out;
-        }
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialize dict");
-                goto out;
-        }
+        dict = data;
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_DELETE_VOLUME, NULL,
-                              this, gf_cli3_1_delete_volume_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_delete_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_DELETE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
-        if (dict)
-                dict_unref (dict);
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_start_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_start_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req = {{0,}};
         int                     ret = 0;
-        cli_local_t             *local = NULL;
         dict_t                  *dict = NULL;
 
         if (!frame || !this ||  !data) {
@@ -2128,27 +3040,11 @@ gf_cli3_1_start_volume (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        local = cli_local_get ();
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize dict");
-                goto out;
-        }
-
-
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_START_VOLUME, NULL,
-                              this, gf_cli3_1_start_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_start_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_START_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2157,12 +3053,11 @@ out:
 }
 
 int32_t
-gf_cli3_1_stop_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_stop_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req             req = {{0,}};
         int                    ret = 0;
-        cli_local_t            *local = NULL;
         dict_t                 *dict = data;
 
         if (!frame || !this ||  !data) {
@@ -2170,28 +3065,12 @@ gf_cli3_1_stop_volume (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        local = cli_local_get ();
         dict = data;
 
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *) &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_STOP_VOLUME, NULL,
-                              this, gf_cli3_1_stop_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_stop_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_STOP_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2200,17 +3079,12 @@ out:
 }
 
 int32_t
-gf_cli3_1_defrag_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_defrag_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req     =  {{0,}};
         int                     ret     = 0;
-        cli_local_t            *local   = NULL;
-        char                   *volname = NULL;
-        char                   *cmd_str = NULL;
         dict_t                 *dict    = NULL;
-        gf_cli_defrag_type      cmd     = 0;
-        dict_t                  *req_dict = NULL;
 
         if (!frame || !this ||  !data) {
                 ret = -1;
@@ -2219,82 +3093,10 @@ gf_cli3_1_defrag_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                gf_log ("", GF_LOG_DEBUG, "error");
-
-        ret = dict_get_str (dict, "command", &cmd_str);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "error");
-                goto out;
-        }
-
-        if (strcmp (cmd_str, "start") == 0) {
-                cmd = GF_DEFRAG_CMD_START;
-                ret = dict_get_str (dict, "start-type", &cmd_str);
-                if (!ret) {
-                        if (strcmp (cmd_str, "fix-layout") == 0) {
-                                cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
-                        }
-                        if (strcmp (cmd_str, "migrate-data") == 0) {
-                                cmd = GF_DEFRAG_CMD_START_MIGRATE_DATA;
-                        }
-                        if (strcmp (cmd_str, "migrate-data-force") == 0) {
-                                cmd = GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE;
-                        }
-                }
-                goto done;
-        }
-        if (strcmp (cmd_str, "stop") == 0) {
-                cmd = GF_DEFRAG_CMD_STOP;
-                goto done;
-        }
-        if (strcmp (cmd_str, "status") == 0) {
-                cmd = GF_DEFRAG_CMD_STATUS;
-        }
-
-done:
-        local = cli_local_get ();
-
-        req_dict = dict_new ();
-        if (!req_dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_str (req_dict, "volname", volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to set dict");
-                goto out;
-        }
-
-        ret = dict_set_int32 (req_dict, "rebalance-command", (int32_t) cmd);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to set dict");
-                goto out;
-        }
-
-        if (local) {
-                local->dict = dict_ref (req_dict);
-                frame->local = local;
-        }
-
-        ret = dict_allocate_and_serialize (req_dict,
-                                           &req.dict.dict_val,
-                                           (size_t *) &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_DEFRAG_VOLUME, NULL,
-                              this, gf_cli3_1_defrag_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_defrag_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2303,7 +3105,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_rename_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_rename_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req = {{0,}};
@@ -2317,9 +3119,8 @@ gf_cli3_1_rename_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *) &req.dict.dict_len);
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to serialize the data");
@@ -2330,7 +3131,7 @@ gf_cli3_1_rename_volume (call_frame_t *frame, xlator_t *this,
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_RENAME_VOLUME, NULL,
-                              this, gf_cli3_1_rename_volume_cbk,
+                              this, gf_cli_rename_volume_cbk,
                               (xdrproc_t) xdr_gf_cli_req);
 
 out:
@@ -2340,7 +3141,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_reset_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_reset_volume (call_frame_t *frame, xlator_t *this,
                         void *data)
 {
         gf_cli_req              req =  {{0,}};
@@ -2354,28 +3155,18 @@ gf_cli3_1_reset_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                               GLUSTER_CLI_RESET_VOLUME, NULL,
-                               this, gf_cli3_1_reset_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_reset_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_RESET_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
-                gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-gf_cli3_1_set_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_set_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req =  {{0,}};
@@ -2389,19 +3180,10 @@ gf_cli3_1_set_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_SET_VOLUME, NULL,
-                              this, gf_cli3_1_set_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_set_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_SET_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2410,7 +3192,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_add_brick (call_frame_t *frame, xlator_t *this,
+gf_cli_add_brick (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req =  {{0,}};
@@ -2435,33 +3217,20 @@ gf_cli3_1_add_brick (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_ADD_BRICK, NULL,
-                              this, gf_cli3_1_add_brick_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_add_brick_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_ADD_BRICK, this, cli_rpc_prog, NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_remove_brick (call_frame_t *frame, xlator_t *this,
+gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req                req =  {{0,}};;
@@ -2488,23 +3257,16 @@ gf_cli3_1_remove_brick (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
-        if (command != GF_OP_CMD_STATUS) {
+        if ((command != GF_OP_CMD_STATUS) &&
+            (command != GF_OP_CMD_STOP)) {
 
-                ret = dict_allocate_and_serialize (dict,
-                                                   &req.dict.dict_val,
-                                                   (size_t *)&req.dict.dict_len);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to get serialized length of dict");
-                        goto out;
-                }
 
-                ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                                      GLUSTER_CLI_REMOVE_BRICK, NULL,
-                                      this, gf_cli3_1_remove_brick_cbk,
-                                      (xdrproc_t) xdr_gf_cli_req);
+                ret = cli_to_glusterd (&req, frame, gf_cli_remove_brick_cbk,
+                                       (xdrproc_t) xdr_gf_cli_req, dict,
+                                       GLUSTER_CLI_REMOVE_BRICK, this,
+                                       cli_rpc_prog, NULL);
         } else {
-                /* Need rebalance status to e sent :-) */
+                /* Need rebalance status to be sent :-) */
                 req_dict = dict_new ();
                 if (!req_dict) {
                         ret = -1;
@@ -2513,60 +3275,49 @@ gf_cli3_1_remove_brick (call_frame_t *frame, xlator_t *this,
 
                 ret = dict_set_str (req_dict, "volname", volname);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict");
                         goto out;
                 }
 
-                cmd |= GF_DEFRAG_CMD_STATUS;
+                if (command == GF_OP_CMD_STATUS)
+                        cmd |= GF_DEFRAG_CMD_STATUS;
+                else
+                        cmd |= GF_DEFRAG_CMD_STOP;
 
                 ret = dict_set_int32 (req_dict, "rebalance-command", (int32_t) cmd);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict");
                         goto out;
                 }
 
-                ret = dict_allocate_and_serialize (req_dict,
-                                                   &status_req.dict.dict_val,
-                                                   (size_t *) &status_req.dict.dict_len);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to serialize the data");
+                ret = cli_to_glusterd (&status_req, frame,
+                                       gf_cli3_remove_brick_status_cbk,
+                                       (xdrproc_t) xdr_gf_cli_req, req_dict,
+                                       GLUSTER_CLI_DEFRAG_VOLUME, this,
+                                       cli_rpc_prog, NULL);
 
-                        goto out;
                 }
 
-                ret = cli_cmd_submit (&status_req, frame, cli_rpc_prog,
-                                      GLUSTER_CLI_DEFRAG_VOLUME, NULL,
-                                      this, gf_cli3_remove_brick_status_cbk,
-                                      (xdrproc_t) xdr_gf_cli_req);
-
-        }
-
 out:
+        if (req_dict)
+                dict_unref (req_dict);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
-
-        if (status_req.dict.dict_val)
-                GF_FREE (status_req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
-        if (req_dict)
-                dict_unref (req_dict);
+        GF_FREE (status_req.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_replace_brick (call_frame_t *frame, xlator_t *this,
+gf_cli_replace_brick (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req                  req        =  {{0,}};
         int                         ret        = 0;
-        cli_local_t                *local      = NULL;
         dict_t                     *dict       = NULL;
         char                       *src_brick  = NULL;
         char                       *dst_brick  = NULL;
@@ -2580,17 +3331,6 @@ gf_cli3_1_replace_brick (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        local = cli_local_get ();
-        if (!local) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto out;
-        }
-
-        local->dict                 = dict_ref (dict);
-        frame->local                = local;
-
         ret = dict_get_int32 (dict, "operation", &op);
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -2623,34 +3363,22 @@ gf_cli3_1_replace_brick (call_frame_t *frame, xlator_t *this,
                 "%s with operation=%d", src_brick,
                 dst_brick, op);
 
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_REPLACE_BRICK, NULL,
-                              this, gf_cli3_1_replace_brick_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_replace_brick_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_REPLACE_BRICK, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
 
 
 int32_t
-gf_cli3_1_log_rotate (call_frame_t *frame, xlator_t *this,
+gf_cli_log_rotate (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
         gf_cli_req                req = {{0,}};
@@ -2664,31 +3392,20 @@ gf_cli3_1_log_rotate (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to serialize dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_LOG_ROTATE, NULL,
-                              this, gf_cli3_1_log_rotate_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
-
+        ret = cli_to_glusterd (&req, frame, gf_cli_log_rotate_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_LOG_ROTATE, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
         return ret;
 }
 
 int32_t
-gf_cli3_1_sync_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_sync_volume (call_frame_t *frame, xlator_t *this,
                        void *data)
 {
         int               ret = 0;
@@ -2701,35 +3418,27 @@ gf_cli3_1_sync_volume (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to serialize dict");
-                goto out;
-        }
 
-        ret = cli_cmd_submit (&req, frame,
-                              cli_rpc_prog, GLUSTER_CLI_SYNC_VOLUME,
-                              NULL, this, gf_cli3_1_sync_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_sync_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_SYNC_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_getspec (call_frame_t *frame, xlator_t *this,
+gf_cli_getspec (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_getspec_req          req = {0,};
         int                     ret = 0;
         dict_t                  *dict = NULL;
+        dict_t                  *op_dict = NULL;
 
         if (!frame || !this ||  !data) {
                 ret = -1;
@@ -2742,19 +3451,52 @@ gf_cli3_1_getspec (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
+        op_dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        // Set the supported min and max op-versions, so glusterd can make a
+        // decision
+        ret = dict_set_int32 (op_dict, "min-op-version", GD_OP_VERSION_MIN);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_set_int32 (op_dict, "max-op-version", GD_OP_VERSION_MAX);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (op_dict, &req.xdata.xdata_val,
+                                           &req.xdata.xdata_len);
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to serialize dictionary");
+                goto out;
+        }
+
         ret = cli_cmd_submit (&req, frame, &cli_handshake_prog,
                               GF_HNDSK_GETSPEC, NULL,
-                              this, gf_cli3_1_getspec_cbk,
+                              this, gf_cli_getspec_cbk,
                               (xdrproc_t) xdr_gf_getspec_req);
 
 out:
+        if (op_dict) {
+                dict_unref(op_dict);
+        }
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_quota (call_frame_t *frame, xlator_t *this,
+gf_cli_quota (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         gf_cli_req          req = {{0,}};
@@ -2768,27 +3510,18 @@ gf_cli3_1_quota (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_QUOTA, NULL,
-                              this, gf_cli3_1_quota_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_quota_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_QUOTA, this, cli_rpc_prog, NULL);
 
-        GF_FREE (req.dict.dict_val);
 out:
+        GF_FREE (req.dict.dict_val);
+
         return ret;
 }
 
 int32_t
-gf_cli3_1_pmap_b2p (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_pmap_b2p (call_frame_t *frame, xlator_t *this, void *data)
 {
         pmap_port_by_brick_req  req = {0,};
         int                     ret = 0;
@@ -2807,7 +3540,7 @@ gf_cli3_1_pmap_b2p (call_frame_t *frame, xlator_t *this, void *data)
 
         ret = cli_cmd_submit (&req, frame, &cli_pmap_prog,
                               GF_PMAP_PORTBYBRICK, NULL,
-                              this, gf_cli3_1_pmap_b2p_cbk,
+                              this, gf_cli_pmap_b2p_cbk,
                               (xdrproc_t) xdr_pmap_port_by_brick_req);
 
 out:
@@ -2817,7 +3550,7 @@ out:
 }
 
 static int
-gf_cli3_1_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
                        int count, void *myframe)
 {
         gf1_cli_fsm_log_rsp        rsp   = {0,};
@@ -2837,14 +3570,15 @@ gf_cli3_1_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         if (rsp.op_ret) {
                 if (strcmp (rsp.op_errstr, ""))
-                        cli_out ("%s", rsp.op_errstr);
-                cli_out ("fsm log unsuccessful");
+                        cli_err ("%s", rsp.op_errstr);
+                cli_err ("fsm log unsuccessful");
                 ret = rsp.op_ret;
                 goto out;
         }
@@ -2860,7 +3594,7 @@ gf_cli3_1_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
                                 &dict);
 
         if (ret) {
-                cli_out ("bad response");
+                cli_err ("bad response");
                 goto out;
         }
 
@@ -2868,7 +3602,7 @@ gf_cli3_1_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
         if (tr_count)
                 cli_out("number of transitions: %d", tr_count);
         else
-                cli_out("No transitions");
+                cli_err("No transitions");
         for (i = 0; i < tr_count; i++) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "log%d-old-state", i);
@@ -2907,7 +3641,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_fsm_log (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_fsm_log (call_frame_t *frame, xlator_t *this, void *data)
 {
         int                        ret = -1;
         gf1_cli_fsm_log_req        req = {0,};
@@ -2921,7 +3655,7 @@ gf_cli3_1_fsm_log (call_frame_t *frame, xlator_t *this, void *data)
         req.name = data;
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_FSM_LOG, NULL,
-                              this, gf_cli3_1_fsm_log_cbk,
+                              this, gf_cli_fsm_log_cbk,
                               (xdrproc_t) xdr_gf1_cli_fsm_log_req);
 
 out:
@@ -2931,14 +3665,17 @@ out:
 }
 
 int
-gf_cli3_1_gsync_config_command (dict_t *dict)
+gf_cli_gsync_config_command (dict_t *dict)
 {
         runner_t runner     = {0,};
         char *subop         = NULL;
         char *gwd           = NULL;
         char *slave         = NULL;
+        char *confpath      = NULL;
         char *master        = NULL;
         char *op_name       = NULL;
+        int   ret           = -1;
+        char  conf_path[PATH_MAX] = "";
 
         if (dict_get_str (dict, "subop", &subop) != 0)
                 return -1;
@@ -2957,9 +3694,17 @@ gf_cli3_1_gsync_config_command (dict_t *dict)
         if (dict_get_str (dict, "op_name", &op_name) != 0)
                 op_name = NULL;
 
+        ret = dict_get_str (dict, "conf_path", &confpath);
+        if (!confpath) {
+                ret = snprintf (conf_path, sizeof(conf_path) - 1,
+                                "%s/"GEOREP"/gsyncd_template.conf", gwd);
+                conf_path[ret] = '\0';
+                confpath = conf_path;
+        }
+
         runinit (&runner);
         runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, gwd);
+        runner_argprintf (&runner, "%s", confpath);
         if (master)
                 runner_argprintf (&runner, ":%s", master);
         runner_add_arg (&runner, slave);
@@ -2971,64 +3716,643 @@ gf_cli3_1_gsync_config_command (dict_t *dict)
 }
 
 int
-gf_cli3_1_gsync_out_status (dict_t *dict)
+gf_cli_fetch_gsyncd_status_values (char *status,
+                                   gf_cli_gsync_status_t *sts_val)
 {
-        int              gsync_count = 0;
-        int              i = 0;
-        int              ret = 0;
-        char             mst[PATH_MAX] = {0, };
-        char             slv[PATH_MAX]= {0, };
-        char             sts[PATH_MAX] = {0, };
-        char             hyphens[81] = {0, };
-        char             *mst_val = NULL;
-        char             *slv_val = NULL;
-        char             *sts_val = NULL;
-
-        cli_out ("%-20s %-50s %-10s", "MASTER", "SLAVE", "STATUS");
-
-        for (i=0; i<sizeof(hyphens)-1; i++)
-                hyphens[i] = '-';
+        int32_t   ret      = -1;
+        char     *tmp      = NULL;
+        char     *save_ptr = NULL;
+        char     *key      = NULL;
+        char     *value    = NULL;
 
-        cli_out ("%s", hyphens);
+        if (!status || !sts_val) {
+                gf_log ("", GF_LOG_ERROR, "status or sts_val is null");
+                goto out;
+        }
 
+        tmp = strtok_r (status, "\n", &save_ptr);
 
-        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
+        if (tmp)
+                sts_val->health = gf_strdup (tmp);
+
+        while (tmp) {
+                key = strtok_r (tmp, "=", &value);
+
+                if ((key) && (!strcmp(key, "Uptime")))
+                        sts_val->uptime = gf_strdup (value);
+
+                if ((key) && (!strcmp(key, "FilesSyncd")))
+                        sts_val->files_syncd = gf_strdup (value);
+
+                if ((key) && (!strcmp(key, "FilesPending")))
+                        sts_val->files_pending = gf_strdup (value);
+
+                if ((key) && (!strcmp(key, "BytesPending"))) {
+                        value = gf_uint64_2human_readable(atol(value));
+                        sts_val->bytes_pending = gf_strdup (value);
+                }
+
+                if ((key) && (!strcmp(key, "DeletesPending")))
+                        sts_val->deletes_pending = gf_strdup (value);
+
+                tmp = strtok_r (NULL, ";", &save_ptr);
+        }
+
+        if (sts_val->health)
+                ret = 0;
+
+        if (!sts_val->uptime)
+                sts_val->uptime = gf_strdup ("N/A");
+
+        if (!sts_val->files_syncd)
+                sts_val->files_syncd = gf_strdup ("N/A");
+
+        if (!sts_val->files_pending)
+                sts_val->files_pending = gf_strdup ("N/A");
+
+        if (!sts_val->bytes_pending)
+                sts_val->bytes_pending = gf_strdup ("N/A");
+
+        if (!sts_val->deletes_pending)
+                sts_val->deletes_pending = gf_strdup ("N/A");
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d.", ret);
+        return ret;
+}
+
+char*
+get_struct_variable (int mem_num, gf_cli_gsync_status_t *sts_val)
+{
+        switch (mem_num) {
+        case 0: return (sts_val->node);
+        case 1: return (sts_val->master);
+        case 2: return (sts_val->slave);
+        case 3: return (sts_val->health);
+        case 4: return (sts_val->uptime);
+        case 5: return (sts_val->files_syncd);
+        case 6: return (sts_val->files_pending);
+        case 7: return (sts_val->bytes_pending);
+        case 8: return (sts_val->deletes_pending);
+        default:
+                 goto out;
+        }
+
+out:
+        return NULL;
+}
+
+int
+gf_cli_print_status (char **title_values,
+                     gf_cli_gsync_status_t **sts_vals,
+                     int *spacing, int gsync_count,
+                     int number_of_fields, int is_detail)
+{
+        int     indents                  = 0;
+        int     i                        = 0;
+        int     j                        = 0;
+        int     ret                      = 0;
+        int     total_spacing            = 0;
+        char  **output_values            = NULL;
+        char   *tmp                      = NULL;
+        char   *hyphens                  = NULL;
+        char    heading[PATH_MAX]        = {0, };
+        char    indent_spaces[PATH_MAX]  = {0, };
+
+        /* calculating spacing for hyphens */
+        for (i = 0; i < number_of_fields; i++) {
+                /* Suppressing master and slave output for status detail */
+                if ((is_detail) && ((i == 1) || (i == 2))) {
+                        total_spacing++;
+                        continue;
+                } else if ((!is_detail) && (i > 4)) {
+                       /* Suppressing detailed output for
+                        * status */
+                        continue;
+                }
+                spacing[i] += 3; /* Adding extra space to
+                                    distinguish between fields */
+                total_spacing += spacing[i];
+        }
+        total_spacing += 4; /* For the spacing between the fields */
+
+        /* char pointers for each field */
+        output_values = GF_CALLOC (number_of_fields, sizeof (char *),
+                                   gf_common_mt_char);
+        if (!output_values) {
+                ret = -1;
+                goto out;
+        }
+        for (i = 0; i < number_of_fields; i++) {
+                output_values[i] = GF_CALLOC (spacing[i] + 1, sizeof (char),
+                                              gf_common_mt_char);
+                if (!output_values[i]) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        hyphens = GF_CALLOC (total_spacing + 1, sizeof (char),
+                             gf_common_mt_char);
+        if (!hyphens) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf(heading, sizeof(heading), "MASTER: %s  SLAVE: %s",
+                       sts_vals[0]->master, sts_vals[0]->slave);
         if (ret) {
-                gf_log ("cli", GF_LOG_INFO, "No active geo-replication sessions"
-                        "present for the selected");
+                if (ret < sizeof(heading))
+                        heading[ret] = '\0';
+                else
+                        heading[sizeof(heading) - 1] = '\0';
                 ret = 0;
+        } else {
+                ret = -1;
                 goto out;
         }
 
-        for (i = 1; i <= gsync_count; i++) {
-                snprintf (mst, sizeof(mst), "master%d", i);
-                snprintf (slv, sizeof(slv), "slave%d", i);
-                snprintf (sts, sizeof(sts), "status%d", i);
+        if (is_detail) {
+                cli_out (" ");
+                if (strlen(heading) > total_spacing)
+                        cli_out ("%s", heading);
+                else {
+                        /* Printing the heading with centre justification */
+                        indents = (total_spacing - strlen(heading)) / 2;
+                        memset (indent_spaces, ' ', indents);
+                        indent_spaces[indents] = '\0';
+                        ret = snprintf (hyphens, total_spacing, "%s%s",
+                                        indent_spaces, heading);
+                        if (ret) {
+                                hyphens[ret] = '\0';
+                                cli_out ("%s", hyphens);
+                                ret = 0;
+                        } else {
+                                ret = -1;
+                                goto out;
+                        }
+                }
+                cli_out (" ");
+        }
+
+        /* setting the title "NODE", "MASTER", etc. from title_values[]
+           and printing the same */
+        for (j = 0; j < number_of_fields; j++) {
+                /* Suppressing master and slave output for status detail */
+                if ((is_detail) && ((j == 1) || (j == 2))) {
+                        output_values[j][0] = '\0';
+                        continue;
+                } else if ((!is_detail) && (j > 4)) {
+                       /* Suppressing detailed output for
+                        * status */
+                       output_values[j][0] = '\0';
+                        continue;
+                }
+                memset (output_values[j], ' ', spacing[j]);
+                memcpy (output_values[j], title_values[j],
+                        strlen(title_values[j]));
+                output_values[j][spacing[j]] = '\0';
+        }
+        cli_out ("%s %s %s %s %s %s %s %s %s", output_values[0],
+                 output_values[1], output_values[2], output_values[3],
+                 output_values[4], output_values[5], output_values[6],
+                 output_values[7], output_values[8]);
+
+        /* setting and printing the hyphens */
+        memset (hyphens, '-', total_spacing);
+        hyphens[total_spacing] = '\0';
+        cli_out ("%s", hyphens);
+
+        for (i = 0; i < gsync_count; i++) {
+                for (j = 0; j < number_of_fields; j++) {
+                        /* Suppressing master and slave output for
+                         * status detail */
+                        if ((is_detail) && ((j == 1) || (j == 2))) {
+                                output_values[j][0] = '\0';
+                                continue;
+                        }  else if ((!is_detail) && (j > 4)) {
+                                /* Suppressing detailed output for
+                                 * status */
+                                output_values[j][0] = '\0';
+                                continue;
+                        }
+                        tmp = get_struct_variable(j, sts_vals[i]);
+                        if (!tmp) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "struct member empty.");
+                                ret = -1;
+                                goto out;
+                        }
+                        memset (output_values[j], ' ', spacing[j]);
+                        memcpy (output_values[j], tmp, strlen (tmp));
+                        output_values[j][spacing[j]] = '\0';
+                }
+
+                cli_out ("%s %s %s %s %s %s %s %s %s", output_values[0],
+                         output_values[1], output_values[2], output_values[3],
+                         output_values[4], output_values[5], output_values[6],
+                         output_values[7], output_values[8]);
+        }
 
-                ret = dict_get_str (dict, mst, &mst_val);
+out:
+        if (output_values) {
+                for (i = 0; i < number_of_fields; i++) {
+                        if (output_values[i])
+                                GF_FREE (output_values[i]);
+                }
+                GF_FREE (output_values);
+        }
+
+        if (hyphens)
+                GF_FREE (hyphens);
+
+        return ret;
+}
+
+int
+gf_cli_read_status_data (dict_t *dict,
+                         gf_cli_gsync_status_t **sts_vals,
+                         int *spacing, int gsync_count,
+                         int number_of_fields)
+{
+        int     ret            = 0;
+        int     i              = 0;
+        int     j              = 0;
+        char    mst[PATH_MAX]  = {0, };
+        char    slv[PATH_MAX]  = {0, };
+        char    sts[PATH_MAX]  = {0, };
+        char    nds[PATH_MAX]  = {0, };
+        char   *status         = NULL;
+        char   *tmp            = NULL;
+
+        /* Storing per node status info in each object */
+        for (i = 0; i < gsync_count; i++) {
+                snprintf (nds, sizeof(nds), "node%d", i + 1);
+                snprintf (mst, sizeof(mst), "master%d", i + 1);
+                snprintf (slv, sizeof(slv), "slave%d", i + 1);
+                snprintf (sts, sizeof(sts), "status%d", i + 1);
+
+                /* Fetching the values from dict, and calculating
+                   the max length for each field */
+                ret = dict_get_str (dict, nds, &(sts_vals[i]->node));
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (dict, mst, &(sts_vals[i]->master));
                 if (ret)
                         goto out;
 
-                ret = dict_get_str (dict, slv, &slv_val);
+                ret = dict_get_str (dict, slv, &(sts_vals[i]->slave));
                 if (ret)
                         goto out;
 
-                ret = dict_get_str (dict, sts, &sts_val);
+                ret = dict_get_str (dict, sts, &status);
                 if (ret)
                         goto out;
 
-                cli_out ("%-20s %-50s %-10s", mst_val,
-                         slv_val, sts_val);
+                /* Fetching health and uptime from sts_val */
+                ret = gf_cli_fetch_gsyncd_status_values (status, sts_vals[i]);
+                if (ret)
+                        goto out;
 
+                for (j = 0; j < number_of_fields; j++) {
+                        tmp = get_struct_variable(j, sts_vals[i]);
+                        if (!tmp) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "struct member empty.");
+                                ret = -1;
+                                goto out;
+                        }
+                        if (strlen (tmp) > spacing[j])
+                                spacing[j] = strlen (tmp);
+                }
         }
 
- out:
+out:
+        return ret;
+}
+
+int
+gf_cli_gsync_status_output (dict_t *dict, int status_detail)
+{
+        int                     gsync_count    = 0;
+        int                     i              = 0;
+        int                     j              = 0;
+        int                     ret            = 0;
+        int                     spacing[10]    = {0};
+        int                     num_of_fields  = 9;
+        char                    errmsg[1024]   = "";
+        char                   *master         = NULL;
+        char                   *slave          = NULL;
+        char                   *tmp            = NULL;
+        char                   *title_values[] = {"NODE", "MASTER", "SLAVE",
+                                                  "HEALTH", "UPTIME",
+                                                  "FILES SYNCD",
+                                                  "FILES PENDING",
+                                                  "BYTES PENDING",
+                                                  "DELETES PENDING"};
+        gf_cli_gsync_status_t **sts_vals = NULL;
+
+        /* Checks if any session is active or not */
+        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
+        if (ret) {
+                ret = dict_get_str (dict, "master", &master);
+
+                ret = dict_get_str (dict, "slave", &slave);
+
+                if (master) {
+                        if (slave)
+                                snprintf (errmsg, sizeof(errmsg), "No active "
+                                          "geo-replication sessions between %s"
+                                          " and %s", master, slave);
+                        else
+                                snprintf (errmsg, sizeof(errmsg), "No active "
+                                          "geo-replication sessions for %s",
+                                          master);
+                } else
+                        snprintf (errmsg, sizeof(errmsg), "No active "
+                                  "geo-replication sessions");
+
+                gf_log ("cli", GF_LOG_INFO, "%s", errmsg);
+                cli_out ("%s", errmsg);
+                ret = 0;
+                goto out;
+        }
+
+        for (i = 0; i < num_of_fields; i++)
+                spacing[i] = strlen(title_values[i]);
+
+        /* gsync_count = number of nodes reporting output.
+           each sts_val object will store output of each
+           node */
+        sts_vals = GF_CALLOC (gsync_count, sizeof (gf_cli_gsync_status_t *),
+                              gf_common_mt_char);
+        if (!sts_vals) {
+                ret = -1;
+                goto out;
+        }
+        for (i = 0; i < gsync_count; i++) {
+                sts_vals[i] = GF_CALLOC (1, sizeof (gf_cli_gsync_status_t),
+                                         gf_common_mt_char);
+                if (!sts_vals[i]) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = gf_cli_read_status_data (dict, sts_vals, spacing,
+                                       gsync_count, num_of_fields);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read status data");
+                goto out;
+        }
+
+        ret = gf_cli_print_status (title_values, sts_vals, spacing, gsync_count,
+                                   num_of_fields, status_detail);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to print status output");
+                goto out;
+        }
+
+out:
+        if (sts_vals) {
+                for (i = 0; i < gsync_count; i++) {
+                        for (j = 3; j < num_of_fields; j++) {
+                                tmp = get_struct_variable(j, sts_vals[i]);
+                                if (tmp)
+                                        GF_FREE (tmp);
+                        }
+                }
+                GF_FREE (sts_vals);
+        }
+
+        return ret;
+}
+
+static int32_t
+write_contents_to_common_pem_file (dict_t *dict, int output_count)
+{
+        char            *workdir                   = NULL;
+        char             common_pem_file[PATH_MAX] = "";
+        char            *output                    = NULL;
+        char             output_name[PATH_MAX]     = "";
+        int              bytes_writen              = 0;
+        int              fd                        = -1;
+        int              ret                       = -1;
+        int              i                         = -1;
+
+        ret = dict_get_str (dict, "glusterd_workdir", &workdir);
+        if (ret || !workdir) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch workdir");
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (common_pem_file, sizeof(common_pem_file),
+                  "%s/geo-replication/common_secret.pem.pub",
+                  workdir);
+
+        unlink (common_pem_file);
+
+        fd = open (common_pem_file, O_WRONLY | O_CREAT, 0600);
+        if (fd == -1) {
+                gf_log ("", GF_LOG_ERROR, "Failed to open %s"
+                        " Error : %s", common_pem_file,
+                        strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 1; i <= output_count; i++) {
+                memset (output_name, '\0', sizeof (output_name));
+                snprintf (output_name, sizeof (output_name),
+                          "output_%d", i);
+                ret = dict_get_str (dict, output_name, &output);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to get %s.",
+                                output_name);
+                        cli_out ("Unable to fetch output.");
+                }
+                if (output) {
+                        bytes_writen = write (fd, output, strlen(output));
+                        if (bytes_writen != strlen(output)) {
+                                gf_log ("", GF_LOG_ERROR, "Failed to write "
+                                        "to %s", common_pem_file);
+                                ret = -1;
+                                goto out;
+                        }
+                        /* Adding the new line character */
+                        bytes_writen = write (fd, "\n", strlen("\n"));
+                        if (bytes_writen != strlen("\n")) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Failed to add new line char");
+                                ret = -1;
+                                goto out;
+                        }
+                        output = NULL;
+                }
+        }
+
+        cli_out ("Common secret pub file present at %s", common_pem_file);
+        ret = 0;
+out:
+        if (fd)
+                close (fd);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
+}
+
+int
+gf_cli_sys_exec_cbk (struct rpc_req *req, struct iovec *iov,
+                     int count, void *myframe)
+{
+        int                     ret     = -1;
+        int                     output_count     = -1;
+        int                     i     = -1;
+        char                   *output  = NULL;
+        char                   *command = NULL;
+        char                    output_name[PATH_MAX] = "";
+        gf_cli_rsp              rsp     = {0, };
+        dict_t                  *dict   = NULL;
+        call_frame_t            *frame  = NULL;
+
+        if (req->rpc_status == -1) {
+                ret = -1;
+                goto out;
+        }
+
+        frame = myframe;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+       }
+
+        dict = dict_new ();
+
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+        if (ret)
+                goto out;
+
+        if (rsp.op_ret) {
+                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
+                         "Command failed.");
+                ret = rsp.op_ret;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "output_count", &output_count);
+        if (ret) {
+                cli_out ("Command executed successfully.");
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "command", &command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get command from dict");
+                goto out;
+        }
+
+        if (!strcmp (command, "gsec_create")) {
+                ret = write_contents_to_common_pem_file (dict, output_count);
+                if (!ret)
+                        goto out;
+        }
+
+        for (i = 1; i <= output_count; i++) {
+                memset (output_name, '\0', sizeof (output_name));
+                snprintf (output_name, sizeof (output_name),
+                          "output_%d", i);
+                ret = dict_get_str (dict, output_name, &output);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to get %s.",
+                                output_name);
+                        cli_out ("Unable to fetch output.");
+                }
+                if (output) {
+                        cli_out ("%s", output);
+                        output = NULL;
+                }
+        }
+
+        ret = 0;
+out:
+        if (dict)
+                dict_unref (dict);
+        cli_cmd_broadcast_response (ret);
+
+        free (rsp.dict.dict_val);
 
+        return ret;
 }
 
 int
-gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_copy_file_cbk (struct rpc_req *req, struct iovec *iov,
+                      int count, void *myframe)
+{
+        int                     ret     = -1;
+        gf_cli_rsp              rsp     = {0, };
+        dict_t                  *dict   = NULL;
+        call_frame_t            *frame  = NULL;
+
+        if (req->rpc_status == -1) {
+                ret = -1;
+                goto out;
+        }
+
+        frame = myframe;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+       }
+
+        dict = dict_new ();
+
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+       if (ret)
+                goto out;
+
+        if (rsp.op_ret) {
+                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
+                         "Copy unsuccessful");
+                ret = rsp.op_ret;
+                goto out;
+        }
+
+        cli_out ("Successfully copied file.");
+
+out:
+        if (dict)
+                dict_unref (dict);
+        cli_cmd_broadcast_response (ret);
+
+        free (rsp.dict.dict_val);
+
+        return ret;
+}
+
+int
+gf_cli_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
                          int count, void *myframe)
 {
         int                     ret     = -1;
@@ -3037,17 +4361,22 @@ gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
         char                    *gsync_status = NULL;
         char                    *master = NULL;
         char                    *slave  = NULL;
-        int32_t                  type    = 0;
+        int32_t                  type   = 0;
+        call_frame_t            *frame  = NULL;
+        gf_boolean_t             status_detail = _gf_false;
+
 
         if (req->rpc_status == -1) {
                 ret = -1;
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to get response structure");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -3063,8 +4392,17 @@ gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
         if (ret)
                 goto out;
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_gsync (dict, rsp.op_ret, rsp.op_errno,
+                                                rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
         if (rsp.op_ret) {
-                cli_out ("%s", rsp.op_errstr ? rsp.op_errstr :
+                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
                          GEOREP" command unsuccessful");
                 ret = rsp.op_ret;
                 goto out;
@@ -3078,7 +4416,7 @@ gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get type");
+                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get type");
                 goto out;
         }
 
@@ -3098,29 +4436,50 @@ gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
                 break;
 
                 case GF_GSYNC_OPTION_TYPE_CONFIG:
-                        ret = gf_cli3_1_gsync_config_command (dict);
+                        ret = gf_cli_gsync_config_command (dict);
                 break;
 
                 case GF_GSYNC_OPTION_TYPE_STATUS:
-                        ret = gf_cli3_1_gsync_out_status (dict);
-                        goto out;
+                        status_detail = dict_get_str_boolean (dict,
+                                                              "status-detail",
+                                                              _gf_false);
+                        ret = gf_cli_gsync_status_output (dict, status_detail);
+                break;
+
+                case GF_GSYNC_OPTION_TYPE_DELETE:
+                        if (dict_get_str (dict, "master", &master) != 0)
+                                master = "???";
+                        if (dict_get_str (dict, "slave", &slave) != 0)
+                                slave = "???";
+                        cli_out ("Deleting " GEOREP " session between %s & %s"
+                                 " has been successful", master, slave);
+                break;
+
+                case GF_GSYNC_OPTION_TYPE_CREATE:
+                        if (dict_get_str (dict, "master", &master) != 0)
+                                master = "???";
+                        if (dict_get_str (dict, "slave", &slave) != 0)
+                                slave = "???";
+                        cli_out ("Creating " GEOREP " session between %s & %s"
+                                 " has been successful", master, slave);
+                break;
+
                 default:
                         cli_out (GEOREP" command executed successfully");
         }
 
 out:
-
+        if (dict)
+                dict_unref (dict);
         cli_cmd_broadcast_response (ret);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_gsync_set (call_frame_t *frame, xlator_t *this,
-                     void *data)
+gf_cli_sys_exec (call_frame_t *frame, xlator_t *this, void *data)
 {
         int                      ret    = 0;
         dict_t                  *dict   = NULL;
@@ -3128,29 +4487,67 @@ gf_cli3_1_gsync_set (call_frame_t *frame, xlator_t *this,
 
         if (!frame || !this || !data) {
                 ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid data");
                 goto out;
         }
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *) &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
+        ret = cli_to_glusterd (&req, frame, gf_cli_sys_exec_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_SYS_EXEC, this, cli_rpc_prog,
+                               NULL);
+out:
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int32_t
+gf_cli_copy_file (call_frame_t *frame, xlator_t *this, void *data)
+{
+        int                      ret    = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req               req = {{0,}};
 
+        if (!frame || !this || !data) {
+                ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid data");
                 goto out;
         }
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_GSYNC_SET, NULL,
-                              this, gf_cli3_1_gsync_set_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        dict = data;
 
+        ret = cli_to_glusterd (&req, frame, gf_cli_copy_file_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_COPY_FILE, this, cli_rpc_prog,
+                               NULL);
 out:
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int32_t
+gf_cli_gsync_set (call_frame_t *frame, xlator_t *this,
+                     void *data)
+{
+        int                      ret    = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req               req = {{0,}};
+
+        if (!frame || !this || !data) {
+                ret = -1;
+                goto out;
+        }
+
+        dict = data;
+
+        ret = cli_to_glusterd (&req, frame, gf_cli_gsync_set_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_GSYNC_SET, this, cli_rpc_prog,
+                               NULL);
+
+out:
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
@@ -3183,7 +4580,6 @@ cmd_profile_volume_brick_out (dict_t *dict, int count, int interval)
         uint64_t                sec = 0;
         uint64_t                r_count = 0;
         uint64_t                w_count = 0;
-        char                    *brick = NULL;
         uint64_t                rb_counts[32] = {0};
         uint64_t                wb_counts[32] = {0};
         cli_profile_info_t      profile_info[GF_FOP_MAXVALUE] = {{0}};
@@ -3196,9 +4592,6 @@ cmd_profile_volume_brick_out (dict_t *dict, int count, int interval)
         int                     ret = 0;
         double                  total_percentage_latency = 0;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-brick", count);
-        ret = dict_get_str (dict, key, &brick);
         for (i = 0; i < 32; i++) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "%d-%d-read-%d", count,
@@ -3233,7 +4626,7 @@ cmd_profile_volume_brick_out (dict_t *dict, int count, int interval)
                 snprintf (key, sizeof (key), "%d-%d-%d-maxlatency", count,
                           interval, i);
                 ret = dict_get_double (dict, key, &profile_info[i].max_latency);
-                profile_info[i].fop_name = gf_fop_list[i];
+                profile_info[i].fop_name = (char *)gf_fop_list[i];
 
                 total_percentage_latency +=
                        (profile_info[i].fop_hits * profile_info[i].avg_latency);
@@ -3261,7 +4654,6 @@ cmd_profile_volume_brick_out (dict_t *dict, int count, int interval)
         ret = dict_get_uint64 (dict, key, &w_count);
 
         if (ret == 0) {
-                cli_out ("Brick: %s", brick);
         }
 
         if (interval == -1)
@@ -3346,7 +4738,7 @@ cmd_profile_volume_brick_out (dict_t *dict, int count, int interval)
 }
 
 int32_t
-gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
                               int count, void *myframe)
 {
         gf_cli_rsp                        rsp   = {0,};
@@ -3358,6 +4750,9 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
         int                               i = 1;
         int32_t                           brick_count = 0;
         char                              *volname = NULL;
+        char                              *brick = NULL;
+        char                              str[1024] = {0,};
+
         if (-1 == req->rpc_status) {
                 goto out;
         }
@@ -3365,7 +4760,8 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
         gf_log ("cli", GF_LOG_DEBUG, "Received resp to profile");
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -3388,6 +4784,16 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 dict->extra_stdfree = rsp.dict.dict_val;
         }
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_profile (dict, rsp.op_ret,
+                                                  rsp.op_errno,
+                                                  rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
         ret = dict_get_str (dict, "volname", &volname);
         if (ret)
                 goto out;
@@ -3397,7 +4803,7 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, "")) {
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("%s", rsp.op_errstr);
         } else {
                 switch (op) {
                 case GF_CLI_STATS_START:
@@ -3435,11 +4841,28 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
 
         if (!brick_count) {
-                cli_out ("All bricks of volume %s are down.", volname);
+                cli_err ("All bricks of volume %s are down.", volname);
                 goto out;
         }
 
         while (i <= brick_count) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%d-brick", i);
+                ret = dict_get_str (dict, key, &brick);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Couldn't get brick name");
+                        goto out;
+                }
+
+                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
+                if (ret)
+                        snprintf (str, sizeof (str), "NFS Server : %s", brick);
+                else
+                        snprintf (str, sizeof (str), "Brick: %s", brick);
+                cli_out ("%s", str);
+                memset (str, '-', strlen (str));
+                cli_out ("%s", str);
+
                 snprintf (key, sizeof (key), "%d-cumulative", i);
                 ret = dict_get_int32 (dict, key, &interval);
                 if (ret == 0) {
@@ -3457,14 +4880,13 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         if (dict)
                 dict_unref (dict);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.op_errstr);
         cli_cmd_broadcast_response (ret);
         return ret;
 }
 
 int32_t
-gf_cli3_1_profile_volume (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_profile_volume (call_frame_t *frame, xlator_t *this, void *data)
 {
         int                        ret   = -1;
         gf_cli_req                 req   = {{0,}};
@@ -3478,58 +4900,47 @@ gf_cli3_1_profile_volume (call_frame_t *frame, xlator_t *this, void *data)
                 goto out;
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_PROFILE_VOLUME, NULL,
-                              this, gf_cli3_1_profile_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_profile_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
         return ret;
 }
 
 int32_t
-gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
                               int count, void *myframe)
 {
         gf_cli_rsp                        rsp   = {0,};
         int                               ret   = -1;
-        dict_t                            *dict = NULL;
-        gf1_cli_stats_op                op = GF_CLI_STATS_NONE;
+        dict_t                           *dict = NULL;
+        gf1_cli_stats_op                  op = GF_CLI_STATS_NONE;
         char                              key[256] = {0};
         int                               i = 0;
         int32_t                           brick_count = 0;
         char                              brick[1024];
         int32_t                           members = 0;
-        char                              *filename;
-        char                              *bricks;
-        uint64_t                           value = 0;
+        char                             *filename;
+        char                             *bricks;
+        uint64_t                          value = 0;
         int32_t                           j = 0;
         gf1_cli_top_op                    top_op = GF_CLI_TOP_NONE;
         uint64_t                          nr_open = 0;
         uint64_t                          max_nr_open = 0;
         double                            throughput = 0;
         double                            time = 0;
-        long int                          time_sec = 0;
-        long int                           time_usec = 0;
-        struct tm                         *tm = NULL;
+        int32_t                           time_sec = 0;
+        long int                          time_usec = 0;
         char                              timestr[256] = {0, };
-        char                              *openfd_str = NULL;
+        char                             *openfd_str = NULL;
+        gf_boolean_t                      nfs = _gf_false;
+        gf_boolean_t                      clear_stats = _gf_false;
+        int                               stats_cleared = 0;
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -3538,14 +4949,15 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
         gf_log ("cli", GF_LOG_DEBUG, "Received resp to top");
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to decode response");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         if (rsp.op_ret) {
                 if (strcmp (rsp.op_errstr, ""))
-                        cli_out ("%s", rsp.op_errstr);
-                cli_out ("volume top unsuccessful");
+                        cli_err ("%s", rsp.op_errstr);
+                cli_err ("volume top unsuccessful");
                 ret = rsp.op_ret;
                 goto out;
         }
@@ -3573,6 +4985,18 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 ret = 0;
                 goto out;
         }
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_top (dict, rsp.op_ret,
+                                              rsp.op_errno,
+                                              rsp.op_errstr);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                }
+                goto out;
+        }
+
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret)
                 goto out;
@@ -3580,13 +5004,35 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
         ret = dict_get_int32 (dict, key, (int32_t*)&top_op);
         if (ret)
                 goto out;
+
+        clear_stats = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+
         while (i < brick_count) {
                 i++;
                 snprintf (brick, sizeof (brick), "%d-brick", i);
                 ret = dict_get_str (dict, brick, &bricks);
                 if (ret)
                         goto out;
-                cli_out ("Brick: %s", bricks);
+
+                nfs = dict_get_str_boolean (dict, "nfs", _gf_false);
+
+                if (clear_stats) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
+                        ret = dict_get_int32 (dict, key, &stats_cleared);
+                        if (ret)
+                                goto out;
+                        cli_out (stats_cleared ? "Cleared stats for %s %s" :
+                                 "Failed to clear stats for %s %s",
+                                 nfs ? "NFS server on" : "brick", bricks);
+                        continue;
+                }
+
+                if (nfs)
+                        cli_out ("NFS Server : %s", bricks);
+                else
+                        cli_out ("Brick: %s", bricks);
+
                 snprintf(key, sizeof (key), "%d-members", i);
                 ret = dict_get_int32 (dict, key, &members);
 
@@ -3663,11 +5109,9 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 ret = dict_get_int32 (dict, key, (int32_t *)&time_usec);
                                 if (ret)
                                         goto out;
-                                tm    = localtime (&time_sec);
-                                if (!tm)
-                                        goto out;
-                                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                                gf_time_fmt (timestr, sizeof timestr,
+                                             time_sec, gf_timefmt_FT);
+                                snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
                                   ".%"GF_PRI_SUSECONDS, time_usec);
                                 if (strlen (filename) < VOL_TOP_PERF_FILENAME_DEF_WIDTH)
                                         cli_out ("%*"PRIu64" %-*s %-*s",
@@ -3699,13 +5143,12 @@ out:
         if (dict)
                 dict_unref (dict);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
         return ret;
 }
 
 int32_t
-gf_cli3_1_top_volume (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_top_volume (call_frame_t *frame, xlator_t *this, void *data)
 {
         int                        ret   = -1;
         gf_cli_req                 req   = {{0,}};
@@ -3719,32 +5162,20 @@ gf_cli3_1_top_volume (call_frame_t *frame, xlator_t *this, void *data)
                 goto out;
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_PROFILE_VOLUME, NULL,
-                              this, gf_cli3_1_top_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_top_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
         return ret;
 }
 
 
 int
-gf_cli3_1_getwd_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_getwd_cbk (struct rpc_req *req, struct iovec *iov,
                        int count, void *myframe)
 {
         gf1_cli_getwd_rsp rsp   = {0,};
@@ -3755,8 +5186,15 @@ gf_cli3_1_getwd_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_getwd_rsp);
-        if (ret < 0 || rsp.op_ret == -1) {
-                gf_log ("", GF_LOG_ERROR, "error");
+        if (ret < 0) {
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        if (rsp.op_ret == -1) {
+                cli_err ("getwd failed");
+                ret = rsp.op_ret;
                 goto out;
         }
 
@@ -3772,7 +5210,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_getwd (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_getwd (call_frame_t *frame, xlator_t *this, void *data)
 {
         int                      ret = -1;
         gf1_cli_getwd_req        req = {0,};
@@ -3785,7 +5223,7 @@ gf_cli3_1_getwd (call_frame_t *frame, xlator_t *this, void *data)
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_GETWD, NULL,
-                              this, gf_cli3_1_getwd_cbk,
+                              this, gf_cli_getwd_cbk,
                               (xdrproc_t) xdr_gf1_cli_getwd_req);
 
 out:
@@ -3805,6 +5243,8 @@ cli_print_volume_status_mempool (dict_t *dict, char *prefix)
         uint64_t        paddedsizeof = 0;
         uint64_t        alloccount = 0;
         int32_t         maxalloc = 0;
+        uint64_t        pool_misses = 0;
+        int32_t         maxstdalloc = 0;
         char            key[1024] = {0,};
         int             i = 0;
 
@@ -3818,10 +5258,12 @@ cli_print_volume_status_mempool (dict_t *dict, char *prefix)
                 goto out;
 
         cli_out ("Mempool Stats\n-------------");
-        cli_out ("%-30s %9s %9s %12s %10s %8s", "Name", "HotCount","ColdCount",
-                 "PaddedSizeof", "AllocCount", "MaxAlloc");
-        cli_out ("%-30s %9s %9s %12s %10s %8s", "----", "--------", "---------",
-                 "------------", "----------", "--------");
+        cli_out ("%-30s %9s %9s %12s %10s %8s %8s %12s", "Name", "HotCount",
+                 "ColdCount", "PaddedSizeof", "AllocCount", "MaxAlloc",
+                 "Misses", "Max-StdAlloc");
+        cli_out ("%-30s %9s %9s %12s %10s %8s %8s %12s", "----", "--------",
+                 "---------", "------------", "----------",
+                 "--------", "--------", "------------");
 
         for (i = 0; i < mempool_count; i++) {
                 memset (key, 0, sizeof (key));
@@ -3861,9 +5303,21 @@ cli_print_volume_status_mempool (dict_t *dict, char *prefix)
                 if (ret)
                         goto out;
 
-                cli_out ("%-30s %9d %9d %12"PRIu64" %10"PRIu64" %8d", name,
-                         hotcount, coldcount, paddedsizeof, alloccount,
-                         maxalloc);
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.max-stdalloc", prefix, i);
+                ret = dict_get_int32 (dict, key, &maxstdalloc);
+                if (ret)
+                        goto out;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.pool-misses", prefix, i);
+                ret = dict_get_uint64 (dict, key, &pool_misses);
+                if (ret)
+                        goto out;
+
+                cli_out ("%-30s %9d %9d %12"PRIu64" %10"PRIu64" %8d %8"PRIu64
+                         " %12d", name, hotcount, coldcount, paddedsizeof,
+                         alloccount, maxalloc, pool_misses, maxstdalloc);
         }
 
 out:
@@ -3872,7 +5326,7 @@ out:
 }
 
 void
-cli_print_volume_status_mem (dict_t *dict)
+cli_print_volume_status_mem (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
@@ -3880,7 +5334,9 @@ cli_print_volume_status_mem (dict_t *dict)
         char            *path = NULL;
         int             online = -1;
         char            key[1024] = {0,};
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         int             val = 0;
         int             i = 0;
 
@@ -3891,24 +5347,32 @@ cli_print_volume_status_mem (dict_t *dict)
                 goto out;
         cli_out ("Memory status for volume : %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for (i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.hostname", i);
                 ret = dict_get_str (dict, key, &hostname);
                 if (ret)
-                        goto out;
+                        continue;
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.path", i);
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
-                        goto out;
-                cli_out ("Brick : %s:%s", hostname, path);
+                        continue;
+                if (notbrick)
+                        cli_out ("%s : %s", hostname, path);
+                else
+                        cli_out ("Brick : %s:%s", hostname, path);
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.status", i);
@@ -3916,7 +5380,10 @@ cli_print_volume_status_mem (dict_t *dict)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4003,11 +5470,13 @@ out:
 }
 
 void
-cli_print_volume_status_clients (dict_t *dict)
+cli_print_volume_status_clients (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -4026,11 +5495,16 @@ cli_print_volume_status_clients (dict_t *dict)
                 goto out;
         cli_out ("Client connections for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for ( i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -4043,7 +5517,11 @@ cli_print_volume_status_clients (dict_t *dict)
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
                         goto out;
-                cli_out ("Brick : %s:%s", hostname, path);
+
+                if (notbrick)
+                        cli_out ("%s : %s", hostname, path);
+                else
+                        cli_out ("Brick : %s:%s", hostname, path);
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.status", i);
@@ -4051,7 +5529,10 @@ cli_print_volume_status_clients (dict_t *dict)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4062,6 +5543,9 @@ cli_print_volume_status_clients (dict_t *dict)
                         goto out;
 
                 cli_out ("Clients connected : %d", client_count);
+                if (client_count == 0)
+                        continue;
+
                 cli_out ("%-48s %15s %15s", "Hostname", "BytesRead",
                          "BytesWritten");
                 cli_out ("%-48s %15s %15s", "--------", "---------",
@@ -4106,6 +5590,7 @@ cli_print_volume_status_inode_entry (dict_t *dict, char *prefix)
         uint64_t        nlookup = 0;
         uint32_t        ref = 0;
         int             ia_type = 0;
+        char            inode_type;
 
         GF_ASSERT (dict);
         GF_ASSERT (prefix);
@@ -4134,8 +5619,35 @@ cli_print_volume_status_inode_entry (dict_t *dict, char *prefix)
         if (ret)
                 goto out;
 
-        cli_out ("%-40s %14"PRIu64" %14"PRIu32" %9d",
-                 gfid, nlookup, ref, ia_type);
+        switch (ia_type) {
+        case IA_IFREG:
+                inode_type = 'R';
+                break;
+        case IA_IFDIR:
+                inode_type = 'D';
+                break;
+        case IA_IFLNK:
+                inode_type = 'L';
+                break;
+        case IA_IFBLK:
+                inode_type = 'B';
+                break;
+        case IA_IFCHR:
+                inode_type = 'C';
+                break;
+        case IA_IFIFO:
+                inode_type = 'F';
+                break;
+        case IA_IFSOCK:
+                inode_type = 'S';
+                break;
+        default:
+                inode_type = 'I';
+                break;
+        }
+
+        cli_out ("%-40s %14"PRIu64" %14"PRIu32" %9c",
+                 gfid, nlookup, ref, inode_type);
 
 out:
         return;
@@ -4216,11 +5728,13 @@ out:
 }
 
 void
-cli_print_volume_status_inode (dict_t *dict)
+cli_print_volume_status_inode (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -4236,11 +5750,16 @@ cli_print_volume_status_inode (dict_t *dict)
                 goto out;
         cli_out ("Inode tables for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for (i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for ( i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -4253,7 +5772,10 @@ cli_print_volume_status_inode (dict_t *dict)
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
                         goto out;
-                cli_out ("Brick : %s:%s", hostname, path);
+                if (notbrick)
+                        cli_out ("%s : %s", hostname, path);
+                else
+                        cli_out ("Brick : %s:%s", hostname, path);
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.status", i);
@@ -4261,7 +5783,10 @@ cli_print_volume_status_inode (dict_t *dict)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4331,7 +5856,7 @@ cli_print_volume_status_fdtable (dict_t *dict, char *prefix)
         if (ret)
                 goto out;
         if (0 == openfds) {
-                cli_out ("No open fds");
+                cli_err ("No open fds");
                 goto out;
         }
 
@@ -4369,11 +5894,13 @@ out:
 }
 
 void
-cli_print_volume_status_fd (dict_t *dict)
+cli_print_volume_status_fd (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -4389,11 +5916,16 @@ cli_print_volume_status_fd (dict_t *dict)
                 goto out;
         cli_out ("FD tables for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
         if (ret)
                 goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
+        if (ret)
+                goto out;
+
+        index_max = brick_index_max + other_count;
 
-        for (i = 0; i < brick_count; i++) {
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -4406,7 +5938,11 @@ cli_print_volume_status_fd (dict_t *dict)
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
                         goto out;
-                cli_out ("Brick : %s:%s", hostname, path);
+
+                if (notbrick)
+                        cli_out ("%s : %s", hostname, path);
+                else
+                        cli_out ("Brick : %s:%s", hostname, path);
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.status", i);
@@ -4414,7 +5950,10 @@ cli_print_volume_status_fd (dict_t *dict)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4583,11 +6122,13 @@ cli_print_volume_status_call_stack (dict_t *dict, char *prefix)
 }
 
 void
-cli_print_volume_status_callpool (dict_t *dict)
+cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -4603,11 +6144,16 @@ cli_print_volume_status_callpool (dict_t *dict)
                 goto out;
         cli_out ("Pending calls for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
         if (ret)
                 goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
+        if (ret)
+                goto out;
+
+        index_max = brick_index_max + other_count;
 
-        for (i = 0; i < brick_count; i++) {
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -4620,7 +6166,11 @@ cli_print_volume_status_callpool (dict_t *dict)
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
                         goto out;
-                cli_out ("Brick : %s:%s", hostname, path);
+
+                if (notbrick)
+                        cli_out ("%s : %s", hostname, path);
+                else
+                        cli_out ("Brick : %s:%s", hostname, path);
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.status", i);
@@ -4628,7 +6178,10 @@ cli_print_volume_status_callpool (dict_t *dict)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4657,37 +6210,194 @@ out:
         return;
 }
 
+static void
+cli_print_volume_status_tasks (dict_t *dict)
+{
+        int             ret         = -1;
+        int             i           = 0;
+        int             j           = 0;
+        int             count       = 0;
+        int             task_count  = 0;
+        int             status      = 0;
+        char           *op          = NULL;
+        char           *task_id_str = NULL;
+        char           *volname     = NULL;
+        char            key[1024]   = {0,};
+        char            task[1024]  = {0,};
+        char           *brick       = NULL;
+        char           *src_brick   = NULL;
+        char           *dest_brick  = NULL;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, "tasks", &task_count);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to get tasks count");
+                return;
+        }
+
+        cli_out ("Task Status of Volume %s", volname);
+        cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
+
+        if (task_count == 0) {
+                cli_out ("There are no active volume tasks");
+                cli_out (" ");
+                return;
+        }
+
+        for (i = 0; i < task_count; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.type", i);
+                ret = dict_get_str(dict, key, &op);
+                if (ret)
+                        return;
+                cli_out ("%-20s : %-20s", "Task", op);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.id", i);
+                ret = dict_get_str (dict, key, &task_id_str);
+                if (ret)
+                        return;
+                cli_out ("%-20s : %-20s", "ID", task_id_str);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.status", i);
+                ret = dict_get_int32 (dict, key, &status);
+                if (ret)
+                        return;
+
+                snprintf (task, sizeof (task), "task%d", i);
+
+                /*
+                   Replace brick only has two states - In progress and Complete
+                   Ref: xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+                */
+
+                if (!strcmp (op, "Replace brick")) {
+                        if (status)
+                                status = GF_DEFRAG_STATUS_COMPLETE;
+                        else
+                                status = GF_DEFRAG_STATUS_STARTED;
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%s.src-brick", task);
+                        ret = dict_get_str (dict, key, &src_brick);
+                        if (ret)
+                                goto out;
+
+                        cli_out ("%-20s : %-20s", "Source Brick", src_brick);
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%s.dst-brick", task);
+                        ret = dict_get_str (dict, key, &dest_brick);
+                        if (ret)
+                                goto out;
+
+                        cli_out ("%-20s : %-20s", "Destination Brick",
+                                 dest_brick);
+
+                } else if (!strcmp (op, "Remove brick")) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%s.count", task);
+                        ret = dict_get_int32 (dict, key, &count);
+                        if (ret)
+                                goto out;
+
+                        cli_out ("%-20s", "Removed bricks:");
+
+                        for (j = 1; j <= count; j++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key),"%s.brick%d",
+                                          task, j);
+                                ret = dict_get_str (dict, key, &brick);
+                                if (ret)
+                                        goto out;
+
+                                cli_out ("%-20s", brick);
+                        }
+                }
+                cli_out ("%-20s : %-20s", "Status",
+                         cli_vol_task_status_str[status]);
+                cli_out (" ");
+        }
+
+out:
+        return;
+}
+
 static int
-gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_status_cbk (struct rpc_req *req, struct iovec *iov,
                       int count, void *myframe)
 {
-        int                             ret            = -1;
-        int                             i              = 0;
-        int                             pid            = -1;
-        uint32_t                        cmd            = 0;
-        char                            key[1024]      = {0,};
-        char                           *hostname       = NULL;
-        char                           *path           = NULL;
-        char                           *volname        = NULL;
-        dict_t                         *dict           = NULL;
-        gf_cli_rsp                      rsp            = {0,};
-        cli_volume_status_t                 status         = {0};
+        int                             ret             = -1;
+        int                             brick_index_max = -1;
+        int                             other_count     = 0;
+        int                             index_max       = 0;
+        int                             i               = 0;
+        int                             pid             = -1;
+        uint32_t                        cmd             = 0;
+        gf_boolean_t                    notbrick        = _gf_false;
+        char                            key[1024]       = {0,};
+        char                           *hostname        = NULL;
+        char                           *path            = NULL;
+        char                           *volname         = NULL;
+        dict_t                         *dict            = NULL;
+        gf_cli_rsp                      rsp             = {0,};
+        cli_volume_status_t             status          = {0};
+        cli_local_t                    *local           = NULL;
+        gf_boolean_t                    wipe_local      = _gf_false;
+        char                            msg[1024]       = {0,};
 
         if (req->rpc_status == -1)
                 goto out;
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Volume status response error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_DEBUG, "Received response to status cmd");
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_out ("Unable to obtain volume status information.");
+        local = ((call_frame_t *)myframe)->local;
+        if (!local) {
+                local = cli_local_get ();
+                if (!local) {
+                        ret = -1;
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to get local");
+                        goto out;
+                }
+                wipe_local = _gf_true;
+        }
+
+        if (rsp.op_ret) {
+                if (strcmp (rsp.op_errstr, ""))
+                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+                else
+                        snprintf (msg, sizeof (msg), "Unable to obtain volume "
+                                  "status information.");
+
+                if (global_state->mode & GLUSTER_MODE_XML) {
+                        if (!local->all)
+                                cli_xml_output_str ("volStatus", msg,
+                                                    rsp.op_ret, rsp.op_errno,
+                                                    rsp.op_errstr);
+                        ret = 0;
+                        goto out;
+                }
+
+                cli_err ("%s", msg);
+                if (local && local->all) {
+                        ret = 0;
+                        cli_out (" ");
+                } else
+                        ret = -1;
+
+                goto out;
+        }
 
         dict = dict_new ();
         if (!dict)
@@ -4704,16 +6414,56 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
 
         if ((cmd & GF_CLI_STATUS_ALL)) {
-                ((call_frame_t *)myframe)->local = dict;
-                ret = 0;
+                if (local && local->dict) {
+                        dict_ref (dict);
+                        ret = dict_set_static_ptr (local->dict, "rsp-dict", dict);
+                        ret = 0;
+                } else {
+                        gf_log ("cli", GF_LOG_ERROR, "local not found");
+                        ret = -1;
+                }
                 goto out;
         }
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                goto out;
-        if (count == 0) {
-                ret = -1;
+        if ((cmd & GF_CLI_STATUS_NFS) || (cmd & GF_CLI_STATUS_SHD))
+                notbrick = _gf_true;
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                if (!local->all) {
+                        ret = cli_xml_output_vol_status_begin (local,
+                                                               rsp.op_ret,
+                                                               rsp.op_errno,
+                                                               rsp.op_errstr);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                                goto out;
+                        }
+                }
+                if (cmd & GF_CLI_STATUS_TASKS) {
+                        ret = cli_xml_output_vol_status_tasks_detail (local,
+                                                                      dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,"Error outputting "
+                                        "to xml");
+                                goto out;
+                        }
+                } else {
+                        ret = cli_xml_output_vol_status (local, dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                                goto out;
+                        }
+                }
+
+                if (!local->all) {
+                        ret = cli_xml_output_vol_status_end (local);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                        }
+                }
                 goto out;
         }
 
@@ -4721,23 +6471,27 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
 
         switch (cmd & GF_CLI_STATUS_MASK) {
                 case GF_CLI_STATUS_MEM:
-                        cli_print_volume_status_mem (dict);
+                        cli_print_volume_status_mem (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_CLIENTS:
-                        cli_print_volume_status_clients (dict);
+                        cli_print_volume_status_clients (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_INODE:
-                        cli_print_volume_status_inode (dict);
+                        cli_print_volume_status_inode (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_FD:
-                        cli_print_volume_status_fd (dict);
+                        cli_print_volume_status_fd (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_CALLPOOL:
-                        cli_print_volume_status_callpool (dict);
+                        cli_print_volume_status_callpool (dict, notbrick);
+                        goto cont;
+                        break;
+                case GF_CLI_STATUS_TASKS:
+                        cli_print_volume_status_tasks (dict);
                         goto cont;
                         break;
                 default:
@@ -4748,48 +6502,68 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
         if (ret)
                 goto out;
 
-        cli_out ("\nSTATUS OF VOLUME: %s", volname);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
 
-        if ((cmd & GF_CLI_STATUS_DETAIL) == 0)
-                cli_out ("BRICK\t\t\t\t\t\t\tPORT\tONLINE\tPID");
+        ret = dict_get_int32 (dict, "other-count", &other_count);
+        if (ret)
+                goto out;
 
-        for (i = 0; i < count; i++) {
+        index_max = brick_index_max + other_count;
+
+
+        cli_out ("Status of volume: %s", volname);
 
+        if ((cmd & GF_CLI_STATUS_DETAIL) == 0) {
+                cli_out ("Gluster process\t\t\t\t\t\tPort\tOnline\tPid");
                 cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
+        }
+
+        for (i = 0; i <= index_max; i++) {
+
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.hostname", i);
                 ret = dict_get_str (dict, key, &hostname);
                 if (ret)
-                        goto out;
+                        continue;
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.path", i);
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
-                        goto out;
+                        continue;
 
+                /* Brick/not-brick is handled seperately here as all
+                 * types of nodes are contained in the default output
+                 */
                 memset (status.brick, 0, PATH_MAX + 255);
-                snprintf (status.brick, PATH_MAX + 255, "%s:%s",
-                          hostname, path);
+                if (!strcmp (hostname, "NFS Server") ||
+                    !strcmp (hostname, "Self-heal Daemon"))
+                        snprintf (status.brick, PATH_MAX + 255, "%s on %s",
+                                  hostname, path);
+                else
+                        snprintf (status.brick, PATH_MAX + 255, "Brick %s:%s",
+                                  hostname, path);
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.port", i);
                 ret = dict_get_int32 (dict, key, &(status.port));
                 if (ret)
-                        goto out;
+                        continue;
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.status", i);
                 ret = dict_get_int32 (dict, key, &(status.online));
                 if (ret)
-                        goto out;
+                        continue;
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.pid", i);
                 ret = dict_get_int32 (dict, key, &pid);
                 if (ret)
-                        goto out;
+                        continue;
                 if (pid == -1)
                         ret = gf_asprintf (&(status.pid_str), "%s", "N/A");
                 else
@@ -4802,26 +6576,34 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
                         ret = cli_get_detail_status (dict, i, &status);
                         if (ret)
                                 goto out;
+                        cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
                         cli_print_detailed_status (&status);
 
                 } else {
                         cli_print_brick_status (&status);
                 }
         }
+        cli_out (" ");
 
+        if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE)
+                cli_print_volume_status_tasks (dict);
 cont:
         ret = rsp.op_ret;
 
 out:
-        if (status.brick)
-                GF_FREE (status.brick);
+        if (dict)
+                dict_unref (dict);
+        GF_FREE (status.brick);
+        if (local && wipe_local) {
+                cli_local_wipe (local);
+        }
 
         cli_cmd_broadcast_response (ret);
         return ret;
 }
 
 int32_t
-gf_cli3_1_status_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_status_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req                      req  = {{0,}};
@@ -4833,20 +6615,10 @@ gf_cli3_1_status_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_STATUS_VOLUME, NULL,
-                              this, gf_cli3_1_status_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
-
+        ret = cli_to_glusterd (&req, frame, gf_cli_status_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_STATUS_VOLUME, this, cli_rpc_prog,
+                               NULL);
  out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning: %d", ret);
         return ret;
@@ -4861,29 +6633,32 @@ gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
         uint32_t         cmd          = 0;
         char             key[1024]    = {0};
         char            *volname      = NULL;
-        dict_t          *vol_dict     = NULL;
+        void            *vol_dict     = NULL;
         dict_t          *dict         = NULL;
+        cli_local_t     *local        = NULL;
 
-        dict = (dict_t *)data;
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
+        if (frame->local) {
+                local = frame->local;
+                local->all = _gf_true;
+        } else
                 goto out;
 
-        ret = gf_cli3_1_status_volume (frame, this, data);
+        ret = dict_get_uint32 (local->dict, "cmd", &cmd);
         if (ret)
                 goto out;
 
-        vol_dict =  (dict_t *)(frame->local);
 
-        ret = dict_get_int32 (vol_dict, "vol_count", &vol_count);
-        if (ret) {
-                cli_out ("Failed to get names of volumes");
+        ret = gf_cli_status_volume (frame, this, data);
+        if (ret)
                 goto out;
-        }
 
-        if (vol_count == 0) {
-                cli_out ("No volumes present");
-                ret = 0;
+        ret = dict_get_ptr (local->dict, "rsp-dict", &vol_dict);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 ((dict_t *)vol_dict, "vol_count", &vol_count);
+        if (ret) {
+                cli_err ("Failed to get names of volumes");
                 goto out;
         }
 
@@ -4891,6 +6666,22 @@ gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
         cmd &= ~GF_CLI_STATUS_ALL;
         cmd |= GF_CLI_STATUS_VOL;
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                //TODO: Pass proper op_* values
+                ret = cli_xml_output_vol_status_begin (local, 0,0, NULL);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                        goto out;
+                }
+        }
+
+        if (vol_count == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
+                cli_err ("No volumes present");
+                ret = 0;
+                goto out;
+        }
+
         for (i = 0; i < vol_count; i++) {
 
                 dict = dict_new ();
@@ -4903,7 +6694,7 @@ gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
                 if (ret)
                         goto out;
 
-                ret = dict_set_dynstr (dict, "volname", volname);
+                ret = dict_set_str (dict, "volname", volname);
                 if (ret)
                         goto out;
 
@@ -4911,23 +6702,38 @@ gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
                 if (ret)
                         goto out;
 
-                ret = gf_cli3_1_status_volume (frame, this, dict);
+                ret = gf_cli_status_volume (frame, this, dict);
                 if (ret)
                         goto out;
 
                 dict_unref (dict);
         }
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_status_end (local);
+        }
+
  out:
         if (ret)
                 gf_log ("cli", GF_LOG_ERROR, "status all failed");
+
+        if (vol_dict)
+                dict_unref (vol_dict);
+
         if (ret && dict)
                 dict_unref (dict);
+
+        if (local)
+                cli_local_wipe (local);
+
+        if (frame)
+                frame->local = NULL;
+
         return ret;
 }
 
 static int
-gf_cli3_1_mount_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_mount_cbk (struct rpc_req *req, struct iovec *iov,
                   int count, void *myframe)
 {
         gf1_cli_mount_rsp rsp   = {0,};
@@ -4939,7 +6745,8 @@ gf_cli3_1_mount_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_mount_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -4950,9 +6757,9 @@ gf_cli3_1_mount_cbk (struct rpc_req *req, struct iovec *iov,
                 cli_out ("%s", rsp.path);
         } else {
                 /* weird sounding but easy to parse... */
-                cli_out ("%d : failed with this errno (%s)",
+                cli_err ("%d : failed with this errno (%s)",
                          rsp.op_errno, strerror (rsp.op_errno));
-                ret = 1;
+                ret = -1;
         }
 
 out:
@@ -4961,7 +6768,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_mount (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_mount (call_frame_t *frame, xlator_t *this, void *data)
 {
         gf1_cli_mount_req  req  = {0,};
         int                ret  = -1;
@@ -4977,7 +6784,7 @@ gf_cli3_1_mount (call_frame_t *frame, xlator_t *this, void *data)
 
         req.label = label;
         ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
+                                           &req.dict.dict_len);
         if (ret) {
                 ret = -1;
                 goto out;
@@ -4985,7 +6792,7 @@ gf_cli3_1_mount (call_frame_t *frame, xlator_t *this, void *data)
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_MOUNT, NULL,
-                              this, gf_cli3_1_mount_cbk,
+                              this, gf_cli_mount_cbk,
                               (xdrproc_t)xdr_gf1_cli_mount_req);
 
 out:
@@ -4994,7 +6801,7 @@ out:
 }
 
 static int
-gf_cli3_1_umount_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_umount_cbk (struct rpc_req *req, struct iovec *iov,
                    int count, void *myframe)
 {
         gf1_cli_umount_rsp rsp   = {0,};
@@ -5006,7 +6813,8 @@ gf_cli3_1_umount_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_umount_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -5015,8 +6823,8 @@ gf_cli3_1_umount_cbk (struct rpc_req *req, struct iovec *iov,
         if (rsp.op_ret == 0)
                 ret = 0;
         else {
-                cli_out ("umount failed");
-                ret = 1;
+                cli_err ("umount failed");
+                ret = -1;
         }
 
 out:
@@ -5025,7 +6833,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_umount (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_umount (call_frame_t *frame, xlator_t *this, void *data)
 {
         gf1_cli_umount_req  req  = {0,};
         int                ret  = -1;
@@ -5047,7 +6855,7 @@ gf_cli3_1_umount (call_frame_t *frame, xlator_t *this, void *data)
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_UMOUNT, NULL,
-                              this, gf_cli3_1_umount_cbk,
+                              this, gf_cli_umount_cbk,
                               (xdrproc_t)xdr_gf1_cli_umount_req);
 
  out:
@@ -5055,8 +6863,212 @@ gf_cli3_1_umount (call_frame_t *frame, xlator_t *this, void *data)
         return ret;
 }
 
+void
+cmd_heal_volume_statistics_out (dict_t *dict, int  brick)
+{
+
+        uint64_t        num_entries = 0;
+        int             ret = 0;
+        char            key[256] = {0};
+        char            *hostname = NULL;
+        uint64_t        i = 0;
+        uint64_t        healed_count = 0;
+        uint64_t        split_brain_count = 0;
+        uint64_t        heal_failed_count = 0;
+        char            *start_time_str = NULL;
+        char            *end_time_str = NULL;
+        char            *crawl_type = NULL;
+        int             progress = -1;
+
+        snprintf (key, sizeof key, "%d-hostname", brick);
+        ret = dict_get_str (dict, key, &hostname);
+        if (ret)
+                goto out;
+        cli_out ("------------------------------------------------");
+        cli_out ("\nCrawl statistics for brick no %d", brick);
+        cli_out ("Hostname of brick %s", hostname);
+
+        snprintf (key, sizeof key, "statistics-%d-count", brick);
+        ret = dict_get_uint64 (dict, key, &num_entries);
+        if (ret)
+                goto out;
+
+        for (i = 0; i < num_entries; i++)
+        {
+                snprintf (key, sizeof key, "statistics_crawl_type-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_str (dict, key, &crawl_type);
+                if (ret)
+                        goto out;
+
+                snprintf (key, sizeof key, "statistics_healed_cnt-%d-%"PRIu64,
+                          brick,i);
+                ret = dict_get_uint64 (dict, key, &healed_count);
+                if (ret)
+                        goto out;
+
+                snprintf (key, sizeof key, "statistics_sb_cnt-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_uint64 (dict, key, &split_brain_count);
+                if (ret)
+                        goto out;
+                snprintf (key, sizeof key, "statistics_heal_failed_cnt-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_uint64 (dict, key, &heal_failed_count);
+                if (ret)
+                        goto out;
+                snprintf (key, sizeof key, "statistics_strt_time-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_str (dict, key,  &start_time_str);
+                if (ret)
+                        goto out;
+                snprintf (key, sizeof key, "statistics_end_time-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_str (dict, key, &end_time_str);
+                if (ret)
+                        goto out;
+                snprintf (key, sizeof key, "statistics_inprogress-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_int32 (dict, key, &progress);
+                if (ret)
+                        goto out;
+
+                cli_out ("\nStarting time of crawl: %s", start_time_str);
+                if (progress == 1)
+                        cli_out ("Crawl is in progress");
+                else
+                        cli_out ("Ending time of crawl: %s", end_time_str);
+
+                cli_out ("Type of crawl: %s", crawl_type);
+                cli_out ("No. of entries healed: %"PRIu64,
+                         healed_count);
+                cli_out ("No. of entries in split-brain: %"PRIu64,
+                        split_brain_count);
+                cli_out ("No. of heal failed entries: %"PRIu64,
+                         heal_failed_count);
+
+        }
+
+
+out:
+        return;
+}
+
+void
+cmd_heal_volume_brick_out (dict_t *dict, int brick)
+{
+        uint64_t        num_entries = 0;
+        int             ret = 0;
+        char            key[256] = {0};
+        char           *hostname = NULL;
+        char           *path = NULL;
+        char           *status = NULL;
+        uint64_t        i = 0;
+        uint32_t        time = 0;
+        char            timestr[32] = {0};
+        char            *shd_status = NULL;
+
+        snprintf (key, sizeof key, "%d-hostname", brick);
+        ret = dict_get_str (dict, key, &hostname);
+        if (ret)
+                goto out;
+        snprintf (key, sizeof key, "%d-path", brick);
+        ret = dict_get_str (dict, key, &path);
+        if (ret)
+                goto out;
+        cli_out ("\nBrick %s:%s", hostname, path);
+
+        snprintf (key, sizeof key, "%d-status", brick);
+        ret = dict_get_str (dict, key, &status);
+        if (status && strlen (status))
+                cli_out ("Status: %s", status);
+
+        snprintf (key, sizeof key, "%d-shd-status",brick);
+        ret = dict_get_str (dict, key, &shd_status);
+
+        if(!shd_status)
+        {
+                snprintf (key, sizeof key, "%d-count", brick);
+                ret = dict_get_uint64 (dict, key, &num_entries);
+                cli_out ("Number of entries: %"PRIu64, num_entries);
+
+
+                for (i = 0; i < num_entries; i++) {
+                        snprintf (key, sizeof key, "%d-%"PRIu64, brick, i);
+                        ret = dict_get_str (dict, key, &path);
+                        if (ret)
+                                continue;
+                        time = 0;
+                        snprintf (key, sizeof key, "%d-%"PRIu64"-time",
+                                  brick, i);
+                        ret = dict_get_uint32 (dict, key, &time);
+                        if (!time) {
+                                cli_out ("%s", path);
+                        } else {
+                                gf_time_fmt (timestr, sizeof timestr,
+                                             time, gf_timefmt_FT);
+                                if (i == 0) {
+                                cli_out ("at                    path on brick");
+                                cli_out ("-----------------------------------");
+                                }
+                                cli_out ("%s %s", timestr, path);
+                        }
+                }
+        }
+
+out:
+        return;
+}
+
+
+void
+cmd_heal_volume_statistics_heal_count_out (dict_t *dict, int brick)
+{
+        uint64_t        num_entries = 0;
+        int             ret = 0;
+        char            key[256] = {0};
+        char           *hostname = NULL;
+        char           *path = NULL;
+        char           *status = NULL;
+        char            *shd_status = NULL;
+
+        snprintf (key, sizeof key, "%d-hostname", brick);
+        ret = dict_get_str (dict, key, &hostname);
+        if (ret)
+                goto out;
+        snprintf (key, sizeof key, "%d-path", brick);
+        ret = dict_get_str (dict, key, &path);
+        if (ret)
+                goto out;
+        cli_out ("\nBrick %s:%s", hostname, path);
+
+        snprintf (key, sizeof key, "%d-status", brick);
+        ret = dict_get_str (dict, key, &status);
+        if (status && strlen (status))
+                cli_out ("Status: %s", status);
+
+        snprintf (key, sizeof key, "%d-shd-status",brick);
+        ret = dict_get_str (dict, key, &shd_status);
+
+        if(!shd_status)
+        {
+                snprintf (key, sizeof key, "%d-hardlinks", brick);
+                ret = dict_get_uint64 (dict, key, &num_entries);
+                if (ret)
+                        cli_out ("No gathered input for this brick");
+                else
+                        cli_out ("Number of entries: %"PRIu64, num_entries);
+
+
+        }
+
+out:
+        return;
+}
+
+
 int
-gf_cli3_1_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp              rsp   = {0,};
@@ -5064,64 +7076,182 @@ gf_cli3_1_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
         cli_local_t             *local = NULL;
         char                    *volname = NULL;
         call_frame_t            *frame = NULL;
+        dict_t                  *input_dict = NULL;
         dict_t                  *dict = NULL;
+        int                     brick_count = 0;
+        int                     i = 0;
+        gf_xl_afr_op_t          heal_op = GF_AFR_OP_INVALID;
+        char                    *operation = NULL;
+        char                    *substr = NULL;
+        char                    *heal_op_str = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        frame = myframe;
-
-        if (frame) {
+        if (frame)
                 local = frame->local;
-                frame->local = NULL;
+
+        if (local) {
+                input_dict = local->dict;
+                ret = dict_get_int32 (input_dict, "heal-op",
+                                      (int32_t*)&heal_op);
+        }
+//TODO: Proper XML output
+//#if (HAVE_LIB_XML)
+//        if (global_state->mode & GLUSTER_MODE_XML) {
+//                ret = cli_xml_output_dict ("volHeal", dict, rsp.op_ret,
+//                                           rsp.op_errno, rsp.op_errstr);
+//                if (ret)
+//                        gf_log ("cli", GF_LOG_ERROR,
+//                                "Error outputting to xml");
+//                goto out;
+//        }
+//#endif
+
+        ret = dict_get_str (input_dict, "volname", &volname);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get volname");
+                goto out;
         }
 
-        if (local)
-                dict = local->dict;
+        gf_log ("cli", GF_LOG_INFO, "Received resp to heal volume");
+
+        switch (heal_op) {
+                case    GF_AFR_OP_HEAL_INDEX:
+                        heal_op_str = "to perform index self heal";
+                        break;
+                case    GF_AFR_OP_HEAL_FULL:
+                        heal_op_str = "to perform full self heal";
+                        break;
+                case    GF_AFR_OP_INDEX_SUMMARY:
+                        heal_op_str = "list of entries to be healed";
+                        break;
+                case    GF_AFR_OP_HEALED_FILES:
+                        heal_op_str = "list of healed entries";
+                        break;
+                case    GF_AFR_OP_HEAL_FAILED_FILES:
+                        heal_op_str = "list of heal failed entries";
+                        break;
+                case    GF_AFR_OP_SPLIT_BRAIN_FILES:
+                        heal_op_str = "list of split brain entries";
+                        break;
+                case    GF_AFR_OP_STATISTICS:
+                        heal_op_str =  "crawl statistics";
+                        break;
+                case    GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                        heal_op_str = "count of entries to be healed";
+                        break;
+                case    GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                        heal_op_str = "count of entries to be healed per replica";
+                        break;
+                case    GF_AFR_OP_INVALID:
+                        heal_op_str = "invalid heal op";
+                        break;
+        }
+
+        if ((heal_op == GF_AFR_OP_HEAL_FULL) ||
+            (heal_op == GF_AFR_OP_HEAL_INDEX)) {
+                operation = "Launching heal operation";
+                substr = "\nUse heal info commands to check status";
+        } else {
+                operation = "Gathering";
+                substr = "";
+        }
+
+        if (rsp.op_ret) {
+                if (strcmp (rsp.op_errstr, "")) {
+                        cli_err ("%s", rsp.op_errstr);
+                } else {
+                        cli_err ("%s %s on volume %s has been unsuccessful",
+                                 operation, heal_op_str, volname);
+                }
+
+                ret = rsp.op_ret;
+                goto out;
+        } else {
+                cli_out ("%s %s on volume %s has been successful %s", operation,
+                         heal_op_str, volname, substr);
+        }
+
+        ret = rsp.op_ret;
+        if ((heal_op == GF_AFR_OP_HEAL_FULL) ||
+            (heal_op == GF_AFR_OP_HEAL_INDEX))
+                goto out;
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.dict.dict_val,
+                                rsp.dict.dict_len,
+                                &dict);
 
-        ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                gf_log ("", GF_LOG_ERROR,
+                                "Unable to allocate memory");
                 goto out;
+        } else {
+                dict->extra_stdfree = rsp.dict.dict_val;
         }
+        ret = dict_get_int32 (dict, "count", &brick_count);
+        if (ret)
+                goto out;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to heal volume");
+        if (!brick_count) {
+                cli_err ("All bricks of volume %s are down.", volname);
+                goto out;
+        }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-        else
-                cli_out ("Starting heal on volume %s has been %s", volname,
-                        (rsp.op_ret) ? "unsuccessful": "successful");
+        switch (heal_op) {
+                case GF_AFR_OP_STATISTICS:
+                        for (i = 0; i < brick_count; i++)
+                                cmd_heal_volume_statistics_out (dict, i);
+                        break;
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                        for (i = 0; i < brick_count; i++)
+                                cmd_heal_volume_statistics_heal_count_out (dict,
+                                                                           i);
+                        break;
+                case GF_AFR_OP_INDEX_SUMMARY:
+                case GF_AFR_OP_HEALED_FILES:
+                case GF_AFR_OP_HEAL_FAILED_FILES:
+                case GF_AFR_OP_SPLIT_BRAIN_FILES:
+                        for (i = 0; i < brick_count; i++)
+                                cmd_heal_volume_brick_out (dict, i);
+                        break;
+                default:
+                        break;
+        }
 
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (local)
-                cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.op_errstr);
         if (dict)
                 dict_unref (dict);
         return ret;
 }
 
 int32_t
-gf_cli3_1_heal_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_heal_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req = {{0,}};
         int                     ret = 0;
-        cli_local_t             *local = NULL;
         dict_t                  *dict = NULL;
 
         if (!frame || !this ||  !data) {
@@ -5130,58 +7260,56 @@ gf_cli3_1_heal_volume (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        local = cli_local_get ();
-
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
 
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_HEAL_VOLUME, NULL,
-                              this, gf_cli3_1_heal_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_heal_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_HEAL_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_statedump_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_statedump_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 int count, void *myframe)
 {
         gf_cli_rsp                      rsp = {0,};
         int                             ret = -1;
+        char                            msg[1024] = {0,};
 
         if (-1 == req->rpc_status)
                 goto out;
         ret = xdr_to_generic (*iov, &rsp,
                               (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "XDR decoding failed");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
-        gf_log ("cli", GF_LOG_DEBUG, "Recieved response to statedump");
+        gf_log ("cli", GF_LOG_DEBUG, "Received response to statedump");
         if (rsp.op_ret)
-                cli_out ("%s", rsp.op_errstr);
+                snprintf (msg, sizeof(msg), "%s", rsp.op_errstr);
         else
-                cli_out ("Volume statedump sucessful");
+                snprintf (msg, sizeof (msg), "Volume statedump successful");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_str ("volStatedump", msg, rsp.op_ret,
+                                          rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
 
+        if (rsp.op_ret)
+                cli_err ("volume statedump: failed: %s", msg);
+        else
+                cli_out ("volume statedump: success");
         ret = rsp.op_ret;
 
 out:
@@ -5190,7 +7318,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_statedump_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_statedump_volume (call_frame_t *frame, xlator_t *this,
                             void *data)
 {
         gf_cli_req                      req = {{0,}};
@@ -5202,65 +7330,1338 @@ gf_cli3_1_statedump_volume (call_frame_t *frame, xlator_t *this,
 
         options = data;
 
-        ret = dict_allocate_and_serialize (options,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
+        ret = cli_to_glusterd (&req, frame, gf_cli_statedump_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, options,
+                               GLUSTER_CLI_STATEDUMP_VOLUME, this, cli_rpc_prog,
+                               NULL);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int32_t
+gf_cli_list_volume_cbk (struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+        int             ret = -1;
+        gf_cli_rsp      rsp = {0,};
+        dict_t          *dict = NULL;
+        int             vol_count = 0;;
+        char            *volname = NULL;
+        char            key[1024] = {0,};
+        int             i = 0;
+
+        if (-1 == req->rpc_status)
+                goto out;
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_list (dict, rsp.op_ret, rsp.op_errno,
+                                               rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
                 goto out;
         }
 
+        if (rsp.op_ret)
+                cli_err ("%s", rsp.op_errstr);
+        else {
+                ret = dict_get_int32 (dict, "count", &vol_count);
+                if (ret)
+                        goto out;
+
+                if (vol_count == 0) {
+                        cli_err ("No volumes present in cluster");
+                        goto out;
+                }
+                for (i = 0; i < vol_count; i++) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "volume%d", i);
+                        ret = dict_get_str (dict, key, &volname);
+                        if (ret)
+                                goto out;
+                        cli_out ("%s", volname);
+                }
+        }
+
+        ret = rsp.op_ret;
+
+out:
+        cli_cmd_broadcast_response (ret);
+        return ret;
+}
+
+int32_t
+gf_cli_list_volume (call_frame_t *frame, xlator_t *this, void *data)
+{
+        int             ret = -1;
+        gf_cli_req      req = {{0,}};
+
+        if (!frame || !this)
+                goto out;
+
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_STATEDUMP_VOLUME, NULL,
-                              this, gf_cli3_1_statedump_volume_cbk,
+                              GLUSTER_CLI_LIST_VOLUME, NULL,
+                              this, gf_cli_list_volume_cbk,
                               (xdrproc_t)xdr_gf_cli_req);
 
 out:
-        if (options)
-                dict_destroy (options);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+gf_cli_clearlocks_volume_cbk (struct rpc_req *req, struct iovec *iov,
+                                  int count, void *myframe)
+{
+        gf_cli_rsp                      rsp = {0,};
+        int                             ret = -1;
+        char                            *lk_summary = NULL;
+        char                            *volname = NULL;
+        dict_t                          *dict = NULL;
+
+        if (-1 == req->rpc_status)
+                goto out;
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+        gf_log ("cli", GF_LOG_DEBUG, "Received response to clear-locks");
+
+        if (rsp.op_ret) {
+                cli_err ("Volume clear-locks unsuccessful");
+                cli_err ("%s", rsp.op_errstr);
+
+        } else {
+                if (!rsp.dict.dict_len) {
+                        cli_err ("Possibly no locks cleared");
+                        ret = 0;
+                        goto out;
+                }
+
+                dict = dict_new ();
+
+                if (!dict) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &dict);
+
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Unable to serialize response dictionary");
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Unable to get volname "
+                                "from dictionary");
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, "lk-summary", &lk_summary);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Unable to get lock "
+                                "summary from dictionary");
+                        goto out;
+                }
+                cli_out ("Volume clear-locks successful");
+                cli_out ("%s", lk_summary);
+
+        }
+
+        ret = rsp.op_ret;
+
+out:
+        if (dict)
+                dict_unref (dict);
+        cli_cmd_broadcast_response (ret);
+        return ret;
+}
+
+int32_t
+gf_cli_clearlocks_volume (call_frame_t *frame, xlator_t *this,
+                             void *data)
+{
+        gf_cli_req                      req = {{0,}};
+        dict_t                          *options = NULL;
+        int                             ret = -1;
+
+        if (!frame || !this || !data)
+                goto out;
+
+        options = data;
+
+        ret = cli_to_glusterd (&req, frame, gf_cli_clearlocks_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, options,
+                               GLUSTER_CLI_CLRLOCKS_VOLUME, this, cli_rpc_prog,
+                               NULL);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int32_t
+cli_snapshot_remove_reply (gf_cli_rsp *rsp, dict_t *dict, call_frame_t *frame)
+{
+        int32_t       ret        = -1;
+        char         *snap_name  = NULL;
+
+        GF_ASSERT (rsp);
+        GF_ASSERT (dict);
+        GF_ASSERT (frame);
+
+        if (rsp->op_ret) {
+                cli_err("snapshot delete: failed: %s",
+                        rsp->op_errstr ? rsp->op_errstr :
+                        "Please check log file for details");
+                ret = rsp->op_ret;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snap_name);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapname");
+                goto out;
+        }
+
+        cli_out ("snapshot delete: %s: snap removed successfully",
+                 snap_name);
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+cli_snapshot_config_display (dict_t *dict, gf_cli_rsp *rsp)
+{
+        char                buf[PATH_MAX]        = "";
+        char               *volname              = NULL;
+        int                 ret                  = -1;
+        int                 config_command       = 0;
+        uint64_t            value                = 0;
+        uint64_t            hard_limit           = 0;
+        uint64_t            soft_limit           = 0;
+        uint64_t            i                    = 0;
+        uint64_t            voldisplaycount      = 0;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp);
+
+        if (rsp->op_ret) {
+                cli_err ("Snapshot Config : failed: %s",
+                         rsp->op_errstr ? rsp->op_errstr :
+                         "Please check log file for details");
+                ret = rsp->op_ret;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "config-command", &config_command);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch config type");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        /* Ignore the error, as volname is optional */
+
+        if (!volname) {
+                volname = "System";
+        }
+
+        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
+        /* Ignore the error, as the key specified is optional */
+        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
+
+        if (!hard_limit && !soft_limit
+                        && config_command != GF_SNAP_CONFIG_DISPLAY) {
+                ret = -1;
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Could not fetch config-key");
+                goto out;
+        }
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+                if (hard_limit && soft_limit) {
+                        cli_out ("snapshot config: snap-max-hard-limit "
+                                "& snap-max-soft-limit for system set "
+                                 "successfully");
+                } else if (hard_limit){
+                        cli_out ("snapshot config: %s "
+                                 "for snap-max-hard-limit set successfully",
+                                 volname);
+                } else if (soft_limit) {
+                        cli_out ("snapshot config: %s "
+                                 "for snap-max-soft-limit set successfully",
+                                 volname);
+                }
+                break;
+
+        case GF_SNAP_CONFIG_DISPLAY :
+                cli_out ("\nSnapshot System Configuration:");
+                ret = dict_get_uint64 (dict, "snap-max-hard-limit",
+                                       &value);
+                if (ret) {
+                         gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                "snap_max_hard_limit for %s", volname);
+                         ret = -1;
+                         goto out;
+                }
+                cli_out ("snap-max-hard-limit : %"PRIu64, value);
+
+                ret = dict_get_uint64 (dict, "snap-max-soft-limit",
+                                       &soft_limit);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                               "snap-max-soft-limit for %s", volname);
+                        ret = -1;
+                        goto out;
+                }
+                cli_out ("snap-max-soft-limit : %"PRIu64"%%\n",
+                         soft_limit);
+
+                cli_out ("Snapshot Volume Configuration:");
+
+                ret = dict_get_uint64 (dict, "voldisplaycount",
+                                       &voldisplaycount);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                               "Could not fetch voldisplaycount");
+                        ret = -1;
+                        goto out;
+                }
+
+                for (i = 0; i < voldisplaycount; i++) {
+                        snprintf (buf, sizeof(buf), "volume%ld-volname", i);
+                        ret = dict_get_str (dict, buf, &volname);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                       " %s", buf);
+                                ret = -1;
+                                goto out;
+                        }
+                        cli_out ("\nVolume : %s", volname);
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-hard-limit", i);
+                        ret = dict_get_uint64 (dict, buf, &value);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                       " %s", buf);
+                                ret = -1;
+                                goto out;
+                        }
+                        cli_out ("snap-max-hard-limit : %"PRIu64, value);
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-active-hard-limit", i);
+                        ret = dict_get_uint64 (dict, buf, &value);
+                        if (ret) {
+                                 gf_log ("cli", GF_LOG_ERROR, "Could not fetch"
+                                        " effective snap_max_hard_limit for "
+                                        "%s", volname);
+                                ret = -1;
+                                goto out;
+                        }
+                        cli_out ("Effective snap-max-hard-limit : %"PRIu64,
+                                 value);
+
+                        snprintf (buf, sizeof(buf),
+                                      "volume%ld-snap-max-soft-limit", i);
+                        ret = dict_get_uint64 (dict, buf, &value);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                       " %s", buf);
+                                ret = -1;
+                                goto out;
+                        }
+                        cli_out ("Effective snap-max-soft-limit : %"PRIu64" "
+                                 "(%"PRIu64"%%)", value, soft_limit);
+                }
+                break;
+        default :
+                break;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/* This function is used to print the volume related information
+ * of a snap.
+ *
+ * arg - 0, dict       : Response Dictionary.
+ * arg - 1, prefix str : snaplist.snap{0..}.vol{0..}.*
+ */
+int
+cli_get_each_volinfo_in_snap (dict_t *dict, char *keyprefix,
+                              gf_boolean_t snap_driven) {
+        char             key[PATH_MAX]  =  "";
+        char            *get_buffer     =  NULL;
+        int              value          =  0;
+        int              ret            = -1;
+        char             indent[5]      = "\t";
+        char            *volname        =  NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (keyprefix);
+
+        if (snap_driven) {
+                ret = snprintf (key, sizeof (key), "%s.volname", keyprefix);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &get_buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
+                        goto out;
+                }
+                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
+                        "Snap Volume Name", ":", get_buffer);
+
+                ret = snprintf (key, sizeof (key),
+                                "%s.origin-volname", keyprefix);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_WARNING, "Failed to get %s", key);
+                        cli_out ("%-12s", "Origin:");
+                }
+                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
+                        "Origin Volume name", ":", volname);
+
+
+                ret = snprintf (key, sizeof (key), "%s.snapcount",
+                                keyprefix);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, key, &value);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
+                        goto out;
+                }
+                cli_out ("%s%s %s      %s %d", indent, "Snaps taken for",
+                        volname, ":", value);
+
+                ret = snprintf (key, sizeof (key), "%s.snaps-available",
+                                keyprefix);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, key, &value);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
+                        goto out;
+                }
+                cli_out ("%s%s %s  %s %d", indent, "Snaps available for",
+                         volname, ":", value);
+        }
+
+
+        ret = snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
+                goto out;
+        }
+        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Status",
+                 ":", get_buffer);
+out :
+        return ret;
+}
+
+/* This function is used to print snap related information
+ * arg - 0, dict       : Response dictionary.
+ * arg - 1, prefix_str : snaplist.snap{0..}.*
+ */
+int
+cli_get_volinfo_in_snap (dict_t *dict, char *keyprefix) {
+
+        char            key[PATH_MAX]   = "";
+        int             i               = 0;
+        int             volcount        = 0;
+        int             ret             = -1;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (keyprefix);
+
+        ret = snprintf (key, sizeof (key), "%s.vol-count", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, key, &volcount);
+        for (i = 1 ; i <= volcount ; i++) {
+                ret = snprintf (key, sizeof (key),
+                                "%s.vol%d", keyprefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+                ret = cli_get_each_volinfo_in_snap (dict, key, _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not list "
+                                "details of volume in a snap");
+                        goto out;
+                }
+                cli_out (" ");
+        }
+
+out :
+        return ret;
+}
+
+int
+cli_get_each_snap_info (dict_t *dict, char *prefix_str,
+                        gf_boolean_t snap_driven) {
+        char            key_buffer[PATH_MAX] = "";
+        char           *get_buffer           = NULL;
+        int             ret                  = -1;
+        char            indent[5]            = "";
+
+        GF_ASSERT (dict);
+        GF_ASSERT (prefix_str);
+
+        if (!snap_driven)
+                strcat (indent, "\t");
+
+        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snapname",
+                        prefix_str);
+        if (ret < 0 ) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ",
+                        key_buffer);
+                goto out;
+        }
+        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Snapshot",
+                ":", get_buffer);
+
+        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-id",
+                        prefix_str);
+        if (ret < 0 ) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ",
+                        key_buffer);
+                goto out;
+        }
+        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Snap UUID",
+                ":", get_buffer);
+
+        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-desc",
+                        prefix_str);
+        if (ret < 0 ) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        if (!ret) {
+                /* Ignore error for description */
+                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
+                         "Description", ":", get_buffer);
+        }
+
+        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-time",
+                        prefix_str);
+        if (ret < 0 ) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ",
+                        prefix_str);
+                goto out;
+        }
+        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Created",
+                ":", get_buffer);
+
+        if (snap_driven) {
+                cli_out ("%-12s", "Snap Volumes:\n");
+                ret = cli_get_volinfo_in_snap (dict, prefix_str);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Unable to list details "
+                                "of the snaps");
+                        goto out;
+                }
+        }
+out :
+        return ret;
+}
+
+/* This is a generic function to print snap related information.
+ * arg - 0, dict : Response Dictionary
+ */
+int
+cli_call_snapshot_info (dict_t *dict, gf_boolean_t bool_snap_driven) {
+        int             snap_count       =  0;
+        char            key[PATH_MAX]    =  "";
+        int             ret              =  -1;
+        int             i                =   0;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_int32 (dict, "snap-count", &snap_count);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to get snap-count");
+                goto out;
+        }
+
+        if (snap_count == 0) {
+                cli_out ("No snapshots present");
+        }
+
+        for (i = 1 ; i <= snap_count ; i++) {
+                ret = snprintf (key, sizeof (key), "snap%d", i);
+                if (ret < 0) {
+                        goto out;
+                }
+                ret = cli_get_each_snap_info (dict, key, bool_snap_driven);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Unable to print snap details");
+                        goto out;
+                }
+        }
+out :
+        return ret;
+}
+
+int
+cli_get_snaps_in_volume (dict_t *dict) {
+        int      ret             =  -1;
+        int      i               =  0;
+        int      count           =  0;
+        int      avail           =  0;
+        char     key[PATH_MAX]   =  "";
+        char    *get_buffer      =  NULL;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_str (dict, "origin-volname", &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch origin-volname");
+                goto out;
+        }
+        cli_out (INDENT_MAIN_HEAD "%s", "Volume Name", ":", get_buffer);
+
+        ret = dict_get_int32 (dict, "snap-count", &avail);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snap-count");
+                goto out;
+        }
+        cli_out (INDENT_MAIN_HEAD "%d", "Snaps Taken", ":", avail);
+
+        ret = dict_get_int32 (dict, "snaps-available", &count);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snaps-available");
+                goto out;
+        }
+        cli_out (INDENT_MAIN_HEAD "%d", "Snaps Available", ":", count);
+
+        for (i = 1 ; i <= avail ; i++) {
+                snprintf (key, sizeof (key), "snap%d", i);
+                ret = cli_get_each_snap_info (dict, key, _gf_false);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Unable to print snap details");
+                        goto out;
+                }
+
+                ret = snprintf (key, sizeof (key), "snap%d.vol1", i);
+                if (ret < 0) {
+                        goto out;
+                }
+                ret = cli_get_each_volinfo_in_snap (dict, key, _gf_false);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not get volume "
+                                "related information");
+                        goto out;
+                }
+
+                cli_out (" ");
+        }
+out :
+        return ret;
+}
+
+int
+cli_snapshot_list (dict_t *dict) {
+        int     snapcount       =       0;
+        char    key[PATH_MAX]   =       "";
+        int     ret             =       -1;
+        int     i               =       0;
+        char    *get_buffer     =       NULL;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_int32 (dict, "snap-count", &snapcount);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snap count");
+                goto out;
+        }
+
+        if (snapcount == 0) {
+                cli_out ("No snapshots present");
+        }
+
+        for (i = 1 ; i <= snapcount ; i++) {
+                ret = snprintf (key, sizeof (key), "snapname%d",i);
+                if (ret < 0) {
+                        goto out;
+                }
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+                ret = dict_get_str (dict, key, &get_buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not get %s ", key);
+                        goto out;
+                } else {
+                        cli_out ("%s", get_buffer);
+                }
+        }
+out :
         return ret;
 }
 
+int
+cli_get_snap_volume_status (dict_t *dict, char *key_prefix)
+{
+        int             ret             =       -1;
+        char            key[PATH_MAX]   =       "";
+        char            *buffer         =       NULL;
+        int             brickcount      =       0;
+        int             i               =       0;
+        int             pid             =       0;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key_prefix);
+
+        ret = snprintf (key, sizeof (key), "%s.brickcount", key_prefix);
+        if (ret < 0) {
+                goto out;
+        }
+        ret = dict_get_int32 (dict, key, &brickcount);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
+                goto out;
+        }
+
+        for ( i = 0 ; i < brickcount ; i++ ) {
+                ret = snprintf (key, sizeof (key), "%s.brick%d.path",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO,
+                                "Unable to get Brick Path");
+                        continue;
+                }
+                cli_out ("\n\t%-17s %s   %s", "Brick Path", ":", buffer);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.vgname",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO,
+                                "Unable to get Volume Group");
+                        cli_out ("\t%-17s %s   %s", "Volume Group", ":", "N/A");
+                } else
+                        cli_out ("\t%-17s %s   %s", "Volume Group", ":", buffer);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.status",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO,
+                                "Unable to get Brick Running");
+                        cli_out ("\t%-17s %s   %s", "Brick Running", ":", "N/A");
+                } else
+                        cli_out ("\t%-17s %s   %s", "Brick Running", ":", buffer);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.pid",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, key, &pid);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO,
+                                "Unable to get pid");
+                        cli_out ("\t%-17s %s   %s", "Brick PID", ":", "N/A");
+                } else
+                        cli_out ("\t%-17s %s   %d", "Brick PID", ":", pid);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.data",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO,
+                                "Unable to get Data Percent");
+                        cli_out ("\t%-17s %s   %s", "Data Percentage", ":", "N/A");
+                } else
+                        cli_out ("\t%-17s %s   %s", "Data Percentage", ":", buffer);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.lvsize",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+                ret = dict_get_str (dict, key, &buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO, "Unable to get LV Size");
+                        cli_out ("\t%-17s %s   %s", "LV Size", ":", "N/A");
+                } else
+                        cli_out ("\t%-17s %s   %s", "LV Size", ":", buffer);
+
+        }
+out :
+        return ret;
+}
+
+
+
+int
+cli_get_single_snap_status (dict_t *dict, char *keyprefix)
+{
+        int             ret             =       -1;
+        char            key[PATH_MAX]   =       "";
+        int             i               =       0;
+        int             volcount        =       0;
+        char            *get_buffer     =       NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (keyprefix);
+
+        ret = snprintf (key, sizeof (key), "%s.snapname", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to get snapname");
+                goto out;
+        }
+        cli_out ("\nSnap Name : %s", get_buffer);
+
+        ret = snprintf (key, sizeof (key), "%s.uuid", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to get snap UUID");
+                goto out;
+        }
+        cli_out ("Snap UUID : %s", get_buffer);
+
+        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, key, &volcount);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to get volume count");
+                goto out;
+        }
+
+        for (i = 0 ; i < volcount ; i++) {
+                ret = snprintf (key, sizeof (key), "%s.vol%d", keyprefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = cli_get_snap_volume_status (dict, key);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Could not get snap volume status");
+                        goto out;
+                }
+        }
+out :
+        return ret;
+}
+
+int
+cli_snap_status_all (dict_t *dict) {
+        int             ret             =       -1;
+        char            key[PATH_MAX]   =       "";
+        int             snapcount       =       0;
+        int             i               =       0;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_int32 (dict, "status.snapcount", &snapcount);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not get snapcount");
+                goto out;
+        }
+
+        if (snapcount == 0) {
+                cli_out ("No snapshots present");
+        }
+
+        for (i = 0 ; i < snapcount; i++) {
+                ret = snprintf (key, sizeof (key), "status.snap%d",i);
+                if (ret < 0) {
+                        goto out;
+                }
+                ret = cli_get_single_snap_status (dict, key);
+        }
+out:
+        return ret;
+}
+
+
+int
+cli_snapshot_status_display (dict_t *dict, gf_cli_rsp *rsp)
+{
+        char            key[PATH_MAX]   =       "";
+        int             ret             =       -1;
+        int             status_cmd      =       -1;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp);
+
+        if (rsp->op_ret) {
+                cli_err ("Snapshot Status : failed: %s",
+                        rsp->op_errstr ? rsp->op_errstr :
+                        "Please check log file for details");
+                ret = rsp->op_ret;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "cmd", &status_cmd);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch status type");
+                goto out;
+        }
+        switch (status_cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL :
+                {
+                        ret = cli_snap_status_all (dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                        "status of all snap");
+                                goto out;
+                        }
+                break;
+                }
+
+                case GF_SNAP_STATUS_TYPE_SNAP :
+                {
+                        ret = snprintf (key, sizeof (key), "status.snap0");
+                        if (ret < 0) {
+                                goto out;
+                        }
+                        ret = cli_get_single_snap_status (dict, key);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                        "status of snap");
+                                goto out;
+                        }
+                break;
+                }
+
+                case GF_SNAP_STATUS_TYPE_VOL :
+                {
+                        ret = cli_snap_status_all (dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                        "status of snap in a volume");
+                                goto out;
+                        }
+                        break;
+                }
+                default :
+                        break;
+        }
+out :
+        return ret;
+}
+
+int
+gf_cli_snapshot_cbk (struct rpc_req *req, struct iovec *iov,
+                     int count, void *myframe)
+{
+        int                   ret                      = -1;
+        gf_cli_rsp            rsp                      = {0, };
+        dict_t               *dict                     = NULL;
+        char                 *snap_name                = NULL;
+        int32_t               type                     =  0;
+        call_frame_t         *frame                    = NULL;
+        gf_boolean_t         snap_driven               = _gf_false;
+
+        if (req->rpc_status == -1) {
+                ret = -1;
+                goto out;
+        }
+
+        frame = myframe;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        dict = dict_new ();
+
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get type");
+                goto out;
+        }
+
+        switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                if (rsp.op_ret) {
+                        cli_err("snapshot create: failed: %s",
+                                 rsp.op_errstr ? rsp.op_errstr :
+                                 "Please check log file for details");
+                                 ret = rsp.op_ret;
+                                 goto out;
+                }
+
+                ret = dict_get_str (dict, "snapname", &snap_name);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Failed to get snap name");
+                        goto out;
+                }
+                cli_out ("snapshot create: %s: snap created successfully",
+                         snap_name);
+                break;
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                /* TODO: Check if rsp.op_ret needs to be checked here. Or is
+                 * it ok to check this in the start of the function where we
+                 * get rsp.*/
+                if (rsp.op_ret) {
+                        cli_err("snapshot restore: failed: %s",
+                                 rsp.op_errstr ? rsp.op_errstr :
+                                 "Please check log file for details");
+                                 ret = rsp.op_ret;
+                                 goto out;
+                }
+
+                ret = dict_get_str (dict, "snapname", &snap_name);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Failed to get snap name");
+                        goto out;
+                }
+
+                cli_out ("Snapshot restore: %s: Snap restored "
+                                 "successfully", snap_name);
+
+                ret = 0;
+                break;
+
+        case GF_SNAP_OPTION_TYPE_INFO:
+                if (rsp.op_ret) {
+                        cli_err ("Snapshot info : failed: %s",
+                                  rsp.op_errstr ? rsp.op_errstr :
+                                  "Please check log file for details");
+                        ret = rsp.op_ret;
+                        goto out;
+                }
+
+                snap_driven = dict_get_str_boolean (dict, "snap-driven",
+                                                    _gf_false);
+                if (snap_driven == _gf_true) {
+                        ret = cli_call_snapshot_info  (dict, snap_driven);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Snapshot info failed");
+                                goto out;
+                        }
+                } else if (snap_driven == _gf_false) {
+                        ret = cli_get_snaps_in_volume (dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Snapshot info failed");
+                                goto out;
+                        }
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                ret = cli_snapshot_config_display (dict, &rsp);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
+                                "snapshot config output.");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_LIST:
+                if (rsp.op_ret) {
+                        cli_err ("Snapshot list : failed: %s",
+                                 rsp.op_errstr ? rsp.op_errstr :
+                                "Please check log file for details");
+                        ret = rsp.op_ret;
+                        goto out;
+                }
+
+                ret = cli_snapshot_list (dict);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
+                                "snapshot list");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = cli_snapshot_remove_reply (&rsp, dict, frame);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Failed to delete snap");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = cli_snapshot_status_display (dict, &rsp);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
+                                "snapshot status output.");
+                        goto out;
+                }
+                break;
+
+        default:
+                cli_err ("Unknown command executed");
+                ret = -1;
+                goto out;
+        }
+out:
+        if (dict)
+                dict_unref (dict);
+        cli_cmd_broadcast_response (ret);
+
+        free (rsp.dict.dict_val);
+        free (rsp.op_errstr);
+
+        return ret;
+}
+
+int32_t
+gf_cli_snapshot (call_frame_t *frame, xlator_t *this,
+                 void *data)
+{
+        gf_cli_req        req     = {{0,}};
+        dict_t           *options = NULL;
+        int               ret     = -1;
+
+        if (!frame || !this || !data)
+                goto out;
+
+        options = data;
+
+        ret = cli_to_glusterd (&req, frame, gf_cli_snapshot_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, options,
+                               GLUSTER_CLI_SNAP, this, cli_rpc_prog,
+                               NULL);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int
+cli_to_glusterd (gf_cli_req *req, call_frame_t *frame,
+                 fop_cbk_fn_t cbkfn, xdrproc_t xdrproc, dict_t *dict,
+                 int procnum, xlator_t *this, rpc_clnt_prog_t *prog,
+                 struct iobref *iobref)
+{
+        int                ret = 0;
+        size_t             len = 0;
+        char               *cmd = NULL;
+        int                i = 0;
+        const char         **words = NULL;
+        cli_local_t        *local = NULL;
+
+        if (!this || !frame || !dict) {
+                ret = -1;
+                goto out;
+        }
+
+        if (!frame->local) {
+                ret = -1;
+                goto out;
+        }
+
+        local = frame->local;
+
+        if (!local->words) {
+                ret = -1;
+                goto out;
+        }
+
+        words = local->words;
+
+        while (words[i])
+                len += strlen (words[i++]) + 1;
+
+        cmd = GF_CALLOC (1, len, gf_common_mt_char);
+
+        if (!cmd) {
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; words[i]; i++) {
+                strncat (cmd, words[i], strlen (words[i]));
+                if (words[i+1] != NULL)
+                        strncat (cmd, " ", strlen (" "));
+        }
+
+        cmd [len - 1] = '\0';
+
+        ret = dict_set_dynstr (dict, "cmd-str", cmd);
+        if (ret)
+                goto out;
+
+        ret = dict_allocate_and_serialize (dict, &(req->dict).dict_val,
+                                           &(req->dict).dict_len);
+
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = cli_cmd_submit (req, frame, prog, procnum, iobref, this,
+                              cbkfn, (xdrproc_t) xdrproc);
+
+out:
+        return ret;
+
+}
+
 struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = {
         [GLUSTER_CLI_NULL]             = {"NULL", NULL },
-        [GLUSTER_CLI_PROBE]            = {"PROBE_QUERY", gf_cli3_1_probe},
-        [GLUSTER_CLI_DEPROBE]          = {"DEPROBE_QUERY", gf_cli3_1_deprobe},
-        [GLUSTER_CLI_LIST_FRIENDS]     = {"LIST_FRIENDS", gf_cli3_1_list_friends},
-        [GLUSTER_CLI_CREATE_VOLUME]    = {"CREATE_VOLUME", gf_cli3_1_create_volume},
-        [GLUSTER_CLI_DELETE_VOLUME]    = {"DELETE_VOLUME", gf_cli3_1_delete_volume},
-        [GLUSTER_CLI_START_VOLUME]     = {"START_VOLUME", gf_cli3_1_start_volume},
-        [GLUSTER_CLI_STOP_VOLUME]      = {"STOP_VOLUME", gf_cli3_1_stop_volume},
-        [GLUSTER_CLI_RENAME_VOLUME]    = {"RENAME_VOLUME", gf_cli3_1_rename_volume},
-        [GLUSTER_CLI_DEFRAG_VOLUME]    = {"DEFRAG_VOLUME", gf_cli3_1_defrag_volume},
-        [GLUSTER_CLI_GET_VOLUME]       = {"GET_VOLUME", gf_cli3_1_get_volume},
-        [GLUSTER_CLI_GET_NEXT_VOLUME]  = {"GET_NEXT_VOLUME", gf_cli3_1_get_next_volume},
-        [GLUSTER_CLI_SET_VOLUME]       = {"SET_VOLUME", gf_cli3_1_set_volume},
-        [GLUSTER_CLI_ADD_BRICK]        = {"ADD_BRICK", gf_cli3_1_add_brick},
-        [GLUSTER_CLI_REMOVE_BRICK]     = {"REMOVE_BRICK", gf_cli3_1_remove_brick},
-        [GLUSTER_CLI_REPLACE_BRICK]    = {"REPLACE_BRICK", gf_cli3_1_replace_brick},
-        [GLUSTER_CLI_LOG_ROTATE]       = {"LOG ROTATE", gf_cli3_1_log_rotate},
-        [GLUSTER_CLI_GETSPEC]          = {"GETSPEC", gf_cli3_1_getspec},
-        [GLUSTER_CLI_PMAP_PORTBYBRICK] = {"PMAP PORTBYBRICK", gf_cli3_1_pmap_b2p},
-        [GLUSTER_CLI_SYNC_VOLUME]      = {"SYNC_VOLUME", gf_cli3_1_sync_volume},
-        [GLUSTER_CLI_RESET_VOLUME]     = {"RESET_VOLUME", gf_cli3_1_reset_volume},
-        [GLUSTER_CLI_FSM_LOG]          = {"FSM_LOG", gf_cli3_1_fsm_log},
-        [GLUSTER_CLI_GSYNC_SET]        = {"GSYNC_SET", gf_cli3_1_gsync_set},
-        [GLUSTER_CLI_PROFILE_VOLUME]   = {"PROFILE_VOLUME", gf_cli3_1_profile_volume},
-        [GLUSTER_CLI_QUOTA]            = {"QUOTA", gf_cli3_1_quota},
-        [GLUSTER_CLI_TOP_VOLUME]       = {"TOP_VOLUME", gf_cli3_1_top_volume},
-        [GLUSTER_CLI_GETWD]            = {"GETWD", gf_cli3_1_getwd},
-        [GLUSTER_CLI_STATUS_VOLUME]    = {"STATUS_VOLUME", gf_cli3_1_status_volume},
+        [GLUSTER_CLI_PROBE]            = {"PROBE_QUERY", gf_cli_probe},
+        [GLUSTER_CLI_DEPROBE]          = {"DEPROBE_QUERY", gf_cli_deprobe},
+        [GLUSTER_CLI_LIST_FRIENDS]     = {"LIST_FRIENDS", gf_cli_list_friends},
+        [GLUSTER_CLI_UUID_RESET]       = {"UUID_RESET", gf_cli3_1_uuid_reset},
+        [GLUSTER_CLI_UUID_GET]       = {"UUID_GET", gf_cli3_1_uuid_get},
+        [GLUSTER_CLI_CREATE_VOLUME]    = {"CREATE_VOLUME", gf_cli_create_volume},
+        [GLUSTER_CLI_DELETE_VOLUME]    = {"DELETE_VOLUME", gf_cli_delete_volume},
+        [GLUSTER_CLI_START_VOLUME]     = {"START_VOLUME", gf_cli_start_volume},
+        [GLUSTER_CLI_STOP_VOLUME]      = {"STOP_VOLUME", gf_cli_stop_volume},
+        [GLUSTER_CLI_RENAME_VOLUME]    = {"RENAME_VOLUME", gf_cli_rename_volume},
+        [GLUSTER_CLI_DEFRAG_VOLUME]    = {"DEFRAG_VOLUME", gf_cli_defrag_volume},
+        [GLUSTER_CLI_GET_VOLUME]       = {"GET_VOLUME", gf_cli_get_volume},
+        [GLUSTER_CLI_GET_NEXT_VOLUME]  = {"GET_NEXT_VOLUME", gf_cli_get_next_volume},
+        [GLUSTER_CLI_SET_VOLUME]       = {"SET_VOLUME", gf_cli_set_volume},
+        [GLUSTER_CLI_ADD_BRICK]        = {"ADD_BRICK", gf_cli_add_brick},
+        [GLUSTER_CLI_REMOVE_BRICK]     = {"REMOVE_BRICK", gf_cli_remove_brick},
+        [GLUSTER_CLI_REPLACE_BRICK]    = {"REPLACE_BRICK", gf_cli_replace_brick},
+        [GLUSTER_CLI_LOG_ROTATE]       = {"LOG ROTATE", gf_cli_log_rotate},
+        [GLUSTER_CLI_GETSPEC]          = {"GETSPEC", gf_cli_getspec},
+        [GLUSTER_CLI_PMAP_PORTBYBRICK] = {"PMAP PORTBYBRICK", gf_cli_pmap_b2p},
+        [GLUSTER_CLI_SYNC_VOLUME]      = {"SYNC_VOLUME", gf_cli_sync_volume},
+        [GLUSTER_CLI_RESET_VOLUME]     = {"RESET_VOLUME", gf_cli_reset_volume},
+        [GLUSTER_CLI_FSM_LOG]          = {"FSM_LOG", gf_cli_fsm_log},
+        [GLUSTER_CLI_GSYNC_SET]        = {"GSYNC_SET", gf_cli_gsync_set},
+        [GLUSTER_CLI_PROFILE_VOLUME]   = {"PROFILE_VOLUME", gf_cli_profile_volume},
+        [GLUSTER_CLI_QUOTA]            = {"QUOTA", gf_cli_quota},
+        [GLUSTER_CLI_TOP_VOLUME]       = {"TOP_VOLUME", gf_cli_top_volume},
+        [GLUSTER_CLI_GETWD]            = {"GETWD", gf_cli_getwd},
+        [GLUSTER_CLI_STATUS_VOLUME]    = {"STATUS_VOLUME", gf_cli_status_volume},
         [GLUSTER_CLI_STATUS_ALL]       = {"STATUS_ALL", gf_cli_status_volume_all},
-        [GLUSTER_CLI_MOUNT]            = {"MOUNT", gf_cli3_1_mount},
-        [GLUSTER_CLI_UMOUNT]           = {"UMOUNT", gf_cli3_1_umount},
-        [GLUSTER_CLI_HEAL_VOLUME]      = {"HEAL_VOLUME", gf_cli3_1_heal_volume},
-        [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME", gf_cli3_1_statedump_volume},
+        [GLUSTER_CLI_MOUNT]            = {"MOUNT", gf_cli_mount},
+        [GLUSTER_CLI_UMOUNT]           = {"UMOUNT", gf_cli_umount},
+        [GLUSTER_CLI_HEAL_VOLUME]      = {"HEAL_VOLUME", gf_cli_heal_volume},
+        [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME", gf_cli_statedump_volume},
+        [GLUSTER_CLI_LIST_VOLUME]      = {"LIST_VOLUME", gf_cli_list_volume},
+        [GLUSTER_CLI_CLRLOCKS_VOLUME]  = {"CLEARLOCKS_VOLUME", gf_cli_clearlocks_volume},
+        [GLUSTER_CLI_COPY_FILE]        = {"COPY_FILE", gf_cli_copy_file},
+        [GLUSTER_CLI_SYS_EXEC]         = {"SYS_EXEC", gf_cli_sys_exec},
+        [GLUSTER_CLI_SNAP]             = {"SNAP", gf_cli_snapshot},
 };
 
 struct rpc_clnt_program cli_prog = {
diff --git a/cli/src/cli-xml-output.c b/cli/src/cli-xml-output.c
new file mode 100644
index 000000000..d8884d44b
--- /dev/null
+++ b/cli/src/cli-xml-output.c
@@ -0,0 +1,3772 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <stdlib.h>
+#include "cli.h"
+#include "cli1-xdr.h"
+#include "run.h"
+#include "compat.h"
+#include "syscall.h"
+
+
+enum gf_task_types {
+    GF_TASK_TYPE_REBALANCE,
+    GF_TASK_TYPE_REMOVE_BRICK
+};
+
+/*
+ * IMPORTANT NOTE:
+ * All exported functions in this file which use libxml need use a
+ * #if (HAVE_LIB_XML), #else, #endif
+ * For eg,
+ *      int exported_func () {
+ *              #if (HAVE_LIB_XML)
+ *                      <Stuff using libxml>
+ *              #else
+ *                      return 0;
+ *              #endif
+ *      }
+ *
+ *  All other functions, which are called internally within this file need to be
+ *  within #if (HAVE_LIB_XML), #endif statements
+ *  For eg,
+ *      #if (HAVE_LIB_XML)
+ *      int internal_func ()
+ *      {
+ *      }
+ *      #endif
+ *
+ *  Following the above formate ensures that all xml related code is compliled
+ *  only when libxml2 is present, and also keeps the rest of the codebase free
+ *  of #if (HAVE_LIB_XML)
+ */
+
+
+#if (HAVE_LIB_XML)
+
+#include <libxml/encoding.h>
+#include <libxml/xmlwriter.h>
+
+#define XML_RET_CHECK_AND_GOTO(ret, label)      do {            \
+                if (ret < 0) {                                  \
+                        ret = -1;                               \
+                        goto label;                             \
+                }                                               \
+                else                                            \
+                        ret = 0;                                \
+        }while (0)                                              \
+
+int
+cli_begin_xml_output (xmlTextWriterPtr *writer, xmlDocPtr *doc)
+{
+        int             ret = -1;
+
+        *writer = xmlNewTextWriterDoc (doc, 0);
+        if (writer == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = xmlTextWriterStartDocument (*writer, "1.0", "UTF-8", "yes");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* <cliOutput> */
+        ret = xmlTextWriterStartElement (*writer, (xmlChar *)"cliOutput");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_end_xml_output (xmlTextWriterPtr writer, xmlDocPtr doc)
+{
+        int             ret = -1;
+
+        /* </cliOutput> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterEndDocument (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+
+        /* Dump xml document to stdout and pretty format it */
+        xmlSaveFormatFileEnc ("-", doc, "UTF-8", 1);
+
+        xmlFreeTextWriter (writer);
+        xmlFreeDoc (doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_common (xmlTextWriterPtr writer, int op_ret, int op_errno,
+                       char *op_errstr)
+{
+        int             ret = -1;
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"opRet",
+                                               "%d", op_ret);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"opErrno",
+                                               "%d", op_errno);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"opErrstr",
+                                                "%s", op_errstr);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_str (char *op, char *str, int op_ret, int op_errno,
+                    char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        if (op) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"cliOp",
+                                                       "%s", op);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        if (str) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"output",
+                                                       "%s", str);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_data_pair (dict_t *this, char *key, data_t *value,
+                          void *data)
+{
+        int                     ret = -1;
+        xmlTextWriterPtr        *writer = NULL;
+
+        writer = (xmlTextWriterPtr *)data;
+
+        ret = xmlTextWriterWriteFormatElement (*writer, (xmlChar *)key,
+                                               "%s", value->data);
+
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_dict ( char *op, dict_t *dict, int op_ret, int op_errno,
+                      char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <"op"> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)op);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        if (dict)
+                dict_foreach (dict, cli_xml_output_data_pair, &writer);
+
+        /* </"op"> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (writer, doc);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_status_common (xmlTextWriterPtr writer, dict_t *dict,
+                                  int   brick_index, int *online,
+                                  gf_boolean_t *node_present)
+{
+        int             ret = -1;
+        char            *hostname = NULL;
+        char            *path = NULL;
+        int             port = 0;
+        int             status = 0;
+        int             pid = 0;
+        char            key[1024] = {0,};
+
+        snprintf (key, sizeof (key), "brick%d.hostname", brick_index);
+        ret = dict_get_str (dict, key, &hostname);
+        if (ret) {
+                *node_present = _gf_false;
+                goto out;
+        }
+        *node_present = _gf_true;
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hostname",
+                                               "%s", hostname);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.path", brick_index);
+        ret = dict_get_str (dict, key, &path);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"path",
+                                               "%s", path);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.status", brick_index);
+        ret = dict_get_int32 (dict, key, &status);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"status",
+                                               "%d", status);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+        *online = status;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.port", brick_index);
+        ret = dict_get_int32 (dict, key, &port);
+        if (ret)
+                goto out;
+
+        /* If the process is either offline or doesn't provide a port (shd)
+         * port = "N/A"
+         * else print the port number of the process.
+         */
+
+        if (*online == 1 && port != 0)
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"port",
+                                                       "%d", port);
+        else
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"port",
+                                                       "%s", "N/A");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.pid", brick_index);
+        ret = dict_get_int32 (dict, key, &pid);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"pid",
+                                               "%d", pid);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_detail (xmlTextWriterPtr writer, dict_t *dict,
+                                  int brick_index)
+{
+        int             ret = -1;
+        uint64_t        size_total = 0;
+        uint64_t        size_free = 0;
+        char            *device = NULL;
+        uint64_t        block_size = 0;
+        char            *mnt_options = NULL;
+        char            *fs_name = NULL;
+        char            *inode_size = NULL;
+        uint64_t        inodes_total = 0;
+        uint64_t        inodes_free = 0;
+        char            key[1024] = {0,};
+
+        snprintf (key, sizeof (key), "brick%d.total", brick_index);
+        ret = dict_get_uint64 (dict, key, &size_total);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"sizeTotal",
+                                               "%"PRIu64, size_total);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.free", brick_index);
+        ret = dict_get_uint64 (dict, key, &size_free);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"sizeFree",
+                                               "%"PRIu64, size_free);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.device", brick_index);
+        ret = dict_get_str (dict, key, &device);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"device",
+                                               "%s", device);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.block_size", brick_index);
+        ret = dict_get_uint64 (dict, key, &block_size);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"blockSize",
+                                               "%"PRIu64, block_size);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.mnt_options", brick_index);
+        ret = dict_get_str (dict, key, &mnt_options);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"mntOptions",
+                                               "%s", mnt_options);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.fs_name", brick_index);
+        ret = dict_get_str (dict, key, &fs_name);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"fsName",
+                                               "%s", fs_name);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* inode details are only available for ext 2/3/4 & xfs */
+        if (!IS_EXT_FS(fs_name) || strcmp (fs_name, "xfs")) {
+                        ret = 0;
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.inode_size", brick_index);
+        ret = dict_get_str (dict, key, &inode_size);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"inodeSize",
+                                               "%s", fs_name);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.total_inodes", brick_index);
+        ret = dict_get_uint64 (dict, key, &inodes_total);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"inodesTotal",
+                                               "%"PRIu64, inodes_total);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.free_inodes", brick_index);
+        ret = dict_get_uint64 (dict, key, &inodes_free);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"inodesFree",
+                                               "%"PRIu64, inodes_free);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_mempool (xmlTextWriterPtr writer, dict_t *dict,
+                                   char *prefix)
+{
+        int             ret = -1;
+        int             mempool_count = 0;
+        char            *name = NULL;
+        int             hotcount = 0;
+        int             coldcount = 0;
+        uint64_t        paddedsizeof = 0;
+        uint64_t        alloccount = 0;
+        int             maxalloc = 0;
+        char            key[1024] = {0,};
+        int             i = 0;
+
+        /* <mempool> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"mempool");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%s.mempool-count", prefix);
+        ret = dict_get_int32 (dict, key, &mempool_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
+                                               "%d", mempool_count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        for (i = 0; i < mempool_count; i++) {
+                /* <pool> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"pool");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.name", prefix, i);
+                ret = dict_get_str (dict, key, &name);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"name",
+                                                       "%s", name);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.hotcount", prefix, i);
+                ret = dict_get_int32 (dict, key, &hotcount);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"hotCount",
+                                                       "%d", hotcount);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.coldcount", prefix, i);
+                ret = dict_get_int32 (dict, key, &coldcount);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"coldCount",
+                                                       "%d", coldcount);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.paddedsizeof",
+                          prefix, i);
+                ret = dict_get_uint64 (dict, key, &paddedsizeof);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"padddedSizeOf", "%"PRIu64,
+                         paddedsizeof);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.alloccount", prefix, i);
+                ret = dict_get_uint64 (dict, key, &alloccount);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"allocCount",
+                                                       "%"PRIu64, alloccount);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.max_alloc", prefix, i);
+                ret = dict_get_int32 (dict, key, &maxalloc);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"maxAlloc",
+                                                       "%d", maxalloc);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.pool-misses", prefix, i);
+                ret = dict_get_uint64 (dict, key, &alloccount);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"poolMisses",
+                                                       "%"PRIu64, alloccount);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.pool%d.max-stdalloc", prefix, i);
+                ret = dict_get_int32 (dict, key, &maxalloc);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"maxStdAlloc",
+                                                       "%d", maxalloc);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+
+                /* </pool> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </mempool> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_mem (xmlTextWriterPtr writer, dict_t *dict,
+                               int brick_index)
+{
+        int             ret = -1;
+        int             arena = 0;
+        int             ordblks = 0;
+        int             smblks = 0;
+        int             hblks = 0;
+        int             hblkhd = 0;
+        int             usmblks = 0;
+        int             fsmblks = 0;
+        int             uordblks = 0;
+        int             fordblks = 0;
+        int             keepcost = 0;
+        char            key[1024] = {0,};
+
+        /* <memStatus> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"memStatus");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* <mallinfo> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"mallinfo");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "brick%d.mallinfo.arena", brick_index);
+        ret = dict_get_int32 (dict, key, &arena);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"arena",
+                                               "%d", arena);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.mallinfo.ordblks", brick_index);
+        ret = dict_get_int32 (dict, key, &ordblks);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"ordblks",
+                                               "%d", ordblks);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.mallinfo.smblks", brick_index);
+        ret = dict_get_int32 (dict, key, &smblks);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"smblks",
+                                               "%d", smblks);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.mallinfo.hblks", brick_index);
+        ret = dict_get_int32 (dict, key, &hblks);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hblks",
+                                               "%d", hblks);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.mallinfo.hblkhd", brick_index);
+        ret = dict_get_int32 (dict, key, &hblkhd);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hblkhd",
+                                               "%d", hblkhd);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.mallinfo.usmblks", brick_index);
+        ret = dict_get_int32 (dict, key, &usmblks);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"usmblks",
+                                               "%d", usmblks);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.mallinfo.fsmblks", brick_index);
+        ret = dict_get_int32 (dict, key, &fsmblks);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"fsmblks",
+                                               "%d", fsmblks);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.mallinfo.uordblks", brick_index);
+        ret = dict_get_int32 (dict, key, &uordblks);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"uordblks",
+                                               "%d", uordblks);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.mallinfo.fordblks", brick_index);
+        ret = dict_get_int32 (dict, key, &fordblks);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"fordblks",
+                                               "%d", fordblks);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.mallinfo.keepcost", brick_index);
+        ret = dict_get_int32 (dict, key, &keepcost);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"keepcost",
+                                               "%d", keepcost);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </mallinfo> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d", brick_index);
+        ret = cli_xml_output_vol_status_mempool (writer, dict, key);
+        if (ret)
+                goto out;
+
+        /* </memStatus> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_clients (xmlTextWriterPtr writer, dict_t *dict,
+                                   int brick_index)
+{
+        int             ret = -1;
+        int             client_count = 0;
+        char            *hostname = NULL;
+        uint64_t        bytes_read = 0;
+        uint64_t        bytes_write = 0;
+        char            key[1024] = {0,};
+        int             i = 0;
+
+        /* <clientsStatus> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"clientsStatus");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "brick%d.clientcount", brick_index);
+        ret = dict_get_int32 (dict, key, &client_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer,
+                                               (xmlChar *)"clientCount",
+                                               "%d", client_count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        for (i = 0; i < client_count; i++) {
+                /* <client> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"client");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "brick%d.client%d.hostname",
+                          brick_index, i);
+                ret = dict_get_str (dict, key, &hostname);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"hostname",
+                                                       "%s", hostname);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "brick%d.client%d.bytesread",
+                          brick_index, i);
+                ret = dict_get_uint64 (dict, key, &bytes_read);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"bytesRead",
+                                                       "%"PRIu64, bytes_read);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "brick%d.client%d.byteswrite",
+                          brick_index, i);
+                ret = dict_get_uint64 (dict, key, &bytes_write);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"bytesWrite",
+                                                       "%"PRIu64, bytes_write);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </client> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </clientsStatus> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_inode_entry (xmlTextWriterPtr writer, dict_t *dict,
+                                       char *prefix)
+{
+        int             ret = -1;
+        char            *gfid = NULL;
+        uint64_t        nlookup = 0;
+        uint32_t        ref = 0;
+        int             ia_type = 0;
+        char            key[1024] = {0,};
+
+        /* <inode> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"inode");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%s.gfid", prefix);
+        ret = dict_get_str (dict, key, &gfid);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"gfid",
+                                               "%s", gfid);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key,0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.nlookup", prefix);
+        ret = dict_get_uint64 (dict, key, &nlookup);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"nLookup",
+                                               "%"PRIu64, nlookup);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key,0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.ref", prefix);
+        ret = dict_get_uint32 (dict, key, &ref);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"ref",
+                                               "%"PRIu32, ref);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key,0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.ia_type", prefix);
+        ret = dict_get_int32 (dict, key, &ia_type);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"iaType",
+                                               "%d", ia_type);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </inode> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_itable (xmlTextWriterPtr writer, dict_t *dict,
+                                  char *prefix)
+{
+        int             ret = -1;
+        uint32_t        active_size = 0;
+        uint32_t        lru_size = 0;
+        uint32_t        purge_size = 0;
+        char            key[1024] = {0,};
+        int             i = 0;
+
+        snprintf (key, sizeof (key), "%s.active_size", prefix);
+        ret = dict_get_uint32 (dict, key, &active_size);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"activeSize",
+                                               "%"PRIu32, active_size);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+        if (active_size != 0) {
+                /* <active> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"active");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                for (i = 0; i < active_size; i++) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%s.active%d", prefix, i);
+                        ret = cli_xml_output_vol_status_inode_entry
+                                (writer, dict, key);
+                        if (ret)
+                                goto out;
+                }
+                /* </active> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.lru_size", prefix);
+        ret = dict_get_uint32 (dict, key, &lru_size);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"lruSize",
+                                               "%"PRIu32, lru_size);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+        if (lru_size != 0) {
+                /* <lru> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"lru");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                for (i = 0; i < lru_size; i++) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%s.lru%d", prefix, i);
+                        ret = cli_xml_output_vol_status_inode_entry
+                                (writer, dict, key);
+                        if (ret)
+                                goto out;
+                }
+                /* </lru> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.purge_size", prefix);
+        ret = dict_get_uint32 (dict, key, &purge_size);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"purgeSize",
+                                               "%"PRIu32, purge_size);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+        if (purge_size != 0) {
+                /* <purge> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"purge");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                for (i = 0; i < purge_size; i++) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%s.purge%d", prefix, i);
+                        ret = cli_xml_output_vol_status_inode_entry
+                                (writer, dict, key);
+                        if (ret)
+                                goto out;
+                }
+                /* </purge> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_inode (xmlTextWriterPtr writer, dict_t *dict,
+                                 int brick_index)
+{
+        int             ret = -1;
+        int             conn_count = 0;
+        char            key[1024] = {0,};
+        int             i = 0;
+
+        /* <inodeStatus> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"inodeStatus");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "brick%d.conncount", brick_index);
+        ret = dict_get_int32 (dict, key, &conn_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"connections",
+                                               "%d", conn_count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        for (i = 0; i < conn_count; i++) {
+                /* <connection> */
+                ret = xmlTextWriterStartElement (writer,
+                                                 (xmlChar *)"connection");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "brick%d.conn%d.itable",
+                          brick_index, i);
+                ret = cli_xml_output_vol_status_itable (writer, dict, key);
+                if (ret)
+                        goto out;
+
+                /* </connection> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </inodeStatus> */
+        ret= xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_fdtable (xmlTextWriterPtr writer, dict_t *dict,
+                                   char *prefix)
+{
+        int             ret = -1;
+        int             refcount = 0;
+        uint32_t        maxfds = 0;
+        int             firstfree = 0;
+        int             openfds = 0;
+        int             fd_pid = 0;
+        int             fd_refcount = 0;
+        int             fd_flags = 0;
+        char            key[1024] = {0,};
+        int             i = 0;
+
+        /* <fdTable> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"fdTable");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%s.refcount", prefix);
+        ret = dict_get_int32 (dict, key, &refcount);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"refCount",
+                                               "%d", refcount);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.maxfds", prefix);
+        ret = dict_get_uint32 (dict, key, &maxfds);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"maxFds",
+                                               "%"PRIu32, maxfds);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.firstfree", prefix);
+        ret = dict_get_int32 (dict, key, &firstfree);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"firstFree",
+                                               "%d", firstfree);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.openfds", prefix);
+        ret = dict_get_int32 (dict, key, &openfds);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"openFds",
+                                               "%d", openfds);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        for (i = 0; i < maxfds; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.fdentry%d.pid", prefix, i);
+                ret = dict_get_int32 (dict, key, &fd_pid);
+                if (ret)
+                        continue;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.fdentry%d.refcount",
+                          prefix, i);
+                ret = dict_get_int32 (dict, key, &fd_refcount);
+                if (ret)
+                        continue;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.fdentry%d.flags", prefix, i);
+                ret = dict_get_int32 (dict, key, &fd_flags);
+                if (ret)
+                        continue;
+
+                /* <fd> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"fd");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"entry",
+                                                       "%d", i+1);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"pid",
+                                                       "%d", fd_pid);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"refCount",
+                                                       "%d", fd_refcount);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"flags",
+                                                       "%d", fd_flags);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </fd> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </fdTable> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_fd (xmlTextWriterPtr writer, dict_t *dict,
+                              int brick_index)
+{
+        int             ret = -1;
+        int             conn_count = 0;
+        char            key[1024] = {0,};
+        int             i = 0;
+
+        /* <fdStatus> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"fdStatus");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "brick%d.conncount", brick_index);
+        ret = dict_get_int32 (dict, key, &conn_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"connections",
+                                               "%d", conn_count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        for (i = 0; i < conn_count; i++) {
+                /* <connection> */
+                ret = xmlTextWriterStartElement (writer,
+                                                 (xmlChar *)"connection");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "brick%d.conn%d.fdtable",
+                          brick_index, i);
+                ret = cli_xml_output_vol_status_fdtable (writer, dict, key);
+                if (ret)
+                        goto out;
+
+                /* </connection> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </fdStatus> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_callframe (xmlTextWriterPtr writer, dict_t *dict,
+                                     char *prefix)
+{
+        int             ret = -1;
+        int             ref_count = 0;
+        char            *translator = NULL;
+        int             complete = 0;
+        char            *parent = NULL;
+        char            *wind_from = NULL;
+        char            *wind_to = NULL;
+        char            *unwind_from = NULL;
+        char            *unwind_to = NULL;
+        char            key[1024] = {0,};
+
+        /* <callFrame> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"callFrame");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%s.refcount", prefix);
+        ret = dict_get_int32 (dict, key, &ref_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"refCount",
+                                               "%d", ref_count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.translator", prefix);
+        ret = dict_get_str (dict, key, &translator);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"translator",
+                                               "%s", translator);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.complete", prefix);
+        ret = dict_get_int32 (dict, key, &complete);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"complete",
+                                               "%d", complete);
+        XML_RET_CHECK_AND_GOTO (ret ,out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.parent", prefix);
+        ret = dict_get_str (dict, key, &parent);
+        if (!ret) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"parent",
+                                                       "%s", parent);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.windfrom", prefix);
+        ret = dict_get_str (dict, key, &wind_from);
+        if (!ret) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"windFrom",
+                                                       "%s", wind_from);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.windto", prefix);
+        ret = dict_get_str (dict, key, &wind_to);
+        if (!ret) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"windTo",
+                                                       "%s", wind_to);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.unwindfrom", prefix);
+        ret = dict_get_str (dict, key, &unwind_from);
+        if (!ret) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"unwindFrom",
+                                                       "%s", unwind_from);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.unwindto", prefix);
+        ret = dict_get_str (dict, key, &unwind_to);
+        if (!ret) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"unwindTo",
+                                                       "%s", unwind_to);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </callFrame> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_callstack (xmlTextWriterPtr writer, dict_t *dict,
+                                     char *prefix)
+{
+        int             ret = -1;
+        int             uid = 0;
+        int             gid = 0;
+        int             pid = 0;
+        uint64_t        unique = 0;
+        int             frame_count = 0;
+        char            key[1024] = {0,};
+        int             i = 0;
+
+        /* <callStack> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"callStack");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%s.uid", prefix);
+        ret = dict_get_int32 (dict, key, &uid);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"uid",
+                                               "%d", uid);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.gid", prefix);
+        ret = dict_get_int32 (dict, key, &gid);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"gid",
+                                               "%d", gid);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.pid", prefix);
+        ret = dict_get_int32 (dict, key, &pid);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"pid",
+                                               "%d", pid);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.unique", prefix);
+        ret = dict_get_uint64 (dict, key, &unique);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"unique",
+                                               "%"PRIu64, unique);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.count", prefix);
+        ret = dict_get_int32 (dict, key, &frame_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"frameCount",
+                                               "%d", frame_count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        for (i = 0; i < frame_count; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.frame%d", prefix, i);
+                ret = cli_xml_output_vol_status_callframe (writer, dict,
+                                                           key);
+                if (ret)
+                        goto out;
+        }
+
+        /* </callStack> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_callpool (xmlTextWriterPtr writer, dict_t *dict,
+                                    int brick_index)
+{
+        int             ret = -1;
+        int             call_count = 0;
+        char            key[1024] = {0,};
+        int             i = 0;
+
+        /* <callpoolStatus> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"callpoolStatus");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "brick%d.callpool.count", brick_index);
+        ret = dict_get_int32 (dict, key, &call_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
+                                               "%d", call_count);
+
+        for (i = 0; i < call_count; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "brick%d.callpool.stack%d",
+                          brick_index, i);
+                ret = cli_xml_output_vol_status_callstack (writer, dict,
+                                                           key);
+                if (ret)
+                        goto out;
+        }
+
+        /* </callpoolStatus> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_status_begin (cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+
+        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
+                                     op_errstr);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* <volStatus> */
+        ret = xmlTextWriterStartElement (local->writer,
+                                         (xmlChar *) "volStatus");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* <volumes> */
+        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volumes");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_status_end (cli_local_t *local)
+{
+#if (HAVE_LIB_XML)
+        int     ret = -1;
+
+        /* </volumes> */
+        ret = xmlTextWriterEndElement (local->writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </volStatus> */
+        ret = xmlTextWriterEndElement (local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = cli_end_xml_output (local->writer, local->doc);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_remove_brick_task_params (xmlTextWriterPtr writer, dict_t *dict,
+                                         char *prefix)
+{
+        int             ret = -1;
+        char            key[1024] = {0,};
+        int             count = 0;
+        int             i = 0;
+        char            *brick = NULL;
+
+        /* <params> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"params");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%s.count", prefix);
+        ret = dict_get_int32 (dict, key, &count);
+        if (ret)
+                goto out;
+
+        for (i = 1; i <= count; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.brick%d", prefix, i);
+                ret = dict_get_str (dict, key, &brick);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"brick",
+                                                       "%s", brick);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+                brick = NULL;
+        }
+
+        /* </param> */
+        ret = xmlTextWriterEndElement (writer);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_replace_brick_task_params (xmlTextWriterPtr writer, dict_t *dict,
+                                          char *prefix)
+{
+
+        int             ret = -1;
+        char            key[1024] = {0,};
+        char            *brick = NULL;
+
+        /* <params> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"params");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%s.src-brick", prefix);
+        ret = dict_get_str (dict, key, &brick);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"srcBrick",
+                                               "%s", brick);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.dst-brick", prefix);
+        ret = dict_get_str (dict, key, &brick);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"dstBrick",
+                                               "%s", brick);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+
+        /* </param> */
+        ret = xmlTextWriterEndElement (writer);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_tasks (cli_local_t *local, dict_t *dict) {
+        int                     ret = -1;
+        char                    *task_type = NULL;
+        char                    *task_id_str = NULL;
+        int                     status = 0;
+        int                     tasks = 0;
+        char                    key[1024] = {0,};
+        int                     i = 0;
+
+        /* <tasks> */
+        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"tasks");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_int32 (dict, "tasks", &tasks);
+        if (ret)
+                goto out;
+
+        for (i = 0; i < tasks; i++) {
+                /* <task> */
+                ret = xmlTextWriterStartElement (local->writer,
+                                                 (xmlChar *)"task");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.type", i);
+                ret = dict_get_str (dict, key, &task_type);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"type",
+                                                       "%s", task_type);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.id", i);
+                ret = dict_get_str (dict, key, &task_id_str);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"id",
+                                                       "%s", task_id_str);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.status", i);
+                ret = dict_get_int32 (dict, key, &status);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"status",
+                                                       "%d", status);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                if (!strcmp (task_type, "Replace brick")) {
+                    if (status) {
+                        status = GF_DEFRAG_STATUS_COMPLETE;
+                    } else {
+                        status = GF_DEFRAG_STATUS_STARTED;
+                    }
+                }
+
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"statusStr",
+                                                       "%s",
+                                             cli_vol_task_status_str[status]);
+
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d", i);
+                if (!strcmp (task_type, "Replace brick")) {
+                        ret = cli_xml_output_replace_brick_task_params
+                                (local->writer, dict, key);
+                        if (ret)
+                                goto out;
+                } else if (!strcmp (task_type, "Remove brick")) {
+                        ret = cli_xml_output_remove_brick_task_params
+                                (local->writer, dict, key);
+                        if (ret)
+                                goto out;
+                }
+
+
+                /* </task> */
+                ret = xmlTextWriterEndElement (local->writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </tasks> */
+        ret = xmlTextWriterEndElement (local->writer);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_tasks_detail (cli_local_t *local, dict_t *dict)
+{
+        int    ret     = -1;
+        char  *volname = NULL;
+
+        /*<volume>*/
+        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (local->writer,
+                                               (xmlChar *)"volName", "%s",
+                                               volname);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_xml_output_vol_status_tasks (local, dict);
+        if (ret)
+                goto out;
+
+        /* </volume> */
+        ret = xmlTextWriterEndElement (local->writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_status (cli_local_t *local, dict_t *dict)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        char                    *volname = NULL;
+        int                     brick_count = 0;
+        int                     brick_index_max = -1;
+        int                     other_count = 0;
+        int                     index_max = 0;
+        uint32_t                cmd = GF_CLI_STATUS_NONE;
+        int                     online = 0;
+        gf_boolean_t            node_present = _gf_true;
+        int                     i;
+
+
+        /* <volume> */
+        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (local->writer,
+                                               (xmlChar *)"volName", "%s",
+                                               volname);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_int32 (dict, "count", &brick_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (local->writer,
+                                               (xmlChar *)"nodeCount", "%d",
+                                               brick_count);
+        if (ret)
+                goto out;
+
+        ret = dict_get_uint32 (dict, "cmd", &cmd);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
+        if (ret)
+                goto out;
+
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
+                /* <node> */
+                ret = xmlTextWriterStartElement (local->writer,
+                                                 (xmlChar *)"node");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = cli_xml_output_vol_status_common (local->writer, dict, i,
+                                                        &online, &node_present);
+                if (ret) {
+                        if (node_present)
+                                goto out;
+                        else
+                                continue;
+                }
+
+                switch (cmd & GF_CLI_STATUS_MASK) {
+                case GF_CLI_STATUS_DETAIL:
+                        ret = cli_xml_output_vol_status_detail (local->writer,
+                                                                dict, i);
+                        if (ret)
+                                goto out;
+                        break;
+
+                case GF_CLI_STATUS_MEM:
+                        if (online) {
+                                ret = cli_xml_output_vol_status_mem
+                                        (local->writer, dict, i);
+                                if (ret)
+                                        goto out;
+                        }
+                        break;
+
+                case GF_CLI_STATUS_CLIENTS:
+                        if (online) {
+                                ret = cli_xml_output_vol_status_clients
+                                        (local->writer, dict, i);
+                                if (ret)
+                                        goto out;
+                        }
+                        break;
+
+                case GF_CLI_STATUS_INODE:
+                        if (online) {
+                                ret = cli_xml_output_vol_status_inode
+                                        (local->writer, dict, i);
+                                if (ret)
+                                        goto out;
+                        }
+                        break;
+
+                case GF_CLI_STATUS_FD:
+                        if (online) {
+                                ret = cli_xml_output_vol_status_fd
+                                        (local->writer, dict, i);
+                                if (ret)
+                                        goto out;
+                        }
+                        break;
+
+                case GF_CLI_STATUS_CALLPOOL:
+                        if (online) {
+                                ret = cli_xml_output_vol_status_callpool
+                                        (local->writer, dict, i);
+                                if (ret)
+                                        goto out;
+                        }
+                        break;
+                default:
+                        break;
+
+                }
+                /* </node> */
+                ret = xmlTextWriterEndElement (local->writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* Tasks are only present when a normal volume status call is done on a
+         * single volume or on all volumes
+         */
+        if (((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) &&
+            (cmd & (GF_CLI_STATUS_VOL|GF_CLI_STATUS_ALL))) {
+                ret = cli_xml_output_vol_status_tasks (local, dict);
+                if (ret)
+                        goto out;
+        }
+
+        /* </volume> */
+        ret = xmlTextWriterEndElement (local->writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_top_rw_perf (xmlTextWriterPtr writer, dict_t *dict,
+                                int brick_index, int member_index)
+{
+        int        ret = -1;
+        char      *filename = NULL;
+        uint64_t   throughput = 0;
+        long int   time_sec = 0;
+        long int   time_usec = 0;
+        char       timestr[256] = {0,};
+        char       key[1024] = {0,};
+
+        /* <file> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"file");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%d-filename-%d", brick_index,
+                  member_index);
+        ret = dict_get_str (dict, key, &filename);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"filename",
+                                               "%s", filename);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-value-%d", brick_index, member_index);
+        ret = dict_get_uint64 (dict, key, &throughput);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
+                                               "%"PRIu64, throughput);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-time-sec-%d", brick_index,
+                  member_index);
+        ret = dict_get_int32 (dict, key, (int32_t *)&time_sec);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-time-usec-%d", brick_index,
+                  member_index);
+        ret = dict_get_int32 (dict, key, (int32_t *)&time_usec);
+        if (ret)
+                goto out;
+
+        gf_time_fmt (timestr, sizeof timestr, time_sec, gf_timefmt_FT);
+        snprintf (timestr + strlen (timestr),
+                  sizeof timestr - strlen (timestr),
+                  ".%"GF_PRI_SUSECONDS, time_usec);
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"time",
+                                               "%s", timestr);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </file> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_top_other (xmlTextWriterPtr writer, dict_t *dict,
+                                int brick_index, int member_index)
+{
+        int             ret = -1;
+        char            *filename = NULL;
+        uint64_t        count = 0;
+        char            key[1024] = {0,};
+
+        /* <file> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"file");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%d-filename-%d", brick_index,
+                  member_index);
+        ret = dict_get_str (dict, key, &filename);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"filename",
+                                               "%s", filename);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-value-%d", brick_index, member_index);
+        ret = dict_get_uint64 (dict, key, &count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
+                                               "%"PRIu64, count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </file> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        int                     brick_count = 0;
+        int                     top_op = GF_CLI_TOP_NONE;
+        char                    *brick_name = NULL;
+        int                     members = 0;
+        uint64_t                current_open = 0;
+        uint64_t                max_open = 0;
+        char                    *max_open_time = NULL;
+        double                  throughput = 0.0;
+        double                  time_taken = 0.0;
+        char                    key[1024] = {0,};
+        int                     i = 0;
+        int                     j = 0;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volTop> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volTop");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_int32 (dict, "count", &brick_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"brickCount",
+                                               "%d", brick_count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_int32 (dict, "1-top-op", &top_op);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"topOp",
+                                               "%d", top_op);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        while (i < brick_count) {
+                i++;
+
+                /* <brick> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"brick");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%d-brick", i);
+                ret = dict_get_str (dict, key, &brick_name);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"name",
+                                                       "%s", brick_name);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key , sizeof (key), "%d-members", i);
+                ret = dict_get_int32 (dict, key, &members);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"members",
+                                                       "%d", members);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                switch (top_op) {
+                case GF_CLI_TOP_OPEN:
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%d-current-open", i);
+                        ret = dict_get_uint64 (dict, key, &current_open);
+                        if (ret)
+                                goto out;
+                        ret = xmlTextWriterWriteFormatElement
+                                (writer, (xmlChar *)"currentOpen", "%"PRIu64,
+                                 current_open);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%d-max-open", i);
+                        ret = dict_get_uint64 (dict, key, &max_open);
+                        if (ret)
+                                goto out;
+                        ret = xmlTextWriterWriteFormatElement
+                                (writer, (xmlChar *)"maxOpen", "%"PRIu64,
+                                 max_open);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%d-max-openfd-time", i);
+                        ret = dict_get_str (dict, key, &max_open_time);
+                        if (ret)
+                                goto out;
+                        ret = xmlTextWriterWriteFormatElement
+                                (writer, (xmlChar *)"maxOpenTime", "%s",
+                                 max_open_time);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                case GF_CLI_TOP_READ:
+                case GF_CLI_TOP_WRITE:
+                case GF_CLI_TOP_OPENDIR:
+                case GF_CLI_TOP_READDIR:
+
+                        break;
+
+                case GF_CLI_TOP_READ_PERF:
+                case GF_CLI_TOP_WRITE_PERF:
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%d-throughput", i);
+                        ret = dict_get_double (dict, key, &throughput);
+                        if (!ret) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key), "%d-time", i);
+                                ret = dict_get_double (dict, key, &time_taken);
+                        }
+
+                        if (!ret) {
+                                ret = xmlTextWriterWriteFormatElement
+                                        (writer, (xmlChar *)"throughput",
+                                         "%f", throughput);
+                                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                                ret = xmlTextWriterWriteFormatElement
+                                        (writer, (xmlChar *)"timeTaken",
+                                         "%f", time_taken);
+                        }
+
+                        break;
+
+                default:
+                        ret = -1;
+                        goto out;
+                }
+
+                for (j = 1; j <= members; j++) {
+                        if (top_op == GF_CLI_TOP_READ_PERF ||
+                            top_op == GF_CLI_TOP_WRITE_PERF) {
+                                ret = cli_xml_output_vol_top_rw_perf
+                                        (writer, dict, i, j);
+                        } else {
+                                ret = cli_xml_output_vol_top_other
+                                        (writer, dict, i, j);
+                        }
+                        if (ret)
+                                goto out;
+                }
+
+
+                /* </brick> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </volTop> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_profile_stats (xmlTextWriterPtr writer, dict_t *dict,
+                                  int brick_index, int interval)
+{
+        int                     ret = -1;
+        uint64_t                read_count = 0;
+        uint64_t                write_count = 0;
+        uint64_t                hits = 0;
+        double                  avg_latency = 0.0;
+        double                  max_latency = 0.0;
+        double                  min_latency = 0.0;
+        uint64_t                duration = 0;
+        uint64_t                total_read = 0;
+        uint64_t                total_write = 0;
+        char                    key[1024] = {0};
+        int                     i = 0;
+
+        /* <cumulativeStats> || <intervalStats> */
+        if (interval == -1)
+                ret = xmlTextWriterStartElement (writer,
+                                                 (xmlChar *)"cumulativeStats");
+        else
+                ret = xmlTextWriterStartElement (writer,
+                                                 (xmlChar *)"intervalStats");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* <blockStats> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"blockStats");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        for (i = 0; i < 32; i++) {
+                /* <block> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"block");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"size", "%"PRIu32, (1 << i));
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%d-%d-read-%d", brick_index,
+                          interval, (1 << i));
+                ret = dict_get_uint64 (dict, key, &read_count);
+                if (ret)
+                        read_count = 0;
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"reads", "%"PRIu64, read_count);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%d-%d-write-%d", brick_index,
+                          interval, (1 << i));
+                ret = dict_get_uint64 (dict, key, &write_count);
+                if (ret)
+                        write_count = 0;
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"writes", "%"PRIu64, write_count);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </block> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </blockStats> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* <fopStats> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"fopStats");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%d-%d-%d-hits", brick_index,
+                          interval, i);
+                ret = dict_get_uint64 (dict, key, &hits);
+                if (ret)
+                        goto cont;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%d-%d-%d-avglatency", brick_index,
+                          interval, i);
+                ret = dict_get_double (dict, key, &avg_latency);
+                if (ret)
+                        goto cont;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%d-%d-%d-minlatency", brick_index,
+                          interval, i);
+                ret = dict_get_double (dict, key, &min_latency);
+                if (ret)
+                        goto cont;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%d-%d-%d-maxlatency", brick_index,
+                          interval, i);
+                ret = dict_get_double (dict, key, &max_latency);
+                if (ret)
+                        goto cont;
+
+                /* <fop> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"fop");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"name","%s", gf_fop_list[i]);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"hits", "%"PRIu64, hits);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"avgLatency", "%f", avg_latency);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"minLatency", "%f", min_latency);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"maxLatency", "%f", max_latency);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </fop> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+cont:
+                hits = 0;
+                avg_latency = 0.0;
+                min_latency = 0.0;
+                max_latency = 0.0;
+        }
+
+        /* </fopStats> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-%d-duration", brick_index, interval);
+        ret = dict_get_uint64 (dict, key, &duration);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"duration",
+                                               "%"PRIu64, duration);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-%d-total-read", brick_index, interval);
+        ret = dict_get_uint64 (dict, key, &total_read);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"totalRead",
+                                               "%"PRIu64, total_read);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-%d-total-write", brick_index, interval);
+        ret = dict_get_uint64 (dict, key, &total_write);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"totalWrite",
+                                               "%"PRIu64, total_write);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </cumulativeStats> || </intervalStats> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_profile (dict_t *dict, int op_ret, int op_errno,
+                            char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *volname = NULL;
+        int                     op = GF_CLI_STATS_NONE;
+        int                     brick_count = 0;
+        char                    *brick_name = NULL;
+        int                     interval = 0;
+        char                    key[1024] = {0,};
+        int                     i = 0;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volProfile> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volProfile");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"volname",
+                                               "%s", volname);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_int32 (dict, "op", &op);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"profileOp",
+                                               "%d", op);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        if (op != GF_CLI_STATS_INFO)
+                goto cont;
+
+        ret = dict_get_int32 (dict, "count", &brick_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"brickCount",
+                                               "%d", brick_count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        while (i < brick_count) {
+                i++;
+
+                /* <brick> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"brick");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                snprintf (key, sizeof (key), "%d-brick", i);
+                ret = dict_get_str (dict, key, &brick_name);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"brickName", "%s", brick_name);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                snprintf (key, sizeof (key), "%d-cumulative", i);
+                ret = dict_get_int32 (dict, key, &interval);
+                if (ret == 0) {
+                        ret = cli_xml_output_vol_profile_stats
+                                (writer, dict, i, interval);
+                        if (ret)
+                                goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%d-interval", i);
+                ret = dict_get_int32 (dict, key, &interval);
+                if (ret == 0) {
+                        ret = cli_xml_output_vol_profile_stats
+                                (writer, dict, i, interval);
+                        if (ret)
+                                goto out;
+                }
+
+                /* </brick> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+cont:
+        /* </volProfile> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (writer, doc);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_list (dict_t *dict, int op_ret, int op_errno,
+                         char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        int                     count = 0;
+        char                    *volname = NULL;
+        char                    key[1024] = {0,};
+        int                     i = 0;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volList> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volList");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
+                                               "%d", count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        for (i = 0; i < count; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"volume",
+                                                       "%s", volname);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </volList> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (writer, doc);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_info_option (xmlTextWriterPtr writer, char *substr,
+                                char *optstr, char *valstr)
+{
+        int             ret = -1;
+        char            *ptr1 = NULL;
+        char            *ptr2 = NULL;
+
+        ptr1 = substr;
+        ptr2 = optstr;
+
+        while (ptr1) {
+                if (*ptr1 != *ptr2)
+                        break;
+                ptr1++;
+                ptr2++;
+                if (!ptr1)
+                        goto out;
+                if (!ptr2)
+                        goto out;
+        }
+        if (*ptr2 == '\0')
+                goto out;
+
+        /* <option> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"option");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"name",
+                                               "%s", ptr2);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"value",
+                                               "%s", valstr);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </option> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+struct tmp_xml_option_logger {
+        char *key;
+        xmlTextWriterPtr writer;
+};
+
+static int
+_output_vol_info_option (dict_t *d, char *k, data_t *v,
+                         void *data)
+{
+        int                           ret   = 0;
+        char                         *ptr   = NULL;
+        struct tmp_xml_option_logger *tmp   = NULL;
+
+        tmp = data;
+
+        ptr = strstr (k, "option.");
+        if (!ptr)
+                goto out;
+
+        if (!v) {
+                ret = -1;
+                goto out;
+        }
+        ret = cli_xml_output_vol_info_option (tmp->writer, tmp->key, k,
+                                              v->data);
+
+out:
+        return ret;
+}
+
+int
+cli_xml_output_vol_info_options (xmlTextWriterPtr writer, dict_t *dict,
+                                 char *prefix)
+{
+        int             ret = -1;
+        int             opt_count = 0;
+        char            key[1024] = {0,};
+        struct tmp_xml_option_logger tmp = {0,};
+
+        snprintf (key, sizeof (key), "%s.opt_count", prefix);
+        ret = dict_get_int32 (dict, key, &opt_count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"optCount",
+                                               "%d", opt_count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* <options> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"options");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf (key, sizeof (key), "%s.option.", prefix);
+
+        tmp.key = key;
+        tmp.writer = writer;
+        ret = dict_foreach (dict, _output_vol_info_option, &tmp);
+        if (ret)
+                goto out;
+
+        /* </options> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_info (cli_local_t *local, dict_t *dict)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = 0;
+        int                     count = 0;
+        char                    *volname = NULL;
+        char                    *volume_id = NULL;
+        char                    *uuid = NULL;
+        int                     type = 0;
+        int                     status = 0;
+        int                     brick_count = 0;
+        int                     dist_count = 0;
+        int                     stripe_count = 0;
+        int                     replica_count = 0;
+        int                     transport = 0;
+        char                    *brick = NULL;
+        char                    key[1024] = {0,};
+        int                     i = 0;
+        int                     j = 1;
+        char                    *caps = NULL;
+        int                     k __attribute__((unused)) = 0;
+        char                    *snap_volume = NULL;
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                goto out;
+
+        for (i = 0; i < count; i++) {
+                /* <volume> */
+                ret = xmlTextWriterStartElement (local->writer,
+                                                 (xmlChar *)"volume");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.name", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"name",
+                                                       "%s", volname);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.volume_id", i);
+                ret = dict_get_str (dict, key, &volume_id);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"id",
+                                                       "%s", volume_id);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.status", i);
+                ret = dict_get_int32 (dict, key, &status);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"status",
+                                                       "%d", status);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.snap_volume", i);
+                ret = dict_get_str (dict, key, &snap_volume);
+                if (ret)
+                        goto out;
+                if (snap_volume) {
+                        ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                         (xmlChar *)"snapVol",
+                                                          "%s", snap_volume);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+                }
+
+                ret =xmlTextWriterWriteFormatElement
+                        (local->writer, (xmlChar *)"statusStr", "%s",
+                         cli_vol_status_str[status]);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.brick_count", i);
+                ret = dict_get_int32 (dict, key, &brick_count);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"brickCount",
+                                                       "%d", brick_count);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.dist_count", i);
+                ret = dict_get_int32 (dict, key, &dist_count);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"distCount",
+                                                       "%d", dist_count);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.stripe_count", i);
+                ret = dict_get_int32 (dict, key, &stripe_count);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"stripeCount",
+                                                       "%d", stripe_count);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.replica_count", i);
+                ret = dict_get_int32 (dict, key, &replica_count);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"replicaCount",
+                                                       "%d", replica_count);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.type", i);
+                ret = dict_get_int32 (dict, key, &type);
+                if (ret)
+                        goto out;
+                /* For Distributed-(stripe,replicate,stipe-replicate) types */
+                if ((type > 0) && (dist_count < brick_count))
+                        type += 3;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"type",
+                                                       "%d", type);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"typeStr",
+                                                       "%s",
+                                                       cli_vol_type_str[type]);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.transport", i);
+                ret = dict_get_int32 (dict, key, &transport);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"transport",
+                                                       "%d", transport);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+#ifdef HAVE_BD_XLATOR
+                /* <xlators> */
+                ret = xmlTextWriterStartElement (local->writer,
+                                                 (xmlChar *)"xlators");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                for (k = 0; ; k++) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key),"volume%d.xlator%d", i, k);
+                        ret = dict_get_str (dict, key, &caps);
+                        if (ret)
+                                break;
+
+                        /* <xlator> */
+                        ret = xmlTextWriterStartElement (local->writer,
+                                                         (xmlChar *)"xlator");
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        ret = xmlTextWriterWriteFormatElement
+                                (local->writer, (xmlChar *)"name", "%s", caps);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        /* <capabilities> */
+                        ret = xmlTextWriterStartElement (local->writer,
+                                                         (xmlChar *)
+                                                         "capabilities");
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        j = 0;
+                        for (j = 0; ;j++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key),
+                                          "volume%d.xlator%d.caps%d", i, k, j);
+                                ret = dict_get_str (dict, key, &caps);
+                                if (ret)
+                                        break;
+                                ret = xmlTextWriterWriteFormatElement
+                                        (local->writer, (xmlChar *)"capability",
+                                         "%s", caps);
+                                XML_RET_CHECK_AND_GOTO (ret, out);
+                        }
+                        /* </capabilities> */
+                        ret = xmlTextWriterEndElement (local->writer);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+                        /* </xlator> */
+                        ret = xmlTextWriterEndElement (local->writer);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+                }
+                ret = xmlTextWriterFullEndElement (local->writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+                /* </xlators> */
+#else
+                caps = 0; /* Avoid compiler warnings when BD not enabled */
+#endif
+                j = 1;
+
+                /* <bricks> */
+                ret = xmlTextWriterStartElement (local->writer,
+                                                 (xmlChar *)"bricks");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+                while (j <= brick_count) {
+                        ret = xmlTextWriterStartElement
+                                (local->writer, (xmlChar *)"brick");
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "volume%d.brick%d.uuid",
+                                  i, j);
+                        ret = dict_get_str (dict, key, &uuid);
+                        if (ret)
+                                goto out;
+                        ret = xmlTextWriterWriteFormatAttribute
+                                (local->writer, (xmlChar *)"uuid", "%s",
+                                 uuid);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "volume%d.brick%d", i, j);
+                        ret = dict_get_str (dict, key, &brick);
+                        if (ret)
+                                goto out;
+                        ret = xmlTextWriterWriteFormatString
+                                (local->writer, "%s", brick);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        /* </brick> */
+                        ret = xmlTextWriterEndElement (local->writer);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        j++;
+                }
+                /* </bricks> */
+                ret = xmlTextWriterEndElement (local->writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d", i);
+                ret = cli_xml_output_vol_info_options (local->writer, dict,
+                                                       key);
+                if (ret)
+                        goto out;
+
+                /* </volume> */
+                ret = xmlTextWriterEndElement (local->writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        if (volname) {
+                GF_FREE (local->get_vol.volname);
+                local->get_vol.volname = gf_strdup (volname);
+                local->vol_count += count;
+        }
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_info_begin (cli_local_t *local, int op_ret, int op_errno,
+                               char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+
+        GF_ASSERT (local);
+
+        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
+                                     op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volInfo> */
+        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volInfo");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* <volumes> */
+        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volumes");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* Init vol count */
+        local->vol_count = 0;
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_info_end (cli_local_t *local)
+{
+#if (HAVE_LIB_XML)
+        int             ret = -1;
+
+        GF_ASSERT (local);
+
+        ret = xmlTextWriterWriteFormatElement (local->writer,
+                                               (xmlChar *)"count",
+                                               "%d", local->vol_count);
+
+        /* </volumes> */
+        ret = xmlTextWriterEndElement (local->writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </volInfo> */
+        ret = xmlTextWriterEndElement (local->writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (local->writer, local->doc);
+
+out:
+        gf_log ("cli", GF_LOG_ERROR, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_quota_limit_list (char *volname, char *limit_list,
+                                     int op_ret, int op_errno,
+                                     char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        int64_t                 size = 0;
+        int64_t                 limit_value = 0;
+        int                     i = 0;
+        int                     j = 0;
+        int                     k = 0;
+        int                     len = 0;
+        char                    *size_str = NULL;
+        char                    path[PATH_MAX] = {0,};
+        char                    ret_str[1024] = {0,};
+        char                    value[1024] = {0,};
+        char                    mountdir[] = "/tmp/mountXXXXXX";
+        char                    abspath[PATH_MAX] = {0,};
+        runner_t                runner = {0,};
+
+        GF_ASSERT (volname);
+        GF_ASSERT (limit_list);
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volQuota> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volQuota");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        if (!limit_list)
+                goto cont;
+
+        len = strlen (limit_list);
+        if (len == 0)
+                goto cont;
+
+        if (mkdtemp (mountdir) == NULL) {
+                gf_log ("cli", GF_LOG_ERROR, "failed to create a temporary"
+                        " mount directory");
+                ret = -1;
+                goto out;
+        }
+
+        ret = runcmd (SBIN_DIR"/glusterfs", "-s", "localhost",
+                      "--volfile-id", volname, "-l",
+                      DEFAULT_LOG_FILE_DIRECTORY"/quota-list-xml.log",
+                      mountdir, NULL);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR,
+                        "failed to mount glusterfs client");
+                ret = -1;
+                goto rm_dir;
+        }
+
+        while (i < len) {
+                j = 0;
+                k = 0;
+                size = 0;
+
+                while (limit_list[i] != ':')
+                        path[k++] = limit_list[i++];
+                path[k] = '\0';
+
+                i++;
+
+                while (limit_list[i] != ',' && limit_list[i] != '\0')
+                        value[j++] = limit_list[i++];
+                i++;
+
+                snprintf (abspath, sizeof (abspath), "%s/%s", mountdir, path);
+                ret = sys_lgetxattr (abspath, "trusted.limit.list",
+                                     (void *)ret_str, 4096);
+                if (ret >= 0) {
+                        sscanf (ret_str, "%"SCNd64",%"SCNd64, &size,
+                                &limit_value);
+                        size_str = gf_uint64_2human_readable ((uint64_t)size);
+                }
+
+                /* <quota> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"quota");
+                XML_RET_CHECK_AND_GOTO (ret, unmount);
+
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"path", "%s", path);
+                XML_RET_CHECK_AND_GOTO (ret, unmount);
+
+                ret = xmlTextWriterWriteFormatElement
+                        (writer, (xmlChar *)"limit", "%s", value);
+                XML_RET_CHECK_AND_GOTO (ret, unmount);
+
+                if (size_str) {
+                        ret = xmlTextWriterWriteFormatElement
+                                (writer, (xmlChar *)"size", "%s", size_str);
+                        XML_RET_CHECK_AND_GOTO (ret, unmount);
+                        GF_FREE (size_str);
+                } else {
+                        ret = xmlTextWriterWriteFormatElement
+                                (writer, (xmlChar *)"size", "%"PRId64, size);
+                        XML_RET_CHECK_AND_GOTO (ret, unmount);
+                }
+
+                /* </quota> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, unmount);
+
+        }
+
+unmount:
+        runinit (&runner);
+        runner_add_args (&runner, "umount",
+#if GF_LINUX_HOST_OS
+                         "-l",
+#endif
+                         mountdir, NULL);
+        ret = runner_run_reuse (&runner);
+        if (ret)
+                runner_log (&runner, "cli", GF_LOG_WARNING, "error executing");
+        runner_end (&runner);
+
+rm_dir:
+        rmdir (mountdir);
+
+cont:
+        /* </volQuota> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        GF_FREE (size_str);
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_peer_status (dict_t *dict, int op_ret, int op_errno,
+                            char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        int                     count = 0;
+        char                    *uuid = NULL;
+        char                    *hostname = NULL;
+        int                     connected = 0;
+        int                     state_id = 0;
+        char                    *state_str = NULL;
+        int                     i = 1;
+        char                    key[1024] = {0,};
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <peerStatus> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"peerStatus");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        if (!dict)
+                goto cont;
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                goto out;
+
+        while (i <= count) {
+                /* <peer> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"peer");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "friend%d.uuid", i);
+                ret = dict_get_str (dict, key, &uuid);
+                if (ret)
+                        goto out;
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"uuid",
+                                                       "%s", uuid);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "friend%d.hostname", i);
+                ret = dict_get_str (dict, key, &hostname);
+                if (ret)
+                        goto out;
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"hostname",
+                                                       "%s", hostname);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "friend%d.connected", i);
+                ret = dict_get_int32 (dict, key, &connected);
+                if (ret)
+                        goto out;
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"connected",
+                                                       "%d", connected);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "friend%d.stateId", i);
+                ret = dict_get_int32 (dict, key, &state_id);
+                if (!ret) {
+                        /* ignore */
+
+                        ret = xmlTextWriterWriteFormatElement (writer,
+                                           (xmlChar *)"state", "%d", state_id);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "friend%d.state", i);
+                ret = dict_get_str (dict, key, &state_str);
+                if (!ret) {
+                        /* ignore */
+
+                        ret = xmlTextWriterWriteFormatElement (writer,
+                                       (xmlChar *)"stateStr", "%s", state_str);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+                }
+
+                /* </peer> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                i++;
+        }
+
+cont:
+        /* </peerStatus> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+/* Used for rebalance stop/status, remove-brick status */
+int
+cli_xml_output_vol_rebalance_status (xmlTextWriterPtr writer, dict_t *dict,
+                                     enum gf_task_types task_type)
+{
+        int                     ret = -1;
+        int                     count = 0;
+        char                    *node_name = NULL;
+        char                    *node_uuid = NULL;
+        uint64_t                files = 0;
+        uint64_t                size = 0;
+        uint64_t                lookups = 0;
+        int                     status_rcd = 0;
+        uint64_t                failures = 0;
+        uint64_t                skipped = 0;
+        uint64_t                total_files = 0;
+        uint64_t                total_size = 0;
+        uint64_t                total_lookups = 0;
+        uint64_t                total_failures = 0;
+        uint64_t                total_skipped = 0;
+        char                    key[1024] = {0,};
+        int                     i = 0;
+        int                     overall_status = -1;
+        double                  elapsed = 0;
+        double                  overall_elapsed = 0;
+
+        if (!dict) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"nodeCount",
+                                               "%d", count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        while (i < count) {
+                i++;
+
+                /* <node> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"node");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "node-name-%d", i);
+                ret = dict_get_str (dict, key, &node_name);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"nodeName",
+                                                       "%s", node_name);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "node-uuid-%d", i);
+                ret = dict_get_str (dict, key, &node_uuid);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"id",
+                                                       "%s", node_uuid);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "files-%d", i);
+                ret = dict_get_uint64 (dict, key, &files);
+                if (ret)
+                        goto out;
+                total_files += files;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"files",
+                                                       "%"PRIu64, files);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "size-%d", i);
+                ret = dict_get_uint64 (dict, key, &size);
+                if (ret)
+                        goto out;
+                total_size += size;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"size",
+                                                       "%"PRIu64,size);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "lookups-%d", i);
+                ret = dict_get_uint64 (dict, key, &lookups);
+                if (ret)
+                        goto out;
+                total_lookups += lookups;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"lookups",
+                                                       "%"PRIu64, lookups);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &failures);
+                if (ret)
+                        goto out;
+                total_failures += failures;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"failures",
+                                                       "%"PRIu64, failures);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* skipped-%d is not available for remove brick in dict,
+                   so using failures as skipped count in case of remove-brick
+                   similar to logic used in CLI(non xml output) */
+                if (task_type == GF_TASK_TYPE_REBALANCE) {
+                    memset (key, 0, sizeof (key));
+                    snprintf (key, sizeof (key), "skipped-%d", i);
+                }
+                else {
+                    memset (key, 0, sizeof (key));
+                    snprintf (key, sizeof (key), "failures-%d", i);
+                }
+
+                ret = dict_get_uint64 (dict, key, &skipped);
+                if (ret)
+                        goto out;
+                total_skipped += skipped;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"skipped",
+                                                       "%"PRIu64, skipped);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "status-%d", i);
+                ret = dict_get_int32 (dict, key, &status_rcd);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"status",
+                                                       "%d", status_rcd);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"statusStr",
+                                                       "%s",
+                                         cli_vol_task_status_str[status_rcd]);
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "run-time-%d", i);
+                ret = dict_get_double (dict, key, &elapsed);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"runtime",
+                                                       "%.2f", elapsed);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                if (elapsed > overall_elapsed) {
+                    overall_elapsed = elapsed;
+                }
+
+                if (-1 == overall_status)
+                        overall_status = status_rcd;
+                else if ((GF_DEFRAG_STATUS_COMPLETE == overall_status ||
+                          status_rcd > overall_status) &&
+                         (status_rcd != GF_DEFRAG_STATUS_COMPLETE))
+                        overall_status = status_rcd;
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </node> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* Aggregate status */
+        /* <aggregate> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"aggregate");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"files",
+                                               "%"PRIu64, total_files);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"size",
+                                               "%"PRIu64, total_size);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"lookups",
+                                               "%"PRIu64, total_lookups);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"failures",
+                                               "%"PRIu64, total_failures);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"skipped",
+                                               "%"PRIu64, total_skipped);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"status",
+                                               "%d", overall_status);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"statusStr",
+                                               "%s",
+                                      cli_vol_task_status_str[overall_status]);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"runtime",
+                                               "%.2f", overall_elapsed);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </aggregate> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_rebalance (gf_cli_defrag_type op, dict_t *dict, int op_ret,
+                              int op_errno, char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *task_id_str = NULL;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volRebalance> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volRebalance");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
+        if (ret == 0) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"task-id",
+                                                       "%s", task_id_str);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"op",
+                                               "%d", op);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        if ((GF_DEFRAG_CMD_STOP == op) || (GF_DEFRAG_CMD_STATUS == op)) {
+                ret = cli_xml_output_vol_rebalance_status (writer, dict,
+                                                      GF_TASK_TYPE_REBALANCE);
+                if (ret)
+                        goto out;
+        }
+
+        /* </volRebalance> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_remove_brick (gf_boolean_t status_op, dict_t *dict,
+                                 int op_ret, int op_errno, char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *task_id_str = NULL;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volRemoveBrick> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volRemoveBrick");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
+        if (ret == 0) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"task-id",
+                                                       "%s", task_id_str);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        if (status_op) {
+                ret = cli_xml_output_vol_rebalance_status (writer, dict,
+                                                   GF_TASK_TYPE_REMOVE_BRICK);
+                if (ret)
+                        goto out;
+        }
+
+        /* </volRemoveBrick> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_replace_brick (gf1_cli_replace_op op, dict_t *dict,
+                                  int op_ret, int op_errno, char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        int                     status = 0;
+        uint64_t                files = 0;
+        char                    *current_file = 0;
+        char                    *task_id_str = NULL;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volReplaceBrick> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volReplaceBrick");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_str (dict, GF_REPLACE_BRICK_TID_KEY, &task_id_str);
+        if (ret == 0) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"task-id",
+                                                       "%s", task_id_str);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"op",
+                                               "%d", op);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        if (GF_REPLACE_OP_STATUS == op) {
+                ret = dict_get_int32 (dict, "status", &status);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"status",
+                                                       "%d", status);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = dict_get_uint64 (dict, "files", &files);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"files",
+                                                       "%"PRIu64, files);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                if (status)
+                        goto cont;
+
+                ret = dict_get_str (dict, "current_file", &current_file);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"currentFile",
+                                                       "%s", current_file);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+cont:
+        /* </volReplaceBrick> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_create (dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *volname = NULL;
+        char                    *volid = NULL;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        if (dict) {
+                /* <volCreate> */
+                ret = xmlTextWriterStartElement (writer,
+                                                 (xmlChar *)"volCreate");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* <volume> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *) "name",
+                                                       "%s", volname);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = dict_get_str (dict, "volume-id", &volid);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"id",
+                                                       "%s", volid);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </volume> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </volCreate> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_generic_volume (char *op, dict_t *dict, int op_ret, int op_errno,
+                               char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *volname = NULL;
+        char                    *volid = NULL;
+
+        GF_ASSERT (op);
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        if (dict) {
+                /* <"op"> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)op);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* <volume> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *) "name",
+                                                       "%s", volname);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = dict_get_str (dict, "vol-id", &volid);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"id",
+                                                       "%s", volid);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </volume> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </"op"> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_gsync_status (dict_t *dict, xmlTextWriterPtr writer)
+{
+        char                    master_key[PATH_MAX] = "";
+        char                    slave_key[PATH_MAX] = "";
+        char                    status_key[PATH_MAX] = "";
+        char                    node_key[PATH_MAX] = "";
+        char                    *master = NULL;
+        char                    *slave = NULL;
+        char                    *status = NULL;
+        char                    *node = NULL;
+        int                     ret = -1;
+        int                     gsync_count = 0;
+        int                     i = 1;
+
+        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
+        if (ret)
+                goto out;
+
+        for (i=1; i <= gsync_count; i++) {
+                snprintf (node_key, sizeof(node_key), "node%d", i);
+                snprintf (master_key, sizeof(master_key), "master%d", i);
+                snprintf (slave_key, sizeof(slave_key), "slave%d", i);
+                snprintf (status_key, sizeof(status_key), "status%d", i);
+
+                ret = dict_get_str (dict, node_key, &node);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (dict, master_key, &master);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (dict, slave_key, &slave);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (dict, status_key, &status);
+                if (ret)
+                        goto out;
+
+                /* <pair> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"pair");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"node",
+                                                       "%s", node);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"master",
+                                                       "%s", master);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"slave",
+                                                       "%s", slave);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"status",
+                                                       "%s", status);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </pair> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+        }
+
+out:
+        gf_log ("cli",GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_gsync (dict_t *dict, int op_ret, int op_errno,
+                          char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *master = NULL;
+        char                    *slave = NULL;
+        int                     type = 0;
+
+        GF_ASSERT (dict);
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <geoRep> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"geoRep");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to get type");
+                goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"type",
+                                               "%d", type);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        switch (type) {
+        case GF_GSYNC_OPTION_TYPE_START:
+        case GF_GSYNC_OPTION_TYPE_STOP:
+                if (dict_get_str (dict, "master", &master) != 0)
+                        master = "???";
+                if (dict_get_str (dict, "slave", &slave) != 0)
+                        slave = "???";
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"master",
+                                                       "%s", master);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"slave",
+                                                       "%s", slave);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+                break;
+        case GF_GSYNC_OPTION_TYPE_STATUS:
+                ret = cli_xml_output_vol_gsync_status(dict, writer);
+                break;
+        default:
+                ret = 0;
+                break;
+        }
+
+        /* </geoRep> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (writer, doc);
+out:
+        gf_log ("cli",GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
diff --git a/cli/src/cli.c b/cli/src/cli.c
index 78c76c53f..91b315ff1 100644
--- a/cli/src/cli.c
+++ b/cli/src/cli.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -99,43 +89,6 @@ rpc_clnt_prog_t *cli_rpc_prog;
 
 extern struct rpc_clnt_program cli_prog;
 
-
-
-
-static char *
-generate_uuid ()
-{
-        char           tmp_str[1024] = {0,};
-        char           hostname[256] = {0,};
-        struct timeval tv = {0,};
-        struct tm      now = {0, };
-        char           now_str[32];
-
-        if (gettimeofday (&tv, NULL) == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "gettimeofday: failed %s",
-                        strerror (errno));
-        }
-
-        if (gethostname (hostname, 256) == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "gethostname: failed %s",
-                        strerror (errno));
-        }
-
-        localtime_r (&tv.tv_sec, &now);
-        strftime (now_str, 32, "%Y/%m/%d-%H:%M:%S", &now);
-        snprintf (tmp_str, 1024, "%s-%d-%s:%"
-#ifdef GF_DARWIN_HOST_OS
-                  PRId32,
-#else
-                  "ld",
-#endif
-                  hostname, getpid(), now_str, tv.tv_usec);
-
-        return gf_strdup (tmp_str);
-}
-
 static int
 glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
 {
@@ -145,7 +98,7 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
 
         xlator_mem_acct_init (THIS, cli_mt_end);
 
-        ctx->process_uuid = generate_uuid ();
+        ctx->process_uuid = generate_glusterfs_ctx_id ();
         if (!ctx->process_uuid)
                 return -1;
 
@@ -164,22 +117,33 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
         if (!pool)
                 return -1;
 
-        /* frame_mem_pool size 112 * 16k */
-        pool->frame_mem_pool = mem_pool_new (call_frame_t, 16384);
-
+        /* frame_mem_pool size 112 * 64 */
+        pool->frame_mem_pool = mem_pool_new (call_frame_t, 32);
         if (!pool->frame_mem_pool)
                 return -1;
 
-        /* stack_mem_pool size 256 * 8k */
-        pool->stack_mem_pool = mem_pool_new (call_stack_t, 8192); 
+        /* stack_mem_pool size 256 * 128 */
+        pool->stack_mem_pool = mem_pool_new (call_stack_t, 16);
 
         if (!pool->stack_mem_pool)
                 return -1;
 
-        ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
+        ctx->stub_mem_pool = mem_pool_new (call_stub_t, 16);
         if (!ctx->stub_mem_pool)
                 return -1;
 
+        ctx->dict_pool = mem_pool_new (dict_t, 32);
+        if (!ctx->dict_pool)
+                return -1;
+
+        ctx->dict_pair_pool = mem_pool_new (data_pair_t, 512);
+        if (!ctx->dict_pair_pool)
+                return -1;
+
+        ctx->dict_data_pool = mem_pool_new (data_t, 512);
+        if (!ctx->dict_data_pool)
+                return -1;
+
         INIT_LIST_HEAD (&pool->all_frames);
         LOCK_INIT (&pool->lock);
         ctx->pool = pool;
@@ -199,12 +163,13 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
 
 
 static int
-logging_init (struct cli_state *state)
+logging_init (glusterfs_ctx_t *ctx, struct cli_state *state)
 {
         char *log_file = state->log_file ? state->log_file :
                          DEFAULT_CLI_LOG_FILE_DIRECTORY "/cli.log";
 
-        if (gf_log_init (log_file) == -1) {
+        /* passing ident as NULL means to use default ident for syslog */
+        if (gf_log_init (ctx, log_file, NULL) == -1) {
                 fprintf (stderr, "ERROR: failed to open logfile %s\n",
                          log_file);
                 return -1;
@@ -273,6 +238,8 @@ cli_submit_request (void *req, call_frame_t *frame,
 out:
         if (new_iobref)
                 iobref_unref (iobref);
+        if (iobuf)
+                iobuf_unref (iobuf);
         return ret;
 }
 
@@ -317,16 +284,44 @@ cli_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
         return ret;
 }
 
+
+/*
+ * ret: 0: option successfully processed
+ *      1: signalling end of option list
+ *     -1: unknown option or other issue
+ */
 int
 cli_opt_parse (char *opt, struct cli_state *state)
 {
         char *oarg;
 
+        if (strcmp (opt, "") == 0)
+                return 1;
+
         if (strcmp (opt, "version") == 0) {
-                puts (argp_program_version);
+                cli_out ("%s", argp_program_version);
                 exit (0);
         }
 
+        if (strcmp (opt, "print-logdir") == 0) {
+                cli_out ("%s", DEFAULT_LOG_FILE_DIRECTORY);
+                exit (0);
+        }
+
+        if (strcmp (opt, "print-statedumpdir") == 0) {
+                cli_out ("%s", DEFAULT_VAR_RUN_DIRECTORY);
+                exit (0);
+        }
+
+        if (strcmp (opt, "xml") == 0) {
+#if (HAVE_LIB_XML)
+                state->mode |= GLUSTER_MODE_XML;
+#else
+                cli_err ("XML output not supported. Ignoring '--xml' option");
+#endif
+                return 0;
+        }
+
         oarg = strtail (opt, "mode=");
         if (oarg) {
                 if (strcmp (oarg, "script") == 0) {
@@ -358,6 +353,12 @@ cli_opt_parse (char *opt, struct cli_state *state)
                 return 0;
         }
 
+        oarg = strtail (opt, "glusterd-sock=");
+        if (oarg) {
+                state->glusterd_sock = oarg;
+                return 0;
+        }
+
         return -1;
 }
 
@@ -385,9 +386,16 @@ parse_cmdline (int argc, char *argv[], struct cli_state *state)
                         state->argc--;
                         /* argv shifted, next check should be at i again */
                         i--;
+                        if (ret == 1) {
+                                /* end of cli options */
+                                ret = 0;
+                                break;
+                        }
                 }
         }
 
+        state->argv[state->argc] = NULL;
+
         return ret;
 }
 
@@ -412,7 +420,6 @@ cli_state_init (struct cli_state *state)
         int                   ret = 0;
 
 
-        state->remote_host = "localhost";
         state->log_level = -1;
 
         tree = &state->tree;
@@ -432,11 +439,35 @@ cli_usage_out (const char *usage)
         if (!usage || usage[0] == '\0')
                 return -1;
 
-        cli_out ("Usage: %s", usage);
+        cli_err ("Usage: %s", usage);
         return 0;
 }
 
 int
+_cli_err (const char *fmt, ...)
+{
+        struct cli_state *state = NULL;
+        va_list           ap;
+        int               ret = 0;
+
+        state = global_state;
+
+        va_start (ap, fmt);
+
+#ifdef HAVE_READLINE
+        if (state->rl_enabled && !state->rl_processing)
+                return cli_rl_err(state, fmt, ap);
+#endif
+
+        ret = vfprintf (stderr, fmt, ap);
+        fprintf (stderr, "\n");
+        va_end (ap);
+
+        return ret;
+}
+
+
+int
 _cli_out (const char *fmt, ...)
 {
         struct cli_state *state = NULL;
@@ -475,23 +506,46 @@ cli_rpc_init (struct cli_state *state)
         if (!options)
                 goto out;
 
-        ret = dict_set_str (options, "remote-host", state->remote_host);
-        if (ret)
-                goto out;
-
-        if (state->remote_port)
-                port = state->remote_port;
+        /* Connect using to glusterd using the specified method, giving
+         * preference to unix socket connection. If nothing is specified connect
+         * to the default glusterd socket
+         */
+        if (state->glusterd_sock) {
+                gf_log ("cli", GF_LOG_INFO, "Connecting to glusterd using "
+                        "sockfile %s", state->glusterd_sock);
+                ret = rpc_transport_unix_options_build (&options,
+                                                        state->glusterd_sock,
+                                                        0);
+                if (ret)
+                        goto out;
+        } else if (state->remote_host) {
+                gf_log ("cli", GF_LOG_INFO, "Connecting to remote glusterd at "
+                        "%s", state->remote_host);
+                ret = dict_set_str (options, "remote-host", state->remote_host);
+                if (ret)
+                        goto out;
 
-        ret = dict_set_int32 (options, "remote-port", port);
-        if (ret)
-                goto out;
+                if (state->remote_port)
+                        port = state->remote_port;
 
-        ret = dict_set_str (options, "transport.address-family", "inet/inet6");
-        if (ret)
-                goto out;
+                ret = dict_set_int32 (options, "remote-port", port);
+                if (ret)
+                        goto out;
 
-        rpc = rpc_clnt_new (options, this->ctx, this->name);
+                ret = dict_set_str (options, "transport.address-family",
+                                    "inet");
+                if (ret)
+                        goto out;
+        } else {
+                gf_log ("cli", GF_LOG_DEBUG, "Connecting to glusterd using "
+                        "default socket");
+                ret = rpc_transport_unix_options_build
+                        (&options, DEFAULT_GLUSTERD_SOCKFILE, 0);
+                if (ret)
+                        goto out;
+        }
 
+        rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
         if (!rpc)
                 goto out;
 
@@ -501,7 +555,7 @@ cli_rpc_init (struct cli_state *state)
                 goto out;
         }
 
-        rpc_clnt_start (rpc);
+        ret = rpc_clnt_start (rpc);
 out:
         if (ret) {
                 if (rpc)
@@ -525,76 +579,15 @@ void
 cli_local_wipe (cli_local_t *local)
 {
         if (local) {
+                GF_FREE (local->get_vol.volname);
+                if (local->dict)
+                        dict_unref (local->dict);
                 GF_FREE (local);
         }
 
         return;
 }
 
-/* If the path exists use realpath(3) to handle extra slashes and to resolve
- * symlinks else strip the extra slashes in the path and return */
-
-int
-cli_canonicalize_path (char *path)
-{
-        struct stat     sb = {0};
-        int             ret = -1;
-        char            *tmppath = NULL;
-        char            *dir = NULL;
-        char            *tmpstr = NULL;
-        int             path_len = 0;
-
-        if (!path)
-                return ret;
-
-        ret = stat (path, &sb);
-        if (ret == -1) {
-                /* Strip the extra slashes and return */
-                tmppath = gf_strdup (path);
-                if (tmppath == NULL) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Out of memory.");
-                        goto out;
-                }
-                bzero (path, strlen(path));
-                path[0] = '/';
-                dir = strtok_r(tmppath, "/", &tmpstr);
-                while (dir) {
-                        strncpy ((path + path_len + 1), dir, strlen(dir));
-                        path_len = strlen (path);
-                        dir = strtok_r(NULL, "/", &tmpstr);
-                        if (dir)
-                                strncpy((path + path_len), "/", 1);
-                }
-                if (path_len == 0)
-                        path[1] = '\0';
-                else
-                        path[path_len] = '\0';
-                ret = 0;
-                goto out;
-        } else {
-                tmppath = gf_strdup(path);
-                if (tmppath == NULL) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Out of memory.");
-                        goto out;
-                }
-                if (realpath (tmppath, path) == NULL) {
-                        cli_out ("Path manipulation failed: %s",
-                                 strerror(errno));
-                        gf_log ("cli", GF_LOG_ERROR, "Path manipulation "
-                                 "failed: %s", strerror(errno));
-                        ret = -1;
-                        goto out;
-                }
-                ret = 0;
-        }
-out:
-        if (tmppath)
-                GF_FREE(tmppath);
-        return ret;
-}
-
 struct cli_state *global_state;
 
 int
@@ -604,13 +597,19 @@ main (int argc, char *argv[])
         int                ret = -1;
         glusterfs_ctx_t   *ctx = NULL;
 
-        ret = glusterfs_globals_init ();
+        ctx = glusterfs_ctx_new ();
+        if (!ctx)
+                return ENOMEM;
+
+#ifdef DEBUG
+        gf_mem_acct_enable_set (ctx);
+#endif
+
+        ret = glusterfs_globals_init (ctx);
         if (ret)
                 return ret;
 
-        ctx = glusterfs_ctx_get ();
-        if (!ctx)
-                return ENOMEM;
+	THIS->ctx = ctx;
 
         ret = glusterfs_ctx_defaults_init (ctx);
         if (ret)
@@ -627,7 +626,7 @@ main (int argc, char *argv[])
         if (ret)
                 goto out;
 
-        ret = logging_init (&state);
+        ret = logging_init (ctx, &state);
         if (ret)
                 goto out;
 
diff --git a/cli/src/cli.h b/cli/src/cli.h
index e091e7004..8daa4b741 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLI_H__
 #define __CLI_H__
 
@@ -28,12 +18,20 @@
 #include "rpc-clnt.h"
 #include "glusterfs.h"
 #include "protocol-common.h"
+#include "logging.h"
+
+#include "cli1-xdr.h"
+
+#if (HAVE_LIB_XML)
+#include <libxml/encoding.h>
+#include <libxml/xmlwriter.h>
+#endif
 
 #define DEFAULT_EVENT_POOL_SIZE            16384
 #define CLI_GLUSTERD_PORT                  24007
 #define CLI_DEFAULT_CONN_TIMEOUT             120
 #define CLI_DEFAULT_CMD_TIMEOUT              120
-#define CLI_TOP_CMD_TIMEOUT                  300 //Longer timeout for volume top
+#define CLI_TEN_MINUTES_TIMEOUT              600 //Longer timeout for volume top
 #define DEFAULT_CLI_LOG_FILE_DIRECTORY     DATADIR "/log/glusterfs"
 #define CLI_VOL_STATUS_BRICK_LEN              55
 #define CLI_TAB_LENGTH                         8
@@ -46,12 +44,16 @@ enum argp_option_keys {
 
 #define GLUSTER_MODE_SCRIPT    (1 << 0)
 #define GLUSTER_MODE_ERR_FATAL (1 << 1)
-
+#define GLUSTER_MODE_XML       (1 << 2)
 struct cli_state;
 struct cli_cmd_word;
 struct cli_cmd_tree;
 struct cli_cmd;
 
+extern char *cli_vol_type_str[];
+extern char *cli_vol_status_str[];
+extern char *cli_vol_task_status_str[];
+
 typedef int (cli_cmd_cbk_t)(struct cli_state *state,
                             struct cli_cmd_word *word,
                             const char **words,
@@ -113,6 +115,8 @@ struct cli_state {
 
         char                 *log_file;
         gf_loglevel_t         log_level;
+
+        char                 *glusterd_sock;
 };
 
 struct cli_local {
@@ -122,6 +126,26 @@ struct cli_local {
         } get_vol;
 
         dict_t          *dict;
+        const char      **words;
+        /* Marker for volume status all */
+        gf_boolean_t    all;
+#if (HAVE_LIB_XML)
+        xmlTextWriterPtr        writer;
+        xmlDocPtr               doc;
+        int                     vol_count;
+#endif
+};
+
+struct gf_cli_gsync_detailed_status_ {
+        char *node;
+        char *master;
+        char *slave;
+        char *health;
+        char *uptime;
+        char *files_syncd;
+        char *files_pending;
+        char *bytes_pending;
+        char *deletes_pending;
 };
 
 struct cli_volume_status {
@@ -132,14 +156,23 @@ struct cli_volume_status {
         uint64_t       free_inodes;
         char          *brick;
         char          *pid_str;
-        char          *fs_name;
         char          *free;
         char          *total;
+#ifdef GF_LINUX_HOST_OS
+        char          *fs_name;
         char          *mount_options;
         char          *device;
         char          *inode_size;
+#endif
 };
 
+struct snap_config_opt_vals_ {
+        char           *op_name;
+        char           *question;
+};
+
+typedef struct gf_cli_gsync_detailed_status_ gf_cli_gsync_status_t;
+
 typedef struct cli_volume_status cli_volume_status_t;
 
 typedef struct cli_local cli_local_t;
@@ -162,10 +195,12 @@ int cli_cmd_process_line (struct cli_state *state, const char *line);
 
 int cli_rl_enable (struct cli_state *state);
 int cli_rl_out (struct cli_state *state, const char *fmt, va_list ap);
+int cli_rl_err (struct cli_state *state, const char *fmt, va_list ap);
 
 int cli_usage_out (const char *usage);
 
 int _cli_out (const char *fmt, ...);
+int _cli_err (const char *fmt, ...);
 
 #define cli_out(fmt...) do {                       \
                 FMT_WARN (fmt);                    \
@@ -174,6 +209,13 @@ int _cli_out (const char *fmt, ...);
                                                    \
         } while (0)
 
+#define cli_err(fmt...) do {                       \
+                FMT_WARN (fmt);                    \
+                                                   \
+                _cli_err(fmt);                     \
+                                                   \
+        } while (0)
+
 int
 cli_submit_request (void *req, call_frame_t *frame,
                     rpc_clnt_prog_t *prog,
@@ -195,7 +237,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **opt);
 
 int32_t
 cli_cmd_volume_set_parse (const char **words, int wordcount,
-                          dict_t **options);
+                          dict_t **options, char **op_errstr);
 
 int32_t
 cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
@@ -219,6 +261,9 @@ cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options)
 int32_t
 cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
                                         dict_t **options);
+int32_t
+cli_cmd_volume_clrlks_opts_parse (const char **words, int wordcount,
+                                  dict_t **options);
 
 cli_local_t * cli_local_get ();
 
@@ -235,9 +280,6 @@ int
 cli_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                 void *data);
 
-int
-cli_canonicalize_path (char *path);
-
 int32_t
 cli_cmd_volume_profile_parse (const char **words, int wordcount,
                               dict_t **options);
@@ -254,6 +296,14 @@ cli_cmd_volume_status_parse (const char **words, int wordcount,
                              dict_t **options);
 
 int
+cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
+                                   dict_t **options);
+
+int
+cli_cmd_volume_defrag_parse (const char **words, int wordcount,
+                             dict_t **options);
+
+int
 cli_print_brick_status (cli_volume_status_t *status);
 
 void
@@ -264,4 +314,87 @@ cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status);
 
 void
 cli_print_line (int len);
+
+int
+cli_xml_output_str (char *op, char *str, int op_ret, int op_errno,
+                    char *op_errstr);
+
+int
+cli_xml_output_dict (char *op, dict_t *dict, int op_ret, int op_errno,
+                     char *op_errstr);
+
+int
+cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr);
+
+int
+cli_xml_output_vol_profile (dict_t *dict, int op_ret, int op_errno,
+                            char *op_errstr);
+
+int
+cli_xml_output_vol_status_begin (cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr);
+
+int
+cli_xml_output_vol_status_end (cli_local_t *local);
+
+int
+cli_xml_output_vol_status (cli_local_t *local, dict_t *dict);
+
+int
+cli_xml_output_vol_list (dict_t *dict, int op_ret, int op_errno,
+                         char *op_errstr);
+
+int
+cli_xml_output_vol_info_begin (cli_local_t *local, int op_ret, int op_errno,
+                               char *op_errstr);
+
+int
+cli_xml_output_vol_info_end (cli_local_t *local);
+
+int
+cli_xml_output_vol_info (cli_local_t *local, dict_t *dict);
+
+int
+cli_xml_output_vol_quota_limit_list (char *volname, char *limit_list,
+                                      int op_ret, int op_errno,
+                                      char *op_errstr);
+
+int
+cli_xml_output_peer_status (dict_t *dict, int op_ret, int op_errno,
+                            char *op_errstr);
+
+int
+cli_xml_output_vol_rebalance (gf_cli_defrag_type op, dict_t *dict, int op_ret,
+                              int op_errno, char *op_errstr);
+
+int
+cli_xml_output_vol_remove_brick (gf_boolean_t status_op, dict_t *dict,
+                                 int op_ret, int op_errno, char *op_errstr);
+
+int
+cli_xml_output_vol_replace_brick (gf1_cli_replace_op op, dict_t *dict,
+                                  int op_ret, int op_errno, char *op_errstr);
+
+int
+cli_xml_output_vol_create (dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
+
+int
+cli_xml_output_generic_volume (char *op, dict_t *dict, int op_ret, int op_errno,
+                               char *op_errstr);
+
+int
+cli_xml_output_vol_gsync (dict_t *dict, int op_ret, int op_errno,
+                          char *op_errstr);
+int
+cli_xml_output_vol_status_tasks_detail (cli_local_t *local, dict_t *dict);
+
+char *
+is_server_debug_xlator (void *myframe);
+
+int32_t
+cli_cmd_snapshot_parse (const char **words, int wordcount, dict_t **options,
+                        struct cli_state *state);
+
 #endif /* __CLI_H__ */
diff --git a/cli/src/input.c b/cli/src/input.c
index a88d35874..a8ea46c6d 100644
--- a/cli/src/input.c
+++ b/cli/src/input.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -44,7 +34,7 @@ cli_batch (void *d)
         ret = cli_cmd_process (state, state->argc, state->argv);
 
         gf_log ("", GF_LOG_INFO, "Exiting with: %d", ret);
-        exit (ret);
+        exit (-ret);
 
         return NULL;
 }
@@ -71,11 +61,11 @@ cli_input (void *d)
                 if (len > 0 && cmd[len - 1] == '\n') //strip trailing \n
                         cmd[len - 1] = '\0';
                 ret = cli_cmd_process_line (state, cmd);
-                if (ret == -1 && state->mode & GLUSTER_MODE_ERR_FATAL)
+                if (ret != 0 && state->mode & GLUSTER_MODE_ERR_FATAL)
                         break;
         }
 
-        exit (ret);
+        exit (-ret);
 
         return NULL;
 }
diff --git a/cli/src/registry.c b/cli/src/registry.c
index 5b63e82b1..c4abe3be0 100644
--- a/cli/src/registry.c
+++ b/cli/src/registry.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -387,7 +377,7 @@ cli_cmd_register (struct cli_cmd_tree *tree, struct cli_cmd *cmd)
         char **tokens = NULL;
         int    ret = 0;
 
-        GF_ASSERT (cmd)
+        GF_ASSERT (cmd);
 
         if (cmd->reg_cbk)
                 cmd->reg_cbk (cmd);
diff --git a/configure.ac b/configure.ac
index d2b034dcb..b3d1ed184 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,21 +1,12 @@
-dnl Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-
-dnl This file is part of GlusterFS.
-dnl
-dnl GlusterFS is free software; you can redistribute it and/or modify
-dnl it under the terms of the GNU General Public License as published by
-dnl the Free Software Foundation; either version 3 of the License, or
-dnl (at your option) any later version.
-dnl
-dnl GlusterFS is distributed in the hope that it will be useful,
-dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
-dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-dnl GNU General Public License for more details.
+dnl  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+dnl  This file is part of GlusterFS.
 dnl
-dnl You should have received a copy of the GNU General Public License
-dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
+dnl  This file is licensed to you under your choice of the GNU Lesser
+dnl  General Public License, version 3 or any later version (LGPLv3 or
+dnl  later), or the GNU General Public License, version 2 (GPLv2), in all
+dnl  cases as published by the Free Software Foundation.
 
-AC_INIT([glusterfs],[3git],[gluster-users@gluster.org])
+AC_INIT([glusterfs],[3git],[gluster-users@gluster.org],,[https://github.com/gluster/glusterfs.git])
 
 AM_INIT_AUTOMAKE
 
@@ -33,13 +24,15 @@ if libtool --help 2>&1 | grep -q quiet; then
    AM_LIBTOOLFLAGS="--quiet";
 fi
 
-AM_CONFIG_HEADER([config.h])
+AC_CONFIG_HEADERS([config.h])
 
 AC_CONFIG_FILES([Makefile
-		libglusterfs/Makefile
-		libglusterfs/src/Makefile
-		glusterfsd/Makefile
-		glusterfsd/src/Makefile
+                libglusterfs/Makefile
+                libglusterfs/src/Makefile
+                geo-replication/src/peer_gsec_create
+                geo-replication/src/peer_add_secret_pub
+                glusterfsd/Makefile
+                glusterfsd/src/Makefile
                 rpc/Makefile
                 rpc/rpc-lib/Makefile
                 rpc/rpc-lib/src/Makefile
@@ -50,112 +43,162 @@ AC_CONFIG_FILES([Makefile
                 rpc/rpc-transport/rdma/src/Makefile
                 rpc/xdr/Makefile
                 rpc/xdr/src/Makefile
-		xlators/Makefile
-		xlators/mount/Makefile
-		xlators/mount/fuse/Makefile
-		xlators/mount/fuse/src/Makefile
-		xlators/mount/fuse/utils/mount.glusterfs
-		xlators/mount/fuse/utils/mount_glusterfs
-		xlators/mount/fuse/utils/Makefile
-		xlators/storage/Makefile
-		xlators/storage/posix/Makefile
-		xlators/storage/posix/src/Makefile
-		xlators/cluster/Makefile
-		xlators/cluster/afr/Makefile
-		xlators/cluster/afr/src/Makefile
-		xlators/cluster/stripe/Makefile
-		xlators/cluster/stripe/src/Makefile
-		xlators/cluster/dht/Makefile
-		xlators/cluster/dht/src/Makefile
-		xlators/performance/Makefile
-		xlators/performance/write-behind/Makefile
-		xlators/performance/write-behind/src/Makefile
-		xlators/performance/read-ahead/Makefile
-		xlators/performance/read-ahead/src/Makefile
-		xlators/performance/io-threads/Makefile
-		xlators/performance/io-threads/src/Makefile
-		xlators/performance/io-cache/Makefile
-		xlators/performance/io-cache/src/Makefile
-		xlators/performance/symlink-cache/Makefile
-		xlators/performance/symlink-cache/src/Makefile
-		xlators/performance/quick-read/Makefile
-		xlators/performance/quick-read/src/Makefile
-                xlators/performance/stat-prefetch/Makefile
-                xlators/performance/stat-prefetch/src/Makefile
-		xlators/debug/Makefile
-		xlators/debug/trace/Makefile
-		xlators/debug/trace/src/Makefile
-		xlators/debug/error-gen/Makefile
-		xlators/debug/error-gen/src/Makefile
-		xlators/debug/io-stats/Makefile
-		xlators/debug/io-stats/src/Makefile
-		xlators/protocol/Makefile
-		xlators/protocol/auth/Makefile
-		xlators/protocol/auth/addr/Makefile
-		xlators/protocol/auth/addr/src/Makefile
-		xlators/protocol/auth/login/Makefile
-		xlators/protocol/auth/login/src/Makefile
-		xlators/protocol/client/Makefile
-		xlators/protocol/client/src/Makefile
-		xlators/protocol/server/Makefile
-		xlators/protocol/server/src/Makefile
-		xlators/features/Makefile
-		xlators/features/locks/Makefile
-		xlators/features/locks/src/Makefile
-		xlators/features/trash/Makefile
-		xlators/features/trash/src/Makefile
-		xlators/features/quota/Makefile
-		xlators/features/quota/src/Makefile
+                xlators/Makefile
+                xlators/mount/Makefile
+                xlators/mount/fuse/Makefile
+                xlators/mount/fuse/src/Makefile
+                xlators/mount/fuse/utils/mount.glusterfs
+                xlators/mount/fuse/utils/mount_glusterfs
+                xlators/mount/fuse/utils/Makefile
+                xlators/storage/Makefile
+                xlators/storage/posix/Makefile
+                xlators/storage/posix/src/Makefile
+                xlators/storage/bd/Makefile
+                xlators/storage/bd/src/Makefile
+                xlators/cluster/Makefile
+                xlators/cluster/afr/Makefile
+                xlators/cluster/afr/src/Makefile
+                xlators/cluster/stripe/Makefile
+                xlators/cluster/stripe/src/Makefile
+                xlators/cluster/dht/Makefile
+                xlators/cluster/dht/src/Makefile
+                xlators/performance/Makefile
+                xlators/performance/write-behind/Makefile
+                xlators/performance/write-behind/src/Makefile
+                xlators/performance/read-ahead/Makefile
+                xlators/performance/read-ahead/src/Makefile
+                xlators/performance/readdir-ahead/Makefile
+                xlators/performance/readdir-ahead/src/Makefile
+                xlators/performance/io-threads/Makefile
+                xlators/performance/io-threads/src/Makefile
+                xlators/performance/io-cache/Makefile
+                xlators/performance/io-cache/src/Makefile
+                xlators/performance/symlink-cache/Makefile
+                xlators/performance/symlink-cache/src/Makefile
+                xlators/performance/quick-read/Makefile
+                xlators/performance/quick-read/src/Makefile
+                xlators/performance/open-behind/Makefile
+                xlators/performance/open-behind/src/Makefile
+                xlators/performance/md-cache/Makefile
+                xlators/performance/md-cache/src/Makefile
+                xlators/debug/Makefile
+                xlators/debug/trace/Makefile
+                xlators/debug/trace/src/Makefile
+                xlators/debug/error-gen/Makefile
+                xlators/debug/error-gen/src/Makefile
+                xlators/debug/io-stats/Makefile
+                xlators/debug/io-stats/src/Makefile
+                xlators/protocol/Makefile
+                xlators/protocol/auth/Makefile
+                xlators/protocol/auth/addr/Makefile
+                xlators/protocol/auth/addr/src/Makefile
+                xlators/protocol/auth/login/Makefile
+                xlators/protocol/auth/login/src/Makefile
+                xlators/protocol/client/Makefile
+                xlators/protocol/client/src/Makefile
+                xlators/protocol/server/Makefile
+                xlators/protocol/server/src/Makefile
+                xlators/features/Makefile
+		xlators/features/changelog/Makefile
+		xlators/features/changelog/src/Makefile
+		xlators/features/changelog/lib/Makefile
+		xlators/features/changelog/lib/src/Makefile
+                xlators/features/glupy/Makefile
+                xlators/features/glupy/src/Makefile
+                xlators/features/locks/Makefile
+                xlators/features/locks/src/Makefile
+                xlators/features/quota/Makefile
+                xlators/features/quota/src/Makefile
                 xlators/features/marker/Makefile
                 xlators/features/marker/src/Makefile
-                xlators/features/marker/utils/Makefile
-                xlators/features/marker/utils/src/Makefile
-                xlators/features/marker/utils/syncdaemon/Makefile
-		xlators/features/read-only/Makefile
-		xlators/features/read-only/src/Makefile
-		xlators/features/mac-compat/Makefile
-		xlators/features/mac-compat/src/Makefile
-		xlators/features/quiesce/Makefile
-		xlators/features/quiesce/src/Makefile
-		xlators/encryption/Makefile
-		xlators/encryption/rot-13/Makefile
-		xlators/encryption/rot-13/src/Makefile
+                xlators/features/read-only/Makefile
+                xlators/features/read-only/src/Makefile
+                xlators/features/compress/Makefile
+                xlators/features/compress/src/Makefile
+                xlators/features/mac-compat/Makefile
+                xlators/features/mac-compat/src/Makefile
+                xlators/features/quiesce/Makefile
+                xlators/features/quiesce/src/Makefile
+                xlators/features/index/Makefile
+                xlators/features/index/src/Makefile
+                xlators/features/protect/Makefile
+                xlators/features/protect/src/Makefile
+                xlators/features/gfid-access/Makefile
+                xlators/features/gfid-access/src/Makefile
+                xlators/playground/Makefile
+                xlators/playground/template/Makefile
+                xlators/playground/template/src/Makefile
+                xlators/encryption/Makefile
+                xlators/encryption/rot-13/Makefile
+                xlators/encryption/rot-13/src/Makefile
+                xlators/encryption/crypt/Makefile
+                xlators/encryption/crypt/src/Makefile
+		xlators/features/qemu-block/Makefile
+		xlators/features/qemu-block/src/Makefile
                 xlators/system/Makefile
                 xlators/system/posix-acl/Makefile
                 xlators/system/posix-acl/src/Makefile
-                cli/Makefile
-                cli/src/Makefile
-		doc/Makefile
-		doc/examples/Makefile
-		doc/hacker-guide/Makefile
-		extras/Makefile
-		extras/init.d/Makefile
-		extras/init.d/glusterd.plist
-                extras/init.d/glusterd-Debian
-                extras/init.d/glusterd-Redhat
-                extras/init.d/glusterd-SuSE
-		extras/benchmarking/Makefile
-		contrib/fuse-util/Makefile
-		contrib/uuid/uuid_types.h
                 xlators/nfs/Makefile
                 xlators/nfs/server/Makefile
                 xlators/nfs/server/src/Makefile
                 xlators/mgmt/Makefile
                 xlators/mgmt/glusterd/Makefile
                 xlators/mgmt/glusterd/src/Makefile
-		glusterfs.spec])
+                cli/Makefile
+                cli/src/Makefile
+                doc/Makefile
+                extras/Makefile
+                extras/init.d/Makefile
+                extras/init.d/glusterd.plist
+                extras/init.d/glusterd-Debian
+                extras/init.d/glusterd-Redhat
+                extras/init.d/glusterd-SuSE
+                extras/systemd/Makefile
+                extras/systemd/glusterd.service
+                extras/benchmarking/Makefile
+                extras/hook-scripts/Makefile
+                extras/ocf/Makefile
+                extras/ocf/glusterd
+                extras/ocf/volume
+                extras/LinuxRPM/Makefile
+                extras/geo-rep/Makefile
+                contrib/fuse-util/Makefile
+                contrib/uuid/uuid_types.h
+                glusterfs-api.pc
+                libgfchangelog.pc
+                api/Makefile
+                api/src/Makefile
+                api/examples/Makefile
+                api/examples/setup.py
+		geo-replication/Makefile
+		geo-replication/src/Makefile
+		geo-replication/syncdaemon/Makefile
+                glusterfs.spec])
 
 AC_CANONICAL_HOST
 
 AC_PROG_CC
+AC_DISABLE_STATIC
 AC_PROG_LIBTOOL
 
+AC_ARG_WITH(pkgconfigdir,
+            [  --with-pkgconfigdir=DIR      pkgconfig file in DIR @<:@LIBDIR/pkgconfig@:>@],
+            [pkgconfigdir=$withval],
+            [pkgconfigdir='${libdir}/pkgconfig'])
+AC_SUBST(pkgconfigdir)
+
 AC_ARG_WITH(mountutildir,
             [  --with-mountutildir=DIR mount helper utility in DIR @<:@/sbin@:>@],
             [mountutildir=$withval],
             [mountutildir='/sbin'])
 AC_SUBST(mountutildir)
 
+AC_ARG_WITH(systemddir,
+            [  --with-systemddir=DIR systemd service files in DIR @<:@/usr/lib/systemd/system@:>@],
+            [systemddir=$withval],
+            [systemddir='/usr/lib/systemd/system'])
+AC_SUBST(systemddir)
+
 AC_ARG_WITH(initdir,
             [  --with-initdir=DIR init.d scripts in DIR @<:@/etc/init.d@:>@],
             [initdir=$withval],
@@ -168,6 +211,13 @@ AC_ARG_WITH(launchddir,
             [launchddir='/Library/LaunchDaemons'])
 AC_SUBST(launchddir)
 
+AC_ARG_WITH([ocf],
+            [AS_HELP_STRING([--without-ocf], [build OCF-compliant cluster resource agents])],
+            ,
+            [OCF_SUBDIR='ocf'],
+            )
+AC_SUBST(OCF_SUBDIR)
+
 # LEX needs a check
 AC_PROG_LEX
 if test  "x${LEX}" != "xflex" -a "x${FLEX}" != "xlex"; then
@@ -198,8 +248,10 @@ fi
 
 AC_CHECK_TOOL([LD],[ld])
 
+AC_CHECK_LIB([crypto], [MD5], , AC_MSG_ERROR([OpenSSL crypto library is required to build glusterfs]))
+
 AC_CHECK_LIB([pthread], [pthread_mutex_init], , AC_MSG_ERROR([Posix threads library is required to build glusterfs]))
-		 
+
 AC_CHECK_FUNC([dlopen], [has_dlopen=yes], AC_CHECK_LIB([dl], [dlopen], , AC_MSG_ERROR([Dynamic linking library required to build glusterfs])))
 
 
@@ -207,6 +259,10 @@ AC_CHECK_HEADERS([sys/xattr.h])
 
 AC_CHECK_HEADERS([sys/extattr.h])
 
+AC_CHECK_HEADERS([openssl/md5.h])
+
+AC_CHECK_HEADERS([linux/falloc.h])
+
 case $host_os in
   darwin*)
     if ! test "`/usr/bin/sw_vers | grep ProductVersion: | cut -f 2 | cut -d. -f2`" -ge 5; then
@@ -240,8 +296,8 @@ fi
 
 # FUSE section
 AC_ARG_ENABLE([fuse-client],
-	      AC_HELP_STRING([--disable-fuse-client],
-			     [Do not build the fuse client. NOTE: you cannot mount glusterfs without the client]))
+              AC_HELP_STRING([--disable-fuse-client],
+                             [Do not build the fuse client. NOTE: you cannot mount glusterfs without the client]))
 
 BUILD_FUSE_CLIENT=no
 if test "x$enable_fuse_client" != "xno"; then
@@ -249,60 +305,153 @@ if test "x$enable_fuse_client" != "xno"; then
    BUILD_FUSE_CLIENT="yes"
 fi
 
+AC_ARG_ENABLE([bd-xlator],
+              AC_HELP_STRING([--enable-bd-xlator], [Build BD xlator]))
+
+if test "x$enable_bd_xlator" != "xno"; then
+  AC_CHECK_LIB([lvm2app],
+               [lvm_init,lvm_lv_from_name],
+               [HAVE_BD_LIB="yes"],
+               [HAVE_BD_LIB="no"])
+
+if test "x$HAVE_BD_LIB" = "xyes"; then
+    # lvm_lv_from_name() has been made public with lvm2-2.02.79
+    AC_CHECK_DECLS(
+                  [lvm_lv_from_name],
+                  [NEED_LVM_LV_FROM_NAME_DECL="no"],
+                  [NEED_LVM_LV_FROM_NAME_DECL="yes"],
+                  [[#include <lvm2app.h>]])
+  fi
+fi
+
+if test "x$enable_bd_xlator" = "xyes" -a "x$HAVE_BD_LIB" = "xno"; then
+   echo "BD xlator requested but required lvm2 development library not found."
+   exit 1
+fi
+
+BUILD_BD_XLATOR=no
+if test "x${enable-bd-xlator}" != "xno" -a "x${HAVE_BD_LIB}" = "xyes"; then
+  BUILD_BD_XLATOR=yes
+  AC_DEFINE(HAVE_BD_XLATOR, 1, [define if lvm2app library found and bd xlator
+                            enabled])
+  if test "x$NEED_LVM_LV_FROM_NAME_DECL" = "xyes"; then
+    AC_DEFINE(NEED_LVM_LV_FROM_NAME_DECL, 1, [defined if lvm_lv_from_name()
+                  was not found in the lvm2app.h header, but can be linked])
+  fi
+fi
+
+AM_CONDITIONAL([ENABLE_BD_XLATOR], [test x$BUILD_BD_XLATOR = xyes])
+
+# start encryption/crypt section
+
+AC_CHECK_HEADERS([openssl/cmac.h], [have_cmac_h=yes], [have_cmac_h=no])
+
+AC_ARG_ENABLE([crypt-xlator],
+	AC_HELP_STRING([--enable-crypt-xlator], [Build crypt encryption xlator]))
+
+if test "x$enable_crypt_xlator" = "xyes" -a "x$have_cmac_h" = "xno"; then
+   echo "Encryption xlator requires OpenSSL with cmac.h"
+   exit 1
+fi
+
+BUILD_CRYPT_XLATOR=no
+if test "x$enable_crypt_xlator" != "xno" -a "x$have_cmac_h" = "xyes"; then
+   BUILD_CRYPT_XLATOR=yes
+   AC_DEFINE(HAVE_CRYPT_XLATOR, 1, [enable building crypt encryption xlator])
+fi
+
+AM_CONDITIONAL([ENABLE_CRYPT_XLATOR], [test x$BUILD_CRYPT_XLATOR = xyes])
+
 AC_SUBST(FUSE_CLIENT_SUBDIR)
 # end FUSE section
 
 
 # FUSERMOUNT section
 AC_ARG_ENABLE([fusermount],
-              AC_HELP_STRING([--enable-fusermount],
-                             [Build fusermount]))
+              AC_HELP_STRING([--disable-fusermount],
+                             [Use system's fusermount]))
 
-BUILD_FUSERMOUNT="no"
-if test "x$enable_fusermount" = "xyes"; then
-  FUSERMOUNT_SUBDIR="contrib/fuse-util"
-  BUILD_FUSERMOUNT="yes"
+BUILD_FUSERMOUNT="yes"
+if test "x$enable_fusermount" = "xno"; then
+  BUILD_FUSERMOUNT="no"
+else
   AC_DEFINE(GF_FUSERMOUNT, 1, [Use our own fusermount])
+  FUSERMOUNT_SUBDIR="contrib/fuse-util"
 fi
 
 AC_SUBST(FUSERMOUNT_SUBDIR)
 #end FUSERMOUNT section
 
+# QEMU_BLOCK section
+
+AC_ARG_ENABLE([qemu-block],
+                AC_HELP_STRING([--enable-qemu-block],
+                        [Build QEMU Block formats translator]))
+
+if test "x$enable_qemu_block" != "xno"; then
+  PKG_CHECK_MODULES([GLIB], [glib-2.0],
+                    [HAVE_GLIB_2="yes"],
+  		    [HAVE_GLIB_2="no"])
+fi
+
+if test "x$enable_qemu_block" = "xyes" -a "x$HAVE_GLIB_2" = "xno"; then
+   echo "QEMU Block formats translator requires libglib-2.0, but missing."
+   exit 1
+fi
+
+BUILD_QEMU_BLOCK=no
+if test "x${enable_qemu_block}" != "xno" -a "x${HAVE_GLIB_2}" = "xyes"; then
+  BUILD_QEMU_BLOCK=yes
+  AC_DEFINE(HAVE_QEMU_BLOCK, 1, [define if libglib-2.0 library found and QEMU
+                  	     Block translator enabled])
+fi
+
+AM_CONDITIONAL([ENABLE_QEMU_BLOCK], [test x$BUILD_QEMU_BLOCK = xyes])
+
+# end QEMU_BLOCK section
 
 # EPOLL section
 AC_ARG_ENABLE([epoll],
-	      AC_HELP_STRING([--disable-epoll],
-			     [Use poll instead of epoll.]))
+              AC_HELP_STRING([--disable-epoll],
+                             [Use poll instead of epoll.]))
 
 BUILD_EPOLL=no
 if test "x$enable_epoll" != "xno"; then
    AC_CHECK_HEADERS([sys/epoll.h],
                     [BUILD_EPOLL=yes],
-		    [BUILD_EPOLL=no])
+                    [BUILD_EPOLL=no])
 fi
 # end EPOLL section
 
 
 # IBVERBS section
 AC_ARG_ENABLE([ibverbs],
-	      AC_HELP_STRING([--disable-ibverbs],
-			     [Do not build the ibverbs transport]))
+              AC_HELP_STRING([--disable-ibverbs],
+                             [Do not build the ibverbs transport]))
 
 if test "x$enable_ibverbs" != "xno"; then
   AC_CHECK_LIB([ibverbs],
                [ibv_get_device_list],
-	       [HAVE_LIBIBVERBS="yes"],
-	       [HAVE_LIBIBVERBS="no"])
+               [HAVE_LIBIBVERBS="yes"],
+               [HAVE_LIBIBVERBS="no"])
+  AC_CHECK_LIB([rdmacm], [rdma_create_id], [HAVE_RDMACM="yes"], [HAVE_RDMACM="no"])
 fi
 
-if test "x$enable_ibverbs" = "xyes" -a "x$HAVE_LIBIBVERBS" = "xno"; then
-   echo "ibverbs requested but not found."
-   exit 1
+if test "x$enable_ibverbs" = "xyes"; then
+   if test "x$HAVE_LIBIBVERBS" = "xno"; then
+      echo "ibverbs-transport requested, but libibverbs is not present."
+      exit 1
+   fi
+
+   if test "x$HAVE_RDMACM" = "xno"; then
+      echo "ibverbs-transport requested, but librdmacm is not present."
+      exit 1
+   fi
 fi
 
 BUILD_RDMA=no
 BUILD_IBVERBS=no
-if test "x$enable_ibverbs" != "xno" -a "x$HAVE_LIBIBVERBS" = "xyes"; then
+if test "x$enable_ibverbs" != "xno" -a "x$HAVE_LIBIBVERBS" = "xyes" -a "x$HAVE_RDMACM" = "xyes"; then
   IBVERBS_SUBDIR=ib-verbs
   BUILD_IBVERBS=yes
   RDMA_SUBDIR=rdma
@@ -316,8 +465,8 @@ AC_SUBST(RDMA_SUBDIR)
 
 # SYNCDAEMON section
 AC_ARG_ENABLE([georeplication],
-	      AC_HELP_STRING([--disable-georeplication],
-			     [Do not install georeplication components]))
+              AC_HELP_STRING([--disable-georeplication],
+                             [Do not install georeplication components]))
 
 BUILD_SYNCDAEMON=no
 case $host_os in
@@ -329,12 +478,12 @@ case $host_os in
        ;;
      *)
 #disabling geo replication for non-linux platforms
-	enable_georeplication=no
+        enable_georeplication=no
         ;;
 esac
 SYNCDAEMON_COMPILE=0
 if test "x$enable_georeplication" != "xno"; then
-  SYNCDAEMON_SUBDIR=utils
+  SYNCDAEMON_SUBDIR=geo-replication
   SYNCDAEMON_COMPILE=1
 
   BUILD_SYNCDAEMON="yes"
@@ -358,15 +507,67 @@ AC_SUBST(SYNCDAEMON_COMPILE)
 AC_SUBST(SYNCDAEMON_SUBDIR)
 # end SYNCDAEMON section
 
-#check if libxml is present if so enable HAVE_LIB_XML
-echo -n "checking if libxml2 is present... "
+# CDC xlator - check if libz is present if so enable HAVE_LIB_Z
+echo -n "checking if libz is present... "
 
-PKG_CHECK_MODULES([LIBXML2], [libxml-2.0 >= 2.6.19],
-                  [echo "yes (features requiring libxml2 enabled)"  AC_DEFINE(HAVE_LIB_XML, 1, [define if libxml2 is present])],
+PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.0],
+                  [echo "yes (features requiring zlib enabled)"  AC_DEFINE(HAVE_LIB_Z, 1, [define if zlib is present])],
                   [echo "no"] )
 
-AC_SUBST(LIBXML2_CFLAGS)
-AC_SUBST(LIBXML2_LIBS)
+AC_SUBST(LIBZ_CFLAGS)
+AC_SUBST(LIBZ_LIBS)
+# end CDC xlator secion
+
+# check for systemtap/dtrace
+BUILD_SYSTEMTAP=no
+AC_MSG_CHECKING([whether to include systemtap tracing support])
+AC_ARG_ENABLE([systemtap],
+              [AS_HELP_STRING([--enable-systemtap],
+              [Enable inclusion of systemtap trace support])],
+              [ENABLE_SYSTEMTAP="${enableval}"], [ENABLE_SYSTEMTAP="def"])
+
+AM_CONDITIONAL([ENABLE_SYSTEMTAP], [test "x${ENABLE_SYSTEMTAP}" = "xyes"])
+AC_MSG_RESULT(${ENABLE_SYSTEMTAP})
+
+if test "x${ENABLE_SYSTEMTAP}" != "xno"; then
+    AC_CHECK_PROG(DTRACE, dtrace, "yes", "no")
+    AC_CHECK_HEADER([sys/sdt.h], [SDT_H_FOUND="yes"],
+                    [SDT_H_FOUND="no"])
+fi
+
+if test "x${ENABLE_SYSTEMTAP}" = "xyes"; then
+    if test "x${DTRACE}" = "xno"; then
+        AC_MSG_ERROR([dtrace not found])
+    elif test "$x{SDT_H_FOUND}" = "xno"; then
+        AC_MSG_ERROR([systemtap support needs sys/sdt.h header])
+    fi
+fi
+
+if test "x${DTRACE}" = "xyes" -a "x${SDT_H_FOUND}" = "xyes"; then
+    AC_MSG_CHECKING([x"${DTRACE}"xy"${SDT_H_FOUND}"y])
+    AC_DEFINE([HAVE_SYSTEMTAP], [1], [Define to 1 if using  probes.])
+    BUILD_SYSTEMTAP=yes
+fi
+# end of systemtap/dtrace
+
+# xml-output
+AC_ARG_ENABLE([xml-output],
+              AC_HELP_STRING([--disable-xml-output],
+                             [Disable the xml output]))
+BUILD_XML_OUTPUT="yes"
+if test "x$enable_xml_output" != "xno"; then
+    #check if libxml is present if so enable HAVE_LIB_XML
+    m4_ifdef([AM_PATH_XML2],[AM_PATH_XML2([2.6.19])], [no_xml=yes])
+    if test "x${no_xml}" = "x"; then
+        AC_DEFINE([HAVE_LIB_XML], [1], [Define to 1 if using libxml2.])
+    else
+        AC_MSG_WARN([libxml2 devel libraries not found disabling XML support])
+        BUILD_XML_OUTPUT="no"
+    fi
+else
+    BUILD_XML_OUTPUT="no"
+fi
+# end of xml-output
 
 dnl FreeBSD > 5 has execinfo as a Ported library for giving a workaround
 dnl solution to GCC backtrace functionality
@@ -391,17 +592,26 @@ dnl Linux, Solaris, Cygwin
 AC_CHECK_MEMBERS([struct stat.st_atim.tv_nsec])
 dnl FreeBSD, NetBSD
 AC_CHECK_MEMBERS([struct stat.st_atimespec.tv_nsec])
+case $host_os in
+        *netbsd*)
+        CFLAGS+=" -D_INCOMPLETE_XOPEN_C063"
+        ;;
+esac
 AC_CHECK_FUNC([linkat], [have_linkat=yes])
 if test "x${have_linkat}" = "xyes"; then
    AC_DEFINE(HAVE_LINKAT, 1, [define if found linkat])
 fi
 AC_SUBST(HAVE_LINKAT)
 
+dnl check for Monotonic clock
+AC_CHECK_FUNC([clock_gettime], [has_monotonic_clock=yes], AC_CHECK_LIB([rt], [clock_gettime], , AC_MSG_WARN([System doesn't have monotonic clock using contrib])))
+
 dnl Check for argp
 AC_CHECK_HEADER([argp.h], AC_DEFINE(HAVE_ARGP, 1, [have argp]))
 AC_CONFIG_SUBDIRS(argp-standalone)
+
 BUILD_ARGP_STANDALONE=no
-if test "x${ac_cv_header_argp_h}" = "xno"; then 
+if test "x${ac_cv_header_argp_h}" = "xno"; then
    BUILD_ARGP_STANDALONE=yes
    ARGP_STANDALONE_CPPFLAGS='-I${top_srcdir}/argp-standalone'
    ARGP_STANDALONE_LDADD='${top_builddir}/argp-standalone/libargp.a'
@@ -422,7 +632,18 @@ if test "x${have_fdatasync}" = "xyes"; then
    AC_DEFINE(HAVE_FDATASYNC, 1, [define if fdatasync exists])
 fi
 
-# Check the distribution where you are compiling glusterfs on 
+AC_CHECK_FUNC([fallocate], [have_fallocate=yes])
+if test "x${have_fallocate}" = "xyes"; then
+   AC_DEFINE(HAVE_FALLOCATE, 1, [define if fallocate exists])
+fi
+
+AC_CHECK_FUNC([posix_fallocate], [have_posix_fallocate=yes])
+if test "x${have_posix_fallocate}" = "xyes"; then
+   AC_DEFINE(HAVE_POSIX_FALLOCATE, 1, [define if posix_fallocate exists])
+fi
+
+
+# Check the distribution where you are compiling glusterfs on
 
 GF_DISTRIBUTION=
 AC_CHECK_FILE([/etc/debian_version])
@@ -444,56 +665,115 @@ AC_SUBST(GF_DISTRIBUTION)
 GF_HOST_OS=""
 GF_LDFLAGS="-rdynamic"
 
+# check for gcc -Werror=format-security
+saved_CFLAGS=$CFLAGS
+CFLAGS="-Wformat -Werror=format-security"
+AC_MSG_CHECKING([whether $CC accepts -Werror=format-security])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [cc_werror_format_security=yes], [cc_werror_format_security=no])
+echo $cc_werror_format_security
+if test "x$cc_werror_format_security" = "xno"; then
+    CFLAGS="$saved_CFLAGS"
+else
+    CFLAGS="$saved_CFLAGS $CFLAGS"
+fi
+
+# check for gcc -Werror=implicit-function-declaration
+saved_CFLAGS=$CFLAGS
+CFLAGS="-Werror=implicit-function-declaration"
+AC_MSG_CHECKING([whether $CC accepts -Werror=implicit-function-declaration])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [cc_werror_implicit=yes], [cc_werror_implicit=no])
+echo $cc_werror_implicit
+if test "x$cc_werror_implicit" = "xno"; then
+    CFLAGS="$saved_CFLAGS"
+else
+    CFLAGS="$saved_CFLAGS $CFLAGS"
+fi
+
 case $host_os in
      linux*)
-     dnl	GF_LINUX_HOST_OS=1
         GF_HOST_OS="GF_LINUX_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -O0"
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(bindir)\\\""
-	;;
+        GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -O0"
+        GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
+        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(bindir)\\\""
+        ;;
      solaris*)
         GF_HOST_OS="GF_SOLARIS_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -O0 -m64"
-	GF_LDFLAGS=""
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	GF_GLUSTERFS_LDFLAGS="-lnsl -lresolv -lsocket"
+        GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -O0 -m64"
+        GF_LDFLAGS=""
+        GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
+        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        GF_GLUSTERFS_LIBS="-lnsl -lresolv -lsocket"
         BUILD_FUSE_CLIENT=no
         FUSE_CLIENT_SUBDIR=""
-	;;
+        ;;
      *netbsd*)
-	GF_HOST_OS="GF_BSD_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D_INCOMPLETE_XOPEN_C063"
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	if test "x$ac_cv_header_execinfo_h" = "xyes"; then
-	   GF_GLUSTERFS_LDFLAGS="-lexecinfo"
-	fi
-	GF_FUSE_LDADD="-liconv -lperfuse"
-	BUILD_FUSE_CLIENT=no
-	;;
+        GF_HOST_OS="GF_BSD_HOST_OS"
+        GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D_INCOMPLETE_XOPEN_C063"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
+        GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
+        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        if test "x$ac_cv_header_execinfo_h" = "xyes"; then
+           GF_GLUSTERFS_LIBS="-lexecinfo"
+        fi
+        GF_FUSE_LDADD="-lperfuse"
+        BUILD_FUSE_CLIENT=yes
+        LEXLIB=""
+        ;;
      *bsd*)
         GF_HOST_OS="GF_BSD_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -O0"
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	if test "x$ac_cv_header_execinfo_h" = "xyes"; then
-	   GF_GLUSTERFS_LDFLAGS="-lexecinfo"
-	fi				      
-	BUILD_FUSE_CLIENT=no
-	;;
+        GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -O0"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
+        GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
+        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        if test "x$ac_cv_header_execinfo_h" = "xyes"; then
+           GF_GLUSTERFS_LIBS="-lexecinfo"
+        fi
+        BUILD_FUSE_CLIENT=no
+        ;;
      darwin*)
         GF_HOST_OS="GF_DARWIN_HOST_OS"
-	LIBTOOL=glibtool
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D__DARWIN_64_BIT_INO_T -bundle -undefined suppress -flat_namespace -D_XOPEN_SOURCE -O0"
-	GF_GLUSTERFS_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D__DARWIN_64_BIT_INO_T -undefined suppress -flat_namespace -O0"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	GF_FUSE_CFLAGS="-I\$(CONTRIBDIR)/macfuse"
-	;;
+        LIBTOOL=glibtool
+        GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D__DARWIN_64_BIT_INO_T -bundle -undefined suppress -flat_namespace -D_XOPEN_SOURCE -O0"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
+        GF_GLUSTERFS_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D__DARWIN_64_BIT_INO_T -undefined suppress -flat_namespace -O0"
+        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        GF_FUSE_CFLAGS="-I\$(CONTRIBDIR)/macfuse"
+        ;;
 esac
 
+# enable debug section
+AC_ARG_ENABLE([debug],
+              AC_HELP_STRING([--enable-debug],
+                             [Enable debug build options.]))
+
+BUILD_DEBUG=no
+if test "x$enable_debug" = "xyes"; then
+   BUILD_DEBUG=yes
+   CFLAGS=`echo $CFLAGS | sed -e s/O2/O0/`
+else
+   BUILD_DEBUG=no
+fi
+AC_SUBST(CFLAGS)
+# end enable debug section
+
+# syslog section
+AC_ARG_ENABLE([syslog],
+	      AC_HELP_STRING([--disable-syslog],
+			     [Disable syslog for logging]))
+
+USE_SYSLOG="yes"
+if test "x$enable_syslog" != "xno"; then
+  AC_DEFINE(GF_USE_SYSLOG, 1, [Use syslog for logging])
+else
+  USE_SYSLOG="no"
+fi
+AM_CONDITIONAL([ENABLE_SYSLOG], [test x$USE_SYSLOG = xyes])
+#end syslog section
+
 BUILD_READLINE=no
 AC_CHECK_LIB([readline -lcurses],[readline],[RLLIBS="-lreadline -lcurses"])
 AC_CHECK_LIB([readline -ltermcap],[readline],[RLLIBS="-lreadline -ltermcap"])
@@ -504,8 +784,66 @@ if test "x$RLLIBS" != "x"; then
    BUILD_READLINE=yes
 fi
 
+BUILD_LIBAIO=no
+AC_CHECK_LIB([aio],[io_setup],[LIBAIO="-laio"])
+
+if test "x$LIBAIO" != "x"; then
+   AC_DEFINE(HAVE_LIBAIO, 1, [libaio based POSIX enabled])
+   BUILD_LIBAIO=yes
+fi
+
+# glupy section
+BUILD_GLUPY=no
+have_python2=no
+have_Python_h=no
+
+AM_PATH_PYTHON()
+if echo $PYTHON_VERSION | grep ^2; then
+  have_python2=yes
+fi
+AC_CHECK_HEADERS([python$PYTHON_VERSION/Python.h],[have_Python_h=yes],[])
+AC_ARG_ENABLE([glupy],
+              AS_HELP_STRING([--enable-glupy],
+                             [build glupy]))
+case x$enable_glupy in
+   xyes)
+      if test "x$have_python2" = "xyes" -a "x$have_Python_h" = "xyes"; then
+         BUILD_GLUPY=yes
+      else
+         AC_MSG_ERROR([glupy requires python-devel/python-dev package and python2.x])
+      fi
+      ;;
+   xno)
+      ;;
+   *)
+      if test "x$have_python2" = "xyes" -a "x$have_Python_h" = "xyes"; then
+         BUILD_GLUPY=yes
+      else
+         AC_MSG_WARN([
+        ---------------------------------------------------------------------------------
+         cannot build glupy. python 2.x and python-devel/python-dev package are required.
+        ---------------------------------------------------------------------------------])
+      fi
+      ;;
+esac
+
+if test "x$BUILD_GLUPY" = "xyes"; then
+   BUILD_PYTHON_INC=`$PYTHON -c "from distutils import sysconfig; print sysconfig.get_python_inc()"`
+   BUILD_PYTHON_LIB=python$PYTHON_VERSION
+   GLUPY_SUBDIR=glupy
+   GLUPY_SUBDIR_MAKEFILE=xlators/features/glupy/Makefile
+   GLUPY_SUBDIR_SRC_MAKEFILE=xlators/features/glupy/src/Makefile
+   echo "building glupy with -isystem $BUILD_PYTHON_INC -l $BUILD_PYTHON_LIB"
+   AC_SUBST(BUILD_PYTHON_INC)
+   AC_SUBST(BUILD_PYTHON_LIB)
+   AC_SUBST(GLUPY_SUBDIR)
+   AC_SUBST(GLUPY_SUBDIR_MAKEFILE)
+   AC_SUBST(GLUPY_SUBDIR_SRC_MAKEFILE)
+fi
+# end glupy section
+
 AC_SUBST(GF_HOST_OS)
-AC_SUBST(GF_GLUSTERFS_LDFLAGS)
+AC_SUBST([GF_GLUSTERFS_LIBS])
 AC_SUBST(GF_GLUSTERFS_CFLAGS)
 AC_SUBST(GF_CFLAGS)
 AC_SUBST(GF_LDFLAGS)
@@ -513,27 +851,41 @@ AC_SUBST(GF_LDADD)
 AC_SUBST(GF_FUSE_LDADD)
 AC_SUBST(GF_FUSE_CFLAGS)
 AC_SUBST(RLLIBS)
+AC_SUBST(LIBAIO)
 AC_SUBST(AM_MAKEFLAGS)
 AC_SUBST(AM_LIBTOOLFLAGS)
 
 CONTRIBDIR='$(top_srcdir)/contrib'
 AC_SUBST(CONTRIBDIR)
 
-INCLUDES='-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/uuid'
-AC_SUBST(INCLUDES)
+GF_CPPDEFINES='-D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)'
+GF_CPPINCLUDES='-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/uuid'
+GF_CPPFLAGS="$GF_CPPDEFINES $GF_CPPINCLUDES"
+AC_SUBST([GF_CPPFLAGS])
+
+AM_CONDITIONAL([GF_DARWIN_HOST_OS], test "${GF_HOST_OS}" = "GF_DARWIN_HOST_OS")
 
-AM_CONDITIONAL([GF_DARWIN_HOST_OS], test "${GF_HOST_OS}" = "GF_DARWIN_HOST_OS")	
+AM_CONDITIONAL([GF_INSTALL_VAR_LIB_GLUSTERD], test ! -d ${localstatedir}/lib/glusterd && test -d ${sysconfdir}/glusterd )
 
 AC_OUTPUT
 
 echo
 echo "GlusterFS configure summary"
 echo "==========================="
-echo "FUSE client        : $BUILD_FUSE_CLIENT"
-echo "Infiniband verbs   : $BUILD_IBVERBS"
-echo "epoll IO multiplex : $BUILD_EPOLL"
-echo "argp-standalone    : $BUILD_ARGP_STANDALONE"
-echo "fusermount         : $BUILD_FUSERMOUNT"
-echo "readline           : $BUILD_READLINE"
-echo "georeplication     : $BUILD_SYNCDAEMON"
+echo "FUSE client          : $BUILD_FUSE_CLIENT"
+echo "Infiniband verbs     : $BUILD_IBVERBS"
+echo "epoll IO multiplex   : $BUILD_EPOLL"
+echo "argp-standalone      : $BUILD_ARGP_STANDALONE"
+echo "fusermount           : $BUILD_FUSERMOUNT"
+echo "readline             : $BUILD_READLINE"
+echo "georeplication       : $BUILD_SYNCDAEMON"
+echo "Linux-AIO            : $BUILD_LIBAIO"
+echo "Enable Debug         : $BUILD_DEBUG"
+echo "systemtap            : $BUILD_SYSTEMTAP"
+echo "Block Device xlator  : $BUILD_BD_XLATOR"
+echo "glupy                : $BUILD_GLUPY"
+echo "Use syslog           : $USE_SYSLOG"
+echo "XML output           : $BUILD_XML_OUTPUT"
+echo "QEMU Block formats   : $BUILD_QEMU_BLOCK"
+echo "Encryption xlator    : $BUILD_CRYPT_XLATOR"
 echo
diff --git a/contrib/aclocal/mkdirp.m4 b/contrib/aclocal/mkdirp.m4
new file mode 100644
index 000000000..d2f7edd5c
--- /dev/null
+++ b/contrib/aclocal/mkdirp.m4
@@ -0,0 +1,146 @@
+# Excerpt from autoconf/autoconf/programs.m4
+# This file is part of Autoconf.                       -*- Autoconf -*-
+# Checking for programs.
+
+# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
+# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software
+# Foundation, Inc.
+
+# This file is part of Autoconf.  This program is free
+# software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# Under Section 7 of GPL version 3, you are granted additional
+# permissions described in the Autoconf Configure Script Exception,
+# version 3.0, as published by the Free Software Foundation.
+#
+# You should have received a copy of the GNU General Public License
+# and a copy of the Autoconf Configure Script Exception along with
+# this program; see the files COPYINGv3 and COPYING.EXCEPTION
+# respectively.  If not, see <http://www.gnu.org/licenses/>.
+
+# Written by David MacKenzie, with help from
+# Franc,ois Pinard, Karl Berry, Richard Pixley, Ian Lance Taylor,
+# Roland McGrath, Noah Friedman, david d zuhn, and many others.
+
+# AC_PROG_MKDIR_P
+# ---------------
+# Check whether `mkdir -p' is known to be thread-safe, and fall back to
+# install-sh -d otherwise.
+#
+# Automake 1.8 used `mkdir -m 0755 -p --' to ensure that directories
+# created by `make install' are always world readable, even if the
+# installer happens to have an overly restrictive umask (e.g. 077).
+# This was a mistake.  There are at least two reasons why we must not
+# use `-m 0755':
+#   - it causes special bits like SGID to be ignored,
+#   - it may be too restrictive (some setups expect 775 directories).
+#
+# Do not use -m 0755 and let people choose whatever they expect by
+# setting umask.
+#
+# We cannot accept any implementation of `mkdir' that recognizes `-p'.
+# Some implementations (such as Solaris 8's) are vulnerable to race conditions:
+# if a parallel make tries to run `mkdir -p a/b' and `mkdir -p a/c'
+# concurrently, both version can detect that a/ is missing, but only
+# one can create it and the other will error out.  Consequently we
+# restrict ourselves to known race-free implementations.
+#
+# Automake used to define mkdir_p as `mkdir -p .', in order to
+# allow $(mkdir_p) to be used without argument.  As in
+#   $(mkdir_p) $(somedir)
+# where $(somedir) is conditionally defined.  However we don't do
+# that for MKDIR_P.
+#  1. before we restricted the check to GNU mkdir, `mkdir -p .' was
+#     reported to fail in read-only directories.  The system where this
+#     happened has been forgotten.
+#  2. in practice we call $(MKDIR_P) on directories such as
+#       $(MKDIR_P) "$(DESTDIR)$(somedir)"
+#     and we don't want to create $(DESTDIR) if $(somedir) is empty.
+#     To support the latter case, we have to write
+#       test -z "$(somedir)" || $(MKDIR_P) "$(DESTDIR)$(somedir)"
+#     so $(MKDIR_P) always has an argument.
+#     We will have better chances of detecting a missing test if
+#     $(MKDIR_P) complains about missing arguments.
+#  3. $(MKDIR_P) is named after `mkdir -p' and we don't expect this
+#     to accept no argument.
+#  4. having something like `mkdir .' in the output is unsightly.
+#
+# On NextStep and OpenStep, the `mkdir' command does not
+# recognize any option.  It will interpret all options as
+# directories to create.
+AN_MAKEVAR([MKDIR_P], [AC_PROG_MKDIR_P])
+AC_DEFUN_ONCE([AC_PROG_MKDIR_P],
+[AC_REQUIRE([AC_CONFIG_AUX_DIR_DEFAULT])dnl
+
+AC_MSG_CHECKING([for a thread-safe mkdir -p])
+if test -z "$MKDIR_P"; then
+  AC_CACHE_VAL([ac_cv_path_mkdir],
+    [_AS_PATH_WALK([$PATH$PATH_SEPARATOR/opt/sfw/bin],
+      [for ac_prog in mkdir gmkdir; do
+	 for ac_exec_ext in '' $ac_executable_extensions; do
+	   AS_EXECUTABLE_P(["$as_dir/$ac_prog$ac_exec_ext"]) || continue
+	   case `"$as_dir/$ac_prog$ac_exec_ext" --version 2>&1` in #(
+	     'mkdir (GNU coreutils) '* | \
+	     'mkdir (coreutils) '* | \
+	     'mkdir (fileutils) '4.1*)
+	       ac_cv_path_mkdir=$as_dir/$ac_prog$ac_exec_ext
+	       break 3;;
+	   esac
+	 done
+       done])])
+  test -d ./--version && rmdir ./--version
+  if test "${ac_cv_path_mkdir+set}" = set; then
+    MKDIR_P="$ac_cv_path_mkdir -p"
+  else
+    # As a last resort, use the slow shell script.  Don't cache a
+    # value for MKDIR_P within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the value is a relative name.
+    MKDIR_P="$ac_install_sh -d"
+  fi
+fi
+dnl status.m4 does special magic for MKDIR_P instead of AC_SUBST,
+dnl to get relative names right.  However, also AC_SUBST here so
+dnl that Automake versions before 1.10 will pick it up (they do not
+dnl trace AC_SUBST_TRACE).
+dnl FIXME: Remove this once we drop support for Automake < 1.10.
+AC_SUBST([MKDIR_P])dnl
+AC_MSG_RESULT([$MKDIR_P])
+])# AC_PROG_MKDIR_P
+
+
+# From automake/m4/mkdirp.m4
+##                                                          -*- Autoconf -*-
+# Copyright (C) 2003, 2004, 2005, 2006  Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_PROG_MKDIR_P
+# ---------------
+# Check for `mkdir -p'.
+AC_DEFUN([AM_PROG_MKDIR_P],
+[
+AC_REQUIRE([AC_PROG_MKDIR_P])dnl
+dnl Automake 1.8 to 1.9.6 used to define mkdir_p.  We now use MKDIR_P,
+dnl while keeping a definition of mkdir_p for backward compatibility.
+dnl @MKDIR_P@ is magic: AC_OUTPUT adjusts its value for each Makefile.
+dnl However we cannot define mkdir_p as $(MKDIR_P) for the sake of
+dnl Makefile.ins that do not define MKDIR_P, so we do our own
+dnl adjustment using top_builddir (which is defined more often than
+dnl MKDIR_P).
+AC_SUBST([mkdir_p], ["$MKDIR_P"])dnl
+case $mkdir_p in
+  [[\\/$]]* | ?:[[\\/]]*) ;;
+  */*) mkdir_p="\$(top_builddir)/$mkdir_p" ;;
+esac
+])
diff --git a/contrib/aclocal/python.m4 b/contrib/aclocal/python.m4
new file mode 100644
index 000000000..a39a90090
--- /dev/null
+++ b/contrib/aclocal/python.m4
@@ -0,0 +1,209 @@
+## ------------------------                                 -*- Autoconf -*-
+## Python file handling
+## From Andrew Dalke
+## Updated by James Henstridge
+## ------------------------
+# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2008, 2009
+# Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_PATH_PYTHON([MINIMUM-VERSION], [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+# ---------------------------------------------------------------------------
+# Adds support for distributing Python modules and packages.  To
+# install modules, copy them to $(pythondir), using the python_PYTHON
+# automake variable.  To install a package with the same name as the
+# automake package, install to $(pkgpythondir), or use the
+# pkgpython_PYTHON automake variable.
+#
+# The variables $(pyexecdir) and $(pkgpyexecdir) are provided as
+# locations to install python extension modules (shared libraries).
+# Another macro is required to find the appropriate flags to compile
+# extension modules.
+#
+# If your package is configured with a different prefix to python,
+# users will have to add the install directory to the PYTHONPATH
+# environment variable, or create a .pth file (see the python
+# documentation for details).
+#
+# If the MINIMUM-VERSION argument is passed, AM_PATH_PYTHON will
+# cause an error if the version of python installed on the system
+# doesn't meet the requirement.  MINIMUM-VERSION should consist of
+# numbers and dots only.
+AC_DEFUN([AM_PATH_PYTHON],
+ [
+  dnl Find a Python interpreter.  Python versions prior to 2.0 are not
+  dnl supported. (2.0 was released on October 16, 2000).
+  m4_define_default([_AM_PYTHON_INTERPRETER_LIST],
+                    [python python2 python3 python3.2 python3.1 python3.0 python2.7 python2.6 python2.5 python2.4 python2.3 python2.2 dnl
+python2.1 python2.0])
+
+  m4_if([$1],[],[
+    dnl No version check is needed.
+    # Find any Python interpreter.
+    if test -z "$PYTHON"; then
+      AC_PATH_PROGS([PYTHON], _AM_PYTHON_INTERPRETER_LIST, :)
+    fi
+    am_display_PYTHON=python
+  ], [
+    dnl A version check is needed.
+    if test -n "$PYTHON"; then
+      # If the user set $PYTHON, use it and don't search something else.
+      AC_MSG_CHECKING([whether $PYTHON version >= $1])
+      AM_PYTHON_CHECK_VERSION([$PYTHON], [$1],
+			      [AC_MSG_RESULT(yes)],
+			      [AC_MSG_ERROR(too old)])
+      am_display_PYTHON=$PYTHON
+    else
+      # Otherwise, try each interpreter until we find one that satisfies
+      # VERSION.
+      AC_CACHE_CHECK([for a Python interpreter with version >= $1],
+	[am_cv_pathless_PYTHON],[
+	for am_cv_pathless_PYTHON in _AM_PYTHON_INTERPRETER_LIST none; do
+	  test "$am_cv_pathless_PYTHON" = none && break
+	  AM_PYTHON_CHECK_VERSION([$am_cv_pathless_PYTHON], [$1], [break])
+	done])
+      # Set $PYTHON to the absolute path of $am_cv_pathless_PYTHON.
+      if test "$am_cv_pathless_PYTHON" = none; then
+	PYTHON=:
+      else
+        AC_PATH_PROG([PYTHON], [$am_cv_pathless_PYTHON])
+      fi
+      am_display_PYTHON=$am_cv_pathless_PYTHON
+    fi
+  ])
+
+  if test "$PYTHON" = :; then
+  dnl Run any user-specified action, or abort.
+    m4_default([$3], [AC_MSG_ERROR([no suitable Python interpreter found])])
+  else
+
+  dnl Query Python for its version number.  Getting [:3] seems to be
+  dnl the best way to do this; it's what "site.py" does in the standard
+  dnl library.
+
+  AC_CACHE_CHECK([for $am_display_PYTHON version], [am_cv_python_version],
+    [am_cv_python_version=`$PYTHON -c "import sys; sys.stdout.write(sys.version[[:3]])"`])
+  AC_SUBST([PYTHON_VERSION], [$am_cv_python_version])
+
+  dnl Use the values of $prefix and $exec_prefix for the corresponding
+  dnl values of PYTHON_PREFIX and PYTHON_EXEC_PREFIX.  These are made
+  dnl distinct variables so they can be overridden if need be.  However,
+  dnl general consensus is that you shouldn't need this ability.
+
+  AC_SUBST([PYTHON_PREFIX], ['${prefix}'])
+  AC_SUBST([PYTHON_EXEC_PREFIX], ['${exec_prefix}'])
+
+  dnl At times (like when building shared libraries) you may want
+  dnl to know which OS platform Python thinks this is.
+
+  AC_CACHE_CHECK([for $am_display_PYTHON platform], [am_cv_python_platform],
+    [am_cv_python_platform=`$PYTHON -c "import sys; sys.stdout.write(sys.platform)"`])
+  AC_SUBST([PYTHON_PLATFORM], [$am_cv_python_platform])
+
+
+  dnl Set up 4 directories:
+
+  dnl pythondir -- where to install python scripts.  This is the
+  dnl   site-packages directory, not the python standard library
+  dnl   directory like in previous automake betas.  This behavior
+  dnl   is more consistent with lispdir.m4 for example.
+  dnl Query distutils for this directory.  distutils does not exist in
+  dnl Python 1.5, so we fall back to the hardcoded directory if it
+  dnl doesn't work.
+  AC_CACHE_CHECK([for $am_display_PYTHON script directory],
+    [am_cv_python_pythondir],
+    [if test "x$prefix" = xNONE
+     then
+       am_py_prefix=$ac_default_prefix
+     else
+       am_py_prefix=$prefix
+     fi
+     am_cv_python_pythondir=`$PYTHON -c "import sys; from distutils import sysconfig; sys.stdout.write(sysconfig.get_python_lib(0,0,prefix='$am_py_prefix'))" 2>/dev/null ||
+     echo "$PYTHON_PREFIX/lib/python$PYTHON_VERSION/site-packages"`
+     case $am_cv_python_pythondir in
+     $am_py_prefix*)
+       am__strip_prefix=`echo "$am_py_prefix" | sed 's|.|.|g'`
+       am_cv_python_pythondir=`echo "$am_cv_python_pythondir" | sed "s,^$am__strip_prefix,$PYTHON_PREFIX,"`
+       ;;
+     *)
+       case $am_py_prefix in
+         /usr|/System*) ;;
+         *)
+	  am_cv_python_pythondir=$PYTHON_PREFIX/lib/python$PYTHON_VERSION/site-packages
+	  ;;
+       esac
+       ;;
+     esac
+    ])
+  AC_SUBST([pythondir], [$am_cv_python_pythondir])
+
+  dnl pkgpythondir -- $PACKAGE directory under pythondir.  Was
+  dnl   PYTHON_SITE_PACKAGE in previous betas, but this naming is
+  dnl   more consistent with the rest of automake.
+
+  AC_SUBST([pkgpythondir], [\${pythondir}/$PACKAGE])
+
+  dnl pyexecdir -- directory for installing python extension modules
+  dnl   (shared libraries)
+  dnl Query distutils for this directory.  distutils does not exist in
+  dnl Python 1.5, so we fall back to the hardcoded directory if it
+  dnl doesn't work.
+  AC_CACHE_CHECK([for $am_display_PYTHON extension module directory],
+    [am_cv_python_pyexecdir],
+    [if test "x$exec_prefix" = xNONE
+     then
+       am_py_exec_prefix=$am_py_prefix
+     else
+       am_py_exec_prefix=$exec_prefix
+     fi
+     am_cv_python_pyexecdir=`$PYTHON -c "import sys; from distutils import sysconfig; sys.stdout.write(sysconfig.get_python_lib(1,0,prefix='$am_py_exec_prefix'))" 2>/dev/null ||
+     echo "$PYTHON_EXEC_PREFIX/lib/python$PYTHON_VERSION/site-packages"`
+     case $am_cv_python_pyexecdir in
+     $am_py_exec_prefix*)
+       am__strip_prefix=`echo "$am_py_exec_prefix" | sed 's|.|.|g'`
+       am_cv_python_pyexecdir=`echo "$am_cv_python_pyexecdir" | sed "s,^$am__strip_prefix,$PYTHON_EXEC_PREFIX,"`
+       ;;
+     *)
+       case $am_py_exec_prefix in
+         /usr|/System*) ;;
+         *)
+	   am_cv_python_pyexecdir=$PYTHON_EXEC_PREFIX/lib/python$PYTHON_VERSION/site-packages
+	   ;;
+       esac
+       ;;
+     esac
+    ])
+  AC_SUBST([pyexecdir], [$am_cv_python_pyexecdir])
+
+  dnl pkgpyexecdir -- $(pyexecdir)/$(PACKAGE)
+
+  AC_SUBST([pkgpyexecdir], [\${pyexecdir}/$PACKAGE])
+
+  dnl Run any user-specified action.
+  $2
+  fi
+
+])
+
+
+# AM_PYTHON_CHECK_VERSION(PROG, VERSION, [ACTION-IF-TRUE], [ACTION-IF-FALSE])
+# ---------------------------------------------------------------------------
+# Run ACTION-IF-TRUE if the Python interpreter PROG has version >= VERSION.
+# Run ACTION-IF-FALSE otherwise.
+# This test uses sys.hexversion instead of the string equivalent (first
+# word of sys.version), in order to cope with versions such as 2.2c1.
+# This supports Python 2.0 or higher. (2.0 was released on October 16, 2000).
+AC_DEFUN([AM_PYTHON_CHECK_VERSION],
+ [prog="import sys
+# split strings by '.' and convert to numeric.  Append some zeros
+# because we need at least 4 digits for the hex conversion.
+# map returns an iterator in Python 3.0 and a list in 2.x
+minver = list(map(int, '$2'.split('.'))) + [[0, 0, 0]]
+minverhex = 0
+# xrange is not present in Python 3.0 and range returns an iterator
+for i in list(range(0, 4)): minverhex = (minverhex << 8) + minver[[i]]
+sys.exit(sys.hexversion < minverhex)"
+  AS_IF([AM_RUN_LOG([$1 -c "$prog"])], [$3], [$4])])
diff --git a/contrib/fuse-include/fuse-mount.h b/contrib/fuse-include/fuse-mount.h
index 9f83faf02..9358ac810 100644
--- a/contrib/fuse-include/fuse-mount.h
+++ b/contrib/fuse-include/fuse-mount.h
@@ -8,5 +8,6 @@
 */
 
 void gf_fuse_unmount (const char *mountpoint, int fd);
-int gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
-                   pid_t *mtab_pid);
+int gf_fuse_mount (const char *mountpoint, char *fsname,
+                   unsigned long mountflags, char *mnt_param,
+                   pid_t *mtab_pid, int status_fd);
diff --git a/contrib/fuse-include/fuse_kernel.h b/contrib/fuse-include/fuse_kernel.h
index 9ae25d6f9..60bb2f9f7 100644
--- a/contrib/fuse-include/fuse_kernel.h
+++ b/contrib/fuse-include/fuse_kernel.h
@@ -60,23 +60,75 @@
  * 7.13
  *  - make max number of background requests and congestion threshold
  *    tunables
+ *
+ * 7.14
+ *  - add splice support to fuse device
+ *
+ * 7.15
+ *  - add store notify
+ *  - add retrieve notify
+ *
+ * 7.16
+ *  - add BATCH_FORGET request
+ *  - FUSE_IOCTL_UNRESTRICTED shall now return with array of 'struct
+ *    fuse_ioctl_iovec' instead of ambiguous 'struct iovec'
+ *  - add FUSE_IOCTL_32BIT flag
+ *
+ * 7.17
+ *  - add FUSE_FLOCK_LOCKS and FUSE_RELEASE_FLOCK_UNLOCK
+ *
+ * 7.18
+ *  - add FUSE_IOCTL_DIR flag
+ *  - add FUSE_NOTIFY_DELETE
+ *
+ * 7.19
+ *  - add FUSE_FALLOCATE
+ *
+ * 7.20
+ *  - add FUSE_AUTO_INVAL_DATA
+ *
+ * 7.21
+ *  - add FUSE_READDIRPLUS
+ *  - send the requested events in POLL request
+ *
+ * 7.22
+ *  - add FUSE_ASYNC_DIO
  */
 
 #ifndef _LINUX_FUSE_H
 #define _LINUX_FUSE_H
 
-#include <sys/types.h>
-#define __u64 uint64_t
-#define __s64 int64_t
-#define __u32 uint32_t
-#define __s32 int32_t
-#define __u16 uint16_t
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdint.h>
+#endif
+
+/*
+ * Version negotiation:
+ *
+ * Both the kernel and userspace send the version they support in the
+ * INIT request and reply respectively.
+ *
+ * If the major versions match then both shall use the smallest
+ * of the two minor versions for communication.
+ *
+ * If the kernel supports a larger major version, then userspace shall
+ * reply with the major version it supports, ignore the rest of the
+ * INIT message and expect a new INIT message from the kernel with a
+ * matching major version.
+ *
+ * If the library supports a larger major version, then it shall fall
+ * back to the major protocol version sent by the kernel for
+ * communication and reply with that major version (and an arbitrary
+ * supported minor version).
+ */
 
 /** Version number of this interface */
 #define FUSE_KERNEL_VERSION 7
 
 /** Minor version number of this interface */
-#define FUSE_KERNEL_MINOR_VERSION 13
+#define FUSE_KERNEL_MINOR_VERSION 22
 
 /** The node ID of the root inode */
 #define FUSE_ROOT_ID 1
@@ -85,42 +137,42 @@
    userspace works under 64bit kernels */
 
 struct fuse_attr {
-	__u64	ino;
-	__u64	size;
-	__u64	blocks;
-	__u64	atime;
-	__u64	mtime;
-	__u64	ctime;
-	__u32	atimensec;
-	__u32	mtimensec;
-	__u32	ctimensec;
-	__u32	mode;
-	__u32	nlink;
-	__u32	uid;
-	__u32	gid;
-	__u32	rdev;
-	__u32	blksize;
-	__u32	padding;
+	uint64_t	ino;
+	uint64_t	size;
+	uint64_t	blocks;
+	uint64_t	atime;
+	uint64_t	mtime;
+	uint64_t	ctime;
+	uint32_t	atimensec;
+	uint32_t	mtimensec;
+	uint32_t	ctimensec;
+	uint32_t	mode;
+	uint32_t	nlink;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	rdev;
+	uint32_t	blksize;
+	uint32_t	padding;
 };
 
 struct fuse_kstatfs {
-	__u64	blocks;
-	__u64	bfree;
-	__u64	bavail;
-	__u64	files;
-	__u64	ffree;
-	__u32	bsize;
-	__u32	namelen;
-	__u32	frsize;
-	__u32	padding;
-	__u32	spare[6];
+	uint64_t	blocks;
+	uint64_t	bfree;
+	uint64_t	bavail;
+	uint64_t	files;
+	uint64_t	ffree;
+	uint32_t	bsize;
+	uint32_t	namelen;
+	uint32_t	frsize;
+	uint32_t	padding;
+	uint32_t	spare[6];
 };
 
 struct fuse_file_lock {
-	__u64	start;
-	__u64	end;
-	__u32	type;
-	__u32	pid; /* tgid */
+	uint64_t	start;
+	uint64_t	end;
+	uint32_t	type;
+	uint32_t	pid; /* tgid */
 };
 
 /**
@@ -151,8 +203,22 @@ struct fuse_file_lock {
 /**
  * INIT request/reply flags
  *
+ * FUSE_ASYNC_READ: asynchronous read requests
+ * FUSE_POSIX_LOCKS: remote locking for POSIX file locks
+ * FUSE_FILE_OPS: kernel sends file handle for fstat, etc... (not yet supported)
+ * FUSE_ATOMIC_O_TRUNC: handles the O_TRUNC open flag in the filesystem
  * FUSE_EXPORT_SUPPORT: filesystem handles lookups of "." and ".."
+ * FUSE_BIG_WRITES: filesystem can handle write size larger than 4kB
  * FUSE_DONT_MASK: don't apply umask to file mode on create operations
+ * FUSE_SPLICE_WRITE: kernel supports splice write on the device
+ * FUSE_SPLICE_MOVE: kernel supports splice move on the device
+ * FUSE_SPLICE_READ: kernel supports splice read on the device
+ * FUSE_FLOCK_LOCKS: remote locking for BSD style file locks
+ * FUSE_HAS_IOCTL_DIR: kernel supports ioctl on directories
+ * FUSE_AUTO_INVAL_DATA: automatically invalidate cached pages
+ * FUSE_DO_READDIRPLUS: do READDIRPLUS (READDIR+LOOKUP in one)
+ * FUSE_READDIRPLUS_AUTO: adaptive readdirplus
+ * FUSE_ASYNC_DIO: asynchronous direct I/O submission
  */
 #define FUSE_ASYNC_READ		(1 << 0)
 #define FUSE_POSIX_LOCKS	(1 << 1)
@@ -161,6 +227,15 @@ struct fuse_file_lock {
 #define FUSE_EXPORT_SUPPORT	(1 << 4)
 #define FUSE_BIG_WRITES		(1 << 5)
 #define FUSE_DONT_MASK		(1 << 6)
+#define FUSE_SPLICE_WRITE	(1 << 7)
+#define FUSE_SPLICE_MOVE	(1 << 8)
+#define FUSE_SPLICE_READ	(1 << 9)
+#define FUSE_FLOCK_LOCKS	(1 << 10)
+#define FUSE_HAS_IOCTL_DIR	(1 << 11)
+#define FUSE_AUTO_INVAL_DATA	(1 << 12)
+#define FUSE_DO_READDIRPLUS	(1 << 13)
+#define FUSE_READDIRPLUS_AUTO	(1 << 14)
+#define FUSE_ASYNC_DIO		(1 << 15)
 
 /**
  * CUSE INIT request/reply flags
@@ -173,6 +248,7 @@ struct fuse_file_lock {
  * Release flags
  */
 #define FUSE_RELEASE_FLUSH	(1 << 0)
+#define FUSE_RELEASE_FLOCK_UNLOCK	(1 << 1)
 
 /**
  * Getattr flags
@@ -204,12 +280,16 @@ struct fuse_file_lock {
  * FUSE_IOCTL_COMPAT: 32bit compat ioctl on 64bit machine
  * FUSE_IOCTL_UNRESTRICTED: not restricted to well-formed ioctls, retry allowed
  * FUSE_IOCTL_RETRY: retry with new iovecs
+ * FUSE_IOCTL_32BIT: 32bit ioctl
+ * FUSE_IOCTL_DIR: is a directory
  *
  * FUSE_IOCTL_MAX_IOV: maximum of in_iovecs + out_iovecs
  */
 #define FUSE_IOCTL_COMPAT	(1 << 0)
 #define FUSE_IOCTL_UNRESTRICTED	(1 << 1)
 #define FUSE_IOCTL_RETRY	(1 << 2)
+#define FUSE_IOCTL_32BIT	(1 << 3)
+#define FUSE_IOCTL_DIR		(1 << 4)
 
 #define FUSE_IOCTL_MAX_IOV	256
 
@@ -259,6 +339,10 @@ enum fuse_opcode {
 	FUSE_DESTROY       = 38,
 	FUSE_IOCTL         = 39,
 	FUSE_POLL          = 40,
+	FUSE_NOTIFY_REPLY  = 41,
+	FUSE_BATCH_FORGET  = 42,
+	FUSE_FALLOCATE     = 43,
+	FUSE_READDIRPLUS   = 44,
 
 	/* CUSE specific operations */
 	CUSE_INIT          = 4096,
@@ -268,6 +352,9 @@ enum fuse_notify_code {
 	FUSE_NOTIFY_POLL   = 1,
 	FUSE_NOTIFY_INVAL_INODE = 2,
 	FUSE_NOTIFY_INVAL_ENTRY = 3,
+	FUSE_NOTIFY_STORE = 4,
+	FUSE_NOTIFY_RETRIEVE = 5,
+	FUSE_NOTIFY_DELETE = 6,
 	FUSE_NOTIFY_CODE_MAX,
 };
 
@@ -277,133 +364,143 @@ enum fuse_notify_code {
 #define FUSE_COMPAT_ENTRY_OUT_SIZE 120
 
 struct fuse_entry_out {
-	__u64	nodeid;		/* Inode ID */
-	__u64	generation;	/* Inode generation: nodeid:gen must
-				   be unique for the fs's lifetime */
-	__u64	entry_valid;	/* Cache timeout for the name */
-	__u64	attr_valid;	/* Cache timeout for the attributes */
-	__u32	entry_valid_nsec;
-	__u32	attr_valid_nsec;
+	uint64_t	nodeid;		/* Inode ID */
+	uint64_t	generation;	/* Inode generation: nodeid:gen must
+					   be unique for the fs's lifetime */
+	uint64_t	entry_valid;	/* Cache timeout for the name */
+	uint64_t	attr_valid;	/* Cache timeout for the attributes */
+	uint32_t	entry_valid_nsec;
+	uint32_t	attr_valid_nsec;
 	struct fuse_attr attr;
 };
 
 struct fuse_forget_in {
-	__u64	nlookup;
+	uint64_t	nlookup;
+};
+
+struct fuse_forget_one {
+	uint64_t	nodeid;
+	uint64_t	nlookup;
+};
+
+struct fuse_batch_forget_in {
+	uint32_t	count;
+	uint32_t	dummy;
 };
 
 struct fuse_getattr_in {
-	__u32	getattr_flags;
-	__u32	dummy;
-	__u64	fh;
+	uint32_t	getattr_flags;
+	uint32_t	dummy;
+	uint64_t	fh;
 };
 
 #define FUSE_COMPAT_ATTR_OUT_SIZE 96
 
 struct fuse_attr_out {
-	__u64	attr_valid;	/* Cache timeout for the attributes */
-	__u32	attr_valid_nsec;
-	__u32	dummy;
+	uint64_t	attr_valid;	/* Cache timeout for the attributes */
+	uint32_t	attr_valid_nsec;
+	uint32_t	dummy;
 	struct fuse_attr attr;
 };
 
 #define FUSE_COMPAT_MKNOD_IN_SIZE 8
 
 struct fuse_mknod_in {
-	__u32	mode;
-	__u32	rdev;
-	__u32	umask;
-	__u32	padding;
+	uint32_t	mode;
+	uint32_t	rdev;
+	uint32_t	umask;
+	uint32_t	padding;
 };
 
 struct fuse_mkdir_in {
-	__u32	mode;
-	__u32	umask;
+	uint32_t	mode;
+	uint32_t	umask;
 };
 
 struct fuse_rename_in {
-	__u64	newdir;
+	uint64_t	newdir;
 };
 
 struct fuse_link_in {
-	__u64	oldnodeid;
+	uint64_t	oldnodeid;
 };
 
 struct fuse_setattr_in {
-	__u32	valid;
-	__u32	padding;
-	__u64	fh;
-	__u64	size;
-	__u64	lock_owner;
-	__u64	atime;
-	__u64	mtime;
-	__u64	unused2;
-	__u32	atimensec;
-	__u32	mtimensec;
-	__u32	unused3;
-	__u32	mode;
-	__u32	unused4;
-	__u32	uid;
-	__u32	gid;
-	__u32	unused5;
+	uint32_t	valid;
+	uint32_t	padding;
+	uint64_t	fh;
+	uint64_t	size;
+	uint64_t	lock_owner;
+	uint64_t	atime;
+	uint64_t	mtime;
+	uint64_t	unused2;
+	uint32_t	atimensec;
+	uint32_t	mtimensec;
+	uint32_t	unused3;
+	uint32_t	mode;
+	uint32_t	unused4;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	unused5;
 };
 
 struct fuse_open_in {
-	__u32	flags;
-	__u32	unused;
+	uint32_t	flags;
+	uint32_t	unused;
 };
 
 struct fuse_create_in {
-	__u32	flags;
-	__u32	mode;
-	__u32	umask;
-	__u32	padding;
+	uint32_t	flags;
+	uint32_t	mode;
+	uint32_t	umask;
+	uint32_t	padding;
 };
 
 struct fuse_open_out {
-	__u64	fh;
-	__u32	open_flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint32_t	open_flags;
+	uint32_t	padding;
 };
 
 struct fuse_release_in {
-	__u64	fh;
-	__u32	flags;
-	__u32	release_flags;
-	__u64	lock_owner;
+	uint64_t	fh;
+	uint32_t	flags;
+	uint32_t	release_flags;
+	uint64_t	lock_owner;
 };
 
 struct fuse_flush_in {
-	__u64	fh;
-	__u32	unused;
-	__u32	padding;
-	__u64	lock_owner;
+	uint64_t	fh;
+	uint32_t	unused;
+	uint32_t	padding;
+	uint64_t	lock_owner;
 };
 
 struct fuse_read_in {
-	__u64	fh;
-	__u64	offset;
-	__u32	size;
-	__u32	read_flags;
-	__u64	lock_owner;
-	__u32	flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	read_flags;
+	uint64_t	lock_owner;
+	uint32_t	flags;
+	uint32_t	padding;
 };
 
 #define FUSE_COMPAT_WRITE_IN_SIZE 24
 
 struct fuse_write_in {
-	__u64	fh;
-	__u64	offset;
-	__u32	size;
-	__u32	write_flags;
-	__u64	lock_owner;
-	__u32	flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	write_flags;
+	uint64_t	lock_owner;
+	uint32_t	flags;
+	uint32_t	padding;
 };
 
 struct fuse_write_out {
-	__u32	size;
-	__u32	padding;
+	uint32_t	size;
+	uint32_t	padding;
 };
 
 #define FUSE_COMPAT_STATFS_SIZE 48
@@ -413,32 +510,32 @@ struct fuse_statfs_out {
 };
 
 struct fuse_fsync_in {
-	__u64	fh;
-	__u32	fsync_flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint32_t	fsync_flags;
+	uint32_t	padding;
 };
 
 struct fuse_setxattr_in {
-	__u32	size;
-	__u32	flags;
+	uint32_t	size;
+	uint32_t	flags;
 };
 
 struct fuse_getxattr_in {
-	__u32	size;
-	__u32	padding;
+	uint32_t	size;
+	uint32_t	padding;
 };
 
 struct fuse_getxattr_out {
-	__u32	size;
-	__u32	padding;
+	uint32_t	size;
+	uint32_t	padding;
 };
 
 struct fuse_lk_in {
-	__u64	fh;
-	__u64	owner;
+	uint64_t	fh;
+	uint64_t	owner;
 	struct fuse_file_lock lk;
-	__u32	lk_flags;
-	__u32	padding;
+	uint32_t	lk_flags;
+	uint32_t	padding;
 };
 
 struct fuse_lk_out {
@@ -446,134 +543,190 @@ struct fuse_lk_out {
 };
 
 struct fuse_access_in {
-	__u32	mask;
-	__u32	padding;
+	uint32_t	mask;
+	uint32_t	padding;
 };
 
 struct fuse_init_in {
-	__u32	major;
-	__u32	minor;
-	__u32	max_readahead;
-	__u32	flags;
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	max_readahead;
+	uint32_t	flags;
 };
 
 struct fuse_init_out {
-	__u32	major;
-	__u32	minor;
-	__u32	max_readahead;
-	__u32	flags;
-	__u16   max_background;
-	__u16   congestion_threshold;
-	__u32	max_write;
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	max_readahead;
+	uint32_t	flags;
+	uint16_t	max_background;
+	uint16_t	congestion_threshold;
+	uint32_t	max_write;
 };
 
 #define CUSE_INIT_INFO_MAX 4096
 
 struct cuse_init_in {
-	__u32	major;
-	__u32	minor;
-	__u32	unused;
-	__u32	flags;
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	unused;
+	uint32_t	flags;
 };
 
 struct cuse_init_out {
-	__u32	major;
-	__u32	minor;
-	__u32	unused;
-	__u32	flags;
-	__u32	max_read;
-	__u32	max_write;
-	__u32	dev_major;		/* chardev major */
-	__u32	dev_minor;		/* chardev minor */
-	__u32	spare[10];
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	unused;
+	uint32_t	flags;
+	uint32_t	max_read;
+	uint32_t	max_write;
+	uint32_t	dev_major;		/* chardev major */
+	uint32_t	dev_minor;		/* chardev minor */
+	uint32_t	spare[10];
 };
 
 struct fuse_interrupt_in {
-	__u64	unique;
+	uint64_t	unique;
 };
 
 struct fuse_bmap_in {
-	__u64	block;
-	__u32	blocksize;
-	__u32	padding;
+	uint64_t	block;
+	uint32_t	blocksize;
+	uint32_t	padding;
 };
 
 struct fuse_bmap_out {
-	__u64	block;
+	uint64_t	block;
 };
 
 struct fuse_ioctl_in {
-	__u64	fh;
-	__u32	flags;
-	__u32	cmd;
-	__u64	arg;
-	__u32	in_size;
-	__u32	out_size;
+	uint64_t	fh;
+	uint32_t	flags;
+	uint32_t	cmd;
+	uint64_t	arg;
+	uint32_t	in_size;
+	uint32_t	out_size;
+};
+
+struct fuse_ioctl_iovec {
+	uint64_t	base;
+	uint64_t	len;
 };
 
 struct fuse_ioctl_out {
-	__s32	result;
-	__u32	flags;
-	__u32	in_iovs;
-	__u32	out_iovs;
+	int32_t		result;
+	uint32_t	flags;
+	uint32_t	in_iovs;
+	uint32_t	out_iovs;
 };
 
 struct fuse_poll_in {
-	__u64	fh;
-	__u64	kh;
-	__u32	flags;
-	__u32   padding;
+	uint64_t	fh;
+	uint64_t	kh;
+	uint32_t	flags;
+	uint32_t	events;
 };
 
 struct fuse_poll_out {
-	__u32	revents;
-	__u32	padding;
+	uint32_t	revents;
+	uint32_t	padding;
 };
 
 struct fuse_notify_poll_wakeup_out {
-	__u64	kh;
+	uint64_t	kh;
+};
+
+struct fuse_fallocate_in {
+	uint64_t	fh;
+	uint64_t	offset;
+	uint64_t	length;
+	uint32_t	mode;
+	uint32_t	padding;
 };
 
 struct fuse_in_header {
-	__u32	len;
-	__u32	opcode;
-	__u64	unique;
-	__u64	nodeid;
-	__u32	uid;
-	__u32	gid;
-	__u32	pid;
-	__u32	padding;
+	uint32_t	len;
+	uint32_t	opcode;
+	uint64_t	unique;
+	uint64_t	nodeid;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	pid;
+	uint32_t	padding;
 };
 
 struct fuse_out_header {
-	__u32	len;
-	__s32	error;
-	__u64	unique;
+	uint32_t	len;
+	int32_t		error;
+	uint64_t	unique;
 };
 
 struct fuse_dirent {
-	__u64	ino;
-	__u64	off;
-	__u32	namelen;
-	__u32	type;
-	char name[0];
+	uint64_t	ino;
+	uint64_t	off;
+	uint32_t	namelen;
+	uint32_t	type;
+	char name[];
 };
 
 #define FUSE_NAME_OFFSET offsetof(struct fuse_dirent, name)
-#define FUSE_DIRENT_ALIGN(x) (((x) + sizeof(__u64) - 1) & ~(sizeof(__u64) - 1))
+#define FUSE_DIRENT_ALIGN(x) \
+	(((x) + sizeof(uint64_t) - 1) & ~(sizeof(uint64_t) - 1))
 #define FUSE_DIRENT_SIZE(d) \
 	FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET + (d)->namelen)
 
+struct fuse_direntplus {
+	struct fuse_entry_out entry_out;
+	struct fuse_dirent dirent;
+};
+
+#define FUSE_NAME_OFFSET_DIRENTPLUS \
+	offsetof(struct fuse_direntplus, dirent.name)
+#define FUSE_DIRENTPLUS_SIZE(d) \
+	FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET_DIRENTPLUS + (d)->dirent.namelen)
+
 struct fuse_notify_inval_inode_out {
-	__u64	ino;
-	__s64	off;
-	__s64	len;
+	uint64_t	ino;
+	int64_t		off;
+	int64_t		len;
 };
 
 struct fuse_notify_inval_entry_out {
-	__u64	parent;
-	__u32	namelen;
-	__u32	padding;
+	uint64_t	parent;
+	uint32_t	namelen;
+	uint32_t	padding;
+};
+
+struct fuse_notify_delete_out {
+	uint64_t	parent;
+	uint64_t	child;
+	uint32_t	namelen;
+	uint32_t	padding;
+};
+
+struct fuse_notify_store_out {
+	uint64_t	nodeid;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	padding;
+};
+
+struct fuse_notify_retrieve_out {
+	uint64_t	notify_unique;
+	uint64_t	nodeid;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	padding;
+};
+
+/* Matches the size of fuse_write_in */
+struct fuse_notify_retrieve_in {
+	uint64_t	dummy1;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	dummy2;
+	uint64_t	dummy3;
+	uint64_t	dummy4;
 };
 
 #endif /* _LINUX_FUSE_H */
diff --git a/contrib/fuse-util/mount_util.h b/contrib/fuse-include/mount_util.h
index f392f99f1..f392f99f1 100644
--- a/contrib/fuse-util/mount_util.h
+++ b/contrib/fuse-include/mount_util.h
diff --git a/contrib/fuse-lib/mount-common.c b/contrib/fuse-lib/mount-common.c
new file mode 100644
index 000000000..fd6cce44e
--- /dev/null
+++ b/contrib/fuse-lib/mount-common.c
@@ -0,0 +1,265 @@
+/*
+  FUSE: Filesystem in Userspace
+  Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
+  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+
+  This program can be distributed under the terms of the GNU LGPLv2.
+  See the file COPYING.LIB.
+*/
+
+#include "mount-gluster-compat.h"
+
+/*
+ * These functions (and gf_fuse_umount() in mount.c)
+ * were originally taken from libfuse as of commit 7960e99e
+ * (http://fuse.git.sourceforge.net/git/gitweb.cgi?p=fuse/fuse;a=commit;h=7960e99e)
+ * almost verbatim. What has been changed upon adoption:
+ * - style adopted to that of glusterfs
+ * - s/fprintf/gf_log/
+ * - s/free/FREE/, s/malloc/MALLOC/
+ * - there are some other minor things
+ *
+ * For changes that were made later and syncs with upstream,
+ * see the commit log and per-function comments.
+ */
+
+#ifndef __NetBSD__
+/* FUSE: cherry-picked bd99f9cf */
+static int
+mtab_needs_update (const char *mnt)
+{
+        int res;
+        struct stat stbuf;
+
+        /* If mtab is within new mount, don't touch it */
+        if (strncmp (mnt, _PATH_MOUNTED, strlen (mnt)) == 0 &&
+            _PATH_MOUNTED[strlen (mnt)] == '/')
+                return 0;
+
+        /*
+         * Skip mtab update if /etc/mtab:
+         *
+         *  - doesn't exist,
+         *  - is a symlink,
+         *  - is on a read-only filesystem.
+         */
+        res = lstat (_PATH_MOUNTED, &stbuf);
+        if (res == -1) {
+                if (errno == ENOENT)
+                        return 0;
+        } else {
+                uid_t ruid;
+                int err;
+
+                if (S_ISLNK (stbuf.st_mode))
+                        return 0;
+
+                ruid = getuid ();
+                if (ruid != 0)
+                        setreuid (0, -1);
+
+                res = access (_PATH_MOUNTED, W_OK);
+                err = (res == -1) ? errno : 0;
+                if (ruid != 0)
+                        setreuid (ruid, -1);
+
+                if (err == EROFS)
+                        return 0;
+        }
+
+        return 1;
+}
+#else /* __NetBSD__ */
+#define mtab_needs_update(x) 1
+#endif /* __NetBSD__ */
+
+/* FUSE: called add_mount_legacy(); R.I.P. as of cbd3a2a8 */
+int
+fuse_mnt_add_mount (const char *progname, const char *fsname,
+                    const char *mnt, const char *type, const char *opts)
+{
+        int res;
+        int status;
+        sigset_t blockmask;
+        sigset_t oldmask;
+
+        if (!mtab_needs_update (mnt))
+                return 0;
+
+        sigemptyset (&blockmask);
+        sigaddset (&blockmask, SIGCHLD);
+        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: sigprocmask: %s",
+                               progname, strerror (errno));
+                return -1;
+        }
+
+        res = fork ();
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
+                goto out_restore;
+        }
+        if (res == 0) {
+                char templ[] = "/tmp/fusermountXXXXXX";
+                char *tmp;
+
+                sigprocmask (SIG_SETMASK, &oldmask, NULL);
+                setuid (geteuid ());
+
+                /*
+                 * hide in a directory, where mount isn't able to resolve
+                 * fsname as a valid path
+                 */
+                tmp = mkdtemp (templ);
+                if (!tmp) {
+                        GFFUSE_LOGERR ("%s: failed to create temporary directory",
+                                       progname);
+                        exit (1);
+                }
+                if (chdir (tmp)) {
+                        GFFUSE_LOGERR ("%s: failed to chdir to %s: %s",
+                                       progname, tmp, strerror (errno));
+                        exit (1);
+                }
+                rmdir (tmp);
+                execl (_PATH_MOUNT, _PATH_MOUNT, "-i", "-f", "-t", type,
+                       "-o", opts, fsname, mnt, NULL);
+                GFFUSE_LOGERR ("%s: failed to execute %s: %s",
+                               progname, _PATH_MOUNT, strerror (errno));
+                exit (1);
+        }
+
+        res = waitpid (res, &status, 0);
+        if (res == -1)
+                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
+        res = (res != -1 && status == 0) ? 0 : -1;
+
+ out_restore:
+        sigprocmask (SIG_SETMASK, &oldmask, NULL);
+        return res;
+}
+
+char *
+fuse_mnt_resolve_path (const char *progname, const char *orig)
+{
+        char buf[PATH_MAX];
+        char *copy;
+        char *dst;
+        char *end;
+        char *lastcomp;
+        const char *toresolv;
+
+        if (!orig[0]) {
+                GFFUSE_LOGERR ("%s: invalid mountpoint '%s'", progname, orig);
+                return NULL;
+        }
+
+        copy = strdup (orig);
+        if (copy == NULL) {
+                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
+                return NULL;
+        }
+
+        toresolv = copy;
+        lastcomp = NULL;
+        for (end = copy + strlen (copy) - 1; end > copy && *end == '/'; end --);
+        if (end[0] != '/') {
+                char *tmp;
+                end[1] = '\0';
+                tmp = strrchr (copy, '/');
+                if (tmp == NULL) {
+                        lastcomp = copy;
+                        toresolv = ".";
+                } else {
+                        lastcomp = tmp + 1;
+                        if (tmp == copy)
+                                toresolv = "/";
+                }
+                if (strcmp (lastcomp, ".") == 0 || strcmp (lastcomp, "..") == 0) {
+                        lastcomp = NULL;
+                        toresolv = copy;
+                }
+                else if (tmp)
+                        tmp[0] = '\0';
+        }
+        if (realpath (toresolv, buf) == NULL) {
+                GFFUSE_LOGERR ("%s: bad mount point %s: %s", progname, orig,
+                               strerror (errno));
+                FREE (copy);
+                return NULL;
+        }
+        if (lastcomp == NULL)
+                dst = strdup (buf);
+        else {
+                dst = (char *) MALLOC (strlen (buf) + 1 + strlen (lastcomp) + 1);
+                if (dst) {
+                        unsigned buflen = strlen (buf);
+                        if (buflen && buf[buflen-1] == '/')
+                                sprintf (dst, "%s%s", buf, lastcomp);
+                        else
+                                sprintf (dst, "%s/%s", buf, lastcomp);
+                }
+        }
+        FREE (copy);
+        if (dst == NULL)
+                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
+        return dst;
+}
+
+/* FUSE: to support some changes that were reverted since
+ * then, it was split in two (fuse_mnt_umount() and
+ * exec_umount()); however the actual code is same as here
+ * since 0197ce40
+ */
+int
+fuse_mnt_umount (const char *progname, const char *abs_mnt,
+                 const char *rel_mnt, int lazy)
+{
+        int res;
+        int status;
+        sigset_t blockmask;
+        sigset_t oldmask;
+
+        if (!mtab_needs_update (abs_mnt)) {
+                res = umount2 (rel_mnt, lazy ? 2 : 0);
+                if (res == -1)
+                        GFFUSE_LOGERR ("%s: failed to unmount %s: %s",
+                                       progname, abs_mnt, strerror (errno));
+                return res;
+        }
+
+        sigemptyset (&blockmask);
+        sigaddset (&blockmask, SIGCHLD);
+        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: sigprocmask: %s", progname,
+                               strerror (errno));
+                return -1;
+        }
+
+        res = fork ();
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
+                goto out_restore;
+        }
+        if (res == 0) {
+                sigprocmask (SIG_SETMASK, &oldmask, NULL);
+                setuid (geteuid ());
+                execl ("/bin/umount", "/bin/umount", "-i", rel_mnt,
+                      lazy ? "-l" : NULL, NULL);
+                GFFUSE_LOGERR ("%s: failed to execute /bin/umount: %s",
+                               progname, strerror (errno));
+                exit (1);
+        }
+        res = waitpid (res, &status, 0);
+        if (res == -1)
+                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
+
+        if (status != 0)
+                res = -1;
+
+ out_restore:
+        sigprocmask (SIG_SETMASK, &oldmask, NULL);
+        return res;
+}
diff --git a/contrib/fuse-lib/mount-gluster-compat.h b/contrib/fuse-lib/mount-gluster-compat.h
new file mode 100644
index 000000000..4fc20623b
--- /dev/null
+++ b/contrib/fuse-lib/mount-gluster-compat.h
@@ -0,0 +1,56 @@
+/*
+  FUSE: Filesystem in Userspace
+  Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
+  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+
+  This program can be distributed under the terms of the GNU LGPLv2.
+  See the file COPYING.LIB.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stddef.h>
+#include <limits.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <dirent.h>
+#include <signal.h>
+#ifndef __NetBSD__
+#include <mntent.h>
+#endif /* __NetBSD__ */
+#include <sys/stat.h>
+#include <sys/poll.h>
+#include <sys/un.h>
+#include <sys/wait.h>
+#include <sys/mount.h>
+
+#ifdef __NetBSD__
+#include <perfuse.h>
+#define umount2(dir, flags) unmount(dir, ((flags) != 0) ? MNT_FORCE : 0)
+#define MS_RDONLY MNT_RDONLY
+#endif
+
+#ifdef linux
+#define _PATH_MOUNT "/bin/mount"
+#else /* NetBSD, MacOS X */
+#define _PATH_MOUNT "/sbin/mount"
+#endif
+
+#ifdef FUSE_UTIL
+#define MALLOC(size) malloc (size)
+#define FREE(ptr) free (ptr)
+#define GFFUSE_LOGERR(...) fprintf (stderr, ## __VA_ARGS__)
+#else /* FUSE_UTIL */
+#include "glusterfs.h"
+#include "logging.h"
+#include "common-utils.h"
+
+#define GFFUSE_LOGERR(...) \
+        gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
+#endif /* !FUSE_UTIL */
diff --git a/contrib/fuse-lib/mount.c b/contrib/fuse-lib/mount.c
index 800fd193e..922d9e464 100644
--- a/contrib/fuse-lib/mount.c
+++ b/contrib/fuse-lib/mount.c
@@ -7,342 +7,140 @@
   See the file COPYING.LIB.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <stddef.h>
-#include <limits.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <dirent.h>
-#include <signal.h>
-#ifndef __NetBSD__
-#include <mntent.h>
-#endif /* __NetBSD__ */
-#include <sys/stat.h>
-#include <sys/poll.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <sys/wait.h>
-#include <sys/mount.h>
-
-#ifdef __NetBSD__
-#include <perfuse.h>
-#define umount2(dir, flags) unmount(dir, ((flags) != 0) ? MNT_FORCE : 0)
-#endif
-
-#ifdef linux
-#define _PATH_MOUNT "/bin/mount"
-#else /* NetBSD, MacOS X */
-#define _PATH_MOUNT "/sbin/mount"
-#endif
-
-#ifdef FUSE_UTIL
-#define MALLOC(size) malloc (size)
-#define FREE(ptr) free (ptr)
-#define GFFUSE_LOGERR(...) fprintf (stderr, ## __VA_ARGS__)
-#else /* FUSE_UTIL */
-#include "glusterfs.h"
-#include "logging.h"
-#include "common-utils.h"
+#include "mount_util.h"
+#include "mount-gluster-compat.h"
 
 #ifdef GF_FUSERMOUNT
 #define FUSERMOUNT_PROG FUSERMOUNT_DIR "/fusermount-glusterfs"
 #else
 #define FUSERMOUNT_PROG "fusermount"
 #endif
-#define FUSE_COMMFD_ENV "_FUSE_COMMFD"
+#define FUSE_DEVFD_ENV "_FUSE_DEVFD"
 
-#define GFFUSE_LOGERR(...) \
-        gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
-#endif /* !FUSE_UTIL */
 
-/*
- * Functions below, until following note, were taken from libfuse
- * (http://git.gluster.com/?p=users/csaba/fuse.git;a=commit;h=b988bbf9)
- * almost verbatim. What has been changed:
- * - style adopted to that of glusterfs
- * - s/fprintf/gf_log/
- * - s/free/FREE/, s/malloc/MALLOC/
- * - there are some other minor things
- */
-
-#ifndef __NetBSD__
-static int
-mtab_needs_update (const char *mnt)
+/* FUSE: function is called fuse_kern_unmount() */
+void
+gf_fuse_unmount (const char *mountpoint, int fd)
 {
         int res;
-        struct stat stbuf;
-
-        /* If mtab is within new mount, don't touch it */
-        if (strncmp (mnt, _PATH_MOUNTED, strlen (mnt)) == 0 &&
-            _PATH_MOUNTED[strlen (mnt)] == '/')
-                return 0;
-
-        /*
-         * Skip mtab update if /etc/mtab:
-         *
-         *  - doesn't exist,
-         *  - is a symlink,
-         *  - is on a read-only filesystem.
-         */
-        res = lstat (_PATH_MOUNTED, &stbuf);
-        if (res == -1) {
-                if (errno == ENOENT)
-                        return 0;
-        } else {
-                if (S_ISLNK (stbuf.st_mode))
-                        return 0;
-
-                res = access (_PATH_MOUNTED, W_OK);
-                if (res == -1 && errno == EROFS)
-                        return 0;
-        }
+        int pid;
 
-        return 1;
-}
-#else /* __NetBSD__ */
-#define mtab_needs_update(x) 1
-#endif /* __NetBSD__ */
+        if (!mountpoint)
+                return;
 
-#ifndef FUSE_UTIL
-static
-#endif
-int
-fuse_mnt_add_mount (const char *progname, const char *fsname,
-                    const char *mnt, const char *type, const char *opts,
-                    pid_t *mtab_pid)
-{
-        int res;
-        int status;
-        sigset_t blockmask;
-        sigset_t oldmask;
-
-        if (!mtab_needs_update (mnt))
-                return 0;
-
-        sigemptyset (&blockmask);
-        sigaddset (&blockmask, SIGCHLD);
-        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: sigprocmask: %s",
-                               progname, strerror (errno));
-                return -1;
-        }
+        if (fd != -1) {
+                struct pollfd pfd;
 
-        res = fork ();
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
-                goto out_restore;
-        }
-        if (res == 0) {
-                char templ[] = "/tmp/fusermountXXXXXX";
-                char *tmp;
-
-                if (!mtab_pid) {
-                        /* mtab update done async, just log if fails */
-                        res = fork ();
-                        if (res)
-                                exit (res == -1 ? 1 : 0);
-                        res = fork ();
-                        if (res) {
-                                if (res != -1) {
-                                        if (!(res == waitpid (res, &status, 0)
-                                              && status == 0))
-                                                GFFUSE_LOGERR ("%s: /etc/mtab "
-                                                               "update failed",
-                                                               progname);
-                                }
-                                exit (0);
-                        }
-                }
+                pfd.fd = fd;
+                pfd.events = 0;
+                res = poll (&pfd, 1, 0);
+                /* If file poll returns POLLERR on the device file descriptor,
+                   then the filesystem is already unmounted */
+                if (res == 1 && (pfd.revents & POLLERR))
+                        return;
 
-                sigprocmask (SIG_SETMASK, &oldmask, NULL);
-                setuid (geteuid ());
-
-                /*
-                 * hide in a directory, where mount isn't able to resolve
-                 * fsname as a valid path
-                 */
-                tmp = mkdtemp (templ);
-                if (!tmp) {
-                        GFFUSE_LOGERR ("%s: failed to create temporary directory",
-                                       progname);
-                        exit (1);
-                }
-                if (chdir (tmp)) {
-                        GFFUSE_LOGERR ("%s: failed to chdir to %s: %s",
-                                       progname, tmp, strerror (errno));
-                        exit (1);
-                }
-                rmdir (tmp);
-                execl (_PATH_MOUNT, _PATH_MOUNT, "-i", "-f", "-t", type,
-                       "-o", opts, fsname, mnt, NULL);
-                GFFUSE_LOGERR ("%s: failed to execute %s: %s",
-                               progname, _PATH_MOUNT, strerror (errno));
-                exit (1);
+                /* Need to close file descriptor, otherwise synchronous umount
+                   would recurse into filesystem, and deadlock */
+                close (fd);
         }
-        if (mtab_pid) {
-                *mtab_pid = res;
-                res = 0;
-        } else {
-                if (!(res == waitpid (res, &status, 0) && status == 0))
-                        res = -1;
+
+        if (geteuid () == 0) {
+                fuse_mnt_umount ("fuse", mountpoint, mountpoint, 1);
+                return;
         }
-        if (res == -1)
-                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
 
- out_restore:
-        sigprocmask (SIG_SETMASK, &oldmask, NULL);
-        return res;
-}
+        res = umount2 (mountpoint, 2);
+        if (res == 0)
+                return;
 
-#ifndef FUSE_UTIL
-static
-#endif
-char
-*fuse_mnt_resolve_path (const char *progname, const char *orig)
-{
-        char buf[PATH_MAX];
-        char *copy;
-        char *dst;
-        char *end;
-        char *lastcomp;
-        const char *toresolv;
-
-        if (!orig[0]) {
-                GFFUSE_LOGERR ("%s: invalid mountpoint '%s'", progname, orig);
-                return NULL;
-        }
+        pid = fork ();
+        if (pid == -1)
+                return;
 
-        copy = strdup (orig);
-        if (copy == NULL) {
-                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
-                return NULL;
-        }
+        if (pid == 0) {
+                const char *argv[] = { FUSERMOUNT_PROG, "-u", "-q", "-z",
+                                       "--", mountpoint, NULL };
 
-        toresolv = copy;
-        lastcomp = NULL;
-        for (end = copy + strlen (copy) - 1; end > copy && *end == '/'; end --);
-        if (end[0] != '/') {
-                char *tmp;
-                end[1] = '\0';
-                tmp = strrchr (copy, '/');
-                if (tmp == NULL) {
-                        lastcomp = copy;
-                        toresolv = ".";
-                } else {
-                        lastcomp = tmp + 1;
-                        if (tmp == copy)
-                                toresolv = "/";
-                }
-                if (strcmp (lastcomp, ".") == 0 || strcmp (lastcomp, "..") == 0) {
-                        lastcomp = NULL;
-                        toresolv = copy;
-                }
-                else if (tmp)
-                        tmp[0] = '\0';
-        }
-        if (realpath (toresolv, buf) == NULL) {
-                GFFUSE_LOGERR ("%s: bad mount point %s: %s", progname, orig,
-                               strerror (errno));
-                FREE (copy);
-                return NULL;
-        }
-        if (lastcomp == NULL)
-                dst = strdup (buf);
-        else {
-                dst = (char *) MALLOC (strlen (buf) + 1 + strlen (lastcomp) + 1);
-                if (dst) {
-                        unsigned buflen = strlen (buf);
-                        if (buflen && buf[buflen-1] == '/')
-                                sprintf (dst, "%s%s", buf, lastcomp);
-                        else
-                                sprintf (dst, "%s/%s", buf, lastcomp);
-                }
+                execvp (FUSERMOUNT_PROG, (char **)argv);
+                _exit (1);
         }
-        FREE (copy);
-        if (dst == NULL)
-                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
-        return dst;
+        waitpid (pid, NULL, 0);
 }
 
-#ifndef FUSE_UTIL
-/* return value:
- * >= 0         => fd
- * -1         => error
- */
-static int
-receive_fd (int fd)
+
+/* gluster-specific routines */
+
+static char *
+escape (char *s)
 {
-        struct msghdr msg;
-        struct iovec iov;
-        char buf[1];
-        int rv;
-        size_t ccmsg[CMSG_SPACE (sizeof (int)) / sizeof (size_t)];
-        struct cmsghdr *cmsg;
-        int *recv_fd;
-
-        iov.iov_base = buf;
-        iov.iov_len = 1;
-
-        msg.msg_name = 0;
-        msg.msg_namelen = 0;
-        msg.msg_iov = &iov;
-        msg.msg_iovlen = 1;
-        /* old BSD implementations should use msg_accrights instead of
-         * msg_control; the interface is different. */
-        msg.msg_control = ccmsg;
-        msg.msg_controllen = sizeof (ccmsg);
-
-        while (((rv = recvmsg (fd, &msg, 0)) == -1) && errno == EINTR);
-        if (rv == -1) {
-                GFFUSE_LOGERR ("recvmsg failed: %s", strerror (errno));
-                return -1;
-        }
-        if (!rv) {
-                /* EOF */
-                return -1;
+        size_t len = 0;
+        char *p = NULL;
+        char *q = NULL;
+        char *e = NULL;
+
+        for (p = s; *p; p++) {
+                if (*p == ',')
+                       len++;
+                len++;
         }
 
-        cmsg = CMSG_FIRSTHDR (&msg);
-        /*
-         * simplify condition expression
-         */
-        if (cmsg->cmsg_type != SCM_RIGHTS) {
-                GFFUSE_LOGERR ("got control message of unknown type %d",
-                               cmsg->cmsg_type);
-                return -1;
+        e = CALLOC (1, len + 1);
+        if (!e)
+                return NULL;
+
+        for (p = s, q = e; *p; p++, q++) {
+                if (*p == ',') {
+                        *q = '\\';
+                        q++;
+                }
+                *q = *p;
         }
 
-        recv_fd = (int *) CMSG_DATA (cmsg);
-        return (*recv_fd);
+        return e;
 }
 
 static int
-fuse_mount_fusermount (const char *mountpoint, const char *opts)
+fuse_mount_fusermount (const char *mountpoint, char *fsname,
+                       unsigned long mountflags, char *mnt_param,
+                       int fd)
 {
-        int fds[2], pid;
-        int res;
-        int rv;
+        int  pid = -1;
+        int  res = 0;
+        int  ret = -1;
+        char *fm_mnt_params = NULL;
+        char *efsname = NULL;
+
+#ifndef GF_FUSERMOUNT
+        GFFUSE_LOGERR ("Mounting via helper utility "
+                       "(unprivileged mounting) is supported "
+                       "only if glusterfs is compiled with "
+                       "--enable-fusermount");
+        return -1;
+#endif
+
+        efsname = escape (fsname);
+        if (!efsname) {
+                GFFUSE_LOGERR ("Out of memory");
 
-        res = socketpair (PF_UNIX, SOCK_STREAM, 0, fds);
-        if (res == -1) {
-                GFFUSE_LOGERR ("socketpair() failed: %s", strerror (errno));
                 return -1;
         }
+        ret = asprintf (&fm_mnt_params,
+                        "%s%s,fsname=%s,nonempty,subtype=glusterfs",
+                        (mountflags & MS_RDONLY) ? "ro," : "",
+                        mnt_param, efsname);
+        FREE (efsname);
+        if (ret == -1) {
+                GFFUSE_LOGERR ("Out of memory");
+
+                goto out;
+        }
 
+        /* fork to exec fusermount */
         pid = fork ();
         if (pid == -1) {
                 GFFUSE_LOGERR ("fork() failed: %s", strerror (errno));
-                close (fds[0]);
-                close (fds[1]);
-                return -1;
+                ret = -1;
+                goto out;
         }
 
         if (pid == 0) {
@@ -351,214 +149,37 @@ fuse_mount_fusermount (const char *mountpoint, const char *opts)
                 int a = 0;
 
                 argv[a++] = FUSERMOUNT_PROG;
-                if (opts) {
-                        argv[a++] = "-o";
-                        argv[a++] = opts;
-                }
+                argv[a++] = "-o";
+                argv[a++] = fm_mnt_params;
                 argv[a++] = "--";
                 argv[a++] = mountpoint;
                 argv[a++] = NULL;
 
-                close (fds[1]);
-                fcntl (fds[0], F_SETFD, 0);
-                snprintf (env, sizeof (env), "%i", fds[0]);
-                setenv (FUSE_COMMFD_ENV, env, 1);
+                snprintf (env, sizeof (env), "%i", fd);
+                setenv (FUSE_DEVFD_ENV, env, 1);
                 execvp (FUSERMOUNT_PROG, (char **)argv);
                 GFFUSE_LOGERR ("failed to exec fusermount: %s",
                                strerror (errno));
                 _exit (1);
         }
 
-        close (fds[0]);
-        rv = receive_fd (fds[1]);
-        close (fds[1]);
-        waitpid (pid, NULL, 0); /* bury zombie */
-
-        return rv;
-}
-#endif
-
-#ifndef FUSE_UTIL
-static
-#endif
-int
-fuse_mnt_umount (const char *progname, const char *abs_mnt,
-                 const char *rel_mnt, int lazy)
-{
-        int res;
-        int status;
-        sigset_t blockmask;
-        sigset_t oldmask;
-
-        if (!mtab_needs_update (abs_mnt)) {
-                res = umount2 (rel_mnt, lazy ? 2 : 0);
-                if (res == -1)
-                        GFFUSE_LOGERR ("%s: failed to unmount %s: %s",
-                                       progname, abs_mnt, strerror (errno));
-                return res;
-        }
-
-        sigemptyset (&blockmask);
-        sigaddset (&blockmask, SIGCHLD);
-        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: sigprocmask: %s", progname,
-                               strerror (errno));
-                return -1;
-        }
-
-        res = fork ();
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
-                goto out_restore;
-        }
-        if (res == 0) {
-                sigprocmask (SIG_SETMASK, &oldmask, NULL);
-                setuid (geteuid ());
-                execl ("/bin/umount", "/bin/umount", "-i", rel_mnt,
-                      lazy ? "-l" : NULL, NULL);
-                GFFUSE_LOGERR ("%s: failed to execute /bin/umount: %s",
-                               progname, strerror (errno));
-                exit (1);
-        }
-        res = waitpid (res, &status, 0);
-        if (res == -1)
-                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
-
-        if (status != 0)
-                res = -1;
-
- out_restore:
-        sigprocmask (SIG_SETMASK, &oldmask, NULL);
-        return res;
-}
-
-#ifdef FUSE_UTIL
-int
-fuse_mnt_check_empty (const char *progname, const char *mnt,
-                      mode_t rootmode, off_t rootsize)
-{
-        int isempty = 1;
-
-        if (S_ISDIR (rootmode)) {
-                struct dirent *ent;
-                DIR *dp = opendir (mnt);
-                if (dp == NULL) {
-                        fprintf (stderr,
-                                 "%s: failed to open mountpoint for reading: %s\n",
-                                 progname, strerror (errno));
-                        return -1;
-                }
-                while ((ent = readdir (dp)) != NULL) {
-                        if (strcmp (ent->d_name, ".") != 0 &&
-                            strcmp (ent->d_name, "..") != 0) {
-                                isempty = 0;
-                                break;
-                        }
-                }
-                closedir (dp);
-        } else if (rootsize)
-                isempty = 0;
-
-        if (!isempty) {
-                fprintf (stderr, "%s: mountpoint is not empty\n", progname);
-                fprintf (stderr, "%s: if you are sure this is safe, "
-                         "use the 'nonempty' mount option\n", progname);
-                return -1;
-        }
-        return 0;
-}
-
-int
-fuse_mnt_check_fuseblk (void)
-{
-        char buf[256];
-        FILE *f = fopen ("/proc/filesystems", "r");
-        if (!f)
-                return 1;
-
-        while (fgets (buf, sizeof (buf), f))
-                if (strstr (buf, "fuseblk\n")) {
-                        fclose (f);
-                        return 1;
-                }
-
-        fclose (f);
-        return 0;
-}
-#endif
-
-#ifndef FUSE_UTIL
-void
-gf_fuse_unmount (const char *mountpoint, int fd)
-{
-        int res;
-        int pid;
-
-        if (!mountpoint)
-                return;
-
-        if (fd != -1) {
-                struct pollfd pfd;
-
-                pfd.fd = fd;
-                pfd.events = 0;
-                res = poll (&pfd, 1, 0);
-                /* If file poll returns POLLERR on the device file descriptor,
-                   then the filesystem is already unmounted */
-                if (res == 1 && (pfd.revents & POLLERR))
-                        return;
-
-                /* Need to close file descriptor, otherwise synchronous umount
-                   would recurse into filesystem, and deadlock */
-                close (fd);
-        }
-
-        if (geteuid () == 0) {
-                fuse_mnt_umount ("fuse", mountpoint, mountpoint, 1);
-                return;
-        }
-
-        res = umount2 (mountpoint, 2);
-        if (res == 0)
-                return;
-
-        pid = fork ();
-        if (pid == -1)
-                return;
-
-        if (pid == 0) {
-                const char *argv[] = { FUSERMOUNT_PROG, "-u", "-q", "-z",
-                                       "--", mountpoint, NULL };
-
-                execvp (FUSERMOUNT_PROG, (char **)argv);
-                _exit (1);
-        }
-        waitpid (pid, NULL, 0);
+        ret = waitpid (pid, &res, 0);
+        ret = (ret == pid && res == 0) ? 0 : -1;
+ out:
+        FREE (fm_mnt_params);
+        return ret;
 }
-#endif
-
-/*
- * Functions below are loosely modelled after similar functions of libfuse
- */
 
-#ifndef FUSE_UTIL
 static int
-fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mtab_pid)
+fuse_mount_sys (const char *mountpoint, char *fsname,
+                unsigned long mountflags, char *mnt_param, int fd)
 {
-        int fd = -1, ret = -1;
+        int ret = -1;
         unsigned mounted = 0;
         char *mnt_param_mnt = NULL;
         char *fstype = "fuse.glusterfs";
         char *source = fsname;
 
-        fd = open ("/dev/fuse", O_RDWR);
-        if (fd == -1) {
-                GFFUSE_LOGERR ("cannot open /dev/fuse (%s)", strerror (errno));
-
-                return -1;
-        }
-
         ret = asprintf (&mnt_param_mnt,
                         "%s,fd=%i,rootmode=%o,user_id=%i,group_id=%i",
                         mnt_param, fd, S_IFDIR, getuid (), getgid ());
@@ -567,7 +188,7 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
 
                 goto out;
         }
-        ret = mount (source, mountpoint, fstype, 0,
+        ret = mount (source, mountpoint, fstype, mountflags,
                      mnt_param_mnt);
         if (ret == -1 && errno == ENODEV) {
                 /* fs subtype support was added by 79c0b2df aka
@@ -580,7 +201,7 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
 
                         goto out;
                 }
-                ret = mount (source, mountpoint, fstype, 0,
+                ret = mount (source, mountpoint, fstype, mountflags,
                              mnt_param_mnt);
         }
         if (ret == -1)
@@ -588,8 +209,10 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
         else
                 mounted = 1;
 
+#ifndef __NetBSD__
         if (geteuid () == 0) {
                 char *newmnt = fuse_mnt_resolve_path ("fuse", mountpoint);
+                char *mnt_param_mtab = NULL;
 
                 if (!newmnt) {
                         ret = -1;
@@ -597,8 +220,17 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
                         goto out;
                 }
 
-                ret = fuse_mnt_add_mount ("fuse", source, newmnt, fstype,
-                                          mnt_param, mtab_pid);
+                ret = asprintf (&mnt_param_mtab, "%s%s",
+                                mountflags & MS_RDONLY ? "ro," : "",
+                                mnt_param);
+                if (ret == -1)
+                        GFFUSE_LOGERR ("Out of memory");
+                else {
+                        ret = fuse_mnt_add_mount ("fuse", source, newmnt,
+                                                  fstype, mnt_param_mtab);
+                        FREE (mnt_param_mtab);
+                }
+
                 FREE (newmnt);
                 if (ret == -1) {
                         GFFUSE_LOGERR ("failed to add mtab entry");
@@ -606,95 +238,76 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
                         goto out;
                 }
         }
+#endif /* __NetBSD__ */
 
- out:
+out:
         if (ret == -1) {
                 if (mounted)
                         umount2 (mountpoint, 2); /* lazy umount */
-                close (fd);
-                fd = -1;
         }
         FREE (mnt_param_mnt);
         if (source != fsname)
                 FREE (source);
-        return fd;
-}
-
-static char *
-escape (char *s)
-{
-        size_t len = 0;
-        char *p = NULL;
-        char *q = NULL;
-        char *e = NULL;
-
-        for (p = s; *p; p++) {
-                if (*p == ',')
-                       len++;
-                len++;
-        }
-
-        e = CALLOC (1, len + 1);
-        if (!e)
-                return NULL;
-
-        for (p = s, q = e; *p; p++, q++) {
-                if (*p == ',') {
-                        *q = '\\';
-                        q++;
-                }
-                *q = *p;
-        }
 
-        return e;
+        return ret;
 }
 
 int
-gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
-               pid_t *mtab_pid)
+gf_fuse_mount (const char *mountpoint, char *fsname,
+               unsigned long mountflags, char *mnt_param,
+               pid_t *mnt_pid, int status_fd)
 {
-        int fd = -1, rv = -1;
-        char *fm_mnt_params = NULL, *p = NULL;
-        char *efsname = NULL;
+        int   fd  = -1;
+        pid_t pid = -1;
+        int   ret = -1;
 
-        fd = fuse_mount_sys (mountpoint, fsname, mnt_param, mtab_pid);
+        fd = open ("/dev/fuse", O_RDWR);
         if (fd == -1) {
-                gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                        "direct mount failed (%s), "
-                        "retry to mount via fusermount",
-                        strerror (errno));
-
-                efsname = escape (fsname);
-                if (!efsname) {
-                        GFFUSE_LOGERR ("Out of memory");
-
-                        return -1;
-                }
-                rv = asprintf (&fm_mnt_params,
-                               "%s,fsname=%s,nonempty,subtype=glusterfs",
-                               mnt_param, efsname);
-                FREE (efsname);
-                if (rv == -1) {
-                        GFFUSE_LOGERR ("Out of memory");
+                GFFUSE_LOGERR ("cannot open /dev/fuse (%s)",
+                                strerror (errno));
+                return -1;
+        }
 
-                        return -1;
+        /* start mount agent */
+        pid = fork();
+        switch (pid) {
+        case 0:
+                /* hello it's mount agent */
+                if (!mnt_pid) {
+                        /* daemonize mount agent, caller is
+                         * not interested in waiting for it
+                         */
+                        pid = fork ();
+                        if (pid)
+                                exit (pid == -1 ? 1 : 0);
                 }
 
-                fd = fuse_mount_fusermount (mountpoint, fm_mnt_params);
-                if (fd == -1) {
-                        p = fm_mnt_params + strlen (fm_mnt_params);
-                        while (*--p != ',');
-                        *p = '\0';
+                ret = fuse_mount_sys (mountpoint, fsname, mountflags, mnt_param, fd);
+                if (ret == -1) {
+                        gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                                "direct mount failed (%s), "
+                                "retry to mount via fusermount",
+                                strerror (errno));
 
-                        fd = fuse_mount_fusermount (mountpoint, fm_mnt_params);
+                        ret = fuse_mount_fusermount (mountpoint, fsname, mountflags,
+                                                     mnt_param, fd);
                 }
 
-                FREE (fm_mnt_params);
+                if (ret == -1)
+                        GFFUSE_LOGERR ("mount of %s to %s (%s) failed",
+                                       fsname, mountpoint, mnt_param);
 
-                if (fd == -1)
-                       GFFUSE_LOGERR ("mount failed");
+                if (status_fd >= 0)
+                        (void)write (status_fd, &ret, sizeof (ret));
+                exit (!!ret);
+                /* bye mount agent */
+        case -1:
+                close (fd);
+                fd = -1;
         }
 
+        if (mnt_pid)
+               *mnt_pid = pid;
+
         return fd;
 }
-#endif
diff --git a/contrib/fuse-util/Makefile.am b/contrib/fuse-util/Makefile.am
index a72e3832b..971d3d220 100644
--- a/contrib/fuse-util/Makefile.am
+++ b/contrib/fuse-util/Makefile.am
@@ -1,9 +1,11 @@
 bin_PROGRAMS = fusermount-glusterfs
 
-fusermount_glusterfs_SOURCES = fusermount.c $(CONTRIBDIR)/fuse-lib/mount.c
-noinst_HEADERS = mount_util.h
+fusermount_glusterfs_SOURCES = fusermount.c mount_util.c $(CONTRIBDIR)/fuse-lib/mount-common.c
+noinst_HEADERS = $(CONTRIBDIR)/fuse-include/mount_util.h
 
-AM_CFLAGS = -Wall -D_FILE_OFFSET_BITS=64 -DFUSE_UTIL $(GF_CFLAGS) -D_GNU_SOURCE
+AM_CPPFLAGS = $(GF_CPPFLAGS) -DFUSE_UTIL -I$(CONTRIBDIR)/fuse-include
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 install-exec-hook:
 	-chown root $(DESTDIR)$(bindir)/fusermount-glusterfs
diff --git a/contrib/fuse-util/fusermount.c b/contrib/fuse-util/fusermount.c
index 39da9b6a0..0ff8d9039 100644
--- a/contrib/fuse-util/fusermount.c
+++ b/contrib/fuse-util/fusermount.c
@@ -19,6 +19,7 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <pwd.h>
+#include <limits.h>
 #include <mntent.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
@@ -28,6 +29,7 @@
 #include <sys/utsname.h>
 #include <sched.h>
 
+#define FUSE_DEVFD_ENV		"_FUSE_DEVFD"
 #define FUSE_COMMFD_ENV		"_FUSE_COMMFD"
 
 #define FUSE_DEV_OLD "/proc/fs/fuse/dev"
@@ -114,8 +116,16 @@ static int lock_umount(void)
 
 static void unlock_umount(int mtablock)
 {
-	lockf(mtablock, F_ULOCK, 0);
-	close(mtablock);
+	if (mtablock >= 0) {
+		int res;
+
+		res = lockf(mtablock, F_ULOCK, 0);
+		if (res < 0) {
+			fprintf(stderr, "%s: error releasing lock: %s\n",
+				progname, strerror(errno));
+		}
+		close(mtablock);
+	}
 }
 
 static int add_mount(const char *source, const char *mnt, const char *type,
@@ -238,7 +248,7 @@ static int check_is_mount_child(void *p)
 	}
 
 	count = 0;
-	while ((entp = getmntent(fp)) != NULL)
+	while (getmntent(fp) != NULL)
 		count++;
 	endmntent(fp);
 
@@ -325,7 +335,7 @@ static int check_is_mount(const char *last, const char *mnt)
 	return 0;
 }
 
-static int chdir_to_parent(char *copy, const char **lastp, int *currdir_fd)
+static int chdir_to_parent(char *copy, const char **lastp)
 {
 	char *tmp;
 	const char *parent;
@@ -350,14 +360,6 @@ static int chdir_to_parent(char *copy, const char **lastp, int *currdir_fd)
 		parent = "/";
 	}
 
-	*currdir_fd = open(".", O_RDONLY);
-	if (*currdir_fd == -1) {
-		fprintf(stderr,
-			"%s: failed to open current directory: %s\n",
-			progname, strerror(errno));
-		return -1;
-	}
-
 	res = chdir(parent);
 	if (res == -1) {
 		fprintf(stderr, "%s: failed to chdir to %s: %s\n",
@@ -382,7 +384,6 @@ static int chdir_to_parent(char *copy, const char **lastp, int *currdir_fd)
 
 static int unmount_fuse_locked(const char *mnt, int quiet, int lazy)
 {
-	int currdir_fd = -1;
 	char *copy;
 	const char *last;
 	int res;
@@ -399,7 +400,7 @@ static int unmount_fuse_locked(const char *mnt, int quiet, int lazy)
 		return -1;
 	}
 
-	res = chdir_to_parent(copy, &last, &currdir_fd);
+	res = chdir_to_parent(copy, &last);
 	if (res == -1)
 		goto out;
 
@@ -411,10 +412,6 @@ static int unmount_fuse_locked(const char *mnt, int quiet, int lazy)
 
 out:
 	free(copy);
-	if (currdir_fd != -1) {
-		fchdir(currdir_fd);
-		close(currdir_fd);
-	}
 
 	return res;
 }
@@ -508,13 +505,13 @@ static void read_conf(void)
 		int isnewline = 1;
 		while (fgets(line, sizeof(line), fp) != NULL) {
 			if (isnewline) {
-				if (line[strlen(line)-1] == '\n') {
+				if (strlen(line) && line[strlen(line)-1] == '\n') {
 					strip_line(line);
 					parse_line(line, linenum);
 				} else {
 					isnewline = 0;
 				}
-			} else if(line[strlen(line)-1] == '\n') {
+			} else if(strlen(line) && line[strlen(line)-1] == '\n') {
 				fprintf(stderr, "%s: reading %s: line %i too long\n", progname, FUSE_CONF, linenum);
 
 				isnewline = 1;
@@ -609,7 +606,7 @@ static int add_option(char **optsp, const char *opt, unsigned expand)
 static int get_mnt_opts(int flags, char *opts, char **mnt_optsp)
 {
 	int i;
-	int l;
+	size_t l;
 
 	if (!(flags & MS_RDONLY) && add_option(mnt_optsp, "rw", 0) == -1)
 		return -1;
@@ -624,7 +621,7 @@ static int get_mnt_opts(int flags, char *opts, char **mnt_optsp)
 		return -1;
 	/* remove comma from end of opts*/
 	l = strlen(*mnt_optsp);
-	if ((*mnt_optsp)[l-1] == ',')
+	if (l && (*mnt_optsp)[l-1] == ',')
 		(*mnt_optsp)[l-1] = '\0';
 	if (getuid() != 0) {
 		const char *user = get_user_name();
@@ -653,8 +650,7 @@ static int get_string_opt(const char *s, unsigned len, const char *opt,
 	unsigned opt_len = strlen(opt);
 	char *d;
 
-	if (*val)
-		free(*val);
+	free(*val);
 	*val = (char *) malloc(len - opt_len + 1);
 	if (!*val) {
 		fprintf(stderr, "%s: failed to allocate memory\n", progname);
@@ -823,15 +819,14 @@ static int do_mount(const char *mnt, char **typep, mode_t rootmode,
 			fprintf(stderr, "%s: mount failed: %s\n", progname,
 				strerror(errno_save));
 		goto err;
-	} else {
-		*sourcep = source;
-		*typep = type;
-		*mnt_optsp = mnt_opts;
 	}
+	*sourcep = source;
+	*typep = type;
+	*mnt_optsp = mnt_opts;
 	free(fsname);
 	free(optbuf);
 
-	return res;
+	return 0;
 
 err:
 	free(fsname);
@@ -874,8 +869,7 @@ static int check_version(const char *dev)
 	return 0;
 }
 
-static int check_perm(const char **mntp, struct stat *stbuf, int *currdir_fd,
-		      int *mountpoint_fd)
+static int check_perm(const char **mntp, struct stat *stbuf, int *mountpoint_fd)
 {
 	int res;
 	const char *mnt = *mntp;
@@ -893,13 +887,6 @@ static int check_perm(const char **mntp, struct stat *stbuf, int *currdir_fd,
 		return 0;
 
 	if (S_ISDIR(stbuf->st_mode)) {
-		*currdir_fd = open(".", O_RDONLY);
-		if (*currdir_fd == -1) {
-			fprintf(stderr,
-				"%s: failed to open current directory: %s\n",
-				progname, strerror(errno));
-			return -1;
-		}
 		res = chdir(mnt);
 		if (res == -1) {
 			fprintf(stderr,
@@ -1016,8 +1003,36 @@ static int open_fuse_device(char **devp)
 	return -1;
 }
 
+static int check_fuse_device(char *devfd, char **devp)
+{
+	int res;
+	char *devlink;
 
-static int mount_fuse(const char *mnt, const char *opts)
+	res = asprintf(&devlink, "/proc/self/fd/%s", devfd);
+	if (res == -1) {
+		fprintf(stderr, "%s: failed to allocate memory\n", progname);
+		return -1;
+	}
+
+	*devp = (char *) calloc(1, PATH_MAX + 1);
+	if (!*devp) {
+		fprintf(stderr, "%s: failed to allocate memory\n", progname);
+		free(devlink);
+		return -1;
+	}
+
+	res = readlink (devlink, *devp, PATH_MAX);
+	free (devlink);
+	if (res == -1) {
+		fprintf(stderr, "%s: specified fuse fd is invalid\n",
+			progname);
+		return -1;
+	}
+
+	return atoi(devfd);
+}
+
+static int mount_fuse(const char *mnt, const char *opts, char *devfd)
 {
 	int res;
 	int fd;
@@ -1027,10 +1042,9 @@ static int mount_fuse(const char *mnt, const char *opts)
 	char *source = NULL;
 	char *mnt_opts = NULL;
 	const char *real_mnt = mnt;
-	int currdir_fd = -1;
 	int mountpoint_fd = -1;
 
-	fd = open_fuse_device(&dev);
+	fd = devfd ? check_fuse_device(devfd, &dev) : open_fuse_device(&dev);
 	if (fd == -1)
 		return -1;
 
@@ -1041,15 +1055,13 @@ static int mount_fuse(const char *mnt, const char *opts)
 		int mount_count = count_fuse_fs();
 		if (mount_count >= mount_max) {
 			fprintf(stderr, "%s: too many FUSE filesystems mounted; mount_max=N can be set in /etc/fuse.conf\n", progname);
-			close(fd);
-			return -1;
+			goto fail_close_fd;
 		}
 	}
 
 	res = check_version(dev);
 	if (res != -1) {
-		res = check_perm(&real_mnt, &stbuf, &currdir_fd,
-				 &mountpoint_fd);
+		res = check_perm(&real_mnt, &stbuf, &mountpoint_fd);
 		restore_privs();
 		if (res != -1)
 			res = do_mount(real_mnt, &type, stbuf.st_mode & S_IFMT,
@@ -1058,33 +1070,38 @@ static int mount_fuse(const char *mnt, const char *opts)
 	} else
 		restore_privs();
 
-	if (currdir_fd != -1) {
-		fchdir(currdir_fd);
-		close(currdir_fd);
-	}
 	if (mountpoint_fd != -1)
 		close(mountpoint_fd);
 
+	if (res == -1)
+		goto fail_close_fd;
+
+	res = chdir("/");
 	if (res == -1) {
-		close(fd);
-		return -1;
+		fprintf(stderr, "%s: failed to chdir to '/'\n", progname);
+		goto fail_close_fd;
 	}
 
 	if (geteuid() == 0) {
 		res = add_mount(source, mnt, type, mnt_opts);
 		if (res == -1) {
-			umount2(mnt, 2); /* lazy umount */
-			close(fd);
-			return -1;
+			/* Can't clean up mount in a non-racy way */
+			goto fail_close_fd;
 		}
 	}
 
+out_free:
 	free(source);
 	free(type);
 	free(mnt_opts);
 	free(dev);
 
 	return fd;
+
+fail_close_fd:
+	close(fd);
+	fd = -1;
+	goto out_free;
 }
 
 static int send_fd(int sock_fd, int fd)
@@ -1154,6 +1171,7 @@ int main(int argc, char *argv[])
 	static int unmount = 0;
 	static int lazy = 0;
 	static int quiet = 0;
+	char *devfd;
 	char *commfd;
 	int cfd;
 	const char *opts = "";
@@ -1222,6 +1240,13 @@ int main(int argc, char *argv[])
 
 	drop_privs();
 	mnt = fuse_mnt_resolve_path(progname, origmnt);
+	if (mnt != NULL) {
+		res = chdir("/");
+		if (res == -1) {
+			fprintf(stderr, "%s: failed to chdir to '/'\n", progname);
+			exit(1);
+		}
+	}
 	restore_privs();
 	if (mnt == NULL)
 		exit(1);
@@ -1242,21 +1267,26 @@ int main(int argc, char *argv[])
 		return 0;
 	}
 
-	commfd = getenv(FUSE_COMMFD_ENV);
-	if (commfd == NULL) {
-		fprintf(stderr, "%s: old style mounting not supported\n",
-			progname);
-		exit(1);
+	devfd = getenv(FUSE_DEVFD_ENV);
+	if (devfd == NULL) {
+		commfd = getenv(FUSE_COMMFD_ENV);
+		if (commfd == NULL) {
+			fprintf(stderr, "%s: old style mounting not supported\n",
+				progname);
+			exit(1);
+		}
 	}
 
-	fd = mount_fuse(mnt, opts);
+	fd = mount_fuse(mnt, opts, devfd);
 	if (fd == -1)
 		exit(1);
 
-	cfd = atoi(commfd);
-	res = send_fd(cfd, fd);
-	if (res == -1)
-		exit(1);
+	if (devfd == NULL) {
+		cfd = atoi(commfd);
+		res = send_fd(cfd, fd);
+		if (res == -1)
+			exit(1);
+	}
 
 	return 0;
 }
diff --git a/contrib/fuse-util/mount_util.c b/contrib/fuse-util/mount_util.c
new file mode 100644
index 000000000..911b84445
--- /dev/null
+++ b/contrib/fuse-util/mount_util.c
@@ -0,0 +1,64 @@
+/*
+  FUSE: Filesystem in Userspace
+  Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
+
+  This program can be distributed under the terms of the GNU LGPLv2.
+  See the file COPYING.LIB.
+*/
+
+#include <dirent.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+int fuse_mnt_check_empty(const char *progname, const char *mnt,
+			 mode_t rootmode, off_t rootsize)
+{
+	int isempty = 1;
+
+	if (S_ISDIR(rootmode)) {
+		struct dirent *ent;
+		DIR *dp = opendir(mnt);
+		if (dp == NULL) {
+			fprintf(stderr,
+				"%s: failed to open mountpoint for reading: %s\n",
+				progname, strerror(errno));
+			return -1;
+		}
+		while ((ent = readdir(dp)) != NULL) {
+			if (strcmp(ent->d_name, ".") != 0 &&
+			    strcmp(ent->d_name, "..") != 0) {
+				isempty = 0;
+				break;
+			}
+		}
+		closedir(dp);
+	} else if (rootsize)
+		isempty = 0;
+
+	if (!isempty) {
+		fprintf(stderr, "%s: mountpoint is not empty\n", progname);
+		fprintf(stderr, "%s: if you are sure this is safe, use the 'nonempty' mount option\n", progname);
+		return -1;
+	}
+	return 0;
+}
+
+int fuse_mnt_check_fuseblk(void)
+{
+	char buf[256];
+	FILE *f = fopen("/proc/filesystems", "r");
+	if (!f)
+		return 1;
+
+	while (fgets(buf, sizeof(buf), f))
+		if (strstr(buf, "fuseblk\n")) {
+			fclose(f);
+			return 1;
+		}
+
+	fclose(f);
+	return 0;
+}
diff --git a/contrib/libgen/basename_r.c b/contrib/libgen/basename_r.c
new file mode 100644
index 000000000..2c3a87afe
--- /dev/null
+++ b/contrib/libgen/basename_r.c
@@ -0,0 +1,40 @@
+/*
+ * borrowed from glibc-2.12.1/string/basename.c
+ * Modified to return "." for NULL or "", as required for SUSv2.
+ */
+#include <string.h>
+#include <stdlib.h>
+#ifdef THREAD_UNSAFE_BASENAME
+
+/* Return the name-within-directory of a file name.
+   Copyright (C) 1996,97,98,2002 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+char *
+basename_r (filename)
+     const char *filename;
+{
+  char *p;
+
+  if ((filename == NULL) || (*filename == '\0'))
+    return ".";
+
+  p = strrchr (filename, '/');
+  return p ? p + 1 : (char *) filename;
+}
+#endif /* THREAD_UNSAFE_BASENAME */
diff --git a/contrib/libgen/dirname_r.c b/contrib/libgen/dirname_r.c
new file mode 100644
index 000000000..131cbcf2a
--- /dev/null
+++ b/contrib/libgen/dirname_r.c
@@ -0,0 +1,243 @@
+/*
+ * Borrowed from glibc-2.12.1/string/memrchr.c
+ * Based on strlen implementation by Torbjorn Granlund (tege@sics.se),
+ * Removed code for long bigger than 32 bytes, renamed __ptr_t as void *
+ * changed reg_char type to char.
+ */
+#include <string.h>
+#include <stdlib.h>
+#ifdef THREAD_UNSAFE_DIRNAME
+
+/* memrchr -- find the last occurrence of a byte in a memory block
+   Copyright (C) 1991, 93, 96, 97, 99, 2000 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Based on strlen implementation by Torbjorn Granlund (tege@sics.se),
+   with help from Dan Sahlin (dan@sics.se) and
+   commentary by Jim Blandy (jimb@ai.mit.edu);
+   adaptation to memchr suggested by Dick Karpinski (dick@cca.ucsf.edu),
+   and implemented by Roland McGrath (roland@ai.mit.edu).
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+void *
+__memrchr (s, c_in, n)
+     const void * s;
+     int c_in;
+     size_t n;
+{
+  const unsigned char *char_ptr;
+  const unsigned long int *longword_ptr;
+  unsigned long int longword, magic_bits, charmask;
+  unsigned char c;
+
+  c = (unsigned char) c_in;
+
+  /* Handle the last few characters by reading one character at a time.
+     Do this until CHAR_PTR is aligned on a longword boundary.  */
+  for (char_ptr = (const unsigned char *) s + n;
+       n > 0 && ((unsigned long int) char_ptr
+		 & (sizeof (longword) - 1)) != 0;
+       --n)
+    if (*--char_ptr == c)
+      return (void *) char_ptr;
+
+  /* All these elucidatory comments refer to 4-byte longwords,
+     but the theory applies equally well to 8-byte longwords.  */
+
+  longword_ptr = (const unsigned long int *) char_ptr;
+
+  /* Bits 31, 24, 16, and 8 of this number are zero.  Call these bits
+     the "holes."  Note that there is a hole just to the left of
+     each byte, with an extra at the end:
+
+     bits:  01111110 11111110 11111110 11111111
+     bytes: AAAAAAAA BBBBBBBB CCCCCCCC DDDDDDDD
+
+     The 1-bits make sure that carries propagate to the next 0-bit.
+     The 0-bits provide holes for carries to fall into.  */
+
+  if (sizeof (longword) != 4 && sizeof (longword) != 8)
+    abort ();
+
+  magic_bits = 0x7efefeff;
+
+  /* Set up a longword, each of whose bytes is C.  */
+  charmask = c | (c << 8);
+  charmask |= charmask << 16;
+
+  /* Instead of the traditional loop which tests each character,
+     we will test a longword at a time.  The tricky part is testing
+     if *any of the four* bytes in the longword in question are zero.  */
+  while (n >= sizeof (longword))
+    {
+      /* We tentatively exit the loop if adding MAGIC_BITS to
+	 LONGWORD fails to change any of the hole bits of LONGWORD.
+
+	 1) Is this safe?  Will it catch all the zero bytes?
+	 Suppose there is a byte with all zeros.  Any carry bits
+	 propagating from its left will fall into the hole at its
+	 least significant bit and stop.  Since there will be no
+	 carry from its most significant bit, the LSB of the
+	 byte to the left will be unchanged, and the zero will be
+	 detected.
+
+	 2) Is this worthwhile?  Will it ignore everything except
+	 zero bytes?  Suppose every byte of LONGWORD has a bit set
+	 somewhere.  There will be a carry into bit 8.  If bit 8
+	 is set, this will carry into bit 16.  If bit 8 is clear,
+	 one of bits 9-15 must be set, so there will be a carry
+	 into bit 16.  Similarly, there will be a carry into bit
+	 24.  If one of bits 24-30 is set, there will be a carry
+	 into bit 31, so all of the hole bits will be changed.
+
+	 The one misfire occurs when bits 24-30 are clear and bit
+	 31 is set; in this case, the hole at bit 31 is not
+	 changed.  If we had access to the processor carry flag,
+	 we could close this loophole by putting the fourth hole
+	 at bit 32!
+
+	 So it ignores everything except 128's, when they're aligned
+	 properly.
+
+	 3) But wait!  Aren't we looking for C, not zero?
+	 Good point.  So what we do is XOR LONGWORD with a longword,
+	 each of whose bytes is C.  This turns each byte that is C
+	 into a zero.  */
+
+      longword = *--longword_ptr ^ charmask;
+
+      /* Add MAGIC_BITS to LONGWORD.  */
+      if ((((longword + magic_bits)
+
+	    /* Set those bits that were unchanged by the addition.  */
+	    ^ ~longword)
+
+	   /* Look at only the hole bits.  If any of the hole bits
+	      are unchanged, most likely one of the bytes was a
+	      zero.  */
+	   & ~magic_bits) != 0)
+	{
+	  /* Which of the bytes was C?  If none of them were, it was
+	     a misfire; continue the search.  */
+
+	  const unsigned char *cp = (const unsigned char *) longword_ptr;
+
+	  if (cp[3] == c)
+	    return (void *) &cp[3];
+	  if (cp[2] == c)
+	    return (void *) &cp[2];
+	  if (cp[1] == c)
+	    return (void *) &cp[1];
+	  if (cp[0] == c)
+	    return (void *) cp;
+	}
+
+      n -= sizeof (longword);
+    }
+
+  char_ptr = (const unsigned char *) longword_ptr;
+
+  while (n-- > 0)
+    {
+      if (*--char_ptr == c)
+	return (void *) char_ptr;
+    }
+
+  return 0;
+}
+
+/*
+ * Borrowed from glibc-2.12.1/misc/dirname.c
+ */
+
+/* dirname - return directory part of PATH.
+   Copyright (C) 1996, 2000, 2001, 2002 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@cygnus.com>, 1996.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+char *
+dirname_r (char *path)
+{
+  static const char dot[] = ".";
+  char *last_slash;
+
+  /* Find last '/'.  */
+  last_slash = path != NULL ? strrchr (path, '/') : NULL;
+
+  if (last_slash != NULL && last_slash != path && last_slash[1] == '\0')
+    {
+      /* Determine whether all remaining characters are slashes.  */
+      char *runp;
+
+      for (runp = last_slash; runp != path; --runp)
+        if (runp[-1] != '/')
+          break;
+
+      /* The '/' is the last character, we have to look further.  */
+      if (runp != path)
+        last_slash = __memrchr (path, '/', runp - path);
+    }
+
+  if (last_slash != NULL)
+    {
+      /* Determine whether all remaining characters are slashes.  */
+      char *runp;
+
+      for (runp = last_slash; runp != path; --runp)
+        if (runp[-1] != '/')
+          break;
+
+      /* Terminate the path.  */
+      if (runp == path)
+        {
+          /* The last slash is the first character in the string.  We have to
+             return "/".  As a special case we have to return "//" if there
+             are exactly two slashes at the beginning of the string.  See
+             XBD 4.10 Path Name Resolution for more information.  */
+          if (last_slash == path + 1)
+            ++last_slash;
+          else
+            last_slash = path + 1;
+        }
+      else
+        last_slash = runp;
+
+      last_slash[0] = '\0';
+    }
+  else
+    /* This assignment is ill-designed but the XPG specs require to
+       return a string containing "." in any case no directory part is
+       found and so a static and constant string is required.  */
+    path = (char *) dot;
+
+  return path;
+}
+#endif /* THREAD_UNSAFE_DIRNAME */
diff --git a/contrib/md5/md5.c b/contrib/md5/md5.c
deleted file mode 100644
index 5f0d0d157..000000000
--- a/contrib/md5/md5.c
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * RFC 1321 compliant MD5 implementation
- *
- * Copyright (C) 2001-2003 Christophe Devine
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, visit the http://fsf.org website.
- */
-
-
-#include <inttypes.h>
-#include <string.h>
-
-#include "md5.h"
-
-void md5_begin(md_context *ctx)
-{
-	ctx->A = 0x67452301;
-	ctx->B = 0xEFCDAB89;
-	ctx->C = 0x98BADCFE;
-	ctx->D = 0x10325476;
-
-	ctx->totalN = ctx->totalN2 = 0;
-}
-
-static void md5_process(md_context *ctx, const uint8_t data[CSUM_CHUNK])
-{
-	uint32_t X[16], A, B, C, D;
-
-	A = ctx->A;
-	B = ctx->B;
-	C = ctx->C;
-	D = ctx->D;
-
-	X[0] = IVAL(data, 0);
-	X[1] = IVAL(data, 4);
-	X[2] = IVAL(data, 8);
-	X[3] = IVAL(data, 12);
-	X[4] = IVAL(data, 16);
-	X[5] = IVAL(data, 20);
-	X[6] = IVAL(data, 24);
-	X[7] = IVAL(data, 28);
-	X[8] = IVAL(data, 32);
-	X[9] = IVAL(data, 36);
-	X[10] = IVAL(data, 40);
-	X[11] = IVAL(data, 44);
-	X[12] = IVAL(data, 48);
-	X[13] = IVAL(data, 52);
-	X[14] = IVAL(data, 56);
-	X[15] = IVAL(data, 60);
-
-#define S(x,n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n)))
-
-#define P(a,b,c,d,k,s,t) a += F(b,c,d) + X[k] + t, a = S(a,s) + b
-
-#define F(x,y,z) (z ^ (x & (y ^ z)))
-
-	P(A, B, C, D,  0,  7, 0xD76AA478);
-	P(D, A, B, C,  1, 12, 0xE8C7B756);
-	P(C, D, A, B,  2, 17, 0x242070DB);
-	P(B, C, D, A,  3, 22, 0xC1BDCEEE);
-	P(A, B, C, D,  4,  7, 0xF57C0FAF);
-	P(D, A, B, C,  5, 12, 0x4787C62A);
-	P(C, D, A, B,  6, 17, 0xA8304613);
-	P(B, C, D, A,  7, 22, 0xFD469501);
-	P(A, B, C, D,  8,  7, 0x698098D8);
-	P(D, A, B, C,  9, 12, 0x8B44F7AF);
-	P(C, D, A, B, 10, 17, 0xFFFF5BB1);
-	P(B, C, D, A, 11, 22, 0x895CD7BE);
-	P(A, B, C, D, 12,  7, 0x6B901122);
-	P(D, A, B, C, 13, 12, 0xFD987193);
-	P(C, D, A, B, 14, 17, 0xA679438E);
-	P(B, C, D, A, 15, 22, 0x49B40821);
-
-#undef F
-#define F(x,y,z) (y ^ (z & (x ^ y)))
-
-	P(A, B, C, D,  1,  5, 0xF61E2562);
-	P(D, A, B, C,  6,  9, 0xC040B340);
-	P(C, D, A, B, 11, 14, 0x265E5A51);
-	P(B, C, D, A,  0, 20, 0xE9B6C7AA);
-	P(A, B, C, D,  5,  5, 0xD62F105D);
-	P(D, A, B, C, 10,  9, 0x02441453);
-	P(C, D, A, B, 15, 14, 0xD8A1E681);
-	P(B, C, D, A,  4, 20, 0xE7D3FBC8);
-	P(A, B, C, D,  9,  5, 0x21E1CDE6);
-	P(D, A, B, C, 14,  9, 0xC33707D6);
-	P(C, D, A, B,  3, 14, 0xF4D50D87);
-	P(B, C, D, A,  8, 20, 0x455A14ED);
-	P(A, B, C, D, 13,  5, 0xA9E3E905);
-	P(D, A, B, C,  2,  9, 0xFCEFA3F8);
-	P(C, D, A, B,  7, 14, 0x676F02D9);
-	P(B, C, D, A, 12, 20, 0x8D2A4C8A);
-
-#undef F
-#define F(x,y,z) (x ^ y ^ z)
-
-	P(A, B, C, D,  5,  4, 0xFFFA3942);
-	P(D, A, B, C,  8, 11, 0x8771F681);
-	P(C, D, A, B, 11, 16, 0x6D9D6122);
-	P(B, C, D, A, 14, 23, 0xFDE5380C);
-	P(A, B, C, D,  1,  4, 0xA4BEEA44);
-	P(D, A, B, C,  4, 11, 0x4BDECFA9);
-	P(C, D, A, B,  7, 16, 0xF6BB4B60);
-	P(B, C, D, A, 10, 23, 0xBEBFBC70);
-	P(A, B, C, D, 13,  4, 0x289B7EC6);
-	P(D, A, B, C,  0, 11, 0xEAA127FA);
-	P(C, D, A, B,  3, 16, 0xD4EF3085);
-	P(B, C, D, A,  6, 23, 0x04881D05);
-	P(A, B, C, D,  9,  4, 0xD9D4D039);
-	P(D, A, B, C, 12, 11, 0xE6DB99E5);
-	P(C, D, A, B, 15, 16, 0x1FA27CF8);
-	P(B, C, D, A,  2, 23, 0xC4AC5665);
-
-#undef F
-#define F(x,y,z) (y ^ (x | ~z))
-
-	P(A, B, C, D,  0,  6, 0xF4292244);
-	P(D, A, B, C,  7, 10, 0x432AFF97);
-	P(C, D, A, B, 14, 15, 0xAB9423A7);
-	P(B, C, D, A,  5, 21, 0xFC93A039);
-	P(A, B, C, D, 12,  6, 0x655B59C3);
-	P(D, A, B, C,  3, 10, 0x8F0CCC92);
-	P(C, D, A, B, 10, 15, 0xFFEFF47D);
-	P(B, C, D, A,  1, 21, 0x85845DD1);
-	P(A, B, C, D,  8,  6, 0x6FA87E4F);
-	P(D, A, B, C, 15, 10, 0xFE2CE6E0);
-	P(C, D, A, B,  6, 15, 0xA3014314);
-	P(B, C, D, A, 13, 21, 0x4E0811A1);
-	P(A, B, C, D,  4,  6, 0xF7537E82);
-	P(D, A, B, C, 11, 10, 0xBD3AF235);
-	P(C, D, A, B,  2, 15, 0x2AD7D2BB);
-	P(B, C, D, A,  9, 21, 0xEB86D391);
-
-#undef F
-
-	ctx->A += A;
-	ctx->B += B;
-	ctx->C += C;
-	ctx->D += D;
-}
-
-void md5_update(md_context *ctx, const uint8_t *input, uint32_t length)
-{
-	uint32_t left, fill;
-
-	if (!length)
-		return;
-
-	left = ctx->totalN & 0x3F;
-	fill = CSUM_CHUNK - left;
-
-	ctx->totalN += length;
-	ctx->totalN &= 0xFFFFFFFF;
-
-	if (ctx->totalN < length)
-		ctx->totalN2++;
-
-	if (left && length >= fill) {
-		memcpy(ctx->buffer + left, input, fill);
-		md5_process(ctx, ctx->buffer);
-		length -= fill;
-		input  += fill;
-		left = 0;
-	}
-
-	while (length >= CSUM_CHUNK) {
-		md5_process(ctx, input);
-		length -= CSUM_CHUNK;
-		input  += CSUM_CHUNK;
-	}
-
-	if (length)
-		memcpy(ctx->buffer + left, input, length);
-}
-
-static uint8_t md5_padding[CSUM_CHUNK] = { 0x80 };
-
-void md5_result(md_context *ctx, uint8_t digest[MD5_DIGEST_LEN])
-{
-	uint32_t last, padn;
-	uint32_t high, low;
-	uint8_t msglen[8];
-
-	high = (ctx->totalN >> 29)
-	     | (ctx->totalN2 <<  3);
-	low  = (ctx->totalN <<  3);
-
-	SIVAL(msglen, 0, low);
-	SIVAL(msglen, 4, high);
-
-	last = ctx->totalN & 0x3F;
-	padn = last < 56 ? 56 - last : 120 - last;
-
-	md5_update(ctx, md5_padding, padn);
-	md5_update(ctx, msglen, 8);
-
-	SIVAL(digest, 0, ctx->A);
-	SIVAL(digest, 4, ctx->B);
-	SIVAL(digest, 8, ctx->C);
-	SIVAL(digest, 12, ctx->D);
-}
-
-void get_md5(uint8_t *out, const uint8_t *input, int n)
-{
-	md_context ctx;
-	md5_begin(&ctx);
-	md5_update(&ctx, input, n);
-	md5_result(&ctx, out);
-}
-
-#ifdef TEST_MD5
-
-#include <stdlib.h>
-#include <stdio.h>
-
-/*
- * those are the standard RFC 1321 test vectors
- */
-
-static struct {
-    char *str, *md5;
-} tests[] = {
- { "",
-   "d41d8cd98f00b204e9800998ecf8427e" },
- { "a",
-   "0cc175b9c0f1b6a831c399e269772661" },
- { "abc",
-   "900150983cd24fb0d6963f7d28e17f72" },
- { "message digest",
-   "f96b697d7cb7938d525a2f31aaf161d0" },
- { "abcdefghijklmnopqrstuvwxyz",
-   "c3fcd3d76192e4007dfb496cca67e13b" },
- { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
-   "d174ab98d277d9f5a5611c2c9f419d9f" },
- { "12345678901234567890123456789012345678901234567890123456789012345678901234567890",
-   "57edf4a22be3c955ac49da2e2107b67a" },
- { NULL, NULL }
-};
-
-int main(int argc, char *argv[])
-{
-	FILE *f;
-	int i, j;
-	char output[33];
-	md_context ctx;
-	uint8_t buf[1000];
-	uint8_t md5sum[MD5_DIGEST_LEN];
-
-	if (argc < 2) {
-		printf("\nMD5 Validation Tests:\n\n");
-
-		for (i = 0; tests[i].str; i++) {
-			char *str = tests[i].str;
-			char *chk = tests[i].md5;
-
-			printf("  Test %d ", i + 1);
-
-			get_md5(md5sum, str, strlen(str));
-
-			for (j = 0; j < MD5_DIGEST_LEN; j++)
-				sprintf(output + j * 2, "%02x", md5sum[j]);
-
-			if (memcmp(output, chk, 32)) {
-				printf("failed!\n");
-				return 1;
-			}
-
-			printf("passed.\n");
-		}
-
-		printf("\n");
-		return 0;
-	}
-
-	while (--argc) {
-		if (!(f = fopen(*++argv, "rb"))) {
-			perror("fopen");
-			return 1;
-		}
-
-		md5_begin(&ctx);
-
-		while ((i = fread(buf, 1, sizeof buf, f)) > 0)
-			md5_update(&ctx, buf, i);
-
-		fclose(f);
-
-		md5_result(&ctx, md5sum);
-
-		for (j = 0; j < MD5_DIGEST_LEN; j++)
-			printf("%02x", md5sum[j]);
-
-		printf("  %s\n", *argv);
-	}
-
-	return 0;
-}
-
-#endif
diff --git a/contrib/md5/md5.h b/contrib/md5/md5.h
deleted file mode 100644
index ba8f08dbc..000000000
--- a/contrib/md5/md5.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/* rsync-3.0.6/byteorder.h */
-
-/*
- * Simple byteorder handling.
- *
- * Copyright (C) 1992-1995 Andrew Tridgell
- * Copyright (C) 2007-2008 Wayne Davison
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, visit the http://fsf.org website.
- */
-
-#undef CAREFUL_ALIGNMENT
-
-/* We know that the x86 can handle misalignment and has the same
- * byte order (LSB-first) as the 32-bit numbers we transmit. */
-
-#ifdef __i386__
-#define CAREFUL_ALIGNMENT 0
-#endif
-
-#ifndef CAREFUL_ALIGNMENT
-#define CAREFUL_ALIGNMENT 1
-#endif
-
-#define CVAL(buf,pos) (((unsigned char *)(buf))[pos])
-#define UVAL(buf,pos) ((uint32_t)CVAL(buf,pos))
-#define SCVAL(buf,pos,val) (CVAL(buf,pos) = (val))
-
-#if CAREFUL_ALIGNMENT
-#define PVAL(buf,pos) (UVAL(buf,pos)|UVAL(buf,(pos)+1)<<8)
-#define IVAL(buf,pos) (PVAL(buf,pos)|PVAL(buf,(pos)+2)<<16)
-#define SSVALX(buf,pos,val) (CVAL(buf,pos)=(val)&0xFF,CVAL(buf,pos+1)=(val)>>8)
-#define SIVALX(buf,pos,val) (SSVALX(buf,pos,val&0xFFFF),SSVALX(buf,pos+2,val>>16))
-#define SIVAL(buf,pos,val) SIVALX((buf),(pos),((uint32_t)(val)))
-#else
-
-/* this handles things for architectures like the 386 that can handle
-   alignment errors */
-
-/*
-   WARNING: This section is dependent on the length of int32
-   being correct. set CAREFUL_ALIGNMENT if it is not.
-*/
-
-#define IVAL(buf,pos) (*(uint32_t *)((char *)(buf) + (pos)))
-#define SIVAL(buf,pos,val) IVAL(buf,pos)=((uint32_t)(val))
-#endif
-
-/* The include file for both the MD4 and MD5 routines. */
-
-#define MD5_DIGEST_LEN 16
-#define MAX_DIGEST_LEN MD5_DIGEST_LEN
-
-#define CSUM_CHUNK 64
-
-typedef struct {
-	uint32_t A, B, C, D;
-	uint32_t totalN;          /* bit count, lower 32 bits */
-	uint32_t totalN2;         /* bit count, upper 32 bits */
-	uint8_t buffer[CSUM_CHUNK];
-} md_context;
-
-void md5_begin(md_context *ctx);
-void md5_update(md_context *ctx, const uint8_t *input, uint32_t length);
-void md5_result(md_context *ctx, uint8_t digest[MD5_DIGEST_LEN]);
-
-void get_md5(uint8_t digest[MD5_DIGEST_LEN], const uint8_t *input, int n);
diff --git a/contrib/qemu/block.c b/contrib/qemu/block.c
new file mode 100644
index 000000000..b56024113
--- /dev/null
+++ b/contrib/qemu/block.c
@@ -0,0 +1,4604 @@
+/*
+ * QEMU System Emulator block driver
+ *
+ * Copyright (c) 2003 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "config-host.h"
+#include "qemu-common.h"
+#include "trace.h"
+#include "monitor/monitor.h"
+#include "block/block_int.h"
+#include "block/blockjob.h"
+#include "qemu/module.h"
+#include "qapi/qmp/qjson.h"
+#include "sysemu/sysemu.h"
+#include "qemu/notify.h"
+#include "block/coroutine.h"
+#include "qmp-commands.h"
+#include "qemu/timer.h"
+
+#ifdef CONFIG_BSD
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/queue.h>
+#ifndef __DragonFly__
+#include <sys/disk.h>
+#endif
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
+
+typedef enum {
+    BDRV_REQ_COPY_ON_READ = 0x1,
+    BDRV_REQ_ZERO_WRITE   = 0x2,
+} BdrvRequestFlags;
+
+static void bdrv_dev_change_media_cb(BlockDriverState *bs, bool load);
+static BlockDriverAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque);
+static BlockDriverAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque);
+static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
+                                         int64_t sector_num, int nb_sectors,
+                                         QEMUIOVector *iov);
+static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs,
+                                         int64_t sector_num, int nb_sectors,
+                                         QEMUIOVector *iov);
+static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
+    BdrvRequestFlags flags);
+static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
+    BdrvRequestFlags flags);
+static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
+                                               int64_t sector_num,
+                                               QEMUIOVector *qiov,
+                                               int nb_sectors,
+                                               BlockDriverCompletionFunc *cb,
+                                               void *opaque,
+                                               bool is_write);
+static void coroutine_fn bdrv_co_do_rw(void *opaque);
+static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors);
+
+static bool bdrv_exceed_bps_limits(BlockDriverState *bs, int nb_sectors,
+        bool is_write, double elapsed_time, uint64_t *wait);
+static bool bdrv_exceed_iops_limits(BlockDriverState *bs, bool is_write,
+        double elapsed_time, uint64_t *wait);
+static bool bdrv_exceed_io_limits(BlockDriverState *bs, int nb_sectors,
+        bool is_write, int64_t *wait);
+
+static QTAILQ_HEAD(, BlockDriverState) bdrv_states =
+    QTAILQ_HEAD_INITIALIZER(bdrv_states);
+
+static QLIST_HEAD(, BlockDriver) bdrv_drivers =
+    QLIST_HEAD_INITIALIZER(bdrv_drivers);
+
+/* If non-zero, use only whitelisted block drivers */
+static int use_bdrv_whitelist;
+
+#ifdef _WIN32
+static int is_windows_drive_prefix(const char *filename)
+{
+    return (((filename[0] >= 'a' && filename[0] <= 'z') ||
+             (filename[0] >= 'A' && filename[0] <= 'Z')) &&
+            filename[1] == ':');
+}
+
+int is_windows_drive(const char *filename)
+{
+    if (is_windows_drive_prefix(filename) &&
+        filename[2] == '\0')
+        return 1;
+    if (strstart(filename, "\\\\.\\", NULL) ||
+        strstart(filename, "//./", NULL))
+        return 1;
+    return 0;
+}
+#endif
+
+/* throttling disk I/O limits */
+void bdrv_io_limits_disable(BlockDriverState *bs)
+{
+    bs->io_limits_enabled = false;
+
+    while (qemu_co_queue_next(&bs->throttled_reqs));
+
+    if (bs->block_timer) {
+        qemu_del_timer(bs->block_timer);
+        qemu_free_timer(bs->block_timer);
+        bs->block_timer = NULL;
+    }
+
+    bs->slice_start = 0;
+    bs->slice_end   = 0;
+}
+
+static void bdrv_block_timer(void *opaque)
+{
+    BlockDriverState *bs = opaque;
+
+    qemu_co_queue_next(&bs->throttled_reqs);
+}
+
+void bdrv_io_limits_enable(BlockDriverState *bs)
+{
+    qemu_co_queue_init(&bs->throttled_reqs);
+    bs->block_timer = qemu_new_timer_ns(vm_clock, bdrv_block_timer, bs);
+    bs->io_limits_enabled = true;
+}
+
+bool bdrv_io_limits_enabled(BlockDriverState *bs)
+{
+    BlockIOLimit *io_limits = &bs->io_limits;
+    return io_limits->bps[BLOCK_IO_LIMIT_READ]
+         || io_limits->bps[BLOCK_IO_LIMIT_WRITE]
+         || io_limits->bps[BLOCK_IO_LIMIT_TOTAL]
+         || io_limits->iops[BLOCK_IO_LIMIT_READ]
+         || io_limits->iops[BLOCK_IO_LIMIT_WRITE]
+         || io_limits->iops[BLOCK_IO_LIMIT_TOTAL];
+}
+
+static void bdrv_io_limits_intercept(BlockDriverState *bs,
+                                     bool is_write, int nb_sectors)
+{
+    int64_t wait_time = -1;
+
+    if (!qemu_co_queue_empty(&bs->throttled_reqs)) {
+        qemu_co_queue_wait(&bs->throttled_reqs);
+    }
+
+    /* In fact, we hope to keep each request's timing, in FIFO mode. The next
+     * throttled requests will not be dequeued until the current request is
+     * allowed to be serviced. So if the current request still exceeds the
+     * limits, it will be inserted to the head. All requests followed it will
+     * be still in throttled_reqs queue.
+     */
+
+    while (bdrv_exceed_io_limits(bs, nb_sectors, is_write, &wait_time)) {
+        qemu_mod_timer(bs->block_timer,
+                       wait_time + qemu_get_clock_ns(vm_clock));
+        qemu_co_queue_wait_insert_head(&bs->throttled_reqs);
+    }
+
+    qemu_co_queue_next(&bs->throttled_reqs);
+}
+
+/* check if the path starts with "<protocol>:" */
+static int path_has_protocol(const char *path)
+{
+    const char *p;
+
+#ifdef _WIN32
+    if (is_windows_drive(path) ||
+        is_windows_drive_prefix(path)) {
+        return 0;
+    }
+    p = path + strcspn(path, ":/\\");
+#else
+    p = path + strcspn(path, ":/");
+#endif
+
+    return *p == ':';
+}
+
+int path_is_absolute(const char *path)
+{
+#ifdef _WIN32
+    /* specific case for names like: "\\.\d:" */
+    if (is_windows_drive(path) || is_windows_drive_prefix(path)) {
+        return 1;
+    }
+    return (*path == '/' || *path == '\\');
+#else
+    return (*path == '/');
+#endif
+}
+
+/* if filename is absolute, just copy it to dest. Otherwise, build a
+   path to it by considering it is relative to base_path. URL are
+   supported. */
+void path_combine(char *dest, int dest_size,
+                  const char *base_path,
+                  const char *filename)
+{
+    const char *p, *p1;
+    int len;
+
+    if (dest_size <= 0)
+        return;
+    if (path_is_absolute(filename)) {
+        pstrcpy(dest, dest_size, filename);
+    } else {
+        p = strchr(base_path, ':');
+        if (p)
+            p++;
+        else
+            p = base_path;
+        p1 = strrchr(base_path, '/');
+#ifdef _WIN32
+        {
+            const char *p2;
+            p2 = strrchr(base_path, '\\');
+            if (!p1 || p2 > p1)
+                p1 = p2;
+        }
+#endif
+        if (p1)
+            p1++;
+        else
+            p1 = base_path;
+        if (p1 > p)
+            p = p1;
+        len = p - base_path;
+        if (len > dest_size - 1)
+            len = dest_size - 1;
+        memcpy(dest, base_path, len);
+        dest[len] = '\0';
+        pstrcat(dest, dest_size, filename);
+    }
+}
+
+void bdrv_get_full_backing_filename(BlockDriverState *bs, char *dest, size_t sz)
+{
+    if (bs->backing_file[0] == '\0' || path_has_protocol(bs->backing_file)) {
+        pstrcpy(dest, sz, bs->backing_file);
+    } else {
+        path_combine(dest, sz, bs->filename, bs->backing_file);
+    }
+}
+
+void bdrv_register(BlockDriver *bdrv)
+{
+    /* Block drivers without coroutine functions need emulation */
+    if (!bdrv->bdrv_co_readv) {
+        bdrv->bdrv_co_readv = bdrv_co_readv_em;
+        bdrv->bdrv_co_writev = bdrv_co_writev_em;
+
+        /* bdrv_co_readv_em()/brdv_co_writev_em() work in terms of aio, so if
+         * the block driver lacks aio we need to emulate that too.
+         */
+        if (!bdrv->bdrv_aio_readv) {
+            /* add AIO emulation layer */
+            bdrv->bdrv_aio_readv = bdrv_aio_readv_em;
+            bdrv->bdrv_aio_writev = bdrv_aio_writev_em;
+        }
+    }
+
+    QLIST_INSERT_HEAD(&bdrv_drivers, bdrv, list);
+}
+
+/* create a new block device (by default it is empty) */
+BlockDriverState *bdrv_new(const char *device_name)
+{
+    BlockDriverState *bs;
+
+    bs = g_malloc0(sizeof(BlockDriverState));
+    pstrcpy(bs->device_name, sizeof(bs->device_name), device_name);
+    if (device_name[0] != '\0') {
+        QTAILQ_INSERT_TAIL(&bdrv_states, bs, list);
+    }
+    bdrv_iostatus_disable(bs);
+    notifier_list_init(&bs->close_notifiers);
+    notifier_with_return_list_init(&bs->before_write_notifiers);
+
+    return bs;
+}
+
+void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify)
+{
+    notifier_list_add(&bs->close_notifiers, notify);
+}
+
+BlockDriver *bdrv_find_format(const char *format_name)
+{
+    BlockDriver *drv1;
+    QLIST_FOREACH(drv1, &bdrv_drivers, list) {
+        if (!strcmp(drv1->format_name, format_name)) {
+            return drv1;
+        }
+    }
+    return NULL;
+}
+
+static int bdrv_is_whitelisted(BlockDriver *drv, bool read_only)
+{
+    static const char *whitelist_rw[] = {
+        CONFIG_BDRV_RW_WHITELIST
+    };
+    static const char *whitelist_ro[] = {
+        CONFIG_BDRV_RO_WHITELIST
+    };
+    const char **p;
+
+    if (!whitelist_rw[0] && !whitelist_ro[0]) {
+        return 1;               /* no whitelist, anything goes */
+    }
+
+    for (p = whitelist_rw; *p; p++) {
+        if (!strcmp(drv->format_name, *p)) {
+            return 1;
+        }
+    }
+    if (read_only) {
+        for (p = whitelist_ro; *p; p++) {
+            if (!strcmp(drv->format_name, *p)) {
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
+                                          bool read_only)
+{
+    BlockDriver *drv = bdrv_find_format(format_name);
+    return drv && bdrv_is_whitelisted(drv, read_only) ? drv : NULL;
+}
+
+typedef struct CreateCo {
+    BlockDriver *drv;
+    char *filename;
+    QEMUOptionParameter *options;
+    int ret;
+} CreateCo;
+
+static void coroutine_fn bdrv_create_co_entry(void *opaque)
+{
+    CreateCo *cco = opaque;
+    assert(cco->drv);
+
+    cco->ret = cco->drv->bdrv_create(cco->filename, cco->options);
+}
+
+int bdrv_create(BlockDriver *drv, const char* filename,
+    QEMUOptionParameter *options)
+{
+    int ret;
+
+    Coroutine *co;
+    CreateCo cco = {
+        .drv = drv,
+        .filename = g_strdup(filename),
+        .options = options,
+        .ret = NOT_DONE,
+    };
+
+    if (!drv->bdrv_create) {
+        ret = -ENOTSUP;
+        goto out;
+    }
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_create_co_entry(&cco);
+    } else {
+        co = qemu_coroutine_create(bdrv_create_co_entry);
+        qemu_coroutine_enter(co, &cco);
+        while (cco.ret == NOT_DONE) {
+            qemu_aio_wait();
+        }
+    }
+
+    ret = cco.ret;
+
+out:
+    g_free(cco.filename);
+    return ret;
+}
+
+int bdrv_create_file(const char* filename, QEMUOptionParameter *options)
+{
+    BlockDriver *drv;
+
+    drv = bdrv_find_protocol(filename, true);
+    if (drv == NULL) {
+        return -ENOENT;
+    }
+
+    return bdrv_create(drv, filename, options);
+}
+
+/*
+ * Create a uniquely-named empty temporary file.
+ * Return 0 upon success, otherwise a negative errno value.
+ */
+int get_tmp_filename(char *filename, int size)
+{
+#ifdef _WIN32
+    char temp_dir[MAX_PATH];
+    /* GetTempFileName requires that its output buffer (4th param)
+       have length MAX_PATH or greater.  */
+    assert(size >= MAX_PATH);
+    return (GetTempPath(MAX_PATH, temp_dir)
+            && GetTempFileName(temp_dir, "qem", 0, filename)
+            ? 0 : -GetLastError());
+#else
+    int fd;
+    const char *tmpdir;
+    tmpdir = getenv("TMPDIR");
+    if (!tmpdir)
+        tmpdir = "/tmp";
+    if (snprintf(filename, size, "%s/vl.XXXXXX", tmpdir) >= size) {
+        return -EOVERFLOW;
+    }
+    fd = mkstemp(filename);
+    if (fd < 0) {
+        return -errno;
+    }
+    if (close(fd) != 0) {
+        unlink(filename);
+        return -errno;
+    }
+    return 0;
+#endif
+}
+
+/*
+ * Detect host devices. By convention, /dev/cdrom[N] is always
+ * recognized as a host CDROM.
+ */
+static BlockDriver *find_hdev_driver(const char *filename)
+{
+    int score_max = 0, score;
+    BlockDriver *drv = NULL, *d;
+
+    QLIST_FOREACH(d, &bdrv_drivers, list) {
+        if (d->bdrv_probe_device) {
+            score = d->bdrv_probe_device(filename);
+            if (score > score_max) {
+                score_max = score;
+                drv = d;
+            }
+        }
+    }
+
+    return drv;
+}
+
+BlockDriver *bdrv_find_protocol(const char *filename,
+                                bool allow_protocol_prefix)
+{
+    BlockDriver *drv1;
+    char protocol[128];
+    int len;
+    const char *p;
+
+    /* TODO Drivers without bdrv_file_open must be specified explicitly */
+
+    /*
+     * XXX(hch): we really should not let host device detection
+     * override an explicit protocol specification, but moving this
+     * later breaks access to device names with colons in them.
+     * Thanks to the brain-dead persistent naming schemes on udev-
+     * based Linux systems those actually are quite common.
+     */
+    drv1 = find_hdev_driver(filename);
+    if (drv1) {
+        return drv1;
+    }
+
+    if (!path_has_protocol(filename) || !allow_protocol_prefix) {
+        return bdrv_find_format("file");
+    }
+
+    p = strchr(filename, ':');
+    assert(p != NULL);
+    len = p - filename;
+    if (len > sizeof(protocol) - 1)
+        len = sizeof(protocol) - 1;
+    memcpy(protocol, filename, len);
+    protocol[len] = '\0';
+    QLIST_FOREACH(drv1, &bdrv_drivers, list) {
+        if (drv1->protocol_name &&
+            !strcmp(drv1->protocol_name, protocol)) {
+            return drv1;
+        }
+    }
+    return NULL;
+}
+
+static int find_image_format(BlockDriverState *bs, const char *filename,
+                             BlockDriver **pdrv)
+{
+    int score, score_max;
+    BlockDriver *drv1, *drv;
+    uint8_t buf[2048];
+    int ret = 0;
+
+    /* Return the raw BlockDriver * to scsi-generic devices or empty drives */
+    if (bs->sg || !bdrv_is_inserted(bs) || bdrv_getlength(bs) == 0) {
+        drv = bdrv_find_format("raw");
+        if (!drv) {
+            ret = -ENOENT;
+        }
+        *pdrv = drv;
+        return ret;
+    }
+
+    ret = bdrv_pread(bs, 0, buf, sizeof(buf));
+    if (ret < 0) {
+        *pdrv = NULL;
+        return ret;
+    }
+
+    score_max = 0;
+    drv = NULL;
+    QLIST_FOREACH(drv1, &bdrv_drivers, list) {
+        if (drv1->bdrv_probe) {
+            score = drv1->bdrv_probe(buf, ret, filename);
+            if (score > score_max) {
+                score_max = score;
+                drv = drv1;
+            }
+        }
+    }
+    if (!drv) {
+        ret = -ENOENT;
+    }
+    *pdrv = drv;
+    return ret;
+}
+
+/**
+ * Set the current 'total_sectors' value
+ */
+static int refresh_total_sectors(BlockDriverState *bs, int64_t hint)
+{
+    BlockDriver *drv = bs->drv;
+
+    /* Do not attempt drv->bdrv_getlength() on scsi-generic devices */
+    if (bs->sg)
+        return 0;
+
+    /* query actual device if possible, otherwise just trust the hint */
+    if (drv->bdrv_getlength) {
+        int64_t length = drv->bdrv_getlength(bs);
+        if (length < 0) {
+            return length;
+        }
+        hint = length >> BDRV_SECTOR_BITS;
+    }
+
+    bs->total_sectors = hint;
+    return 0;
+}
+
+/**
+ * Set open flags for a given discard mode
+ *
+ * Return 0 on success, -1 if the discard mode was invalid.
+ */
+int bdrv_parse_discard_flags(const char *mode, int *flags)
+{
+    *flags &= ~BDRV_O_UNMAP;
+
+    if (!strcmp(mode, "off") || !strcmp(mode, "ignore")) {
+        /* do nothing */
+    } else if (!strcmp(mode, "on") || !strcmp(mode, "unmap")) {
+        *flags |= BDRV_O_UNMAP;
+    } else {
+        return -1;
+    }
+
+    return 0;
+}
+
+/**
+ * Set open flags for a given cache mode
+ *
+ * Return 0 on success, -1 if the cache mode was invalid.
+ */
+int bdrv_parse_cache_flags(const char *mode, int *flags)
+{
+    *flags &= ~BDRV_O_CACHE_MASK;
+
+    if (!strcmp(mode, "off") || !strcmp(mode, "none")) {
+        *flags |= BDRV_O_NOCACHE | BDRV_O_CACHE_WB;
+    } else if (!strcmp(mode, "directsync")) {
+        *flags |= BDRV_O_NOCACHE;
+    } else if (!strcmp(mode, "writeback")) {
+        *flags |= BDRV_O_CACHE_WB;
+    } else if (!strcmp(mode, "unsafe")) {
+        *flags |= BDRV_O_CACHE_WB;
+        *flags |= BDRV_O_NO_FLUSH;
+    } else if (!strcmp(mode, "writethrough")) {
+        /* this is the default */
+    } else {
+        return -1;
+    }
+
+    return 0;
+}
+
+/**
+ * The copy-on-read flag is actually a reference count so multiple users may
+ * use the feature without worrying about clobbering its previous state.
+ * Copy-on-read stays enabled until all users have called to disable it.
+ */
+void bdrv_enable_copy_on_read(BlockDriverState *bs)
+{
+    bs->copy_on_read++;
+}
+
+void bdrv_disable_copy_on_read(BlockDriverState *bs)
+{
+    assert(bs->copy_on_read > 0);
+    bs->copy_on_read--;
+}
+
+static int bdrv_open_flags(BlockDriverState *bs, int flags)
+{
+    int open_flags = flags | BDRV_O_CACHE_WB;
+
+    /*
+     * Clear flags that are internal to the block layer before opening the
+     * image.
+     */
+    open_flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
+
+    /*
+     * Snapshots should be writable.
+     */
+    if (bs->is_temporary) {
+        open_flags |= BDRV_O_RDWR;
+    }
+
+    return open_flags;
+}
+
+/*
+ * Common part for opening disk images and files
+ *
+ * Removes all processed options from *options.
+ */
+static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
+    QDict *options, int flags, BlockDriver *drv)
+{
+    int ret, open_flags;
+    const char *filename;
+
+    assert(drv != NULL);
+    assert(bs->file == NULL);
+    assert(options != NULL && bs->options != options);
+
+    if (file != NULL) {
+        filename = file->filename;
+    } else {
+        filename = qdict_get_try_str(options, "filename");
+    }
+
+    trace_bdrv_open_common(bs, filename ?: "", flags, drv->format_name);
+
+    /* bdrv_open() with directly using a protocol as drv. This layer is already
+     * opened, so assign it to bs (while file becomes a closed BlockDriverState)
+     * and return immediately. */
+    if (file != NULL && drv->bdrv_file_open) {
+        bdrv_swap(file, bs);
+        return 0;
+    }
+
+    bs->open_flags = flags;
+    bs->buffer_alignment = 512;
+    open_flags = bdrv_open_flags(bs, flags);
+    bs->read_only = !(open_flags & BDRV_O_RDWR);
+
+    if (use_bdrv_whitelist && !bdrv_is_whitelisted(drv, bs->read_only)) {
+        return -ENOTSUP;
+    }
+
+    assert(bs->copy_on_read == 0); /* bdrv_new() and bdrv_close() make it so */
+    if (!bs->read_only && (flags & BDRV_O_COPY_ON_READ)) {
+        bdrv_enable_copy_on_read(bs);
+    }
+
+    if (filename != NULL) {
+        pstrcpy(bs->filename, sizeof(bs->filename), filename);
+    } else {
+        bs->filename[0] = '\0';
+    }
+
+    bs->drv = drv;
+    bs->opaque = g_malloc0(drv->instance_size);
+
+    bs->enable_write_cache = !!(flags & BDRV_O_CACHE_WB);
+
+    /* Open the image, either directly or using a protocol */
+    if (drv->bdrv_file_open) {
+        assert(file == NULL);
+        assert(drv->bdrv_parse_filename || filename != NULL);
+        ret = drv->bdrv_file_open(bs, options, open_flags);
+    } else {
+        if (file == NULL) {
+            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Can't use '%s' as a "
+                          "block driver for the protocol level",
+                          drv->format_name);
+            ret = -EINVAL;
+            goto free_and_fail;
+        }
+        assert(file != NULL);
+        bs->file = file;
+        ret = drv->bdrv_open(bs, options, open_flags);
+    }
+
+    if (ret < 0) {
+        goto free_and_fail;
+    }
+
+    ret = refresh_total_sectors(bs, bs->total_sectors);
+    if (ret < 0) {
+        goto free_and_fail;
+    }
+
+#ifndef _WIN32
+    if (bs->is_temporary) {
+        assert(filename != NULL);
+        unlink(filename);
+    }
+#endif
+    return 0;
+
+free_and_fail:
+    bs->file = NULL;
+    g_free(bs->opaque);
+    bs->opaque = NULL;
+    bs->drv = NULL;
+    return ret;
+}
+
+/*
+ * Opens a file using a protocol (file, host_device, nbd, ...)
+ *
+ * options is a QDict of options to pass to the block drivers, or NULL for an
+ * empty set of options. The reference to the QDict belongs to the block layer
+ * after the call (even on failure), so if the caller intends to reuse the
+ * dictionary, it needs to use QINCREF() before calling bdrv_file_open.
+ */
+int bdrv_file_open(BlockDriverState **pbs, const char *filename,
+                   QDict *options, int flags)
+{
+    BlockDriverState *bs;
+    BlockDriver *drv;
+    const char *drvname;
+    bool allow_protocol_prefix = false;
+    int ret;
+
+    /* NULL means an empty set of options */
+    if (options == NULL) {
+        options = qdict_new();
+    }
+
+    bs = bdrv_new("");
+    bs->options = options;
+    options = qdict_clone_shallow(options);
+
+    /* Fetch the file name from the options QDict if necessary */
+    if (!filename) {
+        filename = qdict_get_try_str(options, "filename");
+    } else if (filename && !qdict_haskey(options, "filename")) {
+        qdict_put(options, "filename", qstring_from_str(filename));
+        allow_protocol_prefix = true;
+    } else {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Can't specify 'file' and "
+                      "'filename' options at the same time");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* Find the right block driver */
+    drvname = qdict_get_try_str(options, "driver");
+    if (drvname) {
+        drv = bdrv_find_whitelisted_format(drvname, !(flags & BDRV_O_RDWR));
+        qdict_del(options, "driver");
+    } else if (filename) {
+        drv = bdrv_find_protocol(filename, allow_protocol_prefix);
+        if (!drv) {
+            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Unknown protocol");
+        }
+    } else {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "Must specify either driver or file");
+        drv = NULL;
+    }
+
+    if (!drv) {
+        ret = -ENOENT;
+        goto fail;
+    }
+
+    /* Parse the filename and open it */
+    if (drv->bdrv_parse_filename && filename) {
+        Error *local_err = NULL;
+        drv->bdrv_parse_filename(filename, options, &local_err);
+        if (error_is_set(&local_err)) {
+            qerror_report_err(local_err);
+            error_free(local_err);
+            ret = -EINVAL;
+            goto fail;
+        }
+        qdict_del(options, "filename");
+    } else if (!drv->bdrv_parse_filename && !filename) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "The '%s' block driver requires a file name",
+                      drv->format_name);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    ret = bdrv_open_common(bs, NULL, options, flags, drv);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Check if any unknown options were used */
+    if (qdict_size(options) != 0) {
+        const QDictEntry *entry = qdict_first(options);
+        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Block protocol '%s' doesn't "
+                      "support the option '%s'",
+                      drv->format_name, entry->key);
+        ret = -EINVAL;
+        goto fail;
+    }
+    QDECREF(options);
+
+    bs->growable = 1;
+    *pbs = bs;
+    return 0;
+
+fail:
+    QDECREF(options);
+    if (!bs->drv) {
+        QDECREF(bs->options);
+    }
+    bdrv_delete(bs);
+    return ret;
+}
+
+/*
+ * Opens the backing file for a BlockDriverState if not yet open
+ *
+ * options is a QDict of options to pass to the block drivers, or NULL for an
+ * empty set of options. The reference to the QDict is transferred to this
+ * function (even on failure), so if the caller intends to reuse the dictionary,
+ * it needs to use QINCREF() before calling bdrv_file_open.
+ */
+int bdrv_open_backing_file(BlockDriverState *bs, QDict *options)
+{
+    char backing_filename[PATH_MAX];
+    int back_flags, ret;
+    BlockDriver *back_drv = NULL;
+
+    if (bs->backing_hd != NULL) {
+        QDECREF(options);
+        return 0;
+    }
+
+    /* NULL means an empty set of options */
+    if (options == NULL) {
+        options = qdict_new();
+    }
+
+    bs->open_flags &= ~BDRV_O_NO_BACKING;
+    if (qdict_haskey(options, "file.filename")) {
+        backing_filename[0] = '\0';
+    } else if (bs->backing_file[0] == '\0' && qdict_size(options) == 0) {
+        QDECREF(options);
+        return 0;
+    }
+
+    bs->backing_hd = bdrv_new("");
+    bdrv_get_full_backing_filename(bs, backing_filename,
+                                   sizeof(backing_filename));
+
+    if (bs->backing_format[0] != '\0') {
+        back_drv = bdrv_find_format(bs->backing_format);
+    }
+
+    /* backing files always opened read-only */
+    back_flags = bs->open_flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT);
+
+    ret = bdrv_open(bs->backing_hd,
+                    *backing_filename ? backing_filename : NULL, options,
+                    back_flags, back_drv);
+    if (ret < 0) {
+        bdrv_delete(bs->backing_hd);
+        bs->backing_hd = NULL;
+        bs->open_flags |= BDRV_O_NO_BACKING;
+        return ret;
+    }
+    return 0;
+}
+
+static void extract_subqdict(QDict *src, QDict **dst, const char *start)
+{
+    const QDictEntry *entry, *next;
+    const char *p;
+
+    *dst = qdict_new();
+    entry = qdict_first(src);
+
+    while (entry != NULL) {
+        next = qdict_next(src, entry);
+        if (strstart(entry->key, start, &p)) {
+            qobject_incref(entry->value);
+            qdict_put_obj(*dst, p, entry->value);
+            qdict_del(src, entry->key);
+        }
+        entry = next;
+    }
+}
+
+/*
+ * Opens a disk image (raw, qcow2, vmdk, ...)
+ *
+ * options is a QDict of options to pass to the block drivers, or NULL for an
+ * empty set of options. The reference to the QDict belongs to the block layer
+ * after the call (even on failure), so if the caller intends to reuse the
+ * dictionary, it needs to use QINCREF() before calling bdrv_open.
+ */
+int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
+              int flags, BlockDriver *drv)
+{
+    int ret;
+    /* TODO: extra byte is a hack to ensure MAX_PATH space on Windows. */
+    char tmp_filename[PATH_MAX + 1];
+    BlockDriverState *file = NULL;
+    QDict *file_options = NULL;
+
+    /* NULL means an empty set of options */
+    if (options == NULL) {
+        options = qdict_new();
+    }
+
+    bs->options = options;
+    options = qdict_clone_shallow(options);
+
+    /* For snapshot=on, create a temporary qcow2 overlay */
+    if (flags & BDRV_O_SNAPSHOT) {
+        BlockDriverState *bs1;
+        int64_t total_size;
+        BlockDriver *bdrv_qcow2;
+        QEMUOptionParameter *create_options;
+        char backing_filename[PATH_MAX];
+
+        if (qdict_size(options) != 0) {
+            error_report("Can't use snapshot=on with driver-specific options");
+            ret = -EINVAL;
+            goto fail;
+        }
+        assert(filename != NULL);
+
+        /* if snapshot, we create a temporary backing file and open it
+           instead of opening 'filename' directly */
+
+        /* if there is a backing file, use it */
+        bs1 = bdrv_new("");
+        ret = bdrv_open(bs1, filename, NULL, 0, drv);
+        if (ret < 0) {
+            bdrv_delete(bs1);
+            goto fail;
+        }
+        total_size = bdrv_getlength(bs1) & BDRV_SECTOR_MASK;
+
+        bdrv_delete(bs1);
+
+        ret = get_tmp_filename(tmp_filename, sizeof(tmp_filename));
+        if (ret < 0) {
+            goto fail;
+        }
+
+        /* Real path is meaningless for protocols */
+        if (path_has_protocol(filename)) {
+            snprintf(backing_filename, sizeof(backing_filename),
+                     "%s", filename);
+        } else if (!realpath(filename, backing_filename)) {
+            ret = -errno;
+            goto fail;
+        }
+
+        bdrv_qcow2 = bdrv_find_format("qcow2");
+        create_options = parse_option_parameters("", bdrv_qcow2->create_options,
+                                                 NULL);
+
+        set_option_parameter_int(create_options, BLOCK_OPT_SIZE, total_size);
+        set_option_parameter(create_options, BLOCK_OPT_BACKING_FILE,
+                             backing_filename);
+        if (drv) {
+            set_option_parameter(create_options, BLOCK_OPT_BACKING_FMT,
+                drv->format_name);
+        }
+
+        ret = bdrv_create(bdrv_qcow2, tmp_filename, create_options);
+        free_option_parameters(create_options);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        filename = tmp_filename;
+        drv = bdrv_qcow2;
+        bs->is_temporary = 1;
+    }
+
+    /* Open image file without format layer */
+    if (flags & BDRV_O_RDWR) {
+        flags |= BDRV_O_ALLOW_RDWR;
+    }
+
+    extract_subqdict(options, &file_options, "file.");
+
+    ret = bdrv_file_open(&file, filename, file_options,
+                         bdrv_open_flags(bs, flags | BDRV_O_UNMAP));
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Find the right image format driver */
+    if (!drv) {
+        ret = find_image_format(file, filename, &drv);
+    }
+
+    if (!drv) {
+        goto unlink_and_fail;
+    }
+
+    /* Open the image */
+    ret = bdrv_open_common(bs, file, options, flags, drv);
+    if (ret < 0) {
+        goto unlink_and_fail;
+    }
+
+    if (bs->file != file) {
+        bdrv_delete(file);
+        file = NULL;
+    }
+
+    /* If there is a backing file, use it */
+    if ((flags & BDRV_O_NO_BACKING) == 0) {
+        QDict *backing_options;
+
+        extract_subqdict(options, &backing_options, "backing.");
+        ret = bdrv_open_backing_file(bs, backing_options);
+        if (ret < 0) {
+            goto close_and_fail;
+        }
+    }
+
+    /* Check if any unknown options were used */
+    if (qdict_size(options) != 0) {
+        const QDictEntry *entry = qdict_first(options);
+        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Block format '%s' used by "
+            "device '%s' doesn't support the option '%s'",
+            drv->format_name, bs->device_name, entry->key);
+
+        ret = -EINVAL;
+        goto close_and_fail;
+    }
+    QDECREF(options);
+
+    if (!bdrv_key_required(bs)) {
+        bdrv_dev_change_media_cb(bs, true);
+    }
+
+    /* throttling disk I/O limits */
+    if (bs->io_limits_enabled) {
+        bdrv_io_limits_enable(bs);
+    }
+
+    return 0;
+
+unlink_and_fail:
+    if (file != NULL) {
+        bdrv_delete(file);
+    }
+    if (bs->is_temporary) {
+        unlink(filename);
+    }
+fail:
+    QDECREF(bs->options);
+    QDECREF(options);
+    bs->options = NULL;
+    return ret;
+
+close_and_fail:
+    bdrv_close(bs);
+    QDECREF(options);
+    return ret;
+}
+
+typedef struct BlockReopenQueueEntry {
+     bool prepared;
+     BDRVReopenState state;
+     QSIMPLEQ_ENTRY(BlockReopenQueueEntry) entry;
+} BlockReopenQueueEntry;
+
+/*
+ * Adds a BlockDriverState to a simple queue for an atomic, transactional
+ * reopen of multiple devices.
+ *
+ * bs_queue can either be an existing BlockReopenQueue that has had QSIMPLE_INIT
+ * already performed, or alternatively may be NULL a new BlockReopenQueue will
+ * be created and initialized. This newly created BlockReopenQueue should be
+ * passed back in for subsequent calls that are intended to be of the same
+ * atomic 'set'.
+ *
+ * bs is the BlockDriverState to add to the reopen queue.
+ *
+ * flags contains the open flags for the associated bs
+ *
+ * returns a pointer to bs_queue, which is either the newly allocated
+ * bs_queue, or the existing bs_queue being used.
+ *
+ */
+BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
+                                    BlockDriverState *bs, int flags)
+{
+    assert(bs != NULL);
+
+    BlockReopenQueueEntry *bs_entry;
+    if (bs_queue == NULL) {
+        bs_queue = g_new0(BlockReopenQueue, 1);
+        QSIMPLEQ_INIT(bs_queue);
+    }
+
+    if (bs->file) {
+        bdrv_reopen_queue(bs_queue, bs->file, flags);
+    }
+
+    bs_entry = g_new0(BlockReopenQueueEntry, 1);
+    QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
+
+    bs_entry->state.bs = bs;
+    bs_entry->state.flags = flags;
+
+    return bs_queue;
+}
+
+/*
+ * Reopen multiple BlockDriverStates atomically & transactionally.
+ *
+ * The queue passed in (bs_queue) must have been built up previous
+ * via bdrv_reopen_queue().
+ *
+ * Reopens all BDS specified in the queue, with the appropriate
+ * flags.  All devices are prepared for reopen, and failure of any
+ * device will cause all device changes to be abandonded, and intermediate
+ * data cleaned up.
+ *
+ * If all devices prepare successfully, then the changes are committed
+ * to all devices.
+ *
+ */
+int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp)
+{
+    int ret = -1;
+    BlockReopenQueueEntry *bs_entry, *next;
+    Error *local_err = NULL;
+
+    assert(bs_queue != NULL);
+
+    bdrv_drain_all();
+
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+        if (bdrv_reopen_prepare(&bs_entry->state, bs_queue, &local_err)) {
+            error_propagate(errp, local_err);
+            goto cleanup;
+        }
+        bs_entry->prepared = true;
+    }
+
+    /* If we reach this point, we have success and just need to apply the
+     * changes
+     */
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+        bdrv_reopen_commit(&bs_entry->state);
+    }
+
+    ret = 0;
+
+cleanup:
+    QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+        if (ret && bs_entry->prepared) {
+            bdrv_reopen_abort(&bs_entry->state);
+        }
+        g_free(bs_entry);
+    }
+    g_free(bs_queue);
+    return ret;
+}
+
+
+/* Reopen a single BlockDriverState with the specified flags. */
+int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp)
+{
+    int ret = -1;
+    Error *local_err = NULL;
+    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, bdrv_flags);
+
+    ret = bdrv_reopen_multiple(queue, &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+    }
+    return ret;
+}
+
+
+/*
+ * Prepares a BlockDriverState for reopen. All changes are staged in the
+ * 'opaque' field of the BDRVReopenState, which is used and allocated by
+ * the block driver layer .bdrv_reopen_prepare()
+ *
+ * bs is the BlockDriverState to reopen
+ * flags are the new open flags
+ * queue is the reopen queue
+ *
+ * Returns 0 on success, non-zero on error.  On error errp will be set
+ * as well.
+ *
+ * On failure, bdrv_reopen_abort() will be called to clean up any data.
+ * It is the responsibility of the caller to then call the abort() or
+ * commit() for any other BDS that have been left in a prepare() state
+ *
+ */
+int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
+                        Error **errp)
+{
+    int ret = -1;
+    Error *local_err = NULL;
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    assert(reopen_state->bs->drv != NULL);
+    drv = reopen_state->bs->drv;
+
+    /* if we are to stay read-only, do not allow permission change
+     * to r/w */
+    if (!(reopen_state->bs->open_flags & BDRV_O_ALLOW_RDWR) &&
+        reopen_state->flags & BDRV_O_RDWR) {
+        error_set(errp, QERR_DEVICE_IS_READ_ONLY,
+                  reopen_state->bs->device_name);
+        goto error;
+    }
+
+
+    ret = bdrv_flush(reopen_state->bs);
+    if (ret) {
+        error_set(errp, ERROR_CLASS_GENERIC_ERROR, "Error (%s) flushing drive",
+                  strerror(-ret));
+        goto error;
+    }
+
+    if (drv->bdrv_reopen_prepare) {
+        ret = drv->bdrv_reopen_prepare(reopen_state, queue, &local_err);
+        if (ret) {
+            if (local_err != NULL) {
+                error_propagate(errp, local_err);
+            } else {
+                error_setg(errp, "failed while preparing to reopen image '%s'",
+                           reopen_state->bs->filename);
+            }
+            goto error;
+        }
+    } else {
+        /* It is currently mandatory to have a bdrv_reopen_prepare()
+         * handler for each supported drv. */
+        error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
+                  drv->format_name, reopen_state->bs->device_name,
+                 "reopening of file");
+        ret = -1;
+        goto error;
+    }
+
+    ret = 0;
+
+error:
+    return ret;
+}
+
+/*
+ * Takes the staged changes for the reopen from bdrv_reopen_prepare(), and
+ * makes them final by swapping the staging BlockDriverState contents into
+ * the active BlockDriverState contents.
+ */
+void bdrv_reopen_commit(BDRVReopenState *reopen_state)
+{
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    drv = reopen_state->bs->drv;
+    assert(drv != NULL);
+
+    /* If there are any driver level actions to take */
+    if (drv->bdrv_reopen_commit) {
+        drv->bdrv_reopen_commit(reopen_state);
+    }
+
+    /* set BDS specific flags now */
+    reopen_state->bs->open_flags         = reopen_state->flags;
+    reopen_state->bs->enable_write_cache = !!(reopen_state->flags &
+                                              BDRV_O_CACHE_WB);
+    reopen_state->bs->read_only = !(reopen_state->flags & BDRV_O_RDWR);
+}
+
+/*
+ * Abort the reopen, and delete and free the staged changes in
+ * reopen_state
+ */
+void bdrv_reopen_abort(BDRVReopenState *reopen_state)
+{
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    drv = reopen_state->bs->drv;
+    assert(drv != NULL);
+
+    if (drv->bdrv_reopen_abort) {
+        drv->bdrv_reopen_abort(reopen_state);
+    }
+}
+
+
+void bdrv_close(BlockDriverState *bs)
+{
+    if (bs->job) {
+        block_job_cancel_sync(bs->job);
+    }
+    bdrv_drain_all(); /* complete I/O */
+    bdrv_flush(bs);
+    bdrv_drain_all(); /* in case flush left pending I/O */
+    notifier_list_notify(&bs->close_notifiers, bs);
+
+    if (bs->drv) {
+        if (bs->backing_hd) {
+            bdrv_delete(bs->backing_hd);
+            bs->backing_hd = NULL;
+        }
+        bs->drv->bdrv_close(bs);
+        g_free(bs->opaque);
+#ifdef _WIN32
+        if (bs->is_temporary) {
+            unlink(bs->filename);
+        }
+#endif
+        bs->opaque = NULL;
+        bs->drv = NULL;
+        bs->copy_on_read = 0;
+        bs->backing_file[0] = '\0';
+        bs->backing_format[0] = '\0';
+        bs->total_sectors = 0;
+        bs->encrypted = 0;
+        bs->valid_key = 0;
+        bs->sg = 0;
+        bs->growable = 0;
+        QDECREF(bs->options);
+        bs->options = NULL;
+
+        if (bs->file != NULL) {
+            bdrv_delete(bs->file);
+            bs->file = NULL;
+        }
+    }
+
+    bdrv_dev_change_media_cb(bs, false);
+
+    /*throttling disk I/O limits*/
+    if (bs->io_limits_enabled) {
+        bdrv_io_limits_disable(bs);
+    }
+}
+
+void bdrv_close_all(void)
+{
+    BlockDriverState *bs;
+
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        bdrv_close(bs);
+    }
+}
+
+/*
+ * Wait for pending requests to complete across all BlockDriverStates
+ *
+ * This function does not flush data to disk, use bdrv_flush_all() for that
+ * after calling this function.
+ *
+ * Note that completion of an asynchronous I/O operation can trigger any
+ * number of other I/O operations on other devices---for example a coroutine
+ * can be arbitrarily complex and a constant flow of I/O can come until the
+ * coroutine is complete.  Because of this, it is not possible to have a
+ * function to drain a single device's I/O queue.
+ */
+void bdrv_drain_all(void)
+{
+    BlockDriverState *bs;
+    bool busy;
+
+    do {
+        busy = qemu_aio_wait();
+
+        /* FIXME: We do not have timer support here, so this is effectively
+         * a busy wait.
+         */
+        QTAILQ_FOREACH(bs, &bdrv_states, list) {
+            if (!qemu_co_queue_empty(&bs->throttled_reqs)) {
+                qemu_co_queue_restart_all(&bs->throttled_reqs);
+                busy = true;
+            }
+        }
+    } while (busy);
+
+    /* If requests are still pending there is a bug somewhere */
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        assert(QLIST_EMPTY(&bs->tracked_requests));
+        assert(qemu_co_queue_empty(&bs->throttled_reqs));
+    }
+}
+
+/* make a BlockDriverState anonymous by removing from bdrv_state list.
+   Also, NULL terminate the device_name to prevent double remove */
+void bdrv_make_anon(BlockDriverState *bs)
+{
+    if (bs->device_name[0] != '\0') {
+        QTAILQ_REMOVE(&bdrv_states, bs, list);
+    }
+    bs->device_name[0] = '\0';
+}
+
+static void bdrv_rebind(BlockDriverState *bs)
+{
+    if (bs->drv && bs->drv->bdrv_rebind) {
+        bs->drv->bdrv_rebind(bs);
+    }
+}
+
+static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
+                                     BlockDriverState *bs_src)
+{
+    /* move some fields that need to stay attached to the device */
+    bs_dest->open_flags         = bs_src->open_flags;
+
+    /* dev info */
+    bs_dest->dev_ops            = bs_src->dev_ops;
+    bs_dest->dev_opaque         = bs_src->dev_opaque;
+    bs_dest->dev                = bs_src->dev;
+    bs_dest->buffer_alignment   = bs_src->buffer_alignment;
+    bs_dest->copy_on_read       = bs_src->copy_on_read;
+
+    bs_dest->enable_write_cache = bs_src->enable_write_cache;
+
+    /* i/o timing parameters */
+    bs_dest->slice_start        = bs_src->slice_start;
+    bs_dest->slice_end          = bs_src->slice_end;
+    bs_dest->slice_submitted    = bs_src->slice_submitted;
+    bs_dest->io_limits          = bs_src->io_limits;
+    bs_dest->throttled_reqs     = bs_src->throttled_reqs;
+    bs_dest->block_timer        = bs_src->block_timer;
+    bs_dest->io_limits_enabled  = bs_src->io_limits_enabled;
+
+    /* r/w error */
+    bs_dest->on_read_error      = bs_src->on_read_error;
+    bs_dest->on_write_error     = bs_src->on_write_error;
+
+    /* i/o status */
+    bs_dest->iostatus_enabled   = bs_src->iostatus_enabled;
+    bs_dest->iostatus           = bs_src->iostatus;
+
+    /* dirty bitmap */
+    bs_dest->dirty_bitmap       = bs_src->dirty_bitmap;
+
+    /* job */
+    bs_dest->in_use             = bs_src->in_use;
+    bs_dest->job                = bs_src->job;
+
+    /* keep the same entry in bdrv_states */
+    pstrcpy(bs_dest->device_name, sizeof(bs_dest->device_name),
+            bs_src->device_name);
+    bs_dest->list = bs_src->list;
+}
+
+/*
+ * Swap bs contents for two image chains while they are live,
+ * while keeping required fields on the BlockDriverState that is
+ * actually attached to a device.
+ *
+ * This will modify the BlockDriverState fields, and swap contents
+ * between bs_new and bs_old. Both bs_new and bs_old are modified.
+ *
+ * bs_new is required to be anonymous.
+ *
+ * This function does not create any image files.
+ */
+void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
+{
+    BlockDriverState tmp;
+
+    /* bs_new must be anonymous and shouldn't have anything fancy enabled */
+    assert(bs_new->device_name[0] == '\0');
+    assert(bs_new->dirty_bitmap == NULL);
+    assert(bs_new->job == NULL);
+    assert(bs_new->dev == NULL);
+    assert(bs_new->in_use == 0);
+    assert(bs_new->io_limits_enabled == false);
+    assert(bs_new->block_timer == NULL);
+
+    tmp = *bs_new;
+    *bs_new = *bs_old;
+    *bs_old = tmp;
+
+    /* there are some fields that should not be swapped, move them back */
+    bdrv_move_feature_fields(&tmp, bs_old);
+    bdrv_move_feature_fields(bs_old, bs_new);
+    bdrv_move_feature_fields(bs_new, &tmp);
+
+    /* bs_new shouldn't be in bdrv_states even after the swap!  */
+    assert(bs_new->device_name[0] == '\0');
+
+    /* Check a few fields that should remain attached to the device */
+    assert(bs_new->dev == NULL);
+    assert(bs_new->job == NULL);
+    assert(bs_new->in_use == 0);
+    assert(bs_new->io_limits_enabled == false);
+    assert(bs_new->block_timer == NULL);
+
+    bdrv_rebind(bs_new);
+    bdrv_rebind(bs_old);
+}
+
+/*
+ * Add new bs contents at the top of an image chain while the chain is
+ * live, while keeping required fields on the top layer.
+ *
+ * This will modify the BlockDriverState fields, and swap contents
+ * between bs_new and bs_top. Both bs_new and bs_top are modified.
+ *
+ * bs_new is required to be anonymous.
+ *
+ * This function does not create any image files.
+ */
+void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top)
+{
+    bdrv_swap(bs_new, bs_top);
+
+    /* The contents of 'tmp' will become bs_top, as we are
+     * swapping bs_new and bs_top contents. */
+    bs_top->backing_hd = bs_new;
+    bs_top->open_flags &= ~BDRV_O_NO_BACKING;
+    pstrcpy(bs_top->backing_file, sizeof(bs_top->backing_file),
+            bs_new->filename);
+    pstrcpy(bs_top->backing_format, sizeof(bs_top->backing_format),
+            bs_new->drv ? bs_new->drv->format_name : "");
+}
+
+void bdrv_delete(BlockDriverState *bs)
+{
+    assert(!bs->dev);
+    assert(!bs->job);
+    assert(!bs->in_use);
+
+    /* remove from list, if necessary */
+    bdrv_make_anon(bs);
+
+    bdrv_close(bs);
+
+    g_free(bs);
+}
+
+int bdrv_attach_dev(BlockDriverState *bs, void *dev)
+/* TODO change to DeviceState *dev when all users are qdevified */
+{
+    if (bs->dev) {
+        return -EBUSY;
+    }
+    bs->dev = dev;
+    bdrv_iostatus_reset(bs);
+    return 0;
+}
+
+/* TODO qdevified devices don't use this, remove when devices are qdevified */
+void bdrv_attach_dev_nofail(BlockDriverState *bs, void *dev)
+{
+    if (bdrv_attach_dev(bs, dev) < 0) {
+        abort();
+    }
+}
+
+void bdrv_detach_dev(BlockDriverState *bs, void *dev)
+/* TODO change to DeviceState *dev when all users are qdevified */
+{
+    assert(bs->dev == dev);
+    bs->dev = NULL;
+    bs->dev_ops = NULL;
+    bs->dev_opaque = NULL;
+    bs->buffer_alignment = 512;
+}
+
+/* TODO change to return DeviceState * when all users are qdevified */
+void *bdrv_get_attached_dev(BlockDriverState *bs)
+{
+    return bs->dev;
+}
+
+void bdrv_set_dev_ops(BlockDriverState *bs, const BlockDevOps *ops,
+                      void *opaque)
+{
+    bs->dev_ops = ops;
+    bs->dev_opaque = opaque;
+}
+
+void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
+                               enum MonitorEvent ev,
+                               BlockErrorAction action, bool is_read)
+{
+    QObject *data;
+    const char *action_str;
+
+    switch (action) {
+    case BDRV_ACTION_REPORT:
+        action_str = "report";
+        break;
+    case BDRV_ACTION_IGNORE:
+        action_str = "ignore";
+        break;
+    case BDRV_ACTION_STOP:
+        action_str = "stop";
+        break;
+    default:
+        abort();
+    }
+
+    data = qobject_from_jsonf("{ 'device': %s, 'action': %s, 'operation': %s }",
+                              bdrv->device_name,
+                              action_str,
+                              is_read ? "read" : "write");
+    monitor_protocol_event(ev, data);
+
+    qobject_decref(data);
+}
+
+static void bdrv_emit_qmp_eject_event(BlockDriverState *bs, bool ejected)
+{
+    QObject *data;
+
+    data = qobject_from_jsonf("{ 'device': %s, 'tray-open': %i }",
+                              bdrv_get_device_name(bs), ejected);
+    monitor_protocol_event(QEVENT_DEVICE_TRAY_MOVED, data);
+
+    qobject_decref(data);
+}
+
+static void bdrv_dev_change_media_cb(BlockDriverState *bs, bool load)
+{
+    if (bs->dev_ops && bs->dev_ops->change_media_cb) {
+        bool tray_was_closed = !bdrv_dev_is_tray_open(bs);
+        bs->dev_ops->change_media_cb(bs->dev_opaque, load);
+        if (tray_was_closed) {
+            /* tray open */
+            bdrv_emit_qmp_eject_event(bs, true);
+        }
+        if (load) {
+            /* tray close */
+            bdrv_emit_qmp_eject_event(bs, false);
+        }
+    }
+}
+
+bool bdrv_dev_has_removable_media(BlockDriverState *bs)
+{
+    return !bs->dev || (bs->dev_ops && bs->dev_ops->change_media_cb);
+}
+
+void bdrv_dev_eject_request(BlockDriverState *bs, bool force)
+{
+    if (bs->dev_ops && bs->dev_ops->eject_request_cb) {
+        bs->dev_ops->eject_request_cb(bs->dev_opaque, force);
+    }
+}
+
+bool bdrv_dev_is_tray_open(BlockDriverState *bs)
+{
+    if (bs->dev_ops && bs->dev_ops->is_tray_open) {
+        return bs->dev_ops->is_tray_open(bs->dev_opaque);
+    }
+    return false;
+}
+
+static void bdrv_dev_resize_cb(BlockDriverState *bs)
+{
+    if (bs->dev_ops && bs->dev_ops->resize_cb) {
+        bs->dev_ops->resize_cb(bs->dev_opaque);
+    }
+}
+
+bool bdrv_dev_is_medium_locked(BlockDriverState *bs)
+{
+    if (bs->dev_ops && bs->dev_ops->is_medium_locked) {
+        return bs->dev_ops->is_medium_locked(bs->dev_opaque);
+    }
+    return false;
+}
+
+/*
+ * Run consistency checks on an image
+ *
+ * Returns 0 if the check could be completed (it doesn't mean that the image is
+ * free of errors) or -errno when an internal error occurred. The results of the
+ * check are stored in res.
+ */
+int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix)
+{
+    if (bs->drv->bdrv_check == NULL) {
+        return -ENOTSUP;
+    }
+
+    memset(res, 0, sizeof(*res));
+    return bs->drv->bdrv_check(bs, res, fix);
+}
+
+#define COMMIT_BUF_SECTORS 2048
+
+/* commit COW file into the raw image */
+int bdrv_commit(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+    int64_t sector, total_sectors;
+    int n, ro, open_flags;
+    int ret = 0;
+    uint8_t *buf;
+    char filename[PATH_MAX];
+
+    if (!drv)
+        return -ENOMEDIUM;
+    
+    if (!bs->backing_hd) {
+        return -ENOTSUP;
+    }
+
+    if (bdrv_in_use(bs) || bdrv_in_use(bs->backing_hd)) {
+        return -EBUSY;
+    }
+
+    ro = bs->backing_hd->read_only;
+    /* Use pstrcpy (not strncpy): filename must be NUL-terminated. */
+    pstrcpy(filename, sizeof(filename), bs->backing_hd->filename);
+    open_flags =  bs->backing_hd->open_flags;
+
+    if (ro) {
+        if (bdrv_reopen(bs->backing_hd, open_flags | BDRV_O_RDWR, NULL)) {
+            return -EACCES;
+        }
+    }
+
+    total_sectors = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+    buf = g_malloc(COMMIT_BUF_SECTORS * BDRV_SECTOR_SIZE);
+
+    for (sector = 0; sector < total_sectors; sector += n) {
+        if (bdrv_is_allocated(bs, sector, COMMIT_BUF_SECTORS, &n)) {
+
+            if (bdrv_read(bs, sector, buf, n) != 0) {
+                ret = -EIO;
+                goto ro_cleanup;
+            }
+
+            if (bdrv_write(bs->backing_hd, sector, buf, n) != 0) {
+                ret = -EIO;
+                goto ro_cleanup;
+            }
+        }
+    }
+
+    if (drv->bdrv_make_empty) {
+        ret = drv->bdrv_make_empty(bs);
+        bdrv_flush(bs);
+    }
+
+    /*
+     * Make sure all data we wrote to the backing device is actually
+     * stable on disk.
+     */
+    if (bs->backing_hd)
+        bdrv_flush(bs->backing_hd);
+
+ro_cleanup:
+    g_free(buf);
+
+    if (ro) {
+        /* ignoring error return here */
+        bdrv_reopen(bs->backing_hd, open_flags & ~BDRV_O_RDWR, NULL);
+    }
+
+    return ret;
+}
+
+int bdrv_commit_all(void)
+{
+    BlockDriverState *bs;
+
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        if (bs->drv && bs->backing_hd) {
+            int ret = bdrv_commit(bs);
+            if (ret < 0) {
+                return ret;
+            }
+        }
+    }
+    return 0;
+}
+
+/**
+ * Remove an active request from the tracked requests list
+ *
+ * This function should be called when a tracked request is completing.
+ */
+static void tracked_request_end(BdrvTrackedRequest *req)
+{
+    QLIST_REMOVE(req, list);
+    qemu_co_queue_restart_all(&req->wait_queue);
+}
+
+/**
+ * Add an active request to the tracked requests list
+ */
+static void tracked_request_begin(BdrvTrackedRequest *req,
+                                  BlockDriverState *bs,
+                                  int64_t sector_num,
+                                  int nb_sectors, bool is_write)
+{
+    *req = (BdrvTrackedRequest){
+        .bs = bs,
+        .sector_num = sector_num,
+        .nb_sectors = nb_sectors,
+        .is_write = is_write,
+        .co = qemu_coroutine_self(),
+    };
+
+    qemu_co_queue_init(&req->wait_queue);
+
+    QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
+}
+
+/**
+ * Round a region to cluster boundaries
+ */
+void bdrv_round_to_clusters(BlockDriverState *bs,
+                            int64_t sector_num, int nb_sectors,
+                            int64_t *cluster_sector_num,
+                            int *cluster_nb_sectors)
+{
+    BlockDriverInfo bdi;
+
+    if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
+        *cluster_sector_num = sector_num;
+        *cluster_nb_sectors = nb_sectors;
+    } else {
+        int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
+        *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
+        *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
+                                            nb_sectors, c);
+    }
+}
+
+static bool tracked_request_overlaps(BdrvTrackedRequest *req,
+                                     int64_t sector_num, int nb_sectors) {
+    /*        aaaa   bbbb */
+    if (sector_num >= req->sector_num + req->nb_sectors) {
+        return false;
+    }
+    /* bbbb   aaaa        */
+    if (req->sector_num >= sector_num + nb_sectors) {
+        return false;
+    }
+    return true;
+}
+
+static void coroutine_fn wait_for_overlapping_requests(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors)
+{
+    BdrvTrackedRequest *req;
+    int64_t cluster_sector_num;
+    int cluster_nb_sectors;
+    bool retry;
+
+    /* If we touch the same cluster it counts as an overlap.  This guarantees
+     * that allocating writes will be serialized and not race with each other
+     * for the same cluster.  For example, in copy-on-read it ensures that the
+     * CoR read and write operations are atomic and guest writes cannot
+     * interleave between them.
+     */
+    bdrv_round_to_clusters(bs, sector_num, nb_sectors,
+                           &cluster_sector_num, &cluster_nb_sectors);
+
+    do {
+        retry = false;
+        QLIST_FOREACH(req, &bs->tracked_requests, list) {
+            if (tracked_request_overlaps(req, cluster_sector_num,
+                                         cluster_nb_sectors)) {
+                /* Hitting this means there was a reentrant request, for
+                 * example, a block driver issuing nested requests.  This must
+                 * never happen since it means deadlock.
+                 */
+                assert(qemu_coroutine_self() != req->co);
+
+                qemu_co_queue_wait(&req->wait_queue);
+                retry = true;
+                break;
+            }
+        }
+    } while (retry);
+}
+
+/*
+ * Return values:
+ * 0        - success
+ * -EINVAL  - backing format specified, but no file
+ * -ENOSPC  - can't update the backing file because no space is left in the
+ *            image file header
+ * -ENOTSUP - format driver doesn't support changing the backing file
+ */
+int bdrv_change_backing_file(BlockDriverState *bs,
+    const char *backing_file, const char *backing_fmt)
+{
+    BlockDriver *drv = bs->drv;
+    int ret;
+
+    /* Backing file format doesn't make sense without a backing file */
+    if (backing_fmt && !backing_file) {
+        return -EINVAL;
+    }
+
+    if (drv->bdrv_change_backing_file != NULL) {
+        ret = drv->bdrv_change_backing_file(bs, backing_file, backing_fmt);
+    } else {
+        ret = -ENOTSUP;
+    }
+
+    if (ret == 0) {
+        pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_file ?: "");
+        pstrcpy(bs->backing_format, sizeof(bs->backing_format), backing_fmt ?: "");
+    }
+    return ret;
+}
+
+/*
+ * Finds the image layer in the chain that has 'bs' as its backing file.
+ *
+ * active is the current topmost image.
+ *
+ * Returns NULL if bs is not found in active's image chain,
+ * or if active == bs.
+ */
+BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
+                                    BlockDriverState *bs)
+{
+    BlockDriverState *overlay = NULL;
+    BlockDriverState *intermediate;
+
+    assert(active != NULL);
+    assert(bs != NULL);
+
+    /* if bs is the same as active, then by definition it has no overlay
+     */
+    if (active == bs) {
+        return NULL;
+    }
+
+    intermediate = active;
+    while (intermediate->backing_hd) {
+        if (intermediate->backing_hd == bs) {
+            overlay = intermediate;
+            break;
+        }
+        intermediate = intermediate->backing_hd;
+    }
+
+    return overlay;
+}
+
+typedef struct BlkIntermediateStates {
+    BlockDriverState *bs;
+    QSIMPLEQ_ENTRY(BlkIntermediateStates) entry;
+} BlkIntermediateStates;
+
+
+/*
+ * Drops images above 'base' up to and including 'top', and sets the image
+ * above 'top' to have base as its backing file.
+ *
+ * Requires that the overlay to 'top' is opened r/w, so that the backing file
+ * information in 'bs' can be properly updated.
+ *
+ * E.g., this will convert the following chain:
+ * bottom <- base <- intermediate <- top <- active
+ *
+ * to
+ *
+ * bottom <- base <- active
+ *
+ * It is allowed for bottom==base, in which case it converts:
+ *
+ * base <- intermediate <- top <- active
+ *
+ * to
+ *
+ * base <- active
+ *
+ * Error conditions:
+ *  if active == top, that is considered an error
+ *
+ */
+int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
+                           BlockDriverState *base)
+{
+    BlockDriverState *intermediate;
+    BlockDriverState *base_bs = NULL;
+    BlockDriverState *new_top_bs = NULL;
+    BlkIntermediateStates *intermediate_state, *next;
+    int ret = -EIO;
+
+    QSIMPLEQ_HEAD(states_to_delete, BlkIntermediateStates) states_to_delete;
+    QSIMPLEQ_INIT(&states_to_delete);
+
+    if (!top->drv || !base->drv) {
+        goto exit;
+    }
+
+    new_top_bs = bdrv_find_overlay(active, top);
+
+    if (new_top_bs == NULL) {
+        /* we could not find the image above 'top', this is an error */
+        goto exit;
+    }
+
+    /* special case of new_top_bs->backing_hd already pointing to base - nothing
+     * to do, no intermediate images */
+    if (new_top_bs->backing_hd == base) {
+        ret = 0;
+        goto exit;
+    }
+
+    intermediate = top;
+
+    /* now we will go down through the list, and add each BDS we find
+     * into our deletion queue, until we hit the 'base'
+     */
+    while (intermediate) {
+        intermediate_state = g_malloc0(sizeof(BlkIntermediateStates));
+        intermediate_state->bs = intermediate;
+        QSIMPLEQ_INSERT_TAIL(&states_to_delete, intermediate_state, entry);
+
+        if (intermediate->backing_hd == base) {
+            base_bs = intermediate->backing_hd;
+            break;
+        }
+        intermediate = intermediate->backing_hd;
+    }
+    if (base_bs == NULL) {
+        /* something went wrong, we did not end at the base. safely
+         * unravel everything, and exit with error */
+        goto exit;
+    }
+
+    /* success - we can delete the intermediate states, and link top->base */
+    ret = bdrv_change_backing_file(new_top_bs, base_bs->filename,
+                                   base_bs->drv ? base_bs->drv->format_name : "");
+    if (ret) {
+        goto exit;
+    }
+    new_top_bs->backing_hd = base_bs;
+
+
+    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
+        /* so that bdrv_close() does not recursively close the chain */
+        intermediate_state->bs->backing_hd = NULL;
+        bdrv_delete(intermediate_state->bs);
+    }
+    ret = 0;
+
+exit:
+    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
+        g_free(intermediate_state);
+    }
+    return ret;
+}
+
+
+static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
+                                   size_t size)
+{
+    int64_t len;
+
+    if (!bdrv_is_inserted(bs))
+        return -ENOMEDIUM;
+
+    if (bs->growable)
+        return 0;
+
+    len = bdrv_getlength(bs);
+
+    if (offset < 0)
+        return -EIO;
+
+    if ((offset > len) || (len - offset < size))
+        return -EIO;
+
+    return 0;
+}
+
+static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
+                              int nb_sectors)
+{
+    return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
+                                   nb_sectors * BDRV_SECTOR_SIZE);
+}
+
+typedef struct RwCo {
+    BlockDriverState *bs;
+    int64_t sector_num;
+    int nb_sectors;
+    QEMUIOVector *qiov;
+    bool is_write;
+    int ret;
+} RwCo;
+
+static void coroutine_fn bdrv_rw_co_entry(void *opaque)
+{
+    RwCo *rwco = opaque;
+
+    if (!rwco->is_write) {
+        rwco->ret = bdrv_co_do_readv(rwco->bs, rwco->sector_num,
+                                     rwco->nb_sectors, rwco->qiov, 0);
+    } else {
+        rwco->ret = bdrv_co_do_writev(rwco->bs, rwco->sector_num,
+                                      rwco->nb_sectors, rwco->qiov, 0);
+    }
+}
+
+/*
+ * Process a vectored synchronous request using coroutines
+ */
+static int bdrv_rwv_co(BlockDriverState *bs, int64_t sector_num,
+                       QEMUIOVector *qiov, bool is_write)
+{
+    Coroutine *co;
+    RwCo rwco = {
+        .bs = bs,
+        .sector_num = sector_num,
+        .nb_sectors = qiov->size >> BDRV_SECTOR_BITS,
+        .qiov = qiov,
+        .is_write = is_write,
+        .ret = NOT_DONE,
+    };
+    assert((qiov->size & (BDRV_SECTOR_SIZE - 1)) == 0);
+
+    /**
+     * In sync call context, when the vcpu is blocked, this throttling timer
+     * will not fire; so the I/O throttling function has to be disabled here
+     * if it has been enabled.
+     */
+    if (bs->io_limits_enabled) {
+        fprintf(stderr, "Disabling I/O throttling on '%s' due "
+                        "to synchronous I/O.\n", bdrv_get_device_name(bs));
+        bdrv_io_limits_disable(bs);
+    }
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_rw_co_entry(&rwco);
+    } else {
+        co = qemu_coroutine_create(bdrv_rw_co_entry);
+        qemu_coroutine_enter(co, &rwco);
+        while (rwco.ret == NOT_DONE) {
+            qemu_aio_wait();
+        }
+    }
+    return rwco.ret;
+}
+
+/*
+ * Process a synchronous request using coroutines
+ */
+static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf,
+                      int nb_sectors, bool is_write)
+{
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base = (void *)buf,
+        .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
+    };
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+    return bdrv_rwv_co(bs, sector_num, &qiov, is_write);
+}
+
+/* return < 0 if error. See bdrv_write() for the return codes */
+int bdrv_read(BlockDriverState *bs, int64_t sector_num,
+              uint8_t *buf, int nb_sectors)
+{
+    return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false);
+}
+
+/* Just like bdrv_read(), but with I/O throttling temporarily disabled */
+int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
+                          uint8_t *buf, int nb_sectors)
+{
+    bool enabled;
+    int ret;
+
+    enabled = bs->io_limits_enabled;
+    bs->io_limits_enabled = false;
+    ret = bdrv_read(bs, 0, buf, 1);
+    bs->io_limits_enabled = enabled;
+    return ret;
+}
+
+/* Return < 0 if error. Important errors are:
+  -EIO         generic I/O error (may happen for all errors)
+  -ENOMEDIUM   No media inserted.
+  -EINVAL      Invalid sector number or nb_sectors
+  -EACCES      Trying to write a read-only device
+*/
+int bdrv_write(BlockDriverState *bs, int64_t sector_num,
+               const uint8_t *buf, int nb_sectors)
+{
+    return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true);
+}
+
+int bdrv_writev(BlockDriverState *bs, int64_t sector_num, QEMUIOVector *qiov)
+{
+    return bdrv_rwv_co(bs, sector_num, qiov, true);
+}
+
+int bdrv_pread(BlockDriverState *bs, int64_t offset,
+               void *buf, int count1)
+{
+    uint8_t tmp_buf[BDRV_SECTOR_SIZE];
+    int len, nb_sectors, count;
+    int64_t sector_num;
+    int ret;
+
+    count = count1;
+    /* first read to align to sector start */
+    len = (BDRV_SECTOR_SIZE - offset) & (BDRV_SECTOR_SIZE - 1);
+    if (len > count)
+        len = count;
+    sector_num = offset >> BDRV_SECTOR_BITS;
+    if (len > 0) {
+        if ((ret = bdrv_read(bs, sector_num, tmp_buf, 1)) < 0)
+            return ret;
+        memcpy(buf, tmp_buf + (offset & (BDRV_SECTOR_SIZE - 1)), len);
+        count -= len;
+        if (count == 0)
+            return count1;
+        sector_num++;
+        buf += len;
+    }
+
+    /* read the sectors "in place" */
+    nb_sectors = count >> BDRV_SECTOR_BITS;
+    if (nb_sectors > 0) {
+        if ((ret = bdrv_read(bs, sector_num, buf, nb_sectors)) < 0)
+            return ret;
+        sector_num += nb_sectors;
+        len = nb_sectors << BDRV_SECTOR_BITS;
+        buf += len;
+        count -= len;
+    }
+
+    /* add data from the last sector */
+    if (count > 0) {
+        if ((ret = bdrv_read(bs, sector_num, tmp_buf, 1)) < 0)
+            return ret;
+        memcpy(buf, tmp_buf, count);
+    }
+    return count1;
+}
+
+int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov)
+{
+    uint8_t tmp_buf[BDRV_SECTOR_SIZE];
+    int len, nb_sectors, count;
+    int64_t sector_num;
+    int ret;
+
+    count = qiov->size;
+
+    /* first write to align to sector start */
+    len = (BDRV_SECTOR_SIZE - offset) & (BDRV_SECTOR_SIZE - 1);
+    if (len > count)
+        len = count;
+    sector_num = offset >> BDRV_SECTOR_BITS;
+    if (len > 0) {
+        if ((ret = bdrv_read(bs, sector_num, tmp_buf, 1)) < 0)
+            return ret;
+        qemu_iovec_to_buf(qiov, 0, tmp_buf + (offset & (BDRV_SECTOR_SIZE - 1)),
+                          len);
+        if ((ret = bdrv_write(bs, sector_num, tmp_buf, 1)) < 0)
+            return ret;
+        count -= len;
+        if (count == 0)
+            return qiov->size;
+        sector_num++;
+    }
+
+    /* write the sectors "in place" */
+    nb_sectors = count >> BDRV_SECTOR_BITS;
+    if (nb_sectors > 0) {
+        QEMUIOVector qiov_inplace;
+
+        qemu_iovec_init(&qiov_inplace, qiov->niov);
+        qemu_iovec_concat(&qiov_inplace, qiov, len,
+                          nb_sectors << BDRV_SECTOR_BITS);
+        ret = bdrv_writev(bs, sector_num, &qiov_inplace);
+        qemu_iovec_destroy(&qiov_inplace);
+        if (ret < 0) {
+            return ret;
+        }
+
+        sector_num += nb_sectors;
+        len = nb_sectors << BDRV_SECTOR_BITS;
+        count -= len;
+    }
+
+    /* add data from the last sector */
+    if (count > 0) {
+        if ((ret = bdrv_read(bs, sector_num, tmp_buf, 1)) < 0)
+            return ret;
+        qemu_iovec_to_buf(qiov, qiov->size - count, tmp_buf, count);
+        if ((ret = bdrv_write(bs, sector_num, tmp_buf, 1)) < 0)
+            return ret;
+    }
+    return qiov->size;
+}
+
+int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
+                const void *buf, int count1)
+{
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base   = (void *) buf,
+        .iov_len    = count1,
+    };
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+    return bdrv_pwritev(bs, offset, &qiov);
+}
+
+/*
+ * Writes to the file and ensures that no writes are reordered across this
+ * request (acts as a barrier)
+ *
+ * Returns 0 on success, -errno in error cases.
+ */
+int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
+    const void *buf, int count)
+{
+    int ret;
+
+    ret = bdrv_pwrite(bs, offset, buf, count);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* No flush needed for cache modes that already do it */
+    if (bs->enable_write_cache) {
+        bdrv_flush(bs);
+    }
+
+    return 0;
+}
+
+static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
+{
+    /* Perform I/O through a temporary buffer so that users who scribble over
+     * their read buffer while the operation is in progress do not end up
+     * modifying the image file.  This is critical for zero-copy guest I/O
+     * where anything might happen inside guest memory.
+     */
+    void *bounce_buffer;
+
+    BlockDriver *drv = bs->drv;
+    struct iovec iov;
+    QEMUIOVector bounce_qiov;
+    int64_t cluster_sector_num;
+    int cluster_nb_sectors;
+    size_t skip_bytes;
+    int ret;
+
+    /* Cover entire cluster so no additional backing file I/O is required when
+     * allocating cluster in the image file.
+     */
+    bdrv_round_to_clusters(bs, sector_num, nb_sectors,
+                           &cluster_sector_num, &cluster_nb_sectors);
+
+    trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
+                                   cluster_sector_num, cluster_nb_sectors);
+
+    iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
+    iov.iov_base = bounce_buffer = qemu_blockalign(bs, iov.iov_len);
+    qemu_iovec_init_external(&bounce_qiov, &iov, 1);
+
+    ret = drv->bdrv_co_readv(bs, cluster_sector_num, cluster_nb_sectors,
+                             &bounce_qiov);
+    if (ret < 0) {
+        goto err;
+    }
+
+    if (drv->bdrv_co_write_zeroes &&
+        buffer_is_zero(bounce_buffer, iov.iov_len)) {
+        ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num,
+                                      cluster_nb_sectors);
+    } else {
+        /* This does not change the data on the disk, it is not necessary
+         * to flush even in cache=writethrough mode.
+         */
+        ret = drv->bdrv_co_writev(bs, cluster_sector_num, cluster_nb_sectors,
+                                  &bounce_qiov);
+    }
+
+    if (ret < 0) {
+        /* It might be okay to ignore write errors for guest requests.  If this
+         * is a deliberate copy-on-read then we don't want to ignore the error.
+         * Simply report it in all cases.
+         */
+        goto err;
+    }
+
+    skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE;
+    qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes,
+                        nb_sectors * BDRV_SECTOR_SIZE);
+
+err:
+    qemu_vfree(bounce_buffer);
+    return ret;
+}
+
+/*
+ * Handle a read request in coroutine context
+ */
+static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
+    BdrvRequestFlags flags)
+{
+    BlockDriver *drv = bs->drv;
+    BdrvTrackedRequest req;
+    int ret;
+
+    if (!drv) {
+        return -ENOMEDIUM;
+    }
+    if (bdrv_check_request(bs, sector_num, nb_sectors)) {
+        return -EIO;
+    }
+
+    /* throttling disk read I/O */
+    if (bs->io_limits_enabled) {
+        bdrv_io_limits_intercept(bs, false, nb_sectors);
+    }
+
+    if (bs->copy_on_read) {
+        flags |= BDRV_REQ_COPY_ON_READ;
+    }
+    if (flags & BDRV_REQ_COPY_ON_READ) {
+        bs->copy_on_read_in_flight++;
+    }
+
+    if (bs->copy_on_read_in_flight) {
+        wait_for_overlapping_requests(bs, sector_num, nb_sectors);
+    }
+
+    tracked_request_begin(&req, bs, sector_num, nb_sectors, false);
+
+    if (flags & BDRV_REQ_COPY_ON_READ) {
+        int pnum;
+
+        ret = bdrv_co_is_allocated(bs, sector_num, nb_sectors, &pnum);
+        if (ret < 0) {
+            goto out;
+        }
+
+        if (!ret || pnum != nb_sectors) {
+            ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov);
+            goto out;
+        }
+    }
+
+    ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
+
+out:
+    tracked_request_end(&req);
+
+    if (flags & BDRV_REQ_COPY_ON_READ) {
+        bs->copy_on_read_in_flight--;
+    }
+
+    return ret;
+}
+
+int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
+    int nb_sectors, QEMUIOVector *qiov)
+{
+    trace_bdrv_co_readv(bs, sector_num, nb_sectors);
+
+    return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
+}
+
+int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
+{
+    trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors);
+
+    return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
+                            BDRV_REQ_COPY_ON_READ);
+}
+
+static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors)
+{
+    BlockDriver *drv = bs->drv;
+    QEMUIOVector qiov;
+    struct iovec iov;
+    int ret;
+
+    /* TODO Emulate only part of misaligned requests instead of letting block
+     * drivers return -ENOTSUP and emulate everything */
+
+    /* First try the efficient write zeroes operation */
+    if (drv->bdrv_co_write_zeroes) {
+        ret = drv->bdrv_co_write_zeroes(bs, sector_num, nb_sectors);
+        if (ret != -ENOTSUP) {
+            return ret;
+        }
+    }
+
+    /* Fall back to bounce buffer if write zeroes is unsupported */
+    iov.iov_len  = nb_sectors * BDRV_SECTOR_SIZE;
+    iov.iov_base = qemu_blockalign(bs, iov.iov_len);
+    memset(iov.iov_base, 0, iov.iov_len);
+    qemu_iovec_init_external(&qiov, &iov, 1);
+
+    ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, &qiov);
+
+    qemu_vfree(iov.iov_base);
+    return ret;
+}
+
+/*
+ * Handle a write request in coroutine context
+ */
+static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
+    BdrvRequestFlags flags)
+{
+    BlockDriver *drv = bs->drv;
+    BdrvTrackedRequest req;
+    int ret;
+
+    if (!bs->drv) {
+        return -ENOMEDIUM;
+    }
+    if (bs->read_only) {
+        return -EACCES;
+    }
+    if (bdrv_check_request(bs, sector_num, nb_sectors)) {
+        return -EIO;
+    }
+
+    /* throttling disk write I/O */
+    if (bs->io_limits_enabled) {
+        bdrv_io_limits_intercept(bs, true, nb_sectors);
+    }
+
+    if (bs->copy_on_read_in_flight) {
+        wait_for_overlapping_requests(bs, sector_num, nb_sectors);
+    }
+
+    tracked_request_begin(&req, bs, sector_num, nb_sectors, true);
+
+    ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req);
+
+    if (ret < 0) {
+        /* Do nothing, write notifier decided to fail this request */
+    } else if (flags & BDRV_REQ_ZERO_WRITE) {
+        ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors);
+    } else {
+        ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
+    }
+
+    if (ret == 0 && !bs->enable_write_cache) {
+        ret = bdrv_co_flush(bs);
+    }
+
+    if (bs->dirty_bitmap) {
+        bdrv_set_dirty(bs, sector_num, nb_sectors);
+    }
+
+    if (bs->wr_highest_sector < sector_num + nb_sectors - 1) {
+        bs->wr_highest_sector = sector_num + nb_sectors - 1;
+    }
+
+    tracked_request_end(&req);
+
+    return ret;
+}
+
+int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
+    int nb_sectors, QEMUIOVector *qiov)
+{
+    trace_bdrv_co_writev(bs, sector_num, nb_sectors);
+
+    return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0);
+}
+
+int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
+                                      int64_t sector_num, int nb_sectors)
+{
+    trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors);
+
+    return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL,
+                             BDRV_REQ_ZERO_WRITE);
+}
+
+/**
+ * Truncate file to 'offset' bytes (needed only for file protocols)
+ */
+int bdrv_truncate(BlockDriverState *bs, int64_t offset)
+{
+    BlockDriver *drv = bs->drv;
+    int ret;
+    if (!drv)
+        return -ENOMEDIUM;
+    if (!drv->bdrv_truncate)
+        return -ENOTSUP;
+    if (bs->read_only)
+        return -EACCES;
+    if (bdrv_in_use(bs))
+        return -EBUSY;
+    ret = drv->bdrv_truncate(bs, offset);
+    if (ret == 0) {
+        ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
+        bdrv_dev_resize_cb(bs);
+    }
+    return ret;
+}
+
+/**
+ * Length of a allocated file in bytes. Sparse files are counted by actual
+ * allocated space. Return < 0 if error or unknown.
+ */
+int64_t bdrv_get_allocated_file_size(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+    if (!drv) {
+        return -ENOMEDIUM;
+    }
+    if (drv->bdrv_get_allocated_file_size) {
+        return drv->bdrv_get_allocated_file_size(bs);
+    }
+    if (bs->file) {
+        return bdrv_get_allocated_file_size(bs->file);
+    }
+    return -ENOTSUP;
+}
+
+/**
+ * Length of a file in bytes. Return < 0 if error or unknown.
+ */
+int64_t bdrv_getlength(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+    if (!drv)
+        return -ENOMEDIUM;
+
+    if (bs->growable || bdrv_dev_has_removable_media(bs)) {
+        if (drv->bdrv_getlength) {
+            return drv->bdrv_getlength(bs);
+        }
+    }
+    return bs->total_sectors * BDRV_SECTOR_SIZE;
+}
+
+/* return 0 as number of sectors if no device present or error */
+void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr)
+{
+    int64_t length;
+    length = bdrv_getlength(bs);
+    if (length < 0)
+        length = 0;
+    else
+        length = length >> BDRV_SECTOR_BITS;
+    *nb_sectors_ptr = length;
+}
+
+/* throttling disk io limits */
+void bdrv_set_io_limits(BlockDriverState *bs,
+                        BlockIOLimit *io_limits)
+{
+    bs->io_limits = *io_limits;
+    bs->io_limits_enabled = bdrv_io_limits_enabled(bs);
+}
+
+void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
+                       BlockdevOnError on_write_error)
+{
+    bs->on_read_error = on_read_error;
+    bs->on_write_error = on_write_error;
+}
+
+BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read)
+{
+    return is_read ? bs->on_read_error : bs->on_write_error;
+}
+
+BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error)
+{
+    BlockdevOnError on_err = is_read ? bs->on_read_error : bs->on_write_error;
+
+    switch (on_err) {
+    case BLOCKDEV_ON_ERROR_ENOSPC:
+        return (error == ENOSPC) ? BDRV_ACTION_STOP : BDRV_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_STOP:
+        return BDRV_ACTION_STOP;
+    case BLOCKDEV_ON_ERROR_REPORT:
+        return BDRV_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_IGNORE:
+        return BDRV_ACTION_IGNORE;
+    default:
+        abort();
+    }
+}
+
+/* This is done by device models because, while the block layer knows
+ * about the error, it does not know whether an operation comes from
+ * the device or the block layer (from a job, for example).
+ */
+void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
+                       bool is_read, int error)
+{
+    assert(error >= 0);
+    bdrv_emit_qmp_error_event(bs, QEVENT_BLOCK_IO_ERROR, action, is_read);
+    if (action == BDRV_ACTION_STOP) {
+        vm_stop(RUN_STATE_IO_ERROR);
+        bdrv_iostatus_set_err(bs, error);
+    }
+}
+
+int bdrv_is_read_only(BlockDriverState *bs)
+{
+    return bs->read_only;
+}
+
+int bdrv_is_sg(BlockDriverState *bs)
+{
+    return bs->sg;
+}
+
+int bdrv_enable_write_cache(BlockDriverState *bs)
+{
+    return bs->enable_write_cache;
+}
+
+void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce)
+{
+    bs->enable_write_cache = wce;
+
+    /* so a reopen() will preserve wce */
+    if (wce) {
+        bs->open_flags |= BDRV_O_CACHE_WB;
+    } else {
+        bs->open_flags &= ~BDRV_O_CACHE_WB;
+    }
+}
+
+int bdrv_is_encrypted(BlockDriverState *bs)
+{
+    if (bs->backing_hd && bs->backing_hd->encrypted)
+        return 1;
+    return bs->encrypted;
+}
+
+int bdrv_key_required(BlockDriverState *bs)
+{
+    BlockDriverState *backing_hd = bs->backing_hd;
+
+    if (backing_hd && backing_hd->encrypted && !backing_hd->valid_key)
+        return 1;
+    return (bs->encrypted && !bs->valid_key);
+}
+
+int bdrv_set_key(BlockDriverState *bs, const char *key)
+{
+    int ret;
+    if (bs->backing_hd && bs->backing_hd->encrypted) {
+        ret = bdrv_set_key(bs->backing_hd, key);
+        if (ret < 0)
+            return ret;
+        if (!bs->encrypted)
+            return 0;
+    }
+    if (!bs->encrypted) {
+        return -EINVAL;
+    } else if (!bs->drv || !bs->drv->bdrv_set_key) {
+        return -ENOMEDIUM;
+    }
+    ret = bs->drv->bdrv_set_key(bs, key);
+    if (ret < 0) {
+        bs->valid_key = 0;
+    } else if (!bs->valid_key) {
+        bs->valid_key = 1;
+        /* call the change callback now, we skipped it on open */
+        bdrv_dev_change_media_cb(bs, true);
+    }
+    return ret;
+}
+
+const char *bdrv_get_format_name(BlockDriverState *bs)
+{
+    return bs->drv ? bs->drv->format_name : NULL;
+}
+
+void bdrv_iterate_format(void (*it)(void *opaque, const char *name),
+                         void *opaque)
+{
+    BlockDriver *drv;
+
+    QLIST_FOREACH(drv, &bdrv_drivers, list) {
+        it(opaque, drv->format_name);
+    }
+}
+
+BlockDriverState *bdrv_find(const char *name)
+{
+    BlockDriverState *bs;
+
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        if (!strcmp(name, bs->device_name)) {
+            return bs;
+        }
+    }
+    return NULL;
+}
+
+BlockDriverState *bdrv_next(BlockDriverState *bs)
+{
+    if (!bs) {
+        return QTAILQ_FIRST(&bdrv_states);
+    }
+    return QTAILQ_NEXT(bs, list);
+}
+
+void bdrv_iterate(void (*it)(void *opaque, BlockDriverState *bs), void *opaque)
+{
+    BlockDriverState *bs;
+
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        it(opaque, bs);
+    }
+}
+
+const char *bdrv_get_device_name(BlockDriverState *bs)
+{
+    return bs->device_name;
+}
+
+int bdrv_get_flags(BlockDriverState *bs)
+{
+    return bs->open_flags;
+}
+
+int bdrv_flush_all(void)
+{
+    BlockDriverState *bs;
+    int result = 0;
+
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        int ret = bdrv_flush(bs);
+        if (ret < 0 && !result) {
+            result = ret;
+        }
+    }
+
+    return result;
+}
+
+int bdrv_has_zero_init_1(BlockDriverState *bs)
+{
+    return 1;
+}
+
+int bdrv_has_zero_init(BlockDriverState *bs)
+{
+    assert(bs->drv);
+
+    if (bs->drv->bdrv_has_zero_init) {
+        return bs->drv->bdrv_has_zero_init(bs);
+    }
+
+    /* safe default */
+    return 0;
+}
+
+typedef struct BdrvCoIsAllocatedData {
+    BlockDriverState *bs;
+    BlockDriverState *base;
+    int64_t sector_num;
+    int nb_sectors;
+    int *pnum;
+    int ret;
+    bool done;
+} BdrvCoIsAllocatedData;
+
+/*
+ * Returns true iff the specified sector is present in the disk image. Drivers
+ * not implementing the functionality are assumed to not support backing files,
+ * hence all their sectors are reported as allocated.
+ *
+ * If 'sector_num' is beyond the end of the disk image the return value is 0
+ * and 'pnum' is set to 0.
+ *
+ * 'pnum' is set to the number of sectors (including and immediately following
+ * the specified sector) that are known to be in the same
+ * allocated/unallocated state.
+ *
+ * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
+ * beyond the end of the disk image it will be clamped.
+ */
+int coroutine_fn bdrv_co_is_allocated(BlockDriverState *bs, int64_t sector_num,
+                                      int nb_sectors, int *pnum)
+{
+    int64_t n;
+
+    if (sector_num >= bs->total_sectors) {
+        *pnum = 0;
+        return 0;
+    }
+
+    n = bs->total_sectors - sector_num;
+    if (n < nb_sectors) {
+        nb_sectors = n;
+    }
+
+    if (!bs->drv->bdrv_co_is_allocated) {
+        *pnum = nb_sectors;
+        return 1;
+    }
+
+    return bs->drv->bdrv_co_is_allocated(bs, sector_num, nb_sectors, pnum);
+}
+
+/* Coroutine wrapper for bdrv_is_allocated() */
+static void coroutine_fn bdrv_is_allocated_co_entry(void *opaque)
+{
+    BdrvCoIsAllocatedData *data = opaque;
+    BlockDriverState *bs = data->bs;
+
+    data->ret = bdrv_co_is_allocated(bs, data->sector_num, data->nb_sectors,
+                                     data->pnum);
+    data->done = true;
+}
+
+/*
+ * Synchronous wrapper around bdrv_co_is_allocated().
+ *
+ * See bdrv_co_is_allocated() for details.
+ */
+int bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+                      int *pnum)
+{
+    Coroutine *co;
+    BdrvCoIsAllocatedData data = {
+        .bs = bs,
+        .sector_num = sector_num,
+        .nb_sectors = nb_sectors,
+        .pnum = pnum,
+        .done = false,
+    };
+
+    co = qemu_coroutine_create(bdrv_is_allocated_co_entry);
+    qemu_coroutine_enter(co, &data);
+    while (!data.done) {
+        qemu_aio_wait();
+    }
+    return data.ret;
+}
+
+/*
+ * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
+ *
+ * Return true if the given sector is allocated in any image between
+ * BASE and TOP (inclusive).  BASE can be NULL to check if the given
+ * sector is allocated in any image of the chain.  Return false otherwise.
+ *
+ * 'pnum' is set to the number of sectors (including and immediately following
+ *  the specified sector) that are known to be in the same
+ *  allocated/unallocated state.
+ *
+ */
+int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top,
+                                            BlockDriverState *base,
+                                            int64_t sector_num,
+                                            int nb_sectors, int *pnum)
+{
+    BlockDriverState *intermediate;
+    int ret, n = nb_sectors;
+
+    intermediate = top;
+    while (intermediate && intermediate != base) {
+        int pnum_inter;
+        ret = bdrv_co_is_allocated(intermediate, sector_num, nb_sectors,
+                                   &pnum_inter);
+        if (ret < 0) {
+            return ret;
+        } else if (ret) {
+            *pnum = pnum_inter;
+            return 1;
+        }
+
+        /*
+         * [sector_num, nb_sectors] is unallocated on top but intermediate
+         * might have
+         *
+         * [sector_num+x, nr_sectors] allocated.
+         */
+        if (n > pnum_inter &&
+            (intermediate == top ||
+             sector_num + pnum_inter < intermediate->total_sectors)) {
+            n = pnum_inter;
+        }
+
+        intermediate = intermediate->backing_hd;
+    }
+
+    *pnum = n;
+    return 0;
+}
+
+/* Coroutine wrapper for bdrv_is_allocated_above() */
+static void coroutine_fn bdrv_is_allocated_above_co_entry(void *opaque)
+{
+    BdrvCoIsAllocatedData *data = opaque;
+    BlockDriverState *top = data->bs;
+    BlockDriverState *base = data->base;
+
+    data->ret = bdrv_co_is_allocated_above(top, base, data->sector_num,
+                                           data->nb_sectors, data->pnum);
+    data->done = true;
+}
+
+/*
+ * Synchronous wrapper around bdrv_co_is_allocated_above().
+ *
+ * See bdrv_co_is_allocated_above() for details.
+ */
+int bdrv_is_allocated_above(BlockDriverState *top, BlockDriverState *base,
+                            int64_t sector_num, int nb_sectors, int *pnum)
+{
+    Coroutine *co;
+    BdrvCoIsAllocatedData data = {
+        .bs = top,
+        .base = base,
+        .sector_num = sector_num,
+        .nb_sectors = nb_sectors,
+        .pnum = pnum,
+        .done = false,
+    };
+
+    co = qemu_coroutine_create(bdrv_is_allocated_above_co_entry);
+    qemu_coroutine_enter(co, &data);
+    while (!data.done) {
+        qemu_aio_wait();
+    }
+    return data.ret;
+}
+
+const char *bdrv_get_encrypted_filename(BlockDriverState *bs)
+{
+    if (bs->backing_hd && bs->backing_hd->encrypted)
+        return bs->backing_file;
+    else if (bs->encrypted)
+        return bs->filename;
+    else
+        return NULL;
+}
+
+void bdrv_get_backing_filename(BlockDriverState *bs,
+                               char *filename, int filename_size)
+{
+    pstrcpy(filename, filename_size, bs->backing_file);
+}
+
+int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
+                          const uint8_t *buf, int nb_sectors)
+{
+    BlockDriver *drv = bs->drv;
+    if (!drv)
+        return -ENOMEDIUM;
+    if (!drv->bdrv_write_compressed)
+        return -ENOTSUP;
+    if (bdrv_check_request(bs, sector_num, nb_sectors))
+        return -EIO;
+
+    assert(!bs->dirty_bitmap);
+
+    return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
+}
+
+int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
+{
+    BlockDriver *drv = bs->drv;
+    if (!drv)
+        return -ENOMEDIUM;
+    if (!drv->bdrv_get_info)
+        return -ENOTSUP;
+    memset(bdi, 0, sizeof(*bdi));
+    return drv->bdrv_get_info(bs, bdi);
+}
+
+int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
+                      int64_t pos, int size)
+{
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base   = (void *) buf,
+        .iov_len    = size,
+    };
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+    return bdrv_writev_vmstate(bs, &qiov, pos);
+}
+
+int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
+{
+    BlockDriver *drv = bs->drv;
+
+    if (!drv) {
+        return -ENOMEDIUM;
+    } else if (drv->bdrv_save_vmstate) {
+        return drv->bdrv_save_vmstate(bs, qiov, pos);
+    } else if (bs->file) {
+        return bdrv_writev_vmstate(bs->file, qiov, pos);
+    }
+
+    return -ENOTSUP;
+}
+
+int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
+                      int64_t pos, int size)
+{
+    BlockDriver *drv = bs->drv;
+    if (!drv)
+        return -ENOMEDIUM;
+    if (drv->bdrv_load_vmstate)
+        return drv->bdrv_load_vmstate(bs, buf, pos, size);
+    if (bs->file)
+        return bdrv_load_vmstate(bs->file, buf, pos, size);
+    return -ENOTSUP;
+}
+
+void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event)
+{
+    if (!bs || !bs->drv || !bs->drv->bdrv_debug_event) {
+        return;
+    }
+
+    bs->drv->bdrv_debug_event(bs, event);
+}
+
+int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
+                          const char *tag)
+{
+    while (bs && bs->drv && !bs->drv->bdrv_debug_breakpoint) {
+        bs = bs->file;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_breakpoint) {
+        return bs->drv->bdrv_debug_breakpoint(bs, event, tag);
+    }
+
+    return -ENOTSUP;
+}
+
+int bdrv_debug_resume(BlockDriverState *bs, const char *tag)
+{
+    while (bs && bs->drv && !bs->drv->bdrv_debug_resume) {
+        bs = bs->file;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_resume) {
+        return bs->drv->bdrv_debug_resume(bs, tag);
+    }
+
+    return -ENOTSUP;
+}
+
+bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag)
+{
+    while (bs && bs->drv && !bs->drv->bdrv_debug_is_suspended) {
+        bs = bs->file;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_is_suspended) {
+        return bs->drv->bdrv_debug_is_suspended(bs, tag);
+    }
+
+    return false;
+}
+
+int bdrv_is_snapshot(BlockDriverState *bs)
+{
+    return !!(bs->open_flags & BDRV_O_SNAPSHOT);
+}
+
+/* backing_file can either be relative, or absolute, or a protocol.  If it is
+ * relative, it must be relative to the chain.  So, passing in bs->filename
+ * from a BDS as backing_file should not be done, as that may be relative to
+ * the CWD rather than the chain. */
+BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
+        const char *backing_file)
+{
+    char *filename_full = NULL;
+    char *backing_file_full = NULL;
+    char *filename_tmp = NULL;
+    int is_protocol = 0;
+    BlockDriverState *curr_bs = NULL;
+    BlockDriverState *retval = NULL;
+
+    if (!bs || !bs->drv || !backing_file) {
+        return NULL;
+    }
+
+    filename_full     = g_malloc(PATH_MAX);
+    backing_file_full = g_malloc(PATH_MAX);
+    filename_tmp      = g_malloc(PATH_MAX);
+
+    is_protocol = path_has_protocol(backing_file);
+
+    for (curr_bs = bs; curr_bs->backing_hd; curr_bs = curr_bs->backing_hd) {
+
+        /* If either of the filename paths is actually a protocol, then
+         * compare unmodified paths; otherwise make paths relative */
+        if (is_protocol || path_has_protocol(curr_bs->backing_file)) {
+            if (strcmp(backing_file, curr_bs->backing_file) == 0) {
+                retval = curr_bs->backing_hd;
+                break;
+            }
+        } else {
+            /* If not an absolute filename path, make it relative to the current
+             * image's filename path */
+            path_combine(filename_tmp, PATH_MAX, curr_bs->filename,
+                         backing_file);
+
+            /* We are going to compare absolute pathnames */
+            if (!realpath(filename_tmp, filename_full)) {
+                continue;
+            }
+
+            /* We need to make sure the backing filename we are comparing against
+             * is relative to the current image filename (or absolute) */
+            path_combine(filename_tmp, PATH_MAX, curr_bs->filename,
+                         curr_bs->backing_file);
+
+            if (!realpath(filename_tmp, backing_file_full)) {
+                continue;
+            }
+
+            if (strcmp(backing_file_full, filename_full) == 0) {
+                retval = curr_bs->backing_hd;
+                break;
+            }
+        }
+    }
+
+    g_free(filename_full);
+    g_free(backing_file_full);
+    g_free(filename_tmp);
+    return retval;
+}
+
+int bdrv_get_backing_file_depth(BlockDriverState *bs)
+{
+    if (!bs->drv) {
+        return 0;
+    }
+
+    if (!bs->backing_hd) {
+        return 0;
+    }
+
+    return 1 + bdrv_get_backing_file_depth(bs->backing_hd);
+}
+
+BlockDriverState *bdrv_find_base(BlockDriverState *bs)
+{
+    BlockDriverState *curr_bs = NULL;
+
+    if (!bs) {
+        return NULL;
+    }
+
+    curr_bs = bs;
+
+    while (curr_bs->backing_hd) {
+        curr_bs = curr_bs->backing_hd;
+    }
+    return curr_bs;
+}
+
+/**************************************************************/
+/* async I/Os */
+
+BlockDriverAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
+                                 QEMUIOVector *qiov, int nb_sectors,
+                                 BlockDriverCompletionFunc *cb, void *opaque)
+{
+    trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
+
+    return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors,
+                                 cb, opaque, false);
+}
+
+BlockDriverAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
+                                  QEMUIOVector *qiov, int nb_sectors,
+                                  BlockDriverCompletionFunc *cb, void *opaque)
+{
+    trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
+
+    return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors,
+                                 cb, opaque, true);
+}
+
+
+typedef struct MultiwriteCB {
+    int error;
+    int num_requests;
+    int num_callbacks;
+    struct {
+        BlockDriverCompletionFunc *cb;
+        void *opaque;
+        QEMUIOVector *free_qiov;
+    } callbacks[];
+} MultiwriteCB;
+
+static void multiwrite_user_cb(MultiwriteCB *mcb)
+{
+    int i;
+
+    for (i = 0; i < mcb->num_callbacks; i++) {
+        mcb->callbacks[i].cb(mcb->callbacks[i].opaque, mcb->error);
+        if (mcb->callbacks[i].free_qiov) {
+            qemu_iovec_destroy(mcb->callbacks[i].free_qiov);
+        }
+        g_free(mcb->callbacks[i].free_qiov);
+    }
+}
+
+static void multiwrite_cb(void *opaque, int ret)
+{
+    MultiwriteCB *mcb = opaque;
+
+    trace_multiwrite_cb(mcb, ret);
+
+    if (ret < 0 && !mcb->error) {
+        mcb->error = ret;
+    }
+
+    mcb->num_requests--;
+    if (mcb->num_requests == 0) {
+        multiwrite_user_cb(mcb);
+        g_free(mcb);
+    }
+}
+
+static int multiwrite_req_compare(const void *a, const void *b)
+{
+    const BlockRequest *req1 = a, *req2 = b;
+
+    /*
+     * Note that we can't simply subtract req2->sector from req1->sector
+     * here as that could overflow the return value.
+     */
+    if (req1->sector > req2->sector) {
+        return 1;
+    } else if (req1->sector < req2->sector) {
+        return -1;
+    } else {
+        return 0;
+    }
+}
+
+/*
+ * Takes a bunch of requests and tries to merge them. Returns the number of
+ * requests that remain after merging.
+ */
+static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
+    int num_reqs, MultiwriteCB *mcb)
+{
+    int i, outidx;
+
+    // Sort requests by start sector
+    qsort(reqs, num_reqs, sizeof(*reqs), &multiwrite_req_compare);
+
+    // Check if adjacent requests touch the same clusters. If so, combine them,
+    // filling up gaps with zero sectors.
+    outidx = 0;
+    for (i = 1; i < num_reqs; i++) {
+        int merge = 0;
+        int64_t oldreq_last = reqs[outidx].sector + reqs[outidx].nb_sectors;
+
+        // Handle exactly sequential writes and overlapping writes.
+        if (reqs[i].sector <= oldreq_last) {
+            merge = 1;
+        }
+
+        if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 > IOV_MAX) {
+            merge = 0;
+        }
+
+        if (merge) {
+            size_t size;
+            QEMUIOVector *qiov = g_malloc0(sizeof(*qiov));
+            qemu_iovec_init(qiov,
+                reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1);
+
+            // Add the first request to the merged one. If the requests are
+            // overlapping, drop the last sectors of the first request.
+            size = (reqs[i].sector - reqs[outidx].sector) << 9;
+            qemu_iovec_concat(qiov, reqs[outidx].qiov, 0, size);
+
+            // We should need to add any zeros between the two requests
+            assert (reqs[i].sector <= oldreq_last);
+
+            // Add the second request
+            qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size);
+
+            reqs[outidx].nb_sectors = qiov->size >> 9;
+            reqs[outidx].qiov = qiov;
+
+            mcb->callbacks[i].free_qiov = reqs[outidx].qiov;
+        } else {
+            outidx++;
+            reqs[outidx].sector     = reqs[i].sector;
+            reqs[outidx].nb_sectors = reqs[i].nb_sectors;
+            reqs[outidx].qiov       = reqs[i].qiov;
+        }
+    }
+
+    return outidx + 1;
+}
+
+/*
+ * Submit multiple AIO write requests at once.
+ *
+ * On success, the function returns 0 and all requests in the reqs array have
+ * been submitted. In error case this function returns -1, and any of the
+ * requests may or may not be submitted yet. In particular, this means that the
+ * callback will be called for some of the requests, for others it won't. The
+ * caller must check the error field of the BlockRequest to wait for the right
+ * callbacks (if error != 0, no callback will be called).
+ *
+ * The implementation may modify the contents of the reqs array, e.g. to merge
+ * requests. However, the fields opaque and error are left unmodified as they
+ * are used to signal failure for a single request to the caller.
+ */
+int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs)
+{
+    MultiwriteCB *mcb;
+    int i;
+
+    /* don't submit writes if we don't have a medium */
+    if (bs->drv == NULL) {
+        for (i = 0; i < num_reqs; i++) {
+            reqs[i].error = -ENOMEDIUM;
+        }
+        return -1;
+    }
+
+    if (num_reqs == 0) {
+        return 0;
+    }
+
+    // Create MultiwriteCB structure
+    mcb = g_malloc0(sizeof(*mcb) + num_reqs * sizeof(*mcb->callbacks));
+    mcb->num_requests = 0;
+    mcb->num_callbacks = num_reqs;
+
+    for (i = 0; i < num_reqs; i++) {
+        mcb->callbacks[i].cb = reqs[i].cb;
+        mcb->callbacks[i].opaque = reqs[i].opaque;
+    }
+
+    // Check for mergable requests
+    num_reqs = multiwrite_merge(bs, reqs, num_reqs, mcb);
+
+    trace_bdrv_aio_multiwrite(mcb, mcb->num_callbacks, num_reqs);
+
+    /* Run the aio requests. */
+    mcb->num_requests = num_reqs;
+    for (i = 0; i < num_reqs; i++) {
+        bdrv_aio_writev(bs, reqs[i].sector, reqs[i].qiov,
+            reqs[i].nb_sectors, multiwrite_cb, mcb);
+    }
+
+    return 0;
+}
+
+void bdrv_aio_cancel(BlockDriverAIOCB *acb)
+{
+    acb->aiocb_info->cancel(acb);
+}
+
+/* block I/O throttling */
+static bool bdrv_exceed_bps_limits(BlockDriverState *bs, int nb_sectors,
+                 bool is_write, double elapsed_time, uint64_t *wait)
+{
+    uint64_t bps_limit = 0;
+    uint64_t extension;
+    double   bytes_limit, bytes_base, bytes_res;
+    double   slice_time, wait_time;
+
+    if (bs->io_limits.bps[BLOCK_IO_LIMIT_TOTAL]) {
+        bps_limit = bs->io_limits.bps[BLOCK_IO_LIMIT_TOTAL];
+    } else if (bs->io_limits.bps[is_write]) {
+        bps_limit = bs->io_limits.bps[is_write];
+    } else {
+        if (wait) {
+            *wait = 0;
+        }
+
+        return false;
+    }
+
+    slice_time = bs->slice_end - bs->slice_start;
+    slice_time /= (NANOSECONDS_PER_SECOND);
+    bytes_limit = bps_limit * slice_time;
+    bytes_base  = bs->slice_submitted.bytes[is_write];
+    if (bs->io_limits.bps[BLOCK_IO_LIMIT_TOTAL]) {
+        bytes_base += bs->slice_submitted.bytes[!is_write];
+    }
+
+    /* bytes_base: the bytes of data which have been read/written; and
+     *             it is obtained from the history statistic info.
+     * bytes_res: the remaining bytes of data which need to be read/written.
+     * (bytes_base + bytes_res) / bps_limit: used to calcuate
+     *             the total time for completing reading/writting all data.
+     */
+    bytes_res   = (unsigned) nb_sectors * BDRV_SECTOR_SIZE;
+
+    if (bytes_base + bytes_res <= bytes_limit) {
+        if (wait) {
+            *wait = 0;
+        }
+
+        return false;
+    }
+
+    /* Calc approx time to dispatch */
+    wait_time = (bytes_base + bytes_res) / bps_limit - elapsed_time;
+
+    /* When the I/O rate at runtime exceeds the limits,
+     * bs->slice_end need to be extended in order that the current statistic
+     * info can be kept until the timer fire, so it is increased and tuned
+     * based on the result of experiment.
+     */
+    extension = wait_time * NANOSECONDS_PER_SECOND;
+    extension = DIV_ROUND_UP(extension, BLOCK_IO_SLICE_TIME) *
+                BLOCK_IO_SLICE_TIME;
+    bs->slice_end += extension;
+    if (wait) {
+        *wait = wait_time * NANOSECONDS_PER_SECOND;
+    }
+
+    return true;
+}
+
+static bool bdrv_exceed_iops_limits(BlockDriverState *bs, bool is_write,
+                             double elapsed_time, uint64_t *wait)
+{
+    uint64_t iops_limit = 0;
+    double   ios_limit, ios_base;
+    double   slice_time, wait_time;
+
+    if (bs->io_limits.iops[BLOCK_IO_LIMIT_TOTAL]) {
+        iops_limit = bs->io_limits.iops[BLOCK_IO_LIMIT_TOTAL];
+    } else if (bs->io_limits.iops[is_write]) {
+        iops_limit = bs->io_limits.iops[is_write];
+    } else {
+        if (wait) {
+            *wait = 0;
+        }
+
+        return false;
+    }
+
+    slice_time = bs->slice_end - bs->slice_start;
+    slice_time /= (NANOSECONDS_PER_SECOND);
+    ios_limit  = iops_limit * slice_time;
+    ios_base   = bs->slice_submitted.ios[is_write];
+    if (bs->io_limits.iops[BLOCK_IO_LIMIT_TOTAL]) {
+        ios_base += bs->slice_submitted.ios[!is_write];
+    }
+
+    if (ios_base + 1 <= ios_limit) {
+        if (wait) {
+            *wait = 0;
+        }
+
+        return false;
+    }
+
+    /* Calc approx time to dispatch, in seconds */
+    wait_time = (ios_base + 1) / iops_limit;
+    if (wait_time > elapsed_time) {
+        wait_time = wait_time - elapsed_time;
+    } else {
+        wait_time = 0;
+    }
+
+    /* Exceeded current slice, extend it by another slice time */
+    bs->slice_end += BLOCK_IO_SLICE_TIME;
+    if (wait) {
+        *wait = wait_time * NANOSECONDS_PER_SECOND;
+    }
+
+    return true;
+}
+
+static bool bdrv_exceed_io_limits(BlockDriverState *bs, int nb_sectors,
+                           bool is_write, int64_t *wait)
+{
+    int64_t  now, max_wait;
+    uint64_t bps_wait = 0, iops_wait = 0;
+    double   elapsed_time;
+    int      bps_ret, iops_ret;
+
+    now = qemu_get_clock_ns(vm_clock);
+    if (now > bs->slice_end) {
+        bs->slice_start = now;
+        bs->slice_end   = now + BLOCK_IO_SLICE_TIME;
+        memset(&bs->slice_submitted, 0, sizeof(bs->slice_submitted));
+    }
+
+    elapsed_time  = now - bs->slice_start;
+    elapsed_time  /= (NANOSECONDS_PER_SECOND);
+
+    bps_ret  = bdrv_exceed_bps_limits(bs, nb_sectors,
+                                      is_write, elapsed_time, &bps_wait);
+    iops_ret = bdrv_exceed_iops_limits(bs, is_write,
+                                      elapsed_time, &iops_wait);
+    if (bps_ret || iops_ret) {
+        max_wait = bps_wait > iops_wait ? bps_wait : iops_wait;
+        if (wait) {
+            *wait = max_wait;
+        }
+
+        now = qemu_get_clock_ns(vm_clock);
+        if (bs->slice_end < now + max_wait) {
+            bs->slice_end = now + max_wait;
+        }
+
+        return true;
+    }
+
+    if (wait) {
+        *wait = 0;
+    }
+
+    bs->slice_submitted.bytes[is_write] += (int64_t)nb_sectors *
+                                           BDRV_SECTOR_SIZE;
+    bs->slice_submitted.ios[is_write]++;
+
+    return false;
+}
+
+/**************************************************************/
+/* async block device emulation */
+
+typedef struct BlockDriverAIOCBSync {
+    BlockDriverAIOCB common;
+    QEMUBH *bh;
+    int ret;
+    /* vector translation state */
+    QEMUIOVector *qiov;
+    uint8_t *bounce;
+    int is_write;
+} BlockDriverAIOCBSync;
+
+static void bdrv_aio_cancel_em(BlockDriverAIOCB *blockacb)
+{
+    BlockDriverAIOCBSync *acb =
+        container_of(blockacb, BlockDriverAIOCBSync, common);
+    qemu_bh_delete(acb->bh);
+    acb->bh = NULL;
+    qemu_aio_release(acb);
+}
+
+static const AIOCBInfo bdrv_em_aiocb_info = {
+    .aiocb_size         = sizeof(BlockDriverAIOCBSync),
+    .cancel             = bdrv_aio_cancel_em,
+};
+
+static void bdrv_aio_bh_cb(void *opaque)
+{
+    BlockDriverAIOCBSync *acb = opaque;
+
+    if (!acb->is_write)
+        qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
+    qemu_vfree(acb->bounce);
+    acb->common.cb(acb->common.opaque, acb->ret);
+    qemu_bh_delete(acb->bh);
+    acb->bh = NULL;
+    qemu_aio_release(acb);
+}
+
+static BlockDriverAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs,
+                                            int64_t sector_num,
+                                            QEMUIOVector *qiov,
+                                            int nb_sectors,
+                                            BlockDriverCompletionFunc *cb,
+                                            void *opaque,
+                                            int is_write)
+
+{
+    BlockDriverAIOCBSync *acb;
+
+    acb = qemu_aio_get(&bdrv_em_aiocb_info, bs, cb, opaque);
+    acb->is_write = is_write;
+    acb->qiov = qiov;
+    acb->bounce = qemu_blockalign(bs, qiov->size);
+    acb->bh = qemu_bh_new(bdrv_aio_bh_cb, acb);
+
+    if (is_write) {
+        qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
+        acb->ret = bs->drv->bdrv_write(bs, sector_num, acb->bounce, nb_sectors);
+    } else {
+        acb->ret = bs->drv->bdrv_read(bs, sector_num, acb->bounce, nb_sectors);
+    }
+
+    qemu_bh_schedule(acb->bh);
+
+    return &acb->common;
+}
+
+static BlockDriverAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 0);
+}
+
+static BlockDriverAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 1);
+}
+
+
+typedef struct BlockDriverAIOCBCoroutine {
+    BlockDriverAIOCB common;
+    BlockRequest req;
+    bool is_write;
+    bool *done;
+    QEMUBH* bh;
+} BlockDriverAIOCBCoroutine;
+
+static void bdrv_aio_co_cancel_em(BlockDriverAIOCB *blockacb)
+{
+    BlockDriverAIOCBCoroutine *acb =
+        container_of(blockacb, BlockDriverAIOCBCoroutine, common);
+    bool done = false;
+
+    acb->done = &done;
+    while (!done) {
+        qemu_aio_wait();
+    }
+}
+
+static const AIOCBInfo bdrv_em_co_aiocb_info = {
+    .aiocb_size         = sizeof(BlockDriverAIOCBCoroutine),
+    .cancel             = bdrv_aio_co_cancel_em,
+};
+
+static void bdrv_co_em_bh(void *opaque)
+{
+    BlockDriverAIOCBCoroutine *acb = opaque;
+
+    acb->common.cb(acb->common.opaque, acb->req.error);
+
+    if (acb->done) {
+        *acb->done = true;
+    }
+
+    qemu_bh_delete(acb->bh);
+    qemu_aio_release(acb);
+}
+
+/* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
+static void coroutine_fn bdrv_co_do_rw(void *opaque)
+{
+    BlockDriverAIOCBCoroutine *acb = opaque;
+    BlockDriverState *bs = acb->common.bs;
+
+    if (!acb->is_write) {
+        acb->req.error = bdrv_co_do_readv(bs, acb->req.sector,
+            acb->req.nb_sectors, acb->req.qiov, 0);
+    } else {
+        acb->req.error = bdrv_co_do_writev(bs, acb->req.sector,
+            acb->req.nb_sectors, acb->req.qiov, 0);
+    }
+
+    acb->bh = qemu_bh_new(bdrv_co_em_bh, acb);
+    qemu_bh_schedule(acb->bh);
+}
+
+static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
+                                               int64_t sector_num,
+                                               QEMUIOVector *qiov,
+                                               int nb_sectors,
+                                               BlockDriverCompletionFunc *cb,
+                                               void *opaque,
+                                               bool is_write)
+{
+    Coroutine *co;
+    BlockDriverAIOCBCoroutine *acb;
+
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
+    acb->req.sector = sector_num;
+    acb->req.nb_sectors = nb_sectors;
+    acb->req.qiov = qiov;
+    acb->is_write = is_write;
+    acb->done = NULL;
+
+    co = qemu_coroutine_create(bdrv_co_do_rw);
+    qemu_coroutine_enter(co, acb);
+
+    return &acb->common;
+}
+
+static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
+{
+    BlockDriverAIOCBCoroutine *acb = opaque;
+    BlockDriverState *bs = acb->common.bs;
+
+    acb->req.error = bdrv_co_flush(bs);
+    acb->bh = qemu_bh_new(bdrv_co_em_bh, acb);
+    qemu_bh_schedule(acb->bh);
+}
+
+BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    trace_bdrv_aio_flush(bs, opaque);
+
+    Coroutine *co;
+    BlockDriverAIOCBCoroutine *acb;
+
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
+    acb->done = NULL;
+
+    co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
+    qemu_coroutine_enter(co, acb);
+
+    return &acb->common;
+}
+
+static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
+{
+    BlockDriverAIOCBCoroutine *acb = opaque;
+    BlockDriverState *bs = acb->common.bs;
+
+    acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
+    acb->bh = qemu_bh_new(bdrv_co_em_bh, acb);
+    qemu_bh_schedule(acb->bh);
+}
+
+BlockDriverAIOCB *bdrv_aio_discard(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    Coroutine *co;
+    BlockDriverAIOCBCoroutine *acb;
+
+    trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
+
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
+    acb->req.sector = sector_num;
+    acb->req.nb_sectors = nb_sectors;
+    acb->done = NULL;
+    co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
+    qemu_coroutine_enter(co, acb);
+
+    return &acb->common;
+}
+
+void bdrv_init(void)
+{
+    module_call_init(MODULE_INIT_BLOCK);
+}
+
+void bdrv_init_with_whitelist(void)
+{
+    use_bdrv_whitelist = 1;
+    bdrv_init();
+}
+
+void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
+                   BlockDriverCompletionFunc *cb, void *opaque)
+{
+    BlockDriverAIOCB *acb;
+
+    acb = g_slice_alloc(aiocb_info->aiocb_size);
+    acb->aiocb_info = aiocb_info;
+    acb->bs = bs;
+    acb->cb = cb;
+    acb->opaque = opaque;
+    return acb;
+}
+
+void qemu_aio_release(void *p)
+{
+    BlockDriverAIOCB *acb = p;
+    g_slice_free1(acb->aiocb_info->aiocb_size, acb);
+}
+
+/**************************************************************/
+/* Coroutine block device emulation */
+
+typedef struct CoroutineIOCompletion {
+    Coroutine *coroutine;
+    int ret;
+} CoroutineIOCompletion;
+
+static void bdrv_co_io_em_complete(void *opaque, int ret)
+{
+    CoroutineIOCompletion *co = opaque;
+
+    co->ret = ret;
+    qemu_coroutine_enter(co->coroutine, NULL);
+}
+
+static int coroutine_fn bdrv_co_io_em(BlockDriverState *bs, int64_t sector_num,
+                                      int nb_sectors, QEMUIOVector *iov,
+                                      bool is_write)
+{
+    CoroutineIOCompletion co = {
+        .coroutine = qemu_coroutine_self(),
+    };
+    BlockDriverAIOCB *acb;
+
+    if (is_write) {
+        acb = bs->drv->bdrv_aio_writev(bs, sector_num, iov, nb_sectors,
+                                       bdrv_co_io_em_complete, &co);
+    } else {
+        acb = bs->drv->bdrv_aio_readv(bs, sector_num, iov, nb_sectors,
+                                      bdrv_co_io_em_complete, &co);
+    }
+
+    trace_bdrv_co_io_em(bs, sector_num, nb_sectors, is_write, acb);
+    if (!acb) {
+        return -EIO;
+    }
+    qemu_coroutine_yield();
+
+    return co.ret;
+}
+
+static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
+                                         int64_t sector_num, int nb_sectors,
+                                         QEMUIOVector *iov)
+{
+    return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, false);
+}
+
+static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs,
+                                         int64_t sector_num, int nb_sectors,
+                                         QEMUIOVector *iov)
+{
+    return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, true);
+}
+
+static void coroutine_fn bdrv_flush_co_entry(void *opaque)
+{
+    RwCo *rwco = opaque;
+
+    rwco->ret = bdrv_co_flush(rwco->bs);
+}
+
+int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
+{
+    int ret;
+
+    if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
+        return 0;
+    }
+
+    /* Write back cached data to the OS even with cache=unsafe */
+    BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
+    if (bs->drv->bdrv_co_flush_to_os) {
+        ret = bs->drv->bdrv_co_flush_to_os(bs);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    /* But don't actually force it to the disk with cache=unsafe */
+    if (bs->open_flags & BDRV_O_NO_FLUSH) {
+        goto flush_parent;
+    }
+
+    BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
+    if (bs->drv->bdrv_co_flush_to_disk) {
+        ret = bs->drv->bdrv_co_flush_to_disk(bs);
+    } else if (bs->drv->bdrv_aio_flush) {
+        BlockDriverAIOCB *acb;
+        CoroutineIOCompletion co = {
+            .coroutine = qemu_coroutine_self(),
+        };
+
+        acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
+        if (acb == NULL) {
+            ret = -EIO;
+        } else {
+            qemu_coroutine_yield();
+            ret = co.ret;
+        }
+    } else {
+        /*
+         * Some block drivers always operate in either writethrough or unsafe
+         * mode and don't support bdrv_flush therefore. Usually qemu doesn't
+         * know how the server works (because the behaviour is hardcoded or
+         * depends on server-side configuration), so we can't ensure that
+         * everything is safe on disk. Returning an error doesn't work because
+         * that would break guests even if the server operates in writethrough
+         * mode.
+         *
+         * Let's hope the user knows what he's doing.
+         */
+        ret = 0;
+    }
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
+     * in the case of cache=unsafe, so there are no useless flushes.
+     */
+flush_parent:
+    return bdrv_co_flush(bs->file);
+}
+
+void bdrv_invalidate_cache(BlockDriverState *bs)
+{
+    if (bs->drv && bs->drv->bdrv_invalidate_cache) {
+        bs->drv->bdrv_invalidate_cache(bs);
+    }
+}
+
+void bdrv_invalidate_cache_all(void)
+{
+    BlockDriverState *bs;
+
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        bdrv_invalidate_cache(bs);
+    }
+}
+
+void bdrv_clear_incoming_migration_all(void)
+{
+    BlockDriverState *bs;
+
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        bs->open_flags = bs->open_flags & ~(BDRV_O_INCOMING);
+    }
+}
+
+int bdrv_flush(BlockDriverState *bs)
+{
+    Coroutine *co;
+    RwCo rwco = {
+        .bs = bs,
+        .ret = NOT_DONE,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_flush_co_entry(&rwco);
+    } else {
+        co = qemu_coroutine_create(bdrv_flush_co_entry);
+        qemu_coroutine_enter(co, &rwco);
+        while (rwco.ret == NOT_DONE) {
+            qemu_aio_wait();
+        }
+    }
+
+    return rwco.ret;
+}
+
+static void coroutine_fn bdrv_discard_co_entry(void *opaque)
+{
+    RwCo *rwco = opaque;
+
+    rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors);
+}
+
+int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
+                                 int nb_sectors)
+{
+    if (!bs->drv) {
+        return -ENOMEDIUM;
+    } else if (bdrv_check_request(bs, sector_num, nb_sectors)) {
+        return -EIO;
+    } else if (bs->read_only) {
+        return -EROFS;
+    }
+
+    if (bs->dirty_bitmap) {
+        bdrv_reset_dirty(bs, sector_num, nb_sectors);
+    }
+
+    /* Do nothing if disabled.  */
+    if (!(bs->open_flags & BDRV_O_UNMAP)) {
+        return 0;
+    }
+
+    if (bs->drv->bdrv_co_discard) {
+        return bs->drv->bdrv_co_discard(bs, sector_num, nb_sectors);
+    } else if (bs->drv->bdrv_aio_discard) {
+        BlockDriverAIOCB *acb;
+        CoroutineIOCompletion co = {
+            .coroutine = qemu_coroutine_self(),
+        };
+
+        acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
+                                        bdrv_co_io_em_complete, &co);
+        if (acb == NULL) {
+            return -EIO;
+        } else {
+            qemu_coroutine_yield();
+            return co.ret;
+        }
+    } else {
+        return 0;
+    }
+}
+
+int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
+{
+    Coroutine *co;
+    RwCo rwco = {
+        .bs = bs,
+        .sector_num = sector_num,
+        .nb_sectors = nb_sectors,
+        .ret = NOT_DONE,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_discard_co_entry(&rwco);
+    } else {
+        co = qemu_coroutine_create(bdrv_discard_co_entry);
+        qemu_coroutine_enter(co, &rwco);
+        while (rwco.ret == NOT_DONE) {
+            qemu_aio_wait();
+        }
+    }
+
+    return rwco.ret;
+}
+
+/**************************************************************/
+/* removable device support */
+
+/**
+ * Return TRUE if the media is present
+ */
+int bdrv_is_inserted(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+
+    if (!drv)
+        return 0;
+    if (!drv->bdrv_is_inserted)
+        return 1;
+    return drv->bdrv_is_inserted(bs);
+}
+
+/**
+ * Return whether the media changed since the last call to this
+ * function, or -ENOTSUP if we don't know.  Most drivers don't know.
+ */
+int bdrv_media_changed(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+
+    if (drv && drv->bdrv_media_changed) {
+        return drv->bdrv_media_changed(bs);
+    }
+    return -ENOTSUP;
+}
+
+/**
+ * If eject_flag is TRUE, eject the media. Otherwise, close the tray
+ */
+void bdrv_eject(BlockDriverState *bs, bool eject_flag)
+{
+    BlockDriver *drv = bs->drv;
+
+    if (drv && drv->bdrv_eject) {
+        drv->bdrv_eject(bs, eject_flag);
+    }
+
+    if (bs->device_name[0] != '\0') {
+        bdrv_emit_qmp_eject_event(bs, eject_flag);
+    }
+}
+
+/**
+ * Lock or unlock the media (if it is locked, the user won't be able
+ * to eject it manually).
+ */
+void bdrv_lock_medium(BlockDriverState *bs, bool locked)
+{
+    BlockDriver *drv = bs->drv;
+
+    trace_bdrv_lock_medium(bs, locked);
+
+    if (drv && drv->bdrv_lock_medium) {
+        drv->bdrv_lock_medium(bs, locked);
+    }
+}
+
+/* needed for generic scsi interface */
+
+int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
+{
+    BlockDriver *drv = bs->drv;
+
+    if (drv && drv->bdrv_ioctl)
+        return drv->bdrv_ioctl(bs, req, buf);
+    return -ENOTSUP;
+}
+
+BlockDriverAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
+        unsigned long int req, void *buf,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    BlockDriver *drv = bs->drv;
+
+    if (drv && drv->bdrv_aio_ioctl)
+        return drv->bdrv_aio_ioctl(bs, req, buf, cb, opaque);
+    return NULL;
+}
+
+void bdrv_set_buffer_alignment(BlockDriverState *bs, int align)
+{
+    bs->buffer_alignment = align;
+}
+
+void *qemu_blockalign(BlockDriverState *bs, size_t size)
+{
+    return qemu_memalign((bs && bs->buffer_alignment) ? bs->buffer_alignment : 512, size);
+}
+
+/*
+ * Check if all memory in this vector is sector aligned.
+ */
+bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
+{
+    int i;
+
+    for (i = 0; i < qiov->niov; i++) {
+        if ((uintptr_t) qiov->iov[i].iov_base % bs->buffer_alignment) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void bdrv_set_dirty_tracking(BlockDriverState *bs, int granularity)
+{
+    int64_t bitmap_size;
+
+    assert((granularity & (granularity - 1)) == 0);
+
+    if (granularity) {
+        granularity >>= BDRV_SECTOR_BITS;
+        assert(!bs->dirty_bitmap);
+        bitmap_size = (bdrv_getlength(bs) >> BDRV_SECTOR_BITS);
+        bs->dirty_bitmap = hbitmap_alloc(bitmap_size, ffs(granularity) - 1);
+    } else {
+        if (bs->dirty_bitmap) {
+            hbitmap_free(bs->dirty_bitmap);
+            bs->dirty_bitmap = NULL;
+        }
+    }
+}
+
+int bdrv_get_dirty(BlockDriverState *bs, int64_t sector)
+{
+    if (bs->dirty_bitmap) {
+        return hbitmap_get(bs->dirty_bitmap, sector);
+    } else {
+        return 0;
+    }
+}
+
+void bdrv_dirty_iter_init(BlockDriverState *bs, HBitmapIter *hbi)
+{
+    hbitmap_iter_init(hbi, bs->dirty_bitmap, 0);
+}
+
+void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
+                    int nr_sectors)
+{
+    hbitmap_set(bs->dirty_bitmap, cur_sector, nr_sectors);
+}
+
+void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector,
+                      int nr_sectors)
+{
+    hbitmap_reset(bs->dirty_bitmap, cur_sector, nr_sectors);
+}
+
+int64_t bdrv_get_dirty_count(BlockDriverState *bs)
+{
+    if (bs->dirty_bitmap) {
+        return hbitmap_count(bs->dirty_bitmap);
+    } else {
+        return 0;
+    }
+}
+
+void bdrv_set_in_use(BlockDriverState *bs, int in_use)
+{
+    assert(bs->in_use != in_use);
+    bs->in_use = in_use;
+}
+
+int bdrv_in_use(BlockDriverState *bs)
+{
+    return bs->in_use;
+}
+
+void bdrv_iostatus_enable(BlockDriverState *bs)
+{
+    bs->iostatus_enabled = true;
+    bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+}
+
+/* The I/O status is only enabled if the drive explicitly
+ * enables it _and_ the VM is configured to stop on errors */
+bool bdrv_iostatus_is_enabled(const BlockDriverState *bs)
+{
+    return (bs->iostatus_enabled &&
+           (bs->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
+            bs->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
+            bs->on_read_error == BLOCKDEV_ON_ERROR_STOP));
+}
+
+void bdrv_iostatus_disable(BlockDriverState *bs)
+{
+    bs->iostatus_enabled = false;
+}
+
+void bdrv_iostatus_reset(BlockDriverState *bs)
+{
+    if (bdrv_iostatus_is_enabled(bs)) {
+        bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+        if (bs->job) {
+            block_job_iostatus_reset(bs->job);
+        }
+    }
+}
+
+void bdrv_iostatus_set_err(BlockDriverState *bs, int error)
+{
+    assert(bdrv_iostatus_is_enabled(bs));
+    if (bs->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
+        bs->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
+                                         BLOCK_DEVICE_IO_STATUS_FAILED;
+    }
+}
+
+void
+bdrv_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie, int64_t bytes,
+        enum BlockAcctType type)
+{
+    assert(type < BDRV_MAX_IOTYPE);
+
+    cookie->bytes = bytes;
+    cookie->start_time_ns = get_clock();
+    cookie->type = type;
+}
+
+void
+bdrv_acct_done(BlockDriverState *bs, BlockAcctCookie *cookie)
+{
+    assert(cookie->type < BDRV_MAX_IOTYPE);
+
+    bs->nr_bytes[cookie->type] += cookie->bytes;
+    bs->nr_ops[cookie->type]++;
+    bs->total_time_ns[cookie->type] += get_clock() - cookie->start_time_ns;
+}
+
+void bdrv_img_create(const char *filename, const char *fmt,
+                     const char *base_filename, const char *base_fmt,
+                     char *options, uint64_t img_size, int flags,
+                     Error **errp, bool quiet)
+{
+    QEMUOptionParameter *param = NULL, *create_options = NULL;
+    QEMUOptionParameter *backing_fmt, *backing_file, *size;
+    BlockDriverState *bs = NULL;
+    BlockDriver *drv, *proto_drv;
+    BlockDriver *backing_drv = NULL;
+    int ret = 0;
+
+    /* Find driver and parse its options */
+    drv = bdrv_find_format(fmt);
+    if (!drv) {
+        error_setg(errp, "Unknown file format '%s'", fmt);
+        return;
+    }
+
+    proto_drv = bdrv_find_protocol(filename, true);
+    if (!proto_drv) {
+        error_setg(errp, "Unknown protocol '%s'", filename);
+        return;
+    }
+
+    create_options = append_option_parameters(create_options,
+                                              drv->create_options);
+    create_options = append_option_parameters(create_options,
+                                              proto_drv->create_options);
+
+    /* Create parameter list with default values */
+    param = parse_option_parameters("", create_options, param);
+
+    set_option_parameter_int(param, BLOCK_OPT_SIZE, img_size);
+
+    /* Parse -o options */
+    if (options) {
+        param = parse_option_parameters(options, create_options, param);
+        if (param == NULL) {
+            error_setg(errp, "Invalid options for file format '%s'.", fmt);
+            goto out;
+        }
+    }
+
+    if (base_filename) {
+        if (set_option_parameter(param, BLOCK_OPT_BACKING_FILE,
+                                 base_filename)) {
+            error_setg(errp, "Backing file not supported for file format '%s'",
+                       fmt);
+            goto out;
+        }
+    }
+
+    if (base_fmt) {
+        if (set_option_parameter(param, BLOCK_OPT_BACKING_FMT, base_fmt)) {
+            error_setg(errp, "Backing file format not supported for file "
+                             "format '%s'", fmt);
+            goto out;
+        }
+    }
+
+    backing_file = get_option_parameter(param, BLOCK_OPT_BACKING_FILE);
+    if (backing_file && backing_file->value.s) {
+        if (!strcmp(filename, backing_file->value.s)) {
+            error_setg(errp, "Error: Trying to create an image with the "
+                             "same filename as the backing file");
+            goto out;
+        }
+    }
+
+    backing_fmt = get_option_parameter(param, BLOCK_OPT_BACKING_FMT);
+    if (backing_fmt && backing_fmt->value.s) {
+        backing_drv = bdrv_find_format(backing_fmt->value.s);
+        if (!backing_drv) {
+            error_setg(errp, "Unknown backing file format '%s'",
+                       backing_fmt->value.s);
+            goto out;
+        }
+    }
+
+    // The size for the image must always be specified, with one exception:
+    // If we are using a backing file, we can obtain the size from there
+    size = get_option_parameter(param, BLOCK_OPT_SIZE);
+    if (size && size->value.n == -1) {
+        if (backing_file && backing_file->value.s) {
+            uint64_t size;
+            char buf[32];
+            int back_flags;
+
+            /* backing files always opened read-only */
+            back_flags =
+                flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
+
+            bs = bdrv_new("");
+
+            ret = bdrv_open(bs, backing_file->value.s, NULL, back_flags,
+                            backing_drv);
+            if (ret < 0) {
+                error_setg_errno(errp, -ret, "Could not open '%s'",
+                                 backing_file->value.s);
+                goto out;
+            }
+            bdrv_get_geometry(bs, &size);
+            size *= 512;
+
+            snprintf(buf, sizeof(buf), "%" PRId64, size);
+            set_option_parameter(param, BLOCK_OPT_SIZE, buf);
+        } else {
+            error_setg(errp, "Image creation needs a size parameter");
+            goto out;
+        }
+    }
+
+    if (!quiet) {
+        printf("Formatting '%s', fmt=%s ", filename, fmt);
+        print_option_parameters(param);
+        puts("");
+    }
+    ret = bdrv_create(drv, filename, param);
+    if (ret < 0) {
+        if (ret == -ENOTSUP) {
+            error_setg(errp,"Formatting or formatting option not supported for "
+                            "file format '%s'", fmt);
+        } else if (ret == -EFBIG) {
+            const char *cluster_size_hint = "";
+            if (get_option_parameter(create_options, BLOCK_OPT_CLUSTER_SIZE)) {
+                cluster_size_hint = " (try using a larger cluster size)";
+            }
+            error_setg(errp, "The image size is too large for file format '%s'%s",
+                       fmt, cluster_size_hint);
+        } else {
+            error_setg(errp, "%s: error while creating %s: %s", filename, fmt,
+                       strerror(-ret));
+        }
+    }
+
+out:
+    free_option_parameters(create_options);
+    free_option_parameters(param);
+
+    if (bs) {
+        bdrv_delete(bs);
+    }
+}
+
+AioContext *bdrv_get_aio_context(BlockDriverState *bs)
+{
+    /* Currently BlockDriverState always uses the main loop AioContext */
+    return qemu_get_aio_context();
+}
+
+void bdrv_add_before_write_notifier(BlockDriverState *bs,
+                                    NotifierWithReturn *notifier)
+{
+    notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
+}
diff --git a/contrib/qemu/block/qcow.c b/contrib/qemu/block/qcow.c
new file mode 100644
index 000000000..5239bd68f
--- /dev/null
+++ b/contrib/qemu/block/qcow.c
@@ -0,0 +1,914 @@
+/*
+ * Block driver for the QCOW format
+ *
+ * Copyright (c) 2004-2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include <zlib.h>
+#include "qemu/aes.h"
+#include "migration/migration.h"
+
+/**************************************************************/
+/* QEMU COW block driver with compression and encryption support */
+
+#define QCOW_MAGIC (('Q' << 24) | ('F' << 16) | ('I' << 8) | 0xfb)
+#define QCOW_VERSION 1
+
+#define QCOW_CRYPT_NONE 0
+#define QCOW_CRYPT_AES  1
+
+#define QCOW_OFLAG_COMPRESSED (1LL << 63)
+
+typedef struct QCowHeader {
+    uint32_t magic;
+    uint32_t version;
+    uint64_t backing_file_offset;
+    uint32_t backing_file_size;
+    uint32_t mtime;
+    uint64_t size; /* in bytes */
+    uint8_t cluster_bits;
+    uint8_t l2_bits;
+    uint32_t crypt_method;
+    uint64_t l1_table_offset;
+} QCowHeader;
+
+#define L2_CACHE_SIZE 16
+
+typedef struct BDRVQcowState {
+    int cluster_bits;
+    int cluster_size;
+    int cluster_sectors;
+    int l2_bits;
+    int l2_size;
+    int l1_size;
+    uint64_t cluster_offset_mask;
+    uint64_t l1_table_offset;
+    uint64_t *l1_table;
+    uint64_t *l2_cache;
+    uint64_t l2_cache_offsets[L2_CACHE_SIZE];
+    uint32_t l2_cache_counts[L2_CACHE_SIZE];
+    uint8_t *cluster_cache;
+    uint8_t *cluster_data;
+    uint64_t cluster_cache_offset;
+    uint32_t crypt_method; /* current crypt method, 0 if no key yet */
+    uint32_t crypt_method_header;
+    AES_KEY aes_encrypt_key;
+    AES_KEY aes_decrypt_key;
+    CoMutex lock;
+    Error *migration_blocker;
+} BDRVQcowState;
+
+static int decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset);
+
+static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename)
+{
+    const QCowHeader *cow_header = (const void *)buf;
+
+    if (buf_size >= sizeof(QCowHeader) &&
+        be32_to_cpu(cow_header->magic) == QCOW_MAGIC &&
+        be32_to_cpu(cow_header->version) == QCOW_VERSION)
+        return 100;
+    else
+        return 0;
+}
+
+static int qcow_open(BlockDriverState *bs, QDict *options, int flags)
+{
+    BDRVQcowState *s = bs->opaque;
+    int len, i, shift, ret;
+    QCowHeader header;
+
+    ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
+    if (ret < 0) {
+        goto fail;
+    }
+    be32_to_cpus(&header.magic);
+    be32_to_cpus(&header.version);
+    be64_to_cpus(&header.backing_file_offset);
+    be32_to_cpus(&header.backing_file_size);
+    be32_to_cpus(&header.mtime);
+    be64_to_cpus(&header.size);
+    be32_to_cpus(&header.crypt_method);
+    be64_to_cpus(&header.l1_table_offset);
+
+    if (header.magic != QCOW_MAGIC) {
+        ret = -EMEDIUMTYPE;
+        goto fail;
+    }
+    if (header.version != QCOW_VERSION) {
+        char version[64];
+        snprintf(version, sizeof(version), "QCOW version %d", header.version);
+        qerror_report(QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
+            bs->device_name, "qcow", version);
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    if (header.size <= 1 || header.cluster_bits < 9) {
+        ret = -EINVAL;
+        goto fail;
+    }
+    if (header.crypt_method > QCOW_CRYPT_AES) {
+        ret = -EINVAL;
+        goto fail;
+    }
+    s->crypt_method_header = header.crypt_method;
+    if (s->crypt_method_header) {
+        bs->encrypted = 1;
+    }
+    s->cluster_bits = header.cluster_bits;
+    s->cluster_size = 1 << s->cluster_bits;
+    s->cluster_sectors = 1 << (s->cluster_bits - 9);
+    s->l2_bits = header.l2_bits;
+    s->l2_size = 1 << s->l2_bits;
+    bs->total_sectors = header.size / 512;
+    s->cluster_offset_mask = (1LL << (63 - s->cluster_bits)) - 1;
+
+    /* read the level 1 table */
+    shift = s->cluster_bits + s->l2_bits;
+    s->l1_size = (header.size + (1LL << shift) - 1) >> shift;
+
+    s->l1_table_offset = header.l1_table_offset;
+    s->l1_table = g_malloc(s->l1_size * sizeof(uint64_t));
+
+    ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
+               s->l1_size * sizeof(uint64_t));
+    if (ret < 0) {
+        goto fail;
+    }
+
+    for(i = 0;i < s->l1_size; i++) {
+        be64_to_cpus(&s->l1_table[i]);
+    }
+    /* alloc L2 cache */
+    s->l2_cache = g_malloc(s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t));
+    s->cluster_cache = g_malloc(s->cluster_size);
+    s->cluster_data = g_malloc(s->cluster_size);
+    s->cluster_cache_offset = -1;
+
+    /* read the backing file name */
+    if (header.backing_file_offset != 0) {
+        len = header.backing_file_size;
+        if (len > 1023) {
+            len = 1023;
+        }
+        ret = bdrv_pread(bs->file, header.backing_file_offset,
+                   bs->backing_file, len);
+        if (ret < 0) {
+            goto fail;
+        }
+        bs->backing_file[len] = '\0';
+    }
+
+    /* Disable migration when qcow images are used */
+    error_set(&s->migration_blocker,
+              QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
+              "qcow", bs->device_name, "live migration");
+    migrate_add_blocker(s->migration_blocker);
+
+    qemu_co_mutex_init(&s->lock);
+    return 0;
+
+ fail:
+    g_free(s->l1_table);
+    g_free(s->l2_cache);
+    g_free(s->cluster_cache);
+    g_free(s->cluster_data);
+    return ret;
+}
+
+
+/* We have nothing to do for QCOW reopen, stubs just return
+ * success */
+static int qcow_reopen_prepare(BDRVReopenState *state,
+                               BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
+static int qcow_set_key(BlockDriverState *bs, const char *key)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint8_t keybuf[16];
+    int len, i;
+
+    memset(keybuf, 0, 16);
+    len = strlen(key);
+    if (len > 16)
+        len = 16;
+    /* XXX: we could compress the chars to 7 bits to increase
+       entropy */
+    for(i = 0;i < len;i++) {
+        keybuf[i] = key[i];
+    }
+    s->crypt_method = s->crypt_method_header;
+
+    if (AES_set_encrypt_key(keybuf, 128, &s->aes_encrypt_key) != 0)
+        return -1;
+    if (AES_set_decrypt_key(keybuf, 128, &s->aes_decrypt_key) != 0)
+        return -1;
+    return 0;
+}
+
+/* The crypt function is compatible with the linux cryptoloop
+   algorithm for < 4 GB images. NOTE: out_buf == in_buf is
+   supported */
+static void encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
+                            uint8_t *out_buf, const uint8_t *in_buf,
+                            int nb_sectors, int enc,
+                            const AES_KEY *key)
+{
+    union {
+        uint64_t ll[2];
+        uint8_t b[16];
+    } ivec;
+    int i;
+
+    for(i = 0; i < nb_sectors; i++) {
+        ivec.ll[0] = cpu_to_le64(sector_num);
+        ivec.ll[1] = 0;
+        AES_cbc_encrypt(in_buf, out_buf, 512, key,
+                        ivec.b, enc);
+        sector_num++;
+        in_buf += 512;
+        out_buf += 512;
+    }
+}
+
+/* 'allocate' is:
+ *
+ * 0 to not allocate.
+ *
+ * 1 to allocate a normal cluster (for sector indexes 'n_start' to
+ * 'n_end')
+ *
+ * 2 to allocate a compressed cluster of size
+ * 'compressed_size'. 'compressed_size' must be > 0 and <
+ * cluster_size
+ *
+ * return 0 if not allocated.
+ */
+static uint64_t get_cluster_offset(BlockDriverState *bs,
+                                   uint64_t offset, int allocate,
+                                   int compressed_size,
+                                   int n_start, int n_end)
+{
+    BDRVQcowState *s = bs->opaque;
+    int min_index, i, j, l1_index, l2_index;
+    uint64_t l2_offset, *l2_table, cluster_offset, tmp;
+    uint32_t min_count;
+    int new_l2_table;
+
+    l1_index = offset >> (s->l2_bits + s->cluster_bits);
+    l2_offset = s->l1_table[l1_index];
+    new_l2_table = 0;
+    if (!l2_offset) {
+        if (!allocate)
+            return 0;
+        /* allocate a new l2 entry */
+        l2_offset = bdrv_getlength(bs->file);
+        /* round to cluster size */
+        l2_offset = (l2_offset + s->cluster_size - 1) & ~(s->cluster_size - 1);
+        /* update the L1 entry */
+        s->l1_table[l1_index] = l2_offset;
+        tmp = cpu_to_be64(l2_offset);
+        if (bdrv_pwrite_sync(bs->file,
+                s->l1_table_offset + l1_index * sizeof(tmp),
+                &tmp, sizeof(tmp)) < 0)
+            return 0;
+        new_l2_table = 1;
+    }
+    for(i = 0; i < L2_CACHE_SIZE; i++) {
+        if (l2_offset == s->l2_cache_offsets[i]) {
+            /* increment the hit count */
+            if (++s->l2_cache_counts[i] == 0xffffffff) {
+                for(j = 0; j < L2_CACHE_SIZE; j++) {
+                    s->l2_cache_counts[j] >>= 1;
+                }
+            }
+            l2_table = s->l2_cache + (i << s->l2_bits);
+            goto found;
+        }
+    }
+    /* not found: load a new entry in the least used one */
+    min_index = 0;
+    min_count = 0xffffffff;
+    for(i = 0; i < L2_CACHE_SIZE; i++) {
+        if (s->l2_cache_counts[i] < min_count) {
+            min_count = s->l2_cache_counts[i];
+            min_index = i;
+        }
+    }
+    l2_table = s->l2_cache + (min_index << s->l2_bits);
+    if (new_l2_table) {
+        memset(l2_table, 0, s->l2_size * sizeof(uint64_t));
+        if (bdrv_pwrite_sync(bs->file, l2_offset, l2_table,
+                s->l2_size * sizeof(uint64_t)) < 0)
+            return 0;
+    } else {
+        if (bdrv_pread(bs->file, l2_offset, l2_table, s->l2_size * sizeof(uint64_t)) !=
+            s->l2_size * sizeof(uint64_t))
+            return 0;
+    }
+    s->l2_cache_offsets[min_index] = l2_offset;
+    s->l2_cache_counts[min_index] = 1;
+ found:
+    l2_index = (offset >> s->cluster_bits) & (s->l2_size - 1);
+    cluster_offset = be64_to_cpu(l2_table[l2_index]);
+    if (!cluster_offset ||
+        ((cluster_offset & QCOW_OFLAG_COMPRESSED) && allocate == 1)) {
+        if (!allocate)
+            return 0;
+        /* allocate a new cluster */
+        if ((cluster_offset & QCOW_OFLAG_COMPRESSED) &&
+            (n_end - n_start) < s->cluster_sectors) {
+            /* if the cluster is already compressed, we must
+               decompress it in the case it is not completely
+               overwritten */
+            if (decompress_cluster(bs, cluster_offset) < 0)
+                return 0;
+            cluster_offset = bdrv_getlength(bs->file);
+            cluster_offset = (cluster_offset + s->cluster_size - 1) &
+                ~(s->cluster_size - 1);
+            /* write the cluster content */
+            if (bdrv_pwrite(bs->file, cluster_offset, s->cluster_cache, s->cluster_size) !=
+                s->cluster_size)
+                return -1;
+        } else {
+            cluster_offset = bdrv_getlength(bs->file);
+            if (allocate == 1) {
+                /* round to cluster size */
+                cluster_offset = (cluster_offset + s->cluster_size - 1) &
+                    ~(s->cluster_size - 1);
+                bdrv_truncate(bs->file, cluster_offset + s->cluster_size);
+                /* if encrypted, we must initialize the cluster
+                   content which won't be written */
+                if (s->crypt_method &&
+                    (n_end - n_start) < s->cluster_sectors) {
+                    uint64_t start_sect;
+                    start_sect = (offset & ~(s->cluster_size - 1)) >> 9;
+                    memset(s->cluster_data + 512, 0x00, 512);
+                    for(i = 0; i < s->cluster_sectors; i++) {
+                        if (i < n_start || i >= n_end) {
+                            encrypt_sectors(s, start_sect + i,
+                                            s->cluster_data,
+                                            s->cluster_data + 512, 1, 1,
+                                            &s->aes_encrypt_key);
+                            if (bdrv_pwrite(bs->file, cluster_offset + i * 512,
+                                            s->cluster_data, 512) != 512)
+                                return -1;
+                        }
+                    }
+                }
+            } else if (allocate == 2) {
+                cluster_offset |= QCOW_OFLAG_COMPRESSED |
+                    (uint64_t)compressed_size << (63 - s->cluster_bits);
+            }
+        }
+        /* update L2 table */
+        tmp = cpu_to_be64(cluster_offset);
+        l2_table[l2_index] = tmp;
+        if (bdrv_pwrite_sync(bs->file, l2_offset + l2_index * sizeof(tmp),
+                &tmp, sizeof(tmp)) < 0)
+            return 0;
+    }
+    return cluster_offset;
+}
+
+static int coroutine_fn qcow_co_is_allocated(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, int *pnum)
+{
+    BDRVQcowState *s = bs->opaque;
+    int index_in_cluster, n;
+    uint64_t cluster_offset;
+
+    qemu_co_mutex_lock(&s->lock);
+    cluster_offset = get_cluster_offset(bs, sector_num << 9, 0, 0, 0, 0);
+    qemu_co_mutex_unlock(&s->lock);
+    index_in_cluster = sector_num & (s->cluster_sectors - 1);
+    n = s->cluster_sectors - index_in_cluster;
+    if (n > nb_sectors)
+        n = nb_sectors;
+    *pnum = n;
+    return (cluster_offset != 0);
+}
+
+static int decompress_buffer(uint8_t *out_buf, int out_buf_size,
+                             const uint8_t *buf, int buf_size)
+{
+    z_stream strm1, *strm = &strm1;
+    int ret, out_len;
+
+    memset(strm, 0, sizeof(*strm));
+
+    strm->next_in = (uint8_t *)buf;
+    strm->avail_in = buf_size;
+    strm->next_out = out_buf;
+    strm->avail_out = out_buf_size;
+
+    ret = inflateInit2(strm, -12);
+    if (ret != Z_OK)
+        return -1;
+    ret = inflate(strm, Z_FINISH);
+    out_len = strm->next_out - out_buf;
+    if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) ||
+        out_len != out_buf_size) {
+        inflateEnd(strm);
+        return -1;
+    }
+    inflateEnd(strm);
+    return 0;
+}
+
+static int decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret, csize;
+    uint64_t coffset;
+
+    coffset = cluster_offset & s->cluster_offset_mask;
+    if (s->cluster_cache_offset != coffset) {
+        csize = cluster_offset >> (63 - s->cluster_bits);
+        csize &= (s->cluster_size - 1);
+        ret = bdrv_pread(bs->file, coffset, s->cluster_data, csize);
+        if (ret != csize)
+            return -1;
+        if (decompress_buffer(s->cluster_cache, s->cluster_size,
+                              s->cluster_data, csize) < 0) {
+            return -1;
+        }
+        s->cluster_cache_offset = coffset;
+    }
+    return 0;
+}
+
+static coroutine_fn int qcow_co_readv(BlockDriverState *bs, int64_t sector_num,
+                         int nb_sectors, QEMUIOVector *qiov)
+{
+    BDRVQcowState *s = bs->opaque;
+    int index_in_cluster;
+    int ret = 0, n;
+    uint64_t cluster_offset;
+    struct iovec hd_iov;
+    QEMUIOVector hd_qiov;
+    uint8_t *buf;
+    void *orig_buf;
+
+    if (qiov->niov > 1) {
+        buf = orig_buf = qemu_blockalign(bs, qiov->size);
+    } else {
+        orig_buf = NULL;
+        buf = (uint8_t *)qiov->iov->iov_base;
+    }
+
+    qemu_co_mutex_lock(&s->lock);
+
+    while (nb_sectors != 0) {
+        /* prepare next request */
+        cluster_offset = get_cluster_offset(bs, sector_num << 9,
+                                                 0, 0, 0, 0);
+        index_in_cluster = sector_num & (s->cluster_sectors - 1);
+        n = s->cluster_sectors - index_in_cluster;
+        if (n > nb_sectors) {
+            n = nb_sectors;
+        }
+
+        if (!cluster_offset) {
+            if (bs->backing_hd) {
+                /* read from the base image */
+                hd_iov.iov_base = (void *)buf;
+                hd_iov.iov_len = n * 512;
+                qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
+                qemu_co_mutex_unlock(&s->lock);
+                ret = bdrv_co_readv(bs->backing_hd, sector_num,
+                                    n, &hd_qiov);
+                qemu_co_mutex_lock(&s->lock);
+                if (ret < 0) {
+                    goto fail;
+                }
+            } else {
+                /* Note: in this case, no need to wait */
+                memset(buf, 0, 512 * n);
+            }
+        } else if (cluster_offset & QCOW_OFLAG_COMPRESSED) {
+            /* add AIO support for compressed blocks ? */
+            if (decompress_cluster(bs, cluster_offset) < 0) {
+                goto fail;
+            }
+            memcpy(buf,
+                   s->cluster_cache + index_in_cluster * 512, 512 * n);
+        } else {
+            if ((cluster_offset & 511) != 0) {
+                goto fail;
+            }
+            hd_iov.iov_base = (void *)buf;
+            hd_iov.iov_len = n * 512;
+            qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
+            qemu_co_mutex_unlock(&s->lock);
+            ret = bdrv_co_readv(bs->file,
+                                (cluster_offset >> 9) + index_in_cluster,
+                                n, &hd_qiov);
+            qemu_co_mutex_lock(&s->lock);
+            if (ret < 0) {
+                break;
+            }
+            if (s->crypt_method) {
+                encrypt_sectors(s, sector_num, buf, buf,
+                                n, 0,
+                                &s->aes_decrypt_key);
+            }
+        }
+        ret = 0;
+
+        nb_sectors -= n;
+        sector_num += n;
+        buf += n * 512;
+    }
+
+done:
+    qemu_co_mutex_unlock(&s->lock);
+
+    if (qiov->niov > 1) {
+        qemu_iovec_from_buf(qiov, 0, orig_buf, qiov->size);
+        qemu_vfree(orig_buf);
+    }
+
+    return ret;
+
+fail:
+    ret = -EIO;
+    goto done;
+}
+
+static coroutine_fn int qcow_co_writev(BlockDriverState *bs, int64_t sector_num,
+                          int nb_sectors, QEMUIOVector *qiov)
+{
+    BDRVQcowState *s = bs->opaque;
+    int index_in_cluster;
+    uint64_t cluster_offset;
+    const uint8_t *src_buf;
+    int ret = 0, n;
+    uint8_t *cluster_data = NULL;
+    struct iovec hd_iov;
+    QEMUIOVector hd_qiov;
+    uint8_t *buf;
+    void *orig_buf;
+
+    s->cluster_cache_offset = -1; /* disable compressed cache */
+
+    if (qiov->niov > 1) {
+        buf = orig_buf = qemu_blockalign(bs, qiov->size);
+        qemu_iovec_to_buf(qiov, 0, buf, qiov->size);
+    } else {
+        orig_buf = NULL;
+        buf = (uint8_t *)qiov->iov->iov_base;
+    }
+
+    qemu_co_mutex_lock(&s->lock);
+
+    while (nb_sectors != 0) {
+
+        index_in_cluster = sector_num & (s->cluster_sectors - 1);
+        n = s->cluster_sectors - index_in_cluster;
+        if (n > nb_sectors) {
+            n = nb_sectors;
+        }
+        cluster_offset = get_cluster_offset(bs, sector_num << 9, 1, 0,
+                                            index_in_cluster,
+                                            index_in_cluster + n);
+        if (!cluster_offset || (cluster_offset & 511) != 0) {
+            ret = -EIO;
+            break;
+        }
+        if (s->crypt_method) {
+            if (!cluster_data) {
+                cluster_data = g_malloc0(s->cluster_size);
+            }
+            encrypt_sectors(s, sector_num, cluster_data, buf,
+                            n, 1, &s->aes_encrypt_key);
+            src_buf = cluster_data;
+        } else {
+            src_buf = buf;
+        }
+
+        hd_iov.iov_base = (void *)src_buf;
+        hd_iov.iov_len = n * 512;
+        qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
+        qemu_co_mutex_unlock(&s->lock);
+        ret = bdrv_co_writev(bs->file,
+                             (cluster_offset >> 9) + index_in_cluster,
+                             n, &hd_qiov);
+        qemu_co_mutex_lock(&s->lock);
+        if (ret < 0) {
+            break;
+        }
+        ret = 0;
+
+        nb_sectors -= n;
+        sector_num += n;
+        buf += n * 512;
+    }
+    qemu_co_mutex_unlock(&s->lock);
+
+    if (qiov->niov > 1) {
+        qemu_vfree(orig_buf);
+    }
+    g_free(cluster_data);
+
+    return ret;
+}
+
+static void qcow_close(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+
+    g_free(s->l1_table);
+    g_free(s->l2_cache);
+    g_free(s->cluster_cache);
+    g_free(s->cluster_data);
+
+    migrate_del_blocker(s->migration_blocker);
+    error_free(s->migration_blocker);
+}
+
+static int qcow_create(const char *filename, QEMUOptionParameter *options)
+{
+    int header_size, backing_filename_len, l1_size, shift, i;
+    QCowHeader header;
+    uint8_t *tmp;
+    int64_t total_size = 0;
+    const char *backing_file = NULL;
+    int flags = 0;
+    int ret;
+    BlockDriverState *qcow_bs;
+
+    /* Read out options */
+    while (options && options->name) {
+        if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
+            total_size = options->value.n / 512;
+        } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FILE)) {
+            backing_file = options->value.s;
+        } else if (!strcmp(options->name, BLOCK_OPT_ENCRYPT)) {
+            flags |= options->value.n ? BLOCK_FLAG_ENCRYPT : 0;
+        }
+        options++;
+    }
+
+    ret = bdrv_create_file(filename, options);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = bdrv_file_open(&qcow_bs, filename, NULL, BDRV_O_RDWR);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = bdrv_truncate(qcow_bs, 0);
+    if (ret < 0) {
+        goto exit;
+    }
+
+    memset(&header, 0, sizeof(header));
+    header.magic = cpu_to_be32(QCOW_MAGIC);
+    header.version = cpu_to_be32(QCOW_VERSION);
+    header.size = cpu_to_be64(total_size * 512);
+    header_size = sizeof(header);
+    backing_filename_len = 0;
+    if (backing_file) {
+        if (strcmp(backing_file, "fat:")) {
+            header.backing_file_offset = cpu_to_be64(header_size);
+            backing_filename_len = strlen(backing_file);
+            header.backing_file_size = cpu_to_be32(backing_filename_len);
+            header_size += backing_filename_len;
+        } else {
+            /* special backing file for vvfat */
+            backing_file = NULL;
+        }
+        header.cluster_bits = 9; /* 512 byte cluster to avoid copying
+                                    unmodifyed sectors */
+        header.l2_bits = 12; /* 32 KB L2 tables */
+    } else {
+        header.cluster_bits = 12; /* 4 KB clusters */
+        header.l2_bits = 9; /* 4 KB L2 tables */
+    }
+    header_size = (header_size + 7) & ~7;
+    shift = header.cluster_bits + header.l2_bits;
+    l1_size = ((total_size * 512) + (1LL << shift) - 1) >> shift;
+
+    header.l1_table_offset = cpu_to_be64(header_size);
+    if (flags & BLOCK_FLAG_ENCRYPT) {
+        header.crypt_method = cpu_to_be32(QCOW_CRYPT_AES);
+    } else {
+        header.crypt_method = cpu_to_be32(QCOW_CRYPT_NONE);
+    }
+
+    /* write all the data */
+    ret = bdrv_pwrite(qcow_bs, 0, &header, sizeof(header));
+    if (ret != sizeof(header)) {
+        goto exit;
+    }
+
+    if (backing_file) {
+        ret = bdrv_pwrite(qcow_bs, sizeof(header),
+            backing_file, backing_filename_len);
+        if (ret != backing_filename_len) {
+            goto exit;
+        }
+    }
+
+    tmp = g_malloc0(BDRV_SECTOR_SIZE);
+    for (i = 0; i < ((sizeof(uint64_t)*l1_size + BDRV_SECTOR_SIZE - 1)/
+        BDRV_SECTOR_SIZE); i++) {
+        ret = bdrv_pwrite(qcow_bs, header_size +
+            BDRV_SECTOR_SIZE*i, tmp, BDRV_SECTOR_SIZE);
+        if (ret != BDRV_SECTOR_SIZE) {
+            g_free(tmp);
+            goto exit;
+        }
+    }
+
+    g_free(tmp);
+    ret = 0;
+exit:
+    bdrv_delete(qcow_bs);
+    return ret;
+}
+
+static int qcow_make_empty(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint32_t l1_length = s->l1_size * sizeof(uint64_t);
+    int ret;
+
+    memset(s->l1_table, 0, l1_length);
+    if (bdrv_pwrite_sync(bs->file, s->l1_table_offset, s->l1_table,
+            l1_length) < 0)
+        return -1;
+    ret = bdrv_truncate(bs->file, s->l1_table_offset + l1_length);
+    if (ret < 0)
+        return ret;
+
+    memset(s->l2_cache, 0, s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t));
+    memset(s->l2_cache_offsets, 0, L2_CACHE_SIZE * sizeof(uint64_t));
+    memset(s->l2_cache_counts, 0, L2_CACHE_SIZE * sizeof(uint32_t));
+
+    return 0;
+}
+
+/* XXX: put compressed sectors first, then all the cluster aligned
+   tables to avoid losing bytes in alignment */
+static int qcow_write_compressed(BlockDriverState *bs, int64_t sector_num,
+                                 const uint8_t *buf, int nb_sectors)
+{
+    BDRVQcowState *s = bs->opaque;
+    z_stream strm;
+    int ret, out_len;
+    uint8_t *out_buf;
+    uint64_t cluster_offset;
+
+    if (nb_sectors != s->cluster_sectors) {
+        ret = -EINVAL;
+
+        /* Zero-pad last write if image size is not cluster aligned */
+        if (sector_num + nb_sectors == bs->total_sectors &&
+            nb_sectors < s->cluster_sectors) {
+            uint8_t *pad_buf = qemu_blockalign(bs, s->cluster_size);
+            memset(pad_buf, 0, s->cluster_size);
+            memcpy(pad_buf, buf, nb_sectors * BDRV_SECTOR_SIZE);
+            ret = qcow_write_compressed(bs, sector_num,
+                                        pad_buf, s->cluster_sectors);
+            qemu_vfree(pad_buf);
+        }
+        return ret;
+    }
+
+    out_buf = g_malloc(s->cluster_size + (s->cluster_size / 1000) + 128);
+
+    /* best compression, small window, no zlib header */
+    memset(&strm, 0, sizeof(strm));
+    ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
+                       Z_DEFLATED, -12,
+                       9, Z_DEFAULT_STRATEGY);
+    if (ret != 0) {
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    strm.avail_in = s->cluster_size;
+    strm.next_in = (uint8_t *)buf;
+    strm.avail_out = s->cluster_size;
+    strm.next_out = out_buf;
+
+    ret = deflate(&strm, Z_FINISH);
+    if (ret != Z_STREAM_END && ret != Z_OK) {
+        deflateEnd(&strm);
+        ret = -EINVAL;
+        goto fail;
+    }
+    out_len = strm.next_out - out_buf;
+
+    deflateEnd(&strm);
+
+    if (ret != Z_STREAM_END || out_len >= s->cluster_size) {
+        /* could not compress: write normal cluster */
+        ret = bdrv_write(bs, sector_num, buf, s->cluster_sectors);
+        if (ret < 0) {
+            goto fail;
+        }
+    } else {
+        cluster_offset = get_cluster_offset(bs, sector_num << 9, 2,
+                                            out_len, 0, 0);
+        if (cluster_offset == 0) {
+            ret = -EIO;
+            goto fail;
+        }
+
+        cluster_offset &= s->cluster_offset_mask;
+        ret = bdrv_pwrite(bs->file, cluster_offset, out_buf, out_len);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+    ret = 0;
+fail:
+    g_free(out_buf);
+    return ret;
+}
+
+static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
+{
+    BDRVQcowState *s = bs->opaque;
+    bdi->cluster_size = s->cluster_size;
+    return 0;
+}
+
+
+static QEMUOptionParameter qcow_create_options[] = {
+    {
+        .name = BLOCK_OPT_SIZE,
+        .type = OPT_SIZE,
+        .help = "Virtual disk size"
+    },
+    {
+        .name = BLOCK_OPT_BACKING_FILE,
+        .type = OPT_STRING,
+        .help = "File name of a base image"
+    },
+    {
+        .name = BLOCK_OPT_ENCRYPT,
+        .type = OPT_FLAG,
+        .help = "Encrypt the image"
+    },
+    { NULL }
+};
+
+static BlockDriver bdrv_qcow = {
+    .format_name	= "qcow",
+    .instance_size	= sizeof(BDRVQcowState),
+    .bdrv_probe		= qcow_probe,
+    .bdrv_open		= qcow_open,
+    .bdrv_close		= qcow_close,
+    .bdrv_reopen_prepare = qcow_reopen_prepare,
+    .bdrv_create	= qcow_create,
+    .bdrv_has_zero_init     = bdrv_has_zero_init_1,
+
+    .bdrv_co_readv          = qcow_co_readv,
+    .bdrv_co_writev         = qcow_co_writev,
+    .bdrv_co_is_allocated   = qcow_co_is_allocated,
+
+    .bdrv_set_key           = qcow_set_key,
+    .bdrv_make_empty        = qcow_make_empty,
+    .bdrv_write_compressed  = qcow_write_compressed,
+    .bdrv_get_info          = qcow_get_info,
+
+    .create_options = qcow_create_options,
+};
+
+static void bdrv_qcow_init(void)
+{
+    bdrv_register(&bdrv_qcow);
+}
+
+block_init(bdrv_qcow_init);
diff --git a/contrib/qemu/block/qcow2-cache.c b/contrib/qemu/block/qcow2-cache.c
new file mode 100644
index 000000000..2f3114ecc
--- /dev/null
+++ b/contrib/qemu/block/qcow2-cache.c
@@ -0,0 +1,323 @@
+/*
+ * L2/refcount table cache for the QCOW2 format
+ *
+ * Copyright (c) 2010 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "block/block_int.h"
+#include "qemu-common.h"
+#include "qcow2.h"
+#include "trace.h"
+
+typedef struct Qcow2CachedTable {
+    void*   table;
+    int64_t offset;
+    bool    dirty;
+    int     cache_hits;
+    int     ref;
+} Qcow2CachedTable;
+
+struct Qcow2Cache {
+    Qcow2CachedTable*       entries;
+    struct Qcow2Cache*      depends;
+    int                     size;
+    bool                    depends_on_flush;
+};
+
+Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables)
+{
+    BDRVQcowState *s = bs->opaque;
+    Qcow2Cache *c;
+    int i;
+
+    c = g_malloc0(sizeof(*c));
+    c->size = num_tables;
+    c->entries = g_malloc0(sizeof(*c->entries) * num_tables);
+
+    for (i = 0; i < c->size; i++) {
+        c->entries[i].table = qemu_blockalign(bs, s->cluster_size);
+    }
+
+    return c;
+}
+
+int qcow2_cache_destroy(BlockDriverState* bs, Qcow2Cache *c)
+{
+    int i;
+
+    for (i = 0; i < c->size; i++) {
+        assert(c->entries[i].ref == 0);
+        qemu_vfree(c->entries[i].table);
+    }
+
+    g_free(c->entries);
+    g_free(c);
+
+    return 0;
+}
+
+static int qcow2_cache_flush_dependency(BlockDriverState *bs, Qcow2Cache *c)
+{
+    int ret;
+
+    ret = qcow2_cache_flush(bs, c->depends);
+    if (ret < 0) {
+        return ret;
+    }
+
+    c->depends = NULL;
+    c->depends_on_flush = false;
+
+    return 0;
+}
+
+static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret = 0;
+
+    if (!c->entries[i].dirty || !c->entries[i].offset) {
+        return 0;
+    }
+
+    trace_qcow2_cache_entry_flush(qemu_coroutine_self(),
+                                  c == s->l2_table_cache, i);
+
+    if (c->depends) {
+        ret = qcow2_cache_flush_dependency(bs, c);
+    } else if (c->depends_on_flush) {
+        ret = bdrv_flush(bs->file);
+        if (ret >= 0) {
+            c->depends_on_flush = false;
+        }
+    }
+
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (c == s->refcount_block_cache) {
+        BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_UPDATE_PART);
+    } else if (c == s->l2_table_cache) {
+        BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE);
+    }
+
+    ret = bdrv_pwrite(bs->file, c->entries[i].offset, c->entries[i].table,
+        s->cluster_size);
+    if (ret < 0) {
+        return ret;
+    }
+
+    c->entries[i].dirty = false;
+
+    return 0;
+}
+
+int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c)
+{
+    BDRVQcowState *s = bs->opaque;
+    int result = 0;
+    int ret;
+    int i;
+
+    trace_qcow2_cache_flush(qemu_coroutine_self(), c == s->l2_table_cache);
+
+    for (i = 0; i < c->size; i++) {
+        ret = qcow2_cache_entry_flush(bs, c, i);
+        if (ret < 0 && result != -ENOSPC) {
+            result = ret;
+        }
+    }
+
+    if (result == 0) {
+        ret = bdrv_flush(bs->file);
+        if (ret < 0) {
+            result = ret;
+        }
+    }
+
+    return result;
+}
+
+int qcow2_cache_set_dependency(BlockDriverState *bs, Qcow2Cache *c,
+    Qcow2Cache *dependency)
+{
+    int ret;
+
+    if (dependency->depends) {
+        ret = qcow2_cache_flush_dependency(bs, dependency);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    if (c->depends && (c->depends != dependency)) {
+        ret = qcow2_cache_flush_dependency(bs, c);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    c->depends = dependency;
+    return 0;
+}
+
+void qcow2_cache_depends_on_flush(Qcow2Cache *c)
+{
+    c->depends_on_flush = true;
+}
+
+static int qcow2_cache_find_entry_to_replace(Qcow2Cache *c)
+{
+    int i;
+    int min_count = INT_MAX;
+    int min_index = -1;
+
+
+    for (i = 0; i < c->size; i++) {
+        if (c->entries[i].ref) {
+            continue;
+        }
+
+        if (c->entries[i].cache_hits < min_count) {
+            min_index = i;
+            min_count = c->entries[i].cache_hits;
+        }
+
+        /* Give newer hits priority */
+        /* TODO Check how to optimize the replacement strategy */
+        c->entries[i].cache_hits /= 2;
+    }
+
+    if (min_index == -1) {
+        /* This can't happen in current synchronous code, but leave the check
+         * here as a reminder for whoever starts using AIO with the cache */
+        abort();
+    }
+    return min_index;
+}
+
+static int qcow2_cache_do_get(BlockDriverState *bs, Qcow2Cache *c,
+    uint64_t offset, void **table, bool read_from_disk)
+{
+    BDRVQcowState *s = bs->opaque;
+    int i;
+    int ret;
+
+    trace_qcow2_cache_get(qemu_coroutine_self(), c == s->l2_table_cache,
+                          offset, read_from_disk);
+
+    /* Check if the table is already cached */
+    for (i = 0; i < c->size; i++) {
+        if (c->entries[i].offset == offset) {
+            goto found;
+        }
+    }
+
+    /* If not, write a table back and replace it */
+    i = qcow2_cache_find_entry_to_replace(c);
+    trace_qcow2_cache_get_replace_entry(qemu_coroutine_self(),
+                                        c == s->l2_table_cache, i);
+    if (i < 0) {
+        return i;
+    }
+
+    ret = qcow2_cache_entry_flush(bs, c, i);
+    if (ret < 0) {
+        return ret;
+    }
+
+    trace_qcow2_cache_get_read(qemu_coroutine_self(),
+                               c == s->l2_table_cache, i);
+    c->entries[i].offset = 0;
+    if (read_from_disk) {
+        if (c == s->l2_table_cache) {
+            BLKDBG_EVENT(bs->file, BLKDBG_L2_LOAD);
+        }
+
+        ret = bdrv_pread(bs->file, offset, c->entries[i].table, s->cluster_size);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    /* Give the table some hits for the start so that it won't be replaced
+     * immediately. The number 32 is completely arbitrary. */
+    c->entries[i].cache_hits = 32;
+    c->entries[i].offset = offset;
+
+    /* And return the right table */
+found:
+    c->entries[i].cache_hits++;
+    c->entries[i].ref++;
+    *table = c->entries[i].table;
+
+    trace_qcow2_cache_get_done(qemu_coroutine_self(),
+                               c == s->l2_table_cache, i);
+
+    return 0;
+}
+
+int qcow2_cache_get(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
+    void **table)
+{
+    return qcow2_cache_do_get(bs, c, offset, table, true);
+}
+
+int qcow2_cache_get_empty(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
+    void **table)
+{
+    return qcow2_cache_do_get(bs, c, offset, table, false);
+}
+
+int qcow2_cache_put(BlockDriverState *bs, Qcow2Cache *c, void **table)
+{
+    int i;
+
+    for (i = 0; i < c->size; i++) {
+        if (c->entries[i].table == *table) {
+            goto found;
+        }
+    }
+    return -ENOENT;
+
+found:
+    c->entries[i].ref--;
+    *table = NULL;
+
+    assert(c->entries[i].ref >= 0);
+    return 0;
+}
+
+void qcow2_cache_entry_mark_dirty(Qcow2Cache *c, void *table)
+{
+    int i;
+
+    for (i = 0; i < c->size; i++) {
+        if (c->entries[i].table == table) {
+            goto found;
+        }
+    }
+    abort();
+
+found:
+    c->entries[i].dirty = true;
+}
diff --git a/contrib/qemu/block/qcow2-cluster.c b/contrib/qemu/block/qcow2-cluster.c
new file mode 100644
index 000000000..cca76d4fc
--- /dev/null
+++ b/contrib/qemu/block/qcow2-cluster.c
@@ -0,0 +1,1478 @@
+/*
+ * Block driver for the QCOW version 2 format
+ *
+ * Copyright (c) 2004-2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <zlib.h>
+
+#include "qemu-common.h"
+#include "block/block_int.h"
+#include "block/qcow2.h"
+#include "trace.h"
+
+int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
+                        bool exact_size)
+{
+    BDRVQcowState *s = bs->opaque;
+    int new_l1_size2, ret, i;
+    uint64_t *new_l1_table;
+    int64_t new_l1_table_offset, new_l1_size;
+    uint8_t data[12];
+
+    if (min_size <= s->l1_size)
+        return 0;
+
+    if (exact_size) {
+        new_l1_size = min_size;
+    } else {
+        /* Bump size up to reduce the number of times we have to grow */
+        new_l1_size = s->l1_size;
+        if (new_l1_size == 0) {
+            new_l1_size = 1;
+        }
+        while (min_size > new_l1_size) {
+            new_l1_size = (new_l1_size * 3 + 1) / 2;
+        }
+    }
+
+    if (new_l1_size > INT_MAX) {
+        return -EFBIG;
+    }
+
+#ifdef DEBUG_ALLOC2
+    fprintf(stderr, "grow l1_table from %d to %" PRId64 "\n",
+            s->l1_size, new_l1_size);
+#endif
+
+    new_l1_size2 = sizeof(uint64_t) * new_l1_size;
+    new_l1_table = g_malloc0(align_offset(new_l1_size2, 512));
+    memcpy(new_l1_table, s->l1_table, s->l1_size * sizeof(uint64_t));
+
+    /* write new table (align to cluster) */
+    BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ALLOC_TABLE);
+    new_l1_table_offset = qcow2_alloc_clusters(bs, new_l1_size2);
+    if (new_l1_table_offset < 0) {
+        g_free(new_l1_table);
+        return new_l1_table_offset;
+    }
+
+    ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_WRITE_TABLE);
+    for(i = 0; i < s->l1_size; i++)
+        new_l1_table[i] = cpu_to_be64(new_l1_table[i]);
+    ret = bdrv_pwrite_sync(bs->file, new_l1_table_offset, new_l1_table, new_l1_size2);
+    if (ret < 0)
+        goto fail;
+    for(i = 0; i < s->l1_size; i++)
+        new_l1_table[i] = be64_to_cpu(new_l1_table[i]);
+
+    /* set new table */
+    BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ACTIVATE_TABLE);
+    cpu_to_be32w((uint32_t*)data, new_l1_size);
+    cpu_to_be64wu((uint64_t*)(data + 4), new_l1_table_offset);
+    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_size), data,sizeof(data));
+    if (ret < 0) {
+        goto fail;
+    }
+    g_free(s->l1_table);
+    qcow2_free_clusters(bs, s->l1_table_offset, s->l1_size * sizeof(uint64_t),
+                        QCOW2_DISCARD_OTHER);
+    s->l1_table_offset = new_l1_table_offset;
+    s->l1_table = new_l1_table;
+    s->l1_size = new_l1_size;
+    return 0;
+ fail:
+    g_free(new_l1_table);
+    qcow2_free_clusters(bs, new_l1_table_offset, new_l1_size2,
+                        QCOW2_DISCARD_OTHER);
+    return ret;
+}
+
+/*
+ * l2_load
+ *
+ * Loads a L2 table into memory. If the table is in the cache, the cache
+ * is used; otherwise the L2 table is loaded from the image file.
+ *
+ * Returns a pointer to the L2 table on success, or NULL if the read from
+ * the image file failed.
+ */
+
+static int l2_load(BlockDriverState *bs, uint64_t l2_offset,
+    uint64_t **l2_table)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret;
+
+    ret = qcow2_cache_get(bs, s->l2_table_cache, l2_offset, (void**) l2_table);
+
+    return ret;
+}
+
+/*
+ * Writes one sector of the L1 table to the disk (can't update single entries
+ * and we really don't want bdrv_pread to perform a read-modify-write)
+ */
+#define L1_ENTRIES_PER_SECTOR (512 / 8)
+static int write_l1_entry(BlockDriverState *bs, int l1_index)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t buf[L1_ENTRIES_PER_SECTOR];
+    int l1_start_index;
+    int i, ret;
+
+    l1_start_index = l1_index & ~(L1_ENTRIES_PER_SECTOR - 1);
+    for (i = 0; i < L1_ENTRIES_PER_SECTOR; i++) {
+        buf[i] = cpu_to_be64(s->l1_table[l1_start_index + i]);
+    }
+
+    BLKDBG_EVENT(bs->file, BLKDBG_L1_UPDATE);
+    ret = bdrv_pwrite_sync(bs->file, s->l1_table_offset + 8 * l1_start_index,
+        buf, sizeof(buf));
+    if (ret < 0) {
+        return ret;
+    }
+
+    return 0;
+}
+
+/*
+ * l2_allocate
+ *
+ * Allocate a new l2 entry in the file. If l1_index points to an already
+ * used entry in the L2 table (i.e. we are doing a copy on write for the L2
+ * table) copy the contents of the old L2 table into the newly allocated one.
+ * Otherwise the new table is initialized with zeros.
+ *
+ */
+
+static int l2_allocate(BlockDriverState *bs, int l1_index, uint64_t **table)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t old_l2_offset;
+    uint64_t *l2_table;
+    int64_t l2_offset;
+    int ret;
+
+    old_l2_offset = s->l1_table[l1_index];
+
+    trace_qcow2_l2_allocate(bs, l1_index);
+
+    /* allocate a new l2 entry */
+
+    l2_offset = qcow2_alloc_clusters(bs, s->l2_size * sizeof(uint64_t));
+    if (l2_offset < 0) {
+        return l2_offset;
+    }
+
+    ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* allocate a new entry in the l2 cache */
+
+    trace_qcow2_l2_allocate_get_empty(bs, l1_index);
+    ret = qcow2_cache_get_empty(bs, s->l2_table_cache, l2_offset, (void**) table);
+    if (ret < 0) {
+        return ret;
+    }
+
+    l2_table = *table;
+
+    if ((old_l2_offset & L1E_OFFSET_MASK) == 0) {
+        /* if there was no old l2 table, clear the new table */
+        memset(l2_table, 0, s->l2_size * sizeof(uint64_t));
+    } else {
+        uint64_t* old_table;
+
+        /* if there was an old l2 table, read it from the disk */
+        BLKDBG_EVENT(bs->file, BLKDBG_L2_ALLOC_COW_READ);
+        ret = qcow2_cache_get(bs, s->l2_table_cache,
+            old_l2_offset & L1E_OFFSET_MASK,
+            (void**) &old_table);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        memcpy(l2_table, old_table, s->cluster_size);
+
+        ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &old_table);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+    /* write the l2 table to the file */
+    BLKDBG_EVENT(bs->file, BLKDBG_L2_ALLOC_WRITE);
+
+    trace_qcow2_l2_allocate_write_l2(bs, l1_index);
+    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+    ret = qcow2_cache_flush(bs, s->l2_table_cache);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* update the L1 entry */
+    trace_qcow2_l2_allocate_write_l1(bs, l1_index);
+    s->l1_table[l1_index] = l2_offset | QCOW_OFLAG_COPIED;
+    ret = write_l1_entry(bs, l1_index);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    *table = l2_table;
+    trace_qcow2_l2_allocate_done(bs, l1_index, 0);
+    return 0;
+
+fail:
+    trace_qcow2_l2_allocate_done(bs, l1_index, ret);
+    qcow2_cache_put(bs, s->l2_table_cache, (void**) table);
+    s->l1_table[l1_index] = old_l2_offset;
+    return ret;
+}
+
+/*
+ * Checks how many clusters in a given L2 table are contiguous in the image
+ * file. As soon as one of the flags in the bitmask stop_flags changes compared
+ * to the first cluster, the search is stopped and the cluster is not counted
+ * as contiguous. (This allows it, for example, to stop at the first compressed
+ * cluster which may require a different handling)
+ */
+static int count_contiguous_clusters(uint64_t nb_clusters, int cluster_size,
+        uint64_t *l2_table, uint64_t start, uint64_t stop_flags)
+{
+    int i;
+    uint64_t mask = stop_flags | L2E_OFFSET_MASK;
+    uint64_t offset = be64_to_cpu(l2_table[0]) & mask;
+
+    if (!offset)
+        return 0;
+
+    for (i = start; i < start + nb_clusters; i++) {
+        uint64_t l2_entry = be64_to_cpu(l2_table[i]) & mask;
+        if (offset + (uint64_t) i * cluster_size != l2_entry) {
+            break;
+        }
+    }
+
+	return (i - start);
+}
+
+static int count_contiguous_free_clusters(uint64_t nb_clusters, uint64_t *l2_table)
+{
+    int i;
+
+    for (i = 0; i < nb_clusters; i++) {
+        int type = qcow2_get_cluster_type(be64_to_cpu(l2_table[i]));
+
+        if (type != QCOW2_CLUSTER_UNALLOCATED) {
+            break;
+        }
+    }
+
+    return i;
+}
+
+/* The crypt function is compatible with the linux cryptoloop
+   algorithm for < 4 GB images. NOTE: out_buf == in_buf is
+   supported */
+void qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
+                           uint8_t *out_buf, const uint8_t *in_buf,
+                           int nb_sectors, int enc,
+                           const AES_KEY *key)
+{
+    union {
+        uint64_t ll[2];
+        uint8_t b[16];
+    } ivec;
+    int i;
+
+    for(i = 0; i < nb_sectors; i++) {
+        ivec.ll[0] = cpu_to_le64(sector_num);
+        ivec.ll[1] = 0;
+        AES_cbc_encrypt(in_buf, out_buf, 512, key,
+                        ivec.b, enc);
+        sector_num++;
+        in_buf += 512;
+        out_buf += 512;
+    }
+}
+
+static int coroutine_fn copy_sectors(BlockDriverState *bs,
+                                     uint64_t start_sect,
+                                     uint64_t cluster_offset,
+                                     int n_start, int n_end)
+{
+    BDRVQcowState *s = bs->opaque;
+    QEMUIOVector qiov;
+    struct iovec iov;
+    int n, ret;
+
+    /*
+     * If this is the last cluster and it is only partially used, we must only
+     * copy until the end of the image, or bdrv_check_request will fail for the
+     * bdrv_read/write calls below.
+     */
+    if (start_sect + n_end > bs->total_sectors) {
+        n_end = bs->total_sectors - start_sect;
+    }
+
+    n = n_end - n_start;
+    if (n <= 0) {
+        return 0;
+    }
+
+    iov.iov_len = n * BDRV_SECTOR_SIZE;
+    iov.iov_base = qemu_blockalign(bs, iov.iov_len);
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+
+    BLKDBG_EVENT(bs->file, BLKDBG_COW_READ);
+
+    /* Call .bdrv_co_readv() directly instead of using the public block-layer
+     * interface.  This avoids double I/O throttling and request tracking,
+     * which can lead to deadlock when block layer copy-on-read is enabled.
+     */
+    ret = bs->drv->bdrv_co_readv(bs, start_sect + n_start, n, &qiov);
+    if (ret < 0) {
+        goto out;
+    }
+
+    if (s->crypt_method) {
+        qcow2_encrypt_sectors(s, start_sect + n_start,
+                        iov.iov_base, iov.iov_base, n, 1,
+                        &s->aes_encrypt_key);
+    }
+
+    BLKDBG_EVENT(bs->file, BLKDBG_COW_WRITE);
+    ret = bdrv_co_writev(bs->file, (cluster_offset >> 9) + n_start, n, &qiov);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = 0;
+out:
+    qemu_vfree(iov.iov_base);
+    return ret;
+}
+
+
+/*
+ * get_cluster_offset
+ *
+ * For a given offset of the disk image, find the cluster offset in
+ * qcow2 file. The offset is stored in *cluster_offset.
+ *
+ * on entry, *num is the number of contiguous sectors we'd like to
+ * access following offset.
+ *
+ * on exit, *num is the number of contiguous sectors we can read.
+ *
+ * Returns the cluster type (QCOW2_CLUSTER_*) on success, -errno in error
+ * cases.
+ */
+int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
+    int *num, uint64_t *cluster_offset)
+{
+    BDRVQcowState *s = bs->opaque;
+    unsigned int l2_index;
+    uint64_t l1_index, l2_offset, *l2_table;
+    int l1_bits, c;
+    unsigned int index_in_cluster, nb_clusters;
+    uint64_t nb_available, nb_needed;
+    int ret;
+
+    index_in_cluster = (offset >> 9) & (s->cluster_sectors - 1);
+    nb_needed = *num + index_in_cluster;
+
+    l1_bits = s->l2_bits + s->cluster_bits;
+
+    /* compute how many bytes there are between the offset and
+     * the end of the l1 entry
+     */
+
+    nb_available = (1ULL << l1_bits) - (offset & ((1ULL << l1_bits) - 1));
+
+    /* compute the number of available sectors */
+
+    nb_available = (nb_available >> 9) + index_in_cluster;
+
+    if (nb_needed > nb_available) {
+        nb_needed = nb_available;
+    }
+
+    *cluster_offset = 0;
+
+    /* seek the the l2 offset in the l1 table */
+
+    l1_index = offset >> l1_bits;
+    if (l1_index >= s->l1_size) {
+        ret = QCOW2_CLUSTER_UNALLOCATED;
+        goto out;
+    }
+
+    l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
+    if (!l2_offset) {
+        ret = QCOW2_CLUSTER_UNALLOCATED;
+        goto out;
+    }
+
+    /* load the l2 table in memory */
+
+    ret = l2_load(bs, l2_offset, &l2_table);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* find the cluster offset for the given disk offset */
+
+    l2_index = (offset >> s->cluster_bits) & (s->l2_size - 1);
+    *cluster_offset = be64_to_cpu(l2_table[l2_index]);
+    nb_clusters = size_to_clusters(s, nb_needed << 9);
+
+    ret = qcow2_get_cluster_type(*cluster_offset);
+    switch (ret) {
+    case QCOW2_CLUSTER_COMPRESSED:
+        /* Compressed clusters can only be processed one by one */
+        c = 1;
+        *cluster_offset &= L2E_COMPRESSED_OFFSET_SIZE_MASK;
+        break;
+    case QCOW2_CLUSTER_ZERO:
+        if (s->qcow_version < 3) {
+            return -EIO;
+        }
+        c = count_contiguous_clusters(nb_clusters, s->cluster_size,
+                &l2_table[l2_index], 0,
+                QCOW_OFLAG_COMPRESSED | QCOW_OFLAG_ZERO);
+        *cluster_offset = 0;
+        break;
+    case QCOW2_CLUSTER_UNALLOCATED:
+        /* how many empty clusters ? */
+        c = count_contiguous_free_clusters(nb_clusters, &l2_table[l2_index]);
+        *cluster_offset = 0;
+        break;
+    case QCOW2_CLUSTER_NORMAL:
+        /* how many allocated clusters ? */
+        c = count_contiguous_clusters(nb_clusters, s->cluster_size,
+                &l2_table[l2_index], 0,
+                QCOW_OFLAG_COMPRESSED | QCOW_OFLAG_ZERO);
+        *cluster_offset &= L2E_OFFSET_MASK;
+        break;
+    default:
+        abort();
+    }
+
+    qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+
+    nb_available = (c * s->cluster_sectors);
+
+out:
+    if (nb_available > nb_needed)
+        nb_available = nb_needed;
+
+    *num = nb_available - index_in_cluster;
+
+    return ret;
+}
+
+/*
+ * get_cluster_table
+ *
+ * for a given disk offset, load (and allocate if needed)
+ * the l2 table.
+ *
+ * the l2 table offset in the qcow2 file and the cluster index
+ * in the l2 table are given to the caller.
+ *
+ * Returns 0 on success, -errno in failure case
+ */
+static int get_cluster_table(BlockDriverState *bs, uint64_t offset,
+                             uint64_t **new_l2_table,
+                             int *new_l2_index)
+{
+    BDRVQcowState *s = bs->opaque;
+    unsigned int l2_index;
+    uint64_t l1_index, l2_offset;
+    uint64_t *l2_table = NULL;
+    int ret;
+
+    /* seek the the l2 offset in the l1 table */
+
+    l1_index = offset >> (s->l2_bits + s->cluster_bits);
+    if (l1_index >= s->l1_size) {
+        ret = qcow2_grow_l1_table(bs, l1_index + 1, false);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    assert(l1_index < s->l1_size);
+    l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
+
+    /* seek the l2 table of the given l2 offset */
+
+    if (s->l1_table[l1_index] & QCOW_OFLAG_COPIED) {
+        /* load the l2 table in memory */
+        ret = l2_load(bs, l2_offset, &l2_table);
+        if (ret < 0) {
+            return ret;
+        }
+    } else {
+        /* First allocate a new L2 table (and do COW if needed) */
+        ret = l2_allocate(bs, l1_index, &l2_table);
+        if (ret < 0) {
+            return ret;
+        }
+
+        /* Then decrease the refcount of the old table */
+        if (l2_offset) {
+            qcow2_free_clusters(bs, l2_offset, s->l2_size * sizeof(uint64_t),
+                                QCOW2_DISCARD_OTHER);
+        }
+    }
+
+    /* find the cluster offset for the given disk offset */
+
+    l2_index = (offset >> s->cluster_bits) & (s->l2_size - 1);
+
+    *new_l2_table = l2_table;
+    *new_l2_index = l2_index;
+
+    return 0;
+}
+
+/*
+ * alloc_compressed_cluster_offset
+ *
+ * For a given offset of the disk image, return cluster offset in
+ * qcow2 file.
+ *
+ * If the offset is not found, allocate a new compressed cluster.
+ *
+ * Return the cluster offset if successful,
+ * Return 0, otherwise.
+ *
+ */
+
+uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
+                                               uint64_t offset,
+                                               int compressed_size)
+{
+    BDRVQcowState *s = bs->opaque;
+    int l2_index, ret;
+    uint64_t *l2_table;
+    int64_t cluster_offset;
+    int nb_csectors;
+
+    ret = get_cluster_table(bs, offset, &l2_table, &l2_index);
+    if (ret < 0) {
+        return 0;
+    }
+
+    /* Compression can't overwrite anything. Fail if the cluster was already
+     * allocated. */
+    cluster_offset = be64_to_cpu(l2_table[l2_index]);
+    if (cluster_offset & L2E_OFFSET_MASK) {
+        qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+        return 0;
+    }
+
+    cluster_offset = qcow2_alloc_bytes(bs, compressed_size);
+    if (cluster_offset < 0) {
+        qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+        return 0;
+    }
+
+    nb_csectors = ((cluster_offset + compressed_size - 1) >> 9) -
+                  (cluster_offset >> 9);
+
+    cluster_offset |= QCOW_OFLAG_COMPRESSED |
+                      ((uint64_t)nb_csectors << s->csize_shift);
+
+    /* update L2 table */
+
+    /* compressed clusters never have the copied flag */
+
+    BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE_COMPRESSED);
+    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+    l2_table[l2_index] = cpu_to_be64(cluster_offset);
+    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+    if (ret < 0) {
+        return 0;
+    }
+
+    return cluster_offset;
+}
+
+static int perform_cow(BlockDriverState *bs, QCowL2Meta *m, Qcow2COWRegion *r)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret;
+
+    if (r->nb_sectors == 0) {
+        return 0;
+    }
+
+    qemu_co_mutex_unlock(&s->lock);
+    ret = copy_sectors(bs, m->offset / BDRV_SECTOR_SIZE, m->alloc_offset,
+                       r->offset / BDRV_SECTOR_SIZE,
+                       r->offset / BDRV_SECTOR_SIZE + r->nb_sectors);
+    qemu_co_mutex_lock(&s->lock);
+
+    if (ret < 0) {
+        return ret;
+    }
+
+    /*
+     * Before we update the L2 table to actually point to the new cluster, we
+     * need to be sure that the refcounts have been increased and COW was
+     * handled.
+     */
+    qcow2_cache_depends_on_flush(s->l2_table_cache);
+
+    return 0;
+}
+
+int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
+{
+    BDRVQcowState *s = bs->opaque;
+    int i, j = 0, l2_index, ret;
+    uint64_t *old_cluster, *l2_table;
+    uint64_t cluster_offset = m->alloc_offset;
+
+    trace_qcow2_cluster_link_l2(qemu_coroutine_self(), m->nb_clusters);
+    assert(m->nb_clusters > 0);
+
+    old_cluster = g_malloc(m->nb_clusters * sizeof(uint64_t));
+
+    /* copy content of unmodified sectors */
+    ret = perform_cow(bs, m, &m->cow_start);
+    if (ret < 0) {
+        goto err;
+    }
+
+    ret = perform_cow(bs, m, &m->cow_end);
+    if (ret < 0) {
+        goto err;
+    }
+
+    /* Update L2 table. */
+    if (s->use_lazy_refcounts) {
+        qcow2_mark_dirty(bs);
+    }
+    if (qcow2_need_accurate_refcounts(s)) {
+        qcow2_cache_set_dependency(bs, s->l2_table_cache,
+                                   s->refcount_block_cache);
+    }
+
+    ret = get_cluster_table(bs, m->offset, &l2_table, &l2_index);
+    if (ret < 0) {
+        goto err;
+    }
+    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+
+    for (i = 0; i < m->nb_clusters; i++) {
+        /* if two concurrent writes happen to the same unallocated cluster
+	 * each write allocates separate cluster and writes data concurrently.
+	 * The first one to complete updates l2 table with pointer to its
+	 * cluster the second one has to do RMW (which is done above by
+	 * copy_sectors()), update l2 table with its cluster pointer and free
+	 * old cluster. This is what this loop does */
+        if(l2_table[l2_index + i] != 0)
+            old_cluster[j++] = l2_table[l2_index + i];
+
+        l2_table[l2_index + i] = cpu_to_be64((cluster_offset +
+                    (i << s->cluster_bits)) | QCOW_OFLAG_COPIED);
+     }
+
+
+    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+    if (ret < 0) {
+        goto err;
+    }
+
+    /*
+     * If this was a COW, we need to decrease the refcount of the old cluster.
+     * Also flush bs->file to get the right order for L2 and refcount update.
+     *
+     * Don't discard clusters that reach a refcount of 0 (e.g. compressed
+     * clusters), the next write will reuse them anyway.
+     */
+    if (j != 0) {
+        for (i = 0; i < j; i++) {
+            qcow2_free_any_clusters(bs, be64_to_cpu(old_cluster[i]), 1,
+                                    QCOW2_DISCARD_NEVER);
+        }
+    }
+
+    ret = 0;
+err:
+    g_free(old_cluster);
+    return ret;
+ }
+
+/*
+ * Returns the number of contiguous clusters that can be used for an allocating
+ * write, but require COW to be performed (this includes yet unallocated space,
+ * which must copy from the backing file)
+ */
+static int count_cow_clusters(BDRVQcowState *s, int nb_clusters,
+    uint64_t *l2_table, int l2_index)
+{
+    int i;
+
+    for (i = 0; i < nb_clusters; i++) {
+        uint64_t l2_entry = be64_to_cpu(l2_table[l2_index + i]);
+        int cluster_type = qcow2_get_cluster_type(l2_entry);
+
+        switch(cluster_type) {
+        case QCOW2_CLUSTER_NORMAL:
+            if (l2_entry & QCOW_OFLAG_COPIED) {
+                goto out;
+            }
+            break;
+        case QCOW2_CLUSTER_UNALLOCATED:
+        case QCOW2_CLUSTER_COMPRESSED:
+        case QCOW2_CLUSTER_ZERO:
+            break;
+        default:
+            abort();
+        }
+    }
+
+out:
+    assert(i <= nb_clusters);
+    return i;
+}
+
+/*
+ * Check if there already is an AIO write request in flight which allocates
+ * the same cluster. In this case we need to wait until the previous
+ * request has completed and updated the L2 table accordingly.
+ *
+ * Returns:
+ *   0       if there was no dependency. *cur_bytes indicates the number of
+ *           bytes from guest_offset that can be read before the next
+ *           dependency must be processed (or the request is complete)
+ *
+ *   -EAGAIN if we had to wait for another request, previously gathered
+ *           information on cluster allocation may be invalid now. The caller
+ *           must start over anyway, so consider *cur_bytes undefined.
+ */
+static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
+    uint64_t *cur_bytes, QCowL2Meta **m)
+{
+    BDRVQcowState *s = bs->opaque;
+    QCowL2Meta *old_alloc;
+    uint64_t bytes = *cur_bytes;
+
+    QLIST_FOREACH(old_alloc, &s->cluster_allocs, next_in_flight) {
+
+        uint64_t start = guest_offset;
+        uint64_t end = start + bytes;
+        uint64_t old_start = l2meta_cow_start(old_alloc);
+        uint64_t old_end = l2meta_cow_end(old_alloc);
+
+        if (end <= old_start || start >= old_end) {
+            /* No intersection */
+        } else {
+            if (start < old_start) {
+                /* Stop at the start of a running allocation */
+                bytes = old_start - start;
+            } else {
+                bytes = 0;
+            }
+
+            /* Stop if already an l2meta exists. After yielding, it wouldn't
+             * be valid any more, so we'd have to clean up the old L2Metas
+             * and deal with requests depending on them before starting to
+             * gather new ones. Not worth the trouble. */
+            if (bytes == 0 && *m) {
+                *cur_bytes = 0;
+                return 0;
+            }
+
+            if (bytes == 0) {
+                /* Wait for the dependency to complete. We need to recheck
+                 * the free/allocated clusters when we continue. */
+                qemu_co_mutex_unlock(&s->lock);
+                qemu_co_queue_wait(&old_alloc->dependent_requests);
+                qemu_co_mutex_lock(&s->lock);
+                return -EAGAIN;
+            }
+        }
+    }
+
+    /* Make sure that existing clusters and new allocations are only used up to
+     * the next dependency if we shortened the request above */
+    *cur_bytes = bytes;
+
+    return 0;
+}
+
+/*
+ * Checks how many already allocated clusters that don't require a copy on
+ * write there are at the given guest_offset (up to *bytes). If
+ * *host_offset is not zero, only physically contiguous clusters beginning at
+ * this host offset are counted.
+ *
+ * Note that guest_offset may not be cluster aligned. In this case, the
+ * returned *host_offset points to exact byte referenced by guest_offset and
+ * therefore isn't cluster aligned as well.
+ *
+ * Returns:
+ *   0:     if no allocated clusters are available at the given offset.
+ *          *bytes is normally unchanged. It is set to 0 if the cluster
+ *          is allocated and doesn't need COW, but doesn't have the right
+ *          physical offset.
+ *
+ *   1:     if allocated clusters that don't require a COW are available at
+ *          the requested offset. *bytes may have decreased and describes
+ *          the length of the area that can be written to.
+ *
+ *  -errno: in error cases
+ */
+static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,
+    uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
+{
+    BDRVQcowState *s = bs->opaque;
+    int l2_index;
+    uint64_t cluster_offset;
+    uint64_t *l2_table;
+    unsigned int nb_clusters;
+    unsigned int keep_clusters;
+    int ret, pret;
+
+    trace_qcow2_handle_copied(qemu_coroutine_self(), guest_offset, *host_offset,
+                              *bytes);
+
+    assert(*host_offset == 0 ||    offset_into_cluster(s, guest_offset)
+                                == offset_into_cluster(s, *host_offset));
+
+    /*
+     * Calculate the number of clusters to look for. We stop at L2 table
+     * boundaries to keep things simple.
+     */
+    nb_clusters =
+        size_to_clusters(s, offset_into_cluster(s, guest_offset) + *bytes);
+
+    l2_index = offset_to_l2_index(s, guest_offset);
+    nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+
+    /* Find L2 entry for the first involved cluster */
+    ret = get_cluster_table(bs, guest_offset, &l2_table, &l2_index);
+    if (ret < 0) {
+        return ret;
+    }
+
+    cluster_offset = be64_to_cpu(l2_table[l2_index]);
+
+    /* Check how many clusters are already allocated and don't need COW */
+    if (qcow2_get_cluster_type(cluster_offset) == QCOW2_CLUSTER_NORMAL
+        && (cluster_offset & QCOW_OFLAG_COPIED))
+    {
+        /* If a specific host_offset is required, check it */
+        bool offset_matches =
+            (cluster_offset & L2E_OFFSET_MASK) == *host_offset;
+
+        if (*host_offset != 0 && !offset_matches) {
+            *bytes = 0;
+            ret = 0;
+            goto out;
+        }
+
+        /* We keep all QCOW_OFLAG_COPIED clusters */
+        keep_clusters =
+            count_contiguous_clusters(nb_clusters, s->cluster_size,
+                                      &l2_table[l2_index], 0,
+                                      QCOW_OFLAG_COPIED | QCOW_OFLAG_ZERO);
+        assert(keep_clusters <= nb_clusters);
+
+        *bytes = MIN(*bytes,
+                 keep_clusters * s->cluster_size
+                 - offset_into_cluster(s, guest_offset));
+
+        ret = 1;
+    } else {
+        ret = 0;
+    }
+
+    /* Cleanup */
+out:
+    pret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+    if (pret < 0) {
+        return pret;
+    }
+
+    /* Only return a host offset if we actually made progress. Otherwise we
+     * would make requirements for handle_alloc() that it can't fulfill */
+    if (ret) {
+        *host_offset = (cluster_offset & L2E_OFFSET_MASK)
+                     + offset_into_cluster(s, guest_offset);
+    }
+
+    return ret;
+}
+
+/*
+ * Allocates new clusters for the given guest_offset.
+ *
+ * At most *nb_clusters are allocated, and on return *nb_clusters is updated to
+ * contain the number of clusters that have been allocated and are contiguous
+ * in the image file.
+ *
+ * If *host_offset is non-zero, it specifies the offset in the image file at
+ * which the new clusters must start. *nb_clusters can be 0 on return in this
+ * case if the cluster at host_offset is already in use. If *host_offset is
+ * zero, the clusters can be allocated anywhere in the image file.
+ *
+ * *host_offset is updated to contain the offset into the image file at which
+ * the first allocated cluster starts.
+ *
+ * Return 0 on success and -errno in error cases. -EAGAIN means that the
+ * function has been waiting for another request and the allocation must be
+ * restarted, but the whole request should not be failed.
+ */
+static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
+    uint64_t *host_offset, unsigned int *nb_clusters)
+{
+    BDRVQcowState *s = bs->opaque;
+
+    trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
+                                         *host_offset, *nb_clusters);
+
+    /* Allocate new clusters */
+    trace_qcow2_cluster_alloc_phys(qemu_coroutine_self());
+    if (*host_offset == 0) {
+        int64_t cluster_offset =
+            qcow2_alloc_clusters(bs, *nb_clusters * s->cluster_size);
+        if (cluster_offset < 0) {
+            return cluster_offset;
+        }
+        *host_offset = cluster_offset;
+        return 0;
+    } else {
+        int ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
+        if (ret < 0) {
+            return ret;
+        }
+        *nb_clusters = ret;
+        return 0;
+    }
+}
+
+/*
+ * Allocates new clusters for an area that either is yet unallocated or needs a
+ * copy on write. If *host_offset is non-zero, clusters are only allocated if
+ * the new allocation can match the specified host offset.
+ *
+ * Note that guest_offset may not be cluster aligned. In this case, the
+ * returned *host_offset points to exact byte referenced by guest_offset and
+ * therefore isn't cluster aligned as well.
+ *
+ * Returns:
+ *   0:     if no clusters could be allocated. *bytes is set to 0,
+ *          *host_offset is left unchanged.
+ *
+ *   1:     if new clusters were allocated. *bytes may be decreased if the
+ *          new allocation doesn't cover all of the requested area.
+ *          *host_offset is updated to contain the host offset of the first
+ *          newly allocated cluster.
+ *
+ *  -errno: in error cases
+ */
+static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,
+    uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
+{
+    BDRVQcowState *s = bs->opaque;
+    int l2_index;
+    uint64_t *l2_table;
+    uint64_t entry;
+    unsigned int nb_clusters;
+    int ret;
+
+    uint64_t alloc_cluster_offset;
+
+    trace_qcow2_handle_alloc(qemu_coroutine_self(), guest_offset, *host_offset,
+                             *bytes);
+    assert(*bytes > 0);
+
+    /*
+     * Calculate the number of clusters to look for. We stop at L2 table
+     * boundaries to keep things simple.
+     */
+    nb_clusters =
+        size_to_clusters(s, offset_into_cluster(s, guest_offset) + *bytes);
+
+    l2_index = offset_to_l2_index(s, guest_offset);
+    nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+
+    /* Find L2 entry for the first involved cluster */
+    ret = get_cluster_table(bs, guest_offset, &l2_table, &l2_index);
+    if (ret < 0) {
+        return ret;
+    }
+
+    entry = be64_to_cpu(l2_table[l2_index]);
+
+    /* For the moment, overwrite compressed clusters one by one */
+    if (entry & QCOW_OFLAG_COMPRESSED) {
+        nb_clusters = 1;
+    } else {
+        nb_clusters = count_cow_clusters(s, nb_clusters, l2_table, l2_index);
+    }
+
+    /* This function is only called when there were no non-COW clusters, so if
+     * we can't find any unallocated or COW clusters either, something is
+     * wrong with our code. */
+    assert(nb_clusters > 0);
+
+    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* Allocate, if necessary at a given offset in the image file */
+    alloc_cluster_offset = start_of_cluster(s, *host_offset);
+    ret = do_alloc_cluster_offset(bs, guest_offset, &alloc_cluster_offset,
+                                  &nb_clusters);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Can't extend contiguous allocation */
+    if (nb_clusters == 0) {
+        *bytes = 0;
+        return 0;
+    }
+
+    /*
+     * Save info needed for meta data update.
+     *
+     * requested_sectors: Number of sectors from the start of the first
+     * newly allocated cluster to the end of the (possibly shortened
+     * before) write request.
+     *
+     * avail_sectors: Number of sectors from the start of the first
+     * newly allocated to the end of the last newly allocated cluster.
+     *
+     * nb_sectors: The number of sectors from the start of the first
+     * newly allocated cluster to the end of the area that the write
+     * request actually writes to (excluding COW at the end)
+     */
+    int requested_sectors =
+        (*bytes + offset_into_cluster(s, guest_offset))
+        >> BDRV_SECTOR_BITS;
+    int avail_sectors = nb_clusters
+                        << (s->cluster_bits - BDRV_SECTOR_BITS);
+    int alloc_n_start = offset_into_cluster(s, guest_offset)
+                        >> BDRV_SECTOR_BITS;
+    int nb_sectors = MIN(requested_sectors, avail_sectors);
+    QCowL2Meta *old_m = *m;
+
+    *m = g_malloc0(sizeof(**m));
+
+    **m = (QCowL2Meta) {
+        .next           = old_m,
+
+        .alloc_offset   = alloc_cluster_offset,
+        .offset         = start_of_cluster(s, guest_offset),
+        .nb_clusters    = nb_clusters,
+        .nb_available   = nb_sectors,
+
+        .cow_start = {
+            .offset     = 0,
+            .nb_sectors = alloc_n_start,
+        },
+        .cow_end = {
+            .offset     = nb_sectors * BDRV_SECTOR_SIZE,
+            .nb_sectors = avail_sectors - nb_sectors,
+        },
+    };
+    qemu_co_queue_init(&(*m)->dependent_requests);
+    QLIST_INSERT_HEAD(&s->cluster_allocs, *m, next_in_flight);
+
+    *host_offset = alloc_cluster_offset + offset_into_cluster(s, guest_offset);
+    *bytes = MIN(*bytes, (nb_sectors * BDRV_SECTOR_SIZE)
+                         - offset_into_cluster(s, guest_offset));
+    assert(*bytes != 0);
+
+    return 1;
+
+fail:
+    if (*m && (*m)->nb_clusters > 0) {
+        QLIST_REMOVE(*m, next_in_flight);
+    }
+    return ret;
+}
+
+/*
+ * alloc_cluster_offset
+ *
+ * For a given offset on the virtual disk, find the cluster offset in qcow2
+ * file. If the offset is not found, allocate a new cluster.
+ *
+ * If the cluster was already allocated, m->nb_clusters is set to 0 and
+ * other fields in m are meaningless.
+ *
+ * If the cluster is newly allocated, m->nb_clusters is set to the number of
+ * contiguous clusters that have been allocated. In this case, the other
+ * fields of m are valid and contain information about the first allocated
+ * cluster.
+ *
+ * If the request conflicts with another write request in flight, the coroutine
+ * is queued and will be reentered when the dependency has completed.
+ *
+ * Return 0 on success and -errno in error cases
+ */
+int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
+    int n_start, int n_end, int *num, uint64_t *host_offset, QCowL2Meta **m)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t start, remaining;
+    uint64_t cluster_offset;
+    uint64_t cur_bytes;
+    int ret;
+
+    trace_qcow2_alloc_clusters_offset(qemu_coroutine_self(), offset,
+                                      n_start, n_end);
+
+    assert(n_start * BDRV_SECTOR_SIZE == offset_into_cluster(s, offset));
+    offset = start_of_cluster(s, offset);
+
+again:
+    start = offset + (n_start << BDRV_SECTOR_BITS);
+    remaining = (n_end - n_start) << BDRV_SECTOR_BITS;
+    cluster_offset = 0;
+    *host_offset = 0;
+    cur_bytes = 0;
+    *m = NULL;
+
+    while (true) {
+
+        if (!*host_offset) {
+            *host_offset = start_of_cluster(s, cluster_offset);
+        }
+
+        assert(remaining >= cur_bytes);
+
+        start           += cur_bytes;
+        remaining       -= cur_bytes;
+        cluster_offset  += cur_bytes;
+
+        if (remaining == 0) {
+            break;
+        }
+
+        cur_bytes = remaining;
+
+        /*
+         * Now start gathering as many contiguous clusters as possible:
+         *
+         * 1. Check for overlaps with in-flight allocations
+         *
+         *      a) Overlap not in the first cluster -> shorten this request and
+         *         let the caller handle the rest in its next loop iteration.
+         *
+         *      b) Real overlaps of two requests. Yield and restart the search
+         *         for contiguous clusters (the situation could have changed
+         *         while we were sleeping)
+         *
+         *      c) TODO: Request starts in the same cluster as the in-flight
+         *         allocation ends. Shorten the COW of the in-fight allocation,
+         *         set cluster_offset to write to the same cluster and set up
+         *         the right synchronisation between the in-flight request and
+         *         the new one.
+         */
+        ret = handle_dependencies(bs, start, &cur_bytes, m);
+        if (ret == -EAGAIN) {
+            /* Currently handle_dependencies() doesn't yield if we already had
+             * an allocation. If it did, we would have to clean up the L2Meta
+             * structs before starting over. */
+            assert(*m == NULL);
+            goto again;
+        } else if (ret < 0) {
+            return ret;
+        } else if (cur_bytes == 0) {
+            break;
+        } else {
+            /* handle_dependencies() may have decreased cur_bytes (shortened
+             * the allocations below) so that the next dependency is processed
+             * correctly during the next loop iteration. */
+        }
+
+        /*
+         * 2. Count contiguous COPIED clusters.
+         */
+        ret = handle_copied(bs, start, &cluster_offset, &cur_bytes, m);
+        if (ret < 0) {
+            return ret;
+        } else if (ret) {
+            continue;
+        } else if (cur_bytes == 0) {
+            break;
+        }
+
+        /*
+         * 3. If the request still hasn't completed, allocate new clusters,
+         *    considering any cluster_offset of steps 1c or 2.
+         */
+        ret = handle_alloc(bs, start, &cluster_offset, &cur_bytes, m);
+        if (ret < 0) {
+            return ret;
+        } else if (ret) {
+            continue;
+        } else {
+            assert(cur_bytes == 0);
+            break;
+        }
+    }
+
+    *num = (n_end - n_start) - (remaining >> BDRV_SECTOR_BITS);
+    assert(*num > 0);
+    assert(*host_offset != 0);
+
+    return 0;
+}
+
+static int decompress_buffer(uint8_t *out_buf, int out_buf_size,
+                             const uint8_t *buf, int buf_size)
+{
+    z_stream strm1, *strm = &strm1;
+    int ret, out_len;
+
+    memset(strm, 0, sizeof(*strm));
+
+    strm->next_in = (uint8_t *)buf;
+    strm->avail_in = buf_size;
+    strm->next_out = out_buf;
+    strm->avail_out = out_buf_size;
+
+    ret = inflateInit2(strm, -12);
+    if (ret != Z_OK)
+        return -1;
+    ret = inflate(strm, Z_FINISH);
+    out_len = strm->next_out - out_buf;
+    if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) ||
+        out_len != out_buf_size) {
+        inflateEnd(strm);
+        return -1;
+    }
+    inflateEnd(strm);
+    return 0;
+}
+
+int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret, csize, nb_csectors, sector_offset;
+    uint64_t coffset;
+
+    coffset = cluster_offset & s->cluster_offset_mask;
+    if (s->cluster_cache_offset != coffset) {
+        nb_csectors = ((cluster_offset >> s->csize_shift) & s->csize_mask) + 1;
+        sector_offset = coffset & 511;
+        csize = nb_csectors * 512 - sector_offset;
+        BLKDBG_EVENT(bs->file, BLKDBG_READ_COMPRESSED);
+        ret = bdrv_read(bs->file, coffset >> 9, s->cluster_data, nb_csectors);
+        if (ret < 0) {
+            return ret;
+        }
+        if (decompress_buffer(s->cluster_cache, s->cluster_size,
+                              s->cluster_data + sector_offset, csize) < 0) {
+            return -EIO;
+        }
+        s->cluster_cache_offset = coffset;
+    }
+    return 0;
+}
+
+/*
+ * This discards as many clusters of nb_clusters as possible at once (i.e.
+ * all clusters in the same L2 table) and returns the number of discarded
+ * clusters.
+ */
+static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
+    unsigned int nb_clusters)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t *l2_table;
+    int l2_index;
+    int ret;
+    int i;
+
+    ret = get_cluster_table(bs, offset, &l2_table, &l2_index);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* Limit nb_clusters to one L2 table */
+    nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+
+    for (i = 0; i < nb_clusters; i++) {
+        uint64_t old_offset;
+
+        old_offset = be64_to_cpu(l2_table[l2_index + i]);
+        if ((old_offset & L2E_OFFSET_MASK) == 0) {
+            continue;
+        }
+
+        /* First remove L2 entries */
+        qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+        l2_table[l2_index + i] = cpu_to_be64(0);
+
+        /* Then decrease the refcount */
+        qcow2_free_any_clusters(bs, old_offset, 1, QCOW2_DISCARD_REQUEST);
+    }
+
+    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return nb_clusters;
+}
+
+int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
+    int nb_sectors)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t end_offset;
+    unsigned int nb_clusters;
+    int ret;
+
+    end_offset = offset + (nb_sectors << BDRV_SECTOR_BITS);
+
+    /* Round start up and end down */
+    offset = align_offset(offset, s->cluster_size);
+    end_offset &= ~(s->cluster_size - 1);
+
+    if (offset > end_offset) {
+        return 0;
+    }
+
+    nb_clusters = size_to_clusters(s, end_offset - offset);
+
+    s->cache_discards = true;
+
+    /* Each L2 table is handled by its own loop iteration */
+    while (nb_clusters > 0) {
+        ret = discard_single_l2(bs, offset, nb_clusters);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        nb_clusters -= ret;
+        offset += (ret * s->cluster_size);
+    }
+
+    ret = 0;
+fail:
+    s->cache_discards = false;
+    qcow2_process_discards(bs, ret);
+
+    return ret;
+}
+
+/*
+ * This zeroes as many clusters of nb_clusters as possible at once (i.e.
+ * all clusters in the same L2 table) and returns the number of zeroed
+ * clusters.
+ */
+static int zero_single_l2(BlockDriverState *bs, uint64_t offset,
+    unsigned int nb_clusters)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t *l2_table;
+    int l2_index;
+    int ret;
+    int i;
+
+    ret = get_cluster_table(bs, offset, &l2_table, &l2_index);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* Limit nb_clusters to one L2 table */
+    nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+
+    for (i = 0; i < nb_clusters; i++) {
+        uint64_t old_offset;
+
+        old_offset = be64_to_cpu(l2_table[l2_index + i]);
+
+        /* Update L2 entries */
+        qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+        if (old_offset & QCOW_OFLAG_COMPRESSED) {
+            l2_table[l2_index + i] = cpu_to_be64(QCOW_OFLAG_ZERO);
+            qcow2_free_any_clusters(bs, old_offset, 1, QCOW2_DISCARD_REQUEST);
+        } else {
+            l2_table[l2_index + i] |= cpu_to_be64(QCOW_OFLAG_ZERO);
+        }
+    }
+
+    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return nb_clusters;
+}
+
+int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors)
+{
+    BDRVQcowState *s = bs->opaque;
+    unsigned int nb_clusters;
+    int ret;
+
+    /* The zero flag is only supported by version 3 and newer */
+    if (s->qcow_version < 3) {
+        return -ENOTSUP;
+    }
+
+    /* Each L2 table is handled by its own loop iteration */
+    nb_clusters = size_to_clusters(s, nb_sectors << BDRV_SECTOR_BITS);
+
+    s->cache_discards = true;
+
+    while (nb_clusters > 0) {
+        ret = zero_single_l2(bs, offset, nb_clusters);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        nb_clusters -= ret;
+        offset += (ret * s->cluster_size);
+    }
+
+    ret = 0;
+fail:
+    s->cache_discards = false;
+    qcow2_process_discards(bs, ret);
+
+    return ret;
+}
diff --git a/contrib/qemu/block/qcow2-refcount.c b/contrib/qemu/block/qcow2-refcount.c
new file mode 100644
index 000000000..1244693f3
--- /dev/null
+++ b/contrib/qemu/block/qcow2-refcount.c
@@ -0,0 +1,1374 @@
+/*
+ * Block driver for the QCOW version 2 format
+ *
+ * Copyright (c) 2004-2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu-common.h"
+#include "block/block_int.h"
+#include "block/qcow2.h"
+
+static int64_t alloc_clusters_noref(BlockDriverState *bs, int64_t size);
+static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
+                            int64_t offset, int64_t length,
+                            int addend, enum qcow2_discard_type type);
+
+
+/*********************************************************/
+/* refcount handling */
+
+int qcow2_refcount_init(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret, refcount_table_size2, i;
+
+    refcount_table_size2 = s->refcount_table_size * sizeof(uint64_t);
+    s->refcount_table = g_malloc(refcount_table_size2);
+    if (s->refcount_table_size > 0) {
+        BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_LOAD);
+        ret = bdrv_pread(bs->file, s->refcount_table_offset,
+                         s->refcount_table, refcount_table_size2);
+        if (ret != refcount_table_size2)
+            goto fail;
+        for(i = 0; i < s->refcount_table_size; i++)
+            be64_to_cpus(&s->refcount_table[i]);
+    }
+    return 0;
+ fail:
+    return -ENOMEM;
+}
+
+void qcow2_refcount_close(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    g_free(s->refcount_table);
+}
+
+
+static int load_refcount_block(BlockDriverState *bs,
+                               int64_t refcount_block_offset,
+                               void **refcount_block)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret;
+
+    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_LOAD);
+    ret = qcow2_cache_get(bs, s->refcount_block_cache, refcount_block_offset,
+        refcount_block);
+
+    return ret;
+}
+
+/*
+ * Returns the refcount of the cluster given by its index. Any non-negative
+ * return value is the refcount of the cluster, negative values are -errno
+ * and indicate an error.
+ */
+static int get_refcount(BlockDriverState *bs, int64_t cluster_index)
+{
+    BDRVQcowState *s = bs->opaque;
+    int refcount_table_index, block_index;
+    int64_t refcount_block_offset;
+    int ret;
+    uint16_t *refcount_block;
+    uint16_t refcount;
+
+    refcount_table_index = cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT);
+    if (refcount_table_index >= s->refcount_table_size)
+        return 0;
+    refcount_block_offset = s->refcount_table[refcount_table_index];
+    if (!refcount_block_offset)
+        return 0;
+
+    ret = qcow2_cache_get(bs, s->refcount_block_cache, refcount_block_offset,
+        (void**) &refcount_block);
+    if (ret < 0) {
+        return ret;
+    }
+
+    block_index = cluster_index &
+        ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
+    refcount = be16_to_cpu(refcount_block[block_index]);
+
+    ret = qcow2_cache_put(bs, s->refcount_block_cache,
+        (void**) &refcount_block);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return refcount;
+}
+
+/*
+ * Rounds the refcount table size up to avoid growing the table for each single
+ * refcount block that is allocated.
+ */
+static unsigned int next_refcount_table_size(BDRVQcowState *s,
+    unsigned int min_size)
+{
+    unsigned int min_clusters = (min_size >> (s->cluster_bits - 3)) + 1;
+    unsigned int refcount_table_clusters =
+        MAX(1, s->refcount_table_size >> (s->cluster_bits - 3));
+
+    while (min_clusters > refcount_table_clusters) {
+        refcount_table_clusters = (refcount_table_clusters * 3 + 1) / 2;
+    }
+
+    return refcount_table_clusters << (s->cluster_bits - 3);
+}
+
+
+/* Checks if two offsets are described by the same refcount block */
+static int in_same_refcount_block(BDRVQcowState *s, uint64_t offset_a,
+    uint64_t offset_b)
+{
+    uint64_t block_a = offset_a >> (2 * s->cluster_bits - REFCOUNT_SHIFT);
+    uint64_t block_b = offset_b >> (2 * s->cluster_bits - REFCOUNT_SHIFT);
+
+    return (block_a == block_b);
+}
+
+/*
+ * Loads a refcount block. If it doesn't exist yet, it is allocated first
+ * (including growing the refcount table if needed).
+ *
+ * Returns 0 on success or -errno in error case
+ */
+static int alloc_refcount_block(BlockDriverState *bs,
+    int64_t cluster_index, uint16_t **refcount_block)
+{
+    BDRVQcowState *s = bs->opaque;
+    unsigned int refcount_table_index;
+    int ret;
+
+    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC);
+
+    /* Find the refcount block for the given cluster */
+    refcount_table_index = cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT);
+
+    if (refcount_table_index < s->refcount_table_size) {
+
+        uint64_t refcount_block_offset =
+            s->refcount_table[refcount_table_index] & REFT_OFFSET_MASK;
+
+        /* If it's already there, we're done */
+        if (refcount_block_offset) {
+             return load_refcount_block(bs, refcount_block_offset,
+                 (void**) refcount_block);
+        }
+    }
+
+    /*
+     * If we came here, we need to allocate something. Something is at least
+     * a cluster for the new refcount block. It may also include a new refcount
+     * table if the old refcount table is too small.
+     *
+     * Note that allocating clusters here needs some special care:
+     *
+     * - We can't use the normal qcow2_alloc_clusters(), it would try to
+     *   increase the refcount and very likely we would end up with an endless
+     *   recursion. Instead we must place the refcount blocks in a way that
+     *   they can describe them themselves.
+     *
+     * - We need to consider that at this point we are inside update_refcounts
+     *   and doing the initial refcount increase. This means that some clusters
+     *   have already been allocated by the caller, but their refcount isn't
+     *   accurate yet. free_cluster_index tells us where this allocation ends
+     *   as long as we don't overwrite it by freeing clusters.
+     *
+     * - alloc_clusters_noref and qcow2_free_clusters may load a different
+     *   refcount block into the cache
+     */
+
+    *refcount_block = NULL;
+
+    /* We write to the refcount table, so we might depend on L2 tables */
+    ret = qcow2_cache_flush(bs, s->l2_table_cache);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* Allocate the refcount block itself and mark it as used */
+    int64_t new_block = alloc_clusters_noref(bs, s->cluster_size);
+    if (new_block < 0) {
+        return new_block;
+    }
+
+#ifdef DEBUG_ALLOC2
+    fprintf(stderr, "qcow2: Allocate refcount block %d for %" PRIx64
+        " at %" PRIx64 "\n",
+        refcount_table_index, cluster_index << s->cluster_bits, new_block);
+#endif
+
+    if (in_same_refcount_block(s, new_block, cluster_index << s->cluster_bits)) {
+        /* Zero the new refcount block before updating it */
+        ret = qcow2_cache_get_empty(bs, s->refcount_block_cache, new_block,
+            (void**) refcount_block);
+        if (ret < 0) {
+            goto fail_block;
+        }
+
+        memset(*refcount_block, 0, s->cluster_size);
+
+        /* The block describes itself, need to update the cache */
+        int block_index = (new_block >> s->cluster_bits) &
+            ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
+        (*refcount_block)[block_index] = cpu_to_be16(1);
+    } else {
+        /* Described somewhere else. This can recurse at most twice before we
+         * arrive at a block that describes itself. */
+        ret = update_refcount(bs, new_block, s->cluster_size, 1,
+                              QCOW2_DISCARD_NEVER);
+        if (ret < 0) {
+            goto fail_block;
+        }
+
+        ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+        if (ret < 0) {
+            goto fail_block;
+        }
+
+        /* Initialize the new refcount block only after updating its refcount,
+         * update_refcount uses the refcount cache itself */
+        ret = qcow2_cache_get_empty(bs, s->refcount_block_cache, new_block,
+            (void**) refcount_block);
+        if (ret < 0) {
+            goto fail_block;
+        }
+
+        memset(*refcount_block, 0, s->cluster_size);
+    }
+
+    /* Now the new refcount block needs to be written to disk */
+    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE);
+    qcow2_cache_entry_mark_dirty(s->refcount_block_cache, *refcount_block);
+    ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+    if (ret < 0) {
+        goto fail_block;
+    }
+
+    /* If the refcount table is big enough, just hook the block up there */
+    if (refcount_table_index < s->refcount_table_size) {
+        uint64_t data64 = cpu_to_be64(new_block);
+        BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_HOOKUP);
+        ret = bdrv_pwrite_sync(bs->file,
+            s->refcount_table_offset + refcount_table_index * sizeof(uint64_t),
+            &data64, sizeof(data64));
+        if (ret < 0) {
+            goto fail_block;
+        }
+
+        s->refcount_table[refcount_table_index] = new_block;
+        return 0;
+    }
+
+    ret = qcow2_cache_put(bs, s->refcount_block_cache, (void**) refcount_block);
+    if (ret < 0) {
+        goto fail_block;
+    }
+
+    /*
+     * If we come here, we need to grow the refcount table. Again, a new
+     * refcount table needs some space and we can't simply allocate to avoid
+     * endless recursion.
+     *
+     * Therefore let's grab new refcount blocks at the end of the image, which
+     * will describe themselves and the new refcount table. This way we can
+     * reference them only in the new table and do the switch to the new
+     * refcount table at once without producing an inconsistent state in
+     * between.
+     */
+    BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_GROW);
+
+    /* Calculate the number of refcount blocks needed so far */
+    uint64_t refcount_block_clusters = 1 << (s->cluster_bits - REFCOUNT_SHIFT);
+    uint64_t blocks_used = (s->free_cluster_index +
+        refcount_block_clusters - 1) / refcount_block_clusters;
+
+    /* And now we need at least one block more for the new metadata */
+    uint64_t table_size = next_refcount_table_size(s, blocks_used + 1);
+    uint64_t last_table_size;
+    uint64_t blocks_clusters;
+    do {
+        uint64_t table_clusters =
+            size_to_clusters(s, table_size * sizeof(uint64_t));
+        blocks_clusters = 1 +
+            ((table_clusters + refcount_block_clusters - 1)
+            / refcount_block_clusters);
+        uint64_t meta_clusters = table_clusters + blocks_clusters;
+
+        last_table_size = table_size;
+        table_size = next_refcount_table_size(s, blocks_used +
+            ((meta_clusters + refcount_block_clusters - 1)
+            / refcount_block_clusters));
+
+    } while (last_table_size != table_size);
+
+#ifdef DEBUG_ALLOC2
+    fprintf(stderr, "qcow2: Grow refcount table %" PRId32 " => %" PRId64 "\n",
+        s->refcount_table_size, table_size);
+#endif
+
+    /* Create the new refcount table and blocks */
+    uint64_t meta_offset = (blocks_used * refcount_block_clusters) *
+        s->cluster_size;
+    uint64_t table_offset = meta_offset + blocks_clusters * s->cluster_size;
+    uint16_t *new_blocks = g_malloc0(blocks_clusters * s->cluster_size);
+    uint64_t *new_table = g_malloc0(table_size * sizeof(uint64_t));
+
+    assert(meta_offset >= (s->free_cluster_index * s->cluster_size));
+
+    /* Fill the new refcount table */
+    memcpy(new_table, s->refcount_table,
+        s->refcount_table_size * sizeof(uint64_t));
+    new_table[refcount_table_index] = new_block;
+
+    int i;
+    for (i = 0; i < blocks_clusters; i++) {
+        new_table[blocks_used + i] = meta_offset + (i * s->cluster_size);
+    }
+
+    /* Fill the refcount blocks */
+    uint64_t table_clusters = size_to_clusters(s, table_size * sizeof(uint64_t));
+    int block = 0;
+    for (i = 0; i < table_clusters + blocks_clusters; i++) {
+        new_blocks[block++] = cpu_to_be16(1);
+    }
+
+    /* Write refcount blocks to disk */
+    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE_BLOCKS);
+    ret = bdrv_pwrite_sync(bs->file, meta_offset, new_blocks,
+        blocks_clusters * s->cluster_size);
+    g_free(new_blocks);
+    if (ret < 0) {
+        goto fail_table;
+    }
+
+    /* Write refcount table to disk */
+    for(i = 0; i < table_size; i++) {
+        cpu_to_be64s(&new_table[i]);
+    }
+
+    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE_TABLE);
+    ret = bdrv_pwrite_sync(bs->file, table_offset, new_table,
+        table_size * sizeof(uint64_t));
+    if (ret < 0) {
+        goto fail_table;
+    }
+
+    for(i = 0; i < table_size; i++) {
+        be64_to_cpus(&new_table[i]);
+    }
+
+    /* Hook up the new refcount table in the qcow2 header */
+    uint8_t data[12];
+    cpu_to_be64w((uint64_t*)data, table_offset);
+    cpu_to_be32w((uint32_t*)(data + 8), table_clusters);
+    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE);
+    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, refcount_table_offset),
+        data, sizeof(data));
+    if (ret < 0) {
+        goto fail_table;
+    }
+
+    /* And switch it in memory */
+    uint64_t old_table_offset = s->refcount_table_offset;
+    uint64_t old_table_size = s->refcount_table_size;
+
+    g_free(s->refcount_table);
+    s->refcount_table = new_table;
+    s->refcount_table_size = table_size;
+    s->refcount_table_offset = table_offset;
+
+    /* Free old table. Remember, we must not change free_cluster_index */
+    uint64_t old_free_cluster_index = s->free_cluster_index;
+    qcow2_free_clusters(bs, old_table_offset, old_table_size * sizeof(uint64_t),
+                        QCOW2_DISCARD_OTHER);
+    s->free_cluster_index = old_free_cluster_index;
+
+    ret = load_refcount_block(bs, new_block, (void**) refcount_block);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return 0;
+
+fail_table:
+    g_free(new_table);
+fail_block:
+    if (*refcount_block != NULL) {
+        qcow2_cache_put(bs, s->refcount_block_cache, (void**) refcount_block);
+    }
+    return ret;
+}
+
+void qcow2_process_discards(BlockDriverState *bs, int ret)
+{
+    BDRVQcowState *s = bs->opaque;
+    Qcow2DiscardRegion *d, *next;
+
+    QTAILQ_FOREACH_SAFE(d, &s->discards, next, next) {
+        QTAILQ_REMOVE(&s->discards, d, next);
+
+        /* Discard is optional, ignore the return value */
+        if (ret >= 0) {
+            bdrv_discard(bs->file,
+                         d->offset >> BDRV_SECTOR_BITS,
+                         d->bytes >> BDRV_SECTOR_BITS);
+        }
+
+        g_free(d);
+    }
+}
+
+static void update_refcount_discard(BlockDriverState *bs,
+                                    uint64_t offset, uint64_t length)
+{
+    BDRVQcowState *s = bs->opaque;
+    Qcow2DiscardRegion *d, *p, *next;
+
+    QTAILQ_FOREACH(d, &s->discards, next) {
+        uint64_t new_start = MIN(offset, d->offset);
+        uint64_t new_end = MAX(offset + length, d->offset + d->bytes);
+
+        if (new_end - new_start <= length + d->bytes) {
+            /* There can't be any overlap, areas ending up here have no
+             * references any more and therefore shouldn't get freed another
+             * time. */
+            assert(d->bytes + length == new_end - new_start);
+            d->offset = new_start;
+            d->bytes = new_end - new_start;
+            goto found;
+        }
+    }
+
+    d = g_malloc(sizeof(*d));
+    *d = (Qcow2DiscardRegion) {
+        .bs     = bs,
+        .offset = offset,
+        .bytes  = length,
+    };
+    QTAILQ_INSERT_TAIL(&s->discards, d, next);
+
+found:
+    /* Merge discard requests if they are adjacent now */
+    QTAILQ_FOREACH_SAFE(p, &s->discards, next, next) {
+        if (p == d
+            || p->offset > d->offset + d->bytes
+            || d->offset > p->offset + p->bytes)
+        {
+            continue;
+        }
+
+        /* Still no overlap possible */
+        assert(p->offset == d->offset + d->bytes
+            || d->offset == p->offset + p->bytes);
+
+        QTAILQ_REMOVE(&s->discards, p, next);
+        d->offset = MIN(d->offset, p->offset);
+        d->bytes += p->bytes;
+    }
+}
+
+/* XXX: cache several refcount block clusters ? */
+static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
+    int64_t offset, int64_t length, int addend, enum qcow2_discard_type type)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t start, last, cluster_offset;
+    uint16_t *refcount_block = NULL;
+    int64_t old_table_index = -1;
+    int ret;
+
+#ifdef DEBUG_ALLOC2
+    fprintf(stderr, "update_refcount: offset=%" PRId64 " size=%" PRId64 " addend=%d\n",
+           offset, length, addend);
+#endif
+    if (length < 0) {
+        return -EINVAL;
+    } else if (length == 0) {
+        return 0;
+    }
+
+    if (addend < 0) {
+        qcow2_cache_set_dependency(bs, s->refcount_block_cache,
+            s->l2_table_cache);
+    }
+
+    start = offset & ~(s->cluster_size - 1);
+    last = (offset + length - 1) & ~(s->cluster_size - 1);
+    for(cluster_offset = start; cluster_offset <= last;
+        cluster_offset += s->cluster_size)
+    {
+        int block_index, refcount;
+        int64_t cluster_index = cluster_offset >> s->cluster_bits;
+        int64_t table_index =
+            cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT);
+
+        /* Load the refcount block and allocate it if needed */
+        if (table_index != old_table_index) {
+            if (refcount_block) {
+                ret = qcow2_cache_put(bs, s->refcount_block_cache,
+                    (void**) &refcount_block);
+                if (ret < 0) {
+                    goto fail;
+                }
+            }
+
+            ret = alloc_refcount_block(bs, cluster_index, &refcount_block);
+            if (ret < 0) {
+                goto fail;
+            }
+        }
+        old_table_index = table_index;
+
+        qcow2_cache_entry_mark_dirty(s->refcount_block_cache, refcount_block);
+
+        /* we can update the count and save it */
+        block_index = cluster_index &
+            ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
+
+        refcount = be16_to_cpu(refcount_block[block_index]);
+        refcount += addend;
+        if (refcount < 0 || refcount > 0xffff) {
+            ret = -EINVAL;
+            goto fail;
+        }
+        if (refcount == 0 && cluster_index < s->free_cluster_index) {
+            s->free_cluster_index = cluster_index;
+        }
+        refcount_block[block_index] = cpu_to_be16(refcount);
+
+        if (refcount == 0 && s->discard_passthrough[type]) {
+            update_refcount_discard(bs, cluster_offset, s->cluster_size);
+        }
+    }
+
+    ret = 0;
+fail:
+    if (!s->cache_discards) {
+        qcow2_process_discards(bs, ret);
+    }
+
+    /* Write last changed block to disk */
+    if (refcount_block) {
+        int wret;
+        wret = qcow2_cache_put(bs, s->refcount_block_cache,
+            (void**) &refcount_block);
+        if (wret < 0) {
+            return ret < 0 ? ret : wret;
+        }
+    }
+
+    /*
+     * Try do undo any updates if an error is returned (This may succeed in
+     * some cases like ENOSPC for allocating a new refcount block)
+     */
+    if (ret < 0) {
+        int dummy;
+        dummy = update_refcount(bs, offset, cluster_offset - offset, -addend,
+                                QCOW2_DISCARD_NEVER);
+        (void)dummy;
+    }
+
+    return ret;
+}
+
+/*
+ * Increases or decreases the refcount of a given cluster by one.
+ * addend must be 1 or -1.
+ *
+ * If the return value is non-negative, it is the new refcount of the cluster.
+ * If it is negative, it is -errno and indicates an error.
+ */
+static int update_cluster_refcount(BlockDriverState *bs,
+                                   int64_t cluster_index,
+                                   int addend,
+                                   enum qcow2_discard_type type)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret;
+
+    ret = update_refcount(bs, cluster_index << s->cluster_bits, 1, addend,
+                          type);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return get_refcount(bs, cluster_index);
+}
+
+
+
+/*********************************************************/
+/* cluster allocation functions */
+
+
+
+/* return < 0 if error */
+static int64_t alloc_clusters_noref(BlockDriverState *bs, int64_t size)
+{
+    BDRVQcowState *s = bs->opaque;
+    int i, nb_clusters, refcount;
+
+    nb_clusters = size_to_clusters(s, size);
+retry:
+    for(i = 0; i < nb_clusters; i++) {
+        int64_t next_cluster_index = s->free_cluster_index++;
+        refcount = get_refcount(bs, next_cluster_index);
+
+        if (refcount < 0) {
+            return refcount;
+        } else if (refcount != 0) {
+            goto retry;
+        }
+    }
+#ifdef DEBUG_ALLOC2
+    fprintf(stderr, "alloc_clusters: size=%" PRId64 " -> %" PRId64 "\n",
+            size,
+            (s->free_cluster_index - nb_clusters) << s->cluster_bits);
+#endif
+    return (s->free_cluster_index - nb_clusters) << s->cluster_bits;
+}
+
+int64_t qcow2_alloc_clusters(BlockDriverState *bs, int64_t size)
+{
+    int64_t offset;
+    int ret;
+
+    BLKDBG_EVENT(bs->file, BLKDBG_CLUSTER_ALLOC);
+    offset = alloc_clusters_noref(bs, size);
+    if (offset < 0) {
+        return offset;
+    }
+
+    ret = update_refcount(bs, offset, size, 1, QCOW2_DISCARD_NEVER);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return offset;
+}
+
+int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
+    int nb_clusters)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t cluster_index;
+    uint64_t old_free_cluster_index;
+    int i, refcount, ret;
+
+    /* Check how many clusters there are free */
+    cluster_index = offset >> s->cluster_bits;
+    for(i = 0; i < nb_clusters; i++) {
+        refcount = get_refcount(bs, cluster_index++);
+
+        if (refcount < 0) {
+            return refcount;
+        } else if (refcount != 0) {
+            break;
+        }
+    }
+
+    /* And then allocate them */
+    old_free_cluster_index = s->free_cluster_index;
+    s->free_cluster_index = cluster_index + i;
+
+    ret = update_refcount(bs, offset, i << s->cluster_bits, 1,
+                          QCOW2_DISCARD_NEVER);
+    if (ret < 0) {
+        return ret;
+    }
+
+    s->free_cluster_index = old_free_cluster_index;
+
+    return i;
+}
+
+/* only used to allocate compressed sectors. We try to allocate
+   contiguous sectors. size must be <= cluster_size */
+int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t offset, cluster_offset;
+    int free_in_cluster;
+
+    BLKDBG_EVENT(bs->file, BLKDBG_CLUSTER_ALLOC_BYTES);
+    assert(size > 0 && size <= s->cluster_size);
+    if (s->free_byte_offset == 0) {
+        offset = qcow2_alloc_clusters(bs, s->cluster_size);
+        if (offset < 0) {
+            return offset;
+        }
+        s->free_byte_offset = offset;
+    }
+ redo:
+    free_in_cluster = s->cluster_size -
+        (s->free_byte_offset & (s->cluster_size - 1));
+    if (size <= free_in_cluster) {
+        /* enough space in current cluster */
+        offset = s->free_byte_offset;
+        s->free_byte_offset += size;
+        free_in_cluster -= size;
+        if (free_in_cluster == 0)
+            s->free_byte_offset = 0;
+        if ((offset & (s->cluster_size - 1)) != 0)
+            update_cluster_refcount(bs, offset >> s->cluster_bits, 1,
+                                    QCOW2_DISCARD_NEVER);
+    } else {
+        offset = qcow2_alloc_clusters(bs, s->cluster_size);
+        if (offset < 0) {
+            return offset;
+        }
+        cluster_offset = s->free_byte_offset & ~(s->cluster_size - 1);
+        if ((cluster_offset + s->cluster_size) == offset) {
+            /* we are lucky: contiguous data */
+            offset = s->free_byte_offset;
+            update_cluster_refcount(bs, offset >> s->cluster_bits, 1,
+                                    QCOW2_DISCARD_NEVER);
+            s->free_byte_offset += size;
+        } else {
+            s->free_byte_offset = offset;
+            goto redo;
+        }
+    }
+
+    /* The cluster refcount was incremented, either by qcow2_alloc_clusters()
+     * or explicitly by update_cluster_refcount().  Refcount blocks must be
+     * flushed before the caller's L2 table updates.
+     */
+    qcow2_cache_set_dependency(bs, s->l2_table_cache, s->refcount_block_cache);
+    return offset;
+}
+
+void qcow2_free_clusters(BlockDriverState *bs,
+                          int64_t offset, int64_t size,
+                          enum qcow2_discard_type type)
+{
+    int ret;
+
+    BLKDBG_EVENT(bs->file, BLKDBG_CLUSTER_FREE);
+    ret = update_refcount(bs, offset, size, -1, type);
+    if (ret < 0) {
+        fprintf(stderr, "qcow2_free_clusters failed: %s\n", strerror(-ret));
+        /* TODO Remember the clusters to free them later and avoid leaking */
+    }
+}
+
+/*
+ * Free a cluster using its L2 entry (handles clusters of all types, e.g.
+ * normal cluster, compressed cluster, etc.)
+ */
+void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
+                             int nb_clusters, enum qcow2_discard_type type)
+{
+    BDRVQcowState *s = bs->opaque;
+
+    switch (qcow2_get_cluster_type(l2_entry)) {
+    case QCOW2_CLUSTER_COMPRESSED:
+        {
+            int nb_csectors;
+            nb_csectors = ((l2_entry >> s->csize_shift) &
+                           s->csize_mask) + 1;
+            qcow2_free_clusters(bs,
+                (l2_entry & s->cluster_offset_mask) & ~511,
+                nb_csectors * 512, type);
+        }
+        break;
+    case QCOW2_CLUSTER_NORMAL:
+        qcow2_free_clusters(bs, l2_entry & L2E_OFFSET_MASK,
+                            nb_clusters << s->cluster_bits, type);
+        break;
+    case QCOW2_CLUSTER_UNALLOCATED:
+    case QCOW2_CLUSTER_ZERO:
+        break;
+    default:
+        abort();
+    }
+}
+
+
+
+/*********************************************************/
+/* snapshots and image creation */
+
+
+
+/* update the refcounts of snapshots and the copied flag */
+int qcow2_update_snapshot_refcount(BlockDriverState *bs,
+    int64_t l1_table_offset, int l1_size, int addend)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t *l1_table, *l2_table, l2_offset, offset, l1_size2, l1_allocated;
+    int64_t old_offset, old_l2_offset;
+    int i, j, l1_modified = 0, nb_csectors, refcount;
+    int ret;
+
+    l2_table = NULL;
+    l1_table = NULL;
+    l1_size2 = l1_size * sizeof(uint64_t);
+
+    s->cache_discards = true;
+
+    /* WARNING: qcow2_snapshot_goto relies on this function not using the
+     * l1_table_offset when it is the current s->l1_table_offset! Be careful
+     * when changing this! */
+    if (l1_table_offset != s->l1_table_offset) {
+        l1_table = g_malloc0(align_offset(l1_size2, 512));
+        l1_allocated = 1;
+
+        ret = bdrv_pread(bs->file, l1_table_offset, l1_table, l1_size2);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        for(i = 0;i < l1_size; i++)
+            be64_to_cpus(&l1_table[i]);
+    } else {
+        assert(l1_size == s->l1_size);
+        l1_table = s->l1_table;
+        l1_allocated = 0;
+    }
+
+    for(i = 0; i < l1_size; i++) {
+        l2_offset = l1_table[i];
+        if (l2_offset) {
+            old_l2_offset = l2_offset;
+            l2_offset &= L1E_OFFSET_MASK;
+
+            ret = qcow2_cache_get(bs, s->l2_table_cache, l2_offset,
+                (void**) &l2_table);
+            if (ret < 0) {
+                goto fail;
+            }
+
+            for(j = 0; j < s->l2_size; j++) {
+                offset = be64_to_cpu(l2_table[j]);
+                if (offset != 0) {
+                    old_offset = offset;
+                    offset &= ~QCOW_OFLAG_COPIED;
+                    if (offset & QCOW_OFLAG_COMPRESSED) {
+                        nb_csectors = ((offset >> s->csize_shift) &
+                                       s->csize_mask) + 1;
+                        if (addend != 0) {
+                            int ret;
+                            ret = update_refcount(bs,
+                                (offset & s->cluster_offset_mask) & ~511,
+                                nb_csectors * 512, addend,
+                                QCOW2_DISCARD_SNAPSHOT);
+                            if (ret < 0) {
+                                goto fail;
+                            }
+                        }
+                        /* compressed clusters are never modified */
+                        refcount = 2;
+                    } else {
+                        uint64_t cluster_index = (offset & L2E_OFFSET_MASK) >> s->cluster_bits;
+                        if (addend != 0) {
+                            refcount = update_cluster_refcount(bs, cluster_index, addend,
+                                                               QCOW2_DISCARD_SNAPSHOT);
+                        } else {
+                            refcount = get_refcount(bs, cluster_index);
+                        }
+
+                        if (refcount < 0) {
+                            ret = refcount;
+                            goto fail;
+                        }
+                    }
+
+                    if (refcount == 1) {
+                        offset |= QCOW_OFLAG_COPIED;
+                    }
+                    if (offset != old_offset) {
+                        if (addend > 0) {
+                            qcow2_cache_set_dependency(bs, s->l2_table_cache,
+                                s->refcount_block_cache);
+                        }
+                        l2_table[j] = cpu_to_be64(offset);
+                        qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+                    }
+                }
+            }
+
+            ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+            if (ret < 0) {
+                goto fail;
+            }
+
+
+            if (addend != 0) {
+                refcount = update_cluster_refcount(bs, l2_offset >> s->cluster_bits, addend,
+                                                   QCOW2_DISCARD_SNAPSHOT);
+            } else {
+                refcount = get_refcount(bs, l2_offset >> s->cluster_bits);
+            }
+            if (refcount < 0) {
+                ret = refcount;
+                goto fail;
+            } else if (refcount == 1) {
+                l2_offset |= QCOW_OFLAG_COPIED;
+            }
+            if (l2_offset != old_l2_offset) {
+                l1_table[i] = l2_offset;
+                l1_modified = 1;
+            }
+        }
+    }
+
+    ret = bdrv_flush(bs);
+fail:
+    if (l2_table) {
+        qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
+    }
+
+    s->cache_discards = false;
+    qcow2_process_discards(bs, ret);
+
+    /* Update L1 only if it isn't deleted anyway (addend = -1) */
+    if (ret == 0 && addend >= 0 && l1_modified) {
+        for (i = 0; i < l1_size; i++) {
+            cpu_to_be64s(&l1_table[i]);
+        }
+
+        ret = bdrv_pwrite_sync(bs->file, l1_table_offset, l1_table, l1_size2);
+
+        for (i = 0; i < l1_size; i++) {
+            be64_to_cpus(&l1_table[i]);
+        }
+    }
+    if (l1_allocated)
+        g_free(l1_table);
+    return ret;
+}
+
+
+
+
+/*********************************************************/
+/* refcount checking functions */
+
+
+
+/*
+ * Increases the refcount for a range of clusters in a given refcount table.
+ * This is used to construct a temporary refcount table out of L1 and L2 tables
+ * which can be compared the the refcount table saved in the image.
+ *
+ * Modifies the number of errors in res.
+ */
+static void inc_refcounts(BlockDriverState *bs,
+                          BdrvCheckResult *res,
+                          uint16_t *refcount_table,
+                          int refcount_table_size,
+                          int64_t offset, int64_t size)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t start, last, cluster_offset;
+    int k;
+
+    if (size <= 0)
+        return;
+
+    start = offset & ~(s->cluster_size - 1);
+    last = (offset + size - 1) & ~(s->cluster_size - 1);
+    for(cluster_offset = start; cluster_offset <= last;
+        cluster_offset += s->cluster_size) {
+        k = cluster_offset >> s->cluster_bits;
+        if (k < 0) {
+            fprintf(stderr, "ERROR: invalid cluster offset=0x%" PRIx64 "\n",
+                cluster_offset);
+            res->corruptions++;
+        } else if (k >= refcount_table_size) {
+            fprintf(stderr, "Warning: cluster offset=0x%" PRIx64 " is after "
+                "the end of the image file, can't properly check refcounts.\n",
+                cluster_offset);
+            res->check_errors++;
+        } else {
+            if (++refcount_table[k] == 0) {
+                fprintf(stderr, "ERROR: overflow cluster offset=0x%" PRIx64
+                    "\n", cluster_offset);
+                res->corruptions++;
+            }
+        }
+    }
+}
+
+/* Flags for check_refcounts_l1() and check_refcounts_l2() */
+enum {
+    CHECK_OFLAG_COPIED = 0x1,   /* check QCOW_OFLAG_COPIED matches refcount */
+    CHECK_FRAG_INFO = 0x2,      /* update BlockFragInfo counters */
+};
+
+/*
+ * Increases the refcount in the given refcount table for the all clusters
+ * referenced in the L2 table. While doing so, performs some checks on L2
+ * entries.
+ *
+ * Returns the number of errors found by the checks or -errno if an internal
+ * error occurred.
+ */
+static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
+    uint16_t *refcount_table, int refcount_table_size, int64_t l2_offset,
+    int flags)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t *l2_table, l2_entry;
+    uint64_t next_contiguous_offset = 0;
+    int i, l2_size, nb_csectors, refcount;
+
+    /* Read L2 table from disk */
+    l2_size = s->l2_size * sizeof(uint64_t);
+    l2_table = g_malloc(l2_size);
+
+    if (bdrv_pread(bs->file, l2_offset, l2_table, l2_size) != l2_size)
+        goto fail;
+
+    /* Do the actual checks */
+    for(i = 0; i < s->l2_size; i++) {
+        l2_entry = be64_to_cpu(l2_table[i]);
+
+        switch (qcow2_get_cluster_type(l2_entry)) {
+        case QCOW2_CLUSTER_COMPRESSED:
+            /* Compressed clusters don't have QCOW_OFLAG_COPIED */
+            if (l2_entry & QCOW_OFLAG_COPIED) {
+                fprintf(stderr, "ERROR: cluster %" PRId64 ": "
+                    "copied flag must never be set for compressed "
+                    "clusters\n", l2_entry >> s->cluster_bits);
+                l2_entry &= ~QCOW_OFLAG_COPIED;
+                res->corruptions++;
+            }
+
+            /* Mark cluster as used */
+            nb_csectors = ((l2_entry >> s->csize_shift) &
+                           s->csize_mask) + 1;
+            l2_entry &= s->cluster_offset_mask;
+            inc_refcounts(bs, res, refcount_table, refcount_table_size,
+                l2_entry & ~511, nb_csectors * 512);
+
+            if (flags & CHECK_FRAG_INFO) {
+                res->bfi.allocated_clusters++;
+                res->bfi.compressed_clusters++;
+
+                /* Compressed clusters are fragmented by nature.  Since they
+                 * take up sub-sector space but we only have sector granularity
+                 * I/O we need to re-read the same sectors even for adjacent
+                 * compressed clusters.
+                 */
+                res->bfi.fragmented_clusters++;
+            }
+            break;
+
+        case QCOW2_CLUSTER_ZERO:
+            if ((l2_entry & L2E_OFFSET_MASK) == 0) {
+                break;
+            }
+            /* fall through */
+
+        case QCOW2_CLUSTER_NORMAL:
+        {
+            /* QCOW_OFLAG_COPIED must be set iff refcount == 1 */
+            uint64_t offset = l2_entry & L2E_OFFSET_MASK;
+
+            if (flags & CHECK_OFLAG_COPIED) {
+                refcount = get_refcount(bs, offset >> s->cluster_bits);
+                if (refcount < 0) {
+                    fprintf(stderr, "Can't get refcount for offset %"
+                        PRIx64 ": %s\n", l2_entry, strerror(-refcount));
+                    goto fail;
+                }
+                if ((refcount == 1) != ((l2_entry & QCOW_OFLAG_COPIED) != 0)) {
+                    fprintf(stderr, "ERROR OFLAG_COPIED: offset=%"
+                        PRIx64 " refcount=%d\n", l2_entry, refcount);
+                    res->corruptions++;
+                }
+            }
+
+            if (flags & CHECK_FRAG_INFO) {
+                res->bfi.allocated_clusters++;
+                if (next_contiguous_offset &&
+                    offset != next_contiguous_offset) {
+                    res->bfi.fragmented_clusters++;
+                }
+                next_contiguous_offset = offset + s->cluster_size;
+            }
+
+            /* Mark cluster as used */
+            inc_refcounts(bs, res, refcount_table,refcount_table_size,
+                offset, s->cluster_size);
+
+            /* Correct offsets are cluster aligned */
+            if (offset & (s->cluster_size - 1)) {
+                fprintf(stderr, "ERROR offset=%" PRIx64 ": Cluster is not "
+                    "properly aligned; L2 entry corrupted.\n", offset);
+                res->corruptions++;
+            }
+            break;
+        }
+
+        case QCOW2_CLUSTER_UNALLOCATED:
+            break;
+
+        default:
+            abort();
+        }
+    }
+
+    g_free(l2_table);
+    return 0;
+
+fail:
+    fprintf(stderr, "ERROR: I/O error in check_refcounts_l2\n");
+    g_free(l2_table);
+    return -EIO;
+}
+
+/*
+ * Increases the refcount for the L1 table, its L2 tables and all referenced
+ * clusters in the given refcount table. While doing so, performs some checks
+ * on L1 and L2 entries.
+ *
+ * Returns the number of errors found by the checks or -errno if an internal
+ * error occurred.
+ */
+static int check_refcounts_l1(BlockDriverState *bs,
+                              BdrvCheckResult *res,
+                              uint16_t *refcount_table,
+                              int refcount_table_size,
+                              int64_t l1_table_offset, int l1_size,
+                              int flags)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t *l1_table, l2_offset, l1_size2;
+    int i, refcount, ret;
+
+    l1_size2 = l1_size * sizeof(uint64_t);
+
+    /* Mark L1 table as used */
+    inc_refcounts(bs, res, refcount_table, refcount_table_size,
+        l1_table_offset, l1_size2);
+
+    /* Read L1 table entries from disk */
+    if (l1_size2 == 0) {
+        l1_table = NULL;
+    } else {
+        l1_table = g_malloc(l1_size2);
+        if (bdrv_pread(bs->file, l1_table_offset,
+                       l1_table, l1_size2) != l1_size2)
+            goto fail;
+        for(i = 0;i < l1_size; i++)
+            be64_to_cpus(&l1_table[i]);
+    }
+
+    /* Do the actual checks */
+    for(i = 0; i < l1_size; i++) {
+        l2_offset = l1_table[i];
+        if (l2_offset) {
+            /* QCOW_OFLAG_COPIED must be set iff refcount == 1 */
+            if (flags & CHECK_OFLAG_COPIED) {
+                refcount = get_refcount(bs, (l2_offset & ~QCOW_OFLAG_COPIED)
+                    >> s->cluster_bits);
+                if (refcount < 0) {
+                    fprintf(stderr, "Can't get refcount for l2_offset %"
+                        PRIx64 ": %s\n", l2_offset, strerror(-refcount));
+                    goto fail;
+                }
+                if ((refcount == 1) != ((l2_offset & QCOW_OFLAG_COPIED) != 0)) {
+                    fprintf(stderr, "ERROR OFLAG_COPIED: l2_offset=%" PRIx64
+                        " refcount=%d\n", l2_offset, refcount);
+                    res->corruptions++;
+                }
+            }
+
+            /* Mark L2 table as used */
+            l2_offset &= L1E_OFFSET_MASK;
+            inc_refcounts(bs, res, refcount_table, refcount_table_size,
+                l2_offset, s->cluster_size);
+
+            /* L2 tables are cluster aligned */
+            if (l2_offset & (s->cluster_size - 1)) {
+                fprintf(stderr, "ERROR l2_offset=%" PRIx64 ": Table is not "
+                    "cluster aligned; L1 entry corrupted\n", l2_offset);
+                res->corruptions++;
+            }
+
+            /* Process and check L2 entries */
+            ret = check_refcounts_l2(bs, res, refcount_table,
+                                     refcount_table_size, l2_offset, flags);
+            if (ret < 0) {
+                goto fail;
+            }
+        }
+    }
+    g_free(l1_table);
+    return 0;
+
+fail:
+    fprintf(stderr, "ERROR: I/O error in check_refcounts_l1\n");
+    res->check_errors++;
+    g_free(l1_table);
+    return -EIO;
+}
+
+/*
+ * Checks an image for refcount consistency.
+ *
+ * Returns 0 if no errors are found, the number of errors in case the image is
+ * detected as corrupted, and -errno when an internal error occurred.
+ */
+int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
+                          BdrvCheckMode fix)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t size, i, highest_cluster;
+    int nb_clusters, refcount1, refcount2;
+    QCowSnapshot *sn;
+    uint16_t *refcount_table;
+    int ret;
+
+    size = bdrv_getlength(bs->file);
+    nb_clusters = size_to_clusters(s, size);
+    refcount_table = g_malloc0(nb_clusters * sizeof(uint16_t));
+
+    res->bfi.total_clusters =
+        size_to_clusters(s, bs->total_sectors * BDRV_SECTOR_SIZE);
+
+    /* header */
+    inc_refcounts(bs, res, refcount_table, nb_clusters,
+        0, s->cluster_size);
+
+    /* current L1 table */
+    ret = check_refcounts_l1(bs, res, refcount_table, nb_clusters,
+                             s->l1_table_offset, s->l1_size,
+                             CHECK_OFLAG_COPIED | CHECK_FRAG_INFO);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* snapshots */
+    for(i = 0; i < s->nb_snapshots; i++) {
+        sn = s->snapshots + i;
+        ret = check_refcounts_l1(bs, res, refcount_table, nb_clusters,
+            sn->l1_table_offset, sn->l1_size, 0);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+    inc_refcounts(bs, res, refcount_table, nb_clusters,
+        s->snapshots_offset, s->snapshots_size);
+
+    /* refcount data */
+    inc_refcounts(bs, res, refcount_table, nb_clusters,
+        s->refcount_table_offset,
+        s->refcount_table_size * sizeof(uint64_t));
+
+    for(i = 0; i < s->refcount_table_size; i++) {
+        uint64_t offset, cluster;
+        offset = s->refcount_table[i];
+        cluster = offset >> s->cluster_bits;
+
+        /* Refcount blocks are cluster aligned */
+        if (offset & (s->cluster_size - 1)) {
+            fprintf(stderr, "ERROR refcount block %" PRId64 " is not "
+                "cluster aligned; refcount table entry corrupted\n", i);
+            res->corruptions++;
+            continue;
+        }
+
+        if (cluster >= nb_clusters) {
+            fprintf(stderr, "ERROR refcount block %" PRId64
+                    " is outside image\n", i);
+            res->corruptions++;
+            continue;
+        }
+
+        if (offset != 0) {
+            inc_refcounts(bs, res, refcount_table, nb_clusters,
+                offset, s->cluster_size);
+            if (refcount_table[cluster] != 1) {
+                fprintf(stderr, "ERROR refcount block %" PRId64
+                    " refcount=%d\n",
+                    i, refcount_table[cluster]);
+                res->corruptions++;
+            }
+        }
+    }
+
+    /* compare ref counts */
+    for (i = 0, highest_cluster = 0; i < nb_clusters; i++) {
+        refcount1 = get_refcount(bs, i);
+        if (refcount1 < 0) {
+            fprintf(stderr, "Can't get refcount for cluster %" PRId64 ": %s\n",
+                i, strerror(-refcount1));
+            res->check_errors++;
+            continue;
+        }
+
+        refcount2 = refcount_table[i];
+
+        if (refcount1 > 0 || refcount2 > 0) {
+            highest_cluster = i;
+        }
+
+        if (refcount1 != refcount2) {
+
+            /* Check if we're allowed to fix the mismatch */
+            int *num_fixed = NULL;
+            if (refcount1 > refcount2 && (fix & BDRV_FIX_LEAKS)) {
+                num_fixed = &res->leaks_fixed;
+            } else if (refcount1 < refcount2 && (fix & BDRV_FIX_ERRORS)) {
+                num_fixed = &res->corruptions_fixed;
+            }
+
+            fprintf(stderr, "%s cluster %" PRId64 " refcount=%d reference=%d\n",
+                   num_fixed != NULL     ? "Repairing" :
+                   refcount1 < refcount2 ? "ERROR" :
+                                           "Leaked",
+                   i, refcount1, refcount2);
+
+            if (num_fixed) {
+                ret = update_refcount(bs, i << s->cluster_bits, 1,
+                                      refcount2 - refcount1,
+                                      QCOW2_DISCARD_ALWAYS);
+                if (ret >= 0) {
+                    (*num_fixed)++;
+                    continue;
+                }
+            }
+
+            /* And if we couldn't, print an error */
+            if (refcount1 < refcount2) {
+                res->corruptions++;
+            } else {
+                res->leaks++;
+            }
+        }
+    }
+
+    res->image_end_offset = (highest_cluster + 1) * s->cluster_size;
+    ret = 0;
+
+fail:
+    g_free(refcount_table);
+
+    return ret;
+}
+
diff --git a/contrib/qemu/block/qcow2-snapshot.c b/contrib/qemu/block/qcow2-snapshot.c
new file mode 100644
index 000000000..0caac9055
--- /dev/null
+++ b/contrib/qemu/block/qcow2-snapshot.c
@@ -0,0 +1,660 @@
+/*
+ * Block driver for the QCOW version 2 format
+ *
+ * Copyright (c) 2004-2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu-common.h"
+#include "block/block_int.h"
+#include "block/qcow2.h"
+
+typedef struct QEMU_PACKED QCowSnapshotHeader {
+    /* header is 8 byte aligned */
+    uint64_t l1_table_offset;
+
+    uint32_t l1_size;
+    uint16_t id_str_size;
+    uint16_t name_size;
+
+    uint32_t date_sec;
+    uint32_t date_nsec;
+
+    uint64_t vm_clock_nsec;
+
+    uint32_t vm_state_size;
+    uint32_t extra_data_size; /* for extension */
+    /* extra data follows */
+    /* id_str follows */
+    /* name follows  */
+} QCowSnapshotHeader;
+
+typedef struct QEMU_PACKED QCowSnapshotExtraData {
+    uint64_t vm_state_size_large;
+    uint64_t disk_size;
+} QCowSnapshotExtraData;
+
+void qcow2_free_snapshots(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    int i;
+
+    for(i = 0; i < s->nb_snapshots; i++) {
+        g_free(s->snapshots[i].name);
+        g_free(s->snapshots[i].id_str);
+    }
+    g_free(s->snapshots);
+    s->snapshots = NULL;
+    s->nb_snapshots = 0;
+}
+
+int qcow2_read_snapshots(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    QCowSnapshotHeader h;
+    QCowSnapshotExtraData extra;
+    QCowSnapshot *sn;
+    int i, id_str_size, name_size;
+    int64_t offset;
+    uint32_t extra_data_size;
+    int ret;
+
+    if (!s->nb_snapshots) {
+        s->snapshots = NULL;
+        s->snapshots_size = 0;
+        return 0;
+    }
+
+    offset = s->snapshots_offset;
+    s->snapshots = g_malloc0(s->nb_snapshots * sizeof(QCowSnapshot));
+
+    for(i = 0; i < s->nb_snapshots; i++) {
+        /* Read statically sized part of the snapshot header */
+        offset = align_offset(offset, 8);
+        ret = bdrv_pread(bs->file, offset, &h, sizeof(h));
+        if (ret < 0) {
+            goto fail;
+        }
+
+        offset += sizeof(h);
+        sn = s->snapshots + i;
+        sn->l1_table_offset = be64_to_cpu(h.l1_table_offset);
+        sn->l1_size = be32_to_cpu(h.l1_size);
+        sn->vm_state_size = be32_to_cpu(h.vm_state_size);
+        sn->date_sec = be32_to_cpu(h.date_sec);
+        sn->date_nsec = be32_to_cpu(h.date_nsec);
+        sn->vm_clock_nsec = be64_to_cpu(h.vm_clock_nsec);
+        extra_data_size = be32_to_cpu(h.extra_data_size);
+
+        id_str_size = be16_to_cpu(h.id_str_size);
+        name_size = be16_to_cpu(h.name_size);
+
+        /* Read extra data */
+        ret = bdrv_pread(bs->file, offset, &extra,
+                         MIN(sizeof(extra), extra_data_size));
+        if (ret < 0) {
+            goto fail;
+        }
+        offset += extra_data_size;
+
+        if (extra_data_size >= 8) {
+            sn->vm_state_size = be64_to_cpu(extra.vm_state_size_large);
+        }
+
+        if (extra_data_size >= 16) {
+            sn->disk_size = be64_to_cpu(extra.disk_size);
+        } else {
+            sn->disk_size = bs->total_sectors * BDRV_SECTOR_SIZE;
+        }
+
+        /* Read snapshot ID */
+        sn->id_str = g_malloc(id_str_size + 1);
+        ret = bdrv_pread(bs->file, offset, sn->id_str, id_str_size);
+        if (ret < 0) {
+            goto fail;
+        }
+        offset += id_str_size;
+        sn->id_str[id_str_size] = '\0';
+
+        /* Read snapshot name */
+        sn->name = g_malloc(name_size + 1);
+        ret = bdrv_pread(bs->file, offset, sn->name, name_size);
+        if (ret < 0) {
+            goto fail;
+        }
+        offset += name_size;
+        sn->name[name_size] = '\0';
+    }
+
+    s->snapshots_size = offset - s->snapshots_offset;
+    return 0;
+
+fail:
+    qcow2_free_snapshots(bs);
+    return ret;
+}
+
+/* add at the end of the file a new list of snapshots */
+static int qcow2_write_snapshots(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    QCowSnapshot *sn;
+    QCowSnapshotHeader h;
+    QCowSnapshotExtraData extra;
+    int i, name_size, id_str_size, snapshots_size;
+    struct {
+        uint32_t nb_snapshots;
+        uint64_t snapshots_offset;
+    } QEMU_PACKED header_data;
+    int64_t offset, snapshots_offset;
+    int ret;
+
+    /* compute the size of the snapshots */
+    offset = 0;
+    for(i = 0; i < s->nb_snapshots; i++) {
+        sn = s->snapshots + i;
+        offset = align_offset(offset, 8);
+        offset += sizeof(h);
+        offset += sizeof(extra);
+        offset += strlen(sn->id_str);
+        offset += strlen(sn->name);
+    }
+    snapshots_size = offset;
+
+    /* Allocate space for the new snapshot list */
+    snapshots_offset = qcow2_alloc_clusters(bs, snapshots_size);
+    offset = snapshots_offset;
+    if (offset < 0) {
+        return offset;
+    }
+    ret = bdrv_flush(bs);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* Write all snapshots to the new list */
+    for(i = 0; i < s->nb_snapshots; i++) {
+        sn = s->snapshots + i;
+        memset(&h, 0, sizeof(h));
+        h.l1_table_offset = cpu_to_be64(sn->l1_table_offset);
+        h.l1_size = cpu_to_be32(sn->l1_size);
+        /* If it doesn't fit in 32 bit, older implementations should treat it
+         * as a disk-only snapshot rather than truncate the VM state */
+        if (sn->vm_state_size <= 0xffffffff) {
+            h.vm_state_size = cpu_to_be32(sn->vm_state_size);
+        }
+        h.date_sec = cpu_to_be32(sn->date_sec);
+        h.date_nsec = cpu_to_be32(sn->date_nsec);
+        h.vm_clock_nsec = cpu_to_be64(sn->vm_clock_nsec);
+        h.extra_data_size = cpu_to_be32(sizeof(extra));
+
+        memset(&extra, 0, sizeof(extra));
+        extra.vm_state_size_large = cpu_to_be64(sn->vm_state_size);
+        extra.disk_size = cpu_to_be64(sn->disk_size);
+
+        id_str_size = strlen(sn->id_str);
+        name_size = strlen(sn->name);
+        h.id_str_size = cpu_to_be16(id_str_size);
+        h.name_size = cpu_to_be16(name_size);
+        offset = align_offset(offset, 8);
+
+        ret = bdrv_pwrite(bs->file, offset, &h, sizeof(h));
+        if (ret < 0) {
+            goto fail;
+        }
+        offset += sizeof(h);
+
+        ret = bdrv_pwrite(bs->file, offset, &extra, sizeof(extra));
+        if (ret < 0) {
+            goto fail;
+        }
+        offset += sizeof(extra);
+
+        ret = bdrv_pwrite(bs->file, offset, sn->id_str, id_str_size);
+        if (ret < 0) {
+            goto fail;
+        }
+        offset += id_str_size;
+
+        ret = bdrv_pwrite(bs->file, offset, sn->name, name_size);
+        if (ret < 0) {
+            goto fail;
+        }
+        offset += name_size;
+    }
+
+    /*
+     * Update the header to point to the new snapshot table. This requires the
+     * new table and its refcounts to be stable on disk.
+     */
+    ret = bdrv_flush(bs);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    QEMU_BUILD_BUG_ON(offsetof(QCowHeader, snapshots_offset) !=
+        offsetof(QCowHeader, nb_snapshots) + sizeof(header_data.nb_snapshots));
+
+    header_data.nb_snapshots        = cpu_to_be32(s->nb_snapshots);
+    header_data.snapshots_offset    = cpu_to_be64(snapshots_offset);
+
+    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, nb_snapshots),
+                           &header_data, sizeof(header_data));
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* free the old snapshot table */
+    qcow2_free_clusters(bs, s->snapshots_offset, s->snapshots_size,
+                        QCOW2_DISCARD_SNAPSHOT);
+    s->snapshots_offset = snapshots_offset;
+    s->snapshots_size = snapshots_size;
+    return 0;
+
+fail:
+    return ret;
+}
+
+static void find_new_snapshot_id(BlockDriverState *bs,
+                                 char *id_str, int id_str_size)
+{
+    BDRVQcowState *s = bs->opaque;
+    QCowSnapshot *sn;
+    int i, id, id_max = 0;
+
+    for(i = 0; i < s->nb_snapshots; i++) {
+        sn = s->snapshots + i;
+        id = strtoul(sn->id_str, NULL, 10);
+        if (id > id_max)
+            id_max = id;
+    }
+    snprintf(id_str, id_str_size, "%d", id_max + 1);
+}
+
+static int find_snapshot_by_id(BlockDriverState *bs, const char *id_str)
+{
+    BDRVQcowState *s = bs->opaque;
+    int i;
+
+    for(i = 0; i < s->nb_snapshots; i++) {
+        if (!strcmp(s->snapshots[i].id_str, id_str))
+            return i;
+    }
+    return -1;
+}
+
+static int find_snapshot_by_id_or_name(BlockDriverState *bs, const char *name)
+{
+    BDRVQcowState *s = bs->opaque;
+    int i, ret;
+
+    ret = find_snapshot_by_id(bs, name);
+    if (ret >= 0)
+        return ret;
+    for(i = 0; i < s->nb_snapshots; i++) {
+        if (!strcmp(s->snapshots[i].name, name))
+            return i;
+    }
+    return -1;
+}
+
+/* if no id is provided, a new one is constructed */
+int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
+{
+    BDRVQcowState *s = bs->opaque;
+    QCowSnapshot *new_snapshot_list = NULL;
+    QCowSnapshot *old_snapshot_list = NULL;
+    QCowSnapshot sn1, *sn = &sn1;
+    int i, ret;
+    uint64_t *l1_table = NULL;
+    int64_t l1_table_offset;
+
+    memset(sn, 0, sizeof(*sn));
+
+    /* Generate an ID if it wasn't passed */
+    if (sn_info->id_str[0] == '\0') {
+        find_new_snapshot_id(bs, sn_info->id_str, sizeof(sn_info->id_str));
+    }
+
+    /* Check that the ID is unique */
+    if (find_snapshot_by_id(bs, sn_info->id_str) >= 0) {
+        return -EEXIST;
+    }
+
+    /* Populate sn with passed data */
+    sn->id_str = g_strdup(sn_info->id_str);
+    sn->name = g_strdup(sn_info->name);
+
+    sn->disk_size = bs->total_sectors * BDRV_SECTOR_SIZE;
+    sn->vm_state_size = sn_info->vm_state_size;
+    sn->date_sec = sn_info->date_sec;
+    sn->date_nsec = sn_info->date_nsec;
+    sn->vm_clock_nsec = sn_info->vm_clock_nsec;
+
+    /* Allocate the L1 table of the snapshot and copy the current one there. */
+    l1_table_offset = qcow2_alloc_clusters(bs, s->l1_size * sizeof(uint64_t));
+    if (l1_table_offset < 0) {
+        ret = l1_table_offset;
+        goto fail;
+    }
+
+    sn->l1_table_offset = l1_table_offset;
+    sn->l1_size = s->l1_size;
+
+    l1_table = g_malloc(s->l1_size * sizeof(uint64_t));
+    for(i = 0; i < s->l1_size; i++) {
+        l1_table[i] = cpu_to_be64(s->l1_table[i]);
+    }
+
+    ret = bdrv_pwrite(bs->file, sn->l1_table_offset, l1_table,
+                      s->l1_size * sizeof(uint64_t));
+    if (ret < 0) {
+        goto fail;
+    }
+
+    g_free(l1_table);
+    l1_table = NULL;
+
+    /*
+     * Increase the refcounts of all clusters and make sure everything is
+     * stable on disk before updating the snapshot table to contain a pointer
+     * to the new L1 table.
+     */
+    ret = qcow2_update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, 1);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Append the new snapshot to the snapshot list */
+    new_snapshot_list = g_malloc((s->nb_snapshots + 1) * sizeof(QCowSnapshot));
+    if (s->snapshots) {
+        memcpy(new_snapshot_list, s->snapshots,
+               s->nb_snapshots * sizeof(QCowSnapshot));
+        old_snapshot_list = s->snapshots;
+    }
+    s->snapshots = new_snapshot_list;
+    s->snapshots[s->nb_snapshots++] = *sn;
+
+    ret = qcow2_write_snapshots(bs);
+    if (ret < 0) {
+        g_free(s->snapshots);
+        s->snapshots = old_snapshot_list;
+        goto fail;
+    }
+
+    g_free(old_snapshot_list);
+
+#ifdef DEBUG_ALLOC
+    {
+      BdrvCheckResult result = {0};
+      qcow2_check_refcounts(bs, &result, 0);
+    }
+#endif
+    return 0;
+
+fail:
+    g_free(sn->id_str);
+    g_free(sn->name);
+    g_free(l1_table);
+
+    return ret;
+}
+
+/* copy the snapshot 'snapshot_name' into the current disk image */
+int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
+{
+    BDRVQcowState *s = bs->opaque;
+    QCowSnapshot *sn;
+    int i, snapshot_index;
+    int cur_l1_bytes, sn_l1_bytes;
+    int ret;
+    uint64_t *sn_l1_table = NULL;
+
+    /* Search the snapshot */
+    snapshot_index = find_snapshot_by_id_or_name(bs, snapshot_id);
+    if (snapshot_index < 0) {
+        return -ENOENT;
+    }
+    sn = &s->snapshots[snapshot_index];
+
+    if (sn->disk_size != bs->total_sectors * BDRV_SECTOR_SIZE) {
+        error_report("qcow2: Loading snapshots with different disk "
+            "size is not implemented");
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    /*
+     * Make sure that the current L1 table is big enough to contain the whole
+     * L1 table of the snapshot. If the snapshot L1 table is smaller, the
+     * current one must be padded with zeros.
+     */
+    ret = qcow2_grow_l1_table(bs, sn->l1_size, true);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    cur_l1_bytes = s->l1_size * sizeof(uint64_t);
+    sn_l1_bytes = sn->l1_size * sizeof(uint64_t);
+
+    /*
+     * Copy the snapshot L1 table to the current L1 table.
+     *
+     * Before overwriting the old current L1 table on disk, make sure to
+     * increase all refcounts for the clusters referenced by the new one.
+     * Decrease the refcount referenced by the old one only when the L1
+     * table is overwritten.
+     */
+    sn_l1_table = g_malloc0(cur_l1_bytes);
+
+    ret = bdrv_pread(bs->file, sn->l1_table_offset, sn_l1_table, sn_l1_bytes);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    ret = qcow2_update_snapshot_refcount(bs, sn->l1_table_offset,
+                                         sn->l1_size, 1);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    ret = bdrv_pwrite_sync(bs->file, s->l1_table_offset, sn_l1_table,
+                           cur_l1_bytes);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /*
+     * Decrease refcount of clusters of current L1 table.
+     *
+     * At this point, the in-memory s->l1_table points to the old L1 table,
+     * whereas on disk we already have the new one.
+     *
+     * qcow2_update_snapshot_refcount special cases the current L1 table to use
+     * the in-memory data instead of really using the offset to load a new one,
+     * which is why this works.
+     */
+    ret = qcow2_update_snapshot_refcount(bs, s->l1_table_offset,
+                                         s->l1_size, -1);
+
+    /*
+     * Now update the in-memory L1 table to be in sync with the on-disk one. We
+     * need to do this even if updating refcounts failed.
+     */
+    for(i = 0;i < s->l1_size; i++) {
+        s->l1_table[i] = be64_to_cpu(sn_l1_table[i]);
+    }
+
+    if (ret < 0) {
+        goto fail;
+    }
+
+    g_free(sn_l1_table);
+    sn_l1_table = NULL;
+
+    /*
+     * Update QCOW_OFLAG_COPIED in the active L1 table (it may have changed
+     * when we decreased the refcount of the old snapshot.
+     */
+    ret = qcow2_update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, 0);
+    if (ret < 0) {
+        goto fail;
+    }
+
+#ifdef DEBUG_ALLOC
+    {
+        BdrvCheckResult result = {0};
+        qcow2_check_refcounts(bs, &result, 0);
+    }
+#endif
+    return 0;
+
+fail:
+    g_free(sn_l1_table);
+    return ret;
+}
+
+int qcow2_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
+{
+    BDRVQcowState *s = bs->opaque;
+    QCowSnapshot sn;
+    int snapshot_index, ret;
+
+    /* Search the snapshot */
+    snapshot_index = find_snapshot_by_id_or_name(bs, snapshot_id);
+    if (snapshot_index < 0) {
+        return -ENOENT;
+    }
+    sn = s->snapshots[snapshot_index];
+
+    /* Remove it from the snapshot list */
+    memmove(s->snapshots + snapshot_index,
+            s->snapshots + snapshot_index + 1,
+            (s->nb_snapshots - snapshot_index - 1) * sizeof(sn));
+    s->nb_snapshots--;
+    ret = qcow2_write_snapshots(bs);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /*
+     * The snapshot is now unused, clean up. If we fail after this point, we
+     * won't recover but just leak clusters.
+     */
+    g_free(sn.id_str);
+    g_free(sn.name);
+
+    /*
+     * Now decrease the refcounts of clusters referenced by the snapshot and
+     * free the L1 table.
+     */
+    ret = qcow2_update_snapshot_refcount(bs, sn.l1_table_offset,
+                                         sn.l1_size, -1);
+    if (ret < 0) {
+        return ret;
+    }
+    qcow2_free_clusters(bs, sn.l1_table_offset, sn.l1_size * sizeof(uint64_t),
+                        QCOW2_DISCARD_SNAPSHOT);
+
+    /* must update the copied flag on the current cluster offsets */
+    ret = qcow2_update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, 0);
+    if (ret < 0) {
+        return ret;
+    }
+
+#ifdef DEBUG_ALLOC
+    {
+        BdrvCheckResult result = {0};
+        qcow2_check_refcounts(bs, &result, 0);
+    }
+#endif
+    return 0;
+}
+
+int qcow2_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
+{
+    BDRVQcowState *s = bs->opaque;
+    QEMUSnapshotInfo *sn_tab, *sn_info;
+    QCowSnapshot *sn;
+    int i;
+
+    if (!s->nb_snapshots) {
+        *psn_tab = NULL;
+        return s->nb_snapshots;
+    }
+
+    sn_tab = g_malloc0(s->nb_snapshots * sizeof(QEMUSnapshotInfo));
+    for(i = 0; i < s->nb_snapshots; i++) {
+        sn_info = sn_tab + i;
+        sn = s->snapshots + i;
+        pstrcpy(sn_info->id_str, sizeof(sn_info->id_str),
+                sn->id_str);
+        pstrcpy(sn_info->name, sizeof(sn_info->name),
+                sn->name);
+        sn_info->vm_state_size = sn->vm_state_size;
+        sn_info->date_sec = sn->date_sec;
+        sn_info->date_nsec = sn->date_nsec;
+        sn_info->vm_clock_nsec = sn->vm_clock_nsec;
+    }
+    *psn_tab = sn_tab;
+    return s->nb_snapshots;
+}
+
+int qcow2_snapshot_load_tmp(BlockDriverState *bs, const char *snapshot_name)
+{
+    int i, snapshot_index;
+    BDRVQcowState *s = bs->opaque;
+    QCowSnapshot *sn;
+    uint64_t *new_l1_table;
+    int new_l1_bytes;
+    int ret;
+
+    assert(bs->read_only);
+
+    /* Search the snapshot */
+    snapshot_index = find_snapshot_by_id_or_name(bs, snapshot_name);
+    if (snapshot_index < 0) {
+        return -ENOENT;
+    }
+    sn = &s->snapshots[snapshot_index];
+
+    /* Allocate and read in the snapshot's L1 table */
+    new_l1_bytes = s->l1_size * sizeof(uint64_t);
+    new_l1_table = g_malloc0(align_offset(new_l1_bytes, 512));
+
+    ret = bdrv_pread(bs->file, sn->l1_table_offset, new_l1_table, new_l1_bytes);
+    if (ret < 0) {
+        g_free(new_l1_table);
+        return ret;
+    }
+
+    /* Switch the L1 table */
+    g_free(s->l1_table);
+
+    s->l1_size = sn->l1_size;
+    s->l1_table_offset = sn->l1_table_offset;
+    s->l1_table = new_l1_table;
+
+    for(i = 0;i < s->l1_size; i++) {
+        be64_to_cpus(&s->l1_table[i]);
+    }
+
+    return 0;
+}
diff --git a/contrib/qemu/block/qcow2.c b/contrib/qemu/block/qcow2.c
new file mode 100644
index 000000000..0eceefe2c
--- /dev/null
+++ b/contrib/qemu/block/qcow2.c
@@ -0,0 +1,1825 @@
+/*
+ * Block driver for the QCOW version 2 format
+ *
+ * Copyright (c) 2004-2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include <zlib.h>
+#include "qemu/aes.h"
+#include "block/qcow2.h"
+#include "qemu/error-report.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/qbool.h"
+#include "trace.h"
+
+/*
+  Differences with QCOW:
+
+  - Support for multiple incremental snapshots.
+  - Memory management by reference counts.
+  - Clusters which have a reference count of one have the bit
+    QCOW_OFLAG_COPIED to optimize write performance.
+  - Size of compressed clusters is stored in sectors to reduce bit usage
+    in the cluster offsets.
+  - Support for storing additional data (such as the VM state) in the
+    snapshots.
+  - If a backing store is used, the cluster size is not constrained
+    (could be backported to QCOW).
+  - L2 tables have always a size of one cluster.
+*/
+
+
+typedef struct {
+    uint32_t magic;
+    uint32_t len;
+} QCowExtension;
+
+#define  QCOW2_EXT_MAGIC_END 0
+#define  QCOW2_EXT_MAGIC_BACKING_FORMAT 0xE2792ACA
+#define  QCOW2_EXT_MAGIC_FEATURE_TABLE 0x6803f857
+
+static int qcow2_probe(const uint8_t *buf, int buf_size, const char *filename)
+{
+    const QCowHeader *cow_header = (const void *)buf;
+
+    if (buf_size >= sizeof(QCowHeader) &&
+        be32_to_cpu(cow_header->magic) == QCOW_MAGIC &&
+        be32_to_cpu(cow_header->version) >= 2)
+        return 100;
+    else
+        return 0;
+}
+
+
+/* 
+ * read qcow2 extension and fill bs
+ * start reading from start_offset
+ * finish reading upon magic of value 0 or when end_offset reached
+ * unknown magic is skipped (future extension this version knows nothing about)
+ * return 0 upon success, non-0 otherwise
+ */
+static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
+                                 uint64_t end_offset, void **p_feature_table)
+{
+    BDRVQcowState *s = bs->opaque;
+    QCowExtension ext;
+    uint64_t offset;
+    int ret;
+
+#ifdef DEBUG_EXT
+    printf("qcow2_read_extensions: start=%ld end=%ld\n", start_offset, end_offset);
+#endif
+    offset = start_offset;
+    while (offset < end_offset) {
+
+#ifdef DEBUG_EXT
+        /* Sanity check */
+        if (offset > s->cluster_size)
+            printf("qcow2_read_extension: suspicious offset %lu\n", offset);
+
+        printf("attempting to read extended header in offset %lu\n", offset);
+#endif
+
+        if (bdrv_pread(bs->file, offset, &ext, sizeof(ext)) != sizeof(ext)) {
+            fprintf(stderr, "qcow2_read_extension: ERROR: "
+                    "pread fail from offset %" PRIu64 "\n",
+                    offset);
+            return 1;
+        }
+        be32_to_cpus(&ext.magic);
+        be32_to_cpus(&ext.len);
+        offset += sizeof(ext);
+#ifdef DEBUG_EXT
+        printf("ext.magic = 0x%x\n", ext.magic);
+#endif
+        if (ext.len > end_offset - offset) {
+            error_report("Header extension too large");
+            return -EINVAL;
+        }
+
+        switch (ext.magic) {
+        case QCOW2_EXT_MAGIC_END:
+            return 0;
+
+        case QCOW2_EXT_MAGIC_BACKING_FORMAT:
+            if (ext.len >= sizeof(bs->backing_format)) {
+                fprintf(stderr, "ERROR: ext_backing_format: len=%u too large"
+                        " (>=%zu)\n",
+                        ext.len, sizeof(bs->backing_format));
+                return 2;
+            }
+            if (bdrv_pread(bs->file, offset , bs->backing_format,
+                           ext.len) != ext.len)
+                return 3;
+            bs->backing_format[ext.len] = '\0';
+#ifdef DEBUG_EXT
+            printf("Qcow2: Got format extension %s\n", bs->backing_format);
+#endif
+            break;
+
+        case QCOW2_EXT_MAGIC_FEATURE_TABLE:
+            if (p_feature_table != NULL) {
+                void* feature_table = g_malloc0(ext.len + 2 * sizeof(Qcow2Feature));
+                ret = bdrv_pread(bs->file, offset , feature_table, ext.len);
+                if (ret < 0) {
+                    return ret;
+                }
+
+                *p_feature_table = feature_table;
+            }
+            break;
+
+        default:
+            /* unknown magic - save it in case we need to rewrite the header */
+            {
+                Qcow2UnknownHeaderExtension *uext;
+
+                uext = g_malloc0(sizeof(*uext)  + ext.len);
+                uext->magic = ext.magic;
+                uext->len = ext.len;
+                QLIST_INSERT_HEAD(&s->unknown_header_ext, uext, next);
+
+                ret = bdrv_pread(bs->file, offset , uext->data, uext->len);
+                if (ret < 0) {
+                    return ret;
+                }
+            }
+            break;
+        }
+
+        offset += ((ext.len + 7) & ~7);
+    }
+
+    return 0;
+}
+
+static void cleanup_unknown_header_ext(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    Qcow2UnknownHeaderExtension *uext, *next;
+
+    QLIST_FOREACH_SAFE(uext, &s->unknown_header_ext, next, next) {
+        QLIST_REMOVE(uext, next);
+        g_free(uext);
+    }
+}
+
+static void GCC_FMT_ATTR(2, 3) report_unsupported(BlockDriverState *bs,
+    const char *fmt, ...)
+{
+    char msg[64];
+    va_list ap;
+
+    va_start(ap, fmt);
+    vsnprintf(msg, sizeof(msg), fmt, ap);
+    va_end(ap);
+
+    qerror_report(QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
+        bs->device_name, "qcow2", msg);
+}
+
+static void report_unsupported_feature(BlockDriverState *bs,
+    Qcow2Feature *table, uint64_t mask)
+{
+    while (table && table->name[0] != '\0') {
+        if (table->type == QCOW2_FEAT_TYPE_INCOMPATIBLE) {
+            if (mask & (1 << table->bit)) {
+                report_unsupported(bs, "%.46s",table->name);
+                mask &= ~(1 << table->bit);
+            }
+        }
+        table++;
+    }
+
+    if (mask) {
+        report_unsupported(bs, "Unknown incompatible feature: %" PRIx64, mask);
+    }
+}
+
+/*
+ * Sets the dirty bit and flushes afterwards if necessary.
+ *
+ * The incompatible_features bit is only set if the image file header was
+ * updated successfully.  Therefore it is not required to check the return
+ * value of this function.
+ */
+int qcow2_mark_dirty(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t val;
+    int ret;
+
+    assert(s->qcow_version >= 3);
+
+    if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) {
+        return 0; /* already dirty */
+    }
+
+    val = cpu_to_be64(s->incompatible_features | QCOW2_INCOMPAT_DIRTY);
+    ret = bdrv_pwrite(bs->file, offsetof(QCowHeader, incompatible_features),
+                      &val, sizeof(val));
+    if (ret < 0) {
+        return ret;
+    }
+    ret = bdrv_flush(bs->file);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* Only treat image as dirty if the header was updated successfully */
+    s->incompatible_features |= QCOW2_INCOMPAT_DIRTY;
+    return 0;
+}
+
+/*
+ * Clears the dirty bit and flushes before if necessary.  Only call this
+ * function when there are no pending requests, it does not guard against
+ * concurrent requests dirtying the image.
+ */
+static int qcow2_mark_clean(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+
+    if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) {
+        int ret = bdrv_flush(bs);
+        if (ret < 0) {
+            return ret;
+        }
+
+        s->incompatible_features &= ~QCOW2_INCOMPAT_DIRTY;
+        return qcow2_update_header(bs);
+    }
+    return 0;
+}
+
+static int qcow2_check(BlockDriverState *bs, BdrvCheckResult *result,
+                       BdrvCheckMode fix)
+{
+    int ret = qcow2_check_refcounts(bs, result, fix);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (fix && result->check_errors == 0 && result->corruptions == 0) {
+        return qcow2_mark_clean(bs);
+    }
+    return ret;
+}
+
+static QemuOptsList qcow2_runtime_opts = {
+    .name = "qcow2",
+    .head = QTAILQ_HEAD_INITIALIZER(qcow2_runtime_opts.head),
+    .desc = {
+        {
+            .name = "lazy_refcounts",
+            .type = QEMU_OPT_BOOL,
+            .help = "Postpone refcount updates",
+        },
+        {
+            .name = QCOW2_OPT_DISCARD_REQUEST,
+            .type = QEMU_OPT_BOOL,
+            .help = "Pass guest discard requests to the layer below",
+        },
+        {
+            .name = QCOW2_OPT_DISCARD_SNAPSHOT,
+            .type = QEMU_OPT_BOOL,
+            .help = "Generate discard requests when snapshot related space "
+                    "is freed",
+        },
+        {
+            .name = QCOW2_OPT_DISCARD_OTHER,
+            .type = QEMU_OPT_BOOL,
+            .help = "Generate discard requests when other clusters are freed",
+        },
+        { /* end of list */ }
+    },
+};
+
+static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
+{
+    BDRVQcowState *s = bs->opaque;
+    int len, i, ret = 0;
+    QCowHeader header;
+    QemuOpts *opts;
+    Error *local_err = NULL;
+    uint64_t ext_end;
+    uint64_t l1_vm_state_index;
+
+    ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
+    if (ret < 0) {
+        goto fail;
+    }
+    be32_to_cpus(&header.magic);
+    be32_to_cpus(&header.version);
+    be64_to_cpus(&header.backing_file_offset);
+    be32_to_cpus(&header.backing_file_size);
+    be64_to_cpus(&header.size);
+    be32_to_cpus(&header.cluster_bits);
+    be32_to_cpus(&header.crypt_method);
+    be64_to_cpus(&header.l1_table_offset);
+    be32_to_cpus(&header.l1_size);
+    be64_to_cpus(&header.refcount_table_offset);
+    be32_to_cpus(&header.refcount_table_clusters);
+    be64_to_cpus(&header.snapshots_offset);
+    be32_to_cpus(&header.nb_snapshots);
+
+    if (header.magic != QCOW_MAGIC) {
+        ret = -EMEDIUMTYPE;
+        goto fail;
+    }
+    if (header.version < 2 || header.version > 3) {
+        report_unsupported(bs, "QCOW version %d", header.version);
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    s->qcow_version = header.version;
+
+    /* Initialise version 3 header fields */
+    if (header.version == 2) {
+        header.incompatible_features    = 0;
+        header.compatible_features      = 0;
+        header.autoclear_features       = 0;
+        header.refcount_order           = 4;
+        header.header_length            = 72;
+    } else {
+        be64_to_cpus(&header.incompatible_features);
+        be64_to_cpus(&header.compatible_features);
+        be64_to_cpus(&header.autoclear_features);
+        be32_to_cpus(&header.refcount_order);
+        be32_to_cpus(&header.header_length);
+    }
+
+    if (header.header_length > sizeof(header)) {
+        s->unknown_header_fields_size = header.header_length - sizeof(header);
+        s->unknown_header_fields = g_malloc(s->unknown_header_fields_size);
+        ret = bdrv_pread(bs->file, sizeof(header), s->unknown_header_fields,
+                         s->unknown_header_fields_size);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+    if (header.backing_file_offset) {
+        ext_end = header.backing_file_offset;
+    } else {
+        ext_end = 1 << header.cluster_bits;
+    }
+
+    /* Handle feature bits */
+    s->incompatible_features    = header.incompatible_features;
+    s->compatible_features      = header.compatible_features;
+    s->autoclear_features       = header.autoclear_features;
+
+    if (s->incompatible_features & ~QCOW2_INCOMPAT_MASK) {
+        void *feature_table = NULL;
+        qcow2_read_extensions(bs, header.header_length, ext_end,
+                              &feature_table);
+        report_unsupported_feature(bs, feature_table,
+                                   s->incompatible_features &
+                                   ~QCOW2_INCOMPAT_MASK);
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    /* Check support for various header values */
+    if (header.refcount_order != 4) {
+        report_unsupported(bs, "%d bit reference counts",
+                           1 << header.refcount_order);
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    if (header.cluster_bits < MIN_CLUSTER_BITS ||
+        header.cluster_bits > MAX_CLUSTER_BITS) {
+        ret = -EINVAL;
+        goto fail;
+    }
+    if (header.crypt_method > QCOW_CRYPT_AES) {
+        ret = -EINVAL;
+        goto fail;
+    }
+    s->crypt_method_header = header.crypt_method;
+    if (s->crypt_method_header) {
+        bs->encrypted = 1;
+    }
+    s->cluster_bits = header.cluster_bits;
+    s->cluster_size = 1 << s->cluster_bits;
+    s->cluster_sectors = 1 << (s->cluster_bits - 9);
+    s->l2_bits = s->cluster_bits - 3; /* L2 is always one cluster */
+    s->l2_size = 1 << s->l2_bits;
+    bs->total_sectors = header.size / 512;
+    s->csize_shift = (62 - (s->cluster_bits - 8));
+    s->csize_mask = (1 << (s->cluster_bits - 8)) - 1;
+    s->cluster_offset_mask = (1LL << s->csize_shift) - 1;
+    s->refcount_table_offset = header.refcount_table_offset;
+    s->refcount_table_size =
+        header.refcount_table_clusters << (s->cluster_bits - 3);
+
+    s->snapshots_offset = header.snapshots_offset;
+    s->nb_snapshots = header.nb_snapshots;
+
+    /* read the level 1 table */
+    s->l1_size = header.l1_size;
+
+    l1_vm_state_index = size_to_l1(s, header.size);
+    if (l1_vm_state_index > INT_MAX) {
+        ret = -EFBIG;
+        goto fail;
+    }
+    s->l1_vm_state_index = l1_vm_state_index;
+
+    /* the L1 table must contain at least enough entries to put
+       header.size bytes */
+    if (s->l1_size < s->l1_vm_state_index) {
+        ret = -EINVAL;
+        goto fail;
+    }
+    s->l1_table_offset = header.l1_table_offset;
+    if (s->l1_size > 0) {
+        s->l1_table = g_malloc0(
+            align_offset(s->l1_size * sizeof(uint64_t), 512));
+        ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
+                         s->l1_size * sizeof(uint64_t));
+        if (ret < 0) {
+            goto fail;
+        }
+        for(i = 0;i < s->l1_size; i++) {
+            be64_to_cpus(&s->l1_table[i]);
+        }
+    }
+
+    /* alloc L2 table/refcount block cache */
+    s->l2_table_cache = qcow2_cache_create(bs, L2_CACHE_SIZE);
+    s->refcount_block_cache = qcow2_cache_create(bs, REFCOUNT_CACHE_SIZE);
+
+    s->cluster_cache = g_malloc(s->cluster_size);
+    /* one more sector for decompressed data alignment */
+    s->cluster_data = qemu_blockalign(bs, QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size
+                                  + 512);
+    s->cluster_cache_offset = -1;
+    s->flags = flags;
+
+    ret = qcow2_refcount_init(bs);
+    if (ret != 0) {
+        goto fail;
+    }
+
+    QLIST_INIT(&s->cluster_allocs);
+    QTAILQ_INIT(&s->discards);
+
+    /* read qcow2 extensions */
+    if (qcow2_read_extensions(bs, header.header_length, ext_end, NULL)) {
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* read the backing file name */
+    if (header.backing_file_offset != 0) {
+        len = header.backing_file_size;
+        if (len > 1023) {
+            len = 1023;
+        }
+        ret = bdrv_pread(bs->file, header.backing_file_offset,
+                         bs->backing_file, len);
+        if (ret < 0) {
+            goto fail;
+        }
+        bs->backing_file[len] = '\0';
+    }
+
+    ret = qcow2_read_snapshots(bs);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Clear unknown autoclear feature bits */
+    if (!bs->read_only && s->autoclear_features != 0) {
+        s->autoclear_features = 0;
+        ret = qcow2_update_header(bs);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+    /* Initialise locks */
+    qemu_co_mutex_init(&s->lock);
+
+    /* Repair image if dirty */
+    if (!(flags & BDRV_O_CHECK) && !bs->read_only &&
+        (s->incompatible_features & QCOW2_INCOMPAT_DIRTY)) {
+        BdrvCheckResult result = {0};
+
+        ret = qcow2_check(bs, &result, BDRV_FIX_ERRORS);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+    /* Enable lazy_refcounts according to image and command line options */
+    opts = qemu_opts_create_nofail(&qcow2_runtime_opts);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (error_is_set(&local_err)) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    s->use_lazy_refcounts = qemu_opt_get_bool(opts, QCOW2_OPT_LAZY_REFCOUNTS,
+        (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS));
+
+    s->discard_passthrough[QCOW2_DISCARD_NEVER] = false;
+    s->discard_passthrough[QCOW2_DISCARD_ALWAYS] = true;
+    s->discard_passthrough[QCOW2_DISCARD_REQUEST] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_REQUEST,
+                          flags & BDRV_O_UNMAP);
+    s->discard_passthrough[QCOW2_DISCARD_SNAPSHOT] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_SNAPSHOT, true);
+    s->discard_passthrough[QCOW2_DISCARD_OTHER] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_OTHER, false);
+
+    qemu_opts_del(opts);
+
+    if (s->use_lazy_refcounts && s->qcow_version < 3) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Lazy refcounts require "
+            "a qcow2 image with at least qemu 1.1 compatibility level");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+#ifdef DEBUG_ALLOC
+    {
+        BdrvCheckResult result = {0};
+        qcow2_check_refcounts(bs, &result, 0);
+    }
+#endif
+    return ret;
+
+ fail:
+    g_free(s->unknown_header_fields);
+    cleanup_unknown_header_ext(bs);
+    qcow2_free_snapshots(bs);
+    qcow2_refcount_close(bs);
+    g_free(s->l1_table);
+    if (s->l2_table_cache) {
+        qcow2_cache_destroy(bs, s->l2_table_cache);
+    }
+    g_free(s->cluster_cache);
+    qemu_vfree(s->cluster_data);
+    return ret;
+}
+
+static int qcow2_set_key(BlockDriverState *bs, const char *key)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint8_t keybuf[16];
+    int len, i;
+
+    memset(keybuf, 0, 16);
+    len = strlen(key);
+    if (len > 16)
+        len = 16;
+    /* XXX: we could compress the chars to 7 bits to increase
+       entropy */
+    for(i = 0;i < len;i++) {
+        keybuf[i] = key[i];
+    }
+    s->crypt_method = s->crypt_method_header;
+
+    if (AES_set_encrypt_key(keybuf, 128, &s->aes_encrypt_key) != 0)
+        return -1;
+    if (AES_set_decrypt_key(keybuf, 128, &s->aes_decrypt_key) != 0)
+        return -1;
+#if 0
+    /* test */
+    {
+        uint8_t in[16];
+        uint8_t out[16];
+        uint8_t tmp[16];
+        for(i=0;i<16;i++)
+            in[i] = i;
+        AES_encrypt(in, tmp, &s->aes_encrypt_key);
+        AES_decrypt(tmp, out, &s->aes_decrypt_key);
+        for(i = 0; i < 16; i++)
+            printf(" %02x", tmp[i]);
+        printf("\n");
+        for(i = 0; i < 16; i++)
+            printf(" %02x", out[i]);
+        printf("\n");
+    }
+#endif
+    return 0;
+}
+
+/* We have nothing to do for QCOW2 reopen, stubs just return
+ * success */
+static int qcow2_reopen_prepare(BDRVReopenState *state,
+                                BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
+static int coroutine_fn qcow2_co_is_allocated(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, int *pnum)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t cluster_offset;
+    int ret;
+
+    *pnum = nb_sectors;
+    /* FIXME We can get errors here, but the bdrv_co_is_allocated interface
+     * can't pass them on today */
+    qemu_co_mutex_lock(&s->lock);
+    ret = qcow2_get_cluster_offset(bs, sector_num << 9, pnum, &cluster_offset);
+    qemu_co_mutex_unlock(&s->lock);
+    if (ret < 0) {
+        *pnum = 0;
+    }
+
+    return (cluster_offset != 0) || (ret == QCOW2_CLUSTER_ZERO);
+}
+
+/* handle reading after the end of the backing file */
+int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
+                  int64_t sector_num, int nb_sectors)
+{
+    int n1;
+    if ((sector_num + nb_sectors) <= bs->total_sectors)
+        return nb_sectors;
+    if (sector_num >= bs->total_sectors)
+        n1 = 0;
+    else
+        n1 = bs->total_sectors - sector_num;
+
+    qemu_iovec_memset(qiov, 512 * n1, 0, 512 * (nb_sectors - n1));
+
+    return n1;
+}
+
+static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
+                          int remaining_sectors, QEMUIOVector *qiov)
+{
+    BDRVQcowState *s = bs->opaque;
+    int index_in_cluster, n1;
+    int ret;
+    int cur_nr_sectors; /* number of sectors in current iteration */
+    uint64_t cluster_offset = 0;
+    uint64_t bytes_done = 0;
+    QEMUIOVector hd_qiov;
+    uint8_t *cluster_data = NULL;
+
+    qemu_iovec_init(&hd_qiov, qiov->niov);
+
+    qemu_co_mutex_lock(&s->lock);
+
+    while (remaining_sectors != 0) {
+
+        /* prepare next request */
+        cur_nr_sectors = remaining_sectors;
+        if (s->crypt_method) {
+            cur_nr_sectors = MIN(cur_nr_sectors,
+                QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors);
+        }
+
+        ret = qcow2_get_cluster_offset(bs, sector_num << 9,
+            &cur_nr_sectors, &cluster_offset);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        index_in_cluster = sector_num & (s->cluster_sectors - 1);
+
+        qemu_iovec_reset(&hd_qiov);
+        qemu_iovec_concat(&hd_qiov, qiov, bytes_done,
+            cur_nr_sectors * 512);
+
+        switch (ret) {
+        case QCOW2_CLUSTER_UNALLOCATED:
+
+            if (bs->backing_hd) {
+                /* read from the base image */
+                n1 = qcow2_backing_read1(bs->backing_hd, &hd_qiov,
+                    sector_num, cur_nr_sectors);
+                if (n1 > 0) {
+                    BLKDBG_EVENT(bs->file, BLKDBG_READ_BACKING_AIO);
+                    qemu_co_mutex_unlock(&s->lock);
+                    ret = bdrv_co_readv(bs->backing_hd, sector_num,
+                                        n1, &hd_qiov);
+                    qemu_co_mutex_lock(&s->lock);
+                    if (ret < 0) {
+                        goto fail;
+                    }
+                }
+            } else {
+                /* Note: in this case, no need to wait */
+                qemu_iovec_memset(&hd_qiov, 0, 0, 512 * cur_nr_sectors);
+            }
+            break;
+
+        case QCOW2_CLUSTER_ZERO:
+            qemu_iovec_memset(&hd_qiov, 0, 0, 512 * cur_nr_sectors);
+            break;
+
+        case QCOW2_CLUSTER_COMPRESSED:
+            /* add AIO support for compressed blocks ? */
+            ret = qcow2_decompress_cluster(bs, cluster_offset);
+            if (ret < 0) {
+                goto fail;
+            }
+
+            qemu_iovec_from_buf(&hd_qiov, 0,
+                s->cluster_cache + index_in_cluster * 512,
+                512 * cur_nr_sectors);
+            break;
+
+        case QCOW2_CLUSTER_NORMAL:
+            if ((cluster_offset & 511) != 0) {
+                ret = -EIO;
+                goto fail;
+            }
+
+            if (s->crypt_method) {
+                /*
+                 * For encrypted images, read everything into a temporary
+                 * contiguous buffer on which the AES functions can work.
+                 */
+                if (!cluster_data) {
+                    cluster_data =
+                        qemu_blockalign(bs, QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size);
+                }
+
+                assert(cur_nr_sectors <=
+                    QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors);
+                qemu_iovec_reset(&hd_qiov);
+                qemu_iovec_add(&hd_qiov, cluster_data,
+                    512 * cur_nr_sectors);
+            }
+
+            BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
+            qemu_co_mutex_unlock(&s->lock);
+            ret = bdrv_co_readv(bs->file,
+                                (cluster_offset >> 9) + index_in_cluster,
+                                cur_nr_sectors, &hd_qiov);
+            qemu_co_mutex_lock(&s->lock);
+            if (ret < 0) {
+                goto fail;
+            }
+            if (s->crypt_method) {
+                qcow2_encrypt_sectors(s, sector_num,  cluster_data,
+                    cluster_data, cur_nr_sectors, 0, &s->aes_decrypt_key);
+                qemu_iovec_from_buf(qiov, bytes_done,
+                    cluster_data, 512 * cur_nr_sectors);
+            }
+            break;
+
+        default:
+            g_assert_not_reached();
+            ret = -EIO;
+            goto fail;
+        }
+
+        remaining_sectors -= cur_nr_sectors;
+        sector_num += cur_nr_sectors;
+        bytes_done += cur_nr_sectors * 512;
+    }
+    ret = 0;
+
+fail:
+    qemu_co_mutex_unlock(&s->lock);
+
+    qemu_iovec_destroy(&hd_qiov);
+    qemu_vfree(cluster_data);
+
+    return ret;
+}
+
+static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
+                           int64_t sector_num,
+                           int remaining_sectors,
+                           QEMUIOVector *qiov)
+{
+    BDRVQcowState *s = bs->opaque;
+    int index_in_cluster;
+    int n_end;
+    int ret;
+    int cur_nr_sectors; /* number of sectors in current iteration */
+    uint64_t cluster_offset;
+    QEMUIOVector hd_qiov;
+    uint64_t bytes_done = 0;
+    uint8_t *cluster_data = NULL;
+    QCowL2Meta *l2meta = NULL;
+
+    trace_qcow2_writev_start_req(qemu_coroutine_self(), sector_num,
+                                 remaining_sectors);
+
+    qemu_iovec_init(&hd_qiov, qiov->niov);
+
+    s->cluster_cache_offset = -1; /* disable compressed cache */
+
+    qemu_co_mutex_lock(&s->lock);
+
+    while (remaining_sectors != 0) {
+
+        l2meta = NULL;
+
+        trace_qcow2_writev_start_part(qemu_coroutine_self());
+        index_in_cluster = sector_num & (s->cluster_sectors - 1);
+        n_end = index_in_cluster + remaining_sectors;
+        if (s->crypt_method &&
+            n_end > QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors) {
+            n_end = QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors;
+        }
+
+        ret = qcow2_alloc_cluster_offset(bs, sector_num << 9,
+            index_in_cluster, n_end, &cur_nr_sectors, &cluster_offset, &l2meta);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        assert((cluster_offset & 511) == 0);
+
+        qemu_iovec_reset(&hd_qiov);
+        qemu_iovec_concat(&hd_qiov, qiov, bytes_done,
+            cur_nr_sectors * 512);
+
+        if (s->crypt_method) {
+            if (!cluster_data) {
+                cluster_data = qemu_blockalign(bs, QCOW_MAX_CRYPT_CLUSTERS *
+                                                 s->cluster_size);
+            }
+
+            assert(hd_qiov.size <=
+                   QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size);
+            qemu_iovec_to_buf(&hd_qiov, 0, cluster_data, hd_qiov.size);
+
+            qcow2_encrypt_sectors(s, sector_num, cluster_data,
+                cluster_data, cur_nr_sectors, 1, &s->aes_encrypt_key);
+
+            qemu_iovec_reset(&hd_qiov);
+            qemu_iovec_add(&hd_qiov, cluster_data,
+                cur_nr_sectors * 512);
+        }
+
+        qemu_co_mutex_unlock(&s->lock);
+        BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
+        trace_qcow2_writev_data(qemu_coroutine_self(),
+                                (cluster_offset >> 9) + index_in_cluster);
+        ret = bdrv_co_writev(bs->file,
+                             (cluster_offset >> 9) + index_in_cluster,
+                             cur_nr_sectors, &hd_qiov);
+        qemu_co_mutex_lock(&s->lock);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        while (l2meta != NULL) {
+            QCowL2Meta *next;
+
+            ret = qcow2_alloc_cluster_link_l2(bs, l2meta);
+            if (ret < 0) {
+                goto fail;
+            }
+
+            /* Take the request off the list of running requests */
+            if (l2meta->nb_clusters != 0) {
+                QLIST_REMOVE(l2meta, next_in_flight);
+            }
+
+            qemu_co_queue_restart_all(&l2meta->dependent_requests);
+
+            next = l2meta->next;
+            g_free(l2meta);
+            l2meta = next;
+        }
+
+        remaining_sectors -= cur_nr_sectors;
+        sector_num += cur_nr_sectors;
+        bytes_done += cur_nr_sectors * 512;
+        trace_qcow2_writev_done_part(qemu_coroutine_self(), cur_nr_sectors);
+    }
+    ret = 0;
+
+fail:
+    qemu_co_mutex_unlock(&s->lock);
+
+    while (l2meta != NULL) {
+        QCowL2Meta *next;
+
+        if (l2meta->nb_clusters != 0) {
+            QLIST_REMOVE(l2meta, next_in_flight);
+        }
+        qemu_co_queue_restart_all(&l2meta->dependent_requests);
+
+        next = l2meta->next;
+        g_free(l2meta);
+        l2meta = next;
+    }
+
+    qemu_iovec_destroy(&hd_qiov);
+    qemu_vfree(cluster_data);
+    trace_qcow2_writev_done_req(qemu_coroutine_self(), ret);
+
+    return ret;
+}
+
+static void qcow2_close(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    g_free(s->l1_table);
+
+    qcow2_cache_flush(bs, s->l2_table_cache);
+    qcow2_cache_flush(bs, s->refcount_block_cache);
+
+    qcow2_mark_clean(bs);
+
+    qcow2_cache_destroy(bs, s->l2_table_cache);
+    qcow2_cache_destroy(bs, s->refcount_block_cache);
+
+    g_free(s->unknown_header_fields);
+    cleanup_unknown_header_ext(bs);
+
+    g_free(s->cluster_cache);
+    qemu_vfree(s->cluster_data);
+    qcow2_refcount_close(bs);
+    qcow2_free_snapshots(bs);
+}
+
+static void qcow2_invalidate_cache(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    int flags = s->flags;
+    AES_KEY aes_encrypt_key;
+    AES_KEY aes_decrypt_key;
+    uint32_t crypt_method = 0;
+    QDict *options;
+
+    /*
+     * Backing files are read-only which makes all of their metadata immutable,
+     * that means we don't have to worry about reopening them here.
+     */
+
+    if (s->crypt_method) {
+        crypt_method = s->crypt_method;
+        memcpy(&aes_encrypt_key, &s->aes_encrypt_key, sizeof(aes_encrypt_key));
+        memcpy(&aes_decrypt_key, &s->aes_decrypt_key, sizeof(aes_decrypt_key));
+    }
+
+    qcow2_close(bs);
+
+    options = qdict_new();
+    qdict_put(options, QCOW2_OPT_LAZY_REFCOUNTS,
+              qbool_from_int(s->use_lazy_refcounts));
+
+    memset(s, 0, sizeof(BDRVQcowState));
+    qcow2_open(bs, options, flags);
+
+    QDECREF(options);
+
+    if (crypt_method) {
+        s->crypt_method = crypt_method;
+        memcpy(&s->aes_encrypt_key, &aes_encrypt_key, sizeof(aes_encrypt_key));
+        memcpy(&s->aes_decrypt_key, &aes_decrypt_key, sizeof(aes_decrypt_key));
+    }
+}
+
+static size_t header_ext_add(char *buf, uint32_t magic, const void *s,
+    size_t len, size_t buflen)
+{
+    QCowExtension *ext_backing_fmt = (QCowExtension*) buf;
+    size_t ext_len = sizeof(QCowExtension) + ((len + 7) & ~7);
+
+    if (buflen < ext_len) {
+        return -ENOSPC;
+    }
+
+    *ext_backing_fmt = (QCowExtension) {
+        .magic  = cpu_to_be32(magic),
+        .len    = cpu_to_be32(len),
+    };
+    memcpy(buf + sizeof(QCowExtension), s, len);
+
+    return ext_len;
+}
+
+/*
+ * Updates the qcow2 header, including the variable length parts of it, i.e.
+ * the backing file name and all extensions. qcow2 was not designed to allow
+ * such changes, so if we run out of space (we can only use the first cluster)
+ * this function may fail.
+ *
+ * Returns 0 on success, -errno in error cases.
+ */
+int qcow2_update_header(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    QCowHeader *header;
+    char *buf;
+    size_t buflen = s->cluster_size;
+    int ret;
+    uint64_t total_size;
+    uint32_t refcount_table_clusters;
+    size_t header_length;
+    Qcow2UnknownHeaderExtension *uext;
+
+    buf = qemu_blockalign(bs, buflen);
+
+    /* Header structure */
+    header = (QCowHeader*) buf;
+
+    if (buflen < sizeof(*header)) {
+        ret = -ENOSPC;
+        goto fail;
+    }
+
+    header_length = sizeof(*header) + s->unknown_header_fields_size;
+    total_size = bs->total_sectors * BDRV_SECTOR_SIZE;
+    refcount_table_clusters = s->refcount_table_size >> (s->cluster_bits - 3);
+
+    *header = (QCowHeader) {
+        /* Version 2 fields */
+        .magic                  = cpu_to_be32(QCOW_MAGIC),
+        .version                = cpu_to_be32(s->qcow_version),
+        .backing_file_offset    = 0,
+        .backing_file_size      = 0,
+        .cluster_bits           = cpu_to_be32(s->cluster_bits),
+        .size                   = cpu_to_be64(total_size),
+        .crypt_method           = cpu_to_be32(s->crypt_method_header),
+        .l1_size                = cpu_to_be32(s->l1_size),
+        .l1_table_offset        = cpu_to_be64(s->l1_table_offset),
+        .refcount_table_offset  = cpu_to_be64(s->refcount_table_offset),
+        .refcount_table_clusters = cpu_to_be32(refcount_table_clusters),
+        .nb_snapshots           = cpu_to_be32(s->nb_snapshots),
+        .snapshots_offset       = cpu_to_be64(s->snapshots_offset),
+
+        /* Version 3 fields */
+        .incompatible_features  = cpu_to_be64(s->incompatible_features),
+        .compatible_features    = cpu_to_be64(s->compatible_features),
+        .autoclear_features     = cpu_to_be64(s->autoclear_features),
+        .refcount_order         = cpu_to_be32(3 + REFCOUNT_SHIFT),
+        .header_length          = cpu_to_be32(header_length),
+    };
+
+    /* For older versions, write a shorter header */
+    switch (s->qcow_version) {
+    case 2:
+        ret = offsetof(QCowHeader, incompatible_features);
+        break;
+    case 3:
+        ret = sizeof(*header);
+        break;
+    default:
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    buf += ret;
+    buflen -= ret;
+    memset(buf, 0, buflen);
+
+    /* Preserve any unknown field in the header */
+    if (s->unknown_header_fields_size) {
+        if (buflen < s->unknown_header_fields_size) {
+            ret = -ENOSPC;
+            goto fail;
+        }
+
+        memcpy(buf, s->unknown_header_fields, s->unknown_header_fields_size);
+        buf += s->unknown_header_fields_size;
+        buflen -= s->unknown_header_fields_size;
+    }
+
+    /* Backing file format header extension */
+    if (*bs->backing_format) {
+        ret = header_ext_add(buf, QCOW2_EXT_MAGIC_BACKING_FORMAT,
+                             bs->backing_format, strlen(bs->backing_format),
+                             buflen);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        buf += ret;
+        buflen -= ret;
+    }
+
+    /* Feature table */
+    Qcow2Feature features[] = {
+        {
+            .type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
+            .bit  = QCOW2_INCOMPAT_DIRTY_BITNR,
+            .name = "dirty bit",
+        },
+        {
+            .type = QCOW2_FEAT_TYPE_COMPATIBLE,
+            .bit  = QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR,
+            .name = "lazy refcounts",
+        },
+    };
+
+    ret = header_ext_add(buf, QCOW2_EXT_MAGIC_FEATURE_TABLE,
+                         features, sizeof(features), buflen);
+    if (ret < 0) {
+        goto fail;
+    }
+    buf += ret;
+    buflen -= ret;
+
+    /* Keep unknown header extensions */
+    QLIST_FOREACH(uext, &s->unknown_header_ext, next) {
+        ret = header_ext_add(buf, uext->magic, uext->data, uext->len, buflen);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        buf += ret;
+        buflen -= ret;
+    }
+
+    /* End of header extensions */
+    ret = header_ext_add(buf, QCOW2_EXT_MAGIC_END, NULL, 0, buflen);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    buf += ret;
+    buflen -= ret;
+
+    /* Backing file name */
+    if (*bs->backing_file) {
+        size_t backing_file_len = strlen(bs->backing_file);
+
+        if (buflen < backing_file_len) {
+            ret = -ENOSPC;
+            goto fail;
+        }
+
+        /* Using strncpy is ok here, since buf is not NUL-terminated. */
+        strncpy(buf, bs->backing_file, buflen);
+
+        header->backing_file_offset = cpu_to_be64(buf - ((char*) header));
+        header->backing_file_size   = cpu_to_be32(backing_file_len);
+    }
+
+    /* Write the new header */
+    ret = bdrv_pwrite(bs->file, 0, header, s->cluster_size);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    ret = 0;
+fail:
+    qemu_vfree(header);
+    return ret;
+}
+
+static int qcow2_change_backing_file(BlockDriverState *bs,
+    const char *backing_file, const char *backing_fmt)
+{
+    pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_file ?: "");
+    pstrcpy(bs->backing_format, sizeof(bs->backing_format), backing_fmt ?: "");
+
+    return qcow2_update_header(bs);
+}
+
+static int preallocate(BlockDriverState *bs)
+{
+    uint64_t nb_sectors;
+    uint64_t offset;
+    uint64_t host_offset = 0;
+    int num;
+    int ret;
+    QCowL2Meta *meta;
+
+    nb_sectors = bdrv_getlength(bs) >> 9;
+    offset = 0;
+
+    while (nb_sectors) {
+        num = MIN(nb_sectors, INT_MAX >> 9);
+        ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num,
+                                         &host_offset, &meta);
+        if (ret < 0) {
+            return ret;
+        }
+
+        ret = qcow2_alloc_cluster_link_l2(bs, meta);
+        if (ret < 0) {
+            qcow2_free_any_clusters(bs, meta->alloc_offset, meta->nb_clusters,
+                                    QCOW2_DISCARD_NEVER);
+            return ret;
+        }
+
+        /* There are no dependent requests, but we need to remove our request
+         * from the list of in-flight requests */
+        if (meta != NULL) {
+            QLIST_REMOVE(meta, next_in_flight);
+        }
+
+        /* TODO Preallocate data if requested */
+
+        nb_sectors -= num;
+        offset += num << 9;
+    }
+
+    /*
+     * It is expected that the image file is large enough to actually contain
+     * all of the allocated clusters (otherwise we get failing reads after
+     * EOF). Extend the image to the last allocated sector.
+     */
+    if (host_offset != 0) {
+        uint8_t buf[512];
+        memset(buf, 0, 512);
+        ret = bdrv_write(bs->file, (host_offset >> 9) + num - 1, buf, 1);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static int qcow2_create2(const char *filename, int64_t total_size,
+                         const char *backing_file, const char *backing_format,
+                         int flags, size_t cluster_size, int prealloc,
+                         QEMUOptionParameter *options, int version)
+{
+    /* Calculate cluster_bits */
+    int cluster_bits;
+    cluster_bits = ffs(cluster_size) - 1;
+    if (cluster_bits < MIN_CLUSTER_BITS || cluster_bits > MAX_CLUSTER_BITS ||
+        (1 << cluster_bits) != cluster_size)
+    {
+        error_report(
+            "Cluster size must be a power of two between %d and %dk",
+            1 << MIN_CLUSTER_BITS, 1 << (MAX_CLUSTER_BITS - 10));
+        return -EINVAL;
+    }
+
+    /*
+     * Open the image file and write a minimal qcow2 header.
+     *
+     * We keep things simple and start with a zero-sized image. We also
+     * do without refcount blocks or a L1 table for now. We'll fix the
+     * inconsistency later.
+     *
+     * We do need a refcount table because growing the refcount table means
+     * allocating two new refcount blocks - the seconds of which would be at
+     * 2 GB for 64k clusters, and we don't want to have a 2 GB initial file
+     * size for any qcow2 image.
+     */
+    BlockDriverState* bs;
+    QCowHeader header;
+    uint8_t* refcount_table;
+    int ret;
+
+    ret = bdrv_create_file(filename, options);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = bdrv_file_open(&bs, filename, NULL, BDRV_O_RDWR);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* Write the header */
+    memset(&header, 0, sizeof(header));
+    header.magic = cpu_to_be32(QCOW_MAGIC);
+    header.version = cpu_to_be32(version);
+    header.cluster_bits = cpu_to_be32(cluster_bits);
+    header.size = cpu_to_be64(0);
+    header.l1_table_offset = cpu_to_be64(0);
+    header.l1_size = cpu_to_be32(0);
+    header.refcount_table_offset = cpu_to_be64(cluster_size);
+    header.refcount_table_clusters = cpu_to_be32(1);
+    header.refcount_order = cpu_to_be32(3 + REFCOUNT_SHIFT);
+    header.header_length = cpu_to_be32(sizeof(header));
+
+    if (flags & BLOCK_FLAG_ENCRYPT) {
+        header.crypt_method = cpu_to_be32(QCOW_CRYPT_AES);
+    } else {
+        header.crypt_method = cpu_to_be32(QCOW_CRYPT_NONE);
+    }
+
+    if (flags & BLOCK_FLAG_LAZY_REFCOUNTS) {
+        header.compatible_features |=
+            cpu_to_be64(QCOW2_COMPAT_LAZY_REFCOUNTS);
+    }
+
+    ret = bdrv_pwrite(bs, 0, &header, sizeof(header));
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* Write an empty refcount table */
+    refcount_table = g_malloc0(cluster_size);
+    ret = bdrv_pwrite(bs, cluster_size, refcount_table, cluster_size);
+    g_free(refcount_table);
+
+    if (ret < 0) {
+        goto out;
+    }
+
+    bdrv_close(bs);
+
+    /*
+     * And now open the image and make it consistent first (i.e. increase the
+     * refcount of the cluster that is occupied by the header and the refcount
+     * table)
+     */
+    BlockDriver* drv = bdrv_find_format("qcow2");
+    assert(drv != NULL);
+    ret = bdrv_open(bs, filename, NULL,
+        BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH, drv);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = qcow2_alloc_clusters(bs, 2 * cluster_size);
+    if (ret < 0) {
+        goto out;
+
+    } else if (ret != 0) {
+        error_report("Huh, first cluster in empty image is already in use?");
+        abort();
+    }
+
+    /* Okay, now that we have a valid image, let's give it the right size */
+    ret = bdrv_truncate(bs, total_size * BDRV_SECTOR_SIZE);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* Want a backing file? There you go.*/
+    if (backing_file) {
+        ret = bdrv_change_backing_file(bs, backing_file, backing_format);
+        if (ret < 0) {
+            goto out;
+        }
+    }
+
+    /* And if we're supposed to preallocate metadata, do that now */
+    if (prealloc) {
+        BDRVQcowState *s = bs->opaque;
+        qemu_co_mutex_lock(&s->lock);
+        ret = preallocate(bs);
+        qemu_co_mutex_unlock(&s->lock);
+        if (ret < 0) {
+            goto out;
+        }
+    }
+
+    ret = 0;
+out:
+    bdrv_delete(bs);
+    return ret;
+}
+
+static int qcow2_create(const char *filename, QEMUOptionParameter *options)
+{
+    const char *backing_file = NULL;
+    const char *backing_fmt = NULL;
+    uint64_t sectors = 0;
+    int flags = 0;
+    size_t cluster_size = DEFAULT_CLUSTER_SIZE;
+    int prealloc = 0;
+    int version = 2;
+
+    /* Read out options */
+    while (options && options->name) {
+        if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
+            sectors = options->value.n / 512;
+        } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FILE)) {
+            backing_file = options->value.s;
+        } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FMT)) {
+            backing_fmt = options->value.s;
+        } else if (!strcmp(options->name, BLOCK_OPT_ENCRYPT)) {
+            flags |= options->value.n ? BLOCK_FLAG_ENCRYPT : 0;
+        } else if (!strcmp(options->name, BLOCK_OPT_CLUSTER_SIZE)) {
+            if (options->value.n) {
+                cluster_size = options->value.n;
+            }
+        } else if (!strcmp(options->name, BLOCK_OPT_PREALLOC)) {
+            if (!options->value.s || !strcmp(options->value.s, "off")) {
+                prealloc = 0;
+            } else if (!strcmp(options->value.s, "metadata")) {
+                prealloc = 1;
+            } else {
+                fprintf(stderr, "Invalid preallocation mode: '%s'\n",
+                    options->value.s);
+                return -EINVAL;
+            }
+        } else if (!strcmp(options->name, BLOCK_OPT_COMPAT_LEVEL)) {
+            if (!options->value.s || !strcmp(options->value.s, "0.10")) {
+                version = 2;
+            } else if (!strcmp(options->value.s, "1.1")) {
+                version = 3;
+            } else {
+                fprintf(stderr, "Invalid compatibility level: '%s'\n",
+                    options->value.s);
+                return -EINVAL;
+            }
+        } else if (!strcmp(options->name, BLOCK_OPT_LAZY_REFCOUNTS)) {
+            flags |= options->value.n ? BLOCK_FLAG_LAZY_REFCOUNTS : 0;
+        }
+        options++;
+    }
+
+    if (backing_file && prealloc) {
+        fprintf(stderr, "Backing file and preallocation cannot be used at "
+            "the same time\n");
+        return -EINVAL;
+    }
+
+    if (version < 3 && (flags & BLOCK_FLAG_LAZY_REFCOUNTS)) {
+        fprintf(stderr, "Lazy refcounts only supported with compatibility "
+                "level 1.1 and above (use compat=1.1 or greater)\n");
+        return -EINVAL;
+    }
+
+    return qcow2_create2(filename, sectors, backing_file, backing_fmt, flags,
+                         cluster_size, prealloc, options, version);
+}
+
+static int qcow2_make_empty(BlockDriverState *bs)
+{
+#if 0
+    /* XXX: not correct */
+    BDRVQcowState *s = bs->opaque;
+    uint32_t l1_length = s->l1_size * sizeof(uint64_t);
+    int ret;
+
+    memset(s->l1_table, 0, l1_length);
+    if (bdrv_pwrite(bs->file, s->l1_table_offset, s->l1_table, l1_length) < 0)
+        return -1;
+    ret = bdrv_truncate(bs->file, s->l1_table_offset + l1_length);
+    if (ret < 0)
+        return ret;
+
+    l2_cache_reset(bs);
+#endif
+    return 0;
+}
+
+static coroutine_fn int qcow2_co_write_zeroes(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors)
+{
+    int ret;
+    BDRVQcowState *s = bs->opaque;
+
+    /* Emulate misaligned zero writes */
+    if (sector_num % s->cluster_sectors || nb_sectors % s->cluster_sectors) {
+        return -ENOTSUP;
+    }
+
+    /* Whatever is left can use real zero clusters */
+    qemu_co_mutex_lock(&s->lock);
+    ret = qcow2_zero_clusters(bs, sector_num << BDRV_SECTOR_BITS,
+        nb_sectors);
+    qemu_co_mutex_unlock(&s->lock);
+
+    return ret;
+}
+
+static coroutine_fn int qcow2_co_discard(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors)
+{
+    int ret;
+    BDRVQcowState *s = bs->opaque;
+
+    qemu_co_mutex_lock(&s->lock);
+    ret = qcow2_discard_clusters(bs, sector_num << BDRV_SECTOR_BITS,
+        nb_sectors);
+    qemu_co_mutex_unlock(&s->lock);
+    return ret;
+}
+
+static int qcow2_truncate(BlockDriverState *bs, int64_t offset)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t new_l1_size;
+    int ret;
+
+    if (offset & 511) {
+        error_report("The new size must be a multiple of 512");
+        return -EINVAL;
+    }
+
+    /* cannot proceed if image has snapshots */
+    if (s->nb_snapshots) {
+        error_report("Can't resize an image which has snapshots");
+        return -ENOTSUP;
+    }
+
+    /* shrinking is currently not supported */
+    if (offset < bs->total_sectors * 512) {
+        error_report("qcow2 doesn't support shrinking images yet");
+        return -ENOTSUP;
+    }
+
+    new_l1_size = size_to_l1(s, offset);
+    ret = qcow2_grow_l1_table(bs, new_l1_size, true);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* write updated header.size */
+    offset = cpu_to_be64(offset);
+    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, size),
+                           &offset, sizeof(uint64_t));
+    if (ret < 0) {
+        return ret;
+    }
+
+    s->l1_vm_state_index = new_l1_size;
+    return 0;
+}
+
+/* XXX: put compressed sectors first, then all the cluster aligned
+   tables to avoid losing bytes in alignment */
+static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
+                                  const uint8_t *buf, int nb_sectors)
+{
+    BDRVQcowState *s = bs->opaque;
+    z_stream strm;
+    int ret, out_len;
+    uint8_t *out_buf;
+    uint64_t cluster_offset;
+
+    if (nb_sectors == 0) {
+        /* align end of file to a sector boundary to ease reading with
+           sector based I/Os */
+        cluster_offset = bdrv_getlength(bs->file);
+        cluster_offset = (cluster_offset + 511) & ~511;
+        bdrv_truncate(bs->file, cluster_offset);
+        return 0;
+    }
+
+    if (nb_sectors != s->cluster_sectors) {
+        ret = -EINVAL;
+
+        /* Zero-pad last write if image size is not cluster aligned */
+        if (sector_num + nb_sectors == bs->total_sectors &&
+            nb_sectors < s->cluster_sectors) {
+            uint8_t *pad_buf = qemu_blockalign(bs, s->cluster_size);
+            memset(pad_buf, 0, s->cluster_size);
+            memcpy(pad_buf, buf, nb_sectors * BDRV_SECTOR_SIZE);
+            ret = qcow2_write_compressed(bs, sector_num,
+                                         pad_buf, s->cluster_sectors);
+            qemu_vfree(pad_buf);
+        }
+        return ret;
+    }
+
+    out_buf = g_malloc(s->cluster_size + (s->cluster_size / 1000) + 128);
+
+    /* best compression, small window, no zlib header */
+    memset(&strm, 0, sizeof(strm));
+    ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
+                       Z_DEFLATED, -12,
+                       9, Z_DEFAULT_STRATEGY);
+    if (ret != 0) {
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    strm.avail_in = s->cluster_size;
+    strm.next_in = (uint8_t *)buf;
+    strm.avail_out = s->cluster_size;
+    strm.next_out = out_buf;
+
+    ret = deflate(&strm, Z_FINISH);
+    if (ret != Z_STREAM_END && ret != Z_OK) {
+        deflateEnd(&strm);
+        ret = -EINVAL;
+        goto fail;
+    }
+    out_len = strm.next_out - out_buf;
+
+    deflateEnd(&strm);
+
+    if (ret != Z_STREAM_END || out_len >= s->cluster_size) {
+        /* could not compress: write normal cluster */
+        ret = bdrv_write(bs, sector_num, buf, s->cluster_sectors);
+        if (ret < 0) {
+            goto fail;
+        }
+    } else {
+        cluster_offset = qcow2_alloc_compressed_cluster_offset(bs,
+            sector_num << 9, out_len);
+        if (!cluster_offset) {
+            ret = -EIO;
+            goto fail;
+        }
+        cluster_offset &= s->cluster_offset_mask;
+        BLKDBG_EVENT(bs->file, BLKDBG_WRITE_COMPRESSED);
+        ret = bdrv_pwrite(bs->file, cluster_offset, out_buf, out_len);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+    ret = 0;
+fail:
+    g_free(out_buf);
+    return ret;
+}
+
+static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret;
+
+    qemu_co_mutex_lock(&s->lock);
+    ret = qcow2_cache_flush(bs, s->l2_table_cache);
+    if (ret < 0) {
+        qemu_co_mutex_unlock(&s->lock);
+        return ret;
+    }
+
+    if (qcow2_need_accurate_refcounts(s)) {
+        ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+        if (ret < 0) {
+            qemu_co_mutex_unlock(&s->lock);
+            return ret;
+        }
+    }
+    qemu_co_mutex_unlock(&s->lock);
+
+    return 0;
+}
+
+static int64_t qcow2_vm_state_offset(BDRVQcowState *s)
+{
+	return (int64_t)s->l1_vm_state_index << (s->cluster_bits + s->l2_bits);
+}
+
+static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
+{
+    BDRVQcowState *s = bs->opaque;
+    bdi->cluster_size = s->cluster_size;
+    bdi->vm_state_offset = qcow2_vm_state_offset(s);
+    return 0;
+}
+
+#if 0
+static void dump_refcounts(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t nb_clusters, k, k1, size;
+    int refcount;
+
+    size = bdrv_getlength(bs->file);
+    nb_clusters = size_to_clusters(s, size);
+    for(k = 0; k < nb_clusters;) {
+        k1 = k;
+        refcount = get_refcount(bs, k);
+        k++;
+        while (k < nb_clusters && get_refcount(bs, k) == refcount)
+            k++;
+        printf("%" PRId64 ": refcount=%d nb=%" PRId64 "\n", k, refcount,
+               k - k1);
+    }
+}
+#endif
+
+static int qcow2_save_vmstate(BlockDriverState *bs, QEMUIOVector *qiov,
+                              int64_t pos)
+{
+    BDRVQcowState *s = bs->opaque;
+    int growable = bs->growable;
+    int ret;
+
+    BLKDBG_EVENT(bs->file, BLKDBG_VMSTATE_SAVE);
+    bs->growable = 1;
+    ret = bdrv_pwritev(bs, qcow2_vm_state_offset(s) + pos, qiov);
+    bs->growable = growable;
+
+    return ret;
+}
+
+static int qcow2_load_vmstate(BlockDriverState *bs, uint8_t *buf,
+                              int64_t pos, int size)
+{
+    BDRVQcowState *s = bs->opaque;
+    int growable = bs->growable;
+    int ret;
+
+    BLKDBG_EVENT(bs->file, BLKDBG_VMSTATE_LOAD);
+    bs->growable = 1;
+    ret = bdrv_pread(bs, qcow2_vm_state_offset(s) + pos, buf, size);
+    bs->growable = growable;
+
+    return ret;
+}
+
+static QEMUOptionParameter qcow2_create_options[] = {
+    {
+        .name = BLOCK_OPT_SIZE,
+        .type = OPT_SIZE,
+        .help = "Virtual disk size"
+    },
+    {
+        .name = BLOCK_OPT_COMPAT_LEVEL,
+        .type = OPT_STRING,
+        .help = "Compatibility level (0.10 or 1.1)"
+    },
+    {
+        .name = BLOCK_OPT_BACKING_FILE,
+        .type = OPT_STRING,
+        .help = "File name of a base image"
+    },
+    {
+        .name = BLOCK_OPT_BACKING_FMT,
+        .type = OPT_STRING,
+        .help = "Image format of the base image"
+    },
+    {
+        .name = BLOCK_OPT_ENCRYPT,
+        .type = OPT_FLAG,
+        .help = "Encrypt the image"
+    },
+    {
+        .name = BLOCK_OPT_CLUSTER_SIZE,
+        .type = OPT_SIZE,
+        .help = "qcow2 cluster size",
+        .value = { .n = DEFAULT_CLUSTER_SIZE },
+    },
+    {
+        .name = BLOCK_OPT_PREALLOC,
+        .type = OPT_STRING,
+        .help = "Preallocation mode (allowed values: off, metadata)"
+    },
+    {
+        .name = BLOCK_OPT_LAZY_REFCOUNTS,
+        .type = OPT_FLAG,
+        .help = "Postpone refcount updates",
+    },
+    { NULL }
+};
+
+static BlockDriver bdrv_qcow2 = {
+    .format_name        = "qcow2",
+    .instance_size      = sizeof(BDRVQcowState),
+    .bdrv_probe         = qcow2_probe,
+    .bdrv_open          = qcow2_open,
+    .bdrv_close         = qcow2_close,
+    .bdrv_reopen_prepare  = qcow2_reopen_prepare,
+    .bdrv_create        = qcow2_create,
+    .bdrv_has_zero_init = bdrv_has_zero_init_1,
+    .bdrv_co_is_allocated = qcow2_co_is_allocated,
+    .bdrv_set_key       = qcow2_set_key,
+    .bdrv_make_empty    = qcow2_make_empty,
+
+    .bdrv_co_readv          = qcow2_co_readv,
+    .bdrv_co_writev         = qcow2_co_writev,
+    .bdrv_co_flush_to_os    = qcow2_co_flush_to_os,
+
+    .bdrv_co_write_zeroes   = qcow2_co_write_zeroes,
+    .bdrv_co_discard        = qcow2_co_discard,
+    .bdrv_truncate          = qcow2_truncate,
+    .bdrv_write_compressed  = qcow2_write_compressed,
+
+    .bdrv_snapshot_create   = qcow2_snapshot_create,
+    .bdrv_snapshot_goto     = qcow2_snapshot_goto,
+    .bdrv_snapshot_delete   = qcow2_snapshot_delete,
+    .bdrv_snapshot_list     = qcow2_snapshot_list,
+    .bdrv_snapshot_load_tmp     = qcow2_snapshot_load_tmp,
+    .bdrv_get_info      = qcow2_get_info,
+
+    .bdrv_save_vmstate    = qcow2_save_vmstate,
+    .bdrv_load_vmstate    = qcow2_load_vmstate,
+
+    .bdrv_change_backing_file   = qcow2_change_backing_file,
+
+    .bdrv_invalidate_cache      = qcow2_invalidate_cache,
+
+    .create_options = qcow2_create_options,
+    .bdrv_check = qcow2_check,
+};
+
+static void bdrv_qcow2_init(void)
+{
+    bdrv_register(&bdrv_qcow2);
+}
+
+block_init(bdrv_qcow2_init);
diff --git a/contrib/qemu/block/qcow2.h b/contrib/qemu/block/qcow2.h
new file mode 100644
index 000000000..3b2d5cda7
--- /dev/null
+++ b/contrib/qemu/block/qcow2.h
@@ -0,0 +1,437 @@
+/*
+ * Block driver for the QCOW version 2 format
+ *
+ * Copyright (c) 2004-2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef BLOCK_QCOW2_H
+#define BLOCK_QCOW2_H
+
+#include "qemu/aes.h"
+#include "block/coroutine.h"
+
+//#define DEBUG_ALLOC
+//#define DEBUG_ALLOC2
+//#define DEBUG_EXT
+
+#define QCOW_MAGIC (('Q' << 24) | ('F' << 16) | ('I' << 8) | 0xfb)
+
+#define QCOW_CRYPT_NONE 0
+#define QCOW_CRYPT_AES  1
+
+#define QCOW_MAX_CRYPT_CLUSTERS 32
+
+/* indicate that the refcount of the referenced cluster is exactly one. */
+#define QCOW_OFLAG_COPIED     (1LL << 63)
+/* indicate that the cluster is compressed (they never have the copied flag) */
+#define QCOW_OFLAG_COMPRESSED (1LL << 62)
+/* The cluster reads as all zeros */
+#define QCOW_OFLAG_ZERO (1LL << 0)
+
+#define REFCOUNT_SHIFT 1 /* refcount size is 2 bytes */
+
+#define MIN_CLUSTER_BITS 9
+#define MAX_CLUSTER_BITS 21
+
+#define L2_CACHE_SIZE 16
+
+/* Must be at least 4 to cover all cases of refcount table growth */
+#define REFCOUNT_CACHE_SIZE 4
+
+#define DEFAULT_CLUSTER_SIZE 65536
+
+
+#define QCOW2_OPT_LAZY_REFCOUNTS "lazy_refcounts"
+#define QCOW2_OPT_DISCARD_REQUEST "pass_discard_request"
+#define QCOW2_OPT_DISCARD_SNAPSHOT "pass_discard_snapshot"
+#define QCOW2_OPT_DISCARD_OTHER "pass_discard_other"
+
+typedef struct QCowHeader {
+    uint32_t magic;
+    uint32_t version;
+    uint64_t backing_file_offset;
+    uint32_t backing_file_size;
+    uint32_t cluster_bits;
+    uint64_t size; /* in bytes */
+    uint32_t crypt_method;
+    uint32_t l1_size; /* XXX: save number of clusters instead ? */
+    uint64_t l1_table_offset;
+    uint64_t refcount_table_offset;
+    uint32_t refcount_table_clusters;
+    uint32_t nb_snapshots;
+    uint64_t snapshots_offset;
+
+    /* The following fields are only valid for version >= 3 */
+    uint64_t incompatible_features;
+    uint64_t compatible_features;
+    uint64_t autoclear_features;
+
+    uint32_t refcount_order;
+    uint32_t header_length;
+} QCowHeader;
+
+typedef struct QCowSnapshot {
+    uint64_t l1_table_offset;
+    uint32_t l1_size;
+    char *id_str;
+    char *name;
+    uint64_t disk_size;
+    uint64_t vm_state_size;
+    uint32_t date_sec;
+    uint32_t date_nsec;
+    uint64_t vm_clock_nsec;
+} QCowSnapshot;
+
+struct Qcow2Cache;
+typedef struct Qcow2Cache Qcow2Cache;
+
+typedef struct Qcow2UnknownHeaderExtension {
+    uint32_t magic;
+    uint32_t len;
+    QLIST_ENTRY(Qcow2UnknownHeaderExtension) next;
+    uint8_t data[];
+} Qcow2UnknownHeaderExtension;
+
+enum {
+    QCOW2_FEAT_TYPE_INCOMPATIBLE    = 0,
+    QCOW2_FEAT_TYPE_COMPATIBLE      = 1,
+    QCOW2_FEAT_TYPE_AUTOCLEAR       = 2,
+};
+
+/* Incompatible feature bits */
+enum {
+    QCOW2_INCOMPAT_DIRTY_BITNR   = 0,
+    QCOW2_INCOMPAT_DIRTY         = 1 << QCOW2_INCOMPAT_DIRTY_BITNR,
+
+    QCOW2_INCOMPAT_MASK          = QCOW2_INCOMPAT_DIRTY,
+};
+
+/* Compatible feature bits */
+enum {
+    QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR = 0,
+    QCOW2_COMPAT_LAZY_REFCOUNTS       = 1 << QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR,
+
+    QCOW2_COMPAT_FEAT_MASK            = QCOW2_COMPAT_LAZY_REFCOUNTS,
+};
+
+enum qcow2_discard_type {
+    QCOW2_DISCARD_NEVER = 0,
+    QCOW2_DISCARD_ALWAYS,
+    QCOW2_DISCARD_REQUEST,
+    QCOW2_DISCARD_SNAPSHOT,
+    QCOW2_DISCARD_OTHER,
+    QCOW2_DISCARD_MAX
+};
+
+typedef struct Qcow2Feature {
+    uint8_t type;
+    uint8_t bit;
+    char    name[46];
+} QEMU_PACKED Qcow2Feature;
+
+typedef struct Qcow2DiscardRegion {
+    BlockDriverState *bs;
+    uint64_t offset;
+    uint64_t bytes;
+    QTAILQ_ENTRY(Qcow2DiscardRegion) next;
+} Qcow2DiscardRegion;
+
+typedef struct BDRVQcowState {
+    int cluster_bits;
+    int cluster_size;
+    int cluster_sectors;
+    int l2_bits;
+    int l2_size;
+    int l1_size;
+    int l1_vm_state_index;
+    int csize_shift;
+    int csize_mask;
+    uint64_t cluster_offset_mask;
+    uint64_t l1_table_offset;
+    uint64_t *l1_table;
+
+    Qcow2Cache* l2_table_cache;
+    Qcow2Cache* refcount_block_cache;
+
+    uint8_t *cluster_cache;
+    uint8_t *cluster_data;
+    uint64_t cluster_cache_offset;
+    QLIST_HEAD(QCowClusterAlloc, QCowL2Meta) cluster_allocs;
+
+    uint64_t *refcount_table;
+    uint64_t refcount_table_offset;
+    uint32_t refcount_table_size;
+    int64_t free_cluster_index;
+    int64_t free_byte_offset;
+
+    CoMutex lock;
+
+    uint32_t crypt_method; /* current crypt method, 0 if no key yet */
+    uint32_t crypt_method_header;
+    AES_KEY aes_encrypt_key;
+    AES_KEY aes_decrypt_key;
+    uint64_t snapshots_offset;
+    int snapshots_size;
+    int nb_snapshots;
+    QCowSnapshot *snapshots;
+
+    int flags;
+    int qcow_version;
+    bool use_lazy_refcounts;
+
+    bool discard_passthrough[QCOW2_DISCARD_MAX];
+
+    uint64_t incompatible_features;
+    uint64_t compatible_features;
+    uint64_t autoclear_features;
+
+    size_t unknown_header_fields_size;
+    void* unknown_header_fields;
+    QLIST_HEAD(, Qcow2UnknownHeaderExtension) unknown_header_ext;
+    QTAILQ_HEAD (, Qcow2DiscardRegion) discards;
+    bool cache_discards;
+} BDRVQcowState;
+
+/* XXX: use std qcow open function ? */
+typedef struct QCowCreateState {
+    int cluster_size;
+    int cluster_bits;
+    uint16_t *refcount_block;
+    uint64_t *refcount_table;
+    int64_t l1_table_offset;
+    int64_t refcount_table_offset;
+    int64_t refcount_block_offset;
+} QCowCreateState;
+
+struct QCowAIOCB;
+
+typedef struct Qcow2COWRegion {
+    /**
+     * Offset of the COW region in bytes from the start of the first cluster
+     * touched by the request.
+     */
+    uint64_t    offset;
+
+    /** Number of sectors to copy */
+    int         nb_sectors;
+} Qcow2COWRegion;
+
+/**
+ * Describes an in-flight (part of a) write request that writes to clusters
+ * that are not referenced in their L2 table yet.
+ */
+typedef struct QCowL2Meta
+{
+    /** Guest offset of the first newly allocated cluster */
+    uint64_t offset;
+
+    /** Host offset of the first newly allocated cluster */
+    uint64_t alloc_offset;
+
+    /**
+     * Number of sectors from the start of the first allocated cluster to
+     * the end of the (possibly shortened) request
+     */
+    int nb_available;
+
+    /** Number of newly allocated clusters */
+    int nb_clusters;
+
+    /**
+     * Requests that overlap with this allocation and wait to be restarted
+     * when the allocating request has completed.
+     */
+    CoQueue dependent_requests;
+
+    /**
+     * The COW Region between the start of the first allocated cluster and the
+     * area the guest actually writes to.
+     */
+    Qcow2COWRegion cow_start;
+
+    /**
+     * The COW Region between the area the guest actually writes to and the
+     * end of the last allocated cluster.
+     */
+    Qcow2COWRegion cow_end;
+
+    /** Pointer to next L2Meta of the same write request */
+    struct QCowL2Meta *next;
+
+    QLIST_ENTRY(QCowL2Meta) next_in_flight;
+} QCowL2Meta;
+
+enum {
+    QCOW2_CLUSTER_UNALLOCATED,
+    QCOW2_CLUSTER_NORMAL,
+    QCOW2_CLUSTER_COMPRESSED,
+    QCOW2_CLUSTER_ZERO
+};
+
+#define L1E_OFFSET_MASK 0x00ffffffffffff00ULL
+#define L2E_OFFSET_MASK 0x00ffffffffffff00ULL
+#define L2E_COMPRESSED_OFFSET_SIZE_MASK 0x3fffffffffffffffULL
+
+#define REFT_OFFSET_MASK 0xffffffffffffff00ULL
+
+static inline int64_t start_of_cluster(BDRVQcowState *s, int64_t offset)
+{
+    return offset & ~(s->cluster_size - 1);
+}
+
+static inline int64_t offset_into_cluster(BDRVQcowState *s, int64_t offset)
+{
+    return offset & (s->cluster_size - 1);
+}
+
+static inline int size_to_clusters(BDRVQcowState *s, int64_t size)
+{
+    return (size + (s->cluster_size - 1)) >> s->cluster_bits;
+}
+
+static inline int64_t size_to_l1(BDRVQcowState *s, int64_t size)
+{
+    int shift = s->cluster_bits + s->l2_bits;
+    return (size + (1ULL << shift) - 1) >> shift;
+}
+
+static inline int offset_to_l2_index(BDRVQcowState *s, int64_t offset)
+{
+    return (offset >> s->cluster_bits) & (s->l2_size - 1);
+}
+
+static inline int64_t align_offset(int64_t offset, int n)
+{
+    offset = (offset + n - 1) & ~(n - 1);
+    return offset;
+}
+
+static inline int qcow2_get_cluster_type(uint64_t l2_entry)
+{
+    if (l2_entry & QCOW_OFLAG_COMPRESSED) {
+        return QCOW2_CLUSTER_COMPRESSED;
+    } else if (l2_entry & QCOW_OFLAG_ZERO) {
+        return QCOW2_CLUSTER_ZERO;
+    } else if (!(l2_entry & L2E_OFFSET_MASK)) {
+        return QCOW2_CLUSTER_UNALLOCATED;
+    } else {
+        return QCOW2_CLUSTER_NORMAL;
+    }
+}
+
+/* Check whether refcounts are eager or lazy */
+static inline bool qcow2_need_accurate_refcounts(BDRVQcowState *s)
+{
+    return !(s->incompatible_features & QCOW2_INCOMPAT_DIRTY);
+}
+
+static inline uint64_t l2meta_cow_start(QCowL2Meta *m)
+{
+    return m->offset + m->cow_start.offset;
+}
+
+static inline uint64_t l2meta_cow_end(QCowL2Meta *m)
+{
+    return m->offset + m->cow_end.offset
+        + (m->cow_end.nb_sectors << BDRV_SECTOR_BITS);
+}
+
+// FIXME Need qcow2_ prefix to global functions
+
+/* qcow2.c functions */
+int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
+                  int64_t sector_num, int nb_sectors);
+
+int qcow2_mark_dirty(BlockDriverState *bs);
+int qcow2_update_header(BlockDriverState *bs);
+
+/* qcow2-refcount.c functions */
+int qcow2_refcount_init(BlockDriverState *bs);
+void qcow2_refcount_close(BlockDriverState *bs);
+
+int64_t qcow2_alloc_clusters(BlockDriverState *bs, int64_t size);
+int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
+    int nb_clusters);
+int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size);
+void qcow2_free_clusters(BlockDriverState *bs,
+                          int64_t offset, int64_t size,
+                          enum qcow2_discard_type type);
+void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
+                             int nb_clusters, enum qcow2_discard_type type);
+
+int qcow2_update_snapshot_refcount(BlockDriverState *bs,
+    int64_t l1_table_offset, int l1_size, int addend);
+
+int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
+                          BdrvCheckMode fix);
+
+void qcow2_process_discards(BlockDriverState *bs, int ret);
+
+/* qcow2-cluster.c functions */
+int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
+                        bool exact_size);
+void qcow2_l2_cache_reset(BlockDriverState *bs);
+int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset);
+void qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
+                     uint8_t *out_buf, const uint8_t *in_buf,
+                     int nb_sectors, int enc,
+                     const AES_KEY *key);
+
+int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
+    int *num, uint64_t *cluster_offset);
+int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
+    int n_start, int n_end, int *num, uint64_t *host_offset, QCowL2Meta **m);
+uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
+                                         uint64_t offset,
+                                         int compressed_size);
+
+int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m);
+int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
+    int nb_sectors);
+int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors);
+
+/* qcow2-snapshot.c functions */
+int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info);
+int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id);
+int qcow2_snapshot_delete(BlockDriverState *bs, const char *snapshot_id);
+int qcow2_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab);
+int qcow2_snapshot_load_tmp(BlockDriverState *bs, const char *snapshot_name);
+
+void qcow2_free_snapshots(BlockDriverState *bs);
+int qcow2_read_snapshots(BlockDriverState *bs);
+
+/* qcow2-cache.c functions */
+Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables);
+int qcow2_cache_destroy(BlockDriverState* bs, Qcow2Cache *c);
+
+void qcow2_cache_entry_mark_dirty(Qcow2Cache *c, void *table);
+int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c);
+int qcow2_cache_set_dependency(BlockDriverState *bs, Qcow2Cache *c,
+    Qcow2Cache *dependency);
+void qcow2_cache_depends_on_flush(Qcow2Cache *c);
+
+int qcow2_cache_get(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
+    void **table);
+int qcow2_cache_get_empty(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
+    void **table);
+int qcow2_cache_put(BlockDriverState *bs, Qcow2Cache *c, void **table);
+
+#endif
diff --git a/contrib/qemu/block/qed-check.c b/contrib/qemu/block/qed-check.c
new file mode 100644
index 000000000..b473dcd61
--- /dev/null
+++ b/contrib/qemu/block/qed-check.c
@@ -0,0 +1,248 @@
+/*
+ * QEMU Enhanced Disk Format Consistency Check
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qed.h"
+
+typedef struct {
+    BDRVQEDState *s;
+    BdrvCheckResult *result;
+    bool fix;                           /* whether to fix invalid offsets */
+
+    uint64_t nclusters;
+    uint32_t *used_clusters;            /* referenced cluster bitmap */
+
+    QEDRequest request;
+} QEDCheck;
+
+static bool qed_test_bit(uint32_t *bitmap, uint64_t n) {
+    return !!(bitmap[n / 32] & (1 << (n % 32)));
+}
+
+static void qed_set_bit(uint32_t *bitmap, uint64_t n) {
+    bitmap[n / 32] |= 1 << (n % 32);
+}
+
+/**
+ * Set bitmap bits for clusters
+ *
+ * @check:          Check structure
+ * @offset:         Starting offset in bytes
+ * @n:              Number of clusters
+ */
+static bool qed_set_used_clusters(QEDCheck *check, uint64_t offset,
+                                  unsigned int n)
+{
+    uint64_t cluster = qed_bytes_to_clusters(check->s, offset);
+    unsigned int corruptions = 0;
+
+    while (n-- != 0) {
+        /* Clusters should only be referenced once */
+        if (qed_test_bit(check->used_clusters, cluster)) {
+            corruptions++;
+        }
+
+        qed_set_bit(check->used_clusters, cluster);
+        cluster++;
+    }
+
+    check->result->corruptions += corruptions;
+    return corruptions == 0;
+}
+
+/**
+ * Check an L2 table
+ *
+ * @ret:            Number of invalid cluster offsets
+ */
+static unsigned int qed_check_l2_table(QEDCheck *check, QEDTable *table)
+{
+    BDRVQEDState *s = check->s;
+    unsigned int i, num_invalid = 0;
+    uint64_t last_offset = 0;
+
+    for (i = 0; i < s->table_nelems; i++) {
+        uint64_t offset = table->offsets[i];
+
+        if (qed_offset_is_unalloc_cluster(offset) ||
+            qed_offset_is_zero_cluster(offset)) {
+            continue;
+        }
+        check->result->bfi.allocated_clusters++;
+        if (last_offset && (last_offset + s->header.cluster_size != offset)) {
+            check->result->bfi.fragmented_clusters++;
+        }
+        last_offset = offset;
+
+        /* Detect invalid cluster offset */
+        if (!qed_check_cluster_offset(s, offset)) {
+            if (check->fix) {
+                table->offsets[i] = 0;
+                check->result->corruptions_fixed++;
+            } else {
+                check->result->corruptions++;
+            }
+
+            num_invalid++;
+            continue;
+        }
+
+        qed_set_used_clusters(check, offset, 1);
+    }
+
+    return num_invalid;
+}
+
+/**
+ * Descend tables and check each cluster is referenced once only
+ */
+static int qed_check_l1_table(QEDCheck *check, QEDTable *table)
+{
+    BDRVQEDState *s = check->s;
+    unsigned int i, num_invalid_l1 = 0;
+    int ret, last_error = 0;
+
+    /* Mark L1 table clusters used */
+    qed_set_used_clusters(check, s->header.l1_table_offset,
+                          s->header.table_size);
+
+    for (i = 0; i < s->table_nelems; i++) {
+        unsigned int num_invalid_l2;
+        uint64_t offset = table->offsets[i];
+
+        if (qed_offset_is_unalloc_cluster(offset)) {
+            continue;
+        }
+
+        /* Detect invalid L2 offset */
+        if (!qed_check_table_offset(s, offset)) {
+            /* Clear invalid offset */
+            if (check->fix) {
+                table->offsets[i] = 0;
+                check->result->corruptions_fixed++;
+            } else {
+                check->result->corruptions++;
+            }
+
+            num_invalid_l1++;
+            continue;
+        }
+
+        if (!qed_set_used_clusters(check, offset, s->header.table_size)) {
+            continue; /* skip an invalid table */
+        }
+
+        ret = qed_read_l2_table_sync(s, &check->request, offset);
+        if (ret) {
+            check->result->check_errors++;
+            last_error = ret;
+            continue;
+        }
+
+        num_invalid_l2 = qed_check_l2_table(check,
+                                            check->request.l2_table->table);
+
+        /* Write out fixed L2 table */
+        if (num_invalid_l2 > 0 && check->fix) {
+            ret = qed_write_l2_table_sync(s, &check->request, 0,
+                                          s->table_nelems, false);
+            if (ret) {
+                check->result->check_errors++;
+                last_error = ret;
+                continue;
+            }
+        }
+    }
+
+    /* Drop reference to final table */
+    qed_unref_l2_cache_entry(check->request.l2_table);
+    check->request.l2_table = NULL;
+
+    /* Write out fixed L1 table */
+    if (num_invalid_l1 > 0 && check->fix) {
+        ret = qed_write_l1_table_sync(s, 0, s->table_nelems);
+        if (ret) {
+            check->result->check_errors++;
+            last_error = ret;
+        }
+    }
+
+    return last_error;
+}
+
+/**
+ * Check for unreferenced (leaked) clusters
+ */
+static void qed_check_for_leaks(QEDCheck *check)
+{
+    BDRVQEDState *s = check->s;
+    uint64_t i;
+
+    for (i = s->header.header_size; i < check->nclusters; i++) {
+        if (!qed_test_bit(check->used_clusters, i)) {
+            check->result->leaks++;
+        }
+    }
+}
+
+/**
+ * Mark an image clean once it passes check or has been repaired
+ */
+static void qed_check_mark_clean(BDRVQEDState *s, BdrvCheckResult *result)
+{
+    /* Skip if there were unfixable corruptions or I/O errors */
+    if (result->corruptions > 0 || result->check_errors > 0) {
+        return;
+    }
+
+    /* Skip if image is already marked clean */
+    if (!(s->header.features & QED_F_NEED_CHECK)) {
+        return;
+    }
+
+    /* Ensure fixes reach storage before clearing check bit */
+    bdrv_flush(s->bs);
+
+    s->header.features &= ~QED_F_NEED_CHECK;
+    qed_write_header_sync(s);
+}
+
+int qed_check(BDRVQEDState *s, BdrvCheckResult *result, bool fix)
+{
+    QEDCheck check = {
+        .s = s,
+        .result = result,
+        .nclusters = qed_bytes_to_clusters(s, s->file_size),
+        .request = { .l2_table = NULL },
+        .fix = fix,
+    };
+    int ret;
+
+    check.used_clusters = g_malloc0(((check.nclusters + 31) / 32) *
+                                       sizeof(check.used_clusters[0]));
+
+    check.result->bfi.total_clusters =
+        (s->header.image_size + s->header.cluster_size - 1) /
+            s->header.cluster_size;
+    ret = qed_check_l1_table(&check, s->l1_table);
+    if (ret == 0) {
+        /* Only check for leaks if entire image was scanned successfully */
+        qed_check_for_leaks(&check);
+
+        if (fix) {
+            qed_check_mark_clean(s, result);
+        }
+    }
+
+    g_free(check.used_clusters);
+    return ret;
+}
diff --git a/contrib/qemu/block/qed-cluster.c b/contrib/qemu/block/qed-cluster.c
new file mode 100644
index 000000000..f64b2af8f
--- /dev/null
+++ b/contrib/qemu/block/qed-cluster.c
@@ -0,0 +1,165 @@
+/*
+ * QEMU Enhanced Disk Format Cluster functions
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qed.h"
+
+/**
+ * Count the number of contiguous data clusters
+ *
+ * @s:              QED state
+ * @table:          L2 table
+ * @index:          First cluster index
+ * @n:              Maximum number of clusters
+ * @offset:         Set to first cluster offset
+ *
+ * This function scans tables for contiguous clusters.  A contiguous run of
+ * clusters may be allocated, unallocated, or zero.
+ */
+static unsigned int qed_count_contiguous_clusters(BDRVQEDState *s,
+                                                  QEDTable *table,
+                                                  unsigned int index,
+                                                  unsigned int n,
+                                                  uint64_t *offset)
+{
+    unsigned int end = MIN(index + n, s->table_nelems);
+    uint64_t last = table->offsets[index];
+    unsigned int i;
+
+    *offset = last;
+
+    for (i = index + 1; i < end; i++) {
+        if (qed_offset_is_unalloc_cluster(last)) {
+            /* Counting unallocated clusters */
+            if (!qed_offset_is_unalloc_cluster(table->offsets[i])) {
+                break;
+            }
+        } else if (qed_offset_is_zero_cluster(last)) {
+            /* Counting zero clusters */
+            if (!qed_offset_is_zero_cluster(table->offsets[i])) {
+                break;
+            }
+        } else {
+            /* Counting allocated clusters */
+            if (table->offsets[i] != last + s->header.cluster_size) {
+                break;
+            }
+            last = table->offsets[i];
+        }
+    }
+    return i - index;
+}
+
+typedef struct {
+    BDRVQEDState *s;
+    uint64_t pos;
+    size_t len;
+
+    QEDRequest *request;
+
+    /* User callback */
+    QEDFindClusterFunc *cb;
+    void *opaque;
+} QEDFindClusterCB;
+
+static void qed_find_cluster_cb(void *opaque, int ret)
+{
+    QEDFindClusterCB *find_cluster_cb = opaque;
+    BDRVQEDState *s = find_cluster_cb->s;
+    QEDRequest *request = find_cluster_cb->request;
+    uint64_t offset = 0;
+    size_t len = 0;
+    unsigned int index;
+    unsigned int n;
+
+    if (ret) {
+        goto out;
+    }
+
+    index = qed_l2_index(s, find_cluster_cb->pos);
+    n = qed_bytes_to_clusters(s,
+                              qed_offset_into_cluster(s, find_cluster_cb->pos) +
+                              find_cluster_cb->len);
+    n = qed_count_contiguous_clusters(s, request->l2_table->table,
+                                      index, n, &offset);
+
+    if (qed_offset_is_unalloc_cluster(offset)) {
+        ret = QED_CLUSTER_L2;
+    } else if (qed_offset_is_zero_cluster(offset)) {
+        ret = QED_CLUSTER_ZERO;
+    } else if (qed_check_cluster_offset(s, offset)) {
+        ret = QED_CLUSTER_FOUND;
+    } else {
+        ret = -EINVAL;
+    }
+
+    len = MIN(find_cluster_cb->len, n * s->header.cluster_size -
+              qed_offset_into_cluster(s, find_cluster_cb->pos));
+
+out:
+    find_cluster_cb->cb(find_cluster_cb->opaque, ret, offset, len);
+    g_free(find_cluster_cb);
+}
+
+/**
+ * Find the offset of a data cluster
+ *
+ * @s:          QED state
+ * @request:    L2 cache entry
+ * @pos:        Byte position in device
+ * @len:        Number of bytes
+ * @cb:         Completion function
+ * @opaque:     User data for completion function
+ *
+ * This function translates a position in the block device to an offset in the
+ * image file.  It invokes the cb completion callback to report back the
+ * translated offset or unallocated range in the image file.
+ *
+ * If the L2 table exists, request->l2_table points to the L2 table cache entry
+ * and the caller must free the reference when they are finished.  The cache
+ * entry is exposed in this way to avoid callers having to read the L2 table
+ * again later during request processing.  If request->l2_table is non-NULL it
+ * will be unreferenced before taking on the new cache entry.
+ */
+void qed_find_cluster(BDRVQEDState *s, QEDRequest *request, uint64_t pos,
+                      size_t len, QEDFindClusterFunc *cb, void *opaque)
+{
+    QEDFindClusterCB *find_cluster_cb;
+    uint64_t l2_offset;
+
+    /* Limit length to L2 boundary.  Requests are broken up at the L2 boundary
+     * so that a request acts on one L2 table at a time.
+     */
+    len = MIN(len, (((pos >> s->l1_shift) + 1) << s->l1_shift) - pos);
+
+    l2_offset = s->l1_table->offsets[qed_l1_index(s, pos)];
+    if (qed_offset_is_unalloc_cluster(l2_offset)) {
+        cb(opaque, QED_CLUSTER_L1, 0, len);
+        return;
+    }
+    if (!qed_check_table_offset(s, l2_offset)) {
+        cb(opaque, -EINVAL, 0, 0);
+        return;
+    }
+
+    find_cluster_cb = g_malloc(sizeof(*find_cluster_cb));
+    find_cluster_cb->s = s;
+    find_cluster_cb->pos = pos;
+    find_cluster_cb->len = len;
+    find_cluster_cb->cb = cb;
+    find_cluster_cb->opaque = opaque;
+    find_cluster_cb->request = request;
+
+    qed_read_l2_table(s, request, l2_offset,
+                      qed_find_cluster_cb, find_cluster_cb);
+}
diff --git a/contrib/qemu/block/qed-gencb.c b/contrib/qemu/block/qed-gencb.c
new file mode 100644
index 000000000..7d7ac1ffc
--- /dev/null
+++ b/contrib/qemu/block/qed-gencb.c
@@ -0,0 +1,32 @@
+/*
+ * QEMU Enhanced Disk Format
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qed.h"
+
+void *gencb_alloc(size_t len, BlockDriverCompletionFunc *cb, void *opaque)
+{
+    GenericCB *gencb = g_malloc(len);
+    gencb->cb = cb;
+    gencb->opaque = opaque;
+    return gencb;
+}
+
+void gencb_complete(void *opaque, int ret)
+{
+    GenericCB *gencb = opaque;
+    BlockDriverCompletionFunc *cb = gencb->cb;
+    void *user_opaque = gencb->opaque;
+
+    g_free(gencb);
+    cb(user_opaque, ret);
+}
diff --git a/contrib/qemu/block/qed-l2-cache.c b/contrib/qemu/block/qed-l2-cache.c
new file mode 100644
index 000000000..e9b2aae44
--- /dev/null
+++ b/contrib/qemu/block/qed-l2-cache.c
@@ -0,0 +1,187 @@
+/*
+ * QEMU Enhanced Disk Format L2 Cache
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+/*
+ * L2 table cache usage is as follows:
+ *
+ * An open image has one L2 table cache that is used to avoid accessing the
+ * image file for recently referenced L2 tables.
+ *
+ * Cluster offset lookup translates the logical offset within the block device
+ * to a cluster offset within the image file.  This is done by indexing into
+ * the L1 and L2 tables which store cluster offsets.  It is here where the L2
+ * table cache serves up recently referenced L2 tables.
+ *
+ * If there is a cache miss, that L2 table is read from the image file and
+ * committed to the cache.  Subsequent accesses to that L2 table will be served
+ * from the cache until the table is evicted from the cache.
+ *
+ * L2 tables are also committed to the cache when new L2 tables are allocated
+ * in the image file.  Since the L2 table cache is write-through, the new L2
+ * table is first written out to the image file and then committed to the
+ * cache.
+ *
+ * Multiple I/O requests may be using an L2 table cache entry at any given
+ * time.  That means an entry may be in use across several requests and
+ * reference counting is needed to free the entry at the correct time.  In
+ * particular, an entry evicted from the cache will only be freed once all
+ * references are dropped.
+ *
+ * An in-flight I/O request will hold a reference to a L2 table cache entry for
+ * the period during which it needs to access the L2 table.  This includes
+ * cluster offset lookup, L2 table allocation, and L2 table update when a new
+ * data cluster has been allocated.
+ *
+ * An interesting case occurs when two requests need to access an L2 table that
+ * is not in the cache.  Since the operation to read the table from the image
+ * file takes some time to complete, both requests may see a cache miss and
+ * start reading the L2 table from the image file.  The first to finish will
+ * commit its L2 table into the cache.  When the second tries to commit its
+ * table will be deleted in favor of the existing cache entry.
+ */
+
+#include "trace.h"
+#include "qed.h"
+
+/* Each L2 holds 2GB so this let's us fully cache a 100GB disk */
+#define MAX_L2_CACHE_SIZE 50
+
+/**
+ * Initialize the L2 cache
+ */
+void qed_init_l2_cache(L2TableCache *l2_cache)
+{
+    QTAILQ_INIT(&l2_cache->entries);
+    l2_cache->n_entries = 0;
+}
+
+/**
+ * Free the L2 cache
+ */
+void qed_free_l2_cache(L2TableCache *l2_cache)
+{
+    CachedL2Table *entry, *next_entry;
+
+    QTAILQ_FOREACH_SAFE(entry, &l2_cache->entries, node, next_entry) {
+        qemu_vfree(entry->table);
+        g_free(entry);
+    }
+}
+
+/**
+ * Allocate an uninitialized entry from the cache
+ *
+ * The returned entry has a reference count of 1 and is owned by the caller.
+ * The caller must allocate the actual table field for this entry and it must
+ * be freeable using qemu_vfree().
+ */
+CachedL2Table *qed_alloc_l2_cache_entry(L2TableCache *l2_cache)
+{
+    CachedL2Table *entry;
+
+    entry = g_malloc0(sizeof(*entry));
+    entry->ref++;
+
+    trace_qed_alloc_l2_cache_entry(l2_cache, entry);
+
+    return entry;
+}
+
+/**
+ * Decrease an entry's reference count and free if necessary when the reference
+ * count drops to zero.
+ */
+void qed_unref_l2_cache_entry(CachedL2Table *entry)
+{
+    if (!entry) {
+        return;
+    }
+
+    entry->ref--;
+    trace_qed_unref_l2_cache_entry(entry, entry->ref);
+    if (entry->ref == 0) {
+        qemu_vfree(entry->table);
+        g_free(entry);
+    }
+}
+
+/**
+ * Find an entry in the L2 cache.  This may return NULL and it's up to the
+ * caller to satisfy the cache miss.
+ *
+ * For a cached entry, this function increases the reference count and returns
+ * the entry.
+ */
+CachedL2Table *qed_find_l2_cache_entry(L2TableCache *l2_cache, uint64_t offset)
+{
+    CachedL2Table *entry;
+
+    QTAILQ_FOREACH(entry, &l2_cache->entries, node) {
+        if (entry->offset == offset) {
+            trace_qed_find_l2_cache_entry(l2_cache, entry, offset, entry->ref);
+            entry->ref++;
+            return entry;
+        }
+    }
+    return NULL;
+}
+
+/**
+ * Commit an L2 cache entry into the cache.  This is meant to be used as part of
+ * the process to satisfy a cache miss.  A caller would allocate an entry which
+ * is not actually in the L2 cache and then once the entry was valid and
+ * present on disk, the entry can be committed into the cache.
+ *
+ * Since the cache is write-through, it's important that this function is not
+ * called until the entry is present on disk and the L1 has been updated to
+ * point to the entry.
+ *
+ * N.B. This function steals a reference to the l2_table from the caller so the
+ * caller must obtain a new reference by issuing a call to
+ * qed_find_l2_cache_entry().
+ */
+void qed_commit_l2_cache_entry(L2TableCache *l2_cache, CachedL2Table *l2_table)
+{
+    CachedL2Table *entry;
+
+    entry = qed_find_l2_cache_entry(l2_cache, l2_table->offset);
+    if (entry) {
+        qed_unref_l2_cache_entry(entry);
+        qed_unref_l2_cache_entry(l2_table);
+        return;
+    }
+
+    /* Evict an unused cache entry so we have space.  If all entries are in use
+     * we can grow the cache temporarily and we try to shrink back down later.
+     */
+    if (l2_cache->n_entries >= MAX_L2_CACHE_SIZE) {
+        CachedL2Table *next;
+        QTAILQ_FOREACH_SAFE(entry, &l2_cache->entries, node, next) {
+            if (entry->ref > 1) {
+                continue;
+            }
+
+            QTAILQ_REMOVE(&l2_cache->entries, entry, node);
+            l2_cache->n_entries--;
+            qed_unref_l2_cache_entry(entry);
+
+            /* Stop evicting when we've shrunk back to max size */
+            if (l2_cache->n_entries < MAX_L2_CACHE_SIZE) {
+                break;
+            }
+        }
+    }
+
+    l2_cache->n_entries++;
+    QTAILQ_INSERT_TAIL(&l2_cache->entries, l2_table, node);
+}
diff --git a/contrib/qemu/block/qed-table.c b/contrib/qemu/block/qed-table.c
new file mode 100644
index 000000000..76d2dcccf
--- /dev/null
+++ b/contrib/qemu/block/qed-table.c
@@ -0,0 +1,296 @@
+/*
+ * QEMU Enhanced Disk Format Table I/O
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "trace.h"
+#include "qemu/sockets.h" /* for EINPROGRESS on Windows */
+#include "qed.h"
+
+typedef struct {
+    GenericCB gencb;
+    BDRVQEDState *s;
+    QEDTable *table;
+
+    struct iovec iov;
+    QEMUIOVector qiov;
+} QEDReadTableCB;
+
+static void qed_read_table_cb(void *opaque, int ret)
+{
+    QEDReadTableCB *read_table_cb = opaque;
+    QEDTable *table = read_table_cb->table;
+    int noffsets = read_table_cb->qiov.size / sizeof(uint64_t);
+    int i;
+
+    /* Handle I/O error */
+    if (ret) {
+        goto out;
+    }
+
+    /* Byteswap offsets */
+    for (i = 0; i < noffsets; i++) {
+        table->offsets[i] = le64_to_cpu(table->offsets[i]);
+    }
+
+out:
+    /* Completion */
+    trace_qed_read_table_cb(read_table_cb->s, read_table_cb->table, ret);
+    gencb_complete(&read_table_cb->gencb, ret);
+}
+
+static void qed_read_table(BDRVQEDState *s, uint64_t offset, QEDTable *table,
+                           BlockDriverCompletionFunc *cb, void *opaque)
+{
+    QEDReadTableCB *read_table_cb = gencb_alloc(sizeof(*read_table_cb),
+                                                cb, opaque);
+    QEMUIOVector *qiov = &read_table_cb->qiov;
+
+    trace_qed_read_table(s, offset, table);
+
+    read_table_cb->s = s;
+    read_table_cb->table = table;
+    read_table_cb->iov.iov_base = table->offsets,
+    read_table_cb->iov.iov_len = s->header.cluster_size * s->header.table_size,
+
+    qemu_iovec_init_external(qiov, &read_table_cb->iov, 1);
+    bdrv_aio_readv(s->bs->file, offset / BDRV_SECTOR_SIZE, qiov,
+                   qiov->size / BDRV_SECTOR_SIZE,
+                   qed_read_table_cb, read_table_cb);
+}
+
+typedef struct {
+    GenericCB gencb;
+    BDRVQEDState *s;
+    QEDTable *orig_table;
+    QEDTable *table;
+    bool flush;             /* flush after write? */
+
+    struct iovec iov;
+    QEMUIOVector qiov;
+} QEDWriteTableCB;
+
+static void qed_write_table_cb(void *opaque, int ret)
+{
+    QEDWriteTableCB *write_table_cb = opaque;
+
+    trace_qed_write_table_cb(write_table_cb->s,
+                             write_table_cb->orig_table,
+                             write_table_cb->flush,
+                             ret);
+
+    if (ret) {
+        goto out;
+    }
+
+    if (write_table_cb->flush) {
+        /* We still need to flush first */
+        write_table_cb->flush = false;
+        bdrv_aio_flush(write_table_cb->s->bs, qed_write_table_cb,
+                       write_table_cb);
+        return;
+    }
+
+out:
+    qemu_vfree(write_table_cb->table);
+    gencb_complete(&write_table_cb->gencb, ret);
+}
+
+/**
+ * Write out an updated part or all of a table
+ *
+ * @s:          QED state
+ * @offset:     Offset of table in image file, in bytes
+ * @table:      Table
+ * @index:      Index of first element
+ * @n:          Number of elements
+ * @flush:      Whether or not to sync to disk
+ * @cb:         Completion function
+ * @opaque:     Argument for completion function
+ */
+static void qed_write_table(BDRVQEDState *s, uint64_t offset, QEDTable *table,
+                            unsigned int index, unsigned int n, bool flush,
+                            BlockDriverCompletionFunc *cb, void *opaque)
+{
+    QEDWriteTableCB *write_table_cb;
+    unsigned int sector_mask = BDRV_SECTOR_SIZE / sizeof(uint64_t) - 1;
+    unsigned int start, end, i;
+    size_t len_bytes;
+
+    trace_qed_write_table(s, offset, table, index, n);
+
+    /* Calculate indices of the first and one after last elements */
+    start = index & ~sector_mask;
+    end = (index + n + sector_mask) & ~sector_mask;
+
+    len_bytes = (end - start) * sizeof(uint64_t);
+
+    write_table_cb = gencb_alloc(sizeof(*write_table_cb), cb, opaque);
+    write_table_cb->s = s;
+    write_table_cb->orig_table = table;
+    write_table_cb->flush = flush;
+    write_table_cb->table = qemu_blockalign(s->bs, len_bytes);
+    write_table_cb->iov.iov_base = write_table_cb->table->offsets;
+    write_table_cb->iov.iov_len = len_bytes;
+    qemu_iovec_init_external(&write_table_cb->qiov, &write_table_cb->iov, 1);
+
+    /* Byteswap table */
+    for (i = start; i < end; i++) {
+        uint64_t le_offset = cpu_to_le64(table->offsets[i]);
+        write_table_cb->table->offsets[i - start] = le_offset;
+    }
+
+    /* Adjust for offset into table */
+    offset += start * sizeof(uint64_t);
+
+    bdrv_aio_writev(s->bs->file, offset / BDRV_SECTOR_SIZE,
+                    &write_table_cb->qiov,
+                    write_table_cb->qiov.size / BDRV_SECTOR_SIZE,
+                    qed_write_table_cb, write_table_cb);
+}
+
+/**
+ * Propagate return value from async callback
+ */
+static void qed_sync_cb(void *opaque, int ret)
+{
+    *(int *)opaque = ret;
+}
+
+int qed_read_l1_table_sync(BDRVQEDState *s)
+{
+    int ret = -EINPROGRESS;
+
+    qed_read_table(s, s->header.l1_table_offset,
+                   s->l1_table, qed_sync_cb, &ret);
+    while (ret == -EINPROGRESS) {
+        qemu_aio_wait();
+    }
+
+    return ret;
+}
+
+void qed_write_l1_table(BDRVQEDState *s, unsigned int index, unsigned int n,
+                        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    BLKDBG_EVENT(s->bs->file, BLKDBG_L1_UPDATE);
+    qed_write_table(s, s->header.l1_table_offset,
+                    s->l1_table, index, n, false, cb, opaque);
+}
+
+int qed_write_l1_table_sync(BDRVQEDState *s, unsigned int index,
+                            unsigned int n)
+{
+    int ret = -EINPROGRESS;
+
+    qed_write_l1_table(s, index, n, qed_sync_cb, &ret);
+    while (ret == -EINPROGRESS) {
+        qemu_aio_wait();
+    }
+
+    return ret;
+}
+
+typedef struct {
+    GenericCB gencb;
+    BDRVQEDState *s;
+    uint64_t l2_offset;
+    QEDRequest *request;
+} QEDReadL2TableCB;
+
+static void qed_read_l2_table_cb(void *opaque, int ret)
+{
+    QEDReadL2TableCB *read_l2_table_cb = opaque;
+    QEDRequest *request = read_l2_table_cb->request;
+    BDRVQEDState *s = read_l2_table_cb->s;
+    CachedL2Table *l2_table = request->l2_table;
+    uint64_t l2_offset = read_l2_table_cb->l2_offset;
+
+    if (ret) {
+        /* can't trust loaded L2 table anymore */
+        qed_unref_l2_cache_entry(l2_table);
+        request->l2_table = NULL;
+    } else {
+        l2_table->offset = l2_offset;
+
+        qed_commit_l2_cache_entry(&s->l2_cache, l2_table);
+
+        /* This is guaranteed to succeed because we just committed the entry
+         * to the cache.
+         */
+        request->l2_table = qed_find_l2_cache_entry(&s->l2_cache, l2_offset);
+        assert(request->l2_table != NULL);
+    }
+
+    gencb_complete(&read_l2_table_cb->gencb, ret);
+}
+
+void qed_read_l2_table(BDRVQEDState *s, QEDRequest *request, uint64_t offset,
+                       BlockDriverCompletionFunc *cb, void *opaque)
+{
+    QEDReadL2TableCB *read_l2_table_cb;
+
+    qed_unref_l2_cache_entry(request->l2_table);
+
+    /* Check for cached L2 entry */
+    request->l2_table = qed_find_l2_cache_entry(&s->l2_cache, offset);
+    if (request->l2_table) {
+        cb(opaque, 0);
+        return;
+    }
+
+    request->l2_table = qed_alloc_l2_cache_entry(&s->l2_cache);
+    request->l2_table->table = qed_alloc_table(s);
+
+    read_l2_table_cb = gencb_alloc(sizeof(*read_l2_table_cb), cb, opaque);
+    read_l2_table_cb->s = s;
+    read_l2_table_cb->l2_offset = offset;
+    read_l2_table_cb->request = request;
+
+    BLKDBG_EVENT(s->bs->file, BLKDBG_L2_LOAD);
+    qed_read_table(s, offset, request->l2_table->table,
+                   qed_read_l2_table_cb, read_l2_table_cb);
+}
+
+int qed_read_l2_table_sync(BDRVQEDState *s, QEDRequest *request, uint64_t offset)
+{
+    int ret = -EINPROGRESS;
+
+    qed_read_l2_table(s, request, offset, qed_sync_cb, &ret);
+    while (ret == -EINPROGRESS) {
+        qemu_aio_wait();
+    }
+
+    return ret;
+}
+
+void qed_write_l2_table(BDRVQEDState *s, QEDRequest *request,
+                        unsigned int index, unsigned int n, bool flush,
+                        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    BLKDBG_EVENT(s->bs->file, BLKDBG_L2_UPDATE);
+    qed_write_table(s, request->l2_table->offset,
+                    request->l2_table->table, index, n, flush, cb, opaque);
+}
+
+int qed_write_l2_table_sync(BDRVQEDState *s, QEDRequest *request,
+                            unsigned int index, unsigned int n, bool flush)
+{
+    int ret = -EINPROGRESS;
+
+    qed_write_l2_table(s, request, index, n, flush, qed_sync_cb, &ret);
+    while (ret == -EINPROGRESS) {
+        qemu_aio_wait();
+    }
+
+    return ret;
+}
diff --git a/contrib/qemu/block/qed.c b/contrib/qemu/block/qed.c
new file mode 100644
index 000000000..f767b0528
--- /dev/null
+++ b/contrib/qemu/block/qed.c
@@ -0,0 +1,1596 @@
+/*
+ * QEMU Enhanced Disk Format
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qemu/timer.h"
+#include "trace.h"
+#include "qed.h"
+#include "qapi/qmp/qerror.h"
+#include "migration/migration.h"
+
+static void qed_aio_cancel(BlockDriverAIOCB *blockacb)
+{
+    QEDAIOCB *acb = (QEDAIOCB *)blockacb;
+    bool finished = false;
+
+    /* Wait for the request to finish */
+    acb->finished = &finished;
+    while (!finished) {
+        qemu_aio_wait();
+    }
+}
+
+static const AIOCBInfo qed_aiocb_info = {
+    .aiocb_size         = sizeof(QEDAIOCB),
+    .cancel             = qed_aio_cancel,
+};
+
+static int bdrv_qed_probe(const uint8_t *buf, int buf_size,
+                          const char *filename)
+{
+    const QEDHeader *header = (const QEDHeader *)buf;
+
+    if (buf_size < sizeof(*header)) {
+        return 0;
+    }
+    if (le32_to_cpu(header->magic) != QED_MAGIC) {
+        return 0;
+    }
+    return 100;
+}
+
+/**
+ * Check whether an image format is raw
+ *
+ * @fmt:    Backing file format, may be NULL
+ */
+static bool qed_fmt_is_raw(const char *fmt)
+{
+    return fmt && strcmp(fmt, "raw") == 0;
+}
+
+static void qed_header_le_to_cpu(const QEDHeader *le, QEDHeader *cpu)
+{
+    cpu->magic = le32_to_cpu(le->magic);
+    cpu->cluster_size = le32_to_cpu(le->cluster_size);
+    cpu->table_size = le32_to_cpu(le->table_size);
+    cpu->header_size = le32_to_cpu(le->header_size);
+    cpu->features = le64_to_cpu(le->features);
+    cpu->compat_features = le64_to_cpu(le->compat_features);
+    cpu->autoclear_features = le64_to_cpu(le->autoclear_features);
+    cpu->l1_table_offset = le64_to_cpu(le->l1_table_offset);
+    cpu->image_size = le64_to_cpu(le->image_size);
+    cpu->backing_filename_offset = le32_to_cpu(le->backing_filename_offset);
+    cpu->backing_filename_size = le32_to_cpu(le->backing_filename_size);
+}
+
+static void qed_header_cpu_to_le(const QEDHeader *cpu, QEDHeader *le)
+{
+    le->magic = cpu_to_le32(cpu->magic);
+    le->cluster_size = cpu_to_le32(cpu->cluster_size);
+    le->table_size = cpu_to_le32(cpu->table_size);
+    le->header_size = cpu_to_le32(cpu->header_size);
+    le->features = cpu_to_le64(cpu->features);
+    le->compat_features = cpu_to_le64(cpu->compat_features);
+    le->autoclear_features = cpu_to_le64(cpu->autoclear_features);
+    le->l1_table_offset = cpu_to_le64(cpu->l1_table_offset);
+    le->image_size = cpu_to_le64(cpu->image_size);
+    le->backing_filename_offset = cpu_to_le32(cpu->backing_filename_offset);
+    le->backing_filename_size = cpu_to_le32(cpu->backing_filename_size);
+}
+
+int qed_write_header_sync(BDRVQEDState *s)
+{
+    QEDHeader le;
+    int ret;
+
+    qed_header_cpu_to_le(&s->header, &le);
+    ret = bdrv_pwrite(s->bs->file, 0, &le, sizeof(le));
+    if (ret != sizeof(le)) {
+        return ret;
+    }
+    return 0;
+}
+
+typedef struct {
+    GenericCB gencb;
+    BDRVQEDState *s;
+    struct iovec iov;
+    QEMUIOVector qiov;
+    int nsectors;
+    uint8_t *buf;
+} QEDWriteHeaderCB;
+
+static void qed_write_header_cb(void *opaque, int ret)
+{
+    QEDWriteHeaderCB *write_header_cb = opaque;
+
+    qemu_vfree(write_header_cb->buf);
+    gencb_complete(write_header_cb, ret);
+}
+
+static void qed_write_header_read_cb(void *opaque, int ret)
+{
+    QEDWriteHeaderCB *write_header_cb = opaque;
+    BDRVQEDState *s = write_header_cb->s;
+
+    if (ret) {
+        qed_write_header_cb(write_header_cb, ret);
+        return;
+    }
+
+    /* Update header */
+    qed_header_cpu_to_le(&s->header, (QEDHeader *)write_header_cb->buf);
+
+    bdrv_aio_writev(s->bs->file, 0, &write_header_cb->qiov,
+                    write_header_cb->nsectors, qed_write_header_cb,
+                    write_header_cb);
+}
+
+/**
+ * Update header in-place (does not rewrite backing filename or other strings)
+ *
+ * This function only updates known header fields in-place and does not affect
+ * extra data after the QED header.
+ */
+static void qed_write_header(BDRVQEDState *s, BlockDriverCompletionFunc cb,
+                             void *opaque)
+{
+    /* We must write full sectors for O_DIRECT but cannot necessarily generate
+     * the data following the header if an unrecognized compat feature is
+     * active.  Therefore, first read the sectors containing the header, update
+     * them, and write back.
+     */
+
+    int nsectors = (sizeof(QEDHeader) + BDRV_SECTOR_SIZE - 1) /
+                   BDRV_SECTOR_SIZE;
+    size_t len = nsectors * BDRV_SECTOR_SIZE;
+    QEDWriteHeaderCB *write_header_cb = gencb_alloc(sizeof(*write_header_cb),
+                                                    cb, opaque);
+
+    write_header_cb->s = s;
+    write_header_cb->nsectors = nsectors;
+    write_header_cb->buf = qemu_blockalign(s->bs, len);
+    write_header_cb->iov.iov_base = write_header_cb->buf;
+    write_header_cb->iov.iov_len = len;
+    qemu_iovec_init_external(&write_header_cb->qiov, &write_header_cb->iov, 1);
+
+    bdrv_aio_readv(s->bs->file, 0, &write_header_cb->qiov, nsectors,
+                   qed_write_header_read_cb, write_header_cb);
+}
+
+static uint64_t qed_max_image_size(uint32_t cluster_size, uint32_t table_size)
+{
+    uint64_t table_entries;
+    uint64_t l2_size;
+
+    table_entries = (table_size * cluster_size) / sizeof(uint64_t);
+    l2_size = table_entries * cluster_size;
+
+    return l2_size * table_entries;
+}
+
+static bool qed_is_cluster_size_valid(uint32_t cluster_size)
+{
+    if (cluster_size < QED_MIN_CLUSTER_SIZE ||
+        cluster_size > QED_MAX_CLUSTER_SIZE) {
+        return false;
+    }
+    if (cluster_size & (cluster_size - 1)) {
+        return false; /* not power of 2 */
+    }
+    return true;
+}
+
+static bool qed_is_table_size_valid(uint32_t table_size)
+{
+    if (table_size < QED_MIN_TABLE_SIZE ||
+        table_size > QED_MAX_TABLE_SIZE) {
+        return false;
+    }
+    if (table_size & (table_size - 1)) {
+        return false; /* not power of 2 */
+    }
+    return true;
+}
+
+static bool qed_is_image_size_valid(uint64_t image_size, uint32_t cluster_size,
+                                    uint32_t table_size)
+{
+    if (image_size % BDRV_SECTOR_SIZE != 0) {
+        return false; /* not multiple of sector size */
+    }
+    if (image_size > qed_max_image_size(cluster_size, table_size)) {
+        return false; /* image is too large */
+    }
+    return true;
+}
+
+/**
+ * Read a string of known length from the image file
+ *
+ * @file:       Image file
+ * @offset:     File offset to start of string, in bytes
+ * @n:          String length in bytes
+ * @buf:        Destination buffer
+ * @buflen:     Destination buffer length in bytes
+ * @ret:        0 on success, -errno on failure
+ *
+ * The string is NUL-terminated.
+ */
+static int qed_read_string(BlockDriverState *file, uint64_t offset, size_t n,
+                           char *buf, size_t buflen)
+{
+    int ret;
+    if (n >= buflen) {
+        return -EINVAL;
+    }
+    ret = bdrv_pread(file, offset, buf, n);
+    if (ret < 0) {
+        return ret;
+    }
+    buf[n] = '\0';
+    return 0;
+}
+
+/**
+ * Allocate new clusters
+ *
+ * @s:          QED state
+ * @n:          Number of contiguous clusters to allocate
+ * @ret:        Offset of first allocated cluster
+ *
+ * This function only produces the offset where the new clusters should be
+ * written.  It updates BDRVQEDState but does not make any changes to the image
+ * file.
+ */
+static uint64_t qed_alloc_clusters(BDRVQEDState *s, unsigned int n)
+{
+    uint64_t offset = s->file_size;
+    s->file_size += n * s->header.cluster_size;
+    return offset;
+}
+
+QEDTable *qed_alloc_table(BDRVQEDState *s)
+{
+    /* Honor O_DIRECT memory alignment requirements */
+    return qemu_blockalign(s->bs,
+                           s->header.cluster_size * s->header.table_size);
+}
+
+/**
+ * Allocate a new zeroed L2 table
+ */
+static CachedL2Table *qed_new_l2_table(BDRVQEDState *s)
+{
+    CachedL2Table *l2_table = qed_alloc_l2_cache_entry(&s->l2_cache);
+
+    l2_table->table = qed_alloc_table(s);
+    l2_table->offset = qed_alloc_clusters(s, s->header.table_size);
+
+    memset(l2_table->table->offsets, 0,
+           s->header.cluster_size * s->header.table_size);
+    return l2_table;
+}
+
+static void qed_aio_next_io(void *opaque, int ret);
+
+static void qed_plug_allocating_write_reqs(BDRVQEDState *s)
+{
+    assert(!s->allocating_write_reqs_plugged);
+
+    s->allocating_write_reqs_plugged = true;
+}
+
+static void qed_unplug_allocating_write_reqs(BDRVQEDState *s)
+{
+    QEDAIOCB *acb;
+
+    assert(s->allocating_write_reqs_plugged);
+
+    s->allocating_write_reqs_plugged = false;
+
+    acb = QSIMPLEQ_FIRST(&s->allocating_write_reqs);
+    if (acb) {
+        qed_aio_next_io(acb, 0);
+    }
+}
+
+static void qed_finish_clear_need_check(void *opaque, int ret)
+{
+    /* Do nothing */
+}
+
+static void qed_flush_after_clear_need_check(void *opaque, int ret)
+{
+    BDRVQEDState *s = opaque;
+
+    bdrv_aio_flush(s->bs, qed_finish_clear_need_check, s);
+
+    /* No need to wait until flush completes */
+    qed_unplug_allocating_write_reqs(s);
+}
+
+static void qed_clear_need_check(void *opaque, int ret)
+{
+    BDRVQEDState *s = opaque;
+
+    if (ret) {
+        qed_unplug_allocating_write_reqs(s);
+        return;
+    }
+
+    s->header.features &= ~QED_F_NEED_CHECK;
+    qed_write_header(s, qed_flush_after_clear_need_check, s);
+}
+
+static void qed_need_check_timer_cb(void *opaque)
+{
+    BDRVQEDState *s = opaque;
+
+    /* The timer should only fire when allocating writes have drained */
+    assert(!QSIMPLEQ_FIRST(&s->allocating_write_reqs));
+
+    trace_qed_need_check_timer_cb(s);
+
+    qed_plug_allocating_write_reqs(s);
+
+    /* Ensure writes are on disk before clearing flag */
+    bdrv_aio_flush(s->bs, qed_clear_need_check, s);
+}
+
+static void qed_start_need_check_timer(BDRVQEDState *s)
+{
+    trace_qed_start_need_check_timer(s);
+
+    /* Use vm_clock so we don't alter the image file while suspended for
+     * migration.
+     */
+    qemu_mod_timer(s->need_check_timer, qemu_get_clock_ns(vm_clock) +
+                   get_ticks_per_sec() * QED_NEED_CHECK_TIMEOUT);
+}
+
+/* It's okay to call this multiple times or when no timer is started */
+static void qed_cancel_need_check_timer(BDRVQEDState *s)
+{
+    trace_qed_cancel_need_check_timer(s);
+    qemu_del_timer(s->need_check_timer);
+}
+
+static void bdrv_qed_rebind(BlockDriverState *bs)
+{
+    BDRVQEDState *s = bs->opaque;
+    s->bs = bs;
+}
+
+static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags)
+{
+    BDRVQEDState *s = bs->opaque;
+    QEDHeader le_header;
+    int64_t file_size;
+    int ret;
+
+    s->bs = bs;
+    QSIMPLEQ_INIT(&s->allocating_write_reqs);
+
+    ret = bdrv_pread(bs->file, 0, &le_header, sizeof(le_header));
+    if (ret < 0) {
+        return ret;
+    }
+    qed_header_le_to_cpu(&le_header, &s->header);
+
+    if (s->header.magic != QED_MAGIC) {
+        return -EMEDIUMTYPE;
+    }
+    if (s->header.features & ~QED_FEATURE_MASK) {
+        /* image uses unsupported feature bits */
+        char buf[64];
+        snprintf(buf, sizeof(buf), "%" PRIx64,
+            s->header.features & ~QED_FEATURE_MASK);
+        qerror_report(QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
+            bs->device_name, "QED", buf);
+        return -ENOTSUP;
+    }
+    if (!qed_is_cluster_size_valid(s->header.cluster_size)) {
+        return -EINVAL;
+    }
+
+    /* Round down file size to the last cluster */
+    file_size = bdrv_getlength(bs->file);
+    if (file_size < 0) {
+        return file_size;
+    }
+    s->file_size = qed_start_of_cluster(s, file_size);
+
+    if (!qed_is_table_size_valid(s->header.table_size)) {
+        return -EINVAL;
+    }
+    if (!qed_is_image_size_valid(s->header.image_size,
+                                 s->header.cluster_size,
+                                 s->header.table_size)) {
+        return -EINVAL;
+    }
+    if (!qed_check_table_offset(s, s->header.l1_table_offset)) {
+        return -EINVAL;
+    }
+
+    s->table_nelems = (s->header.cluster_size * s->header.table_size) /
+                      sizeof(uint64_t);
+    s->l2_shift = ffs(s->header.cluster_size) - 1;
+    s->l2_mask = s->table_nelems - 1;
+    s->l1_shift = s->l2_shift + ffs(s->table_nelems) - 1;
+
+    if ((s->header.features & QED_F_BACKING_FILE)) {
+        if ((uint64_t)s->header.backing_filename_offset +
+            s->header.backing_filename_size >
+            s->header.cluster_size * s->header.header_size) {
+            return -EINVAL;
+        }
+
+        ret = qed_read_string(bs->file, s->header.backing_filename_offset,
+                              s->header.backing_filename_size, bs->backing_file,
+                              sizeof(bs->backing_file));
+        if (ret < 0) {
+            return ret;
+        }
+
+        if (s->header.features & QED_F_BACKING_FORMAT_NO_PROBE) {
+            pstrcpy(bs->backing_format, sizeof(bs->backing_format), "raw");
+        }
+    }
+
+    /* Reset unknown autoclear feature bits.  This is a backwards
+     * compatibility mechanism that allows images to be opened by older
+     * programs, which "knock out" unknown feature bits.  When an image is
+     * opened by a newer program again it can detect that the autoclear
+     * feature is no longer valid.
+     */
+    if ((s->header.autoclear_features & ~QED_AUTOCLEAR_FEATURE_MASK) != 0 &&
+        !bdrv_is_read_only(bs->file) && !(flags & BDRV_O_INCOMING)) {
+        s->header.autoclear_features &= QED_AUTOCLEAR_FEATURE_MASK;
+
+        ret = qed_write_header_sync(s);
+        if (ret) {
+            return ret;
+        }
+
+        /* From here on only known autoclear feature bits are valid */
+        bdrv_flush(bs->file);
+    }
+
+    s->l1_table = qed_alloc_table(s);
+    qed_init_l2_cache(&s->l2_cache);
+
+    ret = qed_read_l1_table_sync(s);
+    if (ret) {
+        goto out;
+    }
+
+    /* If image was not closed cleanly, check consistency */
+    if (!(flags & BDRV_O_CHECK) && (s->header.features & QED_F_NEED_CHECK)) {
+        /* Read-only images cannot be fixed.  There is no risk of corruption
+         * since write operations are not possible.  Therefore, allow
+         * potentially inconsistent images to be opened read-only.  This can
+         * aid data recovery from an otherwise inconsistent image.
+         */
+        if (!bdrv_is_read_only(bs->file) &&
+            !(flags & BDRV_O_INCOMING)) {
+            BdrvCheckResult result = {0};
+
+            ret = qed_check(s, &result, true);
+            if (ret) {
+                goto out;
+            }
+        }
+    }
+
+    s->need_check_timer = qemu_new_timer_ns(vm_clock,
+                                            qed_need_check_timer_cb, s);
+
+out:
+    if (ret) {
+        qed_free_l2_cache(&s->l2_cache);
+        qemu_vfree(s->l1_table);
+    }
+    return ret;
+}
+
+/* We have nothing to do for QED reopen, stubs just return
+ * success */
+static int bdrv_qed_reopen_prepare(BDRVReopenState *state,
+                                   BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
+static void bdrv_qed_close(BlockDriverState *bs)
+{
+    BDRVQEDState *s = bs->opaque;
+
+    qed_cancel_need_check_timer(s);
+    qemu_free_timer(s->need_check_timer);
+
+    /* Ensure writes reach stable storage */
+    bdrv_flush(bs->file);
+
+    /* Clean shutdown, no check required on next open */
+    if (s->header.features & QED_F_NEED_CHECK) {
+        s->header.features &= ~QED_F_NEED_CHECK;
+        qed_write_header_sync(s);
+    }
+
+    qed_free_l2_cache(&s->l2_cache);
+    qemu_vfree(s->l1_table);
+}
+
+static int qed_create(const char *filename, uint32_t cluster_size,
+                      uint64_t image_size, uint32_t table_size,
+                      const char *backing_file, const char *backing_fmt)
+{
+    QEDHeader header = {
+        .magic = QED_MAGIC,
+        .cluster_size = cluster_size,
+        .table_size = table_size,
+        .header_size = 1,
+        .features = 0,
+        .compat_features = 0,
+        .l1_table_offset = cluster_size,
+        .image_size = image_size,
+    };
+    QEDHeader le_header;
+    uint8_t *l1_table = NULL;
+    size_t l1_size = header.cluster_size * header.table_size;
+    int ret = 0;
+    BlockDriverState *bs = NULL;
+
+    ret = bdrv_create_file(filename, NULL);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = bdrv_file_open(&bs, filename, NULL, BDRV_O_RDWR | BDRV_O_CACHE_WB);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* File must start empty and grow, check truncate is supported */
+    ret = bdrv_truncate(bs, 0);
+    if (ret < 0) {
+        goto out;
+    }
+
+    if (backing_file) {
+        header.features |= QED_F_BACKING_FILE;
+        header.backing_filename_offset = sizeof(le_header);
+        header.backing_filename_size = strlen(backing_file);
+
+        if (qed_fmt_is_raw(backing_fmt)) {
+            header.features |= QED_F_BACKING_FORMAT_NO_PROBE;
+        }
+    }
+
+    qed_header_cpu_to_le(&header, &le_header);
+    ret = bdrv_pwrite(bs, 0, &le_header, sizeof(le_header));
+    if (ret < 0) {
+        goto out;
+    }
+    ret = bdrv_pwrite(bs, sizeof(le_header), backing_file,
+                      header.backing_filename_size);
+    if (ret < 0) {
+        goto out;
+    }
+
+    l1_table = g_malloc0(l1_size);
+    ret = bdrv_pwrite(bs, header.l1_table_offset, l1_table, l1_size);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = 0; /* success */
+out:
+    g_free(l1_table);
+    bdrv_delete(bs);
+    return ret;
+}
+
+static int bdrv_qed_create(const char *filename, QEMUOptionParameter *options)
+{
+    uint64_t image_size = 0;
+    uint32_t cluster_size = QED_DEFAULT_CLUSTER_SIZE;
+    uint32_t table_size = QED_DEFAULT_TABLE_SIZE;
+    const char *backing_file = NULL;
+    const char *backing_fmt = NULL;
+
+    while (options && options->name) {
+        if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
+            image_size = options->value.n;
+        } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FILE)) {
+            backing_file = options->value.s;
+        } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FMT)) {
+            backing_fmt = options->value.s;
+        } else if (!strcmp(options->name, BLOCK_OPT_CLUSTER_SIZE)) {
+            if (options->value.n) {
+                cluster_size = options->value.n;
+            }
+        } else if (!strcmp(options->name, BLOCK_OPT_TABLE_SIZE)) {
+            if (options->value.n) {
+                table_size = options->value.n;
+            }
+        }
+        options++;
+    }
+
+    if (!qed_is_cluster_size_valid(cluster_size)) {
+        fprintf(stderr, "QED cluster size must be within range [%u, %u] and power of 2\n",
+                QED_MIN_CLUSTER_SIZE, QED_MAX_CLUSTER_SIZE);
+        return -EINVAL;
+    }
+    if (!qed_is_table_size_valid(table_size)) {
+        fprintf(stderr, "QED table size must be within range [%u, %u] and power of 2\n",
+                QED_MIN_TABLE_SIZE, QED_MAX_TABLE_SIZE);
+        return -EINVAL;
+    }
+    if (!qed_is_image_size_valid(image_size, cluster_size, table_size)) {
+        fprintf(stderr, "QED image size must be a non-zero multiple of "
+                        "cluster size and less than %" PRIu64 " bytes\n",
+                qed_max_image_size(cluster_size, table_size));
+        return -EINVAL;
+    }
+
+    return qed_create(filename, cluster_size, image_size, table_size,
+                      backing_file, backing_fmt);
+}
+
+typedef struct {
+    Coroutine *co;
+    int is_allocated;
+    int *pnum;
+} QEDIsAllocatedCB;
+
+static void qed_is_allocated_cb(void *opaque, int ret, uint64_t offset, size_t len)
+{
+    QEDIsAllocatedCB *cb = opaque;
+    *cb->pnum = len / BDRV_SECTOR_SIZE;
+    cb->is_allocated = (ret == QED_CLUSTER_FOUND || ret == QED_CLUSTER_ZERO);
+    if (cb->co) {
+        qemu_coroutine_enter(cb->co, NULL);
+    }
+}
+
+static int coroutine_fn bdrv_qed_co_is_allocated(BlockDriverState *bs,
+                                                 int64_t sector_num,
+                                                 int nb_sectors, int *pnum)
+{
+    BDRVQEDState *s = bs->opaque;
+    uint64_t pos = (uint64_t)sector_num * BDRV_SECTOR_SIZE;
+    size_t len = (size_t)nb_sectors * BDRV_SECTOR_SIZE;
+    QEDIsAllocatedCB cb = {
+        .is_allocated = -1,
+        .pnum = pnum,
+    };
+    QEDRequest request = { .l2_table = NULL };
+
+    qed_find_cluster(s, &request, pos, len, qed_is_allocated_cb, &cb);
+
+    /* Now sleep if the callback wasn't invoked immediately */
+    while (cb.is_allocated == -1) {
+        cb.co = qemu_coroutine_self();
+        qemu_coroutine_yield();
+    }
+
+    qed_unref_l2_cache_entry(request.l2_table);
+
+    return cb.is_allocated;
+}
+
+static int bdrv_qed_make_empty(BlockDriverState *bs)
+{
+    return -ENOTSUP;
+}
+
+static BDRVQEDState *acb_to_s(QEDAIOCB *acb)
+{
+    return acb->common.bs->opaque;
+}
+
+/**
+ * Read from the backing file or zero-fill if no backing file
+ *
+ * @s:          QED state
+ * @pos:        Byte position in device
+ * @qiov:       Destination I/O vector
+ * @cb:         Completion function
+ * @opaque:     User data for completion function
+ *
+ * This function reads qiov->size bytes starting at pos from the backing file.
+ * If there is no backing file then zeroes are read.
+ */
+static void qed_read_backing_file(BDRVQEDState *s, uint64_t pos,
+                                  QEMUIOVector *qiov,
+                                  BlockDriverCompletionFunc *cb, void *opaque)
+{
+    uint64_t backing_length = 0;
+    size_t size;
+
+    /* If there is a backing file, get its length.  Treat the absence of a
+     * backing file like a zero length backing file.
+     */
+    if (s->bs->backing_hd) {
+        int64_t l = bdrv_getlength(s->bs->backing_hd);
+        if (l < 0) {
+            cb(opaque, l);
+            return;
+        }
+        backing_length = l;
+    }
+
+    /* Zero all sectors if reading beyond the end of the backing file */
+    if (pos >= backing_length ||
+        pos + qiov->size > backing_length) {
+        qemu_iovec_memset(qiov, 0, 0, qiov->size);
+    }
+
+    /* Complete now if there are no backing file sectors to read */
+    if (pos >= backing_length) {
+        cb(opaque, 0);
+        return;
+    }
+
+    /* If the read straddles the end of the backing file, shorten it */
+    size = MIN((uint64_t)backing_length - pos, qiov->size);
+
+    BLKDBG_EVENT(s->bs->file, BLKDBG_READ_BACKING_AIO);
+    bdrv_aio_readv(s->bs->backing_hd, pos / BDRV_SECTOR_SIZE,
+                   qiov, size / BDRV_SECTOR_SIZE, cb, opaque);
+}
+
+typedef struct {
+    GenericCB gencb;
+    BDRVQEDState *s;
+    QEMUIOVector qiov;
+    struct iovec iov;
+    uint64_t offset;
+} CopyFromBackingFileCB;
+
+static void qed_copy_from_backing_file_cb(void *opaque, int ret)
+{
+    CopyFromBackingFileCB *copy_cb = opaque;
+    qemu_vfree(copy_cb->iov.iov_base);
+    gencb_complete(&copy_cb->gencb, ret);
+}
+
+static void qed_copy_from_backing_file_write(void *opaque, int ret)
+{
+    CopyFromBackingFileCB *copy_cb = opaque;
+    BDRVQEDState *s = copy_cb->s;
+
+    if (ret) {
+        qed_copy_from_backing_file_cb(copy_cb, ret);
+        return;
+    }
+
+    BLKDBG_EVENT(s->bs->file, BLKDBG_COW_WRITE);
+    bdrv_aio_writev(s->bs->file, copy_cb->offset / BDRV_SECTOR_SIZE,
+                    &copy_cb->qiov, copy_cb->qiov.size / BDRV_SECTOR_SIZE,
+                    qed_copy_from_backing_file_cb, copy_cb);
+}
+
+/**
+ * Copy data from backing file into the image
+ *
+ * @s:          QED state
+ * @pos:        Byte position in device
+ * @len:        Number of bytes
+ * @offset:     Byte offset in image file
+ * @cb:         Completion function
+ * @opaque:     User data for completion function
+ */
+static void qed_copy_from_backing_file(BDRVQEDState *s, uint64_t pos,
+                                       uint64_t len, uint64_t offset,
+                                       BlockDriverCompletionFunc *cb,
+                                       void *opaque)
+{
+    CopyFromBackingFileCB *copy_cb;
+
+    /* Skip copy entirely if there is no work to do */
+    if (len == 0) {
+        cb(opaque, 0);
+        return;
+    }
+
+    copy_cb = gencb_alloc(sizeof(*copy_cb), cb, opaque);
+    copy_cb->s = s;
+    copy_cb->offset = offset;
+    copy_cb->iov.iov_base = qemu_blockalign(s->bs, len);
+    copy_cb->iov.iov_len = len;
+    qemu_iovec_init_external(&copy_cb->qiov, &copy_cb->iov, 1);
+
+    qed_read_backing_file(s, pos, &copy_cb->qiov,
+                          qed_copy_from_backing_file_write, copy_cb);
+}
+
+/**
+ * Link one or more contiguous clusters into a table
+ *
+ * @s:              QED state
+ * @table:          L2 table
+ * @index:          First cluster index
+ * @n:              Number of contiguous clusters
+ * @cluster:        First cluster offset
+ *
+ * The cluster offset may be an allocated byte offset in the image file, the
+ * zero cluster marker, or the unallocated cluster marker.
+ */
+static void qed_update_l2_table(BDRVQEDState *s, QEDTable *table, int index,
+                                unsigned int n, uint64_t cluster)
+{
+    int i;
+    for (i = index; i < index + n; i++) {
+        table->offsets[i] = cluster;
+        if (!qed_offset_is_unalloc_cluster(cluster) &&
+            !qed_offset_is_zero_cluster(cluster)) {
+            cluster += s->header.cluster_size;
+        }
+    }
+}
+
+static void qed_aio_complete_bh(void *opaque)
+{
+    QEDAIOCB *acb = opaque;
+    BlockDriverCompletionFunc *cb = acb->common.cb;
+    void *user_opaque = acb->common.opaque;
+    int ret = acb->bh_ret;
+    bool *finished = acb->finished;
+
+    qemu_bh_delete(acb->bh);
+    qemu_aio_release(acb);
+
+    /* Invoke callback */
+    cb(user_opaque, ret);
+
+    /* Signal cancel completion */
+    if (finished) {
+        *finished = true;
+    }
+}
+
+static void qed_aio_complete(QEDAIOCB *acb, int ret)
+{
+    BDRVQEDState *s = acb_to_s(acb);
+
+    trace_qed_aio_complete(s, acb, ret);
+
+    /* Free resources */
+    qemu_iovec_destroy(&acb->cur_qiov);
+    qed_unref_l2_cache_entry(acb->request.l2_table);
+
+    /* Free the buffer we may have allocated for zero writes */
+    if (acb->flags & QED_AIOCB_ZERO) {
+        qemu_vfree(acb->qiov->iov[0].iov_base);
+        acb->qiov->iov[0].iov_base = NULL;
+    }
+
+    /* Arrange for a bh to invoke the completion function */
+    acb->bh_ret = ret;
+    acb->bh = qemu_bh_new(qed_aio_complete_bh, acb);
+    qemu_bh_schedule(acb->bh);
+
+    /* Start next allocating write request waiting behind this one.  Note that
+     * requests enqueue themselves when they first hit an unallocated cluster
+     * but they wait until the entire request is finished before waking up the
+     * next request in the queue.  This ensures that we don't cycle through
+     * requests multiple times but rather finish one at a time completely.
+     */
+    if (acb == QSIMPLEQ_FIRST(&s->allocating_write_reqs)) {
+        QSIMPLEQ_REMOVE_HEAD(&s->allocating_write_reqs, next);
+        acb = QSIMPLEQ_FIRST(&s->allocating_write_reqs);
+        if (acb) {
+            qed_aio_next_io(acb, 0);
+        } else if (s->header.features & QED_F_NEED_CHECK) {
+            qed_start_need_check_timer(s);
+        }
+    }
+}
+
+/**
+ * Commit the current L2 table to the cache
+ */
+static void qed_commit_l2_update(void *opaque, int ret)
+{
+    QEDAIOCB *acb = opaque;
+    BDRVQEDState *s = acb_to_s(acb);
+    CachedL2Table *l2_table = acb->request.l2_table;
+    uint64_t l2_offset = l2_table->offset;
+
+    qed_commit_l2_cache_entry(&s->l2_cache, l2_table);
+
+    /* This is guaranteed to succeed because we just committed the entry to the
+     * cache.
+     */
+    acb->request.l2_table = qed_find_l2_cache_entry(&s->l2_cache, l2_offset);
+    assert(acb->request.l2_table != NULL);
+
+    qed_aio_next_io(opaque, ret);
+}
+
+/**
+ * Update L1 table with new L2 table offset and write it out
+ */
+static void qed_aio_write_l1_update(void *opaque, int ret)
+{
+    QEDAIOCB *acb = opaque;
+    BDRVQEDState *s = acb_to_s(acb);
+    int index;
+
+    if (ret) {
+        qed_aio_complete(acb, ret);
+        return;
+    }
+
+    index = qed_l1_index(s, acb->cur_pos);
+    s->l1_table->offsets[index] = acb->request.l2_table->offset;
+
+    qed_write_l1_table(s, index, 1, qed_commit_l2_update, acb);
+}
+
+/**
+ * Update L2 table with new cluster offsets and write them out
+ */
+static void qed_aio_write_l2_update(QEDAIOCB *acb, int ret, uint64_t offset)
+{
+    BDRVQEDState *s = acb_to_s(acb);
+    bool need_alloc = acb->find_cluster_ret == QED_CLUSTER_L1;
+    int index;
+
+    if (ret) {
+        goto err;
+    }
+
+    if (need_alloc) {
+        qed_unref_l2_cache_entry(acb->request.l2_table);
+        acb->request.l2_table = qed_new_l2_table(s);
+    }
+
+    index = qed_l2_index(s, acb->cur_pos);
+    qed_update_l2_table(s, acb->request.l2_table->table, index, acb->cur_nclusters,
+                         offset);
+
+    if (need_alloc) {
+        /* Write out the whole new L2 table */
+        qed_write_l2_table(s, &acb->request, 0, s->table_nelems, true,
+                            qed_aio_write_l1_update, acb);
+    } else {
+        /* Write out only the updated part of the L2 table */
+        qed_write_l2_table(s, &acb->request, index, acb->cur_nclusters, false,
+                            qed_aio_next_io, acb);
+    }
+    return;
+
+err:
+    qed_aio_complete(acb, ret);
+}
+
+static void qed_aio_write_l2_update_cb(void *opaque, int ret)
+{
+    QEDAIOCB *acb = opaque;
+    qed_aio_write_l2_update(acb, ret, acb->cur_cluster);
+}
+
+/**
+ * Flush new data clusters before updating the L2 table
+ *
+ * This flush is necessary when a backing file is in use.  A crash during an
+ * allocating write could result in empty clusters in the image.  If the write
+ * only touched a subregion of the cluster, then backing image sectors have
+ * been lost in the untouched region.  The solution is to flush after writing a
+ * new data cluster and before updating the L2 table.
+ */
+static void qed_aio_write_flush_before_l2_update(void *opaque, int ret)
+{
+    QEDAIOCB *acb = opaque;
+    BDRVQEDState *s = acb_to_s(acb);
+
+    if (!bdrv_aio_flush(s->bs->file, qed_aio_write_l2_update_cb, opaque)) {
+        qed_aio_complete(acb, -EIO);
+    }
+}
+
+/**
+ * Write data to the image file
+ */
+static void qed_aio_write_main(void *opaque, int ret)
+{
+    QEDAIOCB *acb = opaque;
+    BDRVQEDState *s = acb_to_s(acb);
+    uint64_t offset = acb->cur_cluster +
+                      qed_offset_into_cluster(s, acb->cur_pos);
+    BlockDriverCompletionFunc *next_fn;
+
+    trace_qed_aio_write_main(s, acb, ret, offset, acb->cur_qiov.size);
+
+    if (ret) {
+        qed_aio_complete(acb, ret);
+        return;
+    }
+
+    if (acb->find_cluster_ret == QED_CLUSTER_FOUND) {
+        next_fn = qed_aio_next_io;
+    } else {
+        if (s->bs->backing_hd) {
+            next_fn = qed_aio_write_flush_before_l2_update;
+        } else {
+            next_fn = qed_aio_write_l2_update_cb;
+        }
+    }
+
+    BLKDBG_EVENT(s->bs->file, BLKDBG_WRITE_AIO);
+    bdrv_aio_writev(s->bs->file, offset / BDRV_SECTOR_SIZE,
+                    &acb->cur_qiov, acb->cur_qiov.size / BDRV_SECTOR_SIZE,
+                    next_fn, acb);
+}
+
+/**
+ * Populate back untouched region of new data cluster
+ */
+static void qed_aio_write_postfill(void *opaque, int ret)
+{
+    QEDAIOCB *acb = opaque;
+    BDRVQEDState *s = acb_to_s(acb);
+    uint64_t start = acb->cur_pos + acb->cur_qiov.size;
+    uint64_t len =
+        qed_start_of_cluster(s, start + s->header.cluster_size - 1) - start;
+    uint64_t offset = acb->cur_cluster +
+                      qed_offset_into_cluster(s, acb->cur_pos) +
+                      acb->cur_qiov.size;
+
+    if (ret) {
+        qed_aio_complete(acb, ret);
+        return;
+    }
+
+    trace_qed_aio_write_postfill(s, acb, start, len, offset);
+    qed_copy_from_backing_file(s, start, len, offset,
+                                qed_aio_write_main, acb);
+}
+
+/**
+ * Populate front untouched region of new data cluster
+ */
+static void qed_aio_write_prefill(void *opaque, int ret)
+{
+    QEDAIOCB *acb = opaque;
+    BDRVQEDState *s = acb_to_s(acb);
+    uint64_t start = qed_start_of_cluster(s, acb->cur_pos);
+    uint64_t len = qed_offset_into_cluster(s, acb->cur_pos);
+
+    trace_qed_aio_write_prefill(s, acb, start, len, acb->cur_cluster);
+    qed_copy_from_backing_file(s, start, len, acb->cur_cluster,
+                                qed_aio_write_postfill, acb);
+}
+
+/**
+ * Check if the QED_F_NEED_CHECK bit should be set during allocating write
+ */
+static bool qed_should_set_need_check(BDRVQEDState *s)
+{
+    /* The flush before L2 update path ensures consistency */
+    if (s->bs->backing_hd) {
+        return false;
+    }
+
+    return !(s->header.features & QED_F_NEED_CHECK);
+}
+
+static void qed_aio_write_zero_cluster(void *opaque, int ret)
+{
+    QEDAIOCB *acb = opaque;
+
+    if (ret) {
+        qed_aio_complete(acb, ret);
+        return;
+    }
+
+    qed_aio_write_l2_update(acb, 0, 1);
+}
+
+/**
+ * Write new data cluster
+ *
+ * @acb:        Write request
+ * @len:        Length in bytes
+ *
+ * This path is taken when writing to previously unallocated clusters.
+ */
+static void qed_aio_write_alloc(QEDAIOCB *acb, size_t len)
+{
+    BDRVQEDState *s = acb_to_s(acb);
+    BlockDriverCompletionFunc *cb;
+
+    /* Cancel timer when the first allocating request comes in */
+    if (QSIMPLEQ_EMPTY(&s->allocating_write_reqs)) {
+        qed_cancel_need_check_timer(s);
+    }
+
+    /* Freeze this request if another allocating write is in progress */
+    if (acb != QSIMPLEQ_FIRST(&s->allocating_write_reqs)) {
+        QSIMPLEQ_INSERT_TAIL(&s->allocating_write_reqs, acb, next);
+    }
+    if (acb != QSIMPLEQ_FIRST(&s->allocating_write_reqs) ||
+        s->allocating_write_reqs_plugged) {
+        return; /* wait for existing request to finish */
+    }
+
+    acb->cur_nclusters = qed_bytes_to_clusters(s,
+            qed_offset_into_cluster(s, acb->cur_pos) + len);
+    qemu_iovec_concat(&acb->cur_qiov, acb->qiov, acb->qiov_offset, len);
+
+    if (acb->flags & QED_AIOCB_ZERO) {
+        /* Skip ahead if the clusters are already zero */
+        if (acb->find_cluster_ret == QED_CLUSTER_ZERO) {
+            qed_aio_next_io(acb, 0);
+            return;
+        }
+
+        cb = qed_aio_write_zero_cluster;
+    } else {
+        cb = qed_aio_write_prefill;
+        acb->cur_cluster = qed_alloc_clusters(s, acb->cur_nclusters);
+    }
+
+    if (qed_should_set_need_check(s)) {
+        s->header.features |= QED_F_NEED_CHECK;
+        qed_write_header(s, cb, acb);
+    } else {
+        cb(acb, 0);
+    }
+}
+
+/**
+ * Write data cluster in place
+ *
+ * @acb:        Write request
+ * @offset:     Cluster offset in bytes
+ * @len:        Length in bytes
+ *
+ * This path is taken when writing to already allocated clusters.
+ */
+static void qed_aio_write_inplace(QEDAIOCB *acb, uint64_t offset, size_t len)
+{
+    /* Allocate buffer for zero writes */
+    if (acb->flags & QED_AIOCB_ZERO) {
+        struct iovec *iov = acb->qiov->iov;
+
+        if (!iov->iov_base) {
+            iov->iov_base = qemu_blockalign(acb->common.bs, iov->iov_len);
+            memset(iov->iov_base, 0, iov->iov_len);
+        }
+    }
+
+    /* Calculate the I/O vector */
+    acb->cur_cluster = offset;
+    qemu_iovec_concat(&acb->cur_qiov, acb->qiov, acb->qiov_offset, len);
+
+    /* Do the actual write */
+    qed_aio_write_main(acb, 0);
+}
+
+/**
+ * Write data cluster
+ *
+ * @opaque:     Write request
+ * @ret:        QED_CLUSTER_FOUND, QED_CLUSTER_L2, QED_CLUSTER_L1,
+ *              or -errno
+ * @offset:     Cluster offset in bytes
+ * @len:        Length in bytes
+ *
+ * Callback from qed_find_cluster().
+ */
+static void qed_aio_write_data(void *opaque, int ret,
+                               uint64_t offset, size_t len)
+{
+    QEDAIOCB *acb = opaque;
+
+    trace_qed_aio_write_data(acb_to_s(acb), acb, ret, offset, len);
+
+    acb->find_cluster_ret = ret;
+
+    switch (ret) {
+    case QED_CLUSTER_FOUND:
+        qed_aio_write_inplace(acb, offset, len);
+        break;
+
+    case QED_CLUSTER_L2:
+    case QED_CLUSTER_L1:
+    case QED_CLUSTER_ZERO:
+        qed_aio_write_alloc(acb, len);
+        break;
+
+    default:
+        qed_aio_complete(acb, ret);
+        break;
+    }
+}
+
+/**
+ * Read data cluster
+ *
+ * @opaque:     Read request
+ * @ret:        QED_CLUSTER_FOUND, QED_CLUSTER_L2, QED_CLUSTER_L1,
+ *              or -errno
+ * @offset:     Cluster offset in bytes
+ * @len:        Length in bytes
+ *
+ * Callback from qed_find_cluster().
+ */
+static void qed_aio_read_data(void *opaque, int ret,
+                              uint64_t offset, size_t len)
+{
+    QEDAIOCB *acb = opaque;
+    BDRVQEDState *s = acb_to_s(acb);
+    BlockDriverState *bs = acb->common.bs;
+
+    /* Adjust offset into cluster */
+    offset += qed_offset_into_cluster(s, acb->cur_pos);
+
+    trace_qed_aio_read_data(s, acb, ret, offset, len);
+
+    if (ret < 0) {
+        goto err;
+    }
+
+    qemu_iovec_concat(&acb->cur_qiov, acb->qiov, acb->qiov_offset, len);
+
+    /* Handle zero cluster and backing file reads */
+    if (ret == QED_CLUSTER_ZERO) {
+        qemu_iovec_memset(&acb->cur_qiov, 0, 0, acb->cur_qiov.size);
+        qed_aio_next_io(acb, 0);
+        return;
+    } else if (ret != QED_CLUSTER_FOUND) {
+        qed_read_backing_file(s, acb->cur_pos, &acb->cur_qiov,
+                              qed_aio_next_io, acb);
+        return;
+    }
+
+    BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
+    bdrv_aio_readv(bs->file, offset / BDRV_SECTOR_SIZE,
+                   &acb->cur_qiov, acb->cur_qiov.size / BDRV_SECTOR_SIZE,
+                   qed_aio_next_io, acb);
+    return;
+
+err:
+    qed_aio_complete(acb, ret);
+}
+
+/**
+ * Begin next I/O or complete the request
+ */
+static void qed_aio_next_io(void *opaque, int ret)
+{
+    QEDAIOCB *acb = opaque;
+    BDRVQEDState *s = acb_to_s(acb);
+    QEDFindClusterFunc *io_fn = (acb->flags & QED_AIOCB_WRITE) ?
+                                qed_aio_write_data : qed_aio_read_data;
+
+    trace_qed_aio_next_io(s, acb, ret, acb->cur_pos + acb->cur_qiov.size);
+
+    /* Handle I/O error */
+    if (ret) {
+        qed_aio_complete(acb, ret);
+        return;
+    }
+
+    acb->qiov_offset += acb->cur_qiov.size;
+    acb->cur_pos += acb->cur_qiov.size;
+    qemu_iovec_reset(&acb->cur_qiov);
+
+    /* Complete request */
+    if (acb->cur_pos >= acb->end_pos) {
+        qed_aio_complete(acb, 0);
+        return;
+    }
+
+    /* Find next cluster and start I/O */
+    qed_find_cluster(s, &acb->request,
+                      acb->cur_pos, acb->end_pos - acb->cur_pos,
+                      io_fn, acb);
+}
+
+static BlockDriverAIOCB *qed_aio_setup(BlockDriverState *bs,
+                                       int64_t sector_num,
+                                       QEMUIOVector *qiov, int nb_sectors,
+                                       BlockDriverCompletionFunc *cb,
+                                       void *opaque, int flags)
+{
+    QEDAIOCB *acb = qemu_aio_get(&qed_aiocb_info, bs, cb, opaque);
+
+    trace_qed_aio_setup(bs->opaque, acb, sector_num, nb_sectors,
+                        opaque, flags);
+
+    acb->flags = flags;
+    acb->finished = NULL;
+    acb->qiov = qiov;
+    acb->qiov_offset = 0;
+    acb->cur_pos = (uint64_t)sector_num * BDRV_SECTOR_SIZE;
+    acb->end_pos = acb->cur_pos + nb_sectors * BDRV_SECTOR_SIZE;
+    acb->request.l2_table = NULL;
+    qemu_iovec_init(&acb->cur_qiov, qiov->niov);
+
+    /* Start request */
+    qed_aio_next_io(acb, 0);
+    return &acb->common;
+}
+
+static BlockDriverAIOCB *bdrv_qed_aio_readv(BlockDriverState *bs,
+                                            int64_t sector_num,
+                                            QEMUIOVector *qiov, int nb_sectors,
+                                            BlockDriverCompletionFunc *cb,
+                                            void *opaque)
+{
+    return qed_aio_setup(bs, sector_num, qiov, nb_sectors, cb, opaque, 0);
+}
+
+static BlockDriverAIOCB *bdrv_qed_aio_writev(BlockDriverState *bs,
+                                             int64_t sector_num,
+                                             QEMUIOVector *qiov, int nb_sectors,
+                                             BlockDriverCompletionFunc *cb,
+                                             void *opaque)
+{
+    return qed_aio_setup(bs, sector_num, qiov, nb_sectors, cb,
+                         opaque, QED_AIOCB_WRITE);
+}
+
+typedef struct {
+    Coroutine *co;
+    int ret;
+    bool done;
+} QEDWriteZeroesCB;
+
+static void coroutine_fn qed_co_write_zeroes_cb(void *opaque, int ret)
+{
+    QEDWriteZeroesCB *cb = opaque;
+
+    cb->done = true;
+    cb->ret = ret;
+    if (cb->co) {
+        qemu_coroutine_enter(cb->co, NULL);
+    }
+}
+
+static int coroutine_fn bdrv_qed_co_write_zeroes(BlockDriverState *bs,
+                                                 int64_t sector_num,
+                                                 int nb_sectors)
+{
+    BlockDriverAIOCB *blockacb;
+    BDRVQEDState *s = bs->opaque;
+    QEDWriteZeroesCB cb = { .done = false };
+    QEMUIOVector qiov;
+    struct iovec iov;
+
+    /* Refuse if there are untouched backing file sectors */
+    if (bs->backing_hd) {
+        if (qed_offset_into_cluster(s, sector_num * BDRV_SECTOR_SIZE) != 0) {
+            return -ENOTSUP;
+        }
+        if (qed_offset_into_cluster(s, nb_sectors * BDRV_SECTOR_SIZE) != 0) {
+            return -ENOTSUP;
+        }
+    }
+
+    /* Zero writes start without an I/O buffer.  If a buffer becomes necessary
+     * then it will be allocated during request processing.
+     */
+    iov.iov_base = NULL,
+    iov.iov_len  = nb_sectors * BDRV_SECTOR_SIZE,
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+    blockacb = qed_aio_setup(bs, sector_num, &qiov, nb_sectors,
+                             qed_co_write_zeroes_cb, &cb,
+                             QED_AIOCB_WRITE | QED_AIOCB_ZERO);
+    if (!blockacb) {
+        return -EIO;
+    }
+    if (!cb.done) {
+        cb.co = qemu_coroutine_self();
+        qemu_coroutine_yield();
+    }
+    assert(cb.done);
+    return cb.ret;
+}
+
+static int bdrv_qed_truncate(BlockDriverState *bs, int64_t offset)
+{
+    BDRVQEDState *s = bs->opaque;
+    uint64_t old_image_size;
+    int ret;
+
+    if (!qed_is_image_size_valid(offset, s->header.cluster_size,
+                                 s->header.table_size)) {
+        return -EINVAL;
+    }
+
+    /* Shrinking is currently not supported */
+    if ((uint64_t)offset < s->header.image_size) {
+        return -ENOTSUP;
+    }
+
+    old_image_size = s->header.image_size;
+    s->header.image_size = offset;
+    ret = qed_write_header_sync(s);
+    if (ret < 0) {
+        s->header.image_size = old_image_size;
+    }
+    return ret;
+}
+
+static int64_t bdrv_qed_getlength(BlockDriverState *bs)
+{
+    BDRVQEDState *s = bs->opaque;
+    return s->header.image_size;
+}
+
+static int bdrv_qed_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
+{
+    BDRVQEDState *s = bs->opaque;
+
+    memset(bdi, 0, sizeof(*bdi));
+    bdi->cluster_size = s->header.cluster_size;
+    bdi->is_dirty = s->header.features & QED_F_NEED_CHECK;
+    return 0;
+}
+
+static int bdrv_qed_change_backing_file(BlockDriverState *bs,
+                                        const char *backing_file,
+                                        const char *backing_fmt)
+{
+    BDRVQEDState *s = bs->opaque;
+    QEDHeader new_header, le_header;
+    void *buffer;
+    size_t buffer_len, backing_file_len;
+    int ret;
+
+    /* Refuse to set backing filename if unknown compat feature bits are
+     * active.  If the image uses an unknown compat feature then we may not
+     * know the layout of data following the header structure and cannot safely
+     * add a new string.
+     */
+    if (backing_file && (s->header.compat_features &
+                         ~QED_COMPAT_FEATURE_MASK)) {
+        return -ENOTSUP;
+    }
+
+    memcpy(&new_header, &s->header, sizeof(new_header));
+
+    new_header.features &= ~(QED_F_BACKING_FILE |
+                             QED_F_BACKING_FORMAT_NO_PROBE);
+
+    /* Adjust feature flags */
+    if (backing_file) {
+        new_header.features |= QED_F_BACKING_FILE;
+
+        if (qed_fmt_is_raw(backing_fmt)) {
+            new_header.features |= QED_F_BACKING_FORMAT_NO_PROBE;
+        }
+    }
+
+    /* Calculate new header size */
+    backing_file_len = 0;
+
+    if (backing_file) {
+        backing_file_len = strlen(backing_file);
+    }
+
+    buffer_len = sizeof(new_header);
+    new_header.backing_filename_offset = buffer_len;
+    new_header.backing_filename_size = backing_file_len;
+    buffer_len += backing_file_len;
+
+    /* Make sure we can rewrite header without failing */
+    if (buffer_len > new_header.header_size * new_header.cluster_size) {
+        return -ENOSPC;
+    }
+
+    /* Prepare new header */
+    buffer = g_malloc(buffer_len);
+
+    qed_header_cpu_to_le(&new_header, &le_header);
+    memcpy(buffer, &le_header, sizeof(le_header));
+    buffer_len = sizeof(le_header);
+
+    if (backing_file) {
+        memcpy(buffer + buffer_len, backing_file, backing_file_len);
+        buffer_len += backing_file_len;
+    }
+
+    /* Write new header */
+    ret = bdrv_pwrite_sync(bs->file, 0, buffer, buffer_len);
+    g_free(buffer);
+    if (ret == 0) {
+        memcpy(&s->header, &new_header, sizeof(new_header));
+    }
+    return ret;
+}
+
+static void bdrv_qed_invalidate_cache(BlockDriverState *bs)
+{
+    BDRVQEDState *s = bs->opaque;
+
+    bdrv_qed_close(bs);
+    memset(s, 0, sizeof(BDRVQEDState));
+    bdrv_qed_open(bs, NULL, bs->open_flags);
+}
+
+static int bdrv_qed_check(BlockDriverState *bs, BdrvCheckResult *result,
+                          BdrvCheckMode fix)
+{
+    BDRVQEDState *s = bs->opaque;
+
+    return qed_check(s, result, !!fix);
+}
+
+static QEMUOptionParameter qed_create_options[] = {
+    {
+        .name = BLOCK_OPT_SIZE,
+        .type = OPT_SIZE,
+        .help = "Virtual disk size (in bytes)"
+    }, {
+        .name = BLOCK_OPT_BACKING_FILE,
+        .type = OPT_STRING,
+        .help = "File name of a base image"
+    }, {
+        .name = BLOCK_OPT_BACKING_FMT,
+        .type = OPT_STRING,
+        .help = "Image format of the base image"
+    }, {
+        .name = BLOCK_OPT_CLUSTER_SIZE,
+        .type = OPT_SIZE,
+        .help = "Cluster size (in bytes)",
+        .value = { .n = QED_DEFAULT_CLUSTER_SIZE },
+    }, {
+        .name = BLOCK_OPT_TABLE_SIZE,
+        .type = OPT_SIZE,
+        .help = "L1/L2 table size (in clusters)"
+    },
+    { /* end of list */ }
+};
+
+static BlockDriver bdrv_qed = {
+    .format_name              = "qed",
+    .instance_size            = sizeof(BDRVQEDState),
+    .create_options           = qed_create_options,
+
+    .bdrv_probe               = bdrv_qed_probe,
+    .bdrv_rebind              = bdrv_qed_rebind,
+    .bdrv_open                = bdrv_qed_open,
+    .bdrv_close               = bdrv_qed_close,
+    .bdrv_reopen_prepare      = bdrv_qed_reopen_prepare,
+    .bdrv_create              = bdrv_qed_create,
+    .bdrv_has_zero_init       = bdrv_has_zero_init_1,
+    .bdrv_co_is_allocated     = bdrv_qed_co_is_allocated,
+    .bdrv_make_empty          = bdrv_qed_make_empty,
+    .bdrv_aio_readv           = bdrv_qed_aio_readv,
+    .bdrv_aio_writev          = bdrv_qed_aio_writev,
+    .bdrv_co_write_zeroes     = bdrv_qed_co_write_zeroes,
+    .bdrv_truncate            = bdrv_qed_truncate,
+    .bdrv_getlength           = bdrv_qed_getlength,
+    .bdrv_get_info            = bdrv_qed_get_info,
+    .bdrv_change_backing_file = bdrv_qed_change_backing_file,
+    .bdrv_invalidate_cache    = bdrv_qed_invalidate_cache,
+    .bdrv_check               = bdrv_qed_check,
+};
+
+static void bdrv_qed_init(void)
+{
+    bdrv_register(&bdrv_qed);
+}
+
+block_init(bdrv_qed_init);
diff --git a/contrib/qemu/block/qed.h b/contrib/qemu/block/qed.h
new file mode 100644
index 000000000..2b4ddedf3
--- /dev/null
+++ b/contrib/qemu/block/qed.h
@@ -0,0 +1,344 @@
+/*
+ * QEMU Enhanced Disk Format
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef BLOCK_QED_H
+#define BLOCK_QED_H
+
+#include "block/block_int.h"
+
+/* The layout of a QED file is as follows:
+ *
+ * +--------+----------+----------+----------+-----+
+ * | header | L1 table | cluster0 | cluster1 | ... |
+ * +--------+----------+----------+----------+-----+
+ *
+ * There is a 2-level pagetable for cluster allocation:
+ *
+ *                     +----------+
+ *                     | L1 table |
+ *                     +----------+
+ *                ,------'  |  '------.
+ *           +----------+   |    +----------+
+ *           | L2 table |  ...   | L2 table |
+ *           +----------+        +----------+
+ *       ,------'  |  '------.
+ *  +----------+   |    +----------+
+ *  |   Data   |  ...   |   Data   |
+ *  +----------+        +----------+
+ *
+ * The L1 table is fixed size and always present.  L2 tables are allocated on
+ * demand.  The L1 table size determines the maximum possible image size; it
+ * can be influenced using the cluster_size and table_size values.
+ *
+ * All fields are little-endian on disk.
+ */
+
+enum {
+    QED_MAGIC = 'Q' | 'E' << 8 | 'D' << 16 | '\0' << 24,
+
+    /* The image supports a backing file */
+    QED_F_BACKING_FILE = 0x01,
+
+    /* The image needs a consistency check before use */
+    QED_F_NEED_CHECK = 0x02,
+
+    /* The backing file format must not be probed, treat as raw image */
+    QED_F_BACKING_FORMAT_NO_PROBE = 0x04,
+
+    /* Feature bits must be used when the on-disk format changes */
+    QED_FEATURE_MASK = QED_F_BACKING_FILE | /* supported feature bits */
+                       QED_F_NEED_CHECK |
+                       QED_F_BACKING_FORMAT_NO_PROBE,
+    QED_COMPAT_FEATURE_MASK = 0,            /* supported compat feature bits */
+    QED_AUTOCLEAR_FEATURE_MASK = 0,         /* supported autoclear feature bits */
+
+    /* Data is stored in groups of sectors called clusters.  Cluster size must
+     * be large to avoid keeping too much metadata.  I/O requests that have
+     * sub-cluster size will require read-modify-write.
+     */
+    QED_MIN_CLUSTER_SIZE = 4 * 1024, /* in bytes */
+    QED_MAX_CLUSTER_SIZE = 64 * 1024 * 1024,
+    QED_DEFAULT_CLUSTER_SIZE = 64 * 1024,
+
+    /* Allocated clusters are tracked using a 2-level pagetable.  Table size is
+     * a multiple of clusters so large maximum image sizes can be supported
+     * without jacking up the cluster size too much.
+     */
+    QED_MIN_TABLE_SIZE = 1,        /* in clusters */
+    QED_MAX_TABLE_SIZE = 16,
+    QED_DEFAULT_TABLE_SIZE = 4,
+
+    /* Delay to flush and clean image after last allocating write completes */
+    QED_NEED_CHECK_TIMEOUT = 5,    /* in seconds */
+};
+
+typedef struct {
+    uint32_t magic;                 /* QED\0 */
+
+    uint32_t cluster_size;          /* in bytes */
+    uint32_t table_size;            /* for L1 and L2 tables, in clusters */
+    uint32_t header_size;           /* in clusters */
+
+    uint64_t features;              /* format feature bits */
+    uint64_t compat_features;       /* compatible feature bits */
+    uint64_t autoclear_features;    /* self-resetting feature bits */
+
+    uint64_t l1_table_offset;       /* in bytes */
+    uint64_t image_size;            /* total logical image size, in bytes */
+
+    /* if (features & QED_F_BACKING_FILE) */
+    uint32_t backing_filename_offset; /* in bytes from start of header */
+    uint32_t backing_filename_size;   /* in bytes */
+} QEDHeader;
+
+typedef struct {
+    uint64_t offsets[0];            /* in bytes */
+} QEDTable;
+
+/* The L2 cache is a simple write-through cache for L2 structures */
+typedef struct CachedL2Table {
+    QEDTable *table;
+    uint64_t offset;    /* offset=0 indicates an invalidate entry */
+    QTAILQ_ENTRY(CachedL2Table) node;
+    int ref;
+} CachedL2Table;
+
+typedef struct {
+    QTAILQ_HEAD(, CachedL2Table) entries;
+    unsigned int n_entries;
+} L2TableCache;
+
+typedef struct QEDRequest {
+    CachedL2Table *l2_table;
+} QEDRequest;
+
+enum {
+    QED_AIOCB_WRITE = 0x0001,       /* read or write? */
+    QED_AIOCB_ZERO  = 0x0002,       /* zero write, used with QED_AIOCB_WRITE */
+};
+
+typedef struct QEDAIOCB {
+    BlockDriverAIOCB common;
+    QEMUBH *bh;
+    int bh_ret;                     /* final return status for completion bh */
+    QSIMPLEQ_ENTRY(QEDAIOCB) next;  /* next request */
+    int flags;                      /* QED_AIOCB_* bits ORed together */
+    bool *finished;                 /* signal for cancel completion */
+    uint64_t end_pos;               /* request end on block device, in bytes */
+
+    /* User scatter-gather list */
+    QEMUIOVector *qiov;
+    size_t qiov_offset;             /* byte count already processed */
+
+    /* Current cluster scatter-gather list */
+    QEMUIOVector cur_qiov;
+    uint64_t cur_pos;               /* position on block device, in bytes */
+    uint64_t cur_cluster;           /* cluster offset in image file */
+    unsigned int cur_nclusters;     /* number of clusters being accessed */
+    int find_cluster_ret;           /* used for L1/L2 update */
+
+    QEDRequest request;
+} QEDAIOCB;
+
+typedef struct {
+    BlockDriverState *bs;           /* device */
+    uint64_t file_size;             /* length of image file, in bytes */
+
+    QEDHeader header;               /* always cpu-endian */
+    QEDTable *l1_table;
+    L2TableCache l2_cache;          /* l2 table cache */
+    uint32_t table_nelems;
+    uint32_t l1_shift;
+    uint32_t l2_shift;
+    uint32_t l2_mask;
+
+    /* Allocating write request queue */
+    QSIMPLEQ_HEAD(, QEDAIOCB) allocating_write_reqs;
+    bool allocating_write_reqs_plugged;
+
+    /* Periodic flush and clear need check flag */
+    QEMUTimer *need_check_timer;
+} BDRVQEDState;
+
+enum {
+    QED_CLUSTER_FOUND,         /* cluster found */
+    QED_CLUSTER_ZERO,          /* zero cluster found */
+    QED_CLUSTER_L2,            /* cluster missing in L2 */
+    QED_CLUSTER_L1,            /* cluster missing in L1 */
+};
+
+/**
+ * qed_find_cluster() completion callback
+ *
+ * @opaque:     User data for completion callback
+ * @ret:        QED_CLUSTER_FOUND   Success
+ *              QED_CLUSTER_L2      Data cluster unallocated in L2
+ *              QED_CLUSTER_L1      L2 unallocated in L1
+ *              -errno              POSIX error occurred
+ * @offset:     Data cluster offset
+ * @len:        Contiguous bytes starting from cluster offset
+ *
+ * This function is invoked when qed_find_cluster() completes.
+ *
+ * On success ret is QED_CLUSTER_FOUND and offset/len are a contiguous range
+ * in the image file.
+ *
+ * On failure ret is QED_CLUSTER_L2 or QED_CLUSTER_L1 for missing L2 or L1
+ * table offset, respectively.  len is number of contiguous unallocated bytes.
+ */
+typedef void QEDFindClusterFunc(void *opaque, int ret, uint64_t offset, size_t len);
+
+/**
+ * Generic callback for chaining async callbacks
+ */
+typedef struct {
+    BlockDriverCompletionFunc *cb;
+    void *opaque;
+} GenericCB;
+
+void *gencb_alloc(size_t len, BlockDriverCompletionFunc *cb, void *opaque);
+void gencb_complete(void *opaque, int ret);
+
+/**
+ * Header functions
+ */
+int qed_write_header_sync(BDRVQEDState *s);
+
+/**
+ * L2 cache functions
+ */
+void qed_init_l2_cache(L2TableCache *l2_cache);
+void qed_free_l2_cache(L2TableCache *l2_cache);
+CachedL2Table *qed_alloc_l2_cache_entry(L2TableCache *l2_cache);
+void qed_unref_l2_cache_entry(CachedL2Table *entry);
+CachedL2Table *qed_find_l2_cache_entry(L2TableCache *l2_cache, uint64_t offset);
+void qed_commit_l2_cache_entry(L2TableCache *l2_cache, CachedL2Table *l2_table);
+
+/**
+ * Table I/O functions
+ */
+int qed_read_l1_table_sync(BDRVQEDState *s);
+void qed_write_l1_table(BDRVQEDState *s, unsigned int index, unsigned int n,
+                        BlockDriverCompletionFunc *cb, void *opaque);
+int qed_write_l1_table_sync(BDRVQEDState *s, unsigned int index,
+                            unsigned int n);
+int qed_read_l2_table_sync(BDRVQEDState *s, QEDRequest *request,
+                           uint64_t offset);
+void qed_read_l2_table(BDRVQEDState *s, QEDRequest *request, uint64_t offset,
+                       BlockDriverCompletionFunc *cb, void *opaque);
+void qed_write_l2_table(BDRVQEDState *s, QEDRequest *request,
+                        unsigned int index, unsigned int n, bool flush,
+                        BlockDriverCompletionFunc *cb, void *opaque);
+int qed_write_l2_table_sync(BDRVQEDState *s, QEDRequest *request,
+                            unsigned int index, unsigned int n, bool flush);
+
+/**
+ * Cluster functions
+ */
+void qed_find_cluster(BDRVQEDState *s, QEDRequest *request, uint64_t pos,
+                      size_t len, QEDFindClusterFunc *cb, void *opaque);
+
+/**
+ * Consistency check
+ */
+int qed_check(BDRVQEDState *s, BdrvCheckResult *result, bool fix);
+
+QEDTable *qed_alloc_table(BDRVQEDState *s);
+
+/**
+ * Round down to the start of a cluster
+ */
+static inline uint64_t qed_start_of_cluster(BDRVQEDState *s, uint64_t offset)
+{
+    return offset & ~(uint64_t)(s->header.cluster_size - 1);
+}
+
+static inline uint64_t qed_offset_into_cluster(BDRVQEDState *s, uint64_t offset)
+{
+    return offset & (s->header.cluster_size - 1);
+}
+
+static inline uint64_t qed_bytes_to_clusters(BDRVQEDState *s, uint64_t bytes)
+{
+    return qed_start_of_cluster(s, bytes + (s->header.cluster_size - 1)) /
+           (s->header.cluster_size - 1);
+}
+
+static inline unsigned int qed_l1_index(BDRVQEDState *s, uint64_t pos)
+{
+    return pos >> s->l1_shift;
+}
+
+static inline unsigned int qed_l2_index(BDRVQEDState *s, uint64_t pos)
+{
+    return (pos >> s->l2_shift) & s->l2_mask;
+}
+
+/**
+ * Test if a cluster offset is valid
+ */
+static inline bool qed_check_cluster_offset(BDRVQEDState *s, uint64_t offset)
+{
+    uint64_t header_size = (uint64_t)s->header.header_size *
+                           s->header.cluster_size;
+
+    if (offset & (s->header.cluster_size - 1)) {
+        return false;
+    }
+    return offset >= header_size && offset < s->file_size;
+}
+
+/**
+ * Test if a table offset is valid
+ */
+static inline bool qed_check_table_offset(BDRVQEDState *s, uint64_t offset)
+{
+    uint64_t end_offset = offset + (s->header.table_size - 1) *
+                          s->header.cluster_size;
+
+    /* Overflow check */
+    if (end_offset <= offset) {
+        return false;
+    }
+
+    return qed_check_cluster_offset(s, offset) &&
+           qed_check_cluster_offset(s, end_offset);
+}
+
+static inline bool qed_offset_is_cluster_aligned(BDRVQEDState *s,
+                                                 uint64_t offset)
+{
+    if (qed_offset_into_cluster(s, offset)) {
+        return false;
+    }
+    return true;
+}
+
+static inline bool qed_offset_is_unalloc_cluster(uint64_t offset)
+{
+    if (offset == 0) {
+        return true;
+    }
+    return false;
+}
+
+static inline bool qed_offset_is_zero_cluster(uint64_t offset)
+{
+    if (offset == 1) {
+        return true;
+    }
+    return false;
+}
+
+#endif /* BLOCK_QED_H */
diff --git a/contrib/qemu/block/snapshot.c b/contrib/qemu/block/snapshot.c
new file mode 100644
index 000000000..6c6d9deea
--- /dev/null
+++ b/contrib/qemu/block/snapshot.c
@@ -0,0 +1,157 @@
+/*
+ * Block layer snapshot related functions
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "block/snapshot.h"
+#include "block/block_int.h"
+
+int bdrv_snapshot_find(BlockDriverState *bs, QEMUSnapshotInfo *sn_info,
+                       const char *name)
+{
+    QEMUSnapshotInfo *sn_tab, *sn;
+    int nb_sns, i, ret;
+
+    ret = -ENOENT;
+    nb_sns = bdrv_snapshot_list(bs, &sn_tab);
+    if (nb_sns < 0) {
+        return ret;
+    }
+    for (i = 0; i < nb_sns; i++) {
+        sn = &sn_tab[i];
+        if (!strcmp(sn->id_str, name) || !strcmp(sn->name, name)) {
+            *sn_info = *sn;
+            ret = 0;
+            break;
+        }
+    }
+    g_free(sn_tab);
+    return ret;
+}
+
+int bdrv_can_snapshot(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+    if (!drv || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
+        return 0;
+    }
+
+    if (!drv->bdrv_snapshot_create) {
+        if (bs->file != NULL) {
+            return bdrv_can_snapshot(bs->file);
+        }
+        return 0;
+    }
+
+    return 1;
+}
+
+int bdrv_snapshot_create(BlockDriverState *bs,
+                         QEMUSnapshotInfo *sn_info)
+{
+    BlockDriver *drv = bs->drv;
+    if (!drv) {
+        return -ENOMEDIUM;
+    }
+    if (drv->bdrv_snapshot_create) {
+        return drv->bdrv_snapshot_create(bs, sn_info);
+    }
+    if (bs->file) {
+        return bdrv_snapshot_create(bs->file, sn_info);
+    }
+    return -ENOTSUP;
+}
+
+int bdrv_snapshot_goto(BlockDriverState *bs,
+                       const char *snapshot_id)
+{
+    BlockDriver *drv = bs->drv;
+    int ret, open_ret;
+
+    if (!drv) {
+        return -ENOMEDIUM;
+    }
+    if (drv->bdrv_snapshot_goto) {
+        return drv->bdrv_snapshot_goto(bs, snapshot_id);
+    }
+
+    if (bs->file) {
+        drv->bdrv_close(bs);
+        ret = bdrv_snapshot_goto(bs->file, snapshot_id);
+        open_ret = drv->bdrv_open(bs, NULL, bs->open_flags);
+        if (open_ret < 0) {
+            bdrv_delete(bs->file);
+            bs->drv = NULL;
+            return open_ret;
+        }
+        return ret;
+    }
+
+    return -ENOTSUP;
+}
+
+int bdrv_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
+{
+    BlockDriver *drv = bs->drv;
+    if (!drv) {
+        return -ENOMEDIUM;
+    }
+    if (drv->bdrv_snapshot_delete) {
+        return drv->bdrv_snapshot_delete(bs, snapshot_id);
+    }
+    if (bs->file) {
+        return bdrv_snapshot_delete(bs->file, snapshot_id);
+    }
+    return -ENOTSUP;
+}
+
+int bdrv_snapshot_list(BlockDriverState *bs,
+                       QEMUSnapshotInfo **psn_info)
+{
+    BlockDriver *drv = bs->drv;
+    if (!drv) {
+        return -ENOMEDIUM;
+    }
+    if (drv->bdrv_snapshot_list) {
+        return drv->bdrv_snapshot_list(bs, psn_info);
+    }
+    if (bs->file) {
+        return bdrv_snapshot_list(bs->file, psn_info);
+    }
+    return -ENOTSUP;
+}
+
+int bdrv_snapshot_load_tmp(BlockDriverState *bs,
+        const char *snapshot_name)
+{
+    BlockDriver *drv = bs->drv;
+    if (!drv) {
+        return -ENOMEDIUM;
+    }
+    if (!bs->read_only) {
+        return -EINVAL;
+    }
+    if (drv->bdrv_snapshot_load_tmp) {
+        return drv->bdrv_snapshot_load_tmp(bs, snapshot_name);
+    }
+    return -ENOTSUP;
+}
diff --git a/contrib/qemu/config-host.h b/contrib/qemu/config-host.h
new file mode 100644
index 000000000..46b1595a8
--- /dev/null
+++ b/contrib/qemu/config-host.h
@@ -0,0 +1,73 @@
+/* Automatically generated by create_config - do not modify */
+#define CONFIG_QEMU_CONFDIR "/usr/local/etc/qemu"
+#define CONFIG_QEMU_DATADIR "/usr/local/share/qemu"
+#define CONFIG_QEMU_DOCDIR "/usr/local/share/doc/qemu"
+#define CONFIG_QEMU_LOCALSTATEDIR "/usr/local/var"
+#define CONFIG_QEMU_HELPERDIR "/usr/local/libexec"
+#define CONFIG_QEMU_LOCALEDIR "/usr/local/share/locale"
+#define HOST_X86_64 1
+#define CONFIG_QEMU_LDST_OPTIMIZATION 1
+#define CONFIG_POSIX 1
+#define CONFIG_LINUX 1
+#define CONFIG_SLIRP 1
+#define CONFIG_SMBD_COMMAND "/usr/sbin/smbd"
+#define CONFIG_AUDIO_DRIVERS \
+    &oss_audio_driver,\
+
+#define CONFIG_OSS 1
+#define CONFIG_BDRV_RW_WHITELIST\
+    NULL
+#define CONFIG_BDRV_RO_WHITELIST\
+    NULL
+#define CONFIG_VNC 1
+#define CONFIG_VNC_TLS 1
+#define CONFIG_VNC_SASL 1
+#define CONFIG_VNC_WS 1
+#define CONFIG_FNMATCH 1
+#define CONFIG_UUID 1
+#define CONFIG_XFS 1
+#define QEMU_VERSION "1.5.50"
+#define QEMU_PKGVERSION ""
+#define CONFIG_CURSES 1
+#define CONFIG_UTIMENSAT 1
+#define CONFIG_PIPE2 1
+#define CONFIG_ACCEPT4 1
+#define CONFIG_SPLICE 1
+#define CONFIG_EVENTFD 1
+#define CONFIG_FALLOCATE 1
+#define CONFIG_FALLOCATE_PUNCH_HOLE 1
+#define CONFIG_SYNC_FILE_RANGE 1
+#define CONFIG_FIEMAP 1
+#define CONFIG_DUP3 1
+#define CONFIG_EPOLL 1
+#define CONFIG_EPOLL_CREATE1 1
+#define CONFIG_EPOLL_PWAIT 1
+#define CONFIG_SENDFILE 1
+#define CONFIG_INOTIFY 1
+#define CONFIG_INOTIFY1 1
+#define CONFIG_BYTESWAP_H 1
+#define CONFIG_CURL 1
+#define CONFIG_LINUX_AIO 1
+#define CONFIG_ATTR 1
+#define CONFIG_VHOST_SCSI 1
+#define CONFIG_IOVEC 1
+#define CONFIG_PREADV 1
+#define CONFIG_FDT 1
+#define CONFIG_SIGNALFD 1
+#define CONFIG_FDATASYNC 1
+#define CONFIG_MADVISE 1
+#define CONFIG_POSIX_MADVISE 1
+#define CONFIG_SIGEV_THREAD_ID 1
+#define CONFIG_UNAME_RELEASE ""
+#define CONFIG_QOM_CAST_DEBUG 1
+#define CONFIG_COROUTINE_BACKEND ucontext
+#define CONFIG_OPEN_BY_HANDLE 1
+#define CONFIG_LINUX_MAGIC_H 1
+#define CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE 1
+#define CONFIG_HAS_ENVIRON 1
+#define CONFIG_CPUID_H 1
+#define CONFIG_INT128 1
+#define CONFIG_VIRTIO_BLK_DATA_PLANE $(CONFIG_VIRTIO)
+#define CONFIG_TRACE_NOP 1
+#define CONFIG_TRACE_FILE trace
+#define CONFIG_TRACE_DEFAULT 1
diff --git a/contrib/qemu/coroutine-ucontext.c b/contrib/qemu/coroutine-ucontext.c
new file mode 100644
index 000000000..4bf2cde27
--- /dev/null
+++ b/contrib/qemu/coroutine-ucontext.c
@@ -0,0 +1,225 @@
+/*
+ * ucontext coroutine initialization code
+ *
+ * Copyright (C) 2006  Anthony Liguori <anthony@codemonkey.ws>
+ * Copyright (C) 2011  Kevin Wolf <kwolf@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.0 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */
+#ifdef _FORTIFY_SOURCE
+#undef _FORTIFY_SOURCE
+#endif
+#include <stdlib.h>
+#include <setjmp.h>
+#include <stdint.h>
+#include <pthread.h>
+#include <ucontext.h>
+#include "qemu-common.h"
+#include "block/coroutine_int.h"
+
+#ifdef CONFIG_VALGRIND_H
+#include <valgrind/valgrind.h>
+#endif
+
+typedef struct {
+    Coroutine base;
+    void *stack;
+    sigjmp_buf env;
+
+#ifdef CONFIG_VALGRIND_H
+    unsigned int valgrind_stack_id;
+#endif
+
+} CoroutineUContext;
+
+/**
+ * Per-thread coroutine bookkeeping
+ */
+typedef struct {
+    /** Currently executing coroutine */
+    Coroutine *current;
+
+    /** The default coroutine */
+    CoroutineUContext leader;
+} CoroutineThreadState;
+
+static pthread_key_t thread_state_key;
+
+/*
+ * va_args to makecontext() must be type 'int', so passing
+ * the pointer we need may require several int args. This
+ * union is a quick hack to let us do that
+ */
+union cc_arg {
+    void *p;
+    int i[2];
+};
+
+static CoroutineThreadState *coroutine_get_thread_state(void)
+{
+    CoroutineThreadState *s = pthread_getspecific(thread_state_key);
+
+    if (!s) {
+        s = g_malloc0(sizeof(*s));
+        s->current = &s->leader.base;
+        pthread_setspecific(thread_state_key, s);
+    }
+    return s;
+}
+
+static void qemu_coroutine_thread_cleanup(void *opaque)
+{
+    CoroutineThreadState *s = opaque;
+
+    g_free(s);
+}
+
+static void __attribute__((constructor)) coroutine_init(void)
+{
+    int ret;
+
+    ret = pthread_key_create(&thread_state_key, qemu_coroutine_thread_cleanup);
+    if (ret != 0) {
+        fprintf(stderr, "unable to create leader key: %s\n", strerror(errno));
+        abort();
+    }
+}
+
+static void coroutine_trampoline(int i0, int i1)
+{
+    union cc_arg arg;
+    CoroutineUContext *self;
+    Coroutine *co;
+
+    arg.i[0] = i0;
+    arg.i[1] = i1;
+    self = arg.p;
+    co = &self->base;
+
+    /* Initialize longjmp environment and switch back the caller */
+    if (!sigsetjmp(self->env, 0)) {
+        siglongjmp(*(sigjmp_buf *)co->entry_arg, 1);
+    }
+
+    while (true) {
+        co->entry(co->entry_arg);
+        qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
+    }
+}
+
+Coroutine *qemu_coroutine_new(void)
+{
+    const size_t stack_size = 1 << 20;
+    CoroutineUContext *co;
+    ucontext_t old_uc, uc;
+    sigjmp_buf old_env;
+    union cc_arg arg = {0};
+
+    /* The ucontext functions preserve signal masks which incurs a
+     * system call overhead.  sigsetjmp(buf, 0)/siglongjmp() does not
+     * preserve signal masks but only works on the current stack.
+     * Since we need a way to create and switch to a new stack, use
+     * the ucontext functions for that but sigsetjmp()/siglongjmp() for
+     * everything else.
+     */
+
+    if (getcontext(&uc) == -1) {
+        abort();
+    }
+
+    co = g_malloc0(sizeof(*co));
+    co->stack = g_malloc(stack_size);
+    co->base.entry_arg = &old_env; /* stash away our jmp_buf */
+
+    uc.uc_link = &old_uc;
+    uc.uc_stack.ss_sp = co->stack;
+    uc.uc_stack.ss_size = stack_size;
+    uc.uc_stack.ss_flags = 0;
+
+#ifdef CONFIG_VALGRIND_H
+    co->valgrind_stack_id =
+        VALGRIND_STACK_REGISTER(co->stack, co->stack + stack_size);
+#endif
+
+    arg.p = co;
+
+    makecontext(&uc, (void (*)(void))coroutine_trampoline,
+                2, arg.i[0], arg.i[1]);
+
+    /* swapcontext() in, siglongjmp() back out */
+    if (!sigsetjmp(old_env, 0)) {
+        swapcontext(&old_uc, &uc);
+    }
+    return &co->base;
+}
+
+#ifdef CONFIG_VALGRIND_H
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+/* Work around an unused variable in the valgrind.h macro... */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
+#endif
+static inline void valgrind_stack_deregister(CoroutineUContext *co)
+{
+    VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id);
+}
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+#pragma GCC diagnostic pop
+#endif
+#endif
+
+void qemu_coroutine_delete(Coroutine *co_)
+{
+    CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_);
+
+#ifdef CONFIG_VALGRIND_H
+    valgrind_stack_deregister(co);
+#endif
+
+    g_free(co->stack);
+    g_free(co);
+}
+
+CoroutineAction qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
+                                      CoroutineAction action)
+{
+    CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_);
+    CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_);
+    CoroutineThreadState *s = coroutine_get_thread_state();
+    int ret;
+
+    s->current = to_;
+
+    ret = sigsetjmp(from->env, 0);
+    if (ret == 0) {
+        siglongjmp(to->env, action);
+    }
+    return ret;
+}
+
+Coroutine *qemu_coroutine_self(void)
+{
+    CoroutineThreadState *s = coroutine_get_thread_state();
+
+    return s->current;
+}
+
+bool qemu_in_coroutine(void)
+{
+    CoroutineThreadState *s = pthread_getspecific(thread_state_key);
+
+    return s && s->current->caller;
+}
diff --git a/contrib/qemu/include/block/aio.h b/contrib/qemu/include/block/aio.h
new file mode 100644
index 000000000..183679374
--- /dev/null
+++ b/contrib/qemu/include/block/aio.h
@@ -0,0 +1,247 @@
+/*
+ * QEMU aio implementation
+ *
+ * Copyright IBM, Corp. 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_AIO_H
+#define QEMU_AIO_H
+
+#include "qemu-common.h"
+#include "qemu/queue.h"
+#include "qemu/event_notifier.h"
+
+typedef struct BlockDriverAIOCB BlockDriverAIOCB;
+typedef void BlockDriverCompletionFunc(void *opaque, int ret);
+
+typedef struct AIOCBInfo {
+    void (*cancel)(BlockDriverAIOCB *acb);
+    size_t aiocb_size;
+} AIOCBInfo;
+
+struct BlockDriverAIOCB {
+    const AIOCBInfo *aiocb_info;
+    BlockDriverState *bs;
+    BlockDriverCompletionFunc *cb;
+    void *opaque;
+};
+
+void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
+                   BlockDriverCompletionFunc *cb, void *opaque);
+void qemu_aio_release(void *p);
+
+typedef struct AioHandler AioHandler;
+typedef void QEMUBHFunc(void *opaque);
+typedef void IOHandler(void *opaque);
+
+typedef struct AioContext {
+    GSource source;
+
+    /* The list of registered AIO handlers */
+    QLIST_HEAD(, AioHandler) aio_handlers;
+
+    /* This is a simple lock used to protect the aio_handlers list.
+     * Specifically, it's used to ensure that no callbacks are removed while
+     * we're walking and dispatching callbacks.
+     */
+    int walking_handlers;
+
+    /* Anchor of the list of Bottom Halves belonging to the context */
+    struct QEMUBH *first_bh;
+
+    /* A simple lock used to protect the first_bh list, and ensure that
+     * no callbacks are removed while we're walking and dispatching callbacks.
+     */
+    int walking_bh;
+
+    /* Used for aio_notify.  */
+    EventNotifier notifier;
+
+    /* GPollFDs for aio_poll() */
+    GArray *pollfds;
+
+    /* Thread pool for performing work and receiving completion callbacks */
+    struct ThreadPool *thread_pool;
+} AioContext;
+
+/* Returns 1 if there are still outstanding AIO requests; 0 otherwise */
+typedef int (AioFlushEventNotifierHandler)(EventNotifier *e);
+
+/**
+ * aio_context_new: Allocate a new AioContext.
+ *
+ * AioContext provide a mini event-loop that can be waited on synchronously.
+ * They also provide bottom halves, a service to execute a piece of code
+ * as soon as possible.
+ */
+AioContext *aio_context_new(void);
+
+/**
+ * aio_context_ref:
+ * @ctx: The AioContext to operate on.
+ *
+ * Add a reference to an AioContext.
+ */
+void aio_context_ref(AioContext *ctx);
+
+/**
+ * aio_context_unref:
+ * @ctx: The AioContext to operate on.
+ *
+ * Drop a reference to an AioContext.
+ */
+void aio_context_unref(AioContext *ctx);
+
+/**
+ * aio_bh_new: Allocate a new bottom half structure.
+ *
+ * Bottom halves are lightweight callbacks whose invocation is guaranteed
+ * to be wait-free, thread-safe and signal-safe.  The #QEMUBH structure
+ * is opaque and must be allocated prior to its use.
+ */
+QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque);
+
+/**
+ * aio_notify: Force processing of pending events.
+ *
+ * Similar to signaling a condition variable, aio_notify forces
+ * aio_wait to exit, so that the next call will re-examine pending events.
+ * The caller of aio_notify will usually call aio_wait again very soon,
+ * or go through another iteration of the GLib main loop.  Hence, aio_notify
+ * also has the side effect of recalculating the sets of file descriptors
+ * that the main loop waits for.
+ *
+ * Calling aio_notify is rarely necessary, because for example scheduling
+ * a bottom half calls it already.
+ */
+void aio_notify(AioContext *ctx);
+
+/**
+ * aio_bh_poll: Poll bottom halves for an AioContext.
+ *
+ * These are internal functions used by the QEMU main loop.
+ */
+int aio_bh_poll(AioContext *ctx);
+
+/**
+ * qemu_bh_schedule: Schedule a bottom half.
+ *
+ * Scheduling a bottom half interrupts the main loop and causes the
+ * execution of the callback that was passed to qemu_bh_new.
+ *
+ * Bottom halves that are scheduled from a bottom half handler are instantly
+ * invoked.  This can create an infinite loop if a bottom half handler
+ * schedules itself.
+ *
+ * @bh: The bottom half to be scheduled.
+ */
+void qemu_bh_schedule(QEMUBH *bh);
+
+/**
+ * qemu_bh_cancel: Cancel execution of a bottom half.
+ *
+ * Canceling execution of a bottom half undoes the effect of calls to
+ * qemu_bh_schedule without freeing its resources yet.  While cancellation
+ * itself is also wait-free and thread-safe, it can of course race with the
+ * loop that executes bottom halves unless you are holding the iothread
+ * mutex.  This makes it mostly useless if you are not holding the mutex.
+ *
+ * @bh: The bottom half to be canceled.
+ */
+void qemu_bh_cancel(QEMUBH *bh);
+
+/**
+ *qemu_bh_delete: Cancel execution of a bottom half and free its resources.
+ *
+ * Deleting a bottom half frees the memory that was allocated for it by
+ * qemu_bh_new.  It also implies canceling the bottom half if it was
+ * scheduled.
+ *
+ * @bh: The bottom half to be deleted.
+ */
+void qemu_bh_delete(QEMUBH *bh);
+
+/* Return whether there are any pending callbacks from the GSource
+ * attached to the AioContext.
+ *
+ * This is used internally in the implementation of the GSource.
+ */
+bool aio_pending(AioContext *ctx);
+
+/* Progress in completing AIO work to occur.  This can issue new pending
+ * aio as a result of executing I/O completion or bh callbacks.
+ *
+ * If there is no pending AIO operation or completion (bottom half),
+ * return false.  If there are pending AIO operations of bottom halves,
+ * return true.
+ *
+ * If there are no pending bottom halves, but there are pending AIO
+ * operations, it may not be possible to make any progress without
+ * blocking.  If @blocking is true, this function will wait until one
+ * or more AIO events have completed, to ensure something has moved
+ * before returning.
+ */
+bool aio_poll(AioContext *ctx, bool blocking);
+
+#ifdef CONFIG_POSIX
+/* Returns 1 if there are still outstanding AIO requests; 0 otherwise */
+typedef int (AioFlushHandler)(void *opaque);
+
+/* Register a file descriptor and associated callbacks.  Behaves very similarly
+ * to qemu_set_fd_handler2.  Unlike qemu_set_fd_handler2, these callbacks will
+ * be invoked when using qemu_aio_wait().
+ *
+ * Code that invokes AIO completion functions should rely on this function
+ * instead of qemu_set_fd_handler[2].
+ */
+void aio_set_fd_handler(AioContext *ctx,
+                        int fd,
+                        IOHandler *io_read,
+                        IOHandler *io_write,
+                        AioFlushHandler *io_flush,
+                        void *opaque);
+#endif
+
+/* Register an event notifier and associated callbacks.  Behaves very similarly
+ * to event_notifier_set_handler.  Unlike event_notifier_set_handler, these callbacks
+ * will be invoked when using qemu_aio_wait().
+ *
+ * Code that invokes AIO completion functions should rely on this function
+ * instead of event_notifier_set_handler.
+ */
+void aio_set_event_notifier(AioContext *ctx,
+                            EventNotifier *notifier,
+                            EventNotifierHandler *io_read,
+                            AioFlushEventNotifierHandler *io_flush);
+
+/* Return a GSource that lets the main loop poll the file descriptors attached
+ * to this AioContext.
+ */
+GSource *aio_get_g_source(AioContext *ctx);
+
+/* Return the ThreadPool bound to this AioContext */
+struct ThreadPool *aio_get_thread_pool(AioContext *ctx);
+
+/* Functions to operate on the main QEMU AioContext.  */
+
+bool qemu_aio_wait(void);
+void qemu_aio_set_event_notifier(EventNotifier *notifier,
+                                 EventNotifierHandler *io_read,
+                                 AioFlushEventNotifierHandler *io_flush);
+
+#ifdef CONFIG_POSIX
+void qemu_aio_set_fd_handler(int fd,
+                             IOHandler *io_read,
+                             IOHandler *io_write,
+                             AioFlushHandler *io_flush,
+                             void *opaque);
+#endif
+
+#endif
diff --git a/contrib/qemu/include/block/block.h b/contrib/qemu/include/block/block.h
new file mode 100644
index 000000000..b6b9014a9
--- /dev/null
+++ b/contrib/qemu/include/block/block.h
@@ -0,0 +1,443 @@
+#ifndef BLOCK_H
+#define BLOCK_H
+
+#include "block/aio.h"
+#include "qemu-common.h"
+#include "qemu/option.h"
+#include "block/coroutine.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi-types.h"
+
+/* block.c */
+typedef struct BlockDriver BlockDriver;
+typedef struct BlockJob BlockJob;
+
+typedef struct BlockDriverInfo {
+    /* in bytes, 0 if irrelevant */
+    int cluster_size;
+    /* offset at which the VM state can be saved (0 if not possible) */
+    int64_t vm_state_offset;
+    bool is_dirty;
+} BlockDriverInfo;
+
+typedef struct BlockFragInfo {
+    uint64_t allocated_clusters;
+    uint64_t total_clusters;
+    uint64_t fragmented_clusters;
+    uint64_t compressed_clusters;
+} BlockFragInfo;
+
+/* Callbacks for block device models */
+typedef struct BlockDevOps {
+    /*
+     * Runs when virtual media changed (monitor commands eject, change)
+     * Argument load is true on load and false on eject.
+     * Beware: doesn't run when a host device's physical media
+     * changes.  Sure would be useful if it did.
+     * Device models with removable media must implement this callback.
+     */
+    void (*change_media_cb)(void *opaque, bool load);
+    /*
+     * Runs when an eject request is issued from the monitor, the tray
+     * is closed, and the medium is locked.
+     * Device models that do not implement is_medium_locked will not need
+     * this callback.  Device models that can lock the medium or tray might
+     * want to implement the callback and unlock the tray when "force" is
+     * true, even if they do not support eject requests.
+     */
+    void (*eject_request_cb)(void *opaque, bool force);
+    /*
+     * Is the virtual tray open?
+     * Device models implement this only when the device has a tray.
+     */
+    bool (*is_tray_open)(void *opaque);
+    /*
+     * Is the virtual medium locked into the device?
+     * Device models implement this only when device has such a lock.
+     */
+    bool (*is_medium_locked)(void *opaque);
+    /*
+     * Runs when the size changed (e.g. monitor command block_resize)
+     */
+    void (*resize_cb)(void *opaque);
+} BlockDevOps;
+
+#define BDRV_O_RDWR        0x0002
+#define BDRV_O_SNAPSHOT    0x0008 /* open the file read only and save writes in a snapshot */
+#define BDRV_O_NOCACHE     0x0020 /* do not use the host page cache */
+#define BDRV_O_CACHE_WB    0x0040 /* use write-back caching */
+#define BDRV_O_NATIVE_AIO  0x0080 /* use native AIO instead of the thread pool */
+#define BDRV_O_NO_BACKING  0x0100 /* don't open the backing file */
+#define BDRV_O_NO_FLUSH    0x0200 /* disable flushing on this disk */
+#define BDRV_O_COPY_ON_READ 0x0400 /* copy read backing sectors into image */
+#define BDRV_O_INCOMING    0x0800  /* consistency hint for incoming migration */
+#define BDRV_O_CHECK       0x1000  /* open solely for consistency check */
+#define BDRV_O_ALLOW_RDWR  0x2000  /* allow reopen to change from r/o to r/w */
+#define BDRV_O_UNMAP       0x4000  /* execute guest UNMAP/TRIM operations */
+
+#define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH)
+
+#define BDRV_SECTOR_BITS   9
+#define BDRV_SECTOR_SIZE   (1ULL << BDRV_SECTOR_BITS)
+#define BDRV_SECTOR_MASK   ~(BDRV_SECTOR_SIZE - 1)
+
+typedef enum {
+    BDRV_ACTION_REPORT, BDRV_ACTION_IGNORE, BDRV_ACTION_STOP
+} BlockErrorAction;
+
+typedef QSIMPLEQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) BlockReopenQueue;
+
+typedef struct BDRVReopenState {
+    BlockDriverState *bs;
+    int flags;
+    void *opaque;
+} BDRVReopenState;
+
+
+void bdrv_iostatus_enable(BlockDriverState *bs);
+void bdrv_iostatus_reset(BlockDriverState *bs);
+void bdrv_iostatus_disable(BlockDriverState *bs);
+bool bdrv_iostatus_is_enabled(const BlockDriverState *bs);
+void bdrv_iostatus_set_err(BlockDriverState *bs, int error);
+void bdrv_info_print(Monitor *mon, const QObject *data);
+void bdrv_info(Monitor *mon, QObject **ret_data);
+void bdrv_stats_print(Monitor *mon, const QObject *data);
+void bdrv_info_stats(Monitor *mon, QObject **ret_data);
+
+/* disk I/O throttling */
+void bdrv_io_limits_enable(BlockDriverState *bs);
+void bdrv_io_limits_disable(BlockDriverState *bs);
+bool bdrv_io_limits_enabled(BlockDriverState *bs);
+
+void bdrv_init(void);
+void bdrv_init_with_whitelist(void);
+BlockDriver *bdrv_find_protocol(const char *filename,
+                                bool allow_protocol_prefix);
+BlockDriver *bdrv_find_format(const char *format_name);
+BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
+                                          bool readonly);
+int bdrv_create(BlockDriver *drv, const char* filename,
+    QEMUOptionParameter *options);
+int bdrv_create_file(const char* filename, QEMUOptionParameter *options);
+BlockDriverState *bdrv_new(const char *device_name);
+void bdrv_make_anon(BlockDriverState *bs);
+void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old);
+void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top);
+void bdrv_delete(BlockDriverState *bs);
+int bdrv_parse_cache_flags(const char *mode, int *flags);
+int bdrv_parse_discard_flags(const char *mode, int *flags);
+int bdrv_file_open(BlockDriverState **pbs, const char *filename,
+                   QDict *options, int flags);
+int bdrv_open_backing_file(BlockDriverState *bs, QDict *options);
+int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
+              int flags, BlockDriver *drv);
+BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
+                                    BlockDriverState *bs, int flags);
+int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp);
+int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp);
+int bdrv_reopen_prepare(BDRVReopenState *reopen_state,
+                        BlockReopenQueue *queue, Error **errp);
+void bdrv_reopen_commit(BDRVReopenState *reopen_state);
+void bdrv_reopen_abort(BDRVReopenState *reopen_state);
+void bdrv_close(BlockDriverState *bs);
+void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify);
+int bdrv_attach_dev(BlockDriverState *bs, void *dev);
+void bdrv_attach_dev_nofail(BlockDriverState *bs, void *dev);
+void bdrv_detach_dev(BlockDriverState *bs, void *dev);
+void *bdrv_get_attached_dev(BlockDriverState *bs);
+void bdrv_set_dev_ops(BlockDriverState *bs, const BlockDevOps *ops,
+                      void *opaque);
+void bdrv_dev_eject_request(BlockDriverState *bs, bool force);
+bool bdrv_dev_has_removable_media(BlockDriverState *bs);
+bool bdrv_dev_is_tray_open(BlockDriverState *bs);
+bool bdrv_dev_is_medium_locked(BlockDriverState *bs);
+int bdrv_read(BlockDriverState *bs, int64_t sector_num,
+              uint8_t *buf, int nb_sectors);
+int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
+                          uint8_t *buf, int nb_sectors);
+int bdrv_write(BlockDriverState *bs, int64_t sector_num,
+               const uint8_t *buf, int nb_sectors);
+int bdrv_writev(BlockDriverState *bs, int64_t sector_num, QEMUIOVector *qiov);
+int bdrv_pread(BlockDriverState *bs, int64_t offset,
+               void *buf, int count);
+int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
+                const void *buf, int count);
+int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov);
+int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
+    const void *buf, int count);
+int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
+    int nb_sectors, QEMUIOVector *qiov);
+int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
+int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
+    int nb_sectors, QEMUIOVector *qiov);
+/*
+ * Efficiently zero a region of the disk image.  Note that this is a regular
+ * I/O request like read or write and should have a reasonable size.  This
+ * function is not suitable for zeroing the entire image in a single request
+ * because it may allocate memory for the entire region.
+ */
+int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs, int64_t sector_num,
+    int nb_sectors);
+int coroutine_fn bdrv_co_is_allocated(BlockDriverState *bs, int64_t sector_num,
+    int nb_sectors, int *pnum);
+int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top,
+                                            BlockDriverState *base,
+                                            int64_t sector_num,
+                                            int nb_sectors, int *pnum);
+BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
+    const char *backing_file);
+int bdrv_get_backing_file_depth(BlockDriverState *bs);
+int bdrv_truncate(BlockDriverState *bs, int64_t offset);
+int64_t bdrv_getlength(BlockDriverState *bs);
+int64_t bdrv_get_allocated_file_size(BlockDriverState *bs);
+void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr);
+int bdrv_commit(BlockDriverState *bs);
+int bdrv_commit_all(void);
+int bdrv_change_backing_file(BlockDriverState *bs,
+    const char *backing_file, const char *backing_fmt);
+void bdrv_register(BlockDriver *bdrv);
+int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
+                           BlockDriverState *base);
+BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
+                                    BlockDriverState *bs);
+BlockDriverState *bdrv_find_base(BlockDriverState *bs);
+
+
+typedef struct BdrvCheckResult {
+    int corruptions;
+    int leaks;
+    int check_errors;
+    int corruptions_fixed;
+    int leaks_fixed;
+    int64_t image_end_offset;
+    BlockFragInfo bfi;
+} BdrvCheckResult;
+
+typedef enum {
+    BDRV_FIX_LEAKS    = 1,
+    BDRV_FIX_ERRORS   = 2,
+} BdrvCheckMode;
+
+int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix);
+
+/* async block I/O */
+typedef void BlockDriverDirtyHandler(BlockDriverState *bs, int64_t sector,
+                                     int sector_num);
+BlockDriverAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
+                                 QEMUIOVector *iov, int nb_sectors,
+                                 BlockDriverCompletionFunc *cb, void *opaque);
+BlockDriverAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
+                                  QEMUIOVector *iov, int nb_sectors,
+                                  BlockDriverCompletionFunc *cb, void *opaque);
+BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
+                                 BlockDriverCompletionFunc *cb, void *opaque);
+BlockDriverAIOCB *bdrv_aio_discard(BlockDriverState *bs,
+                                   int64_t sector_num, int nb_sectors,
+                                   BlockDriverCompletionFunc *cb, void *opaque);
+void bdrv_aio_cancel(BlockDriverAIOCB *acb);
+
+typedef struct BlockRequest {
+    /* Fields to be filled by multiwrite caller */
+    int64_t sector;
+    int nb_sectors;
+    QEMUIOVector *qiov;
+    BlockDriverCompletionFunc *cb;
+    void *opaque;
+
+    /* Filled by multiwrite implementation */
+    int error;
+} BlockRequest;
+
+int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs,
+    int num_reqs);
+
+/* sg packet commands */
+int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf);
+BlockDriverAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
+        unsigned long int req, void *buf,
+        BlockDriverCompletionFunc *cb, void *opaque);
+
+/* Invalidate any cached metadata used by image formats */
+void bdrv_invalidate_cache(BlockDriverState *bs);
+void bdrv_invalidate_cache_all(void);
+
+void bdrv_clear_incoming_migration_all(void);
+
+/* Ensure contents are flushed to disk.  */
+int bdrv_flush(BlockDriverState *bs);
+int coroutine_fn bdrv_co_flush(BlockDriverState *bs);
+int bdrv_flush_all(void);
+void bdrv_close_all(void);
+void bdrv_drain_all(void);
+
+int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors);
+int bdrv_co_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors);
+int bdrv_has_zero_init_1(BlockDriverState *bs);
+int bdrv_has_zero_init(BlockDriverState *bs);
+int bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+                      int *pnum);
+int bdrv_is_allocated_above(BlockDriverState *top, BlockDriverState *base,
+                            int64_t sector_num, int nb_sectors, int *pnum);
+
+void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
+                       BlockdevOnError on_write_error);
+BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read);
+BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error);
+void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
+                       bool is_read, int error);
+int bdrv_is_read_only(BlockDriverState *bs);
+int bdrv_is_sg(BlockDriverState *bs);
+int bdrv_enable_write_cache(BlockDriverState *bs);
+void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce);
+int bdrv_is_inserted(BlockDriverState *bs);
+int bdrv_media_changed(BlockDriverState *bs);
+void bdrv_lock_medium(BlockDriverState *bs, bool locked);
+void bdrv_eject(BlockDriverState *bs, bool eject_flag);
+const char *bdrv_get_format_name(BlockDriverState *bs);
+BlockDriverState *bdrv_find(const char *name);
+BlockDriverState *bdrv_next(BlockDriverState *bs);
+void bdrv_iterate(void (*it)(void *opaque, BlockDriverState *bs),
+                  void *opaque);
+int bdrv_is_encrypted(BlockDriverState *bs);
+int bdrv_key_required(BlockDriverState *bs);
+int bdrv_set_key(BlockDriverState *bs, const char *key);
+int bdrv_query_missing_keys(void);
+void bdrv_iterate_format(void (*it)(void *opaque, const char *name),
+                         void *opaque);
+const char *bdrv_get_device_name(BlockDriverState *bs);
+int bdrv_get_flags(BlockDriverState *bs);
+int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
+                          const uint8_t *buf, int nb_sectors);
+int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi);
+void bdrv_round_to_clusters(BlockDriverState *bs,
+                            int64_t sector_num, int nb_sectors,
+                            int64_t *cluster_sector_num,
+                            int *cluster_nb_sectors);
+
+const char *bdrv_get_encrypted_filename(BlockDriverState *bs);
+void bdrv_get_backing_filename(BlockDriverState *bs,
+                               char *filename, int filename_size);
+void bdrv_get_full_backing_filename(BlockDriverState *bs,
+                                    char *dest, size_t sz);
+int bdrv_is_snapshot(BlockDriverState *bs);
+
+int path_is_absolute(const char *path);
+void path_combine(char *dest, int dest_size,
+                  const char *base_path,
+                  const char *filename);
+
+int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos);
+int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
+                      int64_t pos, int size);
+
+int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
+                      int64_t pos, int size);
+
+void bdrv_img_create(const char *filename, const char *fmt,
+                     const char *base_filename, const char *base_fmt,
+                     char *options, uint64_t img_size, int flags,
+                     Error **errp, bool quiet);
+
+void bdrv_set_buffer_alignment(BlockDriverState *bs, int align);
+void *qemu_blockalign(BlockDriverState *bs, size_t size);
+bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov);
+
+struct HBitmapIter;
+void bdrv_set_dirty_tracking(BlockDriverState *bs, int granularity);
+int bdrv_get_dirty(BlockDriverState *bs, int64_t sector);
+void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
+void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
+void bdrv_dirty_iter_init(BlockDriverState *bs, struct HBitmapIter *hbi);
+int64_t bdrv_get_dirty_count(BlockDriverState *bs);
+
+void bdrv_enable_copy_on_read(BlockDriverState *bs);
+void bdrv_disable_copy_on_read(BlockDriverState *bs);
+
+void bdrv_set_in_use(BlockDriverState *bs, int in_use);
+int bdrv_in_use(BlockDriverState *bs);
+
+#ifdef CONFIG_LINUX_AIO
+int raw_get_aio_fd(BlockDriverState *bs);
+#else
+static inline int raw_get_aio_fd(BlockDriverState *bs)
+{
+    return -ENOTSUP;
+}
+#endif
+
+enum BlockAcctType {
+    BDRV_ACCT_READ,
+    BDRV_ACCT_WRITE,
+    BDRV_ACCT_FLUSH,
+    BDRV_MAX_IOTYPE,
+};
+
+typedef struct BlockAcctCookie {
+    int64_t bytes;
+    int64_t start_time_ns;
+    enum BlockAcctType type;
+} BlockAcctCookie;
+
+void bdrv_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
+        int64_t bytes, enum BlockAcctType type);
+void bdrv_acct_done(BlockDriverState *bs, BlockAcctCookie *cookie);
+
+typedef enum {
+    BLKDBG_L1_UPDATE,
+
+    BLKDBG_L1_GROW_ALLOC_TABLE,
+    BLKDBG_L1_GROW_WRITE_TABLE,
+    BLKDBG_L1_GROW_ACTIVATE_TABLE,
+
+    BLKDBG_L2_LOAD,
+    BLKDBG_L2_UPDATE,
+    BLKDBG_L2_UPDATE_COMPRESSED,
+    BLKDBG_L2_ALLOC_COW_READ,
+    BLKDBG_L2_ALLOC_WRITE,
+
+    BLKDBG_READ_AIO,
+    BLKDBG_READ_BACKING_AIO,
+    BLKDBG_READ_COMPRESSED,
+
+    BLKDBG_WRITE_AIO,
+    BLKDBG_WRITE_COMPRESSED,
+
+    BLKDBG_VMSTATE_LOAD,
+    BLKDBG_VMSTATE_SAVE,
+
+    BLKDBG_COW_READ,
+    BLKDBG_COW_WRITE,
+
+    BLKDBG_REFTABLE_LOAD,
+    BLKDBG_REFTABLE_GROW,
+
+    BLKDBG_REFBLOCK_LOAD,
+    BLKDBG_REFBLOCK_UPDATE,
+    BLKDBG_REFBLOCK_UPDATE_PART,
+    BLKDBG_REFBLOCK_ALLOC,
+    BLKDBG_REFBLOCK_ALLOC_HOOKUP,
+    BLKDBG_REFBLOCK_ALLOC_WRITE,
+    BLKDBG_REFBLOCK_ALLOC_WRITE_BLOCKS,
+    BLKDBG_REFBLOCK_ALLOC_WRITE_TABLE,
+    BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE,
+
+    BLKDBG_CLUSTER_ALLOC,
+    BLKDBG_CLUSTER_ALLOC_BYTES,
+    BLKDBG_CLUSTER_FREE,
+
+    BLKDBG_FLUSH_TO_OS,
+    BLKDBG_FLUSH_TO_DISK,
+
+    BLKDBG_EVENT_MAX,
+} BlkDebugEvent;
+
+#define BLKDBG_EVENT(bs, evt) bdrv_debug_event(bs, evt)
+void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event);
+
+int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
+                           const char *tag);
+int bdrv_debug_resume(BlockDriverState *bs, const char *tag);
+bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag);
+
+#endif
diff --git a/contrib/qemu/include/block/block_int.h b/contrib/qemu/include/block/block_int.h
new file mode 100644
index 000000000..c6ac871e2
--- /dev/null
+++ b/contrib/qemu/include/block/block_int.h
@@ -0,0 +1,421 @@
+/*
+ * QEMU System Emulator block driver
+ *
+ * Copyright (c) 2003 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef BLOCK_INT_H
+#define BLOCK_INT_H
+
+#include "block/block.h"
+#include "qemu/option.h"
+#include "qemu/queue.h"
+#include "block/coroutine.h"
+#include "qemu/timer.h"
+#include "qapi-types.h"
+#include "qapi/qmp/qerror.h"
+#include "monitor/monitor.h"
+#include "qemu/hbitmap.h"
+#include "block/snapshot.h"
+
+#define BLOCK_FLAG_ENCRYPT          1
+#define BLOCK_FLAG_COMPAT6          4
+#define BLOCK_FLAG_LAZY_REFCOUNTS   8
+
+#define BLOCK_IO_LIMIT_READ     0
+#define BLOCK_IO_LIMIT_WRITE    1
+#define BLOCK_IO_LIMIT_TOTAL    2
+
+#define BLOCK_IO_SLICE_TIME     100000000
+#define NANOSECONDS_PER_SECOND  1000000000.0
+
+#define BLOCK_OPT_SIZE              "size"
+#define BLOCK_OPT_ENCRYPT           "encryption"
+#define BLOCK_OPT_COMPAT6           "compat6"
+#define BLOCK_OPT_BACKING_FILE      "backing_file"
+#define BLOCK_OPT_BACKING_FMT       "backing_fmt"
+#define BLOCK_OPT_CLUSTER_SIZE      "cluster_size"
+#define BLOCK_OPT_TABLE_SIZE        "table_size"
+#define BLOCK_OPT_PREALLOC          "preallocation"
+#define BLOCK_OPT_SUBFMT            "subformat"
+#define BLOCK_OPT_COMPAT_LEVEL      "compat"
+#define BLOCK_OPT_LAZY_REFCOUNTS    "lazy_refcounts"
+#define BLOCK_OPT_ADAPTER_TYPE      "adapter_type"
+
+typedef struct BdrvTrackedRequest {
+    BlockDriverState *bs;
+    int64_t sector_num;
+    int nb_sectors;
+    bool is_write;
+    QLIST_ENTRY(BdrvTrackedRequest) list;
+    Coroutine *co; /* owner, used for deadlock detection */
+    CoQueue wait_queue; /* coroutines blocked on this request */
+} BdrvTrackedRequest;
+
+
+typedef struct BlockIOLimit {
+    int64_t bps[3];
+    int64_t iops[3];
+} BlockIOLimit;
+
+typedef struct BlockIOBaseValue {
+    uint64_t bytes[2];
+    uint64_t ios[2];
+} BlockIOBaseValue;
+
+struct BlockDriver {
+    const char *format_name;
+    int instance_size;
+    int (*bdrv_probe)(const uint8_t *buf, int buf_size, const char *filename);
+    int (*bdrv_probe_device)(const char *filename);
+
+    /* Any driver implementing this callback is expected to be able to handle
+     * NULL file names in its .bdrv_open() implementation */
+    void (*bdrv_parse_filename)(const char *filename, QDict *options, Error **errp);
+
+    /* For handling image reopen for split or non-split files */
+    int (*bdrv_reopen_prepare)(BDRVReopenState *reopen_state,
+                               BlockReopenQueue *queue, Error **errp);
+    void (*bdrv_reopen_commit)(BDRVReopenState *reopen_state);
+    void (*bdrv_reopen_abort)(BDRVReopenState *reopen_state);
+
+    int (*bdrv_open)(BlockDriverState *bs, QDict *options, int flags);
+    int (*bdrv_file_open)(BlockDriverState *bs, QDict *options, int flags);
+    int (*bdrv_read)(BlockDriverState *bs, int64_t sector_num,
+                     uint8_t *buf, int nb_sectors);
+    int (*bdrv_write)(BlockDriverState *bs, int64_t sector_num,
+                      const uint8_t *buf, int nb_sectors);
+    void (*bdrv_close)(BlockDriverState *bs);
+    void (*bdrv_rebind)(BlockDriverState *bs);
+    int (*bdrv_create)(const char *filename, QEMUOptionParameter *options);
+    int (*bdrv_set_key)(BlockDriverState *bs, const char *key);
+    int (*bdrv_make_empty)(BlockDriverState *bs);
+    /* aio */
+    BlockDriverAIOCB *(*bdrv_aio_readv)(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque);
+    BlockDriverAIOCB *(*bdrv_aio_writev)(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque);
+    BlockDriverAIOCB *(*bdrv_aio_flush)(BlockDriverState *bs,
+        BlockDriverCompletionFunc *cb, void *opaque);
+    BlockDriverAIOCB *(*bdrv_aio_discard)(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque);
+
+    int coroutine_fn (*bdrv_co_readv)(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
+    int coroutine_fn (*bdrv_co_writev)(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
+    /*
+     * Efficiently zero a region of the disk image.  Typically an image format
+     * would use a compact metadata representation to implement this.  This
+     * function pointer may be NULL and .bdrv_co_writev() will be called
+     * instead.
+     */
+    int coroutine_fn (*bdrv_co_write_zeroes)(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors);
+    int coroutine_fn (*bdrv_co_discard)(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors);
+    int coroutine_fn (*bdrv_co_is_allocated)(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, int *pnum);
+
+    /*
+     * Invalidate any cached meta-data.
+     */
+    void (*bdrv_invalidate_cache)(BlockDriverState *bs);
+
+    /*
+     * Flushes all data that was already written to the OS all the way down to
+     * the disk (for example raw-posix calls fsync()).
+     */
+    int coroutine_fn (*bdrv_co_flush_to_disk)(BlockDriverState *bs);
+
+    /*
+     * Flushes all internal caches to the OS. The data may still sit in a
+     * writeback cache of the host OS, but it will survive a crash of the qemu
+     * process.
+     */
+    int coroutine_fn (*bdrv_co_flush_to_os)(BlockDriverState *bs);
+
+    const char *protocol_name;
+    int (*bdrv_truncate)(BlockDriverState *bs, int64_t offset);
+    int64_t (*bdrv_getlength)(BlockDriverState *bs);
+    int64_t (*bdrv_get_allocated_file_size)(BlockDriverState *bs);
+    int (*bdrv_write_compressed)(BlockDriverState *bs, int64_t sector_num,
+                                 const uint8_t *buf, int nb_sectors);
+
+    int (*bdrv_snapshot_create)(BlockDriverState *bs,
+                                QEMUSnapshotInfo *sn_info);
+    int (*bdrv_snapshot_goto)(BlockDriverState *bs,
+                              const char *snapshot_id);
+    int (*bdrv_snapshot_delete)(BlockDriverState *bs, const char *snapshot_id);
+    int (*bdrv_snapshot_list)(BlockDriverState *bs,
+                              QEMUSnapshotInfo **psn_info);
+    int (*bdrv_snapshot_load_tmp)(BlockDriverState *bs,
+                                  const char *snapshot_name);
+    int (*bdrv_get_info)(BlockDriverState *bs, BlockDriverInfo *bdi);
+
+    int (*bdrv_save_vmstate)(BlockDriverState *bs, QEMUIOVector *qiov,
+                             int64_t pos);
+    int (*bdrv_load_vmstate)(BlockDriverState *bs, uint8_t *buf,
+                             int64_t pos, int size);
+
+    int (*bdrv_change_backing_file)(BlockDriverState *bs,
+        const char *backing_file, const char *backing_fmt);
+
+    /* removable device specific */
+    int (*bdrv_is_inserted)(BlockDriverState *bs);
+    int (*bdrv_media_changed)(BlockDriverState *bs);
+    void (*bdrv_eject)(BlockDriverState *bs, bool eject_flag);
+    void (*bdrv_lock_medium)(BlockDriverState *bs, bool locked);
+
+    /* to control generic scsi devices */
+    int (*bdrv_ioctl)(BlockDriverState *bs, unsigned long int req, void *buf);
+    BlockDriverAIOCB *(*bdrv_aio_ioctl)(BlockDriverState *bs,
+        unsigned long int req, void *buf,
+        BlockDriverCompletionFunc *cb, void *opaque);
+
+    /* List of options for creating images, terminated by name == NULL */
+    QEMUOptionParameter *create_options;
+
+
+    /*
+     * Returns 0 for completed check, -errno for internal errors.
+     * The check results are stored in result.
+     */
+    int (*bdrv_check)(BlockDriverState* bs, BdrvCheckResult *result,
+        BdrvCheckMode fix);
+
+    void (*bdrv_debug_event)(BlockDriverState *bs, BlkDebugEvent event);
+
+    /* TODO Better pass a option string/QDict/QemuOpts to add any rule? */
+    int (*bdrv_debug_breakpoint)(BlockDriverState *bs, const char *event,
+        const char *tag);
+    int (*bdrv_debug_resume)(BlockDriverState *bs, const char *tag);
+    bool (*bdrv_debug_is_suspended)(BlockDriverState *bs, const char *tag);
+
+    /*
+     * Returns 1 if newly created images are guaranteed to contain only
+     * zeros, 0 otherwise.
+     */
+    int (*bdrv_has_zero_init)(BlockDriverState *bs);
+
+    QLIST_ENTRY(BlockDriver) list;
+};
+
+/*
+ * Note: the function bdrv_append() copies and swaps contents of
+ * BlockDriverStates, so if you add new fields to this struct, please
+ * inspect bdrv_append() to determine if the new fields need to be
+ * copied as well.
+ */
+struct BlockDriverState {
+    int64_t total_sectors; /* if we are reading a disk image, give its
+                              size in sectors */
+    int read_only; /* if true, the media is read only */
+    int open_flags; /* flags used to open the file, re-used for re-open */
+    int encrypted; /* if true, the media is encrypted */
+    int valid_key; /* if true, a valid encryption key has been set */
+    int sg;        /* if true, the device is a /dev/sg* */
+    int copy_on_read; /* if true, copy read backing sectors into image
+                         note this is a reference count */
+
+    BlockDriver *drv; /* NULL means no media */
+    void *opaque;
+
+    void *dev;                  /* attached device model, if any */
+    /* TODO change to DeviceState when all users are qdevified */
+    const BlockDevOps *dev_ops;
+    void *dev_opaque;
+
+    char filename[1024];
+    char backing_file[1024]; /* if non zero, the image is a diff of
+                                this file image */
+    char backing_format[16]; /* if non-zero and backing_file exists */
+    int is_temporary;
+
+    BlockDriverState *backing_hd;
+    BlockDriverState *file;
+
+    NotifierList close_notifiers;
+
+    /* Callback before write request is processed */
+    NotifierWithReturnList before_write_notifiers;
+
+    /* number of in-flight copy-on-read requests */
+    unsigned int copy_on_read_in_flight;
+
+    /* the time for latest disk I/O */
+    int64_t slice_start;
+    int64_t slice_end;
+    BlockIOLimit io_limits;
+    BlockIOBaseValue slice_submitted;
+    CoQueue      throttled_reqs;
+    QEMUTimer    *block_timer;
+    bool         io_limits_enabled;
+
+    /* I/O stats (display with "info blockstats"). */
+    uint64_t nr_bytes[BDRV_MAX_IOTYPE];
+    uint64_t nr_ops[BDRV_MAX_IOTYPE];
+    uint64_t total_time_ns[BDRV_MAX_IOTYPE];
+    uint64_t wr_highest_sector;
+
+    /* Whether the disk can expand beyond total_sectors */
+    int growable;
+
+    /* the memory alignment required for the buffers handled by this driver */
+    int buffer_alignment;
+
+    /* do we need to tell the quest if we have a volatile write cache? */
+    int enable_write_cache;
+
+    /* NOTE: the following infos are only hints for real hardware
+       drivers. They are not used by the block driver */
+    BlockdevOnError on_read_error, on_write_error;
+    bool iostatus_enabled;
+    BlockDeviceIoStatus iostatus;
+    char device_name[32];
+    HBitmap *dirty_bitmap;
+    int in_use; /* users other than guest access, eg. block migration */
+    QTAILQ_ENTRY(BlockDriverState) list;
+
+    QLIST_HEAD(, BdrvTrackedRequest) tracked_requests;
+
+    /* long-running background operation */
+    BlockJob *job;
+
+    QDict *options;
+};
+
+int get_tmp_filename(char *filename, int size);
+
+void bdrv_set_io_limits(BlockDriverState *bs,
+                        BlockIOLimit *io_limits);
+
+/**
+ * bdrv_add_before_write_notifier:
+ *
+ * Register a callback that is invoked before write requests are processed but
+ * after any throttling or waiting for overlapping requests.
+ */
+void bdrv_add_before_write_notifier(BlockDriverState *bs,
+                                    NotifierWithReturn *notifier);
+
+/**
+ * bdrv_get_aio_context:
+ *
+ * Returns: the currently bound #AioContext
+ */
+AioContext *bdrv_get_aio_context(BlockDriverState *bs);
+
+#ifdef _WIN32
+int is_windows_drive(const char *filename);
+#endif
+void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
+                               enum MonitorEvent ev,
+                               BlockErrorAction action, bool is_read);
+
+/**
+ * stream_start:
+ * @bs: Block device to operate on.
+ * @base: Block device that will become the new base, or %NULL to
+ * flatten the whole backing file chain onto @bs.
+ * @base_id: The file name that will be written to @bs as the new
+ * backing file if the job completes.  Ignored if @base is %NULL.
+ * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @on_error: The action to take upon error.
+ * @cb: Completion function for the job.
+ * @opaque: Opaque pointer value passed to @cb.
+ * @errp: Error object.
+ *
+ * Start a streaming operation on @bs.  Clusters that are unallocated
+ * in @bs, but allocated in any image between @base and @bs (both
+ * exclusive) will be written to @bs.  At the end of a successful
+ * streaming job, the backing file of @bs will be changed to
+ * @base_id in the written image and to @base in the live BlockDriverState.
+ */
+void stream_start(BlockDriverState *bs, BlockDriverState *base,
+                  const char *base_id, int64_t speed, BlockdevOnError on_error,
+                  BlockDriverCompletionFunc *cb,
+                  void *opaque, Error **errp);
+
+/**
+ * commit_start:
+ * @bs: Top Block device
+ * @base: Block device that will be written into, and become the new top
+ * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @on_error: The action to take upon error.
+ * @cb: Completion function for the job.
+ * @opaque: Opaque pointer value passed to @cb.
+ * @errp: Error object.
+ *
+ */
+void commit_start(BlockDriverState *bs, BlockDriverState *base,
+                 BlockDriverState *top, int64_t speed,
+                 BlockdevOnError on_error, BlockDriverCompletionFunc *cb,
+                 void *opaque, Error **errp);
+
+/*
+ * mirror_start:
+ * @bs: Block device to operate on.
+ * @target: Block device to write to.
+ * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @granularity: The chosen granularity for the dirty bitmap.
+ * @buf_size: The amount of data that can be in flight at one time.
+ * @mode: Whether to collapse all images in the chain to the target.
+ * @on_source_error: The action to take upon error reading from the source.
+ * @on_target_error: The action to take upon error writing to the target.
+ * @cb: Completion function for the job.
+ * @opaque: Opaque pointer value passed to @cb.
+ * @errp: Error object.
+ *
+ * Start a mirroring operation on @bs.  Clusters that are allocated
+ * in @bs will be written to @bs until the job is cancelled or
+ * manually completed.  At the end of a successful mirroring job,
+ * @bs will be switched to read from @target.
+ */
+void mirror_start(BlockDriverState *bs, BlockDriverState *target,
+                  int64_t speed, int64_t granularity, int64_t buf_size,
+                  MirrorSyncMode mode, BlockdevOnError on_source_error,
+                  BlockdevOnError on_target_error,
+                  BlockDriverCompletionFunc *cb,
+                  void *opaque, Error **errp);
+
+/*
+ * backup_start:
+ * @bs: Block device to operate on.
+ * @target: Block device to write to.
+ * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @on_source_error: The action to take upon error reading from the source.
+ * @on_target_error: The action to take upon error writing to the target.
+ * @cb: Completion function for the job.
+ * @opaque: Opaque pointer value passed to @cb.
+ *
+ * Start a backup operation on @bs.  Clusters in @bs are written to @target
+ * until the job is cancelled or manually completed.
+ */
+void backup_start(BlockDriverState *bs, BlockDriverState *target,
+                  int64_t speed, BlockdevOnError on_source_error,
+                  BlockdevOnError on_target_error,
+                  BlockDriverCompletionFunc *cb, void *opaque,
+                  Error **errp);
+
+#endif /* BLOCK_INT_H */
diff --git a/contrib/qemu/include/block/blockjob.h b/contrib/qemu/include/block/blockjob.h
new file mode 100644
index 000000000..c290d07bb
--- /dev/null
+++ b/contrib/qemu/include/block/blockjob.h
@@ -0,0 +1,278 @@
+/*
+ * Declarations for long-running block device operations
+ *
+ * Copyright (c) 2011 IBM Corp.
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef BLOCKJOB_H
+#define BLOCKJOB_H 1
+
+#include "block/block.h"
+
+/**
+ * BlockJobType:
+ *
+ * A class type for block job objects.
+ */
+typedef struct BlockJobType {
+    /** Derived BlockJob struct size */
+    size_t instance_size;
+
+    /** String describing the operation, part of query-block-jobs QMP API */
+    const char *job_type;
+
+    /** Optional callback for job types that support setting a speed limit */
+    void (*set_speed)(BlockJob *job, int64_t speed, Error **errp);
+
+    /** Optional callback for job types that need to forward I/O status reset */
+    void (*iostatus_reset)(BlockJob *job);
+
+    /**
+     * Optional callback for job types whose completion must be triggered
+     * manually.
+     */
+    void (*complete)(BlockJob *job, Error **errp);
+} BlockJobType;
+
+/**
+ * BlockJob:
+ *
+ * Long-running operation on a BlockDriverState.
+ */
+struct BlockJob {
+    /** The job type, including the job vtable.  */
+    const BlockJobType *job_type;
+
+    /** The block device on which the job is operating.  */
+    BlockDriverState *bs;
+
+    /**
+     * The coroutine that executes the job.  If not NULL, it is
+     * reentered when busy is false and the job is cancelled.
+     */
+    Coroutine *co;
+
+    /**
+     * Set to true if the job should cancel itself.  The flag must
+     * always be tested just before toggling the busy flag from false
+     * to true.  After a job has been cancelled, it should only yield
+     * if #qemu_aio_wait will ("sooner or later") reenter the coroutine.
+     */
+    bool cancelled;
+
+    /**
+     * Set to true if the job is either paused, or will pause itself
+     * as soon as possible (if busy == true).
+     */
+    bool paused;
+
+    /**
+     * Set to false by the job while it is in a quiescent state, where
+     * no I/O is pending and the job has yielded on any condition
+     * that is not detected by #qemu_aio_wait, such as a timer.
+     */
+    bool busy;
+
+    /** Status that is published by the query-block-jobs QMP API */
+    BlockDeviceIoStatus iostatus;
+
+    /** Offset that is published by the query-block-jobs QMP API */
+    int64_t offset;
+
+    /** Length that is published by the query-block-jobs QMP API */
+    int64_t len;
+
+    /** Speed that was set with @block_job_set_speed.  */
+    int64_t speed;
+
+    /** The completion function that will be called when the job completes.  */
+    BlockDriverCompletionFunc *cb;
+
+    /** The opaque value that is passed to the completion function.  */
+    void *opaque;
+};
+
+/**
+ * block_job_create:
+ * @job_type: The class object for the newly-created job.
+ * @bs: The block
+ * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @cb: Completion function for the job.
+ * @opaque: Opaque pointer value passed to @cb.
+ * @errp: Error object.
+ *
+ * Create a new long-running block device job and return it.  The job
+ * will call @cb asynchronously when the job completes.  Note that
+ * @bs may have been closed at the time the @cb it is called.  If
+ * this is the case, the job may be reported as either cancelled or
+ * completed.
+ *
+ * This function is not part of the public job interface; it should be
+ * called from a wrapper that is specific to the job type.
+ */
+void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
+                       int64_t speed, BlockDriverCompletionFunc *cb,
+                       void *opaque, Error **errp);
+
+/**
+ * block_job_sleep_ns:
+ * @job: The job that calls the function.
+ * @clock: The clock to sleep on.
+ * @ns: How many nanoseconds to stop for.
+ *
+ * Put the job to sleep (assuming that it wasn't canceled) for @ns
+ * nanoseconds.  Canceling the job will interrupt the wait immediately.
+ */
+void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns);
+
+/**
+ * block_job_completed:
+ * @job: The job being completed.
+ * @ret: The status code.
+ *
+ * Call the completion function that was registered at creation time, and
+ * free @job.
+ */
+void block_job_completed(BlockJob *job, int ret);
+
+/**
+ * block_job_set_speed:
+ * @job: The job to set the speed for.
+ * @speed: The new value
+ * @errp: Error object.
+ *
+ * Set a rate-limiting parameter for the job; the actual meaning may
+ * vary depending on the job type.
+ */
+void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp);
+
+/**
+ * block_job_cancel:
+ * @job: The job to be canceled.
+ *
+ * Asynchronously cancel the specified job.
+ */
+void block_job_cancel(BlockJob *job);
+
+/**
+ * block_job_complete:
+ * @job: The job to be completed.
+ * @errp: Error object.
+ *
+ * Asynchronously complete the specified job.
+ */
+void block_job_complete(BlockJob *job, Error **errp);
+
+/**
+ * block_job_is_cancelled:
+ * @job: The job being queried.
+ *
+ * Returns whether the job is scheduled for cancellation.
+ */
+bool block_job_is_cancelled(BlockJob *job);
+
+/**
+ * block_job_query:
+ * @job: The job to get information about.
+ *
+ * Return information about a job.
+ */
+BlockJobInfo *block_job_query(BlockJob *job);
+
+/**
+ * block_job_pause:
+ * @job: The job to be paused.
+ *
+ * Asynchronously pause the specified job.
+ */
+void block_job_pause(BlockJob *job);
+
+/**
+ * block_job_resume:
+ * @job: The job to be resumed.
+ *
+ * Resume the specified job.
+ */
+void block_job_resume(BlockJob *job);
+
+/**
+ * qobject_from_block_job:
+ * @job: The job whose information is requested.
+ *
+ * Return a QDict corresponding to @job's query-block-jobs entry.
+ */
+QObject *qobject_from_block_job(BlockJob *job);
+
+/**
+ * block_job_ready:
+ * @job: The job which is now ready to complete.
+ *
+ * Send a BLOCK_JOB_READY event for the specified job.
+ */
+void block_job_ready(BlockJob *job);
+
+/**
+ * block_job_is_paused:
+ * @job: The job being queried.
+ *
+ * Returns whether the job is currently paused, or will pause
+ * as soon as it reaches a sleeping point.
+ */
+bool block_job_is_paused(BlockJob *job);
+
+/**
+ * block_job_cancel_sync:
+ * @job: The job to be canceled.
+ *
+ * Synchronously cancel the job.  The completion callback is called
+ * before the function returns.  The job may actually complete
+ * instead of canceling itself; the circumstances under which this
+ * happens depend on the kind of job that is active.
+ *
+ * Returns the return value from the job if the job actually completed
+ * during the call, or -ECANCELED if it was canceled.
+ */
+int block_job_cancel_sync(BlockJob *job);
+
+/**
+ * block_job_iostatus_reset:
+ * @job: The job whose I/O status should be reset.
+ *
+ * Reset I/O status on @job and on BlockDriverState objects it uses,
+ * other than job->bs.
+ */
+void block_job_iostatus_reset(BlockJob *job);
+
+/**
+ * block_job_error_action:
+ * @job: The job to signal an error for.
+ * @bs: The block device on which to set an I/O error.
+ * @on_err: The error action setting.
+ * @is_read: Whether the operation was a read.
+ * @error: The error that was reported.
+ *
+ * Report an I/O error for a block job and possibly stop the VM.  Return the
+ * action that was selected based on @on_err and @error.
+ */
+BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
+                                        BlockdevOnError on_err,
+                                        int is_read, int error);
+#endif
diff --git a/contrib/qemu/include/block/coroutine.h b/contrib/qemu/include/block/coroutine.h
new file mode 100644
index 000000000..377805a3b
--- /dev/null
+++ b/contrib/qemu/include/block/coroutine.h
@@ -0,0 +1,218 @@
+/*
+ * QEMU coroutine implementation
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Stefan Hajnoczi    <stefanha@linux.vnet.ibm.com>
+ *  Kevin Wolf         <kwolf@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_COROUTINE_H
+#define QEMU_COROUTINE_H
+
+#include <stdbool.h>
+#include "qemu/queue.h"
+#include "qemu/timer.h"
+
+/**
+ * Coroutines are a mechanism for stack switching and can be used for
+ * cooperative userspace threading.  These functions provide a simple but
+ * useful flavor of coroutines that is suitable for writing sequential code,
+ * rather than callbacks, for operations that need to give up control while
+ * waiting for events to complete.
+ *
+ * These functions are re-entrant and may be used outside the global mutex.
+ */
+
+/**
+ * Mark a function that executes in coroutine context
+ *
+ * Functions that execute in coroutine context cannot be called directly from
+ * normal functions.  In the future it would be nice to enable compiler or
+ * static checker support for catching such errors.  This annotation might make
+ * it possible and in the meantime it serves as documentation.
+ *
+ * For example:
+ *
+ *   static void coroutine_fn foo(void) {
+ *       ....
+ *   }
+ */
+#define coroutine_fn
+
+typedef struct Coroutine Coroutine;
+
+/**
+ * Coroutine entry point
+ *
+ * When the coroutine is entered for the first time, opaque is passed in as an
+ * argument.
+ *
+ * When this function returns, the coroutine is destroyed automatically and
+ * execution continues in the caller who last entered the coroutine.
+ */
+typedef void coroutine_fn CoroutineEntry(void *opaque);
+
+/**
+ * Create a new coroutine
+ *
+ * Use qemu_coroutine_enter() to actually transfer control to the coroutine.
+ */
+Coroutine *qemu_coroutine_create(CoroutineEntry *entry);
+
+/**
+ * Transfer control to a coroutine
+ *
+ * The opaque argument is passed as the argument to the entry point when
+ * entering the coroutine for the first time.  It is subsequently ignored.
+ */
+void qemu_coroutine_enter(Coroutine *coroutine, void *opaque);
+
+/**
+ * Transfer control back to a coroutine's caller
+ *
+ * This function does not return until the coroutine is re-entered using
+ * qemu_coroutine_enter().
+ */
+void coroutine_fn qemu_coroutine_yield(void);
+
+/**
+ * Get the currently executing coroutine
+ */
+Coroutine *coroutine_fn qemu_coroutine_self(void);
+
+/**
+ * Return whether or not currently inside a coroutine
+ *
+ * This can be used to write functions that work both when in coroutine context
+ * and when not in coroutine context.  Note that such functions cannot use the
+ * coroutine_fn annotation since they work outside coroutine context.
+ */
+bool qemu_in_coroutine(void);
+
+
+
+/**
+ * CoQueues are a mechanism to queue coroutines in order to continue executing
+ * them later. They provide the fundamental primitives on which coroutine locks
+ * are built.
+ */
+typedef struct CoQueue {
+    QTAILQ_HEAD(, Coroutine) entries;
+    AioContext *ctx;
+} CoQueue;
+
+/**
+ * Initialise a CoQueue. This must be called before any other operation is used
+ * on the CoQueue.
+ */
+void qemu_co_queue_init(CoQueue *queue);
+
+/**
+ * Adds the current coroutine to the CoQueue and transfers control to the
+ * caller of the coroutine.
+ */
+void coroutine_fn qemu_co_queue_wait(CoQueue *queue);
+
+/**
+ * Adds the current coroutine to the head of the CoQueue and transfers control to the
+ * caller of the coroutine.
+ */
+void coroutine_fn qemu_co_queue_wait_insert_head(CoQueue *queue);
+
+/**
+ * Restarts the next coroutine in the CoQueue and removes it from the queue.
+ *
+ * Returns true if a coroutine was restarted, false if the queue is empty.
+ */
+bool qemu_co_queue_next(CoQueue *queue);
+
+/**
+ * Restarts all coroutines in the CoQueue and leaves the queue empty.
+ */
+void qemu_co_queue_restart_all(CoQueue *queue);
+
+/**
+ * Checks if the CoQueue is empty.
+ */
+bool qemu_co_queue_empty(CoQueue *queue);
+
+
+/**
+ * Provides a mutex that can be used to synchronise coroutines
+ */
+typedef struct CoMutex {
+    bool locked;
+    CoQueue queue;
+} CoMutex;
+
+/**
+ * Initialises a CoMutex. This must be called before any other operation is used
+ * on the CoMutex.
+ */
+void qemu_co_mutex_init(CoMutex *mutex);
+
+/**
+ * Locks the mutex. If the lock cannot be taken immediately, control is
+ * transferred to the caller of the current coroutine.
+ */
+void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex);
+
+/**
+ * Unlocks the mutex and schedules the next coroutine that was waiting for this
+ * lock to be run.
+ */
+void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex);
+
+typedef struct CoRwlock {
+    bool writer;
+    int reader;
+    CoQueue queue;
+} CoRwlock;
+
+/**
+ * Initialises a CoRwlock. This must be called before any other operation
+ * is used on the CoRwlock
+ */
+void qemu_co_rwlock_init(CoRwlock *lock);
+
+/**
+ * Read locks the CoRwlock. If the lock cannot be taken immediately because
+ * of a parallel writer, control is transferred to the caller of the current
+ * coroutine.
+ */
+void qemu_co_rwlock_rdlock(CoRwlock *lock);
+
+/**
+ * Write Locks the mutex. If the lock cannot be taken immediately because
+ * of a parallel reader, control is transferred to the caller of the current
+ * coroutine.
+ */
+void qemu_co_rwlock_wrlock(CoRwlock *lock);
+
+/**
+ * Unlocks the read/write lock and schedules the next coroutine that was
+ * waiting for this lock to be run.
+ */
+void qemu_co_rwlock_unlock(CoRwlock *lock);
+
+/**
+ * Yield the coroutine for a given duration
+ *
+ * Note this function uses timers and hence only works when a main loop is in
+ * use.  See main-loop.h and do not use from qemu-tool programs.
+ */
+void coroutine_fn co_sleep_ns(QEMUClock *clock, int64_t ns);
+
+/**
+ * Yield until a file descriptor becomes readable
+ *
+ * Note that this function clobbers the handlers for the file descriptor.
+ */
+void coroutine_fn yield_until_fd_readable(int fd);
+#endif /* QEMU_COROUTINE_H */
diff --git a/contrib/qemu/include/block/coroutine_int.h b/contrib/qemu/include/block/coroutine_int.h
new file mode 100644
index 000000000..f133d65af
--- /dev/null
+++ b/contrib/qemu/include/block/coroutine_int.h
@@ -0,0 +1,53 @@
+/*
+ * Coroutine internals
+ *
+ * Copyright (c) 2011 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef QEMU_COROUTINE_INT_H
+#define QEMU_COROUTINE_INT_H
+
+#include "qemu/queue.h"
+#include "block/coroutine.h"
+
+typedef enum {
+    COROUTINE_YIELD = 1,
+    COROUTINE_TERMINATE = 2,
+} CoroutineAction;
+
+struct Coroutine {
+    CoroutineEntry *entry;
+    void *entry_arg;
+    Coroutine *caller;
+    QSLIST_ENTRY(Coroutine) pool_next;
+
+    /* Coroutines that should be woken up when we yield or terminate */
+    QTAILQ_HEAD(, Coroutine) co_queue_wakeup;
+    QTAILQ_ENTRY(Coroutine) co_queue_next;
+};
+
+Coroutine *qemu_coroutine_new(void);
+void qemu_coroutine_delete(Coroutine *co);
+CoroutineAction qemu_coroutine_switch(Coroutine *from, Coroutine *to,
+                                      CoroutineAction action);
+void coroutine_fn qemu_co_queue_run_restart(Coroutine *co);
+
+#endif
diff --git a/contrib/qemu/include/block/snapshot.h b/contrib/qemu/include/block/snapshot.h
new file mode 100644
index 000000000..eaf61f032
--- /dev/null
+++ b/contrib/qemu/include/block/snapshot.h
@@ -0,0 +1,53 @@
+/*
+ * Block layer snapshot related functions
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef SNAPSHOT_H
+#define SNAPSHOT_H
+
+#include "qemu-common.h"
+
+typedef struct QEMUSnapshotInfo {
+    char id_str[128]; /* unique snapshot id */
+    /* the following fields are informative. They are not needed for
+       the consistency of the snapshot */
+    char name[256]; /* user chosen name */
+    uint64_t vm_state_size; /* VM state info size */
+    uint32_t date_sec; /* UTC date of the snapshot */
+    uint32_t date_nsec;
+    uint64_t vm_clock_nsec; /* VM clock relative to boot */
+} QEMUSnapshotInfo;
+
+int bdrv_snapshot_find(BlockDriverState *bs, QEMUSnapshotInfo *sn_info,
+                       const char *name);
+int bdrv_can_snapshot(BlockDriverState *bs);
+int bdrv_snapshot_create(BlockDriverState *bs,
+                         QEMUSnapshotInfo *sn_info);
+int bdrv_snapshot_goto(BlockDriverState *bs,
+                       const char *snapshot_id);
+int bdrv_snapshot_delete(BlockDriverState *bs, const char *snapshot_id);
+int bdrv_snapshot_list(BlockDriverState *bs,
+                       QEMUSnapshotInfo **psn_info);
+int bdrv_snapshot_load_tmp(BlockDriverState *bs,
+                           const char *snapshot_name);
+#endif
diff --git a/contrib/qemu/include/config.h b/contrib/qemu/include/config.h
new file mode 100644
index 000000000..e20f78696
--- /dev/null
+++ b/contrib/qemu/include/config.h
@@ -0,0 +1,2 @@
+#include "config-host.h"
+#include "config-target.h"
diff --git a/contrib/qemu/include/exec/cpu-common.h b/contrib/qemu/include/exec/cpu-common.h
new file mode 100644
index 000000000..e4996e19c
--- /dev/null
+++ b/contrib/qemu/include/exec/cpu-common.h
@@ -0,0 +1,124 @@
+#ifndef CPU_COMMON_H
+#define CPU_COMMON_H 1
+
+/* CPU interfaces that are target independent.  */
+
+#ifndef CONFIG_USER_ONLY
+#include "exec/hwaddr.h"
+#endif
+
+#ifndef NEED_CPU_H
+#include "exec/poison.h"
+#endif
+
+#include "qemu/bswap.h"
+#include "qemu/queue.h"
+
+/**
+ * CPUListState:
+ * @cpu_fprintf: Print function.
+ * @file: File to print to using @cpu_fprint.
+ *
+ * State commonly used for iterating over CPU models.
+ */
+typedef struct CPUListState {
+    fprintf_function cpu_fprintf;
+    FILE *file;
+} CPUListState;
+
+#if !defined(CONFIG_USER_ONLY)
+
+enum device_endian {
+    DEVICE_NATIVE_ENDIAN,
+    DEVICE_BIG_ENDIAN,
+    DEVICE_LITTLE_ENDIAN,
+};
+
+/* address in the RAM (different from a physical address) */
+#if defined(CONFIG_XEN_BACKEND)
+typedef uint64_t ram_addr_t;
+#  define RAM_ADDR_MAX UINT64_MAX
+#  define RAM_ADDR_FMT "%" PRIx64
+#else
+typedef uintptr_t ram_addr_t;
+#  define RAM_ADDR_MAX UINTPTR_MAX
+#  define RAM_ADDR_FMT "%" PRIxPTR
+#endif
+
+/* memory API */
+
+typedef void CPUWriteMemoryFunc(void *opaque, hwaddr addr, uint32_t value);
+typedef uint32_t CPUReadMemoryFunc(void *opaque, hwaddr addr);
+
+void qemu_ram_remap(ram_addr_t addr, ram_addr_t length);
+/* This should not be used by devices.  */
+MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr);
+void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev);
+
+void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
+                            int len, int is_write);
+static inline void cpu_physical_memory_read(hwaddr addr,
+                                            void *buf, int len)
+{
+    cpu_physical_memory_rw(addr, buf, len, 0);
+}
+static inline void cpu_physical_memory_write(hwaddr addr,
+                                             const void *buf, int len)
+{
+    cpu_physical_memory_rw(addr, (void *)buf, len, 1);
+}
+void *cpu_physical_memory_map(hwaddr addr,
+                              hwaddr *plen,
+                              int is_write);
+void cpu_physical_memory_unmap(void *buffer, hwaddr len,
+                               int is_write, hwaddr access_len);
+void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque));
+
+bool cpu_physical_memory_is_io(hwaddr phys_addr);
+
+/* Coalesced MMIO regions are areas where write operations can be reordered.
+ * This usually implies that write operations are side-effect free.  This allows
+ * batching which can make a major impact on performance when using
+ * virtualization.
+ */
+void qemu_flush_coalesced_mmio_buffer(void);
+
+uint32_t ldub_phys(hwaddr addr);
+uint32_t lduw_le_phys(hwaddr addr);
+uint32_t lduw_be_phys(hwaddr addr);
+uint32_t ldl_le_phys(hwaddr addr);
+uint32_t ldl_be_phys(hwaddr addr);
+uint64_t ldq_le_phys(hwaddr addr);
+uint64_t ldq_be_phys(hwaddr addr);
+void stb_phys(hwaddr addr, uint32_t val);
+void stw_le_phys(hwaddr addr, uint32_t val);
+void stw_be_phys(hwaddr addr, uint32_t val);
+void stl_le_phys(hwaddr addr, uint32_t val);
+void stl_be_phys(hwaddr addr, uint32_t val);
+void stq_le_phys(hwaddr addr, uint64_t val);
+void stq_be_phys(hwaddr addr, uint64_t val);
+
+#ifdef NEED_CPU_H
+uint32_t lduw_phys(hwaddr addr);
+uint32_t ldl_phys(hwaddr addr);
+uint64_t ldq_phys(hwaddr addr);
+void stl_phys_notdirty(hwaddr addr, uint32_t val);
+void stw_phys(hwaddr addr, uint32_t val);
+void stl_phys(hwaddr addr, uint32_t val);
+void stq_phys(hwaddr addr, uint64_t val);
+#endif
+
+void cpu_physical_memory_write_rom(hwaddr addr,
+                                   const uint8_t *buf, int len);
+
+extern struct MemoryRegion io_mem_rom;
+extern struct MemoryRegion io_mem_notdirty;
+
+typedef void (RAMBlockIterFunc)(void *host_addr,
+    ram_addr_t offset, ram_addr_t length, void *opaque);
+
+void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque);
+
+#endif
+
+#endif /* !CPU_COMMON_H */
diff --git a/contrib/qemu/include/exec/hwaddr.h b/contrib/qemu/include/exec/hwaddr.h
new file mode 100644
index 000000000..c9eb78fba
--- /dev/null
+++ b/contrib/qemu/include/exec/hwaddr.h
@@ -0,0 +1,20 @@
+/* Define hwaddr if it exists.  */
+
+#ifndef HWADDR_H
+#define HWADDR_H
+
+#define HWADDR_BITS 64
+/* hwaddr is the type of a physical address (its size can
+   be different from 'target_ulong').  */
+
+typedef uint64_t hwaddr;
+#define HWADDR_MAX UINT64_MAX
+#define TARGET_FMT_plx "%016" PRIx64
+#define HWADDR_PRId PRId64
+#define HWADDR_PRIi PRIi64
+#define HWADDR_PRIo PRIo64
+#define HWADDR_PRIu PRIu64
+#define HWADDR_PRIx PRIx64
+#define HWADDR_PRIX PRIX64
+
+#endif
diff --git a/contrib/qemu/include/exec/poison.h b/contrib/qemu/include/exec/poison.h
new file mode 100644
index 000000000..2341a7504
--- /dev/null
+++ b/contrib/qemu/include/exec/poison.h
@@ -0,0 +1,63 @@
+/* Poison identifiers that should not be used when building
+   target independent device code.  */
+
+#ifndef HW_POISON_H
+#define HW_POISON_H
+#ifdef __GNUC__
+
+#pragma GCC poison TARGET_I386
+#pragma GCC poison TARGET_X86_64
+#pragma GCC poison TARGET_ALPHA
+#pragma GCC poison TARGET_ARM
+#pragma GCC poison TARGET_CRIS
+#pragma GCC poison TARGET_LM32
+#pragma GCC poison TARGET_M68K
+#pragma GCC poison TARGET_MIPS
+#pragma GCC poison TARGET_MIPS64
+#pragma GCC poison TARGET_OPENRISC
+#pragma GCC poison TARGET_PPC
+#pragma GCC poison TARGET_PPCEMB
+#pragma GCC poison TARGET_PPC64
+#pragma GCC poison TARGET_ABI32
+#pragma GCC poison TARGET_SH4
+#pragma GCC poison TARGET_SPARC
+#pragma GCC poison TARGET_SPARC64
+
+#pragma GCC poison TARGET_WORDS_BIGENDIAN
+#pragma GCC poison BSWAP_NEEDED
+
+#pragma GCC poison TARGET_LONG_BITS
+#pragma GCC poison TARGET_FMT_lx
+#pragma GCC poison TARGET_FMT_ld
+
+#pragma GCC poison TARGET_PAGE_SIZE
+#pragma GCC poison TARGET_PAGE_MASK
+#pragma GCC poison TARGET_PAGE_BITS
+#pragma GCC poison TARGET_PAGE_ALIGN
+
+#pragma GCC poison CPUArchState
+#pragma GCC poison env
+
+#pragma GCC poison lduw_phys
+#pragma GCC poison ldl_phys
+#pragma GCC poison ldq_phys
+#pragma GCC poison stl_phys_notdirty
+#pragma GCC poison stw_phys
+#pragma GCC poison stl_phys
+#pragma GCC poison stq_phys
+
+#pragma GCC poison CPU_INTERRUPT_HARD
+#pragma GCC poison CPU_INTERRUPT_EXITTB
+#pragma GCC poison CPU_INTERRUPT_HALT
+#pragma GCC poison CPU_INTERRUPT_DEBUG
+#pragma GCC poison CPU_INTERRUPT_TGT_EXT_0
+#pragma GCC poison CPU_INTERRUPT_TGT_EXT_1
+#pragma GCC poison CPU_INTERRUPT_TGT_EXT_2
+#pragma GCC poison CPU_INTERRUPT_TGT_EXT_3
+#pragma GCC poison CPU_INTERRUPT_TGT_EXT_4
+#pragma GCC poison CPU_INTERRUPT_TGT_INT_0
+#pragma GCC poison CPU_INTERRUPT_TGT_INT_1
+#pragma GCC poison CPU_INTERRUPT_TGT_INT_2
+
+#endif
+#endif
diff --git a/contrib/qemu/include/fpu/softfloat.h b/contrib/qemu/include/fpu/softfloat.h
new file mode 100644
index 000000000..f3927e241
--- /dev/null
+++ b/contrib/qemu/include/fpu/softfloat.h
@@ -0,0 +1,641 @@
+/*
+ * QEMU float support
+ *
+ * Derived from SoftFloat.
+ */
+
+/*============================================================================
+
+This C header file is part of the SoftFloat IEC/IEEE Floating-point Arithmetic
+Package, Release 2b.
+
+Written by John R. Hauser.  This work was made possible in part by the
+International Computer Science Institute, located at Suite 600, 1947 Center
+Street, Berkeley, California 94704.  Funding was partially provided by the
+National Science Foundation under grant MIP-9311980.  The original version
+of this code was written as part of a project to build a fixed-point vector
+processor in collaboration with the University of California at Berkeley,
+overseen by Profs. Nelson Morgan and John Wawrzynek.  More information
+is available through the Web page `http://www.cs.berkeley.edu/~jhauser/
+arithmetic/SoftFloat.html'.
+
+THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort has
+been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT TIMES
+RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO PERSONS
+AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ALL LOSSES,
+COSTS, OR OTHER PROBLEMS THEY INCUR DUE TO THE SOFTWARE, AND WHO FURTHERMORE
+EFFECTIVELY INDEMNIFY JOHN HAUSER AND THE INTERNATIONAL COMPUTER SCIENCE
+INSTITUTE (possibly via similar legal warning) AGAINST ALL LOSSES, COSTS, OR
+OTHER PROBLEMS INCURRED BY THEIR CUSTOMERS AND CLIENTS DUE TO THE SOFTWARE.
+
+Derivative works are acceptable, even for commercial purposes, so long as
+(1) the source code for the derivative work includes prominent notice that
+the work is derivative, and (2) the source code includes prominent notice with
+these four paragraphs for those parts of this code that are retained.
+
+=============================================================================*/
+
+#ifndef SOFTFLOAT_H
+#define SOFTFLOAT_H
+
+#if defined(CONFIG_SOLARIS) && defined(CONFIG_NEEDS_LIBSUNMATH)
+#include <sunmath.h>
+#endif
+
+#include <inttypes.h>
+#include "config-host.h"
+#include "qemu/osdep.h"
+
+/*----------------------------------------------------------------------------
+| Each of the following `typedef's defines the most convenient type that holds
+| integers of at least as many bits as specified.  For example, `uint8' should
+| be the most convenient type that can hold unsigned integers of as many as
+| 8 bits.  The `flag' type must be able to hold either a 0 or 1.  For most
+| implementations of C, `flag', `uint8', and `int8' should all be `typedef'ed
+| to the same as `int'.
+*----------------------------------------------------------------------------*/
+typedef uint8_t flag;
+typedef uint8_t uint8;
+typedef int8_t int8;
+typedef unsigned int uint32;
+typedef signed int int32;
+typedef uint64_t uint64;
+typedef int64_t int64;
+
+#define LIT64( a ) a##LL
+#define INLINE static inline
+
+#define STATUS_PARAM , float_status *status
+#define STATUS(field) status->field
+#define STATUS_VAR , status
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE floating-point ordering relations
+*----------------------------------------------------------------------------*/
+enum {
+    float_relation_less      = -1,
+    float_relation_equal     =  0,
+    float_relation_greater   =  1,
+    float_relation_unordered =  2
+};
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE floating-point types.
+*----------------------------------------------------------------------------*/
+/* Use structures for soft-float types.  This prevents accidentally mixing
+   them with native int/float types.  A sufficiently clever compiler and
+   sane ABI should be able to see though these structs.  However
+   x86/gcc 3.x seems to struggle a bit, so leave them disabled by default.  */
+//#define USE_SOFTFLOAT_STRUCT_TYPES
+#ifdef USE_SOFTFLOAT_STRUCT_TYPES
+typedef struct {
+    uint16_t v;
+} float16;
+#define float16_val(x) (((float16)(x)).v)
+#define make_float16(x) __extension__ ({ float16 f16_val = {x}; f16_val; })
+#define const_float16(x) { x }
+typedef struct {
+    uint32_t v;
+} float32;
+/* The cast ensures an error if the wrong type is passed.  */
+#define float32_val(x) (((float32)(x)).v)
+#define make_float32(x) __extension__ ({ float32 f32_val = {x}; f32_val; })
+#define const_float32(x) { x }
+typedef struct {
+    uint64_t v;
+} float64;
+#define float64_val(x) (((float64)(x)).v)
+#define make_float64(x) __extension__ ({ float64 f64_val = {x}; f64_val; })
+#define const_float64(x) { x }
+#else
+typedef uint16_t float16;
+typedef uint32_t float32;
+typedef uint64_t float64;
+#define float16_val(x) (x)
+#define float32_val(x) (x)
+#define float64_val(x) (x)
+#define make_float16(x) (x)
+#define make_float32(x) (x)
+#define make_float64(x) (x)
+#define const_float16(x) (x)
+#define const_float32(x) (x)
+#define const_float64(x) (x)
+#endif
+typedef struct {
+    uint64_t low;
+    uint16_t high;
+} floatx80;
+#define make_floatx80(exp, mant) ((floatx80) { mant, exp })
+#define make_floatx80_init(exp, mant) { .low = mant, .high = exp }
+typedef struct {
+#ifdef HOST_WORDS_BIGENDIAN
+    uint64_t high, low;
+#else
+    uint64_t low, high;
+#endif
+} float128;
+#define make_float128(high_, low_) ((float128) { .high = high_, .low = low_ })
+#define make_float128_init(high_, low_) { .high = high_, .low = low_ }
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE floating-point underflow tininess-detection mode.
+*----------------------------------------------------------------------------*/
+enum {
+    float_tininess_after_rounding  = 0,
+    float_tininess_before_rounding = 1
+};
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE floating-point rounding mode.
+*----------------------------------------------------------------------------*/
+enum {
+    float_round_nearest_even = 0,
+    float_round_down         = 1,
+    float_round_up           = 2,
+    float_round_to_zero      = 3
+};
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE floating-point exception flags.
+*----------------------------------------------------------------------------*/
+enum {
+    float_flag_invalid   =  1,
+    float_flag_divbyzero =  4,
+    float_flag_overflow  =  8,
+    float_flag_underflow = 16,
+    float_flag_inexact   = 32,
+    float_flag_input_denormal = 64,
+    float_flag_output_denormal = 128
+};
+
+typedef struct float_status {
+    signed char float_detect_tininess;
+    signed char float_rounding_mode;
+    signed char float_exception_flags;
+    signed char floatx80_rounding_precision;
+    /* should denormalised results go to zero and set the inexact flag? */
+    flag flush_to_zero;
+    /* should denormalised inputs go to zero and set the input_denormal flag? */
+    flag flush_inputs_to_zero;
+    flag default_nan_mode;
+} float_status;
+
+void set_float_rounding_mode(int val STATUS_PARAM);
+void set_float_exception_flags(int val STATUS_PARAM);
+INLINE void set_float_detect_tininess(int val STATUS_PARAM)
+{
+    STATUS(float_detect_tininess) = val;
+}
+INLINE void set_flush_to_zero(flag val STATUS_PARAM)
+{
+    STATUS(flush_to_zero) = val;
+}
+INLINE void set_flush_inputs_to_zero(flag val STATUS_PARAM)
+{
+    STATUS(flush_inputs_to_zero) = val;
+}
+INLINE void set_default_nan_mode(flag val STATUS_PARAM)
+{
+    STATUS(default_nan_mode) = val;
+}
+INLINE int get_float_exception_flags(float_status *status)
+{
+    return STATUS(float_exception_flags);
+}
+void set_floatx80_rounding_precision(int val STATUS_PARAM);
+
+/*----------------------------------------------------------------------------
+| Routine to raise any or all of the software IEC/IEEE floating-point
+| exception flags.
+*----------------------------------------------------------------------------*/
+void float_raise( int8 flags STATUS_PARAM);
+
+/*----------------------------------------------------------------------------
+| Options to indicate which negations to perform in float*_muladd()
+| Using these differs from negating an input or output before calling
+| the muladd function in that this means that a NaN doesn't have its
+| sign bit inverted before it is propagated.
+*----------------------------------------------------------------------------*/
+enum {
+    float_muladd_negate_c = 1,
+    float_muladd_negate_product = 2,
+    float_muladd_negate_result = 4,
+};
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE integer-to-floating-point conversion routines.
+*----------------------------------------------------------------------------*/
+float32 int32_to_float32( int32 STATUS_PARAM );
+float64 int32_to_float64( int32 STATUS_PARAM );
+float32 uint32_to_float32( uint32 STATUS_PARAM );
+float64 uint32_to_float64( uint32 STATUS_PARAM );
+floatx80 int32_to_floatx80( int32 STATUS_PARAM );
+float128 int32_to_float128( int32 STATUS_PARAM );
+float32 int64_to_float32( int64 STATUS_PARAM );
+float32 uint64_to_float32( uint64 STATUS_PARAM );
+float64 int64_to_float64( int64 STATUS_PARAM );
+float64 uint64_to_float64( uint64 STATUS_PARAM );
+floatx80 int64_to_floatx80( int64 STATUS_PARAM );
+float128 int64_to_float128( int64 STATUS_PARAM );
+float128 uint64_to_float128( uint64 STATUS_PARAM );
+
+/*----------------------------------------------------------------------------
+| Software half-precision conversion routines.
+*----------------------------------------------------------------------------*/
+float16 float32_to_float16( float32, flag STATUS_PARAM );
+float32 float16_to_float32( float16, flag STATUS_PARAM );
+
+/*----------------------------------------------------------------------------
+| Software half-precision operations.
+*----------------------------------------------------------------------------*/
+int float16_is_quiet_nan( float16 );
+int float16_is_signaling_nan( float16 );
+float16 float16_maybe_silence_nan( float16 );
+
+INLINE int float16_is_any_nan(float16 a)
+{
+    return ((float16_val(a) & ~0x8000) > 0x7c00);
+}
+
+/*----------------------------------------------------------------------------
+| The pattern for a default generated half-precision NaN.
+*----------------------------------------------------------------------------*/
+extern const float16 float16_default_nan;
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE single-precision conversion routines.
+*----------------------------------------------------------------------------*/
+int_fast16_t float32_to_int16_round_to_zero(float32 STATUS_PARAM);
+uint_fast16_t float32_to_uint16_round_to_zero(float32 STATUS_PARAM);
+int32 float32_to_int32( float32 STATUS_PARAM );
+int32 float32_to_int32_round_to_zero( float32 STATUS_PARAM );
+uint32 float32_to_uint32( float32 STATUS_PARAM );
+uint32 float32_to_uint32_round_to_zero( float32 STATUS_PARAM );
+int64 float32_to_int64( float32 STATUS_PARAM );
+int64 float32_to_int64_round_to_zero( float32 STATUS_PARAM );
+float64 float32_to_float64( float32 STATUS_PARAM );
+floatx80 float32_to_floatx80( float32 STATUS_PARAM );
+float128 float32_to_float128( float32 STATUS_PARAM );
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE single-precision operations.
+*----------------------------------------------------------------------------*/
+float32 float32_round_to_int( float32 STATUS_PARAM );
+float32 float32_add( float32, float32 STATUS_PARAM );
+float32 float32_sub( float32, float32 STATUS_PARAM );
+float32 float32_mul( float32, float32 STATUS_PARAM );
+float32 float32_div( float32, float32 STATUS_PARAM );
+float32 float32_rem( float32, float32 STATUS_PARAM );
+float32 float32_muladd(float32, float32, float32, int STATUS_PARAM);
+float32 float32_sqrt( float32 STATUS_PARAM );
+float32 float32_exp2( float32 STATUS_PARAM );
+float32 float32_log2( float32 STATUS_PARAM );
+int float32_eq( float32, float32 STATUS_PARAM );
+int float32_le( float32, float32 STATUS_PARAM );
+int float32_lt( float32, float32 STATUS_PARAM );
+int float32_unordered( float32, float32 STATUS_PARAM );
+int float32_eq_quiet( float32, float32 STATUS_PARAM );
+int float32_le_quiet( float32, float32 STATUS_PARAM );
+int float32_lt_quiet( float32, float32 STATUS_PARAM );
+int float32_unordered_quiet( float32, float32 STATUS_PARAM );
+int float32_compare( float32, float32 STATUS_PARAM );
+int float32_compare_quiet( float32, float32 STATUS_PARAM );
+float32 float32_min(float32, float32 STATUS_PARAM);
+float32 float32_max(float32, float32 STATUS_PARAM);
+int float32_is_quiet_nan( float32 );
+int float32_is_signaling_nan( float32 );
+float32 float32_maybe_silence_nan( float32 );
+float32 float32_scalbn( float32, int STATUS_PARAM );
+
+INLINE float32 float32_abs(float32 a)
+{
+    /* Note that abs does *not* handle NaN specially, nor does
+     * it flush denormal inputs to zero.
+     */
+    return make_float32(float32_val(a) & 0x7fffffff);
+}
+
+INLINE float32 float32_chs(float32 a)
+{
+    /* Note that chs does *not* handle NaN specially, nor does
+     * it flush denormal inputs to zero.
+     */
+    return make_float32(float32_val(a) ^ 0x80000000);
+}
+
+INLINE int float32_is_infinity(float32 a)
+{
+    return (float32_val(a) & 0x7fffffff) == 0x7f800000;
+}
+
+INLINE int float32_is_neg(float32 a)
+{
+    return float32_val(a) >> 31;
+}
+
+INLINE int float32_is_zero(float32 a)
+{
+    return (float32_val(a) & 0x7fffffff) == 0;
+}
+
+INLINE int float32_is_any_nan(float32 a)
+{
+    return ((float32_val(a) & ~(1 << 31)) > 0x7f800000UL);
+}
+
+INLINE int float32_is_zero_or_denormal(float32 a)
+{
+    return (float32_val(a) & 0x7f800000) == 0;
+}
+
+INLINE float32 float32_set_sign(float32 a, int sign)
+{
+    return make_float32((float32_val(a) & 0x7fffffff) | (sign << 31));
+}
+
+#define float32_zero make_float32(0)
+#define float32_one make_float32(0x3f800000)
+#define float32_ln2 make_float32(0x3f317218)
+#define float32_pi make_float32(0x40490fdb)
+#define float32_half make_float32(0x3f000000)
+#define float32_infinity make_float32(0x7f800000)
+
+
+/*----------------------------------------------------------------------------
+| The pattern for a default generated single-precision NaN.
+*----------------------------------------------------------------------------*/
+extern const float32 float32_default_nan;
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE double-precision conversion routines.
+*----------------------------------------------------------------------------*/
+int_fast16_t float64_to_int16_round_to_zero(float64 STATUS_PARAM);
+uint_fast16_t float64_to_uint16_round_to_zero(float64 STATUS_PARAM);
+int32 float64_to_int32( float64 STATUS_PARAM );
+int32 float64_to_int32_round_to_zero( float64 STATUS_PARAM );
+uint32 float64_to_uint32( float64 STATUS_PARAM );
+uint32 float64_to_uint32_round_to_zero( float64 STATUS_PARAM );
+int64 float64_to_int64( float64 STATUS_PARAM );
+int64 float64_to_int64_round_to_zero( float64 STATUS_PARAM );
+uint64 float64_to_uint64 (float64 a STATUS_PARAM);
+uint64 float64_to_uint64_round_to_zero (float64 a STATUS_PARAM);
+float32 float64_to_float32( float64 STATUS_PARAM );
+floatx80 float64_to_floatx80( float64 STATUS_PARAM );
+float128 float64_to_float128( float64 STATUS_PARAM );
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE double-precision operations.
+*----------------------------------------------------------------------------*/
+float64 float64_round_to_int( float64 STATUS_PARAM );
+float64 float64_trunc_to_int( float64 STATUS_PARAM );
+float64 float64_add( float64, float64 STATUS_PARAM );
+float64 float64_sub( float64, float64 STATUS_PARAM );
+float64 float64_mul( float64, float64 STATUS_PARAM );
+float64 float64_div( float64, float64 STATUS_PARAM );
+float64 float64_rem( float64, float64 STATUS_PARAM );
+float64 float64_muladd(float64, float64, float64, int STATUS_PARAM);
+float64 float64_sqrt( float64 STATUS_PARAM );
+float64 float64_log2( float64 STATUS_PARAM );
+int float64_eq( float64, float64 STATUS_PARAM );
+int float64_le( float64, float64 STATUS_PARAM );
+int float64_lt( float64, float64 STATUS_PARAM );
+int float64_unordered( float64, float64 STATUS_PARAM );
+int float64_eq_quiet( float64, float64 STATUS_PARAM );
+int float64_le_quiet( float64, float64 STATUS_PARAM );
+int float64_lt_quiet( float64, float64 STATUS_PARAM );
+int float64_unordered_quiet( float64, float64 STATUS_PARAM );
+int float64_compare( float64, float64 STATUS_PARAM );
+int float64_compare_quiet( float64, float64 STATUS_PARAM );
+float64 float64_min(float64, float64 STATUS_PARAM);
+float64 float64_max(float64, float64 STATUS_PARAM);
+int float64_is_quiet_nan( float64 a );
+int float64_is_signaling_nan( float64 );
+float64 float64_maybe_silence_nan( float64 );
+float64 float64_scalbn( float64, int STATUS_PARAM );
+
+INLINE float64 float64_abs(float64 a)
+{
+    /* Note that abs does *not* handle NaN specially, nor does
+     * it flush denormal inputs to zero.
+     */
+    return make_float64(float64_val(a) & 0x7fffffffffffffffLL);
+}
+
+INLINE float64 float64_chs(float64 a)
+{
+    /* Note that chs does *not* handle NaN specially, nor does
+     * it flush denormal inputs to zero.
+     */
+    return make_float64(float64_val(a) ^ 0x8000000000000000LL);
+}
+
+INLINE int float64_is_infinity(float64 a)
+{
+    return (float64_val(a) & 0x7fffffffffffffffLL ) == 0x7ff0000000000000LL;
+}
+
+INLINE int float64_is_neg(float64 a)
+{
+    return float64_val(a) >> 63;
+}
+
+INLINE int float64_is_zero(float64 a)
+{
+    return (float64_val(a) & 0x7fffffffffffffffLL) == 0;
+}
+
+INLINE int float64_is_any_nan(float64 a)
+{
+    return ((float64_val(a) & ~(1ULL << 63)) > 0x7ff0000000000000ULL);
+}
+
+INLINE int float64_is_zero_or_denormal(float64 a)
+{
+    return (float64_val(a) & 0x7ff0000000000000LL) == 0;
+}
+
+INLINE float64 float64_set_sign(float64 a, int sign)
+{
+    return make_float64((float64_val(a) & 0x7fffffffffffffffULL)
+                        | ((int64_t)sign << 63));
+}
+
+#define float64_zero make_float64(0)
+#define float64_one make_float64(0x3ff0000000000000LL)
+#define float64_ln2 make_float64(0x3fe62e42fefa39efLL)
+#define float64_pi make_float64(0x400921fb54442d18LL)
+#define float64_half make_float64(0x3fe0000000000000LL)
+#define float64_infinity make_float64(0x7ff0000000000000LL)
+
+/*----------------------------------------------------------------------------
+| The pattern for a default generated double-precision NaN.
+*----------------------------------------------------------------------------*/
+extern const float64 float64_default_nan;
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE extended double-precision conversion routines.
+*----------------------------------------------------------------------------*/
+int32 floatx80_to_int32( floatx80 STATUS_PARAM );
+int32 floatx80_to_int32_round_to_zero( floatx80 STATUS_PARAM );
+int64 floatx80_to_int64( floatx80 STATUS_PARAM );
+int64 floatx80_to_int64_round_to_zero( floatx80 STATUS_PARAM );
+float32 floatx80_to_float32( floatx80 STATUS_PARAM );
+float64 floatx80_to_float64( floatx80 STATUS_PARAM );
+float128 floatx80_to_float128( floatx80 STATUS_PARAM );
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE extended double-precision operations.
+*----------------------------------------------------------------------------*/
+floatx80 floatx80_round_to_int( floatx80 STATUS_PARAM );
+floatx80 floatx80_add( floatx80, floatx80 STATUS_PARAM );
+floatx80 floatx80_sub( floatx80, floatx80 STATUS_PARAM );
+floatx80 floatx80_mul( floatx80, floatx80 STATUS_PARAM );
+floatx80 floatx80_div( floatx80, floatx80 STATUS_PARAM );
+floatx80 floatx80_rem( floatx80, floatx80 STATUS_PARAM );
+floatx80 floatx80_sqrt( floatx80 STATUS_PARAM );
+int floatx80_eq( floatx80, floatx80 STATUS_PARAM );
+int floatx80_le( floatx80, floatx80 STATUS_PARAM );
+int floatx80_lt( floatx80, floatx80 STATUS_PARAM );
+int floatx80_unordered( floatx80, floatx80 STATUS_PARAM );
+int floatx80_eq_quiet( floatx80, floatx80 STATUS_PARAM );
+int floatx80_le_quiet( floatx80, floatx80 STATUS_PARAM );
+int floatx80_lt_quiet( floatx80, floatx80 STATUS_PARAM );
+int floatx80_unordered_quiet( floatx80, floatx80 STATUS_PARAM );
+int floatx80_compare( floatx80, floatx80 STATUS_PARAM );
+int floatx80_compare_quiet( floatx80, floatx80 STATUS_PARAM );
+int floatx80_is_quiet_nan( floatx80 );
+int floatx80_is_signaling_nan( floatx80 );
+floatx80 floatx80_maybe_silence_nan( floatx80 );
+floatx80 floatx80_scalbn( floatx80, int STATUS_PARAM );
+
+INLINE floatx80 floatx80_abs(floatx80 a)
+{
+    a.high &= 0x7fff;
+    return a;
+}
+
+INLINE floatx80 floatx80_chs(floatx80 a)
+{
+    a.high ^= 0x8000;
+    return a;
+}
+
+INLINE int floatx80_is_infinity(floatx80 a)
+{
+    return (a.high & 0x7fff) == 0x7fff && a.low == 0x8000000000000000LL;
+}
+
+INLINE int floatx80_is_neg(floatx80 a)
+{
+    return a.high >> 15;
+}
+
+INLINE int floatx80_is_zero(floatx80 a)
+{
+    return (a.high & 0x7fff) == 0 && a.low == 0;
+}
+
+INLINE int floatx80_is_zero_or_denormal(floatx80 a)
+{
+    return (a.high & 0x7fff) == 0;
+}
+
+INLINE int floatx80_is_any_nan(floatx80 a)
+{
+    return ((a.high & 0x7fff) == 0x7fff) && (a.low<<1);
+}
+
+#define floatx80_zero make_floatx80(0x0000, 0x0000000000000000LL)
+#define floatx80_one make_floatx80(0x3fff, 0x8000000000000000LL)
+#define floatx80_ln2 make_floatx80(0x3ffe, 0xb17217f7d1cf79acLL)
+#define floatx80_pi make_floatx80(0x4000, 0xc90fdaa22168c235LL)
+#define floatx80_half make_floatx80(0x3ffe, 0x8000000000000000LL)
+#define floatx80_infinity make_floatx80(0x7fff, 0x8000000000000000LL)
+
+/*----------------------------------------------------------------------------
+| The pattern for a default generated extended double-precision NaN.
+*----------------------------------------------------------------------------*/
+extern const floatx80 floatx80_default_nan;
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE quadruple-precision conversion routines.
+*----------------------------------------------------------------------------*/
+int32 float128_to_int32( float128 STATUS_PARAM );
+int32 float128_to_int32_round_to_zero( float128 STATUS_PARAM );
+int64 float128_to_int64( float128 STATUS_PARAM );
+int64 float128_to_int64_round_to_zero( float128 STATUS_PARAM );
+float32 float128_to_float32( float128 STATUS_PARAM );
+float64 float128_to_float64( float128 STATUS_PARAM );
+floatx80 float128_to_floatx80( float128 STATUS_PARAM );
+
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE quadruple-precision operations.
+*----------------------------------------------------------------------------*/
+float128 float128_round_to_int( float128 STATUS_PARAM );
+float128 float128_add( float128, float128 STATUS_PARAM );
+float128 float128_sub( float128, float128 STATUS_PARAM );
+float128 float128_mul( float128, float128 STATUS_PARAM );
+float128 float128_div( float128, float128 STATUS_PARAM );
+float128 float128_rem( float128, float128 STATUS_PARAM );
+float128 float128_sqrt( float128 STATUS_PARAM );
+int float128_eq( float128, float128 STATUS_PARAM );
+int float128_le( float128, float128 STATUS_PARAM );
+int float128_lt( float128, float128 STATUS_PARAM );
+int float128_unordered( float128, float128 STATUS_PARAM );
+int float128_eq_quiet( float128, float128 STATUS_PARAM );
+int float128_le_quiet( float128, float128 STATUS_PARAM );
+int float128_lt_quiet( float128, float128 STATUS_PARAM );
+int float128_unordered_quiet( float128, float128 STATUS_PARAM );
+int float128_compare( float128, float128 STATUS_PARAM );
+int float128_compare_quiet( float128, float128 STATUS_PARAM );
+int float128_is_quiet_nan( float128 );
+int float128_is_signaling_nan( float128 );
+float128 float128_maybe_silence_nan( float128 );
+float128 float128_scalbn( float128, int STATUS_PARAM );
+
+INLINE float128 float128_abs(float128 a)
+{
+    a.high &= 0x7fffffffffffffffLL;
+    return a;
+}
+
+INLINE float128 float128_chs(float128 a)
+{
+    a.high ^= 0x8000000000000000LL;
+    return a;
+}
+
+INLINE int float128_is_infinity(float128 a)
+{
+    return (a.high & 0x7fffffffffffffffLL) == 0x7fff000000000000LL && a.low == 0;
+}
+
+INLINE int float128_is_neg(float128 a)
+{
+    return a.high >> 63;
+}
+
+INLINE int float128_is_zero(float128 a)
+{
+    return (a.high & 0x7fffffffffffffffLL) == 0 && a.low == 0;
+}
+
+INLINE int float128_is_zero_or_denormal(float128 a)
+{
+    return (a.high & 0x7fff000000000000LL) == 0;
+}
+
+INLINE int float128_is_any_nan(float128 a)
+{
+    return ((a.high >> 48) & 0x7fff) == 0x7fff &&
+        ((a.low != 0) || ((a.high & 0xffffffffffffLL) != 0));
+}
+
+#define float128_zero make_float128(0, 0)
+
+/*----------------------------------------------------------------------------
+| The pattern for a default generated quadruple-precision NaN.
+*----------------------------------------------------------------------------*/
+extern const float128 float128_default_nan;
+
+#endif /* !SOFTFLOAT_H */
diff --git a/contrib/qemu/include/glib-compat.h b/contrib/qemu/include/glib-compat.h
new file mode 100644
index 000000000..8aa77afd6
--- /dev/null
+++ b/contrib/qemu/include/glib-compat.h
@@ -0,0 +1,27 @@
+/*
+ * GLIB Compatibility Functions
+ *
+ * Copyright IBM, Corp. 2013
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_GLIB_COMPAT_H
+#define QEMU_GLIB_COMPAT_H
+
+#include <glib.h>
+
+#if !GLIB_CHECK_VERSION(2, 14, 0)
+static inline guint g_timeout_add_seconds(guint interval, GSourceFunc function,
+                                          gpointer data)
+{
+    return g_timeout_add(interval * 1000, function, data);
+}
+#endif
+
+#endif
diff --git a/contrib/qemu/include/migration/migration.h b/contrib/qemu/include/migration/migration.h
new file mode 100644
index 000000000..bc9fde0b2
--- /dev/null
+++ b/contrib/qemu/include/migration/migration.h
@@ -0,0 +1,157 @@
+/*
+ * QEMU live migration
+ *
+ * Copyright IBM, Corp. 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_MIGRATION_H
+#define QEMU_MIGRATION_H
+
+#include "qapi/qmp/qdict.h"
+#include "qemu-common.h"
+#include "qemu/thread.h"
+#include "qemu/notify.h"
+#include "qapi/error.h"
+#include "migration/vmstate.h"
+#include "qapi-types.h"
+#include "exec/cpu-common.h"
+
+struct MigrationParams {
+    bool blk;
+    bool shared;
+};
+
+typedef struct MigrationState MigrationState;
+
+struct MigrationState
+{
+    int64_t bandwidth_limit;
+    size_t bytes_xfer;
+    size_t xfer_limit;
+    QemuThread thread;
+    QEMUBH *cleanup_bh;
+    QEMUFile *file;
+
+    int state;
+    MigrationParams params;
+    double mbps;
+    int64_t total_time;
+    int64_t downtime;
+    int64_t expected_downtime;
+    int64_t dirty_pages_rate;
+    int64_t dirty_bytes_rate;
+    bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
+    int64_t xbzrle_cache_size;
+};
+
+void process_incoming_migration(QEMUFile *f);
+
+void qemu_start_incoming_migration(const char *uri, Error **errp);
+
+uint64_t migrate_max_downtime(void);
+
+void do_info_migrate_print(Monitor *mon, const QObject *data);
+
+void do_info_migrate(Monitor *mon, QObject **ret_data);
+
+void exec_start_incoming_migration(const char *host_port, Error **errp);
+
+void exec_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp);
+
+void tcp_start_incoming_migration(const char *host_port, Error **errp);
+
+void tcp_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp);
+
+void unix_start_incoming_migration(const char *path, Error **errp);
+
+void unix_start_outgoing_migration(MigrationState *s, const char *path, Error **errp);
+
+void fd_start_incoming_migration(const char *path, Error **errp);
+
+void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **errp);
+
+void migrate_fd_error(MigrationState *s);
+
+void migrate_fd_connect(MigrationState *s);
+
+int migrate_fd_close(MigrationState *s);
+
+void add_migration_state_change_notifier(Notifier *notify);
+void remove_migration_state_change_notifier(Notifier *notify);
+bool migration_is_active(MigrationState *);
+bool migration_has_finished(MigrationState *);
+bool migration_has_failed(MigrationState *);
+MigrationState *migrate_get_current(void);
+
+uint64_t ram_bytes_remaining(void);
+uint64_t ram_bytes_transferred(void);
+uint64_t ram_bytes_total(void);
+
+void acct_update_position(QEMUFile *f, size_t size, bool zero);
+
+extern SaveVMHandlers savevm_ram_handlers;
+
+uint64_t dup_mig_bytes_transferred(void);
+uint64_t dup_mig_pages_transferred(void);
+uint64_t skipped_mig_bytes_transferred(void);
+uint64_t skipped_mig_pages_transferred(void);
+uint64_t norm_mig_bytes_transferred(void);
+uint64_t norm_mig_pages_transferred(void);
+uint64_t xbzrle_mig_bytes_transferred(void);
+uint64_t xbzrle_mig_pages_transferred(void);
+uint64_t xbzrle_mig_pages_overflow(void);
+uint64_t xbzrle_mig_pages_cache_miss(void);
+
+/**
+ * @migrate_add_blocker - prevent migration from proceeding
+ *
+ * @reason - an error to be returned whenever migration is attempted
+ */
+void migrate_add_blocker(Error *reason);
+
+/**
+ * @migrate_del_blocker - remove a blocking error from migration
+ *
+ * @reason - the error blocking migration
+ */
+void migrate_del_blocker(Error *reason);
+
+bool migrate_rdma_pin_all(void);
+
+bool migrate_auto_converge(void);
+
+int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen,
+                         uint8_t *dst, int dlen);
+int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen);
+
+int migrate_use_xbzrle(void);
+int64_t migrate_xbzrle_cache_size(void);
+
+int64_t xbzrle_cache_resize(int64_t new_size);
+
+void ram_control_before_iterate(QEMUFile *f, uint64_t flags);
+void ram_control_after_iterate(QEMUFile *f, uint64_t flags);
+void ram_control_load_hook(QEMUFile *f, uint64_t flags);
+
+/* Whenever this is found in the data stream, the flags
+ * will be passed to ram_control_load_hook in the incoming-migration
+ * side. This lets before_ram_iterate/after_ram_iterate add
+ * transport-specific sections to the RAM migration data.
+ */
+#define RAM_SAVE_FLAG_HOOK     0x80
+
+#define RAM_SAVE_CONTROL_NOT_SUPP -1000
+#define RAM_SAVE_CONTROL_DELAYED  -2000
+
+size_t ram_control_save_page(QEMUFile *f, ram_addr_t block_offset,
+                             ram_addr_t offset, size_t size,
+                             int *bytes_sent);
+
+#endif
diff --git a/contrib/qemu/include/migration/qemu-file.h b/contrib/qemu/include/migration/qemu-file.h
new file mode 100644
index 000000000..0f757fbeb
--- /dev/null
+++ b/contrib/qemu/include/migration/qemu-file.h
@@ -0,0 +1,266 @@
+/*
+ * QEMU System Emulator
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef QEMU_FILE_H
+#define QEMU_FILE_H 1
+#include "exec/cpu-common.h"
+
+/* This function writes a chunk of data to a file at the given position.
+ * The pos argument can be ignored if the file is only being used for
+ * streaming.  The handler should try to write all of the data it can.
+ */
+typedef int (QEMUFilePutBufferFunc)(void *opaque, const uint8_t *buf,
+                                    int64_t pos, int size);
+
+/* Read a chunk of data from a file at the given position.  The pos argument
+ * can be ignored if the file is only be used for streaming.  The number of
+ * bytes actually read should be returned.
+ */
+typedef int (QEMUFileGetBufferFunc)(void *opaque, uint8_t *buf,
+                                    int64_t pos, int size);
+
+/* Close a file
+ *
+ * Return negative error number on error, 0 or positive value on success.
+ *
+ * The meaning of return value on success depends on the specific back-end being
+ * used.
+ */
+typedef int (QEMUFileCloseFunc)(void *opaque);
+
+/* Called to return the OS file descriptor associated to the QEMUFile.
+ */
+typedef int (QEMUFileGetFD)(void *opaque);
+
+/*
+ * This function writes an iovec to file.
+ */
+typedef ssize_t (QEMUFileWritevBufferFunc)(void *opaque, struct iovec *iov,
+                                           int iovcnt, int64_t pos);
+
+/*
+ * This function provides hooks around different
+ * stages of RAM migration.
+ */
+typedef int (QEMURamHookFunc)(QEMUFile *f, void *opaque, uint64_t flags);
+
+/*
+ * Constants used by ram_control_* hooks
+ */
+#define RAM_CONTROL_SETUP    0
+#define RAM_CONTROL_ROUND    1
+#define RAM_CONTROL_HOOK     2
+#define RAM_CONTROL_FINISH   3
+
+/*
+ * This function allows override of where the RAM page
+ * is saved (such as RDMA, for example.)
+ */
+typedef size_t (QEMURamSaveFunc)(QEMUFile *f, void *opaque,
+                               ram_addr_t block_offset,
+                               ram_addr_t offset,
+                               size_t size,
+                               int *bytes_sent);
+
+typedef struct QEMUFileOps {
+    QEMUFilePutBufferFunc *put_buffer;
+    QEMUFileGetBufferFunc *get_buffer;
+    QEMUFileCloseFunc *close;
+    QEMUFileGetFD *get_fd;
+    QEMUFileWritevBufferFunc *writev_buffer;
+    QEMURamHookFunc *before_ram_iterate;
+    QEMURamHookFunc *after_ram_iterate;
+    QEMURamHookFunc *hook_ram_load;
+    QEMURamSaveFunc *save_page;
+} QEMUFileOps;
+
+QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops);
+QEMUFile *qemu_fopen(const char *filename, const char *mode);
+QEMUFile *qemu_fdopen(int fd, const char *mode);
+QEMUFile *qemu_fopen_socket(int fd, const char *mode);
+QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
+int qemu_get_fd(QEMUFile *f);
+int qemu_fclose(QEMUFile *f);
+int64_t qemu_ftell(QEMUFile *f);
+void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size);
+void qemu_put_byte(QEMUFile *f, int v);
+/*
+ * put_buffer without copying the buffer.
+ * The buffer should be available till it is sent asynchronously.
+ */
+void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size);
+bool qemu_file_mode_is_not_valid(const char *mode);
+
+static inline void qemu_put_ubyte(QEMUFile *f, unsigned int v)
+{
+    qemu_put_byte(f, (int)v);
+}
+
+#define qemu_put_sbyte qemu_put_byte
+
+void qemu_put_be16(QEMUFile *f, unsigned int v);
+void qemu_put_be32(QEMUFile *f, unsigned int v);
+void qemu_put_be64(QEMUFile *f, uint64_t v);
+int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size);
+int qemu_get_byte(QEMUFile *f);
+void qemu_update_position(QEMUFile *f, size_t size);
+
+static inline unsigned int qemu_get_ubyte(QEMUFile *f)
+{
+    return (unsigned int)qemu_get_byte(f);
+}
+
+#define qemu_get_sbyte qemu_get_byte
+
+unsigned int qemu_get_be16(QEMUFile *f);
+unsigned int qemu_get_be32(QEMUFile *f);
+uint64_t qemu_get_be64(QEMUFile *f);
+
+int qemu_file_rate_limit(QEMUFile *f);
+void qemu_file_reset_rate_limit(QEMUFile *f);
+void qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate);
+int64_t qemu_file_get_rate_limit(QEMUFile *f);
+int qemu_file_get_error(QEMUFile *f);
+void qemu_fflush(QEMUFile *f);
+
+static inline void qemu_put_be64s(QEMUFile *f, const uint64_t *pv)
+{
+    qemu_put_be64(f, *pv);
+}
+
+static inline void qemu_put_be32s(QEMUFile *f, const uint32_t *pv)
+{
+    qemu_put_be32(f, *pv);
+}
+
+static inline void qemu_put_be16s(QEMUFile *f, const uint16_t *pv)
+{
+    qemu_put_be16(f, *pv);
+}
+
+static inline void qemu_put_8s(QEMUFile *f, const uint8_t *pv)
+{
+    qemu_put_byte(f, *pv);
+}
+
+static inline void qemu_get_be64s(QEMUFile *f, uint64_t *pv)
+{
+    *pv = qemu_get_be64(f);
+}
+
+static inline void qemu_get_be32s(QEMUFile *f, uint32_t *pv)
+{
+    *pv = qemu_get_be32(f);
+}
+
+static inline void qemu_get_be16s(QEMUFile *f, uint16_t *pv)
+{
+    *pv = qemu_get_be16(f);
+}
+
+static inline void qemu_get_8s(QEMUFile *f, uint8_t *pv)
+{
+    *pv = qemu_get_byte(f);
+}
+
+// Signed versions for type safety
+static inline void qemu_put_sbuffer(QEMUFile *f, const int8_t *buf, int size)
+{
+    qemu_put_buffer(f, (const uint8_t *)buf, size);
+}
+
+static inline void qemu_put_sbe16(QEMUFile *f, int v)
+{
+    qemu_put_be16(f, (unsigned int)v);
+}
+
+static inline void qemu_put_sbe32(QEMUFile *f, int v)
+{
+    qemu_put_be32(f, (unsigned int)v);
+}
+
+static inline void qemu_put_sbe64(QEMUFile *f, int64_t v)
+{
+    qemu_put_be64(f, (uint64_t)v);
+}
+
+static inline size_t qemu_get_sbuffer(QEMUFile *f, int8_t *buf, int size)
+{
+    return qemu_get_buffer(f, (uint8_t *)buf, size);
+}
+
+static inline int qemu_get_sbe16(QEMUFile *f)
+{
+    return (int)qemu_get_be16(f);
+}
+
+static inline int qemu_get_sbe32(QEMUFile *f)
+{
+    return (int)qemu_get_be32(f);
+}
+
+static inline int64_t qemu_get_sbe64(QEMUFile *f)
+{
+    return (int64_t)qemu_get_be64(f);
+}
+
+static inline void qemu_put_s8s(QEMUFile *f, const int8_t *pv)
+{
+    qemu_put_8s(f, (const uint8_t *)pv);
+}
+
+static inline void qemu_put_sbe16s(QEMUFile *f, const int16_t *pv)
+{
+    qemu_put_be16s(f, (const uint16_t *)pv);
+}
+
+static inline void qemu_put_sbe32s(QEMUFile *f, const int32_t *pv)
+{
+    qemu_put_be32s(f, (const uint32_t *)pv);
+}
+
+static inline void qemu_put_sbe64s(QEMUFile *f, const int64_t *pv)
+{
+    qemu_put_be64s(f, (const uint64_t *)pv);
+}
+
+static inline void qemu_get_s8s(QEMUFile *f, int8_t *pv)
+{
+    qemu_get_8s(f, (uint8_t *)pv);
+}
+
+static inline void qemu_get_sbe16s(QEMUFile *f, int16_t *pv)
+{
+    qemu_get_be16s(f, (uint16_t *)pv);
+}
+
+static inline void qemu_get_sbe32s(QEMUFile *f, int32_t *pv)
+{
+    qemu_get_be32s(f, (uint32_t *)pv);
+}
+
+static inline void qemu_get_sbe64s(QEMUFile *f, int64_t *pv)
+{
+    qemu_get_be64s(f, (uint64_t *)pv);
+}
+#endif
diff --git a/contrib/qemu/include/migration/vmstate.h b/contrib/qemu/include/migration/vmstate.h
new file mode 100644
index 000000000..1c31b5d6f
--- /dev/null
+++ b/contrib/qemu/include/migration/vmstate.h
@@ -0,0 +1,740 @@
+/*
+ * QEMU migration/snapshot declarations
+ *
+ * Copyright (c) 2009-2011 Red Hat, Inc.
+ *
+ * Original author: Juan Quintela <quintela@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef QEMU_VMSTATE_H
+#define QEMU_VMSTATE_H 1
+
+#ifndef CONFIG_USER_ONLY
+#include <migration/qemu-file.h>
+#endif
+
+typedef void SaveStateHandler(QEMUFile *f, void *opaque);
+typedef int LoadStateHandler(QEMUFile *f, void *opaque, int version_id);
+
+typedef struct SaveVMHandlers {
+    /* This runs inside the iothread lock.  */
+    void (*set_params)(const MigrationParams *params, void * opaque);
+    SaveStateHandler *save_state;
+
+    void (*cancel)(void *opaque);
+    int (*save_live_complete)(QEMUFile *f, void *opaque);
+
+    /* This runs both outside and inside the iothread lock.  */
+    bool (*is_active)(void *opaque);
+
+    /* This runs outside the iothread lock in the migration case, and
+     * within the lock in the savevm case.  The callback had better only
+     * use data that is local to the migration thread or protected
+     * by other locks.
+     */
+    int (*save_live_iterate)(QEMUFile *f, void *opaque);
+
+    /* This runs outside the iothread lock!  */
+    int (*save_live_setup)(QEMUFile *f, void *opaque);
+    uint64_t (*save_live_pending)(QEMUFile *f, void *opaque, uint64_t max_size);
+
+    LoadStateHandler *load_state;
+} SaveVMHandlers;
+
+int register_savevm(DeviceState *dev,
+                    const char *idstr,
+                    int instance_id,
+                    int version_id,
+                    SaveStateHandler *save_state,
+                    LoadStateHandler *load_state,
+                    void *opaque);
+
+int register_savevm_live(DeviceState *dev,
+                         const char *idstr,
+                         int instance_id,
+                         int version_id,
+                         SaveVMHandlers *ops,
+                         void *opaque);
+
+void unregister_savevm(DeviceState *dev, const char *idstr, void *opaque);
+void register_device_unmigratable(DeviceState *dev, const char *idstr,
+                                                                void *opaque);
+
+
+typedef struct VMStateInfo VMStateInfo;
+typedef struct VMStateDescription VMStateDescription;
+
+struct VMStateInfo {
+    const char *name;
+    int (*get)(QEMUFile *f, void *pv, size_t size);
+    void (*put)(QEMUFile *f, void *pv, size_t size);
+};
+
+enum VMStateFlags {
+    VMS_SINGLE           = 0x001,
+    VMS_POINTER          = 0x002,
+    VMS_ARRAY            = 0x004,
+    VMS_STRUCT           = 0x008,
+    VMS_VARRAY_INT32     = 0x010,  /* Array with size in int32_t field*/
+    VMS_BUFFER           = 0x020,  /* static sized buffer */
+    VMS_ARRAY_OF_POINTER = 0x040,
+    VMS_VARRAY_UINT16    = 0x080,  /* Array with size in uint16_t field */
+    VMS_VBUFFER          = 0x100,  /* Buffer with size in int32_t field */
+    VMS_MULTIPLY         = 0x200,  /* multiply "size" field by field_size */
+    VMS_VARRAY_UINT8     = 0x400,  /* Array with size in uint8_t field*/
+    VMS_VARRAY_UINT32    = 0x800,  /* Array with size in uint32_t field*/
+};
+
+typedef struct {
+    const char *name;
+    size_t offset;
+    size_t size;
+    size_t start;
+    int num;
+    size_t num_offset;
+    size_t size_offset;
+    const VMStateInfo *info;
+    enum VMStateFlags flags;
+    const VMStateDescription *vmsd;
+    int version_id;
+    bool (*field_exists)(void *opaque, int version_id);
+} VMStateField;
+
+typedef struct VMStateSubsection {
+    const VMStateDescription *vmsd;
+    bool (*needed)(void *opaque);
+} VMStateSubsection;
+
+struct VMStateDescription {
+    const char *name;
+    int unmigratable;
+    int version_id;
+    int minimum_version_id;
+    int minimum_version_id_old;
+    LoadStateHandler *load_state_old;
+    int (*pre_load)(void *opaque);
+    int (*post_load)(void *opaque, int version_id);
+    void (*pre_save)(void *opaque);
+    VMStateField *fields;
+    const VMStateSubsection *subsections;
+};
+
+#ifdef CONFIG_USER_ONLY
+extern const VMStateDescription vmstate_dummy;
+#endif
+
+extern const VMStateInfo vmstate_info_bool;
+
+extern const VMStateInfo vmstate_info_int8;
+extern const VMStateInfo vmstate_info_int16;
+extern const VMStateInfo vmstate_info_int32;
+extern const VMStateInfo vmstate_info_int64;
+
+extern const VMStateInfo vmstate_info_uint8_equal;
+extern const VMStateInfo vmstate_info_uint16_equal;
+extern const VMStateInfo vmstate_info_int32_equal;
+extern const VMStateInfo vmstate_info_uint32_equal;
+extern const VMStateInfo vmstate_info_uint64_equal;
+extern const VMStateInfo vmstate_info_int32_le;
+
+extern const VMStateInfo vmstate_info_uint8;
+extern const VMStateInfo vmstate_info_uint16;
+extern const VMStateInfo vmstate_info_uint32;
+extern const VMStateInfo vmstate_info_uint64;
+
+extern const VMStateInfo vmstate_info_float64;
+
+extern const VMStateInfo vmstate_info_timer;
+extern const VMStateInfo vmstate_info_buffer;
+extern const VMStateInfo vmstate_info_unused_buffer;
+extern const VMStateInfo vmstate_info_bitmap;
+
+#define type_check_2darray(t1,t2,n,m) ((t1(*)[n][m])0 - (t2*)0)
+#define type_check_array(t1,t2,n) ((t1(*)[n])0 - (t2*)0)
+#define type_check_pointer(t1,t2) ((t1**)0 - (t2*)0)
+
+#define vmstate_offset_value(_state, _field, _type)                  \
+    (offsetof(_state, _field) +                                      \
+     type_check(_type, typeof_field(_state, _field)))
+
+#define vmstate_offset_pointer(_state, _field, _type)                \
+    (offsetof(_state, _field) +                                      \
+     type_check_pointer(_type, typeof_field(_state, _field)))
+
+#define vmstate_offset_array(_state, _field, _type, _num)            \
+    (offsetof(_state, _field) +                                      \
+     type_check_array(_type, typeof_field(_state, _field), _num))
+
+#define vmstate_offset_2darray(_state, _field, _type, _n1, _n2)      \
+    (offsetof(_state, _field) +                                      \
+     type_check_2darray(_type, typeof_field(_state, _field), _n1, _n2))
+
+#define vmstate_offset_sub_array(_state, _field, _type, _start)      \
+    (offsetof(_state, _field[_start]))
+
+#define vmstate_offset_buffer(_state, _field)                        \
+    vmstate_offset_array(_state, _field, uint8_t,                    \
+                         sizeof(typeof_field(_state, _field)))
+
+#define VMSTATE_SINGLE_TEST(_field, _state, _test, _version, _info, _type) { \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                      \
+    .field_exists = (_test),                                         \
+    .size         = sizeof(_type),                                   \
+    .info         = &(_info),                                        \
+    .flags        = VMS_SINGLE,                                      \
+    .offset       = vmstate_offset_value(_state, _field, _type),     \
+}
+
+#define VMSTATE_POINTER(_field, _state, _version, _info, _type) {    \
+    .name       = (stringify(_field)),                               \
+    .version_id = (_version),                                        \
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_SINGLE|VMS_POINTER,                            \
+    .offset     = vmstate_offset_value(_state, _field, _type),       \
+}
+
+#define VMSTATE_POINTER_TEST(_field, _state, _test, _info, _type) {  \
+    .name       = (stringify(_field)),                               \
+    .info       = &(_info),                                          \
+    .field_exists = (_test),                                         \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_SINGLE|VMS_POINTER,                            \
+    .offset     = vmstate_offset_value(_state, _field, _type),       \
+}
+
+#define VMSTATE_ARRAY(_field, _state, _num, _version, _info, _type) {\
+    .name       = (stringify(_field)),                               \
+    .version_id = (_version),                                        \
+    .num        = (_num),                                            \
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_ARRAY,                                         \
+    .offset     = vmstate_offset_array(_state, _field, _type, _num), \
+}
+
+#define VMSTATE_2DARRAY(_field, _state, _n1, _n2, _version, _info, _type) { \
+    .name       = (stringify(_field)),                                      \
+    .version_id = (_version),                                               \
+    .num        = (_n1) * (_n2),                                            \
+    .info       = &(_info),                                                 \
+    .size       = sizeof(_type),                                            \
+    .flags      = VMS_ARRAY,                                                \
+    .offset     = vmstate_offset_2darray(_state, _field, _type, _n1, _n2),  \
+}
+
+#define VMSTATE_ARRAY_TEST(_field, _state, _num, _test, _info, _type) {\
+    .name         = (stringify(_field)),                              \
+    .field_exists = (_test),                                          \
+    .num          = (_num),                                           \
+    .info         = &(_info),                                         \
+    .size         = sizeof(_type),                                    \
+    .flags        = VMS_ARRAY,                                        \
+    .offset       = vmstate_offset_array(_state, _field, _type, _num),\
+}
+
+#define VMSTATE_SUB_ARRAY(_field, _state, _start, _num, _version, _info, _type) { \
+    .name       = (stringify(_field)),                               \
+    .version_id = (_version),                                        \
+    .num        = (_num),                                            \
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_ARRAY,                                         \
+    .offset     = vmstate_offset_sub_array(_state, _field, _type, _start), \
+}
+
+#define VMSTATE_ARRAY_INT32_UNSAFE(_field, _state, _field_num, _info, _type) {\
+    .name       = (stringify(_field)),                               \
+    .num_offset = vmstate_offset_value(_state, _field_num, int32_t), \
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_VARRAY_INT32,                                  \
+    .offset     = offsetof(_state, _field),                          \
+}
+
+#define VMSTATE_VARRAY_INT32(_field, _state, _field_num, _version, _info, _type) {\
+    .name       = (stringify(_field)),                               \
+    .version_id = (_version),                                        \
+    .num_offset = vmstate_offset_value(_state, _field_num, int32_t), \
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_VARRAY_INT32|VMS_POINTER,                      \
+    .offset     = vmstate_offset_pointer(_state, _field, _type),     \
+}
+
+#define VMSTATE_VARRAY_UINT32(_field, _state, _field_num, _version, _info, _type) {\
+    .name       = (stringify(_field)),                               \
+    .version_id = (_version),                                        \
+    .num_offset = vmstate_offset_value(_state, _field_num, uint32_t),\
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_VARRAY_UINT32|VMS_POINTER,                     \
+    .offset     = vmstate_offset_pointer(_state, _field, _type),     \
+}
+
+#define VMSTATE_VARRAY_UINT16_UNSAFE(_field, _state, _field_num, _version, _info, _type) {\
+    .name       = (stringify(_field)),                               \
+    .version_id = (_version),                                        \
+    .num_offset = vmstate_offset_value(_state, _field_num, uint16_t),\
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_VARRAY_UINT16,                                 \
+    .offset     = offsetof(_state, _field),                          \
+}
+
+#define VMSTATE_STRUCT_TEST(_field, _state, _test, _version, _vmsd, _type) { \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                      \
+    .field_exists = (_test),                                         \
+    .vmsd         = &(_vmsd),                                        \
+    .size         = sizeof(_type),                                   \
+    .flags        = VMS_STRUCT,                                      \
+    .offset       = vmstate_offset_value(_state, _field, _type),     \
+}
+
+#define VMSTATE_STRUCT_POINTER_TEST(_field, _state, _test, _vmsd, _type) { \
+    .name         = (stringify(_field)),                             \
+    .field_exists = (_test),                                         \
+    .vmsd         = &(_vmsd),                                        \
+    .size         = sizeof(_type),                                   \
+    .flags        = VMS_STRUCT|VMS_POINTER,                          \
+    .offset       = vmstate_offset_value(_state, _field, _type),     \
+}
+
+#define VMSTATE_ARRAY_OF_POINTER(_field, _state, _num, _version, _info, _type) {\
+    .name       = (stringify(_field)),                               \
+    .version_id = (_version),                                        \
+    .num        = (_num),                                            \
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_ARRAY|VMS_ARRAY_OF_POINTER,                    \
+    .offset     = vmstate_offset_array(_state, _field, _type, _num), \
+}
+
+#define VMSTATE_STRUCT_ARRAY_TEST(_field, _state, _num, _test, _version, _vmsd, _type) { \
+    .name         = (stringify(_field)),                             \
+    .num          = (_num),                                          \
+    .field_exists = (_test),                                         \
+    .version_id   = (_version),                                      \
+    .vmsd         = &(_vmsd),                                        \
+    .size         = sizeof(_type),                                   \
+    .flags        = VMS_STRUCT|VMS_ARRAY,                            \
+    .offset       = vmstate_offset_array(_state, _field, _type, _num),\
+}
+
+#define VMSTATE_STRUCT_VARRAY_UINT8(_field, _state, _field_num, _version, _vmsd, _type) { \
+    .name       = (stringify(_field)),                               \
+    .num_offset = vmstate_offset_value(_state, _field_num, uint8_t), \
+    .version_id = (_version),                                        \
+    .vmsd       = &(_vmsd),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_STRUCT|VMS_VARRAY_UINT8,                       \
+    .offset     = offsetof(_state, _field),                          \
+}
+
+#define VMSTATE_STRUCT_VARRAY_POINTER_INT32(_field, _state, _field_num, _vmsd, _type) { \
+    .name       = (stringify(_field)),                               \
+    .version_id = 0,                                                 \
+    .num_offset = vmstate_offset_value(_state, _field_num, int32_t), \
+    .size       = sizeof(_type),                                     \
+    .vmsd       = &(_vmsd),                                          \
+    .flags      = VMS_POINTER | VMS_VARRAY_INT32 | VMS_STRUCT,       \
+    .offset     = vmstate_offset_pointer(_state, _field, _type),     \
+}
+
+#define VMSTATE_STRUCT_VARRAY_POINTER_UINT32(_field, _state, _field_num, _vmsd, _type) { \
+    .name       = (stringify(_field)),                               \
+    .version_id = 0,                                                 \
+    .num_offset = vmstate_offset_value(_state, _field_num, uint32_t),\
+    .size       = sizeof(_type),                                     \
+    .vmsd       = &(_vmsd),                                          \
+    .flags      = VMS_POINTER | VMS_VARRAY_INT32 | VMS_STRUCT,       \
+    .offset     = vmstate_offset_pointer(_state, _field, _type),     \
+}
+
+#define VMSTATE_STRUCT_VARRAY_POINTER_UINT16(_field, _state, _field_num, _vmsd, _type) { \
+    .name       = (stringify(_field)),                               \
+    .version_id = 0,                                                 \
+    .num_offset = vmstate_offset_value(_state, _field_num, uint16_t),\
+    .size       = sizeof(_type),                                     \
+    .vmsd       = &(_vmsd),                                          \
+    .flags      = VMS_POINTER | VMS_VARRAY_UINT16 | VMS_STRUCT,      \
+    .offset     = vmstate_offset_pointer(_state, _field, _type),     \
+}
+
+#define VMSTATE_STRUCT_VARRAY_INT32(_field, _state, _field_num, _version, _vmsd, _type) { \
+    .name       = (stringify(_field)),                               \
+    .num_offset = vmstate_offset_value(_state, _field_num, int32_t), \
+    .version_id = (_version),                                        \
+    .vmsd       = &(_vmsd),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_STRUCT|VMS_VARRAY_INT32,                       \
+    .offset     = offsetof(_state, _field),                          \
+}
+
+#define VMSTATE_STRUCT_VARRAY_UINT32(_field, _state, _field_num, _version, _vmsd, _type) { \
+    .name       = (stringify(_field)),                               \
+    .num_offset = vmstate_offset_value(_state, _field_num, uint32_t), \
+    .version_id = (_version),                                        \
+    .vmsd       = &(_vmsd),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_STRUCT|VMS_VARRAY_UINT32,                      \
+    .offset     = offsetof(_state, _field),                          \
+}
+
+#define VMSTATE_STATIC_BUFFER(_field, _state, _version, _test, _start, _size) { \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                      \
+    .field_exists = (_test),                                         \
+    .size         = (_size - _start),                                \
+    .info         = &vmstate_info_buffer,                            \
+    .flags        = VMS_BUFFER,                                      \
+    .offset       = vmstate_offset_buffer(_state, _field) + _start,  \
+}
+
+#define VMSTATE_VBUFFER_MULTIPLY(_field, _state, _version, _test, _start, _field_size, _multiply) { \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                      \
+    .field_exists = (_test),                                         \
+    .size_offset  = vmstate_offset_value(_state, _field_size, uint32_t),\
+    .size         = (_multiply),                                      \
+    .info         = &vmstate_info_buffer,                            \
+    .flags        = VMS_VBUFFER|VMS_POINTER|VMS_MULTIPLY,            \
+    .offset       = offsetof(_state, _field),                        \
+    .start        = (_start),                                        \
+}
+
+#define VMSTATE_VBUFFER(_field, _state, _version, _test, _start, _field_size) { \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                      \
+    .field_exists = (_test),                                         \
+    .size_offset  = vmstate_offset_value(_state, _field_size, int32_t),\
+    .info         = &vmstate_info_buffer,                            \
+    .flags        = VMS_VBUFFER|VMS_POINTER,                         \
+    .offset       = offsetof(_state, _field),                        \
+    .start        = (_start),                                        \
+}
+
+#define VMSTATE_VBUFFER_UINT32(_field, _state, _version, _test, _start, _field_size) { \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                      \
+    .field_exists = (_test),                                         \
+    .size_offset  = vmstate_offset_value(_state, _field_size, uint32_t),\
+    .info         = &vmstate_info_buffer,                            \
+    .flags        = VMS_VBUFFER|VMS_POINTER,                         \
+    .offset       = offsetof(_state, _field),                        \
+    .start        = (_start),                                        \
+}
+
+#define VMSTATE_BUFFER_UNSAFE_INFO(_field, _state, _version, _info, _size) { \
+    .name       = (stringify(_field)),                               \
+    .version_id = (_version),                                        \
+    .size       = (_size),                                           \
+    .info       = &(_info),                                          \
+    .flags      = VMS_BUFFER,                                        \
+    .offset     = offsetof(_state, _field),                          \
+}
+
+#define VMSTATE_BUFFER_POINTER_UNSAFE(_field, _state, _version, _size) { \
+    .name       = (stringify(_field)),                               \
+    .version_id = (_version),                                        \
+    .size       = (_size),                                           \
+    .info       = &vmstate_info_buffer,                              \
+    .flags      = VMS_BUFFER|VMS_POINTER,                            \
+    .offset     = offsetof(_state, _field),                          \
+}
+
+#define VMSTATE_UNUSED_BUFFER(_test, _version, _size) {              \
+    .name         = "unused",                                        \
+    .field_exists = (_test),                                         \
+    .version_id   = (_version),                                      \
+    .size         = (_size),                                         \
+    .info         = &vmstate_info_unused_buffer,                     \
+    .flags        = VMS_BUFFER,                                      \
+}
+
+/* _field_size should be a int32_t field in the _state struct giving the
+ * size of the bitmap _field in bits.
+ */
+#define VMSTATE_BITMAP(_field, _state, _version, _field_size) {      \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                      \
+    .size_offset  = vmstate_offset_value(_state, _field_size, int32_t),\
+    .info         = &vmstate_info_bitmap,                            \
+    .flags        = VMS_VBUFFER|VMS_POINTER,                         \
+    .offset       = offsetof(_state, _field),                        \
+}
+
+/* _f : field name
+   _f_n : num of elements field_name
+   _n : num of elements
+   _s : struct state name
+   _v : version
+*/
+
+#define VMSTATE_SINGLE(_field, _state, _version, _info, _type)        \
+    VMSTATE_SINGLE_TEST(_field, _state, NULL, _version, _info, _type)
+
+#define VMSTATE_STRUCT(_field, _state, _version, _vmsd, _type)        \
+    VMSTATE_STRUCT_TEST(_field, _state, NULL, _version, _vmsd, _type)
+
+#define VMSTATE_STRUCT_POINTER(_field, _state, _vmsd, _type)          \
+    VMSTATE_STRUCT_POINTER_TEST(_field, _state, NULL, _vmsd, _type)
+
+#define VMSTATE_STRUCT_ARRAY(_field, _state, _num, _version, _vmsd, _type) \
+    VMSTATE_STRUCT_ARRAY_TEST(_field, _state, _num, NULL, _version,   \
+            _vmsd, _type)
+
+#define VMSTATE_BOOL_V(_f, _s, _v)                                    \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_bool, bool)
+
+#define VMSTATE_INT8_V(_f, _s, _v)                                    \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_int8, int8_t)
+#define VMSTATE_INT16_V(_f, _s, _v)                                   \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_int16, int16_t)
+#define VMSTATE_INT32_V(_f, _s, _v)                                   \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_int32, int32_t)
+#define VMSTATE_INT64_V(_f, _s, _v)                                   \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_int64, int64_t)
+
+#define VMSTATE_UINT8_V(_f, _s, _v)                                   \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_uint8, uint8_t)
+#define VMSTATE_UINT16_V(_f, _s, _v)                                  \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_uint16, uint16_t)
+#define VMSTATE_UINT32_V(_f, _s, _v)                                  \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_uint32, uint32_t)
+#define VMSTATE_UINT64_V(_f, _s, _v)                                  \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_uint64, uint64_t)
+
+#define VMSTATE_BOOL(_f, _s)                                          \
+    VMSTATE_BOOL_V(_f, _s, 0)
+
+#define VMSTATE_INT8(_f, _s)                                          \
+    VMSTATE_INT8_V(_f, _s, 0)
+#define VMSTATE_INT16(_f, _s)                                         \
+    VMSTATE_INT16_V(_f, _s, 0)
+#define VMSTATE_INT32(_f, _s)                                         \
+    VMSTATE_INT32_V(_f, _s, 0)
+#define VMSTATE_INT64(_f, _s)                                         \
+    VMSTATE_INT64_V(_f, _s, 0)
+
+#define VMSTATE_UINT8(_f, _s)                                         \
+    VMSTATE_UINT8_V(_f, _s, 0)
+#define VMSTATE_UINT16(_f, _s)                                        \
+    VMSTATE_UINT16_V(_f, _s, 0)
+#define VMSTATE_UINT32(_f, _s)                                        \
+    VMSTATE_UINT32_V(_f, _s, 0)
+#define VMSTATE_UINT64(_f, _s)                                        \
+    VMSTATE_UINT64_V(_f, _s, 0)
+
+#define VMSTATE_UINT8_EQUAL(_f, _s)                                   \
+    VMSTATE_SINGLE(_f, _s, 0, vmstate_info_uint8_equal, uint8_t)
+
+#define VMSTATE_UINT16_EQUAL(_f, _s)                                  \
+    VMSTATE_SINGLE(_f, _s, 0, vmstate_info_uint16_equal, uint16_t)
+
+#define VMSTATE_UINT16_EQUAL_V(_f, _s, _v)                            \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_uint16_equal, uint16_t)
+
+#define VMSTATE_INT32_EQUAL(_f, _s)                                   \
+    VMSTATE_SINGLE(_f, _s, 0, vmstate_info_int32_equal, int32_t)
+
+#define VMSTATE_UINT32_EQUAL_V(_f, _s, _v)                            \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_uint32_equal, uint32_t)
+
+#define VMSTATE_UINT32_EQUAL(_f, _s)                                  \
+    VMSTATE_UINT32_EQUAL_V(_f, _s, 0)
+
+#define VMSTATE_UINT64_EQUAL_V(_f, _s, _v)                            \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_uint64_equal, uint64_t)
+
+#define VMSTATE_UINT64_EQUAL(_f, _s)                                  \
+    VMSTATE_UINT64_EQUAL_V(_f, _s, 0)
+
+#define VMSTATE_INT32_LE(_f, _s)                                   \
+    VMSTATE_SINGLE(_f, _s, 0, vmstate_info_int32_le, int32_t)
+
+#define VMSTATE_UINT8_TEST(_f, _s, _t)                               \
+    VMSTATE_SINGLE_TEST(_f, _s, _t, 0, vmstate_info_uint8, uint8_t)
+
+#define VMSTATE_UINT16_TEST(_f, _s, _t)                               \
+    VMSTATE_SINGLE_TEST(_f, _s, _t, 0, vmstate_info_uint16, uint16_t)
+
+#define VMSTATE_UINT32_TEST(_f, _s, _t)                                  \
+    VMSTATE_SINGLE_TEST(_f, _s, _t, 0, vmstate_info_uint32, uint32_t)
+
+
+#define VMSTATE_FLOAT64_V(_f, _s, _v)                                 \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_float64, float64)
+
+#define VMSTATE_FLOAT64(_f, _s)                                       \
+    VMSTATE_FLOAT64_V(_f, _s, 0)
+
+#define VMSTATE_TIMER_TEST(_f, _s, _test)                             \
+    VMSTATE_POINTER_TEST(_f, _s, _test, vmstate_info_timer, QEMUTimer *)
+
+#define VMSTATE_TIMER_V(_f, _s, _v)                                   \
+    VMSTATE_POINTER(_f, _s, _v, vmstate_info_timer, QEMUTimer *)
+
+#define VMSTATE_TIMER(_f, _s)                                         \
+    VMSTATE_TIMER_V(_f, _s, 0)
+
+#define VMSTATE_TIMER_ARRAY(_f, _s, _n)                              \
+    VMSTATE_ARRAY_OF_POINTER(_f, _s, _n, 0, vmstate_info_timer, QEMUTimer *)
+
+#define VMSTATE_BOOL_ARRAY_V(_f, _s, _n, _v)                         \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_bool, bool)
+
+#define VMSTATE_BOOL_ARRAY(_f, _s, _n)                               \
+    VMSTATE_BOOL_ARRAY_V(_f, _s, _n, 0)
+
+#define VMSTATE_UINT16_ARRAY_V(_f, _s, _n, _v)                         \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_uint16, uint16_t)
+
+#define VMSTATE_UINT16_2DARRAY_V(_f, _s, _n1, _n2, _v)                \
+    VMSTATE_2DARRAY(_f, _s, _n1, _n2, _v, vmstate_info_uint16, uint16_t)
+
+#define VMSTATE_UINT16_ARRAY(_f, _s, _n)                               \
+    VMSTATE_UINT16_ARRAY_V(_f, _s, _n, 0)
+
+#define VMSTATE_UINT16_2DARRAY(_f, _s, _n1, _n2)                      \
+    VMSTATE_UINT16_2DARRAY_V(_f, _s, _n1, _n2, 0)
+
+#define VMSTATE_UINT8_2DARRAY_V(_f, _s, _n1, _n2, _v)                 \
+    VMSTATE_2DARRAY(_f, _s, _n1, _n2, _v, vmstate_info_uint8, uint8_t)
+
+#define VMSTATE_UINT8_ARRAY_V(_f, _s, _n, _v)                         \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_uint8, uint8_t)
+
+#define VMSTATE_UINT8_ARRAY(_f, _s, _n)                               \
+    VMSTATE_UINT8_ARRAY_V(_f, _s, _n, 0)
+
+#define VMSTATE_UINT8_2DARRAY(_f, _s, _n1, _n2)                       \
+    VMSTATE_UINT8_2DARRAY_V(_f, _s, _n1, _n2, 0)
+
+#define VMSTATE_UINT32_ARRAY_V(_f, _s, _n, _v)                        \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_uint32, uint32_t)
+
+#define VMSTATE_UINT32_ARRAY(_f, _s, _n)                              \
+    VMSTATE_UINT32_ARRAY_V(_f, _s, _n, 0)
+
+#define VMSTATE_UINT64_ARRAY_V(_f, _s, _n, _v)                        \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_uint64, uint64_t)
+
+#define VMSTATE_UINT64_ARRAY(_f, _s, _n)                              \
+    VMSTATE_UINT64_ARRAY_V(_f, _s, _n, 0)
+
+#define VMSTATE_INT16_ARRAY_V(_f, _s, _n, _v)                         \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_int16, int16_t)
+
+#define VMSTATE_INT16_ARRAY(_f, _s, _n)                               \
+    VMSTATE_INT16_ARRAY_V(_f, _s, _n, 0)
+
+#define VMSTATE_INT32_ARRAY_V(_f, _s, _n, _v)                         \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_int32, int32_t)
+
+#define VMSTATE_INT32_ARRAY(_f, _s, _n)                               \
+    VMSTATE_INT32_ARRAY_V(_f, _s, _n, 0)
+
+#define VMSTATE_UINT32_SUB_ARRAY(_f, _s, _start, _num)                \
+    VMSTATE_SUB_ARRAY(_f, _s, _start, _num, 0, vmstate_info_uint32, uint32_t)
+
+#define VMSTATE_UINT32_ARRAY(_f, _s, _n)                              \
+    VMSTATE_UINT32_ARRAY_V(_f, _s, _n, 0)
+
+#define VMSTATE_INT64_ARRAY_V(_f, _s, _n, _v)                         \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_int64, int64_t)
+
+#define VMSTATE_INT64_ARRAY(_f, _s, _n)                               \
+    VMSTATE_INT64_ARRAY_V(_f, _s, _n, 0)
+
+#define VMSTATE_FLOAT64_ARRAY_V(_f, _s, _n, _v)                       \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_float64, float64)
+
+#define VMSTATE_FLOAT64_ARRAY(_f, _s, _n)                             \
+    VMSTATE_FLOAT64_ARRAY_V(_f, _s, _n, 0)
+
+#define VMSTATE_BUFFER_V(_f, _s, _v)                                  \
+    VMSTATE_STATIC_BUFFER(_f, _s, _v, NULL, 0, sizeof(typeof_field(_s, _f)))
+
+#define VMSTATE_BUFFER(_f, _s)                                        \
+    VMSTATE_BUFFER_V(_f, _s, 0)
+
+#define VMSTATE_PARTIAL_BUFFER(_f, _s, _size)                         \
+    VMSTATE_STATIC_BUFFER(_f, _s, 0, NULL, 0, _size)
+
+#define VMSTATE_BUFFER_START_MIDDLE(_f, _s, _start) \
+    VMSTATE_STATIC_BUFFER(_f, _s, 0, NULL, _start, sizeof(typeof_field(_s, _f)))
+
+#define VMSTATE_PARTIAL_VBUFFER(_f, _s, _size)                        \
+    VMSTATE_VBUFFER(_f, _s, 0, NULL, 0, _size)
+
+#define VMSTATE_PARTIAL_VBUFFER_UINT32(_f, _s, _size)                        \
+    VMSTATE_VBUFFER_UINT32(_f, _s, 0, NULL, 0, _size)
+
+#define VMSTATE_SUB_VBUFFER(_f, _s, _start, _size)                    \
+    VMSTATE_VBUFFER(_f, _s, 0, NULL, _start, _size)
+
+#define VMSTATE_BUFFER_TEST(_f, _s, _test)                            \
+    VMSTATE_STATIC_BUFFER(_f, _s, 0, _test, 0, sizeof(typeof_field(_s, _f)))
+
+#define VMSTATE_BUFFER_UNSAFE(_field, _state, _version, _size)        \
+    VMSTATE_BUFFER_UNSAFE_INFO(_field, _state, _version, vmstate_info_buffer, _size)
+
+#define VMSTATE_UNUSED_V(_v, _size)                                   \
+    VMSTATE_UNUSED_BUFFER(NULL, _v, _size)
+
+#define VMSTATE_UNUSED(_size)                                         \
+    VMSTATE_UNUSED_V(0, _size)
+
+#define VMSTATE_UNUSED_TEST(_test, _size)                             \
+    VMSTATE_UNUSED_BUFFER(_test, 0, _size)
+
+#define VMSTATE_END_OF_LIST()                                         \
+    {}
+
+int vmstate_load_state(QEMUFile *f, const VMStateDescription *vmsd,
+                       void *opaque, int version_id);
+void vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd,
+                        void *opaque);
+
+int vmstate_register_with_alias_id(DeviceState *dev, int instance_id,
+                                   const VMStateDescription *vmsd,
+                                   void *base, int alias_id,
+                                   int required_for_version);
+
+static inline int vmstate_register(DeviceState *dev, int instance_id,
+                                   const VMStateDescription *vmsd,
+                                   void *opaque)
+{
+    return vmstate_register_with_alias_id(dev, instance_id, vmsd,
+                                          opaque, -1, 0);
+}
+
+void vmstate_unregister(DeviceState *dev, const VMStateDescription *vmsd,
+                        void *opaque);
+
+struct MemoryRegion;
+void vmstate_register_ram(struct MemoryRegion *memory, DeviceState *dev);
+void vmstate_unregister_ram(struct MemoryRegion *memory, DeviceState *dev);
+void vmstate_register_ram_global(struct MemoryRegion *memory);
+
+#endif
diff --git a/contrib/qemu/include/monitor/monitor.h b/contrib/qemu/include/monitor/monitor.h
new file mode 100644
index 000000000..1942cc42f
--- /dev/null
+++ b/contrib/qemu/include/monitor/monitor.h
@@ -0,0 +1,104 @@
+#ifndef MONITOR_H
+#define MONITOR_H
+
+#include "qemu-common.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/qdict.h"
+#include "block/block.h"
+#include "monitor/readline.h"
+
+extern Monitor *cur_mon;
+extern Monitor *default_mon;
+
+/* flags for monitor_init */
+#define MONITOR_IS_DEFAULT    0x01
+#define MONITOR_USE_READLINE  0x02
+#define MONITOR_USE_CONTROL   0x04
+#define MONITOR_USE_PRETTY    0x08
+
+/* flags for monitor commands */
+#define MONITOR_CMD_ASYNC       0x0001
+
+/* QMP events */
+typedef enum MonitorEvent {
+    QEVENT_SHUTDOWN,
+    QEVENT_RESET,
+    QEVENT_POWERDOWN,
+    QEVENT_STOP,
+    QEVENT_RESUME,
+    QEVENT_VNC_CONNECTED,
+    QEVENT_VNC_INITIALIZED,
+    QEVENT_VNC_DISCONNECTED,
+    QEVENT_BLOCK_IO_ERROR,
+    QEVENT_RTC_CHANGE,
+    QEVENT_WATCHDOG,
+    QEVENT_SPICE_CONNECTED,
+    QEVENT_SPICE_INITIALIZED,
+    QEVENT_SPICE_DISCONNECTED,
+    QEVENT_BLOCK_JOB_COMPLETED,
+    QEVENT_BLOCK_JOB_CANCELLED,
+    QEVENT_BLOCK_JOB_ERROR,
+    QEVENT_BLOCK_JOB_READY,
+    QEVENT_DEVICE_DELETED,
+    QEVENT_DEVICE_TRAY_MOVED,
+    QEVENT_NIC_RX_FILTER_CHANGED,
+    QEVENT_SUSPEND,
+    QEVENT_SUSPEND_DISK,
+    QEVENT_WAKEUP,
+    QEVENT_BALLOON_CHANGE,
+    QEVENT_SPICE_MIGRATE_COMPLETED,
+    QEVENT_GUEST_PANICKED,
+
+    /* Add to 'monitor_event_names' array in monitor.c when
+     * defining new events here */
+
+    QEVENT_MAX,
+} MonitorEvent;
+
+int monitor_cur_is_qmp(void);
+
+void monitor_protocol_event(MonitorEvent event, QObject *data);
+void monitor_init(CharDriverState *chr, int flags);
+
+int monitor_suspend(Monitor *mon);
+void monitor_resume(Monitor *mon);
+
+int monitor_read_bdrv_key_start(Monitor *mon, BlockDriverState *bs,
+                                BlockDriverCompletionFunc *completion_cb,
+                                void *opaque);
+int monitor_read_block_device_key(Monitor *mon, const char *device,
+                                  BlockDriverCompletionFunc *completion_cb,
+                                  void *opaque);
+
+int monitor_get_fd(Monitor *mon, const char *fdname, Error **errp);
+int monitor_handle_fd_param(Monitor *mon, const char *fdname);
+
+void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
+    GCC_FMT_ATTR(2, 0);
+void monitor_printf(Monitor *mon, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
+void monitor_print_filename(Monitor *mon, const char *filename);
+void monitor_flush(Monitor *mon);
+int monitor_set_cpu(int cpu_index);
+int monitor_get_cpu_index(void);
+
+typedef void (MonitorCompletion)(void *opaque, QObject *ret_data);
+
+void monitor_set_error(Monitor *mon, QError *qerror);
+void monitor_read_command(Monitor *mon, int show_prompt);
+ReadLineState *monitor_get_rs(Monitor *mon);
+int monitor_read_password(Monitor *mon, ReadLineFunc *readline_func,
+                          void *opaque);
+
+int qmp_qom_set(Monitor *mon, const QDict *qdict, QObject **ret);
+
+int qmp_qom_get(Monitor *mon, const QDict *qdict, QObject **ret);
+
+AddfdInfo *monitor_fdset_add_fd(int fd, bool has_fdset_id, int64_t fdset_id,
+                                bool has_opaque, const char *opaque,
+                                Error **errp);
+int monitor_fdset_get_fd(int64_t fdset_id, int flags);
+int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd);
+int monitor_fdset_dup_fd_remove(int dup_fd);
+int monitor_fdset_dup_fd_find(int dup_fd);
+
+#endif /* !MONITOR_H */
diff --git a/contrib/qemu/include/monitor/readline.h b/contrib/qemu/include/monitor/readline.h
new file mode 100644
index 000000000..fc9806ecf
--- /dev/null
+++ b/contrib/qemu/include/monitor/readline.h
@@ -0,0 +1,55 @@
+#ifndef READLINE_H
+#define READLINE_H
+
+#include "qemu-common.h"
+
+#define READLINE_CMD_BUF_SIZE 4095
+#define READLINE_MAX_CMDS 64
+#define READLINE_MAX_COMPLETIONS 256
+
+typedef void ReadLineFunc(Monitor *mon, const char *str, void *opaque);
+typedef void ReadLineCompletionFunc(const char *cmdline);
+
+typedef struct ReadLineState {
+    char cmd_buf[READLINE_CMD_BUF_SIZE + 1];
+    int cmd_buf_index;
+    int cmd_buf_size;
+
+    char last_cmd_buf[READLINE_CMD_BUF_SIZE + 1];
+    int last_cmd_buf_index;
+    int last_cmd_buf_size;
+
+    int esc_state;
+    int esc_param;
+
+    char *history[READLINE_MAX_CMDS];
+    int hist_entry;
+
+    ReadLineCompletionFunc *completion_finder;
+    char *completions[READLINE_MAX_COMPLETIONS];
+    int nb_completions;
+    int completion_index;
+
+    ReadLineFunc *readline_func;
+    void *readline_opaque;
+    int read_password;
+    char prompt[256];
+    Monitor *mon;
+} ReadLineState;
+
+void readline_add_completion(ReadLineState *rs, const char *str);
+void readline_set_completion_index(ReadLineState *rs, int completion_index);
+
+const char *readline_get_history(ReadLineState *rs, unsigned int index);
+
+void readline_handle_byte(ReadLineState *rs, int ch);
+
+void readline_start(ReadLineState *rs, const char *prompt, int read_password,
+                    ReadLineFunc *readline_func, void *opaque);
+void readline_restart(ReadLineState *rs);
+void readline_show_prompt(ReadLineState *rs);
+
+ReadLineState *readline_init(Monitor *mon,
+                             ReadLineCompletionFunc *completion_finder);
+
+#endif /* !READLINE_H */
diff --git a/contrib/qemu/include/qapi/error.h b/contrib/qemu/include/qapi/error.h
new file mode 100644
index 000000000..ffd1cea47
--- /dev/null
+++ b/contrib/qemu/include/qapi/error.h
@@ -0,0 +1,85 @@
+/*
+ * QEMU Error Objects
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.  See
+ * the COPYING.LIB file in the top-level directory.
+ */
+#ifndef ERROR_H
+#define ERROR_H
+
+#include "qemu/compiler.h"
+#include "qapi-types.h"
+#include <stdbool.h>
+
+/**
+ * A class representing internal errors within QEMU.  An error has a ErrorClass
+ * code and a human message.
+ */
+typedef struct Error Error;
+
+/**
+ * Set an indirect pointer to an error given a ErrorClass value and a
+ * printf-style human message.  This function is not meant to be used outside
+ * of QEMU.
+ */
+void error_set(Error **err, ErrorClass err_class, const char *fmt, ...) GCC_FMT_ATTR(3, 4);
+
+/**
+ * Set an indirect pointer to an error given a ErrorClass value and a
+ * printf-style human message, followed by a strerror() string if
+ * @os_error is not zero.
+ */
+void error_set_errno(Error **err, int os_error, ErrorClass err_class, const char *fmt, ...) GCC_FMT_ATTR(4, 5);
+
+/**
+ * Same as error_set(), but sets a generic error
+ */
+#define error_setg(err, fmt, ...) \
+    error_set(err, ERROR_CLASS_GENERIC_ERROR, fmt, ## __VA_ARGS__)
+#define error_setg_errno(err, os_error, fmt, ...) \
+    error_set_errno(err, os_error, ERROR_CLASS_GENERIC_ERROR, fmt, ## __VA_ARGS__)
+
+/**
+ * Helper for open() errors
+ */
+void error_setg_file_open(Error **errp, int os_errno, const char *filename);
+
+/**
+ * Returns true if an indirect pointer to an error is pointing to a valid
+ * error object.
+ */
+bool error_is_set(Error **err);
+
+/*
+ * Get the error class of an error object.
+ */
+ErrorClass error_get_class(const Error *err);
+
+/**
+ * Returns an exact copy of the error passed as an argument.
+ */
+Error *error_copy(const Error *err);
+
+/**
+ * Get a human readable representation of an error object.
+ */
+const char *error_get_pretty(Error *err);
+
+/**
+ * Propagate an error to an indirect pointer to an error.  This function will
+ * always transfer ownership of the error reference and handles the case where
+ * dst_err is NULL correctly.  Errors after the first are discarded.
+ */
+void error_propagate(Error **dst_err, Error *local_err);
+
+/**
+ * Free an error object.
+ */
+void error_free(Error *err);
+
+#endif
diff --git a/contrib/qemu/include/qapi/qmp/json-lexer.h b/contrib/qemu/include/qapi/qmp/json-lexer.h
new file mode 100644
index 000000000..cdff0460a
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/json-lexer.h
@@ -0,0 +1,51 @@
+/*
+ * JSON lexer
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_JSON_LEXER_H
+#define QEMU_JSON_LEXER_H
+
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qlist.h"
+
+typedef enum json_token_type {
+    JSON_OPERATOR = 100,
+    JSON_INTEGER,
+    JSON_FLOAT,
+    JSON_KEYWORD,
+    JSON_STRING,
+    JSON_ESCAPE,
+    JSON_SKIP,
+    JSON_ERROR,
+} JSONTokenType;
+
+typedef struct JSONLexer JSONLexer;
+
+typedef void (JSONLexerEmitter)(JSONLexer *, QString *, JSONTokenType, int x, int y);
+
+struct JSONLexer
+{
+    JSONLexerEmitter *emit;
+    int state;
+    QString *token;
+    int x, y;
+};
+
+void json_lexer_init(JSONLexer *lexer, JSONLexerEmitter func);
+
+int json_lexer_feed(JSONLexer *lexer, const char *buffer, size_t size);
+
+int json_lexer_flush(JSONLexer *lexer);
+
+void json_lexer_destroy(JSONLexer *lexer);
+
+#endif
diff --git a/contrib/qemu/include/qapi/qmp/json-parser.h b/contrib/qemu/include/qapi/qmp/json-parser.h
new file mode 100644
index 000000000..44d88f346
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/json-parser.h
@@ -0,0 +1,24 @@
+/*
+ * JSON Parser 
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_JSON_PARSER_H
+#define QEMU_JSON_PARSER_H
+
+#include "qemu-common.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/error.h"
+
+QObject *json_parser_parse(QList *tokens, va_list *ap);
+QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp);
+
+#endif
diff --git a/contrib/qemu/include/qapi/qmp/json-streamer.h b/contrib/qemu/include/qapi/qmp/json-streamer.h
new file mode 100644
index 000000000..823f7d7fa
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/json-streamer.h
@@ -0,0 +1,40 @@
+/*
+ * JSON streaming support
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_JSON_STREAMER_H
+#define QEMU_JSON_STREAMER_H
+
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/json-lexer.h"
+
+typedef struct JSONMessageParser
+{
+    void (*emit)(struct JSONMessageParser *parser, QList *tokens);
+    JSONLexer lexer;
+    int brace_count;
+    int bracket_count;
+    QList *tokens;
+    uint64_t token_size;
+} JSONMessageParser;
+
+void json_message_parser_init(JSONMessageParser *parser,
+                              void (*func)(JSONMessageParser *, QList *));
+
+int json_message_parser_feed(JSONMessageParser *parser,
+                             const char *buffer, size_t size);
+
+int json_message_parser_flush(JSONMessageParser *parser);
+
+void json_message_parser_destroy(JSONMessageParser *parser);
+
+#endif
diff --git a/contrib/qemu/include/qapi/qmp/qbool.h b/contrib/qemu/include/qapi/qmp/qbool.h
new file mode 100644
index 000000000..c4eaab9bb
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/qbool.h
@@ -0,0 +1,29 @@
+/*
+ * QBool Module
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QBOOL_H
+#define QBOOL_H
+
+#include <stdint.h>
+#include "qapi/qmp/qobject.h"
+
+typedef struct QBool {
+    QObject_HEAD;
+    int value;
+} QBool;
+
+QBool *qbool_from_int(int value);
+int qbool_get_int(const QBool *qb);
+QBool *qobject_to_qbool(const QObject *obj);
+
+#endif /* QBOOL_H */
diff --git a/contrib/qemu/include/qapi/qmp/qdict.h b/contrib/qemu/include/qapi/qmp/qdict.h
new file mode 100644
index 000000000..685b2e3fc
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/qdict.h
@@ -0,0 +1,69 @@
+/*
+ * QDict Module
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#ifndef QDICT_H
+#define QDICT_H
+
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qlist.h"
+#include "qemu/queue.h"
+#include <stdint.h>
+
+#define QDICT_BUCKET_MAX 512
+
+typedef struct QDictEntry {
+    char *key;
+    QObject *value;
+    QLIST_ENTRY(QDictEntry) next;
+} QDictEntry;
+
+typedef struct QDict {
+    QObject_HEAD;
+    size_t size;
+    QLIST_HEAD(,QDictEntry) table[QDICT_BUCKET_MAX];
+} QDict;
+
+/* Object API */
+QDict *qdict_new(void);
+const char *qdict_entry_key(const QDictEntry *entry);
+QObject *qdict_entry_value(const QDictEntry *entry);
+size_t qdict_size(const QDict *qdict);
+void qdict_put_obj(QDict *qdict, const char *key, QObject *value);
+void qdict_del(QDict *qdict, const char *key);
+int qdict_haskey(const QDict *qdict, const char *key);
+QObject *qdict_get(const QDict *qdict, const char *key);
+QDict *qobject_to_qdict(const QObject *obj);
+void qdict_iter(const QDict *qdict,
+                void (*iter)(const char *key, QObject *obj, void *opaque),
+                void *opaque);
+const QDictEntry *qdict_first(const QDict *qdict);
+const QDictEntry *qdict_next(const QDict *qdict, const QDictEntry *entry);
+
+/* Helper to qdict_put_obj(), accepts any object */
+#define qdict_put(qdict, key, obj) \
+        qdict_put_obj(qdict, key, QOBJECT(obj))
+
+/* High level helpers */
+double qdict_get_double(const QDict *qdict, const char *key);
+int64_t qdict_get_int(const QDict *qdict, const char *key);
+int qdict_get_bool(const QDict *qdict, const char *key);
+QList *qdict_get_qlist(const QDict *qdict, const char *key);
+QDict *qdict_get_qdict(const QDict *qdict, const char *key);
+const char *qdict_get_str(const QDict *qdict, const char *key);
+int64_t qdict_get_try_int(const QDict *qdict, const char *key,
+                          int64_t def_value);
+int qdict_get_try_bool(const QDict *qdict, const char *key, int def_value);
+const char *qdict_get_try_str(const QDict *qdict, const char *key);
+
+QDict *qdict_clone_shallow(const QDict *src);
+
+#endif /* QDICT_H */
diff --git a/contrib/qemu/include/qapi/qmp/qerror.h b/contrib/qemu/include/qapi/qmp/qerror.h
new file mode 100644
index 000000000..c30c2f6d7
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/qerror.h
@@ -0,0 +1,249 @@
+/*
+ * QError Module
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+#ifndef QERROR_H
+#define QERROR_H
+
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qstring.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
+#include "qapi-types.h"
+#include <stdarg.h>
+
+typedef struct QError {
+    QObject_HEAD;
+    Location loc;
+    char *err_msg;
+    ErrorClass err_class;
+} QError;
+
+QString *qerror_human(const QError *qerror);
+void qerror_report(ErrorClass err_class, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
+void qerror_report_err(Error *err);
+void assert_no_error(Error *err);
+
+/*
+ * QError class list
+ * Please keep the definitions in alphabetical order.
+ * Use scripts/check-qerror.sh to check.
+ */
+#define QERR_ADD_CLIENT_FAILED \
+    ERROR_CLASS_GENERIC_ERROR, "Could not add client"
+
+#define QERR_AMBIGUOUS_PATH \
+    ERROR_CLASS_GENERIC_ERROR, "Path '%s' does not uniquely identify an object"
+
+#define QERR_BAD_BUS_FOR_DEVICE \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' can't go on a %s bus"
+
+#define QERR_BASE_NOT_FOUND \
+    ERROR_CLASS_GENERIC_ERROR, "Base '%s' not found"
+
+#define QERR_BLOCK_JOB_NOT_ACTIVE \
+    ERROR_CLASS_DEVICE_NOT_ACTIVE, "No active block job on device '%s'"
+
+#define QERR_BLOCK_JOB_PAUSED \
+    ERROR_CLASS_GENERIC_ERROR, "The block job for device '%s' is currently paused"
+
+#define QERR_BLOCK_JOB_NOT_READY \
+    ERROR_CLASS_GENERIC_ERROR, "The active block job for device '%s' cannot be completed"
+
+#define QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED \
+    ERROR_CLASS_GENERIC_ERROR, "Block format '%s' used by device '%s' does not support feature '%s'"
+
+#define QERR_BUFFER_OVERRUN \
+    ERROR_CLASS_GENERIC_ERROR, "An internal buffer overran"
+
+#define QERR_BUS_NO_HOTPLUG \
+    ERROR_CLASS_GENERIC_ERROR, "Bus '%s' does not support hotplugging"
+
+#define QERR_BUS_NOT_FOUND \
+    ERROR_CLASS_GENERIC_ERROR, "Bus '%s' not found"
+
+#define QERR_COMMAND_DISABLED \
+    ERROR_CLASS_GENERIC_ERROR, "The command %s has been disabled for this instance"
+
+#define QERR_COMMAND_NOT_FOUND \
+    ERROR_CLASS_COMMAND_NOT_FOUND, "The command %s has not been found"
+
+#define QERR_DEVICE_ENCRYPTED \
+    ERROR_CLASS_DEVICE_ENCRYPTED, "'%s' (%s) is encrypted"
+
+#define QERR_DEVICE_FEATURE_BLOCKS_MIGRATION \
+    ERROR_CLASS_GENERIC_ERROR, "Migration is disabled when using feature '%s' in device '%s'"
+
+#define QERR_DEVICE_HAS_NO_MEDIUM \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' has no medium"
+
+#define QERR_DEVICE_INIT_FAILED \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' could not be initialized"
+
+#define QERR_DEVICE_IN_USE \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' is in use"
+
+#define QERR_DEVICE_IS_READ_ONLY \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' is read only"
+
+#define QERR_DEVICE_LOCKED \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' is locked"
+
+#define QERR_DEVICE_MULTIPLE_BUSSES \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' has multiple child busses"
+
+#define QERR_DEVICE_NO_BUS \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' has no child bus"
+
+#define QERR_DEVICE_NO_HOTPLUG \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' does not support hotplugging"
+
+#define QERR_DEVICE_NOT_ACTIVE \
+    ERROR_CLASS_DEVICE_NOT_ACTIVE, "Device '%s' has not been activated"
+
+#define QERR_DEVICE_NOT_ENCRYPTED \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' is not encrypted"
+
+#define QERR_DEVICE_NOT_FOUND \
+    ERROR_CLASS_DEVICE_NOT_FOUND, "Device '%s' not found"
+
+#define QERR_DEVICE_NOT_REMOVABLE \
+    ERROR_CLASS_GENERIC_ERROR, "Device '%s' is not removable"
+
+#define QERR_DUPLICATE_ID \
+    ERROR_CLASS_GENERIC_ERROR, "Duplicate ID '%s' for %s"
+
+#define QERR_FD_NOT_FOUND \
+    ERROR_CLASS_GENERIC_ERROR, "File descriptor named '%s' not found"
+
+#define QERR_FD_NOT_SUPPLIED \
+    ERROR_CLASS_GENERIC_ERROR, "No file descriptor supplied via SCM_RIGHTS"
+
+#define QERR_FEATURE_DISABLED \
+    ERROR_CLASS_GENERIC_ERROR, "The feature '%s' is not enabled"
+
+#define QERR_INVALID_BLOCK_FORMAT \
+    ERROR_CLASS_GENERIC_ERROR, "Invalid block format '%s'"
+
+#define QERR_INVALID_OPTION_GROUP \
+    ERROR_CLASS_GENERIC_ERROR, "There is no option group '%s'"
+
+#define QERR_INVALID_PARAMETER \
+    ERROR_CLASS_GENERIC_ERROR, "Invalid parameter '%s'"
+
+#define QERR_INVALID_PARAMETER_COMBINATION \
+    ERROR_CLASS_GENERIC_ERROR, "Invalid parameter combination"
+
+#define QERR_INVALID_PARAMETER_TYPE \
+    ERROR_CLASS_GENERIC_ERROR, "Invalid parameter type for '%s', expected: %s"
+
+#define QERR_INVALID_PARAMETER_VALUE \
+    ERROR_CLASS_GENERIC_ERROR, "Parameter '%s' expects %s"
+
+#define QERR_INVALID_PASSWORD \
+    ERROR_CLASS_GENERIC_ERROR, "Password incorrect"
+
+#define QERR_IO_ERROR \
+    ERROR_CLASS_GENERIC_ERROR, "An IO error has occurred"
+
+#define QERR_JSON_PARSE_ERROR \
+    ERROR_CLASS_GENERIC_ERROR, "JSON parse error, %s"
+
+#define QERR_JSON_PARSING \
+    ERROR_CLASS_GENERIC_ERROR, "Invalid JSON syntax"
+
+#define QERR_KVM_MISSING_CAP \
+    ERROR_CLASS_K_V_M_MISSING_CAP, "Using KVM without %s, %s unavailable"
+
+#define QERR_MIGRATION_ACTIVE \
+    ERROR_CLASS_GENERIC_ERROR, "There's a migration process in progress"
+
+#define QERR_MIGRATION_NOT_SUPPORTED \
+    ERROR_CLASS_GENERIC_ERROR, "State blocked by non-migratable device '%s'"
+
+#define QERR_MISSING_PARAMETER \
+    ERROR_CLASS_GENERIC_ERROR, "Parameter '%s' is missing"
+
+#define QERR_NO_BUS_FOR_DEVICE \
+    ERROR_CLASS_GENERIC_ERROR, "No '%s' bus found for device '%s'"
+
+#define QERR_NOT_SUPPORTED \
+    ERROR_CLASS_GENERIC_ERROR, "Not supported"
+
+#define QERR_PERMISSION_DENIED \
+    ERROR_CLASS_GENERIC_ERROR, "Insufficient permission to perform this operation"
+
+#define QERR_PROPERTY_NOT_FOUND \
+    ERROR_CLASS_GENERIC_ERROR, "Property '%s.%s' not found"
+
+#define QERR_PROPERTY_VALUE_BAD \
+    ERROR_CLASS_GENERIC_ERROR, "Property '%s.%s' doesn't take value '%s'"
+
+#define QERR_PROPERTY_VALUE_IN_USE \
+    ERROR_CLASS_GENERIC_ERROR, "Property '%s.%s' can't take value '%s', it's in use"
+
+#define QERR_PROPERTY_VALUE_NOT_FOUND \
+    ERROR_CLASS_GENERIC_ERROR, "Property '%s.%s' can't find value '%s'"
+
+#define QERR_PROPERTY_VALUE_NOT_POWER_OF_2 \
+    ERROR_CLASS_GENERIC_ERROR, "Property %s.%s doesn't take value '%" PRId64 "', it's not a power of 2"
+
+#define QERR_PROPERTY_VALUE_OUT_OF_RANGE \
+    ERROR_CLASS_GENERIC_ERROR, "Property %s.%s doesn't take value %" PRId64 " (minimum: %" PRId64 ", maximum: %" PRId64 ")"
+
+#define QERR_QGA_COMMAND_FAILED \
+    ERROR_CLASS_GENERIC_ERROR, "Guest agent command failed, error was '%s'"
+
+#define QERR_QGA_LOGGING_FAILED \
+    ERROR_CLASS_GENERIC_ERROR, "Guest agent failed to log non-optional log statement"
+
+#define QERR_QMP_BAD_INPUT_OBJECT \
+    ERROR_CLASS_GENERIC_ERROR, "Expected '%s' in QMP input"
+
+#define QERR_QMP_BAD_INPUT_OBJECT_MEMBER \
+    ERROR_CLASS_GENERIC_ERROR, "QMP input object member '%s' expects '%s'"
+
+#define QERR_QMP_EXTRA_MEMBER \
+    ERROR_CLASS_GENERIC_ERROR, "QMP input object member '%s' is unexpected"
+
+#define QERR_RESET_REQUIRED \
+    ERROR_CLASS_GENERIC_ERROR, "Resetting the Virtual Machine is required"
+
+#define QERR_SET_PASSWD_FAILED \
+    ERROR_CLASS_GENERIC_ERROR, "Could not set password"
+
+#define QERR_TOO_MANY_FILES \
+    ERROR_CLASS_GENERIC_ERROR, "Too many open files"
+
+#define QERR_UNDEFINED_ERROR \
+    ERROR_CLASS_GENERIC_ERROR, "An undefined error has occurred"
+
+#define QERR_UNKNOWN_BLOCK_FORMAT_FEATURE \
+    ERROR_CLASS_GENERIC_ERROR, "'%s' uses a %s feature which is not supported by this qemu version: %s"
+
+#define QERR_UNSUPPORTED \
+    ERROR_CLASS_GENERIC_ERROR, "this feature or command is not currently supported"
+
+#define QERR_VIRTFS_FEATURE_BLOCKS_MIGRATION \
+    ERROR_CLASS_GENERIC_ERROR, "Migration is disabled when VirtFS export path '%s' is mounted in the guest using mount_tag '%s'"
+
+#define QERR_SOCKET_CONNECT_FAILED \
+    ERROR_CLASS_GENERIC_ERROR, "Failed to connect to socket"
+
+#define QERR_SOCKET_LISTEN_FAILED \
+    ERROR_CLASS_GENERIC_ERROR, "Failed to set socket to listening mode"
+
+#define QERR_SOCKET_BIND_FAILED \
+    ERROR_CLASS_GENERIC_ERROR, "Failed to bind socket"
+
+#define QERR_SOCKET_CREATE_FAILED \
+    ERROR_CLASS_GENERIC_ERROR, "Failed to create socket"
+
+#endif /* QERROR_H */
diff --git a/contrib/qemu/include/qapi/qmp/qfloat.h b/contrib/qemu/include/qapi/qmp/qfloat.h
new file mode 100644
index 000000000..a8658443d
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/qfloat.h
@@ -0,0 +1,29 @@
+/*
+ * QFloat Module
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QFLOAT_H
+#define QFLOAT_H
+
+#include <stdint.h>
+#include "qapi/qmp/qobject.h"
+
+typedef struct QFloat {
+    QObject_HEAD;
+    double value;
+} QFloat;
+
+QFloat *qfloat_from_double(double value);
+double qfloat_get_double(const QFloat *qi);
+QFloat *qobject_to_qfloat(const QObject *obj);
+
+#endif /* QFLOAT_H */
diff --git a/contrib/qemu/include/qapi/qmp/qint.h b/contrib/qemu/include/qapi/qmp/qint.h
new file mode 100644
index 000000000..48a41b0f2
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/qint.h
@@ -0,0 +1,28 @@
+/*
+ * QInt Module
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#ifndef QINT_H
+#define QINT_H
+
+#include <stdint.h>
+#include "qapi/qmp/qobject.h"
+
+typedef struct QInt {
+    QObject_HEAD;
+    int64_t value;
+} QInt;
+
+QInt *qint_from_int(int64_t value);
+int64_t qint_get_int(const QInt *qi);
+QInt *qobject_to_qint(const QObject *obj);
+
+#endif /* QINT_H */
diff --git a/contrib/qemu/include/qapi/qmp/qjson.h b/contrib/qemu/include/qapi/qmp/qjson.h
new file mode 100644
index 000000000..73351ed6d
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/qjson.h
@@ -0,0 +1,29 @@
+/*
+ * QObject JSON integration
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QJSON_H
+#define QJSON_H
+
+#include <stdarg.h>
+#include "qemu/compiler.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qstring.h"
+
+QObject *qobject_from_json(const char *string) GCC_FMT_ATTR(1, 0);
+QObject *qobject_from_jsonf(const char *string, ...) GCC_FMT_ATTR(1, 2);
+QObject *qobject_from_jsonv(const char *string, va_list *ap) GCC_FMT_ATTR(1, 0);
+
+QString *qobject_to_json(const QObject *obj);
+QString *qobject_to_json_pretty(const QObject *obj);
+
+#endif /* QJSON_H */
diff --git a/contrib/qemu/include/qapi/qmp/qlist.h b/contrib/qemu/include/qapi/qmp/qlist.h
new file mode 100644
index 000000000..6cc4831df
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/qlist.h
@@ -0,0 +1,63 @@
+/*
+ * QList Module
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#ifndef QLIST_H
+#define QLIST_H
+
+#include "qapi/qmp/qobject.h"
+#include "qemu/queue.h"
+
+typedef struct QListEntry {
+    QObject *value;
+    QTAILQ_ENTRY(QListEntry) next;
+} QListEntry;
+
+typedef struct QList {
+    QObject_HEAD;
+    QTAILQ_HEAD(,QListEntry) head;
+} QList;
+
+#define qlist_append(qlist, obj) \
+        qlist_append_obj(qlist, QOBJECT(obj))
+
+#define QLIST_FOREACH_ENTRY(qlist, var)             \
+        for ((var) = ((qlist)->head.tqh_first);     \
+            (var);                                  \
+            (var) = ((var)->next.tqe_next))
+
+static inline QObject *qlist_entry_obj(const QListEntry *entry)
+{
+    return entry->value;
+}
+
+QList *qlist_new(void);
+QList *qlist_copy(QList *src);
+void qlist_append_obj(QList *qlist, QObject *obj);
+void qlist_iter(const QList *qlist,
+                void (*iter)(QObject *obj, void *opaque), void *opaque);
+QObject *qlist_pop(QList *qlist);
+QObject *qlist_peek(QList *qlist);
+int qlist_empty(const QList *qlist);
+size_t qlist_size(const QList *qlist);
+QList *qobject_to_qlist(const QObject *obj);
+
+static inline const QListEntry *qlist_first(const QList *qlist)
+{
+    return QTAILQ_FIRST(&qlist->head);
+}
+
+static inline const QListEntry *qlist_next(const QListEntry *entry)
+{
+    return QTAILQ_NEXT(entry, next);
+}
+
+#endif /* QLIST_H */
diff --git a/contrib/qemu/include/qapi/qmp/qobject.h b/contrib/qemu/include/qapi/qmp/qobject.h
new file mode 100644
index 000000000..9124649ed
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/qobject.h
@@ -0,0 +1,112 @@
+/*
+ * QEMU Object Model.
+ *
+ * Based on ideas by Avi Kivity <avi@redhat.com>
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ * QObject Reference Counts Terminology
+ * ------------------------------------
+ *
+ *  - Returning references: A function that returns an object may
+ *  return it as either a weak or a strong reference.  If the reference
+ *  is strong, you are responsible for calling QDECREF() on the reference
+ *  when you are done.
+ *
+ *  If the reference is weak, the owner of the reference may free it at
+ *  any time in the future.  Before storing the reference anywhere, you
+ *  should call QINCREF() to make the reference strong.
+ *
+ *  - Transferring ownership: when you transfer ownership of a reference
+ *  by calling a function, you are no longer responsible for calling
+ *  QDECREF() when the reference is no longer needed.  In other words,
+ *  when the function returns you must behave as if the reference to the
+ *  passed object was weak.
+ */
+#ifndef QOBJECT_H
+#define QOBJECT_H
+
+#include <stddef.h>
+#include <assert.h>
+
+typedef enum {
+    QTYPE_NONE,
+    QTYPE_QINT,
+    QTYPE_QSTRING,
+    QTYPE_QDICT,
+    QTYPE_QLIST,
+    QTYPE_QFLOAT,
+    QTYPE_QBOOL,
+    QTYPE_QERROR,
+} qtype_code;
+
+struct QObject;
+
+typedef struct QType {
+    qtype_code code;
+    void (*destroy)(struct QObject *);
+} QType;
+
+typedef struct QObject {
+    const QType *type;
+    size_t refcnt;
+} QObject;
+
+/* Objects definitions must include this */
+#define QObject_HEAD  \
+    QObject base
+
+/* Get the 'base' part of an object */
+#define QOBJECT(obj) (&(obj)->base)
+
+/* High-level interface for qobject_incref() */
+#define QINCREF(obj)      \
+    qobject_incref(QOBJECT(obj))
+
+/* High-level interface for qobject_decref() */
+#define QDECREF(obj)              \
+    qobject_decref(obj ? QOBJECT(obj) : NULL)
+
+/* Initialize an object to default values */
+#define QOBJECT_INIT(obj, qtype_type)   \
+    obj->base.refcnt = 1;               \
+    obj->base.type   = qtype_type
+
+/**
+ * qobject_incref(): Increment QObject's reference count
+ */
+static inline void qobject_incref(QObject *obj)
+{
+    if (obj)
+        obj->refcnt++;
+}
+
+/**
+ * qobject_decref(): Decrement QObject's reference count, deallocate
+ * when it reaches zero
+ */
+static inline void qobject_decref(QObject *obj)
+{
+    if (obj && --obj->refcnt == 0) {
+        assert(obj->type != NULL);
+        assert(obj->type->destroy != NULL);
+        obj->type->destroy(obj);
+    }
+}
+
+/**
+ * qobject_type(): Return the QObject's type
+ */
+static inline qtype_code qobject_type(const QObject *obj)
+{
+    assert(obj->type != NULL);
+    return obj->type->code;
+}
+
+#endif /* QOBJECT_H */
diff --git a/contrib/qemu/include/qapi/qmp/qstring.h b/contrib/qemu/include/qapi/qmp/qstring.h
new file mode 100644
index 000000000..1bc366610
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/qstring.h
@@ -0,0 +1,36 @@
+/*
+ * QString Module
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#ifndef QSTRING_H
+#define QSTRING_H
+
+#include <stdint.h>
+#include "qapi/qmp/qobject.h"
+
+typedef struct QString {
+    QObject_HEAD;
+    char *string;
+    size_t length;
+    size_t capacity;
+} QString;
+
+QString *qstring_new(void);
+QString *qstring_from_str(const char *str);
+QString *qstring_from_substr(const char *str, int start, int end);
+size_t qstring_get_length(const QString *qstring);
+const char *qstring_get_str(const QString *qstring);
+void qstring_append_int(QString *qstring, int64_t value);
+void qstring_append(QString *qstring, const char *str);
+void qstring_append_chr(QString *qstring, int c);
+QString *qobject_to_qstring(const QObject *obj);
+
+#endif /* QSTRING_H */
diff --git a/contrib/qemu/include/qapi/qmp/types.h b/contrib/qemu/include/qapi/qmp/types.h
new file mode 100644
index 000000000..7782ec5a6
--- /dev/null
+++ b/contrib/qemu/include/qapi/qmp/types.h
@@ -0,0 +1,25 @@
+/*
+ * Include all QEMU objects.
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#ifndef QEMU_OBJECTS_H
+#define QEMU_OBJECTS_H
+
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qjson.h"
+
+#endif /* QEMU_OBJECTS_H */
diff --git a/contrib/qemu/include/qemu-common.h b/contrib/qemu/include/qemu-common.h
new file mode 100644
index 000000000..6948bb917
--- /dev/null
+++ b/contrib/qemu/include/qemu-common.h
@@ -0,0 +1,478 @@
+
+/* Common header file that is included by all of QEMU.
+ *
+ * This file is supposed to be included only by .c files. No header file should
+ * depend on qemu-common.h, as this would easily lead to circular header
+ * dependencies.
+ *
+ * If a header file uses a definition from qemu-common.h, that definition
+ * must be moved to a separate header file, and the header that uses it
+ * must include that header.
+ */
+#ifndef QEMU_COMMON_H
+#define QEMU_COMMON_H
+
+#include "qemu/compiler.h"
+#include "config-host.h"
+#include "qemu/typedefs.h"
+
+#if defined(__arm__) || defined(__sparc__) || defined(__mips__) || defined(__hppa__) || defined(__ia64__)
+#define WORDS_ALIGNED
+#endif
+
+#define TFR(expr) do { if ((expr) != -1) break; } while (errno == EINTR)
+
+/* we put basic includes here to avoid repeating them in device drivers */
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <string.h>
+#include <strings.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <time.h>
+#include <ctype.h>
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <assert.h>
+#include <signal.h>
+#include "glib-compat.h"
+
+#ifdef _WIN32
+#include "sysemu/os-win32.h"
+#endif
+
+#ifdef CONFIG_POSIX
+#include "sysemu/os-posix.h"
+#endif
+
+#ifndef O_LARGEFILE
+#define O_LARGEFILE 0
+#endif
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+#ifndef ENOMEDIUM
+#define ENOMEDIUM ENODEV
+#endif
+#if !defined(ENOTSUP)
+#define ENOTSUP 4096
+#endif
+#if !defined(ECANCELED)
+#define ECANCELED 4097
+#endif
+#if !defined(EMEDIUMTYPE)
+#define EMEDIUMTYPE 4098
+#endif
+#ifndef TIME_MAX
+#define TIME_MAX LONG_MAX
+#endif
+
+/* HOST_LONG_BITS is the size of a native pointer in bits. */
+#if UINTPTR_MAX == UINT32_MAX
+# define HOST_LONG_BITS 32
+#elif UINTPTR_MAX == UINT64_MAX
+# define HOST_LONG_BITS 64
+#else
+# error Unknown pointer size
+#endif
+
+typedef int (*fprintf_function)(FILE *f, const char *fmt, ...)
+    GCC_FMT_ATTR(2, 3);
+
+#ifdef _WIN32
+#define fsync _commit
+#if !defined(lseek)
+# define lseek _lseeki64
+#endif
+int qemu_ftruncate64(int, int64_t);
+#if !defined(ftruncate)
+# define ftruncate qemu_ftruncate64
+#endif
+
+static inline char *realpath(const char *path, char *resolved_path)
+{
+    _fullpath(resolved_path, path, _MAX_PATH);
+    return resolved_path;
+}
+#endif
+
+/* icount */
+void configure_icount(const char *option);
+extern int use_icount;
+
+#include "qemu/osdep.h"
+#include "qemu/bswap.h"
+
+/* FIXME: Remove NEED_CPU_H.  */
+#ifdef NEED_CPU_H
+#include "cpu.h"
+#endif /* !defined(NEED_CPU_H) */
+
+/* main function, renamed */
+#if defined(CONFIG_COCOA)
+int qemu_main(int argc, char **argv, char **envp);
+#endif
+
+void qemu_get_timedate(struct tm *tm, int offset);
+int qemu_timedate_diff(struct tm *tm);
+
+#if !GLIB_CHECK_VERSION(2, 20, 0)
+/*
+ * Glib before 2.20.0 doesn't implement g_poll, so wrap it to compile properly
+ * on older systems.
+ */
+static inline gint g_poll(GPollFD *fds, guint nfds, gint timeout)
+{
+    GMainContext *ctx = g_main_context_default();
+    return g_main_context_get_poll_func(ctx)(fds, nfds, timeout);
+}
+#endif
+
+/**
+ * is_help_option:
+ * @s: string to test
+ *
+ * Check whether @s is one of the standard strings which indicate
+ * that the user is asking for a list of the valid values for a
+ * command option like -cpu or -M. The current accepted strings
+ * are 'help' and '?'. '?' is deprecated (it is a shell wildcard
+ * which makes it annoying to use in a reliable way) but provided
+ * for backwards compatibility.
+ *
+ * Returns: true if @s is a request for a list.
+ */
+static inline bool is_help_option(const char *s)
+{
+    return !strcmp(s, "?") || !strcmp(s, "help");
+}
+
+/* cutils.c */
+void pstrcpy(char *buf, int buf_size, const char *str);
+void strpadcpy(char *buf, int buf_size, const char *str, char pad);
+char *pstrcat(char *buf, int buf_size, const char *s);
+int strstart(const char *str, const char *val, const char **ptr);
+int stristart(const char *str, const char *val, const char **ptr);
+int qemu_strnlen(const char *s, int max_len);
+char *qemu_strsep(char **input, const char *delim);
+time_t mktimegm(struct tm *tm);
+int qemu_fls(int i);
+int qemu_fdatasync(int fd);
+int fcntl_setfl(int fd, int flag);
+int qemu_parse_fd(const char *param);
+
+int parse_uint(const char *s, unsigned long long *value, char **endptr,
+               int base);
+int parse_uint_full(const char *s, unsigned long long *value, int base);
+
+/*
+ * strtosz() suffixes used to specify the default treatment of an
+ * argument passed to strtosz() without an explicit suffix.
+ * These should be defined using upper case characters in the range
+ * A-Z, as strtosz() will use qemu_toupper() on the given argument
+ * prior to comparison.
+ */
+#define STRTOSZ_DEFSUFFIX_EB	'E'
+#define STRTOSZ_DEFSUFFIX_PB	'P'
+#define STRTOSZ_DEFSUFFIX_TB	'T'
+#define STRTOSZ_DEFSUFFIX_GB	'G'
+#define STRTOSZ_DEFSUFFIX_MB	'M'
+#define STRTOSZ_DEFSUFFIX_KB	'K'
+#define STRTOSZ_DEFSUFFIX_B	'B'
+int64_t strtosz(const char *nptr, char **end);
+int64_t strtosz_suffix(const char *nptr, char **end, const char default_suffix);
+int64_t strtosz_suffix_unit(const char *nptr, char **end,
+                            const char default_suffix, int64_t unit);
+
+/* path.c */
+void init_paths(const char *prefix);
+const char *path(const char *pathname);
+
+#define qemu_isalnum(c)		isalnum((unsigned char)(c))
+#define qemu_isalpha(c)		isalpha((unsigned char)(c))
+#define qemu_iscntrl(c)		iscntrl((unsigned char)(c))
+#define qemu_isdigit(c)		isdigit((unsigned char)(c))
+#define qemu_isgraph(c)		isgraph((unsigned char)(c))
+#define qemu_islower(c)		islower((unsigned char)(c))
+#define qemu_isprint(c)		isprint((unsigned char)(c))
+#define qemu_ispunct(c)		ispunct((unsigned char)(c))
+#define qemu_isspace(c)		isspace((unsigned char)(c))
+#define qemu_isupper(c)		isupper((unsigned char)(c))
+#define qemu_isxdigit(c)	isxdigit((unsigned char)(c))
+#define qemu_tolower(c)		tolower((unsigned char)(c))
+#define qemu_toupper(c)		toupper((unsigned char)(c))
+#define qemu_isascii(c)		isascii((unsigned char)(c))
+#define qemu_toascii(c)		toascii((unsigned char)(c))
+
+void *qemu_oom_check(void *ptr);
+
+ssize_t qemu_write_full(int fd, const void *buf, size_t count)
+    QEMU_WARN_UNUSED_RESULT;
+ssize_t qemu_send_full(int fd, const void *buf, size_t count, int flags)
+    QEMU_WARN_UNUSED_RESULT;
+ssize_t qemu_recv_full(int fd, void *buf, size_t count, int flags)
+    QEMU_WARN_UNUSED_RESULT;
+
+#ifndef _WIN32
+int qemu_pipe(int pipefd[2]);
+/* like openpty() but also makes it raw; return master fd */
+int qemu_openpty_raw(int *aslave, char *pty_name);
+#endif
+
+#ifdef _WIN32
+/* MinGW needs type casts for the 'buf' and 'optval' arguments. */
+#define qemu_getsockopt(sockfd, level, optname, optval, optlen) \
+    getsockopt(sockfd, level, optname, (void *)optval, optlen)
+#define qemu_setsockopt(sockfd, level, optname, optval, optlen) \
+    setsockopt(sockfd, level, optname, (const void *)optval, optlen)
+#define qemu_recv(sockfd, buf, len, flags) recv(sockfd, (void *)buf, len, flags)
+#define qemu_sendto(sockfd, buf, len, flags, destaddr, addrlen) \
+    sendto(sockfd, (const void *)buf, len, flags, destaddr, addrlen)
+#else
+#define qemu_getsockopt(sockfd, level, optname, optval, optlen) \
+    getsockopt(sockfd, level, optname, optval, optlen)
+#define qemu_setsockopt(sockfd, level, optname, optval, optlen) \
+    setsockopt(sockfd, level, optname, optval, optlen)
+#define qemu_recv(sockfd, buf, len, flags) recv(sockfd, buf, len, flags)
+#define qemu_sendto(sockfd, buf, len, flags, destaddr, addrlen) \
+    sendto(sockfd, buf, len, flags, destaddr, addrlen)
+#endif
+
+/* Error handling.  */
+
+void QEMU_NORETURN hw_error(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
+
+struct ParallelIOArg {
+    void *buffer;
+    int count;
+};
+
+typedef int (*DMA_transfer_handler) (void *opaque, int nchan, int pos, int size);
+
+typedef uint64_t pcibus_t;
+
+typedef enum LostTickPolicy {
+    LOST_TICK_DISCARD,
+    LOST_TICK_DELAY,
+    LOST_TICK_MERGE,
+    LOST_TICK_SLEW,
+    LOST_TICK_MAX
+} LostTickPolicy;
+
+typedef struct PCIHostDeviceAddress {
+    unsigned int domain;
+    unsigned int bus;
+    unsigned int slot;
+    unsigned int function;
+} PCIHostDeviceAddress;
+
+void tcg_exec_init(unsigned long tb_size);
+bool tcg_enabled(void);
+
+void cpu_exec_init_all(void);
+
+/* CPU save/load.  */
+#ifdef CPU_SAVE_VERSION
+void cpu_save(QEMUFile *f, void *opaque);
+int cpu_load(QEMUFile *f, void *opaque, int version_id);
+#endif
+
+/* Unblock cpu */
+void qemu_cpu_kick_self(void);
+
+/* work queue */
+struct qemu_work_item {
+    struct qemu_work_item *next;
+    void (*func)(void *data);
+    void *data;
+    int done;
+    bool free;
+};
+
+
+/**
+ * Sends a (part of) iovec down a socket, yielding when the socket is full, or
+ * Receives data into a (part of) iovec from a socket,
+ * yielding when there is no data in the socket.
+ * The same interface as qemu_sendv_recvv(), with added yielding.
+ * XXX should mark these as coroutine_fn
+ */
+ssize_t qemu_co_sendv_recvv(int sockfd, struct iovec *iov, unsigned iov_cnt,
+                            size_t offset, size_t bytes, bool do_send);
+#define qemu_co_recvv(sockfd, iov, iov_cnt, offset, bytes) \
+  qemu_co_sendv_recvv(sockfd, iov, iov_cnt, offset, bytes, false)
+#define qemu_co_sendv(sockfd, iov, iov_cnt, offset, bytes) \
+  qemu_co_sendv_recvv(sockfd, iov, iov_cnt, offset, bytes, true)
+
+/**
+ * The same as above, but with just a single buffer
+ */
+ssize_t qemu_co_send_recv(int sockfd, void *buf, size_t bytes, bool do_send);
+#define qemu_co_recv(sockfd, buf, bytes) \
+  qemu_co_send_recv(sockfd, buf, bytes, false)
+#define qemu_co_send(sockfd, buf, bytes) \
+  qemu_co_send_recv(sockfd, buf, bytes, true)
+
+typedef struct QEMUIOVector {
+    struct iovec *iov;
+    int niov;
+    int nalloc;
+    size_t size;
+} QEMUIOVector;
+
+void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint);
+void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov);
+void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len);
+void qemu_iovec_concat(QEMUIOVector *dst,
+                       QEMUIOVector *src, size_t soffset, size_t sbytes);
+void qemu_iovec_concat_iov(QEMUIOVector *dst,
+                           struct iovec *src_iov, unsigned int src_cnt,
+                           size_t soffset, size_t sbytes);
+void qemu_iovec_destroy(QEMUIOVector *qiov);
+void qemu_iovec_reset(QEMUIOVector *qiov);
+size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
+                         void *buf, size_t bytes);
+size_t qemu_iovec_from_buf(QEMUIOVector *qiov, size_t offset,
+                           const void *buf, size_t bytes);
+size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
+                         int fillc, size_t bytes);
+
+bool buffer_is_zero(const void *buf, size_t len);
+
+void qemu_progress_init(int enabled, float min_skip);
+void qemu_progress_end(void);
+void qemu_progress_print(float delta, int max);
+const char *qemu_get_vm_name(void);
+
+#define QEMU_FILE_TYPE_BIOS   0
+#define QEMU_FILE_TYPE_KEYMAP 1
+char *qemu_find_file(int type, const char *name);
+
+/* OS specific functions */
+void os_setup_early_signal_handling(void);
+char *os_find_datadir(const char *argv0);
+void os_parse_cmd_args(int index, const char *optarg);
+void os_pidfile_error(void);
+
+/* Convert a byte between binary and BCD.  */
+static inline uint8_t to_bcd(uint8_t val)
+{
+    return ((val / 10) << 4) | (val % 10);
+}
+
+static inline uint8_t from_bcd(uint8_t val)
+{
+    return ((val >> 4) * 10) + (val & 0x0f);
+}
+
+/* compute with 96 bit intermediate result: (a*b)/c */
+static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
+{
+    union {
+        uint64_t ll;
+        struct {
+#ifdef HOST_WORDS_BIGENDIAN
+            uint32_t high, low;
+#else
+            uint32_t low, high;
+#endif
+        } l;
+    } u, res;
+    uint64_t rl, rh;
+
+    u.ll = a;
+    rl = (uint64_t)u.l.low * (uint64_t)b;
+    rh = (uint64_t)u.l.high * (uint64_t)b;
+    rh += (rl >> 32);
+    res.l.high = rh / c;
+    res.l.low = (((rh % c) << 32) + (rl & 0xffffffff)) / c;
+    return res.ll;
+}
+
+/* Round number down to multiple */
+#define QEMU_ALIGN_DOWN(n, m) ((n) / (m) * (m))
+
+/* Round number up to multiple */
+#define QEMU_ALIGN_UP(n, m) QEMU_ALIGN_DOWN((n) + (m) - 1, (m))
+
+static inline bool is_power_of_2(uint64_t value)
+{
+    if (!value) {
+        return 0;
+    }
+
+    return !(value & (value - 1));
+}
+
+/* round down to the nearest power of 2*/
+int64_t pow2floor(int64_t value);
+
+#include "qemu/module.h"
+
+/*
+ * Implementation of ULEB128 (http://en.wikipedia.org/wiki/LEB128)
+ * Input is limited to 14-bit numbers
+ */
+
+int uleb128_encode_small(uint8_t *out, uint32_t n);
+int uleb128_decode_small(const uint8_t *in, uint32_t *n);
+
+/* unicode.c */
+int mod_utf8_codepoint(const char *s, size_t n, char **end);
+
+/*
+ * Hexdump a buffer to a file. An optional string prefix is added to every line
+ */
+
+void qemu_hexdump(const char *buf, FILE *fp, const char *prefix, size_t size);
+
+/* vector definitions */
+#ifdef __ALTIVEC__
+#include <altivec.h>
+/* The altivec.h header says we're allowed to undef these for
+ * C++ compatibility.  Here we don't care about C++, but we
+ * undef them anyway to avoid namespace pollution.
+ */
+#undef vector
+#undef pixel
+#undef bool
+#define VECTYPE        __vector unsigned char
+#define SPLAT(p)       vec_splat(vec_ld(0, p), 0)
+#define ALL_EQ(v1, v2) vec_all_eq(v1, v2)
+/* altivec.h may redefine the bool macro as vector type.
+ * Reset it to POSIX semantics. */
+#define bool _Bool
+#elif defined __SSE2__
+#include <emmintrin.h>
+#define VECTYPE        __m128i
+#define SPLAT(p)       _mm_set1_epi8(*(p))
+#define ALL_EQ(v1, v2) (_mm_movemask_epi8(_mm_cmpeq_epi8(v1, v2)) == 0xFFFF)
+#else
+#define VECTYPE        unsigned long
+#define SPLAT(p)       (*(p) * (~0UL / 255))
+#define ALL_EQ(v1, v2) ((v1) == (v2))
+#endif
+
+#define BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR 8
+static inline bool
+can_use_buffer_find_nonzero_offset(const void *buf, size_t len)
+{
+    return (len % (BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR
+                   * sizeof(VECTYPE)) == 0
+            && ((uintptr_t) buf) % sizeof(VECTYPE) == 0);
+}
+size_t buffer_find_nonzero_offset(const void *buf, size_t len);
+
+/*
+ * helper to parse debug environment variables
+ */
+int parse_debug_env(const char *name, int max, int initial);
+
+#endif
diff --git a/contrib/qemu/include/qemu/aes.h b/contrib/qemu/include/qemu/aes.h
new file mode 100644
index 000000000..e79c70743
--- /dev/null
+++ b/contrib/qemu/include/qemu/aes.h
@@ -0,0 +1,45 @@
+#ifndef QEMU_AES_H
+#define QEMU_AES_H
+
+#define AES_MAXNR 14
+#define AES_BLOCK_SIZE 16
+
+struct aes_key_st {
+    uint32_t rd_key[4 *(AES_MAXNR + 1)];
+    int rounds;
+};
+typedef struct aes_key_st AES_KEY;
+
+int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
+	AES_KEY *key);
+int AES_set_decrypt_key(const unsigned char *userKey, const int bits,
+	AES_KEY *key);
+
+void AES_encrypt(const unsigned char *in, unsigned char *out,
+	const AES_KEY *key);
+void AES_decrypt(const unsigned char *in, unsigned char *out,
+	const AES_KEY *key);
+void AES_cbc_encrypt(const unsigned char *in, unsigned char *out,
+		     const unsigned long length, const AES_KEY *key,
+		     unsigned char *ivec, const int enc);
+
+/*
+AES_Te0[x] = S [x].[02, 01, 01, 03];
+AES_Te1[x] = S [x].[03, 02, 01, 01];
+AES_Te2[x] = S [x].[01, 03, 02, 01];
+AES_Te3[x] = S [x].[01, 01, 03, 02];
+AES_Te4[x] = S [x].[01, 01, 01, 01];
+
+AES_Td0[x] = Si[x].[0e, 09, 0d, 0b];
+AES_Td1[x] = Si[x].[0b, 0e, 09, 0d];
+AES_Td2[x] = Si[x].[0d, 0b, 0e, 09];
+AES_Td3[x] = Si[x].[09, 0d, 0b, 0e];
+AES_Td4[x] = Si[x].[01, 01, 01, 01];
+*/
+
+extern const uint32_t AES_Te0[256], AES_Te1[256], AES_Te2[256],
+                      AES_Te3[256], AES_Te4[256];
+extern const uint32_t AES_Td0[256], AES_Td1[256], AES_Td2[256],
+                      AES_Td3[256], AES_Td4[256];
+
+#endif
diff --git a/contrib/qemu/include/qemu/atomic.h b/contrib/qemu/include/qemu/atomic.h
new file mode 100644
index 000000000..0aa891330
--- /dev/null
+++ b/contrib/qemu/include/qemu/atomic.h
@@ -0,0 +1,202 @@
+/*
+ * Simple interface for atomic operations.
+ *
+ * Copyright (C) 2013 Red Hat, Inc.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef __QEMU_ATOMIC_H
+#define __QEMU_ATOMIC_H 1
+
+#include "qemu/compiler.h"
+
+/* For C11 atomic ops */
+
+/* Compiler barrier */
+#define barrier()   ({ asm volatile("" ::: "memory"); (void)0; })
+
+#ifndef __ATOMIC_RELAXED
+
+/*
+ * We use GCC builtin if it's available, as that can use mfence on
+ * 32-bit as well, e.g. if built with -march=pentium-m. However, on
+ * i386 the spec is buggy, and the implementation followed it until
+ * 4.3 (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793).
+ */
+#if defined(__i386__) || defined(__x86_64__)
+#if !QEMU_GNUC_PREREQ(4, 4)
+#if defined __x86_64__
+#define smp_mb()    ({ asm volatile("mfence" ::: "memory"); (void)0; })
+#else
+#define smp_mb()    ({ asm volatile("lock; addl $0,0(%%esp) " ::: "memory"); (void)0; })
+#endif
+#endif
+#endif
+
+
+#ifdef __alpha__
+#define smp_read_barrier_depends()   asm volatile("mb":::"memory")
+#endif
+
+#if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
+
+/*
+ * Because of the strongly ordered storage model, wmb() and rmb() are nops
+ * here (a compiler barrier only).  QEMU doesn't do accesses to write-combining
+ * qemu memory or non-temporal load/stores from C code.
+ */
+#define smp_wmb()   barrier()
+#define smp_rmb()   barrier()
+
+/*
+ * __sync_lock_test_and_set() is documented to be an acquire barrier only,
+ * but it is a full barrier at the hardware level.  Add a compiler barrier
+ * to make it a full barrier also at the compiler level.
+ */
+#define atomic_xchg(ptr, i)    (barrier(), __sync_lock_test_and_set(ptr, i))
+
+/*
+ * Load/store with Java volatile semantics.
+ */
+#define atomic_mb_set(ptr, i)  ((void)atomic_xchg(ptr, i))
+
+#elif defined(_ARCH_PPC)
+
+/*
+ * We use an eieio() for wmb() on powerpc.  This assumes we don't
+ * need to order cacheable and non-cacheable stores with respect to
+ * each other.
+ *
+ * smp_mb has the same problem as on x86 for not-very-new GCC
+ * (http://patchwork.ozlabs.org/patch/126184/, Nov 2011).
+ */
+#define smp_wmb()   ({ asm volatile("eieio" ::: "memory"); (void)0; })
+#if defined(__powerpc64__)
+#define smp_rmb()   ({ asm volatile("lwsync" ::: "memory"); (void)0; })
+#else
+#define smp_rmb()   ({ asm volatile("sync" ::: "memory"); (void)0; })
+#endif
+#define smp_mb()    ({ asm volatile("sync" ::: "memory"); (void)0; })
+
+#endif /* _ARCH_PPC */
+
+#endif /* C11 atomics */
+
+/*
+ * For (host) platforms we don't have explicit barrier definitions
+ * for, we use the gcc __sync_synchronize() primitive to generate a
+ * full barrier.  This should be safe on all platforms, though it may
+ * be overkill for smp_wmb() and smp_rmb().
+ */
+#ifndef smp_mb
+#define smp_mb()    __sync_synchronize()
+#endif
+
+#ifndef smp_wmb
+#ifdef __ATOMIC_RELEASE
+#define smp_wmb()   __atomic_thread_fence(__ATOMIC_RELEASE)
+#else
+#define smp_wmb()   __sync_synchronize()
+#endif
+#endif
+
+#ifndef smp_rmb
+#ifdef __ATOMIC_ACQUIRE
+#define smp_rmb()   __atomic_thread_fence(__ATOMIC_ACQUIRE)
+#else
+#define smp_rmb()   __sync_synchronize()
+#endif
+#endif
+
+#ifndef smp_read_barrier_depends
+#ifdef __ATOMIC_CONSUME
+#define smp_read_barrier_depends()   __atomic_thread_fence(__ATOMIC_CONSUME)
+#else
+#define smp_read_barrier_depends()   barrier()
+#endif
+#endif
+
+#ifndef atomic_read
+#define atomic_read(ptr)       (*(__typeof__(*ptr) *volatile) (ptr))
+#endif
+
+#ifndef atomic_set
+#define atomic_set(ptr, i)     ((*(__typeof__(*ptr) *volatile) (ptr)) = (i))
+#endif
+
+/* These have the same semantics as Java volatile variables.
+ * See http://gee.cs.oswego.edu/dl/jmm/cookbook.html:
+ * "1. Issue a StoreStore barrier (wmb) before each volatile store."
+ *  2. Issue a StoreLoad barrier after each volatile store.
+ *     Note that you could instead issue one before each volatile load, but
+ *     this would be slower for typical programs using volatiles in which
+ *     reads greatly outnumber writes. Alternatively, if available, you
+ *     can implement volatile store as an atomic instruction (for example
+ *     XCHG on x86) and omit the barrier. This may be more efficient if
+ *     atomic instructions are cheaper than StoreLoad barriers.
+ *  3. Issue LoadLoad and LoadStore barriers after each volatile load."
+ *
+ * If you prefer to think in terms of "pairing" of memory barriers,
+ * an atomic_mb_read pairs with an atomic_mb_set.
+ *
+ * And for the few ia64 lovers that exist, an atomic_mb_read is a ld.acq,
+ * while an atomic_mb_set is a st.rel followed by a memory barrier.
+ *
+ * These are a bit weaker than __atomic_load/store with __ATOMIC_SEQ_CST
+ * (see docs/atomics.txt), and I'm not sure that __ATOMIC_ACQ_REL is enough.
+ * Just always use the barriers manually by the rules above.
+ */
+#ifndef atomic_mb_read
+#define atomic_mb_read(ptr)    ({           \
+    typeof(*ptr) _val = atomic_read(ptr);   \
+    smp_rmb();                              \
+    _val;                                   \
+})
+#endif
+
+#ifndef atomic_mb_set
+#define atomic_mb_set(ptr, i)  do {         \
+    smp_wmb();                              \
+    atomic_set(ptr, i);                     \
+    smp_mb();                               \
+} while (0)
+#endif
+
+#ifndef atomic_xchg
+#ifdef __ATOMIC_SEQ_CST
+#define atomic_xchg(ptr, i)    ({                           \
+    typeof(*ptr) _new = (i), _old;                          \
+    __atomic_exchange(ptr, &_new, &_old, __ATOMIC_SEQ_CST); \
+    _old;                                                   \
+})
+#elif defined __clang__
+#define atomic_xchg(ptr, i)    __sync_exchange(ptr, i)
+#else
+/* __sync_lock_test_and_set() is documented to be an acquire barrier only.  */
+#define atomic_xchg(ptr, i)    (smp_mb(), __sync_lock_test_and_set(ptr, i))
+#endif
+#endif
+
+/* Provide shorter names for GCC atomic builtins.  */
+#define atomic_fetch_inc(ptr)  __sync_fetch_and_add(ptr, 1)
+#define atomic_fetch_dec(ptr)  __sync_fetch_and_add(ptr, -1)
+#define atomic_fetch_add       __sync_fetch_and_add
+#define atomic_fetch_sub       __sync_fetch_and_sub
+#define atomic_fetch_and       __sync_fetch_and_and
+#define atomic_fetch_or        __sync_fetch_and_or
+#define atomic_cmpxchg         __sync_val_compare_and_swap
+
+/* And even shorter names that return void.  */
+#define atomic_inc(ptr)        ((void) __sync_fetch_and_add(ptr, 1))
+#define atomic_dec(ptr)        ((void) __sync_fetch_and_add(ptr, -1))
+#define atomic_add(ptr, n)     ((void) __sync_fetch_and_add(ptr, n))
+#define atomic_sub(ptr, n)     ((void) __sync_fetch_and_sub(ptr, n))
+#define atomic_and(ptr, n)     ((void) __sync_fetch_and_and(ptr, n))
+#define atomic_or(ptr, n)      ((void) __sync_fetch_and_or(ptr, n))
+
+#endif
diff --git a/contrib/qemu/include/qemu/bitmap.h b/contrib/qemu/include/qemu/bitmap.h
new file mode 100644
index 000000000..308bbb71e
--- /dev/null
+++ b/contrib/qemu/include/qemu/bitmap.h
@@ -0,0 +1,222 @@
+/*
+ * Bitmap Module
+ *
+ * Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com>
+ *
+ * Mostly inspired by (stolen from) linux/bitmap.h and linux/bitops.h
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#ifndef BITMAP_H
+#define BITMAP_H
+
+#include "qemu-common.h"
+#include "qemu/bitops.h"
+
+/*
+ * The available bitmap operations and their rough meaning in the
+ * case that the bitmap is a single unsigned long are thus:
+ *
+ * Note that nbits should be always a compile time evaluable constant.
+ * Otherwise many inlines will generate horrible code.
+ *
+ * bitmap_zero(dst, nbits)			*dst = 0UL
+ * bitmap_fill(dst, nbits)			*dst = ~0UL
+ * bitmap_copy(dst, src, nbits)			*dst = *src
+ * bitmap_and(dst, src1, src2, nbits)		*dst = *src1 & *src2
+ * bitmap_or(dst, src1, src2, nbits)		*dst = *src1 | *src2
+ * bitmap_xor(dst, src1, src2, nbits)		*dst = *src1 ^ *src2
+ * bitmap_andnot(dst, src1, src2, nbits)	*dst = *src1 & ~(*src2)
+ * bitmap_complement(dst, src, nbits)		*dst = ~(*src)
+ * bitmap_equal(src1, src2, nbits)		Are *src1 and *src2 equal?
+ * bitmap_intersects(src1, src2, nbits) 	Do *src1 and *src2 overlap?
+ * bitmap_empty(src, nbits)			Are all bits zero in *src?
+ * bitmap_full(src, nbits)			Are all bits set in *src?
+ * bitmap_set(dst, pos, nbits)			Set specified bit area
+ * bitmap_clear(dst, pos, nbits)		Clear specified bit area
+ * bitmap_find_next_zero_area(buf, len, pos, n, mask)	Find bit free area
+ */
+
+/*
+ * Also the following operations apply to bitmaps.
+ *
+ * set_bit(bit, addr)			*addr |= bit
+ * clear_bit(bit, addr)			*addr &= ~bit
+ * change_bit(bit, addr)		*addr ^= bit
+ * test_bit(bit, addr)			Is bit set in *addr?
+ * test_and_set_bit(bit, addr)		Set bit and return old value
+ * test_and_clear_bit(bit, addr)	Clear bit and return old value
+ * test_and_change_bit(bit, addr)	Change bit and return old value
+ * find_first_zero_bit(addr, nbits)	Position first zero bit in *addr
+ * find_first_bit(addr, nbits)		Position first set bit in *addr
+ * find_next_zero_bit(addr, nbits, bit)	Position next zero bit in *addr >= bit
+ * find_next_bit(addr, nbits, bit)	Position next set bit in *addr >= bit
+ */
+
+#define BITMAP_LAST_WORD_MASK(nbits)                                    \
+    (                                                                   \
+        ((nbits) % BITS_PER_LONG) ?                                     \
+        (1UL<<((nbits) % BITS_PER_LONG))-1 : ~0UL                       \
+        )
+
+#define DECLARE_BITMAP(name,bits)                  \
+	unsigned long name[BITS_TO_LONGS(bits)]
+
+#define small_nbits(nbits)                      \
+	((nbits) <= BITS_PER_LONG)
+
+int slow_bitmap_empty(const unsigned long *bitmap, int bits);
+int slow_bitmap_full(const unsigned long *bitmap, int bits);
+int slow_bitmap_equal(const unsigned long *bitmap1,
+                   const unsigned long *bitmap2, int bits);
+void slow_bitmap_complement(unsigned long *dst, const unsigned long *src,
+                         int bits);
+void slow_bitmap_shift_right(unsigned long *dst,
+                          const unsigned long *src, int shift, int bits);
+void slow_bitmap_shift_left(unsigned long *dst,
+                         const unsigned long *src, int shift, int bits);
+int slow_bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
+                 const unsigned long *bitmap2, int bits);
+void slow_bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
+                 const unsigned long *bitmap2, int bits);
+void slow_bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
+                  const unsigned long *bitmap2, int bits);
+int slow_bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
+                    const unsigned long *bitmap2, int bits);
+int slow_bitmap_intersects(const unsigned long *bitmap1,
+			const unsigned long *bitmap2, int bits);
+
+static inline unsigned long *bitmap_new(int nbits)
+{
+    int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
+    return g_malloc0(len);
+}
+
+static inline void bitmap_zero(unsigned long *dst, int nbits)
+{
+    if (small_nbits(nbits)) {
+        *dst = 0UL;
+    } else {
+        int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
+        memset(dst, 0, len);
+    }
+}
+
+static inline void bitmap_fill(unsigned long *dst, int nbits)
+{
+    size_t nlongs = BITS_TO_LONGS(nbits);
+    if (!small_nbits(nbits)) {
+        int len = (nlongs - 1) * sizeof(unsigned long);
+        memset(dst, 0xff,  len);
+    }
+    dst[nlongs - 1] = BITMAP_LAST_WORD_MASK(nbits);
+}
+
+static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,
+                               int nbits)
+{
+    if (small_nbits(nbits)) {
+        *dst = *src;
+    } else {
+        int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
+        memcpy(dst, src, len);
+    }
+}
+
+static inline int bitmap_and(unsigned long *dst, const unsigned long *src1,
+                             const unsigned long *src2, int nbits)
+{
+    if (small_nbits(nbits)) {
+        return (*dst = *src1 & *src2) != 0;
+    }
+    return slow_bitmap_and(dst, src1, src2, nbits);
+}
+
+static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
+			const unsigned long *src2, int nbits)
+{
+    if (small_nbits(nbits)) {
+        *dst = *src1 | *src2;
+    } else {
+        slow_bitmap_or(dst, src1, src2, nbits);
+    }
+}
+
+static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
+			const unsigned long *src2, int nbits)
+{
+    if (small_nbits(nbits)) {
+        *dst = *src1 ^ *src2;
+    } else {
+        slow_bitmap_xor(dst, src1, src2, nbits);
+    }
+}
+
+static inline int bitmap_andnot(unsigned long *dst, const unsigned long *src1,
+			const unsigned long *src2, int nbits)
+{
+    if (small_nbits(nbits)) {
+        return (*dst = *src1 & ~(*src2)) != 0;
+    }
+    return slow_bitmap_andnot(dst, src1, src2, nbits);
+}
+
+static inline void bitmap_complement(unsigned long *dst, const unsigned long *src,
+			int nbits)
+{
+    if (small_nbits(nbits)) {
+        *dst = ~(*src) & BITMAP_LAST_WORD_MASK(nbits);
+    } else {
+        slow_bitmap_complement(dst, src, nbits);
+    }
+}
+
+static inline int bitmap_equal(const unsigned long *src1,
+			const unsigned long *src2, int nbits)
+{
+    if (small_nbits(nbits)) {
+        return ! ((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
+    } else {
+        return slow_bitmap_equal(src1, src2, nbits);
+    }
+}
+
+static inline int bitmap_empty(const unsigned long *src, int nbits)
+{
+    if (small_nbits(nbits)) {
+        return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
+    } else {
+        return slow_bitmap_empty(src, nbits);
+    }
+}
+
+static inline int bitmap_full(const unsigned long *src, int nbits)
+{
+    if (small_nbits(nbits)) {
+        return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
+    } else {
+        return slow_bitmap_full(src, nbits);
+    }
+}
+
+static inline int bitmap_intersects(const unsigned long *src1,
+			const unsigned long *src2, int nbits)
+{
+    if (small_nbits(nbits)) {
+        return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
+    } else {
+        return slow_bitmap_intersects(src1, src2, nbits);
+    }
+}
+
+void bitmap_set(unsigned long *map, int i, int len);
+void bitmap_clear(unsigned long *map, int start, int nr);
+unsigned long bitmap_find_next_zero_area(unsigned long *map,
+					 unsigned long size,
+					 unsigned long start,
+					 unsigned int nr,
+					 unsigned long align_mask);
+
+#endif /* BITMAP_H */
diff --git a/contrib/qemu/include/qemu/bitops.h b/contrib/qemu/include/qemu/bitops.h
new file mode 100644
index 000000000..affcc969d
--- /dev/null
+++ b/contrib/qemu/include/qemu/bitops.h
@@ -0,0 +1,276 @@
+/*
+ * Bitops Module
+ *
+ * Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com>
+ *
+ * Mostly inspired by (stolen from) linux/bitmap.h and linux/bitops.h
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#ifndef BITOPS_H
+#define BITOPS_H
+
+#include "qemu-common.h"
+#include "host-utils.h"
+
+#define BITS_PER_BYTE           CHAR_BIT
+#define BITS_PER_LONG           (sizeof (unsigned long) * BITS_PER_BYTE)
+
+#define BIT(nr)			(1UL << (nr))
+#define BIT_MASK(nr)		(1UL << ((nr) % BITS_PER_LONG))
+#define BIT_WORD(nr)		((nr) / BITS_PER_LONG)
+#define BITS_TO_LONGS(nr)	DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(long))
+
+/**
+ * set_bit - Set a bit in memory
+ * @nr: the bit to set
+ * @addr: the address to start counting from
+ */
+static inline void set_bit(int nr, unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+        unsigned long *p = addr + BIT_WORD(nr);
+
+	*p  |= mask;
+}
+
+/**
+ * clear_bit - Clears a bit in memory
+ * @nr: Bit to clear
+ * @addr: Address to start counting from
+ */
+static inline void clear_bit(int nr, unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+        unsigned long *p = addr + BIT_WORD(nr);
+
+	*p &= ~mask;
+}
+
+/**
+ * change_bit - Toggle a bit in memory
+ * @nr: Bit to change
+ * @addr: Address to start counting from
+ */
+static inline void change_bit(int nr, unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+        unsigned long *p = addr + BIT_WORD(nr);
+
+	*p ^= mask;
+}
+
+/**
+ * test_and_set_bit - Set a bit and return its old value
+ * @nr: Bit to set
+ * @addr: Address to count from
+ */
+static inline int test_and_set_bit(int nr, unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+        unsigned long *p = addr + BIT_WORD(nr);
+	unsigned long old = *p;
+
+	*p = old | mask;
+	return (old & mask) != 0;
+}
+
+/**
+ * test_and_clear_bit - Clear a bit and return its old value
+ * @nr: Bit to clear
+ * @addr: Address to count from
+ */
+static inline int test_and_clear_bit(int nr, unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+        unsigned long *p = addr + BIT_WORD(nr);
+	unsigned long old = *p;
+
+	*p = old & ~mask;
+	return (old & mask) != 0;
+}
+
+/**
+ * test_and_change_bit - Change a bit and return its old value
+ * @nr: Bit to change
+ * @addr: Address to count from
+ */
+static inline int test_and_change_bit(int nr, unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+        unsigned long *p = addr + BIT_WORD(nr);
+	unsigned long old = *p;
+
+	*p = old ^ mask;
+	return (old & mask) != 0;
+}
+
+/**
+ * test_bit - Determine whether a bit is set
+ * @nr: bit number to test
+ * @addr: Address to start counting from
+ */
+static inline int test_bit(int nr, const unsigned long *addr)
+{
+	return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
+}
+
+/**
+ * find_last_bit - find the last set bit in a memory region
+ * @addr: The address to start the search at
+ * @size: The maximum size to search
+ *
+ * Returns the bit number of the first set bit, or size.
+ */
+unsigned long find_last_bit(const unsigned long *addr,
+                            unsigned long size);
+
+/**
+ * find_next_bit - find the next set bit in a memory region
+ * @addr: The address to base the search on
+ * @offset: The bitnumber to start searching at
+ * @size: The bitmap size in bits
+ */
+unsigned long find_next_bit(const unsigned long *addr,
+				   unsigned long size, unsigned long offset);
+
+/**
+ * find_next_zero_bit - find the next cleared bit in a memory region
+ * @addr: The address to base the search on
+ * @offset: The bitnumber to start searching at
+ * @size: The bitmap size in bits
+ */
+
+unsigned long find_next_zero_bit(const unsigned long *addr,
+                                 unsigned long size,
+                                 unsigned long offset);
+
+/**
+ * find_first_bit - find the first set bit in a memory region
+ * @addr: The address to start the search at
+ * @size: The maximum size to search
+ *
+ * Returns the bit number of the first set bit.
+ */
+static inline unsigned long find_first_bit(const unsigned long *addr,
+                                           unsigned long size)
+{
+    return find_next_bit(addr, size, 0);
+}
+
+/**
+ * find_first_zero_bit - find the first cleared bit in a memory region
+ * @addr: The address to start the search at
+ * @size: The maximum size to search
+ *
+ * Returns the bit number of the first cleared bit.
+ */
+static inline unsigned long find_first_zero_bit(const unsigned long *addr,
+                                                unsigned long size)
+{
+    return find_next_zero_bit(addr, size, 0);
+}
+
+static inline unsigned long hweight_long(unsigned long w)
+{
+    unsigned long count;
+
+    for (count = 0; w; w >>= 1) {
+        count += w & 1;
+    }
+    return count;
+}
+
+/**
+ * extract32:
+ * @value: the value to extract the bit field from
+ * @start: the lowest bit in the bit field (numbered from 0)
+ * @length: the length of the bit field
+ *
+ * Extract from the 32 bit input @value the bit field specified by the
+ * @start and @length parameters, and return it. The bit field must
+ * lie entirely within the 32 bit word. It is valid to request that
+ * all 32 bits are returned (ie @length 32 and @start 0).
+ *
+ * Returns: the value of the bit field extracted from the input value.
+ */
+static inline uint32_t extract32(uint32_t value, int start, int length)
+{
+    assert(start >= 0 && length > 0 && length <= 32 - start);
+    return (value >> start) & (~0U >> (32 - length));
+}
+
+/**
+ * extract64:
+ * @value: the value to extract the bit field from
+ * @start: the lowest bit in the bit field (numbered from 0)
+ * @length: the length of the bit field
+ *
+ * Extract from the 64 bit input @value the bit field specified by the
+ * @start and @length parameters, and return it. The bit field must
+ * lie entirely within the 64 bit word. It is valid to request that
+ * all 64 bits are returned (ie @length 64 and @start 0).
+ *
+ * Returns: the value of the bit field extracted from the input value.
+ */
+static inline uint64_t extract64(uint64_t value, int start, int length)
+{
+    assert(start >= 0 && length > 0 && length <= 64 - start);
+    return (value >> start) & (~0ULL >> (64 - length));
+}
+
+/**
+ * deposit32:
+ * @value: initial value to insert bit field into
+ * @start: the lowest bit in the bit field (numbered from 0)
+ * @length: the length of the bit field
+ * @fieldval: the value to insert into the bit field
+ *
+ * Deposit @fieldval into the 32 bit @value at the bit field specified
+ * by the @start and @length parameters, and return the modified
+ * @value. Bits of @value outside the bit field are not modified.
+ * Bits of @fieldval above the least significant @length bits are
+ * ignored. The bit field must lie entirely within the 32 bit word.
+ * It is valid to request that all 32 bits are modified (ie @length
+ * 32 and @start 0).
+ *
+ * Returns: the modified @value.
+ */
+static inline uint32_t deposit32(uint32_t value, int start, int length,
+                                 uint32_t fieldval)
+{
+    uint32_t mask;
+    assert(start >= 0 && length > 0 && length <= 32 - start);
+    mask = (~0U >> (32 - length)) << start;
+    return (value & ~mask) | ((fieldval << start) & mask);
+}
+
+/**
+ * deposit64:
+ * @value: initial value to insert bit field into
+ * @start: the lowest bit in the bit field (numbered from 0)
+ * @length: the length of the bit field
+ * @fieldval: the value to insert into the bit field
+ *
+ * Deposit @fieldval into the 64 bit @value at the bit field specified
+ * by the @start and @length parameters, and return the modified
+ * @value. Bits of @value outside the bit field are not modified.
+ * Bits of @fieldval above the least significant @length bits are
+ * ignored. The bit field must lie entirely within the 64 bit word.
+ * It is valid to request that all 64 bits are modified (ie @length
+ * 64 and @start 0).
+ *
+ * Returns: the modified @value.
+ */
+static inline uint64_t deposit64(uint64_t value, int start, int length,
+                                 uint64_t fieldval)
+{
+    uint64_t mask;
+    assert(start >= 0 && length > 0 && length <= 64 - start);
+    mask = (~0ULL >> (64 - length)) << start;
+    return (value & ~mask) | ((fieldval << start) & mask);
+}
+
+#endif
diff --git a/contrib/qemu/include/qemu/bswap.h b/contrib/qemu/include/qemu/bswap.h
new file mode 100644
index 000000000..14a5f657c
--- /dev/null
+++ b/contrib/qemu/include/qemu/bswap.h
@@ -0,0 +1,478 @@
+#ifndef BSWAP_H
+#define BSWAP_H
+
+#include "config-host.h"
+#include <inttypes.h>
+#include <limits.h>
+#include <string.h>
+#include "fpu/softfloat.h"
+
+#ifdef CONFIG_MACHINE_BSWAP_H
+# include <sys/endian.h>
+# include <sys/types.h>
+# include <machine/bswap.h>
+#elif defined(CONFIG_BYTESWAP_H)
+# include <byteswap.h>
+
+static inline uint16_t bswap16(uint16_t x)
+{
+    return bswap_16(x);
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return bswap_32(x);
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return bswap_64(x);
+}
+# else
+static inline uint16_t bswap16(uint16_t x)
+{
+    return (((x & 0x00ff) << 8) |
+            ((x & 0xff00) >> 8));
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return (((x & 0x000000ffU) << 24) |
+            ((x & 0x0000ff00U) <<  8) |
+            ((x & 0x00ff0000U) >>  8) |
+            ((x & 0xff000000U) >> 24));
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return (((x & 0x00000000000000ffULL) << 56) |
+            ((x & 0x000000000000ff00ULL) << 40) |
+            ((x & 0x0000000000ff0000ULL) << 24) |
+            ((x & 0x00000000ff000000ULL) <<  8) |
+            ((x & 0x000000ff00000000ULL) >>  8) |
+            ((x & 0x0000ff0000000000ULL) >> 24) |
+            ((x & 0x00ff000000000000ULL) >> 40) |
+            ((x & 0xff00000000000000ULL) >> 56));
+}
+#endif /* ! CONFIG_MACHINE_BSWAP_H */
+
+static inline void bswap16s(uint16_t *s)
+{
+    *s = bswap16(*s);
+}
+
+static inline void bswap32s(uint32_t *s)
+{
+    *s = bswap32(*s);
+}
+
+static inline void bswap64s(uint64_t *s)
+{
+    *s = bswap64(*s);
+}
+
+#if defined(HOST_WORDS_BIGENDIAN)
+#define be_bswap(v, size) (v)
+#define le_bswap(v, size) glue(bswap, size)(v)
+#define be_bswaps(v, size)
+#define le_bswaps(p, size) do { *p = glue(bswap, size)(*p); } while(0)
+#else
+#define le_bswap(v, size) (v)
+#define be_bswap(v, size) glue(bswap, size)(v)
+#define le_bswaps(v, size)
+#define be_bswaps(p, size) do { *p = glue(bswap, size)(*p); } while(0)
+#endif
+
+#define CPU_CONVERT(endian, size, type)\
+static inline type endian ## size ## _to_cpu(type v)\
+{\
+    return glue(endian, _bswap)(v, size);\
+}\
+\
+static inline type cpu_to_ ## endian ## size(type v)\
+{\
+    return glue(endian, _bswap)(v, size);\
+}\
+\
+static inline void endian ## size ## _to_cpus(type *p)\
+{\
+    glue(endian, _bswaps)(p, size);\
+}\
+\
+static inline void cpu_to_ ## endian ## size ## s(type *p)\
+{\
+    glue(endian, _bswaps)(p, size);\
+}\
+\
+static inline type endian ## size ## _to_cpup(const type *p)\
+{\
+    return glue(glue(endian, size), _to_cpu)(*p);\
+}\
+\
+static inline void cpu_to_ ## endian ## size ## w(type *p, type v)\
+{\
+    *p = glue(glue(cpu_to_, endian), size)(v);\
+}
+
+CPU_CONVERT(be, 16, uint16_t)
+CPU_CONVERT(be, 32, uint32_t)
+CPU_CONVERT(be, 64, uint64_t)
+
+CPU_CONVERT(le, 16, uint16_t)
+CPU_CONVERT(le, 32, uint32_t)
+CPU_CONVERT(le, 64, uint64_t)
+
+/* len must be one of 1, 2, 4 */
+static inline uint32_t qemu_bswap_len(uint32_t value, int len)
+{
+    return bswap32(value) >> (32 - 8 * len);
+}
+
+/* Unions for reinterpreting between floats and integers.  */
+
+typedef union {
+    float32 f;
+    uint32_t l;
+} CPU_FloatU;
+
+typedef union {
+    float64 d;
+#if defined(HOST_WORDS_BIGENDIAN)
+    struct {
+        uint32_t upper;
+        uint32_t lower;
+    } l;
+#else
+    struct {
+        uint32_t lower;
+        uint32_t upper;
+    } l;
+#endif
+    uint64_t ll;
+} CPU_DoubleU;
+
+typedef union {
+     floatx80 d;
+     struct {
+         uint64_t lower;
+         uint16_t upper;
+     } l;
+} CPU_LDoubleU;
+
+typedef union {
+    float128 q;
+#if defined(HOST_WORDS_BIGENDIAN)
+    struct {
+        uint32_t upmost;
+        uint32_t upper;
+        uint32_t lower;
+        uint32_t lowest;
+    } l;
+    struct {
+        uint64_t upper;
+        uint64_t lower;
+    } ll;
+#else
+    struct {
+        uint32_t lowest;
+        uint32_t lower;
+        uint32_t upper;
+        uint32_t upmost;
+    } l;
+    struct {
+        uint64_t lower;
+        uint64_t upper;
+    } ll;
+#endif
+} CPU_QuadU;
+
+/* unaligned/endian-independent pointer access */
+
+/*
+ * the generic syntax is:
+ *
+ * load: ld{type}{sign}{size}{endian}_p(ptr)
+ *
+ * store: st{type}{size}{endian}_p(ptr, val)
+ *
+ * Note there are small differences with the softmmu access API!
+ *
+ * type is:
+ * (empty): integer access
+ *   f    : float access
+ *
+ * sign is:
+ * (empty): for floats or 32 bit size
+ *   u    : unsigned
+ *   s    : signed
+ *
+ * size is:
+ *   b: 8 bits
+ *   w: 16 bits
+ *   l: 32 bits
+ *   q: 64 bits
+ *
+ * endian is:
+ * (empty): host endian
+ *   be   : big endian
+ *   le   : little endian
+ */
+
+static inline int ldub_p(const void *ptr)
+{
+    return *(uint8_t *)ptr;
+}
+
+static inline int ldsb_p(const void *ptr)
+{
+    return *(int8_t *)ptr;
+}
+
+static inline void stb_p(void *ptr, int v)
+{
+    *(uint8_t *)ptr = v;
+}
+
+/* Any compiler worth its salt will turn these memcpy into native unaligned
+   operations.  Thus we don't need to play games with packed attributes, or
+   inline byte-by-byte stores.  */
+
+static inline int lduw_p(const void *ptr)
+{
+    uint16_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline int ldsw_p(const void *ptr)
+{
+    int16_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline void stw_p(void *ptr, uint16_t v)
+{
+    memcpy(ptr, &v, sizeof(v));
+}
+
+static inline int ldl_p(const void *ptr)
+{
+    int32_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline void stl_p(void *ptr, uint32_t v)
+{
+    memcpy(ptr, &v, sizeof(v));
+}
+
+static inline uint64_t ldq_p(const void *ptr)
+{
+    uint64_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline void stq_p(void *ptr, uint64_t v)
+{
+    memcpy(ptr, &v, sizeof(v));
+}
+
+static inline int lduw_le_p(const void *ptr)
+{
+    return (uint16_t)le_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldsw_le_p(const void *ptr)
+{
+    return (int16_t)le_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldl_le_p(const void *ptr)
+{
+    return le_bswap(ldl_p(ptr), 32);
+}
+
+static inline uint64_t ldq_le_p(const void *ptr)
+{
+    return le_bswap(ldq_p(ptr), 64);
+}
+
+static inline void stw_le_p(void *ptr, int v)
+{
+    stw_p(ptr, le_bswap(v, 16));
+}
+
+static inline void stl_le_p(void *ptr, int v)
+{
+    stl_p(ptr, le_bswap(v, 32));
+}
+
+static inline void stq_le_p(void *ptr, uint64_t v)
+{
+    stq_p(ptr, le_bswap(v, 64));
+}
+
+/* float access */
+
+static inline float32 ldfl_le_p(const void *ptr)
+{
+    CPU_FloatU u;
+    u.l = ldl_le_p(ptr);
+    return u.f;
+}
+
+static inline void stfl_le_p(void *ptr, float32 v)
+{
+    CPU_FloatU u;
+    u.f = v;
+    stl_le_p(ptr, u.l);
+}
+
+static inline float64 ldfq_le_p(const void *ptr)
+{
+    CPU_DoubleU u;
+    u.ll = ldq_le_p(ptr);
+    return u.d;
+}
+
+static inline void stfq_le_p(void *ptr, float64 v)
+{
+    CPU_DoubleU u;
+    u.d = v;
+    stq_le_p(ptr, u.ll);
+}
+
+static inline int lduw_be_p(const void *ptr)
+{
+    return (uint16_t)be_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldsw_be_p(const void *ptr)
+{
+    return (int16_t)be_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldl_be_p(const void *ptr)
+{
+    return be_bswap(ldl_p(ptr), 32);
+}
+
+static inline uint64_t ldq_be_p(const void *ptr)
+{
+    return be_bswap(ldq_p(ptr), 64);
+}
+
+static inline void stw_be_p(void *ptr, int v)
+{
+    stw_p(ptr, be_bswap(v, 16));
+}
+
+static inline void stl_be_p(void *ptr, int v)
+{
+    stl_p(ptr, be_bswap(v, 32));
+}
+
+static inline void stq_be_p(void *ptr, uint64_t v)
+{
+    stq_p(ptr, be_bswap(v, 64));
+}
+
+/* float access */
+
+static inline float32 ldfl_be_p(const void *ptr)
+{
+    CPU_FloatU u;
+    u.l = ldl_be_p(ptr);
+    return u.f;
+}
+
+static inline void stfl_be_p(void *ptr, float32 v)
+{
+    CPU_FloatU u;
+    u.f = v;
+    stl_be_p(ptr, u.l);
+}
+
+static inline float64 ldfq_be_p(const void *ptr)
+{
+    CPU_DoubleU u;
+    u.ll = ldq_be_p(ptr);
+    return u.d;
+}
+
+static inline void stfq_be_p(void *ptr, float64 v)
+{
+    CPU_DoubleU u;
+    u.d = v;
+    stq_be_p(ptr, u.ll);
+}
+
+/* Legacy unaligned versions.  Note that we never had a complete set.  */
+
+static inline void cpu_to_le16wu(uint16_t *p, uint16_t v)
+{
+    stw_le_p(p, v);
+}
+
+static inline void cpu_to_le32wu(uint32_t *p, uint32_t v)
+{
+    stl_le_p(p, v);
+}
+
+static inline uint16_t le16_to_cpupu(const uint16_t *p)
+{
+    return lduw_le_p(p);
+}
+
+static inline uint32_t le32_to_cpupu(const uint32_t *p)
+{
+    return ldl_le_p(p);
+}
+
+static inline uint32_t be32_to_cpupu(const uint32_t *p)
+{
+    return ldl_be_p(p);
+}
+
+static inline void cpu_to_be16wu(uint16_t *p, uint16_t v)
+{
+    stw_be_p(p, v);
+}
+
+static inline void cpu_to_be32wu(uint32_t *p, uint32_t v)
+{
+    stl_be_p(p, v);
+}
+
+static inline void cpu_to_be64wu(uint64_t *p, uint64_t v)
+{
+    stq_be_p(p, v);
+}
+
+static inline void cpu_to_32wu(uint32_t *p, uint32_t v)
+{
+    stl_p(p, v);
+}
+
+static inline unsigned long leul_to_cpu(unsigned long v)
+{
+    /* In order to break an include loop between here and
+       qemu-common.h, don't rely on HOST_LONG_BITS.  */
+#if ULONG_MAX == UINT32_MAX
+    return le_bswap(v, 32);
+#elif ULONG_MAX == UINT64_MAX
+    return le_bswap(v, 64);
+#else
+# error Unknown sizeof long
+#endif
+}
+
+#undef le_bswap
+#undef be_bswap
+#undef le_bswaps
+#undef be_bswaps
+
+#endif /* BSWAP_H */
diff --git a/contrib/qemu/include/qemu/compiler.h b/contrib/qemu/include/qemu/compiler.h
new file mode 100644
index 000000000..155b35896
--- /dev/null
+++ b/contrib/qemu/include/qemu/compiler.h
@@ -0,0 +1,55 @@
+/* public domain */
+
+#ifndef COMPILER_H
+#define COMPILER_H
+
+#include "config-host.h"
+
+/*----------------------------------------------------------------------------
+| The macro QEMU_GNUC_PREREQ tests for minimum version of the GNU C compiler.
+| The code is a copy of SOFTFLOAT_GNUC_PREREQ, see softfloat-macros.h.
+*----------------------------------------------------------------------------*/
+#if defined(__GNUC__) && defined(__GNUC_MINOR__)
+# define QEMU_GNUC_PREREQ(maj, min) \
+         ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+#else
+# define QEMU_GNUC_PREREQ(maj, min) 0
+#endif
+
+#define QEMU_NORETURN __attribute__ ((__noreturn__))
+
+#if QEMU_GNUC_PREREQ(3, 4)
+#define QEMU_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+#else
+#define QEMU_WARN_UNUSED_RESULT
+#endif
+
+#if defined(_WIN32)
+# define QEMU_PACKED __attribute__((gcc_struct, packed))
+#else
+# define QEMU_PACKED __attribute__((packed))
+#endif
+
+#define cat(x,y) x ## y
+#define cat2(x,y) cat(x,y)
+#define QEMU_BUILD_BUG_ON(x) \
+    typedef char cat2(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused));
+
+#if defined __GNUC__
+# if !QEMU_GNUC_PREREQ(4, 4)
+   /* gcc versions before 4.4.x don't support gnu_printf, so use printf. */
+#  define GCC_FMT_ATTR(n, m) __attribute__((format(printf, n, m)))
+# else
+   /* Use gnu_printf when supported (qemu uses standard format strings). */
+#  define GCC_FMT_ATTR(n, m) __attribute__((format(gnu_printf, n, m)))
+#  if defined(_WIN32)
+    /* Map __printf__ to __gnu_printf__ because we want standard format strings
+     * even when MinGW or GLib include files use __printf__. */
+#   define __printf__ __gnu_printf__
+#  endif
+# endif
+#else
+#define GCC_FMT_ATTR(n, m)
+#endif
+
+#endif /* COMPILER_H */
diff --git a/contrib/qemu/include/qemu/error-report.h b/contrib/qemu/include/qemu/error-report.h
new file mode 100644
index 000000000..3b098a917
--- /dev/null
+++ b/contrib/qemu/include/qemu/error-report.h
@@ -0,0 +1,46 @@
+/*
+ * Error reporting
+ *
+ * Copyright (C) 2010 Red Hat Inc.
+ *
+ * Authors:
+ *  Markus Armbruster <armbru@redhat.com>,
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_ERROR_H
+#define QEMU_ERROR_H
+
+#include <stdarg.h>
+#include <stdbool.h>
+#include "qemu/compiler.h"
+
+typedef struct Location {
+    /* all members are private to qemu-error.c */
+    enum { LOC_NONE, LOC_CMDLINE, LOC_FILE } kind;
+    int num;
+    const void *ptr;
+    struct Location *prev;
+} Location;
+
+Location *loc_push_restore(Location *loc);
+Location *loc_push_none(Location *loc);
+Location *loc_pop(Location *loc);
+Location *loc_save(Location *loc);
+void loc_restore(Location *loc);
+void loc_set_none(void);
+void loc_set_cmdline(char **argv, int idx, int cnt);
+void loc_set_file(const char *fname, int lno);
+
+void error_vprintf(const char *fmt, va_list ap) GCC_FMT_ATTR(1, 0);
+void error_printf(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
+void error_printf_unless_qmp(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
+void error_print_loc(void);
+void error_set_progname(const char *argv0);
+void error_report(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
+const char *error_get_progname(void);
+extern bool enable_timestamp_msg;
+
+#endif
diff --git a/contrib/qemu/include/qemu/event_notifier.h b/contrib/qemu/include/qemu/event_notifier.h
new file mode 100644
index 000000000..88b57af7c
--- /dev/null
+++ b/contrib/qemu/include/qemu/event_notifier.h
@@ -0,0 +1,46 @@
+/*
+ * event notifier support
+ *
+ * Copyright Red Hat, Inc. 2010
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_EVENT_NOTIFIER_H
+#define QEMU_EVENT_NOTIFIER_H
+
+#include "qemu-common.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+struct EventNotifier {
+#ifdef _WIN32
+    HANDLE event;
+#else
+    int rfd;
+    int wfd;
+#endif
+};
+
+typedef void EventNotifierHandler(EventNotifier *);
+
+int event_notifier_init(EventNotifier *, int active);
+void event_notifier_cleanup(EventNotifier *);
+int event_notifier_set(EventNotifier *);
+int event_notifier_test_and_clear(EventNotifier *);
+int event_notifier_set_handler(EventNotifier *, EventNotifierHandler *);
+
+#ifdef CONFIG_POSIX
+void event_notifier_init_fd(EventNotifier *, int fd);
+int event_notifier_get_fd(EventNotifier *);
+#else
+HANDLE event_notifier_get_handle(EventNotifier *);
+#endif
+
+#endif
diff --git a/contrib/qemu/include/qemu/hbitmap.h b/contrib/qemu/include/qemu/hbitmap.h
new file mode 100644
index 000000000..550d7ce2c
--- /dev/null
+++ b/contrib/qemu/include/qemu/hbitmap.h
@@ -0,0 +1,209 @@
+/*
+ * Hierarchical Bitmap Data Type
+ *
+ * Copyright Red Hat, Inc., 2012
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#ifndef HBITMAP_H
+#define HBITMAP_H 1
+
+#include <limits.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include "bitops.h"
+#include "host-utils.h"
+
+typedef struct HBitmap HBitmap;
+typedef struct HBitmapIter HBitmapIter;
+
+#define BITS_PER_LEVEL         (BITS_PER_LONG == 32 ? 5 : 6)
+
+/* For 32-bit, the largest that fits in a 4 GiB address space.
+ * For 64-bit, the number of sectors in 1 PiB.  Good luck, in
+ * either case... :)
+ */
+#define HBITMAP_LOG_MAX_SIZE   (BITS_PER_LONG == 32 ? 34 : 41)
+
+/* We need to place a sentinel in level 0 to speed up iteration.  Thus,
+ * we do this instead of HBITMAP_LOG_MAX_SIZE / BITS_PER_LEVEL.  The
+ * difference is that it allocates an extra level when HBITMAP_LOG_MAX_SIZE
+ * is an exact multiple of BITS_PER_LEVEL.
+ */
+#define HBITMAP_LEVELS         ((HBITMAP_LOG_MAX_SIZE / BITS_PER_LEVEL) + 1)
+
+struct HBitmapIter {
+    const HBitmap *hb;
+
+    /* Copied from hb for access in the inline functions (hb is opaque).  */
+    int granularity;
+
+    /* Entry offset into the last-level array of longs.  */
+    size_t pos;
+
+    /* The currently-active path in the tree.  Each item of cur[i] stores
+     * the bits (i.e. the subtrees) yet to be processed under that node.
+     */
+    unsigned long cur[HBITMAP_LEVELS];
+};
+
+/**
+ * hbitmap_alloc:
+ * @size: Number of bits in the bitmap.
+ * @granularity: Granularity of the bitmap.  Aligned groups of 2^@granularity
+ * bits will be represented by a single bit.  Each operation on a
+ * range of bits first rounds the bits to determine which group they land
+ * in, and then affect the entire set; iteration will only visit the first
+ * bit of each group.
+ *
+ * Allocate a new HBitmap.
+ */
+HBitmap *hbitmap_alloc(uint64_t size, int granularity);
+
+/**
+ * hbitmap_empty:
+ * @hb: HBitmap to operate on.
+ *
+ * Return whether the bitmap is empty.
+ */
+bool hbitmap_empty(const HBitmap *hb);
+
+/**
+ * hbitmap_granularity:
+ * @hb: HBitmap to operate on.
+ *
+ * Return the granularity of the HBitmap.
+ */
+int hbitmap_granularity(const HBitmap *hb);
+
+/**
+ * hbitmap_count:
+ * @hb: HBitmap to operate on.
+ *
+ * Return the number of bits set in the HBitmap.
+ */
+uint64_t hbitmap_count(const HBitmap *hb);
+
+/**
+ * hbitmap_set:
+ * @hb: HBitmap to operate on.
+ * @start: First bit to set (0-based).
+ * @count: Number of bits to set.
+ *
+ * Set a consecutive range of bits in an HBitmap.
+ */
+void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t count);
+
+/**
+ * hbitmap_reset:
+ * @hb: HBitmap to operate on.
+ * @start: First bit to reset (0-based).
+ * @count: Number of bits to reset.
+ *
+ * Reset a consecutive range of bits in an HBitmap.
+ */
+void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count);
+
+/**
+ * hbitmap_get:
+ * @hb: HBitmap to operate on.
+ * @item: Bit to query (0-based).
+ *
+ * Return whether the @item-th bit in an HBitmap is set.
+ */
+bool hbitmap_get(const HBitmap *hb, uint64_t item);
+
+/**
+ * hbitmap_free:
+ * @hb: HBitmap to operate on.
+ *
+ * Free an HBitmap and all of its associated memory.
+ */
+void hbitmap_free(HBitmap *hb);
+
+/**
+ * hbitmap_iter_init:
+ * @hbi: HBitmapIter to initialize.
+ * @hb: HBitmap to iterate on.
+ * @first: First bit to visit (0-based, must be strictly less than the
+ * size of the bitmap).
+ *
+ * Set up @hbi to iterate on the HBitmap @hb.  hbitmap_iter_next will return
+ * the lowest-numbered bit that is set in @hb, starting at @first.
+ *
+ * Concurrent setting of bits is acceptable, and will at worst cause the
+ * iteration to miss some of those bits.  Resetting bits before the current
+ * position of the iterator is also okay.  However, concurrent resetting of
+ * bits can lead to unexpected behavior if the iterator has not yet reached
+ * those bits.
+ */
+void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first);
+
+/* hbitmap_iter_skip_words:
+ * @hbi: HBitmapIter to operate on.
+ *
+ * Internal function used by hbitmap_iter_next and hbitmap_iter_next_word.
+ */
+unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi);
+
+/**
+ * hbitmap_iter_next:
+ * @hbi: HBitmapIter to operate on.
+ *
+ * Return the next bit that is set in @hbi's associated HBitmap,
+ * or -1 if all remaining bits are zero.
+ */
+static inline int64_t hbitmap_iter_next(HBitmapIter *hbi)
+{
+    unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1];
+    int64_t item;
+
+    if (cur == 0) {
+        cur = hbitmap_iter_skip_words(hbi);
+        if (cur == 0) {
+            return -1;
+        }
+    }
+
+    /* The next call will resume work from the next bit.  */
+    hbi->cur[HBITMAP_LEVELS - 1] = cur & (cur - 1);
+    item = ((uint64_t)hbi->pos << BITS_PER_LEVEL) + ctzl(cur);
+
+    return item << hbi->granularity;
+}
+
+/**
+ * hbitmap_iter_next_word:
+ * @hbi: HBitmapIter to operate on.
+ * @p_cur: Location where to store the next non-zero word.
+ *
+ * Return the index of the next nonzero word that is set in @hbi's
+ * associated HBitmap, and set *p_cur to the content of that word
+ * (bits before the index that was passed to hbitmap_iter_init are
+ * trimmed on the first call).  Return -1, and set *p_cur to zero,
+ * if all remaining words are zero.
+ */
+static inline size_t hbitmap_iter_next_word(HBitmapIter *hbi, unsigned long *p_cur)
+{
+    unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1];
+
+    if (cur == 0) {
+        cur = hbitmap_iter_skip_words(hbi);
+        if (cur == 0) {
+            *p_cur = 0;
+            return -1;
+        }
+    }
+
+    /* The next call will resume work from the next word.  */
+    hbi->cur[HBITMAP_LEVELS - 1] = 0;
+    *p_cur = cur;
+    return hbi->pos;
+}
+
+
+#endif
diff --git a/contrib/qemu/include/qemu/host-utils.h b/contrib/qemu/include/qemu/host-utils.h
new file mode 100644
index 000000000..0f688c1c0
--- /dev/null
+++ b/contrib/qemu/include/qemu/host-utils.h
@@ -0,0 +1,322 @@
+/*
+ * Utility compute operations used by translated code.
+ *
+ * Copyright (c) 2007 Thiemo Seufer
+ * Copyright (c) 2007 Jocelyn Mayer
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef HOST_UTILS_H
+#define HOST_UTILS_H 1
+
+#include "qemu/compiler.h"   /* QEMU_GNUC_PREREQ */
+#include <limits.h>
+
+#ifdef CONFIG_INT128
+static inline void mulu64(uint64_t *plow, uint64_t *phigh,
+                          uint64_t a, uint64_t b)
+{
+    __uint128_t r = (__uint128_t)a * b;
+    *plow = r;
+    *phigh = r >> 64;
+}
+
+static inline void muls64(uint64_t *plow, uint64_t *phigh,
+                          int64_t a, int64_t b)
+{
+    __int128_t r = (__int128_t)a * b;
+    *plow = r;
+    *phigh = r >> 64;
+}
+#else
+void muls64(uint64_t *phigh, uint64_t *plow, int64_t a, int64_t b);
+void mulu64(uint64_t *phigh, uint64_t *plow, uint64_t a, uint64_t b);
+#endif
+
+/**
+ * clz32 - count leading zeros in a 32-bit value.
+ * @val: The value to search
+ *
+ * Returns 32 if the value is zero.  Note that the GCC builtin is
+ * undefined if the value is zero.
+ */
+static inline int clz32(uint32_t val)
+{
+#if QEMU_GNUC_PREREQ(3, 4)
+    return val ? __builtin_clz(val) : 32;
+#else
+    /* Binary search for the leading one bit.  */
+    int cnt = 0;
+
+    if (!(val & 0xFFFF0000U)) {
+        cnt += 16;
+        val <<= 16;
+    }
+    if (!(val & 0xFF000000U)) {
+        cnt += 8;
+        val <<= 8;
+    }
+    if (!(val & 0xF0000000U)) {
+        cnt += 4;
+        val <<= 4;
+    }
+    if (!(val & 0xC0000000U)) {
+        cnt += 2;
+        val <<= 2;
+    }
+    if (!(val & 0x80000000U)) {
+        cnt++;
+        val <<= 1;
+    }
+    if (!(val & 0x80000000U)) {
+        cnt++;
+    }
+    return cnt;
+#endif
+}
+
+/**
+ * clo32 - count leading ones in a 32-bit value.
+ * @val: The value to search
+ *
+ * Returns 32 if the value is -1.
+ */
+static inline int clo32(uint32_t val)
+{
+    return clz32(~val);
+}
+
+/**
+ * clz64 - count leading zeros in a 64-bit value.
+ * @val: The value to search
+ *
+ * Returns 64 if the value is zero.  Note that the GCC builtin is
+ * undefined if the value is zero.
+ */
+static inline int clz64(uint64_t val)
+{
+#if QEMU_GNUC_PREREQ(3, 4)
+    return val ? __builtin_clzll(val) : 64;
+#else
+    int cnt = 0;
+
+    if (!(val >> 32)) {
+        cnt += 32;
+    } else {
+        val >>= 32;
+    }
+
+    return cnt + clz32(val);
+#endif
+}
+
+/**
+ * clo64 - count leading ones in a 64-bit value.
+ * @val: The value to search
+ *
+ * Returns 64 if the value is -1.
+ */
+static inline int clo64(uint64_t val)
+{
+    return clz64(~val);
+}
+
+/**
+ * ctz32 - count trailing zeros in a 32-bit value.
+ * @val: The value to search
+ *
+ * Returns 32 if the value is zero.  Note that the GCC builtin is
+ * undefined if the value is zero.
+ */
+static inline int ctz32(uint32_t val)
+{
+#if QEMU_GNUC_PREREQ(3, 4)
+    return val ? __builtin_ctz(val) : 32;
+#else
+    /* Binary search for the trailing one bit.  */
+    int cnt;
+
+    cnt = 0;
+    if (!(val & 0x0000FFFFUL)) {
+        cnt += 16;
+        val >>= 16;
+    }
+    if (!(val & 0x000000FFUL)) {
+        cnt += 8;
+        val >>= 8;
+    }
+    if (!(val & 0x0000000FUL)) {
+        cnt += 4;
+        val >>= 4;
+    }
+    if (!(val & 0x00000003UL)) {
+        cnt += 2;
+        val >>= 2;
+    }
+    if (!(val & 0x00000001UL)) {
+        cnt++;
+        val >>= 1;
+    }
+    if (!(val & 0x00000001UL)) {
+        cnt++;
+    }
+
+    return cnt;
+#endif
+}
+
+/**
+ * cto32 - count trailing ones in a 32-bit value.
+ * @val: The value to search
+ *
+ * Returns 32 if the value is -1.
+ */
+static inline int cto32(uint32_t val)
+{
+    return ctz32(~val);
+}
+
+/**
+ * ctz64 - count trailing zeros in a 64-bit value.
+ * @val: The value to search
+ *
+ * Returns 64 if the value is zero.  Note that the GCC builtin is
+ * undefined if the value is zero.
+ */
+static inline int ctz64(uint64_t val)
+{
+#if QEMU_GNUC_PREREQ(3, 4)
+    return val ? __builtin_ctzll(val) : 64;
+#else
+    int cnt;
+
+    cnt = 0;
+    if (!((uint32_t)val)) {
+        cnt += 32;
+        val >>= 32;
+    }
+
+    return cnt + ctz32(val);
+#endif
+}
+
+/**
+ * ctz64 - count trailing ones in a 64-bit value.
+ * @val: The value to search
+ *
+ * Returns 64 if the value is -1.
+ */
+static inline int cto64(uint64_t val)
+{
+    return ctz64(~val);
+}
+
+/**
+ * ctpop8 - count the population of one bits in an 8-bit value.
+ * @val: The value to search
+ */
+static inline int ctpop8(uint8_t val)
+{
+#if QEMU_GNUC_PREREQ(3, 4)
+    return __builtin_popcount(val);
+#else
+    val = (val & 0x55) + ((val >> 1) & 0x55);
+    val = (val & 0x33) + ((val >> 2) & 0x33);
+    val = (val & 0x0f) + ((val >> 4) & 0x0f);
+
+    return val;
+#endif
+}
+
+/**
+ * ctpop16 - count the population of one bits in a 16-bit value.
+ * @val: The value to search
+ */
+static inline int ctpop16(uint16_t val)
+{
+#if QEMU_GNUC_PREREQ(3, 4)
+    return __builtin_popcount(val);
+#else
+    val = (val & 0x5555) + ((val >> 1) & 0x5555);
+    val = (val & 0x3333) + ((val >> 2) & 0x3333);
+    val = (val & 0x0f0f) + ((val >> 4) & 0x0f0f);
+    val = (val & 0x00ff) + ((val >> 8) & 0x00ff);
+
+    return val;
+#endif
+}
+
+/**
+ * ctpop32 - count the population of one bits in a 32-bit value.
+ * @val: The value to search
+ */
+static inline int ctpop32(uint32_t val)
+{
+#if QEMU_GNUC_PREREQ(3, 4)
+    return __builtin_popcount(val);
+#else
+    val = (val & 0x55555555) + ((val >>  1) & 0x55555555);
+    val = (val & 0x33333333) + ((val >>  2) & 0x33333333);
+    val = (val & 0x0f0f0f0f) + ((val >>  4) & 0x0f0f0f0f);
+    val = (val & 0x00ff00ff) + ((val >>  8) & 0x00ff00ff);
+    val = (val & 0x0000ffff) + ((val >> 16) & 0x0000ffff);
+
+    return val;
+#endif
+}
+
+/**
+ * ctpop64 - count the population of one bits in a 64-bit value.
+ * @val: The value to search
+ */
+static inline int ctpop64(uint64_t val)
+{
+#if QEMU_GNUC_PREREQ(3, 4)
+    return __builtin_popcountll(val);
+#else
+    val = (val & 0x5555555555555555ULL) + ((val >>  1) & 0x5555555555555555ULL);
+    val = (val & 0x3333333333333333ULL) + ((val >>  2) & 0x3333333333333333ULL);
+    val = (val & 0x0f0f0f0f0f0f0f0fULL) + ((val >>  4) & 0x0f0f0f0f0f0f0f0fULL);
+    val = (val & 0x00ff00ff00ff00ffULL) + ((val >>  8) & 0x00ff00ff00ff00ffULL);
+    val = (val & 0x0000ffff0000ffffULL) + ((val >> 16) & 0x0000ffff0000ffffULL);
+    val = (val & 0x00000000ffffffffULL) + ((val >> 32) & 0x00000000ffffffffULL);
+
+    return val;
+#endif
+}
+
+/* Host type specific sizes of these routines.  */
+
+#if ULONG_MAX == UINT32_MAX
+# define clzl   clz32
+# define ctzl   ctz32
+# define clol   clo32
+# define ctol   cto32
+# define ctpopl ctpop32
+#elif ULONG_MAX == UINT64_MAX
+# define clzl   clz64
+# define ctzl   ctz64
+# define clol   clo64
+# define ctol   cto64
+# define ctpopl ctpop64
+#else
+# error Unknown sizeof long
+#endif
+
+#endif
diff --git a/contrib/qemu/include/qemu/iov.h b/contrib/qemu/include/qemu/iov.h
new file mode 100644
index 000000000..68d25f29b
--- /dev/null
+++ b/contrib/qemu/include/qemu/iov.h
@@ -0,0 +1,115 @@
+/*
+ * Helpers for using (partial) iovecs.
+ *
+ * Copyright (C) 2010 Red Hat, Inc.
+ *
+ * Author(s):
+ *  Amit Shah <amit.shah@redhat.com>
+ *  Michael Tokarev <mjt@tls.msk.ru>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#ifndef IOV_H
+#define IOV_H
+
+#include "qemu-common.h"
+
+/**
+ * count and return data size, in bytes, of an iovec
+ * starting at `iov' of `iov_cnt' number of elements.
+ */
+size_t iov_size(const struct iovec *iov, const unsigned int iov_cnt);
+
+/**
+ * Copy from single continuous buffer to scatter-gather vector of buffers
+ * (iovec) and back like memcpy() between two continuous memory regions.
+ * Data in single continuous buffer starting at address `buf' and
+ * `bytes' bytes long will be copied to/from an iovec `iov' with
+ * `iov_cnt' number of elements, starting at byte position `offset'
+ * within the iovec.  If the iovec does not contain enough space,
+ * only part of data will be copied, up to the end of the iovec.
+ * Number of bytes actually copied will be returned, which is
+ *  min(bytes, iov_size(iov)-offset)
+ * `Offset' must point to the inside of iovec.
+ * It is okay to use very large value for `bytes' since we're
+ * limited by the size of the iovec anyway, provided that the
+ * buffer pointed to by buf has enough space.  One possible
+ * such "large" value is -1 (sinice size_t is unsigned),
+ * so specifying `-1' as `bytes' means 'up to the end of iovec'.
+ */
+size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
+                    size_t offset, const void *buf, size_t bytes);
+size_t iov_to_buf(const struct iovec *iov, const unsigned int iov_cnt,
+                  size_t offset, void *buf, size_t bytes);
+
+/**
+ * Set data bytes pointed out by iovec `iov' of size `iov_cnt' elements,
+ * starting at byte offset `start', to value `fillc', repeating it
+ * `bytes' number of times.  `Offset' must point to the inside of iovec.
+ * If `bytes' is large enough, only last bytes portion of iovec,
+ * up to the end of it, will be filled with the specified value.
+ * Function return actual number of bytes processed, which is
+ * min(size, iov_size(iov) - offset).
+ * Again, it is okay to use large value for `bytes' to mean "up to the end".
+ */
+size_t iov_memset(const struct iovec *iov, const unsigned int iov_cnt,
+                  size_t offset, int fillc, size_t bytes);
+
+/*
+ * Send/recv data from/to iovec buffers directly
+ *
+ * `offset' bytes in the beginning of iovec buffer are skipped and
+ * next `bytes' bytes are used, which must be within data of iovec.
+ *
+ *   r = iov_send_recv(sockfd, iov, iovcnt, offset, bytes, true);
+ *
+ * is logically equivalent to
+ *
+ *   char *buf = malloc(bytes);
+ *   iov_to_buf(iov, iovcnt, offset, buf, bytes);
+ *   r = send(sockfd, buf, bytes, 0);
+ *   free(buf);
+ *
+ * For iov_send_recv() _whole_ area being sent or received
+ * should be within the iovec, not only beginning of it.
+ */
+ssize_t iov_send_recv(int sockfd, struct iovec *iov, unsigned iov_cnt,
+                      size_t offset, size_t bytes, bool do_send);
+#define iov_recv(sockfd, iov, iov_cnt, offset, bytes) \
+  iov_send_recv(sockfd, iov, iov_cnt, offset, bytes, false)
+#define iov_send(sockfd, iov, iov_cnt, offset, bytes) \
+  iov_send_recv(sockfd, iov, iov_cnt, offset, bytes, true)
+
+/**
+ * Produce a text hexdump of iovec `iov' with `iov_cnt' number of elements
+ * in file `fp', prefixing each line with `prefix' and processing not more
+ * than `limit' data bytes.
+ */
+void iov_hexdump(const struct iovec *iov, const unsigned int iov_cnt,
+                 FILE *fp, const char *prefix, size_t limit);
+
+/*
+ * Partial copy of vector from iov to dst_iov (data is not copied).
+ * dst_iov overlaps iov at a specified offset.
+ * size of dst_iov is at most bytes. dst vector count is returned.
+ */
+unsigned iov_copy(struct iovec *dst_iov, unsigned int dst_iov_cnt,
+                 const struct iovec *iov, unsigned int iov_cnt,
+                 size_t offset, size_t bytes);
+
+/*
+ * Remove a given number of bytes from the front or back of a vector.
+ * This may update iov and/or iov_cnt to exclude iovec elements that are
+ * no longer required.
+ *
+ * The number of bytes actually discarded is returned.  This number may be
+ * smaller than requested if the vector is too small.
+ */
+size_t iov_discard_front(struct iovec **iov, unsigned int *iov_cnt,
+                         size_t bytes);
+size_t iov_discard_back(struct iovec *iov, unsigned int *iov_cnt,
+                        size_t bytes);
+
+#endif
diff --git a/contrib/qemu/include/qemu/main-loop.h b/contrib/qemu/include/qemu/main-loop.h
new file mode 100644
index 000000000..6f0200a7a
--- /dev/null
+++ b/contrib/qemu/include/qemu/main-loop.h
@@ -0,0 +1,311 @@
+/*
+ * QEMU System Emulator
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef QEMU_MAIN_LOOP_H
+#define QEMU_MAIN_LOOP_H 1
+
+#include "block/aio.h"
+
+#define SIG_IPI SIGUSR1
+
+/**
+ * qemu_init_main_loop: Set up the process so that it can run the main loop.
+ *
+ * This includes setting up signal handlers.  It should be called before
+ * any other threads are created.  In addition, threads other than the
+ * main one should block signals that are trapped by the main loop.
+ * For simplicity, you can consider these signals to be safe: SIGUSR1,
+ * SIGUSR2, thread signals (SIGFPE, SIGILL, SIGSEGV, SIGBUS) and real-time
+ * signals if available.  Remember that Windows in practice does not have
+ * signals, though.
+ *
+ * In the case of QEMU tools, this will also start/initialize timers.
+ */
+int qemu_init_main_loop(void);
+
+/**
+ * main_loop_wait: Run one iteration of the main loop.
+ *
+ * If @nonblocking is true, poll for events, otherwise suspend until
+ * one actually occurs.  The main loop usually consists of a loop that
+ * repeatedly calls main_loop_wait(false).
+ *
+ * Main loop services include file descriptor callbacks, bottom halves
+ * and timers (defined in qemu-timer.h).  Bottom halves are similar to timers
+ * that execute immediately, but have a lower overhead and scheduling them
+ * is wait-free, thread-safe and signal-safe.
+ *
+ * It is sometimes useful to put a whole program in a coroutine.  In this
+ * case, the coroutine actually should be started from within the main loop,
+ * so that the main loop can run whenever the coroutine yields.  To do this,
+ * you can use a bottom half to enter the coroutine as soon as the main loop
+ * starts:
+ *
+ *     void enter_co_bh(void *opaque) {
+ *         QEMUCoroutine *co = opaque;
+ *         qemu_coroutine_enter(co, NULL);
+ *     }
+ *
+ *     ...
+ *     QEMUCoroutine *co = qemu_coroutine_create(coroutine_entry);
+ *     QEMUBH *start_bh = qemu_bh_new(enter_co_bh, co);
+ *     qemu_bh_schedule(start_bh);
+ *     while (...) {
+ *         main_loop_wait(false);
+ *     }
+ *
+ * (In the future we may provide a wrapper for this).
+ *
+ * @nonblocking: Whether the caller should block until an event occurs.
+ */
+int main_loop_wait(int nonblocking);
+
+/**
+ * qemu_get_aio_context: Return the main loop's AioContext
+ */
+AioContext *qemu_get_aio_context(void);
+
+/**
+ * qemu_notify_event: Force processing of pending events.
+ *
+ * Similar to signaling a condition variable, qemu_notify_event forces
+ * main_loop_wait to look at pending events and exit.  The caller of
+ * main_loop_wait will usually call it again very soon, so qemu_notify_event
+ * also has the side effect of recalculating the sets of file descriptors
+ * that the main loop waits for.
+ *
+ * Calling qemu_notify_event is rarely necessary, because main loop
+ * services (bottom halves and timers) call it themselves.  One notable
+ * exception occurs when using qemu_set_fd_handler2 (see below).
+ */
+void qemu_notify_event(void);
+
+#ifdef _WIN32
+/* return TRUE if no sleep should be done afterwards */
+typedef int PollingFunc(void *opaque);
+
+/**
+ * qemu_add_polling_cb: Register a Windows-specific polling callback
+ *
+ * Currently, under Windows some events are polled rather than waited for.
+ * Polling callbacks do not ensure that @func is called timely, because
+ * the main loop might wait for an arbitrarily long time.  If possible,
+ * you should instead create a separate thread that does a blocking poll
+ * and set a Win32 event object.  The event can then be passed to
+ * qemu_add_wait_object.
+ *
+ * Polling callbacks really have nothing Windows specific in them, but
+ * as they are a hack and are currently not necessary under POSIX systems,
+ * they are only available when QEMU is running under Windows.
+ *
+ * @func: The function that does the polling, and returns 1 to force
+ * immediate completion of main_loop_wait.
+ * @opaque: A pointer-size value that is passed to @func.
+ */
+int qemu_add_polling_cb(PollingFunc *func, void *opaque);
+
+/**
+ * qemu_del_polling_cb: Unregister a Windows-specific polling callback
+ *
+ * This function removes a callback that was registered with
+ * qemu_add_polling_cb.
+ *
+ * @func: The function that was passed to qemu_add_polling_cb.
+ * @opaque: A pointer-size value that was passed to qemu_add_polling_cb.
+ */
+void qemu_del_polling_cb(PollingFunc *func, void *opaque);
+
+/* Wait objects handling */
+typedef void WaitObjectFunc(void *opaque);
+
+/**
+ * qemu_add_wait_object: Register a callback for a Windows handle
+ *
+ * Under Windows, the iohandler mechanism can only be used with sockets.
+ * QEMU must use the WaitForMultipleObjects API to wait on other handles.
+ * This function registers a #HANDLE with QEMU, so that it will be included
+ * in the main loop's calls to WaitForMultipleObjects.  When the handle
+ * is in a signaled state, QEMU will call @func.
+ *
+ * @handle: The Windows handle to be observed.
+ * @func: A function to be called when @handle is in a signaled state.
+ * @opaque: A pointer-size value that is passed to @func.
+ */
+int qemu_add_wait_object(HANDLE handle, WaitObjectFunc *func, void *opaque);
+
+/**
+ * qemu_del_wait_object: Unregister a callback for a Windows handle
+ *
+ * This function removes a callback that was registered with
+ * qemu_add_wait_object.
+ *
+ * @func: The function that was passed to qemu_add_wait_object.
+ * @opaque: A pointer-size value that was passed to qemu_add_wait_object.
+ */
+void qemu_del_wait_object(HANDLE handle, WaitObjectFunc *func, void *opaque);
+#endif
+
+/* async I/O support */
+
+typedef void IOReadHandler(void *opaque, const uint8_t *buf, int size);
+typedef int IOCanReadHandler(void *opaque);
+
+/**
+ * qemu_set_fd_handler2: Register a file descriptor with the main loop
+ *
+ * This function tells the main loop to wake up whenever one of the
+ * following conditions is true:
+ *
+ * 1) if @fd_write is not %NULL, when the file descriptor is writable;
+ *
+ * 2) if @fd_read is not %NULL, when the file descriptor is readable.
+ *
+ * @fd_read_poll can be used to disable the @fd_read callback temporarily.
+ * This is useful to avoid calling qemu_set_fd_handler2 every time the
+ * client becomes interested in reading (or dually, stops being interested).
+ * A typical example is when @fd is a listening socket and you want to bound
+ * the number of active clients.  Remember to call qemu_notify_event whenever
+ * the condition may change from %false to %true.
+ *
+ * The callbacks that are set up by qemu_set_fd_handler2 are level-triggered.
+ * If @fd_read does not read from @fd, or @fd_write does not write to @fd
+ * until its buffers are full, they will be called again on the next
+ * iteration.
+ *
+ * @fd: The file descriptor to be observed.  Under Windows it must be
+ * a #SOCKET.
+ *
+ * @fd_read_poll: A function that returns 1 if the @fd_read callback
+ * should be fired.  If the function returns 0, the main loop will not
+ * end its iteration even if @fd becomes readable.
+ *
+ * @fd_read: A level-triggered callback that is fired if @fd is readable
+ * at the beginning of a main loop iteration, or if it becomes readable
+ * during one.
+ *
+ * @fd_write: A level-triggered callback that is fired when @fd is writable
+ * at the beginning of a main loop iteration, or if it becomes writable
+ * during one.
+ *
+ * @opaque: A pointer-sized value that is passed to @fd_read_poll,
+ * @fd_read and @fd_write.
+ */
+int qemu_set_fd_handler2(int fd,
+                         IOCanReadHandler *fd_read_poll,
+                         IOHandler *fd_read,
+                         IOHandler *fd_write,
+                         void *opaque);
+
+/**
+ * qemu_set_fd_handler: Register a file descriptor with the main loop
+ *
+ * This function tells the main loop to wake up whenever one of the
+ * following conditions is true:
+ *
+ * 1) if @fd_write is not %NULL, when the file descriptor is writable;
+ *
+ * 2) if @fd_read is not %NULL, when the file descriptor is readable.
+ *
+ * The callbacks that are set up by qemu_set_fd_handler are level-triggered.
+ * If @fd_read does not read from @fd, or @fd_write does not write to @fd
+ * until its buffers are full, they will be called again on the next
+ * iteration.
+ *
+ * @fd: The file descriptor to be observed.  Under Windows it must be
+ * a #SOCKET.
+ *
+ * @fd_read: A level-triggered callback that is fired if @fd is readable
+ * at the beginning of a main loop iteration, or if it becomes readable
+ * during one.
+ *
+ * @fd_write: A level-triggered callback that is fired when @fd is writable
+ * at the beginning of a main loop iteration, or if it becomes writable
+ * during one.
+ *
+ * @opaque: A pointer-sized value that is passed to @fd_read and @fd_write.
+ */
+int qemu_set_fd_handler(int fd,
+                        IOHandler *fd_read,
+                        IOHandler *fd_write,
+                        void *opaque);
+
+#ifdef CONFIG_POSIX
+/**
+ * qemu_add_child_watch: Register a child process for reaping.
+ *
+ * Under POSIX systems, a parent process must read the exit status of
+ * its child processes using waitpid, or the operating system will not
+ * free some of the resources attached to that process.
+ *
+ * This function directs the QEMU main loop to observe a child process
+ * and call waitpid as soon as it exits; the watch is then removed
+ * automatically.  It is useful whenever QEMU forks a child process
+ * but will find out about its termination by other means such as a
+ * "broken pipe".
+ *
+ * @pid: The pid that QEMU should observe.
+ */
+int qemu_add_child_watch(pid_t pid);
+#endif
+
+/**
+ * qemu_mutex_lock_iothread: Lock the main loop mutex.
+ *
+ * This function locks the main loop mutex.  The mutex is taken by
+ * qemu_init_main_loop and always taken except while waiting on
+ * external events (such as with select).  The mutex should be taken
+ * by threads other than the main loop thread when calling
+ * qemu_bh_new(), qemu_set_fd_handler() and basically all other
+ * functions documented in this file.
+ *
+ * NOTE: tools currently are single-threaded and qemu_mutex_lock_iothread
+ * is a no-op there.
+ */
+void qemu_mutex_lock_iothread(void);
+
+/**
+ * qemu_mutex_unlock_iothread: Unlock the main loop mutex.
+ *
+ * This function unlocks the main loop mutex.  The mutex is taken by
+ * qemu_init_main_loop and always taken except while waiting on
+ * external events (such as with select).  The mutex should be unlocked
+ * as soon as possible by threads other than the main loop thread,
+ * because it prevents the main loop from processing callbacks,
+ * including timers and bottom halves.
+ *
+ * NOTE: tools currently are single-threaded and qemu_mutex_unlock_iothread
+ * is a no-op there.
+ */
+void qemu_mutex_unlock_iothread(void);
+
+/* internal interfaces */
+
+void qemu_fd_register(int fd);
+void qemu_iohandler_fill(GArray *pollfds);
+void qemu_iohandler_poll(GArray *pollfds, int rc);
+
+QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque);
+void qemu_bh_schedule_idle(QEMUBH *bh);
+
+#endif
diff --git a/contrib/qemu/include/qemu/module.h b/contrib/qemu/include/qemu/module.h
new file mode 100644
index 000000000..c4ccd5716
--- /dev/null
+++ b/contrib/qemu/include/qemu/module.h
@@ -0,0 +1,40 @@
+/*
+ * QEMU Module Infrastructure
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_MODULE_H
+#define QEMU_MODULE_H
+
+/* This should not be used directly.  Use block_init etc. instead.  */
+#define module_init(function, type)                                         \
+static void __attribute__((constructor)) do_qemu_init_ ## function(void) {  \
+    register_module_init(function, type);                                   \
+}
+
+typedef enum {
+    MODULE_INIT_BLOCK,
+    MODULE_INIT_MACHINE,
+    MODULE_INIT_QAPI,
+    MODULE_INIT_QOM,
+    MODULE_INIT_MAX
+} module_init_type;
+
+#define block_init(function) module_init(function, MODULE_INIT_BLOCK)
+#define machine_init(function) module_init(function, MODULE_INIT_MACHINE)
+#define qapi_init(function) module_init(function, MODULE_INIT_QAPI)
+#define type_init(function) module_init(function, MODULE_INIT_QOM)
+
+void register_module_init(void (*fn)(void), module_init_type type);
+
+void module_call_init(module_init_type type);
+
+#endif
diff --git a/contrib/qemu/include/qemu/notify.h b/contrib/qemu/include/qemu/notify.h
new file mode 100644
index 000000000..a3d73e4bc
--- /dev/null
+++ b/contrib/qemu/include/qemu/notify.h
@@ -0,0 +1,72 @@
+/*
+ * Notifier lists
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_NOTIFY_H
+#define QEMU_NOTIFY_H
+
+#include "qemu/queue.h"
+
+typedef struct Notifier Notifier;
+
+struct Notifier
+{
+    void (*notify)(Notifier *notifier, void *data);
+    QLIST_ENTRY(Notifier) node;
+};
+
+typedef struct NotifierList
+{
+    QLIST_HEAD(, Notifier) notifiers;
+} NotifierList;
+
+#define NOTIFIER_LIST_INITIALIZER(head) \
+    { QLIST_HEAD_INITIALIZER((head).notifiers) }
+
+void notifier_list_init(NotifierList *list);
+
+void notifier_list_add(NotifierList *list, Notifier *notifier);
+
+void notifier_remove(Notifier *notifier);
+
+void notifier_list_notify(NotifierList *list, void *data);
+
+/* Same as Notifier but allows .notify() to return errors */
+typedef struct NotifierWithReturn NotifierWithReturn;
+
+struct NotifierWithReturn {
+    /**
+     * Return 0 on success (next notifier will be invoked), otherwise
+     * notifier_with_return_list_notify() will stop and return the value.
+     */
+    int (*notify)(NotifierWithReturn *notifier, void *data);
+    QLIST_ENTRY(NotifierWithReturn) node;
+};
+
+typedef struct NotifierWithReturnList {
+    QLIST_HEAD(, NotifierWithReturn) notifiers;
+} NotifierWithReturnList;
+
+#define NOTIFIER_WITH_RETURN_LIST_INITIALIZER(head) \
+    { QLIST_HEAD_INITIALIZER((head).notifiers) }
+
+void notifier_with_return_list_init(NotifierWithReturnList *list);
+
+void notifier_with_return_list_add(NotifierWithReturnList *list,
+                                   NotifierWithReturn *notifier);
+
+void notifier_with_return_remove(NotifierWithReturn *notifier);
+
+int notifier_with_return_list_notify(NotifierWithReturnList *list,
+                                     void *data);
+
+#endif
diff --git a/contrib/qemu/include/qemu/option.h b/contrib/qemu/include/qemu/option.h
new file mode 100644
index 000000000..a83c70032
--- /dev/null
+++ b/contrib/qemu/include/qemu/option.h
@@ -0,0 +1,157 @@
+/*
+ * Commandline option parsing functions
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2009 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef QEMU_OPTIONS_H
+#define QEMU_OPTIONS_H
+
+#include <stdint.h>
+#include "qemu/queue.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qdict.h"
+
+enum QEMUOptionParType {
+    OPT_FLAG,
+    OPT_NUMBER,
+    OPT_SIZE,
+    OPT_STRING,
+};
+
+typedef struct QEMUOptionParameter {
+    const char *name;
+    enum QEMUOptionParType type;
+    union {
+        uint64_t n;
+        char* s;
+    } value;
+    const char *help;
+} QEMUOptionParameter;
+
+
+const char *get_opt_name(char *buf, int buf_size, const char *p, char delim);
+const char *get_opt_value(char *buf, int buf_size, const char *p);
+int get_next_param_value(char *buf, int buf_size,
+                         const char *tag, const char **pstr);
+int get_param_value(char *buf, int buf_size,
+                    const char *tag, const char *str);
+
+
+/*
+ * The following functions take a parameter list as input. This is a pointer to
+ * the first element of a QEMUOptionParameter array which is terminated by an
+ * entry with entry->name == NULL.
+ */
+
+QEMUOptionParameter *get_option_parameter(QEMUOptionParameter *list,
+    const char *name);
+int set_option_parameter(QEMUOptionParameter *list, const char *name,
+    const char *value);
+int set_option_parameter_int(QEMUOptionParameter *list, const char *name,
+    uint64_t value);
+QEMUOptionParameter *append_option_parameters(QEMUOptionParameter *dest,
+    QEMUOptionParameter *list);
+QEMUOptionParameter *parse_option_parameters(const char *param,
+    QEMUOptionParameter *list, QEMUOptionParameter *dest);
+void free_option_parameters(QEMUOptionParameter *list);
+void print_option_parameters(QEMUOptionParameter *list);
+void print_option_help(QEMUOptionParameter *list);
+
+/* ------------------------------------------------------------------ */
+
+typedef struct QemuOpt QemuOpt;
+typedef struct QemuOpts QemuOpts;
+typedef struct QemuOptsList QemuOptsList;
+
+enum QemuOptType {
+    QEMU_OPT_STRING = 0,  /* no parsing (use string as-is)                        */
+    QEMU_OPT_BOOL,        /* on/off                                               */
+    QEMU_OPT_NUMBER,      /* simple number                                        */
+    QEMU_OPT_SIZE,        /* size, accepts (K)ilo, (M)ega, (G)iga, (T)era postfix */
+};
+
+typedef struct QemuOptDesc {
+    const char *name;
+    enum QemuOptType type;
+    const char *help;
+} QemuOptDesc;
+
+struct QemuOptsList {
+    const char *name;
+    const char *implied_opt_name;
+    bool merge_lists;  /* Merge multiple uses of option into a single list? */
+    QTAILQ_HEAD(, QemuOpts) head;
+    QemuOptDesc desc[];
+};
+
+const char *qemu_opt_get(QemuOpts *opts, const char *name);
+/**
+ * qemu_opt_has_help_opt:
+ * @opts: options to search for a help request
+ *
+ * Check whether the options specified by @opts include one of the
+ * standard strings which indicate that the user is asking for a
+ * list of the valid values for a command line option (as defined
+ * by is_help_option()).
+ *
+ * Returns: true if @opts includes 'help' or equivalent.
+ */
+bool qemu_opt_has_help_opt(QemuOpts *opts);
+bool qemu_opt_get_bool(QemuOpts *opts, const char *name, bool defval);
+uint64_t qemu_opt_get_number(QemuOpts *opts, const char *name, uint64_t defval);
+uint64_t qemu_opt_get_size(QemuOpts *opts, const char *name, uint64_t defval);
+int qemu_opt_set(QemuOpts *opts, const char *name, const char *value);
+void qemu_opt_set_err(QemuOpts *opts, const char *name, const char *value,
+                      Error **errp);
+int qemu_opt_set_bool(QemuOpts *opts, const char *name, bool val);
+int qemu_opt_set_number(QemuOpts *opts, const char *name, int64_t val);
+typedef int (*qemu_opt_loopfunc)(const char *name, const char *value, void *opaque);
+int qemu_opt_foreach(QemuOpts *opts, qemu_opt_loopfunc func, void *opaque,
+                     int abort_on_failure);
+
+QemuOpts *qemu_opts_find(QemuOptsList *list, const char *id);
+QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
+                           int fail_if_exists, Error **errp);
+QemuOpts *qemu_opts_create_nofail(QemuOptsList *list);
+void qemu_opts_reset(QemuOptsList *list);
+void qemu_opts_loc_restore(QemuOpts *opts);
+int qemu_opts_set(QemuOptsList *list, const char *id,
+                  const char *name, const char *value);
+const char *qemu_opts_id(QemuOpts *opts);
+void qemu_opts_del(QemuOpts *opts);
+void qemu_opts_validate(QemuOpts *opts, const QemuOptDesc *desc, Error **errp);
+int qemu_opts_do_parse(QemuOpts *opts, const char *params, const char *firstname);
+QemuOpts *qemu_opts_parse(QemuOptsList *list, const char *params, int permit_abbrev);
+void qemu_opts_set_defaults(QemuOptsList *list, const char *params,
+                            int permit_abbrev);
+QemuOpts *qemu_opts_from_qdict(QemuOptsList *list, const QDict *qdict,
+                               Error **errp);
+QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict);
+void qemu_opts_absorb_qdict(QemuOpts *opts, QDict *qdict, Error **errp);
+
+typedef int (*qemu_opts_loopfunc)(QemuOpts *opts, void *opaque);
+int qemu_opts_print(QemuOpts *opts, void *dummy);
+int qemu_opts_foreach(QemuOptsList *list, qemu_opts_loopfunc func, void *opaque,
+                      int abort_on_failure);
+
+#endif
diff --git a/contrib/qemu/include/qemu/option_int.h b/contrib/qemu/include/qemu/option_int.h
new file mode 100644
index 000000000..8212fa4a4
--- /dev/null
+++ b/contrib/qemu/include/qemu/option_int.h
@@ -0,0 +1,54 @@
+/*
+ * Commandline option parsing functions
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2009 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef QEMU_OPTIONS_INTERNAL_H
+#define QEMU_OPTIONS_INTERNAL_H
+
+#include "qemu/option.h"
+#include "qemu/error-report.h"
+
+struct QemuOpt {
+    const char   *name;
+    const char   *str;
+
+    const QemuOptDesc *desc;
+    union {
+        bool boolean;
+        uint64_t uint;
+    } value;
+
+    QemuOpts     *opts;
+    QTAILQ_ENTRY(QemuOpt) next;
+};
+
+struct QemuOpts {
+    char *id;
+    QemuOptsList *list;
+    Location loc;
+    QTAILQ_HEAD(QemuOptHead, QemuOpt) head;
+    QTAILQ_ENTRY(QemuOpts) next;
+};
+
+#endif
diff --git a/contrib/qemu/include/qemu/osdep.h b/contrib/qemu/include/qemu/osdep.h
new file mode 100644
index 000000000..26136f16e
--- /dev/null
+++ b/contrib/qemu/include/qemu/osdep.h
@@ -0,0 +1,218 @@
+#ifndef QEMU_OSDEP_H
+#define QEMU_OSDEP_H
+
+#include "config-host.h"
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include <sys/types.h>
+#ifdef __OpenBSD__
+#include <sys/signal.h>
+#endif
+
+#ifndef _WIN32
+#include <sys/wait.h>
+#else
+#define WIFEXITED(x)   1
+#define WEXITSTATUS(x) (x)
+#endif
+
+#include <sys/time.h>
+
+#if defined(CONFIG_SOLARIS) && CONFIG_SOLARIS_VERSION < 10
+/* [u]int_fast*_t not in <sys/int_types.h> */
+typedef unsigned char           uint_fast8_t;
+typedef unsigned int            uint_fast16_t;
+typedef signed int              int_fast16_t;
+#endif
+
+#ifndef glue
+#define xglue(x, y) x ## y
+#define glue(x, y) xglue(x, y)
+#define stringify(s)	tostring(s)
+#define tostring(s)	#s
+#endif
+
+#ifndef likely
+#if __GNUC__ < 3
+#define __builtin_expect(x, n) (x)
+#endif
+
+#define likely(x)   __builtin_expect(!!(x), 1)
+#define unlikely(x)   __builtin_expect(!!(x), 0)
+#endif
+
+#ifndef container_of
+#define container_of(ptr, type, member) ({                      \
+        const typeof(((type *) 0)->member) *__mptr = (ptr);     \
+        (type *) ((char *) __mptr - offsetof(type, member));})
+#endif
+
+/* Convert from a base type to a parent type, with compile time checking.  */
+#ifdef __GNUC__
+#define DO_UPCAST(type, field, dev) ( __extension__ ( { \
+    char __attribute__((unused)) offset_must_be_zero[ \
+        -offsetof(type, field)]; \
+    container_of(dev, type, field);}))
+#else
+#define DO_UPCAST(type, field, dev) container_of(dev, type, field)
+#endif
+
+#define typeof_field(type, field) typeof(((type *)0)->field)
+#define type_check(t1,t2) ((t1*)0 - (t2*)0)
+
+#ifndef MIN
+#define MIN(a, b) (((a) < (b)) ? (a) : (b))
+#endif
+#ifndef MAX
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
+#endif
+
+#ifndef ROUND_UP
+#define ROUND_UP(n,d) (((n) + (d) - 1) & -(d))
+#endif
+
+#ifndef DIV_ROUND_UP
+#define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
+#endif
+
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+#endif
+
+#ifndef always_inline
+#if !((__GNUC__ < 3) || defined(__APPLE__))
+#ifdef __OPTIMIZE__
+#undef inline
+#define inline __attribute__ (( always_inline )) __inline__
+#endif
+#endif
+#else
+#undef inline
+#define inline always_inline
+#endif
+
+#define qemu_printf printf
+
+int qemu_daemon(int nochdir, int noclose);
+void *qemu_memalign(size_t alignment, size_t size);
+void *qemu_anon_ram_alloc(size_t size);
+void qemu_vfree(void *ptr);
+void qemu_anon_ram_free(void *ptr, size_t size);
+
+#define QEMU_MADV_INVALID -1
+
+#if defined(CONFIG_MADVISE)
+
+#define QEMU_MADV_WILLNEED  MADV_WILLNEED
+#define QEMU_MADV_DONTNEED  MADV_DONTNEED
+#ifdef MADV_DONTFORK
+#define QEMU_MADV_DONTFORK  MADV_DONTFORK
+#else
+#define QEMU_MADV_DONTFORK  QEMU_MADV_INVALID
+#endif
+#ifdef MADV_MERGEABLE
+#define QEMU_MADV_MERGEABLE MADV_MERGEABLE
+#else
+#define QEMU_MADV_MERGEABLE QEMU_MADV_INVALID
+#endif
+#ifdef MADV_DONTDUMP
+#define QEMU_MADV_DONTDUMP MADV_DONTDUMP
+#else
+#define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
+#endif
+#ifdef MADV_HUGEPAGE
+#define QEMU_MADV_HUGEPAGE MADV_HUGEPAGE
+#else
+#define QEMU_MADV_HUGEPAGE QEMU_MADV_INVALID
+#endif
+
+#elif defined(CONFIG_POSIX_MADVISE)
+
+#define QEMU_MADV_WILLNEED  POSIX_MADV_WILLNEED
+#define QEMU_MADV_DONTNEED  POSIX_MADV_DONTNEED
+#define QEMU_MADV_DONTFORK  QEMU_MADV_INVALID
+#define QEMU_MADV_MERGEABLE QEMU_MADV_INVALID
+#define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
+#define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
+
+#else /* no-op */
+
+#define QEMU_MADV_WILLNEED  QEMU_MADV_INVALID
+#define QEMU_MADV_DONTNEED  QEMU_MADV_INVALID
+#define QEMU_MADV_DONTFORK  QEMU_MADV_INVALID
+#define QEMU_MADV_MERGEABLE QEMU_MADV_INVALID
+#define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
+#define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
+
+#endif
+
+int qemu_madvise(void *addr, size_t len, int advice);
+
+int qemu_open(const char *name, int flags, ...);
+int qemu_close(int fd);
+
+#if defined(__HAIKU__) && defined(__i386__)
+#define FMT_pid "%ld"
+#elif defined(WIN64)
+#define FMT_pid "%" PRId64
+#else
+#define FMT_pid "%d"
+#endif
+
+int qemu_create_pidfile(const char *filename);
+int qemu_get_thread_id(void);
+
+#ifndef CONFIG_IOVEC
+struct iovec {
+    void *iov_base;
+    size_t iov_len;
+};
+/*
+ * Use the same value as Linux for now.
+ */
+#define IOV_MAX 1024
+
+ssize_t readv(int fd, const struct iovec *iov, int iov_cnt);
+ssize_t writev(int fd, const struct iovec *iov, int iov_cnt);
+#else
+#include <sys/uio.h>
+#endif
+
+#ifdef _WIN32
+static inline void qemu_timersub(const struct timeval *val1,
+                                 const struct timeval *val2,
+                                 struct timeval *res)
+{
+    res->tv_sec = val1->tv_sec - val2->tv_sec;
+    if (val1->tv_usec < val2->tv_usec) {
+        res->tv_sec--;
+        res->tv_usec = val1->tv_usec - val2->tv_usec + 1000 * 1000;
+    } else {
+        res->tv_usec = val1->tv_usec - val2->tv_usec;
+    }
+}
+#else
+#define qemu_timersub timersub
+#endif
+
+void qemu_set_cloexec(int fd);
+
+void qemu_set_version(const char *);
+const char *qemu_get_version(void);
+
+void fips_set_state(bool requested);
+bool fips_get_state(void);
+
+/* Return a dynamically allocated pathname denoting a file or directory that is
+ * appropriate for storing local state.
+ *
+ * @relative_pathname need not start with a directory separator; one will be
+ * added automatically.
+ *
+ * The caller is responsible for releasing the value returned with g_free()
+ * after use.
+ */
+char *qemu_get_local_state_pathname(const char *relative_pathname);
+
+#endif
diff --git a/contrib/qemu/include/qemu/queue.h b/contrib/qemu/include/qemu/queue.h
new file mode 100644
index 000000000..d433b9017
--- /dev/null
+++ b/contrib/qemu/include/qemu/queue.h
@@ -0,0 +1,414 @@
+/*      $NetBSD: queue.h,v 1.52 2009/04/20 09:56:08 mschuett Exp $ */
+
+/*
+ * QEMU version: Copy from netbsd, removed debug code, removed some of
+ * the implementations.  Left in singly-linked lists, lists, simple
+ * queues, and tail queues.
+ */
+
+/*
+ * Copyright (c) 1991, 1993
+ *      The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *      @(#)queue.h     8.5 (Berkeley) 8/20/94
+ */
+
+#ifndef QEMU_SYS_QUEUE_H_
+#define QEMU_SYS_QUEUE_H_
+
+/*
+ * This file defines four types of data structures: singly-linked lists,
+ * lists, simple queues, and tail queues.
+ *
+ * A singly-linked list is headed by a single forward pointer. The
+ * elements are singly linked for minimum space and pointer manipulation
+ * overhead at the expense of O(n) removal for arbitrary elements. New
+ * elements can be added to the list after an existing element or at the
+ * head of the list.  Elements being removed from the head of the list
+ * should use the explicit macro for this purpose for optimum
+ * efficiency. A singly-linked list may only be traversed in the forward
+ * direction.  Singly-linked lists are ideal for applications with large
+ * datasets and few or no removals or for implementing a LIFO queue.
+ *
+ * A list is headed by a single forward pointer (or an array of forward
+ * pointers for a hash table header). The elements are doubly linked
+ * so that an arbitrary element can be removed without a need to
+ * traverse the list. New elements can be added to the list before
+ * or after an existing element or at the head of the list. A list
+ * may only be traversed in the forward direction.
+ *
+ * A simple queue is headed by a pair of pointers, one the head of the
+ * list and the other to the tail of the list. The elements are singly
+ * linked to save space, so elements can only be removed from the
+ * head of the list. New elements can be added to the list after
+ * an existing element, at the head of the list, or at the end of the
+ * list. A simple queue may only be traversed in the forward direction.
+ *
+ * A tail queue is headed by a pair of pointers, one to the head of the
+ * list and the other to the tail of the list. The elements are doubly
+ * linked so that an arbitrary element can be removed without a need to
+ * traverse the list. New elements can be added to the list before or
+ * after an existing element, at the head of the list, or at the end of
+ * the list. A tail queue may be traversed in either direction.
+ *
+ * For details on the use of these macros, see the queue(3) manual page.
+ */
+
+#include "qemu/atomic.h" /* for smp_wmb() */
+
+/*
+ * List definitions.
+ */
+#define QLIST_HEAD(name, type)                                          \
+struct name {                                                           \
+        struct type *lh_first;  /* first element */                     \
+}
+
+#define QLIST_HEAD_INITIALIZER(head)                                    \
+        { NULL }
+
+#define QLIST_ENTRY(type)                                               \
+struct {                                                                \
+        struct type *le_next;   /* next element */                      \
+        struct type **le_prev;  /* address of previous next element */  \
+}
+
+/*
+ * List functions.
+ */
+#define QLIST_INIT(head) do {                                           \
+        (head)->lh_first = NULL;                                        \
+} while (/*CONSTCOND*/0)
+
+#define QLIST_INSERT_AFTER(listelm, elm, field) do {                    \
+        if (((elm)->field.le_next = (listelm)->field.le_next) != NULL)  \
+                (listelm)->field.le_next->field.le_prev =               \
+                    &(elm)->field.le_next;                              \
+        (listelm)->field.le_next = (elm);                               \
+        (elm)->field.le_prev = &(listelm)->field.le_next;               \
+} while (/*CONSTCOND*/0)
+
+#define QLIST_INSERT_BEFORE(listelm, elm, field) do {                   \
+        (elm)->field.le_prev = (listelm)->field.le_prev;                \
+        (elm)->field.le_next = (listelm);                               \
+        *(listelm)->field.le_prev = (elm);                              \
+        (listelm)->field.le_prev = &(elm)->field.le_next;               \
+} while (/*CONSTCOND*/0)
+
+#define QLIST_INSERT_HEAD(head, elm, field) do {                        \
+        if (((elm)->field.le_next = (head)->lh_first) != NULL)          \
+                (head)->lh_first->field.le_prev = &(elm)->field.le_next;\
+        (head)->lh_first = (elm);                                       \
+        (elm)->field.le_prev = &(head)->lh_first;                       \
+} while (/*CONSTCOND*/0)
+
+#define QLIST_INSERT_HEAD_RCU(head, elm, field) do {                    \
+        (elm)->field.le_prev = &(head)->lh_first;                       \
+        (elm)->field.le_next = (head)->lh_first;                        \
+        smp_wmb(); /* fill elm before linking it */                     \
+        if ((head)->lh_first != NULL)  {                                \
+            (head)->lh_first->field.le_prev = &(elm)->field.le_next;    \
+        }                                                               \
+        (head)->lh_first = (elm);                                       \
+        smp_wmb();                                                      \
+} while (/* CONSTCOND*/0)
+
+#define QLIST_REMOVE(elm, field) do {                                   \
+        if ((elm)->field.le_next != NULL)                               \
+                (elm)->field.le_next->field.le_prev =                   \
+                    (elm)->field.le_prev;                               \
+        *(elm)->field.le_prev = (elm)->field.le_next;                   \
+} while (/*CONSTCOND*/0)
+
+#define QLIST_FOREACH(var, head, field)                                 \
+        for ((var) = ((head)->lh_first);                                \
+                (var);                                                  \
+                (var) = ((var)->field.le_next))
+
+#define QLIST_FOREACH_SAFE(var, head, field, next_var)                  \
+        for ((var) = ((head)->lh_first);                                \
+                (var) && ((next_var) = ((var)->field.le_next), 1);      \
+                (var) = (next_var))
+
+/*
+ * List access methods.
+ */
+#define QLIST_EMPTY(head)                ((head)->lh_first == NULL)
+#define QLIST_FIRST(head)                ((head)->lh_first)
+#define QLIST_NEXT(elm, field)           ((elm)->field.le_next)
+
+
+/*
+ * Singly-linked List definitions.
+ */
+#define QSLIST_HEAD(name, type)                                          \
+struct name {                                                           \
+        struct type *slh_first; /* first element */                     \
+}
+
+#define QSLIST_HEAD_INITIALIZER(head)                                    \
+        { NULL }
+
+#define QSLIST_ENTRY(type)                                               \
+struct {                                                                \
+        struct type *sle_next;  /* next element */                      \
+}
+
+/*
+ * Singly-linked List functions.
+ */
+#define QSLIST_INIT(head) do {                                           \
+        (head)->slh_first = NULL;                                       \
+} while (/*CONSTCOND*/0)
+
+#define QSLIST_INSERT_AFTER(slistelm, elm, field) do {                   \
+        (elm)->field.sle_next = (slistelm)->field.sle_next;             \
+        (slistelm)->field.sle_next = (elm);                             \
+} while (/*CONSTCOND*/0)
+
+#define QSLIST_INSERT_HEAD(head, elm, field) do {                        \
+        (elm)->field.sle_next = (head)->slh_first;                      \
+        (head)->slh_first = (elm);                                      \
+} while (/*CONSTCOND*/0)
+
+#define QSLIST_REMOVE_HEAD(head, field) do {                             \
+        (head)->slh_first = (head)->slh_first->field.sle_next;          \
+} while (/*CONSTCOND*/0)
+
+#define QSLIST_REMOVE_AFTER(slistelm, field) do {                        \
+        (slistelm)->field.sle_next =                                    \
+            QSLIST_NEXT(QSLIST_NEXT((slistelm), field), field);           \
+} while (/*CONSTCOND*/0)
+
+#define QSLIST_FOREACH(var, head, field)                                 \
+        for((var) = (head)->slh_first; (var); (var) = (var)->field.sle_next)
+
+#define QSLIST_FOREACH_SAFE(var, head, field, tvar)                      \
+        for ((var) = QSLIST_FIRST((head));                               \
+            (var) && ((tvar) = QSLIST_NEXT((var), field), 1);            \
+            (var) = (tvar))
+
+/*
+ * Singly-linked List access methods.
+ */
+#define QSLIST_EMPTY(head)       ((head)->slh_first == NULL)
+#define QSLIST_FIRST(head)       ((head)->slh_first)
+#define QSLIST_NEXT(elm, field)  ((elm)->field.sle_next)
+
+
+/*
+ * Simple queue definitions.
+ */
+#define QSIMPLEQ_HEAD(name, type)                                       \
+struct name {                                                           \
+    struct type *sqh_first;    /* first element */                      \
+    struct type **sqh_last;    /* addr of last next element */          \
+}
+
+#define QSIMPLEQ_HEAD_INITIALIZER(head)                                 \
+    { NULL, &(head).sqh_first }
+
+#define QSIMPLEQ_ENTRY(type)                                            \
+struct {                                                                \
+    struct type *sqe_next;    /* next element */                        \
+}
+
+/*
+ * Simple queue functions.
+ */
+#define QSIMPLEQ_INIT(head) do {                                        \
+    (head)->sqh_first = NULL;                                           \
+    (head)->sqh_last = &(head)->sqh_first;                              \
+} while (/*CONSTCOND*/0)
+
+#define QSIMPLEQ_INSERT_HEAD(head, elm, field) do {                     \
+    if (((elm)->field.sqe_next = (head)->sqh_first) == NULL)            \
+        (head)->sqh_last = &(elm)->field.sqe_next;                      \
+    (head)->sqh_first = (elm);                                          \
+} while (/*CONSTCOND*/0)
+
+#define QSIMPLEQ_INSERT_TAIL(head, elm, field) do {                     \
+    (elm)->field.sqe_next = NULL;                                       \
+    *(head)->sqh_last = (elm);                                          \
+    (head)->sqh_last = &(elm)->field.sqe_next;                          \
+} while (/*CONSTCOND*/0)
+
+#define QSIMPLEQ_INSERT_AFTER(head, listelm, elm, field) do {           \
+    if (((elm)->field.sqe_next = (listelm)->field.sqe_next) == NULL)    \
+        (head)->sqh_last = &(elm)->field.sqe_next;                      \
+    (listelm)->field.sqe_next = (elm);                                  \
+} while (/*CONSTCOND*/0)
+
+#define QSIMPLEQ_REMOVE_HEAD(head, field) do {                          \
+    if (((head)->sqh_first = (head)->sqh_first->field.sqe_next) == NULL)\
+        (head)->sqh_last = &(head)->sqh_first;                          \
+} while (/*CONSTCOND*/0)
+
+#define QSIMPLEQ_REMOVE(head, elm, type, field) do {                    \
+    if ((head)->sqh_first == (elm)) {                                   \
+        QSIMPLEQ_REMOVE_HEAD((head), field);                            \
+    } else {                                                            \
+        struct type *curelm = (head)->sqh_first;                        \
+        while (curelm->field.sqe_next != (elm))                         \
+            curelm = curelm->field.sqe_next;                            \
+        if ((curelm->field.sqe_next =                                   \
+            curelm->field.sqe_next->field.sqe_next) == NULL)            \
+                (head)->sqh_last = &(curelm)->field.sqe_next;           \
+    }                                                                   \
+} while (/*CONSTCOND*/0)
+
+#define QSIMPLEQ_FOREACH(var, head, field)                              \
+    for ((var) = ((head)->sqh_first);                                   \
+        (var);                                                          \
+        (var) = ((var)->field.sqe_next))
+
+#define QSIMPLEQ_FOREACH_SAFE(var, head, field, next)                   \
+    for ((var) = ((head)->sqh_first);                                   \
+        (var) && ((next = ((var)->field.sqe_next)), 1);                 \
+        (var) = (next))
+
+#define QSIMPLEQ_CONCAT(head1, head2) do {                              \
+    if (!QSIMPLEQ_EMPTY((head2))) {                                     \
+        *(head1)->sqh_last = (head2)->sqh_first;                        \
+        (head1)->sqh_last = (head2)->sqh_last;                          \
+        QSIMPLEQ_INIT((head2));                                         \
+    }                                                                   \
+} while (/*CONSTCOND*/0)
+
+#define QSIMPLEQ_LAST(head, type, field)                                \
+    (QSIMPLEQ_EMPTY((head)) ?                                           \
+        NULL :                                                          \
+            ((struct type *)(void *)                                    \
+        ((char *)((head)->sqh_last) - offsetof(struct type, field))))
+
+/*
+ * Simple queue access methods.
+ */
+#define QSIMPLEQ_EMPTY(head)        ((head)->sqh_first == NULL)
+#define QSIMPLEQ_FIRST(head)        ((head)->sqh_first)
+#define QSIMPLEQ_NEXT(elm, field)   ((elm)->field.sqe_next)
+
+
+/*
+ * Tail queue definitions.
+ */
+#define Q_TAILQ_HEAD(name, type, qual)                                  \
+struct name {                                                           \
+        qual type *tqh_first;           /* first element */             \
+        qual type *qual *tqh_last;      /* addr of last next element */ \
+}
+#define QTAILQ_HEAD(name, type)  Q_TAILQ_HEAD(name, struct type,)
+
+#define QTAILQ_HEAD_INITIALIZER(head)                                   \
+        { NULL, &(head).tqh_first }
+
+#define Q_TAILQ_ENTRY(type, qual)                                       \
+struct {                                                                \
+        qual type *tqe_next;            /* next element */              \
+        qual type *qual *tqe_prev;      /* address of previous next element */\
+}
+#define QTAILQ_ENTRY(type)       Q_TAILQ_ENTRY(struct type,)
+
+/*
+ * Tail queue functions.
+ */
+#define QTAILQ_INIT(head) do {                                          \
+        (head)->tqh_first = NULL;                                       \
+        (head)->tqh_last = &(head)->tqh_first;                          \
+} while (/*CONSTCOND*/0)
+
+#define QTAILQ_INSERT_HEAD(head, elm, field) do {                       \
+        if (((elm)->field.tqe_next = (head)->tqh_first) != NULL)        \
+                (head)->tqh_first->field.tqe_prev =                     \
+                    &(elm)->field.tqe_next;                             \
+        else                                                            \
+                (head)->tqh_last = &(elm)->field.tqe_next;              \
+        (head)->tqh_first = (elm);                                      \
+        (elm)->field.tqe_prev = &(head)->tqh_first;                     \
+} while (/*CONSTCOND*/0)
+
+#define QTAILQ_INSERT_TAIL(head, elm, field) do {                       \
+        (elm)->field.tqe_next = NULL;                                   \
+        (elm)->field.tqe_prev = (head)->tqh_last;                       \
+        *(head)->tqh_last = (elm);                                      \
+        (head)->tqh_last = &(elm)->field.tqe_next;                      \
+} while (/*CONSTCOND*/0)
+
+#define QTAILQ_INSERT_AFTER(head, listelm, elm, field) do {             \
+        if (((elm)->field.tqe_next = (listelm)->field.tqe_next) != NULL)\
+                (elm)->field.tqe_next->field.tqe_prev =                 \
+                    &(elm)->field.tqe_next;                             \
+        else                                                            \
+                (head)->tqh_last = &(elm)->field.tqe_next;              \
+        (listelm)->field.tqe_next = (elm);                              \
+        (elm)->field.tqe_prev = &(listelm)->field.tqe_next;             \
+} while (/*CONSTCOND*/0)
+
+#define QTAILQ_INSERT_BEFORE(listelm, elm, field) do {                  \
+        (elm)->field.tqe_prev = (listelm)->field.tqe_prev;              \
+        (elm)->field.tqe_next = (listelm);                              \
+        *(listelm)->field.tqe_prev = (elm);                             \
+        (listelm)->field.tqe_prev = &(elm)->field.tqe_next;             \
+} while (/*CONSTCOND*/0)
+
+#define QTAILQ_REMOVE(head, elm, field) do {                            \
+        if (((elm)->field.tqe_next) != NULL)                            \
+                (elm)->field.tqe_next->field.tqe_prev =                 \
+                    (elm)->field.tqe_prev;                              \
+        else                                                            \
+                (head)->tqh_last = (elm)->field.tqe_prev;               \
+        *(elm)->field.tqe_prev = (elm)->field.tqe_next;                 \
+} while (/*CONSTCOND*/0)
+
+#define QTAILQ_FOREACH(var, head, field)                                \
+        for ((var) = ((head)->tqh_first);                               \
+                (var);                                                  \
+                (var) = ((var)->field.tqe_next))
+
+#define QTAILQ_FOREACH_SAFE(var, head, field, next_var)                 \
+        for ((var) = ((head)->tqh_first);                               \
+                (var) && ((next_var) = ((var)->field.tqe_next), 1);     \
+                (var) = (next_var))
+
+#define QTAILQ_FOREACH_REVERSE(var, head, headname, field)              \
+        for ((var) = (*(((struct headname *)((head)->tqh_last))->tqh_last));    \
+                (var);                                                  \
+                (var) = (*(((struct headname *)((var)->field.tqe_prev))->tqh_last)))
+
+/*
+ * Tail queue access methods.
+ */
+#define QTAILQ_EMPTY(head)               ((head)->tqh_first == NULL)
+#define QTAILQ_FIRST(head)               ((head)->tqh_first)
+#define QTAILQ_NEXT(elm, field)          ((elm)->field.tqe_next)
+
+#define QTAILQ_LAST(head, headname) \
+        (*(((struct headname *)((head)->tqh_last))->tqh_last))
+#define QTAILQ_PREV(elm, headname, field) \
+        (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last))
+
+#endif  /* !QEMU_SYS_QUEUE_H_ */
diff --git a/contrib/qemu/include/qemu/sockets.h b/contrib/qemu/include/qemu/sockets.h
new file mode 100644
index 000000000..c5174d76a
--- /dev/null
+++ b/contrib/qemu/include/qemu/sockets.h
@@ -0,0 +1,83 @@
+/* headers to use the BSD sockets */
+#ifndef QEMU_SOCKET_H
+#define QEMU_SOCKET_H
+
+#ifdef _WIN32
+#include <windows.h>
+#include <winsock2.h>
+#include <ws2tcpip.h>
+
+#define socket_error() WSAGetLastError()
+
+int inet_aton(const char *cp, struct in_addr *ia);
+
+#else
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <sys/un.h>
+
+#define socket_error() errno
+#define closesocket(s) close(s)
+
+#endif /* !_WIN32 */
+
+#include "qemu/option.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
+
+extern QemuOptsList socket_optslist;
+
+/* misc helpers */
+int qemu_socket(int domain, int type, int protocol);
+int qemu_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
+int socket_set_cork(int fd, int v);
+int socket_set_nodelay(int fd);
+void qemu_set_block(int fd);
+void qemu_set_nonblock(int fd);
+int send_all(int fd, const void *buf, int len1);
+int recv_all(int fd, void *buf, int len1, bool single_read);
+
+/* callback function for nonblocking connect
+ * valid fd on success, negative error code on failure
+ */
+typedef void NonBlockingConnectHandler(int fd, void *opaque);
+
+InetSocketAddress *inet_parse(const char *str, Error **errp);
+int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp);
+int inet_listen(const char *str, char *ostr, int olen,
+                int socktype, int port_offset, Error **errp);
+int inet_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque);
+int inet_connect(const char *str, Error **errp);
+int inet_nonblocking_connect(const char *str,
+                             NonBlockingConnectHandler *callback,
+                             void *opaque, Error **errp);
+
+int inet_dgram_opts(QemuOpts *opts, Error **errp);
+const char *inet_strfamily(int family);
+
+int unix_listen_opts(QemuOpts *opts, Error **errp);
+int unix_listen(const char *path, char *ostr, int olen, Error **errp);
+int unix_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque);
+int unix_connect(const char *path, Error **errp);
+int unix_nonblocking_connect(const char *str,
+                             NonBlockingConnectHandler *callback,
+                             void *opaque, Error **errp);
+
+SocketAddress *socket_parse(const char *str, Error **errp);
+int socket_connect(SocketAddress *addr, Error **errp,
+                   NonBlockingConnectHandler *callback, void *opaque);
+int socket_listen(SocketAddress *addr, Error **errp);
+int socket_dgram(SocketAddress *remote, SocketAddress *local, Error **errp);
+
+/* Old, ipv4 only bits.  Don't use for new code. */
+int parse_host_port(struct sockaddr_in *saddr, const char *str);
+int socket_init(void);
+
+#endif /* QEMU_SOCKET_H */
diff --git a/contrib/qemu/include/qemu/thread-posix.h b/contrib/qemu/include/qemu/thread-posix.h
new file mode 100644
index 000000000..0f30dccb5
--- /dev/null
+++ b/contrib/qemu/include/qemu/thread-posix.h
@@ -0,0 +1,28 @@
+#ifndef __QEMU_THREAD_POSIX_H
+#define __QEMU_THREAD_POSIX_H 1
+#include "pthread.h"
+#include <semaphore.h>
+
+struct QemuMutex {
+    pthread_mutex_t lock;
+};
+
+struct QemuCond {
+    pthread_cond_t cond;
+};
+
+struct QemuSemaphore {
+#if defined(__APPLE__) || defined(__NetBSD__)
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
+    int count;
+#else
+    sem_t sem;
+#endif
+};
+
+struct QemuThread {
+    pthread_t thread;
+};
+
+#endif
diff --git a/contrib/qemu/include/qemu/thread.h b/contrib/qemu/include/qemu/thread.h
new file mode 100644
index 000000000..c02404b9f
--- /dev/null
+++ b/contrib/qemu/include/qemu/thread.h
@@ -0,0 +1,56 @@
+#ifndef __QEMU_THREAD_H
+#define __QEMU_THREAD_H 1
+
+#include <inttypes.h>
+#include <stdbool.h>
+
+typedef struct QemuMutex QemuMutex;
+typedef struct QemuCond QemuCond;
+typedef struct QemuSemaphore QemuSemaphore;
+typedef struct QemuThread QemuThread;
+
+#ifdef _WIN32
+#include "qemu/thread-win32.h"
+#else
+#include "qemu/thread-posix.h"
+#endif
+
+#define QEMU_THREAD_JOINABLE 0
+#define QEMU_THREAD_DETACHED 1
+
+void qemu_mutex_init(QemuMutex *mutex);
+void qemu_mutex_destroy(QemuMutex *mutex);
+void qemu_mutex_lock(QemuMutex *mutex);
+int qemu_mutex_trylock(QemuMutex *mutex);
+void qemu_mutex_unlock(QemuMutex *mutex);
+
+#define rcu_read_lock() do { } while (0)
+#define rcu_read_unlock() do { } while (0)
+
+void qemu_cond_init(QemuCond *cond);
+void qemu_cond_destroy(QemuCond *cond);
+
+/*
+ * IMPORTANT: The implementation does not guarantee that pthread_cond_signal
+ * and pthread_cond_broadcast can be called except while the same mutex is
+ * held as in the corresponding pthread_cond_wait calls!
+ */
+void qemu_cond_signal(QemuCond *cond);
+void qemu_cond_broadcast(QemuCond *cond);
+void qemu_cond_wait(QemuCond *cond, QemuMutex *mutex);
+
+void qemu_sem_init(QemuSemaphore *sem, int init);
+void qemu_sem_post(QemuSemaphore *sem);
+void qemu_sem_wait(QemuSemaphore *sem);
+int qemu_sem_timedwait(QemuSemaphore *sem, int ms);
+void qemu_sem_destroy(QemuSemaphore *sem);
+
+void qemu_thread_create(QemuThread *thread,
+                        void *(*start_routine)(void *),
+                        void *arg, int mode);
+void *qemu_thread_join(QemuThread *thread);
+void qemu_thread_get_self(QemuThread *thread);
+bool qemu_thread_is_self(QemuThread *thread);
+void qemu_thread_exit(void *retval);
+
+#endif
diff --git a/contrib/qemu/include/qemu/timer.h b/contrib/qemu/include/qemu/timer.h
new file mode 100644
index 000000000..9dd206ce7
--- /dev/null
+++ b/contrib/qemu/include/qemu/timer.h
@@ -0,0 +1,305 @@
+#ifndef QEMU_TIMER_H
+#define QEMU_TIMER_H
+
+#include "qemu-common.h"
+#include "qemu/main-loop.h"
+#include "qemu/notify.h"
+
+/* timers */
+
+#define SCALE_MS 1000000
+#define SCALE_US 1000
+#define SCALE_NS 1
+
+typedef struct QEMUClock QEMUClock;
+typedef void QEMUTimerCB(void *opaque);
+
+/* The real time clock should be used only for stuff which does not
+   change the virtual machine state, as it is run even if the virtual
+   machine is stopped. The real time clock has a frequency of 1000
+   Hz. */
+extern QEMUClock *rt_clock;
+
+/* The virtual clock is only run during the emulation. It is stopped
+   when the virtual machine is stopped. Virtual timers use a high
+   precision clock, usually cpu cycles (use ticks_per_sec). */
+extern QEMUClock *vm_clock;
+
+/* The host clock should be use for device models that emulate accurate
+   real time sources. It will continue to run when the virtual machine
+   is suspended, and it will reflect system time changes the host may
+   undergo (e.g. due to NTP). The host clock has the same precision as
+   the virtual clock. */
+extern QEMUClock *host_clock;
+
+int64_t qemu_get_clock_ns(QEMUClock *clock);
+int64_t qemu_clock_has_timers(QEMUClock *clock);
+int64_t qemu_clock_expired(QEMUClock *clock);
+int64_t qemu_clock_deadline(QEMUClock *clock);
+void qemu_clock_enable(QEMUClock *clock, bool enabled);
+void qemu_clock_warp(QEMUClock *clock);
+
+void qemu_register_clock_reset_notifier(QEMUClock *clock, Notifier *notifier);
+void qemu_unregister_clock_reset_notifier(QEMUClock *clock,
+                                          Notifier *notifier);
+
+QEMUTimer *qemu_new_timer(QEMUClock *clock, int scale,
+                          QEMUTimerCB *cb, void *opaque);
+void qemu_free_timer(QEMUTimer *ts);
+void qemu_del_timer(QEMUTimer *ts);
+void qemu_mod_timer_ns(QEMUTimer *ts, int64_t expire_time);
+void qemu_mod_timer(QEMUTimer *ts, int64_t expire_time);
+bool qemu_timer_pending(QEMUTimer *ts);
+bool qemu_timer_expired(QEMUTimer *timer_head, int64_t current_time);
+uint64_t qemu_timer_expire_time_ns(QEMUTimer *ts);
+
+void qemu_run_timers(QEMUClock *clock);
+void qemu_run_all_timers(void);
+void configure_alarms(char const *opt);
+void init_clocks(void);
+int init_timer_alarm(void);
+
+int64_t cpu_get_ticks(void);
+void cpu_enable_ticks(void);
+void cpu_disable_ticks(void);
+
+static inline QEMUTimer *qemu_new_timer_ns(QEMUClock *clock, QEMUTimerCB *cb,
+                                           void *opaque)
+{
+    return qemu_new_timer(clock, SCALE_NS, cb, opaque);
+}
+
+static inline QEMUTimer *qemu_new_timer_ms(QEMUClock *clock, QEMUTimerCB *cb,
+                                           void *opaque)
+{
+    return qemu_new_timer(clock, SCALE_MS, cb, opaque);
+}
+
+static inline int64_t qemu_get_clock_ms(QEMUClock *clock)
+{
+    return qemu_get_clock_ns(clock) / SCALE_MS;
+}
+
+static inline int64_t get_ticks_per_sec(void)
+{
+    return 1000000000LL;
+}
+
+/* real time host monotonic timer */
+static inline int64_t get_clock_realtime(void)
+{
+    struct timeval tv;
+
+    gettimeofday(&tv, NULL);
+    return tv.tv_sec * 1000000000LL + (tv.tv_usec * 1000);
+}
+
+/* Warning: don't insert tracepoints into these functions, they are
+   also used by simpletrace backend and tracepoints would cause
+   an infinite recursion! */
+#ifdef _WIN32
+extern int64_t clock_freq;
+
+static inline int64_t get_clock(void)
+{
+    LARGE_INTEGER ti;
+    QueryPerformanceCounter(&ti);
+    return muldiv64(ti.QuadPart, get_ticks_per_sec(), clock_freq);
+}
+
+#else
+
+extern int use_rt_clock;
+
+static inline int64_t get_clock(void)
+{
+#ifdef CLOCK_MONOTONIC
+    if (use_rt_clock) {
+        struct timespec ts;
+        clock_gettime(CLOCK_MONOTONIC, &ts);
+        return ts.tv_sec * 1000000000LL + ts.tv_nsec;
+    } else
+#endif
+    {
+        /* XXX: using gettimeofday leads to problems if the date
+           changes, so it should be avoided. */
+        return get_clock_realtime();
+    }
+}
+#endif
+
+void qemu_get_timer(QEMUFile *f, QEMUTimer *ts);
+void qemu_put_timer(QEMUFile *f, QEMUTimer *ts);
+
+/* icount */
+int64_t cpu_get_icount(void);
+int64_t cpu_get_clock(void);
+
+/*******************************************/
+/* host CPU ticks (if available) */
+
+#if defined(_ARCH_PPC)
+
+static inline int64_t cpu_get_real_ticks(void)
+{
+    int64_t retval;
+#ifdef _ARCH_PPC64
+    /* This reads timebase in one 64bit go and includes Cell workaround from:
+       http://ozlabs.org/pipermail/linuxppc-dev/2006-October/027052.html
+    */
+    __asm__ __volatile__ ("mftb    %0\n\t"
+                          "cmpwi   %0,0\n\t"
+                          "beq-    $-8"
+                          : "=r" (retval));
+#else
+    /* http://ozlabs.org/pipermail/linuxppc-dev/1999-October/003889.html */
+    unsigned long junk;
+    __asm__ __volatile__ ("mfspr   %1,269\n\t"  /* mftbu */
+                          "mfspr   %L0,268\n\t" /* mftb */
+                          "mfspr   %0,269\n\t"  /* mftbu */
+                          "cmpw    %0,%1\n\t"
+                          "bne     $-16"
+                          : "=r" (retval), "=r" (junk));
+#endif
+    return retval;
+}
+
+#elif defined(__i386__)
+
+static inline int64_t cpu_get_real_ticks(void)
+{
+    int64_t val;
+    asm volatile ("rdtsc" : "=A" (val));
+    return val;
+}
+
+#elif defined(__x86_64__)
+
+static inline int64_t cpu_get_real_ticks(void)
+{
+    uint32_t low,high;
+    int64_t val;
+    asm volatile("rdtsc" : "=a" (low), "=d" (high));
+    val = high;
+    val <<= 32;
+    val |= low;
+    return val;
+}
+
+#elif defined(__hppa__)
+
+static inline int64_t cpu_get_real_ticks(void)
+{
+    int val;
+    asm volatile ("mfctl %%cr16, %0" : "=r"(val));
+    return val;
+}
+
+#elif defined(__ia64)
+
+static inline int64_t cpu_get_real_ticks(void)
+{
+    int64_t val;
+    asm volatile ("mov %0 = ar.itc" : "=r"(val) :: "memory");
+    return val;
+}
+
+#elif defined(__s390__)
+
+static inline int64_t cpu_get_real_ticks(void)
+{
+    int64_t val;
+    asm volatile("stck 0(%1)" : "=m" (val) : "a" (&val) : "cc");
+    return val;
+}
+
+#elif defined(__sparc__)
+
+static inline int64_t cpu_get_real_ticks (void)
+{
+#if defined(_LP64)
+    uint64_t        rval;
+    asm volatile("rd %%tick,%0" : "=r"(rval));
+    return rval;
+#else
+    /* We need an %o or %g register for this.  For recent enough gcc
+       there is an "h" constraint for that.  Don't bother with that.  */
+    union {
+        uint64_t i64;
+        struct {
+            uint32_t high;
+            uint32_t low;
+        }       i32;
+    } rval;
+    asm volatile("rd %%tick,%%g1; srlx %%g1,32,%0; mov %%g1,%1"
+                 : "=r"(rval.i32.high), "=r"(rval.i32.low) : : "g1");
+    return rval.i64;
+#endif
+}
+
+#elif defined(__mips__) && \
+    ((defined(__mips_isa_rev) && __mips_isa_rev >= 2) || defined(__linux__))
+/*
+ * binutils wants to use rdhwr only on mips32r2
+ * but as linux kernel emulate it, it's fine
+ * to use it.
+ *
+ */
+#define MIPS_RDHWR(rd, value) {                         \
+        __asm__ __volatile__ (".set   push\n\t"         \
+                              ".set mips32r2\n\t"       \
+                              "rdhwr  %0, "rd"\n\t"     \
+                              ".set   pop"              \
+                              : "=r" (value));          \
+    }
+
+static inline int64_t cpu_get_real_ticks(void)
+{
+    /* On kernels >= 2.6.25 rdhwr <reg>, $2 and $3 are emulated */
+    uint32_t count;
+    static uint32_t cyc_per_count = 0;
+
+    if (!cyc_per_count) {
+        MIPS_RDHWR("$3", cyc_per_count);
+    }
+
+    MIPS_RDHWR("$2", count);
+    return (int64_t)(count * cyc_per_count);
+}
+
+#elif defined(__alpha__)
+
+static inline int64_t cpu_get_real_ticks(void)
+{
+    uint64_t cc;
+    uint32_t cur, ofs;
+
+    asm volatile("rpcc %0" : "=r"(cc));
+    cur = cc;
+    ofs = cc >> 32;
+    return cur - ofs;
+}
+
+#else
+/* The host CPU doesn't have an easily accessible cycle counter.
+   Just return a monotonically increasing value.  This will be
+   totally wrong, but hopefully better than nothing.  */
+static inline int64_t cpu_get_real_ticks (void)
+{
+    static int64_t ticks = 0;
+    return ticks++;
+}
+#endif
+
+#ifdef CONFIG_PROFILER
+static inline int64_t profile_getclock(void)
+{
+    return cpu_get_real_ticks();
+}
+
+extern int64_t qemu_time, qemu_time_start;
+extern int64_t tlb_flush_time;
+extern int64_t dev_time;
+#endif
+
+#endif
diff --git a/contrib/qemu/include/qemu/typedefs.h b/contrib/qemu/include/qemu/typedefs.h
new file mode 100644
index 000000000..ac9f8d41a
--- /dev/null
+++ b/contrib/qemu/include/qemu/typedefs.h
@@ -0,0 +1,69 @@
+#ifndef QEMU_TYPEDEFS_H
+#define QEMU_TYPEDEFS_H
+
+/* A load of opaque types so that device init declarations don't have to
+   pull in all the real definitions.  */
+typedef struct QEMUTimer QEMUTimer;
+typedef struct QEMUFile QEMUFile;
+typedef struct QEMUBH QEMUBH;
+
+struct Monitor;
+typedef struct Monitor Monitor;
+typedef struct MigrationParams MigrationParams;
+
+typedef struct Property Property;
+typedef struct PropertyInfo PropertyInfo;
+typedef struct CompatProperty CompatProperty;
+typedef struct DeviceState DeviceState;
+typedef struct BusState BusState;
+typedef struct BusClass BusClass;
+
+typedef struct AddressSpace AddressSpace;
+typedef struct MemoryRegion MemoryRegion;
+typedef struct MemoryRegionSection MemoryRegionSection;
+
+typedef struct MemoryMappingList MemoryMappingList;
+
+typedef struct NICInfo NICInfo;
+typedef struct HCIInfo HCIInfo;
+typedef struct AudioState AudioState;
+typedef struct BlockDriverState BlockDriverState;
+typedef struct DriveInfo DriveInfo;
+typedef struct DisplayState DisplayState;
+typedef struct DisplayChangeListener DisplayChangeListener;
+typedef struct DisplaySurface DisplaySurface;
+typedef struct PixelFormat PixelFormat;
+typedef struct QemuConsole QemuConsole;
+typedef struct CharDriverState CharDriverState;
+typedef struct MACAddr MACAddr;
+typedef struct NetClientState NetClientState;
+typedef struct i2c_bus i2c_bus;
+typedef struct ISABus ISABus;
+typedef struct ISADevice ISADevice;
+typedef struct SMBusDevice SMBusDevice;
+typedef struct PCIHostState PCIHostState;
+typedef struct PCIExpressHost PCIExpressHost;
+typedef struct PCIBus PCIBus;
+typedef struct PCIDevice PCIDevice;
+typedef struct PCIExpressDevice PCIExpressDevice;
+typedef struct PCIBridge PCIBridge;
+typedef struct PCIEAERMsg PCIEAERMsg;
+typedef struct PCIEAERLog PCIEAERLog;
+typedef struct PCIEAERErr PCIEAERErr;
+typedef struct PCIEPort PCIEPort;
+typedef struct PCIESlot PCIESlot;
+typedef struct MSIMessage MSIMessage;
+typedef struct SerialState SerialState;
+typedef struct PCMCIACardState PCMCIACardState;
+typedef struct MouseTransformInfo MouseTransformInfo;
+typedef struct uWireSlave uWireSlave;
+typedef struct I2SCodec I2SCodec;
+typedef struct SSIBus SSIBus;
+typedef struct EventNotifier EventNotifier;
+typedef struct VirtIODevice VirtIODevice;
+typedef struct QEMUSGList QEMUSGList;
+typedef struct SHPCDevice SHPCDevice;
+typedef struct FWCfgState FWCfgState;
+typedef struct PcGuestInfo PcGuestInfo;
+
+#endif /* QEMU_TYPEDEFS_H */
diff --git a/contrib/qemu/include/sysemu/os-posix.h b/contrib/qemu/include/sysemu/os-posix.h
new file mode 100644
index 000000000..25d0b2a73
--- /dev/null
+++ b/contrib/qemu/include/sysemu/os-posix.h
@@ -0,0 +1,52 @@
+/*
+ * posix specific declarations
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2010 Jes Sorensen <Jes.Sorensen@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef QEMU_OS_POSIX_H
+#define QEMU_OS_POSIX_H
+
+void os_set_line_buffering(void);
+void os_set_proc_name(const char *s);
+void os_setup_signal_handling(void);
+void os_daemonize(void);
+void os_setup_post(void);
+int os_mlock(void);
+
+typedef struct timeval qemu_timeval;
+#define qemu_gettimeofday(tp) gettimeofday(tp, NULL)
+
+#ifndef CONFIG_UTIMENSAT
+#ifndef UTIME_NOW
+# define UTIME_NOW     ((1l << 30) - 1l)
+#endif
+#ifndef UTIME_OMIT
+# define UTIME_OMIT    ((1l << 30) - 2l)
+#endif
+#endif
+typedef struct timespec qemu_timespec;
+int qemu_utimens(const char *path, const qemu_timespec *times);
+
+bool is_daemonized(void);
+
+#endif
diff --git a/contrib/qemu/include/sysemu/sysemu.h b/contrib/qemu/include/sysemu/sysemu.h
new file mode 100644
index 000000000..3caeb66eb
--- /dev/null
+++ b/contrib/qemu/include/sysemu/sysemu.h
@@ -0,0 +1,200 @@
+#ifndef SYSEMU_H
+#define SYSEMU_H
+/* Misc. things related to the system emulator.  */
+
+#include "qemu/typedefs.h"
+#include "qemu/option.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
+#include "qapi-types.h"
+#include "qemu/notify.h"
+#include "qemu/main-loop.h"
+
+/* vl.c */
+
+extern const char *bios_name;
+
+extern const char *qemu_name;
+extern uint8_t qemu_uuid[];
+int qemu_uuid_parse(const char *str, uint8_t *uuid);
+#define UUID_FMT "%02hhx%02hhx%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx"
+
+bool runstate_check(RunState state);
+void runstate_set(RunState new_state);
+int runstate_is_running(void);
+bool runstate_needs_reset(void);
+typedef struct vm_change_state_entry VMChangeStateEntry;
+typedef void VMChangeStateHandler(void *opaque, int running, RunState state);
+
+VMChangeStateEntry *qemu_add_vm_change_state_handler(VMChangeStateHandler *cb,
+                                                     void *opaque);
+void qemu_del_vm_change_state_handler(VMChangeStateEntry *e);
+void vm_state_notify(int running, RunState state);
+
+#define VMRESET_SILENT   false
+#define VMRESET_REPORT   true
+
+void vm_start(void);
+int vm_stop(RunState state);
+int vm_stop_force_state(RunState state);
+
+typedef enum WakeupReason {
+    QEMU_WAKEUP_REASON_OTHER = 0,
+    QEMU_WAKEUP_REASON_RTC,
+    QEMU_WAKEUP_REASON_PMTIMER,
+} WakeupReason;
+
+void qemu_system_reset_request(void);
+void qemu_system_suspend_request(void);
+void qemu_register_suspend_notifier(Notifier *notifier);
+void qemu_system_wakeup_request(WakeupReason reason);
+void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
+void qemu_register_wakeup_notifier(Notifier *notifier);
+void qemu_system_shutdown_request(void);
+void qemu_system_powerdown_request(void);
+void qemu_register_powerdown_notifier(Notifier *notifier);
+void qemu_system_debug_request(void);
+void qemu_system_vmstop_request(RunState reason);
+int qemu_shutdown_requested_get(void);
+int qemu_reset_requested_get(void);
+void qemu_system_killed(int signal, pid_t pid);
+void qemu_devices_reset(void);
+void qemu_system_reset(bool report);
+
+void qemu_add_exit_notifier(Notifier *notify);
+void qemu_remove_exit_notifier(Notifier *notify);
+
+void qemu_add_machine_init_done_notifier(Notifier *notify);
+
+void do_savevm(Monitor *mon, const QDict *qdict);
+int load_vmstate(const char *name);
+void do_delvm(Monitor *mon, const QDict *qdict);
+void do_info_snapshots(Monitor *mon, const QDict *qdict);
+
+void qemu_announce_self(void);
+
+bool qemu_savevm_state_blocked(Error **errp);
+void qemu_savevm_state_begin(QEMUFile *f,
+                             const MigrationParams *params);
+int qemu_savevm_state_iterate(QEMUFile *f);
+void qemu_savevm_state_complete(QEMUFile *f);
+void qemu_savevm_state_cancel(void);
+uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size);
+int qemu_loadvm_state(QEMUFile *f);
+
+/* SLIRP */
+void do_info_slirp(Monitor *mon);
+
+typedef enum DisplayType
+{
+    DT_DEFAULT,
+    DT_CURSES,
+    DT_SDL,
+    DT_GTK,
+    DT_NOGRAPHIC,
+    DT_NONE,
+} DisplayType;
+
+extern int autostart;
+
+typedef enum {
+    VGA_NONE, VGA_STD, VGA_CIRRUS, VGA_VMWARE, VGA_XENFB, VGA_QXL,
+} VGAInterfaceType;
+
+extern int vga_interface_type;
+#define xenfb_enabled (vga_interface_type == VGA_XENFB)
+#define qxl_enabled (vga_interface_type == VGA_QXL)
+
+extern int graphic_width;
+extern int graphic_height;
+extern int graphic_depth;
+extern DisplayType display_type;
+extern const char *keyboard_layout;
+extern int win2k_install_hack;
+extern int alt_grab;
+extern int ctrl_grab;
+extern int smp_cpus;
+extern int max_cpus;
+extern int cursor_hide;
+extern int graphic_rotate;
+extern int no_quit;
+extern int no_shutdown;
+extern int semihosting_enabled;
+extern int old_param;
+extern int boot_menu;
+extern uint8_t *boot_splash_filedata;
+extern size_t boot_splash_filedata_size;
+extern uint8_t qemu_extra_params_fw[2];
+extern QEMUClock *rtc_clock;
+
+#define MAX_NODES 64
+#define MAX_CPUMASK_BITS 255
+extern int nb_numa_nodes;
+extern uint64_t node_mem[MAX_NODES];
+extern unsigned long *node_cpumask[MAX_NODES];
+
+#define MAX_OPTION_ROMS 16
+typedef struct QEMUOptionRom {
+    const char *name;
+    int32_t bootindex;
+} QEMUOptionRom;
+extern QEMUOptionRom option_rom[MAX_OPTION_ROMS];
+extern int nb_option_roms;
+
+#define MAX_PROM_ENVS 128
+extern const char *prom_envs[MAX_PROM_ENVS];
+extern unsigned int nb_prom_envs;
+
+/* pci-hotplug */
+void pci_device_hot_add(Monitor *mon, const QDict *qdict);
+int pci_drive_hot_add(Monitor *mon, const QDict *qdict, DriveInfo *dinfo);
+void do_pci_device_hot_remove(Monitor *mon, const QDict *qdict);
+
+/* generic hotplug */
+void drive_hot_add(Monitor *mon, const QDict *qdict);
+
+/* CPU hotplug */
+void qemu_register_cpu_added_notifier(Notifier *notifier);
+
+/* pcie aer error injection */
+void pcie_aer_inject_error_print(Monitor *mon, const QObject *data);
+int do_pcie_aer_inject_error(Monitor *mon,
+                             const QDict *qdict, QObject **ret_data);
+
+/* serial ports */
+
+#define MAX_SERIAL_PORTS 4
+
+extern CharDriverState *serial_hds[MAX_SERIAL_PORTS];
+
+/* parallel ports */
+
+#define MAX_PARALLEL_PORTS 3
+
+extern CharDriverState *parallel_hds[MAX_PARALLEL_PORTS];
+
+void do_usb_add(Monitor *mon, const QDict *qdict);
+void do_usb_del(Monitor *mon, const QDict *qdict);
+void usb_info(Monitor *mon, const QDict *qdict);
+
+void rtc_change_mon_event(struct tm *tm);
+
+void add_boot_device_path(int32_t bootindex, DeviceState *dev,
+                          const char *suffix);
+char *get_boot_devices_list(size_t *size);
+
+DeviceState *get_boot_device(uint32_t position);
+
+QemuOpts *qemu_get_machine_opts(void);
+
+bool usb_enabled(bool default_usb);
+
+extern QemuOptsList qemu_drive_opts;
+extern QemuOptsList qemu_chardev_opts;
+extern QemuOptsList qemu_device_opts;
+extern QemuOptsList qemu_netdev_opts;
+extern QemuOptsList qemu_net_opts;
+extern QemuOptsList qemu_global_opts;
+extern QemuOptsList qemu_mon_opts;
+
+#endif
diff --git a/contrib/qemu/include/trace.h b/contrib/qemu/include/trace.h
new file mode 100644
index 000000000..c15f49812
--- /dev/null
+++ b/contrib/qemu/include/trace.h
@@ -0,0 +1,6 @@
+#ifndef TRACE_H
+#define TRACE_H
+
+#include "trace/generated-tracers.h"
+
+#endif  /* TRACE_H */
diff --git a/contrib/qemu/nop-symbols.c b/contrib/qemu/nop-symbols.c
new file mode 100644
index 000000000..ae93a3d3b
--- /dev/null
+++ b/contrib/qemu/nop-symbols.c
@@ -0,0 +1,12 @@
+int notifier_with_return_list_init () { return 0; }
+int notifier_with_return_list_notify () { return 0; }
+int notifier_with_return_list_add () { return 0; }
+int notifier_list_init () { return 0; }
+int notifier_list_notify () { return 0; }
+int notifier_list_add () { return 0; }
+int monitor_protocol_event () { return 0; }
+int block_job_cancel_sync () { return 0; }
+int block_job_iostatus_reset () { return 0; }
+int vm_stop () { return 0; }
+int qemu_get_aio_context () { return 0; }
+
diff --git a/contrib/qemu/qapi-types.h b/contrib/qemu/qapi-types.h
new file mode 100644
index 000000000..082b06d1c
--- /dev/null
+++ b/contrib/qemu/qapi-types.h
@@ -0,0 +1,2746 @@
+/* AUTOMATICALLY GENERATED, DO NOT MODIFY */
+
+/*
+ * schema-defined QAPI types
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QAPI_TYPES_H
+#define QAPI_TYPES_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+
+#ifndef QAPI_TYPES_BUILTIN_STRUCT_DECL_H
+#define QAPI_TYPES_BUILTIN_STRUCT_DECL_H
+
+
+typedef struct strList
+{
+    union {
+        char * value;
+        uint64_t padding;
+    };
+    struct strList *next;
+} strList;
+
+typedef struct intList
+{
+    union {
+        int64_t value;
+        uint64_t padding;
+    };
+    struct intList *next;
+} intList;
+
+typedef struct numberList
+{
+    union {
+        double value;
+        uint64_t padding;
+    };
+    struct numberList *next;
+} numberList;
+
+typedef struct boolList
+{
+    union {
+        bool value;
+        uint64_t padding;
+    };
+    struct boolList *next;
+} boolList;
+
+typedef struct int8List
+{
+    union {
+        int8_t value;
+        uint64_t padding;
+    };
+    struct int8List *next;
+} int8List;
+
+typedef struct int16List
+{
+    union {
+        int16_t value;
+        uint64_t padding;
+    };
+    struct int16List *next;
+} int16List;
+
+typedef struct int32List
+{
+    union {
+        int32_t value;
+        uint64_t padding;
+    };
+    struct int32List *next;
+} int32List;
+
+typedef struct int64List
+{
+    union {
+        int64_t value;
+        uint64_t padding;
+    };
+    struct int64List *next;
+} int64List;
+
+typedef struct uint8List
+{
+    union {
+        uint8_t value;
+        uint64_t padding;
+    };
+    struct uint8List *next;
+} uint8List;
+
+typedef struct uint16List
+{
+    union {
+        uint16_t value;
+        uint64_t padding;
+    };
+    struct uint16List *next;
+} uint16List;
+
+typedef struct uint32List
+{
+    union {
+        uint32_t value;
+        uint64_t padding;
+    };
+    struct uint32List *next;
+} uint32List;
+
+typedef struct uint64List
+{
+    union {
+        uint64_t value;
+        uint64_t padding;
+    };
+    struct uint64List *next;
+} uint64List;
+
+#endif /* QAPI_TYPES_BUILTIN_STRUCT_DECL_H */
+
+
+extern const char *ErrorClass_lookup[];
+typedef enum ErrorClass
+{
+    ERROR_CLASS_GENERIC_ERROR = 0,
+    ERROR_CLASS_COMMAND_NOT_FOUND = 1,
+    ERROR_CLASS_DEVICE_ENCRYPTED = 2,
+    ERROR_CLASS_DEVICE_NOT_ACTIVE = 3,
+    ERROR_CLASS_DEVICE_NOT_FOUND = 4,
+    ERROR_CLASS_K_V_M_MISSING_CAP = 5,
+    ERROR_CLASS_MAX = 6,
+} ErrorClass;
+
+typedef struct ErrorClassList
+{
+    ErrorClass value;
+    struct ErrorClassList *next;
+} ErrorClassList;
+
+
+typedef struct NameInfo NameInfo;
+
+typedef struct NameInfoList
+{
+    union {
+        NameInfo *value;
+        uint64_t padding;
+    };
+    struct NameInfoList *next;
+} NameInfoList;
+
+
+typedef struct VersionInfo VersionInfo;
+
+typedef struct VersionInfoList
+{
+    union {
+        VersionInfo *value;
+        uint64_t padding;
+    };
+    struct VersionInfoList *next;
+} VersionInfoList;
+
+
+typedef struct KvmInfo KvmInfo;
+
+typedef struct KvmInfoList
+{
+    union {
+        KvmInfo *value;
+        uint64_t padding;
+    };
+    struct KvmInfoList *next;
+} KvmInfoList;
+
+extern const char *RunState_lookup[];
+typedef enum RunState
+{
+    RUN_STATE_DEBUG = 0,
+    RUN_STATE_INMIGRATE = 1,
+    RUN_STATE_INTERNAL_ERROR = 2,
+    RUN_STATE_IO_ERROR = 3,
+    RUN_STATE_PAUSED = 4,
+    RUN_STATE_POSTMIGRATE = 5,
+    RUN_STATE_PRELAUNCH = 6,
+    RUN_STATE_FINISH_MIGRATE = 7,
+    RUN_STATE_RESTORE_VM = 8,
+    RUN_STATE_RUNNING = 9,
+    RUN_STATE_SAVE_VM = 10,
+    RUN_STATE_SHUTDOWN = 11,
+    RUN_STATE_SUSPENDED = 12,
+    RUN_STATE_WATCHDOG = 13,
+    RUN_STATE_GUEST_PANICKED = 14,
+    RUN_STATE_MAX = 15,
+} RunState;
+
+typedef struct RunStateList
+{
+    RunState value;
+    struct RunStateList *next;
+} RunStateList;
+
+
+typedef struct SnapshotInfo SnapshotInfo;
+
+typedef struct SnapshotInfoList
+{
+    union {
+        SnapshotInfo *value;
+        uint64_t padding;
+    };
+    struct SnapshotInfoList *next;
+} SnapshotInfoList;
+
+
+typedef struct ImageInfo ImageInfo;
+
+typedef struct ImageInfoList
+{
+    union {
+        ImageInfo *value;
+        uint64_t padding;
+    };
+    struct ImageInfoList *next;
+} ImageInfoList;
+
+
+typedef struct ImageCheck ImageCheck;
+
+typedef struct ImageCheckList
+{
+    union {
+        ImageCheck *value;
+        uint64_t padding;
+    };
+    struct ImageCheckList *next;
+} ImageCheckList;
+
+
+typedef struct StatusInfo StatusInfo;
+
+typedef struct StatusInfoList
+{
+    union {
+        StatusInfo *value;
+        uint64_t padding;
+    };
+    struct StatusInfoList *next;
+} StatusInfoList;
+
+
+typedef struct UuidInfo UuidInfo;
+
+typedef struct UuidInfoList
+{
+    union {
+        UuidInfo *value;
+        uint64_t padding;
+    };
+    struct UuidInfoList *next;
+} UuidInfoList;
+
+
+typedef struct ChardevInfo ChardevInfo;
+
+typedef struct ChardevInfoList
+{
+    union {
+        ChardevInfo *value;
+        uint64_t padding;
+    };
+    struct ChardevInfoList *next;
+} ChardevInfoList;
+
+extern const char *DataFormat_lookup[];
+typedef enum DataFormat
+{
+    DATA_FORMAT_UTF8 = 0,
+    DATA_FORMAT_BASE64 = 1,
+    DATA_FORMAT_MAX = 2,
+} DataFormat;
+
+typedef struct DataFormatList
+{
+    DataFormat value;
+    struct DataFormatList *next;
+} DataFormatList;
+
+
+typedef struct CommandInfo CommandInfo;
+
+typedef struct CommandInfoList
+{
+    union {
+        CommandInfo *value;
+        uint64_t padding;
+    };
+    struct CommandInfoList *next;
+} CommandInfoList;
+
+
+typedef struct EventInfo EventInfo;
+
+typedef struct EventInfoList
+{
+    union {
+        EventInfo *value;
+        uint64_t padding;
+    };
+    struct EventInfoList *next;
+} EventInfoList;
+
+
+typedef struct MigrationStats MigrationStats;
+
+typedef struct MigrationStatsList
+{
+    union {
+        MigrationStats *value;
+        uint64_t padding;
+    };
+    struct MigrationStatsList *next;
+} MigrationStatsList;
+
+
+typedef struct XBZRLECacheStats XBZRLECacheStats;
+
+typedef struct XBZRLECacheStatsList
+{
+    union {
+        XBZRLECacheStats *value;
+        uint64_t padding;
+    };
+    struct XBZRLECacheStatsList *next;
+} XBZRLECacheStatsList;
+
+
+typedef struct MigrationInfo MigrationInfo;
+
+typedef struct MigrationInfoList
+{
+    union {
+        MigrationInfo *value;
+        uint64_t padding;
+    };
+    struct MigrationInfoList *next;
+} MigrationInfoList;
+
+extern const char *MigrationCapability_lookup[];
+typedef enum MigrationCapability
+{
+    MIGRATION_CAPABILITY_XBZRLE = 0,
+    MIGRATION_CAPABILITY_X_RDMA_PIN_ALL = 1,
+    MIGRATION_CAPABILITY_AUTO_CONVERGE = 2,
+    MIGRATION_CAPABILITY_MAX = 3,
+} MigrationCapability;
+
+typedef struct MigrationCapabilityList
+{
+    MigrationCapability value;
+    struct MigrationCapabilityList *next;
+} MigrationCapabilityList;
+
+
+typedef struct MigrationCapabilityStatus MigrationCapabilityStatus;
+
+typedef struct MigrationCapabilityStatusList
+{
+    union {
+        MigrationCapabilityStatus *value;
+        uint64_t padding;
+    };
+    struct MigrationCapabilityStatusList *next;
+} MigrationCapabilityStatusList;
+
+
+typedef struct MouseInfo MouseInfo;
+
+typedef struct MouseInfoList
+{
+    union {
+        MouseInfo *value;
+        uint64_t padding;
+    };
+    struct MouseInfoList *next;
+} MouseInfoList;
+
+
+typedef struct CpuInfo CpuInfo;
+
+typedef struct CpuInfoList
+{
+    union {
+        CpuInfo *value;
+        uint64_t padding;
+    };
+    struct CpuInfoList *next;
+} CpuInfoList;
+
+
+typedef struct BlockDeviceInfo BlockDeviceInfo;
+
+typedef struct BlockDeviceInfoList
+{
+    union {
+        BlockDeviceInfo *value;
+        uint64_t padding;
+    };
+    struct BlockDeviceInfoList *next;
+} BlockDeviceInfoList;
+
+extern const char *BlockDeviceIoStatus_lookup[];
+typedef enum BlockDeviceIoStatus
+{
+    BLOCK_DEVICE_IO_STATUS_OK = 0,
+    BLOCK_DEVICE_IO_STATUS_FAILED = 1,
+    BLOCK_DEVICE_IO_STATUS_NOSPACE = 2,
+    BLOCK_DEVICE_IO_STATUS_MAX = 3,
+} BlockDeviceIoStatus;
+
+typedef struct BlockDeviceIoStatusList
+{
+    BlockDeviceIoStatus value;
+    struct BlockDeviceIoStatusList *next;
+} BlockDeviceIoStatusList;
+
+
+typedef struct BlockDirtyInfo BlockDirtyInfo;
+
+typedef struct BlockDirtyInfoList
+{
+    union {
+        BlockDirtyInfo *value;
+        uint64_t padding;
+    };
+    struct BlockDirtyInfoList *next;
+} BlockDirtyInfoList;
+
+
+typedef struct BlockInfo BlockInfo;
+
+typedef struct BlockInfoList
+{
+    union {
+        BlockInfo *value;
+        uint64_t padding;
+    };
+    struct BlockInfoList *next;
+} BlockInfoList;
+
+
+typedef struct BlockDeviceStats BlockDeviceStats;
+
+typedef struct BlockDeviceStatsList
+{
+    union {
+        BlockDeviceStats *value;
+        uint64_t padding;
+    };
+    struct BlockDeviceStatsList *next;
+} BlockDeviceStatsList;
+
+
+typedef struct BlockStats BlockStats;
+
+typedef struct BlockStatsList
+{
+    union {
+        BlockStats *value;
+        uint64_t padding;
+    };
+    struct BlockStatsList *next;
+} BlockStatsList;
+
+
+typedef struct VncClientInfo VncClientInfo;
+
+typedef struct VncClientInfoList
+{
+    union {
+        VncClientInfo *value;
+        uint64_t padding;
+    };
+    struct VncClientInfoList *next;
+} VncClientInfoList;
+
+
+typedef struct VncInfo VncInfo;
+
+typedef struct VncInfoList
+{
+    union {
+        VncInfo *value;
+        uint64_t padding;
+    };
+    struct VncInfoList *next;
+} VncInfoList;
+
+
+typedef struct SpiceChannel SpiceChannel;
+
+typedef struct SpiceChannelList
+{
+    union {
+        SpiceChannel *value;
+        uint64_t padding;
+    };
+    struct SpiceChannelList *next;
+} SpiceChannelList;
+
+extern const char *SpiceQueryMouseMode_lookup[];
+typedef enum SpiceQueryMouseMode
+{
+    SPICE_QUERY_MOUSE_MODE_CLIENT = 0,
+    SPICE_QUERY_MOUSE_MODE_SERVER = 1,
+    SPICE_QUERY_MOUSE_MODE_UNKNOWN = 2,
+    SPICE_QUERY_MOUSE_MODE_MAX = 3,
+} SpiceQueryMouseMode;
+
+typedef struct SpiceQueryMouseModeList
+{
+    SpiceQueryMouseMode value;
+    struct SpiceQueryMouseModeList *next;
+} SpiceQueryMouseModeList;
+
+
+typedef struct SpiceInfo SpiceInfo;
+
+typedef struct SpiceInfoList
+{
+    union {
+        SpiceInfo *value;
+        uint64_t padding;
+    };
+    struct SpiceInfoList *next;
+} SpiceInfoList;
+
+
+typedef struct BalloonInfo BalloonInfo;
+
+typedef struct BalloonInfoList
+{
+    union {
+        BalloonInfo *value;
+        uint64_t padding;
+    };
+    struct BalloonInfoList *next;
+} BalloonInfoList;
+
+
+typedef struct PciMemoryRange PciMemoryRange;
+
+typedef struct PciMemoryRangeList
+{
+    union {
+        PciMemoryRange *value;
+        uint64_t padding;
+    };
+    struct PciMemoryRangeList *next;
+} PciMemoryRangeList;
+
+
+typedef struct PciMemoryRegion PciMemoryRegion;
+
+typedef struct PciMemoryRegionList
+{
+    union {
+        PciMemoryRegion *value;
+        uint64_t padding;
+    };
+    struct PciMemoryRegionList *next;
+} PciMemoryRegionList;
+
+
+typedef struct PciBridgeInfo PciBridgeInfo;
+
+typedef struct PciBridgeInfoList
+{
+    union {
+        PciBridgeInfo *value;
+        uint64_t padding;
+    };
+    struct PciBridgeInfoList *next;
+} PciBridgeInfoList;
+
+
+typedef struct PciDeviceInfo PciDeviceInfo;
+
+typedef struct PciDeviceInfoList
+{
+    union {
+        PciDeviceInfo *value;
+        uint64_t padding;
+    };
+    struct PciDeviceInfoList *next;
+} PciDeviceInfoList;
+
+
+typedef struct PciInfo PciInfo;
+
+typedef struct PciInfoList
+{
+    union {
+        PciInfo *value;
+        uint64_t padding;
+    };
+    struct PciInfoList *next;
+} PciInfoList;
+
+extern const char *BlockdevOnError_lookup[];
+typedef enum BlockdevOnError
+{
+    BLOCKDEV_ON_ERROR_REPORT = 0,
+    BLOCKDEV_ON_ERROR_IGNORE = 1,
+    BLOCKDEV_ON_ERROR_ENOSPC = 2,
+    BLOCKDEV_ON_ERROR_STOP = 3,
+    BLOCKDEV_ON_ERROR_MAX = 4,
+} BlockdevOnError;
+
+typedef struct BlockdevOnErrorList
+{
+    BlockdevOnError value;
+    struct BlockdevOnErrorList *next;
+} BlockdevOnErrorList;
+
+extern const char *MirrorSyncMode_lookup[];
+typedef enum MirrorSyncMode
+{
+    MIRROR_SYNC_MODE_TOP = 0,
+    MIRROR_SYNC_MODE_FULL = 1,
+    MIRROR_SYNC_MODE_NONE = 2,
+    MIRROR_SYNC_MODE_MAX = 3,
+} MirrorSyncMode;
+
+typedef struct MirrorSyncModeList
+{
+    MirrorSyncMode value;
+    struct MirrorSyncModeList *next;
+} MirrorSyncModeList;
+
+
+typedef struct BlockJobInfo BlockJobInfo;
+
+typedef struct BlockJobInfoList
+{
+    union {
+        BlockJobInfo *value;
+        uint64_t padding;
+    };
+    struct BlockJobInfoList *next;
+} BlockJobInfoList;
+
+extern const char *NewImageMode_lookup[];
+typedef enum NewImageMode
+{
+    NEW_IMAGE_MODE_EXISTING = 0,
+    NEW_IMAGE_MODE_ABSOLUTE_PATHS = 1,
+    NEW_IMAGE_MODE_MAX = 2,
+} NewImageMode;
+
+typedef struct NewImageModeList
+{
+    NewImageMode value;
+    struct NewImageModeList *next;
+} NewImageModeList;
+
+
+typedef struct BlockdevSnapshot BlockdevSnapshot;
+
+typedef struct BlockdevSnapshotList
+{
+    union {
+        BlockdevSnapshot *value;
+        uint64_t padding;
+    };
+    struct BlockdevSnapshotList *next;
+} BlockdevSnapshotList;
+
+
+typedef struct DriveBackup DriveBackup;
+
+typedef struct DriveBackupList
+{
+    union {
+        DriveBackup *value;
+        uint64_t padding;
+    };
+    struct DriveBackupList *next;
+} DriveBackupList;
+
+
+typedef struct Abort Abort;
+
+typedef struct AbortList
+{
+    union {
+        Abort *value;
+        uint64_t padding;
+    };
+    struct AbortList *next;
+} AbortList;
+
+
+typedef struct TransactionAction TransactionAction;
+
+typedef struct TransactionActionList
+{
+    union {
+        TransactionAction *value;
+        uint64_t padding;
+    };
+    struct TransactionActionList *next;
+} TransactionActionList;
+
+extern const char *TransactionActionKind_lookup[];
+typedef enum TransactionActionKind
+{
+    TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC = 0,
+    TRANSACTION_ACTION_KIND_DRIVE_BACKUP = 1,
+    TRANSACTION_ACTION_KIND_ABORT = 2,
+    TRANSACTION_ACTION_KIND_MAX = 3,
+} TransactionActionKind;
+
+
+typedef struct ObjectPropertyInfo ObjectPropertyInfo;
+
+typedef struct ObjectPropertyInfoList
+{
+    union {
+        ObjectPropertyInfo *value;
+        uint64_t padding;
+    };
+    struct ObjectPropertyInfoList *next;
+} ObjectPropertyInfoList;
+
+
+typedef struct ObjectTypeInfo ObjectTypeInfo;
+
+typedef struct ObjectTypeInfoList
+{
+    union {
+        ObjectTypeInfo *value;
+        uint64_t padding;
+    };
+    struct ObjectTypeInfoList *next;
+} ObjectTypeInfoList;
+
+
+typedef struct DevicePropertyInfo DevicePropertyInfo;
+
+typedef struct DevicePropertyInfoList
+{
+    union {
+        DevicePropertyInfo *value;
+        uint64_t padding;
+    };
+    struct DevicePropertyInfoList *next;
+} DevicePropertyInfoList;
+
+
+typedef struct NetdevNoneOptions NetdevNoneOptions;
+
+typedef struct NetdevNoneOptionsList
+{
+    union {
+        NetdevNoneOptions *value;
+        uint64_t padding;
+    };
+    struct NetdevNoneOptionsList *next;
+} NetdevNoneOptionsList;
+
+
+typedef struct NetLegacyNicOptions NetLegacyNicOptions;
+
+typedef struct NetLegacyNicOptionsList
+{
+    union {
+        NetLegacyNicOptions *value;
+        uint64_t padding;
+    };
+    struct NetLegacyNicOptionsList *next;
+} NetLegacyNicOptionsList;
+
+
+typedef struct String String;
+
+typedef struct StringList
+{
+    union {
+        String *value;
+        uint64_t padding;
+    };
+    struct StringList *next;
+} StringList;
+
+
+typedef struct NetdevUserOptions NetdevUserOptions;
+
+typedef struct NetdevUserOptionsList
+{
+    union {
+        NetdevUserOptions *value;
+        uint64_t padding;
+    };
+    struct NetdevUserOptionsList *next;
+} NetdevUserOptionsList;
+
+
+typedef struct NetdevTapOptions NetdevTapOptions;
+
+typedef struct NetdevTapOptionsList
+{
+    union {
+        NetdevTapOptions *value;
+        uint64_t padding;
+    };
+    struct NetdevTapOptionsList *next;
+} NetdevTapOptionsList;
+
+
+typedef struct NetdevSocketOptions NetdevSocketOptions;
+
+typedef struct NetdevSocketOptionsList
+{
+    union {
+        NetdevSocketOptions *value;
+        uint64_t padding;
+    };
+    struct NetdevSocketOptionsList *next;
+} NetdevSocketOptionsList;
+
+
+typedef struct NetdevVdeOptions NetdevVdeOptions;
+
+typedef struct NetdevVdeOptionsList
+{
+    union {
+        NetdevVdeOptions *value;
+        uint64_t padding;
+    };
+    struct NetdevVdeOptionsList *next;
+} NetdevVdeOptionsList;
+
+
+typedef struct NetdevDumpOptions NetdevDumpOptions;
+
+typedef struct NetdevDumpOptionsList
+{
+    union {
+        NetdevDumpOptions *value;
+        uint64_t padding;
+    };
+    struct NetdevDumpOptionsList *next;
+} NetdevDumpOptionsList;
+
+
+typedef struct NetdevBridgeOptions NetdevBridgeOptions;
+
+typedef struct NetdevBridgeOptionsList
+{
+    union {
+        NetdevBridgeOptions *value;
+        uint64_t padding;
+    };
+    struct NetdevBridgeOptionsList *next;
+} NetdevBridgeOptionsList;
+
+
+typedef struct NetdevHubPortOptions NetdevHubPortOptions;
+
+typedef struct NetdevHubPortOptionsList
+{
+    union {
+        NetdevHubPortOptions *value;
+        uint64_t padding;
+    };
+    struct NetdevHubPortOptionsList *next;
+} NetdevHubPortOptionsList;
+
+
+typedef struct NetClientOptions NetClientOptions;
+
+typedef struct NetClientOptionsList
+{
+    union {
+        NetClientOptions *value;
+        uint64_t padding;
+    };
+    struct NetClientOptionsList *next;
+} NetClientOptionsList;
+
+extern const char *NetClientOptionsKind_lookup[];
+typedef enum NetClientOptionsKind
+{
+    NET_CLIENT_OPTIONS_KIND_NONE = 0,
+    NET_CLIENT_OPTIONS_KIND_NIC = 1,
+    NET_CLIENT_OPTIONS_KIND_USER = 2,
+    NET_CLIENT_OPTIONS_KIND_TAP = 3,
+    NET_CLIENT_OPTIONS_KIND_SOCKET = 4,
+    NET_CLIENT_OPTIONS_KIND_VDE = 5,
+    NET_CLIENT_OPTIONS_KIND_DUMP = 6,
+    NET_CLIENT_OPTIONS_KIND_BRIDGE = 7,
+    NET_CLIENT_OPTIONS_KIND_HUBPORT = 8,
+    NET_CLIENT_OPTIONS_KIND_MAX = 9,
+} NetClientOptionsKind;
+
+
+typedef struct NetLegacy NetLegacy;
+
+typedef struct NetLegacyList
+{
+    union {
+        NetLegacy *value;
+        uint64_t padding;
+    };
+    struct NetLegacyList *next;
+} NetLegacyList;
+
+
+typedef struct Netdev Netdev;
+
+typedef struct NetdevList
+{
+    union {
+        Netdev *value;
+        uint64_t padding;
+    };
+    struct NetdevList *next;
+} NetdevList;
+
+
+typedef struct InetSocketAddress InetSocketAddress;
+
+typedef struct InetSocketAddressList
+{
+    union {
+        InetSocketAddress *value;
+        uint64_t padding;
+    };
+    struct InetSocketAddressList *next;
+} InetSocketAddressList;
+
+
+typedef struct UnixSocketAddress UnixSocketAddress;
+
+typedef struct UnixSocketAddressList
+{
+    union {
+        UnixSocketAddress *value;
+        uint64_t padding;
+    };
+    struct UnixSocketAddressList *next;
+} UnixSocketAddressList;
+
+
+typedef struct SocketAddress SocketAddress;
+
+typedef struct SocketAddressList
+{
+    union {
+        SocketAddress *value;
+        uint64_t padding;
+    };
+    struct SocketAddressList *next;
+} SocketAddressList;
+
+extern const char *SocketAddressKind_lookup[];
+typedef enum SocketAddressKind
+{
+    SOCKET_ADDRESS_KIND_INET = 0,
+    SOCKET_ADDRESS_KIND_UNIX = 1,
+    SOCKET_ADDRESS_KIND_FD = 2,
+    SOCKET_ADDRESS_KIND_MAX = 3,
+} SocketAddressKind;
+
+
+typedef struct MachineInfo MachineInfo;
+
+typedef struct MachineInfoList
+{
+    union {
+        MachineInfo *value;
+        uint64_t padding;
+    };
+    struct MachineInfoList *next;
+} MachineInfoList;
+
+
+typedef struct CpuDefinitionInfo CpuDefinitionInfo;
+
+typedef struct CpuDefinitionInfoList
+{
+    union {
+        CpuDefinitionInfo *value;
+        uint64_t padding;
+    };
+    struct CpuDefinitionInfoList *next;
+} CpuDefinitionInfoList;
+
+
+typedef struct AddfdInfo AddfdInfo;
+
+typedef struct AddfdInfoList
+{
+    union {
+        AddfdInfo *value;
+        uint64_t padding;
+    };
+    struct AddfdInfoList *next;
+} AddfdInfoList;
+
+
+typedef struct FdsetFdInfo FdsetFdInfo;
+
+typedef struct FdsetFdInfoList
+{
+    union {
+        FdsetFdInfo *value;
+        uint64_t padding;
+    };
+    struct FdsetFdInfoList *next;
+} FdsetFdInfoList;
+
+
+typedef struct FdsetInfo FdsetInfo;
+
+typedef struct FdsetInfoList
+{
+    union {
+        FdsetInfo *value;
+        uint64_t padding;
+    };
+    struct FdsetInfoList *next;
+} FdsetInfoList;
+
+
+typedef struct TargetInfo TargetInfo;
+
+typedef struct TargetInfoList
+{
+    union {
+        TargetInfo *value;
+        uint64_t padding;
+    };
+    struct TargetInfoList *next;
+} TargetInfoList;
+
+extern const char *QKeyCode_lookup[];
+typedef enum QKeyCode
+{
+    Q_KEY_CODE_SHIFT = 0,
+    Q_KEY_CODE_SHIFT_R = 1,
+    Q_KEY_CODE_ALT = 2,
+    Q_KEY_CODE_ALT_R = 3,
+    Q_KEY_CODE_ALTGR = 4,
+    Q_KEY_CODE_ALTGR_R = 5,
+    Q_KEY_CODE_CTRL = 6,
+    Q_KEY_CODE_CTRL_R = 7,
+    Q_KEY_CODE_MENU = 8,
+    Q_KEY_CODE_ESC = 9,
+    Q_KEY_CODE_1 = 10,
+    Q_KEY_CODE_2 = 11,
+    Q_KEY_CODE_3 = 12,
+    Q_KEY_CODE_4 = 13,
+    Q_KEY_CODE_5 = 14,
+    Q_KEY_CODE_6 = 15,
+    Q_KEY_CODE_7 = 16,
+    Q_KEY_CODE_8 = 17,
+    Q_KEY_CODE_9 = 18,
+    Q_KEY_CODE_0 = 19,
+    Q_KEY_CODE_MINUS = 20,
+    Q_KEY_CODE_EQUAL = 21,
+    Q_KEY_CODE_BACKSPACE = 22,
+    Q_KEY_CODE_TAB = 23,
+    Q_KEY_CODE_Q = 24,
+    Q_KEY_CODE_W = 25,
+    Q_KEY_CODE_E = 26,
+    Q_KEY_CODE_R = 27,
+    Q_KEY_CODE_T = 28,
+    Q_KEY_CODE_Y = 29,
+    Q_KEY_CODE_U = 30,
+    Q_KEY_CODE_I = 31,
+    Q_KEY_CODE_O = 32,
+    Q_KEY_CODE_P = 33,
+    Q_KEY_CODE_BRACKET_LEFT = 34,
+    Q_KEY_CODE_BRACKET_RIGHT = 35,
+    Q_KEY_CODE_RET = 36,
+    Q_KEY_CODE_A = 37,
+    Q_KEY_CODE_S = 38,
+    Q_KEY_CODE_D = 39,
+    Q_KEY_CODE_F = 40,
+    Q_KEY_CODE_G = 41,
+    Q_KEY_CODE_H = 42,
+    Q_KEY_CODE_J = 43,
+    Q_KEY_CODE_K = 44,
+    Q_KEY_CODE_L = 45,
+    Q_KEY_CODE_SEMICOLON = 46,
+    Q_KEY_CODE_APOSTROPHE = 47,
+    Q_KEY_CODE_GRAVE_ACCENT = 48,
+    Q_KEY_CODE_BACKSLASH = 49,
+    Q_KEY_CODE_Z = 50,
+    Q_KEY_CODE_X = 51,
+    Q_KEY_CODE_C = 52,
+    Q_KEY_CODE_V = 53,
+    Q_KEY_CODE_B = 54,
+    Q_KEY_CODE_N = 55,
+    Q_KEY_CODE_M = 56,
+    Q_KEY_CODE_COMMA = 57,
+    Q_KEY_CODE_DOT = 58,
+    Q_KEY_CODE_SLASH = 59,
+    Q_KEY_CODE_ASTERISK = 60,
+    Q_KEY_CODE_SPC = 61,
+    Q_KEY_CODE_CAPS_LOCK = 62,
+    Q_KEY_CODE_F1 = 63,
+    Q_KEY_CODE_F2 = 64,
+    Q_KEY_CODE_F3 = 65,
+    Q_KEY_CODE_F4 = 66,
+    Q_KEY_CODE_F5 = 67,
+    Q_KEY_CODE_F6 = 68,
+    Q_KEY_CODE_F7 = 69,
+    Q_KEY_CODE_F8 = 70,
+    Q_KEY_CODE_F9 = 71,
+    Q_KEY_CODE_F10 = 72,
+    Q_KEY_CODE_NUM_LOCK = 73,
+    Q_KEY_CODE_SCROLL_LOCK = 74,
+    Q_KEY_CODE_KP_DIVIDE = 75,
+    Q_KEY_CODE_KP_MULTIPLY = 76,
+    Q_KEY_CODE_KP_SUBTRACT = 77,
+    Q_KEY_CODE_KP_ADD = 78,
+    Q_KEY_CODE_KP_ENTER = 79,
+    Q_KEY_CODE_KP_DECIMAL = 80,
+    Q_KEY_CODE_SYSRQ = 81,
+    Q_KEY_CODE_KP_0 = 82,
+    Q_KEY_CODE_KP_1 = 83,
+    Q_KEY_CODE_KP_2 = 84,
+    Q_KEY_CODE_KP_3 = 85,
+    Q_KEY_CODE_KP_4 = 86,
+    Q_KEY_CODE_KP_5 = 87,
+    Q_KEY_CODE_KP_6 = 88,
+    Q_KEY_CODE_KP_7 = 89,
+    Q_KEY_CODE_KP_8 = 90,
+    Q_KEY_CODE_KP_9 = 91,
+    Q_KEY_CODE_LESS = 92,
+    Q_KEY_CODE_F11 = 93,
+    Q_KEY_CODE_F12 = 94,
+    Q_KEY_CODE_PRINT = 95,
+    Q_KEY_CODE_HOME = 96,
+    Q_KEY_CODE_PGUP = 97,
+    Q_KEY_CODE_PGDN = 98,
+    Q_KEY_CODE_END = 99,
+    Q_KEY_CODE_LEFT = 100,
+    Q_KEY_CODE_UP = 101,
+    Q_KEY_CODE_DOWN = 102,
+    Q_KEY_CODE_RIGHT = 103,
+    Q_KEY_CODE_INSERT = 104,
+    Q_KEY_CODE_DELETE = 105,
+    Q_KEY_CODE_STOP = 106,
+    Q_KEY_CODE_AGAIN = 107,
+    Q_KEY_CODE_PROPS = 108,
+    Q_KEY_CODE_UNDO = 109,
+    Q_KEY_CODE_FRONT = 110,
+    Q_KEY_CODE_COPY = 111,
+    Q_KEY_CODE_OPEN = 112,
+    Q_KEY_CODE_PASTE = 113,
+    Q_KEY_CODE_FIND = 114,
+    Q_KEY_CODE_CUT = 115,
+    Q_KEY_CODE_LF = 116,
+    Q_KEY_CODE_HELP = 117,
+    Q_KEY_CODE_META_L = 118,
+    Q_KEY_CODE_META_R = 119,
+    Q_KEY_CODE_COMPOSE = 120,
+    Q_KEY_CODE_MAX = 121,
+} QKeyCode;
+
+typedef struct QKeyCodeList
+{
+    QKeyCode value;
+    struct QKeyCodeList *next;
+} QKeyCodeList;
+
+
+typedef struct KeyValue KeyValue;
+
+typedef struct KeyValueList
+{
+    union {
+        KeyValue *value;
+        uint64_t padding;
+    };
+    struct KeyValueList *next;
+} KeyValueList;
+
+extern const char *KeyValueKind_lookup[];
+typedef enum KeyValueKind
+{
+    KEY_VALUE_KIND_NUMBER = 0,
+    KEY_VALUE_KIND_QCODE = 1,
+    KEY_VALUE_KIND_MAX = 2,
+} KeyValueKind;
+
+
+typedef struct ChardevFile ChardevFile;
+
+typedef struct ChardevFileList
+{
+    union {
+        ChardevFile *value;
+        uint64_t padding;
+    };
+    struct ChardevFileList *next;
+} ChardevFileList;
+
+
+typedef struct ChardevHostdev ChardevHostdev;
+
+typedef struct ChardevHostdevList
+{
+    union {
+        ChardevHostdev *value;
+        uint64_t padding;
+    };
+    struct ChardevHostdevList *next;
+} ChardevHostdevList;
+
+
+typedef struct ChardevSocket ChardevSocket;
+
+typedef struct ChardevSocketList
+{
+    union {
+        ChardevSocket *value;
+        uint64_t padding;
+    };
+    struct ChardevSocketList *next;
+} ChardevSocketList;
+
+
+typedef struct ChardevUdp ChardevUdp;
+
+typedef struct ChardevUdpList
+{
+    union {
+        ChardevUdp *value;
+        uint64_t padding;
+    };
+    struct ChardevUdpList *next;
+} ChardevUdpList;
+
+
+typedef struct ChardevMux ChardevMux;
+
+typedef struct ChardevMuxList
+{
+    union {
+        ChardevMux *value;
+        uint64_t padding;
+    };
+    struct ChardevMuxList *next;
+} ChardevMuxList;
+
+
+typedef struct ChardevStdio ChardevStdio;
+
+typedef struct ChardevStdioList
+{
+    union {
+        ChardevStdio *value;
+        uint64_t padding;
+    };
+    struct ChardevStdioList *next;
+} ChardevStdioList;
+
+
+typedef struct ChardevSpiceChannel ChardevSpiceChannel;
+
+typedef struct ChardevSpiceChannelList
+{
+    union {
+        ChardevSpiceChannel *value;
+        uint64_t padding;
+    };
+    struct ChardevSpiceChannelList *next;
+} ChardevSpiceChannelList;
+
+
+typedef struct ChardevSpicePort ChardevSpicePort;
+
+typedef struct ChardevSpicePortList
+{
+    union {
+        ChardevSpicePort *value;
+        uint64_t padding;
+    };
+    struct ChardevSpicePortList *next;
+} ChardevSpicePortList;
+
+
+typedef struct ChardevVC ChardevVC;
+
+typedef struct ChardevVCList
+{
+    union {
+        ChardevVC *value;
+        uint64_t padding;
+    };
+    struct ChardevVCList *next;
+} ChardevVCList;
+
+
+typedef struct ChardevMemory ChardevMemory;
+
+typedef struct ChardevMemoryList
+{
+    union {
+        ChardevMemory *value;
+        uint64_t padding;
+    };
+    struct ChardevMemoryList *next;
+} ChardevMemoryList;
+
+
+typedef struct ChardevDummy ChardevDummy;
+
+typedef struct ChardevDummyList
+{
+    union {
+        ChardevDummy *value;
+        uint64_t padding;
+    };
+    struct ChardevDummyList *next;
+} ChardevDummyList;
+
+
+typedef struct ChardevBackend ChardevBackend;
+
+typedef struct ChardevBackendList
+{
+    union {
+        ChardevBackend *value;
+        uint64_t padding;
+    };
+    struct ChardevBackendList *next;
+} ChardevBackendList;
+
+extern const char *ChardevBackendKind_lookup[];
+typedef enum ChardevBackendKind
+{
+    CHARDEV_BACKEND_KIND_FILE = 0,
+    CHARDEV_BACKEND_KIND_SERIAL = 1,
+    CHARDEV_BACKEND_KIND_PARALLEL = 2,
+    CHARDEV_BACKEND_KIND_PIPE = 3,
+    CHARDEV_BACKEND_KIND_SOCKET = 4,
+    CHARDEV_BACKEND_KIND_UDP = 5,
+    CHARDEV_BACKEND_KIND_PTY = 6,
+    CHARDEV_BACKEND_KIND_NULL = 7,
+    CHARDEV_BACKEND_KIND_MUX = 8,
+    CHARDEV_BACKEND_KIND_MSMOUSE = 9,
+    CHARDEV_BACKEND_KIND_BRAILLE = 10,
+    CHARDEV_BACKEND_KIND_STDIO = 11,
+    CHARDEV_BACKEND_KIND_CONSOLE = 12,
+    CHARDEV_BACKEND_KIND_SPICEVMC = 13,
+    CHARDEV_BACKEND_KIND_SPICEPORT = 14,
+    CHARDEV_BACKEND_KIND_VC = 15,
+    CHARDEV_BACKEND_KIND_MEMORY = 16,
+    CHARDEV_BACKEND_KIND_MAX = 17,
+} ChardevBackendKind;
+
+
+typedef struct ChardevReturn ChardevReturn;
+
+typedef struct ChardevReturnList
+{
+    union {
+        ChardevReturn *value;
+        uint64_t padding;
+    };
+    struct ChardevReturnList *next;
+} ChardevReturnList;
+
+extern const char *TpmModel_lookup[];
+typedef enum TpmModel
+{
+    TPM_MODEL_TPM_TIS = 0,
+    TPM_MODEL_MAX = 1,
+} TpmModel;
+
+typedef struct TpmModelList
+{
+    TpmModel value;
+    struct TpmModelList *next;
+} TpmModelList;
+
+extern const char *TpmType_lookup[];
+typedef enum TpmType
+{
+    TPM_TYPE_PASSTHROUGH = 0,
+    TPM_TYPE_MAX = 1,
+} TpmType;
+
+typedef struct TpmTypeList
+{
+    TpmType value;
+    struct TpmTypeList *next;
+} TpmTypeList;
+
+
+typedef struct TPMPassthroughOptions TPMPassthroughOptions;
+
+typedef struct TPMPassthroughOptionsList
+{
+    union {
+        TPMPassthroughOptions *value;
+        uint64_t padding;
+    };
+    struct TPMPassthroughOptionsList *next;
+} TPMPassthroughOptionsList;
+
+
+typedef struct TpmTypeOptions TpmTypeOptions;
+
+typedef struct TpmTypeOptionsList
+{
+    union {
+        TpmTypeOptions *value;
+        uint64_t padding;
+    };
+    struct TpmTypeOptionsList *next;
+} TpmTypeOptionsList;
+
+extern const char *TpmTypeOptionsKind_lookup[];
+typedef enum TpmTypeOptionsKind
+{
+    TPM_TYPE_OPTIONS_KIND_PASSTHROUGH = 0,
+    TPM_TYPE_OPTIONS_KIND_MAX = 1,
+} TpmTypeOptionsKind;
+
+
+typedef struct TPMInfo TPMInfo;
+
+typedef struct TPMInfoList
+{
+    union {
+        TPMInfo *value;
+        uint64_t padding;
+    };
+    struct TPMInfoList *next;
+} TPMInfoList;
+
+
+typedef struct AcpiTableOptions AcpiTableOptions;
+
+typedef struct AcpiTableOptionsList
+{
+    union {
+        AcpiTableOptions *value;
+        uint64_t padding;
+    };
+    struct AcpiTableOptionsList *next;
+} AcpiTableOptionsList;
+
+extern const char *CommandLineParameterType_lookup[];
+typedef enum CommandLineParameterType
+{
+    COMMAND_LINE_PARAMETER_TYPE_STRING = 0,
+    COMMAND_LINE_PARAMETER_TYPE_BOOLEAN = 1,
+    COMMAND_LINE_PARAMETER_TYPE_NUMBER = 2,
+    COMMAND_LINE_PARAMETER_TYPE_SIZE = 3,
+    COMMAND_LINE_PARAMETER_TYPE_MAX = 4,
+} CommandLineParameterType;
+
+typedef struct CommandLineParameterTypeList
+{
+    CommandLineParameterType value;
+    struct CommandLineParameterTypeList *next;
+} CommandLineParameterTypeList;
+
+
+typedef struct CommandLineParameterInfo CommandLineParameterInfo;
+
+typedef struct CommandLineParameterInfoList
+{
+    union {
+        CommandLineParameterInfo *value;
+        uint64_t padding;
+    };
+    struct CommandLineParameterInfoList *next;
+} CommandLineParameterInfoList;
+
+
+typedef struct CommandLineOptionInfo CommandLineOptionInfo;
+
+typedef struct CommandLineOptionInfoList
+{
+    union {
+        CommandLineOptionInfo *value;
+        uint64_t padding;
+    };
+    struct CommandLineOptionInfoList *next;
+} CommandLineOptionInfoList;
+
+extern const char *X86CPURegister32_lookup[];
+typedef enum X86CPURegister32
+{
+    X86_C_P_U_REGISTER32_EAX = 0,
+    X86_C_P_U_REGISTER32_EBX = 1,
+    X86_C_P_U_REGISTER32_ECX = 2,
+    X86_C_P_U_REGISTER32_EDX = 3,
+    X86_C_P_U_REGISTER32_ESP = 4,
+    X86_C_P_U_REGISTER32_EBP = 5,
+    X86_C_P_U_REGISTER32_ESI = 6,
+    X86_C_P_U_REGISTER32_EDI = 7,
+    X86_C_P_U_REGISTER32_MAX = 8,
+} X86CPURegister32;
+
+typedef struct X86CPURegister32List
+{
+    X86CPURegister32 value;
+    struct X86CPURegister32List *next;
+} X86CPURegister32List;
+
+
+typedef struct X86CPUFeatureWordInfo X86CPUFeatureWordInfo;
+
+typedef struct X86CPUFeatureWordInfoList
+{
+    union {
+        X86CPUFeatureWordInfo *value;
+        uint64_t padding;
+    };
+    struct X86CPUFeatureWordInfoList *next;
+} X86CPUFeatureWordInfoList;
+
+extern const char *RxState_lookup[];
+typedef enum RxState
+{
+    RX_STATE_NORMAL = 0,
+    RX_STATE_NONE = 1,
+    RX_STATE_ALL = 2,
+    RX_STATE_MAX = 3,
+} RxState;
+
+typedef struct RxStateList
+{
+    RxState value;
+    struct RxStateList *next;
+} RxStateList;
+
+
+typedef struct RxFilterInfo RxFilterInfo;
+
+typedef struct RxFilterInfoList
+{
+    union {
+        RxFilterInfo *value;
+        uint64_t padding;
+    };
+    struct RxFilterInfoList *next;
+} RxFilterInfoList;
+
+#ifndef QAPI_TYPES_BUILTIN_CLEANUP_DECL_H
+#define QAPI_TYPES_BUILTIN_CLEANUP_DECL_H
+
+void qapi_free_strList(strList * obj);
+void qapi_free_intList(intList * obj);
+void qapi_free_numberList(numberList * obj);
+void qapi_free_boolList(boolList * obj);
+void qapi_free_int8List(int8List * obj);
+void qapi_free_int16List(int16List * obj);
+void qapi_free_int32List(int32List * obj);
+void qapi_free_int64List(int64List * obj);
+void qapi_free_uint8List(uint8List * obj);
+void qapi_free_uint16List(uint16List * obj);
+void qapi_free_uint32List(uint32List * obj);
+void qapi_free_uint64List(uint64List * obj);
+
+#endif /* QAPI_TYPES_BUILTIN_CLEANUP_DECL_H */
+
+
+void qapi_free_ErrorClassList(ErrorClassList * obj);
+
+struct NameInfo
+{
+    bool has_name;
+    char * name;
+};
+
+void qapi_free_NameInfoList(NameInfoList * obj);
+void qapi_free_NameInfo(NameInfo * obj);
+
+struct VersionInfo
+{
+    struct 
+    {
+        int64_t major;
+        int64_t minor;
+        int64_t micro;
+    } qemu;
+    char * package;
+};
+
+void qapi_free_VersionInfoList(VersionInfoList * obj);
+void qapi_free_VersionInfo(VersionInfo * obj);
+
+struct KvmInfo
+{
+    bool enabled;
+    bool present;
+};
+
+void qapi_free_KvmInfoList(KvmInfoList * obj);
+void qapi_free_KvmInfo(KvmInfo * obj);
+
+void qapi_free_RunStateList(RunStateList * obj);
+
+struct SnapshotInfo
+{
+    char * id;
+    char * name;
+    int64_t vm_state_size;
+    int64_t date_sec;
+    int64_t date_nsec;
+    int64_t vm_clock_sec;
+    int64_t vm_clock_nsec;
+};
+
+void qapi_free_SnapshotInfoList(SnapshotInfoList * obj);
+void qapi_free_SnapshotInfo(SnapshotInfo * obj);
+
+struct ImageInfo
+{
+    char * filename;
+    char * format;
+    bool has_dirty_flag;
+    bool dirty_flag;
+    bool has_actual_size;
+    int64_t actual_size;
+    int64_t virtual_size;
+    bool has_cluster_size;
+    int64_t cluster_size;
+    bool has_encrypted;
+    bool encrypted;
+    bool has_backing_filename;
+    char * backing_filename;
+    bool has_full_backing_filename;
+    char * full_backing_filename;
+    bool has_backing_filename_format;
+    char * backing_filename_format;
+    bool has_snapshots;
+    SnapshotInfoList * snapshots;
+    bool has_backing_image;
+    ImageInfo * backing_image;
+};
+
+void qapi_free_ImageInfoList(ImageInfoList * obj);
+void qapi_free_ImageInfo(ImageInfo * obj);
+
+struct ImageCheck
+{
+    char * filename;
+    char * format;
+    int64_t check_errors;
+    bool has_image_end_offset;
+    int64_t image_end_offset;
+    bool has_corruptions;
+    int64_t corruptions;
+    bool has_leaks;
+    int64_t leaks;
+    bool has_corruptions_fixed;
+    int64_t corruptions_fixed;
+    bool has_leaks_fixed;
+    int64_t leaks_fixed;
+    bool has_total_clusters;
+    int64_t total_clusters;
+    bool has_allocated_clusters;
+    int64_t allocated_clusters;
+    bool has_fragmented_clusters;
+    int64_t fragmented_clusters;
+    bool has_compressed_clusters;
+    int64_t compressed_clusters;
+};
+
+void qapi_free_ImageCheckList(ImageCheckList * obj);
+void qapi_free_ImageCheck(ImageCheck * obj);
+
+struct StatusInfo
+{
+    bool running;
+    bool singlestep;
+    RunState status;
+};
+
+void qapi_free_StatusInfoList(StatusInfoList * obj);
+void qapi_free_StatusInfo(StatusInfo * obj);
+
+struct UuidInfo
+{
+    char * UUID;
+};
+
+void qapi_free_UuidInfoList(UuidInfoList * obj);
+void qapi_free_UuidInfo(UuidInfo * obj);
+
+struct ChardevInfo
+{
+    char * label;
+    char * filename;
+};
+
+void qapi_free_ChardevInfoList(ChardevInfoList * obj);
+void qapi_free_ChardevInfo(ChardevInfo * obj);
+
+void qapi_free_DataFormatList(DataFormatList * obj);
+
+struct CommandInfo
+{
+    char * name;
+};
+
+void qapi_free_CommandInfoList(CommandInfoList * obj);
+void qapi_free_CommandInfo(CommandInfo * obj);
+
+struct EventInfo
+{
+    char * name;
+};
+
+void qapi_free_EventInfoList(EventInfoList * obj);
+void qapi_free_EventInfo(EventInfo * obj);
+
+struct MigrationStats
+{
+    int64_t transferred;
+    int64_t remaining;
+    int64_t total;
+    int64_t duplicate;
+    int64_t skipped;
+    int64_t normal;
+    int64_t normal_bytes;
+    int64_t dirty_pages_rate;
+    double mbps;
+};
+
+void qapi_free_MigrationStatsList(MigrationStatsList * obj);
+void qapi_free_MigrationStats(MigrationStats * obj);
+
+struct XBZRLECacheStats
+{
+    int64_t cache_size;
+    int64_t bytes;
+    int64_t pages;
+    int64_t cache_miss;
+    int64_t overflow;
+};
+
+void qapi_free_XBZRLECacheStatsList(XBZRLECacheStatsList * obj);
+void qapi_free_XBZRLECacheStats(XBZRLECacheStats * obj);
+
+struct MigrationInfo
+{
+    bool has_status;
+    char * status;
+    bool has_ram;
+    MigrationStats * ram;
+    bool has_disk;
+    MigrationStats * disk;
+    bool has_xbzrle_cache;
+    XBZRLECacheStats * xbzrle_cache;
+    bool has_total_time;
+    int64_t total_time;
+    bool has_expected_downtime;
+    int64_t expected_downtime;
+    bool has_downtime;
+    int64_t downtime;
+};
+
+void qapi_free_MigrationInfoList(MigrationInfoList * obj);
+void qapi_free_MigrationInfo(MigrationInfo * obj);
+
+void qapi_free_MigrationCapabilityList(MigrationCapabilityList * obj);
+
+struct MigrationCapabilityStatus
+{
+    MigrationCapability capability;
+    bool state;
+};
+
+void qapi_free_MigrationCapabilityStatusList(MigrationCapabilityStatusList * obj);
+void qapi_free_MigrationCapabilityStatus(MigrationCapabilityStatus * obj);
+
+struct MouseInfo
+{
+    char * name;
+    int64_t index;
+    bool current;
+    bool absolute;
+};
+
+void qapi_free_MouseInfoList(MouseInfoList * obj);
+void qapi_free_MouseInfo(MouseInfo * obj);
+
+struct CpuInfo
+{
+    int64_t CPU;
+    bool current;
+    bool halted;
+    bool has_pc;
+    int64_t pc;
+    bool has_nip;
+    int64_t nip;
+    bool has_npc;
+    int64_t npc;
+    bool has_PC;
+    int64_t PC;
+    int64_t thread_id;
+};
+
+void qapi_free_CpuInfoList(CpuInfoList * obj);
+void qapi_free_CpuInfo(CpuInfo * obj);
+
+struct BlockDeviceInfo
+{
+    char * file;
+    bool ro;
+    char * drv;
+    bool has_backing_file;
+    char * backing_file;
+    int64_t backing_file_depth;
+    bool encrypted;
+    bool encryption_key_missing;
+    int64_t bps;
+    int64_t bps_rd;
+    int64_t bps_wr;
+    int64_t iops;
+    int64_t iops_rd;
+    int64_t iops_wr;
+    ImageInfo * image;
+};
+
+void qapi_free_BlockDeviceInfoList(BlockDeviceInfoList * obj);
+void qapi_free_BlockDeviceInfo(BlockDeviceInfo * obj);
+
+void qapi_free_BlockDeviceIoStatusList(BlockDeviceIoStatusList * obj);
+
+struct BlockDirtyInfo
+{
+    int64_t count;
+    int64_t granularity;
+};
+
+void qapi_free_BlockDirtyInfoList(BlockDirtyInfoList * obj);
+void qapi_free_BlockDirtyInfo(BlockDirtyInfo * obj);
+
+struct BlockInfo
+{
+    char * device;
+    char * type;
+    bool removable;
+    bool locked;
+    bool has_inserted;
+    BlockDeviceInfo * inserted;
+    bool has_tray_open;
+    bool tray_open;
+    bool has_io_status;
+    BlockDeviceIoStatus io_status;
+    bool has_dirty;
+    BlockDirtyInfo * dirty;
+};
+
+void qapi_free_BlockInfoList(BlockInfoList * obj);
+void qapi_free_BlockInfo(BlockInfo * obj);
+
+struct BlockDeviceStats
+{
+    int64_t rd_bytes;
+    int64_t wr_bytes;
+    int64_t rd_operations;
+    int64_t wr_operations;
+    int64_t flush_operations;
+    int64_t flush_total_time_ns;
+    int64_t wr_total_time_ns;
+    int64_t rd_total_time_ns;
+    int64_t wr_highest_offset;
+};
+
+void qapi_free_BlockDeviceStatsList(BlockDeviceStatsList * obj);
+void qapi_free_BlockDeviceStats(BlockDeviceStats * obj);
+
+struct BlockStats
+{
+    bool has_device;
+    char * device;
+    BlockDeviceStats * stats;
+    bool has_parent;
+    BlockStats * parent;
+};
+
+void qapi_free_BlockStatsList(BlockStatsList * obj);
+void qapi_free_BlockStats(BlockStats * obj);
+
+struct VncClientInfo
+{
+    char * host;
+    char * family;
+    char * service;
+    bool has_x509_dname;
+    char * x509_dname;
+    bool has_sasl_username;
+    char * sasl_username;
+};
+
+void qapi_free_VncClientInfoList(VncClientInfoList * obj);
+void qapi_free_VncClientInfo(VncClientInfo * obj);
+
+struct VncInfo
+{
+    bool enabled;
+    bool has_host;
+    char * host;
+    bool has_family;
+    char * family;
+    bool has_service;
+    char * service;
+    bool has_auth;
+    char * auth;
+    bool has_clients;
+    VncClientInfoList * clients;
+};
+
+void qapi_free_VncInfoList(VncInfoList * obj);
+void qapi_free_VncInfo(VncInfo * obj);
+
+struct SpiceChannel
+{
+    char * host;
+    char * family;
+    char * port;
+    int64_t connection_id;
+    int64_t channel_type;
+    int64_t channel_id;
+    bool tls;
+};
+
+void qapi_free_SpiceChannelList(SpiceChannelList * obj);
+void qapi_free_SpiceChannel(SpiceChannel * obj);
+
+void qapi_free_SpiceQueryMouseModeList(SpiceQueryMouseModeList * obj);
+
+struct SpiceInfo
+{
+    bool enabled;
+    bool migrated;
+    bool has_host;
+    char * host;
+    bool has_port;
+    int64_t port;
+    bool has_tls_port;
+    int64_t tls_port;
+    bool has_auth;
+    char * auth;
+    bool has_compiled_version;
+    char * compiled_version;
+    SpiceQueryMouseMode mouse_mode;
+    bool has_channels;
+    SpiceChannelList * channels;
+};
+
+void qapi_free_SpiceInfoList(SpiceInfoList * obj);
+void qapi_free_SpiceInfo(SpiceInfo * obj);
+
+struct BalloonInfo
+{
+    int64_t actual;
+};
+
+void qapi_free_BalloonInfoList(BalloonInfoList * obj);
+void qapi_free_BalloonInfo(BalloonInfo * obj);
+
+struct PciMemoryRange
+{
+    int64_t base;
+    int64_t limit;
+};
+
+void qapi_free_PciMemoryRangeList(PciMemoryRangeList * obj);
+void qapi_free_PciMemoryRange(PciMemoryRange * obj);
+
+struct PciMemoryRegion
+{
+    int64_t bar;
+    char * type;
+    int64_t address;
+    int64_t size;
+    bool has_prefetch;
+    bool prefetch;
+    bool has_mem_type_64;
+    bool mem_type_64;
+};
+
+void qapi_free_PciMemoryRegionList(PciMemoryRegionList * obj);
+void qapi_free_PciMemoryRegion(PciMemoryRegion * obj);
+
+struct PciBridgeInfo
+{
+    struct 
+    {
+        int64_t number;
+        int64_t secondary;
+        int64_t subordinate;
+        PciMemoryRange * io_range;
+        PciMemoryRange * memory_range;
+        PciMemoryRange * prefetchable_range;
+    } bus;
+    bool has_devices;
+    PciDeviceInfoList * devices;
+};
+
+void qapi_free_PciBridgeInfoList(PciBridgeInfoList * obj);
+void qapi_free_PciBridgeInfo(PciBridgeInfo * obj);
+
+struct PciDeviceInfo
+{
+    int64_t bus;
+    int64_t slot;
+    int64_t function;
+    struct 
+    {
+        bool has_desc;
+        char * desc;
+        int64_t class;
+    } class_info;
+    struct 
+    {
+        int64_t device;
+        int64_t vendor;
+    } id;
+    bool has_irq;
+    int64_t irq;
+    char * qdev_id;
+    bool has_pci_bridge;
+    PciBridgeInfo * pci_bridge;
+    PciMemoryRegionList * regions;
+};
+
+void qapi_free_PciDeviceInfoList(PciDeviceInfoList * obj);
+void qapi_free_PciDeviceInfo(PciDeviceInfo * obj);
+
+struct PciInfo
+{
+    int64_t bus;
+    PciDeviceInfoList * devices;
+};
+
+void qapi_free_PciInfoList(PciInfoList * obj);
+void qapi_free_PciInfo(PciInfo * obj);
+
+void qapi_free_BlockdevOnErrorList(BlockdevOnErrorList * obj);
+
+void qapi_free_MirrorSyncModeList(MirrorSyncModeList * obj);
+
+struct BlockJobInfo
+{
+    char * type;
+    char * device;
+    int64_t len;
+    int64_t offset;
+    bool busy;
+    bool paused;
+    int64_t speed;
+    BlockDeviceIoStatus io_status;
+};
+
+void qapi_free_BlockJobInfoList(BlockJobInfoList * obj);
+void qapi_free_BlockJobInfo(BlockJobInfo * obj);
+
+void qapi_free_NewImageModeList(NewImageModeList * obj);
+
+struct BlockdevSnapshot
+{
+    char * device;
+    char * snapshot_file;
+    bool has_format;
+    char * format;
+    bool has_mode;
+    NewImageMode mode;
+};
+
+void qapi_free_BlockdevSnapshotList(BlockdevSnapshotList * obj);
+void qapi_free_BlockdevSnapshot(BlockdevSnapshot * obj);
+
+struct DriveBackup
+{
+    char * device;
+    char * target;
+    bool has_format;
+    char * format;
+    MirrorSyncMode sync;
+    bool has_mode;
+    NewImageMode mode;
+    bool has_speed;
+    int64_t speed;
+    bool has_on_source_error;
+    BlockdevOnError on_source_error;
+    bool has_on_target_error;
+    BlockdevOnError on_target_error;
+};
+
+void qapi_free_DriveBackupList(DriveBackupList * obj);
+void qapi_free_DriveBackup(DriveBackup * obj);
+
+struct Abort
+{
+};
+
+void qapi_free_AbortList(AbortList * obj);
+void qapi_free_Abort(Abort * obj);
+
+struct TransactionAction
+{
+    TransactionActionKind kind;
+    union {
+        void *data;
+        BlockdevSnapshot * blockdev_snapshot_sync;
+        DriveBackup * drive_backup;
+        Abort * abort;
+    };
+};
+void qapi_free_TransactionActionList(TransactionActionList * obj);
+void qapi_free_TransactionAction(TransactionAction * obj);
+
+struct ObjectPropertyInfo
+{
+    char * name;
+    char * type;
+};
+
+void qapi_free_ObjectPropertyInfoList(ObjectPropertyInfoList * obj);
+void qapi_free_ObjectPropertyInfo(ObjectPropertyInfo * obj);
+
+struct ObjectTypeInfo
+{
+    char * name;
+};
+
+void qapi_free_ObjectTypeInfoList(ObjectTypeInfoList * obj);
+void qapi_free_ObjectTypeInfo(ObjectTypeInfo * obj);
+
+struct DevicePropertyInfo
+{
+    char * name;
+    char * type;
+};
+
+void qapi_free_DevicePropertyInfoList(DevicePropertyInfoList * obj);
+void qapi_free_DevicePropertyInfo(DevicePropertyInfo * obj);
+
+struct NetdevNoneOptions
+{
+};
+
+void qapi_free_NetdevNoneOptionsList(NetdevNoneOptionsList * obj);
+void qapi_free_NetdevNoneOptions(NetdevNoneOptions * obj);
+
+struct NetLegacyNicOptions
+{
+    bool has_netdev;
+    char * netdev;
+    bool has_macaddr;
+    char * macaddr;
+    bool has_model;
+    char * model;
+    bool has_addr;
+    char * addr;
+    bool has_vectors;
+    uint32_t vectors;
+};
+
+void qapi_free_NetLegacyNicOptionsList(NetLegacyNicOptionsList * obj);
+void qapi_free_NetLegacyNicOptions(NetLegacyNicOptions * obj);
+
+struct String
+{
+    char * str;
+};
+
+void qapi_free_StringList(StringList * obj);
+void qapi_free_String(String * obj);
+
+struct NetdevUserOptions
+{
+    bool has_hostname;
+    char * hostname;
+    bool has_q_restrict;
+    bool q_restrict;
+    bool has_ip;
+    char * ip;
+    bool has_net;
+    char * net;
+    bool has_host;
+    char * host;
+    bool has_tftp;
+    char * tftp;
+    bool has_bootfile;
+    char * bootfile;
+    bool has_dhcpstart;
+    char * dhcpstart;
+    bool has_dns;
+    char * dns;
+    bool has_dnssearch;
+    StringList * dnssearch;
+    bool has_smb;
+    char * smb;
+    bool has_smbserver;
+    char * smbserver;
+    bool has_hostfwd;
+    StringList * hostfwd;
+    bool has_guestfwd;
+    StringList * guestfwd;
+};
+
+void qapi_free_NetdevUserOptionsList(NetdevUserOptionsList * obj);
+void qapi_free_NetdevUserOptions(NetdevUserOptions * obj);
+
+struct NetdevTapOptions
+{
+    bool has_ifname;
+    char * ifname;
+    bool has_fd;
+    char * fd;
+    bool has_fds;
+    char * fds;
+    bool has_script;
+    char * script;
+    bool has_downscript;
+    char * downscript;
+    bool has_helper;
+    char * helper;
+    bool has_sndbuf;
+    uint64_t sndbuf;
+    bool has_vnet_hdr;
+    bool vnet_hdr;
+    bool has_vhost;
+    bool vhost;
+    bool has_vhostfd;
+    char * vhostfd;
+    bool has_vhostfds;
+    char * vhostfds;
+    bool has_vhostforce;
+    bool vhostforce;
+    bool has_queues;
+    uint32_t queues;
+};
+
+void qapi_free_NetdevTapOptionsList(NetdevTapOptionsList * obj);
+void qapi_free_NetdevTapOptions(NetdevTapOptions * obj);
+
+struct NetdevSocketOptions
+{
+    bool has_fd;
+    char * fd;
+    bool has_listen;
+    char * listen;
+    bool has_connect;
+    char * connect;
+    bool has_mcast;
+    char * mcast;
+    bool has_localaddr;
+    char * localaddr;
+    bool has_udp;
+    char * udp;
+};
+
+void qapi_free_NetdevSocketOptionsList(NetdevSocketOptionsList * obj);
+void qapi_free_NetdevSocketOptions(NetdevSocketOptions * obj);
+
+struct NetdevVdeOptions
+{
+    bool has_sock;
+    char * sock;
+    bool has_port;
+    uint16_t port;
+    bool has_group;
+    char * group;
+    bool has_mode;
+    uint16_t mode;
+};
+
+void qapi_free_NetdevVdeOptionsList(NetdevVdeOptionsList * obj);
+void qapi_free_NetdevVdeOptions(NetdevVdeOptions * obj);
+
+struct NetdevDumpOptions
+{
+    bool has_len;
+    uint64_t len;
+    bool has_file;
+    char * file;
+};
+
+void qapi_free_NetdevDumpOptionsList(NetdevDumpOptionsList * obj);
+void qapi_free_NetdevDumpOptions(NetdevDumpOptions * obj);
+
+struct NetdevBridgeOptions
+{
+    bool has_br;
+    char * br;
+    bool has_helper;
+    char * helper;
+};
+
+void qapi_free_NetdevBridgeOptionsList(NetdevBridgeOptionsList * obj);
+void qapi_free_NetdevBridgeOptions(NetdevBridgeOptions * obj);
+
+struct NetdevHubPortOptions
+{
+    int32_t hubid;
+};
+
+void qapi_free_NetdevHubPortOptionsList(NetdevHubPortOptionsList * obj);
+void qapi_free_NetdevHubPortOptions(NetdevHubPortOptions * obj);
+
+struct NetClientOptions
+{
+    NetClientOptionsKind kind;
+    union {
+        void *data;
+        NetdevNoneOptions * none;
+        NetLegacyNicOptions * nic;
+        NetdevUserOptions * user;
+        NetdevTapOptions * tap;
+        NetdevSocketOptions * socket;
+        NetdevVdeOptions * vde;
+        NetdevDumpOptions * dump;
+        NetdevBridgeOptions * bridge;
+        NetdevHubPortOptions * hubport;
+    };
+};
+void qapi_free_NetClientOptionsList(NetClientOptionsList * obj);
+void qapi_free_NetClientOptions(NetClientOptions * obj);
+
+struct NetLegacy
+{
+    bool has_vlan;
+    int32_t vlan;
+    bool has_id;
+    char * id;
+    bool has_name;
+    char * name;
+    NetClientOptions * opts;
+};
+
+void qapi_free_NetLegacyList(NetLegacyList * obj);
+void qapi_free_NetLegacy(NetLegacy * obj);
+
+struct Netdev
+{
+    char * id;
+    NetClientOptions * opts;
+};
+
+void qapi_free_NetdevList(NetdevList * obj);
+void qapi_free_Netdev(Netdev * obj);
+
+struct InetSocketAddress
+{
+    char * host;
+    char * port;
+    bool has_to;
+    uint16_t to;
+    bool has_ipv4;
+    bool ipv4;
+    bool has_ipv6;
+    bool ipv6;
+};
+
+void qapi_free_InetSocketAddressList(InetSocketAddressList * obj);
+void qapi_free_InetSocketAddress(InetSocketAddress * obj);
+
+struct UnixSocketAddress
+{
+    char * path;
+};
+
+void qapi_free_UnixSocketAddressList(UnixSocketAddressList * obj);
+void qapi_free_UnixSocketAddress(UnixSocketAddress * obj);
+
+struct SocketAddress
+{
+    SocketAddressKind kind;
+    union {
+        void *data;
+        InetSocketAddress * inet;
+        UnixSocketAddress * q_unix;
+        String * fd;
+    };
+};
+void qapi_free_SocketAddressList(SocketAddressList * obj);
+void qapi_free_SocketAddress(SocketAddress * obj);
+
+struct MachineInfo
+{
+    char * name;
+    bool has_alias;
+    char * alias;
+    bool has_is_default;
+    bool is_default;
+    int64_t cpu_max;
+};
+
+void qapi_free_MachineInfoList(MachineInfoList * obj);
+void qapi_free_MachineInfo(MachineInfo * obj);
+
+struct CpuDefinitionInfo
+{
+    char * name;
+};
+
+void qapi_free_CpuDefinitionInfoList(CpuDefinitionInfoList * obj);
+void qapi_free_CpuDefinitionInfo(CpuDefinitionInfo * obj);
+
+struct AddfdInfo
+{
+    int64_t fdset_id;
+    int64_t fd;
+};
+
+void qapi_free_AddfdInfoList(AddfdInfoList * obj);
+void qapi_free_AddfdInfo(AddfdInfo * obj);
+
+struct FdsetFdInfo
+{
+    int64_t fd;
+    bool has_opaque;
+    char * opaque;
+};
+
+void qapi_free_FdsetFdInfoList(FdsetFdInfoList * obj);
+void qapi_free_FdsetFdInfo(FdsetFdInfo * obj);
+
+struct FdsetInfo
+{
+    int64_t fdset_id;
+    FdsetFdInfoList * fds;
+};
+
+void qapi_free_FdsetInfoList(FdsetInfoList * obj);
+void qapi_free_FdsetInfo(FdsetInfo * obj);
+
+struct TargetInfo
+{
+    char * arch;
+};
+
+void qapi_free_TargetInfoList(TargetInfoList * obj);
+void qapi_free_TargetInfo(TargetInfo * obj);
+
+void qapi_free_QKeyCodeList(QKeyCodeList * obj);
+
+struct KeyValue
+{
+    KeyValueKind kind;
+    union {
+        void *data;
+        int64_t number;
+        QKeyCode qcode;
+    };
+};
+void qapi_free_KeyValueList(KeyValueList * obj);
+void qapi_free_KeyValue(KeyValue * obj);
+
+struct ChardevFile
+{
+    bool has_in;
+    char * in;
+    char * out;
+};
+
+void qapi_free_ChardevFileList(ChardevFileList * obj);
+void qapi_free_ChardevFile(ChardevFile * obj);
+
+struct ChardevHostdev
+{
+    char * device;
+};
+
+void qapi_free_ChardevHostdevList(ChardevHostdevList * obj);
+void qapi_free_ChardevHostdev(ChardevHostdev * obj);
+
+struct ChardevSocket
+{
+    SocketAddress * addr;
+    bool has_server;
+    bool server;
+    bool has_wait;
+    bool wait;
+    bool has_nodelay;
+    bool nodelay;
+    bool has_telnet;
+    bool telnet;
+};
+
+void qapi_free_ChardevSocketList(ChardevSocketList * obj);
+void qapi_free_ChardevSocket(ChardevSocket * obj);
+
+struct ChardevUdp
+{
+    SocketAddress * remote;
+    bool has_local;
+    SocketAddress * local;
+};
+
+void qapi_free_ChardevUdpList(ChardevUdpList * obj);
+void qapi_free_ChardevUdp(ChardevUdp * obj);
+
+struct ChardevMux
+{
+    char * chardev;
+};
+
+void qapi_free_ChardevMuxList(ChardevMuxList * obj);
+void qapi_free_ChardevMux(ChardevMux * obj);
+
+struct ChardevStdio
+{
+    bool has_signal;
+    bool signal;
+};
+
+void qapi_free_ChardevStdioList(ChardevStdioList * obj);
+void qapi_free_ChardevStdio(ChardevStdio * obj);
+
+struct ChardevSpiceChannel
+{
+    char * type;
+};
+
+void qapi_free_ChardevSpiceChannelList(ChardevSpiceChannelList * obj);
+void qapi_free_ChardevSpiceChannel(ChardevSpiceChannel * obj);
+
+struct ChardevSpicePort
+{
+    char * fqdn;
+};
+
+void qapi_free_ChardevSpicePortList(ChardevSpicePortList * obj);
+void qapi_free_ChardevSpicePort(ChardevSpicePort * obj);
+
+struct ChardevVC
+{
+    bool has_width;
+    int64_t width;
+    bool has_height;
+    int64_t height;
+    bool has_cols;
+    int64_t cols;
+    bool has_rows;
+    int64_t rows;
+};
+
+void qapi_free_ChardevVCList(ChardevVCList * obj);
+void qapi_free_ChardevVC(ChardevVC * obj);
+
+struct ChardevMemory
+{
+    bool has_size;
+    int64_t size;
+};
+
+void qapi_free_ChardevMemoryList(ChardevMemoryList * obj);
+void qapi_free_ChardevMemory(ChardevMemory * obj);
+
+struct ChardevDummy
+{
+};
+
+void qapi_free_ChardevDummyList(ChardevDummyList * obj);
+void qapi_free_ChardevDummy(ChardevDummy * obj);
+
+struct ChardevBackend
+{
+    ChardevBackendKind kind;
+    union {
+        void *data;
+        ChardevFile * file;
+        ChardevHostdev * serial;
+        ChardevHostdev * parallel;
+        ChardevHostdev * pipe;
+        ChardevSocket * socket;
+        ChardevUdp * udp;
+        ChardevDummy * pty;
+        ChardevDummy * null;
+        ChardevMux * mux;
+        ChardevDummy * msmouse;
+        ChardevDummy * braille;
+        ChardevStdio * stdio;
+        ChardevDummy * console;
+        ChardevSpiceChannel * spicevmc;
+        ChardevSpicePort * spiceport;
+        ChardevVC * vc;
+        ChardevMemory * memory;
+    };
+};
+void qapi_free_ChardevBackendList(ChardevBackendList * obj);
+void qapi_free_ChardevBackend(ChardevBackend * obj);
+
+struct ChardevReturn
+{
+    bool has_pty;
+    char * pty;
+};
+
+void qapi_free_ChardevReturnList(ChardevReturnList * obj);
+void qapi_free_ChardevReturn(ChardevReturn * obj);
+
+void qapi_free_TpmModelList(TpmModelList * obj);
+
+void qapi_free_TpmTypeList(TpmTypeList * obj);
+
+struct TPMPassthroughOptions
+{
+    bool has_path;
+    char * path;
+    bool has_cancel_path;
+    char * cancel_path;
+};
+
+void qapi_free_TPMPassthroughOptionsList(TPMPassthroughOptionsList * obj);
+void qapi_free_TPMPassthroughOptions(TPMPassthroughOptions * obj);
+
+struct TpmTypeOptions
+{
+    TpmTypeOptionsKind kind;
+    union {
+        void *data;
+        TPMPassthroughOptions * passthrough;
+    };
+};
+void qapi_free_TpmTypeOptionsList(TpmTypeOptionsList * obj);
+void qapi_free_TpmTypeOptions(TpmTypeOptions * obj);
+
+struct TPMInfo
+{
+    char * id;
+    TpmModel model;
+    TpmTypeOptions * options;
+};
+
+void qapi_free_TPMInfoList(TPMInfoList * obj);
+void qapi_free_TPMInfo(TPMInfo * obj);
+
+struct AcpiTableOptions
+{
+    bool has_sig;
+    char * sig;
+    bool has_rev;
+    uint8_t rev;
+    bool has_oem_id;
+    char * oem_id;
+    bool has_oem_table_id;
+    char * oem_table_id;
+    bool has_oem_rev;
+    uint32_t oem_rev;
+    bool has_asl_compiler_id;
+    char * asl_compiler_id;
+    bool has_asl_compiler_rev;
+    uint32_t asl_compiler_rev;
+    bool has_file;
+    char * file;
+    bool has_data;
+    char * data;
+};
+
+void qapi_free_AcpiTableOptionsList(AcpiTableOptionsList * obj);
+void qapi_free_AcpiTableOptions(AcpiTableOptions * obj);
+
+void qapi_free_CommandLineParameterTypeList(CommandLineParameterTypeList * obj);
+
+struct CommandLineParameterInfo
+{
+    char * name;
+    CommandLineParameterType type;
+    bool has_help;
+    char * help;
+};
+
+void qapi_free_CommandLineParameterInfoList(CommandLineParameterInfoList * obj);
+void qapi_free_CommandLineParameterInfo(CommandLineParameterInfo * obj);
+
+struct CommandLineOptionInfo
+{
+    char * option;
+    CommandLineParameterInfoList * parameters;
+};
+
+void qapi_free_CommandLineOptionInfoList(CommandLineOptionInfoList * obj);
+void qapi_free_CommandLineOptionInfo(CommandLineOptionInfo * obj);
+
+void qapi_free_X86CPURegister32List(X86CPURegister32List * obj);
+
+struct X86CPUFeatureWordInfo
+{
+    int64_t cpuid_input_eax;
+    bool has_cpuid_input_ecx;
+    int64_t cpuid_input_ecx;
+    X86CPURegister32 cpuid_register;
+    int64_t features;
+};
+
+void qapi_free_X86CPUFeatureWordInfoList(X86CPUFeatureWordInfoList * obj);
+void qapi_free_X86CPUFeatureWordInfo(X86CPUFeatureWordInfo * obj);
+
+void qapi_free_RxStateList(RxStateList * obj);
+
+struct RxFilterInfo
+{
+    char * name;
+    bool promiscuous;
+    RxState multicast;
+    RxState unicast;
+    bool broadcast_allowed;
+    bool multicast_overflow;
+    bool unicast_overflow;
+    char * main_mac;
+    intList * vlan_table;
+    strList * unicast_table;
+    strList * multicast_table;
+};
+
+void qapi_free_RxFilterInfoList(RxFilterInfoList * obj);
+void qapi_free_RxFilterInfo(RxFilterInfo * obj);
+
+#endif
diff --git a/contrib/qemu/qemu-coroutine-lock.c b/contrib/qemu/qemu-coroutine-lock.c
new file mode 100644
index 000000000..d9fea4989
--- /dev/null
+++ b/contrib/qemu/qemu-coroutine-lock.c
@@ -0,0 +1,178 @@
+/*
+ * coroutine queues and locks
+ *
+ * Copyright (c) 2011 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu-common.h"
+#include "block/coroutine.h"
+#include "block/coroutine_int.h"
+#include "qemu/queue.h"
+#include "trace.h"
+
+void qemu_co_queue_init(CoQueue *queue)
+{
+    QTAILQ_INIT(&queue->entries);
+}
+
+void coroutine_fn qemu_co_queue_wait(CoQueue *queue)
+{
+    Coroutine *self = qemu_coroutine_self();
+    QTAILQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
+    qemu_coroutine_yield();
+    assert(qemu_in_coroutine());
+}
+
+void coroutine_fn qemu_co_queue_wait_insert_head(CoQueue *queue)
+{
+    Coroutine *self = qemu_coroutine_self();
+    QTAILQ_INSERT_HEAD(&queue->entries, self, co_queue_next);
+    qemu_coroutine_yield();
+    assert(qemu_in_coroutine());
+}
+
+/**
+ * qemu_co_queue_run_restart:
+ *
+ * Enter each coroutine that was previously marked for restart by
+ * qemu_co_queue_next() or qemu_co_queue_restart_all().  This function is
+ * invoked by the core coroutine code when the current coroutine yields or
+ * terminates.
+ */
+void qemu_co_queue_run_restart(Coroutine *co)
+{
+    Coroutine *next;
+
+    trace_qemu_co_queue_run_restart(co);
+    while ((next = QTAILQ_FIRST(&co->co_queue_wakeup))) {
+        QTAILQ_REMOVE(&co->co_queue_wakeup, next, co_queue_next);
+        qemu_coroutine_enter(next, NULL);
+    }
+}
+
+static bool qemu_co_queue_do_restart(CoQueue *queue, bool single)
+{
+    Coroutine *self = qemu_coroutine_self();
+    Coroutine *next;
+
+    if (QTAILQ_EMPTY(&queue->entries)) {
+        return false;
+    }
+
+    while ((next = QTAILQ_FIRST(&queue->entries)) != NULL) {
+        QTAILQ_REMOVE(&queue->entries, next, co_queue_next);
+        QTAILQ_INSERT_TAIL(&self->co_queue_wakeup, next, co_queue_next);
+        trace_qemu_co_queue_next(next);
+        if (single) {
+            break;
+        }
+    }
+    return true;
+}
+
+bool qemu_co_queue_next(CoQueue *queue)
+{
+    return qemu_co_queue_do_restart(queue, true);
+}
+
+void qemu_co_queue_restart_all(CoQueue *queue)
+{
+    qemu_co_queue_do_restart(queue, false);
+}
+
+bool qemu_co_queue_empty(CoQueue *queue)
+{
+    return (QTAILQ_FIRST(&queue->entries) == NULL);
+}
+
+void qemu_co_mutex_init(CoMutex *mutex)
+{
+    memset(mutex, 0, sizeof(*mutex));
+    qemu_co_queue_init(&mutex->queue);
+}
+
+void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex)
+{
+    Coroutine *self = qemu_coroutine_self();
+
+    trace_qemu_co_mutex_lock_entry(mutex, self);
+
+    while (mutex->locked) {
+        qemu_co_queue_wait(&mutex->queue);
+    }
+
+    mutex->locked = true;
+
+    trace_qemu_co_mutex_lock_return(mutex, self);
+}
+
+void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex)
+{
+    Coroutine *self = qemu_coroutine_self();
+
+    trace_qemu_co_mutex_unlock_entry(mutex, self);
+
+    assert(mutex->locked == true);
+    assert(qemu_in_coroutine());
+
+    mutex->locked = false;
+    qemu_co_queue_next(&mutex->queue);
+
+    trace_qemu_co_mutex_unlock_return(mutex, self);
+}
+
+void qemu_co_rwlock_init(CoRwlock *lock)
+{
+    memset(lock, 0, sizeof(*lock));
+    qemu_co_queue_init(&lock->queue);
+}
+
+void qemu_co_rwlock_rdlock(CoRwlock *lock)
+{
+    while (lock->writer) {
+        qemu_co_queue_wait(&lock->queue);
+    }
+    lock->reader++;
+}
+
+void qemu_co_rwlock_unlock(CoRwlock *lock)
+{
+    assert(qemu_in_coroutine());
+    if (lock->writer) {
+        lock->writer = false;
+        qemu_co_queue_restart_all(&lock->queue);
+    } else {
+        lock->reader--;
+        assert(lock->reader >= 0);
+        /* Wakeup only one waiting writer */
+        if (!lock->reader) {
+            qemu_co_queue_next(&lock->queue);
+        }
+    }
+}
+
+void qemu_co_rwlock_wrlock(CoRwlock *lock)
+{
+    while (lock->writer || lock->reader) {
+        qemu_co_queue_wait(&lock->queue);
+    }
+    lock->writer = true;
+}
diff --git a/contrib/qemu/qemu-coroutine-sleep.c b/contrib/qemu/qemu-coroutine-sleep.c
new file mode 100644
index 000000000..169ce5ccc
--- /dev/null
+++ b/contrib/qemu/qemu-coroutine-sleep.c
@@ -0,0 +1,39 @@
+/*
+ * QEMU coroutine sleep
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Stefan Hajnoczi    <stefanha@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "block/coroutine.h"
+#include "qemu/timer.h"
+
+typedef struct CoSleepCB {
+    QEMUTimer *ts;
+    Coroutine *co;
+} CoSleepCB;
+
+static void co_sleep_cb(void *opaque)
+{
+    CoSleepCB *sleep_cb = opaque;
+
+    qemu_coroutine_enter(sleep_cb->co, NULL);
+}
+
+void coroutine_fn co_sleep_ns(QEMUClock *clock, int64_t ns)
+{
+    CoSleepCB sleep_cb = {
+        .co = qemu_coroutine_self(),
+    };
+    sleep_cb.ts = qemu_new_timer(clock, SCALE_NS, co_sleep_cb, &sleep_cb);
+    qemu_mod_timer(sleep_cb.ts, qemu_get_clock_ns(clock) + ns);
+    qemu_coroutine_yield();
+    qemu_del_timer(sleep_cb.ts);
+    qemu_free_timer(sleep_cb.ts);
+}
diff --git a/contrib/qemu/qemu-coroutine.c b/contrib/qemu/qemu-coroutine.c
new file mode 100644
index 000000000..5e19307ee
--- /dev/null
+++ b/contrib/qemu/qemu-coroutine.c
@@ -0,0 +1,135 @@
+/*
+ * QEMU coroutines
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Stefan Hajnoczi    <stefanha@linux.vnet.ibm.com>
+ *  Kevin Wolf         <kwolf@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "trace.h"
+#include "qemu-common.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
+#include "block/coroutine_int.h"
+
+enum {
+    /* Maximum free pool size prevents holding too many freed coroutines */
+    POOL_MAX_SIZE = 0,
+};
+
+/** Free list to speed up creation */
+static QemuMutex pool_lock;
+static QSLIST_HEAD(, Coroutine) pool = QSLIST_HEAD_INITIALIZER(pool);
+static unsigned int pool_size;
+
+Coroutine *qemu_coroutine_create(CoroutineEntry *entry)
+{
+    Coroutine *co;
+
+    qemu_mutex_lock(&pool_lock);
+    co = QSLIST_FIRST(&pool);
+    if (co) {
+        QSLIST_REMOVE_HEAD(&pool, pool_next);
+        pool_size--;
+    }
+    qemu_mutex_unlock(&pool_lock);
+
+    if (!co) {
+        co = qemu_coroutine_new();
+    }
+
+    co->entry = entry;
+    QTAILQ_INIT(&co->co_queue_wakeup);
+    return co;
+}
+
+static void coroutine_delete(Coroutine *co)
+{
+    qemu_mutex_lock(&pool_lock);
+    if (pool_size < POOL_MAX_SIZE) {
+        QSLIST_INSERT_HEAD(&pool, co, pool_next);
+        co->caller = NULL;
+        pool_size++;
+        qemu_mutex_unlock(&pool_lock);
+        return;
+    }
+    qemu_mutex_unlock(&pool_lock);
+
+    qemu_coroutine_delete(co);
+}
+
+static void __attribute__((constructor)) coroutine_pool_init(void)
+{
+    qemu_mutex_init(&pool_lock);
+}
+
+static void __attribute__((destructor)) coroutine_pool_cleanup(void)
+{
+    Coroutine *co;
+    Coroutine *tmp;
+
+    QSLIST_FOREACH_SAFE(co, &pool, pool_next, tmp) {
+        QSLIST_REMOVE_HEAD(&pool, pool_next);
+        qemu_coroutine_delete(co);
+    }
+
+    qemu_mutex_destroy(&pool_lock);
+}
+
+static void coroutine_swap(Coroutine *from, Coroutine *to)
+{
+    CoroutineAction ret;
+
+    ret = qemu_coroutine_switch(from, to, COROUTINE_YIELD);
+
+    qemu_co_queue_run_restart(to);
+
+    switch (ret) {
+    case COROUTINE_YIELD:
+        return;
+    case COROUTINE_TERMINATE:
+        trace_qemu_coroutine_terminate(to);
+        coroutine_delete(to);
+        return;
+    default:
+        abort();
+    }
+}
+
+void qemu_coroutine_enter(Coroutine *co, void *opaque)
+{
+    Coroutine *self = qemu_coroutine_self();
+
+    trace_qemu_coroutine_enter(self, co, opaque);
+
+    if (co->caller) {
+        fprintf(stderr, "Co-routine re-entered recursively\n");
+        abort();
+    }
+
+    co->caller = self;
+    co->entry_arg = opaque;
+    coroutine_swap(self, co);
+}
+
+void coroutine_fn qemu_coroutine_yield(void)
+{
+    Coroutine *self = qemu_coroutine_self();
+    Coroutine *to = self->caller;
+
+    trace_qemu_coroutine_yield(self, to);
+
+    if (!to) {
+        fprintf(stderr, "Co-routine is yielding to no one\n");
+        abort();
+    }
+
+    self->caller = NULL;
+    coroutine_swap(self, to);
+}
diff --git a/contrib/qemu/qmp-commands.h b/contrib/qemu/qmp-commands.h
new file mode 100644
index 000000000..fcc0ff0f7
--- /dev/null
+++ b/contrib/qemu/qmp-commands.h
@@ -0,0 +1,204 @@
+/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT MODIFY */
+
+/*
+ * schema-defined QAPI function prototypes
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QMP_COMMANDS_H
+#define QMP_COMMANDS_H
+
+#include "qapi-types.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/error.h"
+
+void qmp_add_client(const char * protocol, const char * fdname, bool has_skipauth, bool skipauth, bool has_tls, bool tls, Error **errp);
+int qmp_marshal_input_add_client(Monitor *mon, const QDict *qdict, QObject **ret);
+NameInfo * qmp_query_name(Error **errp);
+int qmp_marshal_input_query_name(Monitor *mon, const QDict *qdict, QObject **ret);
+VersionInfo * qmp_query_version(Error **errp);
+int qmp_marshal_input_query_version(Monitor *mon, const QDict *qdict, QObject **ret);
+KvmInfo * qmp_query_kvm(Error **errp);
+int qmp_marshal_input_query_kvm(Monitor *mon, const QDict *qdict, QObject **ret);
+StatusInfo * qmp_query_status(Error **errp);
+int qmp_marshal_input_query_status(Monitor *mon, const QDict *qdict, QObject **ret);
+UuidInfo * qmp_query_uuid(Error **errp);
+int qmp_marshal_input_query_uuid(Monitor *mon, const QDict *qdict, QObject **ret);
+ChardevInfoList * qmp_query_chardev(Error **errp);
+int qmp_marshal_input_query_chardev(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_ringbuf_write(const char * device, const char * data, bool has_format, DataFormat format, Error **errp);
+int qmp_marshal_input_ringbuf_write(Monitor *mon, const QDict *qdict, QObject **ret);
+char * qmp_ringbuf_read(const char * device, int64_t size, bool has_format, DataFormat format, Error **errp);
+int qmp_marshal_input_ringbuf_read(Monitor *mon, const QDict *qdict, QObject **ret);
+CommandInfoList * qmp_query_commands(Error **errp);
+int qmp_marshal_input_query_commands(Monitor *mon, const QDict *qdict, QObject **ret);
+EventInfoList * qmp_query_events(Error **errp);
+int qmp_marshal_input_query_events(Monitor *mon, const QDict *qdict, QObject **ret);
+MigrationInfo * qmp_query_migrate(Error **errp);
+int qmp_marshal_input_query_migrate(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_migrate_set_capabilities(MigrationCapabilityStatusList * capabilities, Error **errp);
+int qmp_marshal_input_migrate_set_capabilities(Monitor *mon, const QDict *qdict, QObject **ret);
+MigrationCapabilityStatusList * qmp_query_migrate_capabilities(Error **errp);
+int qmp_marshal_input_query_migrate_capabilities(Monitor *mon, const QDict *qdict, QObject **ret);
+MouseInfoList * qmp_query_mice(Error **errp);
+int qmp_marshal_input_query_mice(Monitor *mon, const QDict *qdict, QObject **ret);
+CpuInfoList * qmp_query_cpus(Error **errp);
+int qmp_marshal_input_query_cpus(Monitor *mon, const QDict *qdict, QObject **ret);
+BlockInfoList * qmp_query_block(Error **errp);
+int qmp_marshal_input_query_block(Monitor *mon, const QDict *qdict, QObject **ret);
+BlockStatsList * qmp_query_blockstats(Error **errp);
+int qmp_marshal_input_query_blockstats(Monitor *mon, const QDict *qdict, QObject **ret);
+VncInfo * qmp_query_vnc(Error **errp);
+int qmp_marshal_input_query_vnc(Monitor *mon, const QDict *qdict, QObject **ret);
+SpiceInfo * qmp_query_spice(Error **errp);
+int qmp_marshal_input_query_spice(Monitor *mon, const QDict *qdict, QObject **ret);
+BalloonInfo * qmp_query_balloon(Error **errp);
+int qmp_marshal_input_query_balloon(Monitor *mon, const QDict *qdict, QObject **ret);
+PciInfoList * qmp_query_pci(Error **errp);
+int qmp_marshal_input_query_pci(Monitor *mon, const QDict *qdict, QObject **ret);
+BlockJobInfoList * qmp_query_block_jobs(Error **errp);
+int qmp_marshal_input_query_block_jobs(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_quit(Error **errp);
+int qmp_marshal_input_quit(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_stop(Error **errp);
+int qmp_marshal_input_stop(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_system_reset(Error **errp);
+int qmp_marshal_input_system_reset(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_system_powerdown(Error **errp);
+int qmp_marshal_input_system_powerdown(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_cpu(int64_t index, Error **errp);
+int qmp_marshal_input_cpu(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_cpu_add(int64_t id, Error **errp);
+int qmp_marshal_input_cpu_add(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_memsave(int64_t val, int64_t size, const char * filename, bool has_cpu_index, int64_t cpu_index, Error **errp);
+int qmp_marshal_input_memsave(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_pmemsave(int64_t val, int64_t size, const char * filename, Error **errp);
+int qmp_marshal_input_pmemsave(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_cont(Error **errp);
+int qmp_marshal_input_cont(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_system_wakeup(Error **errp);
+int qmp_marshal_input_system_wakeup(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_inject_nmi(Error **errp);
+int qmp_marshal_input_inject_nmi(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_set_link(const char * name, bool up, Error **errp);
+int qmp_marshal_input_set_link(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_block_passwd(const char * device, const char * password, Error **errp);
+int qmp_marshal_input_block_passwd(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_balloon(int64_t value, Error **errp);
+int qmp_marshal_input_balloon(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_block_resize(const char * device, int64_t size, Error **errp);
+int qmp_marshal_input_block_resize(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_transaction(TransactionActionList * actions, Error **errp);
+int qmp_marshal_input_transaction(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_blockdev_snapshot_sync(const char * device, const char * snapshot_file, bool has_format, const char * format, bool has_mode, NewImageMode mode, Error **errp);
+int qmp_marshal_input_blockdev_snapshot_sync(Monitor *mon, const QDict *qdict, QObject **ret);
+char * qmp_human_monitor_command(const char * command_line, bool has_cpu_index, int64_t cpu_index, Error **errp);
+int qmp_marshal_input_human_monitor_command(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_block_commit(const char * device, bool has_base, const char * base, const char * top, bool has_speed, int64_t speed, Error **errp);
+int qmp_marshal_input_block_commit(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_drive_backup(const char * device, const char * target, bool has_format, const char * format, MirrorSyncMode sync, bool has_mode, NewImageMode mode, bool has_speed, int64_t speed, bool has_on_source_error, BlockdevOnError on_source_error, bool has_on_target_error, BlockdevOnError on_target_error, Error **errp);
+int qmp_marshal_input_drive_backup(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_drive_mirror(const char * device, const char * target, bool has_format, const char * format, MirrorSyncMode sync, bool has_mode, NewImageMode mode, bool has_speed, int64_t speed, bool has_granularity, uint32_t granularity, bool has_buf_size, int64_t buf_size, bool has_on_source_error, BlockdevOnError on_source_error, bool has_on_target_error, BlockdevOnError on_target_error, Error **errp);
+int qmp_marshal_input_drive_mirror(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_migrate_cancel(Error **errp);
+int qmp_marshal_input_migrate_cancel(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_migrate_set_downtime(double value, Error **errp);
+int qmp_marshal_input_migrate_set_downtime(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_migrate_set_speed(int64_t value, Error **errp);
+int qmp_marshal_input_migrate_set_speed(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_migrate_set_cache_size(int64_t value, Error **errp);
+int qmp_marshal_input_migrate_set_cache_size(Monitor *mon, const QDict *qdict, QObject **ret);
+int64_t qmp_query_migrate_cache_size(Error **errp);
+int qmp_marshal_input_query_migrate_cache_size(Monitor *mon, const QDict *qdict, QObject **ret);
+ObjectPropertyInfoList * qmp_qom_list(const char * path, Error **errp);
+int qmp_marshal_input_qom_list(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_set_password(const char * protocol, const char * password, bool has_connected, const char * connected, Error **errp);
+int qmp_marshal_input_set_password(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_expire_password(const char * protocol, const char * time, Error **errp);
+int qmp_marshal_input_expire_password(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_eject(const char * device, bool has_force, bool force, Error **errp);
+int qmp_marshal_input_eject(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_change_vnc_password(const char * password, Error **errp);
+int qmp_marshal_input_change_vnc_password(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_change(const char * device, const char * target, bool has_arg, const char * arg, Error **errp);
+int qmp_marshal_input_change(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_block_set_io_throttle(const char * device, int64_t bps, int64_t bps_rd, int64_t bps_wr, int64_t iops, int64_t iops_rd, int64_t iops_wr, Error **errp);
+int qmp_marshal_input_block_set_io_throttle(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_block_stream(const char * device, bool has_base, const char * base, bool has_speed, int64_t speed, bool has_on_error, BlockdevOnError on_error, Error **errp);
+int qmp_marshal_input_block_stream(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_block_job_set_speed(const char * device, int64_t speed, Error **errp);
+int qmp_marshal_input_block_job_set_speed(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_block_job_cancel(const char * device, bool has_force, bool force, Error **errp);
+int qmp_marshal_input_block_job_cancel(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_block_job_pause(const char * device, Error **errp);
+int qmp_marshal_input_block_job_pause(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_block_job_resume(const char * device, Error **errp);
+int qmp_marshal_input_block_job_resume(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_block_job_complete(const char * device, Error **errp);
+int qmp_marshal_input_block_job_complete(Monitor *mon, const QDict *qdict, QObject **ret);
+ObjectTypeInfoList * qmp_qom_list_types(bool has_implements, const char * implements, bool has_abstract, bool abstract, Error **errp);
+int qmp_marshal_input_qom_list_types(Monitor *mon, const QDict *qdict, QObject **ret);
+DevicePropertyInfoList * qmp_device_list_properties(const char * typename, Error **errp);
+int qmp_marshal_input_device_list_properties(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_migrate(const char * uri, bool has_blk, bool blk, bool has_inc, bool inc, bool has_detach, bool detach, Error **errp);
+int qmp_marshal_input_migrate(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_xen_save_devices_state(const char * filename, Error **errp);
+int qmp_marshal_input_xen_save_devices_state(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_xen_set_global_dirty_log(bool enable, Error **errp);
+int qmp_marshal_input_xen_set_global_dirty_log(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_device_del(const char * id, Error **errp);
+int qmp_marshal_input_device_del(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_dump_guest_memory(bool paging, const char * protocol, bool has_begin, int64_t begin, bool has_length, int64_t length, Error **errp);
+int qmp_marshal_input_dump_guest_memory(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_netdev_del(const char * id, Error **errp);
+int qmp_marshal_input_netdev_del(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_getfd(const char * fdname, Error **errp);
+int qmp_marshal_input_getfd(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_closefd(const char * fdname, Error **errp);
+int qmp_marshal_input_closefd(Monitor *mon, const QDict *qdict, QObject **ret);
+MachineInfoList * qmp_query_machines(Error **errp);
+int qmp_marshal_input_query_machines(Monitor *mon, const QDict *qdict, QObject **ret);
+CpuDefinitionInfoList * qmp_query_cpu_definitions(Error **errp);
+int qmp_marshal_input_query_cpu_definitions(Monitor *mon, const QDict *qdict, QObject **ret);
+AddfdInfo * qmp_add_fd(bool has_fdset_id, int64_t fdset_id, bool has_opaque, const char * opaque, Error **errp);
+int qmp_marshal_input_add_fd(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_remove_fd(int64_t fdset_id, bool has_fd, int64_t fd, Error **errp);
+int qmp_marshal_input_remove_fd(Monitor *mon, const QDict *qdict, QObject **ret);
+FdsetInfoList * qmp_query_fdsets(Error **errp);
+int qmp_marshal_input_query_fdsets(Monitor *mon, const QDict *qdict, QObject **ret);
+TargetInfo * qmp_query_target(Error **errp);
+int qmp_marshal_input_query_target(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_send_key(KeyValueList * keys, bool has_hold_time, int64_t hold_time, Error **errp);
+int qmp_marshal_input_send_key(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_screendump(const char * filename, Error **errp);
+int qmp_marshal_input_screendump(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_nbd_server_start(SocketAddress * addr, Error **errp);
+int qmp_marshal_input_nbd_server_start(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_nbd_server_add(const char * device, bool has_writable, bool writable, Error **errp);
+int qmp_marshal_input_nbd_server_add(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_nbd_server_stop(Error **errp);
+int qmp_marshal_input_nbd_server_stop(Monitor *mon, const QDict *qdict, QObject **ret);
+ChardevReturn * qmp_chardev_add(const char * id, ChardevBackend * backend, Error **errp);
+int qmp_marshal_input_chardev_add(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_chardev_remove(const char * id, Error **errp);
+int qmp_marshal_input_chardev_remove(Monitor *mon, const QDict *qdict, QObject **ret);
+TpmModelList * qmp_query_tpm_models(Error **errp);
+int qmp_marshal_input_query_tpm_models(Monitor *mon, const QDict *qdict, QObject **ret);
+TpmTypeList * qmp_query_tpm_types(Error **errp);
+int qmp_marshal_input_query_tpm_types(Monitor *mon, const QDict *qdict, QObject **ret);
+TPMInfoList * qmp_query_tpm(Error **errp);
+int qmp_marshal_input_query_tpm(Monitor *mon, const QDict *qdict, QObject **ret);
+CommandLineOptionInfoList * qmp_query_command_line_options(bool has_option, const char * option, Error **errp);
+int qmp_marshal_input_query_command_line_options(Monitor *mon, const QDict *qdict, QObject **ret);
+RxFilterInfoList * qmp_query_rx_filter(bool has_name, const char * name, Error **errp);
+int qmp_marshal_input_query_rx_filter(Monitor *mon, const QDict *qdict, QObject **ret);
+
+#endif
diff --git a/contrib/qemu/qobject/json-lexer.c b/contrib/qemu/qobject/json-lexer.c
new file mode 100644
index 000000000..440df6039
--- /dev/null
+++ b/contrib/qemu/qobject/json-lexer.c
@@ -0,0 +1,373 @@
+/*
+ * JSON lexer
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qint.h"
+#include "qemu-common.h"
+#include "qapi/qmp/json-lexer.h"
+
+#define MAX_TOKEN_SIZE (64ULL << 20)
+
+/*
+ * \"([^\\\"]|(\\\"\\'\\\\\\/\\b\\f\\n\\r\\t\\u[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]))*\"
+ * '([^\\']|(\\\"\\'\\\\\\/\\b\\f\\n\\r\\t\\u[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]))*'
+ * 0|([1-9][0-9]*(.[0-9]+)?([eE]([-+])?[0-9]+))
+ * [{}\[\],:]
+ * [a-z]+
+ *
+ */
+
+enum json_lexer_state {
+    IN_ERROR = 0,
+    IN_DQ_UCODE3,
+    IN_DQ_UCODE2,
+    IN_DQ_UCODE1,
+    IN_DQ_UCODE0,
+    IN_DQ_STRING_ESCAPE,
+    IN_DQ_STRING,
+    IN_SQ_UCODE3,
+    IN_SQ_UCODE2,
+    IN_SQ_UCODE1,
+    IN_SQ_UCODE0,
+    IN_SQ_STRING_ESCAPE,
+    IN_SQ_STRING,
+    IN_ZERO,
+    IN_DIGITS,
+    IN_DIGIT,
+    IN_EXP_E,
+    IN_MANTISSA,
+    IN_MANTISSA_DIGITS,
+    IN_NONZERO_NUMBER,
+    IN_NEG_NONZERO_NUMBER,
+    IN_KEYWORD,
+    IN_ESCAPE,
+    IN_ESCAPE_L,
+    IN_ESCAPE_LL,
+    IN_ESCAPE_I,
+    IN_ESCAPE_I6,
+    IN_ESCAPE_I64,
+    IN_WHITESPACE,
+    IN_START,
+};
+
+#define TERMINAL(state) [0 ... 0x7F] = (state)
+
+/* Return whether TERMINAL is a terminal state and the transition to it
+   from OLD_STATE required lookahead.  This happens whenever the table
+   below uses the TERMINAL macro.  */
+#define TERMINAL_NEEDED_LOOKAHEAD(old_state, terminal) \
+            (json_lexer[(old_state)][0] == (terminal))
+
+static const uint8_t json_lexer[][256] =  {
+    /* double quote string */
+    [IN_DQ_UCODE3] = {
+        ['0' ... '9'] = IN_DQ_STRING,
+        ['a' ... 'f'] = IN_DQ_STRING,
+        ['A' ... 'F'] = IN_DQ_STRING,
+    },
+    [IN_DQ_UCODE2] = {
+        ['0' ... '9'] = IN_DQ_UCODE3,
+        ['a' ... 'f'] = IN_DQ_UCODE3,
+        ['A' ... 'F'] = IN_DQ_UCODE3,
+    },
+    [IN_DQ_UCODE1] = {
+        ['0' ... '9'] = IN_DQ_UCODE2,
+        ['a' ... 'f'] = IN_DQ_UCODE2,
+        ['A' ... 'F'] = IN_DQ_UCODE2,
+    },
+    [IN_DQ_UCODE0] = {
+        ['0' ... '9'] = IN_DQ_UCODE1,
+        ['a' ... 'f'] = IN_DQ_UCODE1,
+        ['A' ... 'F'] = IN_DQ_UCODE1,
+    },
+    [IN_DQ_STRING_ESCAPE] = {
+        ['b'] = IN_DQ_STRING,
+        ['f'] =  IN_DQ_STRING,
+        ['n'] =  IN_DQ_STRING,
+        ['r'] =  IN_DQ_STRING,
+        ['t'] =  IN_DQ_STRING,
+        ['/'] = IN_DQ_STRING,
+        ['\\'] = IN_DQ_STRING,
+        ['\''] = IN_DQ_STRING,
+        ['\"'] = IN_DQ_STRING,
+        ['u'] = IN_DQ_UCODE0,
+    },
+    [IN_DQ_STRING] = {
+        [1 ... 0xBF] = IN_DQ_STRING,
+        [0xC2 ... 0xF4] = IN_DQ_STRING,
+        ['\\'] = IN_DQ_STRING_ESCAPE,
+        ['"'] = JSON_STRING,
+    },
+
+    /* single quote string */
+    [IN_SQ_UCODE3] = {
+        ['0' ... '9'] = IN_SQ_STRING,
+        ['a' ... 'f'] = IN_SQ_STRING,
+        ['A' ... 'F'] = IN_SQ_STRING,
+    },
+    [IN_SQ_UCODE2] = {
+        ['0' ... '9'] = IN_SQ_UCODE3,
+        ['a' ... 'f'] = IN_SQ_UCODE3,
+        ['A' ... 'F'] = IN_SQ_UCODE3,
+    },
+    [IN_SQ_UCODE1] = {
+        ['0' ... '9'] = IN_SQ_UCODE2,
+        ['a' ... 'f'] = IN_SQ_UCODE2,
+        ['A' ... 'F'] = IN_SQ_UCODE2,
+    },
+    [IN_SQ_UCODE0] = {
+        ['0' ... '9'] = IN_SQ_UCODE1,
+        ['a' ... 'f'] = IN_SQ_UCODE1,
+        ['A' ... 'F'] = IN_SQ_UCODE1,
+    },
+    [IN_SQ_STRING_ESCAPE] = {
+        ['b'] = IN_SQ_STRING,
+        ['f'] =  IN_SQ_STRING,
+        ['n'] =  IN_SQ_STRING,
+        ['r'] =  IN_SQ_STRING,
+        ['t'] =  IN_SQ_STRING,
+        ['/'] = IN_DQ_STRING,
+        ['\\'] = IN_DQ_STRING,
+        ['\''] = IN_SQ_STRING,
+        ['\"'] = IN_SQ_STRING,
+        ['u'] = IN_SQ_UCODE0,
+    },
+    [IN_SQ_STRING] = {
+        [1 ... 0xBF] = IN_SQ_STRING,
+        [0xC2 ... 0xF4] = IN_SQ_STRING,
+        ['\\'] = IN_SQ_STRING_ESCAPE,
+        ['\''] = JSON_STRING,
+    },
+
+    /* Zero */
+    [IN_ZERO] = {
+        TERMINAL(JSON_INTEGER),
+        ['0' ... '9'] = IN_ERROR,
+        ['.'] = IN_MANTISSA,
+    },
+
+    /* Float */
+    [IN_DIGITS] = {
+        TERMINAL(JSON_FLOAT),
+        ['0' ... '9'] = IN_DIGITS,
+    },
+
+    [IN_DIGIT] = {
+        ['0' ... '9'] = IN_DIGITS,
+    },
+
+    [IN_EXP_E] = {
+        ['-'] = IN_DIGIT,
+        ['+'] = IN_DIGIT,
+        ['0' ... '9'] = IN_DIGITS,
+    },
+
+    [IN_MANTISSA_DIGITS] = {
+        TERMINAL(JSON_FLOAT),
+        ['0' ... '9'] = IN_MANTISSA_DIGITS,
+        ['e'] = IN_EXP_E,
+        ['E'] = IN_EXP_E,
+    },
+
+    [IN_MANTISSA] = {
+        ['0' ... '9'] = IN_MANTISSA_DIGITS,
+    },
+
+    /* Number */
+    [IN_NONZERO_NUMBER] = {
+        TERMINAL(JSON_INTEGER),
+        ['0' ... '9'] = IN_NONZERO_NUMBER,
+        ['e'] = IN_EXP_E,
+        ['E'] = IN_EXP_E,
+        ['.'] = IN_MANTISSA,
+    },
+
+    [IN_NEG_NONZERO_NUMBER] = {
+        ['0'] = IN_ZERO,
+        ['1' ... '9'] = IN_NONZERO_NUMBER,
+    },
+
+    /* keywords */
+    [IN_KEYWORD] = {
+        TERMINAL(JSON_KEYWORD),
+        ['a' ... 'z'] = IN_KEYWORD,
+    },
+
+    /* whitespace */
+    [IN_WHITESPACE] = {
+        TERMINAL(JSON_SKIP),
+        [' '] = IN_WHITESPACE,
+        ['\t'] = IN_WHITESPACE,
+        ['\r'] = IN_WHITESPACE,
+        ['\n'] = IN_WHITESPACE,
+    },        
+
+    /* escape */
+    [IN_ESCAPE_LL] = {
+        ['d'] = JSON_ESCAPE,
+    },
+
+    [IN_ESCAPE_L] = {
+        ['d'] = JSON_ESCAPE,
+        ['l'] = IN_ESCAPE_LL,
+    },
+
+    [IN_ESCAPE_I64] = {
+        ['d'] = JSON_ESCAPE,
+    },
+
+    [IN_ESCAPE_I6] = {
+        ['4'] = IN_ESCAPE_I64,
+    },
+
+    [IN_ESCAPE_I] = {
+        ['6'] = IN_ESCAPE_I6,
+    },
+
+    [IN_ESCAPE] = {
+        ['d'] = JSON_ESCAPE,
+        ['i'] = JSON_ESCAPE,
+        ['p'] = JSON_ESCAPE,
+        ['s'] = JSON_ESCAPE,
+        ['f'] = JSON_ESCAPE,
+        ['l'] = IN_ESCAPE_L,
+        ['I'] = IN_ESCAPE_I,
+    },
+
+    /* top level rule */
+    [IN_START] = {
+        ['"'] = IN_DQ_STRING,
+        ['\''] = IN_SQ_STRING,
+        ['0'] = IN_ZERO,
+        ['1' ... '9'] = IN_NONZERO_NUMBER,
+        ['-'] = IN_NEG_NONZERO_NUMBER,
+        ['{'] = JSON_OPERATOR,
+        ['}'] = JSON_OPERATOR,
+        ['['] = JSON_OPERATOR,
+        [']'] = JSON_OPERATOR,
+        [','] = JSON_OPERATOR,
+        [':'] = JSON_OPERATOR,
+        ['a' ... 'z'] = IN_KEYWORD,
+        ['%'] = IN_ESCAPE,
+        [' '] = IN_WHITESPACE,
+        ['\t'] = IN_WHITESPACE,
+        ['\r'] = IN_WHITESPACE,
+        ['\n'] = IN_WHITESPACE,
+    },
+};
+
+void json_lexer_init(JSONLexer *lexer, JSONLexerEmitter func)
+{
+    lexer->emit = func;
+    lexer->state = IN_START;
+    lexer->token = qstring_new();
+    lexer->x = lexer->y = 0;
+}
+
+static int json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
+{
+    int char_consumed, new_state;
+
+    lexer->x++;
+    if (ch == '\n') {
+        lexer->x = 0;
+        lexer->y++;
+    }
+
+    do {
+        new_state = json_lexer[lexer->state][(uint8_t)ch];
+        char_consumed = !TERMINAL_NEEDED_LOOKAHEAD(lexer->state, new_state);
+        if (char_consumed) {
+            qstring_append_chr(lexer->token, ch);
+        }
+
+        switch (new_state) {
+        case JSON_OPERATOR:
+        case JSON_ESCAPE:
+        case JSON_INTEGER:
+        case JSON_FLOAT:
+        case JSON_KEYWORD:
+        case JSON_STRING:
+            lexer->emit(lexer, lexer->token, new_state, lexer->x, lexer->y);
+            /* fall through */
+        case JSON_SKIP:
+            QDECREF(lexer->token);
+            lexer->token = qstring_new();
+            new_state = IN_START;
+            break;
+        case IN_ERROR:
+            /* XXX: To avoid having previous bad input leaving the parser in an
+             * unresponsive state where we consume unpredictable amounts of
+             * subsequent "good" input, percolate this error state up to the
+             * tokenizer/parser by forcing a NULL object to be emitted, then
+             * reset state.
+             *
+             * Also note that this handling is required for reliable channel
+             * negotiation between QMP and the guest agent, since chr(0xFF)
+             * is placed at the beginning of certain events to ensure proper
+             * delivery when the channel is in an unknown state. chr(0xFF) is
+             * never a valid ASCII/UTF-8 sequence, so this should reliably
+             * induce an error/flush state.
+             */
+            lexer->emit(lexer, lexer->token, JSON_ERROR, lexer->x, lexer->y);
+            QDECREF(lexer->token);
+            lexer->token = qstring_new();
+            new_state = IN_START;
+            lexer->state = new_state;
+            return 0;
+        default:
+            break;
+        }
+        lexer->state = new_state;
+    } while (!char_consumed && !flush);
+
+    /* Do not let a single token grow to an arbitrarily large size,
+     * this is a security consideration.
+     */
+    if (lexer->token->length > MAX_TOKEN_SIZE) {
+        lexer->emit(lexer, lexer->token, lexer->state, lexer->x, lexer->y);
+        QDECREF(lexer->token);
+        lexer->token = qstring_new();
+        lexer->state = IN_START;
+    }
+
+    return 0;
+}
+
+int json_lexer_feed(JSONLexer *lexer, const char *buffer, size_t size)
+{
+    size_t i;
+
+    for (i = 0; i < size; i++) {
+        int err;
+
+        err = json_lexer_feed_char(lexer, buffer[i], false);
+        if (err < 0) {
+            return err;
+        }
+    }
+
+    return 0;
+}
+
+int json_lexer_flush(JSONLexer *lexer)
+{
+    return lexer->state == IN_START ? 0 : json_lexer_feed_char(lexer, 0, true);
+}
+
+void json_lexer_destroy(JSONLexer *lexer)
+{
+    QDECREF(lexer->token);
+}
diff --git a/contrib/qemu/qobject/json-parser.c b/contrib/qemu/qobject/json-parser.c
new file mode 100644
index 000000000..e7947b340
--- /dev/null
+++ b/contrib/qemu/qobject/json-parser.c
@@ -0,0 +1,724 @@
+/*
+ * JSON Parser 
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include <stdarg.h>
+
+#include "qemu-common.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/json-parser.h"
+#include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/qerror.h"
+
+typedef struct JSONParserContext
+{
+    Error *err;
+    struct {
+        QObject **buf;
+        size_t pos;
+        size_t count;
+    } tokens;
+} JSONParserContext;
+
+#define BUG_ON(cond) assert(!(cond))
+
+/**
+ * TODO
+ *
+ * 0) make errors meaningful again
+ * 1) add geometry information to tokens
+ * 3) should we return a parsed size?
+ * 4) deal with premature EOI
+ */
+
+static QObject *parse_value(JSONParserContext *ctxt, va_list *ap);
+
+/**
+ * Token manipulators
+ *
+ * tokens are dictionaries that contain a type, a string value, and geometry information
+ * about a token identified by the lexer.  These are routines that make working with
+ * these objects a bit easier.
+ */
+static const char *token_get_value(QObject *obj)
+{
+    return qdict_get_str(qobject_to_qdict(obj), "token");
+}
+
+static JSONTokenType token_get_type(QObject *obj)
+{
+    return qdict_get_int(qobject_to_qdict(obj), "type");
+}
+
+static int token_is_operator(QObject *obj, char op)
+{
+    const char *val;
+
+    if (token_get_type(obj) != JSON_OPERATOR) {
+        return 0;
+    }
+
+    val = token_get_value(obj);
+
+    return (val[0] == op) && (val[1] == 0);
+}
+
+static int token_is_keyword(QObject *obj, const char *value)
+{
+    if (token_get_type(obj) != JSON_KEYWORD) {
+        return 0;
+    }
+
+    return strcmp(token_get_value(obj), value) == 0;
+}
+
+static int token_is_escape(QObject *obj, const char *value)
+{
+    if (token_get_type(obj) != JSON_ESCAPE) {
+        return 0;
+    }
+
+    return (strcmp(token_get_value(obj), value) == 0);
+}
+
+/**
+ * Error handler
+ */
+static void GCC_FMT_ATTR(3, 4) parse_error(JSONParserContext *ctxt,
+                                           QObject *token, const char *msg, ...)
+{
+    va_list ap;
+    char message[1024];
+    va_start(ap, msg);
+    vsnprintf(message, sizeof(message), msg, ap);
+    va_end(ap);
+    if (ctxt->err) {
+        error_free(ctxt->err);
+        ctxt->err = NULL;
+    }
+    error_set(&ctxt->err, QERR_JSON_PARSE_ERROR, message);
+}
+
+/**
+ * String helpers
+ *
+ * These helpers are used to unescape strings.
+ */
+static void wchar_to_utf8(uint16_t wchar, char *buffer, size_t buffer_length)
+{
+    if (wchar <= 0x007F) {
+        BUG_ON(buffer_length < 2);
+
+        buffer[0] = wchar & 0x7F;
+        buffer[1] = 0;
+    } else if (wchar <= 0x07FF) {
+        BUG_ON(buffer_length < 3);
+
+        buffer[0] = 0xC0 | ((wchar >> 6) & 0x1F);
+        buffer[1] = 0x80 | (wchar & 0x3F);
+        buffer[2] = 0;
+    } else {
+        BUG_ON(buffer_length < 4);
+
+        buffer[0] = 0xE0 | ((wchar >> 12) & 0x0F);
+        buffer[1] = 0x80 | ((wchar >> 6) & 0x3F);
+        buffer[2] = 0x80 | (wchar & 0x3F);
+        buffer[3] = 0;
+    }
+}
+
+static int hex2decimal(char ch)
+{
+    if (ch >= '0' && ch <= '9') {
+        return (ch - '0');
+    } else if (ch >= 'a' && ch <= 'f') {
+        return 10 + (ch - 'a');
+    } else if (ch >= 'A' && ch <= 'F') {
+        return 10 + (ch - 'A');
+    }
+
+    return -1;
+}
+
+/**
+ * parse_string(): Parse a json string and return a QObject
+ *
+ *  string
+ *      ""
+ *      " chars "
+ *  chars
+ *      char
+ *      char chars
+ *  char
+ *      any-Unicode-character-
+ *          except-"-or-\-or-
+ *          control-character
+ *      \"
+ *      \\
+ *      \/
+ *      \b
+ *      \f
+ *      \n
+ *      \r
+ *      \t
+ *      \u four-hex-digits 
+ */
+static QString *qstring_from_escaped_str(JSONParserContext *ctxt, QObject *token)
+{
+    const char *ptr = token_get_value(token);
+    QString *str;
+    int double_quote = 1;
+
+    if (*ptr == '"') {
+        double_quote = 1;
+    } else {
+        double_quote = 0;
+    }
+    ptr++;
+
+    str = qstring_new();
+    while (*ptr && 
+           ((double_quote && *ptr != '"') || (!double_quote && *ptr != '\''))) {
+        if (*ptr == '\\') {
+            ptr++;
+
+            switch (*ptr) {
+            case '"':
+                qstring_append(str, "\"");
+                ptr++;
+                break;
+            case '\'':
+                qstring_append(str, "'");
+                ptr++;
+                break;
+            case '\\':
+                qstring_append(str, "\\");
+                ptr++;
+                break;
+            case '/':
+                qstring_append(str, "/");
+                ptr++;
+                break;
+            case 'b':
+                qstring_append(str, "\b");
+                ptr++;
+                break;
+            case 'f':
+                qstring_append(str, "\f");
+                ptr++;
+                break;
+            case 'n':
+                qstring_append(str, "\n");
+                ptr++;
+                break;
+            case 'r':
+                qstring_append(str, "\r");
+                ptr++;
+                break;
+            case 't':
+                qstring_append(str, "\t");
+                ptr++;
+                break;
+            case 'u': {
+                uint16_t unicode_char = 0;
+                char utf8_char[4];
+                int i = 0;
+
+                ptr++;
+
+                for (i = 0; i < 4; i++) {
+                    if (qemu_isxdigit(*ptr)) {
+                        unicode_char |= hex2decimal(*ptr) << ((3 - i) * 4);
+                    } else {
+                        parse_error(ctxt, token,
+                                    "invalid hex escape sequence in string");
+                        goto out;
+                    }
+                    ptr++;
+                }
+
+                wchar_to_utf8(unicode_char, utf8_char, sizeof(utf8_char));
+                qstring_append(str, utf8_char);
+            }   break;
+            default:
+                parse_error(ctxt, token, "invalid escape sequence in string");
+                goto out;
+            }
+        } else {
+            char dummy[2];
+
+            dummy[0] = *ptr++;
+            dummy[1] = 0;
+
+            qstring_append(str, dummy);
+        }
+    }
+
+    return str;
+
+out:
+    QDECREF(str);
+    return NULL;
+}
+
+static QObject *parser_context_pop_token(JSONParserContext *ctxt)
+{
+    QObject *token;
+    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
+    token = ctxt->tokens.buf[ctxt->tokens.pos];
+    ctxt->tokens.pos++;
+    return token;
+}
+
+/* Note: parser_context_{peek|pop}_token do not increment the
+ * token object's refcount. In both cases the references will continue
+ * to be tracked and cleaned up in parser_context_free(), so do not
+ * attempt to free the token object.
+ */
+static QObject *parser_context_peek_token(JSONParserContext *ctxt)
+{
+    QObject *token;
+    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
+    token = ctxt->tokens.buf[ctxt->tokens.pos];
+    return token;
+}
+
+static JSONParserContext parser_context_save(JSONParserContext *ctxt)
+{
+    JSONParserContext saved_ctxt = {0};
+    saved_ctxt.tokens.pos = ctxt->tokens.pos;
+    saved_ctxt.tokens.count = ctxt->tokens.count;
+    saved_ctxt.tokens.buf = ctxt->tokens.buf;
+    return saved_ctxt;
+}
+
+static void parser_context_restore(JSONParserContext *ctxt,
+                                   JSONParserContext saved_ctxt)
+{
+    ctxt->tokens.pos = saved_ctxt.tokens.pos;
+    ctxt->tokens.count = saved_ctxt.tokens.count;
+    ctxt->tokens.buf = saved_ctxt.tokens.buf;
+}
+
+static void tokens_append_from_iter(QObject *obj, void *opaque)
+{
+    JSONParserContext *ctxt = opaque;
+    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
+    ctxt->tokens.buf[ctxt->tokens.pos++] = obj;
+    qobject_incref(obj);
+}
+
+static JSONParserContext *parser_context_new(QList *tokens)
+{
+    JSONParserContext *ctxt;
+    size_t count;
+
+    if (!tokens) {
+        return NULL;
+    }
+
+    count = qlist_size(tokens);
+    if (count == 0) {
+        return NULL;
+    }
+
+    ctxt = g_malloc0(sizeof(JSONParserContext));
+    ctxt->tokens.pos = 0;
+    ctxt->tokens.count = count;
+    ctxt->tokens.buf = g_malloc(count * sizeof(QObject *));
+    qlist_iter(tokens, tokens_append_from_iter, ctxt);
+    ctxt->tokens.pos = 0;
+
+    return ctxt;
+}
+
+/* to support error propagation, ctxt->err must be freed separately */
+static void parser_context_free(JSONParserContext *ctxt)
+{
+    int i;
+    if (ctxt) {
+        for (i = 0; i < ctxt->tokens.count; i++) {
+            qobject_decref(ctxt->tokens.buf[i]);
+        }
+        g_free(ctxt->tokens.buf);
+        g_free(ctxt);
+    }
+}
+
+/**
+ * Parsing rules
+ */
+static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
+{
+    QObject *key = NULL, *token = NULL, *value, *peek;
+    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+
+    peek = parser_context_peek_token(ctxt);
+    if (peek == NULL) {
+        parse_error(ctxt, NULL, "premature EOI");
+        goto out;
+    }
+
+    key = parse_value(ctxt, ap);
+    if (!key || qobject_type(key) != QTYPE_QSTRING) {
+        parse_error(ctxt, peek, "key is not a string in object");
+        goto out;
+    }
+
+    token = parser_context_pop_token(ctxt);
+    if (token == NULL) {
+        parse_error(ctxt, NULL, "premature EOI");
+        goto out;
+    }
+
+    if (!token_is_operator(token, ':')) {
+        parse_error(ctxt, token, "missing : in object pair");
+        goto out;
+    }
+
+    value = parse_value(ctxt, ap);
+    if (value == NULL) {
+        parse_error(ctxt, token, "Missing value in dict");
+        goto out;
+    }
+
+    qdict_put_obj(dict, qstring_get_str(qobject_to_qstring(key)), value);
+
+    qobject_decref(key);
+
+    return 0;
+
+out:
+    parser_context_restore(ctxt, saved_ctxt);
+    qobject_decref(key);
+
+    return -1;
+}
+
+static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
+{
+    QDict *dict = NULL;
+    QObject *token, *peek;
+    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+
+    token = parser_context_pop_token(ctxt);
+    if (token == NULL) {
+        goto out;
+    }
+
+    if (!token_is_operator(token, '{')) {
+        goto out;
+    }
+    token = NULL;
+
+    dict = qdict_new();
+
+    peek = parser_context_peek_token(ctxt);
+    if (peek == NULL) {
+        parse_error(ctxt, NULL, "premature EOI");
+        goto out;
+    }
+
+    if (!token_is_operator(peek, '}')) {
+        if (parse_pair(ctxt, dict, ap) == -1) {
+            goto out;
+        }
+
+        token = parser_context_pop_token(ctxt);
+        if (token == NULL) {
+            parse_error(ctxt, NULL, "premature EOI");
+            goto out;
+        }
+
+        while (!token_is_operator(token, '}')) {
+            if (!token_is_operator(token, ',')) {
+                parse_error(ctxt, token, "expected separator in dict");
+                goto out;
+            }
+            token = NULL;
+
+            if (parse_pair(ctxt, dict, ap) == -1) {
+                goto out;
+            }
+
+            token = parser_context_pop_token(ctxt);
+            if (token == NULL) {
+                parse_error(ctxt, NULL, "premature EOI");
+                goto out;
+            }
+        }
+        token = NULL;
+    } else {
+        token = parser_context_pop_token(ctxt);
+        token = NULL;
+    }
+
+    return QOBJECT(dict);
+
+out:
+    parser_context_restore(ctxt, saved_ctxt);
+    QDECREF(dict);
+    return NULL;
+}
+
+static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
+{
+    QList *list = NULL;
+    QObject *token, *peek;
+    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+
+    token = parser_context_pop_token(ctxt);
+    if (token == NULL) {
+        goto out;
+    }
+
+    if (!token_is_operator(token, '[')) {
+        token = NULL;
+        goto out;
+    }
+    token = NULL;
+
+    list = qlist_new();
+
+    peek = parser_context_peek_token(ctxt);
+    if (peek == NULL) {
+        parse_error(ctxt, NULL, "premature EOI");
+        goto out;
+    }
+
+    if (!token_is_operator(peek, ']')) {
+        QObject *obj;
+
+        obj = parse_value(ctxt, ap);
+        if (obj == NULL) {
+            parse_error(ctxt, token, "expecting value");
+            goto out;
+        }
+
+        qlist_append_obj(list, obj);
+
+        token = parser_context_pop_token(ctxt);
+        if (token == NULL) {
+            parse_error(ctxt, NULL, "premature EOI");
+            goto out;
+        }
+
+        while (!token_is_operator(token, ']')) {
+            if (!token_is_operator(token, ',')) {
+                parse_error(ctxt, token, "expected separator in list");
+                goto out;
+            }
+
+            token = NULL;
+
+            obj = parse_value(ctxt, ap);
+            if (obj == NULL) {
+                parse_error(ctxt, token, "expecting value");
+                goto out;
+            }
+
+            qlist_append_obj(list, obj);
+
+            token = parser_context_pop_token(ctxt);
+            if (token == NULL) {
+                parse_error(ctxt, NULL, "premature EOI");
+                goto out;
+            }
+        }
+
+        token = NULL;
+    } else {
+        token = parser_context_pop_token(ctxt);
+        token = NULL;
+    }
+
+    return QOBJECT(list);
+
+out:
+    parser_context_restore(ctxt, saved_ctxt);
+    QDECREF(list);
+    return NULL;
+}
+
+static QObject *parse_keyword(JSONParserContext *ctxt)
+{
+    QObject *token, *ret;
+    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+
+    token = parser_context_pop_token(ctxt);
+    if (token == NULL) {
+        goto out;
+    }
+
+    if (token_get_type(token) != JSON_KEYWORD) {
+        goto out;
+    }
+
+    if (token_is_keyword(token, "true")) {
+        ret = QOBJECT(qbool_from_int(true));
+    } else if (token_is_keyword(token, "false")) {
+        ret = QOBJECT(qbool_from_int(false));
+    } else {
+        parse_error(ctxt, token, "invalid keyword `%s'", token_get_value(token));
+        goto out;
+    }
+
+    return ret;
+
+out: 
+    parser_context_restore(ctxt, saved_ctxt);
+
+    return NULL;
+}
+
+static QObject *parse_escape(JSONParserContext *ctxt, va_list *ap)
+{
+    QObject *token = NULL, *obj;
+    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+
+    if (ap == NULL) {
+        goto out;
+    }
+
+    token = parser_context_pop_token(ctxt);
+    if (token == NULL) {
+        goto out;
+    }
+
+    if (token_is_escape(token, "%p")) {
+        obj = va_arg(*ap, QObject *);
+    } else if (token_is_escape(token, "%i")) {
+        obj = QOBJECT(qbool_from_int(va_arg(*ap, int)));
+    } else if (token_is_escape(token, "%d")) {
+        obj = QOBJECT(qint_from_int(va_arg(*ap, int)));
+    } else if (token_is_escape(token, "%ld")) {
+        obj = QOBJECT(qint_from_int(va_arg(*ap, long)));
+    } else if (token_is_escape(token, "%lld") ||
+               token_is_escape(token, "%I64d")) {
+        obj = QOBJECT(qint_from_int(va_arg(*ap, long long)));
+    } else if (token_is_escape(token, "%s")) {
+        obj = QOBJECT(qstring_from_str(va_arg(*ap, const char *)));
+    } else if (token_is_escape(token, "%f")) {
+        obj = QOBJECT(qfloat_from_double(va_arg(*ap, double)));
+    } else {
+        goto out;
+    }
+
+    return obj;
+
+out:
+    parser_context_restore(ctxt, saved_ctxt);
+
+    return NULL;
+}
+
+static QObject *parse_literal(JSONParserContext *ctxt)
+{
+    QObject *token, *obj;
+    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+
+    token = parser_context_pop_token(ctxt);
+    if (token == NULL) {
+        goto out;
+    }
+
+    switch (token_get_type(token)) {
+    case JSON_STRING:
+        obj = QOBJECT(qstring_from_escaped_str(ctxt, token));
+        break;
+    case JSON_INTEGER: {
+        /* A possibility exists that this is a whole-valued float where the
+         * fractional part was left out due to being 0 (.0). It's not a big
+         * deal to treat these as ints in the parser, so long as users of the
+         * resulting QObject know to expect a QInt in place of a QFloat in
+         * cases like these.
+         *
+         * However, in some cases these values will overflow/underflow a
+         * QInt/int64 container, thus we should assume these are to be handled
+         * as QFloats/doubles rather than silently changing their values.
+         *
+         * strtoll() indicates these instances by setting errno to ERANGE
+         */
+        int64_t value;
+
+        errno = 0; /* strtoll doesn't set errno on success */
+        value = strtoll(token_get_value(token), NULL, 10);
+        if (errno != ERANGE) {
+            obj = QOBJECT(qint_from_int(value));
+            break;
+        }
+        /* fall through to JSON_FLOAT */
+    }
+    case JSON_FLOAT:
+        /* FIXME dependent on locale */
+        obj = QOBJECT(qfloat_from_double(strtod(token_get_value(token), NULL)));
+        break;
+    default:
+        goto out;
+    }
+
+    return obj;
+
+out:
+    parser_context_restore(ctxt, saved_ctxt);
+
+    return NULL;
+}
+
+static QObject *parse_value(JSONParserContext *ctxt, va_list *ap)
+{
+    QObject *obj;
+
+    obj = parse_object(ctxt, ap);
+    if (obj == NULL) {
+        obj = parse_array(ctxt, ap);
+    }
+    if (obj == NULL) {
+        obj = parse_escape(ctxt, ap);
+    }
+    if (obj == NULL) {
+        obj = parse_keyword(ctxt);
+    } 
+    if (obj == NULL) {
+        obj = parse_literal(ctxt);
+    }
+
+    return obj;
+}
+
+QObject *json_parser_parse(QList *tokens, va_list *ap)
+{
+    return json_parser_parse_err(tokens, ap, NULL);
+}
+
+QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp)
+{
+    JSONParserContext *ctxt = parser_context_new(tokens);
+    QObject *result;
+
+    if (!ctxt) {
+        return NULL;
+    }
+
+    result = parse_value(ctxt, ap);
+
+    error_propagate(errp, ctxt->err);
+
+    parser_context_free(ctxt);
+
+    return result;
+}
diff --git a/contrib/qemu/qobject/json-streamer.c b/contrib/qemu/qobject/json-streamer.c
new file mode 100644
index 000000000..1b2f9b1d1
--- /dev/null
+++ b/contrib/qemu/qobject/json-streamer.c
@@ -0,0 +1,122 @@
+/*
+ * JSON streaming support
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qdict.h"
+#include "qemu-common.h"
+#include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/json-streamer.h"
+
+#define MAX_TOKEN_SIZE (64ULL << 20)
+#define MAX_NESTING (1ULL << 10)
+
+static void json_message_process_token(JSONLexer *lexer, QString *token, JSONTokenType type, int x, int y)
+{
+    JSONMessageParser *parser = container_of(lexer, JSONMessageParser, lexer);
+    QDict *dict;
+
+    if (type == JSON_OPERATOR) {
+        switch (qstring_get_str(token)[0]) {
+        case '{':
+            parser->brace_count++;
+            break;
+        case '}':
+            parser->brace_count--;
+            break;
+        case '[':
+            parser->bracket_count++;
+            break;
+        case ']':
+            parser->bracket_count--;
+            break;
+        default:
+            break;
+        }
+    }
+
+    dict = qdict_new();
+    qdict_put(dict, "type", qint_from_int(type));
+    QINCREF(token);
+    qdict_put(dict, "token", token);
+    qdict_put(dict, "x", qint_from_int(x));
+    qdict_put(dict, "y", qint_from_int(y));
+
+    parser->token_size += token->length;
+
+    qlist_append(parser->tokens, dict);
+
+    if (type == JSON_ERROR) {
+        goto out_emit_bad;
+    } else if (parser->brace_count < 0 ||
+        parser->bracket_count < 0 ||
+        (parser->brace_count == 0 &&
+         parser->bracket_count == 0)) {
+        goto out_emit;
+    } else if (parser->token_size > MAX_TOKEN_SIZE ||
+               parser->bracket_count > MAX_NESTING ||
+               parser->brace_count > MAX_NESTING) {
+        /* Security consideration, we limit total memory allocated per object
+         * and the maximum recursion depth that a message can force.
+         */
+        goto out_emit;
+    }
+
+    return;
+
+out_emit_bad:
+    /* clear out token list and tell the parser to emit and error
+     * indication by passing it a NULL list
+     */
+    QDECREF(parser->tokens);
+    parser->tokens = NULL;
+out_emit:
+    /* send current list of tokens to parser and reset tokenizer */
+    parser->brace_count = 0;
+    parser->bracket_count = 0;
+    parser->emit(parser, parser->tokens);
+    if (parser->tokens) {
+        QDECREF(parser->tokens);
+    }
+    parser->tokens = qlist_new();
+    parser->token_size = 0;
+}
+
+void json_message_parser_init(JSONMessageParser *parser,
+                              void (*func)(JSONMessageParser *, QList *))
+{
+    parser->emit = func;
+    parser->brace_count = 0;
+    parser->bracket_count = 0;
+    parser->tokens = qlist_new();
+    parser->token_size = 0;
+
+    json_lexer_init(&parser->lexer, json_message_process_token);
+}
+
+int json_message_parser_feed(JSONMessageParser *parser,
+                             const char *buffer, size_t size)
+{
+    return json_lexer_feed(&parser->lexer, buffer, size);
+}
+
+int json_message_parser_flush(JSONMessageParser *parser)
+{
+    return json_lexer_flush(&parser->lexer);
+}
+
+void json_message_parser_destroy(JSONMessageParser *parser)
+{
+    json_lexer_destroy(&parser->lexer);
+    QDECREF(parser->tokens);
+}
diff --git a/contrib/qemu/qobject/qbool.c b/contrib/qemu/qobject/qbool.c
new file mode 100644
index 000000000..a3d2afa82
--- /dev/null
+++ b/contrib/qemu/qobject/qbool.c
@@ -0,0 +1,68 @@
+/*
+ * QBool Module
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu-common.h"
+
+static void qbool_destroy_obj(QObject *obj);
+
+static const QType qbool_type = {
+    .code = QTYPE_QBOOL,
+    .destroy = qbool_destroy_obj,
+};
+
+/**
+ * qbool_from_int(): Create a new QBool from an int
+ *
+ * Return strong reference.
+ */
+QBool *qbool_from_int(int value)
+{
+    QBool *qb;
+
+    qb = g_malloc(sizeof(*qb));
+    qb->value = value;
+    QOBJECT_INIT(qb, &qbool_type);
+
+    return qb;
+}
+
+/**
+ * qbool_get_int(): Get the stored int
+ */
+int qbool_get_int(const QBool *qb)
+{
+    return qb->value;
+}
+
+/**
+ * qobject_to_qbool(): Convert a QObject into a QBool
+ */
+QBool *qobject_to_qbool(const QObject *obj)
+{
+    if (qobject_type(obj) != QTYPE_QBOOL)
+        return NULL;
+
+    return container_of(obj, QBool, base);
+}
+
+/**
+ * qbool_destroy_obj(): Free all memory allocated by a
+ * QBool object
+ */
+static void qbool_destroy_obj(QObject *obj)
+{
+    assert(obj != NULL);
+    g_free(qobject_to_qbool(obj));
+}
diff --git a/contrib/qemu/qobject/qdict.c b/contrib/qemu/qobject/qdict.c
new file mode 100644
index 000000000..ed381f9a5
--- /dev/null
+++ b/contrib/qemu/qobject/qdict.c
@@ -0,0 +1,478 @@
+/*
+ * QDict Module
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu/queue.h"
+#include "qemu-common.h"
+
+static void qdict_destroy_obj(QObject *obj);
+
+static const QType qdict_type = {
+    .code = QTYPE_QDICT,
+    .destroy = qdict_destroy_obj,
+};
+
+/**
+ * qdict_new(): Create a new QDict
+ *
+ * Return strong reference.
+ */
+QDict *qdict_new(void)
+{
+    QDict *qdict;
+
+    qdict = g_malloc0(sizeof(*qdict));
+    QOBJECT_INIT(qdict, &qdict_type);
+
+    return qdict;
+}
+
+/**
+ * qobject_to_qdict(): Convert a QObject into a QDict
+ */
+QDict *qobject_to_qdict(const QObject *obj)
+{
+    if (qobject_type(obj) != QTYPE_QDICT)
+        return NULL;
+
+    return container_of(obj, QDict, base);
+}
+
+/**
+ * tdb_hash(): based on the hash agorithm from gdbm, via tdb
+ * (from module-init-tools)
+ */
+static unsigned int tdb_hash(const char *name)
+{
+    unsigned value;	/* Used to compute the hash value.  */
+    unsigned   i;	/* Used to cycle through random values. */
+
+    /* Set the initial value from the key size. */
+    for (value = 0x238F13AF * strlen(name), i=0; name[i]; i++)
+        value = (value + (((const unsigned char *)name)[i] << (i*5 % 24)));
+
+    return (1103515243 * value + 12345);
+}
+
+/**
+ * alloc_entry(): allocate a new QDictEntry
+ */
+static QDictEntry *alloc_entry(const char *key, QObject *value)
+{
+    QDictEntry *entry;
+
+    entry = g_malloc0(sizeof(*entry));
+    entry->key = g_strdup(key);
+    entry->value = value;
+
+    return entry;
+}
+
+/**
+ * qdict_entry_value(): Return qdict entry value
+ *
+ * Return weak reference.
+ */
+QObject *qdict_entry_value(const QDictEntry *entry)
+{
+    return entry->value;
+}
+
+/**
+ * qdict_entry_key(): Return qdict entry key
+ *
+ * Return a *pointer* to the string, it has to be duplicated before being
+ * stored.
+ */
+const char *qdict_entry_key(const QDictEntry *entry)
+{
+    return entry->key;
+}
+
+/**
+ * qdict_find(): List lookup function
+ */
+static QDictEntry *qdict_find(const QDict *qdict,
+                              const char *key, unsigned int bucket)
+{
+    QDictEntry *entry;
+
+    QLIST_FOREACH(entry, &qdict->table[bucket], next)
+        if (!strcmp(entry->key, key))
+            return entry;
+
+    return NULL;
+}
+
+/**
+ * qdict_put_obj(): Put a new QObject into the dictionary
+ *
+ * Insert the pair 'key:value' into 'qdict', if 'key' already exists
+ * its 'value' will be replaced.
+ *
+ * This is done by freeing the reference to the stored QObject and
+ * storing the new one in the same entry.
+ *
+ * NOTE: ownership of 'value' is transferred to the QDict
+ */
+void qdict_put_obj(QDict *qdict, const char *key, QObject *value)
+{
+    unsigned int bucket;
+    QDictEntry *entry;
+
+    bucket = tdb_hash(key) % QDICT_BUCKET_MAX;
+    entry = qdict_find(qdict, key, bucket);
+    if (entry) {
+        /* replace key's value */
+        qobject_decref(entry->value);
+        entry->value = value;
+    } else {
+        /* allocate a new entry */
+        entry = alloc_entry(key, value);
+        QLIST_INSERT_HEAD(&qdict->table[bucket], entry, next);
+        qdict->size++;
+    }
+}
+
+/**
+ * qdict_get(): Lookup for a given 'key'
+ *
+ * Return a weak reference to the QObject associated with 'key' if
+ * 'key' is present in the dictionary, NULL otherwise.
+ */
+QObject *qdict_get(const QDict *qdict, const char *key)
+{
+    QDictEntry *entry;
+
+    entry = qdict_find(qdict, key, tdb_hash(key) % QDICT_BUCKET_MAX);
+    return (entry == NULL ? NULL : entry->value);
+}
+
+/**
+ * qdict_haskey(): Check if 'key' exists
+ *
+ * Return 1 if 'key' exists in the dict, 0 otherwise
+ */
+int qdict_haskey(const QDict *qdict, const char *key)
+{
+    unsigned int bucket = tdb_hash(key) % QDICT_BUCKET_MAX;
+    return (qdict_find(qdict, key, bucket) == NULL ? 0 : 1);
+}
+
+/**
+ * qdict_size(): Return the size of the dictionary
+ */
+size_t qdict_size(const QDict *qdict)
+{
+    return qdict->size;
+}
+
+/**
+ * qdict_get_obj(): Get a QObject of a specific type
+ */
+static QObject *qdict_get_obj(const QDict *qdict, const char *key,
+                              qtype_code type)
+{
+    QObject *obj;
+
+    obj = qdict_get(qdict, key);
+    assert(obj != NULL);
+    assert(qobject_type(obj) == type);
+
+    return obj;
+}
+
+/**
+ * qdict_get_double(): Get an number mapped by 'key'
+ *
+ * This function assumes that 'key' exists and it stores a
+ * QFloat or QInt object.
+ *
+ * Return number mapped by 'key'.
+ */
+double qdict_get_double(const QDict *qdict, const char *key)
+{
+    QObject *obj = qdict_get(qdict, key);
+
+    assert(obj);
+    switch (qobject_type(obj)) {
+    case QTYPE_QFLOAT:
+        return qfloat_get_double(qobject_to_qfloat(obj));
+    case QTYPE_QINT:
+        return qint_get_int(qobject_to_qint(obj));
+    default:
+        abort();
+    }
+}
+
+/**
+ * qdict_get_int(): Get an integer mapped by 'key'
+ *
+ * This function assumes that 'key' exists and it stores a
+ * QInt object.
+ *
+ * Return integer mapped by 'key'.
+ */
+int64_t qdict_get_int(const QDict *qdict, const char *key)
+{
+    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QINT);
+    return qint_get_int(qobject_to_qint(obj));
+}
+
+/**
+ * qdict_get_bool(): Get a bool mapped by 'key'
+ *
+ * This function assumes that 'key' exists and it stores a
+ * QBool object.
+ *
+ * Return bool mapped by 'key'.
+ */
+int qdict_get_bool(const QDict *qdict, const char *key)
+{
+    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QBOOL);
+    return qbool_get_int(qobject_to_qbool(obj));
+}
+
+/**
+ * qdict_get_qlist(): Get the QList mapped by 'key'
+ *
+ * This function assumes that 'key' exists and it stores a
+ * QList object.
+ *
+ * Return QList mapped by 'key'.
+ */
+QList *qdict_get_qlist(const QDict *qdict, const char *key)
+{
+    return qobject_to_qlist(qdict_get_obj(qdict, key, QTYPE_QLIST));
+}
+
+/**
+ * qdict_get_qdict(): Get the QDict mapped by 'key'
+ *
+ * This function assumes that 'key' exists and it stores a
+ * QDict object.
+ *
+ * Return QDict mapped by 'key'.
+ */
+QDict *qdict_get_qdict(const QDict *qdict, const char *key)
+{
+    return qobject_to_qdict(qdict_get_obj(qdict, key, QTYPE_QDICT));
+}
+
+/**
+ * qdict_get_str(): Get a pointer to the stored string mapped
+ * by 'key'
+ *
+ * This function assumes that 'key' exists and it stores a
+ * QString object.
+ *
+ * Return pointer to the string mapped by 'key'.
+ */
+const char *qdict_get_str(const QDict *qdict, const char *key)
+{
+    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QSTRING);
+    return qstring_get_str(qobject_to_qstring(obj));
+}
+
+/**
+ * qdict_get_try_int(): Try to get integer mapped by 'key'
+ *
+ * Return integer mapped by 'key', if it is not present in
+ * the dictionary or if the stored object is not of QInt type
+ * 'def_value' will be returned.
+ */
+int64_t qdict_get_try_int(const QDict *qdict, const char *key,
+                          int64_t def_value)
+{
+    QObject *obj;
+
+    obj = qdict_get(qdict, key);
+    if (!obj || qobject_type(obj) != QTYPE_QINT)
+        return def_value;
+
+    return qint_get_int(qobject_to_qint(obj));
+}
+
+/**
+ * qdict_get_try_bool(): Try to get a bool mapped by 'key'
+ *
+ * Return bool mapped by 'key', if it is not present in the
+ * dictionary or if the stored object is not of QBool type
+ * 'def_value' will be returned.
+ */
+int qdict_get_try_bool(const QDict *qdict, const char *key, int def_value)
+{
+    QObject *obj;
+
+    obj = qdict_get(qdict, key);
+    if (!obj || qobject_type(obj) != QTYPE_QBOOL)
+        return def_value;
+
+    return qbool_get_int(qobject_to_qbool(obj));
+}
+
+/**
+ * qdict_get_try_str(): Try to get a pointer to the stored string
+ * mapped by 'key'
+ *
+ * Return a pointer to the string mapped by 'key', if it is not present
+ * in the dictionary or if the stored object is not of QString type
+ * NULL will be returned.
+ */
+const char *qdict_get_try_str(const QDict *qdict, const char *key)
+{
+    QObject *obj;
+
+    obj = qdict_get(qdict, key);
+    if (!obj || qobject_type(obj) != QTYPE_QSTRING)
+        return NULL;
+
+    return qstring_get_str(qobject_to_qstring(obj));
+}
+
+/**
+ * qdict_iter(): Iterate over all the dictionary's stored values.
+ *
+ * This function allows the user to provide an iterator, which will be
+ * called for each stored value in the dictionary.
+ */
+void qdict_iter(const QDict *qdict,
+                void (*iter)(const char *key, QObject *obj, void *opaque),
+                void *opaque)
+{
+    int i;
+    QDictEntry *entry;
+
+    for (i = 0; i < QDICT_BUCKET_MAX; i++) {
+        QLIST_FOREACH(entry, &qdict->table[i], next)
+            iter(entry->key, entry->value, opaque);
+    }
+}
+
+static QDictEntry *qdict_next_entry(const QDict *qdict, int first_bucket)
+{
+    int i;
+
+    for (i = first_bucket; i < QDICT_BUCKET_MAX; i++) {
+        if (!QLIST_EMPTY(&qdict->table[i])) {
+            return QLIST_FIRST(&qdict->table[i]);
+        }
+    }
+
+    return NULL;
+}
+
+/**
+ * qdict_first(): Return first qdict entry for iteration.
+ */
+const QDictEntry *qdict_first(const QDict *qdict)
+{
+    return qdict_next_entry(qdict, 0);
+}
+
+/**
+ * qdict_next(): Return next qdict entry in an iteration.
+ */
+const QDictEntry *qdict_next(const QDict *qdict, const QDictEntry *entry)
+{
+    QDictEntry *ret;
+
+    ret = QLIST_NEXT(entry, next);
+    if (!ret) {
+        unsigned int bucket = tdb_hash(entry->key) % QDICT_BUCKET_MAX;
+        ret = qdict_next_entry(qdict, bucket + 1);
+    }
+
+    return ret;
+}
+
+/**
+ * qdict_clone_shallow(): Clones a given QDict. Its entries are not copied, but
+ * another reference is added.
+ */
+QDict *qdict_clone_shallow(const QDict *src)
+{
+    QDict *dest;
+    QDictEntry *entry;
+    int i;
+
+    dest = qdict_new();
+
+    for (i = 0; i < QDICT_BUCKET_MAX; i++) {
+        QLIST_FOREACH(entry, &src->table[i], next) {
+            qobject_incref(entry->value);
+            qdict_put_obj(dest, entry->key, entry->value);
+        }
+    }
+
+    return dest;
+}
+
+/**
+ * qentry_destroy(): Free all the memory allocated by a QDictEntry
+ */
+static void qentry_destroy(QDictEntry *e)
+{
+    assert(e != NULL);
+    assert(e->key != NULL);
+    assert(e->value != NULL);
+
+    qobject_decref(e->value);
+    g_free(e->key);
+    g_free(e);
+}
+
+/**
+ * qdict_del(): Delete a 'key:value' pair from the dictionary
+ *
+ * This will destroy all data allocated by this entry.
+ */
+void qdict_del(QDict *qdict, const char *key)
+{
+    QDictEntry *entry;
+
+    entry = qdict_find(qdict, key, tdb_hash(key) % QDICT_BUCKET_MAX);
+    if (entry) {
+        QLIST_REMOVE(entry, next);
+        qentry_destroy(entry);
+        qdict->size--;
+    }
+}
+
+/**
+ * qdict_destroy_obj(): Free all the memory allocated by a QDict
+ */
+static void qdict_destroy_obj(QObject *obj)
+{
+    int i;
+    QDict *qdict;
+
+    assert(obj != NULL);
+    qdict = qobject_to_qdict(obj);
+
+    for (i = 0; i < QDICT_BUCKET_MAX; i++) {
+        QDictEntry *entry = QLIST_FIRST(&qdict->table[i]);
+        while (entry) {
+            QDictEntry *tmp = QLIST_NEXT(entry, next);
+            QLIST_REMOVE(entry, next);
+            qentry_destroy(entry);
+            entry = tmp;
+        }
+    }
+
+    g_free(qdict);
+}
diff --git a/contrib/qemu/qobject/qerror.c b/contrib/qemu/qobject/qerror.c
new file mode 100644
index 000000000..3aee1cf6a
--- /dev/null
+++ b/contrib/qemu/qobject/qerror.c
@@ -0,0 +1,156 @@
+/*
+ * QError Module
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "monitor/monitor.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu-common.h"
+
+static void qerror_destroy_obj(QObject *obj);
+
+static const QType qerror_type = {
+    .code = QTYPE_QERROR,
+    .destroy = qerror_destroy_obj,
+};
+
+/**
+ * qerror_new(): Create a new QError
+ *
+ * Return strong reference.
+ */
+static QError *qerror_new(void)
+{
+    QError *qerr;
+
+    qerr = g_malloc0(sizeof(*qerr));
+    QOBJECT_INIT(qerr, &qerror_type);
+
+    return qerr;
+}
+
+/**
+ * qerror_from_info(): Create a new QError from error information
+ *
+ * Return strong reference.
+ */
+static QError *qerror_from_info(ErrorClass err_class, const char *fmt,
+                                va_list *va)
+{
+    QError *qerr;
+
+    qerr = qerror_new();
+    loc_save(&qerr->loc);
+
+    qerr->err_msg = g_strdup_vprintf(fmt, *va);
+    qerr->err_class = err_class;
+
+    return qerr;
+}
+
+/**
+ * qerror_human(): Format QError data into human-readable string.
+ */
+QString *qerror_human(const QError *qerror)
+{
+    return qstring_from_str(qerror->err_msg);
+}
+
+/**
+ * qerror_print(): Print QError data
+ *
+ * This function will print the member 'desc' of the specified QError object,
+ * it uses error_report() for this, so that the output is routed to the right
+ * place (ie. stderr or Monitor's device).
+ */
+static void qerror_print(QError *qerror)
+{
+    QString *qstring = qerror_human(qerror);
+    loc_push_restore(&qerror->loc);
+    error_report("%s", qstring_get_str(qstring));
+    loc_pop(&qerror->loc);
+    QDECREF(qstring);
+}
+
+void qerror_report(ErrorClass eclass, const char *fmt, ...)
+{
+    va_list va;
+    QError *qerror;
+
+    va_start(va, fmt);
+    qerror = qerror_from_info(eclass, fmt, &va);
+    va_end(va);
+
+    if (monitor_cur_is_qmp()) {
+        monitor_set_error(cur_mon, qerror);
+    } else {
+        qerror_print(qerror);
+        QDECREF(qerror);
+    }
+}
+
+/* Evil... */
+struct Error
+{
+    char *msg;
+    ErrorClass err_class;
+};
+
+void qerror_report_err(Error *err)
+{
+    QError *qerr;
+
+    qerr = qerror_new();
+    loc_save(&qerr->loc);
+    qerr->err_msg = g_strdup(err->msg);
+    qerr->err_class = err->err_class;
+
+    if (monitor_cur_is_qmp()) {
+        monitor_set_error(cur_mon, qerr);
+    } else {
+        qerror_print(qerr);
+        QDECREF(qerr);
+    }
+}
+
+void assert_no_error(Error *err)
+{
+    if (err) {
+        qerror_report_err(err);
+        abort();
+    }
+}
+
+/**
+ * qobject_to_qerror(): Convert a QObject into a QError
+ */
+static QError *qobject_to_qerror(const QObject *obj)
+{
+    if (qobject_type(obj) != QTYPE_QERROR) {
+        return NULL;
+    }
+
+    return container_of(obj, QError, base);
+}
+
+/**
+ * qerror_destroy_obj(): Free all memory allocated by a QError
+ */
+static void qerror_destroy_obj(QObject *obj)
+{
+    QError *qerr;
+
+    assert(obj != NULL);
+    qerr = qobject_to_qerror(obj);
+
+    g_free(qerr->err_msg);
+    g_free(qerr);
+}
diff --git a/contrib/qemu/qobject/qfloat.c b/contrib/qemu/qobject/qfloat.c
new file mode 100644
index 000000000..7de0992db
--- /dev/null
+++ b/contrib/qemu/qobject/qfloat.c
@@ -0,0 +1,68 @@
+/*
+ * QFloat Module
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu-common.h"
+
+static void qfloat_destroy_obj(QObject *obj);
+
+static const QType qfloat_type = {
+    .code = QTYPE_QFLOAT,
+    .destroy = qfloat_destroy_obj,
+};
+
+/**
+ * qfloat_from_int(): Create a new QFloat from a float
+ *
+ * Return strong reference.
+ */
+QFloat *qfloat_from_double(double value)
+{
+    QFloat *qf;
+
+    qf = g_malloc(sizeof(*qf));
+    qf->value = value;
+    QOBJECT_INIT(qf, &qfloat_type);
+
+    return qf;
+}
+
+/**
+ * qfloat_get_double(): Get the stored float
+ */
+double qfloat_get_double(const QFloat *qf)
+{
+    return qf->value;
+}
+
+/**
+ * qobject_to_qfloat(): Convert a QObject into a QFloat
+ */
+QFloat *qobject_to_qfloat(const QObject *obj)
+{
+    if (qobject_type(obj) != QTYPE_QFLOAT)
+        return NULL;
+
+    return container_of(obj, QFloat, base);
+}
+
+/**
+ * qfloat_destroy_obj(): Free all memory allocated by a
+ * QFloat object
+ */
+static void qfloat_destroy_obj(QObject *obj)
+{
+    assert(obj != NULL);
+    g_free(qobject_to_qfloat(obj));
+}
diff --git a/contrib/qemu/qobject/qint.c b/contrib/qemu/qobject/qint.c
new file mode 100644
index 000000000..86b9b04f0
--- /dev/null
+++ b/contrib/qemu/qobject/qint.c
@@ -0,0 +1,67 @@
+/*
+ * QInt Module
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu-common.h"
+
+static void qint_destroy_obj(QObject *obj);
+
+static const QType qint_type = {
+    .code = QTYPE_QINT,
+    .destroy = qint_destroy_obj,
+};
+
+/**
+ * qint_from_int(): Create a new QInt from an int64_t
+ *
+ * Return strong reference.
+ */
+QInt *qint_from_int(int64_t value)
+{
+    QInt *qi;
+
+    qi = g_malloc(sizeof(*qi));
+    qi->value = value;
+    QOBJECT_INIT(qi, &qint_type);
+
+    return qi;
+}
+
+/**
+ * qint_get_int(): Get the stored integer
+ */
+int64_t qint_get_int(const QInt *qi)
+{
+    return qi->value;
+}
+
+/**
+ * qobject_to_qint(): Convert a QObject into a QInt
+ */
+QInt *qobject_to_qint(const QObject *obj)
+{
+    if (qobject_type(obj) != QTYPE_QINT)
+        return NULL;
+
+    return container_of(obj, QInt, base);
+}
+
+/**
+ * qint_destroy_obj(): Free all memory allocated by a
+ * QInt object
+ */
+static void qint_destroy_obj(QObject *obj)
+{
+    assert(obj != NULL);
+    g_free(qobject_to_qint(obj));
+}
diff --git a/contrib/qemu/qobject/qjson.c b/contrib/qemu/qobject/qjson.c
new file mode 100644
index 000000000..19085a1bb
--- /dev/null
+++ b/contrib/qemu/qobject/qjson.c
@@ -0,0 +1,282 @@
+/*
+ * QObject JSON integration
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/json-parser.h"
+#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qdict.h"
+
+typedef struct JSONParsingState
+{
+    JSONMessageParser parser;
+    va_list *ap;
+    QObject *result;
+} JSONParsingState;
+
+static void parse_json(JSONMessageParser *parser, QList *tokens)
+{
+    JSONParsingState *s = container_of(parser, JSONParsingState, parser);
+    s->result = json_parser_parse(tokens, s->ap);
+}
+
+QObject *qobject_from_jsonv(const char *string, va_list *ap)
+{
+    JSONParsingState state = {};
+
+    state.ap = ap;
+
+    json_message_parser_init(&state.parser, parse_json);
+    json_message_parser_feed(&state.parser, string, strlen(string));
+    json_message_parser_flush(&state.parser);
+    json_message_parser_destroy(&state.parser);
+
+    return state.result;
+}
+
+QObject *qobject_from_json(const char *string)
+{
+    return qobject_from_jsonv(string, NULL);
+}
+
+/*
+ * IMPORTANT: This function aborts on error, thus it must not
+ * be used with untrusted arguments.
+ */
+QObject *qobject_from_jsonf(const char *string, ...)
+{
+    QObject *obj;
+    va_list ap;
+
+    va_start(ap, string);
+    obj = qobject_from_jsonv(string, &ap);
+    va_end(ap);
+
+    assert(obj != NULL);
+    return obj;
+}
+
+typedef struct ToJsonIterState
+{
+    int indent;
+    int pretty;
+    int count;
+    QString *str;
+} ToJsonIterState;
+
+static void to_json(const QObject *obj, QString *str, int pretty, int indent);
+
+static void to_json_dict_iter(const char *key, QObject *obj, void *opaque)
+{
+    ToJsonIterState *s = opaque;
+    QString *qkey;
+    int j;
+
+    if (s->count)
+        qstring_append(s->str, ", ");
+
+    if (s->pretty) {
+        qstring_append(s->str, "\n");
+        for (j = 0 ; j < s->indent ; j++)
+            qstring_append(s->str, "    ");
+    }
+
+    qkey = qstring_from_str(key);
+    to_json(QOBJECT(qkey), s->str, s->pretty, s->indent);
+    QDECREF(qkey);
+
+    qstring_append(s->str, ": ");
+    to_json(obj, s->str, s->pretty, s->indent);
+    s->count++;
+}
+
+static void to_json_list_iter(QObject *obj, void *opaque)
+{
+    ToJsonIterState *s = opaque;
+    int j;
+
+    if (s->count)
+        qstring_append(s->str, ", ");
+
+    if (s->pretty) {
+        qstring_append(s->str, "\n");
+        for (j = 0 ; j < s->indent ; j++)
+            qstring_append(s->str, "    ");
+    }
+
+    to_json(obj, s->str, s->pretty, s->indent);
+    s->count++;
+}
+
+static void to_json(const QObject *obj, QString *str, int pretty, int indent)
+{
+    switch (qobject_type(obj)) {
+    case QTYPE_QINT: {
+        QInt *val = qobject_to_qint(obj);
+        char buffer[1024];
+
+        snprintf(buffer, sizeof(buffer), "%" PRId64, qint_get_int(val));
+        qstring_append(str, buffer);
+        break;
+    }
+    case QTYPE_QSTRING: {
+        QString *val = qobject_to_qstring(obj);
+        const char *ptr;
+        int cp;
+        char buf[16];
+        char *end;
+
+        ptr = qstring_get_str(val);
+        qstring_append(str, "\"");
+
+        for (; *ptr; ptr = end) {
+            cp = mod_utf8_codepoint(ptr, 6, &end);
+            switch (cp) {
+            case '\"':
+                qstring_append(str, "\\\"");
+                break;
+            case '\\':
+                qstring_append(str, "\\\\");
+                break;
+            case '\b':
+                qstring_append(str, "\\b");
+                break;
+            case '\f':
+                qstring_append(str, "\\f");
+                break;
+            case '\n':
+                qstring_append(str, "\\n");
+                break;
+            case '\r':
+                qstring_append(str, "\\r");
+                break;
+            case '\t':
+                qstring_append(str, "\\t");
+                break;
+            default:
+                if (cp < 0) {
+                    cp = 0xFFFD; /* replacement character */
+                }
+                if (cp > 0xFFFF) {
+                    /* beyond BMP; need a surrogate pair */
+                    snprintf(buf, sizeof(buf), "\\u%04X\\u%04X",
+                             0xD800 + ((cp - 0x10000) >> 10),
+                             0xDC00 + ((cp - 0x10000) & 0x3FF));
+                } else if (cp < 0x20 || cp >= 0x7F) {
+                    snprintf(buf, sizeof(buf), "\\u%04X", cp);
+                } else {
+                    buf[0] = cp;
+                    buf[1] = 0;
+                }
+                qstring_append(str, buf);
+            }
+        };
+
+        qstring_append(str, "\"");
+        break;
+    }
+    case QTYPE_QDICT: {
+        ToJsonIterState s;
+        QDict *val = qobject_to_qdict(obj);
+
+        s.count = 0;
+        s.str = str;
+        s.indent = indent + 1;
+        s.pretty = pretty;
+        qstring_append(str, "{");
+        qdict_iter(val, to_json_dict_iter, &s);
+        if (pretty) {
+            int j;
+            qstring_append(str, "\n");
+            for (j = 0 ; j < indent ; j++)
+                qstring_append(str, "    ");
+        }
+        qstring_append(str, "}");
+        break;
+    }
+    case QTYPE_QLIST: {
+        ToJsonIterState s;
+        QList *val = qobject_to_qlist(obj);
+
+        s.count = 0;
+        s.str = str;
+        s.indent = indent + 1;
+        s.pretty = pretty;
+        qstring_append(str, "[");
+        qlist_iter(val, (void *)to_json_list_iter, &s);
+        if (pretty) {
+            int j;
+            qstring_append(str, "\n");
+            for (j = 0 ; j < indent ; j++)
+                qstring_append(str, "    ");
+        }
+        qstring_append(str, "]");
+        break;
+    }
+    case QTYPE_QFLOAT: {
+        QFloat *val = qobject_to_qfloat(obj);
+        char buffer[1024];
+        int len;
+
+        len = snprintf(buffer, sizeof(buffer), "%f", qfloat_get_double(val));
+        while (len > 0 && buffer[len - 1] == '0') {
+            len--;
+        }
+
+        if (len && buffer[len - 1] == '.') {
+            buffer[len - 1] = 0;
+        } else {
+            buffer[len] = 0;
+        }
+        
+        qstring_append(str, buffer);
+        break;
+    }
+    case QTYPE_QBOOL: {
+        QBool *val = qobject_to_qbool(obj);
+
+        if (qbool_get_int(val)) {
+            qstring_append(str, "true");
+        } else {
+            qstring_append(str, "false");
+        }
+        break;
+    }
+    case QTYPE_QERROR:
+        /* XXX: should QError be emitted? */
+    case QTYPE_NONE:
+        break;
+    }
+}
+
+QString *qobject_to_json(const QObject *obj)
+{
+    QString *str = qstring_new();
+
+    to_json(obj, str, 0, 0);
+
+    return str;
+}
+
+QString *qobject_to_json_pretty(const QObject *obj)
+{
+    QString *str = qstring_new();
+
+    to_json(obj, str, 1, 0);
+
+    return str;
+}
diff --git a/contrib/qemu/qobject/qlist.c b/contrib/qemu/qobject/qlist.c
new file mode 100644
index 000000000..1ced0de58
--- /dev/null
+++ b/contrib/qemu/qobject/qlist.c
@@ -0,0 +1,170 @@
+/*
+ * QList Module
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu/queue.h"
+#include "qemu-common.h"
+
+static void qlist_destroy_obj(QObject *obj);
+
+static const QType qlist_type = {
+    .code = QTYPE_QLIST,
+    .destroy = qlist_destroy_obj,
+};
+ 
+/**
+ * qlist_new(): Create a new QList
+ *
+ * Return strong reference.
+ */
+QList *qlist_new(void)
+{
+    QList *qlist;
+
+    qlist = g_malloc(sizeof(*qlist));
+    QTAILQ_INIT(&qlist->head);
+    QOBJECT_INIT(qlist, &qlist_type);
+
+    return qlist;
+}
+
+static void qlist_copy_elem(QObject *obj, void *opaque)
+{
+    QList *dst = opaque;
+
+    qobject_incref(obj);
+    qlist_append_obj(dst, obj);
+}
+
+QList *qlist_copy(QList *src)
+{
+    QList *dst = qlist_new();
+
+    qlist_iter(src, qlist_copy_elem, dst);
+
+    return dst;
+}
+
+/**
+ * qlist_append_obj(): Append an QObject into QList
+ *
+ * NOTE: ownership of 'value' is transferred to the QList
+ */
+void qlist_append_obj(QList *qlist, QObject *value)
+{
+    QListEntry *entry;
+
+    entry = g_malloc(sizeof(*entry));
+    entry->value = value;
+
+    QTAILQ_INSERT_TAIL(&qlist->head, entry, next);
+}
+
+/**
+ * qlist_iter(): Iterate over all the list's stored values.
+ *
+ * This function allows the user to provide an iterator, which will be
+ * called for each stored value in the list.
+ */
+void qlist_iter(const QList *qlist,
+                void (*iter)(QObject *obj, void *opaque), void *opaque)
+{
+    QListEntry *entry;
+
+    QTAILQ_FOREACH(entry, &qlist->head, next)
+        iter(entry->value, opaque);
+}
+
+QObject *qlist_pop(QList *qlist)
+{
+    QListEntry *entry;
+    QObject *ret;
+
+    if (qlist == NULL || QTAILQ_EMPTY(&qlist->head)) {
+        return NULL;
+    }
+
+    entry = QTAILQ_FIRST(&qlist->head);
+    QTAILQ_REMOVE(&qlist->head, entry, next);
+
+    ret = entry->value;
+    g_free(entry);
+
+    return ret;
+}
+
+QObject *qlist_peek(QList *qlist)
+{
+    QListEntry *entry;
+    QObject *ret;
+
+    if (qlist == NULL || QTAILQ_EMPTY(&qlist->head)) {
+        return NULL;
+    }
+
+    entry = QTAILQ_FIRST(&qlist->head);
+
+    ret = entry->value;
+
+    return ret;
+}
+
+int qlist_empty(const QList *qlist)
+{
+    return QTAILQ_EMPTY(&qlist->head);
+}
+
+static void qlist_size_iter(QObject *obj, void *opaque)
+{
+    size_t *count = opaque;
+    (*count)++;
+}
+
+size_t qlist_size(const QList *qlist)
+{
+    size_t count = 0;
+    qlist_iter(qlist, qlist_size_iter, &count);
+    return count;
+}
+
+/**
+ * qobject_to_qlist(): Convert a QObject into a QList
+ */
+QList *qobject_to_qlist(const QObject *obj)
+{
+    if (qobject_type(obj) != QTYPE_QLIST) {
+        return NULL;
+    }
+
+    return container_of(obj, QList, base);
+}
+
+/**
+ * qlist_destroy_obj(): Free all the memory allocated by a QList
+ */
+static void qlist_destroy_obj(QObject *obj)
+{
+    QList *qlist;
+    QListEntry *entry, *next_entry;
+
+    assert(obj != NULL);
+    qlist = qobject_to_qlist(obj);
+
+    QTAILQ_FOREACH_SAFE(entry, &qlist->head, next, next_entry) {
+        QTAILQ_REMOVE(&qlist->head, entry, next);
+        qobject_decref(entry->value);
+        g_free(entry);
+    }
+
+    g_free(qlist);
+}
diff --git a/contrib/qemu/qobject/qstring.c b/contrib/qemu/qobject/qstring.c
new file mode 100644
index 000000000..607b7a142
--- /dev/null
+++ b/contrib/qemu/qobject/qstring.c
@@ -0,0 +1,149 @@
+/*
+ * QString Module
+ *
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * Authors:
+ *  Luiz Capitulino <lcapitulino@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qstring.h"
+#include "qemu-common.h"
+
+static void qstring_destroy_obj(QObject *obj);
+
+static const QType qstring_type = {
+    .code = QTYPE_QSTRING,
+    .destroy = qstring_destroy_obj,
+};
+
+/**
+ * qstring_new(): Create a new empty QString
+ *
+ * Return strong reference.
+ */
+QString *qstring_new(void)
+{
+    return qstring_from_str("");
+}
+
+/**
+ * qstring_get_length(): Get the length of a QString
+ */
+size_t qstring_get_length(const QString *qstring)
+{
+    return qstring->length;
+}
+
+/**
+ * qstring_from_substr(): Create a new QString from a C string substring
+ *
+ * Return string reference
+ */
+QString *qstring_from_substr(const char *str, int start, int end)
+{
+    QString *qstring;
+
+    qstring = g_malloc(sizeof(*qstring));
+
+    qstring->length = end - start + 1;
+    qstring->capacity = qstring->length;
+
+    qstring->string = g_malloc(qstring->capacity + 1);
+    memcpy(qstring->string, str + start, qstring->length);
+    qstring->string[qstring->length] = 0;
+
+    QOBJECT_INIT(qstring, &qstring_type);
+
+    return qstring;
+}
+
+/**
+ * qstring_from_str(): Create a new QString from a regular C string
+ *
+ * Return strong reference.
+ */
+QString *qstring_from_str(const char *str)
+{
+    return qstring_from_substr(str, 0, strlen(str) - 1);
+}
+
+static void capacity_increase(QString *qstring, size_t len)
+{
+    if (qstring->capacity < (qstring->length + len)) {
+        qstring->capacity += len;
+        qstring->capacity *= 2; /* use exponential growth */
+
+        qstring->string = g_realloc(qstring->string, qstring->capacity + 1);
+    }
+}
+
+/* qstring_append(): Append a C string to a QString
+ */
+void qstring_append(QString *qstring, const char *str)
+{
+    size_t len = strlen(str);
+
+    capacity_increase(qstring, len);
+    memcpy(qstring->string + qstring->length, str, len);
+    qstring->length += len;
+    qstring->string[qstring->length] = 0;
+}
+
+void qstring_append_int(QString *qstring, int64_t value)
+{
+    char num[32];
+
+    snprintf(num, sizeof(num), "%" PRId64, value);
+    qstring_append(qstring, num);
+}
+
+/**
+ * qstring_append_chr(): Append a C char to a QString
+ */
+void qstring_append_chr(QString *qstring, int c)
+{
+    capacity_increase(qstring, 1);
+    qstring->string[qstring->length++] = c;
+    qstring->string[qstring->length] = 0;
+}
+
+/**
+ * qobject_to_qstring(): Convert a QObject to a QString
+ */
+QString *qobject_to_qstring(const QObject *obj)
+{
+    if (qobject_type(obj) != QTYPE_QSTRING)
+        return NULL;
+
+    return container_of(obj, QString, base);
+}
+
+/**
+ * qstring_get_str(): Return a pointer to the stored string
+ *
+ * NOTE: Should be used with caution, if the object is deallocated
+ * this pointer becomes invalid.
+ */
+const char *qstring_get_str(const QString *qstring)
+{
+    return qstring->string;
+}
+
+/**
+ * qstring_destroy_obj(): Free all memory allocated by a QString
+ * object
+ */
+static void qstring_destroy_obj(QObject *obj)
+{
+    QString *qs;
+
+    assert(obj != NULL);
+    qs = qobject_to_qstring(obj);
+    g_free(qs->string);
+    g_free(qs);
+}
diff --git a/contrib/qemu/trace/generated-tracers.h b/contrib/qemu/trace/generated-tracers.h
new file mode 100644
index 000000000..b512660f3
--- /dev/null
+++ b/contrib/qemu/trace/generated-tracers.h
@@ -0,0 +1,3759 @@
+/* This file is autogenerated by tracetool, do not edit. */
+
+#ifndef TRACE__GENERATED_TRACERS_H
+#define TRACE__GENERATED_TRACERS_H
+
+#include "qemu-common.h"
+
+static inline void trace_qxl_interface_set_mm_time(int qid, uint32_t mm_time)
+{
+}
+
+static inline void trace_qxl_io_write_vga(int qid, const char * mode, uint32_t addr, uint32_t val)
+{
+}
+
+static inline void trace_g_malloc(size_t size, void * ptr)
+{
+}
+
+static inline void trace_g_realloc(void * ptr, size_t size, void * newptr)
+{
+}
+
+static inline void trace_g_free(void * ptr)
+{
+}
+
+static inline void trace_qemu_memalign(size_t alignment, size_t size, void * ptr)
+{
+}
+
+static inline void trace_qemu_anon_ram_alloc(size_t size, void * ptr)
+{
+}
+
+static inline void trace_qemu_vfree(void * ptr)
+{
+}
+
+static inline void trace_qemu_anon_ram_free(void * ptr, size_t size)
+{
+}
+
+static inline void trace_virtqueue_fill(void * vq, const void * elem, unsigned int len, unsigned int idx)
+{
+}
+
+static inline void trace_virtqueue_flush(void * vq, unsigned int count)
+{
+}
+
+static inline void trace_virtqueue_pop(void * vq, void * elem, unsigned int in_num, unsigned int out_num)
+{
+}
+
+static inline void trace_virtio_queue_notify(void * vdev, int n, void * vq)
+{
+}
+
+static inline void trace_virtio_irq(void * vq)
+{
+}
+
+static inline void trace_virtio_notify(void * vdev, void * vq)
+{
+}
+
+static inline void trace_virtio_set_status(void * vdev, uint8_t val)
+{
+}
+
+static inline void trace_virtio_serial_send_control_event(unsigned int port, uint16_t event, uint16_t value)
+{
+}
+
+static inline void trace_virtio_serial_throttle_port(unsigned int port, bool throttle)
+{
+}
+
+static inline void trace_virtio_serial_handle_control_message(uint16_t event, uint16_t value)
+{
+}
+
+static inline void trace_virtio_serial_handle_control_message_port(unsigned int port)
+{
+}
+
+static inline void trace_virtio_console_flush_buf(unsigned int port, size_t len, ssize_t ret)
+{
+}
+
+static inline void trace_virtio_console_chr_read(unsigned int port, int size)
+{
+}
+
+static inline void trace_virtio_console_chr_event(unsigned int port, int event)
+{
+}
+
+static inline void trace_bdrv_open_common(void * bs, const char * filename, int flags, const char * format_name)
+{
+}
+
+static inline void trace_multiwrite_cb(void * mcb, int ret)
+{
+}
+
+static inline void trace_bdrv_aio_multiwrite(void * mcb, int num_callbacks, int num_reqs)
+{
+}
+
+static inline void trace_bdrv_aio_discard(void * bs, int64_t sector_num, int nb_sectors, void * opaque)
+{
+}
+
+static inline void trace_bdrv_aio_flush(void * bs, void * opaque)
+{
+}
+
+static inline void trace_bdrv_aio_readv(void * bs, int64_t sector_num, int nb_sectors, void * opaque)
+{
+}
+
+static inline void trace_bdrv_aio_writev(void * bs, int64_t sector_num, int nb_sectors, void * opaque)
+{
+}
+
+static inline void trace_bdrv_lock_medium(void * bs, bool locked)
+{
+}
+
+static inline void trace_bdrv_co_readv(void * bs, int64_t sector_num, int nb_sector)
+{
+}
+
+static inline void trace_bdrv_co_copy_on_readv(void * bs, int64_t sector_num, int nb_sector)
+{
+}
+
+static inline void trace_bdrv_co_writev(void * bs, int64_t sector_num, int nb_sector)
+{
+}
+
+static inline void trace_bdrv_co_write_zeroes(void * bs, int64_t sector_num, int nb_sector)
+{
+}
+
+static inline void trace_bdrv_co_io_em(void * bs, int64_t sector_num, int nb_sectors, int is_write, void * acb)
+{
+}
+
+static inline void trace_bdrv_co_do_copy_on_readv(void * bs, int64_t sector_num, int nb_sectors, int64_t cluster_sector_num, int cluster_nb_sectors)
+{
+}
+
+static inline void trace_stream_one_iteration(void * s, int64_t sector_num, int nb_sectors, int is_allocated)
+{
+}
+
+static inline void trace_stream_start(void * bs, void * base, void * s, void * co, void * opaque)
+{
+}
+
+static inline void trace_commit_one_iteration(void * s, int64_t sector_num, int nb_sectors, int is_allocated)
+{
+}
+
+static inline void trace_commit_start(void * bs, void * base, void * top, void * s, void * co, void * opaque)
+{
+}
+
+static inline void trace_mirror_start(void * bs, void * s, void * co, void * opaque)
+{
+}
+
+static inline void trace_mirror_restart_iter(void * s, int64_t cnt)
+{
+}
+
+static inline void trace_mirror_before_flush(void * s)
+{
+}
+
+static inline void trace_mirror_before_drain(void * s, int64_t cnt)
+{
+}
+
+static inline void trace_mirror_before_sleep(void * s, int64_t cnt, int synced)
+{
+}
+
+static inline void trace_mirror_one_iteration(void * s, int64_t sector_num, int nb_sectors)
+{
+}
+
+static inline void trace_mirror_cow(void * s, int64_t sector_num)
+{
+}
+
+static inline void trace_mirror_iteration_done(void * s, int64_t sector_num, int nb_sectors, int ret)
+{
+}
+
+static inline void trace_mirror_yield(void * s, int64_t cnt, int buf_free_count, int in_flight)
+{
+}
+
+static inline void trace_mirror_yield_in_flight(void * s, int64_t sector_num, int in_flight)
+{
+}
+
+static inline void trace_mirror_yield_buf_busy(void * s, int nb_chunks, int in_flight)
+{
+}
+
+static inline void trace_mirror_break_buf_busy(void * s, int nb_chunks, int in_flight)
+{
+}
+
+static inline void trace_backup_do_cow_enter(void * job, int64_t start, int64_t sector_num, int nb_sectors)
+{
+}
+
+static inline void trace_backup_do_cow_return(void * job, int64_t sector_num, int nb_sectors, int ret)
+{
+}
+
+static inline void trace_backup_do_cow_skip(void * job, int64_t start)
+{
+}
+
+static inline void trace_backup_do_cow_process(void * job, int64_t start)
+{
+}
+
+static inline void trace_backup_do_cow_read_fail(void * job, int64_t start, int ret)
+{
+}
+
+static inline void trace_backup_do_cow_write_fail(void * job, int64_t start, int ret)
+{
+}
+
+static inline void trace_qmp_block_job_cancel(void * job)
+{
+}
+
+static inline void trace_qmp_block_job_pause(void * job)
+{
+}
+
+static inline void trace_qmp_block_job_resume(void * job)
+{
+}
+
+static inline void trace_qmp_block_job_complete(void * job)
+{
+}
+
+static inline void trace_block_job_cb(void * bs, void * job, int ret)
+{
+}
+
+static inline void trace_qmp_block_stream(void * bs, void * job)
+{
+}
+
+static inline void trace_virtio_blk_req_complete(void * req, int status)
+{
+}
+
+static inline void trace_virtio_blk_rw_complete(void * req, int ret)
+{
+}
+
+static inline void trace_virtio_blk_handle_write(void * req, uint64_t sector, size_t nsectors)
+{
+}
+
+static inline void trace_virtio_blk_handle_read(void * req, uint64_t sector, size_t nsectors)
+{
+}
+
+static inline void trace_virtio_blk_data_plane_start(void * s)
+{
+}
+
+static inline void trace_virtio_blk_data_plane_stop(void * s)
+{
+}
+
+static inline void trace_virtio_blk_data_plane_process_request(void * s, unsigned int out_num, unsigned int in_num, unsigned int head)
+{
+}
+
+static inline void trace_virtio_blk_data_plane_complete_request(void * s, unsigned int head, int ret)
+{
+}
+
+static inline void trace_vring_setup(uint64_t physical, void * desc, void * avail, void * used)
+{
+}
+
+static inline void trace_thread_pool_submit(void * pool, void * req, void * opaque)
+{
+}
+
+static inline void trace_thread_pool_complete(void * pool, void * req, void * opaque, int ret)
+{
+}
+
+static inline void trace_thread_pool_cancel(void * req, void * opaque)
+{
+}
+
+static inline void trace_paio_submit(void * acb, void * opaque, int64_t sector_num, int nb_sectors, int type)
+{
+}
+
+static inline void trace_paio_complete(void * acb, void * opaque, int ret)
+{
+}
+
+static inline void trace_paio_cancel(void * acb, void * opaque)
+{
+}
+
+static inline void trace_cpu_in(unsigned int addr, unsigned int val)
+{
+}
+
+static inline void trace_cpu_out(unsigned int addr, unsigned int val)
+{
+}
+
+static inline void trace_balloon_event(void * opaque, unsigned long addr)
+{
+}
+
+static inline void trace_apic_local_deliver(int vector, uint32_t lvt)
+{
+}
+
+static inline void trace_apic_deliver_irq(uint8_t dest, uint8_t dest_mode, uint8_t delivery_mode, uint8_t vector_num, uint8_t trigger_mode)
+{
+}
+
+static inline void trace_cpu_set_apic_base(uint64_t val)
+{
+}
+
+static inline void trace_cpu_get_apic_base(uint64_t val)
+{
+}
+
+static inline void trace_apic_mem_readl(uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_apic_mem_writel(uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_apic_report_irq_delivered(int apic_irq_delivered)
+{
+}
+
+static inline void trace_apic_reset_irq_delivered(int apic_irq_delivered)
+{
+}
+
+static inline void trace_apic_get_irq_delivered(int apic_irq_delivered)
+{
+}
+
+static inline void trace_cs4231_mem_readl_dreg(uint32_t reg, uint32_t ret)
+{
+}
+
+static inline void trace_cs4231_mem_readl_reg(uint32_t reg, uint32_t ret)
+{
+}
+
+static inline void trace_cs4231_mem_writel_reg(uint32_t reg, uint32_t old, uint32_t val)
+{
+}
+
+static inline void trace_cs4231_mem_writel_dreg(uint32_t reg, uint32_t old, uint32_t val)
+{
+}
+
+static inline void trace_nvram_read(uint32_t addr, uint32_t ret)
+{
+}
+
+static inline void trace_nvram_write(uint32_t addr, uint32_t old, uint32_t val)
+{
+}
+
+static inline void trace_ecc_mem_writel_mer(uint32_t val)
+{
+}
+
+static inline void trace_ecc_mem_writel_mdr(uint32_t val)
+{
+}
+
+static inline void trace_ecc_mem_writel_mfsr(uint32_t val)
+{
+}
+
+static inline void trace_ecc_mem_writel_vcr(uint32_t val)
+{
+}
+
+static inline void trace_ecc_mem_writel_dr(uint32_t val)
+{
+}
+
+static inline void trace_ecc_mem_writel_ecr0(uint32_t val)
+{
+}
+
+static inline void trace_ecc_mem_writel_ecr1(uint32_t val)
+{
+}
+
+static inline void trace_ecc_mem_readl_mer(uint32_t ret)
+{
+}
+
+static inline void trace_ecc_mem_readl_mdr(uint32_t ret)
+{
+}
+
+static inline void trace_ecc_mem_readl_mfsr(uint32_t ret)
+{
+}
+
+static inline void trace_ecc_mem_readl_vcr(uint32_t ret)
+{
+}
+
+static inline void trace_ecc_mem_readl_mfar0(uint32_t ret)
+{
+}
+
+static inline void trace_ecc_mem_readl_mfar1(uint32_t ret)
+{
+}
+
+static inline void trace_ecc_mem_readl_dr(uint32_t ret)
+{
+}
+
+static inline void trace_ecc_mem_readl_ecr0(uint32_t ret)
+{
+}
+
+static inline void trace_ecc_mem_readl_ecr1(uint32_t ret)
+{
+}
+
+static inline void trace_ecc_diag_mem_writeb(uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_ecc_diag_mem_readb(uint64_t addr, uint32_t ret)
+{
+}
+
+static inline void trace_fw_cfg_write(void * s, uint8_t value)
+{
+}
+
+static inline void trace_fw_cfg_select(void * s, uint16_t key, int ret)
+{
+}
+
+static inline void trace_fw_cfg_read(void * s, uint8_t ret)
+{
+}
+
+static inline void trace_fw_cfg_add_file_dupe(void * s, char * name)
+{
+}
+
+static inline void trace_fw_cfg_add_file(void * s, int index, char * name, size_t len)
+{
+}
+
+static inline void trace_hd_geometry_lchs_guess(void * bs, int cyls, int heads, int secs)
+{
+}
+
+static inline void trace_hd_geometry_guess(void * bs, uint32_t cyls, uint32_t heads, uint32_t secs, int trans)
+{
+}
+
+static inline void trace_jazz_led_read(uint64_t addr, uint8_t val)
+{
+}
+
+static inline void trace_jazz_led_write(uint64_t addr, uint8_t new)
+{
+}
+
+static inline void trace_lance_mem_readw(uint64_t addr, uint32_t ret)
+{
+}
+
+static inline void trace_lance_mem_writew(uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_slavio_intctl_mem_readl(uint32_t cpu, uint64_t addr, uint32_t ret)
+{
+}
+
+static inline void trace_slavio_intctl_mem_writel(uint32_t cpu, uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_slavio_intctl_mem_writel_clear(uint32_t cpu, uint32_t val, uint32_t intreg_pending)
+{
+}
+
+static inline void trace_slavio_intctl_mem_writel_set(uint32_t cpu, uint32_t val, uint32_t intreg_pending)
+{
+}
+
+static inline void trace_slavio_intctlm_mem_readl(uint64_t addr, uint32_t ret)
+{
+}
+
+static inline void trace_slavio_intctlm_mem_writel(uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_slavio_intctlm_mem_writel_enable(uint32_t val, uint32_t intregm_disabled)
+{
+}
+
+static inline void trace_slavio_intctlm_mem_writel_disable(uint32_t val, uint32_t intregm_disabled)
+{
+}
+
+static inline void trace_slavio_intctlm_mem_writel_target(uint32_t cpu)
+{
+}
+
+static inline void trace_slavio_check_interrupts(uint32_t pending, uint32_t intregm_disabled)
+{
+}
+
+static inline void trace_slavio_set_irq(uint32_t target_cpu, int irq, uint32_t pil, int level)
+{
+}
+
+static inline void trace_slavio_set_timer_irq_cpu(int cpu, int level)
+{
+}
+
+static inline void trace_slavio_misc_update_irq_raise(void)
+{
+}
+
+static inline void trace_slavio_misc_update_irq_lower(void)
+{
+}
+
+static inline void trace_slavio_set_power_fail(int power_failing, uint8_t config)
+{
+}
+
+static inline void trace_slavio_cfg_mem_writeb(uint32_t val)
+{
+}
+
+static inline void trace_slavio_cfg_mem_readb(uint32_t ret)
+{
+}
+
+static inline void trace_slavio_diag_mem_writeb(uint32_t val)
+{
+}
+
+static inline void trace_slavio_diag_mem_readb(uint32_t ret)
+{
+}
+
+static inline void trace_slavio_mdm_mem_writeb(uint32_t val)
+{
+}
+
+static inline void trace_slavio_mdm_mem_readb(uint32_t ret)
+{
+}
+
+static inline void trace_slavio_aux1_mem_writeb(uint32_t val)
+{
+}
+
+static inline void trace_slavio_aux1_mem_readb(uint32_t ret)
+{
+}
+
+static inline void trace_slavio_aux2_mem_writeb(uint32_t val)
+{
+}
+
+static inline void trace_slavio_aux2_mem_readb(uint32_t ret)
+{
+}
+
+static inline void trace_apc_mem_writeb(uint32_t val)
+{
+}
+
+static inline void trace_apc_mem_readb(uint32_t ret)
+{
+}
+
+static inline void trace_slavio_sysctrl_mem_writel(uint32_t val)
+{
+}
+
+static inline void trace_slavio_sysctrl_mem_readl(uint32_t ret)
+{
+}
+
+static inline void trace_slavio_led_mem_writew(uint32_t val)
+{
+}
+
+static inline void trace_slavio_led_mem_readw(uint32_t ret)
+{
+}
+
+static inline void trace_slavio_timer_get_out(uint64_t limit, uint32_t counthigh, uint32_t count)
+{
+}
+
+static inline void trace_slavio_timer_irq(uint32_t counthigh, uint32_t count)
+{
+}
+
+static inline void trace_slavio_timer_mem_readl_invalid(uint64_t addr)
+{
+}
+
+static inline void trace_slavio_timer_mem_readl(uint64_t addr, uint32_t ret)
+{
+}
+
+static inline void trace_slavio_timer_mem_writel(uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_slavio_timer_mem_writel_limit(unsigned int timer_index, uint64_t count)
+{
+}
+
+static inline void trace_slavio_timer_mem_writel_counter_invalid(void)
+{
+}
+
+static inline void trace_slavio_timer_mem_writel_status_start(unsigned int timer_index)
+{
+}
+
+static inline void trace_slavio_timer_mem_writel_status_stop(unsigned int timer_index)
+{
+}
+
+static inline void trace_slavio_timer_mem_writel_mode_user(unsigned int timer_index)
+{
+}
+
+static inline void trace_slavio_timer_mem_writel_mode_counter(unsigned int timer_index)
+{
+}
+
+static inline void trace_slavio_timer_mem_writel_mode_invalid(void)
+{
+}
+
+static inline void trace_slavio_timer_mem_writel_invalid(uint64_t addr)
+{
+}
+
+static inline void trace_ledma_memory_read(uint64_t addr)
+{
+}
+
+static inline void trace_ledma_memory_write(uint64_t addr)
+{
+}
+
+static inline void trace_sparc32_dma_set_irq_raise(void)
+{
+}
+
+static inline void trace_sparc32_dma_set_irq_lower(void)
+{
+}
+
+static inline void trace_espdma_memory_read(uint32_t addr)
+{
+}
+
+static inline void trace_espdma_memory_write(uint32_t addr)
+{
+}
+
+static inline void trace_sparc32_dma_mem_readl(uint64_t addr, uint32_t ret)
+{
+}
+
+static inline void trace_sparc32_dma_mem_writel(uint64_t addr, uint32_t old, uint32_t val)
+{
+}
+
+static inline void trace_sparc32_dma_enable_raise(void)
+{
+}
+
+static inline void trace_sparc32_dma_enable_lower(void)
+{
+}
+
+static inline void trace_sun4m_cpu_interrupt(unsigned int level)
+{
+}
+
+static inline void trace_sun4m_cpu_reset_interrupt(unsigned int level)
+{
+}
+
+static inline void trace_sun4m_cpu_set_irq_raise(int level)
+{
+}
+
+static inline void trace_sun4m_cpu_set_irq_lower(int level)
+{
+}
+
+static inline void trace_sun4m_iommu_mem_readl(uint64_t addr, uint32_t ret)
+{
+}
+
+static inline void trace_sun4m_iommu_mem_writel(uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_sun4m_iommu_mem_writel_ctrl(uint64_t iostart)
+{
+}
+
+static inline void trace_sun4m_iommu_mem_writel_tlbflush(uint32_t val)
+{
+}
+
+static inline void trace_sun4m_iommu_mem_writel_pgflush(uint32_t val)
+{
+}
+
+static inline void trace_sun4m_iommu_page_get_flags(uint64_t pa, uint64_t iopte, uint32_t ret)
+{
+}
+
+static inline void trace_sun4m_iommu_translate_pa(uint64_t addr, uint64_t pa, uint32_t iopte)
+{
+}
+
+static inline void trace_sun4m_iommu_bad_addr(uint64_t addr)
+{
+}
+
+static inline void trace_usb_packet_state_change(int bus, const char * port, int ep, void * p, const char * o, const char * n)
+{
+}
+
+static inline void trace_usb_packet_state_fault(int bus, const char * port, int ep, void * p, const char * o, const char * n)
+{
+}
+
+static inline void trace_usb_port_claim(int bus, const char * port)
+{
+}
+
+static inline void trace_usb_port_attach(int bus, const char * port, const char * devspeed, const char * portspeed)
+{
+}
+
+static inline void trace_usb_port_detach(int bus, const char * port)
+{
+}
+
+static inline void trace_usb_port_release(int bus, const char * port)
+{
+}
+
+static inline void trace_usb_ehci_reset(void)
+{
+}
+
+static inline void trace_usb_ehci_opreg_read(uint32_t addr, const char * str, uint32_t val)
+{
+}
+
+static inline void trace_usb_ehci_opreg_write(uint32_t addr, const char * str, uint32_t val)
+{
+}
+
+static inline void trace_usb_ehci_opreg_change(uint32_t addr, const char * str, uint32_t new, uint32_t old)
+{
+}
+
+static inline void trace_usb_ehci_portsc_read(uint32_t addr, uint32_t port, uint32_t val)
+{
+}
+
+static inline void trace_usb_ehci_portsc_write(uint32_t addr, uint32_t port, uint32_t val)
+{
+}
+
+static inline void trace_usb_ehci_portsc_change(uint32_t addr, uint32_t port, uint32_t new, uint32_t old)
+{
+}
+
+static inline void trace_usb_ehci_usbsts(const char * sts, int state)
+{
+}
+
+static inline void trace_usb_ehci_state(const char * schedule, const char * state)
+{
+}
+
+static inline void trace_usb_ehci_qh_ptrs(void * q, uint32_t addr, uint32_t nxt, uint32_t c_qtd, uint32_t n_qtd, uint32_t a_qtd)
+{
+}
+
+static inline void trace_usb_ehci_qh_fields(uint32_t addr, int rl, int mplen, int eps, int ep, int devaddr)
+{
+}
+
+static inline void trace_usb_ehci_qh_bits(uint32_t addr, int c, int h, int dtc, int i)
+{
+}
+
+static inline void trace_usb_ehci_qtd_ptrs(void * q, uint32_t addr, uint32_t nxt, uint32_t altnext)
+{
+}
+
+static inline void trace_usb_ehci_qtd_fields(uint32_t addr, int tbytes, int cpage, int cerr, int pid)
+{
+}
+
+static inline void trace_usb_ehci_qtd_bits(uint32_t addr, int ioc, int active, int halt, int babble, int xacterr)
+{
+}
+
+static inline void trace_usb_ehci_itd(uint32_t addr, uint32_t nxt, uint32_t mplen, uint32_t mult, uint32_t ep, uint32_t devaddr)
+{
+}
+
+static inline void trace_usb_ehci_sitd(uint32_t addr, uint32_t nxt, uint32_t active)
+{
+}
+
+static inline void trace_usb_ehci_port_attach(uint32_t port, const char * owner, const char * device)
+{
+}
+
+static inline void trace_usb_ehci_port_detach(uint32_t port, const char * owner)
+{
+}
+
+static inline void trace_usb_ehci_port_reset(uint32_t port, int enable)
+{
+}
+
+static inline void trace_usb_ehci_data(int rw, uint32_t cpage, uint32_t offset, uint32_t addr, uint32_t len, uint32_t bufpos)
+{
+}
+
+static inline void trace_usb_ehci_queue_action(void * q, const char * action)
+{
+}
+
+static inline void trace_usb_ehci_packet_action(void * q, void * p, const char * action)
+{
+}
+
+static inline void trace_usb_ehci_irq(uint32_t level, uint32_t frindex, uint32_t sts, uint32_t mask)
+{
+}
+
+static inline void trace_usb_ehci_guest_bug(const char * reason)
+{
+}
+
+static inline void trace_usb_ehci_doorbell_ring(void)
+{
+}
+
+static inline void trace_usb_ehci_doorbell_ack(void)
+{
+}
+
+static inline void trace_usb_ehci_dma_error(void)
+{
+}
+
+static inline void trace_usb_uhci_reset(void)
+{
+}
+
+static inline void trace_usb_uhci_schedule_start(void)
+{
+}
+
+static inline void trace_usb_uhci_schedule_stop(void)
+{
+}
+
+static inline void trace_usb_uhci_frame_start(uint32_t num)
+{
+}
+
+static inline void trace_usb_uhci_frame_stop_bandwidth(void)
+{
+}
+
+static inline void trace_usb_uhci_frame_loop_stop_idle(void)
+{
+}
+
+static inline void trace_usb_uhci_frame_loop_continue(void)
+{
+}
+
+static inline void trace_usb_uhci_mmio_readw(uint32_t addr, uint32_t val)
+{
+}
+
+static inline void trace_usb_uhci_mmio_writew(uint32_t addr, uint32_t val)
+{
+}
+
+static inline void trace_usb_uhci_queue_add(uint32_t token)
+{
+}
+
+static inline void trace_usb_uhci_queue_del(uint32_t token, const char * reason)
+{
+}
+
+static inline void trace_usb_uhci_packet_add(uint32_t token, uint32_t addr)
+{
+}
+
+static inline void trace_usb_uhci_packet_link_async(uint32_t token, uint32_t addr)
+{
+}
+
+static inline void trace_usb_uhci_packet_unlink_async(uint32_t token, uint32_t addr)
+{
+}
+
+static inline void trace_usb_uhci_packet_cancel(uint32_t token, uint32_t addr, int done)
+{
+}
+
+static inline void trace_usb_uhci_packet_complete_success(uint32_t token, uint32_t addr)
+{
+}
+
+static inline void trace_usb_uhci_packet_complete_shortxfer(uint32_t token, uint32_t addr)
+{
+}
+
+static inline void trace_usb_uhci_packet_complete_stall(uint32_t token, uint32_t addr)
+{
+}
+
+static inline void trace_usb_uhci_packet_complete_babble(uint32_t token, uint32_t addr)
+{
+}
+
+static inline void trace_usb_uhci_packet_complete_error(uint32_t token, uint32_t addr)
+{
+}
+
+static inline void trace_usb_uhci_packet_del(uint32_t token, uint32_t addr)
+{
+}
+
+static inline void trace_usb_uhci_qh_load(uint32_t qh)
+{
+}
+
+static inline void trace_usb_uhci_td_load(uint32_t qh, uint32_t td, uint32_t ctrl, uint32_t token)
+{
+}
+
+static inline void trace_usb_uhci_td_queue(uint32_t td, uint32_t ctrl, uint32_t token)
+{
+}
+
+static inline void trace_usb_uhci_td_nextqh(uint32_t qh, uint32_t td)
+{
+}
+
+static inline void trace_usb_uhci_td_async(uint32_t qh, uint32_t td)
+{
+}
+
+static inline void trace_usb_uhci_td_complete(uint32_t qh, uint32_t td)
+{
+}
+
+static inline void trace_usb_xhci_reset(void)
+{
+}
+
+static inline void trace_usb_xhci_run(void)
+{
+}
+
+static inline void trace_usb_xhci_stop(void)
+{
+}
+
+static inline void trace_usb_xhci_cap_read(uint32_t off, uint32_t val)
+{
+}
+
+static inline void trace_usb_xhci_oper_read(uint32_t off, uint32_t val)
+{
+}
+
+static inline void trace_usb_xhci_port_read(uint32_t port, uint32_t off, uint32_t val)
+{
+}
+
+static inline void trace_usb_xhci_runtime_read(uint32_t off, uint32_t val)
+{
+}
+
+static inline void trace_usb_xhci_doorbell_read(uint32_t off, uint32_t val)
+{
+}
+
+static inline void trace_usb_xhci_oper_write(uint32_t off, uint32_t val)
+{
+}
+
+static inline void trace_usb_xhci_port_write(uint32_t port, uint32_t off, uint32_t val)
+{
+}
+
+static inline void trace_usb_xhci_runtime_write(uint32_t off, uint32_t val)
+{
+}
+
+static inline void trace_usb_xhci_doorbell_write(uint32_t off, uint32_t val)
+{
+}
+
+static inline void trace_usb_xhci_irq_intx(uint32_t level)
+{
+}
+
+static inline void trace_usb_xhci_irq_msi(uint32_t nr)
+{
+}
+
+static inline void trace_usb_xhci_irq_msix(uint32_t nr)
+{
+}
+
+static inline void trace_usb_xhci_irq_msix_use(uint32_t nr)
+{
+}
+
+static inline void trace_usb_xhci_irq_msix_unuse(uint32_t nr)
+{
+}
+
+static inline void trace_usb_xhci_queue_event(uint32_t vector, uint32_t idx, const char * trb, const char * evt, uint64_t param, uint32_t status, uint32_t control)
+{
+}
+
+static inline void trace_usb_xhci_fetch_trb(uint64_t addr, const char * name, uint64_t param, uint32_t status, uint32_t control)
+{
+}
+
+static inline void trace_usb_xhci_port_reset(uint32_t port)
+{
+}
+
+static inline void trace_usb_xhci_port_link(uint32_t port, uint32_t pls)
+{
+}
+
+static inline void trace_usb_xhci_port_notify(uint32_t port, uint32_t pls)
+{
+}
+
+static inline void trace_usb_xhci_slot_enable(uint32_t slotid)
+{
+}
+
+static inline void trace_usb_xhci_slot_disable(uint32_t slotid)
+{
+}
+
+static inline void trace_usb_xhci_slot_address(uint32_t slotid)
+{
+}
+
+static inline void trace_usb_xhci_slot_configure(uint32_t slotid)
+{
+}
+
+static inline void trace_usb_xhci_slot_evaluate(uint32_t slotid)
+{
+}
+
+static inline void trace_usb_xhci_slot_reset(uint32_t slotid)
+{
+}
+
+static inline void trace_usb_xhci_ep_enable(uint32_t slotid, uint32_t epid)
+{
+}
+
+static inline void trace_usb_xhci_ep_disable(uint32_t slotid, uint32_t epid)
+{
+}
+
+static inline void trace_usb_xhci_ep_set_dequeue(uint32_t slotid, uint32_t epid, uint32_t streamid, uint64_t param)
+{
+}
+
+static inline void trace_usb_xhci_ep_kick(uint32_t slotid, uint32_t epid, uint32_t streamid)
+{
+}
+
+static inline void trace_usb_xhci_ep_stop(uint32_t slotid, uint32_t epid)
+{
+}
+
+static inline void trace_usb_xhci_ep_reset(uint32_t slotid, uint32_t epid)
+{
+}
+
+static inline void trace_usb_xhci_xfer_start(void * xfer, uint32_t slotid, uint32_t epid, uint32_t streamid)
+{
+}
+
+static inline void trace_usb_xhci_xfer_async(void * xfer)
+{
+}
+
+static inline void trace_usb_xhci_xfer_nak(void * xfer)
+{
+}
+
+static inline void trace_usb_xhci_xfer_retry(void * xfer)
+{
+}
+
+static inline void trace_usb_xhci_xfer_success(void * xfer, uint32_t bytes)
+{
+}
+
+static inline void trace_usb_xhci_xfer_error(void * xfer, uint32_t ret)
+{
+}
+
+static inline void trace_usb_xhci_unimplemented(const char * item, int nr)
+{
+}
+
+static inline void trace_usb_desc_device(int addr, int len, int ret)
+{
+}
+
+static inline void trace_usb_desc_device_qualifier(int addr, int len, int ret)
+{
+}
+
+static inline void trace_usb_desc_config(int addr, int index, int len, int ret)
+{
+}
+
+static inline void trace_usb_desc_other_speed_config(int addr, int index, int len, int ret)
+{
+}
+
+static inline void trace_usb_desc_string(int addr, int index, int len, int ret)
+{
+}
+
+static inline void trace_usb_desc_bos(int addr, int len, int ret)
+{
+}
+
+static inline void trace_usb_set_addr(int addr)
+{
+}
+
+static inline void trace_usb_set_config(int addr, int config, int ret)
+{
+}
+
+static inline void trace_usb_set_interface(int addr, int iface, int alt, int ret)
+{
+}
+
+static inline void trace_usb_clear_device_feature(int addr, int feature, int ret)
+{
+}
+
+static inline void trace_usb_set_device_feature(int addr, int feature, int ret)
+{
+}
+
+static inline void trace_usb_hub_reset(int addr)
+{
+}
+
+static inline void trace_usb_hub_control(int addr, int request, int value, int index, int length)
+{
+}
+
+static inline void trace_usb_hub_get_port_status(int addr, int nr, int status, int changed)
+{
+}
+
+static inline void trace_usb_hub_set_port_feature(int addr, int nr, const char * f)
+{
+}
+
+static inline void trace_usb_hub_clear_port_feature(int addr, int nr, const char * f)
+{
+}
+
+static inline void trace_usb_hub_attach(int addr, int nr)
+{
+}
+
+static inline void trace_usb_hub_detach(int addr, int nr)
+{
+}
+
+static inline void trace_usb_uas_reset(int addr)
+{
+}
+
+static inline void trace_usb_uas_command(int addr, uint16_t tag, int lun, uint32_t lun64_1, uint32_t lun64_2)
+{
+}
+
+static inline void trace_usb_uas_response(int addr, uint16_t tag, uint8_t code)
+{
+}
+
+static inline void trace_usb_uas_sense(int addr, uint16_t tag, uint8_t status)
+{
+}
+
+static inline void trace_usb_uas_read_ready(int addr, uint16_t tag)
+{
+}
+
+static inline void trace_usb_uas_write_ready(int addr, uint16_t tag)
+{
+}
+
+static inline void trace_usb_uas_xfer_data(int addr, uint16_t tag, uint32_t copy, uint32_t uoff, uint32_t usize, uint32_t soff, uint32_t ssize)
+{
+}
+
+static inline void trace_usb_uas_scsi_data(int addr, uint16_t tag, uint32_t bytes)
+{
+}
+
+static inline void trace_usb_uas_scsi_complete(int addr, uint16_t tag, uint32_t status, uint32_t resid)
+{
+}
+
+static inline void trace_usb_uas_tmf_abort_task(int addr, uint16_t tag, uint16_t task_tag)
+{
+}
+
+static inline void trace_usb_uas_tmf_logical_unit_reset(int addr, uint16_t tag, int lun)
+{
+}
+
+static inline void trace_usb_uas_tmf_unsupported(int addr, uint16_t tag, uint32_t function)
+{
+}
+
+static inline void trace_usb_host_open_started(int bus, int addr)
+{
+}
+
+static inline void trace_usb_host_open_success(int bus, int addr)
+{
+}
+
+static inline void trace_usb_host_open_failure(int bus, int addr)
+{
+}
+
+static inline void trace_usb_host_disconnect(int bus, int addr)
+{
+}
+
+static inline void trace_usb_host_close(int bus, int addr)
+{
+}
+
+static inline void trace_usb_host_attach_kernel(int bus, int addr, int interface)
+{
+}
+
+static inline void trace_usb_host_detach_kernel(int bus, int addr, int interface)
+{
+}
+
+static inline void trace_usb_host_set_address(int bus, int addr, int config)
+{
+}
+
+static inline void trace_usb_host_set_config(int bus, int addr, int config)
+{
+}
+
+static inline void trace_usb_host_set_interface(int bus, int addr, int interface, int alt)
+{
+}
+
+static inline void trace_usb_host_claim_interfaces(int bus, int addr, int config, int nif)
+{
+}
+
+static inline void trace_usb_host_claim_interface(int bus, int addr, int config, int interface)
+{
+}
+
+static inline void trace_usb_host_release_interfaces(int bus, int addr)
+{
+}
+
+static inline void trace_usb_host_release_interface(int bus, int addr, int interface)
+{
+}
+
+static inline void trace_usb_host_req_control(int bus, int addr, void * p, int req, int value, int index)
+{
+}
+
+static inline void trace_usb_host_req_data(int bus, int addr, void * p, int in, int ep, int size)
+{
+}
+
+static inline void trace_usb_host_req_complete(int bus, int addr, void * p, int status, int length)
+{
+}
+
+static inline void trace_usb_host_req_emulated(int bus, int addr, void * p, int status)
+{
+}
+
+static inline void trace_usb_host_req_canceled(int bus, int addr, void * p)
+{
+}
+
+static inline void trace_usb_host_urb_submit(int bus, int addr, void * aurb, int length, int more)
+{
+}
+
+static inline void trace_usb_host_urb_complete(int bus, int addr, void * aurb, int status, int length, int more)
+{
+}
+
+static inline void trace_usb_host_urb_canceled(int bus, int addr, void * aurb)
+{
+}
+
+static inline void trace_usb_host_ep_set_halt(int bus, int addr, int ep)
+{
+}
+
+static inline void trace_usb_host_ep_clear_halt(int bus, int addr, int ep)
+{
+}
+
+static inline void trace_usb_host_iso_start(int bus, int addr, int ep)
+{
+}
+
+static inline void trace_usb_host_iso_stop(int bus, int addr, int ep)
+{
+}
+
+static inline void trace_usb_host_iso_out_of_bufs(int bus, int addr, int ep)
+{
+}
+
+static inline void trace_usb_host_iso_many_urbs(int bus, int addr, int count)
+{
+}
+
+static inline void trace_usb_host_reset(int bus, int addr)
+{
+}
+
+static inline void trace_usb_host_auto_scan_enabled(void)
+{
+}
+
+static inline void trace_usb_host_auto_scan_disabled(void)
+{
+}
+
+static inline void trace_usb_host_claim_port(int bus, int hub, int port)
+{
+}
+
+static inline void trace_usb_host_parse_device(int bus, int addr, int vendor, int product)
+{
+}
+
+static inline void trace_usb_host_parse_config(int bus, int addr, int value, int active)
+{
+}
+
+static inline void trace_usb_host_parse_interface(int bus, int addr, int num, int alt, int active)
+{
+}
+
+static inline void trace_usb_host_parse_endpoint(int bus, int addr, int ep, const char * dir, const char * type, int active)
+{
+}
+
+static inline void trace_usb_host_parse_unknown(int bus, int addr, int len, int type)
+{
+}
+
+static inline void trace_usb_host_parse_error(int bus, int addr, const char * errmsg)
+{
+}
+
+static inline void trace_scsi_req_alloc(int target, int lun, int tag)
+{
+}
+
+static inline void trace_scsi_req_cancel(int target, int lun, int tag)
+{
+}
+
+static inline void trace_scsi_req_data(int target, int lun, int tag, int len)
+{
+}
+
+static inline void trace_scsi_req_data_canceled(int target, int lun, int tag, int len)
+{
+}
+
+static inline void trace_scsi_req_dequeue(int target, int lun, int tag)
+{
+}
+
+static inline void trace_scsi_req_continue(int target, int lun, int tag)
+{
+}
+
+static inline void trace_scsi_req_continue_canceled(int target, int lun, int tag)
+{
+}
+
+static inline void trace_scsi_req_parsed(int target, int lun, int tag, int cmd, int mode, int xfer)
+{
+}
+
+static inline void trace_scsi_req_parsed_lba(int target, int lun, int tag, int cmd, uint64_t lba)
+{
+}
+
+static inline void trace_scsi_req_parse_bad(int target, int lun, int tag, int cmd)
+{
+}
+
+static inline void trace_scsi_req_build_sense(int target, int lun, int tag, int key, int asc, int ascq)
+{
+}
+
+static inline void trace_scsi_device_set_ua(int target, int lun, int key, int asc, int ascq)
+{
+}
+
+static inline void trace_scsi_report_luns(int target, int lun, int tag)
+{
+}
+
+static inline void trace_scsi_inquiry(int target, int lun, int tag, int cdb1, int cdb2)
+{
+}
+
+static inline void trace_scsi_test_unit_ready(int target, int lun, int tag)
+{
+}
+
+static inline void trace_scsi_request_sense(int target, int lun, int tag)
+{
+}
+
+static inline void trace_vm_state_notify(int running, int reason)
+{
+}
+
+static inline void trace_load_file(const char * name, const char * path)
+{
+}
+
+static inline void trace_runstate_set(int new_state)
+{
+}
+
+static inline void trace_qcow2_writev_start_req(void * co, int64_t sector, int nb_sectors)
+{
+}
+
+static inline void trace_qcow2_writev_done_req(void * co, int ret)
+{
+}
+
+static inline void trace_qcow2_writev_start_part(void * co)
+{
+}
+
+static inline void trace_qcow2_writev_done_part(void * co, int cur_nr_sectors)
+{
+}
+
+static inline void trace_qcow2_writev_data(void * co, uint64_t offset)
+{
+}
+
+static inline void trace_qcow2_alloc_clusters_offset(void * co, uint64_t offset, int n_start, int n_end)
+{
+}
+
+static inline void trace_qcow2_handle_copied(void * co, uint64_t guest_offset, uint64_t host_offset, uint64_t bytes)
+{
+}
+
+static inline void trace_qcow2_handle_alloc(void * co, uint64_t guest_offset, uint64_t host_offset, uint64_t bytes)
+{
+}
+
+static inline void trace_qcow2_do_alloc_clusters_offset(void * co, uint64_t guest_offset, uint64_t host_offset, int nb_clusters)
+{
+}
+
+static inline void trace_qcow2_cluster_alloc_phys(void * co)
+{
+}
+
+static inline void trace_qcow2_cluster_link_l2(void * co, int nb_clusters)
+{
+}
+
+static inline void trace_qcow2_l2_allocate(void * bs, int l1_index)
+{
+}
+
+static inline void trace_qcow2_l2_allocate_get_empty(void * bs, int l1_index)
+{
+}
+
+static inline void trace_qcow2_l2_allocate_write_l2(void * bs, int l1_index)
+{
+}
+
+static inline void trace_qcow2_l2_allocate_write_l1(void * bs, int l1_index)
+{
+}
+
+static inline void trace_qcow2_l2_allocate_done(void * bs, int l1_index, int ret)
+{
+}
+
+static inline void trace_qcow2_cache_get(void * co, int c, uint64_t offset, bool read_from_disk)
+{
+}
+
+static inline void trace_qcow2_cache_get_replace_entry(void * co, int c, int i)
+{
+}
+
+static inline void trace_qcow2_cache_get_read(void * co, int c, int i)
+{
+}
+
+static inline void trace_qcow2_cache_get_done(void * co, int c, int i)
+{
+}
+
+static inline void trace_qcow2_cache_flush(void * co, int c)
+{
+}
+
+static inline void trace_qcow2_cache_entry_flush(void * co, int c, int i)
+{
+}
+
+static inline void trace_qed_alloc_l2_cache_entry(void * l2_cache, void * entry)
+{
+}
+
+static inline void trace_qed_unref_l2_cache_entry(void * entry, int ref)
+{
+}
+
+static inline void trace_qed_find_l2_cache_entry(void * l2_cache, void * entry, uint64_t offset, int ref)
+{
+}
+
+static inline void trace_qed_read_table(void * s, uint64_t offset, void * table)
+{
+}
+
+static inline void trace_qed_read_table_cb(void * s, void * table, int ret)
+{
+}
+
+static inline void trace_qed_write_table(void * s, uint64_t offset, void * table, unsigned int index, unsigned int n)
+{
+}
+
+static inline void trace_qed_write_table_cb(void * s, void * table, int flush, int ret)
+{
+}
+
+static inline void trace_qed_need_check_timer_cb(void * s)
+{
+}
+
+static inline void trace_qed_start_need_check_timer(void * s)
+{
+}
+
+static inline void trace_qed_cancel_need_check_timer(void * s)
+{
+}
+
+static inline void trace_qed_aio_complete(void * s, void * acb, int ret)
+{
+}
+
+static inline void trace_qed_aio_setup(void * s, void * acb, int64_t sector_num, int nb_sectors, void * opaque, int flags)
+{
+}
+
+static inline void trace_qed_aio_next_io(void * s, void * acb, int ret, uint64_t cur_pos)
+{
+}
+
+static inline void trace_qed_aio_read_data(void * s, void * acb, int ret, uint64_t offset, size_t len)
+{
+}
+
+static inline void trace_qed_aio_write_data(void * s, void * acb, int ret, uint64_t offset, size_t len)
+{
+}
+
+static inline void trace_qed_aio_write_prefill(void * s, void * acb, uint64_t start, size_t len, uint64_t offset)
+{
+}
+
+static inline void trace_qed_aio_write_postfill(void * s, void * acb, uint64_t start, size_t len, uint64_t offset)
+{
+}
+
+static inline void trace_qed_aio_write_main(void * s, void * acb, int ret, uint64_t offset, size_t len)
+{
+}
+
+static inline void trace_g364fb_read(uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_g364fb_write(uint64_t addr, uint32_t new)
+{
+}
+
+static inline void trace_grlib_gptimer_enable(int id, uint32_t count)
+{
+}
+
+static inline void trace_grlib_gptimer_disabled(int id, uint32_t config)
+{
+}
+
+static inline void trace_grlib_gptimer_restart(int id, uint32_t reload)
+{
+}
+
+static inline void trace_grlib_gptimer_set_scaler(uint32_t scaler, uint32_t freq)
+{
+}
+
+static inline void trace_grlib_gptimer_hit(int id)
+{
+}
+
+static inline void trace_grlib_gptimer_readl(int id, uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_grlib_gptimer_writel(int id, uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_grlib_irqmp_check_irqs(uint32_t pend, uint32_t force, uint32_t mask, uint32_t lvl1, uint32_t lvl2)
+{
+}
+
+static inline void trace_grlib_irqmp_ack(int intno)
+{
+}
+
+static inline void trace_grlib_irqmp_set_irq(int irq)
+{
+}
+
+static inline void trace_grlib_irqmp_readl_unknown(uint64_t addr)
+{
+}
+
+static inline void trace_grlib_irqmp_writel_unknown(uint64_t addr, uint32_t value)
+{
+}
+
+static inline void trace_grlib_apbuart_event(int event)
+{
+}
+
+static inline void trace_grlib_apbuart_writel_unknown(uint64_t addr, uint32_t value)
+{
+}
+
+static inline void trace_grlib_apbuart_readl_unknown(uint64_t addr)
+{
+}
+
+static inline void trace_leon3_set_irq(int intno)
+{
+}
+
+static inline void trace_leon3_reset_irq(int intno)
+{
+}
+
+static inline void trace_spice_vmc_write(ssize_t out, int len)
+{
+}
+
+static inline void trace_spice_vmc_read(int bytes, int len)
+{
+}
+
+static inline void trace_spice_vmc_register_interface(void * scd)
+{
+}
+
+static inline void trace_spice_vmc_unregister_interface(void * scd)
+{
+}
+
+static inline void trace_spice_vmc_event(int event)
+{
+}
+
+static inline void trace_lm32_pic_raise_irq(void)
+{
+}
+
+static inline void trace_lm32_pic_lower_irq(void)
+{
+}
+
+static inline void trace_lm32_pic_interrupt(int irq, int level)
+{
+}
+
+static inline void trace_lm32_pic_set_im(uint32_t im)
+{
+}
+
+static inline void trace_lm32_pic_set_ip(uint32_t ip)
+{
+}
+
+static inline void trace_lm32_pic_get_im(uint32_t im)
+{
+}
+
+static inline void trace_lm32_pic_get_ip(uint32_t ip)
+{
+}
+
+static inline void trace_lm32_juart_get_jtx(uint32_t value)
+{
+}
+
+static inline void trace_lm32_juart_set_jtx(uint32_t value)
+{
+}
+
+static inline void trace_lm32_juart_get_jrx(uint32_t value)
+{
+}
+
+static inline void trace_lm32_juart_set_jrx(uint32_t value)
+{
+}
+
+static inline void trace_lm32_timer_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_lm32_timer_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_lm32_timer_hit(void)
+{
+}
+
+static inline void trace_lm32_timer_irq_state(int level)
+{
+}
+
+static inline void trace_lm32_uart_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_lm32_uart_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_lm32_uart_irq_state(int level)
+{
+}
+
+static inline void trace_lm32_sys_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_megasas_init_firmware(uint64_t pa)
+{
+}
+
+static inline void trace_megasas_init_queue(uint64_t queue_pa, int queue_len, uint64_t head, uint64_t tail, uint32_t flags)
+{
+}
+
+static inline void trace_megasas_initq_map_failed(int frame)
+{
+}
+
+static inline void trace_megasas_initq_mismatch(int queue_len, int fw_cmds)
+{
+}
+
+static inline void trace_megasas_qf_found(unsigned int index, uint64_t pa)
+{
+}
+
+static inline void trace_megasas_qf_new(unsigned int index, void * cmd)
+{
+}
+
+static inline void trace_megasas_qf_failed(unsigned long pa)
+{
+}
+
+static inline void trace_megasas_qf_enqueue(unsigned int index, unsigned int count, uint64_t context, unsigned int tail, int busy)
+{
+}
+
+static inline void trace_megasas_qf_update(unsigned int head, unsigned int busy)
+{
+}
+
+static inline void trace_megasas_qf_dequeue(unsigned int index)
+{
+}
+
+static inline void trace_megasas_qf_map_failed(int cmd, unsigned long frame)
+{
+}
+
+static inline void trace_megasas_qf_complete_noirq(uint64_t context)
+{
+}
+
+static inline void trace_megasas_qf_complete(uint64_t context, unsigned int tail, unsigned int offset, int busy, unsigned int doorbell)
+{
+}
+
+static inline void trace_megasas_handle_frame(const char * cmd, uint64_t addr, uint64_t context, uint32_t count)
+{
+}
+
+static inline void trace_megasas_frame_busy(uint64_t addr)
+{
+}
+
+static inline void trace_megasas_unhandled_frame_cmd(int cmd, uint8_t frame_cmd)
+{
+}
+
+static inline void trace_megasas_handle_scsi(const char * frame, int bus, int dev, int lun, void * sdev, unsigned long size)
+{
+}
+
+static inline void trace_megasas_scsi_target_not_present(const char * frame, int bus, int dev, int lun)
+{
+}
+
+static inline void trace_megasas_scsi_invalid_cdb_len(const char * frame, int bus, int dev, int lun, int len)
+{
+}
+
+static inline void trace_megasas_iov_read_overflow(int cmd, int bytes, int len)
+{
+}
+
+static inline void trace_megasas_iov_write_overflow(int cmd, int bytes, int len)
+{
+}
+
+static inline void trace_megasas_iov_read_underflow(int cmd, int bytes, int len)
+{
+}
+
+static inline void trace_megasas_iov_write_underflow(int cmd, int bytes, int len)
+{
+}
+
+static inline void trace_megasas_scsi_req_alloc_failed(const char * frame, int dev, int lun)
+{
+}
+
+static inline void trace_megasas_scsi_read_start(int cmd, int len)
+{
+}
+
+static inline void trace_megasas_scsi_write_start(int cmd, int len)
+{
+}
+
+static inline void trace_megasas_scsi_nodata(int cmd)
+{
+}
+
+static inline void trace_megasas_scsi_complete(int cmd, uint32_t status, int len, int xfer)
+{
+}
+
+static inline void trace_megasas_command_complete(int cmd, uint32_t status, uint32_t resid)
+{
+}
+
+static inline void trace_megasas_handle_io(int cmd, const char * frame, int dev, int lun, unsigned long lba, unsigned long count)
+{
+}
+
+static inline void trace_megasas_io_target_not_present(int cmd, const char * frame, int dev, int lun)
+{
+}
+
+static inline void trace_megasas_io_read_start(int cmd, unsigned long lba, unsigned long count, unsigned long len)
+{
+}
+
+static inline void trace_megasas_io_write_start(int cmd, unsigned long lba, unsigned long count, unsigned long len)
+{
+}
+
+static inline void trace_megasas_io_complete(int cmd, uint32_t len)
+{
+}
+
+static inline void trace_megasas_io_read(int cmd, int bytes, int len, unsigned long offset)
+{
+}
+
+static inline void trace_megasas_io_write(int cmd, int bytes, int len, unsigned long offset)
+{
+}
+
+static inline void trace_megasas_io_continue(int cmd, int bytes)
+{
+}
+
+static inline void trace_megasas_iovec_map_failed(int cmd, int index, unsigned long iov_size)
+{
+}
+
+static inline void trace_megasas_iovec_sgl_overflow(int cmd, int index, int limit)
+{
+}
+
+static inline void trace_megasas_iovec_sgl_underflow(int cmd, int index)
+{
+}
+
+static inline void trace_megasas_iovec_sgl_invalid(int cmd, int index, uint64_t pa, uint32_t len)
+{
+}
+
+static inline void trace_megasas_iovec_overflow(int cmd, int len, int limit)
+{
+}
+
+static inline void trace_megasas_iovec_underflow(int cmd, int len, int limit)
+{
+}
+
+static inline void trace_megasas_handle_dcmd(int cmd, int opcode)
+{
+}
+
+static inline void trace_megasas_finish_dcmd(int cmd, int size)
+{
+}
+
+static inline void trace_megasas_dcmd_req_alloc_failed(int cmd, const char * desc)
+{
+}
+
+static inline void trace_megasas_dcmd_internal_submit(int cmd, const char * desc, int dev)
+{
+}
+
+static inline void trace_megasas_dcmd_internal_finish(int cmd, int opcode, int lun)
+{
+}
+
+static inline void trace_megasas_dcmd_internal_invalid(int cmd, int opcode)
+{
+}
+
+static inline void trace_megasas_dcmd_unhandled(int cmd, int opcode, int len)
+{
+}
+
+static inline void trace_megasas_dcmd_zero_sge(int cmd)
+{
+}
+
+static inline void trace_megasas_dcmd_invalid_sge(int cmd, int count)
+{
+}
+
+static inline void trace_megasas_dcmd_map_failed(int cmd)
+{
+}
+
+static inline void trace_megasas_dcmd_invalid_xfer_len(int cmd, unsigned long size, unsigned long max)
+{
+}
+
+static inline void trace_megasas_dcmd_enter(int cmd, const char * dcmd, int len)
+{
+}
+
+static inline void trace_megasas_dcmd_dummy(int cmd, unsigned long size)
+{
+}
+
+static inline void trace_megasas_dcmd_set_fw_time(int cmd, unsigned long time)
+{
+}
+
+static inline void trace_megasas_dcmd_pd_get_list(int cmd, int num, int max, int offset)
+{
+}
+
+static inline void trace_megasas_dcmd_ld_get_list(int cmd, int num, int max)
+{
+}
+
+static inline void trace_megasas_dcmd_ld_get_info(int cmd, int ld_id)
+{
+}
+
+static inline void trace_megasas_dcmd_pd_get_info(int cmd, int pd_id)
+{
+}
+
+static inline void trace_megasas_dcmd_pd_list_query(int cmd, int flags)
+{
+}
+
+static inline void trace_megasas_dcmd_unsupported(int cmd, unsigned long size)
+{
+}
+
+static inline void trace_megasas_abort_frame(int cmd, int abort_cmd)
+{
+}
+
+static inline void trace_megasas_abort_no_cmd(int cmd, uint64_t context)
+{
+}
+
+static inline void trace_megasas_abort_invalid_context(int cmd, uint64_t context, int abort_cmd)
+{
+}
+
+static inline void trace_megasas_reset(void)
+{
+}
+
+static inline void trace_megasas_init(int sges, int cmds, const char * intr, const char * mode)
+{
+}
+
+static inline void trace_megasas_msix_raise(int vector)
+{
+}
+
+static inline void trace_megasas_irq_lower(void)
+{
+}
+
+static inline void trace_megasas_irq_raise(void)
+{
+}
+
+static inline void trace_megasas_intr_enabled(void)
+{
+}
+
+static inline void trace_megasas_intr_disabled(void)
+{
+}
+
+static inline void trace_megasas_mmio_readl(unsigned long addr, uint32_t val)
+{
+}
+
+static inline void trace_megasas_mmio_invalid_readl(unsigned long addr)
+{
+}
+
+static inline void trace_megasas_mmio_writel(uint32_t addr, uint32_t val)
+{
+}
+
+static inline void trace_megasas_mmio_invalid_writel(uint32_t addr, uint32_t val)
+{
+}
+
+static inline void trace_milkymist_ac97_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_ac97_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_ac97_pulse_irq_crrequest(void)
+{
+}
+
+static inline void trace_milkymist_ac97_pulse_irq_crreply(void)
+{
+}
+
+static inline void trace_milkymist_ac97_pulse_irq_dmaw(void)
+{
+}
+
+static inline void trace_milkymist_ac97_pulse_irq_dmar(void)
+{
+}
+
+static inline void trace_milkymist_ac97_in_cb(int avail, uint32_t remaining)
+{
+}
+
+static inline void trace_milkymist_ac97_in_cb_transferred(int transferred)
+{
+}
+
+static inline void trace_milkymist_ac97_out_cb(int free, uint32_t remaining)
+{
+}
+
+static inline void trace_milkymist_ac97_out_cb_transferred(int transferred)
+{
+}
+
+static inline void trace_milkymist_hpdmc_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_hpdmc_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_memcard_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_memcard_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_minimac2_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_minimac2_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_minimac2_mdio_write(uint8_t phy_addr, uint8_t addr, uint16_t value)
+{
+}
+
+static inline void trace_milkymist_minimac2_mdio_read(uint8_t phy_addr, uint8_t addr, uint16_t value)
+{
+}
+
+static inline void trace_milkymist_minimac2_tx_frame(uint32_t length)
+{
+}
+
+static inline void trace_milkymist_minimac2_rx_frame(const void * buf, uint32_t length)
+{
+}
+
+static inline void trace_milkymist_minimac2_drop_rx_frame(const void * buf)
+{
+}
+
+static inline void trace_milkymist_minimac2_rx_transfer(const void * buf, uint32_t length)
+{
+}
+
+static inline void trace_milkymist_minimac2_raise_irq_rx(void)
+{
+}
+
+static inline void trace_milkymist_minimac2_lower_irq_rx(void)
+{
+}
+
+static inline void trace_milkymist_minimac2_pulse_irq_tx(void)
+{
+}
+
+static inline void trace_milkymist_pfpu_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_pfpu_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_pfpu_vectout(uint32_t a, uint32_t b, uint32_t dma_ptr)
+{
+}
+
+static inline void trace_milkymist_pfpu_pulse_irq(void)
+{
+}
+
+static inline void trace_milkymist_softusb_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_softusb_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_softusb_mevt(uint8_t m)
+{
+}
+
+static inline void trace_milkymist_softusb_kevt(uint8_t m)
+{
+}
+
+static inline void trace_milkymist_softusb_mouse_event(int dx, int dy, int dz, int bs)
+{
+}
+
+static inline void trace_milkymist_softusb_pulse_irq(void)
+{
+}
+
+static inline void trace_milkymist_sysctl_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_sysctl_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_sysctl_icap_write(uint32_t value)
+{
+}
+
+static inline void trace_milkymist_sysctl_start_timer0(void)
+{
+}
+
+static inline void trace_milkymist_sysctl_stop_timer0(void)
+{
+}
+
+static inline void trace_milkymist_sysctl_start_timer1(void)
+{
+}
+
+static inline void trace_milkymist_sysctl_stop_timer1(void)
+{
+}
+
+static inline void trace_milkymist_sysctl_pulse_irq_timer0(void)
+{
+}
+
+static inline void trace_milkymist_sysctl_pulse_irq_timer1(void)
+{
+}
+
+static inline void trace_milkymist_tmu2_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_tmu2_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_tmu2_start(void)
+{
+}
+
+static inline void trace_milkymist_tmu2_pulse_irq(void)
+{
+}
+
+static inline void trace_milkymist_uart_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_uart_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_uart_raise_irq(void)
+{
+}
+
+static inline void trace_milkymist_uart_lower_irq(void)
+{
+}
+
+static inline void trace_milkymist_vgafb_memory_read(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_milkymist_vgafb_memory_write(uint32_t addr, uint32_t value)
+{
+}
+
+static inline void trace_mipsnet_send(uint32_t size)
+{
+}
+
+static inline void trace_mipsnet_receive(uint32_t size)
+{
+}
+
+static inline void trace_mipsnet_read(uint64_t addr, uint32_t val)
+{
+}
+
+static inline void trace_mipsnet_write(uint64_t addr, uint64_t val)
+{
+}
+
+static inline void trace_mipsnet_irq(uint32_t isr, uint32_t intctl)
+{
+}
+
+static inline void trace_pc87312_io_read(uint32_t addr, uint32_t val)
+{
+}
+
+static inline void trace_pc87312_io_write(uint32_t addr, uint32_t val)
+{
+}
+
+static inline void trace_pc87312_info_floppy(uint32_t base)
+{
+}
+
+static inline void trace_pc87312_info_ide(uint32_t base)
+{
+}
+
+static inline void trace_pc87312_info_parallel(uint32_t base, uint32_t irq)
+{
+}
+
+static inline void trace_pc87312_info_serial(int n, uint32_t base, uint32_t irq)
+{
+}
+
+static inline void trace_pvscsi_ring_init_data(uint32_t txr_len_log2, uint32_t rxr_len_log2)
+{
+}
+
+static inline void trace_pvscsi_ring_init_msg(uint32_t len_log2)
+{
+}
+
+static inline void trace_pvscsi_ring_flush_cmp(uint64_t filled_cmp_ptr)
+{
+}
+
+static inline void trace_pvscsi_ring_flush_msg(uint64_t filled_cmp_ptr)
+{
+}
+
+static inline void trace_pvscsi_update_irq_level(bool raise, uint64_t mask, uint64_t status)
+{
+}
+
+static inline void trace_pvscsi_update_irq_msi(void)
+{
+}
+
+static inline void trace_pvscsi_cmp_ring_put(unsigned long addr)
+{
+}
+
+static inline void trace_pvscsi_msg_ring_put(unsigned long addr)
+{
+}
+
+static inline void trace_pvscsi_complete_request(uint64_t context, uint64_t len, uint8_t sense_key)
+{
+}
+
+static inline void trace_pvscsi_get_sg_list(int nsg, size_t size)
+{
+}
+
+static inline void trace_pvscsi_get_next_sg_elem(uint32_t flags)
+{
+}
+
+static inline void trace_pvscsi_command_complete_not_found(uint32_t tag)
+{
+}
+
+static inline void trace_pvscsi_command_complete_data_run(void)
+{
+}
+
+static inline void trace_pvscsi_command_complete_sense_len(int len)
+{
+}
+
+static inline void trace_pvscsi_convert_sglist(uint64_t context, unsigned long addr, uint32_t resid)
+{
+}
+
+static inline void trace_pvscsi_process_req_descr(uint8_t cmd, uint64_t ctx)
+{
+}
+
+static inline void trace_pvscsi_process_req_descr_unknown_device(void)
+{
+}
+
+static inline void trace_pvscsi_process_req_descr_invalid_dir(void)
+{
+}
+
+static inline void trace_pvscsi_process_io(unsigned long addr)
+{
+}
+
+static inline void trace_pvscsi_on_cmd_noimpl(const char* cmd)
+{
+}
+
+static inline void trace_pvscsi_on_cmd_reset_dev(uint32_t tgt, int lun, void* dev)
+{
+}
+
+static inline void trace_pvscsi_on_cmd_arrived(const char* cmd)
+{
+}
+
+static inline void trace_pvscsi_on_cmd_abort(uint64_t ctx, uint32_t tgt)
+{
+}
+
+static inline void trace_pvscsi_on_cmd_unknown(uint64_t cmd_id)
+{
+}
+
+static inline void trace_pvscsi_on_cmd_unknown_data(uint32_t data)
+{
+}
+
+static inline void trace_pvscsi_io_write(const char* cmd, uint64_t val)
+{
+}
+
+static inline void trace_pvscsi_io_write_unknown(unsigned long addr, unsigned sz, uint64_t val)
+{
+}
+
+static inline void trace_pvscsi_io_read(const char* cmd, uint64_t status)
+{
+}
+
+static inline void trace_pvscsi_io_read_unknown(unsigned long addr, unsigned sz)
+{
+}
+
+static inline void trace_pvscsi_init_msi_fail(int res)
+{
+}
+
+static inline void trace_pvscsi_state(const char* state)
+{
+}
+
+static inline void trace_pvscsi_tx_rings_ppn(const char* label, uint64_t ppn)
+{
+}
+
+static inline void trace_pvscsi_tx_rings_num_pages(const char* label, uint32_t num)
+{
+}
+
+static inline void trace_xen_ram_alloc(unsigned long ram_addr, unsigned long size)
+{
+}
+
+static inline void trace_xen_client_set_memory(uint64_t start_addr, unsigned long size, bool log_dirty)
+{
+}
+
+static inline void trace_xen_map_cache(uint64_t phys_addr)
+{
+}
+
+static inline void trace_xen_remap_bucket(uint64_t index)
+{
+}
+
+static inline void trace_xen_map_cache_return(void* ptr)
+{
+}
+
+static inline void trace_xen_map_block(uint64_t phys_addr, uint64_t size)
+{
+}
+
+static inline void trace_xen_unmap_block(void* addr, unsigned long size)
+{
+}
+
+static inline void trace_xen_platform_log(char * s)
+{
+}
+
+static inline void trace_qemu_coroutine_enter(void * from, void * to, void * opaque)
+{
+}
+
+static inline void trace_qemu_coroutine_yield(void * from, void * to)
+{
+}
+
+static inline void trace_qemu_coroutine_terminate(void * co)
+{
+}
+
+static inline void trace_qemu_co_queue_run_restart(void * co)
+{
+}
+
+static inline void trace_qemu_co_queue_next(void * nxt)
+{
+}
+
+static inline void trace_qemu_co_mutex_lock_entry(void * mutex, void * self)
+{
+}
+
+static inline void trace_qemu_co_mutex_lock_return(void * mutex, void * self)
+{
+}
+
+static inline void trace_qemu_co_mutex_unlock_entry(void * mutex, void * self)
+{
+}
+
+static inline void trace_qemu_co_mutex_unlock_return(void * mutex, void * self)
+{
+}
+
+static inline void trace_escc_put_queue(char channel, int b)
+{
+}
+
+static inline void trace_escc_get_queue(char channel, int val)
+{
+}
+
+static inline void trace_escc_update_irq(int irq)
+{
+}
+
+static inline void trace_escc_update_parameters(char channel, int speed, int parity, int data_bits, int stop_bits)
+{
+}
+
+static inline void trace_escc_mem_writeb_ctrl(char channel, uint32_t reg, uint32_t val)
+{
+}
+
+static inline void trace_escc_mem_writeb_data(char channel, uint32_t val)
+{
+}
+
+static inline void trace_escc_mem_readb_ctrl(char channel, uint32_t reg, uint8_t val)
+{
+}
+
+static inline void trace_escc_mem_readb_data(char channel, uint32_t ret)
+{
+}
+
+static inline void trace_escc_serial_receive_byte(char channel, int ch)
+{
+}
+
+static inline void trace_escc_sunkbd_event_in(int ch)
+{
+}
+
+static inline void trace_escc_sunkbd_event_out(int ch)
+{
+}
+
+static inline void trace_escc_kbd_command(int val)
+{
+}
+
+static inline void trace_escc_sunmouse_event(int dx, int dy, int buttons_state)
+{
+}
+
+static inline void trace_iscsi_aio_write16_cb(void * iscsi, int status, void * acb, int canceled)
+{
+}
+
+static inline void trace_iscsi_aio_writev(void * iscsi, int64_t sector_num, int nb_sectors, void * opaque, void * acb)
+{
+}
+
+static inline void trace_iscsi_aio_read16_cb(void * iscsi, int status, void * acb, int canceled)
+{
+}
+
+static inline void trace_iscsi_aio_readv(void * iscsi, int64_t sector_num, int nb_sectors, void * opaque, void * acb)
+{
+}
+
+static inline void trace_esp_error_fifo_overrun(void)
+{
+}
+
+static inline void trace_esp_error_unhandled_command(uint32_t val)
+{
+}
+
+static inline void trace_esp_error_invalid_write(uint32_t val, uint32_t addr)
+{
+}
+
+static inline void trace_esp_raise_irq(void)
+{
+}
+
+static inline void trace_esp_lower_irq(void)
+{
+}
+
+static inline void trace_esp_dma_enable(void)
+{
+}
+
+static inline void trace_esp_dma_disable(void)
+{
+}
+
+static inline void trace_esp_get_cmd(uint32_t dmalen, int target)
+{
+}
+
+static inline void trace_esp_do_busid_cmd(uint8_t busid)
+{
+}
+
+static inline void trace_esp_handle_satn_stop(uint32_t cmdlen)
+{
+}
+
+static inline void trace_esp_write_response(uint32_t status)
+{
+}
+
+static inline void trace_esp_do_dma(uint32_t cmdlen, uint32_t len)
+{
+}
+
+static inline void trace_esp_command_complete(void)
+{
+}
+
+static inline void trace_esp_command_complete_unexpected(void)
+{
+}
+
+static inline void trace_esp_command_complete_fail(void)
+{
+}
+
+static inline void trace_esp_transfer_data(uint32_t dma_left, int32_t ti_size)
+{
+}
+
+static inline void trace_esp_handle_ti(uint32_t minlen)
+{
+}
+
+static inline void trace_esp_handle_ti_cmd(uint32_t cmdlen)
+{
+}
+
+static inline void trace_esp_mem_readb(uint32_t saddr, uint8_t reg)
+{
+}
+
+static inline void trace_esp_mem_writeb(uint32_t saddr, uint8_t reg, uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_nop(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_flush(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_reset(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_bus_reset(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_iccs(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_msgacc(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_pad(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_satn(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_rstatn(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_sel(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_selatn(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_selatns(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_ensel(uint32_t val)
+{
+}
+
+static inline void trace_esp_mem_writeb_cmd_dissel(uint32_t val)
+{
+}
+
+static inline void trace_esp_pci_error_invalid_dma_direction(void)
+{
+}
+
+static inline void trace_esp_pci_error_invalid_read(uint32_t reg)
+{
+}
+
+static inline void trace_esp_pci_error_invalid_write(uint32_t reg)
+{
+}
+
+static inline void trace_esp_pci_error_invalid_write_dma(uint32_t val, uint32_t addr)
+{
+}
+
+static inline void trace_esp_pci_dma_read(uint32_t saddr, uint32_t reg)
+{
+}
+
+static inline void trace_esp_pci_dma_write(uint32_t saddr, uint32_t reg, uint32_t val)
+{
+}
+
+static inline void trace_esp_pci_dma_idle(uint32_t val)
+{
+}
+
+static inline void trace_esp_pci_dma_blast(uint32_t val)
+{
+}
+
+static inline void trace_esp_pci_dma_abort(uint32_t val)
+{
+}
+
+static inline void trace_esp_pci_dma_start(uint32_t val)
+{
+}
+
+static inline void trace_esp_pci_sbac_read(uint32_t reg)
+{
+}
+
+static inline void trace_esp_pci_sbac_write(uint32_t reg, uint32_t val)
+{
+}
+
+static inline void trace_handle_qmp_command(void * mon, const char * cmd_name)
+{
+}
+
+static inline void trace_monitor_protocol_emitter(void * mon)
+{
+}
+
+static inline void trace_monitor_protocol_event(uint32_t event, const char * evname, void * data)
+{
+}
+
+static inline void trace_monitor_protocol_event_handler(uint32_t event, void * data, uint64_t last, uint64_t now)
+{
+}
+
+static inline void trace_monitor_protocol_event_emit(uint32_t event, void * data)
+{
+}
+
+static inline void trace_monitor_protocol_event_queue(uint32_t event, void * data, uint64_t rate, uint64_t last, uint64_t now)
+{
+}
+
+static inline void trace_monitor_protocol_event_throttle(uint32_t event, uint64_t rate)
+{
+}
+
+static inline void trace_open_eth_mii_write(unsigned idx, uint16_t v)
+{
+}
+
+static inline void trace_open_eth_mii_read(unsigned idx, uint16_t v)
+{
+}
+
+static inline void trace_open_eth_update_irq(uint32_t v)
+{
+}
+
+static inline void trace_open_eth_receive(unsigned len)
+{
+}
+
+static inline void trace_open_eth_receive_mcast(unsigned idx, uint32_t h0, uint32_t h1)
+{
+}
+
+static inline void trace_open_eth_receive_reject(void)
+{
+}
+
+static inline void trace_open_eth_receive_desc(uint32_t addr, uint32_t len_flags)
+{
+}
+
+static inline void trace_open_eth_start_xmit(uint32_t addr, unsigned len, unsigned tx_len)
+{
+}
+
+static inline void trace_open_eth_reg_read(uint32_t addr, uint32_t v)
+{
+}
+
+static inline void trace_open_eth_reg_write(uint32_t addr, uint32_t v)
+{
+}
+
+static inline void trace_open_eth_desc_read(uint32_t addr, uint32_t v)
+{
+}
+
+static inline void trace_open_eth_desc_write(uint32_t addr, uint32_t v)
+{
+}
+
+static inline void trace_v9fs_rerror(uint16_t tag, uint8_t id, int err)
+{
+}
+
+static inline void trace_v9fs_version(uint16_t tag, uint8_t id, int32_t msize, char* version)
+{
+}
+
+static inline void trace_v9fs_version_return(uint16_t tag, uint8_t id, int32_t msize, char* version)
+{
+}
+
+static inline void trace_v9fs_attach(uint16_t tag, uint8_t id, int32_t fid, int32_t afid, char* uname, char* aname)
+{
+}
+
+static inline void trace_v9fs_attach_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path)
+{
+}
+
+static inline void trace_v9fs_stat(uint16_t tag, uint8_t id, int32_t fid)
+{
+}
+
+static inline void trace_v9fs_stat_return(uint16_t tag, uint8_t id, int32_t mode, int32_t atime, int32_t mtime, int64_t length)
+{
+}
+
+static inline void trace_v9fs_getattr(uint16_t tag, uint8_t id, int32_t fid, uint64_t request_mask)
+{
+}
+
+static inline void trace_v9fs_getattr_return(uint16_t tag, uint8_t id, uint64_t result_mask, uint32_t mode, uint32_t uid, uint32_t gid)
+{
+}
+
+static inline void trace_v9fs_walk(uint16_t tag, uint8_t id, int32_t fid, int32_t newfid, uint16_t nwnames)
+{
+}
+
+static inline void trace_v9fs_walk_return(uint16_t tag, uint8_t id, uint16_t nwnames, void* qids)
+{
+}
+
+static inline void trace_v9fs_open(uint16_t tag, uint8_t id, int32_t fid, int32_t mode)
+{
+}
+
+static inline void trace_v9fs_open_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int iounit)
+{
+}
+
+static inline void trace_v9fs_lcreate(uint16_t tag, uint8_t id, int32_t dfid, int32_t flags, int32_t mode, uint32_t gid)
+{
+}
+
+static inline void trace_v9fs_lcreate_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int32_t iounit)
+{
+}
+
+static inline void trace_v9fs_fsync(uint16_t tag, uint8_t id, int32_t fid, int datasync)
+{
+}
+
+static inline void trace_v9fs_clunk(uint16_t tag, uint8_t id, int32_t fid)
+{
+}
+
+static inline void trace_v9fs_read(uint16_t tag, uint8_t id, int32_t fid, uint64_t off, uint32_t max_count)
+{
+}
+
+static inline void trace_v9fs_read_return(uint16_t tag, uint8_t id, int32_t count, ssize_t err)
+{
+}
+
+static inline void trace_v9fs_readdir(uint16_t tag, uint8_t id, int32_t fid, uint64_t offset, uint32_t max_count)
+{
+}
+
+static inline void trace_v9fs_readdir_return(uint16_t tag, uint8_t id, uint32_t count, ssize_t retval)
+{
+}
+
+static inline void trace_v9fs_write(uint16_t tag, uint8_t id, int32_t fid, uint64_t off, uint32_t count, int cnt)
+{
+}
+
+static inline void trace_v9fs_write_return(uint16_t tag, uint8_t id, int32_t total, ssize_t err)
+{
+}
+
+static inline void trace_v9fs_create(uint16_t tag, uint8_t id, int32_t fid, char* name, int32_t perm, int8_t mode)
+{
+}
+
+static inline void trace_v9fs_create_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int iounit)
+{
+}
+
+static inline void trace_v9fs_symlink(uint16_t tag, uint8_t id, int32_t fid, char* name, char* symname, uint32_t gid)
+{
+}
+
+static inline void trace_v9fs_symlink_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path)
+{
+}
+
+static inline void trace_v9fs_flush(uint16_t tag, uint8_t id, int16_t flush_tag)
+{
+}
+
+static inline void trace_v9fs_link(uint16_t tag, uint8_t id, int32_t dfid, int32_t oldfid, char* name)
+{
+}
+
+static inline void trace_v9fs_remove(uint16_t tag, uint8_t id, int32_t fid)
+{
+}
+
+static inline void trace_v9fs_wstat(uint16_t tag, uint8_t id, int32_t fid, int32_t mode, int32_t atime, int32_t mtime)
+{
+}
+
+static inline void trace_v9fs_mknod(uint16_t tag, uint8_t id, int32_t fid, int mode, int major, int minor)
+{
+}
+
+static inline void trace_v9fs_mknod_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path)
+{
+}
+
+static inline void trace_v9fs_lock(uint16_t tag, uint8_t id, int32_t fid, uint8_t type, uint64_t start, uint64_t length)
+{
+}
+
+static inline void trace_v9fs_lock_return(uint16_t tag, uint8_t id, int8_t status)
+{
+}
+
+static inline void trace_v9fs_getlock(uint16_t tag, uint8_t id, int32_t fid, uint8_t type, uint64_t start, uint64_t length)
+{
+}
+
+static inline void trace_v9fs_getlock_return(uint16_t tag, uint8_t id, uint8_t type, uint64_t start, uint64_t length, uint32_t proc_id)
+{
+}
+
+static inline void trace_v9fs_mkdir(uint16_t tag, uint8_t id, int32_t fid, char* name, int mode, uint32_t gid)
+{
+}
+
+static inline void trace_v9fs_mkdir_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int err)
+{
+}
+
+static inline void trace_v9fs_xattrwalk(uint16_t tag, uint8_t id, int32_t fid, int32_t newfid, char* name)
+{
+}
+
+static inline void trace_v9fs_xattrwalk_return(uint16_t tag, uint8_t id, int64_t size)
+{
+}
+
+static inline void trace_v9fs_xattrcreate(uint16_t tag, uint8_t id, int32_t fid, char* name, int64_t size, int flags)
+{
+}
+
+static inline void trace_v9fs_readlink(uint16_t tag, uint8_t id, int32_t fid)
+{
+}
+
+static inline void trace_v9fs_readlink_return(uint16_t tag, uint8_t id, char* target)
+{
+}
+
+static inline void trace_mmu_helper_dfault(uint64_t address, uint64_t context, int mmu_idx, uint32_t tl)
+{
+}
+
+static inline void trace_mmu_helper_dprot(uint64_t address, uint64_t context, int mmu_idx, uint32_t tl)
+{
+}
+
+static inline void trace_mmu_helper_dmiss(uint64_t address, uint64_t context)
+{
+}
+
+static inline void trace_mmu_helper_tfault(uint64_t address, uint64_t context)
+{
+}
+
+static inline void trace_mmu_helper_tmiss(uint64_t address, uint64_t context)
+{
+}
+
+static inline void trace_mmu_helper_get_phys_addr_code(uint32_t tl, int mmu_idx, uint64_t prim_context, uint64_t sec_context, uint64_t address)
+{
+}
+
+static inline void trace_mmu_helper_get_phys_addr_data(uint32_t tl, int mmu_idx, uint64_t prim_context, uint64_t sec_context, uint64_t address)
+{
+}
+
+static inline void trace_mmu_helper_mmu_fault(uint64_t address, uint64_t paddr, int mmu_idx, uint32_t tl, uint64_t prim_context, uint64_t sec_context)
+{
+}
+
+static inline void trace_int_helper_set_softint(uint32_t softint)
+{
+}
+
+static inline void trace_int_helper_clear_softint(uint32_t softint)
+{
+}
+
+static inline void trace_int_helper_write_softint(uint32_t softint)
+{
+}
+
+static inline void trace_int_helper_icache_freeze(void)
+{
+}
+
+static inline void trace_int_helper_dcache_freeze(void)
+{
+}
+
+static inline void trace_win_helper_gregset_error(uint32_t pstate)
+{
+}
+
+static inline void trace_win_helper_switch_pstate(uint32_t pstate_regs, uint32_t new_pstate_regs)
+{
+}
+
+static inline void trace_win_helper_no_switch_pstate(uint32_t new_pstate_regs)
+{
+}
+
+static inline void trace_win_helper_wrpil(uint32_t psrpil, uint32_t new_pil)
+{
+}
+
+static inline void trace_win_helper_done(uint32_t tl)
+{
+}
+
+static inline void trace_win_helper_retry(uint32_t tl)
+{
+}
+
+static inline void trace_dma_bdrv_io(void * dbs, void * bs, int64_t sector_num, bool to_dev)
+{
+}
+
+static inline void trace_dma_aio_cancel(void * dbs)
+{
+}
+
+static inline void trace_dma_complete(void * dbs, int ret, void * cb)
+{
+}
+
+static inline void trace_dma_bdrv_cb(void * dbs, int ret)
+{
+}
+
+static inline void trace_dma_map_wait(void * dbs)
+{
+}
+
+static inline void trace_console_gfx_new(void)
+{
+}
+
+static inline void trace_console_txt_new(int w, int h)
+{
+}
+
+static inline void trace_console_select(int nr)
+{
+}
+
+static inline void trace_console_refresh(int interval)
+{
+}
+
+static inline void trace_displaysurface_create(void * display_surface, int w, int h)
+{
+}
+
+static inline void trace_displaysurface_create_from(void * display_surface, int w, int h, int bpp, int swap)
+{
+}
+
+static inline void trace_displaysurface_free(void * display_surface)
+{
+}
+
+static inline void trace_displaychangelistener_register(void * dcl, const char * name)
+{
+}
+
+static inline void trace_displaychangelistener_unregister(void * dcl, const char * name)
+{
+}
+
+static inline void trace_ppm_save(const char * filename, void * display_surface)
+{
+}
+
+static inline void trace_vmware_value_read(uint32_t index, uint32_t value)
+{
+}
+
+static inline void trace_vmware_value_write(uint32_t index, uint32_t value)
+{
+}
+
+static inline void trace_vmware_palette_read(uint32_t index, uint32_t value)
+{
+}
+
+static inline void trace_vmware_palette_write(uint32_t index, uint32_t value)
+{
+}
+
+static inline void trace_vmware_scratch_read(uint32_t index, uint32_t value)
+{
+}
+
+static inline void trace_vmware_scratch_write(uint32_t index, uint32_t value)
+{
+}
+
+static inline void trace_vmware_setmode(uint32_t w, uint32_t h, uint32_t bpp)
+{
+}
+
+static inline void trace_savevm_section_start(void)
+{
+}
+
+static inline void trace_savevm_section_end(unsigned int section_id)
+{
+}
+
+static inline void trace_migration_bitmap_sync_start(void)
+{
+}
+
+static inline void trace_migration_bitmap_sync_end(uint64_t dirty_pages)
+{
+}
+
+static inline void trace_migration_throttle(void)
+{
+}
+
+static inline void trace_qxl_create_guest_primary(int qid, uint32_t width, uint32_t height, uint64_t mem, uint32_t format, uint32_t position)
+{
+}
+
+static inline void trace_qxl_create_guest_primary_rest(int qid, int32_t stride, uint32_t type, uint32_t flags)
+{
+}
+
+static inline void trace_qxl_destroy_primary(int qid)
+{
+}
+
+static inline void trace_qxl_enter_vga_mode(int qid)
+{
+}
+
+static inline void trace_qxl_exit_vga_mode(int qid)
+{
+}
+
+static inline void trace_qxl_hard_reset(int qid, int64_t loadvm)
+{
+}
+
+static inline void trace_qxl_interface_async_complete_io(int qid, uint32_t current_async, void * cookie)
+{
+}
+
+static inline void trace_qxl_interface_attach_worker(int qid)
+{
+}
+
+static inline void trace_qxl_interface_get_init_info(int qid)
+{
+}
+
+static inline void trace_qxl_interface_set_compression_level(int qid, int64_t level)
+{
+}
+
+static inline void trace_qxl_interface_update_area_complete(int qid, uint32_t surface_id, uint32_t dirty_left, uint32_t dirty_right, uint32_t dirty_top, uint32_t dirty_bottom)
+{
+}
+
+static inline void trace_qxl_interface_update_area_complete_rest(int qid, uint32_t num_updated_rects)
+{
+}
+
+static inline void trace_qxl_interface_update_area_complete_overflow(int qid, int max)
+{
+}
+
+static inline void trace_qxl_interface_update_area_complete_schedule_bh(int qid, uint32_t num_dirty)
+{
+}
+
+static inline void trace_qxl_io_destroy_primary_ignored(int qid, const char * mode)
+{
+}
+
+static inline void trace_qxl_io_log(int qid, const uint8_t * log_buf)
+{
+}
+
+static inline void trace_qxl_io_read_unexpected(int qid)
+{
+}
+
+static inline void trace_qxl_io_unexpected_vga_mode(int qid, uint64_t addr, uint64_t val, const char * desc)
+{
+}
+
+static inline void trace_qxl_io_write(int qid, const char * mode, uint64_t addr, uint64_t val, unsigned size, int async)
+{
+}
+
+static inline void trace_qxl_memslot_add_guest(int qid, uint32_t slot_id, uint64_t guest_start, uint64_t guest_end)
+{
+}
+
+static inline void trace_qxl_post_load(int qid, const char * mode)
+{
+}
+
+static inline void trace_qxl_pre_load(int qid)
+{
+}
+
+static inline void trace_qxl_pre_save(int qid)
+{
+}
+
+static inline void trace_qxl_reset_surfaces(int qid)
+{
+}
+
+static inline void trace_qxl_ring_command_check(int qid, const char * mode)
+{
+}
+
+static inline void trace_qxl_ring_command_get(int qid, const char * mode)
+{
+}
+
+static inline void trace_qxl_ring_command_req_notification(int qid)
+{
+}
+
+static inline void trace_qxl_ring_cursor_check(int qid, const char * mode)
+{
+}
+
+static inline void trace_qxl_ring_cursor_get(int qid, const char * mode)
+{
+}
+
+static inline void trace_qxl_ring_cursor_req_notification(int qid)
+{
+}
+
+static inline void trace_qxl_ring_res_push(int qid, const char * mode, uint32_t surface_count, uint32_t free_res, void * last_release, const char * notify)
+{
+}
+
+static inline void trace_qxl_ring_res_push_rest(int qid, uint32_t ring_has, uint32_t ring_size, uint32_t prod, uint32_t cons)
+{
+}
+
+static inline void trace_qxl_ring_res_put(int qid, uint32_t free_res)
+{
+}
+
+static inline void trace_qxl_set_mode(int qid, int modenr, uint32_t x_res, uint32_t y_res, uint32_t bits, uint64_t devmem)
+{
+}
+
+static inline void trace_qxl_soft_reset(int qid)
+{
+}
+
+static inline void trace_qemu_spice_add_memslot(int qid, uint32_t slot_id, unsigned long virt_start, unsigned long virt_end, int async)
+{
+}
+
+static inline void trace_qemu_spice_del_memslot(int qid, uint32_t gid, uint32_t slot_id)
+{
+}
+
+static inline void trace_qemu_spice_create_primary_surface(int qid, uint32_t sid, void * surface, int async)
+{
+}
+
+static inline void trace_qemu_spice_destroy_primary_surface(int qid, uint32_t sid, int async)
+{
+}
+
+static inline void trace_qemu_spice_wakeup(uint32_t qid)
+{
+}
+
+static inline void trace_qemu_spice_start(uint32_t qid)
+{
+}
+
+static inline void trace_qemu_spice_stop(uint32_t qid)
+{
+}
+
+static inline void trace_qemu_spice_create_update(uint32_t left, uint32_t right, uint32_t top, uint32_t bottom)
+{
+}
+
+static inline void trace_qxl_spice_destroy_surfaces_complete(int qid)
+{
+}
+
+static inline void trace_qxl_spice_destroy_surfaces(int qid, int async)
+{
+}
+
+static inline void trace_qxl_spice_destroy_surface_wait_complete(int qid, uint32_t id)
+{
+}
+
+static inline void trace_qxl_spice_destroy_surface_wait(int qid, uint32_t id, int async)
+{
+}
+
+static inline void trace_qxl_spice_flush_surfaces_async(int qid, uint32_t surface_count, uint32_t num_free_res)
+{
+}
+
+static inline void trace_qxl_spice_monitors_config(int qid)
+{
+}
+
+static inline void trace_qxl_spice_loadvm_commands(int qid, void * ext, uint32_t count)
+{
+}
+
+static inline void trace_qxl_spice_oom(int qid)
+{
+}
+
+static inline void trace_qxl_spice_reset_cursor(int qid)
+{
+}
+
+static inline void trace_qxl_spice_reset_image_cache(int qid)
+{
+}
+
+static inline void trace_qxl_spice_reset_memslots(int qid)
+{
+}
+
+static inline void trace_qxl_spice_update_area(int qid, uint32_t surface_id, uint32_t left, uint32_t right, uint32_t top, uint32_t bottom)
+{
+}
+
+static inline void trace_qxl_spice_update_area_rest(int qid, uint32_t num_dirty_rects, uint32_t clear_dirty_region)
+{
+}
+
+static inline void trace_qxl_surfaces_dirty(int qid, int surface, int offset, int size)
+{
+}
+
+static inline void trace_qxl_send_events(int qid, uint32_t events)
+{
+}
+
+static inline void trace_qxl_send_events_vm_stopped(int qid, uint32_t events)
+{
+}
+
+static inline void trace_qxl_set_guest_bug(int qid)
+{
+}
+
+static inline void trace_qxl_interrupt_client_monitors_config(int qid, int num_heads, void * heads)
+{
+}
+
+static inline void trace_qxl_client_monitors_config_unsupported_by_guest(int qid, uint32_t int_mask, void * client_monitors_config)
+{
+}
+
+static inline void trace_qxl_client_monitors_config_unsupported_by_device(int qid, int revision)
+{
+}
+
+static inline void trace_qxl_client_monitors_config_capped(int qid, int requested, int limit)
+{
+}
+
+static inline void trace_qxl_client_monitors_config_crc(int qid, unsigned size, uint32_t crc32)
+{
+}
+
+static inline void trace_qxl_set_client_capabilities_unsupported_by_revision(int qid, int revision)
+{
+}
+
+static inline void trace_qxl_render_blit_guest_primary_initialized(void)
+{
+}
+
+static inline void trace_qxl_render_blit(int32_t stride, int32_t left, int32_t right, int32_t top, int32_t bottom)
+{
+}
+
+static inline void trace_qxl_render_guest_primary_resized(int32_t width, int32_t height, int32_t stride, int32_t bytes_pp, int32_t bits_pp)
+{
+}
+
+static inline void trace_qxl_render_update_area_done(void * cookie)
+{
+}
+
+static inline void trace_spapr_pci_msi(const char * msg, uint32_t n, uint32_t ca)
+{
+}
+
+static inline void trace_spapr_pci_msi_setup(const char * name, unsigned vector, uint64_t addr)
+{
+}
+
+static inline void trace_spapr_pci_rtas_ibm_change_msi(unsigned func, unsigned req)
+{
+}
+
+static inline void trace_spapr_pci_rtas_ibm_query_interrupt_source_number(unsigned ioa, unsigned intr)
+{
+}
+
+static inline void trace_spapr_pci_msi_write(uint64_t addr, uint64_t data, uint32_t dt_irq)
+{
+}
+
+static inline void trace_spapr_pci_lsi_set(const char * busname, int pin, uint32_t irq)
+{
+}
+
+static inline void trace_xics_icp_check_ipi(int server, uint8_t mfrr)
+{
+}
+
+static inline void trace_xics_icp_accept(uint32_t old_xirr, uint32_t new_xirr)
+{
+}
+
+static inline void trace_xics_icp_eoi(int server, uint32_t xirr, uint32_t new_xirr)
+{
+}
+
+static inline void trace_xics_icp_irq(int server, int nr, uint8_t priority)
+{
+}
+
+static inline void trace_xics_icp_raise(uint32_t xirr, uint8_t pending_priority)
+{
+}
+
+static inline void trace_xics_set_irq_msi(int srcno, int nr)
+{
+}
+
+static inline void trace_xics_masked_pending(void)
+{
+}
+
+static inline void trace_xics_set_irq_lsi(int srcno, int nr)
+{
+}
+
+static inline void trace_xics_ics_write_xive(int nr, int srcno, int server, uint8_t priority)
+{
+}
+
+static inline void trace_xics_ics_reject(int nr, int srcno)
+{
+}
+
+static inline void trace_xics_ics_eoi(int nr)
+{
+}
+
+static inline void trace_hbitmap_iter_skip_words(const void * hb, void * hbi, uint64_t pos, unsigned long cur)
+{
+}
+
+static inline void trace_hbitmap_reset(void * hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit)
+{
+}
+
+static inline void trace_hbitmap_set(void * hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit)
+{
+}
+
+static inline void trace_ioinst(const char * insn)
+{
+}
+
+static inline void trace_ioinst_sch_id(const char * insn, int cssid, int ssid, int schid)
+{
+}
+
+static inline void trace_ioinst_chp_id(const char * insn, int cssid, int chpid)
+{
+}
+
+static inline void trace_ioinst_chsc_cmd(uint16_t cmd, uint16_t len)
+{
+}
+
+static inline void trace_css_enable_facility(const char * facility)
+{
+}
+
+static inline void trace_css_crw(uint8_t rsc, uint8_t erc, uint16_t rsid, const char * chained)
+{
+}
+
+static inline void trace_css_chpid_add(uint8_t cssid, uint8_t chpid, uint8_t type)
+{
+}
+
+static inline void trace_css_new_image(uint8_t cssid, const char * default_cssid)
+{
+}
+
+static inline void trace_css_assign_subch(const char * do_assign, uint8_t cssid, uint8_t ssid, uint16_t schid, uint16_t devno)
+{
+}
+
+static inline void trace_css_io_interrupt(int cssid, int ssid, int schid, uint32_t intparm, uint8_t isc, const char * conditional)
+{
+}
+
+static inline void trace_virtio_ccw_interpret_ccw(int cssid, int ssid, int schid, int cmd_code)
+{
+}
+
+static inline void trace_virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char * devno_mode)
+{
+}
+
+static inline void trace_migrate_set_state(int new_state)
+{
+}
+
+static inline void trace_kvm_ioctl(int type, void * arg)
+{
+}
+
+static inline void trace_kvm_vm_ioctl(int type, void * arg)
+{
+}
+
+static inline void trace_kvm_vcpu_ioctl(int cpu_index, int type, void * arg)
+{
+}
+
+static inline void trace_kvm_run_exit(int cpu_index, uint32_t reason)
+{
+}
+
+static inline void trace_object_dynamic_cast_assert(const char * type, const char * target, const char * file, int line, const char * func)
+{
+}
+
+static inline void trace_object_class_dynamic_cast_assert(const char * type, const char * target, const char * file, int line, const char * func)
+{
+}
+#endif /* TRACE__GENERATED_TRACERS_H */
diff --git a/contrib/qemu/util/aes.c b/contrib/qemu/util/aes.c
new file mode 100644
index 000000000..91e97fa6e
--- /dev/null
+++ b/contrib/qemu/util/aes.c
@@ -0,0 +1,1314 @@
+/**
+ *
+ * aes.c - integrated in QEMU by Fabrice Bellard from the OpenSSL project.
+ */
+/*
+ * rijndael-alg-fst.c
+ *
+ * @version 3.0 (December 2000)
+ *
+ * Optimised ANSI C code for the Rijndael cipher (now AES)
+ *
+ * @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be>
+ * @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be>
+ * @author Paulo Barreto <paulo.barreto@terra.com.br>
+ *
+ * This code is hereby placed in the public domain.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "qemu-common.h"
+#include "qemu/aes.h"
+
+#ifndef NDEBUG
+#define NDEBUG
+#endif
+
+typedef uint32_t u32;
+typedef uint16_t u16;
+typedef uint8_t u8;
+
+/* This controls loop-unrolling in aes_core.c */
+#undef FULL_UNROLL
+# define GETU32(pt) (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] <<  8) ^ ((u32)(pt)[3]))
+# define PUTU32(ct, st) { (ct)[0] = (u8)((st) >> 24); (ct)[1] = (u8)((st) >> 16); (ct)[2] = (u8)((st) >>  8); (ct)[3] = (u8)(st); }
+
+/*
+AES_Te0[x] = S [x].[02, 01, 01, 03];
+AES_Te1[x] = S [x].[03, 02, 01, 01];
+AES_Te2[x] = S [x].[01, 03, 02, 01];
+AES_Te3[x] = S [x].[01, 01, 03, 02];
+AES_Te4[x] = S [x].[01, 01, 01, 01];
+
+AES_Td0[x] = Si[x].[0e, 09, 0d, 0b];
+AES_Td1[x] = Si[x].[0b, 0e, 09, 0d];
+AES_Td2[x] = Si[x].[0d, 0b, 0e, 09];
+AES_Td3[x] = Si[x].[09, 0d, 0b, 0e];
+AES_Td4[x] = Si[x].[01, 01, 01, 01];
+*/
+
+const uint32_t AES_Te0[256] = {
+    0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
+    0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
+    0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
+    0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
+    0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
+    0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
+    0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
+    0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
+    0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
+    0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
+    0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
+    0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
+    0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
+    0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
+    0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
+    0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
+    0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
+    0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
+    0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
+    0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
+    0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
+    0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
+    0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
+    0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
+    0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
+    0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
+    0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
+    0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
+    0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
+    0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
+    0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
+    0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
+    0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
+    0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
+    0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
+    0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
+    0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
+    0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
+    0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
+    0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
+    0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
+    0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
+    0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
+    0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
+    0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
+    0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
+    0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
+    0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
+    0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
+    0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
+    0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
+    0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
+    0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
+    0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
+    0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
+    0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
+    0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
+    0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
+    0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
+    0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
+    0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
+    0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
+    0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
+    0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
+};
+const uint32_t AES_Te1[256] = {
+    0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
+    0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
+    0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
+    0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
+    0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
+    0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
+    0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
+    0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
+    0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
+    0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
+    0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
+    0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
+    0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
+    0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
+    0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
+    0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
+    0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
+    0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
+    0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
+    0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
+    0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
+    0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
+    0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
+    0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
+    0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
+    0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
+    0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
+    0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
+    0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
+    0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
+    0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
+    0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
+    0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
+    0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
+    0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
+    0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
+    0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
+    0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
+    0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
+    0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
+    0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
+    0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
+    0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
+    0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
+    0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
+    0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
+    0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
+    0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
+    0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
+    0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
+    0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
+    0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
+    0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
+    0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
+    0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
+    0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
+    0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
+    0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
+    0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
+    0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
+    0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
+    0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
+    0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
+    0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
+};
+const uint32_t AES_Te2[256] = {
+    0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
+    0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
+    0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
+    0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
+    0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
+    0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
+    0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
+    0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
+    0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
+    0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
+    0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
+    0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
+    0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
+    0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
+    0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
+    0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
+    0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
+    0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
+    0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
+    0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
+    0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
+    0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
+    0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
+    0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
+    0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
+    0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
+    0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
+    0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
+    0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
+    0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
+    0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
+    0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
+    0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
+    0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
+    0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
+    0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
+    0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
+    0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
+    0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
+    0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
+    0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
+    0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
+    0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
+    0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
+    0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
+    0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
+    0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
+    0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
+    0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
+    0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
+    0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
+    0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
+    0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
+    0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
+    0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
+    0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
+    0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
+    0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
+    0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
+    0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
+    0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
+    0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
+    0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
+    0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
+};
+const uint32_t AES_Te3[256] = {
+
+    0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
+    0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
+    0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
+    0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
+    0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
+    0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
+    0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
+    0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
+    0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
+    0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
+    0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
+    0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
+    0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
+    0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
+    0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
+    0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
+    0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
+    0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
+    0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
+    0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
+    0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
+    0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
+    0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
+    0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
+    0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
+    0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
+    0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
+    0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
+    0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
+    0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
+    0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
+    0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
+    0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
+    0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
+    0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
+    0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
+    0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
+    0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
+    0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
+    0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
+    0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
+    0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
+    0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
+    0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
+    0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
+    0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
+    0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
+    0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
+    0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
+    0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
+    0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
+    0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
+    0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
+    0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
+    0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
+    0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
+    0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
+    0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
+    0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
+    0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
+    0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
+    0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
+    0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
+    0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
+};
+const uint32_t AES_Te4[256] = {
+    0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU,
+    0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U,
+    0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU,
+    0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U,
+    0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU,
+    0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U,
+    0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU,
+    0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U,
+    0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U,
+    0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU,
+    0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U,
+    0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U,
+    0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U,
+    0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU,
+    0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U,
+    0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U,
+    0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU,
+    0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U,
+    0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U,
+    0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U,
+    0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU,
+    0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU,
+    0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U,
+    0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU,
+    0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU,
+    0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U,
+    0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU,
+    0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U,
+    0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU,
+    0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U,
+    0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U,
+    0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U,
+    0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU,
+    0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U,
+    0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU,
+    0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U,
+    0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU,
+    0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U,
+    0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U,
+    0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU,
+    0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU,
+    0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU,
+    0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U,
+    0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U,
+    0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU,
+    0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U,
+    0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU,
+    0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U,
+    0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU,
+    0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U,
+    0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU,
+    0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU,
+    0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U,
+    0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU,
+    0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U,
+    0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU,
+    0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U,
+    0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U,
+    0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U,
+    0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU,
+    0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU,
+    0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U,
+    0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU,
+    0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U,
+};
+const uint32_t AES_Td0[256] = {
+    0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
+    0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
+    0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
+    0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU,
+    0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U,
+    0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U,
+    0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU,
+    0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U,
+    0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU,
+    0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U,
+    0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U,
+    0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U,
+    0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U,
+    0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU,
+    0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U,
+    0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU,
+    0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U,
+    0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU,
+    0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U,
+    0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U,
+    0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U,
+    0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU,
+    0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U,
+    0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU,
+    0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U,
+    0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU,
+    0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U,
+    0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU,
+    0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU,
+    0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U,
+    0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU,
+    0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U,
+    0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU,
+    0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U,
+    0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U,
+    0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U,
+    0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU,
+    0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U,
+    0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U,
+    0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU,
+    0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U,
+    0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U,
+    0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U,
+    0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U,
+    0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U,
+    0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU,
+    0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U,
+    0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U,
+    0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U,
+    0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U,
+    0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U,
+    0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU,
+    0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU,
+    0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU,
+    0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU,
+    0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U,
+    0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U,
+    0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU,
+    0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU,
+    0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U,
+    0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU,
+    0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U,
+    0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
+    0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U,
+};
+const uint32_t AES_Td1[256] = {
+    0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
+    0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
+    0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
+    0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U,
+    0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U,
+    0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U,
+    0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U,
+    0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U,
+    0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U,
+    0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU,
+    0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU,
+    0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU,
+    0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U,
+    0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU,
+    0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U,
+    0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U,
+    0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U,
+    0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU,
+    0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU,
+    0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U,
+    0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU,
+    0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U,
+    0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU,
+    0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU,
+    0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U,
+    0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U,
+    0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U,
+    0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU,
+    0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U,
+    0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU,
+    0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U,
+    0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U,
+    0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U,
+    0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU,
+    0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U,
+    0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U,
+    0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U,
+    0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U,
+    0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U,
+    0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U,
+    0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU,
+    0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU,
+    0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U,
+    0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU,
+    0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U,
+    0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU,
+    0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU,
+    0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U,
+    0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU,
+    0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U,
+    0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U,
+    0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U,
+    0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U,
+    0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U,
+    0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U,
+    0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U,
+    0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU,
+    0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U,
+    0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U,
+    0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU,
+    0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U,
+    0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U,
+    0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
+    0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U,
+};
+const uint32_t AES_Td2[256] = {
+    0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
+    0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
+    0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
+    0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U,
+    0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU,
+    0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U,
+    0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U,
+    0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U,
+    0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U,
+    0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU,
+    0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U,
+    0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U,
+    0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU,
+    0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U,
+    0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U,
+    0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U,
+    0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U,
+    0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U,
+    0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U,
+    0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU,
+
+    0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U,
+    0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U,
+    0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U,
+    0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U,
+    0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U,
+    0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU,
+    0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU,
+    0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U,
+    0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU,
+    0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U,
+    0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU,
+    0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU,
+    0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU,
+    0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU,
+    0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U,
+    0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U,
+    0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U,
+    0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U,
+    0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U,
+    0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U,
+    0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U,
+    0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU,
+    0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU,
+    0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U,
+    0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U,
+    0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU,
+    0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU,
+    0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U,
+    0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U,
+    0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U,
+    0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U,
+    0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U,
+    0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U,
+    0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U,
+    0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU,
+    0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U,
+    0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U,
+    0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U,
+    0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U,
+    0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U,
+    0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U,
+    0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU,
+    0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
+    0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U,
+};
+const uint32_t AES_Td3[256] = {
+    0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
+    0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
+    0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
+    0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U,
+    0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU,
+    0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU,
+    0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U,
+    0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU,
+    0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U,
+    0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU,
+    0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U,
+    0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U,
+    0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U,
+    0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U,
+    0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U,
+    0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU,
+    0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU,
+    0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U,
+    0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U,
+    0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU,
+    0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU,
+    0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U,
+    0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U,
+    0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U,
+    0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U,
+    0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU,
+    0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U,
+    0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U,
+    0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU,
+    0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU,
+    0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U,
+    0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U,
+    0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U,
+    0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU,
+    0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U,
+    0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U,
+    0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U,
+    0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U,
+    0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U,
+    0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U,
+    0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U,
+    0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU,
+    0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U,
+    0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U,
+    0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU,
+    0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU,
+    0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U,
+    0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU,
+    0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U,
+    0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U,
+    0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U,
+    0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U,
+    0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U,
+    0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U,
+    0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU,
+    0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU,
+    0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU,
+    0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU,
+    0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U,
+    0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U,
+    0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U,
+    0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU,
+    0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
+    0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U,
+};
+const uint32_t AES_Td4[256] = {
+    0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U,
+    0x30303030U, 0x36363636U, 0xa5a5a5a5U, 0x38383838U,
+    0xbfbfbfbfU, 0x40404040U, 0xa3a3a3a3U, 0x9e9e9e9eU,
+    0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, 0xfbfbfbfbU,
+    0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U,
+    0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U,
+    0x34343434U, 0x8e8e8e8eU, 0x43434343U, 0x44444444U,
+    0xc4c4c4c4U, 0xdedededeU, 0xe9e9e9e9U, 0xcbcbcbcbU,
+    0x54545454U, 0x7b7b7b7bU, 0x94949494U, 0x32323232U,
+    0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU,
+    0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU,
+    0x42424242U, 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU,
+    0x08080808U, 0x2e2e2e2eU, 0xa1a1a1a1U, 0x66666666U,
+    0x28282828U, 0xd9d9d9d9U, 0x24242424U, 0xb2b2b2b2U,
+    0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U,
+    0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U,
+    0x72727272U, 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U,
+    0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U,
+    0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, 0xccccccccU,
+    0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U,
+    0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U,
+    0xfdfdfdfdU, 0xededededU, 0xb9b9b9b9U, 0xdadadadaU,
+    0x5e5e5e5eU, 0x15151515U, 0x46464646U, 0x57575757U,
+    0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, 0x84848484U,
+    0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U,
+    0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU,
+    0xf7f7f7f7U, 0xe4e4e4e4U, 0x58585858U, 0x05050505U,
+    0xb8b8b8b8U, 0xb3b3b3b3U, 0x45454545U, 0x06060606U,
+    0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, 0x8f8f8f8fU,
+    0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U,
+    0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U,
+    0x01010101U, 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU,
+    0x3a3a3a3aU, 0x91919191U, 0x11111111U, 0x41414141U,
+    0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, 0xeaeaeaeaU,
+    0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU,
+    0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U,
+    0x96969696U, 0xacacacacU, 0x74747474U, 0x22222222U,
+    0xe7e7e7e7U, 0xadadadadU, 0x35353535U, 0x85858585U,
+    0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, 0xe8e8e8e8U,
+    0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU,
+    0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U,
+    0x1d1d1d1dU, 0x29292929U, 0xc5c5c5c5U, 0x89898989U,
+    0x6f6f6f6fU, 0xb7b7b7b7U, 0x62626262U, 0x0e0e0e0eU,
+    0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, 0x1b1b1b1bU,
+    0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU,
+    0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U,
+    0x9a9a9a9aU, 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU,
+    0x78787878U, 0xcdcdcdcdU, 0x5a5a5a5aU, 0xf4f4f4f4U,
+    0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, 0x33333333U,
+    0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U,
+    0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U,
+    0x27272727U, 0x80808080U, 0xececececU, 0x5f5f5f5fU,
+    0x60606060U, 0x51515151U, 0x7f7f7f7fU, 0xa9a9a9a9U,
+    0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, 0x0d0d0d0dU,
+    0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU,
+    0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU,
+    0xa0a0a0a0U, 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU,
+    0xaeaeaeaeU, 0x2a2a2a2aU, 0xf5f5f5f5U, 0xb0b0b0b0U,
+    0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, 0x3c3c3c3cU,
+    0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U,
+    0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU,
+    0xbabababaU, 0x77777777U, 0xd6d6d6d6U, 0x26262626U,
+    0xe1e1e1e1U, 0x69696969U, 0x14141414U, 0x63636363U,
+    0x55555555U, 0x21212121U, 0x0c0c0c0cU, 0x7d7d7d7dU,
+};
+static const u32 rcon[] = {
+	0x01000000, 0x02000000, 0x04000000, 0x08000000,
+	0x10000000, 0x20000000, 0x40000000, 0x80000000,
+	0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
+};
+
+/**
+ * Expand the cipher key into the encryption key schedule.
+ */
+int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
+			AES_KEY *key) {
+
+	u32 *rk;
+   	int i = 0;
+	u32 temp;
+
+	if (!userKey || !key)
+		return -1;
+	if (bits != 128 && bits != 192 && bits != 256)
+		return -2;
+
+	rk = key->rd_key;
+
+	if (bits==128)
+		key->rounds = 10;
+	else if (bits==192)
+		key->rounds = 12;
+	else
+		key->rounds = 14;
+
+	rk[0] = GETU32(userKey     );
+	rk[1] = GETU32(userKey +  4);
+	rk[2] = GETU32(userKey +  8);
+	rk[3] = GETU32(userKey + 12);
+	if (bits == 128) {
+		while (1) {
+			temp  = rk[3];
+			rk[4] = rk[0] ^
+                                (AES_Te4[(temp >> 16) & 0xff] & 0xff000000) ^
+                                (AES_Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
+                                (AES_Te4[(temp      ) & 0xff] & 0x0000ff00) ^
+                                (AES_Te4[(temp >> 24)       ] & 0x000000ff) ^
+				rcon[i];
+			rk[5] = rk[1] ^ rk[4];
+			rk[6] = rk[2] ^ rk[5];
+			rk[7] = rk[3] ^ rk[6];
+			if (++i == 10) {
+				return 0;
+			}
+			rk += 4;
+		}
+	}
+	rk[4] = GETU32(userKey + 16);
+	rk[5] = GETU32(userKey + 20);
+	if (bits == 192) {
+		while (1) {
+			temp = rk[ 5];
+			rk[ 6] = rk[ 0] ^
+                                (AES_Te4[(temp >> 16) & 0xff] & 0xff000000) ^
+                                (AES_Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
+                                (AES_Te4[(temp      ) & 0xff] & 0x0000ff00) ^
+                                (AES_Te4[(temp >> 24)       ] & 0x000000ff) ^
+				rcon[i];
+			rk[ 7] = rk[ 1] ^ rk[ 6];
+			rk[ 8] = rk[ 2] ^ rk[ 7];
+			rk[ 9] = rk[ 3] ^ rk[ 8];
+			if (++i == 8) {
+				return 0;
+			}
+			rk[10] = rk[ 4] ^ rk[ 9];
+			rk[11] = rk[ 5] ^ rk[10];
+			rk += 6;
+		}
+	}
+	rk[6] = GETU32(userKey + 24);
+	rk[7] = GETU32(userKey + 28);
+	if (bits == 256) {
+		while (1) {
+			temp = rk[ 7];
+			rk[ 8] = rk[ 0] ^
+                                (AES_Te4[(temp >> 16) & 0xff] & 0xff000000) ^
+                                (AES_Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
+                                (AES_Te4[(temp      ) & 0xff] & 0x0000ff00) ^
+                                (AES_Te4[(temp >> 24)       ] & 0x000000ff) ^
+				rcon[i];
+			rk[ 9] = rk[ 1] ^ rk[ 8];
+			rk[10] = rk[ 2] ^ rk[ 9];
+			rk[11] = rk[ 3] ^ rk[10];
+			if (++i == 7) {
+				return 0;
+			}
+			temp = rk[11];
+			rk[12] = rk[ 4] ^
+                                (AES_Te4[(temp >> 24)       ] & 0xff000000) ^
+                                (AES_Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^
+                                (AES_Te4[(temp >>  8) & 0xff] & 0x0000ff00) ^
+                                (AES_Te4[(temp      ) & 0xff] & 0x000000ff);
+			rk[13] = rk[ 5] ^ rk[12];
+			rk[14] = rk[ 6] ^ rk[13];
+			rk[15] = rk[ 7] ^ rk[14];
+
+			rk += 8;
+        	}
+	}
+	return 0;
+}
+
+/**
+ * Expand the cipher key into the decryption key schedule.
+ */
+int AES_set_decrypt_key(const unsigned char *userKey, const int bits,
+			 AES_KEY *key) {
+
+        u32 *rk;
+	int i, j, status;
+	u32 temp;
+
+	/* first, start with an encryption schedule */
+	status = AES_set_encrypt_key(userKey, bits, key);
+	if (status < 0)
+		return status;
+
+	rk = key->rd_key;
+
+	/* invert the order of the round keys: */
+	for (i = 0, j = 4*(key->rounds); i < j; i += 4, j -= 4) {
+		temp = rk[i    ]; rk[i    ] = rk[j    ]; rk[j    ] = temp;
+		temp = rk[i + 1]; rk[i + 1] = rk[j + 1]; rk[j + 1] = temp;
+		temp = rk[i + 2]; rk[i + 2] = rk[j + 2]; rk[j + 2] = temp;
+		temp = rk[i + 3]; rk[i + 3] = rk[j + 3]; rk[j + 3] = temp;
+	}
+	/* apply the inverse MixColumn transform to all round keys but the first and the last: */
+	for (i = 1; i < (key->rounds); i++) {
+		rk += 4;
+		rk[0] =
+                        AES_Td0[AES_Te4[(rk[0] >> 24)       ] & 0xff] ^
+                        AES_Td1[AES_Te4[(rk[0] >> 16) & 0xff] & 0xff] ^
+                        AES_Td2[AES_Te4[(rk[0] >>  8) & 0xff] & 0xff] ^
+                        AES_Td3[AES_Te4[(rk[0]      ) & 0xff] & 0xff];
+		rk[1] =
+                        AES_Td0[AES_Te4[(rk[1] >> 24)       ] & 0xff] ^
+                        AES_Td1[AES_Te4[(rk[1] >> 16) & 0xff] & 0xff] ^
+                        AES_Td2[AES_Te4[(rk[1] >>  8) & 0xff] & 0xff] ^
+                        AES_Td3[AES_Te4[(rk[1]      ) & 0xff] & 0xff];
+		rk[2] =
+                        AES_Td0[AES_Te4[(rk[2] >> 24)       ] & 0xff] ^
+                        AES_Td1[AES_Te4[(rk[2] >> 16) & 0xff] & 0xff] ^
+                        AES_Td2[AES_Te4[(rk[2] >>  8) & 0xff] & 0xff] ^
+                        AES_Td3[AES_Te4[(rk[2]      ) & 0xff] & 0xff];
+		rk[3] =
+                        AES_Td0[AES_Te4[(rk[3] >> 24)       ] & 0xff] ^
+                        AES_Td1[AES_Te4[(rk[3] >> 16) & 0xff] & 0xff] ^
+                        AES_Td2[AES_Te4[(rk[3] >>  8) & 0xff] & 0xff] ^
+                        AES_Td3[AES_Te4[(rk[3]      ) & 0xff] & 0xff];
+	}
+	return 0;
+}
+
+#ifndef AES_ASM
+/*
+ * Encrypt a single block
+ * in and out can overlap
+ */
+void AES_encrypt(const unsigned char *in, unsigned char *out,
+		 const AES_KEY *key) {
+
+	const u32 *rk;
+	u32 s0, s1, s2, s3, t0, t1, t2, t3;
+#ifndef FULL_UNROLL
+	int r;
+#endif /* ?FULL_UNROLL */
+
+	assert(in && out && key);
+	rk = key->rd_key;
+
+	/*
+	 * map byte array block to cipher state
+	 * and add initial round key:
+	 */
+	s0 = GETU32(in     ) ^ rk[0];
+	s1 = GETU32(in +  4) ^ rk[1];
+	s2 = GETU32(in +  8) ^ rk[2];
+	s3 = GETU32(in + 12) ^ rk[3];
+#ifdef FULL_UNROLL
+	/* round 1: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[ 4];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[ 5];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[ 6];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[ 7];
+   	/* round 2: */
+        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[ 8];
+        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[ 9];
+        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[10];
+        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[11];
+	/* round 3: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[12];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[13];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[14];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[15];
+   	/* round 4: */
+        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[16];
+        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[17];
+        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[18];
+        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[19];
+	/* round 5: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[20];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[21];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[22];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[23];
+   	/* round 6: */
+        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[24];
+        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[25];
+        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[26];
+        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[27];
+	/* round 7: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[28];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[29];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[30];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[31];
+   	/* round 8: */
+        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[32];
+        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[33];
+        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[34];
+        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[35];
+	/* round 9: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[36];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[37];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[38];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[39];
+    if (key->rounds > 10) {
+        /* round 10: */
+        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[40];
+        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[41];
+        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[42];
+        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[43];
+        /* round 11: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[44];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[45];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[46];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[47];
+        if (key->rounds > 12) {
+            /* round 12: */
+            s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[48];
+            s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[49];
+            s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[50];
+            s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[51];
+            /* round 13: */
+            t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[52];
+            t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[53];
+            t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[54];
+            t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[55];
+        }
+    }
+    rk += key->rounds << 2;
+#else  /* !FULL_UNROLL */
+    /*
+     * Nr - 1 full rounds:
+     */
+    r = key->rounds >> 1;
+    for (;;) {
+        t0 =
+            AES_Te0[(s0 >> 24)       ] ^
+            AES_Te1[(s1 >> 16) & 0xff] ^
+            AES_Te2[(s2 >>  8) & 0xff] ^
+            AES_Te3[(s3      ) & 0xff] ^
+            rk[4];
+        t1 =
+            AES_Te0[(s1 >> 24)       ] ^
+            AES_Te1[(s2 >> 16) & 0xff] ^
+            AES_Te2[(s3 >>  8) & 0xff] ^
+            AES_Te3[(s0      ) & 0xff] ^
+            rk[5];
+        t2 =
+            AES_Te0[(s2 >> 24)       ] ^
+            AES_Te1[(s3 >> 16) & 0xff] ^
+            AES_Te2[(s0 >>  8) & 0xff] ^
+            AES_Te3[(s1      ) & 0xff] ^
+            rk[6];
+        t3 =
+            AES_Te0[(s3 >> 24)       ] ^
+            AES_Te1[(s0 >> 16) & 0xff] ^
+            AES_Te2[(s1 >>  8) & 0xff] ^
+            AES_Te3[(s2      ) & 0xff] ^
+            rk[7];
+
+        rk += 8;
+        if (--r == 0) {
+            break;
+        }
+
+        s0 =
+            AES_Te0[(t0 >> 24)       ] ^
+            AES_Te1[(t1 >> 16) & 0xff] ^
+            AES_Te2[(t2 >>  8) & 0xff] ^
+            AES_Te3[(t3      ) & 0xff] ^
+            rk[0];
+        s1 =
+            AES_Te0[(t1 >> 24)       ] ^
+            AES_Te1[(t2 >> 16) & 0xff] ^
+            AES_Te2[(t3 >>  8) & 0xff] ^
+            AES_Te3[(t0      ) & 0xff] ^
+            rk[1];
+        s2 =
+            AES_Te0[(t2 >> 24)       ] ^
+            AES_Te1[(t3 >> 16) & 0xff] ^
+            AES_Te2[(t0 >>  8) & 0xff] ^
+            AES_Te3[(t1      ) & 0xff] ^
+            rk[2];
+        s3 =
+            AES_Te0[(t3 >> 24)       ] ^
+            AES_Te1[(t0 >> 16) & 0xff] ^
+            AES_Te2[(t1 >>  8) & 0xff] ^
+            AES_Te3[(t2      ) & 0xff] ^
+            rk[3];
+    }
+#endif /* ?FULL_UNROLL */
+    /*
+	 * apply last round and
+	 * map cipher state to byte array block:
+	 */
+	s0 =
+                (AES_Te4[(t0 >> 24)       ] & 0xff000000) ^
+                (AES_Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Te4[(t2 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Te4[(t3      ) & 0xff] & 0x000000ff) ^
+		rk[0];
+	PUTU32(out     , s0);
+	s1 =
+                (AES_Te4[(t1 >> 24)       ] & 0xff000000) ^
+                (AES_Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Te4[(t3 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Te4[(t0      ) & 0xff] & 0x000000ff) ^
+		rk[1];
+	PUTU32(out +  4, s1);
+	s2 =
+                (AES_Te4[(t2 >> 24)       ] & 0xff000000) ^
+                (AES_Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Te4[(t0 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Te4[(t1      ) & 0xff] & 0x000000ff) ^
+		rk[2];
+	PUTU32(out +  8, s2);
+	s3 =
+                (AES_Te4[(t3 >> 24)       ] & 0xff000000) ^
+                (AES_Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Te4[(t1 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Te4[(t2      ) & 0xff] & 0x000000ff) ^
+		rk[3];
+	PUTU32(out + 12, s3);
+}
+
+/*
+ * Decrypt a single block
+ * in and out can overlap
+ */
+void AES_decrypt(const unsigned char *in, unsigned char *out,
+		 const AES_KEY *key) {
+
+	const u32 *rk;
+	u32 s0, s1, s2, s3, t0, t1, t2, t3;
+#ifndef FULL_UNROLL
+	int r;
+#endif /* ?FULL_UNROLL */
+
+	assert(in && out && key);
+	rk = key->rd_key;
+
+	/*
+	 * map byte array block to cipher state
+	 * and add initial round key:
+	 */
+    s0 = GETU32(in     ) ^ rk[0];
+    s1 = GETU32(in +  4) ^ rk[1];
+    s2 = GETU32(in +  8) ^ rk[2];
+    s3 = GETU32(in + 12) ^ rk[3];
+#ifdef FULL_UNROLL
+    /* round 1: */
+    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[ 4];
+    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[ 5];
+    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[ 6];
+    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[ 7];
+    /* round 2: */
+    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[ 8];
+    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[ 9];
+    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[10];
+    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[11];
+    /* round 3: */
+    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[12];
+    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[13];
+    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[14];
+    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[15];
+    /* round 4: */
+    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[16];
+    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[17];
+    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[18];
+    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[19];
+    /* round 5: */
+    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[20];
+    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[21];
+    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[22];
+    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[23];
+    /* round 6: */
+    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[24];
+    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[25];
+    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[26];
+    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[27];
+    /* round 7: */
+    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[28];
+    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[29];
+    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[30];
+    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[31];
+    /* round 8: */
+    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[32];
+    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[33];
+    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[34];
+    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[35];
+    /* round 9: */
+    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[36];
+    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[37];
+    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[38];
+    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[39];
+    if (key->rounds > 10) {
+        /* round 10: */
+        s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[40];
+        s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[41];
+        s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[42];
+        s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[43];
+        /* round 11: */
+        t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[44];
+        t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[45];
+        t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[46];
+        t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[47];
+        if (key->rounds > 12) {
+            /* round 12: */
+            s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[48];
+            s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[49];
+            s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[50];
+            s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[51];
+            /* round 13: */
+            t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[52];
+            t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[53];
+            t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[54];
+            t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[55];
+        }
+    }
+	rk += key->rounds << 2;
+#else  /* !FULL_UNROLL */
+    /*
+     * Nr - 1 full rounds:
+     */
+    r = key->rounds >> 1;
+    for (;;) {
+        t0 =
+            AES_Td0[(s0 >> 24)       ] ^
+            AES_Td1[(s3 >> 16) & 0xff] ^
+            AES_Td2[(s2 >>  8) & 0xff] ^
+            AES_Td3[(s1      ) & 0xff] ^
+            rk[4];
+        t1 =
+            AES_Td0[(s1 >> 24)       ] ^
+            AES_Td1[(s0 >> 16) & 0xff] ^
+            AES_Td2[(s3 >>  8) & 0xff] ^
+            AES_Td3[(s2      ) & 0xff] ^
+            rk[5];
+        t2 =
+            AES_Td0[(s2 >> 24)       ] ^
+            AES_Td1[(s1 >> 16) & 0xff] ^
+            AES_Td2[(s0 >>  8) & 0xff] ^
+            AES_Td3[(s3      ) & 0xff] ^
+            rk[6];
+        t3 =
+            AES_Td0[(s3 >> 24)       ] ^
+            AES_Td1[(s2 >> 16) & 0xff] ^
+            AES_Td2[(s1 >>  8) & 0xff] ^
+            AES_Td3[(s0      ) & 0xff] ^
+            rk[7];
+
+        rk += 8;
+        if (--r == 0) {
+            break;
+        }
+
+        s0 =
+            AES_Td0[(t0 >> 24)       ] ^
+            AES_Td1[(t3 >> 16) & 0xff] ^
+            AES_Td2[(t2 >>  8) & 0xff] ^
+            AES_Td3[(t1      ) & 0xff] ^
+            rk[0];
+        s1 =
+            AES_Td0[(t1 >> 24)       ] ^
+            AES_Td1[(t0 >> 16) & 0xff] ^
+            AES_Td2[(t3 >>  8) & 0xff] ^
+            AES_Td3[(t2      ) & 0xff] ^
+            rk[1];
+        s2 =
+            AES_Td0[(t2 >> 24)       ] ^
+            AES_Td1[(t1 >> 16) & 0xff] ^
+            AES_Td2[(t0 >>  8) & 0xff] ^
+            AES_Td3[(t3      ) & 0xff] ^
+            rk[2];
+        s3 =
+            AES_Td0[(t3 >> 24)       ] ^
+            AES_Td1[(t2 >> 16) & 0xff] ^
+            AES_Td2[(t1 >>  8) & 0xff] ^
+            AES_Td3[(t0      ) & 0xff] ^
+            rk[3];
+    }
+#endif /* ?FULL_UNROLL */
+    /*
+	 * apply last round and
+	 * map cipher state to byte array block:
+	 */
+   	s0 =
+                (AES_Td4[(t0 >> 24)       ] & 0xff000000) ^
+                (AES_Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Td4[(t2 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Td4[(t1      ) & 0xff] & 0x000000ff) ^
+   		rk[0];
+	PUTU32(out     , s0);
+   	s1 =
+                (AES_Td4[(t1 >> 24)       ] & 0xff000000) ^
+                (AES_Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Td4[(t3 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Td4[(t2      ) & 0xff] & 0x000000ff) ^
+   		rk[1];
+	PUTU32(out +  4, s1);
+   	s2 =
+                (AES_Td4[(t2 >> 24)       ] & 0xff000000) ^
+                (AES_Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Td4[(t0 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Td4[(t3      ) & 0xff] & 0x000000ff) ^
+   		rk[2];
+	PUTU32(out +  8, s2);
+   	s3 =
+                (AES_Td4[(t3 >> 24)       ] & 0xff000000) ^
+                (AES_Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Td4[(t1 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Td4[(t0      ) & 0xff] & 0x000000ff) ^
+   		rk[3];
+	PUTU32(out + 12, s3);
+}
+
+#endif /* AES_ASM */
+
+void AES_cbc_encrypt(const unsigned char *in, unsigned char *out,
+		     const unsigned long length, const AES_KEY *key,
+		     unsigned char *ivec, const int enc)
+{
+
+	unsigned long n;
+	unsigned long len = length;
+	unsigned char tmp[AES_BLOCK_SIZE];
+
+	assert(in && out && key && ivec);
+
+	if (enc) {
+		while (len >= AES_BLOCK_SIZE) {
+			for(n=0; n < AES_BLOCK_SIZE; ++n)
+				tmp[n] = in[n] ^ ivec[n];
+			AES_encrypt(tmp, out, key);
+			memcpy(ivec, out, AES_BLOCK_SIZE);
+			len -= AES_BLOCK_SIZE;
+			in += AES_BLOCK_SIZE;
+			out += AES_BLOCK_SIZE;
+		}
+		if (len) {
+			for(n=0; n < len; ++n)
+				tmp[n] = in[n] ^ ivec[n];
+			for(n=len; n < AES_BLOCK_SIZE; ++n)
+				tmp[n] = ivec[n];
+			AES_encrypt(tmp, tmp, key);
+			memcpy(out, tmp, AES_BLOCK_SIZE);
+			memcpy(ivec, tmp, AES_BLOCK_SIZE);
+		}
+	} else {
+		while (len >= AES_BLOCK_SIZE) {
+			memcpy(tmp, in, AES_BLOCK_SIZE);
+			AES_decrypt(in, out, key);
+			for(n=0; n < AES_BLOCK_SIZE; ++n)
+				out[n] ^= ivec[n];
+			memcpy(ivec, tmp, AES_BLOCK_SIZE);
+			len -= AES_BLOCK_SIZE;
+			in += AES_BLOCK_SIZE;
+			out += AES_BLOCK_SIZE;
+		}
+		if (len) {
+			memcpy(tmp, in, AES_BLOCK_SIZE);
+			AES_decrypt(tmp, tmp, key);
+			for(n=0; n < len; ++n)
+				out[n] = tmp[n] ^ ivec[n];
+			memcpy(ivec, tmp, AES_BLOCK_SIZE);
+		}
+	}
+}
diff --git a/contrib/qemu/util/bitmap.c b/contrib/qemu/util/bitmap.c
new file mode 100644
index 000000000..687841dce
--- /dev/null
+++ b/contrib/qemu/util/bitmap.c
@@ -0,0 +1,256 @@
+/*
+ * Bitmap Module
+ *
+ * Stolen from linux/src/lib/bitmap.c
+ *
+ * Copyright (C) 2010 Corentin Chary
+ *
+ * This source code is licensed under the GNU General Public License,
+ * Version 2.
+ */
+
+#include "qemu/bitops.h"
+#include "qemu/bitmap.h"
+
+/*
+ * bitmaps provide an array of bits, implemented using an an
+ * array of unsigned longs.  The number of valid bits in a
+ * given bitmap does _not_ need to be an exact multiple of
+ * BITS_PER_LONG.
+ *
+ * The possible unused bits in the last, partially used word
+ * of a bitmap are 'don't care'.  The implementation makes
+ * no particular effort to keep them zero.  It ensures that
+ * their value will not affect the results of any operation.
+ * The bitmap operations that return Boolean (bitmap_empty,
+ * for example) or scalar (bitmap_weight, for example) results
+ * carefully filter out these unused bits from impacting their
+ * results.
+ *
+ * These operations actually hold to a slightly stronger rule:
+ * if you don't input any bitmaps to these ops that have some
+ * unused bits set, then they won't output any set unused bits
+ * in output bitmaps.
+ *
+ * The byte ordering of bitmaps is more natural on little
+ * endian architectures.
+ */
+
+int slow_bitmap_empty(const unsigned long *bitmap, int bits)
+{
+    int k, lim = bits/BITS_PER_LONG;
+
+    for (k = 0; k < lim; ++k) {
+        if (bitmap[k]) {
+            return 0;
+        }
+    }
+    if (bits % BITS_PER_LONG) {
+        if (bitmap[k] & BITMAP_LAST_WORD_MASK(bits)) {
+            return 0;
+        }
+    }
+
+    return 1;
+}
+
+int slow_bitmap_full(const unsigned long *bitmap, int bits)
+{
+    int k, lim = bits/BITS_PER_LONG;
+
+    for (k = 0; k < lim; ++k) {
+        if (~bitmap[k]) {
+            return 0;
+        }
+    }
+
+    if (bits % BITS_PER_LONG) {
+        if (~bitmap[k] & BITMAP_LAST_WORD_MASK(bits)) {
+            return 0;
+        }
+    }
+
+    return 1;
+}
+
+int slow_bitmap_equal(const unsigned long *bitmap1,
+                      const unsigned long *bitmap2, int bits)
+{
+    int k, lim = bits/BITS_PER_LONG;
+
+    for (k = 0; k < lim; ++k) {
+        if (bitmap1[k] != bitmap2[k]) {
+            return 0;
+        }
+    }
+
+    if (bits % BITS_PER_LONG) {
+        if ((bitmap1[k] ^ bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) {
+            return 0;
+        }
+    }
+
+    return 1;
+}
+
+void slow_bitmap_complement(unsigned long *dst, const unsigned long *src,
+                            int bits)
+{
+    int k, lim = bits/BITS_PER_LONG;
+
+    for (k = 0; k < lim; ++k) {
+        dst[k] = ~src[k];
+    }
+
+    if (bits % BITS_PER_LONG) {
+        dst[k] = ~src[k] & BITMAP_LAST_WORD_MASK(bits);
+    }
+}
+
+int slow_bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
+                    const unsigned long *bitmap2, int bits)
+{
+    int k;
+    int nr = BITS_TO_LONGS(bits);
+    unsigned long result = 0;
+
+    for (k = 0; k < nr; k++) {
+        result |= (dst[k] = bitmap1[k] & bitmap2[k]);
+    }
+    return result != 0;
+}
+
+void slow_bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
+                    const unsigned long *bitmap2, int bits)
+{
+    int k;
+    int nr = BITS_TO_LONGS(bits);
+
+    for (k = 0; k < nr; k++) {
+        dst[k] = bitmap1[k] | bitmap2[k];
+    }
+}
+
+void slow_bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
+                     const unsigned long *bitmap2, int bits)
+{
+    int k;
+    int nr = BITS_TO_LONGS(bits);
+
+    for (k = 0; k < nr; k++) {
+        dst[k] = bitmap1[k] ^ bitmap2[k];
+    }
+}
+
+int slow_bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
+                       const unsigned long *bitmap2, int bits)
+{
+    int k;
+    int nr = BITS_TO_LONGS(bits);
+    unsigned long result = 0;
+
+    for (k = 0; k < nr; k++) {
+        result |= (dst[k] = bitmap1[k] & ~bitmap2[k]);
+    }
+    return result != 0;
+}
+
+#define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) % BITS_PER_LONG))
+
+void bitmap_set(unsigned long *map, int start, int nr)
+{
+    unsigned long *p = map + BIT_WORD(start);
+    const int size = start + nr;
+    int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG);
+    unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start);
+
+    while (nr - bits_to_set >= 0) {
+        *p |= mask_to_set;
+        nr -= bits_to_set;
+        bits_to_set = BITS_PER_LONG;
+        mask_to_set = ~0UL;
+        p++;
+    }
+    if (nr) {
+        mask_to_set &= BITMAP_LAST_WORD_MASK(size);
+        *p |= mask_to_set;
+    }
+}
+
+void bitmap_clear(unsigned long *map, int start, int nr)
+{
+    unsigned long *p = map + BIT_WORD(start);
+    const int size = start + nr;
+    int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG);
+    unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start);
+
+    while (nr - bits_to_clear >= 0) {
+        *p &= ~mask_to_clear;
+        nr -= bits_to_clear;
+        bits_to_clear = BITS_PER_LONG;
+        mask_to_clear = ~0UL;
+        p++;
+    }
+    if (nr) {
+        mask_to_clear &= BITMAP_LAST_WORD_MASK(size);
+        *p &= ~mask_to_clear;
+    }
+}
+
+#define ALIGN_MASK(x,mask)      (((x)+(mask))&~(mask))
+
+/**
+ * bitmap_find_next_zero_area - find a contiguous aligned zero area
+ * @map: The address to base the search on
+ * @size: The bitmap size in bits
+ * @start: The bitnumber to start searching at
+ * @nr: The number of zeroed bits we're looking for
+ * @align_mask: Alignment mask for zero area
+ *
+ * The @align_mask should be one less than a power of 2; the effect is that
+ * the bit offset of all zero areas this function finds is multiples of that
+ * power of 2. A @align_mask of 0 means no alignment is required.
+ */
+unsigned long bitmap_find_next_zero_area(unsigned long *map,
+					 unsigned long size,
+					 unsigned long start,
+					 unsigned int nr,
+					 unsigned long align_mask)
+{
+    unsigned long index, end, i;
+again:
+    index = find_next_zero_bit(map, size, start);
+
+    /* Align allocation */
+    index = ALIGN_MASK(index, align_mask);
+
+    end = index + nr;
+    if (end > size) {
+        return end;
+    }
+    i = find_next_bit(map, end, index);
+    if (i < end) {
+        start = i + 1;
+        goto again;
+    }
+    return index;
+}
+
+int slow_bitmap_intersects(const unsigned long *bitmap1,
+                           const unsigned long *bitmap2, int bits)
+{
+    int k, lim = bits/BITS_PER_LONG;
+
+    for (k = 0; k < lim; ++k) {
+        if (bitmap1[k] & bitmap2[k]) {
+            return 1;
+        }
+    }
+
+    if (bits % BITS_PER_LONG) {
+        if ((bitmap1[k] & bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) {
+            return 1;
+        }
+    }
+    return 0;
+}
diff --git a/contrib/qemu/util/bitops.c b/contrib/qemu/util/bitops.c
new file mode 100644
index 000000000..227c38b88
--- /dev/null
+++ b/contrib/qemu/util/bitops.c
@@ -0,0 +1,158 @@
+/*
+ * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ * Copyright (C) 2008 IBM Corporation
+ * Written by Rusty Russell <rusty@rustcorp.com.au>
+ * (Inspired by David Howell's find_next_bit implementation)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include "qemu/bitops.h"
+
+#define BITOP_WORD(nr)		((nr) / BITS_PER_LONG)
+
+/*
+ * Find the next set bit in a memory region.
+ */
+unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
+			    unsigned long offset)
+{
+    const unsigned long *p = addr + BITOP_WORD(offset);
+    unsigned long result = offset & ~(BITS_PER_LONG-1);
+    unsigned long tmp;
+
+    if (offset >= size) {
+        return size;
+    }
+    size -= result;
+    offset %= BITS_PER_LONG;
+    if (offset) {
+        tmp = *(p++);
+        tmp &= (~0UL << offset);
+        if (size < BITS_PER_LONG) {
+            goto found_first;
+        }
+        if (tmp) {
+            goto found_middle;
+        }
+        size -= BITS_PER_LONG;
+        result += BITS_PER_LONG;
+    }
+    while (size >= 4*BITS_PER_LONG) {
+        unsigned long d1, d2, d3;
+        tmp = *p;
+        d1 = *(p+1);
+        d2 = *(p+2);
+        d3 = *(p+3);
+        if (tmp) {
+            goto found_middle;
+        }
+        if (d1 | d2 | d3) {
+            break;
+        }
+        p += 4;
+        result += 4*BITS_PER_LONG;
+        size -= 4*BITS_PER_LONG;
+    }
+    while (size >= BITS_PER_LONG) {
+        if ((tmp = *(p++))) {
+            goto found_middle;
+        }
+        result += BITS_PER_LONG;
+        size -= BITS_PER_LONG;
+    }
+    if (!size) {
+        return result;
+    }
+    tmp = *p;
+
+found_first:
+    tmp &= (~0UL >> (BITS_PER_LONG - size));
+    if (tmp == 0UL) {		/* Are any bits set? */
+        return result + size;	/* Nope. */
+    }
+found_middle:
+    return result + ctzl(tmp);
+}
+
+/*
+ * This implementation of find_{first,next}_zero_bit was stolen from
+ * Linus' asm-alpha/bitops.h.
+ */
+unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
+				 unsigned long offset)
+{
+    const unsigned long *p = addr + BITOP_WORD(offset);
+    unsigned long result = offset & ~(BITS_PER_LONG-1);
+    unsigned long tmp;
+
+    if (offset >= size) {
+        return size;
+    }
+    size -= result;
+    offset %= BITS_PER_LONG;
+    if (offset) {
+        tmp = *(p++);
+        tmp |= ~0UL >> (BITS_PER_LONG - offset);
+        if (size < BITS_PER_LONG) {
+            goto found_first;
+        }
+        if (~tmp) {
+            goto found_middle;
+        }
+        size -= BITS_PER_LONG;
+        result += BITS_PER_LONG;
+    }
+    while (size & ~(BITS_PER_LONG-1)) {
+        if (~(tmp = *(p++))) {
+            goto found_middle;
+        }
+        result += BITS_PER_LONG;
+        size -= BITS_PER_LONG;
+    }
+    if (!size) {
+        return result;
+    }
+    tmp = *p;
+
+found_first:
+    tmp |= ~0UL << size;
+    if (tmp == ~0UL) {	/* Are any bits zero? */
+        return result + size;	/* Nope. */
+    }
+found_middle:
+    return result + ctzl(~tmp);
+}
+
+unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
+{
+    unsigned long words;
+    unsigned long tmp;
+
+    /* Start at final word. */
+    words = size / BITS_PER_LONG;
+
+    /* Partial final word? */
+    if (size & (BITS_PER_LONG-1)) {
+        tmp = (addr[words] & (~0UL >> (BITS_PER_LONG
+                                       - (size & (BITS_PER_LONG-1)))));
+        if (tmp) {
+            goto found;
+        }
+    }
+
+    while (words) {
+        tmp = addr[--words];
+        if (tmp) {
+        found:
+            return words * BITS_PER_LONG + BITS_PER_LONG - 1 - clzl(tmp);
+        }
+    }
+
+    /* Not found */
+    return size;
+}
diff --git a/contrib/qemu/util/cutils.c b/contrib/qemu/util/cutils.c
new file mode 100644
index 000000000..0116fcde7
--- /dev/null
+++ b/contrib/qemu/util/cutils.c
@@ -0,0 +1,532 @@
+/*
+ * Simple C functions to supplement the C library
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "qemu/host-utils.h"
+#include <math.h>
+
+#include "qemu/sockets.h"
+#include "qemu/iov.h"
+
+void strpadcpy(char *buf, int buf_size, const char *str, char pad)
+{
+    int len = qemu_strnlen(str, buf_size);
+    memcpy(buf, str, len);
+    memset(buf + len, pad, buf_size - len);
+}
+
+void pstrcpy(char *buf, int buf_size, const char *str)
+{
+    int c;
+    char *q = buf;
+
+    if (buf_size <= 0)
+        return;
+
+    for(;;) {
+        c = *str++;
+        if (c == 0 || q >= buf + buf_size - 1)
+            break;
+        *q++ = c;
+    }
+    *q = '\0';
+}
+
+/* strcat and truncate. */
+char *pstrcat(char *buf, int buf_size, const char *s)
+{
+    int len;
+    len = strlen(buf);
+    if (len < buf_size)
+        pstrcpy(buf + len, buf_size - len, s);
+    return buf;
+}
+
+int strstart(const char *str, const char *val, const char **ptr)
+{
+    const char *p, *q;
+    p = str;
+    q = val;
+    while (*q != '\0') {
+        if (*p != *q)
+            return 0;
+        p++;
+        q++;
+    }
+    if (ptr)
+        *ptr = p;
+    return 1;
+}
+
+int stristart(const char *str, const char *val, const char **ptr)
+{
+    const char *p, *q;
+    p = str;
+    q = val;
+    while (*q != '\0') {
+        if (qemu_toupper(*p) != qemu_toupper(*q))
+            return 0;
+        p++;
+        q++;
+    }
+    if (ptr)
+        *ptr = p;
+    return 1;
+}
+
+/* XXX: use host strnlen if available ? */
+int qemu_strnlen(const char *s, int max_len)
+{
+    int i;
+
+    for(i = 0; i < max_len; i++) {
+        if (s[i] == '\0') {
+            break;
+        }
+    }
+    return i;
+}
+
+char *qemu_strsep(char **input, const char *delim)
+{
+    char *result = *input;
+    if (result != NULL) {
+        char *p;
+
+        for (p = result; *p != '\0'; p++) {
+            if (strchr(delim, *p)) {
+                break;
+            }
+        }
+        if (*p == '\0') {
+            *input = NULL;
+        } else {
+            *p = '\0';
+            *input = p + 1;
+        }
+    }
+    return result;
+}
+
+time_t mktimegm(struct tm *tm)
+{
+    time_t t;
+    int y = tm->tm_year + 1900, m = tm->tm_mon + 1, d = tm->tm_mday;
+    if (m < 3) {
+        m += 12;
+        y--;
+    }
+    t = 86400ULL * (d + (153 * m - 457) / 5 + 365 * y + y / 4 - y / 100 + 
+                 y / 400 - 719469);
+    t += 3600 * tm->tm_hour + 60 * tm->tm_min + tm->tm_sec;
+    return t;
+}
+
+int qemu_fls(int i)
+{
+    return 32 - clz32(i);
+}
+
+/*
+ * Make sure data goes on disk, but if possible do not bother to
+ * write out the inode just for timestamp updates.
+ *
+ * Unfortunately even in 2009 many operating systems do not support
+ * fdatasync and have to fall back to fsync.
+ */
+int qemu_fdatasync(int fd)
+{
+#ifdef CONFIG_FDATASYNC
+    return fdatasync(fd);
+#else
+    return fsync(fd);
+#endif
+}
+
+/*
+ * Searches for an area with non-zero content in a buffer
+ *
+ * Attention! The len must be a multiple of
+ * BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR * sizeof(VECTYPE)
+ * and addr must be a multiple of sizeof(VECTYPE) due to
+ * restriction of optimizations in this function.
+ *
+ * can_use_buffer_find_nonzero_offset() can be used to check
+ * these requirements.
+ *
+ * The return value is the offset of the non-zero area rounded
+ * down to a multiple of sizeof(VECTYPE) for the first
+ * BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR chunks and down to
+ * BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR * sizeof(VECTYPE)
+ * afterwards.
+ *
+ * If the buffer is all zero the return value is equal to len.
+ */
+
+size_t buffer_find_nonzero_offset(const void *buf, size_t len)
+{
+    const VECTYPE *p = buf;
+    const VECTYPE zero = (VECTYPE){0};
+    size_t i;
+
+    assert(can_use_buffer_find_nonzero_offset(buf, len));
+
+    if (!len) {
+        return 0;
+    }
+
+    for (i = 0; i < BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR; i++) {
+        if (!ALL_EQ(p[i], zero)) {
+            return i * sizeof(VECTYPE);
+        }
+    }
+
+    for (i = BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR;
+         i < len / sizeof(VECTYPE);
+         i += BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR) {
+        VECTYPE tmp0 = p[i + 0] | p[i + 1];
+        VECTYPE tmp1 = p[i + 2] | p[i + 3];
+        VECTYPE tmp2 = p[i + 4] | p[i + 5];
+        VECTYPE tmp3 = p[i + 6] | p[i + 7];
+        VECTYPE tmp01 = tmp0 | tmp1;
+        VECTYPE tmp23 = tmp2 | tmp3;
+        if (!ALL_EQ(tmp01 | tmp23, zero)) {
+            break;
+        }
+    }
+
+    return i * sizeof(VECTYPE);
+}
+
+/*
+ * Checks if a buffer is all zeroes
+ *
+ * Attention! The len must be a multiple of 4 * sizeof(long) due to
+ * restriction of optimizations in this function.
+ */
+bool buffer_is_zero(const void *buf, size_t len)
+{
+    /*
+     * Use long as the biggest available internal data type that fits into the
+     * CPU register and unroll the loop to smooth out the effect of memory
+     * latency.
+     */
+
+    size_t i;
+    long d0, d1, d2, d3;
+    const long * const data = buf;
+
+    /* use vector optimized zero check if possible */
+    if (can_use_buffer_find_nonzero_offset(buf, len)) {
+        return buffer_find_nonzero_offset(buf, len) == len;
+    }
+
+    assert(len % (4 * sizeof(long)) == 0);
+    len /= sizeof(long);
+
+    for (i = 0; i < len; i += 4) {
+        d0 = data[i + 0];
+        d1 = data[i + 1];
+        d2 = data[i + 2];
+        d3 = data[i + 3];
+
+        if (d0 || d1 || d2 || d3) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+#ifndef _WIN32
+/* Sets a specific flag */
+int fcntl_setfl(int fd, int flag)
+{
+    int flags;
+
+    flags = fcntl(fd, F_GETFL);
+    if (flags == -1)
+        return -errno;
+
+    if (fcntl(fd, F_SETFL, flags | flag) == -1)
+        return -errno;
+
+    return 0;
+}
+#endif
+
+static int64_t suffix_mul(char suffix, int64_t unit)
+{
+    switch (qemu_toupper(suffix)) {
+    case STRTOSZ_DEFSUFFIX_B:
+        return 1;
+    case STRTOSZ_DEFSUFFIX_KB:
+        return unit;
+    case STRTOSZ_DEFSUFFIX_MB:
+        return unit * unit;
+    case STRTOSZ_DEFSUFFIX_GB:
+        return unit * unit * unit;
+    case STRTOSZ_DEFSUFFIX_TB:
+        return unit * unit * unit * unit;
+    case STRTOSZ_DEFSUFFIX_PB:
+        return unit * unit * unit * unit * unit;
+    case STRTOSZ_DEFSUFFIX_EB:
+        return unit * unit * unit * unit * unit * unit;
+    }
+    return -1;
+}
+
+/*
+ * Convert string to bytes, allowing either B/b for bytes, K/k for KB,
+ * M/m for MB, G/g for GB or T/t for TB. End pointer will be returned
+ * in *end, if not NULL. Return -ERANGE on overflow, Return -EINVAL on
+ * other error.
+ */
+int64_t strtosz_suffix_unit(const char *nptr, char **end,
+                            const char default_suffix, int64_t unit)
+{
+    int64_t retval = -EINVAL;
+    char *endptr;
+    unsigned char c;
+    int mul_required = 0;
+    double val, mul, integral, fraction;
+
+    errno = 0;
+    val = strtod(nptr, &endptr);
+    if (isnan(val) || endptr == nptr || errno != 0) {
+        goto fail;
+    }
+    fraction = modf(val, &integral);
+    if (fraction != 0) {
+        mul_required = 1;
+    }
+    c = *endptr;
+    mul = suffix_mul(c, unit);
+    if (mul >= 0) {
+        endptr++;
+    } else {
+        mul = suffix_mul(default_suffix, unit);
+        assert(mul >= 0);
+    }
+    if (mul == 1 && mul_required) {
+        goto fail;
+    }
+    if ((val * mul >= INT64_MAX) || val < 0) {
+        retval = -ERANGE;
+        goto fail;
+    }
+    retval = val * mul;
+
+fail:
+    if (end) {
+        *end = endptr;
+    }
+
+    return retval;
+}
+
+int64_t strtosz_suffix(const char *nptr, char **end, const char default_suffix)
+{
+    return strtosz_suffix_unit(nptr, end, default_suffix, 1024);
+}
+
+int64_t strtosz(const char *nptr, char **end)
+{
+    return strtosz_suffix(nptr, end, STRTOSZ_DEFSUFFIX_MB);
+}
+
+/**
+ * parse_uint:
+ *
+ * @s: String to parse
+ * @value: Destination for parsed integer value
+ * @endptr: Destination for pointer to first character not consumed
+ * @base: integer base, between 2 and 36 inclusive, or 0
+ *
+ * Parse unsigned integer
+ *
+ * Parsed syntax is like strtoull()'s: arbitrary whitespace, a single optional
+ * '+' or '-', an optional "0x" if @base is 0 or 16, one or more digits.
+ *
+ * If @s is null, or @base is invalid, or @s doesn't start with an
+ * integer in the syntax above, set *@value to 0, *@endptr to @s, and
+ * return -EINVAL.
+ *
+ * Set *@endptr to point right beyond the parsed integer (even if the integer
+ * overflows or is negative, all digits will be parsed and *@endptr will
+ * point right beyond them).
+ *
+ * If the integer is negative, set *@value to 0, and return -ERANGE.
+ *
+ * If the integer overflows unsigned long long, set *@value to
+ * ULLONG_MAX, and return -ERANGE.
+ *
+ * Else, set *@value to the parsed integer, and return 0.
+ */
+int parse_uint(const char *s, unsigned long long *value, char **endptr,
+               int base)
+{
+    int r = 0;
+    char *endp = (char *)s;
+    unsigned long long val = 0;
+
+    if (!s) {
+        r = -EINVAL;
+        goto out;
+    }
+
+    errno = 0;
+    val = strtoull(s, &endp, base);
+    if (errno) {
+        r = -errno;
+        goto out;
+    }
+
+    if (endp == s) {
+        r = -EINVAL;
+        goto out;
+    }
+
+    /* make sure we reject negative numbers: */
+    while (isspace((unsigned char)*s)) {
+        s++;
+    }
+    if (*s == '-') {
+        val = 0;
+        r = -ERANGE;
+        goto out;
+    }
+
+out:
+    *value = val;
+    *endptr = endp;
+    return r;
+}
+
+/**
+ * parse_uint_full:
+ *
+ * @s: String to parse
+ * @value: Destination for parsed integer value
+ * @base: integer base, between 2 and 36 inclusive, or 0
+ *
+ * Parse unsigned integer from entire string
+ *
+ * Have the same behavior of parse_uint(), but with an additional check
+ * for additional data after the parsed number. If extra characters are present
+ * after the parsed number, the function will return -EINVAL, and *@v will
+ * be set to 0.
+ */
+int parse_uint_full(const char *s, unsigned long long *value, int base)
+{
+    char *endp;
+    int r;
+
+    r = parse_uint(s, value, &endp, base);
+    if (r < 0) {
+        return r;
+    }
+    if (*endp) {
+        *value = 0;
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+int qemu_parse_fd(const char *param)
+{
+    int fd;
+    char *endptr = NULL;
+
+    fd = strtol(param, &endptr, 10);
+    if (*endptr || (fd == 0 && param == endptr)) {
+        return -1;
+    }
+    return fd;
+}
+
+/* round down to the nearest power of 2*/
+int64_t pow2floor(int64_t value)
+{
+    if (!is_power_of_2(value)) {
+        value = 0x8000000000000000ULL >> clz64(value);
+    }
+    return value;
+}
+
+/*
+ * Implementation of  ULEB128 (http://en.wikipedia.org/wiki/LEB128)
+ * Input is limited to 14-bit numbers
+ */
+int uleb128_encode_small(uint8_t *out, uint32_t n)
+{
+    g_assert(n <= 0x3fff);
+    if (n < 0x80) {
+        *out++ = n;
+        return 1;
+    } else {
+        *out++ = (n & 0x7f) | 0x80;
+        *out++ = n >> 7;
+        return 2;
+    }
+}
+
+int uleb128_decode_small(const uint8_t *in, uint32_t *n)
+{
+    if (!(*in & 0x80)) {
+        *n = *in++;
+        return 1;
+    } else {
+        *n = *in++ & 0x7f;
+        /* we exceed 14 bit number */
+        if (*in & 0x80) {
+            return -1;
+        }
+        *n |= *in++ << 7;
+        return 2;
+    }
+}
+
+/*
+ * helper to parse debug environment variables
+ */
+int parse_debug_env(const char *name, int max, int initial)
+{
+    char *debug_env = getenv(name);
+    char *inv = NULL;
+    int debug;
+
+    if (!debug_env) {
+        return initial;
+    }
+    debug = strtol(debug_env, &inv, 10);
+    if (inv == debug_env) {
+        return initial;
+    }
+    if (debug < 0 || debug > max) {
+        fprintf(stderr, "warning: %s not in [0, %d]", name, max);
+        return initial;
+    }
+    return debug;
+}
diff --git a/contrib/qemu/util/error.c b/contrib/qemu/util/error.c
new file mode 100644
index 000000000..53b04354a
--- /dev/null
+++ b/contrib/qemu/util/error.c
@@ -0,0 +1,120 @@
+/*
+ * QEMU Error Objects
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.  See
+ * the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu-common.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi-types.h"
+#include "qapi/qmp/qerror.h"
+
+struct Error
+{
+    char *msg;
+    ErrorClass err_class;
+};
+
+void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
+{
+    Error *err;
+    va_list ap;
+
+    if (errp == NULL) {
+        return;
+    }
+    assert(*errp == NULL);
+
+    err = g_malloc0(sizeof(*err));
+
+    va_start(ap, fmt);
+    err->msg = g_strdup_vprintf(fmt, ap);
+    va_end(ap);
+    err->err_class = err_class;
+
+    *errp = err;
+}
+
+void error_set_errno(Error **errp, int os_errno, ErrorClass err_class,
+                     const char *fmt, ...)
+{
+    Error *err;
+    char *msg1;
+    va_list ap;
+
+    if (errp == NULL) {
+        return;
+    }
+    assert(*errp == NULL);
+
+    err = g_malloc0(sizeof(*err));
+
+    va_start(ap, fmt);
+    msg1 = g_strdup_vprintf(fmt, ap);
+    if (os_errno != 0) {
+        err->msg = g_strdup_printf("%s: %s", msg1, strerror(os_errno));
+        g_free(msg1);
+    } else {
+        err->msg = msg1;
+    }
+    va_end(ap);
+    err->err_class = err_class;
+
+    *errp = err;
+}
+
+void error_setg_file_open(Error **errp, int os_errno, const char *filename)
+{
+    error_setg_errno(errp, os_errno, "Could not open '%s'", filename);
+}
+
+Error *error_copy(const Error *err)
+{
+    Error *err_new;
+
+    err_new = g_malloc0(sizeof(*err));
+    err_new->msg = g_strdup(err->msg);
+    err_new->err_class = err->err_class;
+
+    return err_new;
+}
+
+bool error_is_set(Error **errp)
+{
+    return (errp && *errp);
+}
+
+ErrorClass error_get_class(const Error *err)
+{
+    return err->err_class;
+}
+
+const char *error_get_pretty(Error *err)
+{
+    return err->msg;
+}
+
+void error_free(Error *err)
+{
+    if (err) {
+        g_free(err->msg);
+        g_free(err);
+    }
+}
+
+void error_propagate(Error **dst_err, Error *local_err)
+{
+    if (dst_err && !*dst_err) {
+        *dst_err = local_err;
+    } else if (local_err) {
+        error_free(local_err);
+    }
+}
diff --git a/contrib/qemu/util/hbitmap.c b/contrib/qemu/util/hbitmap.c
new file mode 100644
index 000000000..d93683128
--- /dev/null
+++ b/contrib/qemu/util/hbitmap.c
@@ -0,0 +1,402 @@
+/*
+ * Hierarchical Bitmap Data Type
+ *
+ * Copyright Red Hat, Inc., 2012
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include <string.h>
+#include <glib.h>
+#include <assert.h>
+#include "qemu/osdep.h"
+#include "qemu/hbitmap.h"
+#include "qemu/host-utils.h"
+#include "trace.h"
+
+/* HBitmaps provides an array of bits.  The bits are stored as usual in an
+ * array of unsigned longs, but HBitmap is also optimized to provide fast
+ * iteration over set bits; going from one bit to the next is O(logB n)
+ * worst case, with B = sizeof(long) * CHAR_BIT: the result is low enough
+ * that the number of levels is in fact fixed.
+ *
+ * In order to do this, it stacks multiple bitmaps with progressively coarser
+ * granularity; in all levels except the last, bit N is set iff the N-th
+ * unsigned long is nonzero in the immediately next level.  When iteration
+ * completes on the last level it can examine the 2nd-last level to quickly
+ * skip entire words, and even do so recursively to skip blocks of 64 words or
+ * powers thereof (32 on 32-bit machines).
+ *
+ * Given an index in the bitmap, it can be split in group of bits like
+ * this (for the 64-bit case):
+ *
+ *   bits 0-57 => word in the last bitmap     | bits 58-63 => bit in the word
+ *   bits 0-51 => word in the 2nd-last bitmap | bits 52-57 => bit in the word
+ *   bits 0-45 => word in the 3rd-last bitmap | bits 46-51 => bit in the word
+ *
+ * So it is easy to move up simply by shifting the index right by
+ * log2(BITS_PER_LONG) bits.  To move down, you shift the index left
+ * similarly, and add the word index within the group.  Iteration uses
+ * ffs (find first set bit) to find the next word to examine; this
+ * operation can be done in constant time in most current architectures.
+ *
+ * Setting or clearing a range of m bits on all levels, the work to perform
+ * is O(m + m/W + m/W^2 + ...), which is O(m) like on a regular bitmap.
+ *
+ * When iterating on a bitmap, each bit (on any level) is only visited
+ * once.  Hence, The total cost of visiting a bitmap with m bits in it is
+ * the number of bits that are set in all bitmaps.  Unless the bitmap is
+ * extremely sparse, this is also O(m + m/W + m/W^2 + ...), so the amortized
+ * cost of advancing from one bit to the next is usually constant (worst case
+ * O(logB n) as in the non-amortized complexity).
+ */
+
+struct HBitmap {
+    /* Number of total bits in the bottom level.  */
+    uint64_t size;
+
+    /* Number of set bits in the bottom level.  */
+    uint64_t count;
+
+    /* A scaling factor.  Given a granularity of G, each bit in the bitmap will
+     * will actually represent a group of 2^G elements.  Each operation on a
+     * range of bits first rounds the bits to determine which group they land
+     * in, and then affect the entire page; iteration will only visit the first
+     * bit of each group.  Here is an example of operations in a size-16,
+     * granularity-1 HBitmap:
+     *
+     *    initial state            00000000
+     *    set(start=0, count=9)    11111000 (iter: 0, 2, 4, 6, 8)
+     *    reset(start=1, count=3)  00111000 (iter: 4, 6, 8)
+     *    set(start=9, count=2)    00111100 (iter: 4, 6, 8, 10)
+     *    reset(start=5, count=5)  00000000
+     *
+     * From an implementation point of view, when setting or resetting bits,
+     * the bitmap will scale bit numbers right by this amount of bits.  When
+     * iterating, the bitmap will scale bit numbers left by this amount of
+     * bits.
+     */
+    int granularity;
+
+    /* A number of progressively less coarse bitmaps (i.e. level 0 is the
+     * coarsest).  Each bit in level N represents a word in level N+1 that
+     * has a set bit, except the last level where each bit represents the
+     * actual bitmap.
+     *
+     * Note that all bitmaps have the same number of levels.  Even a 1-bit
+     * bitmap will still allocate HBITMAP_LEVELS arrays.
+     */
+    unsigned long *levels[HBITMAP_LEVELS];
+};
+
+static inline int popcountl(unsigned long l)
+{
+    return BITS_PER_LONG == 32 ? ctpop32(l) : ctpop64(l);
+}
+
+/* Advance hbi to the next nonzero word and return it.  hbi->pos
+ * is updated.  Returns zero if we reach the end of the bitmap.
+ */
+unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi)
+{
+    size_t pos = hbi->pos;
+    const HBitmap *hb = hbi->hb;
+    unsigned i = HBITMAP_LEVELS - 1;
+
+    unsigned long cur;
+    do {
+        cur = hbi->cur[--i];
+        pos >>= BITS_PER_LEVEL;
+    } while (cur == 0);
+
+    /* Check for end of iteration.  We always use fewer than BITS_PER_LONG
+     * bits in the level 0 bitmap; thus we can repurpose the most significant
+     * bit as a sentinel.  The sentinel is set in hbitmap_alloc and ensures
+     * that the above loop ends even without an explicit check on i.
+     */
+
+    if (i == 0 && cur == (1UL << (BITS_PER_LONG - 1))) {
+        return 0;
+    }
+    for (; i < HBITMAP_LEVELS - 1; i++) {
+        /* Shift back pos to the left, matching the right shifts above.
+         * The index of this word's least significant set bit provides
+         * the low-order bits.
+         */
+        assert(cur);
+        pos = (pos << BITS_PER_LEVEL) + ctzl(cur);
+        hbi->cur[i] = cur & (cur - 1);
+
+        /* Set up next level for iteration.  */
+        cur = hb->levels[i + 1][pos];
+    }
+
+    hbi->pos = pos;
+    trace_hbitmap_iter_skip_words(hbi->hb, hbi, pos, cur);
+
+    assert(cur);
+    return cur;
+}
+
+void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first)
+{
+    unsigned i, bit;
+    uint64_t pos;
+
+    hbi->hb = hb;
+    pos = first >> hb->granularity;
+    assert(pos < hb->size);
+    hbi->pos = pos >> BITS_PER_LEVEL;
+    hbi->granularity = hb->granularity;
+
+    for (i = HBITMAP_LEVELS; i-- > 0; ) {
+        bit = pos & (BITS_PER_LONG - 1);
+        pos >>= BITS_PER_LEVEL;
+
+        /* Drop bits representing items before first.  */
+        hbi->cur[i] = hb->levels[i][pos] & ~((1UL << bit) - 1);
+
+        /* We have already added level i+1, so the lowest set bit has
+         * been processed.  Clear it.
+         */
+        if (i != HBITMAP_LEVELS - 1) {
+            hbi->cur[i] &= ~(1UL << bit);
+        }
+    }
+}
+
+bool hbitmap_empty(const HBitmap *hb)
+{
+    return hb->count == 0;
+}
+
+int hbitmap_granularity(const HBitmap *hb)
+{
+    return hb->granularity;
+}
+
+uint64_t hbitmap_count(const HBitmap *hb)
+{
+    return hb->count << hb->granularity;
+}
+
+/* Count the number of set bits between start and end, not accounting for
+ * the granularity.  Also an example of how to use hbitmap_iter_next_word.
+ */
+static uint64_t hb_count_between(HBitmap *hb, uint64_t start, uint64_t last)
+{
+    HBitmapIter hbi;
+    uint64_t count = 0;
+    uint64_t end = last + 1;
+    unsigned long cur;
+    size_t pos;
+
+    hbitmap_iter_init(&hbi, hb, start << hb->granularity);
+    for (;;) {
+        pos = hbitmap_iter_next_word(&hbi, &cur);
+        if (pos >= (end >> BITS_PER_LEVEL)) {
+            break;
+        }
+        count += popcountl(cur);
+    }
+
+    if (pos == (end >> BITS_PER_LEVEL)) {
+        /* Drop bits representing the END-th and subsequent items.  */
+        int bit = end & (BITS_PER_LONG - 1);
+        cur &= (1UL << bit) - 1;
+        count += popcountl(cur);
+    }
+
+    return count;
+}
+
+/* Setting starts at the last layer and propagates up if an element
+ * changes from zero to non-zero.
+ */
+static inline bool hb_set_elem(unsigned long *elem, uint64_t start, uint64_t last)
+{
+    unsigned long mask;
+    bool changed;
+
+    assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL));
+    assert(start <= last);
+
+    mask = 2UL << (last & (BITS_PER_LONG - 1));
+    mask -= 1UL << (start & (BITS_PER_LONG - 1));
+    changed = (*elem == 0);
+    *elem |= mask;
+    return changed;
+}
+
+/* The recursive workhorse (the depth is limited to HBITMAP_LEVELS)... */
+static void hb_set_between(HBitmap *hb, int level, uint64_t start, uint64_t last)
+{
+    size_t pos = start >> BITS_PER_LEVEL;
+    size_t lastpos = last >> BITS_PER_LEVEL;
+    bool changed = false;
+    size_t i;
+
+    i = pos;
+    if (i < lastpos) {
+        uint64_t next = (start | (BITS_PER_LONG - 1)) + 1;
+        changed |= hb_set_elem(&hb->levels[level][i], start, next - 1);
+        for (;;) {
+            start = next;
+            next += BITS_PER_LONG;
+            if (++i == lastpos) {
+                break;
+            }
+            changed |= (hb->levels[level][i] == 0);
+            hb->levels[level][i] = ~0UL;
+        }
+    }
+    changed |= hb_set_elem(&hb->levels[level][i], start, last);
+
+    /* If there was any change in this layer, we may have to update
+     * the one above.
+     */
+    if (level > 0 && changed) {
+        hb_set_between(hb, level - 1, pos, lastpos);
+    }
+}
+
+void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t count)
+{
+    /* Compute range in the last layer.  */
+    uint64_t last = start + count - 1;
+
+    trace_hbitmap_set(hb, start, count,
+                      start >> hb->granularity, last >> hb->granularity);
+
+    start >>= hb->granularity;
+    last >>= hb->granularity;
+    count = last - start + 1;
+
+    hb->count += count - hb_count_between(hb, start, last);
+    hb_set_between(hb, HBITMAP_LEVELS - 1, start, last);
+}
+
+/* Resetting works the other way round: propagate up if the new
+ * value is zero.
+ */
+static inline bool hb_reset_elem(unsigned long *elem, uint64_t start, uint64_t last)
+{
+    unsigned long mask;
+    bool blanked;
+
+    assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL));
+    assert(start <= last);
+
+    mask = 2UL << (last & (BITS_PER_LONG - 1));
+    mask -= 1UL << (start & (BITS_PER_LONG - 1));
+    blanked = *elem != 0 && ((*elem & ~mask) == 0);
+    *elem &= ~mask;
+    return blanked;
+}
+
+/* The recursive workhorse (the depth is limited to HBITMAP_LEVELS)... */
+static void hb_reset_between(HBitmap *hb, int level, uint64_t start, uint64_t last)
+{
+    size_t pos = start >> BITS_PER_LEVEL;
+    size_t lastpos = last >> BITS_PER_LEVEL;
+    bool changed = false;
+    size_t i;
+
+    i = pos;
+    if (i < lastpos) {
+        uint64_t next = (start | (BITS_PER_LONG - 1)) + 1;
+
+        /* Here we need a more complex test than when setting bits.  Even if
+         * something was changed, we must not blank bits in the upper level
+         * unless the lower-level word became entirely zero.  So, remove pos
+         * from the upper-level range if bits remain set.
+         */
+        if (hb_reset_elem(&hb->levels[level][i], start, next - 1)) {
+            changed = true;
+        } else {
+            pos++;
+        }
+
+        for (;;) {
+            start = next;
+            next += BITS_PER_LONG;
+            if (++i == lastpos) {
+                break;
+            }
+            changed |= (hb->levels[level][i] != 0);
+            hb->levels[level][i] = 0UL;
+        }
+    }
+
+    /* Same as above, this time for lastpos.  */
+    if (hb_reset_elem(&hb->levels[level][i], start, last)) {
+        changed = true;
+    } else {
+        lastpos--;
+    }
+
+    if (level > 0 && changed) {
+        hb_reset_between(hb, level - 1, pos, lastpos);
+    }
+}
+
+void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count)
+{
+    /* Compute range in the last layer.  */
+    uint64_t last = start + count - 1;
+
+    trace_hbitmap_reset(hb, start, count,
+                        start >> hb->granularity, last >> hb->granularity);
+
+    start >>= hb->granularity;
+    last >>= hb->granularity;
+
+    hb->count -= hb_count_between(hb, start, last);
+    hb_reset_between(hb, HBITMAP_LEVELS - 1, start, last);
+}
+
+bool hbitmap_get(const HBitmap *hb, uint64_t item)
+{
+    /* Compute position and bit in the last layer.  */
+    uint64_t pos = item >> hb->granularity;
+    unsigned long bit = 1UL << (pos & (BITS_PER_LONG - 1));
+
+    return (hb->levels[HBITMAP_LEVELS - 1][pos >> BITS_PER_LEVEL] & bit) != 0;
+}
+
+void hbitmap_free(HBitmap *hb)
+{
+    unsigned i;
+    for (i = HBITMAP_LEVELS; i-- > 0; ) {
+        g_free(hb->levels[i]);
+    }
+    g_free(hb);
+}
+
+HBitmap *hbitmap_alloc(uint64_t size, int granularity)
+{
+    HBitmap *hb = g_malloc0(sizeof (struct HBitmap));
+    unsigned i;
+
+    assert(granularity >= 0 && granularity < 64);
+    size = (size + (1ULL << granularity) - 1) >> granularity;
+    assert(size <= ((uint64_t)1 << HBITMAP_LOG_MAX_SIZE));
+
+    hb->size = size;
+    hb->granularity = granularity;
+    for (i = HBITMAP_LEVELS; i-- > 0; ) {
+        size = MAX((size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
+        hb->levels[i] = g_malloc0(size * sizeof(unsigned long));
+    }
+
+    /* We necessarily have free bits in level 0 due to the definition
+     * of HBITMAP_LEVELS, so use one for a sentinel.  This speeds up
+     * hbitmap_iter_skip_words.
+     */
+    assert(size == 1);
+    hb->levels[0][0] |= 1UL << (BITS_PER_LONG - 1);
+    return hb;
+}
diff --git a/contrib/qemu/util/hexdump.c b/contrib/qemu/util/hexdump.c
new file mode 100644
index 000000000..969b3406c
--- /dev/null
+++ b/contrib/qemu/util/hexdump.c
@@ -0,0 +1,37 @@
+/*
+ * Helper to hexdump a buffer
+ *
+ * Copyright (c) 2013 Red Hat, Inc.
+ * Copyright (c) 2013 Gerd Hoffmann <kraxel@redhat.com>
+ * Copyright (c) 2013 Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ * Copyright (c) 2013 Xilinx, Inc
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu-common.h"
+
+void qemu_hexdump(const char *buf, FILE *fp, const char *prefix, size_t size)
+{
+    unsigned int b;
+
+    for (b = 0; b < size; b++) {
+        if ((b % 16) == 0) {
+            fprintf(fp, "%s: %04x:", prefix, b);
+        }
+        if ((b % 4) == 0) {
+            fprintf(fp, " ");
+        }
+        fprintf(fp, " %02x", (unsigned char)buf[b]);
+        if ((b % 16) == 15) {
+            fprintf(fp, "\n");
+        }
+    }
+    if ((b % 16) != 0) {
+        fprintf(fp, "\n");
+    }
+}
diff --git a/contrib/qemu/util/iov.c b/contrib/qemu/util/iov.c
new file mode 100644
index 000000000..cc6e837c8
--- /dev/null
+++ b/contrib/qemu/util/iov.c
@@ -0,0 +1,426 @@
+/*
+ * Helpers for getting linearized buffers from iov / filling buffers into iovs
+ *
+ * Copyright IBM, Corp. 2007, 2008
+ * Copyright (C) 2010 Red Hat, Inc.
+ *
+ * Author(s):
+ *  Anthony Liguori <aliguori@us.ibm.com>
+ *  Amit Shah <amit.shah@redhat.com>
+ *  Michael Tokarev <mjt@tls.msk.ru>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu/iov.h"
+
+#ifdef _WIN32
+# include <windows.h>
+# include <winsock2.h>
+#else
+# include <sys/types.h>
+# include <sys/socket.h>
+#endif
+
+size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
+                    size_t offset, const void *buf, size_t bytes)
+{
+    size_t done;
+    unsigned int i;
+    for (i = 0, done = 0; (offset || done < bytes) && i < iov_cnt; i++) {
+        if (offset < iov[i].iov_len) {
+            size_t len = MIN(iov[i].iov_len - offset, bytes - done);
+            memcpy(iov[i].iov_base + offset, buf + done, len);
+            done += len;
+            offset = 0;
+        } else {
+            offset -= iov[i].iov_len;
+        }
+    }
+    assert(offset == 0);
+    return done;
+}
+
+size_t iov_to_buf(const struct iovec *iov, const unsigned int iov_cnt,
+                  size_t offset, void *buf, size_t bytes)
+{
+    size_t done;
+    unsigned int i;
+    for (i = 0, done = 0; (offset || done < bytes) && i < iov_cnt; i++) {
+        if (offset < iov[i].iov_len) {
+            size_t len = MIN(iov[i].iov_len - offset, bytes - done);
+            memcpy(buf + done, iov[i].iov_base + offset, len);
+            done += len;
+            offset = 0;
+        } else {
+            offset -= iov[i].iov_len;
+        }
+    }
+    assert(offset == 0);
+    return done;
+}
+
+size_t iov_memset(const struct iovec *iov, const unsigned int iov_cnt,
+                  size_t offset, int fillc, size_t bytes)
+{
+    size_t done;
+    unsigned int i;
+    for (i = 0, done = 0; (offset || done < bytes) && i < iov_cnt; i++) {
+        if (offset < iov[i].iov_len) {
+            size_t len = MIN(iov[i].iov_len - offset, bytes - done);
+            memset(iov[i].iov_base + offset, fillc, len);
+            done += len;
+            offset = 0;
+        } else {
+            offset -= iov[i].iov_len;
+        }
+    }
+    assert(offset == 0);
+    return done;
+}
+
+size_t iov_size(const struct iovec *iov, const unsigned int iov_cnt)
+{
+    size_t len;
+    unsigned int i;
+
+    len = 0;
+    for (i = 0; i < iov_cnt; i++) {
+        len += iov[i].iov_len;
+    }
+    return len;
+}
+
+/* helper function for iov_send_recv() */
+static ssize_t
+do_send_recv(int sockfd, struct iovec *iov, unsigned iov_cnt, bool do_send)
+{
+#ifdef CONFIG_POSIX
+    ssize_t ret;
+    struct msghdr msg;
+    memset(&msg, 0, sizeof(msg));
+    msg.msg_iov = iov;
+    msg.msg_iovlen = iov_cnt;
+    do {
+        ret = do_send
+            ? sendmsg(sockfd, &msg, 0)
+            : recvmsg(sockfd, &msg, 0);
+    } while (ret < 0 && errno == EINTR);
+    return ret;
+#else
+    /* else send piece-by-piece */
+    /*XXX Note: windows has WSASend() and WSARecv() */
+    unsigned i = 0;
+    ssize_t ret = 0;
+    while (i < iov_cnt) {
+        ssize_t r = do_send
+            ? send(sockfd, iov[i].iov_base, iov[i].iov_len, 0)
+            : recv(sockfd, iov[i].iov_base, iov[i].iov_len, 0);
+        if (r > 0) {
+            ret += r;
+        } else if (!r) {
+            break;
+        } else if (errno == EINTR) {
+            continue;
+        } else {
+            /* else it is some "other" error,
+             * only return if there was no data processed. */
+            if (ret == 0) {
+                ret = -1;
+            }
+            break;
+        }
+        i++;
+    }
+    return ret;
+#endif
+}
+
+ssize_t iov_send_recv(int sockfd, struct iovec *iov, unsigned iov_cnt,
+                      size_t offset, size_t bytes,
+                      bool do_send)
+{
+    ssize_t total = 0;
+    ssize_t ret;
+    size_t orig_len, tail;
+    unsigned niov;
+
+    while (bytes > 0) {
+        /* Find the start position, skipping `offset' bytes:
+         * first, skip all full-sized vector elements, */
+        for (niov = 0; niov < iov_cnt && offset >= iov[niov].iov_len; ++niov) {
+            offset -= iov[niov].iov_len;
+        }
+
+        /* niov == iov_cnt would only be valid if bytes == 0, which
+         * we already ruled out in the loop condition.  */
+        assert(niov < iov_cnt);
+        iov += niov;
+        iov_cnt -= niov;
+
+        if (offset) {
+            /* second, skip `offset' bytes from the (now) first element,
+             * undo it on exit */
+            iov[0].iov_base += offset;
+            iov[0].iov_len -= offset;
+        }
+        /* Find the end position skipping `bytes' bytes: */
+        /* first, skip all full-sized elements */
+        tail = bytes;
+        for (niov = 0; niov < iov_cnt && iov[niov].iov_len <= tail; ++niov) {
+            tail -= iov[niov].iov_len;
+        }
+        if (tail) {
+            /* second, fixup the last element, and remember the original
+             * length */
+            assert(niov < iov_cnt);
+            assert(iov[niov].iov_len > tail);
+            orig_len = iov[niov].iov_len;
+            iov[niov++].iov_len = tail;
+        }
+
+        ret = do_send_recv(sockfd, iov, niov, do_send);
+
+        /* Undo the changes above before checking for errors */
+        if (tail) {
+            iov[niov-1].iov_len = orig_len;
+        }
+        if (offset) {
+            iov[0].iov_base -= offset;
+            iov[0].iov_len += offset;
+        }
+
+        if (ret < 0) {
+            assert(errno != EINTR);
+            if (errno == EAGAIN && total > 0) {
+                return total;
+            }
+            return -1;
+        }
+
+        /* Prepare for the next iteration */
+        offset += ret;
+        total += ret;
+        bytes -= ret;
+    }
+
+    return total;
+}
+
+
+void iov_hexdump(const struct iovec *iov, const unsigned int iov_cnt,
+                 FILE *fp, const char *prefix, size_t limit)
+{
+    int v;
+    size_t size = 0;
+    char *buf;
+
+    for (v = 0; v < iov_cnt; v++) {
+        size += iov[v].iov_len;
+    }
+    size = size > limit ? limit : size;
+    buf = g_malloc(size);
+    iov_to_buf(iov, iov_cnt, 0, buf, size);
+    qemu_hexdump(buf, fp, prefix, size);
+    g_free(buf);
+}
+
+unsigned iov_copy(struct iovec *dst_iov, unsigned int dst_iov_cnt,
+                 const struct iovec *iov, unsigned int iov_cnt,
+                 size_t offset, size_t bytes)
+{
+    size_t len;
+    unsigned int i, j;
+    for (i = 0, j = 0; i < iov_cnt && j < dst_iov_cnt && bytes; i++) {
+        if (offset >= iov[i].iov_len) {
+            offset -= iov[i].iov_len;
+            continue;
+        }
+        len = MIN(bytes, iov[i].iov_len - offset);
+
+        dst_iov[j].iov_base = iov[i].iov_base + offset;
+        dst_iov[j].iov_len = len;
+        j++;
+        bytes -= len;
+        offset = 0;
+    }
+    assert(offset == 0);
+    return j;
+}
+
+/* io vectors */
+
+void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint)
+{
+    qiov->iov = g_malloc(alloc_hint * sizeof(struct iovec));
+    qiov->niov = 0;
+    qiov->nalloc = alloc_hint;
+    qiov->size = 0;
+}
+
+void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov)
+{
+    int i;
+
+    qiov->iov = iov;
+    qiov->niov = niov;
+    qiov->nalloc = -1;
+    qiov->size = 0;
+    for (i = 0; i < niov; i++)
+        qiov->size += iov[i].iov_len;
+}
+
+void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len)
+{
+    assert(qiov->nalloc != -1);
+
+    if (qiov->niov == qiov->nalloc) {
+        qiov->nalloc = 2 * qiov->nalloc + 1;
+        qiov->iov = g_realloc(qiov->iov, qiov->nalloc * sizeof(struct iovec));
+    }
+    qiov->iov[qiov->niov].iov_base = base;
+    qiov->iov[qiov->niov].iov_len = len;
+    qiov->size += len;
+    ++qiov->niov;
+}
+
+/*
+ * Concatenates (partial) iovecs from src_iov to the end of dst.
+ * It starts copying after skipping `soffset' bytes at the
+ * beginning of src and adds individual vectors from src to
+ * dst copies up to `sbytes' bytes total, or up to the end
+ * of src_iov if it comes first.  This way, it is okay to specify
+ * very large value for `sbytes' to indicate "up to the end
+ * of src".
+ * Only vector pointers are processed, not the actual data buffers.
+ */
+void qemu_iovec_concat_iov(QEMUIOVector *dst,
+                           struct iovec *src_iov, unsigned int src_cnt,
+                           size_t soffset, size_t sbytes)
+{
+    int i;
+    size_t done;
+
+    if (!sbytes) {
+        return;
+    }
+    assert(dst->nalloc != -1);
+    for (i = 0, done = 0; done < sbytes && i < src_cnt; i++) {
+        if (soffset < src_iov[i].iov_len) {
+            size_t len = MIN(src_iov[i].iov_len - soffset, sbytes - done);
+            qemu_iovec_add(dst, src_iov[i].iov_base + soffset, len);
+            done += len;
+            soffset = 0;
+        } else {
+            soffset -= src_iov[i].iov_len;
+        }
+    }
+    assert(soffset == 0); /* offset beyond end of src */
+}
+
+/*
+ * Concatenates (partial) iovecs from src to the end of dst.
+ * It starts copying after skipping `soffset' bytes at the
+ * beginning of src and adds individual vectors from src to
+ * dst copies up to `sbytes' bytes total, or up to the end
+ * of src if it comes first.  This way, it is okay to specify
+ * very large value for `sbytes' to indicate "up to the end
+ * of src".
+ * Only vector pointers are processed, not the actual data buffers.
+ */
+void qemu_iovec_concat(QEMUIOVector *dst,
+                       QEMUIOVector *src, size_t soffset, size_t sbytes)
+{
+    qemu_iovec_concat_iov(dst, src->iov, src->niov, soffset, sbytes);
+}
+
+void qemu_iovec_destroy(QEMUIOVector *qiov)
+{
+    assert(qiov->nalloc != -1);
+
+    qemu_iovec_reset(qiov);
+    g_free(qiov->iov);
+    qiov->nalloc = 0;
+    qiov->iov = NULL;
+}
+
+void qemu_iovec_reset(QEMUIOVector *qiov)
+{
+    assert(qiov->nalloc != -1);
+
+    qiov->niov = 0;
+    qiov->size = 0;
+}
+
+size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
+                         void *buf, size_t bytes)
+{
+    return iov_to_buf(qiov->iov, qiov->niov, offset, buf, bytes);
+}
+
+size_t qemu_iovec_from_buf(QEMUIOVector *qiov, size_t offset,
+                           const void *buf, size_t bytes)
+{
+    return iov_from_buf(qiov->iov, qiov->niov, offset, buf, bytes);
+}
+
+size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
+                         int fillc, size_t bytes)
+{
+    return iov_memset(qiov->iov, qiov->niov, offset, fillc, bytes);
+}
+
+size_t iov_discard_front(struct iovec **iov, unsigned int *iov_cnt,
+                         size_t bytes)
+{
+    size_t total = 0;
+    struct iovec *cur;
+
+    for (cur = *iov; *iov_cnt > 0; cur++) {
+        if (cur->iov_len > bytes) {
+            cur->iov_base += bytes;
+            cur->iov_len -= bytes;
+            total += bytes;
+            break;
+        }
+
+        bytes -= cur->iov_len;
+        total += cur->iov_len;
+        *iov_cnt -= 1;
+    }
+
+    *iov = cur;
+    return total;
+}
+
+size_t iov_discard_back(struct iovec *iov, unsigned int *iov_cnt,
+                        size_t bytes)
+{
+    size_t total = 0;
+    struct iovec *cur;
+
+    if (*iov_cnt == 0) {
+        return 0;
+    }
+
+    cur = iov + (*iov_cnt - 1);
+
+    while (*iov_cnt > 0) {
+        if (cur->iov_len > bytes) {
+            cur->iov_len -= bytes;
+            total += bytes;
+            break;
+        }
+
+        bytes -= cur->iov_len;
+        total += cur->iov_len;
+        cur--;
+        *iov_cnt -= 1;
+    }
+
+    return total;
+}
diff --git a/contrib/qemu/util/module.c b/contrib/qemu/util/module.c
new file mode 100644
index 000000000..7acc33d07
--- /dev/null
+++ b/contrib/qemu/util/module.c
@@ -0,0 +1,81 @@
+/*
+ * QEMU Module Infrastructure
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu-common.h"
+#include "qemu/queue.h"
+#include "qemu/module.h"
+
+typedef struct ModuleEntry
+{
+    void (*init)(void);
+    QTAILQ_ENTRY(ModuleEntry) node;
+} ModuleEntry;
+
+typedef QTAILQ_HEAD(, ModuleEntry) ModuleTypeList;
+
+static ModuleTypeList init_type_list[MODULE_INIT_MAX];
+
+static void init_types(void)
+{
+    static int inited;
+    int i;
+
+    if (inited) {
+        return;
+    }
+
+    for (i = 0; i < MODULE_INIT_MAX; i++) {
+        QTAILQ_INIT(&init_type_list[i]);
+    }
+
+    inited = 1;
+}
+
+
+static ModuleTypeList *find_type(module_init_type type)
+{
+    ModuleTypeList *l;
+
+    init_types();
+
+    l = &init_type_list[type];
+
+    return l;
+}
+
+void register_module_init(void (*fn)(void), module_init_type type)
+{
+    ModuleEntry *e;
+    ModuleTypeList *l;
+
+    e = g_malloc0(sizeof(*e));
+    e->init = fn;
+
+    l = find_type(type);
+
+    QTAILQ_INSERT_TAIL(l, e, node);
+}
+
+void module_call_init(module_init_type type)
+{
+    ModuleTypeList *l;
+    ModuleEntry *e;
+
+    l = find_type(type);
+
+    QTAILQ_FOREACH(e, l, node) {
+        e->init();
+    }
+}
diff --git a/contrib/qemu/util/oslib-posix.c b/contrib/qemu/util/oslib-posix.c
new file mode 100644
index 000000000..3dc8b1b07
--- /dev/null
+++ b/contrib/qemu/util/oslib-posix.c
@@ -0,0 +1,243 @@
+/*
+ * os-posix-lib.c
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2010 Red Hat, Inc.
+ *
+ * QEMU library functions on POSIX which are shared between QEMU and
+ * the QEMU tools.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/* The following block of code temporarily renames the daemon() function so the
+   compiler does not see the warning associated with it in stdlib.h on OSX */
+#ifdef __APPLE__
+#define daemon qemu_fake_daemon_function
+#include <stdlib.h>
+#undef daemon
+extern int daemon(int, int);
+#endif
+
+#if defined(__linux__) && (defined(__x86_64__) || defined(__arm__))
+   /* Use 2 MiB alignment so transparent hugepages can be used by KVM.
+      Valgrind does not support alignments larger than 1 MiB,
+      therefore we need special code which handles running on Valgrind. */
+#  define QEMU_VMALLOC_ALIGN (512 * 4096)
+#elif defined(__linux__) && defined(__s390x__)
+   /* Use 1 MiB (segment size) alignment so gmap can be used by KVM. */
+#  define QEMU_VMALLOC_ALIGN (256 * 4096)
+#else
+#  define QEMU_VMALLOC_ALIGN getpagesize()
+#endif
+
+#include <glib/gprintf.h>
+
+#include "config-host.h"
+#include "sysemu/sysemu.h"
+#include "trace.h"
+#include "qemu/sockets.h"
+#include <sys/mman.h>
+
+#ifdef CONFIG_LINUX
+#include <sys/syscall.h>
+#endif
+
+int qemu_get_thread_id(void)
+{
+#if defined(__linux__)
+    return syscall(SYS_gettid);
+#else
+    return getpid();
+#endif
+}
+
+int qemu_daemon(int nochdir, int noclose)
+{
+    return daemon(nochdir, noclose);
+}
+
+void *qemu_oom_check(void *ptr)
+{
+    if (ptr == NULL) {
+        fprintf(stderr, "Failed to allocate memory: %s\n", strerror(errno));
+        abort();
+    }
+    return ptr;
+}
+
+void *qemu_memalign(size_t alignment, size_t size)
+{
+    void *ptr;
+#if defined(_POSIX_C_SOURCE) && !defined(__sun__)
+    int ret;
+    ret = posix_memalign(&ptr, alignment, size);
+    if (ret != 0) {
+        fprintf(stderr, "Failed to allocate %zu B: %s\n",
+                size, strerror(ret));
+        abort();
+    }
+#elif defined(CONFIG_BSD)
+    ptr = qemu_oom_check(valloc(size));
+#else
+    ptr = qemu_oom_check(memalign(alignment, size));
+#endif
+    trace_qemu_memalign(alignment, size, ptr);
+    return ptr;
+}
+
+/* alloc shared memory pages */
+void *qemu_anon_ram_alloc(size_t size)
+{
+    size_t align = QEMU_VMALLOC_ALIGN;
+    size_t total = size + align - getpagesize();
+    void *ptr = mmap(0, total, PROT_READ | PROT_WRITE,
+                     MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+    size_t offset = QEMU_ALIGN_UP((uintptr_t)ptr, align) - (uintptr_t)ptr;
+
+    if (ptr == MAP_FAILED) {
+        fprintf(stderr, "Failed to allocate %zu B: %s\n",
+                size, strerror(errno));
+        abort();
+    }
+
+    ptr += offset;
+    total -= offset;
+
+    if (offset > 0) {
+        munmap(ptr - offset, offset);
+    }
+    if (total > size) {
+        munmap(ptr + size, total - size);
+    }
+
+    trace_qemu_anon_ram_alloc(size, ptr);
+    return ptr;
+}
+
+void qemu_vfree(void *ptr)
+{
+    trace_qemu_vfree(ptr);
+    free(ptr);
+}
+
+void qemu_anon_ram_free(void *ptr, size_t size)
+{
+    trace_qemu_anon_ram_free(ptr, size);
+    if (ptr) {
+        munmap(ptr, size);
+    }
+}
+
+void qemu_set_block(int fd)
+{
+    int f;
+    f = fcntl(fd, F_GETFL);
+    fcntl(fd, F_SETFL, f & ~O_NONBLOCK);
+}
+
+void qemu_set_nonblock(int fd)
+{
+    int f;
+    f = fcntl(fd, F_GETFL);
+    fcntl(fd, F_SETFL, f | O_NONBLOCK);
+}
+
+void qemu_set_cloexec(int fd)
+{
+    int f;
+    f = fcntl(fd, F_GETFD);
+    fcntl(fd, F_SETFD, f | FD_CLOEXEC);
+}
+
+/*
+ * Creates a pipe with FD_CLOEXEC set on both file descriptors
+ */
+int qemu_pipe(int pipefd[2])
+{
+    int ret;
+
+#ifdef CONFIG_PIPE2
+    ret = pipe2(pipefd, O_CLOEXEC);
+    if (ret != -1 || errno != ENOSYS) {
+        return ret;
+    }
+#endif
+    ret = pipe(pipefd);
+    if (ret == 0) {
+        qemu_set_cloexec(pipefd[0]);
+        qemu_set_cloexec(pipefd[1]);
+    }
+
+    return ret;
+}
+
+int qemu_utimens(const char *path, const struct timespec *times)
+{
+    struct timeval tv[2], tv_now;
+    struct stat st;
+    int i;
+#ifdef CONFIG_UTIMENSAT
+    int ret;
+
+    ret = utimensat(AT_FDCWD, path, times, AT_SYMLINK_NOFOLLOW);
+    if (ret != -1 || errno != ENOSYS) {
+        return ret;
+    }
+#endif
+    /* Fallback: use utimes() instead of utimensat() */
+
+    /* happy if special cases */
+    if (times[0].tv_nsec == UTIME_OMIT && times[1].tv_nsec == UTIME_OMIT) {
+        return 0;
+    }
+    if (times[0].tv_nsec == UTIME_NOW && times[1].tv_nsec == UTIME_NOW) {
+        return utimes(path, NULL);
+    }
+
+    /* prepare for hard cases */
+    if (times[0].tv_nsec == UTIME_NOW || times[1].tv_nsec == UTIME_NOW) {
+        gettimeofday(&tv_now, NULL);
+    }
+    if (times[0].tv_nsec == UTIME_OMIT || times[1].tv_nsec == UTIME_OMIT) {
+        stat(path, &st);
+    }
+
+    for (i = 0; i < 2; i++) {
+        if (times[i].tv_nsec == UTIME_NOW) {
+            tv[i].tv_sec = tv_now.tv_sec;
+            tv[i].tv_usec = tv_now.tv_usec;
+        } else if (times[i].tv_nsec == UTIME_OMIT) {
+            tv[i].tv_sec = (i == 0) ? st.st_atime : st.st_mtime;
+            tv[i].tv_usec = 0;
+        } else {
+            tv[i].tv_sec = times[i].tv_sec;
+            tv[i].tv_usec = times[i].tv_nsec / 1000;
+        }
+    }
+
+    return utimes(path, &tv[0]);
+}
+
+char *
+qemu_get_local_state_pathname(const char *relative_pathname)
+{
+    return g_strdup_printf("%s/%s", CONFIG_QEMU_LOCALSTATEDIR,
+                           relative_pathname);
+}
diff --git a/contrib/qemu/util/qemu-error.c b/contrib/qemu/util/qemu-error.c
new file mode 100644
index 000000000..fec02c607
--- /dev/null
+++ b/contrib/qemu/util/qemu-error.c
@@ -0,0 +1,225 @@
+/*
+ * Error reporting
+ *
+ * Copyright (C) 2010 Red Hat Inc.
+ *
+ * Authors:
+ *  Markus Armbruster <armbru@redhat.com>,
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include <stdio.h>
+#include "monitor/monitor.h"
+
+/*
+ * Print to current monitor if we have one, else to stderr.
+ * TODO should return int, so callers can calculate width, but that
+ * requires surgery to monitor_vprintf().  Left for another day.
+ */
+void error_vprintf(const char *fmt, va_list ap)
+{
+    if (cur_mon) {
+        monitor_vprintf(cur_mon, fmt, ap);
+    } else {
+        vfprintf(stderr, fmt, ap);
+    }
+}
+
+/*
+ * Print to current monitor if we have one, else to stderr.
+ * TODO just like error_vprintf()
+ */
+void error_printf(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_vprintf(fmt, ap);
+    va_end(ap);
+}
+
+void error_printf_unless_qmp(const char *fmt, ...)
+{
+    va_list ap;
+
+    if (!monitor_cur_is_qmp()) {
+        va_start(ap, fmt);
+        error_vprintf(fmt, ap);
+        va_end(ap);
+    }
+}
+
+static Location std_loc = {
+    .kind = LOC_NONE
+};
+static Location *cur_loc = &std_loc;
+
+/*
+ * Push location saved in LOC onto the location stack, return it.
+ * The top of that stack is the current location.
+ * Needs a matching loc_pop().
+ */
+Location *loc_push_restore(Location *loc)
+{
+    assert(!loc->prev);
+    loc->prev = cur_loc;
+    cur_loc = loc;
+    return loc;
+}
+
+/*
+ * Initialize *LOC to "nowhere", push it onto the location stack.
+ * The top of that stack is the current location.
+ * Needs a matching loc_pop().
+ * Return LOC.
+ */
+Location *loc_push_none(Location *loc)
+{
+    loc->kind = LOC_NONE;
+    loc->prev = NULL;
+    return loc_push_restore(loc);
+}
+
+/*
+ * Pop the location stack.
+ * LOC must be the current location, i.e. the top of the stack.
+ */
+Location *loc_pop(Location *loc)
+{
+    assert(cur_loc == loc && loc->prev);
+    cur_loc = loc->prev;
+    loc->prev = NULL;
+    return loc;
+}
+
+/*
+ * Save the current location in LOC, return LOC.
+ */
+Location *loc_save(Location *loc)
+{
+    *loc = *cur_loc;
+    loc->prev = NULL;
+    return loc;
+}
+
+/*
+ * Change the current location to the one saved in LOC.
+ */
+void loc_restore(Location *loc)
+{
+    Location *prev = cur_loc->prev;
+    assert(!loc->prev);
+    *cur_loc = *loc;
+    cur_loc->prev = prev;
+}
+
+/*
+ * Change the current location to "nowhere in particular".
+ */
+void loc_set_none(void)
+{
+    cur_loc->kind = LOC_NONE;
+}
+
+/*
+ * Change the current location to argument ARGV[IDX..IDX+CNT-1].
+ */
+void loc_set_cmdline(char **argv, int idx, int cnt)
+{
+    cur_loc->kind = LOC_CMDLINE;
+    cur_loc->num = cnt;
+    cur_loc->ptr = argv + idx;
+}
+
+/*
+ * Change the current location to file FNAME, line LNO.
+ */
+void loc_set_file(const char *fname, int lno)
+{
+    assert (fname || cur_loc->kind == LOC_FILE);
+    cur_loc->kind = LOC_FILE;
+    cur_loc->num = lno;
+    if (fname) {
+        cur_loc->ptr = fname;
+    }
+}
+
+static const char *progname;
+
+/*
+ * Set the program name for error_print_loc().
+ */
+void error_set_progname(const char *argv0)
+{
+    const char *p = strrchr(argv0, '/');
+    progname = p ? p + 1 : argv0;
+}
+
+const char *error_get_progname(void)
+{
+    return progname;
+}
+
+/*
+ * Print current location to current monitor if we have one, else to stderr.
+ */
+void error_print_loc(void)
+{
+    const char *sep = "";
+    int i;
+    const char *const *argp;
+
+    if (!cur_mon && progname) {
+        fprintf(stderr, "%s:", progname);
+        sep = " ";
+    }
+    switch (cur_loc->kind) {
+    case LOC_CMDLINE:
+        argp = cur_loc->ptr;
+        for (i = 0; i < cur_loc->num; i++) {
+            error_printf("%s%s", sep, argp[i]);
+            sep = " ";
+        }
+        error_printf(": ");
+        break;
+    case LOC_FILE:
+        error_printf("%s:", (const char *)cur_loc->ptr);
+        if (cur_loc->num) {
+            error_printf("%d:", cur_loc->num);
+        }
+        error_printf(" ");
+        break;
+    default:
+        error_printf("%s", sep);
+    }
+}
+
+bool enable_timestamp_msg;
+/*
+ * Print an error message to current monitor if we have one, else to stderr.
+ * Format arguments like sprintf().  The result should not contain
+ * newlines.
+ * Prepend the current location and append a newline.
+ * It's wrong to call this in a QMP monitor.  Use qerror_report() there.
+ */
+void error_report(const char *fmt, ...)
+{
+    va_list ap;
+    GTimeVal tv;
+    gchar *timestr;
+
+    if (enable_timestamp_msg) {
+        g_get_current_time(&tv);
+        timestr = g_time_val_to_iso8601(&tv);
+        error_printf("%s ", timestr);
+        g_free(timestr);
+    }
+
+    error_print_loc();
+    va_start(ap, fmt);
+    error_vprintf(fmt, ap);
+    va_end(ap);
+    error_printf("\n");
+}
diff --git a/contrib/qemu/util/qemu-option.c b/contrib/qemu/util/qemu-option.c
new file mode 100644
index 000000000..e0ef426da
--- /dev/null
+++ b/contrib/qemu/util/qemu-option.c
@@ -0,0 +1,1126 @@
+/*
+ * Commandline option parsing functions
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2009 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "qapi/qmp/types.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/option_int.h"
+
+/*
+ * Extracts the name of an option from the parameter string (p points at the
+ * first byte of the option name)
+ *
+ * The option name is delimited by delim (usually , or =) or the string end
+ * and is copied into buf. If the option name is longer than buf_size, it is
+ * truncated. buf is always zero terminated.
+ *
+ * The return value is the position of the delimiter/zero byte after the option
+ * name in p.
+ */
+const char *get_opt_name(char *buf, int buf_size, const char *p, char delim)
+{
+    char *q;
+
+    q = buf;
+    while (*p != '\0' && *p != delim) {
+        if (q && (q - buf) < buf_size - 1)
+            *q++ = *p;
+        p++;
+    }
+    if (q)
+        *q = '\0';
+
+    return p;
+}
+
+/*
+ * Extracts the value of an option from the parameter string p (p points at the
+ * first byte of the option value)
+ *
+ * This function is comparable to get_opt_name with the difference that the
+ * delimiter is fixed to be comma which starts a new option. To specify an
+ * option value that contains commas, double each comma.
+ */
+const char *get_opt_value(char *buf, int buf_size, const char *p)
+{
+    char *q;
+
+    q = buf;
+    while (*p != '\0') {
+        if (*p == ',') {
+            if (*(p + 1) != ',')
+                break;
+            p++;
+        }
+        if (q && (q - buf) < buf_size - 1)
+            *q++ = *p;
+        p++;
+    }
+    if (q)
+        *q = '\0';
+
+    return p;
+}
+
+int get_next_param_value(char *buf, int buf_size,
+                         const char *tag, const char **pstr)
+{
+    const char *p;
+    char option[128];
+
+    p = *pstr;
+    for(;;) {
+        p = get_opt_name(option, sizeof(option), p, '=');
+        if (*p != '=')
+            break;
+        p++;
+        if (!strcmp(tag, option)) {
+            *pstr = get_opt_value(buf, buf_size, p);
+            if (**pstr == ',') {
+                (*pstr)++;
+            }
+            return strlen(buf);
+        } else {
+            p = get_opt_value(NULL, 0, p);
+        }
+        if (*p != ',')
+            break;
+        p++;
+    }
+    return 0;
+}
+
+int get_param_value(char *buf, int buf_size,
+                    const char *tag, const char *str)
+{
+    return get_next_param_value(buf, buf_size, tag, &str);
+}
+
+/*
+ * Searches an option list for an option with the given name
+ */
+QEMUOptionParameter *get_option_parameter(QEMUOptionParameter *list,
+    const char *name)
+{
+    while (list && list->name) {
+        if (!strcmp(list->name, name)) {
+            return list;
+        }
+        list++;
+    }
+
+    return NULL;
+}
+
+static void parse_option_bool(const char *name, const char *value, bool *ret,
+                              Error **errp)
+{
+    if (value != NULL) {
+        if (!strcmp(value, "on")) {
+            *ret = 1;
+        } else if (!strcmp(value, "off")) {
+            *ret = 0;
+        } else {
+            error_set(errp,QERR_INVALID_PARAMETER_VALUE, name, "'on' or 'off'");
+        }
+    } else {
+        *ret = 1;
+    }
+}
+
+static void parse_option_number(const char *name, const char *value,
+                                uint64_t *ret, Error **errp)
+{
+    char *postfix;
+    uint64_t number;
+
+    if (value != NULL) {
+        number = strtoull(value, &postfix, 0);
+        if (*postfix != '\0') {
+            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name, "a number");
+            return;
+        }
+        *ret = number;
+    } else {
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE, name, "a number");
+    }
+}
+
+static void parse_option_size(const char *name, const char *value,
+                              uint64_t *ret, Error **errp)
+{
+    char *postfix;
+    double sizef;
+
+    if (value != NULL) {
+        sizef = strtod(value, &postfix);
+        switch (*postfix) {
+        case 'T':
+            sizef *= 1024;
+            /* fall through */
+        case 'G':
+            sizef *= 1024;
+            /* fall through */
+        case 'M':
+            sizef *= 1024;
+            /* fall through */
+        case 'K':
+        case 'k':
+            sizef *= 1024;
+            /* fall through */
+        case 'b':
+        case '\0':
+            *ret = (uint64_t) sizef;
+            break;
+        default:
+            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name, "a size");
+#if 0 /* conversion from qerror_report() to error_set() broke this: */
+            error_printf_unless_qmp("You may use k, M, G or T suffixes for "
+                    "kilobytes, megabytes, gigabytes and terabytes.\n");
+#endif
+            return;
+        }
+    } else {
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE, name, "a size");
+    }
+}
+
+/*
+ * Sets the value of a parameter in a given option list. The parsing of the
+ * value depends on the type of option:
+ *
+ * OPT_FLAG (uses value.n):
+ *      If no value is given, the flag is set to 1.
+ *      Otherwise the value must be "on" (set to 1) or "off" (set to 0)
+ *
+ * OPT_STRING (uses value.s):
+ *      value is strdup()ed and assigned as option value
+ *
+ * OPT_SIZE (uses value.n):
+ *      The value is converted to an integer. Suffixes for kilobytes etc. are
+ *      allowed (powers of 1024).
+ *
+ * Returns 0 on succes, -1 in error cases
+ */
+int set_option_parameter(QEMUOptionParameter *list, const char *name,
+    const char *value)
+{
+    bool flag;
+    Error *local_err = NULL;
+
+    // Find a matching parameter
+    list = get_option_parameter(list, name);
+    if (list == NULL) {
+        fprintf(stderr, "Unknown option '%s'\n", name);
+        return -1;
+    }
+
+    // Process parameter
+    switch (list->type) {
+    case OPT_FLAG:
+        parse_option_bool(name, value, &flag, &local_err);
+        if (!error_is_set(&local_err)) {
+            list->value.n = flag;
+        }
+        break;
+
+    case OPT_STRING:
+        if (value != NULL) {
+            list->value.s = g_strdup(value);
+        } else {
+            fprintf(stderr, "Option '%s' needs a parameter\n", name);
+            return -1;
+        }
+        break;
+
+    case OPT_SIZE:
+        parse_option_size(name, value, &list->value.n, &local_err);
+        break;
+
+    default:
+        fprintf(stderr, "Bug: Option '%s' has an unknown type\n", name);
+        return -1;
+    }
+
+    if (error_is_set(&local_err)) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Sets the given parameter to an integer instead of a string.
+ * This function cannot be used to set string options.
+ *
+ * Returns 0 on success, -1 in error cases
+ */
+int set_option_parameter_int(QEMUOptionParameter *list, const char *name,
+    uint64_t value)
+{
+    // Find a matching parameter
+    list = get_option_parameter(list, name);
+    if (list == NULL) {
+        fprintf(stderr, "Unknown option '%s'\n", name);
+        return -1;
+    }
+
+    // Process parameter
+    switch (list->type) {
+    case OPT_FLAG:
+    case OPT_NUMBER:
+    case OPT_SIZE:
+        list->value.n = value;
+        break;
+
+    default:
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Frees a option list. If it contains strings, the strings are freed as well.
+ */
+void free_option_parameters(QEMUOptionParameter *list)
+{
+    QEMUOptionParameter *cur = list;
+
+    while (cur && cur->name) {
+        if (cur->type == OPT_STRING) {
+            g_free(cur->value.s);
+        }
+        cur++;
+    }
+
+    g_free(list);
+}
+
+/*
+ * Count valid options in list
+ */
+static size_t count_option_parameters(QEMUOptionParameter *list)
+{
+    size_t num_options = 0;
+
+    while (list && list->name) {
+        num_options++;
+        list++;
+    }
+
+    return num_options;
+}
+
+/*
+ * Append an option list (list) to an option list (dest).
+ *
+ * If dest is NULL, a new copy of list is created.
+ *
+ * Returns a pointer to the first element of dest (or the newly allocated copy)
+ */
+QEMUOptionParameter *append_option_parameters(QEMUOptionParameter *dest,
+    QEMUOptionParameter *list)
+{
+    size_t num_options, num_dest_options;
+
+    num_options = count_option_parameters(dest);
+    num_dest_options = num_options;
+
+    num_options += count_option_parameters(list);
+
+    dest = g_realloc(dest, (num_options + 1) * sizeof(QEMUOptionParameter));
+    dest[num_dest_options].name = NULL;
+
+    while (list && list->name) {
+        if (get_option_parameter(dest, list->name) == NULL) {
+            dest[num_dest_options++] = *list;
+            dest[num_dest_options].name = NULL;
+        }
+        list++;
+    }
+
+    return dest;
+}
+
+/*
+ * Parses a parameter string (param) into an option list (dest).
+ *
+ * list is the template option list. If dest is NULL, a new copy of list is
+ * created. If list is NULL, this function fails.
+ *
+ * A parameter string consists of one or more parameters, separated by commas.
+ * Each parameter consists of its name and possibly of a value. In the latter
+ * case, the value is delimited by an = character. To specify a value which
+ * contains commas, double each comma so it won't be recognized as the end of
+ * the parameter.
+ *
+ * For more details of the parsing see above.
+ *
+ * Returns a pointer to the first element of dest (or the newly allocated copy)
+ * or NULL in error cases
+ */
+QEMUOptionParameter *parse_option_parameters(const char *param,
+    QEMUOptionParameter *list, QEMUOptionParameter *dest)
+{
+    QEMUOptionParameter *allocated = NULL;
+    char name[256];
+    char value[256];
+    char *param_delim, *value_delim;
+    char next_delim;
+
+    if (list == NULL) {
+        return NULL;
+    }
+
+    if (dest == NULL) {
+        dest = allocated = append_option_parameters(NULL, list);
+    }
+
+    while (*param) {
+
+        // Find parameter name and value in the string
+        param_delim = strchr(param, ',');
+        value_delim = strchr(param, '=');
+
+        if (value_delim && (value_delim < param_delim || !param_delim)) {
+            next_delim = '=';
+        } else {
+            next_delim = ',';
+            value_delim = NULL;
+        }
+
+        param = get_opt_name(name, sizeof(name), param, next_delim);
+        if (value_delim) {
+            param = get_opt_value(value, sizeof(value), param + 1);
+        }
+        if (*param != '\0') {
+            param++;
+        }
+
+        // Set the parameter
+        if (set_option_parameter(dest, name, value_delim ? value : NULL)) {
+            goto fail;
+        }
+    }
+
+    return dest;
+
+fail:
+    // Only free the list if it was newly allocated
+    free_option_parameters(allocated);
+    return NULL;
+}
+
+/*
+ * Prints all options of a list that have a value to stdout
+ */
+void print_option_parameters(QEMUOptionParameter *list)
+{
+    while (list && list->name) {
+        switch (list->type) {
+            case OPT_STRING:
+                 if (list->value.s != NULL) {
+                     printf("%s='%s' ", list->name, list->value.s);
+                 }
+                break;
+            case OPT_FLAG:
+                printf("%s=%s ", list->name, list->value.n ? "on" : "off");
+                break;
+            case OPT_SIZE:
+            case OPT_NUMBER:
+                printf("%s=%" PRId64 " ", list->name, list->value.n);
+                break;
+            default:
+                printf("%s=(unknown type) ", list->name);
+                break;
+        }
+        list++;
+    }
+}
+
+/*
+ * Prints an overview of all available options
+ */
+void print_option_help(QEMUOptionParameter *list)
+{
+    printf("Supported options:\n");
+    while (list && list->name) {
+        printf("%-16s %s\n", list->name,
+            list->help ? list->help : "No description available");
+        list++;
+    }
+}
+
+/* ------------------------------------------------------------------ */
+
+static QemuOpt *qemu_opt_find(QemuOpts *opts, const char *name)
+{
+    QemuOpt *opt;
+
+    QTAILQ_FOREACH_REVERSE(opt, &opts->head, QemuOptHead, next) {
+        if (strcmp(opt->name, name) != 0)
+            continue;
+        return opt;
+    }
+    return NULL;
+}
+
+const char *qemu_opt_get(QemuOpts *opts, const char *name)
+{
+    QemuOpt *opt = qemu_opt_find(opts, name);
+    return opt ? opt->str : NULL;
+}
+
+bool qemu_opt_has_help_opt(QemuOpts *opts)
+{
+    QemuOpt *opt;
+
+    QTAILQ_FOREACH_REVERSE(opt, &opts->head, QemuOptHead, next) {
+        if (is_help_option(opt->name)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool qemu_opt_get_bool(QemuOpts *opts, const char *name, bool defval)
+{
+    QemuOpt *opt = qemu_opt_find(opts, name);
+
+    if (opt == NULL)
+        return defval;
+    assert(opt->desc && opt->desc->type == QEMU_OPT_BOOL);
+    return opt->value.boolean;
+}
+
+uint64_t qemu_opt_get_number(QemuOpts *opts, const char *name, uint64_t defval)
+{
+    QemuOpt *opt = qemu_opt_find(opts, name);
+
+    if (opt == NULL)
+        return defval;
+    assert(opt->desc && opt->desc->type == QEMU_OPT_NUMBER);
+    return opt->value.uint;
+}
+
+uint64_t qemu_opt_get_size(QemuOpts *opts, const char *name, uint64_t defval)
+{
+    QemuOpt *opt = qemu_opt_find(opts, name);
+
+    if (opt == NULL)
+        return defval;
+    assert(opt->desc && opt->desc->type == QEMU_OPT_SIZE);
+    return opt->value.uint;
+}
+
+static void qemu_opt_parse(QemuOpt *opt, Error **errp)
+{
+    if (opt->desc == NULL)
+        return;
+
+    switch (opt->desc->type) {
+    case QEMU_OPT_STRING:
+        /* nothing */
+        return;
+    case QEMU_OPT_BOOL:
+        parse_option_bool(opt->name, opt->str, &opt->value.boolean, errp);
+        break;
+    case QEMU_OPT_NUMBER:
+        parse_option_number(opt->name, opt->str, &opt->value.uint, errp);
+        break;
+    case QEMU_OPT_SIZE:
+        parse_option_size(opt->name, opt->str, &opt->value.uint, errp);
+        break;
+    default:
+        abort();
+    }
+}
+
+static void qemu_opt_del(QemuOpt *opt)
+{
+    QTAILQ_REMOVE(&opt->opts->head, opt, next);
+    g_free((/* !const */ char*)opt->name);
+    g_free((/* !const */ char*)opt->str);
+    g_free(opt);
+}
+
+static bool opts_accepts_any(const QemuOpts *opts)
+{
+    return opts->list->desc[0].name == NULL;
+}
+
+static const QemuOptDesc *find_desc_by_name(const QemuOptDesc *desc,
+                                            const char *name)
+{
+    int i;
+
+    for (i = 0; desc[i].name != NULL; i++) {
+        if (strcmp(desc[i].name, name) == 0) {
+            return &desc[i];
+        }
+    }
+
+    return NULL;
+}
+
+static void opt_set(QemuOpts *opts, const char *name, const char *value,
+                    bool prepend, Error **errp)
+{
+    QemuOpt *opt;
+    const QemuOptDesc *desc;
+    Error *local_err = NULL;
+
+    desc = find_desc_by_name(opts->list->desc, name);
+    if (!desc && !opts_accepts_any(opts)) {
+        error_set(errp, QERR_INVALID_PARAMETER, name);
+        return;
+    }
+
+    opt = g_malloc0(sizeof(*opt));
+    opt->name = g_strdup(name);
+    opt->opts = opts;
+    if (prepend) {
+        QTAILQ_INSERT_HEAD(&opts->head, opt, next);
+    } else {
+        QTAILQ_INSERT_TAIL(&opts->head, opt, next);
+    }
+    opt->desc = desc;
+    opt->str = g_strdup(value);
+    qemu_opt_parse(opt, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+        qemu_opt_del(opt);
+    }
+}
+
+int qemu_opt_set(QemuOpts *opts, const char *name, const char *value)
+{
+    Error *local_err = NULL;
+
+    opt_set(opts, name, value, false, &local_err);
+    if (error_is_set(&local_err)) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+        return -1;
+    }
+
+    return 0;
+}
+
+void qemu_opt_set_err(QemuOpts *opts, const char *name, const char *value,
+                      Error **errp)
+{
+    opt_set(opts, name, value, false, errp);
+}
+
+int qemu_opt_set_bool(QemuOpts *opts, const char *name, bool val)
+{
+    QemuOpt *opt;
+    const QemuOptDesc *desc = opts->list->desc;
+
+    opt = g_malloc0(sizeof(*opt));
+    opt->desc = find_desc_by_name(desc, name);
+    if (!opt->desc && !opts_accepts_any(opts)) {
+        qerror_report(QERR_INVALID_PARAMETER, name);
+        g_free(opt);
+        return -1;
+    }
+
+    opt->name = g_strdup(name);
+    opt->opts = opts;
+    opt->value.boolean = !!val;
+    opt->str = g_strdup(val ? "on" : "off");
+    QTAILQ_INSERT_TAIL(&opts->head, opt, next);
+
+    return 0;
+}
+
+int qemu_opt_set_number(QemuOpts *opts, const char *name, int64_t val)
+{
+    QemuOpt *opt;
+    const QemuOptDesc *desc = opts->list->desc;
+
+    opt = g_malloc0(sizeof(*opt));
+    opt->desc = find_desc_by_name(desc, name);
+    if (!opt->desc && !opts_accepts_any(opts)) {
+        qerror_report(QERR_INVALID_PARAMETER, name);
+        g_free(opt);
+        return -1;
+    }
+
+    opt->name = g_strdup(name);
+    opt->opts = opts;
+    opt->value.uint = val;
+    opt->str = g_strdup_printf("%" PRId64, val);
+    QTAILQ_INSERT_TAIL(&opts->head, opt, next);
+
+    return 0;
+}
+
+int qemu_opt_foreach(QemuOpts *opts, qemu_opt_loopfunc func, void *opaque,
+                     int abort_on_failure)
+{
+    QemuOpt *opt;
+    int rc = 0;
+
+    QTAILQ_FOREACH(opt, &opts->head, next) {
+        rc = func(opt->name, opt->str, opaque);
+        if (abort_on_failure  &&  rc != 0)
+            break;
+    }
+    return rc;
+}
+
+QemuOpts *qemu_opts_find(QemuOptsList *list, const char *id)
+{
+    QemuOpts *opts;
+
+    QTAILQ_FOREACH(opts, &list->head, next) {
+        if (!opts->id && !id) {
+            return opts;
+        }
+        if (opts->id && id && !strcmp(opts->id, id)) {
+            return opts;
+        }
+    }
+    return NULL;
+}
+
+static int id_wellformed(const char *id)
+{
+    int i;
+
+    if (!qemu_isalpha(id[0])) {
+        return 0;
+    }
+    for (i = 1; id[i]; i++) {
+        if (!qemu_isalnum(id[i]) && !strchr("-._", id[i])) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
+                           int fail_if_exists, Error **errp)
+{
+    QemuOpts *opts = NULL;
+
+    if (id) {
+        if (!id_wellformed(id)) {
+            error_set(errp,QERR_INVALID_PARAMETER_VALUE, "id", "an identifier");
+#if 0 /* conversion from qerror_report() to error_set() broke this: */
+            error_printf_unless_qmp("Identifiers consist of letters, digits, '-', '.', '_', starting with a letter.\n");
+#endif
+            return NULL;
+        }
+        opts = qemu_opts_find(list, id);
+        if (opts != NULL) {
+            if (fail_if_exists && !list->merge_lists) {
+                error_set(errp, QERR_DUPLICATE_ID, id, list->name);
+                return NULL;
+            } else {
+                return opts;
+            }
+        }
+    } else if (list->merge_lists) {
+        opts = qemu_opts_find(list, NULL);
+        if (opts) {
+            return opts;
+        }
+    }
+    opts = g_malloc0(sizeof(*opts));
+    opts->id = g_strdup(id);
+    opts->list = list;
+    loc_save(&opts->loc);
+    QTAILQ_INIT(&opts->head);
+    QTAILQ_INSERT_TAIL(&list->head, opts, next);
+    return opts;
+}
+
+QemuOpts *qemu_opts_create_nofail(QemuOptsList *list)
+{
+    QemuOpts *opts;
+    Error *errp = NULL;
+    opts = qemu_opts_create(list, NULL, 0, &errp);
+    assert_no_error(errp);
+    return opts;
+}
+
+void qemu_opts_reset(QemuOptsList *list)
+{
+    QemuOpts *opts, *next_opts;
+
+    QTAILQ_FOREACH_SAFE(opts, &list->head, next, next_opts) {
+        qemu_opts_del(opts);
+    }
+}
+
+void qemu_opts_loc_restore(QemuOpts *opts)
+{
+    loc_restore(&opts->loc);
+}
+
+int qemu_opts_set(QemuOptsList *list, const char *id,
+                  const char *name, const char *value)
+{
+    QemuOpts *opts;
+    Error *local_err = NULL;
+
+    opts = qemu_opts_create(list, id, 1, &local_err);
+    if (error_is_set(&local_err)) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+        return -1;
+    }
+    return qemu_opt_set(opts, name, value);
+}
+
+const char *qemu_opts_id(QemuOpts *opts)
+{
+    return opts->id;
+}
+
+void qemu_opts_del(QemuOpts *opts)
+{
+    QemuOpt *opt;
+
+    for (;;) {
+        opt = QTAILQ_FIRST(&opts->head);
+        if (opt == NULL)
+            break;
+        qemu_opt_del(opt);
+    }
+    QTAILQ_REMOVE(&opts->list->head, opts, next);
+    g_free(opts->id);
+    g_free(opts);
+}
+
+int qemu_opts_print(QemuOpts *opts, void *dummy)
+{
+    QemuOpt *opt;
+
+    fprintf(stderr, "%s: %s:", opts->list->name,
+            opts->id ? opts->id : "<noid>");
+    QTAILQ_FOREACH(opt, &opts->head, next) {
+        fprintf(stderr, " %s=\"%s\"", opt->name, opt->str);
+    }
+    fprintf(stderr, "\n");
+    return 0;
+}
+
+static int opts_do_parse(QemuOpts *opts, const char *params,
+                         const char *firstname, bool prepend)
+{
+    char option[128], value[1024];
+    const char *p,*pe,*pc;
+    Error *local_err = NULL;
+
+    for (p = params; *p != '\0'; p++) {
+        pe = strchr(p, '=');
+        pc = strchr(p, ',');
+        if (!pe || (pc && pc < pe)) {
+            /* found "foo,more" */
+            if (p == params && firstname) {
+                /* implicitly named first option */
+                pstrcpy(option, sizeof(option), firstname);
+                p = get_opt_value(value, sizeof(value), p);
+            } else {
+                /* option without value, probably a flag */
+                p = get_opt_name(option, sizeof(option), p, ',');
+                if (strncmp(option, "no", 2) == 0) {
+                    memmove(option, option+2, strlen(option+2)+1);
+                    pstrcpy(value, sizeof(value), "off");
+                } else {
+                    pstrcpy(value, sizeof(value), "on");
+                }
+            }
+        } else {
+            /* found "foo=bar,more" */
+            p = get_opt_name(option, sizeof(option), p, '=');
+            if (*p != '=') {
+                break;
+            }
+            p++;
+            p = get_opt_value(value, sizeof(value), p);
+        }
+        if (strcmp(option, "id") != 0) {
+            /* store and parse */
+            opt_set(opts, option, value, prepend, &local_err);
+            if (error_is_set(&local_err)) {
+                qerror_report_err(local_err);
+                error_free(local_err);
+                return -1;
+            }
+        }
+        if (*p != ',') {
+            break;
+        }
+    }
+    return 0;
+}
+
+int qemu_opts_do_parse(QemuOpts *opts, const char *params, const char *firstname)
+{
+    return opts_do_parse(opts, params, firstname, false);
+}
+
+static QemuOpts *opts_parse(QemuOptsList *list, const char *params,
+                            int permit_abbrev, bool defaults)
+{
+    const char *firstname;
+    char value[1024], *id = NULL;
+    const char *p;
+    QemuOpts *opts;
+    Error *local_err = NULL;
+
+    assert(!permit_abbrev || list->implied_opt_name);
+    firstname = permit_abbrev ? list->implied_opt_name : NULL;
+
+    if (strncmp(params, "id=", 3) == 0) {
+        get_opt_value(value, sizeof(value), params+3);
+        id = value;
+    } else if ((p = strstr(params, ",id=")) != NULL) {
+        get_opt_value(value, sizeof(value), p+4);
+        id = value;
+    }
+    opts = qemu_opts_create(list, id, !defaults, &local_err);
+    if (opts == NULL) {
+        if (error_is_set(&local_err)) {
+            qerror_report_err(local_err);
+            error_free(local_err);
+        }
+        return NULL;
+    }
+
+    if (opts_do_parse(opts, params, firstname, defaults) != 0) {
+        qemu_opts_del(opts);
+        return NULL;
+    }
+
+    return opts;
+}
+
+QemuOpts *qemu_opts_parse(QemuOptsList *list, const char *params,
+                          int permit_abbrev)
+{
+    return opts_parse(list, params, permit_abbrev, false);
+}
+
+void qemu_opts_set_defaults(QemuOptsList *list, const char *params,
+                            int permit_abbrev)
+{
+    QemuOpts *opts;
+
+    opts = opts_parse(list, params, permit_abbrev, true);
+    assert(opts);
+}
+
+typedef struct OptsFromQDictState {
+    QemuOpts *opts;
+    Error **errp;
+} OptsFromQDictState;
+
+static void qemu_opts_from_qdict_1(const char *key, QObject *obj, void *opaque)
+{
+    OptsFromQDictState *state = opaque;
+    char buf[32];
+    const char *value;
+    int n;
+
+    if (!strcmp(key, "id") || error_is_set(state->errp)) {
+        return;
+    }
+
+    switch (qobject_type(obj)) {
+    case QTYPE_QSTRING:
+        value = qstring_get_str(qobject_to_qstring(obj));
+        break;
+    case QTYPE_QINT:
+        n = snprintf(buf, sizeof(buf), "%" PRId64,
+                     qint_get_int(qobject_to_qint(obj)));
+        assert(n < sizeof(buf));
+        value = buf;
+        break;
+    case QTYPE_QFLOAT:
+        n = snprintf(buf, sizeof(buf), "%.17g",
+                     qfloat_get_double(qobject_to_qfloat(obj)));
+        assert(n < sizeof(buf));
+        value = buf;
+        break;
+    case QTYPE_QBOOL:
+        pstrcpy(buf, sizeof(buf),
+                qbool_get_int(qobject_to_qbool(obj)) ? "on" : "off");
+        value = buf;
+        break;
+    default:
+        return;
+    }
+
+    qemu_opt_set_err(state->opts, key, value, state->errp);
+}
+
+/*
+ * Create QemuOpts from a QDict.
+ * Use value of key "id" as ID if it exists and is a QString.
+ * Only QStrings, QInts, QFloats and QBools are copied.  Entries with
+ * other types are silently ignored.
+ */
+QemuOpts *qemu_opts_from_qdict(QemuOptsList *list, const QDict *qdict,
+                               Error **errp)
+{
+    OptsFromQDictState state;
+    Error *local_err = NULL;
+    QemuOpts *opts;
+
+    opts = qemu_opts_create(list, qdict_get_try_str(qdict, "id"), 1,
+                            &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+        return NULL;
+    }
+
+    assert(opts != NULL);
+
+    state.errp = &local_err;
+    state.opts = opts;
+    qdict_iter(qdict, qemu_opts_from_qdict_1, &state);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+        qemu_opts_del(opts);
+        return NULL;
+    }
+
+    return opts;
+}
+
+/*
+ * Adds all QDict entries to the QemuOpts that can be added and removes them
+ * from the QDict. When this function returns, the QDict contains only those
+ * entries that couldn't be added to the QemuOpts.
+ */
+void qemu_opts_absorb_qdict(QemuOpts *opts, QDict *qdict, Error **errp)
+{
+    const QDictEntry *entry, *next;
+
+    entry = qdict_first(qdict);
+
+    while (entry != NULL) {
+        Error *local_err = NULL;
+        OptsFromQDictState state = {
+            .errp = &local_err,
+            .opts = opts,
+        };
+
+        next = qdict_next(qdict, entry);
+
+        if (find_desc_by_name(opts->list->desc, entry->key)) {
+            qemu_opts_from_qdict_1(entry->key, entry->value, &state);
+            if (error_is_set(&local_err)) {
+                error_propagate(errp, local_err);
+                return;
+            } else {
+                qdict_del(qdict, entry->key);
+            }
+        }
+
+        entry = next;
+    }
+}
+
+/*
+ * Convert from QemuOpts to QDict.
+ * The QDict values are of type QString.
+ * TODO We'll want to use types appropriate for opt->desc->type, but
+ * this is enough for now.
+ */
+QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict)
+{
+    QemuOpt *opt;
+    QObject *val;
+
+    if (!qdict) {
+        qdict = qdict_new();
+    }
+    if (opts->id) {
+        qdict_put(qdict, "id", qstring_from_str(opts->id));
+    }
+    QTAILQ_FOREACH(opt, &opts->head, next) {
+        val = QOBJECT(qstring_from_str(opt->str));
+        qdict_put_obj(qdict, opt->name, val);
+    }
+    return qdict;
+}
+
+/* Validate parsed opts against descriptions where no
+ * descriptions were provided in the QemuOptsList.
+ */
+void qemu_opts_validate(QemuOpts *opts, const QemuOptDesc *desc, Error **errp)
+{
+    QemuOpt *opt;
+    Error *local_err = NULL;
+
+    assert(opts_accepts_any(opts));
+
+    QTAILQ_FOREACH(opt, &opts->head, next) {
+        opt->desc = find_desc_by_name(desc, opt->name);
+        if (!opt->desc) {
+            error_set(errp, QERR_INVALID_PARAMETER, opt->name);
+            return;
+        }
+
+        qemu_opt_parse(opt, &local_err);
+        if (error_is_set(&local_err)) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+}
+
+int qemu_opts_foreach(QemuOptsList *list, qemu_opts_loopfunc func, void *opaque,
+                      int abort_on_failure)
+{
+    Location loc;
+    QemuOpts *opts;
+    int rc = 0;
+
+    loc_push_none(&loc);
+    QTAILQ_FOREACH(opts, &list->head, next) {
+        loc_restore(&opts->loc);
+        rc |= func(opts, opaque);
+        if (abort_on_failure  &&  rc != 0)
+            break;
+    }
+    loc_pop(&loc);
+    return rc;
+}
diff --git a/contrib/qemu/util/qemu-thread-posix.c b/contrib/qemu/util/qemu-thread-posix.c
new file mode 100644
index 000000000..4489abf1d
--- /dev/null
+++ b/contrib/qemu/util/qemu-thread-posix.c
@@ -0,0 +1,327 @@
+/*
+ * Wrappers around mutex/cond/thread functions
+ *
+ * Copyright Red Hat, Inc. 2009
+ *
+ * Author:
+ *  Marcelo Tosatti <mtosatti@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <time.h>
+#include <signal.h>
+#include <stdint.h>
+#include <string.h>
+#include <limits.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include "qemu/thread.h"
+
+static void error_exit(int err, const char *msg)
+{
+    fprintf(stderr, "qemu: %s: %s\n", msg, strerror(err));
+    abort();
+}
+
+void qemu_mutex_init(QemuMutex *mutex)
+{
+    int err;
+    pthread_mutexattr_t mutexattr;
+
+    pthread_mutexattr_init(&mutexattr);
+    pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_ERRORCHECK);
+    err = pthread_mutex_init(&mutex->lock, &mutexattr);
+    pthread_mutexattr_destroy(&mutexattr);
+    if (err)
+        error_exit(err, __func__);
+}
+
+void qemu_mutex_destroy(QemuMutex *mutex)
+{
+    int err;
+
+    err = pthread_mutex_destroy(&mutex->lock);
+    if (err)
+        error_exit(err, __func__);
+}
+
+void qemu_mutex_lock(QemuMutex *mutex)
+{
+    int err;
+
+    err = pthread_mutex_lock(&mutex->lock);
+    if (err)
+        error_exit(err, __func__);
+}
+
+int qemu_mutex_trylock(QemuMutex *mutex)
+{
+    return pthread_mutex_trylock(&mutex->lock);
+}
+
+void qemu_mutex_unlock(QemuMutex *mutex)
+{
+    int err;
+
+    err = pthread_mutex_unlock(&mutex->lock);
+    if (err)
+        error_exit(err, __func__);
+}
+
+void qemu_cond_init(QemuCond *cond)
+{
+    int err;
+
+    err = pthread_cond_init(&cond->cond, NULL);
+    if (err)
+        error_exit(err, __func__);
+}
+
+void qemu_cond_destroy(QemuCond *cond)
+{
+    int err;
+
+    err = pthread_cond_destroy(&cond->cond);
+    if (err)
+        error_exit(err, __func__);
+}
+
+void qemu_cond_signal(QemuCond *cond)
+{
+    int err;
+
+    err = pthread_cond_signal(&cond->cond);
+    if (err)
+        error_exit(err, __func__);
+}
+
+void qemu_cond_broadcast(QemuCond *cond)
+{
+    int err;
+
+    err = pthread_cond_broadcast(&cond->cond);
+    if (err)
+        error_exit(err, __func__);
+}
+
+void qemu_cond_wait(QemuCond *cond, QemuMutex *mutex)
+{
+    int err;
+
+    err = pthread_cond_wait(&cond->cond, &mutex->lock);
+    if (err)
+        error_exit(err, __func__);
+}
+
+void qemu_sem_init(QemuSemaphore *sem, int init)
+{
+    int rc;
+
+#if defined(__APPLE__) || defined(__NetBSD__)
+    rc = pthread_mutex_init(&sem->lock, NULL);
+    if (rc != 0) {
+        error_exit(rc, __func__);
+    }
+    rc = pthread_cond_init(&sem->cond, NULL);
+    if (rc != 0) {
+        error_exit(rc, __func__);
+    }
+    if (init < 0) {
+        error_exit(EINVAL, __func__);
+    }
+    sem->count = init;
+#else
+    rc = sem_init(&sem->sem, 0, init);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
+void qemu_sem_destroy(QemuSemaphore *sem)
+{
+    int rc;
+
+#if defined(__APPLE__) || defined(__NetBSD__)
+    rc = pthread_cond_destroy(&sem->cond);
+    if (rc < 0) {
+        error_exit(rc, __func__);
+    }
+    rc = pthread_mutex_destroy(&sem->lock);
+    if (rc < 0) {
+        error_exit(rc, __func__);
+    }
+#else
+    rc = sem_destroy(&sem->sem);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
+void qemu_sem_post(QemuSemaphore *sem)
+{
+    int rc;
+
+#if defined(__APPLE__) || defined(__NetBSD__)
+    pthread_mutex_lock(&sem->lock);
+    if (sem->count == INT_MAX) {
+        rc = EINVAL;
+    } else if (sem->count++ < 0) {
+        rc = pthread_cond_signal(&sem->cond);
+    } else {
+        rc = 0;
+    }
+    pthread_mutex_unlock(&sem->lock);
+    if (rc != 0) {
+        error_exit(rc, __func__);
+    }
+#else
+    rc = sem_post(&sem->sem);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
+static void compute_abs_deadline(struct timespec *ts, int ms)
+{
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    ts->tv_nsec = tv.tv_usec * 1000 + (ms % 1000) * 1000000;
+    ts->tv_sec = tv.tv_sec + ms / 1000;
+    if (ts->tv_nsec >= 1000000000) {
+        ts->tv_sec++;
+        ts->tv_nsec -= 1000000000;
+    }
+}
+
+int qemu_sem_timedwait(QemuSemaphore *sem, int ms)
+{
+    int rc;
+    struct timespec ts;
+
+#if defined(__APPLE__) || defined(__NetBSD__)
+    compute_abs_deadline(&ts, ms);
+    pthread_mutex_lock(&sem->lock);
+    --sem->count;
+    while (sem->count < 0) {
+        rc = pthread_cond_timedwait(&sem->cond, &sem->lock, &ts);
+        if (rc == ETIMEDOUT) {
+            ++sem->count;
+            break;
+        }
+        if (rc != 0) {
+            error_exit(rc, __func__);
+        }
+    }
+    pthread_mutex_unlock(&sem->lock);
+    return (rc == ETIMEDOUT ? -1 : 0);
+#else
+    if (ms <= 0) {
+        /* This is cheaper than sem_timedwait.  */
+        do {
+            rc = sem_trywait(&sem->sem);
+        } while (rc == -1 && errno == EINTR);
+        if (rc == -1 && errno == EAGAIN) {
+            return -1;
+        }
+    } else {
+        compute_abs_deadline(&ts, ms);
+        do {
+            rc = sem_timedwait(&sem->sem, &ts);
+        } while (rc == -1 && errno == EINTR);
+        if (rc == -1 && errno == ETIMEDOUT) {
+            return -1;
+        }
+    }
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+    return 0;
+#endif
+}
+
+void qemu_sem_wait(QemuSemaphore *sem)
+{
+#if defined(__APPLE__) || defined(__NetBSD__)
+    pthread_mutex_lock(&sem->lock);
+    --sem->count;
+    while (sem->count < 0) {
+        pthread_cond_wait(&sem->cond, &sem->lock);
+    }
+    pthread_mutex_unlock(&sem->lock);
+#else
+    int rc;
+
+    do {
+        rc = sem_wait(&sem->sem);
+    } while (rc == -1 && errno == EINTR);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
+void qemu_thread_create(QemuThread *thread,
+                       void *(*start_routine)(void*),
+                       void *arg, int mode)
+{
+    sigset_t set, oldset;
+    int err;
+    pthread_attr_t attr;
+
+    err = pthread_attr_init(&attr);
+    if (err) {
+        error_exit(err, __func__);
+    }
+    if (mode == QEMU_THREAD_DETACHED) {
+        err = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+        if (err) {
+            error_exit(err, __func__);
+        }
+    }
+
+    /* Leave signal handling to the iothread.  */
+    sigfillset(&set);
+    pthread_sigmask(SIG_SETMASK, &set, &oldset);
+    err = pthread_create(&thread->thread, &attr, start_routine, arg);
+    if (err)
+        error_exit(err, __func__);
+
+    pthread_sigmask(SIG_SETMASK, &oldset, NULL);
+
+    pthread_attr_destroy(&attr);
+}
+
+void qemu_thread_get_self(QemuThread *thread)
+{
+    thread->thread = pthread_self();
+}
+
+bool qemu_thread_is_self(QemuThread *thread)
+{
+   return pthread_equal(pthread_self(), thread->thread);
+}
+
+void qemu_thread_exit(void *retval)
+{
+    pthread_exit(retval);
+}
+
+void *qemu_thread_join(QemuThread *thread)
+{
+    int err;
+    void *ret;
+
+    err = pthread_join(thread->thread, &ret);
+    if (err) {
+        error_exit(err, __func__);
+    }
+    return ret;
+}
diff --git a/contrib/qemu/util/unicode.c b/contrib/qemu/util/unicode.c
new file mode 100644
index 000000000..d1c865885
--- /dev/null
+++ b/contrib/qemu/util/unicode.c
@@ -0,0 +1,100 @@
+/*
+ * Dealing with Unicode
+ *
+ * Copyright (C) 2013 Red Hat, Inc.
+ *
+ * Authors:
+ *  Markus Armbruster <armbru@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu-common.h"
+
+/**
+ * mod_utf8_codepoint:
+ * @s: string encoded in modified UTF-8
+ * @n: maximum number of bytes to read from @s, if less than 6
+ * @end: set to end of sequence on return
+ *
+ * Convert the modified UTF-8 sequence at the start of @s.  Modified
+ * UTF-8 is exactly like UTF-8, except U+0000 is encoded as
+ * "\xC0\x80".
+ *
+ * If @n is zero or @s points to a zero byte, the sequence is invalid,
+ * and @end is set to @s.
+ *
+ * If @s points to an impossible byte (0xFE or 0xFF) or a continuation
+ * byte, the sequence is invalid, and @end is set to @s + 1
+ *
+ * Else, the first byte determines how many continuation bytes are
+ * expected.  If there are fewer, the sequence is invalid, and @end is
+ * set to @s + 1 + actual number of continuation bytes.  Else, the
+ * sequence is well-formed, and @end is set to @s + 1 + expected
+ * number of continuation bytes.
+ *
+ * A well-formed sequence is valid unless it encodes a codepoint
+ * outside the Unicode range U+0000..U+10FFFF, one of Unicode's 66
+ * noncharacters, a surrogate codepoint, or is overlong.  Except the
+ * overlong sequence "\xC0\x80" is valid.
+ *
+ * Conversion succeeds if and only if the sequence is valid.
+ *
+ * Returns: the Unicode codepoint on success, -1 on failure.
+ */
+int mod_utf8_codepoint(const char *s, size_t n, char **end)
+{
+    static int min_cp[5] = { 0x80, 0x800, 0x10000, 0x200000, 0x4000000 };
+    const unsigned char *p;
+    unsigned byte, mask, len, i;
+    int cp;
+
+    if (n == 0 || *s == 0) {
+        /* empty sequence */
+        *end = (char *)s;
+        return -1;
+    }
+
+    p = (const unsigned char *)s;
+    byte = *p++;
+    if (byte < 0x80) {
+        cp = byte;              /* one byte sequence */
+    } else if (byte >= 0xFE) {
+        cp = -1;                /* impossible bytes 0xFE, 0xFF */
+    } else if ((byte & 0x40) == 0) {
+        cp = -1;                /* unexpected continuation byte */
+    } else {
+        /* multi-byte sequence */
+        len = 0;
+        for (mask = 0x80; byte & mask; mask >>= 1) {
+            len++;
+        }
+        assert(len > 1 && len < 7);
+        cp = byte & (mask - 1);
+        for (i = 1; i < len; i++) {
+            byte = i < n ? *p : 0;
+            if ((byte & 0xC0) != 0x80) {
+                cp = -1;        /* continuation byte missing */
+                goto out;
+            }
+            p++;
+            cp <<= 6;
+            cp |= byte & 0x3F;
+        }
+        if (cp > 0x10FFFF) {
+            cp = -1;            /* beyond Unicode range */
+        } else if ((cp >= 0xFDD0 && cp <= 0xFDEF)
+                   || (cp & 0xFFFE) == 0xFFFE) {
+            cp = -1;            /* noncharacter */
+        } else if (cp >= 0xD800 && cp <= 0xDFFF) {
+            cp = -1;            /* surrogate code point */
+        } else if (cp < min_cp[len - 2] && !(cp == 0 && len == 2)) {
+            cp = -1;            /* overlong, not \xC0\x80 */
+        }
+    }
+
+out:
+    *end = (char *)p;
+    return cp;
+}
diff --git a/contrib/rbtree/rb.c b/contrib/rbtree/rb.c
index d1339b97d..6184c507e 100644
--- a/contrib/rbtree/rb.c
+++ b/contrib/rbtree/rb.c
@@ -1,26 +1,30 @@
 /* Produced by texiweb from libavl.w. */
 
 /* libavl - library for manipulation of binary trees.
-   Copyright (C) 1998-2002, 2004 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2004 Free Software
+   Foundation, Inc.
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+   02110-1301 USA.
+
+   This code is also covered by the following earlier license notice:
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License as
    published by the Free Software Foundation; either version 2 of the
    License, or (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-   See the GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
-   02111-1307, USA.
-
-   The author may be contacted at <blp@gnu.org> on the Internet, or
-   write to Ben Pfaff, Stanford University, Computer Science Dept., 353
-   Serra Mall, Stanford CA 94305, USA.
 */
 
 #include <assert.h>
diff --git a/contrib/rbtree/rb.h b/contrib/rbtree/rb.h
index c8858b556..97b44cfd4 100644
--- a/contrib/rbtree/rb.h
+++ b/contrib/rbtree/rb.h
@@ -1,26 +1,30 @@
 /* Produced by texiweb from libavl.w. */
 
 /* libavl - library for manipulation of binary trees.
-   Copyright (C) 1998-2002, 2004 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2004 Free Software
+   Foundation, Inc.
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+   02110-1301 USA.
+
+   This code is also covered by the following earlier license notice:
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License as
    published by the Free Software Foundation; either version 2 of the
    License, or (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-   See the GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
-   02111-1307, USA.
-
-   The author may be contacted at <blp@gnu.org> on the Internet, or
-   write to Ben Pfaff, Stanford University, Computer Science Dept., 353
-   Serra Mall, Stanford CA 94305, USA.
 */
 
 #ifndef RB_H
@@ -51,7 +55,7 @@ void rb_free (struct libavl_allocator *, void *);
 
 /* Maximum RB height. */
 #ifndef RB_MAX_HEIGHT
-#define RB_MAX_HEIGHT 48
+#define RB_MAX_HEIGHT 128
 #endif
 
 /* Tree data structure. */
diff --git a/contrib/stdlib/gf_mkostemp.c b/contrib/stdlib/gf_mkostemp.c
new file mode 100644
index 000000000..931249a45
--- /dev/null
+++ b/contrib/stdlib/gf_mkostemp.c
@@ -0,0 +1,107 @@
+/* Borrowed from glibc-2.16/sysdeps/posix/tempname.c */
+
+/* Copyright (C) 1991-2001, 2006, 2007, 2009 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/time.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <time.h>
+#include <inttypes.h>
+
+static const char letters[] =
+"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+
+/* Generate a temporary file name based on TMPL.  TMPL must match the
+   rules for mk[s]temp (i.e. end in "XXXXXX", possibly with a suffix).
+*/
+
+#if !defined(TMP_MAX)
+#define TMP_MAX 238328
+#endif
+
+int
+gf_mkostemp (char *tmpl, int suffixlen, int flags)
+{
+        int len;
+        char *XXXXXX;
+        static uint64_t value;
+        uint64_t random_time_bits;
+        unsigned int count;
+        int fd = -1;
+
+  /* A lower bound on the number of temporary files to attempt to
+     generate.  The maximum total number of temporary file names that
+     can exist for a given template is 62**6.  It should never be
+     necessary to try all these combinations.  Instead if a reasonable
+     number of names is tried (we define reasonable as 62**3) fail to
+     give the system administrator the chance to remove the problems.  */
+
+        unsigned int attempts = TMP_MAX; /* TMP_MAX == 62³ */
+
+        len = strlen (tmpl);
+        if (len < 6 + suffixlen || memcmp (&tmpl[len - 6 - suffixlen],
+                                           "XXXXXX", 6))
+                return -1;
+
+  /* This is where the Xs start.  */
+        XXXXXX = &tmpl[len - 6 - suffixlen];
+
+  /* Get some more or less random data.  */
+# if HAVE_GETTIMEOFDAY
+        struct timeval tv;
+        gettimeofday (&tv, NULL);
+        random_time_bits = ((uint64_t) tv.tv_usec << 16) ^ tv.tv_sec;
+# else
+        random_time_bits = time (NULL);
+# endif
+
+        value += random_time_bits ^ getpid ();
+
+        for (count = 0; count < attempts; value += 7777, ++count) {
+                uint64_t v = value;
+
+                /* Fill in the random bits.  */
+                XXXXXX[0] = letters[v % 62];
+                v /= 62;
+                XXXXXX[1] = letters[v % 62];
+                v /= 62;
+                XXXXXX[2] = letters[v % 62];
+                v /= 62;
+                XXXXXX[3] = letters[v % 62];
+                v /= 62;
+                XXXXXX[4] = letters[v % 62];
+                v /= 62;
+                XXXXXX[5] = letters[v % 62];
+
+                fd = open (tmpl, (flags & ~O_ACCMODE)
+                           | O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
+
+                if (fd >= 0)
+                        return fd;
+                else if (errno != EEXIST)
+                        return -1;
+        }
+
+        /* We got out of the loop because we ran out of combinations to try.  */
+        return -1;
+}
diff --git a/doc/Makefile.am b/doc/Makefile.am
index aee63b289..1103b607d 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1,13 +1,6 @@
-EXTRA_DIST = glusterfs.vol.sample glusterfsd.vol.sample glusterfs.8 mount.glusterfs.8\
-	porting_guide.txt authentication.txt coding-standard.pdf get_put_api_using_xattr.txt \
-	translator-options.txt mac-related-xattrs.txt replicate.pdf glusterd.vol gluster.8 \
-        glusterd.8 glusterfsd.8
-
-SUBDIRS = examples hacker-guide
-
-voldir = $(sysconfdir)/glusterfs
-vol_DATA = glusterd.vol
+EXTRA_DIST = glusterfs.8 mount.glusterfs.8 gluster.8 \
+			glusterd.8 glusterfsd.8
 
 man8_MANS = glusterfs.8 mount.glusterfs.8 gluster.8 glusterd.8 glusterfsd.8
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/doc/admin-guide/en-US/images/640px-GlusterFS_Architecture.png b/doc/admin-guide/en-US/images/640px-GlusterFS_Architecture.png
new file mode 100644
index 000000000..95f89ec82
--- /dev/null
+++ b/doc/admin-guide/en-US/images/640px-GlusterFS_Architecture.png
diff --git a/doc/admin-guide/en-US/images/Distributed_Replicated_Volume.png b/doc/admin-guide/en-US/images/Distributed_Replicated_Volume.png
new file mode 100644
index 000000000..dfc0a2c56
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Distributed_Replicated_Volume.png
diff --git a/doc/admin-guide/en-US/images/Distributed_Striped_Replicated_Volume.png b/doc/admin-guide/en-US/images/Distributed_Striped_Replicated_Volume.png
new file mode 100644
index 000000000..d286fa99e
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Distributed_Striped_Replicated_Volume.png
diff --git a/doc/admin-guide/en-US/images/Distributed_Striped_Volume.png b/doc/admin-guide/en-US/images/Distributed_Striped_Volume.png
new file mode 100644
index 000000000..752fa982f
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Distributed_Striped_Volume.png
diff --git a/doc/admin-guide/en-US/images/Distributed_Volume.png b/doc/admin-guide/en-US/images/Distributed_Volume.png
new file mode 100644
index 000000000..4386ca935
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Distributed_Volume.png
diff --git a/doc/admin-guide/en-US/images/Geo-Rep03_Internet.png b/doc/admin-guide/en-US/images/Geo-Rep03_Internet.png
new file mode 100644
index 000000000..3cd0eaded
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Geo-Rep03_Internet.png
diff --git a/doc/admin-guide/en-US/images/Geo-Rep04_Cascading.png b/doc/admin-guide/en-US/images/Geo-Rep04_Cascading.png
new file mode 100644
index 000000000..54bf9f05c
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Geo-Rep04_Cascading.png
diff --git a/doc/admin-guide/en-US/images/Geo-Rep_LAN.png b/doc/admin-guide/en-US/images/Geo-Rep_LAN.png
new file mode 100644
index 000000000..a74f6dbb5
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Geo-Rep_LAN.png
diff --git a/doc/admin-guide/en-US/images/Geo-Rep_WAN.png b/doc/admin-guide/en-US/images/Geo-Rep_WAN.png
new file mode 100644
index 000000000..d72d72768
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Geo-Rep_WAN.png
diff --git a/doc/admin-guide/en-US/images/GlusterFS_Architecture.png b/doc/admin-guide/en-US/images/GlusterFS_Architecture.png
new file mode 100644
index 000000000..b506db1f4
--- /dev/null
+++ b/doc/admin-guide/en-US/images/GlusterFS_Architecture.png
diff --git a/doc/admin-guide/en-US/images/Hadoop_Architecture.png b/doc/admin-guide/en-US/images/Hadoop_Architecture.png
new file mode 100644
index 000000000..8725bd330
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Hadoop_Architecture.png
diff --git a/doc/admin-guide/en-US/images/Replicated_Volume.png b/doc/admin-guide/en-US/images/Replicated_Volume.png
new file mode 100644
index 000000000..135a63f34
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Replicated_Volume.png
diff --git a/doc/admin-guide/en-US/images/Striped_Replicated_Volume.png b/doc/admin-guide/en-US/images/Striped_Replicated_Volume.png
new file mode 100644
index 000000000..ee88af731
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Striped_Replicated_Volume.png
diff --git a/doc/admin-guide/en-US/images/Striped_Volume.png b/doc/admin-guide/en-US/images/Striped_Volume.png
new file mode 100644
index 000000000..63a84b242
--- /dev/null
+++ b/doc/admin-guide/en-US/images/Striped_Volume.png
diff --git a/doc/admin-guide/en-US/images/UFO_Architecture.png b/doc/admin-guide/en-US/images/UFO_Architecture.png
new file mode 100644
index 000000000..be85d7b28
--- /dev/null
+++ b/doc/admin-guide/en-US/images/UFO_Architecture.png
diff --git a/doc/admin-guide/en-US/images/VSA_Architecture.png b/doc/admin-guide/en-US/images/VSA_Architecture.png
new file mode 100644
index 000000000..c3ab80cf3
--- /dev/null
+++ b/doc/admin-guide/en-US/images/VSA_Architecture.png
diff --git a/doc/admin-guide/en-US/images/icon.svg b/doc/admin-guide/en-US/images/icon.svg
new file mode 100644
index 000000000..b2f16d0f6
--- /dev/null
+++ b/doc/admin-guide/en-US/images/icon.svg
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.0" width="32" height="32" id="svg3017">
+  <defs id="defs3019">
+    <linearGradient id="linearGradient2381">
+      <stop id="stop2383" style="stop-color:#ffffff;stop-opacity:1" offset="0"/>
+      <stop id="stop2385" style="stop-color:#ffffff;stop-opacity:0" offset="1"/>
+    </linearGradient>
+    <linearGradient x1="296.4996" y1="188.81061" x2="317.32471" y2="209.69398" id="linearGradient2371" xlink:href="#linearGradient2381" gradientUnits="userSpaceOnUse" gradientTransform="matrix(0.90776,0,0,0.90776,24.35648,49.24131)"/>
+  </defs>
+  <g transform="matrix(0.437808,-0.437808,0.437808,0.437808,-220.8237,43.55311)" id="g5089">
+    <path d="m 8.4382985,-6.28125 c -0.6073916,0 -4.3132985,5.94886271 -4.3132985,8.25 l 0,26.71875 c 0,0.846384 0.5818159,1.125 1.15625,1.125 l 25.5625,0 c 0.632342,0 1.125001,-0.492658 1.125,-1.125 l 0,-5.21875 0.28125,0 c 0.49684,0 0.906249,-0.409411 0.90625,-0.90625 l 0,-27.9375 c 0,-0.4968398 -0.40941,-0.90625 -0.90625,-0.90625 l -23.8117015,0 z" transform="translate(282.8327,227.1903)" id="path5091" style="fill:#5c5c4f;stroke:#000000;stroke-width:3.23021388;stroke-miterlimit:4;stroke-dasharray:none"/>
+    <rect width="27.85074" height="29.369793" rx="1.1414107" ry="1.1414107" x="286.96509" y="227.63805" id="rect5093" style="fill:#032c87"/>
+    <path d="m 288.43262,225.43675 25.2418,0 0,29.3698 -26.37615,0.0241 1.13435,-29.39394 z" id="rect5095" style="fill:#ffffff"/>
+    <path d="m 302.44536,251.73726 c 1.38691,7.85917 -0.69311,11.28365 -0.69311,11.28365 2.24384,-1.60762 3.96426,-3.47694 4.90522,-5.736 0.96708,2.19264 1.83294,4.42866 4.27443,5.98941 0,0 -1.59504,-7.2004 -1.71143,-11.53706 l -6.77511,0 z" id="path5097" style="fill:#a70000;fill-opacity:1;stroke-width:2"/>
+    <rect width="25.241802" height="29.736675" rx="0.89682275" ry="0.89682275" x="290.73544" y="220.92249" id="rect5099" style="fill:#809cc9"/>
+    <path d="m 576.47347,725.93939 6.37084,0.41502 0.4069,29.51809 c -1.89202,-1.31785 -6.85427,-3.7608 -8.26232,-1.68101 l 0,-26.76752 c 0,-0.82246 0.66212,-1.48458 1.48458,-1.48458 z" transform="matrix(0.499065,-0.866565,0,1,0,0)" id="rect5101" style="fill:#4573b3;fill-opacity:1"/>
+    <path d="m 293.2599,221.89363 20.73918,0 c 0.45101,0 0.8141,0.3631 0.8141,0.81411 0.21547,6.32836 -19.36824,21.7635 -22.36739,17.59717 l 0,-17.59717 c 0,-0.45101 0.3631,-0.81411 0.81411,-0.81411 z" id="path5103" style="opacity:0.65536726;fill:url(#linearGradient2371);fill-opacity:1"/>
+  </g>
+</svg>
diff --git a/doc/admin-guide/en-US/markdown/Administration_Guide.md b/doc/admin-guide/en-US/markdown/Administration_Guide.md
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/Administration_Guide.md
@@ -0,0 +1 @@
+
diff --git a/doc/admin-guide/en-US/markdown/Author_Group.md b/doc/admin-guide/en-US/markdown/Author_Group.md
new file mode 100644
index 000000000..ef2a5e677
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/Author_Group.md
@@ -0,0 +1,5 @@
+Divya
+Muntimadugu
+Red Hat
+Engineering Content Services
+divya@redhat.com
diff --git a/doc/admin-guide/en-US/markdown/Book_Info.md b/doc/admin-guide/en-US/markdown/Book_Info.md
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/Book_Info.md
@@ -0,0 +1 @@
+
diff --git a/doc/admin-guide/en-US/markdown/Chapter.md b/doc/admin-guide/en-US/markdown/Chapter.md
new file mode 100644
index 000000000..8420259c4
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/Chapter.md
@@ -0,0 +1,18 @@
+Test Chapter
+============
+
+This is a test paragraph
+
+Test Section 1
+==============
+
+This is a test paragraph in a section
+
+Test Section 2
+==============
+
+This is a test paragraph in Section 2
+
+1.  listitem text
+
+
diff --git a/doc/admin-guide/en-US/markdown/Preface.md b/doc/admin-guide/en-US/markdown/Preface.md
new file mode 100644
index 000000000..f7e934ae8
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/Preface.md
@@ -0,0 +1,22 @@
+Preface
+=======
+
+This guide describes how to configure, operate, and manage Gluster File
+System (GlusterFS).
+
+Audience
+========
+
+This guide is intended for Systems Administrators interested in
+configuring and managing GlusterFS.
+
+This guide assumes that you are familiar with the Linux operating
+system, concepts of File System, GlusterFS concepts, and GlusterFS
+Installation
+
+License
+=======
+
+The License information is available at [][].
+
+  []: http://www.redhat.com/licenses/rhel_rha_eula.html
diff --git a/doc/admin-guide/en-US/markdown/Revision_History.md b/doc/admin-guide/en-US/markdown/Revision_History.md
new file mode 100644
index 000000000..2084309d1
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/Revision_History.md
@@ -0,0 +1,4 @@
+Revision History
+================
+
+1-0 Thu Apr 5 2012 Divya Muntimadugu <divya@redhat.com> Draft
diff --git a/doc/admin-guide/en-US/markdown/admin_ACLs.md b/doc/admin-guide/en-US/markdown/admin_ACLs.md
new file mode 100644
index 000000000..308e069ca
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_ACLs.md
@@ -0,0 +1,197 @@
+POSIX Access Control Lists
+==========================
+
+POSIX Access Control Lists (ACLs) allows you to assign different
+permissions for different users or groups even though they do not
+correspond to the original owner or the owning group.
+
+For example: User john creates a file but does not want to allow anyone
+to do anything with this file, except another user, antony (even though
+there are other users that belong to the group john).
+
+This means, in addition to the file owner, the file group, and others,
+additional users and groups can be granted or denied access by using
+POSIX ACLs.
+
+Activating POSIX ACLs Support
+=============================
+
+To use POSIX ACLs for a file or directory, the partition of the file or
+directory must be mounted with POSIX ACLs support.
+
+Activating POSIX ACLs Support on Sever
+--------------------------------------
+
+To mount the backend export directories for POSIX ACLs support, use the
+following command:
+
+`# mount -o acl `
+
+For example:
+
+`# mount -o acl /dev/sda1 /export1 `
+
+Alternatively, if the partition is listed in the /etc/fstab file, add
+the following entry for the partition to include the POSIX ACLs option:
+
+`LABEL=/work /export1 ext3 rw, acl 14 `
+
+Activating POSIX ACLs Support on Client
+---------------------------------------
+
+To mount the glusterfs volumes for POSIX ACLs support, use the following
+command:
+
+`# mount –t glusterfs -o acl `
+
+For example:
+
+`# mount -t glusterfs -o acl 198.192.198.234:glustervolume /mnt/gluster`
+
+Setting POSIX ACLs
+==================
+
+You can set two types of POSIX ACLs, that is, access ACLs and default
+ACLs. You can use access ACLs to grant permission for a specific file or
+directory. You can use default ACLs only on a directory but if a file
+inside that directory does not have an ACLs, it inherits the permissions
+of the default ACLs of the directory.
+
+You can set ACLs for per user, per group, for users not in the user
+group for the file, and via the effective right mask.
+
+Setting Access ACLs
+-------------------
+
+You can apply access ACLs to grant permission for both files and
+directories.
+
+**To set or modify Access ACLs**
+
+You can set or modify access ACLs use the following command:
+
+`# setfacl –m  file `
+
+The ACL entry types are the POSIX ACLs representations of owner, group,
+and other.
+
+Permissions must be a combination of the characters `r` (read), `w`
+(write), and `x` (execute). You must specify the ACL entry in the
+following format and can specify multiple entry types separated by
+commas.
+
+  ACL Entry              Description
+  ---------------------- --------------------------------------------------------------------------------------------------------------------------------------------------
+  u:uid:\<permission\>   Sets the access ACLs for a user. You can specify user name or UID
+  g:gid:\<permission\>   Sets the access ACLs for a group. You can specify group name or GID.
+  m:\<permission\>       Sets the effective rights mask. The mask is the combination of all access permissions of the owning group and all of the user and group entries.
+  o:\<permission\>       Sets the access ACLs for users other than the ones in the group for the file.
+
+If a file or directory already has an POSIX ACLs, and the setfacl
+command is used, the additional permissions are added to the existing
+POSIX ACLs or the existing rule is modified.
+
+For example, to give read and write permissions to user antony:
+
+`# setfacl -m u:antony:rw /mnt/gluster/data/testfile `
+
+Setting Default ACLs
+--------------------
+
+You can apply default ACLs only to directories. They determine the
+permissions of a file system objects that inherits from its parent
+directory when it is created.
+
+To set default ACLs
+
+You can set default ACLs for files and directories using the following
+command:
+
+`# setfacl –m –-set `
+
+For example, to set the default ACLs for the /data directory to read for
+users not in the user group:
+
+`# setfacl –m --set o::r /mnt/gluster/data `
+
+> **Note**
+>
+> An access ACLs set for an individual file can override the default
+> ACLs permissions.
+
+**Effects of a Default ACLs**
+
+The following are the ways in which the permissions of a directory's
+default ACLs are passed to the files and subdirectories in it:
+
+-   A subdirectory inherits the default ACLs of the parent directory
+    both as its default ACLs and as an access ACLs.
+
+-   A file inherits the default ACLs as its access ACLs.
+
+Retrieving POSIX ACLs
+=====================
+
+You can view the existing POSIX ACLs for a file or directory.
+
+**To view existing POSIX ACLs**
+
+-   View the existing access ACLs of a file using the following command:
+
+    `# getfacl `
+
+    For example, to view the existing POSIX ACLs for sample.jpg
+
+        # getfacl /mnt/gluster/data/test/sample.jpg
+        # owner: antony
+        # group: antony
+        user::rw-
+        group::rw-
+        other::r--
+
+-   View the default ACLs of a directory using the following command:
+
+    `# getfacl `
+
+    For example, to view the existing ACLs for /data/doc
+
+        # getfacl /mnt/gluster/data/doc
+        # owner: antony
+        # group: antony
+        user::rw-
+        user:john:r--
+        group::r--
+        mask::r--
+        other::r--
+        default:user::rwx
+        default:user:antony:rwx
+        default:group::r-x
+        default:mask::rwx
+        default:other::r-x
+
+Removing POSIX ACLs
+===================
+
+To remove all the permissions for a user, groups, or others, use the
+following command:
+
+`# setfacl -x `
+
+For example, to remove all permissions from the user antony:
+
+`# setfacl -x u:antony /mnt/gluster/data/test-file`
+
+Samba and ACLs
+==============
+
+If you are using Samba to access GlusterFS FUSE mount, then POSIX ACLs
+are enabled by default. Samba has been compiled with the
+`--with-acl-support` option, so no special flags are required when
+accessing or mounting a Samba share.
+
+NFS and ACLs
+============
+
+Currently we do not support ACLs configuration through NFS, i.e. setfacl
+and getfacl commands do not work. However, ACLs permissions set using
+Gluster Native Client is applicable on NFS mounts.
diff --git a/doc/admin-guide/en-US/markdown/admin_Hadoop.md b/doc/admin-guide/en-US/markdown/admin_Hadoop.md
new file mode 100644
index 000000000..2894fa713
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_Hadoop.md
@@ -0,0 +1,170 @@
+Managing Hadoop Compatible Storage
+==================================
+
+GlusterFS provides compatibility for Apache Hadoop and it uses the
+standard file system APIs available in Hadoop to provide a new storage
+option for Hadoop deployments. Existing MapReduce based applications can
+use GlusterFS seamlessly. This new functionality opens up data within
+Hadoop deployments to any file-based or object-based application.
+
+Architecture Overview
+=====================
+
+The following diagram illustrates Hadoop integration with GlusterFS:
+
+Advantages
+==========
+
+The following are the advantages of Hadoop Compatible Storage with
+GlusterFS:
+
+-   Provides simultaneous file-based and object-based access within
+    Hadoop.
+
+-   Eliminates the centralized metadata server.
+
+-   Provides compatibility with MapReduce applications and rewrite is
+    not required.
+
+-   Provides a fault tolerant file system.
+
+Preparing to Install Hadoop Compatible Storage
+==============================================
+
+This section provides information on pre-requisites and list of
+dependencies that will be installed during installation of Hadoop
+compatible storage.
+
+Pre-requisites
+--------------
+
+The following are the pre-requisites to install Hadoop Compatible
+Storage :
+
+-   Hadoop 0.20.2 is installed, configured, and is running on all the
+    machines in the cluster.
+
+-   Java Runtime Environment
+
+-   Maven (mandatory only if you are building the plugin from the
+    source)
+
+-   JDK (mandatory only if you are building the plugin from the source)
+
+-   getfattr - command line utility
+
+Installing, and Configuring Hadoop Compatible Storage
+=====================================================
+
+This section describes how to install and configure Hadoop Compatible
+Storage in your storage environment and verify that it is functioning
+correctly.
+
+1.  Download `glusterfs-hadoop-0.20.2-0.1.x86_64.rpm` file to each
+    server on your cluster. You can download the file from [][].
+
+2.  To install Hadoop Compatible Storage on all servers in your cluster,
+    run the following command:
+
+    `# rpm –ivh --nodeps glusterfs-hadoop-0.20.2-0.1.x86_64.rpm`
+
+    The following files will be extracted:
+
+    -   /usr/local/lib/glusterfs-Hadoop-version-gluster\_plugin\_version.jar
+
+    -   /usr/local/lib/conf/core-site.xml
+
+3.  (Optional) To install Hadoop Compatible Storage in a different
+    location, run the following command:
+
+    `# rpm –ivh --nodeps –prefix /usr/local/glusterfs/hadoop glusterfs-hadoop- 0.20.2-0.1.x86_64.rpm`
+
+4.  Edit the `conf/core-site.xml` file. The following is the sample
+    `conf/core-site.xml` file:
+
+        <configuration>
+          <property>
+            <name>fs.glusterfs.impl</name>
+            <value>org.apache.hadoop.fs.glusterfs.Gluster FileSystem</value>
+        </property>
+
+        <property>
+           <name>fs.default.name</name>
+           <value>glusterfs://fedora1:9000</value>
+        </property>
+
+        <property>
+           <name>fs.glusterfs.volname</name>
+           <value>hadoopvol</value>
+        </property>  
+         
+        <property>
+           <name>fs.glusterfs.mount</name>
+           <value>/mnt/glusterfs</value>
+        </property>
+
+        <property>
+           <name>fs.glusterfs.server</name>
+           <value>fedora2</value>
+        </property>
+
+        <property>
+           <name>quick.slave.io</name>
+           <value>Off</value>
+        </property>
+        </configuration>
+
+    The following are the configurable fields:
+
+      -----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      Property Name          Default Value              Description
+      ---------------------- -------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      fs.default.name        glusterfs://fedora1:9000   Any hostname in the cluster as the server and any port number.
+
+      fs.glusterfs.volname   hadoopvol                  GlusterFS volume to mount.
+
+      fs.glusterfs.mount     /mnt/glusterfs             The directory used to fuse mount the volume.
+
+      fs.glusterfs.server    fedora2                    Any hostname or IP address on the cluster except the client/master.
+
+      quick.slave.io         Off                        Performance tunable option. If this option is set to On, the plugin will try to perform I/O directly from the disk file system (like ext3 or ext4) the file resides on. Hence read performance will improve and job would run faster.
+                                                        > **Note**
+                                                        >
+                                                        > This option is not tested widely
+      -----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+5.  Create a soft link in Hadoop’s library and configuration directory
+    for the downloaded files (in Step 3) using the following commands:
+
+    `# ln -s >`
+
+    For example,
+
+    `# ln –s /usr/local/lib/glusterfs-0.20.2-0.1.jar /lib/glusterfs-0.20.2-0.1.jar`
+
+    `# ln –s /usr/local/lib/conf/core-site.xml /conf/core-site.xml `
+
+6.  (Optional) You can run the following command on Hadoop master to
+    build the plugin and deploy it along with core-site.xml file,
+    instead of repeating the above steps:
+
+    `# build-deploy-jar.py -d  -c `
+
+Starting and Stopping the Hadoop MapReduce Daemon
+=================================================
+
+To start and stop MapReduce daemon
+
+-   To start MapReduce daemon manually, enter the following command:
+
+    `# /bin/start-mapred.sh`
+
+-   To stop MapReduce daemon manually, enter the following command:
+
+    `# /bin/stop-mapred.sh `
+
+> **Note**
+>
+> You must start Hadoop MapReduce daemon on all servers.
+
+  []: http://download.gluster.com/pub/gluster/glusterfs/qa-releases/3.3-beta-2/glusterfs-hadoop-0.20.2-0.1.x86_64.rpm
diff --git a/doc/admin-guide/en-US/markdown/admin_UFO.md b/doc/admin-guide/en-US/markdown/admin_UFO.md
new file mode 100644
index 000000000..3311eff01
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_UFO.md
@@ -0,0 +1,1219 @@
+Managing Unified File and Object Storage
+========================================
+
+Unified File and Object Storage (UFO) unifies NAS and object storage
+technology. It provides a system for data storage that enables users to
+access the same data, both as an object and as a file, thus simplifying
+management and controlling storage costs.
+
+Unified File and Object Storage is built upon Openstack's Object Storage
+Swift. Open Stack Object Storage allows users to store and retrieve
+files and content through a simple Web Service (REST: Representational
+State Transfer) interface as objects and GlusterFS, allows users to
+store and retrieve files using Native Fuse and NFS mounts. It uses
+GlusterFS as a backend file system for Open Stack Swift. It also
+leverages on Open Stack Swift's web interface for storing and retrieving
+files over the web combined with GlusterFS features like scalability and
+high availability, replication, elastic volume management for data
+management at disk level.
+
+Unified File and Object Storage technology enables enterprises to adopt
+and deploy cloud storage solutions. It allows users to access and modify
+data as objects from a REST interface along with the ability to access
+and modify files from NAS interfaces including NFS and CIFS. In addition
+to decreasing cost and making it faster and easier to access object
+data, it also delivers massive scalability, high availability and
+replication of object storage. Infrastructure as a Service (IaaS)
+providers can utilize GlusterFS Unified File and Object Storage
+technology to enable their own cloud storage service. Enterprises can
+use this technology to accelerate the process of preparing file-based
+applications for the cloud and simplify new application development for
+cloud computing environments.
+
+OpenStack Object Storage is scalable object storage system and it is not
+a traditional file system. You will not be able to mount this system
+like traditional SAN or NAS volumes and perform POSIX compliant
+operations.
+
+Components of Object Storage
+============================
+
+The major components of Object Storage are:
+
+**Proxy Server**
+
+All REST requests to the UFO are routed through the Proxy Server.
+
+**Objects and Containers**
+
+An object is the basic storage entity and any optional metadata that
+represents the data you store. When you upload data, the data is stored
+as-is (with no compression or encryption).
+
+A container is a storage compartment for your data and provides a way
+for you to organize your data. Containers can be visualized as
+directories in a Linux system. Data must be stored in a container and
+hence objects are created within a container.
+
+It implements objects as files and directories under the container. The
+object name is a '/' separated path and UFO maps it to directories until
+the last name in the path, which is marked as a file. With this
+approach, objects can be accessed as files and directories from native
+GlusterFS (FUSE) or NFS mounts by providing the '/' separated path.
+
+**Accounts and Account Servers**
+
+The OpenStack Object Storage system is designed to be used by many
+different storage consumers. Each user is associated with one or more
+accounts and must identify themselves using an authentication system.
+While authenticating, users must provide the name of the account for
+which the authentication is requested.
+
+UFO implements accounts as GlusterFS volumes. So, when a user is granted
+read/write permission on an account, it means that that user has access
+to all the data available on that GlusterFS volume.
+
+**Authentication and Access Permissions**
+
+You must authenticate against an authentication service to receive
+OpenStack Object Storage connection parameters and an authentication
+token. The token must be passed in for all subsequent container or
+object operations. One authentication service that you can use as a
+middleware example is called `tempauth`.
+
+By default, each user has their own storage account and has full access
+to that account. Users must authenticate with their credentials as
+described above, but once authenticated they can manage containers and
+objects within that account. If a user wants to access the content from
+another account, they must have API access key or a session token
+provided by their authentication system.
+
+Advantages of using GlusterFS Unified File and Object Storage
+=============================================================
+
+The following are the advantages of using GlusterFS UFO:
+
+-   No limit on upload and download files sizes as compared to Open
+    Stack Swift which limits the object size to 5GB.
+
+-   A unified view of data across NAS and Object Storage technologies.
+
+-   Using GlusterFS's UFO has other advantages like the following:
+
+    -   High availability
+
+    -   Scalability
+
+    -   Replication
+
+    -   Elastic Volume management
+
+Preparing to Deploy Unified File and Object Storage
+===================================================
+
+This section provides information on pre-requisites and list of
+dependencies that will be installed during the installation of Unified
+File and Object Storage.
+
+Pre-requisites
+--------------
+
+GlusterFS's Unified File and Object Storage needs `user_xattr` support
+from the underlying disk file system. Use the following command to
+enable `user_xattr` for GlusterFS bricks backend:
+
+`# mount –o remount,user_xattr `
+
+For example,
+
+`# mount –o remount,user_xattr /dev/hda1 `
+
+Dependencies
+------------
+
+The following packages are installed on GlusterFS when you install
+Unified File and Object Storage:
+
+-   curl
+
+-   memcached
+
+-   openssl
+
+-   xfsprogs
+
+-   python2.6
+
+-   pyxattr
+
+-   python-configobj
+
+-   python-setuptools
+
+-   python-simplejson
+
+-   python-webob
+
+-   python-eventlet
+
+-   python-greenlet
+
+-   python-pastedeploy
+
+-   python-netifaces
+
+Installing and Configuring Unified File and Object Storage
+==========================================================
+
+This section provides instructions on how to install and configure
+Unified File and Object Storage in your storage environment.
+
+Installing Unified File and Object Storage
+------------------------------------------
+
+To install Unified File and Object Storage:
+
+1.  Download `rhel_install.sh` install script from [][] .
+
+2.  Run `rhel_install.sh` script using the following command:
+
+    `# sh rhel_install.sh`
+
+3.  Download `swift-1.4.5-1.noarch.rpm` and
+    `swift-plugin-1.0.-1.el6.noarch.rpm` files from [][].
+
+4.  Install `swift-1.4.5-1.noarch.rpm` and
+    `swift-plugin-1.0.-1.el6.noarch.rpm` using the following commands:
+
+    `# rpm -ivh swift-1.4.5-1.noarch.rpm`
+
+    `# rpm -ivh swift-plugin-1.0.-1.el6.noarch.rpm`
+
+    > **Note**
+    >
+    > You must repeat the above steps on all the machines on which you
+    > want to install Unified File and Object Storage. If you install
+    > the Unified File and Object Storage on multiple servers, you can
+    > use a load balancer like pound, nginx, and so on to distribute the
+    > request across the machines.
+
+Adding Users
+------------
+
+The authentication system allows the administrator to grant different
+levels of access to different users based on the requirement. The
+following are the types of user permissions:
+
+-   admin user
+
+-   normal user
+
+Admin user has read and write permissions on the account. By default, a
+normal user has no read or write permissions. A normal user can only
+authenticate itself to get a Auth-Token. Read or write permission are
+provided through ACLs by the admin users.
+
+Add a new user by adding the following entry in
+`/etc/swift/proxy-server.conf` file:
+
+`user_<account-name>_<user-name> = <password> [.admin]`
+
+For example,
+
+`user_test_tester = testing .admin`
+
+> **Note**
+>
+> During installation, the installation script adds few sample users to
+> the `proxy-server.conf` file. It is highly recommended that you remove
+> all the default sample user entries from the configuration file.
+
+For more information on setting ACLs, see ?.
+
+Configuring Proxy Server
+------------------------
+
+The Proxy Server is responsible for connecting to the rest of the
+OpenStack Object Storage architecture. For each request, it looks up the
+location of the account, container, or object in the ring and route the
+request accordingly. The public API is also exposed through the proxy
+server. When objects are streamed to or from an object server, they are
+streamed directly through the proxy server to or from the user – the
+proxy server does not spool them.
+
+The configurable options pertaining to proxy server are stored in
+`/etc/swift/proxy-server.conf`. The following is the sample
+`proxy-server.conf` file:
+
+    [app:proxy-server]
+    use = egg:swift#proxy
+    allow_account_management=true
+    account_autocreate=true
+
+    [filter:tempauth]
+    use = egg:swift#tempauth                                   user_admin_admin=admin.admin.reseller_admin
+    user_test_tester=testing.admin
+    user_test2_tester2=testing2.admin
+    user_test_tester3=testing3
+
+    [filter:healthcheck]
+    use = egg:swift#healthcheck 
+
+    [filter:cache]
+    use = egg:swift#memcache
+
+By default, GlusterFS's Unified File and Object Storage is configured to
+support HTTP protocol and uses temporary authentication to authenticate
+the HTTP requests.
+
+Configuring Authentication System
+---------------------------------
+
+Proxy server must be configured to authenticate using `
+          
+        `.
+
+Configuring Proxy Server for HTTPS
+----------------------------------
+
+By default, proxy server only handles HTTP request. To configure the
+proxy server to process HTTPS requests, perform the following steps:
+
+1.  Create self-signed cert for SSL using the following commands:
+
+        cd /etc/swift
+        openssl req -new -x509 -nodes -out cert.crt -keyout cert.key
+
+2.  Add the following lines to `/etc/swift/proxy-server.conf `under
+    [DEFAULT]
+
+        bind_port = 443
+         cert_file = /etc/swift/cert.crt
+         key_file = /etc/swift/cert.key
+
+3.  Restart the servers using the following commands:
+
+        swift-init main stop
+        swift-init main start
+
+The following are the configurable options:
+
+  Option       Default      Description
+  ------------ ------------ -------------------------------
+  bind\_ip     0.0.0.0      IP Address for server to bind
+  bind\_port   80           Port for server to bind
+  swift\_dir   /etc/swift   Swift configuration directory
+  workers      1            Number of workers to fork
+  user         swift        swift user
+  cert\_file                Path to the ssl .crt
+  key\_file                 Path to the ssl .key
+
+  : proxy-server.conf Default Options in the [DEFAULT] section
+
+  Option                          Default           Description
+  ------------------------------- ----------------- -----------------------------------------------------------------------------------------------------------
+  use                                               paste.deploy entry point for the container server. For most cases, this should be `egg:swift#container`.
+  log\_name                       proxy-server      Label used when logging
+  log\_facility                   LOG\_LOCAL0       Syslog log facility
+  log\_level                      INFO              Log level
+  log\_headers                    True              If True, log headers in each request
+  recheck\_account\_existence     60                Cache timeout in seconds to send memcached for account existence
+  recheck\_container\_existence   60                Cache timeout in seconds to send memcached for container existence
+  object\_chunk\_size             65536             Chunk size to read from object servers
+  client\_chunk\_size             65536             Chunk size to read from clients
+  memcache\_servers               127.0.0.1:11211   Comma separated list of memcached servers ip:port
+  node\_timeout                   10                Request timeout to external services
+  client\_timeout                 60                Timeout to read one chunk from a client
+  conn\_timeout                   0.5               Connection timeout to external services
+  error\_suppression\_interval    60                Time in seconds that must elapse since the last error for a node to be considered no longer error limited
+  error\_suppression\_limit       10                Error count to consider a node error limited
+  allow\_account\_management      false             Whether account `PUT`s and `DELETE`s are even callable
+
+  : proxy-server.conf Server Options in the [proxy-server] section
+
+Configuring Object Server
+-------------------------
+
+The Object Server is a very simple blob storage server that can store,
+retrieve, and delete objects stored on local devices. Objects are stored
+as binary files on the file system with metadata stored in the file’s
+extended attributes (xattrs). This requires that the underlying file
+system choice for object servers support xattrs on files.
+
+The configurable options pertaining Object Server are stored in the file
+`/etc/swift/object-server/1.conf`. The following is the sample
+`object-server/1.conf` file:
+
+    [DEFAULT]
+    devices = /srv/1/node
+    mount_check = false
+    bind_port = 6010
+    user = root
+    log_facility = LOG_LOCAL2
+
+    [pipeline:main]
+    pipeline = gluster object-server
+
+    [app:object-server]
+    use = egg:swift#object 
+
+    [filter:gluster]
+    use = egg:swift#gluster
+
+    [object-replicator]
+    vm_test_mode = yes
+
+    [object-updater]
+    [object-auditor]
+
+The following are the configurable options:
+
+  Option         Default      Description
+  -------------- ------------ ----------------------------------------------------------------------------------------------------
+  swift\_dir     /etc/swift   Swift configuration directory
+  devices        /srv/node    Mount parent directory where devices are mounted
+  mount\_check   true         Whether or not check if the devices are mounted to prevent accidentally writing to the root device
+  bind\_ip       0.0.0.0      IP Address for server to bind
+  bind\_port     6000         Port for server to bind
+  workers        1            Number of workers to fork
+
+  : object-server.conf Default Options in the [DEFAULT] section
+
+  Option                 Default         Description
+  ---------------------- --------------- ----------------------------------------------------------------------------------------------------
+  use                                    paste.deploy entry point for the object server. For most cases, this should be `egg:swift#object`.
+  log\_name              object-server   log name used when logging
+  log\_facility          LOG\_LOCAL0     Syslog log facility
+  log\_level             INFO            Logging level
+  log\_requests          True            Whether or not to log each request
+  user                   swift           swift user
+  node\_timeout          3               Request timeout to external services
+  conn\_timeout          0.5             Connection timeout to external services
+  network\_chunk\_size   65536           Size of chunks to read or write over the network
+  disk\_chunk\_size      65536           Size of chunks to read or write to disk
+  max\_upload\_time      65536           Maximum time allowed to upload an object
+  slow                   0               If \> 0, Minimum time in seconds for a `PUT` or `DELETE` request to complete
+
+  : object-server.conf Server Options in the [object-server] section
+
+Configuring Container Server
+----------------------------
+
+The Container Server’s primary job is to handle listings of objects. The
+listing is done by querying the GlusterFS mount point with path. This
+query returns a list of all files and directories present under that
+container.
+
+The configurable options pertaining to container server are stored in
+`/etc/swift/container-server/1.conf` file. The following is the sample
+`container-server/1.conf` file:
+
+    [DEFAULT]
+    devices = /srv/1/node
+    mount_check = false
+    bind_port = 6011
+    user = root
+    log_facility = LOG_LOCAL2
+
+    [pipeline:main]
+    pipeline = gluster container-server
+
+    [app:container-server]
+    use = egg:swift#container
+
+    [filter:gluster]
+    use = egg:swift#gluster
+
+    [container-replicator]
+    [container-updater]
+    [container-auditor]
+
+The following are the configurable options:
+
+  Option         Default      Description
+  -------------- ------------ ----------------------------------------------------------------------------------------------------
+  swift\_dir     /etc/swift   Swift configuration directory
+  devices        /srv/node    Mount parent directory where devices are mounted
+  mount\_check   true         Whether or not check if the devices are mounted to prevent accidentally writing to the root device
+  bind\_ip       0.0.0.0      IP Address for server to bind
+  bind\_port     6001         Port for server to bind
+  workers        1            Number of workers to fork
+  user           swift        Swift user
+
+  : container-server.conf Default Options in the [DEFAULT] section
+
+  Option          Default            Description
+  --------------- ------------------ ----------------------------------------------------------------------------------------------------------
+  use                                paste.deploy entry point for the container server. For most cases, this should be `egg:swift#container`.
+  log\_name       container-server   Label used when logging
+  log\_facility   LOG\_LOCAL0        Syslog log facility
+  log\_level      INFO               Logging level
+  node\_timeout   3                  Request timeout to external services
+  conn\_timeout   0.5                Connection timeout to external services
+
+  : container-server.conf Server Options in the [container-server]
+  section
+
+Configuring Account Server
+--------------------------
+
+The Account Server is very similar to the Container Server, except that
+it is responsible for listing of containers rather than objects. In UFO,
+each gluster volume is an account.
+
+The configurable options pertaining to account server are stored in
+`/etc/swift/account-server/1.conf` file. The following is the sample
+`account-server/1.conf` file:
+
+    [DEFAULT]
+    devices = /srv/1/node
+    mount_check = false
+    bind_port = 6012
+    user = root
+    log_facility = LOG_LOCAL2
+
+    [pipeline:main]
+    pipeline = gluster account-server
+
+    [app:account-server]
+    use = egg:swift#account
+
+    [filter:gluster]
+    use = egg:swift#gluster 
+
+    [account-replicator]
+    vm_test_mode = yes
+
+    [account-auditor]
+    [account-reaper]
+
+The following are the configurable options:
+
+  Option         Default      Description
+  -------------- ------------ ----------------------------------------------------------------------------------------------------
+  swift\_dir     /etc/swift   Swift configuration directory
+  devices        /srv/node    mount parent directory where devices are mounted
+  mount\_check   true         Whether or not check if the devices are mounted to prevent accidentally writing to the root device
+  bind\_ip       0.0.0.0      IP Address for server to bind
+  bind\_port     6002         Port for server to bind
+  workers        1            Number of workers to fork
+  user           swift        Swift user
+
+  : account-server.conf Default Options in the [DEFAULT] section
+
+  Option          Default          Description
+  --------------- ---------------- ----------------------------------------------------------------------------------------------------------
+  use                              paste.deploy entry point for the container server. For most cases, this should be `egg:swift#container`.
+  log\_name       account-server   Label used when logging
+  log\_facility   LOG\_LOCAL0      Syslog log facility
+  log\_level      INFO             Logging level
+
+  : account-server.conf Server Options in the [account-server] section
+
+Starting and Stopping Server
+----------------------------
+
+You must start the server manually when system reboots and whenever you
+update/modify the configuration files.
+
+-   To start the server, enter the following command:
+
+    `# swift_init main start`
+
+-   To stop the server, enter the following command:
+
+    `# swift_init main stop`
+
+Working with Unified File and Object Storage
+============================================
+
+This section describes the REST API for administering and managing
+Object Storage. All requests will be directed to the host and URL
+described in the `X-Storage-URL HTTP` header obtained during successful
+authentication.
+
+Configuring Authenticated Access
+--------------------------------
+
+Authentication is the process of proving identity to the system. To use
+the REST interface, you must obtain an authorization token using GET
+method and supply it with v1.0 as the path.
+
+Each REST request against the Object Storage system requires the
+addition of a specific authorization token HTTP x-header, defined as
+X-Auth-Token. The storage URL and authentication token are returned in
+the headers of the response.
+
+-   To authenticate, run the following command:
+
+        GET auth/v1.0 HTTP/1.1
+        Host: <auth URL>
+        X-Auth-User: <account name>:<user name>
+        X-Auth-Key: <user-Password>
+
+    For example,
+
+        GET auth/v1.0 HTTP/1.1
+        Host: auth.example.com
+        X-Auth-User: test:tester
+        X-Auth-Key: testing
+
+        HTTP/1.1 200 OK
+        X-Storage-Url: https:/example.storage.com:443/v1/AUTH_test
+        X-Storage-Token: AUTH_tkde3ad38b087b49bbbac0494f7600a554
+        X-Auth-Token: AUTH_tkde3ad38b087b49bbbac0494f7600a554
+        Content-Length: 0
+        Date: Wed, 10 jul 2011 06:11:51 GMT
+
+    To authenticate access using cURL (for the above example), run the
+    following command:
+
+        curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass:testing' -k
+        https://auth.example.com:443/auth/v1.0
+
+    The X-Auth-Url has to be parsed and used in the connection and
+    request line of all subsequent requests to the server. In the
+    example output, users connecting to server will send most
+    container/object requests with a host header of example.storage.com
+    and the request line's version and account as v1/AUTH\_test.
+
+> **Note**
+>
+> The authentication tokens are valid for a 24 hour period.
+
+Working with Accounts
+---------------------
+
+This section describes the list of operations you can perform at the
+account level of the URL.
+
+### Displaying Container Information
+
+You can list the objects of a specific container, or all containers, as
+needed using GET command. You can use the following optional parameters
+with GET request to refine the results:
+
+  Parameter   Description
+  ----------- --------------------------------------------------------------------------
+  limit       Limits the number of results to at most *n* value.
+  marker      Returns object names greater in value than the specified marker.
+  format      Specify either json or xml to return the respective serialized response.
+
+**To display container information**
+
+-   List all the containers of an account using the following command:
+
+        GET /<apiversion>/<account> HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <authentication-token-key>
+
+    For example,
+
+        GET /v1/AUTH_test HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+
+        HTTP/1.1 200 Ok
+        Date: Wed, 13 Jul 2011 16:32:21 GMT
+        Server: Apache
+        Content-Type: text/plain; charset=UTF-8
+        Content-Length: 39
+
+        songs
+        movies
+        documents
+        reports
+
+To display container information using cURL (for the above example), run
+the following command:
+
+    curl -v -X GET -H 'X-Auth-Token: AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+    https://example.storage.com:443/v1/AUTH_test -k
+
+### Displaying Account Metadata Information
+
+You can issue HEAD command to the storage service to view the number of
+containers and the total bytes stored in the account.
+
+-   To display containers and storage used, run the following command:
+
+        HEAD /<apiversion>/<account> HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <authentication-token-key>
+
+    For example,
+
+        HEAD /v1/AUTH_test HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+
+        HTTP/1.1 204 No Content
+        Date: Wed, 13 Jul 2011 16:52:21 GMT
+        Server: Apache
+        X-Account-Container-Count: 4
+        X-Account-Total-Bytes-Used: 394792
+
+    To display account metadata information using cURL (for the above
+    example), run the following command:
+
+        curl -v -X HEAD -H 'X-Auth-Token:
+        AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+        https://example.storage.com:443/v1/AUTH_test -k
+
+Working with Containers
+-----------------------
+
+This section describes the list of operations you can perform at the
+container level of the URL.
+
+### Creating Containers
+
+You can use PUT command to create containers. Containers are the storage
+folders for your data. The URL encoded name must be less than 256 bytes
+and cannot contain a forward slash '/' character.
+
+-   To create a container, run the following command:
+
+        PUT /<apiversion>/<account>/<container>/ HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <authentication-token-key>
+
+    For example,
+
+        PUT /v1/AUTH_test/pictures/ HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+        HTTP/1.1 201 Created
+
+        Date: Wed, 13 Jul 2011 17:32:21 GMT
+        Server: Apache
+        Content-Type: text/plain; charset=UTF-8
+
+    To create container using cURL (for the above example), run the
+    following command:
+
+        curl -v -X PUT -H 'X-Auth-Token:
+        AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+        https://example.storage.com:443/v1/AUTH_test/pictures -k
+
+    The status code of 201 (Created) indicates that you have
+    successfully created the container. If a container with same is
+    already existed, the status code of 202 is displayed.
+
+### Displaying Objects of a Container
+
+You can list the objects of a container using GET command. You can use
+the following optional parameters with GET request to refine the
+results:
+
+  Parameter   Description
+  ----------- --------------------------------------------------------------------------------------------------------------
+  limit       Limits the number of results to at most *n* value.
+  marker      Returns object names greater in value than the specified marker.
+  prefix      Displays the results limited to object names beginning with the substring x. beginning with the substring x.
+  path        Returns the object names nested in the pseudo path.
+  format      Specify either json or xml to return the respective serialized response.
+  delimiter   Returns all the object names nested in the container.
+
+To display objects of a container
+
+-   List objects of a specific container using the following command:
+
+<!-- -->
+
+    GET /<apiversion>/<account>/<container>[parm=value] HTTP/1.1
+    Host: <storage URL>
+    X-Auth-Token: <authentication-token-key>
+
+For example,
+
+    GET /v1/AUTH_test/images HTTP/1.1
+    Host: example.storage.com
+    X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+
+    HTTP/1.1 200 Ok
+    Date: Wed, 13 Jul 2011 15:42:21 GMT
+    Server: Apache
+    Content-Type: text/plain; charset=UTF-8
+    Content-Length: 139
+
+    sample file.jpg
+    test-file.pdf
+    You and Me.pdf
+    Puddle of Mudd.mp3
+    Test Reports.doc
+
+To display objects of a container using cURL (for the above example),
+run the following command:
+
+    curl -v -X GET-H 'X-Auth-Token: AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+    https://example.storage.com:443/v1/AUTH_test/images -k
+
+### Displaying Container Metadata Information
+
+You can issue HEAD command to the storage service to view the number of
+objects in a container and the total bytes of all the objects stored in
+the container.
+
+-   To display list of objects and storage used, run the following
+    command:
+
+        HEAD /<apiversion>/<account>/<container> HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <authentication-token-key>
+
+    For example,
+
+        HEAD /v1/AUTH_test/images HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+
+        HTTP/1.1 204 No Content
+        Date: Wed, 13 Jul 2011 19:52:21 GMT
+        Server: Apache
+        X-Account-Object-Count: 8
+        X-Container-Bytes-Used: 472
+
+    To display list of objects and storage used in a container using
+    cURL (for the above example), run the following command:
+
+        curl -v -X HEAD -H 'X-Auth-Token:
+        AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+        https://example.storage.com:443/v1/AUTH_test/images -k
+
+### Deleting Container
+
+You can use DELETE command to permanently delete containers. The
+container must be empty before it can be deleted.
+
+You can issue HEAD command to determine if it contains any objects.
+
+-   To delete a container, run the following command:
+
+        DELETE /<apiversion>/<account>/<container>/ HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <authentication-token-key>
+
+    For example,
+
+        DELETE /v1/AUTH_test/pictures HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+
+        HTTP/1.1 204 No Content
+        Date: Wed, 13 Jul 2011 17:52:21 GMT
+        Server: Apache
+        Content-Length: 0
+        Content-Type: text/plain; charset=UTF-8
+
+    To delete a container using cURL (for the above example), run the
+    following command:
+
+        curl -v -X DELETE -H 'X-Auth-Token:
+        AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+        https://example.storage.com:443/v1/AUTH_test/pictures -k
+
+    The status code of 204 (No Content) indicates that you have
+    successfully deleted the container. If that container does not
+    exist, the status code 404 (Not Found) is displayed, and if the
+    container is not empty, the status code 409 (Conflict) is displayed.
+
+### Updating Container Metadata
+
+You can update the metadata of container using POST operation, metadata
+keys should be prefixed with 'x-container-meta'.
+
+-   To update the metadata of the object, run the following command:
+
+        POST /<apiversion>/<account>/<container> HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <Authentication-token-key>
+        X-Container-Meta-<key>: <new value>
+        X-Container-Meta-<key>: <new value>
+
+    For example,
+
+        POST /v1/AUTH_test/images HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+        X-Container-Meta-Zoo: Lion
+        X-Container-Meta-Home: Dog
+
+        HTTP/1.1 204 No Content
+        Date: Wed, 13 Jul 2011 20:52:21 GMT
+        Server: Apache
+        Content-Type: text/plain; charset=UTF-8
+
+    To update the metadata of the object using cURL (for the above
+    example), run the following command:
+
+        curl -v -X POST -H 'X-Auth-Token:
+        AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+        https://example.storage.com:443/v1/AUTH_test/images -H ' X-Container-Meta-Zoo: Lion' -H 'X-Container-Meta-Home: Dog' -k
+
+    The status code of 204 (No Content) indicates the container's
+    metadata is updated successfully. If that object does not exist, the
+    status code 404 (Not Found) is displayed.
+
+### Setting ACLs on Container
+
+You can set the container access control list by using POST command on
+container with `x- container-read` and` x-container-write` keys.
+
+The ACL format is `[item[,item...]]`. Each item can be a group name to
+give access to or a referrer designation to grant or deny based on the
+HTTP Referer header.
+
+The referrer designation format is:` .r:[-]value`.
+
+The .r can also be `.ref, .referer, `or .`referrer`; though it will be
+shortened to.r for decreased character count usage. The value can be `*`
+to specify any referrer host is allowed access. The leading minus sign
+(-) indicates referrer hosts that should be denied access.
+
+Examples of valid ACLs:
+
+    .r:*
+    .r:*,bobs_account,sues_account:sue
+    bobs_account,sues_account:sue
+
+Examples of invalid ACLs:
+
+    .r:
+    .r:-
+
+By default, allowing read access via `r `will not allow listing objects
+in the container but allows retrieving objects from the container. To
+turn on listings, use the .`rlistings` directive. Also, `.r`
+designations are not allowed in headers whose names include the word
+write.
+
+For example, to set all the objects access rights to "public" inside the
+container using cURL (for the above example), run the following command:
+
+    curl -v -X POST -H 'X-Auth-Token:
+    AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+    https://example.storage.com:443/v1/AUTH_test/images
+    -H 'X-Container-Read: .r:*' -k
+
+Working with Objects
+--------------------
+
+An object represents the data and any metadata for the files stored in
+the system. Through the REST interface, metadata for an object can be
+included by adding custom HTTP headers to the request and the data
+payload as the request body. Objects name should not exceed 1024 bytes
+after URL encoding.
+
+This section describes the list of operations you can perform at the
+object level of the URL.
+
+### Creating or Updating Object
+
+You can use PUT command to write or update an object's content and
+metadata.
+
+You can verify the data integrity by including an MD5checksum for the
+object's data in the ETag header. ETag header is optional and can be
+used to ensure that the object's contents are stored successfully in the
+storage system.
+
+You can assign custom metadata to objects by including additional HTTP
+headers on the PUT request. The objects created with custom metadata via
+HTTP headers are identified with the`X-Object- Meta`- prefix.
+
+-   To create or update an object, run the following command:
+
+        PUT /<apiversion>/<account>/<container>/<object> HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <authentication-token-key>
+        ETag: da1e100dc9e7becc810986e37875ae38
+        Content-Length: 342909
+        X-Object-Meta-PIN: 2343
+
+    For example,
+
+        PUT /v1/AUTH_test/pictures/dog HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+        ETag: da1e100dc9e7becc810986e37875ae38
+
+        HTTP/1.1 201 Created
+        Date: Wed, 13 Jul 2011 18:32:21 GMT
+        Server: Apache
+        ETag: da1e100dc9e7becc810986e37875ae38
+        Content-Length: 0
+        Content-Type: text/plain; charset=UTF-8
+
+    To create or update an object using cURL (for the above example),
+    run the following command:
+
+        curl -v -X PUT -H 'X-Auth-Token:
+        AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+        https://example.storage.com:443/v1/AUTH_test/pictures/dog -H 'Content-
+        Length: 0' -k
+
+    The status code of 201 (Created) indicates that you have
+    successfully created or updated the object. If there is a missing
+    content-Length or Content-Type header in the request, the status
+    code of 412 (Length Required) is displayed. (Optionally) If the MD5
+    checksum of the data written to the storage system does not match
+    the ETag value, the status code of 422 (Unprocessable Entity) is
+    displayed.
+
+#### Chunked Transfer Encoding
+
+You can upload data without knowing the size of the data to be uploaded.
+You can do this by specifying an HTTP header of Transfer-Encoding:
+chunked and without using a Content-Length header.
+
+You can use this feature while doing a DB dump, piping the output
+through gzip, and then piping the data directly into Object Storage
+without having to buffer the data to disk to compute the file size.
+
+-   To create or update an object, run the following command:
+
+        PUT /<apiversion>/<account>/<container>/<object> HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <authentication-token-key>
+        Transfer-Encoding: chunked
+        X-Object-Meta-PIN: 2343
+
+    For example,
+
+        PUT /v1/AUTH_test/pictures/cat HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+        Transfer-Encoding: chunked
+        X-Object-Meta-PIN: 2343
+        19
+        A bunch of data broken up
+        D
+        into chunks.
+        0
+
+### Copying Object
+
+You can copy object from one container to another or add a new object
+and then add reference to designate the source of the data from another
+container.
+
+**To copy object from one container to another**
+
+-   To add a new object and designate the source of the data from
+    another container, run the following command:
+
+        COPY /<apiversion>/<account>/<container>/<sourceobject> HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: < authentication-token-key>
+        Destination: /<container>/<destinationobject>
+
+    For example,
+
+        COPY /v1/AUTH_test/images/dogs HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+        Destination: /photos/cats
+
+        HTTP/1.1 201 Created
+        Date: Wed, 13 Jul 2011 18:32:21 GMT
+        Server: Apache
+        Content-Length: 0
+        Content-Type: text/plain; charset=UTF-8
+
+    To copy an object using cURL (for the above example), run the
+    following command:
+
+        curl -v -X COPY -H 'X-Auth-Token:
+        AUTH_tkde3ad38b087b49bbbac0494f7600a554' -H 'Destination: /photos/cats' -k https://example.storage.com:443/v1/AUTH_test/images/dogs
+
+    The status code of 201 (Created) indicates that you have
+    successfully copied the object. If there is a missing content-Length
+    or Content-Type header in the request, the status code of 412
+    (Length Required) is displayed.
+
+    You can also use PUT command to copy object by using additional
+    header `X-Copy-From: container/obj`.
+
+-   To use PUT command to copy an object, run the following command:
+
+        PUT /v1/AUTH_test/photos/cats HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+        X-Copy-From: /images/dogs
+
+        HTTP/1.1 201 Created
+        Date: Wed, 13 Jul 2011 18:32:21 GMT
+        Server: Apache
+        Content-Type: text/plain; charset=UTF-8
+
+    To copy an object using cURL (for the above example), run the
+    following command:
+
+        curl -v -X PUT -H 'X-Auth-Token: AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+        -H 'X-Copy-From: /images/dogs' –k
+        https://example.storage.com:443/v1/AUTH_test/images/cats
+
+    The status code of 201 (Created) indicates that you have
+    successfully copied the object.
+
+### Displaying Object Information
+
+You can issue GET command on an object to view the object data of the
+object.
+
+-   To display the content of an object run the following command:
+
+        GET /<apiversion>/<account>/<container>/<object> HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <Authentication-token-key>
+
+    For example,
+
+        GET /v1/AUTH_test/images/cat HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+
+        HTTP/1.1 200 Ok
+        Date: Wed, 13 Jul 2011 23:52:21 GMT
+        Server: Apache
+        Last-Modified: Thu, 14 Jul 2011 13:40:18 GMT
+        ETag: 8a964ee2a5e88be344f36c22562a6486
+        Content-Length: 534210
+        [.........]
+
+    To display the content of an object using cURL (for the above
+    example), run the following command:
+
+        curl -v -X GET -H 'X-Auth-Token:
+        AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+        https://example.storage.com:443/v1/AUTH_test/images/cat -k
+
+    The status code of 200 (Ok) indicates the object's data is displayed
+    successfully. If that object does not exist, the status code 404
+    (Not Found) is displayed.
+
+### Displaying Object Metadata
+
+You can issue HEAD command on an object to view the object metadata and
+other standard HTTP headers. You must send only authorization token as
+header.
+
+-   To display the metadata of the object, run the following command:
+
+<!-- -->
+
+    HEAD /<apiversion>/<account>/<container>/<object> HTTP/1.1
+    Host: <storage URL>
+    X-Auth-Token: <Authentication-token-key>
+
+For example,
+
+    HEAD /v1/AUTH_test/images/cat HTTP/1.1
+    Host: example.storage.com
+    X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+
+    HTTP/1.1 204 No Content
+    Date: Wed, 13 Jul 2011 21:52:21 GMT
+    Server: Apache
+    Last-Modified: Thu, 14 Jul 2011 13:40:18 GMT
+    ETag: 8a964ee2a5e88be344f36c22562a6486
+    Content-Length: 512000
+    Content-Type: text/plain; charset=UTF-8
+    X-Object-Meta-House: Cat
+    X-Object-Meta-Zoo: Cat
+    X-Object-Meta-Home: Cat
+    X-Object-Meta-Park: Cat
+
+To display the metadata of the object using cURL (for the above
+example), run the following command:
+
+    curl -v -X HEAD -H 'X-Auth-Token:
+    AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+    https://example.storage.com:443/v1/AUTH_test/images/cat -k
+
+The status code of 204 (No Content) indicates the object's metadata is
+displayed successfully. If that object does not exist, the status code
+404 (Not Found) is displayed.
+
+### Updating Object Metadata
+
+You can issue POST command on an object name only to set or overwrite
+arbitrary key metadata. You cannot change the object's other headers
+such as Content-Type, ETag and others using POST operation. The POST
+command will delete all the existing metadata and replace it with the
+new arbitrary key metadata.
+
+You must prefix **X-Object-Meta-** to the key names.
+
+-   To update the metadata of an object, run the following command:
+
+        POST /<apiversion>/<account>/<container>/<object> HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <Authentication-token-key>
+        X-Object-Meta-<key>: <new value>
+        X-Object-Meta-<key>: <new value>
+
+    For example,
+
+        POST /v1/AUTH_test/images/cat HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+        X-Object-Meta-Zoo: Lion
+        X-Object-Meta-Home: Dog
+
+        HTTP/1.1 202 Accepted
+        Date: Wed, 13 Jul 2011 22:52:21 GMT
+        Server: Apache
+        Content-Length: 0
+        Content-Type: text/plain; charset=UTF-8
+
+    To update the metadata of an object using cURL (for the above
+    example), run the following command:
+
+        curl -v -X POST -H 'X-Auth-Token:
+        AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+        https://example.storage.com:443/v1/AUTH_test/images/cat -H ' X-Object-
+        Meta-Zoo: Lion' -H 'X-Object-Meta-Home: Dog' -k
+
+    The status code of 202 (Accepted) indicates that you have
+    successfully updated the object's metadata. If that object does not
+    exist, the status code 404 (Not Found) is displayed.
+
+### Deleting Object
+
+You can use DELETE command to permanently delete the object.
+
+The DELETE command on an object will be processed immediately and any
+subsequent operations like GET, HEAD, POST, or DELETE on the object will
+display 404 (Not Found) error.
+
+-   To delete an object, run the following command:
+
+        DELETE /<apiversion>/<account>/<container>/<object> HTTP/1.1
+        Host: <storage URL>
+        X-Auth-Token: <Authentication-token-key>
+
+    For example,
+
+        DELETE /v1/AUTH_test/pictures/cat HTTP/1.1
+        Host: example.storage.com
+        X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554
+
+        HTTP/1.1 204 No Content
+        Date: Wed, 13 Jul 2011 20:52:21 GMT
+        Server: Apache
+        Content-Type: text/plain; charset=UTF-8
+
+    To delete an object using cURL (for the above example), run the
+    following command:
+
+        curl -v -X DELETE -H 'X-Auth-Token:
+        AUTH_tkde3ad38b087b49bbbac0494f7600a554'
+        https://example.storage.com:443/v1/AUTH_test/pictures/cat -k
+
+    The status code of 204 (No Content) indicates that you have
+    successfully deleted the object. If that object does not exist, the
+    status code 404 (Not Found) is displayed.
+
+  []: http://download.gluster.com/pub/gluster/glusterfs/3.2/UFO/
diff --git a/doc/admin-guide/en-US/markdown/admin_commandref.md b/doc/admin-guide/en-US/markdown/admin_commandref.md
new file mode 100644
index 000000000..4ff05f4ef
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_commandref.md
@@ -0,0 +1,180 @@
+Command Reference
+=================
+
+This section describes the available commands and includes the following
+section:
+
+-   gluster Command
+
+    Gluster Console Manager (command line interpreter)
+
+-   glusterd Daemon
+
+    Gluster elastic volume management daemon
+
+gluster Command
+===============
+
+**NAME**
+
+gluster - Gluster Console Manager (command line interpreter)
+
+**SYNOPSIS**
+
+To run the program and display the gluster prompt:
+
+**gluster**
+
+To specify a command directly: gluster [COMMANDS] [OPTIONS]
+
+**DESCRIPTION**
+
+The Gluster Console Manager is a command line utility for elastic volume
+management. You can run the gluster command on any export server. The
+command enables administrators to perform cloud operations such as
+creating, expanding, shrinking, rebalancing, and migrating volumes
+without needing to schedule server downtime.
+
+**COMMANDS**
+
+  ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+  Command                                                                                                    Description
+  ---------------------------------------------------------------------------------------------------------- ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+  **Volume**
+
+  volume info [all | VOLNAME]                                                                                Displays information about all volumes, or the specified volume.
+
+  volume create NEW-VOLNAME [stripe COUNT] [replica COUNT] [transport tcp | rdma | tcp,rdma] NEW-BRICK ...   Creates a new volume of the specified type using the specified bricks and transport type (the default transport type is tcp).
+
+  volume delete VOLNAME                                                                                      Deletes the specified volume.
+
+  volume start VOLNAME                                                                                       Starts the specified volume.
+
+  volume stop VOLNAME [force]                                                                                Stops the specified volume.
+
+  volume rename VOLNAME NEW-VOLNAME                                                                          Renames the specified volume.
+
+  volume help                                                                                                Displays help for the volume command.
+
+  **Brick**
+
+  volume add-brick VOLNAME NEW-BRICK ...                                                                     Adds the specified brick to the specified volume.
+
+  volume replace-brick VOLNAME (BRICK NEW-BRICK) start | pause | abort | status                              Replaces the specified brick.
+
+  volume remove-brick VOLNAME [(replica COUNT)|(stripe COUNT)] BRICK ...                                     Removes the specified brick from the specified volume.
+
+  **Rebalance**
+
+  volume rebalance VOLNAME start                                                                             Starts rebalancing the specified volume.
+
+  volume rebalance VOLNAME stop                                                                              Stops rebalancing the specified volume.
+
+  volume rebalance VOLNAME status                                                                            Displays the rebalance status of the specified volume.
+
+  **Log**
+
+  volume log filename VOLNAME [BRICK] DIRECTORY                                                              Sets the log directory for the corresponding volume/brick.
+
+  volume log rotate VOLNAME [BRICK]                                                                          Rotates the log file for corresponding volume/brick.
+
+  volume log locate VOLNAME [BRICK]                                                                          Locates the log file for corresponding volume/brick.
+
+  **Peer**
+
+  peer probe HOSTNAME                                                                                        Probes the specified peer.
+
+  peer detach HOSTNAME                                                                                       Detaches the specified peer.
+
+  peer status                                                                                                Displays the status of peers.
+
+  peer help                                                                                                  Displays help for the peer command.
+
+  **Geo-replication**
+
+  volume geo-replication MASTER SLAVE start                                                                  Start geo-replication between the hosts specified by MASTER and SLAVE. You can specify a local master volume as :VOLNAME.
+                                                                                                             
+                                                                                                             You can specify a local slave volume as :VOLUME and a local slave directory as /DIRECTORY/SUB-DIRECTORY. You can specify a remote slave volume as DOMAIN::VOLNAME and a remote slave directory as DOMAIN:/DIRECTORY/SUB-DIRECTORY.
+
+  volume geo-replication MASTER SLAVE stop                                                                   Stop geo-replication between the hosts specified by MASTER and SLAVE. You can specify a local master volume as :VOLNAME and a local master directory as /DIRECTORY/SUB-DIRECTORY.
+                                                                                                             
+                                                                                                             You can specify a local slave volume as :VOLNAME and a local slave directory as /DIRECTORY/SUB-DIRECTORY. You can specify a remote slave volume as DOMAIN::VOLNAME and a remote slave directory as DOMAIN:/DIRECTORY/SUB-DIRECTORY.
+
+  volume geo-replication MASTER SLAVE config [options]                                                                                                                                                                                                                                                                                             Configure geo-replication options between the hosts specified by MASTER and SLAVE.
+
+  gluster-command COMMAND                                                                                    The path where the gluster command is installed.
+
+  gluster-log-level LOGFILELEVEL                                                                             The log level for gluster processes.
+
+  log-file LOGFILE                                                                                           The path to the geo-replication log file.
+
+  log-level LOGFILELEVEL                                                                                     The log level for geo-replication.
+
+  remote-gsyncd COMMAND                                                                                      The path where the gsyncd binary is installed on the remote machine.
+
+  ssh-command COMMAND                                                                                        The ssh command to use to connect to the remote machine (the default is ssh).
+
+  rsync-command COMMAND                                                                                      The rsync command to use for synchronizing the files (the default is rsync).
+
+  volume\_id= UID                                                                                            The command to delete the existing master UID for the intermediate/slave node.
+
+  timeout SECONDS                                                                                            The timeout period.
+
+  sync-jobs N                                                                                                The number of simultaneous files/directories that can be synchronized.
+
+                                                                                                             ignore-deletes                                                                                                                                                                                                                        If this option is set to 1, a file deleted on master will not trigger a delete operation on the slave. Hence, the slave will remain as a superset of the master and can be used to recover the master in case of crash and/or accidental delete.
+
+  **Other**
+
+  help                                                                                                                                                                                                                                                                                                                                             Display the command options.
+
+  quit                                                                                                                                                                                                                                                                                                                                             Exit the gluster command line interface.
+  ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+**FILES**
+
+/var/lib/glusterd/\*
+
+**SEE ALSO**
+
+fusermount(1), mount.glusterfs(8), glusterfs-volgen(8), glusterfs(8),
+glusterd(8)
+
+glusterd Daemon
+===============
+
+**NAME**
+
+glusterd - Gluster elastic volume management daemon
+
+**SYNOPSIS**
+
+glusterd [OPTION...]
+
+**DESCRIPTION**
+
+The glusterd daemon is used for elastic volume management. The daemon
+must be run on all export servers.
+
+**OPTIONS**
+
+  Option                              Description
+  ----------------------------------- ----------------------------------------------------------------------------------------------------------------
+  **Basic**
+  -l=LOGFILE, --log-file=LOGFILE      Files to use for logging (the default is /usr/local/var/log/glusterfs/glusterfs.log).
+  -L=LOGLEVEL, --log-level=LOGLEVEL   Logging severity. Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is INFO).
+  --debug                             Runs the program in debug mode. This option sets --no-daemon, --log-level to DEBUG, and --log-file to console.
+  -N, --no-daemon                     Runs the program in the foreground.
+  **Miscellaneous**
+  -?, --help                          Displays this help.
+  --usage                             Displays a short usage message.
+  -V, --version                       Prints the program version.
+
+**FILES**
+
+/var/lib/glusterd/\*
+
+**SEE ALSO**
+
+fusermount(1), mount.glusterfs(8), glusterfs-volgen(8), glusterfs(8),
+gluster(8)
diff --git a/doc/admin-guide/en-US/markdown/admin_console.md b/doc/admin-guide/en-US/markdown/admin_console.md
new file mode 100644
index 000000000..9b69de02d
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_console.md
@@ -0,0 +1,51 @@
+Using the Gluster Console Manager – Command Line Utility
+========================================================
+
+The Gluster Console Manager is a single command line utility that
+simplifies configuration and management of your storage environment. The
+Gluster Console Manager is similar to the LVM (Logical Volume Manager)
+CLI or ZFS Command Line Interface, but across multiple storage servers.
+You can use the Gluster Console Manager online, while volumes are
+mounted and active. Gluster automatically synchronizes volume
+configuration information across all Gluster servers.
+
+Using the Gluster Console Manager, you can create new volumes, start
+volumes, and stop volumes, as required. You can also add bricks to
+volumes, remove bricks from existing volumes, as well as change
+translator settings, among other operations.
+
+You can also use the commands to create scripts for automation, as well
+as use the commands as an API to allow integration with third-party
+applications.
+
+**Running the Gluster Console Manager**
+
+You can run the Gluster Console Manager on any GlusterFS server either
+by invoking the commands or by running the Gluster CLI in interactive
+mode. You can also use the gluster command remotely using SSH.
+
+-   To run commands directly:
+
+    ` # gluster peer `
+
+    For example:
+
+    ` # gluster peer status `
+
+-   To run the Gluster Console Manager in interactive mode
+
+    `# gluster`
+
+    You can execute gluster commands from the Console Manager prompt:
+
+    ` gluster> `
+
+    For example, to view the status of the peer server:
+
+    \# `gluster `
+
+    `gluster > peer status `
+
+    Display the status of the peer.
+
+
diff --git a/doc/admin-guide/en-US/markdown/admin_directory_Quota.md b/doc/admin-guide/en-US/markdown/admin_directory_Quota.md
new file mode 100644
index 000000000..09c757781
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_directory_Quota.md
@@ -0,0 +1,172 @@
+Managing Directory Quota
+========================
+
+Directory quotas in GlusterFS allow you to set limits on usage of disk
+space by directories or volumes. The storage administrators can control
+the disk space utilization at the directory and/or volume levels in
+GlusterFS by setting limits to allocatable disk space at any level in
+the volume and directory hierarchy. This is particularly useful in cloud
+deployments to facilitate utility billing model.
+
+> **Note**
+>
+> For now, only Hard limit is supported. Here, the limit cannot be
+> exceeded and attempts to use more disk space or inodes beyond the set
+> limit will be denied.
+
+System administrators can also monitor the resource utilization to limit
+the storage for the users depending on their role in the organization.
+
+You can set the quota at the following levels:
+
+-   Directory level – limits the usage at the directory level
+
+-   Volume level – limits the usage at the volume level
+
+> **Note**
+>
+> You can set the disk limit on the directory even if it is not created.
+> The disk limit is enforced immediately after creating that directory.
+> For more information on setting disk limit, see ?.
+
+Enabling Quota
+==============
+
+You must enable Quota to set disk limits.
+
+**To enable quota**
+
+-   Enable the quota using the following command:
+
+    `# gluster volume quota  enable `
+
+    For example, to enable quota on test-volume:
+
+        # gluster volume quota test-volume enable
+        Quota is enabled on /test-volume
+
+Disabling Quota
+===============
+
+You can disable Quota, if needed.
+
+**To disable quota:**
+
+-   Disable the quota using the following command:
+
+    `# gluster volume quota  disable `
+
+    For example, to disable quota translator on test-volume:
+
+        # gluster volume quota test-volume disable
+        Quota translator is disabled on /test-volume
+
+Setting or Replacing Disk Limit
+===============================
+
+You can create new directories in your storage environment and set the
+disk limit or set disk limit for the existing directories. The directory
+name should be relative to the volume with the export directory/mount
+being treated as "/".
+
+**To set or replace disk limit**
+
+-   Set the disk limit using the following command:
+
+    `# gluster volume quota  limit-usage /`
+
+    For example, to set limit on data directory on test-volume where
+    data is a directory under the export directory:
+
+        # gluster volume quota test-volume limit-usage /data 10GB
+        Usage limit has been set on /data
+
+    > **Note**
+    >
+    > In a multi-level directory hierarchy, the strictest disk limit
+    > will be considered for enforcement.
+
+Displaying Disk Limit Information
+=================================
+
+You can display disk limit information on all the directories on which
+the limit is set.
+
+**To display disk limit information**
+
+-   Display disk limit information of all the directories on which limit
+    is set, using the following command:
+
+    `# gluster volume quota  list`
+
+    For example, to see the set disks limit on test-volume:
+
+        # gluster volume quota test-volume list
+
+
+        /Test/data    10 GB       6 GB
+        /Test/data1   10 GB       4 GB
+
+-   Display disk limit information on a particular directory on which
+    limit is set, using the following command:
+
+    `# gluster volume quota  list `
+
+    For example, to see the set limit on /data directory of test-volume:
+
+        # gluster volume quota test-volume list /data
+
+
+        /Test/data    10 GB       6 GB
+
+Updating Memory Cache Size
+==========================
+
+For performance reasons, quota caches the directory sizes on client. You
+can set timeout indicating the maximum valid duration of directory sizes
+in cache, from the time they are populated.
+
+For example: If there are multiple clients writing to a single
+directory, there are chances that some other client might write till the
+quota limit is exceeded. However, this new file-size may not get
+reflected in the client till size entry in cache has become stale
+because of timeout. If writes happen on this client during this
+duration, they are allowed even though they would lead to exceeding of
+quota-limits, since size in cache is not in sync with the actual size.
+When timeout happens, the size in cache is updated from servers and will
+be in sync and no further writes will be allowed. A timeout of zero will
+force fetching of directory sizes from server for every operation that
+modifies file data and will effectively disables directory size caching
+on client side.
+
+**To update the memory cache size**
+
+-   Update the memory cache size using the following command:
+
+    `# gluster volume set  features.quota-timeout`
+
+    For example, to update the memory cache size for every 5 seconds on
+    test-volume:
+
+        # gluster volume set test-volume features.quota-timeout 5
+        Set volume successful
+
+Removing Disk Limit
+===================
+
+You can remove set disk limit, if you do not want quota anymore.
+
+**To remove disk limit**
+
+-   Remove disk limit set on a particular directory using the following
+    command:
+
+    `# gluster volume quota  remove `
+
+    For example, to remove the disk limit on /data directory of
+    test-volume:
+
+        # gluster volume quota test-volume remove /data
+        Usage limit set on /data is removed
+
+
diff --git a/doc/admin-guide/en-US/markdown/admin_geo-replication.md b/doc/admin-guide/en-US/markdown/admin_geo-replication.md
new file mode 100644
index 000000000..849957244
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_geo-replication.md
@@ -0,0 +1,738 @@
+Managing Geo-replication
+========================
+
+Geo-replication provides a continuous, asynchronous, and incremental
+replication service from one site to another over Local Area Networks
+(LANs), Wide Area Network (WANs), and across the Internet.
+
+Geo-replication uses a master–slave model, whereby replication and
+mirroring occurs between the following partners:
+
+-   Master – a GlusterFS volume
+
+-   Slave – a slave which can be of the following types:
+
+    -   A local directory which can be represented as file URL like
+        `file:///path/to/dir`. You can use shortened form, for example,
+        ` /path/to/dir`.
+
+    -   A GlusterFS Volume - Slave volume can be either a local volume
+        like `gluster://localhost:volname` (shortened form - `:volname`)
+        or a volume served by different host like
+        `gluster://host:volname` (shortened form - `host:volname`).
+
+    > **Note**
+    >
+    > Both of the above types can be accessed remotely using SSH tunnel.
+    > To use SSH, add an SSH prefix to either a file URL or gluster type
+    > URL. For example, ` ssh://root@remote-host:/path/to/dir`
+    > (shortened form - `root@remote-host:/path/to/dir`) or
+    > `ssh://root@remote-host:gluster://localhost:volname` (shortened
+    > from - `root@remote-host::volname`).
+
+This section introduces Geo-replication, illustrates the various
+deployment scenarios, and explains how to configure the system to
+provide replication and mirroring in your environment.
+
+Replicated Volumes vs Geo-replication
+=====================================
+
+The following table lists the difference between replicated volumes and
+geo-replication:
+
+  Replicated Volumes                                                                      Geo-replication
+  --------------------------------------------------------------------------------------- -----------------------------------------------------------------------------------------------------------------
+  Mirrors data across clusters                                                            Mirrors data across geographically distributed clusters
+  Provides high-availability                                                              Ensures backing up of data for disaster recovery
+  Synchronous replication (each and every file operation is sent across all the bricks)   Asynchronous replication (checks for the changes in files periodically and syncs them on detecting differences)
+
+Preparing to Deploy Geo-replication
+===================================
+
+This section provides an overview of the Geo-replication deployment
+scenarios, describes how you can check the minimum system requirements,
+and explores common deployment scenarios.
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+Exploring Geo-replication Deployment Scenarios
+----------------------------------------------
+
+Geo-replication provides an incremental replication service over Local
+Area Networks (LANs), Wide Area Network (WANs), and across the Internet.
+This section illustrates the most common deployment scenarios for
+Geo-replication, including the following:
+
+-   Geo-replication over LAN
+
+-   Geo-replication over WAN
+
+-   Geo-replication over the Internet
+
+-   Multi-site cascading Geo-replication
+
+**Geo-replication over LAN**
+
+You can configure Geo-replication to mirror data over a Local Area
+Network.
+
+![ Geo-replication over LAN ][]
+
+**Geo-replication over WAN**
+
+You can configure Geo-replication to replicate data over a Wide Area
+Network.
+
+![ Geo-replication over WAN ][]
+
+**Geo-replication over Internet**
+
+You can configure Geo-replication to mirror data over the Internet.
+
+![ Geo-replication over Internet ][]
+
+**Multi-site cascading Geo-replication**
+
+You can configure Geo-replication to mirror data in a cascading fashion
+across multiple sites.
+
+![ Multi-site cascading Geo-replication ][]
+
+Geo-replication Deployment Overview
+-----------------------------------
+
+Deploying Geo-replication involves the following steps:
+
+1.  Verify that your environment matches the minimum system requirement.
+    For more information, see ?.
+
+2.  Determine the appropriate deployment scenario. For more information,
+    see ?.
+
+3.  Start Geo-replication on master and slave systems, as required. For
+    more information, see ?.
+
+Checking Geo-replication Minimum Requirements
+---------------------------------------------
+
+Before deploying GlusterFS Geo-replication, verify that your systems
+match the minimum requirements.
+
+The following table outlines the minimum requirements for both master
+and slave nodes within your environment:
+
+  Component                Master                                                                Slave
+  ------------------------ --------------------------------------------------------------------- --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+  Operating System         GNU/Linux                                                             GNU/Linux
+  Filesystem               GlusterFS 3.2 or higher                                               GlusterFS 3.2 or higher (GlusterFS needs to be installed, but does not need to be running), ext3, ext4, or XFS (any other POSIX compliant file system would work, but has not been tested extensively)
+  Python                   Python 2.4 (with ctypes external module), or Python 2.5 (or higher)   Python 2.4 (with ctypes external module), or Python 2.5 (or higher)
+  Secure shell             OpenSSH version 4.0 (or higher)                                       SSH2-compliant daemon
+  Remote synchronization   rsync 3.0.7 or higher                                                 rsync 3.0.7 or higher
+  FUSE                     GlusterFS supported versions                                          GlusterFS supported versions
+
+Setting Up the Environment for Geo-replication
+----------------------------------------------
+
+**Time Synchronization**
+
+-   On bricks of a geo-replication master volume, all the servers' time
+    must be uniform. You are recommended to set up NTP (Network Time
+    Protocol) service to keep the bricks sync in time and avoid
+    out-of-time sync effect.
+
+    For example: In a Replicated volume where brick1 of the master is at
+    12.20 hrs and brick 2 of the master is at 12.10 hrs with 10 minutes
+    time lag, all the changes in brick2 between this period may go
+    unnoticed during synchronization of files with Slave.
+
+    For more information on setting up NTP, see [][].
+
+**To setup Geo-replication for SSH**
+
+Password-less login has to be set up between the host machine (where
+geo-replication Start command will be issued) and the remote machine
+(where slave process should be launched through SSH).
+
+1.  On the node where geo-replication sessions are to be set up, run the
+    following command:
+
+    `# ssh-keygen -f /var/lib/glusterd/geo-replication/secret.pem`
+
+    Press Enter twice to avoid passphrase.
+
+2.  Run the following command on master for all the slave hosts:
+
+    `# ssh-copy-id -i /var/lib/glusterd/geo-replication/secret.pem.pub @`
+
+Setting Up the Environment for a Secure Geo-replication Slave
+-------------------------------------------------------------
+
+You can configure a secure slave using SSH so that master is granted a
+restricted access. With GlusterFS, you need not specify configuration
+parameters regarding the slave on the master-side configuration. For
+example, the master does not require the location of the rsync program
+on slave but the slave must ensure that rsync is in the PATH of the user
+which the master connects using SSH. The only information that master
+and slave have to negotiate are the slave-side user account, slave's
+resources that master uses as slave resources, and the master's public
+key. Secure access to the slave can be established using the following
+options:
+
+-   Restricting Remote Command Execution
+
+-   Using `Mountbroker` for Slaves
+
+-   Using IP based Access Control
+
+**Backward Compatibility**
+
+Your existing Ge-replication environment will work with GlusterFS,
+except for the following:
+
+-   The process of secure reconfiguration affects only the glusterfs
+    instance on slave. The changes are transparent to master with the
+    exception that you may have to change the SSH target to an
+    unprivileged account on slave.
+
+-   The following are the some exceptions where this might not work:
+
+    -   Geo-replication URLs which specify the slave resource when
+        configuring master will include the following special
+        characters: space, \*, ?, [;
+
+    -   Slave must have a running instance of glusterd, even if there is
+        no gluster volume among the mounted slave resources (that is,
+        file tree slaves are used exclusively) .
+
+### Restricting Remote Command Execution
+
+If you restrict remote command execution, then the Slave audits commands
+coming from the master and the commands related to the given
+geo-replication session is allowed. The Slave also provides access only
+to the files within the slave resource which can be read or manipulated
+by the Master.
+
+To restrict remote command execution:
+
+1.  Identify the location of the gsyncd helper utility on Slave. This
+    utility is installed in `PREFIX/libexec/glusterfs/gsyncd`, where
+    PREFIX is a compile-time parameter of glusterfs. For example,
+    `--prefix=PREFIX` to the configure script with the following common
+    values` /usr, /usr/local, and /opt/glusterfs/glusterfs_version`.
+
+2.  Ensure that command invoked from master to slave passed through the
+    slave's gsyncd utility.
+
+    You can use either of the following two options:
+
+    -   Set gsyncd with an absolute path as the shell for the account
+        which the master connects through SSH. If you need to use a
+        privileged account, then set it up by creating a new user with
+        UID 0.
+
+    -   Setup key authentication with command enforcement to gsyncd. You
+        must prefix the copy of master's public key in the Slave
+        account's `authorized_keys` file with the following command:
+
+        `command=<path to gsyncd>`.
+
+        For example,
+        `command="PREFIX/glusterfs/gsyncd" ssh-rsa AAAAB3Nza....`
+
+### Using Mountbroker for Slaves
+
+`mountbroker` is a new service of glusterd. This service allows an
+unprivileged process to own a GlusterFS mount by registering a label
+(and DSL (Domain-specific language) options ) with glusterd through a
+glusterd volfile. Using CLI, you can send a mount request to glusterd to
+receive an alias (symlink) of the mounted volume.
+
+A request from the agent , the unprivileged slave agents use the
+mountbroker service of glusterd to set up an auxiliary gluster mount for
+the agent in a special environment which ensures that the agent is only
+allowed to access with special parameters that provide administrative
+level access to the particular volume.
+
+**To setup an auxiliary gluster mount for the agent**:
+
+1.  Create a new group. For example, `geogroup`.
+
+2.  Create a unprivileged account. For example, ` geoaccount`. Make it a
+    member of ` geogroup`.
+
+3.  Create a new directory owned by root and with permissions *0711.*
+    For example, create a create mountbroker-root directory
+    `/var/mountbroker-root`.
+
+4.  Add the following options to the glusterd volfile, assuming the name
+    of the slave gluster volume as `slavevol`:
+
+    `option mountbroker-root /var/mountbroker-root `
+
+    `option mountbroker-geo-replication.geoaccount slavevol`
+
+    `option geo-replication-log-group geogroup`
+
+    If you are unable to locate the glusterd volfile at
+    `/etc/glusterfs/glusterd.vol`, you can create a volfile containing
+    both the default configuration and the above options and place it at
+    `/etc/glusterfs/`.
+
+    A sample glusterd volfile along with default options:
+
+        volume management
+            type mgmt/glusterd
+            option working-directory /var/lib/glusterd
+            option transport-type socket,rdma
+            option transport.socket.keepalive-time 10
+            option transport.socket.keepalive-interval 2
+            option transport.socket.read-fail-log off
+
+            option mountbroker-root /var/mountbroker-root 
+            option mountbroker-geo-replication.geoaccount slavevol
+            option geo-replication-log-group geogroup
+        end-volume
+
+    If you host multiple slave volumes on Slave, you can repeat step 2.
+    for each of them and add the following options to the `volfile`:
+
+        option mountbroker-geo-replication.geoaccount2 slavevol2
+        option mountbroker-geo-replication.geoaccount3 slavevol3
+
+5.  Setup Master to access Slave as `geoaccount@Slave`.
+
+    You can add multiple slave volumes within the same account
+    (geoaccount) by providing comma-separated list (without spaces) as
+    the argument of `mountbroker-geo-replication.geogroup`. You can also
+    have multiple options of the form `mountbroker-geo-replication.*`.
+    It is recommended to use one service account per Master machine. For
+    example, if there are multiple slave volumes on Slave for the master
+    machines Master1, Master2, and Master3, then create a dedicated
+    service user on Slave for them by repeating Step 2. for each (like
+    geogroup1, geogroup2, and geogroup3), and then add the following
+    corresponding options to the volfile:
+
+    `option mountbroker-geo-replication.geoaccount1 slavevol11,slavevol12,slavevol13`
+
+    `option mountbroker-geo-replication.geoaccount2 slavevol21,slavevol22`
+
+    `option mountbroker-geo-replication.geoaccount3 slavevol31`
+
+    Now set up Master1 to ssh to geoaccount1@Slave, etc.
+
+    You must restart glusterd after making changes in the configuration
+    to effect the updates.
+
+### Using IP based Access Control
+
+You can use IP based access control method to provide access control for
+the slave resources using IP address. You can use method for both Slave
+and file tree slaves, but in the section, we are focusing on file tree
+slaves using this method.
+
+To set access control based on IP address for file tree slaves:
+
+1.  Set a general restriction for accessibility of file tree resources:
+
+    `# gluster volume geo-replication '/*' config allow-network ::1,127.0.0.1 `
+
+    This will refuse all requests for spawning slave agents except for
+    requests initiated locally.
+
+2.  If you want the to lease file tree at `/data/slave-tree` to Master,
+    enter the following command:
+
+    `# gluster volume geo-replicationconfig allow-network `
+
+    `MasterIP` is the IP address of Master. The slave agent spawn
+    request from master will be accepted if it is executed at
+    `/data/slave-tree`.
+
+If the Master side network configuration does not enable the Slave to
+recognize the exact IP address of Master, you can use CIDR notation to
+specify a subnet instead of a single IP address as MasterIP or even
+comma-separated lists of CIDR subnets.
+
+If you want to extend IP based access control to gluster slaves, use the
+following command:
+
+`# gluster volume geo-replication '*' config allow-network ::1,127.0.0.1`
+
+Starting Geo-replication
+========================
+
+This section describes how to configure and start Gluster
+Geo-replication in your storage environment, and verify that it is
+functioning correctly.
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+Starting Geo-replication
+------------------------
+
+To start Gluster Geo-replication
+
+-   Start geo-replication between the hosts using the following command:
+
+    `# gluster volume geo-replication  start`
+
+    For example:
+
+        # gluster volume geo-replication Volume1 example.com:/data/remote_dir start
+        Starting geo-replication session between Volume1
+        example.com:/data/remote_dir has been successful
+
+    > **Note**
+    >
+    > You may need to configure the service before starting Gluster
+    > Geo-replication. For more information, see ?.
+
+Verifying Successful Deployment
+-------------------------------
+
+You can use the gluster command to verify the status of Gluster
+Geo-replication in your environment.
+
+**To verify the status Gluster Geo-replication**
+
+-   Verify the status by issuing the following command on host:
+
+    `# gluster volume geo-replication  status`
+
+    For example:
+
+    `# gluster volume geo-replication Volume1 example.com:/data/remote_dir status`
+
+        # gluster volume geo-replication Volume1 example.com:/data/remote_dir status
+
+        MASTER    SLAVE                            STATUS
+        ______    ______________________________   ____________
+        Volume1 root@example.com:/data/remote_dir  Starting....
+
+Displaying Geo-replication Status Information
+---------------------------------------------
+
+You can display status information about a specific geo-replication
+master session, or a particular master-slave session, or all
+geo-replication sessions, as needed.
+
+**To display geo-replication status information**
+
+-   Display information of all geo-replication sessions using the
+    following command:
+
+        # gluster volume geo-replication Volume1 example.com:/data/remote_dir status
+
+        MASTER    SLAVE                            STATUS
+        ______    ______________________________   ____________
+        Volume1 root@example.com:/data/remote_dir  Starting....
+
+-   Display information of a particular master slave session using the
+    following command:
+
+    `# gluster volume geo-replication  status`
+
+    For example, to display information of Volume1 and
+    example.com:/data/remote\_dir
+
+    `# gluster volume geo-replication Volume1 example.com:/data/remote_dir status`
+
+    The status of the geo-replication between Volume1 and
+    example.com:/data/remote\_dir is displayed.
+
+-   Display information of all geo-replication sessions belonging to a
+    master
+
+    `# gluster volume geo-replication MASTER status`
+
+    For example, to display information of Volume1
+
+        # gluster volume geo-replication Volume1 example.com:/data/remote_dir status
+
+        MASTER    SLAVE                            STATUS
+        ______    ______________________________   ____________
+        Volume1 ssh://example.com:gluster://127.0.0.1:remove_volume  OK
+
+        Volume1 ssh://example.com:file:///data/remote_dir  OK
+
+    The status of a session could be one of the following four:
+
+-   **Starting**: This is the initial phase of the Geo-replication
+    session; it remains in this state for a minute, to make sure no
+    abnormalities are present.
+
+-   **OK**: The geo-replication session is in a stable state.
+
+-   **Faulty**: The geo-replication session has witnessed some
+    abnormality and the situation has to be investigated further. For
+    further information, see ? section.
+
+-   **Corrupt**: The monitor thread which is monitoring the
+    geo-replication session has died. This situation should not occur
+    normally, if it persists contact Red Hat Support[][1].
+
+Configuring Geo-replication
+---------------------------
+
+To configure Gluster Geo-replication
+
+-   Use the following command at the Gluster command line:
+
+    `# gluster volume geo-replication  config [options]`
+
+    For more information about the options, see ?.
+
+    For example:
+
+    To view list of all option/value pair, use the following command:
+
+    `# gluster volume geo-replication Volume1 example.com:/data/remote_dir config`
+
+Stopping Geo-replication
+------------------------
+
+You can use the gluster command to stop Gluster Geo-replication (syncing
+of data from Master to Slave) in your environment.
+
+**To stop Gluster Geo-replication**
+
+-   Stop geo-replication between the hosts using the following command:
+
+    `# gluster volume geo-replication  stop `
+
+    For example:
+
+        # gluster volume geo-replication Volume1 example.com:/data/remote_dir stop
+        Stopping geo-replication session between Volume1 and
+        example.com:/data/remote_dir has been successful
+
+    See ? for more information about the gluster command.
+
+Restoring Data from the Slave
+=============================
+
+You can restore data from the slave to the master volume, whenever the
+master volume becomes faulty for reasons like hardware failure.
+
+The example in this section assumes that you are using the Master Volume
+(Volume1) with the following configuration:
+
+    machine1# gluster volume info
+    Type: Distribute
+    Status: Started
+    Number of Bricks: 2
+    Transport-type: tcp
+    Bricks:
+    Brick1: machine1:/export/dir16
+    Brick2: machine2:/export/dir16
+    Options Reconfigured:
+    geo-replication.indexing: on
+
+The data is syncing from master volume (Volume1) to slave directory
+(example.com:/data/remote\_dir). To view the status of this
+geo-replication session run the following command on Master:
+
+    # gluster volume geo-replication Volume1 root@example.com:/data/remote_dir status
+
+    MASTER    SLAVE                             STATUS
+    ______    ______________________________    ____________
+    Volume1  root@example.com:/data/remote_dir   OK
+
+**Before Failure**
+
+Assume that the Master volume had 100 files and was mounted at
+/mnt/gluster on one of the client machines (client). Run the following
+command on Client machine to view the list of files:
+
+    client# ls /mnt/gluster | wc –l
+    100
+
+The slave directory (example.com) will have same data as in the master
+volume and same can be viewed by running the following command on slave:
+
+    example.com# ls /data/remote_dir/ | wc –l
+    100
+
+**After Failure**
+
+If one of the bricks (machine2) fails, then the status of
+Geo-replication session is changed from "OK" to "Faulty". To view the
+status of this geo-replication session run the following command on
+Master:
+
+    # gluster volume geo-replication Volume1 root@example.com:/data/remote_dir status
+
+    MASTER    SLAVE                              STATUS
+    ______    ______________________________     ____________
+    Volume1   root@example.com:/data/remote_dir  Faulty
+
+Machine2 is failed and now you can see discrepancy in number of files
+between master and slave. Few files will be missing from the master
+volume but they will be available only on slave as shown below.
+
+Run the following command on Client:
+
+    client # ls /mnt/gluster | wc –l
+    52
+
+Run the following command on slave (example.com):
+
+    Example.com# # ls /data/remote_dir/ | wc –l
+    100
+
+**To restore data from the slave machine**
+
+1.  Stop all Master's geo-replication sessions using the following
+    command:
+
+    `# gluster volume geo-replication  stop`
+
+    For example:
+
+        machine1# gluster volume geo-replication Volume1
+        example.com:/data/remote_dir stop
+
+        Stopping geo-replication session between Volume1 &
+        example.com:/data/remote_dir has been successful
+
+    > **Note**
+    >
+    > Repeat `# gluster volume geo-replication  stop `command on all
+    > active geo-replication sessions of master volume.
+
+2.  Replace the faulty brick in the master by using the following
+    command:
+
+    `# gluster volume replace-brick  start`
+
+    For example:
+
+        machine1# gluster volume replace-brick Volume1 machine2:/export/dir16 machine3:/export/dir16 start
+        Replace-brick started successfully
+
+3.  Commit the migration of data using the following command:
+
+    `# gluster volume replace-brick  commit force `
+
+    For example:
+
+        machine1# gluster volume replace-brick Volume1 machine2:/export/dir16 machine3:/export/dir16 commit force
+        Replace-brick commit successful
+
+4.  Verify the migration of brick by viewing the volume info using the
+    following command:
+
+    `# gluster volume info `
+
+    For example:
+
+        machine1# gluster volume info
+        Volume Name: Volume1
+        Type: Distribute
+        Status: Started
+        Number of Bricks: 2
+        Transport-type: tcp
+        Bricks:
+        Brick1: machine1:/export/dir16
+        Brick2: machine3:/export/dir16
+        Options Reconfigured:
+        geo-replication.indexing: on
+
+5.  Run rsync command manually to sync data from slave to master
+    volume's client (mount point).
+
+    For example:
+
+    `example.com# rsync -PavhS --xattrs --ignore-existing /data/remote_dir/ client:/mnt/gluster`
+
+    Verify that the data is synced by using the following command:
+
+    On master volume, run the following command:
+
+        Client # ls | wc –l
+        100
+
+    On the Slave run the following command:
+
+        example.com# ls /data/remote_dir/ | wc –l
+        100
+
+    Now Master volume and Slave directory is synced.
+
+6.  Restart geo-replication session from master to slave using the
+    following command:
+
+    `# gluster volume geo-replication  start `
+
+    For example:
+
+        machine1# gluster volume geo-replication Volume1
+        example.com:/data/remote_dir start
+        Starting geo-replication session between Volume1 &
+        example.com:/data/remote_dir has been successful
+
+Best Practices
+==============
+
+**Manually Setting Time**
+
+If you have to change the time on your bricks manually, then you must
+set uniform time on all bricks. This avoids the out-of-time sync issue
+described in ?. Setting time backward corrupts the geo-replication
+index, so the recommended way to set the time manually is:
+
+1.  Stop geo-replication between the master and slave using the
+    following command:
+
+    `# gluster volume geo-replication  sto`p
+
+2.  Stop the geo-replication indexing using the following command:
+
+    `# gluster volume set  geo-replication.indexing of`f
+
+3.  Set uniform time on all bricks.s
+
+4.  Restart your geo-replication sessions by using the following
+    command:
+
+    `# gluster volume geo-replication start `
+
+**Running Geo-replication commands in one system**
+
+It is advisable to run the geo-replication commands in one of the bricks
+in the trusted storage pool. This is because, the log files for the
+geo-replication session would be stored in the \*Server\* where the
+Geo-replication start is initiated. Hence it would be easier to locate
+the log-files when required.
+
+**Isolation**
+
+Geo-replication slave operation is not sandboxed as of now and is ran as
+a privileged service. So for the security reason, it is advised to
+create a sandbox environment (dedicated machine / dedicated virtual
+machine / chroot/container type solution) by the administrator to run
+the geo-replication slave in it. Enhancement in this regard will be
+available in follow-up minor release.
+
+  [ Geo-replication over LAN ]: images/Geo-Rep_LAN.png
+  [ Geo-replication over WAN ]: images/Geo-Rep_WAN.png
+  [ Geo-replication over Internet ]: images/Geo-Rep03_Internet.png
+  [ Multi-site cascading Geo-replication ]: images/Geo-Rep04_Cascading.png
+  []: http://docs.redhat.com/docs/en-US/Red_Hat_Enterprise_Linux/6/html/Migration_Planning_Guide/ch04s07.html
+  [1]: www.redhat.com/support/
diff --git a/doc/admin-guide/en-US/markdown/admin_managing_snapshots.md b/doc/admin-guide/en-US/markdown/admin_managing_snapshots.md
new file mode 100644
index 000000000..e76ee9151
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_managing_snapshots.md
@@ -0,0 +1,66 @@
+Managing GlusterFS Volume Snapshots
+==========================
+
+This section describes how to perform common GlusterFS volume snapshot
+management operations
+
+Pre-requisites
+=====================
+
+GlusterFS volume snapshot feature is based on thinly provisioned LVM snapshot.
+To make use of snapshot feature GlusterFS volume should fulfill following
+pre-requisites:
+
+* Each brick should be on an independent thinly provisioned LVM.
+* Brick LVM should not contain any other data other than brick.
+* None of the brick should be on a thick LVM.
+
+
+Snapshot Management
+=====================
+
+
+**Snapshot creation**
+
+*gluster snapshot create \<vol-name\> \[-n \<snap-name\>\] \[-d \<description\>\]*
+
+This command will create a snapshot of a GlusterFS volume. User can provide a snap-name and a description to identify the snap. The description cannot be more than 1024 characters.
+
+Volume should be present and it should be in started state.
+
+**Restoring snaps**
+
+*gluster snapshot restore -v \<vol-name\> \<snap-name\>*
+
+This command restores an already taken snapshot of a GlusterFS volume. Snapshot restore is an offline activity therefore if the volume is online then the restore operation will fail.
+
+Once the snapshot is restored  it will be deleted from the list of snapshot.
+
+**Deleting snaps**
+
+*gluster snapshot delete \<volname\>\ -s \<snap-name\> \[force\]*
+
+This command will delete the specified snapshot.
+
+**Listing of available snaps**
+
+*gluster snapshot list \[\<volname\> \[-s \<snap-name>\]\]*
+
+This command is used to list all snapshots taken, or for a specified volume. If snap-name is provided then it will list the details of that snap.
+
+**Configuring the snapshot behavior**
+
+*gluster snapshot config \[\<vol-name | all\>\]*
+
+This command will display existing config values for a volume. If volume name is not provided then config values of all the volume is displayed. System config is displayed irrespective of volume name.
+
+*gluster snapshot config \<vol-name | all\> \[\<snap-max-hard-limit\> \<count\>\] \[\<snap-max-soft-limit\> \<percentage\>\]*
+
+The above command can be used to change the existing config values. If vol-name is provided then config value of that volume is changed, else it will set/change the system limit.
+
+The system limit is the default value of the config for all the volume. Volume specific limit cannot cross the system limit. If a volume specific limit is not provided then system limit will be considered.
+
+If any of this limit is decreased and the current snap count of the system/volume is more than the limit then the command will fail. If user still want to decrease the limit then force option should be used.
+
+
+
diff --git a/doc/admin-guide/en-US/markdown/admin_managing_volumes.md b/doc/admin-guide/en-US/markdown/admin_managing_volumes.md
new file mode 100644
index 000000000..6c06e27a0
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_managing_volumes.md
@@ -0,0 +1,710 @@
+Managing GlusterFS Volumes
+==========================
+
+This section describes how to perform common GlusterFS management
+operations, including the following:
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+Tuning Volume Options
+=====================
+
+You can tune volume options, as needed, while the cluster is online and
+available.
+
+> **Note**
+>
+> Red Hat recommends you to set server.allow-insecure option to ON if
+> there are too many bricks in each volume or if there are too many
+> services which have already utilized all the privileged ports in the
+> system. Turning this option ON allows ports to accept/reject messages
+> from insecure ports. So, use this option only if your deployment
+> requires it.
+
+To tune volume options
+
+-   Tune volume options using the following command:
+
+    `# gluster volume set `
+
+    For example, to specify the performance cache size for test-volume:
+
+        # gluster volume set test-volume performance.cache-size 256MB
+        Set volume successful
+
+    The following table lists the Volume options along with its
+    description and default value:
+
+    > **Note**
+    >
+    > The default options given here are subject to modification at any
+    > given time and may not be the same for all versions.
+
+      -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      Option                                 Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     Default Value                      Available Options
+      -------------------------------------- ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------- ---------------------------------------------------------------------------------------
+      auth.allow                             IP addresses of the clients which should be allowed to access the volume.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \* (allow all)                     Valid IP address which includes wild card patterns including \*, such as 192.168.1.\*
+
+      auth.reject                            IP addresses of the clients which should be denied to access the volume.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        NONE (reject none)                 Valid IP address which includes wild card patterns including \*, such as 192.168.2.\*
+
+      client.grace-timeout                   Specifies the duration for the lock state to be maintained on the client after a network disconnection.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         10                                 10 - 1800 secs
+
+      cluster.self-heal-window-size          Specifies the maximum number of blocks per file on which self-heal would happen simultaneously.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 16                                 0 - 1025 blocks
+
+      cluster.data-self-heal-algorithm       Specifies the type of self-heal. If you set the option as "full", the entire file is copied from source to destinations. If the option is set to "diff" the file blocks that are not in sync are copied to destinations. Reset uses a heuristic model. If the file does not exist on one of the subvolumes, or a zero-byte file exists (created by entry self-heal) the entire content has to be copied anyway, so there is no benefit from using the "diff" algorithm. If the file size is about the same as page size, the entire file can be read and written with a few operations, which will be faster than "diff" which has to read checksums and then read and write.   reset                              full | diff | reset
+
+      cluster.min-free-disk                  Specifies the percentage of disk space that must be kept free. Might be useful for non-uniform bricks.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          10%                                Percentage of required minimum free disk space
+
+      cluster.stripe-block-size              Specifies the size of the stripe unit that will be read from or written to.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     128 KB (for all files)             size in bytes
+
+      cluster.self-heal-daemon               Allows you to turn-off proactive self-heal on replicated volumes.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               on                                 On | Off
+
+      cluster.ensure-durability              This option makes sure the data/metadata is durable across abrupt shutdown of the brick.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        on                                 On | Off
+
+      diagnostics.brick-log-level            Changes the log-level of the bricks.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            INFO                               DEBUG|WARNING|ERROR|CRITICAL|NONE|TRACE
+
+      diagnostics.client-log-level           Changes the log-level of the clients.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           INFO                               DEBUG|WARNING|ERROR|CRITICAL|NONE|TRACE
+
+      diagnostics.latency-measurement        Statistics related to the latency of each operation would be tracked.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           off                                On | Off
+
+      diagnostics.dump-fd-stats              Statistics related to file-operations would be tracked.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         off                                On | Off
+
+      feature.read-only                      Enables you to mount the entire volume as read-only for all the clients (including NFS clients) accessing it.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   off                                On | Off
+
+      features.lock-heal                     Enables self-healing of locks when the network disconnects.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     on                                 On | Off
+
+      features.quota-timeout                 For performance reasons, quota caches the directory sizes on client. You can set timeout indicating the maximum duration of directory sizes in cache, from the time they are populated, during which they are considered valid.                                                                                                                                                                                                                                                                                                                                                                                                                                                 0                                  0 - 3600 secs
+
+      geo-replication.indexing               Use this option to automatically sync the changes in the filesystem from Master to Slave.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       off                                On | Off
+
+      network.frame-timeout                  The time frame after which the operation has to be declared as dead, if the server does not respond for a particular operation.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 1800 (30 mins)                     1800 secs
+
+      network.ping-timeout                   The time duration for which the client waits to check if the server is responsive. When a ping timeout happens, there is a network disconnect between the client and server. All resources held by server on behalf of the client get cleaned up. When a reconnection happens, all resources will need to be re-acquired before the client can resume its operations on the server. Additionally, the locks will be acquired and the lock tables updated.                                                                                                                                                                                                                       42 Secs                            42 Secs
+                                             This reconnect is a very expensive operation and should be avoided.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
+
+      nfs.enable-ino32                       For 32-bit nfs clients or applications that do not support 64-bit inode numbers or large files, use this option from the CLI to make Gluster NFS return 32-bit inode numbers instead of 64-bit inode numbers. Applications that will benefit are those that were either:                                                                                                                                                                                                                                                                                                                                                                                                        off                                On | Off
+                                             \* Built 32-bit and run on 32-bit machines.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
+                                             \* Built 32-bit on 64-bit systems.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
+                                             \* Built 64-bit but use a library built 32-bit, especially relevant for python and perl scripts.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
+                                             Either of the conditions above can lead to application on Linux NFS clients failing with "Invalid argument" or "Value too large for defined data type" errors.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     
+
+      nfs.volume-access                      Set the access type for the specified sub-volume.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               read-write                         read-write|read-only
+
+      nfs.trusted-write                      If there is an UNSTABLE write from the client, STABLE flag will be returned to force the client to not send a COMMIT request.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   off                                On | Off
+                                             In some environments, combined with a replicated GlusterFS setup, this option can improve write performance. This flag allows users to trust Gluster replication logic to sync data to the disks and recover when required. COMMIT requests if received will be handled in a default manner by fsyncing. STABLE writes are still handled in a sync manner.                                                                                                                                                                                                                                                                                                                                                         
+
+      nfs.trusted-sync                       All writes and COMMIT requests are treated as async. This implies that no write requests are guaranteed to be on server disks when the write reply is received at the NFS client. Trusted sync includes trusted-write behavior.                                                                                                                                                                                                                                                                                                                                                                                                                                                 off                                On | Off
+
+      nfs.export-dir                         This option can be used to export specified comma separated subdirectories in the volume. The path must be an absolute path. Along with path allowed list of IPs/hostname can be associated with each subdirectory. If provided connection will allowed only from these IPs. Format: \<dir\>[(hostspec[|hostspec|...])][,...]. Where hostspec can be an IP address, hostname or an IP range in CIDR notation. **Note**: Care must be taken while configuring this option as invalid entries and/or unreachable DNS servers can introduce unwanted delay in all the mount calls.                                                                                                     No sub directory exported.                        Absolute path with allowed list of IP/hostname.
+
+      nfs.export-volumes                     Enable/Disable exporting entire volumes, instead if used in conjunction with nfs3.export-dir, can allow setting up only subdirectories as exports.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              on                                 On | Off
+
+      nfs.rpc-auth-unix                      Enable/Disable the AUTH\_UNIX authentication type. This option is enabled by default for better interoperability. However, you can disable it if required.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      on                                 On | Off
+
+      nfs.rpc-auth-null                      Enable/Disable the AUTH\_NULL authentication type. It is not recommended to change the default value for this option.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           on                                 On | Off
+
+      nfs.rpc-auth-allow\<IP- Addresses\>    Allow a comma separated list of addresses and/or hostnames to connect to the server. By default, all clients are disallowed. This allows you to define a general rule for all exported volumes.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 Reject All                         IP address or Host name
+
+      nfs.rpc-auth-reject IP- Addresses      Reject a comma separated list of addresses and/or hostnames from connecting to the server. By default, all connections are disallowed. This allows you to define a general rule for all exported volumes.                                                                                                                                                                                                                                                                                                                                                                                                                                                                       Reject All                         IP address or Host name
+
+      nfs.ports-insecure                     Allow client connections from unprivileged ports. By default only privileged ports are allowed. This is a global setting in case insecure ports are to be enabled for all exports using a single option.                                                                                                                                                                                                                                                                                                                                                                                                                                                                        off                                On | Off
+
+      nfs.addr-namelookup                    Turn-off name lookup for incoming client connections using this option. In some setups, the name server can take too long to reply to DNS queries resulting in timeouts of mount requests. Use this option to turn off name lookups during address authentication. Note, turning this off will prevent you from using hostnames in rpc-auth.addr.\* filters.                                                                                                                                                                                                                                                                                                                    on                                 On | Off
+
+      nfs.register-with- portmap             For systems that need to run multiple NFS servers, you need to prevent more than one from registering with portmap service. Use this option to turn off portmap registration for Gluster NFS.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   on                                 On | Off
+
+      nfs.port \<PORT- NUMBER\>              Use this option on systems that need Gluster NFS to be associated with a non-default port number.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               38465- 38467                       
+
+      nfs.disable                            Turn-off volume being exported by NFS                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           off                                On | Off
+
+      performance.write-behind-window-size   Size of the per-file write-behind buffer.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       1 MB                               Write-behind cache size
+
+      performance.io-thread-count            The number of threads in IO threads translator.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 16                                 0 - 65
+
+      performance.flush-behind               If this option is set ON, instructs write-behind translator to perform flush in background, by returning success (or any errors, if any of previous writes were failed) to application even before flush is sent to backend filesystem.                                                                                                                                                                                                                                                                                                                                                                                                                                         On                                 On | Off
+
+      performance.cache-max-file-size        Sets the maximum file size cached by the io-cache translator. Can use the normal size descriptors of KB, MB, GB,TB or PB (for example, 6GB). Maximum size uint64.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               2 \^ 64 -1 bytes                   size in bytes
+
+      performance.cache-min-file-size        Sets the minimum file size cached by the io-cache translator. Values same as "max" above.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       0B                                 size in bytes
+
+      performance.cache-refresh-timeout      The cached data for a file will be retained till 'cache-refresh-timeout' seconds, after which data re-validation is performed.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  1 sec                              0 - 61
+
+      performance.cache-size                 Size of the read cache.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         32 MB                              size in bytes
+
+      server.allow-insecure                  Allow client connections from unprivileged ports. By default only privileged ports are allowed. This is a global setting in case insecure ports are to be enabled for all exports using a single option.                                                                                                                                                                                                                                                                                                                                                                                                                                                                        on                                 On | Off
+
+      server.grace-timeout                   Specifies the duration for the lock state to be maintained on the server after a network disconnection.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         10                                 10 - 1800 secs
+
+      server.statedump-path                  Location of the state dump file.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                /tmp directory of the brick        New directory path
+
+      storage.health-check-interval          Number of seconds between health-checks done on the filesystem that is used for the brick(s). Defaults to 30 seconds, set to 0 to disable.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      /tmp directory of the brick        New directory path
+      -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+    You can view the changed volume options using
+    the` # gluster volume info ` command. For more information, see ?.
+
+Expanding Volumes
+=================
+
+You can expand volumes, as needed, while the cluster is online and
+available. For example, you might want to add a brick to a distributed
+volume, thereby increasing the distribution and adding to the capacity
+of the GlusterFS volume.
+
+Similarly, you might want to add a group of bricks to a distributed
+replicated volume, increasing the capacity of the GlusterFS volume.
+
+> **Note**
+>
+> When expanding distributed replicated and distributed striped volumes,
+> you need to add a number of bricks that is a multiple of the replica
+> or stripe count. For example, to expand a distributed replicated
+> volume with a replica count of 2, you need to add bricks in multiples
+> of 2 (such as 4, 6, 8, etc.).
+
+**To expand a volume**
+
+1.  On the first server in the cluster, probe the server to which you
+    want to add the new brick using the following command:
+
+    `# gluster peer probe `
+
+    For example:
+
+        # gluster peer probe server4
+        Probe successful
+
+2.  Add the brick using the following command:
+
+    `# gluster volume add-brick `
+
+    For example:
+
+        # gluster volume add-brick test-volume server4:/exp4
+        Add Brick successful
+
+3.  Check the volume information using the following command:
+
+    `# gluster volume info `
+
+    The command displays information similar to the following:
+
+        Volume Name: test-volume
+        Type: Distribute
+        Status: Started
+        Number of Bricks: 4
+        Bricks:
+        Brick1: server1:/exp1
+        Brick2: server2:/exp2
+        Brick3: server3:/exp3
+        Brick4: server4:/exp4
+
+4.  Rebalance the volume to ensure that all files are distributed to the
+    new brick.
+
+    You can use the rebalance command as described in ?.
+
+Shrinking Volumes
+=================
+
+You can shrink volumes, as needed, while the cluster is online and
+available. For example, you might need to remove a brick that has become
+inaccessible in a distributed volume due to hardware or network failure.
+
+> **Note**
+>
+> Data residing on the brick that you are removing will no longer be
+> accessible at the Gluster mount point. Note however that only the
+> configuration information is removed - you can continue to access the
+> data directly from the brick, as necessary.
+
+When shrinking distributed replicated and distributed striped volumes,
+you need to remove a number of bricks that is a multiple of the replica
+or stripe count. For example, to shrink a distributed striped volume
+with a stripe count of 2, you need to remove bricks in multiples of 2
+(such as 4, 6, 8, etc.). In addition, the bricks you are trying to
+remove must be from the same sub-volume (the same replica or stripe
+set).
+
+**To shrink a volume**
+
+1.  Remove the brick using the following command:
+
+    `# gluster volume remove-brick ` `start`
+
+    For example, to remove server2:/exp2:
+
+        # gluster volume remove-brick test-volume server2:/exp2
+
+        Removing brick(s) can result in data loss. Do you want to Continue? (y/n)
+
+2.  Enter "y" to confirm the operation. The command displays the
+    following message indicating that the remove brick operation is
+    successfully started:
+
+        Remove Brick successful 
+
+3.  (Optional) View the status of the remove brick operation using the
+    following command:
+
+    `# gluster volume remove-brick `` status`
+
+    For example, to view the status of remove brick operation on
+    server2:/exp2 brick:
+
+        # gluster volume remove-brick test-volume server2:/exp2 status
+                                        Node  Rebalanced-files  size  scanned       status
+                                   ---------  ----------------  ----  -------  -----------
+        617c923e-6450-4065-8e33-865e28d9428f               34   340      162   in progress
+
+4.  Check the volume information using the following command:
+
+    `# gluster volume info `
+
+    The command displays information similar to the following:
+
+        # gluster volume info
+        Volume Name: test-volume
+        Type: Distribute
+        Status: Started
+        Number of Bricks: 3
+        Bricks:
+        Brick1: server1:/exp1
+        Brick3: server3:/exp3
+        Brick4: server4:/exp4
+
+5.  Rebalance the volume to ensure that all files are distributed to the
+    new brick.
+
+    You can use the rebalance command as described in ?.
+
+Migrating Volumes
+=================
+
+You can migrate the data from one brick to another, as needed, while the
+cluster is online and available.
+
+**To migrate a volume**
+
+1.  Make sure the new brick, server5 in this example, is successfully
+    added to the cluster.
+
+    For more information, see ?.
+
+2.  Migrate the data from one brick to another using the following
+    command:
+
+    ` # gluster volume replace-brick  start`
+
+    For example, to migrate the data in server3:/exp3 to server5:/exp5
+    in test-volume:
+
+        # gluster volume replace-brick test-volume server3:/exp3  server5:exp5 start
+        Replace brick start operation successful
+
+    > **Note**
+    >
+    > You need to have the FUSE package installed on the server on which
+    > you are running the replace-brick command for the command to work.
+
+3.  To pause the migration operation, if needed, use the following
+    command:
+
+    `# gluster volume replace-brick  pause `
+
+    For example, to pause the data migration from server3:/exp3 to
+    server5:/exp5 in test-volume:
+
+        # gluster volume replace-brick test-volume server3:/exp3 server5:exp5 pause
+        Replace brick pause operation successful
+
+4.  To abort the migration operation, if needed, use the following
+    command:
+
+    ` # gluster volume replace-brick abort `
+
+    For example, to abort the data migration from server3:/exp3 to
+    server5:/exp5 in test-volume:
+
+        # gluster volume replace-brick test-volume server3:/exp3 server5:exp5 abort
+        Replace brick abort operation successful
+
+5.  Check the status of the migration operation using the following
+    command:
+
+    ` # gluster volume replace-brick status `
+
+    For example, to check the data migration status from server3:/exp3
+    to server5:/exp5 in test-volume:
+
+        # gluster volume replace-brick test-volume server3:/exp3 server5:/exp5 status
+        Current File = /usr/src/linux-headers-2.6.31-14/block/Makefile 
+        Number of files migrated = 10567
+        Migration complete
+
+    The status command shows the current file being migrated along with
+    the current total number of files migrated. After completion of
+    migration, it displays Migration complete.
+
+6.  Commit the migration of data from one brick to another using the
+    following command:
+
+    ` # gluster volume replace-brick commit `
+
+    For example, to commit the data migration from server3:/exp3 to
+    server5:/exp5 in test-volume:
+
+        # gluster volume replace-brick test-volume server3:/exp3 server5:/exp5 commit
+        replace-brick commit successful
+
+7.  Verify the migration of brick by viewing the volume info using the
+    following command:
+
+    `# gluster volume info `
+
+    For example, to check the volume information of new brick
+    server5:/exp5 in test-volume:
+
+        # gluster volume info test-volume
+        Volume Name: testvolume
+        Type: Replicate
+        Status: Started
+        Number of Bricks: 4
+        Transport-type: tcp
+        Bricks:
+        Brick1: server1:/exp1
+        Brick2: server2:/exp2
+        Brick3: server4:/exp4
+        Brick4: server5:/exp5
+
+        The new volume details are displayed.
+
+    The new volume details are displayed.
+
+    In the above example, previously, there were bricks; 1,2,3, and 4
+    and now brick 3 is replaced by brick 5.
+
+Rebalancing Volumes
+===================
+
+After expanding or shrinking a volume (using the add-brick and
+remove-brick commands respectively), you need to rebalance the data
+among the servers. New directories created after expanding or shrinking
+of the volume will be evenly distributed automatically. For all the
+existing directories, the distribution can be fixed by rebalancing the
+layout and/or data.
+
+This section describes how to rebalance GlusterFS volumes in your
+storage environment, using the following common scenarios:
+
+-   Fix Layout - Fixes the layout changes so that the files can actually
+    go to newly added nodes. For more information, see ?.
+
+-   Fix Layout and Migrate Data - Rebalances volume by fixing the layout
+    changes and migrating the existing data. For more information, see
+    ?.
+
+Rebalancing Volume to Fix Layout Changes
+----------------------------------------
+
+Fixing the layout is necessary because the layout structure is static
+for a given directory. In a scenario where new bricks have been added to
+the existing volume, newly created files in existing directories will
+still be distributed only among the old bricks. The
+`# gluster volume rebalance fix-layout start `command will fix the
+layout information so that the files can also go to newly added nodes.
+When this command is issued, all the file stat information which is
+already cached will get revalidated.
+
+A fix-layout rebalance will only fix the layout changes and does not
+migrate data. If you want to migrate the existing data,
+use`# gluster volume rebalance  start ` command to rebalance data among
+the servers.
+
+**To rebalance a volume to fix layout changes**
+
+-   Start the rebalance operation on any one of the server using the
+    following command:
+
+    `# gluster volume rebalance fix-layout start`
+
+    For example:
+
+        # gluster volume rebalance test-volume fix-layout start
+        Starting rebalance on volume test-volume has been successful
+
+Rebalancing Volume to Fix Layout and Migrate Data
+-------------------------------------------------
+
+After expanding or shrinking a volume (using the add-brick and
+remove-brick commands respectively), you need to rebalance the data
+among the servers.
+
+**To rebalance a volume to fix layout and migrate the existing data**
+
+-   Start the rebalance operation on any one of the server using the
+    following command:
+
+    `# gluster volume rebalance start`
+
+    For example:
+
+        # gluster volume rebalance test-volume start
+        Starting rebalancing on volume test-volume has been successful
+
+-   Start the migration operation forcefully on any one of the server
+    using the following command:
+
+    `# gluster volume rebalance start force`
+
+    For example:
+
+        # gluster volume rebalance test-volume start force
+        Starting rebalancing on volume test-volume has been successful
+
+Displaying Status of Rebalance Operation
+----------------------------------------
+
+You can display the status information about rebalance volume operation,
+as needed.
+
+**To view status of rebalance volume**
+
+-   Check the status of the rebalance operation, using the following
+    command:
+
+    `# gluster volume rebalance  status`
+
+    For example:
+
+        # gluster volume rebalance test-volume status
+                                        Node  Rebalanced-files  size  scanned       status
+                                   ---------  ----------------  ----  -------  -----------
+        617c923e-6450-4065-8e33-865e28d9428f               416  1463      312  in progress
+
+    The time to complete the rebalance operation depends on the number
+    of files on the volume along with the corresponding file sizes.
+    Continue checking the rebalance status, verifying that the number of
+    files rebalanced or total files scanned keeps increasing.
+
+    For example, running the status command again might display a result
+    similar to the following:
+
+        # gluster volume rebalance test-volume status
+                                        Node  Rebalanced-files  size  scanned       status
+                                   ---------  ----------------  ----  -------  -----------
+        617c923e-6450-4065-8e33-865e28d9428f               498  1783      378  in progress
+
+    The rebalance status displays the following when the rebalance is
+    complete:
+
+        # gluster volume rebalance test-volume status
+                                        Node  Rebalanced-files  size  scanned       status
+                                   ---------  ----------------  ----  -------  -----------
+        617c923e-6450-4065-8e33-865e28d9428f               502  1873      334   completed
+
+Stopping Rebalance Operation
+----------------------------
+
+You can stop the rebalance operation, as needed.
+
+**To stop rebalance**
+
+-   Stop the rebalance operation using the following command:
+
+    `# gluster volume rebalance  stop`
+
+    For example:
+
+        # gluster volume rebalance test-volume stop
+                                        Node  Rebalanced-files  size  scanned       status
+                                   ---------  ----------------  ----  -------  -----------
+        617c923e-6450-4065-8e33-865e28d9428f               59   590      244       stopped
+        Stopped rebalance process on volume test-volume 
+
+Stopping Volumes
+================
+
+To stop a volume
+
+1.  Stop the volume using the following command:
+
+    `# gluster volume stop `
+
+    For example, to stop test-volume:
+
+        # gluster volume stop test-volume
+        Stopping volume will make its data inaccessible. Do you want to continue? (y/n)
+
+2.  Enter `y` to confirm the operation. The output of the command
+    displays the following:
+
+        Stopping volume test-volume has been successful
+
+Deleting Volumes
+================
+
+To delete a volume
+
+1.  Delete the volume using the following command:
+
+    `# gluster volume delete `
+
+    For example, to delete test-volume:
+
+        # gluster volume delete test-volume
+        Deleting volume will erase all information about the volume. Do you want to continue? (y/n)
+
+2.  Enter `y` to confirm the operation. The command displays the
+    following:
+
+        Deleting volume test-volume has been successful
+
+Triggering Self-Heal on Replicate
+=================================
+
+In replicate module, previously you had to manually trigger a self-heal
+when a brick goes offline and comes back online, to bring all the
+replicas in sync. Now the pro-active self-heal daemon runs in the
+background, diagnoses issues and automatically initiates self-healing
+every 10 minutes on the files which requires*healing*.
+
+You can view the list of files that need *healing*, the list of files
+which are currently/previously *healed*, list of files which are in
+split-brain state, and you can manually trigger self-heal on the entire
+volume or only on the files which need *healing*.
+
+-   Trigger self-heal only on the files which requires *healing*:
+
+    `# gluster volume heal `
+
+    For example, to trigger self-heal on files which requires *healing*
+    of test-volume:
+
+        # gluster volume heal test-volume
+        Heal operation on volume test-volume has been successful
+
+-   Trigger self-heal on all the files of a volume:
+
+    `# gluster volume heal ` `full`
+
+    For example, to trigger self-heal on all the files of of
+    test-volume:
+
+        # gluster volume heal test-volume full
+        Heal operation on volume test-volume has been successful
+
+-   View the list of files that needs *healing*:
+
+    `# gluster volume heal ` `info`
+
+    For example, to view the list of files on test-volume that needs
+    *healing*:
+
+        # gluster volume heal test-volume info
+        Brick :/gfs/test-volume_0
+        Number of entries: 0
+         
+        Brick :/gfs/test-volume_1
+        Number of entries: 101 
+        /95.txt
+        /32.txt
+        /66.txt
+        /35.txt
+        /18.txt
+        /26.txt
+        /47.txt
+        /55.txt
+        /85.txt
+        ...
+
+-   View the list of files that are self-healed:
+
+    `# gluster volume heal ` `info healed`
+
+    For example, to view the list of files on test-volume that are
+    self-healed:
+
+        # gluster volume heal test-volume info healed
+        Brick :/gfs/test-volume_0 
+        Number of entries: 0
+
+        Brick :/gfs/test-volume_1 
+        Number of entries: 69
+        /99.txt
+        /93.txt
+        /76.txt
+        /11.txt
+        /27.txt
+        /64.txt
+        /80.txt
+        /19.txt
+        /41.txt
+        /29.txt
+        /37.txt
+        /46.txt
+        ...
+
+-   View the list of files of a particular volume on which the self-heal
+    failed:
+
+    `# gluster volume heal ` `info failed`
+
+    For example, to view the list of files of test-volume that are not
+    self-healed:
+
+        # gluster volume heal test-volume info failed
+        Brick :/gfs/test-volume_0
+        Number of entries: 0 
+
+        Brick server2:/gfs/test-volume_3 
+        Number of entries: 72
+        /90.txt
+        /95.txt
+        /77.txt
+        /71.txt
+        /87.txt
+        /24.txt
+        ...
+
+-   View the list of files of a particular volume which are in
+    split-brain state:
+
+    `# gluster volume heal ` `info split-brain`
+
+    For example, to view the list of files of test-volume which are in
+    split-brain state:
+
+        # gluster volume heal test-volume info split-brain
+        Brick server1:/gfs/test-volume_2 
+        Number of entries: 12
+        /83.txt
+        /28.txt
+        /69.txt
+        ...
+
+        Brick :/gfs/test-volume_2
+        Number of entries: 12
+        /83.txt
+        /28.txt
+        /69.txt
+        ...
+
+
diff --git a/doc/admin-guide/en-US/markdown/admin_monitoring_workload.md b/doc/admin-guide/en-US/markdown/admin_monitoring_workload.md
new file mode 100644
index 000000000..0312bd048
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_monitoring_workload.md
@@ -0,0 +1,931 @@
+Monitoring your GlusterFS Workload
+==================================
+
+You can monitor the GlusterFS volumes on different parameters.
+Monitoring volumes helps in capacity planning and performance tuning
+tasks of the GlusterFS volume. Using these information, you can identify
+and troubleshoot issues.
+
+You can use Volume Top and Profile commands to view the performance and
+identify bottlenecks/hotspots of each brick of a volume. This helps
+system administrators to get vital performance information whenever
+performance needs to be probed.
+
+You can also perform statedump of the brick processes and nfs server
+process of a volume, and also view volume status and volume information.
+
+Running GlusterFS Volume Profile Command
+========================================
+
+GlusterFS Volume Profile command provides an interface to get the
+per-brick I/O information for each File Operation (FOP) of a volume. The
+per brick information helps in identifying bottlenecks in the storage
+system.
+
+This section describes how to run GlusterFS Volume Profile command by
+performing the following operations:
+
+-   ?
+
+-   ?
+
+-   ?
+
+Start Profiling
+---------------
+
+You must start the Profiling to view the File Operation information for
+each brick.
+
+**To start profiling:**
+
+-   Start profiling using the following command:
+
+`# gluster volume profile  start `
+
+For example, to start profiling on test-volume:
+
+    # gluster volume profile test-volume start
+    Profiling started on test-volume
+
+When profiling on the volume is started, the following additional
+options are displayed in the Volume Info:
+
+    diagnostics.count-fop-hits: on
+
+    diagnostics.latency-measurement: on
+
+Displaying the I/0 Information
+------------------------------
+
+You can view the I/O information of each brick.
+
+To display I/O information:
+
+-   Display the I/O information using the following command:
+
+`# gluster volume profile  info`
+
+For example, to see the I/O information on test-volume:
+
+    # gluster volume profile test-volume info
+    Brick: Test:/export/2
+    Cumulative Stats:
+
+    Block                     1b+           32b+           64b+
+    Size:
+           Read:                0              0              0
+           Write:             908             28              8
+
+    Block                   128b+           256b+         512b+
+    Size:
+           Read:                0               6             4
+           Write:               5              23            16
+
+    Block                  1024b+          2048b+        4096b+
+    Size:
+           Read:                 0              52           17
+           Write:               15             120          846
+
+    Block                   8192b+         16384b+      32768b+
+    Size:
+           Read:                52               8           34
+           Write:              234             134          286
+
+    Block                                  65536b+     131072b+
+    Size:
+           Read:                               118          622
+           Write:                             1341          594
+
+
+    %-latency  Avg-      Min-       Max-       calls     Fop
+              latency   Latency    Latency  
+    ___________________________________________________________
+    4.82      1132.28   21.00      800970.00   4575    WRITE
+    5.70       156.47    9.00      665085.00   39163   READDIRP
+    11.35      315.02    9.00     1433947.00   38698   LOOKUP
+    11.88     1729.34   21.00     2569638.00    7382   FXATTROP
+    47.35   104235.02 2485.00     7789367.00     488   FSYNC
+
+    ------------------
+
+    ------------------
+
+    Duration     : 335
+
+    BytesRead    : 94505058
+
+    BytesWritten : 195571980
+
+Stop Profiling
+--------------
+
+You can stop profiling the volume, if you do not need profiling
+information anymore.
+
+**To stop profiling**
+
+-   Stop profiling using the following command:
+
+    `# gluster volume profile  stop`
+
+    For example, to stop profiling on test-volume:
+
+    `# gluster volume profile  stop`
+
+    `Profiling stopped on test-volume`
+
+Running GlusterFS Volume TOP Command
+====================================
+
+GlusterFS Volume Top command allows you to view the glusterfs bricks’
+performance metrics like read, write, file open calls, file read calls,
+file write calls, directory open calls, and directory real calls. The
+top command displays up to 100 results.
+
+This section describes how to run and view the results for the following
+GlusterFS Top commands:
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+-   ?
+
+Viewing Open fd Count and Maximum fd Count
+------------------------------------------
+
+You can view both current open fd count (list of files that are
+currently the most opened and the count) on the brick and the maximum
+open fd count (count of files that are the currently open and the count
+of maximum number of files opened at any given point of time, since the
+servers are up and running). If the brick name is not specified, then
+open fd metrics of all the bricks belonging to the volume will be
+displayed.
+
+**To view open fd count and maximum fd count:**
+
+-   View open fd count and maximum fd count using the following command:
+
+    `# gluster volume top  open [brick ] [list-cnt ]`
+
+    For example, to view open fd count and maximum fd count on brick
+    server:/export of test-volume and list top 10 open calls:
+
+    `# gluster volume top  open brick  list-cnt `
+
+    `Brick: server:/export/dir1 `
+
+    `Current open fd's: 34 Max open fd's: 209 `
+
+                     ==========Open file stats========
+
+        open            file name
+        call count     
+
+        2               /clients/client0/~dmtmp/PARADOX/
+                        COURSES.DB
+
+        11              /clients/client0/~dmtmp/PARADOX/
+                        ENROLL.DB
+
+        11              /clients/client0/~dmtmp/PARADOX/
+                        STUDENTS.DB
+
+        10              /clients/client0/~dmtmp/PWRPNT/
+                        TIPS.PPT
+
+        10              /clients/client0/~dmtmp/PWRPNT/
+                        PCBENCHM.PPT
+
+        9               /clients/client7/~dmtmp/PARADOX/
+                        STUDENTS.DB
+
+        9               /clients/client1/~dmtmp/PARADOX/
+                        STUDENTS.DB
+
+        9               /clients/client2/~dmtmp/PARADOX/
+                        STUDENTS.DB
+
+        9               /clients/client0/~dmtmp/PARADOX/
+                        STUDENTS.DB
+
+        9               /clients/client8/~dmtmp/PARADOX/
+                        STUDENTS.DB
+
+Viewing Highest File Read Calls
+-------------------------------
+
+You can view highest read calls on each brick. If brick name is not
+specified, then by default, list of 100 files will be displayed.
+
+**To view highest file Read calls:**
+
+-   View highest file Read calls using the following command:
+
+    `# gluster volume top  read [brick ] [list-cnt ] `
+
+    For example, to view highest Read calls on brick server:/export of
+    test-volume:
+
+    `# gluster volume top  read brick  list-cnt `
+
+    `Brick:` server:/export/dir1
+
+                  ==========Read file stats========
+
+        read              filename
+        call count
+
+        116              /clients/client0/~dmtmp/SEED/LARGE.FIL
+
+        64               /clients/client0/~dmtmp/SEED/MEDIUM.FIL
+
+        54               /clients/client2/~dmtmp/SEED/LARGE.FIL
+
+        54               /clients/client6/~dmtmp/SEED/LARGE.FIL
+
+        54               /clients/client5/~dmtmp/SEED/LARGE.FIL
+
+        54               /clients/client0/~dmtmp/SEED/LARGE.FIL
+
+        54               /clients/client3/~dmtmp/SEED/LARGE.FIL
+
+        54               /clients/client4/~dmtmp/SEED/LARGE.FIL
+
+        54               /clients/client9/~dmtmp/SEED/LARGE.FIL
+
+        54               /clients/client8/~dmtmp/SEED/LARGE.FIL
+
+Viewing Highest File Write Calls
+--------------------------------
+
+You can view list of files which has highest file write calls on each
+brick. If brick name is not specified, then by default, list of 100
+files will be displayed.
+
+**To view highest file Write calls:**
+
+-   View highest file Write calls using the following command:
+
+    `# gluster volume top  write [brick ] [list-cnt ] `
+
+    For example, to view highest Write calls on brick server:/export of
+    test-volume:
+
+    `# gluster volume top  write brick  list-cnt `
+
+    `Brick: server:/export/dir1 `
+
+                       ==========Write file stats========
+        write call count   filename
+
+        83                /clients/client0/~dmtmp/SEED/LARGE.FIL
+
+        59                /clients/client7/~dmtmp/SEED/LARGE.FIL
+
+        59                /clients/client1/~dmtmp/SEED/LARGE.FIL
+
+        59                /clients/client2/~dmtmp/SEED/LARGE.FIL
+
+        59                /clients/client0/~dmtmp/SEED/LARGE.FIL
+
+        59                /clients/client8/~dmtmp/SEED/LARGE.FIL
+
+        59                /clients/client5/~dmtmp/SEED/LARGE.FIL
+
+        59                /clients/client4/~dmtmp/SEED/LARGE.FIL
+
+        59                /clients/client6/~dmtmp/SEED/LARGE.FIL
+
+        59                /clients/client3/~dmtmp/SEED/LARGE.FIL
+
+Viewing Highest Open Calls on Directories
+-----------------------------------------
+
+You can view list of files which has highest open calls on directories
+of each brick. If brick name is not specified, then the metrics of all
+the bricks belonging to that volume will be displayed.
+
+To view list of open calls on each directory
+
+-   View list of open calls on each directory using the following
+    command:
+
+    `# gluster volume top  opendir [brick ] [list-cnt ] `
+
+    For example, to view open calls on brick server:/export/ of
+    test-volume:
+
+    `# gluster volume top  opendir brick  list-cnt `
+
+    `Brick: server:/export/dir1 `
+
+                 ==========Directory open stats========
+
+        Opendir count     directory name
+
+        1001              /clients/client0/~dmtmp
+
+        454               /clients/client8/~dmtmp
+
+        454               /clients/client2/~dmtmp
+         
+        454               /clients/client6/~dmtmp
+
+        454               /clients/client5/~dmtmp
+
+        454               /clients/client9/~dmtmp
+
+        443               /clients/client0/~dmtmp/PARADOX
+
+        408               /clients/client1/~dmtmp
+
+        408               /clients/client7/~dmtmp
+
+        402               /clients/client4/~dmtmp
+
+Viewing Highest Read Calls on Directory
+---------------------------------------
+
+You can view list of files which has highest directory read calls on
+each brick. If brick name is not specified, then the metrics of all the
+bricks belonging to that volume will be displayed.
+
+**To view list of highest directory read calls on each brick**
+
+-   View list of highest directory read calls on each brick using the
+    following command:
+
+    `# gluster volume top  readdir [brick ] [list-cnt ] `
+
+    For example, to view highest directory read calls on brick
+    server:/export of test-volume:
+
+    `# gluster volume top  readdir brick  list-cnt `
+
+    `Brick: `
+
+        ==========Directory readdirp stats========
+
+        readdirp count           directory name
+
+        1996                    /clients/client0/~dmtmp
+
+        1083                    /clients/client0/~dmtmp/PARADOX
+
+        904                     /clients/client8/~dmtmp
+
+        904                     /clients/client2/~dmtmp
+
+        904                     /clients/client6/~dmtmp
+
+        904                     /clients/client5/~dmtmp
+
+        904                     /clients/client9/~dmtmp
+
+        812                     /clients/client1/~dmtmp
+
+        812                     /clients/client7/~dmtmp
+
+        800                     /clients/client4/~dmtmp
+
+Viewing List of Read Performance on each Brick
+----------------------------------------------
+
+You can view the read throughput of files on each brick. If brick name
+is not specified, then the metrics of all the bricks belonging to that
+volume will be displayed. The output will be the read throughput.
+
+           ==========Read throughput file stats========
+
+    read         filename                         Time
+    through
+    put(MBp
+    s)
+     
+    2570.00    /clients/client0/~dmtmp/PWRPNT/      -2011-01-31
+               TRIDOTS.POT                      15:38:36.894610  
+    2570.00    /clients/client0/~dmtmp/PWRPNT/      -2011-01-31
+               PCBENCHM.PPT                     15:38:39.815310                                                             
+    2383.00    /clients/client2/~dmtmp/SEED/        -2011-01-31
+               MEDIUM.FIL                       15:52:53.631499
+                                                  
+    2340.00    /clients/client0/~dmtmp/SEED/        -2011-01-31
+               MEDIUM.FIL                       15:38:36.926198
+                                                  
+    2299.00   /clients/client0/~dmtmp/SEED/         -2011-01-31
+              LARGE.FIL                         15:38:36.930445
+                                                 
+    2259.00   /clients/client0/~dmtmp/PARADOX/      -2011-01-31
+              COURSES.X04                       15:38:40.549919
+
+    2221.00   /clients/client0/~dmtmp/PARADOX/      -2011-01-31
+              STUDENTS.VAL                      15:52:53.298766
+                                                  
+    2221.00   /clients/client3/~dmtmp/SEED/         -2011-01-31
+              COURSES.DB                        15:39:11.776780
+
+    2184.00   /clients/client3/~dmtmp/SEED/         -2011-01-31
+              MEDIUM.FIL                        15:39:10.251764
+                                                  
+    2184.00   /clients/client5/~dmtmp/WORD/         -2011-01-31
+              BASEMACH.DOC                      15:39:09.336572                                          
+
+This command will initiate a dd for the specified count and block size
+and measures the corresponding throughput.
+
+**To view list of read performance on each brick**
+
+-   View list of read performance on each brick using the following
+    command:
+
+    `# gluster volume top  read-perf [bs  count ] [brick ] [list-cnt ]`
+
+    For example, to view read performance on brick server:/export/ of
+    test-volume, 256 block size of count 1, and list count 10:
+
+    `# gluster volume top  read-perf bs 256 count 1 brick list-cnt `
+
+    `Brick: server:/export/dir1 256 bytes (256 B) copied, Throughput: 4.1 MB/s `
+
+               ==========Read throughput file stats========
+
+        read         filename                         Time
+        through
+        put(MBp
+        s)
+
+        2912.00   /clients/client0/~dmtmp/PWRPNT/    -2011-01-31
+                   TRIDOTS.POT                   15:38:36.896486
+                                                   
+        2570.00   /clients/client0/~dmtmp/PWRPNT/    -2011-01-31
+                   PCBENCHM.PPT                  15:38:39.815310
+                                                   
+        2383.00   /clients/client2/~dmtmp/SEED/      -2011-01-31
+                   MEDIUM.FIL                    15:52:53.631499
+                                                   
+        2340.00   /clients/client0/~dmtmp/SEED/      -2011-01-31
+                   MEDIUM.FIL                    15:38:36.926198
+
+        2299.00   /clients/client0/~dmtmp/SEED/      -2011-01-31
+                   LARGE.FIL                     15:38:36.930445
+                                                              
+        2259.00  /clients/client0/~dmtmp/PARADOX/    -2011-01-31
+                  COURSES.X04                    15:38:40.549919
+                                                   
+        2221.00  /clients/client9/~dmtmp/PARADOX/    -2011-01-31
+                  STUDENTS.VAL                   15:52:53.298766
+                                                   
+        2221.00  /clients/client8/~dmtmp/PARADOX/    -2011-01-31
+                 COURSES.DB                      15:39:11.776780
+                                                   
+        2184.00  /clients/client3/~dmtmp/SEED/       -2011-01-31
+                  MEDIUM.FIL                     15:39:10.251764
+                                                   
+        2184.00  /clients/client5/~dmtmp/WORD/       -2011-01-31
+                 BASEMACH.DOC                    15:39:09.336572
+                                                   
+
+Viewing List of Write Performance on each Brick
+-----------------------------------------------
+
+You can view list of write throughput of files on each brick. If brick
+name is not specified, then the metrics of all the bricks belonging to
+that volume will be displayed. The output will be the write throughput.
+
+This command will initiate a dd for the specified count and block size
+and measures the corresponding throughput. To view list of write
+performance on each brick:
+
+-   View list of write performance on each brick using the following
+    command:
+
+    `# gluster volume top  write-perf [bs  count ] [brick ] [list-cnt ] `
+
+    For example, to view write performance on brick server:/export/ of
+    test-volume, 256 block size of count 1, and list count 10:
+
+    `# gluster volume top  write-perf bs 256 count 1 brick list-cnt `
+
+    `Brick`: server:/export/dir1
+
+    `256 bytes (256 B) copied, Throughput: 2.8 MB/s `
+
+               ==========Write throughput file stats========
+
+        write                filename                 Time
+        throughput
+        (MBps)
+         
+        1170.00    /clients/client0/~dmtmp/SEED/     -2011-01-31
+                   SMALL.FIL                     15:39:09.171494
+
+        1008.00    /clients/client6/~dmtmp/SEED/     -2011-01-31
+                   LARGE.FIL                      15:39:09.73189
+
+        949.00    /clients/client0/~dmtmp/SEED/      -2011-01-31
+                  MEDIUM.FIL                     15:38:36.927426
+
+        936.00   /clients/client0/~dmtmp/SEED/       -2011-01-31
+                 LARGE.FIL                        15:38:36.933177    
+        897.00   /clients/client5/~dmtmp/SEED/       -2011-01-31
+                 MEDIUM.FIL                       15:39:09.33628
+
+        897.00   /clients/client6/~dmtmp/SEED/       -2011-01-31
+                 MEDIUM.FIL                       15:39:09.27713
+
+        885.00   /clients/client0/~dmtmp/SEED/       -2011-01-31
+                  SMALL.FIL                      15:38:36.924271
+
+        528.00   /clients/client5/~dmtmp/SEED/       -2011-01-31
+                 LARGE.FIL                        15:39:09.81893
+
+        516.00   /clients/client6/~dmtmp/ACCESS/    -2011-01-31
+                 FASTENER.MDB                    15:39:01.797317
+
+Displaying Volume Information
+=============================
+
+You can display information about a specific volume, or all volumes, as
+needed.
+
+**To display volume information**
+
+-   Display information about a specific volume using the following
+    command:
+
+    `# gluster volume info ``VOLNAME`
+
+    For example, to display information about test-volume:
+
+        # gluster volume info test-volume
+        Volume Name: test-volume
+        Type: Distribute
+        Status: Created
+        Number of Bricks: 4
+        Bricks:
+        Brick1: server1:/exp1
+        Brick2: server2:/exp2
+        Brick3: server3:/exp3
+        Brick4: server4:/exp4
+
+-   Display information about all volumes using the following command:
+
+    `# gluster volume info all`
+
+        # gluster volume info all
+
+        Volume Name: test-volume
+        Type: Distribute
+        Status: Created
+        Number of Bricks: 4
+        Bricks:
+        Brick1: server1:/exp1
+        Brick2: server2:/exp2
+        Brick3: server3:/exp3
+        Brick4: server4:/exp4
+
+        Volume Name: mirror
+        Type: Distributed-Replicate
+        Status: Started
+        Number of Bricks: 2 X 2 = 4
+        Bricks:
+        Brick1: server1:/brick1
+        Brick2: server2:/brick2
+        Brick3: server3:/brick3
+        Brick4: server4:/brick4
+
+        Volume Name: Vol
+        Type: Distribute
+        Status: Started
+        Number of Bricks: 1
+        Bricks:
+        Brick: server:/brick6
+
+Performing Statedump on a Volume
+================================
+
+Statedump is a mechanism through which you can get details of all
+internal variables and state of the glusterfs process at the time of
+issuing the command.You can perform statedumps of the brick processes
+and nfs server process of a volume using the statedump command. The
+following options can be used to determine what information is to be
+dumped:
+
+-   **mem** - Dumps the memory usage and memory pool details of the
+    bricks.
+
+-   **iobuf** - Dumps iobuf details of the bricks.
+
+-   **priv** - Dumps private information of loaded translators.
+
+-   **callpool** - Dumps the pending calls of the volume.
+
+-   **fd** - Dumps the open fd tables of the volume.
+
+-   **inode** - Dumps the inode tables of the volume.
+
+**To display volume statedump**
+
+-   Display statedump of a volume or NFS server using the following
+    command:
+
+    `# gluster volume statedump  [nfs] [all|mem|iobuf|callpool|priv|fd|inode]`
+
+    For example, to display statedump of test-volume:
+
+        # gluster volume statedump test-volume
+        Volume statedump successful
+
+    The statedump files are created on the brick servers in the` /tmp`
+    directory or in the directory set using `server.statedump-path`
+    volume option. The naming convention of the dump file is
+    `<brick-path>.<brick-pid>.dump`.
+
+-   By defult, the output of the statedump is stored at
+    ` /tmp/<brickname.PID.dump>` file on that particular server. Change
+    the directory of the statedump file using the following command:
+
+    `# gluster volume set  server.statedump-path `
+
+    For example, to change the location of the statedump file of
+    test-volume:
+
+        # gluster volume set test-volume server.statedump-path /usr/local/var/log/glusterfs/dumps/
+        Set volume successful
+
+    You can view the changed path of the statedump file using the
+    following command:
+
+    `# gluster volume info `
+
+Displaying Volume Status
+========================
+
+You can display the status information about a specific volume, brick or
+all volumes, as needed. Status information can be used to understand the
+current status of the brick, nfs processes, and overall file system.
+Status information can also be used to monitor and debug the volume
+information. You can view status of the volume along with the following
+details:
+
+-   **detail** - Displays additional information about the bricks.
+
+-   **clients** - Displays the list of clients connected to the volume.
+
+-   **mem** - Displays the memory usage and memory pool details of the
+    bricks.
+
+-   **inode** - Displays the inode tables of the volume.
+
+-   **fd** - Displays the open fd (file descriptors) tables of the
+    volume.
+
+-   **callpool** - Displays the pending calls of the volume.
+
+**To display volume status**
+
+-   Display information about a specific volume using the following
+    command:
+
+    `# gluster volume status [all| []] [detail|clients|mem|inode|fd|callpool]`
+
+    For example, to display information about test-volume:
+
+        # gluster volume status test-volume
+        STATUS OF VOLUME: test-volume
+        BRICK                           PORT   ONLINE   PID
+        --------------------------------------------------------
+        arch:/export/1                  24009   Y       22445
+        --------------------------------------------------------
+        arch:/export/2                  24010   Y       22450
+
+-   Display information about all volumes using the following command:
+
+    `# gluster volume status all`
+
+        # gluster volume status all
+        STATUS OF VOLUME: volume-test
+        BRICK                           PORT   ONLINE   PID
+        --------------------------------------------------------
+        arch:/export/4                  24010   Y       22455
+
+        STATUS OF VOLUME: test-volume
+        BRICK                           PORT   ONLINE   PID
+        --------------------------------------------------------
+        arch:/export/1                  24009   Y       22445
+        --------------------------------------------------------
+        arch:/export/2                  24010   Y       22450
+
+-   Display additional information about the bricks using the following
+    command:
+
+    `# gluster volume status  detail`
+
+    For example, to display additional information about the bricks of
+    test-volume:
+
+        # gluster volume status test-volume details
+        STATUS OF VOLUME: test-volume
+        -------------------------------------------
+        Brick                : arch:/export/1      
+        Port                 : 24009              
+        Online               : Y                  
+        Pid                  : 16977              
+        File System          : rootfs              
+        Device               : rootfs              
+        Mount Options        : rw                  
+        Disk Space Free      : 13.8GB              
+        Total Disk Space     : 46.5GB              
+        Inode Size           : N/A                
+        Inode Count          : N/A                
+        Free Inodes          : N/A
+
+        Number of Bricks: 1
+        Bricks:
+        Brick: server:/brick6
+
+-   Display the list of clients accessing the volumes using the
+    following command:
+
+    `# gluster volume status  clients`
+
+    For example, to display the list of clients connected to
+    test-volume:
+
+        # gluster volume status test-volume clients
+        Brick : arch:/export/1
+        Clients connected : 2
+        Hostname          Bytes Read   BytesWritten
+        --------          ---------    ------------
+        127.0.0.1:1013    776          676
+        127.0.0.1:1012    50440        51200
+
+-   Display the memory usage and memory pool details of the bricks using
+    the following command:
+
+    `# gluster volume status  mem`
+
+    For example, to display the memory usage and memory pool details of
+    the bricks of test-volume:
+
+        Memory status for volume : test-volume
+        ----------------------------------------------
+        Brick : arch:/export/1
+        Mallinfo
+        --------
+        Arena    : 434176
+        Ordblks  : 2
+        Smblks   : 0
+        Hblks    : 12
+        Hblkhd   : 40861696
+        Usmblks  : 0
+        Fsmblks  : 0
+        Uordblks : 332416
+        Fordblks : 101760
+        Keepcost : 100400
+
+        Mempool Stats
+        -------------
+        Name                               HotCount ColdCount PaddedSizeof AllocCount MaxAlloc
+        ----                               -------- --------- ------------ ---------- --------
+        test-volume-server:fd_t                0     16384           92         57        5
+        test-volume-server:dentry_t           59       965           84         59       59
+        test-volume-server:inode_t            60       964          148         60       60
+        test-volume-server:rpcsvc_request_t    0       525         6372        351        2
+        glusterfs:struct saved_frame           0      4096          124          2        2
+        glusterfs:struct rpc_req               0      4096         2236          2        2
+        glusterfs:rpcsvc_request_t             1       524         6372          2        1
+        glusterfs:call_stub_t                  0      1024         1220        288        1
+        glusterfs:call_stack_t                 0      8192         2084        290        2
+        glusterfs:call_frame_t                 0     16384          172       1728        6
+
+-   Display the inode tables of the volume using the following command:
+
+    `# gluster volume status  inode`
+
+    For example, to display the inode tables of the test-volume:
+
+        # gluster volume status test-volume inode
+        inode tables for volume test-volume
+        ----------------------------------------------
+        Brick : arch:/export/1
+        Active inodes:
+        GFID                                            Lookups            Ref   IA type
+        ----                                            -------            ---   -------
+        6f3fe173-e07a-4209-abb6-484091d75499                  1              9         2
+        370d35d7-657e-44dc-bac4-d6dd800ec3d3                  1              1         2
+
+        LRU inodes: 
+        GFID                                            Lookups            Ref   IA type
+        ----                                            -------            ---   -------
+        80f98abe-cdcf-4c1d-b917-ae564cf55763                  1              0         1
+        3a58973d-d549-4ea6-9977-9aa218f233de                  1              0         1
+        2ce0197d-87a9-451b-9094-9baa38121155                  1              0         2
+
+-   Display the open fd tables of the volume using the following
+    command:
+
+    `# gluster volume status  fd`
+
+    For example, to display the open fd tables of the test-volume:
+
+        # gluster volume status test-volume fd
+
+        FD tables for volume test-volume
+        ----------------------------------------------
+        Brick : arch:/export/1
+        Connection 1:
+        RefCount = 0  MaxFDs = 128  FirstFree = 4
+        FD Entry            PID                 RefCount            Flags              
+        --------            ---                 --------            -----              
+        0                   26311               1                   2                  
+        1                   26310               3                   2                  
+        2                   26310               1                   2                  
+        3                   26311               3                   2                  
+         
+        Connection 2:
+        RefCount = 0  MaxFDs = 128  FirstFree = 0
+        No open fds
+         
+        Connection 3:
+        RefCount = 0  MaxFDs = 128  FirstFree = 0
+        No open fds
+
+-   Display the pending calls of the volume using the following command:
+
+    `# gluster volume status  callpool`
+
+    Each call has a call stack containing call frames.
+
+    For example, to display the pending calls of test-volume:
+
+        # gluster volume status test-volume
+
+        Pending calls for volume test-volume
+        ----------------------------------------------
+        Brick : arch:/export/1
+        Pending calls: 2
+        Call Stack1
+         UID    : 0
+         GID    : 0
+         PID    : 26338
+         Unique : 192138
+         Frames : 7
+         Frame 1
+          Ref Count   = 1
+          Translator  = test-volume-server
+          Completed   = No
+         Frame 2
+          Ref Count   = 0
+          Translator  = test-volume-posix
+          Completed   = No
+          Parent      = test-volume-access-control
+          Wind From   = default_fsync
+          Wind To     = FIRST_CHILD(this)->fops->fsync
+         Frame 3
+          Ref Count   = 1
+          Translator  = test-volume-access-control
+          Completed   = No
+          Parent      = repl-locks
+          Wind From   = default_fsync
+          Wind To     = FIRST_CHILD(this)->fops->fsync
+         Frame 4
+          Ref Count   = 1
+          Translator  = test-volume-locks
+          Completed   = No
+          Parent      = test-volume-io-threads
+          Wind From   = iot_fsync_wrapper
+          Wind To     = FIRST_CHILD (this)->fops->fsync
+         Frame 5
+          Ref Count   = 1
+          Translator  = test-volume-io-threads
+          Completed   = No
+          Parent      = test-volume-marker
+          Wind From   = default_fsync
+          Wind To     = FIRST_CHILD(this)->fops->fsync
+         Frame 6
+          Ref Count   = 1
+          Translator  = test-volume-marker
+          Completed   = No
+          Parent      = /export/1
+          Wind From   = io_stats_fsync
+          Wind To     = FIRST_CHILD(this)->fops->fsync
+         Frame 7
+          Ref Count   = 1
+          Translator  = /export/1
+          Completed   = No
+          Parent      = test-volume-server
+          Wind From   = server_fsync_resume
+          Wind To     = bound_xl->fops->fsync
+
+
diff --git a/doc/admin-guide/en-US/markdown/admin_setting_volumes.md b/doc/admin-guide/en-US/markdown/admin_setting_volumes.md
new file mode 100644
index 000000000..4038523c8
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_setting_volumes.md
@@ -0,0 +1,419 @@
+Setting up GlusterFS Server Volumes
+===================================
+
+A volume is a logical collection of bricks where each brick is an export
+directory on a server in the trusted storage pool. Most of the gluster
+management operations are performed on the volume.
+
+To create a new volume in your storage environment, specify the bricks
+that comprise the volume. After you have created a new volume, you must
+start it before attempting to mount it.
+
+-   Volumes of the following types can be created in your storage
+    environment:
+
+    -   Distributed - Distributed volumes distributes files throughout
+        the bricks in the volume. You can use distributed volumes where
+        the requirement is to scale storage and the redundancy is either
+        not important or is provided by other hardware/software layers.
+        For more information, see ? .
+
+    -   Replicated – Replicated volumes replicates files across bricks
+        in the volume. You can use replicated volumes in environments
+        where high-availability and high-reliability are critical. For
+        more information, see ?.
+
+    -   Striped – Striped volumes stripes data across bricks in the
+        volume. For best results, you should use striped volumes only in
+        high concurrency environments accessing very large files. For
+        more information, see ?.
+
+    -   Distributed Striped - Distributed striped volumes stripe data
+        across two or more nodes in the cluster. You should use
+        distributed striped volumes where the requirement is to scale
+        storage and in high concurrency environments accessing very
+        large files is critical. For more information, see ?.
+
+    -   Distributed Replicated - Distributed replicated volumes
+        distributes files across replicated bricks in the volume. You
+        can use distributed replicated volumes in environments where the
+        requirement is to scale storage and high-reliability is
+        critical. Distributed replicated volumes also offer improved
+        read performance in most environments. For more information, see
+        ?.
+
+    -   Distributed Striped Replicated – Distributed striped replicated
+        volumes distributes striped data across replicated bricks in the
+        cluster. For best results, you should use distributed striped
+        replicated volumes in highly concurrent environments where
+        parallel access of very large files and performance is critical.
+        In this release, configuration of this volume type is supported
+        only for Map Reduce workloads. For more information, see ?.
+
+    -   Striped Replicated – Striped replicated volumes stripes data
+        across replicated bricks in the cluster. For best results, you
+        should use striped replicated volumes in highly concurrent
+        environments where there is parallel access of very large files
+        and performance is critical. In this release, configuration of
+        this volume type is supported only for Map Reduce workloads. For
+        more information, see ?.
+
+**To create a new volume**
+
+-   Create a new volume :
+
+    `# gluster volume create [stripe  | replica ] [transport tcp | rdma | tcp, rdma] `
+
+    For example, to create a volume called test-volume consisting of
+    server3:/exp3 and server4:/exp4:
+
+        # gluster volume create test-volume server3:/exp3 server4:/exp4
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+Creating Distributed Volumes
+============================
+
+In a distributed volumes files are spread randomly across the bricks in
+the volume. Use distributed volumes where you need to scale storage and
+redundancy is either not important or is provided by other
+hardware/software layers.
+
+> **Note**
+>
+> Disk/server failure in distributed volumes can result in a serious
+> loss of data because directory contents are spread randomly across the
+> bricks in the volume.
+
+![][]
+
+**To create a distributed volume**
+
+1.  Create a trusted storage pool as described earlier in ?.
+
+2.  Create the distributed volume:
+
+    `# gluster volume create  [transport tcp | rdma | tcp,rdma] `
+
+    For example, to create a distributed volume with four storage
+    servers using tcp:
+
+        # gluster volume create test-volume server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+    (Optional) You can display the volume information:
+
+        # gluster volume info
+        Volume Name: test-volume
+        Type: Distribute
+        Status: Created
+        Number of Bricks: 4
+        Transport-type: tcp
+        Bricks:
+        Brick1: server1:/exp1
+        Brick2: server2:/exp2
+        Brick3: server3:/exp3
+        Brick4: server4:/exp4
+
+    For example, to create a distributed volume with four storage
+    servers over InfiniBand:
+
+        # gluster volume create test-volume transport rdma server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+    If the transport type is not specified, *tcp* is used as the
+    default. You can also set additional options if required, such as
+    auth.allow or auth.reject. For more information, see ?
+
+    > **Note**
+    >
+    > Make sure you start your volumes before you try to mount them or
+    > else client operations after the mount will hang, see ? for
+    > details.
+
+Creating Replicated Volumes
+===========================
+
+Replicated volumes create copies of files across multiple bricks in the
+volume. You can use replicated volumes in environments where
+high-availability and high-reliability are critical.
+
+> **Note**
+>
+> The number of bricks should be equal to of the replica count for a
+> replicated volume. To protect against server and disk failures, it is
+> recommended that the bricks of the volume are from different servers.
+
+![][1]
+
+**To create a replicated volume**
+
+1.  Create a trusted storage pool as described earlier in ?.
+
+2.  Create the replicated volume:
+
+    `# gluster volume create  [replica ] [transport tcp | rdma tcp,rdma] `
+
+    For example, to create a replicated volume with two storage servers:
+
+        # gluster volume create test-volume replica 2 transport tcp server1:/exp1 server2:/exp2
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+    If the transport type is not specified, *tcp* is used as the
+    default. You can also set additional options if required, such as
+    auth.allow or auth.reject. For more information, see ?
+
+    > **Note**
+    >
+    > Make sure you start your volumes before you try to mount them or
+    > else client operations after the mount will hang, see ? for
+    > details.
+
+Creating Striped Volumes
+========================
+
+Striped volumes stripes data across bricks in the volume. For best
+results, you should use striped volumes only in high concurrency
+environments accessing very large files.
+
+> **Note**
+>
+> The number of bricks should be a equal to the stripe count for a
+> striped volume.
+
+![][2]
+
+**To create a striped volume**
+
+1.  Create a trusted storage pool as described earlier in ?.
+
+2.  Create the striped volume:
+
+    `# gluster volume create  [stripe ] [transport tcp | rdma | tcp,rdma] `
+
+    For example, to create a striped volume across two storage servers:
+
+        # gluster volume create test-volume stripe 2 transport tcp server1:/exp1 server2:/exp2
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+    If the transport type is not specified, *tcp* is used as the
+    default. You can also set additional options if required, such as
+    auth.allow or auth.reject. For more information, see ?
+
+    > **Note**
+    >
+    > Make sure you start your volumes before you try to mount them or
+    > else client operations after the mount will hang, see ? for
+    > details.
+
+Creating Distributed Striped Volumes
+====================================
+
+Distributed striped volumes stripes files across two or more nodes in
+the cluster. For best results, you should use distributed striped
+volumes where the requirement is to scale storage and in high
+concurrency environments accessing very large files is critical.
+
+> **Note**
+>
+> The number of bricks should be a multiple of the stripe count for a
+> distributed striped volume.
+
+![][3]
+
+**To create a distributed striped volume**
+
+1.  Create a trusted storage pool as described earlier in ?.
+
+2.  Create the distributed striped volume:
+
+    `# gluster volume create  [stripe ] [transport tcp | rdma | tcp,rdma] `
+
+    For example, to create a distributed striped volume across eight
+    storage servers:
+
+        # gluster volume create test-volume stripe 4 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 server5:/exp5 server6:/exp6 server7:/exp7 server8:/exp8
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+    If the transport type is not specified, *tcp* is used as the
+    default. You can also set additional options if required, such as
+    auth.allow or auth.reject. For more information, see ?
+
+    > **Note**
+    >
+    > Make sure you start your volumes before you try to mount them or
+    > else client operations after the mount will hang, see ? for
+    > details.
+
+Creating Distributed Replicated Volumes
+=======================================
+
+Distributes files across replicated bricks in the volume. You can use
+distributed replicated volumes in environments where the requirement is
+to scale storage and high-reliability is critical. Distributed
+replicated volumes also offer improved read performance in most
+environments.
+
+> **Note**
+>
+> The number of bricks should be a multiple of the replica count for a
+> distributed replicated volume. Also, the order in which bricks are
+> specified has a great effect on data protection. Each replica\_count
+> consecutive bricks in the list you give will form a replica set, with
+> all replica sets combined into a volume-wide distribute set. To make
+> sure that replica-set members are not placed on the same node, list
+> the first brick on every server, then the second brick on every server
+> in the same order, and so on.
+
+![][4]
+
+**To create a distributed replicated volume**
+
+1.  Create a trusted storage pool as described earlier in ?.
+
+2.  Create the distributed replicated volume:
+
+    `# gluster volume create  [replica ] [transport tcp | rdma | tcp,rdma] `
+
+    For example, four node distributed (replicated) volume with a
+    two-way mirror:
+
+        # gluster volume create test-volume replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+    For example, to create a six node distributed (replicated) volume
+    with a two-way mirror:
+
+        # gluster volume create test-volume replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 server5:/exp5 server6:/exp6
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+    If the transport type is not specified, *tcp* is used as the
+    default. You can also set additional options if required, such as
+    auth.allow or auth.reject. For more information, see ?
+
+    > **Note**
+    >
+    > Make sure you start your volumes before you try to mount them or
+    > else client operations after the mount will hang, see ? for
+    > details.
+
+Creating Distributed Striped Replicated Volumes
+===============================================
+
+Distributed striped replicated volumes distributes striped data across
+replicated bricks in the cluster. For best results, you should use
+distributed striped replicated volumes in highly concurrent environments
+where parallel access of very large files and performance is critical.
+In this release, configuration of this volume type is supported only for
+Map Reduce workloads.
+
+> **Note**
+>
+> The number of bricks should be a multiples of number of stripe count
+> and replica count for a distributed striped replicated volume.
+
+**To create a distributed striped replicated volume**
+
+1.  Create a trusted storage pool as described earlier in ?.
+
+2.  Create a distributed striped replicated volume using the following
+    command:
+
+    `# gluster volume create  [stripe ] [replica ] [transport tcp | rdma | tcp,rdma] `
+
+    For example, to create a distributed replicated striped volume
+    across eight storage servers:
+
+        # gluster volume create test-volume stripe 2 replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 server5:/exp5 server6:/exp6 server7:/exp7 server8:/exp8
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+    If the transport type is not specified, *tcp* is used as the
+    default. You can also set additional options if required, such as
+    auth.allow or auth.reject. For more information, see ?
+
+    > **Note**
+    >
+    > Make sure you start your volumes before you try to mount them or
+    > else client operations after the mount will hang, see ? for
+    > details.
+
+Creating Striped Replicated Volumes
+===================================
+
+Striped replicated volumes stripes data across replicated bricks in the
+cluster. For best results, you should use striped replicated volumes in
+highly concurrent environments where there is parallel access of very
+large files and performance is critical. In this release, configuration
+of this volume type is supported only for Map Reduce workloads.
+
+> **Note**
+>
+> The number of bricks should be a multiple of the replicate count and
+> stripe count for a striped replicated volume.
+
+![][5]
+
+**To create a striped replicated volume**
+
+1.  Create a trusted storage pool consisting of the storage servers that
+    will comprise the volume.
+
+    For more information, see ?.
+
+2.  Create a striped replicated volume :
+
+    `# gluster volume create  [stripe ] [replica ] [transport tcp | rdma | tcp,rdma] `
+
+    For example, to create a striped replicated volume across four
+    storage servers:
+
+        # gluster volume create test-volume stripe 2 replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+    To create a striped replicated volume across six storage servers:
+
+        # gluster volume create test-volume stripe 3 replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 server5:/exp5 server6:/exp6
+        Creation of test-volume has been successful
+        Please start the volume to access data.
+
+    If the transport type is not specified, *tcp* is used as the
+    default. You can also set additional options if required, such as
+    auth.allow or auth.reject. For more information, see ?
+
+    > **Note**
+    >
+    > Make sure you start your volumes before you try to mount them or
+    > else client operations after the mount will hang, see ? for
+    > details.
+
+Starting Volumes
+================
+
+You must start your volumes before you try to mount them.
+
+**To start a volume**
+
+-   Start a volume:
+
+    `# gluster volume start `
+
+    For example, to start test-volume:
+
+        # gluster volume start test-volume
+        Starting test-volume has been successful
+
+  []: images/Distributed_Volume.png
+  [1]: images/Replicated_Volume.png
+  [2]: images/Striped_Volume.png
+  [3]: images/Distributed_Striped_Volume.png
+  [4]: images/Distributed_Replicated_Volume.png
+  [5]: images/Striped_Replicated_Volume.png
diff --git a/doc/admin-guide/en-US/markdown/admin_settingup_clients.md b/doc/admin-guide/en-US/markdown/admin_settingup_clients.md
new file mode 100644
index 000000000..85b28c952
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_settingup_clients.md
@@ -0,0 +1,641 @@
+Accessing Data - Setting Up GlusterFS Client
+============================================
+
+You can access gluster volumes in multiple ways. You can use Gluster
+Native Client method for high concurrency, performance and transparent
+failover in GNU/Linux clients. You can also use NFS v3 to access gluster
+volumes. Extensive testing has be done on GNU/Linux clients and NFS
+implementation in other operating system, such as FreeBSD, and Mac OS X,
+as well as Windows 7 (Professional and Up) and Windows Server 2003.
+Other NFS client implementations may work with gluster NFS server.
+
+You can use CIFS to access volumes when using Microsoft Windows as well
+as SAMBA clients. For this access method, Samba packages need to be
+present on the client side.
+
+Gluster Native Client
+=====================
+
+The Gluster Native Client is a FUSE-based client running in user space.
+Gluster Native Client is the recommended method for accessing volumes
+when high concurrency and high write performance is required.
+
+This section introduces the Gluster Native Client and explains how to
+install the software on client machines. This section also describes how
+to mount volumes on clients (both manually and automatically) and how to
+verify that the volume has mounted successfully.
+
+Installing the Gluster Native Client
+------------------------------------
+
+Before you begin installing the Gluster Native Client, you need to
+verify that the FUSE module is loaded on the client and has access to
+the required modules as follows:
+
+1.  Add the FUSE loadable kernel module (LKM) to the Linux kernel:
+
+    `# modprobe fuse`
+
+2.  Verify that the FUSE module is loaded:
+
+    `# dmesg | grep -i fuse `
+
+    `fuse init (API version 7.13)`
+
+### Installing on Red Hat Package Manager (RPM) Distributions
+
+To install Gluster Native Client on RPM distribution-based systems
+
+1.  Install required prerequisites on the client using the following
+    command:
+
+    `$ sudo yum -y install openssh-server wget fuse fuse-libs openib libibverbs`
+
+2.  Ensure that TCP and UDP ports 24007 and 24008 are open on all
+    Gluster servers. Apart from these ports, you need to open one port
+    for each brick starting from port 24009. For example: if you have
+    five bricks, you need to have ports 24009 to 24013 open.
+
+    You can use the following chains with iptables:
+
+    `$ sudo iptables -A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 24007:24008 -j ACCEPT `
+
+    `$ sudo iptables -A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 24009:24014 -j ACCEPT`
+
+    > **Note**
+    >
+    > If you already have iptable chains, make sure that the above
+    > ACCEPT rules precede the DROP rules. This can be achieved by
+    > providing a lower rule number than the DROP rule.
+
+3.  Download the latest glusterfs, glusterfs-fuse, and glusterfs-rdma
+    RPM files to each client. The glusterfs package contains the Gluster
+    Native Client. The glusterfs-fuse package contains the FUSE
+    translator required for mounting on client systems and the
+    glusterfs-rdma packages contain OpenFabrics verbs RDMA module for
+    Infiniband.
+
+    You can download the software at [][].
+
+4.  Install Gluster Native Client on the client.
+
+    `$ sudo rpm -i glusterfs-3.3.0qa30-1.x86_64.rpm `
+
+    `$ sudo rpm -i glusterfs-fuse-3.3.0qa30-1.x86_64.rpm `
+
+    `$ sudo rpm -i glusterfs-rdma-3.3.0qa30-1.x86_64.rpm`
+
+    > **Note**
+    >
+    > The RDMA module is only required when using Infiniband.
+
+### Installing on Debian-based Distributions
+
+To install Gluster Native Client on Debian-based distributions
+
+1.  Install OpenSSH Server on each client using the following command:
+
+    `$ sudo apt-get install openssh-server vim wget`
+
+2.  Download the latest GlusterFS .deb file and checksum to each client.
+
+    You can download the software at [][1].
+
+3.  For each .deb file, get the checksum (using the following command)
+    and compare it against the checksum for that file in the md5sum
+    file.
+
+    `$ md5sum GlusterFS_DEB_file.deb `
+
+    The md5sum of the packages is available at: [][2]
+
+4.  Uninstall GlusterFS v3.1 (or an earlier version) from the client
+    using the following command:
+
+    `$ sudo dpkg -r glusterfs `
+
+    (Optional) Run `$ sudo dpkg -purge glusterfs `to purge the
+    configuration files.
+
+5.  Install Gluster Native Client on the client using the following
+    command:
+
+    `$ sudo dpkg -i GlusterFS_DEB_file `
+
+    For example:
+
+    `$ sudo dpkg -i glusterfs-3.3.x.deb `
+
+6.  Ensure that TCP and UDP ports 24007 and 24008 are open on all
+    Gluster servers. Apart from these ports, you need to open one port
+    for each brick starting from port 24009. For example: if you have
+    five bricks, you need to have ports 24009 to 24013 open.
+
+    You can use the following chains with iptables:
+
+    `$ sudo iptables -A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 24007:24008 -j ACCEPT `
+
+    `$ sudo iptables -A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 24009:24014 -j ACCEPT`
+
+    > **Note**
+    >
+    > If you already have iptable chains, make sure that the above
+    > ACCEPT rules precede the DROP rules. This can be achieved by
+    > providing a lower rule number than the DROP rule.
+
+### Performing a Source Installation
+
+To build and install Gluster Native Client from the source code
+
+1.  Create a new directory using the following commands:
+
+    `# mkdir glusterfs `
+
+    `# cd glusterfs`
+
+2.  Download the source code.
+
+    You can download the source at [][1].
+
+3.  Extract the source code using the following command:
+
+    `# tar -xvzf SOURCE-FILE `
+
+4.  Run the configuration utility using the following command:
+
+    `# ./configure `
+
+    `GlusterFS configure summary `
+
+    `================== `
+
+    `FUSE client : yes `
+
+    `Infiniband verbs : yes `
+
+    `epoll IO multiplex : yes `
+
+    `argp-standalone : no `
+
+    `fusermount : no `
+
+    `readline : yes`
+
+    The configuration summary shows the components that will be built
+    with Gluster Native Client.
+
+5.  Build the Gluster Native Client software using the following
+    commands:
+
+    `# make `
+
+    `# make install`
+
+6.  Verify that the correct version of Gluster Native Client is
+    installed, using the following command:
+
+    `# glusterfs –-version`
+
+Mounting Volumes
+----------------
+
+After installing the Gluster Native Client, you need to mount Gluster
+volumes to access data. There are two methods you can choose:
+
+-   ?
+
+-   ?
+
+After mounting a volume, you can test the mounted volume using the
+procedure described in ?.
+
+> **Note**
+>
+> Server names selected during creation of Volumes should be resolvable
+> in the client machine. You can use appropriate /etc/hosts entries or
+> DNS server to resolve server names to IP addresses.
+
+### Manually Mounting Volumes
+
+To manually mount a Gluster volume
+
+-   To mount a volume, use the following command:
+
+    `# mount -t glusterfs HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR`
+
+    For example:
+
+    `# mount -t glusterfs server1:/test-volume /mnt/glusterfs`
+
+    > **Note**
+    >
+    > The server specified in the mount command is only used to fetch
+    > the gluster configuration volfile describing the volume name.
+    > Subsequently, the client will communicate directly with the
+    > servers mentioned in the volfile (which might not even include the
+    > one used for mount).
+    >
+    > If you see a usage message like "Usage: mount.glusterfs", mount
+    > usually requires you to create a directory to be used as the mount
+    > point. Run "mkdir /mnt/glusterfs" before you attempt to run the
+    > mount command listed above.
+
+**Mounting Options**
+
+You can specify the following options when using the
+`mount -t glusterfs` command. Note that you need to separate all options
+with commas.
+
+backupvolfile-server=server-name
+
+volfile-max-fetch-attempts=number of attempts
+
+log-level=loglevel
+
+log-file=logfile
+
+transport=transport-type
+
+direct-io-mode=[enable|disable]
+
+For example:
+
+`# mount -t glusterfs -o backupvolfile-server=volfile_server2 --volfile-max-fetch-attempts=2 log-level=WARNING,log-file=/var/log/gluster.log server1:/test-volume /mnt/glusterfs`
+
+If `backupvolfile-server` option is added while mounting fuse client,
+when the first volfile server fails, then the server specified in
+`backupvolfile-server` option is used as volfile server to mount the
+client.
+
+In `--volfile-max-fetch-attempts=X` option, specify the number of
+attempts to fetch volume files while mounting a volume. This option is
+useful when you mount a server with multiple IP addresses or when
+round-robin DNS is configured for the server-name..
+
+### Automatically Mounting Volumes
+
+You can configure your system to automatically mount the Gluster volume
+each time your system starts.
+
+The server specified in the mount command is only used to fetch the
+gluster configuration volfile describing the volume name. Subsequently,
+the client will communicate directly with the servers mentioned in the
+volfile (which might not even include the one used for mount).
+
+**To automatically mount a Gluster volume**
+
+-   To mount a volume, edit the /etc/fstab file and add the following
+    line:
+
+    `HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR glusterfs defaults,_netdev 0 0 `
+
+    For example:
+
+    `server1:/test-volume /mnt/glusterfs glusterfs defaults,_netdev 0 0`
+
+**Mounting Options**
+
+You can specify the following options when updating the /etc/fstab file.
+Note that you need to separate all options with commas.
+
+log-level=loglevel
+
+log-file=logfile
+
+transport=transport-type
+
+direct-io-mode=[enable|disable]
+
+For example:
+
+`HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR glusterfs defaults,_netdev,log-level=WARNING,log-file=/var/log/gluster.log 0 0 `
+
+### Testing Mounted Volumes
+
+To test mounted volumes
+
+-   Use the following command:
+
+    `# mount `
+
+    If the gluster volume was successfully mounted, the output of the
+    mount command on the client will be similar to this example:
+
+    `server1:/test-volume on /mnt/glusterfs type fuse.glusterfs (rw,allow_other,default_permissions,max_read=131072`
+
+-   Use the following command:
+
+    `# df`
+
+    The output of df command on the client will display the aggregated
+    storage space from all the bricks in a volume similar to this
+    example:
+
+    `# df -h /mnt/glusterfs Filesystem Size Used Avail Use% Mounted on server1:/test-volume 28T 22T 5.4T 82% /mnt/glusterfs`
+
+-   Change to the directory and list the contents by entering the
+    following:
+
+    `# cd MOUNTDIR `
+
+    `# ls`
+
+-   For example,
+
+    `# cd /mnt/glusterfs `
+
+    `# ls`
+
+NFS
+===
+
+You can use NFS v3 to access to gluster volumes. Extensive testing has
+be done on GNU/Linux clients and NFS implementation in other operating
+system, such as FreeBSD, and Mac OS X, as well as Windows 7
+(Professional and Up), Windows Server 2003, and others, may work with
+gluster NFS server implementation.
+
+GlusterFS now includes network lock manager (NLM) v4. NLM enables
+applications on NFSv3 clients to do record locking on files on NFS
+server. It is started automatically whenever the NFS server is run.
+
+You must install nfs-common package on both servers and clients (only
+for Debian-based) distribution.
+
+This section describes how to use NFS to mount Gluster volumes (both
+manually and automatically) and how to verify that the volume has been
+mounted successfully.
+
+Using NFS to Mount Volumes
+--------------------------
+
+You can use either of the following methods to mount Gluster volumes:
+
+-   ?
+
+-   ?
+
+**Prerequisite**: Install nfs-common package on both servers and clients
+(only for Debian-based distribution), using the following command:
+
+`$ sudo aptitude install nfs-common `
+
+After mounting a volume, you can test the mounted volume using the
+procedure described in ?.
+
+### Manually Mounting Volumes Using NFS
+
+To manually mount a Gluster volume using NFS
+
+-   To mount a volume, use the following command:
+
+    `# mount -t nfs -o vers=3 HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR`
+
+    For example:
+
+    `# mount -t nfs -o vers=3 server1:/test-volume /mnt/glusterfs`
+
+    > **Note**
+    >
+    > Gluster NFS server does not support UDP. If the NFS client you are
+    > using defaults to connecting using UDP, the following message
+    > appears:
+    >
+    > `requested NFS version or transport protocol is not supported`.
+
+    **To connect using TCP**
+
+-   Add the following option to the mount command:
+
+    `-o mountproto=tcp `
+
+    For example:
+
+    `# mount -o mountproto=tcp -t nfs server1:/test-volume /mnt/glusterfs`
+
+**To mount Gluster NFS server from a Solaris client**
+
+-   Use the following command:
+
+    `# mount -o proto=tcp,vers=3 nfs://HOSTNAME-OR-IPADDRESS:38467/VOLNAME MOUNTDIR`
+
+    For example:
+
+    ` # mount -o proto=tcp,vers=3 nfs://server1:38467/test-volume /mnt/glusterfs`
+
+### Automatically Mounting Volumes Using NFS
+
+You can configure your system to automatically mount Gluster volumes
+using NFS each time the system starts.
+
+**To automatically mount a Gluster volume using NFS**
+
+-   To mount a volume, edit the /etc/fstab file and add the following
+    line:
+
+    `HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR nfs defaults,_netdev,vers=3 0 0`
+
+    For example,
+
+    `server1:/test-volume /mnt/glusterfs nfs defaults,_netdev,vers=3 0 0`
+
+    > **Note**
+    >
+    > Gluster NFS server does not support UDP. If the NFS client you are
+    > using defaults to connecting using UDP, the following message
+    > appears:
+    >
+    > `requested NFS version or transport protocol is not supported.`
+
+    To connect using TCP
+
+-   Add the following entry in /etc/fstab file :
+
+    `HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR nfs defaults,_netdev,mountproto=tcp 0 0`
+
+    For example,
+
+    `server1:/test-volume /mnt/glusterfs nfs defaults,_netdev,mountproto=tcp 0 0`
+
+**To automount NFS mounts**
+
+Gluster supports \*nix standard method of automounting NFS mounts.
+Update the /etc/auto.master and /etc/auto.misc and restart the autofs
+service. After that, whenever a user or process attempts to access the
+directory it will be mounted in the background.
+
+### Testing Volumes Mounted Using NFS
+
+You can confirm that Gluster directories are mounting successfully.
+
+**To test mounted volumes**
+
+-   Use the mount command by entering the following:
+
+    `# mount`
+
+    For example, the output of the mount command on the client will
+    display an entry like the following:
+
+    `server1:/test-volume on /mnt/glusterfs type nfs (rw,vers=3,addr=server1)`
+
+-   Use the df command by entering the following:
+
+    `# df`
+
+    For example, the output of df command on the client will display the
+    aggregated storage space from all the bricks in a volume.
+
+        # df -h /mnt/glusterfs 
+        Filesystem              Size Used Avail Use% Mounted on 
+        server1:/test-volume    28T  22T  5.4T  82%  /mnt/glusterfs
+
+-   Change to the directory and list the contents by entering the
+    following:
+
+    `# cd MOUNTDIR`
+
+    `# ls`
+
+    For example,
+
+    `
+                    
+                  `
+
+    `# ls`
+
+CIFS
+====
+
+You can use CIFS to access to volumes when using Microsoft Windows as
+well as SAMBA clients. For this access method, Samba packages need to be
+present on the client side. You can export glusterfs mount point as the
+samba export, and then mount it using CIFS protocol.
+
+This section describes how to mount CIFS shares on Microsoft
+Windows-based clients (both manually and automatically) and how to
+verify that the volume has mounted successfully.
+
+> **Note**
+>
+> CIFS access using the Mac OS X Finder is not supported, however, you
+> can use the Mac OS X command line to access Gluster volumes using
+> CIFS.
+
+Using CIFS to Mount Volumes
+---------------------------
+
+You can use either of the following methods to mount Gluster volumes:
+
+-   ?
+
+-   ?
+
+After mounting a volume, you can test the mounted volume using the
+procedure described in ?.
+
+You can also use Samba for exporting Gluster Volumes through CIFS
+protocol.
+
+### Exporting Gluster Volumes Through Samba
+
+We recommend you to use Samba for exporting Gluster volumes through the
+CIFS protocol.
+
+**To export volumes through CIFS protocol**
+
+1.  Mount a Gluster volume. For more information on mounting volumes,
+    see ?.
+
+2.  Setup Samba configuration to export the mount point of the Gluster
+    volume.
+
+    For example, if a Gluster volume is mounted on /mnt/gluster, you
+    must edit smb.conf file to enable exporting this through CIFS. Open
+    smb.conf file in an editor and add the following lines for a simple
+    configuration:
+
+    [glustertest]
+
+    comment = For testing a Gluster volume exported through CIFS
+
+    path = /mnt/glusterfs
+
+    read only = no
+
+    guest ok = yes
+
+Save the changes and start the smb service using your systems init
+scripts (/etc/init.d/smb [re]start).
+
+> **Note**
+>
+> To be able mount from any server in the trusted storage pool, you must
+> repeat these steps on each Gluster node. For more advanced
+> configurations, see Samba documentation.
+
+### Manually Mounting Volumes Using CIFS
+
+You can manually mount Gluster volumes using CIFS on Microsoft
+Windows-based client machines.
+
+**To manually mount a Gluster volume using CIFS**
+
+1.  Using Windows Explorer, choose **Tools \> Map Network Drive…** from
+    the menu. The **Map Network Drive**window appears.
+
+2.  Choose the drive letter using the **Drive** drop-down list.
+
+3.  Click **Browse**, select the volume to map to the network drive, and
+    click **OK**.
+
+4.  Click **Finish.**
+
+The network drive (mapped to the volume) appears in the Computer window.
+
+**Alternatively, to manually mount a Gluster volume using CIFS.**
+
+-   Click **Start \> Run** and enter the following:
+
+    `
+                    
+                  `
+
+    For example:
+
+    `
+                    
+                  `
+
+### Automatically Mounting Volumes Using CIFS
+
+You can configure your system to automatically mount Gluster volumes
+using CIFS on Microsoft Windows-based clients each time the system
+starts.
+
+**To automatically mount a Gluster volume using CIFS**
+
+The network drive (mapped to the volume) appears in the Computer window
+and is reconnected each time the system starts.
+
+1.  Using Windows Explorer, choose **Tools \> Map Network Drive…** from
+    the menu. The **Map Network Drive**window appears.
+
+2.  Choose the drive letter using the **Drive** drop-down list.
+
+3.  Click **Browse**, select the volume to map to the network drive, and
+    click **OK**.
+
+4.  Click the **Reconnect** at logon checkbox.
+
+5.  Click **Finish.**
+
+### Testing Volumes Mounted Using CIFS
+
+You can confirm that Gluster directories are mounting successfully by
+navigating to the directory using Windows Explorer.
+
+  []: http://bits.gluster.com/gluster/glusterfs/3.3.0qa30/x86_64/
+  [1]: http://www.gluster.org/download/
+  [2]: http://download.gluster.com/pub/gluster/glusterfs
diff --git a/doc/admin-guide/en-US/markdown/admin_start_stop_daemon.md b/doc/admin-guide/en-US/markdown/admin_start_stop_daemon.md
new file mode 100644
index 000000000..43251cd01
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_start_stop_daemon.md
@@ -0,0 +1,70 @@
+Managing the glusterd Service
+=============================
+
+After installing GlusterFS, you must start glusterd service. The
+glusterd service serves as the Gluster elastic volume manager,
+overseeing glusterfs processes, and co-ordinating dynamic volume
+operations, such as adding and removing volumes across multiple storage
+servers non-disruptively.
+
+This section describes how to start the glusterd service in the
+following ways:
+
+-   ?
+
+-   ?
+
+> **Note**
+>
+> You must start glusterd on all GlusterFS servers.
+
+Starting and Stopping glusterd Manually
+=======================================
+
+This section describes how to start and stop glusterd manually
+
+-   To start glusterd manually, enter the following command:
+
+    `# /etc/init.d/glusterd start `
+
+-   To stop glusterd manually, enter the following command:
+
+    `# /etc/init.d/glusterd stop`
+
+Starting glusterd Automatically
+===============================
+
+This section describes how to configure the system to automatically
+start the glusterd service every time the system boots.
+
+To automatically start the glusterd service every time the system boots,
+enter the following from the command line:
+
+`# chkconfig glusterd on `
+
+Red Hat-based Systems
+---------------------
+
+To configure Red Hat-based systems to automatically start the glusterd
+service every time the system boots, enter the following from the
+command line:
+
+`# chkconfig glusterd on `
+
+Debian-based Systems
+--------------------
+
+To configure Debian-based systems to automatically start the glusterd
+service every time the system boots, enter the following from the
+command line:
+
+`# update-rc.d glusterd defaults`
+
+Systems Other than Red Hat and Debain
+-------------------------------------
+
+To configure systems other than Red Hat or Debian to automatically start
+the glusterd service every time the system boots, enter the following
+entry to the*/etc/rc.local* file:
+
+`# echo "glusterd" >> /etc/rc.local `
diff --git a/doc/admin-guide/en-US/markdown/admin_storage_pools.md b/doc/admin-guide/en-US/markdown/admin_storage_pools.md
new file mode 100644
index 000000000..2a35cbea5
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_storage_pools.md
@@ -0,0 +1,73 @@
+Setting up Trusted Storage Pools
+================================
+
+Before you can configure a GlusterFS volume, you must create a trusted
+storage pool consisting of the storage servers that provides bricks to a
+volume.
+
+A storage pool is a trusted network of storage servers. When you start
+the first server, the storage pool consists of that server alone. To add
+additional storage servers to the storage pool, you can use the probe
+command from a storage server that is already trusted.
+
+> **Note**
+>
+> Do not self-probe the first server/localhost.
+
+The GlusterFS service must be running on all storage servers that you
+want to add to the storage pool. See ? for more information.
+
+Adding Servers to Trusted Storage Pool
+======================================
+
+To create a trusted storage pool, add servers to the trusted storage
+pool
+
+1.  The hostnames used to create the storage pool must be resolvable by
+    DNS.
+
+    To add a server to the storage pool:
+
+    `# gluster peer probe `
+
+    For example, to create a trusted storage pool of four servers, add
+    three servers to the storage pool from server1:
+
+        # gluster peer probe server2
+        Probe successful
+
+        # gluster peer probe server3
+        Probe successful
+
+        # gluster peer probe server4
+        Probe successful
+
+2.  Verify the peer status from the first server using the following
+    commands:
+
+        # gluster peer status
+        Number of Peers: 3
+
+        Hostname: server2
+        Uuid: 5e987bda-16dd-43c2-835b-08b7d55e94e5
+        State: Peer in Cluster (Connected)
+
+        Hostname: server3
+        Uuid: 1e0ca3aa-9ef7-4f66-8f15-cbc348f29ff7
+        State: Peer in Cluster (Connected)
+
+        Hostname: server4
+        Uuid: 3e0caba-9df7-4f66-8e5d-cbc348f29ff7
+        State: Peer in Cluster (Connected)
+
+Removing Servers from the Trusted Storage Pool
+==============================================
+
+To remove a server from the storage pool:
+
+`# gluster peer detach`
+
+For example, to remove server4 from the trusted storage pool:
+
+    # gluster peer detach server4
+    Detach successful
diff --git a/doc/admin-guide/en-US/markdown/admin_troubleshooting.md b/doc/admin-guide/en-US/markdown/admin_troubleshooting.md
new file mode 100644
index 000000000..88fb85c24
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/admin_troubleshooting.md
@@ -0,0 +1,543 @@
+Troubleshooting GlusterFS
+=========================
+
+This section describes how to manage GlusterFS logs and most common
+troubleshooting scenarios related to GlusterFS.
+
+Managing GlusterFS Logs
+=======================
+
+This section describes how to manage GlusterFS logs by performing the
+following operation:
+
+-   Rotating Logs
+
+Rotating Logs
+-------------
+
+Administrators can rotate the log file in a volume, as needed.
+
+**To rotate a log file**
+
+-   Rotate the log file using the following command:
+
+    `# gluster volume log rotate `
+
+    For example, to rotate the log file on test-volume:
+
+        # gluster volume log rotate test-volume
+        log rotate successful
+
+    > **Note**
+    >
+    > When a log file is rotated, the contents of the current log file
+    > are moved to log-file- name.epoch-time-stamp.
+
+Troubleshooting Geo-replication
+===============================
+
+This section describes the most common troubleshooting scenarios related
+to GlusterFS Geo-replication.
+
+Locating Log Files
+------------------
+
+For every Geo-replication session, the following three log files are
+associated to it (four, if the slave is a gluster volume):
+
+-   Master-log-file - log file for the process which monitors the Master
+    volume
+
+-   Slave-log-file - log file for process which initiates the changes in
+    slave
+
+-   Master-gluster-log-file - log file for the maintenance mount point
+    that Geo-replication module uses to monitor the master volume
+
+-   Slave-gluster-log-file - is the slave's counterpart of it
+
+**Master Log File**
+
+To get the Master-log-file for geo-replication, use the following
+command:
+
+`gluster volume geo-replication  config log-file`
+
+For example:
+
+`# gluster volume geo-replication Volume1 example.com:/data/remote_dir config log-file `
+
+**Slave Log File**
+
+To get the log file for Geo-replication on slave (glusterd must be
+running on slave machine), use the following commands:
+
+1.  On master, run the following command:
+
+    `# gluster volume geo-replication Volume1 example.com:/data/remote_dir config session-owner 5f6e5200-756f-11e0-a1f0-0800200c9a66 `
+
+    Displays the session owner details.
+
+2.  On slave, run the following command:
+
+    `# gluster volume geo-replication /data/remote_dir config log-file /var/log/gluster/${session-owner}:remote-mirror.log `
+
+3.  Replace the session owner details (output of Step 1) to the output
+    of the Step 2 to get the location of the log file.
+
+    `/var/log/gluster/5f6e5200-756f-11e0-a1f0-0800200c9a66:remote-mirror.log`
+
+Rotating Geo-replication Logs
+-----------------------------
+
+Administrators can rotate the log file of a particular master-slave
+session, as needed. When you run geo-replication's ` log-rotate`
+command, the log file is backed up with the current timestamp suffixed
+to the file name and signal is sent to gsyncd to start logging to a new
+log file.
+
+**To rotate a geo-replication log file**
+
+-   Rotate log file for a particular master-slave session using the
+    following command:
+
+    `# gluster volume geo-replication  log-rotate`
+
+    For example, to rotate the log file of master `Volume1` and slave
+    `example.com:/data/remote_dir` :
+
+        # gluster volume geo-replication Volume1 example.com:/data/remote_dir log rotate
+        log rotate successful
+
+-   Rotate log file for all sessions for a master volume using the
+    following command:
+
+    `# gluster volume geo-replication  log-rotate`
+
+    For example, to rotate the log file of master `Volume1`:
+
+        # gluster volume geo-replication Volume1 log rotate
+        log rotate successful
+
+-   Rotate log file for all sessions using the following command:
+
+    `# gluster volume geo-replication log-rotate`
+
+    For example, to rotate the log file for all sessions:
+
+        # gluster volume geo-replication log rotate
+        log rotate successful
+
+Synchronization is not complete
+-------------------------------
+
+**Description**: GlusterFS Geo-replication did not synchronize the data
+completely but still the geo- replication status displayed is OK.
+
+**Solution**: You can enforce a full sync of the data by erasing the
+index and restarting GlusterFS Geo- replication. After restarting,
+GlusterFS Geo-replication begins synchronizing all the data. All files
+are compared using checksum, which can be a lengthy and high resource
+utilization operation on large data sets. If the error situation
+persists, contact Red Hat Support.
+
+For more information about erasing index, see ?.
+
+Issues in Data Synchronization
+------------------------------
+
+**Description**: Geo-replication display status as OK, but the files do
+not get synced, only directories and symlink gets synced with the
+following error message in the log:
+
+[2011-05-02 13:42:13.467644] E [master:288:regjob] GMaster: failed to
+sync ./some\_file\`
+
+**Solution**: Geo-replication invokes rsync v3.0.0 or higher on the host
+and the remote machine. You must verify if you have installed the
+required version.
+
+Geo-replication status displays Faulty very often
+-------------------------------------------------
+
+**Description**: Geo-replication displays status as faulty very often
+with a backtrace similar to the following:
+
+2011-04-28 14:06:18.378859] E [syncdutils:131:log\_raise\_exception]
+\<top\>: FAIL: Traceback (most recent call last): File
+"/usr/local/libexec/glusterfs/python/syncdaemon/syncdutils.py", line
+152, in twraptf(\*aa) File
+"/usr/local/libexec/glusterfs/python/syncdaemon/repce.py", line 118, in
+listen rid, exc, res = recv(self.inf) File
+"/usr/local/libexec/glusterfs/python/syncdaemon/repce.py", line 42, in
+recv return pickle.load(inf) EOFError
+
+**Solution**: This error indicates that the RPC communication between
+the master gsyncd module and slave gsyncd module is broken and this can
+happen for various reasons. Check if it satisfies all the following
+pre-requisites:
+
+-   Password-less SSH is set up properly between the host and the remote
+    machine.
+
+-   If FUSE is installed in the machine, because geo-replication module
+    mounts the GlusterFS volume using FUSE to sync data.
+
+-   If the **Slave** is a volume, check if that volume is started.
+
+-   If the Slave is a plain directory, verify if the directory has been
+    created already with the required permissions.
+
+-   If GlusterFS 3.2 or higher is not installed in the default location
+    (in Master) and has been prefixed to be installed in a custom
+    location, configure the `gluster-command` for it to point to the
+    exact location.
+
+-   If GlusterFS 3.2 or higher is not installed in the default location
+    (in slave) and has been prefixed to be installed in a custom
+    location, configure the `remote-gsyncd-command` for it to point to
+    the exact place where gsyncd is located.
+
+Intermediate Master goes to Faulty State
+----------------------------------------
+
+**Description**: In a cascading set-up, the intermediate master goes to
+faulty state with the following log:
+
+raise RuntimeError ("aborting on uuid change from %s to %s" % \\
+RuntimeError: aborting on uuid change from af07e07c-427f-4586-ab9f-
+4bf7d299be81 to de6b5040-8f4e-4575-8831-c4f55bd41154
+
+**Solution**: In a cascading set-up the Intermediate master is loyal to
+the original primary master. The above log means that the
+geo-replication module has detected change in primary master. If this is
+the desired behavior, delete the config option volume-id in the session
+initiated from the intermediate master.
+
+Troubleshooting POSIX ACLs
+==========================
+
+This section describes the most common troubleshooting issues related to
+POSIX ACLs.
+
+setfacl command fails with “setfacl: \<file or directory name\>: Operation not supported” error
+-----------------------------------------------------------------------------------------------
+
+You may face this error when the backend file systems in one of the
+servers is not mounted with the "-o acl" option. The same can be
+confirmed by viewing the following error message in the log file of the
+server "Posix access control list is not supported".
+
+**Solution**: Remount the backend file system with "-o acl" option. For
+more information, see ?.
+
+Troubleshooting Hadoop Compatible Storage
+=========================================
+
+This section describes the most common troubleshooting issues related to
+Hadoop Compatible Storage.
+
+Time Sync
+---------
+
+Running MapReduce job may throw exceptions if the time is out-of-sync on
+the hosts in the cluster.
+
+**Solution**: Sync the time on all hosts using ntpd program.
+
+Troubleshooting NFS
+===================
+
+This section describes the most common troubleshooting issues related to
+NFS .
+
+mount command on NFS client fails with “RPC Error: Program not registered”
+--------------------------------------------------------------------------
+
+Start portmap or rpcbind service on the NFS server.
+
+This error is encountered when the server has not started correctly.
+
+On most Linux distributions this is fixed by starting portmap:
+
+`$ /etc/init.d/portmap start`
+
+On some distributions where portmap has been replaced by rpcbind, the
+following command is required:
+
+`$ /etc/init.d/rpcbind start `
+
+After starting portmap or rpcbind, gluster NFS server needs to be
+restarted.
+
+NFS server start-up fails with “Port is already in use” error in the log file."
+-------------------------------------------------------------------------------
+
+Another Gluster NFS server is running on the same machine.
+
+This error can arise in case there is already a Gluster NFS server
+running on the same machine. This situation can be confirmed from the
+log file, if the following error lines exist:
+
+    [2010-05-26 23:40:49] E [rpc-socket.c:126:rpcsvc_socket_listen] rpc-socket: binding socket failed:Address already in use
+    [2010-05-26 23:40:49] E [rpc-socket.c:129:rpcsvc_socket_listen] rpc-socket: Port is already in use 
+    [2010-05-26 23:40:49] E [rpcsvc.c:2636:rpcsvc_stage_program_register] rpc-service: could not create listening connection 
+    [2010-05-26 23:40:49] E [rpcsvc.c:2675:rpcsvc_program_register] rpc-service: stage registration of program failed 
+    [2010-05-26 23:40:49] E [rpcsvc.c:2695:rpcsvc_program_register] rpc-service: Program registration failed: MOUNT3, Num: 100005, Ver: 3, Port: 38465 
+    [2010-05-26 23:40:49] E [nfs.c:125:nfs_init_versions] nfs: Program init failed 
+    [2010-05-26 23:40:49] C [nfs.c:531:notify] nfs: Failed to initialize protocols
+
+To resolve this error one of the Gluster NFS servers will have to be
+shutdown. At this time, Gluster NFS server does not support running
+multiple NFS servers on the same machine.
+
+mount command fails with “rpc.statd” related error message
+----------------------------------------------------------
+
+If the mount command fails with the following error message:
+
+mount.nfs: rpc.statd is not running but is required for remote locking.
+mount.nfs: Either use '-o nolock' to keep locks local, or start statd.
+
+Start rpc.statd
+
+For NFS clients to mount the NFS server, rpc.statd service must be
+running on the clients.
+
+Start rpc.statd service by running the following command:
+
+`$ rpc.statd `
+
+mount command takes too long to finish.
+---------------------------------------
+
+Start rpcbind service on the NFS client.
+
+The problem is that the rpcbind or portmap service is not running on the
+NFS client. The resolution for this is to start either of these services
+by running the following command:
+
+`$ /etc/init.d/portmap start`
+
+On some distributions where portmap has been replaced by rpcbind, the
+following command is required:
+
+`$ /etc/init.d/rpcbind start`
+
+NFS server glusterfsd starts but initialization fails with “nfsrpc- service: portmap registration of program failed” error message in the log.
+----------------------------------------------------------------------------------------------------------------------------------------------
+
+NFS start-up can succeed but the initialization of the NFS service can
+still fail preventing clients from accessing the mount points. Such a
+situation can be confirmed from the following error messages in the log
+file:
+
+    [2010-05-26 23:33:47] E [rpcsvc.c:2598:rpcsvc_program_register_portmap] rpc-service: Could notregister with portmap 
+    [2010-05-26 23:33:47] E [rpcsvc.c:2682:rpcsvc_program_register] rpc-service: portmap registration of program failed
+    [2010-05-26 23:33:47] E [rpcsvc.c:2695:rpcsvc_program_register] rpc-service: Program registration failed: MOUNT3, Num: 100005, Ver: 3, Port: 38465
+    [2010-05-26 23:33:47] E [nfs.c:125:nfs_init_versions] nfs: Program init failed
+    [2010-05-26 23:33:47] C [nfs.c:531:notify] nfs: Failed to initialize protocols
+    [2010-05-26 23:33:49] E [rpcsvc.c:2614:rpcsvc_program_unregister_portmap] rpc-service: Could not unregister with portmap
+    [2010-05-26 23:33:49] E [rpcsvc.c:2731:rpcsvc_program_unregister] rpc-service: portmap unregistration of program failed
+    [2010-05-26 23:33:49] E [rpcsvc.c:2744:rpcsvc_program_unregister] rpc-service: Program unregistration failed: MOUNT3, Num: 100005, Ver: 3, Port: 38465
+
+1.  Start portmap or rpcbind service on the NFS server.
+
+    On most Linux distributions, portmap can be started using the
+    following command:
+
+    `$ /etc/init.d/portmap start `
+
+    On some distributions where portmap has been replaced by rpcbind,
+    run the following command:
+
+    `$ /etc/init.d/rpcbind start `
+
+    After starting portmap or rpcbind, gluster NFS server needs to be
+    restarted.
+
+2.  Stop another NFS server running on the same machine.
+
+    Such an error is also seen when there is another NFS server running
+    on the same machine but it is not the Gluster NFS server. On Linux
+    systems, this could be the kernel NFS server. Resolution involves
+    stopping the other NFS server or not running the Gluster NFS server
+    on the machine. Before stopping the kernel NFS server, ensure that
+    no critical service depends on access to that NFS server's exports.
+
+    On Linux, kernel NFS servers can be stopped by using either of the
+    following commands depending on the distribution in use:
+
+    `$ /etc/init.d/nfs-kernel-server stop`
+
+    `$ /etc/init.d/nfs stop`
+
+3.  Restart Gluster NFS server.
+
+mount command fails with NFS server failed error.
+-------------------------------------------------
+
+mount command fails with following error
+
+*mount: mount to NFS server '10.1.10.11' failed: timed out (retrying).*
+
+Perform one of the following to resolve this issue:
+
+1.  Disable name lookup requests from NFS server to a DNS server.
+
+    The NFS server attempts to authenticate NFS clients by performing a
+    reverse DNS lookup to match hostnames in the volume file with the
+    client IP addresses. There can be a situation where the NFS server
+    either is not able to connect to the DNS server or the DNS server is
+    taking too long to responsd to DNS request. These delays can result
+    in delayed replies from the NFS server to the NFS client resulting
+    in the timeout error seen above.
+
+    NFS server provides a work-around that disables DNS requests,
+    instead relying only on the client IP addresses for authentication.
+    The following option can be added for successful mounting in such
+    situations:
+
+    `option rpc-auth.addr.namelookup off `
+
+    > **Note**
+    >
+    > Note: Remember that disabling the NFS server forces authentication
+    > of clients to use only IP addresses and if the authentication
+    > rules in the volume file use hostnames, those authentication rules
+    > will fail and disallow mounting for those clients.
+
+    or
+
+2.  NFS version used by the NFS client is other than version 3.
+
+    Gluster NFS server supports version 3 of NFS protocol. In recent
+    Linux kernels, the default NFS version has been changed from 3 to 4.
+    It is possible that the client machine is unable to connect to the
+    Gluster NFS server because it is using version 4 messages which are
+    not understood by Gluster NFS server. The timeout can be resolved by
+    forcing the NFS client to use version 3. The **vers** option to
+    mount command is used for this purpose:
+
+    `$ mount  -o vers=3 `
+
+showmount fails with clnt\_create: RPC: Unable to receive
+---------------------------------------------------------
+
+Check your firewall setting to open ports 111 for portmap
+requests/replies and Gluster NFS server requests/replies. Gluster NFS
+server operates over the following port numbers: 38465, 38466, and
+38467.
+
+For more information, see ?.
+
+Application fails with "Invalid argument" or "Value too large for defined data type" error.
+-------------------------------------------------------------------------------------------
+
+These two errors generally happen for 32-bit nfs clients or applications
+that do not support 64-bit inode numbers or large files. Use the
+following option from the CLI to make Gluster NFS return 32-bit inode
+numbers instead: nfs.enable-ino32 \<on|off\>
+
+Applications that will benefit are those that were either:
+
+-   built 32-bit and run on 32-bit machines such that they do not
+    support large files by default
+
+-   built 32-bit on 64-bit systems
+
+This option is disabled by default so NFS returns 64-bit inode numbers
+by default.
+
+Applications which can be rebuilt from source are recommended to rebuild
+using the following flag with gcc:
+
+` -D_FILE_OFFSET_BITS=64`
+
+Troubleshooting File Locks
+==========================
+
+In GlusterFS 3.3 you can use `statedump` command to list the locks held
+on files. The statedump output also provides information on each lock
+with its range, basename, PID of the application holding the lock, and
+so on. You can analyze the output to know about the locks whose
+owner/application is no longer running or interested in that lock. After
+ensuring that the no application is using the file, you can clear the
+lock using the following `clear lock` command:
+
+`# `
+
+For more information on performing `statedump`, see ?
+
+**To identify locked file and clear locks**
+
+1.  Perform statedump on the volume to view the files that are locked
+    using the following command:
+
+    `# gluster volume statedump  inode`
+
+    For example, to display statedump of test-volume:
+
+        # gluster volume statedump test-volume
+        Volume statedump successful
+
+    The statedump files are created on the brick servers in the` /tmp`
+    directory or in the directory set using `server.statedump-path`
+    volume option. The naming convention of the dump file is
+    `<brick-path>.<brick-pid>.dump`.
+
+    The following are the sample contents of the statedump file. It
+    indicates that GlusterFS has entered into a state where there is an
+    entry lock (entrylk) and an inode lock (inodelk). Ensure that those
+    are stale locks and no resources own them.
+
+        [xlator.features.locks.vol-locks.inode]
+        path=/
+        mandatory=0
+        entrylk-count=1
+        lock-dump.domain.domain=vol-replicate-0
+        xlator.feature.locks.lock-dump.domain.entrylk.entrylk[0](ACTIVE)=type=ENTRYLK_WRLCK on basename=file1, pid = 714782904, owner=ffffff2a3c7f0000, transport=0x20e0670, , granted at Mon Feb 27 16:01:01 2012
+
+        conn.2.bound_xl./gfs/brick1.hashsize=14057
+        conn.2.bound_xl./gfs/brick1.name=/gfs/brick1/inode
+        conn.2.bound_xl./gfs/brick1.lru_limit=16384
+        conn.2.bound_xl./gfs/brick1.active_size=2
+        conn.2.bound_xl./gfs/brick1.lru_size=0
+        conn.2.bound_xl./gfs/brick1.purge_size=0
+
+        [conn.2.bound_xl./gfs/brick1.active.1]
+        gfid=538a3d4a-01b0-4d03-9dc9-843cd8704d07
+        nlookup=1
+        ref=2
+        ia_type=1
+        [xlator.features.locks.vol-locks.inode]
+        path=/file1
+        mandatory=0
+        inodelk-count=1
+        lock-dump.domain.domain=vol-replicate-0
+        inodelk.inodelk[0](ACTIVE)=type=WRITE, whence=0, start=0, len=0, pid = 714787072, owner=00ffff2a3c7f0000, transport=0x20e0670, , granted at Mon Feb 27 16:01:01 2012
+
+2.  Clear the lock using the following command:
+
+    `# `
+
+    For example, to clear the entry lock on `file1` of test-volume:
+
+        # gluster volume clear-locks test-volume / kind granted entry file1
+        Volume clear-locks successful
+        vol-locks: entry blocked locks=0 granted locks=1
+
+3.  Clear the inode lock using the following command:
+
+    `# `
+
+    For example, to clear the inode lock on `file1` of test-volume:
+
+        # gluster  volume clear-locks test-volume /file1 kind granted inode 0,0-0
+        Volume clear-locks successful
+        vol-locks: inode blocked locks=0 granted locks=1
+
+    You can perform statedump on test-volume again to verify that the
+    above inode and entry locks are cleared.
+
+
diff --git a/doc/admin-guide/en-US/markdown/gfs_introduction.md b/doc/admin-guide/en-US/markdown/gfs_introduction.md
new file mode 100644
index 000000000..fd2c53dc9
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/gfs_introduction.md
@@ -0,0 +1,50 @@
+Introducing Gluster File System
+===============================
+
+GlusterFS is an open source, clustered file system capable of scaling to
+several petabytes and handling thousands of clients. GlusterFS can be
+flexibly combined with commodity physical, virtual, and cloud resources
+to deliver highly available and performant enterprise storage at a
+fraction of the cost of traditional solutions.
+
+GlusterFS clusters together storage building blocks over Infiniband RDMA
+and/or TCP/IP interconnect, aggregating disk and memory resources and
+managing data in a single global namespace. GlusterFS is based on a
+stackable user space design, delivering exceptional performance for
+diverse workloads.
+
+![ Virtualized Cloud Environments ][]
+
+GlusterFS is designed for today's high-performance, virtualized cloud
+environments. Unlike traditional data centers, cloud environments
+require multi-tenancy along with the ability to grow or shrink resources
+on demand. Enterprises can scale capacity, performance, and availability
+on demand, with no vendor lock-in, across on-premise, public cloud, and
+hybrid environments.
+
+GlusterFS is in production at thousands of enterprises spanning media,
+healthcare, government, education, web 2.0, and financial services. The
+following table lists the commercial offerings and its documentation
+location:
+
+  ------------------------------------------------------------------------
+  Product     Documentation Location
+  ----------- ------------------------------------------------------------
+  Red Hat     [][]
+  Storage     
+  Software    
+  Appliance   
+
+  Red Hat     [][1]
+  Virtual     
+  Storage     
+  Appliance   
+
+  Red Hat     [][2]
+  Storage     
+  ------------------------------------------------------------------------
+
+  [ Virtualized Cloud Environments ]: images/640px-GlusterFS_Architecture.png
+  []: http://docs.redhat.com/docs/en-US/Red_Hat_Storage_Software_Appliance/index.html
+  [1]: http://docs.redhat.com/docs/en-US/Red_Hat_Virtual_Storage_Appliance/index.html
+  [2]: http://docs.redhat.com/docs/en-US/Red_Hat_Storage/index.html
diff --git a/doc/admin-guide/en-US/markdown/glossary.md b/doc/admin-guide/en-US/markdown/glossary.md
new file mode 100644
index 000000000..0febaff8f
--- /dev/null
+++ b/doc/admin-guide/en-US/markdown/glossary.md
@@ -0,0 +1,134 @@
+Glossary
+========
+
+Brick
+:   A Brick is the GlusterFS basic unit of storage, represented by an
+    export directory on a server in the trusted storage pool. A Brick is
+    expressed by combining a server with an export directory in the
+    following format:
+
+    `SERVER:EXPORT`
+
+    For example:
+
+    `myhostname:/exports/myexportdir/`
+
+Cluster
+:   A cluster is a group of linked computers, working together closely
+    thus in many respects forming a single computer.
+
+Distributed File System
+:   A file system that allows multiple clients to concurrently access
+    data over a computer network.
+
+Filesystem
+:   A method of storing and organizing computer files and their data.
+    Essentially, it organizes these files into a database for the
+    storage, organization, manipulation, and retrieval by the computer's
+    operating system.
+
+    Source: [Wikipedia][]
+
+FUSE
+:   Filesystem in Userspace (FUSE) is a loadable kernel module for
+    Unix-like computer operating systems that lets non-privileged users
+    create their own file systems without editing kernel code. This is
+    achieved by running file system code in user space while the FUSE
+    module provides only a "bridge" to the actual kernel interfaces.
+
+    Source: [Wikipedia][1]
+
+Geo-Replication
+:   Geo-replication provides a continuous, asynchronous, and incremental
+    replication service from site to another over Local Area Networks
+    (LAN), Wide Area Network (WAN), and across the Internet.
+
+glusterd
+:   The Gluster management daemon that needs to run on all servers in
+    the trusted storage pool.
+
+Metadata
+:   Metadata is data providing information about one or more other
+    pieces of data.
+
+Namespace
+:   Namespace is an abstract container or environment created to hold a
+    logical grouping of unique identifiers or symbols. Each Gluster
+    volume exposes a single namespace as a POSIX mount point that
+    contains every file in the cluster.
+
+Open Source
+:   Open source describes practices in production and development that
+    promote access to the end product's source materials. Some consider
+    open source a philosophy, others consider it a pragmatic
+    methodology.
+
+    Before the term open source became widely adopted, developers and
+    producers used a variety of phrases to describe the concept; open
+    source gained hold with the rise of the Internet, and the attendant
+    need for massive retooling of the computing source code.
+
+    Opening the source code enabled a self-enhancing diversity of
+    production models, communication paths, and interactive communities.
+    Subsequently, a new, three-word phrase "open source software" was
+    born to describe the environment that the new copyright, licensing,
+    domain, and consumer issues created.
+
+    Source: [Wikipedia][2]
+
+Petabyte
+:   A petabyte (derived from the SI prefix peta- ) is a unit of
+    information equal to one quadrillion (short scale) bytes, or 1000
+    terabytes. The unit symbol for the petabyte is PB. The prefix peta-
+    (P) indicates a power of 1000:
+
+    1 PB = 1,000,000,000,000,000 B = 10005 B = 1015 B.
+
+    The term "pebibyte" (PiB), using a binary prefix, is used for the
+    corresponding power of 1024.
+
+    Source: [Wikipedia][3]
+
+POSIX
+:   Portable Operating System Interface (for Unix) is the name of a
+    family of related standards specified by the IEEE to define the
+    application programming interface (API), along with shell and
+    utilities interfaces for software compatible with variants of the
+    Unix operating system. Gluster exports a fully POSIX compliant file
+    system.
+
+RAID
+:   Redundant Array of Inexpensive Disks (RAID) is a technology that
+    provides increased storage reliability through redundancy, combining
+    multiple low-cost, less-reliable disk drives components into a
+    logical unit where all drives in the array are interdependent.
+
+RRDNS
+:   Round Robin Domain Name Service (RRDNS) is a method to distribute
+    load across application servers. RRDNS is implemented by creating
+    multiple A records with the same name and different IP addresses in
+    the zone file of a DNS server.
+
+Trusted Storage Pool
+:   A storage pool is a trusted network of storage servers. When you
+    start the first server, the storage pool consists of that server
+    alone.
+
+Userspace
+:   Applications running in user space don’t directly interact with
+    hardware, instead using the kernel to moderate access. Userspace
+    applications are generally more portable than applications in kernel
+    space. Gluster is a user space application.
+
+Volfile
+:   Volfile is a configuration file used by glusterfs process. Volfile
+    will be usually located at `/var/lib/glusterd/vols/VOLNAME`.
+
+Volume
+:   A volume is a logical collection of bricks. Most of the gluster
+    management operations happen on the volume.
+
+  [Wikipedia]: http://en.wikipedia.org/wiki/Filesystem
+  [1]: http://en.wikipedia.org/wiki/Filesystem_in_Userspace
+  [2]: http://en.wikipedia.org/wiki/Open_source
+  [3]: http://en.wikipedia.org/wiki/Petabyte
diff --git a/doc/authentication.txt b/doc/authentication.txt
index 70aafd933..73cb21d73 100644
--- a/doc/authentication.txt
+++ b/doc/authentication.txt
@@ -48,7 +48,7 @@
       protocol/client:
 	option remote-subvolume foo-brick
 	* Client is connecting from a.b.c.d
-      
+
       protocol/server:
 	option auth.addr.foo-brick.allow a.b.c.d,e.f.g.h,i.j.k.l #, other ip addresses from which clients are allowed to connect to foo-brick
 
@@ -79,19 +79,19 @@
   * reject only "user shoo from a.b.c.d"
     protcol/client:
 	option remote-subvolume shoo-brick
-    
+
     protocol/server:
 	# observe that no "option auth.login.shoo-brick.allow shoo" given
 	# Also other users from a.b.c.d have to be explicitly allowed using auth.login.shoo-brick.allow ...
 	option auth.addr.shoo-brick.allow !a.b.c.d
 
-  * reject only "user shoo" from a.b.c.d i.e., user shoo from a.b.c.d has to be rejected. 
+  * reject only "user shoo" from a.b.c.d i.e., user shoo from a.b.c.d has to be rejected.
     * same as reject only "user shoo from a.b.c.d" above, but rules have to be added whether to allow ip addresses (and users from those ips) other than a.b.c.d
 
 ****************************************************************************************************
 
 * ip or username/password based authentication
-  
+
   * allow user foo or clients from a.b.c.d
     protocol/client:
 	option remote-subvolume foo-brick
@@ -104,7 +104,7 @@
   * reject user shoo or clients from a.b.c.d
     protocol/client:
 	option remote-subvolume shoo-brick
-   
+
     protocol/server:
 	option auth.login.shoo-brick.allow <usernames other than shoo>
 	#for each username mentioned in the above <usernames other than shoo> list, specify password as below
diff --git a/doc/coding-standard.tex b/doc/coding-standard.tex
index 92f799c01..30d412a91 100644
--- a/doc/coding-standard.tex
+++ b/doc/coding-standard.tex
@@ -27,7 +27,7 @@ purpose. The comment should be descriptive without being overly verbose.
 \textsl{Good}:
 
 \begin{verbatim}
-        DBTYPE      access_mode;    /* access mode for accessing 
+        DBTYPE      access_mode;    /* access mode for accessing
                                      * the databases, can be
                                      * DB_HASH, DB_BTREE
                                      * (option access-mode <mode>)
@@ -69,7 +69,7 @@ Never use a non-constant expression as the initialization value for a variable.
 
 \section*{$\bullet$ Validate all arguments to a function}
 All pointer arguments to a function must be checked for \texttt{NULL}.
-A macro named \texttt{VALIDATE} (in \texttt{common-utils.h}) 
+A macro named \texttt{VALIDATE} (in \texttt{common-utils.h})
 takes one argument, and if it is \texttt{NULL}, writes a log message and
 jumps to a label called \texttt{err} after setting op\_ret and op\_errno
 appropriately. It is recommended to use this template.
@@ -142,8 +142,8 @@ for success or failure.
 \begin{verbatim}
         op_ret = close (_fd);
         if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, 
-                        "close on file %s failed (%s)", real_path, 
+                gf_log (this->name, GF_LOG_ERROR,
+                        "close on file %s failed (%s)", real_path,
                         strerror (errno));
                 op_errno = errno;
                 goto out;
@@ -157,9 +157,9 @@ memory allocation fails, the call should be unwound and an error
 returned to the user.
 
 \section*{$\bullet$ Use result args and reserve the return value to indicate success or failure}
-The return value of every functions must indicate success or failure (unless 
-it is impossible for the function to fail --- e.g., boolean functions). If 
-the function needs to return additional data, it must be returned using a 
+The return value of every functions must indicate success or failure (unless
+it is impossible for the function to fail --- e.g., boolean functions). If
+the function needs to return additional data, it must be returned using a
 result (pointer) argument.
 
 \vspace{2ex}
@@ -192,11 +192,11 @@ Unless impossible, use the length-limited versions of the string functions.
 \end{verbatim}
 
 \section*{$\bullet$ No dead or commented code}
-There must be no dead code (code to which control can never be passed) or 
+There must be no dead code (code to which control can never be passed) or
 commented out code in the codebase.
 
 \section*{$\bullet$ Only one unwind and return per function}
-There must be only one exit out of a function. \texttt{UNWIND} and return 
+There must be only one exit out of a function. \texttt{UNWIND} and return
 should happen at only point in the function.
 
 \section*{$\bullet$ Function length or Keep functions small}
@@ -305,7 +305,7 @@ documentation.
 \end{verbatim}
 
 \subsection*{Indicating critical sections}
-To clearly show regions of code which execute with locks held, use 
+To clearly show regions of code which execute with locks held, use
 the following format:
 
 \begin{verbatim}
@@ -324,7 +324,7 @@ point, \texttt{out}.  At that point, the code should detect the error
 that has occured and do appropriate cleanup.
 
 \begin{verbatim}
-int32_t 
+int32_t
 sample_fop (call_frame_t *frame, xlator_t *this, ...)
 {
         char *            var1     = NULL;
@@ -337,13 +337,13 @@ sample_fop (call_frame_t *frame, xlator_t *this, ...)
         VALIDATE_OR_GOTO (this, out);
 
         /* other validations */
-        
+
         dir = opendir (...);
 
         if (dir == NULL) {
                 op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR, 
-                        "opendir failed on %s (%s)", loc->path, 
+                gf_log (this->name, GF_LOG_ERROR,
+                        "opendir failed on %s (%s)", loc->path,
                         strerror (op_errno));
                 goto out;
         }
@@ -367,11 +367,10 @@ sample_fop (call_frame_t *frame, xlator_t *this, ...)
                 if (dir) {
                         closedir (dir);
                         dir = NULL;
-                }     
-        
+                }
+
                 if (pfd) {
-                        if (pfd->path)
-                                FREE (pfd->path);
+                        FREE (pfd->path);
                         FREE (pfd);
                         pfd = NULL;
                 }
diff --git a/doc/examples/Makefile.am b/doc/examples/Makefile.am
deleted file mode 100644
index af8fe1fd8..000000000
--- a/doc/examples/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-EXTRA = README replicate.vol stripe.vol protocol-client.vol protocol-server.vol posix-locks.vol trash.vol write-behind.vol io-threads.vol io-cache.vol read-ahead.vol filter.vol trace.vol
-EXTRA_DIST = $(EXTRA)
-
-docdir = $(datadir)/doc/$(PACKAGE_NAME)
-Examplesdir = $(docdir)/examples
-Examples_DATA = $(EXTRA)
-
-CLEANFILES = 
diff --git a/doc/examples/README b/doc/examples/README
deleted file mode 100644
index 732751571..000000000
--- a/doc/examples/README
+++ /dev/null
@@ -1,13 +0,0 @@
-GlusterFS's translator feature is very flexible and there are quite a lot of
-ways one can configure their filesystem to behave like.
-
-Volume Specification is a way in which GlusterFS understands how it has to work,
-based on what is written there.
-
-Going through the following URLs may give you more idea about all these.
-
-* http://www.gluster.org/docs/index.php/GlusterFS
-* http://www.gluster.org/docs/index.php/GlusterFS_Volume_Specification
-* http://www.gluster.org/docs/index.php/GlusterFS_Translators
-
-Mail us any doubts, suggestions on 'gluster-devel(at)nongnu.org'
diff --git a/doc/examples/filter.vol b/doc/examples/filter.vol
deleted file mode 100644
index ca5c59837..000000000
--- a/doc/examples/filter.vol
+++ /dev/null
@@ -1,23 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-## In normal clustered storage type, any of the cluster translators can come here.
-#
-# Definition of other clients
-# 
-# Definition of cluster translator (may be unify, afr, or unify over afr)
-# 
-
-### 'Filter' translator is used on client side (or server side according to needs). This traslator makes all the below translators, (or say volumes) as read-only. Hence if one wants a 'read-only' filesystem, using filter as the top most volume will make it really fast as the fops are returned from this level itself.
-
-volume filter-ro
- type features/filter
- option root-squashing enable
-# option completely-read-only yes
-# translate-uid 1-99=0
- subvolumes client
-end-volume
-\ No newline at end of file
diff --git a/doc/examples/io-cache.vol b/doc/examples/io-cache.vol
deleted file mode 100644
index a71745017..000000000
--- a/doc/examples/io-cache.vol
+++ /dev/null
@@ -1,31 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-## In normal clustered storage type, any of the cluster translators can come
-# here.
-#
-# Definition of other clients
-#
-# Definition of cluster translator (may be distribute, replicate, or distribute
-# over replicate)
-#
-
-### 'IO-Cache' translator is best used on client side when a filesystem has file
-# which are not modified frequently but read several times. For example, while
-# compiling a kernel, *.h files are read while compiling every *.c file, in
-# these case, io-cache translator comes very handy, as it keeps the whole file
-# content in the cache, and serves from the cache.
-# One can provide the priority of the cache too.
-
-volume ioc
-  type performance/io-cache
-  subvolumes client         # In this example it is 'client' you may have to
-                            #  change it according to your spec file.
-  option cache-size 64MB    # 32MB is default
-  option force-revalidate-timeout 5 # 1second is default
-  option priority *.html:2,*:1 # default is *:0
-end-volume
diff --git a/doc/examples/io-threads.vol b/doc/examples/io-threads.vol
deleted file mode 100644
index 236f5b8b1..000000000
--- a/doc/examples/io-threads.vol
+++ /dev/null
@@ -1,22 +0,0 @@
-volume brick
-  type storage/posix                   # POSIX FS translator
-  option directory /home/export        # Export this directory
-end-volume
-
-###  'IO-threads' translator gives a threading behaviour to File I/O calls. All
-# other normal fops are having default behaviour. Loading this on server side
-# helps to reduce the contension of network. (Which is assumed as a GlusterFS
-# hang).
-
-volume iot
-  type performance/io-threads
-  subvolumes brick
-  option thread-count 4 # default value is 1
-end-volume
-
-volume server
-  type protocol/server
-  subvolumes iot
-  option transport-type tcp     # For TCP/IP transport
-  option auth.addr.iot.allow 192.168.*
-end-volume
diff --git a/doc/examples/posix-locks.vol b/doc/examples/posix-locks.vol
deleted file mode 100644
index 673afa3f8..000000000
--- a/doc/examples/posix-locks.vol
+++ /dev/null
@@ -1,19 +0,0 @@
-volume brick
-  type storage/posix                   # POSIX FS translator
-  option directory /home/export        # Export this directory
-end-volume
-
-# 'Posix-locks' feature should be added on the server side.
-
-volume p-locks
-  type features/posix-locks
-  subvolumes brick
-  option mandatory on
-end-volume
-
-volume server
-  type protocol/server
-  subvolumes p-locks
-  option transport-type tcp
-  option auth.addr.p-locks.allow 192.168.* # Allow access to "p-locks" volume
-end-volume
diff --git a/doc/examples/protocol-client.vol b/doc/examples/protocol-client.vol
deleted file mode 100644
index c34ef790d..000000000
--- a/doc/examples/protocol-client.vol
+++ /dev/null
@@ -1,12 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-# option transport.socket.remote-port 24016
-
-# option transport-type rdma # for Infiniband verbs transport
-# option transport.rdma.work-request-send-count 16
-# option transport.rdma.work-request-recv-count 16
-# option transport.rdma.remote-port 24016
-  option remote-subvolume brick        # name of the remote volume
-end-volume
diff --git a/doc/examples/protocol-server.vol b/doc/examples/protocol-server.vol
deleted file mode 100644
index 195e49657..000000000
--- a/doc/examples/protocol-server.vol
+++ /dev/null
@@ -1,21 +0,0 @@
-### Export volume "brick" with the contents of "/home/export" directory.
-volume brick
-  type storage/posix                   # POSIX FS translator
-  option directory /home/export        # Export this directory
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp      # For TCP/IP transport
-# option transport.socket.listen-port 24016
-
-# option transport-type rdma
-# option transport.rdma.work-request-send-count 64
-# option transport.rdma.work-request-recv-count 64
-# option transport.rdma.listen-port 24016
-
-# option bind-address 192.168.1.10     # Default is to listen on all interfaces
-  subvolumes brick
-  option auth.addr.brick.allow 192.168.* # Allow access to "brick" volume
-end-volume
diff --git a/doc/examples/read-ahead.vol b/doc/examples/read-ahead.vol
deleted file mode 100644
index 9e4dba556..000000000
--- a/doc/examples/read-ahead.vol
+++ /dev/null
@@ -1,24 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-## In normal clustered storage type, any of the cluster translators can come here.
-#
-# Definition of other clients
-#
-# Definition of cluster translator (may be distribute, replicate, or distribute
-# over replicate)
-#
-
-# 'Read-Ahead' translator is best utilized on client side, as it prefetches
-# the file contents when the first read() call is issued.
-
-volume ra
-  type performance/read-ahead
-  subvolumes client
-  option page-count 4          # default is 2
-  option force-atime-update no # defalut is 'no'
-end-volume
diff --git a/doc/examples/replicate.vol b/doc/examples/replicate.vol
deleted file mode 100644
index 10626d46f..000000000
--- a/doc/examples/replicate.vol
+++ /dev/null
@@ -1,118 +0,0 @@
-### 'NOTE'
-# This file has both server spec and client spec to get an understanding of
-# replicate spec file. Hence can't be used as it is, as a GlusterFS spec file.
-# One need to seperate out server spec and client spec to get it working.
-
-#=========================================================================
-
-# **** server1 spec file ****
-
-### Export volume "brick" with the contents of "/home/export" directory.
-volume posix1
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export1        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick1
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix1
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp                      # For TCP/IP transport
-  option transport.socket.listen-port 24016
-  subvolumes brick1
-  option auth.addr.brick1.allow * 		# access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** server2 spec file ****
-volume posix2
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export2        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick2
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix2
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp                  # For TCP/IP transport
-  option transport.socket.listen-port 24017
-  subvolumes brick2
-  option auth.addr.brick2.allow * # Allow access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** server3 spec file ****
-
-volume posix3
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export3        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick3
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix3
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp                 # For TCP/IP transport
-  option transport.socket.listen-port 24018
-  subvolumes brick3
-  option auth.addr.brick3.allow * 		# access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** Clustered Client config file ****
-
-### Add client feature and attach to remote subvolume of server1
-volume client1
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24016
-  option remote-subvolume brick1        # name of the remote volume
-end-volume
-
-### Add client feature and attach to remote subvolume of server2
-volume client2
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24017
-  option remote-subvolume brick2        # name of the remote volume
-end-volume
-
-volume client3
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24018
-  option remote-subvolume brick3        # name of the remote volume
-end-volume
-
-## Add replicate feature.
-volume replicate
-  type cluster/replicate
-  subvolumes client1 client2 client3
-end-volume
diff --git a/doc/examples/stripe.vol b/doc/examples/stripe.vol
deleted file mode 100644
index 9524e8198..000000000
--- a/doc/examples/stripe.vol
+++ /dev/null
@@ -1,120 +0,0 @@
-
-### 'NOTE'
-# This file has both server spec and client spec to get an understanding of
-# stripe's spec file. Hence can't be used as it is, as a GlusterFS spec file.
-# One need to seperate out server spec and client spec to get it working.
-
-#=========================================================================
-
-# **** server1 spec file ****
-
-### Export volume "brick" with the contents of "/home/export" directory.
-volume posix1
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export1        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick1
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix1
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp     # For TCP/IP transport
-  option transport.socket.listen-port 24016
-  subvolumes brick1
-  option auth.addr.brick1.allow * 		# access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** server2 spec file ****
-volume posix2
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export2        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick2
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix2
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp     # For TCP/IP transport
-  option transport.socket.listen-port 24017
-  subvolumes brick2
-  option auth.addr.brick2.allow * # Allow access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** server3 spec file ****
-
-volume posix3
-  type storage/posix                    # POSIX FS translator
-  option directory /home/export3        # Export this directory
-end-volume
-
-### Add POSIX record locking support to the storage brick
-volume brick3
-  type features/posix-locks
-  option mandatory on          # enables mandatory locking on all files
-  subvolumes posix3
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp     # For TCP/IP transport
-  option transport.socket.listen-port 24018
-  subvolumes brick3
-  option auth.addr.brick3.allow * 		# access to "brick" volume
-end-volume
-
-
-#=========================================================================
-
-# **** Clustered Client config file ****
-
-### Add client feature and attach to remote subvolume of server1
-volume client1
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24016
-  option remote-subvolume brick1        # name of the remote volume
-end-volume
-
-### Add client feature and attach to remote subvolume of server2
-volume client2
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24017
-  option remote-subvolume brick2        # name of the remote volume
-end-volume
-
-volume client3
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 127.0.0.1      # IP address of the remote brick
-  option transport.socket.remote-port 24018
-  option remote-subvolume brick3        # name of the remote volume
-end-volume
-
-## Add Stripe Feature.
-volume stripe
-  type cluster/stripe
-  subvolumes client1 client2 client3
-  option block-size 1MB
-end-volume
diff --git a/doc/examples/trace.vol b/doc/examples/trace.vol
deleted file mode 100644
index 59830f26a..000000000
--- a/doc/examples/trace.vol
+++ /dev/null
@@ -1,21 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-### 'Trace' translator is a very handy debug tool for GlusterFS, as it can be
-# loaded between any of the two volumes without changing the behaviour of the
-# filesystem.
-# On client side it can be the top most volume in spec (like now) to understand
-# what calls are made on FUSE filesystem, when a mounted filesystem is
-# accessed.
-
-volume trace
-  type debug/trace
-  subvolumes client
-end-volume
-
-# 'NOTE:' By loading 'debug/trace' translator, filesystem will be very slow as
-# it logs each and every calls to the log file.
diff --git a/doc/examples/trash.vol b/doc/examples/trash.vol
deleted file mode 100644
index 3fcf315af..000000000
--- a/doc/examples/trash.vol
+++ /dev/null
@@ -1,20 +0,0 @@
-volume brick
-  type storage/posix                   # POSIX FS translator
-  option directory /home/export        # Export this directory
-end-volume
-
-### 'Trash' translator is best used on server side as it just renames the
-# deleted file inside 'trash-dir', and it makes 4 seperate fops for one unlink
-# call.
-volume trashcan
-  type features/trash
-  subvolumes brick
-  option trash-dir /.trashcan
-end-volume
-
-volume server
-  type protocol/server
-  subvolumes trashcan
-  option transport-type tcp     # For TCP/IP transport
-  option auth.addr.brick.allow 192.168.* # Allow access to "brick" volume
-end-volume
diff --git a/doc/examples/write-behind.vol b/doc/examples/write-behind.vol
deleted file mode 100644
index 2b5ed4139..000000000
--- a/doc/examples/write-behind.vol
+++ /dev/null
@@ -1,27 +0,0 @@
-volume client
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-  option remote-host 192.168.1.10      # IP address of the remote brick
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-## In normal clustered storage type, any of the cluster translators can come here.
-#
-# Definition of other clients
-#
-# Definition of cluster translator (may be unify, replicate, or unify over replicate)
-#
-
-
-# 'Write-behind' translator is a performance booster for write operation. Best
-# used on client side, as its main intension is to reduce the network latency
-# caused for each write operation.
-
-volume wb
-  type performance/write-behind
-  subvolumes client
-  option flush-behind on    # default value is 'off'
-  option window-size 2MB
-  option enable-O_SYNC no  # default is no
-  option disable-for-first-nbytes 128KB #default is 1
-end-volume
diff --git a/doc/features/rdma-cm-in-3.4.0.txt b/doc/features/rdma-cm-in-3.4.0.txt
new file mode 100644
index 000000000..fd953e56b
--- /dev/null
+++ b/doc/features/rdma-cm-in-3.4.0.txt
@@ -0,0 +1,9 @@
+Following is the impact of http://review.gluster.org/#change,149.
+
+New userspace packages needed:
+librdmacm
+librdmacm-devel
+
+rdmacm needs an IPoIB address for connection establishment. This requirement results in following issues:
+* Because of bug #890502, we've to probe the peer on an IPoIB address. This imposes a restriction that all volumes created in the future have to communicate over IPoIB address (irrespective of whether they use gluster's tcp or rdma transport).
+* Currently client has an independence to choose b/w tcp and rdma transports while communicating with the server (by creating volumes with transport-type tcp,rdma). This independence was a byproduct of our ability use the normal channel used with transport-type tcp for rdma connectiion establishment handshake too. However, with new requirement of IPoIB address for connection establishment, we loose this independence (till we bring in multi-network support - where a brick can be identified by a set of ip-addresses and we can choose different pairs of ip-addresses for communication based on our requirements - in glusterd).
diff --git a/doc/features/rebalance.md b/doc/features/rebalance.md
new file mode 100644
index 000000000..29b993008
--- /dev/null
+++ b/doc/features/rebalance.md
@@ -0,0 +1,74 @@
+## Background
+
+
+For a more detailed description, view Jeff Darcy's blog post [here] 
+(http://hekafs.org/index.php/2012/03/glusterfs-algorithms-distribution/) 
+
+GlusterFS uses the distribute translator (DHT) to aggregate space of multiple servers. DHT distributes files among its subvolumes using a consistent hashing method providing 32-bit hashes. Each DHT subvolume is given a range in the 32-bit hash space. A hash value is calculated for every file using a combination of its name. The file is then placed in the subvolume with the hash range that contains the hash value. 
+
+## What is rebalance?
+
+The rebalance process migrates files between the DHT subvolumes when necessary.
+
+## When is rebalance required?
+
+Rebalancing is required for two main cases.
+
+1. Addition/Removal of bricks
+
+2. Renaming of a file 
+
+## Addition/Removal of  bricks
+
+Whenever the number or order of DHT subvolumes change, the hash range given to each subvolume is recalculated. When this happens, already existing files on the volume will need to be moved to the correct subvolume based on their hash. Rebalance does this activity.
+
+Addition of bricks which increase the size of a volume will increase the number of DHT subvolumes and lead to recalculation of hash ranges (This doesn't happen when bricks are added to a volume to increase redundancy, i.e. increase replica count of a volume). This will require an explicit rebalance command to be issued to migrate the files.
+
+Removal of bricks which decrease the size of a volumes also causes the hash ranges of DHT to be recalculated. But we don't need to issue an explicit rebalance command in this case, as rebalance is done automatically by the remove-brick process if needed.
+
+## Renaming of a file
+
+Renaming of file will cause its hash to change. The file now needs to be moved to the correct subvolume based on its new hash. Rebalance does this.
+
+## How does rebalance work?
+
+At a high level, the rebalance process consists of the following 3 steps:
+
+1. Crawl the volume to access all files
+2. Calculate the hash for the file
+3. If needed move the migrate the file to the correct subvolume.
+
+
+The rebalance process has been optimized by making it distributed across the trusted storage pool. With distributed rebalance, a rebalance process is launched on each peer in the cluster. Each rebalance process will crawl files on only those bricks of the volume which are present on it, and migrate the files which need migration to the correct brick. This speeds up the rebalance process considerably.
+
+## What will happen if rebalance is not run?
+
+### Addition of bricks
+
+With the current implementation of add-brick, when the size of a volume is augmented by adding new bricks, the new bricks are not put into use immediately i.e., the hash ranges there not recalculated immediately. This means that the files will still be placed only onto the existing bricks, leaving the newly added storage space unused. Starting a rebalance process on the volume will cause the hash ranges to be recalculated with the new bricks included, which allows the newly added storage space to be used.
+
+### Renaming a file
+
+When a file rename causes the file to be hashed to a new subvolume, DHT writes a link file on the new subvolume leaving the actual file on the original subvolume. A link file is an empty file, which has an extended attribute set that points to the subvolume on which the actual file exists. So, when a client accesses the renamed file, DHT first looks for the file in the hashed subvolume and gets the link file. DHT understands the link file, and gets the actual file from the subvolume pointed to by the link file. This leads to a slight reduction in performance. A rebalance will move the actual file to the hashed subvolume, allowing clients to access the file directly once again.
+
+## Are clients affected during a rebalance process?
+
+The rebalance process is transparent to applications on the clients. Applications which have open files on the volume will not be affected by the rebalance process, even if the open file requires migration. The DHT translator on the client will hide the migration from the applications.
+
+##How are open files migrated?
+
+(A more technical description of the algorithm used can be seen in the commit message of commit a07bb18c8adeb8597f62095c5d1361c5bad01f09.)
+
+To achieve migration of open files, two things need to be assured of,
+a) any writes or changes happening to the file during migration are correctly synced to destination subvolume after the migration is complete.
+b) any further changes should be made to the destination subvolume
+
+Both of these requirements require sending notificatoins to clients. Clients are notified by overloading an attribute used in every callback functions. DHT understands these attributes in the callbacks and can be notified if a file is being migrated or not.
+
+During rebalance, a file will be in two phases
+
+1. Migration in process - In this phase the file is being migrated by the rebalance process from the source subvolume to the destination subvolume. The rebalance process will set a 'in-migration' attribute on the file, which will notify the clients' DHT translator. The clients' DHT translator will then take care to send any further changes to the destination subvolume as well. This way we satisfy the first requirement
+
+2. Migration completed - Once the file has been migrated, the rebalance process will set a 'migration-complete' attribute on the file. The clients will be notified of the completion and all further operations on the file will happen on the destination subvolume.
+
+The DHT translator handles the above and allows the applications on the clients to continue working on a file under migration.
diff --git a/doc/gluster.8 b/doc/gluster.8
index 87c812081..3c78fb8b1 100644
--- a/doc/gluster.8
+++ b/doc/gluster.8
@@ -1,18 +1,11 @@
-.\"  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+
+.\"  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
 .\"  This file is part of GlusterFS.
 .\"
-.\"  GlusterFS is free software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU General Public License as published
-.\"  by the Free Software Foundation; either version 3 of the License,
-.\"  or (at your option) any later version.
-.\"
-.\"  GlusterFS is distributed in the hope that it will be useful, but
-.\"  WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-.\"  General Public License for more details.
-.\"
-.\"  You should have received a copy of the GNU General Public License
-.\"  along with this program.  If not, see " <http://www.gnu.org/licenses/>.
+.\"  This file is licensed to you under your choice of the GNU Lesser
+.\"  General Public License, version 3 or any later version (LGPLv3 or
+.\"  later), or the GNU General Public License, version 2 (GPLv2), in all
+.\"  cases as published by the Free Software Foundation.
 .\"
 .\"
 .TH Gluster 8 "Gluster command line utility" "07 March 2011" "Gluster Inc."
@@ -121,7 +114,7 @@ Display the command options.
 Exit the gluster command line interface.
 
 .SH FILES
-/etc/glusterd/*
+/var/lib/glusterd/*
 .SH SEE ALSO
 .nf
 \fBfusermount\fR(1), \fBmount.glusterfs\fR(8), \fBglusterfs\fR(8), \fBglusterd\fR(8)
diff --git a/doc/glusterd.8 b/doc/glusterd.8
index 267a7e00f..04a43481e 100644
--- a/doc/glusterd.8
+++ b/doc/glusterd.8
@@ -1,20 +1,11 @@
 .\"
-.\"  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+.\"  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
 .\"  This file is part of GlusterFS.
 .\"
-.\"  GlusterFS is free software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU General Public License as published
-.\"  by the Free Software Foundation; either version 3 of the License,
-.\"  or (at your option) any later version.
-.\"
-.\"  GlusterFS is distributed in the hope that it will be useful, but
-.\"  WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-.\"  General Public License for more details.
-.\"
-.\"  You should have received a copy of the GNU General Public License
-.\"  along with this program.  If not, see
-.\"  <http://www.gnu.org/licenses/>.
+.\"  This file is licensed to you under your choice of the GNU Lesser
+.\"  General Public License, version 3 or any later version (LGPLv3 or
+.\"  later), or the GNU General Public License, version 2 (GPLv2), in all
+.\"  cases as published by the Free Software Foundation.
 .\"
 .\"
 
@@ -59,7 +50,7 @@ Print the program version.
 
 .PP
 .SH FILES
-/etc/glusterd/*
+/var/lib/glusterd/*
 
 .SH SEE ALSO
 .nf
diff --git a/doc/glusterfs.8 b/doc/glusterfs.8
index b21fd675e..60ad5709b 100644
--- a/doc/glusterfs.8
+++ b/doc/glusterfs.8
@@ -1,19 +1,10 @@
-.\"  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+.\"  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
 .\"  This file is part of GlusterFS.
 .\"
-.\"  GlusterFS is free software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU General Public License as published
-.\"  by the Free Software Foundation; either version 3 of the License,
-.\"  or (at your option) any later version.
-.\"
-.\"  GlusterFS is distributed in the hope that it will be useful, but
-.\"  WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-.\"  General Public License for more details.
-.\"
-.\"  You should have received a copy of the GNU General Public License
-.\"  long with this program.  If not, see
-.\"  <http://www.gnu.org/licenses/>.
+.\"  This file is licensed to you under your choice of the GNU Lesser
+.\"  General Public License, version 3 or any later version (LGPLv3 or
+.\"  later), or the GNU General Public License, version 2 (GPLv2), in all
+.\"  cases as published by the Free Software Foundation.
 .\"
 .\"
 .\"
@@ -32,7 +23,8 @@ be made of any commodity hardware, such as x86-64 server with SATA-II RAID and
 Infiniband HBA.
 
 GlusterFS is fully POSIX compliant file system. On client side, it has dependency
-on FUSE package, on server side, it works seemlessly on different operating systems. Currently supported on GNU/Linux and Solaris.
+on FUSE package, on server side, it works seemlessly on different operating systems.
+Currently supported on GNU/Linux and Solaris.
 
 .SH OPTIONS
 
@@ -40,33 +32,55 @@ on FUSE package, on server side, it works seemlessly on different operating syst
 .PP
 .TP
 \fB\-f, \fB\-\-volfile=VOLUME-FILE\fR
-File to use as VOLUME-FILE (the default is /etc/glusterfs/glusterfs.vol).
+File to use as VOLUME-FILE.
 .TP
 \fB\-l, \fB\-\-log\-file=LOGFILE\fR
-File to use for logging.
+File to use for logging (the default is <INSTALL-DIR>/var/log/glusterfs/<MOUNT-POINT>.log).
 .TP
 \fB\-L, \fB\-\-log\-level=LOGLEVEL\fR
-Logging severity.  Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is WARNING).
+Logging severity.  Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is INFO).
 .TP
 \fB\-s, \fB\-\-volfile\-server=SERVER\fR
 Server to get the volume from.  This option overrides \fB\-\-volfile \fR option.
+.TP
+\fB\-\-volfile\-max\-fetch\-attempts=MAX\-ATTEMPTS\fR
+Maximum number of connect attempts to server. This option should be provided with
+\fB\-\-volfile\-server\fR option (the default is 1).
 
 .SS "Advanced options"
 .PP
 .TP
+\fB\-\-acl\fR
+Mount the filesystem with POSIX ACL support.
+.TP
 \fB\-\-debug\fR
 Run in debug mode.  This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG,
 and \fB\-\-log\-file\fR to console.
 .TP
+\fB\-\-enable\-ino32=BOOL\fR
+Use 32-bit inodes when mounting to workaround application that doesn't support 64-bit inodes.
+.TP
+\fB\-\-fopen\-keep\-cache\fR
+Do not purge the cache on file open.
+.TP
+\fB\-\-mac\-compat=BOOL\fR
+Provide stubs for attributes needed for seamless operation on Macs (the default is off).
+.TP
 \fB\-N, \fB\-\-no\-daemon\fR
 Run in the foreground.
 .TP
-\fB\-\-read\-only\fR
-Make the file system read-only.
-.TP
 \fB\-p, \fB\-\-pid\-file=PIDFILE\fR
 File to use as PID file.
 .TP
+\fB\-\-read\-only\fR
+Mount the file system in 'read-only' mode.
+.TP
+\fB\-\-selinux\fR
+Enable SELinux label (extended attributes) support on inodes.
+.TP
+\fB\-S, \fB\-\-socket\-file=SOCKFILE\fR
+File to use as unix-socket.
+.TP
 \fB\-\-volfile\-id=KEY\fR
 Key of the volume file to be fetched from the server.
 .TP
@@ -74,11 +88,14 @@ Key of the volume file to be fetched from the server.
 Port number of volfile server.
 .TP
 \fB\-\-volfile\-server\-transport=TRANSPORT\fR
-Transport type to get volume file from server (the default is socket).
+Transport type to get volume file from server (the default is tcp).
 .TP
 \fB\-\-volume\-name=VOLUME\-NAME\fR
 Volume name to be used for MOUNT-POINT (the default is top most volume in VOLUME-FILE).
 .TP
+\fB\-\-worm\fR
+Mount the filesystem in 'worm' mode.
+.TP
 \fB\-\-xlator\-option=VOLUME\-NAME.OPTION=VALUE\fR
 Add/Override a translator option for a volume with the specified value.
 
@@ -89,11 +106,29 @@ Add/Override a translator option for a volume with the specified value.
 \fB\-\-attribute\-timeout=SECONDS\fR
 Set attribute timeout to SECONDS for inodes in fuse kernel module (the default is 1).
 .TP
-\fB\-\-entry\-timeout=SECONDS\fR
-Set entry timeout to SECONDS in fuse kernel module (the default is 1).
+\fB\-\-background\-qlen=N\fR
+Set fuse module's background queue length to N (the default is 64).
+.TP
+\fB\-\-congestion\-threshold=N\fR
+Set fuse module's congestion threshold to N (the default is 48).
 .TP
 \fB\-\-direct\-io\-mode=BOOL\fR
 Enable/Disable the direct-I/O mode in fuse module (the default is enable).
+.TP
+\fB\-\-dump-fuse=PATH\f\R
+Dump fuse traffic to PATH
+.TP
+\fB\-\-entry\-timeout=SECONDS\fR
+Set entry timeout to SECONDS in fuse kernel module (the default is 1).
+.TP
+\fB\-\-gid\-timeout=SECONDS\fR
+Set auxilary group list timeout to SECONDS for fuse translator (the default is 0).
+.TP
+\fB\-\-negative\-timeout=SECONDS\fR
+Set negative timeout to SECONDS in fuse kernel module (the default is 0).
+.TP
+\fB\-\-volfile-check\fR
+Enable strict volume file checking.
 
 .SS "Miscellaneous Options"
 .PP
@@ -110,12 +145,12 @@ Print the program version.
 
 .PP
 .SH FILES
-/etc/glusterfs/*.vol, /etc/glusterd/vols/*/*.vol
+/var/lib/glusterd/vols/*/*.vol
 .SH EXAMPLES
 mount a volume named foo on server bar with log level DEBUG on mount point
 /mnt/foo
 
-# glusterfs --log-level=DEBUG --volfile-id=foo --volfile-server=bar /mnt/foo
+# glusterfs \-\-log\-level=DEBUG \-\-volfile\-id=foo \-\-volfile\-server=bar /mnt/foo
 
 .SH SEE ALSO
 .nf
@@ -124,6 +159,6 @@ mount a volume named foo on server bar with log level DEBUG on mount point
 .fi
 .SH COPYRIGHT
 .nf
-Copyright(c) 2006-2011  Gluster, Inc.  <http://www.gluster.com>
+Copyright(c) 2006-2011  Red Hat, Inc.  <http://www.redhat.com>
 \fR
 .fi
diff --git a/doc/glusterfs.vol.sample b/doc/glusterfs.vol.sample
deleted file mode 100644
index 977363b92..000000000
--- a/doc/glusterfs.vol.sample
+++ /dev/null
@@ -1,53 +0,0 @@
-### file: client-volume.vol.sample
-
-#####################################
-###  GlusterFS Client Volume File  ##
-#####################################
-
-#### CONFIG FILE RULES:
-### "#" is comment character.
-### - Config file is case sensitive
-### - Options within a volume block can be in any order.
-### - Spaces or tabs are used as delimitter within a line.
-### - Each option should end within a line.
-### - Missing or commented fields will assume default values.
-### - Blank/commented lines are allowed.
-### - Sub-volumes should already be defined above before referring.
-
-### Add client feature and attach to remote subvolume
-volume client
-  type protocol/client
-  option transport-type tcp
-# option transport-type unix
-# option transport-type ib-sdp
-  option remote-host 127.0.0.1         # IP address of the remote brick
-# option transport.socket.remote-port 24016
-
-# option transport-type rdma
-# option transport.rdma.remote-port 24016
-# option transport.rdma.work-request-send-count 16
-# option transport.rdma.work-request-recv-count 16
-
-  option remote-subvolume brick        # name of the remote volume
-end-volume
-
-### Add readahead feature
-#volume readahead
-#  type performance/read-ahead
-#  option page-count 2       # cache per file  = (page-count x page-size)
-#  subvolumes client
-#end-volume
-
-### Add IO-Cache feature
-#volume iocache
-#  type performance/io-cache
-#  subvolumes readahead
-#end-volume
-
-### Add writeback feature
-#volume writeback
-#  type performance/write-behind
-#  option window-size 2MB
-#  option flush-behind off
-#  subvolumes iocache
-#end-volume
diff --git a/doc/glusterfsd.8 b/doc/glusterfsd.8
index f4261e3ad..176d04236 100644
--- a/doc/glusterfsd.8
+++ b/doc/glusterfsd.8
@@ -1,19 +1,10 @@
-.\"  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+.\"  Copyright (c) 20088888888-2012 Red Hat, Inc. <http://www.redhat.com>
 .\"  This file is part of GlusterFS.
 .\"
-.\"  GlusterFS is free software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU General Public License as published
-.\"  by the Free Software Foundation; either version 3 of the License,
-.\"  or (at your option) any later version.
-.\"
-.\"  GlusterFS is distributed in the hope that it will be useful, but
-.\"  WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-.\"  General Public License for more details.
-.\"
-.\"  You should have received a copy of the GNU General Public License
-.\"  long with this program.  If not, see
-.\"  <http://www.gnu.org/licenses/>.
+.\"  This file is licensed to you under your choice of the GNU Lesser
+.\"  General Public License, version 3 or any later version (LGPLv3 or
+.\"  later), or the GNU General Public License, version 2 (GPLv2), in all
+.\"  cases as published by the Free Software Foundation.
 .\"
 .\"
 .\"
@@ -89,7 +80,7 @@ KEY of the volume file to be fetched from server
 Port number of volfile server
 .TP
 \fB\-\-volfile\-server\-transport=TRANSPORT\fR
-Transport type to get volume file from server [default: socket]
+Transport type to get volume file from server [default: tcp]
 .TP
 \fB\-\-volume\-name=VOLUME\-NAME\fR
 Volume name to be used for MOUNT-POINT [default: top most volume in
@@ -131,7 +122,7 @@ Print program version
 .SH EXAMPLES
 Start a GlusterFS server on localhost with volume name foo
 
-glusterfsd -s localhost --volfile-id foo.server.media-disk-1 -p /etc/glusterd/vols/foo/run/server-media-disk-1.pid -S /tmp/<uniqueid>.socket --brick-name /media/disk-1 -l /var/log/glusterfs/bricks/media-disk-1.log --brick-port 24009 --xlator-option foo-server.listen-port=24009
+glusterfsd \-s localhost \-\-volfile\-id foo.server.media-disk\-1 \-p /var/lib/glusterd/vols/foo/run/server\-media\-disk\-1.pid \-S /tmp/<uniqueid>.socket \-\-brick-name /media/disk\-1 \-l /var/log/glusterfs/bricks/media\-disk\-1.log \-\-brick\-port 24009 \-\-xlator\-option foo\-server.listen-port=24009
 
 .SH SEE ALSO
 .nf
diff --git a/doc/glusterfsd.vol.sample b/doc/glusterfsd.vol.sample
deleted file mode 100644
index ec2fd341e..000000000
--- a/doc/glusterfsd.vol.sample
+++ /dev/null
@@ -1,44 +0,0 @@
-### file: server-volume.vol.sample
-
-#####################################
-###  GlusterFS Server Volume File  ##
-#####################################
-
-#### CONFIG FILE RULES:
-### "#" is comment character.
-### - Config file is case sensitive
-### - Options within a volume block can be in any order.
-### - Spaces or tabs are used as delimitter within a line.
-### - Multiple values to options will be : delimited.
-### - Each option should end within a line.
-### - Missing or commented fields will assume default values.
-### - Blank/commented lines are allowed.
-### - Sub-volumes should already be defined above before referring.
-
-### Export volume "brick" with the contents of "/home/export" directory.
-volume brick
-  type storage/posix                   # POSIX FS translator
-  option directory /home/export        # Export this directory
-end-volume
-
-### Add network serving capability to above brick.
-volume server
-  type protocol/server
-  option transport-type tcp
-# option transport-type unix
-# option transport-type ib-sdp
-# option transport.socket.bind-address 192.168.1.10     # Default is to listen
-                                                        # on all interfaces
-# option transport.socket.listen-port 24016
-
-# option transport-type rdma
-# option transport.rdma.listen-port 24016
-# option transport.rdma.work-request-send-count 64
-# option transport.rdma.work-request-recv-count 64
-
-  subvolumes brick
-# NOTE: Access to any volume through protocol/server is denied by
-# default. You need to explicitly grant access through # "auth"
-# option.
-  option auth.addr.brick.allow * # Allow access to "brick" volume
-end-volume
diff --git a/doc/hacker-guide/adding-fops.txt b/doc/hacker-guide/adding-fops.txt
deleted file mode 100644
index e70dbbdc8..000000000
--- a/doc/hacker-guide/adding-fops.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-		  HOW TO ADD A NEW FOP TO GlusterFS
-		  =================================
-
-Steps to be followed when adding a new FOP to GlusterFS:
-
-1. Edit glusterfs.h and add a GF_FOP_* constant.
-
-2. Edit xlator.[ch] and:
-   2a. add the new prototype for fop and callback.
-   2b. edit xlator_fops structure.
-
-3. Edit xlator.c and add to fill_defaults.
-
-4. Edit protocol.h and add struct necessary for the new FOP.
-
-5. Edit defaults.[ch] and provide default implementation.
-
-6. Edit call-stub.[ch] and provide stub implementation.
-
-7. Edit common-utils.c and add to gf_global_variable_init().
-
-8. Edit client-protocol and add your FOP.
-
-9. Edit server-protocol and add your FOP.
-
-10. Implement your FOP in any translator for which the default implementation
-    is not sufficient.
-
-==========================================
-Last updated: Mon Oct 27 21:35:49 IST 2008
-
-Author: Vikas Gorur <vikas@gluster.com>
-==========================================
diff --git a/doc/hacker-guide/bdb.txt b/doc/hacker-guide/bdb.txt
deleted file mode 100644
index fd0bd3652..000000000
--- a/doc/hacker-guide/bdb.txt
+++ /dev/null
@@ -1,70 +0,0 @@
-
-* How does file translates to key/value pair?
----------------------------------------------
-
-  in bdb a file is identified by key (obtained by taking basename() of the path of
-the file) and file contents are stored as value corresponding to the key in database
-file (defaults to glusterfs_storage.db under dirname() directory).
-
-* symlinks, directories
------------------------
-
-  symlinks and directories are stored as is.
-
-* db (database) files
----------------------
-
-  every directory, including root directory, contains a database file called
-glusterfs_storage.db. all the regular files contained in the directory are stored
-as key/value pair inside the glusterfs_storage.db.
-
-* internal data cache
----------------------
-
-  db does not provide a way to find out the size of the value corresponding to a key. 
-so, bdb makes DB->get() call for key and takes the length of the value returned. 
-since DB->get() also returns file contents for key, bdb maintains an internal cache and
-stores the file contents in the cache.
-  every directory maintains a seperate cache.
-  
-* inode number transformation
------------------------------
-
-  bdb allocates a inode number to each file and directory on its own. bdb maintains a 
-global counter and increments it after allocating inode number for each file
-(regular, symlink or directory). NOTE: bdb does not guarantee persistent inode numbers.
-
-* checkpoint thread
--------------------
-
-  bdb creates a checkpoint thread at the time of init(). checkpoint thread does a 
-periodic checkpoint on the DB_ENV. checkpoint is the mechanism, provided by db, to 
-forcefully commit the logged transactions to the storage.
-
-NOTES ABOUT FOPS:
------------------
-
-lookup() -
- 1> do lstat() on the path, if lstat fails, we assume that the file being looked up
-    is either a regular file or doesn't exist.
- 2> lookup in the DB of parent directory for key corresponding to path. if key exists,
-    return key, with. 
-    NOTE: 'struct stat' stat()ed from DB file is used as a container for 'struct stat'
-           of the regular file. st_ino, st_size, st_blocks are updated with file's values.
-
-readv() -
- 1> do a lookup in bctx cache. if successful, return the requested data from cache.
- 2> if cache missed, do a DB->get() the entire file content and insert to cache.
-
-writev():
- 1> flush any cached content of this file.
- 2> do a DB->put(), with DB_DBT_PARTIAL flag. 
-    NOTE: DB_DBT_PARTIAL is used to do partial update of a value in DB.
-
-readdir():
- 1> regular readdir() in a loop, and vomit all DB_ENV log files and DB files that
-    we encounter.
- 2> if the readdir() buffer still has space, open a DB cursor and do a sequential
-    DBC->get() to fill the reaadir buffer.
-
-
diff --git a/doc/hacker-guide/en-US/markdown/adding-fops.md b/doc/hacker-guide/en-US/markdown/adding-fops.md
new file mode 100644
index 000000000..3f72ed3e2
--- /dev/null
+++ b/doc/hacker-guide/en-US/markdown/adding-fops.md
@@ -0,0 +1,18 @@
+Adding a new FOP
+================
+
+Steps to be followed when adding a new FOP to GlusterFS:
+
+1. Edit `glusterfs.h` and add a `GF_FOP_*` constant.
+2. Edit `xlator.[ch]` and:
+    * add the new prototype for fop and callback.
+    * edit `xlator_fops` structure.
+3. Edit `xlator.c` and add to fill_defaults.
+4. Edit `protocol.h` and add struct necessary for the new FOP.
+5. Edit `defaults.[ch]` and provide default implementation.
+6. Edit `call-stub.[ch]` and provide stub implementation.
+7. Edit `common-utils.c` and add to gf_global_variable_init().
+8. Edit client-protocol and add your FOP.
+9. Edit server-protocol and add your FOP.
+10. Implement your FOP in any translator for which the default implementation
+    is not sufficient.
diff --git a/doc/hacker-guide/en-US/markdown/afr.md b/doc/hacker-guide/en-US/markdown/afr.md
new file mode 100644
index 000000000..1be7e39f2
--- /dev/null
+++ b/doc/hacker-guide/en-US/markdown/afr.md
@@ -0,0 +1,191 @@
+cluster/afr translator
+======================
+
+Locking
+-------
+
+Before understanding replicate, one must understand two internal FOPs:
+
+### `GF_FILE_LK`
+
+This is exactly like `fcntl(2)` locking, except the locks are in a
+separate domain from locks held by applications.
+
+### `GF_DIR_LK (loc_t *loc, char *basename)`
+
+This allows one to lock a name under a directory. For example,
+to lock /mnt/glusterfs/foo, one would use the call:
+
+```
+GF_DIR_LK ({loc_t for "/mnt/glusterfs"}, "foo")
+```
+
+If one wishes to lock *all* the names under a particular directory,
+supply the basename argument as `NULL`.
+
+The locks can either be read locks or write locks; consult the
+function prototype for more details.
+
+Both these operations are implemented by the features/locks (earlier
+known as posix-locks) translator.
+
+Basic design
+------------
+
+All FOPs can be classified into four major groups:
+
+### inode-read
+
+Operations that read an inode's data (file contents) or metadata (perms, etc.).
+
+access, getxattr, fstat, readlink, readv, stat.
+
+### inode-write
+
+Operations that modify an inode's data or metadata.
+
+chmod, chown, truncate, writev, utimens.
+
+### dir-read
+
+Operations that read a directory's contents or metadata.
+
+readdir, getdents, checksum.
+
+### dir-write
+
+Operations that modify a directory's contents or metadata.
+
+create, link, mkdir, mknod, rename, rmdir, symlink, unlink.
+
+Some of these make a subgroup in that they modify *two* different entries:
+link, rename, symlink.
+
+### Others
+
+Other operations.
+
+flush, lookup, open, opendir, statfs.
+
+Algorithms
+----------
+
+Each of the four major groups has its own algorithm:
+
+### inode-read, dir-read
+
+1. Send a request to the first child that is up:
+    * if it fails:
+        * try the next available child
+    * if we have exhausted all children:
+        * return failure
+
+### inode-write
+
+ All operations are done in parallel unless specified otherwise.
+
+1. Send a ``GF_FILE_LK`` request on all children for a write lock on the 
+   appropriate region
+   (for metadata operations: entire file (0, 0) for writev:
+   (offset, offset+size of buffer))
+    * If a lock request fails on a child:
+        * unlock all children
+        * try to acquire a blocking lock (`F_SETLKW`) on each child, serially.
+	  If this fails (due to `ENOTCONN` or `EINVAL`):
+          Consider this child as dead for rest of transaction.
+2. Mark all children as "pending" on all (alive) children (see below for 
+meaning of "pending").
+    * If it fails on any child:
+        * mark it as dead (in transaction local state).
+3. Perform operation on all (alive) children.
+    * If it fails on any child:
+        * mark it as dead (in transaction local state).
+4. Unmark all successful children as not "pending" on all nodes.
+5. Unlock region on all (alive) children.
+
+### dir-write
+
+ The algorithm for dir-write is same as above except instead of holding
+ `GF_FILE_LK` locks we hold a GF_DIR_LK lock on the name being operated upon.
+ In case of link-type calls, we hold locks on both the operand names.
+
+"pending"
+---------
+
+The "pending" number is like a journal entry. A pending entry is an
+array of 32-bit integers stored in network byte-order as the extended
+attribute of an inode (which can be a directory as well).
+
+There are three keys corresponding to three types of pending operations:
+
+### `AFR_METADATA_PENDING`
+
+There are some metadata operations pending on this inode (perms, ctime/mtime,
+xattr, etc.).
+
+### `AFR_DATA_PENDING`
+
+There is some data pending on this inode (writev).
+
+### `AFR_ENTRY_PENDING`
+
+There are some directory operations pending on this directory
+(create, unlink, etc.).
+
+Self heal
+---------
+
+* On lookup, gather extended attribute data:
+    * If entry is a regular file:
+        * If an entry is present on one child and not on others:
+            * create entry on others.
+        * If entries exist but have different metadata (perms, etc.):
+            * consider the entry with the highest `AFR_METADATA_PENDING` number as
+              definitive and replicate its attributes on children.
+    * If entry is a directory:
+        * Consider the entry with the higest `AFR_ENTRY_PENDING` number as
+          definitive and replicate its contents on all children.
+    * If any two entries have non-matching types (i.e., one is file and
+      other is directory):
+        * Announce to the user via log that a split-brain situation has been
+          detected, and do nothing.
+* On open, gather extended attribute data:
+    * Consider the file with the highest `AFR_DATA_PENDING` number as
+      the definitive one and replicate its contents on all other
+      children.
+
+During all self heal operations, appropriate locks must be held on all
+regions/entries being affected.
+
+Inode scaling
+-------------
+
+Inode scaling is necessary because if a situation arises where an inode number 
+is returned for a directory (by lookup) which was previously the inode number 
+of a file (as per FUSE's table), then FUSE gets horribly confused (consult a 
+FUSE expert for more details).
+
+To avoid such a situation, we distribute the 64-bit inode space equally
+among all children of replicate.
+
+To illustrate:
+
+If c1, c2, c3 are children of replicate, they each get 1/3 of the available
+inode space:
+
+-------------  --  --  --  --  --  --  --  --  --  --  --  ---
+Child:         c1  c2  c3  c1  c2  c3  c1  c2  c3  c1  c2  ...
+Inode number:  1    2   3   4   5   6   7   8   9  10  11  ...
+-------------  --  --  --  --  --  --  --  --  --  --  --  ---
+
+Thus, if lookup on c1 returns an inode number "2", it is scaled to "4"
+(which is the second inode number in c1's space).
+
+This way we ensure that there is never a collision of inode numbers from
+two different children.
+
+This reduction of inode space doesn't really reduce the usability of
+replicate since even if we assume replicate has 1024 children (which would be a
+highly unusual scenario), each child still has a 54-bit inode space:
+$2^{54} \sim 1.8 \times 10^{16}$, which is much larger than any real
+world requirement.
diff --git a/doc/hacker-guide/en-US/markdown/coding-standard.md b/doc/hacker-guide/en-US/markdown/coding-standard.md
new file mode 100644
index 000000000..178dc142a
--- /dev/null
+++ b/doc/hacker-guide/en-US/markdown/coding-standard.md
@@ -0,0 +1,402 @@
+GlusterFS Coding Standards
+==========================
+
+Structure definitions should have a comment per member
+------------------------------------------------------
+
+Every member in a structure definition must have a comment about its
+purpose. The comment should be descriptive without being overly verbose.
+
+*Bad:*
+
+```
+gf_lock_t   lock;           /* lock */
+```
+
+*Good:*
+
+```
+DBTYPE      access_mode;    /* access mode for accessing
+                             * the databases, can be
+                             * DB_HASH, DB_BTREE
+                             * (option access-mode <mode>)
+                             */
+```
+
+Declare all variables at the beginning of the function
+------------------------------------------------------
+
+All local variables in a function must be declared immediately after the
+opening brace. This makes it easy to keep track of memory that needs to be freed
+during exit. It also helps debugging, since gdb cannot handle variables
+declared inside loops or other such blocks.
+
+Always initialize local variables
+---------------------------------
+
+Every local variable should be initialized to a sensible default value
+at the point of its declaration. All pointers should be initialized to NULL,
+and all integers should be zero or (if it makes sense) an error value.
+
+
+*Good:*
+
+```
+int ret       = 0;
+char *databuf = NULL;
+int _fd       = -1;
+```
+
+Initialization should always be done with a constant value
+----------------------------------------------------------
+
+Never use a non-constant expression as the initialization value for a variable.
+
+
+*Bad:*
+
+```
+pid_t pid     = frame->root->pid;
+char *databuf = malloc (1024);
+```
+
+Validate all arguments to a function
+------------------------------------
+
+All pointer arguments to a function must be checked for `NULL`.
+A macro named `VALIDATE` (in `common-utils.h`)
+takes one argument, and if it is `NULL`, writes a log message and
+jumps to a label called `err` after setting op_ret and op_errno
+appropriately. It is recommended to use this template.
+
+
+*Good:*
+
+```
+VALIDATE(frame);
+VALIDATE(this);
+VALIDATE(inode);
+```
+
+Never rely on precedence of operators
+-------------------------------------
+
+Never write code that relies on the precedence of operators to execute
+correctly.  Such code can be hard to read and someone else might not
+know the precedence of operators as accurately as you do.
+
+*Bad:*
+
+```
+if (op_ret == -1 && errno != ENOENT)
+```
+
+*Good:*
+
+```
+if ((op_ret == -1) && (errno != ENOENT))
+```
+
+Use exactly matching types
+--------------------------
+
+Use a variable of the exact type declared in the manual to hold the
+return value of a function. Do not use an ``equivalent'' type.
+
+
+*Bad:*
+
+```
+int len = strlen (path);
+```
+
+*Good:*
+
+```
+size_t len = strlen (path);
+```
+
+Never write code such as `foo->bar->baz`; check every pointer
+-------------------------------------------------------------
+
+Do not write code that blindly follows a chain of pointer
+references. Any pointer in the chain may be `NULL` and thus
+cause a crash. Verify that each pointer is non-null before following
+it.
+
+Check return value of all functions and system calls
+----------------------------------------------------
+
+The return value of all system calls and API functions must be checked
+for success or failure.
+
+*Bad:*
+
+```
+close (fd);
+```
+
+*Good:*
+
+```
+op_ret = close (_fd);
+if (op_ret == -1) {
+        gf_log (this->name, GF_LOG_ERROR,
+                "close on file %s failed (%s)", real_path,
+                strerror (errno));
+        op_errno = errno;
+        goto out;
+}
+```
+
+
+Gracefully handle failure of malloc
+-----------------------------------
+
+GlusterFS should never crash or exit due to lack of memory. If a
+memory allocation fails, the call should be unwound and an error
+returned to the user.
+
+*Use result args and reserve the return value to indicate success or failure:*
+
+The return value of every functions must indicate success or failure (unless
+it is impossible for the function to fail --- e.g., boolean functions). If
+the function needs to return additional data, it must be returned using a
+result (pointer) argument.
+
+*Bad:*
+
+```
+int32_t dict_get_int32 (dict_t *this, char *key);
+```
+
+*Good:*
+
+```
+int dict_get_int32 (dict_t *this, char *key, int32_t *val);
+```
+
+Always use the `n' versions of string functions
+-----------------------------------------------
+
+Unless impossible, use the length-limited versions of the string functions.
+
+*Bad:*
+
+```
+strcpy (entry_path, real_path);
+```
+
+*Good:*
+
+```
+strncpy (entry_path, real_path, entry_path_len);
+```
+
+No dead or commented code
+-------------------------
+
+There must be no dead code (code to which control can never be passed) or
+commented out code in the codebase.
+
+Only one unwind and return per function
+---------------------------------------
+
+There must be only one exit out of a function. `UNWIND` and return
+should happen at only point in the function.
+
+Function length or Keep functions small
+---------------------------------------
+
+We live in the UNIX-world where modules do one thing and do it well.
+This rule should apply to our functions also. If a function is very long, try splitting it
+into many little helper functions. The question is, in a coding
+spree, how do we know a function is long and unreadable. One rule of
+thumb given by Linus Torvalds is that, a function should be broken-up
+if you have 4 or more levels of indentation going on for more than 3-4
+lines.
+
+*Example for a helper function:*
+```
+static int
+same_owner (posix_lock_t *l1, posix_lock_t *l2)
+{
+        return ((l1->client_pid == l2->client_pid) &&
+               (l1->transport  == l2->transport));
+}
+```
+
+Defining functions as static
+----------------------------
+
+Define internal functions as static only if you're
+very sure that there will not be a crash(..of any kind..) emanating in
+that function. If there is even a remote possibility, perhaps due to
+pointer derefering, etc, declare the function as non-static. This
+ensures that when a crash does happen, the function name shows up the
+in the back-trace generated by libc. However, doing so has potential
+for polluting the function namespace, so to avoid conflicts with other
+components in other parts, ensure that the function names are
+prepended with a prefix that identify the component to which it
+belongs. For eg. non-static functions in io-threads translator start
+with iot_.
+
+Ensure function calls wrap around after 80-columns
+--------------------------------------------------
+
+Place remaining arguments on the next line if needed.
+
+Functions arguments and function definition
+-------------------------------------------
+
+Place all the arguments of a function definition on the same line
+until the line goes beyond 80-cols. Arguments that extend beyind
+80-cols should be placed on the next line.
+
+Style issues
+------------
+
+### Brace placement
+
+Use K&R/Linux style of brace placement for blocks.
+
+*Good:*
+
+```
+int some_function (...)
+{
+        if (...) {
+                /* ... */
+        } else if (...) {
+                /* ... */
+        } else {
+                /* ... */
+        }
+
+        do {
+                /* ... */
+        } while (cond);
+}
+```
+
+### Indentation
+
+Use *eight* spaces for indenting blocks. Ensure that your
+file contains only spaces and not tab characters. You can do this
+in Emacs by selecting the entire file (`C-x h`) and
+running `M-x untabify`.
+
+To make Emacs indent lines automatically by eight spaces, add this
+line to your `.emacs`:
+
+```
+(add-hook 'c-mode-hook (lambda () (c-set-style "linux")))
+```
+
+### Comments
+
+Write a comment before every function describing its purpose (one-line),
+its arguments, and its return value. Mention whether it is an internal
+function or an exported function.
+
+Write a comment before every structure describing its purpose, and
+write comments about each of its members.
+
+Follow the style shown below for comments, since such comments
+can then be automatically extracted by doxygen to generate
+documentation.
+
+*Good:*
+
+```
+/**
+* hash_name -hash function for filenames
+* @par:  parent inode number
+* @name: basename of inode
+* @mod:  number of buckets in the hashtable
+*
+* @return: success: bucket number
+*          failure: -1
+*
+* Not for external use.
+*/
+```
+
+### Indicating critical sections
+
+To clearly show regions of code which execute with locks held, use
+the following format:
+
+```
+pthread_mutex_lock (&mutex);
+{
+        /* code */
+}
+pthread_mutex_unlock (&mutex);
+```
+
+*A skeleton fop function:*
+
+This is the recommended template for any fop. In the beginning come
+the initializations. After that, the `success' control flow should be
+linear.  Any error conditions should cause a `goto` to a single
+point, `out`.  At that point, the code should detect the error
+that has occured and do appropriate cleanup.
+
+```
+int32_t
+sample_fop (call_frame_t *frame, xlator_t *this, ...)
+{
+        char *            var1     = NULL;
+        int32_t           op_ret   = -1;
+        int32_t           op_errno = 0;
+        DIR *             dir      = NULL;
+        struct posix_fd * pfd      = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+
+        /* other validations */
+
+        dir = opendir (...);
+
+        if (dir == NULL) {
+                op_errno = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "opendir failed on %s (%s)", loc->path,
+                        strerror (op_errno));
+                goto out;
+        }
+
+        /* another system call */
+        if (...) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "out of memory :(");
+                goto out;
+        }
+
+        /* ... */
+
+ out:
+        if (op_ret == -1) {
+
+          /* check for all the cleanup that needs to be
+             done */
+
+                if (dir) {
+                        closedir (dir);
+                        dir = NULL;
+                }
+
+                if (pfd) {
+                        FREE (pfd->path);
+                        FREE (pfd);
+                        pfd = NULL;
+                }
+        }
+
+        STACK_UNWIND (frame, op_ret, op_errno, fd);
+        return 0;
+}
+```
diff --git a/doc/hacker-guide/en-US/markdown/posix.md b/doc/hacker-guide/en-US/markdown/posix.md
new file mode 100644
index 000000000..84c813e55
--- /dev/null
+++ b/doc/hacker-guide/en-US/markdown/posix.md
@@ -0,0 +1,59 @@
+storage/posix translator
+========================
+
+Notes
+-----
+
+### `SET_FS_ID`
+
+This is so that all filesystem checks are done with the user's
+uid/gid and not GlusterFS's uid/gid.
+
+### `MAKE_REAL_PATH`
+
+This macro concatenates the base directory of the posix volume
+('option directory') with the given path.
+
+### `need_xattr` in lookup
+
+If this flag is passed, lookup returns a xattr dictionary that contains
+the file's create time, the file's contents, and the version number
+of the file.
+
+This is a hack to increase small file performance. If an application
+wants to read a small file, it can finish its job with just a lookup
+call instead of a lookup followed by read.
+
+### `getdents`/`setdents`
+
+These are used by unify to set and get directory entries.
+
+### `ALIGN_BUF`
+
+Macro to align an address to a page boundary (4K).
+
+### `priv->export_statfs`
+
+In some cases, two exported volumes may reside on the same
+partition on the server. Sending statvfs info for both
+the volumes will lead to erroneous df output at the client,
+since free space on the partition will be counted twice.
+
+In such cases, user can disable exporting statvfs info
+on one of the volumes by setting this option.
+
+### `xattrop`
+
+This fop is used by replicate to set version numbers on files.
+
+### `getxattr`/`setxattr` hack to read/write files
+
+A key, `GLUSTERFS_FILE_CONTENT_STRING`, is handled in a special way by
+`getxattr`/`setxattr`. A getxattr with the key will return the entire
+content of the file as the value. A `setxattr` with the key will write
+the value as the entire content of the file.
+
+### `posix_checksum`
+
+This calculates a simple XOR checksum on all entry names in a
+directory that is used by unify to compare directory contents.
diff --git a/doc/hacker-guide/en-US/markdown/translator-development.md b/doc/hacker-guide/en-US/markdown/translator-development.md
new file mode 100644
index 000000000..77d1b606a
--- /dev/null
+++ b/doc/hacker-guide/en-US/markdown/translator-development.md
@@ -0,0 +1,666 @@
+Translator development
+======================
+
+Setting the Stage
+-----------------
+
+This is the first post in a series that will explain some of the details of
+writing a GlusterFS translator, using some actual code to illustrate.
+
+Before we begin, a word about environments. GlusterFS is over 300K lines of
+code spread across a few hundred files. That's no Linux kernel or anything, but
+ you're still going to be navigating through a lot of code in every
+code-editing session, so some kind of cross-referencing is *essential*. I use
+cscope with the vim bindings, and if I couldn't do Crtl+G and such to jump
+between definitions all the time my productivity would be cut in half. You may
+prefer different tools, but as I go through these examples you'll need
+something functionally similar to follow on. OK, on with the show.
+
+The first thing you need to know is that translators are not just bags of
+functions and variables. They need to have a very definite internal structure
+so that the translator-loading code can figure out where all the pieces are.
+The way it does this is to use dlsym to look for specific names within your
+shared-object file, as follow (from `xlator.c`):
+
+```
+if (!(xl->fops = dlsym (handle, "fops"))) {
+    gf_log ("xlator", GF_LOG_WARNING, "dlsym(fops) on %s",
+        dlerror ());
+    goto out;
+}
+
+if (!(xl->cbks = dlsym (handle, "cbks"))) {
+    gf_log ("xlator", GF_LOG_WARNING, "dlsym(cbks) on %s",
+        dlerror ());
+    goto out;
+}
+
+if (!(xl->init = dlsym (handle, "init"))) {
+    gf_log ("xlator", GF_LOG_WARNING, "dlsym(init) on %s",
+        dlerror ());
+    goto out;
+}
+
+if (!(xl->fini = dlsym (handle, "fini"))) {
+    gf_log ("xlator", GF_LOG_WARNING, "dlsym(fini) on %s",
+        dlerror ());
+    goto out;
+}
+```
+
+In this example, `xl` is a pointer to the in-memory object for the translator
+we're loading. As you can see, it's looking up various symbols *by name* in the
+ shared object it just loaded, and storing pointers to those symbols. Some of
+them (e.g. init are functions, while others e.g. fops are dispatch tables
+containing pointers to many functions. Together, these make up the translator's
+ public interface.
+
+Most of this glue or boilerplate can easily be found at the bottom of one of
+the source files that make up each translator. We're going to use the `rot-13`
+translator just for fun, so in this case you'd look in `rot-13.c` to see this:
+
+```
+struct xlator_fops fops = {
+    .readv          = rot13_readv,
+    .writev         = rot13_writev
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+{ .key    = {"encrypt-write"},
+  .type = GF_OPTION_TYPE_BOOL
+},
+{ .key    = {"decrypt-read"},
+  .type = GF_OPTION_TYPE_BOOL
+},
+{ .key    = {NULL} },
+};
+```
+
+The `fops` table, defined in `xlator.h`, is one of the most important pieces.
+This table contains a pointer to each of the filesystem functions that your
+translator might implement -- `open`, `read`, `stat`, `chmod`, and so on. There
+are 82 such functions in all, but don't worry; any that you don't specify here
+will be see as null and filled with defaults from `defaults.c` when your
+translator is loaded. In this particular example, since `rot-13` is an
+exceptionally simple translator, we only fill in two entries for `readv` and
+`writev`.
+
+There are actually two other tables, also required to have predefined names,
+that are also used to find translator functions: `cbks` (which is empty in this
+ snippet) and `dumpops` (which is missing entirely). The first of these specify
+ entry points for when inodes are forgotten or file descriptors are released.
+In other words, they're destructors for objects in which your translator might
+ have an interest. Mostly you can ignore them, because the default behavior
+handles even the simpler cases of translator-specific inode/fd context
+automatically. However, if the context you attach is a complex structure
+requiring complex cleanup, you'll need to supply these functions. As for
+dumpops, that's just used if you want to provide functions to pretty-print
+various structures in logs. I've never used it myself, though I probably
+should. What's noteworthy here is that we don't even define dumpops. That's
+because all of the functions that might use these dispatch functions will check
+ for `xl->dumpops` being `NULL` before calling through it. This is in sharp
+contrast to the behavior for `fops` and `cbks1`, which *must* be present. If
+they're not, translator loading will fail because these pointers are not
+checked every time and if they're `NULL` then we'll segfault. That's why we
+provide an empty definition for cbks; it's OK for the individual function
+pointers to be NULL, but not for the whole table to be absent.
+
+The last piece I'll cover today is options. As you can see, this is a table of
+translator-specific option names and some information about their types.
+GlusterFS actually provides a pretty rich set of types (`volume_option_type_t`
+in `options.`h) which includes paths, translator names, percentages, and times
+in addition to the obvious integers and strings. Also, the `volume_option_t`
+structure can include information about alternate names, min/max/default
+values, enumerated string values, and descriptions. We don't see any of these
+here, so let's take a quick look at some more complex examples from afr.c and
+then come back to `rot-13`.
+
+```
+{ .key  = {"data-self-heal-algorithm"},
+  .type = GF_OPTION_TYPE_STR,
+  .default_value = "",
+  .description   = "Select between \"full\", \"diff\". The "
+           "\"full\" algorithm copies the entire file from "
+           "source to sink. The \"diff\" algorithm copies to "
+           "sink only those blocks whose checksums don't match "
+           "with those of source.",
+  .value = { "diff", "full", "" }
+},
+{ .key  = {"data-self-heal-window-size"},
+  .type = GF_OPTION_TYPE_INT,
+  .min  = 1,
+  .max  = 1024,
+  .default_value = "1",
+  .description = "Maximum number blocks per file for which "
+         "self-heal process would be applied simultaneously."
+},
+```
+
+When your translator is loaded, all of this information is used to parse the
+options actually provided in the volfile, and then the result is turned into a
+dictionary and stored as `xl->options`. This dictionary is then processed by
+your init function, which you can see being looked up in the first code
+fragment above. We're only going to look at a small part of the `rot-13`'s
+init for now.
+
+```
+priv->decrypt_read = 1;
+priv->encrypt_write = 1;
+
+data = dict_get (this->options, "encrypt-write");
+if (data) {
+    if (gf_string2boolean (data->data, &priv->encrypt_write
+        == -1) {
+        gf_log (this->name, GF_LOG_ERROR,
+            "encrypt-write takes only boolean options");
+        return -1;
+    }
+}
+```
+
+What we can see here is that we're setting some defaults in our priv structure,
+then looking to see if an `encrypt-write` option was actually provided. If so,
+we convert and store it. This is a pretty classic use of dict_get to fetch a
+field from a dictionary, and of using one of many conversion functions in
+`common-utils.c` to convert `data->data` into something we can use.
+
+So far we've covered the basic of how a translator gets loaded, how we find its
+various parts, and how we process its options. In my next Translator 101 post,
+we'll go a little deeper into other things that init and its companion fini
+might do, and how some other fields in our `xlator_t` structure (commonly
+referred to as this) are commonly used.
+
+`init`, `fini`, and private context
+-----------------------------------
+
+In the previous Translator 101 post, we looked at some of the dispatch tables
+and options processing in a translator. This time we're going to cover the rest
+ of the "shell" of a translator -- i.e. the other global parts not specific to
+handling a particular request.
+
+Let's start by looking at the relationship between a translator and its shared
+library. At a first approximation, this is the relationship between an object
+and a class in just about any object-oriented programming language. The class
+defines behaviors, but has to be instantiated as an object to have any kind of
+existence. In our case the object is an `xlator_t`. Several of these might be
+created within the same daemon, sharing all of the same code through init/fini
+and dispatch tables, but sharing *no data*. You could implement shared data (as
+ static variables in your shared libraries) but that's strongly discouraged.
+Every function in your shared library will get an `xlator_t` as an argument,
+and should use it. This lack of class-level data is one of the points where
+the analogy to common OOP systems starts to break down. Another place is the
+complete lack of inheritance. Translators inherit behavior (code) from exactly
+one shared library -- looked up and loaded using the `type` field in a volfile
+`volume ... end-volume` block -- and that's it -- not even single inheritance,
+no subclasses or superclasses, no mixins or prototypes, just the relationship
+between an object and its class. With that in mind, let's turn to the init
+function that we just barely touched on last time.
+
+```
+int32_t
+init (xlator_t *this)
+{
+    data_t *data = NULL;
+    rot_13_private_t *priv = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_log ("rot13", GF_LOG_ERROR,
+            "FATAL: rot13 should have exactly one child");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_log (this->name, GF_LOG_WARNING,
+            "dangling volume. check volfile ");
+    }
+
+    priv = GF_CALLOC (sizeof (rot_13_private_t), 1, 0);
+    if (!priv)
+        return -1;
+```
+
+At the very top, we see the function signature -- we get a pointer to the
+`xlator_t` object that we're initializing, and we return an `int32_t` status.
+As with most functions in the translator API, this should be zero to indicate
+success. In this case it's safe to return -1 for failure, but watch out: in
+dispatch-table functions, the return value means the status of the *function
+call* rather than the *request*. A request error should be reflected as a
+callback with a non-zero `op_re`t value, but the dispatch function itself
+should still return zero. In fact, the handling of a non-zero return from a
+dispatch function is not all that robust (we recently had a bug report in
+HekaFS related to this) so it's something you should probably avoid
+altogether. This only underscores the difference between dispatch functions
+and `init`/`fini` functions, where non-zero returns *are* expected and handled
+logically by aborting the translator setup. We can see that down at the
+bottom, where we return -1 to indicate that we couldn't allocate our
+private-data area (more about that later).
+
+The first thing this init function does is check that the translator is being
+set up in the right kind of environment. Translators are called by parents and
+in turn call children. Some translators are "initial" translators that inject
+requests into the system from elsewhere -- e.g. mount/fuse injecting requests
+from the kernel, protocol/server injecting requests from the network. Those
+translators don't need parents, but `rot-13` does and so we check for that.
+Similarly, some translators are "final" translators that (from the perspective
+of the current process) terminate requests instead of passing them on -- e.g.
+`protocol/client` passing them to another node, `storage/posix` passing them to
+a local filesystem. Other translators "multiplex" between multiple children --
+ passing each parent request on to one (`cluster/dht`), some
+(`cluster/stripe`), or all (`cluster/afr`) of those children. `rot-13` fits
+into none of those categories either, so it checks that it has *exactly one*
+child. It might be more convenient or robust if translator shared libraries
+had standard variables describing these requirements, to be checked in a
+consistent way by the translator-loading infrastructure itself instead of by
+each separate init function, but this is the way translators work today.
+
+The last thing we see in this fragment is allocating our private data area.
+This can literally be anything we want; the infrastructure just provides the
+priv pointer as a convenience but takes no responsibility for how it's used. In
+ this case we're using `GF_CALLOC` to allocate our own `rot_13_private_t`
+structure. This gets us all the benefits of GlusterFS's memory-leak detection
+infrastructure, but the way we're calling it is not quite ideal. For one thing,
+ the first two arguments -- from `calloc(3)` -- are kind of reversed. For
+another, notice how the last argument is zero. That can actually be an
+enumerated value, to tell the GlusterFS allocator *what* type we're
+allocating. This can be very useful information for memory profiling and leak
+detection, so it's recommended that you follow the example of any
+x`xx-mem-types.h` file elsewhere in the source tree instead of just passing
+zero here (even though that works).
+
+To finish our tour of standard initialization/termination, let's look at the
+end of `init` and the beginning of `fini`:
+
+```
+    this->private = priv;
+    gf_log ("rot13", GF_LOG_DEBUG, "rot13 xlator loaded");
+    return 0;
+}
+
+void
+fini (xlator_t *this)
+{
+    rot_13_private_t *priv = this->private;
+
+    if (!priv)
+        return;
+    this->private = NULL;
+    GF_FREE (priv);
+```
+
+At the end of init we're just storing our private-data pointer in the `priv`
+field of our `xlator_t`, then returning zero to indicate that initialization
+succeeded. As is usually the case, our fini is even simpler. All it really has
+to do is `GF_FREE` our private-data pointer, which we do in a slightly
+roundabout way here. Notice how we don't even have a return value here, since
+there's nothing obvious and useful that the infrastructure could do if `fini`
+failed.
+
+That's practically everything we need to know to get our translator through
+loading, initialization, options processing, and termination. If we had defined
+ no dispatch functions, we could actually configure a daemon to use our
+translator and it would work as a basic pass-through from its parent to a
+single child. In the next post I'll cover how to build the translator and
+configure a daemon to use it, so that we can actually step through it in a
+debugger and see how it all fits together before we actually start adding
+functionality.
+
+This Time For Real
+------------------
+
+In the first two parts of this series, we learned how to write a basic
+translator skeleton that can get through loading, initialization, and option
+processing. This time we'll cover how to build that translator, configure a
+volume to use it, and run the glusterfs daemon in debug mode.
+
+Unfortunately, there's not much direct support for writing new translators. You
+can check out a GlusterFS tree and splice in your own translator directory, but
+ that's a bit painful because you'll have to update multiple makefiles plus a
+bunch of autoconf garbage. As part of the HekaFS project, I basically reverse
+engineered the truly necessary parts of the translator-building process and
+then pestered one of the Fedora glusterfs package maintainers (thanks
+daMaestro!) to add a `glusterfs-devel` package with the required headers. Since
+ then the complexity level in the HekaFS tree has crept back up a bit, but I
+still remember the simple method and still consider it the easiest way to get
+started on a new translator. For the sake of those not using Fedora, I'm going
+to describe a method that doesn't depend on that header package. What it does
+depend on is a GlusterFS source tree, much as you might have cloned from GitHub
+ or the Gluster review site. This tree doesn't have to be fully built, but you
+do need to run `autogen.sh` and configure in it. Then you can take the
+following simple makefile and put it in a directory with your actual source.
+
+```
+# Change these to match your source code.
+TARGET  = rot-13.so
+OBJECTS = rot-13.o
+
+# Change these to match your environment.
+GLFS_SRC = /srv/glusterfs
+GLFS_LIB = /usr/lib64
+HOST_OS  = GF_LINUX_HOST_OS
+
+# You shouldn't need to change anything below here.
+
+CFLAGS  = -fPIC -Wall -O0 -g \
+      -DHAVE_CONFIG_H -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE \
+      -D$(HOST_OS) -I$(GLFS_SRC) -I$(GLFS_SRC)/contrib/uuid \
+      -I$(GLFS_SRC)/libglusterfs/src
+LDFLAGS = -shared -nostartfiles -L$(GLFS_LIB) -lglusterfs \
+      -lpthread
+
+$(TARGET): $(OBJECTS)
+    $(CC) $(OBJECTS) $(LDFLAGS) -o $(TARGET)
+```
+
+Yes, it's still Linux-specific. Mea culpa. As you can see, we're sticking with
+the `rot-13` example, so you can just copy the files from
+`xlators/encryption/rot-13/src` in your GlusterFS tree to follow on. Type
+`make` and you should be rewarded with a nice little `.so` file.
+
+```
+xlator_example$ ls -l rot-13.so
+-rwxr-xr-x. 1 jeff jeff 40784 Nov 16 16:41 rot-13.so
+```
+
+Notice that we've built with optimization level zero and debugging symbols
+included, which would not typically be the case for a packaged version of
+GlusterFS. Let's put our version of `rot-13.so` into a slightly different file
+on our system, so that it doesn't stomp on the installed version (not that
+you'd ever want to use that anyway).
+
+```
+xlator_example# ls /usr/lib64/glusterfs/3git/xlator/encryption/
+crypt.so crypt.so.0 crypt.so.0.0.0 rot-13.so rot-13.so.0
+rot-13.so.0.0.0
+xlator_example# cp rot-13.so \
+    /usr/lib64/glusterfs/3git/xlator/encryption/my-rot-13.so
+```
+
+These paths represent the current Gluster filesystem layout, which is likely to
+be deprecated in favor of the Fedora layout; your paths may vary. At this point
+ we're ready to configure a volume using our new translator. To do that, I'm
+going to suggest something that's strongly discouraged except during
+development (the Gluster guys are going to hate me for this): write our own
+volfile. Here's just about the simplest volfile you'll ever see.
+
+```
+volume my-posix
+    type storage/posix
+    option directory /srv/export
+end-volume
+
+volume my-rot13
+    type encryption/my-rot-13
+    subvolumes my-posix
+end-volume
+```
+
+All we have here is a basic brick using `/srv/export` for its data, and then
+an instance of our translator layered on top -- no client or server is
+necessary for what we're doing, and the system will automatically push a
+mount/fuse translator on top if there's no server translator. To try this out,
+all we need is the following command (assuming the directories involved already
+ exist).
+
+```
+xlator_example$ glusterfs --debug -f my.vol /srv/import
+```
+
+You should be rewarded with a whole lot of log output, including the text of
+the volfile (this is very useful for debugging problems in the field). If you
+go to another window on the same machine, you can see that you have a new
+filesystem mounted.
+
+```
+~$ df /srv/import
+Filesystem   1K-blocks     Used    Available Use% Mounted on
+/srv/xlator_example/my.vol
+             114506240     2706176 105983488   3% /srv/import
+```
+
+Just for fun, write something into a file in `/srv/import`, then look at the
+corresponding file in `/srv/export` to see it all `rot-13`'ed for you.
+
+```
+~$ echo hello > /srv/import/a_file
+~$ cat /srv/export/a_file
+uryyb
+```
+
+There you have it -- functionality you control, implemented easily, layered on
+top of local storage. Now you could start adding functionality -- real
+encryption, perhaps -- and inevitably having to debug it. You could do that the
+ old-school way, with `gf_log` (preferred) or even plain old `printf`, or you
+could  run daemons under `gdb` instead. Alternatively, you could wait for the
+next  Translator 101 post, where we'll be doing exactly that.
+
+Debugging a Translator
+----------------------
+
+Now that we've learned what a translator looks like and how to build one, it's
+time to run one and actually watch it work. The best way to do this is good
+old-fashioned `gdb`, as follows (using some of the examples from last time).
+
+```
+xlator_example# gdb glusterfs
+GNU gdb (GDB) Red Hat Enterprise Linux (7.2-50.el6)
+...
+(gdb) r --debug -f my.vol /srv/import
+Starting program: /usr/sbin/glusterfs --debug -f my.vol /srv/import
+...
+[2011-11-23 11:23:16.495516] I [fuse-bridge.c:2971:fuse_init]
+    0-glusterfs-fuse: FUSE inited with protocol versions:
+    glusterfs 7.13 kernel 7.13
+```
+
+If you get to this point, your glusterfs client process is already running. You
+can go to another window to see the mountpoint, do file operations, etc.
+
+```
+~# df /srv/import
+Filesystem   1K-blocks     Used    Available Use% Mounted on
+/root/xlator_example/my.vol
+             114506240     2643968 106045568   3% /srv/import
+~# ls /srv/import
+a_file
+~# cat /srv/import/a_file
+hello
+```
+
+Now let's interrupt the process and see where we are.
+
+```
+^C
+Program received signal SIGINT, Interrupt.
+0x0000003a0060b3dc in pthread_cond_wait@@GLIBC_2.3.2 ()
+                   from /lib64/libpthread.so.0
+(gdb) info threads
+  5 Thread 0x7fffeffff700 (LWP 27206)  0x0000003a002dd8c7
+    in readv ()
+    from /lib64/libc.so.6
+  4 Thread 0x7ffff50e3700 (LWP 27205)  0x0000003a0060b75b
+    in pthread_cond_timedwait@@GLIBC_2.3.2 ()
+    from /lib64/libpthread.so.0
+  3 Thread 0x7ffff5f02700 (LWP 27204)  0x0000003a0060b3dc
+    in pthread_cond_wait@@GLIBC_2.3.2 ()
+    from /lib64/libpthread.so.0
+  2 Thread 0x7ffff6903700 (LWP 27203)  0x0000003a0060f245
+    in sigwait ()
+    from /lib64/libpthread.so.0
+* 1 Thread 0x7ffff7957700 (LWP 27196)  0x0000003a0060b3dc
+    in pthread_cond_wait@@GLIBC_2.3.2 ()
+    from /lib64/libpthread.so.0
+```
+
+Like any non-toy server, this one has multiple threads. What are they all
+doing? Honestly, even I don't know. Thread 1 turns out to be in
+`event_dispatch_epoll`, which means it's the one handling all of our network
+I/O. Note that with socket multi-threading patch this will change, with one
+thread in `socket_poller` per connection. Thread 2 is in `glusterfs_sigwaiter`
+which means signals will be isolated to that thread. Thread 3 is in
+`syncenv_task`, so it's a worker process for synchronous requests such as
+those used by the rebalance and repair code. Thread 4 is in
+`janitor_get_next_fd`, so it's waiting for a chance to close no-longer-needed
+file descriptors on the local filesystem. (I admit I had to look that one up,
+BTW.) Lastly, thread 5 is in `fuse_thread_proc`, so it's the one fetching
+requests from our FUSE interface. You'll often see many more threads than
+this, but it's a pretty good basic set. Now, let's set a breakpoint so we can
+actually watch a request.
+
+```
+(gdb) b rot13_writev
+Breakpoint 1 at 0x7ffff50e4f0b: file rot-13.c, line 119.
+(gdb) c
+Continuing.
+```
+
+At this point we go into our other window and do something that will involve a write.
+
+```
+~# echo goodbye > /srv/import/another_file
+(back to the first window)
+[Switching to Thread 0x7fffeffff700 (LWP 27206)]
+
+Breakpoint 1, rot13_writev (frame=0x7ffff6e4402c, this=0x638440,
+    fd=0x7ffff409802c, vector=0x7fffe8000cd8, count=1, offset=0,
+    iobref=0x7fffe8001070) at rot-13.c:119
+119   rot_13_private_t *priv = (rot_13_private_t *)this->private;
+```
+
+Remember how we built with debugging symbols enabled and no optimization? That
+will be pretty important for the next few steps. As you can see, we're in
+`rot13_writev`, with several parameters.
+
+* `frame` is our always-present frame pointer for this request. Also,
+  `frame->local` will point to any local data we created and attached to the
+  request ourselves.
+* `this` is a pointer to our instance of the `rot-13` translator. You can examine
+  it if you like to see the name, type, options, parent/children, inode table,
+  and other stuff associated with it.
+* `fd` is a pointer to a file-descriptor *object* (`fd_t`, not just a
+  file-descriptor index which is what most people use "fd" for). This in turn
+  points to an inode object (`inode_t`) and we can associate our own
+  `rot-13`-specific data with either of these.
+* `vector` and `count` together describe the data buffers for this write, which
+  we'll get to in a moment.
+* `offset` is the offset into the file at which we're writing.
+* `iobref` is a buffer-reference object, which is used to track the life cycle
+  of buffers containing read/write data. If you look closely, you'll notice that
+  `vector[0].iov_base` points to the same address as `iobref->iobrefs[0].ptr`, which
+  should give you some idea of the inter-relationships between vector and iobref.
+
+OK, now what about that `vector`? We can use it to examine the data being
+written, like this.
+
+```
+(gdb) p vector[0]
+$2 = {iov_base = 0x7ffff7936000, iov_len = 8}
+(gdb) x/s 0x7ffff7936000
+0x7ffff7936000:     "goodbye\n"
+```
+
+It's not always safe to view this data as a string, because it might just as
+well be binary data, but since we're generating the write this time it's safe
+and convenient. With that knowledge, let's step through things a bit.
+
+```
+(gdb) s
+120        if (priv->encrypt_write)
+(gdb)
+121            rot13_iovec (vector, count);
+(gdb)
+rot13_iovec (vector=0x7fffe8000cd8, count=1) at rot-13.c:57
+57        for (i = 0; i < count; i++) {
+(gdb)
+58            rot13 (vector[i].iov_base, vector[i].iov_len);
+(gdb)
+rot13 (buf=0x7ffff7936000 "goodbye\n", len=8) at rot-13.c:45
+45        for (i = 0; i < len; i++) {
+(gdb)
+46            if (buf[i] >= 'a' && buf[i] <= 'z')
+(gdb)
+47                buf[i] = 'a' + ((buf[i] - 'a' + 13) % 26);
+```
+
+Here we've stepped into `rot13_iovec`, which iterates through our vector
+calling `rot13`, which in turn iterates through the characters in that chunk
+doing the  `rot-13` operation if/as appropriate. This is pretty straightforward
+stuff, so  let's skip to the next interesting bit.
+
+```
+(gdb) fin
+Run till exit from #0  rot13 (buf=0x7ffff7936000 "goodbye\n",
+    len=8) at rot-13.c:47
+rot13_iovec (vector=0x7fffe8000cd8, count=1) at rot-13.c:57
+57        for (i = 0; i < count; i++) {
+(gdb) fin
+Run till exit from #0  rot13_iovec (vector=0x7fffe8000cd8,
+    count=1) at rot-13.c:57
+rot13_writev (frame=0x7ffff6e4402c, this=0x638440,
+    fd=0x7ffff409802c, vector=0x7fffe8000cd8, count=1,
+    offset=0, iobref=0x7fffe8001070) at rot-13.c:123
+123        STACK_WIND (frame,
+(gdb) b 129
+Breakpoint 2 at 0x7ffff50e4f35: file rot-13.c, line 129.
+(gdb) b rot13_writev_cbk
+Breakpoint 3 at 0x7ffff50e4db3: file rot-13.c, line 106.
+(gdb) c
+```
+
+So we've set breakpoints on both the callback and the statement following the
+`STACK_WIND`. Which one will we hit first?
+
+```
+Breakpoint 3, rot13_writev_cbk (frame=0x7ffff6e4402c,
+    cookie=0x7ffff6e440d8, this=0x638440, op_ret=8, op_errno=0,
+    prebuf=0x7fffefffeca0, postbuf=0x7fffefffec30)
+    at rot-13.c:106
+106  STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno,
+                          prebuf, postbuf);
+(gdb) bt
+#0  rot13_writev_cbk (frame=0x7ffff6e4402c,
+    cookie=0x7ffff6e440d8, this=0x638440, op_ret=8, op_errno=0,
+    prebuf=0x7fffefffeca0, postbuf=0x7fffefffec30)
+    at rot-13.c:106
+#1  0x00007ffff52f1b37 in posix_writev (frame=0x7ffff6e440d8,
+    this=<value optimized out>, fd=<value optimized out>,
+    vector=<value optimized out>, count=1,
+    offset=<value optimized out>, iobref=0x7fffe8001070)
+    at posix.c:2217
+#2  0x00007ffff50e513e in rot13_writev (frame=0x7ffff6e4402c,
+    this=0x638440, fd=0x7ffff409802c, vector=0x7fffe8000cd8,
+    count=1, offset=0, iobref=0x7fffe8001070) at rot-13.c:123
+```
+
+Surprise! We're in `rot13_writev_cbk` now, called (indirectly) while we're
+still in `rot13_writev` before `STACK_WIND` returns (still at rot-13.c:123). If
+ you did any request cleanup here, then you need to be careful about what you
+do in the remainder of `rot13_writev` because data may have been freed etc.
+It's tempting to say you should just do the cleanup in `rot13_writev` after
+the `STACK_WIND,` but that's not valid because it's also possible that some
+other translator returned without calling `STACK_UNWIND` -- i.e. before
+`rot13_writev` is called, so then it would be the one getting null-pointer
+errors instead. To put it another way, the callback and the return from
+`STACK_WIND` can occur in either order or even simultaneously on different
+threads. Even if you were to use reference counts, you'd have to make sure to
+use locking or atomic operations to avoid races, and it's not worth it. Unless
+you *really* understand the possible flows of control and know what you're
+doing, it's better to do cleanup in the callback and nothing after
+`STACK_WIND.`
+
+At this point all that's left is a `STACK_UNWIND` and a return. The
+`STACK_UNWIND` invokes our parent's completion callback, and in this case our
+parent is FUSE so at that point the VFS layer is notified of the write being
+complete. Finally, we return through several levels of normal function calls
+until we come back to fuse_thread_proc, which waits for the next request.
+
+So that's it. For extra fun, you might want to repeat this exercise by stepping
+through some other call -- stat or setxattr might be good choices -- but you'll
+ have to use a translator that actually implements those calls to see much
+that's interesting. Then you'll pretty much know everything I knew when I
+started writing my first for-real translators, and probably even a bit more. I
+hope you've enjoyed this series, or at least found it useful, and if you have
+any suggestions for other topics I should cover please let me know (via
+comments or email, IRC or Twitter).
diff --git a/doc/hacker-guide/en-US/markdown/write-behind.md b/doc/hacker-guide/en-US/markdown/write-behind.md
new file mode 100644
index 000000000..e20682249
--- /dev/null
+++ b/doc/hacker-guide/en-US/markdown/write-behind.md
@@ -0,0 +1,56 @@
+performance/write-behind translator
+===================================
+
+Basic working
+--------------
+
+Write behind is basically a translator to lie to the application that the 
+write-requests are finished, even before it is actually finished.
+
+On a regular translator tree without write-behind, control flow is like this:
+
+1. application makes a `write()` system call.
+2. VFS ==> FUSE ==> `/dev/fuse`.
+3. fuse-bridge initiates a glusterfs `writev()` call.
+4. `writev()` is `STACK_WIND()`ed upto client-protocol or storage translator.
+5. client-protocol, on receiving reply from server, starts `STACK_UNWIND()` towards the fuse-bridge.
+
+On a translator tree with write-behind, control flow is like this:
+
+1. application makes a `write()` system call.
+2. VFS ==> FUSE ==> `/dev/fuse`.
+3. fuse-bridge initiates a glusterfs `writev()` call.
+4. `writev()` is `STACK_WIND()`ed upto write-behind translator.
+5. write-behind adds the write buffer to its internal queue and does a `STACK_UNWIND()` towards the fuse-bridge.
+
+write call is completed in application's percepective. after 
+`STACK_UNWIND()`ing towards the fuse-bridge, write-behind initiates a fresh 
+writev() call to its child translator, whose replies will be consumed by 
+write-behind itself. Write-behind _doesn't_ cache the write buffer, unless 
+`option flush-behind on` is specified in volume specification file.
+
+Windowing
+---------
+
+With respect to write-behind, each write-buffer has three flags: `stack_wound`, `write_behind` and `got_reply`.
+
+* `stack_wound`: if set, indicates that write-behind has initiated `STACK_WIND()` towards child translator.
+* `write_behind`: if set, indicates that write-behind has done `STACK_UNWIND()` towards fuse-bridge.
+* `got_reply`: if set, indicates that write-behind has received reply from child translator for a `writev()` `STACK_WIND()`. a request will be destroyed by write-behind only if this flag is set.
+
+Currently pending write requests = aggregate size of requests with write_behind = 1 and got_reply = 0.
+
+window size limits the aggregate size of currently pending write requests. once 
+the pending requests' size has reached the window size, write-behind blocks  
+writev() calls from fuse-bridge. Blocking is only from application's 
+perspective. Write-behind does `STACK_WIND()` to child translator 
+straight-away, but hold behind the `STACK_UNWIND()` towards fuse-bridge. 
+`STACK_UNWIND()` is done only once write-behind gets enough replies to 
+accomodate for currently blocked request.
+
+Flush behind
+------------
+
+If `option flush-behind on` is specified in volume specification file, then 
+write-behind sends aggregate write requests to child translator, instead of 
+regular per request `STACK_WIND()`s.
diff --git a/doc/hacker-guide/lock-ahead.txt b/doc/hacker-guide/lock-ahead.txt
deleted file mode 100644
index 63392b7fa..000000000
--- a/doc/hacker-guide/lock-ahead.txt
+++ /dev/null
@@ -1,80 +0,0 @@
-			Lock-ahead translator
-                        ---------------------
-
-The objective of the lock-ahead translator is to speculatively
-hold locks (inodelk and entrylk) on the universal set (0 - infinity
-in case of inodelk and all basenames in case of entrylk) even
-when a lock is requested only on a subset, in anticipation that
-further locks will be requested within the same universal set.
-
-So, for example, when cluster/replicate locks a region before
-writing to it, lock-ahead would instead lock the entire file.
-On further writes, lock-ahead can immediately return success for 
-the lock requests, since the entire file has been previously locked.
-
-To avoid starvation of other clients/mountpoints, we employ a
-notify mechanism, described below.
-
-typedef struct {
-        struct list_head subset_locks;
-} la_universal_lock_t;
-
-Universal lock structure is stored in the inode context.
-
-typedef struct {
-	enum {LOCK_AHEAD_ENTRYLK, LOCK_AHEAD_FENTRYLK,
-              LOCK_AHEAD_INODELK, LOCK_AHEAD_FINODELK};
-
-        union {
-	      fd_t *fd;
-              loc_t loc;
-	};
-
-	off_t l_start;
-        off_t l_len;
-
-	const char *basename;
-
-	struct list_head universal_lock;
-} la_subset_lock_t;
-
-
-fops implemented:
-
-* inodelk/finodelk/entrylk/fentrylk:
-
-lock:
-  if universal lock held:
-    add subset to it (save loc_t or fd) and return success
-  else:
-    send lock-notify fop
-    hold universal lock and return
-      (set inode context, add subset to it, save loc_t or fd)
-    
-    if this fails:
-      forward the lock request
-
-unlock:
-  if subset exists in universal lock:
-    delete subset lock from list
-  else:
-    forward it
-
-* release:
-    hold subset locks (each subset lock using the saved loc_t or fd)
-    and release universal lock
-
-* lock-notify (on unwind) (new fop)
-    hold subset locks and release universal lock
-
-
-lock-notify in locks translator:
-
-if a subset lock in entrylk/inodelk cannot be satisfied
-because of a universal lock held by someone else:
-    unwind the lock-notify fop
-
-==============================================
-$ Last updated: Tue Feb 17 11:31:18 IST 2009 $
-$ Author: Vikas Gorur <vikas@gluster.com>  $
-==============================================
diff --git a/doc/hacker-guide/posix.txt b/doc/hacker-guide/posix.txt
deleted file mode 100644
index d0132abfe..000000000
--- a/doc/hacker-guide/posix.txt
+++ /dev/null
@@ -1,59 +0,0 @@
----------------
-* storage/posix
----------------
-
-- SET_FS_ID
-
-  This is so that all filesystem checks are done with the user's 
-  uid/gid and not GlusterFS's uid/gid.
-
-- MAKE_REAL_PATH
- 
-  This macro concatenates the base directory of the posix volume
-  ('option directory') with the given path.
-
-- need_xattr in lookup
-
-  If this flag is passed, lookup returns a xattr dictionary that contains
-  the file's create time, the file's contents, and the version number
-  of the file.
-
-  This is a hack to increase small file performance. If an application 
-  wants to read a small file, it can finish its job with just a lookup 
-  call instead of a lookup followed by read.
-
-- getdents/setdents
-
-  These are used by unify to set and get directory entries.
-
-- ALIGN_BUF
-  
-  Macro to align an address to a page boundary (4K).
-
-- priv->export_statfs
-
-  In some cases, two exported volumes may reside on the same
-  partition on the server. Sending statvfs info for both
-  the volumes will lead to erroneous df output at the client,
-  since free space on the partition will be counted twice.
-
-  In such cases, user can disable exporting statvfs info
-  on one of the volumes by setting this option.
-
-- xattrop
-
-  This fop is used by replicate to set version numbers on files. 
-
-- getxattr/setxattr hack to read/write files
-
-  A key, GLUSTERFS_FILE_CONTENT_STRING, is handled in a special way by
-  getxattr/setxattr. A getxattr with the key will return the entire
-  content of the file as the value. A setxattr with the key will write
-  the value as the entire content of the file.
-
-- posix_checksum
-  
-  This calculates a simple XOR checksum on all entry names in a
-  directory that is used by unify to compare directory contents.
-
-
diff --git a/doc/hacker-guide/write-behind.txt b/doc/hacker-guide/write-behind.txt
deleted file mode 100644
index a6e9a8890..000000000
--- a/doc/hacker-guide/write-behind.txt
+++ /dev/null
@@ -1,45 +0,0 @@
-basic working
---------------
-
-  write behind is basically a translator to lie to the application that the write-requests are finished, even before it is actually finished.
-
-  on a regular translator tree without write-behind, control flow is like this:
-  
-  1. application makes a write() system call.
-  2. VFS ==> FUSE ==> /dev/fuse.
-  3. fuse-bridge initiates a glusterfs writev() call.
-  4. writev() is STACK_WIND()ed upto client-protocol or storage translator.
-  5. client-protocol, on receiving reply from server, starts STACK_UNWIND() towards the fuse-bridge.
-
-  on a translator tree with write-behind, control flow is like this:
-  
-  1. application makes a write() system call.
-  2. VFS ==> FUSE ==> /dev/fuse.
-  3. fuse-bridge initiates a glusterfs writev() call.
-  4. writev() is STACK_WIND()ed upto write-behind translator.
-  5. write-behind adds the write buffer to its internal queue and does a STACK_UNWIND() towards the fuse-bridge.
-  
-  write call is completed in application's percepective. after STACK_UNWIND()ing towards the fuse-bridge, write-behind initiates a fresh writev() call to its child translator, whose replies will be consumed by write-behind itself. write-behind _doesn't_ cache the write buffer, unless 'option flush-behind on' is specified in volume specification file.
-
-windowing
----------
-
-  write respect to write-behind, each write-buffer has three flags: 'stack_wound', 'write_behind' and 'got_reply'.
-
-  stack_wound: if set, indicates that write-behind has initiated STACK_WIND() towards child translator. 
-
-  write_behind: if set, indicates that write-behind has done STACK_UNWIND() towards fuse-bridge.
-
-  got_reply: if set, indicates that write-behind has received reply from child translator for a writev() STACK_WIND(). a request will be destroyed by write-behind only if this flag is set.
-
-  currently pending write requests = aggregate size of requests with write_behind = 1 and got_reply = 0.
-  
-  window size limits the aggregate size of currently pending write requests. once the pending requests' size has reached the window size, write-behind blocks  writev() calls from fuse-bridge. 
-  blocking is only from application's perspective. write-behind does STACK_WIND() to child translator straight-away, but hold behind the STACK_UNWIND() towards fuse-bridge. STACK_UNWIND() is done only once write-behind gets enough replies to accomodate for currently blocked request.
-  
-flush behind
-------------
-
-  if 'option flush-behind on' is specified in volume specification file, then write-behind sends aggregate write requests to child translator, instead of regular per request STACK_WIND()s.
-  
-
diff --git a/doc/user-guide/Makefile.am b/doc/legacy/Makefile.am
index 800e7321d..b2caabaa2 100644
--- a/doc/user-guide/Makefile.am
+++ b/doc/legacy/Makefile.am
@@ -1,3 +1,3 @@
 info_TEXINFOS = user-guide.texi
-CLEANFILES = *~ 
+CLEANFILES = *~
 DISTCLEANFILES = .deps/*.P *.info *vti
diff --git a/doc/user-guide/advanced-stripe.odg b/doc/legacy/advanced-stripe.odg
index 7686d7091..7686d7091 100644
--- a/doc/user-guide/advanced-stripe.odg
+++ b/doc/legacy/advanced-stripe.odg
diff --git a/doc/user-guide/advanced-stripe.pdf b/doc/legacy/advanced-stripe.pdf
index ec8b03dcf..ec8b03dcf 100644
--- a/doc/user-guide/advanced-stripe.pdf
+++ b/doc/legacy/advanced-stripe.pdf
diff --git a/doc/booster.txt b/doc/legacy/booster.txt
index 684ac8965..051401a28 100644
--- a/doc/booster.txt
+++ b/doc/legacy/booster.txt
@@ -1,6 +1,6 @@
 Introduction
 ============
-* booster is a LD_PRELOADable library which boosts read/write performance by bypassing fuse for 
+* booster is a LD_PRELOADable library which boosts read/write performance by bypassing fuse for
   read() and write() calls.
 
 Requirements
@@ -14,29 +14,29 @@ Design
   * contents of client volume-file.
   * mount point.
 
-* LD_PRELOADed booster.so maintains an hash table storing mount-points and libglusterfsclient handles 
-  so that handles are reused for files from same mount point. 
+* LD_PRELOADed booster.so maintains an hash table storing mount-points and libglusterfsclient handles
+  so that handles are reused for files from same mount point.
 
 * it also maintains a fdtable. fdtable maps the fd (integer) returned to application to fd (pointer to fd struct)
   used by libglusterfsclient. application is returned the same fd as the one returned from libc apis.
 
 * During fork, these tables are overwritten to enable creation of fresh glusterfs context in child.
- 
+
 Working
 =======
-* application willing to use booster LD_PRELOADs booster.so which is a wrapper library implementing 
+* application willing to use booster LD_PRELOADs booster.so which is a wrapper library implementing
   open, read and write.
 
-* application should specify the path to logfile through the environment variable GLFS_BOOSTER_LOGFILE. If 
+* application should specify the path to logfile through the environment variable GLFS_BOOSTER_LOGFILE. If
   not specified, logging is done to /dev/stderr.
 
 * open call does,
   * real_open on the file.
   * fgetxattr(fd).
-  * store the volume-file content got in the dictionary to a temparory file.
-  * look in the hashtable for the mount-point, if already present get the libglusterfsclient handle from the 
-    hashtable. Otherwise get a new handle from libglusterfsclient (be careful about mount point not present in 
-    the hashtable and multiple glusterfs_inits running simultaneously for the same mount-point there by using 
+  * store the volume-file content got in the dictionary to a temporary file.
+  * look in the hashtable for the mount-point, if already present get the libglusterfsclient handle from the
+    hashtable. Otherwise get a new handle from libglusterfsclient (be careful about mount point not present in
+    the hashtable and multiple glusterfs_inits running simultaneously for the same mount-point there by using
     multiple handles for the same mount point).
   * real_close (fd).
   * delete temporary volume-volfile.
@@ -51,4 +51,4 @@ Working
 * close call does,
   * remove the fd from the fdtable.
 
-* other calls use real_calls. 
+* other calls use real_calls.
diff --git a/doc/user-guide/colonO-icon.jpg b/doc/legacy/colonO-icon.jpg
index 3e66f7a27..3e66f7a27 100644
--- a/doc/user-guide/colonO-icon.jpg
+++ b/doc/legacy/colonO-icon.jpg
diff --git a/doc/errno.list.bsd.txt b/doc/legacy/errno.list.bsd.txt
index 350af25e4..350af25e4 100644
--- a/doc/errno.list.bsd.txt
+++ b/doc/legacy/errno.list.bsd.txt
diff --git a/doc/errno.list.linux.txt b/doc/legacy/errno.list.linux.txt
index baa50792d..cc868644b 100644
--- a/doc/errno.list.linux.txt
+++ b/doc/legacy/errno.list.linux.txt
@@ -95,7 +95,7 @@ extern "C" {
 
 /**
  * @defgroup apr_errno Error Codes
- * @ingroup APR 
+ * @ingroup APR
  * @{
  */
 
@@ -110,7 +110,7 @@ typedef int apr_status_t;
  * @param buf A buffer to hold the error string.
  * @param bufsize Size of the buffer to hold the string.
  */
-APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf, 
+APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
                                  apr_size_t bufsize);
 
 #if defined(DOXYGEN)
@@ -130,7 +130,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * Fold an apr_status_t code back to the native platform defined error.
  * @param e The apr_status_t folded platform os error code.
  * @warning  macro implementation; the statcode argument may be evaluated
- *      multiple times.  If the statcode was not created by apr_get_os_error 
+ *      multiple times.  If the statcode was not created by apr_get_os_error
  *      or APR_FROM_OS_ERROR, the results are undefined.
  */
 #define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR)
@@ -166,7 +166,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * @warning This is a macro implementation; the statcode argument may be evaluated
  *      multiple times.  If the statcode was not created by apr_get_os_error
  *      or APR_FROM_OS_ERROR, the results are undefined.  This macro sets
- *      errno, or calls a WSASetLastError() style function, unfolding 
+ *      errno, or calls a WSASetLastError() style function, unfolding
  *      socketcode with APR_TO_OS_ERROR.
  */
 
@@ -206,12 +206,12 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_OS_START_CANONERR  (APR_OS_START_USERERR \
                                  + (APR_OS_ERRSPACE_SIZE * 10))
 /**
- * APR_OS_START_EAIERR folds EAI_ error codes from getaddrinfo() into 
+ * APR_OS_START_EAIERR folds EAI_ error codes from getaddrinfo() into
  *     apr_status_t values.
  */
 #define APR_OS_START_EAIERR    (APR_OS_START_CANONERR + APR_OS_ERRSPACE_SIZE)
 /**
- * APR_OS_START_SYSERR folds platform-specific system error values into 
+ * APR_OS_START_SYSERR folds platform-specific system error values into
  *     apr_status_t values.
  */
 #define APR_OS_START_SYSERR    (APR_OS_START_EAIERR + APR_OS_ERRSPACE_SIZE)
@@ -219,13 +219,13 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 /** no error. */
 #define APR_SUCCESS 0
 
-/** 
+/**
  * @defgroup APR_Error APR Error Values
  * <PRE>
  * <b>APR ERROR VALUES</b>
- * APR_ENOSTAT      APR was unable to perform a stat on the file 
+ * APR_ENOSTAT      APR was unable to perform a stat on the file
  * APR_ENOPOOL      APR was not provided a pool with which to allocate memory
- * APR_EBADDATE     APR was given an invalid date 
+ * APR_EBADDATE     APR was given an invalid date
  * APR_EINVALSOCK   APR was given an invalid socket
  * APR_ENOPROC      APR was not given a process structure
  * APR_ENOTIME      APR was not given a time structure
@@ -236,7 +236,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * APR_ENOTHREAD    APR was not given a thread structure
  * APR_ENOTHDKEY    APR was not given a thread key structure
  * APR_ENOSHMAVAIL  There is no more shared memory available
- * APR_EDSOOPEN     APR was unable to open the dso object.  For more 
+ * APR_EDSOOPEN     APR was unable to open the dso object.  For more
  *                  information call apr_dso_error().
  * APR_EGENERAL     General failure (specific information not available)
  * APR_EBADIP       The specified IP address is invalid
@@ -256,17 +256,17 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * APR_INCOMPLETE     The operation was incomplete although some processing
  *                    was performed and the results are partially valid
  * APR_BADCH          Getopt found an option not in the option string
- * APR_BADARG         Getopt found an option that is missing an argument 
+ * APR_BADARG         Getopt found an option that is missing an argument
  *                    and an argument was specified in the option string
  * APR_EOF            APR has encountered the end of the file
  * APR_NOTFOUND       APR was unable to find the socket in the poll structure
  * APR_ANONYMOUS      APR is using anonymous shared memory
  * APR_FILEBASED      APR is using a file name as the key to the shared memory
  * APR_KEYBASED       APR is using a shared key as the key to the shared memory
- * APR_EINIT          Ininitalizer value.  If no option has been found, but 
+ * APR_EINIT          Ininitalizer value.  If no option has been found, but
  *                    the status variable requires a value, this should be used
- * APR_ENOTIMPL       The APR function has not been implemented on this 
- *                    platform, either because nobody has gotten to it yet, 
+ * APR_ENOTIMPL       The APR function has not been implemented on this
+ *                    platform, either because nobody has gotten to it yet,
  *                    or the function is impossible on this platform.
  * APR_EMISMATCH      Two passwords do not match.
  * APR_EABSOLUTE      The given path was absolute.
@@ -334,7 +334,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_ENOTENOUGHENTROPY (APR_OS_START_ERROR + 28)
 /** @} */
 
-/** 
+/**
  * @defgroup APR_STATUS_IS Status Value Tests
  * @warning For any particular error condition, more than one of these tests
  *      may match. This is because platform-specific error codes may not
@@ -345,16 +345,16 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  *      adjust the order of the tests accordingly.
  * @{
  */
-/** 
- * APR was unable to perform a stat on the file 
+/**
+ * APR was unable to perform a stat on the file
  * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_ENOSTAT(s)        ((s) == APR_ENOSTAT)
-/** 
- * APR was not provided a pool with which to allocate memory 
+/**
+ * APR was not provided a pool with which to allocate memory
  * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_ENOPOOL(s)        ((s) == APR_ENOPOOL)
 /** APR was given an invalid date  */
@@ -386,8 +386,8 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 /** The specified netmask is invalid */
 #define APR_STATUS_IS_EBADMASK(s)       ((s) == APR_EBADMASK)
 /* empty slot: +18 */
-/** 
- * APR was unable to open the dso object.  
+/**
+ * APR was unable to open the dso object.
  * For more information call apr_dso_error().
  */
 #if defined(WIN32)
@@ -425,7 +425,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 
 /** @} */
 
-/** 
+/**
  * @addtogroup APR_Error
  * @{
  */
@@ -466,7 +466,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 /** @see APR_STATUS_IS_KEYBASED */
 #define APR_KEYBASED       (APR_OS_START_STATUS + 21)
 /** @see APR_STATUS_IS_EINIT */
-#define APR_EINIT          (APR_OS_START_STATUS + 22)  
+#define APR_EINIT          (APR_OS_START_STATUS + 22)
 /** @see APR_STATUS_IS_ENOTIMPL */
 #define APR_ENOTIMPL       (APR_OS_START_STATUS + 23)
 /** @see APR_STATUS_IS_EMISMATCH */
@@ -475,156 +475,156 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_EBUSY          (APR_OS_START_STATUS + 25)
 /** @} */
 
-/** 
+/**
  * @addtogroup APR_STATUS_IS
  * @{
  */
-/** 
- * Program is currently executing in the child 
+/**
+ * Program is currently executing in the child
  * @warning
  * always use this test, as platform-specific variances may meet this
  * more than one error code */
 #define APR_STATUS_IS_INCHILD(s)        ((s) == APR_INCHILD)
-/** 
- * Program is currently executing in the parent 
+/**
+ * Program is currently executing in the parent
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_INPARENT(s)       ((s) == APR_INPARENT)
-/** 
- * The thread is detached 
+/**
+ * The thread is detached
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_DETACH(s)         ((s) == APR_DETACH)
-/** 
- * The thread is not detached 
+/**
+ * The thread is not detached
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_NOTDETACH(s)      ((s) == APR_NOTDETACH)
-/** 
+/**
  * The child has finished executing
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_CHILD_DONE(s)     ((s) == APR_CHILD_DONE)
-/** 
+/**
  * The child has not finished executing
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_CHILD_NOTDONE(s)  ((s) == APR_CHILD_NOTDONE)
-/** 
+/**
  * The operation did not finish before the timeout
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP)
-/** 
+/**
  * The operation was incomplete although some processing was performed
  * and the results are partially valid.
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_INCOMPLETE(s)     ((s) == APR_INCOMPLETE)
 /* empty slot: +9 */
 /* empty slot: +10 */
 /* empty slot: +11 */
-/** 
+/**
  * Getopt found an option not in the option string
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_BADCH(s)          ((s) == APR_BADCH)
-/** 
- * Getopt found an option not in the option string and an argument was 
+/**
+ * Getopt found an option not in the option string and an argument was
  * specified in the option string
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_BADARG(s)         ((s) == APR_BADARG)
-/** 
+/**
  * APR has encountered the end of the file
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_EOF(s)            ((s) == APR_EOF)
-/** 
+/**
  * APR was unable to find the socket in the poll structure
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_NOTFOUND(s)       ((s) == APR_NOTFOUND)
 /* empty slot: +16 */
 /* empty slot: +17 */
 /* empty slot: +18 */
-/** 
+/**
  * APR is using anonymous shared memory
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_ANONYMOUS(s)      ((s) == APR_ANONYMOUS)
-/** 
+/**
  * APR is using a file name as the key to the shared memory
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_FILEBASED(s)      ((s) == APR_FILEBASED)
-/** 
+/**
  * APR is using a shared key as the key to the shared memory
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_KEYBASED(s)       ((s) == APR_KEYBASED)
-/** 
- * Ininitalizer value.  If no option has been found, but 
+/**
+ * Ininitalizer value.  If no option has been found, but
  * the status variable requires a value, this should be used
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_EINIT(s)          ((s) == APR_EINIT)
-/** 
- * The APR function has not been implemented on this 
- * platform, either because nobody has gotten to it yet, 
+/**
+ * The APR function has not been implemented on this
+ * platform, either because nobody has gotten to it yet,
  * or the function is impossible on this platform.
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_ENOTIMPL(s)       ((s) == APR_ENOTIMPL)
-/** 
+/**
  * Two passwords do not match.
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_EMISMATCH(s)      ((s) == APR_EMISMATCH)
-/** 
+/**
  * The given lock was busy
  * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_EBUSY(s)          ((s) == APR_EBUSY)
 
 /** @} */
 
-/** 
+/**
  * @addtogroup APR_Error APR Error Values
  * @{
  */
@@ -713,8 +713,8 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_ESPIPE         (APR_OS_START_CANONERR + 12)
 #endif
 
-/** 
- * @see APR_STATUS_IS_EAGAIN 
+/**
+ * @see APR_STATUS_IS_EAGAIN
  * @warning use APR_STATUS_IS_EAGAIN instead of just testing this value
  */
 #ifdef EAGAIN
@@ -753,7 +753,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_EINPROGRESS    (APR_OS_START_CANONERR + 17)
 #endif
 
-/** 
+/**
  * @see APR_STATUS_IS_ECONNABORTED
  * @warning use APR_STATUS_IS_ECONNABORTED instead of just testing this value
  */
@@ -771,7 +771,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_ECONNRESET     (APR_OS_START_CANONERR + 19)
 #endif
 
-/** @see APR_STATUS_IS_ETIMEDOUT 
+/** @see APR_STATUS_IS_ETIMEDOUT
  *  @deprecated */
 #ifdef ETIMEDOUT
 #define APR_ETIMEDOUT ETIMEDOUT
@@ -849,7 +849,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  */
 #define APR_OS2_STATUS(e) (APR_FROM_OS_ERROR(e))
 
-/* These can't sit in a private header, so in spite of the extra size, 
+/* These can't sit in a private header, so in spite of the extra size,
  * they need to be made available here.
  */
 #define SOCBASEERR              10000
@@ -946,10 +946,10 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
                 || (s) == APR_OS_START_SYSERR + SOCECONNRESET)
 /* XXX deprecated */
 #define APR_STATUS_IS_ETIMEDOUT(s)         ((s) == APR_ETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)    
+                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)
 #undef APR_STATUS_IS_TIMEUP
 #define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP \
-                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)    
+                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)
 #define APR_STATUS_IS_EHOSTUNREACH(s)   ((s) == APR_EHOSTUNREACH \
                 || (s) == APR_OS_START_SYSERR + SOCEHOSTUNREACH)
 #define APR_STATUS_IS_ENETUNREACH(s)    ((s) == APR_ENETUNREACH \
@@ -1182,7 +1182,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define apr_get_netos_error() (errno)
 #define apr_set_netos_error(e) (errno = (e))
 
-/** 
+/**
  * @addtogroup APR_STATUS_IS
  * @{
  */
@@ -1246,15 +1246,15 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 /** operation now in progress */
 #define APR_STATUS_IS_EINPROGRESS(s)    ((s) == APR_EINPROGRESS)
 
-/** 
- * Software caused connection abort 
+/**
+ * Software caused connection abort
  * @remark
- * EPROTO on certain older kernels really means ECONNABORTED, so we need to 
+ * EPROTO on certain older kernels really means ECONNABORTED, so we need to
  * ignore it for them.  See discussion in new-httpd archives nh.9701 & nh.9603
  *
- * There is potentially a bug in Solaris 2.x x<6, and other boxes that 
+ * There is potentially a bug in Solaris 2.x x<6, and other boxes that
  * implement tcp sockets in userland (i.e. on top of STREAMS).  On these
- * systems, EPROTO can actually result in a fatal loop.  See PR#981 for 
+ * systems, EPROTO can actually result in a fatal loop.  See PR#981 for
  * example.  It's hard to handle both uses of EPROTO.
  */
 #ifdef EPROTO
diff --git a/doc/errno.list.macosx.txt b/doc/legacy/errno.list.macosx.txt
index 728753ac7..4954e03d8 100644
--- a/doc/errno.list.macosx.txt
+++ b/doc/legacy/errno.list.macosx.txt
@@ -34,7 +34,7 @@ extern "C" {
 
 /**
  * @defgroup apr_errno Error Codes
- * @ingroup APR 
+ * @ingroup APR
  * @{
  */
 
@@ -49,7 +49,7 @@ typedef int apr_status_t;
  * @param buf A buffer to hold the error string.
  * @param bufsize Size of the buffer to hold the string.
  */
-APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf, 
+APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
                                  apr_size_t bufsize);
 
 #if defined(DOXYGEN)
@@ -69,7 +69,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * Fold an apr_status_t code back to the native platform defined error.
  * @param e The apr_status_t folded platform os error code.
  * @warning  macro implementation; the statcode argument may be evaluated
- *      multiple times.  If the statcode was not created by apr_get_os_error 
+ *      multiple times.  If the statcode was not created by apr_get_os_error
  *      or APR_FROM_OS_ERROR, the results are undefined.
  */
 #define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR)
@@ -105,7 +105,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * @warning This is a macro implementation; the statcode argument may be evaluated
  *      multiple times.  If the statcode was not created by apr_get_os_error
  *      or APR_FROM_OS_ERROR, the results are undefined.  This macro sets
- *      errno, or calls a WSASetLastError() style function, unfolding 
+ *      errno, or calls a WSASetLastError() style function, unfolding
  *      socketcode with APR_TO_OS_ERROR.
  */
 
@@ -145,12 +145,12 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_OS_START_CANONERR  (APR_OS_START_USERERR \
                                  + (APR_OS_ERRSPACE_SIZE * 10))
 /**
- * APR_OS_START_EAIERR folds EAI_ error codes from getaddrinfo() into 
+ * APR_OS_START_EAIERR folds EAI_ error codes from getaddrinfo() into
  *     apr_status_t values.
  */
 #define APR_OS_START_EAIERR    (APR_OS_START_CANONERR + APR_OS_ERRSPACE_SIZE)
 /**
- * APR_OS_START_SYSERR folds platform-specific system error values into 
+ * APR_OS_START_SYSERR folds platform-specific system error values into
  *     apr_status_t values.
  */
 #define APR_OS_START_SYSERR    (APR_OS_START_EAIERR + APR_OS_ERRSPACE_SIZE)
@@ -158,13 +158,13 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 /** no error. */
 #define APR_SUCCESS 0
 
-/** 
+/**
  * @defgroup APR_Error APR Error Values
  * <PRE>
  * <b>APR ERROR VALUES</b>
- * APR_ENOSTAT      APR was unable to perform a stat on the file 
+ * APR_ENOSTAT      APR was unable to perform a stat on the file
  * APR_ENOPOOL      APR was not provided a pool with which to allocate memory
- * APR_EBADDATE     APR was given an invalid date 
+ * APR_EBADDATE     APR was given an invalid date
  * APR_EINVALSOCK   APR was given an invalid socket
  * APR_ENOPROC      APR was not given a process structure
  * APR_ENOTIME      APR was not given a time structure
@@ -175,7 +175,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * APR_ENOTHREAD    APR was not given a thread structure
  * APR_ENOTHDKEY    APR was not given a thread key structure
  * APR_ENOSHMAVAIL  There is no more shared memory available
- * APR_EDSOOPEN     APR was unable to open the dso object.  For more 
+ * APR_EDSOOPEN     APR was unable to open the dso object.  For more
  *                  information call apr_dso_error().
  * APR_EGENERAL     General failure (specific information not available)
  * APR_EBADIP       The specified IP address is invalid
@@ -195,17 +195,17 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * APR_INCOMPLETE     The operation was incomplete although some processing
  *                    was performed and the results are partially valid
  * APR_BADCH          Getopt found an option not in the option string
- * APR_BADARG         Getopt found an option that is missing an argument 
+ * APR_BADARG         Getopt found an option that is missing an argument
  *                    and an argument was specified in the option string
  * APR_EOF            APR has encountered the end of the file
  * APR_NOTFOUND       APR was unable to find the socket in the poll structure
  * APR_ANONYMOUS      APR is using anonymous shared memory
  * APR_FILEBASED      APR is using a file name as the key to the shared memory
  * APR_KEYBASED       APR is using a shared key as the key to the shared memory
- * APR_EINIT          Ininitalizer value.  If no option has been found, but 
+ * APR_EINIT          Ininitalizer value.  If no option has been found, but
  *                    the status variable requires a value, this should be used
- * APR_ENOTIMPL       The APR function has not been implemented on this 
- *                    platform, either because nobody has gotten to it yet, 
+ * APR_ENOTIMPL       The APR function has not been implemented on this
+ *                    platform, either because nobody has gotten to it yet,
  *                    or the function is impossible on this platform.
  * APR_EMISMATCH      Two passwords do not match.
  * APR_EABSOLUTE      The given path was absolute.
@@ -273,7 +273,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_ENOTENOUGHENTROPY (APR_OS_START_ERROR + 28)
 /** @} */
 
-/** 
+/**
  * @defgroup APR_STATUS_IS Status Value Tests
  * @warning For any particular error condition, more than one of these tests
  *      may match. This is because platform-specific error codes may not
@@ -284,16 +284,16 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  *      adjust the order of the tests accordingly.
  * @{
  */
-/** 
- * APR was unable to perform a stat on the file 
+/**
+ * APR was unable to perform a stat on the file
  * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_ENOSTAT(s)        ((s) == APR_ENOSTAT)
-/** 
- * APR was not provided a pool with which to allocate memory 
+/**
+ * APR was not provided a pool with which to allocate memory
  * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_ENOPOOL(s)        ((s) == APR_ENOPOOL)
 /** APR was given an invalid date  */
@@ -325,8 +325,8 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 /** The specified netmask is invalid */
 #define APR_STATUS_IS_EBADMASK(s)       ((s) == APR_EBADMASK)
 /* empty slot: +18 */
-/** 
- * APR was unable to open the dso object.  
+/**
+ * APR was unable to open the dso object.
  * For more information call apr_dso_error().
  */
 #if defined(WIN32)
@@ -364,7 +364,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 
 /** @} */
 
-/** 
+/**
  * @addtogroup APR_Error
  * @{
  */
@@ -405,7 +405,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 /** @see APR_STATUS_IS_KEYBASED */
 #define APR_KEYBASED       (APR_OS_START_STATUS + 21)
 /** @see APR_STATUS_IS_EINIT */
-#define APR_EINIT          (APR_OS_START_STATUS + 22)  
+#define APR_EINIT          (APR_OS_START_STATUS + 22)
 /** @see APR_STATUS_IS_ENOTIMPL */
 #define APR_ENOTIMPL       (APR_OS_START_STATUS + 23)
 /** @see APR_STATUS_IS_EMISMATCH */
@@ -414,156 +414,156 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_EBUSY          (APR_OS_START_STATUS + 25)
 /** @} */
 
-/** 
+/**
  * @addtogroup APR_STATUS_IS
  * @{
  */
-/** 
- * Program is currently executing in the child 
+/**
+ * Program is currently executing in the child
  * @warning
  * always use this test, as platform-specific variances may meet this
  * more than one error code */
 #define APR_STATUS_IS_INCHILD(s)        ((s) == APR_INCHILD)
-/** 
- * Program is currently executing in the parent 
+/**
+ * Program is currently executing in the parent
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_INPARENT(s)       ((s) == APR_INPARENT)
-/** 
- * The thread is detached 
+/**
+ * The thread is detached
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_DETACH(s)         ((s) == APR_DETACH)
-/** 
- * The thread is not detached 
+/**
+ * The thread is not detached
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_NOTDETACH(s)      ((s) == APR_NOTDETACH)
-/** 
+/**
  * The child has finished executing
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_CHILD_DONE(s)     ((s) == APR_CHILD_DONE)
-/** 
+/**
  * The child has not finished executing
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_CHILD_NOTDONE(s)  ((s) == APR_CHILD_NOTDONE)
-/** 
+/**
  * The operation did not finish before the timeout
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP)
-/** 
+/**
  * The operation was incomplete although some processing was performed
  * and the results are partially valid.
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_INCOMPLETE(s)     ((s) == APR_INCOMPLETE)
 /* empty slot: +9 */
 /* empty slot: +10 */
 /* empty slot: +11 */
-/** 
+/**
  * Getopt found an option not in the option string
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_BADCH(s)          ((s) == APR_BADCH)
-/** 
- * Getopt found an option not in the option string and an argument was 
+/**
+ * Getopt found an option not in the option string and an argument was
  * specified in the option string
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_BADARG(s)         ((s) == APR_BADARG)
-/** 
+/**
  * APR has encountered the end of the file
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_EOF(s)            ((s) == APR_EOF)
-/** 
+/**
  * APR was unable to find the socket in the poll structure
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_NOTFOUND(s)       ((s) == APR_NOTFOUND)
 /* empty slot: +16 */
 /* empty slot: +17 */
 /* empty slot: +18 */
-/** 
+/**
  * APR is using anonymous shared memory
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_ANONYMOUS(s)      ((s) == APR_ANONYMOUS)
-/** 
+/**
  * APR is using a file name as the key to the shared memory
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_FILEBASED(s)      ((s) == APR_FILEBASED)
-/** 
+/**
  * APR is using a shared key as the key to the shared memory
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_KEYBASED(s)       ((s) == APR_KEYBASED)
-/** 
- * Ininitalizer value.  If no option has been found, but 
+/**
+ * Ininitalizer value.  If no option has been found, but
  * the status variable requires a value, this should be used
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_EINIT(s)          ((s) == APR_EINIT)
-/** 
- * The APR function has not been implemented on this 
- * platform, either because nobody has gotten to it yet, 
+/**
+ * The APR function has not been implemented on this
+ * platform, either because nobody has gotten to it yet,
  * or the function is impossible on this platform.
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_ENOTIMPL(s)       ((s) == APR_ENOTIMPL)
-/** 
+/**
  * Two passwords do not match.
  * @warning
  * always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_EMISMATCH(s)      ((s) == APR_EMISMATCH)
-/** 
+/**
  * The given lock was busy
  * @warning always use this test, as platform-specific variances may meet this
- * more than one error code 
+ * more than one error code
  */
 #define APR_STATUS_IS_EBUSY(s)          ((s) == APR_EBUSY)
 
 /** @} */
 
-/** 
+/**
  * @addtogroup APR_Error APR Error Values
  * @{
  */
@@ -652,8 +652,8 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_ESPIPE         (APR_OS_START_CANONERR + 12)
 #endif
 
-/** 
- * @see APR_STATUS_IS_EAGAIN 
+/**
+ * @see APR_STATUS_IS_EAGAIN
  * @warning use APR_STATUS_IS_EAGAIN instead of just testing this value
  */
 #ifdef EAGAIN
@@ -692,7 +692,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_EINPROGRESS    (APR_OS_START_CANONERR + 17)
 #endif
 
-/** 
+/**
  * @see APR_STATUS_IS_ECONNABORTED
  * @warning use APR_STATUS_IS_ECONNABORTED instead of just testing this value
  */
@@ -710,7 +710,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define APR_ECONNRESET     (APR_OS_START_CANONERR + 19)
 #endif
 
-/** @see APR_STATUS_IS_ETIMEDOUT 
+/** @see APR_STATUS_IS_ETIMEDOUT
  *  @deprecated */
 #ifdef ETIMEDOUT
 #define APR_ETIMEDOUT ETIMEDOUT
@@ -788,7 +788,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  */
 #define APR_OS2_STATUS(e) (APR_FROM_OS_ERROR(e))
 
-/* These can't sit in a private header, so in spite of the extra size, 
+/* These can't sit in a private header, so in spite of the extra size,
  * they need to be made available here.
  */
 #define SOCBASEERR              10000
@@ -885,10 +885,10 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
                 || (s) == APR_OS_START_SYSERR + SOCECONNRESET)
 /* XXX deprecated */
 #define APR_STATUS_IS_ETIMEDOUT(s)         ((s) == APR_ETIMEDOUT \
-                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)    
+                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)
 #undef APR_STATUS_IS_TIMEUP
 #define APR_STATUS_IS_TIMEUP(s)         ((s) == APR_TIMEUP \
-                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)    
+                || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT)
 #define APR_STATUS_IS_EHOSTUNREACH(s)   ((s) == APR_EHOSTUNREACH \
                 || (s) == APR_OS_START_SYSERR + SOCEHOSTUNREACH)
 #define APR_STATUS_IS_ENETUNREACH(s)    ((s) == APR_ENETUNREACH \
@@ -1121,7 +1121,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 #define apr_get_netos_error() (errno)
 #define apr_set_netos_error(e) (errno = (e))
 
-/** 
+/**
  * @addtogroup APR_STATUS_IS
  * @{
  */
@@ -1185,15 +1185,15 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
 /** operation now in progress */
 #define APR_STATUS_IS_EINPROGRESS(s)    ((s) == APR_EINPROGRESS)
 
-/** 
- * Software caused connection abort 
+/**
+ * Software caused connection abort
  * @remark
- * EPROTO on certain older kernels really means ECONNABORTED, so we need to 
+ * EPROTO on certain older kernels really means ECONNABORTED, so we need to
  * ignore it for them.  See discussion in new-httpd archives nh.9701 & nh.9603
  *
- * There is potentially a bug in Solaris 2.x x<6, and other boxes that 
+ * There is potentially a bug in Solaris 2.x x<6, and other boxes that
  * implement tcp sockets in userland (i.e. on top of STREAMS).  On these
- * systems, EPROTO can actually result in a fatal loop.  See PR#981 for 
+ * systems, EPROTO can actually result in a fatal loop.  See PR#981 for
  * example.  It's hard to handle both uses of EPROTO.
  */
 #ifdef EPROTO
@@ -1236,14 +1236,14 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_LICENSE_HEADER_START@
- * 
+ *
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * compliance with the License. Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this
  * file.
- * 
+ *
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
@@ -1251,7 +1251,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- * 
+ *
  * @APPLE_LICENSE_HEADER_END@
  */
 #include <sys/errno.h>
@@ -1261,7 +1261,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
+ *
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
@@ -1270,10 +1270,10 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
- * 
+ *
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
+ *
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
@@ -1281,7 +1281,7 @@ APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf,
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- * 
+ *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
@@ -1484,7 +1484,7 @@ __END_DECLS
 #define	ECANCELED	89		/* Operation canceled */
 
 #define EIDRM		90		/* Identifier removed */
-#define ENOMSG		91		/* No message of desired type */   
+#define ENOMSG		91		/* No message of desired type */
 #define EILSEQ		92		/* Illegal byte sequence */
 #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define ENOATTR		93		/* Attribute not found */
diff --git a/doc/errno.list.solaris.txt b/doc/legacy/errno.list.solaris.txt
index 23601e9d3..23601e9d3 100644
--- a/doc/errno.list.solaris.txt
+++ b/doc/legacy/errno.list.solaris.txt
diff --git a/doc/user-guide/fdl.texi b/doc/legacy/fdl.texi
index e33c687cd..e33c687cd 100644
--- a/doc/user-guide/fdl.texi
+++ b/doc/legacy/fdl.texi
diff --git a/doc/user-guide/fuse.odg b/doc/legacy/fuse.odg
index 61bd103c7..61bd103c7 100644
--- a/doc/user-guide/fuse.odg
+++ b/doc/legacy/fuse.odg
diff --git a/doc/user-guide/fuse.pdf b/doc/legacy/fuse.pdf
index a7d13faff..a7d13faff 100644
--- a/doc/user-guide/fuse.pdf
+++ b/doc/legacy/fuse.pdf
diff --git a/doc/get_put_api_using_xattr.txt b/doc/legacy/get_put_api_using_xattr.txt
index 58951f5bf..243f9f1ae 100644
--- a/doc/get_put_api_using_xattr.txt
+++ b/doc/legacy/get_put_api_using_xattr.txt
@@ -16,7 +16,7 @@ internals:
 * posix handling setxattr/getxattr
   - setxattr
      posix setxattr does a open with O_CREAT|O_TRUNC on the <path>/<name>, writes value of the setxattr as data into the file and closes the file. when data is null, posix setxattr avoids doing write. file is closed after write.
-     
+
   - getxattr
      posix getxattr does open with O_RDONLY on the <path>/<name>, reads the complete content of the file. file is closed after read.
 
diff --git a/doc/user-guide/ha.odg b/doc/legacy/ha.odg
index e4b8b72d0..e4b8b72d0 100644
--- a/doc/user-guide/ha.odg
+++ b/doc/legacy/ha.odg
diff --git a/doc/user-guide/ha.pdf b/doc/legacy/ha.pdf
index e372c0ab0..e372c0ab0 100644
--- a/doc/user-guide/ha.pdf
+++ b/doc/legacy/ha.pdf
diff --git a/doc/hacker-guide/Makefile.am b/doc/legacy/hacker-guide/Makefile.am
index 65c92ac23..65c92ac23 100644
--- a/doc/hacker-guide/Makefile.am
+++ b/doc/legacy/hacker-guide/Makefile.am
diff --git a/doc/hacker-guide/call-stub.txt b/doc/legacy/hacker-guide/call-stub.txt
index bca1579b2..021037a35 100644
--- a/doc/hacker-guide/call-stub.txt
+++ b/doc/legacy/hacker-guide/call-stub.txt
@@ -2,8 +2,8 @@ creating a call stub and pausing a call
 ---------------------------------------
 libglusterfs provides seperate API to pause each of the fop. parameters to each API is
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
-       NOTE: @fn should exactly take the same type and number of parameters that 
+@fn    - procedure to call during call_resume().
+       NOTE: @fn should exactly take the same type and number of parameters that
              the corresponding regular fop takes.
 rest will be the regular parameters to corresponding fop.
 
@@ -17,7 +17,7 @@ specific parameters.
 here is the list of stub creation APIs for xlator fops.
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @loc       - pointer to location structure.
              NOTE: @loc will be copied to a different location, with inode_ref() to
 	           @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -30,7 +30,7 @@ fop_lookup_stub (call_frame_t *frame,
 		 int32_t need_xattr);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -41,7 +41,7 @@ fop_stat_stub (call_frame_t *frame,
 	       loc_t *loc);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @fd    - file descriptor parameter to lk fop.
          NOTE: @fd is stored with a fd_ref().
 call_stub_t *
@@ -50,7 +50,7 @@ fop_fstat_stub (call_frame_t *frame,
 		fd_t *fd);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to @loc->inode and
 	       @loc->parent, if not NULL. also @loc->path will be copied to a different location.
@@ -62,7 +62,7 @@ fop_chmod_stub (call_frame_t *frame,
 		mode_t mode);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @fd    - file descriptor parameter to lk fop.
          NOTE: @fd is stored with a fd_ref().
 @mode  - mode parameter for fchmod fop.
@@ -73,7 +73,7 @@ fop_fchmod_stub (call_frame_t *frame,
 		 mode_t mode);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to @loc->inode and
 	       @loc->parent, if not NULL. also @loc->path will be copied to a different location.
@@ -87,7 +87,7 @@ fop_chown_stub (call_frame_t *frame,
 		gid_t gid);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @fd    - file descriptor parameter to lk fop.
          NOTE: @fd is stored with a fd_ref().
 @uid   - uid parameter to fchown.
@@ -100,7 +100,7 @@ fop_fchown_stub (call_frame_t *frame,
 		 gid_t gid);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -113,7 +113,7 @@ fop_truncate_stub (call_frame_t *frame,
 		   off_t off);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @fd    - file descriptor parameter to lk fop.
          NOTE: @fd is stored with a fd_ref().
 @off   - offset parameter to ftruncate fop.
@@ -124,7 +124,7 @@ fop_ftruncate_stub (call_frame_t *frame,
 		    off_t off);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -137,7 +137,7 @@ fop_utimens_stub (call_frame_t *frame,
 		  struct timespec tv[2]);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -150,7 +150,7 @@ fop_access_stub (call_frame_t *frame,
 		 int32_t mask);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -163,7 +163,7 @@ fop_readlink_stub (call_frame_t *frame,
 		   size_t size);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -178,7 +178,7 @@ fop_mknod_stub (call_frame_t *frame,
 		dev_t rdev);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -191,7 +191,7 @@ fop_mkdir_stub (call_frame_t *frame,
 		mode_t mode);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -202,7 +202,7 @@ fop_unlink_stub (call_frame_t *frame,
 		 loc_t *loc);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -213,7 +213,7 @@ fop_rmdir_stub (call_frame_t *frame,
 		loc_t *loc);
 
 @frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
+@fn       - procedure to call during call_resume().
 @linkname - linkname parameter to symlink fop.
 @loc      - pointer to location structure.
             NOTE: @loc will be copied to a different location, with inode_ref() to
@@ -226,10 +226,10 @@ fop_symlink_stub (call_frame_t *frame,
 		  loc_t *loc);
 
 @frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
+@fn       - procedure to call during call_resume().
 @oldloc   - pointer to location structure.
-            NOTE: @oldloc will be copied to a different location, with inode_ref() to 
-	          @oldloc->inode and @oldloc->parent, if not NULL. also @oldloc->path will 
+            NOTE: @oldloc will be copied to a different location, with inode_ref() to
+	          @oldloc->inode and @oldloc->parent, if not NULL. also @oldloc->path will
 		  be copied to a different location, if not NULL.
 @newloc   - pointer to location structure.
             NOTE: @newloc will be copied to a different location, with inode_ref() to
@@ -242,7 +242,7 @@ fop_rename_stub (call_frame_t *frame,
 		 loc_t *newloc);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc     - pointer to location structure.
            NOTE: @loc will be copied to a different location, with inode_ref() to
 	         @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -255,7 +255,7 @@ fop_link_stub (call_frame_t *frame,
 	       const char *newpath);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -272,7 +272,7 @@ fop_create_stub (call_frame_t *frame,
 		 mode_t mode, fd_t *fd);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @flags - flags parameter to open fop.
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
@@ -286,7 +286,7 @@ fop_open_stub (call_frame_t *frame,
 	       fd_t *fd);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @fd    - file descriptor parameter to lk fop.
          NOTE: @fd is stored with a fd_ref().
 @size  - size parameter to readv fop.
@@ -299,10 +299,10 @@ fop_readv_stub (call_frame_t *frame,
 		off_t off);
 
 @frame  - call frame which has to be used to resume the call at call_resume().
-@fn     - procedure to call during call_resume(). 
+@fn     - procedure to call during call_resume().
 @fd     - file descriptor parameter to lk fop.
           NOTE: @fd is stored with a fd_ref().
-@vector - vector parameter to writev fop. 
+@vector - vector parameter to writev fop.
 	  NOTE: @vector is iov_dup()ed while creating stub. and frame->root->req_refs
                 dictionary is dict_ref()ed.
 @count  - count parameter to writev fop.
@@ -316,7 +316,7 @@ fop_writev_stub (call_frame_t *frame,
 		 off_t off);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @fd    - file descriptor parameter to flush fop.
          NOTE: @fd is stored with a fd_ref().
 call_stub_t *
@@ -326,7 +326,7 @@ fop_flush_stub (call_frame_t *frame,
 
 
 @frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
+@fn       - procedure to call during call_resume().
 @fd       - file descriptor parameter to lk fop.
             NOTE: @fd is stored with a fd_ref().
 @datasync - datasync parameter to fsync fop.
@@ -337,7 +337,7 @@ fop_fsync_stub (call_frame_t *frame,
 		int32_t datasync);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to @loc->inode and
 	       @loc->parent, if not NULL. also @loc->path will be copied to a different location.
@@ -346,11 +346,11 @@ fop_fsync_stub (call_frame_t *frame,
 call_stub_t *
 fop_opendir_stub (call_frame_t *frame,
 		  fop_opendir_t fn,
-		  loc_t *loc, 
+		  loc_t *loc,
 		  fd_t *fd);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @fd    - file descriptor parameter to getdents fop.
          NOTE: @fd is stored with a fd_ref().
 @size  - size parameter to getdents fop.
@@ -365,7 +365,7 @@ fop_getdents_stub (call_frame_t *frame,
 		   int32_t flag);
 
 @frame   - call frame which has to be used to resume the call at call_resume().
-@fn      - procedure to call during call_resume(). 
+@fn      - procedure to call during call_resume().
 @fd      - file descriptor parameter to setdents fop.
            NOTE: @fd is stored with a fd_ref().
 @flags   - flags parameter to setdents fop.
@@ -379,7 +379,7 @@ fop_setdents_stub (call_frame_t *frame,
 		   int32_t count);
 
 @frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
+@fn       - procedure to call during call_resume().
 @fd       - file descriptor parameter to setdents fop.
             NOTE: @fd is stored with a fd_ref().
 @datasync - datasync parameter to fsyncdir fop.
@@ -390,7 +390,7 @@ fop_fsyncdir_stub (call_frame_t *frame,
 		   int32_t datasync);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -401,9 +401,9 @@ fop_statfs_stub (call_frame_t *frame,
 		 loc_t *loc);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
-         NOTE: @loc will be copied to a different location, with inode_ref() to 
+         NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
 	       copied to a different location.
 @dict  - dict parameter to setxattr fop.
@@ -416,7 +416,7 @@ fop_setxattr_stub (call_frame_t *frame,
 		   int32_t flags);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -429,7 +429,7 @@ fop_getxattr_stub (call_frame_t *frame,
 		   const char *name);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -443,7 +443,7 @@ fop_removexattr_stub (call_frame_t *frame,
 		      const char *name);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @fd    - file descriptor parameter to lk fop.
          NOTE: @fd is stored with a fd_ref().
 @cmd   - command parameter to lk fop.
@@ -457,13 +457,13 @@ fop_lk_stub (call_frame_t *frame,
 	     struct flock *lock);
 
 @frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
+@fn       - procedure to call during call_resume().
 @fd       - fd parameter to gf_lk fop.
 	    NOTE: @fd is fd_ref()ed while creating stub, if not NULL.
 @cmd      - cmd parameter to gf_lk fop.
 @lock     - lock paramater to gf_lk fop.
 	    NOTE: @lock is copied to a different memory location while creating
-	          stub. 
+	          stub.
 call_stub_t *
 fop_gf_lk_stub (call_frame_t *frame,
 		fop_gf_lk_t fn,
@@ -472,7 +472,7 @@ fop_gf_lk_stub (call_frame_t *frame,
 		struct flock *lock);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @fd    - file descriptor parameter to readdir fop.
          NOTE: @fd is stored with a fd_ref().
 @size  - size parameter to readdir fop.
@@ -485,7 +485,7 @@ fop_readdir_stub (call_frame_t *frame,
 		  off_t off);
 
 @frame - call frame which has to be used to resume the call at call_resume().
-@fn    - procedure to call during call_resume(). 
+@fn    - procedure to call during call_resume().
 @loc   - pointer to location structure.
          NOTE: @loc will be copied to a different location, with inode_ref() to
 	       @loc->inode and @loc->parent, if not NULL. also @loc->path will be
@@ -498,7 +498,7 @@ fop_checksum_stub (call_frame_t *frame,
 		   int32_t flags);
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @inode     - inode parameter to @fn.
@@ -516,7 +516,7 @@ fop_lookup_cbk_stub (call_frame_t *frame,
 		     struct stat *buf,
 		     dict_t *dict);
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -529,7 +529,7 @@ fop_stat_cbk_stub (call_frame_t *frame,
 		   struct stat *buf);
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -542,7 +542,7 @@ fop_fstat_cbk_stub (call_frame_t *frame,
 		    struct stat *buf);
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -555,7 +555,7 @@ fop_chmod_cbk_stub (call_frame_t *frame,
 		    struct stat *buf);
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -568,7 +568,7 @@ fop_fchmod_cbk_stub (call_frame_t *frame,
 		     struct stat *buf);
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -582,7 +582,7 @@ fop_chown_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -596,7 +596,7 @@ fop_fchown_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -610,7 +610,7 @@ fop_truncate_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -624,7 +624,7 @@ fop_ftruncate_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -638,7 +638,7 @@ fop_utimens_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 call_stub_t *
@@ -649,7 +649,7 @@ fop_access_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @path      - path parameter to @fn.
@@ -663,7 +663,7 @@ fop_readlink_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @inode     - inode parameter to @fn.
@@ -680,7 +680,7 @@ fop_mknod_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @inode     - inode parameter to @fn.
@@ -697,7 +697,7 @@ fop_mkdir_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 call_stub_t *
@@ -708,7 +708,7 @@ fop_unlink_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 call_stub_t *
@@ -719,7 +719,7 @@ fop_rmdir_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @inode     - inode parameter to @fn.
@@ -736,7 +736,7 @@ fop_symlink_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -750,7 +750,7 @@ fop_rename_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @inode     - inode parameter to @fn.
@@ -766,7 +766,7 @@ fop_link_cbk_stub (call_frame_t *frame,
 		   struct stat *buf);
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @fd        - fd parameter to @fn.
@@ -786,7 +786,7 @@ fop_create_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @fd        - fd parameter to @fn.
@@ -800,10 +800,10 @@ fop_open_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
-@vector    - vector parameter to @fn.	
+@vector    - vector parameter to @fn.
 	     NOTE: @vector is copied to a different memory location, if not NULL. also
 	           frame->root->rsp_refs is dict_ref()ed.
 @stbuf     - stbuf parameter to @fn.
@@ -819,7 +819,7 @@ fop_readv_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @stbuf     - stbuf parameter to @fn.
@@ -833,7 +833,7 @@ fop_writev_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 call_stub_t *
@@ -843,7 +843,7 @@ fop_flush_cbk_stub (call_frame_t *frame,
 		    int32_t op_errno);
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 call_stub_t *
@@ -854,7 +854,7 @@ fop_fsync_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @fd        - fd parameter to @fn.
@@ -868,7 +868,7 @@ fop_opendir_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @entries   - entries parameter to @fn.
@@ -883,7 +883,7 @@ fop_getdents_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 call_stub_t *
@@ -893,7 +893,7 @@ fop_setdents_cbk_stub (call_frame_t *frame,
 		       int32_t op_errno);
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 call_stub_t *
@@ -904,7 +904,7 @@ fop_fsyncdir_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @buf       - buf parameter to @fn.
@@ -918,7 +918,7 @@ fop_statfs_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 call_stub_t *
@@ -929,7 +929,7 @@ fop_setxattr_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 @value     - value dictionary parameter to @fn.
@@ -943,7 +943,7 @@ fop_getxattr_cbk_stub (call_frame_t *frame,
 
 
 @frame     - call frame which has to be used to resume the call at call_resume().
-@fn        - procedure to call during call_resume(). 
+@fn        - procedure to call during call_resume().
 @op_ret    - op_ret parameter to @fn.
 @op_errno  - op_errno parameter to @fn.
 call_stub_t *
@@ -954,12 +954,12 @@ fop_removexattr_cbk_stub (call_frame_t *frame,
 
 
 @frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
+@fn       - procedure to call during call_resume().
 @op_ret   - op_ret parameter to @fn.
 @op_errno - op_errno parameter to @fn.
 @lock     - lock parameter to @fn.
 	    NOTE: @lock is copied to a different memory location while creating
-	          stub. 
+	          stub.
 call_stub_t *
 fop_lk_cbk_stub (call_frame_t *frame,
 		 fop_lk_cbk_t fn,
@@ -968,12 +968,12 @@ fop_lk_cbk_stub (call_frame_t *frame,
 		 struct flock *lock);
 
 @frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
+@fn       - procedure to call during call_resume().
 @op_ret   - op_ret parameter to @fn.
 @op_errno - op_errno parameter to @fn.
 @lock     - lock parameter to @fn.
 	    NOTE: @lock is copied to a different memory location while creating
-	          stub. 
+	          stub.
 call_stub_t *
 fop_gf_lk_cbk_stub (call_frame_t *frame,
 		    fop_gf_lk_cbk_t fn,
@@ -983,7 +983,7 @@ fop_gf_lk_cbk_stub (call_frame_t *frame,
 
 
 @frame    - call frame which has to be used to resume the call at call_resume().
-@fn       - procedure to call during call_resume(). 
+@fn       - procedure to call during call_resume().
 @op_ret   - op_ret parameter to @fn.
 @op_errno - op_errno parameter to @fn.
 @entries  - entries parameter to @fn.
@@ -996,14 +996,14 @@ fop_readdir_cbk_stub (call_frame_t *frame,
 
 
 @frame         - call frame which has to be used to resume the call at call_resume().
-@fn            - procedure to call during call_resume(). 
+@fn            - procedure to call during call_resume().
 @op_ret        - op_ret parameter to @fn.
 @op_errno      - op_errno parameter to @fn.
 @file_checksum - file_checksum parameter to @fn.
-                 NOTE: file_checksum will be copied to a different memory location 
+                 NOTE: file_checksum will be copied to a different memory location
 		       while creating stub.
 @dir_checksum  - dir_checksum parameter to @fn.
-                 NOTE: file_checksum will be copied to a different memory location 
+                 NOTE: file_checksum will be copied to a different memory location
 		       while creating stub.
 call_stub_t *
 fop_checksum_cbk_stub (call_frame_t *frame,
@@ -1025,9 +1025,9 @@ resuming a call:
         in  stub->args.<operation>.<fd_t-or-inode_t-or-dict_t>. so, if any fd_t, dict_t or
 	inode_t pointers are assigned at stub->args.<operation>.<fd_t-or-inode_t-or-dict_t> after
 	fop_<operation>_stub() call, they must be <fd_t-or-inode_t-or-dict_t>_ref()ed.
-	
+
 	call_resume does not STACK_DESTROY() for any fop.
-	
+
   if stub->fn is NULL, call_resume does STACK_WIND() or STACK_UNWIND() using the stub->frame.
 
   return - call resume fails only if stub is NULL. call resume fails with errno set to EINVAL.
diff --git a/doc/hacker-guide/hacker-guide.tex b/doc/legacy/hacker-guide/hacker-guide.tex
index c2d7255d7..11101e7a8 100644
--- a/doc/hacker-guide/hacker-guide.tex
+++ b/doc/legacy/hacker-guide/hacker-guide.tex
@@ -31,7 +31,7 @@ most part.
 
 \chapter{Major components}
 \section{libglusterfs}
-\texttt{libglusterfs} contains supporting code used by all the other components. 
+\texttt{libglusterfs} contains supporting code used by all the other components.
 The important files here are:
 
 \texttt{dict.c}: This is an implementation of a serializable dictionary type. It is
@@ -165,8 +165,8 @@ First we include the requisite headers.
 #include "logging.h"
 
 /*
- * This is a rot13 ``encryption'' xlator. It rot13's data when 
- * writing to disk and rot13's it back when reading it. 
+ * This is a rot13 ``encryption'' xlator. It rot13's data when
+ * writing to disk and rot13's it back when reading it.
  * This xlator is meant as an example, not for production
  *  use ;) (hence no error-checking)
  */
@@ -178,7 +178,7 @@ letters. Any other byte is passed through as it is.
 
 \begin{verbatim}
 /* We only handle lower case letters for simplicity */
-static void 
+static void
 rot13 (char *buf, int len)
 {
   int i;
@@ -252,12 +252,12 @@ rot13_writev (call_frame_t *frame,
               xlator_t *this,
               dict_t *ctx,
               struct iovec *vector,
-              int32_t count, 
+              int32_t count,
               off_t offset)
 {
   rot13_iovec (vector, count);
 
-  STACK_WIND (frame, 
+  STACK_WIND (frame,
               rot13_writev_cbk,
               FIRST_CHILD (this),
               FIRST_CHILD (this)->fops->writev,
@@ -267,9 +267,9 @@ rot13_writev (call_frame_t *frame,
 
 \end{verbatim}
 
-Every xlator must define two functions and two external symbols. The functions are 
+Every xlator must define two functions and two external symbols. The functions are
 \texttt{init} and \texttt{fini}, and the symbols are \texttt{fops} and \texttt{mops}.
-The \texttt{init} function is called when the xlator is loaded by GlusterFS, and 
+The \texttt{init} function is called when the xlator is loaded by GlusterFS, and
 contains code for the xlator to initialize itself. Note that if an xlator is present
 multiple times in the spec tree, the \texttt{init} function will be called each time
 the xlator is loaded.
@@ -279,7 +279,7 @@ int32_t
 init (xlator_t *this)
 {
   if (!this->children) {
-    gf_log ("rot13", GF_LOG_ERROR, 
+    gf_log ("rot13", GF_LOG_ERROR,
             "FATAL: rot13 should have exactly one child");
     return -1;
   }
@@ -291,7 +291,7 @@ init (xlator_t *this)
 
 \begin{verbatim}
 
-void 
+void
 fini (xlator_t *this)
 {
   return;
diff --git a/doc/hacker-guide/replicate.txt b/doc/legacy/hacker-guide/replicate.txt
index fd1ef2747..133c72afa 100644
--- a/doc/hacker-guide/replicate.txt
+++ b/doc/legacy/hacker-guide/replicate.txt
@@ -5,7 +5,7 @@
 Before understanding replicate, one must understand two internal FOPs:
 
 GF_FILE_LK:
-  This is exactly like fcntl(2) locking, except the locks are in a 
+  This is exactly like fcntl(2) locking, except the locks are in a
   separate domain from locks held by applications.
 
 GF_DIR_LK (loc_t *loc, char *basename):
@@ -17,7 +17,7 @@ GF_DIR_LK (loc_t *loc, char *basename):
   If one wishes to lock *all* the names under a particular directory,
   supply the basename argument as NULL.
 
-  The locks can either be read locks or write locks; consult the 
+  The locks can either be read locks or write locks; consult the
   function prototype for more details.
 
 Both these operations are implemented by the features/locks (earlier
@@ -79,7 +79,7 @@ Each of the four major groups has its own algorithm:
 
  All operations are done in parallel unless specified otherwise.
 
- (1) Send a GF_FILE_LK request on all children for a write lock on 
+ (1) Send a GF_FILE_LK request on all children for a write lock on
      the appropriate region
             (for metadata operations: entire file (0, 0)
              for writev: (offset, offset+size of buffer))
@@ -87,11 +87,11 @@ Each of the four major groups has its own algorithm:
      - If a lock request fails on a child:
          unlock all children
          try to acquire a blocking lock (F_SETLKW) on each child, serially.
-	 
+
 	 If this fails (due to ENOTCONN or EINVAL):
            Consider this child as dead for rest of transaction.
 
- (2) Mark all children as "pending" on all (alive) children 
+ (2) Mark all children as "pending" on all (alive) children
      (see below for meaning of "pending").
 
      - If it fails on any child:
@@ -105,7 +105,7 @@ Each of the four major groups has its own algorithm:
  (4) Unmark all successful children as not "pending" on all nodes.
 
  (5) Unlock region on all (alive) children.
-     
+
  -----------
  - dir-write
  -----------
@@ -121,11 +121,11 @@ Each of the four major groups has its own algorithm:
  The "pending" number is like a journal entry. A pending entry is an
  array of 32-bit integers stored in network byte-order as the extended
  attribute of an inode (which can be a directory as well).
- 
+
  There are three keys corresponding to three types of pending operations:
 
  - AFR_METADATA_PENDING
-     There are some metadata operations pending on this inode (perms, ctime/mtime, 
+     There are some metadata operations pending on this inode (perms, ctime/mtime,
      xattr, etc.).
 
  - AFR_DATA_PENDING
@@ -134,7 +134,7 @@ Each of the four major groups has its own algorithm:
  - AFR_ENTRY_PENDING
      There are some directory operations pending on this directory
      (create, unlink, etc.).
-    
+
 -----------
 * Self heal
 -----------
@@ -155,7 +155,7 @@ Each of the four major groups has its own algorithm:
      other is directory):
      - Announce to the user via log that a split-brain situation has been
        detected, and do nothing.
- 
+
  - On open, gather extended attribute data:
    - Consider the file with the highest AFR_DATA_PENDING number as
      the definitive one and replicate its contents on all other
@@ -190,7 +190,7 @@ Thus, if lookup on c1 returns an inode number "2", it is scaled to "4"
 This way we ensure that there is never a collision of inode numbers from
 two different children.
 
-This reduction of inode space doesn't really reduce the usability of 
+This reduction of inode space doesn't really reduce the usability of
 replicate since even if we assume replicate has 1024 children (which would be a
 highly unusual scenario), each child still has a 54-bit inode space.
 
diff --git a/doc/handling-options.txt b/doc/legacy/handling-options.txt
index cac1fe939..9a3b2510a 100644
--- a/doc/handling-options.txt
+++ b/doc/legacy/handling-options.txt
@@ -2,12 +2,12 @@
 How to add a new option to a given volume ?
 ===========================================
 
-* Add a entry in 'struct volume_options options[]' with your key, what is 
+* Add a entry in 'struct volume_options options[]' with your key, what is
   the type of the 'key', etc.
 
-* The 'key' and corresponding 'value' given for the same by user are validated 
+* The 'key' and corresponding 'value' given for the same by user are validated
   before calling init() of the translator/transport/scheduler/auth-module.
 
-* Once the complete init() is successful, user will get a warning if he has 
+* Once the complete init() is successful, user will get a warning if he has
   given a 'key' which is not defined in these modules.
 
diff --git a/doc/mac-related-xattrs.txt b/doc/legacy/mac-related-xattrs.txt
index 805658334..92bb2ceef 100644
--- a/doc/mac-related-xattrs.txt
+++ b/doc/legacy/mac-related-xattrs.txt
@@ -1,21 +1,21 @@
 
-This document is intended to briefly explain how the Extended Attributes on 
+This document is intended to briefly explain how the Extended Attributes on
 Darwin 10.5.x releases works
 ----
 
-On Darwin other than all the normal filesystem operations, 'Finder' (like 
-Explorer in Windows but a little more) keeps its information in two extended 
-attributes named 'com.apple.FinderInfo' and 'com.apple.ResourceFork'. If these 
-xattrs are not implemented the filesystem won't be shown on Finder, and if they 
-are not implemented properly there may be issues when some of the file operations 
-are done through GUI of Finder. But when a filesystem is used over mountpoint in a 
-terminal, everything is fine and these xattrs are not required. 
+On Darwin other than all the normal filesystem operations, 'Finder' (like
+Explorer in Windows but a little more) keeps its information in two extended
+attributes named 'com.apple.FinderInfo' and 'com.apple.ResourceFork'. If these
+xattrs are not implemented the filesystem won't be shown on Finder, and if they
+are not implemented properly there may be issues when some of the file operations
+are done through GUI of Finder. But when a filesystem is used over mountpoint in a
+terminal, everything is fine and these xattrs are not required.
 
-Currently the way these xattrs are implemented is simple. All the xattr calls 
+Currently the way these xattrs are implemented is simple. All the xattr calls
 (getxattr, setxattr, listxattr, removexattr) are passed down to underlaying filesystem,
 most of the cases when exported FS is on MacOS X itself, these keys are supported, hence
-the fops succeed. But in the case of using exports of different OS on Darwin the issue is 
-extended attribute prefix like 'com.apple.' may not be supported, hence the problem with 
+the fops succeed. But in the case of using exports of different OS on Darwin the issue is
+extended attribute prefix like 'com.apple.' may not be supported, hence the problem with
 Finder. To solve this issue, GlusterFS returns virtual default values to these keys, which
 works fine on most of the cases.
 
diff --git a/doc/porting_guide.txt b/doc/legacy/porting_guide.txt
index 905bb4228..5705cd964 100644
--- a/doc/porting_guide.txt
+++ b/doc/legacy/porting_guide.txt
@@ -3,7 +3,7 @@
 
 * General setup
 
-The configure script will detect the target platform for the build. 
+The configure script will detect the target platform for the build.
 All platform-specific CFLAGS, macro definitions should be done
 in configure.ac
 
@@ -15,11 +15,11 @@ Platform-specific code can be written like this:
 
 * Coding guidelines
 
-In general, avoid glibc extensions. For example, nested functions don't work 
+In general, avoid glibc extensions. For example, nested functions don't work
 on Mac OS X. It is best to stick to C99.
 
 When using library calls and system calls, pay attention to the
-portability notes. As far as possible stick to POSIX-specified behavior. 
+portability notes. As far as possible stick to POSIX-specified behavior.
 Do not use anything expressly permitted by the specification. For example,
 some fields in structures may be present only on certain platforms. Avoid
 use of such things.
@@ -27,14 +27,14 @@ use of such things.
 Do not pass values of constants such as F_*, O_*, errno values, etc. across
 platforms.
 
-Please refer compat-errno.h for more details about errno handling inside 
-glusterfs for cross platform. 
+Please refer compat-errno.h for more details about errno handling inside
+glusterfs for cross platform.
 
 * Specific issues
 
 - The argp library is available only on Linux through glibc, but for other
   platforms glusterfs has already included argp-standalone library which will
-  statically linked during the glusterfs build. 
+  statically linked during the glusterfs build.
 
 - Extended attribute calls (setxattr, listxattr, etc.) have differing prototypes
   on different platforms. See compat.h for macro definitions to resolve this, also
diff --git a/doc/replicate.lyx b/doc/legacy/replicate.lyx
index d11a92bee..58ba6b2e0 100644
--- a/doc/replicate.lyx
+++ b/doc/legacy/replicate.lyx
@@ -36,7 +36,7 @@ Automatic File Replication (replicate) in GlusterFS
 \end_layout
 
 \begin_layout Author
-Vikas Gorur 
+Vikas Gorur
 \family typewriter
 \size larger
 <vikas@gluster.com>
@@ -77,7 +77,7 @@ The replicate translator of GlusterFS aims to keep identical copies of a file
 \end_layout
 
 \begin_layout Standard
-In the rest of the document the terms 
+In the rest of the document the terms
 \begin_inset Quotes eld
 \end_inset
 
@@ -85,7 +85,7 @@ subvolume
 \begin_inset Quotes erd
 \end_inset
 
- and 
+ and
 \begin_inset Quotes eld
 \end_inset
 
@@ -167,7 +167,7 @@ end{verbatim}
 
 \begin_layout Standard
 This defines an replicate volume with two subvolumes, brick1, and brick2.
- For replicate to work properly, it is essential that its subvolumes support 
+ For replicate to work properly, it is essential that its subvolumes support
 \series bold
 extended attributes
 \series default
@@ -177,7 +177,7 @@ extended attributes
 \end_layout
 
 \begin_layout Standard
-The storage volumes used as backend for replicate 
+The storage volumes used as backend for replicate
 \emph on
 must
 \emph default
@@ -262,7 +262,7 @@ replicate divides all filesystem write operations into three classes:
 \begin_layout Itemize
 
 \series bold
-data: 
+data:
 \series default
 Operations that modify the contents of a file (write, truncate).
 \end_layout
@@ -270,7 +270,7 @@ Operations that modify the contents of a file (write, truncate).
 \begin_layout Itemize
 
 \series bold
-metadata: 
+metadata:
 \series default
 Operations that modify attributes of a file or directory (permissions, ownership
 , etc.).
@@ -279,7 +279,7 @@ Operations that modify attributes of a file or directory (permissions, ownership
 \begin_layout Itemize
 
 \series bold
-entry: 
+entry:
 \series default
 Operations that create or delete directory entries (mkdir, create, rename,
  rmdir, unlink, etc.).
@@ -345,7 +345,7 @@ Self-Heal
 \begin_layout Standard
 replicate automatically tries to fix any inconsistencies it detects among different
  copies of a file.
- It uses information in the change log to determine which copy is the 
+ It uses information in the change log to determine which copy is the
 \begin_inset Quotes eld
 \end_inset
 
@@ -357,7 +357,7 @@ correct
 \end_layout
 
 \begin_layout Standard
-Self-heal is triggered when a file or directory is first 
+Self-heal is triggered when a file or directory is first
 \begin_inset Quotes eld
 \end_inset
 
@@ -374,7 +374,7 @@ If the entry being accessed is a directory:
 \end_layout
 
 \begin_layout Itemize
-The contents of the 
+The contents of the
 \begin_inset Quotes eld
 \end_inset
 
@@ -412,7 +412,7 @@ It may happen that one replicate client can access only some of the servers in
  a cluster and another replicate client can access the remaining servers.
  Or it may happen that in a cluster of two servers, one server goes down
  and comes back up, but the other goes down immediately.
- Both these scenarios result in a 
+ Both these scenarios result in a
 \begin_inset Quotes eld
 \end_inset
 
@@ -425,7 +425,7 @@ split-brain
 
 \begin_layout Standard
 In a split-brain situation, there will be two or more copies of a file,
- all of which are 
+ all of which are
 \begin_inset Quotes eld
 \end_inset
 
@@ -484,12 +484,12 @@ split-brain
 ).
  This means if a discrepancy is noticed in the attributes or content of
  a file, the copy on the `favorite-child' will be considered the definitive
- version and its contents will 
+ version and its contents will
 \emph on
-overwrite 
+overwrite
 \emph default
 the contents of all other copies.
- Use this option with caution! It is possible to 
+ Use this option with caution! It is possible to
 \emph on
 lose data
 \emph default
@@ -502,7 +502,7 @@ Self-heal options
 \end_layout
 
 \begin_layout Standard
-Setting any of these options to 
+Setting any of these options to
 \begin_inset Quotes eld
 \end_inset
 
@@ -549,7 +549,7 @@ If any of these options is turned off, it disables writing of change log
  entries for that class of file operations.
  That is, steps 2 and 4 of the write algorithm (see above) are not done.
  Note that if the change log is not written, the self-heal algorithm cannot
- determine the 
+ determine the
 \begin_inset Quotes eld
 \end_inset
 
@@ -557,7 +557,7 @@ correct
 \begin_inset Quotes erd
 \end_inset
 
- version of a file and hence self-heal will only be able to fix 
+ version of a file and hence self-heal will only be able to fix
 \begin_inset Quotes eld
 \end_inset
 
@@ -602,7 +602,7 @@ These options let you specify the number of lock servers to use for each
  The default values are satisfactory in most cases.
  If you are extra paranoid, you may want to increase the values.
  However, be very cautious if you set the data- or entry- lock server counts
- to zero, since this can result in 
+ to zero, since this can result in
 \emph on
 lost data.
 
@@ -610,11 +610,11 @@ lost data.
  For example, if you set the data-lock-server-count to zero, and two application
 s write to the same region of a file, there is a possibility that none of
  your servers will have all the data.
- In other words, the copies will be 
+ In other words, the copies will be
 \emph on
 inconsistent
 \emph default
-, and 
+, and
 \emph on
 incomplete
 \emph default
diff --git a/doc/replicate.pdf b/doc/legacy/replicate.pdf
index b7212af2b..b7212af2b 100644
--- a/doc/replicate.pdf
+++ b/doc/legacy/replicate.pdf
diff --git a/doc/solaris-related-xattrs.txt b/doc/legacy/solaris-related-xattrs.txt
index e26efa5d1..3a4643948 100644
--- a/doc/solaris-related-xattrs.txt
+++ b/doc/legacy/solaris-related-xattrs.txt
@@ -1,42 +1,42 @@
                    Solaris Extended Attributes
 
 In solaris extended attributes are logically supported as files
-within the filesystem.  The file system is therefore augmented 
+within the filesystem.  The file system is therefore augmented
 with an orthogonal namespace of file attributes. Attribute values
 are accessed by file descriptors obtained through a special attribute
-interface. This type of logical view of "attributes as files" allows 
-the leveraging of existing file system interface functionality to 
-support the construction, deletion and manipulation of attributes. 
+interface. This type of logical view of "attributes as files" allows
+the leveraging of existing file system interface functionality to
+support the construction, deletion and manipulation of attributes.
 
 But as we have tested through this functionality provided by Solaris
 we have come accross two major issues as written below.
 
-1. Symlink XATTR_NOFOLLOW not present for creating extended attributes 
+1. Symlink XATTR_NOFOLLOW not present for creating extended attributes
    directly on the symlinks like other platforms Linux,MAC-OSX,BSD etc.
-   An implementation is present for O_NOFOLLOW for "openat()" call sets 
-   up errno ELOOP whenever encountered with a symlink and also another 
+   An implementation is present for O_NOFOLLOW for "openat()" call sets
+   up errno ELOOP whenever encountered with a symlink and also another
    implementation AT_SYMLINK_NOFOLLOW which is not present for calls like
    "attropen(), openat()"
 
    a snippet of test code which helped us understand this behaviour
    --------------------------------------
-    attrfd = attropen (path, key, 
+    attrfd = attropen (path, key,
     flags|AT_SYMLINK_NOFOLLOW|O_CREAT|O_WRONLY|O_NOFOLLOW, 0777);
     if (attrfd >= 0) {
       ftruncate (attrfd, 0);
       ret = write (attrfd, value, size);
       close (attrfd);
     } else {
-       fprintf (stderr, "Couldn't set extended attribute for %s (%d)\n", 
+       fprintf (stderr, "Couldn't set extended attribute for %s (%d)\n",
        	        path, errno);
-    }		
+    }
     --------------------------------------
 
 2. Extended attribute support for special files like device files, fifo files
-   is not supported under solaris. 
+   is not supported under solaris.
 
 Apart from these glitches almost everything regarding porting functionality
-for extended attribute calls has been properly implemented in compat.c 
+for extended attribute calls has been properly implemented in compat.c
 with writing wrapper around functions over
 "attropen()", "openat()", "unlinkat()"
 
diff --git a/doc/stat-prefetch-design.txt b/doc/legacy/stat-prefetch-design.txt
index 06d0ad37e..68ed423d3 100644
--- a/doc/stat-prefetch-design.txt
+++ b/doc/legacy/stat-prefetch-design.txt
@@ -1,64 +1,64 @@
 what is stat-prefetch?
 ======================
 It is a translator which caches the dentries read in readdir. This dentry
-list is stored in the context of fd. Later when lookup happens on 
+list is stored in the context of fd. Later when lookup happens on
 [parent-inode, basename (path)] combination, this list is searched for the
 basename. The dentry thus searched is used to fill up the stat corresponding
 to path being looked upon, thereby short-cutting lookup calls. This cache is
-preserved till closedir is called on the fd. The purpose of this translator 
-is to optimize operations like 'ls -l', where a readdir is followed by 
+preserved till closedir is called on the fd. The purpose of this translator
+is to optimize operations like 'ls -l', where a readdir is followed by
 lookup (stat) calls on each directory entry.
 
-1. stat-prefetch harnesses the efficiency of short lookup calls 
-   (saves network roundtrip time for lookup calls from being accounted to 
+1. stat-prefetch harnesses the efficiency of short lookup calls
+   (saves network roundtrip time for lookup calls from being accounted to
    the stat call).
-2. To maintain the correctness, it does lookup-behind - lookup is winded to 
-   underlying translators after it is unwound to upper translators. 
+2. To maintain the correctness, it does lookup-behind - lookup is winded to
+   underlying translators after it is unwound to upper translators.
    lookup-behind is necessary as inode gets populated in server inode table
-   only in lookup-cbk and also because various translators store their 
+   only in lookup-cbk and also because various translators store their
    contexts in inode contexts during lookup calls.
 
 fops to be implemented:
 =======================
 * lookup
-  1. check the dentry cache stored in context of fds opened by the same process 
+  1. check the dentry cache stored in context of fds opened by the same process
      on parent inode for basename. If found unwind with cached stat, else wind
-     the lookup call to underlying translators. 
-  2. stat is stored in the context of inode if the path being looked upon 
+     the lookup call to underlying translators.
+  2. stat is stored in the context of inode if the path being looked upon
      happens to be directory. This stat will be used to fill postparent stat
      when lookup happens on any of the directory contents.
 
 * readdir
   1. cache the direntries returned in readdir_cbk in the context of fd.
-  2. if the readdir is happening on non-expected offsets (means a seekdir/rewinddir 
+  2. if the readdir is happening on non-expected offsets (means a seekdir/rewinddir
      has happened), cache has to be flushed.
-  3. delete the entry corresponding to basename of path on which fd is opened 
+  3. delete the entry corresponding to basename of path on which fd is opened
      from cache stored in parent.
 
 * chmod/fchmod
   delete the entry corresponding to basename from cache stored in context of
-  fds opened on parent inode, since these calls change st_mode and st_ctime of 
+  fds opened on parent inode, since these calls change st_mode and st_ctime of
   stat.
- 
+
 * chown/fchown
-  delete the entry corresponding to basename from cache stored in context of 
-  fds opened on parent inode, since these calls change st_uid/st_gid and 
+  delete the entry corresponding to basename from cache stored in context of
+  fds opened on parent inode, since these calls change st_uid/st_gid and
   st_ctime of stat.
 
 * truncate/ftruncate
-  delete the entry corresponding to basename from cache stored in context of 
+  delete the entry corresponding to basename from cache stored in context of
   fds opened on parent inode, since these calls change st_size/st_mtime of stat.
 
 * utimens
-  delete the entry corresponding to basename from cache stored in context of 
+  delete the entry corresponding to basename from cache stored in context of
   fds opened on parent inode, since this call changes st_atime/st_mtime of stat.
 
 * readlink
   delete the entry corresponding to basename from cache stored in context of fds
   opened on parent inode, since this call changes st_atime of stat.
- 
+
 * unlink
-  1. delete the entry corresponding to basename from cache stored in context of 
+  1. delete the entry corresponding to basename from cache stored in context of
      fds, opened on parent directory containing the file being unlinked.
   2. delete the entry corresponding to basename of parent directory from cache
      of grand-parent.
@@ -66,14 +66,14 @@ fops to be implemented:
 * rmdir
   1. delete the entry corresponding to basename from cache stored in context of
      fds opened on parent inode.
-  2. remove the entire cache from all fds opened on inode corresponding to 
+  2. remove the entire cache from all fds opened on inode corresponding to
      directory being removed.
   3. delete the entry correspondig to basename of parent from cache stored in
      grand-parent.
 
 * readv
   delete the entry corresponding to basename from cache stored in context of fds
-  opened on parent inode, since readv changes st_atime of file. 
+  opened on parent inode, since readv changes st_atime of file.
 
 * writev
   delete the entry corresponding to basename from cache stored in context of fds
@@ -82,29 +82,29 @@ fops to be implemented:
 
 * fsync
   there is a confusion here as to whether fsync updates mtime/ctimes. Disk based
-  filesystems (atleast ext2) just writes the times stored in inode to disk 
-  during fsync and not the time at which fsync is being done. But in glusterfs, 
-  a translator like write-behind actually sends writes during fsync which will 
-  change mtime/ctime. Hence stat-prefetch implements fsync to delete the entry 
+  filesystems (atleast ext2) just writes the times stored in inode to disk
+  during fsync and not the time at which fsync is being done. But in glusterfs,
+  a translator like write-behind actually sends writes during fsync which will
+  change mtime/ctime. Hence stat-prefetch implements fsync to delete the entry
   corresponding to basename from cache stored in context of fds opened on parent
   inode.
- 
+
 * rename
-  1. remove entry corresponding to oldname from cache stored in fd contexts of 
+  1. remove entry corresponding to oldname from cache stored in fd contexts of
      oldparent.
   2. remove entry corresponding to newname from cache stored in fd contexts of
-     newparent. 
-  3. remove entry corresponding to oldparent from cache stored in 
+     newparent.
+  3. remove entry corresponding to oldparent from cache stored in
      old-grand-parent, since removing oldname changes st_mtime and st_ctime
      of oldparent stat.
-  4. remove entry corresponding to newparent from cache stored in 
+  4. remove entry corresponding to newparent from cache stored in
      new-grand-parent, since adding newname changes st_mtime and st_ctime
      of newparent stat.
-  5. if oldname happens to be a directory, remove entire cache from all fds 
+  5. if oldname happens to be a directory, remove entire cache from all fds
      opened on it.
 
 * create/mknod/mkdir/symlink/link
-  delete entry corresponding to basename of parent directory in which these 
+  delete entry corresponding to basename of parent directory in which these
   operations are happening, from cache stored in context of fds opened on
   grand-parent, since adding a new entry to a directory changes st_mtime
   and st_ctime of parent directory.
@@ -116,13 +116,13 @@ fops to be implemented:
 * setdents
   1. remove entry corresponding to basename of path on which fd is opened from
      cache stored in context of fds opened on parent.
-  2. for each of the entry in the direntry list, delete from cache stored in 
+  2. for each of the entry in the direntry list, delete from cache stored in
      context of fd, the entry corresponding to basename of path being passed.
 
 * getdents
   1. remove entry corresponding to basename of path on which fd is opened from
-     cache stored in parent, since getdents changes st_atime. 
-  2. remove entries corresponding to symbolic links from cache, since readlink 
+     cache stored in parent, since getdents changes st_atime.
+  2. remove entries corresponding to symbolic links from cache, since readlink
      would've changed st_atime.
 
 * checksum
@@ -144,11 +144,11 @@ callbacks to be implemented:
 limitations:
 ============
 * since a readdir does not return extended attributes of file, if need_xattr is
-  set, short-cutting of lookup does not happen and lookup is passed to 
+  set, short-cutting of lookup does not happen and lookup is passed to
   underlying translators.
 
 * posix_readdir does not check whether the dentries are spanning across multiple
-  mount points. Hence it is not transforming inode numbers in stat buffers if 
+  mount points. Hence it is not transforming inode numbers in stat buffers if
   posix is configured to allow export directory spanning on multiple mountpoints.
-  This is a bug which needs to be fixed. posix_readdir should treat dentries the 
+  This is a bug which needs to be fixed. posix_readdir should treat dentries the
   same way as if lookup is happening on dentries.
diff --git a/doc/user-guide/stripe.odg b/doc/legacy/stripe.odg
index 79441bf14..79441bf14 100644
--- a/doc/user-guide/stripe.odg
+++ b/doc/legacy/stripe.odg
diff --git a/doc/user-guide/stripe.pdf b/doc/legacy/stripe.pdf
index b94446feb..b94446feb 100644
--- a/doc/user-guide/stripe.pdf
+++ b/doc/legacy/stripe.pdf
diff --git a/doc/translator-options.txt b/doc/legacy/translator-options.txt
index 278ef5b00..3422c058a 100644
--- a/doc/translator-options.txt
+++ b/doc/legacy/translator-options.txt
@@ -1,7 +1,7 @@
 mount/fuse:
 	* direct-io-mode            GF_OPTION_TYPE_BOOL   on|off|yes|no
 	* mount-point (mountpoint)  GF_OPTION_TYPE_PATH   <any-posix-valid-path>
-	* attribute-timeout         GF_OPTION_TYPE_DOUBLE   0.0 
+	* attribute-timeout         GF_OPTION_TYPE_DOUBLE   0.0
 	* entry-timeout             GF_OPTION_TYPE_DOUBLE   0.0
 
 protocol/server:
@@ -13,20 +13,20 @@ protocol/server:
 
 protocol/client:
 	* username                  GF_OPTION_TYPE_ANY
-	* password                  GF_OPTION_TYPE_ANY 
+	* password                  GF_OPTION_TYPE_ANY
 	* transport-type            GF_OPTION_TYPE_STR   tcp|socket|ib-verbs|unix|ib-sdp|
 	  			    			 tcp/client|ib-verbs/client
-	* remote-host               GF_OPTION_TYPE_ANY 
-	* remote-subvolume          GF_OPTION_TYPE_ANY 
-	* transport-timeout         GF_OPTION_TYPE_TIME  5-1013 
+	* remote-host               GF_OPTION_TYPE_ANY
+	* remote-subvolume          GF_OPTION_TYPE_ANY
+	* transport-timeout         GF_OPTION_TYPE_TIME  5-1013
 
 cluster/replicate:
 	* read-subvolume	    GF_OPTION_TYPE_XLATOR
 	* favorite-child 	    GF_OPTION_TYPE_XLATOR
-	* data-self-heal 	    GF_OPTION_TYPE_BOOL 
+	* data-self-heal 	    GF_OPTION_TYPE_BOOL
 	* metadata-self-heal 	    GF_OPTION_TYPE_BOOL
-	* entry-self-heal 	    GF_OPTION_TYPE_BOOL 
-	* data-change-log 	    GF_OPTION_TYPE_BOOL 
+	* entry-self-heal 	    GF_OPTION_TYPE_BOOL
+	* data-change-log 	    GF_OPTION_TYPE_BOOL
 	* metadata-change-log 	    GF_OPTION_TYPE_BOOL
 	* entry-change-log 	    GF_OPTION_TYPE_BOOL
 	* data-lock-server-count    GF_OPTION_TYPE_INT    0
@@ -34,54 +34,54 @@ cluster/replicate:
 	* entry-lock-server-count    GF_OPTION_TYPE_INT   0
 
 cluster/distribute:
-	* lookup-unhashed           GF_OPTION_TYPE_BOOL 
+	* lookup-unhashed           GF_OPTION_TYPE_BOOL
 
 cluster/unify:
-	* namespace		    GF_OPTION_TYPE_XLATOR 
-	* scheduler 		    GF_OPTION_TYPE_STR    alu|rr|random|nufa|switch 
+	* namespace		    GF_OPTION_TYPE_XLATOR
+	* scheduler 		    GF_OPTION_TYPE_STR    alu|rr|random|nufa|switch
 	* self-heal 		    GF_OPTION_TYPE_STR    foreground|background|off
-	* optimist  		    GF_OPTION_TYPE_BOOL 
+	* optimist  		    GF_OPTION_TYPE_BOOL
 
 cluster/nufa:
-	local-volume-name           GF_OPTION_TYPE_XLATOR 
+	local-volume-name           GF_OPTION_TYPE_XLATOR
 
 cluster/stripe:
-	* block-size		    GF_OPTION_TYPE_ANY 
+	* block-size		    GF_OPTION_TYPE_ANY
 	* use-xattr  		    GF_OPTION_TYPE_BOOL
 
 debug/trace:
 	* include-ops (include)     GF_OPTION_TYPE_STR
-	* exclude-ops (exclude)     GF_OPTION_TYPE_STR 
+	* exclude-ops (exclude)     GF_OPTION_TYPE_STR
 
 encryption/rot-13:
 	* encrypt-write             GF_OPTION_TYPE_BOOL
-	* decrypt-read  	    GF_OPTION_TYPE_BOOL 
+	* decrypt-read  	    GF_OPTION_TYPE_BOOL
 
 features/path-convertor:
-	* start-offset              GF_OPTION_TYPE_INT     0-4095 
-	* end-offset 		    GF_OPTION_TYPE_INT	   1-4096 
+	* start-offset              GF_OPTION_TYPE_INT     0-4095
+	* end-offset 		    GF_OPTION_TYPE_INT	   1-4096
 	* replace-with		    GF_OPTION_TYPE_ANY
 
 features/trash:
-	* trash-dir	            GF_OPTION_TYPE_PATH 
+	* trash-dir	            GF_OPTION_TYPE_PATH
 
 features/locks:
-	* mandatory-locks (mandatory) GF_OPTION_TYPE_BOOL 
+	* mandatory-locks (mandatory) GF_OPTION_TYPE_BOOL
 
 features/filter:
-	* root-squashing	    GF_OPTION_TYPE_BOOL 
+	* root-squashing	    GF_OPTION_TYPE_BOOL
 	* read-only                 GF_OPTION_TYPE_BOOL
 	* fixed-uid		    GF_OPTION_TYPE_INT
 	* fixed-gid		    GF_OPTION_TYPE_INT
-	* translate-uid             GF_OPTION_TYPE_ANY 
+	* translate-uid             GF_OPTION_TYPE_ANY
 	* translate-gid		    GF_OPTION_TYPE_ANY
-	* filter-uid		    GF_OPTION_TYPE_ANY 
-	* filter-gid		    GF_OPTION_TYPE_ANY 
+	* filter-uid		    GF_OPTION_TYPE_ANY
+	* filter-gid		    GF_OPTION_TYPE_ANY
 
 features/quota:
 	* min-free-disk-limit	    GF_OPTION_TYPE_PERCENT
 	* refresh-interval	    GF_OPTION_TYPE_TIME
-	* disk-usage-limit	    GF_OPTION_TYPE_SIZET 
+	* disk-usage-limit	    GF_OPTION_TYPE_SIZET
 
 storage/posix:
 	* o-direct		    GF_OPTION_TYPE_BOOL
@@ -104,16 +104,16 @@ storage/bdb:
 	* access-mode		    GF_OPTION_TYPE_STR
 
 performance/read-ahead:
-	* force-atime-update        GF_OPTION_TYPE_BOOL 
+	* force-atime-update        GF_OPTION_TYPE_BOOL
 	* page-size		    GF_OPTION_TYPE_SIZET (64 * GF_UNIT_KB)-(2 * GF_UNIT_MB)
-	* page-count		    GF_OPTION_TYPE_INT   1-16 
+	* page-count		    GF_OPTION_TYPE_INT   1-16
 
 performance/write-behind:
 	* flush-behind		    GF_OPTION_TYPE_BOOL
-	* aggregate-size	    GF_OPTION_TYPE_SIZET  (128 * GF_UNIT_KB)-(4 * GF_UNIT_MB) 
-	* window-size		    GF_OPTION_TYPE_SIZET  (512 * GF_UNIT_KB)-(1 * GF_UNIT_GB) 
-	* enable-O_SYNC		    GF_OPTION_TYPE_BOOL  
-	* disable-for-first-nbytes  GF_OPTION_TYPE_SIZET  1 - (1 * GF_UNIT_MB) 
+	* aggregate-size	    GF_OPTION_TYPE_SIZET  (128 * GF_UNIT_KB)-(4 * GF_UNIT_MB)
+	* window-size		    GF_OPTION_TYPE_SIZET  (512 * GF_UNIT_KB)-(1 * GF_UNIT_GB)
+	* enable-O_SYNC		    GF_OPTION_TYPE_BOOL
+	* disable-for-first-nbytes  GF_OPTION_TYPE_SIZET  1 - (1 * GF_UNIT_MB)
 
 performance/symlink-cache:
 
@@ -121,9 +121,9 @@ performance/io-threads:
 	* thread-count	            GF_OPTION_TYPE_INT    1-32
 
 performance/io-cache:
-	* priority	            GF_OPTION_TYPE_ANY 
-	* cache-timeout (force-revalidate-timeout) GF_OPTION_TYPE_INT 0-60 
-	* page-size	            GF_OPTION_TYPE_SIZET  (16 * GF_UNIT_KB)-(4 * GF_UNIT_MB) 
+	* priority	            GF_OPTION_TYPE_ANY
+	* cache-timeout (force-revalidate-timeout) GF_OPTION_TYPE_INT 0-60
+	* page-size	            GF_OPTION_TYPE_SIZET  (16 * GF_UNIT_KB)-(4 * GF_UNIT_MB)
         * cache-size                GF_OPTION_TYPE_SIZET  (4 * GF_UNIT_MB)-(6 * GF_UNIT_GB)
 
 performance/quick-read:
@@ -132,16 +132,16 @@ performance/quick-read:
 
 auth:
 - addr:
-	* auth.addr.*.allow	    GF_OPTION_TYPE_ANY 
-	* auth.addr.*.reject	    GF_OPTION_TYPE_ANY 
+	* auth.addr.*.allow	    GF_OPTION_TYPE_ANY
+	* auth.addr.*.reject	    GF_OPTION_TYPE_ANY
 
 - login:
-	* auth.login.*.allow	    GF_OPTION_TYPE_ANY 
+	* auth.login.*.allow	    GF_OPTION_TYPE_ANY
 	* auth.login.*.password	    GF_OPTION_TYPE_ANY
 
 scheduler/alu:
-	* scheduler.alu.order (alu.order) 
-	                            GF_OPTION_TYPE_ANY 
+	* scheduler.alu.order (alu.order)
+	                            GF_OPTION_TYPE_ANY
 	* scheduler.alu.disk-usage.entry-threshold (alu.disk-usage.entry-threshold)
 	  			    GF_OPTION_TYPE_SIZET
         * scheduler.alu.disk-usage.exit-threshold (alu.disk-usage.exit-threshold)
@@ -149,17 +149,17 @@ scheduler/alu:
         * scheduler.alu.write-usage.entry-threshold (alu.write-usage.entry-threshold)
                                     GF_OPTION_TYPE_SIZET
         * scheduler.alu.write-usage.exit-threshold (alu.write-usage.exit-threshold)
-                                    GF_OPTION_TYPE_SIZET 
+                                    GF_OPTION_TYPE_SIZET
         * scheduler.alu.read-usage.entry-threshold (alu.read-usage.entry-threshold)
                                     GF_OPTION_TYPE_SIZET
         * scheduler.alu.read-usage.exit-threshold (alu.read-usage.exit-threshold)
-                                    GF_OPTION_TYPE_SIZET 
+                                    GF_OPTION_TYPE_SIZET
         * scheduler.alu.open-files-usage.entry-threshold (alu.open-files-usage.entry-threshold)
                                     GF_OPTION_TYPE_INT
         * scheduler.alu.open-files-usage.exit-threshold (alu.open-files-usage.exit-threshold)
-                                    GF_OPTION_TYPE_INT 
+                                    GF_OPTION_TYPE_INT
         * scheduler.read-only-subvolumes (alu.read-only-subvolumes)
-                                    GF_OPTION_TYPE_ANY 
+                                    GF_OPTION_TYPE_ANY
         * scheduler.refresh-interval (alu.refresh-interval)
                                     GF_OPTION_TYPE_TIME
         * scheduler.limits.min-free-disk (alu.limits.min-free-disk)
@@ -168,11 +168,11 @@ scheduler/alu:
                                     GF_OPTION_TYPE_INT
 
 scheduler/nufa:
-	* scheduler.refresh-interval (nufa.refresh-interval) 
+	* scheduler.refresh-interval (nufa.refresh-interval)
 	                            GF_OPTION_TYPE_TIME
-	* scheduler.limits.min-free-disk (nufa.limits.min-free-disk) 
+	* scheduler.limits.min-free-disk (nufa.limits.min-free-disk)
 	                            GF_OPTION_TYPE_PERCENT
-	* scheduler.local-volume-name (nufa.local-volume-name) 
+	* scheduler.local-volume-name (nufa.local-volume-name)
 	                            GF_OPTION_TYPE_XLATOR
 
 scheduler/random:
@@ -204,20 +204,20 @@ transport/ib-verbs:
         * transport.ib-verbs.work-request-recv-count (ib-verbs-work-request-recv-count)
                                  GF_OPTION_TYPE_INT
         * remote-port (transport.remote-port,transport.ib-verbs.remote-port)
-	                         GF_OPTION_TYPE_INT 
-        * transport.ib-verbs.listen-port  GF_OPTION_TYPE_INT 
-        * transport.ib-verbs.connect-path (connect-path) GF_OPTION_TYPE_ANY 
-        * transport.ib-verbs.bind-path (bind-path) GF_OPTION_TYPE_ANY 
-        * transport.ib-verbs.listen-path (listen-path) GF_OPTION_TYPE_ANY 
+	                         GF_OPTION_TYPE_INT
+        * transport.ib-verbs.listen-port  GF_OPTION_TYPE_INT
+        * transport.ib-verbs.connect-path (connect-path) GF_OPTION_TYPE_ANY
+        * transport.ib-verbs.bind-path (bind-path) GF_OPTION_TYPE_ANY
+        * transport.ib-verbs.listen-path (listen-path) GF_OPTION_TYPE_ANY
         * transport.address-family (address-family) GF_OPTION_TYPE_STR inet|inet6|inet/inet6|
                                                                        inet6/inet|unix|inet-sdp
 
 transport/socket:
-	* transport.remote-port (remote-port,transport.socket.remote-port) GF_OPTION_TYPE_INT 
-	* transport.socket.listen-port (listen-port) GF_OPTION_TYPE_INT 
-	* transport.socket.bind-address (bind-address) GF_OPTION_TYPE_ANY 
-	* transport.socket.connect-path (connect-path) GF_OPTION_TYPE_ANY 
-	* transport.socket.bind-path (bind-path)       GF_OPTION_TYPE_ANY 
+	* transport.remote-port (remote-port,transport.socket.remote-port) GF_OPTION_TYPE_INT
+	* transport.socket.listen-port (listen-port) GF_OPTION_TYPE_INT
+	* transport.socket.bind-address (bind-address) GF_OPTION_TYPE_ANY
+	* transport.socket.connect-path (connect-path) GF_OPTION_TYPE_ANY
+	* transport.socket.bind-path (bind-path)       GF_OPTION_TYPE_ANY
 	* transport.socket.listen-path (listen-path)   GF_OPTION_TYPE_ANY
 	* transport.address-family (address-family)    GF_OPTION_TYPE_STR  inet|inet6|
 	  			   		       			   inet/inet6|inet6/inet|
diff --git a/doc/user-guide/unify.odg b/doc/legacy/unify.odg
index ccaa9bf16..ccaa9bf16 100644
--- a/doc/user-guide/unify.odg
+++ b/doc/legacy/unify.odg
diff --git a/doc/user-guide/unify.pdf b/doc/legacy/unify.pdf
index c22027f66..c22027f66 100644
--- a/doc/user-guide/unify.pdf
+++ b/doc/legacy/unify.pdf
diff --git a/doc/user-guide/user-guide.info b/doc/legacy/user-guide.info
index e46645099..2bbadb351 100644
--- a/doc/user-guide/user-guide.info
+++ b/doc/legacy/user-guide.info
@@ -130,7 +130,7 @@ suggestions.  A huge thanks to them all.
 
    Patrick Negri - for TCP non-blocking connect.
         http://gluster.org/core-team.php (<list-hacking@gluster.com>)
-                                                          Gluster 
+                                                          Gluster
 
 
 File: user-guide.info,  Node: Introduction,  Next: Installation and Invocation,  Prev: Acknowledgements,  Up: Top
@@ -160,15 +160,15 @@ makes them all appear to be a part of the same filesystem.
 =================
 
 You can reach us through the mailing list *gluster-devel*
-(<gluster-devel@nongnu.org>).  
+(<gluster-devel@nongnu.org>).
 
    You can also find many of the developers on IRC, on the `#gluster'
-channel on Freenode (<irc.freenode.net>).  
+channel on Freenode (<irc.freenode.net>).
 
    The GlusterFS documentation wiki is also useful:
 <http://gluster.org/docs/index.php/GlusterFS>
 
-   For commercial support, you can contact Gluster at: 
+   For commercial support, you can contact Gluster at:
 
      3194 Winding Vista Common
      Fremont, CA 94539
@@ -397,8 +397,8 @@ command-line options accepted by it.
 `--volfile-server-port=<port-number>'
      Listening port number of volfile server.
 
-`--volfile-server-transport=[socket|ib-verbs]'
-     Transport type to get volfile from server. [default: `socket']
+`--volfile-server-transport=[tcp|ib-verbs]'
+     Transport type to get volfile from server. [default: `tcp']
 
 `--xlator-options=<volume-name.option=value>'
      Add/override a translator option for a volume with specified value.
@@ -467,8 +467,8 @@ filesystem to appear. Example:
 `--volfile-server-port=<port-number>'
      Listening port number of volfile server.
 
-`--volfile-server-transport=[socket|ib-verbs]'
-     Transport type to get volfile from server. [default: `socket']
+`--volfile-server-transport=[tcp|ib-verbs]'
+     Transport type to get volfile from server. [default: `tcp']
 
 `--xlator-options=<volume-name.option=value>'
      Add/override a translator option for a volume with specified value.
@@ -907,7 +907,7 @@ and the client.
      Whether to make the connection attempt asynchronous.
 
 `remote-port <n> (24007)'
-     Server port to connect to.  
+     Server port to connect to.
 
 `remote-host <hostname> *'
      Hostname or IP address of the server. If the host name resolves to
@@ -952,7 +952,7 @@ always best to use `ib-verbs'. Use `ib-sdp' only if you cannot get
      Whether to make the connection attempt asynchronous.
 
 `remote-port <n> (24007)'
-     Server port to connect to.  
+     Server port to connect to.
 
 `remote-host <hostname> *'
      Hostname or IP address of the server. If the host name resolves to
@@ -2057,7 +2057,7 @@ the `--log-file' option (See *note Client::).
 =======================
 
 `modprobe fuse' fails with: "Unknown symbol in module, or unknown
-parameter".  
+parameter".
 
    If you are using fuse-2.6.x on Redhat Enterprise Linux Work Station 4
 and Advanced Server 4 with 2.6.9-42.ELlargesmp, 2.6.9-42.ELsmp,
diff --git a/doc/user-guide/user-guide.pdf b/doc/legacy/user-guide.pdf
index ed7bd2a99..ed7bd2a99 100644
--- a/doc/user-guide/user-guide.pdf
+++ b/doc/legacy/user-guide.pdf
diff --git a/doc/user-guide/user-guide.texi b/doc/legacy/user-guide.texi
index f864c186a..8e429853f 100644
--- a/doc/user-guide/user-guide.texi
+++ b/doc/legacy/user-guide.texi
@@ -23,7 +23,7 @@ Documentation License''.
 @title GlusterFS 2.0 User Guide [DRAFT]
 @subtitle January 15, 2008
 @author http://gluster.org/core-team.php
-@author @email{@b{Gluster}} 
+@author @email{@b{Gluster}}
 @page
 @vskip 0pt plus 1filll
 @insertcopying
@@ -36,78 +36,78 @@ Documentation License''.
 
 @insertcopying
 @menu
-* Acknowledgements::            
-* Introduction::                
-* Installation and Invocation::  
-* Concepts::                    
-* Translators::                 
-* Usage Scenarios::             
-* Troubleshooting::             
-* GNU Free Documentation Licence::  
-* Index::                       
+* Acknowledgements::
+* Introduction::
+* Installation and Invocation::
+* Concepts::
+* Translators::
+* Usage Scenarios::
+* Troubleshooting::
+* GNU Free Documentation Licence::
+* Index::
 
 @detailmenu
  --- The Detailed Node Listing ---
 
 Installation and Invocation
 
-* Pre requisites::              
-* Getting GlusterFS::           
-* Building::                    
-* Running GlusterFS::           
-* A Tutorial Introduction::     
+* Pre requisites::
+* Getting GlusterFS::
+* Building::
+* Running GlusterFS::
+* A Tutorial Introduction::
 
 Running GlusterFS
 
-* Server::                      
-* Client::                      
+* Server::
+* Client::
 
 Concepts
 
-* Filesystems in Userspace::                
-* Translator::                  
-* Volume specification file::   
+* Filesystems in Userspace::
+* Translator::
+* Volume specification file::
 
 Translators
 
-* Storage Translators::         
-* Client and Server Translators::  
-* Clustering Translators::      
-* Performance Translators::     
-* Features Translators::        
+* Storage Translators::
+* Client and Server Translators::
+* Clustering Translators::
+* Performance Translators::
+* Features Translators::
 
 Storage Translators
 
-* POSIX::        
+* POSIX::
 
 Client and Server Translators
 
-* Transport modules::           
-* Client protocol::             
-* Server protocol::             
+* Transport modules::
+* Client protocol::
+* Server protocol::
 
 Clustering Translators
 
-* Unify::                       
-* Replicate::  
-* Stripe::                      
+* Unify::
+* Replicate::
+* Stripe::
 
 Performance Translators
 
-* Read Ahead::                  
-* Write Behind::                
-* IO Threads::                  
-* IO Cache::                    
+* Read Ahead::
+* Write Behind::
+* IO Threads::
+* IO Cache::
 
-Features Translators 
+Features Translators
 
-* POSIX Locks::                 
-* Fixed ID::                    
+* POSIX Locks::
+* Fixed ID::
 
 Miscellaneous Translators
 
-* ROT-13::                      
-* Trace::                       
+* ROT-13::
+* Trace::
 
 @end detailmenu
 @end menu
@@ -120,7 +120,7 @@ Miscellaneous Translators
 @node Acknowledgements
 @unnumbered Acknowledgements
 GlusterFS continues to be a wonderful and enriching experience for all
-of us involved. 
+of us involved.
 
 GlusterFS development would not have been possible at this pace if
 not for our enthusiastic users. People from around the world have
@@ -142,7 +142,7 @@ Jacques Mattheij - for Europe mirror.
 Patrick Negri - for TCP non-blocking connect.
 @flushright
 http://gluster.org/core-team.php (@email{list-hacking@@gluster.com})
-@email{@b{Gluster}} 
+@email{@b{Gluster}}
 @end flushright
 
 @node Introduction
@@ -166,7 +166,7 @@ Need for distributed filesystems
 @end itemize
 
 @section Contacting us
-You can reach us through the mailing list @strong{gluster-devel} 
+You can reach us through the mailing list @strong{gluster-devel}
 (@email{gluster-devel@@nongnu.org}).
 @cindex GlusterFS mailing list
 
@@ -197,11 +197,11 @@ You can also email us at @email{support@@gluster.com}.
 @chapter Installation and Invocation
 
 @menu
-* Pre requisites::              
-* Getting GlusterFS::           
-* Building::                    
-* Running GlusterFS::           
-* A Tutorial Introduction::     
+* Pre requisites::
+* Getting GlusterFS::
+* Building::
+* Running GlusterFS::
+* A Tutorial Introduction::
 @end menu
 
 @node Pre requisites
@@ -247,7 +247,7 @@ our patched version of the @acronym{FUSE} kernel module. See Patched FUSE for de
 
 @subsection Patched FUSE
 
-The GlusterFS project maintains a patched version of @acronym{FUSE} meant to be used 
+The GlusterFS project maintains a patched version of @acronym{FUSE} meant to be used
 with GlusterFS. The patches increase GlusterFS performance. It is recommended that
 all users use the patched @acronym{FUSE}.
 
@@ -311,7 +311,7 @@ $ cd glusterfs-<version>
 
 If you checked out the source from the Arch repository, you'll need
 to run @command{./autogen.sh} first. Note that you'll need to have
-Autoconf and Automake installed for this. 
+Autoconf and Automake installed for this.
 
 Run @command{configure}.
 
@@ -371,8 +371,8 @@ paths with the prefix.
 @section Running GlusterFS
 
 @menu
-* Server::                      
-* Client::                      
+* Server::
+* Client::
 @end menu
 
 @node Server
@@ -386,7 +386,7 @@ of the GlusterFS server program and all the command-line options accepted by it.
 @cartouche
 @table @code
 Basic Options
-@item -f, --volfile=<path>   
+@item -f, --volfile=<path>
       Use the volume file as the volume specification.
 
 @item -s, --volfile-server=<hostname>
@@ -396,7 +396,7 @@ Basic Options
       Specify the path for the log file.
 
 @item -L, --log-level=<level>
-      Set the log level for the server. Log level should be one of @acronym{DEBUG}, 
+      Set the log level for the server. Log level should be one of @acronym{DEBUG},
 @acronym{WARNING}, @acronym{ERROR}, @acronym{CRITICAL}, or @acronym{NONE}.
 
 Advanced Options
@@ -404,10 +404,10 @@ Advanced Options
       Run in debug mode. This option sets --no-daemon, --log-level to DEBUG and
       --log-file to console.
 
-@item  -N, --no-daemon            
+@item  -N, --no-daemon
       Run glusterfsd as a foreground process.
 
-@item  -p, --pid-file=<path>      
+@item  -p, --pid-file=<path>
       Path for the @acronym{PID} file.
 
 @item --volfile-id=<key>
@@ -416,20 +416,20 @@ Advanced Options
 @item --volfile-server-port=<port-number>
       Listening port number of volfile server.
 
-@item --volfile-server-transport=[socket|ib-verbs]
-      Transport type to get volfile from server. [default: @command{socket}]
+@item --volfile-server-transport=[tcp|ib-verbs]
+      Transport type to get volfile from server. [default: @command{tcp}]
 
 @item --xlator-options=<volume-name.option=value>
       Add/override a translator option for a volume with specified value.
 
 Miscellaneous Options
-@item  -?, --help                 
+@item  -?, --help
        Show this help text.
 
-@item  --usage                
+@item  --usage
        Display a short usage message.
 
-@item  -V, --version              
+@item  -V, --version
        Show version information.
 @end table
 @end cartouche
@@ -464,7 +464,7 @@ The command-line options are detailed below.
 @table @code
 
 Basic Options
-@item -f, --volfile=<path>   
+@item -f, --volfile=<path>
       Use the volume file as the volume specification.
 
 @item -s, --volfile-server=<hostname>
@@ -474,7 +474,7 @@ Basic Options
       Specify the path for the log file.
 
 @item -L, --log-level=<level>
-      Set the log level for the server. Log level should be one of @acronym{DEBUG}, 
+      Set the log level for the server. Log level should be one of @acronym{DEBUG},
 @acronym{WARNING}, @acronym{ERROR}, @acronym{CRITICAL}, or @acronym{NONE}.
 
 Advanced Options
@@ -482,10 +482,10 @@ Advanced Options
       Run in debug mode. This option sets --no-daemon, --log-level to DEBUG and
       --log-file to console.
 
-@item  -N, --no-daemon            
+@item  -N, --no-daemon
       Run @command{glusterfs} as a foreground process.
 
-@item  -p, --pid-file=<path>      
+@item  -p, --pid-file=<path>
       Path for the @acronym{PID} file.
 
 @item --volfile-id=<key>
@@ -494,8 +494,8 @@ Advanced Options
 @item --volfile-server-port=<port-number>
       Listening port number of volfile server.
 
-@item --volfile-server-transport=[socket|ib-verbs]
-      Transport type to get volfile from server. [default: @command{socket}]
+@item --volfile-server-transport=[tcp|ib-verbs]
+      Transport type to get volfile from server. [default: @command{tcp}]
 
 @item --xlator-options=<volume-name.option=value>
       Add/override a translator option for a volume with specified value.
@@ -512,14 +512,14 @@ Advanced Options
        automatically if kernel supports big writes (>= 2.6.26).
 
 @item  -e, --entry-timeout=<n>
-       Entry timeout for directory entries in the kernel, in seconds. 
+       Entry timeout for directory entries in the kernel, in seconds.
        Defaults to 1 second.
 
 Missellaneous Options
-@item  -?, --help                 
+@item  -?, --help
        Show this help information.
 
-@item  -V, --version              
+@item  -V, --version
        Show version information.
 @end table
 @end cartouche
@@ -527,7 +527,7 @@ Missellaneous Options
 @node A Tutorial Introduction
 @section A Tutorial Introduction
 
-This section will show you how to quickly get GlusterFS up and running. We'll 
+This section will show you how to quickly get GlusterFS up and running. We'll
 configure GlusterFS as a simple network filesystem, with one server and one client.
 In this mode of usage, GlusterFS can serve as a replacement for NFS.
 
@@ -545,18 +545,18 @@ be run on the server will be shown with the prompt:
 Our goal is to make a directory on the @emph{server} (say, @command{/export})
 accessible to the @emph{client}.
 
-First of all, get GlusterFS installed on both the machines, as described in the 
+First of all, get GlusterFS installed on both the machines, as described in the
 previous sections. Make sure you have the @acronym{FUSE} kernel module loaded. You
-can ensure this by running: 
+can ensure this by running:
 
 @example
 [root@@server]# modprobe fuse
 @end example
 
 Before we can run the GlusterFS client or server programs, we need to write
-two files called @emph{volume specifications} (equivalently refered to as @emph{volfiles}). 
+two files called @emph{volume specifications} (equivalently refered to as @emph{volfiles}).
 The volfile describes the @emph{translator tree} on a node. The next chapter will
-explain the concepts of `translator' and `volume specification' in detail. For now, 
+explain the concepts of `translator' and `volume specification' in detail. For now,
 just assume that the volfile is like an NFS @command{/etc/export} file.
 
 On the server, create a text file somewhere (we'll assume the path
@@ -572,7 +572,7 @@ end-volume
 volume server
   type protocol/server
   subvolumes colon-o
-  option transport-type tcp     
+  option transport-type tcp
   option auth.addr.colon-o.allow *
 end-volume
 @end example
@@ -625,9 +625,9 @@ working as a network file system.
 @chapter Concepts
 
 @menu
-* Filesystems in Userspace::                
-* Translator::                  
-* Volume specification file::   
+* Filesystems in Userspace::
+* Translator::
+* Volume specification file::
 @end menu
 
 @node Filesystems in Userspace
@@ -639,16 +639,16 @@ is a kernel module/library that allows us to write a filesystem
 completely in userspace.
 
 @acronym{FUSE} consists of a kernel module which interacts with the userspace
-implementation using a device file @code{/dev/fuse}. When a process 
+implementation using a device file @code{/dev/fuse}. When a process
 makes a syscall on a @acronym{FUSE} filesystem, @acronym{VFS} hands the request to the
 @acronym{FUSE} module, which writes the request to @code{/dev/fuse}. The
 userspace implementation polls @code{/dev/fuse}, and when a request arrives,
 processes it and writes the result back to @code{/dev/fuse}. The kernel then
-reads from the device file and returns the result to the user process. 
+reads from the device file and returns the result to the user process.
 
 In case of GlusterFS, the userspace program is the GlusterFS client.
 The control flow is shown in the diagram below. The GlusterFS client
-services the request by sending it to the server, which in turn 
+services the request by sending it to the server, which in turn
 hands it to the local @acronym{POSIX} filesystem.
 
 @center @image{fuse,44pc,,,.pdf}
@@ -752,7 +752,7 @@ or ``forty-two''.
 line is considered the value; it is up to the translator to parse it.
 
 @item @emph{subvolume1}, @emph{subvolume2}, @dots{}
-  Volume names of sub-volumes. The sub-volumes must already have been defined earlier 
+  Volume names of sub-volumes. The sub-volumes must already have been defined earlier
 in the file.
 @end table
 
@@ -797,11 +797,11 @@ end-volume
 @chapter Translators
 
 @menu
-* Storage Translators::         
-* Client and Server Translators::  
-* Clustering Translators::      
-* Performance Translators::     
-* Features Translators::        
+* Storage Translators::
+* Client and Server Translators::
+* Clustering Translators::
+* Performance Translators::
+* Features Translators::
 * Miscellaneous Translators::
 @end menu
 
@@ -823,12 +823,12 @@ Other storage backends are planned for the future. One of the possibilities is a
 Amazon S3 translator. Amazon S3 is an unlimited online storage service accessible
 through a web services @acronym{API}. The S3 translator will allow you to access
 the storage as a normal @acronym{POSIX} filesystem.
-@footnote{Some more discussion about this can be found at: 
+@footnote{Some more discussion about this can be found at:
 
 http://developer.amazonwebservices.com/connect/message.jspa?messageID=52873}
 
 @menu
-* POSIX::        
+* POSIX::
 * BDB::
 @end menu
 
@@ -843,14 +843,14 @@ filesystem as its ``backend'' to actually store files and
 directories. This can be any filesystem that supports extended
 attributes (@acronym{EXT3}, ReiserFS, @acronym{XFS}, ...). Extended
 attributes are used by some translators to store metadata, for
-example, by the replicate and stripe translators. See 
+example, by the replicate and stripe translators. See
 @ref{Replicate} and @ref{Stripe}, respectively for details.
 
 @cartouche
 @table @code
 @item directory <path>
 The directory on the local filesystem which is to be used for storage.
-@end table               
+@end table
 @end cartouche
 
 @node BDB
@@ -862,7 +862,7 @@ type storage/bdb
 The @command{BDB} translator uses a @acronym{Berkeley DB} database as its
 ``backend'' to actually store files as key-value pair in the database and
 directories as regular @acronym{POSIX} directories. Note that @acronym{BDB}
-does not provide extended attribute support for regular files. Do not use 
+does not provide extended attribute support for regular files. Do not use
 @acronym{BDB} as storage translator while using any translator that demands
 extended attributes on ``backend''.
 
@@ -892,9 +892,9 @@ translator tree over the network or access a remote GlusterFS
 server. These two translators implement GlusterFS's network protocol.
 
 @menu
-* Transport modules::           
-* Client protocol::             
-* Server protocol::             
+* Transport modules::
+* Client protocol::
+* Server protocol::
 @end menu
 
 @node Transport modules
@@ -962,7 +962,7 @@ This module accepts the same options as @command{tcp}
 
 @cindex infiniband transport
 
-InfiniBand is a scalable switched fabric interconnect mechanism 
+InfiniBand is a scalable switched fabric interconnect mechanism
 primarily used in high-performance computing. InfiniBand can deliver
 data throughput of the order of 10 Gbit/s, with latencies of 4-5 ms.
 
@@ -970,7 +970,7 @@ The @command{ib-verbs} transport accesses the InfiniBand hardware through
 the ``verbs'' @acronym{API}, which is the lowest level of software access possible
 and which gives the highest performance. On InfiniBand hardware, it is always
 best to use @command{ib-verbs}. Use @command{ib-sdp} only if you cannot get
-@command{ib-verbs} working for some reason. 
+@command{ib-verbs} working for some reason.
 
 The @command{ib-verbs} client module accepts the following options:
 
@@ -1049,7 +1049,7 @@ translator tree.
 
 @item transport-type [tcp,ib-sdp,ib-verbs] (tcp)
 The transport type to use. You should use the client versions of all the
-transport modules (@command{tcp}, @command{ib-sdp}, 
+transport modules (@command{tcp}, @command{ib-sdp},
 @command{ib-verbs}).
 @item remote-subvolume <volume_name> *
 The name of the volume on the remote host to attach to. Note that
@@ -1075,7 +1075,7 @@ remote GlusterFS clients.
 @table @code
 @item client-volume-filename <path> (<CONFDIR>/glusterfs-client.vol)
 The volume specification file to use for the client. This is the file the
-client will receive when it is invoked with the @command{--server} option 
+client will receive when it is invoked with the @command{--server} option
 (@ref{Client}).
 
 @item transport-type [tcp,ib-verbs,ib-sdp] (tcp)
@@ -1106,9 +1106,9 @@ translator allows a file to be spread across many server nodes. The following se
 look at each of these translators in detail.
 
 @menu
-* Unify::                       
-* Replicate::  
-* Stripe::                      
+* Unify::
+* Replicate::
+* Stripe::
 @end menu
 
 @node Unify
@@ -1121,7 +1121,7 @@ type cluster/unify
 
 The unify translator presents a `unified' view of all its sub-volumes. That is,
 it makes the union of all its sub-volumes appear as a single volume. It is the
-unify translator that gives GlusterFS the ability to access an arbitrarily 
+unify translator that gives GlusterFS the ability to access an arbitrarily
 large amount of storage.
 
 For unify to work correctly, certain invariants need to be maintained across
@@ -1144,7 +1144,7 @@ Looking at the second requirement, you might wonder how one can
 accomplish storing redundant copies of a file, if no file can exist
 multiple times.  To answer, we must remember that these invariants are
 from @emph{unify's perspective}.  A translator such as replicate at a lower
-level in the translator tree than unify may subvert this picture. 
+level in the translator tree than unify may subvert this picture.
 
 The first invariant might seem quite tedious to ensure. We shall see
 later that this is not so, since unify's @emph{self-heal} mechanism
@@ -1241,8 +1241,8 @@ are allowed. For example: @command{option alu.limits.min-free-disk 5GB}.
 @item alu.read-usage.exit-threshold <%> (5)
 @item alu.open-files-usage.entry-threshold <n> (1000)
 @item alu.open-files-usage.exit-threshold <n> (100)
-@item alu.limits.min-free-disk <%> 
-@item alu.limits.max-open-files <n> 
+@item alu.limits.min-free-disk <%>
+@item alu.limits.max-open-files <n>
 @end table
 @end cartouche
 
@@ -1259,7 +1259,7 @@ files are mostly similar in size and I/O access pattern, this
 scheduler is a good choice. RR scheduler checks for free disk space
 on the server before scheduling, so you can know when to add
 another server node. The default value of min-free-disk is 5% and is
-checked on file creation calls, with atleast 10 seconds (by default) 
+checked on file creation calls, with atleast 10 seconds (by default)
 elapsing between two checks.
 
 Options:
@@ -1315,7 +1315,7 @@ than a specified amount (5% by default) then @acronym{NUFA} schedules files
 among the other child volumes in a round-robin fashion.
 
 @acronym{NUFA} is named after the similar strategy used for memory access,
-@acronym{NUMA}@footnote{Non-Uniform Memory Access: 
+@acronym{NUMA}@footnote{Non-Uniform Memory Access:
 @indicateurl{http://en.wikipedia.org/wiki/Non-Uniform_Memory_Access}}.
 
 @cartouche
@@ -1325,7 +1325,7 @@ Minimum disk space that must be free (local or remote) for @acronym{NUFA} to sch
 file to it.
 @item nufa.refresh-interval <t> (10 seconds)
 Time between two successive free disk space checks.
-@item nufa.local-volume-name <volume> 
+@item nufa.local-volume-name <volume>
 The name of the volume corresponding to the local system. This volume must be
 one of the children of the unify volume. This option is mandatory.
 @end table
@@ -1397,13 +1397,13 @@ end-volume
 @end example
 
 This sample configuration will replicate all directories and files on
-brick1, brick2 and brick3. 
+brick1, brick2 and brick3.
 
 All the read operations happen from the first alive child. If all the
 three sub-volumes are up, reads will be done from brick1; if brick1 is
 down read will be done from brick2. In case read() was being done on
 brick1 and it goes down, replicate transparently falls back to
-brick2. 
+brick2.
 
 The next release of GlusterFS will add the following features:
 @itemize
@@ -1496,7 +1496,7 @@ type cluster/stripe
 The stripe translator distributes the contents of a file over its
 sub-volumes.  It does this by creating a file equal in size to the
 total size of the file on each of its sub-volumes. It then writes only
-a part of the file to each sub-volume, leaving the rest of it empty. 
+a part of the file to each sub-volume, leaving the rest of it empty.
 These empty regions are called `holes' in Unix terminology. The holes
 do not consume any disk space.
 
@@ -1504,14 +1504,14 @@ The diagram below makes this clear.
 
 @center @image{stripe,44pc,,,.pdf}
 
-You can configure stripe so that only filenames matching a pattern 
-are striped. You can also configure the size of the data to be stored 
+You can configure stripe so that only filenames matching a pattern
+are striped. You can also configure the size of the data to be stored
 on each sub-volume.
 
 @cartouche
 @table @code
 @item block-size <pattern>:<size>  (*:0 no striping)
-Distribute files matching @command{<pattern>} over the sub-volumes, 
+Distribute files matching @command{<pattern>} over the sub-volumes,
 storing at least @command{<size>} on each sub-volume. For example,
 
 @example
@@ -1530,9 +1530,9 @@ different sizes for different file name patterns.
 @section Performance Translators
 
 @menu
-* Read Ahead::                  
-* Write Behind::                
-* IO Threads::                  
+* Read Ahead::
+* Write Behind::
+* IO Threads::
 * IO Cache::
 * Booster::
 @end menu
@@ -1547,9 +1547,9 @@ type performance/read-ahead
 The read-ahead translator pre-fetches data in advance on every read.
 This benefits applications that mostly process files in sequential order,
 since the next block of data will already be available by the time the
-application is done with the current one. 
+application is done with the current one.
 
-Additionally, the read-ahead translator also behaves as a read-aggregator. 
+Additionally, the read-ahead translator also behaves as a read-aggregator.
 Many small read operations are combined and issued as fewer, larger read
 requests to the server.
 
@@ -1557,7 +1557,7 @@ Read-ahead deals in ``pages'' as the unit of data fetched. The page size
 is configurable, as is the ``page count'', which is the number of pages
 that are pre-fetched.
 
-Read-ahead is best used with InfiniBand (using the ib-verbs transport). 
+Read-ahead is best used with InfiniBand (using the ib-verbs transport).
 On FastEthernet and Gigabit Ethernet networks,
 GlusterFS can achieve the link-maximum throughput even without
 read-ahead, making it quite superflous.
@@ -1577,7 +1577,7 @@ The unit of data that is pre-fetched.
 The number of pages that are pre-fetched.
 @item force-atime-update [on|off|yes|no] (off|no)
 Whether to force an access time (atime) update on the file on every read. Without
-this, the atime will be slightly imprecise, as it will reflect the time when 
+this, the atime will be slightly imprecise, as it will reflect the time when
 the read-ahead translator read the data, not when the application actually read it.
 @end table
 @end cartouche
@@ -1596,7 +1596,7 @@ write-behind translator, successive write requests can be pipelined.
 This mode of write-behind operation is best used on the client side, to
 enable decreased write latency for the application.
 
-The write-behind translator can also aggregate write requests. If the 
+The write-behind translator can also aggregate write requests. If the
 @command{aggregate-size} option is specified, then successive writes upto that
 size are accumulated and written in a single operation. This mode of operation
 is best used on the server side, as this will decrease the disk's head movement
@@ -1659,7 +1659,7 @@ It caches data upto @command{cache-size} bytes. The cache is maintained as
 a prioritized least-recently-used (@acronym{LRU}) list, with priorities determined
 by user-specified patterns to match filenames.
 
-When the IO cache translator detects a write operation, the 
+When the IO cache translator detects a write operation, the
 cache for that file is flushed.
 
 The IO cache translator periodically verifies the consistency of
@@ -1715,11 +1715,11 @@ can start your application as:
 The booster translator accepts no options.
 
 @node Features Translators
-@section Features Translators 
+@section Features Translators
 
 @menu
-* POSIX Locks::                 
-* Fixed ID::                    
+* POSIX Locks::
+* Fixed ID::
 @end menu
 
 @node POSIX Locks
@@ -1783,8 +1783,8 @@ The @acronym{GID} to send to the server
 @section Miscellaneous Translators
 
 @menu
-* ROT-13::                      
-* Trace::                       
+* ROT-13::
+* Trace::
 @end menu
 
 @node ROT-13
@@ -1799,7 +1799,7 @@ contents using the @acronym{ROT-13} algorithm. @acronym{ROT-13} is a trivial
 algorithm that rotates each alphabet by thirteen places. Thus, 'A' becomes 'N',
 'B' becomes 'O', and 'Z' becomes 'M'.
 
-It goes without saying that you shouldn't use this translator if you need 
+It goes without saying that you shouldn't use this translator if you need
 @emph{real} encryption (a future release of GlusterFS will have real encryption
 translators).
 
@@ -1816,7 +1816,7 @@ Whether to decrypt on read
 @subsection Trace
 @cindex trace (translator)
 @example
-type debug/trace     
+type debug/trace
 @end example
 
 The trace translator is intended for debugging purposes. When loaded, it
@@ -1827,23 +1827,23 @@ level of DEBUG (See @ref{Running GlusterFS}) for trace to work.
 Sample trace output (lines have been wrapped for readability):
 @cartouche
 @example
-2007-10-30 00:08:58 D [trace.c:1579:trace_opendir] trace: callid: 68 
-(*this=0x8059e40, loc=0x8091984 @{path=/iozone3_283, inode=0x8091f00@}, 
+2007-10-30 00:08:58 D [trace.c:1579:trace_opendir] trace: callid: 68
+(*this=0x8059e40, loc=0x8091984 @{path=/iozone3_283, inode=0x8091f00@},
  fd=0x8091d50)
 
-2007-10-30 00:08:58 D [trace.c:630:trace_opendir_cbk] trace: 
+2007-10-30 00:08:58 D [trace.c:630:trace_opendir_cbk] trace:
 (*this=0x8059e40, op_ret=4, op_errno=1, fd=0x8091d50)
 
-2007-10-30 00:08:58 D [trace.c:1602:trace_readdir] trace: callid: 69 
+2007-10-30 00:08:58 D [trace.c:1602:trace_readdir] trace: callid: 69
 (*this=0x8059e40, size=4096, offset=0 fd=0x8091d50)
 
-2007-10-30 00:08:58 D [trace.c:215:trace_readdir_cbk] trace: 
+2007-10-30 00:08:58 D [trace.c:215:trace_readdir_cbk] trace:
 (*this=0x8059e40, op_ret=0, op_errno=0, count=4)
 
-2007-10-30 00:08:58 D [trace.c:1624:trace_closedir] trace: callid: 71 
+2007-10-30 00:08:58 D [trace.c:1624:trace_closedir] trace: callid: 71
 (*this=0x8059e40, *fd=0x8091d50)
 
-2007-10-30 00:08:58 D [trace.c:809:trace_closedir_cbk] trace: 
+2007-10-30 00:08:58 D [trace.c:809:trace_closedir_cbk] trace:
 (*this=0x8059e40, op_ret=0, op_errno=1)
 @end example
 @end cartouche
@@ -1871,7 +1871,7 @@ scheduling on a single storage volume. Alternatively users can choose
 to have two separate volumes and hence two mount points, but the
 applications may demand a single storage system to host both.
 
-This document explains how to mix file level scheduling with stripe. 
+This document explains how to mix file level scheduling with stripe.
 
 @subsection Configuration Brief
 
@@ -1904,17 +1904,17 @@ addresses / access control fields to match your environment.
            type storage/posix
            option directory /export/for-unify
    end-volume
- 
+
    volume posix-stripe
            type storage/posix
            option directory /export/for-stripe
    end-volume
- 
+
    volume posix-namespace
            type storage/posix
            option directory /export/for-namespace
    end-volume
-  
+
    volume server
            type protocol/server
            option transport-type tcp
@@ -1963,7 +1963,7 @@ addresses / access control fields to match your environment.
      option remote-host 192.168.1.4
      option remote-subvolume posix-unify
    end-volume
- 
+
    volume client-stripe-1
      type protocol/client
      option transport-type tcp
@@ -1991,13 +1991,13 @@ addresses / access control fields to match your environment.
      option remote-host 192.168.1.4
      option remote-subvolume posix-stripe
    end-volume
-  
+
    volume unify
      type cluster/unify
      option scheduler rr
      subvolumes cluster-unify-1 cluster-unify-2 cluster-unify-3 cluster-unify-4
    end-volume
- 
+
    volume stripe
      type cluster/stripe
      option block-size *.img:2MB # All files ending with .img are striped with 2MB stripe block size.
@@ -2046,13 +2046,13 @@ concludes with the suggested procedure to report bugs in GlusterFS.
 @subsection Server errors
 
 @example
-glusterfsd: FATAL: could not open specfile: 
+glusterfsd: FATAL: could not open specfile:
 '/etc/glusterfs/glusterfsd.vol'
 @end example
 
-The GlusterFS server expects the volume specification file to be 
+The GlusterFS server expects the volume specification file to be
 at @command{/etc/glusterfs/glusterfsd.vol}. The example
-specification file will be installed as 
+specification file will be installed as
 @command{/etc/glusterfs/glusterfsd.vol.sample}. You need to edit
 it and rename it, or provide a different specification file using
 the @command{--spec-file} command line option (See @ref{Server}).
@@ -2060,7 +2060,7 @@ the @command{--spec-file} command line option (See @ref{Server}).
 @vskip 4ex
 
 @example
-gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfsd.log" 
+gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfsd.log"
              (Permission denied)
 @end example
 
@@ -2072,7 +2072,7 @@ file using the @command{--log-file} option (See @ref{Server}).
 @subsection Client errors
 
 @example
-fusermount: failed to access mountpoint /mnt: 
+fusermount: failed to access mountpoint /mnt:
             Transport endpoint is not connected
 @end example
 
@@ -2114,7 +2114,7 @@ port instead.
 @vskip 4ex
 
 @example
-gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfs.log" 
+gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfs.log"
              (Permission denied)
 @end example
 
diff --git a/doc/user-guide/xlator.odg b/doc/legacy/xlator.odg
index 179a65f6e..179a65f6e 100644
--- a/doc/user-guide/xlator.odg
+++ b/doc/legacy/xlator.odg
diff --git a/doc/user-guide/xlator.pdf b/doc/legacy/xlator.pdf
index a07e14d67..a07e14d67 100644
--- a/doc/user-guide/xlator.pdf
+++ b/doc/legacy/xlator.pdf
diff --git a/doc/logging.txt b/doc/logging.txt
new file mode 100644
index 000000000..b4ee45996
--- /dev/null
+++ b/doc/logging.txt
@@ -0,0 +1,66 @@
+
+New logging framework in glusterfs is targeted for end users like
+customers, community members, testers etc.  This aims to bring clear,
+understandable logs called user logs whereas the current logging are
+considered as developer logs.  The new framework brings with following
+features
+
+* Each message is logged with proper well defined error code and each
+  error code has well known error message.
+* A logbook has defined error code and error messages.  It helps to
+  keep track of possible causes and remedies
+* Log are sent to syslog.  The syslog application can be configured to
+  pass them to centralized logging system
+* It brings
+  - Remove repeated log messages
+  - Send alerts to users on certain events
+  - Run a program on events
+  - Call home service on events
+
+
+Log book:
+=========
+A log book is a JSON formatted file error-codes.json located in top
+level of glusterfs source repository.  At compile time, gen-headers.py
+generates libglusterfs/src/gf-error-codes.h using the log book and
+gf-error-codes.h.template file.  libglusterfs/src/gf-error-codes.h
+consists of header definitions and helper functions to get message by
+code for given locale. Currently it has _gf_get_message() function
+returns message for locale 'en'.
+
+New entry to log book is added like
+
+{
+    "IO_ERROR": {"code": 2233,
+                 "message": {"en": "I/O error occurred"}},
+    "SETUP_ERROR": {"code": 2240,
+                 "message": {"en": "Setup error"}},
+}
+
+
+Logging:
+========
+The framework provides two functions
+
+void gf_openlog (const char *ident, int option, int facility);
+void gf_syslog (int error_code, int facility_priority, char *format, ...);
+
+Consumers need to call gf_openlog() prior to gf_syslog() like the way
+traditional syslog function calls.  error_code is mandatory when using
+gf_syslog().  For example,
+
+gf_openlog (NULL, -1, -1);
+gf_syslog (GF_ERR_DEV, LOG_ERR, "error reading configuration file");
+
+The logs are sent in CEE format (http://cee.mitre.org/) to syslog.
+Its targeted to rsyslog syslog server.
+
+This log framework is enabled at compile time by default.  This can be
+disabled by passing '--disable-syslog' to ./configure or '--without
+syslog' to rpmbuild
+
+Even though its enabled at compile time, its required to have
+/etc/glusterfs/logger.conf file to make it into effect before starting
+gluster services
+
+Currently all gluster logs are sent with error code GF_ERR_DEV.
diff --git a/doc/mount.glusterfs.8 b/doc/mount.glusterfs.8
index 10e1d59f0..01b7f7554 100644
--- a/doc/mount.glusterfs.8
+++ b/doc/mount.glusterfs.8
@@ -1,19 +1,10 @@
-.\"  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+.\"  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
 .\"  This file is part of GlusterFS.
 .\"
-.\"  GlusterFS is free software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU General Public License as published
-.\"  by the Free Software Foundation; either version 3 of the License,
-.\"  or (at your option) any later version.
-.\"
-.\"  GlusterFS is distributed in the hope that it will be useful, but
-.\"  WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-.\"  General Public License for more details.
-.\"
-.\"  You should have received a copy of the GNU General Public License
-.\"  long with this program.  If not, see
-.\"  <http://www.gnu.org/licenses/>.
+.\"  This file is licensed to you under your choice of the GNU Lesser
+.\"  General Public License, version 3 or any later version (LGPLv3 or
+.\"  later), or the GNU General Public License, version 2 (GPLv2), in all
+.\"  cases as published by the Free Software Foundation.
 .\"
 .\"
 .\"
@@ -57,7 +48,7 @@ Mount the filesystem read-only
 Volume key or name of the volume file to be fetched from server
 .TP
 \fBtransport=\fRTRANSPORT-TYPE
-Transport type to get volume file from server [default: socket]
+Transport type to get volume file from server [default: tcp]
 .TP
 \fBvolume\-name=\fRVOLUME-NAME
 Volume name to be used for MOUNT-POINT [default: top most volume in
diff --git a/doc/qa/qa-client.vol b/doc/qa/qa-client.vol
deleted file mode 100644
index 176dda589..000000000
--- a/doc/qa/qa-client.vol
+++ /dev/null
@@ -1,170 +0,0 @@
-# This spec file should be used for testing before any release
-# 
-
-# 1st client
-volume client1
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-# option transport.ib-verbs.work-request-send-size  131072
-# option transport.ib-verbs.work-request-send-count 64
-# option transport.ib-verbs.work-request-recv-size  131072
-# option transport.ib-verbs.work-request-recv-count 64
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra1
-end-volume
-
-# 2nd client
-volume client2
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra2
-end-volume
-
-# 3rd client
-volume client3
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra3
-end-volume
-
-# 4th client
-volume client4
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra4
-end-volume
-
-# 5th client
-volume client5
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra5
-end-volume
-
-# 6th client
-volume client6
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra6
-end-volume
-
-# 7th client
-volume client7
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra7
-end-volume
-
-# 8th client 
-volume client8
-  type protocol/client
-  option transport-type tcp     # for TCP/IP transport
-# option transport-type ib-sdp  # for Infiniband transport
-# option transport-type ib-verbs # for ib-verbs transport 
-  option remote-host 127.0.0.1 
-  option remote-subvolume ra8
-end-volume
-
-# 1st Stripe (client1 client2)
-volume stripe1
-  type cluster/stripe
-  subvolumes client1 client2
-  option block-size 128KB  # all striped in 128kB block
-end-volume
-
-# 2st Stripe (client3 client4)
-volume stripe2
-  type cluster/stripe
-  subvolumes client3 client4
-  option block-size 128KB  # all striped in 128kB block
-end-volume
-
-# 3st Stripe (client5 client6)
-volume stripe3
-  type cluster/stripe
-  subvolumes client5 client6
-  option block-size 128KB  # all striped in 128kB block
-end-volume
-
-# 4st Stripe (client7 client8)
-volume stripe4
-  type cluster/stripe
-  subvolumes client7 client8
-  option block-size 128KB  # all striped in 128kB block
-end-volume
-
-
-# 1st replicate
-volume replicate1
-  type cluster/replicate
-  subvolumes stripe1 stripe2
-end-volume
-
-# 2nd replicate
-volume replicate2
-  type cluster/replicate
-  subvolumes stripe3 stripe4
-end-volume
-
-volume ns
-  type protocol/client
-  option transport-type tcp
-  option remote-host 127.0.0.1
-  option remote-subvolume brick-ns
-end-volume
-
-# Unify
-volume unify0
-  type cluster/unify
-  subvolumes replicate1 replicate2
-#  subvolumes stripe1 stripe3
-  option namespace ns
-  option scheduler rr # random # alu # nufa
-  option rr.limits.min-free-disk 1GB
-# option alu.order x
-# option alu.x.entry-threshold
-# option alu.x.exit-threshold
-end-volume
-
-
-# ==== Performance Translators ====
-# The default options for performance translators should be the best for 90+% of the cases
-volume iot
-  type performance/io-threads
-  subvolumes unify0
-end-volume
-
-volume wb
-  type performance/write-behind
-  subvolumes iot
-end-volume
-
-volume ioc
- type performance/io-cache
- subvolumes wb
-end-volume
-
-volume ra
-  type performance/read-ahead
-  subvolumes ioc
-end-volume
diff --git a/doc/qa/qa-high-avail-client.vol b/doc/qa/qa-high-avail-client.vol
deleted file mode 100644
index 69cb8dd30..000000000
--- a/doc/qa/qa-high-avail-client.vol
+++ /dev/null
@@ -1,17 +0,0 @@
-volume client
- type protocol/client
- option transport-type tcp
- option remote-host localhost
- option transport.socket.remote-port 7001
- option remote-subvolume server1-iot
-end-volume
-
-volume ra
- type performance/read-ahead
- subvolumes client
-end-volume
-
-volume wb
- type performance/write-behind
- subvolumes ra
-end-volume
diff --git a/doc/qa/qa-high-avail-server.vol b/doc/qa/qa-high-avail-server.vol
deleted file mode 100644
index 3556b9dae..000000000
--- a/doc/qa/qa-high-avail-server.vol
+++ /dev/null
@@ -1,344 +0,0 @@
-
-# -- server 1 --
-volume server1-posix1
- type storage/posix
- option directory /tmp/ha-export1/
-end-volume
-
-volume server1-ns1
- type storage/posix
- option directory /tmp/ha-export-ns1/
-end-volume
-
-volume server1-client2
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7002
- option remote-subvolume server2-posix2
-end-volume
-
-volume server1-ns2
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7002
- option remote-subvolume server2-ns2
-end-volume
-
-volume server1-client3
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7003
- option remote-subvolume server3-posix3
-end-volume
-
-volume server1-ns3
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7003
- option remote-subvolume server3-ns3
-end-volume
-
-volume server1-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server1-posix1
-end-volume
-
-
-volume server1-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server1-client2
-end-volume
-
-volume server1-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server1-client3
-end-volume
-
-volume server1-ns-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server1-ns1
-end-volume
-
-volume server1-ns-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server1-ns2
-end-volume
-
-volume server1-ns-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server1-ns3
-end-volume
-
-volume server1-ns-replicate
- type cluster/replicate
- subvolumes server1-ns-io1 server1-ns-io2 server1-ns-io3
-end-volume
-
-volume server1-storage-replicate
- type cluster/replicate
- subvolumes server1-io1 server1-io2 server1-io3
-end-volume
-
-volume server1-unify
- type cluster/unify
- #option self-heal off
- subvolumes server1-storage-replicate
- option namespace server1-ns-replicate
- option scheduler rr
-end-volume
-
-volume server1-iot
- type performance/io-threads
- option thread-count 8
- subvolumes server1-unify
-end-volume
-
-volume server1
- type protocol/server
- option transport-type tcp
- subvolumes server1-iot
- option transport.socket.listen-port 7001
- option auth.addr.server1-posix1.allow *
- option auth.addr.server1-ns1.allow *
- option auth.addr.server1-iot.allow * 
-end-volume
-
-
-# == Server2 ==
-volume server2-client1
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7001
- option remote-subvolume server1-posix1
-end-volume
-
-volume server2-ns1
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7001
- option remote-subvolume server1-ns1
-end-volume
-
-volume server2-posix2
- type storage/posix
- option directory /tmp/ha-export2/
-end-volume
-
-volume server2-ns2
- type storage/posix
- option directory /tmp/ha-export-ns2/
-end-volume
-
-volume server2-client3
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7003
- option remote-subvolume server3-posix3
-end-volume
-
-volume server2-ns3
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7003
- option remote-subvolume server3-ns3
-end-volume
-
-volume server2-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server2-client1
-end-volume
-
-
-volume server2-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server2-posix2
-end-volume
-
-volume server2-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server2-client3
-end-volume
-
-volume server2-ns-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server2-ns1
-end-volume
-
-volume server2-ns-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server2-ns2
-end-volume
-
-volume server2-ns-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server2-ns3
-end-volume
-
-volume server2-ns-replicate
- type cluster/replicate
- subvolumes server2-ns-io1 server2-ns-io2 server2-ns-io3
-end-volume
-
-volume server2-storage-replicate
- type cluster/replicate
- subvolumes server2-io2 server2-io3 server2-io1
-end-volume
-
-volume server2-unify
- type cluster/unify
- option self-heal off
- subvolumes server2-storage-replicate
- option namespace server2-ns-replicate
- option scheduler rr
-end-volume
-
-volume server2-iot
- type performance/io-threads
- option thread-count 8
- subvolumes server2-unify
-end-volume
-
-volume server2
- type protocol/server
- option transport-type tcp
- subvolumes server2-iot
- option transport.socket.listen-port 7002
- option auth.addr.server2-posix2.allow *
- option auth.addr.server2-ns2.allow *
- option auth.addr.server2-iot.allow * 
-end-volume
-
-# == server 3 ==
-volume server3-client1
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7001
- option remote-subvolume server1-posix1
-end-volume
-
-volume server3-ns1
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7001
- option remote-subvolume server1-ns1
-end-volume
-
-volume server3-client2
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7002
- option remote-subvolume server2-posix2
-end-volume
-
-volume server3-ns2
- type protocol/client
- option transport-type tcp
- option remote-host 127.0.0.1
- option transport.socket.remote-port 7002
- option remote-subvolume server2-ns2
-end-volume
-
-volume server3-posix3
- type storage/posix
- option directory /tmp/ha-export3/
-end-volume
-
-volume server3-ns3
- type storage/posix
- option directory /tmp/ha-export-ns3/
-end-volume
-
-volume server3-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server3-client1
-end-volume
-
-
-volume server3-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server3-client2
-end-volume
-
-volume server3-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server3-posix3
-end-volume
-
-volume server3-ns-io1
- type performance/io-threads
- option thread-count 8
- subvolumes server3-ns1
-end-volume
-
-volume server3-ns-io2
- type performance/io-threads
- option thread-count 8
- subvolumes server3-ns2
-end-volume
-
-volume server3-ns-io3
- type performance/io-threads
- option thread-count 8
- subvolumes server3-ns3
-end-volume
-
-volume server3-ns-replicate
- type cluster/replicate
- subvolumes server3-ns-io1 server3-ns-io2 server3-ns-io3
-end-volume
-
-volume server3-storage-replicate
- type cluster/replicate
- subvolumes server3-io3 server3-io2 server3-io1
-end-volume
-
-volume server3-unify
- type cluster/unify
- option self-heal off
- subvolumes server3-storage-replicate
- option namespace server3-ns-replicate
- option scheduler rr
-end-volume
-
-volume server3-iot
- type performance/io-threads
- option thread-count 8
- subvolumes server3-unify
-end-volume
-
-volume server3
- type protocol/server
- option transport-type tcp
- subvolumes server3-iot
- option transport.socket.listen-port 7003
- option auth.addr.server3-posix3.allow *
- option auth.addr.server3-ns3.allow *
- option auth.addr.server3-iot.allow * 
-end-volume
-
diff --git a/doc/qa/qa-server.vol b/doc/qa/qa-server.vol
deleted file mode 100644
index 1c245c324..000000000
--- a/doc/qa/qa-server.vol
+++ /dev/null
@@ -1,284 +0,0 @@
-# This spec file should be used for testing before any release
-# 
-
-# Namespace posix
-volume brick-ns
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export-ns        # Export this directory
-end-volume
-
-# 1st server
-
-volume brick1
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export1        # Export this directory
-end-volume
-
-# == Posix-Locks ==
- volume plocks1
-   type features/posix-locks
-#  option mandatory on
-   subvolumes brick1
- end-volume
-
-volume iot1
-  type performance/io-threads
-  subvolumes plocks1 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb1
-  type performance/write-behind
-  subvolumes iot1
-# option <key> <value>
-end-volume
-
-volume ra1
-  type performance/read-ahead
-  subvolumes wb1
-# option <key> <value>
-end-volume
-
-volume brick2
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export2        # Export this directory
-end-volume
-
-# == TrashCan Translator ==
-# volume trash2
-#   type features/trash
-#   option trash-dir /.trashcan
-#   subvolumes brick2
-# end-volume
-
-# == Posix-Locks ==
-volume plocks2
-  type features/posix-locks
-#  option <something> <something>
-  subvolumes brick2
-end-volume
-
-volume iot2
-  type performance/io-threads
-  subvolumes plocks2 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb2
-  type performance/write-behind
-  subvolumes iot2
-# option <key> <value>
-end-volume
-
-volume ra2
-  type performance/read-ahead
-  subvolumes wb2
-# option <key> <value>
-end-volume
-
-volume brick3
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export3        # Export this directory
-end-volume
-
-# == TrashCan Translator ==
-# volume trash3
-#   type features/trash
-#   option trash-dir /.trashcan
-#   subvolumes brick3
-# end-volume
-
-# == Posix-Locks ==
-volume plocks3
-  type features/posix-locks
-#  option <something> <something>
-  subvolumes brick3
-end-volume
-
-volume iot3
-  type performance/io-threads
-  subvolumes plocks3 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb3
-  type performance/write-behind
-  subvolumes iot3
-# option <key> <value>
-end-volume
-
-volume ra3
-  type performance/read-ahead
-  subvolumes wb3
-# option <key> <value>
-end-volume
-
-volume brick4
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export4        # Export this directory
-end-volume
-
-# == Posix-Locks ==
-volume plocks4
-  type features/posix-locks
-# option <something> <something>
-  subvolumes brick4
-end-volume
-
-volume iot4
-  type performance/io-threads
-  subvolumes plocks4 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb4
-  type performance/write-behind
-  subvolumes iot4
-# option <key> <value>
-end-volume
-
-volume ra4
-  type performance/read-ahead
-  subvolumes wb4
-# option <key> <value>
-end-volume
-
-volume brick5
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export5        # Export this directory
-end-volume
-
-
-# == Posix-Locks ==
-volume plocks5
-  type features/posix-locks
-# option <something> <something>
-  subvolumes brick5
-end-volume
-
-volume iot5
-  type performance/io-threads
-  subvolumes plocks5 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb5
-  type performance/write-behind
-  subvolumes iot5
-# option <key> <value>
-end-volume
-
-volume ra5
-  type performance/read-ahead
-  subvolumes wb5
-# option <key> <value>
-end-volume
-
-volume brick6
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export6        # Export this directory
-end-volume
-
-# == Posix-Locks ==
-volume plocks6
-  type features/posix-locks
-#   option <something> <something>
-  subvolumes brick6
-end-volume
-
-volume iot6
-  type performance/io-threads
-  subvolumes plocks6 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb6
-  type performance/write-behind
-  subvolumes iot6
-# option <key> <value>
-end-volume
-
-volume ra6
-  type performance/read-ahead
-  subvolumes wb6
-# option <key> <value>
-end-volume
-
-volume brick7
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export7        # Export this directory
-end-volume
-
-# == Posix-Locks ==
-volume plocks7
-  type features/posix-locks
-#   option <something> <something>
-  subvolumes brick7
-end-volume
-
-volume iot7
-  type performance/io-threads
-  subvolumes plocks7 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb7
-  type performance/write-behind
-  subvolumes iot7
-# option <key> <value>
-end-volume
-
-volume ra7
-  type performance/read-ahead
-  subvolumes wb7
-# option <key> <value>
-end-volume
-
-volume brick8
-  type storage/posix                   # POSIX FS translator
-  option directory /tmp/export8        # Export this directory
-end-volume
-
-# == Posix-Locks ==
-volume plocks8
-  type features/posix-locks
-#   option <something> <something>
-  subvolumes brick8
-end-volume
-
-volume iot8
-  type performance/io-threads
-  subvolumes plocks8 # change properly if above commented volumes needs to be included
-# option <key> <value>
-end-volume
-
-volume wb8
-  type performance/write-behind
-  subvolumes iot8
-# option <key> <value>
-end-volume
-
-volume ra8
-  type performance/read-ahead
-  subvolumes wb8
-# option <key> <value>
-end-volume
-
-volume server8
-  type protocol/server
-  subvolumes ra8 ra1 ra2 ra3 ra4 ra5 ra6 ra7 brick-ns
-  option transport-type tcp     # For TCP/IP transport
-# option transport-type ib-sdp  # For Infiniband transport
-# option transport-type ib-verbs # For ib-verbs transport
-  option client-volume-filename /examples/qa-client.vol
-  option auth.addr.ra1.allow * # Allow access to "stat8" volume
-  option auth.addr.ra2.allow * # Allow access to "stat8" volume
-  option auth.addr.ra3.allow * # Allow access to "stat8" volume
-  option auth.addr.ra4.allow * # Allow access to "stat8" volume
-  option auth.addr.ra5.allow * # Allow access to "stat8" volume
-  option auth.addr.ra6.allow * # Allow access to "stat8" volume
-  option auth.addr.ra7.allow * # Allow access to "stat8" volume
-  option auth.addr.ra8.allow * # Allow access to "stat8" volume
-  option auth.addr.brick-ns.allow * # Allow access to "stat8" volume
-end-volume
-
diff --git a/doc/split-brain.md b/doc/split-brain.md
new file mode 100644
index 000000000..b0d938e26
--- /dev/null
+++ b/doc/split-brain.md
@@ -0,0 +1,251 @@
+Steps to recover from File split-brain.
+======================================
+
+Quick Start:
+============
+1. Get the path of the file that is in split-brain:  
+>  It can be obtained either by  
+>       a) The command `gluster volume heal info split-brain`.  
+>       b) Identify the files for which file operations performed
+           from the client keep failing with Input/Output error.
+
+2. Close the applications that opened this file from the mount point.
+In case of VMs, they need to be powered-off.
+
+3. Decide on the correct copy:  
+> This is done by observing the afr changelog extended attributes of the file on
+the bricks using the getfattr command; then identifying the type of split-brain 
+(data split-brain, metadata split-brain, entry split-brain or split-brain due to
+gfid-mismatch); and finally determining which of the bricks contains the 'good copy'
+of the file.  
+>   `getfattr -d -m . -e hex <file-path-on-brick>`.  
+It is also possible that one brick might contain the correct data while the
+other might contain the correct metadata.
+
+4. Reset the relevant extended attribute on the brick(s) that contains the
+'bad copy' of the file data/metadata using the setfattr command.  
+>   `setfattr -n <attribute-name> -v <attribute-value> <file-path-on-brick>`
+
+5. Trigger self-heal on the file by performing lookup from the client:  
+>   `ls -l <file-path-on-gluster-mount>`
+
+Detailed Instructions for steps 3 through 5:  
+===========================================
+To understand how to resolve split-brain we need to know how to interpret the
+afr changelog extended attributes.
+
+Execute `getfattr -d -m . -e hex <file-path-on-brick>`
+
+* Example:  
+[root@store3 ~]# getfattr -d -e hex -m. brick-a/file.txt  
+\#file: brick-a/file.txt  
+security.selinux=0x726f6f743a6f626a6563745f723a66696c655f743a733000  
+trusted.afr.vol-client-2=0x000000000000000000000000  
+trusted.afr.vol-client-3=0x000000000200000000000000  
+trusted.gfid=0x307a5c9efddd4e7c96e94fd4bcdcbd1b  
+
+The extended attributes with `trusted.afr.<volname>-client-<subvolume-index>`
+are used by afr to maintain changelog of the file.The values of the
+`trusted.afr.<volname>-client-<subvolume-index>` are calculated by the glusterfs
+client (fuse or nfs-server) processes. When the glusterfs client modifies a file
+or directory, the client contacts each brick and updates the changelog extended 
+attribute according to the response of the brick.
+
+'subvolume-index' is nothing but (brick number - 1) in
+`gluster volume info <volname>` output.
+
+* Example:  
+[root@pranithk-laptop ~]# gluster volume info vol  
+ Volume Name: vol  
+ Type: Distributed-Replicate  
+ Volume ID: 4f2d7849-fbd6-40a2-b346-d13420978a01  
+ Status: Created  
+ Number of Bricks: 4 x 2 = 8  
+ Transport-type: tcp  
+ Bricks:  
+ brick-a: pranithk-laptop:/gfs/brick-a  
+ brick-b: pranithk-laptop:/gfs/brick-b  
+ brick-c: pranithk-laptop:/gfs/brick-c  
+ brick-d: pranithk-laptop:/gfs/brick-d  
+ brick-e: pranithk-laptop:/gfs/brick-e  
+ brick-f: pranithk-laptop:/gfs/brick-f  
+ brick-g: pranithk-laptop:/gfs/brick-g  
+ brick-h: pranithk-laptop:/gfs/brick-h  
+
+In the example above:  
+```
+Brick             |    Replica set        |    Brick subvolume index
+----------------------------------------------------------------------------
+-/gfs/brick-a     |       0               |       0
+-/gfs/brick-b     |       0               |       1
+-/gfs/brick-c     |       1               |       2
+-/gfs/brick-d     |       1               |       3
+-/gfs/brick-e     |       2               |       4
+-/gfs/brick-f     |       2               |       5
+-/gfs/brick-g     |       3               |       6
+-/gfs/brick-h     |       3               |       7
+```
+
+Each file in a brick maintains the changelog of itself and that of the files
+present in all the other bricks in it's replica set as seen by that brick.
+
+In the example volume given above, all files in brick-a will have 2 entries, 
+one for itself and the other for the file present in it's replica pair, i.e.brick-b:  
+trusted.afr.vol-client-0=0x000000000000000000000000 -->changelog for itself (brick-a)  
+trusted.afr.vol-client-1=0x000000000000000000000000 -->changelog for brick-b as seen by brick-a  
+
+Likewise, all files in brick-b will have:  
+trusted.afr.vol-client-0=0x000000000000000000000000 -->changelog for brick-a as seen by brick-b  
+trusted.afr.vol-client-1=0x000000000000000000000000 -->changelog for itself (brick-b)  
+
+The same can be extended for other replica pairs.  
+
+Interpreting Changelog (roughly pending operation count) Value:  
+Each extended attribute has a value which is 24 hexa decimal digits.  
+First 8 digits represent changelog of data. Second 8 digits represent changelog
+of metadata. Last 8 digits represent Changelog of directory entries.  
+
+Pictorially representing the same, we have:
+```
+0x 000003d7 00000001 00000000
+        |      |       |
+        |      |        \_ changelog of directory entries
+        |       \_ changelog of metadata
+         \ _ changelog of data
+```
+         
+
+For Directories metadata and entry changelogs are valid.
+For regular files data and metadata changelogs are valid.
+For special files like device files etc metadata changelog is valid.
+When a file split-brain happens it could be either data split-brain or
+meta-data split-brain or both. When a split-brain happens the changelog of the
+file would be something like this:  
+
+* Example:(Lets consider both data, metadata split-brain on same file).  
+[root@pranithk-laptop vol]# getfattr -d -m . -e hex /gfs/brick-?/a  
+getfattr: Removing leading '/' from absolute path names  
+\#file: gfs/brick-a/a  
+trusted.afr.vol-client-0=0x000000000000000000000000  
+trusted.afr.vol-client-1=0x000003d70000000100000000  
+trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57   
+\#file: gfs/brick-b/a  
+trusted.afr.vol-client-0=0x000003b00000000100000000  
+trusted.afr.vol-client-1=0x000000000000000000000000  
+trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57  
+
+###Observations:
+
+####According to changelog extended attributes on file /gfs/brick-a/a:  
+The first 8 digits of trusted.afr.vol-client-0 are all
+zeros (0x00000000................), and the first 8 digits of
+trusted.afr.vol-client-1 are not all zeros (0x000003d7................).
+So the changelog on /gfs/brick-a/a implies that some data operations succeeded
+on itself but failed on /gfs/brick-b/a.
+
+The second 8 digits of trusted.afr.vol-client-0 are
+all zeros (0x........00000000........), and the second 8 digits of
+trusted.afr.vol-client-1 are not all zeros (0x........00000001........).
+So the changelog on /gfs/brick-a/a implies that some metadata operations succeeded 
+on itself but failed on /gfs/brick-b/a.
+
+####According to Changelog extended attributes on file /gfs/brick-b/a:  
+The first 8 digits of trusted.afr.vol-client-0 are not all
+zeros (0x000003b0................), and the first 8 digits of
+trusted.afr.vol-client-1 are all zeros (0x00000000................).
+So the changelog on /gfs/brick-b/a implies that some data operations succeeded
+on itself but failed on /gfs/brick-a/a.
+
+The second 8 digits of trusted.afr.vol-client-0 are not
+all zeros (0x........00000001........), and the second 8 digits of
+trusted.afr.vol-client-1 are all zeros (0x........00000000........).
+So the changelog on /gfs/brick-b/a implies that some metadata operations succeeded
+on itself but failed on /gfs/brick-a/a.
+
+Since both the copies have data, metadata changes that are not on the other
+file, it is in both data and metadata split-brain.
+
+Deciding on the correct copy:  
+-----------------------------
+The user may have to inspect stat,getfattr output of the files to decide which 
+metadata to retain and contents of the file to decide which data to retain.
+Continuing with the example above, lets say we want to retain the data
+of /gfs/brick-a/a and metadata of /gfs/brick-b/a.
+
+Resetting the relevant changelogs to resolve the split-brain:  
+-------------------------------------------------------------
+For resolving data-split-brain:  
+We need to change the changelog extended attributes on the files as if some data
+operations succeeded on /gfs/brick-a/a but failed on /gfs/brick-b/a. But
+/gfs/brick-b/a should NOT have any changelog which says some data operations
+succeeded on /gfs/brick-b/a but failed on /gfs/brick-a/a. We need to reset the
+data part of the changelog on trusted.afr.vol-client-0 of /gfs/brick-b/a.
+
+For resolving metadata-split-brain:  
+We need to change the changelog extended attributes on the files as if some
+metadata operations succeeded on /gfs/brick-b/a but failed on /gfs/brick-a/a.
+But /gfs/brick-a/a should NOT have any changelog which says some metadata
+operations succeeded on /gfs/brick-a/a but failed on /gfs/brick-b/a.
+We need to reset metadata part of the changelog on
+trusted.afr.vol-client-1 of /gfs/brick-a/a
+
+So, the intended changes are:  
+On /gfs/brick-b/a:  
+For trusted.afr.vol-client-0  
+0x000003b00000000100000000 to 0x000000000000000100000000  
+(Note that the metadata part is still not all zeros)  
+Hence execute
+`setfattr -n trusted.afr.vol-client-0 -v 0x000000000000000100000000 /gfs/brick-b/a`
+
+On /gfs/brick-a/a:  
+For trusted.afr.vol-client-1  
+0x0000000000000000ffffffff to 0x000003d70000000000000000  
+(Note that the data part is still not all zeros)  
+Hence execute  
+`setfattr -n trusted.afr.vol-client-1 -v 0x000003d70000000000000000 /gfs/brick-a/a`
+
+Thus after the above operations are done, the changelogs look like this:  
+[root@pranithk-laptop vol]# getfattr -d -m . -e hex /gfs/brick-?/a  
+getfattr: Removing leading '/' from absolute path names  
+\#file: gfs/brick-a/a  
+trusted.afr.vol-client-0=0x000000000000000000000000  
+trusted.afr.vol-client-1=0x000003d70000000000000000  
+trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57  
+
+\#file: gfs/brick-b/a  
+trusted.afr.vol-client-0=0x000000000000000100000000  
+trusted.afr.vol-client-1=0x000000000000000000000000  
+trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57  
+
+
+Triggering Self-heal:
+---------------------
+Perform `ls -l <file-path-on-gluster-mount>` to trigger healing.
+
+Fixing Directory entry split-brain:
+----------------------------------
+Afr has the ability to conservatively merge different entries in the directories
+when there is a split-brain on directory.
+If on one brick directory 'd' has entries '1', '2' and has entries '3', '4' on
+the other brick then afr will merge all of the entries in the directory to have
+'1', '2', '3', '4' entries in the same directory.
+(Note: this may result in deleted files to re-appear in case the split-brain
+happens because of deletion of files on the directory)
+Split-brain resolution needs human intervention when there is at least one entry
+which has same file name but different gfid in that directory.
+Example:  
+On brick-a the directory has entries '1' (with gfid g1), '2' and on brick-b
+directory has entries '1' (with gfid g2) and '3'.
+These kinds of directory split-brains need human intervention to resolve.
+The user needs to remove either file '1' on brick-a or the file '1' on brick-b
+to resolve the split-brain. In addition, the corresponding gfid-link file also
+needs to be removed.The gfid-link files are present in the .glusterfs folder
+in the top-level directory of the brick. If the gfid of the file is
+0x307a5c9efddd4e7c96e94fd4bcdcbd1b (the trusted.gfid extended attribute got
+from the getfattr command earlier),the gfid-link file can be found at
+> /gfs/brick-a/.glusterfs/30/7a/307a5c9efddd4e7c96e94fd4bcdcbd1b
+
+####Word of caution:
+Before deleting the gfid-link, we have to ensure that there are no hard links
+to the file present on that brick. If hard-links exist,they must be deleted as
+well.
diff --git a/error-codes.json b/error-codes.json
new file mode 100644
index 000000000..5121049d3
--- /dev/null
+++ b/error-codes.json
@@ -0,0 +1,4 @@
+{
+    "ERR_DEV": {"code": 9999,
+		"message": {"en": "devel error"}}
+}
diff --git a/extras/LinuxRPM/Makefile.am b/extras/LinuxRPM/Makefile.am
new file mode 100644
index 000000000..1dafa982b
--- /dev/null
+++ b/extras/LinuxRPM/Makefile.am
@@ -0,0 +1,57 @@
+
+GFS_TAR = ../../glusterfs-$(VERSION).tar.gz
+
+.PHONY: all
+
+all:
+	@echo "To build RPMS run 'make glusterrpms'"
+
+.PHONY: glusterrpms prep srcrpm testsrpm clean
+
+glusterrpms: prep srcrpm rpms
+	-rm -rf rpmbuild
+
+prep:
+	if [ ! -e $(GFS_TAR) ]; then \
+		$(MAKE) -C ../.. dist; \
+	fi
+	-mkdir -p rpmbuild/SPECS
+	-mkdir -p rpmbuild/RPMS
+	-mkdir -p rpmbuild/SRPMS
+	-rm -rf rpmbuild/SOURCES
+	@if [ -d /d/cache/glusterfs -a -e /d/cache/glusterfs/sources ]; then \
+		echo "copying glusterfs rpm files from local cache..."  ; \
+		mkdir -p ./rpmbuild/SOURCES; \
+		cp /d/cache/glusterfs/* ./rpmbuild/SOURCES/ ; \
+	elif [ -x /usr/bin/git ]; then \
+		echo "fetching glusterfs rpm files from fedora git repo..."; \
+		cd ./rpmbuild && git clone git://pkgs.fedoraproject.org/glusterfs.git > /dev/null && mv glusterfs SOURCES; \
+	else \
+		echo "glusterfs rpm files not fetched, you don't have git installed!" ; \
+		exit 1 ; \
+	fi
+	cp ../../*.tar.gz ./rpmbuild/SOURCES
+	cp ../../glusterfs.spec ./rpmbuild/SPECS
+
+srcrpm:
+	rpmbuild --define '_topdir $(shell pwd)/rpmbuild' -bs rpmbuild/SPECS/glusterfs.spec
+	mv rpmbuild/SRPMS/* .
+
+rpms:
+	rpmbuild --define '_topdir $(shell pwd)/rpmbuild' -bb rpmbuild/SPECS/glusterfs.spec
+	mv rpmbuild/RPMS/*/* .
+
+# EPEL-5 does not like new versions of rpmbuild and requires some
+# _source_* defines
+
+testsrpm: prep
+	rpmbuild --define '_topdir $(shell pwd)/rpmbuild' \
+		--define '_source_payload w9.gzdio' \
+		--define '_source_filedigest_algorithm 1' \
+		-bs rpmbuild/SPECS/glusterfs.spec
+	mv rpmbuild/SRPMS/* ../..
+	-rm -rf rpmbuild
+
+clean:
+	-rm -rf rpmbuild
+	-rm -f *.rpm
diff --git a/extras/Makefile.am b/extras/Makefile.am
index 5c77ce73d..cf619329b 100644
--- a/extras/Makefile.am
+++ b/extras/Makefile.am
@@ -1,12 +1,20 @@
 
-docdir = $(datadir)/doc/glusterfs/
-EditorModedir = $(docdir)/
+EditorModedir = $(docdir)
 EditorMode_DATA = glusterfs-mode.el glusterfs.vim
 
-SUBDIRS = init.d benchmarking
+SUBDIRS = init.d systemd benchmarking hook-scripts $(OCF_SUBDIR) LinuxRPM geo-rep
+
+confdir = $(sysconfdir)/glusterfs
+conf_DATA = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \
+	logger.conf.example glusterfs-georep-logrotate
+
+voldir = $(sysconfdir)/glusterfs
+vol_DATA = glusterd.vol
 
 EXTRA_DIST = specgen.scm MacOSX/Portfile glusterfs-mode.el glusterfs.vim  \
 	migrate-unify-to-distribute.sh backend-xattr-sanitize.sh          \
 	backend-cleanup.sh disk_usage_sync.sh quota-remove-xattr.sh       \
-	quota-metadata-cleanup.sh glusterfs-logrotate
-
+	quota-metadata-cleanup.sh glusterfs-logrotate clear_xattrs.sh     \
+	group-virt.example glusterd-sysconfig gluster-rsyslog-7.2.conf    \
+	gluster-rsyslog-5.8.conf logger.conf.example glusterd.vol         \
+	glusterfs-georep-logrotate
diff --git a/extras/Ubuntu/README.Ubuntu b/extras/Ubuntu/README.Ubuntu
index 651bd046e..0c5b7828d 100644
--- a/extras/Ubuntu/README.Ubuntu
+++ b/extras/Ubuntu/README.Ubuntu
@@ -1,20 +1,24 @@
-Bug 3282 - Mounting from localhost in fstab fails at boot on ubuntu
-(http://bugs.gluster.com/show_bug.cgi?id=3282)
+Bug 765014 - Mounting from localhost in fstab fails at boot on ubuntu
+(https://bugzilla.redhat.com/show_bug.cgi?id=765014)
+(https://bugs.launchpad.net/ubuntu/+source/glusterfs/+bug/876648)
 
 Ubuntu uses upstart instead of init to bootstrap the system and it has a unique
-way of handling fstab, using a program called mountall(8).  As a result,
-glusterfs mounts in fstab are tried before the glusterd service is running.  In
-the case where the client is also a server and the volume is mounted from
-localhost, the mount fails at boot time.  An upstart job for glusterd is needed
-to correct this.
+way of handling fstab, using a program called mountall(8).  As a result of using
+a debian initscript to start glusterd, glusterfs mounts in fstab are tried before
+the glusterd service is running.  In the case where the client is also a server
+and the volume is mounted from localhost, the mount fails at boot time.  To
+correct this we need to launch glusterd using upstart and block the glusterfs
+mounting event until glusterd is started.
 
 The glusterd.conf file contains the necessary configuration for upstart to
 manage the glusterd service.  It should be placed in /etc/init/glusterd.conf
 on Ubuntu systems, and then the old initscript /etc/init.d/glusterd can be
-removed.
+removed.  An additional upstart job, mounting-glusterfs.conf, is also required
+to block mounting glusterfs volumes until the glusterd service is available.
+Both of these upstart jobs need to be placed in /etc/init to resolve the issue.
 
-It can also be added to Ubuntu deb packages by placing it in debian/upstart
-inside the source package (or debian/glusterd.upstart if the source package
-builds multiple binary packages.)
+Starting with Ubuntu 12.04, Precise Pangolin, these upstart jobs will be
+included with the glusterfs-server package in the Ubuntu repository. 
 
-This affects all versions of glusterfs on the ubuntu platform.
+This affects all versions of glusterfs on the Ubuntu platform since at least
+10.04, Lucid Lynx.
diff --git a/extras/Ubuntu/glusterd.conf b/extras/Ubuntu/glusterd.conf
index b987ffa13..aa99502b0 100644
--- a/extras/Ubuntu/glusterd.conf
+++ b/extras/Ubuntu/glusterd.conf
@@ -1,17 +1,10 @@
-# glusterd service upstart job
-#
-# Author: Louis Zuckerman <me@louiszuckerman.com>
+author "Louis Zuckerman <me@louiszuckerman.com>"
+description "GlusterFS Management Daemon"
 
-description     "GlusterFS Management Daemon"
-
-start on (local-filesystems and net-device-up IFACE=lo and net-device-up IFACE=eth0) or (mounting TYPE=glusterfs)
+start on runlevel [2345]
 stop on runlevel [016]
 
-respawn
-
-exec /usr/sbin/glusterd -N -p /var/run/glusterd.pid
+expect fork
 
-post-start script
-    sleep 1
-end script
+exec /usr/sbin/glusterd -p /var/run/glusterd.pid
 
diff --git a/extras/Ubuntu/mounting-glusterfs.conf b/extras/Ubuntu/mounting-glusterfs.conf
new file mode 100644
index 000000000..3c59c0f63
--- /dev/null
+++ b/extras/Ubuntu/mounting-glusterfs.conf
@@ -0,0 +1,7 @@
+author "Louis Zuckerman <me@louiszuckerman.com>"
+description "Block the mounting event for glusterfs filesystems until glusterd is running"
+
+start on mounting TYPE=glusterfs
+task
+exec start wait-for-state WAIT_FOR=glusterd WAITER=mounting-glusterfs
+
diff --git a/extras/benchmarking/Makefile.am b/extras/benchmarking/Makefile.am
index 04cc06182..bfcc59277 100644
--- a/extras/benchmarking/Makefile.am
+++ b/extras/benchmarking/Makefile.am
@@ -1,7 +1,5 @@
 
-docdir = $(datadir)/doc/$(PACKAGE_NAME)/benchmarking
-
-benchmarkingdir = $(docdir)
+benchmarkingdir = $(docdir)/benchmarking
 
 benchmarking_DATA = rdd.c glfs-bm.c README launch-script.sh local-script.sh
 
diff --git a/extras/benchmarking/glfs-bm.c b/extras/benchmarking/glfs-bm.c
index 035d055df..dc717f33c 100644
--- a/extras/benchmarking/glfs-bm.c
+++ b/extras/benchmarking/glfs-bm.c
@@ -1,21 +1,12 @@
-/*                                                                  
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com> 
-  This file is part of GlusterFS.                             
-  GlusterFS is free software; you can redistribute it and/or modify 
-  it under the terms of the GNU General Public License as published 
-  by the Free Software Foundation; either version 3 of the License, 
-  or (at your option) any later version.                            
-
-  GlusterFS is distributed in the hope that it will be useful, but 
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details. 
-                                                                                   
-  You should have received a copy of the GNU General Public License 
-  along with this program.  If not, see 
-  <http://www.gnu.org/licenses/>.  
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #define _GNU_SOURCE
 #define __USE_FILE_OFFSET64
 #define _FILE_OFFSET_BITS 64
diff --git a/extras/benchmarking/rdd.c b/extras/benchmarking/rdd.c
index b06333370..a667c6a1d 100644
--- a/extras/benchmarking/rdd.c
+++ b/extras/benchmarking/rdd.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <sys/stat.h>
 #include <sys/types.h>
diff --git a/extras/generate-xdr-files.sh b/extras/generate-xdr-files.sh
index e52321cd3..bc02f77c9 100755
--- a/extras/generate-xdr-files.sh
+++ b/extras/generate-xdr-files.sh
@@ -22,22 +22,13 @@ append_licence_header ()
 
     cat >$dst_file <<EOF
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
diff --git a/extras/geo-rep/Makefile.am b/extras/geo-rep/Makefile.am
new file mode 100644
index 000000000..fc5f56d54
--- /dev/null
+++ b/extras/geo-rep/Makefile.am
@@ -0,0 +1,2 @@
+EXTRA_DIST = gsync-sync-gfid.c gsync-upgrade.sh generate-gfid-file.sh \
+	get-gfid.sh slave-upgrade.sh
diff --git a/extras/geo-rep/generate-gfid-file.sh b/extras/geo-rep/generate-gfid-file.sh
new file mode 100644
index 000000000..c6739fbf1
--- /dev/null
+++ b/extras/geo-rep/generate-gfid-file.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+#Usage: generate-gfid-file.sh <master-volfile-server:master-volume> <path-to-get-gfid.sh> <output-file>
+
+function get_gfids()
+{
+    GET_GFID_CMD=$1
+    OUTPUT_FILE=$2
+    find . -exec $GET_GFID_CMD {} \; > $OUTPUT_FILE
+}
+
+function mount_client()
+{
+    local T; # temporary mount
+    local i; # inode number
+
+    VOLFILE_SERVER=$1;
+    VOLUME=$2;
+    GFID_CMD=$3;
+    OUTPUT=$4;
+
+    T=$(mktemp -d);
+
+    glusterfs -s $VOLFILE_SERVER --volfile-id $VOLUME $T;
+
+    i=$(stat -c '%i' $T);
+
+    [ "x$i" = "x1" ] || fatal "could not mount volume $MASTER on $T";
+
+    cd $T;
+
+    get_gfids $GFID_CMD $OUTPUT
+
+    cd -;
+
+    umount $T || fatal "could not umount $MASTER from $T";
+
+    rmdir $T || warn "rmdir of $T failed";
+}
+
+
+function main()
+{
+    SLAVE=$1
+    GET_GFID_CMD=$2
+    OUTPUT=$3
+
+    VOLFILE_SERVER=`echo $SLAVE | sed -e 's/\(.*\):.*/\1/'`
+    VOLUME_NAME=`echo $SLAVE | sed -e 's/.*:\(.*\)/\1/'`
+
+    mount_client $VOLFILE_SERVER $VOLUME_NAME $GET_GFID_CMD $OUTPUT
+}
+
+main "$@";
diff --git a/extras/geo-rep/get-gfid.sh b/extras/geo-rep/get-gfid.sh
new file mode 100755
index 000000000..a4d609b0b
--- /dev/null
+++ b/extras/geo-rep/get-gfid.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+ATTR_STR=`getfattr -h $1 -n glusterfs.gfid.string`
+GLFS_PATH=`echo $ATTR_STR | sed -e 's/# file: \(.*\) glusterfs.gfid.string*/\1/g'`
+GFID=`echo $ATTR_STR | sed -e 's/.*glusterfs.gfid.string="\(.*\)"/\1/g'`
+
+echo "$GFID $GLFS_PATH"
diff --git a/extras/geo-rep/gsync-sync-gfid.c b/extras/geo-rep/gsync-sync-gfid.c
new file mode 100644
index 000000000..601f4720e
--- /dev/null
+++ b/extras/geo-rep/gsync-sync-gfid.c
@@ -0,0 +1,106 @@
+
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <limits.h>
+#include <sys/types.h>
+#include <attr/xattr.h>
+#include <libgen.h>
+#include <ctype.h>
+#include <stdlib.h>
+
+#ifndef UUID_CANONICAL_FORM_LEN
+#define UUID_CANONICAL_FORM_LEN 36
+#endif
+
+#ifndef GF_FUSE_AUX_GFID_HEAL
+#define GF_FUSE_AUX_GFID_HEAL    "glusterfs.gfid.heal"
+#endif
+
+#define GLFS_LINE_MAX           (PATH_MAX + (2 * UUID_CANONICAL_FORM_LEN))
+
+int
+main (int argc, char *argv[])
+{
+        char *file                = NULL;
+        char *tmp = NULL;
+        char *tmp1 = NULL;
+        char *parent_dir = NULL;
+        char *gfid                = NULL;
+        char *bname = NULL;
+        int   ret                 = -1;
+        int len = 0;
+        FILE *fp                  = NULL;
+        char  line[GLFS_LINE_MAX] = {0,};
+        char *path = NULL;
+        void  *blob               = NULL;
+        void  *tmp_blob               = NULL;
+
+        if (argc != 2) {
+                /* each line in the file has the following format
+                 * uuid-in-canonical-form path-relative-to-gluster-mount.
+                 * Both uuid and relative path are from master mount.
+                 */
+                fprintf (stderr, "usage: %s <file-of-paths-to-be-synced>\n",
+                         argv[0]);
+                goto out;
+        }
+
+        file = argv[1];
+
+        fp = fopen (file, "r");
+        if (fp == NULL) {
+                fprintf (stderr, "cannot open %s for reading (%s)\n",
+                         file, strerror (errno));
+                goto out;
+        }
+
+        while (fgets (line, GLFS_LINE_MAX, fp) != NULL) {
+                tmp = line;
+                path = gfid = line;
+
+                path += UUID_CANONICAL_FORM_LEN + 1;
+
+                while(isspace (*path))
+                        path++;
+
+                if ((strlen (line) < GLFS_LINE_MAX) &&
+                    (line[strlen (line) - 1] == '\n'))
+                        line[strlen (line) - 1] = '\0';
+
+                line[UUID_CANONICAL_FORM_LEN] = '\0';
+
+                tmp = strdup (path);
+                tmp1 = strdup (path);
+                parent_dir = dirname (tmp);
+                bname = basename (tmp1);
+
+                /* gfid + '\0' + bname + '\0' */
+                len = UUID_CANONICAL_FORM_LEN + 1 + strlen (bname) + 1;
+
+                blob = calloc (1, len);
+
+                memcpy (blob, gfid, UUID_CANONICAL_FORM_LEN);
+
+                tmp_blob = blob + UUID_CANONICAL_FORM_LEN + 1;
+
+                memcpy (tmp_blob, bname, strlen (bname));
+
+                ret = setxattr (parent_dir, GF_FUSE_AUX_GFID_HEAL, blob, len,
+                                0);
+                if (ret < 0) {
+                        fprintf (stderr, "setxattr on %s/%s failed (%s)\n",
+                                 parent_dir, bname, strerror (errno));
+                }
+                memset (line, 0, GLFS_LINE_MAX);
+
+                free (blob);
+                free (tmp); free (tmp1);
+                blob = NULL;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
diff --git a/extras/geo-rep/gsync-upgrade.sh b/extras/geo-rep/gsync-upgrade.sh
new file mode 100644
index 000000000..b17948736
--- /dev/null
+++ b/extras/geo-rep/gsync-upgrade.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+#usage: gsync-upgrade.sh <slave-volfile-server:slave-volume> <gfid-file>
+#                        <path-to-gsync-sync-gfid> <ssh-identity-file>
+#<slave-volfile-server>: a machine on which gluster cli can fetch slave volume info.
+#                        slave-volfile-server defaults to localhost.
+#
+#<gfid-file>: a file containing paths and their associated gfids
+#            on master. The paths are relative to master mount point
+#            (not absolute). An example extract of <gfid-file> can be,
+#
+#            <extract>
+#            22114455-57c5-46e9-a783-c40f83a72b09 /dir
+#            25772386-3eb8-4550-a802-c3fdc938ca80 /dir/file
+#            </extract>
+#
+#<ssh-identity-file>: file from which the identity (private key) for public key authentication is read.
+
+SLAVE_MOUNT='/tmp/glfs_slave'
+
+function SSH()
+{
+    HOST=$1
+    SSHKEY=$2
+
+    shift 2
+
+    ssh -qi $SSHKEY \
+        -oPasswordAuthentication=no \
+        -oStrictHostKeyChecking=no \
+        "$HOST" "$@";
+}
+
+function get_bricks()
+{
+    SSHKEY=$3
+
+    SSH $1 $SSHKEY "gluster volume info $2" | grep -E 'Brick[0-9]+' | sed -e 's/[^:]*:\(.*\)/\1/g'
+}
+
+function cleanup_brick()
+{
+    HOST=$1
+    BRICK=$2
+    SSHKEY=$3
+
+    # TODO: write a C program to receive a list of files and does cleanup on
+    # them instead of spawning a new setfattr process for each file if
+    # performance is bad.
+    SSH -i $SSHKEY $HOST  "rm -rf $BRICK/.glusterfs/* && find $BRICK -exec setfattr -x trusted.gfid {} \;"
+}
+
+function cleanup_slave()
+{
+    SSHKEY=$2
+
+    VOLFILE_SERVER=`echo $1 | sed -e 's/\(.*\):.*/\1/'`
+    VOLUME_NAME=`echo $1 | sed -e 's/.*:\(.*\)/\1/'`
+
+    BRICKS=`get_bricks $VOLFILE_SERVER $VOLUME_NAME $SSHKEY`
+
+    for i in $BRICKS; do
+	HOST=`echo $i | sed -e 's/\(.*\):.*/\1/'`
+	BRICK=`echo $i | sed -e 's/.*:\(.*\)/\1/'`
+	cleanup_brick $HOST $BRICK $SSHKEY
+    done
+
+    SSH -i $SSHKEY $VOLFILE_SERVER "gluster --mode=script volume stop $VOLUME_NAME; gluster volume start $VOLUME_NAME";
+
+}
+
+function mount_client()
+{
+    local T; # temporary mount
+    local i; # inode number
+    GFID_FILE=$3
+    SYNC_CMD=$4
+
+    T=$(mktemp -d);
+
+    glusterfs --aux-gfid-mount -s $1 --volfile-id $2 $T;
+
+    i=$(stat -c '%i' $T);
+
+    [ "x$i" = "x1" ] || fatal "could not mount volume $MASTER on $T";
+
+    cd $T;
+
+    $SYNC_CMD $GFID_FILE
+
+    cd -;
+
+    umount -l $T || fatal "could not umount $MASTER from $T";
+
+    rmdir $T || warn "rmdir of $T failed";
+}
+
+function sync_gfids()
+{
+    SLAVE=$1
+    GFID_FILE=$2
+
+    SLAVE_VOLFILE_SERVER=`echo $SLAVE | sed -e 's/\(.*\):.*/\1/'`
+    SLAVE_VOLUME_NAME=`echo $SLAVE | sed -e 's/.*:\(.*\)/\1/'`
+
+    if [ "x$SLAVE_VOLFILE_SERVER" = "x" ]; then
+        SLAVE_VOLFILE_SERVER="localhost"
+    fi
+
+    mount_client $SLAVE_VOLFILE_SERVER $SLAVE_VOLUME_NAME $GFID_FILE $3
+}
+
+function upgrade()
+{
+    SLAVE=$1
+    GFID_FILE=$2
+    SYNC_CMD=$3
+    SSHKEY=$4
+
+    cleanup_slave $SLAVE $SSHKEY
+    sync_gfids $SLAVE $GFID_FILE $SYNC_CMD
+}
+
+upgrade "$@"
diff --git a/extras/geo-rep/slave-upgrade.sh b/extras/geo-rep/slave-upgrade.sh
new file mode 100644
index 000000000..6198f408a
--- /dev/null
+++ b/extras/geo-rep/slave-upgrade.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+#usage: slave-upgrade.sh <volfile-server:volname> <gfid-file>
+#                        <path-to-gsync-sync-gfid>
+#<slave-volfile-server>: a machine on which gluster cli can fetch slave volume info.
+#                        slave-volfile-server defaults to localhost.
+#
+#<gfid-file>: a file containing paths and their associated gfids
+#            on master. The paths are relative to master mount point
+#            (not absolute). An example extract of <gfid-file> can be,
+#
+#            <extract>
+#            22114455-57c5-46e9-a783-c40f83a72b09 /dir
+#            25772386-3eb8-4550-a802-c3fdc938ca80 /dir/file
+#            </extract>
+
+function get_bricks()
+{
+    gluster volume info $1 | grep -E 'Brick[0-9]+' | sed -e 's/[^:]*:\(.*\)/\1/g'
+}
+
+function cleanup_brick()
+{
+    HOST=$1
+    BRICK=$2
+
+    # TODO: write a C program to receive a list of files and does cleanup on
+    # them instead of spawning a new setfattr process for each file if
+    # performance is bad.
+    ssh $HOST "rm -rf $BRICK/.glusterfs/* && find $BRICK -exec setfattr -x trusted.gfid {} \; 2>/dev/null"
+}
+
+function cleanup_slave()
+{
+    VOLUME_NAME=`echo $1 | sed -e 's/.*:\(.*\)/\1/'`
+
+    BRICKS=`get_bricks $VOLUME_NAME`
+
+    for i in $BRICKS; do
+	HOST=`echo $i | sed -e 's/\(.*\):.*/\1/'`
+	BRICK=`echo $i | sed -e 's/.*:\(.*\)/\1/'`
+	cleanup_brick $HOST $BRICK
+    done
+
+    # Now restart the volume
+    gluster --mode=script volume stop $VOLUME_NAME;
+    gluster volume start $VOLUME_NAME;
+}
+
+function mount_client()
+{
+    local T; # temporary mount
+    local i; # inode number
+
+    VOLUME_NAME=$2;
+    GFID_FILE=$3
+    SYNC_CMD=$4
+
+    T=$(mktemp -d);
+
+    glusterfs --aux-gfid-mount -s $1 --volfile-id $VOLUME_NAME $T;
+
+    i=$(stat -c '%i' $T);
+
+    cd $T;
+
+    $SYNC_CMD $GFID_FILE
+
+    cd -;
+
+    umount $T || fatal "could not umount $MASTER from $T";
+
+    rmdir $T || warn "rmdir of $T failed";
+}
+
+function sync_gfids()
+{
+    SLAVE=$1
+    GFID_FILE=$2
+    SYNC_CMD=$3
+
+    SLAVE_VOLFILE_SERVER=`echo $SLAVE | sed -e 's/\(.*\):.*/\1/'`
+    SLAVE_VOLUME_NAME=`echo $SLAVE | sed -e 's/.*:\(.*\)/\1/'`
+
+    if [ "x$SLAVE_VOLFILE_SERVER" = "x" ]; then
+        SLAVE_VOLFILE_SERVER="localhost"
+    fi
+
+    mount_client $SLAVE_VOLFILE_SERVER $SLAVE_VOLUME_NAME $GFID_FILE $SYNC_CMD
+}
+
+function upgrade()
+{
+    SLAVE=$1
+    GFID_FILE=$2
+    SYNC_CMD=$3
+
+    cleanup_slave $SLAVE
+
+    sync_gfids $SLAVE $GFID_FILE $SYNC_CMD
+}
+
+upgrade "$@"
diff --git a/extras/gluster-rsyslog-5.8.conf b/extras/gluster-rsyslog-5.8.conf
new file mode 100644
index 000000000..2519999bc
--- /dev/null
+++ b/extras/gluster-rsyslog-5.8.conf
@@ -0,0 +1,51 @@
+##### gluster.conf #####
+
+#
+## If you want to log every message to the log file instead of
+## intelligently suppressing repeated messages, set off to
+## RepeatedMsgReduction.  This change requires rsyslog restart
+## (eg. run 'service rsyslog restart')
+#
+#$RepeatedMsgReduction off
+$RepeatedMsgReduction on
+
+#
+## The mmcount module provides the capability to count log messages by 
+## severity or json property of given app-name.  The count value is added
+## into the log message as json property named '$msgid'
+#
+$ModLoad mmcount
+$mmcountKey gf_code # start counting value of gf_code
+
+$template Glusterfsd_dynLogFile,"/var/log/glusterfs/bricks/%app-name%.log"
+$template Gluster_dynLogFile,"/var/log/glusterfs/%app-name%.log"
+
+$template GLFS_Template,"%msgid%/%syslogfacility-text:::uppercase%/%syslogseverity-text:::uppercase% [%TIMESTAMP:::date-rfc3339%] %msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+
+#
+## Pass logs to mmcount if app-name is 'gluster'
+#
+if $app-name contains 'gluster' then :mmcount:
+
+if $app-name contains 'glusterfsd' then ?Glusterfsd_dynLogFile;GLFS_Template
+if $app-name contains 'gluster' and not ( $app-name contains 'glusterfsd' ) then ?Gluster_dynLogFile;GLFS_Template
+
+#
+## Sample configuration to send a email alert for every 50th mmcount
+#
+#$ModLoad ommail
+#$ActionMailSMTPServer smtp.example.com
+#$ActionMailFrom rsyslog@example.com
+#$ActionMailTo glusteradmin@example.com
+#$template mailSubject,"50th message of gf_code=9999 on %hostname%"
+#$template mailBody,"RSYSLOG Alert\r\nmsg='%msg%'"
+#$ActionMailSubject mailSubject
+#$ActionExecOnlyOnceEveryInterval 30
+#if $app-name == 'glusterfsd' and $msgid != 0 and $msgid % 50 == 0 \
+#then :ommail:;RSYSLOG_SyslogProtocol23Format
+#
+
+#
+## discard logs where app-name is 'gluster' as we processed already
+#
+if $app-name contains 'gluster' then ~
diff --git a/extras/gluster-rsyslog-7.2.conf b/extras/gluster-rsyslog-7.2.conf
new file mode 100644
index 000000000..8b2841543
--- /dev/null
+++ b/extras/gluster-rsyslog-7.2.conf
@@ -0,0 +1,76 @@
+##### gluster.conf #####
+#
+## If you want to log every message to the log file instead of
+## intelligently suppressing repeated messages, set off to
+## RepeatedMsgReduction.  This change requires rsyslog restart
+## (eg. run 'service rsyslog restart')
+#
+#$RepeatedMsgReduction off
+$RepeatedMsgReduction on
+
+$ModLoad mmjsonparse
+*.* :mmjsonparse:
+
+#
+## The mmcount module provides the capability to count log messages by 
+## severity or json property of given app-name.  The count value is added
+## into the log message as json property named 'mmcount'
+##
+## More info at http://www.rsyslog.com/doc/mmcount.html
+#
+#module(load="mmcount")
+#action(type="mmcount" appname="glusterd" key="!gf_code") # count each value of gf_code of appname glusterd
+#action(type="mmcount" appname="glusterfsd" key="!gf_code") # count each value of gf_code of appname glusterfsd
+#action(type="mmcount" appname="glusterfs" key="!gf_code") # count each value of gf_code of appname glusterfs
+
+template (name="Glusterfsd_dynLogFile" type="string" string="/var/log/glusterfs/bricks/%app-name%.log")
+template (name="Gluster_dynLogFile" type="string" string="/var/log/glusterfs/%app-name%.log")
+
+template(name="GLFS_template" type="list") {
+   property(name="$!mmcount")
+   constant(value="/")
+   property(name="syslogfacility-text" caseConversion="upper")
+   constant(value="/")
+   property(name="syslogseverity-text" caseConversion="upper")
+   constant(value=" ")
+   constant(value="[")
+   property(name="timereported" dateFormat="rfc3339")
+   constant(value="] ")
+   constant(value="[")
+   property(name="$!gf_code")
+   constant(value="] ")
+   constant(value="[")
+   property(name="$!gf_message")
+   constant(value="] ")
+   property(name="$!msg")
+   constant(value="\n")
+}
+
+if $app-name contains 'glusterfsd' then {
+   action(type="omfile"
+          DynaFile="Glusterfsd_dynLogFile"
+          Template="GLFS_template")
+   stop
+}
+
+if $app-name contains 'gluster' then {
+   action(type="omfile"
+          DynaFile="Gluster_dynLogFile"
+          Template="GLFS_template")
+   stop
+}
+
+#
+## send email for every 50th mmcount
+#$ModLoad ommail
+#if $app-name == 'glusterfsd' and $!mmcount <> 0 and $!mmcount % 50 == 0 then {
+#   $ActionMailSMTPServer smtp.example.com
+#   $ActionMailFrom rsyslog@example.com
+#   $ActionMailTo glusteradmin@example.com
+#   $template mailSubject,"50th message of gf_code=9999 on %hostname%"
+#   $template mailBody,"RSYSLOG Alert\r\nmsg='%msg%'"
+#   $ActionMailSubject mailSubject
+#   $ActionExecOnlyOnceEveryInterval 30
+#   :ommail:;RSYSLOG_SyslogProtocol23Format
+#}
+#
diff --git a/extras/glusterd-sysconfig b/extras/glusterd-sysconfig
new file mode 100644
index 000000000..8237c5711
--- /dev/null
+++ b/extras/glusterd-sysconfig
@@ -0,0 +1,6 @@
+## Set custom log file and log level (bellow are defaults)
+# LOG_FILE='/var/log/glusterfs/glusterd.log'
+# LOG_LEVEL='INFO'
+
+## Set custom options for glusterd
+# GLUSTERD_OPTIONS=''
diff --git a/doc/glusterd.vol b/extras/glusterd.vol
index 809042cbd..9bac52ab7 100644
--- a/doc/glusterd.vol
+++ b/extras/glusterd.vol
@@ -1,8 +1,9 @@
 volume management
     type mgmt/glusterd
-    option working-directory /etc/glusterd
+    option working-directory /var/lib/glusterd
     option transport-type socket,rdma
     option transport.socket.keepalive-time 10
     option transport.socket.keepalive-interval 2
     option transport.socket.read-fail-log off
+#   option base-port 49152
 end-volume
diff --git a/extras/glusterfs-georep-logrotate b/extras/glusterfs-georep-logrotate
new file mode 100644
index 000000000..6a69ab1e3
--- /dev/null
+++ b/extras/glusterfs-georep-logrotate
@@ -0,0 +1,18 @@
+
+rotate 52
+missingok
+
+compress
+delaycompress
+notifempty
+
+/var/log/glusterfs/geo-replication/*/*.log {
+}
+
+
+/var/log/glusterfs/geo-replication-slaves/*.log {
+}
+
+
+/var/log/glusterfs/geo-replication-slaves/*/*.log {
+}
diff --git a/extras/glusterfs.vim b/extras/glusterfs.vim
index 62102c1ee..899cc6551 100644
--- a/extras/glusterfs.vim
+++ b/extras/glusterfs.vim
@@ -1,20 +1,11 @@
 " glusterfs.vim: GNU Vim Syntax file for GlusterFS .vol specification
-" Copyright (c) 2017-2011 Gluster, Inc. <http://www.gluster.com>
-" This file is part of GlusterFS.
+"  Copyright (c) 2007777777Red Hat, Inc. <http://www.redhat.com>
+"  This file is part of GlusterFS.
 "
-" GlusterFS is free software; you can redistribute it and/or modify
-" it under the terms of the GNU General Public License as published
-" by the Free Software Foundation; either version 3 of the License,
-" or (at your option) any later version.
-"
-" GlusterFS is distributed in the hope that it will be useful, but
-" WITHOUT ANY WARRANTY; without even the implied warranty of
-" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-" General Public License for more details.
-"
-" You should have received a copy of the GNU General Public License
-" along with this program.  If not, see
-" <http://www.gnu.org/licenses/>.
+"  This file is licensed to you under your choice of the GNU Lesser
+"  General Public License, version 3 or any later version (LGPLv3 or
+"  later), or the GNU General Public License, version 2 (GPLv2), in all
+"  cases as published by the Free Software Foundation.
 "
 " Last Modified: Wed Aug  1 00:47:10 IST 2007
 " Version: 0.8 
diff --git a/extras/gnfs-loganalyse.py b/extras/gnfs-loganalyse.py
index 6b24e5a71..71e79b6be 100644
--- a/extras/gnfs-loganalyse.py
+++ b/extras/gnfs-loganalyse.py
@@ -1,21 +1,13 @@
 #!/bin/python
 """
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
 """
 
 import os
diff --git a/extras/group-virt.example b/extras/group-virt.example
new file mode 100644
index 000000000..7dc777f2d
--- /dev/null
+++ b/extras/group-virt.example
@@ -0,0 +1,6 @@
+quick-read=off
+read-ahead=off
+io-cache=off
+stat-prefetch=off
+eager-lock=enable
+remote-dio=enable
diff --git a/extras/hook-scripts/Makefile.am b/extras/hook-scripts/Makefile.am
new file mode 100644
index 000000000..f6bded20c
--- /dev/null
+++ b/extras/hook-scripts/Makefile.am
@@ -0,0 +1 @@
+EXTRA_DIST = S29CTDBsetup.sh S30samba-start.sh S30samba-stop.sh S30samba-set.sh S56glusterd-geo-rep-create-post.sh
diff --git a/extras/hook-scripts/S29CTDBsetup.sh b/extras/hook-scripts/S29CTDBsetup.sh
new file mode 100644
index 000000000..e256be1f3
--- /dev/null
+++ b/extras/hook-scripts/S29CTDBsetup.sh
@@ -0,0 +1,69 @@
+#! /bin/bash
+#non-portable - RHS-2.0 only
+# - The script mounts the 'meta-vol' on start 'event' on a known
+#   directory (eg. /gluster/lock)
+# - Adds the necessary configuration changes for ctdb in smb.conf and
+#   restarts smb service.
+# - P.S: There are other 'tasks' that need to be done outside this script
+#   to get CTDB based failover up and running.
+
+SMB_CONF=/etc/samba/smb.conf
+
+CTDB_MNT=/gluster/lock
+PROGNAME="ctdb"
+OPTSPEC="volname:"
+VOL=
+# $META is the volume that will be used by CTDB as a shared filesystem.
+# It is not desirable to use this volume for storing 'data' as well.
+# META is set to 'all' (viz. a keyword and hence not a legal volume name)
+# to prevent the script from running for volumes it was not intended.
+# User needs to set META to the volume that serves CTDB lockfile.
+META="all"
+
+function sighup_samba () {
+        pid=`cat /var/run/smbd.pid`
+        if [ $pid != " " ]
+        then
+                kill -HUP $pid;
+        else
+                /etc/init.d/smb start
+        fi
+}
+
+function parse_args () {
+        ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+        eval set -- "$ARGS"
+
+        while true; do
+            case $1 in
+                --volname)
+                    shift
+                    VOL=$1
+                ;;
+
+                *)
+                     shift
+                     break
+                ;;
+
+            esac
+
+            shift
+        done
+}
+
+function add_glusterfs_ctdb_options () {
+        PAT="Share Definitions"
+        GLUSTER_CTDB_CONFIG="# ctdb config for glusterfs\n\tclustering = yes\n\tidmap backend = tdb2\n\tprivate dir = "$CTDB_MNT"\n"
+
+        sed -i /"$PAT"/i\ "$GLUSTER_CTDB_CONFIG" $SMB_CONF
+}
+
+parse_args $@
+if [ "$META" = "$VOL" ]
+then
+        add_glusterfs_ctdb_options
+        sighup_samba
+        mount -t glusterfs `hostname`:$VOL "$CTDB_MNT" &
+fi
+
diff --git a/extras/hook-scripts/S30samba-set.sh b/extras/hook-scripts/S30samba-set.sh
new file mode 100755
index 000000000..6b11f5a4f
--- /dev/null
+++ b/extras/hook-scripts/S30samba-set.sh
@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#Need to be copied to hooks/<HOOKS_VER>/set/post/
+
+#TODO: All gluster and samba paths are assumed for fedora like systems.
+#Some efforts are required to make it work on other distros.
+
+#The preferred way of creating a smb share of a gluster volume has changed.
+#The old method was to create a fuse mount of the volume and share the mount
+#point through samba.
+#
+#New method eliminates the requirement of fuse mount and changes in fstab.
+#glusterfs_vfs plugin for samba makes call to libgfapi to access the volume.
+#
+#This hook script enables user to enable or disable smb share by volume set
+#option. Keys "user.cifs" and "user.smb" both are valid, but user.smb is
+#preferred.
+
+
+PROGNAME="Ssamba-set"
+OPTSPEC="volname:"
+VOL=
+
+enable_smb=""
+
+function parse_args () {
+        ARGS=$(getopt -l $OPTSPEC  -o "o" -name $PROGNAME $@)
+        eval set -- "$ARGS"
+
+        while true; do
+            case $1 in
+            --volname)
+                shift
+                VOL=$1
+                ;;
+            *)
+                shift
+                for pair in $@; do
+                        read key value < <(echo "$pair" | tr "=" " ")
+                        case "$key" in
+                            "user.cifs")
+                                    enable_smb=$value
+                                    ;;
+                            "user.smb")
+                                    enable_smb=$value
+                                    ;;
+                            *)
+                                    ;;
+                        esac
+                done
+
+                shift
+                break
+                ;;
+            esac
+            shift
+        done
+}
+
+function add_samba_share () {
+        volname=$1
+        STRING="\n[gluster-$volname]\n"
+        STRING+="comment = For samba share of volume $volname\n"
+        STRING+="vfs objects = glusterfs\n"
+        STRING+="glusterfs:volume = $volname\n"
+        STRING+="glusterfs:logfile = /var/log/samba/glusterfs-$volname.%%M.log\n"
+        STRING+="glusterfs:loglevel = 7\n"
+        STRING+="path = /\n"
+        STRING+="read only = no\n"
+        STRING+="guest ok = yes\n"
+        printf "$STRING"  >> /etc/samba/smb.conf
+}
+
+function sighup_samba () {
+        pid=`cat /var/run/smbd.pid`
+        if [ "x$pid" != "x" ]
+        then
+                kill -HUP "$pid";
+        else
+                /etc/init.d/smb start
+        fi
+}
+
+function del_samba_share () {
+        volname=$1
+        sed -i "/\[gluster-$volname\]/,/^$/d" /etc/samba/smb.conf
+}
+
+function is_volume_started () {
+        volname=$1
+        echo "$(grep status /var/lib/glusterd/vols/"$volname"/info |\
+                cut -d"=" -f2)"
+}
+
+parse_args $@
+if [ "0" = $(is_volume_started "$VOL") ]; then
+    exit 0
+fi
+
+if [ "$enable_smb" = "enable" ]; then
+    if ! grep --quiet "\[gluster-$VOL\]" /etc/samba/smb.conf ; then
+            add_samba_share $VOL
+            sighup_samba
+    fi
+
+elif [ "$enable_smb" = "disable" ]; then
+    del_samba_share $VOL
+    sighup_samba
+fi
diff --git a/extras/hook-scripts/S30samba-start.sh b/extras/hook-scripts/S30samba-start.sh
new file mode 100755
index 000000000..34fde0ef8
--- /dev/null
+++ b/extras/hook-scripts/S30samba-start.sh
@@ -0,0 +1,110 @@
+#!/bin/bash
+
+#Need to be copied to hooks/<HOOKS_VER>/start/post
+
+#TODO: All gluster and samba paths are assumed for fedora like systems.
+#Some efforts are required to make it work on other distros.
+
+#The preferred way of creating a smb share of a gluster volume has changed.
+#The old method was to create a fuse mount of the volume and share the mount
+#point through samba.
+#
+#New method eliminates the requirement of fuse mount and changes in fstab.
+#glusterfs_vfs plugin for samba makes call to libgfapi to access the volume.
+#
+#This hook script automagically creates shares for volume on every volume start
+#event by adding the entries in smb.conf file and sending SIGHUP to samba.
+#
+#In smb.conf:
+#glusterfs vfs plugin has to be specified as required vfs object.
+#Path value is relative to the root of gluster volume;"/" signifies complete
+#volume.
+
+PROGNAME="Ssamba-start"
+OPTSPEC="volname:"
+VOL=
+CONFIGFILE=
+LOGFILEBASE=
+PIDDIR=
+
+function parse_args () {
+        ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+        eval set -- "$ARGS"
+
+        while true; do
+        case $1 in
+        --volname)
+         shift
+         VOL=$1
+         ;;
+        *)
+         shift
+         break
+         ;;
+        esac
+        shift
+        done
+}
+
+function find_config_info () {
+        cmdout=`smbd -b | grep smb.conf`
+        if [ $? -ne 0 ];then
+                echo "Samba is not installed"
+                exit 1
+        fi
+        CONFIGFILE=`echo $cmdout | awk {'print $2'}`
+        PIDDIR=`smbd -b | grep PIDDIR | awk {'print $2'}`
+        LOGFILEBASE=`smbd -b | grep 'LOGFILEBASE' | awk '{print $2}'`
+}
+
+function add_samba_share () {
+        volname=$1
+        STRING="\n[gluster-$volname]\n"
+        STRING+="comment = For samba share of volume $volname\n"
+        STRING+="vfs objects = glusterfs\n"
+        STRING+="glusterfs:volume = $volname\n"
+        STRING+="glusterfs:logfile = $LOGFILEBASE/glusterfs-$volname.%%M.log\n"
+        STRING+="glusterfs:loglevel = 7\n"
+        STRING+="path = /\n"
+        STRING+="read only = no\n"
+        STRING+="guest ok = yes\n"
+        printf "$STRING"  >> ${CONFIGFILE}
+}
+
+function sighup_samba () {
+        pid=`cat ${PIDDIR}/smbd.pid`
+        if [ "x$pid" != "x" ]
+        then
+                kill -HUP "$pid";
+        else
+                /etc/init.d/smb condrestart
+        fi
+}
+
+function get_smb () {
+        volname=$1
+        uservalue=
+
+        usercifsvalue=$(grep user.cifs /var/lib/glusterd/vols/"$volname"/info |\
+                        cut -d"=" -f2)
+        usersmbvalue=$(grep user.smb /var/lib/glusterd/vols/"$volname"/info |\
+                       cut -d"=" -f2)
+
+        if [[ $usercifsvalue = "disable" || $usersmbvalue = "disable" ]]; then
+                uservalue="disable"
+        fi
+        echo "$uservalue"
+}
+
+parse_args $@
+if [ $(get_smb "$VOL") = "disable" ]; then
+        exit 0
+fi
+
+#Find smb.conf, smbd pid directory and smbd logfile path
+find_config_info
+
+if ! grep --quiet "\[gluster-$VOL\]" ${CONFIGFILE} ; then
+        add_samba_share $VOL
+        sighup_samba
+fi
diff --git a/extras/hook-scripts/S30samba-stop.sh b/extras/hook-scripts/S30samba-stop.sh
new file mode 100755
index 000000000..8950eea43
--- /dev/null
+++ b/extras/hook-scripts/S30samba-stop.sh
@@ -0,0 +1,71 @@
+#! /bin/bash
+
+#Need to be copied to hooks/<HOOKS_VER>/stop/pre
+
+#TODO: All gluster and samba paths are assumed for fedora like systems.
+#Some efforts are required to make it work on other distros.
+
+#The preferred way of creating a smb share of a gluster volume has changed.
+#The old method was to create a fuse mount of the volume and share the mount
+#point through samba.
+#
+#New method eliminates the requirement of fuse mount and changes in fstab.
+#glusterfs_vfs plugin for samba makes call to libgfapi to access the volume.
+#
+#This hook script automagically removes shares for volume on every volume stop
+#event by removing the volume related entries(if any) in smb.conf file.
+
+PROGNAME="Ssamba-stop"
+OPTSPEC="volname:"
+VOL=
+CONFIGFILE=
+PIDDIR=
+
+function parse_args () {
+        ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+        eval set -- "$ARGS"
+
+        while true; do
+        case $1 in
+        --volname)
+         shift
+         VOL=$1
+         ;;
+        *)
+         shift
+         break
+         ;;
+        esac
+        shift
+        done
+}
+
+function find_config_info () {
+        cmdout=`smbd -b | grep smb.conf`
+        if [ $? -ne 0 ];then
+                echo "Samba is not installed"
+                exit 1
+        fi
+        CONFIGFILE=`echo $cmdout | awk {'print $2'}`
+        PIDDIR=`smbd -b | grep PIDDIR | awk {'print $2'}`
+}
+
+function del_samba_share () {
+        volname=$1
+        sed -i "/\[gluster-$volname\]/,/^$/d" ${CONFIGFILE}
+}
+
+function sighup_samba () {
+        pid=`cat ${PIDDIR}/smbd.pid`
+        if [ "x$pid" != "x" ]
+        then
+                kill -HUP $pid;
+        else
+                /etc/init.d/smb condrestart
+        fi
+}
+
+parse_args $@
+find_config_info
+del_samba_share $VOL
+sighup_samba
diff --git a/extras/hook-scripts/S40ufo-stop.py b/extras/hook-scripts/S40ufo-stop.py
new file mode 100755
index 000000000..107f19683
--- /dev/null
+++ b/extras/hook-scripts/S40ufo-stop.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+import os
+from optparse import OptionParser
+
+if __name__ == '__main__':
+    # check if swift is installed
+    try:
+        from gluster.swift.common.Glusterfs import get_mnt_point, unmount
+    except ImportError:
+        import sys
+        sys.exit("Openstack Swift does not appear to be installed properly");
+
+    op = OptionParser(usage="%prog [options...]")
+    op.add_option('--volname', dest='vol', type=str)
+    op.add_option('--last', dest='last', type=str)
+    (opts, args) = op.parse_args()
+
+
+    mnt_point = get_mnt_point(opts.vol)
+    if mnt_point:
+        unmount(mnt_point)
+    else:
+        sys.exit("get_mnt_point returned none for mount point")
diff --git a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh
new file mode 100755
index 000000000..1369c22fc
--- /dev/null
+++ b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+key_val_pair1=`echo $2 | cut -d ' ' -f 1`
+key_val_pair2=`echo $2 | cut -d ' ' -f 2`
+key_val_pair3=`echo $2 | cut -d ' ' -f 3`
+
+key=`echo $key_val_pair1 | cut -d '=' -f 1`
+val=`echo $key_val_pair1 | cut -d '=' -f 2`
+if [ "$key" != "is_push_pem" ]; then
+    exit;
+fi
+if [ "$val" != '1' ]; then
+    exit;
+fi
+
+key=`echo $key_val_pair2 | cut -d '=' -f 1`
+val=`echo $key_val_pair2 | cut -d '=' -f 2`
+if [ "$key" != "pub_file" ]; then
+    exit;
+fi
+if [ "$val" == "" ]; then
+    exit;
+fi
+pub_file=`echo $val`
+pub_file_tmp=`echo $val`_tmp
+
+key=`echo $key_val_pair3 | cut -d '=' -f 1`
+val=`echo $key_val_pair3 | cut -d '=' -f 2`
+if [ "$key" != "slave_ip" ]; then
+    exit;
+fi
+if [ "$val" == "" ]; then
+    exit;
+fi
+slave_ip=`echo $val`
+
+if [ -f $pub_file ]; then
+    scp $pub_file $slave_ip:$pub_file_tmp
+    ssh $slave_ip "mv $pub_file_tmp $pub_file"
+    ssh $slave_ip "gluster system:: copy file /geo-replication/common_secret.pem.pub > /dev/null"
+    ssh $slave_ip "gluster system:: execute add_secret_pub > /dev/null"
+fi
diff --git a/extras/init.d/Makefile.am b/extras/init.d/Makefile.am
index 66715f431..38898fddd 100644
--- a/extras/init.d/Makefile.am
+++ b/extras/init.d/Makefile.am
@@ -1,19 +1,22 @@
 
-EXTRA_DIST = glusterd-Debian glusterd-Redhat glusterd-SuSE glusterd.plist
+EXTRA_DIST = glusterd-Debian glusterd-Redhat glusterd-SuSE glusterd.plist rhel5-load-fuse.modules
 
 CLEANFILES = 
 
-initdir = @initdir@
-launchddir = @launchddir@
+INIT_DIR = @initdir@
+SYSTEMD_DIR = @systemddir@
+LAUNCHD_DIR = @launchddir@
 
 $(GF_DISTRIBUTION): 
-	$(mkdir_p) $(DESTDIR)$(initdir)	
-	$(INSTALL_PROGRAM) glusterd-$(GF_DISTRIBUTION) $(DESTDIR)$(initdir)/glusterd
+	@if [ ! -d $(SYSTEMD_DIR) ]; then \
+		$(mkdir_p) $(DESTDIR)$(INIT_DIR); \
+		$(INSTALL_PROGRAM) glusterd-$(GF_DISTRIBUTION) $(DESTDIR)$(INIT_DIR)/glusterd; \
+	fi
 
 install-exec-local: $(GF_DISTRIBUTION)
 
 install-data-local: 
 if GF_DARWIN_HOST_OS
-	$(mkdir_p) $(DESTDIR)$(launchddir)
-	$(INSTALL_PROGRAM) glusterd.plist $(DESTDIR)$(launchddir)/com.gluster.glusterd.plist
+	$(mkdir_p) $(DESTDIR)$(LAUNCHD_DIR)
+	$(INSTALL_PROGRAM) glusterd.plist $(DESTDIR)$(LAUNCHD_DIR)/com.gluster.glusterd.plist
 endif
diff --git a/extras/init.d/glusterd-Redhat.in b/extras/init.d/glusterd-Redhat.in
index cf9d390f0..e320708ae 100755
--- a/extras/init.d/glusterd-Redhat.in
+++ b/extras/init.d/glusterd-Redhat.in
@@ -1,89 +1,142 @@
 #!/bin/bash
 #
-# chkconfig: 35 20 80
-# description: Gluster File System service for volume management
+# glusterd   Startup script for the glusterfs server
 #
+# chkconfig:   - 20 80
+# description: Clustered file-system server
 
-# Get function from functions library
+### BEGIN INIT INFO
+# Provides: glusterd
+# Required-Start: $local_fs $network
+# Required-Stop: $local_fs $network
+# Should-Start: 
+# Should-Stop: 
+# Default-Start: 
+# Default-Stop: 0 1 2 3 4 5 6
+# Short-Description: glusterfs server
+# Description:       Clustered file-system server
+### END INIT INFO
+#
+
+# Source function library.
 . /etc/rc.d/init.d/functions
 
 BASE=glusterd
-PIDFILE=/var/run/$BASE.pid
+
+# Fedora File System Layout dictates /run
+[ -e /run ] && RUNDIR="/run"
+PIDFILE="${RUNDIR:-/var/run}/${BASE}.pid"
+
 PID=`test -f $PIDFILE && cat $PIDFILE`
+
+# Overwriteable from sysconfig
+LOG_LEVEL=''
+LOG_FILE=''
+GLUSTERD_OPTIONS=''
+GLUSTERD_NOFILE='65536'
+
+[ -f /etc/sysconfig/${BASE} ] && . /etc/sysconfig/${BASE}
+
+[ ! -z $LOG_LEVEL ] && GLUSTERD_OPTIONS="${GLUSTERD_OPTIONS} --log-level ${LOG_LEVEL}"
+[ ! -z $LOG_FILE ] && GLUSTERD_OPTIONS="${GLUSTERD_OPTIONS} --log-file ${LOG_FILE}"
+
 GLUSTERFSD=glusterfsd
 GLUSTERFS=glusterfs
 GLUSTERD_BIN=@prefix@/sbin/$BASE
-GLUSTERD_OPTS="--pid-file=$PIDFILE"
+GLUSTERD_OPTS="--pid-file=$PIDFILE ${GLUSTERD_OPTIONS}"
 GLUSTERD="$GLUSTERD_BIN $GLUSTERD_OPTS"
 RETVAL=0
 
+LOCKFILE=/var/lock/subsys/${BASE}
+
 # Start the service $BASE
 start()
 {
-       pidofproc -p $PIDFILE $GLUSTERD_BIN &> /dev/null
-       status=$?
-       if [ $status -eq 0 ]; then
+       if pidofproc -p $PIDFILE $GLUSTERD_BIN &> /dev/null; then
            echo "glusterd service is already running with pid $PID"
-           exit 1
+           return 0
        else
+           ulimit -n $GLUSTERD_NOFILE
            echo -n $"Starting $BASE:"
            daemon $GLUSTERD
            RETVAL=$?
            echo
-           [ $RETVAL -ne 0 ] && exit $RETVAL
+           [ $RETVAL -eq 0 ] && touch $LOCKFILE
+           return $RETVAL
        fi
-
 }
 
 # Stop the service $BASE
 stop()
 {
     echo -n $"Stopping $BASE:"
-    pidofproc -p $PIDFILE $GLUSTERD_BIN &> /dev/null
-    status=$?
-    if [ $status -eq 0 ]; then
+    if pidofproc -p $PIDFILE $GLUSTERD_BIN &> /dev/null; then
         killproc -p $PIDFILE $BASE
-        [ -w $PIDFILE ] && rm -f $PIDFILE
     else
         killproc $BASE
     fi
+    RETVAL=$?
+    [ $RETVAL -eq 0 ] && rm -f $LOCKFILE
+    return $RETVAL
+}
 
-    echo
-    pidof -c -o %PPID -x $GLUSTERFSD &> /dev/null
-    [ $? -eq 0 ] &&  killproc $GLUSTERFSD &> /dev/null
+restart()
+{
+    stop
+    start
+}
 
-       #pidof -c -o %PPID -x $GLUSTERFS &> /dev/null
-       #[ $? -eq 0 ] &&  killproc $GLUSTERFS &> /dev/null
+reload()
+{
+    restart
+}
 
-    if [ -f /etc/glusterd/nfs/run/nfs.pid ] ;then
-        pid=`cat /etc/glusterd/nfs/run/nfs.pid`;
-        cmd=`ps -p $pid -o comm=`
+force_reload()
+{
+    restart
+}
 
-        if [ $cmd == "glusterfs" ]; then
-            kill `cat /etc/glusterd/nfs/run/nfs.pid`
-        fi
-    fi
+rh_status()
+{
+    status $BASE
+}
+
+rh_status_q()
+{
+    rh_status &>/dev/null
 }
 
 
 ### service arguments ###
 case $1 in
     start)
-        start
+        rh_status_q && exit 0
+        $1
         ;;
     stop)
-        stop
+        rh_status_q || exit 0
+        $1
+        ;;
+    restart)
+        $1
+        ;;
+    reload)
+        rh_status_q || exit 7
+        $1
+        ;;
+    force-reload)
+        force_reload
         ;;
     status)
-        status $BASE
+        rh_status
         ;;
-    restart)
-        $0 stop
-        $0 start
+    condrestart|try-restart)
+        rh_status_q || exit 0
+        restart
         ;;
     *)
-        echo $"Usage: $0 {start|stop|status|restart}."
+        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
         exit 1
 esac
 
-exit 0
+exit $?
diff --git a/extras/init.d/glusterd-SuSE.in b/extras/init.d/glusterd-SuSE.in
index 16cf8de6a..6259bab00 100755
--- a/extras/init.d/glusterd-SuSE.in
+++ b/extras/init.d/glusterd-SuSE.in
@@ -2,8 +2,8 @@
 #
 ### BEGIN INIT INFO
 # Provides:       glusterd
-# Required-Start: $local_fs $network
-# Required-Stop:
+# Required-Start: $remote_fs $network
+# Required-Stop: $remote_fs $network
 # Default-Start:  3 5
 # Default-Stop:
 # Short-Description: Gluster File System service for volume management
@@ -61,13 +61,17 @@ case $1 in
 	fi
 	rc_status -v
        ;;
+ reload)
+       rc_failed 3
+       rc_status -v
+       ;;
  restart)
        $0 stop
        $0 start
        rc_status
        ;;
  *)
-       echo $"Usage: $0 {start|stop|status|restart}."
+       echo $"Usage: $0 {start|stop|status|reload|restart}."
        exit 1
 esac
 
diff --git a/extras/init.d/rhel5-load-fuse.modules b/extras/init.d/rhel5-load-fuse.modules
new file mode 100755
index 000000000..ee194db99
--- /dev/null
+++ b/extras/init.d/rhel5-load-fuse.modules
@@ -0,0 +1,7 @@
+#!/bin/sh
+#
+# fusermount-glusterfs requires the /dev/fuse character device. The fuse module
+# provides this and is loaded on demand in newer Linux distributions.
+#
+
+[ -c /dev/fuse ] || /sbin/modprobe fuse
diff --git a/extras/logger.conf.example b/extras/logger.conf.example
new file mode 100644
index 000000000..248be5bda
--- /dev/null
+++ b/extras/logger.conf.example
@@ -0,0 +1,13 @@
+#
+# Sample logger.conf file to configure enhanced Logging in GlusterFS
+#
+# To enable enhanced logging capabilities,
+#
+# 1. rename this file to /etc/glusterfs/logger.conf
+#
+# 2. rename /etc/rsyslog.d/gluster.conf.example to
+#    /etc/rsyslog.d/gluster.conf
+#
+# This change requires restart of all gluster services/volumes and
+# rsyslog.
+#
diff --git a/extras/ocf/Makefile.am b/extras/ocf/Makefile.am
new file mode 100644
index 000000000..c49a835fb
--- /dev/null
+++ b/extras/ocf/Makefile.am
@@ -0,0 +1,11 @@
+EXTRA_DIST = glusterd.in volume.in
+
+# The root of the OCF resource agent hierarchy
+# Per the OCF standard, it's always "lib",
+# not "lib64" (even on 64-bit platforms).
+ocfdir = $(prefix)/lib/ocf
+
+# The ceph provider directory
+radir = $(ocfdir)/resource.d/$(PACKAGE_NAME)
+
+ra_SCRIPTS = glusterd volume
diff --git a/extras/ocf/glusterd.in b/extras/ocf/glusterd.in
new file mode 100755
index 000000000..c119a285d
--- /dev/null
+++ b/extras/ocf/glusterd.in
@@ -0,0 +1,212 @@
+#!/bin/sh
+#
+# glusterd
+#
+# Description:  Manages a glusterd server as a (typically cloned)
+#               HA resource
+#
+# Authors:      Florian Haas (hastexo Professional Services GmbH)
+#
+# License:      GNU General Public License (GPL)
+
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+# Convenience variables
+# When sysconfdir and localstatedir aren't passed in as
+# configure flags, they're defined in terms of prefix
+prefix=@prefix@
+#######################################################################
+
+
+OCF_RESKEY_binary_default="glusterd"
+OCF_RESKEY_pid_default="@localstatedir@/run/glusterd.pid"
+OCF_RESKEY_socket_default=""
+OCF_RESKEY_additional_parameters_default=""
+
+: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}}
+: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}}
+
+glusterd_meta_data() {
+    cat <<EOF
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="glusterd" version="0.1">
+  <version>0.1</version>
+  <longdesc lang="en">
+  </longdesc>
+  <shortdesc lang="en">Manages a Gluster server</shortdesc>
+  <parameters>
+    <parameter name="binary">
+      <longdesc lang="en">
+      Name of the glusterd executable. Specify a full absolute
+      path if the binary is not in your \$PATH.
+      </longdesc>
+      <shortdesc lang="en">glusterd executable</shortdesc>
+      <content type="string" default="$OCF_RESKEY_binary_default"/>
+    </parameter>
+    <parameter name="pid">
+      <longdesc lang="en">
+      Path to the glusterd PID file.
+      </longdesc>
+      <shortdesc lang="en">PID file</shortdesc>
+      <content type="string" default="$OCF_RESKEY_pid_default"/>
+    </parameter>
+    <parameter name="socket">
+      <longdesc lang="en">
+      Path to the glusterd UNIX socket file. If unspecified,
+      glusterd will not listen on any socket.
+      </longdesc>
+      <shortdesc lang="en">Socket file</shortdesc>
+      <content type="string"/>
+    </parameter>
+  </parameters>
+  <actions>
+    <action name="start"        timeout="20" />
+    <action name="stop"         timeout="20" />
+    <action name="monitor"      timeout="20" interval="10" />
+    <action name="reload"       timeout="20" />
+    <action name="meta-data"    timeout="5" />
+    <action name="validate-all"   timeout="20" />
+  </actions>
+</resource-agent>
+EOF
+
+}
+
+glusterd_start() {
+    local glusterd_options
+    # exit immediately if configuration is not valid
+    glusterd_validate_all || exit $?
+
+    # if resource is already running, bail out early
+    if glusterd_monitor; then
+        ocf_log info "Resource is already running"
+        return $OCF_SUCCESS
+    fi
+
+    # actually start up the resource here (make sure to immediately
+    # exit with an $OCF_ERR_ error code if anything goes seriously
+    # wrong)
+    glusterd_options="-p $OCF_RESKEY_pid"
+    if [ -n "$OCF_RESKEY_socket" ]; then
+	glusterd_options="$glusterd_options -S $OCF_RESKEY_socket"
+    fi
+    if [ -n "$OCF_RESKEY_additional_parameters" ]; then
+	glusterd_options="$glusterd_options $OCF_RESKEY_additional_parameters"
+    fi
+
+    ocf_run $OCF_RESKEY_binary $glusterd_options || exit $OCF_ERR_GENERIC
+
+    # After the resource has been started, check whether it started up
+    # correctly. If the resource starts asynchronously, the agent may
+    # spin on the monitor function here -- if the resource does not
+    # start up within the defined timeout, the cluster manager will
+    # consider the start action failed
+    while ! glusterd_monitor; do
+        ocf_log debug "Resource has not started yet, waiting"
+        sleep 1
+    done
+
+    # only return $OCF_SUCCESS if _everything_ succeeded as expected
+    return $OCF_SUCCESS
+}
+
+glusterd_stop() {
+    local rc
+    local pid
+
+    # exit immediately if configuration is not valid
+    glusterd_validate_all || exit $?
+
+    glusterd_monitor
+    rc=$?
+    case "$rc" in
+        "$OCF_SUCCESS")
+            # Currently running. Normal, expected behavior.
+            ocf_log debug "Resource is currently running"
+            ;;
+        "$OCF_NOT_RUNNING")
+            # Currently not running. Nothing to do.
+            ocf_log info "Resource is already stopped"
+            return $OCF_SUCCESS
+            ;;
+    esac
+
+    # actually shut down the resource here (make sure to immediately
+    # exit with an $OCF_ERR_ error code if anything goes seriously
+    # wrong)
+    pid=`cat $OCF_RESKEY_pid`
+    ocf_run kill -s TERM $pid || exit OCF_ERR_GENERIC
+
+    # After the resource has been stopped, check whether it shut down
+    # correctly. If the resource stops asynchronously, the agent may
+    # spin on the monitor function here -- if the resource does not
+    # shut down within the defined timeout, the cluster manager will
+    # consider the stop action failed
+    while glusterd_monitor; do
+        ocf_log debug "Resource has not stopped yet, waiting"
+        sleep 1
+    done
+
+    # only return $OCF_SUCCESS if _everything_ succeeded as expected
+    return $OCF_SUCCESS
+
+}
+
+glusterd_monitor() {
+    local pid
+
+    [ -e $OCF_RESKEY_pid ] || return $OCF_NOT_RUNNING
+
+    pid=`cat $OCF_RESKEY_pid`
+    ocf_run kill -s 0 $pid || return $OCF_NOT_RUNNING
+
+    ocf_log debug "$OCF_RESKEY_binary running with PID $pid"
+    return $OCF_SUCCESS
+}
+
+glusterd_validate_all() {
+    # Test for required binaries
+    check_binary $OCF_RESKEY_binary
+
+    return $OCF_SUCCESS
+}
+
+
+
+# Make sure meta-data and usage always succeed
+case $__OCF_ACTION in
+meta-data)      glusterd_meta_data
+                exit $OCF_SUCCESS
+                ;;
+usage|help)     glusterd_usage
+                exit $OCF_SUCCESS
+                ;;
+esac
+
+# Anything other than meta-data and usage must pass validation
+glusterd_validate_all || exit $?
+
+# Translate each action into the appropriate function call
+case $__OCF_ACTION in
+start)          glusterd_start;;
+stop)           glusterd_stop;;
+status|monitor) glusterd_monitor;;
+reload)         ocf_log info "Reloading..."
+                glusterd_start
+                ;;
+validate-all)   ;;
+notify)		exit $OCF_SUCCESS;;
+*)              glusterd_usage
+                exit $OCF_ERR_UNIMPLEMENTED
+                ;;
+esac
+rc=$?
+
+# The resource agent may optionally log a debug message
+ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION returned $rc"
+exit $rc
diff --git a/extras/ocf/volume.in b/extras/ocf/volume.in
new file mode 100755
index 000000000..72fd1213a
--- /dev/null
+++ b/extras/ocf/volume.in
@@ -0,0 +1,246 @@
+#!/bin/sh
+#
+# glusterd
+#
+# Description:  Manages a glusterd server as a (typically cloned)
+#               HA resource
+#
+# Authors:      Florian Haas (hastexo Professional Services GmbH)
+#
+# License:      GNU General Public License (GPL)
+
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+# Convenience variables
+# When sysconfdir and localstatedir aren't passed in as
+# configure flags, they're defined in terms of prefix
+prefix=@prefix@
+SHORTHOSTNAME=`hostname -s`
+#######################################################################
+
+OCF_RESKEY_binary_default="gluster"
+
+: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}}
+
+volume_meta_data() {
+    cat <<EOF
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="volume" version="0.1">
+  <version>0.1</version>
+  <longdesc lang="en">
+Manages a GlusterFS volume and monitors its bricks. When a resource of
+this type is configured as a clone (as is commonly the case), then it
+must have clone ordering enabled.
+  </longdesc>
+  <shortdesc lang="en">Manages a GlusterFS volume</shortdesc>
+  <parameters>
+    <parameter name="volname" required="1">
+      <longdesc lang="en">
+      The name of the volume to manage.
+      </longdesc>
+      <shortdesc lang="en">volume name</shortdesc>
+      <content type="string"/>
+    </parameter>
+    <parameter name="binary">
+      <longdesc lang="en">
+      Name of the gluster executable. Specify a full absolute
+      path if the binary is not in your \$PATH.
+      </longdesc>
+      <shortdesc lang="en">gluster executable</shortdesc>
+      <content type="string" default="$OCF_RESKEY_binary_default"/>
+    </parameter>
+  </parameters>
+  <actions>
+    <action name="start"        timeout="20" />
+    <action name="stop"         timeout="20" />
+    <action name="monitor"      timeout="20" interval="10" />
+    <action name="reload"       timeout="20" />
+    <action name="meta-data"    timeout="5" />
+    <action name="validate-all"   timeout="20" />
+  </actions>
+</resource-agent>
+EOF
+
+}
+
+volume_getdir() {
+    local voldir
+    voldir="@sysconfdir@/glusterd/vols/${OCF_RESKEY_volname}"
+
+    [ -d ${voldir} ] || return 1
+
+    echo "${voldir}"
+    return 0
+}
+
+volume_getbricks() {
+    local infofile
+    local voldir
+    voldir=`volume_getdir`
+    infofile="${voldir}/info"
+
+    [ -e ${infofile} ] || return 1
+
+    echo "`sed -n -e "s/^brick-.\+=${SHORTHOSTNAME}://p" < ${infofile}`"
+    return 0
+}
+
+volume_getpids() {
+    local bricks
+    local piddir
+    local pidfile
+    local infofile
+    local voldir
+
+    voldir=`volume_getdir`
+    bricks=`volume_getbricks`
+    piddir="${voldir}/run"
+
+    for brick in ${bricks}; do
+	pidfile="${piddir}/${SHORTHOSTNAME}${brick}.pid"
+	[ -e $pidfile ] || return 1
+	cat $pidfile
+    done
+
+    return 0
+}
+
+volume_start() {
+    local volume_options
+
+    # exit immediately if configuration is not valid
+    volume_validate_all || exit $?
+
+    # if resource is already running, bail out early
+    if volume_monitor; then
+        ocf_log info "Resource is already running"
+        return $OCF_SUCCESS
+    fi
+
+    # actually start up the resource here
+    ocf_run "$OCF_RESKEY_binary" \
+	volume start "$OCF_RESKEY_volname" force || exit $OCF_ERR_GENERIC
+
+    # After the resource has been started, check whether it started up
+    # correctly. If the resource starts asynchronously, the agent may
+    # spin on the monitor function here -- if the resource does not
+    # start up within the defined timeout, the cluster manager will
+    # consider the start action failed
+    while ! volume_monitor; do
+        ocf_log debug "Resource has not started yet, waiting"
+        sleep 1
+    done
+
+    # only return $OCF_SUCCESS if _everything_ succeeded as expected
+    return $OCF_SUCCESS
+}
+
+volume_stop() {
+    local rc
+    local pid
+
+    # exit immediately if configuration is not valid
+    volume_validate_all || exit $?
+
+    volume_monitor
+    rc=$?
+    case "$rc" in
+        "$OCF_SUCCESS")
+            # Currently running. Normal, expected behavior.
+            ocf_log debug "Resource is currently running"
+            ;;
+        "$OCF_NOT_RUNNING")
+            # Currently not running. Nothing to do.
+            ocf_log info "Resource is already stopped"
+            return $OCF_SUCCESS
+            ;;
+    esac
+
+    # actually shut down the resource here (make sure to immediately
+    # exit with an $OCF_ERR_ error code if anything goes seriously
+    # wrong)
+    pids=`volume_getpids`
+    for pid in $pids; do
+	ocf_run kill -s TERM $pid
+    done
+
+    # After the resource has been stopped, check whether it shut down
+    # correctly. If the resource stops asynchronously, the agent may
+    # spin on the monitor function here -- if the resource does not
+    # shut down within the defined timeout, the cluster manager will
+    # consider the stop action failed
+    while volume_monitor; do
+        ocf_log debug "Resource has not stopped yet, waiting"
+        sleep 1
+    done
+
+    # only return $OCF_SUCCESS if _everything_ succeeded as expected
+    return $OCF_SUCCESS
+
+}
+
+volume_monitor() {
+    local pid
+
+    pids=`volume_getpids` || return $OCF_NOT_RUNNING
+
+    for pid in $pids; do
+	ocf_run kill -s 0 $pid || return $OCF_NOT_RUNNING
+    done
+
+    ocf_log debug "Local bricks for volume ${OCF_RESKEY_volname} running with PIDs $pids"
+    return $OCF_SUCCESS
+}
+
+volume_validate_all() {
+    # Test for configuration errors first
+    if [ -z "${OCF_RESKEY_volname}" ]; then
+	ocf_log err 'Missing required parameter "volname"'
+	return $OCF_ERR_CONFIGURED
+    fi
+
+    # Test for required binaries
+    check_binary $OCF_RESKEY_binary
+
+    return $OCF_SUCCESS
+}
+
+
+
+# Make sure meta-data and usage always succeed
+case $__OCF_ACTION in
+meta-data)      volume_meta_data
+                exit $OCF_SUCCESS
+                ;;
+usage|help)     volume_usage
+                exit $OCF_SUCCESS
+                ;;
+esac
+
+# Anything other than meta-data and usage must pass validation
+volume_validate_all || exit $?
+
+# Translate each action into the appropriate function call
+case $__OCF_ACTION in
+start)          volume_start;;
+stop)           volume_stop;;
+status|monitor) volume_monitor;;
+reload)         ocf_log info "Reloading..."
+                volume_start
+                ;;
+validate-all)   ;;
+notify)		exit $OCF_SUCCESS;;
+*)              volume_usage
+                exit $OCF_ERR_UNIMPLEMENTED
+                ;;
+esac
+rc=$?
+
+# The resource agent may optionally log a debug message
+ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION returned $rc"
+exit $rc
diff --git a/extras/profiler/glusterfs-profiler b/extras/profiler/glusterfs-profiler
index 042eadcf2..65d445864 100755
--- a/extras/profiler/glusterfs-profiler
+++ b/extras/profiler/glusterfs-profiler
@@ -1,22 +1,15 @@
 #!/usr/bin/env python
 
-# texttable - module for creating simple ASCII tables
-# Copyright (C) 2003-2010 Gerome Fournier <jefke(at)free.fr>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
+#  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
 
+
+# texttable - module for creating simple ASCII tables
 # Incorporated from texttable.py downloaded from
 # http://jefke.free.fr/stuff/python/texttable/texttable-0.7.0.tar.gz
 
diff --git a/extras/prot_filter.py b/extras/prot_filter.py
new file mode 100755
index 000000000..7dccacf15
--- /dev/null
+++ b/extras/prot_filter.py
@@ -0,0 +1,144 @@
+#!/usr/bin/python
+
+"""
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+"""
+
+"""
+  INSTRUCTIONS
+  Put this in /usr/lib64/glusterfs/$version/filter to have it run automatically,
+  or else you'll have to run it by hand every time you change the volume
+  configuration.  Give it a list of volume names on which to enable the
+  protection functionality; it will deliberately ignore client volfiles for
+  other volumes, and all server volfiles.  It *will* include internal client
+  volfiles such as those used for NFS or rebalance/self-heal; this is a
+  deliberate choice so that it will catch deletions from those sources as well.
+"""
+
+volume_list = [ "jdtest" ]
+
+import copy
+import string
+import sys
+import types
+
+class Translator:
+    def __init__ (self, name):
+        self.name = name
+        self.xl_type = ""
+        self.opts = {}
+        self.subvols = []
+        self.dumped = False
+    def __repr__ (self):
+        return "<Translator %s>" % self.name
+
+def load (path):
+    # If it's a string, open it; otherwise, assume it's already a
+    # file-like object (most notably from urllib*).
+    if type(path) in types.StringTypes:
+        fp = file(path,"r")
+    else:
+        fp = path
+    all_xlators = {}
+    xlator = None
+    last_xlator = None
+    while True:
+        text = fp.readline()
+        if text == "":
+            break
+        text = text.split()
+        if not len(text):
+            continue
+        if text[0] == "volume":
+            if xlator:
+                raise RuntimeError, "nested volume definition"
+            xlator = Translator(text[1])
+            continue
+        if not xlator:
+            raise RuntimeError, "text outside volume definition"
+        if text[0] == "type":
+            xlator.xl_type = text[1]
+            continue
+        if text[0] == "option":
+            xlator.opts[text[1]] = string.join(text[2:])
+            continue
+        if text[0] == "subvolumes":
+            for sv in text[1:]:
+                xlator.subvols.append(all_xlators[sv])
+            continue
+        if text[0] == "end-volume":
+            all_xlators[xlator.name] = xlator
+            last_xlator = xlator
+            xlator = None
+            continue
+        raise RuntimeError, "unrecognized keyword %s" % text[0]
+    if xlator:
+        raise RuntimeError, "unclosed volume definition"
+    return all_xlators, last_xlator
+
+def generate (graph, last, stream=sys.stdout):
+    for sv in last.subvols:
+        if not sv.dumped:
+            generate(graph,sv,stream)
+            print >> stream, ""
+            sv.dumped = True
+    print >> stream, "volume %s" % last.name
+    print >> stream, "    type %s" % last.xl_type
+    for k, v in last.opts.iteritems():
+        print >> stream, "    option %s %s" % (k, v)
+    if last.subvols:
+        print >> stream, "    subvolumes %s" % string.join(
+            [ sv.name for sv in last.subvols ])
+    print >> stream, "end-volume"
+
+def push_filter (graph, old_xl, filt_type, opts={}):
+    new_type = "-" + filt_type.split("/")[1]
+    old_type = "-" + old_xl.xl_type.split("/")[1]
+    pos = old_xl.name.find(old_type)
+    if pos >= 0:
+        new_name = old_xl.name
+        old_name = new_name[:pos] + new_type + new_name[len(old_type)+pos:]
+    else:
+        new_name = old_xl.name + old_type
+        old_name = old_xl.name + new_type
+    new_xl = Translator(new_name)
+    new_xl.xl_type = old_xl.xl_type
+    new_xl.opts = old_xl.opts
+    new_xl.subvols = old_xl.subvols
+    graph[new_xl.name] = new_xl
+    old_xl.name = old_name
+    old_xl.xl_type = filt_type
+    old_xl.opts = opts
+    old_xl.subvols = [new_xl]
+    graph[old_xl.name] = old_xl
+
+if __name__ == "__main__":
+    path = sys.argv[1]
+    # Alow an override for debugging.
+    for extra in sys.argv[2:]:
+        volume_list.append(extra)
+    graph, last = load(path)
+    for v in volume_list:
+        if graph.has_key(v):
+            break
+    else:
+        print "No configured volumes found - aborting."
+        sys.exit(0)
+    for v in graph.values():
+        if v.xl_type == "cluster/distribute":
+            push_filter(graph,v,"features/prot_dht")
+        elif v.xl_type == "protocol/client":
+            push_filter(graph,v,"features/prot_client")
+    # We push debug/trace so that every fop gets a real frame, because DHT
+    # gets confused if STACK_WIND_TAIL causes certain fops to be invoked
+    # from anything other than a direct child.
+    for v in graph.values():
+        if v.xl_type == "features/prot_client":
+            push_filter(graph,v,"debug/trace")
+    generate(graph,last,stream=open(path,"w"))
diff --git a/extras/rebalance.py b/extras/rebalance.py
new file mode 100755
index 000000000..80c614c5d
--- /dev/null
+++ b/extras/rebalance.py
@@ -0,0 +1,299 @@
+#!/usr/bin/python
+
+import atexit
+import copy
+import optparse
+import os
+import pipes
+import shutil
+import string
+import subprocess
+import sys
+import tempfile
+import volfilter
+
+# It's just more convenient to have named fields.
+class Brick:
+        def __init__ (self, path, name):
+                self.path = path
+                self.sv_name = name
+                self.size = 0
+                self.curr_size = 0
+                self.good_size = 0
+        def set_size (self, size):
+                self.size = size
+        def set_range (self, rs, re):
+                self.r_start = rs
+                self.r_end = re
+                self.curr_size = self.r_end - self.r_start + 1
+        def __repr__ (self):
+                value = self.path[:]
+                value += "(%d," % self.size
+                if self.curr_size:
+                        value += "0x%x,0x%x)" % (self.r_start, self.r_end)
+                else:
+                        value += "-)"
+                return value
+
+def get_bricks (host, vol):
+        t = pipes.Template()
+        t.prepend("gluster --remote-host=%s system getspec %s"%(host,vol),".-")
+        return t.open(None,"r")
+
+def generate_stanza (vf, all_xlators, cur_subvol):
+        sv_list = []
+        for sv in cur_subvol.subvols:
+                generate_stanza(vf,all_xlators,sv)
+                sv_list.append(sv.name)
+        vf.write("volume %s\n"%cur_subvol.name)
+        vf.write("  type %s\n"%cur_subvol.type)
+        for kvpair in cur_subvol.opts.iteritems():
+                vf.write("  option %s %s\n"%kvpair)
+        if sv_list:
+                vf.write("  subvolumes %s\n"%string.join(sv_list))
+        vf.write("end-volume\n\n")
+
+
+def mount_brick (localpath, all_xlators, dht_subvol):
+
+        # Generate a volfile.
+        vf_name = localpath + ".vol"
+        vf = open(vf_name,"w")
+        generate_stanza(vf,all_xlators,dht_subvol)
+        vf.flush()
+        vf.close()
+
+        # Create a brick directory and mount the brick there.
+        os.mkdir(localpath)
+        subprocess.call(["glusterfs","-f",vf_name,localpath])
+
+# We use the command-line tools because there's no getxattr support in the
+# Python standard library (which is ridiculous IMO).  Adding the xattr package
+# from PyPI would create a new and difficult dependency because the bits to
+# satisfy it don't seem to exist in Fedora.  We already expect the command-line
+# tools to be there, so it's safer just to rely on them.
+#
+# We might have to revisit this if we get as far as actually issuing millions
+# of setxattr requests.  Even then, it might be better to do that part with a C
+# program which has only a build-time dependency.
+def get_range (brick):
+        t = pipes.Template()
+        cmd = "getfattr -e hex -n trusted.glusterfs.dht %s 2> /dev/null"
+        t.prepend(cmd%brick,".-")
+        t.append("grep ^trusted.glusterfs.dht=","--")
+        f = t.open(None,"r")
+        try:
+                value = f.readline().rstrip().split('=')[1][2:]
+        except:
+                print "could not get layout for %s (might be OK)" % brick
+                return None
+        v_start = int("0x"+value[16:24],16)
+        v_end = int("0x"+value[24:32],16)
+        return (v_start, v_end)
+
+def calc_sizes (bricks, total):
+        leftover = 1 << 32
+        for b in bricks:
+               if b.size:
+                        b.good_size = (b.size << 32) / total
+                        leftover -= b.good_size
+               else:
+                        b.good_size = 0
+        if leftover:
+                # Add the leftover to an old brick if we can.
+                for b in bricks:
+                        if b.good_size:
+                                b.good_size += leftover
+                                break
+                else:
+                        # Fine, just add it wherever.
+                        bricks[0].good_size += leftover
+
+# Normalization means sorting the bricks by r_start and (b) ensuring that there
+# are no gaps.
+def normalize (in_bricks):
+        out_bricks = []
+        curr_hash = 0
+        used = 0
+        while curr_hash < (1<<32):
+                curr_best = None
+                for b in in_bricks:
+                        if b.r_start == curr_hash:
+                                used += 1
+                                out_bricks.append(b)
+                                in_bricks.remove(b)
+                                curr_hash = b.r_end + 1
+                                break
+                else:
+                        print "gap found at 0x%08x" % curr_hash
+                        sys.exit(1)
+        return out_bricks + in_bricks, used
+
+def get_score (bricks):
+        score = 0
+        curr_hash = 0
+        for b in bricks:
+                if not b.curr_size:
+                        curr_hash += b.good_size
+                        continue
+                new_start = curr_hash
+                curr_hash += b.good_size
+                new_end = curr_hash - 1
+                if new_start > b.r_start:
+                        max_start = new_start
+                else:
+                        max_start = b.r_start
+                if new_end < b.r_end:
+                        min_end = new_end
+                else:
+                        min_end = b.r_end
+                if max_start <= min_end:
+                        score += (min_end - max_start + 1)
+        return score
+
+if __name__ == "__main__":
+
+	my_usage = "%prog [options] server volume [directory]"
+	parser = optparse.OptionParser(usage=my_usage)
+        parser.add_option("-f", "--free-space", dest="free_space",
+                          default=False, action="store_true",
+                          help="use free space instead of total space")
+        parser.add_option("-l", "--leave-mounted", dest="leave_mounted",
+                          default=False, action="store_true",
+                          help="leave subvolumes mounted")
+        parser.add_option("-v", "--verbose", dest="verbose",
+                          default=False, action="store_true",
+                          help="verbose output")
+	options, args = parser.parse_args()
+
+        if len(args) == 3:
+                fix_dir = args[2]
+        else:
+                if len(args) != 2:
+                        parser.print_help()
+                        sys.exit(1)
+                fix_dir = None
+        hostname, volname = args[:2]
+
+        # Make sure stuff gets cleaned up, even if there are exceptions.
+        orig_dir = os.getcwd()
+        work_dir = tempfile.mkdtemp()
+        bricks = []
+        def cleanup_workdir ():
+                os.chdir(orig_dir)
+                if options.verbose:
+                        print "Cleaning up %s" % work_dir
+                for b in bricks:
+                        subprocess.call(["umount",b.path])
+                shutil.rmtree(work_dir)
+        if not options.leave_mounted:
+                atexit.register(cleanup_workdir)
+        os.chdir(work_dir)
+
+        # Mount each brick individually, so we can issue brick-specific calls.
+        if options.verbose:
+                print "Mounting subvolumes..."
+        index = 0
+        volfile_pipe = get_bricks(hostname,volname)
+        all_xlators, last_xlator = volfilter.load(volfile_pipe)
+        for dht_vol in all_xlators.itervalues():
+                if dht_vol.type == "cluster/distribute":
+                        break
+        else:
+                print "no DHT volume found"
+                sys.exit(1)
+        for sv in dht_vol.subvols:
+                #print "found subvol %s" % sv.name
+                lpath = "%s/brick%s" % (work_dir, index)
+                index += 1
+                mount_brick(lpath,all_xlators,sv)
+                bricks.append(Brick(lpath,sv.name))
+        if index == 0:
+                print "no bricks"
+                sys.exit(1)
+
+        # Collect all of the sizes.
+        if options.verbose:
+                print "Collecting information..."
+        total = 0
+        for b in bricks:
+                info = os.statvfs(b.path)
+                # We want a standard unit even if different bricks use
+                # different block sizes.  The size is chosen to avoid overflows
+                # for very large bricks with very small block sizes, but also
+                # accommodate filesystems which use very large block sizes to
+                # cheat on benchmarks.
+                blocksper100mb = 104857600 / info[0]
+                if options.free_space:
+                        size = info[3] / blocksper100mb
+                else:
+                        size = info[2] / blocksper100mb
+                if size <= 0:
+                        print "brick %s has invalid size %d" % (b.path, size)
+                        sys.exit(1)
+                b.set_size(size)
+                total += size
+
+        # Collect all of the layout information.
+        for b in bricks:
+                hash_range = get_range(b.path)
+                if hash_range is not None:
+                        rs, re = hash_range
+                        if rs > re:
+                                print "%s has backwards hash range" % b.path
+                                sys.exit(1)
+                        b.set_range(hash_range[0],hash_range[1])
+
+        if options.verbose:
+                print "Calculating new layouts..."
+        calc_sizes(bricks,total)
+        bricks, used = normalize(bricks)
+
+        # We can't afford O(n!) here, but O(n^2) should be OK and the result
+        # should be almost as good.
+        while used < len(bricks):
+                best_place = used
+                best_score = get_score(bricks)
+                for i in xrange(used):
+                        new_bricks = bricks[:]
+                        del new_bricks[used]
+                        new_bricks.insert(i,bricks[used])
+                        new_score = get_score(new_bricks)
+                        if new_score > best_score:
+                                best_place = i
+                                best_score = new_score
+                if best_place != used:
+                        nb = bricks[used]
+                        del bricks[used]
+                        bricks.insert(best_place,nb)
+                used += 1
+
+        # Finalize whatever we decided on.
+        curr_hash = 0
+        for b in bricks:
+                b.r_start = curr_hash
+                curr_hash += b.good_size
+                b.r_end = curr_hash - 1
+
+        print "Here are the xattr values for your size-weighted layout:"
+        for b in bricks:
+                print "  %s: 0x0000000200000000%08x%08x" % (
+                        b.sv_name, b.r_start, b.r_end)
+
+        if fix_dir:
+                if options.verbose:
+                        print "Fixing layout for %s" % fix_dir
+                for b in bricks:
+                        value = "0x0000000200000000%08x%08x" % (
+                                b.r_start, b.r_end)
+                        path = "%s/%s" % (b.path, fix_dir)
+                        cmd = "setfattr -n trusted.glusterfs.dht -v %s %s" % (
+                                value, path)
+                        print cmd
+
+        if options.leave_mounted:
+                print "The following subvolumes are still mounted:"
+                for b in bricks:
+                        print "%s on %s" % (b.sv_name, b.path)
+                print "Don't forget to clean up when you're done."
+
diff --git a/extras/rpc-coverage.sh b/extras/rpc-coverage.sh
index ebb92a2a1..9148a73e2 100755
--- a/extras/rpc-coverage.sh
+++ b/extras/rpc-coverage.sh
@@ -398,8 +398,7 @@ function test_getxattr()
 function test_removexattr()
 {
     setfattr -x trusted.testing $PFX/dir/file || fail "setfattr remove"
-    getfattr -n trusted.testing $PFX/dir/file 2>&1 | grep -q 'No such attribute' \
-	|| fail "getfattr"
+    getfattr -n trusted.testing $PFXf/dir/file 2>&1 | grep -q "No such attribute"
 }
 
 
diff --git a/extras/stripe-merge.c b/extras/stripe-merge.c
index 3f8e4b124..74bd47e30 100644
--- a/extras/stripe-merge.c
+++ b/extras/stripe-merge.c
@@ -1,48 +1,494 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+ * stripe-merge.c
+ *
+ * This program recovers an original file based on the striped files stored on
+ * the individual bricks of a striped volume. The file format and stripe
+ * geometry is validated through the extended attributes stored in the file.
+ *
+ * TODO: Support optional xattr recovery (i.e., user xattrs). Perhaps provide a
+ * 	 command-line flag to toggle this behavior.
+ */
+
 #include <stdio.h>
-#include <unistd.h>
-#include <fcntl.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <errno.h>
+#include <string.h>
+#include <attr/xattr.h>
+#include <fnmatch.h>
 
-int
-main (int argc, char *argv[])
+#define ATTRNAME_STRIPE_INDEX "trusted.*.stripe-index"
+#define ATTRNAME_STRIPE_COUNT "trusted.*.stripe-count"
+#define ATTRNAME_STRIPE_SIZE "trusted.*.stripe-size"
+#define ATTRNAME_STRIPE_COALESCE "trusted.*.stripe-coalesce"
+
+#define INVALID_FD -1
+#define INVALID_MODE UINT32_MAX
+
+struct file_stripe_info {
+	int stripe_count;
+	int stripe_size;
+	int coalesce;
+	mode_t mode;
+	int fd[0];
+};
+
+static int close_files(struct file_stripe_info *);
+
+static struct
+file_stripe_info *alloc_file_stripe_info(int count)
 {
-	int fds[argc-1];
-	char buf[argc-1][4096];
 	int i;
-	int max_ret, ret;
+	struct file_stripe_info *finfo;
 
-	if (argc < 2) {
-		printf ("Usage: %s file1 file2 ... >file\n", argv[0]);
-		return 1;
+	finfo = calloc(1, sizeof(struct file_stripe_info) +
+		(sizeof(int) * count));
+	if (!finfo)
+		return NULL;
+
+	for (i = 0; i < count; i++)
+		finfo->fd[i] = INVALID_FD;
+
+	finfo->mode = INVALID_MODE;
+	finfo->coalesce = INVALID_FD;
+
+	return finfo;
+}
+
+/*
+ * Search for an attribute matching the provided pattern. Return a count for
+ * the total number of matching entries (including 0). Allocate a buffer for
+ * the first matching entry found.
+ */
+static int
+get_stripe_attr_name(const char *path, const char *pattern, char **attrname)
+{
+	char attrbuf[4096];
+	char *ptr, *match = NULL;
+	int len, r, match_count = 0;
+
+	if (!path || !pattern || !attrname)
+		return -1;
+
+	len = listxattr(path, attrbuf, sizeof(attrbuf));
+	if (len < 0)
+		return len;
+
+	ptr = attrbuf;
+	while (ptr) {
+		r = fnmatch(pattern, ptr, 0);
+		if (!r) {
+			if (!match)
+				match = ptr;
+			match_count++;
+		} else if (r != FNM_NOMATCH) {
+			return -1;
+		}
+
+		len -= strlen(ptr) + 1;
+		if (len > 0)
+			ptr += strlen(ptr) + 1;
+		else
+			ptr = NULL;
 	}
 
-	for (i=0; i<argc-1; i++) {
-		fds[i] = open (argv[i+1], O_RDONLY);
-		if (fds[i] == -1) {
-			perror (argv[i+1]);
-			return 1;
+	if (match)
+		*attrname = strdup(match);
+
+	return match_count;
+}
+
+/*
+ * Get the integer representation of a named attribute.
+ */
+static int
+get_stripe_attr_val(const char *path, const char *attr, int *val)
+{
+	char attrbuf[4096];
+	int len;
+
+	if (!path || !attr || !val)
+		return -1;
+
+	len = getxattr(path, attr, attrbuf, sizeof(attrbuf));
+	if (len < 0)
+		return len;
+
+	*val = atoi(attrbuf);
+
+	return 0;
+}
+
+/*
+ * Get an attribute name/value (assumed to be an integer) pair based on a
+ * specified search pattern. A buffer is allocated for the exact attr name
+ * returned. Optionally, skip the pattern search if a buffer is provided
+ * (which should contain an attribute name).
+ *
+ * Returns the attribute count or -1 on error. The value parameter is set only
+ * when a single attribute is found.
+ */
+static int
+get_attr(const char *path, const char *pattern, char **buf, int *val)
+{
+	int count = 1;
+
+	if (!buf)
+		return -1;
+
+	if (!*buf) {
+		count = get_stripe_attr_name(path, pattern, buf);
+		if (count > 1) {
+			/* pattern isn't good enough */
+			fprintf(stderr, "ERROR: duplicate attributes found "
+				"matching pattern: %s\n", pattern);
+			free(*buf);
+			*buf = NULL;
+			return count;
+		} else if (count < 1) {
+			return count;
 		}
 	}
 
-	max_ret = 0;
+	if (get_stripe_attr_val(path, *buf, val) < 0)
+		return -1;
+
+	return count;
+}
+
+/*
+ * validate_and_open_files()
+ *
+ * Open the provided source files and validate the extended attributes. Verify
+ * that the geometric attributes are consistent across all of the files and
+ * print a warning if any files are missing. We proceed without error in the
+ * latter case to support partial recovery.
+ */
+static struct
+file_stripe_info *validate_and_open_files(char *paths[], int count)
+{
+	int i, val, tmp;
+	struct stat sbuf;
+	char *stripe_count_attr = NULL;
+	char *stripe_size_attr = NULL;
+	char *stripe_index_attr = NULL;
+	char *stripe_coalesce_attr = NULL;
+	struct file_stripe_info *finfo = NULL;
+
+	for (i = 0; i < count; i++) {
+		if (!paths[i])
+			goto err;
+
+		/*
+		 * Check the stripe count first so we can allocate the info
+		 * struct with the appropriate number of fds.
+		 */
+		if (get_attr(paths[i], ATTRNAME_STRIPE_COUNT,
+				&stripe_count_attr, &val) != 1) {
+			fprintf(stderr, "ERROR: %s: attribute: '%s'\n",
+				paths[i], ATTRNAME_STRIPE_COUNT);
+			goto err;
+		}
+		if (!finfo) {
+			finfo = alloc_file_stripe_info(val);
+			if (!finfo)
+				goto err;
+
+			if (val != count)
+				fprintf(stderr, "WARNING: %s: stripe-count "
+					"(%d) != file count (%d). Result may "
+					"be incomplete.\n", paths[i], val,
+					count);
+
+			finfo->stripe_count = val;
+		} else if (val != finfo->stripe_count) {
+			fprintf(stderr, "ERROR %s: invalid stripe count: %d "
+				"(expected %d)\n", paths[i], val,
+				finfo->stripe_count);
+			goto err;
+		}
+
+		/*
+		 * Get and validate the chunk size.
+		 */
+		if (get_attr(paths[i], ATTRNAME_STRIPE_SIZE, &stripe_size_attr,
+				&val) != 1) {
+			fprintf(stderr, "ERROR: %s: attribute: '%s'\n",
+				paths[i], ATTRNAME_STRIPE_SIZE);
+			goto err;
+		}
+
+		if (!finfo->stripe_size) {
+			finfo->stripe_size = val;
+		} else if (val != finfo->stripe_size) {
+			fprintf(stderr, "ERROR: %s: invalid stripe size: %d "
+				"(expected %d)\n", paths[i], val,
+				finfo->stripe_size);
+			goto err;
+		}
+
+		/*
+		 * stripe-coalesce is a backward compatible attribute. If the
+		 * attribute does not exist, assume a value of zero for the
+		 * traditional stripe format.
+		 */
+		tmp = get_attr(paths[i], ATTRNAME_STRIPE_COALESCE,
+				&stripe_coalesce_attr, &val);
+		if (!tmp) {
+			val = 0;
+		} else if (tmp != 1) {
+			fprintf(stderr, "ERROR: %s: attribute: '%s'\n",
+				paths[i], ATTRNAME_STRIPE_COALESCE);
+			goto err;
+		}
+
+		if (finfo->coalesce == INVALID_FD) {
+			finfo->coalesce = val;
+		} else if (val != finfo->coalesce) {
+			fprintf(stderr, "ERROR: %s: invalid coalesce flag\n",
+				paths[i]);
+			goto err;
+		}
+
+		/*
+		 * Get/validate the stripe index and open the file in the
+		 * appropriate fd slot.
+		 */
+		if (get_attr(paths[i], ATTRNAME_STRIPE_INDEX,
+				&stripe_index_attr, &val) != 1) {
+			fprintf(stderr, "ERROR: %s: attribute: '%s'\n",
+				paths[i], ATTRNAME_STRIPE_INDEX);
+			goto err;
+		}
+		if (finfo->fd[val] != INVALID_FD) {
+			fprintf(stderr, "ERROR: %s: duplicate stripe index: "
+				"%d\n", paths[i], val);
+			goto err;
+		}
+
+		finfo->fd[val] = open(paths[i], O_RDONLY);
+		if (finfo->fd[val] < 0)
+			goto err;
+
+		/*
+		 * Get the creation mode for the file.
+		 */
+		if (fstat(finfo->fd[val], &sbuf) < 0)
+			goto err;
+		if (finfo->mode == INVALID_MODE) {
+			finfo->mode = sbuf.st_mode;
+		} else if (sbuf.st_mode != finfo->mode) {
+			fprintf(stderr, "ERROR: %s: invalid mode\n", paths[i]);
+			goto err;
+		}
+	}
+
+	free(stripe_count_attr);
+	free(stripe_size_attr);
+	free(stripe_index_attr);
+	free(stripe_coalesce_attr);
+
+	return finfo;
+err:
+
+	free(stripe_count_attr);
+	free(stripe_size_attr);
+	free(stripe_index_attr);
+	free(stripe_coalesce_attr);
+
+	if (finfo) {
+		close_files(finfo);
+		free(finfo);
+	}
+
+	return NULL;
+}
+
+static int
+close_files(struct file_stripe_info *finfo)
+{
+	int i, ret;
+
+	if (!finfo)
+		return -1;
+
+	for (i = 0; i < finfo->stripe_count; i++) {
+		if (finfo->fd[i] == INVALID_FD)
+			continue;
+
+		ret = close(finfo->fd[i]);
+		if (ret < 0)
+			return ret;
+	}
+
+	return ret;
+}
+
+/*
+ * Generate the original file using files striped in the coalesced format.
+ * Data in the striped files is stored at a coalesced offset based on the
+ * stripe number.
+ *
+ * Walk through the finfo fds (which are already ordered) and and iteratively
+ * copy stripe_size bytes from the source files to the target file. If a source
+ * file is missing, seek past the associated stripe_size bytes in the target
+ * file.
+ */
+static int
+generate_file_coalesce(int target, struct file_stripe_info *finfo)
+{
+	char *buf;
+	int ret = 0;
+	int r, w, i;
+
+	buf = malloc(finfo->stripe_size);
+	if (!buf)
+		return -1;
+
+	i = 0;
+	while (1) {
+		if (finfo->fd[i] == INVALID_FD) {
+			if (lseek(target, finfo->stripe_size, SEEK_CUR) < 0)
+				break;
+
+			i = (i + 1) % finfo->stripe_count;
+			continue;
+		}
+
+		r = read(finfo->fd[i], buf, finfo->stripe_size);
+		if (r < 0) {
+			ret = r;
+			break;
+		}
+		if (!r)
+			break;
+
+		w = write(target, buf, r);
+		if (w < 0) {
+			ret = w;
+			break;
+		}
+
+		i = (i + 1) % finfo->stripe_count;
+	}
+
+	free(buf);
+	return ret;
+}
+
+/*
+ * Generate the original file using files striped with the traditional stripe
+ * format. Data in the striped files is stored at the equivalent offset from
+ * the source file.
+ */
+static int
+generate_file_traditional(int target, struct file_stripe_info *finfo)
+{
+	int i, j, max_ret, ret;
+	char buf[finfo->stripe_count][4096];
+
 	do {
 		char newbuf[4096] = {0, };
-		int j;
 
 		max_ret = 0;
-		for (i=0; i<argc-1; i++) {
-			memset (buf[i], 0, 4096);
-			ret = read (fds[i], buf[i], 4096); 
+		for (i = 0; i < finfo->stripe_count; i++) {
+			memset(buf[i], 0, 4096);
+			ret = read(finfo->fd[i], buf[i], 4096);
 			if (ret > max_ret)
 				max_ret = ret;
 		}
-		for (i=0; i<max_ret;i++)
-			for (j=0; j<argc-1; j++)
+		for (i = 0; i < max_ret; i++)
+			for (j = 0; j < finfo->stripe_count; j++)
 				newbuf[i] |= buf[j][i];
-		write (1, newbuf, max_ret);
+			write(target, newbuf, max_ret);
 	} while (max_ret);
 
 	return 0;
 }
 
+static int
+generate_file(int target, struct file_stripe_info *finfo)
+{
+	if (finfo->coalesce)
+		return generate_file_coalesce(target, finfo);
+
+	return generate_file_traditional(target, finfo);
+}
+
+static void
+usage(char *name)
+{
+	fprintf(stderr, "Usage: %s [-o <outputfile>] <inputfile1> "
+		"<inputfile2> ...\n", name);
+}
+
+int
+main(int argc, char *argv[])
+{
+	int file_count, opt;
+	char *opath = NULL;
+	int targetfd;
+	struct file_stripe_info *finfo;
+
+	while ((opt = getopt(argc, argv, "o:")) != -1) {
+		switch (opt) {
+		case 'o':
+			opath = optarg;
+			break;
+		default:
+			usage(argv[0]);
+			return -1;
+		}
+	}
+
+	file_count = argc - optind;
+
+	if (!opath || !file_count) {
+		usage(argv[0]);
+		return -1;
+	}
+
+	finfo = validate_and_open_files(&argv[optind], file_count);
+	if (!finfo)
+		goto err;
+
+	targetfd = open(opath, O_RDWR|O_CREAT, finfo->mode);
+	if (targetfd < 0)
+		goto err;
+
+	if (generate_file(targetfd, finfo) < 0)
+		goto err;
+
+	if (fsync(targetfd) < 0)
+		fprintf(stderr, "ERROR: %s\n", strerror(errno));
+	if (close(targetfd) < 0)
+		fprintf(stderr, "ERROR: %s\n", strerror(errno));
+
+	close_files(finfo);
+	free(finfo);
+
+	return 0;
+
+err:
+	if (finfo) {
+		close_files(finfo);
+		free(finfo);
+	}
+
+	return -1;
+}
+
diff --git a/extras/systemd/Makefile.am b/extras/systemd/Makefile.am
new file mode 100644
index 000000000..3fc656b82
--- /dev/null
+++ b/extras/systemd/Makefile.am
@@ -0,0 +1,11 @@
+
+CLEANFILES =
+
+SYSTEMD_DIR = @systemddir@
+
+install-exec-local:
+	@if [ -d $(SYSTEMD_DIR) ]; then \
+		$(mkdir_p) $(DESTDIR)$(SYSTEMD_DIR); \
+		$(INSTALL_PROGRAM) glusterd.service $(DESTDIR)$(SYSTEMD_DIR)/; \
+	fi
+
diff --git a/extras/systemd/glusterd.service.in b/extras/systemd/glusterd.service.in
new file mode 100644
index 000000000..fc8d8c9a2
--- /dev/null
+++ b/extras/systemd/glusterd.service.in
@@ -0,0 +1,14 @@
+[Unit]
+Description=GlusterFS, a clustered file-system server
+After=network.target rpcbind.service
+Before=network-online.target
+
+[Service]
+Type=forking
+PIDFile=/run/glusterd.pid
+LimitNOFILE=65536
+ExecStart=@prefix@/sbin/glusterd -p /run/glusterd.pid
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target
diff --git a/extras/test/bug-920583.t b/extras/test/bug-920583.t
new file mode 100755
index 000000000..eedbb800a
--- /dev/null
+++ b/extras/test/bug-920583.t
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+##Copy this file to tests/bugs before running run.sh (cp extras/test/bug-920583.t tests/bugs/)
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+logdir=`gluster --print-logdir`
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+function log-file-name()
+{
+    logfilename=$M0".log"
+    echo ${logfilename:1} | tr / -
+}
+
+log_file=$logdir"/"`log-file-name`
+
+lookup_unhashed_count=`grep "adding option 'lookup-unhashed'" $log_file | wc -l`
+no_child_down_count=`grep "adding option 'assert-no-child-down'" $log_file | wc -l`
+mount -t glusterfs $H0:/$V0 $M0 -o "xlator-option=*dht.assert-no-child-down=yes,xlator-option=*dht.lookup-unhashed=yes"
+touch $M0/file1;
+
+new_lookup_unhashed_count=`grep "adding option 'lookup-unhashed'" $log_file | wc -l`
+new_no_child_down_count=`grep "adding option 'assert-no-child-down'" $log_file | wc -l`
+EXPECT "1" expr $new_lookup_unhashed_count - $lookup_unhashed_count
+EXPECT "1" expr $new_no_child_down_count - $no_child_down_count
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/extras/test/gluster_commands.sh b/extras/test/gluster_commands.sh
index e1e396020..cb2a55fd5 100755
--- a/extras/test/gluster_commands.sh
+++ b/extras/test/gluster_commands.sh
@@ -1,21 +1,13 @@
 #!/bin/bash
 
-#   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-#   This file is part of GlusterFS.
 
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
 
 
 # This script tests the basics gluster cli commands.
@@ -54,13 +46,13 @@ gluster volume start vol
 gluster volume info
 sleep 1
 mount -t glusterfs `hostname`:vol /mnt/client
-sleep 1 
+sleep 1
 df -h
 
 echo "adding-brick......."
 gluster volume add-brick vol `hostname`:/exports/exp2
 gluster volume info
-sleep 1 
+sleep 1
 umount /mnt/client
 mount -t glusterfs `hostname`:vol /mnt/client
 df -h
@@ -102,7 +94,7 @@ sleep 1
 echo "removing brick......."
 gluster --mode=script volume remove-brick vol `hostname`:/exports/exp2
 gluster volume info
-sleep 1 
+sleep 1
 df -h
 sleep 1
 
@@ -127,7 +119,7 @@ sleep 1
 echo "starting replicate volume......"
 gluster volume start mirror
 gluster volume info
-sleep 1 
+sleep 1
 mount -t glusterfs `hostname`:mirror /mnt/client
 sleep 1
 df -h
@@ -136,7 +128,7 @@ sleep 1
 echo "adding-brick......."
 gluster volume add-brick mirror `hostname`:/exports/exp3 `hostname`:/exports/exp4
 gluster volume info
-sleep 1 
+sleep 1
 df -h
 sleep 1
 
@@ -178,14 +170,14 @@ sleep 1
 echo "removeing-brick....."
 gluster --mode=script volume remove-brick mirror `hostname`:/exports/exp3 `hostname`:/exports/exp4
 gluster volume info
-sleep 1 
+sleep 1
 df -h
 sleep 1
 
 echo "stopping replicate volume....."
 gluster --mode=script volume stop mirror
 gluster volume info
-sleep 1 
+sleep 1
 umount /mnt/client
 df -h
 
@@ -206,14 +198,14 @@ gluster volume start str
 gluster volume info
 sleep 1
 mount -t glusterfs `hostname`:str /mnt/client
-sleep 1 
+sleep 1
 df -h
 sleep 1
 
 echo "adding brick...."
 gluster volume add-brick str `hostname`:/exports/exp3 `hostname`:/exports/exp4
 gluster volume info
-sleep 1 
+sleep 1
 df -h
 sleep 1
 
diff --git a/extras/test/ld-preload-test/ld-preload-lib.c b/extras/test/ld-preload-test/ld-preload-lib.c
index e17305a4a..88afd14c3 100644
--- a/extras/test/ld-preload-test/ld-preload-lib.c
+++ b/extras/test/ld-preload-test/ld-preload-lib.c
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 /* LD PRELOAD'able library
  * A very simple library that intercepts booster supported system calls
  * and prints a log message to stdout.
diff --git a/extras/test/ld-preload-test/ld-preload-test.c b/extras/test/ld-preload-test/ld-preload-test.c
index 55dd98805..78772f598 100644
--- a/extras/test/ld-preload-test/ld-preload-test.c
+++ b/extras/test/ld-preload-test/ld-preload-test.c
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 /*
  * LD PRELOAD Test Tool
  *
diff --git a/extras/test/open-fd-tests.c b/extras/test/open-fd-tests.c
new file mode 100644
index 000000000..4184079d0
--- /dev/null
+++ b/extras/test/open-fd-tests.c
@@ -0,0 +1,64 @@
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <attr/xattr.h>
+#include <errno.h>
+#include <string.h>
+
+int
+main (int argc, char *argv[])
+{
+        int          ret      = -1;
+        int          fd       = 0;
+        char        *filename = NULL;
+        int          loop     = 0;
+        struct stat  stbuf    = {0,};
+        char         string[1024] = {0,};
+
+        if (argc > 1)
+                filename = argv[1];
+
+        if (!filename)
+                filename = "temp-fd-test-file";
+
+        fd = open (filename, O_RDWR|O_CREAT|O_TRUNC);
+        if (fd < 0) {
+                fd = 0;
+                fprintf (stderr, "open failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        while (loop < 1000) {
+                /* Use it as a mechanism to test time delays */
+                memset (string, 0, 1024);
+                scanf ("%s", string);
+
+                ret = write (fd, string, strlen (string));
+                if (ret != strlen (string)) {
+                        fprintf (stderr, "write failed : %s (%s %d)\n",
+                                 strerror (errno), string, loop);
+                        goto out;
+                }
+
+                ret = write (fd, "\n", 1);
+                if (ret != 1) {
+                        fprintf (stderr, "write failed : %s (%d)\n",
+                                 strerror (errno), loop);
+                        goto out;
+                }
+
+                loop++;
+        }
+
+        fprintf (stdout, "finishing the test after %d loops\n", loop);
+
+        ret = 0;
+out:
+        if (fd)
+                close (fd);
+
+        return ret;
+}
diff --git a/extras/test/run.sh b/extras/test/run.sh
index 2440af237..4b3839cf6 100755
--- a/extras/test/run.sh
+++ b/extras/test/run.sh
@@ -1,21 +1,12 @@
 #!/bin/sh
 
-#   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-#   This file is part of GlusterFS.
-
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
 
 # Running gluster sanity test which starts glusterd and runs gluster commands, and exit at the first failure.
 $PWD/gluster_commands.sh
diff --git a/extras/test/stop_glusterd.sh b/extras/test/stop_glusterd.sh
index a84689beb..a2db13f42 100755
--- a/extras/test/stop_glusterd.sh
+++ b/extras/test/stop_glusterd.sh
@@ -1,21 +1,12 @@
 #!/bin/bash
 
-#   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-#   This file is part of GlusterFS.
-
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
+#
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
 
 #This script stops the glusterd running on the machine. Helpful for gluster sanity script
 
diff --git a/extras/test/test-ffop.c b/extras/test/test-ffop.c
index 2c47ab004..2d174d452 100644
--- a/extras/test/test-ffop.c
+++ b/extras/test/test-ffop.c
@@ -6,20 +6,82 @@
 #include <attr/xattr.h>
 #include <errno.h>
 #include <string.h>
+#include <dirent.h>
+
+int fd_based_fops_1 (char *filename); //for fd based fops after unlink
+int fd_based_fops_2 (char *filename); //for fd based fops before unlink
+int dup_fd_based_fops (char *filename); // fops based on fd after dup
+int path_based_fops (char *filename); //for fops based on path
+int dir_based_fops (char *filename); // for fops which operate on directory
+int link_based_fops (char *filename); //for fops which operate in link files (symlinks)
+int test_open_modes (char *filename); // to test open syscall with open modes available.
+int generic_open_read_write (char *filename, int flag); // generic function which does open write and read.
 
 int
 main (int argc, char *argv[])
 {
-        int          ret      = -1;
-        int          fd       = 0;
-        char        *filename = NULL;
-        struct stat  stbuf    = {0,};
+        int   ret           = -1;
+        char  filename[255] = {0,};
 
         if (argc > 1)
-                filename = argv[1];
+                strcpy(filename, argv[1]);
+        else
+                strcpy(filename, "temp-xattr-test-file");
+
+        ret = fd_based_fops_1 (strcat(filename, "_1"));
+        if (ret < 0)
+                fprintf (stderr, "fd based file operation 1 failed\n");
+        else
+                fprintf (stdout, "fd based file operation 1 passed\n");
+
+        ret = fd_based_fops_2 (strcat(filename, "_2"));
+        if (ret < 0)
+                fprintf (stderr, "fd based file operation 2 failed\n");
+        else
+                fprintf (stdout, "fd based file operation 2 passed\n");
+
+        ret = dup_fd_based_fops (strcat (filename, "_3"));
+        if (ret < 0)
+                fprintf (stderr, "dup fd based file operation failed\n");
+        else
+                fprintf (stdout, "dup fd based file operation passed\n");
 
-        if (!filename)
-                filename = "temp-xattr-test-file";
+        ret = path_based_fops (strcat (filename, "_4"));
+        if (ret < 0)
+                fprintf (stderr, "path based file operation failed\n");
+        else
+                fprintf (stdout, "path based file operation passed\n");
+
+        ret = dir_based_fops (strcat (filename, "_5"));
+        if (ret < 0)
+                fprintf (stderr, "directory based file operation failed\n");
+        else
+                fprintf (stdout, "directory based file operation passed\n");
+
+        ret = link_based_fops (strcat (filename, "_5"));
+        if (ret < 0)
+                fprintf (stderr, "link based file operation failed\n");
+        else
+                fprintf (stdout, "link based file operation passed\n");
+
+        ret = test_open_modes (strcat (filename, "_5"));
+        if (ret < 0)
+                fprintf (stderr, "testing modes of 'open' call failed\n");
+        else
+                fprintf (stdout, "testing modes of 'open' call passed\n");
+
+out:
+        return ret;
+}
+
+int
+fd_based_fops_1 (char *filename)
+{
+        int         fd        = 0;
+        int         ret       = -1;
+        struct stat stbuf     = {0,};
+        char        wstr[50]  = {0,};
+        char        rstr[50]  = {0,};
 
         fd = open (filename, O_RDWR|O_CREAT);
         if (fd < 0) {
@@ -34,6 +96,34 @@ main (int argc, char *argv[])
                 goto out;
         }
 
+        strcpy (wstr, "This is my string\n");
+        ret = write (fd, wstr, strlen(wstr));
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "write failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = lseek (fd, 0, SEEK_SET);
+        if (ret < 0) {
+                fprintf (stderr, "lseek failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = read (fd, rstr, strlen(wstr));
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "read failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = memcmp (rstr, wstr, strlen (wstr));
+        if (ret != 0) {
+                ret = -1;
+                fprintf (stderr, "read returning junk\n");
+                goto out;
+        }
+
         ret = ftruncate (fd, 0);
         if (ret < 0) {
                 fprintf (stderr, "ftruncate failed : %s\n", strerror (errno));
@@ -103,3 +193,678 @@ out:
 
         return ret;
 }
+
+
+int
+fd_based_fops_2 (char *filename)
+{
+        int         fd    = 0;
+        int         ret   = -1;
+        struct stat stbuf = {0,};
+        char        wstr[50]  = {0,};
+        char        rstr[50]  = {0,};
+
+        fd = open (filename, O_RDWR|O_CREAT);
+        if (fd < 0) {
+                fd = 0;
+                fprintf (stderr, "open failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = ftruncate (fd, 0);
+
+        if (ret < 0) {
+                fprintf (stderr, "ftruncate failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        strcpy (wstr, "This is my second string\n");
+        ret = write (fd, wstr, strlen (wstr));
+        if (ret < 0) {
+                ret = -1;
+                fprintf (stderr, "write failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        lseek (fd, 0, SEEK_SET);
+        if (ret < 0) {
+                fprintf (stderr, "lseek failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = read (fd, rstr, strlen (wstr));
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "read failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = memcmp (rstr, wstr, strlen (wstr));
+        if (ret != 0) {
+                ret = -1;
+                fprintf (stderr, "read returning junk\n");
+                goto out;
+        }
+
+        ret = fstat (fd, &stbuf);
+        if (ret < 0) {
+                fprintf (stderr, "fstat failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fchmod (fd, 0640);
+        if (ret < 0) {
+                fprintf (stderr, "fchmod failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fchown (fd, 10001, 10001);
+        if (ret < 0) {
+                fprintf (stderr, "fchown failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fsync (fd);
+        if (ret < 0) {
+                fprintf (stderr, "fsync failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fsetxattr (fd, "trusted.xattr-test", "working", 8, 0);
+        if (ret < 0) {
+                fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fdatasync (fd);
+        if (ret < 0) {
+                fprintf (stderr, "fdatasync failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = flistxattr (fd, NULL, 0);
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "flistxattr failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fgetxattr (fd, "trusted.xattr-test", NULL, 0);
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fremovexattr (fd, "trusted.xattr-test");
+        if (ret < 0) {
+                fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+out:
+        if (fd)
+                close (fd);
+        unlink (filename);
+
+        return ret;
+}
+
+int
+path_based_fops (char *filename)
+{
+        int         ret              = -1;
+        int         fd               = 0;
+        struct stat stbuf            = {0,};
+        char        newfilename[255] = {0,};
+
+        fd = creat (filename, 0644);
+        if (fd < 0) {
+                fprintf (stderr, "creat failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = truncate (filename, 0);
+        if (ret < 0) {
+                fprintf (stderr, "truncate failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = stat (filename, &stbuf);
+        if (ret < 0) {
+                fprintf (stderr, "stat failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = chmod (filename, 0640);
+        if (ret < 0) {
+                fprintf (stderr, "chmod failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = chown (filename, 10001, 10001);
+        if (ret < 0) {
+                fprintf (stderr, "chown failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = setxattr (filename, "trusted.xattr-test", "working", 8, 0);
+        if (ret < 0) {
+                fprintf (stderr, "setxattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = listxattr (filename, NULL, 0);
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "listxattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = getxattr (filename, "trusted.xattr-test", NULL, 0);
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "getxattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = removexattr (filename, "trusted.xattr-test");
+        if (ret < 0) {
+                fprintf (stderr, "removexattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = access (filename, R_OK|W_OK);
+        if (ret < 0) {
+                fprintf (stderr, "access failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        strcpy (newfilename, filename);
+        strcat(newfilename, "_new");
+        ret = rename (filename, newfilename);
+        if (ret < 0) {
+                fprintf (stderr, "rename failed: %s\n", strerror (errno));
+                goto out;
+        }
+        unlink (newfilename);
+
+out:
+        if (fd)
+                close (fd);
+
+        unlink (filename);
+        return ret;
+}
+
+int
+dup_fd_based_fops (char *filename)
+{
+        int         fd        = 0;
+        int         newfd     = 0;
+        int         ret       = -1;
+        struct stat stbuf     = {0,};
+        char        wstr[50]  = {0,};
+        char        rstr[50]  = {0,};
+
+        fd = open (filename, O_RDWR|O_CREAT);
+        if (fd < 0) {
+                fd = 0;
+                fprintf (stderr, "open failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        newfd = dup (fd);
+        if (newfd < 0) {
+                ret = -1;
+                fprintf (stderr, "dup failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        close (fd);
+
+        strcpy (wstr, "This is my string\n");
+        ret = write (newfd, wstr, strlen(wstr));
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "write failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = lseek (newfd, 0, SEEK_SET);
+        if (ret < 0) {
+                fprintf (stderr, "lseek failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = read (newfd, rstr, strlen(wstr));
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "read failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = memcmp (rstr, wstr, strlen (wstr));
+        if (ret != 0) {
+                ret = -1;
+                fprintf (stderr, "read returning junk\n");
+                goto out;
+        }
+
+        ret = ftruncate (newfd, 0);
+        if (ret < 0) {
+                fprintf (stderr, "ftruncate failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fstat (newfd, &stbuf);
+        if (ret < 0) {
+                fprintf (stderr, "fstat failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fchmod (newfd, 0640);
+        if (ret < 0) {
+                fprintf (stderr, "fchmod failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fchown (newfd, 10001, 10001);
+        if (ret < 0) {
+                fprintf (stderr, "fchown failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fsync (newfd);
+        if (ret < 0) {
+                fprintf (stderr, "fsync failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fsetxattr (newfd, "trusted.xattr-test", "working", 8, 0);
+        if (ret < 0) {
+                fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fdatasync (newfd);
+        if (ret < 0) {
+                fprintf (stderr, "fdatasync failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = flistxattr (newfd, NULL, 0);
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "flistxattr failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fgetxattr (newfd, "trusted.xattr-test", NULL, 0);
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = fremovexattr (newfd, "trusted.xattr-test");
+        if (ret < 0) {
+                fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (newfd)
+                close (newfd);
+        ret = unlink (filename);
+        if (ret < 0) {
+                fprintf (stderr, "unlink failed : %s\n", strerror (errno));
+                goto out;
+        }
+
+        return ret;
+}
+
+int
+dir_based_fops (char *dirname)
+{
+        int            ret           = -1;
+        DIR           *dp            = NULL;
+        char           buff[255]     = {0,};
+        struct dirent *dbuff         = {0,};
+        struct stat    stbuff        = {0,};
+        char           newdname[255] = {0,};
+        char          *cwd           = NULL;
+
+        ret = mkdir (dirname, 0755);
+        if (ret < 0) {
+                fprintf (stderr, "mkdir failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        dp = opendir (dirname);
+        if (dp == NULL) {
+                fprintf (stderr, "opendir failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        dbuff = readdir (dp);
+        if (NULL == dbuff) {
+                fprintf (stderr, "readdir failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = closedir (dp);
+        if (ret < 0) {
+                fprintf (stderr, "closedir failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = stat (dirname, &stbuff);
+        if (ret < 0) {
+                fprintf (stderr, "stat failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = chmod (dirname, 0744);
+        if (ret < 0) {
+                fprintf (stderr, "chmod failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = chown (dirname, 10001, 10001);
+        if (ret < 0) {
+                fprintf (stderr, "chmod failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = setxattr (dirname, "trusted.xattr-test", "working", 8, 0);
+        if (ret < 0) {
+                fprintf (stderr, "setxattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = listxattr (dirname, NULL, 0);
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "listxattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = getxattr (dirname, "trusted.xattr-test", NULL, 0);
+        if (ret <= 0) {
+                ret = -1;
+                fprintf (stderr, "getxattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = removexattr (dirname, "trusted.xattr-test");
+        if (ret < 0) {
+                fprintf (stderr, "removexattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        strcpy (newdname, dirname);
+        strcat (newdname, "/../");
+        ret = chdir (newdname);
+        if (ret < 0) {
+                fprintf (stderr, "chdir failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        cwd = getcwd (buff, 255);
+        if (NULL == cwd) {
+                fprintf (stderr, "getcwd failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        strcpy (newdname, dirname);
+        strcat (newdname, "new");
+        ret = rename (dirname, newdname);
+        if (ret < 0) {
+                fprintf (stderr, "rename failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = rmdir (newdname);
+        if (ret < 0) {
+                fprintf (stderr, "rmdir failed: %s\n", strerror (errno));
+                return ret;
+        }
+
+out:
+        rmdir (dirname);
+        return ret;
+}
+
+int
+link_based_fops (char *filename)
+{
+        int         ret           = -1;
+        int         fd            = 0;
+        char        newname[255]  = {0,};
+        char        linkname[255] = {0,};
+        struct stat lstbuf        = {0,};
+
+        fd = creat (filename, 0644);
+        if (fd < 0) {
+                fd = 0;
+                fprintf (stderr, "creat failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        strcpy (newname, filename);
+        strcat (newname, "_hlink");
+        ret = link (filename, newname);
+        if (ret < 0) {
+                fprintf (stderr, "link failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = unlink (filename);
+        if (ret < 0) {
+                fprintf (stderr, "unlink failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        strcpy (linkname, filename);
+        strcat (linkname, "_slink");
+        ret = symlink (newname, linkname);
+        if (ret < 0) {
+                fprintf (stderr, "symlink failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = lstat (linkname, &lstbuf);
+        if (ret < 0) {
+                fprintf (stderr, "lstbuf failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = lchown (linkname, 10001, 10001);
+        if (ret < 0) {
+                fprintf (stderr, "lchown failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = lsetxattr (linkname, "trusted.lxattr-test", "working", 8, 0);
+        if (ret < 0) {
+                fprintf (stderr, "lsetxattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = llistxattr (linkname, NULL, 0);
+        if (ret < 0) {
+                ret = -1;
+                fprintf (stderr, "llistxattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = lgetxattr (linkname, "trusted.lxattr-test", NULL, 0);
+        if (ret < 0) {
+                ret = -1;
+                fprintf (stderr, "lgetxattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+        ret = lremovexattr (linkname, "trusted.lxattr-test");
+        if (ret < 0) {
+                fprintf (stderr, "lremovexattr failed: %s\n", strerror (errno));
+                goto out;
+        }
+
+
+out:
+        if (fd)
+                close(fd);
+        unlink (linkname);
+        unlink (newname);
+}
+
+int
+test_open_modes (char *filename)
+{
+        int ret = -1;
+
+        ret = generic_open_read_write (filename, O_CREAT|O_WRONLY);
+        if (3 != ret) {
+               fprintf (stderr, "flag O_CREAT|O_WRONLY failed: \n");
+               goto out;
+        }
+
+        ret = generic_open_read_write (filename, O_CREAT|O_RDWR);
+        if (ret != 0) {
+               fprintf (stderr, "flag O_CREAT|O_RDWR failed\n");
+               goto out;
+        }
+
+        ret = generic_open_read_write (filename, O_CREAT|O_RDONLY);
+        if (ret != 0) {
+                fprintf (stderr, "flag O_CREAT|O_RDONLY failed\n");
+                goto out;
+        }
+
+        ret = creat (filename, 0644);
+        close (ret);
+        ret = generic_open_read_write (filename, O_WRONLY);
+        if (3 != ret) {
+               fprintf (stderr, "flag O_WRONLY failed\n");
+               goto out;
+        }
+
+        ret = creat (filename, 0644);
+        close (ret);
+        ret = generic_open_read_write (filename, O_RDWR);
+        if (0 != ret) {
+               fprintf (stderr, "flag O_RDWR failed\n");
+               goto out;
+        }
+
+        ret = creat (filename, 0644);
+        close (ret);
+        ret = generic_open_read_write (filename, O_RDONLY);
+        if (0 != ret) {
+               fprintf (stderr, "flag O_RDONLY failed\n");
+               goto out;
+        }
+
+        ret = creat (filename, 0644);
+        close (ret);
+        ret = generic_open_read_write (filename, O_TRUNC|O_WRONLY);
+        if (3 != ret) {
+               fprintf (stderr, "flag O_TRUNC|O_WRONLY failed\n");
+               goto out;
+        }
+
+#if 0 /* undefined behaviour, unable to reliably test */
+        ret = creat (filename, 0644);
+        close (ret);
+        ret = generic_open_read_write (filename, O_TRUNC|O_RDONLY);
+        if (0 != ret) {
+               fprintf (stderr, "flag O_TRUNC|O_RDONLY failed\n");
+               goto out;
+        }
+#endif
+
+        ret = generic_open_read_write (filename, O_CREAT|O_RDWR|O_SYNC);
+        if (0 != ret) {
+               fprintf (stderr, "flag O_CREAT|O_RDWR|O_SYNC failed\n");
+               goto out;
+        }
+
+        ret = creat (filename, 0644);
+        close (ret);
+        ret = generic_open_read_write (filename, O_CREAT|O_EXCL);
+        if (0 != ret) {
+                fprintf (stderr, "flag O_CREAT|O_EXCL failed\n");
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int generic_open_read_write (char *filename, int flag)
+{
+        int  fd          = 0;
+        int  ret         = -1;
+        char wstring[50] = {0,};
+        char rstring[50] = {0,};
+
+        fd = open (filename, flag);
+        if (fd < 0) {
+                if (flag == O_CREAT|O_EXCL && errno == EEXIST) {
+                        unlink (filename);
+                        return 0;
+                }
+                else {
+                        fd = 0;
+                        fprintf (stderr, "open failed: %s\n", strerror (errno));
+                        return 1;
+                }
+        }
+
+        strcpy (wstring, "My string to write\n");
+        ret = write (fd, wstring, strlen(wstring));
+        if (ret <= 0) {
+                if (errno != EBADF) {
+                        fprintf (stderr, "write failed: %s\n", strerror (errno));
+                        close (fd);
+                        unlink(filename);
+                        return 2;
+                }
+        }
+
+        ret = lseek (fd, 0, SEEK_SET);
+        if (ret < 0) {
+                close (fd);
+                unlink(filename);
+                return 4;
+        }
+
+        ret = read (fd, rstring, strlen(wstring));
+        if (ret < 0) {
+                close (fd);
+                unlink (filename);
+                return 3;
+        }
+
+        /* Compare the rstring with wstring. But we do not want to return
+         * error when the flag is either O_RDONLY, O_CREAT|O_RDONLY or
+         * O_TRUNC|O_RDONLY. Because in that case we are not writing
+         * anything to the file.*/
+
+        ret = memcmp (wstring, rstring, strlen (wstring));
+        if (0 != ret  && !(flag == O_CREAT|O_RDONLY || flag == O_RDONLY ||\
+            flag == O_TRUNC|O_RDONLY)) {
+                fprintf (stderr, "read is returning junk\n");
+                close (fd);
+                unlink (filename);
+                return 4;
+        }
+
+        close (fd);
+        unlink (filename);
+        return 0;
+}
diff --git a/extras/volfilter.py b/extras/volfilter.py
new file mode 100644
index 000000000..0ca456a78
--- /dev/null
+++ b/extras/volfilter.py
@@ -0,0 +1,167 @@
+# Copyright (c) 2010-2011 Red Hat, Inc.
+#
+# This file is part of HekaFS.
+#
+# HekaFS is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License, version 3, as published by the Free
+# Software Foundation.
+#
+# HekaFS is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License * along
+# with HekaFS.  If not, see <http://www.gnu.org/licenses/>.
+
+import copy
+import string
+import sys
+import types
+
+good_xlators = [
+	"cluster/afr",
+	"cluster/dht",
+	"cluster/distribute",
+	"cluster/replicate",
+	"cluster/stripe",
+	"debug/io-stats",
+	"features/access-control",
+	"features/locks",
+	"features/marker",
+	"features/uidmap",
+	"performance/io-threads",
+	"protocol/client",
+	"protocol/server",
+	"storage/posix",
+]
+
+def copy_stack (old_xl,suffix,recursive=False):
+	if recursive:
+		new_name = old_xl.name + "-" + suffix
+	else:
+		new_name = suffix
+	new_xl = Translator(new_name)
+	new_xl.type = old_xl.type
+	# The results with normal assignment here are . . . amusing.
+	new_xl.opts = copy.deepcopy(old_xl.opts)
+	for sv in old_xl.subvols:
+		new_xl.subvols.append(copy_stack(sv,suffix,True))
+	# Patch up the path at the bottom.
+	if new_xl.type == "storage/posix":
+		new_xl.opts["directory"] += ("/" + suffix)
+	return new_xl
+
+def cleanup (parent, graph):
+	if parent.type in good_xlators:
+		# Temporary fix so that HekaFS volumes can use the
+		# SSL-enabled multi-threaded socket transport.
+		if parent.type == "protocol/server":
+			parent.type = "protocol/server2"
+			parent.opts["transport-type"] = "ssl"
+		elif parent.type == "protocol/client":
+			parent.type = "protocol/client2"
+			parent.opts["transport-type"] = "ssl"
+		sv = []
+		for child in parent.subvols:
+			sv.append(cleanup(child,graph))
+		parent.subvols = sv
+	else:
+		parent = cleanup(parent.subvols[0],graph)
+	return parent
+
+class Translator:
+	def __init__ (self, name):
+		self.name = name
+		self.type = ""
+		self.opts = {}
+		self.subvols = []
+		self.dumped = False
+	def __repr__ (self):
+		return "<Translator %s>" % self.name
+
+def load (path):
+	# If it's a string, open it; otherwise, assume it's already a
+	# file-like object (most notably from urllib*).
+	if type(path) in types.StringTypes:
+		fp = file(path,"r")
+	else:
+		fp = path
+	all_xlators = {}
+	xlator = None
+	last_xlator = None
+	while True:
+		text = fp.readline()
+		if text == "":
+			break
+		text = text.split()
+		if not len(text):
+			continue
+		if text[0] == "volume":
+			if xlator:
+				raise RuntimeError, "nested volume definition"
+			xlator = Translator(text[1])
+			continue
+		if not xlator:
+			raise RuntimeError, "text outside volume definition"
+		if text[0] == "type":
+			xlator.type = text[1]
+			continue
+		if text[0] == "option":
+			xlator.opts[text[1]] = string.join(text[2:])
+			continue
+		if text[0] == "subvolumes":
+			for sv in text[1:]:
+				xlator.subvols.append(all_xlators[sv])
+			continue
+		if text[0] == "end-volume":
+			all_xlators[xlator.name] = xlator
+			last_xlator = xlator
+			xlator = None
+			continue
+		raise RuntimeError, "unrecognized keyword %s" % text[0]
+	if xlator:
+		raise RuntimeError, "unclosed volume definition"
+	return all_xlators, last_xlator
+
+def generate (graph, last, stream=sys.stdout):
+	for sv in last.subvols:
+		if not sv.dumped:
+			generate(graph,sv,stream)
+			print >> stream, ""
+			sv.dumped = True
+	print >> stream, "volume %s" % last.name
+	print >> stream, "    type %s" % last.type
+	for k, v in last.opts.iteritems():
+		print >> stream, "    option %s %s" % (k, v)
+	if last.subvols:
+		print >> stream, "    subvolumes %s" % string.join(
+			[ sv.name for sv in last.subvols ])
+	print >> stream, "end-volume"
+
+def push_filter (graph, old_xl, filt_type, opts={}):
+	suffix = "-" + old_xl.type.split("/")[1]
+	if len(old_xl.name) > len(suffix):
+		if old_xl.name[-len(suffix):] == suffix:
+			old_xl.name = old_xl.name[:-len(suffix)]
+	new_xl = Translator(old_xl.name+suffix)
+	new_xl.type = old_xl.type
+	new_xl.opts = old_xl.opts
+	new_xl.subvols = old_xl.subvols
+	graph[new_xl.name] = new_xl
+	old_xl.name += ("-" + filt_type.split("/")[1])
+	old_xl.type = filt_type
+	old_xl.opts = opts
+	old_xl.subvols = [new_xl]
+	graph[old_xl.name] = old_xl
+
+def delete (graph, victim):
+	if len(victim.subvols) != 1:
+		raise RuntimeError, "attempt to delete non-unary translator"
+	for xl in graph.itervalues():
+		while xl.subvols.count(victim):
+			i = xl.subvols.index(victim)
+			xl.subvols[i] = victim.subvols[0]
+
+if __name__ == "__main__":
+	graph, last = load(sys.argv[1])
+	generate(graph,last)
diff --git a/extras/who-wrote-glusterfs/gitdm.aliases b/extras/who-wrote-glusterfs/gitdm.aliases
new file mode 100644
index 000000000..784a3e3bc
--- /dev/null
+++ b/extras/who-wrote-glusterfs/gitdm.aliases
@@ -0,0 +1,48 @@
+#
+# This is the email aliases file, mapping secondary addresses onto a single,
+# canonical address. This file should probably match the contents of .mailmap
+# in the root of the git repository.
+#
+# Format: <alias> <real>
+
+amar@gluster.com	amarts@redhat.com
+amar@del.gluster.com	amarts@redhat.com
+avati@amp.gluster.com	avati@redhat.com
+avati@blackhole.gluster.com	avati@redhat.com
+avati@dev.gluster.com	avati@redhat.com
+avati@gluster.com	avati@redhat.com
+wheelear@gmail.com      awheeler@redhat.com
+anush@gluster.com	ashetty@redhat.com
+csaba@gluster.com	csaba@redhat.com
+csaba@lowlife.hu	csaba@redhat.com
+csaba@zresearch.com	csaba@redhat.com
+harsha@gluster.com	fharshav@redhat.com
+harsha@zresearch.com	fharshav@redhat.com
+harsha@dev.gluster.com	fharshav@redhat.com
+harsha@harshavardhana.net	fharshav@redhat.com
+kkeithle@f16node1.kkeithle.usersys.redhat.com	kkeithle@redhat.com
+kaushal@gluster.com	kaushal@redhat.com
+kaushikbv@gluster.com	kbudiger@redhat.com
+krishna@gluster.com	ksriniva@redhat.com
+krishna@zresearch.com	ksriniva@redhat.com
+krishna@guest-laptop	ksriniva@redhat.com
+kp@gluster.com	kparthas@redhat.com
+me@louiszuckerman.com	louiszuckerman@gmail.com
+msvbhat@gmail.com	vbhat@redhat.com
+vishwanath@gluster.com	vbhat@redhat.com
+pavan@dev.gluster.com	pavan@gluster.com
+zaitcev@yahoo.com	zaitcev@kotori.zaitcev.us
+pranithk@gluster.com	pkarampu@redhat.com
+raghavendrabhat@gluster.com	raghavendra@redhat.com
+raghavendra@gluster.com	rgowdapp@redhat.com
+raghavendra@zresearch.com	rgowdapp@redhat.com
+rahulcssjce@gmail.com	rahulcs@redhat.com
+rajesh@gluster.com	rajesh@redhat.com
+rajesh.amaravathi@gmail.com	rajesh@redhat.com
+shehjart@zresearch.com	shehjart@gluster.com
+venky@gluster.com	vshankar@redhat.com
+vijay@gluster.com	vbellur@redhat.com
+vijay@dev.gluster.com	vbellur@redhat.com
+vijaykumar.koppad@gmail.com	vkoppad@redhat.com
+vikas@zresearch.com	vikas@gluster.com
+shishirng@gluster.com	sgowda@redhat.com
diff --git a/extras/who-wrote-glusterfs/gitdm.config b/extras/who-wrote-glusterfs/gitdm.config
new file mode 100644
index 000000000..e1ff2bd5b
--- /dev/null
+++ b/extras/who-wrote-glusterfs/gitdm.config
@@ -0,0 +1,8 @@
+#
+# This is the gitdm configuration file for GlusterFS.
+# See the gitdm.config in the gitdm repositofy for additional options and
+# comments.
+#
+
+EmailAliases gitdm.aliases
+EmailMap gitdm.domain-map
diff --git a/extras/who-wrote-glusterfs/gitdm.domain-map b/extras/who-wrote-glusterfs/gitdm.domain-map
new file mode 100644
index 000000000..f1c305898
--- /dev/null
+++ b/extras/who-wrote-glusterfs/gitdm.domain-map
@@ -0,0 +1,15 @@
+#
+# Here is a set of mappings of domain names onto employer names.
+#
+active.by       ActiveCloud
+cern.ch         CERN
+gluster.com     Red Hat
+gooddata.com	GoodData
+hastexo.com     hastexo
+ibm.com		IBM
+linbit.com	LINBIT
+netbsd.org	NetBSD
+netdirect.ca	Net Direct
+redhat.com	Red Hat
+stepping-stone.ch       stepping stone GmbH
+zresearch.com   Red Hat
diff --git a/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh b/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh
new file mode 100755
index 000000000..487f5874b
--- /dev/null
+++ b/extras/who-wrote-glusterfs/who-wrote-glusterfs.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+#
+# Gather statistics on "Who wrote GlusterFS". The idea comes from the excellent
+# articles on http://lwn.net/ named "Who wrote <linux-version>?".
+#
+# gitdm comes from git://git.lwn.net/gitdm.git by Jonathan Corbet.
+#
+# Confguration files used:
+#  - gitdm.config: main configuration file, pointing to the others
+#  - gitdm.aliases: merge users with different emailaddresses into one
+#  - gitdm.domain-map: map domain names from emailaddresses to companies
+#
+
+DIRNAME=$(dirname $0)
+
+GITDM_REPO=git://git.lwn.net/gitdm.git
+GITDM_DIR=${DIRNAME}/gitdm
+GITDM_CMD="python ${GITDM_DIR}/gitdm"
+
+error()
+{
+        local ret=${?}
+        printf "${@}\n" > /dev/stderr
+        return ${ret}
+}
+
+check_gitdm()
+{
+        if [ ! -e "${GITDM_DIR}/gitdm" ]
+        then
+                git clone --quiet git://git.lwn.net/gitdm.git ${DIRNAME}/gitdm
+        fi
+}
+
+# The first argument is the revision-range (see 'git rev-list --help').
+# REV can be empty, and the statistics will be calculated over the whole
+# current branch.
+REV=${1}
+shift
+# all remaining options are passed to gitdm, see the gitdm script for an
+# explanation of the accepted options.
+GITDM_OPTS=${@}
+
+if ! check_gitdm
+then
+        error "Could not find 'gitdm', exiting..."
+        exit 1
+fi
+
+git log --numstat -M ${REV} | ${GITDM_CMD} -b ${DIRNAME} -n ${GITDM_OPTS}
diff --git a/gen-headers.py b/gen-headers.py
new file mode 100755
index 000000000..ef9fa7711
--- /dev/null
+++ b/gen-headers.py
@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+import sys
+try:
+    import json
+except ImportError:
+    import simplejson as json
+from string import Template
+
+
+def getLogBook(logFile='error-codes.json'):
+    fp = open(logFile)
+    return json.load(fp)
+
+
+def genCHeader(logBook,
+               infile='gf-error-codes.h.template',
+               outfile='gf-error-codes.h'):
+    fp = open('gf-error-codes.h.template')
+    s = fp.read()
+    fp.close()
+    template = Template(s)
+
+    defineLines = []
+    caseLines = []
+    for name, value in logBook.iteritems():
+        nameDef = "GF_%s" % (name.upper(),)
+        code = value['code']
+        msgNameDef = "%s_MSG" % (nameDef,)
+        msg = value['message']['en']
+
+        defineLines.append("#define %-20s %d" % (nameDef, code))
+        defineLines.append("#define %-20s %s" % (msgNameDef,
+                                                 json.dumps(msg)))
+        caseLines.append("        case %s: return _(%s);" % \
+                             (nameDef, msgNameDef))
+
+    d = {'DEFINES': "\n".join(defineLines),
+         'CASES': "\n".join(caseLines)}
+    #print template.substitute(d)
+
+    fp = open(outfile, 'w')
+    fp.write(template.substitute(d))
+    fp.close()
+
+
+if __name__ == "__main__":
+    try:
+        logBook = getLogBook()
+        genCHeader(logBook)
+        sys.exit(0)
+    except IOError, e:
+        print str(e)
+        sys.exit(-1)
diff --git a/xlators/features/marker/utils/Makefile.am b/geo-replication/Makefile.am
index 556951d9f..556951d9f 100644
--- a/xlators/features/marker/utils/Makefile.am
+++ b/geo-replication/Makefile.am
diff --git a/geo-replication/src/Makefile.am b/geo-replication/src/Makefile.am
new file mode 100644
index 000000000..324d8869f
--- /dev/null
+++ b/geo-replication/src/Makefile.am
@@ -0,0 +1,33 @@
+
+gsyncddir = $(libexecdir)/glusterfs
+
+gsyncd_SCRIPTS = gverify.sh peer_add_secret_pub peer_gsec_create
+
+# peer_gsec_create and peer_add_secret_pub are not added to
+# EXTRA_DIST as it's derived from a .in file
+EXTRA_DIST = gverify.sh
+
+gsyncd_PROGRAMS = gsyncd
+
+gsyncd_SOURCES = gsyncd.c procdiggy.c
+
+gsyncd_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	$(GF_GLUSTERFS_LIBS)
+
+gsyncd_LDFLAGS = $(GF_LDFLAGS)
+
+noinst_HEADERS = procdiggy.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src\
+	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
+	-DUSE_LIBGLUSTERFS\
+	-DSBIN_DIR=\"$(sbindir)\" -DPYTHON=\"$(PYTHON)\"
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+
+CLEANFILES =
+
+$(top_builddir)/libglusterfs/src/libglusterfs.la:
+	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
diff --git a/xlators/features/marker/utils/src/gsyncd.c b/geo-replication/src/gsyncd.c
index d554e562b..0830e7f9b 100644
--- a/xlators/features/marker/utils/src/gsyncd.c
+++ b/geo-replication/src/gsyncd.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -27,6 +17,19 @@
 #include <stdio.h>
 #include <unistd.h>
 #include <string.h>
+#include <sys/param.h> /* for PATH_MAX */
+
+/* NOTE (USE_LIBGLUSTERFS):
+ * ------------------------
+ * When USE_LIBGLUSTERFS debugging sumbol is passed; perform
+ * glusterfs translator like initialization so that glusterfs
+ * globals, contexts are valid when glustefs api's are invoked.
+ * We unconditionally pass then while building gsyncd binary.
+ */
+#ifdef USE_LIBGLUSTERFS
+#include "glusterfs.h"
+#include "globals.h"
+#endif
 
 #include "common-utils.h"
 #include "run.h"
@@ -34,7 +37,7 @@
 
 #define _GLUSTERD_CALLED_ "_GLUSTERD_CALLED_"
 #define _GSYNCD_DISPATCHED_ "_GSYNCD_DISPATCHED_"
-#define GSYNCD_CONF "geo-replication/gsyncd.conf"
+#define GSYNCD_CONF_TEMPLATE "geo-replication/gsyncd_template.conf"
 #define GSYNCD_PY "gsyncd.py"
 #define RSYNC "rsync"
 
@@ -44,12 +47,14 @@ static int
 duplexpand (void **buf, size_t tsiz, size_t *len)
 {
         size_t osiz = tsiz * *len;
-
-        *buf = realloc (*buf, osiz << 1);
-        if (!buf)
+        char *p = realloc (*buf, osiz << 1);
+        if (!p) {
+                free(*buf);
                 return -1;
+        }
 
-        memset ((char *)*buf + osiz, 0, osiz);
+        memset (p + osiz, 0, osiz);
+        *buf = p;
         *len <<= 1;
 
         return 0;
@@ -59,6 +64,7 @@ static int
 str2argv (char *str, char ***argv)
 {
         char *p         = NULL;
+        char *savetok   = NULL;
         int argc        = 0;
         size_t argv_len = 32;
         int ret         = 0;
@@ -72,7 +78,7 @@ str2argv (char *str, char ***argv)
         if (!*argv)
                 goto error;
 
-        while ((p = strtok (str, " "))) {
+        while ((p = strtok_r (str, " ", &savetok))) {
                 str = NULL;
 
                 argc++;
@@ -102,8 +108,10 @@ invoke_gsyncd (int argc, char **argv)
         int i                      = 0;
         int j                      = 0;
         char *nargv[argc + 4];
+        char *python = NULL;
 
         if (restricted) {
+                size_t len;
                 /* in restricted mode we forcibly use the system-wide config */
                 runinit (&runner);
                 runner_add_args (&runner, SBIN_DIR"/gluster",
@@ -113,16 +121,17 @@ invoke_gsyncd (int argc, char **argv)
                 if (runner_start (&runner) == 0 &&
                     fgets (config_file, PATH_MAX,
                            runner_chio (&runner, STDOUT_FILENO)) != NULL &&
-                    config_file[strlen (config_file) - 1] == '\n' &&
+                    (len = strlen (config_file)) &&
+                    config_file[len - 1] == '\n' &&
                     runner_end (&runner) == 0)
-                        gluster_workdir_len = strlen (config_file) - 1;
+                        gluster_workdir_len = len - 1;
 
                 if (gluster_workdir_len) {
-                        if (gluster_workdir_len + 1 + strlen (GSYNCD_CONF) + 1 >
+                        if (gluster_workdir_len + 1 + strlen (GSYNCD_CONF_TEMPLATE) + 1 >
                             PATH_MAX)
                                 goto error;
                         config_file[gluster_workdir_len] = '/';
-                        strcat (config_file, GSYNCD_CONF);
+                        strcat (config_file, GSYNCD_CONF_TEMPLATE);
                 } else
                         goto error;
 
@@ -134,7 +143,10 @@ invoke_gsyncd (int argc, char **argv)
                 goto error;
 
         j = 0;
-        nargv[j++] = PYTHON;
+        python = getenv("PYTHON");
+        if(!python)
+                python = PYTHON;
+        nargv[j++] = python;
         nargv[j++] = GSYNCD_PREFIX"/python/syncdaemon/"GSYNCD_PY;
         for (i = 1; i < argc; i++)
                 nargv[j++] = argv[i];
@@ -144,9 +156,9 @@ invoke_gsyncd (int argc, char **argv)
         }
         nargv[j++] = NULL;
 
-        execvp (PYTHON, nargv);
+        execvp (python, nargv);
 
-        fprintf (stderr, "exec of "PYTHON" failed\n");
+        fprintf (stderr, "exec of '%s' failed\n", python);
         return 127;
 
  error:
@@ -159,6 +171,7 @@ static int
 find_gsyncd (pid_t pid, pid_t ppid, char *name, void *data)
 {
         char buf[NAME_MAX * 2] = {0,};
+        char path[PATH_MAX]    = {0,};
         char *p                = NULL;
         int zeros              = 0;
         int ret                = 0;
@@ -168,13 +181,8 @@ find_gsyncd (pid_t pid, pid_t ppid, char *name, void *data)
         if (ppid != pida[0])
                 return 0;
 
-        ret = gf_asprintf (&p, PROC"/%d/cmdline", pid);
-        if (ret == -1) {
-                fprintf (stderr, "out of memory\n");
-                return -1;
-        }
-
-        fd = open (p, O_RDONLY);
+        sprintf (path, PROC"/%d/cmdline", pid);
+        fd = open (path, O_RDONLY);
         if (fd == -1)
                 return 0;
         ret = read (fd, buf, sizeof (buf));
@@ -213,7 +221,7 @@ static int
 invoke_rsync (int argc, char **argv)
 {
         int i                  = 0;
-        char *p                = NULL;
+        char path[PATH_MAX]    = {0,};
         pid_t pid              = -1;
         pid_t ppid             = -1;
         pid_t pida[]           = {-1, -1};
@@ -240,8 +248,11 @@ invoke_rsync (int argc, char **argv)
                         fprintf (stderr, "sshd ancestor not found\n");
                         goto error;
                 }
-                if (strcmp (name, "sshd") == 0)
+                if (strcmp (name, "sshd") == 0) {
+                        GF_FREE (name);
                         break;
+                }
+                GF_FREE (name);
         }
         /* look up "ssh-sibling" gsyncd */
         pida[0] = pid;
@@ -251,15 +262,12 @@ invoke_rsync (int argc, char **argv)
                 goto error;
         }
         /* check if rsync target matches gsyncd target */
-        if (gf_asprintf (&p, PROC"/%d/cwd", pida[1]) == -1) {
-                fprintf (stderr, "out of memory\n");
-                goto error;
-        }
-        ret = readlink (p, buf, sizeof (buf));
+        sprintf (path, PROC"/%d/cwd", pida[1]);
+        ret = readlink (path, buf, sizeof (buf));
         if (ret == -1 || ret == sizeof (buf))
                 goto error;
         if (strcmp (argv[argc - 1], "/") == 0 /* root dir cannot be a target */ ||
-            (strcmp (argv[argc - 1], p) /* match against gluster target */ &&
+            (strcmp (argv[argc - 1], path) /* match against gluster target */ &&
              strcmp (argv[argc - 1], buf) /* match against file target */) != 0) {
                 fprintf (stderr, "rsync target does not match "GEOREP" session\n");
                 goto error;
@@ -277,6 +285,46 @@ invoke_rsync (int argc, char **argv)
         return 1;
 }
 
+static int
+invoke_gluster (int argc, char **argv)
+{
+        int i = 0;
+        int j = 0;
+        int optsover = 0;
+        char *ov = NULL;
+
+        for (i = 1; i < argc; i++) {
+                ov = strtail (argv[i], "--");
+                if (ov && !optsover) {
+                        if (*ov == '\0')
+                                optsover = 1;
+                        continue;
+                }
+                switch (++j) {
+                case 1:
+                        if (strcmp (argv[i], "volume") != 0)
+                                goto error;
+                        break;
+                case 2:
+                        if (strcmp (argv[i], "info") != 0)
+                                goto error;
+                        break;
+                case 3:
+                        break;
+                default:
+                        goto error;
+                }
+        }
+
+        argv[0] = "gluster";
+        execvp (SBIN_DIR"/gluster", argv);
+        fprintf (stderr, "exec of gluster failed\n");
+        return 127;
+
+ error:
+        fprintf (stderr, "disallowed gluster invocation\n");
+        return 1;
+}
 
 struct invocable {
         char *name;
@@ -284,8 +332,9 @@ struct invocable {
 };
 
 struct invocable invocables[] = {
-        { "rsync",  invoke_rsync  },
-        { "gsyncd", invoke_gsyncd },
+        { "rsync",   invoke_rsync  },
+        { "gsyncd",  invoke_gsyncd },
+        { "gluster", invoke_gluster },
         { NULL, NULL}
 };
 
@@ -297,6 +346,19 @@ main (int argc, char **argv)
         char *b             = NULL;
         char *sargv         = NULL;
 
+#ifdef USE_LIBGLUSTERFS
+        glusterfs_ctx_t *ctx = NULL;
+
+        ctx = glusterfs_ctx_new ();
+        if (!ctx)
+                return ENOMEM;
+
+        if (glusterfs_globals_init (ctx))
+                return 1;
+
+        THIS->ctx = ctx;
+#endif
+
         evas = getenv (_GLUSTERD_CALLED_);
         if (evas && strcmp (evas, "1") == 0)
                 /* OK, we know glusterd called us, no need to look for further config
diff --git a/geo-replication/src/gverify.sh b/geo-replication/src/gverify.sh
new file mode 100755
index 000000000..bd1b25f24
--- /dev/null
+++ b/geo-replication/src/gverify.sh
@@ -0,0 +1,160 @@
+#!/bin/bash
+
+# Script to verify the Master and Slave Gluster compatibility.
+# To use ./gverify <master volume> <slave host> <slave volume>
+# Returns 0 if master and slave compatible.
+
+# Considering buffer_size 100MB
+BUFFER_SIZE=104857600;
+slave_log_file=`gluster --print-logdir`/geo-replication-slaves/slave.log
+
+function SSHM()
+{
+    ssh -q \
+	-oPasswordAuthentication=no \
+	-oStrictHostKeyChecking=no \
+	-oControlMaster=yes \
+	"$@";
+}
+
+function cmd_master()
+{
+    VOL=$1;
+    local cmd_line;
+    cmd_line=$(cat <<EOF
+function do_verify() {
+v=\$1;
+d=\$(mktemp -d 2>/dev/null);
+glusterfs -s localhost --xlator-option="*dht.lookup-unhashed=off" --volfile-id \$v -l $slave_log_file \$d;
+i=\$(stat -c "%i" \$d);
+if [[ "\$i" -ne "1" ]]; then
+echo 0:0;
+exit 1;
+fi;
+cd \$d;
+available_size=\$(df \$d | tail -1 | awk "{print \\\$2}");
+umount -l \$d;
+rmdir \$d;
+ver=\$(gluster --version | head -1 | cut -f2 -d " ");
+echo \$available_size:\$ver;
+};
+cd /tmp;
+[ x$VOL != x ] && do_verify $VOL;
+EOF
+);
+
+echo $cmd_line;
+}
+
+function cmd_slave()
+{
+    VOL=$1;
+    local cmd_line;
+    cmd_line=$(cat <<EOF
+function do_verify() {
+v=\$1;
+d=\$(mktemp -d 2>/dev/null);
+glusterfs -s localhost --xlator-option="*dht.lookup-unhashed=off" --volfile-id \$v -l $slave_log_file \$d;
+i=\$(stat -c "%i" \$d);
+if [[ "\$i" -ne "1" ]]; then
+echo 0:0;
+exit 1;
+fi;
+cd \$d;
+available_size=\$(df \$d | tail -1 | awk "{print \\\$4}");
+no_of_files=\$(find  \$d -maxdepth 0 -empty);
+umount -l \$d;
+rmdir \$d;
+ver=\$(gluster --version | head -1 | cut -f2 -d " ");
+echo \$available_size:\$ver:\$no_of_files:;
+};
+cd /tmp;
+[ x$VOL != x ] && do_verify $VOL;
+EOF
+);
+
+echo $cmd_line;
+}
+
+function master_stats()
+{
+    MASTERVOL=$1;
+    local cmd_line;
+    cmd_line=$(cmd_master $MASTERVOL);
+    bash -c "$cmd_line";
+}
+
+
+function slave_stats()
+{
+    SLAVEHOST=$1;
+    SLAVEVOL=$2;
+    local cmd_line;
+    cmd_line=$(cmd_slave $SLAVEVOL);
+    SSHM $SLAVEHOST bash -c "'$cmd_line'";
+}
+
+
+function main()
+{
+    log_file=$4
+    > $log_file
+
+    # Use FORCE_BLOCKER flag in the error message to differentiate
+    # between the errors which the force command should bypass
+
+    ping -w 5 $2;
+    if [ $? -ne 0 ]; then
+        echo "FORCE_BLOCKER|$2 not reachable." > $log_file
+        exit 1;
+    fi;
+
+    ssh -oNumberOfPasswordPrompts=0 $2 "echo Testing_Passwordless_SSH";
+    if [ $? -ne 0 ]; then
+        echo "FORCE_BLOCKER|Passwordless ssh login has not been setup with $2." > $log_file
+        exit 1;
+    fi;
+
+    ERRORS=0;
+    master_data=$(master_stats $1);
+    slave_data=$(slave_stats $2 $3);
+    master_size=$(echo $master_data | cut -f1 -d':');
+    slave_size=$(echo $slave_data | cut -f1 -d':');
+    master_version=$(echo $master_data | cut -f2 -d':');
+    slave_version=$(echo $slave_data | cut -f2 -d':');
+    slave_no_of_files=$(echo $slave_data | cut -f3 -d':');
+
+    if [[ "x$master_size" = "x" || "x$master_version" = "x" || "$master_size" -eq "0" ]]; then
+	echo "FORCE_BLOCKER|Unable to fetch master volume details. Please check the master cluster and master volume." > $log_file;
+	exit 1;
+    fi;
+
+    if [[ "x$slave_size" = "x" || "x$slave_version" = "x" || "$slave_size" -eq "0" ]]; then
+	echo "FORCE_BLOCKER|Unable to fetch slave volume details. Please check the slave cluster and slave volume." > $log_file;
+	exit 1;
+    fi;
+
+    # The above checks are mandatory and force command should be blocked
+    # if they fail. The checks below can be bypassed if force option is
+    # provided hence no FORCE_BLOCKER flag.
+
+    if [ ! $slave_size -ge $(($master_size - $BUFFER_SIZE )) ]; then
+	echo "Total size of master is greater than available size of slave." >> $log_file;
+	ERRORS=$(($ERRORS + 1));
+    fi;
+
+    if [ -z $slave_no_of_files ]; then
+        echo "$2::$3 is not empty. Please delete existing files in $2::$3 and retry, or use force to continue without deleting the existing files." >> $log_file;
+        ERRORS=$(($ERRORS + 1));
+    fi;
+ 
+    if [[ $master_version > $slave_version ]]; then
+	echo "Gluster version mismatch between master and slave." >> $log_file;
+	ERRORS=$(($ERRORS + 1));
+    fi;
+
+    exit $ERRORS;
+}
+
+
+main "$@";
diff --git a/geo-replication/src/peer_add_secret_pub.in b/geo-replication/src/peer_add_secret_pub.in
new file mode 100644
index 000000000..c036cf334
--- /dev/null
+++ b/geo-replication/src/peer_add_secret_pub.in
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ ! -d ~/.ssh ]; then
+        mkdir ~/.ssh;
+        chmod 700 ~/.ssh
+        chown root:root ~/.ssh
+fi
+
+cat "$GLUSTERD_WORKING_DIR"/geo-replication/common_secret.pem.pub >> ~/.ssh/authorized_keys
diff --git a/geo-replication/src/peer_gsec_create.in b/geo-replication/src/peer_gsec_create.in
new file mode 100755
index 000000000..ef630bd44
--- /dev/null
+++ b/geo-replication/src/peer_gsec_create.in
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+
+if [ ! -f "$GLUSTERD_WORKING_DIR"/geo-replication/secret.pem.pub ]; then
+    \rm -rf "$GLUSTERD_WORKING_DIR"/geo-replication/secret.pem*
+    ssh-keygen -N '' -f "$GLUSTERD_WORKING_DIR"/geo-replication/secret.pem > /dev/null
+fi
+
+output=`echo command=\"@libexecdir@/glusterfs/gsyncd\" " "``cat "$GLUSTERD_WORKING_DIR"/geo-replication/secret.pem.pub`
+echo $output
diff --git a/xlators/features/marker/utils/src/procdiggy.c b/geo-replication/src/procdiggy.c
index fc0f97999..1eba414c1 100644
--- a/xlators/features/marker/utils/src/procdiggy.c
+++ b/geo-replication/src/procdiggy.c
@@ -1,23 +1,13 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -28,6 +18,7 @@
 #include <unistd.h>
 #include <string.h>
 #include <ctype.h>
+#include <sys/param.h> /* for PATH_MAX */
 
 #include "common-utils.h"
 #include "procdiggy.h"
@@ -37,35 +28,38 @@ pidinfo (pid_t pid, char **name)
 {
         char buf[NAME_MAX * 2] = {0,};
         FILE *f                = NULL;
+        char path[PATH_MAX]    = {0,};
         char *p                = NULL;
         int ret                = 0;
 
-        ret = gf_asprintf (&p, PROC"/%d/status", pid);
-        if (ret == -1)
-                goto oom;
+        sprintf (path, PROC"/%d/status", pid);
 
-        f = fopen (p, "r");
+        f = fopen (path, "r");
         if (!f)
                 return -1;
 
         if (name)
                 *name = NULL;
         for (;;) {
+                size_t len;
                 memset (buf, 0, sizeof (buf));
                 if (fgets (buf, sizeof (buf), f) == NULL ||
-                    buf[strlen (buf) - 1] != '\n') {
+                    (len = strlen (buf)) == 0 ||
+                    buf[len - 1] != '\n') {
                         pid = -1;
                         goto out;
                 }
-                buf[strlen (buf) -1] = '\0';
+                buf[len - 1] = '\0';
 
                 if (name && !*name) {
                         p = strtail (buf, "Name:");
                         if (p) {
                                 while (isspace (*++p));
                                 *name = gf_strdup (p);
-                                if (!*name)
-                                        goto oom;
+                                if (!*name) {
+                                        pid = -2;
+                                        goto out;
+                                }
                                 continue;
                         }
                 }
@@ -82,16 +76,15 @@ pidinfo (pid_t pid, char **name)
 
  out:
         fclose (f);
+        if (pid == -1 && name && *name)
+                GF_FREE (name);
+        if (pid == -2)
+                fprintf (stderr, "out of memory\n");
         return pid;
-
- oom:
-        fclose (f);
-        fprintf (stderr, "out of memory\n");
-        return -2;
 }
 
 int
-prociter (int (*proch) (pid_t pid, pid_t ppid, char *name, void *data),
+prociter (int (*proch) (pid_t pid, pid_t ppid, char *tmpname, void *data),
           void *data)
 {
         char *name        = NULL;
@@ -102,23 +95,27 @@ prociter (int (*proch) (pid_t pid, pid_t ppid, char *name, void *data),
         int ret           = 0;
 
         d = opendir (PROC);
+        if (!d)
+                return -1;
         while (errno = 0, de = readdir (d)) {
                 if (gf_string2int (de->d_name, &pid) != -1 && pid >= 0) {
                         ppid = pidinfo (pid, &name);
                         switch (ppid) {
                         case -1: continue;
-                        case -2: return -1;
+                        case -2: ret = -1; break;
                         }
                         ret = proch (pid, ppid, name, data);
+                        GF_FREE (name);
                         if (ret)
-                                return ret;
+                                break;
                 }
         }
-        if (errno) {
+        closedir (d);
+        if (!de && errno) {
                 fprintf (stderr, "failed to traverse "PROC" (%s)\n",
                          strerror (errno));
-                return -1;
+                ret = -1;
         }
 
-        return 0;
+        return ret;
 }
diff --git a/geo-replication/src/procdiggy.h b/geo-replication/src/procdiggy.h
new file mode 100644
index 000000000..56dfc4eb2
--- /dev/null
+++ b/geo-replication/src/procdiggy.h
@@ -0,0 +1,20 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifdef __NetBSD__
+#include <sys/syslimits.h>
+#endif /* __NetBSD__ */
+
+#define PROC "/proc"
+
+pid_t pidinfo (pid_t pid, char **name);
+
+int prociter (int (*proch) (pid_t pid, pid_t ppid, char *name, void *data),
+              void *data);
+
diff --git a/geo-replication/syncdaemon/Makefile.am b/geo-replication/syncdaemon/Makefile.am
new file mode 100644
index 000000000..83f969639
--- /dev/null
+++ b/geo-replication/syncdaemon/Makefile.am
@@ -0,0 +1,7 @@
+syncdaemondir = $(libexecdir)/glusterfs/python/syncdaemon
+
+syncdaemon_PYTHON = gconf.py gsyncd.py __init__.py master.py README.md repce.py \
+	resource.py configinterface.py syncdutils.py monitor.py libcxattr.py \
+	$(top_builddir)/contrib/ipaddr-py/ipaddr.py libgfchangelog.py
+
+CLEANFILES =
diff --git a/xlators/features/marker/utils/syncdaemon/README.md b/geo-replication/syncdaemon/README.md
index d45006932..67f346ace 100644
--- a/xlators/features/marker/utils/syncdaemon/README.md
+++ b/geo-replication/syncdaemon/README.md
@@ -11,13 +11,13 @@ Requirements are categorized according to this.
 * Python >= 2.5, or 2.4 with Ctypes (see below) (both)
 * OpenSSH >= 4.0 (master) / SSH2 compliant sshd (eg. openssh) (slave)
 * rsync (both)
-* glusterfs with marker support (master); glusterfs (optional on slave)
-* FUSE; for supported versions consult glusterfs
+* glusterfs: with marker and changelog support (master & slave);
+* FUSE: glusterfs fuse module with auxilary gfid based access support
 
 INSTALLATION
 ------------
 
-As of now, the supported way of operation is running from the source directory.
+As of now, the supported way of operation is running from the source directory or using the RPMs given.
 
 If you use Python 2.4.x, you need to install the [Ctypes module](http://python.net/crew/theller/ctypes/).
 
@@ -39,41 +39,18 @@ The config file format matches the following syntax:
       <option2>: <value2>
       # comment
 
-By default (unless specified by the option `-c`), gsyncd looks for config file at _conf/gsyncd.conf_
+By default (unless specified by the option `-c`), gsyncd looks for config file at _conf/gsyncd_template.conf_
 in the source tree.
 
 USAGE
 -----
 
 gsyncd is a utilitly for continous mirroring, ie. it mirrors master to slave incrementally.
-Assume we have a gluster volume _pop_ at localhost. We try to set up the following mirrors
-for it with gysncd:
+Assume we have a gluster volume _pop_ at localhost. We try to set up the mirroring for volume
+_pop_ using gsyncd for gluster volume _moz_ on remote machine/cluster @ example.com. The
+respective gsyncd invocations are (demoing some syntax sugaring):
 
-1. _/data/mirror_
-2. local gluster volume _yow_
-3. _/data/far_mirror_ at example.com
-4. gluster volume _moz_ at example.com
-
-The respective gsyncd invocations are (demoing some syntax sugaring):
-
-1.
-
-      gsyncd.py gluster://localhost:pop file:///data/mirror
-
-  or short form
-
-      gsyncd.py :pop /data/mirror
-
-2. `gsyncd :pop :yow`
-3.
-
-       gsyncd.py :pop ssh://example.com:/data/far_mirror
-
-  or short form
-
-       gsyncd.py :pop example.com:/data/far_mirror
-
-4. `gsyncd.py :pop example.com::moz`
+`gsyncd.py :pop example.com::moz`
 
 gsyncd has to be available on both sides; it's location on the remote side has to be specified
 via the "--remote-gsyncd" option (or "remote-gsyncd" config file parameter). (This option can also be
diff --git a/xlators/features/marker/utils/syncdaemon/__codecheck.py b/geo-replication/syncdaemon/__codecheck.py
index e3386afba..e3386afba 100644
--- a/xlators/features/marker/utils/syncdaemon/__codecheck.py
+++ b/geo-replication/syncdaemon/__codecheck.py
diff --git a/xlators/features/marker/utils/syncdaemon/__init__.py b/geo-replication/syncdaemon/__init__.py
index e69de29bb..e69de29bb 100644
--- a/xlators/features/marker/utils/syncdaemon/__init__.py
+++ b/geo-replication/syncdaemon/__init__.py
diff --git a/xlators/features/marker/utils/syncdaemon/configinterface.py b/geo-replication/syncdaemon/configinterface.py
index e55bec519..a326e8246 100644
--- a/xlators/features/marker/utils/syncdaemon/configinterface.py
+++ b/geo-replication/syncdaemon/configinterface.py
@@ -24,9 +24,9 @@ class MultiDict(object):
     def __getitem__(self, key):
         val = None
         for d in self.dicts:
-            if d.get(key):
+            if d.get(key) != None:
                 val = d[key]
-        if not val:
+        if val == None:
             raise KeyError(key)
         return val
 
diff --git a/xlators/features/marker/utils/syncdaemon/gconf.py b/geo-replication/syncdaemon/gconf.py
index 146c72a18..fe5795f16 100644
--- a/xlators/features/marker/utils/syncdaemon/gconf.py
+++ b/geo-replication/syncdaemon/gconf.py
@@ -12,9 +12,4 @@ class GConf(object):
     permanent_handles = []
     log_metadata = {}
 
-    @classmethod
-    def setup_ssh_ctl(cls, ctld):
-        cls.ssh_ctl_dir = ctld
-        cls.ssh_ctl_args = ["-oControlMaster=auto", "-S", os.path.join(ctld, "gsycnd-ssh-%r@%h:%p")]
-
 gconf = GConf()
diff --git a/xlators/features/marker/utils/syncdaemon/gsyncd.py b/geo-replication/syncdaemon/gsyncd.py
index f145cbc6d..7fcc3165a 100644
--- a/xlators/features/marker/utils/syncdaemon/gsyncd.py
+++ b/geo-replication/syncdaemon/gsyncd.py
@@ -2,22 +2,24 @@
 
 import os
 import os.path
+import glob
 import sys
 import time
 import logging
 import signal
+import shutil
 import optparse
 import fcntl
 import fnmatch
 from optparse import OptionParser, SUPPRESS_HELP
-from logging import Logger
+from logging import Logger, handlers
 from errno import EEXIST, ENOENT
 
 from ipaddr import IPAddress, IPNetwork
 
 from gconf import gconf
 from syncdutils import FreeObject, norm, grabpidfile, finalize, log_raise_exception
-from syncdutils import GsyncdError, select
+from syncdutils import GsyncdError, select, set_term_handler, privileged, update_file
 from configinterface import GConffile
 import resource
 from monitor import monitor
@@ -56,7 +58,43 @@ class GLogger(Logger):
         logging.root = cls("root", lvl)
         logging.setLoggerClass(cls)
         logging.getLogger().handlers = []
-        logging.basicConfig(**lprm)
+        logging.getLogger().setLevel(lprm['level'])
+
+        if 'filename' in lprm:
+            try:
+                logging_handler = handlers.WatchedFileHandler(lprm['filename'])
+                formatter = logging.Formatter(fmt=lprm['format'],
+                datefmt=lprm['datefmt'])
+                logging_handler.setFormatter(formatter)
+                logging.getLogger().addHandler(logging_handler)
+            except AttributeError:
+                # Python version < 2.6 will not have WatchedFileHandler
+                # so fallback to logging without any handler.
+                # Note: logrotate will not work if Python version is < 2.6
+                logging.basicConfig(**lprm)
+        else:
+            # If filename not passed(not available in lprm) then it may be
+            # streaming.(Ex: {"stream": "/dev/stdout"})
+            logging.basicConfig(**lprm)
+
+    @classmethod
+    def _gsyncd_loginit(cls, **kw):
+        lkw = {}
+        if gconf.log_level:
+            lkw['level'] = gconf.log_level
+        if kw.get('log_file'):
+            if kw['log_file'] in ('-', '/dev/stderr'):
+                lkw['stream'] = sys.stderr
+            elif kw['log_file'] == '/dev/stdout':
+                lkw['stream'] = sys.stdout
+            else:
+                lkw['filename'] = kw['log_file']
+
+        cls.setup(label=kw.get('label'), **lkw)
+
+        lkw.update({'saved_label': kw.get('label')})
+        gconf.log_metadata = lkw
+        gconf.log_exit = True
 
 def startup(**kw):
     """set up logging, pidfile grabbing, daemonization"""
@@ -88,26 +126,23 @@ def startup(**kw):
         select((x,), (), ())
         os.close(x)
 
-    lkw = {}
-    if gconf.log_level:
-        lkw['level'] = gconf.log_level
-    if kw.get('log_file'):
-        if kw['log_file'] in ('-', '/dev/stderr'):
-            lkw['stream'] = sys.stderr
-        elif kw['log_file'] == '/dev/stdout':
-            lkw['stream'] = sys.stdout
-        else:
-            lkw['filename'] = kw['log_file']
+    GLogger._gsyncd_loginit(**kw)
+
 
-    GLogger.setup(label=kw.get('label'), **lkw)
+def _unlink(path):
+    try:
+        os.unlink(path)
+    except (OSError, IOError):
+        if sys.exc_info()[1].errno == ENOENT:
+            pass
+        else:
+            raise GsyncdError('Unlink error: %s' % path)
 
-    lkw.update({'saved_label': kw.get('label')})
-    gconf.log_metadata = lkw
-    gconf.log_exit = True
 
 def main():
     """main routine, signal/exception handling boilerplates"""
-    signal.signal(signal.SIGTERM, lambda *a: finalize(*a, **{'exval': 1}))
+    gconf.starttime = time.time()
+    set_term_handler()
     GLogger.setup()
     excont = FreeObject(exval = 0)
     try:
@@ -148,33 +183,67 @@ def main_i():
     op.add_option('--gluster-log-file',    metavar='LOGF',  default=os.devnull, type=str, action='callback', callback=store_abs)
     op.add_option('--gluster-log-level',   metavar='LVL')
     op.add_option('--gluster-params',      metavar='PRMS',  default='')
+    op.add_option('--glusterd-uuid',       metavar='UUID',  type=str, default='', help=SUPPRESS_HELP)
     op.add_option('--gluster-cli-options', metavar='OPTS',  default='--log-file=-')
     op.add_option('--mountbroker',         metavar='LABEL')
     op.add_option('-p', '--pid-file',      metavar='PIDF',  type=str, action='callback', callback=store_abs)
     op.add_option('-l', '--log-file',      metavar='LOGF',  type=str, action='callback', callback=store_abs)
+    op.add_option('--log-file-mbr',        metavar='LOGF',  type=str, action='callback', callback=store_abs)
     op.add_option('--state-file',          metavar='STATF', type=str, action='callback', callback=store_abs)
+    op.add_option('--state-detail-file',   metavar='STATF', type=str, action='callback', callback=store_abs)
     op.add_option('--ignore-deletes',      default=False, action='store_true')
+    op.add_option('--isolated-slave',      default=False, action='store_true')
+    op.add_option('--use-rsync-xattrs',    default=False, action='store_true')
     op.add_option('-L', '--log-level',     metavar='LVL')
     op.add_option('-r', '--remote-gsyncd', metavar='CMD',   default=os.path.abspath(sys.argv[0]))
     op.add_option('--volume-id',           metavar='UUID')
+    op.add_option('--slave-id',            metavar='ID')
     op.add_option('--session-owner',       metavar='ID')
+    op.add_option('--local-id',            metavar='ID',    help=SUPPRESS_HELP, default='')
+    op.add_option('--local-path',          metavar='PATH',  help=SUPPRESS_HELP, default='')
     op.add_option('-s', '--ssh-command',   metavar='CMD',   default='ssh')
     op.add_option('--rsync-command',       metavar='CMD',   default='rsync')
-    op.add_option('--rsync-extra',         metavar='ARGS',  default='-S', help=SUPPRESS_HELP)
+    op.add_option('--rsync-options',       metavar='OPTS',  default='')
+    op.add_option('--rsync-ssh-options',   metavar='OPTS',  default='--compress')
     op.add_option('--timeout',             metavar='SEC',   type=int, default=120)
+    op.add_option('--connection-timeout',  metavar='SEC',   type=int, default=60, help=SUPPRESS_HELP)
     op.add_option('--sync-jobs',           metavar='N',     type=int, default=3)
     op.add_option('--turns',               metavar='N',     type=int, default=0, help=SUPPRESS_HELP)
     op.add_option('--allow-network',       metavar='IPS',   default='')
+    op.add_option('--socketdir',           metavar='DIR')
+    op.add_option('--state-socket-unencoded', metavar='SOCKF', type=str, action='callback', callback=store_abs)
+    op.add_option('--checkpoint',          metavar='LABEL', default='')
+    # tunables for failover/failback mechanism:
+    # None   - gsyncd behaves as normal
+    # blind  - gsyncd works with xtime pairs to identify
+    #          candidates for synchronization
+    # wrapup - same as normal mode but does not assign
+    #          xtimes to orphaned files
+    # see crawl() for usage of the above tunables
+    op.add_option('--special-sync-mode', type=str, help=SUPPRESS_HELP)
+
+    # changelog or xtime? (TODO: Change the default)
+    op.add_option('--change-detector', metavar='MODE', type=str, default='xtime')
+    # sleep interval for change detection (xtime crawl uses a hardcoded 1 second sleep time)
+    op.add_option('--change-interval', metavar='SEC', type=int, default=3)
+    # working directory for changelog based mechanism
+    op.add_option('--working-dir', metavar='DIR', type=str, action='callback', callback=store_abs)
 
     op.add_option('-c', '--config-file',   metavar='CONF',  type=str, action='callback', callback=store_local)
     # duh. need to specify dest or value will be mapped to None :S
     op.add_option('--monitor', dest='monitor', action='callback', callback=store_local_curry(True))
+    op.add_option('--resource-local', dest='resource_local', type=str, action='callback', callback=store_local)
+    op.add_option('--resource-remote', dest='resource_remote', type=str, action='callback', callback=store_local)
     op.add_option('--feedback-fd', dest='feedback_fd', type=int, help=SUPPRESS_HELP, action='callback', callback=store_local)
     op.add_option('--listen', dest='listen', help=SUPPRESS_HELP,      action='callback', callback=store_local_curry(True))
     op.add_option('-N', '--no-daemon', dest="go_daemon",    action='callback', callback=store_local_curry('dont'))
+    op.add_option('--verify', type=str, dest="verify", action='callback', callback=store_local)
+    op.add_option('--create', type=str, dest="create", action='callback', callback=store_local)
+    op.add_option('--delete', dest='delete', action='callback', callback=store_local_curry(True))
     op.add_option('--debug', dest="go_daemon",              action='callback', callback=lambda *a: (store_local_curry('dont')(*a),
                                                                                                     setattr(a[-1].values, 'log_file', '-'),
                                                                                                     setattr(a[-1].values, 'log_level', 'DEBUG'))),
+    op.add_option('--path', type=str, action='append')
 
     for a in ('check', 'get'):
         op.add_option('--config-' + a,      metavar='OPT',  type=str, dest='config', action='callback',
@@ -196,7 +265,7 @@ def main_i():
     op.add_option('--canonicalize-escape-url', dest='url_print', action='callback', callback=store_local_curry('canon_esc'))
 
     tunables = [ norm(o.get_opt_string()[2:]) for o in op.option_list if o.callback in (store_abs, 'store_true', None) and o.get_opt_string() not in ('--version', '--help') ]
-    remote_tunables = [ 'listen', 'go_daemon', 'timeout', 'session_owner', 'config_file' ]
+    remote_tunables = [ 'listen', 'go_daemon', 'timeout', 'session_owner', 'config_file', 'use_rsync_xattrs' ]
     rq_remote_tunables = { 'listen': True }
 
     # precedence for sources of values: 1) commandline, 2) cfg file, 3) defaults
@@ -205,6 +274,19 @@ def main_i():
     # values container.
     defaults = op.get_default_values()
     opts, args = op.parse_args(values=optparse.Values())
+    args_orig = args[:]
+    r = rconf.get('resource_local')
+    if r:
+        if len(args) == 0:
+            args.append(None)
+        args[0] = r
+    r = rconf.get('resource_remote')
+    if r:
+        if len(args) == 0:
+            raise GsyncdError('local resource unspecfied')
+        elif len(args) == 1:
+            args.append(None)
+        args[1] = r
     confdata = rconf.get('config')
     if not (len(args) == 2 or \
             (len(args) == 1 and rconf.get('listen')) or \
@@ -214,6 +296,12 @@ def main_i():
         sys.stderr.write(op.get_usage() + "\n")
         sys.exit(1)
 
+    verify = rconf.get('verify')
+    if verify:
+        logging.info (verify)
+        logging.info ("Able to spawn gsyncd.py")
+        return
+
     restricted = os.getenv('_GSYNCD_RESTRICTED_')
 
     if restricted:
@@ -230,6 +318,17 @@ def main_i():
                                   (k, v))
 
     confrx = getattr(confdata, 'rx', None)
+    def makersc(aa, check=True):
+        if not aa:
+            return ([], None, None)
+        ra = [resource.parse_url(u) for u in aa]
+        local = ra[0]
+        remote = None
+        if len(ra) > 1:
+            remote = ra[1]
+        if check and not local.can_connect_to(remote):
+            raise GsyncdError("%s cannot work with %s" % (local.path, remote and remote.path))
+        return (ra, local, remote)
     if confrx:
         # peers are regexen, don't try to parse them
         if confrx == 'glob':
@@ -237,27 +336,20 @@ def main_i():
         canon_peers = args
         namedict = {}
     else:
-        rscs = [resource.parse_url(u) for u in args]
         dc = rconf.get('url_print')
+        rscs, local, remote = makersc(args_orig, not dc)
         if dc:
             for r in rscs:
                 print(r.get_url(**{'normal': {},
                                    'canon': {'canonical': True},
                                    'canon_esc': {'canonical': True, 'escaped': True}}[dc]))
             return
-        local = remote = None
-        if rscs:
-            local = rscs[0]
-            if len(rscs) > 1:
-                remote = rscs[1]
-            if not local.can_connect_to(remote):
-                raise GsyncdError("%s cannot work with %s" % (local.path, remote and remote.path))
         pa = ([], [], [])
         urlprms = ({}, {'canonical': True}, {'canonical': True, 'escaped': True})
         for x in rscs:
             for i in range(len(pa)):
                 pa[i].append(x.get_url(**urlprms[i]))
-        peers, canon_peers, canon_esc_peers = pa
+        _, canon_peers, canon_esc_peers = pa
         # creating the namedict, a dict representing various ways of referring to / repreenting
         # peers to be fillable in config templates
         mods = (lambda x: x, lambda x: x[0].upper() + x[1:], lambda x: 'e' + x[0].upper() + x[1:])
@@ -271,12 +363,14 @@ def main_i():
             for name in rmap[x]:
                 for j in range(3):
                     namedict[mods[j](name)] = pa[j][i]
-                if x.scheme == 'gluster':
-                    namedict[name + 'vol'] = x.volume
+                namedict[name + 'vol'] = x.volume
+                if name == 'remote':
+                    namedict['remotehost'] = x.remotehost
     if not 'config_file' in rconf:
-        rconf['config_file'] = os.path.join(os.path.dirname(sys.argv[0]), "conf/gsyncd.conf")
+        rconf['config_file'] = os.path.join(os.path.dirname(sys.argv[0]), "conf/gsyncd_template.conf")
     gcnf = GConffile(rconf['config_file'], canon_peers, defaults.__dict__, opts.__dict__, namedict)
 
+    checkpoint_change = False
     if confdata:
         opt_ok = norm(confdata.opt) in tunables + [None]
         if confdata.op == 'check':
@@ -292,13 +386,55 @@ def main_i():
             gcnf.set(confdata.opt, confdata.val, confdata.rx)
         elif confdata.op == 'del':
             gcnf.delete(confdata.opt, confdata.rx)
-        return
+        # when modifying checkpoint, it's important to make a log
+        # of that, so in that case we go on to set up logging even
+        # if its just config invocation
+        if confdata.opt == 'checkpoint' and confdata.op in ('set', 'del') and \
+           not confdata.rx:
+            checkpoint_change = True
+        if not checkpoint_change:
+            return
 
     gconf.__dict__.update(defaults.__dict__)
     gcnf.update_to(gconf.__dict__)
     gconf.__dict__.update(opts.__dict__)
     gconf.configinterface = gcnf
 
+    delete = rconf.get('delete')
+    if delete:
+        logging.info ('geo-replication delete')
+        # Delete pid file, status file, socket file
+        cleanup_paths = []
+        if getattr(gconf, 'pid_file', None):
+            cleanup_paths.append(gconf.pid_file)
+
+        if getattr(gconf, 'state_file', None):
+            cleanup_paths.append(gconf.state_file)
+
+        if getattr(gconf, 'state_detail_file', None):
+            cleanup_paths.append(gconf.state_detail_file)
+
+        if getattr(gconf, 'state_socket_unencoded', None):
+            cleanup_paths.append(gconf.state_socket_unencoded)
+
+        cleanup_paths.append(rconf['config_file'][:-11] + "*");
+
+        # Cleanup changelog working dirs
+        if getattr(gconf, 'working_dir', None):
+            try:
+                shutil.rmtree(gconf.working_dir)
+            except (IOError, OSError):
+                if sys.exc_info()[1].errno == ENOENT:
+                    pass
+                else:
+                    raise GsyncdError('Error while removing working dir: %s' % gconf.working_dir)
+
+        for path in cleanup_paths:
+            # To delete temp files
+            for f in glob.glob(path + "*"):
+                _unlink(f)
+        return
+
     if restricted and gconf.allow_network:
         ssh_conn = os.getenv('SSH_CONNECTION')
         if not ssh_conn:
@@ -330,9 +466,38 @@ def main_i():
             raise GsyncdError('cannot recognize log level "%s"' % lvl0)
         gconf.log_level = lvl2
 
+    if not privileged() and gconf.log_file_mbr:
+        gconf.log_file = gconf.log_file_mbr
+
+    if checkpoint_change:
+        try:
+            GLogger._gsyncd_loginit(log_file=gconf.log_file, label='conf')
+            if confdata.op == 'set':
+                logging.info('checkpoint %s set' % confdata.val)
+            elif confdata.op == 'del':
+                logging.info('checkpoint info was reset')
+        except IOError:
+            if sys.exc_info()[1].errno == ENOENT:
+                # directory of log path is not present,
+                # which happens if we get here from
+                # a peer-multiplexed "config-set checkpoint"
+                # (as that directory is created only on the
+                # original node)
+                pass
+            else:
+                raise
+        return
+
+    create = rconf.get('create')
+    if create:
+        if getattr(gconf, 'state_file', None):
+            update_file(gconf.state_file, lambda f: f.write(create + '\n'))
+        return
+
     go_daemon = rconf['go_daemon']
     be_monitor = rconf.get('monitor')
 
+    rscs, local, remote = makersc(args)
     if not be_monitor and isinstance(remote, resource.SSH) and \
        go_daemon == 'should':
         go_daemon = 'postconn'
@@ -343,16 +508,16 @@ def main_i():
         label = 'monitor'
     elif remote:
         #master
-        label = ''
+        label = gconf.local_path
     else:
         label = 'slave'
     startup(go_daemon=go_daemon, log_file=log_file, label=label)
+    resource.Popen.init_errhandler()
 
     if be_monitor:
-        return monitor()
+        return monitor(*rscs)
 
-    logging.info("syncing: %s" % " -> ".join(peers))
-    resource.Popen.init_errhandler()
+    logging.info("syncing: %s" % " -> ".join(r.url for r in rscs))
     if remote:
         go_daemon = remote.connect_remote(go_daemon=go_daemon)
         if go_daemon:
diff --git a/xlators/features/marker/utils/syncdaemon/libcxattr.py b/geo-replication/syncdaemon/libcxattr.py
index f0a9d2292..b5b6956ae 100644
--- a/xlators/features/marker/utils/syncdaemon/libcxattr.py
+++ b/geo-replication/syncdaemon/libcxattr.py
@@ -43,6 +43,16 @@ class Xattr(object):
         return cls._query_xattr( path, siz, 'lgetxattr', attr)
 
     @classmethod
+    def lgetxattr_buf(cls, path, attr):
+        """lgetxattr variant with size discovery"""
+        size = cls.lgetxattr(path, attr)
+        if size == -1:
+            cls.raise_oserr()
+        if size == 0:
+            return ''
+        return cls.lgetxattr(path, attr, size)
+
+    @classmethod
     def llistxattr(cls, path, siz=0):
         ret = cls._query_xattr(path, siz, 'llistxattr')
         if isinstance(ret, str):
@@ -56,6 +66,11 @@ class Xattr(object):
             cls.raise_oserr()
 
     @classmethod
+    def lsetxattr_l(cls, path, attr, val):
+        """ lazy lsetxattr(): caller handles errno """
+        cls.libc.lsetxattr(path, attr, val, len(val), 0)
+
+    @classmethod
     def lremovexattr(cls, path, attr):
         ret = cls.libc.lremovexattr(path, attr)
         if ret == -1:
diff --git a/geo-replication/syncdaemon/libgfchangelog.py b/geo-replication/syncdaemon/libgfchangelog.py
new file mode 100644
index 000000000..68ec3baf1
--- /dev/null
+++ b/geo-replication/syncdaemon/libgfchangelog.py
@@ -0,0 +1,64 @@
+import os
+from ctypes import *
+from ctypes.util import find_library
+
+class Changes(object):
+    libgfc = CDLL(find_library("gfchangelog"), use_errno=True)
+
+    @classmethod
+    def geterrno(cls):
+        return get_errno()
+
+    @classmethod
+    def raise_oserr(cls):
+        errn = cls.geterrno()
+        raise OSError(errn, os.strerror(errn))
+
+    @classmethod
+    def _get_api(cls, call):
+        return getattr(cls.libgfc, call)
+
+    @classmethod
+    def cl_register(cls, brick, path, log_file, log_level, retries = 0):
+        ret = cls._get_api('gf_changelog_register')(brick, path,
+                                                    log_file, log_level, retries)
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_scan(cls):
+        ret = cls._get_api('gf_changelog_scan')()
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_startfresh(cls):
+        ret = cls._get_api('gf_changelog_start_fresh')()
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_getchanges(cls):
+        """ remove hardcoding for path name length """
+        def clsort(f):
+            return f.split('.')[-1]
+        changes = []
+        buf = create_string_buffer('\0', 4096)
+        call = cls._get_api('gf_changelog_next_change')
+
+        while True:
+            ret = call(buf, 4096)
+            if ret in (0, -1):
+                break;
+            changes.append(buf.raw[:ret-1])
+        if ret == -1:
+            cls.raise_oserr()
+        # cleanup tracker
+        cls.cl_startfresh()
+        return sorted(changes, key=clsort)
+
+    @classmethod
+    def cl_done(cls, clfile):
+        ret = cls._get_api('gf_changelog_done')(clfile)
+        if ret == -1:
+            cls.raise_oserr()
diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py
new file mode 100644
index 000000000..95810a61e
--- /dev/null
+++ b/geo-replication/syncdaemon/master.py
@@ -0,0 +1,1022 @@
+import os
+import sys
+import time
+import stat
+import random
+import signal
+import json
+import logging
+import socket
+import string
+import errno
+from shutil import copyfileobj
+from errno import ENOENT, ENODATA, EPIPE, EEXIST
+from threading import currentThread, Condition, Lock
+from datetime import datetime
+
+from gconf import gconf
+from tempfile import mkdtemp, NamedTemporaryFile
+from syncdutils import FreeObject, Thread, GsyncdError, boolify, escape, \
+                       unescape, select, gauxpfx, md5hex, selfkill, entry2pb, \
+                       lstat, errno_wrap
+
+URXTIME = (-1, 0)
+
+# Utility functions to help us to get to closer proximity
+# of the DRY principle (no, don't look for elevated or
+# perspectivistic things here)
+
+def _xtime_now():
+    t = time.time()
+    sec = int(t)
+    nsec = int((t - sec) * 1000000)
+    return (sec, nsec)
+
+def _volinfo_hook_relax_foreign(self):
+    volinfo_sys = self.get_sys_volinfo()
+    fgn_vi = volinfo_sys[self.KFGN]
+    if fgn_vi:
+        expiry = fgn_vi['timeout'] - int(time.time()) + 1
+        logging.info('foreign volume info found, waiting %d sec for expiry' % \
+                     expiry)
+        time.sleep(expiry)
+        volinfo_sys = self.get_sys_volinfo()
+    return volinfo_sys
+
+
+# The API!
+
+def gmaster_builder(excrawl=None):
+    """produce the GMaster class variant corresponding
+       to sync mode"""
+    this = sys.modules[__name__]
+    modemixin = gconf.special_sync_mode
+    if not modemixin:
+        modemixin = 'normal'
+    changemixin = isinstance(excrawl, str) and excrawl or gconf.change_detector
+    logging.info('setting up %s change detection mode' % changemixin)
+    modemixin = getattr(this, modemixin.capitalize() + 'Mixin')
+    crawlmixin = getattr(this, 'GMaster' + changemixin.capitalize() + 'Mixin')
+    sendmarkmixin = boolify(gconf.use_rsync_xattrs) and SendmarkRsyncMixin or SendmarkNormalMixin
+    purgemixin = boolify(gconf.ignore_deletes) and PurgeNoopMixin or PurgeNormalMixin
+    class _GMaster(crawlmixin, modemixin, sendmarkmixin, purgemixin):
+        pass
+    return _GMaster
+
+
+# Mixin classes that implement the data format
+# and logic particularities of the certain
+# sync modes
+
+class NormalMixin(object):
+    """normal geo-rep behavior"""
+
+    minus_infinity = URXTIME
+
+    # following staticmethods ideally would be
+    # methods of an xtime object (in particular,
+    # implementing the hooks needed for comparison
+    # operators), but at this point we don't yet
+    # have a dedicated xtime class
+
+    @staticmethod
+    def serialize_xtime(xt):
+        return "%d.%d" % tuple(xt)
+
+    @staticmethod
+    def deserialize_xtime(xt):
+        return tuple(int(x) for x in xt.split("."))
+
+    @staticmethod
+    def native_xtime(xt):
+        return xt
+
+    @staticmethod
+    def xtime_geq(xt0, xt1):
+        return xt0 >= xt1
+
+    def make_xtime_opts(self, is_master, opts):
+        if not 'create' in opts:
+            opts['create'] = is_master
+        if not 'default_xtime' in opts:
+            opts['default_xtime'] = URXTIME
+
+    def xtime_low(self, server, path, **opts):
+        xt = server.xtime(path, self.uuid)
+        if isinstance(xt, int) and xt != ENODATA:
+            return xt
+        if xt == ENODATA or xt < self.volmark:
+            if opts['create']:
+                xt = _xtime_now()
+                server.aggregated.set_xtime(path, self.uuid, xt)
+            else:
+                xt = opts['default_xtime']
+        return xt
+
+    def keepalive_payload_hook(self, timo, gap):
+        # first grab a reference as self.volinfo
+        # can be changed in main thread
+        vi = self.volinfo
+        if vi:
+            # then have a private copy which we can mod
+            vi = vi.copy()
+            vi['timeout'] = int(time.time()) + timo
+        else:
+            # send keep-alives more frequently to
+            # avoid a delay in announcing our volume info
+            # to slave if it becomes established in the
+            # meantime
+            gap = min(10, gap)
+        return (vi, gap)
+
+    def volinfo_hook(self):
+        return self.get_sys_volinfo()
+
+    def xtime_reversion_hook(self, path, xtl, xtr):
+        if xtr > xtl:
+            raise GsyncdError("timestamp corruption for " + path)
+
+    def need_sync(self, e, xte, xtrd):
+        return xte > xtrd
+
+    def set_slave_xtime(self, path, mark):
+        self.slave.server.set_xtime(path, self.uuid, mark)
+        self.slave.server.set_xtime_remote(path, self.uuid, mark)
+
+class PartialMixin(NormalMixin):
+    """a variant tuned towards operation with a master
+       that has partial info of the slave (brick typically)"""
+
+    def xtime_reversion_hook(self, path, xtl, xtr):
+        pass
+
+class RecoverMixin(NormalMixin):
+    """a variant that differs from normal in terms
+       of ignoring non-indexed files"""
+
+    @staticmethod
+    def make_xtime_opts(is_master, opts):
+        if not 'create' in opts:
+            opts['create'] = False
+        if not 'default_xtime' in opts:
+            opts['default_xtime'] = URXTIME
+
+    def keepalive_payload_hook(self, timo, gap):
+        return (None, gap)
+
+    def volinfo_hook(self):
+        return _volinfo_hook_relax_foreign(self)
+
+# Further mixins for certain tunable behaviors
+
+class SendmarkNormalMixin(object):
+
+    def sendmark_regular(self, *a, **kw):
+        return self.sendmark(*a, **kw)
+
+class SendmarkRsyncMixin(object):
+
+    def sendmark_regular(self, *a, **kw):
+        pass
+
+
+class PurgeNormalMixin(object):
+
+    def purge_missing(self, path, names):
+        self.slave.server.purge(path, names)
+
+class PurgeNoopMixin(object):
+
+    def purge_missing(self, path, names):
+        pass
+
+class GMasterCommon(object):
+    """abstract class impementling master role"""
+
+    KFGN = 0
+    KNAT = 1
+
+    def get_sys_volinfo(self):
+        """query volume marks on fs root
+
+        err out on multiple foreign masters
+        """
+        fgn_vis, nat_vi = self.master.server.aggregated.foreign_volume_infos(), \
+                          self.master.server.aggregated.native_volume_info()
+        fgn_vi = None
+        if fgn_vis:
+            if len(fgn_vis) > 1:
+                raise GsyncdError("cannot work with multiple foreign masters")
+            fgn_vi = fgn_vis[0]
+        return fgn_vi, nat_vi
+
+    @property
+    def uuid(self):
+        if self.volinfo:
+            return self.volinfo['uuid']
+
+    @property
+    def volmark(self):
+        if self.volinfo:
+            return self.volinfo['volume_mark']
+
+    def xtime(self, path, *a, **opts):
+        """get amended xtime
+
+        as of amending, we can create missing xtime, or
+        determine a valid value if what we get is expired
+        (as of the volume mark expiry); way of amendig
+        depends on @opts and on subject of query (master
+        or slave).
+        """
+        if a:
+            rsc = a[0]
+        else:
+            rsc = self.master
+        self.make_xtime_opts(rsc == self.master, opts)
+        return self.xtime_low(rsc.server, path, **opts)
+
+    def get_initial_crawl_data(self):
+        # while persisting only 'files_syncd' is non-zero, rest of
+        # the stats are nulls. lets keep it that way in case they
+        # are needed to be used some day...
+        default_data = {'files_syncd': 0,
+                        'files_remaining': 0,
+                        'bytes_remaining': 0,
+                        'purges_remaining': 0}
+        if getattr(gconf, 'state_detail_file', None):
+            try:
+                return json.load(open(gconf.state_detail_file))
+            except (IOError, OSError):
+                ex = sys.exc_info()[1]
+                if ex.errno == ENOENT:
+                    # Create file with initial data
+                    with open(gconf.state_detail_file, 'wb') as f:
+                        json.dump(default_data, f)
+                    return default_data
+                else:
+                    raise
+        return default_data
+
+    def update_crawl_data(self):
+        if getattr(gconf, 'state_detail_file', None):
+            try:
+                same_dir = os.path.dirname(gconf.state_detail_file)
+                with NamedTemporaryFile(dir=same_dir, delete=False) as tmp:
+                    json.dump(self.total_crawl_stats, tmp)
+                    os.rename(tmp.name, gconf.state_detail_file)
+            except (IOError, OSError):
+                raise
+
+    def __init__(self, master, slave):
+        self.master = master
+        self.slave = slave
+        self.jobtab = {}
+        self.syncer = Syncer(slave)
+        # crawls vs. turns:
+        # - self.crawls is simply the number of crawl() invocations on root
+        # - one turn is a maximal consecutive sequence of crawls so that each
+        #   crawl in it detects a change to be synced
+        # - self.turns is the number of turns since start
+        # - self.total_turns is a limit so that if self.turns reaches it, then
+        #   we exit (for diagnostic purposes)
+        # so, eg., if the master fs changes unceasingly, self.turns will remain 0.
+        self.crawls = 0
+        self.turns = 0
+        self.total_turns = int(gconf.turns)
+        self.crawl_start = datetime.now()
+        self.lastreport = {'crawls': 0, 'turns': 0, 'time': 0}
+        self.total_crawl_stats = None
+        self.start = None
+        self.change_seen = None
+        # the actual volinfo we make use of
+        self.volinfo = None
+        self.terminate = False
+        self.sleep_interval = 1
+        self.checkpoint_thread = None
+
+    def init_keep_alive(cls):
+        """start the keep-alive thread """
+        timo = int(gconf.timeout or 0)
+        if timo > 0:
+            def keep_alive():
+                while True:
+                    vi, gap = cls.keepalive_payload_hook(timo, timo * 0.5)
+                    cls.slave.server.keep_alive(vi)
+                    time.sleep(gap)
+            t = Thread(target=keep_alive)
+            t.start()
+
+    def should_crawl(cls):
+        return (gconf.glusterd_uuid in cls.master.server.node_uuid())
+
+    def register(self):
+        self.register()
+
+    def crawlwrap(self, oneshot=False):
+        if oneshot:
+            # it's important to do this during the oneshot crawl as
+            # for a passive gsyncd (ie. in a replicate scenario)
+            # the keepalive thread would keep the connection alive.
+            self.init_keep_alive()
+
+        # no need to maintain volinfo state machine.
+        # in a cascading setup, each geo-replication session is
+        # independent (ie. 'volume-mark' and 'xtime' are not
+        # propogated). This is beacuse the slave's xtime is now
+        # stored on the master itself. 'volume-mark' just identifies
+        # that we are in a cascading setup and need to enable
+        # 'geo-replication.ignore-pid-check' option.
+        volinfo_sys = self.volinfo_hook()
+        self.volinfo = volinfo_sys[self.KNAT]
+        inter_master = volinfo_sys[self.KFGN]
+        logging.info("%s master with volume id %s ..." % \
+                         (inter_master and "intermediate" or "primary",
+                          self.uuid))
+        gconf.configinterface.set('volume_id', self.uuid)
+        if self.volinfo:
+            if self.volinfo['retval']:
+                raise GsyncdError("master is corrupt")
+            self.start_checkpoint_thread()
+        else:
+            raise GsyncdError("master volinfo unavailable")
+	self.total_crawl_stats = self.get_initial_crawl_data()
+        self.lastreport['time'] = time.time()
+        logging.info('crawl interval: %d seconds' % self.sleep_interval)
+
+        t0 = time.time()
+        crawl = self.should_crawl()
+        while not self.terminate:
+            if self.start:
+                logging.debug("... crawl #%d done, took %.6f seconds" % \
+                                  (self.crawls, time.time() - self.start))
+            self.start = time.time()
+            should_display_info = self.start - self.lastreport['time'] >= 60
+            if should_display_info:
+                logging.info("%d crawls, %d turns",
+                             self.crawls - self.lastreport['crawls'],
+                             self.turns - self.lastreport['turns'])
+                self.lastreport.update(crawls = self.crawls,
+                                       turns = self.turns,
+                                       time = self.start)
+            t1 = time.time()
+            if int(t1 - t0) >= 60: #lets hardcode this check to 60 seconds
+                crawl = self.should_crawl()
+                t0 = t1
+            if not crawl:
+                time.sleep(5)
+                continue
+            self.crawl()
+            if oneshot:
+                return
+            time.sleep(self.sleep_interval)
+
+    @classmethod
+    def _checkpt_param(cls, chkpt, prm, xtimish=True):
+        """use config backend to lookup a parameter belonging to
+           checkpoint @chkpt"""
+        cprm = getattr(gconf, 'checkpoint_' + prm, None)
+        if not cprm:
+            return
+        chkpt_mapped, val = cprm.split(':', 1)
+        if unescape(chkpt_mapped) != chkpt:
+            return
+        if xtimish:
+            val = cls.deserialize_xtime(val)
+        return val
+
+    @classmethod
+    def _set_checkpt_param(cls, chkpt, prm, val, xtimish=True):
+        """use config backend to store a parameter associated
+           with checkpoint @chkpt"""
+        if xtimish:
+            val = cls.serialize_xtime(val)
+        gconf.configinterface.set('checkpoint_' + prm, "%s:%s" % (escape(chkpt), val))
+
+    @staticmethod
+    def humantime(*tpair):
+        """format xtime-like (sec, nsec) pair to human readable format"""
+        ts = datetime.fromtimestamp(float('.'.join(str(n) for n in tpair))).\
+               strftime("%Y-%m-%d %H:%M:%S")
+        if len(tpair) > 1:
+            ts += '.' + str(tpair[1])
+        return ts
+
+    def get_extra_info(self):
+        str_info = '\nUptime=%s;FilesSyncd=%d;FilesPending=%d;BytesPending=%d;DeletesPending=%d;' % \
+            (self._crawl_time_format(datetime.now() - self.crawl_start), \
+                 self.total_crawl_stats['files_syncd'], \
+                 self.total_crawl_stats['files_remaining'], \
+                 self.total_crawl_stats['bytes_remaining'], \
+                 self.total_crawl_stats['purges_remaining'])
+        str_info += '\0'
+        logging.debug(str_info)
+        return str_info
+
+    def _crawl_time_format(self, crawl_time):
+        # Ex: 5 years, 4 days, 20:23:10
+        years, days = divmod(crawl_time.days, 365.25)
+        years = int(years)
+        days = int(days)
+
+        date=""
+        m, s = divmod(crawl_time.seconds, 60)
+        h, m = divmod(m, 60)
+
+        if years != 0:
+            date += "%s %s " % (years, "year" if years == 1 else "years")
+        if days != 0:
+            date += "%s %s " % (days, "day" if days == 1 else "days")
+
+        date += "%s:%s:%s" % (string.zfill(h, 2), string.zfill(m, 2), string.zfill(s, 2))
+        return date
+
+    def checkpt_service(self, chan, chkpt, tgt):
+        """checkpoint service loop
+
+        monitor and verify checkpoint status for @chkpt, and listen
+        for incoming requests for whom we serve a pretty-formatted
+        status report"""
+        if not chkpt:
+            # dummy loop for the case when there is no checkpt set
+            while True:
+                select([chan], [], [])
+                conn, _ = chan.accept()
+                conn.send(self.get_extra_info())
+                conn.close()
+        completed = self._checkpt_param(chkpt, 'completed', xtimish=False)
+        if completed:
+            completed = tuple(int(x) for x in completed.split('.'))
+        while True:
+            s,_,_ = select([chan], [], [], (not completed) and 5 or None)
+            # either request made and we re-check to not
+            # give back stale data, or we still hunting for completion
+            if self.native_xtime(tgt) and self.native_xtime(tgt) < self.volmark:
+                # indexing has been reset since setting the checkpoint
+                status = "is invalid"
+            else:
+                xtr = self.xtime('.', self.slave)
+                if isinstance(xtr, int):
+                    raise GsyncdError("slave root directory is unaccessible (%s)",
+                                      os.strerror(xtr))
+                ncompleted = self.xtime_geq(xtr, tgt)
+                if completed and not ncompleted: # stale data
+                    logging.warn("completion time %s for checkpoint %s became stale" % \
+                                 (self.humantime(*completed), chkpt))
+                    completed = None
+                    gconf.confdata.delete('checkpoint-completed')
+                if ncompleted and not completed: # just reaching completion
+                    completed = "%.6f" % time.time()
+                    self._set_checkpt_param(chkpt, 'completed', completed, xtimish=False)
+                    completed = tuple(int(x) for x in completed.split('.'))
+                    logging.info("checkpoint %s completed" % chkpt)
+                status = completed and \
+                  "completed at " + self.humantime(completed[0]) or \
+                  "not reached yet"
+            if s:
+                conn = None
+                try:
+                    conn, _ = chan.accept()
+                    try:
+                        conn.send("  | checkpoint %s %s %s" % (chkpt, status, self.get_extra_info()))
+                    except:
+                        exc = sys.exc_info()[1]
+                        if (isinstance(exc, OSError) or isinstance(exc, IOError)) and \
+                           exc.errno == EPIPE:
+                            logging.debug('checkpoint client disconnected')
+                        else:
+                            raise
+                finally:
+                    if conn:
+                        conn.close()
+
+    def start_checkpoint_thread(self):
+        """prepare and start checkpoint service"""
+        if self.checkpoint_thread or not (
+          getattr(gconf, 'state_socket_unencoded', None) and getattr(gconf, 'socketdir', None)
+        ):
+            return
+        chan = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        state_socket = os.path.join(gconf.socketdir, md5hex(gconf.state_socket_unencoded) + ".socket")
+        try:
+            os.unlink(state_socket)
+        except:
+            if sys.exc_info()[0] == OSError:
+                pass
+        chan.bind(state_socket)
+        chan.listen(1)
+        checkpt_tgt = None
+        if gconf.checkpoint:
+            checkpt_tgt = self._checkpt_param(gconf.checkpoint, 'target')
+            if not checkpt_tgt:
+                checkpt_tgt = self.xtime('.')
+                if isinstance(checkpt_tgt, int):
+                    raise GsyncdError("master root directory is unaccessible (%s)",
+                                      os.strerror(checkpt_tgt))
+                self._set_checkpt_param(gconf.checkpoint, 'target', checkpt_tgt)
+            logging.debug("checkpoint target %s has been determined for checkpoint %s" % \
+                          (repr(checkpt_tgt), gconf.checkpoint))
+        t = Thread(target=self.checkpt_service, args=(chan, gconf.checkpoint, checkpt_tgt))
+        t.start()
+        self.checkpoint_thread = t
+
+    def add_job(self, path, label, job, *a, **kw):
+        """insert @job function to job table at @path with @label"""
+        if self.jobtab.get(path) == None:
+            self.jobtab[path] = []
+        self.jobtab[path].append((label, a, lambda : job(*a, **kw)))
+
+    def add_failjob(self, path, label):
+        """invoke .add_job with a job that does nothing just fails"""
+        logging.debug('salvaged: ' + label)
+        self.add_job(path, label, lambda: False)
+
+    def wait(self, path, *args):
+        """perform jobs registered for @path
+
+        Reset jobtab entry for @path,
+        determine success as the conjuction of
+        success of all the jobs. In case of
+        success, call .sendmark on @path
+        """
+        jobs = self.jobtab.pop(path, [])
+        succeed = True
+        for j in jobs:
+            ret = j[-1]()
+            if not ret:
+                succeed = False
+        if succeed and not args[0] == None:
+            self.sendmark(path, *args)
+        return succeed
+
+    def sendmark(self, path, mark, adct=None):
+        """update slave side xtime for @path to master side xtime
+
+        also can send a setattr payload (see Server.setattr).
+        """
+        if adct:
+            self.slave.server.setattr(path, adct)
+        self.set_slave_xtime(path, mark)
+
+class GMasterChangelogMixin(GMasterCommon):
+    """ changelog based change detection and syncing """
+
+    # index for change type and entry
+    IDX_START = 0
+    IDX_END   = 2
+
+    POS_GFID   = 0
+    POS_TYPE   = 1
+    POS_ENTRY1 = 2
+    POS_ENTRY2 = 3  # renames
+
+    _CL_TYPE_DATA_PFX     = "D "
+    _CL_TYPE_METADATA_PFX = "M "
+    _CL_TYPE_ENTRY_PFX    = "E "
+
+    TYPE_GFID  = [_CL_TYPE_DATA_PFX] # ignoring metadata ops
+    TYPE_ENTRY = [_CL_TYPE_ENTRY_PFX]
+
+    # flat directory heirarchy for gfid based access
+    FLAT_DIR_HIERARCHY = '.'
+
+    # maximum retries per changelog before giving up
+    MAX_RETRIES = 10
+
+    def fallback_xsync(self):
+        logging.info('falling back to xsync mode')
+        gconf.configinterface.set('change-detector', 'xsync')
+        selfkill()
+
+    def setup_working_dir(self):
+        workdir = os.path.join(gconf.working_dir, md5hex(gconf.local_path))
+        logfile = os.path.join(workdir, 'changes.log')
+        logging.debug('changelog working dir %s (log: %s)' % (workdir, logfile))
+        return (workdir, logfile)
+
+    # update stats from *this* crawl
+    def update_cumulative_stats(self, files_pending):
+        self.total_crawl_stats['files_remaining']  = files_pending['count']
+        self.total_crawl_stats['bytes_remaining']  = files_pending['bytes']
+        self.total_crawl_stats['purges_remaining'] = files_pending['purge']
+
+    # sync data
+    def syncdata(self, datas):
+        logging.debug('datas: %s' % (datas))
+        for data in datas:
+            logging.debug('candidate for syncing %s' % data)
+            pb = self.syncer.add(data)
+            def regjob(se, xte, pb):
+                rv = pb.wait()
+                if rv[0]:
+                    logging.debug('synced ' + se)
+                    return True
+                else:
+                    if rv[1] in [23, 24]:
+                        # stat to check if the file exist
+                        st = lstat(se)
+                        if isinstance(st, int):
+                            # file got unlinked in the interim
+                            return True
+                    logging.warn('Rsync: %s [errcode: %d]' % (se, rv[1]))
+            self.add_job(self.FLAT_DIR_HIERARCHY, 'reg', regjob, data, None, pb)
+        if self.wait(self.FLAT_DIR_HIERARCHY, None):
+            return True
+
+    def process_change(self, change, done, retry):
+        pfx = gauxpfx()
+        clist   = []
+        entries = []
+        datas = set()
+
+        # basic crawl stats: files and bytes
+        files_pending  = {'count': 0, 'purge': 0, 'bytes': 0, 'files': []}
+        try:
+            f = open(change, "r")
+            clist = f.readlines()
+            f.close()
+        except IOError:
+            raise
+
+        def edct(op, **ed):
+            dct = {}
+            dct['op'] = op
+            for k in ed:
+                if k == 'stat':
+                    st = ed[k]
+                    dst = dct['stat'] = {}
+                    dst['uid'] = st.st_uid
+                    dst['gid'] = st.st_gid
+                    dst['mode'] = st.st_mode
+                else:
+                    dct[k] = ed[k]
+            return dct
+
+        # regular file update: bytes & count
+        def _update_reg(entry, size):
+            if not entry in files_pending['files']:
+                files_pending['count'] += 1
+                files_pending['bytes'] += size
+                files_pending['files'].append(entry)
+        # updates for directories, symlinks etc..
+        def _update_rest():
+            files_pending['count'] += 1
+
+        # entry count
+        def entry_update(entry, size, mode):
+            if stat.S_ISREG(mode):
+                _update_reg(entry, size)
+            else:
+                _update_rest()
+        # purge count
+        def purge_update():
+            files_pending['purge'] += 1
+
+        for e in clist:
+            e = e.strip()
+            et = e[self.IDX_START:self.IDX_END]
+            ec = e[self.IDX_END:].split(' ')
+            if et in self.TYPE_ENTRY:
+                ty = ec[self.POS_TYPE]
+                en = unescape(os.path.join(pfx, ec[self.POS_ENTRY1]))
+                gfid = ec[self.POS_GFID]
+                # definitely need a better way bucketize entry ops
+                if ty in ['UNLINK', 'RMDIR']:
+                    purge_update()
+                    entries.append(edct(ty, gfid=gfid, entry=en))
+                    continue
+                go = os.path.join(pfx, gfid)
+                st = lstat(go)
+                if isinstance(st, int):
+		    if ty == 'RENAME':
+                        entries.append(edct('UNLINK', gfid=gfid, entry=en))
+		    else:
+                        logging.debug('file %s got purged in the interim' % go)
+                    continue
+                entry_update(go, st.st_size, st.st_mode)
+                if ty in ['CREATE', 'MKDIR', 'MKNOD']:
+                    entries.append(edct(ty, stat=st, entry=en, gfid=gfid))
+                elif ty == 'LINK':
+                    entries.append(edct(ty, stat=st, entry=en, gfid=gfid))
+                elif ty == 'SYMLINK':
+                    rl = errno_wrap(os.readlink, [en], [ENOENT])
+                    if isinstance(rl, int):
+                        continue
+                    entries.append(edct(ty, stat=st, entry=en, gfid=gfid, link=rl))
+                elif ty == 'RENAME':
+                    e2 = unescape(os.path.join(pfx, ec[self.POS_ENTRY2]))
+                    entries.append(edct(ty, gfid=gfid, entry=en, entry1=e2, stat=st))
+                else:
+                    logging.warn('ignoring %s [op %s]' % (gfid, ty))
+            elif et in self.TYPE_GFID:
+                go = os.path.join(pfx, ec[0])
+                st = lstat(go)
+                if isinstance(st, int):
+                    logging.debug('file %s got purged in the interim' % go)
+                    continue
+                entry_update(go, st.st_size, st.st_mode)
+                datas.update([go])
+        logging.debug('entries: %s' % repr(entries))
+        if not retry:
+            self.update_cumulative_stats(files_pending)
+        # sync namespace
+        if (entries):
+            self.slave.server.entry_ops(entries)
+        # sync data
+        if self.syncdata(datas):
+            if done:
+                self.master.server.changelog_done(change)
+            return True
+
+    def sync_done(self):
+        self.total_crawl_stats['files_syncd'] += self.total_crawl_stats['files_remaining']
+        self.total_crawl_stats['files_remaining']  = 0
+        self.total_crawl_stats['bytes_remaining']  = 0
+        self.total_crawl_stats['purges_remaining'] = 0
+        self.update_crawl_data()
+
+    def process(self, changes, done=1):
+        for change in changes:
+            tries = 0
+            retry = False
+            while True:
+                logging.debug('processing change %s' % change)
+                if self.process_change(change, done, retry):
+                    self.sync_done()
+                    break
+                retry = True
+                tries += 1
+                if tries == self.MAX_RETRIES:
+                    logging.warn('changelog %s could not be processed - moving on...' % os.path.basename(change))
+                    self.sync_done()
+                    if done:
+                        self.master.server.changelog_done(change)
+                    break
+                # it's either entry_ops() or Rsync that failed to do it's
+                # job. Mostly it's entry_ops() [which currently has a problem
+                # of failing to create an entry but failing to return an errno]
+                # Therefore we do not know if it's either Rsync or the freaking
+                # entry_ops() that failed... so we retry the _whole_ changelog
+                # again.
+                # TODO: remove entry retries when it's gets fixed.
+                logging.warn('incomplete sync, retrying changelog: %s' % change)
+                time.sleep(0.5)
+            self.turns += 1
+
+    def upd_stime(self, stime):
+        if not stime == URXTIME:
+            self.sendmark(self.FLAT_DIR_HIERARCHY, stime)
+
+    def crawl(self):
+        changes = []
+        try:
+            self.master.server.changelog_scan()
+            self.crawls += 1
+        except OSError:
+            self.fallback_xsync()
+        changes = self.master.server.changelog_getchanges()
+        if changes:
+            xtl = self.xtime(self.FLAT_DIR_HIERARCHY)
+            if isinstance(xtl, int):
+                raise GsyncdError('master is corrupt')
+            logging.debug('processing changes %s' % repr(changes))
+            self.process(changes)
+            self.upd_stime(xtl)
+
+    def register(self):
+        (workdir, logfile) = self.setup_working_dir()
+        self.sleep_interval = int(gconf.change_interval)
+        # register with the changelog library
+        try:
+            # 9 == log level (DEBUG)
+            # 5 == connection retries
+            self.master.server.changelog_register(gconf.local_path,
+                                                  workdir, logfile, 9, 5)
+        except OSError:
+            self.fallback_xsync()
+            # control should not reach here
+            raise
+
+class GMasterXsyncMixin(GMasterChangelogMixin):
+    """
+
+    This crawl needs to be xtime based (as of now
+    it's not. this is beacuse we generate CHANGELOG
+    file during each crawl which is then processed
+    by process_change()).
+    For now it's used as a one-shot initial sync
+    mechanism and only syncs directories, regular
+    files and symlinks.
+    """
+
+    def register(self):
+        self.sleep_interval = 60
+        self.tempdir = self.setup_working_dir()[0]
+        self.tempdir = os.path.join(self.tempdir, 'xsync')
+        logging.info('xsync temp directory: %s' % self.tempdir)
+        try:
+            os.makedirs(self.tempdir)
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno == EEXIST and os.path.isdir(self.tempdir):
+                pass
+            else:
+                raise
+
+    def write_entry_change(self, prefix, data=[]):
+        self.fh.write("%s %s\n" % (prefix, ' '.join(data)))
+
+    def open(self):
+        try:
+            self.xsync_change = os.path.join(self.tempdir, 'XSYNC-CHANGELOG.' + str(int(time.time())))
+            self.fh = open(self.xsync_change, 'w')
+        except IOError:
+            raise
+
+    def close(self):
+        self.fh.close()
+
+    def fname(self):
+        return self.xsync_change
+
+    def crawl(self, path='.', xtr=None, done=0):
+        """ generate a CHANGELOG file consumable by process_change """
+        if path == '.':
+            self.open()
+            self.crawls += 1
+        if not xtr:
+            # get the root stime and use it for all comparisons
+            xtr = self.xtime('.', self.slave)
+            if isinstance(xtr, int):
+                if xtr != ENOENT:
+                    raise GsyncdError('slave is corrupt')
+                xtr = self.minus_infinity
+        xtl = self.xtime(path)
+        if isinstance(xtl, int):
+            raise GsyncdError('master is corrupt')
+        if xtr == xtl:
+            if path == '.':
+                self.close()
+            return
+        self.xtime_reversion_hook(path, xtl, xtr)
+        logging.debug("entering " + path)
+        dem = self.master.server.entries(path)
+        pargfid = self.master.server.gfid(path)
+        if isinstance(pargfid, int):
+            logging.warn('skipping directory %s' % (path))
+        for e in dem:
+            bname = e
+            e = os.path.join(path, e)
+            st = lstat(e)
+            if isinstance(st, int):
+                logging.warn('%s got purged in the interim..' % e)
+                continue
+            gfid = self.master.server.gfid(e)
+            if isinstance(gfid, int):
+                logging.warn('skipping entry %s..' % (e))
+                continue
+            xte = self.xtime(e)
+            if isinstance(xte, int):
+                raise GsyncdError('master is corrupt')
+            if not self.need_sync(e, xte, xtr):
+                continue
+            mo = st.st_mode
+            if stat.S_ISDIR(mo):
+                self.write_entry_change("E", [gfid, 'MKDIR', escape(os.path.join(pargfid, bname))])
+                self.crawl(e, xtr)
+            elif stat.S_ISLNK(mo):
+                rl = errno_wrap(os.readlink, [en], [ENOENT])
+                if isinstance(rl, int):
+                    continue
+                self.write_entry_change("E", [gfid, 'SYMLINK', escape(os.path.join(pargfid, bname)), rl])
+            else:
+                # if a file has a hardlink, create a Changelog entry as 'LINK' so the slave
+                # side will decide if to create the new entry, or to create link.
+                if st.st_nlink == 1:
+                    self.write_entry_change("E", [gfid, 'MKNOD', escape(os.path.join(pargfid, bname))])
+                else:
+                    self.write_entry_change("E", [gfid, 'LINK', escape(os.path.join(pargfid, bname))])
+                if stat.S_ISREG(mo):
+                    self.write_entry_change("D", [gfid])
+
+        if path == '.':
+            logging.info('processing xsync changelog %s' % self.fname())
+            self.close()
+            self.process([self.fname()], done)
+            self.upd_stime(xtl)
+
+class BoxClosedErr(Exception):
+    pass
+
+class PostBox(list):
+    """synchronized collection for storing things thought of as "requests" """
+
+    def __init__(self, *a):
+        list.__init__(self, *a)
+        # too bad Python stdlib does not have read/write locks...
+        # it would suffivce to grab the lock in .append as reader, in .close as writer
+        self.lever = Condition()
+        self.open = True
+        self.done = False
+
+    def wait(self):
+        """wait on requests to be processed"""
+        self.lever.acquire()
+        if not self.done:
+            self.lever.wait()
+        self.lever.release()
+        return self.result
+
+    def wakeup(self, data):
+        """wake up requestors with the result"""
+        self.result = data
+        self.lever.acquire()
+        self.done = True
+        self.lever.notifyAll()
+        self.lever.release()
+
+    def append(self, e):
+        """post a request"""
+        self.lever.acquire()
+        if not self.open:
+            raise BoxClosedErr
+        list.append(self, e)
+        self.lever.release()
+
+    def close(self):
+        """prohibit the posting of further requests"""
+        self.lever.acquire()
+        self.open = False
+        self.lever.release()
+
+class Syncer(object):
+    """a staged queue to relay rsync requests to rsync workers
+
+    By "staged queue" its meant that when a consumer comes to the
+    queue, it takes _all_ entries, leaving the queue empty.
+    (I don't know if there is an official term for this pattern.)
+
+    The queue uses a PostBox to accumulate incoming items.
+    When a consumer (rsync worker) comes, a new PostBox is
+    set up and the old one is passed on to the consumer.
+
+    Instead of the simplistic scheme of having one big lock
+    which synchronizes both the addition of new items and
+    PostBox exchanges, use a separate lock to arbitrate consumers,
+    and rely on PostBox's synchronization mechanisms take
+    care about additions.
+
+    There is a corner case racy situation, producers vs. consumers,
+    which is not handled by this scheme: namely, when the PostBox
+    exchange occurs in between being passed to the producer for posting
+    and the post placement. But that's what Postbox.close is for:
+    such a posting will find the PostBox closed, in which case
+    the producer can re-try posting against the actual PostBox of
+    the queue.
+
+    To aid accumlation of items in the PostBoxen before grabbed
+    by an rsync worker, the worker goes to sleep a bit after
+    each completed syncjob.
+    """
+
+    def __init__(self, slave):
+        """spawn worker threads"""
+        self.slave = slave
+        self.lock = Lock()
+        self.pb = PostBox()
+        self.bytes_synced = 0
+        for i in range(int(gconf.sync_jobs)):
+            t = Thread(target=self.syncjob)
+            t.start()
+
+    def syncjob(self):
+        """the life of a worker"""
+        while True:
+            pb = None
+            while True:
+                self.lock.acquire()
+                if self.pb:
+                    pb, self.pb = self.pb, PostBox()
+                self.lock.release()
+                if pb:
+                    break
+                time.sleep(0.5)
+            pb.close()
+            po = self.slave.rsync(pb)
+            if po.returncode == 0:
+                ret = (True, 0)
+            elif po.returncode in (23, 24):
+                # partial transfer (cf. rsync(1)), that's normal
+                ret = (False, po.returncode)
+            else:
+                po.errfail()
+            pb.wakeup(ret)
+
+    def add(self, e):
+        while True:
+            pb = self.pb
+            try:
+                pb.append(e)
+                return pb
+            except BoxClosedErr:
+                pass
diff --git a/geo-replication/syncdaemon/monitor.py b/geo-replication/syncdaemon/monitor.py
new file mode 100644
index 000000000..0c3a42fa6
--- /dev/null
+++ b/geo-replication/syncdaemon/monitor.py
@@ -0,0 +1,244 @@
+import os
+import sys
+import time
+import signal
+import logging
+import uuid
+import xml.etree.ElementTree as XET
+from subprocess import PIPE
+from resource import Popen, FILE, GLUSTER, SSH
+from threading import Lock
+from gconf import gconf
+from syncdutils import update_file, select, waitpid, set_term_handler, is_host_local, GsyncdError
+from syncdutils import escape, Thread, finalize, memoize
+
+class Volinfo(object):
+    def __init__(self, vol, host='localhost', prelude=[]):
+        po = Popen(prelude + ['gluster', '--xml', '--remote-host=' + host, 'volume', 'info', vol],
+                   stdout=PIPE, stderr=PIPE)
+        vix = po.stdout.read()
+        po.wait()
+        po.terminate_geterr()
+        vi = XET.fromstring(vix)
+        if vi.find('opRet').text != '0':
+            if prelude:
+                via = '(via %s) ' % prelude.join(' ')
+            else:
+                via = ' '
+            raise GsyncdError('getting volume info of %s%s failed with errorcode %s',
+                              (vol, via, vi.find('opErrno').text))
+        self.tree = vi
+        self.volume = vol
+        self.host = host
+
+    def get(self, elem):
+        return self.tree.findall('.//' + elem)
+
+    @property
+    @memoize
+    def bricks(self):
+        def bparse(b):
+            host, dirp = b.text.split(':', 2)
+            return {'host': host, 'dir': dirp}
+        return [ bparse(b) for b in self.get('brick') ]
+
+    @property
+    @memoize
+    def uuid(self):
+        ids = self.get('id')
+        if len(ids) != 1:
+            raise GsyncdError("volume info of %s obtained from %s: ambiguous uuid",
+                              self.volume, self.host)
+        return ids[0].text
+
+
+class Monitor(object):
+    """class which spawns and manages gsyncd workers"""
+
+    ST_INIT     = 'Initializing...'
+    ST_STABLE   = 'Stable'
+    ST_FAULTY   = 'faulty'
+    ST_INCON    = 'inconsistent'
+    _ST_ORD = [ST_STABLE, ST_INIT, ST_FAULTY, ST_INCON]
+
+    def __init__(self):
+        self.lock = Lock()
+        self.state = {}
+
+    def set_state(self, state, w=None):
+        """set the state that can be used by external agents
+           like glusterd for status reporting"""
+        computestate = lambda: self.state and self._ST_ORD[max(self._ST_ORD.index(s) for s in self.state.values())]
+        if w:
+            self.lock.acquire()
+            old_state = computestate()
+            self.state[w] = state
+            state = computestate()
+            self.lock.release()
+            if state != old_state:
+                self.set_state(state)
+        else:
+            logging.info('new state: %s' % state)
+            if getattr(gconf, 'state_file', None):
+                update_file(gconf.state_file, lambda f: f.write(state + '\n'))
+
+    @staticmethod
+    def terminate():
+        # relax one SIGTERM by setting a handler that sets back
+        # standard handler
+        set_term_handler(lambda *a: set_term_handler())
+        # give a chance to graceful exit
+        os.kill(-os.getpid(), signal.SIGTERM)
+
+    def monitor(self, w, argv, cpids):
+        """the monitor loop
+
+        Basic logic is a blantantly simple blunt heuristics:
+        if spawned client survives 60 secs, it's considered OK.
+        This servers us pretty well as it's not vulneralbe to
+        any kind of irregular behavior of the child...
+
+        ... well, except for one: if children is hung up on
+        waiting for some event, it can survive aeons, still
+        will be defunct. So we tweak the above logic to
+        expect the worker to send us a signal within 60 secs
+        (in the form of closing its end of a pipe). The worker
+        does this when it's done with the setup stage
+        ready to enter the service loop (note it's the setup
+        stage which is vulnerable to hangs -- the full
+        blown worker blows up on EPIPE if the net goes down,
+        due to the keep-alive thread)
+        """
+
+        self.set_state(self.ST_INIT, w)
+        ret = 0
+        def nwait(p, o=0):
+            p2, r = waitpid(p, o)
+            if not p2:
+                return
+            return r
+        def exit_signalled(s):
+            """ child teminated due to receipt of SIGUSR1 """
+            return (os.WIFSIGNALED(s) and (os.WTERMSIG(s) == signal.SIGUSR1))
+        def exit_status(s):
+            if os.WIFEXITED(s):
+                return os.WEXITSTATUS(s)
+            return 1
+        conn_timeout = int(gconf.connection_timeout)
+        while ret in (0, 1):
+            logging.info('-' * conn_timeout)
+            logging.info('starting gsyncd worker')
+            pr, pw = os.pipe()
+            cpid = os.fork()
+            if cpid == 0:
+                os.close(pr)
+                os.execv(sys.executable, argv + ['--feedback-fd', str(pw),
+                                                 '--local-path', w[0],
+                                                 '--local-id', '.' + escape(w[0]),
+                                                 '--resource-remote', w[1]])
+            self.lock.acquire()
+            cpids.add(cpid)
+            self.lock.release()
+            os.close(pw)
+            t0 = time.time()
+            so = select((pr,), (), (), conn_timeout)[0]
+            os.close(pr)
+            if so:
+                ret = nwait(cpid, os.WNOHANG)
+                if ret != None:
+                    logging.debug("worker died before establishing connection")
+                else:
+                    logging.debug("worker seems to be connected (?? racy check)")
+                    while time.time() < t0 + conn_timeout:
+                        ret = nwait(cpid, os.WNOHANG)
+                        if ret != None:
+                            logging.debug("worker died in startup phase")
+                            break
+                        time.sleep(1)
+            else:
+                logging.debug("worker not confirmed in %d sec, aborting it" % \
+                              conn_timeout)
+                self.terminate()
+                time.sleep(1)
+                os.kill(cpid, signal.SIGKILL)
+                ret = nwait(cpid)
+            if ret == None:
+                self.set_state(self.ST_STABLE, w)
+                ret = nwait(cpid)
+            if exit_signalled(ret):
+                ret = 0
+            else:
+                ret = exit_status(ret)
+                if ret in (0,1):
+                    self.set_state(self.ST_FAULTY, w)
+            time.sleep(10)
+        self.set_state(self.ST_INCON, w)
+        return ret
+
+    def multiplex(self, wspx, suuid):
+        argv = sys.argv[:]
+        for o in ('-N', '--no-daemon', '--monitor'):
+            while o in argv:
+                argv.remove(o)
+        argv.extend(('-N', '-p', '', '--slave-id', suuid))
+        argv.insert(0, os.path.basename(sys.executable))
+
+        cpids = set()
+        ta = []
+        for wx in wspx:
+            def wmon(w):
+                cpid, _ = self.monitor(w, argv, cpids)
+                terminate()
+                time.sleep(1)
+                self.lock.acquire()
+                for cpid in cpids:
+                    os.kill(cpid, signal.SIGKILL)
+                self.lock.release()
+                finalize(exval=1)
+            t = Thread(target = wmon, args=[wx])
+            t.start()
+            ta.append(t)
+        for t in ta:
+            t.join()
+
+def distribute(*resources):
+    master, slave = resources
+    mvol = Volinfo(master.volume, master.host)
+    logging.debug('master bricks: ' + repr(mvol.bricks))
+    prelude  = []
+    si = slave
+    if isinstance(slave, SSH):
+        prelude = gconf.ssh_command.split() + [slave.remote_addr]
+        si = slave.inner_rsc
+        logging.debug('slave SSH gateway: ' + slave.remote_addr)
+    if isinstance(si, FILE):
+        sbricks = {'host': 'localhost', 'dir': si.path}
+        suuid = uuid.uuid5(uuid.NAMESPACE_URL, slave.get_url(canonical=True))
+    elif isinstance(si, GLUSTER):
+        svol = Volinfo(si.volume, si.host, prelude)
+        sbricks = svol.bricks
+        suuid = svol.uuid
+    else:
+        raise GsyncdError("unkown slave type " + slave.url)
+    logging.info('slave bricks: ' + repr(sbricks))
+    if isinstance(si, FILE):
+        slaves = [ slave.url ]
+    else:
+        slavenodes = set(b['host'] for b in sbricks)
+        if isinstance(slave, SSH) and not gconf.isolated_slave:
+            rap = SSH.parse_ssh_address(slave)
+            slaves = [ 'ssh://' + rap['user'] + '@' + h + ':' + si.url for h in slavenodes ]
+        else:
+            slavevols = [ h + ':' + si.volume for h in slavenodes ]
+            if isinstance(slave, SSH):
+                slaves = [ 'ssh://' + rap.remote_addr + ':' + v for v in slavevols ]
+            else:
+                slaves = slavevols
+
+    workerspex = [ (brick['dir'], slaves[idx % len(slaves)]) for idx, brick in enumerate(mvol.bricks) if is_host_local(brick['host']) ]
+    logging.info('worker specs: ' + repr(workerspex))
+    return workerspex, suuid
+
+def monitor(*resources):
+    """oh yeah, actually Monitor is used as singleton, too"""
+    return Monitor().multiplex(*distribute(*resources))
diff --git a/xlators/features/marker/utils/syncdaemon/repce.py b/geo-replication/syncdaemon/repce.py
index 755fb61df..755fb61df 100644
--- a/xlators/features/marker/utils/syncdaemon/repce.py
+++ b/geo-replication/syncdaemon/repce.py
diff --git a/xlators/features/marker/utils/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py
index 3454c3823..faf62f868 100644
--- a/xlators/features/marker/utils/syncdaemon/resource.py
+++ b/geo-replication/syncdaemon/resource.py
@@ -3,21 +3,25 @@ import os
 import sys
 import stat
 import time
+import fcntl
 import errno
+import types
 import struct
 import socket
 import logging
 import tempfile
 import threading
 import subprocess
-from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EISDIR
+from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EISDIR, ENOTEMPTY, ESTALE, EINVAL
+from select import error as SelectError
 
 from gconf import gconf
 import repce
 from repce import RepceServer, RepceClient
-from master import GMaster
+from  master import gmaster_builder
 import syncdutils
-from syncdutils import GsyncdError, select
+from syncdutils import GsyncdError, select, privileged, boolify, funcode
+from syncdutils import umask, entry2pb, gauxpfx, errno_wrap, lstat
 
 UrlRX  = re.compile('\A(\w+)://([^ *?[]*)\Z')
 HostRX = re.compile('[a-z\d](?:[a-z\d.-]*[a-z\d])?', re.I)
@@ -103,12 +107,23 @@ class _MetaXattr(object):
             setattr(self, m, getattr(LXattr, m))
         return getattr(self, meth)
 
+class _MetaChangelog(object):
+    def __getattr__(self, meth):
+        from libgfchangelog import Changes as LChanges
+        xmeth = [ m for m in dir(LChanges) if m[0] != '_' ]
+        if not meth in xmeth:
+            return
+        for m in xmeth:
+            setattr(self, m, getattr(LChanges, m))
+        return getattr(self, meth)
+
 Xattr = _MetaXattr()
+Changes = _MetaChangelog()
 
 
 class Popen(subprocess.Popen):
     """customized subclass of subprocess.Popen with a ring
-    buffer for children error ouput"""
+    buffer for children error output"""
 
     @classmethod
     def init_errhandler(cls):
@@ -116,24 +131,46 @@ class Popen(subprocess.Popen):
         cls.errstore = {}
         def tailer():
             while True:
-                for po in select([po.stderr for po in cls.errstore], [], []):
+                errstore = cls.errstore.copy()
+                try:
+                    poe, _ ,_ = select([po.stderr for po in errstore], [], [], 1)
+                except (ValueError, SelectError):
+                    continue
+                for po in errstore:
+                    if po.stderr not in poe:
+                        continue
                     po.lock.acquire()
                     try:
-                        la = cls.errstore.get(po)
-                        if la == None:
+                        if po.on_death_row:
+                            continue
+                        la = errstore[po]
+                        try:
+                            fd = po.stderr.fileno()
+                        except ValueError:  # file is already closed
+                            continue
+                        l = os.read(fd, 1024)
+                        if not l:
                             continue
-                        l = os.read(po.stderr.fileno(), 1024)
                         tots = len(l)
                         for lx in la:
                             tots += len(lx)
                         while tots > 1<<20 and la:
                             tots -= len(la.pop(0))
+                        la.append(l)
                     finally:
                         po.lock.release()
         t = syncdutils.Thread(target = tailer)
         t.start()
         cls.errhandler = t
 
+    @classmethod
+    def fork(cls):
+        """fork wrapper that restarts errhandler thread in child"""
+        pid = os.fork()
+        if not pid:
+            cls.init_errhandler()
+        return pid
+
     def __init__(self, args, *a, **kw):
         """customizations for subprocess.Popen instantiation
 
@@ -145,6 +182,7 @@ class Popen(subprocess.Popen):
         if 'close_fds' not in kw:
             kw['close_fds'] = True
         self.lock = threading.Lock()
+        self.on_death_row = False
         try:
             sup(self, args, *a, **kw)
         except:
@@ -153,20 +191,32 @@ class Popen(subprocess.Popen):
                 raise
             raise GsyncdError("""execution of "%s" failed with %s (%s)""" % \
                               (args[0], errno.errorcode[ex.errno], os.strerror(ex.errno)))
-        if kw['stderr'] == subprocess.PIPE:
+        if kw.get('stderr') == subprocess.PIPE:
             assert(getattr(self, 'errhandler', None))
             self.errstore[self] = []
 
-    def errfail(self):
-        """fail nicely if child did not terminate with success"""
-        filling = None
+    def errlog(self):
+        """make a log about child's failure event"""
+        filling = ""
         if self.elines:
             filling = ", saying:"
-        logging.error("""command "%s" returned with %d%s""" % \
-                      (" ".join(self.args), self.returncode, filling))
+        logging.error("""command "%s" returned with %s%s""" % \
+                      (" ".join(self.args), repr(self.returncode), filling))
+        lp = ''
+        def logerr(l):
+            logging.error(self.args[0] + "> " + l)
         for l in self.elines:
-            for ll in l.rstrip().split("\n"):
-                logging.error(self.args[0] + "> " + ll.rstrip())
+            ls = l.split('\n')
+            ls[0] = lp + ls[0]
+            lp = ls.pop()
+            for ll in ls:
+                logerr(ll)
+        if lp:
+            logerr(lp)
+
+    def errfail(self):
+        """fail nicely if child did not terminate with success"""
+        self.errlog()
         syncdutils.finalize(exval = 1)
 
     def terminate_geterr(self, fail_on_err = True):
@@ -176,15 +226,19 @@ class Popen(subprocess.Popen):
         """
         self.lock.acquire()
         try:
-            elines = self.errstore.pop(self)
+            self.on_death_row = True
         finally:
             self.lock.release()
+        elines = self.errstore.pop(self)
         if self.poll() == None:
             self.terminate()
-            if sp.poll() == None:
+            if self.poll() == None:
                 time.sleep(0.1)
-            sp.kill()
+                self.kill()
+                self.wait()
         while True:
+            if not select([self.stderr],[],[],0.1)[0]:
+                break
             b = os.read(self.stderr.fileno(), 1024)
             if b:
                 elines.append(b)
@@ -204,10 +258,24 @@ class Server(object):
     and classmethods and is used directly, without instantiation.)
     """
 
-    GX_NSPACE = "trusted.glusterfs"
+    GX_NSPACE_PFX = (privileged() and "trusted" or "system")
+    GX_NSPACE = GX_NSPACE_PFX + ".glusterfs"
     NTV_FMTSTR = "!" + "B"*19 + "II"
     FRGN_XTRA_FMT = "I"
     FRGN_FMTSTR = NTV_FMTSTR + FRGN_XTRA_FMT
+    GX_GFID_CANONICAL_LEN = 37 # canonical gfid len + '\0'
+
+    local_path = ''
+
+    @classmethod
+    def _fmt_mknod(cls, l):
+        return "!II%dsI%dsIII" % (cls.GX_GFID_CANONICAL_LEN, l+1)
+    @classmethod
+    def _fmt_mkdir(cls, l):
+        return "!II%dsI%dsII" % (cls.GX_GFID_CANONICAL_LEN, l+1)
+    @classmethod
+    def _fmt_symlink(cls, l1, l2):
+        return "!II%dsI%ds%ds" % (cls.GX_GFID_CANONICAL_LEN, l1+1, l2+1)
 
     def _pathguard(f):
         """decorator method that checks
@@ -216,22 +284,21 @@ class Server(object):
         point out of the managed tree
         """
 
-        fc = getattr(f, 'func_code', None)
-        if not fc:
-            # python 3
-            fc = f.__code__
+        fc = funcode(f)
         pi = list(fc.co_varnames).index('path')
         def ff(*a):
             path = a[pi]
             ps = path.split('/')
             if path[0] == '/' or '..' in ps:
                 raise ValueError('unsafe path')
+            a = list(a)
+            a[pi] = os.path.join(a[0].local_path, path)
             return f(*a)
         return ff
 
-    @staticmethod
+    @classmethod
     @_pathguard
-    def entries(path):
+    def entries(cls, path):
         """directory entries in an array"""
         # prevent symlinks being followed
         if not stat.S_ISDIR(os.lstat(path).st_mode):
@@ -330,14 +397,162 @@ class Server(object):
                 raise
 
     @classmethod
+    def gfid(cls, gfidpath):
+        return errno_wrap(Xattr.lgetxattr, [gfidpath, 'glusterfs.gfid.string', cls.GX_GFID_CANONICAL_LEN], [ENOENT])
+
+    @classmethod
+    def node_uuid(cls, path='.'):
+        try:
+            uuid_l = Xattr.lgetxattr_buf(path, '.'.join([cls.GX_NSPACE, 'node-uuid']))
+            return uuid_l[:-1].split(' ')
+        except OSError:
+            raise
+
+    @classmethod
+    def xtime_vec(cls, path, *uuids):
+        """vectored version of @xtime
+
+        accepts a list of uuids and returns a dictionary
+        with uuid as key(s) and xtime as value(s)
+        """
+        xt = {}
+        for uuid in uuids:
+            xtu = cls.xtime(path, uuid)
+            if xtu == ENODATA:
+                xtu = None
+            if isinstance(xtu, int):
+                return xtu
+            xt[uuid] = xtu
+        return xt
+
+    @classmethod
     @_pathguard
     def set_xtime(cls, path, uuid, mark):
         """set @mark as xtime for @uuid on @path"""
         Xattr.lsetxattr(path, '.'.join([cls.GX_NSPACE, uuid, 'xtime']), struct.pack('!II', *mark))
 
-    @staticmethod
+    @classmethod
+    @_pathguard
+    def set_xtime_remote(cls, path, uuid, mark):
+        """
+        set @mark as xtime for @uuid on @path
+        the difference b/w this and set_xtime() being
+        set_xtime() being overloaded to set the xtime
+        on the brick (this method sets xtime on the
+        remote slave)
+        """
+        Xattr.lsetxattr(path, '.'.join([cls.GX_NSPACE, uuid, 'xtime']), struct.pack('!II', *mark))
+
+    @classmethod
+    def set_xtime_vec(cls, path, mark_dct):
+        """vectored (or dictered) version of set_xtime
+
+        ignore values that match @ignore
+        """
+        for u,t in mark_dct.items():
+            cls.set_xtime(path, u, t)
+
+    @classmethod
+    def entry_ops(cls, entries):
+        pfx = gauxpfx()
+        logging.debug('entries: %s' % repr(entries))
+        # regular file
+        def entry_pack_reg(gf, bn, st):
+            blen = len(bn)
+            mo = st['mode']
+            return struct.pack(cls._fmt_mknod(blen),
+                               st['uid'], st['gid'],
+                               gf, mo, bn,
+                               stat.S_IMODE(mo), 0, umask())
+        # mkdir
+        def entry_pack_mkdir(gf, bn, st):
+            blen = len(bn)
+            mo = st['mode']
+            return struct.pack(cls._fmt_mkdir(blen),
+                               st['uid'], st['gid'],
+                               gf, mo, bn,
+                               stat.S_IMODE(mo), umask())
+        #symlink
+        def entry_pack_symlink(gf, bn, lnk, st):
+            blen = len(bn)
+            llen = len(lnk)
+            return struct.pack(cls._fmt_symlink(blen, llen),
+                               st['uid'], st['gid'],
+                               gf, st['mode'], bn, lnk)
+        def entry_purge(entry, gfid):
+            # This is an extremely racy code and needs to be fixed ASAP.
+            # The GFID check here is to be sure that the pargfid/bname
+            # to be purged is the GFID gotten from the changelog.
+            # (a stat(changelog_gfid) would also be valid here)
+            # The race here is between the GFID check and the purge.
+            disk_gfid = cls.gfid(entry)
+            if isinstance(disk_gfid, int):
+                return
+            if not gfid == disk_gfid:
+                return
+            er = errno_wrap(os.unlink, [entry], [ENOENT, EISDIR])
+            if isinstance(er, int):
+                if er == EISDIR:
+                    er = errno_wrap(os.rmdir, [entry], [ENOENT, ENOTEMPTY])
+                    if er == ENOTEMPTY:
+                        return er
+        for e in entries:
+            blob = None
+            op = e['op']
+            gfid = e['gfid']
+            entry = e['entry']
+            (pg, bname) = entry2pb(entry)
+            if op in ['RMDIR', 'UNLINK']:
+                while True:
+                    er = entry_purge(entry, gfid)
+                    if isinstance(er, int):
+                        time.sleep(1)
+                    else:
+                        break
+            elif op in ['CREATE', 'MKNOD']:
+                blob = entry_pack_reg(gfid, bname, e['stat'])
+            elif op == 'MKDIR':
+                blob = entry_pack_mkdir(gfid, bname, e['stat'])
+            elif op == 'LINK':
+                slink = os.path.join(pfx, gfid)
+                st = lstat(slink)
+                if isinstance(st, int):
+                    (pg, bname) = entry2pb(entry)
+                    blob = entry_pack_reg(gfid, bname, e['stat'])
+                else:
+                    errno_wrap(os.link, [slink, entry], [ENOENT, EEXIST])
+            elif op == 'SYMLINK':
+                blob = entry_pack_symlink(gfid, bname, e['link'], e['stat'])
+            elif op == 'RENAME':
+                en = e['entry1']
+                st = lstat(entry)
+                if isinstance(st, int):
+                    (pg, bname) = entry2pb(en)
+                    blob = entry_pack_reg(gfid, bname, e['stat'])
+                else:
+                    errno_wrap(os.rename, [entry, en], [ENOENT, EEXIST])
+            if blob:
+                errno_wrap(Xattr.lsetxattr_l, [pg, 'glusterfs.gfid.newfile', blob], [EEXIST], [ENOENT, ESTALE, EINVAL])
+
+    @classmethod
+    def changelog_register(cls, cl_brick, cl_dir, cl_log, cl_level, retries = 0):
+        Changes.cl_register(cl_brick, cl_dir, cl_log, cl_level, retries)
+
+    @classmethod
+    def changelog_scan(cls):
+        Changes.cl_scan()
+
+    @classmethod
+    def changelog_getchanges(cls):
+        return Changes.cl_getchanges()
+
+    @classmethod
+    def changelog_done(cls, clfile):
+        Changes.cl_done(clfile)
+
+    @classmethod
     @_pathguard
-    def setattr(path, adct):
+    def setattr(cls, path, adct):
         """set file attributes
 
         @adct is a dict, where 'own', 'mode' and 'times'
@@ -405,6 +620,10 @@ class SlaveLocal(object):
         stop servicing if a timeout is configured and got no
         keep-alime in that inteval
         """
+
+        if boolify(gconf.use_rsync_xattrs) and not privileged():
+            raise GsyncdError("using rsync for extended attributes is not supported")
+
         repce = RepceServer(self.server, sys.stdin, sys.stdout, int(gconf.sync_jobs))
         t = syncdutils.Thread(target=lambda: (repce.service_loop(),
                                               syncdutils.finalize()))
@@ -431,12 +650,13 @@ class SlaveRemote(object):
         communicate throuh its stdio.
         """
         slave = opts.get('slave', self.url)
+        extra_opts = []
         so = getattr(gconf, 'session_owner', None)
         if so:
-            so_args = ['--session-owner', so]
-        else:
-            so_args = []
-        po = Popen(rargs + gconf.remote_gsyncd.split() + so_args + \
+            extra_opts += ['--session-owner', so]
+        if boolify(gconf.use_rsync_xattrs):
+            extra_opts.append('--use-rsync-xattrs')
+        po = Popen(rargs + gconf.remote_gsyncd.split() + extra_opts + \
                    ['-N', '--listen', '--timeout', str(gconf.timeout), slave],
                    stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         gconf.transport = po
@@ -464,10 +684,19 @@ class SlaveRemote(object):
         if not files:
             raise GsyncdError("no files to sync")
         logging.debug("files: " + ", ".join(files))
-        argv = gconf.rsync_command.split() + gconf.rsync_extra.split() + ['-aR'] + files + list(args)
-        po = Popen(argv, stderr=subprocess.PIPE)
+        argv = gconf.rsync_command.split() + \
+               ['-avR0', '--inplace', '--files-from=-', '--super','--stats', '--numeric-ids', '--no-implied-dirs'] + \
+               gconf.rsync_options.split() + (boolify(gconf.use_rsync_xattrs) and ['--xattrs'] or []) + \
+               ['.'] + list(args)
+        po = Popen(argv, stdin=subprocess.PIPE,stderr=subprocess.PIPE)
+        for f in files:
+            po.stdin.write(f)
+            po.stdin.write('\0')
+
+        po.stdin.close()
         po.wait()
         po.terminate_geterr(fail_on_err = False)
+
         return po
 
 
@@ -575,6 +804,7 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
                     if x[0] > now:
                         logging.debug("volinfo[%s] expires: %d (%d sec later)" % \
                                       (d['uuid'], x[0], x[0] - now))
+                        d['timeout'] = x[0]
                         dict_list.append(d)
                     else:
                         try:
@@ -603,13 +833,15 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
 
     def can_connect_to(self, remote):
         """determine our position in the connectibility matrix"""
-        return True
+        return not remote or \
+               (isinstance(remote, SSH) and isinstance(remote.inner_rsc, GLUSTER))
 
     class Mounter(object):
         """Abstract base class for mounter backends"""
 
         def __init__(self, params):
             self.params = params
+            self.mntpt = None
 
         @classmethod
         def get_glusterprog(cls):
@@ -628,7 +860,7 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
         def make_mount_argv(self, *a):
             raise NotImplementedError
 
-        def cleanup_mntpt(self):
+        def cleanup_mntpt(self, *a):
             pass
 
         def handle_mounter(self, po):
@@ -641,24 +873,75 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
             change into the mount, and lazy unmount the
             filesystem.
             """
-            mounted = False
-            try:
-                po = Popen(self.make_mount_argv(*a), **self.mountkw)
+
+            mpi, mpo = os.pipe()
+            mh = Popen.fork()
+            if mh:
+                os.close(mpi)
+                fcntl.fcntl(mpo, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
+                d = None
+                margv = self.make_mount_argv(*a)
+                if self.mntpt:
+                    # mntpt is determined pre-mount
+                    d = self.mntpt
+                    os.write(mpo, d + '\0')
+                po = Popen(margv, **self.mountkw)
                 self.handle_mounter(po)
                 po.terminate_geterr()
-                d = self.mntpt
-                mounted = True
                 logging.debug('auxiliary glusterfs mount in place')
-                os.chdir(d)
-                self.umount_l(d).terminate_geterr()
-                mounted = False
-            finally:
+                if not d:
+                    # mntpt is determined during mount
+                    d = self.mntpt
+                    os.write(mpo, d + '\0')
+                os.write(mpo, 'M')
+                t = syncdutils.Thread(target=lambda: os.chdir(d))
+                t.start()
+                tlim = gconf.starttime + int(gconf.connection_timeout)
+                while True:
+                    if not t.isAlive():
+                        break
+                    if time.time() >= tlim:
+                        syncdutils.finalize(exval = 1)
+                    time.sleep(1)
+                os.close(mpo)
+                _, rv = syncdutils.waitpid(mh, 0)
+                if rv:
+                    rv = (os.WIFEXITED(rv) and os.WEXITSTATUS(rv) or 0) - \
+                         (os.WIFSIGNALED(rv) and os.WTERMSIG(rv) or 0)
+                    logging.warn('stale mount possibly left behind on ' + d)
+                    raise GsyncdError("cleaning up temp mountpoint %s failed with status %d" % \
+                                      (d, rv))
+            else:
+                rv = 0
                 try:
-                    if mounted:
-                        self.umount_l(d).terminate_geterr(fail_on_err = False)
-                    self.cleanup_mntpt()
+                    os.setsid()
+                    os.close(mpo)
+                    mntdata = ''
+                    while True:
+                        c = os.read(mpi, 1)
+                        if not c:
+                            break
+                        mntdata += c
+                    if mntdata:
+                        mounted = False
+                        if mntdata[-1] == 'M':
+                            mntdata = mntdata[:-1]
+                            assert(mntdata)
+                            mounted = True
+                        assert(mntdata[-1] == '\0')
+                        mntpt = mntdata[:-1]
+                        assert(mntpt)
+                        if mounted:
+                            po = self.umount_l(mntpt)
+                            po.terminate_geterr(fail_on_err = False)
+                            if po.returncode != 0:
+                                po.errlog()
+                                rv = po.returncode
+                        self.cleanup_mntpt(mntpt)
                 except:
-                    logging.warn('stale mount possibly left behind on ' + d)
+                    logging.exception('mount cleanup failure:')
+                    rv = 200
+                os._exit(rv)
             logging.debug('auxiliary glusterfs mount prepared')
 
     class DirectMounter(Mounter):
@@ -675,8 +958,10 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
             self.mntpt = tempfile.mkdtemp(prefix = 'gsyncd-aux-mount-')
             return [self.get_glusterprog()] + ['--' + p for p in self.params] + [self.mntpt]
 
-        def cleanup_mntpt(self):
-            os.rmdir(self.mntpt)
+        def cleanup_mntpt(self, mntpt = None):
+            if not mntpt:
+                mntpt = self.mntpt
+            os.rmdir(mntpt)
 
     class MountbrokerMounter(Mounter):
         """mounter backend using the mountbroker gluster service"""
@@ -715,10 +1000,8 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
         """
 
         label = getattr(gconf, 'mountbroker', None)
-        if not label:
-            uid = os.geteuid()
-            if uid != 0:
-                label = syncdutils.getusername(uid)
+        if not label and not privileged():
+            label = syncdutils.getusername()
         mounter = label and self.MountbrokerMounter or self.DirectMounter
         params = gconf.gluster_params.split() + \
                    (gconf.gluster_log_level and ['log-level=' + gconf.gluster_log_level] or []) + \
@@ -730,6 +1013,10 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
         sup(self, *a, **kw)
         self.slavedir = "/proc/%d/cwd" % self.server.pid()
 
+    def gmaster_instantiate_tuple(self, slave):
+        """return a tuple of the 'one shot' and the 'main crawl' class instance"""
+        return (gmaster_builder('xsync')(self, slave), gmaster_builder()(self, slave))
+
     def service_loop(self, *args):
         """enter service loop
 
@@ -739,7 +1026,41 @@ class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
         - else do that's what's inherited
         """
         if args:
-            GMaster(self, args[0]).crawl_loop()
+            slave = args[0]
+            if gconf.local_path:
+                class brickserver(FILE.FILEServer):
+                    local_path = gconf.local_path
+                    aggregated = self.server
+                    @classmethod
+                    def entries(cls, path):
+                        e = super(brickserver, cls).entries(path)
+                        # on the brick don't mess with /.glusterfs
+                        if path == '.':
+                            try:
+                                e.remove('.glusterfs')
+                            except ValueError:
+                                pass
+                        return e
+                if gconf.slave_id:
+                    # define {,set_}xtime in slave, thus preempting
+                    # the call to remote, so that it takes data from
+                    # the local brick
+                    slave.server.xtime = types.MethodType(lambda _self, path, uuid: brickserver.xtime(path, uuid + '.' + gconf.slave_id), slave.server)
+                    slave.server.set_xtime = types.MethodType(lambda _self, path, uuid, mark: brickserver.set_xtime(path, uuid + '.' + gconf.slave_id, mark), slave.server)
+                (g1, g2) = self.gmaster_instantiate_tuple(slave)
+                g1.master.server = brickserver
+                g2.master.server = brickserver
+            else:
+                (g1, g2) = self.gmaster_instantiate_tuple(slave)
+                g1.master.server.aggregated = gmaster.master.server
+                g2.master.server.aggregated = gmaster.master.server
+            # bad bad bad: bad way to do things like this
+            # need to make this elegant
+            # register the crawlers and start crawling
+            g1.register()
+            g2.register()
+            g1.crawlwrap(oneshot=True)
+            g2.crawlwrap()
         else:
             sup(self, *args)
 
@@ -758,14 +1079,21 @@ class SSH(AbstractUrl, SlaveRemote):
         self.remote_addr, inner_url = sup(self, path,
                                           '^((?:%s@)?%s):(.+)' % tuple([ r.pattern for r in (UserRX, HostRX) ]))
         self.inner_rsc = parse_url(inner_url)
+        self.volume = inner_url[1:]
 
-    def canonical_path(self):
+    @staticmethod
+    def parse_ssh_address(self):
         m = re.match('([^@]+)@(.+)', self.remote_addr)
         if m:
             u, h = m.groups()
         else:
             u, h = syncdutils.getusername(), self.remote_addr
-        remote_addr = '@'.join([u, gethostbyname(h)])
+        self.remotehost = h
+        return {'user': u, 'host': h}
+
+    def canonical_path(self):
+        rap = self.parse_ssh_address(self)
+        remote_addr = '@'.join([rap['user'], gethostbyname(rap['host'])])
         return ':'.join([remote_addr, self.inner_rsc.get_url(canonical=True)])
 
     def can_connect_to(self, remote):
@@ -811,9 +1139,15 @@ class SSH(AbstractUrl, SlaveRemote):
         """
         if go_daemon == 'done':
             return self.start_fd_client(*self.fd_pair)
-        gconf.setup_ssh_ctl(tempfile.mkdtemp(prefix='gsyncd-aux-ssh-'))
+
+        syncdutils.setup_ssh_ctl(tempfile.mkdtemp(prefix='gsyncd-aux-ssh-'),
+                                 self.remote_addr,
+                                 self.inner_rsc.url)
+
         deferred = go_daemon == 'postconn'
-        ret = sup(self, gconf.ssh_command.split() + gconf.ssh_ctl_args + [self.remote_addr], slave=self.inner_rsc.url, deferred=deferred)
+        ret = sup(self, gconf.ssh_command.split() + gconf.ssh_ctl_args + [self.remote_addr],
+                  slave=self.inner_rsc.url, deferred=deferred)
+
         if deferred:
             # send a message to peer so that we can wait for
             # the answer from which we know connection is
@@ -834,4 +1168,5 @@ class SSH(AbstractUrl, SlaveRemote):
             return 'should'
 
     def rsync(self, files):
-        return sup(self, files, '-ze', " ".join(gconf.ssh_command.split() + gconf.ssh_ctl_args), self.slaveurl)
+        return sup(self, files, '-e', " ".join(gconf.ssh_command.split() + gconf.ssh_ctl_args),
+                   *(gconf.rsync_ssh_options.split() + [self.slaveurl]))
diff --git a/xlators/features/marker/utils/syncdaemon/syncdutils.py b/geo-replication/syncdaemon/syncdutils.py
index b91b328b9..348eb38c1 100644
--- a/xlators/features/marker/utils/syncdaemon/syncdutils.py
+++ b/geo-replication/syncdaemon/syncdutils.py
@@ -5,12 +5,14 @@ import time
 import fcntl
 import shutil
 import logging
+import socket
 from threading import Lock, Thread as baseThread
-from errno import EACCES, EAGAIN, EPIPE, ENOTCONN, EINTR
-from signal import SIGTERM, SIGKILL
+from errno import EACCES, EAGAIN, EPIPE, ENOTCONN, ECONNABORTED, EINTR, ENOENT, EPERM, ESTALE, errorcode
+from signal import signal, SIGTERM, SIGKILL
 from time import sleep
 import select as oselect
 from os import waitpid as owaitpid
+
 try:
     from cPickle import PickleError
 except ImportError:
@@ -25,6 +27,16 @@ try:
 except ImportError:
     import urllib
 
+try:
+    from hashlib import md5 as md5
+except ImportError:
+    # py 2.4
+    from md5 import new as md5
+
+# auxillary gfid based access prefix
+_CL_AUX_GFID_PFX = ".gfid/"
+GF_OP_RETRIES = 20
+
 def escape(s):
     """the chosen flavor of string escaping, used all over
        to turn whatever data to creatable representation"""
@@ -68,6 +80,38 @@ def update_file(path, updater, merger = lambda f: True):
             if fx:
                 fx.close()
 
+def create_manifest(fname, content):
+    """
+    Create manifest file for SSH Control Path
+    """
+    fd = None
+    try:
+        fd = os.open(fname, os.O_CREAT|os.O_RDWR)
+        try:
+            os.write(fd, content)
+        except:
+            os.close(fd)
+            raise
+    finally:
+        if fd != None:
+            os.close(fd)
+
+def setup_ssh_ctl(ctld, remote_addr, resource_url):
+    """
+    Setup GConf ssh control path parameters
+    """
+    gconf.ssh_ctl_dir = ctld
+    content = "SLAVE_HOST=%s\nSLAVE_RESOURCE_URL=%s" % (remote_addr,
+                                                            resource_url)
+    content_md5 = md5hex(content)
+    fname = os.path.join(gconf.ssh_ctl_dir,
+                         "%s.mft" % content_md5)
+
+    create_manifest(fname, content)
+    ssh_ctl_path = os.path.join(gconf.ssh_ctl_dir,
+                                "%s.sock" % content_md5)
+    gconf.ssh_ctl_args = ["-oControlMaster=auto", "-S", ssh_ctl_path]
+
 def grabfile(fname, content=None):
     """open @fname + contest for its fcntl lock
 
@@ -138,6 +182,12 @@ def finalize(*a, **kw):
                     raise
     if gconf.ssh_ctl_dir and not gconf.cpid:
         shutil.rmtree(gconf.ssh_ctl_dir)
+    if getattr(gconf, 'state_socket', None):
+        try:
+            os.unlink(gconf.state_socket)
+        except:
+            if sys.exc_info()[0] == OSError:
+                pass
     if gconf.log_exit:
         logging.info("exiting.")
     sys.stdout.flush()
@@ -165,17 +215,23 @@ def log_raise_exception(excont):
         logtag = None
         if isinstance(exc, GsyncdError):
             if is_filelog:
-                logging.error(exc.message)
-            sys.stderr.write('failure: ' + exc.message + "\n")
+                logging.error(exc.args[0])
+            sys.stderr.write('failure: ' + exc.args[0] + '\n')
         elif isinstance(exc, PickleError) or isinstance(exc, EOFError) or \
              ((isinstance(exc, OSError) or isinstance(exc, IOError)) and \
               exc.errno == EPIPE):
             logging.error('connection to peer is broken')
             if hasattr(gconf, 'transport'):
                 gconf.transport.wait()
+                if gconf.transport.returncode == 127:
+                    logging.warn("!!!!!!!!!!!!!")
+                    logging.warn('!!! getting "No such file or directory" errors '
+                                 "is most likely due to MISCONFIGURATION, please consult "
+                                 "http://access.redhat.com/knowledge/docs/en-US/Red_Hat_Storage/2.0/html/Administration_Guide/chap-User_Guide-Geo_Rep-Preparation-Settingup_Environment.html")
+                    logging.warn("!!!!!!!!!!!!!")
                 gconf.transport.terminate_geterr()
-        elif isinstance(exc, OSError) and exc.errno == ENOTCONN:
-            logging.error('glusterfs session went down')
+        elif isinstance(exc, OSError) and exc.errno in (ENOTCONN, ECONNABORTED):
+            logging.error('glusterfs session went down [%s]', errorcode[exc.errno])
         else:
             logtag = "FAIL"
         if not logtag and logging.getLogger().isEnabledFor(logging.DEBUG):
@@ -225,6 +281,9 @@ def getusername(uid = None):
         uid = os.geteuid()
     return pwd.getpwuid(uid).pw_name
 
+def privileged():
+    return os.geteuid() == 0
+
 def boolify(s):
     """
     Generic string to boolean converter
@@ -261,10 +320,119 @@ def eintr_wrap(func, exc, *a):
         except exc:
             ex = sys.exc_info()[1]
             if not ex.args[0] == EINTR:
-                raise GsyncdError(ex.args[1])
+                raise
 
 def select(*a):
     return eintr_wrap(oselect.select, oselect.error, *a)
 
 def waitpid (*a):
     return eintr_wrap(owaitpid, OSError, *a)
+
+def set_term_handler(hook=lambda *a: finalize(*a, **{'exval': 1})):
+    signal(SIGTERM, hook)
+
+def is_host_local(host):
+    locaddr = False
+    for ai in socket.getaddrinfo(host, None):
+        # cf. http://github.com/gluster/glusterfs/blob/ce111f47/xlators/mgmt/glusterd/src/glusterd-utils.c#L125
+        if ai[0] == socket.AF_INET:
+            if ai[-1][0].split(".")[0] == "127":
+                locaddr = True
+                break
+        elif ai[0] == socket.AF_INET6:
+            if ai[-1][0] == "::1":
+                locaddr = True
+                break
+        else:
+            continue
+        try:
+            # use ICMP socket to avoid net.ipv4.ip_nonlocal_bind issue,
+            # cf. https://bugzilla.redhat.com/show_bug.cgi?id=890587
+            s = socket.socket(ai[0], socket.SOCK_RAW, socket.IPPROTO_ICMP)
+        except socket.error:
+            ex = sys.exc_info()[1]
+            if ex.errno != EPERM:
+                raise
+            f = None
+            try:
+                f = open("/proc/sys/net/ipv4/ip_nonlocal_bind")
+                if int(f.read()) != 0:
+                    raise GsyncdError(
+                            "non-local bind is set and not allowed to create raw sockets, "
+                            "cannot determine if %s is local" % host)
+                s = socket.socket(ai[0], socket.SOCK_DGRAM)
+            finally:
+                if f:
+                    f.close()
+        try:
+            s.bind(ai[-1])
+            locaddr = True
+            break
+        except:
+            pass
+        s.close()
+    return locaddr
+
+def funcode(f):
+    fc = getattr(f, 'func_code', None)
+    if not fc:
+        # python 3
+        fc = f.__code__
+    return fc
+
+def memoize(f):
+    fc = funcode(f)
+    fn = fc.co_name
+    def ff(self, *a, **kw):
+        rv = getattr(self, '_' + fn, None)
+        if rv == None:
+            rv = f(self, *a, **kw)
+            setattr(self, '_' + fn, rv)
+        return rv
+    return ff
+
+def umask():
+    return os.umask(0)
+
+def entry2pb(e):
+    return e.rsplit('/', 1)
+
+def gauxpfx():
+    return _CL_AUX_GFID_PFX
+
+def md5hex(s):
+    return md5(s).hexdigest()
+
+def selfkill(sig=SIGTERM):
+    os.kill(os.getpid(), sig)
+
+def errno_wrap(call, arg=[], errnos=[], retry_errnos=[ESTALE]):
+    """ wrapper around calls resilient to errnos.
+    retry in case of ESTALE by default.
+    """
+    nr_tries = 0
+    while True:
+        try:
+            return call(*arg)
+        except OSError:
+            ex = sys.exc_info()[1]
+            if ex.errno in errnos:
+                return ex.errno
+            if not ex.errno in retry_errnos:
+                raise
+            nr_tries += 1
+            if nr_tries == GF_OP_RETRIES:
+                # probably a screwed state, cannot do much...
+                logging.warn('reached maximum retries (%s)...' % repr(arg))
+                return
+            time.sleep(0.250)  # retry the call
+
+def lstat(e):
+    try:
+        return os.lstat(e)
+    except (IOError, OSError):
+        ex = sys.exc_info()[1]
+        if ex.errno == ENOENT:
+            return ex.errno
+        else:
+            raise
diff --git a/gf-error-codes.h.template b/gf-error-codes.h.template
new file mode 100644
index 000000000..ab6020d64
--- /dev/null
+++ b/gf-error-codes.h.template
@@ -0,0 +1,33 @@
+/***************************************************************/
+/**                                                           **/
+/**                   DO NOT EDIT THIS FILE                   **/
+/**           THIS IS AUTO-GENERATED FROM LOG BOOK            **/
+/**         YOUR CHANGES WILL BE LOST IN NEXT BUILD           **/
+/**                                                           **/
+/***************************************************************/
+
+#ifndef _GF_ERROR_CODES_H
+#define _GF_ERROR_CODES_H
+
+#include <libintl.h>
+
+#define _(STRING)    gettext(STRING)
+
+
+/** START: ERROR CODE DEFINITIONS **/
+$DEFINES
+/** END: ERROR CODE DEFINITIONS **/
+
+
+/** START: FUNCTION RETURNS MESSAGE OF GIVEN ERROR CODE **/
+const char *
+_gf_get_message (int code) {
+        switch (code) {
+$CASES
+        default: return NULL;
+        }
+}
+/** END: FUNCTION RETURNS MESSAGE OF GIVEN ERROR CODE **/
+
+
+#endif
diff --git a/glusterfs-api.pc.in b/glusterfs-api.pc.in
new file mode 100644
index 000000000..fab4a57d5
--- /dev/null
+++ b/glusterfs-api.pc.in
@@ -0,0 +1,12 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+
+Name: glusterfs-api
+Description: GlusterFS API
+/* This is the API version, NOT package version */
+Version: 6
+Libs: -L${libdir} -lgfapi -lglusterfs -lgfrpc -lgfxdr
+Cflags: -I${includedir}/glusterfs -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64
diff --git a/glusterfs-hadoop/0.20.2/conf/core-site.xml b/glusterfs-hadoop/0.20.2/conf/core-site.xml
new file mode 100644
index 000000000..d7f75fca7
--- /dev/null
+++ b/glusterfs-hadoop/0.20.2/conf/core-site.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+  <property>
+    <name>fs.glusterfs.impl</name>
+    <value>org.apache.hadoop.fs.glusterfs.GlusterFileSystem</value>
+  </property>
+
+  <property>
+    <name>fs.default.name</name>
+    <value>glusterfs://192.168.1.36:9000</value>
+  </property>
+
+  <property>
+    <name>fs.glusterfs.volname</name>
+    <value>volume-dist-rep</value>
+  </property>
+
+  <property>
+    <name>fs.glusterfs.mount</name>
+    <value>/mnt/glusterfs</value>
+  </property>
+
+  <property>
+    <name>fs.glusterfs.server</name>
+    <value>192.168.1.36</value>
+  </property>
+
+  <property>
+    <name>quick.slave.io</name>
+    <value>Off</value>
+  </property>
+
+</configuration>
diff --git a/glusterfs-hadoop/0.20.2/pom.xml b/glusterfs-hadoop/0.20.2/pom.xml
new file mode 100644
index 000000000..fe661d40f
--- /dev/null
+++ b/glusterfs-hadoop/0.20.2/pom.xml
@@ -0,0 +1,36 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.hadoop.fs.glusterfs</groupId>
+  <artifactId>glusterfs</artifactId>
+  <packaging>jar</packaging>
+  <version>0.20.2-0.1</version>
+  <name>glusterfs</name>
+  <url>http://maven.apache.org</url>
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>3.8.1</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-core</artifactId>
+      <version>0.20.2</version>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>2.3.2</version>
+        <configuration>
+          <source>1.5</source>
+          <target>1.5</target>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSBrickClass.java b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSBrickClass.java
new file mode 100644
index 000000000..e633b8aae
--- /dev/null
+++ b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSBrickClass.java
@@ -0,0 +1,109 @@
+/**
+ *
+ * Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.glusterfs;
+
+import java.io.*;
+
+public class GlusterFSBrickClass {
+        String  host;
+        String  exportedFile;
+        long    start;
+        long    end;
+        boolean isChunked;
+        int     stripeSize;     // Stripe size in bytes
+        int     nrStripes;      // number of stripes
+        int     switchCount;    // for SR, DSR - number of replicas of each stripe
+                                // -1 for others
+
+        public GlusterFSBrickClass (String brick, long start, long len, boolean flag,
+                                    int stripeSize, int nrStripes, int switchCount)
+                throws IOException {
+                this.host = brick2host(brick);
+                this.exportedFile = brick2file(brick);
+                this.start = start;
+                this.end = start + len;
+                this.isChunked = flag;
+                this.stripeSize = stripeSize;
+                this.nrStripes = nrStripes;
+                this.switchCount = switchCount;
+        }
+
+        public boolean isChunked () {
+                return isChunked;
+        }
+
+        public String brickIsLocal(String hostname) {
+                String path = null;
+                File f = null;
+                if (host.equals(hostname))
+                        path = exportedFile;
+
+                return path;
+        }
+
+        public int[] getBrickNumberInTree(long start, int len) {
+                long end = len;
+                int startNodeInTree = ((int) (start / stripeSize)) % nrStripes;
+                int endNodeInTree = ((int) ((start + len) / stripeSize)) % nrStripes;
+
+                if (startNodeInTree != endNodeInTree) {
+                        end = (start - (start % stripeSize)) + stripeSize;
+                        end -= start;
+                }
+
+                return new int[] {startNodeInTree, endNodeInTree, (int) end};
+        }
+
+        public boolean brickHasFilePart(int nodeInTree, int nodeLoc) {
+                if (switchCount == -1)
+                        return (nodeInTree == nodeLoc);
+
+                nodeInTree *= switchCount;
+                for (int i = nodeInTree; i < (nodeInTree + switchCount); i++) {
+                        if (i == nodeLoc)
+                                return true;
+                }
+
+                return false;
+        }
+
+        public String brick2host (String brick)
+                throws IOException {
+                String[] hf = null;
+
+                hf = brick.split(":");
+                if (hf.length != 2)
+                        throw new IOException("Error getting hostname from brick");
+
+                return hf[0];
+        }
+
+        public String brick2file (String brick)
+                throws IOException {
+                String[] hf = null;
+
+                hf = brick.split(":");
+                if (hf.length != 2)
+                        throw new IOException("Error getting hostname from brick");
+
+                return hf[1];
+        }
+
+}
diff --git a/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSBrickRepl.java b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSBrickRepl.java
new file mode 100644
index 000000000..11454e636
--- /dev/null
+++ b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSBrickRepl.java
@@ -0,0 +1,52 @@
+/**
+ *
+ * Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.glusterfs;
+
+import java.io.*;
+
+public class GlusterFSBrickRepl {
+        private String[] replHost;
+        private long start;
+        private long len;
+        private int cnt;
+
+        GlusterFSBrickRepl(int replCount, long start, long len) {
+                this.replHost = new String[replCount];
+                this.start = start;
+                this.len = len;
+                this.cnt = 0;
+        }
+
+        public void addHost (String host) {
+                this.replHost[cnt++] = host;
+        }
+
+        public String[] getReplHosts () {
+                return this.replHost;
+        }
+
+        public long getStartLen () {
+                return this.start;
+        }
+
+        public long getOffLen () {
+                return this.len;
+        }
+}
diff --git a/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSXattr.java b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSXattr.java
new file mode 100644
index 000000000..455dda97e
--- /dev/null
+++ b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSXattr.java
@@ -0,0 +1,472 @@
+/**
+ *
+ * Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.glusterfs;
+
+import java.net.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.io.*;
+import java.util.HashMap;
+import java.util.TreeMap;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.hadoop.fs.BlockLocation;
+
+public class GlusterFSXattr {
+
+	public enum LAYOUT { D, S, R, DS, DR, SR, DSR }
+        public enum CMD { GET_HINTS, GET_REPLICATION, GET_BLOCK_SIZE, CHECK_FOR_QUICK_IO }
+
+        private static String hostname;
+
+        public GlusterFSXattr() { }
+
+        public static String brick2host (String brick)
+        throws IOException {
+                String[] hf = null;
+
+                hf = brick.split(":");
+                if (hf.length != 2) {
+                        System.out.println("brick not of format hostname:path");
+                        throw new IOException("Error getting hostname from brick");
+                }
+
+                return hf[0];
+        }
+
+        public static String brick2file (String brick)
+        throws IOException {
+                String[] hf = null;
+
+                hf = brick.split(":");
+                if (hf.length != 2) {
+                        System.out.println("brick not of format hostname:path");
+                        throw new IOException("Error getting hostname from brick");
+                }
+
+                return hf[1];
+        }
+
+        public static BlockLocation[] getPathInfo (String filename, long start, long len)
+                throws IOException {
+                HashMap<String, ArrayList<String>> vol = null;
+                HashMap<String, Integer> meta          = new HashMap<String, Integer>();
+
+                vol = execGetFattr(filename, meta, CMD.GET_HINTS);
+
+                return getHints(vol, meta, start, len, null);
+        }
+
+        public static long getBlockSize (String filename)
+                throws IOException {
+                HashMap<String, ArrayList<String>> vol = null;
+                HashMap<String, Integer> meta          = new HashMap<String, Integer>();
+
+                vol = execGetFattr(filename, meta, CMD.GET_BLOCK_SIZE);
+
+                if (!meta.containsKey("block-size"))
+                        return 0;
+
+                return (long) meta.get("block-size");
+
+        }
+
+        public static short getReplication (String filename)
+                throws IOException {
+                HashMap<String, ArrayList<String>> vol = null;
+                HashMap<String, Integer> meta          = new HashMap<String, Integer>();
+
+                vol = execGetFattr(filename, meta, CMD.GET_REPLICATION);
+
+                return (short) getReplicationFromLayout(vol, meta);
+
+        }
+
+        public static TreeMap<Integer, GlusterFSBrickClass> quickIOPossible (String filename, long start,
+                                                                             long len)
+                throws IOException {
+                String                   realpath      = null;
+                HashMap<String, ArrayList<String>> vol = null;
+                HashMap<String, Integer> meta          = new HashMap<String, Integer>();
+                TreeMap<Integer, GlusterFSBrickClass> hnts = new TreeMap<Integer, GlusterFSBrickClass>();
+
+                vol = execGetFattr(filename, meta, CMD.GET_HINTS);
+                getHints(vol, meta, start, len, hnts);
+
+                if (hnts.size() == 0)
+                        return null; // BOOM !!
+
+                // DEBUG - dump hnts here
+                return hnts;
+        }
+
+        public static HashMap<String, ArrayList<String>> execGetFattr (String filename,
+                                                                       HashMap<String, Integer> meta,
+                                                                       CMD cmd)
+                throws IOException {
+                Process        p              = null;
+                BufferedReader brInput        = null;
+                String         s              = null;
+                String         cmdOut         = null;
+                String         getfattrCmd    = null;
+                String         xlator         = null;
+                String         enclosingXl    = null;
+                String         enclosingXlVol = null;
+                String         key            = null;
+                String         layout         = "";
+                int            rcount         = 0;
+                int            scount         = 0;
+                int            dcount         = 0;
+                int            count          = 0;
+
+                HashMap<String, ArrayList<String>> vol = new HashMap<String, ArrayList<String>>();
+
+                getfattrCmd = "getfattr -m . -n trusted.glusterfs.pathinfo " + filename;
+
+                p = Runtime.getRuntime().exec(getfattrCmd);
+                brInput = new BufferedReader(new InputStreamReader(p.getInputStream()));
+
+                cmdOut = "";
+                while ( (s = brInput.readLine()) != null )
+                        cmdOut += s;
+
+                /**
+                 * TODO: Use a single regex for extracting posix paths as well
+                 * as xlator counts for layout matching.
+                 */
+
+                Pattern pattern = Pattern.compile("<(.*?)[:\\(](.*?)>");
+                Matcher matcher = pattern.matcher(cmdOut);
+
+                Pattern p_px = Pattern.compile(".*?:(.*)");
+                Matcher m_px;
+                String gibberish_path;
+
+                s = null;
+                while (matcher.find()) {
+                        xlator = matcher.group(1);
+                        if (xlator.equalsIgnoreCase("posix")) {
+                                if (enclosingXl.equalsIgnoreCase("replicate"))
+                                        count = rcount;
+                                else if (enclosingXl.equalsIgnoreCase("stripe"))
+                                        count = scount;
+                                else if (enclosingXl.equalsIgnoreCase("distribute"))
+                                        count = dcount;
+                                else
+                                        throw new IOException("Unknown Translator: " + enclosingXl);
+
+                                key = enclosingXl + "-" + count;
+
+                                if (vol.get(key) == null)
+                                        vol.put(key, new ArrayList<String>());
+
+                                gibberish_path = matcher.group(2);
+
+                                /* extract posix path from the gibberish string */
+                                m_px = p_px.matcher(gibberish_path);
+                                if (!m_px.find())
+                                        throw new IOException("Cannot extract posix path");
+
+                                vol.get(key).add(m_px.group(1));
+                                continue;
+                        }
+
+                        enclosingXl = xlator;
+                        enclosingXlVol = matcher.group(2);
+
+                        if (xlator.equalsIgnoreCase("replicate"))
+                                if (rcount++ != 0)
+                                        continue;
+
+                        if (xlator.equalsIgnoreCase("stripe")) {
+                                if (scount++ != 0)
+                                        continue;
+
+
+                                Pattern ps = Pattern.compile("\\[(\\d+)\\]");
+                                Matcher ms = ps.matcher(enclosingXlVol);
+
+                                if (ms.find()) {
+                                        if (((cmd == CMD.GET_BLOCK_SIZE) || (cmd == CMD.GET_HINTS))
+                                            && (meta != null))
+                                            meta.put("block-size", Integer.parseInt(ms.group(1)));
+                                } else
+                                        throw new IOException("Cannot get stripe size");
+                        }
+
+                        if (xlator.equalsIgnoreCase("distribute"))
+                                if (dcount++ != 0)
+                                        continue;
+
+                        layout += xlator.substring(0, 1);
+                }
+
+                if ((dcount == 0) && (scount == 0) && (rcount == 0))
+                        throw new IOException("Cannot get layout");
+
+                if (meta != null) {
+                        meta.put("dcount", dcount);
+                        meta.put("scount", scount);
+                        meta.put("rcount", rcount);
+                }
+
+                vol.put("layout", new ArrayList<String>(1));
+                vol.get("layout").add(layout);
+
+                return vol;
+        }
+
+        static BlockLocation[] getHints (HashMap<String, ArrayList<String>> vol,
+                                         HashMap<String, Integer> meta,
+                                         long start, long len,
+                                         TreeMap<Integer, GlusterFSBrickClass> hnts)
+                throws IOException {
+                String            brick         = null;
+                String            key           = null;
+                boolean           done          = false;
+                int               i             = 0;
+                int               counter       = 0;
+                int               stripeSize    = 0;
+                long              stripeStart   = 0;
+                long              stripeEnd     = 0;
+                int               nrAllocs      = 0;
+                int               allocCtr      = 0;
+                BlockLocation[] result          = null;
+                ArrayList<String> brickList     = null;
+                ArrayList<String> stripedBricks = null;
+                Iterator<String>  it            = null;
+
+                String[] blks = null;
+                GlusterFSBrickRepl[] repl = null;
+                int dcount, scount, rcount;
+
+                LAYOUT l = LAYOUT.valueOf(vol.get("layout").get(0));
+                dcount = meta.get("dcount");
+                scount = meta.get("scount");
+                rcount = meta.get("rcount");
+
+                switch (l) {
+                case D:
+                        key = "DISTRIBUTE-" + dcount;
+                        brick = vol.get(key).get(0);
+
+                        if (hnts == null) {
+                                result = new BlockLocation[1];
+                                result[0] = new BlockLocation(null, new String[] {brick2host(brick)}, start, len);
+                        } else
+                                hnts.put(0, new GlusterFSBrickClass(brick, start, len, false, -1, -1, -1));
+                        break;
+
+                case R:
+                case DR:
+                        /* just the name says it's striped - the volume isn't */
+                        stripedBricks = new ArrayList<String>();
+
+                        for (i = 1; i <= rcount; i++) {
+                                key = "REPLICATE-" + i;
+                                brickList = vol.get(key);
+                                it = brickList.iterator();
+                                while (it.hasNext()) {
+                                        stripedBricks.add(it.next());
+                                }
+                        }
+
+                        nrAllocs = stripedBricks.size();
+                        if (hnts == null) {
+                                result = new BlockLocation[1];
+                                blks = new String[nrAllocs];
+                        }
+
+                        for (i = 0; i < nrAllocs; i++) {
+                                if (hnts == null)
+                                        blks[i] = brick2host(stripedBricks.get(i));
+                                else
+                                        hnts.put(i, new GlusterFSBrickClass(stripedBricks.get(i), start, len, false, -1, -1, -1));
+                        }
+
+                        if (hnts == null)
+                                result[0] = new BlockLocation(null, blks, start, len);
+
+                        break;
+
+                case SR:
+                case DSR:
+                        int rsize = 0;
+                        ArrayList<ArrayList<String>> replicas = new ArrayList<ArrayList<String>>();
+
+                        stripedBricks = new ArrayList<String>();
+
+                        if (rcount == 0)
+                                throw new IOException("got replicated volume with replication count 0");
+
+                        for (i = 1; i <= rcount; i++) {
+                                key = "REPLICATE-" + i;
+                                brickList = vol.get(key);
+                                it = brickList.iterator();
+                                replicas.add(i - 1, new ArrayList<String>());
+                                while (it.hasNext()) {
+                                        replicas.get(i - 1).add(it.next());
+                                }
+                        }
+
+                        stripeSize = meta.get("block-size");
+
+                        nrAllocs = (int) (((len - start) / stripeSize) + 1);
+                        if (hnts == null) {
+                                result = new BlockLocation[nrAllocs];
+                                repl = new GlusterFSBrickRepl[nrAllocs];
+                        }
+
+                        // starting stripe position
+                        counter = (int) ((start / stripeSize) % rcount);
+                        stripeStart = start;
+
+                        key = null;
+                        int currAlloc = 0;
+                        boolean hntsDone = false;
+                        while ((stripeStart < len) && !done) {
+                                stripeEnd = (stripeStart - (stripeStart % stripeSize)) + stripeSize - 1;
+                                if (stripeEnd > start + len) {
+                                        stripeEnd = start + len - 1;
+                                        done = true;
+                                }
+
+                                rsize = replicas.get(counter).size();
+
+                                if (hnts == null)
+                                        repl[allocCtr] = new GlusterFSBrickRepl(rsize, stripeStart, (stripeEnd - stripeStart));
+
+                                for (i = 0; i < rsize; i++) {
+                                        brick = replicas.get(counter).get(i);
+                                        currAlloc = (allocCtr * rsize) + i;
+
+                                        if (hnts == null)
+                                                repl[allocCtr].addHost(brick2host(brick));
+                                        else
+                                                if (currAlloc <= (rsize * rcount) - 1) {
+                                                        hnts.put(currAlloc, new GlusterFSBrickClass(brick, stripeStart,
+                                                                                                    (stripeEnd - stripeStart),
+                                                                                                    true, stripeSize, rcount, rsize));
+                                                } else
+                                                        hntsDone = true;
+                                }
+
+                                if (hntsDone)
+                                        break;
+
+                                stripeStart = stripeEnd + 1;
+
+                                allocCtr++;
+                                counter++;
+
+                                if (counter >= replicas.size())
+                                        counter = 0;
+                        }
+
+                        if (hnts == null)
+                                for (int k = 0; k < nrAllocs; k++)
+                                        result[k] = new BlockLocation(null, repl[k].getReplHosts(), repl[k].getStartLen(), repl[k].getOffLen());
+
+                        break;
+
+                case S:
+                case DS:
+                        if (scount == 0)
+                                throw new IOException("got striped volume with stripe count 0");
+
+                        stripedBricks = new ArrayList<String>();
+                        stripeSize = meta.get("block-size");
+
+                        key = "STRIPE-" + scount;
+                        brickList = vol.get(key);
+                        it = brickList.iterator();
+                        while (it.hasNext()) {
+                                stripedBricks.add(it.next());
+                        }
+
+                        nrAllocs = (int) ((len - start) / stripeSize) + 1;
+                        if (hnts == null)
+                                result = new BlockLocation[nrAllocs];
+
+                        // starting stripe position
+                        counter = (int) ((start / stripeSize) % stripedBricks.size());
+                        stripeStart = start;
+
+                        key = null;
+                        while ((stripeStart < len) && !done) {
+                                brick = stripedBricks.get(counter);
+
+                                stripeEnd = (stripeStart - (stripeStart % stripeSize)) + stripeSize - 1;
+                                if (stripeEnd > start + len) {
+                                        stripeEnd = start + len - 1;
+                                        done = true;
+                                }
+
+                                if (hnts == null)
+                                        result[allocCtr] = new BlockLocation(null, new String[] {brick2host(brick)},
+                                                                             stripeStart, (stripeEnd - stripeStart));
+                                else
+                                        if (allocCtr <= stripedBricks.size()) {
+                                                hnts.put(allocCtr, new GlusterFSBrickClass(brick, stripeStart, (stripeEnd - stripeStart),
+                                                                                           true, stripeSize, stripedBricks.size(), -1));
+                                        } else
+                                                break;
+
+                                stripeStart = stripeEnd + 1;
+
+                                counter++;
+                                allocCtr++;
+
+                                if (counter >= stripedBricks.size())
+                                        counter = 0;
+                        }
+
+                        break;
+                }
+
+                return result;
+        }
+
+        /* TODO: use meta{dcount,scount,rcount} for checking */
+        public static int getReplicationFromLayout (HashMap<String, ArrayList<String>> vol,
+                                                    HashMap<String, Integer> meta)
+                throws IOException {
+                int replication = 0;
+                LAYOUT l = LAYOUT.valueOf(vol.get("layout").get(0));
+
+                switch (l) {
+                case D:
+                case S:
+                case DS:
+                        replication = 1;
+                        break;
+
+                case R:
+                case DR:
+                case SR:
+                case DSR:
+                        final String key = "REPLICATION-1";
+                        replication = vol.get(key).size();
+                }
+
+                return replication;
+        }
+}
diff --git a/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFUSEInputStream.java b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFUSEInputStream.java
new file mode 100644
index 000000000..e92237aee
--- /dev/null
+++ b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFUSEInputStream.java
@@ -0,0 +1,205 @@
+/**
+ *
+ * Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.glusterfs;
+
+import java.io.*;
+import java.util.TreeMap;
+
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem;
+
+
+public class GlusterFUSEInputStream extends FSInputStream {
+        File                                  f;
+        boolean                               lastActive;
+        long                                  pos;
+        boolean                               closed;
+        String                                thisHost;
+        RandomAccessFile                      fuseInputStream;
+        RandomAccessFile                      fsInputStream;
+        GlusterFSBrickClass                   thisBrick;
+        int                                   nodeLocation;
+        TreeMap<Integer, GlusterFSBrickClass> hnts;
+
+        public GlusterFUSEInputStream (File f, TreeMap<Integer, GlusterFSBrickClass> hnts,
+                                       String hostname) throws IOException {
+                this.f = f;
+                this.pos = 0;
+                this.closed = false;
+                this.hnts = hnts;
+                this.thisHost = hostname;
+                this.fsInputStream = null;
+                this.fuseInputStream = new RandomAccessFile(f.getPath(), "r");
+
+                this.lastActive = true; // true == FUSE, false == backed file
+
+                String directFilePath = null;
+                if (this.hnts != null) {
+                        directFilePath = findLocalFile(f.getPath(), this.hnts);
+                        if (directFilePath != null) {
+                                this.fsInputStream = new RandomAccessFile(directFilePath, "r");
+                                this.lastActive = !this.lastActive;
+                        }
+                }
+        }
+
+        public String findLocalFile (String path, TreeMap<Integer, GlusterFSBrickClass> hnts) {
+                int i = 0;
+                String actFilePath = null;
+                GlusterFSBrickClass gfsBrick = null;
+
+                gfsBrick = hnts.get(0);
+
+                /* do a linear search for the matching host not worrying
+                   about file stripes */
+                for (i = 0; i < hnts.size(); i++) {
+                        gfsBrick = hnts.get(i);
+                        actFilePath = gfsBrick.brickIsLocal(this.thisHost);
+                        if (actFilePath != null) {
+                                this.thisBrick = gfsBrick;
+                                this.nodeLocation = i;
+                                break;
+                        }
+                }
+
+                return actFilePath;
+        }
+
+        public long getPos () throws IOException {
+                return pos;
+        }
+
+        public synchronized int available () throws IOException {
+                return (int) ((f.length()) - getPos());
+        }
+
+        public void seek (long pos) throws IOException {
+                fuseInputStream.seek(pos);
+                if (fsInputStream != null)
+                        fsInputStream.seek(pos);
+        }
+
+        public boolean seekToNewSource (long pos) throws IOException {
+                return false;
+        }
+
+        public RandomAccessFile chooseStream (long start, int[] nlen)
+                throws IOException {
+                GlusterFSBrickClass gfsBrick = null;
+                RandomAccessFile in = fuseInputStream;
+                boolean oldActiveStream = lastActive;
+                lastActive = true;
+
+                if ((hnts != null) && (fsInputStream != null)) {
+                        gfsBrick = hnts.get(0);
+                        if (!gfsBrick.isChunked()) {
+                                in = fsInputStream;
+                                lastActive = false;
+                        } else {
+                                // find the current location in the tree and the amount of data it can serve
+                                int[] nodeInTree = thisBrick.getBrickNumberInTree(start, nlen[0]);
+
+                                // does this node hold the byte ranges we have been requested for ?
+                                if ((nodeInTree[2] != 0) && thisBrick.brickHasFilePart(nodeInTree[0], nodeLocation)) {
+                                        in = fsInputStream;
+                                        nlen[0] = nodeInTree[2]; // the amount of data that can be read from the stripe
+                                        lastActive = false;
+                                }
+                        }
+                }
+
+                return in;
+        }
+
+        public synchronized int read () throws IOException {
+                int byteRead = 0;
+                RandomAccessFile in = null;
+
+                if (closed)
+                        throw new IOException("Stream Closed.");
+
+                int[] nlen = { 1 };
+
+                in = chooseStream(getPos(), nlen);
+
+                byteRead = in.read();
+                if (byteRead >= 0) {
+                        pos++;
+                        syncStreams(1);
+                }
+
+                return byteRead;
+        }
+
+        public synchronized int read (byte buff[], int off, int len) throws
+                IOException {
+                int result = 0;
+                RandomAccessFile in = null;
+
+                if (closed)
+                        throw new IOException("Stream Closed.");
+
+                int[] nlen = {len}; // hack to make len mutable
+                in = chooseStream(pos, nlen);
+
+                result = in.read(buff, off, nlen[0]);
+                if (result > 0) {
+                        pos += result;
+                        syncStreams(result);
+                }
+
+                return result;
+        }
+
+        /**
+         * TODO: use seek() insted of skipBytes(); skipBytes does I/O
+         */
+        public void syncStreams (int bytes) throws IOException {
+                if ((hnts != null) && (hnts.get(0).isChunked()) && (fsInputStream != null))
+                        if (!this.lastActive)
+                                fuseInputStream.skipBytes(bytes);
+                        else
+                                fsInputStream.skipBytes(bytes);
+        }
+
+        public synchronized void close () throws IOException {
+                if (closed)
+                        throw new IOException("Stream closed.");
+
+                super.close();
+                if (fsInputStream != null)
+                        fsInputStream.close();
+                fuseInputStream.close();
+
+                closed = true;
+        }
+
+        // Not supported - mark () and reset ()
+
+        public boolean markSupported () {
+                return false;
+        }
+
+        public void mark (int readLimit) {}
+
+        public void reset () throws IOException {
+                throw new IOException("Mark/Reset not supported.");
+        }
+}
diff --git a/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFUSEOutputStream.java b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFUSEOutputStream.java
new file mode 100644
index 000000000..5192a0a56
--- /dev/null
+++ b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFUSEOutputStream.java
@@ -0,0 +1,86 @@
+/**
+ *
+ * Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.glusterfs;
+
+import java.io.*;
+
+import org.apache.hadoop.fs.FSOutputSummer;
+import org.apache.hadoop.fs.FileSystem;
+
+public class GlusterFUSEOutputStream extends OutputStream {
+        File         f;
+        long         pos;
+        boolean      closed;
+        OutputStream fuseOutputStream;
+
+        public GlusterFUSEOutputStream (String file, boolean append) throws
+                IOException {
+                this.f = new File(file); /* not needed ? */
+                this.pos = 0;
+                this.fuseOutputStream = new FileOutputStream(file, append);
+                this.closed = false;
+        }
+
+        public long getPos () throws IOException {
+                return pos;
+        }
+
+        public void write (int v) throws IOException {
+                if (closed)
+                        throw new IOException("Stream closed.");
+
+                byte[] b = new byte[1];
+                b[0] = (byte) v;
+
+                write(b, 0, 1);
+        }
+
+        public void write (byte b[]) throws IOException {
+                if (closed)
+                        throw new IOException("Stream closed.");
+
+                fuseOutputStream.write(b, 0, b.length);
+                pos += (long) b.length;
+        }
+
+        public void write (byte b[], int off, int len) throws IOException {
+                if (closed)
+                        throw new IOException("Stream closed.");
+
+                fuseOutputStream.write(b, off, len);
+                pos += (long) len;
+        }
+
+        public void flush () throws IOException {
+                if (closed)
+                        throw new IOException("Stream closed.");
+
+                fuseOutputStream.flush();
+        }
+
+        public void close () throws IOException {
+                if (closed)
+                        throw new IOException("Stream closed.");
+
+                flush();
+                fuseOutputStream.close();
+                closed = true;
+        }
+}
diff --git a/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java
new file mode 100644
index 000000000..b0501cced
--- /dev/null
+++ b/glusterfs-hadoop/0.20.2/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java
@@ -0,0 +1,492 @@
+/**
+ *
+ * Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+/**
+ * Implements the Hadoop FileSystem Interface to allow applications to store
+ * files on GlusterFS and run Map/Reduce jobs on the data.
+ */
+
+package org.apache.hadoop.fs.glusterfs;
+
+import java.io.*;
+import java.net.*;
+
+import java.util.regex.*;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
+
+import java.util.TreeMap;
+
+/*
+ * This package provides interface for hadoop jobs (incl. Map/Reduce)
+ * to access files in GlusterFS backed file system via FUSE mount
+ */
+public class GlusterFileSystem extends FileSystem {
+
+        private FileSystem glusterFs    = null;
+        private URI        uri          = null;
+        private Path       workingDir   = null;
+        private String     glusterMount = null;
+        private boolean    mounted      = false;
+
+        /* for quick IO */
+        private boolean quickSlaveIO = false;
+
+        /* extended attribute class */
+        private GlusterFSXattr xattr = null;
+
+        /* hostname of this machine */
+        private static String hostname;
+
+        public GlusterFileSystem () {
+
+        }
+
+        public URI getUri () {
+                return uri;
+        }
+
+        public boolean FUSEMount (String volname, String server, String mount)
+                throws IOException, InterruptedException  {
+                boolean        ret      = true;
+                int            retVal   = 0;
+                Process        p        = null;
+                String         s        = null;
+                String         mountCmd = null;
+
+                mountCmd = "mount -t glusterfs " + server + ":" + "/" + volname + " " + mount;
+
+                try {
+                        p = Runtime.getRuntime().exec(mountCmd);
+
+                        retVal = p.waitFor();
+                        if (retVal != 0)
+                                ret = false;
+
+                } catch (IOException e) {
+                        System.out.println ("Problem mounting FUSE mount on: " + mount);
+                        e.printStackTrace();
+                        System.exit(-1);
+                }
+
+                return ret;
+        }
+
+        public void initialize (URI uri, Configuration conf) throws IOException {
+                boolean ret             = false;
+                String  volName         = null;
+                String  remoteGFSServer = null;
+                String  needQuickRead   = null;
+
+                if (this.mounted)
+                        return;
+
+                System.out.println("Initializing GlusterFS");
+
+                try {
+                        volName = conf.get("fs.glusterfs.volname", "");
+                        glusterMount = conf.get("fs.glusterfs.mount", "");
+                        remoteGFSServer = conf.get("fs.glusterfs.server", "");
+                        needQuickRead = conf.get("quick.slave.io", "");
+
+                        /*
+                         * bail out if we do not have enough information to do a FUSE
+                         * mount
+                         */
+                        if ( (volName.length() == 0) || (remoteGFSServer.length() == 0) ||
+                             (glusterMount.length() == 0) )
+                                System.exit (-1);
+
+                        ret = FUSEMount(volName, remoteGFSServer, glusterMount);
+                        if (!ret) {
+                                System.out.println("Failed to initialize GlusterFS");
+                                System.exit(-1);
+                        }
+
+                        if ((needQuickRead.length() != 0)
+                            && (needQuickRead.equalsIgnoreCase("yes")
+                                || needQuickRead.equalsIgnoreCase("on")
+                                || needQuickRead.equals("1")))
+                                this.quickSlaveIO = true;
+
+                        this.mounted = true;
+                        this.glusterFs = FileSystem.getLocal(conf);
+                        this.workingDir = new Path(glusterMount);
+                        this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
+
+                        this.xattr = new GlusterFSXattr();
+
+                        InetAddress addr = InetAddress.getLocalHost();
+                        this.hostname = addr.getHostName();
+
+                        setConf(conf);
+
+                } catch (Exception e) {
+                        e.printStackTrace();
+                        System.out.println("Unable to initialize GlusterFS");
+                        System.exit(-1);
+                }
+        }
+
+        @Deprecated
+        public String getName () {
+                return getUri().toString();
+        }
+
+        public Path getWorkingDirectory () {
+                return this.workingDir;
+        }
+
+        public Path getHomeDirectory () {
+                return this.workingDir;
+        }
+
+        public Path makeAbsolute (Path path) {
+                String pth = path.toUri().getPath();
+                if (pth.startsWith(workingDir.toUri().getPath())) {
+                        return path;
+                }
+
+                return new Path(workingDir + "/" + pth);
+        }
+
+        public void setWorkingDirectory (Path dir) {
+                this.workingDir = makeAbsolute(dir);
+        }
+
+        public boolean exists (Path path) throws IOException {
+                Path absolute = makeAbsolute(path);
+                File f = new File(absolute.toUri().getPath());
+
+                return f.exists();
+        }
+
+        public boolean mkdirs (Path path, FsPermission permission
+                               ) throws IOException {
+                boolean created  = false;
+                Path    absolute = makeAbsolute(path);
+                File    f        = new File(absolute.toUri().getPath());
+
+                if (f.exists()) {
+                        System.out.println("Directory " + f.getPath() + " already exist");
+                        return true;
+                }
+
+                return f.mkdirs();
+        }
+
+        @Deprecated
+        public boolean isDirectory (Path path) throws IOException {
+                Path absolute = makeAbsolute(path);
+                File f = new File(absolute.toUri().getPath());
+
+                return f.isDirectory();
+        }
+
+        public boolean isFile (Path path) throws IOException {
+                return !isDirectory(path);
+        }
+
+        public Path[] listPaths (Path path) throws IOException {
+                Path   absolute   = makeAbsolute(path);
+                File   f          = new File (absolute.toUri().getPath());
+                String relPath    = path.toUri().getPath();
+                String[] fileList = null;
+                Path[] filePath   = null;
+                int    fileCnt    = 0;
+
+                fileList = f.list();
+
+                filePath = new Path[fileList.length];
+
+                for (; fileCnt < fileList.length; fileCnt++) {
+                        filePath[fileCnt] = new Path(relPath + "/" + fileList[fileCnt]);
+                }
+
+                return filePath;
+        }
+
+        public FileStatus[] listStatus (Path path) throws IOException {
+                int    fileCnt          = 0;
+                Path   absolute         = makeAbsolute(path);
+                String relpath          = path.toUri().getPath();
+                String[] strFileList    = null;
+                FileStatus[] fileStatus = null;
+                File   f                = new File(absolute.toUri().getPath());
+
+                if (!f.exists()) {
+                        return null;
+                }
+
+                if (f.isFile())
+                        return new FileStatus[] {
+                                getFileStatus(path)
+                        };
+
+                if (relpath.charAt(relpath.length() - 1) != '/')
+                        relpath += "/";
+
+                strFileList = f.list();
+
+                fileStatus = new FileStatus[strFileList.length];
+
+                for (; fileCnt < strFileList.length; fileCnt++) {
+                        fileStatus[fileCnt] = getFileStatusFromFileString(relpath + strFileList[fileCnt]);
+                }
+
+                return fileStatus;
+        }
+
+        public FileStatus getFileStatusFromFileString (String path)
+                throws IOException {
+                Path nPath = new Path(path);
+                return getFileStatus(nPath);
+        }
+
+        public FileStatus getFileStatus (Path path) throws IOException {
+                Path absolute = makeAbsolute(path);
+                File f = new File(absolute.toUri().getPath());
+
+                if (!f.exists ())
+                        throw new FileNotFoundException("File " + f.getPath() + " does not exist.");
+
+                if (f.isDirectory ())
+                        return new FileStatus(0, true, 1, 0, f.lastModified(), path.makeQualified(this));
+                else
+                        return new FileStatus(f.length(), false, 0, getDefaultBlockSize(),
+                                              f.lastModified(), path.makeQualified(this));
+
+        }
+
+        /*
+         * creates a new file in glusterfs namespace. internally the file
+         * descriptor is an instance of OutputStream class.
+         */
+        public FSDataOutputStream create (Path path, FsPermission permission,
+                                          boolean overwrite, int bufferSize,
+                                          short replication, long blockSize,
+                                          Progressable progress)
+        throws IOException {
+                Path               absolute          = makeAbsolute(path);
+                Path               parent            = null;
+                File               f                 = null;
+                File               fParent           = null;
+                FSDataOutputStream glusterFileStream = null;
+
+                f = new File(absolute.toUri().getPath());
+
+                if (f.exists ()) {
+                        if (overwrite)
+                                f.delete ();
+                        else
+                                throw new IOException(f.getPath() + " already exist");
+                }
+
+                parent = path.getParent();
+                fParent = new File ((makeAbsolute(parent)).toUri().getPath());
+                if ((parent != null) && (fParent != null) && (!fParent.exists()))
+                        if (!fParent.mkdirs())
+                                throw new IOException("cannot create parent directory: " + fParent.getPath());
+
+                glusterFileStream = new FSDataOutputStream(new GlusterFUSEOutputStream
+                                                           (f.getPath(), false));
+
+                return glusterFileStream;
+        }
+
+        /*
+         * open the file in read mode (internally the file descriptor is an
+         * instance of InputStream class).
+         *
+         * if quick read mode is set then read the file by by-passing FUSE
+         * if we are on same slave where the file exist
+         */
+        public FSDataInputStream open (Path path) throws IOException {
+                Path              absolute          = makeAbsolute(path);
+                File              f                 = new File(absolute.toUri().getPath());
+                FSDataInputStream glusterFileStream = null;
+                TreeMap<Integer, GlusterFSBrickClass> hnts = null;
+
+                if (!f.exists())
+                        throw new IOException("File " + f.getPath() + " does not exist.");
+
+                if (quickSlaveIO)
+                        hnts = xattr.quickIOPossible(f.getPath(), 0, f.length());
+
+                glusterFileStream = new FSDataInputStream(new GlusterFUSEInputStream(f, hnts, hostname));
+                return glusterFileStream;
+        }
+
+        public FSDataInputStream open (Path path, int bufferSize) throws IOException {
+                return open(path);
+        }
+
+        public FSDataOutputStream append (Path f, int bufferSize, Progressable progress)
+                throws IOException {
+                throw new IOException ("append not supported (as yet).");
+        }
+
+        public boolean rename (Path src, Path dst) throws IOException {
+                Path absoluteSrc = makeAbsolute(src);
+                Path absoluteDst = makeAbsolute(dst);
+
+                File fSrc = new File(absoluteSrc.toUri().getPath());
+                File fDst = new File(absoluteDst.toUri().getPath());
+
+                if (fDst.isDirectory()) {
+                        fDst = null;
+                        String newPath = absoluteDst.toUri().getPath() + "/" + fSrc.getName();
+                        fDst = new File(newPath);
+                }
+                return fSrc.renameTo(fDst);
+        }
+
+        @Deprecated
+        public boolean delete (Path path) throws IOException {
+                return delete(path, true);
+        }
+
+        public boolean delete (Path path, boolean recursive) throws IOException {
+                Path absolute = makeAbsolute(path);
+                File f = new File(absolute.toUri().getPath());
+
+                if (f.isFile())
+                        return f.delete();
+
+                FileStatus[] dirEntries = listStatus(absolute);
+                if ((!recursive) && (dirEntries != null) && (dirEntries.length != 0))
+                        throw new IOException ("Directory " + path.toString() + " is not empty");
+
+                if (dirEntries != null)
+                        for (int i = 0; i < dirEntries.length; i++)
+                                delete(new Path(absolute, dirEntries[i].getPath()), recursive);
+
+                return f.delete();
+        }
+
+        @Deprecated
+        public long getLength (Path path) throws IOException {
+                Path absolute = makeAbsolute(path);
+                File f = new File(absolute.toUri().getPath());
+
+                if (!f.exists())
+                        throw new IOException(f.getPath() + " does not exist.");
+
+                return f.length();
+        }
+
+        @Deprecated
+        public short getReplication (Path path) throws IOException {
+                Path absolute = makeAbsolute(path);
+                File f = new File(absolute.toUri().getPath());
+
+                if (!f.exists())
+                        throw new IOException(f.getPath() + " does not exist.");
+
+                return xattr.getReplication(f.getPath());
+        }
+
+        public short getDefaultReplication (Path path) throws IOException {
+                return getReplication(path);
+        }
+
+        public boolean setReplication (Path path, short replication)
+                throws IOException {
+                return true;
+        }
+
+        public long getBlockSize (Path path) throws IOException {
+                long blkSz;
+                Path absolute = makeAbsolute(path);
+                File f = new File(absolute.toUri().getPath());
+
+                blkSz = xattr.getBlockSize(f.getPath());
+                if (blkSz == 0)
+                        blkSz = getLength(path);
+
+                return blkSz;
+        }
+
+        public long getDefaultBlockSize () {
+                return 1 << 26; /* default's from hdfs, kfs */
+        }
+
+        @Deprecated
+        public void lock (Path path, boolean shared) throws IOException {
+        }
+
+        @Deprecated
+        public void release (Path path) throws IOException {
+        }
+
+        public BlockLocation[] getFileBlockLocations (FileStatus file, long start, long len)
+                throws IOException {
+
+                Path absolute          = makeAbsolute(file.getPath());
+                File f                 = new File(absolute.toUri().getPath());
+                BlockLocation[] result = null;
+
+                if (file == null)
+                        return null;
+
+                result = xattr.getPathInfo(f.getPath(), start, len);
+                if (result == null) {
+                        System.out.println("Problem getting destination host for file "
+                                           + f.getPath());
+                        return null;
+                }
+
+                return result;
+        }
+
+        // getFileBlockLocations (FileStatus, long, long) is called by hadoop
+        public BlockLocation[] getFileBlockLocations (Path p, long start, long len)
+                throws IOException {
+                return null;
+        }
+
+        public void copyFromLocalFile (boolean delSrc, Path src, Path dst)
+                throws IOException {
+                FileUtil.copy(glusterFs, src, this, dst, delSrc, getConf());
+        }
+
+        public void copyToLocalFile (boolean delSrc, Path src, Path dst)
+                throws IOException {
+                FileUtil.copy(this, src, glusterFs, dst, delSrc, getConf());
+        }
+
+        public Path startLocalOutput (Path fsOutputFile, Path tmpLocalFile)
+                throws IOException {
+                return tmpLocalFile;
+        }
+
+        public void completeLocalOutput (Path fsOutputFile, Path tmpLocalFile)
+                throws IOException {
+                moveFromLocalFile(tmpLocalFile, fsOutputFile);
+        }
+}
diff --git a/glusterfs-hadoop/0.20.2/src/test/java/org/apache/hadoop/fs/glusterfs/AppTest.java b/glusterfs-hadoop/0.20.2/src/test/java/org/apache/hadoop/fs/glusterfs/AppTest.java
new file mode 100644
index 000000000..21e188c52
--- /dev/null
+++ b/glusterfs-hadoop/0.20.2/src/test/java/org/apache/hadoop/fs/glusterfs/AppTest.java
@@ -0,0 +1,38 @@
+package org.apache.hadoop.fs.glusterfs;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Unit test for simple App.
+ */
+public class AppTest
+    extends TestCase
+{
+    /**
+     * Create the test case
+     *
+     * @param testName name of the test case
+     */
+    public AppTest( String testName )
+    {
+        super( testName );
+    }
+
+    /**
+     * @return the suite of tests being tested
+     */
+    public static Test suite()
+    {
+        return new TestSuite( AppTest.class );
+    }
+
+    /**
+     * Rigourous Test :-)
+     */
+    public void testApp()
+    {
+        assertTrue( true );
+    }
+}
diff --git a/glusterfs-hadoop/0.20.2/tools/build-deploy-jar.py b/glusterfs-hadoop/0.20.2/tools/build-deploy-jar.py
new file mode 100755
index 000000000..c20e53b39
--- /dev/null
+++ b/glusterfs-hadoop/0.20.2/tools/build-deploy-jar.py
@@ -0,0 +1,212 @@
+#!/usr/bin/python
+
+##
+ #
+ # Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+ # This file is part of GlusterFS.
+ #
+ # Licensed under the Apache License, Version 2.0
+ # (the "License"); you may not use this file except in compliance with
+ # the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ # implied. See the License for the specific language governing
+ # permissions and limitations under the License.
+ #
+ ##
+
+import getopt
+import glob
+import sys, os
+import shutil
+import subprocess, shlex
+
+def usage():
+    print "usage: python build-deploy-jar.py [-b/--build] -d/--dir <hadoop-home> [-c/--core] [-m/--mapred] [-h/--henv]"
+
+def addSlash(s):
+    if not (s[-1] == '/'):
+        s = s + '/'
+
+    return s
+
+def whereis(program):
+    abspath = None
+    for path in (os.environ.get('PATH', '')).split(':'):
+        abspath = os.path.join(path, program)
+        if os.path.exists(abspath) and not os.path.isdir(abspath):
+            return abspath
+
+    return None
+
+def getLatestJar(targetdir):
+    glusterfsJar = glob.glob(targetdir + "*.jar")
+    if len(glusterfsJar) == 0:
+        print "No GlusterFS jar file found in %s ... exiting" % (targetdir)
+        return None
+
+    # pick up the latest jar file - just in case ...
+    stat = latestJar = None
+    ctime = 0
+
+    for jar in glusterfsJar:
+        stat = os.stat(jar)
+        if stat.st_ctime > ctime:
+           latestJar = jar
+           ctime = stat.st_ctime
+
+    return latestJar
+
+# build the glusterfs hadoop plugin using maven
+def build_jar(targetdir):
+    location = whereis('mvn')
+
+    if location == None:
+        print "Cannot find maven to build glusterfs hadoop jar"
+        print "please install maven or if it's already installed then fix your PATH environ"
+        return None
+
+    # do a clean packaging
+    if os.path.exists(targetdir) and os.path.isdir(targetdir):
+        print "Cleaning up directories ... [ " + targetdir + " ]"
+        shutil.rmtree(targetdir)
+
+    print "Building glusterfs jar ..."
+    process = subprocess.Popen(['package'], shell=True,
+                               executable=location, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    process.wait()
+    if not process.returncode == 0:
+        print "Building glusterfs jar failed ... exiting"
+        return None
+
+    latestJar = getLatestJar(targetdir)
+    return latestJar
+
+def rcopy(f, host, libdir):
+    print "   * doing remote copy to host %s" % (host)
+    scpCmd = "scp %s %s:%s" % (f, host, libdir)
+
+    os.system(scpCmd);
+
+def deployInSlave(f, confdir, libdir, cc, cm, he):
+    slavefile = confdir + "slaves"
+
+    ccFile = confdir + "core-site.xml"
+    cmFile = confdir + "mapred-site.xml"
+    heFile = confdir + "hadoop-env.sh"
+
+    sf = open(slavefile, 'r')
+    for host in sf:
+        host = host.rstrip('\n')
+        print "  >>> Deploying %s on %s ..." % (os.path.basename(f), host)
+        rcopy(f, host, libdir)
+
+        if cc:
+            print "  >>> Deploying [%s] on %s ..." % (os.path.basename(ccFile), host)
+            rcopy(ccFile, host, confdir)
+
+        if cm:
+            print "  >>> Deploying [%s] on %s ..." % (os.path.basename(cmFile), host)
+            rcopy(cmFile, host, confdir)
+
+        if he:
+            print "  >>> Deploying [%s] on %s ..." % (os.path.basename(heFile), host)
+            rcopy(heFile, host, confdir);
+
+        print "<<< Done\n"
+
+    sf.close()
+
+def deployInMaster(f, confdir, libdir):
+    import socket
+    masterfile = confdir + "masters"
+
+    mf = open(masterfile, 'r')
+    for host in mf:
+        host = host.rstrip('\n')
+        print "  >>> Deploying %s on %s ..." % (os.path.basename(f), host)
+        h = host
+        try:
+            socket.inet_aton(host)
+            h = socket.getfqdn(host)
+        except socket.error:
+            pass
+
+        if h == socket.gethostname() or h == 'localhost':
+            # local cp
+            print "   * doing local copy"
+            shutil.copy(f, libdir)
+        else:
+            # scp the file
+            rcopy(f, h, libdir)
+
+        print "<<< Done\n"
+
+    mf.close()
+
+if __name__ == '__main__':
+    opt = args = []
+    try:
+        opt, args = getopt.getopt(sys.argv[1:], "bd:cmh", ["build", "dir=", "core", "mapred", "henv"]);
+    except getopt.GetoptError, err:
+        print str(err)
+        usage()
+        sys.exit(1)
+
+    needbuild = hadoop_dir = copyCore = copyMapred = copyHadoopEnv = None
+
+    for k, v in opt:
+        if k in ("-b", "--build"):
+            needbuild = True
+        elif k in ("-d", "--dir"):
+            hadoop_dir = v
+        elif k in ("-c", "--core"):
+            copyCore = True
+        elif k in ("-m", "--mapred"):
+            copyMapred = True
+        elif k in ("-h", "--henv"):
+            copyHadoopEnv = True
+        else:
+            pass
+
+    if hadoop_dir == None:
+        print 'hadoop directory missing'
+        usage()
+        sys.exit(1)
+
+    os.chdir(os.path.dirname(sys.argv[0]) + '/..')
+    targetdir = './target/'
+
+    if needbuild:
+        jar = build_jar(targetdir)
+        if jar == None:
+            sys.exit(1)
+    else:
+        jar = getLatestJar(targetdir)
+        if jar == None:
+            print "Maybe you want to build it ? with -b option"
+            sys.exit(1)
+
+    print ""
+    print "*** Deploying %s *** " % (jar)
+
+    # copy jar to local hadoop distribution (master)
+    hadoop_home = addSlash(hadoop_dir)
+    if not (os.path.exists(hadoop_home) and os.path.isdir(hadoop_home)):
+        print "path " + hadoop_home + " does not exist or is not adiretory";
+        sys.exit(1);
+
+    hadoop_conf = hadoop_home + "conf/"
+    hadoop_lib = hadoop_home + "lib/"
+
+    print " >>> Scanning hadoop master file for host(s) to deploy"
+    deployInMaster(jar, hadoop_conf, hadoop_lib)
+
+    print ""
+    print " >>> Scanning hadoop slave file for host(s) to deploy"
+    deployInSlave(jar, hadoop_conf, hadoop_lib, copyCore, copyMapred, copyHadoopEnv)
diff --git a/glusterfs-hadoop/COPYING b/glusterfs-hadoop/COPYING
new file mode 100644
index 000000000..d64569567
--- /dev/null
+++ b/glusterfs-hadoop/COPYING
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/glusterfs-hadoop/README b/glusterfs-hadoop/README
new file mode 100644
index 000000000..3026f11c0
--- /dev/null
+++ b/glusterfs-hadoop/README
@@ -0,0 +1,182 @@
+GlusterFS Hadoop Plugin
+=======================
+
+INTRODUCTION
+------------
+
+This document describes how to use GlusterFS (http://www.gluster.org/) as a backing store with Hadoop.
+
+
+REQUIREMENTS
+------------
+
+* Supported OS is GNU/Linux
+* GlusterFS and Hadoop installed on all machines in the cluster
+* Java Runtime Environment (JRE)
+* Maven (needed if you are building the plugin from source)
+* JDK (needed if you are building the plugin from source)
+
+NOTE: Plugin relies on two *nix command line utilities to function properly. They are:
+
+* mount: Used to mount GlusterFS volumes.
+* getfattr: Used to fetch Extended Attributes of a file
+
+Make sure they are installed on all hosts in the cluster and their locations are in $PATH
+environment variable.
+
+
+INSTALLATION
+------------
+
+** NOTE: Example below is for Hadoop version 0.20.2 ($GLUSTER_HOME/hdfs/0.20.2) **
+
+* Building the plugin from source [Maven (http://maven.apache.org/) and JDK is required to build the plugin]
+
+  Change to glusterfs-hadoop directory in the GlusterFS source tree and build the plugin.
+
+  # cd $GLUSTER_HOME/hdfs/0.20.2
+  # mvn package
+
+  On a successful build the plugin will be present in the `target` directory.
+  (NOTE: version number will be a part of the plugin)
+
+  # ls target/
+  classes  glusterfs-0.20.2-0.1.jar  maven-archiver  surefire-reports  test-classes
+               ^^^^^^^^^^^^^^^^^^
+
+  Copy the plugin to lib/ directory in your $HADOOP_HOME dir.
+
+  # cp target/glusterfs-0.20.2-0.1.jar $HADOOP_HOME/lib
+
+  Copy the sample configuration file that ships with this source (conf/core-site.xml) to conf
+  directory in your $HADOOP_HOME dir.
+
+  # cp conf/core-site.xml $HADOOP_HOME/conf
+
+* Installing the plugin from RPM
+
+  See the plugin documentation for installing from RPM.
+
+
+CLUSTER INSTALLATION
+--------------------
+
+  In case it is tedious to do the above steps(s) on all hosts in the cluster; use the build-and-deploy.py script to
+  build the plugin in one place and deploy it (along with the configuration file on all other hosts).
+
+  This should be run on the host which is that hadoop master [Job Tracker].
+
+* STEPS (You would have done Step 1 and 2 anyway while deploying Hadoop)
+
+  1. Edit conf/slaves file in your hadoop distribution; one line for each slave.
+  2. Setup password-less ssh b/w hadoop master and slave(s).
+  3. Edit conf/core-site.xml with all glusterfs related configurations (see CONFIGURATION)
+  4. Run the following
+     # cd $GLUSTER_HOME/hdfs/0.20.2/tools
+     # python ./build-and-deploy.py -b -d /path/to/hadoop/home -c
+
+     This will build the plugin and copy it (and the config file) to all slaves (mentioned in $HADOOP_HOME/conf/slaves).
+
+   Script options:
+     -b : build the plugin
+     -d : location of hadoop directory
+     -c : deploy core-site.xml
+     -m : deploy mapred-site.xml
+     -h : deploy hadoop-env.sh
+
+
+CONFIGURATION
+-------------
+
+  All plugin configuration is done in a single XML file (core-site.xml) with <name><value> tags in each <property>
+  block.
+
+  Brief explanation of the tunables and the values they accept (change them where-ever needed) are mentioned below
+
+  name:  fs.glusterfs.impl
+  value: org.apache.hadoop.fs.glusterfs.GlusterFileSystem
+
+         The default FileSystem API to use (there is little reason to modify this).
+
+  name:  fs.default.name
+  value: glusterfs://server:port
+
+         The default name that hadoop uses to represent file as a URI (typically a server:port tuple). Use any host
+         in the cluster as the server and any port number. This option has to be in server:port format for hadoop
+         to create file URI; but is not used by plugin.
+
+  name:  fs.glusterfs.volname
+  value: volume-dist-rep
+
+         The volume to mount.
+
+
+  name:  fs.glusterfs.mount
+  value: /mnt/glusterfs
+
+         This is the directory that the plugin will use to mount (FUSE mount) the volume.
+
+  name:  fs.glusterfs.server
+  value: 192.168.1.36, hackme.zugzug.org
+
+         To mount a volume the plugin needs to know the hostname or the IP of a GlusterFS server in the cluster.
+         Mention it here.
+
+  name:  quick.slave.io
+  value: [On/Off], [Yes/No], [1/0]
+
+         NOTE: This option is not tested as of now.
+
+         This is a performance tunable option. Hadoop schedules jobs to hosts that contain the file data part. The job
+         then does I/O on the file (via FUSE in case of GlusterFS). When this option is set, the plugin will try to
+         do I/O directly from the backed filesystem (ext3, ext4 etc..) the file resides on. Hence read performance
+         will improve and job would run faster.
+
+
+USAGE
+-----
+
+  Once configured, start Hadoop Map/Reduce daemons
+
+  # cd $HADOOP_HOME
+  # ./bin/start-mapred.sh
+
+  If the map/reduce job/task trackers are up, all I/O will be done to GlusterFS.
+
+
+FOR HACKERS
+-----------
+
+* Source Layout
+
+** version specific: hdfs/<version> **
+./src
+./src/main
+./src/main/java
+./src/main/java/org
+./src/main/java/org/apache
+./src/main/java/org/apache/hadoop
+./src/main/java/org/apache/hadoop/fs
+./src/main/java/org/apache/hadoop/fs/glusterfs
+./src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSBrickClass.java
+./src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSXattr.java            <--- Fetch/Parse Extended Attributes of a file
+./src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFUSEInputStream.java    <--- Input Stream (instantiated during open() calls; quick read from backed FS)
+./src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSBrickRepl.java
+./src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFUSEOutputStream.java   <--- Output Stream (instantiated during creat() calls)
+./src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java         <--- Entry Point for the plugin (extends Hadoop FileSystem class)
+./src/test
+./src/test/java
+./src/test/java/org
+./src/test/java/org/apache
+./src/test/java/org/apache/hadoop
+./src/test/java/org/apache/hadoop/fs
+./src/test/java/org/apache/hadoop/fs/glusterfs
+./src/test/java/org/apache/hadoop/fs/glusterfs/AppTest.java                  <--- Your test cases go here (if any :-))
+./tools/build-deploy-jar.py                                                  <--- Build and Deployment Script
+./conf
+./conf/core-site.xml                                                         <--- Sample configuration file
+./pom.xml                                                                    <--- build XML file (used by maven)
+
+** toplevel: hdfs/ **
+./COPYING                                                                    <--- License
+./README                                                                     <--- This file
diff --git a/glusterfs.spec.in b/glusterfs.spec.in
index c55da7f2d..040500c5f 100644
--- a/glusterfs.spec.in
+++ b/glusterfs.spec.in
@@ -1,99 +1,269 @@
-# if you make changes, the it is advised to increment this number, and provide
-# a descriptive suffix to identify who owns or what the change represents
-# e.g. release_version 2.MSW
-%define release 1%{?dist}
-%define _libexecdir %{_prefix}/local/libexec/
+%global _hardened_build 1
+
+%global _for_fedora_koji_builds 0
+
+# uncomment and add '%' to use the prereltag for pre-releases
+# global prereltag beta4
 
 # if you wish to compile an rpm without rdma support, compile like this...
 # rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without rdma
-%{?_without_rdma:%define _without_rdma --disable-ibverbs}
+%{?_without_rdma:%global _without_rdma --disable-ibverbs}
+
+# No RDMA Support on s390(x)
+%ifarch s390 s390x
+%global _without_rdma --disable-ibverbs
+%endif
 
 # if you wish to compile an rpm without epoll...
 # rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without epoll
-%{?_without_epoll:%define _without_epoll --disable-epoll}
+%{?_without_epoll:%global _without_epoll --disable-epoll}
 
-# if you wish to compile an rpm with fusermount...
-# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --with fusermount
-%{?_with_fusermount:%define _with_fusermount --enable-fusermount}
+# if you wish to compile an rpm without fusermount...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without fusermount
+%{?_without_fusermount:%global _without_fusermount --disable-fusermount}
 
 # if you wish to compile an rpm without geo-replication support, compile like this...
 # rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without georeplication
-%{?_without_georeplication:%define _without_georeplication --disable-geo-replication}
-
-Summary: Cluster File System
-Name: @PACKAGE_NAME@
-Version: @PACKAGE_VERSION@
-Release: %{release}
-License: GPLv3+
-Group: System Environment/Base
-Vendor: Gluster Inc
-Packager: @PACKAGE_BUGREPORT@
-URL: http://www.gluster.org/docs/index.php/GlusterFS
-Source0: ftp://ftp.gluster.com/pub/gluster/glusterfs/@PACKAGE_VERSION@/@PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz
-BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
-Requires(post): /sbin/chkconfig
-Requires(preun): /sbin/service, /sbin/chkconfig
+%{?_without_georeplication:%global _without_georeplication --disable-geo-replication}
+
+# if you wish to compile an rpm without the OCF resource agents...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without ocf
+%{?_without_ocf:%global _without_ocf --without-ocf}
+
+# if you wish to build rpms without syslog logging, compile like this
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@tar.gz --without syslog
+%{?_without_syslog:%global _without_syslog --disable-syslog}
+
+# disable syslog forcefully as rhel <= 6 doesn't have rsyslog or rsyslog-mmcount
+%if ( 0%{?rhel} && 0%{?rhel} <= 6 )
+%global _without_syslog --disable-syslog
+%endif
+
+# there is no systemtap support! Perhaps some day there will be
+%global _without_systemtap --enable-systemtap=no
+
+# if you wish to compile an rpm without the BD map support...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without bd
+%{?_without_bd:%global _without_bd --disable-bd-xlator}
+
+%if ( 0%{?rhel} && 0%{?rhel} < 6 )
+%define _without_bd --disable-bd-xlator
+%endif
+
+# if you wish to compile an rpm without the qemu-block support...
+# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without qemu-block
+%{?_without_qemu_block:%global _without_qemu_block --disable-qemu-block}
+
+%if ( 0%{?rhel} && 0%{?rhel} < 6 )
+# xlators/features/qemu-block fails to build on RHEL5, disable it
+%define _without_qemu_block --disable-qemu-block
+%endif
+
+%if ( 0%{?fedora} && 0%{?fedora} > 16 ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
+%global           _with_systemd true
+%endif
+
+# From https://fedoraproject.org/wiki/Packaging:Python#Macros
+%if ( 0%{?rhel} && 0%{?rhel} <= 5 )
+%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
+%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")}
+%endif
+
+Summary:          Cluster File System
+%if ( 0%{_for_fedora_koji_builds} )
+Name:             glusterfs
+Version:          3.4.1
+Release:          3%{?prereltag:.%{prereltag}}%{?dist}
+Vendor:           Fedora Project
+%else
+Name:             @PACKAGE_NAME@
+Version:          @PACKAGE_VERSION@
+Release:          1%{?dist}
+Vendor:           glusterfs.org
+%endif
+License:          GPLv2 or LGPLv3+
+Group:            System Environment/Base
+URL:              http://www.gluster.org/docs/index.php/GlusterFS
+%if ( 0%{_for_fedora_koji_builds} )
+Source0:          http://download.gluster.org/pub/gluster/glusterfs/3.4/%{version}%{?prereltag}/glusterfs-%{version}%{?prereltag}.tar.gz
+Source1:          glusterd.sysconfig
+Source2:          glusterfsd.sysconfig
+Source3:          glusterfs-fuse.logrotate
+Source4:          glusterd.logrotate
+Source5:          glusterfsd.logrotate
+Source6:          rhel5-load-fuse-modules
+Source11:         glusterfsd.service
+Source13:         glusterfsd.init
+Patch0:           %{name}-3.2.5.configure.ac.patch
+Patch1:           %{name}-3.3.0.libglusterfs.Makefile.patch
+Patch2:           %{name}-3.3.1.rpc.rpcxprt.rdma.name.c.patch
+%else
+Source0:          @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz
+%endif
+
+BuildRoot:        %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
+
+%if ( 0%{?rhel} && 0%{?rhel} <= 5 )
+BuildRequires:    python-simplejson
+%endif
+%if ( 0%{?_with_systemd:1} )
+%if ( 0%{_for_fedora_koji_builds} )
+%global glusterfsd_service %{S:%{SOURCE11}}
+%endif
+BuildRequires:    systemd-units
+Requires(post):   systemd-units
+Requires(preun):  systemd-units
+Requires(postun): systemd-units
+%define _init_enable()  /bin/systemctl enable %1.service ;
+%define _init_disable() /bin/systemctl disable %1.service ;
+%define _init_restart() /bin/systemctl try-restart %1.service ;
+%define _init_stop()    /bin/systemctl stop %1.service ;
+%define _init_install() %{__install} -D -p -m 0644 %1 %{buildroot}%{_unitdir}/%2.service ;
+# can't seem to make a generic macro that works
+%define _init_glusterd   %{_unitdir}/glusterd.service
+%define _init_glusterfsd %{_unitdir}/glusterfsd.service
+%else
+%if ( 0%{_for_fedora_koji_builds} )
+%global glusterfsd_service %{S:%{SOURCE13}}
+%endif
+Requires(post):   /sbin/chkconfig
+Requires(preun):  /sbin/service
+Requires(preun):  /sbin/chkconfig
 Requires(postun): /sbin/service
+%define _init_enable()  /sbin/chkconfig --add %1 ;
+%define _init_disable() /sbin/chkconfig --del %1 ;
+%define _init_restart() /sbin/service %1 condrestart &>/dev/null ;
+%define _init_stop()    /sbin/service %1 stop &>/dev/null ;
+%define _init_install() %{__install} -D -p -m 0755 %1 %{buildroot}%{_sysconfdir}/init.d/%2 ;
+# can't seem to make a generic macro that works
+%define _init_glusterd   %{_sysconfdir}/init.d/glusterd
+%define _init_glusterfsd %{_sysconfdir}/init.d/glusterfsd
+%endif
+
+Requires:         %{name}-libs = %{version}-%{release}
+BuildRequires:    bison flex
+BuildRequires:    gcc make automake libtool
+BuildRequires:    ncurses-devel readline-devel
+BuildRequires:    libxml2-devel openssl-devel
+BuildRequires:    libaio-devel
+BuildRequires:    python-devel
+BuildRequires:    python-ctypes
+%if ( 0%{!?_without_systemtap:1} )
+BuildRequires:    systemtap-sdt-devel
+%endif
+%if ( 0%{!?_without_bd:1} )
+BuildRequires:    lvm2-devel
+%endif
+%if ( 0%{!?_without_qemu_block:1} )
+BuildRequires:    glib2-devel
+%endif
+
+Obsoletes:        hekafs <= 0.7
+Obsoletes:        %{name}-libs <= 2.0.0
+Obsoletes:        %{name}-common < %{version}-%{release}
+Obsoletes:        %{name}-core < %{version}-%{release}
+Provides:         %{name}-libs = %{version}-%{release}
+Provides:         %{name}-common = %{version}-%{release}
+Provides:         %{name}-core = %{version}-%{release}
 
-BuildRequires: bison flex
-BuildRequires: gcc make
-%if 0%{?suse_version}
-BuildRequires: python-devel
+# We do not want to generate useless provides and requires for xlator .so files
+# Filter all generated:
+#
+# TODO: RHEL5 does not have a convenient solution
+%if ( 0%{?rhel} == 6 )
+    # filter_setup exists in RHEL6 only
+    %filter_provides_in %{_libdir}/glusterfs/%{version}/
+    %global __filter_from_req %{?__filter_from_req} | %{__grep} -v -P '^(?!lib).*\.so.*$'
+    %filter_setup
+%else
+    # modern rpm and current Fedora do not generate requires if the provides
+    # are filtered
+    %global __provides_exclude_from ^%{_libdir}/glusterfs/%{version}/.*$
+%endif
+
+%if ( 0%{?rhel} && 0%{?rhel} < 6 )
+   # _sharedstatedir is not provided by RHEL5
+   %define _sharedstatedir /var/lib
 %endif
-%define _prefix /opt/%{name}/%{version}
 
 %description
 GlusterFS is a clustered file-system capable of scaling to several
-peta-bytes. It aggregates various storage bricks over Infiniband RDMA
+petabytes. It aggregates various storage bricks over Infiniband RDMA
 or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file system in
+system. GlusterFS is one of the most sophisticated file systems in
 terms of features and extensibility.  It borrows a powerful concept
 called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in userspace and easily manageable.
+is in user space and easily manageable.
+
+This package includes the glusterfs binary, the glusterfsd daemon and the
+gluster command line, libglusterfs and glusterfs translator modules common to
+both GlusterFS server and client framework.
 
-%package core
-Summary: GlusterFS common files for both the client and the server
-Group: System Environment/Libraries
-Obsoletes: glusterfs-libs <= 2.0.0 glusterfs-common < 3.1.0 glusterfs-server < 3.1.0
-Provides: glusterfs-libs = %{version}-%{release}
+%package libs
+Summary:          GlusterFS common libraries
+Group:            Applications/File
+%if ( 0%{!?_without_syslog:1} )
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
+Requires:         rsyslog-mmjsonparse
+%endif
+%if ( 0%{?rhel} && 0%{?rhel} == 6 )
+Requires:         rsyslog-mmcount
+%endif
+%endif
 
-%description core
+%description libs
 GlusterFS is a clustered file-system capable of scaling to several
-peta-bytes. It aggregates various storage bricks over Infiniband RDMA
+petabytes. It aggregates various storage bricks over Infiniband RDMA
 or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file system in
+system. GlusterFS is one of the most sophisticated file systems in
 terms of features and extensibility.  It borrows a powerful concept
 called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in userspace and easily manageable.
+is in user space and easily manageable.
+
+This package provides the base GlusterFS libraries
+
+%package cli
+Summary:          GlusterFS CLI
+Group:            Applications/File
+Requires:         %{name}-libs = %{version}-%{release}
+
+%description cli
+GlusterFS is a clustered file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over Infiniband RDMA
+or TCP/IP interconnect into one large parallel network file
+system. GlusterFS is one of the most sophisticated file systems in
+terms of features and extensibility.  It borrows a powerful concept
+called Translators from GNU Hurd kernel. Much of the code in GlusterFS
+is in user space and easily manageable.
 
-This package includes the glusterfs binary, the glusterd daemon and the gluster
-command line, libglusterfs and glusterfs translator modules common to both
-GlusterFS server and client framework.
+This package provides the GlusterFS CLI application and its man page
 
-%if 0%{!?_without_rdma:1}
+%if ( 0%{!?_without_rdma:1} )
 %package rdma
-Summary: GlusterFS rdma
-Group: Applications/File
-Requires: glusterfs-core >= %{version}-%{release}
-BuildRequires: libibverbs-devel
+Summary:          GlusterFS rdma support for ib-verbs
+Group:            Applications/File
+BuildRequires:    libibverbs-devel
+BuildRequires:    librdmacm-devel
+Requires:         %{name} = %{version}-%{release}
 
 %description rdma
 GlusterFS is a clustered file-system capable of scaling to several
-peta-bytes. It aggregates various storage bricks over Infiniband RDMA
+petabytes. It aggregates various storage bricks over Infiniband RDMA
 or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file system in
+system. GlusterFS is one of the most sophisticated file systems in
 terms of features and extensibility.  It borrows a powerful concept
 called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in userspace and easily manageable.
+is in user space and easily manageable.
 
 This package provides support to ib-verbs library.
 %endif
 
-%if 0%{!?_without_georeplication:1}
+%if ( 0%{!?_without_georeplication:1} )
 %package geo-replication
-Summary: GlusterFS Geo-replication
-Group: Applications/File
-Requires: glusterfs-core >= %{version}-%{release} , python-ctypes , rsync >= 3.0.0
+Summary:          GlusterFS Geo-replication
+Group:            Applications/File
+Requires:         %{name} = %{version}-%{release}
+Requires:         %{name}-server = %{version}-%{release}
 
 %description geo-replication
 GlusterFS is a clustered file-system capable of scaling to several
@@ -108,177 +278,896 @@ This package provides support to geo-replication.
 %endif
 
 %package fuse
-Summary: GlusterFS Fuse
-Group: Applications/File
-Requires: glusterfs-core >= %{version}-%{release}
-Obsoletes: glusterfs-client < 3.1.0
+Summary:          Fuse client
+Group:            Applications/File
+BuildRequires:    fuse-devel
+
+Requires:         %{name} = %{version}-%{release}
+
+Obsoletes:        %{name}-client < %{version}-%{release}
+Provides:         %{name}-client = %{version}-%{release}
 
 %description fuse
 GlusterFS is a clustered file-system capable of scaling to several
-peta-bytes. It aggregates various storage bricks over Infiniband RDMA
+petabytes. It aggregates various storage bricks over Infiniband RDMA
 or TCP/IP interconnect into one large parallel network file
-system. GlusterFS is one of the most sophisticated file system in
+system. GlusterFS is one of the most sophisticated file systems in
 terms of features and extensibility.  It borrows a powerful concept
 called Translators from GNU Hurd kernel. Much of the code in GlusterFS
-is in userspace and easily manageable.
+is in user space and easily manageable.
 
 This package provides support to FUSE based clients.
 
+%package server
+Summary:          Clustered file-system server
+Group:            System Environment/Daemons
+Requires:         %{name} = %{version}-%{release}
+Requires:         %{name}-cli = %{version}-%{release}
+Requires:         %{name}-libs = %{version}-%{release}
+Requires:         %{name}-fuse = %{version}-%{release}
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
+Requires:         rpcbind
+%else
+Requires:         portmap
+%endif
+
+%description server
+GlusterFS is a clustered file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over Infiniband RDMA
+or TCP/IP interconnect into one large parallel network file
+system. GlusterFS is one of the most sophisticated file systems in
+terms of features and extensibility.  It borrows a powerful concept
+called Translators from GNU Hurd kernel. Much of the code in GlusterFS
+is in user space and easily manageable.
+
+This package provides the glusterfs server daemon.
+
+%package api
+Summary:          Clustered file-system api library
+Group:            System Environment/Daemons
+Requires:         %{name} = %{version}-%{release}
+Requires:         %{name}-devel = %{version}-%{release}
+# we provide the Python package/namespace 'gluster'
+Provides:         python-gluster = %{version}-%{release}
+
+%description api
+GlusterFS is a clustered file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over Infiniband RDMA
+or TCP/IP interconnect into one large parallel network file
+system. GlusterFS is one of the most sophisticated file systems in
+terms of features and extensibility.  It borrows a powerful concept
+called Translators from GNU Hurd kernel. Much of the code in GlusterFS
+is in user space and easily manageable.
+
+This package provides the glusterfs libgfapi library.
+
+%if ( 0%{!?_without_ocf:1} )
+%package resource-agents
+Summary:          OCF Resource Agents for GlusterFS
+License:          GPLv3+
+%if ( ! ( 0%{?rhel} && 0%{?rhel} < 6 ) )
+# EL5 does not support noarch sub-packages
+BuildArch:        noarch
+%endif
+# this Group handling comes from the Fedora resource-agents package
+%if ( 0%{?fedora} || 0%{?centos_version} || 0%{?rhel} )
+Group:            System Environment/Base
+%else
+Group:            Productivity/Clustering/HA
+%endif
+# for glusterd
+Requires:         glusterfs-server
+# depending on the distribution, we need pacemaker or resource-agents
+Requires:         %{_prefix}/lib/ocf/resource.d
+
+%description resource-agents
+GlusterFS is a clustered file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over Infiniband RDMA
+or TCP/IP interconnect into one large parallel network file
+system. GlusterFS is one of the most sophisticated file systems in
+terms of features and extensibility.  It borrows a powerful concept
+called Translators from GNU Hurd kernel. Much of the code in GlusterFS
+is in user space and easily manageable.
+
+This package provides the resource agents which plug glusterd into
+Open Cluster Framework (OCF) compliant cluster resource managers,
+like Pacemaker.
+%endif
+
+%package devel
+Summary:          Development Libraries
+Group:            Development/Libraries
+Requires:         %{name} = %{version}-%{release}
+
+%description devel
+GlusterFS is a clustered file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over Infiniband RDMA
+or TCP/IP interconnect into one large parallel network file
+system. GlusterFS is one of the most sophisticated file systems in
+terms of features and extensibility.  It borrows a powerful concept
+called Translators from GNU Hurd kernel. Much of the code in GlusterFS
+is in user space and easily manageable.
+
+This package provides the development libraries and include files.
+
+%package api-devel
+Summary:          Development Libraries
+Group:            Development/Libraries
+Requires:         %{name} = %{version}-%{release}
+Requires:         %{name}-devel = %{version}-%{release}
+
+%description api-devel
+GlusterFS is a clustered file-system capable of scaling to several
+petabytes. It aggregates various storage bricks over Infiniband RDMA
+or TCP/IP interconnect into one large parallel network file
+system. GlusterFS is one of the most sophisticated file systems in
+terms of features and extensibility.  It borrows a powerful concept
+called Translators from GNU Hurd kernel. Much of the code in GlusterFS
+is in user space and easily manageable.
+
+This package provides the api include files.
+
+%package regression-tests
+Summary:          Development Tools
+Group:            Development/Tools
+Requires:         %{name} = %{version}-%{release}
+Requires:         %{name}-fuse = %{version}-%{release}
+Requires:         %{name}-server = %{version}-%{release}
+Requires:         perl(App::Prove) perl(Test::Harness) gcc util-linux-ng lvm2
+Requires:         python attr dbench git nfs-utils xfsprogs
+
+%description regression-tests
+The Gluster Test Framework, is a suite of scripts used for
+regression testing of Gluster.
+
 %prep
-%setup -q -n %{name}-%{version}
+%setup -q -n %{name}-%{version}%{?prereltag}
+%if ( 0%{_for_fedora_koji_builds} )
+#%patch0 -p0
+%patch1 -p0 -F4
+%if ( "%{version}" == "3.3.1" )
+%patch2 -p1
+%endif
+%endif
 
 %build
+./autogen.sh
+%configure \
+        %{?_without_rdma} \
+        %{?_without_epoll} \
+        %{?_without_fusermount} \
+        %{?_without_georeplication} \
+        %{?_without_ocf} \
+        %{?_without_syslog} \
+        %{?_without_bd} \
+        %{?_without_qemu_block} \
+        %{?_without_systemtap}
 
-%configure %{?_without_rdma} %{?_without_epoll} %{?_with_fusermount} %{?_without_georeplication}
+# fix hardening and remove rpath in shlibs
+%if ( 0%{?fedora} && 0%{?fedora} > 17 ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
+%{__sed} -i 's| \\\$compiler_flags |&\\\$LDFLAGS |' libtool
+%endif
+%{__sed} -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|' libtool
+%{__sed} -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|' libtool
 
-# Remove rpath
-sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' libtool
-sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool
 %{__make} %{?_smp_mflags}
 
+pushd api/examples
+FLAGS="$RPM_OPT_FLAGS" %{__python} setup.py build
+popd
 
 %install
 %{__rm} -rf %{buildroot}
 %{__make} install DESTDIR=%{buildroot}
-%{__mkdir_p} %{buildroot}/usr/sbin
-%{__mkdir_p} %{buildroot}/usr/local/libexec/glusterfs
-%{__ln_s} ../..%{_sbindir}/glusterd %{buildroot}/usr/sbin/glusterd
-%{__ln_s} ../..%{_sbindir}/glusterfsd %{buildroot}/usr/sbin/glusterfsd
-%{__ln_s} ../..%{_sbindir}/gluster %{buildroot}/usr/sbin/gluster
-%{__ln_s} ../..%{_sbindir}/glusterfs %{buildroot}/usr/sbin/glusterfs
-%{__ln_s} %{_libexecdir}/glusterfs/gsyncd %{buildroot}/usr/local/libexec/glusterfs/gsyncd
+# install the gfapi Python library in /usr/lib/python*/site-packages
+pushd api/examples
+%{__python} setup.py install --skip-build --verbose --root %{buildroot}
+popd
+# Install include directory
 %{__mkdir_p} %{buildroot}%{_includedir}/glusterfs
-%{__mkdir_p} %{buildroot}/var/log/glusterfs
 %{__install} -p -m 0644 libglusterfs/src/*.h \
     %{buildroot}%{_includedir}/glusterfs/
+%{__install} -p -m 0644 contrib/uuid/*.h \
+    %{buildroot}%{_includedir}/glusterfs/
+# Following needed by hekafs multi-tenant translator
+%{__mkdir_p} %{buildroot}%{_includedir}/glusterfs/rpc
+%{__install} -p -m 0644 rpc/rpc-lib/src/*.h \
+    %{buildroot}%{_includedir}/glusterfs/rpc/
+%{__install} -p -m 0644 rpc/xdr/src/*.h \
+    %{buildroot}%{_includedir}/glusterfs/rpc/
+%{__mkdir_p} %{buildroot}%{_includedir}/glusterfs/server
+%{__install} -p -m 0644 xlators/protocol/server/src/*.h \
+    %{buildroot}%{_includedir}/glusterfs/server/
+%if ( 0%{_for_fedora_koji_builds} )
+%{__install} -D -p -m 0644 %{SOURCE1} \
+    %{buildroot}%{_sysconfdir}/sysconfig/glusterd
+%{__install} -D -p -m 0644 %{SOURCE2} \
+    %{buildroot}%{_sysconfdir}/sysconfig/glusterfsd
+%else
+%{__install} -D -p -m 0644 extras/glusterd-sysconfig \
+    %{buildroot}%{_sysconfdir}/sysconfig/glusterd
+%endif
+
+%if ( 0%{_for_fedora_koji_builds} )
+%if ( 0%{?rhel} && 0%{?rhel} <= 5 )
+%{__install} -D -p -m 0755 %{SOURCE6} \
+    %{buildroot}%{_sysconfdir}/sysconfig/modules/glusterfs-fuse.modules
+%endif
+%endif
+
+%{__mkdir_p} %{buildroot}%{_localstatedir}/log/glusterd
+%{__mkdir_p} %{buildroot}%{_localstatedir}/log/glusterfs
+%{__mkdir_p} %{buildroot}%{_localstatedir}/log/glusterfsd
+%{__mkdir_p} %{buildroot}%{_localstatedir}/run/gluster
 
 # Remove unwanted files from all the shared libraries
-find %{buildroot}%{_libdir} -name '*.la' | xargs rm -f
-find %{buildroot}%{_libdir} -name '*.a' | xargs rm -f
+find %{buildroot}%{_libdir} -name '*.a' -delete
+find %{buildroot}%{_libdir} -name '*.la' -delete
 
-#update  /etc/ld.so.conf.d
-%{__mkdir_p} %{buildroot}%{_sysconfdir}/ld.so.conf.d
-%ifarch x86_64
-echo "%{_prefix}/lib64" > %{buildroot}%{_sysconfdir}/ld.so.conf.d/glusterfs.conf
+# Remove installed docs, they're included by %%doc
+%{__rm} -rf %{buildroot}%{_datadir}/doc/glusterfs/
+head -50 ChangeLog > ChangeLog.head && mv ChangeLog.head ChangeLog
+cat << EOM >> ChangeLog
+
+More commit messages for this ChangeLog can be found at
+https://forge.gluster.org/glusterfs-core/glusterfs/commits/v%{version}%{?prereltag}
+EOM
+
+# Remove benchmarking and other unpackaged files
+%{__rm} -rf %{buildroot}/benchmarking
+%{__rm} -f %{buildroot}/glusterfs-mode.el
+%{__rm} -f %{buildroot}/glusterfs.vim
+
+# Create working directory
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd
+
+# Update configuration file to /var/lib working directory
+sed -i 's|option working-directory /etc/glusterd|option working-directory %{_sharedstatedir}/glusterd|g' \
+    %{buildroot}%{_sysconfdir}/glusterfs/glusterd.vol
+
+# Install glusterfsd .service or init.d file
+%if ( 0%{_for_fedora_koji_builds} )
+%_init_install %{glusterfsd_service} glusterfsd
+%endif
+
+%if ( 0%{_for_fedora_koji_builds} )
+# Client logrotate entry
+%{__install} -D -p -m 0644 %{SOURCE3} \
+    %{buildroot}%{_sysconfdir}/logrotate.d/glusterfs-fuse
+
+# Server logrotate entry
+%{__install} -D -p -m 0644 %{SOURCE4} \
+    %{buildroot}%{_sysconfdir}/logrotate.d/glusterd
+# Legacy server logrotate entry
+%{__install} -D -p -m 0644 %{SOURCE5} \
+    %{buildroot}%{_sysconfdir}/logrotate.d/glusterfsd
 %else
-echo "%{_prefix}/lib" > %{buildroot}%{_sysconfdir}/ld.so.conf.d/glusterfs.conf
+%{__install} -D -p -m 0644 extras/glusterfs-logrotate \
+    %{buildroot}%{_sysconfdir}/logrotate.d/glusterfs
 %endif
 
-%clean
-%{__rm} -rf %{buildroot}
+%if ( 0%{!?_without_georeplication:1} )
+# geo-rep ghosts
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/geo-replication
+touch %{buildroot}%{_sharedstatedir}/glusterd/geo-replication/gsyncd_template.conf
+%{__install} -D -p -m 0644 extras/glusterfs-georep-logrotate \
+    %{buildroot}%{_sysconfdir}/logrotate.d/glusterfs-georep
+%endif
 
-%post core
-/sbin/ldconfig
-/sbin/chkconfig --add glusterd
+%if ( 0%{!?_without_syslog:1} )
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
+%{__install} -D -p -m 0644 extras/gluster-rsyslog-7.2.conf \
+    %{buildroot}%{_sysconfdir}/rsyslog.d/gluster.conf.example
+%endif
 
+%if ( 0%{?rhel} && 0%{?rhel} == 6 )
+%{__install} -D -p -m 0644 extras/gluster-rsyslog-5.8.conf \
+    %{buildroot}%{_sysconfdir}/rsyslog.d/gluster.conf.example
+%endif
 
-/sbin/ldconfig
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
+%{__install} -D -p -m 0644 extras/logger.conf.example \
+    %{buildroot}%{_sysconfdir}/glusterfs/logger.conf.example
+%endif
+%endif
 
-if [ -d /etc/glusterd/vols ]; then
-  # Rename old volfiles in an RPM-standard way.  These aren't actually
-  # considered package config files, so %config doesn't work for them.
-  #
-  # NB: we used to exempt client-side volfiles from this treatment.  The
-  # only reason we do it at all (I'm guessing) is because they might be
-  # rewritten differently by a new version of code, and the old versions might
-  # be useful for a merge or rollback (especially if they were tweaked).
-  # Since that rationale does apply to client-side volfiles as well, the
-  # exception has been removed.
-  for file in $(find /etc/glusterd/vols -name '*.vol'); do
-    newfile=${file}.rpmsave
-    echo "warning: ${file} saved as ${newfile}"
-    cp ${file} ${newfile}
-  done
-fi
+# the rest of the ghosts
+touch %{buildroot}%{_sharedstatedir}/glusterd/glusterd.info
+touch %{buildroot}%{_sharedstatedir}/glusterd/options
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/stop
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/stop/post
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/stop/pre
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/start
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/start/post
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/start/pre
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/remove-brick
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/remove-brick/post
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/remove-brick/pre
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/add-brick
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/add-brick/post
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/add-brick/pre
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/set
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/set/post
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/set/pre
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/create
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/create/post
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/create/pre
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/delete
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/delete/post
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/delete/pre
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/copy-file
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/copy-file/post
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/copy-file/pre
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/gsync-create
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/gsync-create/post
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/hooks/1/gsync-create/pre
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/glustershd
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/peers
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/vols
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/groups
+%{__mkdir_p} %{buildroot}%{_sharedstatedir}/glusterd/nfs/run
+touch %{buildroot}%{_sharedstatedir}/glusterd/nfs/nfs-server.vol
+touch %{buildroot}%{_sharedstatedir}/glusterd/nfs/run/nfs.pid
 
-pidof -c -o %PPID -x glusterd &> /dev/null
-if [ $? -eq 0 ];
-then
-kill -9 `pgrep -f gsyncd.py` &> /dev/null
+find ./tests ./run-tests.sh -type f | cpio -pd %{buildroot}%{_prefix}/share/glusterfs
 
-killall glusterd &> /dev/null
-#add marker translator
-glusterd --xlator-option *.upgrade=on
-else
-glusterd --xlator-option *.upgrade=on
-sleep 10
-killall glusterd &> /dev/null
-fi
+%clean
+%{__rm} -rf %{buildroot}
 
-%postun core
-/sbin/ldconfig -n %{_libdir}
+%post
+/sbin/ldconfig
+%if ( 0%{!?_without_syslog:1} )
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
+%_init_restart rsyslog
+%endif
+%endif
 
-%preun core
-if [ $1 -eq 0 ]; then
-    /sbin/chkconfig --del glusterd
-fi
+%postun
+/sbin/ldconfig
+%if ( 0%{!?_without_syslog:1} )
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
+%_init_restart rsyslog
+%endif
+%endif
 
-%files core
-%defattr(-,root,root)
-%doc AUTHORS ChangeLog COPYING INSTALL NEWS README THANKS
-%doc %{_datadir}/doc/glusterfs
-%if 0%{?_with_fusermount:1}
-%{_bindir}/fusermount-glusterfs
+%files
+%defattr(-,root,root,-)
+%doc ChangeLog COPYING-GPLV2 COPYING-LGPLV3 INSTALL README THANKS
+%config(noreplace) %{_sysconfdir}/logrotate.d/*
+%config(noreplace) %{_sysconfdir}/sysconfig/*
+%if ( 0%{!?_without_syslog:1} )
+%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
+%{_sysconfdir}/rsyslog.d/gluster.conf.example
+%endif
 %endif
 %{_libdir}/glusterfs
-%{_libdir}/*.so.*
 %{_sbindir}/glusterfs*
+%{_mandir}/man8/*gluster*.8*
+%exclude %{_mandir}/man8/gluster.8*
+%dir %{_localstatedir}/log/glusterfs
+%dir %{_localstatedir}/run/gluster
+%dir %{_sharedstatedir}/glusterd
+%if ( 0%{!?_without_rdma:1} )
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/rpc-transport/rdma*
+%endif
+# server-side, etc., xlators in other RPMs
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/api*
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/fuse*
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/storage*
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/posix*
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol/server*
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mgmt*
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/nfs*
+# sample xlators not generally used or usable
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/encryption/rot-13*
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/mac-compat*
+%exclude %{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/testing/performance/symlink-cache*
+
+%post libs
+/sbin/ldconfig
+
+%postun libs
+/sbin/ldconfig
+
+%files libs
+%{_libdir}/*.so.*
+%exclude %{_libdir}/libgfapi.*
+
+%files cli
 %{_sbindir}/gluster
-%{_sbindir}/glusterd
-/usr/sbin
-%{_mandir}/man8/*glusterfs.8*
-%{_mandir}/man8/*glusterfsd.8*
-%{_mandir}/man8/*gluster.8*
-%{_mandir}/man8/*glusterd.8*
-%dir /var/log/glusterfs
-%if 0%{!?_without_rdma:1}
-%exclude %{_libdir}/glusterfs/%{version}/rpc-transport/rdma*
-%endif
-%exclude %{_libdir}/glusterfs/%{version}/xlator/mount/fuse*
-%config %{_sysconfdir}/glusterfs
-%{_sysconfdir}/init.d/glusterd
-%{_includedir}/glusterfs
-%exclude %{_includedir}/glusterfs/y.tab.h
-%{_sysconfdir}/ld.so.conf.d/glusterfs.conf
-%{_libdir}/*.so
-%exclude %{_libexecdir}/glusterfs/gsyncd
-%exclude %{_libexecdir}/glusterfs/python/syncdaemon/*
-%exclude /usr/local/libexec/glusterfs/gsyncd
+%{_mandir}/man8/gluster.8*
 
-%if 0%{!?_without_rdma:1}
+%if ( 0%{!?_without_rdma:1} )
 %files rdma
-%defattr(-,root,root)
-%{_libdir}/glusterfs/%{version}/rpc-transport/rdma*
+%defattr(-,root,root,-)
+%{_libdir}/glusterfs/%{version}%{?prereltag}/rpc-transport/rdma*
 %endif
 
-%if 0%{!?_without_georeplication:1}
+%if ( 0%{!?_without_georeplication:1} )
 %post geo-replication
 #restart glusterd.
-/etc/init.d/glusterd restart &> /dev/null
-%endif
+if [ $1 -ge 1 ]; then
+    %_init_restart glusterd
+fi
 
-%if 0%{!?_without_georeplication:1}
 %files geo-replication
 %defattr(-,root,root)
 %{_libexecdir}/glusterfs/gsyncd
 %{_libexecdir}/glusterfs/python/syncdaemon/*
-/usr/local/libexec/glusterfs/gsyncd
+%{_libexecdir}/glusterfs/gverify.sh
+%{_libexecdir}/glusterfs/peer_add_secret_pub
+%{_libexecdir}/glusterfs/peer_gsec_create
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/geo-replication
+%ghost %attr(0644,-,-) %{_sharedstatedir}/glusterd/geo-replication/gsyncd_template.conf
 %endif
 
-
 %files fuse
-%defattr(-,root,root)
-%{_libdir}/glusterfs/%{version}/xlator/mount/fuse*
+%defattr(-,root,root,-)
+%if ( 0%{_for_fedora_koji_builds} )
+%config(noreplace) %{_sysconfdir}/logrotate.d/glusterfs-fuse
+%endif
+%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/fuse*
 /sbin/mount.glusterfs
+%if ( 0%{!?_without_fusermount:1} )
+%{_bindir}/fusermount-glusterfs
+%endif
+%if ( 0%{_for_fedora_koji_builds} )
+%if ( 0%{?rhel} && 0%{?rhel} <= 5 )
+%{_sysconfdir}/sysconfig/modules/glusterfs-fuse.modules
+%endif
+%endif
+
+%files server
+%defattr(-,root,root,-)
+%doc extras/clear_xattrs.sh
+%if ( 0%{_for_fedora_koji_builds} )
+%config(noreplace) %{_sysconfdir}/logrotate.d/glusterd
+%endif
+%config(noreplace) %{_sysconfdir}/sysconfig/glusterd
+%config(noreplace) %{_sysconfdir}/glusterfs
+# Legacy configs
+%if ( 0%{_for_fedora_koji_builds} )
+%config(noreplace) %{_sysconfdir}/logrotate.d/glusterfsd
+%config(noreplace) %{_sysconfdir}/sysconfig/glusterfsd
+%endif
+# init files
+%_init_glusterd
+%if ( 0%{_for_fedora_koji_builds} )
+%_init_glusterfsd
+%endif
+# binaries
+%{_sbindir}/glusterd
+%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/storage*
+%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/features/posix*
+%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/protocol/server*
+%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mgmt*
+%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/nfs*
+%ghost %attr(0644,-,-) %config(noreplace) %{_sharedstatedir}/glusterd/glusterd.info
+%ghost %attr(0600,-,-) %{_sharedstatedir}/glusterd/options
+# This is really ugly, but I have no idea how to mark these directories in an
+# other way. They should belong to the glusterfs-server package, but don't
+# exist after installation. They are generated on the first start...
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop/post
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/stop/pre
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/start
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/start/post
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/start/pre
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/remove-brick
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/remove-brick/post
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/remove-brick/pre
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/post
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/add-brick/pre
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/set
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/set/post
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/set/pre
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create/post
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/create/pre
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete/post
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/hooks/1/delete/pre
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/glustershd
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/vols
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/peers
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/groups
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/nfs
+%ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/nfs/nfs-server.vol
+%ghost %dir %attr(0755,-,-) %{_sharedstatedir}/glusterd/nfs/run
+%ghost      %attr(0600,-,-) %{_sharedstatedir}/glusterd/nfs/run/nfs.pid
+
+%post api
+/sbin/ldconfig
+
+%postun api
+/sbin/ldconfig
+
+%files api
+%exclude %{_libdir}/*.so
+%{_libdir}/libgfapi.*
+%{_libdir}/glusterfs/%{version}%{?prereltag}/xlator/mount/api*
+%{python_sitelib}/*
+
+%if ( 0%{!?_without_ocf:1} )
+%files resource-agents
+%defattr(-,root,root)
+# /usr/lib is the standard for OCF, also on x86_64
+%{_prefix}/lib/ocf/resource.d/glusterfs
+%endif
+
+%files devel
+%defattr(-,root,root,-)
+%{_includedir}/glusterfs
+%exclude %{_includedir}/glusterfs/y.tab.h
+%exclude %{_includedir}/glusterfs/api
+%exclude %{_libdir}/libgfapi.so
+%{_libdir}/*.so
+
+%files api-devel
+%{_libdir}/pkgconfig/glusterfs-api.pc
+%{_libdir}/pkgconfig/libgfchangelog.pc
+%{_libdir}/libgfapi.so
+%{_includedir}/glusterfs/api/*
+
+%files regression-tests
+%defattr(-,root,root,-)
+%{_prefix}/share/glusterfs/*
+%exclude %{_prefix}/share/glusterfs/tests/basic/rpm.t
+
+%post server
+# Legacy server
+%_init_enable glusterd
+%_init_enable glusterfsd
+
+# Genuine Fedora (and EPEL) builds never put gluster files in /etc; if
+# there are any files in /etc from a prior gluster.org install, move them
+# to /var/lib. (N.B. Starting with 3.3.0 all gluster files are in /var/lib
+# in gluster.org RPMs.) Be careful to copy them on the off chance that
+# /etc and /var/lib are on separate file systems
+if [ -d /etc/glusterd -a ! -h %{_sharedstatedir}/glusterd ]; then
+    %{__mkdir_p} %{_sharedstatedir}/glusterd
+    cp -a /etc/glusterd %{_sharedstatedir}/glusterd
+    rm -rf /etc/glusterd
+    ln -sf %{_sharedstatedir}/glusterd /etc/glusterd
+fi
+
+# Rename old volfiles in an RPM-standard way.  These aren't actually
+# considered package config files, so %config doesn't work for them.
+if [ -d %{_sharedstatedir}/glusterd/vols ]; then
+    for file in $(find %{_sharedstatedir}/glusterd/vols -name '*.vol'); do
+        newfile=${file}.rpmsave
+        echo "warning: ${file} saved as ${newfile}"
+        cp ${file} ${newfile}
+    done
+fi
+
+# add marker translator
+# but first make certain that there are no old libs around to bite us
+# BZ 834847
+if [ -e /etc/ld.so.conf.d/glusterfs.conf ]; then
+    rm -f /etc/ld.so.conf.d/glusterfs.conf
+    /sbin/ldconfig
+fi
+pidof -c -o %PPID -x glusterd &> /dev/null
+if [ $? -eq 0 ]; then
+    kill -9 `pgrep -f gsyncd.py` &> /dev/null
+
+    killall glusterd &> /dev/null
+    glusterd --xlator-option *.upgrade=on -N
+else
+    glusterd --xlator-option *.upgrade=on -N
+fi
+
+%preun server
+if [ $1 -eq 0 ]; then
+    if [ -f %_init_glusterfsd ]; then
+        %_init_stop glusterfsd
+    fi
+    %_init_stop glusterd
+    if [ -f %_init_glusterfsd ]; then
+        %_init_disable glusterfsd
+    fi
+    %_init_disable glusterd
+fi
+if [ $1 -ge 1 ]; then
+    if [ -f %_init_glusterfsd ]; then
+        %_init_restart glusterfsd
+    fi
+    %_init_restart glusterd
+fi
 
 %changelog
-* Wed Jul 01 2009 Harshavardhana <harsha@gluster.com> - 2.1
-- Removed mod_glusterfs.so and added new --without epoll build
-  option.
-
-* Thu Apr 16 2009 Harshavardhana <harsha@gluster.com> - 2.0
-- Galore of updates including new packages added common,
-  client,server splitting the original package. rpmbuild
-  fully restructured to adhere to Fedora rpm standards.
-  Older changelog removed as there were warnings when
-  tried with 'rpmlint'.
+* Wed Oct 11 2013 Harshavardhana <fharshav@redhat.com>
+- Add '_sharedstatedir' macro to `/var/lib` on <= RHEL5 (#1003184)
+
+* Wed Oct 9 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec 3.4.1-2+
+
+* Wed Oct 9 2013 Niels de Vos <ndevos@redhat.com>
+- glusterfs-api-devel requires glusterfs-devel (#1016938, #1017094)
+
+* Mon Sep 30 2013 Niels de Vos <ndevos@redhat.com>
+- Package gfapi.py into the Python site-packages path (#1005146)
+
+* Tue Sep 17 2013 Harshavardhana <fharshav@redhat.com>
+- Provide a new package called "glusterfs-regression-tests" for standalone
+  regression testing.
+
+* Thu Aug 22 2013 Niels de Vos <ndevos@redhat.com>
+- Correct the day/date for some entries in this changelog (#1000019)
+
+* Wed Aug 7 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec
+-  add Requires
+-  add -cli subpackage,
+-  fix other minor differences with Fedora glusterfs.spec
+
+* Tue Jul 30 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec, add glusterfs-libs RPM for oVirt/qemu-kvm
+
+* Thu Jul 25 2013 Csaba Henk <csaba@redhat.com>
+- Added peer_add_secret_pub and peer_gsec_create to %{_libexecdir}/glusterfs
+
+* Thu Jul 25 2013 Aravinda VK <avishwan@redhat.com>
+- Added gverify.sh to %{_libexecdir}/glusterfs directory.
+
+* Thu Jul 25 2013 Harshavardhana <fharshav@redhat.com>
+- Allow to build with '--without bd' to disable 'bd' xlator
+
+* Thu Jun 27 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- fix the hardening fix for shlibs, use %%{__sed} macro, shorter ChangeLog
+
+* Wed Jun 26 2013 Niels de Vos <ndevos@redhat.com>
+- move the mount/api xlator to glusterfs-api
+
+* Fri Jun 7 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec, remove G4S/UFO and Swift
+
+* Mon Mar 4 2013 Niels de Vos <ndevos@redhat.com>
+- Package /var/run/gluster so that statedumps can be created
+
+* Wed Feb 6 2013 Kaleb S. KEITHLEY <kkeithle@redhat.com>
+- Sync with Fedora glusterfs.spec
+
+* Tue Dec 11 2012 Filip Pytloun <filip.pytloun@gooddata.com>
+- add sysconfig file
+
+* Thu Oct 25 2012 Niels de Vos <ndevos@redhat.com>
+- Add a sub-package for the OCF resource agents
+
+* Wed Sep 05 2012 Niels de Vos <ndevos@redhat.com>
+- Don't use python-ctypes on SLES (from Jörg Petersen)
+
+* Tue Jul 10 2012 Niels de Vos <ndevos@redhat.com>
+- Include extras/clear_xattrs.sh in the glusterfs-server sub-package
+
+* Thu Jun 07 2012 Niels de Vos <ndevos@redhat.com>
+- Mark /var/lib/glusterd as owned by glusterfs, subdirs belong to -server
+
+* Wed May 9 2012 Kaleb S. KEITHLEY <kkeithle[at]redhat.com>
+- Add BuildRequires: libxml2-devel so that configure will DTRT on for
+- Fedora's Koji build system
+
+* Wed Nov 9 2011 Joe Julian <me@joejulian.name> - git master
+- Merge fedora specfile into gluster's spec.in.
+- Add conditionals to allow the same spec file to be used for both 3.1 and 3.2
+- http://bugs.gluster.com/show_bug.cgi?id=2970
+
+* Wed Oct  5 2011 Joe Julian <me@joejulian.name> - 3.2.4-1
+- Update to 3.2.4
+- Removed the $local_fs requirement from the init scripts as in RHEL/CentOS that's provided
+- by netfs, which needs to be started after glusterd.
+
+* Sun Sep 25 2011 Joe Julian <me@joejulian.name> - 3.2.3-2
+- Merged in upstream changes
+- Fixed version reporting 3.2git
+- Added nfs init script (disabled by default)
+
+* Thu Sep  1 2011 Joe Julian <me@joejulian.name> - 3.2.3-1
+- Update to 3.2.3
+
+* Tue Jul 19 2011 Joe Julian <me@joejulian.name> - 3.2.2-3
+- Add readline and libtermcap dependencies
+
+* Tue Jul 19 2011 Joe Julian <me@joejulian.name> - 3.2.2-2
+- Critical patch to prevent glusterd from walking outside of its own volume during rebalance
+
+* Thu Jul 14 2011 Joe Julian <me@joejulian.name> - 3.2.2-1
+- Update to 3.2.2
+
+* Wed Jul 13 2011 Joe Julian <me@joejulian.name> - 3.2.1-2
+- fix hardcoded path to gsyncd in source to match the actual file location
+
+* Tue Jun 21 2011 Joe Julian <me@joejulian.name> - 3.2.1
+- Update to 3.2.1
+
+* Mon Jun 20 2011 Joe Julian <me@joejulian.name> - 3.1.5
+- Update to 3.1.5
+
+* Tue May 31 2011 Joe Julian <me@joejulian.name> - 3.1.5-qa1.4
+- Current git
+
+* Sun May 29 2011 Joe Julian <me@joejulian.name> - 3.1.5-qa1.2
+- set _sharedstatedir to /var/lib for FHS compliance in RHEL5/CentOS5
+- mv /etc/glusterd, if it exists, to the new state dir for upgrading from gluster packaging
+
+* Sat May 28 2011 Joe Julian <me@joejulian.name> - 3.1.5-qa1.1
+- Update to 3.1.5-qa1
+- Add patch to remove optimization disabling
+- Add patch to remove forced 64 bit compile
+- Obsolete glusterfs-core to allow for upgrading from gluster packaging
+
+* Sat Mar 19 2011 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.1.3-1
+- Update to 3.1.3
+- Merge in more upstream SPEC changes
+- Remove patches from GlusterFS bugzilla #2309 and #2311
+- Remove inode-gen.patch
+
+* Sun Feb 06 2011 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.1.2-3
+- Add back in legacy SPEC elements to support older branches
+
+* Thu Feb 03 2011 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.1.2-2
+- Add patches from CloudFS project
+
+* Tue Jan 25 2011 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.1.2-1
+- Update to 3.1.2
+
+* Wed Jan 5 2011 Dan Horák <dan[at]danny.cz> - 3.1.1-3
+- no InfiniBand on s390(x)
+
+* Sat Jan 1 2011 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.1.1-2
+- Update to support readline
+- Update to not parallel build
+
+* Mon Dec 27 2010 Silas Sewell <silas@sewell.ch> - 3.1.1-1
+- Update to 3.1.1
+- Change package names to mirror upstream
+
+* Mon Dec 20 2010 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.0.7-1
+- Update to 3.0.7
+
+* Wed Jul 28 2010 Jonathan Steffan <jsteffan@fedoraproject.org> - 3.0.5-1
+- Update to 3.0.x
+
+* Sat Apr 10 2010 Jonathan Steffan <jsteffan@fedoraproject.org> - 2.0.9-2
+- Move python version requires into a proper BuildRequires otherwise
+  the spec always turned off python bindings as python is not part
+  of buildsys-build and the chroot will never have python unless we
+  require it
+- Temporarily set -D_FORTIFY_SOURCE=1 until upstream fixes code
+  GlusterFS Bugzilla #197 (#555728)
+- Move glusterfs-volgen to devel subpackage (#555724)
+- Update description (#554947)
+
+* Sat Jan 2 2010 Jonathan Steffan <jsteffan@fedoraproject.org> - 2.0.9-1
+- Update to 2.0.9
+
+* Sun Nov 8 2009 Jonathan Steffan <jsteffan@fedoraproject.org> - 2.0.8-1
+- Update to 2.0.8
+- Remove install of glusterfs-volgen, it's properly added to
+  automake upstream now
+
+* Sat Oct 31 2009 Jonathan Steffan <jsteffan@fedoraproject.org> - 2.0.7-1
+- Update to 2.0.7
+- Install glusterfs-volgen, until it's properly added to automake
+  by upstream
+- Add macro to be able to ship more docs
+
+* Thu Sep 17 2009 Peter Lemenkov <lemenkov@gmail.com> 2.0.6-2
+- Rebuilt with new fuse
+
+* Sat Sep 12 2009 Matthias Saou <http://freshrpms.net/> 2.0.6-1
+- Update to 2.0.6.
+- No longer default to disable the client on RHEL5 (#522192).
+- Update spec file URLs.
+
+* Mon Jul 27 2009 Matthias Saou <http://freshrpms.net/> 2.0.4-1
+- Update to 2.0.4.
+
+* Thu Jun 11 2009 Matthias Saou <http://freshrpms.net/> 2.0.1-2
+- Remove libglusterfs/src/y.tab.c to fix koji F11/devel builds.
+
+* Sat May 16 2009 Matthias Saou <http://freshrpms.net/> 2.0.1-1
+- Update to 2.0.1.
+
+* Thu May  7 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-1
+- Update to 2.0.0 final.
+
+* Wed Apr 29 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-0.3.rc8
+- Move glusterfsd to common, since the client has a symlink to it.
+
+* Fri Apr 24 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-0.2.rc8
+- Update to 2.0.0rc8.
+
+* Sun Apr 12 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-0.2.rc7
+- Update glusterfsd init script to the new style init.
+- Update files to match the new default vol file names.
+- Include logrotate for glusterfsd, use a pid file by default.
+- Include logrotate for glusterfs, using killall for lack of anything better.
+
+* Sat Apr 11 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-0.1.rc7
+- Update to 2.0.0rc7.
+- Rename "libs" to "common" and move the binary, man page and log dir there.
+
+* Tue Feb 24 2009 Fedora Release Engineering <rel-eng@lists.fedoraproject.org>
+- Rebuilt for https://fedoraproject.org/wiki/Fedora_11_Mass_Rebuild
+
+* Mon Feb 16 2009 Matthias Saou <http://freshrpms.net/> 2.0.0-0.1.rc1
+- Update to 2.0.0rc1.
+- Include new libglusterfsclient.h.
+
+* Mon Feb 16 2009 Matthias Saou <http://freshrpms.net/> 1.3.12-1
+- Update to 1.3.12.
+- Remove no longer needed ocreat patch.
+
+* Thu Jul 17 2008 Matthias Saou <http://freshrpms.net/> 1.3.10-1
+- Update to 1.3.10.
+- Remove mount patch, it's been included upstream now.
+
+* Fri May 16 2008 Matthias Saou <http://freshrpms.net/> 1.3.9-1
+- Update to 1.3.9.
+
+* Fri May  9 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-1
+- Update to 1.3.8 final.
+
+* Wed Apr 23 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.10
+- Include short patch to include fixes from latest TLA 751.
+
+* Tue Apr 22 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.9
+- Update to 1.3.8pre6.
+- Include glusterfs binary in both the client and server packages, now that
+  glusterfsd is a symlink to it instead of a separate binary.
+* Sun Feb  3 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.8
+- Add python version check and disable bindings for version < 2.4.
+
+* Sun Feb  3 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.7
+- Add --without client rpmbuild option, make it the default for RHEL (no fuse).
+  (I hope "rhel" is the proper default macro name, couldn't find it...)
+
+* Wed Jan 30 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.6
+- Add --without ibverbs rpmbuild option to the package.
+
+* Mon Jan 14 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.5
+- Update to current TLA again, patch-636 which fixes the known segfaults.
+
+* Thu Jan 10 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.4
+- Downgrade to glusterfs--mainline--2.5--patch-628 which is more stable.
+
+* Tue Jan  8 2008 Matthias Saou <http://freshrpms.net/> 1.3.8-0.3
+- Update to current TLA snapshot.
+- Include umount.glusterfs wrapper script (really needed? dunno).
+- Include patch to mount wrapper to avoid multiple identical mounts.
+
+* Sun Dec 30 2007 Matthias Saou <http://freshrpms.net/> 1.3.8-0.1
+- Update to current TLA snapshot, which includes "volume-name=" fstab option.
+
+* Mon Dec  3 2007 Matthias Saou <http://freshrpms.net/> 1.3.7-6
+- Re-add the /var/log/glusterfs directory in the client sub-package (required).
+- Include custom patch to support vol= in fstab for -n glusterfs client option.
+
+* Mon Nov 26 2007 Matthias Saou <http://freshrpms.net/> 1.3.7-4
+- Re-enable libibverbs.
+- Check and update License field to GPLv3+.
+- Add glusterfs-common obsoletes, to provide upgrade path from old packages.
+- Include patch to add mode to O_CREATE opens.
+
+* Thu Nov 22 2007 Matthias Saou <http://freshrpms.net/> 1.3.7-3
+- Remove Makefile* files from examples.
+- Include RHEL/Fedora type init script, since the included ones don't do.
+
+* Wed Nov 21 2007 Matthias Saou <http://freshrpms.net/> 1.3.7-1
+- Major spec file cleanup.
+- Add missing %%clean section.
+- Fix ldconfig calls (weren't set for the proper sub-package).
+
+* Sat Aug 4 2007 Matt Paine <matt@mattsoftware.com> - 1.3.pre7
+- Added support to build rpm without ibverbs support (use --without ibverbs
+  switch)
+
+* Sun Jul 15 2007 Matt Paine <matt@mattsoftware.com> - 1.3.pre6
+- Initial spec file
diff --git a/glusterfsd/src/Makefile.am b/glusterfsd/src/Makefile.am
index 17d7a4a81..05a10dee3 100644
--- a/glusterfsd/src/Makefile.am
+++ b/glusterfsd/src/Makefile.am
@@ -4,15 +4,17 @@ glusterfsd_SOURCES = glusterfsd.c glusterfsd-mgmt.c
 glusterfsd_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
-	$(GF_LDADD)
-glusterfsd_LDFLAGS = $(GF_LDFLAGS) $(GF_GLUSTERFS_LDFLAGS)
+	$(GF_LDADD) $(GF_GLUSTERFS_LIBS)
+glusterfsd_LDFLAGS = $(GF_LDFLAGS)
 noinst_HEADERS = glusterfsd.h glusterfsd-mem-types.h
 
-AM_CFLAGS = -fPIC -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -DDATADIR=\"$(localstatedir)\" \
-	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_GLUSTERFS_CFLAGS) \
+	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
 	-I$(top_srcdir)/rpc/rpc-lib/src -I$(top_srcdir)/rpc/xdr/src
 
+AM_CFLAGS = -Wall $(GF_GLUSTERFS_CFLAGS)
+
 CLEANFILES = 
 
 $(top_builddir)/libglusterfs/src/libglusterfs.la:
@@ -24,7 +26,9 @@ uninstall-local:
 
 install-data-local:
 	$(INSTALL) -d -m 755 $(DESTDIR)$(localstatedir)/run
+	$(INSTALL) -d -m 755 $(DESTDIR)$(localstatedir)/run/gluster
 	$(INSTALL) -d -m 755 $(DESTDIR)$(localstatedir)/log/glusterfs
+	$(INSTALL) -d -m 755 $(DESTDIR)$(sbindir)
 	rm -f $(DESTDIR)$(sbindir)/glusterfs
 	rm -f $(DESTDIR)$(sbindir)/glusterd
 	ln -s glusterfsd $(DESTDIR)$(sbindir)/glusterfs
diff --git a/glusterfsd/src/glusterfsd-mem-types.h b/glusterfsd/src/glusterfsd-mem-types.h
index a28a7b2e3..7135c0ada 100644
--- a/glusterfsd/src/glusterfsd-mem-types.h
+++ b/glusterfsd/src/glusterfsd-mem-types.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
-
 #ifndef __GLUSTERFSD_MEM_TYPES_H__
 #define __GLUSTERFSD_MEM_TYPES_H__
 
@@ -27,10 +17,10 @@
 enum gfd_mem_types_ {
         gfd_mt_xlator_list_t = GF_MEM_TYPE_START,
         gfd_mt_xlator_t,
+        gfd_mt_server_cmdline_t,
         gfd_mt_xlator_cmdline_option_t,
         gfd_mt_char,
         gfd_mt_call_pool_t,
-        gfd_mt_vol_top_priv_t,
         gfd_mt_end
 
 };
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index e2b658a98..1c9220927 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1,28 +1,17 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <stdlib.h>
 #include <signal.h>
-#include <pthread.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -42,30 +31,41 @@
 #include "xdr-generic.h"
 
 #include "glusterfsd.h"
-#include "glusterfsd-mem-types.h"
 #include "rpcsvc.h"
 #include "cli1-xdr.h"
 #include "statedump.h"
+#include "syncop.h"
+#include "xlator.h"
 
-static char is_mgmt_rpc_reconnect;
+static gf_boolean_t is_mgmt_rpc_reconnect = _gf_false;
 
 int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx);
 int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx);
 int glusterfs_process_volfp (glusterfs_ctx_t *ctx, FILE *fp);
 int glusterfs_graph_unknown_options (glusterfs_graph_t *graph);
+int emancipate(glusterfs_ctx_t *ctx, int ret);
 
 int
-mgmt_cbk_spec (void *data)
+mgmt_cbk_spec (struct rpc_clnt *rpc, void *mydata, void *data)
 {
         glusterfs_ctx_t *ctx = NULL;
+        xlator_t *this = NULL;
 
-        ctx = glusterfs_ctx_get ();
+        this = mydata;
+        ctx = glusterfsd_ctx;
         gf_log ("mgmt", GF_LOG_INFO, "Volume file changed");
 
         glusterfs_volfile_fetch (ctx);
         return 0;
 }
 
+
+int
+mgmt_cbk_event (struct rpc_clnt *rpc, void *mydata, void *data)
+{
+        return 0;
+}
+
 struct iobuf *
 glusterfs_serialize_reply (rpcsvc_request_t *req, void *arg,
                            struct iovec *outmsg, xdrproc_t xdrproc)
@@ -100,7 +100,6 @@ glusterfs_serialize_reply (rpcsvc_request_t *req, void *arg,
         outmsg->iov_len = retlen;
 ret:
         if (retlen == -1) {
-                iobuf_unref (iob);
                 iob = NULL;
         }
 
@@ -122,7 +121,6 @@ glusterfs_submit_reply (rpcsvc_request_t *req, void *arg,
                 goto out;
         }
 
-
         if (!iobref) {
                 iobref = iobref_new ();
                 if (!iobref) {
@@ -135,12 +133,11 @@ glusterfs_submit_reply (rpcsvc_request_t *req, void *arg,
 
         iob = glusterfs_serialize_reply (req, arg, &rsp, xdrproc);
         if (!iob) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to serialize reply");
-                goto out;
+                gf_log_callingfn (THIS->name, GF_LOG_ERROR, "Failed to serialize reply");
+        } else {
+                iobref_add (iobref, iob);
         }
 
-        iobref_add (iobref, iob);
-
         ret = rpcsvc_submit_generic (req, &rsp, 1, payload, payloadcount,
                                      iobref);
 
@@ -148,7 +145,6 @@ glusterfs_submit_reply (rpcsvc_request_t *req, void *arg,
          * we can safely unref the iob in the hope that RPC layer must have
          * ref'ed the iob on receiving into the txlist.
          */
-        iobuf_unref (iob);
         if (ret == -1) {
                 gf_log (THIS->name, GF_LOG_ERROR, "Reply submission failed");
                 goto out;
@@ -156,10 +152,11 @@ glusterfs_submit_reply (rpcsvc_request_t *req, void *arg,
 
         ret = 0;
 out:
+        if (iob)
+                iobuf_unref (iob);
 
-        if (new_iobref) {
+        if (new_iobref && iobref)
                 iobref_unref (iobref);
-        }
 
         return ret;
 }
@@ -178,52 +175,23 @@ glusterfs_terminate_response_send (rpcsvc_request_t *req, int op_ret)
 
         if (dict)
                 ret = dict_allocate_and_serialize (dict, &rsp.output.output_val,
-                                                (size_t *)&rsp.output.output_len);
+                                                   &rsp.output.output_len);
 
 
         if (ret == 0)
                 ret = glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
                                               (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
 
-        if (rsp.output.output_val)
-                GF_FREE (rsp.output.output_val);
+        GF_FREE (rsp.output.output_val);
         if (dict)
                 dict_unref (dict);
         return ret;
 }
 
 int
-glusterfs_listener_stop (void)
-{
-        glusterfs_ctx_t  *ctx = NULL;
-        cmd_args_t       *cmd_args = NULL;
-        int              ret = 0;
-        xlator_t         *this = NULL;
-
-        ctx = glusterfs_ctx_get ();
-        GF_ASSERT (ctx);
-        cmd_args = &ctx->cmd_args;
-        if (cmd_args->sock_file) {
-                ret = unlink (cmd_args->sock_file);
-                if (ret && (ENOENT == errno)) {
-                        ret = 0;
-                }
-        }
-
-        if (ret) {
-                this = THIS;
-                gf_log (this->name, GF_LOG_ERROR, "Failed to unlink linstener "
-                        "socket %s, error: %s", cmd_args->sock_file,
-                        strerror (errno));
-        }
-        return ret;
-}
-
-int
 glusterfs_handle_terminate (rpcsvc_request_t *req)
 {
 
-        (void) glusterfs_listener_stop ();
         glusterfs_terminate_response_send (req, 0);
         cleanup_and_exit (SIGTERM);
         return 0;
@@ -234,118 +202,61 @@ glusterfs_translator_info_response_send (rpcsvc_request_t *req, int ret,
                                          char *msg, dict_t *output)
 {
         gd1_mgmt_brick_op_rsp    rsp = {0,};
-        GF_ASSERT (msg);
+        gf_boolean_t             free_ptr = _gf_false;
         GF_ASSERT (req);
-        GF_ASSERT (output);
 
         rsp.op_ret = ret;
         rsp.op_errno = 0;
-        if (ret && msg[0])
+        if (ret && msg && msg[0])
                 rsp.op_errstr = msg;
         else
                 rsp.op_errstr = "";
 
-        ret = dict_allocate_and_serialize (output, &rsp.output.output_val,
-                                        (size_t *)&rsp.output.output_len);
-
-        ret = glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
-        if (rsp.output.output_val)
-                GF_FREE (rsp.output.output_val);
-        return ret;
-}
-
-int
-glusterfs_handle_translator_info_get_cont (gfd_vol_top_priv_t *priv)
-{
-        int                     ret = -1;
-        xlator_t                *any = NULL;
-        xlator_t                *xlator = NULL;
-        glusterfs_graph_t       *active = NULL;
-        glusterfs_ctx_t         *ctx = NULL;
-        char                    msg[2048] = {0,};
-        dict_t                  *output = NULL;
-        dict_t                  *dict = NULL;
-
-        GF_ASSERT (priv);
-
-        dict = dict_new ();
-        ret = dict_unserialize (priv->xlator_req.input.input_val,
-                                priv->xlator_req.input.input_len, &dict);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to unserialize dict");
-                goto cont;
-        }
-        ret = dict_set_double (dict, "time", priv->time);
-        if (ret)
-                goto cont;
-        ret = dict_set_double (dict, "throughput", priv->throughput);
-        if (ret)
-                goto cont;
-
-cont:
-        ctx = glusterfs_ctx_get ();
-        GF_ASSERT (ctx);
-        active = ctx->active;
-        any = active->first;
-
-        xlator = xlator_search_by_name (any, priv->xlator_req.name);
-        if (!xlator) {
-                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
-                          priv->xlator_req.name);
-                goto out;
+        ret = -1;
+        if (output) {
+                ret = dict_allocate_and_serialize (output,
+                                                   &rsp.output.output_val,
+                                                   &rsp.output.output_len);
         }
+        if (!ret)
+                free_ptr = _gf_true;
 
-        output = dict_new ();
-        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
-
-out:
-        ret = glusterfs_translator_info_response_send (priv->req, ret,
-                                                       msg, output);
-
-        if (priv->xlator_req.name)
-                free (priv->xlator_req.name);
-        if (priv->xlator_req.input.input_val)
-                free (priv->xlator_req.input.input_val);
-        if (dict)
-                dict_unref (dict);
-        if (output)
-                dict_unref (output);
-        GF_FREE (priv);
-
+        glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+        ret = 0;
+        if (free_ptr)
+                GF_FREE (rsp.output.output_val);
         return ret;
 }
 
 int
-glusterfs_translator_heal_response_send (rpcsvc_request_t *req, int op_ret,
-                                         char *msg, dict_t *output)
+glusterfs_xlator_op_response_send (rpcsvc_request_t *req, int op_ret,
+                                   char *msg, dict_t *output)
 {
         gd1_mgmt_brick_op_rsp    rsp = {0,};
         int                      ret = -1;
-        GF_ASSERT (msg);
+        gf_boolean_t             free_ptr = _gf_false;
         GF_ASSERT (req);
-        GF_ASSERT (output);
 
         rsp.op_ret = op_ret;
         rsp.op_errno = 0;
-        if (ret && msg[0])
+        if (op_ret && msg && msg[0])
                 rsp.op_errstr = msg;
         else
                 rsp.op_errstr = "";
 
-        ret = dict_allocate_and_serialize (output, &rsp.output.output_val,
-                                        (size_t *)&rsp.output.output_len);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't serialize "
-                        "output dict.");
-                goto out;
+        if (output) {
+                ret = dict_allocate_and_serialize (output,
+                                                   &rsp.output.output_val,
+                                                   &rsp.output.output_len);
         }
+        if (!ret)
+                free_ptr = _gf_true;
 
         ret = glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
 
-out:
-        if (rsp.output.output_val)
+        if (free_ptr)
                 GF_FREE (rsp.output.output_val);
 
         return ret;
@@ -354,28 +265,35 @@ out:
 int
 glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
 {
-        int32_t                  ret     = -1;
-        gd1_mgmt_brick_op_req    xlator_req = {0,};
-        dict_t                   *dict    = NULL;
-        xlator_t                 *this = NULL;
-        gf1_cli_top_op            top_op = 0;
-        int32_t                  blk_size = 0;
-        int32_t                  blk_count = 0;
-        gfd_vol_top_priv_t       *priv = NULL;
-        pthread_t                tid = -1;
+        int32_t                 ret     = -1;
+        gd1_mgmt_brick_op_req   xlator_req = {0,};
+        dict_t                  *dict    = NULL;
+        xlator_t                *this = NULL;
+        gf1_cli_top_op          top_op = 0;
+        uint32_t                blk_size = 0;
+        uint32_t                blk_count = 0;
+        double                  time = 0;
+        double                  throughput = 0;
+        xlator_t                *any = NULL;
+        xlator_t                *xlator = NULL;
+        glusterfs_graph_t       *active = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
+        char                    msg[2048] = {0,};
+        dict_t                  *output = NULL;
 
         GF_ASSERT (req);
         this = THIS;
         GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &xlator_req,
-                             (xdrproc_t)xdr_gd1_mgmt_brick_op_req)) {
+        ret = xdr_to_generic (req->msg[0], &xlator_req,
+                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
-        dict = dict_new ();
 
+        dict = dict_new ();
         ret = dict_unserialize (xlator_req.input.input_val,
                                 xlator_req.input.input_len,
                                 &dict);
@@ -386,78 +304,82 @@ glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
                 goto out;
         }
 
-        priv = GF_MALLOC (sizeof (gfd_vol_top_priv_t), gfd_mt_vol_top_priv_t);
-        if (!priv) {
-                gf_log ("glusterd", GF_LOG_ERROR, "failed to allocate memory");
-                goto out;
-        }
-        priv->xlator_req = xlator_req;
-        priv->req = req;
-
         ret = dict_get_int32 (dict, "top-op", (int32_t *)&top_op);
         if ((!ret) && (GF_CLI_TOP_READ_PERF == top_op ||
             GF_CLI_TOP_WRITE_PERF == top_op)) {
-                ret = dict_get_int32 (dict, "blk-size", &blk_size);
+                ret = dict_get_uint32 (dict, "blk-size", &blk_size);
                 if (ret)
                         goto cont;
-                ret = dict_get_int32 (dict, "blk-cnt", &blk_count);
+                ret = dict_get_uint32 (dict, "blk-cnt", &blk_count);
                 if (ret)
                         goto cont;
-                priv->blk_size = blk_size;
-                priv->blk_count = blk_count;
+
                 if (GF_CLI_TOP_READ_PERF == top_op) {
-                        ret = pthread_create (&tid, NULL,
-                                              glusterfs_volume_top_read_perf,
-                                              priv);
+                        ret = glusterfs_volume_top_read_perf
+                                (blk_size, blk_count, xlator_req.name,
+                                 &throughput, &time);
                 } else if ( GF_CLI_TOP_WRITE_PERF == top_op) {
-                        ret = pthread_create (&tid, NULL,
-                                              glusterfs_volume_top_write_perf,
-                                              priv);
+                        ret = glusterfs_volume_top_write_perf
+                                (blk_size, blk_count, xlator_req.name,
+                                 &throughput, &time);
                 }
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "Thread create failed");
+                ret = dict_set_double (dict, "time", time);
+                if (ret)
+                        goto cont;
+                ret = dict_set_double (dict, "throughput", throughput);
+                if (ret)
                         goto cont;
-                }
-                gf_log ("glusterd", GF_LOG_DEBUG, "Created new thread with "
-                        "tid %u", (unsigned int)tid);
-                goto out;
         }
 cont:
-        priv->throughput = 0;
-        priv->time = 0;
-        ret = glusterfs_handle_translator_info_get_cont (priv);
+        ctx = glusterfsd_ctx;
+        GF_ASSERT (ctx);
+        active = ctx->active;
+        any = active->first;
+
+        xlator = xlator_search_by_name (any, xlator_req.name);
+        if (!xlator) {
+                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
+                          xlator_req.name);
+                goto out;
+        }
+
+        output = dict_new ();
+        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
+
 out:
+        ret = glusterfs_translator_info_response_send (req, ret, msg, output);
+
+        free (xlator_req.name);
+        free (xlator_req.input.input_val);
+        if (output)
+                dict_unref (output);
         if (dict)
                 dict_unref (dict);
         return ret;
 }
 
-void *
-glusterfs_volume_top_write_perf (void *args)
+int
+glusterfs_volume_top_write_perf (uint32_t blk_size, uint32_t blk_count,
+                                 char *brick_path, double *throughput,
+                                 double *time)
 {
         int32_t                 fd = -1;
         int32_t                 input_fd = -1;
         char                    export_path[PATH_MAX];
         char                    *buf = NULL;
-        int32_t                 blk_size = 0;
-        int32_t                 blk_count = 0;
         int32_t                 iter = 0;
         int32_t                 ret = -1;
-        int64_t                 total_blks = 0;
+        uint64_t                total_blks = 0;
         struct timeval          begin, end = {0,};
-        double                  throughput = 0;
-        double                  time = 0;
-        gfd_vol_top_priv_t      *priv = NULL;
 
-        GF_ASSERT (args);
-        priv = (gfd_vol_top_priv_t *)args;
-
-        blk_size = priv->blk_size;
-        blk_count = priv->blk_count;
+        GF_ASSERT (brick_path);
+        GF_ASSERT (throughput);
+        GF_ASSERT (time);
+        if (!(blk_size > 0) || ! (blk_count > 0))
+                goto out;
 
         snprintf (export_path, sizeof (export_path), "%s/%s",
-                  priv->xlator_req.name, ".gf-tmp-stats-perf");
+                  brick_path, ".gf-tmp-stats-perf");
 
         fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
         if (-1 == fd) {
@@ -494,62 +416,53 @@ glusterfs_volume_top_write_perf (void *args)
                 total_blks += ret;
         }
         ret = 0;
-        if (total_blks != (blk_size * blk_count)) {
+        if (total_blks != ((uint64_t)blk_size * blk_count)) {
                 gf_log ("glusterd", GF_LOG_WARNING, "Error in write");
                 ret = -1;
                 goto out;
         }
 
         gettimeofday (&end, NULL);
-        time = (end.tv_sec - begin.tv_sec) * 1e6
+        *time = (end.tv_sec - begin.tv_sec) * 1e6
                      + (end.tv_usec - begin.tv_usec);
-        throughput = total_blks / time;
+        *throughput = total_blks / *time;
         gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs "
-                "bytes written %"PRId64, throughput, time, total_blks);
+                "bytes written %"PRId64, *throughput, *time, total_blks);
 
 out:
-        priv->throughput = throughput;
-        priv->time = time;
-
         if (fd >= 0)
                 close (fd);
         if (input_fd >= 0)
                 close (input_fd);
-        if (buf)
-                GF_FREE (buf);
+        GF_FREE (buf);
         unlink (export_path);
 
-        (void)glusterfs_handle_translator_info_get_cont (priv);
-
-        return NULL;
+        return ret;
 }
 
-void *
-glusterfs_volume_top_read_perf (void *args)
+int
+glusterfs_volume_top_read_perf (uint32_t blk_size, uint32_t blk_count,
+                                char *brick_path, double *throughput,
+                                double *time)
 {
         int32_t                 fd = -1;
         int32_t                 input_fd = -1;
         int32_t                 output_fd = -1;
         char                    export_path[PATH_MAX];
         char                    *buf = NULL;
-        int32_t                 blk_size = 0;
-        int32_t                 blk_count = 0;
         int32_t                 iter = 0;
         int32_t                 ret = -1;
-        int64_t                 total_blks = 0;
+        uint64_t                total_blks = 0;
         struct timeval          begin, end = {0,};
-        double                  throughput = 0;
-        double                  time = 0;
-        gfd_vol_top_priv_t      *priv = NULL;
 
-        GF_ASSERT (args);
-        priv = (gfd_vol_top_priv_t *)args;
-
-        blk_size = priv->blk_size;
-        blk_count = priv->blk_count;
+        GF_ASSERT (brick_path);
+        GF_ASSERT (throughput);
+        GF_ASSERT (time);
+        if (!(blk_size > 0) || ! (blk_count > 0))
+                goto out;
 
         snprintf (export_path, sizeof (export_path), "%s/%s",
-                  priv->xlator_req.name, ".gf-tmp-stats-perf");
+                  brick_path, ".gf-tmp-stats-perf");
         fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
         if (-1 == fd) {
                 ret = -1;
@@ -618,48 +531,41 @@ glusterfs_volume_top_read_perf (void *args)
                 total_blks += ret;
         }
         ret = 0;
-        if ((blk_size * blk_count) != total_blks) {
+        if (total_blks != ((uint64_t)blk_size * blk_count)) {
                 ret = -1;
-                gf_log ("glusterd", GF_LOG_WARNING, "Error in write");
+                gf_log ("glusterd", GF_LOG_WARNING, "Error in read");
                 goto out;
         }
 
         gettimeofday (&end, NULL);
-        time = (end.tv_sec - begin.tv_sec) * 1e6
-               + (end.tv_usec - begin.tv_usec);
-        throughput = total_blks / time;
+        *time = (end.tv_sec - begin.tv_sec) * 1e6
+                + (end.tv_usec - begin.tv_usec);
+        *throughput = total_blks / *time;
         gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs "
-                "bytes read %"PRId64, throughput, time, total_blks);
+                "bytes read %"PRId64, *throughput, *time, total_blks);
 
 out:
-        priv->throughput = throughput;
-        priv->time = time;
-
         if (fd >= 0)
                 close (fd);
         if (input_fd >= 0)
                 close (input_fd);
         if (output_fd >= 0)
                 close (output_fd);
-        if (buf)
-                GF_FREE (buf);
+        GF_FREE (buf);
         unlink (export_path);
 
-        (void)glusterfs_handle_translator_info_get_cont (priv);
-
-        return NULL;
+        return ret;
 }
 
 int
-glusterfs_handle_translator_heal (rpcsvc_request_t *req)
+glusterfs_handle_translator_op (rpcsvc_request_t *req)
 {
         int32_t                  ret     = -1;
         gd1_mgmt_brick_op_req    xlator_req = {0,};
-        dict_t                   *dict    = NULL;
+        dict_t                   *input    = NULL;
         xlator_t                 *xlator = NULL;
         xlator_t                 *any = NULL;
         dict_t                   *output = NULL;
-        char                     msg[2048] = {0};
         char                     key[2048] = {0};
         char                    *xname = NULL;
         glusterfs_ctx_t          *ctx = NULL;
@@ -672,70 +578,149 @@ glusterfs_handle_translator_heal (rpcsvc_request_t *req)
         this = THIS;
         GF_ASSERT (this);
 
-        ctx = glusterfs_ctx_get ();
-        GF_ASSERT (ctx);
-
-        active = ctx->active;
-        any = active->first;
-        if (!xdr_to_generic (req->msg[0], &xlator_req,
-                             (xdrproc_t)xdr_gd1_mgmt_brick_op_req)) {
+        ret = xdr_to_generic (req->msg[0], &xlator_req,
+                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
-        dict = dict_new ();
 
+        ctx = glusterfsd_ctx;
+        active = ctx->active;
+        any = active->first;
+        input = dict_new ();
         ret = dict_unserialize (xlator_req.input.input_val,
                                 xlator_req.input.input_len,
-                                &dict);
+                                &input);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to "
                         "unserialize req-buffer to dictionary");
                 goto out;
+        } else {
+                input->extra_stdfree = xlator_req.input.input_val;
         }
 
-        ret = dict_get_int32 (dict, "count", &count);
-        i = 0;
-        while (i < count)  {
-                snprintf (key, sizeof (key), "heal-%d", i);
-                ret = dict_get_str (dict, key, &xname);
+        ret = dict_get_int32 (input, "count", &count);
+
+        output = dict_new ();
+        if (!output) {
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; i < count; i++)  {
+                snprintf (key, sizeof (key), "xl-%d", i);
+                ret = dict_get_str (input, key, &xname);
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR, "Couldn't get "
-                                "replicate xlator %s to trigger "
-                                "self-heal", xname);
+                                "xlator %s ", key);
                         goto out;
                 }
                 xlator = xlator_search_by_name (any, xname);
                 if (!xlator) {
-                        snprintf (msg, sizeof (msg), "xlator %s is not loaded",
-                                  xlator_req.name);
-                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR, "xlator %s is not "
+                                "loaded", xname);
                         goto out;
                 }
+        }
+        for (i = 0; i < count; i++)  {
+                snprintf (key, sizeof (key), "xl-%d", i);
+                ret = dict_get_str (input, key, &xname);
+                xlator = xlator_search_by_name (any, xname);
+                XLATOR_NOTIFY (xlator, GF_EVENT_TRANSLATOR_OP, input, output);
+                if (ret)
+                        break;
+        }
+out:
+        glusterfs_xlator_op_response_send (req, ret, "", output);
+        if (input)
+                dict_unref (input);
+        if (output)
+                dict_unref (output);
+        free (xlator_req.name); //malloced by xdr
+
+        return 0;
+}
+
+
+int
+glusterfs_handle_defrag (rpcsvc_request_t *req)
+{
+        int32_t                  ret     = -1;
+        gd1_mgmt_brick_op_req    xlator_req = {0,};
+        dict_t                   *dict    = NULL;
+        xlator_t                 *xlator = NULL;
+        xlator_t                 *any = NULL;
+        dict_t                   *output = NULL;
+        char                     msg[2048] = {0};
+        glusterfs_ctx_t          *ctx = NULL;
+        glusterfs_graph_t        *active = NULL;
+        xlator_t                 *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ctx = glusterfsd_ctx;
+        GF_ASSERT (ctx);
+
+        active = ctx->active;
+        if (!active) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        any = active->first;
+        ret = xdr_to_generic (req->msg[0], &xlator_req,
+                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+        dict = dict_new ();
+        if (!dict)
+                goto out;
 
-                ret = xlator_notify (xlator, GF_EVENT_TRIGGER_HEAL, dict, NULL);
-                i++;
+        ret = dict_unserialize (xlator_req.input.input_val,
+                                xlator_req.input.input_len,
+                                &dict);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to "
+                        "unserialize req-buffer to dictionary");
+                goto out;
         }
+        xlator = xlator_search_by_name (any, xlator_req.name);
+        if (!xlator) {
+                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
+                          xlator_req.name);
+                goto out;
+        }
+
         output = dict_new ();
-        if (!output)
+        if (!output) {
+                ret = -1;
                 goto out;
+        }
+
+        ret = xlator->notify (xlator, GF_EVENT_VOLUME_DEFRAG, dict, output);
 
-        /* output dict is not used currently, could be used later. */
-        ret = glusterfs_translator_heal_response_send (req, ret, msg, output);
+        ret = glusterfs_translator_info_response_send (req, ret,
+                                                       msg, output);
 out:
         if (dict)
                 dict_unref (dict);
-        if (xlator_req.input.input_val)
-                free (xlator_req.input.input_val); // malloced by xdr
+        free (xlator_req.input.input_val); // malloced by xdr
         if (output)
                 dict_unref (output);
-        if (xlator_req.name)
-                free (xlator_req.name); //malloced by xdr
+        free (xlator_req.name); //malloced by xdr
 
         return ret;
-}
 
+}
 int
 glusterfs_handle_brick_status (rpcsvc_request_t *req)
 {
@@ -751,15 +736,16 @@ glusterfs_handle_brick_status (rpcsvc_request_t *req)
         dict_t                  *output = NULL;
         char                    *volname = NULL;
         char                    *xname = NULL;
-        int32_t                 cmd = 0;
+        uint32_t                cmd = 0;
         char                    *msg = NULL;
 
         GF_ASSERT (req);
         this = THIS;
         GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &brick_req,
-            (xdrproc_t)xdr_gd1_mgmt_brick_op_req)) {
+        ret = xdr_to_generic (req->msg[0], &brick_req,
+                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -773,7 +759,7 @@ glusterfs_handle_brick_status (rpcsvc_request_t *req)
                 goto out;
         }
 
-        ret = dict_get_int32 (dict, "cmd", &cmd);
+        ret = dict_get_uint32 (dict, "cmd", &cmd);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Couldn't get status op");
                 goto out;
@@ -785,7 +771,7 @@ glusterfs_handle_brick_status (rpcsvc_request_t *req)
                 goto out;
         }
 
-        ctx = glusterfs_ctx_get ();
+        ctx = glusterfsd_ctx;
         GF_ASSERT (ctx);
         active = ctx->active;
         any = active->first;
@@ -843,59 +829,389 @@ glusterfs_handle_brick_status (rpcsvc_request_t *req)
                 rsp.op_errstr = "";
 
         ret = dict_allocate_and_serialize (output, &rsp.output.output_val,
-                                           (size_t *)&rsp.output.output_len);
+                                           &rsp.output.output_len);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Failed to serialize output dict to rsp");
                 goto out;
         }
 
-        ret = glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
-                                      (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+        glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+        ret = 0;
 
 out:
         if (dict)
                 dict_unref (dict);
-        if (brick_req.input.input_val)
-                free (brick_req.input.input_val);
-        if (xname)
-                GF_FREE (xname);
-        if (msg)
-                GF_FREE (msg);
-        if (rsp.output.output_val)
-                GF_FREE (rsp.output.output_val);
+        if (output)
+                dict_unref (output);
+        free (brick_req.input.input_val);
+        GF_FREE (xname);
+        GF_FREE (msg);
+        GF_FREE (rsp.output.output_val);
 
         return ret;
 }
 
+
 int
-glusterfs_handle_rpc_msg (rpcsvc_request_t *req)
+glusterfs_handle_node_status (rpcsvc_request_t *req)
+{
+        int                     ret = -1;
+        gd1_mgmt_brick_op_req   node_req = {0,};
+        gd1_mgmt_brick_op_rsp   rsp = {0,};
+        glusterfs_ctx_t         *ctx = NULL;
+        glusterfs_graph_t       *active = NULL;
+        xlator_t                *any = NULL;
+        xlator_t                *node = NULL;
+        xlator_t                *subvol = NULL;
+        dict_t                  *dict = NULL;
+        dict_t                  *output = NULL;
+        char                    *volname = NULL;
+        char                    *node_name = NULL;
+        char                    *subvol_name = NULL;
+        uint32_t                cmd = 0;
+        char                    *msg = NULL;
+
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &node_req,
+                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        dict = dict_new ();
+        ret = dict_unserialize (node_req.input.input_val,
+                                node_req.input.input_len, &dict);
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to unserialize "
+                        "req buffer to dictionary");
+                goto out;
+        }
+
+        ret = dict_get_uint32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get status op");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get volname");
+                goto out;
+        }
+
+        ctx = glusterfsd_ctx;
+        GF_ASSERT (ctx);
+        active = ctx->active;
+        any = active->first;
+
+        if ((cmd & GF_CLI_STATUS_NFS) != 0)
+                ret = gf_asprintf (&node_name, "%s", "nfs-server");
+        else if ((cmd & GF_CLI_STATUS_SHD) != 0)
+                ret = gf_asprintf (&node_name, "%s", "glustershd");
+        else {
+                ret = -1;
+                goto out;
+        }
+        if (ret == -1) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to set node xlator name");
+                goto out;
+        }
+
+        node = xlator_search_by_name (any, node_name);
+        if (!node) {
+                ret = -1;
+                gf_log (THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
+                        node_name);
+                goto out;
+        }
+
+        if ((cmd & GF_CLI_STATUS_NFS) != 0)
+                ret = gf_asprintf (&subvol_name, "%s", volname);
+        else if ((cmd & GF_CLI_STATUS_SHD) != 0)
+                ret = gf_asprintf (&subvol_name, "%s-replicate-0", volname);
+        else {
+                ret = -1;
+                goto out;
+        }
+        if (ret == -1) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to set node xlator name");
+                goto out;
+        }
+
+        subvol = xlator_search_by_name (node, subvol_name);
+        if (!subvol) {
+                ret = -1;
+                gf_log (THIS->name, GF_LOG_ERROR, "%s xlator is not loaded",
+                        subvol_name);
+                goto out;
+        }
+
+        output = dict_new ();
+        switch (cmd & GF_CLI_STATUS_MASK) {
+                case GF_CLI_STATUS_MEM:
+                        ret = 0;
+                        gf_proc_dump_mem_info_to_dict (output);
+                        gf_proc_dump_mempool_info_to_dict (ctx, output);
+                        break;
+
+                case GF_CLI_STATUS_CLIENTS:
+                        // clients not availbale for SHD
+                        if ((cmd & GF_CLI_STATUS_SHD) != 0)
+                                break;
+
+                        ret = dict_set_str (output, "volname", volname);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Error setting volname to dict");
+                                goto out;
+                        }
+                        ret = node->dumpops->priv_to_dict (node, output);
+                        break;
+
+                case GF_CLI_STATUS_INODE:
+                        ret = 0;
+                        inode_table_dump_to_dict (subvol->itable, "conn0",
+                                                  output);
+                        ret = dict_set_int32 (output, "conncount", 1);
+                        break;
+
+                case GF_CLI_STATUS_FD:
+                        // cannot find fd-tables in nfs-server graph
+                        // TODO: finish once found
+                        break;
+
+                case GF_CLI_STATUS_CALLPOOL:
+                        ret = 0;
+                        gf_proc_dump_pending_frames_to_dict (ctx->pool, output);
+                        break;
+
+                default:
+                        ret = -1;
+                        msg = gf_strdup ("Unknown status op");
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                        break;
+        }
+        rsp.op_ret = ret;
+        rsp.op_errno = 0;
+        if (ret && msg)
+                rsp.op_errstr = msg;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (output, &rsp.output.output_val,
+                                           &rsp.output.output_len);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to serialize output dict to rsp");
+                goto out;
+        }
+
+        glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+        ret = 0;
+
+out:
+        if (dict)
+                dict_unref (dict);
+        free (node_req.input.input_val);
+        GF_FREE (msg);
+        GF_FREE (rsp.output.output_val);
+        GF_FREE (node_name);
+        GF_FREE (subvol_name);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterfs_handle_nfs_profile (rpcsvc_request_t *req)
+{
+        int                     ret = -1;
+        gd1_mgmt_brick_op_req   nfs_req = {0,};
+        gd1_mgmt_brick_op_rsp   rsp = {0,};
+        dict_t                  *dict = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
+        glusterfs_graph_t       *active = NULL;
+        xlator_t                *any = NULL;
+        xlator_t                *nfs = NULL;
+        xlator_t                *subvol = NULL;
+        char                    *volname = NULL;
+        dict_t                  *output = NULL;
+
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &nfs_req,
+                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        dict = dict_new ();
+        ret = dict_unserialize (nfs_req.input.input_val,
+                                nfs_req.input.input_len, &dict);
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to "
+                        "unserialize req-buffer to dict");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get volname");
+                goto out;
+        }
+
+        ctx = glusterfsd_ctx;
+        GF_ASSERT (ctx);
+
+        active = ctx->active;
+        any = active->first;
+
+        // is this needed?
+        // are problems possible by searching for subvol directly from "any"?
+        nfs = xlator_search_by_name (any, "nfs-server");
+        if (!nfs) {
+                ret = -1;
+                gf_log (THIS->name, GF_LOG_ERROR, "xlator nfs-server is "
+                        "not loaded");
+                goto out;
+        }
+
+        subvol = xlator_search_by_name (nfs, volname);
+        if (!subvol) {
+                ret = -1;
+                gf_log (THIS->name, GF_LOG_ERROR, "xlator %s is no loaded",
+                        volname);
+                goto out;
+        }
+
+        output = dict_new ();
+        ret = subvol->notify (subvol, GF_EVENT_TRANSLATOR_INFO, dict, output);
+
+        rsp.op_ret = ret;
+        rsp.op_errno = 0;
+        rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (output, &rsp.output.output_val,
+                                           &rsp.output.output_len);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to serialize output dict to rsp");
+                goto out;
+        }
+
+        glusterfs_submit_reply (req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+        ret = 0;
+
+out:
+        free (nfs_req.input.input_val);
+        if (dict)
+                dict_unref (dict);
+        if (output)
+                dict_unref (output);
+        GF_FREE (rsp.output.output_val);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterfs_handle_volume_barrier_op (rpcsvc_request_t *req)
 {
-        int     ret = -1;
-        xlator_t *this = THIS;
+        int32_t                  ret     = -1;
+        gd1_mgmt_brick_op_req    xlator_req = {0,};
+        dict_t                   *dict    = NULL;
+        xlator_t                 *xlator = NULL;
+        xlator_t                 *any = NULL;
+        dict_t                   *output = NULL;
+        char                     msg[2048] = {0};
+        glusterfs_ctx_t          *ctx = NULL;
+        glusterfs_graph_t        *active = NULL;
+        xlator_t                 *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
         GF_ASSERT (this);
-        switch (req->procnum) {
-        case GLUSTERD_BRICK_TERMINATE:
-                ret = glusterfs_handle_terminate (req);
-                break;
-        case GLUSTERD_BRICK_XLATOR_INFO:
-                ret = glusterfs_handle_translator_info_get (req);
-                break;
-        case GLUSTERD_BRICK_XLATOR_HEAL:
-                ret = glusterfs_handle_translator_heal (req);
-                break;
-        case GLUSTERD_BRICK_STATUS:
-                ret = glusterfs_handle_brick_status (req);
-                break;
-        default:
-                break;
+
+        ctx = glusterfsd_ctx;
+        GF_ASSERT (ctx);
+
+        active = ctx->active;
+        if (!active) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
         }
 
+        any = active->first;
+        ret = xdr_to_generic (req->msg[0], &xlator_req,
+                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (xlator_req.input.input_val,
+                                xlator_req.input.input_len,
+                                &dict);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to "
+                        "unserialize req-buffer to dictionary");
+                goto out;
+        }
+        xlator = xlator_search_by_name (any, xlator_req.name);
+        if (!xlator) {
+                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
+                          xlator_req.name);
+                goto out;
+        }
+
+        output = dict_new ();
+        if (!output) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = xlator->notify (xlator, GF_EVENT_VOLUME_BARRIER_OP,
+                              dict, output);
+
+        ret = glusterfs_translator_info_response_send (req, ret,
+                                                       msg, output);
+out:
+        if (dict)
+                dict_unref (dict);
+        free (xlator_req.input.input_val); // malloced by xdr
+        if (output)
+                dict_unref (output);
+        free (xlator_req.name); //malloced by xdr
+
+        return ret;
+
+}
+int
+glusterfs_handle_rpc_msg (rpcsvc_request_t *req)
+{
+        int ret = -1;
+        /* for now, nothing */
         return ret;
 }
 
-rpcclnt_cb_actor_t gluster_cbk_actors[] = {
+rpcclnt_cb_actor_t mgmt_cbk_actors[] = {
         [GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
+        [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
+                                 mgmt_cbk_event},
 };
 
 
@@ -903,7 +1219,7 @@ struct rpcclnt_cb_program mgmt_cbk_prog = {
         .progname  = "GlusterFS Callback",
         .prognum   = GLUSTER_CBK_PROGRAM,
         .progver   = GLUSTER_CBK_VERSION,
-        .actors    = gluster_cbk_actors,
+        .actors    = mgmt_cbk_actors,
         .numactors = GF_CBK_MAXVALUE,
 };
 
@@ -929,6 +1245,7 @@ char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
         [GF_HNDSK_SETVOLUME]    = "SETVOLUME",
         [GF_HNDSK_GETSPEC]      = "GETSPEC",
         [GF_HNDSK_PING]         = "PING",
+        [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
 };
 
 rpc_clnt_prog_t clnt_handshake_prog = {
@@ -939,11 +1256,15 @@ rpc_clnt_prog_t clnt_handshake_prog = {
 };
 
 rpcsvc_actor_t glusterfs_actors[] = {
-        [GLUSTERD_BRICK_NULL]        = { "NULL",    GLUSTERD_BRICK_NULL, glusterfs_handle_rpc_msg, NULL, NULL, 0},
-        [GLUSTERD_BRICK_TERMINATE] = { "TERMINATE", GLUSTERD_BRICK_TERMINATE, glusterfs_handle_rpc_msg, NULL, NULL, 0},
-        [GLUSTERD_BRICK_XLATOR_INFO] = { "TRANSLATOR INFO", GLUSTERD_BRICK_XLATOR_INFO, glusterfs_handle_rpc_msg, NULL, NULL, 0},
-        [GLUSTERD_BRICK_XLATOR_HEAL] = { "TRANSLATOR HEAL", GLUSTERD_BRICK_XLATOR_HEAL, glusterfs_handle_rpc_msg, NULL, NULL, 0},
-        [GLUSTERD_BRICK_STATUS] = {"STATUS", GLUSTERD_BRICK_STATUS, glusterfs_handle_rpc_msg, NULL, NULL, 0}
+        [GLUSTERD_BRICK_NULL]          = {"NULL",              GLUSTERD_BRICK_NULL,          glusterfs_handle_rpc_msg,             NULL, 0, DRC_NA},
+        [GLUSTERD_BRICK_TERMINATE]     = {"TERMINATE",         GLUSTERD_BRICK_TERMINATE,     glusterfs_handle_terminate,           NULL, 0, DRC_NA},
+        [GLUSTERD_BRICK_XLATOR_INFO]   = {"TRANSLATOR INFO",   GLUSTERD_BRICK_XLATOR_INFO,   glusterfs_handle_translator_info_get, NULL, 0, DRC_NA},
+        [GLUSTERD_BRICK_XLATOR_OP]     = {"TRANSLATOR OP",     GLUSTERD_BRICK_XLATOR_OP,     glusterfs_handle_translator_op,       NULL, 0, DRC_NA},
+        [GLUSTERD_BRICK_STATUS]        = {"STATUS",            GLUSTERD_BRICK_STATUS,        glusterfs_handle_brick_status,        NULL, 0, DRC_NA},
+        [GLUSTERD_BRICK_XLATOR_DEFRAG] = {"TRANSLATOR DEFRAG", GLUSTERD_BRICK_XLATOR_DEFRAG, glusterfs_handle_defrag,              NULL, 0, DRC_NA},
+        [GLUSTERD_NODE_PROFILE]        = {"NFS PROFILE",       GLUSTERD_NODE_PROFILE,        glusterfs_handle_nfs_profile,         NULL, 0, DRC_NA},
+        [GLUSTERD_NODE_STATUS]         = {"NFS STATUS",        GLUSTERD_NODE_STATUS,         glusterfs_handle_node_status,         NULL, 0, DRC_NA},
+        [GLUSTERD_VOLUME_BARRIER_OP]   = {"VOLUME BARRIER OP", GLUSTERD_VOLUME_BARRIER_OP,   glusterfs_handle_volume_barrier_op,   NULL, 0, DRC_NA},
 };
 
 struct rpcsvc_program glusterfs_mop_prog = {
@@ -952,6 +1273,7 @@ struct rpcsvc_program glusterfs_mop_prog = {
         .progver   = GD_BRICK_VERSION,
         .actors    = glusterfs_actors,
         .numactors = GLUSTERD_BRICK_MAXVALUE,
+	.synctask  = _gf_true,
 };
 
 int
@@ -1004,161 +1326,17 @@ out:
         if (iobref)
                 iobref_unref (iobref);
 
+        if (iobuf)
+                iobuf_unref (iobuf);
         return ret;
 }
 
 
 /* XXX: move these into @ctx */
-static char oldvolfile[131072];
+static char *oldvolfile = NULL;
 static int oldvollen = 0;
 
-static int
-xlator_equal_rec (xlator_t *xl1, xlator_t *xl2)
-{
-        xlator_list_t *trav1 = NULL;
-        xlator_list_t *trav2 = NULL;
-        int            ret   = 0;
-
-        if (xl1 == NULL || xl2 == NULL) {
-                gf_log ("xlator", GF_LOG_DEBUG, "invalid argument");
-                return -1;
-        }
-
-        trav1 = xl1->children;
-        trav2 = xl2->children;
-
-        while (trav1 && trav2) {
-                ret = xlator_equal_rec (trav1->xlator, trav2->xlator);
-                if (ret) {
-                        gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                                "xlators children not equal");
-                        goto out;
-                }
-
-                trav1 = trav1->next;
-                trav2 = trav2->next;
-        }
-
-        if (trav1 || trav2) {
-                ret = -1;
-                goto out;
-        }
-
-        if (strcmp (xl1->name, xl2->name)) {
-                ret = -1;
-                goto out;
-        }
-out :
-        return ret;
-}
-
-static gf_boolean_t
-is_graph_topology_equal (glusterfs_graph_t *graph1,
-                                glusterfs_graph_t *graph2)
-{
-        xlator_t    *trav1    = NULL;
-        xlator_t    *trav2    = NULL;
-        gf_boolean_t ret      = _gf_true;
-
-        trav1 = graph1->first;
-        trav2 = graph2->first;
-
-        ret = xlator_equal_rec (trav1, trav2);
-
-        if (ret) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                        "graphs are not equal");
-                ret = _gf_false;
-                goto out;
-        }
-
-        ret = _gf_true;
-        gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                "graphs are equal");
-
-out:
-        return ret;
-}
-
-/* Function has 3types of return value 0, -ve , 1
- *   return 0          =======> reconfiguration of options has succeeded
- *   return 1          =======> the graph has to be reconstructed and all the xlators should be inited
- *   return -1(or -ve) =======> Some Internal Error occurred during the operation
- */
-static int
-glusterfs_volfile_reconfigure (FILE *newvolfile_fp)
-{
-        glusterfs_graph_t *oldvolfile_graph = NULL;
-        glusterfs_graph_t *newvolfile_graph = NULL;
-        FILE              *oldvolfile_fp    = NULL;
-        glusterfs_ctx_t   *ctx              = NULL;
-
-        int ret = -1;
-
-        oldvolfile_fp = tmpfile ();
-        if (!oldvolfile_fp)
-                goto out;
-
-        if (!oldvollen) {
-                ret = 1; // Has to call INIT for the whole graph
-                goto out;
-        }
-        fwrite (oldvolfile, oldvollen, 1, oldvolfile_fp);
-        fflush (oldvolfile_fp);
-
-
-        oldvolfile_graph = glusterfs_graph_construct (oldvolfile_fp);
-        if (!oldvolfile_graph) {
-                goto out;
-        }
-
-        newvolfile_graph = glusterfs_graph_construct (newvolfile_fp);
-        if (!newvolfile_graph) {
-                goto out;
-        }
-
-        if (!is_graph_topology_equal (oldvolfile_graph,
-                                      newvolfile_graph)) {
-
-                ret = 1;
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                        "Graph topology not equal(should call INIT)");
-                goto out;
-        }
-
-        gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                "Only options have changed in the new "
-                "graph");
-
-        ctx = glusterfs_ctx_get ();
-
-        if (!ctx) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
-                        "glusterfs_ctx_get() returned NULL");
-                goto out;
-        }
-
-        oldvolfile_graph = ctx->active;
-
-        if (!oldvolfile_graph) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
-                        "glusterfs_ctx->active is NULL");
-                goto out;
-        }
-
-        /* */
-        ret = glusterfs_graph_reconfigure (oldvolfile_graph,
-                                           newvolfile_graph);
-        if (ret) {
-                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
-                        "Could not reconfigure new options in old graph");
-                goto out;
-        }
 
-        ret = 0;
-out:
-        return ret;
-}
 
 int
 mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
@@ -1170,6 +1348,7 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         int                      ret   = 0;
         ssize_t                  size = 0;
         FILE                    *tmpfp = NULL;
+        char                    *volfilebuf = NULL;
 
         frame = myframe;
         ctx = frame->this->ctx;
@@ -1189,7 +1368,7 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         if (-1 == rsp.op_ret) {
                 gf_log (frame->this->name, GF_LOG_ERROR,
                         "failed to get the 'volume file' from server");
-                ret = -1;
+                ret = rsp.op_errno;
                 goto out;
         }
 
@@ -1210,6 +1389,10 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
 
         fwrite (rsp.spec, size, 1, tmpfp);
         fflush (tmpfp);
+        if (ferror (tmpfp)) {
+                ret = -1;
+                goto out;
+        }
 
         /*  Check if only options have changed. No need to reload the
         *  volfile if topology hasn't changed.
@@ -1219,10 +1402,19 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         *  return -1(or -ve) =======> Some Internal Error occurred during the operation
         */
 
-        ret = glusterfs_volfile_reconfigure (tmpfp);
+        ret = glusterfs_volfile_reconfigure (oldvollen, tmpfp, ctx, oldvolfile);
         if (ret == 0) {
                 gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
                         "No need to re-load volfile, reconfigure done");
+                if (oldvolfile)
+                        volfilebuf = GF_REALLOC (oldvolfile, size);
+                else
+                        volfilebuf = GF_CALLOC (1, size, gf_common_mt_char);
+                if (!volfilebuf) {
+                        ret = -1;
+                        goto out;
+                }
+                oldvolfile = volfilebuf;
                 oldvollen = size;
                 memcpy (oldvolfile, rsp.spec, size);
                 goto out;
@@ -1234,21 +1426,41 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
         ret = glusterfs_process_volfp (ctx, tmpfp);
+        /* tmpfp closed */
+        tmpfp = NULL;
         if (ret)
                 goto out;
 
+        if (oldvolfile)
+                volfilebuf = GF_REALLOC (oldvolfile, size);
+        else
+                volfilebuf = GF_CALLOC (1, size, gf_common_mt_char);
+
+        if (!volfilebuf) {
+                ret = -1;
+                goto out;
+        }
+        oldvolfile = volfilebuf;
         oldvollen = size;
         memcpy (oldvolfile, rsp.spec, size);
         if (!is_mgmt_rpc_reconnect) {
                 glusterfs_mgmt_pmap_signin (ctx);
-                is_mgmt_rpc_reconnect = 1;
+                is_mgmt_rpc_reconnect =  _gf_true;
         }
 
 out:
         STACK_DESTROY (frame->root);
 
-        if (rsp.spec)
-                free (rsp.spec);
+        free (rsp.spec);
+
+        emancipate (ctx, ret);
+
+        // Stop if server is running at an unsupported op-version
+        if (ENOTSUP == ret) {
+                gf_log ("mgmt", GF_LOG_ERROR, "Server is operating at an "
+                        "op-version which is not supported");
+                cleanup_and_exit (0);
+        }
 
         if (ret && ctx && !ctx->active) {
                 /* Do it only for the first time */
@@ -1259,6 +1471,11 @@ out:
                         ctx->cmd_args.volfile_id);
                 cleanup_and_exit (0);
         }
+
+
+        if (tmpfp)
+                fclose (tmpfp);
+
         return 0;
 }
 
@@ -1270,6 +1487,7 @@ glusterfs_volfile_fetch (glusterfs_ctx_t *ctx)
         gf_getspec_req    req = {0, };
         int               ret = 0;
         call_frame_t     *frame = NULL;
+        dict_t           *dict = NULL;
 
         cmd_args = &ctx->cmd_args;
 
@@ -1278,59 +1496,235 @@ glusterfs_volfile_fetch (glusterfs_ctx_t *ctx)
         req.key = cmd_args->volfile_id;
         req.flags = 0;
 
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        // Set the supported min and max op-versions, so glusterd can make a
+        // decision
+        ret = dict_set_int32 (dict, "min-op-version", GD_OP_VERSION_MIN);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, "max-op-version", GD_OP_VERSION_MAX);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
+                                           &req.xdata.xdata_len);
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to serialize dictionary");
+                goto out;
+        }
+
         ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
                                    GF_HNDSK_GETSPEC, mgmt_getspec_cbk,
                                    (xdrproc_t)xdr_gf_getspec_req);
+out:
+        return ret;
+}
+
+int32_t
+mgmt_event_notify_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+        gf_event_notify_rsp      rsp   = {0,};
+        call_frame_t            *frame = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
+        int                      ret   = 0;
+
+        frame = myframe;
+        ctx = frame->this->ctx;
+
+        if (-1 == req->rpc_status) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_event_notify_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+                ret   = -1;
+                goto out;
+        }
+
+        if (-1 == rsp.op_ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "failed to get the rsp from server");
+                ret = -1;
+                goto out;
+        }
+out:
+        free (rsp.dict.dict_val); //malloced by xdr
         return ret;
+
 }
 
+int32_t
+glusterfs_rebalance_event_notify_cbk (struct rpc_req *req, struct iovec *iov,
+                                      int count, void *myframe)
+{
+        gf_event_notify_rsp      rsp   = {0,};
+        call_frame_t            *frame = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
+        int                      ret   = 0;
+
+        frame = myframe;
+        ctx = frame->this->ctx;
+
+        if (-1 == req->rpc_status) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "failed to get the rsp from server");
+                ret = -1;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_event_notify_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+                ret   = -1;
+                goto out;
+        }
+
+        if (-1 == rsp.op_ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Received error (%s) from server",
+                        strerror (rsp.op_errno));
+                ret = -1;
+                goto out;
+        }
+out:
+        free (rsp.dict.dict_val); //malloced by xdr
+        return ret;
+
+}
+
+int32_t
+glusterfs_rebalance_event_notify (dict_t *dict)
+{
+        glusterfs_ctx_t         *ctx = NULL;
+        gf_event_notify_req      req = {0,};
+        int32_t                  ret = -1;
+        cmd_args_t              *cmd_args = NULL;
+        call_frame_t            *frame = NULL;
+
+        ctx = glusterfsd_ctx;
+        cmd_args = &ctx->cmd_args;
+
+        frame = create_frame (THIS, ctx->pool);
+
+        req.op = GF_EN_DEFRAG_STATUS;
+
+        if (dict) {
+                ret = dict_set_str (dict, "volname", cmd_args->volfile_id);
+                if (ret)
+                        gf_log ("", GF_LOG_ERROR, "failed to set volname");
+
+                ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                                   &req.dict.dict_len);
+        }
+
+        ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
+                                   GF_HNDSK_EVENT_NOTIFY,
+                                   glusterfs_rebalance_event_notify_cbk,
+                                   (xdrproc_t)xdr_gf_event_notify_req);
+
+        GF_FREE (req.dict.dict_val);
+
+        STACK_DESTROY (frame->root);
+        return ret;
+}
 
 static int
 mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                  void *data)
 {
-        xlator_t        *this = NULL;
-        cmd_args_t      *cmd_args = NULL;
-        glusterfs_ctx_t *ctx = NULL;
+        xlator_t         *this = NULL;
+        glusterfs_ctx_t  *ctx = NULL;
         int              ret = 0;
+        server_cmdline_t *server = NULL;
+        rpc_transport_t  *rpc_trans = NULL;
+        int              need_term = 0;
+        int              emval = 0;
 
         this = mydata;
+        rpc_trans = rpc->conn.trans;
         ctx = this->ctx;
-        cmd_args = &ctx->cmd_args;
+
         switch (event) {
         case RPC_CLNT_DISCONNECT:
                 if (!ctx->active) {
-                        cmd_args->max_connect_attempts--;
                         gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
-                                "failed to connect with remote-host: %s",
+                                "failed to connect with remote-host: %s (%s)",
+                                ctx->cmd_args.volfile_server,
                                 strerror (errno));
+                        server = ctx->cmd_args.curr_server;
+                        if (server->list.next == &ctx->cmd_args.volfile_servers) {
+                                need_term = 1;
+                                emval = ENOTCONN;
+                                gf_log("glusterfsd-mgmt", GF_LOG_INFO,
+                                       "Exhausted all volfile servers");
+                                break;
+                        }
+                        server = list_entry (server->list.next, typeof(*server),
+                                             list);
+                        ctx->cmd_args.curr_server = server;
+                        ctx->cmd_args.volfile_server = server->volfile_server;
+
+                        ret = dict_set_str (rpc_trans->options,
+                                            "remote-host",
+                                            server->volfile_server);
+                        if (ret != 0) {
+                                gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
+                                        "failed to set remote-host: %s",
+                                        server->volfile_server);
+                                need_term = 1;
+                                emval = ENOTCONN;
+                                break;
+                        }
                         gf_log ("glusterfsd-mgmt", GF_LOG_INFO,
-                                "%d connect attempts left",
-                                cmd_args->max_connect_attempts);
-                        if (0 >= cmd_args->max_connect_attempts)
-                                cleanup_and_exit (1);
+                                "connecting to next volfile server %s",
+                                server->volfile_server);
                 }
                 break;
         case RPC_CLNT_CONNECT:
                 rpc_clnt_set_connected (&((struct rpc_clnt*)ctx->mgmt)->conn);
 
                 ret = glusterfs_volfile_fetch (ctx);
-                if (ret && ctx && (ctx->active == NULL)) {
-                        /* Do it only for the first time */
-                        /* Exit the process.. there is some wrong options */
-                        gf_log ("mgmt", GF_LOG_ERROR,
-                                "failed to fetch volume file (key:%s)",
-                                ctx->cmd_args.volfile_id);
-                        cleanup_and_exit (0);
+                if (ret) {
+                        emval = ret;
+                        if (!ctx->active) {
+                                need_term = 1;
+                                gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
+                                        "failed to fetch volume file (key:%s)",
+                                        ctx->cmd_args.volfile_id);
+                                break;
+
+                        }
                 }
 
                 if (is_mgmt_rpc_reconnect)
                         glusterfs_mgmt_pmap_signin (ctx);
+
                 break;
         default:
                 break;
         }
 
+        if (need_term) {
+                emancipate (ctx, emval);
+                cleanup_and_exit (1);
+        }
+
         return 0;
 }
 
@@ -1381,7 +1775,7 @@ glusterfs_listener_init (glusterfs_ctx_t *ctx)
         if (ret)
                 goto out;
 
-        rpc = rpcsvc_init (THIS, ctx, options);
+        rpc = rpcsvc_init (THIS, ctx, options, 8);
         if (rpc == NULL) {
                 goto out;
         }
@@ -1409,6 +1803,66 @@ out:
 }
 
 int
+glusterfs_listener_stop (glusterfs_ctx_t *ctx)
+{
+        cmd_args_t              *cmd_args = NULL;
+        rpcsvc_t                *rpc = NULL;
+        rpcsvc_listener_t       *listener = NULL;
+        rpcsvc_listener_t       *next = NULL;
+        int                     ret = 0;
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (ctx);
+
+        rpc = ctx->listener;
+        ctx->listener = NULL;
+
+        (void) rpcsvc_program_unregister(rpc, &glusterfs_mop_prog);
+
+        list_for_each_entry_safe (listener, next, &rpc->listeners, list) {
+                rpcsvc_listener_destroy (listener);
+        }
+
+        (void) rpcsvc_unregister_notify (rpc, glusterfs_rpcsvc_notify, THIS);
+
+        GF_FREE (rpc);
+
+        cmd_args = &ctx->cmd_args;
+        if (cmd_args->sock_file) {
+                ret = unlink (cmd_args->sock_file);
+                if (ret && (ENOENT == errno)) {
+                        ret = 0;
+                }
+        }
+
+        if (ret) {
+                this = THIS;
+                gf_log (this->name, GF_LOG_ERROR, "Failed to unlink listener "
+                        "socket %s, error: %s", cmd_args->sock_file,
+                        strerror (errno));
+        }
+        return ret;
+}
+
+int
+glusterfs_mgmt_notify (int32_t op, void *data, ...)
+{
+        int ret = 0;
+        switch (op)
+        {
+                case GF_EN_DEFRAG_STATUS:
+                        ret = glusterfs_rebalance_event_notify ((dict_t*) data);
+                        break;
+
+                default:
+                        gf_log ("", GF_LOG_ERROR, "Invalid op");
+                        break;
+        }
+
+        return ret;
+}
+
+int
 glusterfs_mgmt_init (glusterfs_ctx_t *ctx)
 {
         cmd_args_t              *cmd_args = NULL;
@@ -1434,7 +1888,7 @@ glusterfs_mgmt_init (glusterfs_ctx_t *ctx)
         if (ret)
                 goto out;
 
-        rpc = rpc_clnt_new (options, THIS->ctx, THIS->name);
+        rpc = rpc_clnt_new (options, THIS->ctx, THIS->name, 8);
         if (!rpc) {
                 ret = -1;
                 gf_log (THIS->name, GF_LOG_WARNING, "failed to create rpc clnt");
@@ -1443,16 +1897,20 @@ glusterfs_mgmt_init (glusterfs_ctx_t *ctx)
 
         ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "failed to register notify function");
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to register notify function");
                 goto out;
         }
 
-        ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog);
+        ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog, THIS);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "failed to register callback function");
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to register callback function");
                 goto out;
         }
 
+        ctx->notify = glusterfs_mgmt_notify;
+
         /* This value should be set before doing the 'rpc_clnt_start()' as
            the notify function uses this variable */
         ctx->mgmt = rpc;
@@ -1531,7 +1989,7 @@ mgmt_pmap_signin_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
-        ctx = glusterfs_ctx_get ();
+        ctx = glusterfsd_ctx;
         cmd_args = &ctx->cmd_args;
 
         if (!cmd_args->brick_port2) {
@@ -1600,7 +2058,7 @@ mgmt_pmap_signout_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
-        ctx = glusterfs_ctx_get ();
+        ctx = glusterfsd_ctx;
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_pmap_signout_rsp);
         if (ret < 0) {
                 gf_log (THIS->name, GF_LOG_ERROR, "XDR decoding failed");
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index c233a551d..3cb8f0f51 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -77,6 +67,7 @@
 #include <fnmatch.h>
 #include "rpc-clnt.h"
 #include "syncop.h"
+#include "client_t.h"
 
 #include "daemon.h"
 
@@ -89,14 +80,15 @@
 static char gf_doc[] = "";
 static char argp_doc[] = "--volfile-server=SERVER [MOUNT-POINT]\n"       \
         "--volfile=VOLFILE [MOUNT-POINT]";
-const char *argp_program_version = "" \
-        PACKAGE_NAME" "PACKAGE_VERSION" built on "__DATE__" "__TIME__ \
-        "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION "\n"  \
-        "Copyright (c) 2006-2011 Gluster Inc. "             \
-        "<http://www.gluster.com>\n"                                \
-        "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"              \
-        "You may redistribute copies of GlusterFS under the terms of "\
-        "the GNU General Public License.";
+const char *argp_program_version = ""
+        PACKAGE_NAME" "PACKAGE_VERSION" built on "__DATE__" "__TIME__
+        "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION "\n"
+        "Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com/>\n"
+        "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"
+        "It is licensed to you under your choice of the GNU Lesser\n"
+        "General Public License, version 3 or any later version (LGPLv3\n"
+        "or later), or the GNU General Public License, version 2 (GPLv2),\n"
+        "in all cases as published by the Free Software Foundation.";
 const char *argp_program_bug_address = "<" PACKAGE_BUGREPORT ">";
 
 static error_t parse_opts (int32_t key, char *arg, struct argp_state *_state);
@@ -106,10 +98,6 @@ static struct argp_option gf_options[] = {
         {"volfile-server", ARGP_VOLFILE_SERVER_KEY, "SERVER", 0,
          "Server to get the volume file from.  This option overrides "
          "--volfile option"},
-        {"volfile-max-fetch-attempts", ARGP_VOLFILE_MAX_FETCH_ATTEMPTS,
-         "MAX-ATTEMPTS", 0, "Maximum number of connect attempts to server. "
-         "This option should be provided with --volfile-server option"
-         "[default: 1]"},
         {"volfile", ARGP_VOLUME_FILE_KEY, "VOLFILE", 0,
          "File to use as VOLUME_FILE"},
         {"spec-file", ARGP_VOLUME_FILE_KEY, "VOLFILE", OPTION_HIDDEN,
@@ -117,7 +105,7 @@ static struct argp_option gf_options[] = {
 
         {"log-level", ARGP_LOG_LEVEL_KEY, "LOGLEVEL", 0,
          "Logging severity.  Valid options are DEBUG, INFO, WARNING, ERROR, "
-         "CRITICAL and NONE [default: INFO]"},
+         "CRITICAL, TRACE and NONE [default: INFO]"},
         {"log-file", ARGP_LOG_FILE_KEY, "LOGFILE", 0,
          "File to use for logging [default: "
          DEFAULT_LOG_FILE_DIRECTORY "/" PACKAGE_NAME ".log" "]"},
@@ -142,15 +130,25 @@ static struct argp_option gf_options[] = {
         {"debug", ARGP_DEBUG_KEY, 0, 0,
          "Run in debug mode.  This option sets --no-daemon, --log-level "
          "to DEBUG and --log-file to console"},
-        {"volume-name", ARGP_VOLUME_NAME_KEY, "VOLUME-NAME", 0,
-         "Volume name to be used for MOUNT-POINT [default: top most volume "
-         "in VOLFILE]"},
-        {"xlator-option", ARGP_XLATOR_OPTION_KEY,"VOLUME-NAME.OPTION=VALUE", 0,
-         "Add/override a translator option for a volume with specified value"},
+        {"volume-name", ARGP_VOLUME_NAME_KEY, "XLATOR-NAME", 0,
+         "Translator name to be used for MOUNT-POINT [default: top most volume "
+         "definition in VOLFILE]"},
+        {"xlator-option", ARGP_XLATOR_OPTION_KEY,"XLATOR-NAME.OPTION=VALUE", 0,
+         "Add/override an option for a translator in volume file with specified"
+         " value"},
         {"read-only", ARGP_READ_ONLY_KEY, 0, 0,
          "Mount the filesystem in 'read-only' mode"},
         {"acl", ARGP_ACL_KEY, 0, 0,
          "Mount the filesystem with POSIX ACL support"},
+        {"selinux", ARGP_SELINUX_KEY, 0, 0,
+         "Enable SELinux label (extened attributes) support on inodes"},
+#ifdef GF_LINUX_HOST_OS
+        {"aux-gfid-mount", ARGP_AUX_GFID_MOUNT_KEY, 0, 0,
+         "Enable access to filesystem through gfid directly"},
+#endif
+        {"enable-ino32", ARGP_INODE32_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+         "Use 32-bit inodes when mounting to workaround broken applications"
+         "that don't support 64-bit inodes"},
         {"worm", ARGP_WORM_KEY, 0, 0,
          "Mount the filesystem in 'worm' mode"},
         {"mac-compat", ARGP_MAC_COMPAT_KEY, "BOOL", OPTION_ARG_OPTIONAL,
@@ -165,6 +163,8 @@ static struct argp_option gf_options[] = {
          "Brick name to be registered with Gluster portmapper" },
         {"brick-port", ARGP_BRICK_PORT_KEY, "BRICK-PORT", OPTION_HIDDEN,
          "Brick Port to be registered with Gluster portmapper" },
+	{"fopen-keep-cache", ARGP_FOPEN_KEEP_CACHE_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+	 "Do not purge the cache on file open"},
 
         {0, 0, 0, 0, "Fuse options:"},
         {"direct-io-mode", ARGP_DIRECT_IO_MODE_KEY, "BOOL", OPTION_ARG_OPTIONAL,
@@ -173,9 +173,20 @@ static struct argp_option gf_options[] = {
          "\"on\" for fds not opened with O_RDONLY]"},
         {"entry-timeout", ARGP_ENTRY_TIMEOUT_KEY, "SECONDS", 0,
          "Set entry timeout to SECONDS in fuse kernel module [default: 1]"},
+        {"negative-timeout", ARGP_NEGATIVE_TIMEOUT_KEY, "SECONDS", 0,
+         "Set negative timeout to SECONDS in fuse kernel module [default: 0]"},
         {"attribute-timeout", ARGP_ATTRIBUTE_TIMEOUT_KEY, "SECONDS", 0,
          "Set attribute timeout to SECONDS for inodes in fuse kernel module "
          "[default: 1]"},
+	{"gid-timeout", ARGP_GID_TIMEOUT_KEY, "SECONDS", 0,
+	 "Set auxilary group list timeout to SECONDS for fuse translator "
+	 "[default: 0]"},
+	{"background-qlen", ARGP_FUSE_BACKGROUND_QLEN_KEY, "N", 0,
+	 "Set fuse module's background queue length to N "
+	 "[default: 64]"},
+	{"congestion-threshold", ARGP_FUSE_CONGESTION_THRESHOLD_KEY, "N", 0,
+	 "Set fuse module's congestion threshold to N "
+	 "[default: 48]"},
         {"client-pid", ARGP_CLIENT_PID_KEY, "PID", OPTION_HIDDEN,
          "client will authenticate itself with process id PID to server"},
         {"user-map-root", ARGP_USER_MAP_ROOT_KEY, "USER", OPTION_HIDDEN,
@@ -184,6 +195,13 @@ static struct argp_option gf_options[] = {
          "Dump fuse traffic to PATH"},
         {"volfile-check", ARGP_VOLFILE_CHECK_KEY, 0, 0,
          "Enable strict volume file checking"},
+        {"mem-accounting", ARGP_MEM_ACCOUNTING_KEY, 0, OPTION_HIDDEN,
+         "Enable internal memory accounting"},
+        {"fuse-mountopts", ARGP_FUSE_MOUNTOPTS_KEY, "OPTIONS", OPTION_HIDDEN,
+         "Extra mount options to pass to FUSE"},
+        {"use-readdirp", ARGP_FUSE_USE_READDIRP_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+         "Use readdirp mode in fuse kernel module"
+         " [default: \"off\"]"},
         {0, 0, 0, 0, "Miscellaneous Options:"},
         {0, }
 };
@@ -195,51 +213,41 @@ int glusterfs_pidfile_cleanup (glusterfs_ctx_t *ctx);
 int glusterfs_volumes_init (glusterfs_ctx_t *ctx);
 int glusterfs_mgmt_init (glusterfs_ctx_t *ctx);
 int glusterfs_listener_init (glusterfs_ctx_t *ctx);
+int glusterfs_listener_stop (glusterfs_ctx_t *ctx);
 
-int
-create_fuse_mount (glusterfs_ctx_t *ctx)
+
+static int
+set_fuse_mount_options (glusterfs_ctx_t *ctx, dict_t *options)
 {
         int              ret = 0;
         cmd_args_t      *cmd_args = NULL;
-        xlator_t        *master = NULL;
+        char            *mount_point = NULL;
+        char            cwd[PATH_MAX] = {0,};
 
         cmd_args = &ctx->cmd_args;
 
-        if (!cmd_args->mount_point) {
-                gf_log ("", GF_LOG_TRACE,
-                        "mount point not found, not a client process");
-                return 0;
-        }
-
-        if (ctx->process_mode != GF_CLIENT_PROCESS) {
-                gf_log("glusterfsd", GF_LOG_ERROR,
-                       "Not a client process, not performing mount operation");
-                return -1;
-        }
-
-        master = GF_CALLOC (1, sizeof (*master),
-                            gfd_mt_xlator_t);
-        if (!master)
-                goto err;
-
-        master->name = gf_strdup ("fuse");
-        if (!master->name)
-                goto err;
-
-        if (xlator_set_type (master, "mount/fuse") == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "MOUNT-POINT %s initialization failed",
-                        cmd_args->mount_point);
-                goto err;
-        }
-
-        master->ctx      = ctx;
-        master->options  = get_new_dict ();
-        if (!master->options)
-                goto err;
+        /* Check if mount-point is absolute path,
+         * if not convert to absolute path by concating with CWD
+         */
+        if (cmd_args->mount_point[0] != '/') {
+                if (getcwd (cwd, PATH_MAX) != NULL) {
+                        ret = gf_asprintf (&mount_point, "%s/%s", cwd,
+                                           cmd_args->mount_point);
+                        if (ret == -1) {
+                                gf_log ("glusterfsd", GF_LOG_ERROR,
+                                        "Could not create absolute mountpoint "
+                                        "path");
+                                goto err;
+                        }
+                } else {
+                        gf_log ("glusterfsd", GF_LOG_ERROR,
+                                "Could not get current working directory");
+                        goto err;
+                }
+        } else
+                mount_point = gf_strdup (cmd_args->mount_point);
 
-        ret = dict_set_static_ptr (master->options, ZR_MOUNTPOINT_OPT,
-                                   cmd_args->mount_point);
+        ret = dict_set_dynstr (options, ZR_MOUNTPOINT_OPT, mount_point);
         if (ret < 0) {
                 gf_log ("glusterfsd", GF_LOG_ERROR,
                         "failed to set mount-point to options dictionary");
@@ -247,7 +255,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->fuse_attribute_timeout >= 0) {
-                ret = dict_set_double (master->options, ZR_ATTR_TIMEOUT_OPT,
+                ret = dict_set_double (options, ZR_ATTR_TIMEOUT_OPT,
                                        cmd_args->fuse_attribute_timeout);
 
                 if (ret < 0) {
@@ -259,7 +267,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->fuse_entry_timeout >= 0) {
-                ret = dict_set_double (master->options, ZR_ENTRY_TIMEOUT_OPT,
+                ret = dict_set_double (options, ZR_ENTRY_TIMEOUT_OPT,
                                        cmd_args->fuse_entry_timeout);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -269,8 +277,19 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
                 }
         }
 
+        if (cmd_args->fuse_negative_timeout >= 0) {
+                ret = dict_set_double (options, ZR_NEGATIVE_TIMEOUT_OPT,
+                                       cmd_args->fuse_negative_timeout);
+                if (ret < 0) {
+                        gf_log ("glusterfsd", GF_LOG_ERROR,
+                                "failed to set dict value for key %s",
+                                ZR_NEGATIVE_TIMEOUT_OPT);
+                        goto err;
+                }
+        }
+
         if (cmd_args->client_pid_set) {
-                ret = dict_set_int32 (master->options, "client-pid",
+                ret = dict_set_int32 (options, "client-pid",
                                       cmd_args->client_pid);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -281,7 +300,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->uid_map_root) {
-                ret = dict_set_int32 (master->options, "uid-map-root",
+                ret = dict_set_int32 (options, "uid-map-root",
                                       cmd_args->uid_map_root);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -292,7 +311,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->volfile_check) {
-                ret = dict_set_int32 (master->options, ZR_STRICT_VOLFILE_CHECK,
+                ret = dict_set_int32 (options, ZR_STRICT_VOLFILE_CHECK,
                                       cmd_args->volfile_check);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -303,7 +322,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->dump_fuse) {
-                ret = dict_set_static_ptr (master->options, ZR_DUMP_FUSE,
+                ret = dict_set_static_ptr (options, ZR_DUMP_FUSE,
                                            cmd_args->dump_fuse);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -314,7 +333,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->acl) {
-                ret = dict_set_static_ptr (master->options, "acl", "on");
+                ret = dict_set_static_ptr (options, "acl", "on");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key acl");
@@ -322,8 +341,37 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
                 }
         }
 
+        if (cmd_args->selinux) {
+                ret = dict_set_static_ptr (options, "selinux", "on");
+                if (ret < 0) {
+                        gf_log ("glusterfsd", GF_LOG_ERROR,
+                                "failed to set dict value for key selinux");
+                        goto err;
+                }
+        }
+
+        if (cmd_args->aux_gfid_mount) {
+                ret = dict_set_static_ptr (options, "virtual-gfid-access",
+                                           "on");
+                if (ret < 0) {
+                        gf_log ("glusterfsd", GF_LOG_ERROR,
+                                "failed to set dict value for key "
+                                "aux-gfid-mount");
+                        goto err;
+                }
+        }
+
+        if (cmd_args->enable_ino32) {
+                ret = dict_set_static_ptr (options, "enable-ino32", "on");
+                if (ret < 0) {
+                        gf_log ("glusterfsd", GF_LOG_ERROR,
+                                "failed to set dict value for key enable-ino32");
+                        goto err;
+                }
+        }
+
         if (cmd_args->read_only) {
-                ret = dict_set_static_ptr (master->options, "read-only", "on");
+                ret = dict_set_static_ptr (options, "read-only", "on");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key read-only");
@@ -331,9 +379,66 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
                 }
         }
 
+	switch (cmd_args->fopen_keep_cache) {
+	case GF_OPTION_ENABLE:
+		ret = dict_set_static_ptr(options, "fopen-keep-cache",
+			"on");
+		if (ret < 0) {
+			gf_log("glusterfsd", GF_LOG_ERROR,
+				"failed to set dict value for key "
+				"fopen-keep-cache");
+			goto err;
+		}
+		break;
+	case GF_OPTION_DISABLE:
+		ret = dict_set_static_ptr(options, "fopen-keep-cache",
+			"off");
+		if (ret < 0) {
+			gf_log("glusterfsd", GF_LOG_ERROR,
+				"failed to set dict value for key "
+				"fopen-keep-cache");
+			goto err;
+		}
+		break;
+        case GF_OPTION_DEFERRED: /* default */
+        default:
+                gf_log ("glusterfsd", GF_LOG_DEBUG,
+			"fopen-keep-cache mode %d",
+                        cmd_args->fopen_keep_cache);
+                break;
+	}
+
+	if (cmd_args->gid_timeout) {
+		ret = dict_set_int32(options, "gid-timeout",
+			cmd_args->gid_timeout);
+		if (ret < 0) {
+			gf_log("glusterfsd", GF_LOG_ERROR, "failed to set dict "
+				"value for key gid-timeout");
+			goto err;
+		}
+	}
+	if (cmd_args->background_qlen) {
+		ret = dict_set_int32 (options, "background-qlen",
+                                      cmd_args->background_qlen);
+		if (ret < 0) {
+			gf_log("glusterfsd", GF_LOG_ERROR, "failed to set dict "
+                               "value for key background-qlen");
+			goto err;
+		}
+	}
+	if (cmd_args->congestion_threshold) {
+		ret = dict_set_int32 (options, "congestion-threshold",
+                                      cmd_args->congestion_threshold);
+		if (ret < 0) {
+			gf_log("glusterfsd", GF_LOG_ERROR, "failed to set dict "
+                               "value for key congestion-threshold");
+			goto err;
+		}
+	}
+
         switch (cmd_args->fuse_direct_io_mode) {
         case GF_OPTION_DISABLE: /* disable */
-                ret = dict_set_static_ptr (master->options, ZR_DIRECT_IO_OPT,
+                ret = dict_set_static_ptr (options, ZR_DIRECT_IO_OPT,
                                            "disable");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -343,7 +448,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
                 }
                 break;
         case GF_OPTION_ENABLE: /* enable */
-                ret = dict_set_static_ptr (master->options, ZR_DIRECT_IO_OPT,
+                ret = dict_set_static_ptr (options, ZR_DIRECT_IO_OPT,
                                            "enable");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -360,8 +465,8 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (!cmd_args->no_daemon_mode) {
-                 ret = dict_set_static_ptr (master->options, "sync-mtab",
-                                            "enable");
+                ret = dict_set_static_ptr (options, "sync-to-mount",
+                                           "enable");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key sync-mtab");
@@ -369,6 +474,77 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
                 }
         }
 
+        if (cmd_args->use_readdirp) {
+                ret = dict_set_str (options, "use-readdirp",
+                                    cmd_args->use_readdirp);
+                if (ret < 0) {
+                        gf_log ("glusterfsd", GF_LOG_ERROR, "failed to set dict"
+                                " value for key use-readdirp");
+                        goto err;
+                }
+        }
+        ret = 0;
+err:
+        return ret;
+}
+
+int
+create_fuse_mount (glusterfs_ctx_t *ctx)
+{
+        int              ret = 0;
+        cmd_args_t      *cmd_args = NULL;
+        xlator_t        *master = NULL;
+
+        cmd_args = &ctx->cmd_args;
+
+        if (!cmd_args->mount_point) {
+                gf_log ("", GF_LOG_TRACE,
+                        "mount point not found, not a client process");
+                return 0;
+        }
+
+        if (ctx->process_mode != GF_CLIENT_PROCESS) {
+                gf_log("glusterfsd", GF_LOG_ERROR,
+                       "Not a client process, not performing mount operation");
+                return -1;
+        }
+
+        master = GF_CALLOC (1, sizeof (*master),
+                            gfd_mt_xlator_t);
+        if (!master)
+                goto err;
+
+        master->name = gf_strdup ("fuse");
+        if (!master->name)
+                goto err;
+
+        if (xlator_set_type (master, "mount/fuse") == -1) {
+                gf_log ("glusterfsd", GF_LOG_ERROR,
+                        "MOUNT-POINT %s initialization failed",
+                        cmd_args->mount_point);
+                goto err;
+        }
+
+        master->ctx      = ctx;
+        master->options  = get_new_dict ();
+        if (!master->options)
+                goto err;
+
+        ret = set_fuse_mount_options (ctx, master->options);
+        if (ret)
+                goto err;
+
+        if (cmd_args->fuse_mountopts) {
+                ret = dict_set_static_ptr (master->options, ZR_FUSE_MOUNTOPTS,
+                                           cmd_args->fuse_mountopts);
+                if (ret < 0) {
+                        gf_log ("glusterfsd", GF_LOG_ERROR,
+                                "failed to set dict value for key %s",
+                                ZR_FUSE_MOUNTOPTS);
+                        goto err;
+                }
+        }
+
         ret = xlator_init (master);
         if (ret) {
                 gf_log ("", GF_LOG_DEBUG, "failed to initialize fuse translator");
@@ -384,7 +560,7 @@ err:
                 xlator_destroy (master);
         }
 
-        return -1;
+        return 1;
 }
 
 
@@ -420,7 +596,58 @@ get_volfp (glusterfs_ctx_t *ctx)
 }
 
 static int
-gf_remember_xlator_option (struct list_head *options, char *arg)
+gf_remember_backup_volfile_server (char *arg)
+{
+        glusterfs_ctx_t         *ctx = NULL;
+        cmd_args_t              *cmd_args = NULL;
+        int                      ret = -1;
+        server_cmdline_t        *server = NULL;
+
+        ctx = glusterfsd_ctx;
+        if (!ctx)
+                goto out;
+        cmd_args = &ctx->cmd_args;
+
+        if(!cmd_args)
+                goto out;
+
+        server = GF_CALLOC (1, sizeof (server_cmdline_t),
+                            gfd_mt_server_cmdline_t);
+        if (!server)
+                goto out;
+
+        INIT_LIST_HEAD(&server->list);
+
+        server->volfile_server = gf_strdup(arg);
+
+        if (!cmd_args->volfile_server) {
+                cmd_args->volfile_server = server->volfile_server;
+                cmd_args->curr_server = server;
+        }
+
+        if (!server->volfile_server) {
+                gf_log ("", GF_LOG_WARNING,
+                        "xlator option %s is invalid", arg);
+                goto out;
+        }
+
+        list_add_tail (&server->list, &cmd_args->volfile_servers);
+
+        ret = 0;
+out:
+        if (ret == -1) {
+                if (server) {
+                        GF_FREE (server->volfile_server);
+                        GF_FREE (server);
+                }
+        }
+
+        return ret;
+
+}
+
+static int
+gf_remember_xlator_option (char *arg)
 {
         glusterfs_ctx_t         *ctx = NULL;
         cmd_args_t              *cmd_args  = NULL;
@@ -429,7 +656,7 @@ gf_remember_xlator_option (struct list_head *options, char *arg)
         char                    *dot = NULL;
         char                    *equals = NULL;
 
-        ctx = glusterfs_ctx_get ();
+        ctx = glusterfsd_ctx;
         cmd_args = &ctx->cmd_args;
 
         option = GF_CALLOC (1, sizeof (xlator_cmdline_option_t),
@@ -481,12 +708,9 @@ gf_remember_xlator_option (struct list_head *options, char *arg)
 out:
         if (ret == -1) {
                 if (option) {
-                        if (option->volume)
-                                GF_FREE (option->volume);
-                        if (option->key)
-                                GF_FREE (option->key);
-                        if (option->value)
-                                GF_FREE (option->value);
+                        GF_FREE (option->volume);
+                        GF_FREE (option->key);
+                        GF_FREE (option->value);
 
                         GF_FREE (option);
                 }
@@ -514,19 +738,8 @@ parse_opts (int key, char *arg, struct argp_state *state)
 
         switch (key) {
         case ARGP_VOLFILE_SERVER_KEY:
-                cmd_args->volfile_server = gf_strdup (arg);
-                break;
+                gf_remember_backup_volfile_server (arg);
 
-        case ARGP_VOLFILE_MAX_FETCH_ATTEMPTS:
-                n = 0;
-
-                if (gf_string2uint_base10 (arg, &n) == 0) {
-                        cmd_args->max_connect_attempts = n;
-                        break;
-                }
-
-                argp_failure (state, -1, 0,
-                              "Invalid limit on connect attempts %s", arg);
                 break;
 
         case ARGP_READ_ONLY_KEY:
@@ -535,6 +748,20 @@ parse_opts (int key, char *arg, struct argp_state *state)
 
         case ARGP_ACL_KEY:
                 cmd_args->acl = 1;
+                gf_remember_xlator_option ("*-md-cache.cache-posix-acl=true");
+                break;
+
+        case ARGP_SELINUX_KEY:
+                cmd_args->selinux = 1;
+                gf_remember_xlator_option ("*-md-cache.cache-selinux=true");
+                break;
+
+        case ARGP_AUX_GFID_MOUNT_KEY:
+                cmd_args->aux_gfid_mount = 1;
+                break;
+
+        case ARGP_INODE32_KEY:
+                cmd_args->enable_ino32 = 1;
                 break;
 
         case ARGP_WORM_KEY:
@@ -556,8 +783,7 @@ parse_opts (int key, char *arg, struct argp_state *state)
                 break;
 
         case ARGP_VOLUME_FILE_KEY:
-                if (cmd_args->volfile)
-                        GF_FREE (cmd_args->volfile);
+                GF_FREE (cmd_args->volfile);
 
                 if (arg[0] != '/') {
                         pwd = getcwd (NULL, PATH_MAX);
@@ -679,6 +905,18 @@ parse_opts (int key, char *arg, struct argp_state *state)
                 argp_failure (state, -1, 0, "unknown entry timeout %s", arg);
                 break;
 
+        case ARGP_NEGATIVE_TIMEOUT_KEY:
+                d = 0.0;
+
+                gf_string2double (arg, &d);
+                if (!(d < 0.0)) {
+                        cmd_args->fuse_negative_timeout = d;
+                        break;
+                }
+
+                argp_failure (state, -1, 0, "unknown negative timeout %s", arg);
+                break;
+
         case ARGP_ATTRIBUTE_TIMEOUT_KEY:
                 d = 0.0;
 
@@ -720,8 +958,9 @@ parse_opts (int key, char *arg, struct argp_state *state)
                 break;
 
         case ARGP_XLATOR_OPTION_KEY:
-                if (gf_remember_xlator_option (&cmd_args->xlator_options, arg))
-                        argp_failure (state, -1, 0, "invalid xlator option  %s", arg);
+                if (gf_remember_xlator_option (arg))
+                        argp_failure (state, -1, 0, "invalid xlator option  %s",
+                                      arg);
 
                 break;
 
@@ -762,7 +1001,71 @@ parse_opts (int key, char *arg, struct argp_state *state)
                 argp_failure (state, -1, 0,
                               "unknown brick (listen) port %s", arg);
                 break;
-        }
+
+        case ARGP_MEM_ACCOUNTING_KEY:
+                /* TODO: it should have got handled much earlier */
+		//gf_mem_acct_enable_set (THIS->ctx);
+                break;
+
+	case ARGP_FOPEN_KEEP_CACHE_KEY:
+                if (!arg)
+                        arg = "on";
+
+                if (gf_string2boolean (arg, &b) == 0) {
+                        cmd_args->fopen_keep_cache = b;
+
+                        break;
+                }
+
+                argp_failure (state, -1, 0,
+                              "unknown cache setting \"%s\"", arg);
+
+		break;
+
+	case ARGP_GID_TIMEOUT_KEY:
+		if (!gf_string2int(arg, &cmd_args->gid_timeout))
+			break;
+
+		argp_failure(state, -1, 0, "unknown group list timeout %s", arg);
+		break;
+        case ARGP_FUSE_BACKGROUND_QLEN_KEY:
+                if (!gf_string2int (arg, &cmd_args->background_qlen))
+                        break;
+
+                argp_failure (state, -1, 0,
+                              "unknown background qlen option %s", arg);
+                break;
+        case ARGP_FUSE_CONGESTION_THRESHOLD_KEY:
+                if (!gf_string2int (arg, &cmd_args->congestion_threshold))
+                        break;
+
+                argp_failure (state, -1, 0,
+                              "unknown congestion threshold option %s", arg);
+                break;
+
+        case ARGP_FUSE_MOUNTOPTS_KEY:
+                cmd_args->fuse_mountopts = gf_strdup (arg);
+                break;
+
+        case ARGP_FUSE_USE_READDIRP_KEY:
+                if (!arg)
+                        arg = "yes";
+
+                if (gf_string2boolean (arg, &b) == 0) {
+                        if (b) {
+                                cmd_args->use_readdirp = "yes";
+                        } else {
+                                cmd_args->use_readdirp = "no";
+                        }
+
+                        break;
+                }
+
+                argp_failure (state, -1, 0,
+                              "unknown use-readdirp setting \"%s\"", arg);
+                break;
+
+	}
 
         return 0;
 }
@@ -774,7 +1077,7 @@ cleanup_and_exit (int signum)
         glusterfs_ctx_t *ctx      = NULL;
         xlator_t        *trav     = NULL;
 
-        ctx = glusterfs_ctx_get ();
+        ctx = glusterfsd_ctx;
 
         if (!ctx)
                 return;
@@ -787,9 +1090,17 @@ cleanup_and_exit (int signum)
 
         ctx->cleanup_started = 1;
         glusterfs_mgmt_pmap_signout (ctx);
-        if (ctx->listener) {
-                ctx->listener = NULL;
-        }
+
+        /* below part is a racy code where the rpcsvc object is freed.
+         * But in another thread (epoll thread), upon poll error in the
+         * socket the transports are cleaned up where again rpcsvc object
+         * is accessed (which is already freed by the below function).
+         * Since the process is about to be killed dont execute the function
+         * below.
+         */
+        /* if (ctx->listener) { */
+        /*         (void) glusterfs_listener_stop (ctx); */
+        /* } */
 
         /* Call fini() of FUSE xlator first:
          * so there are no more requests coming and
@@ -805,8 +1116,11 @@ cleanup_and_exit (int signum)
         exit (0);
 #if 0
         /* TODO: Properly do cleanup_and_exit(), with synchronization */
-        if (ctx->mgmt)
+        if (ctx->mgmt) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&ctx->mgmt->conn);
                 rpc_clnt_unref (ctx->mgmt);
+        }
 
         /* call fini() of each xlator */
         trav = NULL;
@@ -830,7 +1144,7 @@ reincarnate (int signum)
         glusterfs_ctx_t    *ctx = NULL;
         cmd_args_t         *cmd_args = NULL;
 
-        ctx = glusterfs_ctx_get ();
+        ctx = glusterfsd_ctx;
         cmd_args = &ctx->cmd_args;
 
         if (cmd_args->volfile_server) {
@@ -838,9 +1152,8 @@ reincarnate (int signum)
                         "Fetching the volume file from server...");
                 ret = glusterfs_volfile_fetch (ctx);
         } else {
-                gf_log ("glusterfsd", GF_LOG_INFO,
-                        "Reloading volfile ...");
-                ret = glusterfs_volumes_init (ctx);
+                gf_log ("glusterfsd", GF_LOG_DEBUG,
+                        "Not reloading volume specification file on SIGHUP");
         }
 
         /* Also, SIGHUP should do logrotate */
@@ -853,37 +1166,17 @@ reincarnate (int signum)
         return;
 }
 
-
-static char *
-generate_uuid ()
+void
+emancipate (glusterfs_ctx_t *ctx, int ret)
 {
-        char           tmp_str[1024] = {0,};
-        char           hostname[256] = {0,};
-        struct timeval tv = {0,};
-        struct tm      now = {0, };
-        char           now_str[32];
-
-        if (gettimeofday (&tv, NULL) == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "gettimeofday: failed %s",
-                        strerror (errno));
-        }
-
-        if (gethostname (hostname, 256) == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "gethostname: failed %s",
-                        strerror (errno));
+        /* break free from the parent */
+        if (ctx->daemon_pipe[1] != -1) {
+                write (ctx->daemon_pipe[1], (void *) &ret, sizeof (ret));
+                close (ctx->daemon_pipe[1]);
+                ctx->daemon_pipe[1] = -1;
         }
-
-        localtime_r (&tv.tv_sec, &now);
-        strftime (now_str, 32, "%Y/%m/%d-%H:%M:%S", &now);
-        snprintf (tmp_str, 1024, "%s-%d-%s:%" GF_PRI_SUSECONDS,
-                  hostname, getpid(), now_str, tv.tv_usec);
-
-        return gf_strdup (tmp_str);
 }
 
-
 static uint8_t
 gf_get_process_mode (char *exec_name)
 {
@@ -907,82 +1200,20 @@ gf_get_process_mode (char *exec_name)
 }
 
 
-
-static int
-set_log_file_path (cmd_args_t *cmd_args)
-{
-        int   i = 0;
-        int   j = 0;
-        int   ret = 0;
-        int   port = 0;
-        char *tmp_ptr = NULL;
-        char  tmp_str[1024] = {0,};
-
-        if (cmd_args->mount_point) {
-                j = 0;
-                i = 0;
-                if (cmd_args->mount_point[0] == '/')
-                        i = 1;
-                for (; i < strlen (cmd_args->mount_point); i++,j++) {
-                        tmp_str[j] = cmd_args->mount_point[i];
-                        if (cmd_args->mount_point[i] == '/')
-                                tmp_str[j] = '-';
-                }
-
-                ret = gf_asprintf (&cmd_args->log_file,
-                                   DEFAULT_LOG_FILE_DIRECTORY "/%s.log",
-                                   tmp_str);
-                goto done;
-        }
-
-        if (cmd_args->volfile) {
-                j = 0;
-                i = 0;
-                if (cmd_args->volfile[0] == '/')
-                        i = 1;
-                for (; i < strlen (cmd_args->volfile); i++,j++) {
-                        tmp_str[j] = cmd_args->volfile[i];
-                        if (cmd_args->volfile[i] == '/')
-                                tmp_str[j] = '-';
-                }
-                ret = gf_asprintf (&cmd_args->log_file,
-                                   DEFAULT_LOG_FILE_DIRECTORY "/%s.log",
-                                   tmp_str);
-                goto done;
-        }
-
-        if (cmd_args->volfile_server) {
-                port = 1;
-                tmp_ptr = "default";
-
-                if (cmd_args->volfile_server_port)
-                        port = cmd_args->volfile_server_port;
-                if (cmd_args->volfile_id)
-                        tmp_ptr = cmd_args->volfile_id;
-
-                ret = gf_asprintf (&cmd_args->log_file,
-                                   DEFAULT_LOG_FILE_DIRECTORY "/%s-%s-%d.log",
-                                   cmd_args->volfile_server, tmp_ptr, port);
-        }
-done:
-        return ret;
-}
-
-
 static int
 glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
 {
-        cmd_args_t    *cmd_args = NULL;
-        struct rlimit  lim = {0, };
-        call_pool_t   *pool = NULL;
+        cmd_args_t          *cmd_args = NULL;
+        struct rlimit        lim      = {0, };
+        int                  ret      = -1;
 
         xlator_mem_acct_init (THIS, gfd_mt_end);
 
-        ctx->process_uuid = generate_uuid ();
+        ctx->process_uuid = generate_glusterfs_ctx_id ();
         if (!ctx->process_uuid) {
                 gf_log ("", GF_LOG_CRITICAL,
                         "ERROR: glusterfs uuid generation failed");
-                return -1;
+                goto out;
         }
 
         ctx->page_size  = 128 * GF_UNIT_KB;
@@ -991,54 +1222,67 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
         if (!ctx->iobuf_pool) {
                 gf_log ("", GF_LOG_CRITICAL,
                         "ERROR: glusterfs iobuf pool creation failed");
-                return -1;
+                goto out;
         }
 
         ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE);
         if (!ctx->event_pool) {
                 gf_log ("", GF_LOG_CRITICAL,
                         "ERROR: glusterfs event pool creation failed");
-                return -1;
+                goto out;
         }
 
-        pool = GF_CALLOC (1, sizeof (call_pool_t),
-                          gfd_mt_call_pool_t);
-        if (!pool) {
+        ctx->pool = GF_CALLOC (1, sizeof (call_pool_t), gfd_mt_call_pool_t);
+        if (!ctx->pool) {
                 gf_log ("", GF_LOG_CRITICAL,
                         "ERROR: glusterfs call pool creation failed");
-                return -1;
+                goto out;
         }
 
-        /* frame_mem_pool size 112 * 16k */
-        pool->frame_mem_pool = mem_pool_new (call_frame_t, 16384);
+        INIT_LIST_HEAD (&ctx->pool->all_frames);
+        LOCK_INIT (&ctx->pool->lock);
 
-        if (!pool->frame_mem_pool) {
+        /* frame_mem_pool size 112 * 4k */
+        ctx->pool->frame_mem_pool = mem_pool_new (call_frame_t, 4096);
+        if (!ctx->pool->frame_mem_pool) {
                 gf_log ("", GF_LOG_CRITICAL,
                         "ERROR: glusterfs frame pool creation failed");
-                return -1;
+                goto out;
         }
-        /* stack_mem_pool size 256 * 8k */
-        pool->stack_mem_pool = mem_pool_new (call_stack_t, 8192);
-
-        if (!pool->stack_mem_pool) {
+        /* stack_mem_pool size 256 * 1024 */
+        ctx->pool->stack_mem_pool = mem_pool_new (call_stack_t, 1024);
+        if (!ctx->pool->stack_mem_pool) {
                 gf_log ("", GF_LOG_CRITICAL,
                         "ERROR: glusterfs stack pool creation failed");
-                return -1;
+                goto out;
         }
 
         ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
         if (!ctx->stub_mem_pool) {
                 gf_log ("", GF_LOG_CRITICAL,
                         "ERROR: glusterfs stub pool creation failed");
-                return -1;
+                goto out;
         }
 
-        INIT_LIST_HEAD (&pool->all_frames);
-        LOCK_INIT (&pool->lock);
-        ctx->pool = pool;
+        ctx->dict_pool = mem_pool_new (dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+        if (!ctx->dict_pool)
+                goto out;
+
+        ctx->dict_pair_pool = mem_pool_new (data_pair_t,
+                                            GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+        if (!ctx->dict_pair_pool)
+                goto out;
+
+        ctx->dict_data_pool = mem_pool_new (data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+        if (!ctx->dict_data_pool)
+                goto out;
 
         pthread_mutex_init (&(ctx->lock), NULL);
 
+        ctx->clienttable = gf_clienttable_alloc();
+        if (!ctx->clienttable)
+                goto out;
+
         cmd_args = &ctx->cmd_args;
 
         /* parsing command line arguments */
@@ -1055,33 +1299,68 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
 #endif
         cmd_args->fuse_attribute_timeout = -1;
         cmd_args->fuse_entry_timeout = -1;
+	cmd_args->fopen_keep_cache = GF_OPTION_DEFERRED;
 
         INIT_LIST_HEAD (&cmd_args->xlator_options);
+        INIT_LIST_HEAD (&cmd_args->volfile_servers);
 
         lim.rlim_cur = RLIM_INFINITY;
         lim.rlim_max = RLIM_INFINITY;
         setrlimit (RLIMIT_CORE, &lim);
 
-        return 0;
+        ret = 0;
+out:
+
+        if (ret && ctx) {
+                if (ctx->pool) {
+                        mem_pool_destroy (ctx->pool->frame_mem_pool);
+                        mem_pool_destroy (ctx->pool->stack_mem_pool);
+                }
+                GF_FREE (ctx->pool);
+                mem_pool_destroy (ctx->stub_mem_pool);
+                mem_pool_destroy (ctx->dict_pool);
+                mem_pool_destroy (ctx->dict_data_pool);
+                mem_pool_destroy (ctx->dict_pair_pool);
+        }
+
+        return ret;
 }
 
 static int
-logging_init (glusterfs_ctx_t *ctx)
+logging_init (glusterfs_ctx_t *ctx, const char *progpath)
 {
         cmd_args_t *cmd_args = NULL;
         int         ret = 0;
+        char        ident[1024] = {0,};
+        char       *progname = NULL;
+        char       *ptr = NULL;
 
         cmd_args = &ctx->cmd_args;
 
         if (cmd_args->log_file == NULL) {
-                ret = set_log_file_path (cmd_args);
+                ret = gf_set_log_file_path (cmd_args);
                 if (ret == -1) {
                         fprintf (stderr, "ERROR: failed to set the log file path\n");
                         return -1;
                 }
         }
 
-        if (gf_log_init (cmd_args->log_file) == -1) {
+#ifdef GF_USE_SYSLOG
+        progname  = gf_strdup (progpath);
+        snprintf (ident, 1024, "%s_%s", basename(progname),
+                  basename(cmd_args->log_file));
+        GF_FREE (progname);
+        /* remove .log suffix */
+        if (NULL != (ptr = strrchr(ident, '.'))) {
+                if (strcmp(ptr, ".log") == 0) {
+                        /* note: ptr points to location in ident only */
+                        ptr[0] = '\0';
+                }
+        }
+        ptr = ident;
+#endif
+
+        if (gf_log_init (ctx, cmd_args->log_file, ptr) == -1) {
                 fprintf (stderr, "ERROR: failed to open logfile %s\n",
                          cmd_args->log_file);
                 return -1;
@@ -1092,20 +1371,29 @@ logging_init (glusterfs_ctx_t *ctx)
         return 0;
 }
 
+void
+gf_check_and_set_mem_acct (glusterfs_ctx_t *ctx, int argc, char *argv[])
+{
+        int i = 0;
+        for (i = 0; i < argc; i++) {
+                if (strcmp (argv[i], "--mem-accounting") == 0) {
+			gf_mem_acct_enable_set (ctx);
+                        break;
+                }
+        }
+}
 
 int
 parse_cmdline (int argc, char *argv[], glusterfs_ctx_t *ctx)
 {
-        int               process_mode = 0;
-        int               ret = 0;
-        struct stat       stbuf = {0, };
-        struct tm        *tm = NULL;
-        time_t            utime;
-        char              timestr[256];
-        char              tmp_logfile[1024] = { 0 };
-        char              *tmp_logfile_dyn = NULL;
-        char              *tmp_logfilebase = NULL;
-        cmd_args_t        *cmd_args = NULL;
+        int          process_mode = 0;
+        int          ret = 0;
+        struct stat  stbuf = {0, };
+        char         timestr[32];
+        char         tmp_logfile[1024] = { 0 };
+        char        *tmp_logfile_dyn = NULL;
+        char        *tmp_logfilebase = NULL;
+        cmd_args_t  *cmd_args = NULL;
 
         cmd_args = &ctx->cmd_args;
 
@@ -1138,6 +1426,19 @@ parse_cmdline (int argc, char *argv[], glusterfs_ctx_t *ctx)
                         cmd_args->volfile = gf_strdup (DEFAULT_GLUSTERD_VOLFILE);
                 else
                         cmd_args->volfile = gf_strdup (DEFAULT_CLIENT_VOLFILE);
+
+                /* Check if the volfile exists, if not give usage output
+                   and exit */
+                ret = stat (cmd_args->volfile, &stbuf);
+                if (ret) {
+                        gf_log ("glusterfs", GF_LOG_CRITICAL,
+                                "ERROR: parsing the volfile failed (%s)\n",
+                                strerror (errno));
+                        /* argp_usage (argp.) */
+                        fprintf (stderr, "USAGE: %s [options] [mountpoint]\n",
+                                 argv[0]);
+                        goto out;
+                }
         }
 
         if (cmd_args->run_id) {
@@ -1149,8 +1450,8 @@ parse_cmdline (int argc, char *argv[], glusterfs_ctx_t *ctx)
                      (S_ISREG (stbuf.st_mode) || S_ISLNK (stbuf.st_mode))) ||
                     (ret == -1)) {
                         /* Have separate logfile per run */
-                        tm = localtime (&utime);
-                        strftime (timestr, 256, "%Y%m%d.%H%M%S", tm);
+                        gf_time_fmt (timestr, sizeof timestr, time (NULL),
+                                     gf_timefmt_FT);
                         sprintf (tmp_logfile, "%s.%s.%d",
                                  cmd_args->log_file, timestr, getpid ());
 
@@ -1188,7 +1489,7 @@ int
 glusterfs_pidfile_setup (glusterfs_ctx_t *ctx)
 {
         cmd_args_t  *cmd_args = NULL;
-        int          ret = 0;
+        int          ret = -1;
         FILE        *pidfp = NULL;
 
         cmd_args = &ctx->cmd_args;
@@ -1201,7 +1502,7 @@ glusterfs_pidfile_setup (glusterfs_ctx_t *ctx)
                 gf_log ("glusterfsd", GF_LOG_ERROR,
                         "pidfile %s error (%s)",
                         cmd_args->pid_file, strerror (errno));
-                return -1;
+                goto out;
         }
 
         ret = lockf (fileno (pidfp), F_TLOCK, 0);
@@ -1209,7 +1510,7 @@ glusterfs_pidfile_setup (glusterfs_ctx_t *ctx)
                 gf_log ("glusterfsd", GF_LOG_ERROR,
                         "pidfile %s lock error (%s)",
                         cmd_args->pid_file, strerror (errno));
-                return ret;
+                goto out;
         }
 
         gf_log ("glusterfsd", GF_LOG_TRACE,
@@ -1221,12 +1522,17 @@ glusterfs_pidfile_setup (glusterfs_ctx_t *ctx)
                 gf_log ("glusterfsd", GF_LOG_ERROR,
                         "pidfile %s unlock error (%s)",
                         cmd_args->pid_file, strerror (errno));
-                return ret;
+                goto out;
         }
 
         ctx->pidfp = pidfp;
 
-        return 0;
+        ret = 0;
+out:
+        if (ret && pidfp)
+                fclose (pidfp);
+
+        return ret;
 }
 
 
@@ -1339,10 +1645,10 @@ glusterfs_sigwaiter (void *arg)
                         reincarnate (sig);
                         break;
                 case SIGUSR1:
-                        gf_proc_dump_info (sig);
+                        gf_proc_dump_info (sig, glusterfsd_ctx);
                         break;
                 case SIGUSR2:
-                        gf_latency_toggle (sig);
+                        gf_latency_toggle (sig, glusterfsd_ctx);
                         break;
                 default:
 
@@ -1354,6 +1660,13 @@ glusterfs_sigwaiter (void *arg)
 }
 
 
+void
+glusterfsd_print_trace (int signum)
+{
+	gf_print_trace (signum, glusterfsd_ctx);
+}
+
+
 int
 glusterfs_signals_setup (glusterfs_ctx_t *ctx)
 {
@@ -1363,12 +1676,12 @@ glusterfs_signals_setup (glusterfs_ctx_t *ctx)
         sigemptyset (&set);
 
         /* common setting for all threads */
-        signal (SIGSEGV, gf_print_trace);
-        signal (SIGABRT, gf_print_trace);
-        signal (SIGILL, gf_print_trace);
-        signal (SIGTRAP, gf_print_trace);
-        signal (SIGFPE, gf_print_trace);
-        signal (SIGBUS, gf_print_trace);
+        signal (SIGSEGV, glusterfsd_print_trace);
+        signal (SIGABRT, glusterfsd_print_trace);
+        signal (SIGILL, glusterfsd_print_trace);
+        signal (SIGTRAP, glusterfsd_print_trace);
+        signal (SIGFPE, glusterfsd_print_trace);
+        signal (SIGBUS, glusterfsd_print_trace);
         signal (SIGINT, cleanup_and_exit);
         signal (SIGPIPE, SIG_IGN);
 
@@ -1380,7 +1693,7 @@ glusterfs_signals_setup (glusterfs_ctx_t *ctx)
 
         ret = pthread_sigmask (SIG_BLOCK, &set, NULL);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING,
+                gf_log ("glusterfsd", GF_LOG_WARNING,
                         "failed to execute pthread_signmask  %s",
                         strerror (errno));
                 return ret;
@@ -1394,7 +1707,7 @@ glusterfs_signals_setup (glusterfs_ctx_t *ctx)
                   fallback to signals getting handled by other threads.
                   setup the signal handlers
                 */
-                gf_log ("", GF_LOG_WARNING,
+                gf_log ("glusterfsd", GF_LOG_WARNING,
                         "failed to create pthread  %s",
                         strerror (errno));
                 return ret;
@@ -1410,6 +1723,7 @@ daemonize (glusterfs_ctx_t *ctx)
         int            ret = -1;
         cmd_args_t    *cmd_args = NULL;
         int            cstatus = 0;
+        int            err = 0;
 
         cmd_args = &ctx->cmd_args;
 
@@ -1423,24 +1737,48 @@ daemonize (glusterfs_ctx_t *ctx)
         if (cmd_args->debug_mode)
                 goto postfork;
 
+        ret = pipe (ctx->daemon_pipe);
+        if (ret) {
+                /* If pipe() fails, retain daemon_pipe[] = {-1, -1}
+                   and parent will just not wait for child status
+                */
+                ctx->daemon_pipe[0] = -1;
+                ctx->daemon_pipe[1] = -1;
+        }
+
         ret = os_daemon_return (0, 0);
         switch (ret) {
         case -1:
+                if (ctx->daemon_pipe[0] != -1) {
+                        close (ctx->daemon_pipe[0]);
+                        close (ctx->daemon_pipe[1]);
+                }
+
                 gf_log ("daemonize", GF_LOG_ERROR,
                         "Daemonization failed: %s", strerror(errno));
                 goto out;
         case 0:
+                /* child */
+                /* close read */
+                close (ctx->daemon_pipe[0]);
                 break;
         default:
-                if (ctx->mtab_pid > 0) {
-                        ret = waitpid (ctx->mtab_pid, &cstatus, 0);
-                        if (!(ret == ctx->mtab_pid && cstatus == 0)) {
+                /* parent */
+                /* close write */
+                close (ctx->daemon_pipe[1]);
+
+                if (ctx->mnt_pid > 0) {
+                        ret = waitpid (ctx->mnt_pid, &cstatus, 0);
+                        if (!(ret == ctx->mnt_pid && cstatus == 0)) {
                                 gf_log ("daemonize", GF_LOG_ERROR,
-                                        "/etc/mtab update failed");
+                                        "mount failed");
                                 exit (1);
                         }
                 }
-                _exit (0);
+
+                err = 1;
+                read (ctx->daemon_pipe[0], (void *)&err, sizeof (err));
+                _exit (err);
         }
 
 postfork:
@@ -1489,7 +1827,7 @@ glusterfs_process_volfp (glusterfs_ctx_t *ctx, FILE *fp)
                 goto out;
         }
 
-        gf_log_volume_file (fp);
+        gf_log_dump_graph (fp, graph);
 
         ret = 0;
 out:
@@ -1522,7 +1860,8 @@ glusterfs_volumes_init (glusterfs_ctx_t *ctx)
 
         if (cmd_args->volfile_server) {
                 ret = glusterfs_mgmt_init (ctx);
-                goto out;
+                /* return, do not emancipate() yet */
+                return ret;
         }
 
         fp = get_volfp (ctx);
@@ -1539,26 +1878,41 @@ glusterfs_volumes_init (glusterfs_ctx_t *ctx)
                 goto out;
 
 out:
+        emancipate (ctx, ret);
         return ret;
 }
 
 
+/* This is the only legal global pointer  */
+glusterfs_ctx_t *glusterfsd_ctx;
+
 int
 main (int argc, char *argv[])
 {
         glusterfs_ctx_t  *ctx = NULL;
         int               ret = -1;
+        char              cmdlinestr[PATH_MAX] = {0,};
 
-        ret = glusterfs_globals_init ();
-        if (ret)
-                return ret;
-
-        ctx = glusterfs_ctx_get ();
+	ctx = glusterfs_ctx_new ();
         if (!ctx) {
                 gf_log ("glusterfs", GF_LOG_CRITICAL,
                         "ERROR: glusterfs context not initialized");
                 return ENOMEM;
         }
+	glusterfsd_ctx = ctx;
+
+#ifdef DEBUG
+        gf_mem_acct_enable_set (ctx);
+#else
+        /* Enable memory accounting on the fly based on argument */
+        gf_check_and_set_mem_acct (ctx, argc, argv);
+#endif
+
+        ret = glusterfs_globals_init (ctx);
+        if (ret)
+                return ret;
+
+	THIS->ctx = ctx;
 
         ret = glusterfs_ctx_defaults_init (ctx);
         if (ret)
@@ -1568,14 +1922,23 @@ main (int argc, char *argv[])
         if (ret)
                 goto out;
 
-        ret = logging_init (ctx);
+        ret = logging_init (ctx, argv[0]);
         if (ret)
                 goto out;
 
-        /* log the version of glusterfs running here */
-        gf_log (argv[0], GF_LOG_INFO,
-                "Started running %s version %s",
-                argv[0], PACKAGE_VERSION);
+        /* log the version of glusterfs running here along with the actual
+           command line options. */
+        {
+                int i = 0;
+                strcpy (cmdlinestr, argv[0]);
+                for (i = 1; i < argc; i++) {
+                        strcat (cmdlinestr, " ");
+                        strcat (cmdlinestr, argv[i]);
+                }
+                gf_log (argv[0], GF_LOG_INFO,
+                        "Started running %s version %s (%s)",
+                        argv[0], PACKAGE_VERSION, cmdlinestr);
+        }
 
         gf_proc_dump_init();
 
@@ -1587,7 +1950,7 @@ main (int argc, char *argv[])
         if (ret)
                 goto out;
 
-	ctx->env = syncenv_new (0);
+	ctx->env = syncenv_new (0, 0, 0);
         if (!ctx->env) {
                 gf_log ("", GF_LOG_ERROR,
                         "Could not create new sync-environment");
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index 0e68f7f0e..9e2a0e56e 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __GLUSTERFSD_H__
 #define __GLUSTERFSD_H__
 
@@ -30,8 +20,6 @@
 #define DEFAULT_GLUSTERD_VOLFILE              CONFDIR "/glusterd.vol"
 #define DEFAULT_CLIENT_VOLFILE                CONFDIR "/glusterfs.vol"
 #define DEFAULT_SERVER_VOLFILE                CONFDIR "/glusterfsd.vol"
-#define DEFAULT_LOG_FILE_DIRECTORY            DATADIR "/log/glusterfs"
-#define DEFAULT_LOG_LEVEL                     GF_LOG_INFO
 
 #define DEFAULT_EVENT_POOL_SIZE            16384
 
@@ -40,12 +28,17 @@
 #define ARGP_LOG_LEVEL_CRITICAL_OPTION    "CRITICAL"
 #define ARGP_LOG_LEVEL_ERROR_OPTION       "ERROR"
 #define ARGP_LOG_LEVEL_WARNING_OPTION     "WARNING"
-#define ARGP_LOG_LEVEL_INFO_OPTION      "INFO"
+#define ARGP_LOG_LEVEL_INFO_OPTION        "INFO"
 #define ARGP_LOG_LEVEL_DEBUG_OPTION       "DEBUG"
 
 #define ENABLE_NO_DAEMON_MODE     1
 #define ENABLE_DEBUG_MODE         1
 
+#define GF_MEMPOOL_COUNT_OF_DICT_T        4096
+/* Considering 4 key/value pairs in a dictionary on an average */
+#define GF_MEMPOOL_COUNT_OF_DATA_T        (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T   (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+
 enum argp_option_keys {
         ARGP_VOLFILE_SERVER_KEY           = 's',
         ARGP_VOLUME_FILE_KEY              = 'f',
@@ -58,6 +51,7 @@ enum argp_option_keys {
         ARGP_NO_DAEMON_KEY                = 'N',
         ARGP_RUN_ID_KEY                   = 'r',
         ARGP_DEBUG_KEY                    = 133,
+        ARGP_NEGATIVE_TIMEOUT_KEY         = 134,
         ARGP_ENTRY_TIMEOUT_KEY            = 135,
         ARGP_ATTRIBUTE_TIMEOUT_KEY        = 136,
         ARGP_VOLUME_NAME_KEY              = 137,
@@ -80,13 +74,23 @@ enum argp_option_keys {
         ARGP_ACL_KEY                      = 154,
         ARGP_WORM_KEY                     = 155,
         ARGP_USER_MAP_ROOT_KEY            = 156,
+        ARGP_MEM_ACCOUNTING_KEY           = 157,
+        ARGP_SELINUX_KEY                  = 158,
+	ARGP_FOPEN_KEEP_CACHE_KEY	  = 159,
+	ARGP_GID_TIMEOUT_KEY		  = 160,
+	ARGP_FUSE_BACKGROUND_QLEN_KEY     = 161,
+	ARGP_FUSE_CONGESTION_THRESHOLD_KEY = 162,
+        ARGP_INODE32_KEY                  = 163,
+	ARGP_FUSE_MOUNTOPTS_KEY		  = 164,
+        ARGP_FUSE_USE_READDIRP_KEY        = 165,
+	ARGP_AUX_GFID_MOUNT_KEY		  = 166,
 };
 
 struct _gfd_vol_top_priv_t {
         rpcsvc_request_t        *req;
         gd1_mgmt_brick_op_req   xlator_req;
-        int32_t                 blk_count;
-        int32_t                 blk_size;
+        uint32_t                blk_count;
+        uint32_t                blk_size;
         double                  throughput;
         double                  time;
         int32_t                 ret;
@@ -98,6 +102,12 @@ int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx);
 int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx);
 void cleanup_and_exit (int signum);
 
-void *glusterfs_volume_top_read_perf (void *args);
-void *glusterfs_volume_top_write_perf (void *args);
+int glusterfs_volume_top_write_perf (uint32_t blk_size, uint32_t blk_count,
+                                     char *brick_path, double *throughput,
+                                     double *time);
+int glusterfs_volume_top_read_perf (uint32_t blk_size, uint32_t blk_count,
+                                    char *brick_path, double *throughput,
+                                    double *time);
+
+extern glusterfs_ctx_t *glusterfsd_ctx;
 #endif /* __GLUSTERFSD_H__ */
diff --git a/libgfchangelog.pc.in b/libgfchangelog.pc.in
new file mode 100644
index 000000000..d654280d0
--- /dev/null
+++ b/libgfchangelog.pc.in
@@ -0,0 +1,11 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+
+Name: libgfchangelog
+Description: GlusterFS Changelog Consumer Library
+Version: @VERSION@
+Libs: -L${libdir} -lgfchangelog -lglusterfs
+Cflags: -I${includedir}/glusterfs/gfchangelog -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64
diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am
index 34543f622..907399ae6 100644
--- a/libglusterfs/src/Makefile.am
+++ b/libglusterfs/src/Makefile.am
@@ -1,11 +1,10 @@
-libglusterfs_la_CFLAGS = -fPIC  -Wall -g -shared -nostartfiles $(GF_CFLAGS) \
-	$(GF_DARWIN_LIBGLUSTERFS_CFLAGS)
+libglusterfs_la_CFLAGS = -Wall $(GF_CFLAGS) \
+	$(GF_DARWIN_LIBGLUSTERFS_CFLAGS)    \
+        -DDATADIR=\"$(localstatedir)\"
 
-libglusterfs_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 \
-	-D_GNU_SOURCE -DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \
-	-D$(GF_HOST_OS) -I$(CONTRIBDIR)/rbtree \
-	-DSCHEDULERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/scheduler\" \
-	-I$(CONTRIBDIR)/md5
+libglusterfs_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
+	-DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \
+	-I$(top_srcdir)/rpc/rpc-lib/src/ -I$(CONTRIBDIR)/rbtree
 
 libglusterfs_la_LIBADD = @LEXLIB@
 
@@ -16,36 +15,42 @@ CONTRIB_BUILDDIR = $(top_builddir)/contrib
 libglusterfs_la_SOURCES = dict.c xlator.c logging.c \
 	hashfn.c defaults.c common-utils.c timer.c inode.c call-stub.c \
 	compat.c fd.c compat-errno.c event.c mem-pool.c gf-dirent.c syscall.c \
-	iobuf.c globals.c statedump.c stack.c checksum.c daemon.c \
-	$(CONTRIBDIR)/md5/md5.c $(CONTRIBDIR)/rbtree/rb.c rbthash.c latency.c \
+	iobuf.c globals.c statedump.c stack.c checksum.c daemon.c timespec.c \
+	$(CONTRIBDIR)/rbtree/rb.c rbthash.c store.c latency.c \
 	graph.c $(CONTRIBDIR)/uuid/clear.c $(CONTRIBDIR)/uuid/copy.c \
 	$(CONTRIBDIR)/uuid/gen_uuid.c $(CONTRIBDIR)/uuid/pack.c \
 	$(CONTRIBDIR)/uuid/parse.c $(CONTRIBDIR)/uuid/unparse.c \
 	$(CONTRIBDIR)/uuid/uuid_time.c $(CONTRIBDIR)/uuid/compare.c \
 	$(CONTRIBDIR)/uuid/isnull.c $(CONTRIBDIR)/uuid/unpack.c syncop.c \
-	graph-print.c trie.c run.c options.c
+	graph-print.c trie.c run.c options.c fd-lk.c circ-buff.c \
+	event-history.c gidcache.c ctx.c client_t.c event-poll.c event-epoll.c \
+	$(CONTRIBDIR)/libgen/basename_r.c $(CONTRIBDIR)/libgen/dirname_r.c \
+	$(CONTRIBDIR)/stdlib/gf_mkostemp.c
 
-nodist_libglusterfs_la_SOURCES = y.tab.c graph.lex.c
+
+nodist_libglusterfs_la_SOURCES = y.tab.c graph.lex.c gf-error-codes.h
 
 BUILT_SOURCES = graph.lex.c
 
-noinst_HEADERS = common-utils.h defaults.h dict.h glusterfs.h hashfn.h \
+noinst_HEADERS = common-utils.h defaults.h dict.h glusterfs.h hashfn.h timespec.h \
 	logging.h xlator.h stack.h timer.h list.h inode.h call-stub.h compat.h \
 	fd.h revision.h compat-errno.h event.h mem-pool.h byte-order.h \
 	gf-dirent.h locking.h syscall.h iobuf.h globals.h statedump.h \
-	checksum.h daemon.h $(CONTRIBDIR)/md5/md5.h $(CONTRIBDIR)/rbtree/rb.h \
+	checksum.h daemon.h $(CONTRIBDIR)/rbtree/rb.h store.h\
 	rbthash.h iatt.h latency.h mem-types.h $(CONTRIBDIR)/uuid/uuidd.h \
 	$(CONTRIBDIR)/uuid/uuid.h $(CONTRIBDIR)/uuid/uuidP.h \
-	$(CONTRIB_BUILDDIR)/uuid/uuid_types.h syncop.h graph-utils.h trie.h run.h \
-	options.h lkowner.h
+	$(CONTRIB_BUILDDIR)/uuid/uuid_types.h syncop.h graph-utils.h trie.h \
+	run.h options.h lkowner.h fd-lk.h circ-buff.h event-history.h \
+	gidcache.h client_t.h glusterfs-acl.h
 
 EXTRA_DIST = graph.l graph.y
 
 graph.lex.c: graph.l y.tab.h
-	$(LEX) -t $(srcdir)/graph.l > $@
+	$(LEX) -Pgraphyy -t $(srcdir)/graph.l > $@
 
+y.tab.c: y.tab.h
 y.tab.h: graph.y
-	$(YACC) -d $(srcdir)/graph.y
+	$(YACC) -p graphyy -d $(srcdir)/graph.y
 
 CLEANFILES = graph.lex.c y.tab.c y.tab.h
 CONFIG_CLEAN_FILES = $(CONTRIB_BUILDDIR)/uuid/uuid_types.h
diff --git a/libglusterfs/src/byte-order.h b/libglusterfs/src/byte-order.h
index e325e522d..4101db2c7 100644
--- a/libglusterfs/src/byte-order.h
+++ b/libglusterfs/src/byte-order.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _BYTE_ORDER_H
diff --git a/libglusterfs/src/call-stub.c b/libglusterfs/src/call-stub.c
index cdae9df25..ac79cf071 100644
--- a/libglusterfs/src/call-stub.c
+++ b/libglusterfs/src/call-stub.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -22,9 +13,9 @@
 #include "config.h"
 #endif
 
+#include <openssl/md5.h>
 #include <inttypes.h>
 
-#include "md5.h"
 #include "call-stub.h"
 #include "mem-types.h"
 
@@ -46,16 +37,16 @@ stub_new (call_frame_t *frame,
         new->fop = fop;
         new->stub_mem_pool = frame->this->ctx->stub_mem_pool;
         INIT_LIST_HEAD (&new->list);
+
+        INIT_LIST_HEAD (&new->args_cbk.entries);
 out:
         return new;
 }
 
 
 call_stub_t *
-fop_lookup_stub (call_frame_t *frame,
-                 fop_lookup_t fn,
-                 loc_t *loc,
-                 dict_t *xattr_req)
+fop_lookup_stub (call_frame_t *frame, fop_lookup_t fn, loc_t *loc,
+                 dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -65,26 +56,22 @@ fop_lookup_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_LOOKUP);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.lookup.fn = fn;
+        stub->fn.lookup = fn;
 
-        if (xattr_req)
-                stub->args.lookup.xattr_req = dict_ref (xattr_req);
+        loc_copy (&stub->args.loc, loc);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 
-        loc_copy (&stub->args.lookup.loc, loc);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_lookup_cbk_stub (call_frame_t *frame,
-                     fop_lookup_cbk_t fn,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     inode_t *inode,
-                     struct iatt *buf,
-                     dict_t *dict,
-                     struct iatt *postparent)
+fop_lookup_cbk_stub (call_frame_t *frame, fop_lookup_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno,
+                     inode_t *inode, struct iatt *buf,
+                     dict_t *xdata, struct iatt *postparent)
 {
         call_stub_t *stub = NULL;
 
@@ -93,27 +80,25 @@ fop_lookup_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_LOOKUP);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.lookup_cbk.fn = fn;
-        stub->args.lookup_cbk.op_ret = op_ret;
-        stub->args.lookup_cbk.op_errno = op_errno;
+        stub->fn_cbk.lookup = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (inode)
-                stub->args.lookup_cbk.inode = inode_ref (inode);
+                stub->args_cbk.inode = inode_ref (inode);
         if (buf)
-                stub->args.lookup_cbk.buf = *buf;
-        if (dict)
-                stub->args.lookup_cbk.dict = dict_ref (dict);
+                stub->args_cbk.stat = *buf;
         if (postparent)
-                stub->args.lookup_cbk.postparent = *postparent;
+                stub->args_cbk.postparent = *postparent;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
-
 call_stub_t *
-fop_stat_stub (call_frame_t *frame,
-               fop_stat_t fn,
-               loc_t *loc)
+fop_stat_stub (call_frame_t *frame, fop_stat_t fn,
+               loc_t *loc, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -123,19 +108,19 @@ fop_stat_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_STAT);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.stat.fn = fn;
-        loc_copy (&stub->args.stat.loc, loc);
+        stub->fn.stat = fn;
+        loc_copy (&stub->args.loc, loc);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_stat_cbk_stub (call_frame_t *frame,
-                   fop_stat_cbk_t fn,
-                   int32_t op_ret,
-                   int32_t op_errno,
-                   struct iatt *buf)
+fop_stat_cbk_stub (call_frame_t *frame, fop_stat_cbk_t fn,
+                   int32_t op_ret, int32_t op_errno,
+                   struct iatt *buf, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -144,20 +129,21 @@ fop_stat_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_STAT);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.stat_cbk.fn = fn;
-        stub->args.stat_cbk.op_ret = op_ret;
-        stub->args.stat_cbk.op_errno = op_errno;
+        stub->fn_cbk.stat = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (op_ret == 0)
-                stub->args.stat_cbk.buf = *buf;
+                stub->args_cbk.stat = *buf;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fstat_stub (call_frame_t *frame,
-                fop_fstat_t fn,
-                fd_t *fd)
+fop_fstat_stub (call_frame_t *frame, fop_fstat_t fn,
+                fd_t *fd, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -166,21 +152,21 @@ fop_fstat_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_FSTAT);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fstat.fn = fn;
+        stub->fn.fstat = fn;
 
         if (fd)
-                stub->args.fstat.fd = fd_ref (fd);
+                stub->args.fd = fd_ref (fd);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fstat_cbk_stub (call_frame_t *frame,
-                    fop_fstat_cbk_t fn,
-                    int32_t op_ret,
-                    int32_t op_errno,
-                    struct iatt *buf)
+fop_fstat_cbk_stub (call_frame_t *frame, fop_fstat_cbk_t fn,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iatt *buf, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -189,23 +175,21 @@ fop_fstat_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_FSTAT);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fstat_cbk.fn = fn;
-        stub->args.fstat_cbk.op_ret = op_ret;
-        stub->args.fstat_cbk.op_errno = op_errno;
+        stub->fn_cbk.fstat = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (buf)
-                stub->args.fstat_cbk.buf = *buf;
+                stub->args_cbk.stat = *buf;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
-/* truncate */
-
 call_stub_t *
-fop_truncate_stub (call_frame_t *frame,
-                   fop_truncate_t fn,
-                   loc_t *loc,
-                   off_t off)
+fop_truncate_stub (call_frame_t *frame, fop_truncate_t fn,
+                   loc_t *loc, off_t off, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -215,21 +199,20 @@ fop_truncate_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_TRUNCATE);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.truncate.fn = fn;
-        loc_copy (&stub->args.truncate.loc, loc);
-        stub->args.truncate.off = off;
+        stub->fn.truncate = fn;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.offset = off;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_truncate_cbk_stub (call_frame_t *frame,
-                       fop_truncate_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       struct iatt *prebuf,
-                       struct iatt *postbuf)
+fop_truncate_cbk_stub (call_frame_t *frame, fop_truncate_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		       struct iatt *postbuf, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -238,23 +221,23 @@ fop_truncate_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_TRUNCATE);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.truncate_cbk.fn = fn;
-        stub->args.truncate_cbk.op_ret = op_ret;
-        stub->args.truncate_cbk.op_errno = op_errno;
+        stub->fn_cbk.truncate = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (prebuf)
-                stub->args.truncate_cbk.prebuf = *prebuf;
+                stub->args_cbk.prestat = *prebuf;
         if (postbuf)
-                stub->args.truncate_cbk.postbuf = *postbuf;
+                stub->args_cbk.poststat = *postbuf;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_ftruncate_stub (call_frame_t *frame,
-                    fop_ftruncate_t fn,
-                    fd_t *fd,
-                    off_t off)
+fop_ftruncate_stub (call_frame_t *frame, fop_ftruncate_t fn,
+                    fd_t *fd, off_t off, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -263,23 +246,22 @@ fop_ftruncate_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_FTRUNCATE);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.ftruncate.fn = fn;
+        stub->fn.ftruncate = fn;
         if (fd)
-                stub->args.ftruncate.fd = fd_ref (fd);
+                stub->args.fd = fd_ref (fd);
 
-        stub->args.ftruncate.off = off;
+        stub->args.offset = off;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_ftruncate_cbk_stub (call_frame_t *frame,
-                        fop_ftruncate_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        struct iatt *prebuf,
-                        struct iatt *postbuf)
+fop_ftruncate_cbk_stub (call_frame_t *frame, fop_ftruncate_cbk_t fn,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+			struct iatt *postbuf, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -288,23 +270,24 @@ fop_ftruncate_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_FTRUNCATE);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.ftruncate_cbk.fn = fn;
-        stub->args.ftruncate_cbk.op_ret = op_ret;
-        stub->args.ftruncate_cbk.op_errno = op_errno;
+        stub->fn_cbk.ftruncate = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (prebuf)
-                stub->args.ftruncate_cbk.prebuf = *prebuf;
+                stub->args_cbk.prestat = *prebuf;
         if (postbuf)
-                stub->args.ftruncate_cbk.postbuf = *postbuf;
+                stub->args_cbk.poststat = *postbuf;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
+
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_access_stub (call_frame_t *frame,
-                 fop_access_t fn,
-                 loc_t *loc,
-                 int32_t mask)
+fop_access_stub (call_frame_t *frame, fop_access_t fn,
+                 loc_t *loc, int32_t mask, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -314,19 +297,19 @@ fop_access_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_ACCESS);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.access.fn = fn;
-        loc_copy (&stub->args.access.loc, loc);
-        stub->args.access.mask = mask;
+        stub->fn.access = fn;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.mask = mask;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_access_cbk_stub (call_frame_t *frame,
-                     fop_access_cbk_t fn,
-                     int32_t op_ret,
-                     int32_t op_errno)
+fop_access_cbk_stub (call_frame_t *frame, fop_access_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -335,19 +318,19 @@ fop_access_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_ACCESS);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.access_cbk.fn = fn;
-        stub->args.access_cbk.op_ret = op_ret;
-        stub->args.access_cbk.op_errno = op_errno;
+        stub->fn_cbk.access = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_readlink_stub (call_frame_t *frame,
-                   fop_readlink_t fn,
-                   loc_t *loc,
-                   size_t size)
+fop_readlink_stub (call_frame_t *frame, fop_readlink_t fn,
+                   loc_t *loc, size_t size, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -357,21 +340,20 @@ fop_readlink_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_READLINK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.readlink.fn = fn;
-        loc_copy (&stub->args.readlink.loc, loc);
-        stub->args.readlink.size = size;
+        stub->fn.readlink = fn;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.size = size;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_readlink_cbk_stub (call_frame_t *frame,
-                       fop_readlink_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       const char *path,
-                       struct iatt *sbuf)
+fop_readlink_cbk_stub (call_frame_t *frame, fop_readlink_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno,
+                       const char *path, struct iatt *stbuf, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -380,21 +362,23 @@ fop_readlink_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_READLINK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.readlink_cbk.fn = fn;
-        stub->args.readlink_cbk.op_ret = op_ret;
-        stub->args.readlink_cbk.op_errno = op_errno;
+        stub->fn_cbk.readlink = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (path)
-                stub->args.readlink_cbk.buf = gf_strdup (path);
-        if (sbuf)
-                stub->args.readlink_cbk.sbuf = *sbuf;
+                stub->args_cbk.buf = gf_strdup (path);
+        if (stbuf)
+                stub->args_cbk.stat = *stbuf;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_mknod_stub (call_frame_t *frame, fop_mknod_t fn,
-                loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+fop_mknod_stub (call_frame_t *frame, fop_mknod_t fn, loc_t *loc, mode_t mode,
+                dev_t rdev, mode_t umask, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -404,26 +388,23 @@ fop_mknod_stub (call_frame_t *frame, fop_mknod_t fn,
         stub = stub_new (frame, 1, GF_FOP_MKNOD);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.mknod.fn = fn;
-        loc_copy (&stub->args.mknod.loc, loc);
-        stub->args.mknod.mode = mode;
-        stub->args.mknod.rdev = rdev;
-        if (params)
-                stub->args.mknod.params = dict_ref (params);
+        stub->fn.mknod = fn;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.mode = mode;
+        stub->args.rdev = rdev;
+        stub->args.umask = umask;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_mknod_cbk_stub (call_frame_t *frame,
-                    fop_mknod_cbk_t fn,
-                    int32_t op_ret,
-                    int32_t op_errno,
-                    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
+fop_mknod_cbk_stub (call_frame_t *frame, fop_mknod_cbk_t fn, int32_t op_ret,
+                    int32_t op_errno, inode_t *inode, struct iatt *buf,
+                    struct iatt *preparent, struct iatt *postparent,
+		    dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -432,17 +413,20 @@ fop_mknod_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_MKNOD);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.mknod_cbk.fn = fn;
-        stub->args.mknod_cbk.op_ret = op_ret;
-        stub->args.mknod_cbk.op_errno = op_errno;
+        stub->fn_cbk.mknod = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (inode)
-                stub->args.mknod_cbk.inode = inode_ref (inode);
+                stub->args_cbk.inode = inode_ref (inode);
         if (buf)
-                stub->args.mknod_cbk.buf = *buf;
+                stub->args_cbk.stat = *buf;
         if (preparent)
-                stub->args.mknod_cbk.preparent = *preparent;
+                stub->args_cbk.preparent = *preparent;
         if (postparent)
-                stub->args.mknod_cbk.postparent = *postparent;
+                stub->args_cbk.postparent = *postparent;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
+
 out:
         return stub;
 }
@@ -450,7 +434,7 @@ out:
 
 call_stub_t *
 fop_mkdir_stub (call_frame_t *frame, fop_mkdir_t fn,
-                loc_t *loc, mode_t mode, dict_t *params)
+                loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -460,25 +444,23 @@ fop_mkdir_stub (call_frame_t *frame, fop_mkdir_t fn,
         stub = stub_new (frame, 1, GF_FOP_MKDIR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.mkdir.fn = fn;
-        loc_copy (&stub->args.mkdir.loc, loc);
-        stub->args.mkdir.mode = mode;
-        if (params)
-                stub->args.mkdir.params = dict_ref (params);
+        stub->fn.mkdir = fn;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.mode  = mode;
+        stub->args.umask = umask;
+
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_mkdir_cbk_stub (call_frame_t *frame,
-                    fop_mkdir_cbk_t fn,
-                    int32_t op_ret,
-                    int32_t op_errno,
-                    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
+fop_mkdir_cbk_stub (call_frame_t *frame, fop_mkdir_cbk_t fn,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -487,26 +469,27 @@ fop_mkdir_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_MKDIR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.mkdir_cbk.fn = fn;
-        stub->args.mkdir_cbk.op_ret = op_ret;
-        stub->args.mkdir_cbk.op_errno = op_errno;
+        stub->fn_cbk.mkdir = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (inode)
-                stub->args.mkdir_cbk.inode = inode_ref (inode);
+                stub->args_cbk.inode = inode_ref (inode);
         if (buf)
-                stub->args.mkdir_cbk.buf = *buf;
+                stub->args_cbk.stat = *buf;
         if (preparent)
-                stub->args.mkdir_cbk.preparent = *preparent;
+                stub->args_cbk.preparent = *preparent;
         if (postparent)
-                stub->args.mkdir_cbk.postparent = *postparent;
+                stub->args_cbk.postparent = *postparent;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_unlink_stub (call_frame_t *frame,
-                 fop_unlink_t fn,
-                 loc_t *loc)
+fop_unlink_stub (call_frame_t *frame, fop_unlink_t fn,
+                 loc_t *loc, int xflag, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -516,20 +499,21 @@ fop_unlink_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_UNLINK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.unlink.fn = fn;
-        loc_copy (&stub->args.unlink.loc, loc);
+        stub->fn.unlink = fn;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.xflag = xflag;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_unlink_cbk_stub (call_frame_t *frame,
-                     fop_unlink_cbk_t fn,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     struct iatt *preparent,
-                     struct iatt *postparent)
+fop_unlink_cbk_stub (call_frame_t *frame, fop_unlink_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *preparent, struct iatt *postparent,
+		     dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -538,13 +522,15 @@ fop_unlink_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_UNLINK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.unlink_cbk.fn = fn;
-        stub->args.unlink_cbk.op_ret = op_ret;
-        stub->args.unlink_cbk.op_errno = op_errno;
+        stub->fn_cbk.unlink = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (preparent)
-                stub->args.unlink_cbk.preparent = *preparent;
+                stub->args_cbk.preparent = *preparent;
         if (postparent)
-                stub->args.unlink_cbk.postparent = *postparent;
+                stub->args_cbk.postparent = *postparent;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
@@ -553,7 +539,7 @@ out:
 
 call_stub_t *
 fop_rmdir_stub (call_frame_t *frame, fop_rmdir_t fn,
-                loc_t *loc, int flags)
+                loc_t *loc, int flags, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -563,21 +549,21 @@ fop_rmdir_stub (call_frame_t *frame, fop_rmdir_t fn,
         stub = stub_new (frame, 1, GF_FOP_RMDIR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.rmdir.fn = fn;
-        loc_copy (&stub->args.rmdir.loc, loc);
-        stub->args.rmdir.flags = flags;
+        stub->fn.rmdir = fn;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.flags = flags;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_rmdir_cbk_stub (call_frame_t *frame,
-                    fop_rmdir_cbk_t fn,
-                    int32_t op_ret,
-                    int32_t op_errno,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
+fop_rmdir_cbk_stub (call_frame_t *frame, fop_rmdir_cbk_t fn,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iatt *preparent, struct iatt *postparent,
+		    dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -586,13 +572,15 @@ fop_rmdir_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_RMDIR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.rmdir_cbk.fn = fn;
-        stub->args.rmdir_cbk.op_ret = op_ret;
-        stub->args.rmdir_cbk.op_errno = op_errno;
+        stub->fn_cbk.rmdir = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (preparent)
-                stub->args.rmdir_cbk.preparent = *preparent;
+                stub->args_cbk.preparent = *preparent;
         if (postparent)
-                stub->args.rmdir_cbk.postparent = *postparent;
+                stub->args_cbk.postparent = *postparent;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
@@ -600,7 +588,7 @@ out:
 
 call_stub_t *
 fop_symlink_stub (call_frame_t *frame, fop_symlink_t fn,
-                  const char *linkname, loc_t *loc, dict_t *params)
+                  const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -611,25 +599,23 @@ fop_symlink_stub (call_frame_t *frame, fop_symlink_t fn,
         stub = stub_new (frame, 1, GF_FOP_SYMLINK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.symlink.fn = fn;
-        stub->args.symlink.linkname = gf_strdup (linkname);
-        loc_copy (&stub->args.symlink.loc, loc);
-        if (params)
-                stub->args.symlink.params = dict_ref (params);
+        stub->fn.symlink = fn;
+        stub->args.linkname = gf_strdup (linkname);
+        stub->args.umask = umask;
+        loc_copy (&stub->args.loc, loc);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_symlink_cbk_stub (call_frame_t *frame,
-                      fop_symlink_cbk_t fn,
-                      int32_t op_ret,
-                      int32_t op_errno,
-                      inode_t *inode,
-                      struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent)
+fop_symlink_cbk_stub (call_frame_t *frame, fop_symlink_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno,
+                      inode_t *inode, struct iatt *buf,
+                      struct iatt *preparent, struct iatt *postparent,
+		      dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -638,27 +624,27 @@ fop_symlink_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_SYMLINK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.symlink_cbk.fn = fn;
-        stub->args.symlink_cbk.op_ret = op_ret;
-        stub->args.symlink_cbk.op_errno = op_errno;
+        stub->fn_cbk.symlink = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (inode)
-                stub->args.symlink_cbk.inode = inode_ref (inode);
+                stub->args_cbk.inode = inode_ref (inode);
         if (buf)
-                stub->args.symlink_cbk.buf = *buf;
+                stub->args_cbk.stat = *buf;
         if (preparent)
-                stub->args.symlink_cbk.preparent = *preparent;
+                stub->args_cbk.preparent = *preparent;
         if (postparent)
-                stub->args.symlink_cbk.postparent = *postparent;
+                stub->args_cbk.postparent = *postparent;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_rename_stub (call_frame_t *frame,
-                 fop_rename_t fn,
-                 loc_t *oldloc,
-                 loc_t *newloc)
+fop_rename_stub (call_frame_t *frame, fop_rename_t fn,
+                 loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -669,24 +655,22 @@ fop_rename_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_RENAME);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.rename.fn = fn;
-        loc_copy (&stub->args.rename.old, oldloc);
-        loc_copy (&stub->args.rename.new, newloc);
+        stub->fn.rename = fn;
+        loc_copy (&stub->args.loc, oldloc);
+        loc_copy (&stub->args.loc2, newloc);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_rename_cbk_stub (call_frame_t *frame,
-                     fop_rename_cbk_t fn,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     struct iatt *buf,
-                     struct iatt *preoldparent,
-                     struct iatt *postoldparent,
-                     struct iatt *prenewparent,
-                     struct iatt *postnewparent)
+fop_rename_cbk_stub (call_frame_t *frame, fop_rename_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+		     dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -695,29 +679,29 @@ fop_rename_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_RENAME);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.rename_cbk.fn = fn;
-        stub->args.rename_cbk.op_ret = op_ret;
-        stub->args.rename_cbk.op_errno = op_errno;
+        stub->fn_cbk.rename = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (buf)
-                stub->args.rename_cbk.buf = *buf;
+                stub->args_cbk.stat = *buf;
         if (preoldparent)
-                stub->args.rename_cbk.preoldparent = *preoldparent;
+                stub->args_cbk.preparent = *preoldparent;
         if (postoldparent)
-                stub->args.rename_cbk.postoldparent = *postoldparent;
+                stub->args_cbk.postparent = *postoldparent;
         if (prenewparent)
-                stub->args.rename_cbk.prenewparent = *prenewparent;
+                stub->args_cbk.preparent2 = *prenewparent;
         if (postnewparent)
-                stub->args.rename_cbk.postnewparent = *postnewparent;
+                stub->args_cbk.postparent2 = *postnewparent;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_link_stub (call_frame_t *frame,
-               fop_link_t fn,
-               loc_t *oldloc,
-               loc_t *newloc)
+fop_link_stub (call_frame_t *frame, fop_link_t fn,
+               loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -728,24 +712,23 @@ fop_link_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_LINK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.link.fn = fn;
-        loc_copy (&stub->args.link.oldloc, oldloc);
-        loc_copy (&stub->args.link.newloc, newloc);
+        stub->fn.link = fn;
+        loc_copy (&stub->args.loc, oldloc);
+        loc_copy (&stub->args.loc2, newloc);
 
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_link_cbk_stub (call_frame_t *frame,
-                   fop_link_cbk_t fn,
-                   int32_t op_ret,
-                   int32_t op_errno,
-                   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
+fop_link_cbk_stub (call_frame_t *frame, fop_link_cbk_t fn,
+                   int32_t op_ret, int32_t op_errno,
+                   inode_t *inode, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent,
+		   dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -754,17 +737,19 @@ fop_link_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_LINK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.link_cbk.fn = fn;
-        stub->args.link_cbk.op_ret = op_ret;
-        stub->args.link_cbk.op_errno = op_errno;
+        stub->fn_cbk.link = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (inode)
-                stub->args.link_cbk.inode = inode_ref (inode);
+                stub->args_cbk.inode = inode_ref (inode);
         if (buf)
-                stub->args.link_cbk.buf = *buf;
+                stub->args_cbk.stat = *buf;
         if (preparent)
-                stub->args.link_cbk.preparent = *preparent;
+                stub->args_cbk.preparent = *preparent;
         if (postparent)
-                stub->args.link_cbk.postparent = *postparent;
+                stub->args_cbk.postparent = *postparent;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
@@ -773,7 +758,7 @@ out:
 call_stub_t *
 fop_create_stub (call_frame_t *frame, fop_create_t fn,
                  loc_t *loc, int32_t flags, mode_t mode,
-                 fd_t *fd, dict_t *params)
+                 mode_t umask, fd_t *fd, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -783,29 +768,26 @@ fop_create_stub (call_frame_t *frame, fop_create_t fn,
         stub = stub_new (frame, 1, GF_FOP_CREATE);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.create.fn = fn;
-        loc_copy (&stub->args.create.loc, loc);
-        stub->args.create.flags = flags;
-        stub->args.create.mode = mode;
+        stub->fn.create = fn;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.flags = flags;
+        stub->args.mode = mode;
+        stub->args.umask = umask;
         if (fd)
-                stub->args.create.fd = fd_ref (fd);
-        if (params)
-                stub->args.create.params = dict_ref (params);
+                stub->args.fd = fd_ref (fd);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_create_cbk_stub (call_frame_t *frame,
-                     fop_create_cbk_t fn,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     fd_t *fd,
-                     inode_t *inode,
-                     struct iatt *buf,
-                     struct iatt *preparent,
-                     struct iatt *postparent)
+fop_create_cbk_stub (call_frame_t *frame, fop_create_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno,
+                     fd_t *fd, inode_t *inode, struct iatt *buf,
+                     struct iatt *preparent, struct iatt *postparent,
+		     dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -814,30 +796,29 @@ fop_create_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_CREATE);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.create_cbk.fn = fn;
-        stub->args.create_cbk.op_ret = op_ret;
-        stub->args.create_cbk.op_errno = op_errno;
+        stub->fn_cbk.create = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (fd)
-                stub->args.create_cbk.fd = fd_ref (fd);
+                stub->args_cbk.fd = fd_ref (fd);
         if (inode)
-                stub->args.create_cbk.inode = inode_ref (inode);
+                stub->args_cbk.inode = inode_ref (inode);
         if (buf)
-                stub->args.create_cbk.buf = *buf;
+                stub->args_cbk.stat = *buf;
         if (preparent)
-                stub->args.create_cbk.preparent = *preparent;
+                stub->args_cbk.preparent = *preparent;
         if (postparent)
-                stub->args.create_cbk.postparent = *postparent;
+                stub->args_cbk.postparent = *postparent;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_open_stub (call_frame_t *frame,
-               fop_open_t fn,
-               loc_t *loc,
-               int32_t flags, fd_t *fd,
-               int32_t wbflags)
+fop_open_stub (call_frame_t *frame, fop_open_t fn,
+               loc_t *loc, int32_t flags, fd_t *fd, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -847,24 +828,22 @@ fop_open_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_OPEN);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.open.fn = fn;
-        loc_copy (&stub->args.open.loc, loc);
-        stub->args.open.flags = flags;
-        stub->args.open.wbflags = wbflags;
+        stub->fn.open = fn;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.flags = flags;
         if (fd)
-                stub->args.open.fd = fd_ref (fd);
+                stub->args.fd = fd_ref (fd);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_open_cbk_stub (call_frame_t *frame,
-                   fop_open_cbk_t fn,
-                   int32_t op_ret,
-                   int32_t op_errno,
-                   fd_t *fd)
-
+fop_open_cbk_stub (call_frame_t *frame, fop_open_cbk_t fn,
+                   int32_t op_ret, int32_t op_errno,
+                   fd_t *fd, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -873,22 +852,22 @@ fop_open_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_OPEN);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.open_cbk.fn = fn;
-        stub->args.open_cbk.op_ret = op_ret;
-        stub->args.open_cbk.op_errno = op_errno;
+        stub->fn_cbk.open = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (fd)
-                stub->args.open_cbk.fd = fd_ref (fd);
+                stub->args_cbk.fd = fd_ref (fd);
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_readv_stub (call_frame_t *frame,
-                fop_readv_t fn,
-                fd_t *fd,
-                size_t size,
-                off_t off, uint32_t flags)
+fop_readv_stub (call_frame_t *frame, fop_readv_t fn,
+                fd_t *fd, size_t size, off_t off, uint32_t flags,
+		dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -897,28 +876,25 @@ fop_readv_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_READ);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.readv.fn = fn;
+        stub->fn.readv = fn;
         if (fd)
-                stub->args.readv.fd = fd_ref (fd);
-        stub->args.readv.size  = size;
-        stub->args.readv.off   = off;
-        stub->args.readv.flags = flags;
+                stub->args.fd = fd_ref (fd);
+        stub->args.size  = size;
+        stub->args.offset  = off;
+        stub->args.flags = flags;
 
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_readv_cbk_stub (call_frame_t *frame,
-                    fop_readv_cbk_t fn,
-                    int32_t op_ret,
-                    int32_t op_errno,
-                    struct iovec *vector,
-                    int32_t count,
-                    struct iatt *stbuf,
-                    struct iobref *iobref)
-
+fop_readv_cbk_stub (call_frame_t *frame, fop_readv_cbk_t fn,
+                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
+		    int32_t count, struct iatt *stbuf,
+                    struct iobref *iobref, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -927,28 +903,26 @@ fop_readv_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_READ);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.readv_cbk.fn = fn;
-        stub->args.readv_cbk.op_ret = op_ret;
-        stub->args.readv_cbk.op_errno = op_errno;
+        stub->fn_cbk.readv = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (op_ret >= 0) {
-                stub->args.readv_cbk.vector = iov_dup (vector, count);
-                stub->args.readv_cbk.count = count;
-                stub->args.readv_cbk.stbuf = *stbuf;
-                stub->args.readv_cbk.iobref = iobref_ref (iobref);
+                stub->args_cbk.vector = iov_dup (vector, count);
+                stub->args_cbk.count = count;
+                stub->args_cbk.stat = *stbuf;
+                stub->args_cbk.iobref = iobref_ref (iobref);
         }
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_writev_stub (call_frame_t *frame,
-                 fop_writev_t fn,
-                 fd_t *fd,
-                 struct iovec *vector,
-                 int32_t count,
-                 off_t off, uint32_t flags,
-                 struct iobref *iobref)
+fop_writev_stub (call_frame_t *frame, fop_writev_t fn,
+                 fd_t *fd, struct iovec *vector, int32_t count, off_t off,
+		 uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -958,28 +932,25 @@ fop_writev_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_WRITE);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.writev.fn = fn;
+        stub->fn.writev = fn;
         if (fd)
-                stub->args.writev.fd = fd_ref (fd);
-        stub->args.writev.vector = iov_dup (vector, count);
-        stub->args.writev.count  = count;
-        stub->args.writev.off    = off;
-        stub->args.writev.flags  = flags;
-        stub->args.writev.iobref = iobref_ref (iobref);
-
+                stub->args.fd = fd_ref (fd);
+        stub->args.vector = iov_dup (vector, count);
+        stub->args.count  = count;
+        stub->args.offset = off;
+        stub->args.flags  = flags;
+        stub->args.iobref = iobref_ref (iobref);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_writev_cbk_stub (call_frame_t *frame,
-                     fop_writev_cbk_t fn,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     struct iatt *prebuf,
-                     struct iatt *postbuf)
-
+fop_writev_cbk_stub (call_frame_t *frame, fop_writev_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -988,23 +959,23 @@ fop_writev_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_WRITE);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.writev_cbk.fn = fn;
-        stub->args.writev_cbk.op_ret = op_ret;
-        stub->args.writev_cbk.op_errno = op_errno;
+        stub->fn_cbk.writev = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (op_ret >= 0)
-                stub->args.writev_cbk.postbuf = *postbuf;
+                stub->args_cbk.poststat = *postbuf;
         if (prebuf)
-                stub->args.writev_cbk.prebuf = *prebuf;
+                stub->args_cbk.prestat = *prebuf;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
-
 call_stub_t *
-fop_flush_stub (call_frame_t *frame,
-                fop_flush_t fn,
-                fd_t *fd)
+fop_flush_stub (call_frame_t *frame, fop_flush_t fn,
+                fd_t *fd, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1013,20 +984,19 @@ fop_flush_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_FLUSH);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.flush.fn = fn;
+        stub->fn.flush = fn;
         if (fd)
-                stub->args.flush.fd = fd_ref (fd);
+                stub->args.fd = fd_ref (fd);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_flush_cbk_stub (call_frame_t *frame,
-                    fop_flush_cbk_t fn,
-                    int32_t op_ret,
-                    int32_t op_errno)
-
+fop_flush_cbk_stub (call_frame_t *frame, fop_flush_cbk_t fn,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1035,21 +1005,19 @@ fop_flush_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_FLUSH);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.flush_cbk.fn = fn;
-        stub->args.flush_cbk.op_ret = op_ret;
-        stub->args.flush_cbk.op_errno = op_errno;
+        stub->fn_cbk.flush = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
-
-
 call_stub_t *
-fop_fsync_stub (call_frame_t *frame,
-                fop_fsync_t fn,
-                fd_t *fd,
-                int32_t datasync)
+fop_fsync_stub (call_frame_t *frame, fop_fsync_t fn,
+                fd_t *fd, int32_t datasync, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1058,22 +1026,21 @@ fop_fsync_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_FSYNC);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fsync.fn = fn;
+        stub->fn.fsync = fn;
         if (fd)
-                stub->args.fsync.fd = fd_ref (fd);
-        stub->args.fsync.datasync = datasync;
+                stub->args.fd = fd_ref (fd);
+        stub->args.datasync = datasync;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fsync_cbk_stub (call_frame_t *frame,
-                    fop_fsync_cbk_t fn,
-                    int32_t op_ret,
-                    int32_t op_errno,
-                    struct iatt *prebuf,
-                    struct iatt *postbuf)
+fop_fsync_cbk_stub (call_frame_t *frame, fop_fsync_cbk_t fn,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1082,22 +1049,23 @@ fop_fsync_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_FSYNC);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fsync_cbk.fn = fn;
-        stub->args.fsync_cbk.op_ret = op_ret;
-        stub->args.fsync_cbk.op_errno = op_errno;
+        stub->fn_cbk.fsync = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (prebuf)
-                stub->args.fsync_cbk.prebuf = *prebuf;
+                stub->args_cbk.prestat = *prebuf;
         if (postbuf)
-                stub->args.fsync_cbk.postbuf = *postbuf;
+                stub->args_cbk.poststat = *postbuf;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_opendir_stub (call_frame_t *frame,
-                  fop_opendir_t fn,
-                  loc_t *loc, fd_t *fd)
+fop_opendir_stub (call_frame_t *frame, fop_opendir_t fn,
+                  loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1107,22 +1075,21 @@ fop_opendir_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_OPENDIR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.opendir.fn = fn;
-        loc_copy (&stub->args.opendir.loc, loc);
+        stub->fn.opendir = fn;
+        loc_copy (&stub->args.loc, loc);
         if (fd)
-                stub->args.opendir.fd = fd_ref (fd);
+                stub->args.fd = fd_ref (fd);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_opendir_cbk_stub (call_frame_t *frame,
-                      fop_opendir_cbk_t fn,
-                      int32_t op_ret,
-                      int32_t op_errno,
-                      fd_t *fd)
-
+fop_opendir_cbk_stub (call_frame_t *frame, fop_opendir_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno,
+                      fd_t *fd, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1131,22 +1098,22 @@ fop_opendir_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_OPENDIR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.opendir_cbk.fn = fn;
-        stub->args.opendir_cbk.op_ret = op_ret;
-        stub->args.opendir_cbk.op_errno = op_errno;
+        stub->fn_cbk.opendir = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
         if (fd)
-                stub->args.opendir_cbk.fd = fd_ref (fd);
+                stub->args_cbk.fd = fd_ref (fd);
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fsyncdir_stub (call_frame_t *frame,
-                   fop_fsyncdir_t fn,
-                   fd_t *fd,
-                   int32_t datasync)
+fop_fsyncdir_stub (call_frame_t *frame, fop_fsyncdir_t fn,
+                   fd_t *fd, int32_t datasync, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1155,21 +1122,20 @@ fop_fsyncdir_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_FSYNCDIR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fsyncdir.fn = fn;
+        stub->fn.fsyncdir = fn;
         if (fd)
-                stub->args.fsyncdir.fd = fd_ref (fd);
-        stub->args.fsyncdir.datasync = datasync;
+                stub->args.fd = fd_ref (fd);
+        stub->args.datasync = datasync;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fsyncdir_cbk_stub (call_frame_t *frame,
-                       fop_fsyncdir_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno)
-
+fop_fsyncdir_cbk_stub (call_frame_t *frame, fop_fsyncdir_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1178,18 +1144,19 @@ fop_fsyncdir_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_FSYNCDIR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fsyncdir_cbk.fn = fn;
-        stub->args.fsyncdir_cbk.op_ret = op_ret;
-        stub->args.fsyncdir_cbk.op_errno = op_errno;
+        stub->fn_cbk.fsyncdir = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_statfs_stub (call_frame_t *frame,
-                 fop_statfs_t fn,
-                 loc_t *loc)
+fop_statfs_stub (call_frame_t *frame, fop_statfs_t fn,
+                 loc_t *loc, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1199,20 +1166,19 @@ fop_statfs_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_STATFS);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.statfs.fn = fn;
-        loc_copy (&stub->args.statfs.loc, loc);
+        stub->fn.statfs = fn;
+        loc_copy (&stub->args.loc, loc);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_statfs_cbk_stub (call_frame_t *frame,
-                     fop_statfs_cbk_t fn,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     struct statvfs *buf)
-
+fop_statfs_cbk_stub (call_frame_t *frame, fop_statfs_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno,
+                     struct statvfs *buf, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1221,22 +1187,22 @@ fop_statfs_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_STATFS);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.statfs_cbk.fn = fn;
-        stub->args.statfs_cbk.op_ret = op_ret;
-        stub->args.statfs_cbk.op_errno = op_errno;
+        stub->fn_cbk.statfs = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (op_ret == 0)
-                stub->args.statfs_cbk.buf = *buf;
+                stub->args_cbk.statvfs = *buf;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_setxattr_stub (call_frame_t *frame,
-                   fop_setxattr_t fn,
-                   loc_t *loc,
-                   dict_t *dict,
-                   int32_t flags)
+fop_setxattr_stub (call_frame_t *frame, fop_setxattr_t fn,
+                   loc_t *loc, dict_t *dict,
+                   int32_t flags, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1246,12 +1212,14 @@ fop_setxattr_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_SETXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.setxattr.fn = fn;
-        loc_copy (&stub->args.setxattr.loc, loc);
+        stub->fn.setxattr = fn;
+        loc_copy (&stub->args.loc, loc);
         /* TODO */
         if (dict)
-                stub->args.setxattr.dict = dict_ref (dict);
-        stub->args.setxattr.flags = flags;
+                stub->args.xattr = dict_ref (dict);
+        stub->args.flags = flags;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
@@ -1261,7 +1229,7 @@ call_stub_t *
 fop_setxattr_cbk_stub (call_frame_t *frame,
                        fop_setxattr_cbk_t fn,
                        int32_t op_ret,
-                       int32_t op_errno)
+                       int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1270,18 +1238,19 @@ fop_setxattr_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_SETXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.setxattr_cbk.fn = fn;
-        stub->args.setxattr_cbk.op_ret = op_ret;
-        stub->args.setxattr_cbk.op_errno = op_errno;
+        stub->fn_cbk.setxattr = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
-fop_getxattr_stub (call_frame_t *frame,
-                   fop_getxattr_t fn,
-                   loc_t *loc,
-                   const char *name)
+fop_getxattr_stub (call_frame_t *frame, fop_getxattr_t fn,
+                   loc_t *loc, const char *name, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1291,22 +1260,22 @@ fop_getxattr_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_GETXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.getxattr.fn = fn;
-        loc_copy (&stub->args.getxattr.loc, loc);
+        stub->fn.getxattr = fn;
+        loc_copy (&stub->args.loc, loc);
 
         if (name)
-                stub->args.getxattr.name = gf_strdup (name);
+                stub->args.name = gf_strdup (name);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_getxattr_cbk_stub (call_frame_t *frame,
-                       fop_getxattr_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       dict_t *dict)
+fop_getxattr_cbk_stub (call_frame_t *frame, fop_getxattr_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno,
+                       dict_t *dict, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1315,23 +1284,22 @@ fop_getxattr_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_GETXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.getxattr_cbk.fn = fn;
-        stub->args.getxattr_cbk.op_ret = op_ret;
-        stub->args.getxattr_cbk.op_errno = op_errno;
+        stub->fn_cbk.getxattr = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         /* TODO */
         if (dict)
-                stub->args.getxattr_cbk.dict = dict_ref (dict);
+                stub->args_cbk.xattr = dict_ref (dict);
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fsetxattr_stub (call_frame_t *frame,
-                    fop_fsetxattr_t fn,
-                    fd_t *fd,
-                    dict_t *dict,
-                    int32_t flags)
+fop_fsetxattr_stub (call_frame_t *frame, fop_fsetxattr_t fn,
+                    fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1341,23 +1309,22 @@ fop_fsetxattr_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_FSETXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fsetxattr.fn = fn;
-        stub->args.fsetxattr.fd = fd_ref (fd);
+        stub->fn.fsetxattr = fn;
+        stub->args.fd = fd_ref (fd);
 
-        /* TODO */
         if (dict)
-                stub->args.fsetxattr.dict = dict_ref (dict);
-        stub->args.fsetxattr.flags = flags;
+                stub->args.xattr = dict_ref (dict);
+        stub->args.flags = flags;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fsetxattr_cbk_stub (call_frame_t *frame,
-                        fop_fsetxattr_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno)
+fop_fsetxattr_cbk_stub (call_frame_t *frame, fop_fsetxattr_cbk_t fn,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1366,19 +1333,19 @@ fop_fsetxattr_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_FSETXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fsetxattr_cbk.fn = fn;
-        stub->args.fsetxattr_cbk.op_ret = op_ret;
-        stub->args.fsetxattr_cbk.op_errno = op_errno;
+        stub->fn_cbk.fsetxattr = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fgetxattr_stub (call_frame_t *frame,
-                    fop_fgetxattr_t fn,
-                    fd_t *fd,
-                    const char *name)
+fop_fgetxattr_stub (call_frame_t *frame, fop_fgetxattr_t fn,
+                    fd_t *fd, const char *name, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1388,22 +1355,22 @@ fop_fgetxattr_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_FGETXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fgetxattr.fn = fn;
-        stub->args.fgetxattr.fd = fd_ref (fd);
+        stub->fn.fgetxattr = fn;
+        stub->args.fd = fd_ref (fd);
 
         if (name)
-                stub->args.fgetxattr.name = gf_strdup (name);
+                stub->args.name = gf_strdup (name);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fgetxattr_cbk_stub (call_frame_t *frame,
-                        fop_fgetxattr_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        dict_t *dict)
+fop_fgetxattr_cbk_stub (call_frame_t *frame, fop_fgetxattr_cbk_t fn,
+                        int32_t op_ret, int32_t op_errno,
+                        dict_t *dict, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1412,23 +1379,22 @@ fop_fgetxattr_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_GETXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fgetxattr_cbk.fn = fn;
-        stub->args.fgetxattr_cbk.op_ret = op_ret;
-        stub->args.fgetxattr_cbk.op_errno = op_errno;
+        stub->fn_cbk.fgetxattr = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
-        /* TODO */
         if (dict)
-                stub->args.fgetxattr_cbk.dict = dict_ref (dict);
+                stub->args_cbk.xattr = dict_ref (dict);
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_removexattr_stub (call_frame_t *frame,
-                      fop_removexattr_t fn,
-                      loc_t *loc,
-                      const char *name)
+fop_removexattr_stub (call_frame_t *frame, fop_removexattr_t fn,
+                      loc_t *loc, const char *name, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1439,19 +1405,19 @@ fop_removexattr_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_REMOVEXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.removexattr.fn = fn;
-        loc_copy (&stub->args.removexattr.loc, loc);
-        stub->args.removexattr.name = gf_strdup (name);
+        stub->fn.removexattr = fn;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.name = gf_strdup (name);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_removexattr_cbk_stub (call_frame_t *frame,
-                          fop_removexattr_cbk_t fn,
-                          int32_t op_ret,
-                          int32_t op_errno)
+fop_removexattr_cbk_stub (call_frame_t *frame, fop_removexattr_cbk_t fn,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1460,18 +1426,19 @@ fop_removexattr_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_REMOVEXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.removexattr_cbk.fn = fn;
-        stub->args.removexattr_cbk.op_ret = op_ret;
-        stub->args.removexattr_cbk.op_errno = op_errno;
+        stub->fn_cbk.removexattr = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
-fop_fremovexattr_stub (call_frame_t *frame,
-                       fop_fremovexattr_t fn,
-                       fd_t *fd,
-                       const char *name)
+fop_fremovexattr_stub (call_frame_t *frame, fop_fremovexattr_t fn,
+                       fd_t *fd, const char *name, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1482,19 +1449,19 @@ fop_fremovexattr_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_FREMOVEXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fremovexattr.fn = fn;
-        stub->args.fremovexattr.fd = fd_ref (fd);
-        stub->args.fremovexattr.name = gf_strdup (name);
+        stub->fn.fremovexattr = fn;
+        stub->args.fd = fd_ref (fd);
+        stub->args.name = gf_strdup (name);
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fremovexattr_cbk_stub (call_frame_t *frame,
-                           fop_fremovexattr_cbk_t fn,
-                           int32_t op_ret,
-                           int32_t op_errno)
+fop_fremovexattr_cbk_stub (call_frame_t *frame, fop_fremovexattr_cbk_t fn,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1503,19 +1470,20 @@ fop_fremovexattr_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_FREMOVEXATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fremovexattr_cbk.fn = fn;
-        stub->args.fremovexattr_cbk.op_ret = op_ret;
-        stub->args.fremovexattr_cbk.op_errno = op_errno;
+        stub->fn_cbk.fremovexattr = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
-fop_lk_stub (call_frame_t *frame,
-             fop_lk_t fn,
-             fd_t *fd,
-             int32_t cmd,
-             struct gf_flock *lock)
+fop_lk_stub (call_frame_t *frame, fop_lk_t fn,
+             fd_t *fd, int32_t cmd,
+             struct gf_flock *lock, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1525,23 +1493,22 @@ fop_lk_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_LK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.lk.fn = fn;
+        stub->fn.lk = fn;
         if (fd)
-                stub->args.lk.fd = fd_ref (fd);
-        stub->args.lk.cmd = cmd;
-        stub->args.lk.lock = *lock;
+                stub->args.fd = fd_ref (fd);
+        stub->args.cmd = cmd;
+        stub->args.lock = *lock;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_lk_cbk_stub (call_frame_t *frame,
-                 fop_lk_cbk_t fn,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 struct gf_flock *lock)
-
+fop_lk_cbk_stub (call_frame_t *frame, fop_lk_cbk_t fn,
+                 int32_t op_ret, int32_t op_errno,
+                 struct gf_flock *lock, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1550,18 +1517,22 @@ fop_lk_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_LK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.lk_cbk.fn = fn;
-        stub->args.lk_cbk.op_ret = op_ret;
-        stub->args.lk_cbk.op_errno = op_errno;
+        stub->fn_cbk.lk = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (op_ret == 0)
-                stub->args.lk_cbk.lock = *lock;
+                stub->args_cbk.lock = *lock;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
 fop_inodelk_stub (call_frame_t *frame, fop_inodelk_t fn,
-                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
+                  const char *volume, loc_t *loc, int32_t cmd,
+                  struct gf_flock *lock, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1571,21 +1542,24 @@ fop_inodelk_stub (call_frame_t *frame, fop_inodelk_t fn,
         stub = stub_new (frame, 1, GF_FOP_INODELK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.inodelk.fn = fn;
+        stub->fn.inodelk = fn;
 
         if (volume)
-                stub->args.inodelk.volume = gf_strdup (volume);
+                stub->args.volume = gf_strdup (volume);
 
-        loc_copy (&stub->args.inodelk.loc, loc);
-        stub->args.inodelk.cmd  = cmd;
-        stub->args.inodelk.lock = *lock;
+        loc_copy (&stub->args.loc, loc);
+        stub->args.cmd  = cmd;
+        stub->args.lock = *lock;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
 fop_inodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1594,10 +1568,12 @@ fop_inodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
         stub = stub_new (frame, 0, GF_FOP_INODELK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.inodelk_cbk.fn       = fn;
-        stub->args.inodelk_cbk.op_ret   = op_ret;
-        stub->args.inodelk_cbk.op_errno = op_errno;
+        stub->fn_cbk.inodelk       = fn;
+        stub->args_cbk.op_ret   = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
@@ -1605,7 +1581,8 @@ out:
 
 call_stub_t *
 fop_finodelk_stub (call_frame_t *frame, fop_finodelk_t fn,
-                   const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
+                   const char *volume, fd_t *fd, int32_t cmd,
+                   struct gf_flock *lock, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1615,17 +1592,19 @@ fop_finodelk_stub (call_frame_t *frame, fop_finodelk_t fn,
         stub = stub_new (frame, 1, GF_FOP_FINODELK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.finodelk.fn = fn;
+        stub->fn.finodelk = fn;
 
         if (fd)
-                stub->args.finodelk.fd   = fd_ref (fd);
+                stub->args.fd   = fd_ref (fd);
 
         if (volume)
-                stub->args.finodelk.volume = gf_strdup (volume);
+                stub->args.volume = gf_strdup (volume);
 
-        stub->args.finodelk.cmd  = cmd;
-        stub->args.finodelk.lock = *lock;
+        stub->args.cmd  = cmd;
+        stub->args.lock = *lock;
 
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
@@ -1633,7 +1612,7 @@ out:
 
 call_stub_t *
 fop_finodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1642,10 +1621,12 @@ fop_finodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
         stub = stub_new (frame, 0, GF_FOP_FINODELK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.finodelk_cbk.fn       = fn;
-        stub->args.finodelk_cbk.op_ret   = op_ret;
-        stub->args.finodelk_cbk.op_errno = op_errno;
+        stub->fn_cbk.finodelk       = fn;
+        stub->args_cbk.op_ret   = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
@@ -1654,7 +1635,7 @@ out:
 call_stub_t *
 fop_entrylk_stub (call_frame_t *frame, fop_entrylk_t fn,
                   const char *volume, loc_t *loc, const char *name,
-                  entrylk_cmd cmd, entrylk_type type)
+                  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1663,25 +1644,29 @@ fop_entrylk_stub (call_frame_t *frame, fop_entrylk_t fn,
         stub = stub_new (frame, 1, GF_FOP_ENTRYLK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.entrylk.fn = fn;
+        stub->fn.entrylk = fn;
 
         if (volume)
-                stub->args.entrylk.volume = gf_strdup (volume);
+                stub->args.volume = gf_strdup (volume);
+
+        loc_copy (&stub->args.loc, loc);
 
-        loc_copy (&stub->args.entrylk.loc, loc);
+        stub->args.entrylkcmd = cmd;
+        stub->args.entrylktype = type;
 
-        stub->args.entrylk.cmd = cmd;
-        stub->args.entrylk.type = type;
         if (name)
-                stub->args.entrylk.name = gf_strdup (name);
+                stub->args.name = gf_strdup (name);
 
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
 fop_entrylk_cbk_stub (call_frame_t *frame, fop_entrylk_cbk_t fn,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1690,10 +1675,12 @@ fop_entrylk_cbk_stub (call_frame_t *frame, fop_entrylk_cbk_t fn,
         stub = stub_new (frame, 0, GF_FOP_ENTRYLK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.entrylk_cbk.fn       = fn;
-        stub->args.entrylk_cbk.op_ret   = op_ret;
-        stub->args.entrylk_cbk.op_errno = op_errno;
+        stub->fn_cbk.entrylk       = fn;
+        stub->args_cbk.op_ret   = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
@@ -1702,7 +1689,7 @@ out:
 call_stub_t *
 fop_fentrylk_stub (call_frame_t *frame, fop_fentrylk_t fn,
                    const char *volume, fd_t *fd, const char *name,
-                   entrylk_cmd cmd, entrylk_type type)
+                   entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1711,25 +1698,28 @@ fop_fentrylk_stub (call_frame_t *frame, fop_fentrylk_t fn,
         stub = stub_new (frame, 1, GF_FOP_FENTRYLK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fentrylk.fn = fn;
+        stub->fn.fentrylk = fn;
 
         if (volume)
-                stub->args.fentrylk.volume = gf_strdup (volume);
+                stub->args.volume = gf_strdup (volume);
 
         if (fd)
-                stub->args.fentrylk.fd = fd_ref (fd);
-        stub->args.fentrylk.cmd = cmd;
-        stub->args.fentrylk.type = type;
+                stub->args.fd = fd_ref (fd);
+        stub->args.entrylkcmd = cmd;
+        stub->args.entrylktype = type;
         if (name)
-                stub->args.fentrylk.name = gf_strdup (name);
+                stub->args.name = gf_strdup (name);
 
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
 fop_fentrylk_cbk_stub (call_frame_t *frame, fop_fentrylk_cbk_t fn,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1738,21 +1728,21 @@ fop_fentrylk_cbk_stub (call_frame_t *frame, fop_fentrylk_cbk_t fn,
         stub = stub_new (frame, 0, GF_FOP_FENTRYLK);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fentrylk_cbk.fn       = fn;
-        stub->args.fentrylk_cbk.op_ret   = op_ret;
-        stub->args.fentrylk_cbk.op_errno = op_errno;
+        stub->fn_cbk.fentrylk       = fn;
+        stub->args_cbk.op_ret   = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_readdirp_cbk_stub (call_frame_t *frame,
-                       fop_readdirp_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       gf_dirent_t *entries)
+fop_readdirp_cbk_stub (call_frame_t *frame, fop_readdirp_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno,
+                       gf_dirent_t *entries, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
         gf_dirent_t *stub_entry = NULL, *entry = NULL;
@@ -1762,14 +1752,10 @@ fop_readdirp_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_READDIRP);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.readdirp_cbk.fn = fn;
-        stub->args.readdirp_cbk.op_ret = op_ret;
-        stub->args.readdirp_cbk.op_errno = op_errno;
-        INIT_LIST_HEAD (&stub->args.readdirp_cbk.entries.list);
+        stub->fn_cbk.readdirp = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
-        /* This check must come after the init of head above
-         * so we're sure the list is empty for list_empty.
-         */
         GF_VALIDATE_OR_GOTO ("call-stub", entries, out);
 
         if (op_ret > 0) {
@@ -1780,21 +1766,23 @@ fop_readdirp_cbk_stub (call_frame_t *frame,
                         stub_entry->d_off = entry->d_off;
                         stub_entry->d_ino = entry->d_ino;
                         stub_entry->d_stat = entry->d_stat;
+			if (entry->inode)
+				stub_entry->inode = inode_ref (entry->inode);
                         list_add_tail (&stub_entry->list,
-                                       &stub->args.readdirp_cbk.entries.list);
+                                       &stub->args_cbk.entries.list);
                 }
         }
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_readdir_cbk_stub (call_frame_t *frame,
-                      fop_readdir_cbk_t fn,
-                      int32_t op_ret,
-                      int32_t op_errno,
-                      gf_dirent_t *entries)
+fop_readdir_cbk_stub (call_frame_t *frame, fop_readdir_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno,
+                      gf_dirent_t *entries, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
         gf_dirent_t *stub_entry = NULL, *entry = NULL;
@@ -1804,14 +1792,10 @@ fop_readdir_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_READDIR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.readdir_cbk.fn = fn;
-        stub->args.readdir_cbk.op_ret = op_ret;
-        stub->args.readdir_cbk.op_errno = op_errno;
-        INIT_LIST_HEAD (&stub->args.readdir_cbk.entries.list);
+        stub->fn_cbk.readdir = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
-        /* This check must come after the init of head above
-         * so we're sure the list is empty for list_empty.
-         */
         GF_VALIDATE_OR_GOTO ("call-stub", entries, out);
 
         if (op_ret > 0) {
@@ -1823,62 +1807,61 @@ fop_readdir_cbk_stub (call_frame_t *frame,
                         stub_entry->d_ino = entry->d_ino;
 
                         list_add_tail (&stub_entry->list,
-                                       &stub->args.readdir_cbk.entries.list);
+                                       &stub->args_cbk.entries.list);
                 }
         }
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
-fop_readdir_stub (call_frame_t *frame,
-                  fop_readdir_t fn,
-                  fd_t *fd,
-                  size_t size,
-                  off_t off)
+fop_readdir_stub (call_frame_t *frame, fop_readdir_t fn,
+                  fd_t *fd, size_t size,
+                  off_t off, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
         stub = stub_new (frame, 1, GF_FOP_READDIR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.readdir.fn = fn;
-        stub->args.readdir.fd = fd_ref (fd);
-        stub->args.readdir.size = size;
-        stub->args.readdir.off = off;
+        stub->fn.readdir = fn;
+        stub->args.fd = fd_ref (fd);
+        stub->args.size = size;
+        stub->args.offset = off;
 
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
-fop_readdirp_stub (call_frame_t *frame,
-                   fop_readdirp_t fn,
-                   fd_t *fd,
-                   size_t size,
-                   off_t off,
-                   dict_t *dict)
+fop_readdirp_stub (call_frame_t *frame, fop_readdirp_t fn,
+                   fd_t *fd, size_t size, off_t off, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
         stub = stub_new (frame, 1, GF_FOP_READDIRP);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.readdirp.fn = fn;
-        stub->args.readdirp.fd = fd_ref (fd);
-        stub->args.readdirp.size = size;
-        stub->args.readdirp.off = off;
-        stub->args.readdirp.dict = dict;
-
+        stub->fn.readdirp = fn;
+        stub->args.fd = fd_ref (fd);
+        stub->args.size = size;
+        stub->args.offset = off;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
-fop_rchecksum_stub (call_frame_t *frame,
-                    fop_rchecksum_t fn,
-                    fd_t *fd, off_t offset,
-                    int32_t len)
+fop_rchecksum_stub (call_frame_t *frame, fop_rchecksum_t fn,
+                    fd_t *fd, off_t offset, int32_t len, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1888,22 +1871,22 @@ fop_rchecksum_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_RCHECKSUM);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.rchecksum.fn = fn;
-        stub->args.rchecksum.fd = fd_ref (fd);
-        stub->args.rchecksum.offset = offset;
-        stub->args.rchecksum.len    = len;
+        stub->fn.rchecksum = fn;
+        stub->args.fd = fd_ref (fd);
+        stub->args.offset = offset;
+        stub->args.size    = len;
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_rchecksum_cbk_stub (call_frame_t *frame,
-                        fop_rchecksum_cbk_t fn,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        uint32_t weak_checksum,
-                        uint8_t *strong_checksum)
+fop_rchecksum_cbk_stub (call_frame_t *frame, fop_rchecksum_cbk_t fn,
+                        int32_t op_ret, int32_t op_errno,
+                        uint32_t weak_checksum, uint8_t *strong_checksum,
+			dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1912,28 +1895,27 @@ fop_rchecksum_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_RCHECKSUM);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.rchecksum_cbk.fn = fn;
-        stub->args.rchecksum_cbk.op_ret = op_ret;
-        stub->args.rchecksum_cbk.op_errno = op_errno;
+        stub->fn_cbk.rchecksum = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
-        if (op_ret >= 0)
-        {
-                stub->args.rchecksum_cbk.weak_checksum =
+        if (op_ret >= 0) {
+                stub->args_cbk.weak_checksum =
                         weak_checksum;
-
-                stub->args.rchecksum_cbk.strong_checksum =
-                        memdup (strong_checksum, MD5_DIGEST_LEN);
+                stub->args_cbk.strong_checksum =
+                        memdup (strong_checksum, MD5_DIGEST_LENGTH);
         }
+
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_xattrop_cbk_stub (call_frame_t *frame,
-                      fop_xattrop_cbk_t fn,
-                      int32_t op_ret,
-                      int32_t op_errno)
+fop_xattrop_cbk_stub (call_frame_t *frame, fop_xattrop_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1942,21 +1924,21 @@ fop_xattrop_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_XATTROP);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.xattrop_cbk.fn       = fn;
-        stub->args.xattrop_cbk.op_ret   = op_ret;
-        stub->args.xattrop_cbk.op_errno = op_errno;
+        stub->fn_cbk.xattrop       = fn;
+        stub->args_cbk.op_ret   = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_fxattrop_cbk_stub (call_frame_t *frame,
-                       fop_fxattrop_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       dict_t *xattr)
+fop_fxattrop_cbk_stub (call_frame_t *frame, fop_fxattrop_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno,
+                       dict_t *xattr, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
         GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
@@ -1964,23 +1946,23 @@ fop_fxattrop_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_FXATTROP);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fxattrop_cbk.fn = fn;
-        stub->args.fxattrop_cbk.op_ret = op_ret;
-        stub->args.fxattrop_cbk.op_errno = op_errno;
+        stub->fn_cbk.fxattrop = fn;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
         if (xattr)
-                stub->args.fxattrop_cbk.xattr = dict_ref (xattr);
+                stub->args_cbk.xattr = dict_ref (xattr);
 
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_xattrop_stub (call_frame_t *frame,
-                  fop_xattrop_t fn,
-                  loc_t *loc,
-                  gf_xattrop_flags_t optype,
-                  dict_t *xattr)
+fop_xattrop_stub (call_frame_t *frame, fop_xattrop_t fn,
+                  loc_t *loc, gf_xattrop_flags_t optype,
+                  dict_t *xattr, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -1990,23 +1972,24 @@ fop_xattrop_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_XATTROP);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.xattrop.fn = fn;
+        stub->fn.xattrop = fn;
 
-        loc_copy (&stub->args.xattrop.loc, loc);
+        loc_copy (&stub->args.loc, loc);
 
-        stub->args.xattrop.optype = optype;
-        stub->args.xattrop.xattr = dict_ref (xattr);
+        stub->args.optype = optype;
+        stub->args.xattr = dict_ref (xattr);
 
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
-fop_fxattrop_stub (call_frame_t *frame,
-                   fop_fxattrop_t fn,
-                   fd_t *fd,
-                   gf_xattrop_flags_t optype,
-                   dict_t *xattr)
+fop_fxattrop_stub (call_frame_t *frame, fop_fxattrop_t fn,
+                   fd_t *fd, gf_xattrop_flags_t optype,
+                   dict_t *xattr, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -2016,25 +1999,25 @@ fop_fxattrop_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_FXATTROP);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fxattrop.fn = fn;
+        stub->fn.fxattrop = fn;
 
-        stub->args.fxattrop.fd = fd_ref (fd);
+        stub->args.fd = fd_ref (fd);
 
-        stub->args.fxattrop.optype = optype;
-        stub->args.fxattrop.xattr = dict_ref (xattr);
+        stub->args.optype = optype;
+        stub->args.xattr = dict_ref (xattr);
 
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
 
 call_stub_t *
-fop_setattr_cbk_stub (call_frame_t *frame,
-                      fop_setattr_cbk_t fn,
-                      int32_t op_ret,
-                      int32_t op_errno,
-                      struct iatt *statpre,
-                      struct iatt *statpost)
+fop_setattr_cbk_stub (call_frame_t *frame, fop_setattr_cbk_t fn,
+                      int32_t op_ret, int32_t op_errno,
+                      struct iatt *statpre, struct iatt *statpost,
+		      dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -2043,27 +2026,28 @@ fop_setattr_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_SETATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.setattr_cbk.fn = fn;
+        stub->fn_cbk.setattr = fn;
 
-        stub->args.setattr_cbk.op_ret = op_ret;
-        stub->args.setattr_cbk.op_errno = op_errno;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
         if (statpre)
-                stub->args.setattr_cbk.statpre = *statpre;
+                stub->args_cbk.prestat = *statpre;
         if (statpost)
-                stub->args.setattr_cbk.statpost = *statpost;
+                stub->args_cbk.poststat = *statpost;
 
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
-fop_fsetattr_cbk_stub (call_frame_t *frame,
-                       fop_setattr_cbk_t fn,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       struct iatt *statpre,
-                       struct iatt *statpost)
+fop_fsetattr_cbk_stub (call_frame_t *frame, fop_setattr_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *statpre, struct iatt *statpost,
+		       dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -2072,25 +2056,26 @@ fop_fsetattr_cbk_stub (call_frame_t *frame,
         stub = stub_new (frame, 0, GF_FOP_FSETATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fsetattr_cbk.fn = fn;
+        stub->fn_cbk.fsetattr = fn;
 
-        stub->args.fsetattr_cbk.op_ret = op_ret;
-        stub->args.fsetattr_cbk.op_errno = op_errno;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
         if (statpre)
-                stub->args.setattr_cbk.statpre = *statpre;
+                stub->args_cbk.prestat = *statpre;
         if (statpost)
-                stub->args.fsetattr_cbk.statpost = *statpost;
+                stub->args_cbk.poststat = *statpost;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
-fop_setattr_stub (call_frame_t *frame,
-                  fop_setattr_t fn,
-                  loc_t *loc,
-                  struct iatt *stbuf,
-                  int32_t valid)
+fop_setattr_stub (call_frame_t *frame, fop_setattr_t fn,
+                  loc_t *loc, struct iatt *stbuf,
+                  int32_t valid, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -2100,25 +2085,26 @@ fop_setattr_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_SETATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.setattr.fn = fn;
+        stub->fn.setattr = fn;
 
-        loc_copy (&stub->args.setattr.loc, loc);
+        loc_copy (&stub->args.loc, loc);
 
         if (stbuf)
-                stub->args.setattr.stbuf = *stbuf;
+                stub->args.stat = *stbuf;
 
-        stub->args.setattr.valid = valid;
+        stub->args.valid = valid;
 
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
+
 call_stub_t *
-fop_fsetattr_stub (call_frame_t *frame,
-                   fop_fsetattr_t fn,
-                   fd_t *fd,
-                   struct iatt *stbuf,
-                   int32_t valid)
+fop_fsetattr_stub (call_frame_t *frame, fop_fsetattr_t fn,
+                   fd_t *fd, struct iatt *stbuf,
+                   int32_t valid, dict_t *xdata)
 {
         call_stub_t *stub = NULL;
 
@@ -2128,1779 +2114,695 @@ fop_fsetattr_stub (call_frame_t *frame,
         stub = stub_new (frame, 1, GF_FOP_FSETATTR);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        stub->args.fsetattr.fn = fn;
+        stub->fn.fsetattr = fn;
 
         if (fd)
-                stub->args.fsetattr.fd = fd_ref (fd);
+                stub->args.fd = fd_ref (fd);
 
         if (stbuf)
-                stub->args.fsetattr.stbuf = *stbuf;
+                stub->args.stat = *stbuf;
 
-        stub->args.fsetattr.valid = valid;
+        stub->args.valid = valid;
 
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
         return stub;
 }
 
-static void
-call_resume_wind (call_stub_t *stub)
+call_stub_t *
+fop_fallocate_cbk_stub(call_frame_t *frame, fop_fallocate_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *statpre, struct iatt *statpost,
+		       dict_t *xdata)
 {
+        call_stub_t *stub = NULL;
+
+        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+
+        stub = stub_new (frame, 0, GF_FOP_FALLOCATE);
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        switch (stub->fop) {
-        case GF_FOP_OPEN:
-        {
-                stub->args.open.fn (stub->frame,
-                                    stub->frame->this,
-                                    &stub->args.open.loc,
-                                    stub->args.open.flags, stub->args.open.fd,
-                                    stub->args.open.wbflags);
-                break;
-        }
-        case GF_FOP_CREATE:
-        {
-                stub->args.create.fn (stub->frame,
-                                      stub->frame->this,
-                                      &stub->args.create.loc,
-                                      stub->args.create.flags,
-                                      stub->args.create.mode,
-                                      stub->args.create.fd,
-                                      stub->args.create.params);
-                break;
-        }
-        case GF_FOP_STAT:
-        {
-                stub->args.stat.fn (stub->frame,
-                                    stub->frame->this,
-                                    &stub->args.stat.loc);
-                break;
-        }
-        case GF_FOP_READLINK:
-        {
-                stub->args.readlink.fn (stub->frame,
-                                        stub->frame->this,
-                                        &stub->args.readlink.loc,
-                                        stub->args.readlink.size);
-                break;
-        }
+        stub->fn_cbk.fallocate = fn;
 
-        case GF_FOP_MKNOD:
-        {
-                stub->args.mknod.fn (stub->frame, stub->frame->this,
-                                     &stub->args.mknod.loc,
-                                     stub->args.mknod.mode,
-                                     stub->args.mknod.rdev,
-                                     stub->args.mknod.params);
-        }
-        break;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
-        case GF_FOP_MKDIR:
-        {
-                stub->args.mkdir.fn (stub->frame, stub->frame->this,
-                                     &stub->args.mkdir.loc,
-                                     stub->args.mkdir.mode,
-                                     stub->args.mkdir.params);
-        }
-        break;
+        if (statpre)
+                stub->args_cbk.prestat = *statpre;
+        if (statpost)
+                stub->args_cbk.poststat = *statpost;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
+out:
+        return stub;
+}
 
-        case GF_FOP_UNLINK:
-        {
-                stub->args.unlink.fn (stub->frame,
-                                      stub->frame->this,
-                                      &stub->args.unlink.loc);
-        }
-        break;
+call_stub_t *
+fop_fallocate_stub(call_frame_t *frame, fop_fallocate_t fn, fd_t *fd,
+		   int32_t mode, off_t offset, size_t len, dict_t *xdata)
+{
+        call_stub_t *stub = NULL;
 
-        case GF_FOP_RMDIR:
-        {
-                stub->args.rmdir.fn (stub->frame, stub->frame->this,
-                                     &stub->args.rmdir.loc,
-                                     stub->args.rmdir.flags);
-        }
-        break;
+        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
 
-        case GF_FOP_SYMLINK:
-        {
-                stub->args.symlink.fn (stub->frame,
-                                       stub->frame->this,
-                                       stub->args.symlink.linkname,
-                                       &stub->args.symlink.loc,
-                                       stub->args.symlink.params);
-        }
-        break;
+        stub = stub_new (frame, 1, GF_FOP_FALLOCATE);
+        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        case GF_FOP_RENAME:
-        {
-                stub->args.rename.fn (stub->frame,
-                                      stub->frame->this,
-                                      &stub->args.rename.old,
-                                      &stub->args.rename.new);
-        }
-        break;
+        stub->fn.fallocate = fn;
 
-        case GF_FOP_LINK:
-        {
-                stub->args.link.fn (stub->frame,
-                                    stub->frame->this,
-                                    &stub->args.link.oldloc,
-                                    &stub->args.link.newloc);
-        }
-        break;
+        if (fd)
+                stub->args.fd = fd_ref (fd);
 
-        case GF_FOP_TRUNCATE:
-        {
-                stub->args.truncate.fn (stub->frame,
-                                        stub->frame->this,
-                                        &stub->args.truncate.loc,
-                                        stub->args.truncate.off);
-                break;
-        }
+	stub->args.flags = mode;
+	stub->args.offset = offset;
+	stub->args.size = len;
 
-        case GF_FOP_READ:
-        {
-                stub->args.readv.fn (stub->frame,
-                                     stub->frame->this,
-                                     stub->args.readv.fd,
-                                     stub->args.readv.size,
-                                     stub->args.readv.off,
-                                     stub->args.readv.flags);
-                break;
-        }
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
+out:
+        return stub;
 
-        case GF_FOP_WRITE:
-        {
-                stub->args.writev.fn (stub->frame,
-                                      stub->frame->this,
-                                      stub->args.writev.fd,
-                                      stub->args.writev.vector,
-                                      stub->args.writev.count,
-                                      stub->args.writev.off,
-                                      stub->args.writev.flags,
-                                      stub->args.writev.iobref);
-                break;
-        }
+}
 
-        case GF_FOP_STATFS:
-        {
-                stub->args.statfs.fn (stub->frame,
-                                      stub->frame->this,
-                                      &stub->args.statfs.loc);
-                break;
-        }
-        case GF_FOP_FLUSH:
-        {
-                stub->args.flush.fn (stub->frame,
-                                     stub->frame->this,
-                                     stub->args.flush.fd);
-                break;
-        }
+call_stub_t *
+fop_discard_cbk_stub(call_frame_t *frame, fop_discard_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *statpre, struct iatt *statpost,
+		     dict_t *xdata)
+{
+        call_stub_t *stub = NULL;
 
-        case GF_FOP_FSYNC:
-        {
-                stub->args.fsync.fn (stub->frame,
-                                     stub->frame->this,
-                                     stub->args.fsync.fd,
-                                     stub->args.fsync.datasync);
-                break;
-        }
+        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
 
-        case GF_FOP_SETXATTR:
-        {
-                stub->args.setxattr.fn (stub->frame,
-                                        stub->frame->this,
-                                        &stub->args.setxattr.loc,
-                                        stub->args.setxattr.dict,
-                                        stub->args.setxattr.flags);
-                break;
-        }
+        stub = stub_new (frame, 0, GF_FOP_DISCARD);
+        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        case GF_FOP_GETXATTR:
-        {
-                stub->args.getxattr.fn (stub->frame,
-                                        stub->frame->this,
-                                        &stub->args.getxattr.loc,
-                                        stub->args.getxattr.name);
-                break;
-        }
+        stub->fn_cbk.discard = fn;
 
-        case GF_FOP_FSETXATTR:
-        {
-                stub->args.fsetxattr.fn (stub->frame,
-                                         stub->frame->this,
-                                         stub->args.fsetxattr.fd,
-                                         stub->args.fsetxattr.dict,
-                                         stub->args.fsetxattr.flags);
-                break;
-        }
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
-        case GF_FOP_FGETXATTR:
-        {
-                stub->args.fgetxattr.fn (stub->frame,
-                                         stub->frame->this,
-                                         stub->args.fgetxattr.fd,
-                                         stub->args.fgetxattr.name);
-                break;
-        }
+        if (statpre)
+                stub->args_cbk.prestat = *statpre;
+        if (statpost)
+                stub->args_cbk.poststat = *statpost;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
+out:
+        return stub;
+}
 
-        case GF_FOP_REMOVEXATTR:
-        {
-                stub->args.removexattr.fn (stub->frame,
-                                           stub->frame->this,
-                                           &stub->args.removexattr.loc,
-                                           stub->args.removexattr.name);
-                break;
-        }
+call_stub_t *
+fop_discard_stub(call_frame_t *frame, fop_discard_t fn, fd_t *fd,
+		 off_t offset, size_t len, dict_t *xdata)
+{
+        call_stub_t *stub = NULL;
 
-        case GF_FOP_FREMOVEXATTR:
-        {
-                stub->args.fremovexattr.fn (stub->frame,
-                                            stub->frame->this,
-                                            stub->args.fremovexattr.fd,
-                                            stub->args.fremovexattr.name);
-                break;
-        }
+        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
 
-        case GF_FOP_OPENDIR:
-        {
-                stub->args.opendir.fn (stub->frame,
-                                       stub->frame->this,
-                                       &stub->args.opendir.loc,
-                                       stub->args.opendir.fd);
-                break;
-        }
+        stub = stub_new (frame, 1, GF_FOP_DISCARD);
+        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        case GF_FOP_FSYNCDIR:
-        {
-                stub->args.fsyncdir.fn (stub->frame,
-                                        stub->frame->this,
-                                        stub->args.fsyncdir.fd,
-                                        stub->args.fsyncdir.datasync);
-                break;
-        }
+        stub->fn.discard = fn;
 
-        case GF_FOP_ACCESS:
-        {
-                stub->args.access.fn (stub->frame,
-                                      stub->frame->this,
-                                      &stub->args.access.loc,
-                                      stub->args.access.mask);
-                break;
-        }
+        if (fd)
+                stub->args.fd = fd_ref (fd);
 
-        case GF_FOP_FTRUNCATE:
-        {
-                stub->args.ftruncate.fn (stub->frame,
-                                         stub->frame->this,
-                                         stub->args.ftruncate.fd,
-                                         stub->args.ftruncate.off);
-                break;
-        }
+	stub->args.offset = offset;
+	stub->args.size = len;
 
-        case GF_FOP_FSTAT:
-        {
-                stub->args.fstat.fn (stub->frame,
-                                     stub->frame->this,
-                                     stub->args.fstat.fd);
-                break;
-        }
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
+out:
+        return stub;
 
-        case GF_FOP_LK:
-        {
-                stub->args.lk.fn (stub->frame,
-                                  stub->frame->this,
-                                  stub->args.lk.fd,
-                                  stub->args.lk.cmd,
-                                  &stub->args.lk.lock);
-                break;
-        }
+}
 
-        case GF_FOP_INODELK:
-        {
-                stub->args.inodelk.fn (stub->frame,
-                                       stub->frame->this,
-                                       stub->args.inodelk.volume,
-                                       &stub->args.inodelk.loc,
-                                       stub->args.inodelk.cmd,
-                                       &stub->args.inodelk.lock);
-                break;
-        }
+call_stub_t *
+fop_zerofill_cbk_stub(call_frame_t *frame, fop_zerofill_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *statpre, struct iatt *statpost,
+                     dict_t *xdata)
+{
+        call_stub_t *stub = NULL;
 
-        case GF_FOP_FINODELK:
-        {
-                stub->args.finodelk.fn (stub->frame,
-                                        stub->frame->this,
-                                        stub->args.finodelk.volume,
-                                        stub->args.finodelk.fd,
-                                        stub->args.finodelk.cmd,
-                                        &stub->args.finodelk.lock);
-                break;
-        }
+        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
 
-        case GF_FOP_ENTRYLK:
-        {
-                stub->args.entrylk.fn (stub->frame,
-                                       stub->frame->this,
-                                       stub->args.entrylk.volume,
-                                       &stub->args.entrylk.loc,
-                                       stub->args.entrylk.name,
-                                       stub->args.entrylk.cmd,
-                                       stub->args.entrylk.type);
-                break;
-        }
+        stub = stub_new (frame, 0, GF_FOP_ZEROFILL);
+        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        case GF_FOP_FENTRYLK:
-        {
-                stub->args.fentrylk.fn (stub->frame,
-                                        stub->frame->this,
-                                        stub->args.fentrylk.volume,
-                                        stub->args.fentrylk.fd,
-                                        stub->args.fentrylk.name,
-                                        stub->args.fentrylk.cmd,
-                                        stub->args.fentrylk.type);
-                break;
-        }
+        stub->fn_cbk.zerofill = fn;
 
-        break;
+        stub->args_cbk.op_ret = op_ret;
+        stub->args_cbk.op_errno = op_errno;
 
-        case GF_FOP_LOOKUP:
-        {
-                stub->args.lookup.fn (stub->frame,
-                                      stub->frame->this,
-                                      &stub->args.lookup.loc,
-                                      stub->args.lookup.xattr_req);
-                break;
-        }
+        if (statpre)
+                stub->args_cbk.prestat = *statpre;
+        if (statpost)
+                stub->args_cbk.poststat = *statpost;
+        if (xdata)
+                stub->args_cbk.xdata = dict_ref (xdata);
+out:
+        return stub;
+}
 
-        case GF_FOP_RCHECKSUM:
-        {
-                stub->args.rchecksum.fn (stub->frame,
-                                         stub->frame->this,
-                                         stub->args.rchecksum.fd,
-                                         stub->args.rchecksum.offset,
-                                         stub->args.rchecksum.len);
-                break;
-        }
+call_stub_t *
+fop_zerofill_stub(call_frame_t *frame, fop_zerofill_t fn, fd_t *fd,
+                 off_t offset, size_t len, dict_t *xdata)
+{
+        call_stub_t *stub = NULL;
 
-        case GF_FOP_READDIR:
-        {
-                stub->args.readdir.fn (stub->frame,
-                                       stub->frame->this,
-                                       stub->args.readdir.fd,
-                                       stub->args.readdir.size,
-                                       stub->args.readdir.off);
-                break;
-        }
+        GF_VALIDATE_OR_GOTO ("call-stub", frame, out);
+        GF_VALIDATE_OR_GOTO ("call-stub", fn, out);
 
-        case GF_FOP_READDIRP:
-        {
-                stub->args.readdirp.fn (stub->frame,
-                                        stub->frame->this,
-                                        stub->args.readdirp.fd,
-                                        stub->args.readdirp.size,
-                                        stub->args.readdirp.off,
-                                        stub->args.readdirp.dict);
-                break;
-        }
+        stub = stub_new (frame, 1, GF_FOP_ZEROFILL);
+        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        case GF_FOP_XATTROP:
-        {
-                stub->args.xattrop.fn (stub->frame,
-                                       stub->frame->this,
-                                       &stub->args.xattrop.loc,
-                                       stub->args.xattrop.optype,
-                                       stub->args.xattrop.xattr);
+        stub->fn.zerofill = fn;
 
-                break;
-        }
-        case GF_FOP_FXATTROP:
-        {
-                stub->args.fxattrop.fn (stub->frame,
-                                        stub->frame->this,
-                                        stub->args.fxattrop.fd,
-                                        stub->args.fxattrop.optype,
-                                        stub->args.fxattrop.xattr);
+        if (fd)
+                stub->args.fd = fd_ref (fd);
 
-                break;
-        }
-        case GF_FOP_SETATTR:
-        {
-                stub->args.setattr.fn (stub->frame,
-                                       stub->frame->this,
-                                       &stub->args.setattr.loc,
-                                       &stub->args.setattr.stbuf,
-                                       stub->args.setattr.valid);
-                break;
-        }
-        case GF_FOP_FSETATTR:
-        {
-                stub->args.fsetattr.fn (stub->frame,
-                                        stub->frame->this,
-                                        stub->args.fsetattr.fd,
-                                        &stub->args.fsetattr.stbuf,
-                                        stub->args.fsetattr.valid);
-                break;
-        }
-        default:
-        {
-                gf_log_callingfn ("call-stub", GF_LOG_ERROR,
-                                  "Invalid value of FOP (%d)",
-                                  stub->fop);
-                break;
-        }
+        stub->args.offset = offset;
+        stub->args.size = len;
 
-        }
+        if (xdata)
+                stub->args.xdata = dict_ref (xdata);
 out:
-        return;
-}
+        return stub;
 
+}
 
 
 static void
-call_resume_unwind (call_stub_t *stub)
+call_resume_wind (call_stub_t *stub)
 {
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
         switch (stub->fop) {
         case GF_FOP_OPEN:
-        {
-                if (!stub->args.open_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.open_cbk.op_ret,
-                                      stub->args.open_cbk.op_errno,
-                                      stub->args.open_cbk.fd);
-                else
-                        stub->args.open_cbk.fn (stub->frame,
-                                                stub->frame->cookie,
-                                                stub->frame->this,
-                                                stub->args.open_cbk.op_ret,
-                                                stub->args.open_cbk.op_errno,
-                                                stub->args.open_cbk.fd);
+                stub->fn.open (stub->frame, stub->frame->this,
+			       &stub->args.loc, stub->args.flags,
+			       stub->args.fd, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_CREATE:
-        {
-                if (!stub->args.create_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.create_cbk.op_ret,
-                                      stub->args.create_cbk.op_errno,
-                                      stub->args.create_cbk.fd,
-                                      stub->args.create_cbk.inode,
-                                      &stub->args.create_cbk.buf,
-                                      &stub->args.create_cbk.preparent,
-                                      &stub->args.create_cbk.postparent);
-                else
-                        stub->args.create_cbk.fn (stub->frame,
-                                                  stub->frame->cookie,
-                                                  stub->frame->this,
-                                                  stub->args.create_cbk.op_ret,
-                                                  stub->args.create_cbk.op_errno,
-                                                  stub->args.create_cbk.fd,
-                                                  stub->args.create_cbk.inode,
-                                                  &stub->args.create_cbk.buf,
-                                                  &stub->args.create_cbk.preparent,
-                                                  &stub->args.create_cbk.postparent);
-
+                stub->fn.create (stub->frame, stub->frame->this,
+				 &stub->args.loc, stub->args.flags,
+				 stub->args.mode, stub->args.umask,
+				 stub->args.fd, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_STAT:
-        {
-                if (!stub->args.stat_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.stat_cbk.op_ret,
-                                      stub->args.stat_cbk.op_errno,
-                                      &stub->args.stat_cbk.buf);
-                else
-                        stub->args.stat_cbk.fn (stub->frame,
-                                                stub->frame->cookie,
-                                                stub->frame->this,
-                                                stub->args.stat_cbk.op_ret,
-                                                stub->args.stat_cbk.op_errno,
-                                                &stub->args.stat_cbk.buf);
-
+                stub->fn.stat (stub->frame, stub->frame->this,
+			       &stub->args.loc, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_READLINK:
-        {
-                if (!stub->args.readlink_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.readlink_cbk.op_ret,
-                                      stub->args.readlink_cbk.op_errno,
-                                      stub->args.readlink_cbk.buf,
-                                      &stub->args.readlink_cbk.sbuf);
-                else
-                        stub->args.readlink_cbk.fn (stub->frame,
-                                                    stub->frame->cookie,
-                                                    stub->frame->this,
-                                                    stub->args.readlink_cbk.op_ret,
-                                                    stub->args.readlink_cbk.op_errno,
-                                                    stub->args.readlink_cbk.buf,
-                                                    &stub->args.readlink_cbk.sbuf);
-
+                stub->fn.readlink (stub->frame, stub->frame->this,
+				   &stub->args.loc, stub->args.size,
+				   stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_MKNOD:
-        {
-                if (!stub->args.mknod_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.mknod_cbk.op_ret,
-                                      stub->args.mknod_cbk.op_errno,
-                                      stub->args.mknod_cbk.inode,
-                                      &stub->args.mknod_cbk.buf,
-                                      &stub->args.mknod_cbk.preparent,
-                                      &stub->args.mknod_cbk.postparent);
-                else
-                        stub->args.mknod_cbk.fn (stub->frame,
-                                                 stub->frame->cookie,
-                                                 stub->frame->this,
-                                                 stub->args.mknod_cbk.op_ret,
-                                                 stub->args.mknod_cbk.op_errno,
-                                                 stub->args.mknod_cbk.inode,
-                                                 &stub->args.mknod_cbk.buf,
-                                                 &stub->args.mknod_cbk.preparent,
-                                                 &stub->args.mknod_cbk.postparent);
-                break;
-        }
-
+                stub->fn.mknod (stub->frame, stub->frame->this,
+				&stub->args.loc, stub->args.mode,
+				stub->args.rdev, stub->args.umask,
+				stub->args.xdata);
+		break;
         case GF_FOP_MKDIR:
-        {
-                if (!stub->args.mkdir_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.mkdir_cbk.op_ret,
-                                      stub->args.mkdir_cbk.op_errno,
-                                      stub->args.mkdir_cbk.inode,
-                                      &stub->args.mkdir_cbk.buf,
-                                      &stub->args.mkdir_cbk.preparent,
-                                      &stub->args.mkdir_cbk.postparent);
-                else
-                        stub->args.mkdir_cbk.fn (stub->frame,
-                                                 stub->frame->cookie,
-                                                 stub->frame->this,
-                                                 stub->args.mkdir_cbk.op_ret,
-                                                 stub->args.mkdir_cbk.op_errno,
-                                                 stub->args.mkdir_cbk.inode,
-                                                 &stub->args.mkdir_cbk.buf,
-                                                 &stub->args.mkdir_cbk.preparent,
-                                                 &stub->args.mkdir_cbk.postparent);
-
-                if (stub->args.mkdir_cbk.inode)
-                        inode_unref (stub->args.mkdir_cbk.inode);
-
-                break;
-        }
-
+                stub->fn.mkdir (stub->frame, stub->frame->this,
+				&stub->args.loc, stub->args.mode,
+				stub->args.umask, stub->args.xdata);
+		break;
         case GF_FOP_UNLINK:
-        {
-                if (!stub->args.unlink_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.unlink_cbk.op_ret,
-                                      stub->args.unlink_cbk.op_errno,
-                                      &stub->args.unlink_cbk.preparent,
-                                      &stub->args.unlink_cbk.postparent);
-                else
-                        stub->args.unlink_cbk.fn (stub->frame,
-                                                  stub->frame->cookie,
-                                                  stub->frame->this,
-                                                  stub->args.unlink_cbk.op_ret,
-                                                  stub->args.unlink_cbk.op_errno,
-                                                  &stub->args.unlink_cbk.preparent,
-                                                  &stub->args.unlink_cbk.postparent);
-                break;
-        }
-
+                stub->fn.unlink (stub->frame, stub->frame->this,
+				 &stub->args.loc, stub->args.xflag,
+				 stub->args.xdata);
+		break;
         case GF_FOP_RMDIR:
-        {
-                if (!stub->args.rmdir_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.rmdir_cbk.op_ret,
-                                      stub->args.rmdir_cbk.op_errno,
-                                      &stub->args.rmdir_cbk.preparent,
-                                      &stub->args.rmdir_cbk.postparent);
-                else
-                        stub->args.rmdir_cbk.fn (stub->frame,
-                                                 stub->frame->cookie,
-                                                 stub->frame->this,
-                                                 stub->args.rmdir_cbk.op_ret,
-                                                 stub->args.rmdir_cbk.op_errno,
-                                                 &stub->args.rmdir_cbk.preparent,
-                                                 &stub->args.rmdir_cbk.postparent);
-                break;
-        }
-
+                stub->fn.rmdir (stub->frame, stub->frame->this,
+				&stub->args.loc, stub->args.flags,
+				stub->args.xdata);
+		break;
         case GF_FOP_SYMLINK:
-        {
-                if (!stub->args.symlink_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.symlink_cbk.op_ret,
-                                      stub->args.symlink_cbk.op_errno,
-                                      stub->args.symlink_cbk.inode,
-                                      &stub->args.symlink_cbk.buf,
-                                      &stub->args.symlink_cbk.preparent,
-                                      &stub->args.symlink_cbk.postparent);
-                else
-                        stub->args.symlink_cbk.fn (stub->frame,
-                                                   stub->frame->cookie,
-                                                   stub->frame->this,
-                                                   stub->args.symlink_cbk.op_ret,
-                                                   stub->args.symlink_cbk.op_errno,
-                                                   stub->args.symlink_cbk.inode,
-                                                   &stub->args.symlink_cbk.buf,
-                                                   &stub->args.symlink_cbk.preparent,
-                                                   &stub->args.symlink_cbk.postparent);
-        }
-        break;
-
+                stub->fn.symlink (stub->frame, stub->frame->this,
+				  stub->args.linkname, &stub->args.loc,
+				  stub->args.umask, stub->args.xdata);
+		break;
         case GF_FOP_RENAME:
-        {
-                if (!stub->args.rename_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.rename_cbk.op_ret,
-                                      stub->args.rename_cbk.op_errno,
-                                      &stub->args.rename_cbk.buf,
-                                      &stub->args.rename_cbk.preoldparent,
-                                      &stub->args.rename_cbk.postoldparent,
-                                      &stub->args.rename_cbk.prenewparent,
-                                      &stub->args.rename_cbk.postnewparent);
-                else
-                        stub->args.rename_cbk.fn (stub->frame,
-                                                  stub->frame->cookie,
-                                                  stub->frame->this,
-                                                  stub->args.rename_cbk.op_ret,
-                                                  stub->args.rename_cbk.op_errno,
-                                                  &stub->args.rename_cbk.buf,
-                                                  &stub->args.rename_cbk.preoldparent,
-                                                  &stub->args.rename_cbk.postoldparent,
-                                                  &stub->args.rename_cbk.prenewparent,
-                                                  &stub->args.rename_cbk.postnewparent);
-                break;
-        }
-
+                stub->fn.rename (stub->frame, stub->frame->this,
+				 &stub->args.loc, &stub->args.loc2,
+				 stub->args.xdata);
+		break;
         case GF_FOP_LINK:
-        {
-                if (!stub->args.link_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.link_cbk.op_ret,
-                                      stub->args.link_cbk.op_errno,
-                                      stub->args.link_cbk.inode,
-                                      &stub->args.link_cbk.buf);
-                else
-                        stub->args.link_cbk.fn (stub->frame,
-                                                stub->frame->cookie,
-                                                stub->frame->this,
-                                                stub->args.link_cbk.op_ret,
-                                                stub->args.link_cbk.op_errno,
-                                                stub->args.link_cbk.inode,
-                                                &stub->args.link_cbk.buf,
-                                                &stub->args.link_cbk.preparent,
-                                                &stub->args.link_cbk.postparent);
-                break;
-        }
-
+                stub->fn.link (stub->frame, stub->frame->this,
+			       &stub->args.loc, &stub->args.loc2,
+			       stub->args.xdata);
+		break;
         case GF_FOP_TRUNCATE:
-        {
-                if (!stub->args.truncate_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.truncate_cbk.op_ret,
-                                      stub->args.truncate_cbk.op_errno,
-                                      &stub->args.truncate_cbk.prebuf,
-                                      &stub->args.truncate_cbk.postbuf);
-                else
-                        stub->args.truncate_cbk.fn (stub->frame,
-                                                    stub->frame->cookie,
-                                                    stub->frame->this,
-                                                    stub->args.truncate_cbk.op_ret,
-                                                    stub->args.truncate_cbk.op_errno,
-                                                    &stub->args.truncate_cbk.prebuf,
-                                                    &stub->args.truncate_cbk.postbuf);
+                stub->fn.truncate (stub->frame, stub->frame->this,
+				   &stub->args.loc, stub->args.offset,
+				   stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_READ:
-        {
-                if (!stub->args.readv_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.readv_cbk.op_ret,
-                                      stub->args.readv_cbk.op_errno,
-                                      stub->args.readv_cbk.vector,
-                                      stub->args.readv_cbk.count,
-                                      &stub->args.readv_cbk.stbuf,
-                                      stub->args.readv_cbk.iobref);
-                else
-                        stub->args.readv_cbk.fn (stub->frame,
-                                                 stub->frame->cookie,
-                                                 stub->frame->this,
-                                                 stub->args.readv_cbk.op_ret,
-                                                 stub->args.readv_cbk.op_errno,
-                                                 stub->args.readv_cbk.vector,
-                                                 stub->args.readv_cbk.count,
-                                                 &stub->args.readv_cbk.stbuf,
-                                                 stub->args.readv_cbk.iobref);
-        }
-        break;
-
+                stub->fn.readv (stub->frame, stub->frame->this,
+				stub->args.fd, stub->args.size,
+				stub->args.offset, stub->args.flags,
+				stub->args.xdata);
+                break;
         case GF_FOP_WRITE:
-        {
-                if (!stub->args.writev_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.writev_cbk.op_ret,
-                                      stub->args.writev_cbk.op_errno,
-                                      &stub->args.writev_cbk.prebuf,
-                                      &stub->args.writev_cbk.postbuf);
-                else
-                        stub->args.writev_cbk.fn (stub->frame,
-                                                  stub->frame->cookie,
-                                                  stub->frame->this,
-                                                  stub->args.writev_cbk.op_ret,
-                                                  stub->args.writev_cbk.op_errno,
-                                                  &stub->args.writev_cbk.prebuf,
-                                                  &stub->args.writev_cbk.postbuf);
+                stub->fn.writev (stub->frame, stub->frame->this,
+				 stub->args.fd, stub->args.vector,
+				 stub->args.count, stub->args.offset,
+				 stub->args.flags, stub->args.iobref,
+				 stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_STATFS:
-        {
-                if (!stub->args.statfs_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.statfs_cbk.op_ret,
-                                      stub->args.statfs_cbk.op_errno,
-                                      &(stub->args.statfs_cbk.buf));
-                else
-                        stub->args.statfs_cbk.fn (stub->frame,
-                                                  stub->frame->cookie,
-                                                  stub->frame->this,
-                                                  stub->args.statfs_cbk.op_ret,
-                                                  stub->args.statfs_cbk.op_errno,
-                                                  &(stub->args.statfs_cbk.buf));
-        }
-        break;
-
+                stub->fn.statfs (stub->frame, stub->frame->this,
+				 &stub->args.loc, stub->args.xdata);
+                break;
         case GF_FOP_FLUSH:
-        {
-                if (!stub->args.flush_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.flush_cbk.op_ret,
-                                      stub->args.flush_cbk.op_errno);
-                else
-                        stub->args.flush_cbk.fn (stub->frame,
-                                                 stub->frame->cookie,
-                                                 stub->frame->this,
-                                                 stub->args.flush_cbk.op_ret,
-                                                 stub->args.flush_cbk.op_errno);
-
+                stub->fn.flush (stub->frame, stub->frame->this,
+				stub->args.fd, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_FSYNC:
-        {
-                if (!stub->args.fsync_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.fsync_cbk.op_ret,
-                                      stub->args.fsync_cbk.op_errno,
-                                      &stub->args.fsync_cbk.prebuf,
-                                      &stub->args.fsync_cbk.postbuf);
-                else
-                        stub->args.fsync_cbk.fn (stub->frame,
-                                                 stub->frame->cookie,
-                                                 stub->frame->this,
-                                                 stub->args.fsync_cbk.op_ret,
-                                                 stub->args.fsync_cbk.op_errno,
-                                                 &stub->args.fsync_cbk.prebuf,
-                                                 &stub->args.fsync_cbk.postbuf);
+                stub->fn.fsync (stub->frame, stub->frame->this,
+				stub->args.fd, stub->args.datasync,
+				stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_SETXATTR:
-        {
-                if (!stub->args.setxattr_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.setxattr_cbk.op_ret,
-                                      stub->args.setxattr_cbk.op_errno);
-
-                else
-                        stub->args.setxattr_cbk.fn (stub->frame,
-                                                    stub->frame->cookie,
-                                                    stub->frame->this,
-                                                    stub->args.setxattr_cbk.op_ret,
-                                                    stub->args.setxattr_cbk.op_errno);
-
+                stub->fn.setxattr (stub->frame, stub->frame->this,
+				   &stub->args.loc, stub->args.xattr,
+				   stub->args.flags, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_GETXATTR:
-        {
-                if (!stub->args.getxattr_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.getxattr_cbk.op_ret,
-                                      stub->args.getxattr_cbk.op_errno,
-                                      stub->args.getxattr_cbk.dict);
-                else
-                        stub->args.getxattr_cbk.fn (stub->frame,
-                                                    stub->frame->cookie,
-                                                    stub->frame->this,
-                                                    stub->args.getxattr_cbk.op_ret,
-                                                    stub->args.getxattr_cbk.op_errno,
-                                                    stub->args.getxattr_cbk.dict);
+                stub->fn.getxattr (stub->frame, stub->frame->this,
+				   &stub->args.loc, stub->args.name,
+				   stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_FSETXATTR:
-        {
-                if (!stub->args.fsetxattr_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.fsetxattr_cbk.op_ret,
-                                      stub->args.fsetxattr_cbk.op_errno);
-
-                else
-                        stub->args.fsetxattr_cbk.fn (stub->frame,
-                                                     stub->frame->cookie,
-                                                     stub->frame->this,
-                                                     stub->args.fsetxattr_cbk.op_ret,
-                                                     stub->args.fsetxattr_cbk.op_errno);
-
+                stub->fn.fsetxattr (stub->frame, stub->frame->this,
+				    stub->args.fd, stub->args.xattr,
+				    stub->args.flags, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_FGETXATTR:
-        {
-                if (!stub->args.fgetxattr_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.fgetxattr_cbk.op_ret,
-                                      stub->args.fgetxattr_cbk.op_errno,
-                                      stub->args.fgetxattr_cbk.dict);
-                else
-                        stub->args.fgetxattr_cbk.fn (stub->frame,
-                                                     stub->frame->cookie,
-                                                     stub->frame->this,
-                                                     stub->args.fgetxattr_cbk.op_ret,
-                                                     stub->args.fgetxattr_cbk.op_errno,
-                                                     stub->args.fgetxattr_cbk.dict);
+                stub->fn.fgetxattr (stub->frame, stub->frame->this,
+				    stub->args.fd, stub->args.name,
+				    stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_REMOVEXATTR:
-        {
-                if (!stub->args.removexattr_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.removexattr_cbk.op_ret,
-                                      stub->args.removexattr_cbk.op_errno);
-                else
-                        stub->args.removexattr_cbk.fn (stub->frame,
-                                                       stub->frame->cookie,
-                                                       stub->frame->this,
-                                                       stub->args.removexattr_cbk.op_ret,
-                                                       stub->args.removexattr_cbk.op_errno);
-
+                stub->fn.removexattr (stub->frame, stub->frame->this,
+				      &stub->args.loc, stub->args.name,
+				      stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_FREMOVEXATTR:
-        {
-                if (!stub->args.fremovexattr_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.fremovexattr_cbk.op_ret,
-                                      stub->args.fremovexattr_cbk.op_errno);
-                else
-                        stub->args.fremovexattr_cbk.fn (stub->frame,
-                                                        stub->frame->cookie,
-                                                        stub->frame->this,
-                                                        stub->args.fremovexattr_cbk.op_ret,
-                                                        stub->args.fremovexattr_cbk.op_errno);
-
+                stub->fn.fremovexattr (stub->frame, stub->frame->this,
+				       stub->args.fd, stub->args.name,
+				       stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_OPENDIR:
-        {
-                if (!stub->args.opendir_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.opendir_cbk.op_ret,
-                                      stub->args.opendir_cbk.op_errno,
-                                      stub->args.opendir_cbk.fd);
-                else
-                        stub->args.opendir_cbk.fn (stub->frame,
-                                                   stub->frame->cookie,
-                                                   stub->frame->this,
-                                                   stub->args.opendir_cbk.op_ret,
-                                                   stub->args.opendir_cbk.op_errno,
-                                                   stub->args.opendir_cbk.fd);
+                stub->fn.opendir (stub->frame, stub->frame->this,
+				  &stub->args.loc, stub->args.fd,
+				  stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_FSYNCDIR:
-        {
-                if (!stub->args.fsyncdir_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.fsyncdir_cbk.op_ret,
-                                      stub->args.fsyncdir_cbk.op_errno);
-                else
-                        stub->args.fsyncdir_cbk.fn (stub->frame,
-                                                    stub->frame->cookie,
-                                                    stub->frame->this,
-                                                    stub->args.fsyncdir_cbk.op_ret,
-                                                    stub->args.fsyncdir_cbk.op_errno);
+                stub->fn.fsyncdir (stub->frame, stub->frame->this,
+				   stub->args.fd, stub->args.datasync,
+				   stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_ACCESS:
-        {
-                if (!stub->args.access_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.access_cbk.op_ret,
-                                      stub->args.access_cbk.op_errno);
-                else
-                        stub->args.access_cbk.fn (stub->frame,
-                                                  stub->frame->cookie,
-                                                  stub->frame->this,
-                                                  stub->args.access_cbk.op_ret,
-                                                  stub->args.access_cbk.op_errno);
-
+                stub->fn.access (stub->frame, stub->frame->this,
+				 &stub->args.loc, stub->args.mask,
+				 stub->args.xdata);
                 break;
-        }
 
         case GF_FOP_FTRUNCATE:
-        {
-                if (!stub->args.ftruncate_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.ftruncate_cbk.op_ret,
-                                      stub->args.ftruncate_cbk.op_errno,
-                                      &stub->args.ftruncate_cbk.prebuf,
-                                      &stub->args.ftruncate_cbk.postbuf);
-                else
-                        stub->args.ftruncate_cbk.fn (stub->frame,
-                                                     stub->frame->cookie,
-                                                     stub->frame->this,
-                                                     stub->args.ftruncate_cbk.op_ret,
-                                                     stub->args.ftruncate_cbk.op_errno,
-                                                     &stub->args.ftruncate_cbk.prebuf,
-                                                     &stub->args.ftruncate_cbk.postbuf);
+                stub->fn.ftruncate (stub->frame, stub->frame->this,
+				    stub->args.fd, stub->args.offset,
+				    stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_FSTAT:
-        {
-                if (!stub->args.fstat_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.fstat_cbk.op_ret,
-                                      stub->args.fstat_cbk.op_errno,
-                                      &stub->args.fstat_cbk.buf);
-                else
-                        stub->args.fstat_cbk.fn (stub->frame,
-                                                 stub->frame->cookie,
-                                                 stub->frame->this,
-                                                 stub->args.fstat_cbk.op_ret,
-                                                 stub->args.fstat_cbk.op_errno,
-                                                 &stub->args.fstat_cbk.buf);
-
+                stub->fn.fstat (stub->frame, stub->frame->this,
+				stub->args.fd, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_LK:
-        {
-                if (!stub->args.lk_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.lk_cbk.op_ret,
-                                      stub->args.lk_cbk.op_errno,
-                                      &stub->args.lk_cbk.lock);
-                else
-                        stub->args.lk_cbk.fn (stub->frame,
-                                              stub->frame->cookie,
-                                              stub->frame->this,
-                                              stub->args.lk_cbk.op_ret,
-                                              stub->args.lk_cbk.op_errno,
-                                              &stub->args.lk_cbk.lock);
+                stub->fn.lk (stub->frame, stub->frame->this,
+			     stub->args.fd, stub->args.cmd,
+			     &stub->args.lock, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_INODELK:
-        {
-                if (!stub->args.inodelk_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.inodelk_cbk.op_ret,
-                                      stub->args.inodelk_cbk.op_errno);
-
-                else
-                        stub->args.inodelk_cbk.fn (stub->frame,
-                                                   stub->frame->cookie,
-                                                   stub->frame->this,
-                                                   stub->args.inodelk_cbk.op_ret,
-                                                   stub->args.inodelk_cbk.op_errno);
+                stub->fn.inodelk (stub->frame, stub->frame->this,
+				  stub->args.volume, &stub->args.loc,
+				  stub->args.cmd, &stub->args.lock,
+				  stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_FINODELK:
-        {
-                if (!stub->args.finodelk_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.finodelk_cbk.op_ret,
-                                      stub->args.finodelk_cbk.op_errno);
-
-                else
-                        stub->args.finodelk_cbk.fn (stub->frame,
-                                                    stub->frame->cookie,
-                                                    stub->frame->this,
-                                                    stub->args.finodelk_cbk.op_ret,
-                                                    stub->args.finodelk_cbk.op_errno);
+                stub->fn.finodelk (stub->frame, stub->frame->this,
+				   stub->args.volume, stub->args.fd,
+				   stub->args.cmd, &stub->args.lock,
+				   stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_ENTRYLK:
-        {
-                if (!stub->args.entrylk_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.entrylk_cbk.op_ret,
-                                      stub->args.entrylk_cbk.op_errno);
-
-                else
-                        stub->args.entrylk_cbk.fn (stub->frame,
-                                                   stub->frame->cookie,
-                                                   stub->frame->this,
-                                                   stub->args.entrylk_cbk.op_ret,
-                                                   stub->args.entrylk_cbk.op_errno);
+                stub->fn.entrylk (stub->frame, stub->frame->this,
+				  stub->args.volume, &stub->args.loc,
+				  stub->args.name, stub->args.entrylkcmd,
+				  stub->args.entrylktype, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_FENTRYLK:
-        {
-                if (!stub->args.fentrylk_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.fentrylk_cbk.op_ret,
-                                      stub->args.fentrylk_cbk.op_errno);
-
-                else
-                        stub->args.fentrylk_cbk.fn (stub->frame,
-                                                    stub->frame->cookie,
-                                                    stub->frame->this,
-                                                    stub->args.fentrylk_cbk.op_ret,
-                                                    stub->args.fentrylk_cbk.op_errno);
-                break;
-        }
-
+                stub->fn.fentrylk (stub->frame, stub->frame->this,
+				   stub->args.volume, stub->args.fd,
+				   stub->args.name, stub->args.entrylkcmd,
+				   stub->args.entrylktype, stub->args.xdata);
+		break;
         case GF_FOP_LOOKUP:
-        {
-                if (!stub->args.lookup_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.lookup_cbk.op_ret,
-                                      stub->args.lookup_cbk.op_errno,
-                                      stub->args.lookup_cbk.inode,
-                                      &stub->args.lookup_cbk.buf,
-                                      stub->args.lookup_cbk.dict,
-                                      &stub->args.lookup_cbk.postparent);
-                else
-                        stub->args.lookup_cbk.fn (stub->frame,
-                                                  stub->frame->cookie,
-                                                  stub->frame->this,
-                                                  stub->args.lookup_cbk.op_ret,
-                                                  stub->args.lookup_cbk.op_errno,
-                                                  stub->args.lookup_cbk.inode,
-                                                  &stub->args.lookup_cbk.buf,
-                                                  stub->args.lookup_cbk.dict,
-                                                  &stub->args.lookup_cbk.postparent);
-                /* FIXME NULL should not be passed */
-
-                if (stub->args.lookup_cbk.dict)
-                        dict_unref (stub->args.lookup_cbk.dict);
-                if (stub->args.lookup_cbk.inode)
-                        inode_unref (stub->args.lookup_cbk.inode);
-
+                stub->fn.lookup (stub->frame, stub->frame->this,
+				 &stub->args.loc, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_RCHECKSUM:
-        {
-                if (!stub->args.rchecksum_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.rchecksum_cbk.op_ret,
-                                      stub->args.rchecksum_cbk.op_errno,
-                                      stub->args.rchecksum_cbk.weak_checksum,
-                                      stub->args.rchecksum_cbk.strong_checksum);
-                else
-                        stub->args.rchecksum_cbk.fn (stub->frame,
-                                                     stub->frame->cookie,
-                                                     stub->frame->this,
-                                                     stub->args.rchecksum_cbk.op_ret,
-                                                     stub->args.rchecksum_cbk.op_errno,
-                                                     stub->args.rchecksum_cbk.weak_checksum,
-                                                     stub->args.rchecksum_cbk.strong_checksum);
-                if (stub->args.rchecksum_cbk.op_ret >= 0)
-                {
-                        GF_FREE (stub->args.rchecksum_cbk.strong_checksum);
-                }
-
+                stub->fn.rchecksum (stub->frame, stub->frame->this,
+				    stub->args.fd, stub->args.offset,
+				    stub->args.size, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_READDIR:
-        {
-                if (!stub->args.readdir_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.readdir_cbk.op_ret,
-                                      stub->args.readdir_cbk.op_errno,
-                                      &stub->args.readdir_cbk.entries);
-                else
-                        stub->args.readdir_cbk.fn (stub->frame,
-                                                   stub->frame->cookie,
-                                                   stub->frame->this,
-                                                   stub->args.readdir_cbk.op_ret,
-                                                   stub->args.readdir_cbk.op_errno,
-                                                   &stub->args.readdir_cbk.entries);
-
-                if (stub->args.readdir_cbk.op_ret > 0)
-                        gf_dirent_free (&stub->args.readdir_cbk.entries);
-
+                stub->fn.readdir (stub->frame, stub->frame->this,
+				  stub->args.fd, stub->args.size,
+				  stub->args.offset, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_READDIRP:
-        {
-                if (!stub->args.readdirp_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.readdirp_cbk.op_ret,
-                                      stub->args.readdirp_cbk.op_errno,
-                                      &stub->args.readdirp_cbk.entries);
-                else
-                        stub->args.readdirp_cbk.fn (stub->frame,
-                                                    stub->frame->cookie,
-                                                    stub->frame->this,
-                                                    stub->args.readdirp_cbk.op_ret,
-                                                    stub->args.readdirp_cbk.op_errno,
-                                                    &stub->args.readdirp_cbk.entries);
-
-                if (stub->args.readdirp_cbk.op_ret > 0)
-                        gf_dirent_free (&stub->args.readdirp_cbk.entries);
-
+                stub->fn.readdirp (stub->frame, stub->frame->this,
+				   stub->args.fd, stub->args.size,
+				   stub->args.offset, stub->args.xdata);
                 break;
-        }
-
         case GF_FOP_XATTROP:
-        {
-                if (!stub->args.xattrop_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.xattrop_cbk.op_ret,
-                                      stub->args.xattrop_cbk.op_errno);
-                else
-                        stub->args.xattrop_cbk.fn (stub->frame,
-                                                   stub->frame->cookie,
-                                                   stub->frame->this,
-                                                   stub->args.xattrop_cbk.op_ret,
-                                                   stub->args.xattrop_cbk.op_errno,
-                                                   stub->args.xattrop_cbk.xattr);
-
-                if (stub->args.xattrop_cbk.xattr)
-                        dict_unref (stub->args.xattrop_cbk.xattr);
-
+                stub->fn.xattrop (stub->frame, stub->frame->this,
+				  &stub->args.loc, stub->args.optype,
+				  stub->args.xattr, stub->args.xdata);
                 break;
-        }
         case GF_FOP_FXATTROP:
-        {
-                if (!stub->args.fxattrop_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.fxattrop_cbk.op_ret,
-                                      stub->args.fxattrop_cbk.op_errno);
-                else
-                        stub->args.fxattrop_cbk.fn (stub->frame,
-                                                    stub->frame->cookie,
-                                                    stub->frame->this,
-                                                    stub->args.fxattrop_cbk.op_ret,
-                                                    stub->args.fxattrop_cbk.op_errno,
-                                                    stub->args.fxattrop_cbk.xattr);
-
-                if (stub->args.fxattrop_cbk.xattr)
-                        dict_unref (stub->args.fxattrop_cbk.xattr);
-
+                stub->fn.fxattrop (stub->frame, stub->frame->this,
+				   stub->args.fd, stub->args.optype,
+				   stub->args.xattr, stub->args.xdata);
                 break;
-        }
         case GF_FOP_SETATTR:
-        {
-                if (!stub->args.setattr_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.setattr_cbk.op_ret,
-                                      stub->args.setattr_cbk.op_errno,
-                                      &stub->args.setattr_cbk.statpre,
-                                      &stub->args.setattr_cbk.statpost);
-                else
-                        stub->args.setattr_cbk.fn (
-                                stub->frame,
-                                stub->frame->cookie,
-                                stub->frame->this,
-                                stub->args.setattr_cbk.op_ret,
-                                stub->args.setattr_cbk.op_errno,
-                                &stub->args.setattr_cbk.statpre,
-                                &stub->args.setattr_cbk.statpost);
+                stub->fn.setattr (stub->frame, stub->frame->this,
+				  &stub->args.loc, &stub->args.stat,
+				  stub->args.valid, stub->args.xdata);
                 break;
-        }
         case GF_FOP_FSETATTR:
-        {
-                if (!stub->args.fsetattr_cbk.fn)
-                        STACK_UNWIND (stub->frame,
-                                      stub->args.fsetattr_cbk.op_ret,
-                                      stub->args.fsetattr_cbk.op_errno,
-                                      &stub->args.fsetattr_cbk.statpre,
-                                      &stub->args.fsetattr_cbk.statpost);
-                else
-                        stub->args.fsetattr_cbk.fn (
-                                stub->frame,
-                                stub->frame->cookie,
-                                stub->frame->this,
-                                stub->args.fsetattr_cbk.op_ret,
-                                stub->args.fsetattr_cbk.op_errno,
-                                &stub->args.fsetattr_cbk.statpre,
-                                &stub->args.fsetattr_cbk.statpost);
+                stub->fn.fsetattr (stub->frame, stub->frame->this,
+				   stub->args.fd, &stub->args.stat,
+				   stub->args.valid, stub->args.xdata);
+                break;
+	case GF_FOP_FALLOCATE:
+		stub->fn.fallocate(stub->frame, stub->frame->this,
+				   stub->args.fd, stub->args.flags,
+				   stub->args.offset, stub->args.size,
+				   stub->args.xdata);
+		break;
+	case GF_FOP_DISCARD:
+		stub->fn.discard(stub->frame, stub->frame->this,
+				 stub->args.fd, stub->args.offset,
+				 stub->args.size, stub->args.xdata);
+		break;
+        case GF_FOP_ZEROFILL:
+                stub->fn.zerofill(stub->frame, stub->frame->this,
+                                 stub->args.fd, stub->args.offset,
+                                 stub->args.size, stub->args.xdata);
                 break;
-        }
+
         default:
-        {
                 gf_log_callingfn ("call-stub", GF_LOG_ERROR,
                                   "Invalid value of FOP (%d)",
                                   stub->fop);
                 break;
         }
-        }
 out:
         return;
 }
 
 
+#define STUB_UNWIND(stb, fop, args ...) do {				\
+	if (stb->fn_cbk.fop)						\
+		stb->fn_cbk.fop (stb->frame, stb->frame->cookie,	\
+				 stb->frame->this, stb->args_cbk.op_ret, \
+				 stb->args_cbk.op_errno, args);		\
+	else								\
+		STACK_UNWIND_STRICT (fop, stb->frame, stb->args_cbk.op_ret, \
+				     stb->args_cbk.op_errno, args);	\
+	} while (0)
+
+
 static void
-call_stub_destroy_wind (call_stub_t *stub)
+call_resume_unwind (call_stub_t *stub)
 {
+        GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
+
         switch (stub->fop) {
         case GF_FOP_OPEN:
-        {
-                loc_wipe (&stub->args.open.loc);
-                if (stub->args.open.fd)
-                        fd_unref (stub->args.open.fd);
+		STUB_UNWIND (stub, open, stub->args_cbk.fd,
+			     stub->args_cbk.xdata);
                 break;
-        }
         case GF_FOP_CREATE:
-        {
-                loc_wipe (&stub->args.create.loc);
-                if (stub->args.create.fd)
-                        fd_unref (stub->args.create.fd);
-                if (stub->args.create.params)
-                        dict_unref (stub->args.create.params);
+		STUB_UNWIND (stub, create, stub->args_cbk.fd,
+			     stub->args_cbk.inode, &stub->args_cbk.stat,
+			     &stub->args_cbk.preparent,
+			     &stub->args_cbk.postparent,
+			     stub->args_cbk.xdata);
                 break;
-        }
         case GF_FOP_STAT:
-        {
-                loc_wipe (&stub->args.stat.loc);
+		STUB_UNWIND (stub, stat, &stub->args_cbk.stat,
+			     stub->args_cbk.xdata);
                 break;
-        }
         case GF_FOP_READLINK:
-        {
-                loc_wipe (&stub->args.readlink.loc);
-                break;
-        }
-
+		STUB_UNWIND (stub, readlink, stub->args_cbk.buf,
+			     &stub->args_cbk.stat, stub->args.xdata);
+		break;
         case GF_FOP_MKNOD:
-        {
-                loc_wipe (&stub->args.mknod.loc);
-                if (stub->args.mknod.params)
-                        dict_unref (stub->args.mknod.params);
-        }
-        break;
-
+		STUB_UNWIND (stub, mknod, stub->args_cbk.inode,
+			     &stub->args_cbk.stat, &stub->args_cbk.preparent,
+			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
+		break;
         case GF_FOP_MKDIR:
-        {
-                loc_wipe (&stub->args.mkdir.loc);
-                if (stub->args.mkdir.params)
-                        dict_unref (stub->args.mkdir.params);
-        }
-        break;
-
+		STUB_UNWIND (stub, mkdir, stub->args_cbk.inode,
+			     &stub->args_cbk.stat, &stub->args_cbk.preparent,
+			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
+		break;
         case GF_FOP_UNLINK:
-        {
-                loc_wipe (&stub->args.unlink.loc);
-        }
-        break;
-
+		STUB_UNWIND (stub, unlink, &stub->args_cbk.preparent,
+			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
+		break;
         case GF_FOP_RMDIR:
-        {
-                loc_wipe (&stub->args.rmdir.loc);
-        }
-        break;
-
+		STUB_UNWIND (stub, rmdir, &stub->args_cbk.preparent,
+			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
+		break;
         case GF_FOP_SYMLINK:
-        {
-                GF_FREE ((char *)stub->args.symlink.linkname);
-                loc_wipe (&stub->args.symlink.loc);
-                if (stub->args.symlink.params)
-                        dict_unref (stub->args.symlink.params);
-        }
-        break;
-
+		STUB_UNWIND (stub, symlink, stub->args_cbk.inode,
+			     &stub->args_cbk.stat, &stub->args_cbk.preparent,
+			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
+		break;
         case GF_FOP_RENAME:
-        {
-                loc_wipe (&stub->args.rename.old);
-                loc_wipe (&stub->args.rename.new);
-        }
-        break;
-
+		STUB_UNWIND (stub, rename, &stub->args_cbk.stat,
+			     &stub->args_cbk.preparent,
+			     &stub->args_cbk.postparent,
+			     &stub->args_cbk.preparent2,
+			     &stub->args_cbk.postparent2,
+			     stub->args_cbk.xdata);
+		break;
         case GF_FOP_LINK:
-        {
-                loc_wipe (&stub->args.link.oldloc);
-                loc_wipe (&stub->args.link.newloc);
-        }
-        break;
-
+		STUB_UNWIND (stub, link, stub->args_cbk.inode,
+			     &stub->args_cbk.stat, &stub->args_cbk.preparent,
+			     &stub->args_cbk.postparent, stub->args_cbk.xdata);
+		break;
         case GF_FOP_TRUNCATE:
-        {
-                loc_wipe (&stub->args.truncate.loc);
-                break;
-        }
-
+		STUB_UNWIND (stub, truncate, &stub->args_cbk.prestat,
+			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
+		break;
         case GF_FOP_READ:
-        {
-                if (stub->args.readv.fd)
-                        fd_unref (stub->args.readv.fd);
-                break;
-        }
-
+		STUB_UNWIND (stub, readv, stub->args_cbk.vector,
+			     stub->args_cbk.count, &stub->args_cbk.stat,
+			     stub->args_cbk.iobref, stub->args_cbk.xdata);
+		break;
         case GF_FOP_WRITE:
-        {
-                struct iobref *iobref = stub->args.writev.iobref;
-                if (stub->args.writev.fd)
-                        fd_unref (stub->args.writev.fd);
-                GF_FREE (stub->args.writev.vector);
-                if (iobref)
-                        iobref_unref (iobref);
-                break;
-        }
-
+		STUB_UNWIND (stub, writev, &stub->args_cbk.prestat,
+			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
+		break;
         case GF_FOP_STATFS:
-        {
-                loc_wipe (&stub->args.statfs.loc);
-                break;
-        }
+		STUB_UNWIND (stub, statfs, &stub->args_cbk.statvfs,
+			     stub->args_cbk.xdata);
+		break;
         case GF_FOP_FLUSH:
-        {
-                if (stub->args.flush.fd)
-                        fd_unref (stub->args.flush.fd);
-                break;
-        }
-
+		STUB_UNWIND (stub, flush, stub->args_cbk.xdata);
+		break;
         case GF_FOP_FSYNC:
-        {
-                if (stub->args.fsync.fd)
-                        fd_unref (stub->args.fsync.fd);
-                break;
-        }
-
+		STUB_UNWIND (stub, fsync, &stub->args_cbk.prestat,
+			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
+		break;
         case GF_FOP_SETXATTR:
-        {
-                loc_wipe (&stub->args.setxattr.loc);
-                if (stub->args.setxattr.dict)
-                        dict_unref (stub->args.setxattr.dict);
-                break;
-        }
-
+		STUB_UNWIND (stub, setxattr, stub->args_cbk.xdata);
+		break;
         case GF_FOP_GETXATTR:
-        {
-                if (stub->args.getxattr.name)
-                        GF_FREE ((char *)stub->args.getxattr.name);
-                loc_wipe (&stub->args.getxattr.loc);
-                break;
-        }
-
+		STUB_UNWIND (stub, getxattr, stub->args_cbk.xattr,
+			     stub->args_cbk.xdata);
+		break;
         case GF_FOP_FSETXATTR:
-        {
-                fd_unref (stub->args.fsetxattr.fd);
-                if (stub->args.fsetxattr.dict)
-                        dict_unref (stub->args.fsetxattr.dict);
-                break;
-        }
-
+		STUB_UNWIND (stub, fsetxattr, stub->args_cbk.xdata);
+		break;
         case GF_FOP_FGETXATTR:
-        {
-                if (stub->args.fgetxattr.name)
-                        GF_FREE ((char *)stub->args.fgetxattr.name);
-                fd_unref (stub->args.fgetxattr.fd);
-                break;
-        }
-
+		STUB_UNWIND (stub, fgetxattr, stub->args_cbk.xattr,
+			     stub->args_cbk.xdata);
+		break;
         case GF_FOP_REMOVEXATTR:
-        {
-                loc_wipe (&stub->args.removexattr.loc);
-                GF_FREE ((char *)stub->args.removexattr.name);
-                break;
-        }
-
+		STUB_UNWIND (stub, removexattr, stub->args_cbk.xdata);
+		break;
         case GF_FOP_FREMOVEXATTR:
-        {
-                fd_unref (stub->args.fremovexattr.fd);
-                GF_FREE ((char *)stub->args.fremovexattr.name);
-                break;
-        }
-
+		STUB_UNWIND (stub, fremovexattr, stub->args_cbk.xdata);
+		break;
         case GF_FOP_OPENDIR:
-        {
-                loc_wipe (&stub->args.opendir.loc);
-                if (stub->args.opendir.fd)
-                        fd_unref (stub->args.opendir.fd);
-                break;
-        }
-
+		STUB_UNWIND (stub, opendir, stub->args_cbk.fd,
+			     stub->args_cbk.xdata);
+		break;
         case GF_FOP_FSYNCDIR:
-        {
-                if (stub->args.fsyncdir.fd)
-                        fd_unref (stub->args.fsyncdir.fd);
+		STUB_UNWIND (stub, fsyncdir, stub->args_cbk.xdata);
                 break;
-        }
-
         case GF_FOP_ACCESS:
-        {
-                loc_wipe (&stub->args.access.loc);
+		STUB_UNWIND (stub, access, stub->args_cbk.xdata);
                 break;
-        }
-
         case GF_FOP_FTRUNCATE:
-        {
-                if (stub->args.ftruncate.fd)
-                        fd_unref (stub->args.ftruncate.fd);
+		STUB_UNWIND (stub, ftruncate, &stub->args_cbk.prestat,
+			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
                 break;
-        }
-
         case GF_FOP_FSTAT:
-        {
-                if (stub->args.fstat.fd)
-                        fd_unref (stub->args.fstat.fd);
+		STUB_UNWIND (stub, fstat, &stub->args_cbk.stat,
+			     stub->args_cbk.xdata);
                 break;
-        }
-
         case GF_FOP_LK:
-        {
-                if (stub->args.lk.fd)
-                        fd_unref (stub->args.lk.fd);
-                break;
-        }
-
+		STUB_UNWIND (stub, lk, &stub->args_cbk.lock,
+			     stub->args_cbk.xdata);
+		break;
         case GF_FOP_INODELK:
-        {
-                if (stub->args.inodelk.volume)
-                        GF_FREE ((char *)stub->args.inodelk.volume);
-
-                loc_wipe (&stub->args.inodelk.loc);
+		STUB_UNWIND (stub, inodelk, stub->args_cbk.xdata);
                 break;
-        }
         case GF_FOP_FINODELK:
-        {
-                if (stub->args.finodelk.volume)
-                        GF_FREE ((char *)stub->args.finodelk.volume);
-
-                if (stub->args.finodelk.fd)
-                        fd_unref (stub->args.finodelk.fd);
+		STUB_UNWIND (stub, finodelk, stub->args_cbk.xdata);
                 break;
-        }
         case GF_FOP_ENTRYLK:
-        {
-                if (stub->args.entrylk.volume)
-                        GF_FREE ((char *)stub->args.entrylk.volume);
-
-                if (stub->args.entrylk.name)
-                        GF_FREE ((char *)stub->args.entrylk.name);
-                loc_wipe (&stub->args.entrylk.loc);
-                break;
-        }
+		STUB_UNWIND (stub, entrylk, stub->args_cbk.xdata);
+		break;
         case GF_FOP_FENTRYLK:
-        {
-                if (stub->args.fentrylk.volume)
-                        GF_FREE ((char *)stub->args.fentrylk.volume);
-
-                if (stub->args.fentrylk.name)
-                        GF_FREE ((char *)stub->args.fentrylk.name);
-
-                if (stub->args.fentrylk.fd)
-                        fd_unref (stub->args.fentrylk.fd);
+		STUB_UNWIND (stub, fentrylk, stub->args_cbk.xdata);
                 break;
-        }
-
         case GF_FOP_LOOKUP:
-        {
-                loc_wipe (&stub->args.lookup.loc);
-                if (stub->args.lookup.xattr_req)
-                        dict_unref (stub->args.lookup.xattr_req);
+		STUB_UNWIND (stub, lookup, stub->args_cbk.inode,
+			     &stub->args_cbk.stat, stub->args_cbk.xdata,
+			     &stub->args_cbk.postparent);
                 break;
-        }
-
         case GF_FOP_RCHECKSUM:
-        {
-                if (stub->args.rchecksum.fd)
-                        fd_unref (stub->args.rchecksum.fd);
-                break;
-        }
-
+		STUB_UNWIND (stub, rchecksum, stub->args_cbk.weak_checksum,
+			     stub->args_cbk.strong_checksum, stub->args_cbk.xdata);
+		break;
         case GF_FOP_READDIR:
-        {
-                if (stub->args.readdir.fd)
-                        fd_unref (stub->args.readdir.fd);
-                break;
-        }
-
+		STUB_UNWIND (stub, readdir, &stub->args_cbk.entries,
+			     stub->args_cbk.xdata);
+		break;
         case GF_FOP_READDIRP:
-        {
-                if (stub->args.readdirp.fd)
-                        fd_unref (stub->args.readdirp.fd);
+		STUB_UNWIND (stub, readdir, &stub->args_cbk.entries,
+			     stub->args_cbk.xdata);
                 break;
-        }
-
         case GF_FOP_XATTROP:
-        {
-                loc_wipe (&stub->args.xattrop.loc);
-                dict_unref (stub->args.xattrop.xattr);
-                break;
-        }
+		STUB_UNWIND (stub, xattrop, stub->args_cbk.xattr,
+			     stub->args_cbk.xdata);
+		break;
         case GF_FOP_FXATTROP:
-        {
-                if (stub->args.fxattrop.fd)
-                        fd_unref (stub->args.fxattrop.fd);
-                dict_unref (stub->args.fxattrop.xattr);
-                break;
-        }
+		STUB_UNWIND (stub, fxattrop, stub->args_cbk.xattr,
+			     stub->args_cbk.xdata);
+		break;
         case GF_FOP_SETATTR:
-        {
-                loc_wipe (&stub->args.setattr.loc);
+		STUB_UNWIND (stub, setattr, &stub->args_cbk.prestat,
+			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
                 break;
-        }
         case GF_FOP_FSETATTR:
-        {
-                if (stub->args.fsetattr.fd)
-                        fd_unref (stub->args.fsetattr.fd);
+		STUB_UNWIND (stub, fsetattr, &stub->args_cbk.prestat,
+			     &stub->args_cbk.poststat, stub->args_cbk.xdata);
+                break;
+	case GF_FOP_FALLOCATE:
+		STUB_UNWIND(stub, fallocate, &stub->args_cbk.prestat,
+			    &stub->args_cbk.poststat, stub->args_cbk.xdata);
+		break;
+	case GF_FOP_DISCARD:
+		STUB_UNWIND(stub, discard, &stub->args_cbk.prestat,
+			    &stub->args_cbk.poststat, stub->args_cbk.xdata);
+		break;
+        case GF_FOP_ZEROFILL:
+                STUB_UNWIND(stub, zerofill, &stub->args_cbk.prestat,
+                            &stub->args_cbk.poststat, stub->args_cbk.xdata);
                 break;
-        }
+
         default:
-        {
                 gf_log_callingfn ("call-stub", GF_LOG_ERROR,
                                   "Invalid value of FOP (%d)",
                                   stub->fop);
                 break;
         }
-        }
+out:
+        return;
 }
 
 
 static void
-call_stub_destroy_unwind (call_stub_t *stub)
+call_stub_wipe_args (call_stub_t *stub)
 {
-        switch (stub->fop) {
-        case GF_FOP_OPEN:
-        {
-                if (stub->args.open_cbk.fd)
-                        fd_unref (stub->args.open_cbk.fd);
-        }
-        break;
-
-        case GF_FOP_CREATE:
-        {
-                if (stub->args.create_cbk.fd)
-                        fd_unref (stub->args.create_cbk.fd);
-
-                if (stub->args.create_cbk.inode)
-                        inode_unref (stub->args.create_cbk.inode);
-        }
-        break;
-
-        case GF_FOP_STAT:
-                break;
-
-        case GF_FOP_READLINK:
-        {
-                if (stub->args.readlink_cbk.buf)
-                        GF_FREE ((char *)stub->args.readlink_cbk.buf);
-        }
-        break;
-
-        case GF_FOP_MKNOD:
-        {
-                if (stub->args.mknod_cbk.inode)
-                        inode_unref (stub->args.mknod_cbk.inode);
-        }
-        break;
-
-        case GF_FOP_MKDIR:
-        {
-                if (stub->args.mkdir_cbk.inode)
-                        inode_unref (stub->args.mkdir_cbk.inode);
-        }
-        break;
-
-        case GF_FOP_UNLINK:
-                break;
-
-        case GF_FOP_RMDIR:
-                break;
-
-        case GF_FOP_SYMLINK:
-        {
-                if (stub->args.symlink_cbk.inode)
-                        inode_unref (stub->args.symlink_cbk.inode);
-        }
-        break;
-
-        case GF_FOP_RENAME:
-                break;
-
-        case GF_FOP_LINK:
-        {
-                if (stub->args.link_cbk.inode)
-                        inode_unref (stub->args.link_cbk.inode);
-        }
-        break;
-
-        case GF_FOP_TRUNCATE:
-                break;
-
-        case GF_FOP_READ:
-        {
-                if (stub->args.readv_cbk.op_ret >= 0) {
-                        struct iobref *iobref = stub->args.readv_cbk.iobref;
-                        GF_FREE (stub->args.readv_cbk.vector);
-
-                        if (iobref) {
-                                iobref_unref (iobref);
-                        }
-                }
-        }
-        break;
-
-        case GF_FOP_WRITE:
-                break;
-
-        case GF_FOP_STATFS:
-                break;
-
-        case GF_FOP_FLUSH:
-                break;
-
-        case GF_FOP_FSYNC:
-                break;
-
-        case GF_FOP_SETXATTR:
-                break;
-
-        case GF_FOP_GETXATTR:
-        {
-                if (stub->args.getxattr_cbk.dict)
-                        dict_unref (stub->args.getxattr_cbk.dict);
-        }
-        break;
+	loc_wipe (&stub->args.loc);
 
-        case GF_FOP_FSETXATTR:
-                break;
+	loc_wipe (&stub->args.loc2);
 
-        case GF_FOP_FGETXATTR:
-        {
-                if (stub->args.fgetxattr_cbk.dict)
-                        dict_unref (stub->args.fgetxattr_cbk.dict);
-        }
-        break;
+	if (stub->args.fd)
+		fd_unref (stub->args.fd);
 
-        case GF_FOP_REMOVEXATTR:
-                break;
-        case GF_FOP_FREMOVEXATTR:
-                break;
+	GF_FREE ((char *)stub->args.linkname);
 
-        case GF_FOP_OPENDIR:
-        {
-                if (stub->args.opendir_cbk.fd)
-                        fd_unref (stub->args.opendir_cbk.fd);
-        }
-        break;
+	GF_FREE (stub->args.vector);
 
-        case GF_FOP_FSYNCDIR:
-                break;
+	if (stub->args.iobref)
+		iobref_unref (stub->args.iobref);
 
-        case GF_FOP_ACCESS:
-                break;
+	if (stub->args.xattr)
+		dict_unref (stub->args.xattr);
 
-        case GF_FOP_FTRUNCATE:
-                break;
+	GF_FREE ((char *)stub->args.name);
 
-        case GF_FOP_FSTAT:
-                break;
+	GF_FREE ((char *)stub->args.volume);
 
-        case GF_FOP_LK:
-                break;
+	if (stub->args.xdata)
+		dict_unref (stub->args.xdata);
+}
 
-        case GF_FOP_INODELK:
-                break;
 
-        case GF_FOP_FINODELK:
-                break;
+static void
+call_stub_wipe_args_cbk (call_stub_t *stub)
+{
+	if (stub->args_cbk.inode)
+		inode_unref (stub->args_cbk.inode);
 
-        case GF_FOP_ENTRYLK:
-                break;
+	GF_FREE ((char *)stub->args_cbk.buf);
 
-        case GF_FOP_FENTRYLK:
-                break;
+	GF_FREE (stub->args_cbk.vector);
 
-        case GF_FOP_LOOKUP:
-        {
-                if (stub->args.lookup_cbk.inode)
-                        inode_unref (stub->args.lookup_cbk.inode);
+	if (stub->args_cbk.iobref)
+		iobref_unref (stub->args_cbk.iobref);
 
-                if (stub->args.lookup_cbk.dict)
-                        dict_unref (stub->args.lookup_cbk.dict);
-        }
-        break;
+	if (stub->args_cbk.fd)
+		fd_unref (stub->args_cbk.fd);
 
-        case GF_FOP_RCHECKSUM:
-        {
-                if (stub->args.rchecksum_cbk.op_ret >= 0) {
-                        GF_FREE (stub->args.rchecksum_cbk.strong_checksum);
-                }
-        }
-        break;
+	if (stub->args_cbk.xattr)
+		dict_unref (stub->args_cbk.xattr);
 
-        case GF_FOP_READDIR:
-        {
-                if (stub->args.readdir_cbk.op_ret > 0) {
-                        gf_dirent_free (&stub->args.readdir_cbk.entries);
-                }
-        }
-        break;
+	GF_FREE (stub->args_cbk.strong_checksum);
 
-        case GF_FOP_READDIRP:
-        {
-                if (stub->args.readdirp_cbk.op_ret > 0) {
-                        gf_dirent_free (&stub->args.readdirp_cbk.entries);
-                }
-        }
-        break;
+	if (stub->args_cbk.xdata)
+		dict_unref (stub->args_cbk.xdata);
 
-        case GF_FOP_XATTROP:
-        {
-                if (stub->args.xattrop_cbk.xattr)
-                        dict_unref (stub->args.xattrop_cbk.xattr);
-        }
-        break;
-
-        case GF_FOP_FXATTROP:
-        {
-                if (stub->args.fxattrop_cbk.xattr)
-                        dict_unref (stub->args.fxattrop_cbk.xattr);
-        }
-        break;
-
-        case GF_FOP_SETATTR:
-        {
-                break;
-        }
-
-        case GF_FOP_FSETATTR:
-        {
-                break;
-        }
-
-        default:
-        {
-                gf_log_callingfn ("call-stub", GF_LOG_ERROR,
-                                  "Invalid value of FOP (%d)",
-                                  stub->fop);
-                break;
-        }
-        }
+	if (!list_empty (&stub->args_cbk.entries.list))
+		gf_dirent_free (&stub->args_cbk.entries);
 }
 
 
@@ -3909,18 +2811,19 @@ call_stub_destroy (call_stub_t *stub)
 {
         GF_VALIDATE_OR_GOTO ("call-stub", stub, out);
 
-        if (stub->wind) {
-                call_stub_destroy_wind (stub);
-        } else {
-                call_stub_destroy_unwind (stub);
-        }
+        if (stub->wind)
+                call_stub_wipe_args (stub);
+        else
+                call_stub_wipe_args_cbk (stub);
 
         stub->stub_mem_pool = NULL;
+
         mem_put (stub);
 out:
         return;
 }
 
+
 void
 call_resume (call_stub_t *stub)
 {
@@ -3945,3 +2848,26 @@ call_resume (call_stub_t *stub)
 out:
         return;
 }
+
+
+void
+call_unwind_error (call_stub_t *stub, int op_ret, int op_errno)
+{
+        xlator_t *old_THIS = NULL;
+
+        list_del_init (&stub->list);
+
+        old_THIS = THIS;
+        THIS = stub->frame->this;
+        {
+		stub->args_cbk.op_ret = op_ret;
+		stub->args_cbk.op_errno = op_errno;
+		call_resume_unwind (stub);
+        }
+        THIS = old_THIS;
+
+        call_stub_destroy (stub);
+
+        return;
+
+}
diff --git a/libglusterfs/src/call-stub.h b/libglusterfs/src/call-stub.h
index 081f6d200..45bef8044 100644
--- a/libglusterfs/src/call-stub.h
+++ b/libglusterfs/src/call-stub.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CALL_STUB_H_
@@ -34,576 +25,164 @@ typedef struct {
 	char wind;
 	call_frame_t *frame;
 	glusterfs_fop_t fop;
-       struct mem_pool *stub_mem_pool;    /* pointer to stub mempool in glusterfs ctx */
+        struct mem_pool *stub_mem_pool; /* pointer to stub mempool in ctx_t */
 
 	union {
-		/* lookup */
-		struct {
-			fop_lookup_t fn;
-			loc_t loc;
-			dict_t *xattr_req;
-		} lookup;
-		struct {
-			fop_lookup_cbk_t fn;
-			int32_t op_ret, op_errno;
-			inode_t *inode;
-			struct iatt buf;
-			dict_t *dict;
-                        struct iatt postparent;
-		} lookup_cbk;
-
-		/* stat */
-		struct {
-			fop_stat_t fn;
-			loc_t loc;
-		} stat;
-		struct {
-			fop_stat_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct iatt buf;
-		} stat_cbk;
-
-		/* fstat */
-		struct {
-			fop_fstat_t fn;
-			fd_t *fd;
-		} fstat;
-		struct {
-			fop_fstat_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct iatt buf;
-		} fstat_cbk;
-
-		/* truncate */
-		struct {
-			fop_truncate_t fn;
-			loc_t loc;
-			off_t off;
-		} truncate;
-		struct {
-			fop_truncate_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct iatt prebuf;
-                        struct iatt postbuf;
-		} truncate_cbk;
-
-		/* ftruncate */
-		struct {
-			fop_ftruncate_t fn;
-			fd_t *fd;
-			off_t off;
-		} ftruncate;
-		struct {
-			fop_ftruncate_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct iatt prebuf;
-                        struct iatt postbuf;
-		} ftruncate_cbk;
-
-		/* access */
-		struct {
-			fop_access_t fn;
-			loc_t loc;
-			int32_t mask;
-		} access;
-		struct {
-			fop_access_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} access_cbk;
-
-		/* readlink */
-		struct {
-			fop_readlink_t fn;
-			loc_t loc;
-			size_t size;
-		} readlink;
-		struct {
-			fop_readlink_cbk_t fn;
-			int32_t op_ret, op_errno;
-			const char *buf;
-                        struct iatt sbuf;
-		} readlink_cbk;
-
-		/* mknod */
-		struct {
-			fop_mknod_t fn;
-			loc_t loc;
-			mode_t mode;
-			dev_t rdev;
-                        dict_t *params;
-		} mknod;
-		struct {
-			fop_mknod_cbk_t fn;
-			int32_t op_ret, op_errno;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} mknod_cbk;
-
-		/* mkdir */
-		struct {
-			fop_mkdir_t fn;
-			loc_t loc;
-			mode_t mode;
-                        dict_t *params;
-		} mkdir;
-		struct {
-			fop_mkdir_cbk_t fn;
-			int32_t op_ret, op_errno;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} mkdir_cbk;
-
-		/* unlink */
-		struct {
-			fop_unlink_t fn;
-			loc_t loc;
-		} unlink;
-		struct {
-			fop_unlink_cbk_t fn;
-			int32_t op_ret, op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} unlink_cbk;
-
-		/* rmdir */
-		struct {
-			fop_rmdir_t fn;
-			loc_t loc;
-                        int   flags;
-		} rmdir;
-		struct {
-			fop_rmdir_cbk_t fn;
-			int32_t op_ret, op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} rmdir_cbk;
-
-		/* symlink */
-		struct {
-			fop_symlink_t fn;
-			const char *linkname;
-			loc_t loc;
-                        dict_t *params;
-		} symlink;
-		struct {
-			fop_symlink_cbk_t fn;
-			int32_t op_ret, op_errno;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} symlink_cbk;
-
-		/* rename */
-		struct {
-			fop_rename_t fn;
-			loc_t old;
-			loc_t new;
-		} rename;
-		struct {
-			fop_rename_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct iatt buf;
-                        struct iatt preoldparent;
-                        struct iatt postoldparent;
-                        struct iatt prenewparent;
-                        struct iatt postnewparent;
-		} rename_cbk;
-
-		/* link */
-		struct {
-			fop_link_t fn;
-			loc_t oldloc;
-			loc_t newloc;
-		} link;
-		struct {
-			fop_link_cbk_t fn;
-			int32_t op_ret, op_errno;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} link_cbk;
-
-		/* create */
-		struct {
-			fop_create_t fn;
-			loc_t loc;
-			int32_t flags;
-			mode_t mode;
-			fd_t *fd;
-                        dict_t *params;
-		} create;
-		struct {
-			fop_create_cbk_t fn;
-			int32_t op_ret, op_errno;
-			fd_t *fd;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} create_cbk;
-
-		/* open */
-		struct {
-			fop_open_t fn;
-			loc_t loc;
-			int32_t flags;
-			fd_t *fd;
-                        int32_t wbflags;
-		} open;
-		struct {
-			fop_open_cbk_t fn;
-			int32_t op_ret, op_errno;
-			fd_t *fd;
-		} open_cbk;
-
-		/* readv */
-		struct {
-			fop_readv_t fn;
-			fd_t *fd;
-			size_t size;
-			off_t off;
-                        uint32_t flags;
-		} readv;
-		struct {
-			fop_readv_cbk_t fn;
-			int32_t op_ret;
-			int32_t op_errno;
-			struct iovec *vector;
-			int32_t count;
-			struct iatt stbuf;
-			struct iobref *iobref;
-		} readv_cbk;
-
-		/* writev */
-		struct {
-			fop_writev_t fn;
-			fd_t *fd;
-			struct iovec *vector;
-			int32_t count;
-			off_t off;
-                        uint32_t flags;
-			struct iobref *iobref;
-		} writev;
-		struct {
-			fop_writev_cbk_t fn;
-			int32_t op_ret, op_errno;
-                        struct iatt prebuf;
-			struct iatt postbuf;
-		} writev_cbk;
-
-		/* flush */
-		struct {
-			fop_flush_t fn;
-			fd_t *fd;
-		} flush;
-		struct {
-			fop_flush_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} flush_cbk;
-
-		/* fsync */
-		struct {
-			fop_fsync_t fn;
-			fd_t *fd;
-			int32_t datasync;
-		} fsync;
-		struct {
-			fop_fsync_cbk_t fn;
-			int32_t op_ret, op_errno;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-		} fsync_cbk;
-
-		/* opendir */
-		struct {
-			fop_opendir_t fn;
-			loc_t loc;
-			fd_t *fd;
-		} opendir;
-		struct {
-			fop_opendir_cbk_t fn;
-			int32_t op_ret, op_errno;
-			fd_t *fd;
-		} opendir_cbk;
-
-
-		/* fsyncdir */
-		struct {
-			fop_fsyncdir_t fn;
-			fd_t *fd;
-			int32_t datasync;
-		} fsyncdir;
-		struct {
-			fop_fsyncdir_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} fsyncdir_cbk;
-
-		/* statfs */
-		struct {
-			fop_statfs_t fn;
-			loc_t loc;
-		} statfs;
-		struct {
-			fop_statfs_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct statvfs buf;
-		} statfs_cbk;
-
-		/* setxattr */
-		struct {
-			fop_setxattr_t fn;
-			loc_t loc;
-			dict_t *dict;
-			int32_t flags;
-		} setxattr;
-		struct {
-			fop_setxattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} setxattr_cbk;
-
-		/* getxattr */
-		struct {
-			fop_getxattr_t fn;
-			loc_t loc;
-			const char *name;
-		} getxattr;
-		struct {
-			fop_getxattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-			dict_t *dict;
-		} getxattr_cbk;
-
-		/* fsetxattr */
-		struct {
-			fop_fsetxattr_t fn;
-			fd_t *fd;
-			dict_t *dict;
-			int32_t flags;
-		} fsetxattr;
-		struct {
-			fop_fsetxattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} fsetxattr_cbk;
-
-		/* fgetxattr */
-		struct {
-			fop_fgetxattr_t fn;
-			fd_t *fd;
-			const char *name;
-		} fgetxattr;
-		struct {
-			fop_fgetxattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-			dict_t *dict;
-		} fgetxattr_cbk;
-
-		/* removexattr */
-		struct {
-			fop_removexattr_t fn;
-			loc_t loc;
-			const char *name;
-		} removexattr;
-		struct {
-			fop_removexattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} removexattr_cbk;
-
-
-		/* fremovexattr */
-		struct {
-			fop_fremovexattr_t fn;
-			fd_t *fd;
-			const char *name;
-		} fremovexattr;
-		struct {
-			fop_fremovexattr_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} fremovexattr_cbk;
-
-		/* lk */
-		struct {
-			fop_lk_t fn;
-			fd_t *fd;
-			int32_t cmd;
-			struct gf_flock lock;
-		} lk;
-		struct {
-			fop_lk_cbk_t fn;
-			int32_t op_ret, op_errno;
-			struct gf_flock lock;
-		} lk_cbk;
-
-		/* inodelk */
-		struct {
-			fop_inodelk_t fn;
-                        const char *volume;
-			loc_t loc;
-			int32_t cmd;
-			struct gf_flock lock;
-		} inodelk;
-
-		struct {
-			fop_inodelk_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} inodelk_cbk;
-
-		/* finodelk */
-		struct {
-			fop_finodelk_t fn;
-                        const char *volume;
-			fd_t *fd;
-			int32_t cmd;
-			struct gf_flock lock;
-		} finodelk;
-
-		struct {
-			fop_finodelk_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} finodelk_cbk;
-
-		/* entrylk */
-		struct {
-			fop_entrylk_t fn;
-			loc_t loc;
-                        const char *volume;
-			const char *name;
-			entrylk_cmd cmd;
-			entrylk_type type;
-		} entrylk;
-
-		struct {
-			fop_entrylk_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} entrylk_cbk;
-
-		/* fentrylk */
-		struct {
-			fop_fentrylk_t fn;
-			fd_t *fd;
-                        const char *volume;
-			const char *name;
-			entrylk_cmd cmd;
-			entrylk_type type;
-		} fentrylk;
-
-		struct {
-			fop_fentrylk_cbk_t fn;
-			int32_t op_ret, op_errno;
-		} fentrylk_cbk;
-
-		/* readdir */
-		struct {
-			fop_readdir_t fn;
-			fd_t *fd;
-			size_t size;
-			off_t off;
-		} readdir;
-		struct {
-			fop_readdir_cbk_t fn;
-			int32_t op_ret, op_errno;
-			gf_dirent_t entries;
-		} readdir_cbk;
-
-                /* readdirp */
-		struct {
-			fop_readdirp_t fn;
-			fd_t *fd;
-			size_t size;
-			off_t off;
-                        dict_t *dict;
-		} readdirp;
-		struct {
-			fop_readdirp_cbk_t fn;
-			int32_t op_ret, op_errno;
-			gf_dirent_t entries;
-		} readdirp_cbk;
-
-		/* rchecksum */
-		struct {
-			fop_rchecksum_t fn;
-			fd_t *fd;
-                        off_t offset;
-			int32_t len;
-		} rchecksum;
-		struct {
-			fop_rchecksum_cbk_t fn;
-			int32_t op_ret, op_errno;
-			uint32_t weak_checksum;
-			uint8_t *strong_checksum;
-		} rchecksum_cbk;
-
-		/* xattrop */
-		struct {
-			fop_xattrop_t fn;
-			loc_t loc;
-			gf_xattrop_flags_t optype;
-			dict_t *xattr;
-		} xattrop;
-		struct {
-			fop_xattrop_cbk_t fn;
-			int32_t op_ret;
-			int32_t op_errno;
-			dict_t *xattr;
-		} xattrop_cbk;
-
-		/* fxattrop */
-		struct {
-			fop_fxattrop_t fn;
-			fd_t *fd;
-			gf_xattrop_flags_t optype;
-			dict_t *xattr;
-		} fxattrop;
-		struct {
-			fop_fxattrop_cbk_t fn;
-			int32_t op_ret;
-			int32_t op_errno;
-			dict_t *xattr;
-		} fxattrop_cbk;
-
-                /* setattr */
-                struct {
-                        fop_setattr_t fn;
-                        loc_t loc;
-                        struct iatt stbuf;
-                        int32_t valid;
-                } setattr;
-                struct {
-                        fop_setattr_cbk_t fn;
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt statpre;
-                        struct iatt statpost;
-                } setattr_cbk;
-
-                /* fsetattr */
-                struct {
-                        fop_fsetattr_t fn;
-                        fd_t *fd;
-                        struct iatt stbuf;
-                        int32_t valid;
-                } fsetattr;
-                struct {
-                        fop_fsetattr_cbk_t fn;
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt statpre;
-                        struct iatt statpost;
-                } fsetattr_cbk;
+		fop_lookup_t lookup;
+		fop_stat_t stat;
+		fop_fstat_t fstat;
+		fop_truncate_t truncate;
+		fop_ftruncate_t ftruncate;
+		fop_access_t access;
+		fop_readlink_t readlink;
+		fop_mknod_t mknod;
+		fop_mkdir_t mkdir;
+		fop_unlink_t unlink;
+		fop_rmdir_t rmdir;
+		fop_symlink_t symlink;
+		fop_rename_t rename;
+		fop_link_t link;
+		fop_create_t create;
+		fop_open_t open;
+		fop_readv_t readv;
+		fop_writev_t writev;
+		fop_flush_t flush;
+		fop_fsync_t fsync;
+		fop_opendir_t opendir;
+		fop_fsyncdir_t fsyncdir;
+		fop_statfs_t statfs;
+		fop_setxattr_t setxattr;
+		fop_getxattr_t getxattr;
+		fop_fgetxattr_t fgetxattr;
+		fop_fsetxattr_t fsetxattr;
+		fop_removexattr_t removexattr;
+		fop_fremovexattr_t fremovexattr;
+		fop_lk_t lk;
+		fop_inodelk_t inodelk;
+		fop_finodelk_t finodelk;
+		fop_entrylk_t entrylk;
+		fop_fentrylk_t fentrylk;
+		fop_readdir_t readdir;
+		fop_readdirp_t readdirp;
+		fop_rchecksum_t rchecksum;
+		fop_xattrop_t xattrop;
+		fop_fxattrop_t fxattrop;
+		fop_setattr_t setattr;
+		fop_fsetattr_t fsetattr;
+		fop_fallocate_t fallocate;
+		fop_discard_t discard;
+                fop_zerofill_t zerofill;
+	} fn;
 
+	union {
+		fop_lookup_cbk_t lookup;
+		fop_stat_cbk_t stat;
+		fop_fstat_cbk_t fstat;
+		fop_truncate_cbk_t truncate;
+		fop_ftruncate_cbk_t ftruncate;
+		fop_access_cbk_t access;
+		fop_readlink_cbk_t readlink;
+		fop_mknod_cbk_t mknod;
+		fop_mkdir_cbk_t mkdir;
+		fop_unlink_cbk_t unlink;
+		fop_rmdir_cbk_t rmdir;
+		fop_symlink_cbk_t symlink;
+		fop_rename_cbk_t rename;
+		fop_link_cbk_t link;
+		fop_create_cbk_t create;
+		fop_open_cbk_t open;
+		fop_readv_cbk_t readv;
+		fop_writev_cbk_t writev;
+		fop_flush_cbk_t flush;
+		fop_fsync_cbk_t fsync;
+		fop_opendir_cbk_t opendir;
+		fop_fsyncdir_cbk_t fsyncdir;
+		fop_statfs_cbk_t statfs;
+		fop_setxattr_cbk_t setxattr;
+		fop_getxattr_cbk_t getxattr;
+		fop_fgetxattr_cbk_t fgetxattr;
+		fop_fsetxattr_cbk_t fsetxattr;
+		fop_removexattr_cbk_t removexattr;
+		fop_fremovexattr_cbk_t fremovexattr;
+		fop_lk_cbk_t lk;
+		fop_inodelk_cbk_t inodelk;
+		fop_finodelk_cbk_t finodelk;
+		fop_entrylk_cbk_t entrylk;
+		fop_fentrylk_cbk_t fentrylk;
+		fop_readdir_cbk_t readdir;
+		fop_readdirp_cbk_t readdirp;
+		fop_rchecksum_cbk_t rchecksum;
+		fop_xattrop_cbk_t xattrop;
+		fop_fxattrop_cbk_t fxattrop;
+		fop_setattr_cbk_t setattr;
+		fop_fsetattr_cbk_t fsetattr;
+		fop_fallocate_cbk_t fallocate;
+		fop_discard_cbk_t discard;
+                fop_zerofill_cbk_t zerofill;
+	} fn_cbk;
+
+	struct {
+		loc_t loc; // @old in rename(), link()
+		loc_t loc2; // @new in rename(), link()
+		fd_t *fd;
+		off_t offset;
+		int mask;
+		size_t size;
+		mode_t mode;
+		dev_t rdev;
+		mode_t umask;
+		int xflag;
+		int flags;
+		const char *linkname;
+		struct iovec *vector;
+		int count;
+		struct iobref *iobref;
+		int datasync;
+		dict_t *xattr;
+		const char *name;
+		int cmd;
+		struct gf_flock lock;
+		const char *volume;
+		entrylk_cmd entrylkcmd;
+		entrylk_type entrylktype;
+		gf_xattrop_flags_t optype;
+		int valid;
+		struct iatt stat;
+		dict_t *xdata;
 	} args;
+
+	struct {
+		int op_ret;
+		int op_errno;
+		inode_t *inode;
+		struct iatt stat;
+		struct iatt prestat;
+		struct iatt poststat;
+		struct iatt preparent;   // @preoldparent in rename_cbk
+		struct iatt postparent;  // @postoldparent in rename_cbk
+		struct iatt preparent2;  // @prenewparent in rename_cbk
+		struct iatt postparent2; // @postnewparent in rename_cbk
+		const char *buf;
+		struct iovec *vector;
+		int count;
+		struct iobref *iobref;
+		fd_t *fd;
+		struct statvfs statvfs;
+		dict_t *xattr;
+		struct gf_flock lock;
+		gf_dirent_t entries;
+		uint32_t weak_checksum;
+		uint8_t *strong_checksum;
+		dict_t *xdata;
+	} args_cbk;
 } call_stub_t;
 
+
 call_stub_t *
 fop_lookup_stub (call_frame_t *frame,
 		 fop_lookup_t fn,
 		 loc_t *loc,
-		 dict_t *xattr_req);
+		 dict_t *xdata);
 
 call_stub_t *
 fop_lookup_cbk_stub (call_frame_t *frame,
@@ -612,34 +191,34 @@ fop_lookup_cbk_stub (call_frame_t *frame,
 		     int32_t op_errno,
 		     inode_t *inode,
 		     struct iatt *buf,
-                     dict_t *dict,
+                     dict_t *xdata,
                      struct iatt *postparent);
 call_stub_t *
 fop_stat_stub (call_frame_t *frame,
 	       fop_stat_t fn,
-	       loc_t *loc);
+	       loc_t *loc, dict_t *xdata);
 call_stub_t *
 fop_stat_cbk_stub (call_frame_t *frame,
 		   fop_stat_cbk_t fn,
 		   int32_t op_ret,
 		   int32_t op_errno,
-		   struct iatt *buf);
+		   struct iatt *buf, dict_t *xdata);
 call_stub_t *
 fop_fstat_stub (call_frame_t *frame,
 		fop_fstat_t fn,
-		fd_t *fd);
+		fd_t *fd, dict_t *xdata);
 call_stub_t *
 fop_fstat_cbk_stub (call_frame_t *frame,
 		    fop_fstat_cbk_t fn,
 		    int32_t op_ret,
 		    int32_t op_errno,
-		    struct iatt *buf);
+		    struct iatt *buf, dict_t *xdata);
 
 call_stub_t *
 fop_truncate_stub (call_frame_t *frame,
 		   fop_truncate_t fn,
 		   loc_t *loc,
-		   off_t off);
+		   off_t off, dict_t *xdata);
 
 call_stub_t *
 fop_truncate_cbk_stub (call_frame_t *frame,
@@ -647,13 +226,13 @@ fop_truncate_cbk_stub (call_frame_t *frame,
 		       int32_t op_ret,
 		       int32_t op_errno,
 		       struct iatt *prebuf,
-                       struct iatt *postbuf);
+                       struct iatt *postbuf, dict_t *xdata);
 
 call_stub_t *
 fop_ftruncate_stub (call_frame_t *frame,
 		    fop_ftruncate_t fn,
 		    fd_t *fd,
-		    off_t off);
+		    off_t off, dict_t *xdata);
 
 call_stub_t *
 fop_ftruncate_cbk_stub (call_frame_t *frame,
@@ -661,25 +240,25 @@ fop_ftruncate_cbk_stub (call_frame_t *frame,
 			int32_t op_ret,
 			int32_t op_errno,
 			struct iatt *prebuf,
-                        struct iatt *postbuf);
+                        struct iatt *postbuf, dict_t *xdata);
 
 call_stub_t *
 fop_access_stub (call_frame_t *frame,
 		 fop_access_t fn,
 		 loc_t *loc,
-		 int32_t mask);
+		 int32_t mask, dict_t *xdata);
 
 call_stub_t *
 fop_access_cbk_stub (call_frame_t *frame,
 		     fop_access_cbk_t fn,
 		     int32_t op_ret,
-		     int32_t op_errno);
+		     int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_readlink_stub (call_frame_t *frame,
 		   fop_readlink_t fn,
 		   loc_t *loc,
-		   size_t size);
+		   size_t size, dict_t *xdata);
 
 call_stub_t *
 fop_readlink_cbk_stub (call_frame_t *frame,
@@ -687,11 +266,11 @@ fop_readlink_cbk_stub (call_frame_t *frame,
 		       int32_t op_ret,
 		       int32_t op_errno,
 		       const char *path,
-                       struct iatt *buf);
+                       struct iatt *buf, dict_t *xdata);
 
 call_stub_t *
-fop_mknod_stub (call_frame_t *frame, fop_mknod_t fn,
-		loc_t *loc, mode_t mode, dev_t rdev, dict_t *params);
+fop_mknod_stub (call_frame_t *frame, fop_mknod_t fn, loc_t *loc, mode_t mode,
+                dev_t rdev, mode_t umask, dict_t *xdata);
 
 call_stub_t *
 fop_mknod_cbk_stub (call_frame_t *frame,
@@ -701,11 +280,11 @@ fop_mknod_cbk_stub (call_frame_t *frame,
 		    inode_t *inode,
                     struct iatt *buf,
                     struct iatt *preparent,
-                    struct iatt *postparent);
+                    struct iatt *postparent, dict_t *xdata);
 
 call_stub_t *
-fop_mkdir_stub (call_frame_t *frame, fop_mkdir_t fn,
-		loc_t *loc, mode_t mode, dict_t *params);
+fop_mkdir_stub (call_frame_t *frame, fop_mkdir_t fn, loc_t *loc, mode_t mode,
+                mode_t umask, dict_t *xdata);
 
 call_stub_t *
 fop_mkdir_cbk_stub (call_frame_t *frame,
@@ -715,12 +294,11 @@ fop_mkdir_cbk_stub (call_frame_t *frame,
 		    inode_t *inode,
                     struct iatt *buf,
                     struct iatt *preparent,
-                    struct iatt *postparent);
+                    struct iatt *postparent, dict_t *xdata);
 
 call_stub_t *
-fop_unlink_stub (call_frame_t *frame,
-		 fop_unlink_t fn,
-		 loc_t *loc);
+fop_unlink_stub (call_frame_t *frame, fop_unlink_t fn,
+		 loc_t *loc, int xflag, dict_t *xdata);
 
 call_stub_t *
 fop_unlink_cbk_stub (call_frame_t *frame,
@@ -728,11 +306,11 @@ fop_unlink_cbk_stub (call_frame_t *frame,
 		     int32_t op_ret,
 		     int32_t op_errno,
                      struct iatt *preparent,
-                     struct iatt *postparent);
+                     struct iatt *postparent, dict_t *xdata);
 
 call_stub_t *
 fop_rmdir_stub (call_frame_t *frame, fop_rmdir_t fn,
-		loc_t *loc, int flags);
+		loc_t *loc, int flags, dict_t *xdata);
 
 call_stub_t *
 fop_rmdir_cbk_stub (call_frame_t *frame,
@@ -740,11 +318,11 @@ fop_rmdir_cbk_stub (call_frame_t *frame,
 		    int32_t op_ret,
 		    int32_t op_errno,
                     struct iatt *preparent,
-                    struct iatt *postparent);
+                    struct iatt *postparent, dict_t *xdata);
 
 call_stub_t *
 fop_symlink_stub (call_frame_t *frame, fop_symlink_t fn,
-		  const char *linkname, loc_t *loc, dict_t *params);
+		  const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata);
 
 call_stub_t *
 fop_symlink_cbk_stub (call_frame_t *frame,
@@ -754,13 +332,13 @@ fop_symlink_cbk_stub (call_frame_t *frame,
 		      inode_t *inode,
                       struct iatt *buf,
                       struct iatt *preparent,
-                      struct iatt *postparent);
+                      struct iatt *postparent, dict_t *xdata);
 
 call_stub_t *
 fop_rename_stub (call_frame_t *frame,
 		 fop_rename_t fn,
 		 loc_t *oldloc,
-		 loc_t *newloc);
+		 loc_t *newloc, dict_t *xdata);
 
 call_stub_t *
 fop_rename_cbk_stub (call_frame_t *frame,
@@ -771,13 +349,13 @@ fop_rename_cbk_stub (call_frame_t *frame,
                      struct iatt *preoldparent,
                      struct iatt *postoldparent,
                      struct iatt *prenewparent,
-                     struct iatt *postnewparent);
+                     struct iatt *postnewparent, dict_t *xdata);
 
 call_stub_t *
 fop_link_stub (call_frame_t *frame,
 	       fop_link_t fn,
 	       loc_t *oldloc,
-	       loc_t *newloc);
+	       loc_t *newloc, dict_t *xdata);
 
 call_stub_t *
 fop_link_cbk_stub (call_frame_t *frame,
@@ -787,12 +365,12 @@ fop_link_cbk_stub (call_frame_t *frame,
 		   inode_t *inode,
                    struct iatt *buf,
                    struct iatt *preparent,
-                   struct iatt *postparent);
+                   struct iatt *postparent, dict_t *xdata);
 
 call_stub_t *
 fop_create_stub (call_frame_t *frame, fop_create_t fn,
 		 loc_t *loc, int32_t flags, mode_t mode,
-                 fd_t *fd, dict_t *params);
+                 mode_t umask, fd_t *fd, dict_t *xdata);
 
 call_stub_t *
 fop_create_cbk_stub (call_frame_t *frame,
@@ -803,7 +381,7 @@ fop_create_cbk_stub (call_frame_t *frame,
 		     inode_t *inode,
 		     struct iatt *buf,
                      struct iatt *preparent,
-                     struct iatt *postparent);
+                     struct iatt *postparent, dict_t *xdata);
 
 call_stub_t *
 fop_open_stub (call_frame_t *frame,
@@ -811,21 +389,21 @@ fop_open_stub (call_frame_t *frame,
 	       loc_t *loc,
 	       int32_t flags,
 	       fd_t *fd,
-               int32_t wbflags);
+               dict_t *xdata);
 
 call_stub_t *
 fop_open_cbk_stub (call_frame_t *frame,
 		   fop_open_cbk_t fn,
 		   int32_t op_ret,
 		   int32_t op_errno,
-		   fd_t *fd);
+		   fd_t *fd, dict_t *xdata);
 
 call_stub_t *
 fop_readv_stub (call_frame_t *frame,
 		fop_readv_t fn,
 		fd_t *fd,
 		size_t size,
-		off_t off, uint32_t flags);
+		off_t off, uint32_t flags, dict_t *xdata);
 
 call_stub_t *
 fop_readv_cbk_stub (call_frame_t *frame,
@@ -835,7 +413,7 @@ fop_readv_cbk_stub (call_frame_t *frame,
 		    struct iovec *vector,
 		    int32_t count,
 		    struct iatt *stbuf,
-                    struct iobref *iobref);
+                    struct iobref *iobref, dict_t *xdata);
 
 call_stub_t *
 fop_writev_stub (call_frame_t *frame,
@@ -844,7 +422,7 @@ fop_writev_stub (call_frame_t *frame,
 		 struct iovec *vector,
 		 int32_t count,
 		 off_t off, uint32_t flags,
-                 struct iobref *iobref);
+                 struct iobref *iobref, dict_t *xdata);
 
 call_stub_t *
 fop_writev_cbk_stub (call_frame_t *frame,
@@ -852,24 +430,24 @@ fop_writev_cbk_stub (call_frame_t *frame,
 		     int32_t op_ret,
 		     int32_t op_errno,
                      struct iatt *prebuf,
-                     struct iatt *postbuf);
+                     struct iatt *postbuf, dict_t *xdata);
 
 call_stub_t *
 fop_flush_stub (call_frame_t *frame,
 		fop_flush_t fn,
-		fd_t *fd);
+		fd_t *fd, dict_t *xdata);
 
 call_stub_t *
 fop_flush_cbk_stub (call_frame_t *frame,
 		    fop_flush_cbk_t fn,
 		    int32_t op_ret,
-		    int32_t op_errno);
+		    int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_fsync_stub (call_frame_t *frame,
 		fop_fsync_t fn,
 		fd_t *fd,
-		int32_t datasync);
+		int32_t datasync, dict_t *xdata);
 
 call_stub_t *
 fop_fsync_cbk_stub (call_frame_t *frame,
@@ -877,177 +455,177 @@ fop_fsync_cbk_stub (call_frame_t *frame,
 		    int32_t op_ret,
 		    int32_t op_errno,
                     struct iatt *prebuf,
-                    struct iatt *postbuf);
+                    struct iatt *postbuf, dict_t *xdata);
 
 call_stub_t *
 fop_opendir_stub (call_frame_t *frame,
 		  fop_opendir_t fn,
-		  loc_t *loc, fd_t *fd);
+		  loc_t *loc, fd_t *fd, dict_t *xdata);
 
 call_stub_t *
 fop_opendir_cbk_stub (call_frame_t *frame,
 		      fop_opendir_cbk_t fn,
 		      int32_t op_ret,
 		      int32_t op_errno,
-		      fd_t *fd);
+		      fd_t *fd, dict_t *xdata);
 
 call_stub_t *
 fop_fsyncdir_stub (call_frame_t *frame,
 		   fop_fsyncdir_t fn,
 		   fd_t *fd,
-		   int32_t datasync);
+		   int32_t datasync, dict_t *xdata);
 
 call_stub_t *
 fop_fsyncdir_cbk_stub (call_frame_t *frame,
 		       fop_fsyncdir_cbk_t fn,
 		       int32_t op_ret,
-		       int32_t op_errno);
+		       int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_statfs_stub (call_frame_t *frame,
 		 fop_statfs_t fn,
-		 loc_t *loc);
+		 loc_t *loc, dict_t *xdata);
 
 call_stub_t *
 fop_statfs_cbk_stub (call_frame_t *frame,
 		     fop_statfs_cbk_t fn,
 		     int32_t op_ret,
 		     int32_t op_errno,
-		     struct statvfs *buf);
+		     struct statvfs *buf, dict_t *xdata);
 
 call_stub_t *
 fop_setxattr_stub (call_frame_t *frame,
 		   fop_setxattr_t fn,
 		   loc_t *loc,
 		   dict_t *dict,
-		   int32_t flags);
+		   int32_t flags, dict_t *xdata);
 
 call_stub_t *
 fop_setxattr_cbk_stub (call_frame_t *frame,
 		       fop_setxattr_cbk_t fn,
 		       int32_t op_ret,
-		       int32_t op_errno);
+		       int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_getxattr_stub (call_frame_t *frame,
 		   fop_getxattr_t fn,
 		   loc_t *loc,
-		   const char *name);
+		   const char *name, dict_t *xdata);
 
 call_stub_t *
 fop_getxattr_cbk_stub (call_frame_t *frame,
 		       fop_getxattr_cbk_t fn,
 		       int32_t op_ret,
 		       int32_t op_errno,
-		       dict_t *value);
+		       dict_t *value, dict_t *xdata);
 
 call_stub_t *
 fop_fsetxattr_stub (call_frame_t *frame,
                     fop_fsetxattr_t fn,
                     fd_t *fd,
                     dict_t *dict,
-                    int32_t flags);
+                    int32_t flags, dict_t *xdata);
 
 call_stub_t *
 fop_fsetxattr_cbk_stub (call_frame_t *frame,
                         fop_fsetxattr_cbk_t fn,
                         int32_t op_ret,
-                        int32_t op_errno);
+                        int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_fgetxattr_stub (call_frame_t *frame,
                     fop_fgetxattr_t fn,
                     fd_t *fd,
-                    const char *name);
+                    const char *name, dict_t *xdata);
 
 call_stub_t *
 fop_fgetxattr_cbk_stub (call_frame_t *frame,
                         fop_fgetxattr_cbk_t fn,
                         int32_t op_ret,
                         int32_t op_errno,
-                        dict_t *value);
+                        dict_t *value, dict_t *xdata);
 
 call_stub_t *
 fop_removexattr_stub (call_frame_t *frame,
 		      fop_removexattr_t fn,
 		      loc_t *loc,
-		      const char *name);
+		      const char *name, dict_t *xdata);
 
 call_stub_t *
 fop_removexattr_cbk_stub (call_frame_t *frame,
 			  fop_removexattr_cbk_t fn,
 			  int32_t op_ret,
-			  int32_t op_errno);
+			  int32_t op_errno, dict_t *xdata);
 
 
 call_stub_t *
 fop_fremovexattr_stub (call_frame_t *frame,
                        fop_fremovexattr_t fn,
                        fd_t *fd,
-                       const char *name);
+                       const char *name, dict_t *xdata);
 
 call_stub_t *
 fop_fremovexattr_cbk_stub (call_frame_t *frame,
                            fop_fremovexattr_cbk_t fn,
                            int32_t op_ret,
-                           int32_t op_errno);
+                           int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_lk_stub (call_frame_t *frame,
 	     fop_lk_t fn,
 	     fd_t *fd,
 	     int32_t cmd,
-	     struct gf_flock *lock);
+	     struct gf_flock *lock, dict_t *xdata);
 
 call_stub_t *
 fop_lk_cbk_stub (call_frame_t *frame,
 		 fop_lk_cbk_t fn,
 		 int32_t op_ret,
 		 int32_t op_errno,
-		 struct gf_flock *lock);
+		 struct gf_flock *lock, dict_t *xdata);
 
 call_stub_t *
 fop_inodelk_stub (call_frame_t *frame, fop_inodelk_t fn,
 		  const char *volume, loc_t *loc, int32_t cmd,
-                  struct gf_flock *lock);
+                  struct gf_flock *lock, dict_t *xdata);
 
 call_stub_t *
 fop_finodelk_stub (call_frame_t *frame, fop_finodelk_t fn,
 		   const char *volume, fd_t *fd, int32_t cmd,
-                   struct gf_flock *lock);
+                   struct gf_flock *lock, dict_t *xdata);
 
 call_stub_t *
 fop_entrylk_stub (call_frame_t *frame, fop_entrylk_t fn,
 		  const char *volume, loc_t *loc, const char *basename,
-		  entrylk_cmd cmd, entrylk_type type);
+		  entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 call_stub_t *
 fop_fentrylk_stub (call_frame_t *frame, fop_fentrylk_t fn,
 		   const char *volume, fd_t *fd, const char *basename,
-		   entrylk_cmd cmd, entrylk_type type);
+		   entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 call_stub_t *
 fop_inodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-		      int32_t op_ret, int32_t op_errno);
+		      int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_finodelk_cbk_stub (call_frame_t *frame, fop_inodelk_cbk_t fn,
-		       int32_t op_ret, int32_t op_errno);
+		       int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_entrylk_cbk_stub (call_frame_t *frame, fop_entrylk_cbk_t fn,
-		      int32_t op_ret, int32_t op_errno);
+		      int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_fentrylk_cbk_stub (call_frame_t *frame, fop_entrylk_cbk_t fn,
-		       int32_t op_ret, int32_t op_errno);
+		       int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_readdir_stub (call_frame_t *frame,
 		  fop_readdir_t fn,
 		  fd_t *fd,
 		  size_t size,
-		  off_t off);
+		  off_t off, dict_t *xdata);
 
 call_stub_t *
 fop_readdirp_stub (call_frame_t *frame,
@@ -1055,27 +633,27 @@ fop_readdirp_stub (call_frame_t *frame,
 		   fd_t *fd,
 		   size_t size,
 		   off_t off,
-                   dict_t *dict);
+                   dict_t *xdata);
 
 call_stub_t *
 fop_readdirp_cbk_stub (call_frame_t *frame,
 		       fop_readdir_cbk_t fn,
 		       int32_t op_ret,
 		       int32_t op_errno,
-		       gf_dirent_t *entries);
+		       gf_dirent_t *entries, dict_t *xdata);
 
 call_stub_t *
 fop_readdir_cbk_stub (call_frame_t *frame,
 		      fop_readdir_cbk_t fn,
 		      int32_t op_ret,
 		      int32_t op_errno,
-		      gf_dirent_t *entries);
+		      gf_dirent_t *entries, dict_t *xdata);
 
 call_stub_t *
 fop_rchecksum_stub (call_frame_t *frame,
                     fop_rchecksum_t fn,
                     fd_t *fd, off_t offset,
-                    int32_t len);
+                    int32_t len, dict_t *xdata);
 
 call_stub_t *
 fop_rchecksum_cbk_stub (call_frame_t *frame,
@@ -1083,40 +661,40 @@ fop_rchecksum_cbk_stub (call_frame_t *frame,
                         int32_t op_ret,
                         int32_t op_errno,
                         uint32_t weak_checksum,
-                        uint8_t *strong_checksum);
+                        uint8_t *strong_checksum, dict_t *xdata);
 
 call_stub_t *
 fop_xattrop_stub (call_frame_t *frame,
 		  fop_xattrop_t fn,
 		  loc_t *loc,
 		  gf_xattrop_flags_t optype,
-		  dict_t *xattr);
+		  dict_t *xattr, dict_t *xdata);
 
 call_stub_t *
 fop_xattrop_stub_cbk_stub (call_frame_t *frame,
 			   fop_xattrop_cbk_t fn,
 			   int32_t op_ret,
-			   int32_t op_errno);
+			   int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_fxattrop_stub (call_frame_t *frame,
 		   fop_fxattrop_t fn,
 		   fd_t *fd,
 		   gf_xattrop_flags_t optype,
-		   dict_t *xattr);
+		   dict_t *xattr, dict_t *xdata);
 
 call_stub_t *
 fop_fxattrop_stub_cbk_stub (call_frame_t *frame,
 			    fop_xattrop_cbk_t fn,
 			    int32_t op_ret,
-			    int32_t op_errno);
+			    int32_t op_errno, dict_t *xdata);
 
 call_stub_t *
 fop_setattr_stub (call_frame_t *frame,
                   fop_setattr_t fn,
                   loc_t *loc,
                   struct iatt *stbuf,
-                  int32_t valid);
+                  int32_t valid, dict_t *xdata);
 
 call_stub_t *
 fop_setattr_cbk_stub (call_frame_t *frame,
@@ -1124,14 +702,14 @@ fop_setattr_cbk_stub (call_frame_t *frame,
                       int32_t op_ret,
                       int32_t op_errno,
                       struct iatt *statpre,
-                      struct iatt *statpost);
+                      struct iatt *statpost, dict_t *xdata);
 
 call_stub_t *
 fop_fsetattr_stub (call_frame_t *frame,
                    fop_fsetattr_t fn,
                    fd_t *fd,
                    struct iatt *stbuf,
-                   int32_t valid);
+                   int32_t valid, dict_t *xdata);
 
 call_stub_t *
 fop_fsetattr_cbk_stub (call_frame_t *frame,
@@ -1139,8 +717,51 @@ fop_fsetattr_cbk_stub (call_frame_t *frame,
                        int32_t op_ret,
                        int32_t op_errno,
                        struct iatt *statpre,
-                       struct iatt *statpost);
+                       struct iatt *statpost, dict_t *xdata);
+
+call_stub_t *
+fop_fallocate_stub(call_frame_t *frame,
+		   fop_fallocate_t fn,
+		   fd_t *fd,
+		   int32_t mode, off_t offset,
+		   size_t len, dict_t *xdata);
+
+call_stub_t *
+fop_fallocate_cbk_stub(call_frame_t *frame,
+		       fop_fallocate_cbk_t fn,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *statpre, struct iatt *statpost,
+                       dict_t *xdata);
+
+call_stub_t *
+fop_discard_stub(call_frame_t *frame,
+		 fop_discard_t fn,
+		 fd_t *fd,
+		 off_t offset,
+		 size_t len, dict_t *xdata);
+
+call_stub_t *
+fop_discard_cbk_stub(call_frame_t *frame,
+		     fop_discard_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *statpre, struct iatt *statpost,
+                     dict_t *xdata);
+
+call_stub_t *
+fop_zerofill_stub(call_frame_t *frame,
+                 fop_zerofill_t fn,
+                 fd_t *fd,
+                 off_t offset,
+                 size_t len, dict_t *xdata);
+
+call_stub_t *
+fop_zerofill_cbk_stub(call_frame_t *frame,
+                     fop_zerofill_cbk_t fn,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *statpre, struct iatt *statpost,
+                     dict_t *xdata);
 
 void call_resume (call_stub_t *stub);
 void call_stub_destroy (call_stub_t *stub);
+void call_unwind_error (call_stub_t *stub, int op_ret, int op_errno);
 #endif
diff --git a/libglusterfs/src/checksum.c b/libglusterfs/src/checksum.c
index cea782293..e14a3044c 100644
--- a/libglusterfs/src/checksum.c
+++ b/libglusterfs/src/checksum.c
@@ -1,28 +1,17 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include <inttypes.h>
+#include <openssl/md5.h>
+#include <stdint.h>
 
 #include "glusterfs.h"
-#include "md5.h"
-#include "checksum.h"
-
 
 /*
  * The "weak" checksum required for the rsync algorithm,
@@ -31,22 +20,26 @@
  *
  * "a simple 32 bit checksum that can be upadted from either end
  *  (inspired by Mark Adler's Adler-32 checksum)"
+ *
+ * Note: these functions are only called to compute checksums on
+ * pathnames; they don't need to handle arbitrarily long strings of
+ * data. Thus int32_t and uint32_t are sufficient
  */
 
 uint32_t
-gf_rsync_weak_checksum (char *buf1, int32_t len)
+gf_rsync_weak_checksum (unsigned char *buf, size_t len)
 {
-        int32_t i;
+        int32_t i = 0;
         uint32_t s1, s2;
 
-        signed char *buf = (signed char *) buf1;
         uint32_t csum;
 
         s1 = s2 = 0;
-        for (i = 0; i < (len-4); i+=4) {
-                s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3];
-
-                s1 += buf[i+0] + buf[i+1] + buf[i+2] + buf[i+3];
+        if (len >= 4) {
+                for (; i < (len-4); i+=4) {
+                        s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3];
+                        s1 += buf[i+0] + buf[i+1] + buf[i+2] + buf[i+3];
+                }
         }
 
         for (; i < len; i++) {
@@ -66,13 +59,7 @@ gf_rsync_weak_checksum (char *buf1, int32_t len)
  */
 
 void
-gf_rsync_strong_checksum (char *buf, int32_t len, uint8_t *sum)
+gf_rsync_strong_checksum (unsigned char *data, size_t len, unsigned char *md5)
 {
-        md_context m;
-
-        md5_begin (&m);
-        md5_update (&m, (unsigned char *) buf, len);
-        md5_result (&m, (unsigned char *) sum);
-
-        return;
+        MD5(data, len, md5);
 }
diff --git a/libglusterfs/src/checksum.h b/libglusterfs/src/checksum.h
index 3542036bf..bf7eeede8 100644
--- a/libglusterfs/src/checksum.h
+++ b/libglusterfs/src/checksum.h
@@ -1,27 +1,20 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __CHECKSUM_H__
 #define __CHECKSUM_H__
 
-uint32_t gf_rsync_weak_checksum (char *buf, int32_t len);
+uint32_t
+gf_rsync_weak_checksum (unsigned char *buf, size_t len);
 
-void gf_rsync_strong_checksum (char *buf, int32_t len, uint8_t *sum);
+void
+gf_rsync_strong_checksum (unsigned char *buf, size_t len, unsigned char *sum);
 
 #endif /* __CHECKSUM_H__ */
diff --git a/libglusterfs/src/circ-buff.c b/libglusterfs/src/circ-buff.c
new file mode 100644
index 000000000..484ce7dc9
--- /dev/null
+++ b/libglusterfs/src/circ-buff.c
@@ -0,0 +1,202 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "circ-buff.h"
+
+void
+cb_destroy_data (circular_buffer_t *cb,
+                 void (*destroy_buffer_data) (void *data))
+{
+        if (destroy_buffer_data)
+                destroy_buffer_data (cb->data);
+        GF_FREE (cb->data);
+        return;
+}
+
+
+/* hold lock while calling this function */
+int
+__cb_add_entry_buffer (buffer_t *buffer, void *item)
+{
+        circular_buffer_t   *ptr  = NULL;
+        int    ret   = -1;
+        //DO we really need the assert here?
+        GF_ASSERT (buffer->used_len <= buffer->size_buffer);
+
+        if (buffer->use_once == _gf_true &&
+            buffer->used_len == buffer->size_buffer) {
+                gf_log  ("", GF_LOG_WARNING, "buffer %p is use once buffer",
+                         buffer);
+                return -1;
+        } else {
+                if (buffer->used_len == buffer->size_buffer) {
+                        if (buffer->cb[buffer->w_index]) {
+                                ptr = buffer->cb[buffer->w_index];
+                                if (ptr->data) {
+                                        cb_destroy_data (ptr,
+                                                   buffer->destroy_buffer_data);
+                                        ptr->data = NULL;
+                                        GF_FREE (ptr);
+                                }
+                                buffer->cb[buffer->w_index] = NULL;
+                                ptr = NULL;
+                        }
+                }
+
+                buffer->cb[buffer->w_index] =
+                        GF_CALLOC (1, sizeof (circular_buffer_t),
+                                   gf_common_mt_circular_buffer_t);
+                if (!buffer->cb[buffer->w_index])
+                        return -1;
+
+                buffer->cb[buffer->w_index]->data = item;
+                ret = gettimeofday (&buffer->cb[buffer->w_index]->tv, NULL);
+                if (ret == -1)
+                        gf_log_callingfn ("", GF_LOG_WARNING, "getting time of"
+                                          "the day failed");
+                buffer->w_index++;
+                buffer->w_index %= buffer->size_buffer;
+                //used_buffer size cannot be greater than the total buffer size
+
+                if (buffer->used_len < buffer->size_buffer)
+                        buffer->used_len++;
+                return buffer->w_index;
+        }
+}
+
+int
+cb_add_entry_buffer (buffer_t *buffer, void *item)
+{
+        int write_index = -1;
+
+        pthread_mutex_lock (&buffer->lock);
+        {
+                write_index = __cb_add_entry_buffer (buffer, item);
+        }
+        pthread_mutex_unlock (&buffer->lock);
+
+        return write_index;
+}
+
+void
+cb_buffer_show (buffer_t *buffer)
+{
+        pthread_mutex_lock (&buffer->lock);
+        {
+                gf_log ("", GF_LOG_DEBUG, "w_index: %d, size: %"GF_PRI_SIZET
+                        " used_buffer: %d", buffer->w_index,
+                        buffer->size_buffer,
+                        buffer->used_len);
+        }
+        pthread_mutex_unlock (&buffer->lock);
+}
+
+void
+cb_buffer_dump (buffer_t *buffer, void *data,
+                int (fn) (circular_buffer_t *buffer, void *data))
+{
+        int index = 0;
+        circular_buffer_t *entry = NULL;
+        int  entries = 0;
+        int ul = 0;
+        int w_ind = 0;
+        int size_buff = 0;
+        int i = 0;
+
+        ul = buffer->used_len;
+        w_ind = buffer->w_index;
+        size_buff = buffer->size_buffer;
+
+        pthread_mutex_lock (&buffer->lock);
+        {
+                if (buffer->use_once == _gf_false) {
+                        index = (size_buff + (w_ind - ul))%size_buff;
+                        for (entries = 0; entries < buffer->used_len;
+                             entries++) {
+                                entry = buffer->cb[index];
+                                if (entry)
+                                        fn (entry, data);
+                                else
+                                        gf_log_callingfn ("", GF_LOG_WARNING,
+                                                          "Null entry in "
+                                                          "circular buffer at "
+                                                          "index %d.", index);
+
+                                index++;
+                                index %= buffer->size_buffer;
+                        }
+                } else {
+                        for (i = 0; i < buffer->used_len ; i++) {
+                                entry = buffer->cb[i];
+                                fn (entry, data);
+                        }
+                }
+        }
+        pthread_mutex_unlock (&buffer->lock);
+}
+
+buffer_t *
+cb_buffer_new (size_t buffer_size, gf_boolean_t use_once,
+               void (*destroy_buffer_data) (void *data))
+{
+        buffer_t    *buffer = NULL;
+
+        buffer = GF_CALLOC (1, sizeof (*buffer), gf_common_mt_buffer_t);
+        if (!buffer) {
+                gf_log ("", GF_LOG_ERROR, "could not allocate the "
+                        "buffer");
+                goto out;
+        }
+
+        buffer->cb = GF_CALLOC (buffer_size,
+                                sizeof (circular_buffer_t *),
+                                gf_common_mt_circular_buffer_t);
+        if (!buffer->cb) {
+                gf_log ("", GF_LOG_ERROR, "could not allocate the "
+                        "memory for the circular buffer");
+                GF_FREE (buffer);
+                buffer = NULL;
+                goto out;
+        }
+
+        buffer->w_index = 0;
+        buffer->size_buffer = buffer_size;
+        buffer->use_once = use_once;
+        buffer->used_len = 0;
+        buffer->destroy_buffer_data = destroy_buffer_data;
+        pthread_mutex_init (&buffer->lock, NULL);
+
+out:
+        return buffer;
+}
+
+void
+cb_buffer_destroy (buffer_t *buffer)
+{
+        int                     i = 0;
+        circular_buffer_t       *ptr = NULL;
+        if (buffer) {
+                if (buffer->cb) {
+                        for (i = 0; i < buffer->used_len ; i++) {
+                                ptr = buffer->cb[i];
+                                if (ptr->data) {
+                                        cb_destroy_data (ptr,
+                                                   buffer->destroy_buffer_data);
+                                        ptr->data = NULL;
+                                        GF_FREE (ptr);
+                                }
+                        }
+                        GF_FREE (buffer->cb);
+                }
+                pthread_mutex_destroy (&buffer->lock);
+                GF_FREE (buffer);
+        }
+}
+
diff --git a/libglusterfs/src/circ-buff.h b/libglusterfs/src/circ-buff.h
new file mode 100644
index 000000000..e3459f5e3
--- /dev/null
+++ b/libglusterfs/src/circ-buff.h
@@ -0,0 +1,64 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CB_H
+#define _CB_H
+
+#include "common-utils.h"
+#include "logging.h"
+#include "mem-types.h"
+
+#define BUFFER_SIZE 10
+#define TOTAL_SIZE BUFFER_SIZE + 1
+
+
+struct _circular_buffer {
+        struct timeval tv;
+        void *data;
+};
+
+typedef struct _circular_buffer circular_buffer_t;
+
+struct _buffer {
+        unsigned int w_index;
+        size_t  size_buffer;
+        gf_boolean_t use_once;
+        /* This variable is assigned the proper value at the time of initing */
+        /* the buffer. It indicates, whether the buffer should be used once */
+        /*  it becomes full. */
+
+        int  used_len;
+        /* indicates the amount of circular buffer used. */
+
+        circular_buffer_t **cb;
+        void (*destroy_buffer_data) (void *data);
+        pthread_mutex_t   lock;
+};
+
+typedef struct _buffer buffer_t;
+
+int
+cb_add_entry_buffer (buffer_t *buffer, void *item);
+
+void
+cb_buffer_show (buffer_t *buffer);
+
+buffer_t *
+cb_buffer_new (size_t buffer_size,gf_boolean_t use_buffer_once,
+               void (*destroy_data) (void *data));
+
+void
+cb_buffer_destroy (buffer_t *buffer);
+
+void
+cb_buffer_dump (buffer_t *buffer, void *data,
+                int (fn) (circular_buffer_t *buffer, void *data));
+
+#endif /* _CB_H */
diff --git a/libglusterfs/src/client_t.c b/libglusterfs/src/client_t.c
new file mode 100644
index 000000000..06447dc5d
--- /dev/null
+++ b/libglusterfs/src/client_t.c
@@ -0,0 +1,890 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "glusterfs.h"
+#include "dict.h"
+#include "statedump.h"
+#include "client_t.h"
+#include "list.h"
+#include "rpcsvc.h"
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+
+static int
+gf_client_chain_client_entries (cliententry_t *entries, uint32_t startidx,
+                        uint32_t endcount)
+{
+        uint32_t        i = 0;
+
+        if (!entries) {
+                gf_log_callingfn ("client_t", GF_LOG_WARNING, "!entries");
+                return -1;
+        }
+
+        /* Chain only till the second to last entry because we want to
+         * ensure that the last entry has GF_CLIENTTABLE_END.
+         */
+        for (i = startidx; i < (endcount - 1); i++)
+                entries[i].next_free = i + 1;
+
+        /* i has already been incremented upto the last entry. */
+        entries[i].next_free = GF_CLIENTTABLE_END;
+
+        return 0;
+}
+
+
+static int
+gf_client_clienttable_expand (clienttable_t *clienttable, uint32_t nr)
+{
+        cliententry_t   *oldclients     = NULL;
+        uint32_t         oldmax_clients = -1;
+        int              ret            = -1;
+
+        if (clienttable == NULL || nr < 0) {
+                gf_log_callingfn ("client_t", GF_LOG_ERROR, "invalid argument");
+                ret = EINVAL;
+                goto out;
+        }
+
+        /* expand size by power-of-two...
+           this originally came from .../xlators/protocol/server/src/server.c
+           where it was not commented */
+        nr /= (1024 / sizeof (cliententry_t));
+        nr = gf_roundup_next_power_of_two (nr + 1);
+        nr *= (1024 / sizeof (cliententry_t));
+
+        oldclients = clienttable->cliententries;
+        oldmax_clients = clienttable->max_clients;
+
+        clienttable->cliententries = GF_CALLOC (nr, sizeof (cliententry_t),
+                                                gf_common_mt_cliententry_t);
+        if (!clienttable->cliententries) {
+                ret = ENOMEM;
+                goto out;
+        }
+        clienttable->max_clients = nr;
+
+        if (oldclients) {
+                uint32_t cpy = oldmax_clients * sizeof (cliententry_t);
+                memcpy (clienttable->cliententries, oldclients, cpy);
+        }
+
+        gf_client_chain_client_entries (clienttable->cliententries, oldmax_clients,
+                                clienttable->max_clients);
+
+        /* Now that expansion is done, we must update the client list
+         * head pointer so that the client allocation functions can continue
+         * using the expanded table.
+         */
+        clienttable->first_free = oldmax_clients;
+        GF_FREE (oldclients);
+        ret = 0;
+out:
+        return ret;
+}
+
+
+clienttable_t *
+gf_clienttable_alloc (void)
+{
+        clienttable_t *clienttable = NULL;
+
+        clienttable =
+                GF_CALLOC (1, sizeof (clienttable_t), gf_common_mt_clienttable_t);
+        if (!clienttable)
+                return NULL;
+
+        LOCK_INIT (&clienttable->lock);
+        gf_client_clienttable_expand (clienttable, GF_CLIENTTABLE_INITIAL_SIZE);
+        return clienttable;
+}
+
+
+void
+gf_client_clienttable_destroy (clienttable_t *clienttable)
+{
+        client_t         *client        = NULL;
+        cliententry_t    *cliententries = NULL;
+        uint32_t          client_count  = 0;
+        int32_t           i             = 0;
+
+        if (!clienttable) {
+                gf_log_callingfn ("client_t", GF_LOG_WARNING, "!clienttable");
+                return;
+        }
+
+        LOCK (&clienttable->lock);
+        {
+                client_count = clienttable->max_clients;
+                clienttable->max_clients = 0;
+                cliententries = clienttable->cliententries;
+                clienttable->cliententries = NULL;
+        }
+        UNLOCK (&clienttable->lock);
+
+        if (cliententries != NULL) {
+                for (i = 0; i < client_count; i++) {
+                        client = cliententries[i].client;
+                        if (client != NULL) {
+                                gf_client_unref (client);
+                        }
+                }
+
+                GF_FREE (cliententries);
+                LOCK_DESTROY (&clienttable->lock);
+                GF_FREE (clienttable);
+        }
+}
+
+client_t *
+gf_client_get (xlator_t *this, struct rpcsvc_auth_data *cred, char *client_uid)
+{
+        client_t      *client      = NULL;
+        cliententry_t *cliententry = NULL;
+        clienttable_t *clienttable = NULL;
+        unsigned int   i           = 0;
+
+        if (this == NULL || client_uid == NULL) {
+                gf_log_callingfn ("client_t", GF_LOG_ERROR, "invalid argument");
+                errno = EINVAL;
+                return NULL;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "client_uid=%s", client_uid);
+
+        clienttable = this->ctx->clienttable;
+
+        LOCK (&clienttable->lock);
+        {
+                for (; i < clienttable->max_clients; i++) {
+                        client = clienttable->cliententries[i].client;
+                        if (client == NULL)
+                                continue;
+                        /*
+                         * look for matching client_uid, _and_
+                         * if auth was used, matching auth flavour and data
+                         */
+                        if (strcmp (client_uid, client->client_uid) == 0 &&
+                                (cred->flavour != AUTH_NONE &&
+                                        (cred->flavour == client->auth.flavour &&
+                                        (size_t) cred->datalen == client->auth.len &&
+                                        memcmp (cred->authdata,
+                                                client->auth.data,
+                                                client->auth.len) == 0))) {
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+                                __sync_add_and_fetch(&client->ref.bind, 1);
+#else
+                                LOCK (&client->ref.lock);
+                                {
+                                        ++client->ref.bind;
+                                }
+                                UNLOCK (&client->ref.lock);
+#endif
+                                break;
+                        }
+                }
+                if (client) {
+                        gf_client_ref (client);
+                        goto unlock;
+                }
+                client = GF_CALLOC (1, sizeof(client_t), gf_common_mt_client_t);
+                if (client == NULL) {
+                        errno = ENOMEM;
+                        goto unlock;
+                }
+
+                client->this = this;
+
+                LOCK_INIT (&client->scratch_ctx.lock);
+                LOCK_INIT (&client->ref.lock);
+
+                client->client_uid = gf_strdup (client_uid);
+                if (client->client_uid == NULL) {
+                        GF_FREE (client);
+                        client = NULL;
+                        errno = ENOMEM;
+                        goto unlock;
+                }
+                client->scratch_ctx.count = GF_CLIENTCTX_INITIAL_SIZE;
+                client->scratch_ctx.ctx =
+                        GF_CALLOC (GF_CLIENTCTX_INITIAL_SIZE,
+                                   sizeof (struct client_ctx),
+                                   gf_common_mt_client_ctx);
+                if (client->scratch_ctx.ctx == NULL) {
+                        GF_FREE (client->client_uid);
+                        GF_FREE (client);
+                        client = NULL;
+                        errno = ENOMEM;
+                        goto unlock;
+                }
+
+                /* no need to do these atomically here */
+                client->ref.bind = client->ref.count = 1;
+
+                client->auth.flavour = cred->flavour;
+                if (cred->flavour != AUTH_NONE) {
+                        client->auth.data =
+                                GF_CALLOC (1, cred->datalen,
+                                           gf_common_mt_client_t);
+                        if (client->auth.data == NULL) {
+                                GF_FREE (client->scratch_ctx.ctx);
+                                GF_FREE (client->client_uid);
+                                GF_FREE (client);
+                                client = NULL;
+                                errno = ENOMEM;
+                                goto unlock;
+                        }
+                        memcpy (client->auth.data, cred->authdata,
+                                cred->datalen);
+                        client->auth.len = cred->datalen;
+                }
+
+                client->tbl_index = clienttable->first_free;
+                cliententry = &clienttable->cliententries[client->tbl_index];
+                cliententry->client = client;
+                clienttable->first_free = cliententry->next_free;
+                cliententry->next_free = GF_CLIENTENTRY_ALLOCATED;
+                gf_client_ref (client);
+        }
+unlock:
+        UNLOCK (&clienttable->lock);
+
+        return client;
+}
+
+void
+gf_client_put (client_t *client, gf_boolean_t *detached)
+{
+        gf_boolean_t unref = _gf_false;
+        int bind_ref;
+
+        if (detached)
+                *detached = _gf_false;
+
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+        bind_ref = __sync_sub_and_fetch(&client->ref.bind, 1);
+#else
+        LOCK (&client->ref.lock);
+        {
+                bind_ref = --client->ref.bind;
+        }
+        UNLOCK (&client->ref.lock);
+#endif
+        if (bind_ref == 0)
+                unref = _gf_true;
+
+        if (unref) {
+                gf_log (THIS->name, GF_LOG_INFO, "Shutting down connection %s",
+                        client->client_uid);
+                if (detached)
+                        *detached = _gf_true;
+                gf_client_unref (client);
+        }
+}
+
+
+client_t *
+gf_client_ref (client_t *client)
+{
+        if (!client) {
+                gf_log_callingfn ("client_t", GF_LOG_ERROR, "null client");
+                return NULL;
+        }
+
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+        __sync_add_and_fetch(&client->ref.count, 1);
+#else
+        LOCK (&client->ref.lock);
+        {
+                ++client->ref.count;
+        }
+        UNLOCK (&client->ref.lock);
+#endif
+        return client;
+}
+
+
+static void
+client_destroy (client_t *client)
+{
+        clienttable_t     *clienttable = NULL;
+        glusterfs_graph_t *gtrav       = NULL;
+        xlator_t          *xtrav       = NULL;
+
+        if (client == NULL){
+                gf_log_callingfn ("xlator", GF_LOG_ERROR, "invalid argument");
+                goto out;
+        }
+
+        clienttable = client->this->ctx->clienttable;
+
+        LOCK_DESTROY (&client->scratch_ctx.lock);
+        LOCK_DESTROY (&client->ref.lock);
+
+        LOCK (&clienttable->lock);
+        {
+                clienttable->cliententries[client->tbl_index].client = NULL;
+                clienttable->cliententries[client->tbl_index].next_free =
+                        clienttable->first_free;
+                clienttable->first_free = client->tbl_index;
+        }
+        UNLOCK (&clienttable->lock);
+
+        list_for_each_entry (gtrav, &client->this->ctx->graphs, list) {
+                xtrav = gtrav->top;
+                while (xtrav != NULL) {
+                        if (xtrav->cbks->client_destroy != NULL)
+                                xtrav->cbks->client_destroy (xtrav, client);
+                        xtrav = xtrav->next;
+                }
+        }
+        GF_FREE (client->auth.data);
+        GF_FREE (client->scratch_ctx.ctx);
+        GF_FREE (client->client_uid);
+        GF_FREE (client);
+out:
+        return;
+}
+
+
+int
+gf_client_disconnect (client_t *client)
+{
+        int                ret   = 0;
+        glusterfs_graph_t *gtrav = NULL;
+        xlator_t          *xtrav = NULL;
+
+        list_for_each_entry (gtrav, &client->this->ctx->graphs, list) {
+                xtrav = gtrav->top;
+                while (xtrav != NULL) {
+                        if (xtrav->cbks->client_disconnect != NULL)
+                                if (xtrav->cbks->client_disconnect (xtrav, client) != 0)
+                                        ret = -1;
+                        xtrav = xtrav->next;
+                }
+        }
+
+        return ret;
+}
+
+
+void
+gf_client_unref (client_t *client)
+{
+        int refcount;
+
+        if (!client) {
+                gf_log_callingfn ("client_t", GF_LOG_ERROR, "client is NULL");
+                return;
+        }
+
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+        refcount = __sync_sub_and_fetch(&client->ref.count, 1);
+#else
+        LOCK (&client->ref.lock);
+        {
+                refcount = --client->ref.count;
+        }
+        UNLOCK (&client->ref.lock);
+#endif
+        if (refcount == 0) {
+                client_destroy (client);
+        }
+}
+
+
+static int
+client_ctx_set_int (client_t *client, void *key, void *value)
+{
+        int index   = 0;
+        int ret     = 0;
+        int set_idx = -1;
+
+        for (index = 0; index < client->scratch_ctx.count; index++) {
+                if (!client->scratch_ctx.ctx[index].ctx_key) {
+                        if (set_idx == -1)
+                                set_idx = index;
+                        /* dont break, to check if key already exists
+                           further on */
+                }
+                if (client->scratch_ctx.ctx[index].ctx_key == key) {
+                        set_idx = index;
+                        break;
+                }
+        }
+
+        if (set_idx == -1) {
+                ret = -1;
+                goto out;
+        }
+
+        client->scratch_ctx.ctx[set_idx].ctx_key = key;
+        client->scratch_ctx.ctx[set_idx].ctx_value  = value;
+
+out:
+        return ret;
+}
+
+
+int
+client_ctx_set (client_t *client, void *key, void *value)
+{
+        int ret = 0;
+
+        if (!client || !key)
+                return -1;
+
+        LOCK (&client->scratch_ctx.lock);
+        {
+                ret = client_ctx_set_int (client, key, value);
+        }
+        UNLOCK (&client->scratch_ctx.lock);
+
+        return ret;
+}
+
+
+static int
+client_ctx_get_int (client_t *client, void *key, void **value)
+{
+        int index = 0;
+        int ret   = 0;
+
+        for (index = 0; index < client->scratch_ctx.count; index++) {
+                if (client->scratch_ctx.ctx[index].ctx_key == key)
+                        break;
+        }
+
+        if (index == client->scratch_ctx.count) {
+                ret = -1;
+                goto out;
+        }
+
+        if (value)
+                *value = client->scratch_ctx.ctx[index].ctx_value;
+
+out:
+        return ret;
+}
+
+
+int
+client_ctx_get (client_t *client, void *key, void **value)
+{
+        int ret = 0;
+
+        if (!client || !key)
+                return -1;
+
+        LOCK (&client->scratch_ctx.lock);
+        {
+                ret = client_ctx_get_int (client, key, value);
+        }
+        UNLOCK (&client->scratch_ctx.lock);
+
+        return ret;
+}
+
+
+static int
+client_ctx_del_int (client_t *client, void *key, void **value)
+{
+        int index = 0;
+        int ret   = 0;
+
+        for (index = 0; index < client->scratch_ctx.count; index++) {
+                if (client->scratch_ctx.ctx[index].ctx_key == key)
+                        break;
+        }
+
+        if (index == client->scratch_ctx.count) {
+                ret = -1;
+                goto out;
+        }
+
+        if (value)
+                *value = client->scratch_ctx.ctx[index].ctx_value;
+
+        client->scratch_ctx.ctx[index].ctx_key   = 0;
+        client->scratch_ctx.ctx[index].ctx_value = 0;
+
+out:
+        return ret;
+}
+
+
+int
+client_ctx_del (client_t *client, void *key, void **value)
+{
+        int ret = 0;
+
+        if (!client || !key)
+                return -1;
+
+        LOCK (&client->scratch_ctx.lock);
+        {
+                ret = client_ctx_del_int (client, key, value);
+        }
+        UNLOCK (&client->scratch_ctx.lock);
+
+        return ret;
+}
+
+
+void
+client_dump (client_t *client, char *prefix)
+{
+        char key[GF_DUMP_MAX_BUF_LEN];
+
+        if (!client)
+                return;
+
+        memset(key, 0, sizeof key);
+        gf_proc_dump_write("refcount", "%d", client->ref.count);
+}
+
+
+void
+cliententry_dump (cliententry_t *cliententry, char *prefix)
+{
+        if (!cliententry)
+                return;
+
+        if (GF_CLIENTENTRY_ALLOCATED != cliententry->next_free)
+                return;
+
+        if (cliententry->client)
+                client_dump(cliententry->client, prefix);
+}
+
+
+void
+clienttable_dump (clienttable_t *clienttable, char *prefix)
+{
+        int     i   = 0;
+        int     ret = -1;
+        char    key[GF_DUMP_MAX_BUF_LEN] = {0};
+
+        if (!clienttable)
+                return;
+
+        ret = TRY_LOCK (&clienttable->lock);
+        {
+                if (ret) {
+                        gf_log ("client_t", GF_LOG_WARNING,
+                                "Unable to acquire lock");
+                        return;
+                }
+                memset(key, 0, sizeof key);
+                gf_proc_dump_build_key(key, prefix, "maxclients");
+                gf_proc_dump_write(key, "%d", clienttable->max_clients);
+                gf_proc_dump_build_key(key, prefix, "first_free");
+                gf_proc_dump_write(key, "%d", clienttable->first_free);
+                for ( i = 0 ; i < clienttable->max_clients; i++) {
+                        if (GF_CLIENTENTRY_ALLOCATED ==
+                            clienttable->cliententries[i].next_free) {
+                                gf_proc_dump_build_key(key, prefix,
+                                                       "cliententry[%d]", i);
+                                gf_proc_dump_add_section(key);
+                                cliententry_dump(&clienttable->cliententries[i],
+                                                 key);
+                        }
+                }
+        }
+        UNLOCK(&clienttable->lock);
+}
+
+
+void
+client_ctx_dump (client_t *client, char *prefix)
+{
+#if 0 /* TBD, FIXME */
+        struct client_ctx   *client_ctx = NULL;
+        xlator_t            *xl = NULL;
+        int                  i = 0;
+
+        if ((client == NULL) || (client->ctx == NULL)) {
+                goto out;
+        }
+
+        LOCK (&client->ctx_lock);
+        if (client->ctx != NULL) {
+                client_ctx = GF_CALLOC (client->inode->table->xl->graph->ctx_count,
+                                        sizeof (*client_ctx),
+                                        gf_common_mt_client_ctx);
+                if (client_ctx == NULL) {
+                        goto unlock;
+                }
+
+                for (i = 0; i < client->inode->table->xl->graph->ctx_count; i++) {
+                        client_ctx[i] = client->ctx[i];
+                }
+        }
+unlock:
+        UNLOCK (&client->ctx_lock);
+
+        if (client_ctx == NULL) {
+                goto out;
+        }
+
+        for (i = 0; i < client->inode->table->xl->graph->ctx_count; i++) {
+                if (client_ctx[i].xl_key) {
+                        xl = (xlator_t *)(long)client_ctx[i].xl_key;
+                        if (xl->dumpops && xl->dumpops->clientctx)
+                                xl->dumpops->clientctx (xl, client);
+                }
+        }
+out:
+        GF_FREE (client_ctx);
+#endif
+}
+
+
+/*
+ * the following functions are here to preserve legacy behavior of the
+ * protocol/server xlator dump, but perhaps they should just be folded
+ * into the client dump instead?
+ */
+int
+gf_client_dump_fdtables_to_dict (xlator_t *this, dict_t *dict)
+{
+        clienttable_t  *clienttable = NULL;
+        int             count       = 0;
+        int             ret         = -1;
+#ifdef NOTYET
+        client_t       *client      = NULL;
+        char            key[GF_DUMP_MAX_BUF_LEN] = {0,};
+#endif
+
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+
+        clienttable = this->ctx->clienttable;
+
+        if (!clienttable)
+                return -1;
+
+#ifdef NOTYET
+        ret = TRY_LOCK (&clienttable->lock);
+        {
+                if (ret) {
+                        gf_log ("client_t", GF_LOG_WARNING,
+                                "Unable to acquire lock");
+                        return -1;
+                }
+                for ( ; count < clienttable->max_clients; count++) {
+                        if (GF_CLIENTENTRY_ALLOCATED !=
+                            clienttable->cliententries[count].next_free)
+                                continue;
+                        client = clienttable->cliententries[count].client;
+                        memset(key, 0, sizeof key);
+                        snprintf (key, sizeof key, "conn%d", count++);
+                        fdtable_dump_to_dict (client->server_ctx.fdtable,
+                                              key, dict);
+                }
+        }
+        UNLOCK(&clienttable->lock);
+#endif
+
+        ret = dict_set_int32 (dict, "conncount", count);
+out:
+        return ret;
+}
+
+int
+gf_client_dump_fdtables (xlator_t *this)
+{
+        client_t       *client = NULL;
+        clienttable_t  *clienttable = NULL;
+        int             count = 1;
+        int             ret = -1;
+        char            key[GF_DUMP_MAX_BUF_LEN] = {0,};
+
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+
+        clienttable = this->ctx->clienttable;
+
+        if (!clienttable)
+                return -1;
+
+        ret = TRY_LOCK (&clienttable->lock);
+        {
+                if (ret) {
+                        gf_log ("client_t", GF_LOG_WARNING,
+                                "Unable to acquire lock");
+                        return -1;
+                }
+
+
+                for ( ; count < clienttable->max_clients; count++) {
+                        if (GF_CLIENTENTRY_ALLOCATED !=
+                            clienttable->cliententries[count].next_free)
+                                continue;
+                        client = clienttable->cliententries[count].client;
+                        memset(key, 0, sizeof key);
+                        if (client->client_uid) {
+                                gf_proc_dump_build_key (key, "conn",
+                                                        "%d.id", count);
+                                gf_proc_dump_write (key, "%s",
+                                                    client->client_uid);
+                        }
+
+                        gf_proc_dump_build_key (key, "conn", "%d.ref",
+                                                        count);
+                        gf_proc_dump_write (key, "%d", client->ref.count);
+                        if (client->bound_xl) {
+                                gf_proc_dump_build_key (key, "conn",
+                                                        "%d.bound_xl", count);
+                                gf_proc_dump_write (key, "%s",
+                                                    client->bound_xl->name);
+                        }
+
+#ifdef NOTYET
+                        gf_proc_dump_build_key (key, "conn","%d.id", count);
+                        fdtable_dump (client->server_ctx.fdtable, key);
+#endif
+                }
+        }
+
+        UNLOCK(&clienttable->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+gf_client_dump_inodes_to_dict (xlator_t *this, dict_t *dict)
+{
+        client_t       *client        = NULL;
+        clienttable_t  *clienttable   = NULL;
+        xlator_t       *prev_bound_xl = NULL;
+        char            key[32]       = {0,};
+        int             count         = 0;
+        int             ret           = -1;
+
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+
+        clienttable = this->ctx->clienttable;
+
+        if (!clienttable)
+                return -1;
+
+        ret = TRY_LOCK (&clienttable->lock);
+        {
+                if (ret) {
+                        gf_log ("client_t", GF_LOG_WARNING,
+                                "Unable to acquire lock");
+                        return -1;
+                }
+                for ( ; count < clienttable->max_clients; count++) {
+                        if (GF_CLIENTENTRY_ALLOCATED !=
+                            clienttable->cliententries[count].next_free)
+                                continue;
+                        client = clienttable->cliententries[count].client;
+                        memset(key, 0, sizeof key);
+                        if (client->bound_xl && client->bound_xl->itable) {
+                                /* Presently every brick contains only
+                                 * one bound_xl for all connections.
+                                 * This will lead to duplicating of
+                                 *  the inode lists, if listing is
+                                 * done for every connection. This
+                                 * simple check prevents duplication
+                                 * in the present case. If need arises
+                                 * the check can be improved.
+                                 */
+                                if (client->bound_xl == prev_bound_xl)
+                                        continue;
+                                prev_bound_xl = client->bound_xl;
+
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key), "conn%d", count);
+                                inode_table_dump_to_dict (client->bound_xl->itable,
+                                                          key, dict);
+                        }
+                }
+        }
+        UNLOCK(&clienttable->lock);
+
+        ret = dict_set_int32 (dict, "conncount", count);
+
+out:
+        if (prev_bound_xl)
+                prev_bound_xl = NULL;
+        return ret;
+}
+
+int
+gf_client_dump_inodes (xlator_t *this)
+{
+        client_t       *client        = NULL;
+        clienttable_t  *clienttable   = NULL;
+        xlator_t       *prev_bound_xl = NULL;
+        int             count         = 1;
+        int             ret           = -1;
+        char            key[GF_DUMP_MAX_BUF_LEN] = {0,};
+
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+
+        clienttable = this->ctx->clienttable;
+
+        if (!clienttable)
+                goto out;
+
+        ret = TRY_LOCK (&clienttable->lock);
+        {
+                if (ret) {
+                        gf_log ("client_t", GF_LOG_WARNING,
+                                "Unable to acquire lock");
+                        goto out;
+                }
+
+                for ( ; count < clienttable->max_clients; count++) {
+                        if (GF_CLIENTENTRY_ALLOCATED !=
+                            clienttable->cliententries[count].next_free)
+                                continue;
+                        client = clienttable->cliententries[count].client;
+                        memset(key, 0, sizeof key);
+                        if (client->bound_xl && client->bound_xl->itable) {
+                                /* Presently every brick contains only
+                                 * one bound_xl for all connections.
+                                 * This will lead to duplicating of
+                                 * the inode lists, if listing is
+                                 * done for every connection. This
+                                 * simple check prevents duplication
+                                 * in the present case. If need arises
+                                 * the check can be improved.
+                                 */
+                                if (client->bound_xl == prev_bound_xl)
+                                        continue;
+                                prev_bound_xl = client->bound_xl;
+
+                                gf_proc_dump_build_key(key, "conn",
+                                                       "%d.bound_xl.%s", count,
+                                                       client->bound_xl->name);
+                                inode_table_dump(client->bound_xl->itable,key);
+                        }
+                }
+        }
+        UNLOCK(&clienttable->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
diff --git a/libglusterfs/src/client_t.h b/libglusterfs/src/client_t.h
new file mode 100644
index 000000000..f7812f8f0
--- /dev/null
+++ b/libglusterfs/src/client_t.h
@@ -0,0 +1,135 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CLIENT_T_H
+#define _CLIENT_T_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "locking.h"  /* for gf_lock_t, not included by glusterfs.h */
+
+struct client_ctx {
+        void     *ctx_key;
+        void     *ctx_value;
+};
+
+typedef struct _client_t {
+        struct {
+                /* e.g. protocol/server stashes its ctx here */
+                gf_lock_t            lock;
+                unsigned short       count;
+                struct client_ctx   *ctx;
+        }            scratch_ctx;
+        struct {
+                gf_lock_t            lock;
+                volatile int         bind;
+                volatile int         count;
+        }            ref;
+        xlator_t    *bound_xl;
+        xlator_t    *this;
+        int          tbl_index;
+        char        *client_uid;
+        struct {
+                int                  flavour;
+                size_t               len;
+                char                *data;
+        }            auth;
+} client_t;
+
+#define GF_CLIENTCTX_INITIAL_SIZE 8
+
+struct client_table_entry {
+        client_t            *client;
+        int                  next_free;
+};
+typedef struct client_table_entry cliententry_t;
+
+struct clienttable {
+        unsigned int         max_clients;
+        gf_lock_t            lock;
+        cliententry_t       *cliententries;
+        int                  first_free;
+};
+typedef struct clienttable clienttable_t;
+
+#define GF_CLIENTTABLE_INITIAL_SIZE 32
+
+/* Signifies no more entries in the client table. */
+#define GF_CLIENTTABLE_END  -1
+
+/* This is used to invalidate
+ * the next_free value in an cliententry that has been allocated
+ */
+#define GF_CLIENTENTRY_ALLOCATED    -2
+
+struct rpcsvc_auth_data;
+
+client_t *
+gf_client_get (xlator_t *this, struct rpcsvc_auth_data *cred, char *client_uid);
+
+void
+gf_client_put (client_t *client, gf_boolean_t *detached);
+
+clienttable_t *
+gf_clienttable_alloc (void);
+
+void
+gf_client_clienttable_destroy (clienttable_t *clienttable);
+
+client_t *
+gf_client_ref (client_t *client);
+
+void
+gf_client_unref (client_t *client);
+
+int
+gf_client_dump_fdtable_to_dict (xlator_t *this, dict_t *dict);
+
+int
+gf_client_dump_fdtable (xlator_t *this);
+
+int
+gf_client_dump_inodes_to_dict (xlator_t *this, dict_t *dict);
+
+int
+gf_client_dump_inodes (xlator_t *this);
+
+int
+client_ctx_set (client_t *client, void *key, void *value);
+
+int
+client_ctx_get (client_t *client, void *key, void **value);
+
+int
+client_ctx_del (client_t *client, void *key, void **value);
+
+void
+client_ctx_dump (client_t *client, char *prefix);
+
+int
+gf_client_dump_fdtables_to_dict (xlator_t *this, dict_t *dict);
+
+int
+gf_client_dump_fdtables (xlator_t *this);
+
+int
+gf_client_dump_inodes_to_dict (xlator_t *this, dict_t *dict);
+
+int
+gf_client_dump_inodes (xlator_t *this);
+
+int
+gf_client_disconnect (client_t *client);
+
+#endif /* _CLIENT_T_H */
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 068bd8460..827475282 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -52,8 +43,9 @@
 #include "glusterfs.h"
 #include "stack.h"
 #include "globals.h"
-#include "md5.h"
 #include "lkowner.h"
+#include "syscall.h"
+#include <ifaddrs.h>
 
 #ifndef AI_ADDRCONFIG
 #define AI_ADDRCONFIG 0
@@ -67,6 +59,99 @@ struct dnscache6 {
         struct addrinfo *next;
 };
 
+void
+md5_wrapper(const unsigned char *data, size_t len, char *md5)
+{
+        unsigned short i = 0;
+        unsigned short lim = MD5_DIGEST_LENGTH*2+1;
+        unsigned char scratch[MD5_DIGEST_LENGTH] = {0,};
+        MD5(data, len, scratch);
+        for (; i < MD5_DIGEST_LENGTH; i++)
+                snprintf(md5 + i * 2, lim-i*2, "%02x", scratch[i]);
+}
+
+/* works similar to mkdir(1) -p.
+ */
+int
+mkdir_p (char *path, mode_t mode, gf_boolean_t allow_symlinks)
+{
+        int             i               = 0;
+        int             ret             = -1;
+        char            dir[PATH_MAX]   = {0,};
+        struct stat     stbuf           = {0,};
+
+        strcpy (dir, path);
+        i = (dir[0] == '/')? 1: 0;
+        do {
+                if (path[i] != '/' && path[i] != '\0')
+                        continue;
+
+                dir[i] = '\0';
+                ret = mkdir (dir, mode);
+                if (ret && errno != EEXIST) {
+                        gf_log ("", GF_LOG_ERROR, "Failed due to reason %s",
+                                strerror (errno));
+                        goto out;
+                }
+
+                if (ret && errno == EEXIST && !allow_symlinks) {
+                        ret = lstat (dir, &stbuf);
+                        if (ret)
+                                goto out;
+
+                        if (S_ISLNK (stbuf.st_mode)) {
+                                ret = -1;
+                                gf_log ("", GF_LOG_ERROR, "%s is a symlink",
+                                        dir);
+                                goto out;
+                        }
+                }
+                dir[i] = '/';
+
+        } while (path[i++] != '\0');
+
+        ret = stat (dir, &stbuf);
+        if (ret || !S_ISDIR (stbuf.st_mode)) {
+                ret = -1;
+                gf_log ("", GF_LOG_ERROR, "Failed to create directory, "
+                        "possibly some of the components were not directories");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+
+int
+gf_lstat_dir (const char *path, struct stat *stbuf_in)
+{
+        int ret           = -1;
+        struct stat stbuf = {0,};
+
+        if (path == NULL) {
+                errno = EINVAL;
+                goto out;
+        }
+
+        ret = sys_lstat (path, &stbuf);
+        if (ret)
+                goto out;
+
+        if (!S_ISDIR (stbuf.st_mode)) {
+                errno = ENOTDIR;
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+
+out:
+        if (!ret && stbuf_in)
+                *stbuf_in = stbuf;
+
+        return ret;
+}
 
 int
 log_base2 (unsigned long x)
@@ -123,7 +208,9 @@ gf_resolve_ip6 (const char *hostname,
                 memset(&hints, 0, sizeof(hints));
                 hints.ai_family   = family;
                 hints.ai_socktype = SOCK_STREAM;
+#ifndef __NetBSD__
                 hints.ai_flags    = AI_ADDRCONFIG;
+#endif
 
                 ret = gf_asprintf (&port_str, "%d", port);
                 if (-1 == ret) {
@@ -192,28 +279,101 @@ err:
 }
 
 
+struct xldump {
+	int lineno;
+	FILE *logfp;
+};
+
+
+static int
+nprintf (struct xldump *dump, const char *fmt, ...)
+{
+	va_list ap;
+	int ret = 0;
+
+
+	ret += fprintf (dump->logfp, "%3d: ", ++dump->lineno);
+
+	va_start (ap, fmt);
+	ret += vfprintf (dump->logfp, fmt, ap);
+	va_end (ap);
+
+	ret += fprintf (dump->logfp, "\n");
+
+	return ret;
+}
+
+
+static int
+xldump_options (dict_t *this, char *key, data_t *value,	void *d)
+{
+	nprintf (d, "    option %s %s", key, value->data);
+	return 0;
+}
+
+
+static void
+xldump_subvolumes (xlator_t *this, void *d)
+{
+	xlator_list_t *subv = NULL;
+	int len = 0;
+	char *subvstr = NULL;
+
+	subv = this->children;
+	if (!this->children)
+		return;
+
+	for (subv = this->children; subv; subv = subv->next)
+		len += (strlen (subv->xlator->name) + 1);
+
+	subvstr = GF_CALLOC (1, len, gf_common_mt_strdup);
+
+	len = 0;
+	for (subv = this->children; subv; subv= subv->next)
+		len += sprintf (subvstr + len, "%s%s", subv->xlator->name,
+				subv->next ? " " : "");
+
+	nprintf (d, "    subvolumes %s", subvstr);
+
+	GF_FREE (subvstr);
+}
+
+
+static void
+xldump (xlator_t *each, void *d)
+{
+	nprintf (d, "volume %s", each->name);
+	nprintf (d, "    type %s", each->type);
+	dict_foreach (each->options, xldump_options, d);
+
+	xldump_subvolumes (each, d);
+
+	nprintf (d, "end-volume");
+	nprintf (d, "");
+}
+
+
 void
-gf_log_volume_file (FILE *specfp)
+gf_log_dump_graph (FILE *specfp, glusterfs_graph_t *graph)
 {
-        extern FILE *gf_log_logfile;
-        int          lcount = 0;
-        char         data[GF_UNIT_KB];
+        glusterfs_ctx_t *ctx;
+	struct xldump xld = {0, };
+
 
-        fseek (specfp, 0L, SEEK_SET);
+        ctx = THIS->ctx;
+	xld.logfp = ctx->log.gf_log_logfile;
 
-        fprintf (gf_log_logfile, "Given volfile:\n");
-        fprintf (gf_log_logfile,
+        fprintf (ctx->log.gf_log_logfile, "Final graph:\n");
+        fprintf (ctx->log.gf_log_logfile,
                  "+---------------------------------------"
                  "---------------------------------------+\n");
-        while (fgets (data, GF_UNIT_KB, specfp) != NULL){
-                lcount++;
-                fprintf (gf_log_logfile, "%3d: %s", lcount, data);
-        }
-        fprintf (gf_log_logfile,
-                 "\n+---------------------------------------"
+
+	xlator_foreach_depth_first (graph->top, xldump, &xld);
+
+        fprintf (ctx->log.gf_log_logfile,
+                 "+---------------------------------------"
                  "---------------------------------------+\n");
-        fflush (gf_log_logfile);
-        fseek (specfp, 0L, SEEK_SET);
+        fflush (ctx->log.gf_log_logfile);
 }
 
 static void
@@ -348,26 +508,31 @@ out:
 /* Obtain a backtrace and print it to stdout. */
 /* TODO: It looks like backtrace_symbols allocates memory,
    it may be problem because mostly memory allocation/free causes 'sigsegv' */
+
 void
-gf_print_trace (int32_t signum)
+gf_print_trace (int32_t signum, glusterfs_ctx_t *ctx)
 {
-        extern FILE *gf_log_logfile;
-        struct tm   *tm = NULL;
         char         msg[1024] = {0,};
-        char         timestr[256] = {0,};
-        time_t       utime = 0;
+        char         timestr[64] = {0,};
         int          ret = 0;
         int          fd = 0;
 
-        fd = fileno (gf_log_logfile);
+        fd = fileno (ctx->log.gf_log_logfile);
 
+        /* Now every gf_log call will just write to a buffer and when the
+         * buffer becomes full, its written to the log-file. Suppose the process
+         * crashes and prints the backtrace in the log-file, then the previous
+         * log information will still be in the buffer itself. So flush the
+         * contents of the buffer to the log file before printing the backtrace
+         * which helps in debugging.
+         */
+        fflush (ctx->log.gf_log_logfile);
         /* Pending frames, (if any), list them in order */
         ret = write (fd, "pending frames:\n", 16);
         if (ret < 0)
                 goto out;
 
         {
-                glusterfs_ctx_t *ctx = glusterfs_ctx_get ();
                 struct list_head *trav = ((call_pool_t *)ctx->pool)->all_frames.next;
                 while (trav != (&((call_pool_t *)ctx->pool)->all_frames)) {
                         call_frame_t *tmp = (call_frame_t *)(&((call_stack_t *)trav)->frames);
@@ -375,10 +540,10 @@ gf_print_trace (int32_t signum)
                                 sprintf (msg,"frame : type(%d) op(%s)\n",
                                          tmp->root->type,
                                          gf_fop_list[tmp->root->op]);
-                        if (tmp->root->type == GF_OP_TYPE_MGMT)
-                                sprintf (msg,"frame : type(%d) op(%s)\n",
+                        else
+                                sprintf (msg,"frame : type(%d) op(%d)\n",
                                          tmp->root->type,
-                                         gf_mgmt_list[tmp->root->op]);
+                                         tmp->root->op);
 
                         ret = write (fd, msg, strlen (msg));
                         if (ret < 0)
@@ -404,9 +569,7 @@ gf_print_trace (int32_t signum)
         {
                 /* Dump the timestamp of the crash too, so the previous logs
                    can be related */
-                utime = time (NULL);
-                tm    = localtime (&utime);
-                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S\n", tm);
+                gf_time_fmt (timestr, sizeof timestr, time (NULL), gf_timefmt_FT);
                 ret = write (fd, "time of crash: ", 15);
                 if (ret < 0)
                         goto out;
@@ -562,8 +725,7 @@ gf_strstr (const char *str, const char *delim, const char *match)
         }
 
 out:
-        if (tmp_str)
-                free (tmp_str);
+        free (tmp_str);
 
         return ret;
 
@@ -616,6 +778,8 @@ gf_string2time (const char *str, uint32_t *n)
         old_errno = errno;
         errno = 0;
         value = strtol (str, &tail, 0);
+        if (str == tail)
+                errno = EINVAL;
 
         if (errno == ERANGE || errno == EINVAL)
                 return -1;
@@ -634,11 +798,10 @@ gf_string2time (const char *str, uint32_t *n)
         return 0;
 }
 
-
 int
-gf_string2percent (const char *str, uint32_t *n)
+gf_string2percent (const char *str, double *n)
 {
-        unsigned long value = 0;
+        double value = 0;
         char *tail = NULL;
         int old_errno = 0;
         const char *s = NULL;
@@ -659,7 +822,9 @@ gf_string2percent (const char *str, uint32_t *n)
 
         old_errno = errno;
         errno = 0;
-        value = strtol (str, &tail, 0);
+        value = strtod (str, &tail);
+        if (str == tail)
+                errno = EINVAL;
 
         if (errno == ERANGE || errno == EINVAL)
                 return -1;
@@ -693,6 +858,8 @@ _gf_string2long (const char *str, long *n, int base)
         old_errno = errno;
         errno = 0;
         value = strtol (str, &tail, base);
+        if (str == tail)
+                errno = EINVAL;
 
         if (errno == ERANGE || errno == EINVAL)
                 return -1;
@@ -733,6 +900,8 @@ _gf_string2ulong (const char *str, unsigned long *n, int base)
         old_errno = errno;
         errno = 0;
         value = strtoul (str, &tail, base);
+        if (str == tail)
+                errno = EINVAL;
 
         if (errno == ERANGE || errno == EINVAL)
                 return -1;
@@ -773,6 +942,8 @@ _gf_string2uint (const char *str, unsigned int *n, int base)
         old_errno = errno;
         errno = 0;
         value = strtoul (str, &tail, base);
+        if (str == tail)
+                errno = EINVAL;
 
         if (errno == ERANGE || errno == EINVAL)
                 return -1;
@@ -804,6 +975,8 @@ _gf_string2double (const char *str, double *n)
         old_errno = errno;
         errno = 0;
         value = strtod (str, &tail);
+        if (str == tail)
+                errno = EINVAL;
 
         if (errno == ERANGE || errno == EINVAL)
                 return -1;
@@ -835,6 +1008,8 @@ _gf_string2longlong (const char *str, long long *n, int base)
         old_errno = errno;
         errno = 0;
         value = strtoll (str, &tail, base);
+        if (str == tail)
+                errno = EINVAL;
 
         if (errno == ERANGE || errno == EINVAL)
                 return -1;
@@ -875,6 +1050,8 @@ _gf_string2ulonglong (const char *str, unsigned long long *n, int base)
         old_errno = errno;
         errno = 0;
         value = strtoull (str, &tail, base);
+        if (str == tail)
+                errno = EINVAL;
 
         if (errno == ERANGE || errno == EINVAL)
                 return -1;
@@ -948,7 +1125,7 @@ gf_string2int8 (const char *str, int8_t *n)
         if (rv != 0)
                 return rv;
 
-        if (l >= INT8_MIN && l <= INT8_MAX) {
+        if ((l >= INT8_MIN) && (l <= INT8_MAX)) {
                 *n = (int8_t) l;
                 return 0;
         }
@@ -967,7 +1144,7 @@ gf_string2int16 (const char *str, int16_t *n)
         if (rv != 0)
                 return rv;
 
-        if (l >= INT16_MIN && l <= INT16_MAX) {
+        if ((l >= INT16_MIN) && (l <= INT16_MAX)) {
                 *n = (int16_t) l;
                 return 0;
         }
@@ -986,7 +1163,7 @@ gf_string2int32 (const char *str, int32_t *n)
         if (rv != 0)
                 return rv;
 
-        if (l >= INT32_MIN && l <= INT32_MAX) {
+        if ((l >= INT32_MIN) && (l <= INT32_MAX)) {
                 *n = (int32_t) l;
                 return 0;
         }
@@ -1005,7 +1182,7 @@ gf_string2int64 (const char *str, int64_t *n)
         if (rv != 0)
                 return rv;
 
-        if (l >= INT64_MIN && l <= INT64_MAX) {
+        if ((l >= INT64_MIN) && (l <= INT64_MAX)) {
                 *n = (int64_t) l;
                 return 0;
         }
@@ -1217,7 +1394,7 @@ err:
 int
 gf_string2bytesize (const char *str, uint64_t *n)
 {
-        uint64_t value = 0ULL;
+        double value = 0.0;
         char *tail = NULL;
         int old_errno = 0;
         const char *s = NULL;
@@ -1238,7 +1415,9 @@ gf_string2bytesize (const char *str, uint64_t *n)
 
         old_errno = errno;
         errno = 0;
-        value = strtoull (str, &tail, 10);
+        value = strtod (str, &tail);
+        if (str == tail)
+                errno = EINVAL;
 
         if (errno == ERANGE || errno == EINVAL)
                 return -1;
@@ -1262,7 +1441,77 @@ gf_string2bytesize (const char *str, uint64_t *n)
                         return -1;
         }
 
-        *n = value;
+        if ((UINT64_MAX - value) < 0) {
+                errno = ERANGE;
+                return -1;
+        }
+
+        *n = (uint64_t) value;
+
+        return 0;
+}
+
+int
+gf_string2percent_or_bytesize (const char *str,
+			       uint64_t *n,
+			       gf_boolean_t *is_percent)
+{
+        double value = 0ULL;
+        char *tail = NULL;
+        int old_errno = 0;
+        const char *s = NULL;
+
+        if (str == NULL || n == NULL) {
+                gf_log_callingfn (THIS->name, GF_LOG_WARNING,
+				  "argument invalid");
+                errno = EINVAL;
+                return -1;
+        }
+
+        for (s = str; *s != '\0'; s++) {
+                if (isspace (*s))
+                        continue;
+                if (*s == '-')
+                        return -1;
+                break;
+        }
+
+        old_errno = errno;
+        errno = 0;
+        value = strtod (str, &tail);
+        if (str == tail)
+                errno = EINVAL;
+
+        if (errno == ERANGE || errno == EINVAL)
+                return -1;
+
+        if (errno == 0)
+                errno = old_errno;
+
+        if (tail[0] != '\0') {
+                if (strcasecmp (tail, GF_UNIT_KB_STRING) == 0)
+                        value *= GF_UNIT_KB;
+                else if (strcasecmp (tail, GF_UNIT_MB_STRING) == 0)
+                        value *= GF_UNIT_MB;
+                else if (strcasecmp (tail, GF_UNIT_GB_STRING) == 0)
+                        value *= GF_UNIT_GB;
+                else if (strcasecmp (tail, GF_UNIT_TB_STRING) == 0)
+                        value *= GF_UNIT_TB;
+                else if (strcasecmp (tail, GF_UNIT_PB_STRING) == 0)
+                        value *= GF_UNIT_PB;
+		else if (strcasecmp (tail, GF_UNIT_PERCENT_STRING) == 0)
+			*is_percent = _gf_true;
+                else
+                        return -1;
+        }
+
+        /* Error out if we cannot store the value in uint64 */
+        if ((UINT64_MAX - value) < 0) {
+                errno = ERANGE;
+                return -1;
+        }
+
+        *n = (uint64_t) value;
 
         return 0;
 }
@@ -1528,64 +1777,112 @@ get_nth_word (const char *str, int n)
         return word;
 }
 
-/* RFC 1123 & 952 */
-
-/* The functions below validate given internet addresses and
- * wildcard internet address for correctness.
- * All return 1 on success and 0 on failure
- */
-
+/* Syntax formed according to RFC 1912 (RFC 1123 & 952 are more restrictive)  *
+   <hname> ::= <gen-name>*["."<gen-name>]                                     *
+   <gen-name> ::= <let-or-digit> <[*[<let-or-digit-or-hyphen>]<let-or-digit>] */
 char
 valid_host_name (char *address, int length)
 {
-        int     i = 0;
-        char    ret = 0;
-        int     flag = 0;
+        int             i = 0;
+        int             str_len = 0;
+        char            ret = 1;
+        char            *dup_addr = NULL;
+        char            *temp_str = NULL;
+        char            *save_ptr = NULL;
 
-        if ((length > 255) || (length == 1))
+        if ((length > _POSIX_HOST_NAME_MAX) || (length < 1)) {
+                ret = 0;
                 goto out;
+        }
 
-        if (!isalnum (address[length - 1]))
+        dup_addr = gf_strdup (address);
+        if (!dup_addr) {
+                ret = 0;
                 goto out;
+        }
 
-        for (i = 0; i < length; i++) {
-                if (!isalnum (address[i]) && (address[i] != '.')
-                    && (address[i] != '-'))
-                        goto out;
+        if (!isalnum (dup_addr[length - 1]) && (dup_addr[length - 1] != '*')) {
+                ret = 0;
+                goto out;
+        }
 
-                if (isalpha(address[i]))
-                        flag = 1;
+        /* Check for consecutive dots, which is invalid in a hostname and is
+         * ignored by strtok()
+         */
+        if (strstr (dup_addr, "..")) {
+                ret = 0;
+                goto out;
         }
 
-        if (flag)
-                ret = 1;
+        /* gen-name */
+        temp_str = strtok_r (dup_addr, ".", &save_ptr);
+        do {
+                str_len = strlen (temp_str);
+
+                if (!isalnum (temp_str[0]) ||
+                    !isalnum (temp_str[str_len-1])) {
+                        ret = 0;
+                        goto out;
+                }
+                for (i = 1; i < str_len; i++) {
+                        if (!isalnum (temp_str[i]) && (temp_str[i] != '-')) {
+                                ret = 0;
+                                goto out;
+                        }
+                }
+        } while ((temp_str = strtok_r (NULL, ".", &save_ptr)));
+
 out:
+        GF_FREE (dup_addr);
         return ret;
 }
 
+/*  Matches all ipv4 address, if wildcard_acc is true  '*' wildcard pattern for*
+  subnets is considerd as valid strings as well                               */
 char
-valid_ipv4_address (char *address, int length)
+valid_ipv4_address (char *address, int length, gf_boolean_t wildcard_acc)
 {
         int octets = 0;
         int value = 0;
         char *tmp = NULL, *ptr = NULL, *prev = NULL, *endptr = NULL;
-        char ret = 0;
+        char ret = 1;
+        int is_wildcard = 0;
 
         tmp = gf_strdup (address);
+
+        /*
+         * To prevent cases where last character is '.' and which have
+         * consecutive dots like ".." as strtok ignore consecutive
+         * delimeters.
+         */
+        if (length <= 0 ||
+            (strstr (address, "..")) ||
+            (!isdigit (tmp[length - 1]) && (tmp[length - 1] != '*'))) {
+                ret = 0;
+                goto out;
+        }
+
         prev = tmp;
         prev = strtok_r (tmp, ".", &ptr);
 
         while (prev != NULL) {
                 octets++;
-                value = strtol (prev, &endptr, 10);
-                if ((value > 255) || (value < 0) ||
-                    (endptr != NULL && *endptr != '\0'))
-                        goto out;
+                if (wildcard_acc && !strcmp (prev, "*")) {
+                        is_wildcard = 1;
+                } else {
+                        value = strtol (prev, &endptr, 10);
+                        if ((value > 255) || (value < 0) ||
+                            (endptr != NULL && *endptr != '\0')) {
+                                ret = 0;
+                                goto out;
+                        }
+                }
                 prev = strtok_r (NULL, ".", &ptr);
         }
 
-        if (octets == 4)
-                ret = 1;
+        if ((octets > 4) || (octets < 4 && !is_wildcard)) {
+                ret = 0;
+        }
 
 out:
         GF_FREE (tmp);
@@ -1593,27 +1890,55 @@ out:
 }
 
 char
-valid_ipv6_address (char *address, int length)
+valid_ipv6_address (char *address, int length, gf_boolean_t wildcard_acc)
 {
         int hex_numbers = 0;
         int value = 0;
+        int i = 0;
         char *tmp = NULL, *ptr = NULL, *prev = NULL, *endptr = NULL;
-        char ret = 0;
+        char ret = 1;
+        int is_wildcard = 0;
+        int is_compressed = 0;
 
         tmp = gf_strdup (address);
+
+        /* Check for compressed form */
+        if (length <= 0 || tmp[length - 1] == ':') {
+                ret = 0;
+                goto out;
+        }
+        for (i = 0; i < (length - 1) ; i++) {
+                if (tmp[i] == ':' && tmp[i + 1] == ':') {
+                        if (is_compressed == 0)
+                                is_compressed = 1;
+                        else {
+                                ret = 0;
+                                goto out;
+                        }
+                }
+        }
+
         prev = strtok_r (tmp, ":", &ptr);
 
         while (prev != NULL) {
                 hex_numbers++;
-                value = strtol (prev, &endptr, 16);
-                if ((value > 0xffff) || (value < 0)
-                    || (endptr != NULL && *endptr != '\0'))
-                        goto out;
+                if (wildcard_acc && !strcmp (prev, "*")) {
+                        is_wildcard = 1;
+                } else {
+                        value = strtol (prev, &endptr, 16);
+                        if ((value > 0xffff) || (value < 0)
+                        || (endptr != NULL && *endptr != '\0')) {
+                                ret = 0;
+                                goto out;
+                        }
+                }
                 prev = strtok_r (NULL, ":", &ptr);
         }
 
-        if (hex_numbers <= 8)
-                ret = 1;
+        if ((hex_numbers > 8) || (hex_numbers < 8 && !is_wildcard
+            && !is_compressed)) {
+                ret = 0;
+        }
 
 out:
         GF_FREE (tmp);
@@ -1621,7 +1946,7 @@ out:
 }
 
 char
-valid_internet_address (char *address)
+valid_internet_address (char *address, gf_boolean_t wildcard_acc)
 {
         char ret = 0;
         int length = 0;
@@ -1635,8 +1960,8 @@ valid_internet_address (char *address)
         if (length == 0)
                 goto out;
 
-        if (valid_ipv4_address (address, length)
-            || valid_ipv6_address (address, length)
+        if (valid_ipv4_address (address, length, wildcard_acc)
+            || valid_ipv6_address (address, length, wildcard_acc)
             || valid_host_name (address, length))
                 ret = 1;
 
@@ -1644,109 +1969,55 @@ out:
         return ret;
 }
 
-char
-valid_ipv4_wildcard_check (char *address)
+/**
+ * gf_sock_union_equal_addr - check if two given gf_sock_unions have same addr
+ *
+ * @param a - first sock union
+ * @param b - second sock union
+ * @return _gf_true if a and b have same ipv{4,6} addr, _gf_false otherwise
+ */
+gf_boolean_t
+gf_sock_union_equal_addr (union gf_sock_union *a,
+                          union gf_sock_union *b)
 {
-        char    ret = 0;
-        int     octets = 0;
-        char    *tmp = NULL;
-        char    *prev = NULL;
-        char    *endptr = NULL;
-        int     value = 0;
-        int     is_wildcard = 0;
-
-        tmp = gf_strdup (address);
-        prev = strtok (tmp, ".");
-
-        while (prev != NULL) {
-                octets++;
-
-                if (!strcmp (prev, "*")) {
-                        is_wildcard = 1;
-                } else {
-                        value = strtol (prev, &endptr, 10);
-
-                        if ((value > 255) || (value < 0) ||
-                            (endptr != NULL && *endptr != '\0'))
-                                goto out;
-                }
-                prev = strtok (NULL, ".");
+        if (!a || !b) {
+                gf_log ("common-utils", GF_LOG_ERROR, "Invalid arguments"
+                        " to gf_sock_union_equal_addr");
+                return _gf_false;
         }
 
-        if (is_wildcard && (octets <= 4))
-                ret = 1;
-
-out:
-        if (tmp)
-                GF_FREE (tmp);
-        return ret;
-
-}
+        if (a->storage.ss_family != b->storage.ss_family)
+                return _gf_false;
 
-char
-valid_ipv6_wildcard_check (char *address)
-{
-        char    ret = 0;
-        int     hex_numbers = 0;
-        int     value = 0;
-        char    *tmp = NULL;
-        char    *prev = NULL;
-        char    *endptr = NULL;
-        int     is_wildcard = 0;
-
-        tmp = gf_strdup (address);
-        prev = strtok (tmp, ":");
-
-        while (prev != NULL) {
-                hex_numbers++;
-
-                if (!strcmp (prev, "*")) {
-                        is_wildcard = 1;
-                } else {
-                        value = strtol (prev, &endptr, 16);
-
-                        if ((value > 0xffff) || (value < 0) ||
-                            (endptr != NULL && *endptr != '\0'))
-                                goto out;
-                }
-                prev = strtok (NULL, ":");
-        }
-
-        if (is_wildcard && (hex_numbers <= 8))
-                ret = 1;
-out:
-        if (tmp)
-                GF_FREE (tmp);
-        return ret;
-}
+        switch (a->storage.ss_family) {
+        case AF_INET:
+                if (a->sin.sin_addr.s_addr == b->sin.sin_addr.s_addr)
+                        return _gf_true;
+                else
+                        return _gf_false;
 
-char
-valid_wildcard_internet_address (char *address)
-{
-        char    ret = 0;
+        case AF_INET6:
+                if (memcmp ((void *)(&a->sin6.sin6_addr),
+                            (void *)(&b->sin6.sin6_addr),
+                            sizeof (a->sin6.sin6_addr)))
+                        return _gf_false;
+                else
+                        return _gf_true;
 
-        if (address == NULL) {
-                gf_log_callingfn (THIS->name, GF_LOG_WARNING,
-                                  "argument invalid");
-                goto out;
+        default:
+                gf_log ("common-utils", GF_LOG_DEBUG,
+                        "Unsupported/invalid address family");
+                break;
         }
 
-        if (strlen (address) == 0)
-                goto out;
-
-        if (valid_ipv4_wildcard_check (address) ||
-            valid_ipv6_wildcard_check (address))
-                ret = 1;
-
-out:
-        return ret;
+        return _gf_false;
 }
 
 /*Thread safe conversion function*/
 char *
 uuid_utoa (uuid_t uuid)
 {
-        char *uuid_buffer = glusterfs_uuid_buf_get();
+        char *uuid_buffer = glusterfs_uuid_buf_get(THIS->ctx);
         uuid_unparse (uuid, uuid_buffer);
         return uuid_buffer;
 }
@@ -1765,7 +2036,7 @@ uuid_utoa_r (uuid_t uuid, char *dst)
 char *
 lkowner_utoa (gf_lkowner_t *lkowner)
 {
-        char *lkowner_buffer = glusterfs_lkowner_buf_get();
+        char *lkowner_buffer = glusterfs_lkowner_buf_get(THIS->ctx);
         lkowner_unparse (lkowner, lkowner_buffer, GF_LKOWNER_BUF_SIZE);
         return lkowner_buffer;
 }
@@ -1780,19 +2051,6 @@ lkowner_utoa_r (gf_lkowner_t *lkowner, char *dst, int len)
         return dst;
 }
 
-void _get_md5_str (char *out_str, size_t outlen,
-                   const uint8_t *input, int n)
-{
-        uint8_t out[MD5_DIGEST_LEN] = {0};
-        int     j = 0;
-
-        GF_ASSERT (outlen >= (2*MD5_DIGEST_LEN + 1));
-        get_md5 (out, input, n);
-        for (j = 0; j < MD5_DIGEST_LEN; j++)
-                snprintf(out_str + j * 2, outlen-j*2, "%02x", out[j]);
-
-}
-
 void* gf_array_elem (void *a, int index, size_t elem_size)
 {
         uint8_t* ptr = a;
@@ -1855,8 +2113,7 @@ gf_is_str_int (const char *value)
         }
 
 out:
-        if (fptr)
-                GF_FREE (fptr);
+        GF_FREE (fptr);
 
         return flag;
 }
@@ -1866,9 +2123,9 @@ out:
  */
 
 inline int32_t
-gf_roundup_power_of_two (uint32_t nr)
+gf_roundup_power_of_two (int32_t nr)
 {
-        uint32_t result = 1;
+        int32_t result = 1;
 
         if (nr < 0) {
                 gf_log ("common-utils", GF_LOG_WARNING,
@@ -1889,13 +2146,8 @@ out:
  * power of two is returned.
  */
 
-/*
- * rounds up nr to next power of two. If nr is already a power of two, next
- * power of two is returned.
- */
-
 inline int32_t
-gf_roundup_next_power_of_two (uint32_t nr)
+gf_roundup_next_power_of_two (int32_t nr)
 {
         int32_t result = 1;
 
@@ -1998,10 +2250,659 @@ get_mem_size ()
 	return memsize;
 }
 
+/* Strips all whitespace characters in a string and returns length of new string
+ * on success
+ */
+int
+gf_strip_whitespace (char *str, int len)
+{
+        int     i = 0;
+        int     new_len = 0;
+        char    *new_str = NULL;
+
+        GF_ASSERT (str);
+
+        new_str = GF_CALLOC (1, len + 1, gf_common_mt_char);
+        if (new_str == NULL)
+                return -1;
+
+        for (i = 0; i < len; i++) {
+                if (!isspace (str[i]))
+                        new_str[new_len++] = str[i];
+        }
+        new_str[new_len] = '\0';
+
+        if (new_len != len) {
+                memset (str, 0, len);
+                strncpy (str, new_str, new_len);
+        }
+
+        GF_FREE (new_str);
+        return new_len;
+}
 
 int
-gf_client_pid_check (gf_client_pid_t npid)
+gf_canonicalize_path (char *path)
+{
+        int             ret                  = -1;
+        int             path_len             = 0;
+        int             dir_path_len         = 0;
+        char           *tmppath              = NULL;
+        char           *dir                  = NULL;
+        char           *tmpstr               = NULL;
+
+        if (!path || *path != '/')
+                goto out;
+
+        tmppath = gf_strdup (path);
+        if (!tmppath)
+                goto out;
+
+        /* Strip the extra slashes and return */
+        bzero (path, strlen(path));
+        path[0] = '/';
+        dir = strtok_r(tmppath, "/", &tmpstr);
+
+        while (dir) {
+                dir_path_len = strlen(dir);
+                strncpy ((path + path_len + 1), dir, dir_path_len);
+                path_len += dir_path_len + 1;
+                dir = strtok_r (NULL, "/", &tmpstr);
+                if (dir)
+                        strncpy ((path + path_len), "/", 1);
+        }
+        path[path_len] = '\0';
+        ret = 0;
+
+ out:
+        if (ret)
+                gf_log ("common-utils", GF_LOG_ERROR,
+                        "Path manipulation failed");
+
+        GF_FREE(tmppath);
+
+        return ret;
+}
+
+static const char *__gf_timefmts[] = {
+        "%F %T",
+        "%Y/%m/%d-%T",
+        "%b %d %T",
+        "%F %H%M%S"
+};
+
+static const char *__gf_zerotimes[] = {
+        "0000-00-00 00:00:00",
+        "0000/00/00-00:00:00",
+        "xxx 00 00:00:00",
+        "0000-00-00 000000"
+};
+
+void
+_gf_timestuff (gf_timefmts *fmt, const char ***fmts, const char ***zeros)
+{
+        *fmt = gf_timefmt_last;
+        *fmts = __gf_timefmts;
+        *zeros = __gf_zerotimes;
+}
+
+
+char *
+generate_glusterfs_ctx_id (void)
 {
-        return ( (npid > GF_CLIENT_PID_MIN) && (npid < GF_CLIENT_PID_MAX) )
-                ? 0 : -1;
+        char           tmp_str[1024] = {0,};
+        char           hostname[256] = {0,};
+        struct timeval tv = {0,};
+        char           now_str[32];
+
+        if (gettimeofday (&tv, NULL) == -1) {
+                gf_log ("glusterfsd", GF_LOG_ERROR,
+                        "gettimeofday: failed %s",
+                        strerror (errno));
+        }
+
+        if (gethostname (hostname, 256) == -1) {
+                gf_log ("glusterfsd", GF_LOG_ERROR,
+                        "gethostname: failed %s",
+                        strerror (errno));
+        }
+
+        gf_time_fmt (now_str, sizeof now_str, tv.tv_sec, gf_timefmt_Ymd_T);
+        snprintf (tmp_str, sizeof tmp_str, "%s-%d-%s:%"
+#ifdef GF_DARWIN_HOST_OS
+                  PRId32,
+#else
+                  "ld",
+#endif
+                  hostname, getpid(), now_str, tv.tv_usec);
+
+        return gf_strdup (tmp_str);
+}
+
+char *
+gf_get_reserved_ports ()
+{
+        char    *ports_info  = NULL;
+#if defined GF_LINUX_HOST_OS
+        int     proc_fd      = -1;
+        char    *proc_file   = "/proc/sys/net/ipv4/ip_local_reserved_ports";
+        char    buffer[4096] = {0,};
+        int32_t ret          = -1;
+
+        proc_fd = open (proc_file, O_RDONLY);
+        if (proc_fd == -1) {
+                /* What should be done in this case? error out from here
+                 * and thus stop the glusterfs process from starting or
+                 * continue with older method of using any of the available
+                 * port? For now 2nd option is considered.
+                 */
+                gf_log ("glusterfs", GF_LOG_WARNING, "could not open "
+                        "the file /proc/sys/net/ipv4/ip_local_reserved_ports "
+                        "for getting reserved ports info (%s)",
+                        strerror (errno));
+                goto out;
+        }
+
+        ret = read (proc_fd, buffer, sizeof (buffer));
+        if (ret < 0) {
+                gf_log ("glusterfs", GF_LOG_WARNING, "could not "
+                        "read the file %s for getting reserved ports "
+                        "info (%s)", proc_file, strerror (errno));
+                goto out;
+        }
+        ports_info = gf_strdup (buffer);
+
+out:
+        if (proc_fd != -1)
+                close (proc_fd);
+#endif /* GF_LINUX_HOST_OS */
+        return ports_info;
+}
+
+int
+gf_process_reserved_ports (gf_boolean_t *ports)
+{
+        int      ret         = -1;
+#if defined GF_LINUX_HOST_OS
+        char    *ports_info  = NULL;
+        char    *tmp         = NULL;
+        char    *blocked_port = NULL;
+
+        ports_info = gf_get_reserved_ports ();
+        if (!ports_info) {
+                gf_log ("glusterfs", GF_LOG_WARNING, "Not able to get reserved "
+                        "ports, hence there is a possibility that glusterfs "
+                        "may consume reserved port");
+                goto out;
+        }
+
+        blocked_port = strtok_r (ports_info, ",\n",&tmp);
+
+        while (blocked_port) {
+                gf_ports_reserved (blocked_port, ports);
+                blocked_port = strtok_r (NULL, ",\n", &tmp);
+        }
+
+        ret = 0;
+
+out:
+        GF_FREE (ports_info);
+#endif /* GF_LINUX_HOST_OS */
+        return ret;
+}
+
+gf_boolean_t
+gf_ports_reserved (char *blocked_port, gf_boolean_t *ports)
+{
+        gf_boolean_t    result   = _gf_false;
+        char            *range_port = NULL;
+        int16_t         tmp_port1, tmp_port2 = -1;
+
+        if (strstr (blocked_port, "-") == NULL) {
+                /* get rid of the new line character*/
+                if (blocked_port[strlen(blocked_port) -1] == '\n')
+                        blocked_port[strlen(blocked_port) -1] = '\0';
+                if (gf_string2int16 (blocked_port, &tmp_port1) == 0) {
+                        if (tmp_port1 > (GF_CLIENT_PORT_CEILING - 1)
+                            || tmp_port1 < 0) {
+                                gf_log ("glusterfs-socket", GF_LOG_WARNING,
+                                        "invalid port %d", tmp_port1);
+                                result = _gf_true;
+                                goto out;
+                        } else {
+                                gf_log ("glusterfs", GF_LOG_DEBUG,
+                                        "blocking port %d", tmp_port1);
+                                ports[tmp_port1] = _gf_true;
+                        }
+                } else {
+                        gf_log ("glusterfs-socket", GF_LOG_WARNING, "%s is "
+                                "not a valid port identifier", blocked_port);
+                        result = _gf_true;
+                        goto out;
+                }
+        } else {
+                range_port = strtok (blocked_port, "-");
+                if (!range_port){
+                        result = _gf_true;
+                        goto out;
+                }
+                if (gf_string2int16 (range_port, &tmp_port1) == 0) {
+                        if (tmp_port1 > (GF_CLIENT_PORT_CEILING - 1))
+                                tmp_port1 = GF_CLIENT_PORT_CEILING - 1;
+                        if (tmp_port1 < 0)
+                                tmp_port1 = 0;
+                }
+                range_port = strtok (NULL, "-");
+                if (!range_port) {
+                        result = _gf_true;
+                        goto out;
+                }
+                /* get rid of the new line character*/
+                if (range_port[strlen(range_port) -1] == '\n')
+                        range_port[strlen(range_port) - 1] = '\0';
+                if (gf_string2int16 (range_port, &tmp_port2) == 0) {
+                        if (tmp_port2 >
+                            (GF_CLIENT_PORT_CEILING - 1))
+                                tmp_port2 = GF_CLIENT_PORT_CEILING - 1;
+                        if (tmp_port2 < 0)
+                                tmp_port2 = 0;
+                }
+                gf_log ("glusterfs", GF_LOG_DEBUG, "lower: %d, higher: %d",
+                        tmp_port1, tmp_port2);
+                for (; tmp_port1 <= tmp_port2; tmp_port1++)
+                        ports[tmp_port1] = _gf_true;
+        }
+
+out:
+        return result;
+}
+
+/* Takes in client ip{v4,v6} and returns associated hostname, if any
+ * Also, allocates memory for the hostname.
+ * Returns: 0 for success, -1 for failure
+ */
+int
+gf_get_hostname_from_ip (char *client_ip, char **hostname)
+{
+        int                      ret                          = -1;
+        struct sockaddr         *client_sockaddr              = NULL;
+        struct sockaddr_in       client_sock_in               = {0};
+        struct sockaddr_in6      client_sock_in6              = {0};
+        char                     client_hostname[NI_MAXHOST]  = {0};
+        char                    *client_ip_copy               = NULL;
+        char                    *tmp                          = NULL;
+        char                    *ip                           = NULL;
+
+        /* if ipv4, reverse lookup the hostname to
+         * allow FQDN based rpc authentication
+         */
+        if (valid_ipv4_address (client_ip, strlen (client_ip), 0) == _gf_false) {
+                /* most times, we get a.b.c.d:port form, so check that */
+                client_ip_copy = gf_strdup (client_ip);
+                if (!client_ip_copy)
+                        goto out;
+
+                ip = strtok_r (client_ip_copy, ":", &tmp);
+        } else {
+                ip = client_ip;
+        }
+
+        if (valid_ipv4_address (ip, strlen (ip), 0) == _gf_true) {
+                client_sockaddr = (struct sockaddr *)&client_sock_in;
+                client_sock_in.sin_family = AF_INET;
+                ret = inet_pton (AF_INET, ip,
+                                 (void *)&client_sock_in.sin_addr.s_addr);
+
+        } else if (valid_ipv6_address (ip, strlen (ip), 0) == _gf_true) {
+                client_sockaddr = (struct sockaddr *) &client_sock_in6;
+
+                client_sock_in6.sin6_family = AF_INET6;
+                ret = inet_pton (AF_INET6, ip,
+                                 (void *)&client_sock_in6.sin6_addr);
+        } else {
+                goto out;
+        }
+
+        if (ret != 1) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = getnameinfo (client_sockaddr,
+                           sizeof (*client_sockaddr),
+                           client_hostname, sizeof (client_hostname),
+                           NULL, 0, 0);
+        if (ret) {
+                gf_log ("common-utils", GF_LOG_ERROR,
+                        "Could not lookup hostname of %s : %s",
+                        client_ip, gai_strerror (ret));
+                ret = -1;
+                goto out;
+        }
+
+        *hostname = gf_strdup ((char *)client_hostname);
+ out:
+        if (client_ip_copy)
+                GF_FREE (client_ip_copy);
+
+        return ret;
+}
+
+gf_boolean_t
+gf_interface_search (char *ip)
+{
+        int32_t         ret = -1;
+        gf_boolean_t    found = _gf_false;
+        struct          ifaddrs *ifaddr, *ifa;
+        int             family;
+        char            host[NI_MAXHOST];
+        xlator_t        *this = NULL;
+        char            *pct = NULL;
+
+        this = THIS;
+
+        ret = getifaddrs (&ifaddr);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "getifaddrs() failed: %s\n",
+                        gai_strerror(ret));
+                goto out;
+        }
+
+        for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
+                if (!ifa->ifa_addr) {
+                        /*
+                         * This seemingly happens if an interface hasn't
+                         * been bound to a particular protocol (seen with
+                         * TUN devices).
+                         */
+                        continue;
+                }
+                family = ifa->ifa_addr->sa_family;
+
+                if (family != AF_INET && family != AF_INET6)
+                        continue;
+
+                ret = getnameinfo (ifa->ifa_addr,
+                        (family == AF_INET) ? sizeof(struct sockaddr_in) :
+                                              sizeof(struct sockaddr_in6),
+                        host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
+
+                if (ret != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "getnameinfo() failed: %s\n",
+                                gai_strerror(ret));
+                        goto out;
+                }
+
+                /*
+                 * Sometimes the address comes back as addr%eth0 or
+                 * similar.  Since % is an invalid character, we can
+                 * strip it out with confidence that doing so won't
+                 * harm anything.
+                 */
+                pct = index(host,'%');
+                if (pct) {
+                        *pct = '\0';
+                }
+
+                if (strncmp (ip, host, NI_MAXHOST) == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s is local address at interface %s",
+                                ip, ifa->ifa_name);
+                        found = _gf_true;
+                        goto out;
+                }
+        }
+out:
+        if(ifaddr)
+                freeifaddrs (ifaddr);
+        return found;
+}
+
+char *
+get_ip_from_addrinfo (struct addrinfo *addr, char **ip)
+{
+        char buf[64];
+        void *in_addr = NULL;
+        struct sockaddr_in *s4 = NULL;
+        struct sockaddr_in6 *s6 = NULL;
+
+        switch (addr->ai_family)
+        {
+                case AF_INET:
+                        s4 = (struct sockaddr_in *)addr->ai_addr;
+                        in_addr = &s4->sin_addr;
+                        break;
+
+                case AF_INET6:
+                        s6 = (struct sockaddr_in6 *)addr->ai_addr;
+                        in_addr = &s6->sin6_addr;
+                        break;
+
+                default:
+                        gf_log ("glusterd", GF_LOG_ERROR, "Invalid family");
+                        return NULL;
+        }
+
+        if (!inet_ntop(addr->ai_family, in_addr, buf, sizeof(buf))) {
+                gf_log ("glusterd", GF_LOG_ERROR, "String conversion failed");
+                return NULL;
+        }
+
+        *ip = strdup (buf);
+        return *ip;
+}
+
+gf_boolean_t
+gf_is_loopback_localhost (const struct sockaddr *sa, char *hostname)
+{
+        GF_ASSERT (sa);
+
+        gf_boolean_t is_local = _gf_false;
+        const struct in_addr *addr4 = NULL;
+        const struct in6_addr *addr6 = NULL;
+        uint8_t      *ap   = NULL;
+        struct in6_addr loopbackaddr6 = IN6ADDR_LOOPBACK_INIT;
+
+        switch (sa->sa_family) {
+                case AF_INET:
+                        addr4 = &(((struct sockaddr_in *)sa)->sin_addr);
+                        ap = (uint8_t*)&addr4->s_addr;
+                        if (ap[0] == 127)
+                                is_local = _gf_true;
+                        break;
+
+                case AF_INET6:
+                        addr6 = &(((struct sockaddr_in6 *)sa)->sin6_addr);
+                        if (memcmp (addr6, &loopbackaddr6,
+                                    sizeof (loopbackaddr6)) == 0)
+                                is_local = _gf_true;
+                        break;
+
+                default:
+                        if (hostname)
+                                gf_log ("glusterd", GF_LOG_ERROR,
+                                        "unknown address family %d for %s",
+                                        sa->sa_family, hostname);
+                        break;
+        }
+
+        return is_local;
+}
+
+gf_boolean_t
+gf_is_local_addr (char *hostname)
+{
+        int32_t         ret = -1;
+        struct          addrinfo *result = NULL;
+        struct          addrinfo *res = NULL;
+        gf_boolean_t    found = _gf_false;
+        char            *ip = NULL;
+        xlator_t        *this = NULL;
+
+        this = THIS;
+        ret = getaddrinfo (hostname, NULL, NULL, &result);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "error in getaddrinfo: %s\n",
+                        gai_strerror(ret));
+                goto out;
+        }
+
+        for (res = result; res != NULL; res = res->ai_next) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s ",
+                        get_ip_from_addrinfo (res, &ip));
+
+                found = gf_is_loopback_localhost (res->ai_addr, hostname)
+                        || gf_interface_search (ip);
+                if (found)
+                        goto out;
+        }
+
+out:
+        if (result)
+                freeaddrinfo (result);
+
+        if (!found)
+                gf_log (this->name, GF_LOG_DEBUG, "%s is not local", hostname);
+
+        return found;
+}
+
+gf_boolean_t
+gf_is_same_address (char *name1, char *name2)
+{
+        struct addrinfo         *addr1 = NULL;
+        struct addrinfo         *addr2 = NULL;
+        struct addrinfo         *p = NULL;
+        struct addrinfo         *q = NULL;
+        gf_boolean_t            ret = _gf_false;
+        int                     gai_err = 0;
+
+        gai_err = getaddrinfo(name1,NULL,NULL,&addr1);
+        if (gai_err != 0) {
+                gf_log (name1, GF_LOG_WARNING,
+                        "error in getaddrinfo: %s\n", gai_strerror(gai_err));
+                goto out;
+        }
+
+        gai_err = getaddrinfo(name2,NULL,NULL,&addr2);
+        if (gai_err != 0) {
+                gf_log (name2, GF_LOG_WARNING,
+                        "error in getaddrinfo: %s\n", gai_strerror(gai_err));
+                goto out;
+        }
+
+        for (p = addr1; p; p = p->ai_next) {
+                for (q = addr2; q; q = q->ai_next) {
+                        if (p->ai_addrlen != q->ai_addrlen) {
+                                continue;
+                        }
+                        if (memcmp(p->ai_addr,q->ai_addr,p->ai_addrlen)) {
+                                continue;
+                        }
+                        ret = _gf_true;
+                        goto out;
+                }
+        }
+
+out:
+        if (addr1) {
+                freeaddrinfo(addr1);
+        }
+        if (addr2) {
+                freeaddrinfo(addr2);
+        }
+        return ret;
+
+}
+
+
+/* Sets log file path from user provided arguments */
+int
+gf_set_log_file_path (cmd_args_t *cmd_args)
+{
+        int   i = 0;
+        int   j = 0;
+        int   ret = 0;
+        char  tmp_str[1024] = {0,};
+
+        if (!cmd_args)
+                goto done;
+
+        if (cmd_args->mount_point) {
+                j = 0;
+                i = 0;
+                if (cmd_args->mount_point[0] == '/')
+                        i = 1;
+                for (; i < strlen (cmd_args->mount_point); i++,j++) {
+                        tmp_str[j] = cmd_args->mount_point[i];
+                        if (cmd_args->mount_point[i] == '/')
+                                tmp_str[j] = '-';
+                }
+
+                ret = gf_asprintf (&cmd_args->log_file,
+                                   DEFAULT_LOG_FILE_DIRECTORY "/%s.log",
+                                   tmp_str);
+                if (ret > 0)
+                        ret = 0;
+                goto done;
+        }
+
+        if (cmd_args->volfile) {
+                j = 0;
+                i = 0;
+                if (cmd_args->volfile[0] == '/')
+                        i = 1;
+                for (; i < strlen (cmd_args->volfile); i++,j++) {
+                        tmp_str[j] = cmd_args->volfile[i];
+                        if (cmd_args->volfile[i] == '/')
+                                tmp_str[j] = '-';
+                }
+                ret = gf_asprintf (&cmd_args->log_file,
+                                   DEFAULT_LOG_FILE_DIRECTORY "/%s.log",
+                                   tmp_str);
+                if (ret > 0)
+                        ret = 0;
+                goto done;
+        }
+
+        if (cmd_args->volfile_server) {
+
+                ret = gf_asprintf (&cmd_args->log_file,
+                                   DEFAULT_LOG_FILE_DIRECTORY "/%s-%s-%d.log",
+                                   cmd_args->volfile_server,
+                                   cmd_args->volfile_id, getpid());
+                if (ret > 0)
+                        ret = 0;
+        }
+done:
+        return ret;
+}
+
+int
+gf_thread_create (pthread_t *thread, const pthread_attr_t *attr,
+		  void *(*start_routine)(void *), void *arg)
+{
+	sigset_t set, old;
+	int ret;
+
+	sigemptyset (&set);
+
+	sigfillset (&set);
+	sigdelset (&set, SIGSEGV);
+	sigdelset (&set, SIGBUS);
+	sigdelset (&set, SIGILL);
+	sigdelset (&set, SIGSYS);
+	sigdelset (&set, SIGFPE);
+	sigdelset (&set, SIGABRT);
+
+	pthread_sigmask (SIG_BLOCK, &set, &old);
+
+	ret = pthread_create (thread, attr, start_routine, arg);
+
+	pthread_sigmask (SIG_SETMASK, &old, NULL);
+
+	return ret;
 }
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index 63566fb53..3c99a4212 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _COMMON_UTILS_H
@@ -32,6 +23,7 @@
 #include <string.h>
 #include <assert.h>
 #include <pthread.h>
+#include <openssl/md5.h>
 #ifndef GF_BSD_HOST_OS
 #include <alloca.h>
 #endif
@@ -69,9 +61,13 @@ void trap (void);
 #define GF_UNIT_TB_STRING    "TB"
 #define GF_UNIT_PB_STRING    "PB"
 
+#define GF_UNIT_PERCENT_STRING      "%"
+
 #define GEOREP "geo-replication"
 #define GHADOOP "glusterfs-hadoop"
 
+#define GF_SELINUX_XATTR_KEY "security.selinux"
+
 #define WIPE(statp) do { typeof(*statp) z = {0,}; if (statp) *statp = z; } while (0)
 
 #define IS_EXT_FS(fs_name)          \
@@ -79,6 +75,12 @@ void trap (void);
          !strcmp (fs_name, "ext3") || \
          !strcmp (fs_name, "ext4"))
 
+/* Defining this here as it is needed by glusterd for setting
+ * nfs port in volume status.
+ */
+#define GF_NFS3_PORT    2049
+#define GF_CLIENT_PORT_CEILING 1024
+
 enum _gf_boolean
 {
 	_gf_false = 0,
@@ -97,7 +99,7 @@ enum _gf_client_pid
         GF_CLIENT_PID_MAX    =  0,
         GF_CLIENT_PID_GSYNCD = -1,
         GF_CLIENT_PID_HADOOP = -2,
-        GF_CLIENT_PID_MIN    = -3
+        GF_CLIENT_PID_DEFRAG = -3,
 };
 
 typedef enum _gf_boolean gf_boolean_t;
@@ -108,15 +110,14 @@ void gf_global_variable_init(void);
 
 in_addr_t gf_resolve_ip (const char *hostname, void **dnscache);
 
-void gf_log_volume_file (FILE *specfp);
-void gf_print_trace (int32_t signal);
-
-extern char *gf_fop_list[GF_FOP_MAXVALUE];
-extern char *gf_mgmt_list[GF_MGMT_MAXVALUE];
+void gf_log_dump_graph (FILE *specfp, glusterfs_graph_t *graph);
+void gf_print_trace (int32_t signal, glusterfs_ctx_t *ctx);
+int  gf_set_log_file_path (cmd_args_t *cmd_args);
 
 #define VECTORSIZE(count) (count * (sizeof (struct iovec)))
 
 #define STRLEN_0(str) (strlen(str) + 1)
+
 #define VALIDATE_OR_GOTO(arg,label)   do {				\
 		if (!arg) {						\
 			errno = EINVAL;					\
@@ -126,7 +127,7 @@ extern char *gf_mgmt_list[GF_MGMT_MAXVALUE];
                                           "invalid argument: " #arg);   \
 			goto label;					\
 		}							\
-	} while (0);
+	} while (0)
 
 #define GF_VALIDATE_OR_GOTO(name,arg,label)   do {                      \
 		if (!arg) {                                             \
@@ -135,7 +136,7 @@ extern char *gf_mgmt_list[GF_MGMT_MAXVALUE];
                                           "invalid argument: " #arg);	\
 			goto label;                                     \
 		}                                                       \
-	} while (0);
+	} while (0)
 
 #define GF_VALIDATE_OR_GOTO_WITH_ERROR(name, arg, label, errno, error) do { \
                 if (!arg) {                                                 \
@@ -144,15 +145,15 @@ extern char *gf_mgmt_list[GF_MGMT_MAXVALUE];
                                           "invalid argument: " #arg);   \
                         goto label;                                     \
                 }                                                       \
-        }while (0);
+        }while (0)
 
 #define GF_ASSERT_AND_GOTO_WITH_ERROR(name, arg, label, errno, error) do { \
                 if (!arg) {                                             \
-                        GF_ASSERT (0)                                   \
+                        GF_ASSERT (0);                                  \
                         errno = error;                                  \
                         goto label;                                     \
                 }                                                       \
-        }while (0);
+        }while (0)
 
 #define GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO(name,arg,label)               \
         do {                                                            \
@@ -163,7 +164,7 @@ extern char *gf_mgmt_list[GF_MGMT_MAXVALUE];
                                           "invalid argument: " #arg);	\
                         goto label;                                     \
                 }                                                       \
-	} while (0);
+	} while (0)
 
 #define GF_REMOVE_SLASH_FROM_PATH(path, string)                         \
         do {                                                            \
@@ -173,28 +174,44 @@ extern char *gf_mgmt_list[GF_MGMT_MAXVALUE];
                         if (string[i-1] == '/')                         \
                                 string[i-1] = '-';                      \
                 }                                                       \
-        } while (0);                                                    \
-
+        } while (0)
 
-#define GF_IF_INTERNAL_XATTR_GOTO(pattern, dict, trav, op_errno, label) \
-        do{                                                             \
+#define GF_IF_INTERNAL_XATTR_GOTO(pattern, dict, op_errno, label)       \
+        do {                                                            \
                 if (!dict) {                                            \
-                        gf_log (THIS->name, GF_LOG_ERROR,               \
+                        gf_log (this->name, GF_LOG_ERROR,               \
                                 "setxattr dict is null");               \
                         goto label;                                     \
                 }                                                       \
-                trav = dict->members_list;                              \
-                while (trav) {                                          \
-                        if (!fnmatch (pattern, trav->key, 0)) {         \
-                                gf_log (THIS->name, GF_LOG_ERROR,       \
-                                        "attempt to set internal"       \
-                                        " xattr: %s", trav->key);       \
-                                op_errno = EPERM;                       \
-                                goto label;                             \
-                        }                                               \
-                        trav = trav->next;                              \
+                if (dict_foreach_fnmatch (dict, pattern,                \
+                                          dict_null_foreach_fn,         \
+                                          NULL) > 0) {                  \
+                        op_errno = EPERM;                               \
+                        gf_log (this->name, GF_LOG_ERROR,               \
+                                "attempt to set internal"               \
+                                " xattr: %s: %s", pattern,              \
+                                strerror (op_errno));                   \
+                        goto label;                                     \
                 }                                                       \
-        } while(0);                                                     \
+        } while (0)
+
+#define GF_IF_NATIVE_XATTR_GOTO(pattern, key, op_errno, label)          \
+        do {                                                            \
+                if (!key) {                                             \
+                        gf_log (this->name, GF_LOG_ERROR,               \
+                                "no key for removexattr");              \
+                        goto label;                                     \
+                }                                                       \
+                if (!fnmatch (pattern, key, 0)) {                       \
+                        op_errno = EPERM;                               \
+                        gf_log (this->name, GF_LOG_ERROR,               \
+                                "attempt to remove internal "           \
+                                "xattr: %s: %s", key,                   \
+                                strerror (op_errno));                   \
+                        goto label;                                     \
+                }                                                       \
+        } while (0)
+
 
 #define GF_FILE_CONTENT_REQUESTED(_xattr_req,_content_limit) \
 	(dict_get_uint64 (_xattr_req, "glusterfs.content", _content_limit) == 0)
@@ -208,7 +225,7 @@ extern char *gf_mgmt_list[GF_MGMT_MAXVALUE];
                         gf_log_callingfn ("", GF_LOG_ERROR,             \
                                           "Assertion failed: " #x);     \
                 }                                                       \
-        } while (0);
+        } while (0)
 #endif
 
 #define GF_UUID_ASSERT(u) \
@@ -250,7 +267,7 @@ iov_length (const struct iovec *vector, int count)
 
 
 static inline struct iovec *
-iov_dup (struct iovec *vector, int count)
+iov_dup (const struct iovec *vector, int count)
 {
 	int           bytecount = 0;
 	int           i;
@@ -330,6 +347,65 @@ iov_unload (char *buf, const struct iovec *vector, int count)
 }
 
 
+static inline size_t
+iov_load (const struct iovec *vector, int count, char *buf, int size)
+{
+	size_t left = size;
+	size_t cp = 0;
+	int    ret = 0;
+	int    i = 0;
+
+	while (left && i < count) {
+		cp = min (vector[i].iov_len, left);
+		if (vector[i].iov_base != buf + (size - left))
+			memcpy (vector[i].iov_base, buf + (size - left), cp);
+		ret += cp;
+		left -= cp;
+		if (left)
+			i++;
+	}
+
+	return ret;
+}
+
+
+static inline size_t
+iov_copy (const struct iovec *dst, int dcnt,
+	  const struct iovec *src, int scnt)
+{
+	size_t  ret = 0;
+	size_t  left = 0;
+	size_t  min_i = 0;
+	int     s_i = 0, s_ii = 0;
+	int     d_i = 0, d_ii = 0;
+
+	ret = min (iov_length (dst, dcnt), iov_length (src, scnt));
+	left = ret;
+
+	while (left) {
+		min_i = min (dst[d_i].iov_len - d_ii, src[s_i].iov_len - s_ii);
+		memcpy (dst[d_i].iov_base + d_ii, src[s_i].iov_base + s_ii,
+			min_i);
+
+		d_ii += min_i;
+		if (d_ii == dst[d_i].iov_len) {
+			d_ii = 0;
+			d_i++;
+		}
+
+		s_ii += min_i;
+		if (s_ii == src[s_i].iov_len) {
+			s_ii = 0;
+			s_i++;
+		}
+
+		left -= min_i;
+	}
+
+	return ret;
+}
+
+
 static inline int
 mem_0filled (const char *buf, size_t size)
 {
@@ -375,19 +451,52 @@ memdup (const void *ptr, size_t size)
 	return newptr;
 }
 
+typedef enum {
+        gf_timefmt_default = 0,
+        gf_timefmt_FT = 0,  /* YYYY-MM-DD hh:mm:ss */
+        gf_timefmt_Ymd_T,   /* YYYY/MM-DD-hh:mm:ss */
+        gf_timefmt_bdT,     /* ddd DD hh:mm:ss */
+        gf_timefmt_F_HMS,   /* YYYY-MM-DD hhmmss */
+        gf_timefmt_last
+} gf_timefmts;
+
+static inline void
+gf_time_fmt (char *dst, size_t sz_dst, time_t utime, unsigned int fmt)
+{
+        extern void _gf_timestuff (gf_timefmts *, const char ***, const char ***);
+        static gf_timefmts timefmt_last = (gf_timefmts) -1;
+        static const char **fmts;
+        static const char **zeros;
+        struct tm tm;
+
+        if (timefmt_last == -1)
+                _gf_timestuff (&timefmt_last, &fmts, &zeros);
+        if (timefmt_last < fmt) fmt = gf_timefmt_default;
+        if (utime && gmtime_r (&utime, &tm) != NULL) {
+                strftime (dst, sz_dst, fmts[fmt], &tm);
+        } else {
+                strncpy (dst, "N/A", sz_dst);
+        }
+}
+
+int
+mkdir_p (char *path, mode_t mode, gf_boolean_t allow_symlinks);
 /*
  * rounds up nr to power of two. If nr is already a power of two, just returns
  * nr
  */
 
-int32_t gf_roundup_power_of_two (uint32_t nr);
+int
+gf_lstat_dir (const char *path, struct stat *stbuf_in);
+
+int32_t gf_roundup_power_of_two (int32_t nr);
 
 /*
  * rounds up nr to next power of two. If nr is already a power of two, next
  * power of two is returned.
  */
 
-int32_t gf_roundup_next_power_of_two (uint32_t nr);
+int32_t gf_roundup_next_power_of_two (int32_t nr);
 
 char *gf_trim (char *string);
 int gf_strsplit (const char *str, const char *delim,
@@ -421,9 +530,11 @@ int gf_string2uint32_base10 (const char *str, uint32_t *n);
 int gf_string2uint64_base10 (const char *str, uint64_t *n);
 
 int gf_string2bytesize (const char *str, uint64_t *n);
+int gf_string2percent_or_bytesize (const char *str, uint64_t *n,
+				   gf_boolean_t *is_percent);
 
 int gf_string2boolean (const char *str, gf_boolean_t *b);
-int gf_string2percent (const char *str, uint32_t *n);
+int gf_string2percent (const char *str, double *n);
 int gf_string2time (const char *str, uint32_t *n);
 
 int gf_lockfd (int fd);
@@ -442,20 +553,20 @@ void skip_word (char **str);
 char *get_nth_word (const char *str, int n);
 
 char valid_host_name (char *address, int length);
-char valid_ipv4_address (char *address, int length);
-char valid_ipv6_address (char *address, int length);
-char valid_internet_address (char *address);
+char valid_ipv4_address (char *address, int length, gf_boolean_t wildcard_acc);
+char valid_ipv6_address (char *address, int length, gf_boolean_t wildcard_acc);
+char valid_internet_address (char *address, gf_boolean_t wildcard_acc);
 char valid_ipv4_wildcard_check (char *address);
 char valid_ipv6_wildcard_check (char *address);
 char valid_wildcard_internet_address (char *address);
+gf_boolean_t gf_sock_union_equal_addr (union gf_sock_union *a,
+                                       union gf_sock_union *b);
 
 char *uuid_utoa (uuid_t uuid);
 char *uuid_utoa_r (uuid_t uuid, char *dst);
 char *lkowner_utoa (gf_lkowner_t *lkowner);
 char *lkowner_utoa_r (gf_lkowner_t *lkowner, char *dst, int len);
 
-void _get_md5_str (char *out_str, size_t outlen,
-                   const uint8_t *input, int n);
 void gf_array_insertionsort (void *a, int l, int r, size_t elem_size,
                              gf_cmp cmp);
 int gf_is_str_int (const char *value);
@@ -466,5 +577,18 @@ char *get_host_name (char *word, char **host);
 char *get_path_name (char *word, char **path);
 void gf_path_strip_trailing_slashes (char *path);
 uint64_t get_mem_size ();
-int gf_client_pid_check (gf_client_pid_t npid);
+int gf_strip_whitespace (char *str, int len);
+int gf_canonicalize_path (char *path);
+char *generate_glusterfs_ctx_id (void);
+char *gf_get_reserved_ports();
+int gf_process_reserved_ports (gf_boolean_t ports[]);
+gf_boolean_t gf_ports_reserved (char *blocked_port, gf_boolean_t *ports);
+int gf_get_hostname_from_ip (char *client_ip, char **hostname);
+gf_boolean_t gf_is_local_addr (char *hostname);
+gf_boolean_t gf_is_same_address (char *host1, char *host2);
+void md5_wrapper(const unsigned char *data, size_t len, char *md5);
+
+int gf_thread_create (pthread_t *thread, const pthread_attr_t *attr,
+		      void *(*start_routine)(void *), void *arg);
+
 #endif /* _COMMON_UTILS_H */
diff --git a/libglusterfs/src/compat-errno.c b/libglusterfs/src/compat-errno.c
index 2aeb3d805..fd5cc49ce 100644
--- a/libglusterfs/src/compat-errno.c
+++ b/libglusterfs/src/compat-errno.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
diff --git a/libglusterfs/src/compat-errno.h b/libglusterfs/src/compat-errno.h
index a61457960..65e52081d 100644
--- a/libglusterfs/src/compat-errno.h
+++ b/libglusterfs/src/compat-errno.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __COMPAT_ERRNO_H__
diff --git a/libglusterfs/src/compat.c b/libglusterfs/src/compat.c
index 20e165a7a..eb6d8d4b7 100644
--- a/libglusterfs/src/compat.c
+++ b/libglusterfs/src/compat.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -151,8 +142,7 @@ done:
                 *path = export_path;
         }
 out:
-        if (freeptr)
-                GF_FREE (freeptr);
+        GF_FREE (freeptr);
         if (ret && export_path)
                 GF_FREE (export_path);
 
@@ -204,8 +194,7 @@ solaris_xattr_resolve_path (const char *real_path, char **path)
                 *path = gf_strdup (xattr_path);
         }
 out:
-        if (export_path)
-                GF_FREE (export_path);
+        GF_FREE (export_path);
         if (*path)
                 return 0;
         else
@@ -239,8 +228,7 @@ solaris_setxattr(const char *path, const char* key, const char *value,
                                 path, errno);
                 ret = -1;
         }
-        if (mapped_path)
-                GF_FREE (mapped_path);
+        GF_FREE (mapped_path);
         return ret;
 }
 
@@ -306,8 +294,7 @@ solaris_listxattr(const char *path, char *list, size_t size)
                 close (attrdirfd);
         }
 out:
-        if (mapped_path)
-                GF_FREE (mapped_path);
+        GF_FREE (mapped_path);
         return len;
 }
 
@@ -389,8 +376,7 @@ solaris_removexattr(const char *path, const char* key)
                 ret = -1;
         }
 
-        if (mapped_path)
-                GF_FREE (mapped_path);
+        GF_FREE (mapped_path);
 
         return ret;
 }
@@ -430,8 +416,7 @@ solaris_getxattr(const char *path,
                         errno = ENODATA;
                 ret = -1;
         }
-        if (mapped_path)
-                GF_FREE (mapped_path);
+        GF_FREE (mapped_path);
         return ret;
 }
 
@@ -503,8 +488,7 @@ int solaris_unlink (const char *path)
         }
 
 out:
-        if (mapped_path)
-                GF_FREE (mapped_path);
+        GF_FREE (mapped_path);
 
         return  unlink (path);
 }
diff --git a/libglusterfs/src/compat.h b/libglusterfs/src/compat.h
index cb71d32ce..2bd982541 100644
--- a/libglusterfs/src/compat.h
+++ b/libglusterfs/src/compat.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __COMPAT_H__
@@ -41,6 +32,12 @@
 #include <linux/limits.h>
 #include <sys/xattr.h>
 #include <endian.h>
+#ifdef HAVE_FALLOC_H
+#include <linux/falloc.h>
+#else
+#define FALLOC_FL_KEEP_SIZE     0x01 /* default is extend size */
+#define FALLOC_FL_PUNCH_HOLE    0x02 /* de-allocates range */
+#endif
 
 
 #ifndef HAVE_LLISTXATTR
@@ -116,13 +113,6 @@ enum {
 #define F_SETLK64       F_SETLK
 #define F_SETLKW64      F_SETLKW
 
-#ifdef __NetBSD__
-char *basename_r(const char *);
-char *dirname_r(char *);
-
-#define basename(path) basename_r(path)
-#define dirname(path) dirname_r(path)
-#endif /* __NetBSD__ */
 #endif /* GF_BSD_HOST_OS */
 
 #ifdef GF_DARWIN_HOST_OS
@@ -350,6 +340,18 @@ dirent_size (struct dirent *entry)
         return size;
 }
 
+#ifdef THREAD_UNSAFE_BASENAME
+char *basename_r(const char *);
+#define basename(path) basename_r(path)
+#endif /* THREAD_UNSAFE_BASENAME */
+
+#ifdef THREAD_UNSAFE_DIRNAME
+char *dirname_r(char *path);
+#define dirname(path) dirname_r(path)
+#endif /* THREAD_UNSAFE_DIRNAME */
+
+int gf_mkostemp (char *tmpl, int suffixlen, int flags);
+#define mkostemp(tmpl, flags) gf_mkostemp(tmpl, 0, flags);
 
 #ifdef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC
 /* Linux, Solaris, Cygwin */
diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c
new file mode 100644
index 000000000..0082601d4
--- /dev/null
+++ b/libglusterfs/src/ctx.c
@@ -0,0 +1,48 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif /* !_CONFIG_H */
+
+#include <pthread.h>
+
+#include "glusterfs.h"
+
+glusterfs_ctx_t *
+glusterfs_ctx_new ()
+{
+        int               ret = 0;
+	glusterfs_ctx_t  *ctx = NULL;
+
+	/* no GF_CALLOC here, gf_acct_mem_set_enable is not
+	   yet decided at this point */
+        ctx = calloc (1, sizeof (*ctx));
+        if (!ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&ctx->graphs);
+	INIT_LIST_HEAD (&ctx->mempool_list);
+
+	ctx->daemon_pipe[0] = -1;
+	ctx->daemon_pipe[1] = -1;
+
+	ret = pthread_mutex_init (&ctx->lock, NULL);
+	if (ret) {
+		free (ctx);
+		ctx = NULL;
+	}
+out:
+	return ctx;
+}
+
diff --git a/libglusterfs/src/daemon.c b/libglusterfs/src/daemon.c
index 778196164..348e3ad40 100644
--- a/libglusterfs/src/daemon.c
+++ b/libglusterfs/src/daemon.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <fcntl.h>
diff --git a/libglusterfs/src/daemon.h b/libglusterfs/src/daemon.h
index bf1d67cca..80836a326 100644
--- a/libglusterfs/src/daemon.h
+++ b/libglusterfs/src/daemon.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _DAEMON_H
diff --git a/libglusterfs/src/defaults.c b/libglusterfs/src/defaults.c
index f3a8d2832..2ebb25150 100644
--- a/libglusterfs/src/defaults.c
+++ b/libglusterfs/src/defaults.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /* libglusterfs/src/defaults.c:
@@ -41,18 +32,19 @@
 int32_t
 default_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *dict, struct iatt *postparent)
+                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
         STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             dict, postparent);
+                             xdata, postparent);
         return 0;
 }
 
 int32_t
 default_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
@@ -60,37 +52,41 @@ default_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 default_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf,
+                      dict_t *xdata)
 {
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
         return 0;
 }
 
 int32_t
 default_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf)
+                       struct iatt *postbuf,
+                       dict_t *xdata)
 {
         STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
         return 0;
 }
 
 int32_t
 default_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno,
+                    dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int32_t
 default_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf)
+                      struct iatt *buf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf);
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf,
+                             xdata);
         return 0;
 }
 
@@ -99,10 +95,10 @@ int32_t
 default_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent);
+                             buf, preparent, postparent, xdata);
         return 0;
 }
 
@@ -110,30 +106,31 @@ int32_t
 default_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent);
+                             buf, preparent, postparent, xdata);
         return 0;
 }
 
 int32_t
 default_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent);
+                             postparent, xdata);
         return 0;
 }
 
 int32_t
 default_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent,
+                   dict_t *xdata)
 {
         STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent);
+                             postparent, xdata);
         return 0;
 }
 
@@ -142,10 +139,10 @@ int32_t
 default_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -154,10 +151,11 @@ int32_t
 default_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
                     struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent)
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
         STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent);
+                             postoldparent, prenewparent, postnewparent, xdata);
         return 0;
 }
 
@@ -166,10 +164,11 @@ int32_t
 default_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent,
+                  dict_t *xdata)
 {
         STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -178,28 +177,31 @@ int32_t
 default_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent,
+                    dict_t *xdata)
 {
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 int32_t
 default_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd)
+                  int32_t op_ret, int32_t op_errno, fd_t *fd,
+                  dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
 int32_t
 default_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                   dict_t *xdata)
 {
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
+                             stbuf, iobref, xdata);
         return 0;
 }
 
@@ -207,18 +209,20 @@ default_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 default_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf)
+                    struct iatt *postbuf,
+                    dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
         return 0;
 }
 
 
 int32_t
 default_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno,
+                   dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -227,59 +231,67 @@ default_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 default_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf,
+                   dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
         return 0;
 }
 
 int32_t
 default_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 int32_t
 default_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd)
+                     int32_t op_ret, int32_t op_errno, fd_t *fd,
+                     dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
 int32_t
 default_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno,
+                      dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int32_t
 default_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                    int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                    dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int32_t
 default_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno,
+                      dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int32_t
 default_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno,
+                       dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -287,93 +299,104 @@ default_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 default_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int32_t
 default_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 int32_t
 default_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 int32_t
 default_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int32_t
 default_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno)
+                         int32_t op_ret, int32_t op_errno,
+                         dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int32_t
 default_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno,
+                          dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int32_t
 default_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
         return 0;
 }
 
 int32_t
 default_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno,
+                     dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int32_t
 default_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno,
+                      dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int32_t
 default_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno,
+                     dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int32_t
 default_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno,
+                      dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -381,52 +404,86 @@ default_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 default_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                       uint8_t *strong_checksum)
+                       uint8_t *strong_checksum,
+                       dict_t *xdata)
 {
         STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum);
+                             strong_checksum, xdata);
         return 0;
 }
 
 
 int32_t
 default_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
         return 0;
 }
 
 
 int32_t
 default_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                      dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
         return 0;
 }
 
 int32_t
 default_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost)
+                     struct iatt *statpost,
+                     dict_t *xdata)
 {
         STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+                             statpost, xdata);
         return 0;
 }
 
 int32_t
 default_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                      struct iatt *statpost)
+                      struct iatt *statpost,
+                      dict_t *xdata)
 {
         STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+                             statpost, xdata);
         return 0;
 }
 
 int32_t
+default_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                      struct iatt *post, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, pre, post, xdata);
+	return 0;
+}
+
+int32_t
+default_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                    struct iatt *post, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, pre, post, xdata);
+	return 0;
+}
+
+int32_t
+default_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                    struct iatt *post, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, pre,
+                           post, xdata);
+        return 0;
+}
+
+
+int32_t
 default_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, char *spec_data)
 {
@@ -438,266 +495,271 @@ default_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 default_fgetxattr_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          const char *name)
+                          const char *name, dict_t *xdata)
 {
         STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name);
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
         return 0;
 }
 
 int32_t
 default_fsetxattr_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          dict_t *dict, int32_t flags)
+                          dict_t *dict, int32_t flags, dict_t *xdata)
 {
         STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags);
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
         return 0;
 }
 
 int32_t
 default_setxattr_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         dict_t *dict, int32_t flags)
+                         dict_t *dict, int32_t flags, dict_t *xdata)
 {
         STACK_WIND (frame, default_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags);
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
         return 0;
 }
 
 int32_t
-default_statfs_resume (call_frame_t *frame, xlator_t *this, loc_t *loc)
+default_statfs_resume (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         STACK_WIND (frame, default_statfs_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs, loc);
+                    FIRST_CHILD(this)->fops->statfs, loc, xdata);
         return 0;
 }
 
 int32_t
 default_fsyncdir_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                         int32_t flags)
+                         int32_t flags, dict_t *xdata)
 {
         STACK_WIND (frame, default_fsyncdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsyncdir, fd, flags);
+                    FIRST_CHILD(this)->fops->fsyncdir, fd, flags, xdata);
         return 0;
 }
 
 int32_t
 default_opendir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        fd_t *fd)
+                        fd_t *fd, dict_t *xdata)
 {
         STACK_WIND (frame, default_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd);
+                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
         return 0;
 }
 
 int32_t
-default_fstat_resume (call_frame_t *frame, xlator_t *this, fd_t *fd)
+default_fstat_resume (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
         return 0;
 }
 
 int32_t
 default_fsync_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      int32_t flags)
+                      int32_t flags, dict_t *xdata)
 {
         STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, flags);
+                    FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
         return 0;
 }
 
 int32_t
-default_flush_resume (call_frame_t *frame, xlator_t *this, fd_t *fd)
+default_flush_resume (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd);
+                    FIRST_CHILD(this)->fops->flush, fd, xdata);
         return 0;
 }
 
 int32_t
 default_writev_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
                        struct iovec *vector, int32_t count, off_t off,
-                       uint32_t flags, struct iobref *iobref)
+                       uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         STACK_WIND (frame, default_writev_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
-                    flags, iobref);
+                    flags, iobref, xdata);
         return 0;
 }
 
 int32_t
 default_readv_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      size_t size, off_t offset, uint32_t flags)
+                      size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags);
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
         return 0;
 }
 
 
 int32_t
 default_open_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     int32_t flags, fd_t *fd, int32_t wbflags)
+                     int32_t flags, fd_t *fd, dict_t *xdata)
 {
         STACK_WIND (frame, default_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
         return 0;
 }
 
 int32_t
 default_create_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+                       int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                       dict_t *xdata)
 {
         STACK_WIND (frame, default_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, fd,
-                    params);
+                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
+                    fd, xdata);
         return 0;
 }
 
 int32_t
 default_link_resume (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                     loc_t *newloc)
+                     loc_t *newloc, dict_t *xdata)
 {
         STACK_WIND (frame, default_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc);
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
         return 0;
 }
 
 int32_t
 default_rename_resume (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                       loc_t *newloc)
+                       loc_t *newloc, dict_t *xdata)
 {
         STACK_WIND (frame, default_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc);
+                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
         return 0;
 }
 
 
 int
 default_symlink_resume (call_frame_t *frame, xlator_t *this,
-                        const char *linkpath, loc_t *loc, dict_t *params)
+                        const char *linkpath, loc_t *loc, mode_t umask,
+                        dict_t *xdata)
 {
         STACK_WIND (frame, default_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, params);
+                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                    xdata);
         return 0;
 }
 
 int32_t
 default_rmdir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      int flags)
+                      int flags, dict_t *xdata)
 {
         STACK_WIND (frame, default_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags);
+                    FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
         return 0;
 }
 
 int32_t
-default_unlink_resume (call_frame_t *frame, xlator_t *this, loc_t *loc)
+default_unlink_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       int xflag, dict_t *xdata)
 {
         STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
         return 0;
 }
 
 int
 default_mkdir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      mode_t mode, dict_t *params)
+                      mode_t mode, mode_t umask, dict_t *xdata)
 {
         STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, params);
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
         return 0;
 }
 
 
 int
 default_mknod_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      mode_t mode, dev_t rdev, dict_t *parms)
+                      mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
         STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, parms);
+                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                    xdata);
         return 0;
 }
 
 int32_t
 default_readlink_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         size_t size)
+                         size_t size, dict_t *xdata)
 {
         STACK_WIND (frame, default_readlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink, loc, size);
+                    FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
         return 0;
 }
 
 
 int32_t
 default_access_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       int32_t mask)
+                       int32_t mask, dict_t *xdata)
 {
         STACK_WIND (frame, default_access_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access, loc, mask);
+                    FIRST_CHILD(this)->fops->access, loc, mask, xdata);
         return 0;
 }
 
 int32_t
 default_ftruncate_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          off_t offset)
+                          off_t offset, dict_t *xdata)
 {
         STACK_WIND (frame, default_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
         return 0;
 }
 
 int32_t
 default_getxattr_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         const char *name)
+                         const char *name, dict_t *xdata)
 {
         STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
         return 0;
 }
 
 
 int32_t
 default_xattrop_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        gf_xattrop_flags_t flags, dict_t *dict)
+                        gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         STACK_WIND (frame, default_xattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict);
+                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
         return 0;
 }
 
 int32_t
 default_fxattrop_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                         gf_xattrop_flags_t flags, dict_t *dict)
+                         gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         STACK_WIND (frame, default_fxattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict);
+                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
         return 0;
 }
 
 int32_t
 default_removexattr_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                            const char *name)
+                            const char *name, dict_t *xdata)
 {
         STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name);
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
         return 0;
 }
 
 int32_t
 default_fremovexattr_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                             const char *name)
+                             const char *name, dict_t *xdata)
 {
         STACK_WIND (frame, default_fremovexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr, fd, name);
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
         return 0;
 }
 
 int32_t
 default_lk_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   int32_t cmd, struct gf_flock *lock)
+                   int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         STACK_WIND (frame, default_lk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk, fd, cmd, lock);
+                    FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
         return 0;
 }
 
@@ -705,376 +767,431 @@ default_lk_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int32_t
 default_inodelk_resume (call_frame_t *frame, xlator_t *this,
                         const char *volume, loc_t *loc, int32_t cmd,
-                        struct gf_flock *lock)
+                        struct gf_flock *lock,
+                        dict_t *xdata)
 {
         STACK_WIND (frame, default_inodelk_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    volume, loc, cmd, lock);
+                    volume, loc, cmd, lock, xdata);
         return 0;
 }
 
 int32_t
 default_finodelk_resume (call_frame_t *frame, xlator_t *this,
                          const char *volume, fd_t *fd, int32_t cmd,
-                         struct gf_flock *lock)
+                         struct gf_flock *lock,
+                         dict_t *xdata)
 {
         STACK_WIND (frame, default_finodelk_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->finodelk,
-                    volume, fd, cmd, lock);
+                    volume, fd, cmd, lock, xdata);
         return 0;
 }
 
 int32_t
 default_entrylk_resume (call_frame_t *frame, xlator_t *this,
                         const char *volume, loc_t *loc, const char *basename,
-                        entrylk_cmd cmd, entrylk_type type)
+                        entrylk_cmd cmd, entrylk_type type,
+                        dict_t *xdata)
 {
         STACK_WIND (frame, default_entrylk_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
         return 0;
 }
 
 int32_t
 default_fentrylk_resume (call_frame_t *frame, xlator_t *this,
                          const char *volume, fd_t *fd, const char *basename,
-                         entrylk_cmd cmd, entrylk_type type)
+                         entrylk_cmd cmd, entrylk_type type,
+                         dict_t *xdata)
 {
         STACK_WIND (frame, default_fentrylk_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
+                    volume, fd, basename, cmd, type, xdata);
         return 0;
 }
 
 int32_t
 default_rchecksum_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          off_t offset, int32_t len)
+                          off_t offset, int32_t len,
+                          dict_t *xdata)
 {
         STACK_WIND (frame, default_rchecksum_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rchecksum, fd, offset, len);
+                    FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
         return 0;
 }
 
 
 int32_t
 default_readdir_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        size_t size, off_t off)
+                        size_t size, off_t off,
+                        dict_t *xdata)
 {
         STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir, fd, size, off);
+                    FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
         return 0;
 }
 
 
 int32_t
 default_readdirp_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                         size_t size, off_t off, dict_t *dict)
+                         size_t size, off_t off, dict_t *xdata)
 {
         STACK_WIND (frame, default_readdirp_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp, fd, size, off, dict);
+                    FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
         return 0;
 }
 
 int32_t
 default_setattr_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        struct iatt *stbuf, int32_t valid)
+                        struct iatt *stbuf, int32_t valid,
+                        dict_t *xdata)
 {
         STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid);
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
         return 0;
 }
 
 int32_t
 default_truncate_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         off_t offset)
+                         off_t offset,
+                         dict_t *xdata)
 {
         STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
         return 0;
 }
 
 int32_t
-default_stat_resume (call_frame_t *frame, xlator_t *this, loc_t *loc)
+default_stat_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
 {
         STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
         return 0;
 }
 
 int32_t
 default_lookup_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       dict_t *xattr_req)
+                       dict_t *xdata)
 {
         STACK_WIND (frame, default_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
         return 0;
 }
 
 int32_t
 default_fsetattr_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                         struct iatt *stbuf, int32_t valid)
+                         struct iatt *stbuf, int32_t valid,
+                         dict_t *xdata)
 {
         STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid);
+                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid, xdata);
+        return 0;
+}
+
+int32_t
+default_fallocate_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+			 int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
+{
+	STACK_WIND(frame, default_fallocate_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fallocate, fd, keep_size, offset, len,
+		   xdata);
+        return 0;
+}
+
+int32_t
+default_discard_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		       off_t offset, size_t len, dict_t *xdata)
+{
+	STACK_WIND(frame, default_discard_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->discard, fd, offset, len,
+		   xdata);
+        return 0;
+}
+
+int32_t
+default_zerofill_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       off_t offset, size_t len, dict_t *xdata)
+{
+        STACK_WIND(frame, default_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len,
+                   xdata);
         return 0;
 }
 
+
 /* FOPS */
 
 int32_t
 default_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name)
+                   const char *name, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
         return 0;
 }
 
 int32_t
 default_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                   int32_t flags)
+                   int32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags,
+                         xdata);
         return 0;
 }
 
 int32_t
 default_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                  int32_t flags)
+                  int32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, default_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                         xdata);
         return 0;
 }
 
 int32_t
-default_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+default_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        STACK_WIND (frame, default_statfs_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs, loc);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->statfs, loc, xdata);
         return 0;
 }
 
 int32_t
-default_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+default_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fsyncdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsyncdir, fd, flags);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fsyncdir, fd, flags, xdata);
         return 0;
 }
 
 int32_t
-default_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+default_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, default_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
         return 0;
 }
 
 int32_t
-default_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+default_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fstat, fd, xdata);
         return 0;
 }
 
 int32_t
-default_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+default_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, flags);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
         return 0;
 }
 
 int32_t
-default_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+default_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->flush, fd, xdata);
         return 0;
 }
 
 int32_t
 default_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 struct iovec *vector, int32_t count, off_t off, uint32_t flags,
-                struct iobref *iobref)
+                struct iobref *iobref, dict_t *xdata)
 {
-        STACK_WIND (frame, default_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
-                    flags, iobref);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->writev, fd, vector, count,
+                         off, flags, iobref, xdata);
         return 0;
 }
 
 int32_t
 default_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, uint32_t flags)
+               off_t offset, uint32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->readv, fd, size, offset,
+                         flags, xdata);
         return 0;
 }
 
 
 int32_t
 default_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              fd_t *fd, int32_t wbflags)
+              fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, default_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
         return 0;
 }
 
 int32_t
 default_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-                mode_t mode, fd_t *fd, dict_t *params)
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, default_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, fd,
-                    params);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->create, loc, flags, mode,
+                         umask, fd, xdata);
         return 0;
 }
 
 int32_t
-default_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+default_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        STACK_WIND (frame, default_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
         return 0;
 }
 
 int32_t
 default_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc)
+                loc_t *newloc, dict_t *xdata)
 {
-        STACK_WIND (frame, default_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->rename, oldloc, newloc,
+                         xdata);
         return 0;
 }
 
 
 int
 default_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                 loc_t *loc, dict_t *params)
+                 loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        STACK_WIND (frame, default_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, params);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->symlink, linkpath, loc,
+                         umask, xdata);
         return 0;
 }
 
 int32_t
-default_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+default_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+               dict_t *xdata)
 {
-        STACK_WIND (frame, default_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
         return 0;
 }
 
 int32_t
-default_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+default_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                dict_t *xdata)
 {
-        STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
         return 0;
 }
 
 int
 default_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-               dict_t *params)
+               mode_t umask, dict_t *xdata)
 {
-        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, params);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                         xdata);
         return 0;
 }
 
 
 int
 default_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-               dev_t rdev, dict_t *parms)
+               dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, parms);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->mknod, loc, mode, rdev,
+                         umask, xdata);
         return 0;
 }
 
 int32_t
-default_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+default_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size, dict_t *xdata)
 {
-        STACK_WIND (frame, default_readlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink, loc, size);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
         return 0;
 }
 
 
 int32_t
-default_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+default_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask, dict_t *xdata)
 {
-        STACK_WIND (frame, default_access_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access, loc, mask);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->access, loc, mask, xdata);
         return 0;
 }
 
 int32_t
-default_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+default_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
 {
-        STACK_WIND (frame, default_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
         return 0;
 }
 
 int32_t
 default_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  const char *name)
+                  const char *name, dict_t *xdata)
 {
-        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
         return 0;
 }
 
 
 int32_t
 default_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 gf_xattrop_flags_t flags, dict_t *dict)
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        STACK_WIND (frame, default_xattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->xattrop, loc, flags, dict,
+                         xdata);
         return 0;
 }
 
 int32_t
 default_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  gf_xattrop_flags_t flags, dict_t *dict)
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fxattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict,
+                         xdata);
         return 0;
 }
 
 int32_t
 default_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name)
+                     const char *name, dict_t *xdata)
 {
-        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->removexattr, loc, name,
+                         xdata);
         return 0;
 }
 
 int32_t
 default_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      const char *name)
+                      const char *name, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fremovexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr, fd, name);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fremovexattr, fd, name,
+                         xdata);
         return 0;
 }
 
 int32_t
 default_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            int32_t cmd, struct gf_flock *lock)
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_WIND (frame, default_lk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk, fd, cmd, lock);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
         return 0;
 }
 
@@ -1082,115 +1199,160 @@ default_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int32_t
 default_inodelk (call_frame_t *frame, xlator_t *this,
                  const char *volume, loc_t *loc, int32_t cmd,
-                 struct gf_flock *lock)
+                 struct gf_flock *lock,
+                 dict_t *xdata)
 {
-        STACK_WIND (frame, default_inodelk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    volume, loc, cmd, lock);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd,
+                         lock, xdata);
         return 0;
 }
 
 int32_t
 default_finodelk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
+                  const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock,
+                  dict_t *xdata)
 {
-        STACK_WIND (frame, default_finodelk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->finodelk,
-                    volume, fd, cmd, lock);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd,
+                         lock, xdata);
         return 0;
 }
 
 int32_t
 default_entrylk (call_frame_t *frame, xlator_t *this,
                  const char *volume, loc_t *loc, const char *basename,
-                 entrylk_cmd cmd, entrylk_type type)
+                 entrylk_cmd cmd, entrylk_type type,
+                 dict_t *xdata)
 {
-        STACK_WIND (frame, default_entrylk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->entrylk, volume, loc,
+                         basename, cmd, type, xdata);
         return 0;
 }
 
 int32_t
 default_fentrylk (call_frame_t *frame, xlator_t *this,
                   const char *volume, fd_t *fd, const char *basename,
-                  entrylk_cmd cmd, entrylk_type type)
+                  entrylk_cmd cmd, entrylk_type type,
+                  dict_t *xdata)
 {
-        STACK_WIND (frame, default_fentrylk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fentrylk, volume, fd,
+                         basename, cmd, type, xdata);
         return 0;
 }
 
 int32_t
 default_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                   int32_t len)
+                   int32_t len,
+                   dict_t *xdata)
 {
-        STACK_WIND (frame, default_rchecksum_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rchecksum, fd, offset, len);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->rchecksum, fd, offset, len,
+                         xdata);
         return 0;
 }
 
 
 int32_t
 default_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 size_t size, off_t off)
+                 size_t size, off_t off,
+                 dict_t *xdata)
 {
-        STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir, fd, size, off);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->readdir, fd, size, off,
+                         xdata);
         return 0;
 }
 
 
 int32_t
 default_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  size_t size, off_t off, dict_t *dict)
+                  size_t size, off_t off, dict_t *xdata)
 {
-        STACK_WIND (frame, default_readdirp_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp, fd, size, off, dict);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->readdirp, fd, size, off,
+                         xdata);
         return 0;
 }
 
 int32_t
 default_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 struct iatt *stbuf, int32_t valid)
+                 struct iatt *stbuf, int32_t valid,
+                 dict_t *xdata)
 {
-        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid);
+        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
+                         FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid,
+                         xdata);
         return 0;
 }
 
 int32_t
-default_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+default_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                  dict_t *xdata)
 {
-        STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
         return 0;
 }
 
 int32_t
-default_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+default_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xdata)
 {
-        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->stat, loc, xdata);
         return 0;
 }
 
 int32_t
 default_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                dict_t *xattr_req)
+                dict_t *xdata)
 {
-        STACK_WIND (frame, default_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->lookup, loc, xdata);
         return 0;
 }
 
 int32_t
 default_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  struct iatt *stbuf, int32_t valid)
+                  struct iatt *stbuf, int32_t valid,
+                  dict_t *xdata)
 {
-        STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid);
+        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
+                         FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid,
+                         xdata);
+        return 0;
+}
+
+int32_t
+default_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		  int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
+{
+	STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+			FIRST_CHILD(this)->fops->fallocate, fd, keep_size, offset,
+			len, xdata);
+	return 0;
+}
+
+int32_t
+default_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		off_t offset, size_t len, dict_t *xdata)
+{
+	STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+			FIRST_CHILD(this)->fops->discard, fd, offset, len,
+			xdata);
+	return 0;
+}
+
+int32_t
+default_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                off_t offset, size_t len, dict_t *xdata)
+{
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->zerofill, fd, offset, len,
+                        xdata);
         return 0;
 }
 
@@ -1198,6 +1360,8 @@ default_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int32_t
 default_forget (xlator_t *this, inode_t *inode)
 {
+        gf_log_callingfn (this->name, GF_LOG_WARNING, "xlator does not "
+                          "implement forget_cbk");
         return 0;
 }
 
@@ -1205,12 +1369,16 @@ default_forget (xlator_t *this, inode_t *inode)
 int32_t
 default_releasedir (xlator_t *this, fd_t *fd)
 {
+        gf_log_callingfn (this->name, GF_LOG_WARNING, "xlator does not "
+                          "implement releasedir_cbk");
         return 0;
 }
 
 int32_t
 default_release (xlator_t *this, fd_t *fd)
 {
+        gf_log_callingfn (this->name, GF_LOG_WARNING, "xlator does not "
+                          "implement release_cbk");
         return 0;
 }
 
@@ -1223,8 +1391,8 @@ int32_t
 default_getspec (call_frame_t *frame, xlator_t *this, const char *key,
                  int32_t flags)
 {
-        STACK_WIND (frame, default_getspec_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getspec, key, flags);
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->getspec, key, flags);
         return 0;
 }
 
@@ -1235,6 +1403,7 @@ default_notify (xlator_t *this, int32_t event, void *data, ...)
         switch (event)
         {
         case GF_EVENT_PARENT_UP:
+        case GF_EVENT_PARENT_DOWN:
         {
                 xlator_list_t *list = this->children;
 
diff --git a/libglusterfs/src/defaults.h b/libglusterfs/src/defaults.h
index f30bd8332..0747027bc 100644
--- a/libglusterfs/src/defaults.h
+++ b/libglusterfs/src/defaults.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /* libglusterfs/src/defaults.h:
@@ -31,6 +22,18 @@
 
 #include "xlator.h"
 
+int32_t default_notify (xlator_t *this,
+                        int32_t event,
+                        void *data,
+                        ...);
+
+int32_t default_forget (xlator_t *this, inode_t *inode);
+
+int32_t default_release (xlator_t *this, fd_t *fd);
+
+int32_t default_releasedir (xlator_t *this, fd_t *fd);
+
+
 /* Management Operations */
 
 int32_t default_getspec (call_frame_t *frame,
@@ -41,84 +44,85 @@ int32_t default_getspec (call_frame_t *frame,
 int32_t default_rchecksum (call_frame_t *frame,
                            xlator_t *this,
                            fd_t *fd, off_t offset,
-                           int32_t len);
+                           int32_t len, dict_t *xdata);
 
 /* FileSystem operations */
 int32_t default_lookup (call_frame_t *frame,
                         xlator_t *this,
                         loc_t *loc,
-                        dict_t *xattr_req);
+                        dict_t *xdata);
 
 int32_t default_stat (call_frame_t *frame,
                       xlator_t *this,
-                      loc_t *loc);
+                      loc_t *loc, dict_t *xdata);
 
 int32_t default_fstat (call_frame_t *frame,
                        xlator_t *this,
-                       fd_t *fd);
+                       fd_t *fd, dict_t *xdata);
 
 int32_t default_truncate (call_frame_t *frame,
                           xlator_t *this,
                           loc_t *loc,
-                          off_t offset);
+                          off_t offset, dict_t *xdata);
 
 int32_t default_ftruncate (call_frame_t *frame,
                            xlator_t *this,
                            fd_t *fd,
-                           off_t offset);
+                           off_t offset, dict_t *xdata);
 
 int32_t default_access (call_frame_t *frame,
                         xlator_t *this,
                         loc_t *loc,
-                        int32_t mask);
+                        int32_t mask, dict_t *xdata);
 
 int32_t default_readlink (call_frame_t *frame,
                           xlator_t *this,
                           loc_t *loc,
-                          size_t size);
+                          size_t size, dict_t *xdata);
 
-int32_t default_mknod (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, mode_t mode, dev_t rdev, dict_t *params);
+int32_t default_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
 
 int32_t default_mkdir (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, mode_t mode, dict_t *params);
+                       loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
 
 int32_t default_unlink (call_frame_t *frame,
                         xlator_t *this,
-                        loc_t *loc);
+                        loc_t *loc, int xflag, dict_t *xdata);
 
 int32_t default_rmdir (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, int flags);
+                       loc_t *loc, int xflag, dict_t *xdata);
 
 int32_t default_symlink (call_frame_t *frame, xlator_t *this,
-                         const char *linkpath, loc_t *loc, dict_t *params);
+                         const char *linkpath, loc_t *loc, mode_t umask,
+                         dict_t *xdata);
 
 int32_t default_rename (call_frame_t *frame,
                         xlator_t *this,
                         loc_t *oldloc,
-                        loc_t *newloc);
+                        loc_t *newloc, dict_t *xdata);
 
 int32_t default_link (call_frame_t *frame,
                       xlator_t *this,
                       loc_t *oldloc,
-                      loc_t *newloc);
+                      loc_t *newloc, dict_t *xdata);
 
 int32_t default_create (call_frame_t *frame, xlator_t *this,
                         loc_t *loc, int32_t flags, mode_t mode,
-                        fd_t *fd, dict_t *params);
+                        mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t default_open (call_frame_t *frame,
                       xlator_t *this,
                       loc_t *loc,
                       int32_t flags, fd_t *fd,
-                      int32_t wbflags);
+                      dict_t *xdata);
 
 int32_t default_readv (call_frame_t *frame,
                        xlator_t *this,
                        fd_t *fd,
                        size_t size,
                        off_t offset,
-                       uint32_t flags);
+                       uint32_t flags, dict_t *xdata);
 
 int32_t default_writev (call_frame_t *frame,
                         xlator_t *this,
@@ -127,218 +131,224 @@ int32_t default_writev (call_frame_t *frame,
                         int32_t count,
                         off_t offset,
                         uint32_t flags,
-                        struct iobref *iobref);
+                        struct iobref *iobref, dict_t *xdata);
 
 int32_t default_flush (call_frame_t *frame,
                        xlator_t *this,
-                       fd_t *fd);
+                       fd_t *fd, dict_t *xdata);
 
 int32_t default_fsync (call_frame_t *frame,
                        xlator_t *this,
                        fd_t *fd,
-                       int32_t datasync);
+                       int32_t datasync, dict_t *xdata);
 
 int32_t default_opendir (call_frame_t *frame,
                          xlator_t *this,
-                         loc_t *loc, fd_t *fd);
+                         loc_t *loc, fd_t *fd, dict_t *xdata);
 
 int32_t default_fsyncdir (call_frame_t *frame,
                           xlator_t *this,
                           fd_t *fd,
-                          int32_t datasync);
+                          int32_t datasync, dict_t *xdata);
 
 int32_t default_statfs (call_frame_t *frame,
                         xlator_t *this,
-                        loc_t *loc);
+                        loc_t *loc, dict_t *xdata);
 
 int32_t default_setxattr (call_frame_t *frame,
                           xlator_t *this,
                           loc_t *loc,
                           dict_t *dict,
-                          int32_t flags);
+                          int32_t flags, dict_t *xdata);
 
 int32_t default_getxattr (call_frame_t *frame,
                           xlator_t *this,
                           loc_t *loc,
-                          const char *name);
+                          const char *name, dict_t *xdata);
 
 int32_t default_fsetxattr (call_frame_t *frame,
                            xlator_t *this,
                            fd_t *fd,
                            dict_t *dict,
-                           int32_t flags);
+                           int32_t flags, dict_t *xdata);
 
 int32_t default_fgetxattr (call_frame_t *frame,
                            xlator_t *this,
                            fd_t *fd,
-                           const char *name);
+                           const char *name, dict_t *xdata);
 
 int32_t default_removexattr (call_frame_t *frame,
                              xlator_t *this,
                              loc_t *loc,
-                             const char *name);
+                             const char *name, dict_t *xdata);
 
 int32_t default_fremovexattr (call_frame_t *frame,
                               xlator_t *this,
                               fd_t *fd,
-                              const char *name);
+                              const char *name, dict_t *xdata);
 
 int32_t default_lk (call_frame_t *frame,
                     xlator_t *this,
                     fd_t *fd,
                     int32_t cmd,
-                    struct gf_flock *flock);
+                    struct gf_flock *flock, dict_t *xdata);
 
 int32_t default_inodelk (call_frame_t *frame, xlator_t *this,
                          const char *volume, loc_t *loc, int32_t cmd,
-                         struct gf_flock *flock);
+                         struct gf_flock *flock, dict_t *xdata);
 
 int32_t default_finodelk (call_frame_t *frame, xlator_t *this,
                           const char *volume, fd_t *fd, int32_t cmd,
-                          struct gf_flock *flock);
+                          struct gf_flock *flock, dict_t *xdata);
 
 int32_t default_entrylk (call_frame_t *frame, xlator_t *this,
                          const char *volume, loc_t *loc, const char *basename,
-                         entrylk_cmd cmd, entrylk_type type);
+                         entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 int32_t default_fentrylk (call_frame_t *frame, xlator_t *this,
                           const char *volume, fd_t *fd, const char *basename,
-                          entrylk_cmd cmd, entrylk_type type);
+                          entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 int32_t default_readdir (call_frame_t *frame,
                           xlator_t *this,
                           fd_t *fd,
-                          size_t size, off_t off);
+                          size_t size, off_t off, dict_t *xdata);
 
 int32_t default_readdirp (call_frame_t *frame,
                           xlator_t *this,
                           fd_t *fd,
-                          size_t size, off_t off, dict_t *dict);
+                          size_t size, off_t off, dict_t *xdata);
 
 int32_t default_xattrop (call_frame_t *frame,
                          xlator_t *this,
                          loc_t *loc,
                          gf_xattrop_flags_t flags,
-                         dict_t *dict);
+                         dict_t *dict, dict_t *xdata);
 
 int32_t default_fxattrop (call_frame_t *frame,
                           xlator_t *this,
                           fd_t *fd,
                           gf_xattrop_flags_t flags,
-                          dict_t *dict);
-
-int32_t default_notify (xlator_t *this,
-                        int32_t event,
-                        void *data,
-                        ...);
-
-int32_t default_forget (xlator_t *this,
-                        inode_t *inode);
-
-int32_t default_release (xlator_t *this,
-                         fd_t *fd);
-
-int32_t default_releasedir (xlator_t *this,
-                            fd_t *fd);
+                          dict_t *dict, dict_t *xdata);
 
 int32_t default_setattr (call_frame_t *frame,
                          xlator_t *this,
                          loc_t *loc,
                          struct iatt *stbuf,
-                         int32_t valid);
+                         int32_t valid, dict_t *xdata);
 
 int32_t default_fsetattr (call_frame_t *frame,
                           xlator_t *this,
                           fd_t *fd,
                           struct iatt *stbuf,
-                          int32_t valid);
+                          int32_t valid, dict_t *xdata);
+
+int32_t default_fallocate(call_frame_t *frame,
+			  xlator_t *this,
+			  fd_t *fd,
+			  int32_t keep_size, off_t offset,
+			  size_t len, dict_t *xdata);
+
+int32_t default_discard(call_frame_t *frame,
+			xlator_t *this,
+			fd_t *fd,
+			off_t offset,
+			size_t len, dict_t *xdata);
+
+int32_t default_zerofill(call_frame_t *frame,
+                        xlator_t *this,
+                        fd_t *fd,
+                        off_t offset,
+                        size_t len, dict_t *xdata);
+
 
 /* Resume */
-int32_t default_getspec (call_frame_t *frame,
-                         xlator_t *this,
-                         const char *key,
-                         int32_t flag);
+int32_t default_getspec_resume (call_frame_t *frame,
+                                xlator_t *this,
+                                const char *key,
+                                int32_t flag);
 
-int32_t default_rchecksum (call_frame_t *frame,
+int32_t default_rchecksum_resume (call_frame_t *frame,
                            xlator_t *this,
                            fd_t *fd, off_t offset,
-                           int32_t len);
+                           int32_t len, dict_t *xdata);
 
 /* FileSystem operations */
 int32_t default_lookup_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc,
-                        dict_t *xattr_req);
+                               xlator_t *this,
+                               loc_t *loc,
+                               dict_t *xdata);
 
 int32_t default_stat_resume (call_frame_t *frame,
                       xlator_t *this,
-                      loc_t *loc);
+                      loc_t *loc, dict_t *xdata);
 
 int32_t default_fstat_resume (call_frame_t *frame,
                        xlator_t *this,
-                       fd_t *fd);
+                       fd_t *fd, dict_t *xdata);
 
 int32_t default_truncate_resume (call_frame_t *frame,
                           xlator_t *this,
                           loc_t *loc,
-                          off_t offset);
+                          off_t offset, dict_t *xdata);
 
 int32_t default_ftruncate_resume (call_frame_t *frame,
                            xlator_t *this,
                            fd_t *fd,
-                           off_t offset);
+                           off_t offset, dict_t *xdata);
 
 int32_t default_access_resume (call_frame_t *frame,
                         xlator_t *this,
                         loc_t *loc,
-                        int32_t mask);
+                        int32_t mask, dict_t *xdata);
 
 int32_t default_readlink_resume (call_frame_t *frame,
                           xlator_t *this,
                           loc_t *loc,
-                          size_t size);
+                          size_t size, dict_t *xdata);
 
-int32_t default_mknod_resume (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, mode_t mode, dev_t rdev, dict_t *params);
+int32_t default_mknod_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                              mode_t mode, dev_t rdev, mode_t umask,
+                              dict_t *xdata);
 
-int32_t default_mkdir_resume (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, mode_t mode, dict_t *params);
+int32_t default_mkdir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                              mode_t mode, mode_t umask, dict_t *xdata);
 
 int32_t default_unlink_resume (call_frame_t *frame,
-                        xlator_t *this,
-                        loc_t *loc);
+                               xlator_t *this,
+                               loc_t *loc, int xflag, dict_t *xdata);
 
 int32_t default_rmdir_resume (call_frame_t *frame, xlator_t *this,
-                              loc_t *loc, int flags);
+                              loc_t *loc, int xflag, dict_t *xdata);
 
 int32_t default_symlink_resume (call_frame_t *frame, xlator_t *this,
-                         const char *linkpath, loc_t *loc, dict_t *params);
+                                const char *linkpath, loc_t *loc, mode_t umask,
+                                dict_t *xdata);
 
 int32_t default_rename_resume (call_frame_t *frame,
                         xlator_t *this,
                         loc_t *oldloc,
-                        loc_t *newloc);
+                        loc_t *newloc, dict_t *xdata);
 
 int32_t default_link_resume (call_frame_t *frame,
                       xlator_t *this,
                       loc_t *oldloc,
-                      loc_t *newloc);
+                      loc_t *newloc, dict_t *xdata);
 
 int32_t default_create_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, int32_t flags, mode_t mode,
-                        fd_t *fd, dict_t *params);
+                               loc_t *loc, int32_t flags, mode_t mode,
+                               mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t default_open_resume (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t *loc,
-                      int32_t flags, fd_t *fd,
-                      int32_t wbflags);
+                             xlator_t *this,
+                             loc_t *loc,
+                             int32_t flags, fd_t *fd, dict_t *xdata);
 
 int32_t default_readv_resume (call_frame_t *frame,
                               xlator_t *this,
                               fd_t *fd,
                               size_t size,
-                              off_t offset, uint32_t flags);
+                              off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t default_writev_resume (call_frame_t *frame,
                                xlator_t *this,
@@ -346,345 +356,362 @@ int32_t default_writev_resume (call_frame_t *frame,
                                struct iovec *vector,
                                int32_t count,
                                off_t offset, uint32_t flags,
-                               struct iobref *iobref);
+                               struct iobref *iobref, dict_t *xdata);
 
 int32_t default_flush_resume (call_frame_t *frame,
                        xlator_t *this,
-                       fd_t *fd);
+                       fd_t *fd, dict_t *xdata);
 
 int32_t default_fsync_resume (call_frame_t *frame,
                        xlator_t *this,
                        fd_t *fd,
-                       int32_t datasync);
+                       int32_t datasync, dict_t *xdata);
 
 int32_t default_opendir_resume (call_frame_t *frame,
                          xlator_t *this,
-                         loc_t *loc, fd_t *fd);
+                         loc_t *loc, fd_t *fd, dict_t *xdata);
 
 int32_t default_fsyncdir_resume (call_frame_t *frame,
                           xlator_t *this,
                           fd_t *fd,
-                          int32_t datasync);
+                          int32_t datasync, dict_t *xdata);
 
 int32_t default_statfs_resume (call_frame_t *frame,
                         xlator_t *this,
-                        loc_t *loc);
+                        loc_t *loc, dict_t *xdata);
 
 int32_t default_setxattr_resume (call_frame_t *frame,
                           xlator_t *this,
                           loc_t *loc,
                           dict_t *dict,
-                          int32_t flags);
+                          int32_t flags, dict_t *xdata);
 
 int32_t default_getxattr_resume (call_frame_t *frame,
                           xlator_t *this,
                           loc_t *loc,
-                          const char *name);
+                          const char *name, dict_t *xdata);
 
 int32_t default_fsetxattr_resume (call_frame_t *frame,
                            xlator_t *this,
                            fd_t *fd,
                            dict_t *dict,
-                           int32_t flags);
+                           int32_t flags, dict_t *xdata);
 
 int32_t default_fgetxattr_resume (call_frame_t *frame,
                            xlator_t *this,
                            fd_t *fd,
-                           const char *name);
+                           const char *name, dict_t *xdata);
 
 int32_t default_removexattr_resume (call_frame_t *frame,
                              xlator_t *this,
                              loc_t *loc,
-                             const char *name);
+                             const char *name, dict_t *xdata);
 
 int32_t default_fremovexattr_resume (call_frame_t *frame,
                                      xlator_t *this,
                                      fd_t *fd,
-                                     const char *name);
+                                     const char *name, dict_t *xdata);
 
 int32_t default_lk_resume (call_frame_t *frame,
                     xlator_t *this,
                     fd_t *fd,
                     int32_t cmd,
-                    struct gf_flock *flock);
+                    struct gf_flock *flock, dict_t *xdata);
 
 int32_t default_inodelk_resume (call_frame_t *frame, xlator_t *this,
                          const char *volume, loc_t *loc, int32_t cmd,
-                         struct gf_flock *flock);
+                         struct gf_flock *flock, dict_t *xdata);
 
 int32_t default_finodelk_resume (call_frame_t *frame, xlator_t *this,
                           const char *volume, fd_t *fd, int32_t cmd,
-                          struct gf_flock *flock);
+                          struct gf_flock *flock, dict_t *xdata);
 
 int32_t default_entrylk_resume (call_frame_t *frame, xlator_t *this,
                          const char *volume, loc_t *loc, const char *basename,
-                         entrylk_cmd cmd, entrylk_type type);
+                         entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 int32_t default_fentrylk_resume (call_frame_t *frame, xlator_t *this,
                           const char *volume, fd_t *fd, const char *basename,
-                          entrylk_cmd cmd, entrylk_type type);
+                          entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 int32_t default_readdir_resume (call_frame_t *frame,
                           xlator_t *this,
                           fd_t *fd,
-                          size_t size, off_t off);
+                          size_t size, off_t off, dict_t *xdata);
 
 int32_t default_readdirp_resume (call_frame_t *frame,
                                  xlator_t *this,
                                  fd_t *fd,
-                                 size_t size, off_t off, dict_t *dict);
+                                 size_t size, off_t off, dict_t *xdata);
 
 int32_t default_xattrop_resume (call_frame_t *frame,
                          xlator_t *this,
                          loc_t *loc,
                          gf_xattrop_flags_t flags,
-                         dict_t *dict);
+                         dict_t *dict, dict_t *xdata);
 
 int32_t default_fxattrop_resume (call_frame_t *frame,
                           xlator_t *this,
                           fd_t *fd,
                           gf_xattrop_flags_t flags,
-                          dict_t *dict);
+                          dict_t *dict, dict_t *xdata);
 int32_t default_rchecksum_resume (call_frame_t *frame,
                                   xlator_t *this,
                                   fd_t *fd, off_t offset,
-                                  int32_t len);
-
-int32_t default_notify (xlator_t *this,
-                        int32_t event,
-                        void *data,
-                        ...);
-
-int32_t default_forget (xlator_t *this,
-                        inode_t *inode);
-
-int32_t default_release (xlator_t *this,
-                         fd_t *fd);
-
-int32_t default_releasedir (xlator_t *this,
-                            fd_t *fd);
+                                  int32_t len, dict_t *xdata);
 
 int32_t default_setattr_resume (call_frame_t *frame,
                          xlator_t *this,
                          loc_t *loc,
                          struct iatt *stbuf,
-                         int32_t valid);
+                         int32_t valid, dict_t *xdata);
 
 int32_t default_fsetattr_resume (call_frame_t *frame,
                           xlator_t *this,
                           fd_t *fd,
                           struct iatt *stbuf,
-                          int32_t valid);
+                          int32_t valid, dict_t *xdata);
+
+int32_t default_fallocate_resume(call_frame_t *frame,
+				 xlator_t *this,
+				 fd_t *fd,
+				 int32_t keep_size, off_t offset,
+				 size_t len, dict_t *xdata);
+
+int32_t default_discard_resume(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       off_t offset,
+			       size_t len, dict_t *xdata);
+
+int32_t default_zerofill_resume(call_frame_t *frame,
+                               xlator_t *this,
+                               fd_t *fd,
+                               off_t offset,
+                               size_t len, dict_t *xdata);
+
 
 /* _cbk */
 
 int32_t
 default_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *dict, struct iatt *postparent);
+                    struct iatt *buf, dict_t *xdata, struct iatt *postparent);
 
 int32_t
 default_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf);
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata);
 
 
 int32_t
 default_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf);
+                      struct iatt *postbuf, dict_t *xdata);
 
 int32_t
 default_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf);
+                       struct iatt *postbuf, dict_t *xdata);
 
 int32_t
 default_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno);
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int32_t
 default_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf);
+                      struct iatt *buf, dict_t *xdata);
 
 
 int32_t
 default_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent);
+                   struct iatt *postparent, dict_t *xdata);
 
 int32_t
 default_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent);
+                   struct iatt *postparent, dict_t *xdata);
 
 int32_t
 default_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent);
+                    struct iatt *postparent, dict_t *xdata);
 
 int32_t
 default_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent);
+                   struct iatt *postparent, dict_t *xdata);
 
 
 int32_t
 default_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent);
+                     struct iatt *postparent, dict_t *xdata);
 
 
 int32_t
 default_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
                     struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent);
+                    struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata);
 
 
 int32_t
 default_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent);
+                  struct iatt *postparent, dict_t *xdata);
 
 
 int32_t
 default_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent);
+                    struct iatt *postparent, dict_t *xdata);
 
 int32_t
 default_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd);
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata);
 
 int32_t
 default_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref);
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata);
 
 
 int32_t
 default_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf);
+                    struct iatt *postbuf, dict_t *xdata);
 
 
 int32_t
 default_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno);
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 
 
 int32_t
 default_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf);
+                   struct iatt *postbuf, dict_t *xdata);
 
 int32_t
 default_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf);
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata);
 
 int32_t
 default_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd);
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata);
 
 int32_t
 default_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno);
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int32_t
 default_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf);
+                    int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata);
 
 
 int32_t
 default_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno);
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 
 int32_t
 default_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno);
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 
 
 int32_t
 default_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict);
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata);
 
 
 int32_t
 default_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict);
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata);
 
 int32_t
 default_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict);
+                     int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata);
 
 int32_t
 default_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict);
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata);
 
 
 int32_t
 default_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno);
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int32_t
 default_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno);
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int32_t
 default_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock);
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata);
 
 int32_t
 default_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno);
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 
 int32_t
 default_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno);
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int32_t
 default_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno);
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int32_t
 default_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno);
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 
 int32_t
 default_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                       uint8_t *strong_checksum);
+                       uint8_t *strong_checksum, dict_t *xdata);
 
 
 int32_t
 default_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries);
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata);
 
 
 int32_t
 default_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries);
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata);
 
 int32_t
 default_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost);
+                     struct iatt *statpost, dict_t *xdata);
 
 int32_t
 default_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                      struct iatt *statpost);
+                      struct iatt *statpost, dict_t *xdata);
+
+int32_t default_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+			      int32_t op_ret, int32_t op_errno, struct iatt *pre,
+			      struct iatt *post, dict_t *xdata);
+
+int32_t default_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+			    int32_t op_ret, int32_t op_errno, struct iatt *pre,
+			    struct iatt *post, dict_t *xdata);
+
+int32_t default_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                            struct iatt *post, dict_t *xdata);
 
 int32_t
 default_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
diff --git a/libglusterfs/src/dict.c b/libglusterfs/src/dict.c
index fd9dd1cd7..3b7ddce5e 100644
--- a/libglusterfs/src/dict.c
+++ b/libglusterfs/src/dict.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <unistd.h>
@@ -23,6 +14,7 @@
 #include <stdio.h>
 #include <inttypes.h>
 #include <limits.h>
+#include <fnmatch.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -36,23 +28,14 @@
 #include "logging.h"
 #include "compat.h"
 #include "byte-order.h"
-
-data_pair_t *
-get_new_data_pair ()
-{
-        data_pair_t *data_pair_ptr = NULL;
-
-        data_pair_ptr = (data_pair_t *) GF_CALLOC (1, sizeof (data_pair_t),
-                                                   gf_common_mt_data_pair_t);
-        return data_pair_ptr;
-}
+#include "globals.h"
 
 data_t *
 get_new_data ()
 {
         data_t *data = NULL;
 
-        data = (data_t *) GF_CALLOC (1, sizeof (data_t), gf_common_mt_data_t);
+        data = mem_get0 (THIS->ctx->dict_data_pool);
         if (!data) {
                 return NULL;
         }
@@ -64,19 +47,39 @@ get_new_data ()
 dict_t *
 get_new_dict_full (int size_hint)
 {
-        dict_t *dict = GF_CALLOC (1, sizeof (dict_t), gf_common_mt_dict_t);
+        dict_t *dict = mem_get0 (THIS->ctx->dict_pool);
 
         if (!dict) {
                 return NULL;
         }
 
         dict->hash_size = size_hint;
-        dict->members = GF_CALLOC (size_hint, sizeof (data_pair_t *),
-                                   gf_common_mt_data_pair_t);
-
-        if (!dict->members) {
-                GF_FREE (dict);
-                return NULL;
+        if (size_hint == 1) {
+                /*
+                 * This is the only case we ever see currently.  If we ever
+                 * need to support resizing the hash table, the resize function
+                 * will have to take into account the possibility that
+                 * "members" is not separately allocated (i.e. don't just call
+                 * realloc() blindly.
+                 */
+                dict->members = &dict->members_internal;
+        }
+        else {
+                /*
+                 * We actually need to allocate space for size_hint *pointers*
+                 * but we actually allocate space for one *structure*.  Since
+                 * a data_pair_t consists of five pointers, we're wasting four
+                 * pointers' worth for N=1, and will overrun what we allocated
+                 * for N>5.  If anybody ever starts using size_hint, we'll need
+                 * to fix this.
+                 */
+                GF_ASSERT (size_hint <=
+                           (sizeof(data_pair_t) / sizeof(data_pair_t *)));
+                dict->members = mem_get0 (THIS->ctx->dict_pair_pool);
+                if (!dict->members) {
+                        mem_put (dict);
+                        return NULL;
+                }
         }
 
         LOCK_INIT (&dict->lock);
@@ -143,13 +146,11 @@ data_destroy (data_t *data)
                                 else
                                         GF_FREE (data->data);
                         }
-                        if (data->vec)
-                                GF_FREE (data->vec);
                 }
 
                 data->len = 0xbabababa;
                 if (!data->is_const)
-                        GF_FREE (data);
+                        mem_put (data);
         }
 }
 
@@ -162,9 +163,7 @@ data_copy (data_t *old)
                 return NULL;
         }
 
-        data_t *newdata = (data_t *) GF_CALLOC (1, sizeof (*newdata),
-                                                gf_common_mt_data_t);
-
+        data_t *newdata = mem_get0 (THIS->ctx->dict_data_pool);
         if (!newdata) {
                 return NULL;
         }
@@ -176,12 +175,6 @@ data_copy (data_t *old)
                         if (!newdata->data)
                                 goto err_out;
                 }
-                if (old->vec) {
-                        newdata->vec = memdup (old->vec, old->len * (sizeof (void *) +
-                                                                     sizeof (size_t)));
-                        if (!newdata->vec)
-                                goto err_out;
-                }
         }
 
         LOCK_INIT (&newdata->lock);
@@ -189,11 +182,8 @@ data_copy (data_t *old)
 
 err_out:
 
-        if (newdata->data)
-                FREE (newdata->data);
-        if (newdata->vec)
-                FREE (newdata->vec);
-        GF_FREE (newdata);
+        FREE (newdata->data);
+        mem_put (newdata);
 
         return NULL;
 }
@@ -219,7 +209,7 @@ _dict_lookup (dict_t *this, char *key)
 }
 
 int32_t
-dict_lookup (dict_t *this, char *key, data_pair_t **data)
+dict_lookup (dict_t *this, char *key, data_t **data)
 {
         if (!this || !key || !data) {
                 gf_log_callingfn ("dict", GF_LOG_WARNING,
@@ -227,22 +217,22 @@ dict_lookup (dict_t *this, char *key, data_pair_t **data)
                 return -1;
         }
 
+        data_pair_t *tmp = NULL;
         LOCK (&this->lock);
         {
-                *data = _dict_lookup (this, key);
+                tmp = _dict_lookup (this, key);
         }
         UNLOCK (&this->lock);
-        if (*data)
-                return 0;
-        else
+
+        if (!tmp)
                 return -1;
 
+        *data = tmp->value;
+        return 0;
 }
 
 static int32_t
-_dict_set (dict_t *this,
-           char *key,
-           data_t *value)
+_dict_set (dict_t *this, char *key, data_t *value, gf_boolean_t replace)
 {
         int hashval;
         data_pair_t *pair;
@@ -261,34 +251,54 @@ _dict_set (dict_t *this,
 
         tmp = SuperFastHash (key, strlen (key));
         hashval = (tmp % this->hash_size);
-        pair = _dict_lookup (this, key);
 
-        if (pair) {
-                data_t *unref_data = pair->value;
-                pair->value = data_ref (value);
-                data_unref (unref_data);
-                if (key_free)
-                        GF_FREE (key);
-                /* Indicates duplicate key */
-                return 0;
-        }
-        pair = (data_pair_t *) GF_CALLOC (1, sizeof (*pair),
-                                          gf_common_mt_data_pair_t);
-        if (!pair) {
-                return -1;
-        }
+        /* Search for a existing key if 'replace' is asked for */
+        if (replace) {
+                pair = _dict_lookup (this, key);
 
-        pair->key = (char *) GF_CALLOC (1, strlen (key) + 1,
-                                        gf_common_mt_char);
-        if (!pair->key) {
-                GF_FREE (pair);
+                if (pair) {
+                        data_t *unref_data = pair->value;
+                        pair->value = data_ref (value);
+                        data_unref (unref_data);
+                        if (key_free)
+                                GF_FREE (key);
+                        /* Indicates duplicate key */
+                        return 0;
+                }
+        }
 
-                if (key_free)
-                        GF_FREE (key);
-                return -1;
+        if (this->free_pair_in_use) {
+                pair = mem_get0 (THIS->ctx->dict_pair_pool);
+                if (!pair) {
+                        if (key_free)
+                                GF_FREE (key);
+                        return -1;
+                }
+        }
+        else {
+                pair = &this->free_pair;
+                this->free_pair_in_use = _gf_true;
         }
 
-        strcpy (pair->key, key);
+        if (key_free) {
+                /* It's ours.  Use it. */
+                pair->key = key;
+                key_free = 0;
+        }
+        else {
+                pair->key = (char *) GF_CALLOC (1, strlen (key) + 1,
+                                                gf_common_mt_char);
+                if (!pair->key) {
+                        if (pair == &this->free_pair) {
+                                this->free_pair_in_use = _gf_false;
+                        }
+                        else {
+                                mem_put (pair);
+                        }
+                        return -1;
+                }
+                strcpy (pair->key, key);
+        }
         pair->value = data_ref (value);
 
         pair->hash_next = this->members[hashval];
@@ -321,7 +331,28 @@ dict_set (dict_t *this,
 
         LOCK (&this->lock);
 
-        ret = _dict_set (this, key, value);
+        ret = _dict_set (this, key, value, 1);
+
+        UNLOCK (&this->lock);
+
+        return ret;
+}
+
+
+int32_t
+dict_add (dict_t *this, char *key, data_t *value)
+{
+        int32_t ret;
+
+        if (!this || !value) {
+                gf_log_callingfn ("dict", GF_LOG_WARNING,
+                                  "!this || !value for key=%s", key);
+                return -1;
+        }
+
+        LOCK (&this->lock);
+
+        ret = _dict_set (this, key, value, 0);
 
         UNLOCK (&this->lock);
 
@@ -385,7 +416,12 @@ dict_del (dict_t *this, char *key)
                                 pair->next->prev = pair->prev;
 
                         GF_FREE (pair->key);
-                        GF_FREE (pair);
+                        if (pair == &this->free_pair) {
+                                this->free_pair_in_use = _gf_false;
+                        }
+                        else {
+                                mem_put (pair);
+                        }
                         this->count--;
                         break;
                 }
@@ -416,19 +452,21 @@ dict_destroy (dict_t *this)
                 pair = pair->next;
                 data_unref (prev->value);
                 GF_FREE (prev->key);
-                GF_FREE (prev);
+                if (prev != &this->free_pair) {
+                        mem_put (prev);
+                }
                 prev = pair;
         }
 
-        GF_FREE (this->members);
+        if (this->members != &this->members_internal) {
+                mem_put (this->members);
+        }
 
-        if (this->extra_free)
-                GF_FREE (this->extra_free);
-        if (this->extra_stdfree)
-                free (this->extra_stdfree);
+        GF_FREE (this->extra_free);
+        free (this->extra_stdfree);
 
         if (!this->is_static)
-                GF_FREE (this);
+                mem_put (this);
 
         return;
 }
@@ -509,236 +547,6 @@ data_ref (data_t *this)
         return this;
 }
 
-/*
-  Serialization format:
-  ----
-  Count:8
-  Key_len:8:Value_len:8
-  Key
-  Value
-  .
-  .
-  .
-*/
-
-int32_t
-dict_serialized_length_old (dict_t *this)
-{
-
-        if (!this) {
-                gf_log_callingfn ("dict", GF_LOG_WARNING, "dict is NULL");
-                return -1;
-        }
-
-        int32_t len = 9; /* count + \n */
-        int32_t count = this->count;
-        data_pair_t *pair = this->members_list;
-
-        while (count) {
-                len += 18;
-                len += strlen (pair->key) + 1;
-                if (pair->value->vec) {
-                        int i;
-                        for (i=0; i<pair->value->len; i++) {
-                                len += pair->value->vec[i].iov_len;
-                        }
-                } else {
-                        len += pair->value->len;
-                }
-                pair = pair->next;
-                count--;
-        }
-
-        return len;
-}
-
-int32_t
-dict_serialize_old (dict_t *this, char *buf)
-{
-        if (!this || !buf) {
-                gf_log_callingfn ("dict", GF_LOG_WARNING, "dict is NULL");
-                return -1;
-        }
-
-        data_pair_t *pair = this->members_list;
-        int32_t count = this->count;
-        uint64_t dcount = this->count;
-
-        // FIXME: magic numbers
-
-        sprintf (buf, "%08"PRIx64"\n", dcount);
-        buf += 9;
-        while (count) {
-                uint64_t keylen = strlen (pair->key) + 1;
-                uint64_t vallen = pair->value->len;
-
-                sprintf (buf, "%08"PRIx64":%08"PRIx64"\n", keylen, vallen);
-                buf += 18;
-                memcpy (buf, pair->key, keylen);
-                buf += keylen;
-                memcpy (buf, pair->value->data, pair->value->len);
-                buf += pair->value->len;
-                pair = pair->next;
-                count--;
-        }
-        return (0);
-}
-
-
-dict_t *
-dict_unserialize_old (char *buf, int32_t size, dict_t **fill)
-{
-        int32_t ret = 0;
-        int32_t cnt = 0;
-
-        if (!buf || !fill || !(*fill)) {
-                gf_log_callingfn ("dict", GF_LOG_WARNING, "buf is NULL");
-                return NULL;
-        }
-
-        uint64_t count;
-        ret = sscanf (buf, "%"SCNx64"\n", &count);
-        (*fill)->count = 0;
-
-        if (!ret){
-                gf_log ("dict", GF_LOG_ERROR, "sscanf on buf failed");
-                goto err;
-        }
-        buf += 9;
-
-        if (count == 0) {
-                gf_log ("dict", GF_LOG_ERROR, "count == 0");
-                goto err;
-        }
-
-        for (cnt = 0; cnt < count; cnt++) {
-                data_t *value = NULL;
-                char *key = NULL;
-                uint64_t key_len, value_len;
-
-                ret = sscanf (buf, "%"SCNx64":%"SCNx64"\n", &key_len, &value_len);
-                if (ret != 2) {
-                        gf_log ("dict", GF_LOG_ERROR,
-                                "sscanf for key_len and value_len failed");
-                        goto err;
-                }
-                buf += 18;
-
-                key = buf;
-                buf += key_len;
-
-                value = get_new_data ();
-                value->len = value_len;
-                value->data = buf;
-                value->is_static = 1;
-                buf += value_len;
-
-                dict_set (*fill, key, value);
-        }
-
-        goto ret;
-
-err:
-        GF_FREE (*fill);
-        *fill = NULL;
-
-ret:
-        return *fill;
-}
-
-
-int32_t
-dict_iovec_len (dict_t *this)
-{
-        if (!this) {
-                gf_log_callingfn ("dict", GF_LOG_WARNING, "dict is NULL");
-                return -1;
-        }
-
-        int32_t len = 0;
-        data_pair_t *pair = this->members_list;
-
-        len++; /* initial header */
-        while (pair) {
-                len++; /* pair header */
-                len++; /* key */
-
-                if (pair->value->vec)
-                        len += pair->value->len;
-                else
-                        len++;
-                pair = pair->next;
-        }
-
-        return len;
-}
-
-int32_t
-dict_to_iovec (dict_t *this,
-               struct iovec *vec,
-               int32_t count)
-{
-        if (!this || !vec) {
-                gf_log_callingfn ("dict", GF_LOG_WARNING, "dict is NULL");
-                return -1;
-        }
-
-        int32_t i = 0;
-        data_pair_t *pair = this->members_list;
-
-        vec[0].iov_len = 9;
-        if (vec[0].iov_base)
-                sprintf (vec[0].iov_base,
-                         "%08"PRIx64"\n",
-                         (int64_t)this->count);
-        i++;
-
-        while (pair) {
-                int64_t keylen = strlen (pair->key) + 1;
-                int64_t vallen = 0;
-
-                if (pair->value->vec) {
-                        int i;
-
-                        for (i=0; i<pair->value->len; i++) {
-                                vallen += pair->value->vec[i].iov_len;
-                        }
-                } else {
-                        vallen = pair->value->len;
-                }
-
-                vec[i].iov_len = 18;
-                if (vec[i].iov_base)
-                        sprintf (vec[i].iov_base,
-                                 "%08"PRIx64":%08"PRIx64"\n",
-                                 keylen,
-                                 vallen);
-                i++;
-
-                vec[i].iov_len = keylen;
-                vec[i].iov_base = pair->key;
-                i++;
-
-                if (pair->value->vec) {
-                        int k;
-
-                        for (k=0; k<pair->value->len; k++) {
-                                vec[i].iov_len = pair->value->vec[k].iov_len;
-                                vec[i].iov_base = pair->value->vec[k].iov_base;
-                                i++;
-                        }
-                } else {
-                        vec[i].iov_len = pair->value->len;
-                        vec[i].iov_base = pair->value->data;
-                        i++;
-                }
-
-                pair = pair->next;
-        }
-
-        return 0;
-}
-
 data_t *
 int_to_data (int64_t value)
 {
@@ -1088,6 +896,8 @@ data_to_int32 (data_t *data)
 int16_t
 data_to_int16 (data_t *data)
 {
+        int16_t value = 0;
+
         if (!data) {
                 gf_log_callingfn ("dict", GF_LOG_WARNING, "data is NULL");
                 return -1;
@@ -1100,14 +910,25 @@ data_to_int16 (data_t *data)
         memcpy (str, data->data, data->len);
         str[data->len] = '\0';
 
-        return strtol (str, NULL, 0);
+        errno = 0;
+        value = strtol (str, NULL, 0);
+
+        if ((value > SHRT_MAX) || (value < SHRT_MIN)) {
+                errno = ERANGE;
+                gf_log_callingfn ("dict", GF_LOG_WARNING,
+                                  "Error in data conversion: "
+                                  "detected overflow");
+                return -1;
+        }
+
+        return (int16_t)value;
 }
 
 
 int8_t
 data_to_int8 (data_t *data)
 {
-	int32_t value = 0;
+        int8_t value = 0;
 
         if (!data) {
                 gf_log_callingfn ("dict", GF_LOG_WARNING, "data is NULL");
@@ -1121,15 +942,16 @@ data_to_int8 (data_t *data)
         memcpy (str, data->data, data->len);
         str[data->len] = '\0';
 
-	errno = 0;
-	value = strtol (str, NULL, 0);
+        errno = 0;
+        value = strtol (str, NULL, 0);
 
-	if ((SCHAR_MAX > value) || (SCHAR_MIN < value)) {
-		errno = ERANGE;
+        if ((value > SCHAR_MAX) || (value < SCHAR_MIN)) {
+                errno = ERANGE;
                 gf_log_callingfn ("dict", GF_LOG_WARNING,
-				  "Error in data conversion: detected overflow");
+                                  "Error in data conversion: "
+                                  "detected overflow");
                 return -1;
-	}
+        }
 
         return (int8_t)value;
 }
@@ -1169,6 +991,8 @@ data_to_uint32 (data_t *data)
 uint16_t
 data_to_uint16 (data_t *data)
 {
+	uint16_t value = 0;
+
         if (!data)
                 return -1;
 
@@ -1179,7 +1003,18 @@ data_to_uint16 (data_t *data)
         memcpy (str, data->data, data->len);
         str[data->len] = '\0';
 
-        return strtol (str, NULL, 0);
+	errno = 0;
+	value = strtol (str, NULL, 0);
+
+	if ((USHRT_MAX - value) < 0) {
+		errno = ERANGE;
+		gf_log_callingfn ("dict", GF_LOG_WARNING,
+				  "Error in data conversion: "
+				  "overflow detected");
+		return -1;
+	}
+
+        return (uint16_t)value;
 }
 
 uint8_t
@@ -1243,47 +1078,141 @@ data_to_bin (data_t *data)
         return data->data;
 }
 
-void
+int
+dict_null_foreach_fn (dict_t *d, char *k,
+                      data_t *v, void *tmp)
+{
+        return 0;
+}
+
+int
 dict_foreach (dict_t *dict,
-              void (*fn)(dict_t *this,
-                         char *key,
-                         data_t *value,
-                         void *data),
+              int (*fn)(dict_t *this,
+                        char *key,
+                        data_t *value,
+                        void *data),
               void *data)
 {
         if (!dict) {
                 gf_log_callingfn ("dict", GF_LOG_WARNING,
                                   "dict is NULL");
-                return;
+                return -1;
         }
 
-        data_pair_t *pairs = dict->members_list;
-        data_pair_t *next = NULL;
+        int          ret   = -1;
+        data_pair_t *pairs = NULL;
+        data_pair_t *next  = NULL;
 
+        pairs = dict->members_list;
         while (pairs) {
                 next = pairs->next;
-                fn (dict, pairs->key, pairs->value, data);
+                ret = fn (dict, pairs->key, pairs->value, data);
+                if (ret == -1)
+                        return -1;
                 pairs = next;
         }
+
+        return 0;
+}
+
+/* return values:
+   -1 = failure,
+    0 = no matches found,
+   +n = n number of matches
+*/
+int
+dict_foreach_fnmatch (dict_t *dict, char *pattern,
+                      int (*fn)(dict_t *this,
+                                char *key,
+                                data_t *value,
+                                void *data),
+                      void *data)
+{
+        if (!dict) {
+                gf_log_callingfn ("dict", GF_LOG_WARNING,
+                                  "dict is NULL");
+                return 0;
+        }
+
+        int          ret = -1;
+        int          count = 0;
+        data_pair_t *pairs = NULL;
+        data_pair_t *next  = NULL;
+
+        pairs = dict->members_list;
+        while (pairs) {
+                next = pairs->next;
+                if (!fnmatch (pattern, pairs->key, 0)) {
+                        ret = fn (dict, pairs->key, pairs->value, data);
+                        if (ret == -1)
+                                return -1;
+                        count++;
+                }
+                pairs = next;
+        }
+
+        return count;
 }
 
 
-static void
+/**
+ * dict_keys_join - pack the keys of the dictionary in a buffer.
+ *
+ * @value     : buffer in which the keys will be packed (can be NULL)
+ * @size      : size of the buffer which is sent (can be 0, in which case buffer
+ *              is not packed but only length is returned)
+ * @dict      : dictionary of which all the keys will be packed
+ * @filter_fn : keys matched in filter_fn() is counted.
+ *
+ * @return : @length of string after joining keys.
+ *
+ */
+
+int
+dict_keys_join (void *value, int size, dict_t *dict,
+                int (*filter_fn)(char *k))
+{
+	int          len = 0;
+        data_pair_t *pairs = NULL;
+        data_pair_t *next  = NULL;
+
+        pairs = dict->members_list;
+        while (pairs) {
+                next = pairs->next;
+
+                if (filter_fn && filter_fn (pairs->key)){
+		    pairs = next;
+		    continue;
+		}
+
+		if (value && (size > len))
+			strncpy (value + len, pairs->key, size - len);
+
+                len += (strlen (pairs->key) + 1);
+
+                pairs = next;
+        }
+
+	return len;
+}
+
+static int
 _copy (dict_t *unused,
        char *key,
        data_t *value,
        void *newdict)
 {
-        dict_set ((dict_t *)newdict, key, (value));
+        return dict_set ((dict_t *)newdict, key, (value));
 }
 
-static void
+static int
 _remove (dict_t *dict,
          char *key,
          data_t *value,
          void *unused)
 {
         dict_del ((dict_t *)dict, key);
+        return 0;
 }
 
 
@@ -2330,7 +2259,6 @@ _dict_serialized_length (dict_t *this)
         int ret            = -EINVAL;
         int count          = 0;
         int len            = 0;
-        int i              = 0;
         data_pair_t * pair = NULL;
 
         len = DICT_HDR_LEN;
@@ -2365,28 +2293,15 @@ _dict_serialized_length (dict_t *this)
                         goto out;
                 }
 
-                if (pair->value->vec) {
-                        for (i = 0; i < pair->value->len; i++) {
-                                if (pair->value->vec[i].iov_len < 0) {
-                                        gf_log ("dict", GF_LOG_ERROR,
-                                                "iov_len (%"GF_PRI_SIZET") < 0!",
-                                                pair->value->vec[i].iov_len);
-                                        goto out;
-                                }
-
-                                len += pair->value->vec[i].iov_len;
-                        }
-                } else {
-                        if (pair->value->len < 0) {
-                                gf_log ("dict", GF_LOG_ERROR,
-                                        "value->len (%d) < 0",
-                                        pair->value->len);
-                                goto out;
-                        }
-
-                        len += pair->value->len;
+                if (pair->value->len < 0) {
+                        gf_log ("dict", GF_LOG_ERROR,
+                                "value->len (%d) < 0",
+                                pair->value->len);
+                        goto out;
                 }
 
+                len += pair->value->len;
+
                 pair = pair->next;
                 count--;
         }
@@ -2661,6 +2576,7 @@ dict_unserialize (char *orig_buf, int32_t size, dict_t **fill)
                                           "available (%lu) < required (%lu)",
                                           (long)(orig_buf + size),
                                           (long)(buf + vallen));
+                        goto out;
                 }
                 value = get_new_data ();
                 value->len  = vallen;
@@ -2668,7 +2584,7 @@ dict_unserialize (char *orig_buf, int32_t size, dict_t **fill)
                 value->is_static = 0;
                 buf += vallen;
 
-                dict_set (*fill, key, value);
+                dict_add (*fill, key, value);
         }
 
         ret = 0;
@@ -2689,7 +2605,7 @@ out:
  */
 
 int32_t
-dict_allocate_and_serialize (dict_t *this, char **buf, size_t *length)
+dict_allocate_and_serialize (dict_t *this, char **buf, u_int *length)
 {
         int           ret    = -EINVAL;
         ssize_t       len = 0;
@@ -2826,3 +2742,40 @@ dict_serialize_value_with_delim (dict_t *this, char *buf, int32_t *serz_len,
 out:
         return ret;
 }
+
+void
+dict_dump (dict_t *this)
+{
+        int          ret     = 0;
+        int          dumplen = 0;
+        data_pair_t *trav    = NULL;
+        char         dump[64*1024]; /* This is debug only, hence
+                                       performance should not matter */
+
+        if (!this) {
+                gf_log_callingfn ("dict", GF_LOG_WARNING, "dict NULL");
+                goto out;
+        }
+
+        dump[0] = '\0'; /* the array is not initialized to '\0' */
+
+        /* There is a possibility of issues if data is binary, ignore it
+           for now as debugging is more important */
+        for (trav = this->members_list; trav; trav = trav->next) {
+                ret = snprintf (&dump[dumplen], ((64*1024) - dumplen - 1),
+                                "(%s:%s)", trav->key, trav->value->data);
+                if ((ret == -1) || !ret)
+                        break;
+
+                dumplen += ret;
+                /* snprintf doesn't append a trailing '\0', add it here */
+                dump[dumplen] = '\0';
+        }
+
+        if (dumplen)
+                gf_log_callingfn ("dict", GF_LOG_INFO,
+                                  "dict=%p (%s)", this, dump);
+
+out:
+        return;
+}
diff --git a/libglusterfs/src/dict.h b/libglusterfs/src/dict.h
index 8e81e539b..9b41b5a7d 100644
--- a/libglusterfs/src/dict.h
+++ b/libglusterfs/src/dict.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _DICT_H
@@ -38,13 +29,11 @@ typedef struct _data_pair data_pair_t;
 
 #define GF_PROTOCOL_DICT_SERIALIZE(this,from_dict,to,len,ope,labl) do { \
                 int    ret     = 0;                                     \
-                size_t dictlen = 0;                                     \
                                                                         \
                 if (!from_dict)                                         \
                         break;                                          \
                                                                         \
-                ret = dict_allocate_and_serialize (from_dict, to,       \
-                                                   &dictlen);           \
+                ret = dict_allocate_and_serialize (from_dict, to, &len);\
                 if (ret < 0) {                                          \
                         gf_log (this->name, GF_LOG_WARNING,             \
                                 "failed to get serialized dict (%s)",   \
@@ -52,32 +41,25 @@ typedef struct _data_pair data_pair_t;
                         ope = EINVAL;                                   \
                         goto labl;                                      \
                 }                                                       \
-                len = dictlen;                                          \
         } while (0)
 
 
 #define GF_PROTOCOL_DICT_UNSERIALIZE(xl,to,buff,len,ret,ope,labl) do {  \
-                char *buf = NULL;                                       \
                 if (!len)                                               \
                         break;                                          \
                 to = dict_new();                                        \
                 GF_VALIDATE_OR_GOTO (xl->name, to, labl);               \
                                                                         \
-                buf = memdup (buff, len);                               \
-                GF_VALIDATE_OR_GOTO (xl->name, buf, labl);              \
-                                                                        \
-                ret = dict_unserialize (buf, len, &to);                 \
+                ret = dict_unserialize (buff, len, &to);                 \
                 if (ret < 0) {                                          \
                         gf_log (xl->name, GF_LOG_WARNING,               \
                                 "failed to unserialize dictionary (%s)", \
                                 (#to));                                 \
                                                                         \
                         ope = EINVAL;                                   \
-                        GF_FREE (buf);                                  \
                         goto labl;                                      \
                 }                                                       \
                                                                         \
-                to->extra_free = buf;                                   \
         } while (0)
 
 struct _data {
@@ -85,7 +67,6 @@ struct _data {
         unsigned char  is_const:1;
         unsigned char  is_stdalloc:1;
         int32_t        len;
-        struct iovec  *vec;
         char          *data;
         int32_t        refcount;
         gf_lock_t      lock;
@@ -109,13 +90,20 @@ struct _dict {
         char           *extra_free;
         char           *extra_stdfree;
         gf_lock_t       lock;
+        data_pair_t    *members_internal;
+        data_pair_t     free_pair;
+        gf_boolean_t    free_pair_in_use;
 };
 
 
 int32_t is_data_equal (data_t *one, data_t *two);
 void data_destroy (data_t *data);
 
+/* function to set a key/value pair (overwrite existing if matches the key */
 int32_t dict_set (dict_t *this, char *key, data_t *value);
+/* function to set a new key/value pair (without checking for duplicate) */
+int32_t dict_add (dict_t *this, char *key, data_t *value);
+
 data_t *dict_get (dict_t *this, char *key);
 void dict_del (dict_t *this, char *key);
 int dict_reset (dict_t *dict);
@@ -124,10 +112,7 @@ int32_t dict_serialized_length (dict_t *dict);
 int32_t dict_serialize (dict_t *dict, char *buf);
 int32_t dict_unserialize (char *buf, int32_t size, dict_t **fill);
 
-int32_t dict_allocate_and_serialize (dict_t *this, char **buf, size_t *length);
-
-int32_t dict_iovec_len (dict_t *dict);
-int32_t dict_to_iovec (dict_t *dict, struct iovec *vec, int32_t count);
+int32_t dict_allocate_and_serialize (dict_t *this, char **buf, u_int *length);
 
 void dict_destroy (dict_t *dict);
 void dict_unref (dict_t *dict);
@@ -135,7 +120,7 @@ dict_t *dict_ref (dict_t *dict);
 data_t *data_ref (data_t *data);
 void data_unref (data_t *data);
 
-int32_t dict_lookup  (dict_t *this, char *key, data_pair_t **data);
+int32_t dict_lookup  (dict_t *this, char *key, data_t **data);
 /*
    TODO: provide converts for differnt byte sizes, signedness, and void *
  */
@@ -178,16 +163,26 @@ data_t * data_copy (data_t *old);
 dict_t *get_new_dict_full (int size_hint);
 dict_t *get_new_dict ();
 
-data_pair_t *get_new_data_pair ();
+int dict_foreach (dict_t *this,
+                  int (*fn)(dict_t *this,
+                            char *key,
+                            data_t *value,
+                            void *data),
+                  void *data);
+
+int dict_foreach_fnmatch (dict_t *dict, char *pattern,
+                          int (*fn)(dict_t *this,
+                                    char *key,
+                                    data_t *value,
+                                    void *data),
+                          void *data);
 
-void dict_foreach (dict_t *this,
-		   void (*fn)(dict_t *this,
-			      char *key,
-			      data_t *value,
-			      void *data),
-		   void *data);
+int dict_null_foreach_fn (dict_t *d, char *k,
+                          data_t *v, void *tmp);
 
 dict_t *dict_copy (dict_t *this, dict_t *new);
+int dict_keys_join (void *value, int size, dict_t *dict,
+                    int (*filter_fn)(char *key));
 
 /* CLEANED UP FUNCTIONS DECLARATIONS */
 GF_MUST_CHECK dict_t *dict_new (void);
@@ -237,4 +232,6 @@ GF_MUST_CHECK int dict_get_str (dict_t *this, char *key, char **str);
 GF_MUST_CHECK int dict_get_str_boolean (dict_t *this, char *key, int default_val);
 GF_MUST_CHECK int dict_serialize_value_with_delim (dict_t *this, char *buf, int32_t *serz_len,
                                                     char delimiter);
+
+void dict_dump (dict_t *dict);
 #endif
diff --git a/libglusterfs/src/event-epoll.c b/libglusterfs/src/event-epoll.c
new file mode 100644
index 000000000..06b323624
--- /dev/null
+++ b/libglusterfs/src/event-epoll.c
@@ -0,0 +1,463 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <sys/poll.h>
+#include <pthread.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+
+#include "logging.h"
+#include "event.h"
+#include "mem-pool.h"
+#include "common-utils.h"
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+
+#ifdef HAVE_SYS_EPOLL_H
+#include <sys/epoll.h>
+
+
+static int
+__event_getindex (struct event_pool *event_pool, int fd, int idx)
+{
+        int  ret = -1;
+        int  i = 0;
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        if (idx > -1 && idx < event_pool->used) {
+                if (event_pool->reg[idx].fd == fd)
+                        ret = idx;
+        }
+
+        for (i=0; ret == -1 && i<event_pool->used; i++) {
+                if (event_pool->reg[i].fd == fd) {
+                        ret = i;
+                        break;
+                }
+        }
+
+out:
+        return ret;
+}
+
+
+static struct event_pool *
+event_pool_new_epoll (int count)
+{
+        struct event_pool *event_pool = NULL;
+        int                epfd = -1;
+
+        event_pool = GF_CALLOC (1, sizeof (*event_pool),
+                                gf_common_mt_event_pool);
+
+        if (!event_pool)
+                goto out;
+
+        event_pool->count = count;
+        event_pool->reg = GF_CALLOC (event_pool->count,
+                                     sizeof (*event_pool->reg),
+                                     gf_common_mt_reg);
+
+        if (!event_pool->reg) {
+                GF_FREE (event_pool);
+                event_pool = NULL;
+                goto out;
+        }
+
+        epfd = epoll_create (count);
+
+        if (epfd == -1) {
+                gf_log ("epoll", GF_LOG_ERROR, "epoll fd creation failed (%s)",
+                        strerror (errno));
+                GF_FREE (event_pool->reg);
+                GF_FREE (event_pool);
+                event_pool = NULL;
+                goto out;
+        }
+
+        event_pool->fd = epfd;
+
+        event_pool->count = count;
+
+        pthread_mutex_init (&event_pool->mutex, NULL);
+        pthread_cond_init (&event_pool->cond, NULL);
+
+out:
+        return event_pool;
+}
+
+
+int
+event_register_epoll (struct event_pool *event_pool, int fd,
+                      event_handler_t handler,
+                      void *data, int poll_in, int poll_out)
+{
+        int                 idx = -1;
+        int                 ret = -1;
+        struct epoll_event  epoll_event = {0, };
+        struct event_data  *ev_data = (void *)&epoll_event.data;
+
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                if (event_pool->count == event_pool->used) {
+                        event_pool->count *= 2;
+
+                        event_pool->reg = GF_REALLOC (event_pool->reg,
+                                                      event_pool->count *
+                                                      sizeof (*event_pool->reg));
+
+                        if (!event_pool->reg) {
+                                gf_log ("epoll", GF_LOG_ERROR,
+                                        "event registry re-allocation failed");
+                                goto unlock;
+                        }
+                }
+
+                idx = event_pool->used;
+                event_pool->used++;
+
+                event_pool->reg[idx].fd = fd;
+                event_pool->reg[idx].events = EPOLLPRI;
+                event_pool->reg[idx].handler = handler;
+                event_pool->reg[idx].data = data;
+
+                switch (poll_in) {
+                case 1:
+                        event_pool->reg[idx].events |= EPOLLIN;
+                        break;
+                case 0:
+                        event_pool->reg[idx].events &= ~EPOLLIN;
+                        break;
+                case -1:
+                        /* do nothing */
+                        break;
+                default:
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "invalid poll_in value %d", poll_in);
+                        break;
+                }
+
+                switch (poll_out) {
+                case 1:
+                        event_pool->reg[idx].events |= EPOLLOUT;
+                        break;
+                case 0:
+                        event_pool->reg[idx].events &= ~EPOLLOUT;
+                        break;
+                case -1:
+                        /* do nothing */
+                        break;
+                default:
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "invalid poll_out value %d", poll_out);
+                        break;
+                }
+
+                event_pool->changed = 1;
+
+                epoll_event.events = event_pool->reg[idx].events;
+                ev_data->fd = fd;
+                ev_data->idx = idx;
+
+                ret = epoll_ctl (event_pool->fd, EPOLL_CTL_ADD, fd,
+                                 &epoll_event);
+
+                if (ret == -1) {
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "failed to add fd(=%d) to epoll fd(=%d) (%s)",
+                                fd, event_pool->fd, strerror (errno));
+                        goto unlock;
+                }
+
+                pthread_cond_broadcast (&event_pool->cond);
+        }
+unlock:
+        pthread_mutex_unlock (&event_pool->mutex);
+
+out:
+        return ret;
+}
+
+
+static int
+event_unregister_epoll (struct event_pool *event_pool, int fd, int idx_hint)
+{
+        int  idx = -1;
+        int  ret = -1;
+
+        struct epoll_event epoll_event = {0, };
+        struct event_data *ev_data = (void *)&epoll_event.data;
+        int                lastidx = -1;
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                idx = __event_getindex (event_pool, fd, idx_hint);
+
+                if (idx == -1) {
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "index not found for fd=%d (idx_hint=%d)",
+                                fd, idx_hint);
+                        errno = ENOENT;
+                        goto unlock;
+                }
+
+                ret = epoll_ctl (event_pool->fd, EPOLL_CTL_DEL, fd, NULL);
+
+                /* if ret is -1, this array member should never be accessed */
+                /* if it is 0, the array member might be used by idx_cache
+                 * in which case the member should not be accessed till
+                 * it is reallocated
+                 */
+
+                event_pool->reg[idx].fd = -1;
+
+                if (ret == -1) {
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "fail to del fd(=%d) from epoll fd(=%d) (%s)",
+                                fd, event_pool->fd, strerror (errno));
+                        goto unlock;
+                }
+
+                lastidx = event_pool->used - 1;
+                if (lastidx == idx) {
+                        event_pool->used--;
+                        goto unlock;
+                }
+
+                epoll_event.events = event_pool->reg[lastidx].events;
+                ev_data->fd = event_pool->reg[lastidx].fd;
+                ev_data->idx = idx;
+
+                ret = epoll_ctl (event_pool->fd, EPOLL_CTL_MOD, ev_data->fd,
+                                 &epoll_event);
+                if (ret == -1) {
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "fail to modify fd(=%d) index %d to %d (%s)",
+                                ev_data->fd, event_pool->used, idx,
+                                strerror (errno));
+                        goto unlock;
+                }
+
+                /* just replace the unregistered idx by last one */
+                event_pool->reg[idx] = event_pool->reg[lastidx];
+                event_pool->used--;
+        }
+unlock:
+        pthread_mutex_unlock (&event_pool->mutex);
+
+out:
+        return ret;
+}
+
+
+static int
+event_select_on_epoll (struct event_pool *event_pool, int fd, int idx_hint,
+                       int poll_in, int poll_out)
+{
+        int idx = -1;
+        int ret = -1;
+
+        struct epoll_event epoll_event = {0, };
+        struct event_data *ev_data = (void *)&epoll_event.data;
+
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                idx = __event_getindex (event_pool, fd, idx_hint);
+
+                if (idx == -1) {
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "index not found for fd=%d (idx_hint=%d)",
+                                fd, idx_hint);
+                        errno = ENOENT;
+                        goto unlock;
+                }
+
+                switch (poll_in) {
+                case 1:
+                        event_pool->reg[idx].events |= EPOLLIN;
+                        break;
+                case 0:
+                        event_pool->reg[idx].events &= ~EPOLLIN;
+                        break;
+                case -1:
+                        /* do nothing */
+                        break;
+                default:
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "invalid poll_in value %d", poll_in);
+                        break;
+                }
+
+                switch (poll_out) {
+                case 1:
+                        event_pool->reg[idx].events |= EPOLLOUT;
+                        break;
+                case 0:
+                        event_pool->reg[idx].events &= ~EPOLLOUT;
+                        break;
+                case -1:
+                        /* do nothing */
+                        break;
+                default:
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "invalid poll_out value %d", poll_out);
+                        break;
+                }
+
+                epoll_event.events = event_pool->reg[idx].events;
+                ev_data->fd = fd;
+                ev_data->idx = idx;
+
+                ret = epoll_ctl (event_pool->fd, EPOLL_CTL_MOD, fd,
+                                 &epoll_event);
+                if (ret == -1) {
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "failed to modify fd(=%d) events to %d",
+                                fd, epoll_event.events);
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&event_pool->mutex);
+
+out:
+        return ret;
+}
+
+
+static int
+event_dispatch_epoll_handler (struct event_pool *event_pool,
+                              struct epoll_event *events, int i)
+{
+        struct event_data  *event_data = NULL;
+        event_handler_t     handler = NULL;
+        void               *data = NULL;
+        int                 idx = -1;
+        int                 ret = -1;
+
+
+        event_data = (void *)&events[i].data;
+        handler = NULL;
+        data = NULL;
+
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                idx = __event_getindex (event_pool, event_data->fd,
+                                        event_data->idx);
+
+                if (idx == -1) {
+                        gf_log ("epoll", GF_LOG_ERROR,
+                                "index not found for fd(=%d) (idx_hint=%d)",
+                                event_data->fd, event_data->idx);
+                        goto unlock;
+                }
+
+                handler = event_pool->reg[idx].handler;
+                data = event_pool->reg[idx].data;
+        }
+unlock:
+        pthread_mutex_unlock (&event_pool->mutex);
+
+        if (handler)
+                ret = handler (event_data->fd, event_data->idx, data,
+                               (events[i].events & (EPOLLIN|EPOLLPRI)),
+                               (events[i].events & (EPOLLOUT)),
+                               (events[i].events & (EPOLLERR|EPOLLHUP)));
+        return ret;
+}
+
+
+static int
+event_dispatch_epoll (struct event_pool *event_pool)
+{
+        struct epoll_event *events = NULL;
+        int                 size = 0;
+        int                 i = 0;
+        int                 ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        while (1) {
+                pthread_mutex_lock (&event_pool->mutex);
+                {
+                        while (event_pool->used == 0)
+                                pthread_cond_wait (&event_pool->cond,
+                                                   &event_pool->mutex);
+
+                        if (event_pool->used > event_pool->evcache_size) {
+                                GF_FREE (event_pool->evcache);
+
+                                event_pool->evcache = events = NULL;
+
+                                event_pool->evcache_size =
+                                        event_pool->used + 256;
+
+                                events = GF_CALLOC (event_pool->evcache_size,
+                                                    sizeof (struct epoll_event),
+                                                    gf_common_mt_epoll_event);
+                                if (!events)
+                                        break;
+
+                                event_pool->evcache = events;
+                        }
+                }
+                pthread_mutex_unlock (&event_pool->mutex);
+
+                ret = epoll_wait (event_pool->fd, event_pool->evcache,
+                                  event_pool->evcache_size, -1);
+
+                if (ret == 0)
+                        /* timeout */
+                        continue;
+
+                if (ret == -1 && errno == EINTR)
+                        /* sys call */
+                        continue;
+
+                size = ret;
+
+                for (i = 0; i < size; i++) {
+                        if (!events || !events[i].events)
+                                continue;
+
+                        ret = event_dispatch_epoll_handler (event_pool,
+                                                            events, i);
+                }
+        }
+
+out:
+        return ret;
+}
+
+
+struct event_ops event_ops_epoll = {
+        .new              = event_pool_new_epoll,
+        .event_register   = event_register_epoll,
+        .event_select_on  = event_select_on_epoll,
+        .event_unregister = event_unregister_epoll,
+        .event_dispatch   = event_dispatch_epoll
+};
+
+#endif
diff --git a/libglusterfs/src/event-history.c b/libglusterfs/src/event-history.c
new file mode 100644
index 000000000..82baa521a
--- /dev/null
+++ b/libglusterfs/src/event-history.c
@@ -0,0 +1,83 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "event-history.h"
+
+eh_t *
+eh_new (size_t buffer_size, gf_boolean_t use_buffer_once,
+        void (*destroy_buffer_data) (void *data))
+{
+        eh_t *history = NULL;
+        buffer_t *buffer = NULL;
+
+        history = GF_CALLOC (1, sizeof (eh_t), gf_common_mt_eh_t);
+        if (!history) {
+                gf_log ("", GF_LOG_ERROR, "allocating history failed.");
+                goto out;
+        }
+
+        buffer = cb_buffer_new (buffer_size, use_buffer_once,
+                                destroy_buffer_data);
+        if (!buffer) {
+                gf_log ("", GF_LOG_ERROR, "allocating circular buffer failed");
+                GF_FREE (history);
+                history = NULL;
+        }
+
+        history->buffer = buffer;
+
+        pthread_mutex_init (&history->lock, NULL);
+out:
+        return history;
+}
+
+void
+eh_dump (eh_t *history, void *data,
+         int (dump_fn) (circular_buffer_t *buffer, void *data))
+{
+        if (!history) {
+                gf_log ("", GF_LOG_DEBUG, "history is NULL");
+                goto out;
+        }
+
+        cb_buffer_dump (history->buffer, data, dump_fn);
+
+out:
+        return;
+}
+
+int
+eh_save_history (eh_t *history, void *data)
+{
+        int   ret = -1;
+
+        ret = cb_add_entry_buffer (history->buffer, data);
+
+        return ret;
+}
+
+int
+eh_destroy (eh_t *history)
+{
+        if (!history) {
+                gf_log ("", GF_LOG_INFO, "history for the xlator is "
+                        "NULL");
+                return -1;
+        }
+
+        cb_buffer_destroy (history->buffer);
+        history->buffer = NULL;
+
+        pthread_mutex_destroy (&history->lock);
+
+        GF_FREE (history);
+
+        return 0;
+}
diff --git a/libglusterfs/src/event-history.h b/libglusterfs/src/event-history.h
new file mode 100644
index 000000000..b64f63b5e
--- /dev/null
+++ b/libglusterfs/src/event-history.h
@@ -0,0 +1,44 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _EH_H
+#define _EH_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "mem-types.h"
+#include "circ-buff.h"
+
+struct event_hist
+{
+        buffer_t *buffer;
+        pthread_mutex_t lock;
+};
+
+typedef struct event_hist eh_t;
+
+void
+eh_dump (eh_t *event , void *data,
+         int (fn) (circular_buffer_t *buffer, void *data));
+
+eh_t *
+eh_new (size_t buffer_size, gf_boolean_t use_buffer_once,
+        void (*destroy_data) (void *data));
+
+int
+eh_save_history (eh_t *history, void *string);
+
+int
+eh_destroy (eh_t *history);
+
+#endif /* _EH_H */
diff --git a/libglusterfs/src/event-poll.c b/libglusterfs/src/event-poll.c
new file mode 100644
index 000000000..7f7f560d0
--- /dev/null
+++ b/libglusterfs/src/event-poll.c
@@ -0,0 +1,451 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <sys/poll.h>
+#include <pthread.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+
+#include "logging.h"
+#include "event.h"
+#include "mem-pool.h"
+#include "common-utils.h"
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+static int
+event_register_poll (struct event_pool *event_pool, int fd,
+                     event_handler_t handler,
+                     void *data, int poll_in, int poll_out);
+
+
+static int
+__flush_fd (int fd, int idx, void *data,
+            int poll_in, int poll_out, int poll_err)
+{
+        char buf[64];
+        int ret = -1;
+
+        if (!poll_in)
+                return ret;
+
+        do {
+                ret = read (fd, buf, 64);
+                if (ret == -1 && errno != EAGAIN) {
+                        gf_log ("poll", GF_LOG_ERROR,
+                                "read on %d returned error (%s)",
+                                fd, strerror (errno));
+                }
+        } while (ret == 64);
+
+        return ret;
+}
+
+
+static int
+__event_getindex (struct event_pool *event_pool, int fd, int idx)
+{
+        int  ret = -1;
+        int  i = 0;
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        if (idx > -1 && idx < event_pool->used) {
+                if (event_pool->reg[idx].fd == fd)
+                        ret = idx;
+        }
+
+        for (i=0; ret == -1 && i<event_pool->used; i++) {
+                if (event_pool->reg[i].fd == fd) {
+                        ret = i;
+                        break;
+                }
+        }
+
+out:
+        return ret;
+}
+
+
+static struct event_pool *
+event_pool_new_poll (int count)
+{
+        struct event_pool *event_pool = NULL;
+        int                ret = -1;
+
+        event_pool = GF_CALLOC (1, sizeof (*event_pool),
+                                gf_common_mt_event_pool);
+
+        if (!event_pool)
+                return NULL;
+
+        event_pool->count = count;
+        event_pool->reg = GF_CALLOC (event_pool->count,
+                                     sizeof (*event_pool->reg),
+                                     gf_common_mt_reg);
+
+        if (!event_pool->reg) {
+                GF_FREE (event_pool);
+                return NULL;
+        }
+
+        pthread_mutex_init (&event_pool->mutex, NULL);
+
+        ret = pipe (event_pool->breaker);
+
+        if (ret == -1) {
+                gf_log ("poll", GF_LOG_ERROR,
+                        "pipe creation failed (%s)", strerror (errno));
+                GF_FREE (event_pool->reg);
+                GF_FREE (event_pool);
+                return NULL;
+        }
+
+        ret = fcntl (event_pool->breaker[0], F_SETFL, O_NONBLOCK);
+        if (ret == -1) {
+                gf_log ("poll", GF_LOG_ERROR,
+                        "could not set pipe to non blocking mode (%s)",
+                        strerror (errno));
+                close (event_pool->breaker[0]);
+                close (event_pool->breaker[1]);
+                event_pool->breaker[0] = event_pool->breaker[1] = -1;
+
+                GF_FREE (event_pool->reg);
+                GF_FREE (event_pool);
+                return NULL;
+        }
+
+        ret = fcntl (event_pool->breaker[1], F_SETFL, O_NONBLOCK);
+        if (ret == -1) {
+                gf_log ("poll", GF_LOG_ERROR,
+                        "could not set pipe to non blocking mode (%s)",
+                        strerror (errno));
+
+                close (event_pool->breaker[0]);
+                close (event_pool->breaker[1]);
+                event_pool->breaker[0] = event_pool->breaker[1] = -1;
+
+                GF_FREE (event_pool->reg);
+                GF_FREE (event_pool);
+                return NULL;
+        }
+
+        ret = event_register_poll (event_pool, event_pool->breaker[0],
+                                   __flush_fd, NULL, 1, 0);
+        if (ret == -1) {
+                gf_log ("poll", GF_LOG_ERROR,
+                        "could not register pipe fd with poll event loop");
+                close (event_pool->breaker[0]);
+                close (event_pool->breaker[1]);
+                event_pool->breaker[0] = event_pool->breaker[1] = -1;
+
+                GF_FREE (event_pool->reg);
+                GF_FREE (event_pool);
+                return NULL;
+        }
+
+        return event_pool;
+}
+
+
+static int
+event_register_poll (struct event_pool *event_pool, int fd,
+                     event_handler_t handler,
+                     void *data, int poll_in, int poll_out)
+{
+        int idx = -1;
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                if (event_pool->count == event_pool->used)
+                {
+                        event_pool->count += 256;
+
+                        event_pool->reg = GF_REALLOC (event_pool->reg,
+                                                      event_pool->count *
+                                                      sizeof (*event_pool->reg));
+                        if (!event_pool->reg)
+                                goto unlock;
+                }
+
+                idx = event_pool->used++;
+
+                event_pool->reg[idx].fd = fd;
+                event_pool->reg[idx].events = POLLPRI;
+                event_pool->reg[idx].handler = handler;
+                event_pool->reg[idx].data = data;
+
+                switch (poll_in) {
+                case 1:
+                        event_pool->reg[idx].events |= POLLIN;
+                        break;
+                case 0:
+                        event_pool->reg[idx].events &= ~POLLIN;
+                        break;
+                case -1:
+                        /* do nothing */
+                        break;
+                default:
+                        gf_log ("poll", GF_LOG_ERROR,
+                                "invalid poll_in value %d", poll_in);
+                        break;
+                }
+
+                switch (poll_out) {
+                case 1:
+                        event_pool->reg[idx].events |= POLLOUT;
+                        break;
+                case 0:
+                        event_pool->reg[idx].events &= ~POLLOUT;
+                        break;
+                case -1:
+                        /* do nothing */
+                        break;
+                default:
+                        gf_log ("poll", GF_LOG_ERROR,
+                                "invalid poll_out value %d", poll_out);
+                        break;
+                }
+
+                event_pool->changed = 1;
+
+        }
+unlock:
+        pthread_mutex_unlock (&event_pool->mutex);
+
+out:
+        return idx;
+}
+
+
+static int
+event_unregister_poll (struct event_pool *event_pool, int fd, int idx_hint)
+{
+        int idx = -1;
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                idx = __event_getindex (event_pool, fd, idx_hint);
+
+                if (idx == -1) {
+                        gf_log ("poll", GF_LOG_ERROR,
+                                "index not found for fd=%d (idx_hint=%d)",
+                                fd, idx_hint);
+                        errno = ENOENT;
+                        goto unlock;
+                }
+
+                event_pool->reg[idx] =  event_pool->reg[--event_pool->used];
+                event_pool->changed = 1;
+        }
+unlock:
+        pthread_mutex_unlock (&event_pool->mutex);
+
+out:
+        return idx;
+}
+
+
+static int
+event_select_on_poll (struct event_pool *event_pool, int fd, int idx_hint,
+                      int poll_in, int poll_out)
+{
+        int idx = -1;
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                idx = __event_getindex (event_pool, fd, idx_hint);
+
+                if (idx == -1) {
+                        gf_log ("poll", GF_LOG_ERROR,
+                                "index not found for fd=%d (idx_hint=%d)",
+                                fd, idx_hint);
+                        errno = ENOENT;
+                        goto unlock;
+                }
+
+                switch (poll_in) {
+                case 1:
+                        event_pool->reg[idx].events |= POLLIN;
+                        break;
+                case 0:
+                        event_pool->reg[idx].events &= ~POLLIN;
+                        break;
+                case -1:
+                        /* do nothing */
+                        break;
+                default:
+                        /* TODO: log error */
+                        break;
+                }
+
+                switch (poll_out) {
+                case 1:
+                        event_pool->reg[idx].events |= POLLOUT;
+                        break;
+                case 0:
+                        event_pool->reg[idx].events &= ~POLLOUT;
+                        break;
+                case -1:
+                        /* do nothing */
+                        break;
+                default:
+                        /* TODO: log error */
+                        break;
+                }
+
+                if (poll_in + poll_out > -2)
+                        event_pool->changed = 1;
+        }
+unlock:
+        pthread_mutex_unlock (&event_pool->mutex);
+
+out:
+        return idx;
+}
+
+
+static int
+event_dispatch_poll_handler (struct event_pool *event_pool,
+                             struct pollfd *ufds, int i)
+{
+        event_handler_t  handler = NULL;
+        void            *data = NULL;
+        int              idx = -1;
+        int              ret = 0;
+
+        handler = NULL;
+        data    = NULL;
+
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                idx = __event_getindex (event_pool, ufds[i].fd, i);
+
+                if (idx == -1) {
+                        gf_log ("poll", GF_LOG_ERROR,
+                                "index not found for fd=%d (idx_hint=%d)",
+                                ufds[i].fd, i);
+                        goto unlock;
+                }
+
+                handler = event_pool->reg[idx].handler;
+                data = event_pool->reg[idx].data;
+        }
+unlock:
+        pthread_mutex_unlock (&event_pool->mutex);
+
+        if (handler)
+                ret = handler (ufds[i].fd, idx, data,
+                               (ufds[i].revents & (POLLIN|POLLPRI)),
+                               (ufds[i].revents & (POLLOUT)),
+                               (ufds[i].revents & (POLLERR|POLLHUP|POLLNVAL)));
+
+        return ret;
+}
+
+
+static int
+event_dispatch_poll_resize (struct event_pool *event_pool,
+                            struct pollfd *ufds, int size)
+{
+        int              i = 0;
+
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                if (event_pool->changed == 0) {
+                        goto unlock;
+                }
+
+                if (event_pool->used > event_pool->evcache_size) {
+                        GF_FREE (event_pool->evcache);
+
+                        event_pool->evcache = ufds = NULL;
+
+                        event_pool->evcache_size = event_pool->used;
+
+                        ufds = GF_CALLOC (sizeof (struct pollfd),
+                                          event_pool->evcache_size,
+                                          gf_common_mt_pollfd);
+                        if (!ufds)
+                                goto unlock;
+                        event_pool->evcache = ufds;
+                }
+
+                for (i = 0; i < event_pool->used; i++) {
+                        ufds[i].fd = event_pool->reg[i].fd;
+                        ufds[i].events = event_pool->reg[i].events;
+                        ufds[i].revents = 0;
+                }
+
+                size = i;
+        }
+unlock:
+        pthread_mutex_unlock (&event_pool->mutex);
+
+        return size;
+}
+
+
+static int
+event_dispatch_poll (struct event_pool *event_pool)
+{
+        struct pollfd   *ufds = NULL;
+        int              size = 0;
+        int              i = 0;
+        int              ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        while (1) {
+                size = event_dispatch_poll_resize (event_pool, ufds, size);
+                ufds = event_pool->evcache;
+
+                ret = poll (ufds, size, 1);
+
+                if (ret == 0)
+                        /* timeout */
+                        continue;
+
+                if (ret == -1 && errno == EINTR)
+                        /* sys call */
+                        continue;
+
+                for (i = 0; i < size; i++) {
+                        if (!ufds[i].revents)
+                                continue;
+
+                        event_dispatch_poll_handler (event_pool, ufds, i);
+                }
+        }
+
+out:
+        return -1;
+}
+
+
+struct event_ops event_ops_poll = {
+        .new              = event_pool_new_poll,
+        .event_register   = event_register_poll,
+        .event_select_on  = event_select_on_poll,
+        .event_unregister = event_unregister_poll,
+        .event_dispatch   = event_dispatch_poll
+};
diff --git a/libglusterfs/src/event.c b/libglusterfs/src/event.c
index c117a4ae2..0197e7948 100644
--- a/libglusterfs/src/event.c
+++ b/libglusterfs/src/event.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <sys/poll.h>
@@ -35,851 +26,17 @@
 #include "config.h"
 #endif
 
-static int
-event_register_poll (struct event_pool *event_pool, int fd,
-                     event_handler_t handler,
-                     void *data, int poll_in, int poll_out);
-
-
-static int
-__flush_fd (int fd, int idx, void *data,
-            int poll_in, int poll_out, int poll_err)
-{
-        char buf[64];
-        int ret = -1;
-
-        if (!poll_in)
-                return ret;
-
-        do {
-                ret = read (fd, buf, 64);
-                if (ret == -1 && errno != EAGAIN) {
-                        gf_log ("poll", GF_LOG_ERROR,
-                                "read on %d returned error (%s)",
-                                fd, strerror (errno));
-                }
-        } while (ret == 64);
-
-        return ret;
-}
-
-
-static int
-__event_getindex (struct event_pool *event_pool, int fd, int idx)
-{
-        int  ret = -1;
-        int  i = 0;
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        if (idx > -1 && idx < event_pool->used) {
-                if (event_pool->reg[idx].fd == fd)
-                        ret = idx;
-        }
-
-        for (i=0; ret == -1 && i<event_pool->used; i++) {
-                if (event_pool->reg[i].fd == fd) {
-                        ret = i;
-                        break;
-                }
-        }
-
-out:
-        return ret;
-}
-
-
-static struct event_pool *
-event_pool_new_poll (int count)
-{
-        struct event_pool *event_pool = NULL;
-        int                ret = -1;
-
-        event_pool = GF_CALLOC (1, sizeof (*event_pool),
-                                gf_common_mt_event_pool);
-
-        if (!event_pool)
-                return NULL;
-
-        event_pool->count = count;
-        event_pool->reg = GF_CALLOC (event_pool->count,
-                                     sizeof (*event_pool->reg),
-                                     gf_common_mt_reg);
-
-        if (!event_pool->reg) {
-                GF_FREE (event_pool);
-                return NULL;
-        }
-
-        pthread_mutex_init (&event_pool->mutex, NULL);
-
-        ret = pipe (event_pool->breaker);
-
-        if (ret == -1) {
-                gf_log ("poll", GF_LOG_ERROR,
-                        "pipe creation failed (%s)", strerror (errno));
-                GF_FREE (event_pool->reg);
-                GF_FREE (event_pool);
-                return NULL;
-        }
-
-        ret = fcntl (event_pool->breaker[0], F_SETFL, O_NONBLOCK);
-        if (ret == -1) {
-                gf_log ("poll", GF_LOG_ERROR,
-                        "could not set pipe to non blocking mode (%s)",
-                        strerror (errno));
-                close (event_pool->breaker[0]);
-                close (event_pool->breaker[1]);
-                event_pool->breaker[0] = event_pool->breaker[1] = -1;
-
-                GF_FREE (event_pool->reg);
-                GF_FREE (event_pool);
-                return NULL;
-        }
-
-        ret = fcntl (event_pool->breaker[1], F_SETFL, O_NONBLOCK);
-        if (ret == -1) {
-                gf_log ("poll", GF_LOG_ERROR,
-                        "could not set pipe to non blocking mode (%s)",
-                        strerror (errno));
-
-                close (event_pool->breaker[0]);
-                close (event_pool->breaker[1]);
-                event_pool->breaker[0] = event_pool->breaker[1] = -1;
-
-                GF_FREE (event_pool->reg);
-                GF_FREE (event_pool);
-                return NULL;
-        }
-
-        ret = event_register_poll (event_pool, event_pool->breaker[0],
-                                   __flush_fd, NULL, 1, 0);
-        if (ret == -1) {
-                gf_log ("poll", GF_LOG_ERROR,
-                        "could not register pipe fd with poll event loop");
-                close (event_pool->breaker[0]);
-                close (event_pool->breaker[1]);
-                event_pool->breaker[0] = event_pool->breaker[1] = -1;
-
-                GF_FREE (event_pool->reg);
-                GF_FREE (event_pool);
-                return NULL;
-        }
-
-        return event_pool;
-}
-
-
-static int
-event_register_poll (struct event_pool *event_pool, int fd,
-                     event_handler_t handler,
-                     void *data, int poll_in, int poll_out)
-{
-        int idx = -1;
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                if (event_pool->count == event_pool->used)
-                {
-                        event_pool->count += 256;
-
-                        event_pool->reg = GF_REALLOC (event_pool->reg,
-                                                      event_pool->count *
-                                                      sizeof (*event_pool->reg));
-                        if (!event_pool->reg)
-                                goto unlock;
-                }
-
-                idx = event_pool->used++;
-
-                event_pool->reg[idx].fd = fd;
-                event_pool->reg[idx].events = POLLPRI;
-                event_pool->reg[idx].handler = handler;
-                event_pool->reg[idx].data = data;
-
-                switch (poll_in) {
-                case 1:
-                        event_pool->reg[idx].events |= POLLIN;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~POLLIN;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        gf_log ("poll", GF_LOG_ERROR,
-                                "invalid poll_in value %d", poll_in);
-                        break;
-                }
-
-                switch (poll_out) {
-                case 1:
-                        event_pool->reg[idx].events |= POLLOUT;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~POLLOUT;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        gf_log ("poll", GF_LOG_ERROR,
-                                "invalid poll_out value %d", poll_out);
-                        break;
-                }
-
-                event_pool->changed = 1;
-
-        }
-unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
-
-out:
-        return idx;
-}
-
-
-static int
-event_unregister_poll (struct event_pool *event_pool, int fd, int idx_hint)
-{
-        int idx = -1;
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                idx = __event_getindex (event_pool, fd, idx_hint);
-
-                if (idx == -1) {
-                        gf_log ("poll", GF_LOG_ERROR,
-                                "index not found for fd=%d (idx_hint=%d)",
-                                fd, idx_hint);
-                        errno = ENOENT;
-                        goto unlock;
-                }
-
-                event_pool->reg[idx] =  event_pool->reg[--event_pool->used];
-                event_pool->changed = 1;
-        }
-unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
-
-out:
-        return idx;
-}
-
-
-static int
-event_select_on_poll (struct event_pool *event_pool, int fd, int idx_hint,
-                      int poll_in, int poll_out)
-{
-        int idx = -1;
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                idx = __event_getindex (event_pool, fd, idx_hint);
-
-                if (idx == -1) {
-                        gf_log ("poll", GF_LOG_ERROR,
-                                "index not found for fd=%d (idx_hint=%d)",
-                                fd, idx_hint);
-                        errno = ENOENT;
-                        goto unlock;
-                }
-
-                switch (poll_in) {
-                case 1:
-                        event_pool->reg[idx].events |= POLLIN;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~POLLIN;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        /* TODO: log error */
-                        break;
-                }
-
-                switch (poll_out) {
-                case 1:
-                        event_pool->reg[idx].events |= POLLOUT;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~POLLOUT;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        /* TODO: log error */
-                        break;
-                }
-
-                if (poll_in + poll_out > -2)
-                        event_pool->changed = 1;
-        }
-unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
-
-out:
-        return idx;
-}
-
-
-static int
-event_dispatch_poll_handler (struct event_pool *event_pool,
-                             struct pollfd *ufds, int i)
-{
-        event_handler_t  handler = NULL;
-        void            *data = NULL;
-        int              idx = -1;
-        int              ret = 0;
-
-        handler = NULL;
-        data    = NULL;
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                idx = __event_getindex (event_pool, ufds[i].fd, i);
-
-                if (idx == -1) {
-                        gf_log ("poll", GF_LOG_ERROR,
-                                "index not found for fd=%d (idx_hint=%d)",
-                                ufds[i].fd, i);
-                        goto unlock;
-                }
-
-                handler = event_pool->reg[idx].handler;
-                data = event_pool->reg[idx].data;
-        }
-unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
-
-        if (handler)
-                ret = handler (ufds[i].fd, idx, data,
-                               (ufds[i].revents & (POLLIN|POLLPRI)),
-                               (ufds[i].revents & (POLLOUT)),
-                               (ufds[i].revents & (POLLERR|POLLHUP|POLLNVAL)));
-
-        return ret;
-}
-
-
-static int
-event_dispatch_poll_resize (struct event_pool *event_pool,
-                            struct pollfd *ufds, int size)
-{
-        int              i = 0;
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                if (event_pool->changed == 0) {
-                        goto unlock;
-                }
-
-                if (event_pool->used > event_pool->evcache_size) {
-                        if (event_pool->evcache)
-                                GF_FREE (event_pool->evcache);
-
-                        event_pool->evcache = ufds = NULL;
-
-                        event_pool->evcache_size = event_pool->used;
-
-                        ufds = GF_CALLOC (sizeof (struct pollfd),
-                                          event_pool->evcache_size,
-                                          gf_common_mt_pollfd);
-                        if (!ufds)
-                                goto unlock;
-                        event_pool->evcache = ufds;
-                }
-
-                for (i = 0; i < event_pool->used; i++) {
-                        ufds[i].fd = event_pool->reg[i].fd;
-                        ufds[i].events = event_pool->reg[i].events;
-                        ufds[i].revents = 0;
-                }
-
-                size = i;
-        }
-unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
-
-        return size;
-}
-
-
-static int
-event_dispatch_poll (struct event_pool *event_pool)
-{
-        struct pollfd   *ufds = NULL;
-        int              size = 0;
-        int              i = 0;
-        int              ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        while (1) {
-                size = event_dispatch_poll_resize (event_pool, ufds, size);
-                ufds = event_pool->evcache;
-
-                ret = poll (ufds, size, 1);
-
-                if (ret == 0)
-                        /* timeout */
-                        continue;
-
-                if (ret == -1 && errno == EINTR)
-                        /* sys call */
-                        continue;
-
-                for (i = 0; i < size; i++) {
-                        if (!ufds[i].revents)
-                                continue;
-
-                        event_dispatch_poll_handler (event_pool, ufds, i);
-                }
-        }
-
-out:
-        return -1;
-}
-
-
-static struct event_ops event_ops_poll = {
-        .new              = event_pool_new_poll,
-        .event_register   = event_register_poll,
-        .event_select_on  = event_select_on_poll,
-        .event_unregister = event_unregister_poll,
-        .event_dispatch   = event_dispatch_poll
-};
-
-
-
-#ifdef HAVE_SYS_EPOLL_H
-#include <sys/epoll.h>
-
-
-static struct event_pool *
-event_pool_new_epoll (int count)
-{
-        struct event_pool *event_pool = NULL;
-        int                epfd = -1;
-
-        event_pool = GF_CALLOC (1, sizeof (*event_pool),
-                                gf_common_mt_event_pool);
-
-        if (!event_pool)
-                goto out;
-
-        event_pool->count = count;
-        event_pool->reg = GF_CALLOC (event_pool->count,
-                                     sizeof (*event_pool->reg),
-                                     gf_common_mt_reg);
-
-        if (!event_pool->reg) {
-                GF_FREE (event_pool);
-                event_pool = NULL;
-                goto out;
-        }
-
-        epfd = epoll_create (count);
-
-        if (epfd == -1) {
-                gf_log ("epoll", GF_LOG_ERROR, "epoll fd creation failed (%s)",
-                        strerror (errno));
-                GF_FREE (event_pool->reg);
-                GF_FREE (event_pool);
-                event_pool = NULL;
-                goto out;
-        }
-
-        event_pool->fd = epfd;
-
-        event_pool->count = count;
-
-        pthread_mutex_init (&event_pool->mutex, NULL);
-        pthread_cond_init (&event_pool->cond, NULL);
-
-out:
-        return event_pool;
-}
-
-
-int
-event_register_epoll (struct event_pool *event_pool, int fd,
-                      event_handler_t handler,
-                      void *data, int poll_in, int poll_out)
-{
-        int                 idx = -1;
-        int                 ret = -1;
-        struct epoll_event  epoll_event = {0, };
-        struct event_data  *ev_data = (void *)&epoll_event.data;
-
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                if (event_pool->count == event_pool->used) {
-                        event_pool->count *= 2;
-
-                        event_pool->reg = GF_REALLOC (event_pool->reg,
-                                                      event_pool->count *
-                                                      sizeof (*event_pool->reg));
-
-                        if (!event_pool->reg) {
-                                gf_log ("epoll", GF_LOG_ERROR,
-                                        "event registry re-allocation failed");
-                                goto unlock;
-                        }
-                }
-
-                idx = event_pool->used;
-                event_pool->used++;
-
-                event_pool->reg[idx].fd = fd;
-                event_pool->reg[idx].events = EPOLLPRI;
-                event_pool->reg[idx].handler = handler;
-                event_pool->reg[idx].data = data;
-
-                switch (poll_in) {
-                case 1:
-                        event_pool->reg[idx].events |= EPOLLIN;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~EPOLLIN;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "invalid poll_in value %d", poll_in);
-                        break;
-                }
-
-                switch (poll_out) {
-                case 1:
-                        event_pool->reg[idx].events |= EPOLLOUT;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~EPOLLOUT;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "invalid poll_out value %d", poll_out);
-                        break;
-                }
-
-                event_pool->changed = 1;
-
-                epoll_event.events = event_pool->reg[idx].events;
-                ev_data->fd = fd;
-                ev_data->idx = idx;
-
-                ret = epoll_ctl (event_pool->fd, EPOLL_CTL_ADD, fd,
-                                 &epoll_event);
-
-                if (ret == -1) {
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "failed to add fd(=%d) to epoll fd(=%d) (%s)",
-                                fd, event_pool->fd, strerror (errno));
-                        goto unlock;
-                }
-
-                pthread_cond_broadcast (&event_pool->cond);
-        }
-unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
-
-out:
-        return ret;
-}
-
-
-static int
-event_unregister_epoll (struct event_pool *event_pool, int fd, int idx_hint)
-{
-        int  idx = -1;
-        int  ret = -1;
-
-        struct epoll_event epoll_event = {0, };
-        struct event_data *ev_data = (void *)&epoll_event.data;
-        int                lastidx = -1;
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                idx = __event_getindex (event_pool, fd, idx_hint);
-
-                if (idx == -1) {
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "index not found for fd=%d (idx_hint=%d)",
-                                fd, idx_hint);
-                        errno = ENOENT;
-                        goto unlock;
-                }
-
-                ret = epoll_ctl (event_pool->fd, EPOLL_CTL_DEL, fd, NULL);
-
-                /* if ret is -1, this array member should never be accessed */
-                /* if it is 0, the array member might be used by idx_cache
-                 * in which case the member should not be accessed till
-                 * it is reallocated
-                 */
-
-                event_pool->reg[idx].fd = -1;
-
-                if (ret == -1) {
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "fail to del fd(=%d) from epoll fd(=%d) (%s)",
-                                fd, event_pool->fd, strerror (errno));
-                        goto unlock;
-                }
-
-                lastidx = event_pool->used - 1;
-                if (lastidx == idx) {
-                        event_pool->used--;
-                        goto unlock;
-                }
-
-                epoll_event.events = event_pool->reg[lastidx].events;
-                ev_data->fd = event_pool->reg[lastidx].fd;
-                ev_data->idx = idx;
-
-                ret = epoll_ctl (event_pool->fd, EPOLL_CTL_MOD, ev_data->fd,
-                                 &epoll_event);
-                if (ret == -1) {
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "fail to modify fd(=%d) index %d to %d (%s)",
-                                ev_data->fd, event_pool->used, idx,
-                                strerror (errno));
-                        goto unlock;
-                }
-
-                /* just replace the unregistered idx by last one */
-                event_pool->reg[idx] = event_pool->reg[lastidx];
-                event_pool->used--;
-        }
-unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
-
-out:
-        return ret;
-}
-
-
-static int
-event_select_on_epoll (struct event_pool *event_pool, int fd, int idx_hint,
-                       int poll_in, int poll_out)
-{
-        int idx = -1;
-        int ret = -1;
-
-        struct epoll_event epoll_event = {0, };
-        struct event_data *ev_data = (void *)&epoll_event.data;
-
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                idx = __event_getindex (event_pool, fd, idx_hint);
-
-                if (idx == -1) {
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "index not found for fd=%d (idx_hint=%d)",
-                                fd, idx_hint);
-                        errno = ENOENT;
-                        goto unlock;
-                }
-
-                switch (poll_in) {
-                case 1:
-                        event_pool->reg[idx].events |= EPOLLIN;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~EPOLLIN;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "invalid poll_in value %d", poll_in);
-                        break;
-                }
-
-                switch (poll_out) {
-                case 1:
-                        event_pool->reg[idx].events |= EPOLLOUT;
-                        break;
-                case 0:
-                        event_pool->reg[idx].events &= ~EPOLLOUT;
-                        break;
-                case -1:
-                        /* do nothing */
-                        break;
-                default:
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "invalid poll_out value %d", poll_out);
-                        break;
-                }
-
-                epoll_event.events = event_pool->reg[idx].events;
-                ev_data->fd = fd;
-                ev_data->idx = idx;
-
-                ret = epoll_ctl (event_pool->fd, EPOLL_CTL_MOD, fd,
-                                 &epoll_event);
-                if (ret == -1) {
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "failed to modify fd(=%d) events to %d",
-                                fd, epoll_event.events);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
-
-out:
-        return ret;
-}
-
-
-static int
-event_dispatch_epoll_handler (struct event_pool *event_pool,
-                              struct epoll_event *events, int i)
-{
-        struct event_data  *event_data = NULL;
-        event_handler_t     handler = NULL;
-        void               *data = NULL;
-        int                 idx = -1;
-        int                 ret = -1;
-
-
-        event_data = (void *)&events[i].data;
-        handler = NULL;
-        data = NULL;
-
-        pthread_mutex_lock (&event_pool->mutex);
-        {
-                idx = __event_getindex (event_pool, event_data->fd,
-                                        event_data->idx);
-
-                if (idx == -1) {
-                        gf_log ("epoll", GF_LOG_ERROR,
-                                "index not found for fd(=%d) (idx_hint=%d)",
-                                event_data->fd, event_data->idx);
-                        goto unlock;
-                }
-
-                handler = event_pool->reg[idx].handler;
-                data = event_pool->reg[idx].data;
-        }
-unlock:
-        pthread_mutex_unlock (&event_pool->mutex);
-
-        if (handler)
-                ret = handler (event_data->fd, event_data->idx, data,
-                               (events[i].events & (EPOLLIN|EPOLLPRI)),
-                               (events[i].events & (EPOLLOUT)),
-                               (events[i].events & (EPOLLERR|EPOLLHUP)));
-        return ret;
-}
-
-
-static int
-event_dispatch_epoll (struct event_pool *event_pool)
-{
-        struct epoll_event *events = NULL;
-        int                 size = 0;
-        int                 i = 0;
-        int                 ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
-
-        while (1) {
-                pthread_mutex_lock (&event_pool->mutex);
-                {
-                        while (event_pool->used == 0)
-                                pthread_cond_wait (&event_pool->cond,
-                                                   &event_pool->mutex);
-
-                        if (event_pool->used > event_pool->evcache_size) {
-                                if (event_pool->evcache)
-                                        GF_FREE (event_pool->evcache);
-
-                                event_pool->evcache = events = NULL;
-
-                                event_pool->evcache_size =
-                                        event_pool->used + 256;
-
-                                events = GF_CALLOC (event_pool->evcache_size,
-                                                    sizeof (struct epoll_event),
-                                                    gf_common_mt_epoll_event);
-                                if (!events)
-                                        break;
-
-                                event_pool->evcache = events;
-                        }
-                }
-                pthread_mutex_unlock (&event_pool->mutex);
-
-                ret = epoll_wait (event_pool->fd, event_pool->evcache,
-                                  event_pool->evcache_size, -1);
-
-                if (ret == 0)
-                        /* timeout */
-                        continue;
-
-                if (ret == -1 && errno == EINTR)
-                        /* sys call */
-                        continue;
-
-                size = ret;
-
-                for (i = 0; i < size; i++) {
-                        if (!events || !events[i].events)
-                                continue;
-
-                        ret = event_dispatch_epoll_handler (event_pool,
-                                                            events, i);
-                }
-        }
-
-out:
-        return ret;
-}
-
-
-static struct event_ops event_ops_epoll = {
-        .new              = event_pool_new_epoll,
-        .event_register   = event_register_epoll,
-        .event_select_on  = event_select_on_epoll,
-        .event_unregister = event_unregister_epoll,
-        .event_dispatch   = event_dispatch_epoll
-};
-
-#endif
 
 
 struct event_pool *
 event_pool_new (int count)
 {
         struct event_pool *event_pool = NULL;
+        extern struct event_ops event_ops_poll;
 
 #ifdef HAVE_SYS_EPOLL_H
+        extern struct event_ops event_ops_epoll;
+
         event_pool = event_ops_epoll.new (count);
 
         if (event_pool) {
diff --git a/libglusterfs/src/event.h b/libglusterfs/src/event.h
index 0e87f0cd7..7ed182492 100644
--- a/libglusterfs/src/event.h
+++ b/libglusterfs/src/event.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _EVENT_H_
@@ -30,8 +21,8 @@
 struct event_pool;
 struct event_ops;
 struct event_data {
-  int fd;
-  int idx;
+	int fd;
+	int idx;
 } __attribute__ ((__packed__, __may_alias__));
 
 
@@ -39,28 +30,27 @@ typedef int (*event_handler_t) (int fd, int idx, void *data,
 				int poll_in, int poll_out, int poll_err);
 
 struct event_pool {
-  struct event_ops *ops;
+	struct event_ops *ops;
 
-  int fd;
-  int breaker[2];
+	int fd;
+	int breaker[2];
 
-  int count;
-  struct {
-    int fd;
-    int events;
-    void *data;
-    event_handler_t handler;
-  } *reg;
+	int count;
+	struct {
+		int fd;
+		int events;
+		void *data;
+		event_handler_t handler;
+	} *reg;
 
-  int used;
-  int idx_cache;
-  int changed;
+	int used;
+	int changed;
 
-  pthread_mutex_t mutex;
-  pthread_cond_t cond;
+	pthread_mutex_t mutex;
+	pthread_cond_t cond;
 
-  void *evcache;
-  int evcache_size;
+	void *evcache;
+	int evcache_size;
 };
 
 struct event_ops {
diff --git a/libglusterfs/src/fd-lk.c b/libglusterfs/src/fd-lk.c
new file mode 100644
index 000000000..caf2bb38e
--- /dev/null
+++ b/libglusterfs/src/fd-lk.c
@@ -0,0 +1,490 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "fd-lk.h"
+#include "common-utils.h"
+
+
+int32_t
+_fd_lk_delete_lock (fd_lk_ctx_node_t *lock)
+{
+        int32_t    ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("fd-lk", lock, out);
+
+        list_del_init (&lock->next);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+_fd_lk_destroy_lock (fd_lk_ctx_node_t *lock)
+{
+        int32_t    ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("fd-lk", lock, out);
+
+        GF_FREE (lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+_fd_lk_destroy_lock_list (fd_lk_ctx_t *lk_ctx)
+{
+        int               ret     = -1;
+        fd_lk_ctx_node_t *lk      = NULL;
+        fd_lk_ctx_node_t *tmp     = NULL;
+
+        GF_VALIDATE_OR_GOTO ("fd-lk", lk_ctx, out);
+
+        list_for_each_entry_safe (lk, tmp, &lk_ctx->lk_list, next) {
+                _fd_lk_delete_lock (lk);
+                _fd_lk_destroy_lock (lk);
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+fd_lk_ctx_unref (fd_lk_ctx_t *lk_ctx)
+{
+        int ref = -1;
+
+        GF_VALIDATE_OR_GOTO ("fd-lk", lk_ctx, err);
+
+        LOCK (&lk_ctx->lock);
+        {
+                ref = --lk_ctx->ref;
+                if (ref < 0)
+                        GF_ASSERT (!ref);
+                if (ref == 0)
+                        _fd_lk_destroy_lock_list (lk_ctx);
+        }
+        UNLOCK (&lk_ctx->lock);
+
+        if (ref == 0) {
+                LOCK_DESTROY (&lk_ctx->lock);
+                GF_FREE (lk_ctx);
+        }
+
+        return 0;
+err:
+        return -1;
+}
+
+fd_lk_ctx_t *
+_fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx)
+{
+        if (!lk_ctx) {
+                gf_log_callingfn ("fd", GF_LOG_WARNING,
+                                  "invalid argument");
+                return NULL;
+        }
+
+        ++lk_ctx->ref;
+
+        return lk_ctx;
+}
+
+fd_lk_ctx_t *
+fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx)
+{
+        fd_lk_ctx_t *new_lk_ctx = NULL;
+
+        if (!lk_ctx) {
+                gf_log_callingfn ("fd", GF_LOG_WARNING,
+                                  "invalid argument");
+                return NULL;
+        }
+
+        LOCK (&lk_ctx->lock);
+        {
+                new_lk_ctx = _fd_lk_ctx_ref (lk_ctx);
+        }
+        UNLOCK (&lk_ctx->lock);
+
+        return new_lk_ctx;
+}
+
+fd_lk_ctx_t *
+fd_lk_ctx_try_ref (fd_lk_ctx_t *lk_ctx)
+{
+        int         ret         = -1;
+        fd_lk_ctx_t *new_lk_ctx = NULL;
+
+        if (!lk_ctx) {
+                goto out;
+        }
+
+        ret = TRY_LOCK (&lk_ctx->lock);
+        if (ret)
+                goto out;
+
+        new_lk_ctx = _fd_lk_ctx_ref (lk_ctx);
+        UNLOCK (&lk_ctx->lock);
+
+out:
+        return new_lk_ctx;
+}
+
+fd_lk_ctx_t *
+fd_lk_ctx_create ()
+{
+        fd_lk_ctx_t *fd_lk_ctx = NULL;
+
+        fd_lk_ctx = GF_CALLOC (1, sizeof (fd_lk_ctx_t),
+                               gf_common_mt_fd_lk_ctx_t);
+        if (!fd_lk_ctx)
+                goto out;
+
+        INIT_LIST_HEAD (&fd_lk_ctx->lk_list);
+
+        LOCK_INIT (&fd_lk_ctx->lock);
+
+        fd_lk_ctx = fd_lk_ctx_ref (fd_lk_ctx);
+out:
+        return fd_lk_ctx;
+}
+
+int
+_fd_lk_insert_lock (fd_lk_ctx_t *lk_ctx,
+                    fd_lk_ctx_node_t *lock)
+{
+        list_add_tail (&lock->next, &lk_ctx->lk_list);
+        return 0;
+}
+
+static off_t
+_fd_lk_get_lock_len (off_t start, off_t end)
+{
+        if (end == LLONG_MAX)
+                return 0;
+        else
+                return (end - start + 1);
+}
+
+fd_lk_ctx_node_t *
+fd_lk_ctx_node_new (int32_t cmd, struct gf_flock *flock)
+{
+        fd_lk_ctx_node_t  *new_lock  = NULL;
+
+        /* TODO: get from mem-pool */
+        new_lock = GF_CALLOC (1, sizeof (fd_lk_ctx_node_t),
+                              gf_common_mt_fd_lk_ctx_node_t);
+        if (!new_lock)
+                goto out;
+
+        new_lock->cmd = cmd;
+
+        if (flock) {
+                new_lock->fl_type  = flock->l_type;
+                new_lock->fl_start = flock->l_start;
+
+                if (flock->l_len == 0)
+                        new_lock->fl_end = LLONG_MAX;
+                else
+                        new_lock->fl_end = flock->l_start + flock->l_len - 1;
+
+                memcpy (&new_lock->user_flock, flock,
+                        sizeof (struct gf_flock));
+        }
+
+        INIT_LIST_HEAD (&new_lock->next);
+out:
+        return new_lock;
+}
+
+int32_t
+_fd_lk_delete_unlck_locks (fd_lk_ctx_t *lk_ctx)
+{
+        int32_t            ret   = -1;
+        fd_lk_ctx_node_t  *tmp   = NULL;
+        fd_lk_ctx_node_t  *lk    = NULL;
+
+        GF_VALIDATE_OR_GOTO ("fd-lk", lk_ctx, out);
+
+        list_for_each_entry_safe (lk, tmp, &lk_ctx->lk_list, next) {
+                if (lk->fl_type == F_UNLCK) {
+                        _fd_lk_delete_lock (lk);
+                        _fd_lk_destroy_lock (lk);
+                }
+        }
+out:
+        return ret;
+}
+
+int
+fd_lk_overlap (fd_lk_ctx_node_t *l1,
+               fd_lk_ctx_node_t *l2)
+{
+        if (l1->fl_end >= l2->fl_start &&
+            l2->fl_end >= l1->fl_start)
+                return 1;
+
+        return 0;
+}
+
+fd_lk_ctx_node_t *
+_fd_lk_add_locks (fd_lk_ctx_node_t *l1,
+                  fd_lk_ctx_node_t *l2)
+{
+        fd_lk_ctx_node_t  *sum = NULL;
+
+        sum = fd_lk_ctx_node_new (0, NULL);
+        if (!sum)
+                goto out;
+
+        sum->fl_start = min (l1->fl_start, l2->fl_start);
+        sum->fl_end   = max (l1->fl_end, l2->fl_end);
+
+        sum->user_flock.l_start = sum->fl_start;
+        sum->user_flock.l_len   = _fd_lk_get_lock_len (sum->fl_start,
+                                                       sum->fl_end);
+out:
+        return sum;
+}
+
+/* Subtract two locks */
+struct _values {
+        fd_lk_ctx_node_t *locks[3];
+};
+
+int32_t
+_fd_lk_sub_locks (struct _values *v,
+                  fd_lk_ctx_node_t *big,
+                  fd_lk_ctx_node_t *small)
+{
+        int32_t  ret  = -1;
+
+        if ((big->fl_start == small->fl_start) &&
+            (big->fl_end   == small->fl_end)) {
+                /* both edges coincide with big */
+                v->locks[0] = fd_lk_ctx_node_new (small->cmd, NULL);
+                if (!v->locks[0])
+                        goto out;
+
+                memcpy (v->locks[0], big, sizeof (fd_lk_ctx_node_t));
+
+                v->locks[0]->fl_type            = small->fl_type;
+                v->locks[0]->user_flock.l_type = small->fl_type;
+        } else if ((small->fl_start > big->fl_start) &&
+                   (small->fl_end   < big->fl_end)) {
+                /* small lock is completely inside big lock,
+                   break it down into 3 different locks. */
+                v->locks[0] = fd_lk_ctx_node_new (big->cmd, NULL);
+                if (!v->locks[0])
+                        goto out;
+
+                v->locks[1] = fd_lk_ctx_node_new (small->cmd, NULL);
+                if (!v->locks[1])
+                        goto out;
+
+                v->locks[2] = fd_lk_ctx_node_new (big->cmd, NULL);
+                if (!v->locks[2])
+                        goto out;
+
+                memcpy (v->locks[0], big, sizeof (fd_lk_ctx_node_t));
+                v->locks[0]->fl_end = small->fl_start - 1;
+                v->locks[0]->user_flock.l_len =
+                        _fd_lk_get_lock_len (v->locks[0]->fl_start,
+                                             v->locks[0]->fl_end);
+
+                memcpy (v->locks[1], small, sizeof (fd_lk_ctx_node_t));
+
+                memcpy (v->locks[2], big, sizeof (fd_lk_ctx_node_t));
+                v->locks[2]->fl_start = small->fl_end + 1;
+                v->locks[2]->user_flock.l_len =
+                        _fd_lk_get_lock_len (v->locks[2]->fl_start,
+                                             v->locks[2]->fl_end);
+        } else if (small->fl_start == big->fl_start) {
+                /* One of the ends co-incide, break the
+                   locks into two seperate parts */
+                v->locks[0] = fd_lk_ctx_node_new (small->cmd, NULL);
+                if (!v->locks[0])
+                        goto out;
+
+                v->locks[1] = fd_lk_ctx_node_new (big->cmd, NULL);
+                if (!v->locks[1])
+                        goto out;
+
+                memcpy (v->locks[0], small, sizeof (fd_lk_ctx_node_t));
+
+                memcpy (v->locks[1], big, sizeof (fd_lk_ctx_node_t));
+                v->locks[1]->fl_start = small->fl_end + 1;
+                v->locks[1]->user_flock.l_start = small->fl_end + 1;
+        } else if (small->fl_end == big->fl_end) {
+                /* One of the ends co-incide, break the
+                   locks into two seperate parts */
+                v->locks[0] = fd_lk_ctx_node_new (small->cmd, NULL);
+                if (!v->locks[0])
+                        goto out;
+
+                v->locks[1] = fd_lk_ctx_node_new (big->cmd, NULL);
+                if (!v->locks[1])
+                        goto out;
+
+                memcpy (v->locks[0], big, sizeof (fd_lk_ctx_node_t));
+                v->locks[0]->fl_end = small->fl_start - 1;
+                v->locks[0]->user_flock.l_len =
+                        _fd_lk_get_lock_len (v->locks[0]->fl_start,
+                                             v->locks[0]->fl_end);
+
+                memcpy (v->locks[1], small, sizeof (fd_lk_ctx_node_t));
+        } else {
+                /* We should never come to this case */
+                GF_ASSERT (!"Invalid case");
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+static void
+_fd_lk_insert_and_merge (fd_lk_ctx_t *lk_ctx,
+                         fd_lk_ctx_node_t *lock)
+{
+        int32_t               ret     = -1;
+        int32_t               i       = 0;
+        fd_lk_ctx_node_t     *entry   = NULL;
+        fd_lk_ctx_node_t     *t       = NULL;
+        fd_lk_ctx_node_t     *sum     = NULL;
+        struct _values        v       = {.locks = {0, 0, 0 }};
+
+        list_for_each_entry_safe (entry, t, &lk_ctx->lk_list, next) {
+                if (!fd_lk_overlap (entry, lock))
+                        continue;
+
+                if (entry->fl_type == lock->fl_type) {
+                        sum = _fd_lk_add_locks (entry, lock);
+                        if (!sum)
+                                return;
+                        sum->fl_type = entry->fl_type;
+                        sum->user_flock.l_type = entry->fl_type;
+                        _fd_lk_delete_lock (entry);
+                        _fd_lk_destroy_lock (entry);
+                        _fd_lk_destroy_lock (lock);
+                        _fd_lk_insert_and_merge (lk_ctx, sum);
+                        return;
+                } else {
+                        sum = _fd_lk_add_locks (entry, lock);
+                        sum->fl_type = lock->fl_type;
+                        sum->user_flock.l_type = lock->fl_type;
+                        ret = _fd_lk_sub_locks (&v, sum, lock);
+                        if (ret)
+                                return;
+                        _fd_lk_delete_lock (entry);
+                        _fd_lk_destroy_lock (entry);
+
+                        _fd_lk_delete_lock (lock);
+                        _fd_lk_destroy_lock (lock);
+
+                        _fd_lk_destroy_lock (sum);
+
+                        for (i = 0; i < 3; i++) {
+                                if (!v.locks[i])
+                                        continue;
+
+                                INIT_LIST_HEAD (&v.locks[i]->next);
+                                _fd_lk_insert_and_merge (lk_ctx, v.locks[i]);
+                        }
+                        _fd_lk_delete_unlck_locks (lk_ctx);
+                        return;
+                }
+        }
+
+        /* no conflicts, so just insert */
+        if (lock->fl_type != F_UNLCK) {
+                _fd_lk_insert_lock (lk_ctx, lock);
+        } else {
+                _fd_lk_destroy_lock (lock);
+        }
+}
+
+static void
+print_lock_list (fd_lk_ctx_t *lk_ctx)
+{
+        fd_lk_ctx_node_t    *lk     = NULL;
+
+        gf_log ("fd-lk", GF_LOG_DEBUG, "lock list:");
+
+        list_for_each_entry (lk, &lk_ctx->lk_list, next)
+                gf_log ("fd-lk", GF_LOG_DEBUG, "owner = %s, "
+                        "cmd = %s fl_type = %s, fs_start = %"PRId64", "
+                        "fs_end = %"PRId64", user_flock: l_type = %s, "
+                        "l_start = %"PRId64", l_len = %"PRId64", ",
+                        lkowner_utoa (&lk->user_flock.l_owner),
+                        get_lk_cmd (lk->cmd), get_lk_type (lk->fl_type),
+                        lk->fl_start, lk->fl_end,
+                        get_lk_type (lk->user_flock.l_type),
+                        lk->user_flock.l_start,
+                        lk->user_flock.l_len);
+}
+
+int
+fd_lk_insert_and_merge (fd_t *fd, int32_t cmd,
+                        struct gf_flock *flock)
+{
+        int32_t              ret      = -1;
+        fd_lk_ctx_t         *lk_ctx   = NULL;
+        fd_lk_ctx_node_t    *lk     = NULL;
+
+        GF_VALIDATE_OR_GOTO ("fd-lk", fd, out);
+        GF_VALIDATE_OR_GOTO ("fd-lk", flock, out);
+
+        lk_ctx = fd_lk_ctx_ref (fd->lk_ctx);
+        lk     = fd_lk_ctx_node_new (cmd, flock);
+
+        gf_log ("fd-lk", GF_LOG_DEBUG,
+                "new lock requrest: owner = %s, fl_type = %s, "
+                "fs_start = %"PRId64", fs_end = %"PRId64", "
+                "user_flock: l_type = %s, l_start = %"PRId64", "
+                "l_len = %"PRId64, lkowner_utoa (&flock->l_owner),
+                get_lk_type (lk->fl_type), lk->fl_start,
+                lk->fl_end, get_lk_type (lk->user_flock.l_type),
+                lk->user_flock.l_start,
+                lk->user_flock.l_len);
+
+        LOCK (&lk_ctx->lock);
+        {
+                _fd_lk_insert_and_merge (lk_ctx, lk);
+                print_lock_list (lk_ctx);
+        }
+        UNLOCK (&lk_ctx->lock);
+
+        fd_lk_ctx_unref (lk_ctx);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+gf_boolean_t
+fd_lk_ctx_empty (fd_lk_ctx_t *lk_ctx)
+{
+	gf_boolean_t verdict = _gf_true;
+
+	if (!lk_ctx)
+		return _gf_true;
+
+	LOCK (&lk_ctx->lock);
+	{
+		verdict = list_empty (&lk_ctx->lk_list);
+	}
+	UNLOCK (&lk_ctx->lock);
+
+	return verdict;
+}
diff --git a/libglusterfs/src/fd-lk.h b/libglusterfs/src/fd-lk.h
new file mode 100644
index 000000000..1d2ff794c
--- /dev/null
+++ b/libglusterfs/src/fd-lk.h
@@ -0,0 +1,70 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _FD_LK_H
+#define _FD_LK_H
+
+#include "fd.h"
+#include "locking.h"
+#include "list.h"
+#include "logging.h"
+#include "mem-pool.h"
+#include "mem-types.h"
+#include "glusterfs.h"
+#include "common-utils.h"
+
+#define get_lk_type(type)                                               \
+        type == F_UNLCK ? "F_UNLCK" : (type == F_RDLCK ? "F_RDLCK" : "F_WRLCK")
+
+#define get_lk_cmd(cmd)                                                 \
+        cmd == F_SETLKW ? "F_SETLKW" : (cmd == F_SETLK ? "F_SETLK" : "F_GETLK")
+
+struct _fd;
+
+struct fd_lk_ctx {
+        struct list_head lk_list;
+        int   ref;
+        gf_lock_t lock;
+};
+typedef struct fd_lk_ctx fd_lk_ctx_t;
+
+struct fd_lk_ctx_node {
+        int32_t            cmd;
+        struct gf_flock    user_flock;
+        off_t              fl_start;
+        off_t              fl_end;
+        short              fl_type;
+        struct list_head   next;
+};
+typedef struct fd_lk_ctx_node fd_lk_ctx_node_t;
+
+fd_lk_ctx_t *
+_fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx);
+
+fd_lk_ctx_t *
+fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx);
+
+fd_lk_ctx_t *
+fd_lk_ctx_try_ref (fd_lk_ctx_t *lk_ctx);
+
+fd_lk_ctx_t *
+fd_lk_ctx_create ();
+
+int
+fd_lk_insert_and_merge (struct _fd *lk_ctx, int32_t cmd,
+                        struct gf_flock *flock);
+
+int
+fd_lk_ctx_unref (fd_lk_ctx_t *lk_ctx);
+
+gf_boolean_t
+fd_lk_ctx_empty (fd_lk_ctx_t *lk_ctx);
+
+#endif /* _FD_LK_H */
diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index 62a70c457..36cc4d056 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "fd.h"
@@ -130,7 +121,7 @@ gf_fd_fdtable_alloc (void)
 }
 
 
-fdentry_t *
+static fdentry_t *
 __gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count)
 {
         fdentry_t       *fdentries = NULL;
@@ -168,6 +159,53 @@ gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count)
 }
 
 
+static fdentry_t *
+__gf_fd_fdtable_copy_all_fds (fdtable_t *fdtable, uint32_t *count)
+{
+        fdentry_t *fdentries = NULL;
+        int        i         = 0;
+
+        if (count == NULL) {
+                gf_log_callingfn ("fd", GF_LOG_WARNING, "!count");
+                goto out;
+        }
+
+        fdentries = GF_CALLOC (fdtable->max_fds, sizeof (fdentry_t),
+                               gf_common_mt_fdentry_t);
+        if (fdentries == NULL) {
+                goto out;
+        }
+
+        *count = fdtable->max_fds;
+
+        for (i = 0; i < fdtable->max_fds; i++) {
+                if (fdtable->fdentries[i].fd != NULL) {
+                        fdentries[i].fd = fd_ref (fdtable->fdentries[i].fd);
+                }
+        }
+
+out:
+        return fdentries;
+}
+
+
+fdentry_t *
+gf_fd_fdtable_copy_all_fds (fdtable_t *fdtable, uint32_t *count)
+{
+        fdentry_t *entries = NULL;
+
+        if (fdtable) {
+                pthread_mutex_lock (&fdtable->lock);
+                {
+                        entries = __gf_fd_fdtable_copy_all_fds (fdtable, count);
+                }
+                pthread_mutex_unlock (&fdtable->lock);
+        }
+
+        return entries;
+}
+
+
 void
 gf_fd_fdtable_destroy (fdtable_t *fdtable)
 {
@@ -309,6 +347,54 @@ unlock_out:
 }
 
 
+inline void
+gf_fdptr_put (fdtable_t *fdtable, fd_t *fd)
+{
+        fdentry_t *fde   = NULL;
+        int32_t    i     = 0;
+
+        if ((fdtable == NULL) || (fd == NULL)) {
+                gf_log_callingfn ("fd", GF_LOG_ERROR, "invalid argument");
+                return;
+        }
+
+        pthread_mutex_lock (&fdtable->lock);
+        {
+                for (i = 0; i < fdtable->max_fds; i++) {
+                        if (fdtable->fdentries[i].fd == fd) {
+                                fde = &fdtable->fdentries[i];
+                                break;
+                        }
+                }
+
+                if (fde == NULL) {
+                        gf_log_callingfn ("fd", GF_LOG_WARNING,
+                                "fd (%p) is not present in fdtable", fd);
+                        goto unlock_out;
+                }
+
+                /* If the entry is not allocated, put operation must return
+                 * without doing anything.
+                 * This has the potential of masking out any bugs in a user of
+                 * fd that ends up calling gf_fd_put twice for the same fd or
+                 * for an unallocated fd, but it is a price we have to pay for
+                 * ensuring sanity of our fd-table.
+                 */
+                if (fde->next_free != GF_FDENTRY_ALLOCATED)
+                        goto unlock_out;
+                fde->fd = NULL;
+                fde->next_free = fdtable->first_free;
+                fdtable->first_free = i;
+        }
+unlock_out:
+        pthread_mutex_unlock (&fdtable->lock);
+
+        if ((fd != NULL) && (fde != NULL)) {
+                fd_unref (fd);
+        }
+}
+
+
 fd_t *
 gf_fd_fdptr_get (fdtable_t *fdtable, int64_t fd)
 {
@@ -401,7 +487,7 @@ fd_destroy (fd_t *fd)
                 goto out;
 
         if (IA_ISDIR (fd->inode->ia_type)) {
-                for (i = 0; i < fd->xl_count; i++) {
+                for (i = 0; i <  fd->xl_count; i++) {
                         if (fd->_ctx[i].key) {
                                 xl = fd->_ctx[i].xl_key;
                                 old_THIS = THIS;
@@ -427,8 +513,14 @@ fd_destroy (fd_t *fd)
         LOCK_DESTROY (&fd->lock);
 
         GF_FREE (fd->_ctx);
+        LOCK (&fd->inode->lock);
+        {
+                fd->inode->fd_count--;
+        }
+        UNLOCK (&fd->inode->lock);
         inode_unref (fd->inode);
         fd->inode = (inode_t *)0xaaaaaaaa;
+        fd_lk_ctx_unref (fd->lk_ctx);
         mem_put (fd);
 out:
         return;
@@ -463,7 +555,9 @@ fd_unref (fd_t *fd)
 fd_t *
 __fd_bind (fd_t *fd)
 {
+        list_del_init (&fd->inode_list);
         list_add (&fd->inode_list, &fd->inode->fd_list);
+        fd->inode->fd_count++;
 
         return fd;
 }
@@ -488,7 +582,7 @@ fd_bind (fd_t *fd)
 
 
 static fd_t *
-__fd_create (inode_t *inode, pid_t pid)
+__fd_create (inode_t *inode, uint64_t pid)
 {
         fd_t *fd = NULL;
 
@@ -505,11 +599,12 @@ __fd_create (inode_t *inode, pid_t pid)
 
         fd->_ctx = GF_CALLOC (1, (sizeof (struct _fd_ctx) * fd->xl_count),
                               gf_common_mt_fd_ctx);
-        if (!fd->_ctx) {
-                mem_put (fd);
-                fd = NULL;
-                goto out;
-        }
+        if (!fd->_ctx)
+                goto free_fd;
+
+        fd->lk_ctx = fd_lk_ctx_create ();
+        if (!fd->lk_ctx)
+                goto free_fd_ctx;
 
         fd->inode = inode_ref (inode);
         fd->pid = pid;
@@ -518,6 +613,13 @@ __fd_create (inode_t *inode, pid_t pid)
         LOCK_INIT (&fd->lock);
 out:
         return fd;
+
+free_fd_ctx:
+        GF_FREE (fd->_ctx);
+free_fd:
+        mem_put (fd);
+
+        return NULL;
 }
 
 
@@ -526,6 +628,21 @@ fd_create (inode_t *inode, pid_t pid)
 {
         fd_t *fd = NULL;
 
+        fd = __fd_create (inode, (uint64_t)pid);
+        if (!fd)
+                goto out;
+
+        fd = fd_ref (fd);
+
+out:
+        return fd;
+}
+
+fd_t *
+fd_create_uint64 (inode_t *inode, uint64_t pid)
+{
+        fd_t *fd = NULL;
+
         fd = __fd_create (inode, pid);
         if (!fd)
                 goto out;
@@ -538,7 +655,7 @@ out:
 
 
 static fd_t *
-__fd_lookup (inode_t *inode, pid_t pid)
+__fd_lookup (inode_t *inode, uint64_t pid)
 {
         fd_t *iter_fd = NULL;
         fd_t *fd = NULL;
@@ -548,6 +665,12 @@ __fd_lookup (inode_t *inode, pid_t pid)
 
 
         list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (iter_fd->anonymous)
+                        /* If someone was interested in getting an
+                           anonymous fd (or was OK getting an anonymous fd),
+                           they can as well call fd_anonymous() directly */
+                        continue;
+
                 if (!pid || iter_fd->pid == pid) {
                         fd = __fd_ref (iter_fd);
                         break;
@@ -570,6 +693,25 @@ fd_lookup (inode_t *inode, pid_t pid)
 
         LOCK (&inode->lock);
         {
+                fd = __fd_lookup (inode, (uint64_t)pid);
+        }
+        UNLOCK (&inode->lock);
+
+        return fd;
+}
+
+fd_t *
+fd_lookup_uint64 (inode_t *inode, uint64_t pid)
+{
+        fd_t *fd = NULL;
+
+        if (!inode) {
+                gf_log_callingfn ("fd", GF_LOG_WARNING, "!inode");
+                return NULL;
+        }
+
+        LOCK (&inode->lock);
+        {
                 fd = __fd_lookup (inode, pid);
         }
         UNLOCK (&inode->lock);
@@ -577,25 +719,48 @@ fd_lookup (inode_t *inode, pid_t pid)
         return fd;
 }
 
+static fd_t *
+__fd_lookup_anonymous (inode_t *inode)
+{
+        fd_t *iter_fd = NULL;
+        fd_t *fd = NULL;
 
+        if (list_empty (&inode->fd_list))
+                return NULL;
 
-fd_t *
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (iter_fd->anonymous) {
+                        fd = __fd_ref (iter_fd);
+                        break;
+                }
+        }
+
+        return fd;
+}
+
+static fd_t *
 __fd_anonymous (inode_t *inode)
 {
         fd_t *fd = NULL;
 
-        fd = __fd_lookup (inode, -1);
+        fd = __fd_lookup_anonymous (inode);
 
+        /* if (fd); then we already have increased the refcount in
+           __fd_lookup_anonymous(), so no need of one more fd_ref().
+           if (!fd); then both create and bind wont bump up the ref
+           count, so we have to call fd_ref() after bind. */
         if (!fd) {
-                fd = __fd_create (inode, -1);
+                fd = __fd_create (inode, 0);
 
                 if (!fd)
                         return NULL;
 
+                fd->anonymous = _gf_true;
+
                 __fd_bind (fd);
-        }
 
-        __fd_ref (fd);
+                __fd_ref (fd);
+        }
 
         return fd;
 }
@@ -615,11 +780,28 @@ fd_anonymous (inode_t *inode)
         return fd;
 }
 
+fd_t*
+fd_lookup_anonymous (inode_t *inode)
+{
+        fd_t *fd = NULL;
+
+        if (!inode) {
+                gf_log_callingfn ("fd", GF_LOG_WARNING, "!inode");
+                return NULL;
+        }
+
+        LOCK (&inode->lock);
+        {
+                fd = __fd_lookup_anonymous (inode);
+        }
+        UNLOCK (&inode->lock);
+        return fd;
+}
 
 gf_boolean_t
 fd_is_anonymous (fd_t *fd)
 {
-        return (fd && fd->pid == -1);
+        return (fd && fd->anonymous);
 }
 
 
@@ -641,9 +823,12 @@ fd_list_empty (inode_t *inode)
 int
 __fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value)
 {
-        int index = 0;
-        int ret = 0;
-        int set_idx = -1;
+        int             index   = 0, new_xl_count = 0;
+        int             ret     = 0;
+        int             set_idx = -1;
+        void           *begin   = NULL;
+        size_t          diff    = 0;
+        struct _fd_ctx *tmp     = NULL;
 
 	if (!fd || !xlator)
 		return -1;
@@ -662,9 +847,33 @@ __fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value)
         }
 
         if (set_idx == -1) {
-                gf_log_callingfn ("", GF_LOG_WARNING, "%p %s", fd, xlator->name);
-                ret = -1;
-                goto out;
+                set_idx = fd->xl_count;
+
+                new_xl_count = fd->xl_count + xlator->graph->xl_count;
+
+                tmp = GF_REALLOC (fd->_ctx,
+                                  (sizeof (struct _fd_ctx)
+                                   * new_xl_count));
+                if (tmp == NULL) {
+                        gf_log_callingfn (THIS->name, GF_LOG_WARNING,
+                                          "realloc of fd->_ctx for fd "
+                                          "(ptr: %p) failed, cannot set the key"
+                                          , fd);
+                        ret = -1;
+                        goto out;
+                }
+
+                fd->_ctx = tmp;
+
+                begin = fd->_ctx;
+                begin += (fd->xl_count * sizeof (struct _fd_ctx));
+
+                diff = (new_xl_count - fd->xl_count )
+                        * sizeof (struct _fd_ctx);
+
+                memset (begin, 0, diff);
+
+                fd->xl_count = new_xl_count;
         }
 
         fd->_ctx[set_idx].xl_key = xlator;
@@ -797,9 +1006,16 @@ fd_dump (fd_t *fd, char *prefix)
                 return;
 
         memset(key, 0, sizeof(key));
-        gf_proc_dump_write("pid", "%d", fd->pid);
+        gf_proc_dump_write("pid", "%llu", fd->pid);
         gf_proc_dump_write("refcount", "%d", fd->refcount);
         gf_proc_dump_write("flags", "%d", fd->flags);
+
+        if (fd->inode) {
+                gf_proc_dump_build_key (key, "inode", NULL);
+                gf_proc_dump_add_section(key);
+                inode_dump (fd->inode, key);
+        }
+
 }
 
 
@@ -829,10 +1045,8 @@ fdtable_dump (fdtable_t *fdtable, char *prefix)
 
         ret = pthread_mutex_trylock (&fdtable->lock);
 
-        if (ret) {
-                gf_log ("fd", GF_LOG_WARNING, "Unable to acquire lock");
-                return;
-        }
+        if (ret)
+                goto out;
 
         memset(key, 0, sizeof(key));
         gf_proc_dump_build_key(key, prefix, "refcount");
@@ -852,6 +1066,12 @@ fdtable_dump (fdtable_t *fdtable, char *prefix)
         }
 
         pthread_mutex_unlock(&fdtable->lock);
+
+out:
+        if (ret != 0)
+                gf_proc_dump_write ("Unable to dump the fdtable",
+                                    "(Lock acquistion failed) %p", fdtable);
+        return;
 }
 
 
@@ -870,15 +1090,13 @@ fd_ctx_dump (fd_t *fd, char *prefix)
         LOCK (&fd->lock);
         {
                 if (fd->_ctx != NULL) {
-                        fd_ctx = GF_CALLOC (fd->inode->table->xl->graph->xl_count,
-                                            sizeof (*fd_ctx),
+                        fd_ctx = GF_CALLOC (fd->xl_count, sizeof (*fd_ctx),
                                             gf_common_mt_fd_ctx);
                         if (fd_ctx == NULL) {
                                 goto unlock;
                         }
 
-                        for (i = 0; i < fd->inode->table->xl->graph->xl_count;
-                             i++) {
+                        for (i = 0; i < fd->xl_count; i++) {
                                 fd_ctx[i] = fd->_ctx[i];
                         }
                 }
@@ -890,7 +1108,7 @@ unlock:
                 goto out;
         }
 
-        for (i = 0; i < fd->inode->table->xl->graph->xl_count; i++) {
+        for (i = 0; i < fd->xl_count; i++) {
                 if (fd_ctx[i].xl_key) {
                         xl = (xlator_t *)(long)fd_ctx[i].xl_key;
                         if (xl->dumpops && xl->dumpops->fdctx)
@@ -899,9 +1117,7 @@ unlock:
         }
 
 out:
-        if (fd_ctx != NULL) {
-                GF_FREE (fd_ctx);
-        }
+        GF_FREE (fd_ctx);
 
         return;
 }
diff --git a/libglusterfs/src/fd.h b/libglusterfs/src/fd.h
index d4cd9bd06..c1b9157d8 100644
--- a/libglusterfs/src/fd.h
+++ b/libglusterfs/src/fd.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _FD_H
@@ -30,9 +21,14 @@
 #include <unistd.h>
 #include "glusterfs.h"
 #include "locking.h"
+#include "fd-lk.h"
+#include "common-utils.h"
+
+#define GF_ANON_FD_NO -2
 
 struct _inode;
 struct _dict;
+struct fd_lk_ctx;
 
 struct _fd_ctx {
         union {
@@ -45,12 +41,8 @@ struct _fd_ctx {
         };
 };
 
-/* If this structure changes, please have mercy on the booster maintainer
- * and update the fd_t struct in booster/src/booster-fd.h.
- * See the comment there to know why.
- */
 struct _fd {
-        pid_t             pid;
+        uint64_t          pid;
 	int32_t           flags;
         int32_t           refcount;
         struct list_head  inode_list;
@@ -59,6 +51,8 @@ struct _fd {
                                    'struct _fd_ctx' array (_ctx).*/
 	struct _fd_ctx   *_ctx;
         int               xl_count; /* Number of xl referred in this fd */
+        struct fd_lk_ctx *lk_ctx;
+        gf_boolean_t      anonymous; /* geo-rep anonymous fd */
 };
 typedef struct _fd fd_t;
 
@@ -92,7 +86,7 @@ typedef struct _fdtable fdtable_t;
 #include "xlator.h"
 
 
-inline void
+void
 gf_fd_put (fdtable_t *fdtable, int32_t fd);
 
 
@@ -117,6 +111,10 @@ gf_fd_fdtable_destroy (fdtable_t *fdtable);
 
 
 fd_t *
+__fd_ref (fd_t *fd);
+
+
+fd_t *
 fd_ref (fd_t *fd);
 
 
@@ -127,10 +125,17 @@ fd_unref (fd_t *fd);
 fd_t *
 fd_create (struct _inode *inode, pid_t pid);
 
+fd_t *
+fd_create_uint64 (struct _inode *inode, uint64_t pid);
 
 fd_t *
 fd_lookup (struct _inode *inode, pid_t pid);
 
+fd_t *
+fd_lookup_uint64 (struct _inode *inode, uint64_t pid);
+
+fd_t*
+fd_lookup_anonymous (inode_t *inode);
 
 fd_t *
 fd_anonymous (inode_t *inode);
@@ -159,6 +164,9 @@ fd_ctx_get (fd_t *fd, xlator_t *xlator, uint64_t *value);
 int
 fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value);
 
+int
+__fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value);
+
 
 int
 __fd_ctx_set (fd_t *fd, xlator_t *xlator, uint64_t value);
@@ -168,13 +176,14 @@ int
 __fd_ctx_get (fd_t *fd, xlator_t *xlator, uint64_t *value);
 
 
-int
-__fd_ctx_del (fd_t *fd, xlator_t *xlator, uint64_t *value);
+void
+fd_ctx_dump (fd_t *fd, char *prefix);
+
+fdentry_t *
+gf_fd_fdtable_copy_all_fds (fdtable_t *fdtable, uint32_t *count);
 
-fd_t *
-__fd_ref (fd_t *fd);
 
 void
-fd_ctx_dump (fd_t *fd, char *prefix);
+gf_fdptr_put (fdtable_t *fdtable, fd_t *fd);
 
 #endif /* _FD_H */
diff --git a/libglusterfs/src/gf-dirent.c b/libglusterfs/src/gf-dirent.c
index 8d15b0405..bb028c967 100644
--- a/libglusterfs/src/gf-dirent.c
+++ b/libglusterfs/src/gf-dirent.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
diff --git a/libglusterfs/src/gf-dirent.h b/libglusterfs/src/gf-dirent.h
index 29b4aba80..588d522db 100644
--- a/libglusterfs/src/gf-dirent.h
+++ b/libglusterfs/src/gf-dirent.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -54,9 +45,10 @@ struct _gf_dirent_t {
         struct iatt                          d_stat;
         dict_t                              *dict;
         inode_t                             *inode;
-	char                                 d_name[0];
+	char                                 d_name[];
 };
 
+#define DT_ISDIR(mode) (mode == DT_DIR)
 
 gf_dirent_t *gf_dirent_for_name (const char *name);
 void gf_dirent_free (gf_dirent_t *entries);
diff --git a/libglusterfs/src/gidcache.c b/libglusterfs/src/gidcache.c
new file mode 100644
index 000000000..c5bdda925
--- /dev/null
+++ b/libglusterfs/src/gidcache.c
@@ -0,0 +1,192 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "gidcache.h"
+#include "mem-pool.h"
+
+/*
+ * We treat this as a very simple set-associative LRU cache, with entries aged
+ * out after a configurable interval.  Hardly rocket science, but lots of
+ * details to worry about.
+ */
+#define BUCKET_START(p,n)       ((p) + ((n) * AUX_GID_CACHE_ASSOC))
+
+/*
+ * Initialize the cache.
+ */
+int gid_cache_init(gid_cache_t *cache, uint32_t timeout)
+{
+	if (!cache)
+		return -1;
+
+	LOCK_INIT(&cache->gc_lock);
+	cache->gc_max_age = timeout;
+	cache->gc_nbuckets = AUX_GID_CACHE_BUCKETS;
+	memset(cache->gc_cache, 0, sizeof(gid_list_t) * AUX_GID_CACHE_SIZE);
+
+	return 0;
+}
+
+/*
+ * Reconfigure the cache timeout.
+ */
+int gid_cache_reconf(gid_cache_t *cache, uint32_t timeout)
+{
+        if (!cache)
+                return -1;
+
+        LOCK(&cache->gc_lock);
+        cache->gc_max_age = timeout;
+        UNLOCK(&cache->gc_lock);
+
+        return 0;
+}
+
+/*
+ * Look up an ID in the cache. If found, return the actual cache entry to avoid
+ * an additional allocation and memory copy. The caller should copy the data and
+ * release (unlock) the cache as soon as possible.
+ */
+const gid_list_t *gid_cache_lookup(gid_cache_t *cache, uint64_t id)
+{
+	int bucket;
+	int i;
+	time_t now;
+	const gid_list_t *agl;
+
+	LOCK(&cache->gc_lock);
+	now = time(NULL);
+	bucket = id % cache->gc_nbuckets;
+	agl = BUCKET_START(cache->gc_cache, bucket);
+	for (i = 0; i < AUX_GID_CACHE_ASSOC; i++, agl++) {
+		if (!agl->gl_list)
+			continue;
+		if (agl->gl_id != id)
+			continue;
+
+		/*
+		 * We don't put new entries in the cache when expiration=0, but
+		 * there might be entries still in there if expiration was
+		 * changed very recently.  Writing the check this way ensures
+		 * that they're not used.
+		 */
+		if (now < agl->gl_deadline) {
+			return agl;
+		}
+
+		/*
+		 * We're not going to find any more UID matches, and reaping
+		 * is handled further down to maintain LRU order.
+		 */
+		break;
+	}
+	UNLOCK(&cache->gc_lock);
+	return NULL;
+}
+
+/*
+ * Release an entry found via lookup.
+ */
+void gid_cache_release(gid_cache_t *cache, const gid_list_t *agl)
+{
+	UNLOCK(&cache->gc_lock);
+}
+
+/*
+ * Add a new list entry to the cache. If an entry for this ID already exists,
+ * update it.
+ */
+int gid_cache_add(gid_cache_t *cache, gid_list_t *gl)
+{
+	gid_list_t *agl;
+	int bucket;
+	int i;
+	time_t now;
+
+	if (!gl || !gl->gl_list)
+		return -1;
+
+	if (!cache->gc_max_age)
+		return 0;
+
+	LOCK(&cache->gc_lock);
+	now = time(NULL);
+
+	/*
+	 * Scan for the first free entry or one that matches this id. The id
+	 * check is added to address a bug where the cache might contain an
+	 * expired entry for this id. Since lookup occurs in LRU order and
+	 * does not reclaim entries, it will always return failure on discovery
+	 * of an expired entry. This leads to duplicate entries being added,
+	 * which still do not satisfy lookups until the expired entry (and
+	 * everything before it) is reclaimed.
+	 *
+	 * We address this through reuse of an entry already allocated to this
+	 * id, whether expired or not, since we have obviously already received
+	 * more recent data. The entry is repopulated with the new data and a new
+	 * deadline and is pushed forward to reside as the last populated entry in
+	 * the bucket.
+	 */
+	bucket = gl->gl_id % cache->gc_nbuckets;
+	agl = BUCKET_START(cache->gc_cache, bucket);
+	for (i = 0; i < AUX_GID_CACHE_ASSOC; ++i, ++agl) {
+		if (agl->gl_id == gl->gl_id)
+			break;
+		if (!agl->gl_list)
+			break;
+	}
+
+	/*
+	 * The way we allocate free entries naturally places the newest
+	 * ones at the highest indices, so evicting the lowest makes
+	 * sense, but that also means we can't just replace it with the
+	 * one that caused the eviction.  That would cause us to thrash
+	 * the first entry while others remain idle.  Therefore, we
+	 * need to slide the other entries down and add the new one at
+	 * the end just as if the *last* slot had been free.
+	 *
+	 * Deadline expiration is also handled here, since the oldest
+	 * expired entry will be in the first position.  This does mean
+	 * the bucket can stay full of expired entries if we're idle
+	 * but, if the small amount of extra memory or scan time before
+	 * we decide to evict someone ever become issues, we could
+	 * easily add a reaper thread.
+	 */
+
+	if (i >= AUX_GID_CACHE_ASSOC) {
+		/* cache full, evict the first (LRU) entry */
+		i = 0;
+		agl = BUCKET_START(cache->gc_cache, bucket);
+		GF_FREE(agl->gl_list);
+	} else if (agl->gl_list) {
+		/* evict the old entry we plan to reuse */
+		GF_FREE(agl->gl_list);
+	}
+
+	/*
+	 * If we have evicted an entry, slide the subsequent populated entries
+	 * back and populate the last entry.
+	 */
+	for (; i < AUX_GID_CACHE_ASSOC - 1; i++) {
+		if (!agl[1].gl_list)
+			break;
+		agl[0] = agl[1];
+		agl++;
+	}
+
+	agl->gl_id = gl->gl_id;
+	agl->gl_count = gl->gl_count;
+	agl->gl_list = gl->gl_list;
+	agl->gl_deadline = now + cache->gc_max_age;
+
+	UNLOCK(&cache->gc_lock);
+
+	return 1;
+}
diff --git a/libglusterfs/src/gidcache.h b/libglusterfs/src/gidcache.h
new file mode 100644
index 000000000..9379f8e8b
--- /dev/null
+++ b/libglusterfs/src/gidcache.h
@@ -0,0 +1,53 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __GIDCACHE_H__
+#define __GIDCACHE_H__
+
+#include "glusterfs.h"
+#include "locking.h"
+
+/*
+ * TBD: make the cache size tunable
+ *
+ * The current size represents a pretty trivial amount of memory, and should
+ * provide good hit rates even for quite busy systems.  If we ever want to
+ * support really large cache sizes, we'll need to do dynamic allocation
+ * instead of just defining an array within a private structure. It doesn't make
+ * a whole lot of sense to change the associativity, because it won't improve
+ * hit rates all that much and will increase the maintenance cost as we have
+ * to scan more entries with every lookup/update.
+ */
+
+#define AUX_GID_CACHE_ASSOC     4
+#define AUX_GID_CACHE_BUCKETS   256
+#define AUX_GID_CACHE_SIZE      (AUX_GID_CACHE_ASSOC * AUX_GID_CACHE_BUCKETS)
+
+typedef struct {
+	uint64_t	gl_id;
+	int		gl_count;
+	gid_t		*gl_list;
+	time_t		gl_deadline;
+} gid_list_t;
+
+typedef struct {
+	gf_lock_t	gc_lock;
+	uint32_t	gc_max_age;
+	unsigned int	gc_nbuckets;
+	gid_list_t	gc_cache[AUX_GID_CACHE_SIZE];
+} gid_cache_t;
+
+int gid_cache_init(gid_cache_t *, uint32_t);
+int gid_cache_reconf(gid_cache_t *, uint32_t);
+const gid_list_t *gid_cache_lookup(gid_cache_t *, uint64_t);
+void gid_cache_release(gid_cache_t *, const gid_list_t *);
+int gid_cache_add(gid_cache_t *, gid_list_t *);
+
+#endif /* __GIDCACHE_H__ */
diff --git a/libglusterfs/src/globals.c b/libglusterfs/src/globals.c
index 9355a3a46..259c5c885 100644
--- a/libglusterfs/src/globals.c
+++ b/libglusterfs/src/globals.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -28,115 +19,74 @@
 #include "globals.h"
 #include "xlator.h"
 #include "mem-pool.h"
-
-
-/* gf_*_list[] */
-
-char *gf_fop_list[GF_FOP_MAXVALUE];
-char *gf_mgmt_list[GF_MGMT_MAXVALUE];
-
-
-void
-gf_op_list_init()
-{
-        gf_fop_list[GF_FOP_NULL]        = "NULL";
-        gf_fop_list[GF_FOP_STAT]        = "STAT";
-        gf_fop_list[GF_FOP_READLINK]    = "READLINK";
-        gf_fop_list[GF_FOP_MKNOD]       = "MKNOD";
-        gf_fop_list[GF_FOP_MKDIR]       = "MKDIR";
-        gf_fop_list[GF_FOP_UNLINK]      = "UNLINK";
-        gf_fop_list[GF_FOP_RMDIR]       = "RMDIR";
-        gf_fop_list[GF_FOP_SYMLINK]     = "SYMLINK";
-        gf_fop_list[GF_FOP_RENAME]      = "RENAME";
-        gf_fop_list[GF_FOP_LINK]        = "LINK";
-        gf_fop_list[GF_FOP_TRUNCATE]    = "TRUNCATE";
-        gf_fop_list[GF_FOP_OPEN]        = "OPEN";
-        gf_fop_list[GF_FOP_READ]        = "READ";
-        gf_fop_list[GF_FOP_WRITE]       = "WRITE";
-        gf_fop_list[GF_FOP_STATFS]      = "STATFS";
-        gf_fop_list[GF_FOP_FLUSH]       = "FLUSH";
-        gf_fop_list[GF_FOP_FSYNC]       = "FSYNC";
-        gf_fop_list[GF_FOP_SETXATTR]    = "SETXATTR";
-        gf_fop_list[GF_FOP_GETXATTR]    = "GETXATTR";
-        gf_fop_list[GF_FOP_REMOVEXATTR] = "REMOVEXATTR";
-        gf_fop_list[GF_FOP_OPENDIR]     = "OPENDIR";
-        gf_fop_list[GF_FOP_FSYNCDIR]    = "FSYNCDIR";
-        gf_fop_list[GF_FOP_ACCESS]      = "ACCESS";
-        gf_fop_list[GF_FOP_CREATE]      = "CREATE";
-        gf_fop_list[GF_FOP_FTRUNCATE]   = "FTRUNCATE";
-        gf_fop_list[GF_FOP_FSTAT]       = "FSTAT";
-        gf_fop_list[GF_FOP_LK]          = "LK";
-        gf_fop_list[GF_FOP_LOOKUP]      = "LOOKUP";
-        gf_fop_list[GF_FOP_READDIR]     = "READDIR";
-        gf_fop_list[GF_FOP_INODELK]     = "INODELK";
-        gf_fop_list[GF_FOP_FINODELK]    = "FINODELK";
-        gf_fop_list[GF_FOP_ENTRYLK]     = "ENTRYLK";
-        gf_fop_list[GF_FOP_FENTRYLK]    = "FENTRYLK";
-        gf_fop_list[GF_FOP_XATTROP]     = "XATTROP";
-        gf_fop_list[GF_FOP_FXATTROP]    = "FXATTROP";
-        gf_fop_list[GF_FOP_FSETXATTR]   = "FSETXATTR";
-        gf_fop_list[GF_FOP_FGETXATTR]   = "FGETXATTR";
-        gf_fop_list[GF_FOP_RCHECKSUM]   = "RCHECKSUM";
-        gf_fop_list[GF_FOP_SETATTR]     = "SETATTR";
-        gf_fop_list[GF_FOP_FSETATTR]    = "FSETATTR";
-        gf_fop_list[GF_FOP_READDIRP]    = "READDIRP";
-        gf_fop_list[GF_FOP_GETSPEC]     = "GETSPEC";
-        gf_fop_list[GF_FOP_FORGET]      = "FORGET";
-        gf_fop_list[GF_FOP_RELEASE]     = "RELEASE";
-        gf_fop_list[GF_FOP_RELEASEDIR]  = "RELEASEDIR";
-
-        gf_fop_list[GF_MGMT_NULL]  = "NULL";
-        return;
-}
-
-
-/* CTX */
-static glusterfs_ctx_t *glusterfs_ctx;
-
-
-int
-glusterfs_ctx_init ()
-{
-        int  ret = 0;
-
-        if (glusterfs_ctx) {
-                gf_log_callingfn ("", GF_LOG_WARNING, "init called again");
-                goto out;
-        }
-
-        glusterfs_ctx = CALLOC (1, sizeof (*glusterfs_ctx));
-        if (!glusterfs_ctx) {
-                ret = -1;
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&glusterfs_ctx->graphs);
-        INIT_LIST_HEAD (&glusterfs_ctx->mempool_list);
-        ret = pthread_mutex_init (&glusterfs_ctx->lock, NULL);
-
-out:
-        return ret;
-}
-
-
-glusterfs_ctx_t *
-glusterfs_ctx_get ()
-{
-        return glusterfs_ctx;
-
-}
-
-
+#include "syncop.h"
+
+const char *gf_fop_list[GF_FOP_MAXVALUE] = {
+        [GF_FOP_NULL]        = "NULL",
+        [GF_FOP_STAT]        = "STAT",
+        [GF_FOP_READLINK]    = "READLINK",
+        [GF_FOP_MKNOD]       = "MKNOD",
+        [GF_FOP_MKDIR]       = "MKDIR",
+        [GF_FOP_UNLINK]      = "UNLINK",
+        [GF_FOP_RMDIR]       = "RMDIR",
+        [GF_FOP_SYMLINK]     = "SYMLINK",
+        [GF_FOP_RENAME]      = "RENAME",
+        [GF_FOP_LINK]        = "LINK",
+        [GF_FOP_TRUNCATE]    = "TRUNCATE",
+        [GF_FOP_OPEN]        = "OPEN",
+        [GF_FOP_READ]        = "READ",
+        [GF_FOP_WRITE]       = "WRITE",
+        [GF_FOP_STATFS]      = "STATFS",
+        [GF_FOP_FLUSH]       = "FLUSH",
+        [GF_FOP_FSYNC]       = "FSYNC",
+        [GF_FOP_SETXATTR]    = "SETXATTR",
+        [GF_FOP_GETXATTR]    = "GETXATTR",
+        [GF_FOP_REMOVEXATTR] = "REMOVEXATTR",
+        [GF_FOP_OPENDIR]     = "OPENDIR",
+        [GF_FOP_FSYNCDIR]    = "FSYNCDIR",
+        [GF_FOP_ACCESS]      = "ACCESS",
+        [GF_FOP_CREATE]      = "CREATE",
+        [GF_FOP_FTRUNCATE]   = "FTRUNCATE",
+        [GF_FOP_FSTAT]       = "FSTAT",
+        [GF_FOP_LK]          = "LK",
+        [GF_FOP_LOOKUP]      = "LOOKUP",
+        [GF_FOP_READDIR]     = "READDIR",
+        [GF_FOP_INODELK]     = "INODELK",
+        [GF_FOP_FINODELK]    = "FINODELK",
+        [GF_FOP_ENTRYLK]     = "ENTRYLK",
+        [GF_FOP_FENTRYLK]    = "FENTRYLK",
+        [GF_FOP_XATTROP]     = "XATTROP",
+        [GF_FOP_FXATTROP]    = "FXATTROP",
+        [GF_FOP_FSETXATTR]   = "FSETXATTR",
+        [GF_FOP_FGETXATTR]   = "FGETXATTR",
+        [GF_FOP_RCHECKSUM]   = "RCHECKSUM",
+        [GF_FOP_SETATTR]     = "SETATTR",
+        [GF_FOP_FSETATTR]    = "FSETATTR",
+        [GF_FOP_READDIRP]    = "READDIRP",
+        [GF_FOP_GETSPEC]     = "GETSPEC",
+        [GF_FOP_FORGET]      = "FORGET",
+        [GF_FOP_RELEASE]     = "RELEASE",
+        [GF_FOP_RELEASEDIR]  = "RELEASEDIR",
+        [GF_FOP_FREMOVEXATTR]= "FREMOVEXATTR",
+	[GF_FOP_FALLOCATE]   = "FALLOCATE",
+	[GF_FOP_DISCARD]     = "DISCARD",
+        [GF_FOP_ZEROFILL]     = "ZEROFILL",
+};
 /* THIS */
 
 xlator_t global_xlator;
 static pthread_key_t this_xlator_key;
+static pthread_key_t synctask_key;
+static pthread_key_t uuid_buf_key;
+static char          global_uuid_buf[GF_UUID_BUF_SIZE];
+static pthread_key_t lkowner_buf_key;
+static char          global_lkowner_buf[GF_LKOWNER_BUF_SIZE];
+
 
 void
 glusterfs_this_destroy (void *ptr)
 {
-        if (ptr)
-                FREE (ptr);
+        FREE (ptr);
 }
 
 
@@ -153,7 +103,6 @@ glusterfs_this_init ()
 
         global_xlator.name = "glusterfs";
         global_xlator.type = "global";
-        global_xlator.ctx  = glusterfs_ctx;
 
         INIT_LIST_HEAD (&global_xlator.volume_options);
 
@@ -176,8 +125,6 @@ __glusterfs_this_location ()
 
                 ret = pthread_setspecific (this_xlator_key, this_location);
                 if (ret != 0) {
-                        gf_log ("", GF_LOG_WARNING, "pthread setspecific failed");
-
                         FREE (this_location);
                         this_location = NULL;
                         goto out;
@@ -219,10 +166,55 @@ glusterfs_this_set (xlator_t *this)
         return 0;
 }
 
-/* SYNCTASK */
+/* SYNCOPCTX */
+static pthread_key_t syncopctx_key;
 
-static pthread_key_t synctask_key;
+static void
+syncopctx_key_destroy (void *ptr)
+{
+	struct syncopctx *opctx = ptr;
 
+	if (opctx) {
+		if (opctx->groups)
+			GF_FREE (opctx->groups);
+
+		GF_FREE (opctx);
+	}
+
+	return;
+}
+
+void *
+syncopctx_getctx ()
+{
+	void *opctx = NULL;
+
+	opctx = pthread_getspecific (syncopctx_key);
+
+	return opctx;
+}
+
+int
+syncopctx_setctx (void *ctx)
+{
+	int ret = 0;
+
+	ret = pthread_setspecific (syncopctx_key, ctx);
+
+	return ret;
+}
+
+static int
+syncopctx_init (void)
+{
+	int ret;
+
+	ret = pthread_key_create (&syncopctx_key, syncopctx_key_destroy);
+
+	return ret;
+}
+
+/* SYNCTASK */
 
 int
 synctask_init ()
@@ -234,7 +226,6 @@ synctask_init ()
         return ret;
 }
 
-
 void *
 synctask_get ()
 {
@@ -258,13 +249,10 @@ synctask_set (void *synctask)
 
 //UUID_BUFFER
 
-static pthread_key_t uuid_buf_key;
-static char global_uuid_buf[GF_UUID_BUF_SIZE];
 void
 glusterfs_uuid_buf_destroy (void *ptr)
 {
-        if (ptr)
-                FREE (ptr);
+        FREE (ptr);
 }
 
 int
@@ -287,7 +275,7 @@ glusterfs_uuid_buf_get ()
         if(!buf) {
                 buf = MALLOC (GF_UUID_BUF_SIZE);
                 ret = pthread_setspecific (uuid_buf_key, (void *) buf);
-                if(ret)
+                if (ret)
                         buf = global_uuid_buf;
         }
         return buf;
@@ -295,13 +283,10 @@ glusterfs_uuid_buf_get ()
 
 /* LKOWNER_BUFFER */
 
-static pthread_key_t lkowner_buf_key;
-static char global_lkowner_buf[GF_LKOWNER_BUF_SIZE];
 void
 glusterfs_lkowner_buf_destroy (void *ptr)
 {
-        if (ptr)
-                FREE (ptr);
+        FREE (ptr);
 }
 
 int
@@ -324,27 +309,18 @@ glusterfs_lkowner_buf_get ()
         if(!buf) {
                 buf = MALLOC (GF_LKOWNER_BUF_SIZE);
                 ret = pthread_setspecific (lkowner_buf_key, (void *) buf);
-                if(ret)
+                if (ret)
                         buf = global_lkowner_buf;
         }
         return buf;
 }
 
 int
-glusterfs_globals_init ()
+glusterfs_globals_init (glusterfs_ctx_t *ctx)
 {
         int ret = 0;
 
-        gf_op_list_init ();
-
-        gf_log_globals_init ();
-
-        ret = glusterfs_ctx_init ();
-        if (ret) {
-                gf_log ("", GF_LOG_CRITICAL,
-                        "ERROR: glusterfs context init failed");
-                goto out;
-        }
+        gf_log_globals_init (ctx);
 
         ret = glusterfs_this_init ();
         if (ret) {
@@ -367,42 +343,19 @@ glusterfs_globals_init ()
                 goto out;
         }
 
-        gf_mem_acct_enable_set ();
-
         ret = synctask_init ();
         if (ret) {
                 gf_log ("", GF_LOG_CRITICAL,
                         "ERROR: glusterfs synctask init failed");
                 goto out;
         }
+
+        ret = syncopctx_init ();
+        if (ret) {
+                gf_log ("", GF_LOG_CRITICAL,
+                        "ERROR: glusterfs syncopctx init failed");
+                goto out;
+        }
 out:
         return ret;
 }
-
-
-char eventstring[GF_EVENT_MAXVAL+1][64] = {
-        "Invalid event",
-        "Parent Up",
-        "Poll In",
-        "Poll Out",
-        "Poll Err",
-        "Child Up",
-        "Child Down",
-        "Child Connecting",
-        "Child Modified",
-        "Transport Cleanup",
-        "Transport Connected",
-        "Volfile Modified",
-        "New Volfile",
-        "Translator Info",
-        "Trigger Volume Heal",
-        "Authentication Failed",
-        "Invalid event",
-};
-
-/* Copy the string ptr contents if needed for yourself */
-char *
-glusterfs_strevent (glusterfs_event_t ev)
-{
-        return eventstring[ev];
-}
diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
index 86e1f0324..3085db21c 100644
--- a/libglusterfs/src/globals.h
+++ b/libglusterfs/src/globals.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _GLOBALS_H
@@ -22,33 +13,50 @@
 
 #define GF_DEFAULT_BASE_PORT 24007
 
-#include "glusterfs.h"
-
-/* CTX */
-#define CTX (glusterfs_ctx_get())
-
-glusterfs_ctx_t *glusterfs_ctx_get ();
+#define GD_OP_VERSION_KEY     "operating-version"
+#define GD_MIN_OP_VERSION_KEY "minimum-operating-version"
+#define GD_MAX_OP_VERSION_KEY "maximum-operating-version"
+
+/* Gluster versions - OP-VERSION mapping
+ *
+ * 3.3.0                - 1
+ * 3.4.0                - 2
+ * 3.next (3.5?)        - 3
+ *
+ * TODO: Change above comment once gluster version is finalised
+ * TODO: Finalize the op-version ranges
+ */
+#define GD_OP_VERSION_MIN  1 /* MIN is the fresh start op-version, mostly
+                                should not change */
+#define GD_OP_VERSION_MAX  3 /* MAX VERSION is the maximum count in VME table,
+                                should keep changing with introduction of newer
+                                versions */
 
 #include "xlator.h"
 
 /* THIS */
 #define THIS (*__glusterfs_this_location())
 
-#define GF_UUID_BUF_SIZE 50
-
 xlator_t **__glusterfs_this_location ();
 xlator_t *glusterfs_this_get ();
 int glusterfs_this_set (xlator_t *);
 
+/* syncopctx */
+void *syncopctx_getctx ();
+int syncopctx_setctx (void *ctx);
+
 /* task */
 void *synctask_get ();
 int synctask_set (void *);
 
 /* uuid_buf */
 char *glusterfs_uuid_buf_get();
+/* lkowner_buf */
 char *glusterfs_lkowner_buf_get();
 
 /* init */
-int glusterfs_globals_init (void);
+int glusterfs_globals_init (glusterfs_ctx_t *ctx);
+
+extern const char *gf_fop_list[];
 
 #endif /* !_GLOBALS_H */
diff --git a/libglusterfs/src/glusterfs-acl.h b/libglusterfs/src/glusterfs-acl.h
new file mode 100644
index 000000000..b7de1cdb4
--- /dev/null
+++ b/libglusterfs/src/glusterfs-acl.h
@@ -0,0 +1,81 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERFS_ACL_H
+#define _GLUSTERFS_ACL_H
+
+#include <stdint.h>
+#include <sys/types.h> /* For uid_t */
+
+#include "locking.h" /* For gf_lock_t in struct posix_acl_conf */
+
+#define ACL_PROGRAM    100227
+#define ACLV3_VERSION  3
+
+#define POSIX_ACL_MINIMAL_ACE_COUNT   3
+
+#define POSIX_ACL_READ                (0x04)
+#define POSIX_ACL_WRITE               (0x02)
+#define POSIX_ACL_EXECUTE             (0x01)
+
+#define POSIX_ACL_UNDEFINED_TAG       (0x00)
+#define POSIX_ACL_USER_OBJ            (0x01)
+#define POSIX_ACL_USER                (0x02)
+#define POSIX_ACL_GROUP_OBJ           (0x04)
+#define POSIX_ACL_GROUP               (0x08)
+#define POSIX_ACL_MASK                (0x10)
+#define POSIX_ACL_OTHER               (0x20)
+
+#define POSIX_ACL_UNDEFINED_ID        (-1)
+
+#define POSIX_ACL_VERSION             (0x02)
+
+#define POSIX_ACL_ACCESS_XATTR        "system.posix_acl_access"
+#define POSIX_ACL_DEFAULT_XATTR       "system.posix_acl_default"
+
+struct posix_acl_xattr_entry {
+        uint16_t            tag;
+        uint16_t            perm;
+        uint32_t            id;
+};
+
+struct posix_acl_xattr_header {
+        uint32_t                        version;
+        struct posix_acl_xattr_entry    entries[];
+};
+
+struct posix_ace {
+        uint16_t     tag;
+        uint16_t     perm;
+        uint32_t     id;
+};
+
+
+struct posix_acl {
+        int               refcnt;
+        int               count;
+        struct posix_ace  entries[];
+};
+
+struct posix_acl_ctx {
+        uid_t             uid;
+        gid_t             gid;
+        mode_t            perm;
+        struct posix_acl *acl_access;
+        struct posix_acl *acl_default;
+};
+
+struct posix_acl_conf {
+        gf_lock_t         acl_lock;
+        uid_t             super_uid;
+        struct posix_acl *minimal_acl;
+};
+
+#endif /* _GLUSTERFS_ACL_H */
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 8360077ec..2f1e12ee7 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _GLUSTERFS_H
@@ -41,16 +32,23 @@
 #include <arpa/inet.h>
 #include <sys/poll.h>
 #include <pthread.h>
+#include <limits.h> /* For PATH_MAX */
 
 #include "list.h"
 #include "logging.h"
+#include "lkowner.h"
 
 #define GF_YES 1
 #define GF_NO  0
 
 #ifndef O_LARGEFILE
 /* savannah bug #20053, patch for compiling on darwin */
-#define O_LARGEFILE 0
+#define O_LARGEFILE 0100000 /* from bits/fcntl.h */
+#endif
+
+#ifndef O_FMODE_EXEC
+/* redhat bug 843080, added from linux/fs.h */
+#define O_FMODE_EXEC 040 //0x20
 #endif
 
 #ifndef O_DIRECT
@@ -72,27 +70,63 @@
 #define FNM_EXTMATCH 0
 #endif
 
+#define GLUSTERD_MAX_SNAP_NAME  256
 #define ZR_MOUNTPOINT_OPT       "mountpoint"
 #define ZR_ATTR_TIMEOUT_OPT     "attribute-timeout"
 #define ZR_ENTRY_TIMEOUT_OPT    "entry-timeout"
+#define ZR_NEGATIVE_TIMEOUT_OPT "negative-timeout"
 #define ZR_DIRECT_IO_OPT        "direct-io-mode"
 #define ZR_STRICT_VOLFILE_CHECK "strict-volfile-check"
 #define ZR_DUMP_FUSE            "dump-fuse"
+#define ZR_FUSE_MOUNTOPTS       "fuse-mountopts"
 
 #define GF_XATTR_CLRLK_CMD      "glusterfs.clrlk"
 #define GF_XATTR_PATHINFO_KEY   "trusted.glusterfs.pathinfo"
+#define GF_XATTR_NODE_UUID_KEY  "trusted.glusterfs.node-uuid"
+#define GF_XATTR_VOL_ID_KEY   "trusted.glusterfs.volume-id"
+#define GF_XATTR_LOCKINFO_KEY   "trusted.glusterfs.lockinfo"
+#define GF_XATTR_GET_REAL_FILENAME_KEY "user.glusterfs.get_real_filename:"
+
+#define GF_READDIR_SKIP_DIRS       "readdir-filter-directories"
+
+#define BD_XATTR_KEY             "user.glusterfs"
+
+#define XATTR_IS_PATHINFO(x)  (strncmp (x, GF_XATTR_PATHINFO_KEY,       \
+                                        strlen (GF_XATTR_PATHINFO_KEY)) == 0)
+#define XATTR_IS_NODE_UUID(x) (strncmp (x, GF_XATTR_NODE_UUID_KEY,      \
+                                        strlen (GF_XATTR_NODE_UUID_KEY)) == 0)
+#define XATTR_IS_LOCKINFO(x) (strncmp (x, GF_XATTR_LOCKINFO_KEY,        \
+                                       strlen (GF_XATTR_LOCKINFO_KEY)) == 0)
+
+#define XATTR_IS_BD(x) (strncmp (x, BD_XATTR_KEY, strlen (BD_XATTR_KEY)) == 0)
+
 #define GF_XATTR_LINKINFO_KEY   "trusted.distribute.linkinfo"
-#define GFID_XATTR_KEY "trusted.gfid"
+#define GFID_XATTR_KEY          "trusted.gfid"
+#define VIRTUAL_GFID_XATTR_KEY_STR  "glusterfs.gfid.string"
+#define VIRTUAL_GFID_XATTR_KEY      "glusterfs.gfid"
+#define UUID_CANONICAL_FORM_LEN 36
+
+#define GLUSTERFS_INTERNAL_FOP_KEY  "glusterfs-internal-fop"
 
 #define ZR_FILE_CONTENT_STR     "glusterfs.file."
 #define ZR_FILE_CONTENT_STRLEN 15
 
+#define GLUSTERFS_WRITE_IS_APPEND "glusterfs.write-is-append"
 #define GLUSTERFS_OPEN_FD_COUNT "glusterfs.open-fd-count"
 #define GLUSTERFS_INODELK_COUNT "glusterfs.inodelk-count"
 #define GLUSTERFS_ENTRYLK_COUNT "glusterfs.entrylk-count"
 #define GLUSTERFS_POSIXLK_COUNT "glusterfs.posixlk-count"
+#define GLUSTERFS_PARENT_ENTRYLK "glusterfs.parent-entrylk"
+#define GLUSTERFS_INODELK_DOM_COUNT "glusterfs.inodelk-dom-count"
 #define QUOTA_SIZE_KEY "trusted.glusterfs.quota.size"
+#define GFID_TO_PATH_KEY "glusterfs.gfid2path"
+#define GF_XATTR_STIME_PATTERN "trusted.glusterfs.*.stime"
+
+/* Index xlator related */
+#define GF_XATTROP_INDEX_GFID "glusterfs.xattrop_index_gfid"
+#define GF_BASE_INDICES_HOLDER_GFID "glusterfs.base_indicies_holder_gfid"
 
+#define GF_GFIDLESS_LOOKUP "gfidless-lookup"
 /* replace-brick and pump related internal xattrs */
 #define RB_PUMP_CMD_START       "glusterfs.pump.start"
 #define RB_PUMP_CMD_PAUSE       "glusterfs.pump.pause"
@@ -109,16 +143,29 @@
 #define GLUSTERFS_RPC_REPLY_SIZE               24
 
 #define ZR_FILE_CONTENT_REQUEST(key) (!strncmp(key, ZR_FILE_CONTENT_STR, \
-					       ZR_FILE_CONTENT_STRLEN))
+                                               ZR_FILE_CONTENT_STRLEN))
 
-/* TODO: Should we use PATH-MAX? On some systems it may save space */
-#define ZR_PATH_MAX 4096
+#define DEFAULT_VAR_RUN_DIRECTORY    DATADIR "/run/gluster"
+#define GF_REPLICATE_TRASH_DIR          ".landfill"
 
 /* GlusterFS's maximum supported Auxilary GIDs */
 /* TODO: Keeping it to 200, so that we can fit in 2KB buffer for auth data
  * in RPC server code, if there is ever need for having more aux-gids, then
  * we have to add aux-gid in payload of actors */
-#define GF_MAX_AUX_GROUPS   200
+#define GF_MAX_AUX_GROUPS   65536
+
+#define GF_UUID_BUF_SIZE 50
+
+#define GF_REBALANCE_TID_KEY     "rebalance-id"
+#define GF_REMOVE_BRICK_TID_KEY  "remove-brick-id"
+#define GF_REPLACE_BRICK_TID_KEY "replace-brick-id"
+
+#define UUID_CANONICAL_FORM_LEN  36
+
+/* Adding this here instead of any glusterd*.h files as it is also required by
+ * cli
+ */
+#define DEFAULT_GLUSTERD_SOCKFILE             DATADIR "/run/glusterd.socket"
 
 /* NOTE: add members ONLY at the end (just before _MAXVALUE) */
 typedef enum {
@@ -153,8 +200,8 @@ typedef enum {
         GF_FOP_READDIR,
         GF_FOP_INODELK,
         GF_FOP_FINODELK,
-	GF_FOP_ENTRYLK,
-	GF_FOP_FENTRYLK,
+        GF_FOP_ENTRYLK,
+        GF_FOP_FENTRYLK,
         GF_FOP_XATTROP,
         GF_FOP_FXATTROP,
         GF_FOP_FGETXATTR,
@@ -168,6 +215,9 @@ typedef enum {
         GF_FOP_RELEASEDIR,
         GF_FOP_GETSPEC,
         GF_FOP_FREMOVEXATTR,
+	GF_FOP_FALLOCATE,
+	GF_FOP_DISCARD,
+        GF_FOP_ZEROFILL,
         GF_FOP_MAXVALUE,
 } glusterfs_fop_t;
 
@@ -217,21 +267,23 @@ typedef enum {
 
 
 typedef enum {
-	ENTRYLK_LOCK,
-	ENTRYLK_UNLOCK,
-	ENTRYLK_LOCK_NB
+        ENTRYLK_LOCK,
+        ENTRYLK_UNLOCK,
+        ENTRYLK_LOCK_NB
 } entrylk_cmd;
 
 
 typedef enum {
-	ENTRYLK_RDLCK,
-	ENTRYLK_WRLCK
+        ENTRYLK_RDLCK,
+        ENTRYLK_WRLCK
 } entrylk_type;
 
 
 typedef enum {
-	GF_XATTROP_ADD_ARRAY,
-        GF_XATTROP_ADD_ARRAY64
+        GF_XATTROP_ADD_ARRAY,
+        GF_XATTROP_ADD_ARRAY64,
+        GF_XATTROP_OR_ARRAY,
+        GF_XATTROP_AND_ARRAY
 } gf_xattrop_flags_t;
 
 
@@ -240,68 +292,79 @@ typedef enum {
 #define GF_SET_DIR_ONLY       0x4
 #define GF_SET_EPOCH_TIME     0x8 /* used by afr dir lookup selfheal */
 
-/* Directory into which replicate self-heal will move deleted files and
-   directories into. The storage/posix janitor thread will periodically
-   clean up this directory */
-
-#define GF_REPLICATE_TRASH_DIR          ".landfill"
-
 /* key value which quick read uses to get small files in lookup cbk */
 #define GF_CONTENT_KEY "glusterfs.content"
 
 struct _xlator_cmdline_option {
-	struct list_head    cmd_args;
-	char               *volume;
-	char               *key;
-	char               *value;
+        struct list_head    cmd_args;
+        char               *volume;
+        char               *key;
+        char               *value;
 };
 typedef struct _xlator_cmdline_option xlator_cmdline_option_t;
 
+struct _server_cmdline {
+        struct list_head  list;
+        char              *volfile_server;
+};
+typedef struct _server_cmdline server_cmdline_t;
 
 #define GF_OPTION_ENABLE   _gf_true
 #define GF_OPTION_DISABLE  _gf_false
 #define GF_OPTION_DEFERRED 2
 
 struct _cmd_args {
-	/* basic options */
-	char            *volfile_server;
-	char            *volfile;
-        char            *log_server;
-	gf_loglevel_t    log_level;
-	char            *log_file;
+        /* basic options */
+        char             *volfile_server;
+        server_cmdline_t *curr_server;
+        /* List of backup volfile servers, including original */
+        struct list_head volfile_servers;
+        char             *volfile;
+        char             *log_server;
+        gf_loglevel_t    log_level;
+        char            *log_file;
         int32_t          max_connect_attempts;
-	/* advanced options */
-	uint32_t         volfile_server_port;
-	char            *volfile_server_transport;
+        /* advanced options */
+        uint32_t         volfile_server_port;
+        char            *volfile_server_transport;
         uint32_t         log_server_port;
-	char            *pid_file;
+        char            *pid_file;
         char            *sock_file;
-	int              no_daemon_mode;
-	char            *run_id;
-	int              debug_mode;
+        int              no_daemon_mode;
+        char            *run_id;
+        int              debug_mode;
         int              read_only;
         int              acl;
+        int              selinux;
+        int              enable_ino32;
         int              worm;
         int              mac_compat;
-	struct list_head xlator_options;  /* list of xlator_option_t */
-
-	/* fuse options */
-	int              fuse_direct_io_mode;
+        int              fopen_keep_cache;
+        int              gid_timeout;
+        int              aux_gfid_mount;
+        struct list_head xlator_options;  /* list of xlator_option_t */
+
+        /* fuse options */
+        int              fuse_direct_io_mode;
+        char             *use_readdirp;
         int              volfile_check;
-	double           fuse_entry_timeout;
-	double           fuse_attribute_timeout;
-	char            *volume_name;
-	int              fuse_nodev;
-	int              fuse_nosuid;
-	char            *dump_fuse;
+        double           fuse_entry_timeout;
+        double           fuse_negative_timeout;
+        double           fuse_attribute_timeout;
+        char            *volume_name;
+        int              fuse_nodev;
+        int              fuse_nosuid;
+        char            *dump_fuse;
         pid_t            client_pid;
         int              client_pid_set;
         unsigned         uid_map_root;
+        int              background_qlen;
+        int              congestion_threshold;
+        char             *fuse_mountopts;
 
-
-	/* key args */
-	char            *mount_point;
-	char            *volfile_id;
+        /* key args */
+        char            *mount_point;
+        char            *volfile_id;
 
         /* required for portmap */
         int             brick_port;
@@ -326,17 +389,19 @@ struct _glusterfs_graph {
 typedef struct _glusterfs_graph glusterfs_graph_t;
 
 
+typedef int32_t (*glusterfsd_mgmt_event_notify_fn_t) (int32_t event, void *data,
+                                                      ...);
 struct _glusterfs_ctx {
-	cmd_args_t          cmd_args;
-	char               *process_uuid;
-	FILE               *pidfp;
-	char                fin;
-	void               *timer;
-	void               *ib;
-	void               *pool;
-	void               *event_pool;
+        cmd_args_t          cmd_args;
+        char               *process_uuid;
+        FILE               *pidfp;
+        char                fin;
+        void               *timer;
+        void               *ib;
+        struct call_pool   *pool;
+        void               *event_pool;
         void               *iobuf_pool;
-	pthread_mutex_t     lock;
+        pthread_mutex_t     lock;
         size_t              page_size;
         struct list_head    graphs; /* double linked list of graphs - one per volfile parse */
         glusterfs_graph_t  *active; /* the latest graph in use */
@@ -350,21 +415,34 @@ struct _glusterfs_ctx {
         int                 graph_id; /* Incremented per graph, value should
                                          indicate how many times the graph has
                                          got changed */
-        pid_t               mtab_pid; /* pid of the process which updates the mtab */
+        pid_t               mnt_pid; /* pid of the mount agent */
         int                 process_mode; /*mode in which process is runninng*/
-	struct syncenv      *env;         /* The env pointer to the synctasks */
+        struct syncenv     *env;          /* The env pointer to the synctasks */
 
         struct list_head    mempool_list; /* used to keep a global list of
                                              mempools, used to log details of
                                              mempool in statedump */
-        char                *statedump_path;
+        char               *statedump_path;
+
+        struct mem_pool    *dict_pool;
+        struct mem_pool    *dict_pair_pool;
+        struct mem_pool    *dict_data_pool;
+
+        glusterfsd_mgmt_event_notify_fn_t notify; /* Used for xlators to make
+                                                     call to fsd-mgmt */
+        gf_log_handle_t     log; /* all logging related variables */
+
+        int                 mem_acct_enable;
+
+        int                 daemon_pipe[2];
+
+        struct client_disconnect *client_disconnect;
+        struct clienttable *clienttable;
 };
 typedef struct _glusterfs_ctx glusterfs_ctx_t;
 
+glusterfs_ctx_t *glusterfs_ctx_new (void);
 
-/* If you edit this structure then, make a corresponding change in
- * globals.c in the eventstring.
- */
 typedef enum {
         GF_EVENT_PARENT_UP = 1,
         GF_EVENT_POLLIN,
@@ -379,14 +457,14 @@ typedef enum {
         GF_EVENT_VOLFILE_MODIFIED,
         GF_EVENT_GRAPH_NEW,
         GF_EVENT_TRANSLATOR_INFO,
-        GF_EVENT_TRIGGER_HEAL,
+        GF_EVENT_TRANSLATOR_OP,
         GF_EVENT_AUTH_FAILED,
+        GF_EVENT_VOLUME_DEFRAG,
+        GF_EVENT_PARENT_DOWN,
+        GF_EVENT_VOLUME_BARRIER_OP,
         GF_EVENT_MAXVAL,
 } glusterfs_event_t;
 
-/* gf_lkowner_t is defined in lkowner.h */
-#include "lkowner.h"
-
 struct gf_flock {
         short        l_type;
         short        l_whence;
@@ -396,9 +474,6 @@ struct gf_flock {
         gf_lkowner_t l_owner;
 };
 
-
-extern char *glusterfs_strevent (glusterfs_event_t ev);
-
 #define GF_MUST_CHECK __attribute__((warn_unused_result))
 /*
  * Some macros (e.g. ALLOC_OR_GOTO) set variables in function scope, but the
diff --git a/libglusterfs/src/graph-print.c b/libglusterfs/src/graph-print.c
index 432f9a1de..d860d63b3 100644
--- a/libglusterfs/src/graph-print.c
+++ b/libglusterfs/src/graph-print.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -33,6 +24,7 @@
 struct gf_printer {
         ssize_t (*write) (struct gf_printer *gp, char *buf, size_t len);
         void *priv;
+        int  len;
 };
 
 static ssize_t
@@ -89,19 +81,31 @@ gpprintf (struct gf_printer *gp, const char *format, ...)
         return ret;
 }
 
-static int
-glusterfs_graph_print (struct gf_printer *gp, glusterfs_graph_t *graph)
-{
 #define GPPRINTF(gp, fmt, ...) do {                             \
                 ret = gpprintf (gp, fmt, ## __VA_ARGS__);       \
                 if (ret == -1)                                  \
                         goto out;                               \
                 else                                            \
-                        len += ret;                             \
+                        gp->len += ret;                             \
         } while (0)
 
+static int
+_print_volume_options (dict_t *d, char *k, data_t *v,
+                           void *tmp)
+{
+        struct gf_printer *gp  = tmp;
+        int                ret = 0;
+        GPPRINTF (gp, "    option %s %s\n", k, v->data);
+        return 0;
+out:
+        /* means, it is a failure */
+        return -1;
+}
+
+static int
+glusterfs_graph_print (struct gf_printer *gp, glusterfs_graph_t *graph)
+{
         xlator_t      *trav = NULL;
-        data_pair_t   *pair = NULL;
         xlator_list_t *xch = NULL;
         int            ret = 0;
         ssize_t        len = 0;
@@ -114,11 +118,9 @@ glusterfs_graph_print (struct gf_printer *gp, glusterfs_graph_t *graph)
                 GPPRINTF (gp, "volume %s\n    type %s\n", trav->name,
                           trav->type);
 
-                for (pair =  trav->options->members_list; pair && pair->next;
-                     pair = pair->next);
-                for (; pair; pair = pair->prev)
-                        GPPRINTF (gp, "    option %s %s\n", pair->key,
-                                  pair->value->data);
+                ret = dict_foreach (trav->options, _print_volume_options, gp);
+                if (ret)
+                        goto out;
 
                 if (trav->children) {
                         GPPRINTF (gp, "    subvolumes");
@@ -135,6 +137,7 @@ glusterfs_graph_print (struct gf_printer *gp, glusterfs_graph_t *graph)
         }
 
 out:
+        len = gp->len;
         if (ret == -1) {
                 gf_log ("graph-print", GF_LOG_ERROR, "printing failed");
 
diff --git a/libglusterfs/src/graph-utils.h b/libglusterfs/src/graph-utils.h
index d125fbe0d..207664fdb 100644
--- a/libglusterfs/src/graph-utils.h
+++ b/libglusterfs/src/graph-utils.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _GRAPH_H_
diff --git a/libglusterfs/src/graph.c b/libglusterfs/src/graph.c
index 830c88363..e76df1ca5 100644
--- a/libglusterfs/src/graph.c
+++ b/libglusterfs/src/graph.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -29,28 +20,23 @@
 #include "defaults.h"
 
 
-
-
 #if 0
 static void
 _gf_dump_details (int argc, char **argv)
 {
         extern FILE *gf_log_logfile;
         int          i = 0;
-        char         timestr[256];
+        char         timestr[64];
         time_t       utime = 0;
-        struct tm   *tm = NULL;
         pid_t        mypid = 0;
         struct utsname uname_buf = {{0, }, };
         int            uname_ret = -1;
 
-        utime = time (NULL);
-        tm    = localtime (&utime);
         mypid = getpid ();
         uname_ret   = uname (&uname_buf);
 
-        /* Which git? What time? */
-        strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+        utime = time (NULL);
+        gf_time_fmt (timestr, sizeof timestr, utime, gf_timefmt_FT);
         fprintf (gf_log_logfile,
                  "========================================"
                  "========================================\n");
@@ -131,7 +117,8 @@ glusterfs_graph_set_first (glusterfs_graph_t *graph, xlator_t *xl)
 
 int
 glusterfs_graph_insert (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx,
-                        const char *type, const char *name)
+                        const char *type, const char *name,
+                        gf_boolean_t autoload)
 {
         xlator_t        *ixl = NULL;
 
@@ -157,6 +144,8 @@ glusterfs_graph_insert (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx,
         if (!ixl->name)
                 goto err;
 
+        ixl->is_autoloaded = autoload;
+
         if (xlator_set_type (ixl, type) == -1) {
                 gf_log ("glusterfs", GF_LOG_ERROR,
                         "%s (%s) initialization failed",
@@ -187,7 +176,7 @@ glusterfs_graph_acl (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
                 return 0;
 
         ret = glusterfs_graph_insert (graph, ctx, "system/posix-acl",
-                                      "posix-acl-autoload");
+                                      "posix-acl-autoload", 1);
         return ret;
 }
 
@@ -203,7 +192,7 @@ glusterfs_graph_worm (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
                 return 0;
 
         ret = glusterfs_graph_insert (graph, ctx, "features/worm",
-                                      "worm-autoload");
+                                      "worm-autoload", 1);
         return ret;
 }
 
@@ -219,11 +208,26 @@ glusterfs_graph_mac_compat (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
                 return 0;
 
         ret = glusterfs_graph_insert (graph, ctx, "features/mac-compat",
-                                      "mac-compat-autoload");
+                                      "mac-compat-autoload", 1);
 
         return ret;
 }
 
+int
+glusterfs_graph_gfid_access (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
+{
+        int ret = 0;
+        cmd_args_t      *cmd_args = NULL;
+
+        cmd_args = &ctx->cmd_args;
+
+        if (!cmd_args->aux_gfid_mount)
+                return 0;
+
+        ret = glusterfs_graph_insert (graph, ctx, "features/gfid-access",
+                                      "gfid-access-autoload", 1);
+        return ret;
+}
 
 static void
 gf_add_cmdline_options (glusterfs_graph_t *graph, cmd_args_t *cmd_args)
@@ -310,7 +314,7 @@ glusterfs_graph_init (glusterfs_graph_t *graph)
 }
 
 
-static void
+static int
 _log_if_unknown_option (dict_t *dict, char *key, data_t *value, void *data)
 {
         volume_option_t   *found = NULL;
@@ -325,7 +329,7 @@ _log_if_unknown_option (dict_t *dict, char *key, data_t *value, void *data)
                         "option '%s' is not recognized", key);
         }
 
-        return;
+        return 0;
 }
 
 
@@ -349,8 +353,7 @@ fill_uuid (char *uuid, int size)
 {
         char           hostname[256] = {0,};
         struct timeval tv = {0,};
-        struct tm      now = {0, };
-        char           now_str[32];
+        char           now_str[64];
 
         if (gettimeofday (&tv, NULL) == -1) {
                 gf_log ("graph", GF_LOG_ERROR,
@@ -364,8 +367,7 @@ fill_uuid (char *uuid, int size)
                         strerror (errno));
         }
 
-        localtime_r (&tv.tv_sec, &now);
-        strftime (now_str, 32, "%Y/%m/%d-%H:%M:%S", &now);
+        gf_time_fmt (now_str, sizeof now_str, tv.tv_sec, gf_timefmt_Ymd_T);
         snprintf (uuid, size, "%s-%d-%s:%"GF_PRI_SUSECONDS,
                   hostname, getpid(), now_str, tv.tv_usec);
 
@@ -454,6 +456,14 @@ glusterfs_graph_prepare (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
                 return -1;
         }
 
+        /* XXX: gfid-access */
+        ret = glusterfs_graph_gfid_access (graph, ctx);
+        if (ret) {
+                gf_log ("graph", GF_LOG_ERROR,
+                        "glusterfs graph 'gfid-access' failed");
+                return -1;
+        }
+
         /* XXX: this->ctx setting */
         for (trav = graph->first; trav; trav = trav->next) {
                 trav->ctx = ctx;
@@ -505,11 +515,14 @@ glusterfs_graph_activate (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
         ctx->active = graph;
 
         /* XXX: attach to master and set active pointer */
-        if (ctx->master)
+        if (ctx->master) {
                 ret = xlator_notify (ctx->master, GF_EVENT_GRAPH_NEW, graph);
-        if (ret) {
-                gf_log ("graph", GF_LOG_ERROR, "graph new notification failed");
-                return ret;
+                if (ret) {
+                        gf_log ("graph", GF_LOG_ERROR,
+                                "graph new notification failed");
+                        return ret;
+                }
+                ((xlator_t *)ctx->master)->next = graph->top;
         }
 
         /* XXX: perform parent up */
@@ -522,18 +535,204 @@ glusterfs_graph_activate (glusterfs_graph_t *graph, glusterfs_ctx_t *ctx)
         return 0;
 }
 
+
+int
+xlator_equal_rec (xlator_t *xl1, xlator_t *xl2)
+{
+        xlator_list_t *trav1 = NULL;
+        xlator_list_t *trav2 = NULL;
+        int            ret   = 0;
+
+        if (xl1 == NULL || xl2 == NULL) {
+                gf_log ("xlator", GF_LOG_DEBUG, "invalid argument");
+                return -1;
+        }
+
+        trav1 = xl1->children;
+        trav2 = xl2->children;
+
+        while (trav1 && trav2) {
+                ret = xlator_equal_rec (trav1->xlator, trav2->xlator);
+                if (ret) {
+                        gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+                                "xlators children not equal");
+                        goto out;
+                }
+
+                trav1 = trav1->next;
+                trav2 = trav2->next;
+        }
+
+        if (trav1 || trav2) {
+                ret = -1;
+                goto out;
+        }
+
+        if (strcmp (xl1->name, xl2->name)) {
+                ret = -1;
+                goto out;
+        }
+
+	/* type could have changed even if xlator names match,
+	   e.g cluster/distrubte and cluster/nufa share the same
+	   xlator name
+	*/
+        if (strcmp (xl1->type, xl2->type)) {
+                ret = -1;
+                goto out;
+        }
+out :
+        return ret;
+}
+
+
+gf_boolean_t
+is_graph_topology_equal (glusterfs_graph_t *graph1, glusterfs_graph_t *graph2)
+{
+        xlator_t    *trav1    = NULL;
+        xlator_t    *trav2    = NULL;
+        gf_boolean_t ret      = _gf_true;
+
+        trav1 = graph1->first;
+        trav2 = graph2->first;
+
+        ret = xlator_equal_rec (trav1, trav2);
+
+        if (ret) {
+                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+                        "graphs are not equal");
+                ret = _gf_false;
+                goto out;
+        }
+
+        ret = _gf_true;
+        gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+                "graphs are equal");
+
+out:
+        return ret;
+}
+
+
+/* Function has 3types of return value 0, -ve , 1
+ *   return 0          =======> reconfiguration of options has succeeded
+ *   return 1          =======> the graph has to be reconstructed and all the xlators should be inited
+ *   return -1(or -ve) =======> Some Internal Error occurred during the operation
+ */
+int
+glusterfs_volfile_reconfigure (int oldvollen, FILE *newvolfile_fp,
+                               glusterfs_ctx_t *ctx, const char *oldvolfile)
+{
+        glusterfs_graph_t *oldvolfile_graph = NULL;
+        glusterfs_graph_t *newvolfile_graph = NULL;
+        FILE              *oldvolfile_fp    = NULL;
+        gf_boolean_t      active_graph_found = _gf_true;
+
+        int ret = -1;
+
+        if (!oldvollen) {
+                ret = 1; // Has to call INIT for the whole graph
+                goto out;
+        }
+
+        if (!ctx) {
+                gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
+			"ctx is NULL");
+		goto out;
+	}
+
+        oldvolfile_graph = ctx->active;
+        if (!oldvolfile_graph) {
+                active_graph_found = _gf_false;
+                gf_log ("glusterfsd-mgmt", GF_LOG_ERROR,
+                        "glusterfs_ctx->active is NULL");
+
+                oldvolfile_fp = tmpfile ();
+                if (!oldvolfile_fp) {
+                        gf_log ("glusterfsd-mgmt", GF_LOG_ERROR, "Unable to "
+                                "create temporary volfile: (%s)",
+                                strerror (errno));
+                        goto out;
+                }
+
+                fwrite (oldvolfile, oldvollen, 1, oldvolfile_fp);
+                fflush (oldvolfile_fp);
+                if (ferror (oldvolfile_fp)) {
+                        goto out;
+                }
+
+                oldvolfile_graph = glusterfs_graph_construct (oldvolfile_fp);
+                if (!oldvolfile_graph)
+                        goto out;
+        }
+
+        newvolfile_graph = glusterfs_graph_construct (newvolfile_fp);
+        if (!newvolfile_graph) {
+                goto out;
+        }
+
+        if (!is_graph_topology_equal (oldvolfile_graph,
+                                      newvolfile_graph)) {
+
+                ret = 1;
+                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+                        "Graph topology not equal(should call INIT)");
+                goto out;
+        }
+
+        gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+                "Only options have changed in the new "
+                "graph");
+
+        /* */
+        ret = glusterfs_graph_reconfigure (oldvolfile_graph,
+                                           newvolfile_graph);
+        if (ret) {
+                gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+                        "Could not reconfigure new options in old graph");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (oldvolfile_fp)
+                fclose (oldvolfile_fp);
+
+        /*  Do not simply destroy the old graph here. If the oldgraph
+            is constructed here in this function itself instead of getting
+            it from ctx->active (which happens only of ctx->active is NULL),
+            then destroy the old graph. If some i/o is still happening in
+            the old graph and the old graph is obtained from ctx->active,
+            then destroying the graph will cause problems.
+        */
+        if (!active_graph_found && oldvolfile_graph)
+                glusterfs_graph_destroy (oldvolfile_graph);
+        if (newvolfile_graph)
+                glusterfs_graph_destroy (newvolfile_graph);
+
+        return ret;
+}
+
+
 int
 glusterfs_graph_reconfigure (glusterfs_graph_t *oldgraph,
                              glusterfs_graph_t *newgraph)
 {
-        xlator_t *old_xl = NULL;
-        xlator_t *new_xl = NULL;
+        xlator_t   *old_xl   = NULL;
+        xlator_t   *new_xl   = NULL;
 
         GF_ASSERT (oldgraph);
         GF_ASSERT (newgraph);
 
         old_xl   = oldgraph->first;
+        while (old_xl->is_autoloaded) {
+                old_xl = old_xl->children->xlator;
+        }
+
         new_xl   = newgraph->first;
+        while (new_xl->is_autoloaded) {
+                new_xl = new_xl->children->xlator;
+        }
 
         return xlator_tree_reconfigure (old_xl, new_xl);
 }
@@ -541,5 +740,12 @@ glusterfs_graph_reconfigure (glusterfs_graph_t *oldgraph,
 int
 glusterfs_graph_destroy (glusterfs_graph_t *graph)
 {
+        xlator_tree_free (graph->first);
+
+        if (graph) {
+                list_del_init (&graph->list);
+                GF_FREE (graph);
+        }
+
         return 0;
 }
diff --git a/libglusterfs/src/graph.l b/libglusterfs/src/graph.l
index 04fff2582..e4eba9cbe 100644
--- a/libglusterfs/src/graph.l
+++ b/libglusterfs/src/graph.l
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 %x STRING
 %option yylineno
 %option noinput
@@ -80,10 +70,10 @@ TYPE   [t][y][p][e]
                               yyunput (0, NULL);
                        }
                        BEGIN (INITIAL);
-                       yylval = text;
+                       graphyylval = text;
                        return STRING_TOK;
        }
 }
-[^ \t\r\n\"\\]+ {    yylval = gf_strdup (yytext) ; return ID; }
+[^ \t\r\n\"\\]+ {    graphyylval = gf_strdup (yytext) ; return ID; }
 [ \t\r\n]+             ;
 %%
diff --git a/libglusterfs/src/graph.y b/libglusterfs/src/graph.y
index 62c57dd6c..a220abeb9 100644
--- a/libglusterfs/src/graph.y
+++ b/libglusterfs/src/graph.y
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 %token VOLUME_BEGIN VOLUME_END OPTION NEWLINE SUBVOLUME ID WHITESPACE COMMENT TYPE STRING_TOK
 
 %{
@@ -28,6 +18,7 @@
 #include <sys/mman.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <pthread.h>
 
 #define RELAX_POISONING
 
@@ -47,8 +38,8 @@ static void option_error (void);
 #define YYSTYPE char *
 #define GF_CMD_BUFFER_LEN (8 * GF_UNIT_KB)
 
-int yyerror (const char *);
-int yylex ();
+int graphyyerror (const char *);
+int graphyylex ();
 %}
 
 
@@ -88,11 +79,11 @@ glusterfs_graph_t *construct;
 static void
 type_error (void)
 {
-        extern int yylineno;
+        extern int graphyylineno;
 
         gf_log ("parser", GF_LOG_ERROR,
                 "Volume %s, before line %d: Please specify volume type",
-                curr->name, yylineno);
+                curr->name, graphyylineno);
         return;
 }
 
@@ -100,11 +91,11 @@ type_error (void)
 static void
 sub_error (void)
 {
-        extern int yylineno;
+        extern int graphyylineno;
 
         gf_log ("parser", GF_LOG_ERROR,
                 "Volume %s, before line %d: Please specify subvolumes",
-                curr->name, yylineno);
+                curr->name, graphyylineno);
         return;
 }
 
@@ -112,12 +103,12 @@ sub_error (void)
 static void
 option_error (void)
 {
-        extern int yylineno;
+        extern int graphyylineno;
 
         gf_log ("parser", GF_LOG_ERROR,
                 "Volume %s, before line %d: Please specify "
                 "option <key> <value>",
-                curr->name, yylineno);
+                curr->name, graphyylineno);
         return;
 }
 
@@ -125,7 +116,7 @@ option_error (void)
 static int
 new_volume (char *name)
 {
-        extern int   yylineno;
+        extern int   graphyylineno;
         xlator_t    *trav = NULL;
         int          ret = 0;
 
@@ -139,7 +130,7 @@ new_volume (char *name)
         if (curr) {
                 gf_log ("parser", GF_LOG_ERROR,
                         "new volume (%s) defintion in line %d unexpected",
-                        name, yylineno);
+                        name, graphyylineno);
                 ret = -1;
                 goto out;
         }
@@ -159,7 +150,7 @@ new_volume (char *name)
                 if (!strcmp (name, trav->name)) {
                         gf_log ("parser", GF_LOG_ERROR,
 				"Line %d: volume '%s' defined again",
-                                yylineno, name);
+                                graphyylineno, name);
                         ret = -1;
                         goto out;
                 }
@@ -204,7 +195,7 @@ out:
 static int
 volume_type (char *type)
 {
-        extern int   yylineno;
+        extern int   graphyylineno;
         int32_t      ret = 0;
 
         if (!type) {
@@ -218,7 +209,7 @@ volume_type (char *type)
                 gf_log ("parser", GF_LOG_ERROR,
                         "Volume '%s', line %d: type '%s' is not valid or "
 			"not found on this machine",
-                        curr->name, yylineno, type);
+                        curr->name, graphyylineno, type);
                 ret = -1;
                 goto out;
         }
@@ -235,7 +226,7 @@ out:
 static int
 volume_option (char *key, char *value)
 {
-        extern int  yylineno;
+        extern int  graphyylineno;
         int         ret = 0;
         char       *set_value = NULL;
 
@@ -252,7 +243,7 @@ volume_option (char *key, char *value)
                 gf_log ("parser", GF_LOG_ERROR,
                         "Volume '%s', line %d: duplicate entry "
 			"('option %s') present",
-                        curr->name, yylineno, key);
+                        curr->name, graphyylineno, key);
                 ret = -1;
                 goto out;
         }
@@ -271,7 +262,7 @@ out:
 static int
 volume_sub (char *sub)
 {
-        extern int       yylineno;
+        extern int       graphyylineno;
         xlator_t        *trav = NULL;
         int              ret = 0;
 
@@ -293,7 +284,7 @@ volume_sub (char *sub)
                 gf_log ("parser", GF_LOG_ERROR,
                         "Volume '%s', line %d: subvolume '%s' is not defined "
 			"prior to usage",
-                        curr->name, yylineno, sub);
+                        curr->name, graphyylineno, sub);
                 ret = -1;
                 goto out;
         }
@@ -301,7 +292,7 @@ volume_sub (char *sub)
         if (trav == curr) {
                 gf_log ("parser", GF_LOG_ERROR,
                         "Volume '%s', line %d: has '%s' itself as subvolume",
-                        curr->name, yylineno, sub);
+                        curr->name, graphyylineno, sub);
                 ret = -1;
                 goto out;
         }
@@ -338,46 +329,46 @@ volume_end (void)
 
 
 int
-yywrap ()
+graphyywrap ()
 {
         return 1;
 }
 
 
 int
-yyerror (const char *str)
+graphyyerror (const char *str)
 {
-        extern char  *yytext;
-        extern int    yylineno;
+        extern char  *graphyytext;
+        extern int    graphyylineno;
 
-        if (curr && curr->name && yytext) {
-                if (!strcmp (yytext, "volume")) {
+        if (curr && curr->name && graphyytext) {
+                if (!strcmp (graphyytext, "volume")) {
                         gf_log ("parser", GF_LOG_ERROR,
                                 "'end-volume' not defined for volume '%s'",
 				curr->name);
-                } else if (!strcmp (yytext, "type")) {
+                } else if (!strcmp (graphyytext, "type")) {
                         gf_log ("parser", GF_LOG_ERROR,
                                 "line %d: duplicate 'type' defined for "
 				"volume '%s'",
-                                yylineno, curr->name);
-                } else if (!strcmp (yytext, "subvolumes")) {
+                                graphyylineno, curr->name);
+                } else if (!strcmp (graphyytext, "subvolumes")) {
                         gf_log ("parser", GF_LOG_ERROR,
                                 "line %d: duplicate 'subvolumes' defined for "
 				"volume '%s'",
-                                yylineno, curr->name);
+                                graphyylineno, curr->name);
                 } else if (curr) {
                         gf_log ("parser", GF_LOG_ERROR,
                                 "syntax error: line %d (volume '%s'): \"%s\""
 				"\nallowed tokens are 'volume', 'type', "
 				"'subvolumes', 'option', 'end-volume'()",
-                                yylineno, curr->name,
-				yytext);
+                                graphyylineno, curr->name,
+				graphyytext);
                 } else {
                         gf_log ("parser", GF_LOG_ERROR,
                                 "syntax error: line %d (just after volume "
 				"'%s'): \"%s\"\n(%s)",
-                                yylineno, curr->name,
-				yytext,
+                                graphyylineno, curr->name,
+				graphyytext,
                                 "allowed tokens are 'volume', 'type', "
 				"'subvolumes', 'option', 'end-volume'");
                 }
@@ -386,7 +377,7 @@ yyerror (const char *str)
                         "syntax error in line %d: \"%s\" \n"
                         "(allowed tokens are 'volume', 'type', "
 			"'subvolumes', 'option', 'end-volume')\n",
-                        yylineno, yytext);
+                        graphyylineno, graphyytext);
         }
 
         return -1;
@@ -491,6 +482,7 @@ preprocess (FILE *srcfp, FILE *dstfp)
 					cmd_buf_size *= 2;
 					cmd = GF_REALLOC (cmd, cmd_buf_size);
                                         if (cmd == NULL) {
+                                                GF_FREE (result);
                                                 return -1;
                                         }
 
@@ -532,6 +524,7 @@ preprocess (FILE *srcfp, FILE *dstfp)
 out:
 	fseek (srcfp, 0L, SEEK_SET);
 	fseek (dstfp, 0L, SEEK_SET);
+
 	GF_FREE (cmd);
 	GF_FREE (result);
 
@@ -539,7 +532,7 @@ out:
 }
 
 
-extern FILE *yyin;
+extern FILE *graphyyin;
 
 glusterfs_graph_t *
 glusterfs_graph_new ()
@@ -563,52 +556,65 @@ glusterfs_graph_t *
 glusterfs_graph_construct (FILE *fp)
 {
         int                ret = 0;
+        int                tmp_fd = -1;
         glusterfs_graph_t *graph = NULL;
-	FILE              *tmp_file = NULL;
+        FILE              *tmp_file = NULL;
+        char               template[PATH_MAX] = {0};
+	static pthread_mutex_t graph_mutex = PTHREAD_MUTEX_INITIALIZER;
 
         graph = glusterfs_graph_new ();
         if (!graph)
-                return NULL;
+                goto err;
 
-	tmp_file = tmpfile ();
+        strcpy (template, "/tmp/tmp.XXXXXX");
+        tmp_fd = mkstemp (template);
+        if (-1 == tmp_fd)
+                goto err;
 
-	if (tmp_file == NULL) {
-		gf_log ("parser", GF_LOG_ERROR,
-			"cannot create temparory file");
+        ret = unlink (template);
+        if (ret < 0) {
+                gf_log ("parser", GF_LOG_WARNING, "Unable to delete file: %s",
+                        template);
+        }
 
-                glusterfs_graph_destroy (graph);
-		return NULL;
-	}
+        tmp_file = fdopen (tmp_fd, "w+b");
+        if (!tmp_file)
+                goto err;
 
-	ret = preprocess (fp, tmp_file);
-	if (ret < 0) {
-		gf_log ("parser", GF_LOG_ERROR,
-			"parsing of backticks failed");
+        ret = preprocess (fp, tmp_file);
+        if (ret < 0) {
+                gf_log ("parser", GF_LOG_ERROR, "parsing of backticks failed");
+                goto err;
+        }
 
-                glusterfs_graph_destroy (graph);
-		fclose (tmp_file);
-		return NULL;
+	pthread_mutex_lock (&graph_mutex);
+	{
+		graphyyin = tmp_file;
+		construct = graph;
+		ret = yyparse ();
+		construct = NULL;
 	}
-
-        yyin = tmp_file;
-
-        construct = graph;
-
-        ret = yyparse ();
-
-        construct = NULL;
-
-	fclose (tmp_file);
+	pthread_mutex_unlock (&graph_mutex);
 
         if (ret == 1) {
                 gf_log ("parser", GF_LOG_DEBUG,
-			"parsing of volfile failed, please review it "
-			"once more");
-
-                glusterfs_graph_destroy (graph);
-                return NULL;
+                        "parsing of volfile failed, please review it "
+                        "once more");
+                goto err;
         }
 
+        fclose (tmp_file);
         return graph;
+err:
+        if (tmp_file) {
+                fclose (tmp_file);
+        } else {
+                gf_log ("parser", GF_LOG_ERROR, "cannot create temporary file");
+                if (-1 != tmp_fd)
+                        close (tmp_fd);
+        }
+
+        glusterfs_graph_destroy (graph);
+        return NULL;
 }
 
diff --git a/libglusterfs/src/hashfn.c b/libglusterfs/src/hashfn.c
index b5752956c..f79165b22 100644
--- a/libglusterfs/src/hashfn.c
+++ b/libglusterfs/src/hashfn.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <stdint.h>
diff --git a/libglusterfs/src/hashfn.h b/libglusterfs/src/hashfn.h
index af306745b..06ae37e79 100644
--- a/libglusterfs/src/hashfn.h
+++ b/libglusterfs/src/hashfn.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __HASHFN_H__
diff --git a/libglusterfs/src/iatt.h b/libglusterfs/src/iatt.h
index b84eaa379..60ae59047 100644
--- a/libglusterfs/src/iatt.h
+++ b/libglusterfs/src/iatt.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -103,6 +94,8 @@ struct iatt {
 #define IA_PROT_SGID(prot)      ((prot).sgid == 1)
 #define IA_PROT_STCKY(prot)     ((prot).sticky == 1)
 
+#define IA_FILE_OR_DIR(t)       (IA_ISREG(t) || IA_ISDIR(t))
+
 static inline uint32_t
 ia_major (uint64_t ia_dev)
 {
@@ -272,6 +265,24 @@ iatt_from_stat (struct iatt *iatt, struct stat *stat)
         iatt->ia_blksize    = stat->st_blksize;
         iatt->ia_blocks     = stat->st_blocks;
 
+        /* There is a possibility that the backend FS (like XFS) can
+           allocate blocks beyond EOF for better performance reasons, which
+           results in 'st_blocks' with higher values than what is consumed by
+           the file descriptor. This would break few logic inside GlusterFS,
+           like quota behavior etc, thus we need the exact number of blocks
+           which are consumed by the file to the higher layers inside GlusterFS.
+           Currently, this logic won't work for sparse files (ie, file with
+           holes)
+        */
+        {
+                uint64_t maxblocks;
+
+                maxblocks = (iatt->ia_size + 511) / 512;
+
+                if (iatt->ia_blocks > maxblocks)
+                        iatt->ia_blocks = maxblocks;
+        }
+
         iatt->ia_atime      = stat->st_atime;
         iatt->ia_atime_nsec = ST_ATIM_NSEC (stat);
 
diff --git a/libglusterfs/src/inode.c b/libglusterfs/src/inode.c
index 4685ec8d2..15e0ccf78 100644
--- a/libglusterfs/src/inode.c
+++ b/libglusterfs/src/inode.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -143,8 +134,7 @@ __dentry_unset (dentry_t *dentry)
 
         list_del_init (&dentry->inode_list);
 
-        if (dentry->name)
-                GF_FREE (dentry->name);
+        GF_FREE (dentry->name);
 
         if (dentry->parent) {
                 __inode_unref (dentry->parent);
@@ -320,7 +310,7 @@ __inode_destroy (inode_t *inode)
                 goto noctx;
         }
 
-        for (index = 0; index < inode->table->xl->graph->xl_count; index++) {
+        for (index = 0; index < inode->table->ctxcount; index++) {
                 if (inode->_ctx[index].xl_key) {
                         xl = (xlator_t *)(long)inode->_ctx[index].xl_key;
                         old_THIS = THIS;
@@ -538,10 +528,9 @@ __inode_create (inode_table_t *table)
         INIT_LIST_HEAD (&newi->hash);
         INIT_LIST_HEAD (&newi->dentry_list);
 
-        newi->_ctx = GF_CALLOC (1, (sizeof (struct _inode_ctx) *
-                                    table->xl->graph->xl_count),
+        newi->_ctx = GF_CALLOC (1,
+                                (sizeof (struct _inode_ctx) * table->ctxcount),
                                 gf_common_mt_inode_ctx);
-
         if (newi->_ctx == NULL) {
                 LOCK_DESTROY (&newi->lock);
                 mem_put (newi);
@@ -660,6 +649,50 @@ inode_grep (inode_table_t *table, inode_t *parent, const char *name)
         return inode;
 }
 
+
+inode_t *
+inode_resolve (inode_table_t *table, char *path)
+{
+        char    *tmp   = NULL, *bname = NULL, *str = NULL, *saveptr = NULL;
+        inode_t *inode = NULL, *parent = NULL;
+
+        if ((path == NULL) || (table == NULL)) {
+                goto out;
+        }
+
+        parent = inode_ref (table->root);
+        str = tmp = gf_strdup (path);
+
+        while (1) {
+                bname = strtok_r (str, "/", &saveptr);
+                if (bname == NULL) {
+                        break;
+                }
+
+                if (inode != NULL) {
+                        inode_unref (inode);
+                }
+
+                inode = inode_grep (table, parent, bname);
+                if (inode == NULL) {
+                        break;
+                }
+
+                if (parent != NULL) {
+                        inode_unref (parent);
+                }
+
+                parent = inode_ref (inode);
+                str = NULL;
+        }
+
+        inode_unref (parent);
+        GF_FREE (tmp);
+out:
+        return inode;
+}
+
+
 int
 inode_grep_for_gfid (inode_table_t *table, inode_t *parent, const char *name,
                      uuid_t gfid, ia_type_t *type)
@@ -795,24 +828,36 @@ __inode_link (inode_t *inode, inode_t *parent, const char *name,
                 if (uuid_is_null (iatt->ia_gfid))
                         return NULL;
 
-                uuid_copy (inode->gfid, iatt->ia_gfid);
-                inode->ia_type    = iatt->ia_type;
-
-                old_inode = __inode_find (table, inode->gfid);
+                old_inode = __inode_find (table, iatt->ia_gfid);
 
                 if (old_inode) {
                         link_inode = old_inode;
                 } else {
+                        uuid_copy (inode->gfid, iatt->ia_gfid);
+                        inode->ia_type    = iatt->ia_type;
                         __inode_hash (inode);
                 }
         }
 
+        if (name) {
+                if (!strcmp(name, ".") || !strcmp(name, ".."))
+                        return link_inode;
+        }
+
         /* use only link_inode beyond this point */
         if (parent) {
                 old_dentry = __dentry_grep (table, parent, name);
 
                 if (!old_dentry || old_dentry->inode != link_inode) {
                         dentry = __dentry_create (link_inode, parent, name);
+                        if (!dentry) {
+                                gf_log_callingfn (THIS->name, GF_LOG_ERROR,
+                                                  "dentry create failed on "
+                                                  "inode %s with parent %s",
+                                                  uuid_utoa (link_inode->gfid),
+                                                  uuid_utoa (parent->gfid));
+                                return NULL;
+                        }
                         if (old_inode && __is_dentry_cyclic (dentry)) {
                                 __dentry_unset (dentry);
                                 return NULL;
@@ -902,6 +947,54 @@ inode_forget (inode_t *inode, uint64_t nlookup)
         return 0;
 }
 
+/*
+ * Invalidate an inode. This is invoked when a translator decides that an inode's
+ * cache is no longer valid. Any translator interested in taking action in this
+ * situation can define the invalidate callback.
+ */
+int
+inode_invalidate(inode_t *inode)
+{
+	int ret = 0;
+	xlator_t *xl = NULL;
+	xlator_t *old_THIS = NULL;
+
+	if (!inode) {
+		gf_log_callingfn(THIS->name, GF_LOG_WARNING, "inode not found");
+		return -1;
+	}
+
+	/*
+	 * The master xlator is not in the graph but it can define an invalidate
+	 * handler.
+	 */
+	xl = inode->table->xl->ctx->master;
+	if (xl && xl->cbks->invalidate) {
+		old_THIS = THIS;
+		THIS = xl;
+		ret = xl->cbks->invalidate(xl, inode);
+		THIS = old_THIS;
+		if (ret)
+			return ret;
+	}
+
+	xl = inode->table->xl->graph->first;
+	while (xl) {
+		old_THIS = THIS;
+		THIS = xl;
+		if (xl->cbks->invalidate)
+			ret = xl->cbks->invalidate(xl, inode);
+		THIS = old_THIS;
+
+		if (ret)
+			break;
+
+		xl = xl->next;
+	}
+
+	return ret;
+}
+
 
 static void
 __inode_unlink (inode_t *inode, inode_t *parent, const char *name)
@@ -1038,8 +1131,9 @@ __inode_path (inode_t *inode, const char *name, char **bufp)
         int            len   = 0;
         char          *buf   = NULL;
 
-        if (!inode) {
-                gf_log_callingfn (THIS->name, GF_LOG_WARNING, "inode not found");
+        if (!inode || uuid_is_null (inode->gfid)) {
+                GF_ASSERT (0);
+                gf_log_callingfn (THIS->name, GF_LOG_WARNING, "invalid inode");
                 return -1;
         }
 
@@ -1096,7 +1190,7 @@ __inode_path (inode_t *inode, const char *name, char **bufp)
 
                 if (!__is_root_gfid (itrav->gfid)) {
                         snprintf (&buf[i-GFID_STR_PFX_LEN], GFID_STR_PFX_LEN,
-                                  "<gfid:%s>", uuid_utoa (itrav->gfid));
+                                  INODE_PATH_FMT, uuid_utoa (itrav->gfid));
                         buf[i-1] = '>';
                 }
 
@@ -1108,9 +1202,7 @@ __inode_path (inode_t *inode, const char *name, char **bufp)
 out:
         if (__is_root_gfid (inode->gfid) && !name) {
                 ret = 1;
-                if (buf) {
-                        GF_FREE (buf);
-                }
+                GF_FREE (buf);
                 buf = GF_CALLOC (ret + 1, sizeof (char), gf_common_mt_char);
                 if (buf) {
                         strcpy (buf, "/");
@@ -1206,8 +1298,8 @@ __inode_table_init_root (inode_table_t *table)
         iatt.ia_ino = 1;
         iatt.ia_type = IA_IFDIR;
 
-        table->root = root;
         __inode_link (root, NULL, NULL, &iatt);
+        table->root = root;
 }
 
 
@@ -1223,6 +1315,7 @@ inode_table_new (size_t lru_limit, xlator_t *xl)
                 return NULL;
 
         new->xl = xl;
+        new->ctxcount = xl->graph->xl_count + 1;
 
         new->lru_limit = lru_limit;
 
@@ -1254,7 +1347,9 @@ inode_table_new (size_t lru_limit, xlator_t *xl)
         if (!new->name_hash)
                 goto out;
 
-        new->fd_mem_pool = mem_pool_new (fd_t, 16384);
+        /* if number of fd open in one process is more than this,
+           we may hit perf issues */
+        new->fd_mem_pool = mem_pool_new (fd_t, 1024);
 
         if (!new->fd_mem_pool)
                 goto out;
@@ -1286,10 +1381,8 @@ inode_table_new (size_t lru_limit, xlator_t *xl)
 out:
         if (ret) {
                 if (new) {
-                        if (new->inode_hash)
-                                GF_FREE (new->inode_hash);
-                        if (new->name_hash)
-                                GF_FREE (new->name_hash);
+                        GF_FREE (new->inode_hash);
+                        GF_FREE (new->name_hash);
                         if (new->dentry_pool)
                                 mem_pool_destroy (new->dentry_pool);
                         if (new->inode_pool)
@@ -1355,8 +1448,7 @@ inode_from_path (inode_table_t *itable, const char *path)
         if (parent)
                 inode_unref (parent);
 
-        if (pathname)
-                GF_FREE (pathname);
+        GF_FREE (pathname);
 
 out:
         return inode;
@@ -1374,7 +1466,7 @@ __inode_ctx_set2 (inode_t *inode, xlator_t *xlator, uint64_t *value1_p,
         if (!inode || !xlator)
                 return -1;
 
-        for (index = 0; index < xlator->graph->xl_count; index++) {
+        for (index = 0; index < inode->table->ctxcount; index++) {
                 if (!inode->_ctx[index].xl_key) {
                         if (set_idx == -1)
                                 set_idx = index;
@@ -1401,6 +1493,18 @@ out:
         return ret;
 }
 
+int
+__inode_ctx_set0 (inode_t *inode, xlator_t *xlator, uint64_t *value1_p)
+{
+        return __inode_ctx_set2 (inode, xlator, value1_p, NULL);
+}
+
+int
+__inode_ctx_set1 (inode_t *inode, xlator_t *xlator, uint64_t *value2_p)
+{
+        return __inode_ctx_set2 (inode, xlator, NULL, value2_p);
+}
+
 
 int
 inode_ctx_set2 (inode_t *inode, xlator_t *xlator, uint64_t *value1_p,
@@ -1420,34 +1524,97 @@ inode_ctx_set2 (inode_t *inode, xlator_t *xlator, uint64_t *value1_p,
         return ret;
 }
 
+int
+inode_ctx_set1 (inode_t *inode, xlator_t *xlator, uint64_t *value2_p)
+{
+        int ret = 0;
+
+        if (!inode || !xlator)
+                return -1;
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_set1 (inode, xlator, value2_p);
+        }
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+int
+inode_ctx_set0 (inode_t *inode, xlator_t *xlator, uint64_t *value1_p)
+{
+        int ret = 0;
+
+        if (!inode || !xlator)
+                return -1;
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_set0 (inode, xlator, value1_p);
+        }
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+
 
 int
 __inode_ctx_get2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
                   uint64_t *value2)
 {
         int index = 0;
-        int ret = 0;
+        int ret = -1;
 
         if (!inode || !xlator)
-                return -1;
+                goto out;
 
-        for (index = 0; index < xlator->graph->xl_count; index++) {
+        for (index = 0; index < inode->table->ctxcount; index++) {
                 if (inode->_ctx[index].xl_key == xlator)
                         break;
         }
 
-        if (index == xlator->graph->xl_count) {
-                ret = -1;
+        if (index == inode->table->ctxcount)
                 goto out;
+
+        if (inode->_ctx[index].value1) {
+                if (value1)
+                        *value1 = inode->_ctx[index].value1;
+                ret = 0;
+        }
+        if (inode->_ctx[index].value2) {
+                if (value2)
+                        *value2 = inode->_ctx[index].value2;
+                ret = 0;
         }
+out:
+        return ret;
+}
 
-        if (value1)
-                *value1 = inode->_ctx[index].value1;
 
-        if (value2)
-                *value2 = inode->_ctx[index].value2;
 
-out:
+int
+__inode_ctx_get0 (inode_t *inode, xlator_t *xlator, uint64_t *value1)
+{
+        uint64_t tmp_value = 0;
+        int ret = 0;
+
+        ret =  __inode_ctx_get2 (inode, xlator, &tmp_value, NULL);
+        if (!ret)
+                *value1 = tmp_value;
+
+        return ret;
+}
+
+int
+__inode_ctx_get1 (inode_t *inode, xlator_t *xlator, uint64_t *value2)
+{
+        uint64_t tmp_value = 0;
+        int ret = 0;
+
+        ret =  __inode_ctx_get2 (inode, xlator, NULL, &tmp_value);
+        if (!ret)
+                *value2 = tmp_value;
+
         return ret;
 }
 
@@ -1470,6 +1637,40 @@ inode_ctx_get2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
         return ret;
 }
 
+int
+inode_ctx_get1 (inode_t *inode, xlator_t *xlator, uint64_t *value2)
+{
+        int ret = 0;
+
+        if (!inode || !xlator)
+                return -1;
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get1 (inode, xlator, value2);
+        }
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+
+int
+inode_ctx_get0 (inode_t *inode, xlator_t *xlator, uint64_t *value1)
+{
+        int ret = 0;
+
+        if (!inode || !xlator)
+                return -1;
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get0 (inode, xlator, value1);
+        }
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+
 
 int
 inode_ctx_del2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
@@ -1483,19 +1684,20 @@ inode_ctx_del2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
 
         LOCK (&inode->lock);
         {
-                for (index = 0; index < xlator->graph->xl_count; index++) {
+                for (index = 0; index < inode->table->ctxcount;
+                     index++) {
                         if (inode->_ctx[index].xl_key == xlator)
                                 break;
                 }
 
-                if (index == xlator->graph->xl_count) {
+                if (index == inode->table->ctxcount) {
                         ret = -1;
                         goto unlock;
                 }
 
-                if (value1)
+                if (inode->_ctx[index].value1 && value1)
                         *value1 = inode->_ctx[index].value1;
-                if (value2)
+                if (inode->_ctx[index].value2 && value2)
                         *value2 = inode->_ctx[index].value2;
 
                 inode->_ctx[index].key    = 0;
@@ -1508,6 +1710,97 @@ unlock:
         return ret;
 }
 
+/* function behavior:
+ - if value1 is set, value1 in ctx is reset to 0 with current value passed
+ back in value1 address.
+ - if value2 is set, value2 in ctx is reset to 0 with current value passed
+ back in value2 address.
+ - if both are set, both fields are reset.
+*/
+static int
+__inode_ctx_reset2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
+                    uint64_t *value2)
+{
+        int index = 0;
+        int ret = 0;
+
+        if (!inode || !xlator)
+                return -1;
+
+        LOCK (&inode->lock);
+        {
+                for (index = 0; index < inode->table->ctxcount;
+                     index++) {
+                        if (inode->_ctx[index].xl_key == xlator)
+                                break;
+                }
+
+                if (index == inode->table->ctxcount) {
+                        ret = -1;
+                        goto unlock;
+                }
+
+                if (inode->_ctx[index].value1 && value1) {
+                        *value1 = inode->_ctx[index].value1;
+                        inode->_ctx[index].value1 = 0;
+                }
+                if (inode->_ctx[index].value2 && value2) {
+                        *value2 = inode->_ctx[index].value2;
+                        inode->_ctx[index].value2 = 0;
+                }
+        }
+unlock:
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+
+int
+inode_ctx_reset2 (inode_t *inode, xlator_t *xlator, uint64_t *value1_p,
+                  uint64_t *value2_p)
+{
+        uint64_t tmp_value1 = 0;
+        uint64_t tmp_value2 = 0;
+        int ret = 0;
+
+        ret =  __inode_ctx_reset2 (inode, xlator, &tmp_value1, &tmp_value2);
+        if (!ret) {
+                if (value1_p)
+                        *value1_p = tmp_value1;
+                if (value2_p)
+                        *value2_p = tmp_value2;
+        }
+        return ret;
+}
+
+int
+inode_ctx_reset1 (inode_t *inode, xlator_t *xlator, uint64_t *value2_p)
+{
+        uint64_t tmp_value2 = 0;
+        int ret = 0;
+
+        ret = __inode_ctx_reset2 (inode, xlator, NULL, &tmp_value2);
+
+        if (!ret && value2_p)
+                *value2_p = tmp_value2;
+
+        return ret;
+
+}
+int
+inode_ctx_reset0 (inode_t *inode, xlator_t *xlator, uint64_t *value1_p)
+{
+        uint64_t tmp_value1 = 0;
+        int ret = 0;
+
+        ret = __inode_ctx_reset2 (inode, xlator, &tmp_value1, NULL);
+
+        if (!ret && value1_p)
+                *value1_p = tmp_value1;
+
+        return ret;
+}
+
 
 void
 inode_dump (inode_t *inode, char *prefix)
@@ -1517,10 +1810,6 @@ inode_dump (inode_t *inode, char *prefix)
         int                i         = 0;
         fd_t              *fd        = NULL;
         struct _inode_ctx *inode_ctx = NULL;
-        struct  fd_wrapper {
-                fd_t *fd;
-                struct list_head next;
-        } *fd_wrapper, *tmp;
         struct list_head fd_list;
 
         if (!inode)
@@ -1536,43 +1825,36 @@ inode_dump (inode_t *inode, char *prefix)
         {
                 gf_proc_dump_write("gfid", "%s", uuid_utoa (inode->gfid));
                 gf_proc_dump_write("nlookup", "%ld", inode->nlookup);
+                gf_proc_dump_write("fd-count", "%u", inode->fd_count);
                 gf_proc_dump_write("ref", "%u", inode->ref);
                 gf_proc_dump_write("ia_type", "%d", inode->ia_type);
                 if (inode->_ctx) {
-                        inode_ctx = GF_CALLOC (inode->table->xl->graph->xl_count,
+                        inode_ctx = GF_CALLOC (inode->table->ctxcount,
                                                sizeof (*inode_ctx),
                                                gf_common_mt_inode_ctx);
                         if (inode_ctx == NULL) {
                                 goto unlock;
                         }
 
-                        for (i = 0; i < inode->table->xl->graph->xl_count; i++) {
+                        for (i = 0; i < inode->table->ctxcount;
+                             i++) {
                                 inode_ctx[i] = inode->_ctx[i];
                         }
                 }
 
-                if (list_empty (&inode->fd_list)) {
-                        goto unlock;
-                }
-
-                list_for_each_entry (fd, &inode->fd_list, inode_list) {
-                        fd_wrapper = GF_CALLOC (1, sizeof (*fd_wrapper),
-                                                gf_common_mt_char);
-                        if (fd_wrapper == NULL) {
-                                goto unlock;
-                        }
+		if (dump_options.xl_options.dump_fdctx != _gf_true)
+			goto unlock;
 
-                        INIT_LIST_HEAD (&fd_wrapper->next);
-                        list_add_tail (&fd_wrapper->next, &fd_list);
 
-                        fd_wrapper->fd = __fd_ref (fd);
+                list_for_each_entry (fd, &inode->fd_list, inode_list) {
+                        fd_ctx_dump (fd, prefix);
                 }
         }
 unlock:
         UNLOCK(&inode->lock);
 
         if (inode_ctx && (dump_options.xl_options.dump_inodectx == _gf_true)) {
-                for (i = 0; i < inode->table->xl->graph->xl_count; i++) {
+                for (i = 0; i < inode->table->ctxcount; i++) {
                         if (inode_ctx[i].xl_key) {
                                 xl = (xlator_t *)(long)inode_ctx[i].xl_key;
                                 if (xl->dumpops && xl->dumpops->inodectx)
@@ -1581,21 +1863,7 @@ unlock:
                 }
         }
 
-        if (!list_empty (&fd_list)
-            && (dump_options.xl_options.dump_fdctx == _gf_true)) {
-                list_for_each_entry_safe (fd_wrapper, tmp, &fd_list,
-                                          next) {
-                        list_del (&fd_wrapper->next);
-                        fd_ctx_dump (fd_wrapper->fd, prefix);
-
-                        fd_unref (fd_wrapper->fd);
-                        GF_FREE (fd_wrapper);
-                }
-        }
-
-        if (inode_ctx != NULL) {
-                GF_FREE (inode_ctx);
-        }
+        GF_FREE (inode_ctx);
 
         return;
 }
@@ -1650,7 +1918,7 @@ inode_dump_to_dict (inode_t *inode, char *prefix, dict_t *dict)
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "%s.gfid", prefix);
-        ret = dict_set_str (dict, key, gf_strdup (uuid_utoa (inode->gfid)));
+        ret = dict_set_dynstr (dict, key, gf_strdup (uuid_utoa (inode->gfid)));
         if (ret)
                 goto out;
 
diff --git a/libglusterfs/src/inode.h b/libglusterfs/src/inode.h
index 7dda0401d..a88976265 100644
--- a/libglusterfs/src/inode.h
+++ b/libglusterfs/src/inode.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _INODE_H
@@ -28,7 +19,8 @@
 #include <stdint.h>
 #include <sys/types.h>
 
-#define DEFAULT_INODE_MEMPOOL_ENTRIES   16384
+#define DEFAULT_INODE_MEMPOOL_ENTRIES   32 * 1024
+#define INODE_PATH_FMT "<gfid:%s>"
 struct _inode_table;
 typedef struct _inode_table inode_table_t;
 
@@ -64,6 +56,7 @@ struct _inode_table {
         struct mem_pool   *inode_pool;  /* memory pool for inodes */
         struct mem_pool   *dentry_pool; /* memory pool for dentrys */
         struct mem_pool   *fd_mem_pool; /* memory pool for fd_t */
+        int                ctxcount;    /* number of slots in inode->ctx */
 };
 
 
@@ -80,10 +73,12 @@ struct _inode_ctx {
                 uint64_t    key;
                 xlator_t   *xl_key;
         };
+        /* if value1 is 0, then field is not set.. */
         union {
                 uint64_t    value1;
                 void       *ptr1;
         };
+        /* if value2 is 0, then field is not set.. */
         union {
                 uint64_t    value2;
                 void       *ptr2;
@@ -95,6 +90,7 @@ struct _inode {
         uuid_t               gfid;
         gf_lock_t            lock;
         uint64_t             nlookup;
+        uint32_t             fd_count;      /* Open fd count */
         uint32_t             ref;           /* reference count on this inode */
         ia_type_t            ia_type;       /* what kind of file */
         struct list_head     fd_list;       /* list of open files on this inode */
@@ -139,6 +135,9 @@ int
 inode_forget (inode_t *inode, uint64_t nlookup);
 
 int
+inode_invalidate(inode_t *inode);
+
+int
 inode_rename (inode_table_t *table, inode_t *olddir, const char *oldname,
 	      inode_t *newdir, const char *newname,
 	      inode_t *inode, struct iatt *stbuf);
@@ -162,6 +161,11 @@ __inode_path (inode_t *inode, const char *name, char **bufp);
 inode_t *
 inode_from_path (inode_table_t *table, const char *path);
 
+inode_t *
+inode_resolve (inode_table_t *table, char *path);
+
+/* deal with inode ctx's both values */
+
 int
 inode_ctx_set2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
                 uint64_t *value2);
@@ -180,26 +184,66 @@ int
 inode_ctx_del2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
                 uint64_t *value2);
 
-#define __inode_ctx_set(i,x,v_p) __inode_ctx_set2(i,x,v_p,0)
-#define inode_ctx_set(i,x,v_p) inode_ctx_set2(i,x,v_p,0)
+int
+inode_ctx_reset2 (inode_t *inode, xlator_t *xlator, uint64_t *value1,
+                  uint64_t *value2);
+
+/* deal with inode ctx's 1st value */
+
+int
+inode_ctx_set0 (inode_t *inode, xlator_t *xlator, uint64_t *value1);
+
+int
+__inode_ctx_set0 (inode_t *inode, xlator_t *xlator, uint64_t *value1);
+
+int
+inode_ctx_get0 (inode_t *inode, xlator_t *xlator, uint64_t *value1);
+int
+__inode_ctx_get0 (inode_t *inode, xlator_t *xlator, uint64_t *value1);
+
+int
+inode_ctx_reset0 (inode_t *inode, xlator_t *xlator, uint64_t *value1);
+
+/* deal with inode ctx's 2st value */
+
+int
+inode_ctx_set1 (inode_t *inode, xlator_t *xlator, uint64_t *value2);
+
+int
+__inode_ctx_set1 (inode_t *inode, xlator_t *xlator, uint64_t *value2);
+
+int
+inode_ctx_get1 (inode_t *inode, xlator_t *xlator, uint64_t *value2);
+int
+__inode_ctx_get1 (inode_t *inode, xlator_t *xlator, uint64_t *value2);
+
+int
+inode_ctx_reset1 (inode_t *inode, xlator_t *xlator, uint64_t *value2);
+
 
 static inline int
 __inode_ctx_put(inode_t *inode, xlator_t *this, uint64_t v)
 {
-        return __inode_ctx_set2 (inode, this, &v, 0);
+        return __inode_ctx_set0 (inode, this, &v);
 }
 
 static inline int
 inode_ctx_put(inode_t *inode, xlator_t *this, uint64_t v)
 {
-        return inode_ctx_set2(inode, this, &v, 0);
+        return inode_ctx_set0 (inode, this, &v);
 }
 
-#define __inode_ctx_get(i,x,v) __inode_ctx_get2(i,x,v,0)
-#define inode_ctx_get(i,x,v) inode_ctx_get2(i,x,v,0)
+#define __inode_ctx_set(i,x,v_p) __inode_ctx_set0(i,x,v_p)
 
-#define inode_ctx_del(i,x,v) inode_ctx_del2(i,x,v,0)
+#define inode_ctx_set(i,x,v_p) inode_ctx_set0(i,x,v_p)
 
+#define inode_ctx_reset(i,x,v) inode_ctx_reset0(i,x,v)
+
+#define __inode_ctx_get(i,x,v) __inode_ctx_get0(i,x,v)
+
+#define inode_ctx_get(i,x,v) inode_ctx_get0(i,x,v)
+
+#define inode_ctx_del(i,x,v) inode_ctx_del2(i,x,v,0)
 
 gf_boolean_t
 __is_root_gfid (uuid_t gfid);
diff --git a/libglusterfs/src/iobuf.c b/libglusterfs/src/iobuf.c
index 93bc7732e..a89e96267 100644
--- a/libglusterfs/src/iobuf.c
+++ b/libglusterfs/src/iobuf.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -317,6 +308,31 @@ out:
         return;
 }
 
+static void
+iobuf_create_stdalloc_arena (struct iobuf_pool *iobuf_pool)
+{
+        struct iobuf_arena *iobuf_arena = NULL;
+
+        /* No locking required here as its called only once during init */
+        iobuf_arena = GF_CALLOC (sizeof (*iobuf_arena), 1,
+                                 gf_common_mt_iobuf_arena);
+        if (!iobuf_arena)
+                goto err;
+
+        INIT_LIST_HEAD (&iobuf_arena->list);
+        INIT_LIST_HEAD (&iobuf_arena->active.list);
+        INIT_LIST_HEAD (&iobuf_arena->passive.list);
+
+        iobuf_arena->iobuf_pool = iobuf_pool;
+
+        iobuf_arena->page_size = 0x7fffffff;
+
+        list_add_tail (&iobuf_arena->list,
+                       &iobuf_pool->arenas[IOBUF_ARENA_MAX_INDEX]);
+
+err:
+        return;
+}
 
 struct iobuf_pool *
 iobuf_pool_new (void)
@@ -333,7 +349,7 @@ iobuf_pool_new (void)
                 goto out;
 
         pthread_mutex_init (&iobuf_pool->mutex, NULL);
-        for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
+        for (i = 0; i <= IOBUF_ARENA_MAX_INDEX; i++) {
                 INIT_LIST_HEAD (&iobuf_pool->arenas[i]);
                 INIT_LIST_HEAD (&iobuf_pool->filled[i]);
                 INIT_LIST_HEAD (&iobuf_pool->purge[i]);
@@ -351,6 +367,9 @@ iobuf_pool_new (void)
                 arena_size += page_size * num_pages;
         }
 
+        /* Need an arena to handle all the bigger iobuf requests */
+        iobuf_create_stdalloc_arena (iobuf_pool);
+
         iobuf_pool->arena_size = arena_size;
 out:
 
@@ -509,6 +528,50 @@ out:
 }
 
 struct iobuf *
+iobuf_get_from_stdalloc (struct iobuf_pool *iobuf_pool, size_t page_size)
+{
+        struct iobuf       *iobuf       = NULL;
+        struct iobuf_arena *iobuf_arena = NULL;
+        struct iobuf_arena *trav        = NULL;
+        int                 ret         = -1;
+
+        /* The first arena in the 'MAX-INDEX' will always be used for misc */
+        list_for_each_entry (trav, &iobuf_pool->arenas[IOBUF_ARENA_MAX_INDEX],
+                             list) {
+                iobuf_arena = trav;
+                break;
+        }
+
+        iobuf = GF_CALLOC (1, sizeof (*iobuf), gf_common_mt_iobuf);
+        if (!iobuf)
+                goto out;
+
+        /* 4096 is the alignment */
+        iobuf->free_ptr = GF_CALLOC (1, ((page_size + GF_IOBUF_ALIGN_SIZE) - 1),
+                                     gf_common_mt_char);
+        if (!iobuf->free_ptr)
+                goto out;
+
+        iobuf->ptr = GF_ALIGN_BUF (iobuf->free_ptr, GF_IOBUF_ALIGN_SIZE);
+        iobuf->iobuf_arena = iobuf_arena;
+        LOCK_INIT (&iobuf->lock);
+
+        /* Hold a ref because you are allocating and using it */
+        iobuf->ref = 1;
+
+        ret = 0;
+out:
+        if (ret && iobuf) {
+                GF_FREE (iobuf->free_ptr);
+                GF_FREE (iobuf);
+                iobuf = NULL;
+        }
+
+        return iobuf;
+}
+
+
+struct iobuf *
 iobuf_get2 (struct iobuf_pool *iobuf_pool, size_t page_size)
 {
         struct iobuf       *iobuf        = NULL;
@@ -521,10 +584,17 @@ iobuf_get2 (struct iobuf_pool *iobuf_pool, size_t page_size)
 
         rounded_size = gf_iobuf_get_pagesize (page_size);
         if (rounded_size == -1) {
-                gf_log ("iobuf", GF_LOG_ERROR, "page_size (%zu) of "
-                        "iobufs in arena being requested is greater than max "
-                        "available", page_size);
-                return NULL;
+                /* make sure to provide the requested buffer with standard
+                   memory allocations */
+                iobuf = iobuf_get_from_stdalloc (iobuf_pool, page_size);
+
+                gf_log ("iobuf", GF_LOG_DEBUG, "request for iobuf of size %zu "
+                        "is serviced using standard calloc() (%p) as it "
+                        "exceeds the maximum available buffer size",
+                        page_size, iobuf);
+
+                iobuf_pool->request_misses++;
+                return iobuf;
         }
 
         pthread_mutex_lock (&iobuf_pool->mutex);
@@ -593,9 +663,13 @@ __iobuf_put (struct iobuf *iobuf, struct iobuf_arena *iobuf_arena)
 
         index = gf_iobuf_get_arena_index (iobuf_arena->page_size);
         if (index == -1) {
-                gf_log ("iobuf", GF_LOG_ERROR, "page_size (%zu) of "
-                        "iobufs in arena being added is greater than max "
-                        "available", iobuf_arena->page_size);
+                gf_log ("iobuf", GF_LOG_DEBUG, "freeing the iobuf (%p) "
+                        "allocated with standard calloc()", iobuf);
+
+                /* free up properly without bothering about lists and all */
+                LOCK_DESTROY (&iobuf->lock);
+                GF_FREE (iobuf->free_ptr);
+                GF_FREE (iobuf);
                 return;
         }
 
@@ -767,6 +841,29 @@ out:
 }
 
 
+void
+iobref_clear (struct iobref *iobref)
+{
+        int i = 0;
+
+        GF_VALIDATE_OR_GOTO ("iobuf", iobref, out);
+
+        for (; i < GF_IOBREF_IOBUF_COUNT; i++) {
+                if (iobref->iobrefs[i] != NULL) {
+                        iobuf_unref (iobref->iobrefs[i]);
+                } else {
+                        /** iobuf's are attched serially */
+                        break;
+                }
+        }
+
+        iobref_unref (iobref);
+
+ out:
+        return;
+}
+
+
 int
 __iobref_add (struct iobref *iobref, struct iobuf *iobuf)
 {
@@ -967,6 +1064,8 @@ iobuf_stats_dump (struct iobuf_pool *iobuf_pool)
                            iobuf_pool->arena_size);
         gf_proc_dump_write("iobuf_pool.arena_cnt", "%d",
                            iobuf_pool->arena_cnt);
+        gf_proc_dump_write("iobuf_pool.request_misses", "%"PRId64,
+                           iobuf_pool->request_misses);
 
         for (j = 0; j < IOBUF_ARENA_MAX_INDEX; j++) {
                 list_for_each_entry (trav, &iobuf_pool->arenas[j], list) {
diff --git a/libglusterfs/src/iobuf.h b/libglusterfs/src/iobuf.h
index d0f868286..5595309e1 100644
--- a/libglusterfs/src/iobuf.h
+++ b/libglusterfs/src/iobuf.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _IOBUF_H_
@@ -40,6 +31,11 @@
 #define MAP_ANONYMOUS MAP_ANON
 #endif
 
+#define GF_ALIGN_BUF(ptr,bound) ((void *)((unsigned long)(ptr + bound - 1) & \
+                                          (unsigned long)(~(bound - 1))))
+
+#define GF_IOBUF_ALIGN_SIZE 512
+
 /* one allocatable unit for the consumers of the IOBUF API */
 /* each unit hosts @page_size bytes of memory */
 struct iobuf;
@@ -71,6 +67,9 @@ struct iobuf {
         int                  ref;  /* 0 == passive, >0 == active */
 
         void                *ptr;  /* usable memory region by the consumer */
+
+        void                *free_ptr; /* in case of stdalloc, this is the
+                                          one to be freed */
 };
 
 
@@ -113,21 +112,17 @@ struct iobuf_pool {
 
         int                 arena_cnt;
         struct list_head    arenas[GF_VARIABLE_IOBUF_COUNT];
-        /* array of arenas. Each element of
-           the array is a list of arenas
-           holding iobufs of particular
-           page_size
-        */
+        /* array of arenas. Each element of the array is a list of arenas
+           holding iobufs of particular page_size */
+
         struct list_head    filled[GF_VARIABLE_IOBUF_COUNT];
-        /*
-          array of arenas without free iobufs
-        */
+        /* array of arenas without free iobufs */
 
         struct list_head    purge[GF_VARIABLE_IOBUF_COUNT];
-        /*
-          array of of arenas which can be
-          purged
-        */
+        /* array of of arenas which can be purged */
+
+        uint64_t            request_misses; /* mostly the requests for higher
+                                               value of iobufs */
 };
 
 
@@ -155,7 +150,7 @@ struct iobref *iobref_ref (struct iobref *iobref);
 void iobref_unref (struct iobref *iobref);
 int iobref_add (struct iobref *iobref, struct iobuf *iobuf);
 int iobref_merge (struct iobref *to, struct iobref *from);
-
+void iobref_clear (struct iobref *iobref);
 
 size_t iobuf_size (struct iobuf *iobuf);
 size_t iobref_size (struct iobref *iobref);
diff --git a/libglusterfs/src/latency.c b/libglusterfs/src/latency.c
index dfce235c0..b22f72950 100644
--- a/libglusterfs/src/latency.c
+++ b/libglusterfs/src/latency.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -35,87 +26,87 @@ gf_set_fop_from_fn_pointer (call_frame_t *frame, struct xlator_fops *fops, void
 {
         glusterfs_fop_t fop = -1;
 
-        if (fops->stat == fn)
+        if (fops->stat == *(fop_stat_t *)&fn)
                 fop = GF_FOP_STAT;
-        else if (fops->readlink == fn)
+        else if (fops->readlink == *(fop_readlink_t *)&fn)
                 fop = GF_FOP_READLINK;
-        else if (fops->mknod == fn)
+        else if (fops->mknod == *(fop_mknod_t *)&fn)
                 fop = GF_FOP_MKNOD;
-        else if (fops->mkdir == fn)
+        else if (fops->mkdir == *(fop_mkdir_t *)&fn)
                 fop = GF_FOP_MKDIR;
-        else if (fops->unlink == fn)
+        else if (fops->unlink == *(fop_unlink_t *)&fn)
                 fop = GF_FOP_UNLINK;
-        else if (fops->rmdir == fn)
+        else if (fops->rmdir == *(fop_rmdir_t *)&fn)
                 fop = GF_FOP_RMDIR;
-        else if (fops->symlink == fn)
+        else if (fops->symlink == *(fop_symlink_t *)&fn)
                 fop = GF_FOP_SYMLINK;
-        else if (fops->rename == fn)
+        else if (fops->rename == *(fop_rename_t *)&fn)
                 fop = GF_FOP_RENAME;
-        else if (fops->link == fn)
+        else if (fops->link == *(fop_link_t *)&fn)
                 fop = GF_FOP_LINK;
-        else if (fops->truncate == fn)
+        else if (fops->truncate == *(fop_truncate_t *)&fn)
                 fop = GF_FOP_TRUNCATE;
-        else if (fops->open == fn)
+        else if (fops->open == *(fop_open_t *)&fn)
                 fop = GF_FOP_OPEN;
-        else if (fops->readv == fn)
+        else if (fops->readv == *(fop_readv_t *)&fn)
                 fop = GF_FOP_READ;
-        else if (fops->writev == fn)
+        else if (fops->writev == *(fop_writev_t *)&fn)
                 fop = GF_FOP_WRITE;
-        else if (fops->statfs == fn)
+        else if (fops->statfs == *(fop_statfs_t *)&fn)
                 fop = GF_FOP_STATFS;
-        else if (fops->flush == fn)
+        else if (fops->flush == *(fop_flush_t *)&fn)
                 fop = GF_FOP_FLUSH;
-        else if (fops->fsync == fn)
+        else if (fops->fsync == *(fop_fsync_t *)&fn)
                 fop = GF_FOP_FSYNC;
-        else if (fops->setxattr == fn)
+        else if (fops->setxattr == *(fop_setxattr_t *)&fn)
                 fop = GF_FOP_SETXATTR;
-        else if (fops->getxattr == fn)
+        else if (fops->getxattr == *(fop_getxattr_t *)&fn)
                 fop = GF_FOP_GETXATTR;
-        else if (fops->removexattr == fn)
+        else if (fops->removexattr == *(fop_removexattr_t *)&fn)
                 fop = GF_FOP_REMOVEXATTR;
-        else if (fops->opendir == fn)
+        else if (fops->opendir == *(fop_opendir_t *)&fn)
                 fop = GF_FOP_OPENDIR;
-        else if (fops->fsyncdir == fn)
+        else if (fops->fsyncdir == *(fop_fsyncdir_t *)&fn)
                 fop = GF_FOP_FSYNCDIR;
-        else if (fops->access == fn)
+        else if (fops->access == *(fop_access_t *)&fn)
                 fop = GF_FOP_ACCESS;
-        else if (fops->create == fn)
+        else if (fops->create == *(fop_create_t *)&fn)
                 fop = GF_FOP_CREATE;
-        else if (fops->ftruncate == fn)
+        else if (fops->ftruncate == *(fop_ftruncate_t *)&fn)
                 fop = GF_FOP_FTRUNCATE;
-        else if (fops->fstat == fn)
+        else if (fops->fstat == *(fop_fstat_t *)&fn)
                 fop = GF_FOP_FSTAT;
-        else if (fops->lk == fn)
+        else if (fops->lk == *(fop_lk_t *)&fn)
                 fop = GF_FOP_LK;
-        else if (fops->lookup == fn)
+        else if (fops->lookup == *(fop_lookup_t *)&fn)
                 fop = GF_FOP_LOOKUP;
-        else if (fops->readdir == fn)
+        else if (fops->readdir == *(fop_readdir_t *)&fn)
                 fop = GF_FOP_READDIR;
-        else if (fops->inodelk == fn)
+        else if (fops->inodelk == *(fop_inodelk_t *)&fn)
                 fop = GF_FOP_INODELK;
-        else if (fops->finodelk == fn)
+        else if (fops->finodelk == *(fop_finodelk_t *)&fn)
                 fop = GF_FOP_FINODELK;
-        else if (fops->entrylk == fn)
+        else if (fops->entrylk == *(fop_entrylk_t *)&fn)
                 fop = GF_FOP_ENTRYLK;
-        else if (fops->fentrylk == fn)
+        else if (fops->fentrylk == *(fop_fentrylk_t *)&fn)
                 fop = GF_FOP_FENTRYLK;
-        else if (fops->xattrop == fn)
+        else if (fops->xattrop == *(fop_xattrop_t *)&fn)
                 fop = GF_FOP_XATTROP;
-        else if (fops->fxattrop == fn)
+        else if (fops->fxattrop == *(fop_fxattrop_t *)&fn)
                 fop = GF_FOP_FXATTROP;
-        else if (fops->fgetxattr == fn)
+        else if (fops->fgetxattr == *(fop_fgetxattr_t *)&fn)
                 fop = GF_FOP_FGETXATTR;
-        else if (fops->fsetxattr == fn)
+        else if (fops->fsetxattr == *(fop_fsetxattr_t *)&fn)
                 fop = GF_FOP_FSETXATTR;
-        else if (fops->rchecksum == fn)
+        else if (fops->rchecksum == *(fop_rchecksum_t *)&fn)
                 fop = GF_FOP_RCHECKSUM;
-        else if (fops->setattr == fn)
+        else if (fops->setattr == *(fop_setattr_t *)&fn)
                 fop = GF_FOP_SETATTR;
-        else if (fops->fsetattr == fn)
+        else if (fops->fsetattr == *(fop_fsetattr_t *)&fn)
                 fop = GF_FOP_FSETATTR;
-        else if (fops->readdirp == fn)
+        else if (fops->readdirp == *(fop_readdirp_t *)&fn)
                 fop = GF_FOP_READDIRP;
-        else if (fops->getspec == fn)
+        else if (fops->getspec == *(fop_getspec_t *)&fn)
                 fop = GF_FOP_GETSPEC;
         else
                 fop = -1;
@@ -145,6 +136,22 @@ gf_update_latency (call_frame_t *frame)
         lat->mean = lat->mean + (elapsed - lat->mean) / lat->count;
 }
 
+void
+gf_latency_begin (call_frame_t *frame, void *fn)
+{
+        gf_set_fop_from_fn_pointer (frame, frame->this->fops, fn);
+
+        gettimeofday (&frame->begin, NULL);
+}
+
+
+void
+gf_latency_end (call_frame_t *frame)
+{
+        gettimeofday (&frame->end, NULL);
+
+        gf_update_latency (frame);
+}
 
 void
 gf_proc_dump_latency_info (xlator_t *xl)
@@ -157,23 +164,22 @@ gf_proc_dump_latency_info (xlator_t *xl)
         gf_proc_dump_add_section (key_prefix);
 
         for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                gf_proc_dump_build_key (key, key_prefix, gf_fop_list[i]);
+                gf_proc_dump_build_key (key, key_prefix,
+                                        (char *)gf_fop_list[i]);
 
                 gf_proc_dump_write (key, "%.03f,%"PRId64",%.03f",
                                     xl->latencies[i].mean,
                                     xl->latencies[i].count,
                                     xl->latencies[i].total);
         }
+
+        memset (xl->latencies, 0, sizeof (xl->latencies));
 }
 
 
 void
-gf_latency_toggle (int signum)
+gf_latency_toggle (int signum, glusterfs_ctx_t *ctx)
 {
-        glusterfs_ctx_t *ctx = NULL;
-
-        ctx = glusterfs_ctx_get ();
-
         if (ctx) {
                 ctx->measure_latency = !ctx->measure_latency;
                 gf_log ("[core]", GF_LOG_INFO,
diff --git a/libglusterfs/src/latency.h b/libglusterfs/src/latency.h
index 70499d4ba..81acbf484 100644
--- a/libglusterfs/src/latency.h
+++ b/libglusterfs/src/latency.h
@@ -1,25 +1,17 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __LATENCY_H__
 #define __LATENCY_H__
 
+#include "glusterfs.h"
 
 typedef struct fop_latency {
         uint64_t min;           /* min time for the call (microseconds) */
@@ -31,6 +23,6 @@ typedef struct fop_latency {
 } fop_latency_t;
 
 void
-gf_latency_toggle (int signum);
+gf_latency_toggle (int signum, glusterfs_ctx_t *ctx);
 
 #endif /* __LATENCY_H__ */
diff --git a/libglusterfs/src/list.h b/libglusterfs/src/list.h
index db4e1ab30..392c22ceb 100644
--- a/libglusterfs/src/list.h
+++ b/libglusterfs/src/list.h
@@ -1,26 +1,16 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _LLIST_H
 #define _LLIST_H
 
-
 struct list_head {
 	struct list_head *next;
 	struct list_head *prev;
@@ -54,6 +44,31 @@ list_add_tail (struct list_head *new, struct list_head *head)
 }
 
 
+/* This function will insert the element to the list in a order.
+   Order will be based on the compare function provided as a input.
+   If element to be inserted in ascending order compare should return:
+    0: if both the arguments are equal
+   >0: if first argument is greater than second argument
+   <0: if first argument is less than second argument */
+static inline void
+list_add_order (struct list_head *new, struct list_head *head,
+                int (*compare)(struct list_head *, struct list_head *))
+{
+        struct list_head *pos = head->prev;
+
+        while ( pos != head ) {
+                if (compare(new, pos) >= 0)
+                        break;
+
+                /* Iterate the list in the reverse order. This will have
+                   better efficiency if the elements are inserted in the
+                   ascending order */
+                pos = pos->prev;
+        }
+
+        list_add (new, pos);
+}
+
 static inline void
 list_del (struct list_head *old)
 {
@@ -184,4 +199,16 @@ list_append_init (struct list_head *list, struct list_head *head)
 	     &pos->member != (head); 					\
 	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
 
+#define list_for_each_entry_reverse(pos, head, member)                  \
+	for (pos = list_entry((head)->prev, typeof(*pos), member);      \
+	     &pos->member != (head);                                    \
+	     pos = list_entry(pos->member.prev, typeof(*pos), member))
+
+
+#define list_for_each_entry_safe_reverse(pos, n, head, member)          \
+	for (pos = list_entry((head)->prev, typeof(*pos), member),      \
+	        n = list_entry(pos->member.prev, typeof(*pos), member); \
+	     &pos->member != (head);                                    \
+	     pos = n, n = list_entry(n->member.prev, typeof(*n), member))
+
 #endif /* _LLIST_H */
diff --git a/libglusterfs/src/lkowner.h b/libglusterfs/src/lkowner.h
index 5fee17b3d..969d13e50 100644
--- a/libglusterfs/src/lkowner.h
+++ b/libglusterfs/src/lkowner.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2012 Red Hat <http://www.redhat.com/>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _LK_OWNER_H
@@ -49,7 +40,7 @@ lkowner_unparse (gf_lkowner_t *lkowner, char *buf, int buf_len)
                         buf[j] = '-';
                         j++;
                 }
-                sprintf (&buf[j], "%02x", lkowner->data[i]);
+                sprintf (&buf[j], "%02hhx", lkowner->data[i]);
                 j += 2;
                 if (j == buf_len)
                         break;
diff --git a/libglusterfs/src/locking.h b/libglusterfs/src/locking.h
index 44ea12eee..79c6992af 100644
--- a/libglusterfs/src/locking.h
+++ b/libglusterfs/src/locking.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _LOCKING_H
diff --git a/libglusterfs/src/logging.c b/libglusterfs/src/logging.c
index 5485260bc..5deb90cda 100644
--- a/libglusterfs/src/logging.c
+++ b/libglusterfs/src/logging.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -31,9 +22,22 @@
 #include <string.h>
 #include <stdlib.h>
 
+#ifdef GF_USE_SYSLOG
+#include <libintl.h>
+#include <syslog.h>
+#include <sys/stat.h>
+#include "gf-error-codes.h"
+
+#define GF_JSON_MSG_LENGTH      8192
+#define GF_SYSLOG_CEE_FORMAT    \
+        "@cee: {\"msg\": \"%s\", \"gf_code\": \"%u\", \"gf_message\": \"%s\"}"
+#define GF_LOG_CONTROL_FILE     "/etc/glusterfs/logger.conf"
+#endif /* GF_USE_SYSLOG */
+
 #include "xlator.h"
 #include "logging.h"
 #include "defaults.h"
+#include "glusterfs.h"
 
 #ifdef GF_LINUX_HOST_OS
 #include <syslog.h>
@@ -43,50 +47,44 @@
 #include <execinfo.h>
 #endif
 
+/* Ideally this should get moved to logging.h */
+struct _msg_queue {
+        struct list_head msgs;
+};
 
-static pthread_mutex_t  logfile_mutex;
-static char            *filename = NULL;
-static uint8_t          logrotate = 0;
-static FILE            *logfile = NULL;
-static gf_loglevel_t    loglevel = GF_LOG_INFO;
-static int              gf_log_syslog = 1;
-static gf_loglevel_t    sys_log_level = GF_LOG_CRITICAL;
-
-char                    gf_log_xl_log_set;
-gf_loglevel_t           gf_log_loglevel = GF_LOG_INFO; /* extern'd */
-FILE                   *gf_log_logfile;
-
-static char            *cmd_log_filename = NULL;
-static FILE            *cmdlogfile = NULL;
+struct _log_msg {
+        const char *msg;
+        struct list_head queue;
+};
 
 void
 gf_log_logrotate (int signum)
 {
-        logrotate = 1;
+        THIS->ctx->log.logrotate = 1;
 }
 
 void
 gf_log_enable_syslog (void)
 {
-        gf_log_syslog = 1;
+        THIS->ctx->log.gf_log_syslog = 1;
 }
 
 void
 gf_log_disable_syslog (void)
 {
-        gf_log_syslog = 0;
+        THIS->ctx->log.gf_log_syslog = 0;
 }
 
 gf_loglevel_t
 gf_log_get_loglevel (void)
 {
-        return loglevel;
+        return THIS->ctx->log.loglevel;
 }
 
 void
 gf_log_set_loglevel (gf_loglevel_t level)
 {
-        gf_log_loglevel = loglevel = level;
+        THIS->ctx->log.loglevel = level;
 }
 
 
@@ -105,98 +103,312 @@ gf_log_set_xl_loglevel (void *this, gf_loglevel_t level)
         xlator_t *xl = this;
         if (!xl)
                 return;
-        gf_log_xl_log_set = 1;
+        xl->ctx->log.gf_log_xl_log_set = 1;
         xl->loglevel = level;
 }
 
 void
 gf_log_fini (void)
 {
-        pthread_mutex_destroy (&logfile_mutex);
+        pthread_mutex_destroy (&THIS->ctx->log.logfile_mutex);
+}
+
+
+#ifdef GF_USE_SYSLOG
+/**
+ * gf_get_error_message -function to get error message for given error code
+ * @error_code: error code defined by log book
+ *
+ * @return: success: string
+ *          failure: NULL
+ */
+const char *
+gf_get_error_message (int error_code) {
+        return _gf_get_message (error_code);
+}
+
+
+/**
+ * gf_openlog -function to open syslog specific to gluster based on
+ *             existence of file /etc/glusterfs/logger.conf
+ * @ident:    optional identification string similar to openlog()
+ * @option:   optional value to option to openlog().  Passing -1 uses
+ *            'LOG_PID | LOG_NDELAY' as default
+ * @facility: optional facility code similar to openlog().  Passing -1
+ *            uses LOG_DAEMON as default
+ *
+ * @return: void
+ */
+void
+gf_openlog (const char *ident, int option, int facility)
+{
+        int _option = option;
+        int _facility = facility;
+
+        if (-1 == _option) {
+                _option = LOG_PID | LOG_NDELAY;
+        }
+        if (-1 == _facility) {
+                _facility = LOG_LOCAL1;
+        }
+
+        setlocale(LC_ALL, "");
+        bindtextdomain("gluster", "/usr/share/locale");
+        textdomain("gluster");
+
+        openlog(ident, _option, _facility);
 }
 
 
+/**
+ * _json_escape -function to convert string to json encoded string
+ * @str: input string
+ * @buf: buffer to store encoded string
+ * @len: length of @buf
+ *
+ * @return: success: last unprocessed character position by pointer in @str
+ *          failure: NULL
+ *
+ * Internal function. Heavily inspired by _ul_str_escape() function in
+ * libumberlog
+ *
+ * Sample output:
+ * [1] str = "devel error"
+ *     buf = "devel error"
+ * [2] str = "devel	error"
+ *     buf = "devel\terror"
+ * [3] str = "I/O error on "/tmp/foo" file"
+ *     buf = "I/O error on \"/tmp/foo\" file"
+ * [4] str = "I/O erroron /tmp/bar file"
+ *     buf = "I/O error\u001bon /tmp/bar file"
+ *
+ */
+char *
+_json_escape(const char *str, char *buf, size_t len)
+{
+        static const unsigned char json_exceptions[UCHAR_MAX + 1] =
+                {
+                        [0x01] = 1, [0x02] = 1, [0x03] = 1, [0x04] = 1,
+                        [0x05] = 1, [0x06] = 1, [0x07] = 1, [0x08] = 1,
+                        [0x09] = 1, [0x0a] = 1, [0x0b] = 1, [0x0c] = 1,
+                        [0x0d] = 1, [0x0e] = 1, [0x0f] = 1, [0x10] = 1,
+                        [0x11] = 1, [0x12] = 1, [0x13] = 1, [0x14] = 1,
+                        [0x15] = 1, [0x16] = 1, [0x17] = 1, [0x18] = 1,
+                        [0x19] = 1, [0x1a] = 1, [0x1b] = 1, [0x1c] = 1,
+                        [0x1d] = 1, [0x1e] = 1, [0x1f] = 1,
+                        ['\\'] = 1, ['"'] = 1
+                };
+        static const char  json_hex_chars[16] = "0123456789abcdef";
+        unsigned char     *p = NULL;
+        size_t             pos = 0;
+
+        if (!str || !buf || len <= 0) {
+                return NULL;
+        }
+
+        for (p = (unsigned char *)str;
+             *p && (pos + 1) < len;
+             p++)
+        {
+                if (json_exceptions[*p] == 0) {
+                        buf[pos++] = *p;
+                        continue;
+                }
+
+                if ((pos + 2) >= len) {
+                        break;
+                }
+
+                switch (*p)
+                {
+                case '\b':
+                        buf[pos++] = '\\';
+                        buf[pos++] = 'b';
+                        break;
+                case '\n':
+                        buf[pos++] = '\\';
+                        buf[pos++] = 'n';
+                        break;
+                case '\r':
+                        buf[pos++] = '\\';
+                        buf[pos++] = 'r';
+                        break;
+                case '\t':
+                        buf[pos++] = '\\';
+                        buf[pos++] = 't';
+                        break;
+                case '\\':
+                        buf[pos++] = '\\';
+                        buf[pos++] = '\\';
+                        break;
+                case '"':
+                        buf[pos++] = '\\';
+                        buf[pos++] = '"';
+                        break;
+                default:
+                        if ((pos + 6) >= len) {
+                                buf[pos] = '\0';
+                                return (char *)p;
+                        }
+                        buf[pos++] = '\\';
+                        buf[pos++] = 'u';
+                        buf[pos++] = '0';
+                        buf[pos++] = '0';
+                        buf[pos++] = json_hex_chars[(*p) >> 4];
+                        buf[pos++] = json_hex_chars[(*p) & 0xf];
+                        break;
+                }
+        }
+
+        buf[pos] = '\0';
+        return (char *)p;
+}
+
+
+/**
+ * gf_syslog -function to submit message to syslog specific to gluster
+ * @error_code:        error code defined by log book
+ * @facility_priority: facility_priority of syslog()
+ * @format:            optional format string to syslog()
+ *
+ * @return: void
+ */
+void
+gf_syslog (int error_code, int facility_priority, char *format, ...)
+{
+        char       *msg = NULL;
+        char        json_msg[GF_JSON_MSG_LENGTH];
+        GF_UNUSED char       *p = NULL;
+        const char *error_message = NULL;
+        char        json_error_message[GF_JSON_MSG_LENGTH];
+        va_list     ap;
+
+        error_message = gf_get_error_message (error_code);
+
+        va_start (ap, format);
+        if (format) {
+                vasprintf (&msg, format, ap);
+                p = _json_escape (msg, json_msg, GF_JSON_MSG_LENGTH);
+                if (error_message) {
+                        p = _json_escape (error_message, json_error_message,
+                                          GF_JSON_MSG_LENGTH);
+                        syslog (facility_priority, GF_SYSLOG_CEE_FORMAT,
+                                json_msg, error_code, json_error_message);
+                } else {
+                        /* ignore the error code because no error message for it
+                           and use normal syslog */
+                        syslog (facility_priority, "%s", msg);
+                }
+                free (msg);
+        } else {
+                if (error_message) {
+                        /* no user message: treat error_message as msg */
+                        syslog (facility_priority, GF_SYSLOG_CEE_FORMAT,
+                                json_error_message, error_code,
+                                json_error_message);
+                } else {
+                        /* cannot produce log as neither error_message nor
+                           msg available */
+                }
+        }
+        va_end (ap);
+}
+#endif /* GF_USE_SYSLOG */
+
 void
-gf_log_globals_init (void)
+gf_log_globals_init (void *data)
 {
-        pthread_mutex_init (&logfile_mutex, NULL);
+        glusterfs_ctx_t *ctx = data;
 
+        pthread_mutex_init (&ctx->log.logfile_mutex, NULL);
+
+        ctx->log.loglevel         = GF_LOG_INFO;
+        ctx->log.gf_log_syslog    = 1;
+        ctx->log.sys_log_level    = GF_LOG_CRITICAL;
+
+#ifndef GF_USE_SYSLOG
 #ifdef GF_LINUX_HOST_OS
         /* For the 'syslog' output. one can grep 'GlusterFS' in syslog
            for serious logs */
         openlog ("GlusterFS", LOG_PID, LOG_DAEMON);
 #endif
+#endif
 }
 
 int
-gf_log_init (const char *file)
+gf_log_init (void *data, const char *file, const char *ident)
 {
+        glusterfs_ctx_t *ctx = NULL;
+        int     fd = -1;
+
+        ctx = data;
+
+#if defined(GF_USE_SYSLOG)
+        {
+                /* use default ident and option */
+                /* TODO: make FACILITY configurable than LOG_DAEMON */
+                struct stat buf;
+
+                if (stat (GF_LOG_CONTROL_FILE, &buf) == 0) {
+                        /* use syslog logging */
+                        ctx->log.log_control_file_found = 1;
+                        if (ident) {
+                                /* we need to keep this value as */
+                                /* syslog uses it on every logging */
+                                ctx->log.ident = gf_strdup (ident);
+                                gf_openlog (ctx->log.ident, -1, LOG_DAEMON);
+                        } else {
+                                gf_openlog (NULL, -1, LOG_DAEMON);
+                        }
+                } else {
+                        /* use old style logging */
+                        ctx->log.log_control_file_found = 0;
+                }
+        }
+#endif
+
         if (!file){
                 fprintf (stderr, "ERROR: no filename specified\n");
                 return -1;
         }
 
         if (strcmp (file, "-") == 0) {
-                gf_log_logfile = stderr;
-
+                ctx->log.gf_log_logfile = stderr;
+                ctx->log.logfile = stderr;
                 return 0;
         }
 
-        filename = gf_strdup (file);
-        if (!filename) {
+        ctx->log.filename = gf_strdup (file);
+        if (!ctx->log.filename) {
                 fprintf (stderr, "ERROR: updating log-filename failed: %s\n",
                          strerror (errno));
                 return -1;
         }
 
-        logfile = fopen (file, "a");
-        if (!logfile){
+        fd = open (file, O_CREAT | O_RDONLY, S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+                fprintf (stderr, "ERROR: failed to create logfile \"%s\" (%s)\n",
+                         file, strerror (errno));
+                return -1;
+        }
+        close (fd);
+
+        ctx->log.logfile = fopen (file, "a");
+        if (!ctx->log.logfile){
                 fprintf (stderr, "ERROR: failed to open logfile \"%s\" (%s)\n",
                          file, strerror (errno));
                 return -1;
         }
 
-        gf_log_logfile = logfile;
+        ctx->log.gf_log_logfile = ctx->log.logfile;
 
         return 0;
 }
 
-
-
-struct _msg_queue {
-        struct list_head msgs;
-};
-
-struct _log_msg {
-        const char *msg;
-        struct list_head queue;
-};
-
-
-void
-gf_log_lock (void)
-{
-        pthread_mutex_lock (&logfile_mutex);
-}
-
-
-void
-gf_log_unlock (void)
-{
-        pthread_mutex_unlock (&logfile_mutex);
-}
-
-
-void
-gf_log_cleanup (void)
-{
-        pthread_mutex_destroy (&logfile_mutex);
-}
-
 void
 set_sys_log_level (gf_loglevel_t level)
 {
-        sys_log_level = level;
+        THIS->ctx->log.sys_log_level = level;
 }
 
 int
@@ -205,22 +417,23 @@ _gf_log_nomem (const char *domain, const char *file,
                size_t size)
 {
         const char     *basename        = NULL;
-        struct tm      *tm              = NULL;
         xlator_t       *this            = NULL;
         struct timeval  tv              = {0,};
         int             ret             = 0;
-        char            msg[8092];
-        char            timestr[256];
-        char            callstr[4096];
+        char            msg[8092]       = {0,};
+        char            timestr[256]    = {0,};
+        char            callstr[4096]   = {0,};
+        glusterfs_ctx_t *ctx = NULL;
 
         this = THIS;
+        ctx = this->ctx;
 
-        if (gf_log_xl_log_set) {
+        if (ctx->log.gf_log_xl_log_set) {
                 if (this->loglevel && (level > this->loglevel))
                         goto out;
-                else if (level > gf_log_loglevel)
-                        goto out;
         }
+        if (level > ctx->log.loglevel)
+                goto out;
 
         static char *level_strings[] = {"",  /* NONE */
                                         "M", /* EMERGENCY */
@@ -241,6 +454,12 @@ _gf_log_nomem (const char *domain, const char *file,
                 return -1;
         }
 
+        basename = strrchr (file, '/');
+        if (basename)
+                basename++;
+        else
+                basename = file;
+
 #if HAVE_BACKTRACE
         /* Print 'calling function' */
         do {
@@ -267,36 +486,45 @@ _gf_log_nomem (const char *domain, const char *file,
         } while (0);
 #endif /* HAVE_BACKTRACE */
 
+#if defined(GF_USE_SYSLOG)
+        if (ctx->log.log_control_file_found)
+        {
+                int priority;
+                /* treat GF_LOG_TRACE and GF_LOG_NONE as LOG_DEBUG and
+                   other level as is */
+                if (GF_LOG_TRACE == level || GF_LOG_NONE == level) {
+                        priority = LOG_DEBUG;
+                } else {
+                        priority = level - 1;
+                }
+                gf_syslog (GF_ERR_DEV, priority,
+                           "[%s:%d:%s] %s %s: no memory "
+                           "available for size (%"GF_PRI_SIZET")",
+                           basename, line, function, callstr, domain,
+                           size);
+                goto out;
+        }
+#endif /* GF_USE_SYSLOG */
         ret = gettimeofday (&tv, NULL);
         if (-1 == ret)
                 goto out;
+        gf_time_fmt (timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT);
+        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                  ".%"GF_PRI_SUSECONDS, tv.tv_usec);
 
-        tm    = localtime (&tv.tv_sec);
+        ret = sprintf (msg, "[%s] %s [%s:%d:%s] %s %s: no memory "
+                       "available for size (%"GF_PRI_SIZET")",
+                       timestr, level_strings[level],
+                       basename, line, function, callstr,
+                       domain, size);
+        if (-1 == ret) {
+                goto out;
+        }
 
-        pthread_mutex_lock (&logfile_mutex);
+        pthread_mutex_lock (&ctx->log.logfile_mutex);
         {
-                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
-
-                basename = strrchr (file, '/');
-                if (basename)
-                        basename++;
-                else
-                        basename = file;
-
-                ret = sprintf (msg, "[%s] %s [%s:%d:%s] %s %s: no memory "
-                               "available for size (%"GF_PRI_SIZET")",
-                               timestr, level_strings[level],
-                               basename, line, function, callstr,
-                               domain, size);
-                if (-1 == ret) {
-                        goto unlock;
-                }
-
-                if (logfile) {
-                        fprintf (logfile, "%s\n", msg);
-                        fflush (logfile);
+                if (ctx->log.logfile) {
+                        fprintf (ctx->log.logfile, "%s\n", msg);
                 } else {
                         fprintf (stderr, "%s\n", msg);
                 }
@@ -304,13 +532,13 @@ _gf_log_nomem (const char *domain, const char *file,
 #ifdef GF_LINUX_HOST_OS
                 /* We want only serious log in 'syslog', not our debug
                    and trace logs */
-                if (gf_log_syslog && level && (level <= sys_log_level))
+                if (ctx->log.gf_log_syslog && level &&
+                    (level <= ctx->log.sys_log_level))
                         syslog ((level-1), "%s\n", msg);
 #endif
         }
 
-unlock:
-        pthread_mutex_unlock (&logfile_mutex);
+        pthread_mutex_unlock (&ctx->log.logfile_mutex);
 out:
         return ret;
  }
@@ -320,7 +548,6 @@ _gf_log_callingfn (const char *domain, const char *file, const char *function,
                    int line, gf_loglevel_t level, const char *fmt, ...)
 {
         const char     *basename        = NULL;
-        struct tm      *tm              = NULL;
         xlator_t       *this            = NULL;
         char           *str1            = NULL;
         char           *str2            = NULL;
@@ -331,15 +558,17 @@ _gf_log_callingfn (const char *domain, const char *file, const char *function,
         size_t          len             = 0;
         int             ret             = 0;
         va_list         ap;
+        glusterfs_ctx_t *ctx = NULL;
 
         this = THIS;
+        ctx = this->ctx;
 
-        if (gf_log_xl_log_set) {
+        if (ctx->log.gf_log_xl_log_set) {
                 if (this->loglevel && (level > this->loglevel))
                         goto out;
-                else if (level > gf_log_loglevel)
-                        goto out;
         }
+        if (level > ctx->log.loglevel)
+                goto out;
 
         static char *level_strings[] = {"",  /* NONE */
                                         "M", /* EMERGENCY */
@@ -360,6 +589,12 @@ _gf_log_callingfn (const char *domain, const char *file, const char *function,
                 return -1;
         }
 
+        basename = strrchr (file, '/');
+        if (basename)
+                basename++;
+        else
+                basename = file;
+
 #if HAVE_BACKTRACE
         /* Print 'calling function' */
         do {
@@ -386,50 +621,65 @@ _gf_log_callingfn (const char *domain, const char *file, const char *function,
         } while (0);
 #endif /* HAVE_BACKTRACE */
 
+#if defined(GF_USE_SYSLOG)
+        if (ctx->log.log_control_file_found)
+        {
+                int priority;
+                /* treat GF_LOG_TRACE and GF_LOG_NONE as LOG_DEBUG and
+                   other level as is */
+                if (GF_LOG_TRACE == level || GF_LOG_NONE == level) {
+                        priority = LOG_DEBUG;
+                } else {
+                        priority = level - 1;
+                }
+
+                va_start (ap, fmt);
+                vasprintf (&str2, fmt, ap);
+                va_end (ap);
+
+                gf_syslog (GF_ERR_DEV, priority,
+                           "[%s:%d:%s] %s %d-%s: %s",
+                           basename, line, function,
+                           callstr,
+                           ((this->graph) ? this->graph->id:0), domain,
+                           str2);
+
+                goto out;
+        }
+#endif /* GF_USE_SYSLOG */
         ret = gettimeofday (&tv, NULL);
         if (-1 == ret)
                 goto out;
+        va_start (ap, fmt);
+        gf_time_fmt (timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT);
+        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                  ".%"GF_PRI_SUSECONDS, tv.tv_usec);
 
-        tm    = localtime (&tv.tv_sec);
-
-        pthread_mutex_lock (&logfile_mutex);
-        {
-                va_start (ap, fmt);
-
-                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
-
-                basename = strrchr (file, '/');
-                if (basename)
-                        basename++;
-                else
-                        basename = file;
-
-                ret = gf_asprintf (&str1, "[%s] %s [%s:%d:%s] %s %d-%s: ",
-                                   timestr, level_strings[level],
-                                   basename, line, function, callstr,
-                                   ((this->graph) ? this->graph->id:0), domain);
-                if (-1 == ret) {
-                        goto unlock;
-                }
+        ret = gf_asprintf (&str1, "[%s] %s [%s:%d:%s] %s %d-%s: ",
+                           timestr, level_strings[level],
+                           basename, line, function, callstr,
+                           ((this->graph) ? this->graph->id:0), domain);
+        if (-1 == ret) {
+                goto out;
+        }
 
-                ret = vasprintf (&str2, fmt, ap);
-                if (-1 == ret) {
-                        goto unlock;
-                }
+        ret = vasprintf (&str2, fmt, ap);
+        if (-1 == ret) {
+                goto out;
+        }
 
-                va_end (ap);
+        va_end (ap);
 
-                len = strlen (str1);
-                msg = GF_MALLOC (len + strlen (str2) + 1, gf_common_mt_char);
+        len = strlen (str1);
+        msg = GF_MALLOC (len + strlen (str2) + 1, gf_common_mt_char);
 
-                strcpy (msg, str1);
-                strcpy (msg + len, str2);
+        strcpy (msg, str1);
+        strcpy (msg + len, str2);
 
-                if (logfile) {
-                        fprintf (logfile, "%s\n", msg);
-                        fflush (logfile);
+        pthread_mutex_lock (&ctx->log.logfile_mutex);
+        {
+                if (ctx->log.logfile) {
+                        fprintf (ctx->log.logfile, "%s\n", msg);
                 } else {
                         fprintf (stderr, "%s\n", msg);
                 }
@@ -437,25 +687,21 @@ _gf_log_callingfn (const char *domain, const char *file, const char *function,
 #ifdef GF_LINUX_HOST_OS
                 /* We want only serious log in 'syslog', not our debug
                    and trace logs */
-                if (gf_log_syslog && level && (level <= sys_log_level))
+                if (ctx->log.gf_log_syslog && level &&
+                    (level <= ctx->log.sys_log_level))
                         syslog ((level-1), "%s\n", msg);
 #endif
         }
 
-unlock:
-        pthread_mutex_unlock (&logfile_mutex);
+        pthread_mutex_unlock (&ctx->log.logfile_mutex);
 
-        if (msg) {
-                GF_FREE (msg);
-        }
+out:
+        GF_FREE (msg);
 
-        if (str1)
-                GF_FREE (str1);
+        GF_FREE (str1);
 
-        if (str2)
-                FREE (str2);
+        FREE (str2);
 
-out:
         return ret;
 }
 
@@ -463,28 +709,29 @@ int
 _gf_log (const char *domain, const char *file, const char *function, int line,
          gf_loglevel_t level, const char *fmt, ...)
 {
-        const char  *basename = NULL;
-        FILE        *new_logfile = NULL;
-        va_list      ap;
-        struct tm   *tm = NULL;
-        char         timestr[256];
+        const char    *basename = NULL;
+        FILE          *new_logfile = NULL;
+        va_list        ap;
+        char           timestr[256] = {0,};
         struct timeval tv = {0,};
-
-        char        *str1 = NULL;
-        char        *str2 = NULL;
-        char        *msg  = NULL;
-        size_t       len  = 0;
-        int          ret  = 0;
-        xlator_t    *this = NULL;
+        char          *str1 = NULL;
+        char          *str2 = NULL;
+        char          *msg  = NULL;
+        size_t         len  = 0;
+        int            ret  = 0;
+        int            fd   = -1;
+        xlator_t      *this = NULL;
+        glusterfs_ctx_t *ctx = NULL;
 
         this = THIS;
+        ctx = this->ctx;
 
-        if (gf_log_xl_log_set) {
+        if (ctx->log.gf_log_xl_log_set) {
                 if (this->loglevel && (level > this->loglevel))
                         goto out;
-                else if (level > gf_log_loglevel)
-                        goto out;
         }
+        if (level > ctx->log.loglevel)
+                goto out;
 
         static char *level_strings[] = {"",  /* NONE */
                                         "M", /* EMERGENCY */
@@ -505,124 +752,221 @@ _gf_log (const char *domain, const char *file, const char *function, int line,
                 return -1;
         }
 
+        basename = strrchr (file, '/');
+        if (basename)
+                basename++;
+        else
+                basename = file;
 
-        if (logrotate) {
-                logrotate = 0;
+#if defined(GF_USE_SYSLOG)
+        if (ctx->log.log_control_file_found)
+        {
+                int priority;
+                /* treat GF_LOG_TRACE and GF_LOG_NONE as LOG_DEBUG and
+                   other level as is */
+                if (GF_LOG_TRACE == level || GF_LOG_NONE == level) {
+                        priority = LOG_DEBUG;
+                } else {
+                        priority = level - 1;
+                }
 
-                new_logfile = fopen (filename, "a");
+                va_start (ap, fmt);
+                vasprintf (&str2, fmt, ap);
+                va_end (ap);
+
+                gf_syslog (GF_ERR_DEV, priority,
+                           "[%s:%d:%s] %d-%s: %s",
+                           basename, line, function,
+                           ((this->graph) ? this->graph->id:0), domain, str2);
+                goto err;
+        }
+#endif /* GF_USE_SYSLOG */
+
+        if (ctx->log.logrotate) {
+                ctx->log.logrotate = 0;
+
+                fd = open (ctx->log.filename,
+                           O_CREAT | O_RDONLY, S_IRUSR | S_IWUSR);
+                if (fd < 0) {
+                        gf_log ("logrotate", GF_LOG_ERROR,
+                                "%s", strerror (errno));
+                        return -1;
+                }
+                close (fd);
+
+                new_logfile = fopen (ctx->log.filename, "a");
                 if (!new_logfile) {
                         gf_log ("logrotate", GF_LOG_CRITICAL,
                                 "failed to open logfile %s (%s)",
-                                filename, strerror (errno));
+                                ctx->log.filename, strerror (errno));
                         goto log;
                 }
 
-                if (logfile)
-                        fclose (logfile);
+                pthread_mutex_lock (&ctx->log.logfile_mutex);
+                {
+                        if (ctx->log.logfile)
+                                fclose (ctx->log.logfile);
+
+                        ctx->log.gf_log_logfile = ctx->log.logfile = new_logfile;
+                }
+                pthread_mutex_unlock (&ctx->log.logfile_mutex);
 
-                gf_log_logfile = logfile = new_logfile;
         }
 
 log:
         ret = gettimeofday (&tv, NULL);
         if (-1 == ret)
                 goto out;
+        va_start (ap, fmt);
+        gf_time_fmt (timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT);
+        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                  ".%"GF_PRI_SUSECONDS, tv.tv_usec);
 
-        tm    = localtime (&tv.tv_sec);
-
-        pthread_mutex_lock (&logfile_mutex);
-        {
-                va_start (ap, fmt);
+        ret = gf_asprintf (&str1, "[%s] %s [%s:%d:%s] %d-%s: ",
+                           timestr, level_strings[level],
+                           basename, line, function,
+                           ((this->graph)?this->graph->id:0), domain);
+        if (-1 == ret) {
+                goto err;
+        }
 
-                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
-
-                basename = strrchr (file, '/');
-                if (basename)
-                        basename++;
-                else
-                        basename = file;
-
-                ret = gf_asprintf (&str1, "[%s] %s [%s:%d:%s] %d-%s: ",
-                                   timestr, level_strings[level],
-                                   basename, line, function,
-                                   ((this->graph)?this->graph->id:0), domain);
-                if (-1 == ret) {
-                        goto unlock;
-                }
+        ret = vasprintf (&str2, fmt, ap);
+        if (-1 == ret) {
+                goto err;
+        }
 
-                ret = vasprintf (&str2, fmt, ap);
-                if (-1 == ret) {
-                        goto unlock;
-                }
+        va_end (ap);
 
-                va_end (ap);
+        len = strlen (str1);
+        msg = GF_MALLOC (len + strlen (str2) + 1, gf_common_mt_char);
 
-                len = strlen (str1);
-                msg = GF_MALLOC (len + strlen (str2) + 1, gf_common_mt_char);
+        strcpy (msg, str1);
+        strcpy (msg + len, str2);
 
-                strcpy (msg, str1);
-                strcpy (msg + len, str2);
+        pthread_mutex_lock (&ctx->log.logfile_mutex);
+        {
 
-                if (logfile) {
-                        fprintf (logfile, "%s\n", msg);
-                        fflush (logfile);
+                if (ctx->log.logfile) {
+                        fprintf (ctx->log.logfile, "%s\n", msg);
+                        fflush (ctx->log.logfile);
                 } else {
                         fprintf (stderr, "%s\n", msg);
+                        fflush (stderr);
                 }
 
 #ifdef GF_LINUX_HOST_OS
                 /* We want only serious log in 'syslog', not our debug
                    and trace logs */
-                if (gf_log_syslog && level && (level <= sys_log_level))
+                if (ctx->log.gf_log_syslog && level &&
+                    (level <= ctx->log.sys_log_level))
                         syslog ((level-1), "%s\n", msg);
 #endif
         }
 
-unlock:
-        pthread_mutex_unlock (&logfile_mutex);
+        pthread_mutex_unlock (&ctx->log.logfile_mutex);
 
-        if (msg) {
-                GF_FREE (msg);
-        }
+err:
+        GF_FREE (msg);
 
-        if (str1)
-                GF_FREE (str1);
+        GF_FREE (str1);
 
-        if (str2)
-                FREE (str2);
+        FREE (str2);
 
 out:
         return (0);
 }
 
+int
+_gf_log_eh (const char *function, const char *fmt, ...)
+{
+        int          ret   = -1;
+        va_list      ap;
+        char         *str1 = NULL;
+        char         *str2 = NULL;
+        char         *msg  = NULL;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+
+        ret = gf_asprintf (&str1, "[%d] %s: ",
+                           ((this->graph)?this->graph->id:0),
+                           function);
+        if (-1 == ret) {
+                goto out;
+        }
+
+        va_start (ap, fmt);
+
+        ret = vasprintf (&str2, fmt, ap);
+        if (-1 == ret) {
+                goto out;
+        }
+
+        va_end (ap);
+
+        msg = GF_MALLOC (strlen (str1) + strlen (str2) + 1, gf_common_mt_char);
+        if (!msg) {
+                ret = -1;
+                goto out;
+        }
+
+        strcpy (msg, str1);
+        strcat (msg, str2);
+
+        ret = eh_save_history (this->history, msg);
+
+out:
+        GF_FREE (str1);
+
+        /* Use FREE instead of GF_FREE since str2 was allocated by vasprintf */
+        if (str2)
+                FREE (str2);
+
+         return ret;
+}
 
 int
 gf_cmd_log_init (const char *filename)
 {
+        int         fd   = -1;
+        xlator_t   *this = NULL;
+        glusterfs_ctx_t *ctx = NULL;
+
+        this = THIS;
+        ctx  = this->ctx;
+
         if (!filename){
-                gf_log ("glusterd", GF_LOG_CRITICAL, "gf_cmd_log_init: no "
+                gf_log (this->name, GF_LOG_CRITICAL, "gf_cmd_log_init: no "
                         "filename specified\n");
                 return -1;
         }
 
-        cmd_log_filename = gf_strdup (filename);
-        if (!cmd_log_filename) {
-                gf_log ("glusterd", GF_LOG_CRITICAL, "gf_cmd_log_init: strdup"
-                        " error\n");
+        ctx->log.cmd_log_filename = gf_strdup (filename);
+        if (!ctx->log.cmd_log_filename) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "gf_cmd_log_init: strdup error\n");
                 return -1;
         }
         /* close and reopen cmdlogfile for log rotate*/
-        if (cmdlogfile) {
-                fclose (cmdlogfile);
-                cmdlogfile = NULL;
+        if (ctx->log.cmdlogfile) {
+                fclose (ctx->log.cmdlogfile);
+                ctx->log.cmdlogfile = NULL;
         }
-        cmdlogfile = fopen (cmd_log_filename, "a");
-        if (!cmdlogfile){
-                gf_log ("glusterd", GF_LOG_CRITICAL,
+
+        fd = open (ctx->log.cmd_log_filename,
+                   O_CREAT | O_RDONLY, S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "%s", strerror (errno));
+                return -1;
+        }
+        close (fd);
+
+        ctx->log.cmdlogfile = fopen (ctx->log.cmd_log_filename, "a");
+        if (!ctx->log.cmdlogfile){
+                gf_log (this->name, GF_LOG_CRITICAL,
                         "gf_cmd_log_init: failed to open logfile \"%s\" "
-                        "(%s)\n", cmd_log_filename, strerror (errno));
+                        "(%s)\n", ctx->log.cmd_log_filename, strerror (errno));
                 return -1;
         }
         return 0;
@@ -631,17 +975,18 @@ gf_cmd_log_init (const char *filename)
 int
 gf_cmd_log (const char *domain, const char *fmt, ...)
 {
-        va_list      ap;
-        struct tm   *tm = NULL;
-        char         timestr[256];
+        va_list        ap;
+        char           timestr[64];
         struct timeval tv = {0,};
-        char        *str1 = NULL;
-        char        *str2 = NULL;
-        char        *msg  = NULL;
-        size_t       len  = 0;
-        int          ret  = 0;
-
-        if (!cmdlogfile)
+        char          *str1 = NULL;
+        char          *str2 = NULL;
+        char          *msg  = NULL;
+        size_t         len  = 0;
+        int            ret  = 0;
+        glusterfs_ctx_t *ctx = NULL;
+
+        ctx = THIS->ctx;
+        if (!ctx->log.cmdlogfile)
                 return -1;
 
 
@@ -654,11 +999,8 @@ gf_cmd_log (const char *domain, const char *fmt, ...)
         ret = gettimeofday (&tv, NULL);
         if (ret == -1)
                 goto out;
-
-        tm = localtime (&tv.tv_sec);
-
         va_start (ap, fmt);
-        strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+        gf_time_fmt (timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT);
         snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
                   ".%"GF_PRI_SUSECONDS, tv.tv_usec);
 
@@ -681,19 +1023,15 @@ gf_cmd_log (const char *domain, const char *fmt, ...)
         strcpy (msg, str1);
         strcpy (msg + len, str2);
 
-        fprintf (cmdlogfile, "%s\n", msg);
-        fflush (cmdlogfile);
+        fprintf (ctx->log.cmdlogfile, "%s\n", msg);
+        fflush (ctx->log.cmdlogfile);
 
 out:
-        if (msg) {
-                GF_FREE (msg);
-        }
+        GF_FREE (msg);
 
-        if (str1)
-                GF_FREE (str1);
+        GF_FREE (str1);
 
-        if (str2)
-                FREE (str2);
+        FREE (str2);
 
         return (0);
 }
diff --git a/libglusterfs/src/logging.h b/libglusterfs/src/logging.h
index cb23cb54f..cc806a767 100644
--- a/libglusterfs/src/logging.h
+++ b/libglusterfs/src/logging.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __LOGGING_H__
@@ -28,6 +19,7 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <stdarg.h>
+#include <pthread.h>
 
 #ifdef GF_DARWIN_HOST_OS
 #define GF_PRI_FSBLK       "u"
@@ -43,6 +35,7 @@
 #define GF_PRI_BLKSIZE     PRId32
 #define GF_PRI_SIZET       "zu"
 
+
 #if 0
 /* Syslog definitions :-) */
 #define LOG_EMERG   0           /* system is unusable */
@@ -68,25 +61,67 @@ typedef enum {
         GF_LOG_TRACE,      /* full trace of operation */
 } gf_loglevel_t;
 
-extern gf_loglevel_t gf_log_loglevel;
-extern char gf_log_xl_log_set;
+#define DEFAULT_LOG_FILE_DIRECTORY            DATADIR "/log/glusterfs"
+#define DEFAULT_LOG_LEVEL                     GF_LOG_INFO
+
+typedef struct gf_log_handle_ {
+        pthread_mutex_t  logfile_mutex;
+        uint8_t          logrotate;
+        gf_loglevel_t    loglevel;
+        int              gf_log_syslog;
+        gf_loglevel_t    sys_log_level;
+        char             gf_log_xl_log_set;
+        char            *filename;
+        FILE            *logfile;
+        FILE            *gf_log_logfile;
+        char            *cmd_log_filename;
+        FILE            *cmdlogfile;
+#ifdef GF_USE_SYSLOG
+        int              log_control_file_found;
+        char            *ident;
+#endif /* GF_USE_SYSLOG */
+
+} gf_log_handle_t;
+
+void gf_log_globals_init (void *ctx);
+int gf_log_init (void *data, const char *filename, const char *ident);
+
+void gf_log_logrotate (int signum);
+
+void gf_log_cleanup (void);
+
+int _gf_log (const char *domain, const char *file,
+             const char *function, int32_t line, gf_loglevel_t level,
+             const char *fmt, ...)
+             __attribute__ ((__format__ (__printf__, 6, 7)));
+int _gf_log_callingfn (const char *domain, const char *file,
+                       const char *function, int32_t line, gf_loglevel_t level,
+                       const char *fmt, ...)
+                       __attribute__ ((__format__ (__printf__, 6, 7)));
+
+int _gf_log_nomem (const char *domain, const char *file,
+                   const char *function, int line, gf_loglevel_t level,
+                   size_t size);
+
+int _gf_log_eh (const char *function, const char *fmt, ...);
+
+
 
 #define FMT_WARN(fmt...) do { if (0) printf (fmt); } while (0)
 
 #define gf_log(dom, levl, fmt...) do {                                  \
                 FMT_WARN (fmt);                                         \
-                                                                        \
-                if ((levl > gf_log_loglevel) && !gf_log_xl_log_set)     \
-                        break;                                          \
                 _gf_log (dom, __FILE__, __FUNCTION__, __LINE__,         \
                          levl, ##fmt);                                  \
         } while (0)
 
+#define gf_log_eh(fmt...) do {                                          \
+                FMT_WARN (fmt);                                         \
+                _gf_log_eh (__FUNCTION__, ##fmt);                        \
+        } while (0)
+
 #define gf_log_callingfn(dom, levl, fmt...) do {                        \
                 FMT_WARN (fmt);                                         \
-                                                                        \
-                if ((levl > gf_log_loglevel) && !gf_log_xl_log_set)     \
-                        break;                                          \
                 _gf_log_callingfn (dom, __FILE__, __FUNCTION__, __LINE__, \
                                    levl, ##fmt);                        \
         } while (0)
@@ -94,8 +129,6 @@ extern char gf_log_xl_log_set;
 
 /* No malloc or calloc should be called in this function */
 #define gf_log_nomem(dom, levl, size) do {                              \
-                if ((levl > gf_log_loglevel) && !gf_log_xl_log_set)     \
-                        break;                                          \
                 _gf_log_nomem (dom, __FILE__, __FUNCTION__, __LINE__,   \
                                levl, size);                             \
         } while (0)
@@ -106,31 +139,6 @@ extern char gf_log_xl_log_set;
                 gf_log (args);                                          \
         }
 
-
-void gf_log_logrotate (int signum);
-
-void gf_log_globals_init (void);
-int gf_log_init (const char *filename);
-void gf_log_cleanup (void);
-
-int _gf_log (const char *domain, const char *file,
-             const char *function, int32_t line, gf_loglevel_t level,
-             const char *fmt, ...)
-             __attribute__ ((__format__ (__printf__, 6, 7)));
-int _gf_log_callingfn (const char *domain, const char *file, 
-                       const char *function, int32_t line, gf_loglevel_t level,
-                       const char *fmt, ...)
-                       __attribute__ ((__format__ (__printf__, 6, 7)));
-
-int _gf_log_nomem (const char *domain, const char *file,
-                   const char *function, int line, gf_loglevel_t level,
-                   size_t size);
-
-int gf_log_from_client (const char *msg, char *identifier);
-
-void gf_log_lock (void);
-void gf_log_unlock (void);
-
 void gf_log_disable_syslog (void);
 void gf_log_enable_syslog (void);
 gf_loglevel_t gf_log_get_loglevel (void);
@@ -138,6 +146,13 @@ void gf_log_set_loglevel (gf_loglevel_t level);
 gf_loglevel_t gf_log_get_xl_loglevel (void *xl);
 void gf_log_set_xl_loglevel (void *xl, gf_loglevel_t level);
 
+int gf_cmd_log (const char *domain, const char *fmt, ...)
+                __attribute__ ((__format__ (__printf__, 2, 3)));
+
+int gf_cmd_log_init (const char *filename);
+
+void set_sys_log_level (gf_loglevel_t level);
+
 #define GF_DEBUG(xl, format, args...)                           \
         gf_log ((xl)->name, GF_LOG_DEBUG, format, ##args)
 #define GF_INFO(xl, format, args...)                            \
@@ -147,10 +162,4 @@ void gf_log_set_xl_loglevel (void *xl, gf_loglevel_t level);
 #define GF_ERROR(xl, format, args...)                           \
         gf_log ((xl)->name, GF_LOG_ERROR, format, ##args)
 
-int gf_cmd_log (const char *domain, const char *fmt, ...)
-                __attribute__ ((__format__ (__printf__, 2, 3)));
-
-int gf_cmd_log_init (const char *filename);
-
-void set_sys_log_level (gf_loglevel_t level);
 #endif /* __LOGGING_H__ */
diff --git a/libglusterfs/src/mem-pool.c b/libglusterfs/src/mem-pool.c
index f94723ba3..b901dd7a8 100644
--- a/libglusterfs/src/mem-pool.c
+++ b/libglusterfs/src/mem-pool.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "mem-pool.h"
@@ -39,37 +30,18 @@
 
 #define GLUSTERFS_ENV_MEM_ACCT_STR  "GLUSTERFS_DISABLE_MEM_ACCT"
 
-static int gf_mem_acct_enable = 0;
-
-int
-gf_mem_acct_is_enabled ()
-{
-        return gf_mem_acct_enable;
-}
-
 void
-gf_mem_acct_enable_set ()
+gf_mem_acct_enable_set (void *data)
 {
-        char    *opt = NULL;
-        long    val = -1;
-
-#ifdef DEBUG
-        gf_mem_acct_enable = 1;
-        return;
-#endif
-
-        opt = getenv (GLUSTERFS_ENV_MEM_ACCT_STR);
+        glusterfs_ctx_t *ctx = NULL;
 
-        if (!opt)
-                return;
+        ctx = data;
 
-        val = strtol (opt, NULL, 0);
+        GF_ASSERT (ctx);
 
-        if (val)
-                gf_mem_acct_enable = 0;
-        else
-                gf_mem_acct_enable = 1;
+        ctx->mem_acct_enable = 1;
 
+        return;
 }
 
 void
@@ -84,17 +56,11 @@ gf_mem_set_acct_info (xlator_t *xl, char **alloc_ptr,
 
         ptr = (char *) (*alloc_ptr);
 
-        if (!xl) {
-                GF_ASSERT (0);
-        }
+        GF_ASSERT (xl != NULL);
 
-        if (!(xl->mem_acct.rec)) {
-                GF_ASSERT (0);
-        }
+        GF_ASSERT (xl->mem_acct.rec != NULL);
 
-        if (type > xl->mem_acct.num_types) {
-                GF_ASSERT (0);
-        }
+        GF_ASSERT (type <= xl->mem_acct.num_types);
 
         LOCK(&xl->mem_acct.rec[type].lock);
         {
@@ -134,7 +100,7 @@ __gf_calloc (size_t nmemb, size_t size, uint32_t type)
         char            *ptr = NULL;
         xlator_t        *xl = NULL;
 
-        if (!gf_mem_acct_enable)
+        if (!THIS->ctx->mem_acct_enable)
                 return CALLOC (nmemb, size);
 
         xl = THIS;
@@ -160,7 +126,7 @@ __gf_malloc (size_t size, uint32_t type)
         char            *ptr = NULL;
         xlator_t        *xl = NULL;
 
-        if (!gf_mem_acct_enable)
+        if (!THIS->ctx->mem_acct_enable)
                 return MALLOC (size);
 
         xl = THIS;
@@ -185,7 +151,7 @@ __gf_realloc (void *ptr, size_t size)
         xlator_t        *xl = NULL;
         uint32_t        type = 0;
 
-        if (!gf_mem_acct_enable)
+        if (!THIS->ctx->mem_acct_enable)
                 return REALLOC (ptr, size);
 
         tot_size = size + GF_MEM_HEADER_SIZE + GF_MEM_TRAILER_SIZE;
@@ -258,7 +224,7 @@ __gf_free (void *free_ptr)
         uint32_t        type = 0;
         xlator_t        *xl = NULL;
 
-        if (!gf_mem_acct_enable) {
+        if (!THIS->ctx->mem_acct_enable) {
                 FREE (free_ptr);
                 return;
         }
@@ -268,20 +234,16 @@ __gf_free (void *free_ptr)
 
         ptr = (char *)free_ptr - 8 - 4;
 
-        if (GF_MEM_HEADER_MAGIC != *(uint32_t *)ptr) {
-                //Possible corruption, assert here
-                GF_ASSERT (0);
-        }
+        //Possible corruption, assert here
+        GF_ASSERT (GF_MEM_HEADER_MAGIC == *(uint32_t *)ptr);
 
         *(uint32_t *)ptr = 0;
 
         ptr = ptr - sizeof(xlator_t *);
         memcpy (&xl, ptr, sizeof(xlator_t *));
 
-        if (!xl) {
-                //gf_free expects xl to be available
-                GF_ASSERT (0);
-        }
+        //gf_free expects xl to be available
+        GF_ASSERT (xl != NULL);
 
         if (!xl->mem_acct.rec) {
                 ptr = (char *)free_ptr - GF_MEM_HEADER_SIZE;
@@ -294,11 +256,10 @@ __gf_free (void *free_ptr)
         ptr = ptr - 4;
         type = *(uint32_t *)ptr;
 
-        if (GF_MEM_TRAILER_MAGIC != *(uint32_t *)
-            ((char *)free_ptr + req_size)) {
-                // This points to a memory overrun
-                GF_ASSERT (0);
-        }
+        // This points to a memory overrun
+        GF_ASSERT (GF_MEM_TRAILER_MAGIC ==
+                *(uint32_t *)((char *)free_ptr + req_size));
+
         *(uint32_t *) ((char *)free_ptr + req_size) = 0;
 
         LOCK (&xl->mem_acct.rec[type].lock);
@@ -326,7 +287,7 @@ mem_pool_new_fn (unsigned long sizeof_type,
         glusterfs_ctx_t  *ctx = NULL;
 
         if (!sizeof_type || !count) {
-                gf_log ("mem-pool", GF_LOG_ERROR, "invalid argument");
+                gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                 return NULL;
         }
         padded_sizeof_type = sizeof_type + GF_MEM_POOL_PAD_BOUNDARY;
@@ -369,7 +330,7 @@ mem_pool_new_fn (unsigned long sizeof_type,
         mem_pool->pool_end = pool + (count * (padded_sizeof_type));
 
         /* add this pool to the global list */
-        ctx = glusterfs_ctx_get ();
+        ctx = THIS->ctx;
         if (!ctx)
                 goto out;
 
@@ -385,7 +346,7 @@ mem_get0 (struct mem_pool *mem_pool)
         void             *ptr = NULL;
 
         if (!mem_pool) {
-                gf_log ("mem-pool", GF_LOG_ERROR, "invalid argument");
+                gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                 return NULL;
         }
 
@@ -406,7 +367,7 @@ mem_get (struct mem_pool *mem_pool)
         struct mem_pool **pool_ptr = NULL;
 
         if (!mem_pool) {
-                gf_log ("mem-pool", GF_LOG_ERROR, "invalid argument");
+                gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                 return NULL;
         }
 
@@ -451,10 +412,12 @@ mem_get (struct mem_pool *mem_pool)
                  * because it is too much work knowing that a better slab
                  * allocator is coming RSN.
                  */
+                mem_pool->pool_misses++;
+                mem_pool->curr_stdalloc++;
+                if (mem_pool->max_stdalloc < mem_pool->curr_stdalloc)
+                        mem_pool->max_stdalloc = mem_pool->curr_stdalloc;
                 ptr = GF_CALLOC (1, mem_pool->padded_sizeof_type,
                                  gf_common_mt_mem_pool);
-                gf_log_callingfn ("mem-pool", GF_LOG_DEBUG, "Mem pool is full. "
-                                  "Callocing mem");
 
                 /* Memory coming from the heap need not be transformed from a
                  * chunkhead to a usable pointer since it is not coming from
@@ -475,7 +438,7 @@ static int
 __is_member (struct mem_pool *pool, void *ptr)
 {
         if (!pool || !ptr) {
-                gf_log ("mem-pool", GF_LOG_ERROR, "invalid argument");
+                gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                 return -1;
         }
 
@@ -500,20 +463,22 @@ mem_put (void *ptr)
         struct mem_pool *pool = NULL;
 
         if (!ptr) {
-                gf_log ("mem-pool", GF_LOG_ERROR, "invalid argument");
+                gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                 return;
         }
 
         list = head = mem_pool_ptr2chunkhead (ptr);
         tmp = mem_pool_from_ptr (head);
         if (!tmp) {
-                gf_log ("mem-pool", GF_LOG_ERROR, "ptr header is corrupted");
+                gf_log_callingfn ("mem-pool", GF_LOG_ERROR,
+                                  "ptr header is corrupted");
                 return;
         }
 
         pool = *tmp;
         if (!pool) {
-                gf_log ("mem-pool", GF_LOG_ERROR, "mem-pool ptr is NULL");
+                gf_log_callingfn ("mem-pool", GF_LOG_ERROR,
+                                  "mem-pool ptr is NULL");
                 return;
         }
         LOCK (&pool->lock);
@@ -553,6 +518,7 @@ mem_put (void *ptr)
                          * not have enough info to distinguish between the two
                          * situations.
                          */
+                        pool->curr_stdalloc--;
                         GF_FREE (list);
                         break;
                 default:
diff --git a/libglusterfs/src/mem-pool.h b/libglusterfs/src/mem-pool.h
index a23b12202..31f49f75c 100644
--- a/libglusterfs/src/mem-pool.h
+++ b/libglusterfs/src/mem-pool.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _MEM_POOL_H_
@@ -116,7 +107,26 @@ void* __gf_default_realloc (void *oldptr, size_t size)
 
 #define GF_REALLOC(ptr, size)  __gf_realloc (ptr, size)
 
-#define GF_FREE(free_ptr) __gf_free (free_ptr);
+#define GF_FREE(free_ptr) __gf_free (free_ptr)
+
+static inline
+char *gf_strndup (const char *src, size_t len)
+{
+        char *dup_str = NULL;
+
+        if (!src) {
+                goto out;
+        }
+
+        dup_str = GF_CALLOC (1, len + 1, gf_common_mt_strdup);
+        if (!dup_str) {
+                goto out;
+        }
+
+        memcpy (dup_str, src, len);
+out:
+        return dup_str;
+}
 
 static inline
 char * gf_strdup (const char *src)
@@ -137,6 +147,21 @@ char * gf_strdup (const char *src)
         return dup_str;
 }
 
+static inline void *
+gf_memdup (const void *src, size_t size)
+{
+        void *dup_mem = NULL;
+
+        dup_mem = GF_CALLOC(1, size, gf_common_mt_strdup);
+        if (!dup_mem)
+                goto out;
+
+        memcpy (dup_mem, src, size);
+
+out:
+        return dup_mem;
+}
+
 struct mem_pool {
         struct list_head  list;
         int               hot_count;
@@ -147,7 +172,10 @@ struct mem_pool {
         void             *pool_end;
         int               real_sizeof_type;
         uint64_t          alloc_count;
+        uint64_t          pool_misses;
         int               max_alloc;
+        int               curr_stdalloc;
+        int               max_stdalloc;
         char             *name;
         struct list_head  global_list;
 };
@@ -163,7 +191,6 @@ void *mem_get0 (struct mem_pool *pool);
 
 void mem_pool_destroy (struct mem_pool *pool);
 
-int gf_mem_acct_is_enabled ();
-void gf_mem_acct_enable_set ();
+void gf_mem_acct_enable_set (void *ctx);
 
 #endif /* _MEM_POOL_H */
diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h
index b8c61d689..666bd120a 100644
--- a/libglusterfs/src/mem-types.h
+++ b/libglusterfs/src/mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __MEM_TYPES_H__
@@ -106,6 +97,29 @@ enum gf_common_mem_types_ {
         gf_common_mt_trie_end             = 81,
         gf_common_mt_run_argv             = 82,
         gf_common_mt_run_logbuf           = 83,
-        gf_common_mt_end                  = 84
+        gf_common_mt_fd_lk_ctx_t          = 84,
+        gf_common_mt_fd_lk_ctx_node_t     = 85,
+        gf_common_mt_buffer_t             = 86,
+        gf_common_mt_circular_buffer_t    = 87,
+        gf_common_mt_eh_t                 = 88,
+        gf_common_mt_store_handle_t       = 89,
+        gf_common_mt_store_iter_t         = 90,
+        gf_common_mt_drc_client_t         = 91,
+        gf_common_mt_drc_globals_t        = 92,
+        gf_common_mt_drc_rbtree_node_t    = 93,
+        gf_common_mt_iov_base_t           = 94,
+        gf_common_mt_groups_t             = 95,
+        gf_common_mt_cliententry_t        = 96,
+        gf_common_mt_clienttable_t        = 97,
+        gf_common_mt_client_t             = 98,
+        gf_common_mt_client_ctx           = 99,
+        gf_common_mt_lock_table           = 100,
+        gf_common_mt_locker               = 101,
+        gf_common_mt_auxgids              = 102,
+        gf_common_mt_syncopctx            = 103,
+        gf_common_mt_uuid_t               = 104,
+        gf_common_mt_mgmt_v3_lock_obj_t   = 105,
+        gf_common_mt_txn_opinfo_obj_t     = 106,
+        gf_common_mt_end                  = 107
 };
 #endif
diff --git a/libglusterfs/src/options.c b/libglusterfs/src/options.c
index b8d3b6ae5..842b6413a 100644
--- a/libglusterfs/src/options.c
+++ b/libglusterfs/src/options.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -62,7 +53,6 @@ out:
         return ret;
 }
 
-
 static int
 xlator_option_validate_int (xlator_t *xl, const char *key, const char *value,
                             volume_option_t *opt, char **op_errstr)
@@ -80,18 +70,37 @@ xlator_option_validate_int (xlator_t *xl, const char *key, const char *value,
                 goto out;
         }
 
-        if ((opt->min == 0) && (opt->max == 0)) {
-                gf_log (xl->name, GF_LOG_DEBUG,
+        if ((opt->min == 0) && (opt->max == 0) &&
+            (opt->validate == GF_OPT_VALIDATE_BOTH)) {
+                gf_log (xl->name, GF_LOG_TRACE,
                         "no range check required for 'option %s %s'",
                         key, value);
                 ret = 0;
                 goto out;
         }
 
-        if ((inputll < opt->min) || (inputll > opt->max)) {
+        if ((opt->validate == GF_OPT_VALIDATE_MIN)) {
+                if (inputll < opt->min) {
+                        snprintf (errstr, 256,
+                                  "'%lld' in 'option %s %s' is smaller than "
+                                  "minimum value '%.0f'", inputll, key,
+                                  value, opt->min);
+                        gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+        } else if ((opt->validate == GF_OPT_VALIDATE_MAX)) {
+                if ((inputll > opt->max)) {
+                        snprintf (errstr, 256,
+                                  "'%lld' in 'option %s %s' is greater than "
+                                  "maximum value '%.0f'", inputll, key,
+                                  value, opt->max);
+                        gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+        } else if ((inputll < opt->min) || (inputll > opt->max)) {
                 snprintf (errstr, 256,
                           "'%lld' in 'option %s %s' is out of range "
-                          "[%"PRId64" - %"PRId64"]",
+                          "[%.0f - %.0f]",
                           inputll, key, value, opt->min, opt->max);
                 gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
                 goto out;
@@ -110,7 +119,7 @@ xlator_option_validate_sizet (xlator_t *xl, const char *key, const char *value,
                               volume_option_t *opt, char **op_errstr)
 {
         uint64_t  size = 0;
-        int       ret = -1;
+        int       ret = 0;
         char      errstr[256];
 
         /* Check the range */
@@ -119,37 +128,34 @@ xlator_option_validate_sizet (xlator_t *xl, const char *key, const char *value,
                           "invalid number format \"%s\" in option \"%s\"",
                           value, key);
                 gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
+                ret = -1;
                 goto out;
         }
 
         if ((opt->min == 0) && (opt->max == 0)) {
-                gf_log (xl->name, GF_LOG_DEBUG,
+                gf_log (xl->name, GF_LOG_TRACE,
                         "no range check required for 'option %s %s'",
                         key, value);
-                ret = 0;
                 goto out;
         }
 
         if ((size < opt->min) || (size > opt->max)) {
-                if (strncmp (key, "cache-size", 10) == 0) {
+                if ((strncmp (key, "cache-size", 10) == 0) &&
+                    (size > opt->max)) {
                        snprintf (errstr, 256, "Cache size %"PRId64" is out of "
-                                 "range [%"PRId64" - %"PRId64"]",
+                                 "range [%.0f - %.0f]",
                                  size, opt->min, opt->max);
-                       //*op_errstr = gf_strdup (errstr);
                        gf_log (xl->name, GF_LOG_WARNING, "%s", errstr);
-                       ret = 0;
-                       goto out;
                 } else {
                         snprintf (errstr, 256,
                                   "'%"PRId64"' in 'option %s %s' "
-                                  "is out of range [%"PRId64" - %"PRId64"]",
+                                  "is out of range [%.0f - %.0f]",
                                   size, key, value, opt->min, opt->max);
                         gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
-                        goto out;
+                        ret = -1;
                 }
         }
 
-        ret = 0;
 out:
         if (ret && op_errstr)
                 *op_errstr = gf_strdup (errstr);
@@ -222,6 +228,46 @@ out:
         return ret;
 }
 
+void
+set_error_str (char *errstr, size_t len, volume_option_t *opt, const char *key,
+               const char *value)
+{
+        int i = 0;
+        char given_array[4096] = {0,};
+
+        for (i = 0; (i < ZR_OPTION_MAX_ARRAY_SIZE) && opt->value[i];) {
+                strcat (given_array, opt->value[i]);
+                if (((++i) < ZR_OPTION_MAX_ARRAY_SIZE) &&
+                    (opt->value[i]))
+                        strcat (given_array, ", ");
+                else
+                        strcat (given_array, ".");
+        }
+        snprintf (errstr, len, "option %s %s: '%s' is not valid "
+                  "(possible options are %s)", key, value, value, given_array);
+        return;
+}
+
+int
+is_all_whitespaces (const char *value)
+{
+        int i = 0;
+        size_t len = 0;
+
+        if (value == NULL)
+                return -1;
+
+        len = strlen (value);
+
+        for (i = 0; i < len; i++) {
+                if (value[i] == ' ')
+                        continue;
+                else
+                        return 0;
+        }
+
+        return 1;
+}
 
 static int
 xlator_option_validate_str (xlator_t *xl, const char *key, const char *value,
@@ -229,8 +275,7 @@ xlator_option_validate_str (xlator_t *xl, const char *key, const char *value,
 {
         int          ret = -1;
         int          i = 0;
-        char         errstr[256];
-        char         given_array[4096] = {0,};
+        char         errstr[4096] = {0,};
 
         /* Check if the '*str' is valid */
         if (GF_OPTION_LIST_EMPTY(opt)) {
@@ -238,6 +283,9 @@ xlator_option_validate_str (xlator_t *xl, const char *key, const char *value,
                 goto out;
         }
 
+        if (is_all_whitespaces (value) == 1)
+                goto out;
+
         for (i = 0; (i < ZR_OPTION_MAX_ARRAY_SIZE) && opt->value[i]; i++) {
  #ifdef  GF_DARWIN_HOST_OS
                 if (fnmatch (opt->value[i], value, 0) == 0) {
@@ -252,7 +300,8 @@ xlator_option_validate_str (xlator_t *xl, const char *key, const char *value,
  #endif
         }
 
-        if ((i <= ZR_OPTION_MAX_ARRAY_SIZE) && (!opt->value[i])) {
+        if ((i == ZR_OPTION_MAX_ARRAY_SIZE) || (!opt->value[i]))
+                goto out;
                 /* enter here only if
                  * 1. reached end of opt->value array and haven't
                  *    validated input
@@ -262,26 +311,15 @@ xlator_option_validate_str (xlator_t *xl, const char *key, const char *value,
                  *    matched all possible input values.
                  */
 
-                for (i = 0; (i < ZR_OPTION_MAX_ARRAY_SIZE) && opt->value[i];) {
-                        strcat (given_array, opt->value[i]);
-                        if (((++i) < ZR_OPTION_MAX_ARRAY_SIZE) &&
-                           (opt->value[i]))
-                                strcat (given_array, ", ");
-                        else
-                                strcat (given_array, ".");
-                }
-                snprintf (errstr, 256,
-                          "option %s %s: '%s' is not valid "
-                          "(possible options are %s)",
-                          key, value, value, given_array);
-                gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
-                goto out;
-        }
-
         ret = 0;
+
 out:
-        if (ret && op_errstr)
-                *op_errstr = gf_strdup (errstr);
+        if (ret) {
+                set_error_str (errstr, sizeof (errstr), opt, key, value);
+                gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
+                if (op_errstr)
+                        *op_errstr = gf_strdup (errstr);
+        }
         return ret;
 }
 
@@ -290,10 +328,9 @@ static int
 xlator_option_validate_percent (xlator_t *xl, const char *key, const char *value,
                                 volume_option_t *opt, char **op_errstr)
 {
-        int          ret = -1;
-        char         errstr[256];
-        uint32_t     percent = 0;
-
+        double    percent = 0;
+        int       ret = -1;
+        char      errstr[256];
 
         /* Check if the value is valid percentage */
         if (gf_string2percent (value, &percent) != 0) {
@@ -304,9 +341,9 @@ xlator_option_validate_percent (xlator_t *xl, const char *key, const char *value
                 goto out;
         }
 
-        if ((percent < 0) || (percent > 100)) {
+        if ((percent < 0.0) || (percent > 100.0)) {
                 snprintf (errstr, 256,
-                          "'%d' in 'option %s %s' is out of range [0 - 100]",
+                          "'%lf' in 'option %s %s' is out of range [0 - 100]",
                           percent, key, value);
                 gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
                 goto out;
@@ -319,8 +356,6 @@ out:
         return ret;
 }
 
-
-/* TODO: clean this up */
 static int
 xlator_option_validate_percent_or_sizet (xlator_t *xl, const char *key,
                                          const char *value,
@@ -328,88 +363,43 @@ xlator_option_validate_percent_or_sizet (xlator_t *xl, const char *key,
 {
         int          ret = -1;
         char         errstr[256];
-        uint32_t     percent = 0;
         uint64_t     size = 0;
+	gf_boolean_t is_percent = _gf_false;
+
+	if (gf_string2percent_or_bytesize (value, &size, &is_percent) == 0) {
+		if (is_percent) {
+			ret = 0;
+			goto out;
+		}
+		/* Check the range */
+		if ((opt->min == 0) && (opt->max == 0)) {
+			gf_log (xl->name, GF_LOG_TRACE,
+				"no range check required for "
+				"'option %s %s'",
+				key, value);
+			ret = 0;
+			goto out;
+		}
+		if ((size < opt->min) || (size > opt->max)) {
+			snprintf (errstr, 256,
+				  "'%"PRId64"' in 'option %s %s'"
+				  " is out of range [%.0f - %.0f]",
+				  size, key, value, opt->min, opt->max);
+			gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
+			goto out;
+		}
+		ret = 0;
+		goto out;
+	}
+
+	/* If control reaches here, invalid argument */
+
+	snprintf (errstr, 256,
+		  "invalid number format \"%s\" in \"option %s\"",
+		  value, key);
+	gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
 
-        /* Check if the value is valid percentage */
-        if (gf_string2percent (value, &percent) == 0) {
-                if (percent > 100) {
-                        gf_log (xl->name, GF_LOG_DEBUG,
-                                "value given was greater than 100, "
-                                "assuming this is actually a size");
-                        if (gf_string2bytesize (value, &size) == 0) {
-                                        /* Check the range */
-                                if ((opt->min == 0) && (opt->max == 0)) {
-                                        gf_log (xl->name, GF_LOG_DEBUG,
-                                                "no range check rquired for "
-                                                "'option %s %s'",
-                                                key, value);
-                                                // It is a size
-                                                ret = 0;
-                                                goto out;
-                                }
-                                if ((size < opt->min) || (size > opt->max)) {
-                                        snprintf (errstr, 256,
-                                                  "'%"PRId64"' in 'option %s %s' "
-                                                  "is out of range [%"PRId64" - "
-                                                  "%"PRId64"]",
-                                                  size, key, value,
-                                                  opt->min, opt->max);
-                                        gf_log (xl->name, GF_LOG_ERROR, "%s",
-                                                errstr);
-                                        goto out;
-                                }
-                                // It is a size
-                                ret = 0;
-                                goto out;
-                        } else {
-                                // It's not a percent or size
-                                snprintf (errstr, 256,
-                                          "invalid number format \"%s\" "
-                                          "in \"option %s\"",
-                                          value, key);
-                                gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
-                                goto out;
-                        }
-                }
-                // It is a percent
-                ret = 0;
-                goto out;
-        } else {
-                if (gf_string2bytesize (value, &size) == 0) {
-                        /* Check the range */
-                        if ((opt->min == 0) && (opt->max == 0)) {
-                                gf_log (xl->name, GF_LOG_DEBUG,
-                                        "no range check required for "
-                                        "'option %s %s'",
-                                        key, value);
-                                // It is a size
-                                ret = 0;
-                                goto out;
-                        }
-                        if ((size < opt->min) || (size > opt->max)) {
-                                snprintf (errstr, 256,
-                                          "'%"PRId64"' in 'option %s %s'"
-                                          " is out of range [%"PRId64" -"
-                                          " %"PRId64"]",
-                                          size, key, value, opt->min, opt->max);
-                                gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
-                                goto out;
-                        }
-                } else {
-                        // It's not a percent or size
-                        snprintf (errstr, 256,
-                                  "invalid number format \"%s\" in \"option %s\"",
-                                  value, key);
-                        gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
-                        goto out;
-                }
-                //It is a size
-                ret = 0;
-                goto out;
-        }
 
-        ret = 0;
 out:
         if (ret && op_errstr)
                 *op_errstr = gf_strdup (errstr);
@@ -425,7 +415,7 @@ xlator_option_validate_time (xlator_t *xl, const char *key, const char *value,
         char         errstr[256];
         uint32_t     input_time = 0;
 
-                /* Check if the value is valid percentage */
+	/* Check if the value is valid time */
         if (gf_string2time (value, &input_time) != 0) {
                 snprintf (errstr, 256,
                           "invalid time format \"%s\" in "
@@ -436,7 +426,7 @@ xlator_option_validate_time (xlator_t *xl, const char *key, const char *value,
         }
 
         if ((opt->min == 0) && (opt->max == 0)) {
-                gf_log (xl->name, GF_LOG_DEBUG,
+                gf_log (xl->name, GF_LOG_TRACE,
                         "no range check required for "
                         "'option %s %s'",
                         key, value);
@@ -447,7 +437,7 @@ xlator_option_validate_time (xlator_t *xl, const char *key, const char *value,
         if ((input_time < opt->min) || (input_time > opt->max)) {
                 snprintf (errstr, 256,
                           "'%"PRIu32"' in 'option %s %s' is "
-                          "out of range [%"PRId64" - %"PRId64"]",
+                          "out of range [%.0f - %.0f]",
                           input_time, key, value,
                           opt->min, opt->max);
                 gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
@@ -466,35 +456,55 @@ static int
 xlator_option_validate_double (xlator_t *xl, const char *key, const char *value,
                                volume_option_t *opt, char **op_errstr)
 {
-        int          ret = -1;
-        char         errstr[256];
-        double       val = 0.0;
-
-        /* Check if the value is valid double */
-        if (gf_string2double (value, &val) != 0) {
-                snprintf (errstr, 256,
-                          "invalid double \"%s\" in \"option %s\"",
-                          value, key);
-                gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
-                goto out;
-        }
+        double    input = 0.0;
+        int       ret = -1;
+        char      errstr[256];
 
-        if (val < 0.0) {
+        /* Check the range */
+        if (gf_string2double (value, &input) != 0) {
                 snprintf (errstr, 256,
-                          "invalid double \"%s\" in \"option %s\"",
+                          "invalid number format \"%s\" in option \"%s\"",
                           value, key);
                 gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
                 goto out;
         }
 
-        if ((opt->min == 0) && (opt->max == 0)) {
-                gf_log (xl->name, GF_LOG_DEBUG,
+        if ((opt->min == 0) && (opt->max == 0) &&
+            (opt->validate == GF_OPT_VALIDATE_BOTH)) {
+                gf_log (xl->name, GF_LOG_TRACE,
                         "no range check required for 'option %s %s'",
                         key, value);
                 ret = 0;
                 goto out;
         }
 
+        if ((opt->validate == GF_OPT_VALIDATE_MIN)) {
+                if (input < opt->min) {
+                        snprintf (errstr, 256,
+                                  "'%f' in 'option %s %s' is smaller than "
+                                  "minimum value '%f'", input, key,
+                                  value, opt->min);
+                        gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+        } else if ((opt->validate == GF_OPT_VALIDATE_MAX)) {
+                if ((input > opt->max)) {
+                        snprintf (errstr, 256,
+                                  "'%f' in 'option %s %s' is greater than "
+                                  "maximum value '%f'", input, key,
+                                  value, opt->max);
+                        gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+        } else if ((input < opt->min) || (input > opt->max)) {
+                snprintf (errstr, 256,
+                          "'%f' in 'option %s %s' is out of range "
+                          "[%f - %f]",
+                          input, key, value, opt->min, opt->max);
+                gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
+                goto out;
+        }
+
         ret = 0;
 out:
         if (ret && op_errstr)
@@ -510,10 +520,11 @@ xlator_option_validate_addr (xlator_t *xl, const char *key, const char *value,
         int          ret = -1;
         char         errstr[256];
 
-        if (!valid_internet_address ((char *)value)) {
+        if (!valid_internet_address ((char *)value, _gf_false)) {
                 snprintf (errstr, 256,
-                          "internet address '%s' does not conform to standards.",
-                          value);
+                          "option %s %s: '%s'  is not a valid internet-address,"
+                          " it does not conform to standards.",
+                          key, value, value);
                 gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
                 if (op_errstr)
                         *op_errstr = gf_strdup (errstr);
@@ -524,6 +535,178 @@ xlator_option_validate_addr (xlator_t *xl, const char *key, const char *value,
         return ret;
 }
 
+static int
+xlator_option_validate_addr_list (xlator_t *xl, const char *key,
+                                  const char *value, volume_option_t *opt,
+                                  char **op_errstr)
+{
+        int          ret = -1;
+        char         *dup_val = NULL;
+        char         *addr_tok = NULL;
+        char         *save_ptr = NULL;
+        char         errstr[4096] = {0,};
+
+        dup_val = gf_strdup (value);
+        if (!dup_val)
+                goto out;
+
+        addr_tok = strtok_r (dup_val, ",", &save_ptr);
+        if (addr_tok == NULL)
+                goto out;
+        while (addr_tok) {
+                if (!valid_internet_address (addr_tok, _gf_true))
+                        goto out;
+
+                addr_tok = strtok_r (NULL, ",", &save_ptr);
+        }
+        ret = 0;
+
+out:
+        if (ret) {
+                snprintf (errstr, sizeof (errstr), "option %s %s: '%s' is not "
+                "a valid internet-address-list", key, value, value);
+                gf_log (xl->name, GF_LOG_ERROR, "%s", errstr);
+                if (op_errstr)
+                        *op_errstr = gf_strdup (errstr);
+        }
+        GF_FREE (dup_val);
+
+        return ret;
+}
+
+/*XXX: the rules to validate are as per block-size required for stripe xlator */
+static int
+gf_validate_size (const char *sizestr, volume_option_t *opt)
+{
+        uint64_t                value = 0;
+        int                     ret = 0;
+
+        GF_ASSERT (opt);
+
+        if (gf_string2bytesize (sizestr, &value) != 0 ||
+            value < opt->min ||
+            value % 512) {
+                ret = -1;
+                goto out;
+        }
+
+ out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+gf_validate_number (const char *numstr, volume_option_t *opt)
+{
+        int32_t value;
+        return gf_string2int32 (numstr, &value);
+}
+
+/*  Parses the string to be of the form <key1>:<value1>,<key2>:<value2>...  *
+ *  takes two optional validaters key_validator and value_validator         */
+static int
+validate_list_elements (const char *string, volume_option_t *opt,
+                        int (key_validator)( const char *),
+                        int (value_validator)( const char *, volume_option_t *))
+{
+
+        char                    *dup_string = NULL;
+        char                    *str_sav = NULL;
+        char                    *substr_sav = NULL;
+        char                    *str_ptr = NULL;
+        char                    *key = NULL;
+        char                    *value = NULL;
+        int                     ret = 0;
+
+        GF_ASSERT (string);
+
+        dup_string = gf_strdup (string);
+        if (NULL == dup_string)
+                goto out;
+
+        str_ptr = strtok_r (dup_string, ",", &str_sav);
+        if (str_ptr == NULL) {
+                ret = -1;
+                goto out;
+        }
+        while (str_ptr) {
+
+                key = strtok_r (str_ptr, ":", &substr_sav);
+                if (!key ||
+                    (key_validator && key_validator(key))) {
+                        ret = -1;
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "invalid list '%s', key '%s' not valid.",
+                                string, key);
+                        goto out;
+                }
+
+                value = strtok_r (NULL, ":", &substr_sav);
+                if (!value ||
+                    (value_validator && value_validator(value, opt))) {
+                        ret = -1;
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "invalid list '%s', value '%s' not valid.",
+                                string, key);
+                        goto out;
+                }
+
+                str_ptr = strtok_r (NULL, ",", &str_sav);
+                substr_sav = NULL;
+        }
+
+ out:
+        GF_FREE (dup_string);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+xlator_option_validate_priority_list (xlator_t *xl, const char *key,
+                                      const char *value, volume_option_t *opt,
+                                      char **op_errstr)
+{
+        int                     ret =0;
+        char                    errstr[1024] = {0, };
+
+        GF_ASSERT (value);
+
+        ret = validate_list_elements (value, opt, NULL, &gf_validate_number);
+        if (ret) {
+                snprintf (errstr, 1024,
+                          "option %s %s: '%s' is not a valid "
+                          "priority-list", key, value, value);
+                *op_errstr = gf_strdup (errstr);
+        }
+
+        return ret;
+}
+
+static int
+xlator_option_validate_size_list (xlator_t *xl, const char *key,
+                                  const char *value, volume_option_t *opt,
+                                  char **op_errstr)
+{
+
+        int                    ret = 0;
+        char                   errstr[1024] = {0, };
+
+        GF_ASSERT (value);
+
+        ret = gf_validate_size (value, opt);
+        if (ret)
+                ret = validate_list_elements (value, opt, NULL, &gf_validate_size);
+
+        if (ret) {
+                snprintf (errstr, 1024,
+                          "option %s %s: '%s' is not a valid "
+                          "size-list", key, value, value);
+                *op_errstr = gf_strdup (errstr);
+        }
+
+        return ret;
+
+}
 
 static int
 xlator_option_validate_any (xlator_t *xl, const char *key, const char *value,
@@ -532,7 +715,6 @@ xlator_option_validate_any (xlator_t *xl, const char *key, const char *value,
         return 0;
 }
 
-
 typedef int (xlator_option_validator_t) (xlator_t *xl, const char *key,
                                          const char *value,
                                          volume_option_t *opt, char **operrstr);
@@ -556,6 +738,11 @@ xlator_option_validate (xlator_t *xl, char *key, char *value,
                 [GF_OPTION_TYPE_TIME]        = xlator_option_validate_time,
                 [GF_OPTION_TYPE_DOUBLE]      = xlator_option_validate_double,
                 [GF_OPTION_TYPE_INTERNET_ADDRESS] = xlator_option_validate_addr,
+                [GF_OPTION_TYPE_INTERNET_ADDRESS_LIST] =
+                xlator_option_validate_addr_list,
+                [GF_OPTION_TYPE_PRIORITY_LIST] =
+                xlator_option_validate_priority_list,
+                [GF_OPTION_TYPE_SIZE_LIST]   = xlator_option_validate_size_list,
                 [GF_OPTION_TYPE_ANY]         = xlator_option_validate_any,
                 [GF_OPTION_TYPE_MAX]         = NULL,
         };
@@ -574,7 +761,7 @@ out:
 }
 
 
-static volume_option_t *
+volume_option_t *
 xlator_volume_option_get_list (volume_opt_list_t *vol_list, const char *key)
 {
         volume_option_t         *opt = NULL;
@@ -591,7 +778,7 @@ xlator_volume_option_get_list (volume_opt_list_t *vol_list, const char *key)
         } else
                 opt = vol_list->given_opt;
 
-        for (index = 0; opt[index].key && opt[index].key[0]; index++) {
+        for (index = 0; opt[index].key[0]; index++) {
                 for (i = 0; i < ZR_VOLUME_MAX_NUM_KEY; i++) {
                         cmp_key = opt[index].key[i];
                         if (!cmp_key)
@@ -623,7 +810,7 @@ xlator_volume_option_get (xlator_t *xl, const char *key)
 }
 
 
-static void
+static int
 xl_opt_validate (dict_t *dict, char *key, data_t *value, void *data)
 {
         xlator_t          *xl = NULL;
@@ -644,7 +831,7 @@ xl_opt_validate (dict_t *dict, char *key, data_t *value, void *data)
 
         opt = xlator_volume_option_get_list (vol_opt, key);
         if (!opt)
-                return;
+                return 0;
 
         ret = xlator_option_validate (xl, key, value->data, opt, &errstr);
         if (ret)
@@ -662,7 +849,7 @@ xl_opt_validate (dict_t *dict, char *key, data_t *value, void *data)
                 dict_set (dict, opt->key[0], value);
                 dict_del (dict, key);
         }
-        return;
+        return 0;
 }
 
 
@@ -867,11 +1054,8 @@ out:
 
 
 static int
-not_null (char *in, char **out)
+pass (char *in, char **out)
 {
-        if (!in || !out)
-                return -1;
-
         *out = in;
         return 0;
 }
@@ -892,46 +1076,52 @@ xl_by_name (char *in, xlator_t **out)
 
 
 static int
-pc_or_size (char *in, uint64_t *out)
+pc_or_size (char *in, double *out)
 {
-        uint32_t  pc = 0;
+        double  pc = 0;
         int       ret = 0;
+        uint64_t  size = 0;
 
         if (gf_string2percent (in, &pc) == 0) {
-                if (pc > 100) {
-                        ret = gf_string2bytesize (in, out);
+                if (pc > 100.0) {
+                        ret = gf_string2bytesize (in, &size);
+                        if (!ret)
+                                *out = size;
                 } else {
                         *out = pc;
                 }
         } else {
-                ret = gf_string2bytesize (in, out);
+                ret = gf_string2bytesize (in, &size);
+                if (!ret)
+                        *out = size;
         }
         return ret;
 }
 
-
-DEFINE_INIT_OPT(char *, str, not_null);
+DEFINE_INIT_OPT(char *, str, pass);
 DEFINE_INIT_OPT(uint64_t, uint64, gf_string2uint64);
 DEFINE_INIT_OPT(int64_t, int64, gf_string2int64);
 DEFINE_INIT_OPT(uint32_t, uint32, gf_string2uint32);
 DEFINE_INIT_OPT(int32_t, int32, gf_string2int32);
 DEFINE_INIT_OPT(uint64_t, size, gf_string2bytesize);
-DEFINE_INIT_OPT(uint32_t, percent, gf_string2percent);
-DEFINE_INIT_OPT(uint64_t, percent_or_size, pc_or_size);
+DEFINE_INIT_OPT(double, percent, gf_string2percent);
+DEFINE_INIT_OPT(double, percent_or_size, pc_or_size);
 DEFINE_INIT_OPT(gf_boolean_t, bool, gf_string2boolean);
 DEFINE_INIT_OPT(xlator_t *, xlator, xl_by_name);
-DEFINE_INIT_OPT(char *, path, not_null);
+DEFINE_INIT_OPT(char *, path, pass);
+DEFINE_INIT_OPT(double, double, gf_string2double);
 
 
 
-DEFINE_RECONF_OPT(char *, str, not_null);
+DEFINE_RECONF_OPT(char *, str, pass);
 DEFINE_RECONF_OPT(uint64_t, uint64, gf_string2uint64);
 DEFINE_RECONF_OPT(int64_t, int64, gf_string2int64);
 DEFINE_RECONF_OPT(uint32_t, uint32, gf_string2uint32);
 DEFINE_RECONF_OPT(int32_t, int32, gf_string2int32);
 DEFINE_RECONF_OPT(uint64_t, size, gf_string2bytesize);
-DEFINE_RECONF_OPT(uint32_t, percent, gf_string2percent);
-DEFINE_RECONF_OPT(uint64_t, percent_or_size, pc_or_size);
+DEFINE_RECONF_OPT(double, percent, gf_string2percent);
+DEFINE_RECONF_OPT(double, percent_or_size, pc_or_size);
 DEFINE_RECONF_OPT(gf_boolean_t, bool, gf_string2boolean);
 DEFINE_RECONF_OPT(xlator_t *, xlator, xl_by_name);
-DEFINE_RECONF_OPT(char *, path, not_null);
+DEFINE_RECONF_OPT(char *, path, pass);
+DEFINE_RECONF_OPT(double, double, gf_string2double);
diff --git a/libglusterfs/src/options.h b/libglusterfs/src/options.h
index 40089ac63..e2a25baa9 100644
--- a/libglusterfs/src/options.h
+++ b/libglusterfs/src/options.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _OPTIONS_H
@@ -44,25 +35,38 @@ typedef enum {
         GF_OPTION_TYPE_TIME,
         GF_OPTION_TYPE_DOUBLE,
         GF_OPTION_TYPE_INTERNET_ADDRESS,
+        GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+        GF_OPTION_TYPE_PRIORITY_LIST,
+        GF_OPTION_TYPE_SIZE_LIST,
         GF_OPTION_TYPE_MAX,
 } volume_option_type_t;
 
+typedef enum {
+        GF_OPT_VALIDATE_BOTH = 0,
+        GF_OPT_VALIDATE_MIN,
+        GF_OPT_VALIDATE_MAX,
+} opt_validate_type_t;
 
 #define ZR_VOLUME_MAX_NUM_KEY    4
 #define ZR_OPTION_MAX_ARRAY_SIZE 64
 
 /* Each translator should define this structure */
 typedef struct volume_options {
-        char                *key[ZR_VOLUME_MAX_NUM_KEY];
+        char                    *key[ZR_VOLUME_MAX_NUM_KEY];
         /* different key, same meaning */
-        volume_option_type_t type;
-        int64_t              min;  /* 0 means no range */
-        int64_t              max;  /* 0 means no range */
-        char                *value[ZR_OPTION_MAX_ARRAY_SIZE];
+        volume_option_type_t    type;
+        double                  min;  /* 0 means no range */
+        double                  max;  /* 0 means no range */
+        char                    *value[ZR_OPTION_MAX_ARRAY_SIZE];
         /* If specified, will check for one of
            the value from this array */
-        char                *default_value;
-        char                *description; /* about the key */
+        char                    *default_value;
+        char                    *description; /* about the key */
+        /* Required for int options where only the min value
+         * is given and is 0. This will cause validation not to
+         * happen
+         */
+        opt_validate_type_t     validate;
 } volume_option_t;
 
 
@@ -89,6 +93,9 @@ int xlator_options_validate (xlator_t *xl, dict_t *options, char **errstr);
 volume_option_t *
 xlator_volume_option_get (xlator_t *xl, const char *key);
 
+volume_option_t *
+xlator_volume_option_get_list (volume_opt_list_t *vol_list, const char *key);
+
 
 #define DECLARE_INIT_OPT(type_t, type)                                  \
 int                                                                     \
@@ -101,11 +108,12 @@ DECLARE_INIT_OPT(int64_t, int64);
 DECLARE_INIT_OPT(uint32_t, uint32);
 DECLARE_INIT_OPT(int32_t, int32);
 DECLARE_INIT_OPT(uint64_t, size);
-DECLARE_INIT_OPT(uint32_t, percent);
-DECLARE_INIT_OPT(uint64_t, percent_or_size);
+DECLARE_INIT_OPT(double, percent);
+DECLARE_INIT_OPT(double, percent_or_size);
 DECLARE_INIT_OPT(gf_boolean_t, bool);
 DECLARE_INIT_OPT(xlator_t *, xlator);
 DECLARE_INIT_OPT(char *, path);
+DECLARE_INIT_OPT(double, double);
 
 
 #define DEFINE_INIT_OPT(type_t, type, conv)                             \
@@ -137,10 +145,11 @@ xlator_option_init_##type (xlator_t *this, dict_t *options, char *key,  \
         if (!value) {                                                   \
                 gf_log (this->name, GF_LOG_TRACE, "option %s not set",  \
                         key);                                           \
+                *val_p = (type_t)0;                                     \
                 return 0;                                               \
         }                                                               \
         if (value == def_value) {                                       \
-                gf_log (this->name, GF_LOG_DEBUG,                       \
+                gf_log (this->name, GF_LOG_TRACE,                       \
                         "option %s using default value %s",             \
                         key, value);                                    \
         } else {                                                        \
@@ -179,11 +188,12 @@ DECLARE_RECONF_OPT(int64_t, int64);
 DECLARE_RECONF_OPT(uint32_t, uint32);
 DECLARE_RECONF_OPT(int32_t, int32);
 DECLARE_RECONF_OPT(uint64_t, size);
-DECLARE_RECONF_OPT(uint32_t, percent);
-DECLARE_RECONF_OPT(uint64_t, percent_or_size);
+DECLARE_RECONF_OPT(double, percent);
+DECLARE_RECONF_OPT(double, percent_or_size);
 DECLARE_RECONF_OPT(gf_boolean_t, bool);
 DECLARE_RECONF_OPT(xlator_t *, xlator);
 DECLARE_RECONF_OPT(char *, path);
+DECLARE_RECONF_OPT(double, double);
 
 
 #define DEFINE_RECONF_OPT(type_t, type, conv)                            \
@@ -215,6 +225,7 @@ xlator_option_reconf_##type (xlator_t *this, dict_t *options, char *key, \
         if (!value) {                                                   \
                 gf_log (this->name, GF_LOG_TRACE, "option %s not set",  \
                         key);                                           \
+                *val_p = (type_t)0;                                     \
                 return 0;                                               \
         }                                                               \
         if (value == def_value) {                                       \
diff --git a/libglusterfs/src/rbthash.c b/libglusterfs/src/rbthash.c
index beda672d0..0d7b9e521 100644
--- a/libglusterfs/src/rbthash.c
+++ b/libglusterfs/src/rbthash.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -217,8 +208,7 @@ rbthash_deinit_entry (rbthash_table_t *tbl, rbthash_entry_t *entry)
         if (!entry)
                 return;
 
-        if (entry->key)
-                GF_FREE (entry->key);
+        GF_FREE (entry->key);
 
         if (tbl) {
                 if ((entry->data) && (tbl->dfunc))
@@ -237,7 +227,7 @@ rbthash_deinit_entry (rbthash_table_t *tbl, rbthash_entry_t *entry)
 }
 
 
-inline struct rbthash_bucket *
+static inline struct rbthash_bucket *
 rbthash_entry_bucket (rbthash_table_t *tbl, rbthash_entry_t * entry)
 {
         int     nbucket = 0;
@@ -311,7 +301,7 @@ err:
         return ret;
 }
 
-inline struct rbthash_bucket *
+static inline struct rbthash_bucket *
 rbthash_key_bucket (rbthash_table_t *tbl, void *key, int keylen)
 {
         uint32_t        keyhash = 0;
diff --git a/libglusterfs/src/rbthash.h b/libglusterfs/src/rbthash.h
index 02a65b4a8..b093ce998 100644
--- a/libglusterfs/src/rbthash.h
+++ b/libglusterfs/src/rbthash.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __RBTHASH_TABLE_H_
diff --git a/libglusterfs/src/run.c b/libglusterfs/src/run.c
index 46bb37d73..4fd2a3a0d 100644
--- a/libglusterfs/src/run.c
+++ b/libglusterfs/src/run.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _GNU_SOURCE
@@ -30,7 +21,9 @@
 #include <fcntl.h>
 #include <dirent.h>
 #include <assert.h>
+#include <signal.h>
 #include <sys/wait.h>
+#include <sys/resource.h>
 
 #ifdef RUN_STANDALONE
 #define GF_CALLOC(n, s, t) calloc(n, s)
@@ -74,7 +67,10 @@ runner_chio (runner_t *runner, int fd)
 {
         GF_ASSERT (fd > 0 && fd < 3);
 
-        return runner->chio[fd];
+        if ((fd > 0) && (fd < 3))
+                return runner->chio[fd];
+
+        return NULL;
 }
 
 static void
@@ -191,7 +187,7 @@ runner_log (runner_t *runner, const char *dom, gf_loglevel_t lvl,
         if (len > 0)
                 buf[len - 1] = '\0';
 
-        gf_log (dom, lvl, "%s: %s", msg, buf);
+        gf_log_callingfn (dom, lvl, "%s: %s", msg, buf);
 
         GF_FREE (buf);
 }
@@ -201,7 +197,8 @@ runner_redir (runner_t *runner, int fd, int tgt_fd)
 {
         GF_ASSERT (fd > 0 && fd < 3);
 
-        runner->chfd[fd] = (tgt_fd >= 0) ? tgt_fd : -2;
+        if ((fd > 0) && (fd < 3))
+                runner->chfd[fd] = (tgt_fd >= 0) ? tgt_fd : -2;
 }
 
 int
@@ -293,7 +290,11 @@ runner_start (runner_t *runner)
                         } else
                                 ret = -1;
 #else
-                        for (i = 3; i < 65536; i++) {
+                        struct rlimit rl;
+                        ret = getrlimit (RLIMIT_NOFILE, &rl);
+                        GF_ASSERT (ret == 0);
+
+                        for (i = 3; i < rl.rlim_cur; i++) {
                                 if (i != xpi[1])
                                         close (i);
                         }
@@ -364,9 +365,11 @@ runner_end (runner_t *runner)
 
         ret = runner_end_reuse (runner);
 
-        for (p = runner->argv; *p; p++)
-                GF_FREE (*p);
-        GF_FREE (runner->argv);
+        if (runner->argv) {
+                for (p = runner->argv; *p; p++)
+                        GF_FREE (*p);
+                GF_FREE (runner->argv);
+        }
         for (i = 0; i < 3; i++)
                 close (runner->chfd[i]);
 
@@ -379,10 +382,8 @@ runner_run_generic (runner_t *runner, int (*rfin)(runner_t *runner))
         int ret = 0;
 
         ret = runner_start (runner);
-        if (ret != 0)
-                return -1;
 
-        return rfin (runner) ? -1 : 0;
+        return -(rfin (runner) || ret);
 }
 
 int
@@ -391,6 +392,25 @@ runner_run (runner_t *runner)
         return runner_run_generic (runner, runner_end);
 }
 
+
+int
+runner_run_nowait (runner_t *runner)
+{
+	int pid;
+
+	pid = fork ();
+
+	if (!pid) {
+		setsid ();
+		_exit (runner_start (runner));
+	}
+
+	if (pid > 0)
+		runner->chpid = pid;
+	return runner_end (runner);
+}
+
+
 int
 runner_run_reuse (runner_t *runner)
 {
@@ -422,7 +442,7 @@ TBANNER (const char *txt)
 }
 
 int
-main ()
+main (int argc, char **argv)
 {
         runner_t runner;
         char buf[80];
@@ -430,6 +450,8 @@ main ()
         int ret;
         int fd;
         long pathmax = pathconf ("/", _PC_PATH_MAX);
+        struct timeval tv = {0,};
+        struct timeval *tvp = NULL;
 
         wdbuf = malloc (pathmax);
         assert (wdbuf);
@@ -468,7 +490,7 @@ main ()
         printf ("%d %d [%s]\n", ret, errno, strerror (errno));
 
         TBANNER ("output redirection");
-        fd = open ("/tmp/foof", O_WRONLY|O_CREAT|O_TRUNC, 0600);
+        fd = mkstemp ("/tmp/foof");
         assert (fd != -1);
         runinit (&runner);
         runner_add_args (&runner, "echo", "foo", NULL);
@@ -479,6 +501,13 @@ main ()
                 printf (" %d [%s]", errno, strerror (errno));
         putchar ('\n');
 
+        if (argc > 1) {
+                tv.tv_sec = strtoul (argv[1], NULL, 10);
+                if (tv.tv_sec > 0)
+                        tvp = &tv;
+                select (0, 0, 0, 0, tvp);
+        }
+
         return 0;
 }
 #endif
diff --git a/libglusterfs/src/run.h b/libglusterfs/src/run.h
index 70b61aff8..d7554ef6d 100644
--- a/libglusterfs/src/run.h
+++ b/libglusterfs/src/run.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __RUN_H__
@@ -175,6 +166,12 @@ int runner_end_reuse (runner_t *runner);
 int runner_run (runner_t *runner);
 
 /**
+ * variant for runner_run() which does not wait for acknowledgement
+ * from child, and always assumes it succeeds.
+ */
+int runner_run_nowait (runner_t *runner);
+
+/**
  * variant of runner_run() which does not free internal data
  * so that the runner instance can be run again.
  *
diff --git a/libglusterfs/src/scheduler.c b/libglusterfs/src/scheduler.c
deleted file mode 100644
index f27875dc2..000000000
--- a/libglusterfs/src/scheduler.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <dlfcn.h>
-#include <netdb.h>
-#include "xlator.h"
-#include "scheduler.h"
-#include "list.h"
-
-struct sched_ops *
-get_scheduler (xlator_t *xl, const char *name)
-{
-        struct sched_ops  *tmp_sched = NULL;
-        volume_opt_list_t *vol_opt   = NULL;
-        char *sched_file             = NULL;
-        void *handle                 = NULL;
-        int   ret                    = 0;
-
-        if (name == NULL) {
-                gf_log ("scheduler", GF_LOG_ERROR,
-                        "'name' not specified, EINVAL");
-                return NULL;
-        }
-
-        ret = gf_asprintf (&sched_file, "%s/%s.so", SCHEDULERDIR, name);
-        if (-1 == ret) {
-                gf_log ("scheduler", GF_LOG_ERROR, "asprintf failed");
-                return NULL;
-        }
-
-        gf_log ("scheduler", GF_LOG_DEBUG,
-                "attempt to load file %s.so", name);
-
-        handle = dlopen (sched_file, RTLD_LAZY);
-        if (!handle) {
-                gf_log ("scheduler", GF_LOG_ERROR,
-                        "dlopen(%s): %s", sched_file, dlerror ());
-                GF_FREE(sched_file);
-                return NULL;
-        }
-
-        tmp_sched = dlsym (handle, "sched");
-        if (!tmp_sched) {
-                gf_log ("scheduler", GF_LOG_ERROR,
-                        "dlsym(sched) on %s", dlerror ());
-                GF_FREE(sched_file);
-                return NULL;
-        }
-
-        vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
-                             gf_common_mt_volume_opt_list_t);
-        vol_opt->given_opt = dlsym (handle, "options");
-        if (vol_opt->given_opt == NULL) {
-                gf_log ("scheduler", GF_LOG_DEBUG,
-                        "volume option validation not specified");
-        } else {
-                list_add_tail (&vol_opt->list, &xl->volume_options);
-                if (validate_xlator_volume_options (xl, vol_opt->given_opt)
-                    == -1) {
-                        gf_log ("scheduler", GF_LOG_ERROR,
-                                "volume option validation failed");
-                        GF_FREE(sched_file);
-                        return NULL;
-                }
-        }
-        GF_FREE(sched_file);
-        GF_FREE (vol_opt);
-
-        return tmp_sched;
-}
diff --git a/libglusterfs/src/scheduler.h b/libglusterfs/src/scheduler.h
deleted file mode 100644
index acb6597b9..000000000
--- a/libglusterfs/src/scheduler.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _SCHEDULER_H
-#define _SCHEDULER_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-
-struct sched_ops {
-  int32_t (*init) (xlator_t *this);
-  void (*fini) (xlator_t *this);
-  void (*update) (xlator_t *this);
-  xlator_t *(*schedule) (xlator_t *this, const void *path);
-  void (*notify) (xlator_t *xl, int32_t event, void *data);
-  int32_t (*mem_acct_init) (xlator_t *this);
-};
-
-extern struct sched_ops *get_scheduler (xlator_t *xl, const char *name);
-
-#endif /* _SCHEDULER_H */
diff --git a/libglusterfs/src/stack.c b/libglusterfs/src/stack.c
index 29b44aedf..37b338f51 100644
--- a/libglusterfs/src/stack.c
+++ b/libglusterfs/src/stack.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "statedump.h"
@@ -35,6 +26,44 @@ int call_frames_count (call_frame_t *call_frame)
         return count;
 }
 
+call_frame_t *
+create_frame (xlator_t *xl, call_pool_t *pool)
+{
+        call_stack_t    *stack = NULL;
+
+        if (!xl || !pool) {
+                return NULL;
+        }
+
+        stack = mem_get0 (pool->stack_mem_pool);
+        if (!stack)
+                return NULL;
+
+        stack->pool = pool;
+        stack->frames.root = stack;
+        stack->frames.this = xl;
+        stack->ctx = xl->ctx;
+
+        if (stack->ctx->measure_latency) {
+                if (gettimeofday (&stack->tv, NULL) == -1)
+                        gf_log ("stack", GF_LOG_ERROR, "gettimeofday () failed."
+                                " (%s)", strerror (errno));
+                memcpy (&stack->frames.begin, &stack->tv, sizeof (stack->tv));
+        }
+
+        LOCK (&pool->lock);
+        {
+                list_add (&stack->all_frames, &pool->all_frames);
+                pool->cnt++;
+        }
+        UNLOCK (&pool->lock);
+
+        LOCK_INIT (&stack->frames.lock);
+        LOCK_INIT (&stack->stack_lock);
+
+        return &stack->frames;
+}
+
 void
 gf_proc_dump_call_frame (call_frame_t *call_frame, const char *key_buf,...)
 {
@@ -43,6 +72,7 @@ gf_proc_dump_call_frame (call_frame_t *call_frame, const char *key_buf,...)
         va_list ap;
         call_frame_t my_frame;
         int  ret = -1;
+        char timestr[256] = {0,};
 
         if (!call_frame)
                 return;
@@ -56,15 +86,21 @@ gf_proc_dump_call_frame (call_frame_t *call_frame, const char *key_buf,...)
         va_end(ap);
 
         ret = TRY_LOCK(&call_frame->lock);
-        if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump call frame"
-                       " errno: %s", strerror (errno));
-                return;
-        }
+        if (ret)
+                goto out;
 
         memcpy(&my_frame, call_frame, sizeof(my_frame));
         UNLOCK(&call_frame->lock);
 
+        if (my_frame.this->ctx->measure_latency) {
+                gf_time_fmt (timestr, sizeof timestr, my_frame.begin.tv_sec,
+                             gf_timefmt_FT);
+                snprintf (timestr + strlen (timestr),
+                          sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, my_frame.begin.tv_usec);
+                gf_proc_dump_write("frame-creation-time", "%s", timestr);
+        }
+
         gf_proc_dump_write("ref_count", "%d", my_frame.ref_count);
         gf_proc_dump_write("translator", "%s", my_frame.this->name);
         gf_proc_dump_write("complete", "%d", my_frame.complete);
@@ -82,6 +118,14 @@ gf_proc_dump_call_frame (call_frame_t *call_frame, const char *key_buf,...)
 
         if (my_frame.unwind_to)
                 gf_proc_dump_write("unwind_to", "%s", my_frame.unwind_to);
+
+        ret = 0;
+out:
+        if (ret) {
+                gf_proc_dump_write("Unable to dump the frame information",
+                                   "(Lock acquisition failed) %p", my_frame);
+                return;
+        }
 }
 
 
@@ -92,6 +136,7 @@ gf_proc_dump_call_stack (call_stack_t *call_stack, const char *key_buf,...)
         va_list ap;
         call_frame_t *trav;
         int32_t cnt, i;
+        char timestr[256] = {0,};
 
         if (!call_stack)
                 return;
@@ -105,15 +150,26 @@ gf_proc_dump_call_stack (call_stack_t *call_stack, const char *key_buf,...)
         vsnprintf(prefix, GF_DUMP_MAX_BUF_LEN, key_buf, ap);
         va_end(ap);
 
+        if (call_stack->ctx->measure_latency) {
+                gf_time_fmt (timestr, sizeof timestr, call_stack->tv.tv_sec,
+                             gf_timefmt_FT);
+                snprintf (timestr + strlen (timestr),
+                          sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, call_stack->tv.tv_usec);
+        gf_proc_dump_write("callstack-creation-time", "%s", timestr);
+        }
+
         gf_proc_dump_write("uid", "%d", call_stack->uid);
         gf_proc_dump_write("gid", "%d", call_stack->gid);
         gf_proc_dump_write("pid", "%d", call_stack->pid);
         gf_proc_dump_write("unique", "%Ld", call_stack->unique);
+        gf_proc_dump_write("lk-owner", "%s", lkowner_utoa (&call_stack->lk_owner));
 
         if (call_stack->type == GF_OP_TYPE_FOP)
-                gf_proc_dump_write("op", "%s", gf_fop_list[call_stack->op]);
-        else if (call_stack->type == GF_OP_TYPE_MGMT)
-                gf_proc_dump_write("op", "%s", gf_mgmt_list[call_stack->op]);
+                gf_proc_dump_write("op", "%s",
+                                   (char *)gf_fop_list[call_stack->op]);
+        else
+                gf_proc_dump_write("op", "stack");
 
         gf_proc_dump_write("type", "%d", call_stack->type);
         gf_proc_dump_write("cnt", "%d", cnt);
@@ -136,19 +192,18 @@ gf_proc_dump_pending_frames (call_pool_t *call_pool)
         call_stack_t     *trav = NULL;
         int              i = 1;
         int              ret = -1;
+        gf_boolean_t     section_added = _gf_true;
 
         if (!call_pool)
                 return;
 
         ret = TRY_LOCK (&(call_pool->lock));
-        if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump call pool"
-                       " errno: %d", errno);
-                return;
-        }
+        if (ret)
+                goto out;
 
 
         gf_proc_dump_add_section("global.callpool");
+        section_added = _gf_true;
         gf_proc_dump_write("callpool_address","%p", call_pool);
         gf_proc_dump_write("callpool.cnt","%d", call_pool->cnt);
 
@@ -159,6 +214,17 @@ gf_proc_dump_pending_frames (call_pool_t *call_pool)
                 i++;
         }
         UNLOCK (&(call_pool->lock));
+
+        ret = 0;
+out:
+        if (ret) {
+                if (_gf_false == section_added)
+                        gf_proc_dump_add_section("global.callpool");
+                gf_proc_dump_write("Unable to dump the callpool",
+                                   "(Lock acquisition failed) %p",
+                                   call_pool);
+        }
+        return;
 }
 
 void
@@ -186,7 +252,7 @@ gf_proc_dump_call_frame_to_dict (call_frame_t *call_frame,
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "%s.translator", prefix);
-        ret = dict_set_str (dict, key, gf_strdup (tmp_frame.this->name));
+        ret = dict_set_dynstr (dict, key, gf_strdup (tmp_frame.this->name));
         if (ret)
                 return;
 
@@ -199,7 +265,7 @@ gf_proc_dump_call_frame_to_dict (call_frame_t *call_frame,
         if (tmp_frame.parent) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "%s.parent", prefix);
-                ret = dict_set_str (dict, key,
+                ret = dict_set_dynstr (dict, key,
                                     gf_strdup (tmp_frame.parent->this->name));
                 if (ret)
                         return;
@@ -208,7 +274,8 @@ gf_proc_dump_call_frame_to_dict (call_frame_t *call_frame,
         if (tmp_frame.wind_from) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "%s.windfrom", prefix);
-                ret = dict_set_str (dict, key, gf_strdup (tmp_frame.wind_from));
+                ret = dict_set_dynstr (dict, key,
+                                       gf_strdup (tmp_frame.wind_from));
                 if (ret)
                         return;
         }
@@ -216,7 +283,8 @@ gf_proc_dump_call_frame_to_dict (call_frame_t *call_frame,
         if (tmp_frame.wind_to) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "%s.windto", prefix);
-                ret = dict_set_str (dict, key, gf_strdup (tmp_frame.wind_to));
+                ret = dict_set_dynstr (dict, key,
+                                       gf_strdup (tmp_frame.wind_to));
                 if (ret)
                         return;
         }
@@ -224,7 +292,8 @@ gf_proc_dump_call_frame_to_dict (call_frame_t *call_frame,
         if (tmp_frame.unwind_from) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "%s.unwindfrom", prefix);
-                ret = dict_set_str (dict, key, gf_strdup (tmp_frame.unwind_from));
+                ret = dict_set_dynstr (dict, key,
+                                       gf_strdup (tmp_frame.unwind_from));
                 if (ret)
                         return;
         }
@@ -232,7 +301,8 @@ gf_proc_dump_call_frame_to_dict (call_frame_t *call_frame,
         if (tmp_frame.unwind_to) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "%s.unwind_to", prefix);
-                ret = dict_set_str (dict, key, gf_strdup (tmp_frame.unwind_to));
+                ret = dict_set_dynstr (dict, key,
+                                       gf_strdup (tmp_frame.unwind_to));
         }
 
         return;
@@ -281,10 +351,10 @@ gf_proc_dump_call_stack_to_dict (call_stack_t *call_stack,
         snprintf (key, sizeof (key), "%s.op", prefix);
         if (call_stack->type == GF_OP_TYPE_FOP)
                 ret = dict_set_str (dict, key,
-                                    gf_fop_list[call_stack->op]);
-        else if (call_stack->type == GF_OP_TYPE_MGMT)
-                ret = dict_set_str (dict, key,
-                                    gf_mgmt_list[call_stack->op]);
+                                    (char *)gf_fop_list[call_stack->op]);
+        else
+                ret = dict_set_str (dict, key, "other");
+
         if (ret)
                 return;
 
diff --git a/libglusterfs/src/stack.h b/libglusterfs/src/stack.h
index c84040576..f2d2ef950 100644
--- a/libglusterfs/src/stack.h
+++ b/libglusterfs/src/stack.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /*
@@ -34,8 +25,8 @@ struct _call_stack_t;
 typedef struct _call_stack_t call_stack_t;
 struct _call_frame_t;
 typedef struct _call_frame_t call_frame_t;
-struct _call_pool_t;
-typedef struct _call_pool_t call_pool_t;
+struct call_pool;
+typedef struct call_pool call_pool_t;
 
 #include <sys/time.h>
 
@@ -45,6 +36,7 @@ typedef struct _call_pool_t call_pool_t;
 #include "common-utils.h"
 #include "globals.h"
 #include "lkowner.h"
+#include "client_t.h"
 
 #define NFS_PID 1
 #define LOW_PRIO_PROC_PID -1
@@ -55,7 +47,7 @@ typedef int32_t (*ret_fn_t) (call_frame_t *frame,
                              int32_t op_errno,
                              ...);
 
-struct _call_pool_t {
+struct call_pool {
         union {
                 struct list_head   all_frames;
                 struct {
@@ -91,6 +83,8 @@ struct _call_frame_t {
         const char      *unwind_to;
 };
 
+#define SMALL_GROUP_COUNT 128
+
 struct _call_stack_t {
         union {
                 struct list_head      all_frames;
@@ -101,20 +95,24 @@ struct _call_stack_t {
         };
         call_pool_t                  *pool;
         gf_lock_t                     stack_lock;
-        void                         *trans;
+        client_t                     *client;
         uint64_t                      unique;
         void                         *state;  /* pointer to request state */
         uid_t                         uid;
         gid_t                         gid;
         pid_t                         pid;
         uint16_t                      ngrps;
-        uint32_t                      groups[GF_MAX_AUX_GROUPS];
+        uint32_t                      groups_small[SMALL_GROUP_COUNT];
+	uint32_t                     *groups_large;
+	uint32_t                     *groups;
         gf_lkowner_t                  lk_owner;
+        glusterfs_ctx_t              *ctx;
 
         call_frame_t                  frames;
 
         int32_t                       op;
         int8_t                        type;
+        struct timeval                tv;
 };
 
 
@@ -131,11 +129,10 @@ struct _call_stack_t {
 struct xlator_fops;
 
 void
-gf_set_fop_from_fn_pointer (call_frame_t *frame, struct xlator_fops *fops,
-                            void *fn);
+gf_latency_begin (call_frame_t *frame, void *fn);
 
 void
-gf_update_latency (call_frame_t *frame);
+gf_latency_end (call_frame_t *frame);
 
 static inline void
 FRAME_DESTROY (call_frame_t *frame)
@@ -155,7 +152,7 @@ FRAME_DESTROY (call_frame_t *frame)
         mem_put (frame);
 
         if (local)
-                GF_FREE (local);
+                mem_put (local);
 }
 
 
@@ -177,16 +174,37 @@ STACK_DESTROY (call_stack_t *stack)
         }
 
         LOCK_DESTROY (&stack->frames.lock);
+        LOCK_DESTROY (&stack->stack_lock);
 
         while (stack->frames.next) {
                 FRAME_DESTROY (stack->frames.next);
         }
+
+	GF_FREE (stack->groups_large);
+
         mem_put (stack);
 
         if (local)
-                GF_FREE (local);
+                mem_put (local);
 }
 
+static inline void
+STACK_RESET (call_stack_t *stack)
+{
+        void *local = NULL;
+
+        if (stack->frames.local) {
+                local = stack->frames.local;
+                stack->frames.local = NULL;
+        }
+
+        while (stack->frames.next) {
+                FRAME_DESTROY (stack->frames.next);
+        }
+
+        if (local)
+                mem_put (local);
+}
 
 #define cbk(x) cbk_##x
 
@@ -227,6 +245,7 @@ STACK_DESTROY (call_stack_t *stack)
                 _new->wind_from = __FUNCTION__;                         \
                 _new->wind_to = #fn;                                    \
                 _new->unwind_to = #rfn;                                 \
+                                                                        \
                 LOCK_INIT (&_new->lock);                                \
                 LOCK(&frame->root->stack_lock);                         \
                 {                                                       \
@@ -240,11 +259,27 @@ STACK_DESTROY (call_stack_t *stack)
                 UNLOCK(&frame->root->stack_lock);                       \
                 old_THIS = THIS;                                        \
                 THIS = obj;                                             \
+                if (frame->this->ctx->measure_latency)                  \
+                        gf_latency_begin (_new, fn);                    \
                 fn (_new, obj, params);                                 \
                 THIS = old_THIS;                                        \
         } while (0)
 
 
+/* make a call without switching frames */
+#define STACK_WIND_TAIL(frame, obj, fn, params ...)                     \
+        do {                                                            \
+                xlator_t     *old_THIS = NULL;                          \
+                                                                        \
+                frame->this = obj;                                      \
+                frame->wind_to = #fn;                                   \
+                old_THIS = THIS;                                        \
+                THIS = obj;                                             \
+                fn (frame, obj, params);                                \
+                THIS = old_THIS;                                        \
+        } while (0)
+
+
 /* make a call with a cookie */
 #define STACK_WIND_COOKIE(frame, rfn, cky, obj, fn, params ...)         \
         do {                                                            \
@@ -279,6 +314,8 @@ STACK_DESTROY (call_stack_t *stack)
                 fn##_cbk = rfn;                                         \
                 old_THIS = THIS;                                        \
                 THIS = obj;                                             \
+                if (obj->ctx->measure_latency)                          \
+                        gf_latency_begin (_new, fn);                    \
                 fn (_new, obj, params);                                 \
                 THIS = old_THIS;                                        \
         } while (0)
@@ -305,6 +342,8 @@ STACK_DESTROY (call_stack_t *stack)
                 THIS = _parent->this;                                   \
                 frame->complete = _gf_true;                             \
                 frame->unwind_from = __FUNCTION__;                      \
+                if (frame->this->ctx->measure_latency)                  \
+                        gf_latency_end (frame);                         \
                 fn (_parent, frame->cookie, _parent->this, params);     \
                 THIS = old_THIS;                                        \
         } while (0)
@@ -332,11 +371,31 @@ STACK_DESTROY (call_stack_t *stack)
                 THIS = _parent->this;                                   \
                 frame->complete = _gf_true;                             \
                 frame->unwind_from = __FUNCTION__;                      \
+                if (frame->this->ctx->measure_latency)                  \
+                        gf_latency_end (frame);                         \
                 fn (_parent, frame->cookie, _parent->this, params);     \
                 THIS = old_THIS;                                        \
         } while (0)
 
 
+static inline int
+call_stack_alloc_groups (call_stack_t *stack, int ngrps)
+{
+	if (ngrps <= SMALL_GROUP_COUNT) {
+		stack->groups = stack->groups_small;
+	} else {
+		stack->groups_large = GF_CALLOC (sizeof (gid_t), ngrps,
+						 gf_common_mt_groups_t);
+		if (!stack->groups_large)
+			return -1;
+		stack->groups = stack->groups_large;
+	}
+
+	stack->ngrps = ngrps;
+
+	return 0;
+}
+
 static inline call_frame_t *
 copy_frame (call_frame_t *frame)
 {
@@ -360,14 +419,27 @@ copy_frame (call_frame_t *frame)
         newstack->ngrps = oldstack->ngrps;
         newstack->op  = oldstack->op;
         newstack->type = oldstack->type;
+	if (call_stack_alloc_groups (newstack, oldstack->ngrps) != 0) {
+		mem_put (newstack);
+		return NULL;
+	}
         memcpy (newstack->groups, oldstack->groups,
-                sizeof (gid_t) * GF_MAX_AUX_GROUPS);
+                sizeof (gid_t) * oldstack->ngrps);
         newstack->unique = oldstack->unique;
 
         newstack->frames.this = frame->this;
         newstack->frames.root = newstack;
         newstack->pool = oldstack->pool;
         newstack->lk_owner = oldstack->lk_owner;
+        newstack->ctx = oldstack->ctx;
+
+        if (newstack->ctx->measure_latency) {
+                if (gettimeofday (&newstack->tv, NULL) == -1)
+                        gf_log ("stack", GF_LOG_ERROR, "gettimeofday () failed."
+                                " (%s)", strerror (errno));
+                memcpy (&newstack->frames.begin, &newstack->tv,
+                        sizeof (newstack->tv));
+        }
 
         LOCK_INIT (&newstack->frames.lock);
         LOCK_INIT (&newstack->stack_lock);
@@ -382,39 +454,9 @@ copy_frame (call_frame_t *frame)
         return &newstack->frames;
 }
 
-
-static inline call_frame_t *
-create_frame (xlator_t *xl, call_pool_t *pool)
-{
-        call_stack_t    *stack = NULL;
-
-        if (!xl || !pool) {
-                return NULL;
-        }
-
-        stack = mem_get0 (pool->stack_mem_pool);
-        if (!stack)
-                return NULL;
-
-        stack->pool = pool;
-        stack->frames.root = stack;
-        stack->frames.this = xl;
-
-        LOCK (&pool->lock);
-        {
-                list_add (&stack->all_frames, &pool->all_frames);
-                pool->cnt++;
-        }
-        UNLOCK (&pool->lock);
-
-        LOCK_INIT (&stack->frames.lock);
-        LOCK_INIT (&stack->stack_lock);
-
-        return &stack->frames;
-}
-
 void gf_proc_dump_pending_frames(call_pool_t *call_pool);
 void gf_proc_dump_pending_frames_to_dict (call_pool_t *call_pool,
                                           dict_t *dict);
+call_frame_t *create_frame (xlator_t *xl, call_pool_t *pool);
 gf_boolean_t __is_fuse_call (call_frame_t *frame);
 #endif /* _STACK_H */
diff --git a/libglusterfs/src/statedump.c b/libglusterfs/src/statedump.c
index 8c6fe9215..8175faba4 100644
--- a/libglusterfs/src/statedump.c
+++ b/libglusterfs/src/statedump.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <stdarg.h>
@@ -30,7 +21,8 @@
 #endif /* MALLOC_H */
 
 /* We don't want gf_log in this function because it may cause
-   'deadlock' with statedump */
+   'deadlock' with statedump. This is because statedump happens
+   inside a signal handler and cannot afford to block on a lock.*/
 #ifdef gf_log
 # undef gf_log
 #endif
@@ -61,17 +53,12 @@ gf_proc_dump_unlock (void)
         pthread_mutex_unlock (&gf_proc_dump_mutex);
 }
 
-
 static int
-gf_proc_dump_open (char *dump_dir, char *brickname)
+gf_proc_dump_open (char *tmpname)
 {
-        char path[PATH_MAX] = {0,};
         int  dump_fd = -1;
 
-        snprintf (path, sizeof (path), "%s/%s.%d.dump", (dump_dir ?
-                  dump_dir : "/tmp"), brickname, getpid());
-
-        dump_fd = open (path, O_CREAT|O_RDWR|O_TRUNC|O_APPEND, 0600);
+        dump_fd = mkstemp (tmpname);
         if (dump_fd < 0)
                 return -1;
 
@@ -79,7 +66,6 @@ gf_proc_dump_open (char *dump_dir, char *brickname)
         return 0;
 }
 
-
 static void
 gf_proc_dump_close (void)
 {
@@ -87,6 +73,45 @@ gf_proc_dump_close (void)
         gf_dump_fd = -1;
 }
 
+static int
+gf_proc_dump_set_path (char *dump_options_file)
+{
+        int     ret = -1;
+        FILE    *fp = NULL;
+        char    buf[256];
+        char    *key = NULL, *value = NULL;
+        char    *saveptr = NULL;
+
+        fp = fopen (dump_options_file, "r");
+        if (!fp)
+                goto out;
+
+        ret = fscanf (fp, "%s", buf);
+
+        while (ret != EOF) {
+                key = strtok_r (buf, "=", &saveptr);
+                if (!key) {
+                        ret = fscanf (fp, "%s", buf);
+                        continue;
+                }
+
+                value = strtok_r (NULL, "=", &saveptr);
+
+                if (!value) {
+                        ret = fscanf (fp, "%s", buf);
+                        continue;
+                }
+                if (!strcmp (key, "path")) {
+                        dump_options.dump_path = gf_strdup (value);
+                        break;
+                }
+        }
+
+out:
+        if (fp)
+                fclose (fp);
+        return ret;
+}
 
 int
 gf_proc_dump_add_section (char *key, ...)
@@ -305,6 +330,9 @@ gf_proc_dump_mempool_info (glusterfs_ctx_t *ctx)
                                     pool->padded_sizeof_type);
                 gf_proc_dump_write ("alloc-count", "%"PRIu64, pool->alloc_count);
                 gf_proc_dump_write ("max-alloc", "%d", pool->max_alloc);
+
+                gf_proc_dump_write ("pool-misses", "%"PRIu64, pool->pool_misses);
+                gf_proc_dump_write ("max-stdalloc", "%d", pool->max_stdalloc);
         }
 }
 
@@ -356,6 +384,17 @@ gf_proc_dump_mempool_info_to_dict (glusterfs_ctx_t *ctx, dict_t *dict)
                 if (ret)
                         return;
 
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "pool%d.max-stdalloc", count);
+                ret = dict_set_int32 (dict, key, pool->max_stdalloc);
+                if (ret)
+                        return;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "pool%d.pool-misses", count);
+                ret = dict_set_uint64 (dict, key, pool->pool_misses);
+                if (ret)
+                        return;
                 count++;
         }
         ret = dict_set_int32 (dict, "mempool-count", count);
@@ -375,7 +414,7 @@ gf_proc_dump_xlator_info (xlator_t *top)
         if (!top)
                 return;
 
-        ctx = glusterfs_ctx_get ();
+        ctx = top->ctx;
 
         trav = top;
         while (trav) {
@@ -389,8 +428,6 @@ gf_proc_dump_xlator_info (xlator_t *top)
                     (trav->itable)) {
                         snprintf (itable_key, 1024, "%d.%s.itable",
                                   ctx->graph_id, trav->name);
-
-                        inode_table_dump (trav->itable, itable_key);
                 }
 
                 if (!trav->dumpops) {
@@ -410,6 +447,10 @@ gf_proc_dump_xlator_info (xlator_t *top)
                     GF_PROC_DUMP_IS_XL_OPTION_ENABLED (fd))
                         trav->dumpops->fd (trav);
 
+                if (trav->dumpops->history &&
+                    GF_PROC_DUMP_IS_XL_OPTION_ENABLED (history))
+                        trav->dumpops->history (trav);
+
                 trav = trav->next;
         }
 
@@ -420,24 +461,20 @@ static void
 gf_proc_dump_oldgraph_xlator_info (xlator_t *top)
 {
         xlator_t        *trav = NULL;
-        glusterfs_ctx_t *ctx = NULL;
-        char             itable_key[1024] = {0,};
 
         if (!top)
                 return;
 
-        ctx = glusterfs_ctx_get ();
-
         trav = top;
         while (trav) {
                 gf_proc_dump_xlator_mem_info_only_in_use (trav);
 
                 if (GF_PROC_DUMP_IS_XL_OPTION_ENABLED (inode) &&
                     (trav->itable)) {
-                        snprintf (itable_key, 1024, "%d.%s.itable",
-                                  ctx->graph_id, trav->name);
-
-                        inode_table_dump (trav->itable, itable_key);
+                        /*TODO: dump inode table info if necessary by
+                          printing the graph id (taken by glusterfs_cbtx_t)
+                          in the key
+                        */
                 }
 
                 if (!trav->dumpops) {
@@ -472,6 +509,46 @@ gf_proc_dump_enable_all_options ()
         GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_inodectx,
                                  _gf_true);
         GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_fdctx, _gf_true);
+        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_history,
+                                 _gf_true);
+
+        return 0;
+}
+
+gf_boolean_t
+is_gf_proc_dump_all_disabled ()
+{
+        gf_boolean_t all_disabled = _gf_true;
+
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_mem, all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_iobuf, all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_callpool, all_disabled,
+                                   out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_priv,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_inode,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_fd,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_inodectx,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_fdctx,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_history,
+                                   all_disabled, out);
+
+out:
+        return all_disabled;
+}
+
+/* These options are dumped by default if glusterdump.options
+   file exists and it is emtpty
+*/
+static int
+gf_proc_dump_enable_default_options ()
+{
+        GF_PROC_DUMP_SET_OPTION (dump_options.dump_mem, _gf_true);
+        GF_PROC_DUMP_SET_OPTION (dump_options.dump_callpool, _gf_true);
 
         return 0;
 }
@@ -490,7 +567,8 @@ gf_proc_dump_disable_all_options ()
         GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_inodectx,
                                  _gf_false);
         GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_fdctx, _gf_false);
-
+        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_history,
+                                 _gf_false);
         return 0;
 }
 
@@ -502,25 +580,27 @@ gf_proc_dump_parse_set_option (char *key, char *value)
         char buf[GF_DUMP_MAX_BUF_LEN];
         int ret = -1;
 
-        if (!strncasecmp (key, "all", 3)) {
+        if (!strcasecmp (key, "all")) {
                 (void)gf_proc_dump_enable_all_options ();
                 return 0;
-        } else if (!strncasecmp (key, "mem", 3)) {
+        } else if (!strcasecmp (key, "mem")) {
                 opt_key = &dump_options.dump_mem;
-        } else if (!strncasecmp (key, "iobuf", 5)) {
+        } else if (!strcasecmp (key, "iobuf")) {
                 opt_key = &dump_options.dump_iobuf;
-        } else if (!strncasecmp (key, "callpool", 8)) {
+        } else if (!strcasecmp (key, "callpool")) {
                 opt_key = &dump_options.dump_callpool;
-        } else if (!strncasecmp (key, "priv", 4)) {
+        } else if (!strcasecmp (key, "priv")) {
                 opt_key = &dump_options.xl_options.dump_priv;
-        } else if (!strncasecmp (key, "fd", 2)) {
+        } else if (!strcasecmp (key, "fd")) {
                 opt_key = &dump_options.xl_options.dump_fd;
-        } else if (!strncasecmp (key, "inode", 5)) {
+        } else if (!strcasecmp (key, "inode")) {
                 opt_key = &dump_options.xl_options.dump_inode;
-        } else if (!strncasecmp (key, "inodectx", strlen ("inodectx"))) {
+        } else if (!strcasecmp (key, "inodectx")) {
                 opt_key = &dump_options.xl_options.dump_inodectx;
-        } else if (!strncasecmp (key, "fdctx", strlen ("fdctx"))) {
+        } else if (!strcasecmp (key, "fdctx")) {
                 opt_key = &dump_options.xl_options.dump_fdctx;
+        } else if (!strcasecmp (key, "history")) {
+                opt_key = &dump_options.xl_options.dump_history;
         }
 
         if (!opt_key) {
@@ -529,11 +609,9 @@ gf_proc_dump_parse_set_option (char *key, char *value)
                           "matched key : %s\n", key);
                 ret = write (gf_dump_fd, buf, strlen (buf));
 
-                /* warning suppression */
-                if (ret >= 0) {
+                if (ret >= 0)
                         ret = -1;
-                        goto out;
-                }
+                goto out;
 
         }
 
@@ -548,33 +626,49 @@ out:
 }
 
 static int
-gf_proc_dump_options_init (char *dump_name)
+gf_proc_dump_options_init ()
 {
         int     ret = -1;
         FILE    *fp = NULL;
         char    buf[256];
-        char    dumpbuf[GF_DUMP_MAX_BUF_LEN];
         char    *key = NULL, *value = NULL;
         char    *saveptr = NULL;
         char    dump_option_file[PATH_MAX];
 
+        /* glusterd will create a file glusterdump.<pid>.options and
+           sets the statedump options for the process and the file is removed
+           after the statedump is taken. Direct issue of SIGUSR1 does not have
+           mechanism for considering the statedump options. So to have a way
+           of configuring the statedump of all the glusterfs processes through
+           both cli command and SIGUSR1, glusterdump.options file is searched
+           and the options mentioned in it are given the higher priority.
+        */
         snprintf (dump_option_file, sizeof (dump_option_file),
-                  "/tmp/glusterdump.%d.options", getpid ());
-
+                  DEFAULT_VAR_RUN_DIRECTORY
+                  "/glusterdump.options");
         fp = fopen (dump_option_file, "r");
-
         if (!fp) {
-                //ENOENT, return success
-                (void) gf_proc_dump_enable_all_options ();
-                return 0;
+                snprintf (dump_option_file, sizeof (dump_option_file),
+                          DEFAULT_VAR_RUN_DIRECTORY
+                          "/glusterdump.%d.options", getpid ());
+
+                fp = fopen (dump_option_file, "r");
+
+                if (!fp) {
+                        //ENOENT, return success
+                        (void) gf_proc_dump_enable_all_options ();
+                        return 0;
+                }
         }
 
         (void) gf_proc_dump_disable_all_options ();
 
+        // swallow the errors if setting statedump file path is failed.
+        ret = gf_proc_dump_set_path (dump_option_file);
+
         ret = fscanf (fp, "%s", buf);
 
         while (ret != EOF) {
-
                 key = strtok_r (buf, "=", &saveptr);
                 if (!key) {
                         ret = fscanf (fp, "%s", buf);
@@ -588,28 +682,33 @@ gf_proc_dump_options_init (char *dump_name)
                         continue;
                 }
 
-                snprintf (dumpbuf, sizeof (dumpbuf), "[Debug]:key=%s, value=%s\n",key,value);
-                ret = write (gf_dump_fd, dumpbuf, strlen (dumpbuf));
-
                 gf_proc_dump_parse_set_option (key, value);
-
         }
 
+        if (is_gf_proc_dump_all_disabled ())
+                (void) gf_proc_dump_enable_default_options ();
+
+        if (fp)
+                fclose (fp);
+
         return 0;
 }
 
 void
-gf_proc_dump_info (int signum)
+gf_proc_dump_info (int signum, glusterfs_ctx_t *ctx)
 {
-        int                i    = 0;
-        int                ret  = -1;
-        glusterfs_ctx_t   *ctx  = NULL;
-        glusterfs_graph_t *trav = NULL;
-        char               brick_name[PATH_MAX] = {0,};
+        int                i                       = 0;
+        int                ret                     = -1;
+        glusterfs_graph_t *trav                    = NULL;
+        char               brick_name[PATH_MAX]    = {0,};
+        char               timestr[256]            = {0,};
+        char               sign_string[512]        = {0,};
+        char               tmp_dump_name[PATH_MAX] = {0,};
+        char               path[PATH_MAX]          = {0,};
+        struct timeval     tv                      = {0,};
 
         gf_proc_dump_lock ();
 
-        ctx = glusterfs_ctx_get ();
         if (!ctx)
                 goto out;
 
@@ -618,14 +717,44 @@ gf_proc_dump_info (int signum)
         } else
                 strncpy (brick_name, "glusterdump", sizeof (brick_name));
 
-        ret = gf_proc_dump_options_init (brick_name);
+        ret = gf_proc_dump_options_init ();
         if (ret < 0)
                 goto out;
 
-        ret = gf_proc_dump_open (ctx->statedump_path, brick_name);
+        snprintf (path, sizeof (path), "%s/%s.%d.dump.%"PRIu64,
+                  ((dump_options.dump_path != NULL)?dump_options.dump_path:
+                   ((ctx->statedump_path != NULL)?ctx->statedump_path:
+                    DEFAULT_VAR_RUN_DIRECTORY)), brick_name, getpid(),
+                  (uint64_t) time (NULL));
+
+        snprintf (tmp_dump_name, PATH_MAX, "%s/dumpXXXXXX",
+                  ((dump_options.dump_path != NULL)?dump_options.dump_path:
+                   ((ctx->statedump_path != NULL)?ctx->statedump_path:
+                    DEFAULT_VAR_RUN_DIRECTORY)));
+
+        ret = gf_proc_dump_open (tmp_dump_name);
         if (ret < 0)
                 goto out;
 
+        //continue even though gettimeofday() has failed
+        ret = gettimeofday (&tv, NULL);
+        if (0 == ret) {
+                gf_time_fmt (timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT);
+                snprintf (timestr + strlen (timestr),
+                          sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
+        }
+
+        snprintf (sign_string, sizeof (sign_string), "DUMP-START-TIME: %s\n",
+                  timestr);
+
+        //swallow the errors of write for start and end marker
+        ret = write (gf_dump_fd, sign_string, strlen (sign_string));
+
+        memset (sign_string, 0, sizeof (sign_string));
+        memset (timestr, 0, sizeof (timestr));
+        memset (&tv, 0, sizeof (tv));
+
         if (GF_PROC_DUMP_IS_OPTION_ENABLED (mem)) {
                 gf_proc_dump_mem_info ();
                 gf_proc_dump_mempool_info (ctx);
@@ -657,8 +786,24 @@ gf_proc_dump_info (int signum)
                 i++;
         }
 
-        gf_proc_dump_close ();
+        ret = gettimeofday (&tv, NULL);
+        if (0 == ret) {
+                gf_time_fmt (timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT);
+                snprintf (timestr + strlen (timestr),
+                          sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
+        }
+
+        snprintf (sign_string, sizeof (sign_string), "\nDUMP-END-TIME: %s",
+                  timestr);
+        ret = write (gf_dump_fd, sign_string, strlen (sign_string));
+
 out:
+        if (gf_dump_fd != -1)
+                gf_proc_dump_close ();
+        rename (tmp_dump_name, path);
+        GF_FREE (dump_options.dump_path);
+        dump_options.dump_path = NULL;
         gf_proc_dump_unlock ();
 
         return;
diff --git a/libglusterfs/src/statedump.h b/libglusterfs/src/statedump.h
index cdeb5b7fc..8342b120a 100644
--- a/libglusterfs/src/statedump.h
+++ b/libglusterfs/src/statedump.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -32,6 +23,7 @@ typedef struct gf_dump_xl_options_ {
         gf_boolean_t    dump_fd;
         gf_boolean_t    dump_inodectx;
         gf_boolean_t    dump_fdctx;
+        gf_boolean_t    dump_history;
 } gf_dump_xl_options_t;
 
 typedef struct gf_dump_options_ {
@@ -39,6 +31,7 @@ typedef struct gf_dump_options_ {
         gf_boolean_t            dump_iobuf;
         gf_boolean_t            dump_callpool;
         gf_dump_xl_options_t    xl_options; //options for all xlators
+        char                    *dump_path;
 } gf_dump_options_t;
 
 extern gf_dump_options_t dump_options;
@@ -63,13 +56,21 @@ void _gf_proc_dump_build_key (char *key, const char *prefix, char *fmt,...)
 
 #define GF_PROC_DUMP_SET_OPTION(opt,val) opt = val
 
+#define GF_CHECK_DUMP_OPTION_ENABLED(option_dump, var, label)      \
+        do {                                                    \
+                if (option_dump == _gf_true) {                  \
+                        var = _gf_false;                        \
+                        goto label;                             \
+                }                                               \
+        } while (0);
+
 void gf_proc_dump_init();
 
 void gf_proc_dump_fini(void);
 
 void gf_proc_dump_cleanup(void);
 
-void gf_proc_dump_info(int signum);
+void gf_proc_dump_info(int signum, glusterfs_ctx_t *ctx);
 
 int gf_proc_dump_add_section(char *key,...);
 
diff --git a/libglusterfs/src/store.c b/libglusterfs/src/store.c
new file mode 100644
index 000000000..1e6601837
--- /dev/null
+++ b/libglusterfs/src/store.c
@@ -0,0 +1,709 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <inttypes.h>
+#include <libgen.h>
+
+#include "glusterfs.h"
+#include "store.h"
+#include "dict.h"
+#include "xlator.h"
+
+int32_t
+gf_store_mkdir (char *path)
+{
+        int32_t     ret = -1;
+
+        ret = mkdir (path, 0777);
+
+        if ((-1 == ret) && (EEXIST != errno)) {
+                gf_log ("", GF_LOG_ERROR, "mkdir() failed on path %s,"
+                        "errno: %s", path, strerror (errno));
+        } else {
+                ret = 0;
+        }
+
+        return ret;
+}
+
+int32_t
+gf_store_handle_create_on_absence (gf_store_handle_t **shandle,
+                                   char *path)
+{
+        GF_ASSERT (shandle);
+        int32_t     ret = 0;
+
+        if (*shandle == NULL) {
+                ret = gf_store_handle_new (path, shandle);
+
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to create store"
+                                " handle for path: %s", path);
+                }
+        }
+        return ret;
+}
+
+int32_t
+gf_store_mkstemp (gf_store_handle_t *shandle)
+{
+        int     fd = -1;
+        char    tmppath[PATH_MAX] = {0,};
+
+        GF_ASSERT (shandle);
+        GF_ASSERT (shandle->path);
+
+        snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
+        fd = open (tmppath, O_RDWR | O_CREAT | O_TRUNC | O_SYNC, 0600);
+        if (fd <= 0) {
+                gf_log ("", GF_LOG_ERROR, "Failed to open %s, error: %s",
+                        tmppath, strerror (errno));
+        }
+
+        return fd;
+}
+
+int
+gf_store_sync_direntry (char *path)
+{
+        int             ret     = -1;
+        int             dirfd   = -1;
+        char            *dir    = NULL;
+        char            *pdir   = NULL;
+        xlator_t        *this = NULL;
+
+        this = THIS;
+
+        dir = gf_strdup (path);
+        if (!dir)
+                goto out;
+
+        pdir = dirname (dir);
+        dirfd = open (pdir, O_RDONLY);
+        if (dirfd == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open directory "
+                        "%s, due to %s", pdir, strerror (errno));
+                goto out;
+        }
+
+        ret = fsync (dirfd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to fsync %s, due to "
+                        "%s", pdir, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (dirfd >= 0) {
+                ret = close (dirfd);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to close "
+                                "%s, due to %s", pdir, strerror (errno));
+                }
+        }
+
+        if (dir)
+                GF_FREE (dir);
+
+        return ret;
+}
+
+int32_t
+gf_store_rename_tmppath (gf_store_handle_t *shandle)
+{
+        int32_t         ret = -1;
+        char            tmppath[PATH_MAX] = {0,};
+
+        GF_ASSERT (shandle);
+        GF_ASSERT (shandle->path);
+
+        snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
+        ret = rename (tmppath, shandle->path);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to rename %s to %s, "
+                        "error: %s", tmppath, shandle->path, strerror (errno));
+                goto out;
+        }
+
+        ret = gf_store_sync_direntry (tmppath);
+out:
+        return ret;
+}
+
+int32_t
+gf_store_unlink_tmppath (gf_store_handle_t *shandle)
+{
+        int32_t         ret = -1;
+        char            tmppath[PATH_MAX] = {0,};
+
+        GF_ASSERT (shandle);
+        GF_ASSERT (shandle->path);
+
+        snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
+        ret = unlink (tmppath);
+        if (ret && (errno != ENOENT)) {
+                gf_log ("", GF_LOG_ERROR, "Failed to mv %s to %s, error: %s",
+                        tmppath, shandle->path, strerror (errno));
+        } else {
+                ret = 0;
+        }
+
+        return ret;
+}
+
+int
+gf_store_read_and_tokenize (FILE *file, char *str, char **iter_key,
+                            char **iter_val, gf_store_op_errno_t *store_errno)
+{
+        int32_t     ret         =   -1;
+        char        *savetok    = NULL;
+        char        *key        = NULL;
+        char        *value      = NULL;
+        char        *temp       = NULL;
+        size_t       str_len    =    0;
+
+        GF_ASSERT (file);
+        GF_ASSERT (str);
+        GF_ASSERT (iter_key);
+        GF_ASSERT (iter_val);
+        GF_ASSERT (store_errno);
+
+        temp = fgets (str, PATH_MAX, file);
+        if (temp == NULL || feof (file)) {
+                ret = -1;
+                *store_errno = GD_STORE_EOF;
+                goto out;
+        }
+
+        str_len = strlen(str);
+        str[str_len - 1] = '\0';
+        /* Truncate the "\n", as fgets stores "\n" in str */
+
+        key = strtok_r (str, "=", &savetok);
+        if (!key) {
+                ret = -1;
+                *store_errno = GD_STORE_KEY_NULL;
+                goto out;
+        }
+
+        value = strtok_r (NULL, "=", &savetok);
+        if (!value) {
+                ret = -1;
+                *store_errno = GD_STORE_VALUE_NULL;
+                goto out;
+        }
+
+        *iter_key = key;
+        *iter_val = value;
+        *store_errno = GD_STORE_SUCCESS;
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+gf_store_retrieve_value (gf_store_handle_t *handle, char *key, char **value)
+{
+        int32_t         ret = -1;
+        char            *scan_str = NULL;
+        char            *iter_key = NULL;
+        char            *iter_val = NULL;
+        char            *free_str = NULL;
+        struct stat     st        = {0,};
+        gf_store_op_errno_t store_errno = GD_STORE_SUCCESS;
+
+        GF_ASSERT (handle);
+
+        if (handle->locked == F_ULOCK)
+                /* no locking is used handle->fd gets closed() after usage */
+                handle->fd = open (handle->path, O_RDWR);
+        else
+                /* handle->fd is valid already, kept open for lockf() */
+                lseek (handle->fd, 0, SEEK_SET);
+
+        if (handle->fd == -1) {
+                gf_log ("", GF_LOG_ERROR, "Unable to open file %s errno: %s",
+                        handle->path, strerror (errno));
+                goto out;
+        }
+        if (!handle->read)
+                handle->read = fdopen (dup(handle->fd), "r");
+        else
+                fseek (handle->read, 0, SEEK_SET);
+
+        if (!handle->read) {
+                gf_log ("", GF_LOG_ERROR, "Unable to open file %s errno: %s",
+                        handle->path, strerror (errno));
+                goto out;
+        }
+
+        ret = fstat (handle->fd, &st);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_WARNING, "stat on file %s failed",
+                        handle->path);
+                ret = -1;
+                store_errno = GD_STORE_STAT_FAILED;
+                goto out;
+        }
+
+        /* "st.st_size + 1" is used as we are fetching each
+         * line of a file using fgets, fgets will append "\0"
+         * to the end of the string
+         */
+        scan_str = GF_CALLOC (1, st.st_size + 1,
+                              gf_common_mt_char);
+
+        if (scan_str == NULL) {
+                ret = -1;
+                store_errno = GD_STORE_ENOMEM;
+                goto out;
+        }
+
+        free_str = scan_str;
+
+        do {
+                ret = gf_store_read_and_tokenize (handle->read, scan_str,
+                                                  &iter_key, &iter_val,
+                                                  &store_errno);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_TRACE, "error while reading key "
+                                "'%s': %s", key,
+                                gf_store_strerror (store_errno));
+                        goto out;
+                }
+
+                gf_log ("", GF_LOG_TRACE, "key %s read", iter_key);
+
+                if (!strcmp (key, iter_key)) {
+                        gf_log ("", GF_LOG_DEBUG, "key %s found", key);
+                        ret = 0;
+                        if (iter_val)
+                                *value = gf_strdup (iter_val);
+                        goto out;
+                }
+        } while (1);
+out:
+        if (handle->read) {
+                fclose (handle->read);
+                handle->read = NULL;
+        }
+
+        if (handle->fd > 0 && handle->locked == F_ULOCK) {
+                /* only invalidate handle->fd if not locked */
+                close (handle->fd);
+        }
+
+        GF_FREE (free_str);
+
+        return ret;
+}
+
+int32_t
+gf_store_save_value (int fd, char *key, char *value)
+{
+        int32_t         ret = -1;
+        int             dup_fd = -1;
+        FILE           *fp  = NULL;
+
+        GF_ASSERT (fd > 0);
+        GF_ASSERT (key);
+        GF_ASSERT (value);
+
+        dup_fd = dup (fd);
+        if (dup_fd == -1)
+                goto out;
+
+        fp = fdopen (dup_fd, "a+");
+        if (fp == NULL) {
+                gf_log ("", GF_LOG_WARNING, "fdopen failed.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = fprintf (fp, "%s=%s\n", key, value);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_WARNING, "Unable to store key: %s,"
+                        "value: %s, error: %s", key, value,
+                        strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = fflush (fp);
+        if (feof (fp)) {
+                gf_log ("", GF_LOG_WARNING,
+                        "fflush failed, error: %s",
+                        strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (fp)
+                fclose (fp);
+
+        gf_log ("", GF_LOG_DEBUG, "returning: %d", ret);
+        return ret;
+}
+
+int32_t
+gf_store_handle_new (char *path, gf_store_handle_t **handle)
+{
+        int32_t                 ret = -1;
+        gf_store_handle_t *shandle = NULL;
+        int                     fd = -1;
+        char                    *spath = NULL;
+
+        shandle = GF_CALLOC (1, sizeof (*shandle), gf_common_mt_store_handle_t);
+        if (!shandle)
+                goto out;
+
+        spath = gf_strdup (path);
+
+        if (!spath)
+                goto out;
+
+        fd = open (path, O_RDWR | O_CREAT | O_APPEND, 0600);
+        if (fd <= 0) {
+                gf_log ("", GF_LOG_ERROR, "Failed to open file: %s, error: %s",
+                        path, strerror (errno));
+                goto out;
+        }
+
+        ret = gf_store_sync_direntry (spath);
+        if (ret)
+                goto out;
+
+        shandle->path = spath;
+        shandle->locked = F_ULOCK;
+        *handle = shandle;
+
+        ret = 0;
+out:
+        if (fd > 0)
+                close (fd);
+
+        if (ret == -1) {
+                GF_FREE (spath);
+                GF_FREE (shandle);
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+gf_store_handle_retrieve (char *path, gf_store_handle_t **handle)
+{
+        int32_t                 ret = -1;
+        struct stat statbuf = {0};
+
+        ret = stat (path, &statbuf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Path corresponding to "
+                        "%s, returned error: (%s)",
+                        path, strerror (errno));
+                goto out;
+        }
+        ret =  gf_store_handle_new (path, handle);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+gf_store_handle_destroy (gf_store_handle_t *handle)
+{
+        int32_t                 ret = -1;
+
+        if (!handle) {
+                ret = 0;
+                goto out;
+        }
+
+        GF_FREE (handle->path);
+
+        GF_FREE (handle);
+
+        ret = 0;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int32_t
+gf_store_iter_new (gf_store_handle_t  *shandle, gf_store_iter_t  **iter)
+{
+        int32_t                 ret = -1;
+        FILE                    *fp = NULL;
+        gf_store_iter_t         *tmp_iter = NULL;
+
+        GF_ASSERT (shandle);
+        GF_ASSERT (iter);
+
+        fp = fopen (shandle->path, "r");
+        if (!fp) {
+                gf_log ("", GF_LOG_ERROR, "Unable to open file %s errno: %d",
+                        shandle->path, errno);
+                goto out;
+        }
+
+        tmp_iter = GF_CALLOC (1, sizeof (*tmp_iter),
+                              gf_common_mt_store_iter_t);
+        if (!tmp_iter)
+                goto out;
+
+        strncpy (tmp_iter->filepath, shandle->path, sizeof (tmp_iter->filepath));
+        tmp_iter->filepath[sizeof (tmp_iter->filepath) - 1] = 0;
+        tmp_iter->file = fp;
+
+        *iter = tmp_iter;
+        tmp_iter = NULL;
+        ret = 0;
+
+out:
+        if (ret && fp)
+                fclose (fp);
+
+        GF_FREE (tmp_iter);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+gf_store_validate_key_value (char *storepath, char *key, char *val,
+                             gf_store_op_errno_t *op_errno)
+{
+        int ret = 0;
+
+        GF_ASSERT (op_errno);
+        GF_ASSERT (storepath);
+
+        if ((key == NULL) && (val == NULL)) {
+                ret = -1;
+                gf_log ("", GF_LOG_ERROR, "Glusterd store may be corrupted, "
+                        "Invalid key and value (null) in %s", storepath);
+                *op_errno = GD_STORE_KEY_VALUE_NULL;
+        } else if (key == NULL) {
+                ret = -1;
+                gf_log ("", GF_LOG_ERROR, "Glusterd store may be corrupted, "
+                        "Invalid key (null) in %s", storepath);
+                *op_errno = GD_STORE_KEY_NULL;
+        } else if (val == NULL) {
+                ret = -1;
+                gf_log ("", GF_LOG_ERROR, "Glusterd store may be corrupted, "
+                        "Invalid value (null) for key %s in %s", key,
+                        storepath);
+                *op_errno = GD_STORE_VALUE_NULL;
+        } else {
+                ret = 0;
+                *op_errno = GD_STORE_SUCCESS;
+        }
+
+        return ret;
+}
+
+int32_t
+gf_store_iter_get_next (gf_store_iter_t *iter, char  **key, char **value,
+                        gf_store_op_errno_t *op_errno)
+{
+        int32_t         ret       = -1;
+        char            *scan_str = NULL;
+        char            *iter_key = NULL;
+        char            *iter_val = NULL;
+        struct stat     st        = {0,};
+        gf_store_op_errno_t store_errno = GD_STORE_SUCCESS;
+
+        GF_ASSERT (iter);
+        GF_ASSERT (key);
+        GF_ASSERT (value);
+
+        ret = stat (iter->filepath, &st);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_WARNING, "stat on file failed");
+                ret = -1;
+                store_errno = GD_STORE_STAT_FAILED;
+                goto out;
+        }
+
+        /* "st.st_size + 1" is used as we are fetching each
+         * line of a file using fgets, fgets will append "\0"
+         * to the end of the string
+         */
+        scan_str = GF_CALLOC (1, st.st_size + 1,
+                              gf_common_mt_char);
+        if (!scan_str) {
+                ret = -1;
+                store_errno = GD_STORE_ENOMEM;
+                goto out;
+        }
+
+        ret = gf_store_read_and_tokenize (iter->file, scan_str,
+                                          &iter_key, &iter_val,
+                                          &store_errno);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = gf_store_validate_key_value (iter->filepath, iter_key,
+                                           iter_val, &store_errno);
+        if (ret)
+                goto out;
+
+        *key = gf_strdup (iter_key);
+        if (!*key) {
+                ret = -1;
+                store_errno = GD_STORE_ENOMEM;
+                goto out;
+        }
+        *value = gf_strdup (iter_val);
+        if (!*value) {
+                ret = -1;
+                store_errno = GD_STORE_ENOMEM;
+                goto out;
+        }
+        ret = 0;
+
+out:
+        GF_FREE (scan_str);
+        if (ret) {
+                GF_FREE (*key);
+                GF_FREE (*value);
+                *key = NULL;
+                *value = NULL;
+        }
+        if (op_errno)
+                *op_errno = store_errno;
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+gf_store_iter_get_matching (gf_store_iter_t *iter, char *key, char **value)
+{
+        int32_t ret = -1;
+        char    *tmp_key = NULL;
+        char    *tmp_value = NULL;
+
+        ret = gf_store_iter_get_next (iter, &tmp_key, &tmp_value, NULL);
+        while (!ret) {
+                if (!strncmp (key, tmp_key, strlen (key))){
+                        *value = tmp_value;
+                        GF_FREE (tmp_key);
+                        goto out;
+                }
+                GF_FREE (tmp_key);
+                tmp_key = NULL;
+                GF_FREE (tmp_value);
+                tmp_value = NULL;
+                ret = gf_store_iter_get_next (iter, &tmp_key, &tmp_value,
+                                              NULL);
+        }
+out:
+        return ret;
+}
+
+int32_t
+gf_store_iter_destroy (gf_store_iter_t *iter)
+{
+        int32_t         ret = -1;
+
+        if (!iter)
+                return 0;
+
+        /* gf_store_iter_new will not return a valid iter object with iter->file
+         * being NULL*/
+        ret = fclose (iter->file);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "Unable to close file: %s, ret: %d, "
+                        "errno: %d" ,iter->filepath, ret, errno);
+
+        GF_FREE (iter);
+        return ret;
+}
+
+char*
+gf_store_strerror (gf_store_op_errno_t op_errno)
+{
+        switch (op_errno) {
+        case GD_STORE_SUCCESS:
+                return "Success";
+        case GD_STORE_KEY_NULL:
+                return "Invalid Key";
+        case GD_STORE_VALUE_NULL:
+                return "Invalid Value";
+        case GD_STORE_KEY_VALUE_NULL:
+                return "Invalid Key and Value";
+        case GD_STORE_EOF:
+                return "No data";
+        case GD_STORE_ENOMEM:
+                return "No memory";
+        default:
+                return "Invalid errno";
+        }
+        return "Invalid errno";
+}
+
+int
+gf_store_lock (gf_store_handle_t *sh)
+{
+        int                     ret;
+
+        GF_ASSERT (sh);
+        GF_ASSERT (sh->path);
+        GF_ASSERT (sh->locked == F_ULOCK);
+
+        sh->fd = open (sh->path, O_RDWR);
+        if (sh->fd == -1) {
+                gf_log ("", GF_LOG_ERROR, "Failed to open '%s': %s", sh->path,
+                        strerror (errno));
+                return -1;
+        }
+
+        ret = lockf (sh->fd, F_LOCK, 0);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "Failed to gain lock on '%s': %s",
+                        sh->path, strerror (errno));
+        else
+                /* sh->locked is protected by the lockf(sh->fd) above */
+                sh->locked = F_LOCK;
+
+        return ret;
+}
+
+void
+gf_store_unlock (gf_store_handle_t *sh)
+{
+        GF_ASSERT (sh);
+        GF_ASSERT (sh->locked == F_LOCK);
+
+        sh->locked = F_ULOCK;
+        lockf (sh->fd, F_ULOCK, 0);
+        close (sh->fd);
+}
+
+int
+gf_store_locked_local (gf_store_handle_t *sh)
+{
+        GF_ASSERT (sh);
+
+        return (sh->locked == F_LOCK);
+}
diff --git a/libglusterfs/src/store.h b/libglusterfs/src/store.h
new file mode 100644
index 000000000..337103ff7
--- /dev/null
+++ b/libglusterfs/src/store.h
@@ -0,0 +1,112 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_STORE_H_
+#define _GLUSTERD_STORE_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+
+struct gf_store_handle_ {
+        char    *path;
+        int     fd;
+        FILE    *read;
+        int     locked;   /* state of lockf() */
+};
+
+typedef struct gf_store_handle_ gf_store_handle_t;
+
+struct gf_store_iter_ {
+        FILE    *file;
+        char    filepath[PATH_MAX];
+};
+
+typedef struct gf_store_iter_ gf_store_iter_t;
+
+typedef enum {
+        GD_STORE_SUCCESS,
+        GD_STORE_KEY_NULL,
+        GD_STORE_VALUE_NULL,
+        GD_STORE_KEY_VALUE_NULL,
+        GD_STORE_EOF,
+        GD_STORE_ENOMEM,
+        GD_STORE_STAT_FAILED
+} gf_store_op_errno_t;
+
+int32_t
+gf_store_mkdir (char *path);
+
+int32_t
+gf_store_handle_create_on_absence (gf_store_handle_t **shandle, char *path);
+
+int32_t
+gf_store_mkstemp (gf_store_handle_t *shandle);
+
+int
+gf_store_sync_direntry (char *path);
+
+int32_t
+gf_store_rename_tmppath (gf_store_handle_t *shandle);
+
+int32_t
+gf_store_unlink_tmppath (gf_store_handle_t *shandle);
+
+int
+gf_store_read_and_tokenize (FILE *file, char *str, char **iter_key,
+                            char **iter_val, gf_store_op_errno_t *store_errno);
+
+int32_t
+gf_store_retrieve_value (gf_store_handle_t *handle, char *key, char **value);
+
+int32_t
+gf_store_save_value (int fd, char *key, char *value);
+
+int32_t
+gf_store_handle_new (char *path, gf_store_handle_t **handle);
+
+int
+gf_store_handle_retrieve (char *path, gf_store_handle_t **handle);
+
+int32_t
+gf_store_handle_destroy (gf_store_handle_t *handle);
+
+int32_t
+gf_store_iter_new (gf_store_handle_t  *shandle, gf_store_iter_t  **iter);
+
+int32_t
+gf_store_validate_key_value (char *storepath, char *key, char *val,
+                             gf_store_op_errno_t *op_errno);
+
+int32_t
+gf_store_iter_get_next (gf_store_iter_t *iter, char **key, char **value,
+                        gf_store_op_errno_t *op_errno);
+
+int32_t
+gf_store_iter_get_matching (gf_store_iter_t *iter, char *key, char **value);
+
+int32_t
+gf_store_iter_destroy (gf_store_iter_t *iter);
+
+char*
+gf_store_strerror (gf_store_op_errno_t op_errno);
+
+int
+gf_store_lock (gf_store_handle_t *sh);
+
+void
+gf_store_unlock (gf_store_handle_t *sh);
+
+int
+gf_store_locked_local (gf_store_handle_t *sh);
+
+#endif
diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c
index 7b6f78248..c1620bb70 100644
--- a/libglusterfs/src/syncop.c
+++ b/libglusterfs/src/syncop.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -24,57 +15,250 @@
 
 #include "syncop.h"
 
-call_frame_t *
-syncop_create_frame ()
+int
+syncopctx_setfsuid (void *uid)
 {
-        struct synctask *task = NULL;
-        call_frame_t *frame = NULL;
-
-        task = synctask_get ();
+	struct syncopctx *opctx = NULL;
+	int               ret = 0;
+
+	/* In args check */
+	if (!uid) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	opctx = syncopctx_getctx ();
+
+	/* alloc for this thread the first time */
+	if (!opctx) {
+		opctx = GF_CALLOC (1, sizeof (*opctx), gf_common_mt_syncopctx);
+		if (!opctx) {
+			ret = -1;
+			goto out;
+		}
+
+		ret = syncopctx_setctx (opctx);
+		if (ret != 0) {
+			GF_FREE (opctx);
+			opctx = NULL;
+			goto out;
+		}
+	}
+
+out:
+	if (opctx && uid) {
+		opctx->uid = *(uid_t *)uid;
+		opctx->valid |= SYNCOPCTX_UID;
+	}
+
+	return ret;
+}
 
-        if (task) {
-                frame = task->frame;
-        }
+int
+syncopctx_setfsgid (void *gid)
+{
+	struct syncopctx *opctx = NULL;
+	int               ret = 0;
+
+	/* In args check */
+	if (!gid) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	opctx = syncopctx_getctx ();
+
+	/* alloc for this thread the first time */
+	if (!opctx) {
+		opctx = GF_CALLOC (1, sizeof (*opctx), gf_common_mt_syncopctx);
+		if (!opctx) {
+			ret = -1;
+			goto out;
+		}
+
+		ret = syncopctx_setctx (opctx);
+		if (ret != 0) {
+			GF_FREE (opctx);
+			opctx = NULL;
+			goto out;
+		}
+	}
+
+out:
+	if (opctx && gid) {
+		opctx->gid = *(gid_t *)gid;
+		opctx->valid |= SYNCOPCTX_GID;
+	}
+
+	return ret;
+}
 
-        return (call_frame_t *)frame;
+int
+syncopctx_setfsgroups (int count, const void *groups)
+{
+	struct syncopctx *opctx = NULL;
+	gid_t            *tmpgroups = NULL;
+	int               ret = 0;
+
+	/* In args check */
+	if (count != 0 && !groups) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	opctx = syncopctx_getctx ();
+
+	/* alloc for this thread the first time */
+	if (!opctx) {
+		opctx = GF_CALLOC (1, sizeof (*opctx), gf_common_mt_syncopctx);
+		if (!opctx) {
+			ret = -1;
+			goto out;
+		}
+
+		ret = syncopctx_setctx (opctx);
+		if (ret != 0) {
+			GF_FREE (opctx);
+			opctx = NULL;
+			goto out;
+		}
+	}
+
+	/* resize internal groups as required */
+	if (count && opctx->grpsize < count) {
+		if (opctx->groups) {
+			tmpgroups = GF_REALLOC (opctx->groups,
+						(sizeof (gid_t) * count));
+			/* NOTE: Not really required to zero the reallocation,
+			 * as ngrps controls the validity of data,
+			 * making a note irrespective */
+			if (tmpgroups == NULL) {
+				opctx->grpsize = 0;
+				GF_FREE (opctx->groups);
+				opctx->groups = NULL;
+				ret = -1;
+				goto out;
+			}
+		}
+		else {
+			tmpgroups = GF_CALLOC (count, sizeof (gid_t),
+					       gf_common_mt_syncopctx);
+			if (tmpgroups == NULL) {
+				opctx->grpsize = 0;
+				ret = -1;
+				goto out;
+			}
+		}
+
+		opctx->groups = tmpgroups;
+		opctx->grpsize = count;
+	}
+
+	/* copy out the groups passed */
+	if (count)
+		memcpy (opctx->groups, groups, (sizeof (gid_t) * count));
+
+	/* set/reset the ngrps, this is where reset of groups is handled */
+	opctx->ngrps = count;
+	opctx->valid |= SYNCOPCTX_GROUPS;
+
+out:
+	return ret;
 }
 
-void
-synctask_yield (struct synctask *task)
+static void
+__run (struct synctask *task)
 {
-        struct syncenv   *env = NULL;
+        struct syncenv *env = NULL;
 
         env = task->env;
 
-        if (swapcontext (&task->ctx, &env->sched) < 0) {
-                gf_log ("syncop", GF_LOG_ERROR,
-                        "swapcontext failed (%s)", strerror (errno));
+        list_del_init (&task->all_tasks);
+        switch (task->state) {
+        case SYNCTASK_INIT:
+        case SYNCTASK_SUSPEND:
+                break;
+        case SYNCTASK_RUN:
+                gf_log (task->xl->name, GF_LOG_DEBUG,
+                        "re-running already running task");
+                env->runcount--;
+                break;
+        case SYNCTASK_WAIT:
+                env->waitcount--;
+                break;
+        case SYNCTASK_DONE:
+                gf_log (task->xl->name, GF_LOG_WARNING,
+                        "running completed task");
+		return;
+	case SYNCTASK_ZOMBIE:
+		gf_log (task->xl->name, GF_LOG_WARNING,
+			"attempted to wake up zombie!!");
+		return;
         }
+
+        list_add_tail (&task->all_tasks, &env->runq);
+        env->runcount++;
+        task->state = SYNCTASK_RUN;
 }
 
 
-void
-synctask_yawn (struct synctask *task)
+static void
+__wait (struct synctask *task)
 {
-        struct syncenv  *env = NULL;
+        struct syncenv *env = NULL;
 
-        env  = task->env;
+        env = task->env;
 
-        pthread_mutex_lock (&env->mutex);
-        {
-                list_del_init (&task->all_tasks);
-                list_add (&task->all_tasks, &env->waitq);
+        list_del_init (&task->all_tasks);
+        switch (task->state) {
+        case SYNCTASK_INIT:
+        case SYNCTASK_SUSPEND:
+                break;
+        case SYNCTASK_RUN:
+                env->runcount--;
+                break;
+        case SYNCTASK_WAIT:
+                gf_log (task->xl->name, GF_LOG_WARNING,
+                        "re-waiting already waiting task");
+                env->waitcount--;
+                break;
+        case SYNCTASK_DONE:
+                gf_log (task->xl->name, GF_LOG_WARNING,
+                        "running completed task");
+                return;
+	case SYNCTASK_ZOMBIE:
+		gf_log (task->xl->name, GF_LOG_WARNING,
+			"attempted to sleep a zombie!!");
+		return;
         }
-        pthread_mutex_unlock (&env->mutex);
+
+        list_add_tail (&task->all_tasks, &env->waitq);
+        env->waitcount++;
+        task->state = SYNCTASK_WAIT;
 }
 
 
 void
-synctask_zzzz (struct synctask *task)
+synctask_yield (struct synctask *task)
 {
-        synctask_yawn (task);
+        xlator_t *oldTHIS = THIS;
 
-        synctask_yield (task);
+#if defined(__NetBSD__) && defined(_UC_TLSBASE)
+        /* Preserve pthread private pointer through swapcontex() */
+        task->proc->sched.uc_flags &= ~_UC_TLSBASE;
+#endif
+
+        if (task->state != SYNCTASK_DONE)
+                task->state = SYNCTASK_SUSPEND;
+        if (swapcontext (&task->ctx, &task->proc->sched) < 0) {
+                gf_log ("syncop", GF_LOG_ERROR,
+                        "swapcontext failed (%s)", strerror (errno));
+        }
+
+        THIS = oldTHIS;
 }
 
 
@@ -87,33 +271,30 @@ synctask_wake (struct synctask *task)
 
         pthread_mutex_lock (&env->mutex);
         {
-                list_del_init (&task->all_tasks);
-                list_add_tail (&task->all_tasks, &env->runq);
+                task->woken = 1;
+
+                if (task->slept)
+                        __run (task);
+
+		pthread_cond_broadcast (&env->cond);
         }
         pthread_mutex_unlock (&env->mutex);
-
-        pthread_cond_broadcast (&env->cond);
 }
 
-
 void
 synctask_wrap (struct synctask *old_task)
 {
-        int              ret;
         struct synctask *task = NULL;
 
         /* Do not trust the pointer received. It may be
            wrong and can lead to crashes. */
 
         task = synctask_get ();
-        ret = task->syncfn (task->opaque);
-        task->synccbk (ret, task->frame, task->opaque);
+        task->ret = task->syncfn (task->opaque);
+        if (task->synccbk)
+                task->synccbk (task->ret, task->frame, task->opaque);
 
-        /* cannot destroy @task right here as we are
-           in the execution stack of @task itself
-        */
-        task->complete = 1;
-        synctask_wake (task);
+        task->state = SYNCTASK_DONE;
 
         synctask_yield (task);
 }
@@ -125,36 +306,88 @@ synctask_destroy (struct synctask *task)
         if (!task)
                 return;
 
-        if (task->stack)
-                FREE (task->stack);
+        FREE (task->stack);
+
+        if (task->opframe)
+                STACK_DESTROY (task->opframe->root);
+
+        if (task->synccbk == NULL) {
+               pthread_mutex_destroy (&task->mutex);
+               pthread_cond_destroy (&task->cond);
+        }
+
         FREE (task);
 }
 
 
+void
+synctask_done (struct synctask *task)
+{
+        if (task->synccbk) {
+                synctask_destroy (task);
+                return;
+        }
+
+        pthread_mutex_lock (&task->mutex);
+        {
+		task->state = SYNCTASK_ZOMBIE;
+                task->done = 1;
+                pthread_cond_broadcast (&task->cond);
+        }
+        pthread_mutex_unlock (&task->mutex);
+}
+
+
 int
-synctask_new (struct syncenv *env, synctask_fn_t fn, synctask_cbk_t cbk,
-              call_frame_t *frame, void *opaque)
+synctask_setid (struct synctask *task, uid_t uid, gid_t gid)
+{
+        if (!task)
+                return -1;
+
+        if (uid != -1)
+                task->uid = uid;
+
+        if (gid != -1)
+                task->gid = gid;
+
+        return 0;
+}
+
+
+struct synctask *
+synctask_create (struct syncenv *env, synctask_fn_t fn, synctask_cbk_t cbk,
+		 call_frame_t *frame, void *opaque)
 {
         struct synctask *newtask = NULL;
         xlator_t        *this    = THIS;
 
         VALIDATE_OR_GOTO (env, err);
         VALIDATE_OR_GOTO (fn, err);
-        VALIDATE_OR_GOTO (cbk, err);
-        VALIDATE_OR_GOTO (frame, err);
 
         newtask = CALLOC (1, sizeof (*newtask));
         if (!newtask)
-                return -ENOMEM;
+                return NULL;
 
+        newtask->frame      = frame;
+        if (!frame) {
+                newtask->opframe = create_frame (this, this->ctx->pool);
+        } else {
+                newtask->opframe = copy_frame (frame);
+        }
+        if (!newtask->opframe)
+                goto err;
         newtask->env        = env;
         newtask->xl         = this;
         newtask->syncfn     = fn;
         newtask->synccbk    = cbk;
         newtask->opaque     = opaque;
-        newtask->frame      = frame;
+
+        /* default to the uid/gid of the passed frame */
+        newtask->uid = newtask->opframe->root->uid;
+        newtask->gid = newtask->opframe->root->gid;
 
         INIT_LIST_HEAD (&newtask->all_tasks);
+        INIT_LIST_HEAD (&newtask->waitq);
 
         if (getcontext (&newtask->ctx) < 0) {
                 gf_log ("syncop", GF_LOG_ERROR,
@@ -173,35 +406,114 @@ synctask_new (struct syncenv *env, synctask_fn_t fn, synctask_cbk_t cbk,
         newtask->ctx.uc_stack.ss_sp   = newtask->stack;
         newtask->ctx.uc_stack.ss_size = env->stacksize;
 
-        makecontext (&newtask->ctx, (void *) synctask_wrap, 2, newtask);
+        makecontext (&newtask->ctx, (void (*)(void)) synctask_wrap, 2, newtask);
+
+        newtask->state = SYNCTASK_INIT;
+
+        newtask->slept = 1;
+
+        if (!cbk) {
+                pthread_mutex_init (&newtask->mutex, NULL);
+                pthread_cond_init (&newtask->cond, NULL);
+                newtask->done = 0;
+        }
 
         synctask_wake (newtask);
+        /*
+         * Make sure someone's there to execute anything we just put on the
+         * run queue.
+         */
+        syncenv_scale(env);
 
-        return 0;
+	return newtask;
 err:
         if (newtask) {
-                if (newtask->stack)
-                        FREE (newtask->stack);
+                FREE (newtask->stack);
+                if (newtask->opframe)
+                        STACK_DESTROY (newtask->opframe->root);
                 FREE (newtask);
         }
-        return -1;
+
+        return NULL;
+}
+
+
+int
+synctask_join (struct synctask *task)
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&task->mutex);
+	{
+		while (!task->done)
+			pthread_cond_wait (&task->cond, &task->mutex);
+	}
+	pthread_mutex_unlock (&task->mutex);
+
+	ret = task->ret;
+
+	synctask_destroy (task);
+
+	return ret;
+}
+
+
+int
+synctask_new (struct syncenv *env, synctask_fn_t fn, synctask_cbk_t cbk,
+              call_frame_t *frame, void *opaque)
+{
+	struct synctask *newtask = NULL;
+	int              ret = 0;
+
+	newtask = synctask_create (env, fn, cbk, frame, opaque);
+	if (!newtask)
+		return -1;
+
+        if (!cbk)
+		ret = synctask_join (newtask);
+
+        return ret;
 }
 
 
 struct synctask *
-syncenv_task (struct syncenv *env)
+syncenv_task (struct syncproc *proc)
 {
+        struct syncenv   *env = NULL;
         struct synctask  *task = NULL;
+        struct timespec   sleep_till = {0, };
+        int               ret = 0;
+
+        env = proc->env;
 
         pthread_mutex_lock (&env->mutex);
         {
-                while (list_empty (&env->runq))
-                        pthread_cond_wait (&env->cond, &env->mutex);
+                while (list_empty (&env->runq)) {
+                        sleep_till.tv_sec = time (NULL) + SYNCPROC_IDLE_TIME;
+                        ret = pthread_cond_timedwait (&env->cond, &env->mutex,
+                                                      &sleep_till);
+                        if (!list_empty (&env->runq))
+                                break;
+                        if ((ret == ETIMEDOUT) &&
+                            (env->procs > env->procmin)) {
+                                task = NULL;
+                                env->procs--;
+                                memset (proc, 0, sizeof (*proc));
+                                goto unlock;
+                        }
+                }
 
                 task = list_entry (env->runq.next, struct synctask, all_tasks);
 
                 list_del_init (&task->all_tasks);
+                env->runcount--;
+
+                task->woken = 0;
+                task->slept = 0;
+
+                task->proc = proc;
         }
+unlock:
         pthread_mutex_unlock (&env->mutex);
 
         return task;
@@ -218,30 +530,51 @@ synctask_switchto (struct synctask *task)
         synctask_set (task);
         THIS = task->xl;
 
-        if (swapcontext (&env->sched, &task->ctx) < 0) {
+#if defined(__NetBSD__) && defined(_UC_TLSBASE)
+        /* Preserve pthread private pointer through swapcontex() */
+        task->ctx.uc_flags &= ~_UC_TLSBASE;
+#endif
+
+        if (swapcontext (&task->proc->sched, &task->ctx) < 0) {
                 gf_log ("syncop", GF_LOG_ERROR,
                         "swapcontext failed (%s)", strerror (errno));
         }
-}
 
+        if (task->state == SYNCTASK_DONE) {
+                synctask_done (task);
+                return;
+        }
+
+        pthread_mutex_lock (&env->mutex);
+        {
+                if (task->woken) {
+                        __run (task);
+                } else {
+                        task->slept = 1;
+                        __wait (task);
+                }
+        }
+        pthread_mutex_unlock (&env->mutex);
+}
 
 void *
 syncenv_processor (void *thdata)
 {
         struct syncenv  *env = NULL;
+        struct syncproc *proc = NULL;
         struct synctask *task = NULL;
 
-        env = thdata;
+        proc = thdata;
+        env = proc->env;
 
         for (;;) {
-                task = syncenv_task (env);
-
-                if (task->complete) {
-                        synctask_destroy (task);
-                        continue;
-                }
+                task = syncenv_task (proc);
+                if (!task)
+                        break;
 
                 synctask_switchto (task);
+
+                syncenv_scale (env);
         }
 
         return NULL;
@@ -249,6 +582,45 @@ syncenv_processor (void *thdata)
 
 
 void
+syncenv_scale (struct syncenv *env)
+{
+        int  diff = 0;
+        int  scale = 0;
+        int  i = 0;
+        int  ret = 0;
+
+        pthread_mutex_lock (&env->mutex);
+        {
+                if (env->procs > env->runcount)
+                        goto unlock;
+
+                scale = env->runcount;
+                if (scale > env->procmax)
+                        scale = env->procmax;
+                if (scale > env->procs)
+                        diff = scale - env->procs;
+                while (diff) {
+                        diff--;
+                        for (; (i < env->procmax); i++) {
+                                if (env->proc[i].processor == 0)
+                                        break;
+                        }
+
+                        env->proc[i].env = env;
+                        ret = gf_thread_create (&env->proc[i].processor, NULL,
+						syncenv_processor, &env->proc[i]);
+                        if (ret)
+                                break;
+                        env->procs++;
+                        i++;
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&env->mutex);
+}
+
+
+void
 syncenv_destroy (struct syncenv *env)
 {
 
@@ -256,10 +628,19 @@ syncenv_destroy (struct syncenv *env)
 
 
 struct syncenv *
-syncenv_new (size_t stacksize)
+syncenv_new (size_t stacksize, int procmin, int procmax)
 {
         struct syncenv *newenv = NULL;
         int             ret = 0;
+        int             i = 0;
+
+	if (!procmin || procmin < 0)
+		procmin = SYNCENV_PROC_MIN;
+	if (!procmax || procmax > SYNCENV_PROC_MAX)
+		procmax = SYNCENV_PROC_MAX;
+
+	if (procmin > procmax)
+		return NULL;
 
         newenv = CALLOC (1, sizeof (*newenv));
 
@@ -275,9 +656,17 @@ syncenv_new (size_t stacksize)
         newenv->stacksize    = SYNCENV_DEFAULT_STACKSIZE;
         if (stacksize)
                 newenv->stacksize = stacksize;
-
-        ret = pthread_create (&newenv->processor, NULL,
-                              syncenv_processor, newenv);
+	newenv->procmin = procmin;
+	newenv->procmax = procmax;
+
+        for (i = 0; i < newenv->procmin; i++) {
+                newenv->proc[i].env = newenv;
+                ret = gf_thread_create (&newenv->proc[i].processor, NULL,
+					syncenv_processor, &newenv->proc[i]);
+                if (ret)
+                        break;
+                newenv->procs++;
+        }
 
         if (ret != 0)
                 syncenv_destroy (newenv);
@@ -286,13 +675,275 @@ syncenv_new (size_t stacksize)
 }
 
 
+int
+synclock_init (synclock_t *lock)
+{
+	if (!lock)
+		return -1;
+
+	pthread_cond_init (&lock->cond, 0);
+	lock->lock = 0;
+	INIT_LIST_HEAD (&lock->waitq);
+
+	return pthread_mutex_init (&lock->guard, 0);
+}
+
+
+int
+synclock_destroy (synclock_t *lock)
+{
+	if (!lock)
+		return -1;
+
+	pthread_cond_destroy (&lock->cond);
+	return pthread_mutex_destroy (&lock->guard);
+}
+
+
+static int
+__synclock_lock (struct synclock *lock)
+{
+	struct synctask *task = NULL;
+
+	if (!lock)
+		return -1;
+
+	task = synctask_get ();
+
+	while (lock->lock) {
+		if (task) {
+			/* called within a synctask */
+			list_add_tail (&task->waitq, &lock->waitq);
+                        pthread_mutex_unlock (&lock->guard);
+                        synctask_yield (task);
+                        /* task is removed from waitq in unlock,
+                         * under lock->guard.*/
+                        pthread_mutex_lock (&lock->guard);
+		} else {
+			/* called by a non-synctask */
+			pthread_cond_wait (&lock->cond, &lock->guard);
+		}
+	}
+
+	lock->lock = _gf_true;
+	lock->owner = task;
+
+	return 0;
+}
+
+
+int
+synclock_lock (synclock_t *lock)
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&lock->guard);
+	{
+		ret = __synclock_lock (lock);
+	}
+	pthread_mutex_unlock (&lock->guard);
+
+	return ret;
+}
+
+
+int
+synclock_trylock (synclock_t *lock)
+{
+	int ret = 0;
+
+	errno = 0;
+
+	pthread_mutex_lock (&lock->guard);
+	{
+		if (lock->lock) {
+			errno = EBUSY;
+			ret = -1;
+			goto unlock;
+		}
+
+		ret = __synclock_lock (lock);
+	}
+unlock:
+	pthread_mutex_unlock (&lock->guard);
+
+	return ret;
+}
+
+
+static int
+__synclock_unlock (synclock_t *lock)
+{
+	struct synctask *task = NULL;
+	struct synctask *curr = NULL;
+
+	if (!lock)
+		return -1;
+
+	curr = synctask_get ();
+
+	if (lock->owner != curr) {
+		/* warn ? */
+	}
+
+	lock->lock = _gf_false;
+
+	/* There could be both synctasks and non synctasks
+	   waiting (or none, or either). As a mid-approach
+	   between maintaining too many waiting counters
+	   at one extreme and a thundering herd on unlock
+	   at the other, call a cond_signal (which wakes
+	   one waiter) and first synctask waiter. So at
+	   most we have two threads waking up to grab the
+	   just released lock.
+	*/
+	pthread_cond_signal (&lock->cond);
+	if (!list_empty (&lock->waitq)) {
+		task = list_entry (lock->waitq.next, struct synctask, waitq);
+                list_del_init (&task->waitq);
+		synctask_wake (task);
+	}
+
+	return 0;
+}
+
+
+int
+synclock_unlock (synclock_t *lock)
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&lock->guard);
+	{
+		ret = __synclock_unlock (lock);
+	}
+	pthread_mutex_unlock (&lock->guard);
+
+	return ret;
+}
+
+/* Barriers */
+
+int
+syncbarrier_init (struct syncbarrier *barrier)
+{
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	pthread_cond_init (&barrier->cond, 0);
+	barrier->count = 0;
+	INIT_LIST_HEAD (&barrier->waitq);
+
+	return pthread_mutex_init (&barrier->guard, 0);
+}
+
+
+int
+syncbarrier_destroy (struct syncbarrier *barrier)
+{
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	pthread_cond_destroy (&barrier->cond);
+	return pthread_mutex_destroy (&barrier->guard);
+}
+
+
+static int
+__syncbarrier_wait (struct syncbarrier *barrier, int waitfor)
+{
+	struct synctask *task = NULL;
+
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	task = synctask_get ();
+
+	while (barrier->count < waitfor) {
+		if (task) {
+			/* called within a synctask */
+			list_add_tail (&task->waitq, &barrier->waitq);
+                        pthread_mutex_unlock (&barrier->guard);
+                        synctask_yield (task);
+                        pthread_mutex_lock (&barrier->guard);
+		} else {
+			/* called by a non-synctask */
+			pthread_cond_wait (&barrier->cond, &barrier->guard);
+		}
+	}
+
+	barrier->count = 0;
+
+	return 0;
+}
+
+
+int
+syncbarrier_wait (struct syncbarrier *barrier, int waitfor)
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&barrier->guard);
+	{
+		ret = __syncbarrier_wait (barrier, waitfor);
+	}
+	pthread_mutex_unlock (&barrier->guard);
+
+	return ret;
+}
+
+
+static int
+__syncbarrier_wake (struct syncbarrier *barrier)
+{
+	struct synctask *task = NULL;
+
+	if (!barrier) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	barrier->count++;
+
+	pthread_cond_signal (&barrier->cond);
+	if (!list_empty (&barrier->waitq)) {
+		task = list_entry (barrier->waitq.next, struct synctask, waitq);
+                list_del_init (&task->waitq);
+		synctask_wake (task);
+	}
+
+	return 0;
+}
+
+
+int
+syncbarrier_wake (struct syncbarrier *barrier)
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&barrier->guard);
+	{
+		ret = __syncbarrier_wake (barrier);
+	}
+	pthread_mutex_unlock (&barrier->guard);
+
+	return ret;
+}
+
+
 /* FOPS */
 
 
 int
 syncop_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int op_ret, int op_errno, inode_t *inode,
-                   struct iatt *iatt, dict_t *xattr, struct iatt *parent)
+                   struct iatt *iatt, dict_t *xdata, struct iatt *parent)
 {
         struct syncargs *args = NULL;
 
@@ -304,8 +955,8 @@ syncop_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret == 0) {
                 args->iatt1  = *iatt;
                 args->iatt2  = *parent;
-                if (xattr)
-                        args->xattr  = dict_ref (xattr);
+                if (xdata)
+                        args->xdata  = dict_ref (xdata);
         }
 
         __wake (args);
@@ -315,22 +966,22 @@ syncop_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 
 int
-syncop_lookup (xlator_t *subvol, loc_t *loc, dict_t *xattr_req,
-               struct iatt *iatt, dict_t **xattr_rsp, struct iatt *parent)
+syncop_lookup (xlator_t *subvol, loc_t *loc, dict_t *xdata_req,
+               struct iatt *iatt, dict_t **xdata_rsp, struct iatt *parent)
 {
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_lookup_cbk, subvol->fops->lookup,
-                loc, xattr_req);
+                loc, xdata_req);
 
         if (iatt)
                 *iatt = args.iatt1;
         if (parent)
                 *parent = args.iatt2;
-        if (xattr_rsp)
-                *xattr_rsp = args.xattr;
-        else if (args.xattr)
-                dict_unref (args.xattr);
+        if (xdata_rsp)
+                *xdata_rsp = args.xdata;
+        else if (args.xdata)
+                dict_unref (args.xdata);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -348,6 +999,8 @@ entry_copy (gf_dirent_t *source)
         sink->d_type = source->d_type;
         sink->d_stat = source->d_stat;
 
+	if (source->inode)
+		sink->inode = inode_ref (source->inode);
         return sink;
 }
 
@@ -357,7 +1010,7 @@ syncop_readdirp_cbk (call_frame_t *frame,
                      xlator_t *this,
                      int32_t op_ret,
                      int32_t op_errno,
-                     gf_dirent_t *entries)
+                     gf_dirent_t *entries, dict_t *xdata)
 {
         struct syncargs *args = NULL;
         gf_dirent_t *entry = NULL;
@@ -412,12 +1065,71 @@ syncop_readdirp (xlator_t *subvol,
 }
 
 int32_t
+syncop_readdir_cbk (call_frame_t *frame,
+                    void *cookie,
+                    xlator_t *this,
+                    int32_t op_ret,
+                    int32_t op_errno,
+                    gf_dirent_t *entries, dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+        gf_dirent_t *entry = NULL;
+        gf_dirent_t  *tmp = NULL;
+
+        int count = 0;
+
+        args = cookie;
+
+        INIT_LIST_HEAD (&args->entries.list);
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+
+        if (op_ret >= 0) {
+                list_for_each_entry (entry, &entries->list, list) {
+                        tmp = entry_copy (entry);
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "adding entry=%s, count=%d",
+                                tmp->d_name, count);
+                        list_add_tail (&tmp->list, &(args->entries.list));
+                        count++;
+                }
+        }
+
+        __wake (args);
+
+        return 0;
+
+}
+
+int
+syncop_readdir (xlator_t *subvol,
+                fd_t *fd,
+                size_t size,
+                off_t off,
+                gf_dirent_t *entries)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_readdir_cbk, subvol->fops->readdir,
+                fd, size, off, NULL);
+
+        if (entries)
+                list_splice_init (&args.entries.list, &entries->list);
+        /* TODO: need to free all the 'args.entries' in 'else' case */
+
+        errno = args.op_errno;
+        return args.op_ret;
+
+}
+
+int32_t
 syncop_opendir_cbk (call_frame_t *frame,
                     void *cookie,
                     xlator_t *this,
                     int32_t op_ret,
                     int32_t op_errno,
-                    fd_t *fd)
+                    fd_t *fd, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -439,7 +1151,7 @@ syncop_opendir (xlator_t *subvol,
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_opendir_cbk, subvol->fops->opendir,
-                loc, fd);
+                loc, fd, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -447,8 +1159,36 @@ syncop_opendir (xlator_t *subvol,
 }
 
 int
+syncop_fsyncdir_cbk (call_frame_t *frame, void* cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+
+        __wake (args);
+
+        return 0;
+}
+
+int
+syncop_fsyncdir (xlator_t *subvol, fd_t *fd, int datasync)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_fsyncdir_cbk, subvol->fops->fsyncdir,
+                fd, datasync, NULL);
+
+        errno = args.op_errno;
+        return args.op_ret;
+}
+
+int
 syncop_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno)
+                        int op_ret, int op_errno, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -468,7 +1208,7 @@ syncop_removexattr (xlator_t *subvol, loc_t *loc, const char *name)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_removexattr_cbk, subvol->fops->removexattr,
-                loc, name);
+                loc, name, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -476,7 +1216,7 @@ syncop_removexattr (xlator_t *subvol, loc_t *loc, const char *name)
 
 int
 syncop_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno)
+                         int op_ret, int op_errno, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -496,7 +1236,7 @@ syncop_fremovexattr (xlator_t *subvol, fd_t *fd, const char *name)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_fremovexattr_cbk,
-                subvol->fops->fremovexattr, fd, name);
+                subvol->fops->fremovexattr, fd, name, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -504,7 +1244,7 @@ syncop_fremovexattr (xlator_t *subvol, fd_t *fd, const char *name)
 
 int
 syncop_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno)
+                     int op_ret, int op_errno, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -525,7 +1265,7 @@ syncop_setxattr (xlator_t *subvol, loc_t *loc, dict_t *dict, int32_t flags)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_setxattr_cbk, subvol->fops->setxattr,
-                loc, dict, flags);
+                loc, dict, flags, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -533,7 +1273,7 @@ syncop_setxattr (xlator_t *subvol, loc_t *loc, dict_t *dict, int32_t flags)
 
 int
 syncop_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno)
+                      int op_ret, int op_errno, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -554,7 +1294,7 @@ syncop_fsetxattr (xlator_t *subvol, fd_t *fd, dict_t *dict, int32_t flags)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_fsetxattr_cbk, subvol->fops->fsetxattr,
-                fd, dict, flags);
+                fd, dict, flags, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -562,7 +1302,7 @@ syncop_fsetxattr (xlator_t *subvol, fd_t *fd, dict_t *dict, int32_t flags)
 
 int
 syncop_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *dict)
+                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -584,7 +1324,7 @@ syncop_listxattr (xlator_t *subvol, loc_t *loc, dict_t **dict)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_getxattr_cbk, subvol->fops->getxattr,
-                loc, NULL);
+                loc, NULL, NULL);
 
         if (dict)
                 *dict = args.xattr;
@@ -601,7 +1341,7 @@ syncop_getxattr (xlator_t *subvol, loc_t *loc, dict_t **dict, const char *key)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_getxattr_cbk, subvol->fops->getxattr,
-                loc, key);
+                loc, key, NULL);
 
         if (dict)
                 *dict = args.xattr;
@@ -618,7 +1358,7 @@ syncop_fgetxattr (xlator_t *subvol, fd_t *fd, dict_t **dict, const char *key)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_getxattr_cbk, subvol->fops->fgetxattr,
-                fd, key);
+                fd, key, NULL);
 
         if (dict)
                 *dict = args.xattr;
@@ -632,7 +1372,7 @@ syncop_fgetxattr (xlator_t *subvol, fd_t *fd, dict_t **dict, const char *key)
 int
 syncop_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
-                   struct statvfs *buf)
+                   struct statvfs *buf, dict_t *xdata)
 
 {
         struct syncargs *args = NULL;
@@ -659,7 +1399,7 @@ syncop_statfs (xlator_t *subvol, loc_t *loc, struct statvfs *buf)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_statfs_cbk, subvol->fops->statfs,
-                loc);
+                loc, NULL);
 
         if (buf)
                 *buf = args.statvfs_buf;
@@ -671,7 +1411,7 @@ syncop_statfs (xlator_t *subvol, loc_t *loc, struct statvfs *buf)
 int
 syncop_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int op_ret, int op_errno,
-                    struct iatt *preop, struct iatt *postop)
+                    struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -698,7 +1438,7 @@ syncop_setattr (xlator_t *subvol, loc_t *loc, struct iatt *iatt, int valid,
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_setattr_cbk, subvol->fops->setattr,
-                loc, iatt, valid);
+                loc, iatt, valid, NULL);
 
         if (preop)
                 *preop = args.iatt1;
@@ -717,7 +1457,7 @@ syncop_fsetattr (xlator_t *subvol, fd_t *fd, struct iatt *iatt, int valid,
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_setattr_cbk, subvol->fops->fsetattr,
-                fd, iatt, valid);
+                fd, iatt, valid, NULL);
 
         if (preop)
                 *preop = args.iatt1;
@@ -731,7 +1471,7 @@ syncop_fsetattr (xlator_t *subvol, fd_t *fd, struct iatt *iatt, int valid,
 
 int32_t
 syncop_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd)
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -740,9 +1480,6 @@ syncop_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         args->op_ret   = op_ret;
         args->op_errno = op_errno;
 
-        if (op_ret != -1)
-                fd_ref (fd);
-
         __wake (args);
 
         return 0;
@@ -754,7 +1491,7 @@ syncop_open (xlator_t *subvol, loc_t *loc, int32_t flags, fd_t *fd)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_open_cbk, subvol->fops->open,
-                loc, flags, fd, 0);
+                loc, flags, fd, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -765,7 +1502,8 @@ syncop_open (xlator_t *subvol, loc_t *loc, int32_t flags, fd_t *fd)
 int32_t
 syncop_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                  int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -797,11 +1535,14 @@ syncop_readv (xlator_t *subvol, fd_t *fd, size_t size, off_t off,
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_readv_cbk, subvol->fops->readv,
-                fd, size, off, flags);
+                fd, size, off, flags, NULL);
+
+        if (args.op_ret < 0)
+                goto out;
 
         if (vector)
                 *vector = args.vector;
-        else if (args.vector)
+        else
                 GF_FREE (args.vector);
 
         if (count)
@@ -813,6 +1554,7 @@ syncop_readv (xlator_t *subvol, fd_t *fd, size_t size, off_t off,
         else if (args.iobref)
                 iobref_unref (args.iobref);
 
+out:
         errno = args.op_errno;
         return args.op_ret;
 
@@ -821,7 +1563,7 @@ syncop_readv (xlator_t *subvol, fd_t *fd, size_t size, off_t off,
 int
 syncop_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int op_ret, int op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -836,14 +1578,15 @@ syncop_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 int
-syncop_writev (xlator_t *subvol, fd_t *fd, struct iovec *vector,
+syncop_writev (xlator_t *subvol, fd_t *fd, const struct iovec *vector,
                int32_t count, off_t offset, struct iobref *iobref,
                uint32_t flags)
 {
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_writev_cbk, subvol->fops->writev,
-                fd, vector, count, offset, flags, iobref);
+                fd, (struct iovec *) vector, count, offset, flags, iobref,
+                NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -859,7 +1602,7 @@ int syncop_write (xlator_t *subvol, fd_t *fd, const char *buf, int size,
         vec.iov_base = (void *)buf;
 
         SYNCOP (subvol, (&args), syncop_writev_cbk, subvol->fops->writev,
-                fd, &vec, 1, offset, flags, iobref);
+                fd, &vec, 1, offset, flags, iobref, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -878,7 +1621,7 @@ int32_t
 syncop_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -887,8 +1630,8 @@ syncop_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         args->op_ret   = op_ret;
         args->op_errno = op_errno;
 
-        if (op_ret != -1)
-                fd_ref (fd);
+	if (buf)
+		args->iatt1 = *buf;
 
         __wake (args);
 
@@ -897,14 +1640,17 @@ syncop_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 syncop_create (xlator_t *subvol, loc_t *loc, int32_t flags, mode_t mode,
-               fd_t *fd, dict_t *dict)
+               fd_t *fd, dict_t *xdata, struct iatt *iatt)
 {
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_create_cbk, subvol->fops->create,
-                loc, flags, mode, fd, dict);
+                loc, flags, mode, 0, fd, xdata);
 
         errno = args.op_errno;
+	if (iatt)
+		*iatt = args.iatt1;
+
         return args.op_ret;
 
 }
@@ -912,7 +1658,7 @@ syncop_create (xlator_t *subvol, loc_t *loc, int32_t flags, mode_t mode,
 int
 syncop_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int op_ret, int op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -931,16 +1677,114 @@ syncop_unlink (xlator_t *subvol, loc_t *loc)
 {
         struct syncargs args = {0, };
 
-        SYNCOP (subvol, (&args), syncop_unlink_cbk, subvol->fops->unlink, loc);
+        SYNCOP (subvol, (&args), syncop_unlink_cbk, subvol->fops->unlink, loc,
+                0, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
 }
 
 int
+syncop_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+
+        __wake (args);
+
+        return 0;
+}
+
+int
+syncop_rmdir (xlator_t *subvol, loc_t *loc)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_rmdir_cbk, subvol->fops->rmdir, loc,
+                0, NULL);
+
+        errno = args.op_errno;
+        return args.op_ret;
+}
+
+
+int
+syncop_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret = op_ret;
+        args->op_errno = op_errno;
+
+        __wake (args);
+
+        return 0;
+}
+
+
+int
+syncop_link (xlator_t *subvol, loc_t *oldloc, loc_t *newloc)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_link_cbk, subvol->fops->link,
+                oldloc, newloc, NULL);
+
+        errno = args.op_errno;
+
+        return args.op_ret;
+}
+
+
+int
+syncop_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret = op_ret;
+        args->op_errno = op_errno;
+
+        __wake (args);
+
+        return 0;
+}
+
+
+int
+syncop_rename (xlator_t *subvol, loc_t *oldloc, loc_t *newloc)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_rename_cbk, subvol->fops->rename,
+                oldloc, newloc, NULL);
+
+        errno = args.op_errno;
+
+        return args.op_ret;
+}
+
+
+int
 syncop_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int op_ret, int op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -960,7 +1804,7 @@ syncop_ftruncate (xlator_t *subvol, fd_t *fd, off_t offset)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_ftruncate_cbk, subvol->fops->ftruncate,
-                fd, offset);
+                fd, offset, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -972,7 +1816,7 @@ syncop_truncate (xlator_t *subvol, loc_t *loc, off_t offset)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_ftruncate_cbk, subvol->fops->truncate,
-                loc, offset);
+                loc, offset, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -981,7 +1825,7 @@ syncop_truncate (xlator_t *subvol, loc_t *loc, off_t offset)
 int
 syncop_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf)
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -997,12 +1841,43 @@ syncop_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 int
-syncop_fsync (xlator_t *subvol, fd_t *fd)
+syncop_fsync (xlator_t *subvol, fd_t *fd, int dataonly)
 {
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_fsync_cbk, subvol->fops->fsync,
-                fd, 0);
+                fd, dataonly, NULL);
+
+        errno = args.op_errno;
+        return args.op_ret;
+
+}
+
+
+int
+syncop_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+
+        __wake (args);
+
+        return 0;
+
+}
+
+int
+syncop_flush (xlator_t *subvol, fd_t *fd)
+{
+        struct syncargs args = {0};
+
+        SYNCOP (subvol, (&args), syncop_flush_cbk, subvol->fops->flush,
+                fd, NULL);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -1011,7 +1886,7 @@ syncop_fsync (xlator_t *subvol, fd_t *fd)
 
 int
 syncop_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
+                  int32_t op_ret, int32_t op_errno, struct iatt *stbuf, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -1034,7 +1909,7 @@ syncop_fstat (xlator_t *subvol, fd_t *fd, struct iatt *stbuf)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_fstat_cbk, subvol->fops->fstat,
-                fd);
+                fd, NULL);
 
         if (stbuf)
                 *stbuf = args.iatt1;
@@ -1050,7 +1925,7 @@ syncop_stat (xlator_t *subvol, loc_t *loc, struct iatt *stbuf)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_fstat_cbk, subvol->fops->stat,
-                loc);
+                loc, NULL);
 
         if (stbuf)
                 *stbuf = args.iatt1;
@@ -1064,7 +1939,7 @@ int32_t
 syncop_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -1072,6 +1947,8 @@ syncop_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         args->op_ret   = op_ret;
         args->op_errno = op_errno;
+	if (buf)
+		args->iatt1 = *buf;
 
         __wake (args);
 
@@ -1079,14 +1956,18 @@ syncop_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 int
-syncop_symlink (xlator_t *subvol, loc_t *loc, char *newpath, dict_t *dict)
+syncop_symlink (xlator_t *subvol, loc_t *loc, const char *newpath, dict_t *dict,
+		struct iatt *iatt)
 {
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_symlink_cbk, subvol->fops->symlink,
-                newpath, loc, dict);
+                newpath, loc, 0, dict);
 
         errno = args.op_errno;
+	if (iatt)
+		*iatt = args.iatt1;
+
         return args.op_ret;
 
 }
@@ -1094,7 +1975,7 @@ syncop_symlink (xlator_t *subvol, loc_t *loc, char *newpath, dict_t *dict)
 int
 syncop_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int op_ret, int op_errno, const char *path,
-                     struct iatt *stbuf)
+                     struct iatt *stbuf, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -1117,12 +1998,11 @@ syncop_readlink (xlator_t *subvol, loc_t *loc, char **buffer, size_t size)
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_readlink_cbk, subvol->fops->readlink,
-                loc, size);
+                loc, size, NULL);
 
         if (buffer)
                 *buffer = args.buffer;
-        else if (args.buffer)
-                GF_FREE (args.buffer);
+        else GF_FREE (args.buffer);
 
         errno = args.op_errno;
         return args.op_ret;
@@ -1132,7 +2012,7 @@ int
 syncop_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         struct syncargs *args = NULL;
 
@@ -1141,6 +2021,9 @@ syncop_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         args->op_ret   = op_ret;
         args->op_errno = op_errno;
 
+	if (buf)
+		args->iatt1 = *buf;
+
         __wake (args);
 
         return 0;
@@ -1148,14 +2031,207 @@ syncop_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 syncop_mknod (xlator_t *subvol, loc_t *loc, mode_t mode, dev_t rdev,
-              dict_t *dict)
+              dict_t *dict, struct iatt *iatt)
 {
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_mknod_cbk, subvol->fops->mknod,
-                loc, mode, rdev, dict);
+                loc, mode, rdev, 0, dict);
 
         errno = args.op_errno;
+	if (iatt)
+		*iatt = args.iatt1;
+
         return args.op_ret;
 
 }
+
+
+int
+syncop_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+	if (buf)
+		args->iatt1 = *buf;
+
+        __wake (args);
+
+        return 0;
+}
+
+
+int
+syncop_mkdir (xlator_t *subvol, loc_t *loc, mode_t mode, dict_t *dict,
+	      struct iatt *iatt)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_mkdir_cbk, subvol->fops->mkdir,
+                loc, mode, 0, dict);
+
+        errno = args.op_errno;
+	if (iatt)
+		*iatt = args.iatt1;
+
+        return args.op_ret;
+
+}
+
+int
+syncop_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+        __wake (args);
+
+        return 0;
+}
+
+int
+syncop_access (xlator_t *subvol, loc_t *loc, int32_t mask)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_access_cbk, subvol->fops->access,
+                loc, mask, NULL);
+
+        errno = args.op_errno;
+        return args.op_ret;
+}
+
+
+int
+syncop_fallocate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+	struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+
+        __wake (args);
+
+	return 0;
+}
+
+int
+syncop_fallocate(xlator_t *subvol, fd_t *fd, int32_t keep_size, off_t offset,
+		 size_t len)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_fallocate_cbk, subvol->fops->fallocate,
+                fd, keep_size, offset, len, NULL);
+
+        errno = args.op_errno;
+        return args.op_ret;
+}
+
+
+int
+syncop_discard_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+	struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+
+        __wake (args);
+
+	return 0;
+}
+
+int
+syncop_discard(xlator_t *subvol, fd_t *fd, off_t offset, size_t len)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_discard_cbk, subvol->fops->discard,
+                fd, offset, len, NULL);
+
+        errno = args.op_errno;
+        return args.op_ret;
+}
+
+int
+syncop_zerofill_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+
+        __wake (args);
+
+        return 0;
+}
+
+int
+syncop_zerofill(xlator_t *subvol, fd_t *fd, off_t offset, size_t len)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_zerofill_cbk, subvol->fops->zerofill,
+                fd, offset, len, NULL);
+
+        errno = args.op_errno;
+        return args.op_ret;
+}
+
+
+int
+syncop_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	       int op_ret, int op_errno, struct gf_flock *flock,
+	       dict_t *xdata)
+{
+        struct syncargs *args = NULL;
+
+        args = cookie;
+
+        args->op_ret   = op_ret;
+        args->op_errno = op_errno;
+	if (flock)
+		args->flock = *flock;
+        __wake (args);
+
+        return 0;
+}
+
+
+int
+syncop_lk (xlator_t *subvol, fd_t *fd, int cmd, struct gf_flock *flock)
+{
+        struct syncargs args = {0, };
+
+        SYNCOP (subvol, (&args), syncop_lk_cbk, subvol->fops->lk,
+                fd, cmd, flock, NULL);
+
+        errno = args.op_errno;
+	*flock = args.flock;
+
+        return args.op_ret;
+}
diff --git a/libglusterfs/src/syncop.h b/libglusterfs/src/syncop.h
index 627fb6197..f790981f0 100644
--- a/libglusterfs/src/syncop.h
+++ b/libglusterfs/src/syncop.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _SYNCOP_H
@@ -30,8 +21,19 @@
 #include <pthread.h>
 #include <ucontext.h>
 
+#define SYNCENV_PROC_MAX 16
+#define SYNCENV_PROC_MIN 2
+#define SYNCPROC_IDLE_TIME 600
+
+/*
+ * Flags for syncopctx valid elements
+ */
+#define SYNCOPCTX_UID    0x00000001
+#define SYNCOPCTX_GID    0x00000002
+#define SYNCOPCTX_GROUPS 0x00000004
 
 struct synctask;
+struct syncproc;
 struct syncenv;
 
 
@@ -40,37 +42,91 @@ typedef int (*synctask_cbk_t) (int ret, call_frame_t *frame, void *opaque);
 typedef int (*synctask_fn_t) (void *opaque);
 
 
+typedef enum {
+        SYNCTASK_INIT = 0,
+        SYNCTASK_RUN,
+        SYNCTASK_SUSPEND,
+        SYNCTASK_WAIT,
+        SYNCTASK_DONE,
+	SYNCTASK_ZOMBIE,
+} synctask_state_t;
+
 /* for one sequential execution of @syncfn */
 struct synctask {
         struct list_head    all_tasks;
         struct syncenv     *env;
         xlator_t           *xl;
         call_frame_t       *frame;
+        call_frame_t       *opframe;
         synctask_cbk_t      synccbk;
         synctask_fn_t       syncfn;
+        synctask_state_t    state;
         void               *opaque;
         void               *stack;
-        int                 complete;
+        int                 woken;
+        int                 slept;
+        int                 ret;
+
+        uid_t               uid;
+        gid_t               gid;
 
         ucontext_t          ctx;
+        struct syncproc    *proc;
+
+        pthread_mutex_t     mutex; /* for synchronous spawning of synctask */
+        pthread_cond_t      cond;
+        int                 done;
+
+	struct list_head    waitq; /* can wait only "once" at a time */
 };
 
-/* hosts the scheduler thread and framework for executing synctasks */
-struct syncenv {
+
+struct syncproc {
         pthread_t           processor;
+        ucontext_t          sched;
+        struct syncenv     *env;
         struct synctask    *current;
+};
+
+/* hosts the scheduler thread and framework for executing synctasks */
+struct syncenv {
+        struct syncproc     proc[SYNCENV_PROC_MAX];
+        int                 procs;
 
         struct list_head    runq;
+        int                 runcount;
         struct list_head    waitq;
+        int                 waitcount;
+
+	int                 procmin;
+	int                 procmax;
 
         pthread_mutex_t     mutex;
         pthread_cond_t      cond;
 
-        ucontext_t          sched;
         size_t              stacksize;
 };
 
 
+struct synclock {
+	pthread_mutex_t     guard; /* guard the remaining members, pair @cond */
+	pthread_cond_t      cond;  /* waiting non-synctasks */
+	struct list_head    waitq; /* waiting synctasks */
+	gf_boolean_t        lock;  /* _gf_true or _gf_false, lock status */
+	struct synctask    *owner; /* NULL if current owner is not a synctask */
+};
+typedef struct synclock synclock_t;
+
+
+struct syncbarrier {
+	pthread_mutex_t     guard; /* guard the remaining members, pair @cond */
+	pthread_cond_t      cond;  /* waiting non-synctasks */
+	struct list_head    waitq; /* waiting synctasks */
+	int                 count; /* count the number of wakes */
+};
+typedef struct syncbarrier syncbarrier_t;
+
+
 struct syncargs {
         int                 op_ret;
         int                 op_errno;
@@ -83,82 +139,197 @@ struct syncargs {
         int                 count;
         struct iobref      *iobref;
         char               *buffer;
+        dict_t             *xdata;
+	struct gf_flock     flock;
+
+        /* some more _cbk needs */
+        uuid_t              uuid;
+        char               *errstr;
+        dict_t             *dict;
+        pthread_mutex_t     lock_dict;
+
+	syncbarrier_t       barrier;
 
         /* do not touch */
+        struct synctask    *task;
         pthread_mutex_t     mutex;
-        char                complete;
         pthread_cond_t      cond;
-        struct synctask    *task;
+	int                 done;
 };
 
+struct syncopctx {
+        unsigned int valid;  /* valid flags for elements that are set */
+        uid_t        uid;
+        gid_t        gid;
+        int          grpsize;
+        int          ngrps;
+        gid_t       *groups;
+};
 
-#define __yawn(args) do {                                               \
-                struct synctask *task = NULL;                           \
-                                                                        \
-                task = synctask_get ();                                 \
-                if (task) {                                             \
-                        args->task = task;                              \
-                        synctask_yawn (task);                           \
-                } else {                                                \
-                        pthread_mutex_init (&args->mutex, NULL);        \
-                        pthread_cond_init (&args->cond, NULL);          \
-                }                                                       \
+#define __yawn(args) do {                                       \
+        args->task = synctask_get ();                           \
+        if (args->task)                                         \
+            break;                                              \
+        pthread_mutex_init (&args->mutex, NULL);                \
+        pthread_cond_init (&args->cond, NULL);                  \
+        args->done = 0;                                         \
         } while (0)
 
 
-#define __yield(args) do {                                              \
-                if (args->task) {                                       \
-                        synctask_yield (args->task);                    \
-                } else {                                                \
-                        pthread_mutex_lock (&args->mutex);              \
-                        {                                               \
-                                while (!args->complete)                 \
-                                        pthread_cond_wait (&args->cond, \
-                                                           &args->mutex); \
-                        }                                               \
-                        pthread_mutex_unlock (&args->mutex);            \
-                                                                        \
-                        pthread_mutex_destroy (&args->mutex);           \
-                        pthread_cond_destroy (&args->cond);             \
-                }                                                       \
+#define __wake(args) do {                                       \
+        if (args->task) {                                       \
+                synctask_wake (args->task);                     \
+        } else {                                                \
+                pthread_mutex_lock (&args->mutex);              \
+                {                                               \
+                        args->done = 1;				\
+                        pthread_cond_signal (&args->cond);      \
+                }                                               \
+                pthread_mutex_unlock (&args->mutex);            \
+        }                                                       \
         } while (0)
 
 
-#define __wake(args) do {                                               \
-                if (args->task) {                                       \
-                        synctask_wake (args->task);                     \
-                } else {                                                \
-                        pthread_mutex_lock (&args->mutex);              \
-                        {                                               \
-                                args->complete = 1;                     \
-                                pthread_cond_broadcast (&args->cond);   \
-                        }                                               \
-                        pthread_mutex_unlock (&args->mutex);            \
-                }                                                       \
-        } while (0)
+#define __yield(args) do {						\
+	if (args->task) {				                \
+		synctask_yield (args->task);				\
+	} else {							\
+		pthread_mutex_lock (&args->mutex);			\
+		{							\
+			while (!args->done)				\
+				pthread_cond_wait (&args->cond,		\
+						   &args->mutex);	\
+		}							\
+		pthread_mutex_unlock (&args->mutex);			\
+		pthread_mutex_destroy (&args->mutex);			\
+		pthread_cond_destroy (&args->cond);			\
+	}								\
+	} while (0)
 
 
 #define SYNCOP(subvol, stb, cbk, op, params ...) do {                   \
-                call_frame_t    *frame = NULL;                          \
+                struct  synctask        *task = NULL;                   \
+                call_frame_t            *frame = NULL;                  \
+                                                                        \
+                task = synctask_get ();                                 \
+                stb->task = task;                                       \
+                if (task)                                               \
+                        frame = task->opframe;                          \
+                else                                                    \
+                        frame = syncop_create_frame (THIS);		\
                                                                         \
-                frame = syncop_create_frame ();                         \
+                if (task) {                                             \
+                        frame->root->uid = task->uid;                   \
+                        frame->root->gid = task->gid;                   \
+                }                                                       \
                                                                         \
                 __yawn (stb);                                           \
-                STACK_WIND_COOKIE (frame, cbk, (void *)stb, subvol, op, params); \
+                                                                        \
+                STACK_WIND_COOKIE (frame, cbk, (void *)stb, subvol,     \
+                                   op, params);                         \
+                                                                        \
                 __yield (stb);                                          \
+                if (task)                                               \
+                        STACK_RESET (frame->root);                      \
+                else                                                    \
+                        STACK_DESTROY (frame->root);                    \
         } while (0)
 
 
 #define SYNCENV_DEFAULT_STACKSIZE (2 * 1024 * 1024)
 
-struct syncenv * syncenv_new ();
+struct syncenv * syncenv_new (size_t stacksize, int procmin, int procmax);
 void syncenv_destroy (struct syncenv *);
+void syncenv_scale (struct syncenv *env);
 
 int synctask_new (struct syncenv *, synctask_fn_t, synctask_cbk_t, call_frame_t* frame, void *);
-void synctask_zzzz (struct synctask *task);
-void synctask_yawn (struct synctask *task);
+struct synctask *synctask_create (struct syncenv *, synctask_fn_t,
+				  synctask_cbk_t, call_frame_t *, void *);
+int synctask_join (struct synctask *task);
 void synctask_wake (struct synctask *task);
 void synctask_yield (struct synctask *task);
+void synctask_waitfor (struct synctask *task, int count);
+
+#define synctask_barrier_init(args) syncbarrier_init (&args->barrier)
+#define synctask_barrier_wait(args, n) syncbarrier_wait (&args->barrier, n)
+#define synctask_barrier_wake(args) syncbarrier_wake (&args->barrier)
+
+int synctask_setid (struct synctask *task, uid_t uid, gid_t gid);
+#define SYNCTASK_SETID(uid, gid) synctask_setid (synctask_get(), uid, gid);
+
+int syncopctx_setfsuid (void *uid);
+int syncopctx_setfsgid (void *gid);
+int syncopctx_setfsgroups (int count, const void *groups);
+
+static inline call_frame_t *
+syncop_create_frame (xlator_t *this)
+{
+	call_frame_t     *frame = NULL;
+	int               ngrps = -1;
+	struct syncopctx *opctx = NULL;
+
+	frame = create_frame (this, this->ctx->pool);
+	if (!frame)
+		return NULL;
+
+	frame->root->pid = getpid ();
+
+	opctx = syncopctx_getctx ();
+	if (opctx && (opctx->valid & SYNCOPCTX_UID))
+		frame->root->uid = opctx->uid;
+	else
+		frame->root->uid = geteuid ();
+
+	if (opctx && (opctx->valid & SYNCOPCTX_GID))
+		frame->root->gid = opctx->gid;
+	else
+		frame->root->gid = getegid ();
+
+	if (opctx && (opctx->valid & SYNCOPCTX_GROUPS)) {
+		ngrps = opctx->ngrps;
+
+		if (ngrps != 0 && opctx->groups != NULL) {
+			if (call_stack_alloc_groups (frame->root, ngrps) != 0) {
+				STACK_DESTROY (frame->root);
+				return NULL;
+			}
+
+			memcpy (frame->root->groups, opctx->groups,
+				(sizeof (gid_t) * ngrps));
+		}
+	}
+	else {
+		ngrps = getgroups (0, 0);
+		if (ngrps < 0) {
+			STACK_DESTROY (frame->root);
+			return NULL;
+		}
+
+		if (call_stack_alloc_groups (frame->root, ngrps) != 0) {
+			STACK_DESTROY (frame->root);
+			return NULL;
+		}
+
+		if (getgroups (ngrps, frame->root->groups) < 0) {
+			STACK_DESTROY (frame->root);
+			return NULL;
+		}
+	}
+
+	return frame;
+}
+
+int synclock_init (synclock_t *lock);
+int synclock_destory (synclock_t *lock);
+int synclock_lock (synclock_t *lock);
+int synclock_trylock (synclock_t *lock);
+int synclock_unlock (synclock_t *lock);
+
+
+int syncbarrier_init (syncbarrier_t *barrier);
+int syncbarrier_wait (syncbarrier_t *barrier, int waitfor);
+int syncbarrier_wake (syncbarrier_t *barrier);
+int syncbarrier_destroy (syncbarrier_t *barrier);
 
 int syncop_lookup (xlator_t *subvol, loc_t *loc, dict_t *xattr_req,
                    /* out */
@@ -169,6 +340,9 @@ int syncop_readdirp (xlator_t *subvol, fd_t *fd, size_t size, off_t off,
                      /* out */
                      gf_dirent_t *entries);
 
+int syncop_readdir (xlator_t *subvol, fd_t *fd, size_t size, off_t off,
+                    gf_dirent_t *entries);
+
 int syncop_opendir (xlator_t *subvol, loc_t *loc, fd_t *fd);
 
 int syncop_setattr (xlator_t *subvol, loc_t *loc, struct iatt *iatt, int valid,
@@ -190,13 +364,13 @@ int syncop_removexattr (xlator_t *subvol, loc_t *loc, const char *name);
 int syncop_fremovexattr (xlator_t *subvol, fd_t *fd, const char *name);
 
 int syncop_create (xlator_t *subvol, loc_t *loc, int32_t flags, mode_t mode,
-                   fd_t *fd, dict_t *dict);
+                   fd_t *fd, dict_t *dict, struct iatt *iatt);
 int syncop_open (xlator_t *subvol, loc_t *loc, int32_t flags, fd_t *fd);
 int syncop_close (fd_t *fd);
 
 int syncop_write (xlator_t *subvol, fd_t *fd, const char *buf, int size,
                   off_t offset, struct iobref *iobref, uint32_t flags);
-int syncop_writev (xlator_t *subvol, fd_t *fd, struct iovec *vector,
+int syncop_writev (xlator_t *subvol, fd_t *fd, const struct iovec *vector,
                    int32_t count, off_t offset, struct iobref *iobref,
                    uint32_t flags);
 int syncop_readv (xlator_t *subvol, fd_t *fd, size_t size, off_t off,
@@ -208,14 +382,31 @@ int syncop_ftruncate (xlator_t *subvol, fd_t *fd, off_t offset);
 int syncop_truncate (xlator_t *subvol, loc_t *loc, off_t offset);
 
 int syncop_unlink (xlator_t *subvol, loc_t *loc);
+int syncop_rmdir (xlator_t *subvol, loc_t *loc);
 
-int syncop_fsync (xlator_t *subvol, fd_t *fd);
+int syncop_fsync (xlator_t *subvol, fd_t *fd, int dataonly);
+int syncop_flush (xlator_t *subvol, fd_t *fd);
 int syncop_fstat (xlator_t *subvol, fd_t *fd, struct iatt *stbuf);
 int syncop_stat (xlator_t *subvol, loc_t *loc, struct iatt *stbuf);
 
-int syncop_symlink (xlator_t *subvol, loc_t *loc, char *newpath, dict_t *dict);
+int syncop_symlink (xlator_t *subvol, loc_t *loc, const char *newpath,
+                    dict_t *dict, struct iatt *iatt);
 int syncop_readlink (xlator_t *subvol, loc_t *loc, char **buffer, size_t size);
 int syncop_mknod (xlator_t *subvol, loc_t *loc, mode_t mode, dev_t rdev,
-                  dict_t *dict);
+                  dict_t *dict, struct iatt *iatt);
+int syncop_mkdir (xlator_t *subvol, loc_t *loc, mode_t mode, dict_t *dict,
+		  struct iatt *iatt);
+int syncop_link (xlator_t *subvol, loc_t *oldloc, loc_t *newloc);
+int syncop_fsyncdir (xlator_t *subvol, fd_t *fd, int datasync);
+int syncop_access (xlator_t *subvol, loc_t *loc, int32_t mask);
+int syncop_fallocate(xlator_t *subvol, fd_t *fd, int32_t keep_size, off_t offset,
+		     size_t len);
+int syncop_discard(xlator_t *subvol, fd_t *fd, off_t offset, size_t len);
+
+int syncop_zerofill(xlator_t *subvol, fd_t *fd, off_t offset, size_t len);
+
+int syncop_rename (xlator_t *subvol, loc_t *oldloc, loc_t *newloc);
+
+int syncop_lk (xlator_t *subvol, fd_t *fd, int cmd, struct gf_flock *flock);
 
 #endif /* _SYNCOP_H */
diff --git a/libglusterfs/src/syscall.c b/libglusterfs/src/syscall.c
index 8e2d56b8d..e8954cc23 100644
--- a/libglusterfs/src/syscall.c
+++ b/libglusterfs/src/syscall.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -241,7 +232,12 @@ sys_statvfs (const char *path, struct statvfs *buf)
 int
 sys_close (int fd)
 {
-        return close (fd);
+        int ret = -1;
+
+        if (fd >= 0)
+                ret = close (fd);
+
+        return ret;
 }
 
 
@@ -462,3 +458,26 @@ sys_access (const char *pathname, int mode)
 {
         return access (pathname, mode);
 }
+
+
+int
+sys_fallocate(int fd, int mode, off_t offset, off_t len)
+{
+#ifdef HAVE_FALLOCATE
+	return fallocate(fd, mode, offset, len);
+#endif
+
+#ifdef HAVE_POSIX_FALLOCATE
+	if (mode) {
+		/* keep size not supported */
+		errno = EOPNOTSUPP;
+		return -1;
+	}
+
+	return posix_fallocate(fd, offset, len);
+#endif
+
+	errno = ENOSYS;
+	return -1;
+}
+
diff --git a/libglusterfs/src/syscall.h b/libglusterfs/src/syscall.h
index f5325b4cd..f1c9f58c3 100644
--- a/libglusterfs/src/syscall.h
+++ b/libglusterfs/src/syscall.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __SYSCALL_H__
@@ -148,4 +139,6 @@ sys_access (const char *pathname, int mode);
 int
 sys_ftruncate (int fd, off_t length);
 
+int sys_fallocate(int fd, int mode, off_t offset, off_t len);
+
 #endif /* __SYSCALL_H__ */
diff --git a/libglusterfs/src/timer.c b/libglusterfs/src/timer.c
index d8f53a8d1..a059cc212 100644
--- a/libglusterfs/src/timer.c
+++ b/libglusterfs/src/timer.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -26,19 +17,18 @@
 #include "logging.h"
 #include "common-utils.h"
 #include "globals.h"
-
-#define TS(tv) ((((unsigned long long) tv.tv_sec) * 1000000) + (tv.tv_usec))
+#include "timespec.h"
 
 gf_timer_t *
 gf_timer_call_after (glusterfs_ctx_t *ctx,
-                     struct timeval delta,
+                     struct timespec delta,
                      gf_timer_cbk_t callbk,
                      void *data)
 {
         gf_timer_registry_t *reg = NULL;
         gf_timer_t *event = NULL;
         gf_timer_t *trav = NULL;
-        unsigned long long at = 0L;
+        uint64_t at = 0;
 
         if (ctx == NULL)
         {
@@ -57,10 +47,8 @@ gf_timer_call_after (glusterfs_ctx_t *ctx,
         if (!event) {
                 return NULL;
         }
-        gettimeofday (&event->at, NULL);
-        event->at.tv_usec = ((event->at.tv_usec + delta.tv_usec) % 1000000);
-        event->at.tv_sec += ((event->at.tv_usec + delta.tv_usec) / 1000000);
-        event->at.tv_sec += delta.tv_sec;
+        timespec_now (&event->at);
+        timespec_adjust_delta (&event->at, delta);
         at = TS (event->at);
         event->callbk = callbk;
         event->data = data;
@@ -136,7 +124,7 @@ void *
 gf_timer_proc (void *ctx)
 {
         gf_timer_registry_t *reg = NULL;
-	const struct timespec sleepts = {.tv_sec = 1, .tv_nsec = 0, };
+        const struct timespec sleepts = {.tv_sec = 1, .tv_nsec = 0, };
 
         if (ctx == NULL)
         {
@@ -151,14 +139,14 @@ gf_timer_proc (void *ctx)
         }
 
         while (!reg->fin) {
-                unsigned long long now;
-                struct timeval now_tv;
+                uint64_t now;
+                struct timespec now_ts;
                 gf_timer_t *event = NULL;
 
-                gettimeofday (&now_tv, NULL);
-                now = TS (now_tv);
+                timespec_now (&now_ts);
+                now = TS (now_ts);
                 while (1) {
-                        unsigned long long at;
+                        uint64_t at;
                         char need_cbk = 0;
 
                         pthread_mutex_lock (&reg->lock);
@@ -222,7 +210,7 @@ gf_timer_registry_init (glusterfs_ctx_t *ctx)
                 reg->stale.prev = &reg->stale;
 
                 ctx->timer = reg;
-                pthread_create (&reg->th, NULL, gf_timer_proc, ctx);
+                gf_thread_create (&reg->th, NULL, gf_timer_proc, ctx);
         }
 out:
         return ctx->timer;
diff --git a/libglusterfs/src/timer.h b/libglusterfs/src/timer.h
index 3f1dc7adb..2f963adbf 100644
--- a/libglusterfs/src/timer.h
+++ b/libglusterfs/src/timer.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _TIMER_H
@@ -34,7 +25,7 @@ typedef void (*gf_timer_cbk_t) (void *);
 
 struct _gf_timer {
         struct _gf_timer *next, *prev;
-        struct timeval    at;
+        struct timespec    at;
         gf_timer_cbk_t    callbk;
         void             *data;
         xlator_t         *xl;
@@ -53,7 +44,7 @@ typedef struct _gf_timer_registry gf_timer_registry_t;
 
 gf_timer_t *
 gf_timer_call_after (glusterfs_ctx_t *ctx,
-                     struct timeval delta,
+                     struct timespec delta,
                      gf_timer_cbk_t cbk,
                      void *data);
 
diff --git a/libglusterfs/src/timespec.c b/libglusterfs/src/timespec.c
new file mode 100644
index 000000000..a0c281a1e
--- /dev/null
+++ b/libglusterfs/src/timespec.c
@@ -0,0 +1,68 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <stdio.h>
+#include <inttypes.h>
+#if defined GF_LINUX_HOST_OS || defined GF_SOLARIS_HOST_OS || defined GF_BSD_HOST_OS
+#include <time.h>
+#include <sys/time.h>
+#endif
+
+#if defined GF_DARWIN_HOST_OS
+#include <mach/mach_time.h>
+#endif
+
+#include "logging.h"
+#include "time.h"
+
+
+void tv2ts (struct timeval tv, struct timespec *ts)
+{
+        ts->tv_sec  = tv.tv_sec;
+        ts->tv_nsec = tv.tv_usec * 1000;
+}
+
+void timespec_now (struct timespec *ts)
+{
+#if defined GF_LINUX_HOST_OS || defined GF_SOLARIS_HOST_OS || defined GF_BSD_HOST_OS
+
+        if (0 == clock_gettime(CLOCK_MONOTONIC, ts))
+                return;
+        else {
+                struct timeval tv;
+                if (0 == gettimeofday(&tv, NULL))
+                        tv2ts(tv, ts);
+        }
+#elif defined GF_DARWIN_HOST_OS
+        mach_timebase_info_data_t tb = { 0 };
+        static double timebase = 0.0;
+        uint64_t time = 0;
+        mach_timebase_info (&tb);
+
+        timebase *= info.numer;
+        timebase /= info.denom;
+
+        time = mach_absolute_time();
+        time *= timebase;
+
+        ts->tv_sec = (time * NANO);
+        ts->tv_nsec = (time - (ts.tv_sec * GIGA));
+
+#endif /* Platform verification */
+        gf_log_callingfn ("timer", GF_LOG_DEBUG, "%"PRIu64".%09"PRIu64,
+                          ts->tv_sec, ts->tv_nsec);
+}
+
+void timespec_adjust_delta (struct timespec *ts, struct timespec delta)
+{
+        ts->tv_nsec = ((ts->tv_nsec + delta.tv_nsec) % 1000000000);
+        ts->tv_sec += ((ts->tv_nsec + delta.tv_nsec) / 1000000000);
+        ts->tv_sec += delta.tv_sec;
+}
diff --git a/libglusterfs/src/timespec.h b/libglusterfs/src/timespec.h
new file mode 100644
index 000000000..490255df9
--- /dev/null
+++ b/libglusterfs/src/timespec.h
@@ -0,0 +1,24 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __INCLUDE_TIMESPEC_H__
+#define __INCLUDE_TIMESPEC_H__
+
+#include <stdint.h>
+
+#define TS(ts)  ((ts.tv_sec * 1000000000LL) + ts.tv_nsec)
+#define NANO (+1.0E-9)
+#define GIGA UINT64_C(1000000000)
+
+void tv2ts (struct timeval tv, struct timespec *ts);
+void timespec_now (struct timespec *ts);
+void timespec_adjust_delta (struct timespec *ts, struct timespec delta);
+
+#endif /*  __INCLUDE_TIMESPEC_H__ */
diff --git a/libglusterfs/src/trie.c b/libglusterfs/src/trie.c
index 0572bfd3e..f96bbebf6 100644
--- a/libglusterfs/src/trie.c
+++ b/libglusterfs/src/trie.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <stdio.h>
@@ -123,8 +114,7 @@ trienode_free (trienode_t *node)
                         trienode_free (trav);
         }
 
-        if (node->data)
-                GF_FREE (node->data);
+        GF_FREE (node->data);
         GF_FREE (node);
 }
 
@@ -382,8 +372,7 @@ trie_measure_vec (trie_t *trie, const char *word, struct trienodevec *nodevec)
 static int
 trienode_reset (trienode_t *node, void *data)
 {
-        if (node->data)
-                GF_FREE (node->data);
+        GF_FREE (node->data);
 
         return 0;
 }
diff --git a/libglusterfs/src/trie.h b/libglusterfs/src/trie.h
index 2c2e05320..0356e6621 100644
--- a/libglusterfs/src/trie.h
+++ b/libglusterfs/src/trie.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _TRIE_H_
diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c
index 428357633..a277c58a8 100644
--- a/libglusterfs/src/xlator.c
+++ b/libglusterfs/src/xlator.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -88,6 +79,9 @@ fill_defaults (xlator_t *xl)
         SET_DEFAULT_FOP (fxattrop);
         SET_DEFAULT_FOP (setattr);
         SET_DEFAULT_FOP (fsetattr);
+	SET_DEFAULT_FOP (fallocate);
+	SET_DEFAULT_FOP (discard);
+        SET_DEFAULT_FOP (zerofill);
 
         SET_DEFAULT_FOP (getspec);
 
@@ -128,18 +122,9 @@ xlator_volopt_dynload (char *xlator_type, void **dl_handle,
         int                     ret = -1;
         char                    *name = NULL;
         void                    *handle = NULL;
-        volume_opt_list_t       *vol_opt = NULL;
 
         GF_VALIDATE_OR_GOTO ("xlator", xlator_type, out);
 
-        GF_ASSERT (dl_handle);
-
-        if (*dl_handle)
-                if (dlclose (*dl_handle))
-                        gf_log ("xlator", GF_LOG_WARNING, "Unable to close "
-                                  "previously opened handle( may be stale)."
-                                  "Ignoring the invalid handle");
-
         ret = gf_asprintf (&name, "%s/%s.so", XLATORDIR, xlator_type);
         if (-1 == ret) {
                 gf_log ("xlator", GF_LOG_ERROR, "asprintf failed");
@@ -155,28 +140,20 @@ xlator_volopt_dynload (char *xlator_type, void **dl_handle,
                 gf_log ("xlator", GF_LOG_WARNING, "%s", dlerror ());
                 goto out;
         }
-        *dl_handle = handle;
-
 
-        vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
-                         gf_common_mt_volume_opt_list_t);
-
-        if (!vol_opt) {
-                goto out;
-        }
-
-        if (!(vol_opt->given_opt = dlsym (handle, "options"))) {
+        if (!(opt_list->given_opt = dlsym (handle, "options"))) {
                 dlerror ();
-                gf_log ("xlator", GF_LOG_DEBUG,
-                         "Strict option validation not enforced -- neglecting");
+                gf_log ("xlator", GF_LOG_ERROR,
+                        "Failed to load xlator opt table");
+                goto out;
         }
-        opt_list->given_opt = vol_opt->given_opt;
 
-        INIT_LIST_HEAD (&vol_opt->list);
-        list_add_tail (&vol_opt->list, &opt_list->list);
+        *dl_handle = handle;
 
         ret = 0;
  out:
+        GF_FREE (name);
+
         gf_log ("xlator", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 
@@ -190,7 +167,7 @@ xlator_dynload (xlator_t *xl)
         char              *name = NULL;
         void              *handle = NULL;
         volume_opt_list_t *vol_opt = NULL;
-
+        class_methods_t   *vtbl = NULL;
 
         GF_VALIDATE_OR_GOTO ("xlator", xl, out);
 
@@ -225,40 +202,55 @@ xlator_dynload (xlator_t *xl)
                 goto out;
         }
 
-        if (!(xl->init = dlsym (handle, "init"))) {
-                gf_log ("xlator", GF_LOG_WARNING, "dlsym(init) on %s",
-                        dlerror ());
-                goto out;
-        }
+        /*
+         * If class_methods exists, its contents override any definitions of
+         * init or fini for that translator.  Otherwise, we fall back to the
+         * older method of looking for init and fini directly.
+         */
+        vtbl = dlsym(handle,"class_methods");
+        if (vtbl) {
+                xl->init        = vtbl->init;
+                xl->fini        = vtbl->fini;
+                xl->reconfigure = vtbl->reconfigure;
+                xl->notify      = vtbl->notify;
+        }
+        else {
+                if (!(*VOID(&xl->init) = dlsym (handle, "init"))) {
+                        gf_log ("xlator", GF_LOG_WARNING, "dlsym(init) on %s",
+                                dlerror ());
+                        goto out;
+                }
 
-        if (!(xl->fini = dlsym (handle, "fini"))) {
-                gf_log ("xlator", GF_LOG_WARNING, "dlsym(fini) on %s",
-                        dlerror ());
-                goto out;
-        }
+                if (!(*VOID(&(xl->fini)) = dlsym (handle, "fini"))) {
+                        gf_log ("xlator", GF_LOG_WARNING, "dlsym(fini) on %s",
+                                dlerror ());
+                        goto out;
+                }
+                if (!(*VOID(&(xl->reconfigure)) = dlsym (handle,
+                                                         "reconfigure"))) {
+                        gf_log ("xlator", GF_LOG_TRACE,
+                                "dlsym(reconfigure) on %s -- neglecting",
+                                dlerror());
+                }
+                if (!(*VOID(&(xl->notify)) = dlsym (handle, "notify"))) {
+                        gf_log ("xlator", GF_LOG_TRACE,
+                                "dlsym(notify) on %s -- neglecting",
+                                dlerror ());
+                }
 
-        if (!(xl->notify = dlsym (handle, "notify"))) {
-                gf_log ("xlator", GF_LOG_DEBUG,
-                        "dlsym(notify) on %s -- neglecting", dlerror ());
         }
 
         if (!(xl->dumpops = dlsym (handle, "dumpops"))) {
-                gf_log ("xlator", GF_LOG_DEBUG,
+                gf_log ("xlator", GF_LOG_TRACE,
                         "dlsym(dumpops) on %s -- neglecting", dlerror ());
         }
 
-        if (!(xl->mem_acct_init = dlsym (handle, "mem_acct_init"))) {
-                gf_log (xl->name, GF_LOG_DEBUG,
+        if (!(*VOID(&(xl->mem_acct_init)) = dlsym (handle, "mem_acct_init"))) {
+                gf_log (xl->name, GF_LOG_TRACE,
                         "dlsym(mem_acct_init) on %s -- neglecting",
                         dlerror ());
         }
 
-        if (!(xl->reconfigure = dlsym (handle, "reconfigure"))) {
-                gf_log ("xlator", GF_LOG_DEBUG,
-                        "dlsym(reconfigure) on %s -- neglecting",
-                        dlerror());
-        }
-
         vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
                          gf_common_mt_volume_opt_list_t);
 
@@ -268,9 +260,10 @@ xlator_dynload (xlator_t *xl)
 
         if (!(vol_opt->given_opt = dlsym (handle, "options"))) {
                 dlerror ();
-                gf_log (xl->name, GF_LOG_DEBUG,
+                gf_log (xl->name, GF_LOG_TRACE,
                         "Strict option validation not enforced -- neglecting");
         }
+        INIT_LIST_HEAD (&vol_opt->list);
         list_add_tail (&vol_opt->list, &xl->volume_options);
 
         fill_defaults (xl);
@@ -278,8 +271,7 @@ xlator_dynload (xlator_t *xl)
         ret = 0;
 
 out:
-        if (name)
-                GF_FREE (name);
+        GF_FREE (name);
         return ret;
 }
 
@@ -329,6 +321,24 @@ out:
 }
 
 
+void
+xlator_foreach_depth_first (xlator_t *this,
+			    void (*fn)(xlator_t *each, void *data),
+			    void *data)
+{
+	xlator_list_t *subv = NULL;
+
+	subv = this->children;
+
+	while (subv) {
+		xlator_foreach_depth_first (subv->xlator, fn, data);
+		subv = subv->next;
+	}
+
+	fn (this, data);
+}
+
+
 xlator_t *
 xlator_search_by_name (xlator_t *any, const char *name)
 {
@@ -352,7 +362,6 @@ out:
         return search;
 }
 
-
 static int
 __xlator_init(xlator_t *xl)
 {
@@ -430,6 +439,9 @@ xlator_fini_rec (xlator_t *xl)
 
                         xl->fini (xl);
 
+                        if (xl->local_pool)
+                                mem_pool_destroy (xl->local_pool);
+
                         THIS = old_THIS;
                 } else {
                         gf_log (xl->name, GF_LOG_DEBUG, "No fini() found");
@@ -465,12 +477,12 @@ xlator_mem_acct_init (xlator_t *xl, int num_types)
         int             i = 0;
         int             ret = 0;
 
-        if (!gf_mem_acct_is_enabled())
-                return 0;
-
         if (!xl)
                 return -1;
 
+        if (!xl->ctx->mem_acct_enable)
+                return 0;
+
         xl->mem_acct.num_types = num_types;
 
         xl->mem_acct.rec = CALLOC(num_types, sizeof(struct mem_acct_rec));
@@ -504,10 +516,26 @@ out:
         return;
 }
 
+int
+xlator_list_destroy (xlator_list_t *list)
+{
+        xlator_list_t *next = NULL;
+
+        while (list) {
+                next = list->next;
+                GF_FREE (list);
+                list = next;
+        }
+
+        return 0;
+}
+
 
 int
 xlator_tree_free (xlator_t *tree)
 {
+        volume_opt_list_t *vol_opt = NULL;
+        volume_opt_list_t *tmp     = NULL;
         xlator_t *trav = tree;
         xlator_t *prev = tree;
 
@@ -518,9 +546,19 @@ xlator_tree_free (xlator_t *tree)
 
         while (prev) {
                 trav = prev->next;
-                dict_destroy (prev->options);
+                if (prev->dlhandle)
+                        dlclose (prev->dlhandle);
+                dict_unref (prev->options);
                 GF_FREE (prev->name);
                 GF_FREE (prev->type);
+                xlator_list_destroy (prev->children);
+                xlator_list_destroy (prev->parents);
+
+                list_for_each_entry_safe (vol_opt, tmp, &prev->volume_options,
+                                          list) {
+                        list_del_init (&vol_opt->list);
+                        GF_FREE (vol_opt);
+                }
                 GF_FREE (prev);
                 prev = trav;
         }
@@ -549,6 +587,67 @@ loc_wipe (loc_t *loc)
         memset (loc, 0, sizeof (*loc));
 }
 
+int
+loc_path (loc_t *loc, const char *bname)
+{
+        int     ret = 0;
+
+        if (loc->path)
+                goto out;
+
+        ret = -1;
+
+        if (bname && !strlen (bname))
+                bname = NULL;
+
+        if (!bname)
+                goto inode_path;
+
+        if (loc->parent && !uuid_is_null (loc->parent->gfid)) {
+                ret = inode_path (loc->parent, bname, (char**)&loc->path);
+        } else if (!uuid_is_null (loc->pargfid)) {
+                ret = gf_asprintf ((char**)&loc->path, INODE_PATH_FMT"/%s",
+                                   uuid_utoa (loc->pargfid), bname);
+        }
+
+        if (loc->path)
+                goto out;
+
+inode_path:
+        if (loc->inode && !uuid_is_null (loc->inode->gfid)) {
+                ret = inode_path (loc->inode, NULL, (char **)&loc->path);
+        } else if (!uuid_is_null (loc->gfid)) {
+                ret = gf_asprintf ((char**)&loc->path, INODE_PATH_FMT,
+                                   uuid_utoa (loc->gfid));
+        }
+out:
+        return ret;
+}
+
+void
+loc_gfid (loc_t *loc, uuid_t gfid)
+{
+        if (!gfid)
+                goto out;
+        uuid_clear (gfid);
+
+        if (!loc)
+                goto out;
+        else if (!uuid_is_null (loc->gfid))
+                uuid_copy (gfid, loc->gfid);
+        else if (loc->inode && (!uuid_is_null (loc->inode->gfid)))
+                uuid_copy (gfid, loc->inode->gfid);
+out:
+        return;
+}
+
+char*
+loc_gfid_utoa (loc_t *loc)
+{
+        uuid_t gfid;
+        loc_gfid (loc, gfid);
+        return uuid_utoa (gfid);
+}
 
 int
 loc_copy (loc_t *dst, loc_t *src)
@@ -560,6 +659,7 @@ loc_copy (loc_t *dst, loc_t *src)
 
         uuid_copy (dst->gfid, src->gfid);
         uuid_copy (dst->pargfid, src->pargfid);
+        uuid_copy (dst->gfid, src->gfid);
 
         if (src->inode)
                 dst->inode = inode_ref (src->inode);
@@ -567,45 +667,38 @@ loc_copy (loc_t *dst, loc_t *src)
         if (src->parent)
                 dst->parent = inode_ref (src->parent);
 
-        dst->path = gf_strdup (src->path);
+        if (src->path) {
+                dst->path = gf_strdup (src->path);
 
-        if (!dst->path)
-                goto out;
+                if (!dst->path)
+                        goto out;
 
-        dst->name = strrchr (dst->path, '/');
-        if (dst->name)
-                dst->name++;
+                if (src->name)
+                        dst->name = strrchr (dst->path, '/');
+                if (dst->name)
+                        dst->name++;
+        }
 
         ret = 0;
 out:
-        if (ret == -1) {
-                if (dst->inode)
-                        inode_unref (dst->inode);
-
-                if (dst->parent)
-                        inode_unref (dst->parent);
-        }
+        if (ret == -1)
+                loc_wipe (dst);
 
 err:
         return ret;
 }
 
-
-int
-xlator_list_destroy (xlator_list_t *list)
+gf_boolean_t
+loc_is_root (loc_t *loc)
 {
-        xlator_list_t *next = NULL;
-
-        while (list) {
-                next = list->next;
-                GF_FREE (list);
-                list = next;
+        if (loc && __is_root_gfid (loc->gfid)) {
+                return _gf_true;
+        } else if (loc && loc->inode && __is_root_gfid (loc->inode->gfid)) {
+                return _gf_true;
         }
-
-        return 0;
+        return _gf_false;
 }
 
-
 int
 xlator_destroy (xlator_t *xl)
 {
@@ -615,10 +708,8 @@ xlator_destroy (xlator_t *xl)
         if (!xl)
                 return 0;
 
-        if (xl->name)
-                GF_FREE (xl->name);
-        if (xl->type)
-                GF_FREE (xl->type);
+        GF_FREE (xl->name);
+        GF_FREE (xl->type);
         if (xl->dlhandle)
                 dlclose (xl->dlhandle);
         if (xl->options)
@@ -693,7 +784,7 @@ is_gf_log_command (xlator_t *this, const char *name, char *value)
                 goto out;
         }
 
-        ctx = glusterfs_ctx_get();
+        ctx = this->ctx;
         if (!ctx)
                 goto out;
         if (!ctx->active)
diff --git a/libglusterfs/src/xlator.h b/libglusterfs/src/xlator.h
index c8eff3353..b57e5873e 100644
--- a/libglusterfs/src/xlator.h
+++ b/libglusterfs/src/xlator.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _XLATOR_H
@@ -29,7 +20,7 @@
 #include <stdint.h>
 #include <inttypes.h>
 
-
+#include "event-history.h"
 #include "logging.h"
 #include "common-utils.h"
 #include "dict.h"
@@ -75,6 +66,7 @@ typedef int32_t (*event_notify_fn_t) (xlator_t *this, int32_t event, void *data,
 #include "globals.h"
 #include "iatt.h"
 #include "options.h"
+#include "client_t.h"
 
 
 struct _loc {
@@ -106,7 +98,8 @@ typedef int32_t (*fop_rchecksum_cbk_t) (call_frame_t *frame,
                                         int32_t op_ret,
                                         int32_t op_errno,
                                         uint32_t weak_checksum,
-                                        uint8_t *strong_checksum);
+                                        uint8_t *strong_checksum,
+                                        dict_t *xdata);
 
 
 typedef int32_t (*fop_getspec_t) (call_frame_t *frame,
@@ -117,7 +110,7 @@ typedef int32_t (*fop_getspec_t) (call_frame_t *frame,
 typedef int32_t (*fop_rchecksum_t) (call_frame_t *frame,
                                     xlator_t *this,
                                     fd_t *fd, off_t offset,
-                                    int32_t len);
+                                    int32_t len, dict_t *xdata);
 
 
 typedef int32_t (*fop_lookup_cbk_t) (call_frame_t *frame,
@@ -127,7 +120,7 @@ typedef int32_t (*fop_lookup_cbk_t) (call_frame_t *frame,
                                      int32_t op_errno,
                                      inode_t *inode,
                                      struct iatt *buf,
-                                     dict_t *xattr,
+                                     dict_t *xdata,
                                      struct iatt *postparent);
 
 typedef int32_t (*fop_stat_cbk_t) (call_frame_t *frame,
@@ -135,14 +128,14 @@ typedef int32_t (*fop_stat_cbk_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    int32_t op_ret,
                                    int32_t op_errno,
-                                   struct iatt *buf);
+                                   struct iatt *buf, dict_t *xdata);
 
 typedef int32_t (*fop_fstat_cbk_t) (call_frame_t *frame,
                                     void *cookie,
                                     xlator_t *this,
                                     int32_t op_ret,
                                     int32_t op_errno,
-                                    struct iatt *buf);
+                                    struct iatt *buf, dict_t *xdata);
 
 typedef int32_t (*fop_truncate_cbk_t) (call_frame_t *frame,
                                        void *cookie,
@@ -150,7 +143,7 @@ typedef int32_t (*fop_truncate_cbk_t) (call_frame_t *frame,
                                        int32_t op_ret,
                                        int32_t op_errno,
                                        struct iatt *prebuf,
-                                       struct iatt *postbuf);
+                                       struct iatt *postbuf, dict_t *xdata);
 
 typedef int32_t (*fop_ftruncate_cbk_t) (call_frame_t *frame,
                                         void *cookie,
@@ -158,13 +151,13 @@ typedef int32_t (*fop_ftruncate_cbk_t) (call_frame_t *frame,
                                         int32_t op_ret,
                                         int32_t op_errno,
                                         struct iatt *prebuf,
-                                        struct iatt *postbuf);
+                                        struct iatt *postbuf, dict_t *xdata);
 
 typedef int32_t (*fop_access_cbk_t) (call_frame_t *frame,
                                      void *cookie,
                                      xlator_t *this,
                                      int32_t op_ret,
-                                     int32_t op_errno);
+                                     int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_readlink_cbk_t) (call_frame_t *frame,
                                        void *cookie,
@@ -172,7 +165,7 @@ typedef int32_t (*fop_readlink_cbk_t) (call_frame_t *frame,
                                        int32_t op_ret,
                                        int32_t op_errno,
                                        const char *path,
-                                       struct iatt *buf);
+                                       struct iatt *buf, dict_t *xdata);
 
 typedef int32_t (*fop_mknod_cbk_t) (call_frame_t *frame,
                                     void *cookie,
@@ -182,7 +175,7 @@ typedef int32_t (*fop_mknod_cbk_t) (call_frame_t *frame,
                                     inode_t *inode,
                                     struct iatt *buf,
                                     struct iatt *preparent,
-                                    struct iatt *postparent);
+                                    struct iatt *postparent, dict_t *xdata);
 
 typedef int32_t (*fop_mkdir_cbk_t) (call_frame_t *frame,
                                     void *cookie,
@@ -192,7 +185,7 @@ typedef int32_t (*fop_mkdir_cbk_t) (call_frame_t *frame,
                                     inode_t *inode,
                                     struct iatt *buf,
                                     struct iatt *preparent,
-                                    struct iatt *postparent);
+                                    struct iatt *postparent, dict_t *xdata);
 
 typedef int32_t (*fop_unlink_cbk_t) (call_frame_t *frame,
                                      void *cookie,
@@ -200,7 +193,7 @@ typedef int32_t (*fop_unlink_cbk_t) (call_frame_t *frame,
                                      int32_t op_ret,
                                      int32_t op_errno,
                                      struct iatt *preparent,
-                                     struct iatt *postparent);
+                                     struct iatt *postparent, dict_t *xdata);
 
 typedef int32_t (*fop_rmdir_cbk_t) (call_frame_t *frame,
                                     void *cookie,
@@ -208,7 +201,7 @@ typedef int32_t (*fop_rmdir_cbk_t) (call_frame_t *frame,
                                     int32_t op_ret,
                                     int32_t op_errno,
                                     struct iatt *preparent,
-                                    struct iatt *postparent);
+                                    struct iatt *postparent, dict_t *xdata);
 
 typedef int32_t (*fop_symlink_cbk_t) (call_frame_t *frame,
                                       void *cookie,
@@ -218,7 +211,7 @@ typedef int32_t (*fop_symlink_cbk_t) (call_frame_t *frame,
                                       inode_t *inode,
                                       struct iatt *buf,
                                       struct iatt *preparent,
-                                      struct iatt *postparent);
+                                      struct iatt *postparent, dict_t *xdata);
 
 typedef int32_t (*fop_rename_cbk_t) (call_frame_t *frame,
                                      void *cookie,
@@ -229,7 +222,7 @@ typedef int32_t (*fop_rename_cbk_t) (call_frame_t *frame,
                                      struct iatt *preoldparent,
                                      struct iatt *postoldparent,
                                      struct iatt *prenewparent,
-                                     struct iatt *postnewparent);
+                                     struct iatt *postnewparent, dict_t *xdata);
 
 typedef int32_t (*fop_link_cbk_t) (call_frame_t *frame,
                                    void *cookie,
@@ -239,7 +232,7 @@ typedef int32_t (*fop_link_cbk_t) (call_frame_t *frame,
                                    inode_t *inode,
                                    struct iatt *buf,
                                    struct iatt *preparent,
-                                   struct iatt *postparent);
+                                   struct iatt *postparent, dict_t *xdata);
 
 typedef int32_t (*fop_create_cbk_t) (call_frame_t *frame,
                                      void *cookie,
@@ -250,14 +243,14 @@ typedef int32_t (*fop_create_cbk_t) (call_frame_t *frame,
                                      inode_t *inode,
                                      struct iatt *buf,
                                      struct iatt *preparent,
-                                     struct iatt *postparent);
+                                     struct iatt *postparent, dict_t *xdata);
 
 typedef int32_t (*fop_open_cbk_t) (call_frame_t *frame,
                                    void *cookie,
                                    xlator_t *this,
                                    int32_t op_ret,
                                    int32_t op_errno,
-                                   fd_t *fd);
+                                   fd_t *fd, dict_t *xdata);
 
 typedef int32_t (*fop_readv_cbk_t) (call_frame_t *frame,
                                     void *cookie,
@@ -267,7 +260,7 @@ typedef int32_t (*fop_readv_cbk_t) (call_frame_t *frame,
                                     struct iovec *vector,
                                     int32_t count,
                                     struct iatt *stbuf,
-                                    struct iobref *iobref);
+                                    struct iobref *iobref, dict_t *xdata);
 
 typedef int32_t (*fop_writev_cbk_t) (call_frame_t *frame,
                                      void *cookie,
@@ -275,13 +268,13 @@ typedef int32_t (*fop_writev_cbk_t) (call_frame_t *frame,
                                      int32_t op_ret,
                                      int32_t op_errno,
                                      struct iatt *prebuf,
-                                     struct iatt *postbuf);
+                                     struct iatt *postbuf, dict_t *xdata);
 
 typedef int32_t (*fop_flush_cbk_t) (call_frame_t *frame,
                                     void *cookie,
                                     xlator_t *this,
                                     int32_t op_ret,
-                                    int32_t op_errno);
+                                    int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_fsync_cbk_t) (call_frame_t *frame,
                                     void *cookie,
@@ -289,124 +282,124 @@ typedef int32_t (*fop_fsync_cbk_t) (call_frame_t *frame,
                                     int32_t op_ret,
                                     int32_t op_errno,
                                     struct iatt *prebuf,
-                                    struct iatt *postbuf);
+                                    struct iatt *postbuf, dict_t *xdata);
 
 typedef int32_t (*fop_opendir_cbk_t) (call_frame_t *frame,
                                       void *cookie,
                                       xlator_t *this,
                                       int32_t op_ret,
                                       int32_t op_errno,
-                                      fd_t *fd);
+                                      fd_t *fd, dict_t *xdata);
 
 typedef int32_t (*fop_fsyncdir_cbk_t) (call_frame_t *frame,
                                        void *cookie,
                                        xlator_t *this,
                                        int32_t op_ret,
-                                       int32_t op_errno);
+                                       int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_statfs_cbk_t) (call_frame_t *frame,
                                      void *cookie,
                                      xlator_t *this,
                                      int32_t op_ret,
                                      int32_t op_errno,
-                                     struct statvfs *buf);
+                                     struct statvfs *buf, dict_t *xdata);
 
 typedef int32_t (*fop_setxattr_cbk_t) (call_frame_t *frame,
                                        void *cookie,
                                        xlator_t *this,
                                        int32_t op_ret,
-                                       int32_t op_errno);
+                                       int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_getxattr_cbk_t) (call_frame_t *frame,
                                        void *cookie,
                                        xlator_t *this,
                                        int32_t op_ret,
                                        int32_t op_errno,
-                                       dict_t *dict);
+                                       dict_t *dict, dict_t *xdata);
 
 typedef int32_t (*fop_fsetxattr_cbk_t) (call_frame_t *frame,
                                         void *cookie,
                                         xlator_t *this,
                                         int32_t op_ret,
-                                        int32_t op_errno);
+                                        int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_fgetxattr_cbk_t) (call_frame_t *frame,
                                         void *cookie,
                                         xlator_t *this,
                                         int32_t op_ret,
                                         int32_t op_errno,
-                                        dict_t *dict);
+                                        dict_t *dict, dict_t *xdata);
 
 typedef int32_t (*fop_removexattr_cbk_t) (call_frame_t *frame,
                                           void *cookie,
                                           xlator_t *this,
                                           int32_t op_ret,
-                                          int32_t op_errno);
+                                          int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_fremovexattr_cbk_t) (call_frame_t *frame,
                                            void *cookie,
                                            xlator_t *this,
                                            int32_t op_ret,
-                                           int32_t op_errno);
+                                           int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_lk_cbk_t) (call_frame_t *frame,
                                  void *cookie,
                                  xlator_t *this,
                                  int32_t op_ret,
                                  int32_t op_errno,
-                                 struct gf_flock *flock);
+                                 struct gf_flock *flock, dict_t *xdata);
 
 typedef int32_t (*fop_inodelk_cbk_t) (call_frame_t *frame,
                                       void *cookie,
                                       xlator_t *this,
                                       int32_t op_ret,
-                                      int32_t op_errno);
+                                      int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_finodelk_cbk_t) (call_frame_t *frame,
                                        void *cookie,
                                        xlator_t *this,
                                        int32_t op_ret,
-                                       int32_t op_errno);
+                                       int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_entrylk_cbk_t) (call_frame_t *frame,
                                       void *cookie,
                                       xlator_t *this,
                                       int32_t op_ret,
-                                      int32_t op_errno);
+                                      int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_fentrylk_cbk_t) (call_frame_t *frame,
                                        void *cookie,
                                        xlator_t *this,
                                        int32_t op_ret,
-                                       int32_t op_errno);
+                                       int32_t op_errno, dict_t *xdata);
 
 typedef int32_t (*fop_readdir_cbk_t) (call_frame_t *frame,
                                       void *cookie,
                                       xlator_t *this,
                                       int32_t op_ret,
                                       int32_t op_errno,
-                                      gf_dirent_t *entries);
+                                      gf_dirent_t *entries, dict_t *xdata);
 
 typedef int32_t (*fop_readdirp_cbk_t) (call_frame_t *frame,
                                        void *cookie,
                                        xlator_t *this,
                                        int32_t op_ret,
                                        int32_t op_errno,
-                                       gf_dirent_t *entries);
+                                       gf_dirent_t *entries, dict_t *xdata);
 
 typedef int32_t (*fop_xattrop_cbk_t) (call_frame_t *frame,
                                       void *cookie,
                                       xlator_t *this,
                                       int32_t op_ret,
                                       int32_t op_errno,
-                                      dict_t *xattr);
+                                      dict_t *xattr, dict_t *xdata);
 
 typedef int32_t (*fop_fxattrop_cbk_t) (call_frame_t *frame,
                                        void *cookie,
                                        xlator_t *this,
                                        int32_t op_ret,
                                        int32_t op_errno,
-                                       dict_t *xattr);
+                                       dict_t *xattr, dict_t *xdata);
 
 
 typedef int32_t (*fop_setattr_cbk_t) (call_frame_t *frame,
@@ -415,7 +408,7 @@ typedef int32_t (*fop_setattr_cbk_t) (call_frame_t *frame,
                                       int32_t op_ret,
                                       int32_t op_errno,
                                       struct iatt *preop_stbuf,
-                                      struct iatt *postop_stbuf);
+                                      struct iatt *postop_stbuf, dict_t *xdata);
 
 typedef int32_t (*fop_fsetattr_cbk_t) (call_frame_t *frame,
                                        void *cookie,
@@ -423,72 +416,95 @@ typedef int32_t (*fop_fsetattr_cbk_t) (call_frame_t *frame,
                                        int32_t op_ret,
                                        int32_t op_errno,
                                        struct iatt *preop_stbuf,
-                                       struct iatt *postop_stbuf);
+                                       struct iatt *postop_stbuf, dict_t *xdata);
+
+typedef int32_t (*fop_fallocate_cbk_t) (call_frame_t *frame,
+                                        void *cookie,
+                                        xlator_t *this,
+                                        int32_t op_ret,
+                                        int32_t op_errno,
+                                        struct iatt *preop_stbuf,
+                                        struct iatt *postop_stbuf, dict_t *xdata);
+
+typedef int32_t (*fop_discard_cbk_t) (call_frame_t *frame,
+                                      void *cookie,
+                                      xlator_t *this,
+                                      int32_t op_ret,
+                                      int32_t op_errno,
+                                      struct iatt *preop_stbuf,
+                                      struct iatt *postop_stbuf, dict_t *xdata);
+
+typedef int32_t (*fop_zerofill_cbk_t) (call_frame_t *frame,
+                                      void *cookie,
+                                      xlator_t *this,
+                                      int32_t op_ret,
+                                      int32_t op_errno,
+                                      struct iatt *preop_stbuf,
+                                      struct iatt *postop_stbuf, dict_t *xdata);
 
 typedef int32_t (*fop_lookup_t) (call_frame_t *frame,
                                  xlator_t *this,
                                  loc_t *loc,
-                                 dict_t *xattr_req);
+                                 dict_t *xdata);
 
 typedef int32_t (*fop_stat_t) (call_frame_t *frame,
                                xlator_t *this,
-                               loc_t *loc);
+                               loc_t *loc, dict_t *xdata);
 
 typedef int32_t (*fop_fstat_t) (call_frame_t *frame,
                                 xlator_t *this,
-                                fd_t *fd);
+                                fd_t *fd, dict_t *xdata);
 
 typedef int32_t (*fop_truncate_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    loc_t *loc,
-                                   off_t offset);
+                                   off_t offset, dict_t *xdata);
 
 typedef int32_t (*fop_ftruncate_t) (call_frame_t *frame,
                                     xlator_t *this,
                                     fd_t *fd,
-                                    off_t offset);
+                                    off_t offset, dict_t *xdata);
 
 typedef int32_t (*fop_access_t) (call_frame_t *frame,
                                  xlator_t *this,
                                  loc_t *loc,
-                                 int32_t mask);
+                                 int32_t mask, dict_t *xdata);
 
 typedef int32_t (*fop_readlink_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    loc_t *loc,
-                                   size_t size);
+                                   size_t size, dict_t *xdata);
 
 typedef int32_t (*fop_mknod_t) (call_frame_t *frame, xlator_t *this,
                                 loc_t *loc, mode_t mode, dev_t rdev,
-                                dict_t *params);
+                                mode_t umask, dict_t *xdata);
 
-typedef int32_t (*fop_mkdir_t) (call_frame_t *frame, xlator_t *this,
-                                loc_t *loc, mode_t mode, dict_t *params);
+typedef int32_t (*fop_mkdir_t) (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                mode_t mode, mode_t umask, dict_t *xdata);
 
-typedef int32_t (*fop_unlink_t) (call_frame_t *frame,
-                                 xlator_t *this,
-                                 loc_t *loc);
+typedef int32_t (*fop_unlink_t) (call_frame_t *frame, xlator_t *this,
+                                 loc_t *loc, int xflags, dict_t *xdata);
 
 typedef int32_t (*fop_rmdir_t) (call_frame_t *frame, xlator_t *this,
-                                loc_t *loc, int flags);
+                                loc_t *loc, int xflags, dict_t *xdata);
 
 typedef int32_t (*fop_symlink_t) (call_frame_t *frame, xlator_t *this,
                                   const char *linkname, loc_t *loc,
-                                  dict_t *params);
+                                  mode_t umask, dict_t *xdata);
 
 typedef int32_t (*fop_rename_t) (call_frame_t *frame,
                                  xlator_t *this,
                                  loc_t *oldloc,
-                                 loc_t *newloc);
+                                 loc_t *newloc, dict_t *xdata);
 
 typedef int32_t (*fop_link_t) (call_frame_t *frame,
                                xlator_t *this,
                                loc_t *oldloc,
-                               loc_t *newloc);
+                               loc_t *newloc, dict_t *xdata);
 
 typedef int32_t (*fop_create_t) (call_frame_t *frame, xlator_t *this,
                                  loc_t *loc, int32_t flags, mode_t mode,
-                                 fd_t *fd, dict_t *params);
+                                 mode_t umask, fd_t *fd, dict_t *xdata);
 
 /* Tell subsequent writes on the fd_t to fsync after every writev fop without
  * requiring a fsync fop.
@@ -499,19 +515,16 @@ typedef int32_t (*fop_create_t) (call_frame_t *frame, xlator_t *this,
  */
 #define GF_OPEN_NOWB    0x02
 
-typedef int32_t (*fop_open_t) (call_frame_t *frame,
-                               xlator_t *this,
-                               loc_t *loc,
-                               int32_t flags,
-                               fd_t *fd,
-                               int32_t wbflags);
+typedef int32_t (*fop_open_t) (call_frame_t *frame, xlator_t *this,
+                               loc_t *loc, int32_t flags,
+                               fd_t *fd, dict_t *xdata);
 
 typedef int32_t (*fop_readv_t) (call_frame_t *frame,
                                 xlator_t *this,
                                 fd_t *fd,
                                 size_t size,
                                 off_t offset,
-                                uint32_t flags);
+                                uint32_t flags, dict_t *xdata);
 
 typedef int32_t (*fop_writev_t) (call_frame_t *frame,
                                  xlator_t *this,
@@ -520,132 +533,152 @@ typedef int32_t (*fop_writev_t) (call_frame_t *frame,
                                  int32_t count,
                                  off_t offset,
                                  uint32_t flags,
-                                 struct iobref *iobref);
+                                 struct iobref *iobref, dict_t *xdata);
 
 typedef int32_t (*fop_flush_t) (call_frame_t *frame,
                                 xlator_t *this,
-                                fd_t *fd);
+                                fd_t *fd, dict_t *xdata);
 
 typedef int32_t (*fop_fsync_t) (call_frame_t *frame,
                                 xlator_t *this,
                                 fd_t *fd,
-                                int32_t datasync);
+                                int32_t datasync, dict_t *xdata);
 
 typedef int32_t (*fop_opendir_t) (call_frame_t *frame,
                                   xlator_t *this,
                                   loc_t *loc,
-                                  fd_t *fd);
+                                  fd_t *fd, dict_t *xdata);
 
 typedef int32_t (*fop_fsyncdir_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    fd_t *fd,
-                                   int32_t datasync);
+                                   int32_t datasync, dict_t *xdata);
 
 typedef int32_t (*fop_statfs_t) (call_frame_t *frame,
                                  xlator_t *this,
-                                 loc_t *loc);
+                                 loc_t *loc, dict_t *xdata);
 
 typedef int32_t (*fop_setxattr_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    loc_t *loc,
                                    dict_t *dict,
-                                   int32_t flags);
+                                   int32_t flags, dict_t *xdata);
 
 typedef int32_t (*fop_getxattr_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    loc_t *loc,
-                                   const char *name);
+                                   const char *name, dict_t *xdata);
 
 typedef int32_t (*fop_fsetxattr_t) (call_frame_t *frame,
                                     xlator_t *this,
                                     fd_t *fd,
                                     dict_t *dict,
-                                    int32_t flags);
+                                    int32_t flags, dict_t *xdata);
 
 typedef int32_t (*fop_fgetxattr_t) (call_frame_t *frame,
                                     xlator_t *this,
                                     fd_t *fd,
-                                    const char *name);
+                                    const char *name, dict_t *xdata);
 
 typedef int32_t (*fop_removexattr_t) (call_frame_t *frame,
                                       xlator_t *this,
                                       loc_t *loc,
-                                      const char *name);
+                                      const char *name, dict_t *xdata);
 
 typedef int32_t (*fop_fremovexattr_t) (call_frame_t *frame,
                                        xlator_t *this,
                                        fd_t *fd,
-                                       const char *name);
+                                       const char *name, dict_t *xdata);
 
 typedef int32_t (*fop_lk_t) (call_frame_t *frame,
                              xlator_t *this,
                              fd_t *fd,
                              int32_t cmd,
-                             struct gf_flock *flock);
+                             struct gf_flock *flock, dict_t *xdata);
 
 typedef int32_t (*fop_inodelk_t) (call_frame_t *frame,
                                   xlator_t *this,
                                   const char *volume,
                                   loc_t *loc,
                                   int32_t cmd,
-                                  struct gf_flock *flock);
+                                  struct gf_flock *flock, dict_t *xdata);
 
 typedef int32_t (*fop_finodelk_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    const char *volume,
                                    fd_t *fd,
                                    int32_t cmd,
-                                   struct gf_flock *flock);
+                                   struct gf_flock *flock, dict_t *xdata);
 
 typedef int32_t (*fop_entrylk_t) (call_frame_t *frame,
                                   xlator_t *this,
                                   const char *volume, loc_t *loc,
                                   const char *basename, entrylk_cmd cmd,
-                                  entrylk_type type);
+                                  entrylk_type type, dict_t *xdata);
 
 typedef int32_t (*fop_fentrylk_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    const char *volume, fd_t *fd,
                                    const char *basename, entrylk_cmd cmd,
-                                   entrylk_type type);
+                                   entrylk_type type, dict_t *xdata);
 
 typedef int32_t (*fop_readdir_t) (call_frame_t *frame,
                                   xlator_t *this,
                                   fd_t *fd,
                                   size_t size,
-                                  off_t offset);
+                                  off_t offset, dict_t *xdata);
 
 typedef int32_t (*fop_readdirp_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    fd_t *fd,
                                    size_t size,
                                    off_t offset,
-                                   dict_t *dict);
+                                   dict_t *xdata);
 
 typedef int32_t (*fop_xattrop_t) (call_frame_t *frame,
                                   xlator_t *this,
                                   loc_t *loc,
                                   gf_xattrop_flags_t optype,
-                                  dict_t *xattr);
+                                  dict_t *xattr, dict_t *xdata);
 
 typedef int32_t (*fop_fxattrop_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    fd_t *fd,
                                    gf_xattrop_flags_t optype,
-                                   dict_t *xattr);
+                                   dict_t *xattr, dict_t *xdata);
 
 typedef int32_t (*fop_setattr_t) (call_frame_t *frame,
                                   xlator_t *this,
                                   loc_t *loc,
                                   struct iatt *stbuf,
-                                  int32_t valid);
+                                  int32_t valid, dict_t *xdata);
 
 typedef int32_t (*fop_fsetattr_t) (call_frame_t *frame,
                                    xlator_t *this,
                                    fd_t *fd,
                                    struct iatt *stbuf,
-                                   int32_t valid);
+                                   int32_t valid, dict_t *xdata);
 
+typedef int32_t (*fop_fallocate_t) (call_frame_t *frame,
+                                    xlator_t *this,
+                                    fd_t *fd,
+				    int32_t keep_size,
+				    off_t offset,
+				    size_t len,
+                                    dict_t *xdata);
+
+typedef int32_t (*fop_discard_t) (call_frame_t *frame,
+                                  xlator_t *this,
+                                  fd_t *fd,
+				  off_t offset,
+				  size_t len,
+                                  dict_t *xdata);
+typedef int32_t (*fop_zerofill_t) (call_frame_t *frame,
+                                  xlator_t *this,
+                                  fd_t *fd,
+                                  off_t offset,
+                                  size_t len,
+                                  dict_t *xdata);
 
 struct xlator_fops {
         fop_lookup_t         lookup;
@@ -690,6 +723,9 @@ struct xlator_fops {
         fop_setattr_t        setattr;
         fop_fsetattr_t       fsetattr;
         fop_getspec_t        getspec;
+	fop_fallocate_t	     fallocate;
+	fop_discard_t	     discard;
+        fop_zerofill_t       zerofill;
 
         /* these entries are used for a typechecking hack in STACK_WIND _only_ */
         fop_lookup_cbk_t         lookup_cbk;
@@ -734,6 +770,9 @@ struct xlator_fops {
         fop_setattr_cbk_t        setattr_cbk;
         fop_fsetattr_cbk_t       fsetattr_cbk;
         fop_getspec_cbk_t        getspec_cbk;
+	fop_fallocate_cbk_t	 fallocate_cbk;
+	fop_discard_cbk_t	 discard_cbk;
+        fop_zerofill_cbk_t       zerofill_cbk;
 };
 
 typedef int32_t (*cbk_forget_t) (xlator_t *this,
@@ -742,10 +781,17 @@ typedef int32_t (*cbk_forget_t) (xlator_t *this,
 typedef int32_t (*cbk_release_t) (xlator_t *this,
                                   fd_t *fd);
 
+typedef int32_t (*cbk_invalidate_t)(xlator_t *this, inode_t *inode);
+
+typedef int32_t (*cbk_client_t)(xlator_t *this, client_t *client);
+
 struct xlator_cbks {
-        cbk_forget_t    forget;
-        cbk_release_t   release;
-        cbk_release_t   releasedir;
+        cbk_forget_t             forget;
+        cbk_release_t            release;
+        cbk_release_t            releasedir;
+	cbk_invalidate_t         invalidate;
+        cbk_client_t             client_destroy;
+        cbk_client_t             client_disconnect;
 };
 
 typedef int32_t (*dumpop_priv_t) (xlator_t *this);
@@ -770,6 +816,8 @@ typedef int32_t (*dumpop_inodectx_to_dict_t) (xlator_t *this, inode_t *ino,
 typedef int32_t (*dumpop_fdctx_to_dict_t) (xlator_t *this, fd_t *fd,
                                            dict_t *dict);
 
+typedef int32_t (*dumpop_eh_t) (xlator_t *this);
+
 struct xlator_dumpops {
         dumpop_priv_t                   priv;
         dumpop_inode_t                  inode;
@@ -781,6 +829,7 @@ struct xlator_dumpops {
         dumpop_fd_to_dict_t             fd_to_dict;
         dumpop_inodectx_to_dict_t       inodectx_to_dict;
         dumpop_fdctx_to_dict_t          fdctx_to_dict;
+        dumpop_eh_t                     history;
 };
 
 typedef struct xlator_list {
@@ -818,16 +867,43 @@ struct _xlator {
         fop_latency_t latencies[GF_FOP_MAXVALUE];
 
         /* Misc */
+        eh_t               *history; /* event history context */
         glusterfs_ctx_t    *ctx;
         glusterfs_graph_t  *graph; /* not set for fuse */
         inode_table_t      *itable;
         char                init_succeeded;
         void               *private;
         struct mem_acct     mem_acct;
+        uint64_t            winds;
+        char                switched;
+
+        /* for the memory pool of 'frame->local' */
+        struct mem_pool    *local_pool;
+        gf_boolean_t        is_autoloaded;
 };
 
+typedef struct {
+        int32_t                 (*init) (xlator_t *this);
+        void                    (*fini) (xlator_t *this);
+        int32_t                 (*reconfigure) (xlator_t *this,
+                                                dict_t *options);
+        event_notify_fn_t       notify;
+} class_methods_t;
+
 #define xlator_has_parent(xl) (xl->parents != NULL)
 
+#define XLATOR_NOTIFY(_xl, params ...)          \
+        do {                                    \
+                xlator_t *_old_THIS = NULL;     \
+                                                \
+                _old_THIS = THIS;               \
+                THIS = _xl;                     \
+                                                \
+                ret = _xl->notify (_xl, params);\
+                                                \
+                THIS = _old_THIS;               \
+        } while (0);
+
 int32_t xlator_set_type_virtual (xlator_t *xl, const char *type);
 
 int32_t xlator_set_type (xlator_t *xl, const char *type);
@@ -851,6 +927,11 @@ void xlator_foreach (xlator_t *this,
                                  void *data),
                      void *data);
 
+void xlator_foreach_depth_first (xlator_t *this,
+				 void (*fn) (xlator_t *each,
+					     void *data),
+				 void *data);
+
 xlator_t *xlator_search_by_name (xlator_t *any, const char *name);
 
 void inode_destroy_notify (inode_t *inode, const char *xlname);
@@ -858,9 +939,22 @@ void inode_destroy_notify (inode_t *inode, const char *xlname);
 int loc_copy (loc_t *dst, loc_t *src);
 #define loc_dup(src, dst) loc_copy(dst, src)
 void loc_wipe (loc_t *loc);
+int loc_path (loc_t *loc, const char *bname);
+void loc_gfid (loc_t *loc, uuid_t gfid);
+char* loc_gfid_utoa (loc_t *loc);
+gf_boolean_t loc_is_root (loc_t *loc);
 int xlator_mem_acct_init (xlator_t *xl, int num_types);
 int is_gf_log_command (xlator_t *trans, const char *name, char *value);
 int glusterd_check_log_level (const char *value);
 int xlator_volopt_dynload (char *xlator_type, void **dl_handle,
                            volume_opt_list_t *vol_opt_handle);
+enum gf_hdsk_event_notify_op {
+        GF_EN_DEFRAG_STATUS,
+        GF_EN_MAX,
+};
+gf_boolean_t
+is_graph_topology_equal (glusterfs_graph_t *graph1, glusterfs_graph_t *graph2);
+int
+glusterfs_volfile_reconfigure (int oldvollen, FILE *newvolfile_fp,
+                               glusterfs_ctx_t *ctx, const char *oldvolfile);
 #endif /* _XLATOR_H */
diff --git a/libglusterfsclient/src/Makefile.am b/libglusterfsclient/src/Makefile.am
deleted file mode 100644
index 32811c0d5..000000000
--- a/libglusterfsclient/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-lib_LTLIBRARIES = libglusterfsclient.la
-noinst_HEADERS = libglusterfsclient-internals.h
-libglusterfsclient_HEADERS = libglusterfsclient.h 
-libglusterfsclientdir = $(includedir)
-
-libglusterfsclient_la_SOURCES = libglusterfsclient.c libglusterfsclient-dentry.c
-libglusterfsclient_la_CFLAGS =  -fPIC -Wall
-libglusterfsclient_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-libglusterfsclient_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D$(GF_HOST_OS) -D__USE_FILE_OFFSET64 -D_GNU_SOURCE -I$(top_srcdir)/libglusterfs/src -DDATADIR=\"$(localstatedir)\" -DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_CFLAGS)
-libglusterfsclient_la_LDFLAGS = -shared -nostartfiles
-
-CLEANFILES = 
-
-$(top_builddir)/libglusterfs/src/libglusterfs.la:
-	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
-
diff --git a/libglusterfsclient/src/libglusterfsclient-dentry.c b/libglusterfsclient/src/libglusterfsclient-dentry.c
deleted file mode 100644
index 02448b363..000000000
--- a/libglusterfsclient/src/libglusterfsclient-dentry.c
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
-  Copyright (c) 2008 - 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include "libglusterfsclient.h"
-#include "libglusterfsclient-internals.h"
-#include <libgen.h>
-
-#define LIBGLUSTERFS_CLIENT_DENTRY_LOC_PREPARE(_new_loc, _loc, _parent, \
-                                               _resolved) do {          \
-                size_t pathlen = 0;                                     \
-                size_t resolvedlen = 0;                                 \
-                char *path = NULL;                                      \
-                int pad = 0;                                            \
-                pathlen   = strlen (_loc->path) + 1;                    \
-                path = CALLOC (1, pathlen);                             \
-                _new_loc.parent =  _parent;                             \
-                resolvedlen = strlen (_resolved);                       \
-                strncpy (path, _resolved, resolvedlen);                 \
-                if (resolvedlen == 1) /* only root resolved */          \
-                        pad = 0;                                        \
-                else {                                                  \
-                        pad = 1;                                        \
-                        path[resolvedlen] = '/';                        \
-                }                                                       \
-                strcpy_till (path + resolvedlen + pad,                  \
-                             loc->path + resolvedlen + pad, '/');       \
-                _new_loc.path = path;                                   \
-                _new_loc.name = strrchr (path, '/');                    \
-                if (_new_loc.name)                                      \
-                        _new_loc.name++;                                \
-        }while (0);
-
-
-/* strcpy_till - copy @dname to @dest, until 'delim' is encountered in @dest
- * @dest - destination string
- * @dname - source string
- * @delim - delimiter character
- *
- * return - NULL is returned if '0' is encountered in @dname, otherwise returns
- *          a pointer to remaining string begining in @dest.
- */
-static char *
-strcpy_till (char *dest, const char *dname, char delim)
-{
-        char *src = NULL;
-        int idx = 0;
-        char *ret = NULL;
-
-        src = (char *)dname;
-        while (src[idx] && (src[idx] != delim)) {
-                dest[idx] = src[idx];
-                idx++;
-        }
-
-        dest[idx] = 0;
-
-        if (src[idx] == 0)
-                ret = NULL;
-        else
-                ret = &(src[idx]);
-
-        return ret;
-}
-
-/* __libgf_client_path_to_parenti - derive parent inode for @path. if immediate 
- *                            parent is not available in the dentry cache, return nearest
- *                            available parent inode and set @reslv to the path of
- *                            the returned directory.
- *
- * @itable - inode table
- * @path   - path whose parent has to be looked up.
- * @reslv  - if immediate parent is not available, reslv will be set to path of the
- *           resolved parent.
- *
- * return - should never return NULL. should at least return '/' inode.
- */
-static inode_t *
-__libgf_client_path_to_parenti (libglusterfs_client_ctx_t *ctx,
-                                inode_table_t *itable, const char *path,
-                                char **reslv)
-{
-        char *resolved_till = NULL;
-        char *strtokptr = NULL;
-        char *component = NULL;
-        char *next_component = NULL;
-        char *pathdup = NULL;
-        inode_t *curr = NULL;
-        inode_t *parent = NULL;
-        size_t pathlen = 0;
-        loc_t rootloc = {0, };
-        int ret = -1;
-
-        pathlen = STRLEN_0 (path);
-        resolved_till = CALLOC (1, pathlen);
-
-        GF_VALIDATE_OR_GOTO("libglusterfsclient-dentry", resolved_till, out);
-        pathdup = strdup (path);
-        GF_VALIDATE_OR_GOTO("libglusterfsclient-dentry", pathdup, out);
-
-        parent = inode_ref (itable->root);
-        /* If the root inode's is outdated, send a revalidate on it.
-         * A revalidate on root inode also reduces the window in which an
-         * op will fail over distribute because the layout of the root
-         * directory did not  get constructed when we sent the lookup on
-         * root in glusterfs_init. That can happen when not all children of a
-         * distribute volume were up at the time of glusterfs_init.
-         */
-        if (!libgf_is_iattr_cache_valid (ctx, parent, NULL,
-                                        LIBGF_VALIDATE_LOOKUP)) {
-                libgf_client_loc_fill (&rootloc, ctx, 1, 0, "/");
-                ret = libgf_client_lookup (ctx, &rootloc, NULL, NULL, NULL);
-                if (ret == -1) {
-                        gf_log ("libglusterfsclient-dentry", GF_LOG_ERROR,
-                                "Root inode revalidation failed");
-                        inode_unref (parent);
-                        parent = NULL;
-                        goto out;
-                }
-                libgf_client_loc_wipe (&rootloc);
-        }
-
-        curr = NULL;
-
-        component = strtok_r (pathdup, "/", &strtokptr);
-
-        while (component) {
-                curr = inode_search (itable, parent->ino, component);
-                if (!curr) {
-                        break;
-                }
-                if (!libgf_is_iattr_cache_valid (ctx, curr, NULL,
-                                                LIBGF_VALIDATE_LOOKUP))
-                        break;
-
-                /* It is OK to append the component even if it is the       
-                   last component in the path, because, if 'next_component'
-                   returns NULL, @parent will remain the same and
-                   @resolved_till will not be sent back               
-                */
-                strcat (resolved_till, "/");
-                strcat (resolved_till, component);
-
-                next_component = strtok_r (NULL, "/", &strtokptr);
-
-                if (next_component) {
-                        inode_unref (parent);
-                        parent = curr;
-                        curr = NULL;
-                } else {
-                        /* will break */
-                        inode_unref (curr);
-                }
-
-                component = next_component;
-        }
-
-        if (resolved_till[0] == '\0') {
-                strcat (resolved_till, "/");
-        }
-
-        free (pathdup);
-        
-        if (reslv) {
-                *reslv = resolved_till;
-        } else {
-                FREE (resolved_till);
-        }
-
-out:
-        return parent;
-}
-
-static inline void
-libgf_client_update_resolved (const char *path, char *resolved)
-{
-        int32_t pathlen = 0;
-        char *tmp = NULL, *dest = NULL, *dname = NULL;
-        char append_slash = 0;
-
-        pathlen = strlen (resolved); 
-        tmp = (char *)(resolved + pathlen);
-        if (*((char *) (resolved + pathlen - 1)) != '/') {
-                tmp[0] = '/';
-                append_slash = 1;
-        }
-
-        if (append_slash) {
-                dest = tmp + 1;
-        } else {
-                dest = tmp;
-        }
-
-        if (*((char *) path + pathlen) == '/') {
-                dname = (char *) path + pathlen + 1;
-        } else {
-                dname = (char *) path + pathlen;
-        }
-
-        strcpy_till (dest, dname, '/');
-}
-
-/* __do_path_resolve - resolve @loc->path into @loc->inode and @loc->parent. also
- *                     update the dentry cache
- *
- * @loc             - loc to resolve. 
- * @ctx             - libglusterfsclient context
- * @lookup_basename - flag whether to lookup basename(loc->path)
- *
- * return - 0 on success
- *         -1 on failure 
- *          
- */
-static int32_t
-__do_path_resolve (loc_t *loc, libglusterfs_client_ctx_t *ctx,
-                   char lookup_basename)
-{
-        int32_t         op_ret = -1;
-        char           *resolved  = NULL;
-        inode_t        *parent = NULL, *inode = NULL;
-        dentry_t       *dentry = NULL;
-        loc_t          new_loc = {0, };
-	char           *pathname = NULL, *directory = NULL;
-	char           *file = NULL;   
-        
-        parent = loc->parent;
-        if (parent) {
-                inode_ref (parent);
-                gf_log ("libglusterfsclient-dentry", GF_LOG_DEBUG,
-                        "loc->parent(%"PRId64") already present. sending "
-                        "lookup for %"PRId64"/%s", parent->ino, parent->ino,
-                        loc->path);
-                resolved = strdup (loc->path);
-                resolved = dirname (resolved);
-        } else {
-                parent = __libgf_client_path_to_parenti (ctx, ctx->itable,
-                                                         loc->path, &resolved);
-        }
-
-        if (parent == NULL) {
-                /* fire in the bush.. run! run!! run!!! */
-                gf_log ("libglusterfsclient-dentry",
-                        GF_LOG_CRITICAL,
-                        "failed to get parent inode number");
-                op_ret = -1;
-                goto out;
-        }               
-
-        gf_log ("libglusterfsclient-dentry",
-                GF_LOG_DEBUG,
-                "resolved path(%s) till %"PRId64"(%s). "
-                "sending lookup for remaining path",
-                loc->path, parent->ino, resolved);
-
-	pathname = strdup (loc->path);
-	directory = dirname (pathname);
-        pathname = NULL;
-
-        while (strcmp (resolved, directory) != 0) 
-        {
-                dentry = NULL;
-
-                LIBGLUSTERFS_CLIENT_DENTRY_LOC_PREPARE (new_loc, loc, parent,
-                                                        resolved);
-
-		if (pathname) {
-			free (pathname);
-			pathname = NULL;
-		}
-
-		pathname = strdup (new_loc.path);
-		file = basename (pathname);
-
-                new_loc.inode = inode_search (ctx->itable, parent->ino, file);
-                if (new_loc.inode) {
-                        if (libgf_is_iattr_cache_valid (ctx, new_loc.inode,
-                                                        NULL,
-                                                        LIBGF_VALIDATE_LOOKUP))
-                                dentry = dentry_search_for_inode (new_loc.inode,
-                                                                  parent->ino,
-                                                                  file);
-                }
-
-                if (dentry == NULL) {
-                        op_ret = libgf_client_lookup (ctx, &new_loc, NULL, NULL,
-                                                      0);
-                        if (op_ret == -1) {
-                                inode_ref (new_loc.parent);
-                                libgf_client_loc_wipe (&new_loc);
-                                goto out;
-                        }
-                }
-
-                parent = inode_ref (new_loc.inode);
-                libgf_client_loc_wipe (&new_loc);
-
-                libgf_client_update_resolved (loc->path, resolved);
-        }
-
-	if (pathname) {
-		free (pathname);
-		pathname = NULL;
-	} 
-
-        if (lookup_basename) {
-                pathname = strdup (loc->path);
-                file = basename (pathname);
-
-                inode = inode_search (ctx->itable, parent->ino, file);
-                if (!inode) {
-                        libgf_client_loc_fill (&new_loc, ctx, 0, parent->ino,
-                                               file);
-
-                        op_ret = libgf_client_lookup (ctx, &new_loc, NULL, NULL,
-                                                      0);
-                        if (op_ret == -1) {
-                                libgf_client_loc_wipe (&new_loc);
-                                goto out;
-                        }
-                
-                        inode = inode_ref (new_loc.inode);
-                        libgf_client_loc_wipe (&new_loc);
-                }
-        }
-
-        op_ret = 0;
-out:
-        loc->inode = inode;
-        loc->parent = parent;
-
-        FREE (resolved);
-	if (pathname) {
-		FREE (pathname);
-	}
-
-	if (directory) {
-		FREE (directory);
-	}
-
-        return op_ret;
-}
-
-
-/* resolves loc->path to loc->parent and loc->inode */
-int32_t
-libgf_client_path_lookup (loc_t *loc,
-                          libglusterfs_client_ctx_t *ctx,
-                          char lookup_basename)
-{
-        char       *pathname  = NULL;
-        char       *directory = NULL;
-        inode_t    *inode = NULL;
-        inode_t    *parent = NULL;
-        int32_t     op_ret = 0;
-
-        pathname  = strdup (loc->path);
-        directory = dirname (pathname);
-        parent = inode_from_path (ctx->itable, directory);
-
-        if (parent != NULL) {
-                loc->parent = parent;
-
-                if (!lookup_basename) {
-                        gf_log ("libglusterfsclient",
-                                GF_LOG_DEBUG,
-                                "resolved dirname(%s) to %"PRId64,
-                                loc->path, parent->ino);
-                        goto out;
-                } else {
-                        inode = inode_from_path (ctx->itable, loc->path);
-                        if (inode != NULL) {
-                                gf_log ("libglusterfsclient",
-                                        GF_LOG_DEBUG,
-                                        "resolved path(%s) to %"PRId64"/%"PRId64,
-                                        loc->path, parent->ino, inode->ino);
-                                loc->inode = inode;
-                                goto out;
-                        }
-                }
-        }
-
-        if (parent) {
-                inode_unref (parent);
-        } else if (inode) {
-                inode_unref (inode);
-                gf_log ("libglusterfsclient",
-                        GF_LOG_ERROR,
-                        "undesired behaviour. inode(%"PRId64") for %s "
-                        "exists without parent (%s)",
-                        inode->ino, loc->path, directory);
-        }
-        op_ret = __do_path_resolve (loc, ctx, lookup_basename);
-out:    
-        if (pathname)
-                free (pathname);
-
-        return op_ret;
-}
diff --git a/libglusterfsclient/src/libglusterfsclient-internals.h b/libglusterfsclient/src/libglusterfsclient-internals.h
deleted file mode 100755
index f1bad1ff3..000000000
--- a/libglusterfsclient/src/libglusterfsclient-internals.h
+++ /dev/null
@@ -1,298 +0,0 @@
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __LIBGLUSTERFSCLIENT_INTERNALS_H
-#define __LIBGLUSTERFSCLIENT_INTERNALS_H
-
-#include <glusterfs.h>
-#include <logging.h>
-#include <inode.h>
-#include <pthread.h>
-#include <stack.h>
-#include <list.h>
-#include <signal.h>
-#include <call-stub.h>
-#include <sys/time.h>
-#include <sys/resource.h>
-#include <fd.h>
-#include <dirent.h>
-
-#define LIBGF_IOBUF_SIZE        (128 *GF_UNIT_KB)
-typedef void (*sighandler_t) (int);
-typedef struct list_head list_head_t;
-
-typedef struct libglusterfs_client_ctx {
-        glusterfs_ctx_t gf_ctx;
-        inode_table_t *itable;
-        pthread_t reply_thread;
-        call_pool_t pool;
-        uint32_t counter;
-        time_t lookup_timeout;
-        time_t stat_timeout;
-        /* We generate a fake fsid for the subvolume being
-         * accessed through this context.
-         */
-        dev_t fake_fsid;
-        pid_t pid;
-}libglusterfs_client_ctx_t;
-
-typedef struct signal_handler {
-        int signo;
-        sighandler_t handler;
-        list_head_t next;
-}libgf_client_signal_handler_t ;
-
-typedef struct {
-        pthread_mutex_t lock;
-        pthread_cond_t reply_cond;
-        call_stub_t *reply_stub;
-        char complete;
-        union {
-                struct {
-                        char is_revalidate;
-                        loc_t *loc;
-                        int32_t size;
-                } lookup;
-        }fop;
-        fd_t *fd;          /* Needed here because we need a ref to the dir
-                              fd in the libgf_client_readdir_cbk in order
-                              to process the dirents received, without
-                              having them added to the reply stub.
-                              Also used in updating iattr cache. See
-                              readv_cbk for eg.
-                              */
-}libgf_client_local_t;
-
-typedef struct {
-        pthread_cond_t init_con_established;
-        pthread_mutex_t lock;
-        char complete;
-}libglusterfs_client_private_t;
-
-typedef struct {
-        pthread_mutex_t lock;
-        uint32_t previous_lookup_time;
-        uint32_t previous_stat_time;
-        struct iatt stbuf;
-} libglusterfs_client_inode_ctx_t;
-
-/* Our dirent cache is very simplistic when it comes to directory
- * reading workloads. It assumes that all directory traversal operations happen
- * sequentially and that readdir callers dont go jumping around the directory
- * using seekdir, rewinddir. Thats why you'll notice that seekdir, rewinddir
- * API in libglusterfsclient only set the offset. The consequence is that when
- * libgf_dcache_readdir finds that the offset presented to it, is not
- * the same as the offset of the previous dirent returned by dcache (..stored
- * in struct direntcache->prev_off..), it realises that a non-sequential
- * directory read is in progress and returns 0 to signify that the cache is
- * not valid.
- * This could be made a bit more intelligent by using a data structure like
- * a hash-table or a balanced binary tree that allows us to search for the
- * existence of particular offsets in the cache without performing a list or
- * array traversal.
- * Dont use a simple binary search tree because
- * there is no guarantee that offsets in a sequential reading of the directory
- * will be just random integers. If for some reason they are sequential, a BST
- * will end up becoming a list.
- */
-struct direntcache {
-        gf_dirent_t entries;            /* Head of list of cached dirents. */
-        gf_dirent_t *next;              /* Pointer to the next entry that
-                                         * should be sent by readdir */
-        uint64_t prev_off;              /* Offset where the next read will
-                                         * happen.
-                                         */
-};
-
-typedef struct {
-        pthread_mutex_t lock;
-        off_t offset;
-        libglusterfs_client_ctx_t *ctx;
-        /* `man readdir` says readdir is non-re-entrant
-         * only if two readdirs are racing on the same
-         * handle.
-         */
-	struct dirent dirp;
-        struct direntcache *dcache;
-        char   vpath[PATH_MAX]; 
-} libglusterfs_client_fd_ctx_t;
-
-typedef struct libglusterfs_client_async_local {
-        void *cbk_data;
-        union {
-                struct {
-                        fd_t *fd;
-                        glusterfs_readv_cbk_t cbk;
-                        char update_offset;
-                }readv_cbk;
-    
-                struct {
-                        fd_t *fd;
-                        glusterfs_write_cbk_t cbk;
-                }write_cbk;
-
-                struct {
-                        fd_t *fd;
-                }close_cbk;
-
-                struct {
-                        void *buf;
-                        size_t size;
-                        loc_t *loc;
-                        char is_revalidate;
-                        glusterfs_get_cbk_t cbk;
-                }lookup_cbk;
-        }fop;
-}libglusterfs_client_async_local_t;
-
-#define LIBGF_STACK_WIND_AND_WAIT(frame, rfn, obj, fn, params ...)      \
-        do {                                                            \
-                STACK_WIND (frame, rfn, obj, fn, params);               \
-                pthread_mutex_lock (&local->lock);                      \
-                {                                                       \
-                        while (!local->complete) {                      \
-                                pthread_cond_wait (&local->reply_cond,  \
-                                                   &local->lock);       \
-                        }                                               \
-                }                                                       \
-                pthread_mutex_unlock (&local->lock);                    \
-        } while (0)
-
-
-#define LIBGF_CLIENT_SIGNAL(signal_handler_list, signo, handler)        \
-        do {                                                            \
-                libgf_client_signal_handler_t *libgf_handler = CALLOC (1, \
-                                                    sizeof (*libgf_handler)); \
-                ERR_ABORT (libgf_handler);                              \
-                libgf_handler->signo = signo;                           \
-                libgf_handler->handler = signal (signo, handler);       \
-                list_add (&libgf_handler->next, signal_handler_list);   \
-        } while (0)                                                           
-
-#define LIBGF_INSTALL_SIGNAL_HANDLERS(signal_handlers)                  \
-        do {                                                            \
-                INIT_LIST_HEAD (&signal_handlers);                      \
-                /* Handle SIGABORT and SIGSEGV */                       \
-                LIBGF_CLIENT_SIGNAL (&signal_handlers, SIGSEGV, gf_print_trace); \
-                LIBGF_CLIENT_SIGNAL (&signal_handlers, SIGABRT, gf_print_trace); \
-                LIBGF_CLIENT_SIGNAL (&signal_handlers, SIGHUP, gf_log_logrotate); \
-                /* LIBGF_CLIENT_SIGNAL (SIGTERM, glusterfs_cleanup_and_exit); */ \
-        } while (0)
-
-#define LIBGF_RESTORE_SIGNAL_HANDLERS(local)                            \
-        do {                                                            \
-                libgf_client_signal_handler_t *ptr = NULL, *tmp = NULL; \
-                list_for_each_entry_safe (ptr, tmp, &local->signal_handlers,\
-                                          next) {                       \
-                        signal (ptr->signo, ptr->handler);              \
-                        FREE (ptr);                                     \
-                }                                                       \
-        } while (0)                                       
-
-#define LIBGF_CLIENT_FOP_ASYNC(ctx, local, ret_fn, op, args ...)        \
-        do {                                                            \
-                call_frame_t *frame = get_call_frame_for_req (ctx, 1);  \
-                xlator_t *xl = frame->this->children ?                  \
-                        frame->this->children->xlator : NULL;           \
-                frame->root->state = ctx;                               \
-                frame->local = local;                                   \
-                STACK_WIND (frame, ret_fn, xl, xl->fops->op, args);     \
-        } while (0)
-
-#define LIBGF_CLIENT_FOP(ctx, stub, op, local, args ...)                \
-        do {                                                            \
-                call_frame_t *frame = get_call_frame_for_req (ctx, 1);  \
-                xlator_t *xl = frame->this->children ?                  \
-                        frame->this->children->xlator : NULL;           \
-                if (!local) {                                           \
-                        local = CALLOC (1, sizeof (*local));            \
-                }                                                       \
-                ERR_ABORT (local);                                      \
-                frame->local = local;                                   \
-                frame->root->state = ctx;                               \
-                pthread_cond_init (&local->reply_cond, NULL);           \
-                pthread_mutex_init (&local->lock, NULL);                \
-                LIBGF_STACK_WIND_AND_WAIT (frame, libgf_client_##op##_cbk, xl, \
-                                           xl->fops->op, args);         \
-                stub = local->reply_stub;                               \
-                FREE (frame->local);                                    \
-                frame->local = NULL;                                    \
-                STACK_DESTROY (frame->root);                            \
-        } while (0)
-
-#define LIBGF_REPLY_NOTIFY(local)                                       \
-        do {                                                            \
-                pthread_mutex_lock (&local->lock);                      \
-                {                                                       \
-                        local->complete = 1;                            \
-                        pthread_cond_broadcast (&local->reply_cond);    \
-                }                                                       \
-                pthread_mutex_unlock (&local->lock);                    \
-        } while (0)
-
-
-void
-libgf_client_loc_wipe (loc_t *loc);
-
-int32_t
-libgf_client_loc_fill (loc_t *loc,
-                       libglusterfs_client_ctx_t *ctx,
-                       ino_t ino,
-                       ino_t par,
-                       const char *name);
-
-int32_t
-libgf_client_path_lookup (loc_t *loc,
-                          libglusterfs_client_ctx_t *ctx,
-                          char lookup_basename);
-
-int32_t
-libgf_client_lookup (libglusterfs_client_ctx_t *ctx,
-                     loc_t *loc,
-                     struct iatt *stbuf,
-                     dict_t **dict,
-                     dict_t *xattr_req);
-
-/* We're not expecting more than 10-15
- * VMPs per process so a list is acceptable.
- */
-struct vmp_entry {
-        struct list_head list;
-        char * vmp;
-        int vmplen;
-        glusterfs_handle_t handle;
-};
-
-#define LIBGF_UPDATE_LOOKUP     0x1
-#define LIBGF_UPDATE_STAT       0x2
-#define LIBGF_UPDATE_ALL        (LIBGF_UPDATE_LOOKUP | LIBGF_UPDATE_STAT)
-
-#define LIBGF_VALIDATE_LOOKUP  0x1
-#define LIBGF_VALIDATE_STAT     0x2
-
-#define LIBGF_INVALIDATE_LOOKUP  0x1
-#define LIBGF_INVALIDATE_STAT     0x2
-int
-libgf_is_iattr_cache_valid (libglusterfs_client_ctx_t *ctx, inode_t *inode,
-                            struct iatt *sbuf, int flags);
-
-int
-libgf_update_iattr_cache (inode_t *inode, int flags, struct iatt *buf);
-
-#endif
diff --git a/libglusterfsclient/src/libglusterfsclient.c b/libglusterfsclient/src/libglusterfsclient.c
deleted file mode 100755
index b4c1d1edc..000000000
--- a/libglusterfsclient/src/libglusterfsclient.c
+++ /dev/null
@@ -1,8169 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-
-#include <stdio.h>
-#include <errno.h>
-#include <libgen.h>
-#include <stddef.h>
-
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#ifdef GF_SOLARIS_HOST_OS
-#include <sys/statfs.h>
-#endif
-#include <unistd.h>
-#include <xlator.h>
-#include <timer.h>
-#include "defaults.h"
-#include <time.h>
-#include <poll.h>
-#include "transport.h"
-#include "event.h"
-#include "libglusterfsclient.h"
-#include "libglusterfsclient-internals.h"
-#include "compat.h"
-#include "compat-errno.h"
-#ifndef GF_SOLARIS_HOST_OS
-#include <sys/vfs.h>
-#endif
-#include <utime.h>
-#include <sys/param.h>
-#include <list.h>
-#include <stdarg.h>
-#include <sys/statvfs.h>
-#include "hashfn.h"
-#include <sys/select.h>
-
-#define LIBGF_XL_NAME "libglusterfsclient"
-#define LIBGLUSTERFS_INODE_TABLE_LRU_LIMIT 1000 //14057
-#define LIBGF_SENDFILE_BLOCK_SIZE 4096
-#define LIBGF_READDIR_BLOCK     4096
-#define libgf_path_absolute(path) ((path)[0] == '/')
-
-static inline xlator_t *
-libglusterfs_graph (xlator_t *graph);
-int32_t libgf_client_readlink (libglusterfs_client_ctx_t *ctx, loc_t *loc,
-                                        char *buf, size_t bufsize);
-
-int
-libgf_realpath_loc_fill (libglusterfs_client_ctx_t *ctx, char *link,
-                         loc_t *targetloc);
-static int first_init = 1;
-
-/* The global list of virtual mount points */
-struct {
-        struct list_head list;
-        int              entries;
-}vmplist;
-
-
-/* Protects the VMP list above. */
-pthread_mutex_t vmplock = PTHREAD_MUTEX_INITIALIZER;
-
-/* Ensures only one thread is ever calling glusterfs_mount.
- * Since that function internally calls routines which
- * use the yacc parser code using global vars, this process
- * needs to be syncronised.
- */
-pthread_mutex_t mountlock = PTHREAD_MUTEX_INITIALIZER;
-
-static char cwd[PATH_MAX];
-static char cwd_inited = 0;
-static pthread_mutex_t cwdlock   = PTHREAD_MUTEX_INITIALIZER;
-
-char *
-libgf_vmp_virtual_path (struct vmp_entry *entry, const char *path, char *vpath)
-{
-        char    *tmp = NULL;
-
-        tmp = ((char *)(path + (entry->vmplen-1)));
-        if (strlen (tmp) > 0) {
-                if (tmp[0] != '/') {
-                        vpath[0] = '/';
-                        vpath[1] = '\0';
-                        strcat (&vpath[1], tmp);
-                } else
-                        strcpy (vpath, tmp);
-        } else {
-                vpath[0] = '/';
-                vpath[1] = '\0';
-        }
-
-        return vpath;
-}
-
-char *
-zr_build_process_uuid ()
-{
-	char           tmp_str[1024] = {0,};
-	char           hostname[256] = {0,};
-	struct timeval tv = {0,};
-	struct tm      now = {0, };
-	char           now_str[32];
-
-	if (-1 == gettimeofday(&tv, NULL)) {
-		gf_log ("", GF_LOG_ERROR, 
-			"gettimeofday: failed %s",
-			strerror (errno));		
-	}
-
-	if (-1 == gethostname (hostname, 256)) {
-		gf_log ("", GF_LOG_ERROR, 
-			"gethostname: failed %s",
-			strerror (errno));
-	}
-
-	localtime_r (&tv.tv_sec, &now);
-	strftime (now_str, 32, "%Y/%m/%d-%H:%M:%S", &now);
-	snprintf (tmp_str, 1024, "%s-%d-%s:%ld", 
-		  hostname, getpid(), now_str, tv.tv_usec);
-	
-	return strdup (tmp_str);
-}
-
-
-int32_t
-libgf_client_forget (xlator_t *this,
-		     inode_t *inode)
-{
-        uint64_t ptr = 0;
-	libglusterfs_client_inode_ctx_t *ctx = NULL;
-	
-	inode_ctx_del (inode, this, &ptr);
-        ctx = (libglusterfs_client_inode_ctx_t *)(long) ptr;
-
-	FREE (ctx);
-
-        return 0;
-}
-
-xlator_t *
-libgf_inode_to_xlator (inode_t *inode)
-{
-        if (!inode)
-                return NULL;
-
-        if (!inode->table)
-                return NULL;
-
-        if (!inode->table->xl)
-                return NULL;
-
-        if (!inode->table->xl->ctx)
-                return NULL;
-
-        return inode->table->xl->ctx->top;
-}
-
-libglusterfs_client_fd_ctx_t *
-libgf_get_fd_ctx (fd_t *fd)
-{
-        uint64_t                        ctxaddr = 0;
-        libglusterfs_client_fd_ctx_t    *ctx = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, fd, out);
-
-        if (fd_ctx_get (fd, libgf_inode_to_xlator (fd->inode), &ctxaddr) == -1)
-                goto out;
-
-        ctx = (libglusterfs_client_fd_ctx_t *)(long)ctxaddr;
-
-out:
-        return ctx;
-}
-
-libglusterfs_client_fd_ctx_t *
-libgf_alloc_fd_ctx (libglusterfs_client_ctx_t *ctx, fd_t *fd, char *vpath)
-{
-        libglusterfs_client_fd_ctx_t    *fdctx = NULL;
-        uint64_t                        ctxaddr = 0;
-
-        fdctx = CALLOC (1, sizeof (*fdctx));
-        if (fdctx == NULL) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR,
-                        "memory allocation failure");
-                fdctx = NULL;
-                goto out;
-        }
-
-        pthread_mutex_init (&fdctx->lock, NULL);
-        fdctx->ctx = ctx;
-        ctxaddr = (uint64_t) (long)fdctx;
-
-        if (fd->inode) {
-                if (IA_ISDIR (fd->inode->ia_type)) {
-                        fdctx->dcache = CALLOC (1, sizeof (struct direntcache));
-                        if (fdctx->dcache)
-                                INIT_LIST_HEAD (&fdctx->dcache->entries.list);
-                        /* If the calloc fails, we can still continue
-                         * working as the dcache is not required for correct
-                         * operation.
-                         */
-                }
-        }
-
-        if (vpath != NULL) {
-                strcpy (fdctx->vpath, vpath);
-                if (vpath[strlen(vpath) - 1] != '/') {
-                        strcat (fdctx->vpath, "/");
-                }
-        }
-
-        fd_ctx_set (fd, libgf_inode_to_xlator (fd->inode), ctxaddr);
-out:
-        return fdctx;
-}
-
-libglusterfs_client_fd_ctx_t *
-libgf_del_fd_ctx (fd_t *fd)
-{
-        uint64_t                        ctxaddr = 0;
-        libglusterfs_client_fd_ctx_t    *ctx = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, fd, out);
-
-        if (fd_ctx_del (fd, libgf_inode_to_xlator (fd->inode) , &ctxaddr) == -1)
-                goto out;
-
-        ctx = (libglusterfs_client_fd_ctx_t *)(long)ctxaddr;
-
-out:
-        return ctx;
-}
-
-void
-libgf_dcache_invalidate (fd_t *fd)
-{
-        libglusterfs_client_fd_ctx_t    *fd_ctx = NULL;
-
-        if (!fd)
-                return;
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-                return;
-        }
-
-        if (!fd_ctx->dcache) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No dcache present");
-                return;
-        }
-
-        if (!list_empty (&fd_ctx->dcache->entries.list))
-                gf_dirent_free (&fd_ctx->dcache->entries);
-
-        INIT_LIST_HEAD (&fd_ctx->dcache->entries.list);
-
-        fd_ctx->dcache->next = NULL;
-        fd_ctx->dcache->prev_off = 0;
-
-        return;
-}
-
-/* The first entry in the entries is always a placeholder
- * or the list head. The real entries begin from entries->next.
- */
-int
-libgf_dcache_update (libglusterfs_client_ctx_t *ctx, fd_t *fd,
-                     gf_dirent_t *entries)
-{
-        libglusterfs_client_fd_ctx_t    *fd_ctx = NULL;
-        int                             op_ret = -1;
-
-        if ((!ctx) || (!fd) || (!entries)) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-		goto out;
-        }
-
-        /* dcache is not enabled. */
-        if (!fd_ctx->dcache) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No dcache present");
-                op_ret = 0;
-                goto out;
-        }
-
-        /* If we're updating, we must begin with invalidating any previous
-         * entries.
-         */
-        libgf_dcache_invalidate (fd);
-
-        fd_ctx->dcache->next = entries->next;
-        /* We still need to store a pointer to the head
-         * so we start free'ing from the head when invalidation
-         * is required.
-         *
-         * Need to delink the entries from the list
-         * given to us by an underlying translators. Most translators will
-         * free this list after this call so we must preserve the dirents in
-         * order to cache them.
-         */
-        list_splice_init (&entries->list, &fd_ctx->dcache->entries.list);
-        op_ret = 0;
-out:
-        return op_ret;
-}
-
-int
-libgf_dcache_readdir (libglusterfs_client_ctx_t *ctx, fd_t *fd,
-                      struct dirent *dirp, off_t *offset)
-{
-        libglusterfs_client_fd_ctx_t    *fd_ctx = NULL;
-        int                             cachevalid = 0;
-
-        if ((!ctx) || (!fd) || (!dirp) || (!offset))
-                return 0;
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-                goto out;
-        }
-
-        if (!fd_ctx->dcache) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No dcache present");
-                goto out;
-        }
-
-        /* We've either run out of entries in the cache
-         * or the cache is empty.
-         */
-        if (!fd_ctx->dcache->next) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "No entries present");
-                goto out;
-        }
-
-        /* The dirent list is created as a circular linked list
-         * so this check is needed to ensure, we dont start
-         * reading old entries again.
-         * If we're reached this situation, the cache is exhausted
-         * and we'll need to pre-fetch more entries to continue serving.
-         */
-        if (fd_ctx->dcache->next == &fd_ctx->dcache->entries) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Entries exhausted");
-                goto out;
-        }
-
-        /* During sequential reading we generally expect that the offset
-         * requested is the same as the offset we served in the previous call
-         * to readdir. But, seekdir, rewinddir and libgf_dcache_invalidate
-         * require special handling because seekdir/rewinddir change the offset
-         * in the fd_ctx and libgf_dcache_invalidate changes the prev_off.
-         */
-        if (*offset != fd_ctx->dcache->prev_off) {
-                /* For all cases of the if branch above, we know that the
-                 * cache is now invalid except for the case below. It handles
-                 * the case where the two offset values above are different
-                 * but different because the previous readdir block was
-                 * exhausted, resulting in a prev_off being set to 0 in
-                 * libgf_dcache_invalidate, while the requested offset is non
-                 * zero because that is what we returned for the last dirent
-                 * of the previous readdir block.
-                 */
-                if ((*offset != 0) && (fd_ctx->dcache->prev_off == 0)) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Entries"
-                                " exhausted");
-                        cachevalid = 1;
-                } else
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Dcache"
-                                " invalidated previously");
-        } else
-                cachevalid = 1;
-
-        if (!cachevalid)
-                goto out;
-
-        dirp->d_ino = fd_ctx->dcache->next->d_ino;
-        strncpy (dirp->d_name, fd_ctx->dcache->next->d_name,
-                 fd_ctx->dcache->next->d_len);
-
-        *offset = fd_ctx->dcache->next->d_off;
-        dirp->d_off = *offset;
-        fd_ctx->dcache->prev_off = fd_ctx->dcache->next->d_off;
-        fd_ctx->dcache->next = fd_ctx->dcache->next->next;
-
-out:
-        return cachevalid;
-}
-
-
-int32_t
-libgf_client_release (xlator_t *this,
-		      fd_t *fd)
-{
-	libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (IA_ISDIR (fd->inode->ia_type)) {
-                libgf_dcache_invalidate (fd);
-                FREE (fd_ctx->dcache);
-        }
-
-        libgf_del_fd_ctx (fd);
-        if (fd_ctx != NULL) {
-                pthread_mutex_destroy (&fd_ctx->lock);
-                FREE (fd_ctx);
-        }
-
-	return 0;
-}
-
-libglusterfs_client_inode_ctx_t *
-libgf_get_inode_ctx (inode_t *inode)
-{
-        uint64_t                                ctxaddr = 0;
-        libglusterfs_client_inode_ctx_t         *ictx = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, inode, out);
-        if (inode_ctx_get (inode, libgf_inode_to_xlator (inode), &ctxaddr) < 0)
-                goto out;
-
-        ictx = (libglusterfs_client_inode_ctx_t *)(long)ctxaddr;
-
-out:
-        return ictx;
-}
-
-libglusterfs_client_inode_ctx_t *
-libgf_del_inode_ctx (inode_t *inode)
-{
-        uint64_t                                ctxaddr = 0;
-        libglusterfs_client_inode_ctx_t         *ictx = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, inode, out);
-        if (inode_ctx_del (inode, libgf_inode_to_xlator (inode), &ctxaddr) < 0)
-                goto out;
-
-        ictx = (libglusterfs_client_inode_ctx_t *)(long)ctxaddr;
-
-out:
-        return ictx;
-}
-
-libglusterfs_client_inode_ctx_t *
-libgf_alloc_inode_ctx (libglusterfs_client_ctx_t *ctx, inode_t *inode)
-{
-        uint64_t                                ctxaddr = 0;
-        libglusterfs_client_inode_ctx_t         *ictx = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, inode, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        ictx = CALLOC (1, sizeof (*ictx));
-        if (ictx == NULL) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR,
-                                "memory allocation failure");
-                goto out;
-        }
-
-        pthread_mutex_init (&ictx->lock, NULL);
-        ctxaddr = (uint64_t) (long)ictx;
-        if (inode_ctx_put (inode, libgf_inode_to_xlator (inode), ctxaddr) < 0){
-                FREE (ictx);
-                ictx = NULL;
-        }
-
-out:
-        return ictx;
-}
-
-int
-libgf_transform_iattr (libglusterfs_client_ctx_t *libctx, inode_t *inode,
-                       struct iatt *buf)
-{
-
-        if ((!libctx) || (!buf) || (!inode))
-                return -1;
-
-        buf->ia_dev = libctx->fake_fsid;
-        /* If the inode is root, the inode number must be 1 not the
-         * ino received from the file system.
-         */
-        if ((inode->ino == 1) && (buf))
-                buf->ia_ino = 1;
-
-        return 0;
-}
-
-int
-libgf_update_iattr_cache (inode_t *inode, int flags, struct iatt *buf)
-{
-        libglusterfs_client_inode_ctx_t *inode_ctx = NULL;
-        time_t                          current = 0;
-        int                             op_ret = -1;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, inode, out);
-
-        inode_ctx = libgf_get_inode_ctx (inode);
-        if (!inode_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No inode context"
-                        " present");
-                errno = EINVAL;
-                op_ret = -1;
-                goto out;
-        }
-
-        pthread_mutex_lock (&inode_ctx->lock);
-        {
-                /* Take a timestamp only after we've acquired the
-                 * lock.
-                 */
-                current = time (NULL);
-                if (flags & LIBGF_UPDATE_LOOKUP) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Updating lookup");
-                        inode_ctx->previous_lookup_time = current;
-                }
-
-                if (flags & LIBGF_UPDATE_STAT) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Updating stat");
-
-                        /* Update the cached stat struct only if a new
-                         * stat buf is given.
-                         */
-                        if (buf != NULL) {
-                                inode_ctx->previous_stat_time = current;
-                                memcpy (&inode_ctx->stbuf, buf,
-                                                sizeof (inode_ctx->stbuf));
-                        }
-                }
-        }
-        pthread_mutex_unlock (&inode_ctx->lock);
-        op_ret = 0;
-
-out:
-        return op_ret;
-}
-
-
-int
-libgf_invalidate_iattr_cache (inode_t *inode, int flags)
-{
-        libglusterfs_client_inode_ctx_t         *ictx = NULL;
-
-        if (!inode)
-                return -1;
-
-        ictx = libgf_get_inode_ctx (inode);
-        if (!ictx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No inode context"
-                        " present");
-                return -1;
-        }
-
-        pthread_mutex_lock (&ictx->lock);
-        {
-                if (flags & LIBGF_INVALIDATE_LOOKUP) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Invalidating"
-                                " lookup");
-                        ictx->previous_lookup_time = 0;
-                }
-
-                if (flags & LIBGF_INVALIDATE_STAT) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Invalidating"
-                                " stat");
-                        ictx->previous_stat_time = 0;
-                }
-
-        }
-        pthread_mutex_unlock (&ictx->lock);
-
-        return 0;
-}
-
-
-int
-libgf_is_iattr_cache_valid (libglusterfs_client_ctx_t *ctx, inode_t *inode,
-                            struct iatt *sbuf, int flags)
-{
-        time_t                                  current = 0;
-        time_t                                  prev = 0;
-        libglusterfs_client_inode_ctx_t         *inode_ctx = NULL;
-        int                                     cache_valid = 0;
-        time_t                                  timeout = 0;
-
-        if (inode == NULL)
-                return 0;
-
-        inode_ctx = libgf_get_inode_ctx (inode);
-        if (!inode_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No inode context"
-                        " present\n");
-                return 0;
-        }
-
-        pthread_mutex_lock (&inode_ctx->lock);
-        {
-                current = time (NULL);
-                if (flags & LIBGF_VALIDATE_LOOKUP) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Checking lookup");
-                        prev = inode_ctx->previous_lookup_time;
-                        timeout = ctx->lookup_timeout;
-                } else {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Checking stat");
-                        prev = inode_ctx->previous_stat_time;
-                        timeout = ctx->stat_timeout;
-                }
-
-                /* Even if the timeout is set to -1 to cache
-                 * infinitely, fops like write must invalidate the
-                 * stat cache because writev_cbk cannot update
-                 * the cache using the stat returned to it. This is
-                 * because write-behind can return a stat bufs filled
-                 * with zeroes.
-                 */
-                if (prev == 0) {
-                        cache_valid = 0;
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Cache Invalid");
-                        goto iattr_unlock_out;
-                }
-
-                /* Cache infinitely */
-                if (timeout == (time_t)-1) {
-                        cache_valid = 1;
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Caching On and "
-                                "valid");
-                        goto iattr_unlock_out;
-                }
-
-                /* Disable caching completely */
-                if (timeout == 0) {
-                        cache_valid = 0;
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Cache disabled");
-                        goto iattr_unlock_out;
-                }
-
-                if ((prev > 0) && (timeout >= (current - prev))) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Cache valid");
-                        cache_valid = 1;
-                }
-
-                if (flags & LIBGF_VALIDATE_LOOKUP)
-                        goto iattr_unlock_out;
-
-                if ((cache_valid) && (sbuf))
-                        *sbuf = inode_ctx->stbuf;
-        }
-iattr_unlock_out:
-        pthread_mutex_unlock (&inode_ctx->lock);
-
-        return cache_valid;
-}
-
-int32_t
-libgf_client_releasedir (xlator_t *this,
-			 fd_t *fd)
-{
-	libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (IA_ISDIR (fd->inode->ia_type)) {
-                libgf_dcache_invalidate (fd);
-                FREE (fd_ctx->dcache);
-        }
-
-        libgf_del_fd_ctx (fd);
-        if (fd_ctx != NULL) {
-                pthread_mutex_destroy (&fd_ctx->lock);
-                FREE (fd_ctx);
-        }
-
-	return 0;
-}
-
-void *poll_proc (void *ptr)
-{
-        glusterfs_ctx_t *ctx = ptr;
-
-        event_dispatch (ctx->event_pool);
-
-        return NULL;
-}
-
-
-int32_t
-xlator_graph_init (xlator_t *xl)
-{
-        xlator_t *trav = xl;
-        int32_t ret = -1;
-
-        while (trav->prev)
-                trav = trav->prev;
-
-        while (trav) {
-                if (!trav->ready) {
-                        ret = xlator_tree_init (trav);
-                        if (ret < 0)
-                                break;
-                }
-                trav = trav->next;
-        }
-
-        return ret;
-}
-
-
-void
-xlator_graph_fini (xlator_t *xl)
-{
-	xlator_t *trav = xl;
-	while (trav->prev)
-		trav = trav->prev;
-
-	while (trav) {
-		if (!trav->init_succeeded) {
-			break;
-		}
-
-		xlator_tree_fini (trav);
-		trav = trav->next;
-	}
-}
-
-/* Returns a pointer to the @n'th char matching
- * @c in string @str, starting the search from right or
- * end-of-string, rather than starting from left, as rindex
- * function does.
- */
-char *
-libgf_rrindex (char *str, int c, int n)
-{
-        int     len = 0;
-        int     occurrence = 0;
-
-        if (str == NULL)
-                return NULL;
-
-        len = strlen (str);
-        /* Point to last character of string. */
-        str += (len - 1);
-        while (len > 0) {
-                if ((int)*str == c) {
-                        ++occurrence;
-                        if (occurrence == n)
-                                break;
-                }
-                --len;
-                --str;
-        }
-
-        return str;
-}
-
-char *
-libgf_trim_to_prev_dir (char * path)
-{
-        char    *idx = NULL;
-        int      len = 0;
-
-        if (!path)
-                return NULL;
-
-        /* Check if we're already at root, if yes
-         * then there is no prev dir.
-         */
-        len = strlen (path);
-        if (len == 1)
-                return path;
-
-        if (path[len - 1] == '/') {
-                path[len - 1] = '\0';
-        }
-
-        idx = libgf_rrindex (path, '/', 1);
-        /* Move to the char after the / */
-        ++idx;
-        *idx = '\0';
-
-        return path;
-}
-
-
-char *
-libgf_prepend_cwd (const char *userpath, char *abspath, int size)
-{
-        if ((!userpath) || (!abspath))
-                return NULL;
-
-        if (!getcwd (abspath, size))
-                return NULL;
-
-        strcat (abspath, "/");
-        strcat (abspath, userpath);
-
-        return abspath;
-}
-
-
-/* Performs a lightweight path resolution that only
- * looks for . and  .. and replaces those with the
- * proper names.
- *
- * FIXME: This is a stop-gap measure till we have full
- * fledge path resolution going in here.
- * Function returns path strdup'ed so remember to FREE the
- * string as required.
- */
-char *
-libgf_resolve_path_light (char *path)
-{
-        char            *respath = NULL;
-        char            *saveptr = NULL;
-        char            *tok = NULL;
-        int             len = 0;
-        int             addslash = 0;
-        char            mypath[PATH_MAX];
-
-        if (!path)
-                goto out;
-
-        memset (mypath, 0, PATH_MAX);
-
-        if (!libgf_path_absolute (path))
-                libgf_prepend_cwd (path, mypath, PATH_MAX);
-        else
-                strcpy (mypath, path);
-
-        len = strlen (mypath);
-        if (len == 0) {
-                goto out;
-        }
-
-        respath = calloc (PATH_MAX, sizeof (char));
-        if (respath == NULL) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR,"Memory allocation failed");
-                goto out;
-        }
-
-        /* The path only contains a / or a //, so simply add a /
-         * and return.
-         * This needs special handling because the loop below does
-         * not allow us to do so through strtok.
-         */
-        if (((mypath[0] == '/') && (len == 1))
-                        || (strcmp (mypath, "//") == 0)) {
-                strcat (respath, "/");
-                goto out;
-        }
-
-        tok = strtok_r (mypath, "/", &saveptr);
-        addslash = 0;
-        strcat (respath, "/");
-        while (tok) {
-                if (addslash) {
-                        if ((strcmp (tok, ".") != 0)
-                                        && (strcmp (tok, "..") != 0)) {
-                                strcat (respath, "/");
-                        }
-                }
-
-                if ((strcmp (tok, ".") != 0) && (strcmp (tok, "..") != 0)) {
-                        strcat (respath, tok);
-                        addslash = 1;
-                } else if ((strcmp (tok, "..") == 0)) {
-                        libgf_trim_to_prev_dir (respath);
-                        addslash = 0;
-                }
-
-                tok = strtok_r (NULL, "/", &saveptr);
-        }
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Path: %s, Resolved Path: %s",
-                path, respath);
-out:
-        return respath;
-}
-
-void 
-libgf_client_loc_wipe (loc_t *loc)
-{
-	if (loc->path) {
-		FREE (loc->path);
-	}
-
-	if (loc->parent) { 
-		inode_unref (loc->parent);
-		loc->parent = NULL;
-	}
-
-	if (loc->inode) {
-		inode_unref (loc->inode);
-		loc->inode = NULL;
-	}
-
-	loc->path = loc->name = NULL;
-        loc->ino = 0;
-}
-
-
-int32_t
-libgf_client_loc_fill (loc_t *loc,
-		       libglusterfs_client_ctx_t *ctx,
-		       ino_t ino,
-		       ino_t par,
-		       const char *name)
-{
-        inode_t *inode = NULL, *parent = NULL;
-	int32_t ret = -1;
-	char *path = NULL;
-
-        /* resistance against multiple invocation of loc_fill not to get
-           reference leaks via inode_search() */
-
-        inode = loc->inode;
-	
-        if (!inode) {
-                if (ino)
-                        inode = inode_search (ctx->itable, ino, NULL);
-
-                if (inode)
-                        goto inode_found;
-
-                if (par && name)
-                        inode = inode_search (ctx->itable, par, name);
-        }
-
-inode_found:
-        if (inode) {
-                loc->ino = inode->ino;
-                loc->inode = inode;
-        }
-
-        parent = loc->parent;
-        if (!parent) {
-                if (inode)
-                        parent = inode_parent (inode, par, name);
-                else
-                        parent = inode_search (ctx->itable, par, NULL);
-                loc->parent = parent;
-        }
-  
-	if (!loc->path) {
-		if (name && parent) {
-			ret = inode_path (parent, name, &path);
-			if (ret <= 0) {
-				gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-					"inode_path failed for %"PRId64"/%s",
-					parent->ino, name);
-				goto fail;
-			} else {
-				loc->path = path;
-			}
-		} else 	if (inode) {
-			ret = inode_path (inode, NULL, &path);
-			if (ret <= 0) {
-				gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-					"inode_path failed for %"PRId64,
-					inode->ino);
-				goto fail;
-			} else {
-				loc->path = path;
-			}
-		}
-	}
-
-	if (loc->path) {
-		loc->name = strrchr (loc->path, '/');
-		if (loc->name)
-			loc->name++;
-		else loc->name = "";
-	}
-	
-	if ((ino != 1) &&
-	    (parent == NULL)) {
-		gf_log ("fuse-bridge", GF_LOG_ERROR,
-			"failed to search parent for %"PRId64"/%s (%"PRId64")",
-			(ino_t)par, name, (ino_t)ino);
-		ret = -1;
-		goto fail;
-	}
-	ret = 0;
-fail:
-	return ret;
-}
-
-
-static call_frame_t *
-get_call_frame_for_req (libglusterfs_client_ctx_t *ctx, char d)
-{
-        call_pool_t  *pool = ctx->gf_ctx.pool;
-        xlator_t     *this = ctx->gf_ctx.graph;
-        call_frame_t *frame = NULL;
-  
-
-        frame = create_frame (this, pool);
-
-        frame->root->uid = geteuid ();
-        frame->root->gid = getegid ();
-        frame->root->pid = ctx->pid;
-        frame->root->unique = ctx->counter++;
-  
-        return frame;
-}
-
-void 
-libgf_client_fini (xlator_t *this)
-{
-	FREE (this->private);
-        return;
-}
-
-
-int32_t
-libgf_client_notify (xlator_t *this, 
-                     int32_t event,
-                     void *data, 
-                     ...)
-{
-        libglusterfs_client_private_t *priv = this->private;
-
-        switch (event)
-        {
-        case GF_EVENT_CHILD_UP:
-                pthread_mutex_lock (&priv->lock);
-                {
-                        priv->complete = 1;
-                        pthread_cond_broadcast (&priv->init_con_established);
-                }
-                pthread_mutex_unlock (&priv->lock);
-                break;
-
-        default:
-                default_notify (this, event, data);
-        }
-
-        return 0;
-}
-
-int32_t 
-libgf_client_init (xlator_t *this)
-{
-        return 0;
-}
-
-glusterfs_handle_t 
-glusterfs_init (glusterfs_init_params_t *init_ctx, uint32_t fakefsid)
-{
-        libglusterfs_client_ctx_t *ctx = NULL;
-        libglusterfs_client_private_t *priv = NULL;
-        FILE *specfp = NULL;
-        xlator_t *graph = NULL, *trav = NULL;
-        call_pool_t *pool = NULL;
-        int32_t ret = 0;
-        struct rlimit lim;
-	uint32_t xl_count = 0;
-        loc_t       new_loc = {0, };
-        struct timeval tv = {0, };
-        uint32_t       len = 0;
-        char           buf[PATH_MAX];
-
-        if (!init_ctx || (!init_ctx->specfile && !init_ctx->specfp)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        ctx = CALLOC (1, sizeof (*ctx));
-        if (!ctx) {
-		fprintf (stderr, 
-			 "libglusterfsclient: %s:%s():%d: out of memory\n",
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__);
-
-                errno = ENOMEM;
-                return NULL;
-        }
-
-        ctx->lookup_timeout = init_ctx->lookup_timeout;
-        ctx->stat_timeout = init_ctx->stat_timeout;
-        ctx->fake_fsid = fakefsid;
-        ctx->pid = getpid ();
-        pthread_mutex_init (&ctx->gf_ctx.lock, NULL);
-  
-        pool = ctx->gf_ctx.pool = CALLOC (1, sizeof (call_pool_t));
-        if (!pool) {
-                errno = ENOMEM;
-                FREE (ctx);
-                return NULL;
-        }
-
-        LOCK_INIT (&pool->lock);
-        INIT_LIST_HEAD (&pool->all_frames);
-
-	/* FIXME: why is count hardcoded to 16384 */
-        ctx->gf_ctx.event_pool = event_pool_new (16384);
-        ctx->gf_ctx.page_size  = LIBGF_IOBUF_SIZE;
-        ctx->gf_ctx.iobuf_pool = iobuf_pool_new (8 * 1048576,
-                                                 ctx->gf_ctx.page_size);
-
-        lim.rlim_cur = RLIM_INFINITY;
-        lim.rlim_max = RLIM_INFINITY;
-        setrlimit (RLIMIT_CORE, &lim);
-        setrlimit (RLIMIT_NOFILE, &lim);
-
-        ctx->gf_ctx.cmd_args.log_level = GF_LOG_WARNING;
-
-        if (init_ctx->logfile)
-                ctx->gf_ctx.cmd_args.log_file = strdup (init_ctx->logfile);
-        else
-                ctx->gf_ctx.cmd_args.log_file = strdup ("/dev/stderr");
-
-        if (init_ctx->loglevel) {
-                if (!strncasecmp (init_ctx->loglevel, "DEBUG",
-                                  strlen ("DEBUG"))) {
-                        ctx->gf_ctx.cmd_args.log_level = GF_LOG_DEBUG;
-                } else if (!strncasecmp (init_ctx->loglevel, "WARNING",
-                                         strlen ("WARNING"))) {
-                        ctx->gf_ctx.cmd_args.log_level = GF_LOG_WARNING;
-                } else if (!strncasecmp (init_ctx->loglevel, "CRITICAL",
-                                         strlen ("CRITICAL"))) {
-                        ctx->gf_ctx.cmd_args.log_level = GF_LOG_CRITICAL;
-                } else if (!strncasecmp (init_ctx->loglevel, "NONE",
-                                         strlen ("NONE"))) {
-                        ctx->gf_ctx.cmd_args.log_level = GF_LOG_NONE;
-                } else if (!strncasecmp (init_ctx->loglevel, "ERROR",
-                                         strlen ("ERROR"))) {
-                        ctx->gf_ctx.cmd_args.log_level = GF_LOG_ERROR;
-                } else if (!strncasecmp (init_ctx->loglevel, "TRACE",
-                                         strlen ("TRACE"))) {
-                        ctx->gf_ctx.cmd_args.log_level = GF_LOG_TRACE;
-                } else {
-			fprintf (stderr, 
-				 "libglusterfsclient: %s:%s():%d: Unrecognized log-level \"%s\", possible values are \"DEBUG|WARNING|[ERROR]|CRITICAL|NONE|TRACE\"\n",
-                                 __FILE__, __PRETTY_FUNCTION__, __LINE__,
-                                 init_ctx->loglevel);
-			FREE (ctx->gf_ctx.cmd_args.log_file);
-                        FREE (ctx->gf_ctx.pool);
-                        FREE (ctx->gf_ctx.event_pool);
-                        FREE (ctx);
-                        errno = EINVAL;
-                        return NULL;
-                }
-        }
-
-	if (first_init)
-        {
-                memset (buf, 0, PATH_MAX);
-
-                if (getcwd (buf, PATH_MAX) == NULL) {
-                        fprintf (stderr, "libglusterfsclient: cannot get "
-                                 "current working directory (%s)",
-                                 strerror (errno));
-			FREE (ctx->gf_ctx.cmd_args.log_file);
-                        FREE (ctx->gf_ctx.pool);
-                        FREE (ctx->gf_ctx.event_pool);
-                        FREE (ctx);
-                        return NULL;
-                }
-
-                len = strlen (buf);
-                if ((buf[len - 1] != '/')) {
-                        if ((len + 2) > PATH_MAX) {
-                                errno = ENAMETOOLONG;
-                                fprintf (stderr, "libglusterfsclient: cannot"
-                                         "get current working directory (%s)",
-                                         strerror (errno));
-                                FREE (ctx->gf_ctx.cmd_args.log_file);
-                                FREE (ctx->gf_ctx.pool);
-                                FREE (ctx->gf_ctx.event_pool);
-                                FREE (ctx);
-                                return NULL;
-                        }
-
-                        strcat (buf, "/");
-                }
-
-                pthread_mutex_lock (&cwdlock);
-                {
-                        strcpy (cwd, buf);
-                        cwd_inited = 1;
-                }
-                pthread_mutex_unlock (&cwdlock);
-
-                ret = gf_log_init (ctx->gf_ctx.cmd_args.log_file);
-                if (ret == -1) {
-			fprintf (stderr, 
-				 "libglusterfsclient: %s:%s():%d: failed to open logfile \"%s\"\n", 
-				 __FILE__, __PRETTY_FUNCTION__, __LINE__, 
-				 ctx->gf_ctx.cmd_args.log_file);
-			FREE (ctx->gf_ctx.cmd_args.log_file);
-                        FREE (ctx->gf_ctx.pool);
-                        FREE (ctx->gf_ctx.event_pool);
-                        FREE (ctx);
-                        return NULL;
-                }
-
-                gf_log_set_loglevel (ctx->gf_ctx.cmd_args.log_level);
-        }
-
-        if (init_ctx->specfp) {
-                specfp = init_ctx->specfp;
-                if (fseek (specfp, 0L, SEEK_SET)) {
-			fprintf (stderr, 
-				 "libglusterfsclient: %s:%s():%d: fseek on volume file stream failed (%s)\n",
-                                 __FILE__, __PRETTY_FUNCTION__, __LINE__,
-                                 strerror (errno));
-			FREE (ctx->gf_ctx.cmd_args.log_file);
-                        FREE (ctx->gf_ctx.pool);
-                        FREE (ctx->gf_ctx.event_pool);
-                        FREE (ctx);
-                        return NULL;
-                }
-        } else if (init_ctx->specfile) { 
-                specfp = fopen (init_ctx->specfile, "r");
-                ctx->gf_ctx.cmd_args.volume_file = strdup (init_ctx->specfile);
-        }
-
-        if (!specfp) {
-		fprintf (stderr, 
-			 "libglusterfsclient: %s:%s():%d: could not open volfile: %s\n", 
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__,
-                         strerror (errno));
-		FREE (ctx->gf_ctx.cmd_args.log_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_file);
-                FREE (ctx->gf_ctx.pool);
-                FREE (ctx->gf_ctx.event_pool);
-                FREE (ctx);
-                return NULL;
-        }
-
-        if (init_ctx->volume_name) {
-                ctx->gf_ctx.cmd_args.volume_name = strdup (init_ctx->volume_name);
-        }
-
-	graph = file_to_xlator_tree (&ctx->gf_ctx, specfp);
-        if (!graph) {
-		fprintf (stderr, 
-			 "libglusterfsclient: %s:%s():%d: cannot create configuration graph (%s)\n",
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__,
-                         strerror (errno));
-
-		FREE (ctx->gf_ctx.cmd_args.log_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_name);
-                FREE (ctx->gf_ctx.pool);
-                FREE (ctx->gf_ctx.event_pool);
-                FREE (ctx);
-                return NULL;
-        }
-
-        if (init_ctx->volume_name) {
-                trav = graph;
-                while (trav) {
-                        if (strcmp (trav->name, init_ctx->volume_name) == 0) {
-                                graph = trav;
-                                break;
-                        }
-                        trav = trav->next;
-                }
-        }
-
-        ctx->gf_ctx.graph = libglusterfs_graph (graph);
-        if (!ctx->gf_ctx.graph) {
-		fprintf (stderr, 
-			 "libglusterfsclient: %s:%s():%d: graph creation failed (%s)\n",
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__,
-                         strerror (errno));
-
-		xlator_tree_free (graph);
-		FREE (ctx->gf_ctx.cmd_args.log_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_name);
-                FREE (ctx->gf_ctx.pool);
-                FREE (ctx->gf_ctx.event_pool);
-                FREE (ctx);
-                return NULL;
-        }
-        graph = ctx->gf_ctx.graph;
-        ctx->gf_ctx.top = graph;
-
-	trav = graph;
-	while (trav) {
-		xl_count++;  /* Getting this value right is very important */
-		trav = trav->next;
-	}
-
-	ctx->gf_ctx.xl_count = xl_count + 1;
-
-        priv = CALLOC (1, sizeof (*priv));
-        if (!priv) {
-		fprintf (stderr, 
-			 "libglusterfsclient: %s:%s():%d: cannot allocate memory (%s)\n",
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__,
-                         strerror (errno));
-
-		xlator_tree_free (graph);
-		FREE (ctx->gf_ctx.cmd_args.log_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_name);
-                FREE (ctx->gf_ctx.pool);
-                FREE (ctx->gf_ctx.event_pool);
-                /* inode_table_destroy (ctx->itable); */
-                FREE (ctx);
-         
-                return NULL;
-        }
-
-        pthread_cond_init (&priv->init_con_established, NULL);
-        pthread_mutex_init (&priv->lock, NULL);
-
-        graph->private = priv;
-        ctx->itable = inode_table_new (LIBGLUSTERFS_INODE_TABLE_LRU_LIMIT,
-                                       graph);
-        if (!ctx->itable) {
-		fprintf (stderr, 
-			 "libglusterfsclient: %s:%s():%d: cannot create inode table\n",
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__);
-		xlator_tree_free (graph); 
-		FREE (ctx->gf_ctx.cmd_args.log_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_name);
-
-                FREE (ctx->gf_ctx.pool);
-                FREE (ctx->gf_ctx.event_pool);
-		xlator_tree_free (graph); 
-                /* TODO: destroy graph */
-                /* inode_table_destroy (ctx->itable); */
-                FREE (ctx);
-         
-                return NULL;
-        }
-
-	set_global_ctx_ptr (&ctx->gf_ctx);
-	ctx->gf_ctx.process_uuid = zr_build_process_uuid ();
-
-        if (xlator_graph_init (graph) == -1) {
-		fprintf (stderr, 
-			 "libglusterfsclient: %s:%s():%d: graph initialization failed\n",
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__);
-		xlator_tree_free (graph);
-		FREE (ctx->gf_ctx.cmd_args.log_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_name);
-                FREE (ctx->gf_ctx.pool);
-                FREE (ctx->gf_ctx.event_pool);
-                /* TODO: destroy graph */
-                /* inode_table_destroy (ctx->itable); */
-                FREE (ctx);
-                return NULL;
-        }
-
-	/* Send notify to all translator saying things are ready */
-	graph->notify (graph, GF_EVENT_PARENT_UP, graph);
-
-        if (gf_timer_registry_init (&ctx->gf_ctx) == NULL) {
-		fprintf (stderr, 
-			 "libglusterfsclient: %s:%s():%d: timer init failed (%s)\n", 
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__,
-                         strerror (errno));
-
-		xlator_graph_fini (graph);
-		xlator_tree_free (graph);
-		FREE (ctx->gf_ctx.cmd_args.log_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_name);
-
-                FREE (ctx->gf_ctx.pool);
-                FREE (ctx->gf_ctx.event_pool);
-                /* TODO: destroy graph */
-                /* inode_table_destroy (ctx->itable); */
-                FREE (ctx);
-                return NULL;
-        }
-
-        if ((ret = pthread_create (&ctx->reply_thread, NULL, poll_proc,
-                                   (void *)&ctx->gf_ctx))) {
-		fprintf (stderr, 
-			 "libglusterfsclient: %s:%s():%d: reply thread creation failed\n", 
-			 __FILE__, __PRETTY_FUNCTION__, __LINE__);
-		xlator_graph_fini (graph);
-		xlator_tree_free (graph);
-		FREE (ctx->gf_ctx.cmd_args.log_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_file);
-                FREE (ctx->gf_ctx.cmd_args.volume_name);
-
-                FREE (ctx->gf_ctx.pool);
-                FREE (ctx->gf_ctx.event_pool);
-                /* TODO: destroy graph */
-                /* inode_table_destroy (ctx->itable); */
-                FREE (ctx);
-                return NULL;
-        }
-
-        pthread_mutex_lock (&priv->lock); 
-        {
-                while (!priv->complete) {
-                        pthread_cond_wait (&priv->init_con_established,
-                                           &priv->lock);
-                }
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        /* 
-         * wait for some time to allow initialization of all children of 
-         * distribute before sending lookup on '/'
-         */
-
-        tv.tv_sec = 0;
-        tv.tv_usec = (100 * 1000);
-        select (0, NULL, NULL, NULL, &tv);
-
-        /* workaround for xlators like dht which require lookup to be sent
-         * on / */
-        libgf_client_loc_fill (&new_loc, ctx, 1, 0, "/");
-        ret = libgf_client_lookup (ctx, &new_loc, NULL, NULL, NULL);
-        if (ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR, "lookup of /"
-                        " failed");
-                return NULL;
-        }
-        libgf_client_loc_wipe (&new_loc);
-
-	first_init = 0;
- 
-        return ctx;
-}
-
-struct vmp_entry *
-libgf_init_vmpentry (char *vmp, glusterfs_handle_t *vmphandle)
-{
-        struct vmp_entry        *entry = NULL;
-        int                     vmplen = 0;
-        int                     appendslash = 0;
-        int                     ret = -1;
-
-        entry = CALLOC (1, sizeof (struct vmp_entry));
-        if (!entry) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR,"Memory allocation failed");
-                return NULL;
-        }
-
-        vmplen = strlen (vmp);
-        if (vmp[vmplen - 1] != '/') {
-                vmplen++;
-                appendslash = 1;
-        }
-
-        entry->vmp = CALLOC (vmplen + 1, sizeof (char));
-        if (!entry->vmp) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Memory allocation "
-                        "failed");
-                goto free_entry;
-        }
-
-        strcpy (entry->vmp, vmp);
-        if (appendslash) {
-                entry->vmp[vmplen-1] = '/';
-                entry->vmp[vmplen] = '\0';
-        }
- 
-        entry->vmplen = vmplen;
-        entry->handle = vmphandle;
-        INIT_LIST_HEAD (&entry->list);
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "New VMP entry: %s", vmp);
-
-        ret = 0;
-
-free_entry:
-        if (ret == -1) {
-                if (entry->vmp)
-                        FREE (entry->vmp);
-                if (entry)
-                        FREE (entry);
-                entry = NULL;
-        }
-        return entry;
-}
-
-void
-libgf_free_vmp_entry (struct vmp_entry *entry)
-{
-        FREE (entry->vmp);
-        FREE (entry);
-}
-
-int
-libgf_count_path_components (char *path)
-{
-        int     compos = 0;
-        char    *pathdup = NULL;
-        int     len = 0;
-
-        if (!path)
-                return -1;
-
-        pathdup = strdup (path);
-        if (!pathdup)
-                return -1;
-
-        len = strlen (pathdup);
-        if (pathdup[len - 1] == '/')
-                pathdup[len - 1] = '\0';
-
-        path = pathdup;
-        while ((path = strchr (path, '/'))) {
-                compos++;
-                ++path;
-        }
-
-        free (pathdup);
-        return compos;
-}
-
-/* Returns the number of components that match between
- * the VMP and the path. Assumes string1 is vmp entry.
- * Assumes both are absolute paths.
- */
-int
-libgf_strmatchcount (char *string1, char *string2)
-{
-        int     matchcount = 0;
-        char    *s1dup = NULL, *s2dup = NULL;
-        char    *tok1 = NULL, *saveptr1 = NULL;
-        char    *tok2 = NULL, *saveptr2 = NULL;
-
-        if ((!string1) || (!string2))
-                return 0;
-
-        s1dup = strdup (string1);
-        if (!s1dup)
-                return 0;
-
-        s2dup  = strdup (string2);
-        if (!s2dup)
-                goto free_s1;
-
-        string1 = s1dup;
-        string2 = s2dup;
-
-        tok1 = strtok_r(string1, "/", &saveptr1);
-        tok2 = strtok_r (string2, "/", &saveptr2);
-        while (tok1) {
-                if (!tok2)
-                        break;
-
-                if (strcmp (tok1, tok2) != 0)
-                        break;
-
-                matchcount++;
-                tok1 = strtok_r(NULL, "/", &saveptr1);
-                tok2 = strtok_r (NULL, "/", &saveptr2);
-        }
-
-        free (s2dup);
-free_s1:
-        free (s1dup);
-        return matchcount;
-}
-
-int
-libgf_vmp_entry_match (struct vmp_entry *entry, char *path)
-{
-        return libgf_strmatchcount (entry->vmp, path);
-}
-
-#define LIBGF_VMP_EXACT          1
-#define LIBGF_VMP_LONGESTPREFIX  0
-
-
-/* copies vmp from the vmp-entry having glusterfs handle @handle, into @vmp */
-char *
-libgf_vmp_search_vmp (glusterfs_handle_t handle, char *vmp, size_t vmp_size)
-{
-        char             *res   = NULL;
-        struct vmp_entry *entry = NULL;
-
-        if (handle == NULL) {
-                goto out;
-        }
-
-        pthread_mutex_lock (&vmplock);
-        {
-                if (vmplist.entries == 0) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Virtual Mount Point "
-                                "list is empty.");
-                        goto unlock;
-                }
-
-                list_for_each_entry(entry, &vmplist.list, list) {
-                        if (entry->handle == handle) {
-                                if ((vmp_size) < (strlen (entry->vmp) + 1)) {
-                                        errno = ENAMETOOLONG;
-                                        goto unlock;
-                                }
-
-                                strcpy (vmp, entry->vmp);
-                                res = vmp;
-                                break;
-                        }
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&vmplock);
-
-out:
-        return res;
-}
-
-
-struct vmp_entry *
-_libgf_vmp_search_entry (char *path, int searchtype)
-{
-        struct vmp_entry        *entry = NULL;
-        int                     matchcount = 0;
-        struct vmp_entry        *maxentry = NULL;
-        int                     maxcount = 0;
-        int                     vmpcompcount = 0;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "VMP Search: path %s, type: %s",
-                path, (searchtype == LIBGF_VMP_EXACT)?"Exact":"LongestPrefix");
-        if (vmplist.entries == 0) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Virtual Mount Point "
-                        "list is empty.");
-                goto out;
-        }
-
-        list_for_each_entry(entry, &vmplist.list, list) {
-                vmpcompcount = libgf_count_path_components (entry->vmp);
-                matchcount = libgf_vmp_entry_match (entry, path);
-                gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Candidate VMP:  %s,"
-                        " Matchcount: %d", entry->vmp, matchcount);
-                if ((matchcount > maxcount) && (matchcount == vmpcompcount)) {
-                        maxcount = matchcount;
-                        maxentry = entry;
-                }
-        }
-
-        /* To ensure that the longest prefix matched entry is also an exact
-         * match, this is used to check whether duplicate entries are present
-         * in the vmplist.
-         */
-        vmpcompcount = 0;
-        if ((searchtype == LIBGF_VMP_EXACT) && (maxentry)) {
-                vmpcompcount = libgf_count_path_components (maxentry->vmp);
-                matchcount = libgf_count_path_components (path);
-                gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Exact Check: VMP: %s,"
-                        " CompCount: %d, Path: %s, CompCount: %d",
-                        maxentry->vmp, vmpcompcount, path, matchcount);
-                if (vmpcompcount != matchcount) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "No Match");
-                        maxentry = NULL;
-                } else
-                        gf_log (LIBGF_XL_NAME, GF_LOG_TRACE, "Matches!");
-        }
-
-out:        
-        return maxentry;
-} 
-
-/* Used to search for a exactly matching VMP entry.
- */
-struct vmp_entry *
-libgf_vmp_search_exact_entry (char *path)
-{
-        struct vmp_entry        *entry = NULL;
-
-        if (!path)
-                goto out;
-
-        pthread_mutex_lock (&vmplock);
-        {
-                entry = _libgf_vmp_search_entry (path, LIBGF_VMP_EXACT);
-        }
-        pthread_mutex_unlock (&vmplock);
-
-out:
-        if (entry)
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "VMP Entry found: path :%s"
-                        " vmp: %s", path, entry->vmp);
-        else
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "VMP Entry not found: path"
-                        ": %s", path);
-
-        return entry;
-}
-
-
-/* Used to search for a longest prefix matching VMP entry.
- */
-struct vmp_entry *
-libgf_vmp_search_entry (char *path)
-{
-        struct vmp_entry        *entry = NULL;
-
-        if (!path)
-                goto out;
-
-        pthread_mutex_lock (&vmplock);
-        {
-                entry = _libgf_vmp_search_entry (path, LIBGF_VMP_LONGESTPREFIX);
-        }
-        pthread_mutex_unlock (&vmplock);
-
-out:
-        if (entry)
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "VMP Entry found: path :%s"
-                        " vmp: %s", path, entry->vmp);
-        else
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "VMP Entry not found: path"
-                        ": %s", path);
-
-        return entry;
-}
-
-int
-libgf_vmp_map_ghandle (char *vmp, glusterfs_handle_t *vmphandle)
-{
-        int                     ret = -1;
-        struct vmp_entry        *vmpentry = NULL;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "New Entry: %s", vmp);
-        vmpentry = libgf_init_vmpentry (vmp, vmphandle);
-        if (!vmpentry) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Failed to create VMP"
-                        " entry");
-                goto out;
-        }
-
-        pthread_mutex_lock (&vmplock);
-        {
-                if (vmplist.entries == 0) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Empty list");
-                        INIT_LIST_HEAD (&vmplist.list);
-                }
-
-                list_add_tail (&vmpentry->list, &vmplist.list);
-                ++vmplist.entries;
-        }
-        pthread_mutex_unlock (&vmplock);
-        ret = 0;
-
-out:
-        return ret;
-}
-
-/* Path must be validated already. */
-glusterfs_handle_t
-libgf_vmp_get_ghandle (char * path)
-{
-        struct vmp_entry        *entry = NULL;
-
-        entry = libgf_vmp_search_entry (path);
-
-        if (entry == NULL)
-                return NULL;
-
-        return entry->handle;
-}
-
-
-/* Returns the handle for the path given in @path,
- * @path can be a relative path. The point is, here we
- * perform any path resolution that is needed and then
- * search for the corresponding vmp handle.
- * @vpath is a result-value argument in that the virtual
- * path inside the handle is copied into it.
- */
-glusterfs_handle_t
-libgf_resolved_path_handle (const char *path, char *vpath)
-{
-        char                    *respath = NULL;
-        struct vmp_entry        *entry = NULL;
-        glusterfs_handle_t      handle = NULL;
-        char                    *tmp = NULL;
-
-        if ((!path) || (!vpath))
-                return NULL;
-
-        /* We only want compaction before VMP entry search because the
-         * VMP cannot be search unless we have an absolute path.
-         * For absolute paths, we search for VMP first, then perform the
-         * path compaction on the  given virtual path.
-         */
-        if (!libgf_path_absolute (path)) {
-                respath = libgf_resolve_path_light ((char *)path);
-                if (respath == NULL)
-                        return NULL;
-        }
-
-        /* This condition is needed because in case of absolute paths, the path
-         * would already include the VMP and we want to ensure that any path
-         * compaction that happens does not exclude the VMP. In the absence of
-         * this condition an absolute path might get compacted to "/", i.e.
-         * exclude the VMP, and the search will fail.
-         *
-         * For relative paths, respath will aleady include a potential VMP
-         * as a consequence of us prepending the CWD in resolve_light above.
-         */
-        if (libgf_path_absolute (path)) {
-                entry = libgf_vmp_search_entry ((char *)path);
-                if (!entry)
-                        goto free_respath;
-                tmp = libgf_vmp_virtual_path (entry, path, vpath);
-                if (!tmp)
-                        goto free_respath;
-
-                respath = libgf_resolve_path_light (vpath);
-                strcpy (vpath, respath);
-        } else {
-                entry = libgf_vmp_search_entry (respath);
-                if (!entry)
-                        goto free_respath;
-                tmp = libgf_vmp_virtual_path (entry, respath, vpath);
-                if (!tmp)
-                        goto free_respath;
-        }
-
-        handle = entry->handle;
-free_respath:
-        if (respath)
-                free (respath); /* Alloced in libgf_resolve_path_light */
-
-        return handle;
-}
-
-
-int
-glusterfs_mount (char *vmp, glusterfs_init_params_t *ipars)
-{
-        glusterfs_handle_t      vmphandle = NULL;
-        int                     ret = -1;
-        char                    *vmp_resolved = NULL;
-        struct vmp_entry        *vmp_entry = NULL;
-        uint32_t                vmphash = 0;
-        
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, vmp, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ipars, out);
-
-        vmp_resolved = libgf_resolve_path_light (vmp);
-        if (!vmp_resolved) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Path compaction failed");
-                goto out;
-        }
-
-        vmphash = (dev_t)ReallySimpleHash (vmp, strlen (vmp));
-        pthread_mutex_lock (&mountlock);
-        {
-                vmp_entry = libgf_vmp_search_exact_entry (vmp);
-                if (vmp_entry) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Entry exists");
-                        ret = 0;
-                        goto unlock;
-                }
-
-                vmphandle = glusterfs_init (ipars, vmphash);
-                if (!vmphandle) {
-                        errno = EINVAL;
-                        gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "GlusterFS context"
-                                " init failed");
-                        goto unlock;
-                }
-
-                ret = libgf_vmp_map_ghandle (vmp_resolved, vmphandle);
-                if (ret == -1) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Failed to map new"
-                                " handle: %s", vmp);
-                        glusterfs_fini (vmphandle);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&mountlock);
-
-out:
-        if (vmp_resolved)
-                FREE (vmp_resolved);
-
-        return ret;
-}
-
-inline int
-_libgf_umount (char *vmp)
-{
-        struct vmp_entry *entry= NULL;
-        int               ret = -1;
-
-        entry = _libgf_vmp_search_entry (vmp, LIBGF_VMP_EXACT);
-        if (entry == NULL) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                        "path (%s) not mounted", vmp);
-                goto out;
-        }
-
-        if (entry->handle == NULL) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                        "path (%s) has no corresponding glusterfs handle",
-                        vmp);
-                goto out;
-        }
-
-/*        ret = glusterfs_fini (entry->handle); */
-        list_del_init (&entry->list);
-        libgf_free_vmp_entry (entry);
-
-        vmplist.entries--; 
-
-out:
-        return ret;
-}
-
-inline int
-libgf_umount (char *vmp)
-{
-        int ret = -1;
-
-        pthread_mutex_lock (&vmplock);
-        { 
-                ret = _libgf_umount (vmp);
-        }
-        pthread_mutex_unlock (&vmplock);
-        
-        return ret;
-}
-
-int
-glusterfs_umount (char *vmp)
-{ 
-        int    ret = -1; 
-        char *vmp_resolved = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, vmp, out);
-
-        vmp_resolved = libgf_resolve_path_light (vmp);
-        if (!vmp_resolved) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Path compaction failed");
-                goto out;
-        }
-
-        ret = libgf_umount (vmp_resolved);
-
-out:
-        if (vmp_resolved)
-                FREE (vmp_resolved);
-
-        return ret;
-}
-
-int
-glusterfs_umount_all (void)
-{
-        struct vmp_entry *entry = NULL, *tmp = NULL;
-
-        pthread_mutex_lock (&vmplock);
-        {
-                if (vmplist.entries > 0) {
-                        list_for_each_entry_safe (entry, tmp, &vmplist.list,
-                                                  list) {
-                                /* even if there are errors, continue with other
-                                   mounts
-                                */
-                                _libgf_umount (entry->vmp);
-                        }
-                }
-        }
-        pthread_mutex_unlock (&vmplock);
-        
-        return 0;
-}
-
-void
-glusterfs_reset (void)
-{
-        INIT_LIST_HEAD (&vmplist.list);
-        vmplist.entries = 0;
-
-        memset (&vmplock, 0, sizeof (vmplock));
-        pthread_mutex_init (&vmplock, NULL);
-
-	first_init = 1;
-}
-
-void 
-glusterfs_log_lock (void)
-{
-	gf_log_lock ();
-}
-
-
-void glusterfs_log_unlock (void)
-{
-	gf_log_unlock ();
-}
-
-
-void
-libgf_wait_for_frames_unwind (libglusterfs_client_ctx_t *ctx)
-{
-        call_pool_t     *pool = NULL;
-        int             canreturn = 0;
-
-        if (!ctx)
-                return;
-
-        pool = (call_pool_t *)ctx->gf_ctx.pool;
-        while (1) {
-                LOCK (&pool->lock);
-                {
-                        if (pool->cnt == 0) {
-                                canreturn = 1;
-                                goto unlock_out;
-                        }
-                }
-unlock_out:
-                UNLOCK (&pool->lock);
-
-                if (canreturn)
-                        break;
-
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Waiting for call frames");
-                sleep (1);
-        }
-
-        return;
-}
-
-
-int 
-glusterfs_fini (glusterfs_handle_t handle)
-{
-	libglusterfs_client_ctx_t *ctx = handle;
-
-        libgf_wait_for_frames_unwind (ctx);
-
-	FREE (ctx->gf_ctx.cmd_args.log_file);
-	FREE (ctx->gf_ctx.cmd_args.volume_file);
-	FREE (ctx->gf_ctx.cmd_args.volume_name);
-	FREE (ctx->gf_ctx.pool);
-        FREE (ctx->gf_ctx.event_pool);
-        mem_pool_destroy (ctx->itable->inode_pool);
-	 mem_pool_destroy (ctx->itable->dentry_pool);
-	 mem_pool_destroy (ctx->itable->fd_mem_pool);
-        /* iobuf_pool_destroy (ctx->gf_ctx.iobuf_pool); */
-        ((gf_timer_registry_t *)ctx->gf_ctx.timer)->fin = 1;
-
-	xlator_graph_fini (ctx->gf_ctx.graph);
-	xlator_tree_free (ctx->gf_ctx.graph);
-	ctx->gf_ctx.graph = NULL;
-        pthread_cancel (ctx->reply_thread);
-
-        FREE (ctx);
-
-        return 0;
-}
-
-
-int32_t 
-libgf_client_lookup_cbk (call_frame_t *frame,
-                         void *cookie,
-                         xlator_t *this,
-                         int32_t op_ret,
-                         int32_t op_errno,
-                         inode_t *inode,
-                         struct iatt *buf,
-                         dict_t *dict,
-                         struct iatt *postparent)
-{
-        libgf_client_local_t *local = frame->local;
-        libglusterfs_client_ctx_t *ctx = frame->root->state;
-	dict_t *xattr_req = NULL;
-
-        if (op_ret == 0) {
-		inode_t *parent = NULL;
-
-		if (local->fop.lookup.loc->ino == 1) {
-			buf->ia_ino = 1;
-		}
-
-		parent = local->fop.lookup.loc->parent;
-                if (inode->ino != 1) {
-                        inode = inode_link (inode, parent,
-                                            local->fop.lookup.loc->name, buf);
-                }
-
-                libgf_transform_iattr (ctx, inode, buf);
-		inode_lookup (inode);
-        } else {
-                if ((local->fop.lookup.is_revalidate == 0) 
-                    && (op_errno == ENOENT)) {
-                        gf_log ("libglusterfsclient", GF_LOG_DEBUG,
-                                "%"PRId64": (op_num=%d) %s => -1 (%s)",
-				frame->root->unique, frame->root->op,
-				local->fop.lookup.loc->path,
-				strerror (op_errno));
-                } else {
-                        gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "%"PRId64": (op_num=%d) %s => -1 (%s)",
-				frame->root->unique, frame->root->op,
-				local->fop.lookup.loc->path,
-				strerror (op_errno));
-                }
-
-                if (local->fop.lookup.is_revalidate == 1) {
-			int32_t ret = 0;
-                        inode_unref (local->fop.lookup.loc->inode);
-                        local->fop.lookup.loc->inode = inode_new (ctx->itable);
-                        local->fop.lookup.is_revalidate = 2;
-
-                        if (local->fop.lookup.size > 0) {
-                                xattr_req = dict_new ();
-                                ret = dict_set (xattr_req, "glusterfs.content",
-                                                data_from_uint64 (local->fop.lookup.size));
-                                if (ret == -1) {
-                                        op_ret = -1;
-                                        /* TODO: set proper error code */
-                                        op_errno = errno;
-                                        inode = NULL;
-                                        buf = NULL;
-                                        dict = NULL;
-                                        dict_unref (xattr_req);
-                                        goto out;
-                                }
-                        }
-
-                        STACK_WIND (frame, libgf_client_lookup_cbk,
-                                    FIRST_CHILD (this),
-                                    FIRST_CHILD (this)->fops->lookup,
-                                    local->fop.lookup.loc, xattr_req);
-
-			if (xattr_req) {
-				dict_unref (xattr_req);
-				xattr_req = NULL;
-			}
-
-                        return 0;
-                }
-        }
-
-out:
-        local->reply_stub = fop_lookup_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                 inode, buf, dict, postparent);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int32_t
-libgf_client_lookup (libglusterfs_client_ctx_t *ctx,
-                     loc_t *loc,
-                     struct iatt *stbuf,
-                     dict_t **dict,
-		     dict_t *xattr_req)
-{
-        call_stub_t  *stub = NULL;
-        int32_t op_ret;
-        libgf_client_local_t *local = NULL;
-        inode_t *inode = NULL;
-        
-        local = CALLOC (1, sizeof (*local));
-        if (!local) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Memory allocation"
-                        " failed");
-                errno = ENOMEM;
-                return -1;
-        }
-
-        if (loc->inode) {
-                local->fop.lookup.is_revalidate = 1;
-                loc->ino = loc->inode->ino;
-        }
-        else
-                loc->inode = inode_new (ctx->itable);
-
-        local->fop.lookup.loc = loc;
-
-        LIBGF_CLIENT_FOP(ctx, stub, lookup, local, loc, xattr_req);
-
-        op_ret = stub->args.lookup_cbk.op_ret;
-        errno = stub->args.lookup_cbk.op_errno;
-
-        if (op_ret == -1)
-                goto out;
-
-        inode = stub->args.lookup_cbk.inode;
-        if (!(libgf_get_inode_ctx (inode)))
-                libgf_alloc_inode_ctx (ctx, inode);
-        libgf_transform_iattr (ctx, inode, &stub->args.lookup_cbk.buf);
-        libgf_update_iattr_cache (inode, LIBGF_UPDATE_ALL,
-                                        &stub->args.lookup_cbk.buf);
-        if (stbuf)
-                *stbuf = stub->args.lookup_cbk.buf;
-
-        if (dict)
-                *dict = dict_ref (stub->args.lookup_cbk.dict);
-
-        if (inode != loc->inode) {
-                inode_unref (loc->inode);
-                loc->inode = inode_ref (inode);
-        }
-
-out:
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-int 
-glusterfs_glh_get (glusterfs_handle_t handle, const char *path, void *buf,
-                   size_t size, struct stat *stbuf)
-{
-        int32_t op_ret = -1;
-        loc_t loc = {0, };
-        libglusterfs_client_ctx_t *ctx = handle;
-        dict_t *dict = NULL;
-	dict_t *xattr_req = NULL;
-	char *name = NULL, *pathname = NULL;
-        struct iatt iatt = {0,};
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, size %lu", path,
-                (long unsigned)size);
-        if (size < 0) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Invalid size");
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (size == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Path compaction failed");
-                goto out;
-        }
-
-	op_ret = libgf_client_path_lookup (&loc, ctx, 0);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient", GF_LOG_ERROR,
-			"path lookup failed for (%s)", loc.path);
-		goto out;
-	}
-
-	pathname = strdup (loc.path);
-	name = basename (pathname);
-
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino, name);
-        if (op_ret < 0) {
-                gf_log ("libglusterfsclient",
-                        GF_LOG_ERROR,
-                        "libgf_client_loc_fill returned -1, returning EINVAL");
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (size) { 
-                xattr_req = dict_new ();
-                op_ret = dict_set (xattr_req, "glusterfs.content",
-                                   data_from_uint64 (size));
-                if (op_ret < 0) {
-                        gf_log ("libglusterfsclient",
-                                GF_LOG_ERROR,
-                                "setting requested content size dictionary failed");
-                        goto out;
-                }
-        }
-
-        op_ret = libgf_client_lookup (ctx, &loc, &iatt, &dict, xattr_req);
-        iatt_to_stat (&iatt, stbuf);
-        if (!op_ret && stbuf && (iatt.ia_size <= size) && dict && buf) {
-                data_t *mem_data = NULL;
-                void *mem = NULL;
-                
-                mem_data = dict_get (dict, "glusterfs.content");
-                if (mem_data) {
-                        mem = data_to_ptr (mem_data);
-                }
-                        
-                if (mem != NULL) { 
-                        memcpy (buf, mem, iatt.ia_size);
-                }
-        }
-
-out:
-	if (xattr_req) {
-		dict_unref (xattr_req);
-	}
-        
-	if (dict) {
-		dict_unref (dict);
-	}
-
-	if (pathname) {
-		FREE (pathname);
-	}
-	libgf_client_loc_wipe (&loc);
-
-        return op_ret;
-}
-
-int
-glusterfs_get (const char *path, void *buf, size_t size, struct stat *stbuf)
-{
-        int                     op_ret = -1;
-        glusterfs_handle_t      h      = NULL;
-        char                    vpath[PATH_MAX];
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, buf, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, stbuf, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, size %lu", path,
-                (long unsigned)size);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_get (h, vpath, buf, size, stbuf);
-
-out:
-        return op_ret;
-}
-
-int
-libgf_client_lookup_async_cbk (call_frame_t *frame,
-                               void *cookie,
-                               xlator_t *this,
-                               int32_t op_ret,
-                               int32_t op_errno,
-                               inode_t *inode,
-                               struct iatt *stbuf,
-                               dict_t *dict,
-                               struct iatt *postparent)
-{
-        libglusterfs_client_async_local_t *local = frame->local;
-        glusterfs_get_cbk_t lookup_cbk = local->fop.lookup_cbk.cbk;
-        libglusterfs_client_ctx_t *ctx = frame->root->state;
-	glusterfs_iobuf_t *iobuf = NULL;
-	dict_t *xattr_req = NULL;
-        inode_t *parent = NULL;
-        struct stat stat = {0,};
-
-        if (op_ret == 0) {
-                parent = local->fop.lookup_cbk.loc->parent;
-                inode_link (inode, parent, local->fop.lookup_cbk.loc->name,
-                            stbuf);
-                libgf_transform_iattr (ctx, inode, stbuf);
-                if (!(libgf_get_inode_ctx (inode)))
-                        libgf_alloc_inode_ctx (ctx, inode);
-                libgf_update_iattr_cache (inode, LIBGF_UPDATE_ALL, stbuf);
-                inode_lookup (inode);
-        } else {
-                if ((local->fop.lookup_cbk.is_revalidate == 0) 
-                    && (op_errno == ENOENT)) {
-                        gf_log ("libglusterfsclient", GF_LOG_DEBUG,
-                                "%"PRId64": (op_num=%d) %s => -1 (%s)",
-				frame->root->unique, frame->root->op,
-				local->fop.lookup_cbk.loc->path,
-				strerror (op_errno));
-                } else {
-                        gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "%"PRId64": (op_num=%d) %s => -1 (%s)",
-				frame->root->unique, frame->root->op,
-                                local->fop.lookup_cbk.loc->path,
-				strerror (op_errno));
-                }
-
-                if (local->fop.lookup_cbk.is_revalidate == 1) {
-			int32_t ret = 0;
-                        inode_unref (local->fop.lookup_cbk.loc->inode);
-                        local->fop.lookup_cbk.loc->inode = inode_new (ctx->itable);
-                        local->fop.lookup_cbk.is_revalidate = 2;
-
-                        if (local->fop.lookup_cbk.size > 0) {
-                                xattr_req = dict_new ();
-                                ret = dict_set (xattr_req, "glusterfs.content",
-                                                data_from_uint64 (local->fop.lookup_cbk.size));
-                                if (ret == -1) {
-                                        op_ret = -1;
-                                        /* TODO: set proper error code */
-                                        op_errno = errno;
-                                        inode = NULL;
-                                        stbuf = NULL;
-                                        dict = NULL;
-                                        dict_unref (xattr_req);
-                                        goto out;
-                                }
-                        }
-
-
-                        STACK_WIND (frame, libgf_client_lookup_async_cbk,
-                                    FIRST_CHILD (this),
-                                    FIRST_CHILD (this)->fops->lookup,
-                                    local->fop.lookup_cbk.loc, xattr_req);
-			
-			if (xattr_req) {
-				dict_unref (xattr_req);
-				xattr_req = NULL;
-			}
-
-                        return 0;
-                }
-        }
-
-out:
-        if (!op_ret && local->fop.lookup_cbk.size && dict) {
-                data_t *mem_data = NULL;
-                void *mem = NULL;
-		struct iovec *vector = NULL;
-
-                mem_data = dict_get (dict, "glusterfs.content");
-                if (mem_data) {
-                        mem = data_to_ptr (mem_data);
-                }
-
-                if (mem && stbuf->ia_size <= local->fop.lookup_cbk.size) {
-			iobuf = CALLOC (1, sizeof (*iobuf));
-			ERR_ABORT (iobuf);
-
-			vector = CALLOC (1, sizeof (*vector));
-			ERR_ABORT (vector);
-			vector->iov_base = mem;
-			vector->iov_len = stbuf->ia_size;  
-
-			iobuf->vector = vector;
-			iobuf->count = 1;
-			iobuf->dictref = dict_ref (dict);
-		}
-	}
-
-        iatt_to_stat (stbuf, &stat);
-        lookup_cbk (op_ret, op_errno, iobuf, &stat, local->cbk_data);
-
-	libgf_client_loc_wipe (local->fop.lookup_cbk.loc);
-        free (local->fop.lookup_cbk.loc);
-
-        free (local);
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
-
-        return 0;
-}
-
-/* TODO: implement async dentry lookup */
-
-int
-glusterfs_get_async (glusterfs_handle_t handle, 
-		     const char *path,
-		     size_t size, 
-		     glusterfs_get_cbk_t cbk,
-		     void *cbk_data)
-{
-        loc_t *loc = NULL;
-        libglusterfs_client_ctx_t *ctx = handle;
-        libglusterfs_client_async_local_t *local = NULL;
-	int32_t op_ret = 0;
-	dict_t *xattr_req = NULL;
-	char *name = NULL, *pathname = NULL;
-
-	if (!ctx || !path || path[0] != '/') {
-		errno = EINVAL;
-		op_ret = -1;
-		goto out;
-	}
-
-        if (size < 0) {
-                errno = EINVAL;
-                op_ret = -1;
-                goto out;
-        }
-
-        if (size == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        local = CALLOC (1, sizeof (*local));
-        local->fop.lookup_cbk.is_revalidate = 1;
-
-        loc = CALLOC (1, sizeof (*loc));
-	loc->path = strdup (path);
-	op_ret = libgf_client_path_lookup (loc, ctx, 1);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient",
-			GF_LOG_ERROR,
-			"path lookup failed for (%s)", path);
-		goto out;
-	}
-
-	pathname = strdup (path);
-	name = basename (pathname);
-        op_ret = libgf_client_loc_fill (loc, ctx, 0, loc->parent->ino, name);
-	if (op_ret < 0) {
-		gf_log ("libglusterfsclient",
-			GF_LOG_ERROR,
-			"libgf_client_loc_fill returned -1, returning EINVAL");
-		errno = EINVAL;
-		goto out;
-	}
-
-        if (!loc->inode) {
-                loc->inode = inode_new (ctx->itable);
-                local->fop.lookup_cbk.is_revalidate = 0;
-        } 
-
-        local->fop.lookup_cbk.cbk = cbk;
-        local->fop.lookup_cbk.size = size;
-        local->fop.lookup_cbk.loc = loc;
-        local->cbk_data = cbk_data;
-
-        if (size > 0) {
-                xattr_req = dict_new ();
-                op_ret = dict_set (xattr_req, "glusterfs.content",
-                                   data_from_uint64 (size));
-                if (op_ret < 0) {
-                        dict_unref (xattr_req);
-                        xattr_req = NULL;
-                        goto out;
-                }
-        }
-
-        LIBGF_CLIENT_FOP_ASYNC (ctx,
-                                local,
-                                libgf_client_lookup_async_cbk,
-                                lookup,
-                                loc,
-                                xattr_req);
-	if (xattr_req) {
-		dict_unref (xattr_req);
-		xattr_req = NULL;
-	}
-
-out:
-	if (pathname) {
-		FREE (pathname);
-	}
- 
-        return op_ret;
-}
-
-int32_t
-libgf_client_getxattr_cbk (call_frame_t *frame,
-                           void *cookie,
-                           xlator_t *this,
-                           int32_t op_ret,
-                           int32_t op_errno,
-                           dict_t *dict)
-{
-
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_getxattr_cbk_stub (frame, NULL, op_ret,
-                                                   op_errno, dict);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-size_t 
-libgf_client_getxattr (libglusterfs_client_ctx_t *ctx, 
-                       loc_t *loc,
-                       const char *name,
-                       void *value,
-                       size_t size)
-{
-        call_stub_t  *stub = NULL;
-        int32_t op_ret = 0;
-        libgf_client_local_t *local = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, getxattr, local, loc, name);
-
-        op_ret = stub->args.getxattr_cbk.op_ret;
-        errno = stub->args.getxattr_cbk.op_errno;
-
-        if (op_ret >= 0) {
-                /*
-                  gf_log ("LIBGF_CLIENT", GF_LOG_DEBUG,
-                  "%"PRId64": %s => %d", frame->root->unique,
-                  state->fuse_loc.loc.path, op_ret);
-                */
-
-                data_t *value_data = dict_get (stub->args.getxattr_cbk.dict,
-                                               (char *)name);
-    
-                if (value_data) {
-                        int32_t copy_len = 0;
-
-                        /* Don't return the value for '\0' */
-                        op_ret = value_data->len; 
-                        if ((size > 0) && (value != NULL)) {
-                                copy_len = size < value_data->len ? 
-                                        size : value_data->len;
-                                memcpy (value, value_data->data, copy_len);
-                                op_ret = copy_len;
-                        }
-                } else {
-                        errno = ENODATA;
-                        op_ret = -1;
-                }
-        }
-	
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-#define LIBGF_DO_GETXATTR       1
-#define LIBGF_DO_LGETXATTR      2
-
-ssize_t
-__glusterfs_glh_getxattr (glusterfs_handle_t handle, const char *path,
-                          const char *name, void *value, size_t size,
-                          int whichop)
-{
-        int32_t op_ret = -1;
-        loc_t loc = {0, };
-	libglusterfs_client_ctx_t *ctx = handle;
-	char *file = NULL;
-        char *pathres = NULL, *tmp = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, name, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, name %s, size %lu,"
-                " op %d", path, name, (long unsigned)size, whichop);
-        if (name[0] == '\0') {
-		errno = EINVAL;
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Invalid argument: Name"
-                        " not NULL terminated");
-		goto out;
-	}
-
-        if (size < 0) {
-                errno = EINVAL;
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Invalid argument: size is"
-                        " less than zero");
-                goto out;
-        }
-
-        pathres = strdup (path);
-        if (!pathres) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        loc.path = strdup (pathres);
-	op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-	if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-			"path lookup failed for (%s)", loc.path);
-		goto out;
-	}
-
-	tmp = strdup (pathres);
-	file = basename (tmp);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino, file);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient",
-			GF_LOG_ERROR,
-			"libgf_client_loc_fill returned -1, returning EINVAL");
-		errno = EINVAL;
-		goto out;
-	}
-
-        if (whichop == LIBGF_DO_LGETXATTR)
-                goto do_getx;
-
-        if (!IA_ISLNK (loc.inode->ia_type))
-                goto do_getx;
-
-        libgf_client_loc_wipe (&loc); 
-        op_ret = libgf_realpath_loc_fill (ctx, (char *)pathres, &loc);
-        if (op_ret == -1) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "realpath failed");
-                goto out;
-        }
-
-do_getx:
-	op_ret = libgf_client_getxattr (ctx, &loc, name, value, size);
-
-out:
-	if (tmp) {
-		FREE (tmp);
-	}
-
-        if (pathres)
-                FREE (pathres);
-
-        libgf_client_loc_wipe (&loc);
-
-        return op_ret;
-}
-
-ssize_t
-glusterfs_glh_getxattr (glusterfs_handle_t handle, const char *path,
-                        const char *name, void *value, size_t size)
-{
-        return __glusterfs_glh_getxattr (handle, path, name, value, size,
-                                         LIBGF_DO_GETXATTR);
-}
-
-ssize_t
-glusterfs_glh_lgetxattr (glusterfs_handle_t handle, const char *path,
-                         const char *name, void *value, size_t size)
-{
-        return __glusterfs_glh_getxattr (handle, path, name, value, size,
-                                         LIBGF_DO_LGETXATTR);
-}
-
-ssize_t
-glusterfs_getxattr (const char *path, const char *name, void *value,
-                        size_t size)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, name, out);
-
-        if ((size > 0) && (value == NULL)) {
-                errno = EINVAL;
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Invalid argument value");
-                goto out;
-        }
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, name %s, size %lu",
-                path, name, (long unsigned)size);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = __glusterfs_glh_getxattr (h, vpath, name, value, size,
-                                           LIBGF_DO_GETXATTR);
-
-out:
-        return op_ret;
-}
-
-ssize_t
-glusterfs_lgetxattr (const char *path, const char *name, void *value,
-                     size_t size)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, name, out);
-
-        if ((size > 0) && (value == NULL)) {
-                errno = EINVAL;
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Invalid argument value");
-                goto out;
-        }
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, name %s, size %lu",
-                path, name, (long unsigned)size);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = __glusterfs_glh_getxattr (h, vpath, name, value, size,
-                                           LIBGF_DO_LGETXATTR);
-
-out:
-        return op_ret;
-}
-
-static int32_t
-libgf_client_open_cbk (call_frame_t *frame,
-                       void *cookie,
-                       xlator_t *this,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       fd_t *fd)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_open_cbk_stub (frame, NULL, op_ret, op_errno,
-                                               fd);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-
-int 
-libgf_client_open (libglusterfs_client_ctx_t *ctx, 
-                   loc_t *loc, 
-                   fd_t *fd, 
-                   int flags)
-{
-        call_stub_t *stub = NULL;
-        int32_t op_ret = 0;
-        libgf_client_local_t *local = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, open, local, loc, flags, fd, 0);
-
-        op_ret = stub->args.open_cbk.op_ret;
-        errno = stub->args.open_cbk.op_errno;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "open: path %s, status: %d, errno"
-                " %d", loc->path, op_ret, errno);
-        if (op_ret != -1)
-                fd_bind (fd);
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-static int32_t
-libgf_client_create_cbk (call_frame_t *frame,
-                         void *cookie,
-                         xlator_t *this,
-                         int32_t op_ret,
-                         int32_t op_errno,
-                         fd_t *fd,
-                         inode_t *inode,
-                         struct iatt *buf,
-                         struct iatt *preparent,
-                         struct iatt *postparent)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_create_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                 fd, inode, buf, preparent,
-                                                 postparent);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int 
-libgf_client_creat (libglusterfs_client_ctx_t *ctx,
-                    loc_t *loc,
-                    fd_t *fd,
-                    int flags,
-                    mode_t mode)
-{
-        call_stub_t *stub = NULL;
-        int32_t op_ret = 0;
-        libgf_client_local_t *local = NULL;
-        inode_t *libgf_inode = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, create, local, loc, flags, mode, fd);
-  
-        op_ret = stub->args.create_cbk.op_ret;
-        errno = stub->args.create_cbk.op_errno;
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Create: path %s, status: %d,"
-                " errno: %d", loc->path, op_ret, errno);
-        if (op_ret == -1)
-                goto out;
-
-	libgf_inode = stub->args.create_cbk.inode;
-        inode_link (libgf_inode, loc->parent, loc->name,
-                        &stub->args.create_cbk.buf);
-        libgf_transform_iattr (ctx, libgf_inode, &stub->args.create_cbk.buf);
-
-        inode_lookup (libgf_inode);
-
-        libgf_alloc_inode_ctx (ctx, libgf_inode);
-        libgf_update_iattr_cache (libgf_inode, LIBGF_UPDATE_ALL,
-                                        &stub->args.create_cbk.buf);
-
-out:
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-int32_t
-libgf_client_opendir_cbk (call_frame_t *frame,
-                          void *cookie,
-                          xlator_t *this,
-                          int32_t op_ret,
-                          int32_t op_errno,
-                          fd_t *fd)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_opendir_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                  fd);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int 
-libgf_client_opendir (libglusterfs_client_ctx_t *ctx,
-                      loc_t *loc,
-                      fd_t *fd)
-{
-        call_stub_t *stub = NULL;
-        int32_t op_ret = -1;
-        libgf_client_local_t *local = NULL;
-
-        if (((fd->flags & O_ACCMODE) == O_WRONLY)
-                || ((fd->flags & O_ACCMODE) == O_RDWR)) {
-                errno = EISDIR;
-                goto out;
-        }
-        LIBGF_CLIENT_FOP (ctx, stub, opendir, local, loc, fd);
-
-        op_ret = stub->args.opendir_cbk.op_ret;
-        errno = stub->args.opendir_cbk.op_errno;
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "opendir: path %s, status %d,"
-                " errno %d", loc->path, op_ret, errno);
-        if (op_ret != -1)
-                fd_bind (fd);
-
-	call_stub_destroy (stub);
-out:
-        return op_ret;
-}
-
-glusterfs_file_t 
-glusterfs_glh_open (glusterfs_handle_t handle, const char *path, int flags,...)
-{
-        loc_t loc = {0, };
-        long op_ret = -1;
-        fd_t *fd = NULL;
-	int32_t ret = -1;
-	libglusterfs_client_ctx_t *ctx = handle;
-	char *name = NULL, *pathname = NULL;
-        libglusterfs_client_inode_ctx_t *inode_ctx = NULL;
-        mode_t mode = 0;
-        va_list ap;
-        char *pathres = NULL;
-        char *vpath = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        pathres = strdup (path);
-        if (!pathres) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-	loc.path = strdup (pathres);
-	op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-
-        if ((op_ret == -1) && ((flags & O_CREAT) != O_CREAT)) {
-		gf_log ("libglusterfsclient", GF_LOG_ERROR,
-			"path lookup failed for (%s)", loc.path);
-		goto out;
-	}
-
-        if (!op_ret && ((flags & O_CREAT) == O_CREAT) 
-            && ((flags & O_EXCL) == O_EXCL)) {
-                errno = EEXIST;
-                op_ret = -1;
-                goto out;
-        }
-
-        if (op_ret == 0) {
-                flags &= ~O_CREAT;
-        }
-
-        if ((op_ret == -1) && ((flags & O_CREAT) == O_CREAT)) {
-                libgf_client_loc_wipe (&loc);
-                loc.path = strdup (pathres);
-
-                op_ret = libgf_client_path_lookup (&loc, ctx, 0);
-                if (op_ret == -1) {
-                        gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "path lookup failed for parent while trying to"
-                                " create (%s)", pathres);
-                        goto out;
-                }
-
-                loc.inode = inode_new (ctx->itable);
-        }
-
-	pathname = strdup (pathres);
-	name = basename (pathname);
-
-        ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino, name);
-	if (ret == -1) {
-		gf_log ("libglusterfsclient",
-			GF_LOG_ERROR,
-			"libgf_client_loc_fill returned -1, returning EINVAL");
-		errno = EINVAL;
-		goto out;
-	}
-
-        fd = fd_create (loc.inode, ctx->pid);
-        fd->flags = flags;
-
-        if (((flags & O_CREAT) == O_CREAT)) {
-                /* If we have the st_mode for the basename, check if
-                 * it is a directory here itself, rather than sending
-                 * a network message through libgf_client_creat, and
-                 * then receiving a EISDIR.
-                 */
-                if (IA_ISDIR (loc.inode->ia_type)) {
-                        errno = EISDIR;
-                        op_ret = -1;
-                        goto op_over;
-                }
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-                op_ret = libgf_client_creat (ctx, &loc, fd, flags, mode);
-        } else {
-                if (IA_ISDIR (loc.inode->ia_type))
-                        op_ret = libgf_client_opendir (ctx, &loc, fd);
-                else
-                        op_ret = libgf_client_open (ctx, &loc, fd, flags);
-        }
-
-op_over:
-        if (op_ret == -1) {
-                fd_unref (fd);
-                fd = NULL;
-                goto out;
-        }
-
-        vpath = NULL;
-        if (IA_ISDIR (loc.inode->ia_type)) {
-                vpath = (char *)path;
-        }
-
-        if (!libgf_get_fd_ctx (fd)) {
-                if (!libgf_alloc_fd_ctx (ctx, fd, vpath)) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Failed to"
-                                " allocate fd context");
-                        errno = EINVAL;
-                        op_ret = -1;
-                        goto out;
-                }
-        }
-
-        if ((flags & O_TRUNC) && (((flags & O_ACCMODE) == O_RDWR)
-                                  || ((flags & O_ACCMODE) == O_WRONLY))) {
-                inode_ctx = libgf_get_inode_ctx (fd->inode);
-                if (IA_ISREG (inode_ctx->stbuf.ia_type)) {
-                                inode_ctx->stbuf.ia_size = 0;
-                                inode_ctx->stbuf.ia_blocks = 0;
-                }
-        }
-
-out:
-        libgf_client_loc_wipe (&loc);
-
-	if (pathname) {
-		FREE (pathname);
-	}
-
-        if (pathres)
-                FREE (pathres);
-
-        return fd;
-}
-
-glusterfs_file_t
-glusterfs_open (const char *path, int flags, ...)
-{
-        va_list                 ap;
-        glusterfs_file_t        fh   = NULL;
-        glusterfs_handle_t      h    = NULL; 
-        mode_t                  mode = 0;
-        char                    vpath[PATH_MAX];
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        if (flags & O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-                fh = glusterfs_glh_open (h, vpath, flags, mode);
-        } else
-                fh = glusterfs_glh_open (h, vpath, flags);
-out:
-        return fh;
-}
-
-glusterfs_file_t 
-glusterfs_glh_creat (glusterfs_handle_t handle, const char *path, mode_t mode)
-{
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-	return glusterfs_glh_open (handle, path,
-			       (O_CREAT | O_WRONLY | O_TRUNC), mode);
-}
-
-glusterfs_file_t
-glusterfs_creat (const char *path, mode_t mode)
-{
-        glusterfs_file_t        fh = NULL;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        fh = glusterfs_glh_creat (h, vpath, mode);
-
-out:
-        return fh;
-}
-
-int32_t
-libgf_client_flush_cbk (call_frame_t *frame,
-                        void *cookie,
-                        xlator_t *this,
-                        int32_t op_ret,
-                        int32_t op_errno)
-{
-        libgf_client_local_t *local = frame->local;
-        
-        local->reply_stub = fop_flush_cbk_stub (frame, NULL, op_ret, op_errno);
-        
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-
-int 
-libgf_client_flush (libglusterfs_client_ctx_t *ctx, fd_t *fd)
-{
-        call_stub_t *stub;
-        int32_t op_ret;
-        libgf_client_local_t *local = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, flush, local, fd);
-        
-        op_ret = stub->args.flush_cbk.op_ret;
-        errno = stub->args.flush_cbk.op_errno;
-        
-	call_stub_destroy (stub);        
-        return op_ret;
-}
-
-
-int 
-glusterfs_close (glusterfs_file_t fd)
-{
-        int32_t op_ret = -1;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-
-        if (!fd) {
-                errno = EINVAL;
-		goto out;
-        }
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-                goto out;
-        }
-        ctx = fd_ctx->ctx;
-
-        op_ret = libgf_client_flush (ctx, (fd_t *)fd);
-
-        fd_unref ((fd_t *)fd);
-
-out:
-        return op_ret;
-}
-
-int32_t
-libgf_client_setxattr_cbk (call_frame_t *frame,
-                           void *cookie,
-                           xlator_t *this,
-                           int32_t op_ret,
-                           int32_t op_errno)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_setxattr_cbk_stub (frame, NULL, op_ret,
-                                                   op_errno);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int
-libgf_client_setxattr (libglusterfs_client_ctx_t *ctx, 
-                       loc_t *loc,
-                       const char *name,
-                       const void *value,
-                       size_t size,
-                       int flags)
-{
-        call_stub_t  *stub = NULL;
-        int32_t op_ret = 0;
-        dict_t *dict;
-        libgf_client_local_t *local = NULL;
-
-        dict = get_new_dict ();
-
-        dict_set (dict, (char *)name,
-                  bin_to_data ((void *)value, size));
-        dict_ref (dict);
-
-
-        LIBGF_CLIENT_FOP (ctx, stub, setxattr, local, loc, dict, flags);
-
-        op_ret = stub->args.setxattr_cbk.op_ret;
-        errno = stub->args.setxattr_cbk.op_errno;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "path %s, name %s, status %d,"
-                "errno %d", loc->path, name, op_ret, errno);
-        dict_unref (dict);
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-
-#define LIBGF_DO_SETXATTR       1
-#define LIBGF_DO_LSETXATTR      2
-
-int 
-__glusterfs_glh_setxattr (glusterfs_handle_t handle, const char *path,
-                          const char *name, const void *value,
-                          size_t size, int flags, int whichop)
-{
-        int32_t op_ret = -1;
-        loc_t loc = {0, };
-	libglusterfs_client_ctx_t *ctx = handle;
-        char *tmppath = NULL;
-        loc_t *realloc = NULL;
-        char *pathres = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "path %s, name %s, op %d", path
-                ,name, whichop);
-        if (size <= 0) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        pathres = strdup (path);
-        if (!pathres) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        loc.path = strdup (pathres);
-	op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient", GF_LOG_ERROR,
-			"path lookup failed for (%s)", pathres);
-		goto out;
-	}
-
-        tmppath = strdup (pathres);
-
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                        basename (tmppath));
-        FREE (tmppath);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient",
-			GF_LOG_ERROR,
-			"libgf_client_loc_fill returned -1, returning EINVAL");
-		errno = EINVAL;
-		goto out;
-	}
-
-        realloc = &loc;
-        if (whichop == LIBGF_DO_LSETXATTR)
-                goto do_setx;
-
-        if (!IA_ISLNK (loc.inode->ia_type))
-                goto do_setx;
-
-        libgf_client_loc_wipe (&loc);
-        realloc = &loc;
-        libgf_realpath_loc_fill (ctx, (char *)pathres, realloc);
-
-do_setx:
-        if (!op_ret)
-                op_ret = libgf_client_setxattr (ctx, realloc, name, value,
-                                                size, flags);
-
-out:
-        if (pathres)
-                FREE (pathres);
-
-        libgf_client_loc_wipe (realloc);
-        return op_ret;
-}
-
-int
-glusterfs_glh_setxattr (glusterfs_handle_t handle, const char *path,
-                        const char *name, const void *value, size_t size,
-                        int flags)
-{
-        return __glusterfs_glh_setxattr (handle, path, name, value, size, flags
-                                         , LIBGF_DO_SETXATTR);
-}
-
-int
-glusterfs_glh_lsetxattr (glusterfs_handle_t handle, const char *path,
-                         const char *name, const void *value, size_t size,
-                         int flags)
-{
-        return __glusterfs_glh_setxattr (handle, path, name, value, size, flags
-                                         , LIBGF_DO_LSETXATTR);
-}
-
-int
-glusterfs_setxattr (const char *path, const char *name, const void *value,
-                        size_t size, int flags)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, name, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, value, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "path %s, name %s", path, name);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = __glusterfs_glh_setxattr (h, vpath, name, value, size, flags,
-                                           LIBGF_DO_SETXATTR);
-
-out:
-        return op_ret;
-}
-
-int
-glusterfs_lsetxattr (const char *path, const char *name, const void *value,
-                     size_t size, int flags)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, name, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, value, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "path %s, name %s", path, name);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = __glusterfs_glh_setxattr (h, vpath, name, value, size, flags,
-                                           LIBGF_DO_LSETXATTR);
-
-out:
-        return op_ret;
-}
-
-int32_t
-libgf_client_fsetxattr_cbk (call_frame_t *frame,
-                            void *cookie,
-                            xlator_t *this,
-                            int32_t op_ret,
-                            int32_t op_errno)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_fsetxattr_cbk_stub (frame, NULL, op_ret,
-                                                    op_errno);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int
-libgf_client_fsetxattr (libglusterfs_client_ctx_t *ctx, 
-                        fd_t *fd,
-                        const char *name,
-                        const void *value,
-                        size_t size,
-                        int flags)
-{
-        call_stub_t  *stub = NULL;
-        int32_t op_ret = 0;
-        dict_t *dict;
-        libgf_client_local_t *local = NULL;
-
-        dict = get_new_dict ();
-
-        dict_set (dict, (char *)name,
-                  bin_to_data ((void *)value, size));
-        dict_ref (dict);
-
-        LIBGF_CLIENT_FOP (ctx, stub, fsetxattr, local, fd, dict, flags);
-
-        op_ret = stub->args.fsetxattr_cbk.op_ret;
-        errno = stub->args.fsetxattr_cbk.op_errno;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "name %s, status %d, errno %d",
-                name, op_ret, errno);
-        dict_unref (dict);
-	call_stub_destroy (stub);
-
-        return op_ret;
-}
-
-int 
-glusterfs_fsetxattr (glusterfs_file_t fd, 
-                     const char *name,
-                     const void *value, 
-                     size_t size, 
-                     int flags)
-{
-	int32_t op_ret = 0;
-        fd_t *__fd = fd;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        
-        if (!fd) {
-                errno = EINVAL;
-                op_ret = -1;
-                gf_log("libglusterfsclient",
-                       GF_LOG_ERROR,
-                       "invalid fd");
-                goto out;
-        }
-
-        if (size <= 0) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Invalid argument: size is"
-                        " less than or equal to zero");
-                errno = EINVAL;
-                op_ret = -1;
-                goto out;
-        }
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-		op_ret = -1;
-		goto out;
-        }
-
-        ctx = fd_ctx->ctx;
-        op_ret = libgf_client_fsetxattr (ctx, __fd, name, value, size,
-                                         flags);
-        
-out:
-	return op_ret;
-}
-
-int32_t
-libgf_client_fgetxattr_cbk (call_frame_t *frame,
-                            void *cookie,
-                            xlator_t *this,
-                            int32_t op_ret,
-                            int32_t op_errno,
-                            dict_t *dict)
-{
-
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_fgetxattr_cbk_stub (frame, NULL, op_ret,
-                                                    op_errno, dict);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-size_t 
-libgf_client_fgetxattr (libglusterfs_client_ctx_t *ctx, 
-                        fd_t *fd,
-                        const char *name,
-                        void *value,
-                        size_t size)
-{
-        call_stub_t  *stub = NULL;
-        int32_t op_ret = 0;
-        libgf_client_local_t *local = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, fgetxattr, local, fd, name);
-
-        op_ret = stub->args.fgetxattr_cbk.op_ret;
-        errno = stub->args.fgetxattr_cbk.op_errno;
-
-        if (op_ret >= 0) {
-                /*
-                  gf_log ("LIBGF_CLIENT", GF_LOG_DEBUG,
-                  "%"PRId64": %s => %d", frame->root->unique,
-                  state->fuse_loc.loc.path, op_ret);
-                */
-
-                data_t *value_data = dict_get (stub->args.fgetxattr_cbk.dict,
-                                               (char *)name);
-    
-                if (value_data) {
-                        int32_t copy_len = 0;
-
-                        /* Don't return the value for '\0' */
-                        op_ret = value_data->len; 
-                        copy_len = size < value_data->len ? 
-                                size : value_data->len;
-                        memcpy (value, value_data->data, copy_len);
-                } else {
-                        errno = ENODATA;
-                        op_ret = -1;
-                }
-        }
-	
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "name %s, status %d, errno %d",
-                name, op_ret, errno);
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-ssize_t 
-glusterfs_fgetxattr (glusterfs_file_t fd, 
-                     const char *name,
-                     void *value, 
-                     size_t size)
-{
-	int32_t op_ret = 0;
-        libglusterfs_client_ctx_t *ctx;
-        fd_t *__fd = (fd_t *)fd;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "name %s", name);
-        if (size < 0) {
-                errno = EINVAL;
-                op_ret = -1;
-                goto out;
-        }
-
-        if (size == 0)
-                goto out;
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-		op_ret = -1;
-		goto out;
-        }
-
-        ctx = fd_ctx->ctx;
-        op_ret = libgf_client_fgetxattr (ctx, __fd, name, value, size);
-out:
-	return op_ret;
-}
-
-ssize_t 
-glusterfs_listxattr (glusterfs_handle_t handle,
-                     const char *path, 
-                     char *list,
-                     size_t size)
-{
-        return ENOSYS;
-}
-
-ssize_t 
-glusterfs_llistxattr (glusterfs_handle_t handle,
-                      const char *path, 
-                      char *list,
-                      size_t size)
-{
-        return ENOSYS;
-}
-
-ssize_t 
-glusterfs_flistxattr (glusterfs_file_t fd, 
-                      char *list,
-                      size_t size)
-{
-        return ENOSYS;
-}
-
-int 
-glusterfs_removexattr (glusterfs_handle_t handle, 
-                       const char *path, 
-                       const char *name)
-{
-        return ENOSYS;
-}
-
-int 
-glusterfs_lremovexattr (glusterfs_handle_t handle, 
-                        const char *path, 
-                        const char *name)
-{
-        return ENOSYS;
-}
-
-int 
-glusterfs_fremovexattr (glusterfs_file_t fd, 
-                        const char *name)
-{
-        return ENOSYS;
-}
-
-int32_t
-libgf_client_readv_cbk (call_frame_t *frame,
-                        void *cookie,
-                        xlator_t *this,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        struct iovec *vector,
-                        int32_t count,
-                        struct iatt *stbuf,
-                        struct iobref *iobref)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_readv_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                vector, count, stbuf, iobref);
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int
-libgf_client_iobuf_read (libglusterfs_client_ctx_t *ctx, fd_t *fd, void *buf,
-                         size_t size, off_t offset)
-{
-        call_stub_t          *stub = NULL;
-        struct iovec         *vector = NULL;
-        int32_t               op_ret = -1;
-        int                   count = 0;
-        libgf_client_local_t *local = NULL;
-        struct iatt          *stbuf = NULL;
-
-        local = CALLOC (1, sizeof (*local));
-        ERR_ABORT (local);
-        local->fd = fd;
-        LIBGF_CLIENT_FOP (ctx, stub, readv, local, fd, size, offset);
-
-        op_ret = stub->args.readv_cbk.op_ret;
-        errno = stub->args.readv_cbk.op_errno;
-        count = stub->args.readv_cbk.count;
-        vector = stub->args.readv_cbk.vector;
-        if (op_ret > 0) {
-                int i = 0;
-                op_ret = 0;
-                while (size && (i < count)) {
-                        int len = (size < vector[i].iov_len) ?
-                                size : vector[i].iov_len;
-                        memcpy (buf, vector[i++].iov_base, len);
-                        buf += len;
-                        size -= len;
-                        op_ret += len;
-                }
-                stbuf = &stub->args.readv_cbk.stbuf;
-                libgf_transform_iattr (ctx, fd->inode, stbuf);
-                libgf_invalidate_iattr_cache (fd->inode, LIBGF_INVALIDATE_STAT);
-        }
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "size %lu, offset %"PRIu64,
-                (long unsigned)size, offset);
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-int
-libgf_client_read (libglusterfs_client_ctx_t *ctx, fd_t *fd, void *buf,
-                   size_t size, off_t offset)
-{
-        int32_t op_ret = -1;
-        int32_t ret = 0;
-        size_t  tmp   = 0;
-
-        while (size != 0) {
-                tmp = ((size > LIBGF_IOBUF_SIZE) ? LIBGF_IOBUF_SIZE :
-                       size);
-                op_ret = libgf_client_iobuf_read (ctx, fd, buf, tmp, offset);
-                if (op_ret < 0) {
-                        ret = op_ret;
-                        break;
-                }
-
-                ret += op_ret;
-
-                if (op_ret < tmp)
-                        break;
-
-                size -= op_ret;
-                offset += op_ret;
-                buf = (char *)buf + op_ret;
-        }
-
-        return ret;
-}
-
-ssize_t
-glusterfs_read (glusterfs_file_t fd, void *buf, size_t nbytes)
-{
-        int32_t op_ret = -1;
-        off_t offset = 0;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-
-        if (nbytes < 0) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (nbytes == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        if (fd == 0) {
-                errno = EINVAL;
-		goto out;
-        }
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-		goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                ctx = fd_ctx->ctx;
-                offset = fd_ctx->offset;
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-        op_ret = libgf_client_read (ctx, (fd_t *)fd, buf, nbytes, offset);
-
-        if (op_ret > 0) {
-                offset += op_ret;
-                pthread_mutex_lock (&fd_ctx->lock);
-                {
-                        fd_ctx->offset = offset;
-                }
-                pthread_mutex_unlock (&fd_ctx->lock);
-        }
-
-out:
-        return op_ret;
-}
-
-
-ssize_t
-libgf_client_iobuf_readv (libglusterfs_client_ctx_t *ctx, fd_t *fd,
-                          const struct iovec *dst_vector, int count,
-                          size_t size, off_t offset, int *idx,
-                          off_t *vec_offset)
-{
-        call_stub_t          *stub       = NULL;
-        struct iovec         *src_vector = NULL;
-        int32_t               op_ret     = -1;
-        libgf_client_local_t *local      = NULL;
-        int                   src        = 0, dst = 0;
-        int                   src_count  = 0, dst_count = 0;
-        int                   len        = 0, src_len = 0, dst_len = 0;
-        off_t                 src_offset = 0, dst_offset = 0;
-        struct iatt          *stbuf      = NULL;
-
-        dst = *idx;
-        dst_offset = *vec_offset;
-
-        local = CALLOC (1, sizeof (*local));
-        ERR_ABORT (local);
-        local->fd = fd;
-        LIBGF_CLIENT_FOP (ctx, stub, readv, local, fd, size, offset);
-
-        op_ret = stub->args.readv_cbk.op_ret;
-        errno = stub->args.readv_cbk.op_errno;
-        src_count = stub->args.readv_cbk.count;
-        src_vector = stub->args.readv_cbk.vector;
-        if (op_ret > 0) {
-                while ((size != 0) && (dst < dst_count) && (src < src_count)) {
-                        src_len = src_vector[src].iov_len - src_offset;
-                        dst_len = dst_vector[dst].iov_len - dst_offset;
-
-                        len = (src_len < dst_len) ? src_len : dst_len;
-                        if (len > size) {
-                                len = size;
-                        }
-
-                        memcpy (dst_vector[dst].iov_base + dst_offset,
-				src_vector[src].iov_base + src_offset, len);
-
-                        size -= len;
-                        src_offset += len;
-                        dst_offset += len;
-
-                        if (src_offset == src_vector[src].iov_len) {
-                                src_offset = 0;
-                                src++;
-                        }
-
-                        if (dst_offset == dst_vector[dst].iov_len) {
-                                dst_offset = 0;
-                                dst++;
-                        }
-                }
-
-                stbuf = &stub->args.readv_cbk.stbuf;
-                libgf_transform_iattr (ctx, fd->inode, stbuf);
-                libgf_invalidate_iattr_cache (fd->inode, LIBGF_UPDATE_STAT);
-        }
-
-        *idx = dst;
-        *vec_offset = dst_offset;
-
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-
-ssize_t
-libgf_client_readv (libglusterfs_client_ctx_t *ctx, fd_t *fd,
-                    const struct iovec *dst_vector, int dst_count, off_t offset)
-{
-        int32_t               op_ret     = -1;
-        size_t                size       = 0, tmp = 0, ret = 0;
-        int                   i          = 0;
-        int                   dst_idx    = 0;
-        off_t                 dst_offset = 0;
-
-        for (i = 0; i < dst_count; i++)
-        {
-                size += dst_vector[i].iov_len;
-        }
-
-        while (size != 0) {
-                tmp = ((size > LIBGF_IOBUF_SIZE) ? LIBGF_IOBUF_SIZE : size);
-                op_ret = libgf_client_iobuf_readv (ctx, fd, dst_vector,
-                                                   dst_count, tmp, offset,
-                                                   &dst_idx, &dst_offset);
-                if (op_ret <= 0) {
-                        break;
-                }
-
-                offset += op_ret;
-                size -= op_ret;
-                ret += op_ret;
-        }
-
-        return ret;
-}
-
-
-ssize_t
-glusterfs_readv (glusterfs_file_t fd, const struct iovec *vec, int count)
-{
-        int32_t op_ret = -1;
-        off_t offset = 0;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-
-        if (count < 0) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (count == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        if (!fd) {
-                errno = EINVAL;
-		goto out;
-        }
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                ctx = fd_ctx->ctx;
-                offset = fd_ctx->offset;
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-        op_ret = libgf_client_readv (ctx, (fd_t *)fd, vec, count, offset);
-
-        if (op_ret > 0) {
-                offset += op_ret;
-                pthread_mutex_lock (&fd_ctx->lock);
-                {
-                        fd_ctx->offset = offset;
-                }
-                pthread_mutex_unlock (&fd_ctx->lock);
-        }
-
-out:
-        return op_ret;
-}
-
-
-ssize_t 
-glusterfs_pread (glusterfs_file_t fd, 
-                 void *buf, 
-                 size_t count, 
-                 off_t offset)
-{
-        int32_t op_ret = -1;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-
-        if (count < 0) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (count == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        if (!fd) {
-                errno = EINVAL;
-		goto out;
-        }
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		goto out;
-        }
-
-        ctx = fd_ctx->ctx;
-
-        op_ret = libgf_client_read (ctx, (fd_t *)fd, buf, count, offset);
-
-out:
-        return op_ret;
-}
-
-
-int
-libgf_client_writev_cbk (call_frame_t *frame,
-                         void *cookie,
-                         xlator_t *this,
-                         int32_t op_ret,
-                         int32_t op_errno,
-                         struct iatt *prebuf,
-                         struct iatt *postbuf)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_writev_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                 prebuf, postbuf);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-
-int
-libgf_client_iobuf_write (libglusterfs_client_ctx_t *ctx, fd_t *fd, char *addr,
-                          size_t size, off_t offset)
-{
-        struct iobref        *ioref = NULL;
-        struct iobuf         *iob = NULL;
-        int                   op_ret = -1;
-        struct iovec          iov = {0, };
-        call_stub_t          *stub = NULL;
-        libgf_client_local_t *local = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, fd, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, addr, out);
-
-        ioref = iobref_new ();
-        if (!ioref) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Out of memory");
-                goto out;
-        }
-
-        iob = iobuf_get (ctx->gf_ctx.iobuf_pool);
-        if (!iob) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Out of memory");
-                goto out;
-        }
-
-        memcpy (iob->ptr, addr, size);
-        iobref_add (ioref, iob);
-
-        iov.iov_base = iob->ptr;
-        iov.iov_len = size;
-
-        LIBGF_CLIENT_FOP (ctx, stub, writev, local, fd, &iov,
-                          1, offset, ioref);
-
-        op_ret = stub->args.writev_cbk.op_ret;
-        errno = stub->args.writev_cbk.op_errno;
-
-        /* We need to invalidate because it is possible that write-behind
-         * is a translator below us and returns a stat filled with zeroes.
-         */
-        libgf_invalidate_iattr_cache (fd->inode, LIBGF_INVALIDATE_STAT);
-
-out:
-        if (iob) {
-                iobuf_unref (iob);
-        }
-
-        if (ioref) {
-                iobref_unref (ioref);
-        }
-
-        call_stub_destroy (stub);
-        return op_ret;
-}
-
-int
-libgf_client_writev (libglusterfs_client_ctx_t *ctx, 
-                     fd_t *fd, 
-                     struct iovec *vector, 
-                     int count, 
-                     off_t offset)
-{
-        int                     op_ret = 0;
-        int                     written = 0;
-        int                     writesize = 0;
-        int                     size = 0;
-        char                   *base = NULL;
-        int                     i = 0;
-
-        for (i = 0; i < count; i++) {
-                size = vector[i].iov_len;
-                base = vector[i].iov_base;
-
-                while (size > 0) {
-                        writesize = (size > LIBGF_IOBUF_SIZE) ?
-                                LIBGF_IOBUF_SIZE : size;
-
-                        written = libgf_client_iobuf_write (ctx, fd, base,
-                                                            writesize, offset);
-
-                        if (written == -1)
-                                goto out;
-
-                        op_ret += written;
-                        base += written;
-                        size -= written;
-                        offset += written;
-                }
-        }
-
-out:
-        return op_ret;
-}
-
-
-ssize_t 
-glusterfs_write (glusterfs_file_t fd, 
-                 const void *buf, 
-                 size_t n)
-{
-        int32_t op_ret = -1;
-        off_t offset = 0;
-        struct iovec vector;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-
-        if (n < 0) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (n == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        if (!fd) {
-                errno = EINVAL;
-		goto out;
-        }
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		goto out;
-        }
-
-        ctx = fd_ctx->ctx;
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                offset = fd_ctx->offset;
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-        vector.iov_base = (void *)buf;
-        vector.iov_len = n;
-
-        op_ret = libgf_client_writev (ctx,
-                                      (fd_t *)fd, 
-                                      &vector, 
-                                      1, 
-                                      offset);
-
-        if (op_ret >= 0) {
-                offset += op_ret;
-                pthread_mutex_lock (&fd_ctx->lock);
-                {
-                        fd_ctx->offset = offset;
-                }
-                pthread_mutex_unlock (&fd_ctx->lock);
-        }
-
-out:
-        return op_ret;
-}
-
-ssize_t 
-glusterfs_writev (glusterfs_file_t fd, 
-                  const struct iovec *vector,
-                  int count)
-{
-        int32_t op_ret = -1;
-        off_t offset = 0;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-
-        if (count < 0) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (count == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        if (!fd) {
-                errno = EINVAL;
-		goto out;
-        }
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		goto out;
-        }
-
-        ctx = fd_ctx->ctx;
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                offset = fd_ctx->offset;
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-
-        op_ret = libgf_client_writev (ctx,
-                                      (fd_t *)fd, 
-                                      (struct iovec *)vector, 
-                                      count,
-                                      offset);
-
-        if (op_ret >= 0) {
-                offset += op_ret;
-                pthread_mutex_lock (&fd_ctx->lock);
-                {
-                        fd_ctx->offset = offset;
-                }
-                pthread_mutex_unlock (&fd_ctx->lock);
-        }
-
-out:
-        return op_ret;
-}
-
-
-ssize_t 
-glusterfs_pwrite (glusterfs_file_t fd, 
-                  const void *buf, 
-                  size_t count, 
-                  off_t offset)
-{
-        int32_t op_ret = -1;
-        struct iovec vector;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-
-        if (count < 0) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (count == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        if (!fd) {
-                errno = EINVAL;
-		goto out;
-        }
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		goto out;
-        }
-
-        ctx = fd_ctx->ctx;
-
-        vector.iov_base = (void *)buf;
-        vector.iov_len = count;
-
-        op_ret = libgf_client_writev (ctx,
-                                      (fd_t *)fd, 
-                                      &vector, 
-                                      1, 
-                                      offset);
-
-out:
-        return op_ret;
-}
-
-
-int32_t
-libgf_client_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           gf_dirent_t *entries)
-{
-        libgf_client_local_t *local = frame->local;
-
-        /* Note, we dont let entries reach the stub because there it gets copied
-         * while we can simply delink the entries here and link them into our
-         * dcache, thereby avoiding the need to perform more allocations and
-         * copies.
-         */
-        local->reply_stub = fop_readdirp_cbk_stub (frame, NULL, op_ret,
-                                                   op_errno, NULL);
-        if (op_ret > 0)
-                libgf_dcache_update (frame->root->state, local->fd, entries);
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int 
-libgf_client_readdir (libglusterfs_client_ctx_t *ctx, fd_t *fd,
-                      struct dirent *dirp, off_t *offset)
-{  
-        call_stub_t *stub = NULL;
-        int op_ret = -1;
-        libgf_client_local_t *local = NULL;
-
-        if (libgf_dcache_readdir (ctx, fd, dirp, offset))
-                return 1;
-        local = CALLOC (1, sizeof (*local));
-        ERR_ABORT (local);
-        local->fd = fd;
-        LIBGF_CLIENT_FOP (ctx, stub, readdirp, local, fd,
-                          LIBGF_READDIR_BLOCK, *offset);
-
-        errno = stub->args.readdir_cbk.op_errno;
-
-        op_ret = libgf_dcache_readdir (ctx, fd, dirp, offset);
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-
-int
-glusterfs_readdir_r (glusterfs_dir_t dirfd, struct dirent *entry,
-                     struct dirent **result)
-{
-        int                           op_ret = -1;
-        libglusterfs_client_ctx_t    *ctx = NULL;
-        off_t                         offset = 0;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-        struct dirent                *dirp = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, entry, out);
-
-        fd_ctx = libgf_get_fd_ctx (dirfd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "fd context not present");
-                errno = EBADF;
-		goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                ctx = fd_ctx->ctx;
-                offset = fd_ctx->offset;
-                dirp = &fd_ctx->dirp;
-
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "offset %"PRIu64, offset);
-                memset (dirp, 0, sizeof (struct dirent));
-                op_ret = libgf_client_readdir (ctx, (fd_t *)dirfd, dirp,
-                                               &offset);
-                if (op_ret <= 0) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "readdir failed:"
-                                " %s", strerror (errno));
-                        if (result && (op_ret == 0)) {
-                                *result = NULL;
-                        } else if (op_ret < 0){
-                                op_ret = errno;
-                        }
-                        goto unlock;
-                }
-
-                fd_ctx->offset = offset;
-
-                if (result) {
-                        *result = memcpy (entry, dirp, sizeof (*entry));
-                } else {
-                        memcpy (entry, dirp, sizeof (*entry));
-                }
-
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "new offset %"PRIu64", "
-                        " entry %s", offset, entry->d_name);
-                op_ret = 0;
-        }
-unlock:
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-out:
-        return op_ret;
-}
-
-
-void *
-glusterfs_readdir (glusterfs_dir_t dirfd)
-{
-        int op_ret = -1;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        off_t offset = 0;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-        struct dirent *dirp = NULL;
-
-        fd_ctx = libgf_get_fd_ctx (dirfd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "fd context not present");
-                errno = EBADF;
-		goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                ctx = fd_ctx->ctx;
-                offset = fd_ctx->offset;
-                dirp = &fd_ctx->dirp;
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "offset %"PRIu64, offset);
-        memset (dirp, 0, sizeof (struct dirent));
-        op_ret = libgf_client_readdir (ctx, (fd_t *)dirfd, dirp, &offset);
-
-        if (op_ret <= 0) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "readdir failed: %s",
-                        strerror (errno));
-                dirp = NULL;
-                goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                fd_ctx->offset = offset;
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "new offset %"PRIu64", entry %s",
-                offset, dirp->d_name);
-out:
-        return dirp;
-}
-
-
-int
-glusterfs_getdents (glusterfs_file_t fd, struct dirent *dirp,
-                    unsigned int count)
-{
-        int op_ret = -1;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        off_t offset = 0;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                ctx = fd_ctx->ctx;
-                offset = fd_ctx->offset;
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-        op_ret = libgf_client_readdir (ctx, (fd_t *)fd, dirp, &offset);
-
-        if (op_ret > 0) {
-                pthread_mutex_lock (&fd_ctx->lock);
-                {
-                        fd_ctx->offset = offset;
-                }
-                pthread_mutex_unlock (&fd_ctx->lock);
-        }
-
-out:
-        return op_ret;
-}
-
-
-static int32_t
-libglusterfs_readv_async_cbk (call_frame_t *frame,
-                              void *cookie,
-                              xlator_t *this,
-                              int32_t op_ret,
-                              int32_t op_errno,
-                              struct iovec *vector,
-                              int32_t count,
-                              struct iatt *stbuf,
-                              struct iobref *iobref)
-{
-        glusterfs_iobuf_t *buf;
-        libglusterfs_client_async_local_t *local = frame->local;
-        fd_t *__fd = local->fop.readv_cbk.fd;
-        glusterfs_readv_cbk_t readv_cbk = local->fop.readv_cbk.cbk;
-
-        buf = CALLOC (1, sizeof (*buf));
-        ERR_ABORT (buf);
-
-	if (vector) {
-		buf->vector = iov_dup (vector, count);
-	}
-
-        buf->count = count;
-
-	if (iobref) {
-		buf->iobref = iobref_ref (iobref);
-	}
-
-        if (op_ret > 0) {
-                libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-                fd_ctx = libgf_get_fd_ctx (__fd);
-                
-                /* update offset only if we have used offset stored in fd_ctx */
-                if (local->fop.readv_cbk.update_offset) {
-                        pthread_mutex_lock (&fd_ctx->lock);
-                        {
-                                fd_ctx->offset += op_ret;
-                        }
-                        pthread_mutex_unlock (&fd_ctx->lock);
-                }
-        }
-
-        readv_cbk (op_ret, op_errno, buf, local->cbk_data); 
-
-	FREE (local);
-	frame->local = NULL;
-        STACK_DESTROY (frame->root);
-
-        return 0;
-}
-
-void 
-glusterfs_free (glusterfs_iobuf_t *buf)
-{
-        //iov_free (buf->vector, buf->count);
-        FREE (buf->vector);
-        if (buf->iobref)
-                iobref_unref ((struct iobref *) buf->iobref);
-        if (buf->dictref)
-                dict_unref ((dict_t *) buf->dictref);
-        FREE (buf);
-}
-
-int
-glusterfs_read_async (glusterfs_file_t fd, 
-                      size_t nbytes, 
-                      off_t offset,
-                      glusterfs_readv_cbk_t readv_cbk,
-                      void *cbk_data)
-{
-        libglusterfs_client_ctx_t *ctx;
-        fd_t *__fd = (fd_t *)fd;
-        libglusterfs_client_async_local_t *local = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-	int32_t op_ret = 0;
-
-        if (nbytes < 0) {
-                errno = EINVAL;
-                op_ret = -1;
-                goto out;
-        }
-
-        if (nbytes == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        local = CALLOC (1, sizeof (*local));
-        ERR_ABORT (local);
-        local->fop.readv_cbk.fd = __fd;
-        local->fop.readv_cbk.cbk = readv_cbk;
-        local->cbk_data = cbk_data;
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		op_ret = -1;
-		goto out;
-        }
-
-        ctx = fd_ctx->ctx;
-
-        if (offset < 0) {
-                pthread_mutex_lock (&fd_ctx->lock);
-                {
-                        offset = fd_ctx->offset;
-                        local->fop.readv_cbk.update_offset = 1;
-                }
-                pthread_mutex_unlock (&fd_ctx->lock);
-        }
-
-        LIBGF_CLIENT_FOP_ASYNC (ctx,
-                                local,
-                                libglusterfs_readv_async_cbk,
-                                readv,
-                                __fd,
-                                nbytes,
-                                offset);
-
-out:
-        return op_ret;
-}
-
-static int32_t
-libglusterfs_writev_async_cbk (call_frame_t *frame,
-                               void *cookie,
-                               xlator_t *this,
-                               int32_t op_ret,
-                               int32_t op_errno,
-                               struct iatt *prebuf,
-                               struct iatt *postbuf)
-{
-        libglusterfs_client_async_local_t *local = frame->local;
-        fd_t *fd = NULL;
-        glusterfs_write_cbk_t write_cbk;
-
-        write_cbk = local->fop.write_cbk.cbk;
-        fd = local->fop.write_cbk.fd;
-
-        if (op_ret > 0) {
-                libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-                fd_ctx = libgf_get_fd_ctx (fd);
-                pthread_mutex_lock (&fd_ctx->lock);
-                {
-                        fd_ctx->offset += op_ret;  
-                }
-                pthread_mutex_unlock (&fd_ctx->lock);
-        }
-
-        write_cbk (op_ret, op_errno, local->cbk_data);
-
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-int32_t
-glusterfs_write_async (glusterfs_file_t fd, 
-                       const void *buf, 
-                       size_t nbytes, 
-                       off_t offset,
-                       glusterfs_write_cbk_t write_cbk,
-                       void *cbk_data)
-{
-        fd_t *__fd = (fd_t *)fd;
-        struct iovec vector;
-        off_t __offset = offset;
-        libglusterfs_client_ctx_t *ctx = NULL;
-        libglusterfs_client_async_local_t *local = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-	int32_t op_ret = 0;
-        struct iobref *iobref = NULL;
-
-        if (nbytes == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        if (nbytes < 0) {
-                op_ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        local = CALLOC (1, sizeof (*local));
-        ERR_ABORT (local);
-        local->fop.write_cbk.fd = __fd;
-        local->fop.write_cbk.cbk = write_cbk;
-        local->cbk_data = cbk_data;
-
-        vector.iov_base = (void *)buf;
-        vector.iov_len = nbytes;
-  
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		op_ret = -1;
-		goto out;
-        }
-
-        ctx = fd_ctx->ctx;
- 
-        if (offset < 0) {
-                pthread_mutex_lock (&fd_ctx->lock);
-                {
-                        __offset = fd_ctx->offset;
-                }
-                pthread_mutex_unlock (&fd_ctx->lock);
-        }
-
-        iobref = iobref_new ();
-        LIBGF_CLIENT_FOP_ASYNC (ctx,
-                                local,
-                                libglusterfs_writev_async_cbk,
-                                writev,
-                                __fd,
-                                &vector,
-                                1,
-                                __offset,
-                                iobref);
-        iobref_unref (iobref);
-
-out:
-        return op_ret;
-}
-
-off_t
-glusterfs_lseek (glusterfs_file_t fd, off_t offset, int whence)
-{
-        off_t __offset = 0;
-	int32_t op_ret = -1;
-        fd_t *__fd = (fd_t *)fd;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-	libglusterfs_client_ctx_t *ctx = NULL; 
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		__offset = -1;
-		goto out;
-        }
-
-	ctx = fd_ctx->ctx;
-
-        switch (whence)
-        {
-        case SEEK_SET:
-                __offset = offset;
-                break;
-
-        case SEEK_CUR:
-                pthread_mutex_lock (&fd_ctx->lock);
-                {
-                        __offset = fd_ctx->offset;
-                }
-                pthread_mutex_unlock (&fd_ctx->lock);
-
-                __offset += offset;
-                break;
-
-        case SEEK_END:
-	{
-		char cache_valid = 0;
-		off_t end = 0;
-		loc_t loc = {0, };
-		struct iatt stbuf = {0, };
-
-                cache_valid = libgf_is_iattr_cache_valid (ctx, __fd->inode,
-                                                          &stbuf,
-                                                          LIBGF_VALIDATE_STAT);
-                if (cache_valid) {
-			end = stbuf.ia_size;
-		} else {
-			op_ret = libgf_client_loc_fill (&loc, ctx,
-                                                        __fd->inode->ino, 0,
-                                                        NULL);
-			if (op_ret == -1) {
-				gf_log ("libglusterfsclient",
-					GF_LOG_ERROR,
-					"libgf_client_loc_fill returned -1, returning EINVAL");
-				errno = EINVAL;
-				libgf_client_loc_wipe (&loc);
-				__offset = -1;
-				goto out;
-			}
-			
-			op_ret = libgf_client_lookup (ctx, &loc, &stbuf, NULL,
-                                                      NULL);
-			if (op_ret < 0) {
-				__offset = -1;
-				libgf_client_loc_wipe (&loc);
-				goto out;
-			}
-
-			end = stbuf.ia_size;
-		}
-
-                __offset = end + offset; 
-		libgf_client_loc_wipe (&loc);
-	}
-	break;
-
-	default:
-		gf_log ("libglusterfsclient",
-			GF_LOG_ERROR,
-			"invalid value for whence");
-		__offset = -1;
-		errno = EINVAL;
-		goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                fd_ctx->offset = __offset;
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
- 
-out: 
-        return __offset;
-}
-
-
-int32_t
-libgf_client_stat_cbk (call_frame_t *frame,
-                       void *cookie,
-                       xlator_t *this,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       struct iatt *buf)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_stat_cbk_stub (frame, 
-                                               NULL, 
-                                               op_ret, 
-                                               op_errno, 
-                                               buf);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int32_t 
-libgf_client_stat (libglusterfs_client_ctx_t *ctx, 
-                   loc_t *loc,
-                   struct iatt *stbuf)
-{
-        call_stub_t *stub = NULL;
-        int32_t op_ret = 0;
-        libgf_client_local_t *local = NULL;
-        struct iatt cachedbuf = {0, };
-
-        if (libgf_is_iattr_cache_valid (ctx, loc->inode, &cachedbuf,
-                                        LIBGF_VALIDATE_STAT)) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Cache will be used");
-                if (stbuf)
-                        memcpy (stbuf, &cachedbuf, sizeof (struct stat));
-                goto out;
-        }
-
-        LIBGF_CLIENT_FOP (ctx, stub, stat, local, loc);
- 
-        op_ret = stub->args.stat_cbk.op_ret;
-        errno = stub->args.stat_cbk.op_errno;
-        libgf_transform_iattr (ctx, loc->inode, &stub->args.stat_cbk.buf);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, status %d, errno %d",
-                loc->path, op_ret, errno);
-
-        if (op_ret == 0) {
-                if (stbuf)
-                        *stbuf = stub->args.stat_cbk.buf;
-
-                libgf_update_iattr_cache (loc->inode, LIBGF_UPDATE_STAT,
-                                          &stub->args.stat_cbk.buf);
-        }
-
-	call_stub_destroy (stub);
-
-out:
-        return op_ret;
-}
-
-int
-libgf_realpath_loc_fill (libglusterfs_client_ctx_t *ctx, char *link,
-                                loc_t *targetloc)
-{
-        int             op_ret = -1;
-        char            *target = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, link, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, targetloc, out);
-
-        targetloc->path = glusterfs_glh_realpath (ctx, link, NULL);
-
-        if (targetloc->path == NULL)
-                goto out;
-
-        op_ret = libgf_client_path_lookup (targetloc, ctx, 1);
-        if (op_ret == -1)
-                goto out;
-
-        target = strdup (targetloc->path);
-        op_ret = libgf_client_loc_fill (targetloc, ctx, 0,
-                                               targetloc->parent->ino,
-                                               basename (target));
-        if (op_ret == -1) {
-                errno = EINVAL;
-                goto out;
-        }
-
-out:
-        if (target)
-                FREE (target);
-
-        return op_ret;
-}
-
-#define LIBGF_DO_LSTAT  0x01
-#define LIBGF_DO_STAT   0x02
-
-int
-__glusterfs_stat (glusterfs_handle_t handle, const char *path,
-                  struct stat *buf, int whichstat)
-{
-        int32_t op_ret = -1;
-        loc_t loc = {0, };
-        libglusterfs_client_ctx_t *ctx = handle;
-	char *name = NULL, *pathname = NULL;
-        loc_t targetloc = {0, };
-        loc_t *real_loc = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, op: %d", path,
-                whichstat);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-	op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient", GF_LOG_ERROR,
-			"path lookup failed for (%s)", loc.path);
-		goto out;
-	}
-
-	pathname = strdup (loc.path);
-	name = basename (pathname);
-
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino, name);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient",
-			GF_LOG_ERROR,
-			"libgf_client_loc_fill returned -1, returning EINVAL");
-		errno = EINVAL;
-		goto out;
-	}
-        real_loc = &loc;
-        /* The stat fop in glusterfs calls lstat. So we have to
-         * provide the POSIX compatible stat fop. To do so, we need to ensure
-         * that if the @path is a symlink, we must perform a stat on the
-         * target of that symlink than the symlink itself(..because if
-         * do a stat on the symlink, we're actually doing what lstat
-         * should do. See posix_stat
-         */
-        if (whichstat & LIBGF_DO_LSTAT)
-                goto lstat_fop;
-
-        if (!IA_ISLNK (loc.inode->ia_type))
-                goto lstat_fop;
-
-        op_ret = libgf_realpath_loc_fill (ctx, (char *)loc.path, &targetloc);
-        if (op_ret == -1)
-                goto out;
-        real_loc = &targetloc;
-
-lstat_fop:
-
-        if (!op_ret) {
-                struct iatt iatt;
-                op_ret = libgf_client_stat (ctx, real_loc, &iatt);
-                iatt_to_stat (&iatt, buf);
-        }
-
-out:
-	if (pathname) {
-		FREE (pathname);
-	}
-
-        libgf_client_loc_wipe (&loc);
-        libgf_client_loc_wipe (&targetloc);
-
-        return op_ret;
-}
-
-int
-glusterfs_glh_stat (glusterfs_handle_t handle, const char *path,
-                    struct stat *buf)
-{
-        return __glusterfs_stat (handle, path, buf, LIBGF_DO_STAT);
-}
-
-int
-glusterfs_stat (const char *path, struct stat *buf)
-{
-        glusterfs_handle_t      h      = NULL;
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, buf, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_stat (h, vpath, buf);
-
-out:
-        return op_ret;
-}
-
-int
-glusterfs_glh_lstat (glusterfs_handle_t handle, const char *path, struct stat *buf)
-{
-        return __glusterfs_stat (handle, path, buf, LIBGF_DO_LSTAT);
-}
-
-int
-glusterfs_lstat (const char *path, struct stat *buf)
-{
-        glusterfs_handle_t      h      = NULL;
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, buf, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_lstat (h, vpath, buf);
-out:
-        return op_ret;
-}
-
-static int32_t
-libgf_client_fstat_cbk (call_frame_t *frame,
-                        void *cookie,
-                        xlator_t *this,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        struct iatt *buf)
-{  
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_fstat_cbk_stub (frame, 
-                                                NULL, 
-                                                op_ret, 
-                                                op_errno, 
-                                                buf);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-
-}
-
-int32_t
-libgf_client_fstat (libglusterfs_client_ctx_t *ctx, 
-                    fd_t *fd, 
-                    struct stat *buf)
-{
-        call_stub_t *stub = NULL;
-        int32_t op_ret = 0;
-        libgf_client_local_t *local = NULL;
-        struct iatt cachedbuf = {0, };
-
-        if (libgf_is_iattr_cache_valid (ctx, fd->inode, &cachedbuf,
-                                        LIBGF_VALIDATE_STAT)) {
-                if (buf)
-                        memcpy (buf, &cachedbuf, sizeof (struct stat));
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Cache will be used");
-                goto out;
-        }
-
-        LIBGF_CLIENT_FOP (ctx, stub, fstat, local, fd);
- 
-        op_ret = stub->args.fstat_cbk.op_ret;
-        errno = stub->args.fstat_cbk.op_errno;
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "status %d, errno %d", op_ret,
-                errno);
-
-        if (op_ret == 0) {
-                libgf_transform_iattr (ctx, fd->inode,
-                                       &stub->args.fstat_cbk.buf);
-                if (buf)
-                        iatt_to_stat (&stub->args.fstat_cbk.buf, buf);
-                libgf_update_iattr_cache (fd->inode, LIBGF_UPDATE_STAT,
-                                          &stub->args.fstat_cbk.buf);
-        }
-	call_stub_destroy (stub);
-
-out:
-        return op_ret;
-}
-
-int32_t 
-glusterfs_fstat (glusterfs_file_t fd, struct stat *buf) 
-{
-        libglusterfs_client_ctx_t *ctx;
-        fd_t *__fd = (fd_t *)fd;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-	int32_t op_ret = -1;
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-		op_ret = -1;
-		goto out;
-        }
-
-        ctx = fd_ctx->ctx;
-
-	op_ret = libgf_client_fstat (ctx, __fd, buf);
-
-out:
-	return op_ret;
-}
-
-
-static int32_t
-libgf_client_mkdir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			inode_t *inode,
-                        struct iatt *buf,
-                        struct iatt *preparent,
-                        struct iatt *postparent)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_mkdir_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                inode, buf, preparent,
-                                                postparent);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-
-static int32_t
-libgf_client_mkdir (libglusterfs_client_ctx_t *ctx,
-		    loc_t *loc,
-		    mode_t mode)
-{
-	int32_t op_ret = -1;
-        call_stub_t *stub = NULL;
-        libgf_client_local_t *local = NULL;
-        inode_t *libgf_inode = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, mkdir, local, loc, mode);
-        op_ret = stub->args.mkdir_cbk.op_ret;
-        errno = stub->args.mkdir_cbk.op_errno;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, status %d, errno %d",
-                loc->path, op_ret, errno);
-        if (op_ret == -1)
-                goto out;
-
-	libgf_inode = stub->args.mkdir_cbk.inode;
-        inode_link (libgf_inode, loc->parent, loc->name,
-                        &stub->args.mkdir_cbk.buf);
-        libgf_transform_iattr (ctx, libgf_inode, &stub->args.mkdir_cbk.buf);
-
-        inode_lookup (libgf_inode);
-
-        libgf_alloc_inode_ctx (ctx, libgf_inode);
-        libgf_update_iattr_cache (libgf_inode, LIBGF_UPDATE_ALL,
-                                        &stub->args.mkdir_cbk.buf);
-
-out:
-	call_stub_destroy (stub);
-
-	return op_ret;
-}
-
-
-int32_t
-glusterfs_glh_mkdir (glusterfs_handle_t handle, const char *path, mode_t mode)
-{
-	libglusterfs_client_ctx_t *ctx = handle;
-	loc_t loc = {0, };
-	char *pathname = NULL, *name = NULL;
-	int32_t op_ret = -1;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-	op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-	if (op_ret == 0) {
-                op_ret = -1;
-                errno = EEXIST;
-		goto out;
-	}
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 0);
-        if (op_ret == -1) {
-                errno = ENOENT;
-                goto out;
-        }
-
-	pathname = strdup (loc.path);
-	name = basename (pathname);
-
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino, name);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient",
-			GF_LOG_ERROR,
-			"libgf_client_loc_fill returned -1, returning EINVAL");
-		errno = EINVAL;
-		goto out;
-	}
-
-        loc.inode = inode_new (ctx->itable);
-	op_ret = libgf_client_mkdir (ctx, &loc, mode); 
-	if (op_ret == -1) {
-		goto out;
-	}
-
-out:
-	libgf_client_loc_wipe (&loc);
-	if (pathname) {
-		free (pathname);
-		pathname = NULL;
-	}
-
-	return op_ret;
-}
-
-int32_t
-glusterfs_mkdir (const char *path, mode_t mode)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_mkdir (h, vpath, mode);
-out:
-        return op_ret;
-}
-
-static int32_t
-libgf_client_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,struct iatt *preparent,
-                        struct iatt *postparent)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_rmdir_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                preparent, postparent);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-static int32_t
-libgf_client_rmdir (libglusterfs_client_ctx_t *ctx, loc_t *loc)
-{
-        int32_t op_ret = -1;
-        call_stub_t *stub = NULL;
-        libgf_client_local_t *local = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, rmdir, local, loc);
-
-        op_ret = stub->args.rmdir_cbk.op_ret;
-        errno = stub->args.rmdir_cbk.op_errno;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, status %d, errno %d",
-                loc->path, op_ret, errno);
-        if (stub->args.rmdir_cbk.op_ret != 0)
-                goto out;
-
-        inode_unlink (loc->inode, loc->parent, loc->name);
-
-out:
-	call_stub_destroy (stub);
-
-	return op_ret;
-}
-
-int32_t
-glusterfs_glh_rmdir (glusterfs_handle_t handle, const char *path)
-{
-	libglusterfs_client_ctx_t *ctx = handle;
-	loc_t loc = {0, };
-	char *pathname = NULL, *name = NULL;
-	int32_t op_ret = -1;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        loc.path = libgf_resolve_path_light ((char *)path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Path compaction failed");
-                goto out;
-        }
-
-	op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient", GF_LOG_ERROR,
-			"path lookup failed for (%s)", loc.path);
-		goto out;
-	}
-
-	pathname = strdup (loc.path);
-	name = basename (pathname);
-
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino, name);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient",
-			GF_LOG_ERROR,
-			"libgf_client_loc_fill returned -1, returning EINVAL");
-		errno = EINVAL;
-		goto out;
-	}
-
-	op_ret = libgf_client_rmdir (ctx, &loc);
-	if (op_ret == -1) {
-		goto out;
-	}
-
-out:
-	libgf_client_loc_wipe (&loc);
-
-	if (pathname) {
-		free (pathname);
-		pathname = NULL;
-	}
-
-	return op_ret;
-}
-
-int32_t
-glusterfs_rmdir (const char *path)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_rmdir (h, vpath);
-out:
-        return op_ret;
-}
-
-int
-libgf_client_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt *preop, struct iatt *postop)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_setattr_cbk_stub (frame, NULL,
-                                                  op_ret, op_errno,
-                                                  preop, postop);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int
-libgf_client_setattr (libglusterfs_client_ctx_t *ctx, loc_t * loc,
-                      struct iatt *stbuf, int32_t valid)
-{
-        int                             op_ret = -1;
-        libgf_client_local_t            *local = NULL;
-        call_stub_t                     *stub = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, setattr, local, loc,
-                          stbuf, valid);
-
-        op_ret = stub->args.setattr_cbk.op_ret;
-        errno = stub->args.setattr_cbk.op_errno;
-
-        if (op_ret == -1)
-                goto out;
-
-        libgf_transform_iattr (ctx, loc->inode,
-                               &stub->args.setattr_cbk.statpost);
-        libgf_update_iattr_cache (loc->inode, LIBGF_UPDATE_STAT,
-                                  &stub->args.setattr_cbk.statpost);
-out:
-        call_stub_destroy (stub);
-        return op_ret;
-}
-
-
-int
-glusterfs_glh_chmod (glusterfs_handle_t handle, const char *path, mode_t mode)
-{
-        int                             op_ret = -1;
-        libglusterfs_client_ctx_t       *ctx = handle;
-        loc_t                           loc = {0, };
-        char                            *name = NULL;
-        struct iatt                     stbuf = {0,};
-        int32_t                         valid = 0;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        stbuf.ia_prot = ia_prot_from_st_mode (mode);
-        valid |= GF_SET_ATTR_MODE;
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == -1)
-                goto out;
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                                basename (name));
-        if (op_ret == -1) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        op_ret = libgf_client_setattr (ctx, &loc, &stbuf, valid);
-
-out:
-        if (name)
-                FREE (name);
-
-        libgf_client_loc_wipe (&loc);
-        return op_ret;
-}
-
-int
-glusterfs_chmod (const char *path, mode_t mode)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_chmod (h, vpath, mode);
-out:
-        return op_ret;
-}
-
-
-#define LIBGF_DO_CHOWN  1
-#define LIBGF_DO_LCHOWN 2
-
-int
-__glusterfs_chown (glusterfs_handle_t handle, const char *path, uid_t owner,
-                   gid_t group, int whichop)
-{
-        int                             op_ret = -1;
-        libglusterfs_client_ctx_t       *ctx = handle;
-        loc_t                           loc = {0, };
-        char                            *name = NULL;
-        loc_t                           *oploc = NULL;
-        loc_t                           targetloc = {0, };
-        struct iatt                     stbuf = {0,};
-        int32_t                         valid = 0;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, op %d", path, whichop);
-        stbuf.ia_uid = owner;
-        stbuf.ia_gid = group;
-        valid |= (GF_SET_ATTR_UID | GF_SET_ATTR_GID);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == -1)
-                goto out;
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                        basename ((char *)name));
-        if (op_ret == -1) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        oploc = &loc;
-        if (whichop == LIBGF_DO_LCHOWN)
-                goto do_lchown;
-
-        if (!IA_ISLNK (loc.inode->ia_type))
-                goto do_lchown;
-
-        op_ret = libgf_realpath_loc_fill (ctx, (char *)loc.path, &targetloc);
-        if (op_ret == -1)
-                goto out;
-
-        oploc = &targetloc;
-do_lchown:
-        op_ret = libgf_client_setattr (ctx, oploc, &stbuf, valid);
-out:
-        if (name)
-                FREE (name);
-        libgf_client_loc_wipe (&loc);
-        libgf_client_loc_wipe (&targetloc);
-        return op_ret;
-}
-
-int
-glusterfs_glh_chown (glusterfs_handle_t handle, const char *path, uid_t owner,
-                     gid_t group)
-{
-        return __glusterfs_chown (handle, path, owner, group, LIBGF_DO_CHOWN);
-}
-
-int
-glusterfs_chown (const char *path, uid_t owner, gid_t group)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_chown (h, vpath, owner, group);
-
-out:
-        return op_ret;
-}
-
-int
-glusterfs_glh_lchown (glusterfs_handle_t handle, const char *path, uid_t owner,
-                     gid_t group)
-{
-        return __glusterfs_chown (handle, path, owner, group, LIBGF_DO_LCHOWN);
-}
-
-int
-glusterfs_lchown (const char *path, uid_t owner, gid_t group)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_lchown (h, vpath, owner, group);
-out:
-        return op_ret;
-}
-
-glusterfs_dir_t
-glusterfs_glh_opendir (glusterfs_handle_t handle, const char *path)
-{
-        int                             op_ret = -1;
-        libglusterfs_client_ctx_t       *ctx = handle;
-        loc_t                           loc = {0, };
-        fd_t                            *dirfd = NULL;
-        char                            *name = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-
-        if (op_ret == -1)
-                goto out;
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                        basename (name));
-        if (op_ret == -1) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (!IA_ISDIR (loc.inode->ia_type) && !IA_ISLNK (loc.inode->ia_type)) {
-                errno = ENOTDIR;
-                op_ret = -1;
-                goto out;
-        }
-
-        dirfd = fd_create (loc.inode, ctx->pid);
-        op_ret = libgf_client_opendir (ctx, &loc, dirfd);
-
-        if (op_ret == -1) {
-                fd_unref (dirfd);
-                dirfd = NULL;
-                goto out;
-        }
-
-        if (!libgf_get_fd_ctx (dirfd)) {
-                if (!(libgf_alloc_fd_ctx (ctx, dirfd, (char *)path))) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Context "
-                                "allocation failed");
-                        op_ret = -1;
-                        errno = EINVAL;
-                        goto out;
-                }
-        }
-
-out:
-        if (name)
-                FREE (name);
-
-        if (op_ret == -1) {
-                fd_unref (dirfd);
-                dirfd = NULL;
-        }
-
-        libgf_client_loc_wipe (&loc);
-        return dirfd;
-}
-
-glusterfs_dir_t
-glusterfs_opendir (const char *path)
-{
-        char                    vpath[PATH_MAX];
-        glusterfs_dir_t         dir = NULL;
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        dir = glusterfs_glh_opendir (h, vpath);
-out:
-        return dir;
-}
-
-int
-glusterfs_closedir (glusterfs_dir_t dirfd)
-{
-        int                             op_ret = -1;
-        libglusterfs_client_fd_ctx_t    *fdctx = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, dirfd, out);
-        fdctx = libgf_get_fd_ctx (dirfd);
-
-        if (fdctx == NULL) {
-                errno = EBADF;
-                op_ret = -1;
-                goto out;
-        }
-
-        op_ret = libgf_client_flush (fdctx->ctx, (fd_t *)dirfd);
-        fd_unref ((fd_t *)dirfd);
-
-out:
-        return op_ret;
-}
-
-
-int
-libgf_client_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *preop, struct iatt *postop)
-{
-        libgf_client_local_t    *local = frame->local;
-
-        local->reply_stub = fop_fsetattr_cbk_stub (frame, NULL,
-                                                   op_ret, op_errno,
-                                                   preop, postop);
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-
-int
-libgf_client_fsetattr (libglusterfs_client_ctx_t *ctx, fd_t *fd,
-                       struct iatt *stbuf, int32_t valid)
-{
-        int                     op_ret = -1;
-        libgf_client_local_t    *local = NULL;
-        call_stub_t             *stub = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, fsetattr, local, fd, stbuf, valid);
-
-        op_ret = stub->args.fsetattr_cbk.op_ret;
-        errno = stub->args.fsetattr_cbk.op_errno;
-
-        if (op_ret == -1)
-                goto out;
-
-        libgf_transform_iattr (ctx, fd->inode,
-                               &stub->args.fsetattr_cbk.statpost);
-        libgf_update_iattr_cache (fd->inode, LIBGF_UPDATE_STAT,
-                                  &stub->args.fsetattr_cbk.statpost);
-out:
-        call_stub_destroy (stub);
-        return op_ret;
-}
-
-
-int
-glusterfs_fchmod (glusterfs_file_t fd, mode_t mode)
-{
-        libglusterfs_client_fd_ctx_t    *fdctx = NULL;
-        int                             op_ret = -1;
-        struct iatt                     stbuf = {0,};
-        int32_t                         valid = 0;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, fd, out);
-        fdctx = libgf_get_fd_ctx (fd);
-
-        if (!fdctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "No fd context present");
-                errno = EBADF;
-                goto out;
-        }
-
-        stbuf.ia_prot = ia_prot_from_st_mode (mode);
-        valid |= GF_SET_ATTR_MODE;
-
-        op_ret = libgf_client_fsetattr (fdctx->ctx, fd, &stbuf, valid);
-out:
-        return op_ret;
-}
-
-
-int
-glusterfs_fchown (glusterfs_file_t fd, uid_t uid, gid_t gid)
-{
-        int                             op_ret = -1;
-        libglusterfs_client_fd_ctx_t    *fdctx = NULL;
-        struct iatt                     stbuf = {0,};
-        int32_t                         valid = 0;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, fd, out);
-
-        fdctx = libgf_get_fd_ctx (fd);
-        if (!fd) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-                goto out;
-        }
-        stbuf.ia_uid = uid;
-        stbuf.ia_gid = gid;
-
-        valid |= (GF_SET_ATTR_UID | GF_SET_ATTR_GID);
-
-        op_ret = libgf_client_fsetattr (fdctx->ctx, fd, &stbuf, valid);
-
-out:
-        return op_ret;
-}
-
-int
-libgf_client_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *xlator,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
-{
-        libgf_client_local_t    *local = frame->local;
-
-        local->reply_stub = fop_fsync_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                prebuf, postbuf);
-
-        LIBGF_REPLY_NOTIFY (local);
-
-        return 0;
-}
-
-int
-libgf_client_fsync (libglusterfs_client_ctx_t *ctx, fd_t *fd)
-{
-        libgf_client_local_t    *local = NULL;
-        call_stub_t             *stub = NULL;
-        int                     op_ret = -1;
-
-        LIBGF_CLIENT_FOP (ctx, stub, fsync, local, fd, 0);
-
-        op_ret = stub->args.fsync_cbk.op_ret;
-        errno = stub->args.fsync_cbk.op_errno;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "status %d, errno %d", op_ret,
-                errno);
-        call_stub_destroy (stub);
-
-        return op_ret;
-}
-
-int
-glusterfs_fsync (glusterfs_file_t *fd)
-{
-        libglusterfs_client_fd_ctx_t    *fdctx = NULL;
-        int                             op_ret = -1;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, fd, out);
-
-        fdctx = libgf_get_fd_ctx ((fd_t *)fd);
-        if (!fdctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-                goto out;
-        }
-
-        op_ret = libgf_client_fsync (fdctx->ctx, (fd_t *)fd);
-
-out:
-        return op_ret;
-}
-
-int
-libgf_client_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *xlator
-                                ,int32_t op_ret, int32_t op_errno,
-                                struct iatt *prebuf, struct iatt *postbuf)
-{
-        libgf_client_local_t    *local = frame->local;
-
-        local->reply_stub = fop_ftruncate_cbk_stub (frame, NULL, op_ret,
-                                                    op_errno, prebuf, postbuf);
-
-        LIBGF_REPLY_NOTIFY (local);
-
-        return 0;
-}
-
-int
-libgf_client_ftruncate (libglusterfs_client_ctx_t *ctx, fd_t *fd,
-                                off_t length)
-{
-        libgf_client_local_t            *local = NULL;
-        call_stub_t                     *stub = NULL;
-        int                             op_ret = -1;
-        libglusterfs_client_fd_ctx_t    *fdctx = NULL;
-
-        if (!(((fd->flags & O_ACCMODE) == O_RDWR)
-              || ((fd->flags & O_ACCMODE) == O_WRONLY))) {
-                errno = EBADF;
-                goto out;
-        }
-
-        LIBGF_CLIENT_FOP (ctx, stub, ftruncate, local, fd, length);
-
-        op_ret = stub->args.ftruncate_cbk.op_ret;
-        errno = stub->args.ftruncate_cbk.op_errno;
-
-        if (op_ret == -1)
-                goto out;
-
-        libgf_transform_iattr (ctx, fd->inode,
-                               &stub->args.ftruncate_cbk.postbuf);
-        libgf_update_iattr_cache (fd->inode, LIBGF_UPDATE_STAT,
-                                        &stub->args.ftruncate_cbk.postbuf);
-
-        fdctx = libgf_get_fd_ctx (fd);
-        if (!fd) {
-                errno = EINVAL;
-                op_ret = -1;
-                goto out;
-        }
-
-        pthread_mutex_lock (&fdctx->lock);
-        {
-                fdctx->offset = stub->args.ftruncate_cbk.postbuf.ia_size;
-        }
-        pthread_mutex_unlock (&fdctx->lock);
-
-out:
-        call_stub_destroy (stub);
-        return op_ret;
-}
-
-int
-glusterfs_ftruncate (glusterfs_file_t fd, off_t length)
-{
-        libglusterfs_client_fd_ctx_t    *fdctx = NULL;
-        int                             op_ret = -1;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, fd, out);
-
-        fdctx = libgf_get_fd_ctx (fd);
-        if (!fdctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-                goto out;
-        }
-
-        op_ret = libgf_client_ftruncate (fdctx->ctx, fd, length);
-
-out:
-        return op_ret;
-}
-
-int
-libgf_client_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *buf,
-                                struct iatt *preparent, struct iatt *postparent)
-{
-        libgf_client_local_t            *local = frame->local;
-
-        local->reply_stub = fop_link_cbk_stub (frame, NULL, op_ret, op_errno,
-                                               inode, buf, preparent,
-                                               postparent);
-
-        LIBGF_REPLY_NOTIFY (local);
-
-        return 0;
-}
-
-int
-libgf_client_link (libglusterfs_client_ctx_t *ctx, loc_t *old, loc_t *new)
-{
-        call_stub_t                     *stub = NULL;
-        libgf_client_local_t            *local = NULL;
-        int                             op_ret = -1;
-        inode_t                         *inode = NULL;
-        struct iatt                     *sbuf = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, link, local, old, new);
-
-        op_ret = stub->args.link_cbk.op_ret;
-        errno = stub->args.link_cbk.op_errno;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "old %s, new %s, status %d,"
-                " errno %d", old->path, new->path, op_ret, errno);
-        if (op_ret == -1)
-                goto out;
-
-        inode = stub->args.link_cbk.inode;
-        sbuf = &stub->args.link_cbk.buf;
-        inode_link (inode, new->parent, basename ((char *)new->path), sbuf);
-        libgf_transform_iattr (ctx, inode, sbuf);
-        inode_lookup (inode);
-        libgf_update_iattr_cache (inode, LIBGF_UPDATE_STAT, sbuf);
-
-out:
-        call_stub_destroy (stub);
-        return op_ret;
-}
-
-int
-glusterfs_glh_link (glusterfs_handle_t handle, const char *oldpath,
-                        const char *newpath)
-{
-        libglusterfs_client_ctx_t       *ctx = handle;
-        int                             op_ret = -1;
-        loc_t                           old = {0,};
-        loc_t                           new = {0,};
-        char                            *oldname = NULL;
-        char                            *newname = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, oldpath, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, newpath, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "old %s, new %s", oldpath,
-                newpath);
-
-        old.path = strdup (oldpath);
-        if (!old.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&old, ctx, 1);
-        if (op_ret == -1) {
-                errno = ENOENT;
-                goto out;
-        }
-
-        oldname = strdup (old.path);
-        op_ret = libgf_client_loc_fill (&old, ctx, 0, old.parent->ino,
-                                                basename (oldname));
-        if (op_ret == -1) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (IA_ISDIR (old.inode->ia_type)) {
-                errno = EPERM;
-                op_ret = -1;
-                goto out;
-        }
-
-        new.path = strdup (newpath);
-        if (!new.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&new, ctx, 1);
-        if (op_ret == 0) {
-                errno = EEXIST;
-                op_ret = -1;
-                goto out;
-        }
-
-        newname = strdup (new.path);
-        new.inode = inode_ref (old.inode);
-        libgf_client_loc_fill (&new, ctx, 0, new.parent->ino,
-                        basename (newname));
-        op_ret = libgf_client_link (ctx, &old, &new);
-
-out:
-        if (oldname)
-                FREE (oldname);
-        if (newname)
-                FREE (newname);
-        libgf_client_loc_wipe (&old);
-        libgf_client_loc_wipe (&new);
-
-        return op_ret;
-}
-
-int
-glusterfs_link (const char *oldpath, const char *newpath)
-{
-        int                     op_ret = -1;
-        char                    oldvpath[PATH_MAX];
-        char                    newvpath[PATH_MAX];
-        glusterfs_handle_t      oldh = NULL;
-        glusterfs_handle_t      newh = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, oldpath, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, newpath, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "old %s, new %s", oldpath,
-                newpath);
-
-        oldh = libgf_resolved_path_handle (oldpath, oldvpath);
-        if (!oldh) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        newh = libgf_resolved_path_handle (newpath, newvpath);
-        if (!newh) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        /* Cannot hard link across glusterfs mounts. */
-        if (newh != oldh) {
-                errno = EXDEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_link (newh, oldvpath, newvpath);
-out:
-        return op_ret;
-}
-
-int32_t
-libgf_client_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct statvfs *buf)
-{
-        libgf_client_local_t    *local = frame->local;
-
-        local->reply_stub = fop_statfs_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                 buf);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int32_t
-libgf_client_statvfs (libglusterfs_client_ctx_t *ctx, loc_t *loc,
-                      struct statvfs *buf)
-{
-        call_stub_t             *stub = NULL;
-        libgf_client_local_t    *local = NULL;
-        int32_t                 op_ret = -1;
-
-        /* statfs fop receives struct statvfs as an argument */
-
-        /* libgf_client_statfs_cbk will be the callback, not
-           libgf_client_statvfs_cbk. see definition of LIBGF_CLIENT_FOP
-        */
-        LIBGF_CLIENT_FOP (ctx, stub, statfs, local, loc);
-
-        op_ret = stub->args.statfs_cbk.op_ret;
-        errno = stub->args.statfs_cbk.op_errno;
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, status %d, errno %d",
-                loc->path, op_ret, errno);
-        if (op_ret == -1)
-                goto out;
-
-        if (buf)
-                memcpy (buf, &stub->args.statfs_cbk.buf, sizeof (*buf));
-out:
-        call_stub_destroy (stub);
-        return op_ret;
-}
-
-int
-glusterfs_glh_statfs (glusterfs_handle_t handle, const char *path,
-                        struct statfs *buf)
-{
-        struct statvfs                  stvfs = {0, };
-        int32_t                         op_ret = -1;
-        loc_t                           loc = {0, };
-        libglusterfs_client_ctx_t       *ctx = handle;
-        char                            *name = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                        "path lookup failed for (%s)", loc.path);
-                goto out;
-        }
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                        basename (name));
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1, "
-                                "returning EINVAL");
-                        errno = EINVAL;
-                goto out;
-        }
-
-        op_ret = libgf_client_statvfs (ctx, &loc, &stvfs);
-        if (op_ret == 0) {
-#ifdef GF_SOLARIS_HOST_OS
-		buf->f_fstyp = 0;
-                buf->f_bsize = stvfs.f_bsize;
-                buf->f_blocks = stvfs.f_blocks;
-                buf->f_bfree = stvfs.f_bfree;
-		buf->f_files = stvfs.f_bavail;
-                buf->f_ffree = stvfs.f_ffree;
-#else
-                buf->f_type = 0;
-                buf->f_bsize = stvfs.f_bsize;
-                buf->f_blocks = stvfs.f_blocks;
-                buf->f_bfree = stvfs.f_bfree;
-                buf->f_bavail = stvfs.f_bavail;
-                buf->f_files = stvfs.f_bavail;
-                buf->f_ffree = stvfs.f_ffree;
-                /* FIXME: buf->f_fsid has either "val" or "__val" as member
-                   based on conditional macro expansion. see definition of
-                   fsid_t - Raghu
-                   It seems have different structure member names on
-                   different archs, so I am stepping down to doing a struct
-                   to struct copy. :Shehjar
-                */
-                memcpy (&buf->f_fsid, &stvfs.f_fsid, sizeof (stvfs.f_fsid));
-                buf->f_namelen = stvfs.f_namemax;
-#endif
-        }
-
-out:
-        if (name)
-                FREE (name);
-        libgf_client_loc_wipe (&loc);
-        return op_ret;
-}
-
-int
-glusterfs_statfs (const char *path, struct statfs *buf)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h      = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, buf, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_statfs (h, vpath, buf);
-out:
-        return op_ret;
-}
-
-int
-glusterfs_glh_statvfs (glusterfs_handle_t handle, const char *path,
-                                struct statvfs *buf)
-{
-        int32_t                         op_ret = -1;
-        loc_t                           loc = {0, };
-        libglusterfs_client_ctx_t       *ctx = handle;
-        char                            *name = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                        "path lookup failed for (%s)", loc.path);
-                goto out;
-        }
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                                basename (name));
-	if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1, returning"
-                                " EINVAL");
-                errno = EINVAL;
-                goto out;
-	}
-
-        op_ret = libgf_client_statvfs (ctx, &loc, buf);
-        if (op_ret != -1)
-                /* Should've been a call to libgf_transform_iattr but
-                 * that only handles struct stat
-                 */
-                buf->f_fsid = (unsigned long)ctx->fake_fsid;
-
-out:
-        if (name)
-                FREE (name);
-        libgf_client_loc_wipe (&loc);
-        return op_ret;
-}
-
-int
-glusterfs_statvfs (const char *path, struct statvfs *buf)
-{
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h      = NULL;
-        int                     op_ret = -1;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, buf, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_statvfs (h, vpath, buf);
-out:
-        return op_ret;
-}
-
-int32_t
-libgf_client_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_rename_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                 buf, preoldparent,
-                                                 postoldparent, prenewparent,
-                                                 postnewparent);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int32_t
-libgf_client_rename (libglusterfs_client_ctx_t *ctx, loc_t *oldloc,
-                     loc_t *newloc)
-{
-        int                             op_ret = -1;
-        libgf_client_local_t            *local = NULL;
-        call_stub_t                     *stub = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, rename, local, oldloc, newloc);
-
-        op_ret = stub->args.rename_cbk.op_ret;
-        errno = stub->args.rename_cbk.op_errno;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "old %s, new %s, status %d, errno"
-                " %d", oldloc->path, newloc->path, op_ret, errno);
-        if (op_ret == -1)
-                goto out;
-
-        if (!libgf_get_inode_ctx (newloc->inode))
-                libgf_alloc_inode_ctx (ctx, newloc->inode);
-
-        libgf_transform_iattr (ctx, newloc->inode, &stub->args.rename_cbk.buf);
-        libgf_update_iattr_cache (newloc->inode, LIBGF_UPDATE_STAT,
-                                        &stub->args.rename_cbk.buf);
-
-        inode_unlink (oldloc->inode, oldloc->parent, oldloc->name);
-out:
-        call_stub_destroy (stub);
-        return op_ret;
-}
-
-int
-glusterfs_glh_rename (glusterfs_handle_t handle, const char *oldpath,
-                      const char *newpath)
-{
-        int32_t                         op_ret = -1;
-        loc_t                           oldloc = {0, }, newloc = {0, };
-        libglusterfs_client_ctx_t       *ctx = handle;
-        char                            *newname = NULL;
-        char                            *oldname = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, oldpath, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, newpath, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "old %s, new %s", oldpath,
-                newpath);
-
-        oldloc.path = strdup (oldpath);
-        if (!oldloc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&oldloc, ctx, 1);
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                        "path lookup failed for (%s)", oldloc.path);
-                goto out;
-        }
-
-        newloc.path = strdup (newpath);
-        if (!newloc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&newloc, ctx, 1);
-
-        oldname = strdup (oldloc.path);
-        op_ret = libgf_client_loc_fill (&oldloc, ctx, 0, oldloc.parent->ino,
-                                        basename (oldname));
-	if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1,"
-                                " returning EINVAL");
-                errno = EINVAL;
-                goto out;
-        }
-
-        newname = strdup (newloc.path);
-        op_ret = libgf_client_loc_fill (&newloc, ctx, 0, newloc.parent->ino,
-                                        basename (newname));
-	if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1,"
-                                " returning EINVAL");
-                errno = EINVAL;
-                goto out;
-        }
-        op_ret = libgf_client_rename (ctx, &oldloc, &newloc);
-
-out:
-        if (oldname)
-                FREE (oldname);
-        if (newname)
-                FREE (newname);
-        libgf_client_loc_wipe (&newloc);
-        libgf_client_loc_wipe (&oldloc);
-
-        return op_ret;
-}
-
-
-int
-glusterfs_rename (const char *oldpath, const char *newpath)
-{
-        int                     op_ret = -1;
-        char                    oldvpath[PATH_MAX];
-        char                    newvpath[PATH_MAX];
-        glusterfs_handle_t      oldh = NULL;
-        glusterfs_handle_t      newh = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, oldpath, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, newpath, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Old %s, new %s", oldpath,
-                newpath);
-
-        oldh = libgf_resolved_path_handle (oldpath, oldvpath);
-        if (!oldh) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        newh = libgf_resolved_path_handle (newpath, newvpath);
-        if (!newh) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        if (oldh != newh) {
-                errno = EXDEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_rename (oldh, oldvpath, newvpath);
-out:
-        return op_ret;
-}
-
-
-int
-glusterfs_glh_utimes (glusterfs_handle_t handle, const char *path,
-                        const struct timeval times[2])
-{
-        int32_t                         op_ret = -1;
-        loc_t                           loc = {0, };
-        libglusterfs_client_ctx_t       *ctx = handle;
-        char                            *name = NULL;
-        struct iatt                     stbuf = {0,};
-        int32_t                         valid = 0;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-        stbuf.ia_atime = times[0].tv_sec;
-        stbuf.ia_atime_nsec = times[0].tv_usec * 1000;
-        stbuf.ia_mtime = times[1].tv_sec;
-        stbuf.ia_mtime_nsec = times[1].tv_usec * 1000;
-
-        valid |= (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                        "path lookup failed for (%s)", loc.path);
-                goto out;
-        }
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                                basename (name));
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1"
-                                " returning EINVAL");
-                errno = EINVAL;
-                goto out;
-        }
-
-        op_ret = libgf_client_setattr (ctx, &loc, &stbuf, valid);
-out:
-        if (name)
-                FREE (name);
-        libgf_client_loc_wipe (&loc);
-        return op_ret;
-}
-
-int
-glusterfs_utimes (const char *path, const struct timeval times[2])
-{
-        char                    vpath[PATH_MAX];
-        int                     op_ret = -1;
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_utimes (h, vpath, times);
-out:
-        return op_ret;
-}
-
-int
-glusterfs_glh_utime (glusterfs_handle_t handle, const char *path,
-                        const struct utimbuf *buf)
-{
-        int32_t                         op_ret = -1;
-        loc_t                           loc = {0, };
-        libglusterfs_client_ctx_t       *ctx = handle;
-        char                            *name = NULL;
-        struct iatt                     stbuf = {0,};
-        int32_t                         valid = 0;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-        if (buf) {
-                stbuf.ia_atime = buf->actime;
-                stbuf.ia_atime_nsec = 0;
-
-                stbuf.ia_mtime = buf->modtime;
-                stbuf.ia_mtime_nsec = 0;
-        }
-
-        valid |= (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                        "path lookup failed for (%s)", loc.path);
-                goto out;
-        }
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                                basename (name));
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1,"
-                                " returning EINVAL");
-                errno = EINVAL;
-                goto out;
-        }
-
-        op_ret = libgf_client_setattr (ctx, &loc, &stbuf, valid);
-out:
-        if (name)
-                FREE (name);
-        libgf_client_loc_wipe (&loc);
-        return op_ret;
-}
-
-int
-glusterfs_utime (const char *path, const struct utimbuf *buf)
-{
-        char                    vpath[PATH_MAX];
-        int                     op_ret = -1;
-        glusterfs_handle_t      h      = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, buf, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_utime (h, vpath, buf);
-out:
-        return op_ret;
-}
-
-static int32_t
-libgf_client_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *buf,
-                                struct iatt *preparent, struct iatt *postparent)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_mknod_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                inode, buf, preparent,
-                                                postparent);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-static int32_t
-libgf_client_mknod (libglusterfs_client_ctx_t *ctx, loc_t *loc, mode_t mode,
-                    dev_t rdev)
-{
-        int32_t                 op_ret = -1;
-        call_stub_t             *stub = NULL;
-        libgf_client_local_t    *local = NULL;
-        inode_t                 *inode = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, mknod, local, loc, mode, rdev);
-
-        op_ret = stub->args.mknod_cbk.op_ret;
-        errno = stub->args.mknod_cbk.op_errno;
-        if (op_ret == -1)
-                goto out;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, status %d, errno %d",
-                loc->path, op_ret, errno);
-        inode = stub->args.mknod_cbk.inode;
-        inode_link (inode, loc->parent, loc->name, &stub->args.mknod_cbk.buf);
-        libgf_transform_iattr (ctx, inode, &stub->args.mknod_cbk.buf);
-        inode_lookup (inode);
-
-        if (!libgf_alloc_inode_ctx (ctx, inode))
-                libgf_alloc_inode_ctx (ctx, inode);
-
-        libgf_update_iattr_cache (inode, LIBGF_UPDATE_STAT,
-                                        &stub->args.mknod_cbk.buf);
-
-out:
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-int
-glusterfs_glh_mknod(glusterfs_handle_t handle, const char *path, mode_t mode,
-                        dev_t dev)
-{
-        libglusterfs_client_ctx_t       *ctx = handle;
-        loc_t                           loc = {0, };
-        char                            *name = NULL;
-        int32_t                         op_ret = -1;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == 0) {
-                op_ret = -1;
-                errno = EEXIST;
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 0);
-        if (op_ret == -1) {
-                errno = ENOENT;
-                goto out;
-        }
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                                basename (name));
-	if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1, "
-                                " returning EINVAL");
-                errno = EINVAL;
-                goto out;
-        }
-
-        loc.inode = inode_new (ctx->itable);
-        op_ret = libgf_client_mknod (ctx, &loc, mode, dev);
-
-out:
-	libgf_client_loc_wipe (&loc);
-	if (name)
-                FREE (name);
-
-	return op_ret;
-}
-
-int
-glusterfs_mknod(const char *pathname, mode_t mode, dev_t dev)
-{
-        char                    vpath[PATH_MAX]; 
-        int                     op_ret = -1;
-        glusterfs_handle_t      h      = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, pathname, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", pathname);
-
-        h = libgf_resolved_path_handle (pathname, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-out:
-        return op_ret;
-}
-
-int
-glusterfs_glh_mkfifo (glusterfs_handle_t handle, const char *path, mode_t mode)
-{
-
-        libglusterfs_client_ctx_t       *ctx = handle;
-        loc_t                           loc = {0, };
-        char                            *name = NULL;
-        int32_t                         op_ret = -1;
-        dev_t                           dev = 0;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-        loc.path = libgf_resolve_path_light ((char *)path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Failed to resolve name");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == 0) {
-                op_ret = -1;
-                errno = EEXIST;
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 0);
-        if (op_ret == -1) {
-                errno = ENOENT;
-                goto out;
-        }
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                                basename (name));
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1, "
-                                "returning EINVAL");
-                errno = EINVAL;
-                goto out;
-        }
-
-        loc.inode = inode_new (ctx->itable);
-        op_ret = libgf_client_mknod (ctx, &loc, mode | S_IFIFO, dev);
-
-out:
-	libgf_client_loc_wipe (&loc);
-        if (name)
-                free (name);
-
-        return op_ret;
-}
-
-int
-glusterfs_mkfifo (const char *path, mode_t mode)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_mkfifo (h, vpath, mode);
-out:
-        return op_ret;
-}
-
-int32_t
-libgf_client_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         struct iatt *preparent, struct iatt *postparent)
-{
-        libgf_client_local_t    *local = frame->local;
-
-        local->reply_stub = fop_unlink_cbk_stub (frame, NULL, op_ret, op_errno,
-                                                 preparent, postparent);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int
-libgf_client_unlink (libglusterfs_client_ctx_t *ctx, loc_t *loc)
-{
-        int                             op_ret = -1;
-        libgf_client_local_t            *local = NULL;
-        call_stub_t                     *stub = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, unlink, local, loc);
-
-        op_ret = stub->args.unlink_cbk.op_ret;
-        errno = stub->args.unlink_cbk.op_errno;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", loc->path);
-        if (op_ret == -1)
-                goto out;
-
-        inode_unlink (loc->inode, loc->parent, loc->name);
-
-out:
-        call_stub_destroy (stub);
-        return op_ret;
-}
-
-int
-glusterfs_glh_unlink (glusterfs_handle_t handle, const char *path)
-{
-        int32_t                         op_ret = -1;
-        loc_t                           loc = {0, };
-        libglusterfs_client_ctx_t       *ctx = handle;
-        char                            *name = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                        "path lookup failed for (%s)", loc.path);
-                goto out;
-        }
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                                basename (name));
-	if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1, "
-                                " returning EINVAL");
-                errno = EINVAL;
-                goto out;
-        }
-
-        op_ret = libgf_client_unlink (ctx, &loc);
-
-out:
-        if (name)
-                FREE (name);
-        libgf_client_loc_wipe (&loc);
-        return op_ret;
-}
-
-int
-glusterfs_unlink (const char *path)
-{
-        char                    vpath[PATH_MAX];
-        int                     op_ret = -1;
-        glusterfs_handle_t      h      = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_unlink (h, vpath);
-
-out:
-        return op_ret;
-}
-
-static int32_t
-libgf_client_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *buf,
-                                struct iatt *preparent, struct iatt *postparent)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_symlink_cbk_stub (frame, NULL, op_ret,
-                                                  op_errno, inode, buf,
-                                                  preparent, postparent);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int32_t
-libgf_client_symlink (libglusterfs_client_ctx_t *ctx, const char *linkpath,
-                      loc_t *loc)
-{
-        int                     op_ret = -1;
-        libgf_client_local_t    *local = NULL;
-        call_stub_t             *stub = NULL;
-        inode_t                 *inode = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, symlink, local, linkpath, loc);
-
-        op_ret = stub->args.symlink_cbk.op_ret;
-        errno = stub->args.symlink_cbk.op_errno;
-        if (op_ret == -1)
-                goto out;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "target: %s, link: %s, status %d"
-                " errno %d", linkpath, loc->path, op_ret, errno);
-        inode = stub->args.symlink_cbk.inode;
-        inode_link (inode, loc->parent, loc->name,
-                        &stub->args.symlink_cbk.buf);
-        libgf_transform_iattr (ctx, inode, &stub->args.symlink_cbk.buf);
-        inode_lookup (inode);
-        if (!libgf_get_inode_ctx (inode))
-                libgf_alloc_inode_ctx (ctx, inode);
-
-        libgf_update_iattr_cache (inode, LIBGF_UPDATE_STAT,
-                                        &stub->args.symlink_cbk.buf);
-out:
-        call_stub_destroy (stub);
-        return op_ret;
-}
-
-int
-glusterfs_glh_symlink (glusterfs_handle_t handle, const char *oldpath,
-                       const char *newpath)
-{
-        int32_t                         op_ret = -1;
-        libglusterfs_client_ctx_t       *ctx = handle;
-        loc_t                           oldloc = {0, };
-        loc_t                           newloc = {0, };
-        char                            *oldname = NULL;
-        char                            *newname = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, newpath, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "target: %s, link: %s", oldpath,
-                newpath);
-        /* Old path does not need to be interpreted or looked up */
-        oldloc.path = strdup (oldpath);
-
-	newloc.path = strdup (newpath);
-        if (!newloc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-	op_ret = libgf_client_path_lookup (&newloc, ctx, 1);
-	if (op_ret == 0) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                        "new path (%s) already exists, "
-                        " returning EEXIST", newloc.path);
-                op_ret = -1;
-                errno = EEXIST;
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&newloc, ctx, 0);
-        if (op_ret == -1) {
-                errno = ENOENT;
-                goto out;
-        }
-
-        newloc.inode = inode_new (ctx->itable);
-        newname = strdup (newloc.path);
-        op_ret = libgf_client_loc_fill (&newloc, ctx, 0, newloc.parent->ino,
-                                                basename (newname));
-
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1, "
-                                "returning EINVAL");
-                errno = EINVAL;
-                goto out;
-        }
-
-        op_ret = libgf_client_symlink (ctx, oldpath, &newloc);
-
-out:
-        if (newname)
-                FREE (newname);
-
-        if (oldname)
-                FREE (oldname);
-        libgf_client_loc_wipe (&oldloc);
-        libgf_client_loc_wipe (&newloc);
-        return op_ret;
-}
-
-int
-glusterfs_symlink (const char *oldpath, const char *newpath)
-{
-        char                    vpath[PATH_MAX];
-        int                     op_ret = -1;
-        glusterfs_handle_t      h      = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, oldpath, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, newpath, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "target: %s, link: %s", oldpath,
-                newpath);
-
-        h = libgf_resolved_path_handle (newpath, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_symlink (h, oldpath, vpath);
-out:
-        return op_ret;
-}
-
-
-int32_t
-libgf_client_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                const char *path, struct iatt *sbuf)
-{
-        libgf_client_local_t    *local = frame->local;
-
-        local->reply_stub = fop_readlink_cbk_stub (frame, NULL, op_ret,
-                                                   op_errno, path, sbuf);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int32_t
-libgf_client_readlink (libglusterfs_client_ctx_t *ctx, loc_t *loc, char *buf,
-                       size_t bufsize)
-{
-        int                             op_ret = -1;
-        libgf_client_local_t            *local = NULL;
-        call_stub_t                     *stub = NULL;
-        size_t                           cpy_size = 0;
-
-        LIBGF_CLIENT_FOP (ctx, stub, readlink, local, loc, bufsize);
-
-        op_ret = stub->args.readlink_cbk.op_ret;
-        errno = stub->args.readlink_cbk.op_errno;
-
-        if (op_ret != -1) {
-                cpy_size = ((op_ret <= bufsize) ? op_ret : bufsize);
-                memcpy (buf, stub->args.readlink_cbk.buf, cpy_size);
-                op_ret = cpy_size;
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "link: %s, target: %s,"
-                        " status %d, errno %d", loc->path, buf, op_ret, errno);
-        } else
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "link: %s, status %d, "
-                        "errno %d", loc->path, op_ret, errno);
-
-        call_stub_destroy (stub);
-        return op_ret;
-}
-
-ssize_t
-glusterfs_glh_readlink (glusterfs_handle_t handle, const char *path, char *buf,
-                                size_t bufsize)
-{
-        int32_t                         op_ret = -1;
-        loc_t                           loc = {0, };
-        libglusterfs_client_ctx_t       *ctx = handle;
-        char                            *name = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-        if (bufsize < 0) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (bufsize == 0) {
-                op_ret = 0;
-                goto out;
-        }
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                        "path lookup failed for (%s)", loc.path);
-                goto out;
-        }
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                                basename (name));
-        if (op_ret == -1) {
-                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                "libgf_client_loc_fill returned -1, "
-                                "returning EINVAL");
-                errno = EINVAL;
-                goto out;
-        }
-
-        op_ret = libgf_client_readlink (ctx, &loc, buf, bufsize);
-
-out:
-        if (name)
-                FREE (name);
-
-        libgf_client_loc_wipe (&loc);
-        return op_ret;
-}
-
-ssize_t
-glusterfs_readlink (const char *path, char *buf, size_t bufsize)
-{
-        char                    vpath[PATH_MAX];
-        int                     op_ret = -1;
-        glusterfs_handle_t      h      = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, buf, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_readlink (h, vpath, buf, bufsize);
-out:
-        return op_ret;
-}
-
-char *
-glusterfs_glh_realpath (glusterfs_handle_t handle, const char *path,
-                                char *resolved_path)
-{
-        char                            *buf = NULL;
-        char                            *rpath = NULL;
-        char                            *start = NULL, *end = NULL;
-        char                            *dest = NULL;
-        libglusterfs_client_ctx_t       *ctx = handle;
-        long int                        path_max = 0;
-        char                            *ptr = NULL;
-        struct stat                     stbuf = {0, };
-        long int                        new_size = 0;
-        char                            *new_rpath = NULL;
-        int                             dest_offset = 0;
-        char                            *rpath_limit = 0;
-        int                             ret = 0, num_links = 0;
-        char                            *vpath = NULL, *tmppath = NULL;
-        char                             absolute_path[PATH_MAX];
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-#ifdef PATH_MAX
-        path_max = PATH_MAX;
-#else
-        path_max = pathconf (path, _PC_PATH_MAX);
-        if (path_max <= 0) {
-                path_max = 1024;
-        }
-#endif
-
-        if (resolved_path == NULL) {
-                rpath = CALLOC (1, path_max);
-                if (rpath == NULL) {
-                        errno = ENOMEM;
-                        goto out;
-                }
-        } else {
-                rpath = resolved_path;
-        }
-
-        rpath_limit = rpath + path_max;
-
-        if (path[0] == '/') {
-                rpath[0] = '/';
-                dest = rpath + 1;
-        } else {
-                /*
-                   FIXME: can $CWD be a valid path on glusterfs server? hence is
-                   it better to handle this case or just return EINVAL for
-                   relative paths?
-                */
-                ptr = getcwd (rpath, path_max);
-                if (ptr == NULL) {
-                        goto err;
-                }
-                dest = rpath + strlen (rpath);
-        }
-
-        for (start = end = (char *)path; *end; start = end) {
-                if (dest[-1] != '/') {
-                        *dest++ = '/';
-                }
-
-                while (*start == '/') {
-                        start++;
-                }
-
-                for (end = start; *end && *end != '/'; end++);
-
-                if ((end - start) == 0) {
-                        break;
-                }
-
-                if ((end - start == 1) && (start[0] == '.')) {
-                        /* do nothing */
-                } else if (((end - start) == 2) && (start[0] == '.')
-                           && (start[1] == '.')) {
-                        if (dest > rpath + 1) {
-                                while (--dest[-1] != '/');
-                        }
-                } else {
-                        if ((dest + (end - start + 1)) >= rpath_limit) {
-                                if (resolved_path == NULL) {
-                                        errno = ENAMETOOLONG;
-                                        if (dest > rpath + 1)
-                                                dest--;
-                                        *dest = '\0';
-                                        goto err;
-                                }
-
-                                dest_offset = dest - rpath;
-                                new_size = rpath_limit - rpath;
-                                if ((end - start + 1) > path_max) {
-                                        new_size = (end - start + 1);
-                                } else {
-                                        new_size = path_max;
-                                }
-
-                                new_rpath = realloc (rpath, new_size);
-                                if (new_rpath == NULL) {
-                                        goto err;
-                                }
-
-
-                                dest = new_rpath + dest_offset;
-                                rpath = new_rpath;
-                                rpath_limit = rpath + new_size;
-                        }
-
-                        memcpy (dest, start, end - start);
-                        dest +=  end - start;
-                        *dest = '\0';
-
-                        ret = glusterfs_glh_lstat (handle, rpath, &stbuf);
-                        if (ret == -1) {
-                                gf_log ("libglusterfsclient", GF_LOG_ERROR,
-                                        "glusterfs_glh_stat returned -1 for"
-                                        " path (%s):(%s)", rpath,
-                                        strerror (errno));
-                                goto err;
-                        }
-
-                        if (S_ISLNK (stbuf.st_mode)) {
-                                buf = calloc (1, path_max);
-                                if (buf == NULL) {
-                                        errno = ENOMEM;
-                                        goto err;
-                                }
-
-                                if (++num_links > MAXSYMLINKS)
-                                {
-                                        errno = ELOOP;
-                                        FREE (buf);
-                                        goto err;
-                                }
-
-                                ret = glusterfs_glh_readlink (handle, rpath,
-                                                              buf,
-                                                              path_max - 1);
-                                if (ret < 0) {
-                                        gf_log ("libglusterfsclient",
-                                                GF_LOG_ERROR,
-                                                "glusterfs_readlink returned %d"
-                                                " for path (%s):(%s)",
-                                                ret, rpath, strerror (errno));
-                                        FREE (buf);
-                                        goto err;
-                                }
-                                buf[ret] = '\0';
-
-                                if (buf[0] != '/') {
-                                        tmppath = strdup (rpath);
-                                        tmppath = dirname (tmppath);
-                                        sprintf (absolute_path, "%s/%s",
-                                                 tmppath, buf);
-                                        FREE (buf);
-                                        buf = libgf_resolve_path_light ((char *)absolute_path);
-                                        FREE (tmppath);
-                                }
-
-                                rpath = glusterfs_glh_realpath (handle, buf,
-                                                                rpath);
-                                FREE (buf);
-                                if (rpath == NULL) {
-                                        goto out;
-                                }
-                                dest = rpath + strlen (rpath);
-
-                        } else if (!S_ISDIR (stbuf.st_mode) && *end != '\0') {
-                                errno = ENOTDIR;
-                                goto err;
-                        }
-                }
-        }
-        if (dest > rpath + 1 && dest[-1] == '/')
-                --dest;
-        *dest = '\0';
-
-out:
-        if (vpath)
-                FREE (vpath);
-        return rpath;
-
-err:
-        if (vpath)
-                FREE (vpath);
-        if (resolved_path == NULL) {
-                FREE (rpath);
-        }
-
-        return NULL;
-}
-
-char *
-glusterfs_realpath (const char *path, char *resolved_path)
-{
-        char                    *res   = NULL;
-        char                     vpath[PATH_MAX];
-        glusterfs_handle_t       h     = NULL;
-        char                    *realp = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        realp = CALLOC (PATH_MAX, sizeof (char));
-        if (!realp)
-                goto out;
-
-        libgf_vmp_search_vmp (h, realp, PATH_MAX);
-        res = glusterfs_glh_realpath (h, vpath, resolved_path);
-        if (!res)
-                goto out;
-
-        /* This copy is needed to ensure that when we return the real resolved
-         * path, we return a path that accounts for the app's view of the
-         * path, i.e. it starts with the VMP, in case this is an absolute path.
-         */
-        if (libgf_path_absolute (path)) {
-                strcat (realp, resolved_path);
-                strcpy (resolved_path, realp);
-        }
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, resolved %s", path,
-                resolved_path);
-out:
-        if (realp)
-                FREE (realp);
-
-        return res;
-}
-
-int
-glusterfs_glh_remove (glusterfs_handle_t handle, const char *path)
-{
-        loc_t                           loc = {0, };
-        int                             op_ret = -1;
-        libglusterfs_client_ctx_t       *ctx = handle;
-        char                            *name = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, handle, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", path);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-        op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-        if (op_ret == -1)
-                goto out;
-
-        name = strdup (loc.path);
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino,
-                                        basename (name));
-        if (op_ret == -1)
-                goto out;
-
-        if (IA_ISDIR (loc.inode->ia_type))
-                op_ret = libgf_client_rmdir (ctx, &loc);
-        else
-                op_ret = libgf_client_unlink (ctx, &loc);
-
-out:
-        if (name)
-                FREE (name);
-        return op_ret;
-
-}
-
-int
-glusterfs_remove(const char *pathname)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, pathname, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s", pathname);
-
-        h = libgf_resolved_path_handle (pathname, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_remove (h, vpath);
-out:
-        return op_ret;
-}
-
-void
-glusterfs_rewinddir (glusterfs_dir_t dirfd)
-{
-        libglusterfs_client_fd_ctx_t    *fd_ctx = NULL;
-
-        fd_ctx = libgf_get_fd_ctx ((fd_t *)dirfd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-                goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                fd_ctx->offset = 0;
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Offset: %"PRIu64,
-                        fd_ctx->offset);
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-out:
-        return;
-}
-
-void
-glusterfs_seekdir (glusterfs_dir_t dirfd, off_t offset)
-{
-        libglusterfs_client_fd_ctx_t    *fd_ctx = NULL;
-
-        fd_ctx = libgf_get_fd_ctx ((fd_t *)dirfd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                fd_ctx->offset = offset;
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Offset: %"PRIu64,
-                        fd_ctx->offset);
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-out:
-        return;
-}
-
-off_t
-glusterfs_telldir (glusterfs_dir_t dirfd)
-{
-        libglusterfs_client_fd_ctx_t    *fd_ctx = NULL;
-	off_t                           off = -1;
-
-        fd_ctx = libgf_get_fd_ctx ((fd_t *)dirfd);
-        if (!fd_ctx) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "No fd context present");
-                errno = EBADF;
-                goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                off = fd_ctx->offset;
-                gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "Offset: %"PRIu64,
-                        fd_ctx->offset);
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-out:
-        return off;
-}
-
-struct libgf_client_sendfile_data {
-        int                reads_sent;
-        int                reads_completed;
-        int                out_fd;
-        int32_t            op_ret;
-        int32_t            op_errno;
-        pthread_mutex_t    lock;
-        pthread_cond_t     cond;
-};
-
-int
-libgf_client_sendfile_read_cbk (int op_ret, int op_errno,
-                                glusterfs_iobuf_t *buf, void *cbk_data)
-{
-        struct libgf_client_sendfile_data *sendfile_data = cbk_data;
-        int                                bytes = 0;
-
-        if (op_ret > 0) {
-                bytes = writev (sendfile_data->out_fd, buf->vector, buf->count);
-                if (bytes != op_ret) {
-                        op_ret = -1;
-                        op_errno = errno;
-                }
-
-                glusterfs_free (buf);
-        }
-
-        pthread_mutex_lock (&sendfile_data->lock);
-        {
-                if (sendfile_data->op_ret != -1) {
-                        if (op_ret == -1) {
-                                sendfile_data->op_ret = -1;
-                                sendfile_data->op_errno = op_errno;
-                        } else {
-                                sendfile_data->op_ret += op_ret;
-                        }
-                }
-
-                sendfile_data->reads_completed++;
-
-                if (sendfile_data->reads_completed
-                    == sendfile_data->reads_sent) {
-                        pthread_cond_broadcast (&sendfile_data->cond);
-                } 
-        }
-        pthread_mutex_unlock (&sendfile_data->lock);
-
-        return 0;
-}
-
-
-ssize_t
-glusterfs_sendfile (int out_fd, glusterfs_file_t in_fd, off_t *offset,
-                    size_t count)
-{
-        ssize_t                           ret = -1;
-        struct libgf_client_sendfile_data cbk_data = {0, };
-        off_t                             off = -1;
-        size_t                            size = 0;
-        int                               flags = 0;
-        int                               non_block = 0;
-
-        
-        pthread_mutex_init (&cbk_data.lock, NULL);
-        pthread_cond_init (&cbk_data.cond, NULL);
-        cbk_data.out_fd = out_fd;
-        
-        if (offset) {
-                off = *offset;
-        }
-
-        flags = fcntl (out_fd, F_GETFL);
-
-        if (flags != -1) {
-                non_block = flags & O_NONBLOCK;
-
-                if (non_block) {
-                        ret = fcntl (out_fd, F_SETFL, flags & ~O_NONBLOCK);
-                }
-        }
-
-        while (count != 0) {
-                /* 
-                 * FIXME: what's the optimal size for reads and writes?
-                 */
-                size = (count > LIBGF_SENDFILE_BLOCK_SIZE) ? 
-                        LIBGF_SENDFILE_BLOCK_SIZE : count;
-
-                /* 
-                 * we don't wait for reply to previous read, we just send all
-                 * reads in a single go.
-                 */
-                ret = glusterfs_read_async (in_fd, size, off,
-                                            libgf_client_sendfile_read_cbk,
-                                            &cbk_data);
-                if (ret == -1) {
-                        break;
-                }
-
-                pthread_mutex_lock (&cbk_data.lock);
-                {
-                        cbk_data.reads_sent++;
-                }
-                pthread_mutex_unlock (&cbk_data.lock);
-
-                if (offset) {
-                        off += size;
-                }
-
-                count -= size;
-        }
-
-        pthread_mutex_lock (&cbk_data.lock);
-        {
-                /* 
-                 * if we've not received replies to all the reads we've sent,
-                 * wait for them
-                 */
-                if (cbk_data.reads_sent > cbk_data.reads_completed) {
-                        pthread_cond_wait (&cbk_data.cond,
-                                           &cbk_data.lock);
-                }
-        }
-        pthread_mutex_unlock (&cbk_data.lock);
-
-        if (offset != NULL) {
-                *offset = off;
-        }
- 
-        /* if we were able to stack_wind all the reads */
-
-        if (ret == 0) {
-                ret = cbk_data.op_ret;
-                errno = cbk_data.op_errno;
-        }
-
-        if (non_block) {
-                fcntl (out_fd, F_SETFL, flags);
-        }
-
-        return ret;
-}
-
-
-static int32_t
-libgf_client_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct flock *lock)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_lk_cbk_stub (frame, NULL, op_ret, op_errno,
-                                             lock);
-
-        LIBGF_REPLY_NOTIFY (local);
-
-        return 0;
-}
-
-
-int
-libgf_client_lk (libglusterfs_client_ctx_t *ctx, fd_t *fd, int cmd,
-                 struct flock *lock)
-{
-        call_stub_t          *stub = NULL;
-        int32_t               op_ret;
-        libgf_client_local_t *local = NULL;
-        
-        LIBGF_CLIENT_FOP(ctx, stub, lk, local, fd, cmd, lock);
-
-        op_ret = stub->args.lk_cbk.op_ret;
-        errno = stub->args.lk_cbk.op_errno;
-        if (op_ret == 0) {
-                *lock = stub->args.lk_cbk.lock;
-        }
-
-	call_stub_destroy (stub);
-        return op_ret;
-}
-
-
-int
-glusterfs_fcntl (glusterfs_file_t fd, int cmd, ...)
-{
-        int                           ret = -1;
-        struct flock                 *lock = NULL;
-        va_list                       ap;
-        libglusterfs_client_ctx_t    *ctx = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, fd, out);
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		goto out;
-        }
-
-        ctx = fd_ctx->ctx;
-
-        switch (cmd) {
-        case F_SETLK:
-        case F_SETLKW:
-        case F_GETLK:
-#if F_SETLK != F_SETLK64
-        case F_SETLK64:
-#endif
-#if F_SETLKW != F_SETLKW64
-        case F_SETLKW64:
-#endif
-#if F_GETLK != F_GETLK64
-        case F_GETLK64:
-#endif
-                va_start (ap, cmd);
-                lock = va_arg (ap, struct flock *);
-                va_end (ap);
-
-                if (!lock) {
-                        errno = EINVAL;
-                        goto out;
-                }
-
-                ret = libgf_client_lk (ctx, fd, cmd, lock); 
-                break;
-
-        default:
-                errno = EINVAL;
-                break;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-libgf_client_chdir (const char *path)
-{
-        int                op_ret            = 0;
-        uint32_t           resulting_cwd_len = 0;
-
-        pthread_mutex_lock (&cwdlock);
-        {
-                if (!libgf_path_absolute (path)) {
-                        resulting_cwd_len = strlen (path) + strlen (cwd)
-                                + ((path[strlen (path) - 1] == '/')
-                                   ? 0 : 1) + 1;
-
-                        if (resulting_cwd_len > PATH_MAX) {
-                                op_ret = -1;
-                                errno = ENAMETOOLONG;
-                                goto unlock;
-                        }
-                        strcat (cwd, path);
-                } else {
-                        resulting_cwd_len = strlen (path)
-                                + ((path[strlen (path) - 1] == '/')
-                                   ? 0 : 1) + 1;
-                                
-                        if (resulting_cwd_len > PATH_MAX) {
-                                op_ret = -1;
-                                errno = ENAMETOOLONG;
-                                goto unlock;
-                        }
-
-                        strcpy (cwd, path);
-                }
-
-                if (cwd[strlen (cwd) - 1] != '/') {
-                        strcat (cwd, "/");
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&cwdlock);
-
-        return op_ret;
-}
-
-
-int
-glusterfs_fchdir (glusterfs_file_t fd)
-{
-        int                           op_ret = -1;
-        char                          vpath[PATH_MAX];
-        char                          vmp[PATH_MAX]; 
-        char                         *res    = NULL;
-        libglusterfs_client_fd_ctx_t *fd_ctx = NULL;
-        glusterfs_handle_t            handle = NULL;
-        
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, fd, out);
-
-        /* FIXME: there is a race-condition between glusterfs_fchdir and
-           glusterfs_close. If two threads of application call glusterfs_fchdir
-           and glusterfs_close on the same fd, there is a possibility of
-           glusterfs_fchdir accessing freed memory of fd_ctx.
-        */
-
-        fd_ctx = libgf_get_fd_ctx (fd);
-        if (!fd_ctx) {
-                errno = EBADF;
-		goto out;
-        }
-
-        pthread_mutex_lock (&fd_ctx->lock);
-        {
-                handle = fd_ctx->ctx;
-                strcpy (vpath, fd_ctx->vpath);
-        }
-        pthread_mutex_unlock (&fd_ctx->lock);
-
-        if (vpath[0] == '\0') {
-                errno = ENOTDIR;
-                goto out;
-        }
-
-        res = libgf_vmp_search_vmp (handle, vmp, PATH_MAX);
-        if (res == NULL) {
-                errno = EBADF;
-                goto out;
-        }
-
-        /* both vmp and vpath are terminated with '/'. Also path starts with a
-           '/'. Hence the extra '/' amounts to NULL character at the end of the
-           string.
-        */
-        if ((strlen (vmp) + strlen (vpath)) > PATH_MAX) {
-                errno = ENAMETOOLONG;
-                goto out;
-        }
-
-        pthread_mutex_lock (&cwdlock);
-        {
-                strcpy (cwd, vmp);
-                res = vpath;
-                if (res[0] == '/') {
-                        res++;
-                }
-
-                strcat (cwd, res);
-        }
-        pthread_mutex_unlock (&cwdlock);
-
-        op_ret = 0; 
-out:
-        return op_ret;
-}
-
-
-int
-glusterfs_chdir (const char *path)
-{
-        int32_t            op_ret            = -1;
-        glusterfs_handle_t handle            = NULL;
-        loc_t              loc               = {0, };
-        char               vpath[PATH_MAX]; 
-
-        handle = libgf_resolved_path_handle (path, vpath);
-
-        if (handle != NULL)  {
-                loc.path = strdup (vpath);
-                if (!loc.path) {
-                        gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "Path compaction "
-                                "failed");
-                        goto out;
-                }
-
-                op_ret = libgf_client_path_lookup (&loc, handle, 0);
-        }
-
-        if ((handle == NULL) || (op_ret == 0)) {
-                op_ret = libgf_client_chdir (path);
-        }
-
-out:
-        return op_ret;
-}
-
-
-char *
-glusterfs_getcwd (char *buf, size_t size)
-{
-        char              *res      = NULL;
-        size_t             len      = 0; 
-        loc_t              loc      = {0, };
-        glusterfs_handle_t handle   = NULL;
-        char               vpath[PATH_MAX];
-        int32_t            op_ret   = 0; 
-
-        pthread_mutex_lock (&cwdlock);
-        {
-                if (!cwd_inited) {
-                        errno = ENODEV;
-                        goto unlock;
-                }
-
-                if (buf == NULL) {
-                        buf = CALLOC (1, len);
-                        if (buf == NULL) {
-                                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR,
-                                        "out of memory");
-                                goto unlock;
-                        }
-                } else {
-                        if (size == 0) {
-                                errno = EINVAL;
-                                goto unlock;
-                        }
-
-                        if (len > size) {
-                                errno = ERANGE;
-                                goto unlock;
-                        }
-                }
-
-                strcpy (buf, cwd);
-                res = buf; 
-        }
-unlock:
-        pthread_mutex_unlock (&cwdlock);
-
-        if (res != NULL) {
-                handle = libgf_resolved_path_handle (res, vpath);
-
-                if (handle != NULL)  {
-                        loc.path = strdup (vpath);
-                        if (loc.path == NULL) {
-                                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR,
-                                        "strdup failed");
-                        } else {
-                                op_ret = libgf_client_path_lookup (&loc, handle,
-                                                                   0);
-                                if (op_ret == -1) {
-                                        res = NULL;
-                                }
-                        }
-                }
-        }
-
-        return res;
-}
-
-int32_t
-libgf_client_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *prebuf, struct iatt *postbuf)
-{
-        libgf_client_local_t *local = frame->local;
-
-        local->reply_stub = fop_truncate_cbk_stub (frame, NULL, op_ret,
-                                                   op_errno, prebuf, postbuf);
-
-        LIBGF_REPLY_NOTIFY (local);
-        return 0;
-}
-
-int32_t 
-libgf_client_truncate (libglusterfs_client_ctx_t *ctx, 
-                       loc_t *loc, off_t length)
-{
-        call_stub_t *stub = NULL;
-        int32_t op_ret = 0;
-        libgf_client_local_t *local = NULL;
-
-        LIBGF_CLIENT_FOP (ctx, stub, truncate, local, loc, length);
- 
-        op_ret = stub->args.truncate_cbk.op_ret;
-        errno = stub->args.truncate_cbk.op_errno;
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path %s, status %d, errno %d",
-                loc->path, op_ret, errno);
-
-        if (op_ret == -1) {
-                goto out;
-        }
-
-        libgf_transform_iattr (ctx, loc->inode,
-                               &stub->args.truncate_cbk.postbuf);
-
-        libgf_update_iattr_cache (loc->inode, LIBGF_UPDATE_STAT,
-                                  &stub->args.truncate_cbk.postbuf);
-	call_stub_destroy (stub);
-
-out:
-        return op_ret;
-}
-
-int
-glusterfs_glh_truncate (glusterfs_handle_t handle, const char *path,
-                        off_t length)
-{
-        int32_t op_ret = -1;
-        loc_t loc = {0, };
-        libglusterfs_client_ctx_t *ctx = handle;
-	char *name = NULL, *pathname = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, ctx, out);
-        GF_VALIDATE_ABSOLUTE_PATH_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        loc.path = strdup (path);
-        if (!loc.path) {
-                gf_log (LIBGF_XL_NAME, GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
-
-	op_ret = libgf_client_path_lookup (&loc, ctx, 1);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient", GF_LOG_ERROR,
-			"path lookup failed for (%s)", loc.path);
-		goto out;
-	}
-
-	pathname = strdup (loc.path);
-	name = basename (pathname);
-
-        op_ret = libgf_client_loc_fill (&loc, ctx, 0, loc.parent->ino, name);
-	if (op_ret == -1) {
-		gf_log ("libglusterfsclient",
-			GF_LOG_ERROR,
-			"libgf_client_loc_fill returned -1, returning EINVAL");
-		errno = EINVAL;
-		goto out;
-	}
-
-        op_ret = libgf_client_truncate (ctx, &loc, length);
-
-out:
-        libgf_client_loc_wipe (&loc);
-
-        return op_ret;
-}
-
-int
-glusterfs_truncate (const char *path, off_t length)
-{
-        int                     op_ret = -1;
-        char                    vpath[PATH_MAX];
-        glusterfs_handle_t      h = NULL;
-
-        GF_VALIDATE_OR_GOTO (LIBGF_XL_NAME, path, out);
-
-        gf_log (LIBGF_XL_NAME, GF_LOG_DEBUG, "path:%s length:%"PRIu64, path,
-                length);
-        h = libgf_resolved_path_handle (path, vpath);
-        if (!h) {
-                errno = ENODEV;
-                goto out;
-        }
-
-        op_ret = glusterfs_glh_truncate (h, vpath, length);
-out:
-        return op_ret;
-}
-
-static struct xlator_fops libgf_client_fops = {
-};
-
-static struct xlator_cbks libgf_client_cbks = {
-        .forget      = libgf_client_forget,
-	.release     = libgf_client_release,
-	.releasedir  = libgf_client_releasedir,
-};
-
-static inline xlator_t *
-libglusterfs_graph (xlator_t *graph)
-{
-        int       ret = 0;
-        xlator_t *top = NULL;
-        xlator_list_t *xlchild, *xlparent;
-
-        top = CALLOC (1, sizeof (*top));
-        ERR_ABORT (top);
-
-        xlchild = CALLOC (1, sizeof(*xlchild));
-        ERR_ABORT (xlchild);
-        xlchild->xlator = graph;
-        top->children = xlchild;
-        top->ctx = graph->ctx;
-        top->next = graph;
-        top->name = strdup (LIBGF_XL_NAME);
-
-        xlparent = CALLOC (1, sizeof(*xlparent));
-        xlparent->xlator = top;
-        graph->parents = xlparent;
-        ret = asprintf (&top->type, LIBGF_XL_NAME);
-        if (-1 == ret) {
-                fprintf (stderr, "failed to set the top xl's type");
-        }
-
-        top->init = libgf_client_init;
-        top->fops = &libgf_client_fops;
-        top->mops = &libgf_client_mops;
-        top->cbks = &libgf_client_cbks; 
-        top->notify = libgf_client_notify;
-        top->fini = libgf_client_fini;
-        //  fill_defaults (top);
-
-        return top;
-}
diff --git a/libglusterfsclient/src/libglusterfsclient.h b/libglusterfsclient/src/libglusterfsclient.h
deleted file mode 100755
index e029e839d..000000000
--- a/libglusterfsclient/src/libglusterfsclient.h
+++ /dev/null
@@ -1,1372 +0,0 @@
-/*
-  Copyright (c) 2008 - 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _LIBGLUSTERFSCLIENT_H
-#define _LIBGLUSTERFSCLIENT_H
-
-#ifndef __BEGIN_DECLS
-#ifdef __cplusplus
-#define __BEGIN_DECLS extern "C" {
-#else
-#define __BEGIN_DECLS
-#endif
-#endif
-
-#ifndef __END_DECLS
-#ifdef __cplusplus
-#define __END_DECLS }
-#else
-#define __END_DECLS
-#endif
-#endif
-
-
-__BEGIN_DECLS
-
-#include <stdio.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <dirent.h>
-#include <sys/statfs.h>
-#include <sys/statvfs.h>
-#include <utime.h>
-#include <sys/time.h>
-#include <stdint.h>
-
-typedef struct {
-        struct iovec *vector;
-        int           count;
-        void         *iobref;
-        void         *dictref;
-} glusterfs_iobuf_t;
-
-
-typedef
-int (*glusterfs_readv_cbk_t) (int op_ret, int op_errno, glusterfs_iobuf_t *buf,
-			      void *cbk_data);
-
-typedef
-int (*glusterfs_write_cbk_t) (int op_ret, int op_errno, void *cbk_data);
-
-typedef
-int (*glusterfs_get_cbk_t) (int op_ret, int op_errno, glusterfs_iobuf_t *buf,
-			    struct stat *stbuf, void *cbk_data);
-
-
-/* Data Interface
- * The first section describes the data structures required for
- * using libglusterfsclient.
- */
-
-/* This structure needs to be filled up and
- * passed to te glusterfs_init function which uses
- * the params passed herein to initialize a glusterfs
- * client context and then connect to a glusterfs server.
- */
-typedef struct {
-        char          *logfile;         /* Path to the file which will store
-                                           the log.
-                                           */
-        char          *loglevel;        /* The log level required for
-                                           reporting various events within
-                                           libglusterfsclient.
-                                           */
-        struct {
-                char  *specfile;        /* Users can either open a volume or
-                                           specfile and assign the pointer to
-                                           specfp, or just refer to the volume
-                                           /spec file path in specfile.
-                                           */
-                FILE  *specfp;
-        };
-        char          *volume_name;     /* The volume file could describe many
-                                           volumes but the specific volume
-                                           within that file is chosen by
-                                           specifying the volume name here.
-                                           */
-        unsigned long  lookup_timeout;  /* libglusterclient provides the inode
-                                           numbers to be cached by the library.
-                                           The duration for which these are
-                                           cached are defined by lookup_timeout
-                                           . In Seconds.
-                                           */
-        unsigned long  stat_timeout;    /* The file attributes received from
-                                           a stat syscall can also be cached
-                                           for the duration specified in this
-                                           member. In Seconds.
-                                           */
-} glusterfs_init_params_t;
-
-
-
-/* This is the handle returned by glusterfs_init
- * once the initialization is complete.
- * Users should treat this as an opaque handle.
- */
-typedef void * glusterfs_handle_t;
-
-
-
-/* These identifiers are used as handles for files and dirs.
- * Users of libglusterfsclient should not in anyway try to interpret
- * the actual structures these will point to.
- */
-typedef void * glusterfs_file_t;
-typedef void * glusterfs_dir_t;
-
-
-/* Function Call Interface */
-/* libglusterfsclient initialization function.
- * @ctx : the structure described above filled with required values.
- * @fakefsid: User generated fsid to be used to identify this
- * volume.
- *
- * Returns NULL on failure and the non-NULL pointer on success.
- * On failure, the error description might be present in the logfile
- * depending on the log level.
- */
-glusterfs_handle_t
-glusterfs_init (glusterfs_init_params_t *ctx, uint32_t fakefsid);
-
-
-
-/* Used to destroy a glusterfs client context and the
- * connection to the glusterfs server.
- *
- * @handle      : The glusterfs handle returned by glusterfs_init.
- */
-int
-glusterfs_fini (glusterfs_handle_t handle);
-
-
-
-/* libglusterfs client provides two interfaces.
- * 1. handle-based interface
- * Functions that comprise the handle-based interface accept the
- * glusterfs_handle_t as the first argument. It specifies the
- * glusterfs client context over which to perform the operation.
- *
- * 2. Virtual Mount Point based interface:
- * Functions that do not require a handle to be given in order to
- * identify which client context to operate on. This interface
- * internally determines the corresponding client context for the
- * given path. The down-side is that a virtual mount point (VMP) needs to be
- * registered with the library. A VMP is just a string that maps to a
- * glusterfs_handle_t. The advantage of a VMP based interface is that
- * a user program using multiple client contexts does not need to
- * maintain its own mapping between paths and the corresponding
- * handles.
- */
-
-
-
-/* glusterfs_mount is the function that allows users to register a VMP
- * along with the parameters, which will be used to initialize a
- * context. Applications calling glusterfs_mount do not need to
- * initialized a context using the glusterfs_init interface.
- *
- * @vmp         : The virtual mount point.
- * @ipars       : Initialization parameters populated as described
- *              earlier.
- *
- * Returns 0 on success, and -1 on failure.
- */
-int
-glusterfs_mount (char *vmp, glusterfs_init_params_t *ipars);
-
-
-
-/* glusterfs_umount is the VMP equivalent of glusterfs_fini.
- *
- * @vmp         : The VMP which was initialized using glusterfs_mount.
- *
- * Returns 0 on sucess, and -1 on failure.
- */
-int
-glusterfs_umount (char *vmp);
-
-
-/* glusterfs_umount_all unmounts all the mounts */
-int
-glusterfs_umount_all (void);
-
-
-/* For smaller files, application can use just
- * glusterfs_get/glusterfs_get_async
- * to read the whole content. Limit of the file-sizes to be read in
- * glusterfs_get/glusterfs_get_async is passed in the size argument
- */
-
-/* glusterfs_glh_get:
- * @handle : glusterfs handle
- * @path   : path to be looked upon
- * @size   : upper limit of file-sizes to be read in lookup
- * @stbuf  : attribute buffer
- */
-
-int
-glusterfs_glh_get (glusterfs_handle_t handle, const char *path, void *buf,
-	       size_t size, struct stat *stbuf);
-
-int
-glusterfs_get (const char *path, void *buf, size_t size, struct stat *stbuf);
-
-int
-glusterfs_get_async (glusterfs_handle_t handle, const char *path, size_t size,
-		     glusterfs_get_cbk_t cbk, void *cbk_data);
-
-
-
-/* Opens a file. Corresponds to the open syscall.
- *
- * @handle      : Handle returned from glusterfs_init
- * @path        : Path to the file or directory on the glusterfs
- *              export. Must be absolute to the export on the server.
- * @flags       : flags to control open behaviour.
- * @...         : The mode_t argument that defines the mode for a new
- *              file, in case a new file is being created using the
- *              O_CREAT flag in @flags.
- *
- * Returns a non-NULL handle on success. NULL on failure and sets
- * errno accordingly.
- */
-glusterfs_file_t
-glusterfs_glh_open (glusterfs_handle_t handle, const char *path, int flags,
-                        ...);
-
-
-/* Opens a file without having to specify a handle.
- *
- * @path        : Path to the file to open in the glusterfs export.
- *              The path to the file in glusterfs export must be
- *              pre-fixed with the VMP string registered with
- *              glusterfs_mount.
- * @flags       : flags to control open behaviour.
- * @...         : The mode_t argument that defines the mode for a new
- *              file, in case a new file is being created using the
- *              O_CREAT flag in @flags.
- *
- * Returns 0 on success, -1 on failure with errno set accordingly.
- */
-glusterfs_file_t
-glusterfs_open (const char *path, int flags, ...);
-
-
-
-/* Creates a file. Corresponds to the creat syscall.
- *
- * @handle      : Handle returned from glusterfs_init
- * @path        : Path to the file that needs to be created in the
- *              glusterfs export.
- * @mode        : File creation mode.
- *
- * Returns the file handle on success. NULL on error with errno set as
- * required.
- */
-glusterfs_file_t
-glusterfs_glh_creat (glusterfs_handle_t handle, const char *path, mode_t mode);
-
-
-
-/* VMP-based creat.
- * @path        : Path to the file to be created. Must be
- *              pre-prepended with the VMP string registered with
- *              glusterfs_mount.
- * @mode        : File creation mode.
- *
- * Returns file handle on success. NULL handle on error with errno set
- * accordingly.
- */
-glusterfs_file_t
-glusterfs_creat (const char *path, mode_t mode);
-
-
-
-/* Close the file identified by the handle.
- *
- * @fd          : Closes the file.
- *
- * Returns 0 on success, -1 on error with errno set accordingly.
- */
-int
-glusterfs_close (glusterfs_file_t fd);
-
-
-
-/* Get struct stat for the file in path.
- *
- * @handle      : The handle that identifies a glusterfs client
- *              context.
- * @path        : The file for which we need to get struct stat.
- * @stbuf       : The buffer into which the file's stat is copied.
- *
- * Returns 0 on success and -1 on error with errno set accordingly.
- */
-int
-glusterfs_glh_stat (glusterfs_handle_t handle, const char *path,
-                        struct stat *stbuf);
-
-
-/* Get struct stat for file in path.
- *
- * @path        : The file for which struct stat is required.
- * @sbuf        : The buffer into which the stat structure is copied.
- *
- * Returns 0 on success and -1 on error with errno set accordingly.
- */
-int
-glusterfs_stat (const char *path, struct stat *buf);
-
-
-
-/* Gets stat struct for the file.
- *
- * @handle      : The handle identifying a glusterfs client context.
- * @path        : Path to the file for which stat structure is
- *              required. If path is a symlink, the symlink is
- *              interpreted and the stat structure returned for the
- *              target of the link.
- * @buf         : The buffer into which the stat structure is copied.
- *
- * Returns 0 on success and -1 on error with errno set accordingly.
- */
-int
-glusterfs_glh_lstat (glusterfs_handle_t handle, const char *path,
-                        struct stat *buf);
-
-
-
-/* Gets stat struct for a file.
- *
- * @path        : The file to get the struct stat for.
- * @buf         : The receiving struct stat buffer.
- *
- * Returns 0 on success and -1 on error with errno set accordingly.
- */
-int
-glusterfs_lstat (const char *path, struct stat *buf);
-
-
-
-/* Get stat structure for a file.
- *
- * @fd          : The file handle identifying a file on the glusterfs
- *              server.
- * @stbuf       : The buffer into which the stat data is copied.
- *
- * Returns 0 on success and -1 on error with errno set accordingly.
- */
-int
-glusterfs_fstat (glusterfs_file_t fd, struct stat *stbuf);
-
-int
-glusterfs_glh_setxattr (glusterfs_handle_t handle, const char *path,
-                                const char *name, const void *value,
-                                size_t size, int flags);
-
-int
-glusterfs_glh_lsetxattr (glusterfs_handle_t handle, const char *path,
-                         const char *name, const void *value, size_t size,
-                         int flags);
-
-int
-glusterfs_setxattr (const char *path, const char *name, const void *value,
-                        size_t size, int flags);
-
-int
-glusterfs_lsetxattr (const char *path, const char *name, const void *value,
-                     size_t size, int flags);
-
-int
-glusterfs_fsetxattr (glusterfs_file_t fd, const char *name, const void *value,
-                     size_t size, int flags);
-
-ssize_t
-glusterfs_glh_getxattr (glusterfs_handle_t handle, const char *path,
-		        const char *name, void *value, size_t size);
-
-ssize_t
-glusterfs_glh_lgetxattr (glusterfs_handle_t handle, const char *path,
-		         const char *name, void *value, size_t size);
-
-ssize_t
-glusterfs_getxattr (const char *path, const char *name, void *value,
-                        size_t size);
-
-ssize_t
-glusterfs_lgetxattr (const char *path, const char *name, void *value,
-                     size_t size);
-
-ssize_t
-glusterfs_fgetxattr (glusterfs_file_t fd, const char *name, void *value,
-		     size_t size);
-
-ssize_t
-glusterfs_listxattr (glusterfs_handle_t handle, const char *path, char *list,
-                     size_t size);
-
-ssize_t
-glusterfs_llistxattr (glusterfs_handle_t handle, const char *path, char *list,
-                      size_t size);
-
-ssize_t
-glusterfs_flistxattr (glusterfs_file_t fd, char *list, size_t size);
-
-int
-glusterfs_removexattr (glusterfs_handle_t handle, const char *path,
-		       const char *name);
-
-int
-glusterfs_lremovexattr (glusterfs_handle_t handle, const char *path,
-			const char *name);
-
-int
-glusterfs_fremovexattr (glusterfs_file_t fd, const char *name);
-
-
-
-/* Read data from a file.
- * @fd          : Handle returned by glusterfs_open or
- *              glusterfs_glh_open.
- * @buf         : Buffer to read the data into.
- * @nbytes      : Number of bytes to read.
- *
- * Returns number of bytes actually read on success or -1 on error
- * with errno set to the appropriate error number.
- */
-ssize_t
-glusterfs_read (glusterfs_file_t fd, void *buf, size_t nbytes);
-
-
-
-/* Read data into an array of buffers.
- *
- * @fd          : File handle returned by glusterfs_open or
- *              glusterfs_glh_open.
- * @vec         : Array of buffers into which the data is read.
- * @count       : Number of iovecs referred to by vec.
- *
- * Returns number of bytes read on success or -1 on error with errno
- * set appropriately.
- */
-ssize_t
-glusterfs_readv (glusterfs_file_t fd, const struct iovec *vec, int count);
-
-int
-glusterfs_read_async (glusterfs_file_t fd, size_t nbytes, off_t offset,
-                      glusterfs_readv_cbk_t readv_cbk, void *cbk_data);
-
-
-
-/* Write data into a file.
- *
- * @fd          : File handle returned from glusterfs_open or
- *              glusterfs_glh_open.
- * @buf         : Buffer which is written to the file.
- * @nbytes      : Number bytes of the @buf written to the file.
- *
- * On success, returns number of bytes written. On error, returns -1
- * with errno set appropriately.
- */
-ssize_t
-glusterfs_write (glusterfs_file_t fd, const void *buf, size_t nbytes);
-
-
-
-/* Writes an array of buffers into a file.
- *
- * @fd          : The file handle returned from glusterfs_open or
- *              glusterfs_glh_open.
- * @vector      : Array of buffers to be written to the file.
- * @count       : Number of separate buffers in the @vector array.
- *
- * Returns number of bytes written on success or -1 on error with
- * errno set approriately.
- */
-ssize_t
-glusterfs_writev (glusterfs_file_t fd, const struct iovec *vector, int count);
-
-int
-glusterfs_write_async (glusterfs_file_t fd, const void *buf, size_t nbytes,
-                       off_t offset, glusterfs_write_cbk_t write_cbk,
-		       void *cbk_data);
-
-int
-glusterfs_writev_async (glusterfs_file_t fd, const struct iovec *vector,
-			int count, off_t offset,
-			glusterfs_write_cbk_t write_cbk, void *cbk_data);
-
-
-
-/* Read from a file starting at a given offset.
- *
- * @fd          : File handle returned from glusterfs_open or
- *              glusterfs_glh_open.
- * @buf         : Buffer to read the data into.
- * @nbytes      : Number of bytes to read.
- * @offset      : The offset to start reading @nbytes from.
- *
- * Returns number of bytes read on success or -1 on error with errno
- * set appropriately.
- */
-ssize_t
-glusterfs_pread (glusterfs_file_t fd, void *buf, size_t nbytes, off_t offset);
-
-
-
-/* Write to a file starting at a given offset.
- *
- * @fd          : Flie handle returned from glusterfs_open or
- *              glusterfs_glh_open.
- * @buf         : Buffer that will be written to the file.
- * @nbytes      : Number of bytes to write from @buf.
- * @offset      : The starting offset from where @nbytes will be
- *              written.
- *
- * Returns number of bytes written on success and -1 on error with
- * errno set appropriately.
- */
-ssize_t
-glusterfs_pwrite (glusterfs_file_t fd, const void *buf, size_t nbytes,
-		  off_t offset);
-
-
-
-/* Seek to an offset in the file.
- *
- * @fd          : File handle in which to seek to. File handle
- *              returned by glusterfs_open or glusterfs_glh_open.
- * @offset      : Offset to seek to in the given file.
- * @whence      : Determines how the offset is interpreted by this
- *              syscall. The behaviour is similar to the options
- *              provided by the POSIX lseek system call. See man lseek
- *              for more details.
- *
- * On success, returns the resulting absolute offset in the file after the seek
- * operation is performed. ON error, returns -1 with errno set
- * appropriately.
- */
-off_t
-glusterfs_lseek (glusterfs_file_t fd, off_t offset, int whence);
-
-
-
-/* Create a directory.
- *
- * @handle      : The handle of the glusterfs context in which the
- *              directory needs to be created.
- * @path        : The absolute path within the glusterfs context where
- *              the directory needs to be created.
- * @mode        : The mode bits for the newly created directory.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_mkdir (glusterfs_handle_t handle, const char *path, mode_t mode);
-
-
-
-/* Create a directory.
- *
- * @path        : Path to the directory that needs to be created. This
- *              path must be prefixed with the VMP of the particular glusterfs
- *              context.
- * @mode        : Mode flags for the newly created directory.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_mkdir (const char *path, mode_t mode);
-
-
-
-/* Remove a directory.
- *
- * @handle      : Handle of the glusterfs context from which to remove
- *              the directory.
- * @path        : The path of the directory to be removed in the glusterfs
- *              context.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_rmdir (glusterfs_handle_t handle, const char *path);
-
-
-
-/* Remove a directory.
- *
- * @path        : The absolute path to the directory to be removed.
- *              This path must be pre-fixed with the VMP of the
- *              particular glusterfs context in which this directory
- *              resides.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_rmdir (const char *path);
-
-
-
-/* Read directory entries.
- *
- * @fd          : The handle of the directory to be read. This handle
- *              is the one returned by opendir.
- *
- * Returns the directory entry on success and NULL pointer on error
- * with errno set appropriately.
- */
-void *
-glusterfs_readdir (glusterfs_dir_t dirfd);
-
-
-
-/* re-entrant version of glusterfs_readdir.
- *
- * @dirfd       : The handle of directory to be read. This handle is the one
- *                returned by opendir.
- * @entry       : Pointer to storage to store a directory entry. The storage
- *                pointed to by entry shall be large enough for a dirent with 
- *                an array of char d_name members containing at least
- *                {NAME_MAX}+1 elements.
- * @result      : Upon successful return, the pointer returned at *result shall
- *                have the same value as the argument entry. Upon reaching the
- *                end of the directory stream, this pointer shall have the
- *                value NULL.
- */
-int
-glusterfs_readdir_r (glusterfs_dir_t dirfd, struct dirent *entry,
-                     struct dirent **result);
-
-/* Close a directory handle.
- *
- * @fd          : The directory handle to be closed.
- *
- * Returns 0 on success and -1 on error with errno set to 0.
- */
-int
-glusterfs_closedir (glusterfs_dir_t dirfd);
-/* FIXME: remove getdents */
-int
-glusterfs_getdents (glusterfs_dir_t fd, struct dirent *dirp,
-		    unsigned int count);
-
-
-
-/* Create device node.
- *
- * @handle      : glusterfs context in which to create the device
- *              node.
- * @pathname    : The absolute path of the device to be created in the
- *              given glusterfs context.
- *
- * @mode        : Mode flags to apply to the newly created node.
- * @dev         : Device numbers that will apply to the node.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_mknod(glusterfs_handle_t handle, const char *pathname,
-                        mode_t mode, dev_t dev);
-
-
-
-/* Create a device node.
- *
- * @pathname    : The full path of the node to be created. This path
- *              should be pre-pended with the VMP of the glusterfs
- *              context in which this node is to be created.
- * @mode        : Mode flags that will be applied to the newly created
- *              device file.
- * @dev         : The device numbers that will be associated with the
- *              device node.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_mknod(const char *pathname, mode_t mode, dev_t dev);
-
-
-
-/* Returns the real absolute path of the given path.
- *
- * @handle              : The glusterfs context in which the path resides in.
- * @path                : The path to be resolved.
- * @resolved_path       : The resolved path is stored in this buffer
- *                      provided by the caller.
- *
- * Returns a pointer to resolved_path on success and NULL on error
- * with errno set appropriately.
- *
- * See man realpath for details.
- */
-char *
-glusterfs_glh_realpath (glusterfs_handle_t handle, const char *path,
-                        char *resolved_path);
-
-
-/* Returns the real absolute path of the given path.
- *
- * @path                : The path to be resolved. This path must be
- *                      pre-fixed with the VMP of the glusterfs
- *                      context in which the file resides.
- *
- * @resolved_path       : The resolved path is stored in this user
- *                      provided buffer.
- *
- * Returns a pointer to resolved_path on success, and NULL on error
- * with errno set appropriately.
- */
-char *
-glusterfs_realpath (const char *path, char *resolved_path);
-
-
-
-/* Change mode flags on a path.
- *
- * @handle      : Handle of the glusterfs instance in which the path
- *              resides.
- * @path        : The path whose mode bits need to be changed.
- * @mode        : The new mode bits.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_chmod (glusterfs_handle_t handle, const char *path, mode_t mode);
-
-
-
-/* Change mode flags on a path.
- *
- * @path        : The path whose mode bits need to be changed. The
- *              path should be pre-fixed with the VMP that identifies the
- *              glusterfs context within which the path resides.
- * @mode        : The new mode bits.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_chmod (const char *path, mode_t mode);
-
-
-
-/* Change the owner of a path.
- * If @path is a symlink, it is dereferenced and the ownership change
- * happens on the target.
- *
- * @handle      : Handle of the glusterfs context in which the path
- *              resides.
- * @path        : The path whose owner needs to be changed.
- * @owner       : ID of the new owner.
- * @group       : ID of the new group.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_chown (glusterfs_handle_t handle, const char *path, uid_t owner,
-                        gid_t group);
-
-
-
-/* Change the owner of a path.
- *
- * If @path is a symlink, it is dereferenced and the ownership change
- * happens on the target.
- * @path        : The path whose owner needs to be changed. Path must
- *              be pre-fixed with the VMP that identifies the
- *              glusterfs context in which the path resides.
- * @owner       : ID of the new owner.
- * @group       : ID of the new group.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_chown (const char *path, uid_t owner, gid_t group);
-
-
-
-/* Change the owner of the file.
- *
- * @fd          : Handle of the file whose owner needs to be changed.
- * @owner       : ID of the new owner.
- * @group       : ID of the new group.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_fchown (glusterfs_file_t fd, uid_t owner, gid_t group);
-
-
-
-/* Open a directory.
- *
- * @handle      : Handle that identifies a glusterfs context.
- * @path        : Path to the directory in the glusterfs context.
- *
- * Returns a non-NULL handle on success and NULL on failure with errno
- * set appropriately.
- */
-glusterfs_dir_t
-glusterfs_glh_opendir (glusterfs_handle_t handle, const char *path);
-
-
-
-/* Open a directory.
- *
- * @path        : Path to the directory. The path must be prepended
- *              with the VMP in order to identify the glusterfs
- *              context in which path resides.
- *
- * Returns a non-NULL handle on success and NULL on failure with errno
- * set appropriately.
- */
-glusterfs_dir_t
-glusterfs_opendir (const char *path);
-
-
-
-/* Change the mode bits on an open file.
- *
- * @fd          : The file whose mode bits need to be changed.
- * @mode        : The new mode bits.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_fchmod (glusterfs_file_t fd, mode_t mode);
-
-
-
-/* Sync the file contents to storage.
- *
- * @fd          : The file whose contents need to be sync'ed to
- *              storage.
- *
- * Return 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_fsync (glusterfs_file_t *fd);
-
-
-
-/* Truncate an open file.
- *
- * @fd          : The file to truncate.
- * @length      : The length to truncate to.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_ftruncate (glusterfs_file_t fd, off_t length);
-
-
-
-/* Create a hard link between two paths.
- *
- * @handle      : glusterfs context in which both paths should reside.
- * @oldpath     : The existing path to link to.
- * @newpath     : The new path which will be linked to @oldpath.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_link (glusterfs_handle_t handle, const char *oldpath,
-                        const char *newpath);
-
-
-
-/* Create a hard link between two paths.
- *
- * @oldpath     : The existing path to link to.
- * @newpath     : The new path which will be linked to @oldpath.
- *
- * Both paths should exist on the same glusterfs context and should be
- * prefixed with the same VMP.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_link (const char *oldpath, const char *newpath);
-
-
-
-/* Get stats about the underlying file system.
- *
- * @handle      : Identifies the glusterfs context in which resides
- *              the given path.
- * @path        : stats are returned for the file system on which file
- *              is located.
- * @buf         : The buffer into which the stats are copied.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_statfs (glusterfs_handle_t handle, const char *path,
-                        struct statfs *buf);
-
-
-
-/* Get stats about the underlying file system.
- *
- * @path        : stats are returned for the file system on which file
- *              is located. @path must start with the VMP of the
- *              glusterfs context on which the file reside.
- * @buf         : The buffer into which the stats are copied.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_statfs (const char *path, struct statfs *buf);
-
-
-
-/* Get stats about the underlying file system.
- *
- * @handle      : Identifies the glusterfs context in which resides
- *              the given path.
- * @path        : stats are returned for the file system on which file
- *              is located.
- * @buf         : The buffer into which the stats are copied.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_statvfs (glusterfs_handle_t handle, const char *path,
-                   struct statvfs *buf);
-
-
-
-/* Get stats about the underlying file system.
- *
- * @path        : stats are returned for the file system on which file
- *              is located. @path must start with the VMP of the
- *              glusterfs context on which the file reside.
- * @buf         : The buffer into which the stats are copied.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_statvfs (const char *path, struct statvfs *buf);
-
-
-
-/* Set the atime and mtime values for a given path.
- *
- * @handle      : The handle identifying the glusterfs context.
- * @path        : The path for which the times need to be changed.
- * @times       : The array containing new time stamps for the file.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_utimes (glusterfs_handle_t handle, const char *path,
-                        const struct timeval times[2]);
-
-
-
-/* Set the atime and mtime values for a given path.
- *
- * @path        : The path for which the times need to be changed.
- * @times       : The array containing new time stamps for the file.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_utimes (const char *path, const struct timeval times[2]);
-
-
-
-/* Set the atime and mtime values for a given path.
- *
- * @handle      : The handle identifying the glusterfs context.
- * @path        : The path for which the times need to be changed.
- * @buf         : The structure containing new time stamps for the file.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_utime (glusterfs_handle_t handle, const char *path,
-                        const struct utimbuf *buf);
-
-
-
-/* Set the atime and mtime values for a given path.
- *
- * @path        : The path for which the times need to be changed.
- * @buf         : The structure containing new time stamps for the file.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_utime (const char *path, const struct utimbuf *buf);
-
-
-
-/*  Create  FIFO at the given path.
- *
- *  @handle     : The glusterfs context in which to create that FIFO.
- *  @path       : The path within the context where the FIFO is to be
- *              created.
- * @mode        : The mode bits for the newly create FIFO.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_mkfifo (glusterfs_handle_t handle, const char *path,
-                        mode_t mode);
-
-
-
-/*  Create  FIFO at the given path.
- *
- *  @path       : The path within the context where the FIFO is to be
- *              created. @path should begin with the VMP of the
- *              glusterfs context in which the FIFO needs to be
- *              created.
- * @mode        : The mode bits for the newly create FIFO.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_mkfifo (const char *path, mode_t mode);
-
-
-
-/* Unlink a file.
- *
- * @handle      : Handle that identifies a glusterfs instance.
- * @path        : Path in the glusterfs instance that needs to be
- *              unlinked.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_unlink (glusterfs_handle_t handle, const char *path);
-
-
-
-/* Unlink a file.
- *
- * @path        : Path in the glusterfs instance that needs to be
- *              unlinked.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_unlink (const char *path);
-
-
-
-/* Create a symbolic link.
- *
- * @handle      : The handle identifying the glusterfs context.
- * @oldpath     : The existing path to which a symlink needs to be
- *              created.
- * @newpath     : The new path which will be symlinked to the
- *              @oldpath.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_symlink (glusterfs_handle_t handle, const char *oldpath,
-                                const char *newpath);
-
-
-
-/* Create a symbolic link.
- *
- * @oldpath     : The existing path to which a symlink needs to be
- *              created.
- * @newpath     : The new path which will be symlinked to the
- *              @oldpath.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_symlink (const char *oldpath, const char *newpath);
-
-
-
-/* Read a symbolic link.
- *
- * @handle      : Handle identifying the glusterfs context.
- * @path        : The symlink that needs to be read.
- * @buf         : The buffer into which the target of @path will be
- *              stored.
- * @bufsize     : Size of the buffer allocated to @buf.
- *
- * Returns number of bytes copied into @buf and -1 on error with errno
- * set appropriately.
- */
-ssize_t
-glusterfs_glh_readlink (glusterfs_handle_t handle, const char *path, char *buf,
-                                size_t bufsize);
-
-
-
-/* Read a symbolic link.
- *
- * @path        : The symlink that needs to be read.
- * @buf         : The buffer into which the target of @path will be
- *              stored.
- * @bufsize     : Size of the buffer allocated to @buf.
- *
- * Returns number of bytes copied into @buf and -1 on error with errno
- * set appropriately.
- */
-ssize_t
-glusterfs_readlink (const char *path, char *buf, size_t bufsize);
-
-
-
-/* Rename a file or directory.
- *
- * @handle      : The identifier of a glusterfs context.
- * @oldpath     : The path to be renamed.
- * @newpath     : The new name for the @oldpath.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_rename (glusterfs_handle_t handle, const char *oldpath,
-                      const char *newpath);
-
-
-
-/* Rename a file or directory.
- * @oldpath     : The path to be renamed.
- * @newpath     : The new name for the @oldpath.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_rename (const char *oldpath, const char *newpath);
-
-
-
-/* Remove a file or directory in the given glusterfs context.
- *
- * @handle      : Handle identifying the glusterfs context.
- * @path        : Path of the file or directory to be removed.
- *
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_remove (glusterfs_handle_t handle, const char *path);
-
-
-
-/* Remove a file or directory.
- *
- * @path        : Path of the file or directory to be removed. The
- *              path must be pre-fixed with the VMP.
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_remove (const char *path);
-
-
-
-/* Change the owner of the given path.
- *
- * If @path is a symlink, the ownership change happens on the symlink.
- *
- * @handle      : Handle identifying the glusterfs client context.
- * @path        : Path whose owner needs to be changed.
- * @owner       : New owner ID
- * @group       : New Group ID
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-int
-glusterfs_glh_lchown (glusterfs_handle_t handle, const char *path, uid_t owner,
-                      gid_t group);
-
-
-
-/* Change the owner of the given path.
- *
- * If @path is a symlink, the ownership change happens on the symlink.
- *
- * @path        : Path whose owner needs to be changed.
- * @owner       : New owner ID
- * @group       : New Group ID
- *
- * Returns 0 on success and -1 on error with errno set appropriately.
- */
-
-int
-glusterfs_lchown (const char *path, uid_t owner, gid_t group);
-
-
-
-/* Rewind directory stream pointer to beginning of the directory.
- *
- * @dirfd       : Directory handle returned by glusterfs_open on
- *              glusterfs_opendir.
- *
- * Returns no value.
- */
-void
-glusterfs_rewinddir (glusterfs_dir_t dirfd);
-
-
-
-/* Seek to the given offset in the directory handle.
- *
- * @dirfd       : Directory handle returned by glusterfs_open on
- *              glusterfs_opendir.
- * @offset      : The offset to seek to.
- *
- * Returns no value.
- */
-void
-glusterfs_seekdir (glusterfs_dir_t dirfd, off_t offset);
-
-
-
-/* Return the current offset in a directory stream.
- *
- * @dirfd       : Directory handle returned by glusterfs_open on
- *              glusterfs_opendir.
- *
- * Returns the offset in the directory or -1 on error with errno set
- * appropriately.
- */
-off_t
-glusterfs_telldir (glusterfs_dir_t dirfd);
-
-
-/* Write count bytes from in_fd to out_fd, starting at *offset.
- * glusterfs_sendfile aims at eliminating memory copy at the end of
- * each read from in_fd, copying the file directly to out_fd from the buffer 
- * provided by glusterfs.
- *
- * @out_fd: file descriptor opened for writing
- *
- * @in_fd: glusterfs file handle to the file to be read from.
- *
- * @offset: If offset is not NULL, then it points to a variable holding the file
- *          offset  from  which  glusterfs_sendfile()  will  start reading data
- *          from in_fd.  When glusterfs_sendfile() returns, this variable will 
- *          be set to the offset of the byte following the last byte that was 
- *          read.  If offset is  not  NULL, then glusterfs_sendfile()  does  not
- *          modify the current file offset of in_fd; otherwise the current file
- *          offset is adjusted to reflect the number of bytes read from in_fd.
- *
- * @count:  number of bytes to copy between the file descriptors.
- */
-
-ssize_t
-glusterfs_sendfile (int out_fd, glusterfs_file_t in_fd, off_t *offset,
-                    size_t count);
-
-/* manipulate file descriptor
- * This api can have 3 forms similar to fcntl(2).
- *
- * int
- * glusterfs_fcntl (glusterfs_file_t fd, int cmd)
- *
- * int
- * glusterfs_fcntl (glusterfs_file_t fd, int cmd, long arg)
- *
- * int
- * glusterfs_fcntl (glusterfs_file_t fd, int cmd, struct flock *lock)
- *
- * @fd   : file handle returned by glusterfs_open or glusterfs_create.
- * @cmd  : Though the aim is to implement all possible commands supported by
- *         fcntl(2), currently following commands are supported.
- *         F_SETLK, F_SETLKW, F_GETLK -  used to acquire, release, and test for
- *                                       the existence of record locks (also 
- *                                       known as file-segment or file-region
- *                                       locks). More detailed explanation is
- *                                       found in 'man 2 fcntl'
- */
-   
-int
-glusterfs_fcntl (glusterfs_file_t fd, int cmd, ...);
-
-/*
- * Change the current working directory to @path
- * 
- * @path  : path to change the current working directory to.
- *
- * Returns 0 on success and -1 on failure with errno set appropriately.
- */
-int
-glusterfs_chdir (const char *path);
-
-/*
- * Change the current working directory to the path @fd is opened on.
- *
- * @fd   : current working directory will be changed to path @fd is opened on.
- *
- * Returns 0 on success and -1 on  with errno set appropriately.
- */
-int
-glusterfs_fchdir (glusterfs_file_t fd);
-
-/* copies the current working directory into @buf if it is big enough
- *
- * @buf: buffer to copy into it. If @buf is NULL, a buffer will be allocated.
- *       The size of the buffer will be @size if it is not zero, otherwise the
- *       size will be big enough to hold the current working directory.
- * @size: size of the buffer.
- *
- * Returns the pointer to buffer holding current working directory on success
- * and NULL on failure.
- */
-
-char *
-glusterfs_getcwd (char *buf, size_t size);
-
-/*
- * Truncate the file to a specified length.
- *
- * @path   : path to the file.
- * @length : length to which the file has to be truncated.
- *
- * Returns 0 on success and -1 on failure with errno set appropriately 
- */
-
-int
-glusterfs_truncate (const char *path, off_t length);
-
-
-/* FIXME: review the need for these apis */
-/* added for log related initialization in booster fork implementation */
-void
-glusterfs_reset (void);
-
-void
-glusterfs_log_lock (void);
-
-void
-glusterfs_log_unlock (void);
-/* Used to free the glusterfs_read_buf passed to the application from
-   glusterfs_read_async_cbk
-*/
-void
-glusterfs_free (glusterfs_iobuf_t *buf);
-
-__END_DECLS
-
-#endif /* !_LIBGLUSTERFSCLIENT_H */
diff --git a/mod_glusterfs/Makefile.am b/mod_glusterfs/Makefile.am
deleted file mode 100644
index 0abe8dcfc..000000000
--- a/mod_glusterfs/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = apache lighttpd
-
-CLEANFILES = 
diff --git a/mod_glusterfs/apache/1.3/src/Makefile.am b/mod_glusterfs/apache/1.3/src/Makefile.am
deleted file mode 100644
index 6bb3075f5..000000000
--- a/mod_glusterfs/apache/1.3/src/Makefile.am
+++ /dev/null
@@ -1,30 +0,0 @@
-mod_glusterfs_PROGRAMS = mod_glusterfs.so
-mod_glusterfsdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/apache/1.3
-
-mod_glusterfs_so_SOURCES = mod_glusterfs.c
-
-all: mod_glusterfs.so
-
-mod_glusterfs.so: $(top_srcdir)/mod_glusterfs/apache/1.3/src/mod_glusterfs.c $(top_builddir)/libglusterfsclient/src/libglusterfsclient.la
-	ln -sf $(top_srcdir)/mod_glusterfs/apache/1.3/src/mod_glusterfs.c $(top_builddir)/mod_glusterfs/apache/1.3/src/mod_glusterfs-build.c
-	$(APXS) -c -Wc,-g3 -Wc,-O0 -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D_GNU_SOURCE -I$(top_srcdir)/libglusterfsclient/src -Wl,-rpath,$(libdir) -Wl,-rpath,$(top_builddir)/libglusterfsclient/src/.libs/ $(top_builddir)/libglusterfsclient/src/.libs/libglusterfsclient.so mod_glusterfs-build.c -o $(top_builddir)/mod_glusterfs/apache/1.3/src/mod_glusterfs.so
-
-$(top_builddir)/libglusterfsclient/src/libglusterfsclient.la:
-	$(MAKE) -C $(top_builddir)/libglusterfsclient/src/ all
-
-install-data-local:
-	@echo ""
-	@echo ""
-	@echo "**********************************************************************************"
-	@echo "* TO INSTALL MODGLUSTERFS, PLEASE USE,                                            "
-	@echo "* $(APXS) -n glusterfs -ia $(mod_glusterfsdir)/mod_glusterfs.so                   "
-	@echo "**********************************************************************************"
-	@echo ""
-	@echo ""
-
-#install:
-#	cp -fv mod_glusterfs.so $(HTTPD_LIBEXECDIR)
-#	cp -fv httpd.conf $(HTTPD_CONF_DIR)
-
-clean:
-	-rm -fv *.so *.o mod_glusterfs-build.c
diff --git a/mod_glusterfs/apache/1.3/src/README.txt b/mod_glusterfs/apache/1.3/src/README.txt
deleted file mode 100644
index 378a51d79..000000000
--- a/mod_glusterfs/apache/1.3/src/README.txt
+++ /dev/null
@@ -1,107 +0,0 @@
-What is mod_glusterfs?
-======================
-* mod_glusterfs is a module for apache written for efficient serving of files from glusterfs. 
-  mod_glusterfs interfaces with glusterfs using apis provided by libglusterfsclient.
-
-* this README speaks about installation of apache-1.3.x, where x is any minor version.
-
-Prerequisites for mod_glusterfs
-===============================
-Though mod_glusterfs has been written as a module, with an intent of making no changes to the way apache has 
-been built, currently following points have to be taken care of:
-
-* module "so" has to be enabled, for apache to support modules.
-* since glusterfs is compiled with _FILE_OFFSET_BITS=64 and __USE_FILE_OFFSET64 flags, mod_glusterfs and apache 
-  in turn have to be compiled with the above two flags.
- 
- $ tar xzvf apache-1.3.9.tar.gz
- $ cd apache-1.3.9/
- $ # add -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 to EXTRA_CFLAGS in src/Configuration.
- $ ./configure --prefix=/usr --enable-module=so 
- $ cd src
- $ ./Configure 
- $ cd ../
- $ make install  
- $ httpd -l | grep -i mod_so 
-   mod_so.c
-
-* if multiple apache installations are present, make sure to pass --with-apxs=/path/to/apxs/of/proper/version to configure script while building glusterfs.
-
-Build/Install mod_glusterfs
-===========================
-* mod_glusterfs is provided with glusterfs--mainline--3.0 and all releases from the same branch.
-
-* building glusterfs also builds mod_glusterfs. But 'make install' of glusterfs installs mod_glusterfs.so to 
-  glusterfs install directory instead of the apache modules directory.
-
-* 'make install' of glusterfs will print a message similar to the one given below, which is self explanatory. 
-  Make sure to use apxs of proper apache version in case of multiple apache installations. This will copy 
-  mod_glusterfs.so to modules directory of proper apache version and modify the appropriate httpd.conf to enable
-  mod_glusterfs. 
-
-**********************************************************************************************
-* TO INSTALL MODGLUSTERFS, PLEASE USE,                                            
-* apxs -n mod_glusterfs -ia /usr/lib/glusterfs/1.4.0pre2/apache-1.3/mod_glusterfs.so               
-**********************************************************************************************
-
-Configuration
-=============
-* Following configuration has to be added to httpd.conf.
-
- <Location "/glusterfs">
- 	   GlusterfsLogfile "/var/log/glusterfs/glusterfs.log"
-	   GlusterfsLoglevel "warning"
- 	   GlusterfsVolumeSpecfile "/etc/glusterfs/glusterfs-client.spec"
-	   GlusterfsCacheTimeout "600"
-	   GlusterfsXattrFileSize "65536"
- 	   SetHandler "glusterfs-handler"
- </Location>
-
-* GlusterfsVolumeSpecfile (COMPULSORY)
-  Path to the the glusterfs volume specification file.
-
-* GlusterfsLogfile (COMPULSORY)
-  Path to the glusterfs logfile.
-
-* GlusterfsLoglevel (OPTIONAL, default = warning)
-  Severity of messages that are to be logged. Allowed values are critical, error, warning, debug, none 
-  in the decreasing order of severity.
-
-* GlusterfsCacheTimeOut (OPTIONAL, default = 0)
-  Timeout values for glusterfs stat and lookup cache.
-
-* GlusterfsXattrFileSize (OPTIONAL, default = 0)
-  Files with sizes upto and including this value are fetched through the extended attribute interface of 
-  glusterfs rather than the usual open-read-close set of operations. For files of small sizes, it is recommended 
-  to use extended attribute interface.
-
-* With the above configuration all the requests to httpd of the form www.example.org/glusterfs/path/to/file are 
-  served from glusterfs.
-
-Miscellaneous points
-====================
-* httpd by default runs with username "nobody" and group "nogroup". Permissions of logfile and specfile have to 
-  be set suitably.
-
-* Since mod_glusterfs runs with permissions of nobody.nogroup, glusterfs has to use only login based 
-  authentication. See docs/authentication.txt for more details. 
-
-* To copy the data served by httpd into glusterfs mountpoint, glusterfs can be started with the 
-  volume-specification file provided to mod_glusterfs. Any tool like cp can then be used.
-
-* To run in gdb, apache has to be compiled with -lpthread, since libglusterfsclient is multithreaded. 
-  If not on Linux gdb runs into errors like: 
-  "Error while reading shared library symbols:
-   Cannot find new threads: generic error"
-
-* when used with ib-verbs transport, ib_verbs initialization fails.
-  reason for this is that apache runs as non-privileged user and the amount of memory that can be 
-  locked by default is not sufficient for ib-verbs. to fix this, as root run,
-  
-  # ulimit -l unlimited
-
-  and then start apache.
- 
-TODO
-====
-* directory listing for the directories accessed through mod_glusterfs.
diff --git a/mod_glusterfs/apache/1.3/src/mod_glusterfs.c b/mod_glusterfs/apache/1.3/src/mod_glusterfs.c
deleted file mode 100644
index c1380a4fd..000000000
--- a/mod_glusterfs/apache/1.3/src/mod_glusterfs.c
+++ /dev/null
@@ -1,507 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef CORE_PRIVATE
-#define CORE_PRIVATE
-#endif
-
-#include <httpd.h>
-#include <http_config.h>
-#include <http_core.h>
-#include <http_request.h>
-#include <http_protocol.h>
-#include <http_log.h>
-#include <http_main.h>
-#include <util_script.h>
-#include <libglusterfsclient.h>
-#include <sys/uio.h>
-#include <pthread.h>
-
-#define GLUSTERFS_INVALID_LOGLEVEL "mod_glusterfs: Unrecognized log-level "\
-                                   "\"%s\", possible values are \"DEBUG|"\
-                                   "WARNING|ERROR|CRITICAL|NONE\"\n"
-
-#define GLUSTERFS_HANDLER "glusterfs-handler"
-#define GLUSTERFS_CHUNK_SIZE 131072 
-
-module MODULE_VAR_EXPORT glusterfs_module;
-
-/*TODO: verify error returns to server core */
-
-typedef struct glusterfs_dir_config {
-        char *logfile;
-        char *loglevel;
-        char *specfile;
-        char *mount_dir;
-        char *buf;
-        size_t xattr_file_size;
-        uint32_t cache_timeout;
-} glusterfs_dir_config_t;
-
-typedef struct glusterfs_async_local {
-        int op_ret;
-        int op_errno;
-        char async_read_complete;
-        off_t length;
-        off_t read_bytes;
-        glusterfs_iobuf_t *buf;
-        request_rec *request;
-        pthread_mutex_t lock;
-        pthread_cond_t cond;
-}glusterfs_async_local_t;
-
-#define GLUSTERFS_CMD_PERMS ACCESS_CONF
-
-static glusterfs_dir_config_t *
-mod_glusterfs_dconfig(request_rec *r)
-{
-        glusterfs_dir_config_t *dir_config = NULL;
-        if (r->per_dir_config != NULL) {
-                dir_config = ap_get_module_config (r->per_dir_config,
-                                                   &glusterfs_module);
-        }
-
-        return dir_config;
-}
-
-static 
-const char *add_xattr_file_size(cmd_parms *cmd, void *dummy, char *arg)
-{
-        glusterfs_dir_config_t *dir_config = dummy;
-        dir_config->xattr_file_size = atoi (arg);
-        return NULL;
-}
-
-static
-const char *set_cache_timeout(cmd_parms *cmd, void *dummy, char *arg)
-{
-        glusterfs_dir_config_t *dir_config = dummy;
-        dir_config->cache_timeout = atoi (arg);
-        return NULL;
-}
-
-static
-const char *set_loglevel(cmd_parms *cmd, void *dummy, char *arg)
-{
-        glusterfs_dir_config_t *dir_config = dummy;
-        char *error = NULL;
-        if (strncasecmp (arg, "DEBUG", strlen ("DEBUG")) 
-            && strncasecmp (arg, "WARNING", strlen ("WARNING")) 
-            && strncasecmp (arg, "CRITICAL", strlen ("CRITICAL")) 
-            && strncasecmp (arg, "NONE", strlen ("NONE")) 
-            && strncasecmp (arg, "ERROR", strlen ("ERROR")))
-                error = GLUSTERFS_INVALID_LOGLEVEL;
-        else
-                dir_config->loglevel = arg;
-
-        return error;
-}
-
-static 
-const char *add_logfile(cmd_parms *cmd, void *dummy, char *arg)
-{
-        glusterfs_dir_config_t *dir_config = dummy;
-        dir_config->logfile = arg;
-
-        return NULL;
-}
-
-static 
-const char *add_specfile(cmd_parms *cmd, void *dummy, char *arg)
-{
-        glusterfs_dir_config_t *dir_config = dummy;
-
-        dir_config->specfile = arg;
-
-        return NULL;
-}
-
-static void *
-mod_glusterfs_create_dir_config(pool *p, char *dirspec)
-{
-        glusterfs_dir_config_t *dir_config = NULL;
-
-        dir_config = (glusterfs_dir_config_t *) ap_pcalloc(p,
-                                                           sizeof(*dir_config));
-
-        dir_config->mount_dir = dirspec;
-        dir_config->logfile = dir_config->specfile = (char *)0;
-        dir_config->loglevel = "warning";
-        dir_config->cache_timeout = 0;
-        dir_config->buf = NULL;
-
-        return (void *) dir_config;
-}
-
-static void 
-mod_glusterfs_child_init(server_rec *s, pool *p)
-{
-        void **urls = NULL;
-        int n, i;
-        core_server_config *mod_core_config = ap_get_module_config (s->module_config,
-                                                                    &core_module);
-        glusterfs_dir_config_t *dir_config = NULL;
-        glusterfs_init_params_t params = {0, };
-  
-        n = mod_core_config->sec_url->nelts;
-        urls = (void **)mod_core_config->sec_url->elts;
-        for (i = 0; i < n; i++) {
-                dir_config = ap_get_module_config (urls[i], &glusterfs_module);
-
-                if (dir_config) {
-                        memset (&params, 0, sizeof (params));
-
-                        params.logfile = dir_config->logfile;
-                        params.loglevel = dir_config->loglevel;
-                        params.lookup_timeout = dir_config->cache_timeout;
-                        params.stat_timeout = dir_config->cache_timeout;
-                        params.specfile = dir_config->specfile;
-
-                        glusterfs_mount (dir_config->mount_dir, &params);
-                }
-                dir_config = NULL;
-        }
-}
-
-static void 
-mod_glusterfs_child_exit(server_rec *s, pool *p)
-{
-        void **urls = NULL;
-        int n, i;
-        core_server_config *mod_core_config = NULL;
-        glusterfs_dir_config_t *dir_config = NULL;
-  
-        mod_core_config = ap_get_module_config (s->module_config, &core_module);
-        n = mod_core_config->sec_url->nelts;
-        urls = (void **)mod_core_config->sec_url->elts;
-        for (i = 0; i < n; i++) {
-                dir_config = ap_get_module_config (urls[i], &glusterfs_module);
-                if (dir_config) {
-                        glusterfs_umount (dir_config->mount_dir);
-                }
-        }
-}
-
-static int mod_glusterfs_fixup(request_rec *r)
-{
-        glusterfs_dir_config_t *dir_config = NULL;
-        int access_status;
-        int ret;
-        char *path = NULL;
-
-        dir_config = mod_glusterfs_dconfig(r);
-
-        if (dir_config && dir_config->mount_dir
-            && !(strncmp (ap_pstrcat (r->pool, dir_config->mount_dir, "/",
-                                      NULL),
-                          r->uri, strlen (dir_config->mount_dir) + 1)
-                 && !r->handler)) 
-                r->handler = ap_pstrdup (r->pool, GLUSTERFS_HANDLER);
-
-        if (!r->handler || (r->handler && strcmp (r->handler,
-                                                  GLUSTERFS_HANDLER)))
-                return DECLINED;
-
-        path = r->uri;
-
-        memset (&r->finfo, 0, sizeof (r->finfo));
-
-        dir_config->buf = calloc (1, dir_config->xattr_file_size);
-        if (!dir_config->buf) {
-                return HTTP_INTERNAL_SERVER_ERROR;
-        }
-
-        ret = glusterfs_get (path, dir_config->buf, 
-                             dir_config->xattr_file_size, &r->finfo);
-
-        if (ret == -1 || r->finfo.st_size > dir_config->xattr_file_size
-            || S_ISDIR (r->finfo.st_mode)) {
-                free (dir_config->buf);
-                dir_config->buf = NULL;
-
-                if (ret == -1) {
-                        int error = HTTP_NOT_FOUND;
-                        char *emsg = NULL;
-                        if (r->path_info == NULL) {
-                                emsg = ap_pstrcat(r->pool, strerror (errno),
-                                                  r->filename, NULL);
-                        }
-                        else {
-                                emsg = ap_pstrcat(r->pool, strerror (errno),
-                                                  r->filename, r->path_info,
-                                                  NULL);
-                        }
-                        ap_log_rerror(APLOG_MARK, APLOG_ERR|APLOG_NOERRNO, r,
-                                      "%s", emsg);
-                        if (errno != ENOENT) {
-                                error = HTTP_INTERNAL_SERVER_ERROR;
-                        }
-                        return error;
-                }
-        }
-    
-        if (r->uri && strlen (r->uri) && r->uri[strlen(r->uri) - 1] == '/') 
-                r->handler = NULL;
-
-        r->filename = ap_pstrcat (r->pool, r->filename, r->path_info, NULL);
-
-        if ((access_status = ap_find_types(r)) != 0) {
-                return DECLINED;
-        }
-
-        return OK;
-}
-
-
-int 
-mod_glusterfs_readv_async_cbk (int32_t op_ret, int32_t op_errno,
-                               glusterfs_iobuf_t *buf, void *cbk_data)
-{
-        glusterfs_async_local_t *local = cbk_data;
-
-        pthread_mutex_lock (&local->lock);
-        {
-                local->async_read_complete = 1;
-                local->buf = buf;
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                pthread_cond_signal (&local->cond);
-        }
-        pthread_mutex_unlock (&local->lock);
-
-        return 0;
-}
-
-/* use read_async just to avoid memcpy of read buffer in libglusterfsclient */
-static int
-mod_glusterfs_read_async (request_rec *r, glusterfs_file_t fd, off_t offset,
-                          off_t length)
-{
-        glusterfs_async_local_t local;
-        off_t end;
-        int nbytes;
-        int complete;
-        pthread_cond_init (&local.cond, NULL);
-        pthread_mutex_init (&local.lock, NULL);
-  
-        memset (&local, 0, sizeof (local));
-        local.request = r;
-
-        if (length > 0)
-                end = offset + length;
-
-        do {
-                glusterfs_iobuf_t *buf;
-                int i;
-                if (length > 0) {
-                        nbytes = end - offset;
-                        if (nbytes > GLUSTERFS_CHUNK_SIZE)
-                                nbytes = GLUSTERFS_CHUNK_SIZE;
-                } else
-                        nbytes = GLUSTERFS_CHUNK_SIZE;
-
-                glusterfs_read_async(fd, 
-                                     nbytes,
-                                     offset,
-                                     mod_glusterfs_readv_async_cbk,
-                                     (void *)&local);
-
-                pthread_mutex_lock (&local.lock);
-                {
-                        while (!local.async_read_complete) {
-                                pthread_cond_wait (&local.cond, &local.lock);
-                        }
-
-                        local.async_read_complete = 0;
-                        buf = local.buf;
-
-                        if (length < 0)
-                                complete = (local.op_ret <= 0);
-                        else {
-                                local.read_bytes += local.op_ret;
-                                complete = ((local.read_bytes == length) 
-                                            || (local.op_ret < 0));
-                        }
-                }
-                pthread_mutex_unlock (&local.lock);
-
-                for (i = 0; i < buf->count; i++) {
-                        if (ap_rwrite (buf->vector[i].iov_base,
-                                       buf->vector[i].iov_len, r) < 0) {
-                                local.op_ret = -1;
-                                complete = 1;
-                                break;
-                        }
-                }      
-
-                glusterfs_free (buf);
-
-                offset += nbytes;
-        } while (!complete);
-
-        return (local.op_ret < 0 ? SERVER_ERROR : OK);
-}
-
-static int 
-mod_glusterfs_handler(request_rec *r)
-{
-        glusterfs_dir_config_t *dir_config;
-        char *path = NULL;
-        int error = OK;
-        int rangestatus = 0;
-        int errstatus = OK;
-        glusterfs_file_t fd;
-  
-        if (!r->handler || (r->handler && strcmp (r->handler,
-                                                  GLUSTERFS_HANDLER)))
-                return DECLINED;
-
-        if (r->uri[0] == '\0' || r->uri[strlen(r->uri) - 1] == '/') {
-                return DECLINED;
-        }
-  
-        dir_config = mod_glusterfs_dconfig (r);
-  
-        if (r->method_number != M_GET) {
-                return METHOD_NOT_ALLOWED;
-        }
-
-        ap_update_mtime(r, r->finfo.st_mtime);
-        ap_set_last_modified(r);
-        ap_set_etag(r);
-        ap_table_setn(r->headers_out, "Accept-Ranges", "bytes");
-        if (((errstatus = ap_meets_conditions(r)) != OK)
-            || (errstatus = ap_set_content_length(r, r->finfo.st_size))) {
-                return errstatus;
-        }
-        rangestatus =  ap_set_byterange(r);
-        ap_send_http_header(r);
-
-        if (r->finfo.st_size <= dir_config->xattr_file_size && dir_config->buf) {
-                if (!r->header_only) {
-                        error = OK;
-                        ap_log_rerror (APLOG_MARK, APLOG_NOTICE, r, 
-                                       "fetching data from glusterfs through "
-                                       "xattr interface\n");
-      
-                        if (!rangestatus) {
-                                if (ap_rwrite (dir_config->buf,
-                                               r->finfo.st_size, r) < 0) {
-                                        error = HTTP_INTERNAL_SERVER_ERROR;
-                                }
-                        } else {
-                                long offset, length;
-                                while (ap_each_byterange (r, &offset, &length)) {
-                                        if (ap_rwrite (dir_config->buf + offset,
-                                                       length, r) < 0) {
-                                                error = HTTP_INTERNAL_SERVER_ERROR;
-                                                break;
-                                        }
-                                }
-                        }
-                }
-
-                free (dir_config->buf);
-                dir_config->buf = NULL;
-
-                return error;
-        }
-
-        path = r->uri;
-        fd = glusterfs_open (path , O_RDONLY, 0);
-  
-        if (fd == 0) {
-                ap_log_rerror(APLOG_MARK, APLOG_ERR, r,
-                              "file permissions deny server access: %s",
-                              r->filename);
-                return FORBIDDEN;
-        }
-  
-        if (!r->header_only) {
-                if (!rangestatus) {
-                        mod_glusterfs_read_async (r, fd, 0, -1);
-                } else {
-                        long offset, length;
-                        while (ap_each_byterange(r, &offset, &length)) {
-                                mod_glusterfs_read_async (r, fd, offset, length);
-                        }
-                }
-        }
-  
-        glusterfs_close (fd);
-        return error;
-}
-
-static const command_rec mod_glusterfs_cmds[] =
-{
-        {"GlusterfsLogfile", add_logfile, NULL,
-         GLUSTERFS_CMD_PERMS, TAKE1,
-         "Glusterfs Logfile"},
-        {"GlusterfsLoglevel", set_loglevel, NULL,
-         GLUSTERFS_CMD_PERMS, TAKE1,
-         "Glusterfs Loglevel:anyone of none, critical, error, warning, debug"},
-        {"GlusterfsCacheTimeout", set_cache_timeout, NULL,
-         GLUSTERFS_CMD_PERMS, TAKE1,
-         "Timeout value in seconds for caching lookups and stats"},
-        {"GlusterfsVolumeSpecfile", add_specfile, NULL,
-         GLUSTERFS_CMD_PERMS, TAKE1,
-         "Glusterfs Specfile required to access contents of this directory"},
-        {"GlusterfsXattrFileSize", add_xattr_file_size, NULL, 
-         GLUSTERFS_CMD_PERMS, TAKE1,
-         "Maximum size of the file to be fetched using xattr interface of "
-         "glusterfs"},
-        {NULL}
-};
-
-static const handler_rec mod_glusterfs_handlers[] =
-{
-        {GLUSTERFS_HANDLER, mod_glusterfs_handler},
-        {NULL}
-};
-
-module glusterfs_module =
-{
-        STANDARD_MODULE_STUFF,
-        NULL, 
-        mod_glusterfs_create_dir_config,  /* per-directory config creator */
-        NULL,
-        NULL,       /* server config creator */
-        NULL,        /* server config merger */
-        mod_glusterfs_cmds,               /* command table */
-        mod_glusterfs_handlers,           /* [7] list of handlers */
-        NULL,  /* [2] filename-to-URI translation */
-        NULL,      /* [5] check/validate user_id */
-        NULL,       /* [6] check user_id is valid *here* */
-        NULL,     /* [4] check access by host address */
-        NULL,       /* [7] MIME type checker/setter */
-        mod_glusterfs_fixup,        /* [8] fixups */
-        NULL,             /* [10] logger */
-#if MODULE_MAGIC_NUMBER >= 19970103
-        NULL,      /* [3] header parser */
-#endif
-#if MODULE_MAGIC_NUMBER >= 19970719
-        mod_glusterfs_child_init,         /* process initializer */
-#endif
-#if MODULE_MAGIC_NUMBER >= 19970728
-        mod_glusterfs_child_exit,         /* process exit/cleanup */
-#endif
-#if MODULE_MAGIC_NUMBER >= 19970902
-        NULL   /* [1] post read_request handling */
-#endif
-};
diff --git a/mod_glusterfs/apache/2.2/src/Makefile.am b/mod_glusterfs/apache/2.2/src/Makefile.am
deleted file mode 100644
index 1e8f3a31e..000000000
--- a/mod_glusterfs/apache/2.2/src/Makefile.am
+++ /dev/null
@@ -1,31 +0,0 @@
-mod_glusterfs_PROGRAMS = mod_glusterfs.so
-mod_glusterfsdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/apache/2.2
-
-mod_glusterfs_so_SOURCES = mod_glusterfs.c
-
-all: mod_glusterfs.so
-
-mod_glusterfs.so: $(top_srcdir)/mod_glusterfs/apache/2.2/src/mod_glusterfs.c $(top_builddir)/libglusterfsclient/src/libglusterfsclient.la
-	ln -sf $(top_srcdir)/mod_glusterfs/apache/2.2/src/mod_glusterfs.c $(top_builddir)/mod_glusterfs/apache/2.2/src/mod_glusterfs-build.c
-	$(APXS) -c -o mod_glusterfs.la -Wc,-g3 -Wc,-O0 -DLINUX=2 -D_REENTRANT -D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -I$(top_srcdir)/libglusterfsclient/src -L$(top_builddir)/libglusterfsclient/src/.libs/ -lglusterfsclient mod_glusterfs-build.c 
-	-ln -sf .libs/mod_glusterfs.so mod_glusterfs.so
-
-$(top_builddir)/libglusterfsclient/src/libglusterfsclient.la:
-	$(MAKE) -C $(top_builddir)/libglusterfsclient/src/ all
-
-install-data-local:
-	@echo ""
-	@echo ""
-	@echo "**********************************************************************************"
-	@echo "* TO INSTALL MODGLUSTERFS, PLEASE USE,                                            "
-	@echo "* $(APXS) -n glusterfs -ia $(mod_glusterfsdir)/mod_glusterfs.so                "
-	@echo "**********************************************************************************"
-	@echo ""
-	@echo ""
-
-#install:
-#	cp -fv mod_glusterfs.so $(HTTPD_LIBEXECDIR)
-#	cp -fv httpd.conf $(HTTPD_CONF_DIR)
-
-clean:
-	rm -fv *.so *.o
diff --git a/mod_glusterfs/apache/2.2/src/README.txt b/mod_glusterfs/apache/2.2/src/README.txt
deleted file mode 100644
index 214a2535b..000000000
--- a/mod_glusterfs/apache/2.2/src/README.txt
+++ /dev/null
@@ -1,105 +0,0 @@
-What is mod_glusterfs?
-======================
-* mod_glusterfs is a module for apache written for efficient serving of files from glusterfs. 
-  mod_glusterfs interfaces with glusterfs using apis provided by libglusterfsclient.
-
-* this README speaks about installing mod_glusterfs for httpd-2.2 and higher.
-
-Prerequisites for mod_glusterfs
-===============================
-Though mod_glusterfs has been written as a module, with an intent of making no changes to 
-the way apache has been built, currently following points have to be taken care of:
-
-* since glusterfs is compiled with _FILE_OFFSET_BITS=64 and __USE_FILE_OFFSET64 flags, mod_glusterfs and apache 
-  in turn have to be compiled with the above two flags.
- 
- $ tar xzf httpd-2.2.10.tar.gz 
- $ cd httpd-2.2.10/
- $ export CFLAGS='-D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64'
- $ ./configure --prefix=/usr 
- $ make 
- $ make install
- $ httpd -l | grep -i mod_so 
-   mod_so.c
-
-* if multiple apache installations are present, make sure to pass --with-apxs=/path/to/apxs/of/proper/version 
-  to configure script while building glusterfs.
-
-Build/Install mod_glusterfs
-===========================
-* mod_glusterfs is provided with glusterfs--mainline--3.0 and all releases from the same branch.
-
-* building glusterfs also builds mod_glusterfs. But 'make install' of glusterfs installs mod_glusterfs.so to 
-  glusterfs install directory instead of the apache modules directory.
-
-* 'make install' of glusterfs will print a message similar to the one given below, which is self explanatory. 
-  Make sure to use apxs of proper apache version in case of multiple apache installations. This will copy 
-  mod_glusterfs.so to modules directory of proper apache version and modify the appropriate httpd.conf to enable
-  mod_glusterfs. 
-
-**********************************************************************************
-* TO INSTALL MODGLUSTERFS, PLEASE USE,                                            
-* apxs -n glusterfs -ia /usr/lib/glusterfs/2.0.0rc4/apache/2.2/mod_glusterfs.so                
-**********************************************************************************
-
-Configuration
-=============
-* Following configuration has to be added to httpd.conf.
-
- <Location "/glusterfs">
- 	   GlusterfsLogfile "/var/log/glusterfs/glusterfs.log"
-	   GlusterfsLoglevel "warning"
- 	   GlusterfsVolumeSpecfile "/etc/glusterfs/glusterfs-client.spec"
-	   GlusterfsCacheTimeout "600"
-	   GlusterfsXattrFileSize "65536"
- 	   SetHandler "glusterfs-handler"
- </Location>
-
-* GlusterfsVolumeSpecfile (COMPULSORY)
-  Path to the the glusterfs volume specification file.
-
-* GlusterfsLogfile (COMPULSORY)
-  Path to the glusterfs logfile.
-
-* GlusterfsLoglevel (OPTIONAL, default = warning)
-  Severity of messages that are to be logged. Allowed values are critical, error, warning, debug, none 
-  in the decreasing order of severity.
-
-* GlusterfsCacheTimeOut (OPTIONAL, default = 0)
-  Timeout values for glusterfs stat and lookup cache.
-
-* GlusterfsXattrFileSize (OPTIONAL, default = 0)
-  Files with sizes upto and including this value are fetched through the extended attribute interface of 
-  glusterfs rather than the usual open-read-close set of operations. For files of small sizes, it is recommended 
-  to use extended attribute interface.
-
-* With the above configuration all the requests to httpd of the form www.example.org/glusterfs/path/to/file are 
-  served from glusterfs.
-
-* mod_glusterfs also implements mod_dir and mod_autoindex behaviour for files under glusterfs mount.
-  Hence it also takes the directives related to both of these modules. For more details, refer the 
-  documentation for both of these modules. 
-
-Miscellaneous points
-====================
-* httpd by default runs with username "nobody" and group "nogroup". Permissions of logfile and specfile have to 
-  be set suitably.
-
-* Since mod_glusterfs runs with permissions of nobody.nogroup, glusterfs has to use only login based 
-  authentication. See docs/authentication.txt for more details. 
-
-* To copy the data served by httpd into glusterfs mountpoint, glusterfs can be started with the 
-  volume-specification file provided to mod_glusterfs. Any tool like cp can then be used.
-
-* To run in gdb, apache has to be compiled with -lpthread, since libglusterfsclient is 
-  multithreaded. If not on Linux gdb runs into errors like: 
-  "Error while reading shared library symbols:
-   Cannot find new threads: generic error"
-
-* when used with ib-verbs transport, ib_verbs initialization fails.
-  reason for this is that apache runs as non-privileged user and the amount of memory that can be 
-  locked by default is not sufficient for ib-verbs. to fix this, as root run,
-  
-  # ulimit -l unlimited
-
-  and then start apache.
diff --git a/mod_glusterfs/apache/2.2/src/mod_glusterfs.c b/mod_glusterfs/apache/2.2/src/mod_glusterfs.c
deleted file mode 100644
index d2b9f3232..000000000
--- a/mod_glusterfs/apache/2.2/src/mod_glusterfs.c
+++ /dev/null
@@ -1,3627 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef CORE_PRIVATE
-#define CORE_PRIVATE
-#endif
-
-#ifndef NO_CONTENT_TYPE
-#define NO_CONTENT_TYPE "none"
-#endif
-
-#define BYTERANGE_FMT "%" APR_OFF_T_FMT "-%" APR_OFF_T_FMT "/%" APR_OFF_T_FMT
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include <httpd.h>
-#include <http_config.h>
-#include <http_core.h>
-#include <http_request.h>
-#include <http_protocol.h>
-#include <http_log.h>
-#include <http_main.h>
-#include <util_script.h>
-#include <util_filter.h>
-#include <libglusterfsclient.h>
-#include <sys/uio.h>
-#include <pthread.h>
-#include <apr.h>
-#include <apr_strings.h>
-#include <apr_buckets.h>
-#include <apr_fnmatch.h>
-#include <apr_lib.h>
-
-#define GLUSTERFS_INVALID_LOGLEVEL "mod_glfs: Unrecognized log-level \"%s\", "\
-                                   " possible values are \"DEBUG|WARNING|"\
-                                   "ERROR|CRITICAL|NONE\"\n"
-
-#define GLUSTERFS_HANDLER "glusterfs-handler"
-#define GLUSTERFS_CHUNK_SIZE 131072 
-
-static char c_by_encoding, c_by_type, c_by_path;
-
-#define BY_ENCODING &c_by_encoding
-#define BY_TYPE &c_by_type
-#define BY_PATH &c_by_path
-
-module AP_MODULE_DECLARE_DATA glusterfs_module;
-extern module core_module;
-
-#define NO_OPTIONS          (1 <<  0)  /* Indexing options */
-#define ICONS_ARE_LINKS     (1 <<  1)
-#define SCAN_HTML_TITLES    (1 <<  2)
-#define SUPPRESS_ICON       (1 <<  3)
-#define SUPPRESS_LAST_MOD   (1 <<  4)
-#define SUPPRESS_SIZE       (1 <<  5)
-#define SUPPRESS_DESC       (1 <<  6)
-#define SUPPRESS_PREAMBLE   (1 <<  7)
-#define SUPPRESS_COLSORT    (1 <<  8)
-#define SUPPRESS_RULES      (1 <<  9)
-#define FOLDERS_FIRST       (1 << 10)
-#define VERSION_SORT        (1 << 11)
-#define TRACK_MODIFIED      (1 << 12)
-#define FANCY_INDEXING      (1 << 13)
-#define TABLE_INDEXING      (1 << 14)
-#define IGNORE_CLIENT       (1 << 15)
-#define IGNORE_CASE         (1 << 16)
-#define EMIT_XHTML          (1 << 17)
-#define SHOW_FORBIDDEN      (1 << 18)
-
-#define K_NOADJUST 0
-#define K_ADJUST 1
-#define K_UNSET 2
-
-/*
- * Define keys for sorting.
- */
-#define K_NAME 'N'              /* Sort by file name (default) */
-#define K_LAST_MOD 'M'          /* Last modification date */
-#define K_SIZE 'S'              /* Size (absolute, not as displayed) */
-#define K_DESC 'D'              /* Description */
-#define K_VALID "NMSD"          /* String containing _all_ valid K_ opts */
-
-#define D_ASCENDING 'A'
-#define D_DESCENDING 'D'
-#define D_VALID "AD"            /* String containing _all_ valid D_ opts */
-
-/*
- * These are the dimensions of the default icons supplied with Apache.
- */
-#define DEFAULT_ICON_WIDTH 20
-#define DEFAULT_ICON_HEIGHT 22
-
-/*
- * Other default dimensions.
- */
-#define DEFAULT_NAME_WIDTH 23
-#define DEFAULT_DESC_WIDTH 23
-
-struct mod_glfs_ai_item {
-        char *type;
-        char *apply_to;
-        char *apply_path;
-        char *data;
-};
-
-typedef struct mod_glfs_ai_desc_t {
-        char *pattern;
-        char *description;
-        int full_path;
-        int wildcards;
-} mod_glfs_ai_desc_t;
-
-typedef enum {
-        SLASH_OFF = 0,
-        SLASH_ON,
-        SLASH_UNSET
-} mod_glfs_dir_slash_cfg;
-
-/* static ap_filter_rec_t *mod_glfs_output_filter_handle; */
-
-/*TODO: verify error returns to server core */
-
-typedef struct glusterfs_dir_config {
-        char                   *logfile;
-        char                   *loglevel;
-        char                   *specfile;
-        char                   *mount_dir;
-        char                   *buf;
-
-        size_t                  xattr_file_size;
-        uint32_t                cache_timeout;
-        
-        /* mod_dir options */
-        apr_array_header_t     *index_names;
-        mod_glfs_dir_slash_cfg  do_slash;
-
-        /* autoindex options */
-        char                   *default_icon;
-        char                   *style_sheet;
-        apr_int32_t             opts;
-        apr_int32_t             incremented_opts;
-        apr_int32_t             decremented_opts;
-        int                     name_width;
-        int                     name_adjust;
-        int                     desc_width;
-        int                     desc_adjust;
-        int                     icon_width;
-        int                     icon_height;
-        char                    default_keyid;
-        char                    default_direction;
-
-        apr_array_header_t     *icon_list;
-        apr_array_header_t     *alt_list;
-        apr_array_header_t     *desc_list;
-        apr_array_header_t     *ign_list;
-        apr_array_header_t     *hdr_list;
-        apr_array_header_t     *rdme_list;
-
-        char                   *ctype;
-        char                   *charset;
-} glusterfs_dir_config_t;
-
-typedef struct glusterfs_async_local {
-        int                op_ret;
-        int                op_errno;
-        char               async_read_complete;
-        off_t              length;
-        off_t              read_bytes;
-        glusterfs_iobuf_t *buf;
-        request_rec       *request;
-        pthread_mutex_t    lock;
-        pthread_cond_t     cond;
-}glusterfs_async_local_t;
-
-#define GLUSTERFS_CMD_PERMS ACCESS_CONF
-
-
-static glusterfs_dir_config_t *
-mod_glfs_dconfig (request_rec *r)
-{
-        glusterfs_dir_config_t *dir_config = NULL;
-        if (r->per_dir_config != NULL) {
-                dir_config = ap_get_module_config (r->per_dir_config,
-                                                   &glusterfs_module);
-        }
-
-        return dir_config;
-}
-
-
-static const char *
-cmd_add_xattr_file_size (cmd_parms *cmd, void *dummy, const char *arg)
-{
-        glusterfs_dir_config_t *dir_config = dummy;
-        dir_config->xattr_file_size = atoi (arg);
-        return NULL;
-}
-
-
-static const char *
-cmd_set_cache_timeout (cmd_parms *cmd, void *dummy, const char *arg)
-{
-        glusterfs_dir_config_t *dir_config = dummy;
-        dir_config->cache_timeout = atoi (arg);
-        return NULL;
-}
-
-
-static const char *
-cmd_set_loglevel (cmd_parms *cmd, void *dummy, const char *arg)
-{
-        glusterfs_dir_config_t *dir_config = dummy;
-        char                   *error = NULL;
-        if (strncasecmp (arg, "DEBUG", strlen ("DEBUG")) 
-            && strncasecmp (arg, "WARNING", strlen ("WARNING")) 
-            && strncasecmp (arg, "CRITICAL", strlen ("CRITICAL")) 
-            && strncasecmp (arg, "NONE", strlen ("NONE")) 
-            && strncasecmp (arg, "ERROR", strlen ("ERROR")))
-                error = GLUSTERFS_INVALID_LOGLEVEL;
-        else
-                dir_config->loglevel = apr_pstrdup (cmd->pool, arg);
-
-        return error;
-}
-
-static const char *
-cmd_add_logfile (cmd_parms *cmd, void *dummy, const char *arg)
-{
-        glusterfs_dir_config_t *dir_config = dummy;
-        dir_config->logfile = apr_pstrdup (cmd->pool, arg);
-
-        return NULL;
-}
-
-
-static const char *
-cmd_add_volume_specfile (cmd_parms *cmd, void *dummy, const char *arg)
-{
-        glusterfs_dir_config_t *dir_config = dummy;
-
-        dir_config->specfile = apr_pstrdup (cmd->pool, arg);
-
-        return NULL;
-}
-
-#define WILDCARDS_REQUIRED 0
-
-static const char *
-cmd_add_desc (cmd_parms *cmd, void *d, const char *desc,
-              const char *to)
-{
-        glusterfs_dir_config_t *dcfg = NULL;
-        mod_glfs_ai_desc_t     *desc_entry = NULL;
-        char                   *prefix = "";
-
-        dcfg = (glusterfs_dir_config_t *) d;
-        desc_entry = (mod_glfs_ai_desc_t *) apr_array_push(dcfg->desc_list);
-        desc_entry->full_path = (ap_strchr_c(to, '/') == NULL) ? 0 : 1;
-        desc_entry->wildcards = (WILDCARDS_REQUIRED
-                                 || desc_entry->full_path
-                                 || apr_fnmatch_test(to));
-        if (desc_entry->wildcards) {
-                prefix = desc_entry->full_path ? "*/" : "*";
-                desc_entry->pattern = apr_pstrcat(dcfg->desc_list->pool,
-                                                  prefix, to, "*", NULL);
-        }
-        else {
-                desc_entry->pattern = apr_pstrdup(dcfg->desc_list->pool, to);
-        }
-        desc_entry->description = apr_pstrdup(dcfg->desc_list->pool, desc);
-        return NULL;
-}
-
-
-static void push_item(apr_array_header_t *arr, char *type, const char *to,
-                      const char *path, const char *data)
-{
-        struct mod_glfs_ai_item *p = NULL;
-
-        p = (struct mod_glfs_ai_item *) apr_array_push(arr);
-
-        if (!to) {
-                to = "";
-        }
-        if (!path) {
-                path = "";
-        }
-
-        p->type = type;
-        p->data = data ? apr_pstrdup(arr->pool, data) : NULL;
-        p->apply_path = apr_pstrcat(arr->pool, path, "*", NULL);
-
-        if ((type == BY_PATH) && (!ap_is_matchexp(to))) {
-                p->apply_to = apr_pstrcat(arr->pool, "*", to, NULL);
-        }
-        else if (to) {
-                p->apply_to = apr_pstrdup(arr->pool, to);
-        }
-        else {
-                p->apply_to = NULL;
-        }
-}
-
-
-static const char *
-cmd_add_ignore (cmd_parms *cmd, void *d, const char *ext)
-{
-        push_item(((glusterfs_dir_config_t *) d)->ign_list, 0, ext, cmd->path,
-                  NULL);
-        return NULL;
-}
-
-
-static const char *
-cmd_add_header (cmd_parms *cmd, void *d, const char *name)
-{
-        push_item(((glusterfs_dir_config_t *) d)->hdr_list, 0, NULL, cmd->path,
-                  name);
-        return NULL;
-}
-
-
-static const char *
-cmd_add_readme (cmd_parms *cmd, void *d, const char *name)
-{
-        push_item(((glusterfs_dir_config_t *) d)->rdme_list, 0, NULL, cmd->path,
-                  name);
-        return NULL;
-}
-
-
-static const char *
-cmd_add_opts (cmd_parms *cmd, void *d, int argc, char *const argv[])
-{
-        int                     i = 0, option = 0;
-        char                   *w = NULL;
-        apr_int32_t             opts;
-        apr_int32_t             opts_add;
-        apr_int32_t             opts_remove;
-        char                    action = 0;
-        glusterfs_dir_config_t *d_cfg = (glusterfs_dir_config_t *) d;
-
-        opts = d_cfg->opts;
-        opts_add = d_cfg->incremented_opts;
-        opts_remove = d_cfg->decremented_opts;
-
-        for (i = 0; i < argc; i++) {
-                w = argv[i];
-
-                if ((*w == '+') || (*w == '-')) {
-                        action = *(w++);
-                }
-                else {
-                        action = '\0';
-                }
-                if (!strcasecmp(w, "FancyIndexing")) {
-                        option = FANCY_INDEXING;
-                }
-                else if (!strcasecmp(w, "FoldersFirst")) {
-                        option = FOLDERS_FIRST;
-                }
-                else if (!strcasecmp(w, "HTMLTable")) {
-                        option = TABLE_INDEXING;
-                }
-                else if (!strcasecmp(w, "IconsAreLinks")) {
-                        option = ICONS_ARE_LINKS;
-                }
-                else if (!strcasecmp(w, "IgnoreCase")) {
-                        option = IGNORE_CASE;
-                }
-                else if (!strcasecmp(w, "IgnoreClient")) {
-                        option = IGNORE_CLIENT;
-                }
-                else if (!strcasecmp(w, "ScanHTMLTitles")) {
-                        option = SCAN_HTML_TITLES;
-                }
-                else if (!strcasecmp(w, "SuppressColumnSorting")) {
-                        option = SUPPRESS_COLSORT;
-                }
-                else if (!strcasecmp(w, "SuppressDescription")) {
-                        option = SUPPRESS_DESC;
-                }
-                else if (!strcasecmp(w, "SuppressHTMLPreamble")) {
-                        option = SUPPRESS_PREAMBLE;
-                }
-                else if (!strcasecmp(w, "SuppressIcon")) {
-                        option = SUPPRESS_ICON;
-                }
-                else if (!strcasecmp(w, "SuppressLastModified")) {
-                        option = SUPPRESS_LAST_MOD;
-                }
-                else if (!strcasecmp(w, "SuppressSize")) {
-                        option = SUPPRESS_SIZE;
-                }
-                else if (!strcasecmp(w, "SuppressRules")) {
-                        option = SUPPRESS_RULES;
-                }
-                else if (!strcasecmp(w, "TrackModified")) {
-                        option = TRACK_MODIFIED;
-                }
-                else if (!strcasecmp(w, "VersionSort")) {
-                        option = VERSION_SORT;
-                }
-                else if (!strcasecmp(w, "XHTML")) {
-                        option = EMIT_XHTML;
-                }
-                else if (!strcasecmp(w, "ShowForbidden")) {
-                        option = SHOW_FORBIDDEN;
-                }
-                else if (!strcasecmp(w, "None")) {
-                        if (action != '\0') {
-                                return "Cannot combine '+' or '-' with 'None' "
-                                        "keyword";
-                        }
-                        opts = NO_OPTIONS;
-                        opts_add = 0;
-                        opts_remove = 0;
-                }
-                else if (!strcasecmp(w, "IconWidth")) {
-                        if (action != '-') {
-                                d_cfg->icon_width = DEFAULT_ICON_WIDTH;
-                        }
-                        else {
-                                d_cfg->icon_width = 0;
-                        }
-                }
-                else if (!strncasecmp(w, "IconWidth=", 10)) {
-                        if (action == '-') {
-                                return "Cannot combine '-' with IconWidth=n";
-                        }
-                        d_cfg->icon_width = atoi(&w[10]);
-                }
-                else if (!strcasecmp(w, "IconHeight")) {
-                        if (action != '-') {
-                                d_cfg->icon_height = DEFAULT_ICON_HEIGHT;
-                        }
-                        else {
-                                d_cfg->icon_height = 0;
-                        }
-                }
-                else if (!strncasecmp(w, "IconHeight=", 11)) {
-                        if (action == '-') {
-                                return "Cannot combine '-' with IconHeight=n";
-                        }
-                        d_cfg->icon_height = atoi(&w[11]);
-                }
-                else if (!strcasecmp(w, "NameWidth")) {
-                        if (action != '-') {
-                                return "NameWidth with no value may only appear"
-                                        " as "
-                                        "'-NameWidth'";
-                        }
-                        d_cfg->name_width = DEFAULT_NAME_WIDTH;
-                        d_cfg->name_adjust = K_NOADJUST;
-                }
-                else if (!strncasecmp(w, "NameWidth=", 10)) {
-                        if (action == '-') {
-                                return "Cannot combine '-' with NameWidth=n";
-                        }
-                        if (w[10] == '*') {
-                                d_cfg->name_adjust = K_ADJUST;
-                        }
-                        else {
-                                int width = atoi(&w[10]);
-
-                                if (width && (width < 5)) {
-                                        return "NameWidth value must be greater"
-                                                " than 5";
-                                }
-                                d_cfg->name_width = width;
-                                d_cfg->name_adjust = K_NOADJUST;
-                        }
-                }
-                else if (!strcasecmp(w, "DescriptionWidth")) {
-                        if (action != '-') {
-                                return "DescriptionWidth with no value may only"
-                                        " appear as "
-                                        "'-DescriptionWidth'";
-                        }
-                        d_cfg->desc_width = DEFAULT_DESC_WIDTH;
-                        d_cfg->desc_adjust = K_NOADJUST;
-                }
-                else if (!strncasecmp(w, "DescriptionWidth=", 17)) {
-                        if (action == '-') {
-                                return "Cannot combine '-' with "
-                                        "DescriptionWidth=n";
-                        }
-                        if (w[17] == '*') {
-                                d_cfg->desc_adjust = K_ADJUST;
-                        }
-                        else {
-                                int width = atoi(&w[17]);
-
-                                if (width && (width < 12)) {
-                                        return "DescriptionWidth value must be "
-                                                "greater than 12";
-                                }
-                                d_cfg->desc_width = width;
-                                d_cfg->desc_adjust = K_NOADJUST;
-                        }
-                }
-                else if (!strncasecmp(w, "Type=", 5)) {
-                        d_cfg->ctype = apr_pstrdup(cmd->pool, &w[5]);
-                }
-                else if (!strncasecmp(w, "Charset=", 8)) {
-                        d_cfg->charset = apr_pstrdup(cmd->pool, &w[8]);
-                }
-                else {
-                        return "Invalid directory indexing option";
-                }
-                if (action == '\0') {
-                        opts |= option;
-                        opts_add = 0;
-                        opts_remove = 0;
-                }
-                else if (action == '+') {
-                        opts_add |= option;
-                        opts_remove &= ~option;
-                }
-                else {
-                        opts_remove |= option;
-                        opts_add &= ~option;
-                }
-        }
-        if ((opts & NO_OPTIONS) && (opts & ~NO_OPTIONS)) {
-                return "Cannot combine other IndexOptions keywords with 'None'";
-        }
-        d_cfg->incremented_opts = opts_add;
-        d_cfg->decremented_opts = opts_remove;
-        d_cfg->opts = opts;
-        return NULL;
-}
-
-
-static const char *
-cmd_set_default_order(cmd_parms *cmd, void *m,
-                      const char *direction, const char *key)
-{
-        glusterfs_dir_config_t *d_cfg = (glusterfs_dir_config_t *) m;
-
-        if (!strcasecmp(direction, "Ascending")) {
-                d_cfg->default_direction = D_ASCENDING;
-        }
-        else if (!strcasecmp(direction, "Descending")) {
-                d_cfg->default_direction = D_DESCENDING;
-        }
-        else {
-                return "First keyword must be 'Ascending' or 'Descending'";
-        }
-
-        if (!strcasecmp(key, "Name")) {
-                d_cfg->default_keyid = K_NAME;
-        }
-        else if (!strcasecmp(key, "Date")) {
-                d_cfg->default_keyid = K_LAST_MOD;
-        }
-        else if (!strcasecmp(key, "Size")) {
-                d_cfg->default_keyid = K_SIZE;
-        }
-        else if (!strcasecmp(key, "Description")) {
-                d_cfg->default_keyid = K_DESC;
-        }
-        else {
-                return "Second keyword must be 'Name', 'Date', 'Size', or "
-                        "'Description'";
-        }
-
-        return NULL;
-}
-
-
-static char c_by_encoding, c_by_type, c_by_path;
-
-#define BY_ENCODING &c_by_encoding
-#define BY_TYPE &c_by_type
-#define BY_PATH &c_by_path
-
-/*
- * This routine puts the standard HTML header at the top of the index page.
- * We include the DOCTYPE because we may be using features therefrom (i.e.,
- * HEIGHT and WIDTH attributes on the icons if we're FancyIndexing).
- */
-static void emit_preamble(request_rec *r, int xhtml, const char *title)
-{
-        glusterfs_dir_config_t *d;
-
-        d = (glusterfs_dir_config_t *) ap_get_module_config(r->per_dir_config,
-                                                            &glusterfs_module);
-
-        if (xhtml) {
-                ap_rvputs(r, DOCTYPE_XHTML_1_0T,
-                          "<html xmlns=\"http://www.w3.org/1999/xhtml\">\n"
-                          " <head>\n  <title>Index of ", title,
-                          "</title>\n", NULL);
-        } else {
-                ap_rvputs(r, DOCTYPE_HTML_3_2,
-                          "<html>\n <head>\n"
-                          "  <title>Index of ", title,
-                          "</title>\n", NULL);
-        }
-
-        if (d->style_sheet != NULL) {
-                ap_rvputs(r, "  <link rel=\"stylesheet\" href=\"",
-                          d->style_sheet, "\" type=\"text/css\"",
-                          xhtml ? " />\n" : ">\n", NULL);
-        }
-        ap_rvputs(r, " </head>\n <body>\n", NULL);
-}
-
-
-static const char *cmd_add_alt(cmd_parms *cmd, void *d, const char *alt,
-                               const char *to)
-{
-        if (cmd->info == BY_PATH) {
-                if (!strcmp(to, "**DIRECTORY**")) {
-                        to = "^^DIRECTORY^^";
-                }
-        }
-        if (cmd->info == BY_ENCODING) {
-                char *tmp = apr_pstrdup(cmd->pool, to);
-                ap_str_tolower(tmp);
-                to = tmp;
-        }
-
-        push_item(((glusterfs_dir_config_t *) d)->alt_list, cmd->info, to,
-                  cmd->path, alt);
-        return NULL;
-}
-
-static const char *cmd_add_icon(cmd_parms *cmd, void *d, const char *icon,
-                                const char *to)
-{
-        char *iconbak = apr_pstrdup(cmd->pool, icon);
-        char *alt = NULL, *cl = NULL, *tmp = NULL;
-
-        if (icon[0] == '(') {
-                cl = strchr(iconbak, ')');
-
-                if (cl == NULL) {
-                        return "missing closing paren";
-                }
-                alt = ap_getword_nc(cmd->pool, &iconbak, ',');
-                *cl = '\0';                             /* Lose closing paren */
-                cmd_add_alt(cmd, d, &alt[1], to);
-        }
-        if (cmd->info == BY_PATH) {
-                if (!strcmp(to, "**DIRECTORY**")) {
-                        to = "^^DIRECTORY^^";
-                }
-        }
-        if (cmd->info == BY_ENCODING) {
-                tmp = apr_pstrdup(cmd->pool, to);
-                ap_str_tolower(tmp);
-                to = tmp;
-        }
-
-        push_item(((glusterfs_dir_config_t *) d)->icon_list, cmd->info, to,
-                  cmd->path, iconbak);
-        return NULL;
-}
-
-
-static void *
-mod_glfs_create_dir_config(apr_pool_t *p, char *dirspec)
-{
-        glusterfs_dir_config_t *dir_config = NULL;
-
-        dir_config = (glusterfs_dir_config_t *) apr_pcalloc(p,
-                                                            sizeof(*dir_config));
-
-        dir_config->mount_dir = dirspec;
-        dir_config->logfile = dir_config->specfile = (char *)0;
-        dir_config->loglevel = "warning";
-        dir_config->cache_timeout = 0;
-        dir_config->buf = NULL;
-
-        /* mod_dir options init */
-        dir_config->index_names = NULL;
-        dir_config->do_slash = SLASH_UNSET;
-
-        /* autoindex options init */
-        dir_config->icon_width = 0;
-        dir_config->icon_height = 0;
-        dir_config->name_width = DEFAULT_NAME_WIDTH;
-        dir_config->name_adjust = K_UNSET;
-        dir_config->desc_width = DEFAULT_DESC_WIDTH;
-        dir_config->desc_adjust = K_UNSET;
-        dir_config->icon_list = apr_array_make(p, 4,
-                                               sizeof(struct mod_glfs_ai_item));
-        dir_config->alt_list = apr_array_make(p, 4,
-                                              sizeof(struct mod_glfs_ai_item));
-        dir_config->desc_list = apr_array_make(p, 4,
-                                               sizeof(mod_glfs_ai_desc_t));
-        dir_config->ign_list = apr_array_make(p, 4,
-                                              sizeof(struct mod_glfs_ai_item));
-        dir_config->hdr_list = apr_array_make(p, 4,
-                                              sizeof(struct mod_glfs_ai_item));
-        dir_config->rdme_list = apr_array_make(p, 4,
-                                               sizeof(struct mod_glfs_ai_item));
-        dir_config->opts = 0;
-        dir_config->incremented_opts = 0;
-        dir_config->decremented_opts = 0;
-        dir_config->default_keyid = '\0';
-        dir_config->default_direction = '\0';
-
-        return (void *) dir_config;
-}
-
-
-static void *
-mod_glfs_merge_dir_config(apr_pool_t *p, void *parent_conf,
-                          void *newloc_conf)
-{
-        glusterfs_dir_config_t *new = NULL;
-        glusterfs_dir_config_t *add = NULL;
-        glusterfs_dir_config_t *base = NULL;
-
-        new = (glusterfs_dir_config_t *) 
-                apr_pcalloc(p, sizeof(glusterfs_dir_config_t));
-        add = newloc_conf;
-        base = parent_conf;
-
-        if (add->logfile)
-                new->logfile = apr_pstrdup (p, add->logfile);
-
-        if (add->loglevel)
-                new->loglevel = apr_pstrdup (p, add->loglevel);
-
-        if (add->specfile)
-                new->specfile = apr_pstrdup (p, add->specfile);
-
-        if (add->mount_dir)
-                new->mount_dir = apr_pstrdup (p, add->mount_dir);
-
-        new->xattr_file_size = add->xattr_file_size;
-        new->cache_timeout = add->cache_timeout;
-        new->buf = add->buf;
-
-        /* mod_dir */
-        new->index_names = add->index_names ? 
-                add->index_names : base->index_names;
-        new->do_slash =
-                (add->do_slash == SLASH_UNSET) ? base->do_slash : add->do_slash;
-
-        /* auto index */
-        new->default_icon = add->default_icon ? add->default_icon
-                : base->default_icon;
-        new->style_sheet = add->style_sheet ? add->style_sheet
-                : base->style_sheet;
-        new->icon_height = add->icon_height ? 
-                add->icon_height : base->icon_height;
-        new->icon_width = add->icon_width ? add->icon_width : base->icon_width;
-
-        new->ctype = add->ctype ? add->ctype : base->ctype;
-        new->charset = add->charset ? add->charset : base->charset;
-
-        new->alt_list = apr_array_append(p, add->alt_list, base->alt_list);
-        new->ign_list = apr_array_append(p, add->ign_list, base->ign_list);
-        new->hdr_list = apr_array_append(p, add->hdr_list, base->hdr_list);
-        new->desc_list = apr_array_append(p, add->desc_list, base->desc_list);
-        new->icon_list = apr_array_append(p, add->icon_list, base->icon_list);
-        new->rdme_list = apr_array_append(p, add->rdme_list, base->rdme_list);
-        if (add->opts & NO_OPTIONS) {
-                /*
-                 * If the current directory says 'no options' then we also
-                 * clear any incremental mods from being inheritable further down.
-                 */
-                new->opts = NO_OPTIONS;
-                new->incremented_opts = 0;
-                new->decremented_opts = 0;
-        }
-        else {
-                /*
-                 * If there were any nonincremental options selected for
-                 * this directory, they dominate and we don't inherit *anything.*
-                 * Contrariwise, we *do* inherit if the only settings here are
-                 * incremental ones.
-                 */
-                if (add->opts == 0) {
-                        new->incremented_opts = (base->incremented_opts
-                                                 | add->incremented_opts)
-                                & ~add->decremented_opts;
-                        new->decremented_opts = (base->decremented_opts
-                                                 | add->decremented_opts);
-                        /*
-                         * We may have incremental settings, so make sure we 
-                         * don't inadvertently inherit an IndexOptions None 
-                         * from above.
-                         */
-                        new->opts = (base->opts & ~NO_OPTIONS);
-                }
-                else {
-                        /*
-                         * There are local nonincremental settings, which clear
-                         * all inheritance from above.  They *are* the new 
-                         * base settings.
-                         */
-                        new->opts = add->opts;;
-                }
-                /*
-                 * We're guaranteed that there'll be no overlap between
-                 * the add-options and the remove-options.
-                 */
-                new->opts |= new->incremented_opts;
-                new->opts &= ~new->decremented_opts;
-        }
-        /*
-         * Inherit the NameWidth settings if there aren't any specific to
-         * the new location; otherwise we'll end up using the defaults set 
-         * in the config-rec creation routine.
-         */
-        if (add->name_adjust == K_UNSET) {
-                new->name_width = base->name_width;
-                new->name_adjust = base->name_adjust;
-        }
-        else {
-                new->name_width = add->name_width;
-                new->name_adjust = add->name_adjust;
-        }
-
-        /*
-         * Likewise for DescriptionWidth.
-         */
-        if (add->desc_adjust == K_UNSET) {
-                new->desc_width = base->desc_width;
-                new->desc_adjust = base->desc_adjust;
-        }
-        else {
-                new->desc_width = add->desc_width;
-                new->desc_adjust = add->desc_adjust;
-        }
-
-        new->default_keyid = add->default_keyid ? add->default_keyid
-                : base->default_keyid;
-        new->default_direction = add->default_direction ? add->default_direction
-                : base->default_direction;
-
-        return (void *) new;
-}
-
-
-static void 
-mod_glfs_child_init(apr_pool_t *p, server_rec *s)
-{
-        int                      i = 0, num_sec = 0, ret = 0;
-        core_server_config      *sconf = NULL;
-        ap_conf_vector_t       **sec_ent = NULL;
-        glusterfs_dir_config_t  *dir_config = NULL;
-        glusterfs_init_params_t  ctx = {0, };
-  
-        sconf = (core_server_config *) ap_get_module_config (s->module_config,
-                                                             &core_module);
-        sec_ent = (ap_conf_vector_t **) sconf->sec_url->elts;
-        num_sec = sconf->sec_url->nelts;
-
-        for (i = 0; i < num_sec; i++) {
-                dir_config = ap_get_module_config (sec_ent[i],
-                                                   &glusterfs_module);
-
-                if (dir_config) {
-                        memset (&ctx, 0, sizeof (ctx));
-
-                        ctx.logfile = dir_config->logfile;
-                        ctx.loglevel = dir_config->loglevel;
-                        ctx.lookup_timeout = dir_config->cache_timeout;
-                        ctx.stat_timeout = dir_config->cache_timeout;
-                        ctx.specfile = dir_config->specfile;
-
-                        ret = glusterfs_mount (dir_config->mount_dir, &ctx);
-                        if (ret != 0) {
-                                ap_log_error(APLOG_MARK, APLOG_ERR,
-                                             APR_EGENERAL, s, 
-                                             "mod_glfs_child_init: "
-                                             "glusterfs_init failed, check "
-                                             "glusterfs logfile %s for more "
-                                             "details", 
-                                             dir_config->logfile);
-                        }
-                }
-                dir_config = NULL;
-        }
-}
-
-
-static void 
-mod_glfs_child_exit(server_rec *s, apr_pool_t *p)
-{
-        int                      i = 0, num_sec = 0;
-        core_server_config      *sconf = NULL;
-        ap_conf_vector_t       **sec_ent = NULL;
-        glusterfs_dir_config_t  *dir_config = NULL;
-        glusterfs_init_params_t  ctx = {0, };
-
-        sconf = ap_get_module_config(s->module_config,
-                                     &core_module);
-        sec_ent = (ap_conf_vector_t **) sconf->sec_url->elts;
-        num_sec = sconf->sec_url->nelts;
-        for (i = 0; i < num_sec; i++) {
-                dir_config = ap_get_module_config (sec_ent[i],
-                                                   &glusterfs_module);
-                if (dir_config) {
-                        glusterfs_umount (dir_config->mount_dir);
-                }
-        }
-}
-
-static apr_filetype_e filetype_from_mode(mode_t mode)
-{
-        apr_filetype_e type = APR_NOFILE;
-
-        if (S_ISREG(mode))
-                type = APR_REG;
-        else if (S_ISDIR(mode))
-                type = APR_DIR;
-        else if (S_ISCHR(mode))
-                type = APR_CHR;
-        else if (S_ISBLK(mode))
-                type = APR_BLK;
-        else if (S_ISFIFO(mode))
-                type = APR_PIPE;
-        else if (S_ISLNK(mode))
-                type = APR_LNK;
-        else if (S_ISSOCK(mode))
-                type = APR_SOCK;
-        else
-                type = APR_UNKFILE;
-        return type;
-}
-
-
-static void fill_out_finfo(apr_finfo_t *finfo, struct stat *info,
-                           apr_int32_t wanted)
-{ 
-        finfo->valid = APR_FINFO_MIN | APR_FINFO_IDENT | APR_FINFO_NLINK
-                | APR_FINFO_OWNER | APR_FINFO_PROT;
-        finfo->protection = apr_unix_mode2perms(info->st_mode);
-        finfo->filetype = filetype_from_mode(info->st_mode);
-        finfo->user = info->st_uid;
-        finfo->group = info->st_gid;
-        finfo->size = info->st_size;
-        finfo->device = info->st_dev;
-        finfo->nlink = info->st_nlink;
-
-        /* Check for overflow if storing a 64-bit st_ino in a 32-bit
-         * apr_ino_t for LFS builds: */
-        if (sizeof(apr_ino_t) >= sizeof(info->st_ino)
-            || (apr_ino_t)info->st_ino == info->st_ino) {
-                finfo->inode = info->st_ino;
-        } else {
-                finfo->valid &= ~APR_FINFO_INODE;
-        }
-
-        apr_time_ansi_put(&finfo->atime, info->st_atime);
-#ifdef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC
-        finfo->atime += info->st_atim.tv_nsec / APR_TIME_C(1000);
-#elif defined(HAVE_STRUCT_STAT_ST_ATIMENSEC)
-        finfo->atime += info->st_atimensec / APR_TIME_C(1000);
-#elif defined(HAVE_STRUCT_STAT_ST_ATIME_N)
-        finfo->ctime += info->st_atime_n / APR_TIME_C(1000);
-#endif
-
-        apr_time_ansi_put(&finfo->mtime, info->st_mtime);
-#ifdef HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC
-        finfo->mtime += info->st_mtim.tv_nsec / APR_TIME_C(1000);
-#elif defined(HAVE_STRUCT_STAT_ST_MTIMENSEC)
-        finfo->mtime += info->st_mtimensec / APR_TIME_C(1000);
-#elif defined(HAVE_STRUCT_STAT_ST_MTIME_N)
-        finfo->ctime += info->st_mtime_n / APR_TIME_C(1000);
-#endif
-
-        apr_time_ansi_put(&finfo->ctime, info->st_ctime);
-#ifdef HAVE_STRUCT_STAT_ST_CTIM_TV_NSEC
-        finfo->ctime += info->st_ctim.tv_nsec / APR_TIME_C(1000);
-#elif defined(HAVE_STRUCT_STAT_ST_CTIMENSEC)
-        finfo->ctime += info->st_ctimensec / APR_TIME_C(1000);
-#elif defined(HAVE_STRUCT_STAT_ST_CTIME_N)
-        finfo->ctime += info->st_ctime_n / APR_TIME_C(1000);
-#endif
-
-#ifdef HAVE_STRUCT_STAT_ST_BLOCKS
-#ifdef DEV_BSIZE
-        finfo->csize = (apr_off_t)info->st_blocks * (apr_off_t)DEV_BSIZE;
-#else
-        finfo->csize = (apr_off_t)info->st_blocks * (apr_off_t)512;
-#endif
-        finfo->valid |= APR_FINFO_CSIZE;
-#endif
-}
-
-
-static int 
-mod_glfs_map_to_storage(request_rec *r)
-{
-        glusterfs_dir_config_t *dir_config = NULL, *tmp = NULL;
-        int                     access_status = 0, ret = 0;
-        char                   *path = NULL;
-        struct stat             st = {0, };
-        core_server_config     *sconf = NULL;
-        ap_conf_vector_t      **sec_ent = NULL;
-        int                     num_sec = 0, i = 0;
-
-        sconf = (core_server_config *) ap_get_module_config (r->server->module_config,
-                                                             &core_module);
-        sec_ent = (ap_conf_vector_t **) sconf->sec_url->elts;
-        num_sec = sconf->sec_url->nelts;
-
-        for (i = 0; i < num_sec; i++) {
-                tmp = ap_get_module_config (sec_ent[i], &glusterfs_module);
-
-                if (tmp && !strncmp (tmp->mount_dir, r->uri,
-                                     strlen (tmp->mount_dir))) {
-                        if (!dir_config || 
-                            strlen (tmp->mount_dir) 
-                            > strlen (dir_config->mount_dir)) {
-                                dir_config = tmp;
-                        }
-                }
-
-        }
-
-        if (dir_config && dir_config->mount_dir 
-            && !(strncmp (apr_pstrcat (r->pool, dir_config->mount_dir, "/",
-                                       NULL), r->uri,
-                          strlen (dir_config->mount_dir) + 1)
-                 && !r->handler)) 
-                r->handler = GLUSTERFS_HANDLER;
-
-        if (!r->handler || (r->handler && strcmp (r->handler,
-                                                  GLUSTERFS_HANDLER)))
-                return DECLINED;
-
-        path = r->uri;
-
-        memset (&r->finfo, 0, sizeof (r->finfo));
-
-        dir_config->buf = calloc (1, dir_config->xattr_file_size);
-        if (!dir_config->buf) {
-                return HTTP_INTERNAL_SERVER_ERROR;
-        }
-
-        ret = glusterfs_get (path, dir_config->buf, 
-                             dir_config->xattr_file_size, &st);
-
-        if (ret == -1 || st.st_size > dir_config->xattr_file_size 
-            || S_ISDIR (st.st_mode)) {
-                free (dir_config->buf);
-                dir_config->buf = NULL;
-
-                if (ret == -1) {
-                        int error = HTTP_NOT_FOUND;
-                        char *emsg = NULL;
-                        if (r->path_info == NULL) {
-                                emsg = apr_pstrcat(r->pool, strerror (errno),
-                                                   r->filename, NULL);
-                        }
-                        else {
-                                emsg = apr_pstrcat(r->pool, strerror (errno),
-                                                   r->filename, r->path_info,
-                                                   NULL);
-                        }
-                        ap_log_rerror(APLOG_MARK, APLOG_ERR|APLOG_NOERRNO, 0,
-                                      r, "%s", emsg);
-                        if (errno != ENOENT) {
-                                error = HTTP_INTERNAL_SERVER_ERROR;
-                        }
-                        return error;
-                }
-        }
-
-        r->finfo.pool = r->pool;
-        r->finfo.fname = r->filename;
-        fill_out_finfo (&r->finfo, &st, 
-                        APR_FINFO_MIN | APR_FINFO_IDENT | APR_FINFO_NLINK |
-                        APR_FINFO_OWNER | APR_FINFO_PROT);
-                        
-        /* allow core module to run directory_walk() and location_walk() */
-        return DECLINED;
-}
-
-
-static int 
-mod_glfs_readv_async_cbk (int32_t op_ret, int32_t op_errno,
-                          glusterfs_iobuf_t *buf, void *cbk_data)
-{
-        glusterfs_async_local_t *local = cbk_data;
-
-        pthread_mutex_lock (&local->lock);
-        {
-                local->async_read_complete = 1;
-                local->buf = buf;
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                pthread_cond_signal (&local->cond);
-        }
-        pthread_mutex_unlock (&local->lock);
-
-        return 0;
-}
-
-/* use read_async just to avoid memcpy of read buffer in libglusterfsclient */
-static int
-mod_glfs_read_async (request_rec *r, apr_bucket_brigade *bb,
-                     glusterfs_file_t fd,
-                     apr_off_t offset, apr_off_t length)
-{
-        glusterfs_async_local_t local = {0, };
-        off_t                   end = 0;
-        int                     nbytes = 0, complete = 0;
-        conn_rec               *c = r->connection;
-        apr_bucket             *e = NULL;
-        apr_status_t            status = APR_SUCCESS;
-        glusterfs_iobuf_t      *buf = NULL;
-                
-        if (length == 0) {
-                return 0;
-        }
-
-        pthread_cond_init (&local.cond, NULL);
-        pthread_mutex_init (&local.lock, NULL);
-  
-        memset (&local, 0, sizeof (local));
-        local.request = r;
-
-        if (length > 0)
-                end = offset + length;
-
-        do {
-                if (length > 0) {
-                        nbytes = end - offset;
-                        if (nbytes > GLUSTERFS_CHUNK_SIZE)
-                                nbytes = GLUSTERFS_CHUNK_SIZE;
-                } else
-                        nbytes = GLUSTERFS_CHUNK_SIZE;
-
-                glusterfs_read_async(fd, 
-                                     nbytes,
-                                     offset,
-                                     mod_glfs_readv_async_cbk,
-                                     (void *)&local);
-
-                pthread_mutex_lock (&local.lock);
-                {
-                        while (!local.async_read_complete) {
-                                pthread_cond_wait (&local.cond, &local.lock);
-                        }
-
-                        local.async_read_complete = 0;
-                        buf = local.buf;
-
-                        if (length < 0)
-                                complete = (local.op_ret <= 0);
-                        else {
-                                local.read_bytes += local.op_ret;
-                                complete = ((local.read_bytes == length) ||
-                                            (local.op_ret < 0));
-                        }
-                }
-                pthread_mutex_unlock (&local.lock);
-
-                if (!bb) {
-                        bb = apr_brigade_create (r->pool, c->bucket_alloc); 
-                }
-                apr_brigade_writev (bb, NULL, NULL, buf->vector, buf->count);
-
-                /* 
-                 * make sure all the data is written out, since we call 
-                 * glusterfs_free on buf once ap_pass_brigade returns
-                 */
-                e = apr_bucket_flush_create (c->bucket_alloc);
-                APR_BRIGADE_INSERT_TAIL (bb, e);
-
-                status = ap_pass_brigade (r->output_filters, bb);
-                if (status != APR_SUCCESS) {
-                        /* no way to know what type of error occurred */
-                        ap_log_rerror(APLOG_MARK, APLOG_DEBUG, status, r,
-                                      "mod_glfs_handler: ap_pass_brigade "
-                                      "returned %i",
-                                      status);
-                        complete = 1;
-                        local.op_ret = -1;
-                }
-
-                glusterfs_free (buf);
-
-                /* 
-                 * bb has already been cleaned up by core_output_filter, 
-                 * just being paranoid
-                 */
-                apr_brigade_cleanup (bb);
-
-                offset += nbytes;
-        } while (!complete);
-
-        return (local.op_ret < 0 ? HTTP_INTERNAL_SERVER_ERROR : OK);
-}
-
-static int 
-parse_byterange(char *range, apr_off_t clength,
-                apr_off_t *start, apr_off_t *end)
-{
-        char       *dash = NULL, *errp = NULL;
-        apr_off_t   number;
-
-        dash = strchr(range, '-');
-        if (!dash) {
-                return 0;
-        }
-
-        if ((dash == range)) {
-                /* In the form "-5" */
-                if (apr_strtoff(&number, dash+1, &errp, 10) || *errp) {
-                        return 0;
-                }
-                *start = clength - number;
-                *end = clength - 1; 
-        }
-        else {
-                *dash++ = '\0';
-                if (apr_strtoff(&number, range, &errp, 10) || *errp) {
-                        return 0;
-                }
-                *start = number;
-                if (*dash) {
-                        if (apr_strtoff(&number, dash, &errp, 10) || *errp) {
-                                return 0;
-                        }
-                        *end = number;
-                }
-                else {                  /* "5-" */
-                        *end = clength - 1;
-                }
-        }
-
-        if (*start < 0) {
-                *start = 0;
-        }
-
-        if (*end >= clength) {
-                *end = clength - 1;
-        }
-
-        if (*start > *end) {
-                return -1;
-        }
-
-        return (*start > 0 || *end < clength);
-}
-
-
-static int use_range_x(request_rec *r)
-{
-        const char *ua = NULL;
-        return (apr_table_get(r->headers_in, "Request-Range")
-                || ((ua = apr_table_get(r->headers_in, "User-Agent"))
-                    && ap_strstr_c(ua, "MSIE 3")));
-}
-
-
-static int ap_set_byterange(request_rec *r)
-{
-        const char *range = NULL, *if_range = NULL, *match = NULL, *ct = NULL;
-        int         num_ranges = 0;
-
-        if (r->assbackwards) {
-                return 0;
-        }
-
-        /* Check for Range request-header (HTTP/1.1) or Request-Range for
-         * backwards-compatibility with second-draft Luotonen/Franks
-         * byte-ranges (e.g. Netscape Navigator 2-3).
-         *
-         * We support this form, with Request-Range, and (farther down) we
-         * send multipart/x-byteranges instead of multipart/byteranges for
-         * Request-Range based requests to work around a bug in Netscape
-         * Navigator 2-3 and MSIE 3.
-         */
-
-        if (!(range = apr_table_get(r->headers_in, "Range"))) {
-                range = apr_table_get(r->headers_in, "Request-Range");
-        }
-
-        if (!range || strncasecmp(range, "bytes=", 6) || r->status != HTTP_OK) {
-                return 0;
-        }
-
-        /* is content already a single range? */
-        if (apr_table_get(r->headers_out, "Content-Range")) {
-                return 0;
-        }
-
-        /* is content already a multiple range? */
-        if ((ct = apr_table_get(r->headers_out, "Content-Type"))
-            && (!strncasecmp(ct, "multipart/byteranges", 20)
-                || !strncasecmp(ct, "multipart/x-byteranges", 22))) {
-                return 0;
-        }
-
-        /* Check the If-Range header for Etag or Date.
-         * Note that this check will return false (as required) if either
-         * of the two etags are weak.
-         */
-        if ((if_range = apr_table_get(r->headers_in, "If-Range"))) {
-                if (if_range[0] == '"') {
-                        if (!(match = apr_table_get(r->headers_out, "Etag"))
-                            || (strcmp(if_range, match) != 0)) {
-                                return 0;
-                        }
-                }
-                else if (!(match = apr_table_get(r->headers_out,
-                                                 "Last-Modified"))
-                         || (strcmp(if_range, match) != 0)) {
-                        return 0;
-                }
-        }
-
-        if (!ap_strchr_c(range, ',')) {
-                /* a single range */
-                num_ranges = 1;
-        }
-        else {
-                /* a multiple range */
-                num_ranges = 2;
-        }
-
-        r->status = HTTP_PARTIAL_CONTENT;
-        r->range = range + 6;
-
-        return num_ranges;
-}
-
-
-static void
-mod_glfs_handle_byte_ranges (request_rec *r, glusterfs_file_t fd,
-                             int num_ranges)
-{
-        conn_rec           *c = r->connection;
-        char               *ts = NULL, *boundary = NULL, *bound_head = NULL;
-        const char         *orig_ct = NULL;
-        char               *current = NULL, *end = NULL;
-        apr_bucket_brigade *bsend = NULL;
-        apr_bucket         *e = NULL;
-        apr_off_t           range_start, range_end;
-        apr_status_t        rv = APR_SUCCESS;
-        char                found = 0;
-        apr_bucket         *e2 = NULL, *ec = NULL;
-
-        orig_ct = ap_make_content_type (r, r->content_type);
-
-        if (num_ranges > 1) {
-                boundary = apr_psprintf(r->pool, "%" APR_UINT64_T_HEX_FMT "%lx",
-                                        (apr_uint64_t)r->request_time,
-                                        (long) getpid());
-
-                ap_set_content_type(r, apr_pstrcat(r->pool, "multipart",
-                                                   use_range_x(r) ? "/x-" : "/",
-                                                   "byteranges; boundary=",
-                                                   boundary, NULL));
-
-                if (strcasecmp(orig_ct, NO_CONTENT_TYPE)) {
-                        bound_head = apr_pstrcat(r->pool,
-                                                 CRLF "--", boundary,
-                                                 CRLF "Content-type: ",
-                                                 orig_ct,
-                                                 CRLF "Content-range: bytes ",
-                                                 NULL);
-                }
-                else {
-                        /* if we have no type for the content, do our best */
-                        bound_head = apr_pstrcat(r->pool,
-                                                 CRLF "--", boundary,
-                                                 CRLF "Content-range: bytes ",
-                                                 NULL);
-                }
-        }
-
-        while ((current = ap_getword(r->pool, &r->range, ','))
-               && (rv = parse_byterange(current, r->finfo.size, &range_start,
-                                        &range_end))) {
-                bsend = NULL;
-                if (rv == -1) {
-                        continue;
-                }
-
-                found = 1;
-
-                /* For single range requests, we must produce Content-Range 
-                 * header. Otherwise, we need to produce the multipart 
-                 * boundaries.
-                 */
-                if (num_ranges == 1) {
-                        apr_table_setn(r->headers_out, "Content-Range",
-                                       apr_psprintf(r->pool,
-                                                    "bytes " BYTERANGE_FMT,
-                                                    range_start, range_end,
-                                                    r->finfo.size));
-                }
-                else {
-                        /* this brigade holds what we will be sending */
-                        bsend = apr_brigade_create(r->pool, c->bucket_alloc);
-
-                        e = apr_bucket_pool_create(bound_head,
-                                                   strlen(bound_head),
-                                                   r->pool, c->bucket_alloc);
-                        APR_BRIGADE_INSERT_TAIL(bsend, e);
-
-                        ts = apr_psprintf(r->pool, BYTERANGE_FMT CRLF CRLF,
-                                          range_start, range_end,
-                                          r->finfo.size);
-                        e = apr_bucket_pool_create(ts, strlen(ts), r->pool,
-                                                   c->bucket_alloc);
-                        APR_BRIGADE_INSERT_TAIL(bsend, e);
-                }
-                mod_glfs_read_async (r, bsend, fd, range_start,
-                                     (range_end + 1 - range_start));
-        }
-
-        bsend = apr_brigade_create (r->pool, c->bucket_alloc);
-
-        if (found == 0) {
-                r->status = HTTP_OK;
-                /* bsend is assumed to be empty if we get here. */
-                e = ap_bucket_error_create(HTTP_RANGE_NOT_SATISFIABLE, NULL,
-                                           r->pool, c->bucket_alloc);
-                APR_BRIGADE_INSERT_TAIL(bsend, e);
-                e = apr_bucket_eos_create(c->bucket_alloc);
-                APR_BRIGADE_INSERT_TAIL(bsend, e);
-                ap_pass_brigade (r->output_filters, bsend);
-                return;
-        }
-
-        if (num_ranges > 1) {
-                /* add the final boundary */
-                end = apr_pstrcat(r->pool, CRLF "--", boundary, "--" CRLF,
-                                  NULL);
-//                ap_xlate_proto_to_ascii(end, strlen(end));
-                e = apr_bucket_pool_create(end, strlen(end), r->pool,
-                                           c->bucket_alloc);
-                APR_BRIGADE_INSERT_TAIL(bsend, e);
-        }
-
-        ap_pass_brigade (r->output_filters, bsend);
-}
-
-
-
-/****************************************************************
- *
- * Looking things up in config entries...
- */
-
-/* Structure used to hold entries when we're actually building an index */
-
-struct ent {
-        char       *name;
-        char       *icon;
-        char       *alt;
-        char       *desc;
-        apr_off_t   size;
-        apr_time_t  lm;
-        struct ent *next;
-        int         ascending, ignore_case, version_sort;
-        char        key;
-        int         isdir;
-};
-
-static char *find_item(request_rec *r, apr_array_header_t *list, int path_only)
-{
-        const char              *content_type = NULL;
-        const char              *content_encoding = NULL;
-        char                    *path = NULL;
-        int                      i = 0;
-        struct mod_glfs_ai_item *items = NULL;
-        struct mod_glfs_ai_item *p = NULL;
-
-        content_type = ap_field_noparam(r->pool, r->content_type);
-        content_encoding = r->content_encoding;
-        path = r->filename;
-        items = (struct mod_glfs_ai_item *) list->elts;
-
-        for (i = 0; i < list->nelts; ++i) {
-                p = &items[i];
-                /* Special cased for ^^DIRECTORY^^ and ^^BLANKICON^^ */
-                if ((path[0] == '^') || (!ap_strcmp_match(path,
-                                                          p->apply_path))) {
-                        if (!*(p->apply_to)) {
-                                return p->data;
-                        }
-                        else if (p->type == BY_PATH || path[0] == '^') {
-                                if (!ap_strcmp_match(path, p->apply_to)) {
-                                        return p->data;
-                                }
-                        }
-                        else if (!path_only) {
-                                if (!content_encoding) {
-                                        if (p->type == BY_TYPE) {
-                                                if (content_type
-                                                    && !ap_strcasecmp_match(content_type,
-                                                                            p->apply_to)) {
-                                                        return p->data;
-                                                }
-                                        }
-                                }
-                                else {
-                                        if (p->type == BY_ENCODING) {
-                                                if (!ap_strcasecmp_match(content_encoding,
-                                                                         p->apply_to)) {
-                                                        return p->data;
-                                                }
-                                        }
-                                }
-                        }
-                }
-        }
-        return NULL;
-}
-
-#define find_icon(d,p,t) find_item(p,d->icon_list,t)
-#define find_alt(d,p,t) find_item(p,d->alt_list,t)
-#define find_header(d,p) find_item(p,d->hdr_list,0)
-#define find_readme(d,p) find_item(p,d->rdme_list,0)
-
-static char *find_default_item(char *bogus_name, apr_array_header_t *list)
-{
-        request_rec r;
-        /* Bleah.  I tried to clean up find_item, and it lead to this bit
-         * of ugliness.   Note that the fields initialized are precisely
-         * those that find_item looks at...
-         */
-        r.filename = bogus_name;
-        r.content_type = r.content_encoding = NULL;
-        return find_item(&r, list, 1);
-}
-
-#define find_default_icon(d,n) find_default_item(n, d->icon_list)
-#define find_default_alt(d,n) find_default_item(n, d->alt_list)
-
-/*
- * Look through the list of pattern/description pairs and return the first one
- * if any) that matches the filename in the request.  If multiple patterns
- * match, only the first one is used; since the order in the array is the
- * same as the order in which directives were processed, earlier matching
- * directives will dominate.
- */
-
-#ifdef CASE_BLIND_FILESYSTEM
-#define MATCH_FLAGS APR_FNM_CASE_BLIND
-#else
-#define MATCH_FLAGS 0
-#endif
-
-static char *find_desc(glusterfs_dir_config_t *dcfg, const char *filename_full)
-{
-        int                 i = 0;
-        mod_glfs_ai_desc_t *list = NULL;
-        const char         *filename_only = NULL, *filename = NULL;
-        mod_glfs_ai_desc_t *tuple = &list[i];
-        int                 found = 0;
-
-        list = (mod_glfs_ai_desc_t *) dcfg->desc_list->elts;
-        /*
-         * If the filename includes a path, extract just the name itself
-         * for the simple matches.
-         */
-        if ((filename_only = ap_strrchr_c(filename_full, '/')) == NULL) {
-                filename_only = filename_full;
-        }
-        else {
-                filename_only++;
-        }
-        for (i = 0; i < dcfg->desc_list->nelts; ++i) {
-                /*
-                 * Only use the full-path filename if the pattern contains '/'s.
-                 */
-                filename = (tuple->full_path) ? filename_full : filename_only;
-                /*
-                 * Make the comparison using the cheapest method; only do
-                 * wildcard checking if we must.
-                 */
-                if (tuple->wildcards) {
-                        found = (apr_fnmatch(tuple->pattern, filename,
-                                             MATCH_FLAGS) == 0);
-                }
-                else {
-                        found = (ap_strstr_c(filename, tuple->pattern) != NULL);
-                }
-                if (found) {
-                        return tuple->description;
-                }
-        }
-        return NULL;
-}
-
-static int ignore_entry(glusterfs_dir_config_t *d, char *path)
-{
-        apr_array_header_t      *list = d->ign_list;
-        struct mod_glfs_ai_item *items = (struct mod_glfs_ai_item *) list->elts;
-        char                    *tt = NULL, *ap = NULL;
-        int                      i = 0;
-        struct mod_glfs_ai_item *p = &items[i];
-
-        if ((tt = strrchr(path, '/')) == NULL) {
-                tt = path;
-        }
-        else {
-                tt++;
-        }
-
-        for (i = 0; i < list->nelts; ++i) {
-                p = &items[i];
-                if ((ap = strrchr(p->apply_to, '/')) == NULL) {
-                        ap = p->apply_to;
-                }
-                else {
-                        ap++;
-                }
-
-#ifndef CASE_BLIND_FILESYSTEM
-                if (!ap_strcmp_match(path, p->apply_path)
-                    && !ap_strcmp_match(tt, ap)) {
-                        return 1;
-                }
-#else  /* !CASE_BLIND_FILESYSTEM */
-                /*
-                 * On some platforms, the match must be case-blind.  This is really
-                 * a factor of the filesystem involved, but we can't detect that
-                 * reliably - so we have to granularise at the OS level.
-                 */
-                if (!ap_strcasecmp_match(path, p->apply_path)
-                    && !ap_strcasecmp_match(tt, ap)) {
-                        return 1;
-                }
-#endif /* !CASE_BLIND_FILESYSTEM */
-        }
-        return 0;
-}
-
-/*****************************************************************
- *
- * Actually generating output
- */
-
-/*
- * Elements of the emitted document:
- *      Preamble
- *              Emitted unless SUPPRESS_PREAMBLE is set AND ap_run_sub_req
- *              succeeds for the (content_type == text/html) header file.
- *      Header file
- *              Emitted if found (and able).
- *      H1 tag line
- *              Emitted if a header file is NOT emitted.
- *      Directory stuff
- *              Always emitted.
- *      HR
- *              Emitted if FANCY_INDEXING is set.
- *      Readme file
- *              Emitted if found (and able).
- *      ServerSig
- *              Emitted if ServerSignature is not Off AND a readme file
- *              is NOT emitted.
- *      Postamble
- *              Emitted unless SUPPRESS_PREAMBLE is set AND ap_run_sub_req
- *              succeeds for the (content_type == text/html) readme file.
- */
-
-
-/*
- * emit a plain text file
- */
-static void do_emit_plain(request_rec *r, apr_file_t *f)
-{
-        char         buf[AP_IOBUFSIZE + 1];
-        int          ch = 0;
-        apr_size_t   i = 0, c = 0, n = 0;
-        apr_status_t rv = APR_SUCCESS;
-
-        ap_rputs("<pre>\n", r);
-        while (!apr_file_eof(f)) {
-                do {
-                        n = sizeof(char) * AP_IOBUFSIZE;
-                        rv = apr_file_read(f, buf, &n);
-                } while (APR_STATUS_IS_EINTR(rv));
-                if (n == 0 || rv != APR_SUCCESS) {
-                        /* ###: better error here? */
-                        break;
-                }
-                buf[n] = '\0';
-                c = 0;
-                while (c < n) {
-                        for (i = c; i < n; i++) {
-                                if (buf[i] == '<' || buf[i] == '>'
-                                    || buf[i] == '&') {
-                                        break;
-                                }
-                        }
-                        ch = buf[i];
-                        buf[i] = '\0';
-                        ap_rputs(&buf[c], r);
-                        if (ch == '<') {
-                                ap_rputs("&lt;", r);
-                        }
-                        else if (ch == '>') {
-                                ap_rputs("&gt;", r);
-                        }
-                        else if (ch == '&') {
-                                ap_rputs("&amp;", r);
-                        }
-                        c = i + 1;
-                }
-        }
-        ap_rputs("</pre>\n", r);
-}
-
-/*
- * Handle the preamble through the H1 tag line, inclusive.  Locate
- * the file with a subrequests.  Process text/html documents by actually
- * running the subrequest; text/xxx documents get copied verbatim,
- * and any other content type is ignored.  This means that a non-text
- * document (such as HEADER.gif) might get multiviewed as the result
- * instead of a text document, meaning nothing will be displayed, but
- * oh well.
- */
-static void emit_head(request_rec *r, char *header_fname, int suppress_amble,
-                      int emit_xhtml, char *title)
-{
-        apr_table_t *hdrs = r->headers_in;
-        apr_file_t  *f = NULL;
-        request_rec *rr = NULL;
-        int          emit_amble = 1;
-        int          emit_H1 = 1;
-        const char  *r_accept = NULL;
-        const char  *r_accept_enc = NULL;
-
-        /*
-         * If there's a header file, send a subrequest to look for it.  If it's
-         * found and html do the subrequest, otherwise handle it
-         */
-        r_accept = apr_table_get(hdrs, "Accept");
-        r_accept_enc = apr_table_get(hdrs, "Accept-Encoding");
-        apr_table_setn(hdrs, "Accept", "text/html, text/plain");
-        apr_table_unset(hdrs, "Accept-Encoding");
-
-
-        if ((header_fname != NULL) && r->args) {
-                header_fname = apr_pstrcat(r->pool, header_fname, "?", r->args,
-                                           NULL);
-        }
-
-        if ((header_fname != NULL)
-            && (rr = ap_sub_req_lookup_uri(header_fname, r, r->output_filters))
-            && (rr->status == HTTP_OK)
-            && (rr->filename != NULL)
-            && (rr->finfo.filetype == APR_REG)) {
-                /*
-                 * Check for the two specific cases we allow: text/html and
-                 * text/anything-else.  The former is allowed to be processed for
-                 * SSIs.
-                 */
-                if (rr->content_type != NULL) {
-                        if (!strcasecmp(ap_field_noparam(r->pool,
-                                                         rr->content_type),
-                                        "text/html")) {
-                                ap_filter_t *f = NULL;
-        
-                                /* Hope everything will work... */
-                                emit_amble = 0;
-                                emit_H1 = 0;
-
-                                if (! suppress_amble) {
-                                        emit_preamble(r, emit_xhtml, title);
-                                }
-                                /* This is a hack, but I can't find any better 
-                                 * way to do this. The problem is that we have 
-                                 * already created the sub-request,
-                                 * but we just inserted the OLD_WRITE filter, 
-                                 * and the sub-request needs to pass its data 
-                                 * through the OLD_WRITE filter, or things go 
-                                 * horribly wrong (missing data, data in
-                                 * the wrong order, etc).  To fix it, if you 
-                                 * create a sub-request and then insert the 
-                                 * OLD_WRITE filter before you run the request, 
-                                 * you need to make sure that the sub-request
-                                 * data goes through the OLD_WRITE filter.  Just
-                                 * steal this code.  The long-term solution is 
-                                 * to remove the ap_r* functions.
-                                 */
-                                for (f=rr->output_filters;
-                                     f->frec != ap_subreq_core_filter_handle;
-                                     f = f->next);
-                                f->next = r->output_filters;
-
-                                /*
-                                 * If there's a problem running the subrequest, 
-                                 * display the preamble if we didn't do it 
-                                 * before -- the header file didn't get displayed.
-                                 */
-                                if (ap_run_sub_req(rr) != OK) {
-                                        /* It didn't work */
-                                        emit_amble = suppress_amble;
-                                        emit_H1 = 1;
-                                }
-                        }
-                        else if (!strncasecmp("text/", rr->content_type, 5)) {
-                                /*
-                                 * If we can open the file, prefix it with the 
-                                 * preamble regardless; since we'll be sending 
-                                 * a <pre> block around the file's contents, 
-                                 * any HTML header it had won't end up
-                                 * where it belongs.
-                                 */
-                                if (apr_file_open(&f, rr->filename, APR_READ,
-                                                  APR_OS_DEFAULT, r->pool) 
-                                    == APR_SUCCESS) {
-                                        emit_preamble(r, emit_xhtml, title);
-                                        emit_amble = 0;
-                                        do_emit_plain(r, f);
-                                        apr_file_close(f);
-                                        emit_H1 = 0;
-                                }
-                        }
-                }
-        }
-
-        if (r_accept) {
-                apr_table_setn(hdrs, "Accept", r_accept);
-        }
-        else {
-                apr_table_unset(hdrs, "Accept");
-        }
-
-        if (r_accept_enc) {
-                apr_table_setn(hdrs, "Accept-Encoding", r_accept_enc);
-        }
-
-        if (emit_amble) {
-                emit_preamble(r, emit_xhtml, title);
-        }
-        if (emit_H1) {
-                ap_rvputs(r, "<h1>Index of ", title, "</h1>\n", NULL);
-        }
-        if (rr != NULL) {
-                ap_destroy_sub_req(rr);
-        }
-}
-
-
-/*
- * Handle the Readme file through the postamble, inclusive.  Locate
- * the file with a subrequests.  Process text/html documents by actually
- * running the subrequest; text/xxx documents get copied verbatim,
- * and any other content type is ignored.  This means that a non-text
- * document (such as FOOTER.gif) might get multiviewed as the result
- * instead of a text document, meaning nothing will be displayed, but
- * oh well.
- */
-static void emit_tail(request_rec *r, char *readme_fname, int suppress_amble)
-{
-        apr_file_t  *f = NULL;
-        request_rec *rr = NULL;
-        int          suppress_post = 0, suppress_sig = 0;
-
-        /*
-         * If there's a readme file, send a subrequest to look for it.  If it's
-         * found and a text file, handle it -- otherwise fall through and
-         * pretend there's nothing there.
-         */
-        if ((readme_fname != NULL)
-            && (rr = ap_sub_req_lookup_uri(readme_fname, r, r->output_filters))
-            && (rr->status == HTTP_OK)
-            && (rr->filename != NULL)
-            && rr->finfo.filetype == APR_REG) {
-                /*
-                 * Check for the two specific cases we allow: text/html and
-                 * text/anything-else.  The former is allowed to be processed for
-                 * SSIs.
-                 */
-                if (rr->content_type != NULL) {
-                        if (!strcasecmp(ap_field_noparam(r->pool,
-                                                         rr->content_type),
-                                        "text/html")) {
-                                ap_filter_t *f;
-                                for (f=rr->output_filters;
-                                     f->frec != ap_subreq_core_filter_handle;
-                                     f = f->next);
-                                f->next = r->output_filters;
-
-
-                                if (ap_run_sub_req(rr) == OK) {
-                                        /* worked... */
-                                        suppress_sig = 1;
-                                        suppress_post = suppress_amble;
-                                }
-                        }
-                        else if (!strncasecmp("text/", rr->content_type, 5)) {
-                                /*
-                                 * If we can open the file, suppress the signature.
-                                 */
-                                if (apr_file_open(&f, rr->filename, APR_READ,
-                                                  APR_OS_DEFAULT, r->pool)
-                                    == APR_SUCCESS) {
-                                        do_emit_plain(r, f);
-                                        apr_file_close(f);
-                                        suppress_sig = 1;
-                                }
-                        }
-                }
-        }
-
-        if (!suppress_sig) {
-                ap_rputs(ap_psignature("", r), r);
-        }
-        if (!suppress_post) {
-                ap_rputs("</body></html>\n", r);
-        }
-        if (rr != NULL) {
-                ap_destroy_sub_req(rr);
-        }
-}
-
-
-static char *find_title(request_rec *r)
-{
-        char        titlebuf[MAX_STRING_LEN], *find = "<title>";
-        apr_file_t *thefile = NULL;
-        int         x = 0, y = 0, p = 0;
-        apr_size_t  n;
-
-        if (r->status != HTTP_OK) {
-                return NULL;
-        }
-        if ((r->content_type != NULL)
-            && (!strcasecmp(ap_field_noparam(r->pool, r->content_type),
-                            "text/html")
-                || !strcmp(r->content_type, INCLUDES_MAGIC_TYPE))
-            && !r->content_encoding) {
-                if (apr_file_open(&thefile, r->filename, APR_READ,
-                                  APR_OS_DEFAULT, r->pool) != APR_SUCCESS) {
-                        return NULL;
-                }
-                n = sizeof(char) * (MAX_STRING_LEN - 1);
-                apr_file_read(thefile, titlebuf, &n);
-                if (n <= 0) {
-                        apr_file_close(thefile);
-                        return NULL;
-                }
-                titlebuf[n] = '\0';
-                for (x = 0, p = 0; titlebuf[x]; x++) {
-                        if (apr_tolower(titlebuf[x]) == find[p]) {
-                                if (!find[++p]) {
-                                        if ((p = ap_ind(&titlebuf[++x], '<'))
-                                            != -1) {
-                                                titlebuf[x + p] = '\0';
-                                        }
-                                        /* Scan for line breaks for Tanmoy's 
-                                           secretary
-                                        */
-                                        for (y = x; titlebuf[y]; y++) {
-                                                if ((titlebuf[y] == CR) 
-                                                    || (titlebuf[y] == LF)) {
-                                                        if (y == x) {
-                                                                x++;
-                                                        }
-                                                        else {
-                                                                titlebuf[y] = ' ';
-                                                        }
-                                                }
-                                        }
-                                        apr_file_close(thefile);
-                                        return apr_pstrdup(r->pool,
-                                                           &titlebuf[x]);
-                                }
-                        }
-                        else {
-                                p = 0;
-                        }
-                }
-                apr_file_close(thefile);
-        }
-        return NULL;
-}
-
-static struct ent *make_parent_entry(apr_int32_t autoindex_opts,
-                                     glusterfs_dir_config_t *d,
-                                     request_rec *r, char keyid,
-                                     char direction)
-{
-        struct ent *p = NULL;
-        char       *testpath = NULL;
-        /*
-         * p->name is now the true parent URI.
-         * testpath is a crafted lie, so that the syntax '/some/..'
-         * (or simply '..')be used to describe 'up' from '/some/'
-         * when processeing IndexIgnore, and Icon|Alt|Desc configs.
-         */
-
-        p = (struct ent *) apr_pcalloc(r->pool, sizeof(struct ent));
-        /* The output has always been to the parent.  Don't make ourself
-         * our own parent (worthless cyclical reference).
-         */
-        if (!(p->name = ap_make_full_path(r->pool, r->uri, "../"))) {
-                return (NULL);
-        }
-        ap_getparents(p->name);
-        if (!*p->name) {
-                return (NULL);
-        }
-
-        /* IndexIgnore has always compared "/thispath/.." */
-        testpath = ap_make_full_path(r->pool, r->filename, "..");
-        if (ignore_entry(d, testpath)) {
-                return (NULL);
-        }
-
-        p->size = -1;
-        p->lm = -1;
-        p->key = apr_toupper(keyid);
-        p->ascending = (apr_toupper(direction) == D_ASCENDING);
-        p->version_sort = autoindex_opts & VERSION_SORT;
-        if (autoindex_opts & FANCY_INDEXING) {
-                if (!(p->icon = find_default_icon(d, testpath))) {
-                        p->icon = find_default_icon(d, "^^DIRECTORY^^");
-                }
-                if (!(p->alt = find_default_alt(d, testpath))) {
-                        if (!(p->alt = find_default_alt(d, "^^DIRECTORY^^"))) {
-                                p->alt = "DIR";
-                        }
-                }
-                p->desc = find_desc(d, testpath);
-        }
-        return p;
-}
-
-static struct ent *make_autoindex_entry(const apr_finfo_t *dirent,
-                                        int autoindex_opts,
-                                        glusterfs_dir_config_t *d,
-                                        request_rec *r, char keyid,
-                                        char direction,
-                                        const char *pattern)
-{
-        request_rec *rr = NULL;
-        struct ent  *p = NULL;
-        int          show_forbidden = 0;
-
-        /* Dot is ignored, Parent is handled by make_parent_entry() */
-        if ((dirent->name[0] == '.') && (!dirent->name[1]
-                                         || ((dirent->name[1] == '.')
-                                             && !dirent->name[2])))
-                return (NULL);
-
-        /*
-         * On some platforms, the match must be case-blind.  This is really
-         * a factor of the filesystem involved, but we can't detect that
-         * reliably - so we have to granularise at the OS level.
-         */
-        if (pattern && (apr_fnmatch(pattern, dirent->name,
-                                    APR_FNM_NOESCAPE | APR_FNM_PERIOD
-#ifdef CASE_BLIND_FILESYSTEM
-                                    | APR_FNM_CASE_BLIND
-#endif
-                                )
-                        != APR_SUCCESS)) {
-                return (NULL);
-        }
-
-        if (ignore_entry(d, ap_make_full_path(r->pool,
-                                              r->filename, dirent->name))) {
-                return (NULL);
-        }
-
-        if (!(rr = ap_sub_req_lookup_dirent(dirent, r, AP_SUBREQ_NO_ARGS,
-                                            NULL))) {
-                return (NULL);
-        }
-
-        if((autoindex_opts & SHOW_FORBIDDEN)
-           && (rr->status == HTTP_UNAUTHORIZED
-               || rr->status == HTTP_FORBIDDEN)) {
-                show_forbidden = 1;
-        }
-
-        if ((rr->finfo.filetype != APR_DIR && rr->finfo.filetype != APR_REG)
-            || !(rr->status == OK || ap_is_HTTP_SUCCESS(rr->status)
-                 || ap_is_HTTP_REDIRECT(rr->status)
-                 || show_forbidden == 1)) {
-                ap_destroy_sub_req(rr);
-                return (NULL);
-        }
-
-        p = (struct ent *) apr_pcalloc(r->pool, sizeof(struct ent));
-        if (dirent->filetype == APR_DIR) {
-                p->name = apr_pstrcat(r->pool, dirent->name, "/", NULL);
-        }
-        else {
-                p->name = apr_pstrdup(r->pool, dirent->name);
-        }
-        p->size = -1;
-        p->icon = NULL;
-        p->alt = NULL;
-        p->desc = NULL;
-        p->lm = -1;
-        p->isdir = 0;
-        p->key = apr_toupper(keyid);
-        p->ascending = (apr_toupper(direction) == D_ASCENDING);
-        p->version_sort = !!(autoindex_opts & VERSION_SORT);
-        p->ignore_case = !!(autoindex_opts & IGNORE_CASE);
-
-        if (autoindex_opts & (FANCY_INDEXING | TABLE_INDEXING)) {
-                p->lm = rr->finfo.mtime;
-                if (dirent->filetype == APR_DIR) {
-                        if (autoindex_opts & FOLDERS_FIRST) {
-                                p->isdir = 1;
-                        }
-                        rr->filename = ap_make_dirstr_parent (rr->pool,
-                                                              rr->filename);
-
-                        /* omit the trailing slash (1.3 compat) */
-                        rr->filename[strlen(rr->filename) - 1] = '\0';
-
-                        if (!(p->icon = find_icon(d, rr, 1))) {
-                                p->icon = find_default_icon(d, "^^DIRECTORY^^");
-                        }
-                        if (!(p->alt = find_alt(d, rr, 1))) {
-                                if (!(p->alt = find_default_alt(d,
-                                                                "^^DIRECTORY^^"))) {
-                                        p->alt = "DIR";
-                                }
-                        }
-                }
-                else {
-                        p->icon = find_icon(d, rr, 0);
-                        p->alt = find_alt(d, rr, 0);
-                        p->size = rr->finfo.size;
-                }
-
-                p->desc = find_desc(d, rr->filename);
-
-                if ((!p->desc) && (autoindex_opts & SCAN_HTML_TITLES)) {
-                        p->desc = apr_pstrdup(r->pool, find_title(rr));
-                }
-        }
-        ap_destroy_sub_req(rr);
-        /*
-         * We don't need to take any special action for the file size key.
-         * If we did, it would go here.
-         */
-        if (keyid == K_LAST_MOD) {
-                if (p->lm < 0) {
-                        p->lm = 0;
-                }
-        }
-        return (p);
-}
-
-static char *terminate_description(glusterfs_dir_config_t *d, char *desc,
-                                   apr_int32_t autoindex_opts, int desc_width)
-{
-        int          maxsize = desc_width;
-        register int x = 0;
-
-        /*
-         * If there's no DescriptionWidth in effect, default to the old
-         * behaviour of adjusting the description size depending upon
-         * what else is being displayed.  Otherwise, stick with the
-         * setting.
-         */
-        if (d->desc_adjust == K_UNSET) {
-                if (autoindex_opts & SUPPRESS_ICON) {
-                        maxsize += 6;
-                }
-                if (autoindex_opts & SUPPRESS_LAST_MOD) {
-                        maxsize += 19;
-                }
-                if (autoindex_opts & SUPPRESS_SIZE) {
-                        maxsize += 7;
-                }
-        }
-        for (x = 0; desc[x] && ((maxsize > 0) || (desc[x] == '<')); x++) {
-                if (desc[x] == '<') {
-                        while (desc[x] != '>') {
-                                if (!desc[x]) {
-                                        maxsize = 0;
-                                        break;
-                                }
-                                ++x;
-                        }
-                }
-                else if (desc[x] == '&') {
-                        /* entities like &auml; count as one character */
-                        --maxsize;
-                        for ( ; desc[x] != ';'; ++x) {
-                                if (desc[x] == '\0') {
-                                        maxsize = 0;
-                                        break;
-                                }
-                        }
-                }
-                else {
-                        --maxsize;
-                }
-        }
-        if (!maxsize && desc[x] != '\0') {
-                desc[x - 1] = '>';      /* Grump. */
-                desc[x] = '\0';         /* Double Grump! */
-        }
-        return desc;
-}
-
-/*
- * Emit the anchor for the specified field.  If a field is the key for the
- * current request, the link changes its meaning to reverse the order when
- * selected again.  Non-active fields always start in ascending order.
- */
-static void emit_link(request_rec *r, const char *anchor, char column,
-                      char curkey, char curdirection,
-                      const char *colargs, int nosort)
-{
-        char qvalue[9];
-
-        if (!nosort) {
-
-                qvalue[0] = '?';
-                qvalue[1] = 'C';
-                qvalue[2] = '=';
-                qvalue[3] = column;
-                qvalue[4] = ';';
-                qvalue[5] = 'O';
-                qvalue[6] = '=';
-                /* reverse? */
-                qvalue[7] = ((curkey == column) && (curdirection == D_ASCENDING))
-                        ? D_DESCENDING : D_ASCENDING;
-                qvalue[8] = '\0';
-                ap_rvputs(r, "<a href=\"", qvalue, colargs ? colargs : "",
-                          "\">", anchor, "</a>", NULL);
-        }
-        else {
-                ap_rputs(anchor, r);
-        }
-}
-
-static void output_directories(struct ent **ar, int n,
-                               glusterfs_dir_config_t *d, request_rec *r,
-                               apr_int32_t autoindex_opts, char keyid,
-                               char direction, const char *colargs)
-{
-        int            x = 0;
-        apr_size_t     rv = APR_SUCCESS;
-        char          *name = NULL, *tp = NULL;
-        int            static_columns = 0;
-        apr_pool_t    *scratch = NULL;
-        int            name_width = 0, desc_width = 0, cols = 0;
-        char          *name_scratch = NULL, *pad_scratch = NULL, *breakrow = "";
-        char          *anchor = NULL, *t = NULL, *t2 = NULL;
-        int            nwidth = 0;
-        char           time_str[MAX_STRING_LEN];
-        apr_time_exp_t ts = {0, };
-        char           buf[5];
-
-        name = r->uri;
-        static_columns = !!(autoindex_opts & SUPPRESS_COLSORT);
-        apr_pool_create(&scratch, r->pool);
-        if (name[0] == '\0') {
-                name = "/";
-        }
-
-        name_width = d->name_width;
-        desc_width = d->desc_width;
-
-        if ((autoindex_opts & (FANCY_INDEXING | TABLE_INDEXING))
-            == FANCY_INDEXING) {
-                if (d->name_adjust == K_ADJUST) {
-                        for (x = 0; x < n; x++) {
-                                int t = 0;
-                                t = strlen(ar[x]->name);
-                                if (t > name_width) {
-                                        name_width = t;
-                                }
-                        }
-                }
-
-                if (d->desc_adjust == K_ADJUST) {
-                        for (x = 0; x < n; x++) {
-                                if (ar[x]->desc != NULL) {
-                                        int t = 0;
-                                        t = strlen(ar[x]->desc);
-                                        if (t > desc_width) {
-                                                desc_width = t;
-                                        }
-                                }
-                        }
-                }
-        }
-        name_scratch = apr_palloc(r->pool, name_width + 1);
-        pad_scratch = apr_palloc(r->pool, name_width + 1);
-        memset(pad_scratch, ' ', name_width);
-        pad_scratch[name_width] = '\0';
-
-        if (autoindex_opts & TABLE_INDEXING) {
-                cols = 1;
-                ap_rputs("<table><tr>", r);
-                if (!(autoindex_opts & SUPPRESS_ICON)) {
-                        ap_rputs("<th>", r);
-                        if ((tp = find_default_icon(d, "^^BLANKICON^^"))) {
-                                ap_rvputs(r, "<img src=\"",
-                                          ap_escape_html(scratch, tp),
-                                          "\" alt=\"[ICO]\"", NULL);
-                                if (d->icon_width) {
-                                        ap_rprintf(r, " width=\"%d\"",
-                                                   d->icon_width);
-                                }
-                                if (d->icon_height) {
-                                        ap_rprintf(r, " height=\"%d\"",
-                                                   d->icon_height);
-                                }
-
-                                if (autoindex_opts & EMIT_XHTML) {
-                                        ap_rputs(" /", r);
-                                }
-                                ap_rputs("></th>", r);
-                        }
-                        else {
-                                ap_rputs("&nbsp;</th>", r);
-                        }
-
-                        ++cols;
-                }
-                ap_rputs("<th>", r);
-                emit_link(r, "Name", K_NAME, keyid, direction,
-                          colargs, static_columns);
-                if (!(autoindex_opts & SUPPRESS_LAST_MOD)) {
-                        ap_rputs("</th><th>", r);
-                        emit_link(r, "Last modified", K_LAST_MOD, keyid,
-                                  direction, colargs, static_columns);
-                        ++cols;
-                }
-                if (!(autoindex_opts & SUPPRESS_SIZE)) {
-                        ap_rputs("</th><th>", r);
-                        emit_link(r, "Size", K_SIZE, keyid, direction,
-                                  colargs, static_columns);
-                        ++cols;
-                }
-                if (!(autoindex_opts & SUPPRESS_DESC)) {
-                        ap_rputs("</th><th>", r);
-                        emit_link(r, "Description", K_DESC, keyid, direction,
-                                  colargs, static_columns);
-                        ++cols;
-                }
-                if (!(autoindex_opts & SUPPRESS_RULES)) {
-                        breakrow = apr_psprintf(r->pool,
-                                                "<tr><th colspan=\"%d\">"
-                                                "<hr%s></th></tr>\n", cols,
-                                                (autoindex_opts & EMIT_XHTML) 
-                                                ? " /" : "");
-                }
-                ap_rvputs(r, "</th></tr>", breakrow, NULL);
-        }
-        else if (autoindex_opts & FANCY_INDEXING) {
-                ap_rputs("<pre>", r);
-                if (!(autoindex_opts & SUPPRESS_ICON)) {
-                        if ((tp = find_default_icon(d, "^^BLANKICON^^"))) {
-                                ap_rvputs(r, "<img src=\"",
-                                          ap_escape_html(scratch, tp),
-                                          "\" alt=\"Icon \"", NULL);
-                                if (d->icon_width) {
-                                        ap_rprintf(r, " width=\"%d\"",
-                                                   d->icon_width);
-                                }
-                                if (d->icon_height) {
-                                        ap_rprintf(r, " height=\"%d\"",
-                                                   d->icon_height);
-                                }
-
-                                if (autoindex_opts & EMIT_XHTML) {
-                                        ap_rputs(" /", r);
-                                }
-                                ap_rputs("> ", r);
-                        }
-                        else {
-                                ap_rputs("      ", r);
-                        }
-                }
-                emit_link(r, "Name", K_NAME, keyid, direction,
-                          colargs, static_columns);
-                ap_rputs(pad_scratch + 4, r);
-                /*
-                 * Emit the guaranteed-at-least-one-space-between-columns byte.
-                 */
-                ap_rputs(" ", r);
-                if (!(autoindex_opts & SUPPRESS_LAST_MOD)) {
-                        emit_link(r, "Last modified", K_LAST_MOD, keyid,
-                                  direction, colargs, static_columns);
-                        ap_rputs("      ", r);
-                }
-                if (!(autoindex_opts & SUPPRESS_SIZE)) {
-                        emit_link(r, "Size", K_SIZE, keyid, direction,
-                                  colargs, static_columns);
-                        ap_rputs("  ", r);
-                }
-                if (!(autoindex_opts & SUPPRESS_DESC)) {
-                        emit_link(r, "Description", K_DESC, keyid, direction,
-                                  colargs, static_columns);
-                }
-                if (!(autoindex_opts & SUPPRESS_RULES)) {
-                        ap_rputs("<hr", r);
-                        if (autoindex_opts & EMIT_XHTML) {
-                                ap_rputs(" /", r);
-                        }
-                        ap_rputs(">", r);
-                }
-                else {
-                        ap_rputc('\n', r);
-                }
-        }
-        else {
-                ap_rputs("<ul>", r);
-        }
-
-        for (x = 0; x < n; x++) {
-                apr_pool_clear(scratch);
-
-                t = ar[x]->name;
-                anchor = ap_escape_html(scratch, ap_os_escape_path(scratch, t,
-                                                                   0));
-
-                if (!x && t[0] == '/') {
-                        t2 = "Parent Directory";
-                }
-                else {
-                        t2 = t;
-                }
-
-                if (autoindex_opts & TABLE_INDEXING) {
-                        ap_rputs("<tr>", r);
-                        if (!(autoindex_opts & SUPPRESS_ICON)) {
-                                ap_rputs("<td valign=\"top\">", r);
-                                if (autoindex_opts & ICONS_ARE_LINKS) {
-                                        ap_rvputs(r, "<a href=\"", anchor,
-                                                  "\">", NULL);
-                                }
-                                if ((ar[x]->icon) || d->default_icon) {
-                                        ap_rvputs(r, "<img src=\"",
-                                                  ap_escape_html(scratch,
-                                                                 ar[x]->icon ?
-                                                                 ar[x]->icon
-                                                                 : d->default_icon),
-                                                  "\" alt=\"[",
-                                                  (ar[x]->alt ? 
-                                                   ar[x]->alt : "   "),
-                                                  "]\"", NULL);
-                                        if (d->icon_width) {
-                                                ap_rprintf(r, " width=\"%d\"",
-                                                           d->icon_width);
-                                        }
-                                        if (d->icon_height) {
-                                                ap_rprintf(r, " height=\"%d\"",
-                                                           d->icon_height);
-                                        }
-
-                                        if (autoindex_opts & EMIT_XHTML) {
-                                                ap_rputs(" /", r);
-                                        }
-                                        ap_rputs(">", r);
-                                }
-                                else {
-                                        ap_rputs("&nbsp;", r);
-                                }
-                                if (autoindex_opts & ICONS_ARE_LINKS) {
-                                        ap_rputs("</a></td>", r);
-                                }
-                                else {
-                                        ap_rputs("</td>", r);
-                                }
-                        }
-                        if (d->name_adjust == K_ADJUST) {
-                                ap_rvputs(r, "<td><a href=\"", anchor, "\">",
-                                          ap_escape_html(scratch, t2), "</a>",
-                                          NULL);
-                        }
-                        else {
-                                nwidth = strlen(t2);
-                                if (nwidth > name_width) {
-                                        memcpy(name_scratch, t2, name_width - 3);
-                                        name_scratch[name_width - 3] = '.';
-                                        name_scratch[name_width - 2] = '.';
-                                        name_scratch[name_width - 1] = '>';
-                                        name_scratch[name_width] = 0;
-                                        t2 = name_scratch;
-                                        nwidth = name_width;
-                                }
-                                ap_rvputs(r, "<td><a href=\"", anchor, "\">",
-                                          ap_escape_html(scratch, t2),
-                                          "</a>", pad_scratch + nwidth, NULL);
-                        }
-                        if (!(autoindex_opts & SUPPRESS_LAST_MOD)) {
-                                if (ar[x]->lm != -1) {
-                                        char time_str[MAX_STRING_LEN];
-                                        apr_time_exp_t ts;
-                                        apr_time_exp_lt(&ts, ar[x]->lm);
-                                        apr_strftime(time_str, &rv,
-                                                     MAX_STRING_LEN,
-                                                     "</td><td align=\"right\""
-                                                     ">%d-%b-%Y %H:%M  ",
-                                                     &ts);
-                                        ap_rputs(time_str, r);
-                                }
-                                else {
-                                        ap_rputs("</td><td>&nbsp;", r);
-                                }
-                        }
-                        if (!(autoindex_opts & SUPPRESS_SIZE)) {
-                                ap_rvputs(r, "</td><td align=\"right\">",
-                                          apr_strfsize(ar[x]->size, buf), NULL);
-                        }
-                        if (!(autoindex_opts & SUPPRESS_DESC)) {
-                                if (ar[x]->desc) {
-                                        if (d->desc_adjust == K_ADJUST) {
-                                                ap_rvputs(r, "</td><td>",
-                                                          ar[x]->desc, NULL);
-                                        }
-                                        else {
-                                                ap_rvputs(r, "</td><td>",
-                                                          terminate_description(d, ar[x]->desc,
-                                                                                autoindex_opts,
-                                                                                desc_width), NULL);
-                                        }
-                                }
-                        }
-                        else {
-                                ap_rputs("</td><td>&nbsp;", r);
-                        }
-                        ap_rputs("</td></tr>\n", r);
-                }
-                else if (autoindex_opts & FANCY_INDEXING) {
-                        if (!(autoindex_opts & SUPPRESS_ICON)) {
-                                if (autoindex_opts & ICONS_ARE_LINKS) {
-                                        ap_rvputs(r, "<a href=\"", anchor,
-                                                  "\">", NULL);
-                                }
-                                if ((ar[x]->icon) || d->default_icon) {
-                                        ap_rvputs(r, "<img src=\"",
-                                                  ap_escape_html(scratch,
-                                                                 ar[x]->icon ?
-                                                                 ar[x]->icon
-                                                                 : d->default_icon),
-                                                  "\" alt=\"[",
-                                                  (ar[x]->alt ? ar[x]->alt 
-                                                   : "   "),
-                                                  "]\"", NULL);
-                                        if (d->icon_width) {
-                                                ap_rprintf(r, " width=\"%d\"",
-                                                           d->icon_width);
-                                        }
-                                        if (d->icon_height) {
-                                                ap_rprintf(r, " height=\"%d\"",
-                                                           d->icon_height);
-                                        }
-
-                                        if (autoindex_opts & EMIT_XHTML) {
-                                                ap_rputs(" /", r);
-                                        }
-                                        ap_rputs(">", r);
-                                }
-                                else {
-                                        ap_rputs("     ", r);
-                                }
-                                if (autoindex_opts & ICONS_ARE_LINKS) {
-                                        ap_rputs("</a> ", r);
-                                }
-                                else {
-                                        ap_rputc(' ', r);
-                                }
-                        }
-                        nwidth = strlen(t2);
-                        if (nwidth > name_width) {
-                                memcpy(name_scratch, t2, name_width - 3);
-                                name_scratch[name_width - 3] = '.';
-                                name_scratch[name_width - 2] = '.';
-                                name_scratch[name_width - 1] = '>';
-                                name_scratch[name_width] = 0;
-                                t2 = name_scratch;
-                                nwidth = name_width;
-                        }
-                        ap_rvputs(r, "<a href=\"", anchor, "\">",
-                                  ap_escape_html(scratch, t2),
-                                  "</a>", pad_scratch + nwidth, NULL);
-                        /*
-                         * The blank before the storm.. er, before the next
-                         * field.
-                         */
-                        ap_rputs(" ", r);
-                        if (!(autoindex_opts & SUPPRESS_LAST_MOD)) {
-                                if (ar[x]->lm != -1) {
-                                        apr_time_exp_lt(&ts, ar[x]->lm);
-                                        apr_strftime(time_str, &rv,
-                                                     MAX_STRING_LEN,
-                                                     "%d-%b-%Y %H:%M  ", &ts);
-                                        ap_rputs(time_str, r);
-                                }
-                                else {
-                                        /* Length="22-Feb-1998 23:42  " 
-                                         * (see 4 lines above)
-                                         */
-                                        ap_rputs("                   ", r);
-                                }
-                        }
-                        if (!(autoindex_opts & SUPPRESS_SIZE)) {
-                                ap_rputs(apr_strfsize(ar[x]->size, buf), r);
-                                ap_rputs("  ", r);
-                        }
-                        if (!(autoindex_opts & SUPPRESS_DESC)) {
-                                if (ar[x]->desc) {
-                                        ap_rputs(terminate_description(d,
-                                                                       ar[x]->desc,
-                                                                       autoindex_opts,
-                                                                       desc_width), r);
-                                }
-                        }
-                        ap_rputc('\n', r);
-                }
-                else {
-                        ap_rvputs(r, "<li><a href=\"", anchor, "\"> ",
-                                  ap_escape_html(scratch, t2),
-                                  "</a></li>\n", NULL);
-                }
-        }
-        if (autoindex_opts & TABLE_INDEXING) {
-                ap_rvputs(r, breakrow, "</table>\n", NULL);
-        }
-        else if (autoindex_opts & FANCY_INDEXING) {
-                if (!(autoindex_opts & SUPPRESS_RULES)) {
-                        ap_rputs("<hr", r);
-                        if (autoindex_opts & EMIT_XHTML) {
-                                ap_rputs(" /", r);
-                        }
-                        ap_rputs("></pre>\n", r);
-                }
-                else {
-                        ap_rputs("</pre>\n", r);
-                }
-        }
-        else {
-                ap_rputs("</ul>\n", r);
-        }
-}
-
-/*
- * Compare two file entries according to the sort criteria.  The return
- * is essentially a signum function value.
- */
-
-static int dsortf(struct ent **e1, struct ent **e2)
-{
-        struct ent *c1 = NULL, *c2 = NULL;
-        int         result = 0;
-
-        /*
-         * First, see if either of the entries is for the parent directory.
-         * If so, that *always* sorts lower than anything else.
-         */
-        if ((*e1)->name[0] == '/') {
-                return -1;
-        }
-        if ((*e2)->name[0] == '/') {
-                return 1;
-        }
-        /*
-         * Now see if one's a directory and one isn't, if we're set
-         * isdir for FOLDERS_FIRST.
-         */
-        if ((*e1)->isdir != (*e2)->isdir) {
-                return (*e1)->isdir ? -1 : 1;
-        }
-        /*
-         * All of our comparisons will be of the c1 entry against the c2 one,
-         * so assign them appropriately to take care of the ordering.
-         */
-        if ((*e1)->ascending) {
-                c1 = *e1;
-                c2 = *e2;
-        }
-        else {
-                c1 = *e2;
-                c2 = *e1;
-        }
-
-        switch (c1->key) {
-        case K_LAST_MOD:
-                if (c1->lm > c2->lm) {
-                        return 1;
-                }
-                else if (c1->lm < c2->lm) {
-                        return -1;
-                }
-                break;
-        case K_SIZE:
-                if (c1->size > c2->size) {
-                        return 1;
-                }
-                else if (c1->size < c2->size) {
-                        return -1;
-                }
-                break;
-        case K_DESC:
-                if (c1->version_sort) {
-                        result = apr_strnatcmp(c1->desc ? c1->desc : "",
-                                               c2->desc ? c2->desc : "");
-                }
-                else {
-                        result = strcmp(c1->desc ? c1->desc : "",
-                                        c2->desc ? c2->desc : "");
-                }
-                if (result) {
-                        return result;
-                }
-                break;
-        }
-
-        /* names may identical when treated case-insensitively,
-         * so always fall back on strcmp() flavors to put entries
-         * in deterministic order.  This means that 'ABC' and 'abc'
-         * will always appear in the same order, rather than
-         * variably between 'ABC abc' and 'abc ABC' order.
-         */
-
-        if (c1->version_sort) {
-                if (c1->ignore_case) {
-                        result = apr_strnatcasecmp (c1->name, c2->name);
-                }
-                if (!result) {
-                        result = apr_strnatcmp(c1->name, c2->name);
-                }
-        }
-
-        /* The names may be identical in respects other other than
-         * filename case when strnatcmp is used above, so fall back
-         * to strcmp on conflicts so that fn1.01.zzz and fn1.1.zzz
-         * are also sorted in a deterministic order.
-         */
-
-        if (!result && c1->ignore_case) {
-                result = strcasecmp (c1->name, c2->name);
-        }
-
-        if (!result) {
-                result = strcmp (c1->name, c2->name);
-        }
-
-        return result;
-}
-
-
-static int 
-mod_glfs_index_directory (request_rec *r,
-                          glusterfs_dir_config_t *autoindex_conf)
-{
-        char                   *title_name = NULL, *title_endp = NULL;
-        char                   *pstring = NULL, *colargs = NULL;
-        char                   *path = NULL, *fname = NULL, *charset = NULL;
-        char                   *fullpath = NULL, *name = NULL, *ctype = NULL;
-        apr_finfo_t             dirent;
-        glusterfs_file_t        fd = NULL;
-        apr_status_t            status = APR_SUCCESS;
-        int                     num_ent = 0, x;
-        struct ent             *head = NULL, *p = NULL;
-        struct ent            **ar = NULL;
-        const char             *qstring = NULL;
-        apr_int32_t             autoindex_opts = autoindex_conf->opts;
-        char                    keyid, direction;
-        apr_size_t              dirpathlen;
-        glusterfs_dir_config_t *dir_config = NULL;
-        int                     ret = -1;
-        struct dirent          *entry = NULL;
-        struct stat             st = {0, };
-
-        name = r->filename;
-        title_name = ap_escape_html(r->pool, r->uri);
-        ctype = "text/html";
-        dir_config = mod_glfs_dconfig (r);
-        if (dir_config == NULL)  {
-                return HTTP_INTERNAL_SERVER_ERROR;
-        }
-
-        path = r->uri;
-        fd = glusterfs_open (path, O_RDONLY, 0);
-        if (fd == 0) {
-                ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r,
-                              "file permissions deny server access: %s",
-                              r->filename);
-                return HTTP_FORBIDDEN;
-        }
-
-        if (autoindex_conf->ctype) {
-                ctype = autoindex_conf->ctype;
-        }
-        if (autoindex_conf->charset) {
-                charset = autoindex_conf->charset;
-        }
-        else {
-#if APR_HAS_UNICODE_FS
-                charset = "UTF-8";
-#else
-                charset = "ISO-8859-1";
-#endif
-        }
-        if (*charset) {
-                ap_set_content_type(r, apr_pstrcat(r->pool, ctype, ";charset=",
-                                                   charset, NULL));
-        }
-        else {
-                ap_set_content_type(r, ctype);
-        }
-
-        if (autoindex_opts & TRACK_MODIFIED) {
-                ap_update_mtime(r, r->finfo.mtime);
-                ap_set_last_modified(r);
-                ap_set_etag(r);
-        }
-        if (r->header_only) {
-                glusterfs_close (fd);
-                return 0;
-        }
-
-        /*
-         * If there is no specific ordering defined for this directory,
-         * default to ascending by filename.
-         */
-        keyid = autoindex_conf->default_keyid
-                ? autoindex_conf->default_keyid : K_NAME;
-        direction = autoindex_conf->default_direction
-                ? autoindex_conf->default_direction : D_ASCENDING;
-
-        /*
-         * Figure out what sort of indexing (if any) we're supposed to use.
-         *
-         * If no QUERY_STRING was specified or client query strings have been
-         * explicitly disabled.
-         * If we are ignoring the client, suppress column sorting as well.
-         */
-        if (autoindex_opts & IGNORE_CLIENT) {
-                qstring = NULL;
-                autoindex_opts |= SUPPRESS_COLSORT;
-                colargs = "";
-        }
-        else {
-                char fval[5], vval[5], *ppre = "", *epattern = "";
-                fval[0] = '\0'; vval[0] = '\0';
-                qstring = r->args;
-
-                while (qstring && *qstring) {
-
-                        /* C= First Sort key Column (N, M, S, D) */
-                        if (   qstring[0] == 'C' && qstring[1] == '='
-                               && qstring[2] && strchr(K_VALID, qstring[2])
-                               && (   qstring[3] == '&' || qstring[3] == ';'
-                                      || !qstring[3])) {
-                                keyid = qstring[2];
-                                qstring += qstring[3] ? 4 : 3;
-                        }
-
-                        /* O= Sort order (A, D) */
-                        else if (   qstring[0] == 'O' && qstring[1] == '='
-                                    && (   (qstring[2] == D_ASCENDING)
-                                           || (qstring[2] == D_DESCENDING))
-                                    && (   qstring[3] == '&' 
-                                           || qstring[3] == ';'
-                                           || !qstring[3])) {
-                                direction = qstring[2];
-                                qstring += qstring[3] ? 4 : 3;
-                        }
-
-                        /* F= Output Format (0 plain, 1 fancy (pre), 2 table) */
-                        else if (   qstring[0] == 'F' && qstring[1] == '='
-                                    && qstring[2] && strchr("012", qstring[2])
-                                    && (   qstring[3] == '&' || qstring[3] == ';'
-                                           || !qstring[3])) {
-                                if (qstring[2] == '0') {
-                                        autoindex_opts &= ~(FANCY_INDEXING
-                                                            | TABLE_INDEXING);
-                                }
-                                else if (qstring[2] == '1') {
-                                        autoindex_opts = (autoindex_opts
-                                                          | FANCY_INDEXING)
-                                                & ~TABLE_INDEXING;
-                                }
-                                else if (qstring[2] == '2') {
-                                        autoindex_opts |= FANCY_INDEXING
-                                                | TABLE_INDEXING;
-                                }
-                                strcpy(fval, ";F= ");
-                                fval[3] = qstring[2];
-                                qstring += qstring[3] ? 4 : 3;
-                        }
-
-                        /* V= Version sort (0, 1) */
-                        else if (   qstring[0] == 'V' && qstring[1] == '='
-                                    && (qstring[2] == '0' || qstring[2] == '1')
-                                    && (   qstring[3] == '&' || qstring[3] == ';'
-                                           || !qstring[3])) {
-                                if (qstring[2] == '0') {
-                                        autoindex_opts &= ~VERSION_SORT;
-                                }
-                                else if (qstring[2] == '1') {
-                                        autoindex_opts |= VERSION_SORT;
-                                }
-                                strcpy(vval, ";V= ");
-                                vval[3] = qstring[2];
-                                qstring += qstring[3] ? 4 : 3;
-                        }
-
-                        /* P= wildcard pattern (*.foo) */
-                        else if (qstring[0] == 'P' && qstring[1] == '=') {
-                                const char *eos = qstring += 2; /* for efficiency */
-
-                                while (*eos && *eos != '&' && *eos != ';') {
-                                        ++eos;
-                                }
-
-                                if (eos == qstring) {
-                                        pstring = NULL;
-                                }
-                                else {
-                                        pstring = apr_pstrndup(r->pool, qstring,
-                                                               eos - qstring);
-                                        if (ap_unescape_url(pstring) != OK) {
-                                                /* ignore the pattern, if it's bad. */
-                                                pstring = NULL;
-                                        }
-                                        else {
-                                                ppre = ";P=";
-                                                /* be correct */
-                                                epattern = ap_escape_uri(r->pool,
-                                                                         pstring);
-                                        }
-                                }
-
-                                if (*eos && *++eos) {
-                                        qstring = eos;
-                                }
-                                else {
-                                        qstring = NULL;
-                                }
-                        }
-
-                        /* Syntax error?  Ignore the remainder! */
-                        else {
-                                qstring = NULL;
-                        }
-                }
-                colargs = apr_pstrcat(r->pool, fval, vval, ppre, epattern,
-                                      NULL);
-        }
-
-        /* Spew HTML preamble */
-        title_endp = title_name + strlen(title_name) - 1;
-
-        while (title_endp > title_name && *title_endp == '/') {
-                *title_endp-- = '\0';
-        }
-
-        emit_head(r, find_header(autoindex_conf, r),
-                  autoindex_opts & SUPPRESS_PREAMBLE,
-                  autoindex_opts & EMIT_XHTML, title_name);
-
-        /*
-         * Since we don't know how many dir. entries there are, put them into a
-         * linked list and then arrayificate them so qsort can use them.
-         */
-        head = NULL;
-        p = make_parent_entry(autoindex_opts, autoindex_conf, r, keyid,
-                              direction);
-        if (p != NULL) {
-                p->next = head;
-                head = p;
-                num_ent++;
-        }
-        fullpath = apr_palloc(r->pool, APR_PATH_MAX);
-        dirpathlen = strlen(name);
-        memcpy(fullpath, name, dirpathlen);
-
-        do {
-                entry = glusterfs_readdir (fd);
-                if (entry == NULL) {
-                        break;
-                }
-
-                fname = apr_pstrcat (r->pool, path, entry->d_name, NULL);
-
-                ret = glusterfs_stat (fname, &st);
-                if (ret != 0) {
-                        break;
-                }
-                
-                dirent.fname = fname;
-                dirent.name = apr_pstrdup (r->pool, entry->d_name);
-                fill_out_finfo (&dirent, &st, 
-                                APR_FINFO_MIN | APR_FINFO_IDENT
-                                | APR_FINFO_NLINK | APR_FINFO_OWNER
-                                | APR_FINFO_PROT);
-
-                p = make_autoindex_entry(&dirent, autoindex_opts,
-                                         autoindex_conf, r,
-                                         keyid, direction, pstring);
-                if (p != NULL) {
-                        p->next = head;
-                        head = p;
-                        num_ent++;
-                }
-        } while (1);
-
-        if (num_ent > 0) {
-                ar = (struct ent **) apr_palloc(r->pool,
-                                                num_ent * sizeof(struct ent *));
-                p = head;
-                x = 0;
-                while (p) {
-                        ar[x++] = p;
-                        p = p->next;
-                }
-
-                qsort((void *) ar, num_ent, sizeof(struct ent *),
-                      (int (*)(const void *, const void *)) dsortf);
-        }
-        output_directories(ar, num_ent, autoindex_conf, r, autoindex_opts,
-                           keyid, direction, colargs);
-        glusterfs_close (fd);
-
-        emit_tail(r, find_readme(autoindex_conf, r),
-                  autoindex_opts & SUPPRESS_PREAMBLE);
-
-        return 0;
-}
-
-
-static int 
-handle_autoindex(request_rec *r)
-{
-        glusterfs_dir_config_t *dir_config = NULL;
-        int allow_opts;
-
-        allow_opts = ap_allow_options(r);
-
-        r->allowed |= (AP_METHOD_BIT << M_GET);
-        if (r->method_number != M_GET) {
-                return DECLINED;
-        }
-
-        dir_config = mod_glfs_dconfig (r);
-
-        /* OK, nothing easy.  Trot out the heavy artillery... */
-
-        if (allow_opts & OPT_INDEXES) {
-                int errstatus;
-
-                if ((errstatus = ap_discard_request_body(r)) != OK) {
-                        return errstatus;
-                }
-
-                /* KLUDGE --- make the sub_req lookups happen in the right 
-                 * directory. Fixing this in the sub_req_lookup functions 
-                 * themselves is difficult, and would probably break 
-                 * virtual includes...
-                 */
-
-                if (r->filename[strlen(r->filename) - 1] != '/') {
-                        r->filename = apr_pstrcat(r->pool, r->filename, "/",
-                                                  NULL);
-                }
-                return mod_glfs_index_directory(r, dir_config);
-        } else {
-                ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r,
-                              "Directory index forbidden by "
-                              "Options directive: %s", r->filename);
-                return HTTP_FORBIDDEN;
-        }
-}
-
-
-static int 
-mod_glfs_handler (request_rec *r)
-{
-        conn_rec               *c = r->connection;
-        apr_bucket_brigade     *bb;
-        apr_bucket             *e;
-        core_dir_config        *d;
-        int                     errstatus;
-        glusterfs_file_t        fd = NULL;
-        apr_status_t            status;
-        glusterfs_dir_config_t *dir_config = NULL;
-        char                   *path = NULL;
-        int                     num_ranges = 0;
-        apr_size_t              size = 0;
-        apr_off_t               range_start = 0, range_end = 0;
-        char                   *current = NULL;
-        apr_status_t            rv = 0;
-        core_request_config    *req_cfg = NULL;
-
-        /* XXX if/when somebody writes a content-md5 filter we either need to
-         *     remove this support or coordinate when to use the filter vs.
-         *     when to use this code
-         *     The current choice of when to compute the md5 here matches the 1.3
-         *     support fairly closely (unlike 1.3, we don't handle computing md5
-         *     when the charset is translated).
-         */
-
-        int bld_content_md5;
-        if (!r->handler || (r->handler
-                            && strcmp (r->handler, GLUSTERFS_HANDLER)))
-                return DECLINED;
-
-        if (r->uri[0] == '\0') {
-                return DECLINED;
-        }
-  
-        if (r->finfo.filetype == APR_DIR) {
-                return handle_autoindex (r);
-        }
-
-        dir_config = mod_glfs_dconfig (r);
-
-        ap_allow_standard_methods(r, MERGE_ALLOW, M_GET, -1);
-  
-        /* We understood the (non-GET) method, but it might not be legal for
-           this particular resource. Check to see if the 'deliver_script'
-           flag is set. If so, then we go ahead and deliver the file since
-           it isn't really content (only GET normally returns content).
-
-           Note: based on logic further above, the only possible non-GET
-           method at this point is POST. In the future, we should enable
-           script delivery for all methods.  */
-        if (r->method_number != M_GET) {
-                req_cfg = ap_get_module_config(r->request_config, &core_module);
-                if (!req_cfg->deliver_script) {
-                        /* The flag hasn't been set for this request. Punt. */
-                        ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r,
-                                      "This resource does not accept the %s "
-                                      "method.",
-                                      r->method);
-                        return HTTP_METHOD_NOT_ALLOWED;
-                }
-        }
-
-        d = (core_dir_config *)ap_get_module_config(r->per_dir_config,
-                                                    &core_module);
-        bld_content_md5 = (d->content_md5 & 1)
-                && r->output_filters->frec->ftype != AP_FTYPE_RESOURCE;
-
-        if ((errstatus = ap_discard_request_body(r)) != OK) {
-                return errstatus;
-        }
-
-        if (r->finfo.filetype == 0) {
-                ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r,
-                              "File does not exist: %s", r->filename);
-                return HTTP_NOT_FOUND;
-        }
-
-        if ((r->used_path_info != AP_REQ_ACCEPT_PATH_INFO) &&
-            r->path_info && *r->path_info)
-        {
-                /* default to reject */
-                ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r,
-                              "File does not exist: %s",
-                              apr_pstrcat(r->pool, r->filename, r->path_info,
-                                          NULL));
-                return HTTP_NOT_FOUND;
-        }
-
-        ap_update_mtime (r, r->finfo.mtime);
-        ap_set_last_modified (r);
-        ap_set_etag (r);
-        apr_table_setn (r->headers_out, "Accept-Ranges", "bytes");
-
-        num_ranges = ap_set_byterange(r);
-        if (num_ranges == 0) {
-                size = r->finfo.size;
-        } else {
-                char *tmp = apr_pstrdup (r->pool, r->range);
-                while ((current = ap_getword(r->pool, (const char **)&tmp, ','))
-                       && (rv = parse_byterange(current, r->finfo.size,
-                                                &range_start, &range_end))) {
-                        size += (range_end - range_start);
-                }
-        }
-
-        ap_set_content_length (r, size);
-                        
-        if ((errstatus = ap_meets_conditions(r)) != OK) {
-                r->status = errstatus;
-        }
-
-        /* 
-         * file is small enough to have already got the content in 
-         * glusterfs_lookup
-        */
-        if (r->finfo.size <= dir_config->xattr_file_size && dir_config->buf) {
-                if (bld_content_md5) {
-                        apr_table_setn (r->headers_out, "Content-MD5",
-                                        (const char *)ap_md5_binary(r->pool,
-                                                                    dir_config->buf
-                                                                    , r->finfo.size));
-                }
-
-                ap_log_rerror (APLOG_MARK, APLOG_NOTICE, 0, r, 
-                               "fetching data from glusterfs through xattr "
-                               "interface\n");
-                
-                bb = apr_brigade_create(r->pool, c->bucket_alloc);
-
-                e = apr_bucket_heap_create (dir_config->buf, r->finfo.size,
-                                            free, c->bucket_alloc);
-                APR_BRIGADE_INSERT_TAIL (bb, e);
-                
-                e = apr_bucket_eos_create(c->bucket_alloc);
-                APR_BRIGADE_INSERT_TAIL(bb, e);
-
-                dir_config->buf = NULL;
-
-                /* let the byterange_filter handle multipart requests */
-                status = ap_pass_brigade(r->output_filters, bb);
-                if (status == APR_SUCCESS
-                    || r->status != HTTP_OK
-                    || c->aborted) {
-                        return OK;
-                }
-                else {
-                        /* no way to know what type of error occurred */
-                        ap_log_rerror(APLOG_MARK, APLOG_DEBUG, status, r,
-                                      "mod_glfs_handler: ap_pass_brigade "
-                                      "returned %i",
-                                      status);
-                        return HTTP_INTERNAL_SERVER_ERROR;
-                }
-        }
-        
-        /* do standard open/read/close to fetch content */
-        path = r->uri;
-        
-        fd = glusterfs_open (path, O_RDONLY, 0);
-        if (fd == 0) {
-                ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r,
-                              "file permissions deny server access: %s",
-                              r->filename);
-                return HTTP_FORBIDDEN;
-        }
-
-        /* 
-         * byterange_filter cannot handle range requests, since we are not 
-         * sending the whole data in a single brigade
-         */
-
-
-        if (num_ranges == 0) {
-                mod_glfs_read_async (r, NULL, fd, 0, -1);
-        } else {
-                mod_glfs_handle_byte_ranges (r, fd, num_ranges);
-        }
-                
-        glusterfs_close (fd);
-}
-
-
-#if 0
-static apr_status_t 
-mod_glfs_output_filter (ap_filter_t *f,
-                        apr_bucket_brigade *b)
-{
-        size_t size = 0;
-        apr_bucket_t *e = NULL; 
-        size = atol (apr_table_get (r->notes, MOD_GLFS_SIZE));
-        
-        for (e = APR_BRIGADE_FIRST(b);
-             e != APR_BRIGADE_SENTINEL(b);
-             e = APR_BUCKET_NEXT(e))
-        {
-                /* FIXME: can there be more than one heap buckets? */
-                if (e->type == &apr_bucket_type_heap) {
-                        break;
-                }
-        }
-
-        if (e != APR_BRIGADE_SENTINEL(b)) {
-                e->length = size;
-        }
-
-        return ap_pass_brigade (f->next, b);
-}
-#endif
-
-static int 
-mod_glfs_fixup_dir(request_rec *r)
-{
-        glusterfs_dir_config_t  *d = NULL;
-        char                    *dummy_ptr[1];
-        char                   **names_ptr = NULL, *name_ptr = NULL;
-        int                      num_names;
-        int                      error_notfound = 0;
-        char                    *ifile = NULL;
-        request_rec             *rr = NULL;
-
-        /* only handle requests against directories */
-        if (r->finfo.filetype != APR_DIR) {
-                return DECLINED;
-        }
-
-        if (!r->handler || strcmp (r->handler, GLUSTERFS_HANDLER)) {
-                return DECLINED;
-        }
-
-        /* Never tolerate path_info on dir requests */
-        if (r->path_info && *r->path_info) {
-                return DECLINED;
-        }
-
-        d = (glusterfs_dir_config_t *)ap_get_module_config(r->per_dir_config,
-                                                           &glusterfs_module);
-
-        /* Redirect requests that are not '/' terminated */
-        if (r->uri[0] == '\0' || r->uri[strlen(r->uri) - 1] != '/')
-        {
-                if (!d->do_slash) {
-                        return DECLINED;
-                }
-
-                /* Only redirect non-get requests if we have no note to warn
-                 * that this browser cannot handle redirs on non-GET requests
-                 * (such as Microsoft's WebFolders).
-                 */
-                if ((r->method_number != M_GET)
-                    && apr_table_get(r->subprocess_env, "redirect-carefully")) {
-                        return DECLINED;
-                }
-
-                if (r->args != NULL) {
-                        ifile = apr_pstrcat(r->pool, ap_escape_uri(r->pool,
-                                                                   r->uri),
-                                            "/", "?", r->args, NULL);
-                }
-                else {
-                        ifile = apr_pstrcat(r->pool, ap_escape_uri(r->pool,
-                                                                   r->uri),
-                                            "/", NULL);
-                }
-
-                apr_table_setn(r->headers_out, "Location",
-                               ap_construct_url(r->pool, ifile, r));
-                return HTTP_MOVED_PERMANENTLY;
-        }
-
-        if (d->index_names) {
-                names_ptr = (char **)d->index_names->elts;
-                num_names = d->index_names->nelts;
-        }
-        else {
-                dummy_ptr[0] = AP_DEFAULT_INDEX;
-                names_ptr = dummy_ptr;
-                num_names = 1;
-        }
-
-        for (; num_names; ++names_ptr, --num_names) {
-                /* XXX: Is this name_ptr considered escaped yet, or not??? */
-                name_ptr = *names_ptr;
-
-                /* Once upon a time args were handled _after_ the successful
-                 * redirect. But that redirect might then _refuse_ the 
-                 * given r->args, creating a nasty tangle.  It seems safer to 
-                 * consider the r->args while we determine if name_ptr is our 
-                 * viable index, and therefore set them up correctly on redirect.
-                 */
-                if (r->args != NULL) {
-                        name_ptr = apr_pstrcat(r->pool, name_ptr, "?", r->args,
-                                               NULL);
-                }
-
-                rr = ap_sub_req_lookup_uri(name_ptr, r, NULL);
-
-                /* The sub request lookup is very liberal, and the core 
-                 * map_to_storage handler will almost always result in HTTP_OK 
-                 * as /foo/index.html may be /foo with PATH_INFO="/index.html",
-                 * or even / with PATH_INFO="/foo/index.html". To get around 
-                 * this we insist that the the index be a regular filetype.
-                 *
-                 * Another reason is that the core handler also makes the 
-                 * assumption that if r->finfo is still NULL by the time it 
-                 * gets called, the file does not exist.
-                 */
-                if (rr->status == HTTP_OK
-                    && (   (rr->handler && !strcmp(rr->handler, "proxy-server"))
-                           || rr->finfo.filetype == APR_REG)) {
-                        ap_internal_fast_redirect(rr, r);
-                        return OK;
-                }
-
-                /* If the request returned a redirect, propagate it to the 
-                 * client
-                 */
-
-                if (ap_is_HTTP_REDIRECT(rr->status)
-                    || (rr->status == HTTP_NOT_ACCEPTABLE && num_names == 1)
-                    || (rr->status == HTTP_UNAUTHORIZED && num_names == 1)) {
-
-                        apr_pool_join(r->pool, rr->pool);
-                        error_notfound = rr->status;
-                        r->notes = apr_table_overlay(r->pool, r->notes,
-                                                     rr->notes);
-                        r->headers_out = apr_table_overlay(r->pool,
-                                                           r->headers_out,
-                                                           rr->headers_out);
-                        r->err_headers_out = apr_table_overlay(r->pool,
-                                                               r->err_headers_out,
-                                                               rr->err_headers_out);
-                        return error_notfound;
-                }
-
-                /* If the request returned something other than 404 (or 200),
-                 * it means the module encountered some sort of problem. To be
-                 * secure, we should return the error, rather than allow 
-                 * autoindex to create a (possibly unsafe) directory index.
-                 *
-                 * So we store the error, and if none of the listed files
-                 * exist, we return the last error response we got, instead
-                 * of a directory listing.
-                 */
-                if (rr->status && rr->status != HTTP_NOT_FOUND
-                    && rr->status != HTTP_OK) {
-                        error_notfound = rr->status;
-                }
-
-                ap_destroy_sub_req(rr);
-        }
-
-        if (error_notfound) {
-                return error_notfound;
-        }
-
-        /* nothing for us to do, pass on through */
-        return DECLINED;
-}
-
-
-static void 
-mod_glfs_register_hooks(apr_pool_t *p)
-{
-        ap_hook_child_init (mod_glfs_child_init, NULL, NULL, APR_HOOK_MIDDLE);
-        ap_hook_handler (mod_glfs_handler, NULL, NULL, APR_HOOK_REALLY_FIRST);
-        ap_hook_map_to_storage (mod_glfs_map_to_storage, NULL, NULL,
-                                APR_HOOK_REALLY_FIRST);
-        ap_hook_fixups(mod_glfs_fixup_dir,NULL,NULL,APR_HOOK_LAST);
-
-/*    mod_glfs_output_filter_handle = 
-      ap_register_output_filter ("MODGLFS", mod_glfs_output_filter,
-      NULL, AP_FTYPE_PROTOCOL); */
-}
-
-static const char *
-cmd_add_index (cmd_parms *cmd, void *dummy, const char *arg)
-{
-        glusterfs_dir_config_t *d = dummy;
-
-        if (!d->index_names) {
-                d->index_names = apr_array_make(cmd->pool, 2, sizeof(char *));
-        }
-        *(const char **)apr_array_push(d->index_names) = arg;
-        return NULL;
-}
-
-static const char *
-cmd_configure_slash (cmd_parms *cmd, void *d_, int arg)
-{
-        glusterfs_dir_config_t *d = d_;
-
-        d->do_slash = arg ? SLASH_ON : SLASH_OFF;
-        return NULL;
-}
-
-#define DIR_CMD_PERMS OR_INDEXES
-
-static const 
-command_rec mod_glfs_cmds[] =
-{
-        AP_INIT_TAKE1(
-                "GlusterfsLogfile",
-                cmd_add_logfile,
-                NULL,
-                ACCESS_CONF, /*FIXME: allow overriding in .htaccess files */
-                "Glusterfs logfile"
-                ),
-
-        AP_INIT_TAKE1(
-                "GlusterfsLoglevel",
-                cmd_set_loglevel,
-                NULL,
-                ACCESS_CONF,
-                "Glusterfs loglevel:anyone of none, critical, error, warning, "
-                "debug"
-                ),
-
-        AP_INIT_TAKE1(
-                "GlusterfsCacheTimeout",
-                cmd_set_cache_timeout,
-                NULL,
-                ACCESS_CONF,
-                "Timeout value in seconds for lookup and stat cache of "
-                "libglusterfsclient"
-                ),
-
-        AP_INIT_TAKE1(
-                "GlusterfsVolumeSpecfile",
-                cmd_add_volume_specfile,
-                NULL,
-                ACCESS_CONF,
-                "Glusterfs Volume specfication file specifying filesystem "
-                "under this directory"
-                ),
-
-        AP_INIT_TAKE1(
-                "GlusterfsXattrFileSize",
-                cmd_add_xattr_file_size,
-                NULL,
-                ACCESS_CONF,
-                "Maximum size of the file that can be fetched through "
-                "extended attribute interface of libglusterfsclient"
-                ),
-
-        /* mod_dir cmds */
-        AP_INIT_ITERATE("DirectoryIndex", cmd_add_index, 
-                        NULL, DIR_CMD_PERMS,
-                        "a list of file names"),
-
-        AP_INIT_FLAG("DirectorySlash", cmd_configure_slash, 
-                     NULL, DIR_CMD_PERMS,
-                     "On or Off"),
-
-        /* autoindex cmds */
-        AP_INIT_ITERATE2("AddIcon", cmd_add_icon, 
-                         BY_PATH, DIR_CMD_PERMS,
-                         "an icon URL followed by one or more filenames"),
-
-        AP_INIT_ITERATE2("AddIconByType", cmd_add_icon, 
-                         BY_TYPE, DIR_CMD_PERMS,
-                         "an icon URL followed by one or more MIME types"),
-
-        AP_INIT_ITERATE2("AddIconByEncoding", cmd_add_icon, 
-                         BY_ENCODING, DIR_CMD_PERMS,
-                         "an icon URL followed by one or more content encodings"),
-
-        AP_INIT_ITERATE2("AddAlt", cmd_add_alt, BY_PATH, 
-                         DIR_CMD_PERMS,
-                         "alternate descriptive text followed by one or more "
-                         "filenames"),
-
-        AP_INIT_ITERATE2("AddAltByType", cmd_add_alt, 
-                         BY_TYPE, DIR_CMD_PERMS,
-                         "alternate descriptive text followed by one or more "
-                         "MIME types"),
-
-        AP_INIT_ITERATE2("AddAltByEncoding", cmd_add_alt, 
-                         BY_ENCODING, DIR_CMD_PERMS,
-                         "alternate descriptive text followed by one or more "
-                         "content encodings"),
-
-        AP_INIT_TAKE_ARGV("IndexOptions", cmd_add_opts, 
-                          NULL, DIR_CMD_PERMS,
-                          "one or more index options [+|-][]"),
-
-        AP_INIT_TAKE2("IndexOrderDefault", cmd_set_default_order, 
-                      NULL, DIR_CMD_PERMS,
-                      "{Ascending,Descending} {Name,Size,Description,Date}"),
-
-        AP_INIT_ITERATE("IndexIgnore", cmd_add_ignore, 
-                        NULL, DIR_CMD_PERMS,
-                        "one or more file extensions"),
-
-        AP_INIT_ITERATE2("AddDescription", cmd_add_desc, 
-                         BY_PATH, DIR_CMD_PERMS,
-                         "Descriptive text followed by one or more filenames"),
-
-        AP_INIT_TAKE1("HeaderName", cmd_add_header, 
-                      NULL, DIR_CMD_PERMS,
-                      "a filename"),
-
-        AP_INIT_TAKE1("ReadmeName", cmd_add_readme, 
-                      NULL, DIR_CMD_PERMS,
-                      "a filename"),
-
-        AP_INIT_RAW_ARGS("FancyIndexing", ap_set_deprecated, 
-                         NULL, OR_ALL,
-                         "The FancyIndexing directive is no longer supported. "
-                         "Use IndexOptions FancyIndexing."),
-
-        AP_INIT_TAKE1("DefaultIcon", ap_set_string_slot,
-                      (void *)APR_OFFSETOF(glusterfs_dir_config_t,
-                                           default_icon),
-                      DIR_CMD_PERMS, "an icon URL"),
-
-        AP_INIT_TAKE1("IndexStyleSheet", ap_set_string_slot,
-                      (void *)APR_OFFSETOF(glusterfs_dir_config_t, style_sheet),
-                      DIR_CMD_PERMS, "URL to style sheet"),
-
-        {NULL}
-};
-
-module AP_MODULE_DECLARE_DATA glusterfs_module =
-{
-        STANDARD20_MODULE_STUFF,
-        mod_glfs_create_dir_config,
-        mod_glfs_merge_dir_config,
-        NULL, //mod_glfs_create_server_config,
-        NULL, //mod_glfs_merge_server_config,
-        mod_glfs_cmds,
-        mod_glfs_register_hooks,
-};
diff --git a/mod_glusterfs/apache/Makefile.am b/mod_glusterfs/apache/Makefile.am
deleted file mode 100644
index bda039310..000000000
--- a/mod_glusterfs/apache/Makefile.am
+++ /dev/null
@@ -1,10 +0,0 @@
-SUBDIRS = $(MOD_GLUSTERFS_HTTPD_VERSION)
-
-EXTRA_DIST = 1.3/Makefile.am 1.3/Makefile.in \
-	     1.3/src/Makefile.am 1.3/src/Makefile.in \
-	     1.3/src/mod_glusterfs.c \
-	     1.3/src/README.txt \
-	     2.2/Makefile.am 2.2/Makefile.in \
-	     2.2/src/Makefile.am 2.2/src/Makefile.in \
-	     2.2/src/mod_glusterfs.c
-CLEANFILES = 
diff --git a/mod_glusterfs/lighttpd/1.4/Makefile.am b/mod_glusterfs/lighttpd/1.4/Makefile.am
deleted file mode 100644
index eda329111..000000000
--- a/mod_glusterfs/lighttpd/1.4/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-EXTRA_DIST = Makefile.am.diff mod_glusterfs.c mod_glusterfs.h README.txt
-
-CLEANFILES = 
diff --git a/mod_glusterfs/lighttpd/1.4/Makefile.am.diff b/mod_glusterfs/lighttpd/1.4/Makefile.am.diff
deleted file mode 100644
index 375696b5d..000000000
--- a/mod_glusterfs/lighttpd/1.4/Makefile.am.diff
+++ /dev/null
@@ -1,29 +0,0 @@
---- lighttpd-1.4.19/src/Makefile.am	2008-04-16 18:42:18.000000000 +0400
-+++ lighttpd-1.4.19.mod/src/Makefile.am	2008-04-16 18:41:11.000000000 +0400
-@@ -1,4 +1,4 @@
--AM_CFLAGS = $(FAM_CFLAGS)
-+AM_CFLAGS = $(FAM_CFLAGS)  -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64
- 
- noinst_PROGRAMS=proc_open lemon # simple-fcgi #graphic evalo bench ajp ssl error_test adserver gen-license
- sbin_PROGRAMS=lighttpd lighttpd-angel
-@@ -241,6 +241,11 @@
- mod_accesslog_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined
- mod_accesslog_la_LIBADD = $(common_libadd)
- 
-+lib_LTLIBRARIES += mod_glusterfs.la
-+mod_glusterfs_la_SOURCES = mod_glusterfs.c
-+mod_glusterfs_la_CFLAGS = $(AM_CFLAGS) 
-+mod_glusterfs_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined -lglusterfsclient -lpthread
-+mod_glusterfs_la_LIBADD = $(common_libadd) 
- 
- hdr = server.h buffer.h network.h log.h keyvalue.h \
-       response.h request.h fastcgi.h chunk.h \
-@@ -254,7 +259,7 @@
-       configparser.h mod_ssi_exprparser.h \
-       sys-mmap.h sys-socket.h mod_cml.h mod_cml_funcs.h \
-       splaytree.h proc_open.h status_counter.h \
--      mod_magnet_cache.h
-+      mod_magnet_cache.h mod_glusterfs.h
- 
- DEFS= @DEFS@ -DLIBRARY_DIR="\"$(libdir)\"" -DSBIN_DIR="\"$(sbindir)\""
- 
diff --git a/mod_glusterfs/lighttpd/1.4/README.txt b/mod_glusterfs/lighttpd/1.4/README.txt
deleted file mode 100644
index 786a146e4..000000000
--- a/mod_glusterfs/lighttpd/1.4/README.txt
+++ /dev/null
@@ -1,57 +0,0 @@
-Introduction
-============
-mod_glusterfs is a module written for lighttpd to speed up the access of files present on glusterfs. mod_glusterfs uses libglusterfsclient library provided for glusterfs and hence can be used without fuse (File System in User Space).
-
-Usage
-=====
-To use mod_glusterfs with lighttpd-1.4, copy mod_glusterfs.c and mod_glusterfs.h into src/ of lighttpd-1.4 source tree, and apply the Makefile.am.diff to src/Makefile.am. Re-run ./autogen.sh on the top level of the lighttpd-1.4 build tree and recompile.
-
-# cp mod_glusterfs.[ch] /home/glusterfs/lighttpd-1.4/src/
-# cp Makefile.am.diff /home/glusterfs/lighttpd-1.4/
-# cd /home/glusterfs/lighttpd-1.4
-# patch -p1 < Makefile.am.diff 
-# ./autogen.sh
-# ./configure
-# make
-# make install
-
-Configuration
-=============
-* mod_glusterfs should be listed at the begining of the list server.modules in lighttpd.conf. 
-
-Below is a snippet from lighttpd.conf concerning to mod_glusterfs.
-
-$HTTP["url"] =~ "^/glusterfs" {
-	glusterfs.prefix = "/glusterfs" 
-	glusterfs.document-root = "/home/glusterfs/document-root"
-	glusterfs.logfile = "/var/log/glusterfs-logfile"
-	glusterfs.volume-specfile = "/etc/glusterfs/glusterfs.vol"
-	glusterfs.loglevel = "error"
-	glusterfs.cache-timeout = 300
-	glusterfs.xattr-interface-size-limit = "65536"
-}
-
-* $HTTP["url"] =~ "^/glusterfs"
-  A perl style regular expression used to match against the url. If regular expression matches the url, the url is handled by mod_glusterfs. Note that the pattern given here should match glusterfs.prefix.
-
-* glusterfs.prefix (COMPULSORY)
-  A string to be present at the starting of the file path in the url so that the file would be handled by glusterfs.
-  Eg., A GET request on the url http://www.example.com/glusterfs-prefix/some-dir/example-file will result in fetching of the file "/some-dir/example-file" from glusterfs mount if glusterfs.prefix is set to "/glusterfs-prefix".
-
-* glusterfs.volume-specfile (COMPULSORY)
-  Path to the the glusterfs volume specification file.
-
-* glusterfs.logfile (COMPULSORY)
-  Path to the glusterfs logfile.
-
-* glusterfs.loglevel (OPTIONAL, default = warning)
-  Allowed values are critical, error, warning, debug, none in the decreasing order of severity of error conditions.
-
-* glusterfs.cache-timeout (OPTIONAL, default = 0)
-  Timeout values for glusterfs stat and lookup cache.
-
-* glusterfs.document-root (COMPULSORY)
-  An absolute path, relative to which all the files are fetched from glusterfs.
-
-* glusterfs.xattr-interface-size-limit (OPTIONAL, default = 0)
-  Files with sizes upto and including this value are fetched through the extended attribute interface of glusterfs rather than the usual open-read-close set of operations. For files of small sizes, it is recommended to use extended attribute interface.
diff --git a/mod_glusterfs/lighttpd/1.4/mod_glusterfs.c b/mod_glusterfs/lighttpd/1.4/mod_glusterfs.c
deleted file mode 100644
index 295c9704c..000000000
--- a/mod_glusterfs/lighttpd/1.4/mod_glusterfs.c
+++ /dev/null
@@ -1,1820 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <ctype.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <pthread.h>
-#include <sys/types.h>
-#include <fcntl.h>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-
-#include <errno.h>
-#include <unistd.h>
-#include <assert.h>
-
-#include "base.h"
-#include "log.h"
-#include "buffer.h"
-
-#include "plugin.h"
-
-#include "stat_cache.h"
-#include "mod_glusterfs.h"
-#include "etag.h"
-#include "http_chunk.h"
-#include "response.h"
-
-#include "fdevent.h"
-#include <libglusterfsclient.h>
-
-#ifdef HAVE_ATTR_ATTRIBUTES_H
-#include <attr/attributes.h>
-#endif
-
-#ifdef HAVE_FAM_H
-# include <fam.h>
-#endif
-
-#include "sys-mmap.h"
-
-/* NetBSD 1.3.x needs it */
-#ifndef MAP_FAILED
-# define MAP_FAILED -1
-#endif
-
-#ifndef O_LARGEFILE
-# define O_LARGEFILE 0
-#endif
-
-#ifndef HAVE_LSTAT
-#define lstat stat
-#endif
-
-#if 0
-/* 
-   enables debug code for testing if all nodes in the stat-cache as accessable
-*/
-#define DEBUG_STAT_CACHE
-#endif
-
-#ifdef HAVE_LSTAT
-#undef HAVE_LSTAT
-#endif
-
-#define GLUSTERFS_FILE_CHUNK (FILE_CHUNK + 1)
-
-/* 
-   Keep this value large. Each glusterfs_async_read of GLUSTERFS_CHUNK_SIZE
-   results in a network_backend_write of the read data
-*/
-
-#define GLUSTERFS_CHUNK_SIZE 8192
-
-/**
- * this is a staticfile for a lighttpd plugin
- *
- */
-
-typedef struct glusterfs_async_local {
-        int op_ret;
-        int op_errno;
-        char async_read_complete;
-        off_t length;
-        size_t read_bytes;
-        glusterfs_iobuf_t *buf;
-        pthread_mutex_t lock;
-        pthread_cond_t cond;
-} glusterfs_async_local_t;
-
-
-typedef struct {
-        glusterfs_file_t fd;
-        void *buf;
-        buffer *glusterfs_path;
-        /*  off_t response_content_length; */
-        int prefix;
-}mod_glusterfs_ctx_t;
-
-/* plugin config for all request/connections */
-typedef struct {
-        buffer *logfile;
-        buffer *loglevel;
-        buffer *specfile;
-        buffer *prefix;
-        buffer *xattr_file_size;
-        buffer *document_root;
-        array *exclude_exts;
-        unsigned short cache_timeout;
-        char mounted;
-} plugin_config;
-
-static int (*network_backend_write)(struct server *srv, connection *con, int fd,
-                                    chunkqueue *cq);
-
-typedef struct {
-        PLUGIN_DATA;
-        buffer *range_buf;
-        plugin_config **config_storage;
-  
-        plugin_config conf;
-} plugin_data;
-
-typedef struct {
-        chunkqueue *cq;
-        glusterfs_iobuf_t *buf;
-        size_t length;
-}mod_glusterfs_chunkqueue;
-
-#ifdef HAVE_FAM_H
-typedef struct {
-        FAMRequest *req;
-        FAMConnection *fc;
-
-        buffer *name;
-
-        int version;
-} fam_dir_entry;
-#endif
-
-/* the directory name is too long to always compare on it
- * - we need a hash
- * - the hash-key is used as sorting criteria for a tree
- * - a splay-tree is used as we can use the caching effect of it
- */
-
-/* we want to cleanup the stat-cache every few seconds, let's say 10
- *
- * - remove entries which are outdated since 30s
- * - remove entries which are fresh but havn't been used since 60s
- * - if we don't have a stat-cache entry for a directory, release it from the 
- *   monitor
- */
-
-#ifdef DEBUG_STAT_CACHE
-typedef struct {
-        int *ptr;
-
-        size_t used;
-        size_t size;
-} fake_keys;
-
-static fake_keys ctrl;
-#endif
-
-int 
-mod_glusterfs_readv_async_cbk (int op_ret, int op_errno,
-                               glusterfs_iobuf_t *buf,
-                               void *cbk_data)
-{
-        glusterfs_async_local_t *local = cbk_data;
-        pthread_mutex_lock (&local->lock);
-        {
-                local->async_read_complete = 1;
-                local->buf = buf;
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                pthread_cond_signal (&local->cond);
-        }
-        pthread_mutex_unlock (&local->lock);
-
-        return 0;
-}
-
-static int
-mod_glusterfs_read_async (server *srv, connection *con, chunk *glusterfs_chunk)
-{
-        glusterfs_async_local_t local;
-        off_t end = 0;
-        int nbytes;
-        int complete;
-        chunkqueue *cq = NULL;
-        chunk *c = NULL;
-        off_t offset = glusterfs_chunk->file.start;
-        size_t length = glusterfs_chunk->file.length;
-        glusterfs_file_t fd = glusterfs_chunk->file.name;
-
-        pthread_cond_init (&local.cond, NULL);
-        pthread_mutex_init (&local.lock, NULL);
-  
-        //local.fd = fd;
-        memset (&local, 0, sizeof (local));
-
-        if (length > 0)
-                end = offset + length;
-
-        cq = chunkqueue_init ();
-        if (!cq) {
-                con->http_status = 500;
-                return HANDLER_FINISHED;
-        }
-
-        do {
-                glusterfs_iobuf_t *buf;
-                int i;
-                if (length > 0) {
-                        nbytes = end - offset;
-                        if (nbytes > GLUSTERFS_CHUNK_SIZE)
-                                nbytes = GLUSTERFS_CHUNK_SIZE;
-                } else
-                        nbytes = GLUSTERFS_CHUNK_SIZE;
-
-                glusterfs_read_async(fd, 
-                                     nbytes,
-                                     offset,
-                                     mod_glusterfs_readv_async_cbk,
-                                     (void *)&local);
-
-                pthread_mutex_lock (&local.lock);
-                {
-                        while (!local.async_read_complete) {
-                                pthread_cond_wait (&local.cond, &local.lock);
-                        }
-
-                        local.async_read_complete = 0;
-                        buf = local.buf;
-
-                        if ((int)length < 0)
-                                complete = (local.op_ret <= 0);
-                        else {
-                                local.read_bytes += local.op_ret;
-                                complete = ((local.read_bytes == length)
-                                            || (local.op_ret <= 0));
-                        }
-                }
-                pthread_mutex_unlock (&local.lock);
-
-                if (local.op_ret > 0) {
-                        unsigned long check = 0;
-                        for (i = 0; i < buf->count; i++) {
-                                buffer *nw_write_buf = buffer_init ();
-
-                                check += buf->vector[i].iov_len;        
-
-                                nw_write_buf->used = buf->vector[i].iov_len + 1;
-                                nw_write_buf->size = buf->vector[i].iov_len;
-                                nw_write_buf->ptr = buf->vector[i].iov_base;
-
-                                offset += local.op_ret;
-                                chunkqueue_append_buffer_weak(cq, nw_write_buf);
-                        }
-  
-                        network_backend_write (srv, con, con->fd, cq);
-  
-                        if (chunkqueue_written (cq) != local.op_ret) {
-                                mod_glusterfs_chunkqueue *gf_cq;
-                                glusterfs_chunk->file.start = offset;
-                                if ((int)glusterfs_chunk->file.length > 0)
-                                        glusterfs_chunk->file.length -= local.read_bytes;
-
-                                gf_cq = calloc (1, sizeof (*gf_cq));
-                                /* ERR_ABORT (gf_cq); */
-                                gf_cq->cq = cq;
-                                gf_cq->buf = buf;
-                                gf_cq->length = local.op_ret;
-                                glusterfs_chunk->file.mmap.start =(char *)gf_cq;
-                                return local.read_bytes;
-                        }
-      
-                        for (c = cq->first ; c; c = c->next) 
-                                c->mem->ptr = NULL;
-
-                        chunkqueue_reset (cq);
-                }
-
-                glusterfs_free (buf);
-        } while (!complete);
-
-        chunkqueue_free (cq);
-        glusterfs_close (fd);
-
-        if (local.op_ret < 0)
-                con->http_status = 500;
-
-        return (local.op_ret < 0 ? HANDLER_FINISHED : HANDLER_GO_ON);
-}
-
-int mod_glusterfs_network_backend_write(struct server *srv, connection *con,
-                                        int fd, chunkqueue *cq)
-{
-        chunk *c, *prev, *first;
-        int chunks_written = 0;
-        int error = 0;
-
-        for (first = prev = c = cq->first; c; c = c->next, chunks_written++) {
-
-                if (c->type == MEM_CHUNK && c->mem->used && !c->mem->ptr) {
-                        if (cq->first != c) {
-                                prev->next = NULL;
-
-                                /* call stored network_backend_write */
-                                network_backend_write (srv, con, fd, cq);
-
-                                prev->next = c;
-                        } 
-                        cq->first = c->next;
-
-                        if (c->file.fd < 0) {
-                                error = HANDLER_ERROR;
-                                break;
-                        }
-
-                        if (c->file.mmap.start) {
-                                chunk *tmp;
-                                mod_glusterfs_chunkqueue *gf_cq = NULL;
-
-                                gf_cq = (mod_glusterfs_chunkqueue *)c->file.mmap.start;
-
-                                network_backend_write (srv, con, fd, gf_cq->cq);
-
-                                if ((size_t)chunkqueue_written (gf_cq->cq) 
-                                    != gf_cq->length) {
-                                        cq->first = first;
-                                        return chunks_written;
-                                }
-                                for (tmp = gf_cq->cq->first ; tmp;
-                                     tmp = tmp->next)
-                                        tmp->mem->ptr = NULL;
-
-                                chunkqueue_free (gf_cq->cq);
-                                glusterfs_free (gf_cq->buf);
-                                free (gf_cq);
-                                c->file.mmap.start = NULL;
-                        }
-      
-                        mod_glusterfs_read_async (srv, con, c);
-                        if (c->file.mmap.start) {
-                                /* pending chunkqueue from
-                                   mod_glusterfs_read_async to be written to
-                                   network */
-                                cq->first = first;
-                                return chunks_written;
-                        }
-
-                        buffer_free (c->mem);
-                        c->mem = NULL;
-
-                        c->type = FILE_CHUNK;
-                        c->offset = c->file.length = 0;
-                        c->file.name = NULL;
-
-                        if (first == c)
-                                first = c->next;
-
-                        if (cq->last == c)
-                                cq->last = NULL;
-
-                        prev->next = c->next;
-
-                        free(c);
-                }     
-                prev = c;
-        }
-
-        network_backend_write (srv, con, fd, cq);
-
-        cq->first = first;
-
-        return chunks_written;
-}
-
-int chunkqueue_append_glusterfs_file (connection *con, glusterfs_file_t fd,
-                                      off_t offset, size_t len, size_t buf_size)
-{
-        chunk *c = NULL;
-        c = chunkqueue_get_append_tempfile (con->write_queue);
-  
-        if (c->file.is_temp) {
-                close (c->file.fd);
-                unlink (c->file.name->ptr);
-        }
-
-        c->type = MEM_CHUNK;
-
-        buffer_free (c->mem);
-
-        c->mem = buffer_init ();
-        c->mem->used = len + 1;
-        c->mem->size = buf_size;
-        c->mem->ptr = NULL;
-        c->offset = 0;
-
-        buffer_free (c->file.name);
-
-        /* fd returned by libglusterfsclient is a pointer */
-        c->file.name = (buffer *)fd;
-        c->file.start = offset;
-        c->file.length = len;
-
-        //c->file.fd = fd;
-        c->file.mmap.start = NULL;
-        return 0;
-}
-
-/* init the plugin data */
-INIT_FUNC(mod_glusterfs_init) {
-        plugin_data *p;
-
-        p = calloc(1, sizeof(*p));
-        network_backend_write = NULL;
-
-        return p;
-}
-
-/* detroy the plugin data */
-FREE_FUNC(mod_glusterfs_free) {
-        plugin_data *p = p_d;
-
-        UNUSED (srv);
-
-        if (!p) return HANDLER_GO_ON;
-  
-        if (p->config_storage) {
-                size_t i;
-                for (i = 0; i < srv->config_context->used; i++) {
-                        plugin_config *s = p->config_storage[i];
-      
-                        buffer_free (s->logfile);
-                        buffer_free (s->loglevel);
-                        buffer_free (s->specfile);
-                        buffer_free (s->prefix);
-                        buffer_free (s->xattr_file_size);
-                        buffer_free (s->document_root);
-                        array_free (s->exclude_exts);
-  
-                        free (s);
-                }
-                free (p->config_storage);
-        }
-        buffer_free (p->range_buf);
-
-        free (p);
-  
-        return HANDLER_GO_ON;
-}
-
-SETDEFAULTS_FUNC(mod_glusterfs_set_defaults) {
-        plugin_data *p = p_d;
-        size_t i = 0;
-  
-        config_values_t cv[] = {
-                { "glusterfs.logfile",              NULL, T_CONFIG_STRING,
-                  T_CONFIG_SCOPE_CONNECTION },
-    
-                { "glusterfs.loglevel",             NULL, T_CONFIG_STRING,
-                  T_CONFIG_SCOPE_CONNECTION },    
-
-                { "glusterfs.volume-specfile",      NULL, T_CONFIG_STRING,
-                  T_CONFIG_SCOPE_CONNECTION }, 
-
-                { "glusterfs.cache-timeout",        NULL, T_CONFIG_SHORT,
-                  T_CONFIG_SCOPE_CONNECTION },
-    
-                { "glusterfs.exclude-extensions",   NULL, T_CONFIG_ARRAY,
-                  T_CONFIG_SCOPE_CONNECTION },
-    
-                /*TODO: get the prefix from config_conext and 
-                  remove glusterfs.prefix from conf file */
-                { "glusterfs.prefix",               NULL, T_CONFIG_STRING,
-                  T_CONFIG_SCOPE_CONNECTION },
-    
-                { "glusterfs.xattr-interface-size-limit", NULL, T_CONFIG_STRING,
-                  T_CONFIG_SCOPE_CONNECTION },
-
-                { "glusterfs.document-root",        NULL, T_CONFIG_STRING,
-                  T_CONFIG_SCOPE_CONNECTION },
-    
-                { NULL,                          NULL, T_CONFIG_UNSET,
-                  T_CONFIG_SCOPE_UNSET }
-        };
-  
-        p->config_storage = calloc(1,
-                                   srv->config_context->used
-                                   * sizeof(specific_config *));
-        /* ERR_ABORT (p->config_storage);*/
-        p->range_buf = buffer_init ();
-  
-        for (i = 0; i < srv->config_context->used; i++) {
-                plugin_config *s;
-
-                s = calloc(1, sizeof(plugin_config));
-                /* ERR_ABORT (s); */
-                s->logfile = buffer_init ();
-                s->loglevel = buffer_init ();
-                s->specfile = buffer_init ();
-                s->document_root = buffer_init ();
-                s->exclude_exts = array_init ();
-                s->prefix = buffer_init ();
-                s->xattr_file_size = buffer_init ();
-    
-                cv[0].destination = s->logfile;
-                cv[1].destination = s->loglevel;
-                cv[2].destination = s->specfile;
-                cv[3].destination = &s->cache_timeout;
-                cv[4].destination = s->exclude_exts;
-                cv[5].destination = s->prefix;
-                cv[6].destination = s->xattr_file_size;
-                cv[7].destination = s->document_root;
-                p->config_storage[i] = s;
-    
-                if (0 != config_insert_values_global(srv,
-                                                     ((data_config *)srv->config_context->data[i])->value,
-                                                     cv)) {
-                        return HANDLER_FINISHED;
-                }
-        }
-  
-        return HANDLER_GO_ON;
-}
-
-#define PATCH(x)                                \
-        p->conf.x = s->x;
-
-static int mod_glusterfs_patch_connection(server *srv, connection *con,
-                                          plugin_data *p) {
-        size_t i, j;
-        plugin_config *s;
-
-        /* skip the first, the global context */
-        /* 
-           glusterfs related config can only occur inside 
-           $HTTP["url"] == "<glusterfs-prefix>"
-        */
-        p->conf.logfile = NULL;
-        p->conf.loglevel = NULL;
-        p->conf.specfile = NULL;
-        p->conf.cache_timeout = 0;
-        p->conf.exclude_exts = NULL;
-        p->conf.prefix = NULL;
-        p->conf.xattr_file_size = NULL;
-        p->conf.document_root = NULL;
-
-        for (i = 1; i < srv->config_context->used; i++) {
-                data_config *dc = (data_config *)srv->config_context->data[i];
-                s = p->config_storage[i];
-
-                /* condition didn't match */
-                if (!config_check_cond(srv, con, dc)) continue;
-    
-                /* merge config */
-                for (j = 0; j < dc->value->used; j++) {
-                        data_unset *du = dc->value->data[j];
-      
-                        if (buffer_is_equal_string (du->key,
-                                                    CONST_STR_LEN("glusterfs.logfile"))) {
-                                PATCH (logfile);
-                        } else if (buffer_is_equal_string (du->key,
-                                                           CONST_STR_LEN("glusterfs.loglevel"))) {
-                                PATCH (loglevel);
-                        } else if (buffer_is_equal_string (du->key,
-                                                           CONST_STR_LEN ("glusterfs.volume-specfile"))) {
-                                PATCH (specfile);
-                        } else if (buffer_is_equal_string (du->key,
-                                                           CONST_STR_LEN("glusterfs.cache-timeout"))) {
-                                PATCH (cache_timeout);
-                        } else if (buffer_is_equal_string (du->key,
-                                                           CONST_STR_LEN ("glusterfs.exclude-extensions"))) {
-                                PATCH (exclude_exts);
-                        } else if (buffer_is_equal_string (du->key,
-                                                           CONST_STR_LEN ("glusterfs.prefix"))) {
-                                PATCH (prefix);
-                        } else if (buffer_is_equal_string (du->key,
-                                                           CONST_STR_LEN ("glusterfs.xattr-interface-size-limit"))) {
-                                PATCH (xattr_file_size);
-                        } else if (buffer_is_equal_string (du->key,
-                                                           CONST_STR_LEN ("glusterfs.document-root"))) {
-                                PATCH (document_root);
-                        }
-                }
-        }
-        return 0;
-}
-
-#undef PATCH
-
-static int http_response_parse_range(server *srv, connection *con,
-                                     plugin_data *p) {
-        int multipart = 0;
-        int error;
-        off_t start, end;
-        const char *s, *minus;
-        char *boundary = "fkj49sn38dcn3";
-        data_string *ds;
-        stat_cache_entry *sce = NULL;
-        buffer *content_type = NULL;
-        size_t size = 0;
-        mod_glusterfs_ctx_t *ctx = con->plugin_ctx[p->id];
-
-        if (p->conf.xattr_file_size && p->conf.xattr_file_size->ptr) {
-                size = atoi (p->conf.xattr_file_size->ptr);
-        }
-
-        if (HANDLER_ERROR == stat_cache_get_entry(srv, con, con->physical.path,
-                                                  &sce)) {
-                SEGFAULT();
-        }
-  
-        start = 0;
-        end = sce->st.st_size - 1;
-  
-        con->response.content_length = 0;
-  
-        if (NULL != (ds = (data_string *)array_get_element(con->response.headers,
-                                                           "Content-Type"))) {
-                content_type = ds->value;
-        }
-  
-        for (s = con->request.http_range, error = 0;
-             !error && *s && NULL != (minus = strchr(s, '-')); ) {
-                char *err;
-                off_t la, le;
-    
-                if (s == minus) {
-                        /* -<stop> */
-      
-                        le = strtoll(s, &err, 10);
-      
-                        if (le == 0) {
-                                /* RFC 2616 - 14.35.1 */
-        
-                                con->http_status = 416;
-                                error = 1;
-                        } else if (*err == '\0') {
-                                /* end */
-                                s = err;
-        
-                                end = sce->st.st_size - 1;
-                                start = sce->st.st_size + le;
-                        } else if (*err == ',') {
-                                multipart = 1;
-                                s = err + 1;
-        
-                                end = sce->st.st_size - 1;
-                                start = sce->st.st_size + le;
-                        } else {
-                                error = 1;
-                        }
-      
-                } else if (*(minus+1) == '\0' || *(minus+1) == ',') {
-                        /* <start>- */
-      
-                        la = strtoll(s, &err, 10);
-      
-                        if (err == minus) {
-                                /* ok */
-        
-                                if (*(err + 1) == '\0') {
-                                        s = err + 1;
-          
-                                        end = sce->st.st_size - 1;
-                                        start = la;
-          
-                                } else if (*(err + 1) == ',') {
-                                        multipart = 1;
-                                        s = err + 2;
-          
-                                        end = sce->st.st_size - 1;
-                                        start = la;
-                                } else {
-                                        error = 1;
-                                }
-                        } else {
-                                /* error */
-                                error = 1;
-                        }
-                } else {
-                        /* <start>-<stop> */
-      
-                        la = strtoll(s, &err, 10);
-
-                        if (err == minus) {
-                                le = strtoll(minus+1, &err, 10);
-        
-                                /* RFC 2616 - 14.35.1 */
-                                if (la > le) {
-                                        error = 1;
-                                }
-        
-                                if (*err == '\0') {
-                                        /* ok, end*/
-                                        s = err;
-
-                                        end = le;
-                                        start = la;
-                                } else if (*err == ',') {
-                                        multipart = 1;
-                                        s = err + 1;
-          
-                                        end = le;
-                                        start = la;
-                                } else {
-                                        /* error */
-          
-                                        error = 1;
-                                }
-                        } else {
-                                /* error */
-        
-                                error = 1;
-                        }
-                }
-    
-                if (!error) {
-                        if (start < 0) start = 0;
-      
-                        /* RFC 2616 - 14.35.1 */
-                        if (end > sce->st.st_size - 1) end = sce->st.st_size - 1;
-      
-                        if (start > sce->st.st_size - 1) {
-                                error = 1;
-        
-                                con->http_status = 416;
-                        }
-                }
-    
-                if (!error) {
-                        if (multipart) {
-                                /* write boundary-header */
-                                buffer *b;
-        
-                                b = chunkqueue_get_append_buffer(con->write_queue);
-        
-                                buffer_copy_string(b, "\r\n--");
-                                buffer_append_string(b, boundary);
-        
-                                /* write Content-Range */
-                                buffer_append_string(b, "\r\nContent-Range: "
-                                                     "bytes ");
-                                buffer_append_off_t(b, start);
-                                buffer_append_string(b, "-");
-                                buffer_append_off_t(b, end);
-                                buffer_append_string(b, "/");
-                                buffer_append_off_t(b, sce->st.st_size);
-        
-                                buffer_append_string(b, "\r\nContent-Type: ");
-                                buffer_append_string_buffer(b, content_type);
-        
-                                /* write END-OF-HEADER */
-                                buffer_append_string(b, "\r\n\r\n");
-        
-                                con->response.content_length += b->used - 1;
-
-                        }
-
-                        if ((size_t)sce->st.st_size >= size) {
-                                chunkqueue_append_glusterfs_file(con, ctx->fd,
-                                                                 start,
-                                                                 end - start,
-                                                                 size);
-                        } else {
-                                if (!start) {
-                                        buffer *mem = buffer_init ();
-                                        mem->ptr = ctx->buf;
-                                        mem->used = mem->size = sce->st.st_size;
-                                        http_chunk_append_buffer (srv, con, mem);
-                                        ctx->buf = NULL;
-                                } else {
-                                        chunkqueue_append_mem (con->write_queue,
-                                                               ((char *)ctx->buf)
-                                                               + start,
-                                                               end - start + 1);
-                                }
-                        }
-
-                        con->response.content_length += end - start + 1;
-                }
-        }
-
-        if (ctx->buf) {
-                free (ctx->buf);
-                ctx->buf = NULL;
-        }
-
-        /* something went wrong */
-        if (error) return -1;
-  
-        if (multipart) {
-                /* add boundary end */
-                buffer *b;
-    
-                b = chunkqueue_get_append_buffer(con->write_queue);
-    
-                buffer_copy_string_len(b, "\r\n--", 4);
-                buffer_append_string(b, boundary);
-                buffer_append_string_len(b, "--\r\n", 4);
-    
-                con->response.content_length += b->used - 1;
-    
-                /* set header-fields */
-
-                buffer_copy_string(p->range_buf, "multipart/byteranges; boundary=");
-                buffer_append_string(p->range_buf, boundary);
-
-                /* overwrite content-type */
-                response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"),
-                                          CONST_BUF_LEN(p->range_buf));
-        } else {
-                /* add Content-Range-header */
-    
-                buffer_copy_string(p->range_buf, "bytes ");
-                buffer_append_off_t(p->range_buf, start);
-                buffer_append_string(p->range_buf, "-");
-                buffer_append_off_t(p->range_buf, end);
-                buffer_append_string(p->range_buf, "/");
-                buffer_append_off_t(p->range_buf, sce->st.st_size);
-    
-                response_header_insert(srv, con, CONST_STR_LEN("Content-Range"),
-                                       CONST_BUF_LEN(p->range_buf));
-        }
-        
-        /* ok, the file is set-up */
-        return 0;
-}
-
-PHYSICALPATH_FUNC(mod_glusterfs_handle_physical) {
-        plugin_data *p = p_d;
-        stat_cache_entry *sce;
-        mod_glusterfs_ctx_t *plugin_ctx = NULL;
-        size_t size = 0;
-        int ret = 0;
-
-        if (con->http_status != 0) return HANDLER_GO_ON;
-        if (con->uri.path->used == 0) return HANDLER_GO_ON;
-        if (con->physical.path->used == 0) return HANDLER_GO_ON;
-
-        if (con->mode != DIRECT) return HANDLER_GO_ON;
-
-        /*
-          network_backend_write = srv->network_backend_write;
-          srv->network_backend_write = mod_glusterfs_network_backend_write;
-        */
-
-        switch (con->request.http_method) {
-        case HTTP_METHOD_GET:
-        case HTTP_METHOD_POST:
-        case HTTP_METHOD_HEAD:
-                break;
-
-        default:
-                return HANDLER_GO_ON;
-        }
-
-        mod_glusterfs_patch_connection(srv, con, p);
-        if (!p->conf.prefix || p->conf.prefix->used == 0) {
-                return HANDLER_GO_ON;
-        }
-
-        if (!p->conf.document_root || p->conf.document_root->used == 0) {
-                log_error_write(srv, __FILE__, __LINE__, "s",
-                                "glusterfs.document-root is not specified");
-                con->http_status = 500;
-                return HANDLER_FINISHED;
-        }
-
-        if (!p->conf.mounted) {
-                glusterfs_init_params_t ctx;
-
-                if (!p->conf.specfile || p->conf.specfile->used == 0) {
-                        return HANDLER_GO_ON;
-                }
-                memset (&ctx, 0, sizeof (ctx));
-
-                ctx.specfile = p->conf.specfile->ptr;
-                ctx.logfile = p->conf.logfile->ptr;
-                ctx.loglevel = p->conf.loglevel->ptr;
-                ctx.lookup_timeout = ctx.stat_timeout = p->conf.cache_timeout;
-
-                ret = glusterfs_mount (p->conf.prefix->ptr, &ctx);
-                if (ret != 0) {
-                        con->http_status = 500;
-                        log_error_write(srv, __FILE__, __LINE__,  "sbs",
-                                        "glusterfs initialization failed, "
-                                        "please check your configuration. "
-                                        "Glusterfs logfile ",
-                                        p->conf.logfile,
-                                        "might contain details");
-                        return HANDLER_FINISHED;
-                }
-                p->conf.mounted = 1;
-        }
-
-        size = 0;
-        if (p->conf.xattr_file_size && p->conf.xattr_file_size->ptr) 
-                size = atoi (p->conf.xattr_file_size->ptr);
-
-        if (!con->plugin_ctx[p->id]) {
-/* FIXME: what if multiple files are requested from a single connection? */ 
-/* TODO: check whether this works fine for HTTP protocol 1.1 */
-
-                buffer *tmp_buf = buffer_init_buffer (con->physical.basedir);
-
-                plugin_ctx = calloc (1, sizeof (*plugin_ctx));
-                /* ERR_ABORT (plugin_ctx); */
-                con->plugin_ctx[p->id] = plugin_ctx;
-    
-                buffer_append_string_buffer (tmp_buf, p->conf.prefix);
-                buffer_path_simplify (tmp_buf, tmp_buf);
-
-                plugin_ctx->prefix = tmp_buf->used - 1;
-                if (tmp_buf->ptr[plugin_ctx->prefix - 1] == '/')
-                        plugin_ctx->prefix--;
-
-                buffer_free (tmp_buf);
-        } else 
-                /*FIXME: error!! error!! */
-                plugin_ctx = con->plugin_ctx[p->id];
-
-
-        if (size) 
-        {
-                plugin_ctx->buf = malloc (size);
-                /* ERR_ABORT (plugin_ctx->buf); */
-        }
-
-        plugin_ctx->glusterfs_path = buffer_init ();
-        buffer_copy_string_buffer (plugin_ctx->glusterfs_path,
-                                   p->conf.prefix);
-        buffer_append_string (plugin_ctx->glusterfs_path,
-                              p->conf.document_root->ptr);
-        buffer_append_string (plugin_ctx->glusterfs_path, "/");
-        buffer_append_string (plugin_ctx->glusterfs_path,
-                              con->physical.path->ptr + plugin_ctx->prefix);
-        buffer_path_simplify (plugin_ctx->glusterfs_path,
-                              plugin_ctx->glusterfs_path);
- 
-        if (glusterfs_stat_cache_get_entry (srv, con,
-                                            plugin_ctx->glusterfs_path,
-                                            con->physical.path, plugin_ctx->buf,
-                                            size, &sce) == HANDLER_ERROR) {
-                if (errno == ENOENT)
-                        con->http_status = 404;
-                else 
-                        con->http_status = 403;
-
-                free (plugin_ctx->buf);
-                buffer_free (plugin_ctx->glusterfs_path);
-                plugin_ctx->glusterfs_path = NULL;
-                plugin_ctx->buf = NULL;
-
-                free (plugin_ctx);
-                con->plugin_ctx[p->id] = NULL;
-
-                return HANDLER_FINISHED;
-        }
-
-        if (!(S_ISREG (sce->st.st_mode) && (size_t)sce->st.st_size < size)) {
-                free (plugin_ctx->buf);
-                plugin_ctx->buf = NULL;
-        }
-
-        return HANDLER_GO_ON;
-}
-
-static int http_chunk_append_len(server *srv, connection *con, size_t len) {
-        size_t i, olen = len, j;
-        buffer *b;
-
-        b = srv->tmp_chunk_len;
-
-        if (len == 0) {
-                buffer_copy_string(b, "0");
-        } else {
-                for (i = 0; i < 8 && len; i++) {
-                        len >>= 4;
-                }
-    
-                /* i is the number of hex digits we have */
-                buffer_prepare_copy(b, i + 1);
-    
-                for (j = i-1, len = olen; j+1 > 0; j--) {
-                        b->ptr[j] = (len & 0xf) + (((len & 0xf) <= 9) ? 
-                                                   '0' : 'a' - 10);
-                        len >>= 4;
-                }
-                b->used = i;
-                b->ptr[b->used++] = '\0';
-        }
-  
-        buffer_append_string(b, "\r\n");
-        chunkqueue_append_buffer(con->write_queue, b);
-  
-        return 0;
-}
-
-int http_chunk_append_glusterfs_file_chunk(server *srv, connection *con,
-                                           glusterfs_file_t fd, off_t offset,
-                                           off_t len, size_t buf_size) {
-        chunkqueue *cq;
-
-        if (!con) return -1;
-
-        cq = con->write_queue;
-
-        if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) {
-                http_chunk_append_len(srv, con, len);
-        }
-
-        chunkqueue_append_glusterfs_file (con, fd, offset, len, buf_size);
-
-        if ((con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) 
-            && (len > 0)) {
-                chunkqueue_append_mem(cq, "\r\n", 2 + 1);
-        }
-
-        return 0;
-}
-
-int http_chunk_append_glusterfs_mem(server *srv, connection *con,
-                                    char * mem, size_t len,
-                                    size_t buf_size) 
-{
-        chunkqueue *cq = NULL;
-        buffer *buf = NULL;
- 
-        if (!con) return -1;
-  
-        cq = con->write_queue;
-  
-        if (len == 0) {
-                free (mem);
-                if (con->response.transfer_encoding 
-                    & HTTP_TRANSFER_ENCODING_CHUNKED) {
-                        chunkqueue_append_mem(cq, "0\r\n\r\n", 5 + 1);
-                } else {
-                        chunkqueue_append_mem(cq, "", 1);
-                }
-                return 0;
-        }
-
-        if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) {
-                http_chunk_append_len(srv, con, len - 1);
-        }
-  
-        buf = buffer_init ();
-
-        buf->used = len + 1;
-        buf->size = buf_size;
-        buf->ptr = (char *)mem;
-        chunkqueue_append_buffer_weak (cq, buf);
-
-        if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) {
-                chunkqueue_append_mem(cq, "\r\n", 2 + 1);
-        }
-
-        return 0;
-}
-
-
-
-URIHANDLER_FUNC(mod_glusterfs_subrequest) {
-        plugin_data *p = p_d;
-        stat_cache_entry *sce = NULL;
-        int s_len;
-        char allow_caching = 1;
-        size_t size = 0;
-        mod_glusterfs_ctx_t *ctx = con->plugin_ctx[p->id];
-
-        /* someone else has done a decision for us */
-        if (con->http_status != 0) return HANDLER_GO_ON;
-        if (con->uri.path->used == 0) return HANDLER_GO_ON;
-        if (con->physical.path->used == 0) return HANDLER_GO_ON;
-  
-        /* someone else has handled this request */
-        if (con->mode != DIRECT) return HANDLER_GO_ON;
-  
-        /* we only handle GET, POST and HEAD */
-        switch(con->request.http_method) {
-        case HTTP_METHOD_GET:
-        case HTTP_METHOD_POST:
-        case HTTP_METHOD_HEAD:
-                break;
-        default:
-                return HANDLER_GO_ON;
-        }
-  
-        mod_glusterfs_patch_connection(srv, con, p);
-  
-        if (!p->conf.prefix || !p->conf.prefix->used)
-                return HANDLER_GO_ON;
-
-        s_len = con->uri.path->used - 1;
-        /* ignore certain extensions */
-        /*
-          for (k = 0; k < p->conf.exclude_exts->used; k++) {
-          data_string *ds;
-          ds = (data_string *)p->conf.exclude_exts->data[k];
-          
-          if (ds->value->used == 0) continue;
-    
-          if (!strncmp (ds->value->ptr, con->uri.path->ptr,
-                        strlen (ds->value->ptr)))
-          break;
-          }
-  
-          if (k == p->conf.exclude_exts->used) {
-          return HANDLER_GO_ON;
-          }
-        */
-
-        if (con->conf.log_request_handling) {
-                log_error_write(srv, __FILE__, __LINE__,  "s",
-                                "-- serving file from glusterfs");
-        }
-
-        if (HANDLER_ERROR == stat_cache_get_entry(srv, con, con->physical.path,
-                                                  &sce)) {
-                con->http_status = 403;
-                
-                log_error_write(srv, __FILE__, __LINE__, "sbsb",
-                                "not a regular file:", con->uri.path,
-                                "->", con->physical.path);
-    
-                free (ctx);
-                con->plugin_ctx[p->id] = NULL;
-
-                return HANDLER_FINISHED;
-        }
-
-        if (con->uri.path->ptr[s_len] == '/' || !S_ISREG(sce->st.st_mode)) {
-                free (ctx);
-                con->plugin_ctx[p->id] = NULL;
-                return HANDLER_FINISHED;
-        }
-
-        if (p->conf.xattr_file_size && p->conf.xattr_file_size->ptr)
-                size = atoi (p->conf.xattr_file_size->ptr);
-
-        if ((size_t)sce->st.st_size > size) {
-                ctx->fd = glusterfs_open (ctx->glusterfs_path->ptr, O_RDONLY,
-                                          0);
-    
-                if (((long)ctx->fd) == 0) {
-                        con->http_status = 403;
-                        free (ctx);
-                        con->plugin_ctx[p->id] = NULL;
-                        return HANDLER_FINISHED;
-                }
-        }
-
-        buffer_free (ctx->glusterfs_path);
-        ctx->glusterfs_path = NULL;
-
-        /* we only handline regular files */
-#ifdef HAVE_LSTAT
-        if ((sce->is_symlink == 1) && !con->conf.follow_symlink) {
-                con->http_status = 403;
-          
-                if (con->conf.log_request_handling) {
-                        log_error_write(srv, __FILE__, __LINE__,  "s",
-                                        "-- access denied due symlink "
-                                        "restriction");
-                        log_error_write(srv, __FILE__, __LINE__,  "sb",
-                                        "Path         :", con->physical.path);
-                }
-    
-                buffer_reset(con->physical.path);
-                free (ctx);
-                con->plugin_ctx[p->id] = NULL;
-                return HANDLER_FINISHED;
-        }
-#endif
-        if (!S_ISREG(sce->st.st_mode)) {
-                con->http_status = 404;
-    
-                if (con->conf.log_file_not_found) {
-                        log_error_write(srv, __FILE__, __LINE__, "sbsb",
-                                        "not a regular file:", con->uri.path,
-                                        "->", sce->name);
-                }
-    
-                free (ctx);
-                con->plugin_ctx[p->id] = NULL;
-
-                return HANDLER_FINISHED;
-        }
-
-        /* mod_compress might set several data directly, don't overwrite them */
-
-        /* set response content-type, if not set already */
-  
-        if (NULL == array_get_element(con->response.headers, "Content-Type")) {
-                if (buffer_is_empty(sce->content_type)) {
-                        /* we are setting application/octet-stream, but also "
-                         * announce that this header field might change in "
-                         * the seconds few requests. This should fix the 
-                         * aggressive caching of FF and the script download
-                         * seen by the first installations
-                         */
-                        response_header_overwrite(srv, con,
-                                                  CONST_STR_LEN("Content-Type"),
-                                                  CONST_STR_LEN("application/"
-                                                                "octet-stream"));
-      
-                        allow_caching = 0;
-                } else {
-                        response_header_overwrite(srv, con,
-                                                  CONST_STR_LEN("Content-Type"),
-                                                  CONST_BUF_LEN(sce->content_type));
-                }
-        }
-  
-        if (con->conf.range_requests) {
-                response_header_overwrite(srv, con,
-                                          CONST_STR_LEN("Accept-Ranges"),
-                                          CONST_STR_LEN("bytes"));
-        }
-
-        /* TODO: Allow Cachable requests */     
-#if 0
-        if (allow_caching) {
-                if (p->conf.etags_used && con->etag_flags != 0 
-                    && !buffer_is_empty(sce->etag)) {
-                        if (NULL == array_get_element(con->response.headers,
-                                                      "ETag")) {
-                                /* generate e-tag */
-                                etag_mutate(con->physical.etag, sce->etag);
-        
-                                response_header_overwrite(srv, con,
-                                                          CONST_STR_LEN("ETag"),
-                                                          CONST_BUF_LEN(con->physical.etag));
-                        }
-                }
-    
-                /* prepare header */
-                if (NULL == (ds = (data_string *)array_get_element(con->response.headers,
-                                                                   "Last-Modified"))) {
-                        mtime = strftime_cache_get(srv, sce->st.st_mtime);
-                        response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"),
-                                                  CONST_BUF_LEN(mtime));
-                } else {
-                        mtime = ds->value;
-                }
-    
-                if (HANDLER_FINISHED == http_response_handle_cachable(srv, con,
-                                                                      mtime)) {
-                        free (ctx);
-                        con->plugin_ctx[p->id] = NULL;
-                        return HANDLER_FINISHED;
-                }
-        }
-#endif 
-  
-        /*TODO: Read about etags */
-        if (con->request.http_range && con->conf.range_requests) {
-                int do_range_request = 1;
-                data_string *ds = NULL;
-                buffer *mtime = NULL;
-                /* check if we have a conditional GET */
-    
-                /* prepare header */
-                if (NULL == (ds = (data_string *)array_get_element(con->response.headers,
-                                                                   "Last-Modified"))) {
-                        mtime = strftime_cache_get(srv, sce->st.st_mtime);
-                        response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"),
-                                                  CONST_BUF_LEN(mtime));
-                } else {
-                        mtime = ds->value;
-                }
-
-                if (NULL != (ds = (data_string *)array_get_element(con->request.headers,
-                                                                   "If-Range"))) {
-                        /* if the value is the same as our ETag, we do a Range-request,
-                         * otherwise a full 200 */
-            
-                        if (ds->value->ptr[0] == '"') {
-                                /**
-                                 * client wants a ETag
-                                 */
-                                if (!con->physical.etag) {
-                                        do_range_request = 0;
-                                } else if (!buffer_is_equal(ds->value,
-                                                            con->physical.etag)) {
-                                        do_range_request = 0;
-                                }
-                        } else if (!mtime) {
-                                /**
-                                 * we don't have a Last-Modified and can match 
-                                 * the If-Range: 
-                                 *
-                                 * sending all
-                                 */
-                                do_range_request = 0;
-                        } else if (!buffer_is_equal(ds->value, mtime)) {
-                                do_range_request = 0;
-                        }
-                }
-    
-                if (do_range_request) {
-                        /* content prepared, I'm done */
-                        con->file_finished = 1;
-            
-                        if (0 == http_response_parse_range(srv, con, p)) {
-                                con->http_status = 206;
-                        }
-
-                        free (ctx);
-                        con->plugin_ctx[p->id] = NULL;
-                        return HANDLER_FINISHED;
-                }
-        }
-  
-        /* if we are still here, prepare body */
-
-        /* we add it here for all requests
-         * the HEAD request will drop it afterwards again
-         */
-        /*TODO check whether 1 should be subtracted */
-
-        if (p->conf.xattr_file_size && p->conf.xattr_file_size->ptr)
-                size = atoi (p->conf.xattr_file_size->ptr);
-
-        if (size <= (size_t)sce->st.st_size) {
-                http_chunk_append_glusterfs_file_chunk (srv, con, ctx->fd, 0,
-                                                        sce->st.st_size, size);
-        } else {
-                http_chunk_append_glusterfs_mem (srv, con, ctx->buf,
-                                                 sce->st.st_size, size);
-        }
-        
-        con->http_status = 200;
-        con->file_finished = 1;
-  
-        free (ctx);
-        con->plugin_ctx[p->id] = NULL;
-
-        return HANDLER_FINISHED;
-}
-
-#if 0
-URIHANDLER_FUNC(mod_glusterfs_request_done)
-{
-        mod_glusterfs_iobuf_t *cur = first, *prev;
-        while (cur) {
-                prev =  cur;
-                glusterfs_free (cur->buf);
-                cur = cur->next;
-                free (prev);
-        }
-        first = NULL
-                }
-#endif
-
-/* this function is called at dlopen() time and inits the callbacks */
-CONNECTION_FUNC(mod_glusterfs_connection_reset) 
-{
-        (void) p_d;
-        (void) con;
-        if (!network_backend_write)
-                network_backend_write = srv->network_backend_write;
-
-        srv->network_backend_write = mod_glusterfs_network_backend_write;
-
-        return HANDLER_GO_ON;
-}
-
-int mod_glusterfs_plugin_init(plugin *p) {
-        p->version     = LIGHTTPD_VERSION_ID;
-        p->name        = buffer_init_string("glusterfs");
-        p->init        = mod_glusterfs_init;
-        p->handle_physical = mod_glusterfs_handle_physical;
-        p->handle_subrequest_start = mod_glusterfs_subrequest;
-        //      p->handle_request_done = mod_glusterfs_request_done;
-        p->set_defaults  = mod_glusterfs_set_defaults;
-        p->connection_reset = mod_glusterfs_connection_reset;
-        p->cleanup     = mod_glusterfs_free;
-  
-        p->data        = NULL;
-  
-        return 0;
-}
-
-
-/* mod_glusterfs_stat_cache */
-static stat_cache_entry * stat_cache_entry_init(void) {
-        stat_cache_entry *sce = NULL;
-  
-        sce = calloc(1, sizeof(*sce));
-        /* ERR_ABORT (sce); */
-  
-        sce->name = buffer_init();
-        sce->etag = buffer_init();
-        sce->content_type = buffer_init();
-
-        return sce;
-}
-
-#ifdef HAVE_FAM_H
-static fam_dir_entry * fam_dir_entry_init(void) {
-        fam_dir_entry *fam_dir = NULL;
-  
-        fam_dir = calloc(1, sizeof(*fam_dir));
-        /* ERR_ABORT (fam_dir); */
-  
-        fam_dir->name = buffer_init();
-  
-        return fam_dir;
-}
-
-static void fam_dir_entry_free(void *data) {
-        fam_dir_entry *fam_dir = data;
-
-        if (!fam_dir) return;
-
-        FAMCancelMonitor(fam_dir->fc, fam_dir->req);
-
-        buffer_free(fam_dir->name);
-        free(fam_dir->req);
-
-        free(fam_dir);
-}
-#endif
-
-#ifdef HAVE_XATTR
-static int stat_cache_attr_get(buffer *buf, char *name) {
-        int attrlen;
-        int ret;
-  
-        attrlen = 1024;
-        buffer_prepare_copy(buf, attrlen);
-        attrlen--;
-        if(0 == (ret = attr_get(name, "Content-Type", buf->ptr, &attrlen, 0))) {
-                buf->used = attrlen + 1;
-                buf->ptr[attrlen] = '\0';
-        }
-        return ret;
-}
-#endif
-
-/* the famous DJB hash function for strings */
-static uint32_t hashme(buffer *str) {
-        uint32_t hash = 5381;
-        const char *s;
-        for (s = str->ptr; *s; s++) {
-                hash = ((hash << 5) + hash) + *s;
-        }
-  
-        hash &= ~(1 << 31); /* strip the highest bit */
-  
-        return hash;
-}
-
-
-#ifdef HAVE_LSTAT
-static int stat_cache_lstat(server *srv, buffer *dname, struct stat *lst) {
-        if (lstat(dname->ptr, lst) == 0) {
-                return S_ISLNK(lst->st_mode) ? 0 : 1;
-        }
-        else {
-                log_error_write(srv, __FILE__, __LINE__, "sbs",
-                                "lstat failed for:",
-                                dname, strerror(errno));
-        };
-        return -1;
-}
-#endif
-
-/***
- *
- *
- *
- * returns:
- *  - HANDLER_FINISHED on cache-miss (don't forget to reopen the file)
- *  - HANDLER_ERROR on stat() failed -> see errno for problem
- */
-
-handler_t glusterfs_stat_cache_get_entry(server *srv, 
-                                         connection *con, 
-                                         buffer *glusterfs_path,
-                                         buffer *name, 
-                                         void *buf, 
-                                         size_t size, 
-                                         stat_cache_entry **ret_sce) 
-{
-#ifdef HAVE_FAM_H
-        fam_dir_entry *fam_dir = NULL;
-        int dir_ndx = -1;
-        splay_tree *dir_node = NULL;
-#endif
-        stat_cache_entry *sce = NULL;
-        stat_cache *sc;
-        struct stat st; 
-        size_t k;
-#ifdef DEBUG_STAT_CACHE
-        size_t i;
-#endif
-        int file_ndx;
-        splay_tree *file_node = NULL;
-
-        *ret_sce = NULL;
-        memset (&st, 0, sizeof (st));
-
-        /*
-         * check if the directory for this file has changed
-         */
-
-        sc = srv->stat_cache;
-
-        buffer_copy_string_buffer(sc->hash_key, name);
-        buffer_append_long(sc->hash_key, con->conf.follow_symlink);
-
-        file_ndx = hashme(sc->hash_key);
-        sc->files = splaytree_splay(sc->files, file_ndx);
-
-#ifdef DEBUG_STAT_CACHE
-        for (i = 0; i < ctrl.used; i++) {
-                if (ctrl.ptr[i] == file_ndx) break;
-        }
-#endif
-
-        if (sc->files && (sc->files->key == file_ndx)) {
-#ifdef DEBUG_STAT_CACHE
-                /* it was in the cache */
-                assert(i < ctrl.used);
-#endif
-
-                /* we have seen this file already and
-                 * don't stat() it again in the same second */
-
-                file_node = sc->files;
-
-                sce = file_node->data;
-
-                /* check if the name is the same, we might have a collision */
-
-                if (buffer_is_equal(name, sce->name)) {
-                        if (srv->srvconf.stat_cache_engine 
-                            == STAT_CACHE_ENGINE_SIMPLE) {
-                                if (sce->stat_ts == srv->cur_ts && !buf) {
-                                        *ret_sce = sce;
-                                        return HANDLER_GO_ON;
-                                }
-                        }
-                } else {
-                        /* oops, a collision,
-                         *
-                         * file_node is used by the FAM check below to see if 
-                         * we know this file and if we can save a stat().
-                         *
-                         * BUT, the sce is not reset here as the entry into 
-                         * the cache is ok, we it is just not pointing to 
-                         * our requested file.
-                         */
-
-                        file_node = NULL;
-                }
-        } else {
-#ifdef DEBUG_STAT_CACHE
-                if (i != ctrl.used) {
-                        fprintf(stderr, "%s.%d: %08x was already inserted "
-                                "but not found in cache, %s\n",
-                                __FILE__, __LINE__, file_ndx, name->ptr);
-                }
-                assert(i == ctrl.used);
-#endif
-        }
-        /*
-         * *lol*
-         * - open() + fstat() on a named-pipe results in a (intended) hang.
-         * - stat() if regular file + open() to see if we can read from it is 
-         *   better
-         *
-         * */
-        if (-1 == glusterfs_get (glusterfs_path->ptr, buf, size, &st)) {
-                return HANDLER_ERROR;
-        }
-
-        if (NULL == sce) {
-                int osize = 0;
-
-                if (sc->files) {
-                        osize = sc->files->size;
-                }
-
-                sce = stat_cache_entry_init();
-                buffer_copy_string_buffer(sce->name, name);
-
-                sc->files = splaytree_insert(sc->files, file_ndx, sce);
-#ifdef DEBUG_STAT_CACHE
-                if (ctrl.size == 0) {
-                        ctrl.size = 16;
-                        ctrl.used = 0;
-                        ctrl.ptr = malloc(ctrl.size * sizeof(*ctrl.ptr));
-                        /* ERR_ABORT (ctrl.ptr); */
-                } else if (ctrl.size == ctrl.used) {
-                        ctrl.size += 16;
-                        ctrl.ptr = realloc(ctrl.ptr,
-                                           ctrl.size * sizeof(*ctrl.ptr));
-                        /* ERR_ABORT (ctrl.ptr); */
-                }
-
-                ctrl.ptr[ctrl.used++] = file_ndx;
-
-                assert(sc->files);
-                assert(sc->files->data == sce);
-                assert(osize + 1 == splaytree_size(sc->files));
-#endif
-        }
-
-        sce->st = st;
-        sce->stat_ts = srv->cur_ts;
-
-        /* catch the obvious symlinks
-         *
-         * this is not a secure check as we still have a race-condition between
-         * the stat() and the open. We can only solve this by
-         * 1. open() the file
-         * 2. fstat() the fd
-         *
-         * and keeping the file open for the rest of the time. But this can
-         * only be done at network level.
-         *
-         * per default it is not a symlink
-         * */
-#ifdef HAVE_LSTAT
-        sce->is_symlink = 0;
-
-        /* we want to only check for symlinks if we should block symlinks.
-         */
-        if (!con->conf.follow_symlink) {
-                if (stat_cache_lstat(srv, name, &lst)  == 0) {
-#ifdef DEBUG_STAT_CACHE
-                        log_error_write(srv, __FILE__, __LINE__, "sb",
-                                        "found symlink", name);
-#endif
-                        sce->is_symlink = 1;
-                }
-
-                /*
-                 * we assume "/" can not be symlink, so
-                 * skip the symlink stuff if our path is /
-                 **/
-                else if ((name->used > 2)) {
-                        buffer *dname;
-                        char *s_cur;
-
-                        dname = buffer_init();
-                        buffer_copy_string_buffer(dname, name);
-
-                        while ((s_cur = strrchr(dname->ptr,'/'))) {
-                                *s_cur = '\0';
-                                dname->used = s_cur - dname->ptr + 1;
-                                if (dname->ptr == s_cur) {
-#ifdef DEBUG_STAT_CACHE
-                                        log_error_write(srv, __FILE__, __LINE__,
-                                                        "s", "reached /");
-#endif
-                                        break;
-                                }
-#ifdef DEBUG_STAT_CACHE
-                                log_error_write(srv, __FILE__, __LINE__, "sbs",
-                                                "checking if", dname, "is a "
-                                                "symlink");
-#endif
-                                if (stat_cache_lstat(srv, dname, &lst)  == 0) {
-                                        sce->is_symlink = 1;
-#ifdef DEBUG_STAT_CACHE
-                                        log_error_write(srv, __FILE__, __LINE__,
-                                                        "sb",
-                                                        "found symlink", dname);
-#endif
-                                        break;
-                                };
-                        };
-                        buffer_free(dname);
-                };
-        };
-#endif
-
-        if (S_ISREG(st.st_mode)) {
-                /* determine mimetype */
-                buffer_reset(sce->content_type);
-
-                for (k = 0; k < con->conf.mimetypes->used; k++) {
-                        data_string *ds = NULL;
-                        buffer *type = NULL;
-
-                        ds = (data_string *)con->conf.mimetypes->data[k];
-                        type = ds->key;
-                        if (type->used == 0) continue;
-
-                        /* check if the right side is the same */
-                        if (type->used > name->used) continue;
-
-                        if (0 == strncasecmp(name->ptr + name->used - type->used,
-                                             type->ptr, type->used - 1)) {
-                                buffer_copy_string_buffer(sce->content_type,
-                                                          ds->value);
-                                break;
-                        }
-                }
-                etag_create(sce->etag, &(sce->st), con->etag_flags);
-#ifdef HAVE_XATTR
-                if (con->conf.use_xattr && buffer_is_empty(sce->content_type)) {
-                        stat_cache_attr_get(sce->content_type, name->ptr);
-                }
-#endif
-        } else if (S_ISDIR(st.st_mode)) {
-                etag_create(sce->etag, &(sce->st), con->etag_flags);
-        }
-
-#ifdef HAVE_FAM_H
-        if (sc->fam &&
-            (srv->srvconf.stat_cache_engine == STAT_CACHE_ENGINE_FAM)) {
-                /* is this directory already registered ? */
-                if (!dir_node) {
-                        fam_dir = fam_dir_entry_init();
-                        fam_dir->fc = sc->fam;
-
-                        buffer_copy_string_buffer(fam_dir->name, sc->dir_name);
-
-                        fam_dir->version = 1;
-
-                        fam_dir->req = calloc(1, sizeof(FAMRequest));
-                        /* ERR_ABORT (fam_dir->req); */
-
-                        if (0 != FAMMonitorDirectory(sc->fam, fam_dir->name->ptr,
-                                                     fam_dir->req, fam_dir)) {
-
-                                log_error_write(srv, __FILE__, __LINE__, "sbsbs",
-                                                "monitoring dir failed:",
-                                                fam_dir->name, 
-                                                "file:", name,
-                                                FamErrlist[FAMErrno]);
-
-                                fam_dir_entry_free(fam_dir);
-                        } else {
-                                int osize = 0;
-
-                                if (sc->dirs) {
-                                        osize = sc->dirs->size;
-                                }
-
-                                sc->dirs = splaytree_insert(sc->dirs, dir_ndx,
-                                                            fam_dir);
-                                assert(sc->dirs);
-                                assert(sc->dirs->data == fam_dir);
-                                assert(osize == (sc->dirs->size - 1));
-                        }
-                } else {
-                        fam_dir = dir_node->data;
-                }
-
-                /* bind the fam_fc to the stat() cache entry */
-
-                if (fam_dir) {
-                        sce->dir_version = fam_dir->version;
-                        sce->dir_ndx     = dir_ndx;
-                }
-        }
-#endif
-
-        *ret_sce = sce;
-
-        return HANDLER_GO_ON;
-}
-
-/**
- * remove stat() from cache which havn't been stat()ed for
- * more than 10 seconds
- *
- *
- * walk though the stat-cache, collect the ids which are too old
- * and remove them in a second loop
- */
-
-static int stat_cache_tag_old_entries(server *srv, splay_tree *t, int *keys,
-                                      size_t *ndx) {
-        stat_cache_entry *sce;
-
-        if (!t) return 0;
-
-        stat_cache_tag_old_entries(srv, t->left, keys, ndx);
-        stat_cache_tag_old_entries(srv, t->right, keys, ndx);
-
-        sce = t->data;
-
-        if (srv->cur_ts - sce->stat_ts > 2) {
-                keys[(*ndx)++] = t->key;
-        }
-
-        return 0;
-}
diff --git a/mod_glusterfs/lighttpd/1.4/mod_glusterfs.h b/mod_glusterfs/lighttpd/1.4/mod_glusterfs.h
deleted file mode 100644
index 9d73d6999..000000000
--- a/mod_glusterfs/lighttpd/1.4/mod_glusterfs.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _MOD_GLUSTERFS_FILE_CACHE_H_
-#define _MOD_GLUSTERFS_FILE_CACHE_H_
-
-#include "stat_cache.h"
-#include <libglusterfsclient.h>
-#include "base.h"
-
-handler_t glusterfs_stat_cache_get_entry(server *srv, connection *con,
-                                         buffer *glusterfs_path, buffer *name,
-                                         void *buf, size_t size,
-                                         stat_cache_entry **fce);
-
-#endif
diff --git a/mod_glusterfs/lighttpd/1.5/Makefile.am b/mod_glusterfs/lighttpd/1.5/Makefile.am
deleted file mode 100644
index eda329111..000000000
--- a/mod_glusterfs/lighttpd/1.5/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-EXTRA_DIST = Makefile.am.diff mod_glusterfs.c mod_glusterfs.h README.txt
-
-CLEANFILES = 
diff --git a/mod_glusterfs/lighttpd/1.5/Makefile.am.diff b/mod_glusterfs/lighttpd/1.5/Makefile.am.diff
deleted file mode 100644
index 375696b5d..000000000
--- a/mod_glusterfs/lighttpd/1.5/Makefile.am.diff
+++ /dev/null
@@ -1,29 +0,0 @@
---- lighttpd-1.4.19/src/Makefile.am	2008-04-16 18:42:18.000000000 +0400
-+++ lighttpd-1.4.19.mod/src/Makefile.am	2008-04-16 18:41:11.000000000 +0400
-@@ -1,4 +1,4 @@
--AM_CFLAGS = $(FAM_CFLAGS)
-+AM_CFLAGS = $(FAM_CFLAGS)  -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64
- 
- noinst_PROGRAMS=proc_open lemon # simple-fcgi #graphic evalo bench ajp ssl error_test adserver gen-license
- sbin_PROGRAMS=lighttpd lighttpd-angel
-@@ -241,6 +241,11 @@
- mod_accesslog_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined
- mod_accesslog_la_LIBADD = $(common_libadd)
- 
-+lib_LTLIBRARIES += mod_glusterfs.la
-+mod_glusterfs_la_SOURCES = mod_glusterfs.c
-+mod_glusterfs_la_CFLAGS = $(AM_CFLAGS) 
-+mod_glusterfs_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined -lglusterfsclient -lpthread
-+mod_glusterfs_la_LIBADD = $(common_libadd) 
- 
- hdr = server.h buffer.h network.h log.h keyvalue.h \
-       response.h request.h fastcgi.h chunk.h \
-@@ -254,7 +259,7 @@
-       configparser.h mod_ssi_exprparser.h \
-       sys-mmap.h sys-socket.h mod_cml.h mod_cml_funcs.h \
-       splaytree.h proc_open.h status_counter.h \
--      mod_magnet_cache.h
-+      mod_magnet_cache.h mod_glusterfs.h
- 
- DEFS= @DEFS@ -DLIBRARY_DIR="\"$(libdir)\"" -DSBIN_DIR="\"$(sbindir)\""
- 
diff --git a/mod_glusterfs/lighttpd/1.5/README.txt b/mod_glusterfs/lighttpd/1.5/README.txt
deleted file mode 100644
index bdbdfffbc..000000000
--- a/mod_glusterfs/lighttpd/1.5/README.txt
+++ /dev/null
@@ -1,57 +0,0 @@
-Introduction
-============
-mod_glusterfs is a module written for lighttpd to speed up the access of files present on glusterfs. mod_glusterfs uses libglusterfsclient library provided for glusterfs and hence can be used without fuse (File System in User Space).
-
-Usage
-=====
-To use mod_glusterfs with lighttpd-1.5, copy mod_glusterfs.c and mod_glusterfs.h into src/ of lighttpd-1.5 source tree, and apply the Makefile.am.diff to src/Makefile.am. Re-run ./autogen.sh on the top level of the lighttpd-1.5 build tree and recompile.
-
-# cp mod_glusterfs.[ch] /home/glusterfs/lighttpd-1.5/src/
-# cp Makefile.am.diff /home/glusterfs/lighttpd-1.5/
-# cd /home/glusterfs/lighttpd-1.5
-# patch -p1 < Makefile.am.diff 
-# ./autogen.sh
-# ./configure
-# make
-# make install
-
-Configuration
-=============
-* mod_glusterfs should be listed at the begining of the list server.modules in lighttpd.conf. 
-
-Below is a snippet from lighttpd.conf concerning to mod_glusterfs.
-
-$HTTP["url"] =~ "^/glusterfs" {
-	glusterfs.prefix = "/glusterfs" 
-	glusterfs.logfile = "/var/log/glusterfs-logfile"
-	glusterfs.document-root = "/home/glusterfs/document-root"
-	glusterfs.volume-specfile = "/etc/glusterfs/glusterfs.vol"
-	glusterfs.loglevel = "error"
-	glusterfs.cache-timeout = 300
-	glusterfs.xattr-interface-size-limit = "65536"
-}
-
-* $HTTP["url"] =~ "^/glusterfs"
-  A perl style regular expression used to match against the url. If regular expression matches the url, the url is handled by mod_glusterfs. Note that the pattern given here should match glusterfs.prefix.
-
-* glusterfs.prefix (COMPULSORY)
-  A string to be present at the starting of the file path in the url so that the file would be handled by glusterfs.
-  Eg., A GET request on the url http://www.example.com/glusterfs-prefix/some-dir/example-file will result in fetching of the file "/some-dir/example-file" from glusterfs mount if glusterfs.prefix is set to "/glusterfs-prefix".
-
-* glusterfs.volume-specfile (COMPULSORY)
-  Path to the the glusterfs volume specification file.
-
-* glusterfs.logfile (COMPULSORY)
-  Path to the glusterfs logfile.
-
-* glusterfs.loglevel (OPTIONAL, default = warning)
-  Allowed values are critical, error, warning, debug, none in the decreasing order of severity of error conditions.
-
-* glusterfs.cache-timeout (OPTIONAL, default = 0)
-  Timeout values for glusterfs stat and lookup cache.
-
-* glusterfs.document-root (COMPULSORY)
-  An absolute path, relative to which all the files are fetched from glusterfs.
-
-* glusterfs.xattr-interface-size-limit (OPTIONAL, default = 0)
-  Files with sizes upto and including this value are fetched through the extended attribute interface of glusterfs rather than the usual open-read-close set of operations. For files of small sizes, it is recommended to use extended attribute interface.
diff --git a/mod_glusterfs/lighttpd/1.5/mod_glusterfs.c b/mod_glusterfs/lighttpd/1.5/mod_glusterfs.c
deleted file mode 100644
index 67f7a7eac..000000000
--- a/mod_glusterfs/lighttpd/1.5/mod_glusterfs.c
+++ /dev/null
@@ -1,1476 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <ctype.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <pthread.h>
-#include <sys/types.h>
-#include <fcntl.h>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-
-#include <errno.h>
-#include <unistd.h>
-#include <assert.h>
-
-#include "base.h"
-#include "log.h"
-#include "buffer.h"
-
-#include "plugin.h"
-
-#include "stat_cache.h"
-#include "mod_glusterfs.h"
-#include "etag.h"
-#include "response.h"
-
-#include "fdevent.h"
-#include "joblist.h"
-#include "http_req_range.h"
-#include "connections.h"
-#include "configfile.h"
-
-#include <libglusterfsclient.h>
-
-#ifdef HAVE_ATTR_ATTRIBUTES_H
-#include <attr/attributes.h>
-#endif
-
-#ifdef HAVE_FAM_H
-# include <fam.h>
-#endif
-
-#include "sys-mmap.h"
-
-/* NetBSD 1.3.x needs it */
-#ifndef MAP_FAILED
-# define MAP_FAILED -1
-#endif
-
-#ifndef O_LARGEFILE
-# define O_LARGEFILE 0
-#endif
-
-#ifndef HAVE_LSTAT
-#define lstat stat
-#endif
-
-#if 0
-/* enables debug code for testing if all nodes in the stat-cache as accessable */
-#define DEBUG_STAT_CACHE
-#endif
-
-#ifdef HAVE_LSTAT
-#undef HAVE_LSTAT
-#endif
-
-#define GLUSTERFS_FILE_CHUNK (FILE_CHUNK + 1)
-
-/* Keep this value large. Each glusterfs_async_read of GLUSTERFS_CHUNK_SIZE results in a network_backend_write of the read data*/
-
-#define GLUSTERFS_CHUNK_SIZE 8192
-
-/**
- * this is a staticfile for a lighttpd plugin
- *
- */
-
-
-/* plugin config for all request/connections */
-
-typedef struct {
-        buffer *logfile;
-        buffer *loglevel;
-        buffer *specfile;
-        buffer *prefix;
-        buffer *xattr_file_size;
-        buffer *document_root;
-        array *exclude_exts;
-        unsigned short cache_timeout;
-
-        /* FIXME: its a pointer, hence cant be short */
-        unsigned long handle;
-} plugin_config;
-
-static network_status_t (*network_backend_write)(struct server *srv, connection *con, iosocket *sock, chunkqueue *cq);
-
-typedef struct {
-        PLUGIN_DATA;
-        buffer *range_buf;
-        plugin_config **config_storage;
-        http_req_range *ranges;
-        plugin_config conf;
-} plugin_data;
-
-typedef struct glusterfs_async_local {
-        int op_ret;
-        int op_errno;
-        pthread_mutex_t lock;
-        pthread_cond_t cond;
-        connection *con;
-        server *srv;
-        plugin_data *p;
-
-        union {
-                struct {
-                        char async_read_complete;
-                        off_t length;
-                        size_t read_bytes;
-                        glusterfs_read_buf_t *buf;
-                }readv;
-
-                struct {
-                        buffer *name;
-                        buffer *hash_key;
-                        size_t size;
-                }lookup;
-        }fop;
-} glusterfs_async_local_t;
-
-typedef struct {
-        unsigned long fd;
-        buffer *glusterfs_path;
-        void *buf;
-        off_t response_content_length;
-        int prefix;
-}mod_glusterfs_ctx_t;
-
-typedef struct {
-        chunkqueue *cq;
-        glusterfs_read_buf_t *buf;
-        size_t length;
-}mod_glusterfs_chunkqueue;
-
-#ifdef HAVE_FAM_H
-typedef struct {
-        FAMRequest *req;
-        FAMConnection *fc;
-
-        buffer *name;
-
-        int version;
-} fam_dir_entry;
-#endif
-
-/* the directory name is too long to always compare on it
- * - we need a hash
- * - the hash-key is used as sorting criteria for a tree
- * - a splay-tree is used as we can use the caching effect of it
- */
-
-/* we want to cleanup the stat-cache every few seconds, let's say 10
- *
- * - remove entries which are outdated since 30s
- * - remove entries which are fresh but havn't been used since 60s
- * - if we don't have a stat-cache entry for a directory, release it from the monitor
- */
-
-#ifdef DEBUG_STAT_CACHE
-typedef struct {
-        int *ptr;
-
-        size_t used;
-        size_t size;
-} fake_keys;
-
-static fake_keys ctrl;
-#endif
-
-static stat_cache_entry * 
-stat_cache_entry_init(void) 
-{
-        stat_cache_entry *sce = NULL;
-
-        sce = calloc(1, sizeof(*sce));
-        /* ERR_ABORT (sce); */
-
-        sce->name = buffer_init();
-        sce->etag = buffer_init();
-        sce->content_type = buffer_init();
-
-        return sce;
-}
-
-int chunkqueue_append_glusterfs_mem (chunkqueue *cq, const char * mem, size_t len) {
-        buffer *buf = NULL;
- 
-        buf = chunkqueue_get_append_buffer (cq);
-
-        if (buf->ptr)
-                free (buf->ptr);
-
-        buf->used = len + 1;
-        buf->ptr = (char *)mem;
-        buf->size = len;
-
-        return 0;
-}
-
-static int
-glusterfs_lookup_async_cbk (int op_ret, 
-                            int op_errno,
-                            void *buf,
-                            struct stat *st,
-                            void *cbk_data)
-{
-        glusterfs_async_local_t *local = cbk_data;
-
-        mod_glusterfs_ctx_t *ctx = NULL;
-        ctx = local->con->plugin_ctx[local->p->id];
-
-        assert (ctx->buf== buf);
-
-        if (op_ret || !(S_ISREG (st->st_mode) && (size_t)st->st_size <= local->fop.lookup.size)) {
-
-                free (ctx->buf);
-                ctx->buf = NULL;
-
-                if (op_ret) {
-                        buffer_free (ctx->glusterfs_path);
-                        ctx->glusterfs_path = NULL;
-                        free (ctx);
-                        local->con->plugin_ctx[local->p->id] = NULL;
-
-                        if (op_errno == ENOENT)
-                                local->con->http_status = 404;
-                        else 
-                                local->con->http_status = 403;
-                }
-        }
-
-        if (!op_ret) {
-                stat_cache_entry *sce = NULL;
-                stat_cache *sc = local->srv->stat_cache;
-
-                sce = (stat_cache_entry *)g_hash_table_lookup(sc->files, local->fop.lookup.hash_key);
-
-                if (!sce) {
-                        sce = stat_cache_entry_init();
-
-                        buffer_copy_string_buffer(sce->name, local->fop.lookup.name);
-                        g_hash_table_insert(sc->files, buffer_init_string(BUF_STR(local->fop.lookup.hash_key)), sce);
-                }
-
-                sce->state = STAT_CACHE_ENTRY_STAT_FINISHED;
-                sce->stat_ts = time (NULL);
-                memcpy (&sce->st, st, sizeof (*st));
-        }
-
-        g_async_queue_push (local->srv->joblist_queue, local->con);
-        /*
-          joblist_append (local->srv, local->con);
-          kill (getpid(), SIGUSR1);
-        */
-        free (local);
-        return 0;
-}
-
-static handler_t
-glusterfs_stat_cache_get_entry_async (server *srv, 
-                                      connection *con, 
-                                      plugin_data *p,
-                                      buffer *glusterfs_path,
-                                      buffer *name,
-                                      void *buf,
-                                      size_t size,
-                                      stat_cache_entry **ret_sce)
-{
-        stat_cache_entry *sce = NULL;
-        stat_cache *sc;
-        glusterfs_async_local_t *local = NULL;
-
-        *ret_sce = NULL;
-
-        /*
-         * check if the directory for this file has changed
-         */
-
-        sc = srv->stat_cache;
-
-        buffer_copy_string_buffer(sc->hash_key, name);
-        buffer_append_long(sc->hash_key, con->conf.follow_symlink);
-
-        if ((sce = (stat_cache_entry *)g_hash_table_lookup(sc->files, sc->hash_key))) {
-                /* know this entry already */
-
-                if (sce->state == STAT_CACHE_ENTRY_STAT_FINISHED && 
-                    !buf) {
-                        /* verify that this entry is still fresh */
-
-                        *ret_sce = sce;
-
-                        return HANDLER_GO_ON;
-                }
-        }
-
-
-        /*
-         * *lol*
-         * - open() + fstat() on a named-pipe results in a (intended) hang.
-         * - stat() if regular file + open() to see if we can read from it is better
-         *
-         * */
-
-        /* pass a job to the stat-queue */
-
-        local = calloc (1, sizeof (*local));
-        /* ERR_ABORT (local); */
-        local->con = con;
-        local->srv = srv;
-        local->p = p;
-        local->fop.lookup.name = buffer_init_buffer (name);
-        local->fop.lookup.hash_key = buffer_init_buffer (sc->hash_key);
-        local->fop.lookup.size = size;
-
-        if (glusterfs_lookup_async ((libglusterfs_handle_t )p->conf.handle, glusterfs_path->ptr, buf, size, glusterfs_lookup_async_cbk, (void *) local)) {
-                free (local);
-                return HANDLER_ERROR;
-        }
-
-        return HANDLER_WAIT_FOR_EVENT;
-}
-
-int 
-mod_glusterfs_readv_async_cbk (glusterfs_read_buf_t *buf,
-                               void *cbk_data)
-{
-        glusterfs_async_local_t *local = cbk_data;
-        pthread_mutex_lock (&local->lock);
-        {
-                local->fop.readv.async_read_complete = 1;
-                local->fop.readv.buf = buf;
-
-                pthread_cond_signal (&local->cond);
-        }
-        pthread_mutex_unlock (&local->lock);
-
-        return 0;
-}
-
-network_status_t 
-mod_glusterfs_read_async (server *srv, connection *con, chunk *glusterfs_chunk)
-{
-        glusterfs_async_local_t local;
-        off_t end = 0;
-        int nbytes;
-        int complete;
-        chunkqueue *cq = NULL;
-        chunk *c = NULL;
-        off_t offset = glusterfs_chunk->file.start;
-        size_t length = glusterfs_chunk->file.length;
-        unsigned long fd = (unsigned long)glusterfs_chunk->file.name;
-        network_status_t ret;
-
-        pthread_cond_init (&local.cond, NULL);
-        pthread_mutex_init (&local.lock, NULL);
-  
-        //local.fd = fd;
-        memset (&local, 0, sizeof (local));
-
-        if (length > 0)
-                end = offset + length;
-
-        cq = chunkqueue_init ();
-        if (!cq) {
-                con->http_status = 500;
-                return NETWORK_STATUS_FATAL_ERROR;
-        }
-
-        do {
-                glusterfs_read_buf_t *buf;
-                int i;
-                if (length > 0) {
-                        nbytes = end - offset;
-                        if (nbytes > GLUSTERFS_CHUNK_SIZE)
-                                nbytes = GLUSTERFS_CHUNK_SIZE;
-                } else
-                        nbytes = GLUSTERFS_CHUNK_SIZE;
-
-                glusterfs_read_async(fd, 
-                                     nbytes,
-                                     offset,
-                                     mod_glusterfs_readv_async_cbk,
-                                     (void *)&local);
-
-                pthread_mutex_lock (&local.lock);
-                {
-                        while (!local.fop.readv.async_read_complete) {
-                                pthread_cond_wait (&local.cond, &local.lock);
-                        }
-
-                        local.op_ret = local.fop.readv.buf->op_ret;
-                        local.op_errno = local.fop.readv.buf->op_errno;
-
-                        local.fop.readv.async_read_complete = 0;
-                        buf = local.fop.readv.buf;
-
-                        if ((int)length < 0)
-                                complete = (local.fop.readv.buf->op_ret <= 0);
-                        else {
-                                local.fop.readv.read_bytes += local.fop.readv.buf->op_ret;
-                                complete = ((local.fop.readv.read_bytes == length) || (local.fop.readv.buf->op_ret <= 0));
-                        }
-                }
-                pthread_mutex_unlock (&local.lock);
-
-                if (local.op_ret > 0) {
-                        for (i = 0; i < buf->count; i++) {
-                                buffer *nw_write_buf = chunkqueue_get_append_buffer (cq);
-
-                                nw_write_buf->used = nw_write_buf->size = buf->vector[i].iov_len + 1;
-                                nw_write_buf->ptr = buf->vector[i].iov_base;
-
-                                //      buffer_copy_memory (nw_write_buf, buf->vector[i].iov_base, buf->vector[i].iov_len + 1);
-                                offset += local.op_ret;
-                        }
-  
-                        ret = network_backend_write (srv, con, con->sock, cq);
-  
-                        if (chunkqueue_written (cq) != local.op_ret) {
-                                mod_glusterfs_chunkqueue *gf_cq;
-                                glusterfs_chunk->file.start = offset;
-                                if ((int)glusterfs_chunk->file.length > 0)
-                                        glusterfs_chunk->file.length -= local.fop.readv.read_bytes;
-
-                                gf_cq = calloc (1, sizeof (*gf_cq));
-                                /* ERR_ABORT (qf_cq); */
-                                gf_cq->cq = cq;
-                                gf_cq->buf = buf;
-                                gf_cq->length = local.op_ret;
-                                glusterfs_chunk->file.mmap.start = (char *)gf_cq;
-                                return ret;
-                        }
-      
-                        for (c = cq->first ; c; c = c->next) 
-                                c->mem->ptr = NULL;
-
-                        chunkqueue_reset (cq);
-                }
-
-                glusterfs_free (buf);
-        } while (!complete);
-
-        chunkqueue_free (cq);
-        glusterfs_close (fd);
-
-        if (local.op_ret < 0)
-                con->http_status = 500;
-
-        return (local.op_ret < 0 ? NETWORK_STATUS_FATAL_ERROR : NETWORK_STATUS_SUCCESS);
-}
-
-network_status_t mod_glusterfs_network_backend_write(struct server *srv, connection *con, iosocket *sock, chunkqueue *cq)
-{
-        chunk *c, *prev, *first;
-        int chunks_written = 0;
-        int error = 0;
-        network_status_t ret;
-
-        for (first = prev = c = cq->first; c; c = c->next, chunks_written++) {
-
-                if (c->type == MEM_CHUNK && c->mem->used && !c->mem->ptr) {
-                        if (cq->first != c) {
-                                prev->next = NULL;
-
-                                /* call stored network_backend_write */
-                                ret = network_backend_write (srv, con, sock, cq);
-
-                                prev->next = c;
-                                if (ret != NETWORK_STATUS_SUCCESS) {
-                                        cq->first = first;
-                                        return ret;
-                                }
-                        } 
-                        cq->first = c->next;
-
-                        if (c->file.fd < 0) {
-                                error = HANDLER_ERROR;
-                                break;
-                        }
-
-                        if (c->file.mmap.start) {
-                                chunk *tmp;
-                                size_t len;
-                                mod_glusterfs_chunkqueue *gf_cq = (mod_glusterfs_chunkqueue *)c->file.mmap.start;
-
-                                ret = network_backend_write (srv, con, sock, gf_cq->cq);
-
-                                if ((len = (size_t)chunkqueue_written (gf_cq->cq)) != gf_cq->length) {
-                                        gf_cq->length -= len;
-                                        cq->first = first;
-                                        chunkqueue_remove_finished_chunks (gf_cq->cq);
-                                        return ret;
-                                }
-
-                                for (tmp = gf_cq->cq->first ; tmp; tmp = tmp->next)
-                                        tmp->mem->ptr = NULL;
-
-                                chunkqueue_free (gf_cq->cq);
-                                glusterfs_free (gf_cq->buf);
-                                free (gf_cq);
-                                c->file.mmap.start = NULL;
-                        }
-      
-                        ret = mod_glusterfs_read_async (srv, con, c); //c->file.fd, c->file.start, -1);//c->file.length);
-                        if (c->file.mmap.start) {
-                                /* pending chunkqueue from mod_glusterfs_read_async to be written to network */
-                                cq->first = first;
-                                return ret;
-                        }
-
-                        buffer_free (c->mem);
-                        c->mem = NULL;
-
-                        c->type = FILE_CHUNK;
-                        c->offset = c->file.length = 0;
-                        c->file.name = NULL;
-
-                        if (first == c)
-                                first = c->next;
-
-                        if (cq->last == c)
-                                cq->last = NULL;
-
-                        prev->next = c->next;
-
-                        free(c);
-                }     
-                prev = c;
-        }
-
-        ret = network_backend_write (srv, con, sock, cq);
-
-        cq->first = first;
-
-        return ret;
-}
-
-#if 0
-int chunkqueue_append_glusterfs_file (chunkqueue *cq, unsigned long fd, off_t offset, off_t len)
-{
-        chunk *c = NULL;
-        c = chunkqueue_get_append_tempfile (cq);
-  
-        if (c->file.is_temp) {
-                close (c->file.fd);
-                unlink (c->file.name->ptr);
-        }
-
-        c->type = MEM_CHUNK;
-
-        c->mem = buffer_init ();
-        c->mem->used = len + 1;
-        c->mem->ptr = NULL;
-        c->offset = 0;
-
-        /*  buffer_copy_string_buffer (c->file.name, fn); */
-        c->file.start = offset;
-        c->file.length = len;
-        /*  buffer_free (c->file.name); */
-
-        /* identify chunk as glusterfs related */
-        c->file.mmap.start = MAP_FAILED;
-        /*  c->file.mmap.length = c->file.mmap.offset = len;*/
-
-        return 0;
-}
-#endif
-
-int chunkqueue_append_dummy_mem_chunk (chunkqueue *cq, off_t len)
-{
-        chunk *c = NULL;
-        c = chunkqueue_get_append_tempfile (cq);
-  
-        if (c->file.is_temp) {
-                close (c->file.fd);
-                unlink (c->file.name->ptr);
-                c->file.is_temp = 0;
-        }
-
-        c->type = MEM_CHUNK;
-
-        c->mem->used = len + 1;
-        c->offset = len;
-        c->mem->ptr = NULL;
-
-        return 0;
-}
-
-int chunkqueue_append_glusterfs_file (chunkqueue *cq, unsigned long fd, off_t offset, off_t len)
-{
-        chunk *c = NULL;
-        c = chunkqueue_get_append_tempfile (cq);
-  
-        if (c->file.is_temp) {
-                close (c->file.fd);
-                unlink (c->file.name->ptr);
-                c->file.is_temp = 0;
-        }
-
-        c->type = MEM_CHUNK;
-
-        c->mem = buffer_init ();
-        c->mem->used = len + 1;
-        c->mem->ptr = NULL;
-        c->offset = 0;
-
-        /*  buffer_copy_string_buffer (c->file.name, fn); */
-        buffer_free (c->file.name);
-
-        /* fd returned by libglusterfsclient is a pointer */
-        c->file.name = (buffer *)fd;
-        c->file.start = offset;
-        c->file.length = len;
-
-        //c->file.fd = fd;
-        c->file.mmap.start = NULL;
-        return 0;
-}
-
-/* init the plugin data */
-INIT_FUNC(mod_glusterfs_init) {
-        plugin_data *p;
-
-        UNUSED (srv);
-        p = calloc(1, sizeof(*p));
-        /* ERR_ABORT (p); */
-        network_backend_write = NULL;
-        p->ranges = http_request_range_init();
-        return p;
-}
-
-/* detroy the plugin data */
-FREE_FUNC(mod_glusterfs_free) {
-        plugin_data *p = p_d;
-
-        UNUSED (srv);
-
-        if (!p) return HANDLER_GO_ON;
-  
-        if (p->config_storage) {
-                size_t i;
-                for (i = 0; i < srv->config_context->used; i++) {
-                        plugin_config *s = p->config_storage[i];
-      
-                        buffer_free (s->logfile);
-                        buffer_free (s->loglevel);
-                        buffer_free (s->specfile);
-                        buffer_free (s->prefix);
-                        buffer_free (s->xattr_file_size);
-                        buffer_free (s->document_root);
-                        array_free (s->exclude_exts);
-  
-                        free (s);
-                }
-                free (p->config_storage);
-        }
-        buffer_free (p->range_buf);
-        http_request_range_free (p->ranges);
-
-        free (p);
-  
-        return HANDLER_GO_ON;
-}
-
-SETDEFAULTS_FUNC(mod_glusterfs_set_defaults) {
-        plugin_data *p = p_d;
-        size_t i = 0;
-  
-        config_values_t cv[] = {
-                { "glusterfs.logfile",              NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },
-    
-                { "glusterfs.loglevel",             NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },         
-                { "glusterfs.volume-specfile",             NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },          
-                { "glusterfs.cache-timeout",              NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION },
-    
-                { "glusterfs.exclude-extensions",   NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION },
-    
-                /*TODO: get the prefix from config_conext and remove glusterfs.prefix from conf file */
-                { "glusterfs.prefix", NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },
-    
-                { "glusterfs.xattr-interface-size-limit", NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },
-
-                { "glusterfs.document-root", NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },
-    
-                { NULL,                          NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET }
-        };
-  
-        p->config_storage = calloc(1, srv->config_context->used * sizeof(specific_config *));
-        /* ERR_ABORT (p->config_storage); */
-        p->range_buf = buffer_init ();
-  
-        for (i = 0; i < srv->config_context->used; i++) {
-                plugin_config *s;
-
-                s = calloc(1, sizeof(plugin_config));
-                /* ERR_ABORT (s); */
-                s->logfile = buffer_init ();
-                s->loglevel = buffer_init ();
-                s->specfile = buffer_init ();
-                s->exclude_exts = array_init ();
-                s->prefix = buffer_init ();
-                s->xattr_file_size = buffer_init ();
-                s->document_root = buffer_init ();
-    
-                cv[0].destination = s->logfile;
-                cv[1].destination = s->loglevel;
-                cv[2].destination = s->specfile;
-                cv[3].destination = &s->cache_timeout;
-                cv[4].destination = s->exclude_exts;
-                cv[5].destination = s->prefix;
-                cv[6].destination = s->xattr_file_size;
-                cv[7].destination = s->document_root;
-
-                p->config_storage[i] = s;
-    
-                if (0 != config_insert_values_global(srv, ((data_config *)srv->config_context->data[i])->value, cv)) {
-                        return HANDLER_FINISHED;
-                }
-        }
-  
-        return HANDLER_GO_ON;
-}
-
-#define PATCH(x)                                \
-        p->conf.x = s->x;
-
-static int mod_glusterfs_patch_connection(server *srv, connection *con, plugin_data *p) {
-        size_t i, j;
-        plugin_config *s;
-
-        p->conf.logfile = NULL;
-        p->conf.loglevel = NULL;
-        p->conf.specfile = NULL;
-        p->conf.cache_timeout = 0;
-        p->conf.exclude_exts = NULL;
-        p->conf.prefix = NULL;
-        p->conf.xattr_file_size = NULL;
-        p->conf.exclude_exts = NULL;
-
-        /* skip the first, the global context */
-        /* glusterfs related config can only occur inside $HTTP["url"] == "<glusterfs-prefix>" */
-        for (i = 1; i < srv->config_context->used; i++) {
-                data_config *dc = (data_config *)srv->config_context->data[i];
-                s = p->config_storage[i];
-
-                /* condition didn't match */
-                if (!config_check_cond(srv, con, dc)) continue;
-    
-                /* merge config */
-                for (j = 0; j < dc->value->used; j++) {
-                        data_unset *du = dc->value->data[j];
-      
-                        if (buffer_is_equal_string (du->key, CONST_STR_LEN("glusterfs.logfile"))) {
-                                PATCH (logfile);
-                        } else if (buffer_is_equal_string (du->key, CONST_STR_LEN("glusterfs.loglevel"))) {
-                                PATCH (loglevel);
-                        } else if (buffer_is_equal_string (du->key, CONST_STR_LEN ("glusterfs.volume-specfile"))) {
-                                PATCH (specfile);
-                        } else if (buffer_is_equal_string (du->key, CONST_STR_LEN("glusterfs.cache-timeout"))) {
-                                PATCH (cache_timeout);
-                        } else if (buffer_is_equal_string (du->key, CONST_STR_LEN ("glusterfs.exclude-extensions"))) {
-                                PATCH (exclude_exts);
-                        } else if (buffer_is_equal_string (du->key, CONST_STR_LEN ("glusterfs.prefix"))) {
-                                PATCH (prefix);
-                        } else if (buffer_is_equal_string (du->key, CONST_STR_LEN ("glusterfs.xattr-interface-size-limit"))) {
-                                PATCH (xattr_file_size);
-                        } else if (buffer_is_equal_string (du->key, CONST_STR_LEN ("glusterfs.document-root"))) {
-                                PATCH (document_root);
-                        }
-                }
-        }
-        return 0;
-}
-
-#undef PATCH
-
-static int http_response_parse_range(server *srv, connection *con, plugin_data *p) {
-        int multipart = 0;
-        char *boundary = "fkj49sn38dcn3";
-        data_string *ds;
-        stat_cache_entry *sce = NULL;
-        buffer *content_type = NULL;
-        buffer *range = NULL;
-        http_req_range *ranges, *r;
-        mod_glusterfs_ctx_t *ctx = con->plugin_ctx[p->id];
-        size_t size = 0;
-
-        if (!ctx) {
-                return -1;
-        }
-
-        if (NULL != (ds = (data_string *)array_get_element(con->request.headers, CONST_STR_LEN("Range")))) {
-                range = ds->value;
-        } else {
-                /* we don't have a Range header */
-
-                return -1;
-        }
-
-        if (HANDLER_ERROR == stat_cache_get_entry(srv, con, con->physical.path, &sce)) {
-                SEGFAULT();
-        }
-
-        ctx->response_content_length = con->response.content_length = 0;
-
-        if (NULL != (ds = (data_string *)array_get_element(con->response.headers, CONST_STR_LEN("Content-Type")))) {
-                content_type = ds->value;
-        }
-
-        /* start the range-header parser
-         * bytes=<num>  */
-
-        ranges = p->ranges;
-        http_request_range_reset(ranges);
-        switch (http_request_range_parse(range, ranges)) {
-        case PARSE_ERROR:
-                return -1; /* no range valid Range Header */
-        case PARSE_SUCCESS:
-                break;
-        default:
-                TRACE("%s", "foobar");
-                return -1;
-        }
-
-        if (ranges->next) {
-                multipart = 1;
-        }
-
-        if (p->conf.xattr_file_size && p->conf.xattr_file_size->ptr) {
-                size = atoi (p->conf.xattr_file_size->ptr);
-        }
-
-        /* patch the '-1' */
-        for (r = ranges; r; r = r->next) {
-                if (r->start == -1) {
-                        /* -<end>
-                         *
-                         * the last <end> bytes  */
-                        r->start = sce->st.st_size - r->end;
-                        r->end = sce->st.st_size - 1;
-                }
-                if (r->end == -1) {
-                        /* <start>-
-                         * all but the first <start> bytes */
-
-                        r->end = sce->st.st_size - 1;
-                }
-
-                if (r->end > sce->st.st_size - 1) {
-                        /* RFC 2616 - 14.35.1
-                         *
-                         * if last-byte-pos not present or > size-of-file
-                         * take the size-of-file
-                         *
-                         *  */
-                        r->end = sce->st.st_size - 1;
-                }
-
-                if (r->start > sce->st.st_size - 1) {
-                        /* RFC 2616 - 14.35.1
-                         *
-                         * if first-byte-pos > file-size, 416
-                         */
-
-                        con->http_status = 416;
-                        return -1;
-                }
-
-                if (r->start > r->end) {
-                        /* RFC 2616 - 14.35.1
-                         *
-                         * if last-byte-pos is present, it has to be >= first-byte-pos
-                         *
-                         * invalid ranges have to be handle as no Range specified
-                         *  */
-
-                        return -1;
-                }
-        }
-
-        if (r) {
-                /* we ran into an range violation */
-                return -1;
-        }
-
-        if (multipart) {
-                buffer *b;
-                for (r = ranges; r; r = r->next) {
-                        /* write boundary-header */
-
-                        b = chunkqueue_get_append_buffer(con->send);
-
-                        buffer_copy_string(b, "\r\n--");
-                        buffer_append_string(b, boundary);
-
-                        /* write Content-Range */
-                        buffer_append_string(b, "\r\nContent-Range: bytes ");
-                        buffer_append_off_t(b, r->start);
-                        buffer_append_string(b, "-");
-                        buffer_append_off_t(b, r->end);
-                        buffer_append_string(b, "/");
-                        buffer_append_off_t(b, sce->st.st_size);
-
-                        buffer_append_string(b, "\r\nContent-Type: ");
-                        buffer_append_string_buffer(b, content_type);
-
-                        /* write END-OF-HEADER */
-                        buffer_append_string(b, "\r\n\r\n");
-
-                        con->response.content_length += b->used - 1;
-                        ctx->response_content_length += b->used - 1;
-                        con->send->bytes_in += b->used - 1;
-
-                        if ((size_t)sce->st.st_size > size) {
-                                chunkqueue_append_glusterfs_file(con->send_raw, ctx->fd, r->start, r->end - r->start + 1);
-                                con->send_raw->bytes_in += (r->end - r->start + 1);
-                                chunkqueue_append_dummy_mem_chunk (con->send, r->end - r->start + 1);
-                        } else {
-                                chunkqueue_append_mem (con->send, ((char *)ctx->buf) + r->start, r->end - r->start + 1); 
-                                free (ctx->buf);
-                                ctx->buf = NULL;
-                        }
-                
-                        con->response.content_length += r->end - r->start + 1;
-                        ctx->response_content_length += r->end - r->start + 1;
-                        con->send->bytes_in += r->end - r->start + 1;
-                }
-
-                /* add boundary end */
-                b = chunkqueue_get_append_buffer(con->send);
-
-                buffer_copy_string_len(b, "\r\n--", 4);
-                buffer_append_string(b, boundary);
-                buffer_append_string_len(b, "--\r\n", 4);
-
-                con->response.content_length += b->used - 1;
-                ctx->response_content_length += b->used - 1;
-                con->send->bytes_in += b->used - 1;
-
-                /* set header-fields */
-
-                buffer_copy_string(p->range_buf, "multipart/byteranges; boundary=");
-                buffer_append_string(p->range_buf, boundary);
-
-                /* overwrite content-type */
-                response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(p->range_buf));
-
-        } else {
-                r = ranges;
-
-                chunkqueue_append_glusterfs_file(con->send_raw, ctx->fd, r->start, r->end - r->start + 1);
-                con->send_raw->bytes_in += (r->end - r->start + 1);
-                chunkqueue_append_dummy_mem_chunk (con->send, r->end - r->start + 1);
-                con->response.content_length += r->end - r->start + 1;
-                ctx->response_content_length += r->end - r->start + 1;
-                con->send->bytes_in += r->end - r->start + 1;
-
-                buffer_copy_string(p->range_buf, "bytes ");
-                buffer_append_off_t(p->range_buf, r->start);
-                buffer_append_string(p->range_buf, "-");
-                buffer_append_off_t(p->range_buf, r->end);
-                buffer_append_string(p->range_buf, "/");
-                buffer_append_off_t(p->range_buf, sce->st.st_size);
-
-                response_header_insert(srv, con, CONST_STR_LEN("Content-Range"), CONST_BUF_LEN(p->range_buf));
-        }
-
-        /* ok, the file is set-up */
-        return 0;
-}
-
-PHYSICALPATH_FUNC(mod_glusterfs_handle_physical) {
-        plugin_data *p = p_d;
-        stat_cache_entry *sce;
-        size_t size = 0;
-        handler_t ret = 0;
-        mod_glusterfs_ctx_t *plugin_ctx = NULL;
-
-        if (con->http_status != 0) return HANDLER_GO_ON;
-        if (con->uri.path->used == 0) return HANDLER_GO_ON;
-        if (con->physical.path->used == 0) return HANDLER_GO_ON;
-
-        if (con->mode != DIRECT) return HANDLER_GO_ON;
-
-        /*
-          network_backend_write = srv->network_backend_write;
-          srv->network_backend_write = mod_glusterfs_network_backend_write;
-        */
-
-        switch (con->request.http_method) {
-        case HTTP_METHOD_GET:
-        case HTTP_METHOD_POST:
-        case HTTP_METHOD_HEAD:
-                break;
-
-        default:
-                return HANDLER_GO_ON;
-        }
-
-        mod_glusterfs_patch_connection(srv, con, p);
-
-        if (!p->conf.prefix || !p->conf.prefix->ptr) {
-                return HANDLER_GO_ON;
-        }
-
-        if (!p->conf.document_root || p->conf.document_root->used == 0) {
-                log_error_write(srv, __FILE__, __LINE__, "s", "glusterfs.document-root is not specified");
-                con->http_status = 500;
-                return HANDLER_FINISHED;
-        }
-
-        if (p->conf.handle <= 0) {
-                glusterfs_init_ctx_t ctx;
-
-                if (!p->conf.specfile || p->conf.specfile->used == 0) {
-                        return HANDLER_GO_ON;
-                }
-                memset (&ctx, 0, sizeof (ctx));
-
-                ctx.specfile = p->conf.specfile->ptr;
-                ctx.logfile = p->conf.logfile->ptr;
-                ctx.loglevel = p->conf.loglevel->ptr;
-                ctx.lookup_timeout = ctx.stat_timeout = p->conf.cache_timeout;
-
-                p->conf.handle = (unsigned long)glusterfs_init (&ctx);
-
-                if (p->conf.handle <= 0) {
-                        con->http_status = 500;
-                        log_error_write(srv, __FILE__, __LINE__,  "sbs",  "glusterfs initialization failed, please check your configuration. Glusterfs logfile ", p->conf.logfile, "might contain details");
-                        return HANDLER_FINISHED;
-                }
-        }
-
-        size = 0;
-        if (p->conf.xattr_file_size && p->conf.xattr_file_size->ptr) 
-                size = atoi (p->conf.xattr_file_size->ptr);
-
-        if (!con->plugin_ctx[p->id]) {
-                buffer *tmp_buf = buffer_init_buffer (con->physical.basedir);
-
-                plugin_ctx = calloc (1, sizeof (*plugin_ctx));
-                /* ERR_ABORT (plugin_ctx); */
-                con->plugin_ctx[p->id] = plugin_ctx;
-    
-                buffer_append_string_buffer (tmp_buf, p->conf.prefix);
-                buffer_path_simplify (tmp_buf, tmp_buf);
-
-                plugin_ctx->prefix = tmp_buf->used - 1;
-                if (tmp_buf->ptr[plugin_ctx->prefix - 1] == '/')
-                        plugin_ctx->prefix--;
-
-                buffer_free (tmp_buf);
-        } else 
-                /*FIXME: error!! error!! */
-                plugin_ctx = con->plugin_ctx[p->id];
-
-
-        if (size) 
-        {
-                plugin_ctx->buf = malloc (size);
-                /* ERR_ABORT (plugin_ctx->buf); */
-        }
-
-        plugin_ctx->glusterfs_path = buffer_init ();
-        buffer_copy_string_buffer (plugin_ctx->glusterfs_path, p->conf.document_root);
-        buffer_append_string (plugin_ctx->glusterfs_path, "/");
-        buffer_append_string (plugin_ctx->glusterfs_path, con->physical.path->ptr + plugin_ctx->prefix);
-        buffer_path_simplify (plugin_ctx->glusterfs_path, plugin_ctx->glusterfs_path);
-
-        ret = glusterfs_stat_cache_get_entry_async (srv, con, p, plugin_ctx->glusterfs_path, con->physical.path, plugin_ctx->buf, size, &sce);
-
-        if (ret == HANDLER_ERROR) {
-                free (plugin_ctx->buf);
-                plugin_ctx->buf = NULL;
-
-                buffer_free (plugin_ctx->glusterfs_path);
-                plugin_ctx->glusterfs_path = NULL;
-
-                free (plugin_ctx);
-                con->plugin_ctx[p->id] = NULL;
-
-                con->http_status = 500;
-                ret = HANDLER_FINISHED;
-        }
-
-        return ret;
-}
-
-URIHANDLER_FUNC(mod_glusterfs_subrequest) {
-        plugin_data *p = p_d;
-        stat_cache_entry *sce = NULL;
-        int s_len;
-        unsigned long fd;
-        char allow_caching = 1;
-        size_t size = 0;
-        mod_glusterfs_ctx_t *ctx = con->plugin_ctx[p->id];
-
-        /* someone else has done a decision for us */
-        if (con->http_status != 0) return HANDLER_GO_ON;
-        if (con->uri.path->used == 0) return HANDLER_GO_ON;
-        if (con->physical.path->used == 0) return HANDLER_GO_ON;
-  
-        /* someone else has handled this request */
-        if (con->mode != DIRECT) return HANDLER_GO_ON;
-  
-        /* we only handle GET, POST and HEAD */
-        switch(con->request.http_method) {
-        case HTTP_METHOD_GET:
-        case HTTP_METHOD_POST:
-        case HTTP_METHOD_HEAD:
-                break;
-        default:
-                return HANDLER_GO_ON;
-        }
-  
-        mod_glusterfs_patch_connection(srv, con, p);
-  
-        if (!p->conf.prefix || !p->conf.prefix->ptr)
-                return HANDLER_GO_ON;
-
-        if (!ctx) {
-                con->http_status = 500;
-                return HANDLER_FINISHED;
-        }
-
-        s_len = con->uri.path->used - 1;
-        /* ignore certain extensions */
-        /*
-          for (k = 0; k < p->conf.exclude_exts->used; k++) {
-          data_string *ds;
-          ds = (data_string *)p->conf.exclude_exts->data[k];
-          
-          if (ds->value->used == 0) continue;
-    
-          if (!strncmp (ds->value->ptr, con->uri.path->ptr, strlen (ds->value->ptr)))
-          break;
-          }
-  
-          if (k == p->conf.exclude_exts->used) {
-          return HANDLER_GO_ON;
-          }
-        */
-
-        if (con->conf.log_request_handling) {
-                log_error_write(srv, __FILE__, __LINE__,  "s",  "-- serving file from glusterfs");
-        }
-
-        if (HANDLER_ERROR == stat_cache_get_entry(srv, con, con->physical.path, &sce)) {
-                con->http_status = 403;
-
-                /* this might happen if the sce is removed from stat-cache after a successful glusterfs_lookup */
-                if (ctx) {
-                        if (ctx->buf) {
-                                free (ctx->buf);
-                                ctx->buf = NULL;
-                        }
-
-                        if (ctx->glusterfs_path) {
-                                buffer_free (ctx->glusterfs_path);
-                                ctx->glusterfs_path = NULL;
-                        }               
-                        free (ctx);
-                        con->plugin_ctx[p->id] = NULL;
-                }
-
-                log_error_write(srv, __FILE__, __LINE__, "sbsb",
-                                "not a regular file:", con->uri.path,
-                                "->", con->physical.path);
-    
-                return HANDLER_FINISHED;
-        }
-
-        if (con->uri.path->ptr[s_len] == '/' || !S_ISREG(sce->st.st_mode)) {
-                if (ctx) {
-                        if (ctx->glusterfs_path) {
-                                buffer_free (ctx->glusterfs_path);
-                                ctx->glusterfs_path = NULL;
-                        }
-
-                        free (ctx);
-                        con->plugin_ctx[p->id] = NULL;
-                }
-
-                return HANDLER_FINISHED;
-        }
-
-        if (p->conf.xattr_file_size && p->conf.xattr_file_size->ptr)
-                size = atoi (p->conf.xattr_file_size->ptr);
-
-        if ((size_t)sce->st.st_size > size) {
-    
-                fd = glusterfs_open ((libglusterfs_handle_t ) ((unsigned long)p->conf.handle), ctx->glusterfs_path->ptr, O_RDONLY, 0);
-    
-                if (!fd) {
-                        if (ctx) {
-                                if (ctx->glusterfs_path) {
-                                        buffer_free (ctx->glusterfs_path);
-                                        ctx->glusterfs_path = NULL;
-                                }
-
-                                free (ctx);
-                                con->plugin_ctx[p->id] = NULL;
-                        }
-
-                        con->http_status = 403;
-                        return HANDLER_FINISHED;
-                }
-                ctx->fd = fd;
-        }
-
-        /* we only handline regular files */
-#ifdef HAVE_LSTAT
-        if ((sce->is_symlink == 1) && !con->conf.follow_symlink) {
-                con->http_status = 403;
-          
-                if (con->conf.log_request_handling) {
-                        log_error_write(srv, __FILE__, __LINE__,  "s",  "-- access denied due symlink restriction");
-                        log_error_write(srv, __FILE__, __LINE__,  "sb", "Path         :", con->physical.path);
-                }
-    
-                buffer_reset(con->physical.path);
-                if (ctx) {
-                        if (ctx->glusterfs_path) {
-                                buffer_free (ctx->glusterfs_path);
-                                ctx->glusterfs_path = NULL;
-                        }
-
-                        free (ctx);
-                        con->plugin_ctx[p->id] = NULL;
-                }
-
-                return HANDLER_FINISHED;
-        }
-#endif
-        if (!S_ISREG(sce->st.st_mode)) {
-                con->http_status = 404;
-    
-                if (con->conf.log_file_not_found) {
-                        log_error_write(srv, __FILE__, __LINE__, "sbsb",
-                                        "not a regular file:", con->uri.path,
-                                        "->", sce->name);
-                }
-    
-                if (ctx) {
-                        if (ctx->glusterfs_path) {
-                                buffer_free (ctx->glusterfs_path);
-                                ctx->glusterfs_path = NULL;
-                        }
-
-                        free (ctx);
-                        con->plugin_ctx[p->id] = NULL;
-                }
-
-                return HANDLER_FINISHED;
-        }
-
-        /* mod_compress might set several data directly, don't overwrite them */
-
-        /* set response content-type, if not set already */
-  
-        if (NULL == array_get_element(con->response.headers, CONST_STR_LEN("Content-Type"))) {
-                if (buffer_is_empty(sce->content_type)) {
-                        /* we are setting application/octet-stream, but also announce that
-                         * this header field might change in the seconds few requests 
-                         *
-                         * This should fix the aggressive caching of FF and the script download
-                         * seen by the first installations
-                         */
-                        response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("application/octet-stream"));
-      
-                        allow_caching = 0;
-                } else {
-                        response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type));
-                }
-        }
-  
-        if (con->conf.range_requests) {
-                response_header_overwrite(srv, con, CONST_STR_LEN("Accept-Ranges"), CONST_STR_LEN("bytes"));
-        }
-
-        /* TODO: Allow Cachable requests */     
-#if 0
-        if (allow_caching) {
-                if (p->conf.etags_used && con->etag_flags != 0 && !buffer_is_empty(sce->etag)) {
-                        if (NULL == array_get_element(con->response.headers, "ETag")) {
-                                /* generate e-tag */
-                                etag_mutate(con->physical.etag, sce->etag);
-        
-                                response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag));
-                        }
-                }
-    
-                /* prepare header */
-                if (NULL == (ds = (data_string *)array_get_element(con->response.headers, "Last-Modified"))) {
-                        mtime = strftime_cache_get(srv, sce->st.st_mtime);
-                        response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
-                } else {
-                        mtime = ds->value;
-                }
-    
-                if (HANDLER_FINISHED == http_response_handle_cachable(srv, con, mtime)) {
-                        if (ctx) {
-                                if (ctx->glusterfs_path) {
-                                        buffer_free (ctx->glusterfs_path);
-                                        ctx->glusterfs_path = NULL;
-                                }
-
-                                free (ctx);
-                                con->plugin_ctx[p->id] = NULL;
-                        }
-
-                        return HANDLER_FINISHED;
-                }
-        }
-#endif 
-  
-        /*TODO: Read about etags */
-        if (NULL != array_get_element(con->request.headers, CONST_STR_LEN("Range")) && con->conf.range_requests) {
-                int do_range_request = 1;
-                data_string *ds = NULL;
-                buffer *mtime = NULL;
-                /* check if we have a conditional GET */
-    
-                /* prepare header */
-                if (NULL == (ds = (data_string *)array_get_element(con->response.headers, CONST_STR_LEN("Last-Modified")))) {
-                        mtime = strftime_cache_get(srv, sce->st.st_mtime);
-                        response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime));
-                } else {
-                        mtime = ds->value;
-                }
-
-                if (NULL != (ds = (data_string *)array_get_element(con->request.headers, CONST_STR_LEN("If-Range")))) {
-                        /* if the value is the same as our ETag, we do a Range-request,
-                         * otherwise a full 200 */
-            
-                        if (ds->value->ptr[0] == '"') {
-                                /**
-                                 * client wants a ETag
-                                 */
-                                if (!con->physical.etag) {
-                                        do_range_request = 0;
-                                } else if (!buffer_is_equal(ds->value, con->physical.etag)) {
-                                        do_range_request = 0;
-                                }
-                        } else if (!mtime) {
-                                /**
-                                 * we don't have a Last-Modified and can match the If-Range: 
-                                 *
-                                 * sending all
-                                 */
-                                do_range_request = 0;
-                        } else if (!buffer_is_equal(ds->value, mtime)) {
-                                do_range_request = 0;
-                        }
-                }
-    
-                if (do_range_request) {
-                        /* content prepared, I'm done */
-                        con->send->is_closed = 1; 
-            
-                        if (0 == http_response_parse_range(srv, con, p)) {
-                                con->http_status = 206;
-                        }
-                        if (ctx) {
-                                if (ctx->glusterfs_path) {
-                                        buffer_free (ctx->glusterfs_path);
-                                        ctx->glusterfs_path = NULL;
-                                }
-                                free (ctx);
-                                con->plugin_ctx[p->id] = NULL;
-                        }
-
-                        return HANDLER_FINISHED;
-                }
-        }
-  
-        /* if we are still here, prepare body */
-
-        /* we add it here for all requests
-         * the HEAD request will drop it afterwards again
-         */
-
-        if (p->conf.xattr_file_size && p->conf.xattr_file_size->ptr)
-                size = atoi (p->conf.xattr_file_size->ptr);
-
-        if (size < (size_t)sce->st.st_size) {
-                chunkqueue_append_glusterfs_file (con->send_raw, fd, 0, sce->st.st_size);
-                con->send_raw->bytes_in += sce->st.st_size;
-                chunkqueue_append_dummy_mem_chunk (con->send, sce->st.st_size);
-        } else {
-                if (!ctx->buf) {
-                        con->http_status = 404;
-                        return HANDLER_ERROR;
-                }
-                chunkqueue_append_glusterfs_mem (con->send, ctx->buf, sce->st.st_size);
-                ctx->buf = NULL;
-        }
-        ctx->response_content_length = con->response.content_length = sce->st.st_size;
-  
-        con->send->is_closed = 1;
-        con->send->bytes_in = sce->st.st_size;
-
-        return HANDLER_FINISHED;
-}
-
-/* this function is called at dlopen() time and inits the callbacks */
-CONNECTION_FUNC(mod_glusterfs_connection_reset) 
-{
-        (void) p_d;
-        (void) con;
-        if (!network_backend_write)
-                network_backend_write = srv->network_backend_write;
-
-        srv->network_backend_write = mod_glusterfs_network_backend_write;
-
-        return HANDLER_GO_ON;
-}
-
-URIHANDLER_FUNC(mod_glusterfs_response_done) {
-        plugin_data *p = p_d;
-        UNUSED (srv);
-        mod_glusterfs_ctx_t *ctx = con->plugin_ctx[p->id];
-        
-        con->plugin_ctx[p->id] = NULL;
-        if (ctx->glusterfs_path) {
-                free (ctx->glusterfs_path);
-        }
-
-        free (ctx);
-        return HANDLER_GO_ON;
-}
-
-int mod_glusterfs_plugin_init(plugin *p) {
-        p->version     = LIGHTTPD_VERSION_ID;
-        p->name        = buffer_init_string("glusterfs");
-        p->init        = mod_glusterfs_init;
-        p->handle_physical = mod_glusterfs_handle_physical;
-        p->handle_start_backend = mod_glusterfs_subrequest;
-        p->handle_response_done = mod_glusterfs_response_done;
-        p->set_defaults  = mod_glusterfs_set_defaults;
-        p->connection_reset = mod_glusterfs_connection_reset;
-        p->cleanup     = mod_glusterfs_free;
-  
-        p->data        = NULL;
-  
-        return 0;
-}
diff --git a/mod_glusterfs/lighttpd/1.5/mod_glusterfs.h b/mod_glusterfs/lighttpd/1.5/mod_glusterfs.h
deleted file mode 100644
index 5f9bb2c5b..000000000
--- a/mod_glusterfs/lighttpd/1.5/mod_glusterfs.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _MOD_GLUSTERFS_FILE_CACHE_H_
-#define _MOD_GLUSTERFS_FILE_CACHE_H_
-
-#include "stat_cache.h"
-#include <libglusterfsclient.h>
-#include "base.h"
-
-handler_t glusterfs_stat_cache_get_entry(server *srv, connection *con, libglusterfs_handle_t handle, buffer *glusterfs_path, buffer *name, void *buf, size_t size, stat_cache_entry **fce);
-
-#endif
diff --git a/mod_glusterfs/lighttpd/Makefile.am b/mod_glusterfs/lighttpd/Makefile.am
deleted file mode 100644
index c934412b3..000000000
--- a/mod_glusterfs/lighttpd/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = 1.4 1.5
-
-CLEANFILES = 
diff --git a/rfc.sh b/rfc.sh
index e6cce0d2c..baf700495 100755
--- a/rfc.sh
+++ b/rfc.sh
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 
 
-branch="master";
+branch="development";
 
 
 set_hooks_commit_msg()
@@ -16,6 +16,9 @@ set_hooks_commit_msg()
     curl -o $f $u || wget -O $f $u;
 
     chmod +x .git/hooks/commit-msg;
+
+    # Let the 'Change-Id: ' header get assigned on first run of rfc.sh
+    GIT_EDITOR=true git commit --amend;
 }
 
 
@@ -31,7 +34,7 @@ is_num()
 
 rebase_changes()
 {
-    git fetch;
+    git fetch origin;
 
     GIT_EDITOR=$0 git rebase -i origin/$branch;
 }
@@ -82,13 +85,13 @@ assert_diverge()
 
 main()
 {
+    set_hooks_commit_msg;
+
     if [ -e "$1" ]; then
         editor_mode "$@";
         return;
     fi
 
-    set_hooks_commit_msg;
-
     rebase_changes;
 
     assert_diverge;
diff --git a/rpc/rpc-lib/src/Makefile.am b/rpc/rpc-lib/src/Makefile.am
index 8b087301c..f19c3c8a4 100644
--- a/rpc/rpc-lib/src/Makefile.am
+++ b/rpc/rpc-lib/src/Makefile.am
@@ -1,16 +1,19 @@
 lib_LTLIBRARIES = libgfrpc.la
 
 libgfrpc_la_SOURCES = auth-unix.c rpcsvc-auth.c rpcsvc.c auth-null.c \
-	rpc-transport.c xdr-rpc.c xdr-rpcclnt.c rpc-clnt.c auth-glusterfs.c
+	rpc-transport.c xdr-rpc.c xdr-rpcclnt.c rpc-clnt.c auth-glusterfs.c \
+	rpc-drc.c
 
 libgfrpc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = rpcsvc.h rpc-transport.h xdr-common.h xdr-rpc.h xdr-rpcclnt.h \
-	rpc-clnt.h rpcsvc-common.h protocol-common.h
+	rpc-clnt.h rpcsvc-common.h protocol-common.h rpc-drc.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/rpc/xdr/src \
-	-DRPC_TRANSPORTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/rpc-transport\"
+	-DRPC_TRANSPORTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/rpc-transport\" \
+	-I$(top_srcdir)/contrib/rbtree
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = *~
diff --git a/rpc/rpc-lib/src/auth-glusterfs.c b/rpc/rpc-lib/src/auth-glusterfs.c
index 47d197c05..db488434c 100644
--- a/rpc/rpc-lib/src/auth-glusterfs.c
+++ b/rpc/rpc-lib/src/auth-glusterfs.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -73,9 +64,9 @@ int auth_glusterfs_authenticate (rpcsvc_request_t *req, void *priv)
         struct auth_glusterfs_parms  au = {0,};
 
         int ret      = RPCSVC_AUTH_REJECT;
-        int gidcount = 0;
         int j        = 0;
         int i        = 0;
+        int gidcount = 0;
 
         if (!req)
                 return ret;
@@ -105,9 +96,27 @@ int auth_glusterfs_authenticate (rpcsvc_request_t *req, void *priv)
                 goto err;
         }
 
+	if (req->auxgidcount > SMALL_GROUP_COUNT) {
+		req->auxgidlarge = GF_CALLOC(req->auxgidcount,
+					     sizeof(req->auxgids[0]),
+					     gf_common_mt_auxgids);
+		req->auxgids = req->auxgidlarge;
+	} else {
+		req->auxgids = req->auxgidsmall;
+	}
+
+	if (!req->auxgids) {
+		gf_log ("auth-glusterfs", GF_LOG_WARNING,
+			"cannot allocate gid list");
+		ret = RPCSVC_AUTH_REJECT;
+		goto err;
+	}
+
         for (gidcount = 0; gidcount < au.ngrps; ++gidcount)
                 req->auxgids[gidcount] = au.groups[gidcount];
 
+        RPC_AUTH_ROOT_SQUASH(req);
+
         gf_log (GF_RPCSVC, GF_LOG_TRACE, "Auth Info: pid: %u, uid: %d"
                 ", gid: %d, owner: %s",
                 req->pid, req->uid, req->gid, lkowner_utoa (&req->lk_owner));
@@ -210,22 +219,38 @@ int auth_glusterfs_v2_authenticate (rpcsvc_request_t *req, void *priv)
                 goto err;
         }
 
+	if (req->auxgidcount > SMALL_GROUP_COUNT) {
+		req->auxgidlarge = GF_CALLOC(req->auxgidcount,
+					     sizeof(req->auxgids[0]),
+					     gf_common_mt_auxgids);
+		req->auxgids = req->auxgidlarge;
+	} else {
+		req->auxgids = req->auxgidsmall;
+	}
+
+	if (!req->auxgids) {
+		gf_log ("auth-glusterfs-v2", GF_LOG_WARNING,
+			"cannot allocate gid list");
+		ret = RPCSVC_AUTH_REJECT;
+		goto err;
+	}
+
         for (i = 0; i < req->auxgidcount; ++i)
                 req->auxgids[i] = au.groups.groups_val[i];
 
         for (i = 0; i < au.lk_owner.lk_owner_len; ++i)
                 req->lk_owner.data[i] = au.lk_owner.lk_owner_val[i];
 
+        RPC_AUTH_ROOT_SQUASH(req);
+
         gf_log (GF_RPCSVC, GF_LOG_TRACE, "Auth Info: pid: %u, uid: %d"
                 ", gid: %d, owner: %s",
                 req->pid, req->uid, req->gid, lkowner_utoa (&req->lk_owner));
         ret = RPCSVC_AUTH_ACCEPT;
 err:
         /* TODO: instead use alloca() for these variables */
-        if (au.groups.groups_val)
-                free (au.groups.groups_val);
-        if (au.lk_owner.lk_owner_val)
-                free (au.lk_owner.lk_owner_val);
+        free (au.groups.groups_val);
+        free (au.lk_owner.lk_owner_val);
 
         return ret;
 }
diff --git a/rpc/rpc-lib/src/auth-null.c b/rpc/rpc-lib/src/auth-null.c
index aa6b4c1bd..ebdcc8ff8 100644
--- a/rpc/rpc-lib/src/auth-null.c
+++ b/rpc/rpc-lib/src/auth-null.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
diff --git a/rpc/rpc-lib/src/auth-unix.c b/rpc/rpc-lib/src/auth-unix.c
index c3b58945a..fa5f0576e 100644
--- a/rpc/rpc-lib/src/auth-unix.c
+++ b/rpc/rpc-lib/src/auth-unix.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -51,6 +42,7 @@ int auth_unix_authenticate (rpcsvc_request_t *req, void *priv)
         if (!req)
                 return ret;
 
+	req->auxgids = req->auxgidsmall;
         ret = xdr_to_auth_unix_cred (req->cred.authdata, req->cred.datalen,
                                      &aup, machname, req->auxgids);
         if (ret == -1) {
diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h
index 971996091..8bef906cc 100644
--- a/rpc/rpc-lib/src/protocol-common.h
+++ b/rpc/rpc-lib/src/protocol-common.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _PROTOCOL_COMMON_H
@@ -65,6 +56,9 @@ enum gf_fop_procnum {
         GFS3_OP_RELEASE,
         GFS3_OP_RELEASEDIR,
         GFS3_OP_FREMOVEXATTR,
+	GFS3_OP_FALLOCATE,
+	GFS3_OP_DISCARD,
+        GFS3_OP_ZEROFILL,
         GFS3_OP_MAXVALUE,
 } ;
 
@@ -73,6 +67,8 @@ enum gf_handshake_procnum {
         GF_HNDSK_SETVOLUME,
         GF_HNDSK_GETSPEC,
         GF_HNDSK_PING,
+        GF_HNDSK_SET_LK_VER,
+        GF_HNDSK_EVENT_NOTIFY,
         GF_HNDSK_MAXVALUE,
 };
 
@@ -101,8 +97,10 @@ enum gf_probe_resp {
 	GF_PROBE_FRIEND,
         GF_PROBE_ANOTHER_CLUSTER,
         GF_PROBE_VOLUME_CONFLICT,
+        GF_PROBE_SAME_UUID,
         GF_PROBE_UNKNOWN_PEER,
-        GF_PROBE_ADD_FAILED
+        GF_PROBE_ADD_FAILED,
+        GF_PROBE_QUORUM_NOT_MET
 };
 
 enum gf_deprobe_resp {
@@ -110,13 +108,15 @@ enum gf_deprobe_resp {
         GF_DEPROBE_LOCALHOST,
         GF_DEPROBE_NOT_FRIEND,
         GF_DEPROBE_BRICK_EXIST,
-        GF_DEPROBE_FRIEND_DOWN
+        GF_DEPROBE_FRIEND_DOWN,
+        GF_DEPROBE_QUORUM_NOT_MET,
 };
 
 enum gf_cbk_procnum {
         GF_CBK_NULL = 0,
         GF_CBK_FETCHSPEC,
         GF_CBK_INO_FLUSH,
+        GF_CBK_EVENT_NOTIFY,
         GF_CBK_MAXVALUE,
 };
 
@@ -154,6 +154,13 @@ enum gluster_cli_procnum {
         GLUSTER_CLI_UMOUNT,
         GLUSTER_CLI_HEAL_VOLUME,
         GLUSTER_CLI_STATEDUMP_VOLUME,
+        GLUSTER_CLI_LIST_VOLUME,
+        GLUSTER_CLI_CLRLOCKS_VOLUME,
+        GLUSTER_CLI_UUID_RESET,
+        GLUSTER_CLI_UUID_GET,
+        GLUSTER_CLI_COPY_FILE,
+        GLUSTER_CLI_SYS_EXEC,
+        GLUSTER_CLI_SNAP,
         GLUSTER_CLI_MAXVALUE,
 };
 
@@ -179,14 +186,49 @@ enum glusterd_brick_procnum {
         GLUSTERD_BRICK_NULL,    /* 0 */
         GLUSTERD_BRICK_TERMINATE,
         GLUSTERD_BRICK_XLATOR_INFO,
-        GLUSTERD_BRICK_XLATOR_HEAL,
+        GLUSTERD_BRICK_XLATOR_OP,
         GLUSTERD_BRICK_STATUS,
         GLUSTERD_BRICK_OP,
+        GLUSTERD_BRICK_XLATOR_DEFRAG,
+        GLUSTERD_NODE_PROFILE,
+        GLUSTERD_NODE_STATUS,
+        GLUSTERD_VOLUME_BARRIER_OP,
         GLUSTERD_BRICK_MAXVALUE,
 };
 
+enum glusterd_mgmt_hndsk_procnum {
+        GD_MGMT_HNDSK_NULL,
+        GD_MGMT_HNDSK_VERSIONS,
+        GD_MGMT_HNDSK_VERSIONS_ACK,
+        GD_MGMT_HNDSK_MAXVALUE,
+};
+
+typedef enum {
+        GF_AFR_OP_INVALID,
+        GF_AFR_OP_HEAL_INDEX,
+        GF_AFR_OP_HEAL_FULL,
+        GF_AFR_OP_INDEX_SUMMARY,
+        GF_AFR_OP_HEALED_FILES,
+        GF_AFR_OP_HEAL_FAILED_FILES,
+        GF_AFR_OP_SPLIT_BRAIN_FILES,
+        GF_AFR_OP_STATISTICS,
+        GF_AFR_OP_STATISTICS_HEAL_COUNT,
+        GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA,
+} gf_xl_afr_op_t ;
+
+enum glusterd_mgmt_v3_procnum {
+        GLUSTERD_MGMT_V3_NULL,    /* 0 */
+        GLUSTERD_MGMT_V3_LOCK,
+        GLUSTERD_MGMT_V3_PRE_VALIDATE,
+        GLUSTERD_MGMT_V3_BRICK_OP,
+        GLUSTERD_MGMT_V3_COMMIT,
+        GLUSTERD_MGMT_V3_POST_VALIDATE,
+        GLUSTERD_MGMT_V3_UNLOCK,
+        GLUSTERD_MGMT_V3_MAXVALUE,
+};
+
 #define GLUSTER_HNDSK_PROGRAM    14398633 /* Completely random */
-#define GLUSTER_HNDSK_VERSION    1   /* 0.0.1 */
+#define GLUSTER_HNDSK_VERSION    2   /* 0.0.2 */
 
 #define GLUSTER_PMAP_PROGRAM     34123456
 #define GLUSTER_PMAP_VERSION     1
@@ -194,9 +236,9 @@ enum glusterd_brick_procnum {
 #define GLUSTER_CBK_PROGRAM      52743234 /* Completely random */
 #define GLUSTER_CBK_VERSION      1   /* 0.0.1 */
 
-#define GLUSTER3_1_FOP_PROGRAM   1298437 /* Completely random */
-#define GLUSTER3_1_FOP_VERSION   330 /* 3.3.0 */
-#define GLUSTER3_1_FOP_PROCCNT   GFS3_OP_MAXVALUE
+#define GLUSTER_FOP_PROGRAM   1298437 /* Completely random */
+#define GLUSTER_FOP_VERSION   330 /* 3.3.0 */
+#define GLUSTER_FOP_PROCCNT   GFS3_OP_MAXVALUE
 
 /* Second version */
 #define GD_MGMT_PROGRAM          1238433 /* Completely random */
@@ -211,4 +253,12 @@ enum glusterd_brick_procnum {
 #define GD_BRICK_PROGRAM         4867634 /*Completely random*/
 #define GD_BRICK_VERSION         2
 
+/* Third version */
+#define GD_MGMT_V3_PROGRAM       2210013 /* Completely random */
+#define GD_MGMT_V3_VERSION       3
+
+/* OP-VERSION handshake */
+#define GD_MGMT_HNDSK_PROGRAM    1239873 /* Completely random */
+#define GD_MGMT_HNDSK_VERSION    1
+
 #endif /* !_PROTOCOL_COMMON_H */
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 631d7fcf7..ac98a5c91 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -23,7 +14,7 @@
 #include "config.h"
 #endif
 
-#define RPC_CLNT_DEFAULT_REQUEST_COUNT 4096
+#define RPC_CLNT_DEFAULT_REQUEST_COUNT 512
 
 #include "rpc-clnt.h"
 #include "byte-order.h"
@@ -75,8 +66,8 @@ _is_lock_fop (struct saved_frame *sframe)
 {
         int     fop     = 0;
 
-        if (SFRAME_GET_PROGNUM (sframe) == GLUSTER3_1_FOP_PROGRAM &&
-            SFRAME_GET_PROGVER (sframe) == GLUSTER3_1_FOP_VERSION)
+        if (SFRAME_GET_PROGNUM (sframe) == GLUSTER_FOP_PROGRAM &&
+            SFRAME_GET_PROGVER (sframe) == GLUSTER_FOP_VERSION)
                 fop = SFRAME_GET_PROCNUM (sframe);
 
         return ((fop == GFS3_OP_LK) ||
@@ -153,9 +144,8 @@ call_bail (void *data)
         struct saved_frame    *saved_frame = NULL;
         struct saved_frame    *trav = NULL;
         struct saved_frame    *tmp = NULL;
-        struct tm              frame_sent_tm;
         char                   frame_sent[256] = {0,};
-        struct timeval         timeout = {0,};
+        struct timespec        timeout = {0,};
         struct iovec           iov = {0,};
 
         GF_VALIDATE_OR_GOTO ("client", data, out);
@@ -173,7 +163,7 @@ call_bail (void *data)
                    call-once timer */
                 if (conn->timer) {
                         timeout.tv_sec = 10;
-                        timeout.tv_usec = 0;
+                        timeout.tv_nsec = 0;
 
                         gf_timer_call_cancel (clnt->ctx, conn->timer);
                         conn->timer = gf_timer_call_after (clnt->ctx,
@@ -183,7 +173,8 @@ call_bail (void *data)
 
                         if (conn->timer == NULL) {
                                 gf_log (conn->trans->name, GF_LOG_WARNING,
-                                        "Cannot create bailout timer");
+                                        "Cannot create bailout timer for %s",
+                                        conn->trans->peerinfo.identifier);
                         }
                 }
 
@@ -200,21 +191,21 @@ call_bail (void *data)
         pthread_mutex_unlock (&conn->lock);
 
         list_for_each_entry_safe (trav, tmp, &list, list) {
-                localtime_r (&trav->saved_at.tv_sec, &frame_sent_tm);
-                strftime (frame_sent, 32, "%Y-%m-%d %H:%M:%S", &frame_sent_tm);
+                gf_time_fmt (frame_sent, sizeof frame_sent,
+                             trav->saved_at.tv_sec, gf_timefmt_FT);
                 snprintf (frame_sent + strlen (frame_sent),
                           256 - strlen (frame_sent),
                           ".%"GF_PRI_SUSECONDS, trav->saved_at.tv_usec);
 
 		gf_log (conn->trans->name, GF_LOG_ERROR,
-			"bailing out frame type(%s) op(%s(%d)) xid = 0x%ux "
-                        "sent = %s. timeout = %d",
+			"bailing out frame type(%s) op(%s(%d)) xid = 0x%x "
+                        "sent = %s. timeout = %d for %s",
 			trav->rpcreq->prog->progname,
                         (trav->rpcreq->prog->procnames) ?
                         trav->rpcreq->prog->procnames[trav->rpcreq->procnum] :
                         "--",
                         trav->rpcreq->procnum, trav->rpcreq->xid, frame_sent,
-                        conn->frame_timeout);
+                        conn->frame_timeout, conn->trans->peerinfo.identifier);
 
                 clnt = rpc_clnt_ref (clnt);
                 trav->rpcreq->rpc_status = -1;
@@ -236,7 +227,7 @@ __save_frame (struct rpc_clnt *rpc_clnt, call_frame_t *frame,
               struct rpc_req *rpcreq)
 {
         rpc_clnt_connection_t *conn        = NULL;
-        struct timeval         timeout     = {0, };
+        struct timespec        timeout     = {0, };
         struct saved_frame    *saved_frame = NULL;
 
         conn = &rpc_clnt->conn;
@@ -250,7 +241,7 @@ __save_frame (struct rpc_clnt *rpc_clnt, call_frame_t *frame,
         /* TODO: make timeout configurable */
         if (conn->timer == NULL) {
                 timeout.tv_sec  = 10;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
                 conn->timer = gf_timer_call_after (rpc_clnt->ctx,
                                                    timeout,
                                                    call_bail,
@@ -349,20 +340,16 @@ out:
 void
 saved_frames_unwind (struct saved_frames *saved_frames)
 {
-        struct rpc_clnt      *clnt = NULL;
 	struct saved_frame   *trav = NULL;
 	struct saved_frame   *tmp = NULL;
-        struct tm            *frame_sent_tm = NULL;
-        char                 timestr[256] = {0,};
-
+        char                  timestr[1024] = {0,};
         struct iovec          iov = {0,};
 
         list_splice_init (&saved_frames->lk_sf.list, &saved_frames->sf.list);
 
 	list_for_each_entry_safe (trav, tmp, &saved_frames->sf.list, list) {
-                frame_sent_tm = localtime (&trav->saved_at.tv_sec);
-                strftime (timestr, sizeof(timestr), "%Y-%m-%d %H:%M:%S",
-                          frame_sent_tm);
+                gf_time_fmt (timestr, sizeof timestr,
+                             trav->saved_at.tv_sec, gf_timefmt_FT);
                 snprintf (timestr + strlen (timestr),
                           sizeof(timestr) - strlen (timestr),
                           ".%"GF_PRI_SUSECONDS, trav->saved_at.tv_usec);
@@ -373,22 +360,21 @@ saved_frames_unwind (struct saved_frames *saved_frames)
                 gf_log_callingfn (trav->rpcreq->conn->trans->name,
                                   GF_LOG_ERROR,
                                   "forced unwinding frame type(%s) op(%s(%d)) "
-                                  "called at %s",
+                                  "called at %s (xid=0x%x)",
                                   trav->rpcreq->prog->progname,
                                   ((trav->rpcreq->prog->procnames) ?
                                    trav->rpcreq->prog->procnames[trav->rpcreq->procnum]
                                    : "--"),
-                                  trav->rpcreq->procnum, timestr);
+                                  trav->rpcreq->procnum, timestr,
+                                  trav->rpcreq->xid);
 		saved_frames->count--;
 
-                clnt = rpc_clnt_ref (trav->rpcreq->conn->rpc_clnt);
                 trav->rpcreq->rpc_status = -1;
                 trav->rpcreq->cbkfn (trav->rpcreq, &iov, 1, trav->frame);
 
                 rpc_clnt_reply_deinit (trav->rpcreq,
                                        trav->rpcreq->conn->rpc_clnt->reqpool);
 
-                clnt = rpc_clnt_unref (clnt);
 		list_del_init (&trav->list);
                 mem_put (trav);
 	}
@@ -412,7 +398,7 @@ rpc_clnt_reconnect (void *trans_ptr)
 {
         rpc_transport_t         *trans = NULL;
         rpc_clnt_connection_t   *conn  = NULL;
-        struct timeval           tv    = {0, 0};
+        struct timespec          ts    = {0, 0};
         int32_t                  ret   = 0;
         struct rpc_clnt         *clnt  = NULL;
 
@@ -431,23 +417,15 @@ rpc_clnt_reconnect (void *trans_ptr)
                 conn->reconnect = 0;
 
                 if (conn->connected == 0) {
-                        tv.tv_sec = 3;
+                        ts.tv_sec = 3;
+                        ts.tv_nsec = 0;
 
                         gf_log (trans->name, GF_LOG_TRACE,
                                 "attempting reconnect");
                         ret = rpc_transport_connect (trans,
                                                      conn->config.remote_port);
-                        /* Every time there is a disconnection, processes
-                           should try to connect to 'glusterd' (ie, default
-                           port) or whichever port given as 'option remote-port'
-                           in volume file. */
-                        /* Below code makes sure the (re-)configured port lasts
-                           for just one successful attempt */
-                        if (!ret)
-                                conn->config.remote_port = 0;
-
                         conn->reconnect =
-                                gf_timer_call_after (clnt->ctx, tv,
+                                gf_timer_call_after (clnt->ctx, ts,
                                                      rpc_clnt_reconnect,
                                                      trans);
                 } else {
@@ -468,7 +446,7 @@ rpc_clnt_reconnect (void *trans_ptr)
 int
 rpc_clnt_fill_request_info (struct rpc_clnt *clnt, rpc_request_info_t *info)
 {
-        struct saved_frame  saved_frame = {{}, 0};
+        struct saved_frame  saved_frame;
         int                 ret         = -1;
 
         pthread_mutex_lock (&clnt->conn.lock);
@@ -554,6 +532,12 @@ rpc_clnt_connection_cleanup (rpc_clnt_connection_t *conn)
                 }
 
                 conn->connected = 0;
+
+                if (conn->ping_timer) {
+                        gf_timer_call_cancel (clnt->ctx, conn->ping_timer);
+                        conn->ping_timer = NULL;
+                        conn->ping_started = 0;
+                }
         }
         pthread_mutex_unlock (&conn->lock);
 
@@ -679,15 +663,13 @@ rpc_clnt_reply_init (rpc_clnt_connection_t *conn, rpc_transport_pollin_t *msg,
         }
 
         gf_log (conn->trans->name, GF_LOG_TRACE,
-                "received rpc message (RPC XID: 0x%ux"
+                "received rpc message (RPC XID: 0x%x"
                 " Program: %s, ProgVers: %d, Proc: %d) from rpc-transport (%s)",
                 saved_frame->rpcreq->xid,
                 saved_frame->rpcreq->prog->progname,
                 saved_frame->rpcreq->prog->progver,
                 saved_frame->rpcreq->procnum, conn->trans->name);
 
-        req->rpc_status = 0;
-
 out:
         if (ret != 0) {
                 req->rpc_status = -1;
@@ -744,7 +726,8 @@ rpc_clnt_handle_cbk (struct rpc_clnt *clnt, rpc_transport_pollin_t *msg)
 
         if (found && (procnum < program->numactors) &&
             (program->actors[procnum].actor)) {
-                program->actors[procnum].actor (&progmsg);
+                program->actors[procnum].actor (clnt, program->mydata,
+                                                &progmsg);
         }
 
 out:
@@ -838,17 +821,19 @@ out:
         return;
 }
 
+static void
+rpc_clnt_destroy (struct rpc_clnt *rpc);
 
 int
 rpc_clnt_notify (rpc_transport_t *trans, void *mydata,
                  rpc_transport_event_t event, void *data, ...)
 {
-        rpc_clnt_connection_t  *conn     = NULL;
-        struct rpc_clnt        *clnt     = NULL;
-        int                     ret      = -1;
-        rpc_request_info_t     *req_info = NULL;
-        rpc_transport_pollin_t *pollin   = NULL;
-        struct timeval          tv       = {0, };
+        rpc_clnt_connection_t  *conn        = NULL;
+        struct rpc_clnt        *clnt        = NULL;
+        int                     ret         = -1;
+        rpc_request_info_t     *req_info    = NULL;
+        rpc_transport_pollin_t *pollin      = NULL;
+        struct timespec         ts          = {0, };
 
         conn = mydata;
         if (conn == NULL) {
@@ -865,11 +850,13 @@ rpc_clnt_notify (rpc_transport_t *trans, void *mydata,
 
                 pthread_mutex_lock (&conn->lock);
                 {
-                        if (conn->reconnect == NULL) {
-                                tv.tv_sec = 10;
+                        if (!conn->rpc_clnt->disabled
+                            && (conn->reconnect == NULL)) {
+                                ts.tv_sec = 10;
+                                ts.tv_nsec = 0;
 
                                 conn->reconnect =
-                                        gf_timer_call_after (clnt->ctx, tv,
+                                        gf_timer_call_after (clnt->ctx, ts,
                                                              rpc_clnt_reconnect,
                                                              conn->trans);
                         }
@@ -883,9 +870,7 @@ rpc_clnt_notify (rpc_transport_t *trans, void *mydata,
         }
 
         case RPC_TRANSPORT_CLEANUP:
-                /* this event should not be received on a client for, a
-                 * transport is only disconnected, but never destroyed.
-                 */
+                rpc_clnt_destroy (clnt);
                 ret = 0;
                 break;
 
@@ -929,6 +914,14 @@ rpc_clnt_notify (rpc_transport_t *trans, void *mydata,
 
         case RPC_TRANSPORT_CONNECT:
         {
+                /* Every time there is a disconnection, processes
+                   should try to connect to 'glusterd' (ie, default
+                   port) or whichever port given as 'option remote-port'
+                   in volume file. */
+                /* Below code makes sure the (re-)configured port lasts
+                   for just one successful attempt */
+                conn->config.remote_port = 0;
+
                 if (clnt->notifyfn)
                         ret = clnt->notifyfn (clnt, clnt->mydata,
                                               RPC_CLNT_CONNECT, NULL);
@@ -955,7 +948,7 @@ rpc_clnt_connection_deinit (rpc_clnt_connection_t *conn)
 }
 
 
-inline int
+static inline int
 rpc_clnt_connection_init (struct rpc_clnt *clnt, glusterfs_ctx_t *ctx,
                           dict_t *options, char *name)
 {
@@ -1012,8 +1005,8 @@ out:
 }
 
 struct rpc_clnt *
-rpc_clnt_new (dict_t *options,
-              glusterfs_ctx_t *ctx, char *name)
+rpc_clnt_new (dict_t *options, glusterfs_ctx_t *ctx, char *name,
+              uint32_t reqpool_size)
 {
         int                    ret  = -1;
         struct rpc_clnt       *rpc  = NULL;
@@ -1026,8 +1019,10 @@ rpc_clnt_new (dict_t *options,
         pthread_mutex_init (&rpc->lock, NULL);
         rpc->ctx = ctx;
 
-        rpc->reqpool = mem_pool_new (struct rpc_req,
-                                     RPC_CLNT_DEFAULT_REQUEST_COUNT);
+        if (!reqpool_size)
+                reqpool_size = RPC_CLNT_DEFAULT_REQUEST_COUNT;
+
+        rpc->reqpool = mem_pool_new (struct rpc_req, reqpool_size);
         if (rpc->reqpool == NULL) {
                 pthread_mutex_destroy (&rpc->lock);
                 GF_FREE (rpc);
@@ -1036,7 +1031,7 @@ rpc_clnt_new (dict_t *options,
         }
 
         rpc->saved_frames_pool = mem_pool_new (struct saved_frame,
-                                              RPC_CLNT_DEFAULT_REQUEST_COUNT);
+                                               reqpool_size);
         if (rpc->saved_frames_pool == NULL) {
                 pthread_mutex_destroy (&rpc->lock);
                 mem_pool_destroy (rpc->reqpool);
@@ -1057,6 +1052,8 @@ rpc_clnt_new (dict_t *options,
                 goto out;
         }
 
+        rpc->auth_null = dict_get_str_boolean (options, "auth-null", 0);
+
         rpc = rpc_clnt_ref (rpc);
         INIT_LIST_HEAD (&rpc->programs);
 
@@ -1117,7 +1114,7 @@ ret:
 
 
 int
-rpc_clnt_fill_request (int prognum, int progver, int procnum, int payload,
+rpc_clnt_fill_request (int prognum, int progver, int procnum,
                        uint64_t xid, struct auth_glusterfs_parms_v2 *au,
                        struct rpc_msg *request, char *auth_data)
 {
@@ -1137,19 +1134,26 @@ rpc_clnt_fill_request (int prognum, int progver, int procnum, int payload,
         request->rm_call.cb_vers = progver;
         request->rm_call.cb_proc = procnum;
 
-        /* TODO: Using AUTH_GLUSTERFS for time-being. Make it modular in
-         * future so it is easy to plug-in new authentication schemes.
+        /* TODO: Using AUTH_(GLUSTERFS/NULL) in a kludgy way for time-being.
+         * Make it modular in future so it is easy to plug-in new
+         * authentication schemes.
          */
-        ret = xdr_serialize_glusterfs_auth (auth_data, au);
-        if (ret == -1) {
-                gf_log ("rpc-clnt", GF_LOG_DEBUG, "cannot encode credentials");
-                goto out;
-        }
-
-        request->rm_call.cb_cred.oa_flavor = AUTH_GLUSTERFS_v2;
-        request->rm_call.cb_cred.oa_base   = auth_data;
-        request->rm_call.cb_cred.oa_length = ret;
+        if (auth_data) {
+                ret = xdr_serialize_glusterfs_auth (auth_data, au);
+                if (ret == -1) {
+                        gf_log ("rpc-clnt", GF_LOG_DEBUG,
+                                "cannot encode credentials");
+                        goto out;
+                }
 
+                request->rm_call.cb_cred.oa_flavor = AUTH_GLUSTERFS_v2;
+                request->rm_call.cb_cred.oa_base   = auth_data;
+                request->rm_call.cb_cred.oa_length = ret;
+        } else {
+                request->rm_call.cb_cred.oa_flavor = AUTH_NULL;
+                request->rm_call.cb_cred.oa_base   = NULL;
+                request->rm_call.cb_cred.oa_length = 0;
+        }
         request->rm_call.cb_verf.oa_flavor = AUTH_NONE;
         request->rm_call.cb_verf.oa_base = NULL;
         request->rm_call.cb_verf.oa_length = 0;
@@ -1197,7 +1201,7 @@ out:
 
 struct iobuf *
 rpc_clnt_record_build_record (struct rpc_clnt *clnt, int prognum, int progver,
-                              int procnum, size_t payload, uint64_t xid,
+                              int procnum, size_t hdrsize, uint64_t xid,
                               struct auth_glusterfs_parms_v2 *au,
                               struct iovec *recbuf)
 {
@@ -1207,16 +1211,33 @@ rpc_clnt_record_build_record (struct rpc_clnt *clnt, int prognum, int progver,
         struct iovec    recordhdr                    = {0, };
         size_t          pagesize                     = 0;
         int             ret                          = -1;
+        size_t          xdr_size                     = 0;
         char            auth_data[GF_MAX_AUTH_BYTES] = {0, };
 
         if ((!clnt) || (!recbuf) || (!au)) {
                 goto out;
         }
 
+        /* Fill the rpc structure and XDR it into the buffer got above. */
+        if (clnt->auth_null)
+                ret = rpc_clnt_fill_request (prognum, progver, procnum,
+                                             xid, NULL, &request, NULL);
+        else
+                ret = rpc_clnt_fill_request (prognum, progver, procnum,
+                                             xid, au, &request, auth_data);
+
+        if (ret == -1) {
+                gf_log (clnt->conn.trans->name, GF_LOG_WARNING,
+                        "cannot build a rpc-request xid (%"PRIu64")", xid);
+                goto out;
+        }
+
+        xdr_size = xdr_sizeof ((xdrproc_t)xdr_callmsg, &request);
+
         /* First, try to get a pointer into the buffer which the RPC
          * layer can use.
          */
-        request_iob = iobuf_get (clnt->ctx->iobuf_pool);
+        request_iob = iobuf_get2 (clnt->ctx->iobuf_pool, (xdr_size + hdrsize));
         if (!request_iob) {
                 goto out;
         }
@@ -1225,17 +1246,8 @@ rpc_clnt_record_build_record (struct rpc_clnt *clnt, int prognum, int progver,
 
         record = iobuf_ptr (request_iob);  /* Now we have it. */
 
-        /* Fill the rpc structure and XDR it into the buffer got above. */
-        ret = rpc_clnt_fill_request (prognum, progver, procnum, payload, xid,
-                                     au, &request, auth_data);
-        if (ret == -1) {
-                gf_log (clnt->conn.trans->name, GF_LOG_WARNING,
-                        "cannot build a rpc-request xid (%"PRIu64")", xid);
-                goto out;
-        }
-
         recordhdr = rpc_clnt_record_build_header (record, pagesize, &request,
-                                                  payload);
+                                                  hdrsize);
 
         if (!recordhdr.iov_base) {
                 gf_log (clnt->conn.trans->name, GF_LOG_ERROR,
@@ -1256,7 +1268,7 @@ out:
 
 struct iobuf *
 rpc_clnt_record (struct rpc_clnt *clnt, call_frame_t *call_frame,
-                 rpc_clnt_prog_t *prog,int procnum, size_t payload_len,
+                 rpc_clnt_prog_t *prog, int procnum, size_t hdrlen,
                  struct iovec *rpchdr, uint64_t callid)
 {
         struct auth_glusterfs_parms_v2  au          = {0, };
@@ -1292,12 +1304,9 @@ rpc_clnt_record (struct rpc_clnt *clnt, call_frame_t *call_frame,
                 ", gid: %d, owner: %s", au.pid, au.uid, au.gid,
                 lkowner_utoa (&call_frame->root->lk_owner));
 
-        /* Assuming the client program would like to speak to the same version of
-         * program on server.
-         */
         request_iob = rpc_clnt_record_build_record (clnt, prog->prognum,
                                                     prog->progver,
-                                                    procnum, payload_len,
+                                                    procnum, hdrlen,
                                                     callid, &au,
                                                     rpchdr);
         if (!request_iob) {
@@ -1312,7 +1321,7 @@ out:
 
 int
 rpcclnt_cbk_program_register (struct rpc_clnt *clnt,
-                              rpcclnt_cb_program_t *program)
+                              rpcclnt_cb_program_t *program, void *mydata)
 {
         int                   ret                = -1;
         char                  already_registered = 0;
@@ -1352,6 +1361,8 @@ rpcclnt_cbk_program_register (struct rpc_clnt *clnt,
         memcpy (tmp, program, sizeof (*tmp));
         INIT_LIST_HEAD (&tmp->program);
 
+        tmp->mydata = mydata;
+
         pthread_mutex_lock (&clnt->lock);
         {
                 list_add_tail (&tmp->program, &clnt->programs);
@@ -1399,6 +1410,12 @@ rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
                 goto out;
         }
 
+        conn = &rpc->conn;
+
+        if (conn->trans == NULL) {
+                goto out;
+        }
+
         rpcreq = mem_get (rpc->reqpool);
         if (rpcreq == NULL) {
                 goto out;
@@ -1418,8 +1435,6 @@ rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
 
         callid = rpc_clnt_new_callid (rpc);
 
-        conn = &rpc->conn;
-
         rpcreq->prog = prog;
         rpcreq->procnum = procnum;
         rpcreq->conn = conn;
@@ -1432,11 +1447,6 @@ rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
                 proglen += iov_length (proghdr, proghdrcount);
         }
 
-        if (progpayload) {
-                proglen += iov_length (progpayload,
-                                       progpayloadcount);
-        }
-
         request_iob = rpc_clnt_record (rpc, frame, prog,
                                        procnum, proglen,
                                        &rpchdr, callid);
@@ -1468,10 +1478,6 @@ rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
                 if (conn->connected == 0) {
                         ret = rpc_transport_connect (conn->trans,
                                                      conn->config.remote_port);
-                        /* Below code makes sure the (re-)configured port lasts
-                           for just one successful connect attempt */
-                        if (!ret)
-                                conn->config.remote_port = 0;
                 }
 
                 ret = rpc_transport_submit_request (rpc->conn.trans,
@@ -1479,19 +1485,18 @@ rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
                 if (ret == -1) {
                         gf_log (conn->trans->name, GF_LOG_WARNING,
                                 "failed to submit rpc-request "
-                                "(XID: 0x%ux Program: %s, ProgVers: %d, "
+                                "(XID: 0x%x Program: %s, ProgVers: %d, "
                                 "Proc: %d) to rpc-transport (%s)", rpcreq->xid,
                                 rpcreq->prog->progname, rpcreq->prog->progver,
                                 rpcreq->procnum, rpc->conn.trans->name);
                 }
 
                 if ((ret >= 0) && frame) {
-                        gettimeofday (&conn->last_sent, NULL);
                         /* Save the frame in queue */
                         __save_frame (rpc, frame, rpcreq);
 
                         gf_log ("rpc-clnt", GF_LOG_TRACE, "submitted request "
-                                "(XID: 0x%ux Program: %s, ProgVers: %d, "
+                                "(XID: 0x%x Program: %s, ProgVers: %d, "
                                 "Proc: %d) to rpc-transport (%s)", rpcreq->xid,
                                 rpcreq->prog->progname, rpcreq->prog->progver,
                                 rpcreq->procnum, rpc->conn.trans->name);
@@ -1540,19 +1545,21 @@ rpc_clnt_ref (struct rpc_clnt *rpc)
 
 
 static void
-rpc_clnt_destroy (struct rpc_clnt *rpc)
+rpc_clnt_trigger_destroy (struct rpc_clnt *rpc)
 {
         if (!rpc)
                 return;
 
-        if (rpc->conn.trans) {
-                rpc->conn.trans->mydata = NULL;
-                rpc_transport_unref (rpc->conn.trans);
-                //rpc_transport_destroy (rpc->conn.trans);
-        }
+        rpc_clnt_disable (rpc);
+        rpc_transport_unref (rpc->conn.trans);
+}
+
+static void
+rpc_clnt_destroy (struct rpc_clnt *rpc)
+{
+        if (!rpc)
+                return;
 
-        rpc_clnt_connection_cleanup (&rpc->conn);
-        rpc_clnt_reconnect_cleanup (&rpc->conn);
         saved_frames_destroy (rpc->conn.saved_frames);
         pthread_mutex_destroy (&rpc->lock);
         pthread_mutex_destroy (&rpc->conn.lock);
@@ -1579,13 +1586,78 @@ rpc_clnt_unref (struct rpc_clnt *rpc)
         }
         pthread_mutex_unlock (&rpc->lock);
         if (!count) {
-                rpc_clnt_destroy (rpc);
+                rpc_clnt_trigger_destroy (rpc);
                 return NULL;
         }
         return rpc;
 }
 
 
+char
+rpc_clnt_is_disabled (struct rpc_clnt *rpc)
+{
+
+        rpc_clnt_connection_t *conn = NULL;
+        char                   disabled = 0;
+
+        if (!rpc) {
+                goto out;
+        }
+
+        conn = &rpc->conn;
+
+        pthread_mutex_lock (&conn->lock);
+        {
+                disabled = rpc->disabled;
+        }
+        pthread_mutex_unlock (&conn->lock);
+
+out:
+        return disabled;
+}
+
+void
+rpc_clnt_disable (struct rpc_clnt *rpc)
+{
+        rpc_clnt_connection_t *conn = NULL;
+
+        if (!rpc) {
+                goto out;
+        }
+
+        conn = &rpc->conn;
+
+        pthread_mutex_lock (&conn->lock);
+        {
+                rpc->disabled = 1;
+
+                if (conn->timer) {
+                        gf_timer_call_cancel (rpc->ctx, conn->timer);
+                        conn->timer = NULL;
+                }
+
+                if (conn->reconnect) {
+                        gf_timer_call_cancel (rpc->ctx, conn->reconnect);
+                        conn->reconnect = NULL;
+                }
+                conn->connected = 0;
+
+                if (conn->ping_timer) {
+                        gf_timer_call_cancel (rpc->ctx, conn->ping_timer);
+                        conn->ping_timer = NULL;
+                        conn->ping_started = 0;
+                }
+
+        }
+        pthread_mutex_unlock (&conn->lock);
+
+        rpc_transport_disconnect (rpc->conn.trans);
+
+out:
+        return;
+}
+
+
 void
 rpc_clnt_reconfig (struct rpc_clnt *rpc, struct rpc_clnt_config *config)
 {
@@ -1613,7 +1685,7 @@ rpc_clnt_reconfig (struct rpc_clnt *rpc, struct rpc_clnt_config *config)
                         if (strcmp (rpc->conn.config.remote_host,
                                     config->remote_host))
                                 gf_log (rpc->conn.trans->name, GF_LOG_INFO,
-                                        "changing port to %s (from %s)",
+                                        "changing hostname to %s (from %s)",
                                         config->remote_host,
                                         rpc->conn.config.remote_host);
                         FREE (rpc->conn.config.remote_host);
@@ -1626,54 +1698,3 @@ rpc_clnt_reconfig (struct rpc_clnt *rpc, struct rpc_clnt_config *config)
                 rpc->conn.config.remote_host = gf_strdup (config->remote_host);
         }
 }
-
-int
-rpc_clnt_transport_unix_options_build (dict_t **options, char *filepath)
-{
-        dict_t                  *dict = NULL;
-        char                    *fpath = NULL;
-        int                     ret = -1;
-
-        GF_ASSERT (filepath);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        fpath = gf_strdup (filepath);
-        if (!fpath) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (dict, "transport.socket.connect-path", fpath);
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (dict, "transport.address-family", "unix");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (dict, "transport.socket.nodelay", "off");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (dict, "transport-type", "socket");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (dict, "transport.socket.keepalive", "off");
-        if (ret)
-                goto out;
-
-        *options = dict;
-out:
-        if (ret) {
-                if (fpath)
-                        GF_FREE (fpath);
-                if (dict)
-                        dict_unref (dict);
-        }
-        return ret;
-}
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index 7034da705..584963ad0 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __RPC_CLNT_H
@@ -87,7 +78,7 @@ typedef struct rpc_clnt_program {
         int                   numproc;
 } rpc_clnt_prog_t;
 
-typedef int (*rpcclnt_cb_fn) (void *data);
+typedef int (*rpcclnt_cb_fn) (struct rpc_clnt *rpc, void *mydata, void *data);
 
 /* The descriptor for each procedure/actor that runs
  * over the RPC service.
@@ -115,6 +106,9 @@ typedef struct rpcclnt_cb_program {
 
         /* list member to link to list of registered services with rpc_clnt */
         struct list_head        program;
+
+        /* Needed for passing back in cb_actor */
+        void                   *mydata;
 } rpcclnt_cb_program_t;
 
 
@@ -169,7 +163,7 @@ struct rpc_req {
         void                  *conn_private;
 };
 
-struct rpc_clnt {
+typedef struct rpc_clnt {
         pthread_mutex_t        lock;
         rpc_clnt_notify_t      notifyfn;
         rpc_clnt_connection_t  conn;
@@ -186,11 +180,13 @@ struct rpc_clnt {
 
         glusterfs_ctx_t       *ctx;
         int                   refcount;
-};
+        int                   auth_null;
+        char                  disabled;
+} rpc_clnt_t;
 
 
 struct rpc_clnt *rpc_clnt_new (dict_t *options, glusterfs_ctx_t *ctx,
-                               char *name);
+                               char *name, uint32_t reqpool_size);
 
 int rpc_clnt_start (struct rpc_clnt *rpc);
 
@@ -226,10 +222,11 @@ rpc_clnt_ref (struct rpc_clnt *rpc);
 struct rpc_clnt *
 rpc_clnt_unref (struct rpc_clnt *rpc);
 
+int rpc_clnt_connection_cleanup (rpc_clnt_connection_t *conn);
+
 void rpc_clnt_set_connected (rpc_clnt_connection_t *conn);
 
 void rpc_clnt_unset_connected (rpc_clnt_connection_t *conn);
-
 void rpc_clnt_reconnect (void *trans_ptr);
 
 void rpc_clnt_reconfig (struct rpc_clnt *rpc, struct rpc_clnt_config *config);
@@ -238,8 +235,12 @@ void rpc_clnt_reconfig (struct rpc_clnt *rpc, struct rpc_clnt_config *config);
  * procedure handlers.
  */
 int rpcclnt_cbk_program_register (struct rpc_clnt *svc,
-                                  rpcclnt_cb_program_t *program);
+                                  rpcclnt_cb_program_t *program, void *mydata);
+
+void
+rpc_clnt_disable (struct rpc_clnt *rpc);
+
+char
+rpc_clnt_is_disabled (struct rpc_clnt *rpc);
 
-int
-rpc_clnt_transport_unix_options_build (dict_t **options, char *filepath);
 #endif /* !_RPC_CLNT_H */
diff --git a/rpc/rpc-lib/src/rpc-drc.c b/rpc/rpc-lib/src/rpc-drc.c
new file mode 100644
index 000000000..8181e6aee
--- /dev/null
+++ b/rpc/rpc-lib/src/rpc-drc.c
@@ -0,0 +1,872 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "rpcsvc.h"
+#ifndef RPC_DRC_H
+#include "rpc-drc.h"
+#endif
+#include "locking.h"
+#include "hashfn.h"
+#include "common-utils.h"
+#include "statedump.h"
+#include "mem-pool.h"
+
+#include <netinet/in.h>
+#include <unistd.h>
+
+/**
+ * rpcsvc_drc_op_destroy - Destroys the cached reply
+ *
+ * @param drc - the main drc structure
+ * @param reply - the cached reply to destroy
+ * @return NULL if reply is destroyed, reply otherwise
+ */
+static drc_cached_op_t *
+rpcsvc_drc_op_destroy (rpcsvc_drc_globals_t *drc, drc_cached_op_t *reply)
+{
+        GF_ASSERT (drc);
+        GF_ASSERT (reply);
+
+        if (reply->state == DRC_OP_IN_TRANSIT)
+                return reply;
+
+        iobref_unref (reply->msg.iobref);
+        if (reply->msg.rpchdr)
+                GF_FREE (reply->msg.rpchdr);
+        if (reply->msg.proghdr)
+                GF_FREE (reply->msg.proghdr);
+        if (reply->msg.progpayload)
+                GF_FREE (reply->msg.progpayload);
+
+        list_del (&reply->global_list);
+        reply->client->op_count--;
+        drc->op_count--;
+        mem_put (reply);
+        reply = NULL;
+
+        return reply;
+}
+
+/**
+ * rpcsvc_drc_op_rb_unref - This function is used in rb tree cleanup only
+ *
+ * @param reply - the cached reply to unref
+ * @param drc - the main drc structure
+ * @return void
+ */
+static void
+rpcsvc_drc_rb_op_destroy (void *reply, void *drc)
+{
+        rpcsvc_drc_op_destroy (drc, (drc_cached_op_t *)reply);
+}
+
+/**
+ * rpcsvc_remove_drc_client - Cleanup the drc client
+ *
+ * @param client - the drc client to be removed
+ * @return void
+ */
+static void
+rpcsvc_remove_drc_client (drc_client_t *client)
+{
+        rb_destroy (client->rbtree, rpcsvc_drc_rb_op_destroy);
+        list_del (&client->client_list);
+        GF_FREE (client);
+}
+
+/**
+ * rpcsvc_client_lookup - Given a sockaddr_storage, find the client if it exists
+ *
+ * @param drc - the main drc structure
+ * @param sockaddr - the network address of the client to be looked up
+ * @return drc client if it exists, NULL otherwise
+ */
+static drc_client_t *
+rpcsvc_client_lookup (rpcsvc_drc_globals_t *drc,
+                      struct sockaddr_storage *sockaddr)
+{
+        drc_client_t    *client = NULL;
+
+        GF_ASSERT (drc);
+        GF_ASSERT (sockaddr);
+
+        if (list_empty (&drc->clients_head))
+            return NULL;
+
+        list_for_each_entry (client, &drc->clients_head, client_list) {
+                if (gf_sock_union_equal_addr (&client->sock_union,
+                                              (union gf_sock_union *)sockaddr))
+                        return client;
+        }
+
+        return NULL;
+}
+
+/**
+ * drc_compare_reqs - Used by rbtree to determine if incoming req matches with
+ *                    an existing node(cached reply) in rbtree
+ *
+ * @param item - pointer to the incoming req
+ * @param rb_node_data - pointer to an rbtree node (cached reply)
+ * @param param - drc pointer - unused here, but used in *op_destroy
+ * @return 0 if req matches reply, else (req->xid - reply->xid)
+ */
+int
+drc_compare_reqs (const void *item, const void *rb_node_data, void *param)
+{
+        int               ret      = -1;
+        rpcsvc_request_t *req      = NULL;
+        drc_cached_op_t  *reply    = NULL;
+
+        GF_ASSERT (item);
+        GF_ASSERT (rb_node_data);
+        GF_ASSERT (param);
+
+        req = (rpcsvc_request_t *)item;
+        reply = (drc_cached_op_t *)rb_node_data;
+
+        ret = req->xid - reply->xid;
+        if (ret != 0)
+                return ret;
+
+        if (req->prognum == reply->prognum &&
+            req->procnum == reply->procnum &&
+            req->progver == reply->progversion)
+                return 0;
+
+        return 1;
+}
+
+/**
+ * drc_rb_calloc - used by rbtree api to allocate memory for nodes
+ *
+ * @param allocator - the libavl_allocator structure used by rbtree
+ * @param size - not needed by this function
+ * @return pointer to new cached reply (node in rbtree)
+ */
+static void *
+drc_rb_calloc (struct libavl_allocator *allocator, size_t size)
+{
+        rpcsvc_drc_globals_t *drc = NULL;
+
+        /* get the drc pointer by simple typecast, since allocator
+         * is the first member of rpcsvc_drc_globals_t
+         */
+        drc = (rpcsvc_drc_globals_t *)allocator;
+
+        return mem_get (drc->mempool);
+}
+
+/**
+ * drc_rb_free - used by rbtree api to free a node
+ *
+ * @param a - the libavl_allocator structure used by rbtree api
+ * @param block - node that needs to be freed
+ * @return void
+ */
+static void
+drc_rb_free (struct libavl_allocator *a, void *block)
+{
+        mem_put (block);
+}
+
+/**
+ * drc_init_client_cache - initialize a drc client and its rb tree
+ *
+ * @param drc - the main drc structure
+ * @param client - the drc client to be initialized
+ * @return 0 on success, -1 on failure
+ */
+static int
+drc_init_client_cache (rpcsvc_drc_globals_t *drc, drc_client_t *client)
+{
+        GF_ASSERT (drc);
+        GF_ASSERT (client);
+
+        drc->allocator.libavl_malloc = drc_rb_calloc;
+        drc->allocator.libavl_free = drc_rb_free;
+
+        client->rbtree = rb_create (drc_compare_reqs, drc,
+                                    (struct libavl_allocator *)drc);
+        if (!client->rbtree) {
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "rb tree creation failed");
+                return -1;
+        }
+
+        return 0;
+}
+
+/**
+ * rpcsvc_get_drc_client - find the drc client with given sockaddr, else
+ *                         allocate and initialize a new drc client
+ *
+ * @param drc - the main drc structure
+ * @param sockaddr - network address of client
+ * @return drc client on success, NULL on failure
+ */
+static drc_client_t *
+rpcsvc_get_drc_client (rpcsvc_drc_globals_t *drc,
+                       struct sockaddr_storage *sockaddr)
+{
+        drc_client_t      *client      = NULL;
+
+        GF_ASSERT (drc);
+        GF_ASSERT (sockaddr);
+
+        client = rpcsvc_client_lookup (drc, sockaddr);
+        if (client)
+                goto out;
+
+        /* if lookup fails, allocate cache for the new client */
+        client = GF_CALLOC (1, sizeof (drc_client_t),
+                            gf_common_mt_drc_client_t);
+        if (!client)
+                goto out;
+
+        client->ref = 0;
+        client->sock_union = (union gf_sock_union)*sockaddr;
+        client->op_count = 0;
+
+        if (drc_init_client_cache (drc, client)) {
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG,
+                        "initialization of drc client failed");
+                GF_FREE (client);
+                client = NULL;
+                goto out;
+        }
+        drc->client_count++;
+
+        list_add (&client->client_list, &drc->clients_head);
+
+ out:
+        return client;
+}
+
+/**
+ * rpcsvc_need_drc - Determine if a request needs DRC service
+ *
+ * @param req - incoming request
+ * @return 1 if DRC is needed for req, 0 otherwise
+ */
+int
+rpcsvc_need_drc (rpcsvc_request_t *req)
+{
+        rpcsvc_actor_t           *actor = NULL;
+        rpcsvc_drc_globals_t     *drc   = NULL;
+
+        GF_ASSERT (req);
+        GF_ASSERT (req->svc);
+
+        drc = req->svc->drc;
+
+        if (!drc || drc->status == DRC_UNINITIATED)
+                return 0;
+
+        actor = rpcsvc_program_actor (req);
+        if (!actor)
+                return 0;
+
+        return (actor->op_type == DRC_NON_IDEMPOTENT
+                && drc->type != DRC_TYPE_NONE);
+}
+
+/**
+ * rpcsvc_drc_client_ref - ref the drc client
+ *
+ * @param client - the drc client to ref
+ * @return client
+ */
+static drc_client_t *
+rpcsvc_drc_client_ref (drc_client_t *client)
+{
+        GF_ASSERT (client);
+        client->ref++;
+        return client;
+}
+
+/**
+ * rpcsvc_drc_client_unref - unref the drc client, and destroy
+ *                           the client on last unref
+ *
+ * @param drc - the main drc structure
+ * @param client - the drc client to unref
+ * @return NULL if it is the last unref, client otherwise
+ */
+static drc_client_t *
+rpcsvc_drc_client_unref (rpcsvc_drc_globals_t *drc, drc_client_t *client)
+{
+        GF_ASSERT (drc);
+        GF_ASSERT (client->ref);
+
+        client->ref--;
+        if (!client->ref) {
+                drc->client_count--;
+                rpcsvc_remove_drc_client (client);
+                client = NULL;
+        }
+
+        return client;
+}
+
+/**
+ * rpcsvc_drc_lookup - lookup a request to see if it is already cached
+ *
+ * @param req - incoming request
+ * @return cached reply of req if found, NULL otherwise
+ */
+drc_cached_op_t *
+rpcsvc_drc_lookup (rpcsvc_request_t *req)
+{
+        drc_client_t           *client = NULL;
+        drc_cached_op_t        *reply  = NULL;
+
+        GF_ASSERT (req);
+
+        if (!req->trans->drc_client) {
+                client = rpcsvc_get_drc_client (req->svc->drc,
+                                                &req->trans->peerinfo.sockaddr);
+                if (!client)
+                        goto out;
+                req->trans->drc_client = client;
+        }
+
+        client = rpcsvc_drc_client_ref (req->trans->drc_client);
+
+        if (client->op_count == 0)
+                goto out;
+
+        reply = rb_find (client->rbtree, req);
+
+ out:
+        if (client)
+                rpcsvc_drc_client_unref (req->svc->drc, client);
+
+        return reply;
+}
+
+/**
+ * rpcsvc_send_cached_reply - send the cached reply for the incoming request
+ *
+ * @param req - incoming request (which is a duplicate in this case)
+ * @param reply - the cached reply for req
+ * @return 0 on successful reply submission, -1 or other non-zero value otherwise
+ */
+int
+rpcsvc_send_cached_reply (rpcsvc_request_t *req, drc_cached_op_t *reply)
+{
+        int     ret = 0;
+
+        GF_ASSERT (req);
+        GF_ASSERT (reply);
+
+        gf_log (GF_RPCSVC, GF_LOG_DEBUG, "sending cached reply: xid: %d, "
+                "client: %s", req->xid, req->trans->peerinfo.identifier);
+
+        rpcsvc_drc_client_ref (reply->client);
+        ret = rpcsvc_transport_submit (req->trans,
+                     reply->msg.rpchdr, reply->msg.rpchdrcount,
+                     reply->msg.proghdr, reply->msg.proghdrcount,
+                     reply->msg.progpayload, reply->msg.progpayloadcount,
+                     reply->msg.iobref, req->trans_private);
+        rpcsvc_drc_client_unref (req->svc->drc, reply->client);
+
+        return ret;
+}
+
+/**
+ * rpcsvc_cache_reply - cache the reply for the processed request 'req'
+ *
+ * @param req - processed request
+ * @param iobref - iobref structure of the reply
+ * @param rpchdr - rpc header of the reply
+ * @param rpchdrcount - size of rpchdr
+ * @param proghdr - program header of the reply
+ * @param proghdrcount - size of proghdr
+ * @param payload - payload of the reply if any
+ * @param payloadcount - size of payload
+ * @return 0 on success, -1 on failure
+ */
+int
+rpcsvc_cache_reply (rpcsvc_request_t *req, struct iobref *iobref,
+                    struct iovec *rpchdr, int rpchdrcount,
+                    struct iovec *proghdr, int proghdrcount,
+                    struct iovec *payload, int payloadcount)
+{
+        int                       ret              = -1;
+        drc_cached_op_t          *reply            = NULL;
+
+        GF_ASSERT (req);
+        GF_ASSERT (req->reply);
+
+        reply = req->reply;
+
+        reply->state = DRC_OP_CACHED;
+
+        reply->msg.iobref = iobref_ref (iobref);
+
+        reply->msg.rpchdrcount = rpchdrcount;
+        reply->msg.rpchdr = iov_dup (rpchdr, rpchdrcount);
+
+        reply->msg.proghdrcount = proghdrcount;
+        reply->msg.proghdr = iov_dup (proghdr, proghdrcount);
+
+        reply->msg.progpayloadcount = payloadcount;
+        if (payloadcount)
+                reply->msg.progpayload = iov_dup (payload, payloadcount);
+
+        //        rpcsvc_drc_client_unref (req->svc->drc, req->trans->drc_client);
+        //        rpcsvc_drc_op_unref (req->svc->drc, reply);
+        ret = 0;
+
+        return ret;
+}
+
+/**
+ * rpcsvc_vacate_drc_entries - free up some percentage of drc cache
+ *                             based on the lru factor
+ *
+ * @param drc - the main drc structure
+ * @return void
+ */
+static void
+rpcsvc_vacate_drc_entries (rpcsvc_drc_globals_t *drc)
+{
+        uint32_t            i           = 0;
+        uint32_t            n           = 0;
+        drc_cached_op_t    *reply       = NULL;
+        drc_cached_op_t    *tmp         = NULL;
+        drc_client_t       *client      = NULL;
+
+        GF_ASSERT (drc);
+
+        n = drc->global_cache_size / drc->lru_factor;
+
+        list_for_each_entry_safe_reverse (reply, tmp, &drc->cache_head, global_list) {
+                /* Don't delete ops that are in transit */
+                if (reply->state == DRC_OP_IN_TRANSIT)
+                        continue;
+
+                client = reply->client;
+
+                (void *)rb_delete (client->rbtree, reply);
+
+                rpcsvc_drc_op_destroy (drc, reply);
+                rpcsvc_drc_client_unref (drc, client);
+                i++;
+                if (i >= n)
+                        break;
+        }
+}
+
+/**
+ * rpcsvc_add_op_to_cache - insert the cached op into the client rbtree and drc list
+ *
+ * @param drc - the main drc structure
+ * @param reply - the op to be inserted
+ * @return 0 on success, -1 on failure
+ */
+static int
+rpcsvc_add_op_to_cache (rpcsvc_drc_globals_t *drc, drc_cached_op_t *reply)
+{
+        drc_client_t        *client         = NULL;
+        drc_cached_op_t    **tmp_reply      = NULL;
+
+        GF_ASSERT (drc);
+        GF_ASSERT (reply);
+
+        client = reply->client;
+
+        /* cache is full, free up some space */
+        if (drc->op_count >= drc->global_cache_size)
+                rpcsvc_vacate_drc_entries (drc);
+
+        tmp_reply = (drc_cached_op_t **)rb_probe (client->rbtree, reply);
+        if (*tmp_reply != reply) {
+                /* should never happen */
+                gf_log (GF_RPCSVC, GF_LOG_ERROR,
+                        "DRC failed to detect duplicates");
+                return -1;
+        } else if (*tmp_reply == NULL) {
+                /* mem alloc failed */
+                return -1;
+        }
+
+        client->op_count++;
+        list_add (&reply->global_list, &drc->cache_head);
+        drc->op_count++;
+
+        return 0;
+}
+
+/**
+ * rpcsvc_cache_request - cache the in-transition incoming request
+ *
+ * @param req - incoming request
+ * @return 0 on success, -1 on failure
+ */
+int
+rpcsvc_cache_request (rpcsvc_request_t *req)
+{
+        int                        ret            = -1;
+        drc_client_t              *client         = NULL;
+        drc_cached_op_t           *reply          = NULL;
+        rpcsvc_drc_globals_t      *drc            = NULL;
+
+        GF_ASSERT (req);
+
+        drc = req->svc->drc;
+
+        client = req->trans->drc_client;
+        if (!client) {
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "drc client is NULL");
+                goto out;
+        }
+
+        reply = mem_get (drc->mempool);
+        if (!reply)
+                goto out;
+
+        reply->client = rpcsvc_drc_client_ref (client);
+        reply->xid = req->xid;
+        reply->prognum = req->prognum;
+        reply->progversion = req->progver;
+        reply->procnum = req->procnum;
+        reply->state = DRC_OP_IN_TRANSIT;
+        req->reply = reply;
+
+        ret = rpcsvc_add_op_to_cache (drc, reply);
+        if (ret) {
+                req->reply = NULL;
+                rpcsvc_drc_op_destroy (drc, reply);
+                rpcsvc_drc_client_unref (drc, client);
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Failed to add op to drc cache");
+        }
+
+ out:
+        return ret;
+}
+
+/**
+ *
+ * rpcsvc_drc_priv - function which dumps the drc state
+ *
+ * @param drc - the main drc structure
+ * @return 0 on success, -1 on failure
+ */
+int32_t
+rpcsvc_drc_priv (rpcsvc_drc_globals_t *drc)
+{
+        int                      i                         = 0;
+        char                     key[GF_DUMP_MAX_BUF_LEN]  = {0};
+        drc_client_t            *client                    = NULL;
+        char                     ip[INET6_ADDRSTRLEN]      = {0};
+
+        if (!drc || drc->status == DRC_UNINITIATED) {
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "DRC is "
+                        "uninitialized, not dumping its state");
+                return 0;
+        }
+
+        gf_proc_dump_add_section("rpc.drc");
+
+        if (TRY_LOCK (&drc->lock))
+                return -1;
+
+        gf_proc_dump_build_key (key, "drc", "type");
+        gf_proc_dump_write (key, "%d", drc->type);
+
+        gf_proc_dump_build_key (key, "drc", "client_count");
+        gf_proc_dump_write (key, "%d", drc->client_count);
+
+        gf_proc_dump_build_key (key, "drc", "current_cache_size");
+        gf_proc_dump_write (key, "%d", drc->op_count);
+
+        gf_proc_dump_build_key (key, "drc", "max_cache_size");
+        gf_proc_dump_write (key, "%d", drc->global_cache_size);
+
+        gf_proc_dump_build_key (key, "drc", "lru_factor");
+        gf_proc_dump_write (key, "%d", drc->lru_factor);
+
+        gf_proc_dump_build_key (key, "drc", "duplicate_request_count");
+        gf_proc_dump_write (key, "%d", drc->cache_hits);
+
+        gf_proc_dump_build_key (key, "drc", "in_transit_duplicate_requests");
+        gf_proc_dump_write (key, "%d", drc->intransit_hits);
+
+        list_for_each_entry (client, &drc->clients_head, client_list) {
+                gf_proc_dump_build_key (key, "client", "%d.ip-address", i);
+                memset (ip, 0, INET6_ADDRSTRLEN);
+                switch (client->sock_union.storage.ss_family) {
+                case AF_INET:
+                        gf_proc_dump_write (key, "%s", inet_ntop (AF_INET,
+                                &client->sock_union.sin.sin_addr.s_addr,
+                                ip, INET_ADDRSTRLEN));
+                        break;
+                case AF_INET6:
+                        gf_proc_dump_write (key, "%s", inet_ntop (AF_INET6,
+                                &client->sock_union.sin6.sin6_addr,
+                                ip, INET6_ADDRSTRLEN));
+                        break;
+                default:
+                        gf_proc_dump_write (key, "%s", "N/A");
+                }
+
+                gf_proc_dump_build_key (key, "client", "%d.ref_count", i);
+                gf_proc_dump_write (key, "%d", client->ref);
+                gf_proc_dump_build_key (key, "client", "%d.op_count", i);
+                gf_proc_dump_write (key, "%d", client->op_count);
+                i++;
+        }
+
+        UNLOCK (&drc->lock);
+        return 0;
+}
+
+/**
+ * rpcsvc_drc_notify - function which is notified of RPC transport events
+ *
+ * @param svc - pointer to rpcsvc_t structure of the rpc
+ * @param xl - pointer to the xlator
+ * @param event - the event which triggered this notify
+ * @param data - the transport structure
+ * @return 0 on success, -1 on failure
+ */
+int
+rpcsvc_drc_notify (rpcsvc_t *svc, void *xl,
+                   rpcsvc_event_t event, void *data)
+{
+        int                       ret          = -1;
+        rpc_transport_t          *trans        = NULL;
+        drc_client_t             *client       = NULL;
+        rpcsvc_drc_globals_t     *drc          = NULL;
+
+        GF_ASSERT (svc);
+        GF_ASSERT (svc->drc);
+        GF_ASSERT (data);
+
+        drc = svc->drc;
+
+        if (drc->status == DRC_UNINITIATED ||
+            drc->type == DRC_TYPE_NONE)
+                return 0;
+
+        LOCK (&drc->lock);
+
+        trans = (rpc_transport_t *)data;
+        client = rpcsvc_get_drc_client (drc, &trans->peerinfo.sockaddr);
+        if (!client)
+                goto out;
+
+        switch (event) {
+        case RPCSVC_EVENT_ACCEPT:
+                trans->drc_client = rpcsvc_drc_client_ref (client);
+                ret = 0;
+                break;
+
+        case RPCSVC_EVENT_DISCONNECT:
+                ret = 0;
+                if (list_empty (&drc->clients_head))
+                        break;
+                /* should be the last unref */
+                rpcsvc_drc_client_unref (drc, client);
+                trans->drc_client = NULL;
+                break;
+
+        default:
+                break;
+        }
+
+ out:
+        UNLOCK (&drc->lock);
+        return ret;
+}
+
+/**
+ * rpcsvc_drc_init - Initialize the duplicate request cache service
+ *
+ * @param svc - pointer to rpcsvc_t structure of the rpc
+ * @param options - the options dictionary which configures drc
+ * @return 0 on success, non-zero integer on failure
+ */
+int
+rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options)
+{
+        int                         ret            = 0;
+        uint32_t                    drc_type       = 0;
+        uint32_t                    drc_size       = 0;
+        uint32_t                    drc_factor     = 0;
+        rpcsvc_drc_globals_t       *drc            = NULL;
+        static gf_boolean_t         drc_inited     = _gf_false;
+
+        GF_ASSERT (svc);
+        GF_ASSERT (options);
+
+        /* Already inited */
+        if (drc_inited)
+                return 0;
+
+        if (!svc->drc) {
+                drc = GF_CALLOC (1, sizeof (rpcsvc_drc_globals_t),
+                                 gf_common_mt_drc_globals_t);
+                if (!drc)
+                        return -1;
+
+                svc->drc = drc;
+                LOCK_INIT (&drc->lock);
+        } else {
+                drc = svc->drc;
+        }
+
+        LOCK (&drc->lock);
+        if (drc->type != DRC_TYPE_NONE) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Toggle DRC on/off, when more drc types(persistent/cluster)
+           are added, we shouldn't treat this as boolean */
+        ret = dict_get_str_boolean (options, "nfs.drc", _gf_true);
+        if (ret == -1) {
+                gf_log (GF_RPCSVC, GF_LOG_INFO, "drc user options need second look");
+                ret = _gf_true;
+        }
+        drc->enable_drc = ret;
+
+        if (ret == _gf_false) {
+                /* drc off */
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "DRC is off");
+                ret = 0;
+                goto out;
+        }
+
+        /* Specify type of DRC to be used */
+        ret = dict_get_uint32 (options, "nfs.drc-type", &drc_type);
+        if (ret) {
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "drc type not set."
+                        " Continuing with default");
+                drc_type = DRC_DEFAULT_TYPE;
+        }
+
+        drc->type = drc_type;
+
+        /* Set the global cache size (no. of ops to cache) */
+        ret = dict_get_uint32 (options, "nfs.drc-size", &drc_size);
+        if (ret) {
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "drc size not set."
+                        " Continuing with default size");
+                drc_size = DRC_DEFAULT_CACHE_SIZE;
+        }
+
+        drc->global_cache_size = drc_size;
+
+        /* Mempool for cached ops */
+        drc->mempool = mem_pool_new (drc_cached_op_t, drc->global_cache_size);
+        if (!drc->mempool) {
+                gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to get mempool for"
+                        " DRC, drc-size: %d", drc->global_cache_size);
+                ret = -1;
+                goto out;
+        }
+
+        /* What percent of cache to be evicted whenever it fills up */
+        ret = dict_get_uint32 (options, "nfs.drc-lru-factor", &drc_factor);
+        if (ret) {
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "drc lru factor not set."
+                        " Continuing with policy default");
+                drc_factor = DRC_DEFAULT_LRU_FACTOR;
+        }
+
+        drc->lru_factor = (drc_lru_factor_t) drc_factor;
+
+        INIT_LIST_HEAD (&drc->clients_head);
+        INIT_LIST_HEAD (&drc->cache_head);
+
+        ret = rpcsvc_register_notify (svc, rpcsvc_drc_notify, THIS);
+        if (ret) {
+                gf_log (GF_RPCSVC, GF_LOG_ERROR,
+                        "registration of drc_notify function failed");
+                goto out;
+        }
+
+        gf_log (GF_RPCSVC, GF_LOG_DEBUG, "drc init successful");
+        drc->status = DRC_INITIATED;
+        drc_inited = _gf_true;
+
+ out:
+        UNLOCK (&drc->lock);
+        if (ret == -1) {
+                if (drc->mempool) {
+                        mem_pool_destroy (drc->mempool);
+                        drc->mempool = NULL;
+                }
+                GF_FREE (drc);
+                svc->drc = NULL;
+        }
+        return ret;
+}
+
+int
+rpcsvc_drc_reconfigure (rpcsvc_t *svc, dict_t *options)
+{
+        int                     ret        = -1;
+        gf_boolean_t            enable_drc = _gf_false;
+        rpcsvc_drc_globals_t    *drc       = NULL;
+        uint32_t                drc_size   = 0;
+
+        if ((!svc) || (!options))
+                return (-1);
+
+        drc = svc->drc;
+        /* reconfig for drc-size */
+        if (dict_get_uint32 (options, "nfs.drc-size", &drc_size))
+                drc_size = DRC_DEFAULT_CACHE_SIZE;
+
+        if (drc->global_cache_size != drc_size) {
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "nfs.drc-size size can not "
+                        "be reconfigured without NFS server restart.");
+                return (-1);
+        }
+
+        /* reconfig for nfs.drc */
+        ret = dict_get_str_boolean (options, "nfs.drc", _gf_true);
+        if (ret < 0) {
+                ret = _gf_true;
+        }
+        enable_drc = ret;
+
+        if (drc->enable_drc == enable_drc)
+                return 0;
+
+        drc->enable_drc = enable_drc;
+        if (enable_drc) {
+                if (drc == NULL)
+                        return rpcsvc_drc_init(svc, options);
+        } else {
+                if (drc == NULL)
+                        return (0);
+
+                LOCK (&drc->lock);
+                (void) rpcsvc_unregister_notify (svc, rpcsvc_drc_notify, THIS);
+                if (drc->mempool) {
+                        mem_pool_destroy (drc->mempool);
+                        drc->mempool = NULL;
+                }
+                UNLOCK (&drc->lock);
+                GF_FREE (drc);
+                svc->drc = NULL;
+        }
+
+        return (0);
+}
diff --git a/rpc/rpc-lib/src/rpc-drc.h b/rpc/rpc-lib/src/rpc-drc.h
new file mode 100644
index 000000000..7dfaef978
--- /dev/null
+++ b/rpc/rpc-lib/src/rpc-drc.h
@@ -0,0 +1,104 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef RPC_DRC_H
+#define RPC_DRC_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "rpcsvc-common.h"
+#include "rpcsvc.h"
+#include "locking.h"
+#include "dict.h"
+#include "rb.h"
+
+/* per-client cache structure */
+struct drc_client {
+        uint32_t                   ref;
+        union gf_sock_union        sock_union;
+        /* pointers to the cache */
+        struct rb_table           *rbtree;
+        /* no. of ops currently cached */
+        uint32_t                   op_count;
+        struct list_head           client_list;
+};
+
+struct drc_cached_op {
+        drc_op_state_t                 state;
+        uint32_t                       xid;
+        int                            prognum;
+        int                            progversion;
+        int                            procnum;
+        rpc_transport_msg_t            msg;
+        drc_client_t                  *client;
+        struct list_head               client_list;
+        struct list_head               global_list;
+        int32_t                        ref;
+};
+
+/* global drc definitions */
+enum drc_status {
+        DRC_UNINITIATED,
+        DRC_INITIATED
+};
+typedef enum drc_status drc_status_t;
+
+struct drc_globals {
+        /* allocator must be the first member since
+         * it is used so in gf_libavl_allocator
+         */
+        struct libavl_allocator   allocator;
+        drc_type_t                type;
+        /* configurable size parameter */
+        uint32_t                  global_cache_size;
+        drc_lru_factor_t          lru_factor;
+        gf_lock_t                 lock;
+        drc_status_t              status;
+        uint32_t                  op_count;
+        uint64_t                  cache_hits;
+        uint64_t                  intransit_hits;
+        struct mem_pool          *mempool;
+        struct list_head          cache_head;
+        uint32_t                  client_count;
+        struct list_head          clients_head;
+        gf_boolean_t              enable_drc;
+};
+
+int
+rpcsvc_need_drc (rpcsvc_request_t *req);
+
+drc_cached_op_t *
+rpcsvc_drc_lookup (rpcsvc_request_t *req);
+
+int
+rpcsvc_send_cached_reply (rpcsvc_request_t *req, drc_cached_op_t *reply);
+
+int
+rpcsvc_cache_reply (rpcsvc_request_t *req, struct iobref *iobref,
+                    struct iovec *rpchdr, int rpchdrcount,
+                    struct iovec *proghdr, int proghdrcount,
+                    struct iovec *payload, int payloadcount);
+
+int
+rpcsvc_cache_request (rpcsvc_request_t *req);
+
+int32_t
+rpcsvc_drc_priv (rpcsvc_drc_globals_t *drc);
+
+int
+rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options);
+
+int
+rpcsvc_drc_reconfigure (rpcsvc_t *svc, dict_t *options);
+
+#endif /* RPC_DRC_H */
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index b97ba61bf..c24d41084 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <dlfcn.h>
@@ -78,6 +69,19 @@ out:
         return ret;
 }
 
+int
+rpc_transport_throttle (rpc_transport_t *this, gf_boolean_t onoff)
+{
+        int ret = 0;
+
+        if (!this->ops->throttle)
+                return -ENOSYS;
+
+        ret = this->ops->throttle (this, onoff);
+
+        return ret;
+}
+
 int32_t
 rpc_transport_get_peeraddr (rpc_transport_t *this, char *peeraddr, int addrlen,
                             struct sockaddr_storage *sa, size_t salen)
@@ -154,6 +158,7 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
 	int8_t is_tcp = 0, is_unix = 0, is_ibsdp = 0;
 	volume_opt_list_t *vol_opt = NULL;
         gf_boolean_t bind_insecure = _gf_false;
+        xlator_t   *this = NULL;
 
 	GF_VALIDATE_OR_GOTO("rpc-transport", options, fail);
 	GF_VALIDATE_OR_GOTO("rpc-transport", ctx, fail);
@@ -178,7 +183,7 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
 			gf_log ("dict", GF_LOG_DEBUG,
 				"setting transport-type failed");
                 else
-                        gf_log ("rpc-transport", GF_LOG_WARNING,
+                        gf_log ("rpc-transport", GF_LOG_DEBUG,
                                 "missing 'option transport-type'. defaulting to "
                                 "\"socket\"");
 	} else {
@@ -259,13 +264,15 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
 	handle = dlopen (name, RTLD_NOW|RTLD_GLOBAL);
 	if (handle == NULL) {
 		gf_log ("rpc-transport", GF_LOG_ERROR, "%s", dlerror ());
-		gf_log ("rpc-transport", GF_LOG_ERROR,
+		gf_log ("rpc-transport", GF_LOG_WARNING,
 			"volume '%s': transport-type '%s' is not valid or "
 			"not found on this machine",
 			trans_name, type);
 		goto fail;
 	}
 
+        trans->dl_handle = handle;
+
 	trans->ops = dlsym (handle, "tops");
 	if (trans->ops == NULL) {
 		gf_log ("rpc-transport", GF_LOG_ERROR,
@@ -273,22 +280,22 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
 		goto fail;
 	}
 
-	trans->init = dlsym (handle, "init");
+	*VOID(&(trans->init)) = dlsym (handle, "init");
 	if (trans->init == NULL) {
 		gf_log ("rpc-transport", GF_LOG_ERROR,
 			"dlsym (gf_rpc_transport_init) on %s", dlerror ());
 		goto fail;
 	}
 
-	trans->fini = dlsym (handle, "fini");
+	*VOID(&(trans->fini)) = dlsym (handle, "fini");
 	if (trans->fini == NULL) {
 		gf_log ("rpc-transport", GF_LOG_ERROR,
 			"dlsym (gf_rpc_transport_fini) on %s", dlerror ());
 		goto fail;
 	}
 
-        trans->reconfigure = dlsym (handle, "reconfigure");
-        if (trans->fini == NULL) {
+        *VOID(&(trans->reconfigure)) = dlsym (handle, "reconfigure");
+        if (trans->reconfigure == NULL) {
                 gf_log ("rpc-transport", GF_LOG_DEBUG,
                         "dlsym (gf_rpc_transport_reconfigure) on %s", dlerror());
         }
@@ -299,14 +306,15 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
                 goto fail;
         }
 
+        this = THIS;
 	vol_opt->given_opt = dlsym (handle, "options");
 	if (vol_opt->given_opt == NULL) {
 		gf_log ("rpc-transport", GF_LOG_DEBUG,
 			"volume option validation not specified");
 	} else {
                 INIT_LIST_HEAD (&vol_opt->list);
-		list_add_tail (&vol_opt->list, &(THIS->volume_options));
-                if (xlator_options_validate_list (THIS, options, vol_opt,
+		list_add_tail (&vol_opt->list, &(this->volume_options));
+                if (xlator_options_validate_list (this, options, vol_opt,
                                                   NULL)) {
 			gf_log ("rpc-transport", GF_LOG_ERROR,
 				"volume option validation failed");
@@ -317,7 +325,7 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
         trans->options = options;
 
         pthread_mutex_init (&trans->lock, NULL);
-        trans->xl = THIS;
+        trans->xl = this;
 
 	ret = trans->init (trans);
 	if (ret != 0) {
@@ -326,25 +334,27 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
 		goto fail;
 	}
 
-	return_trans = trans;
+        return_trans = trans;
 
-        if (name) {
-                GF_FREE (name);
-        }
+        GF_FREE (name);
 
 	return return_trans;
 
 fail:
         if (trans) {
-                if (trans->name) {
-                        GF_FREE (trans->name);
-                }
+                GF_FREE (trans->name);
+
+                if (trans->dl_handle)
+                        dlclose (trans->dl_handle);
 
                 GF_FREE (trans);
         }
 
-        if (name) {
-                GF_FREE (name);
+        GF_FREE (name);
+
+        if (vol_opt && !list_empty (&vol_opt->list)) {
+                list_del_init (&vol_opt->list);
+                GF_FREE (vol_opt);
         }
 
         return NULL;
@@ -432,8 +442,10 @@ rpc_transport_destroy (rpc_transport_t *this)
 
 	pthread_mutex_destroy (&this->lock);
 
-        if (this->name)
-                GF_FREE (this->name);
+        GF_FREE (this->name);
+
+        if (this->dl_handle)
+                dlclose (this->dl_handle);
 
 	GF_FREE (this);
 fail:
@@ -470,7 +482,7 @@ rpc_transport_unref (rpc_transport_t *this)
 
 	pthread_mutex_lock (&this->lock);
 	{
-		refcount = --this->refcount;
+                refcount = --this->refcount;
 	}
 	pthread_mutex_unlock (&this->lock);
 
@@ -478,7 +490,9 @@ rpc_transport_unref (rpc_transport_t *this)
                 if (this->mydata)
                         this->notify (this, this->mydata, RPC_TRANSPORT_CLEANUP,
                                       NULL);
-		rpc_transport_destroy (this);
+                this->mydata = NULL;
+                this->notify = NULL;
+                rpc_transport_destroy (this);
 	}
 
 	ret = 0;
@@ -520,6 +534,8 @@ out:
         return ret;
 }
 
+
+
 //give negative values to skip setting that value
 //this function asserts if both the values are negative.
 //why call it if you dont set it.
@@ -546,6 +562,63 @@ out:
 }
 
 int
+rpc_transport_unix_options_build (dict_t **options, char *filepath,
+                                  int frame_timeout)
+{
+        dict_t                  *dict = NULL;
+        char                    *fpath = NULL;
+        int                     ret = -1;
+
+        GF_ASSERT (filepath);
+        GF_ASSERT (options);
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        fpath = gf_strdup (filepath);
+        if (!fpath) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dict, "transport.socket.connect-path", fpath);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dict, "transport.address-family", "unix");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dict, "transport.socket.nodelay", "off");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dict, "transport-type", "socket");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dict, "transport.socket.keepalive", "off");
+        if (ret)
+                goto out;
+
+        if (frame_timeout > 0) {
+                ret = dict_set_int32 (dict, "frame-timeout", frame_timeout);
+                if (ret)
+                        goto out;
+        }
+
+        *options = dict;
+out:
+        if (ret) {
+                GF_FREE (fpath);
+                if (dict)
+                        dict_unref (dict);
+        }
+        return ret;
+}
+
+int
 rpc_transport_inet_options_build (dict_t **options, const char *hostname,
                                   int port)
 {
@@ -580,7 +653,7 @@ rpc_transport_inet_options_build (dict_t **options, const char *hostname,
                         "failed to set remote-port with %d", port);
                 goto out;
         }
-        ret = dict_set_str (dict, "transport.address-family", "inet/inet6");
+        ret = dict_set_str (dict, "transport.address-family", "inet");
         if (ret) {
                 gf_log (THIS->name, GF_LOG_WARNING,
                         "failed to set addr-family with inet");
@@ -597,8 +670,7 @@ rpc_transport_inet_options_build (dict_t **options, const char *hostname,
         *options = dict;
 out:
         if (ret) {
-                if (host)
-                        GF_FREE (host);
+                GF_FREE (host);
                 if (dict)
                         dict_unref (dict);
         }
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index f838d4baf..2db9072ae 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __RPC_TRANSPORT_H__
@@ -80,6 +71,11 @@ struct peer_info {
         struct sockaddr_storage sockaddr;
         socklen_t sockaddr_len;
         char identifier[UNIX_PATH_MAX];
+        // OP-VERSION of clients
+        uint32_t max_op_version;
+        uint32_t min_op_version;
+        //Volume mounted by client
+        char volname[1024];
 };
 typedef struct peer_info peer_info_t;
 
@@ -190,16 +186,19 @@ struct rpc_transport {
                                               */
 
         void                      *private;
-        void                      *xl_private;
+        struct _client_t          *xl_private;
         void                      *xl;       /* Used for THIS */
         void                      *mydata;
         pthread_mutex_t            lock;
         int32_t                    refcount;
 
+        int32_t                    outstanding_rpc_count;
+
         glusterfs_ctx_t           *ctx;
         dict_t                    *options;
         char                      *name;
         void                      *dnscache;
+        void                      *drc_client;
         data_t                    *buf;
         int32_t                  (*init)   (rpc_transport_t *this);
         void                     (*fini)   (rpc_transport_t *this);
@@ -214,6 +213,7 @@ struct rpc_transport {
 
         struct list_head           list;
         int                        bind_insecure;
+        void                      *dl_handle; /* handle of dlopen() */
 };
 
 struct rpc_transport_ops {
@@ -237,6 +237,7 @@ struct rpc_transport_ops {
         int32_t (*get_myaddr)     (rpc_transport_t *this, char *peeraddr,
                                    int addrlen, struct sockaddr_storage *sa,
                                    socklen_t sasize);
+        int32_t (*throttle)       (rpc_transport_t *this, gf_boolean_t onoff);
 };
 
 
@@ -290,6 +291,9 @@ int32_t
 rpc_transport_get_myaddr (rpc_transport_t *this, char *peeraddr, int addrlen,
                           struct sockaddr_storage *sa, size_t salen);
 
+int
+rpc_transport_throttle (rpc_transport_t *this, gf_boolean_t onoff);
+
 rpc_transport_pollin_t *
 rpc_transport_pollin_alloc (rpc_transport_t *this, struct iovec *vector,
                             int count, struct iobuf *hdr_iobuf,
@@ -302,5 +306,9 @@ rpc_transport_keepalive_options_set (dict_t *options, int32_t interval,
                                      int32_t time);
 
 int
+rpc_transport_unix_options_build (dict_t **options, char *filepath,
+                                  int frame_timeout);
+
+int
 rpc_transport_inet_options_build (dict_t **options, const char *hostname, int port);
 #endif /* __RPC_TRANSPORT_H__ */
diff --git a/rpc/rpc-lib/src/rpcsvc-auth.c b/rpc/rpc-lib/src/rpcsvc-auth.c
index 930eabdda..4cb86a758 100644
--- a/rpc/rpc-lib/src/rpcsvc-auth.c
+++ b/rpc/rpc-lib/src/rpcsvc-auth.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "rpcsvc.h"
@@ -187,6 +178,29 @@ err:
 }
 
 int
+rpcsvc_set_addr_namelookup (rpcsvc_t *svc, dict_t *options)
+{
+        int             ret;
+        static char     *addrlookup_key = "rpc-auth.addr.namelookup";
+
+        if (!svc || !options)
+                return (-1);
+
+        /* By default it's disabled */
+        ret = dict_get_str_boolean (options, addrlookup_key, _gf_false);
+        if (ret < 0) {
+                svc->addr_namelookup = _gf_false;
+        } else {
+                svc->addr_namelookup = ret;
+        }
+
+        if (svc->addr_namelookup)
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Addr-Name lookup enabled");
+
+        return (0);
+}
+
+int
 rpcsvc_set_allow_insecure (rpcsvc_t *svc, dict_t *options)
 {
         int             ret = -1;
@@ -213,6 +227,26 @@ rpcsvc_set_allow_insecure (rpcsvc_t *svc, dict_t *options)
 }
 
 int
+rpcsvc_set_root_squash (rpcsvc_t *svc, dict_t *options)
+{
+        int  ret = -1;
+
+        GF_ASSERT (svc);
+        GF_ASSERT (options);
+
+        ret = dict_get_str_boolean (options, "root-squash", 0);
+        if (ret != -1)
+                svc->root_squash = ret;
+        else
+                svc->root_squash = _gf_false;
+
+        if (svc->root_squash)
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "root squashing enabled ");
+
+        return 0;
+}
+
+int
 rpcsvc_auth_init (rpcsvc_t *svc, dict_t *options)
 {
         int             ret = -1;
@@ -221,6 +255,8 @@ rpcsvc_auth_init (rpcsvc_t *svc, dict_t *options)
                 return -1;
 
         (void) rpcsvc_set_allow_insecure (svc, options);
+        (void) rpcsvc_set_root_squash (svc, options);
+        (void) rpcsvc_set_addr_namelookup (svc, options);
         ret = rpcsvc_auth_add_initers (svc);
         if (ret == -1) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to add initers");
@@ -237,6 +273,25 @@ out:
         return ret;
 }
 
+int
+rpcsvc_auth_reconf (rpcsvc_t *svc, dict_t *options)
+{
+        int ret = 0;
+
+        if ((!svc) || (!options))
+                return (-1);
+
+        ret = rpcsvc_set_allow_insecure (svc, options);
+        if (ret)
+                return (-1);
+
+        ret = rpcsvc_set_root_squash (svc, options);
+        if (ret)
+                return (-1);
+
+        return rpcsvc_set_addr_namelookup (svc, options);
+}
+
 
 rpcsvc_auth_t *
 __rpcsvc_auth_get_handler (rpcsvc_request_t *req)
@@ -315,6 +370,9 @@ rpcsvc_auth_request_init (rpcsvc_request_t *req)
         if (!auth->authops->request_init)
                 ret = auth->authops->request_init (req, auth->authprivate);
 
+	req->auxgids = req->auxgidsmall; /* reset to auxgidlarge during
+					    unsersialize if necessary */
+	req->auxgidlarge = NULL;
 err:
         return ret;
 }
@@ -354,14 +412,10 @@ err:
 int
 rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen)
 {
-        int             count = 0;
-        int             gen = RPCSVC_AUTH_REJECT;
-        int             spec = RPCSVC_AUTH_REJECT;
-        int             final = RPCSVC_AUTH_REJECT;
-        char            *srchstr = NULL;
-        char            *valstr = NULL;
-        gf_boolean_t    boolval = _gf_false;
-        int             ret = 0;
+        int             count      = 0;
+        int             result     = RPCSVC_AUTH_REJECT;
+        char           *srchstr    = NULL;
+        int             ret        = 0;
 
         struct rpcsvc_auth_list *auth = NULL;
         struct rpcsvc_auth_list *tmp = NULL;
@@ -379,59 +433,27 @@ rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen)
                 if (count >= arrlen)
                         break;
 
-                gen = gf_asprintf (&srchstr, "rpc-auth.%s", auth->name);
-                if (gen == -1) {
+                result = gf_asprintf (&srchstr, "rpc-auth.%s.%s",
+                                      auth->name, volname);
+                if (result == -1) {
                         count = -1;
                         goto err;
                 }
 
-                gen = RPCSVC_AUTH_REJECT;
-                if (dict_get (svc->options, srchstr)) {
-                        ret = dict_get_str (svc->options, srchstr, &valstr);
-                        if (ret == 0) {
-                                ret = gf_string2boolean (valstr, &boolval);
-                                if (ret == 0) {
-                                        if (boolval == _gf_true)
-                                                gen = RPCSVC_AUTH_ACCEPT;
-                                } else
-                                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "Faile"
-                                                "d to read auth val");
-                        } else
-                                gf_log (GF_RPCSVC, GF_LOG_ERROR, "Faile"
-                                        "d to read auth val");
-                }
-
+                ret = dict_get_str_boolean (svc->options, srchstr, 0xC00FFEE);
                 GF_FREE (srchstr);
-                spec = gf_asprintf (&srchstr, "rpc-auth.%s.%s", auth->name,
-                                    volname);
-                if (spec == -1) {
-                        count = -1;
-                        goto err;
-                }
 
-                spec = RPCSVC_AUTH_DONTCARE;
-                if (dict_get (svc->options, srchstr)) {
-                        ret = dict_get_str (svc->options, srchstr, &valstr);
-                        if (ret == 0) {
-                                ret = gf_string2boolean (valstr, &boolval);
-                                if (ret == 0) {
-                                        if (boolval == _gf_true)
-                                                spec = RPCSVC_AUTH_ACCEPT;
-                                        else
-                                                spec = RPCSVC_AUTH_REJECT;
-                                } else
-                                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "Faile"
-                                                "d to read auth val");
-                        } else
-                                gf_log (GF_RPCSVC, GF_LOG_ERROR, "Faile"
-                                        "d to read auth val");
-                }
-
-                GF_FREE (srchstr);
-                final = rpcsvc_combine_gen_spec_volume_checks (gen, spec);
-                if (final == RPCSVC_AUTH_ACCEPT) {
+                switch (ret) {
+                case _gf_true:
+                        result = RPCSVC_AUTH_ACCEPT;
                         autharr[count] = auth->auth->authnum;
                         ++count;
+                        break;
+                case _gf_false:
+                        result = RPCSVC_AUTH_REJECT;
+                        break;
+                default:
+                        result = RPCSVC_AUTH_DONTCARE;
                 }
         }
 
diff --git a/rpc/rpc-lib/src/rpcsvc-common.h b/rpc/rpc-lib/src/rpcsvc-common.h
index b03776dee..aed55e039 100644
--- a/rpc/rpc-lib/src/rpcsvc-common.h
+++ b/rpc/rpc-lib/src/rpcsvc-common.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _RPCSVC_COMMON_H
@@ -39,6 +30,8 @@ struct rpcsvc_state;
 typedef int (*rpcsvc_notify_t) (struct rpcsvc_state *, void *mydata,
                                 rpcsvc_event_t, void *data);
 
+struct drc_globals;
+typedef struct drc_globals rpcsvc_drc_globals_t;
 
 /* Contains global state required for all the RPC services.
  */
@@ -59,24 +52,75 @@ typedef struct rpcsvc_state {
         dict_t                  *options;
 
         /* Allow insecure ports. */
-        int                     allow_insecure;
+        gf_boolean_t            allow_insecure;
         gf_boolean_t            register_portmap;
+        gf_boolean_t            root_squash;
         glusterfs_ctx_t         *ctx;
 
         /* list of connections which will listen for incoming connections */
-        struct list_head         listeners;
+        struct list_head        listeners;
 
         /* list of programs registered with rpcsvc */
-        struct list_head         programs;
+        struct list_head        programs;
 
         /* list of notification callbacks */
-        struct list_head         notify;
-        int                      notify_count;
+        struct list_head        notify;
+        int                     notify_count;
 
         void                    *mydata; /* This is xlator */
-        rpcsvc_notify_t          notifyfn;
+        rpcsvc_notify_t         notifyfn;
         struct mem_pool         *rxpool;
+        rpcsvc_drc_globals_t    *drc;
+
+	/* per-client limit of outstanding rpc requests */
+        int                     outstanding_rpc_limit;
+        gf_boolean_t            addr_namelookup;
 } rpcsvc_t;
 
+/* DRC START */
+enum drc_op_type {
+        DRC_NA              = 0,
+        DRC_IDEMPOTENT      = 1,
+        DRC_NON_IDEMPOTENT  = 2
+};
+typedef enum drc_op_type drc_op_type_t;
+
+enum drc_type {
+        DRC_TYPE_NONE        = 0,
+        DRC_TYPE_IN_MEMORY   = 1
+};
+typedef enum drc_type drc_type_t;
+
+enum drc_lru_factor {
+        DRC_LRU_5_PC       = 20,
+        DRC_LRU_10_PC      = 10,
+        DRC_LRU_25_PC      = 4,
+        DRC_LRU_50_PC      = 2
+};
+typedef enum drc_lru_factor drc_lru_factor_t;
+
+enum drc_xid_state {
+        DRC_XID_MONOTONOUS  = 0,
+        DRC_XID_WRAPPED     = 1
+};
+typedef enum drc_xid_state drc_xid_state_t;
+
+enum drc_op_state {
+        DRC_OP_IN_TRANSIT    = 0,
+        DRC_OP_CACHED        = 1
+};
+typedef enum drc_op_state drc_op_state_t;
+
+enum drc_policy {
+        DRC_LRU              = 0
+};
+typedef enum drc_policy drc_policy_t;
+
+/* Default policies for DRC */
+#define DRC_DEFAULT_TYPE               DRC_TYPE_IN_MEMORY
+#define DRC_DEFAULT_CACHE_SIZE         0x20000
+#define DRC_DEFAULT_LRU_FACTOR         DRC_LRU_25_PC
+
+/* DRC END */
 
 #endif /* #ifndef _RPCSVC_COMMON_H */
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index e0fc29184..037c157f2 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -36,6 +27,8 @@
 #include "xdr-common.h"
 #include "xdr-generic.h"
 #include "rpc-common-xdr.h"
+#include "syncop.h"
+#include "rpc-drc.h"
 
 #include <errno.h>
 #include <pthread.h>
@@ -49,6 +42,7 @@
 #include <stdio.h>
 
 #include "xdr-rpcclnt.h"
+#include "glusterfs-acl.h"
 
 struct rpcsvc_program gluster_dump_prog;
 
@@ -135,6 +129,37 @@ rpcsvc_get_program_vector_sizer (rpcsvc_t *svc, uint32_t prognum,
                 return NULL;
 }
 
+int
+rpcsvc_request_outstanding (rpcsvc_t *svc, rpc_transport_t *trans, int delta)
+{
+        int ret = 0;
+        int old_count = 0;
+        int new_count = 0;
+        int limit = 0;
+
+        pthread_mutex_lock (&trans->lock);
+        {
+                limit = svc->outstanding_rpc_limit;
+                if (!limit)
+                        goto unlock;
+
+                old_count = trans->outstanding_rpc_count;
+                trans->outstanding_rpc_count += delta;
+                new_count = trans->outstanding_rpc_count;
+
+                if (old_count <= limit && new_count > limit)
+                        ret = rpc_transport_throttle (trans, _gf_true);
+
+                if (old_count > limit && new_count <= limit)
+                        ret = rpc_transport_throttle (trans, _gf_false);
+        }
+unlock:
+        pthread_mutex_unlock (&trans->lock);
+
+        return ret;
+}
+
+
 /* This needs to change to returning errors, since
  * we need to return RPC specific error messages when some
  * of the pointers below are NULL.
@@ -170,7 +195,11 @@ rpcsvc_program_actor (rpcsvc_request_t *req)
 
         if (!found) {
                 if (err != PROG_MISMATCH) {
-                        gf_log (GF_RPCSVC, GF_LOG_WARNING,
+                        /* log in DEBUG when nfs clients try to see if
+                         * ACL requests are accepted by nfs server
+                         */
+                        gf_log (GF_RPCSVC, (req->prognum == ACL_PROGRAM) ?
+                                GF_LOG_DEBUG : GF_LOG_WARNING,
                                 "RPC program not available (req %u %u)",
                                 req->prognum, req->progver);
                         err = PROG_UNAVAIL;
@@ -209,6 +238,8 @@ rpcsvc_program_actor (rpcsvc_request_t *req)
                 goto err;
         }
 
+        req->synctask = program->synctask;
+
         err = SUCCESS;
         gf_log (GF_RPCSVC, GF_LOG_TRACE, "Actor found: %s - %s",
                 program->progname, actor->procname);
@@ -223,7 +254,7 @@ err:
 /* this procedure can only pass 4 arguments to registered notifyfn. To send more
  * arguments call wrapper->notify directly.
  */
-inline void
+static inline void
 rpcsvc_program_notify (rpcsvc_listener_t *listener, rpcsvc_event_t event,
                        void *data)
 {
@@ -246,7 +277,7 @@ out:
 }
 
 
-inline int
+static inline int
 rpcsvc_accept (rpcsvc_t *svc, rpc_transport_t *listen_trans,
                rpc_transport_t *new_trans)
 {
@@ -276,8 +307,20 @@ rpcsvc_request_destroy (rpcsvc_request_t *req)
                 iobref_unref (req->iobref);
         }
 
+        if (req->hdr_iobuf)
+                iobuf_unref (req->hdr_iobuf);
+
+        /* This marks the "end" of an RPC request. Reply is
+           completely written to the socket and is on the way
+           to the client. It is time to decrement the
+           outstanding request counter by 1.
+        */
+        rpcsvc_request_outstanding (req->svc, req->trans, -1);
+
         rpc_transport_unref (req->trans);
 
+	GF_FREE (req->auxgidlarge);
+
         mem_put (req);
 
 out:
@@ -307,7 +350,9 @@ rpcsvc_request_init (rpcsvc_t *svc, rpc_transport_t *trans,
         req->msg[0] = progmsg;
         req->iobref = iobref_ref (msg->iobref);
         if (msg->vectored) {
-                for (i = 1; i < msg->count; i++) {
+                /* msg->vector[2] is defined in structure. prevent a
+                   out of bound access */
+                for (i = 1; i < min (msg->count, 2); i++) {
                         req->msg[i] = msg->vector[i];
                 }
         }
@@ -358,6 +403,12 @@ rpcsvc_request_create (rpcsvc_t *svc, rpc_transport_t *trans,
                 goto err;
         }
 
+        /* We just received a new request from the wire. Account for
+           it in the outsanding request counter to make sure we don't
+           ingest too many concurrent requests from the same client.
+        */
+        ret = rpcsvc_request_outstanding (svc, trans, +1);
+
         msgbuf = msg->vector[0].iov_base;
         msglen = msg->vector[0].iov_len;
 
@@ -417,6 +468,7 @@ rpcsvc_request_create (rpcsvc_t *svc, rpc_transport_t *trans,
          * since we are not handling authentication failures for now.
          */
         req->rpc_status = MSG_ACCEPTED;
+        req->reply = NULL;
         ret = 0;
 err:
         if (ret == -1) {
@@ -432,15 +484,39 @@ err:
 
 
 int
+rpcsvc_check_and_reply_error (int ret, call_frame_t *frame, void *opaque)
+{
+        rpcsvc_request_t  *req = NULL;
+
+        req = opaque;
+
+        if (ret)
+                gf_log ("rpcsvc", GF_LOG_ERROR,
+                        "rpc actor failed to complete successfully");
+
+        if (ret == RPCSVC_ACTOR_ERROR) {
+                ret = rpcsvc_error_reply (req);
+                if (ret)
+                        gf_log ("rpcsvc", GF_LOG_WARNING,
+                                "failed to queue error reply");
+        }
+
+        return 0;
+}
+
+int
 rpcsvc_handle_rpc_call (rpcsvc_t *svc, rpc_transport_t *trans,
                         rpc_transport_pollin_t *msg)
 {
-        rpcsvc_actor_t          *actor = NULL;
-        rpcsvc_request_t        *req = NULL;
-        int                     ret = -1;
-        uint16_t                port = 0;
-        gf_boolean_t            is_unix = _gf_false;
-        gf_boolean_t            unprivileged = _gf_false;
+        rpcsvc_actor_t         *actor          = NULL;
+        rpcsvc_actor            actor_fn       = NULL;
+        rpcsvc_request_t       *req            = NULL;
+        int                     ret            = -1;
+        uint16_t                port           = 0;
+        gf_boolean_t            is_unix        = _gf_false;
+        gf_boolean_t            unprivileged   = _gf_false;
+        drc_cached_op_t        *reply          = NULL;
+        rpcsvc_drc_globals_t   *drc            = NULL;
 
         if (!trans || !svc)
                 return -1;
@@ -476,7 +552,7 @@ rpcsvc_handle_rpc_call (rpcsvc_t *svc, rpc_transport_t *trans,
 
         req = rpcsvc_request_create (svc, trans, msg);
         if (!req)
-                goto err;
+                goto out;
 
         if (!rpcsvc_request_accepted (req))
                 goto err_reply;
@@ -494,40 +570,76 @@ rpcsvc_handle_rpc_call (rpcsvc_t *svc, rpc_transport_t *trans,
                         return -1;
         }
 
+        /* DRC */
+        if (rpcsvc_need_drc (req)) {
+                drc = req->svc->drc;
+
+                LOCK (&drc->lock);
+                reply = rpcsvc_drc_lookup (req);
+
+                /* retransmission of completed request, send cached reply */
+                if (reply && reply->state == DRC_OP_CACHED) {
+                        gf_log (GF_RPCSVC, GF_LOG_INFO, "duplicate request:"
+                                " XID: 0x%x", req->xid);
+                        ret = rpcsvc_send_cached_reply (req, reply);
+                        drc->cache_hits++;
+                        UNLOCK (&drc->lock);
+                        goto out;
+
+                } /* retransmitted request, original op in transit, drop it */
+                else if (reply && reply->state == DRC_OP_IN_TRANSIT) {
+                        gf_log (GF_RPCSVC, GF_LOG_INFO, "op in transit,"
+                                " discarding. XID: 0x%x", req->xid);
+                        ret = 0;
+                        drc->intransit_hits++;
+                        rpcsvc_request_destroy (req);
+                        UNLOCK (&drc->lock);
+                        goto out;
+
+                } /* fresh request, cache it as in-transit and proceed */
+                else {
+                        ret = rpcsvc_cache_request (req);
+                }
+                UNLOCK (&drc->lock);
+        }
+
         if (req->rpc_err == SUCCESS) {
                 /* Before going to xlator code, set the THIS properly */
                 THIS = svc->mydata;
 
-                if (req->count == 2) {
-                        if (actor->vector_actor) {
-                                ret = actor->vector_actor (req, &req->msg[1], 1,
-                                                           req->iobref);
-                        } else {
-                                rpcsvc_request_seterr (req, PROC_UNAVAIL);
-                                /* LOG TODO: print more info about procnum,
-                                   prognum etc, also print transport info */
-                                gf_log (GF_RPCSVC, GF_LOG_ERROR,
-                                        "No vectored handler present");
-                                ret = RPCSVC_ACTOR_ERROR;
-                        }
-                } else if (actor->actor) {
-                        ret = actor->actor (req);
+                actor_fn = actor->actor;
+
+                if (!actor_fn) {
+                        rpcsvc_request_seterr (req, PROC_UNAVAIL);
+                        /* LOG TODO: print more info about procnum,
+                           prognum etc, also print transport info */
+                        gf_log (GF_RPCSVC, GF_LOG_ERROR,
+                                "No vectored handler present");
+                        ret = RPCSVC_ACTOR_ERROR;
+                        goto err_reply;
                 }
-        }
 
-err_reply:
-        if (ret == RPCSVC_ACTOR_ERROR) {
-                ret = rpcsvc_error_reply (req);
+                if (req->synctask) {
+                        if (msg->hdr_iobuf)
+                                req->hdr_iobuf = iobuf_ref (msg->hdr_iobuf);
+
+                        ret = synctask_new (THIS->ctx->env,
+                                            (synctask_fn_t) actor_fn,
+                                            rpcsvc_check_and_reply_error, NULL,
+                                            req);
+                } else {
+                        ret = actor_fn (req);
+                }
         }
 
-        if (ret)
-                gf_log ("rpcsvc", GF_LOG_WARNING, "failed to queue error reply");
+err_reply:
 
+        ret = rpcsvc_check_and_reply_error (ret, NULL, req);
         /* No need to propagate error beyond this function since the reply
          * has now been queued. */
         ret = 0;
 
-err:
+out:
         return ret;
 }
 
@@ -545,6 +657,9 @@ rpcsvc_handle_disconnect (rpcsvc_t *svc, rpc_transport_t *trans)
 
         pthread_mutex_lock (&svc->rpclock);
         {
+                if (!svc->notify_count)
+                        goto unlock;
+
                 wrappers = GF_CALLOC (svc->notify_count, sizeof (*wrapper),
                                       gf_common_mt_rpcsvc_wrapper_t);
                 if (!wrappers) {
@@ -685,7 +800,7 @@ err:
         return txrecord;
 }
 
-inline int
+static inline int
 rpcsvc_get_callid (rpcsvc_t *rpc)
 {
         return GF_UNIVERSAL_ANSWER;
@@ -768,16 +883,28 @@ rpcsvc_callback_build_record (rpcsvc_t *rpc, int prognum, int progver,
         char                    *record      = NULL;
         struct iovec             recordhdr   = {0, };
         size_t                   pagesize    = 0;
+        size_t                   xdr_size    = 0;
         int                      ret         = -1;
 
         if ((!rpc) || (!recbuf)) {
                 goto out;
         }
 
+        /* Fill the rpc structure and XDR it into the buffer got above. */
+        ret = rpcsvc_fill_callback (prognum, progver, procnum, payload, xid,
+                                    &request);
+        if (ret == -1) {
+                gf_log ("rpcsvc", GF_LOG_WARNING, "cannot build a rpc-request "
+                        "xid (%"PRIu64")", xid);
+                goto out;
+        }
+
         /* First, try to get a pointer into the buffer which the RPC
          * layer can use.
          */
-        request_iob = iobuf_get (rpc->ctx->iobuf_pool);
+        xdr_size = xdr_sizeof ((xdrproc_t)xdr_callmsg, &request);
+
+        request_iob = iobuf_get2 (rpc->ctx->iobuf_pool, (xdr_size + payload));
         if (!request_iob) {
                 goto out;
         }
@@ -786,15 +913,6 @@ rpcsvc_callback_build_record (rpcsvc_t *rpc, int prognum, int progver,
 
         record = iobuf_ptr (request_iob);  /* Now we have it. */
 
-        /* Fill the rpc structure and XDR it into the buffer got above. */
-        ret = rpcsvc_fill_callback (prognum, progver, procnum, payload, xid,
-                                    &request);
-        if (ret == -1) {
-                gf_log ("rpcsvc", GF_LOG_WARNING, "cannot build a rpc-request "
-                        "xid (%"PRIu64")", xid);
-                goto out;
-        }
-
         recordhdr = rpcsvc_callback_build_header (record, pagesize, &request,
                                                   payload);
 
@@ -868,21 +986,22 @@ out:
         return ret;
 }
 
-inline int
-rpcsvc_transport_submit (rpc_transport_t *trans, struct iovec *hdrvec,
-                         int hdrcount, struct iovec *proghdr, int proghdrcount,
-                         struct iovec *progpayload, int progpayloadcount,
-                         struct iobref *iobref, void *priv)
+int
+rpcsvc_transport_submit (rpc_transport_t *trans, struct iovec *rpchdr,
+                         int rpchdrcount, struct iovec *proghdr,
+                         int proghdrcount, struct iovec *progpayload,
+                         int progpayloadcount, struct iobref *iobref,
+                         void *priv)
 {
         int                   ret   = -1;
         rpc_transport_reply_t reply = {{0, }};
 
-        if ((!trans) || (!hdrvec) || (!hdrvec->iov_base)) {
+        if ((!trans) || (!rpchdr) || (!rpchdr->iov_base)) {
                 goto out;
         }
 
-        reply.msg.rpchdr = hdrvec;
-        reply.msg.rpchdrcount = hdrcount;
+        reply.msg.rpchdr = rpchdr;
+        reply.msg.rpchdrcount = rpchdrcount;
         reply.msg.proghdr = proghdr;
         reply.msg.proghdrcount = proghdrcount;
         reply.msg.progpayload = progpayload;
@@ -938,13 +1057,14 @@ out:
  */
 struct iobuf *
 rpcsvc_record_build_record (rpcsvc_request_t *req, size_t payload,
-                            struct iovec *recbuf)
+                            size_t hdrlen, struct iovec *recbuf)
 {
         struct rpc_msg          reply;
         struct iobuf            *replyiob = NULL;
         char                    *record = NULL;
         struct iovec            recordhdr = {0, };
         size_t                  pagesize = 0;
+        size_t                  xdr_size = 0;
         rpcsvc_t                *svc = NULL;
         int                     ret = -1;
 
@@ -952,19 +1072,25 @@ rpcsvc_record_build_record (rpcsvc_request_t *req, size_t payload,
                 return NULL;
 
         svc = req->svc;
-        replyiob = iobuf_get (svc->ctx->iobuf_pool);
-        pagesize = iobuf_pagesize (replyiob);
-        if (!replyiob) {
-                goto err_exit;
-        }
-
-        record = iobuf_ptr (replyiob);  /* Now we have it. */
 
         /* Fill the rpc structure and XDR it into the buffer got above. */
         ret = rpcsvc_fill_reply (req, &reply);
         if (ret)
                 goto err_exit;
 
+        xdr_size = xdr_sizeof ((xdrproc_t)xdr_replymsg, &reply);
+
+        /* Payload would include 'readv' size etc too, where as
+           that comes as another payload iobuf */
+        replyiob = iobuf_get2 (svc->ctx->iobuf_pool, (xdr_size + hdrlen));
+        if (!replyiob) {
+                goto err_exit;
+        }
+
+        pagesize = iobuf_pagesize (replyiob);
+
+        record = iobuf_ptr (replyiob);  /* Now we have it. */
+
         recordhdr = rpcsvc_record_build_header (record, pagesize, reply,
                                                 payload);
         if (!recordhdr.iov_base) {
@@ -1019,7 +1145,9 @@ rpcsvc_submit_generic (rpcsvc_request_t *req, struct iovec *proghdr,
         struct iovec            recordhdr  = {0, };
         rpc_transport_t        *trans      = NULL;
         size_t                  msglen     = 0;
+        size_t                  hdrlen     = 0;
         char                    new_iobref = 0;
+        rpcsvc_drc_globals_t   *drc        = NULL;
 
         if ((!req) || (!req->trans))
                 return -1;
@@ -1037,7 +1165,7 @@ rpcsvc_submit_generic (rpcsvc_request_t *req, struct iovec *proghdr,
         gf_log (GF_RPCSVC, GF_LOG_TRACE, "Tx message: %zu", msglen);
 
         /* Build the buffer containing the encoded RPC reply. */
-        replyiob = rpcsvc_record_build_record (req, msglen, &recordhdr);
+        replyiob = rpcsvc_record_build_record (req, msglen, hdrlen, &recordhdr);
         if (!replyiob) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR,"Reply record creation failed");
                 goto disconnect_exit;
@@ -1054,20 +1182,31 @@ rpcsvc_submit_generic (rpcsvc_request_t *req, struct iovec *proghdr,
 
         iobref_add (iobref, replyiob);
 
+        /* cache the request in the duplicate request cache for appropriate ops */
+        if (req->reply) {
+                drc = req->svc->drc;
+
+                LOCK (&drc->lock);
+                ret = rpcsvc_cache_reply (req, iobref, &recordhdr, 1,
+                                          proghdr, hdrcount,
+                                          payload, payloadcount);
+                UNLOCK (&drc->lock);
+        }
+
         ret = rpcsvc_transport_submit (trans, &recordhdr, 1, proghdr, hdrcount,
                                        payload, payloadcount, iobref,
                                        req->trans_private);
 
         if (ret == -1) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "failed to submit message "
-                        "(XID: 0x%ux, Program: %s, ProgVers: %d, Proc: %d) to "
+                        "(XID: 0x%x, Program: %s, ProgVers: %d, Proc: %d) to "
                         "rpc-transport (%s)", req->xid,
                         req->prog ? req->prog->progname : "(not matched)",
                         req->prog ? req->prog->progver : 0,
                         req->procnum, trans->name);
         } else {
                 gf_log (GF_RPCSVC, GF_LOG_TRACE,
-                        "submitted reply for rpc-message (XID: 0x%ux, "
+                        "submitted reply for rpc-message (XID: 0x%x, "
                         "Program: %s, ProgVers: %d, Proc: %d) to rpc-transport "
                         "(%s)", req->xid, req->prog ? req->prog->progname: "-",
                         req->prog ? req->prog->progver : 0,
@@ -1097,7 +1236,7 @@ rpcsvc_error_reply (rpcsvc_request_t *req)
         if (!req)
                 return -1;
 
-        gf_log_callingfn ("", GF_LOG_WARNING, "sending a RPC error reply");
+        gf_log_callingfn ("", GF_LOG_DEBUG, "sending a RPC error reply");
 
         /* At this point the req should already have been filled with the
          * appropriate RPC error numbers.
@@ -1110,12 +1249,13 @@ rpcsvc_error_reply (rpcsvc_request_t *req)
 inline int
 rpcsvc_program_register_portmap (rpcsvc_program_t *newprog, uint32_t port)
 {
-        int                ret   = 0;
+        int                ret   = -1; /* FAIL */
 
         if (!newprog) {
                 goto out;
         }
 
+        /* pmap_set() returns 0 for FAIL and 1 for SUCCESS */
         if (!(pmap_set (newprog->prognum, newprog->progver, IPPROTO_TCP,
                         port))) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "Could not register with"
@@ -1123,7 +1263,7 @@ rpcsvc_program_register_portmap (rpcsvc_program_t *newprog, uint32_t port)
                 goto out;
         }
 
-        ret = 0;
+        ret = 0; /* SUCCESS */
 out:
         return ret;
 }
@@ -1132,7 +1272,7 @@ out:
 inline int
 rpcsvc_program_unregister_portmap (rpcsvc_program_t *prog)
 {
-        int ret = 0;
+        int ret = -1;
 
         if (!prog)
                 goto out;
@@ -1254,28 +1394,44 @@ rpcsvc_submit_message (rpcsvc_request_t *req, struct iovec *proghdr,
 
 
 int
-rpcsvc_program_unregister (rpcsvc_t *svc, rpcsvc_program_t *prog)
+rpcsvc_program_unregister (rpcsvc_t *svc, rpcsvc_program_t *program)
 {
         int                     ret = -1;
-
-        if (!svc || !prog) {
+        rpcsvc_program_t        *prog = NULL;
+        if (!svc || !program) {
                 goto out;
         }
 
-        ret = rpcsvc_program_unregister_portmap (prog);
+        ret = rpcsvc_program_unregister_portmap (program);
         if (ret == -1) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "portmap unregistration of"
                         " program failed");
                 goto out;
         }
 
+        pthread_mutex_lock (&svc->rpclock);
+        {
+                list_for_each_entry (prog, &svc->programs, program) {
+                        if ((prog->prognum == program->prognum)
+                            && (prog->progver == program->progver)) {
+                                break;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&svc->rpclock);
+
+        if (prog == NULL) {
+                ret = -1;
+                goto out;
+        }
+
         gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Program unregistered: %s, Num: %d,"
                 " Ver: %d, Port: %d", prog->progname, prog->prognum,
                 prog->progver, prog->progport);
 
         pthread_mutex_lock (&svc->rpclock);
         {
-                list_del (&prog->program);
+                list_del_init (&prog->program);
         }
         pthread_mutex_unlock (&svc->rpclock);
 
@@ -1283,8 +1439,8 @@ rpcsvc_program_unregister (rpcsvc_t *svc, rpcsvc_program_t *prog)
 out:
         if (ret == -1) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "Program unregistration failed"
-                        ": %s, Num: %d, Ver: %d, Port: %d", prog->progname,
-                        prog->prognum, prog->progver, prog->progport);
+                        ": %s, Num: %d, Ver: %d, Port: %d", program->progname,
+                        program->prognum, program->progver, program->progport);
         }
 
         return ret;
@@ -1473,6 +1629,7 @@ rpcsvc_create_listeners (rpcsvc_t *svc, dict_t *options, char *name)
                 }
 
                 GF_FREE (transport_name);
+                transport_name = NULL;
                 count++;
         }
 
@@ -1484,17 +1641,13 @@ rpcsvc_create_listeners (rpcsvc_t *svc, dict_t *options, char *name)
         transport_type = NULL;
 
 out:
-        if (str != NULL) {
-                GF_FREE (str);
-        }
+        GF_FREE (str);
 
-        if (transport_type != NULL) {
-                GF_FREE (transport_type);
-        }
+        GF_FREE (transport_type);
 
-        if (tmp != NULL) {
-                GF_FREE (tmp);
-        }
+        GF_FREE (tmp);
+
+        GF_FREE (transport_name);
 
         return count;
 }
@@ -1671,15 +1824,17 @@ rpcsvc_dump (rpcsvc_request_t *req)
         uint32_t     dump_rsp_len      = 0;
 
         if (!req)
-                goto fail;
+                goto sendrsp;
 
         ret = build_prog_details (req, &rsp);
         if (ret < 0) {
                 op_errno = -ret;
-                goto fail;
+                goto sendrsp;
         }
 
-fail:
+        op_errno = 0;
+
+sendrsp:
         rsp.op_errno = gf_errno_to_error (op_errno);
         rsp.op_ret   = ret;
 
@@ -1691,15 +1846,12 @@ fail:
 
         ret = xdr_serialize_generic (iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
         if (ret < 0) {
-                if (req)
-                        req->rpc_err = GARBAGE_ARGS;
-                op_errno = EINVAL;
-                goto fail;
+                ret = RPCSVC_ACTOR_ERROR;
+        } else {
+                rpcsvc_submit_generic (req, &iov, 1, NULL, 0, NULL);
+                ret = 0;
         }
 
-        ret = rpcsvc_submit_generic (req, &iov, 1, NULL, 0,
-                                     NULL);
-
         free_prog_details (&rsp);
 
         return ret;
@@ -1738,12 +1890,92 @@ rpcsvc_init_options (rpcsvc_t *svc, dict_t *options)
                 gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Portmap registration "
                         "disabled");
 
-        ret = 0;
+        ret = rpcsvc_set_outstanding_rpc_limit (svc, options);
 out:
         return ret;
 }
 
 int
+rpcsvc_reconfigure_options (rpcsvc_t *svc, dict_t *options)
+{
+        xlator_t         *xlator    = NULL;
+        xlator_list_t    *volentry  = NULL;
+        char             *srchkey   = NULL;
+        char             *keyval    = NULL;
+        int              ret        = -1;
+
+        if ((!svc) || (!svc->options) || (!options))
+                return (-1);
+
+        /* Fetch the xlator from svc */
+        xlator = (xlator_t *) svc->mydata;
+        if (!xlator)
+                return (-1);
+
+        /* Reconfigure the volume specific rpc-auth.addr allow part */
+        volentry = xlator->children;
+        while (volentry) {
+                ret = gf_asprintf (&srchkey, "rpc-auth.addr.%s.allow",
+                                             volentry->xlator->name);
+                if (ret == -1) {
+                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
+                        return (-1);
+                }
+
+                /* If found the srchkey, delete old key/val pair
+                 * and set the key with new value.
+                 */
+                if (!dict_get_str (options, srchkey, &keyval)) {
+                        dict_del (svc->options, srchkey);
+                        ret = dict_set_str (svc->options, srchkey, keyval);
+                        if (ret < 0) {
+                                gf_log (GF_RPCSVC, GF_LOG_ERROR,
+                                        "dict_set_str error");
+                                GF_FREE (srchkey);
+                                return (-1);
+                        }
+                }
+
+                GF_FREE (srchkey);
+                volentry = volentry->next;
+        }
+
+        /* Reconfigure the volume specific rpc-auth.addr reject part */
+        volentry = xlator->children;
+        while (volentry) {
+                ret = gf_asprintf (&srchkey, "rpc-auth.addr.%s.reject",
+                                             volentry->xlator->name);
+                if (ret == -1) {
+                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
+                        return (-1);
+                }
+
+                /* If found the srchkey, delete old key/val pair
+                 * and set the key with new value.
+                 */
+                if (!dict_get_str (options, srchkey, &keyval)) {
+                        dict_del (svc->options, srchkey);
+                        ret = dict_set_str (svc->options, srchkey, keyval);
+                        if (ret < 0) {
+                                gf_log (GF_RPCSVC, GF_LOG_ERROR,
+                                        "dict_set_str error");
+                                GF_FREE (srchkey);
+                                return (-1);
+                        }
+                }
+
+                GF_FREE (srchkey);
+                volentry = volentry->next;
+        }
+
+        ret = rpcsvc_init_options (svc, options);
+        if (ret)
+                return (-1);
+
+        return rpcsvc_auth_reconf (svc, options);
+}
+
+int
 rpcsvc_transport_unix_options_build (dict_t **options, char *filepath)
 {
         dict_t                  *dict = NULL;
@@ -1782,21 +2014,63 @@ rpcsvc_transport_unix_options_build (dict_t **options, char *filepath)
         *options = dict;
 out:
         if (ret) {
-                if (fpath)
-                        GF_FREE (fpath);
+                GF_FREE (fpath);
                 if (dict)
                         dict_unref (dict);
         }
         return ret;
 }
 
+/*
+ * Reconfigure() the rpc.outstanding-rpc-limit param.
+ */
+int
+rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options)
+{
+        int            ret        = -1; /* FAILURE */
+        int            rpclim     = 0;
+        static char    *rpclimkey = "rpc.outstanding-rpc-limit";
+
+        if ((!svc) || (!options))
+                return (-1);
+
+        /* Reconfigure() the rpc.outstanding-rpc-limit param */
+        ret = dict_get_int32 (options, rpclimkey, &rpclim);
+        if (ret < 0) {
+                /* Fall back to default for FAILURE */
+                rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
+        } else {
+                /* SUCCESS: round off to multiple of 8.
+                 * If the input value fails Boundary check, fall back to
+                 * default i.e. RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT.
+                 * NB: value 0 is special, means its unset i.e. unlimited.
+                 */
+                rpclim = ((rpclim + 8 - 1) >> 3) * 8;
+                if (rpclim < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) {
+                        rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
+                } else if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) {
+                        rpclim = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT;
+                }
+        }
+
+        if (svc->outstanding_rpc_limit != rpclim) {
+                svc->outstanding_rpc_limit = rpclim;
+                gf_log (GF_RPCSVC, GF_LOG_INFO,
+                                   "Configured %s with value %d",
+                                   rpclimkey, rpclim);
+        }
+
+        return (0);
+}
+
 /* The global RPC service initializer.
  */
 rpcsvc_t *
-rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options)
+rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options,
+             uint32_t poolcount)
 {
         rpcsvc_t          *svc              = NULL;
-        int                ret              = -1, poolcount = 0;
+        int                ret              = -1;
 
         if ((!ctx) || (!options))
                 return NULL;
@@ -1817,7 +2091,8 @@ rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options)
                 goto free_svc;
         }
 
-        poolcount   = RPCSVC_POOLCOUNT_MULT * svc->memfactor;
+        if (!poolcount)
+                poolcount = RPCSVC_POOLCOUNT_MULT * svc->memfactor;
 
         gf_log (GF_RPCSVC, GF_LOG_TRACE, "rx pool: %d", poolcount);
         svc->rxpool = mem_pool_new (rpcsvc_request_t, poolcount);
@@ -1848,6 +2123,7 @@ rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options)
                         "failed to register DUMP program");
                 goto free_svc;
         }
+
         ret = 0;
 free_svc:
         if (ret == -1) {
@@ -1860,17 +2136,16 @@ free_svc:
 
 
 int
-rpcsvc_transport_peer_check_search (dict_t *options, char *pattern, char *clstr)
+rpcsvc_transport_peer_check_search (dict_t *options, char *pattern,
+                                    char *ip, char *hostname)
 {
-        int                     ret = -1;
-        char                    *addrtok = NULL;
-        char                    *addrstr = NULL;
-        char                    *svptr = NULL;
+        int                      ret           = -1;
+        char                    *addrtok       = NULL;
+        char                    *addrstr       = NULL;
+        char                    *dup_addrstr   = NULL;
+        char                    *svptr         = NULL;
 
-        if ((!options) || (!clstr))
-                return -1;
-
-        if (!dict_get (options, pattern))
+        if ((!options) || (!ip))
                 return -1;
 
         ret = dict_get_str (options, pattern, &addrstr);
@@ -1884,88 +2159,91 @@ rpcsvc_transport_peer_check_search (dict_t *options, char *pattern, char *clstr)
                 goto err;
         }
 
-        addrtok = strtok_r (addrstr, ",", &svptr);
+        dup_addrstr = gf_strdup (addrstr);
+        addrtok = strtok_r (dup_addrstr, ",", &svptr);
         while (addrtok) {
 
                 /* CASEFOLD not present on Solaris */
 #ifdef FNM_CASEFOLD
-                ret = fnmatch (addrtok, clstr, FNM_CASEFOLD);
+                ret = fnmatch (addrtok, ip, FNM_CASEFOLD);
 #else
-                ret = fnmatch (addrtok, clstr, 0);
+                ret = fnmatch (addrtok, ip, 0);
 #endif
                 if (ret == 0)
                         goto err;
 
+                /* compare hostnames if applicable */
+                if (hostname) {
+#ifdef FNM_CASEFOLD
+                        ret = fnmatch (addrtok, hostname, FNM_CASEFOLD);
+#else
+                        ret = fnmatch (addrtok, hostname, 0);
+#endif
+                        if (ret == 0)
+                                goto err;
+                }
+
                 addrtok = strtok_r (NULL, ",", &svptr);
         }
 
         ret = -1;
 err:
+        GF_FREE (dup_addrstr);
 
         return ret;
 }
 
 
-int
-rpcsvc_transport_peer_check_allow (dict_t *options, char *volname, char *clstr)
+static int
+rpcsvc_transport_peer_check_allow (dict_t *options, char *volname,
+                                   char *ip, char *hostname)
 {
-        int     ret = RPCSVC_AUTH_DONTCARE;
+        int      ret     = RPCSVC_AUTH_DONTCARE;
         char    *srchstr = NULL;
-        char    globalrule[] = "rpc-auth.addr.allow";
 
-        if ((!options) || (!clstr))
+        if ((!options) || (!ip) || (!volname))
                 return ret;
 
-        /* If volname is NULL, then we're searching for the general rule to
-         * determine the current address in clstr is allowed or not for all
-         * subvolumes.
-         */
-        if (volname) {
-                ret = gf_asprintf (&srchstr, "rpc-auth.addr.%s.allow", volname);
-                if (ret == -1) {
-                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
-                        ret = RPCSVC_AUTH_DONTCARE;
-                        goto out;
-                }
-        } else
-                srchstr = globalrule;
+        ret = gf_asprintf (&srchstr, "rpc-auth.addr.%s.allow", volname);
+        if (ret == -1) {
+                gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
+                ret = RPCSVC_AUTH_DONTCARE;
+                goto out;
+        }
 
-        ret = rpcsvc_transport_peer_check_search (options, srchstr, clstr);
-        if (volname)
-                GF_FREE (srchstr);
+        ret = rpcsvc_transport_peer_check_search (options, srchstr,
+                                                  ip, hostname);
+        GF_FREE (srchstr);
 
         if (ret == 0)
                 ret = RPCSVC_AUTH_ACCEPT;
         else
-                ret = RPCSVC_AUTH_DONTCARE;
+                ret = RPCSVC_AUTH_REJECT;
 out:
         return ret;
 }
 
-int
-rpcsvc_transport_peer_check_reject (dict_t *options, char *volname, char *clstr)
+static int
+rpcsvc_transport_peer_check_reject (dict_t *options, char *volname,
+                                    char *ip, char *hostname)
 {
-        int     ret = RPCSVC_AUTH_DONTCARE;
+        int      ret     = RPCSVC_AUTH_DONTCARE;
         char    *srchstr = NULL;
-        char    generalrule[] = "rpc-auth.addr.reject";
 
-        if ((!options) || (!clstr))
+        if ((!options) || (!ip) || (!volname))
                 return ret;
 
-        if (volname) {
-                ret = gf_asprintf (&srchstr, "rpc-auth.addr.%s.reject",
-                                   volname);
-                if (ret == -1) {
-                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
-                        ret = RPCSVC_AUTH_REJECT;
-                        goto out;
-                }
-        } else
-                srchstr = generalrule;
+        ret = gf_asprintf (&srchstr, "rpc-auth.addr.%s.reject",
+                           volname);
+        if (ret == -1) {
+                gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
+                ret = RPCSVC_AUTH_REJECT;
+                goto out;
+        }
 
-        ret = rpcsvc_transport_peer_check_search (options, srchstr, clstr);
-        if (volname)
-                GF_FREE (srchstr);
+        ret = rpcsvc_transport_peer_check_search (options, srchstr,
+                                                  ip, hostname);
+        GF_FREE (srchstr);
 
         if (ret == 0)
                 ret = RPCSVC_AUTH_REJECT;
@@ -1976,313 +2254,132 @@ out:
 }
 
 
-/* This function tests the results of the allow rule and the reject rule to
- * combine them into a single result that can be used to determine if the
- * connection should be allowed to proceed.
- * Heres the test matrix we need to follow in this function.
- *
- * A -  Allow, the result of the allow test. Never returns R.
- * R - Reject, result of the reject test. Never returns A.
- * Both can return D or dont care if no rule was given.
- *
- * | @allow | @reject | Result |
- * |    A   |   R     | R      |
- * |    D   |   D     | D      |
- * |    A   |   D     | A      |
- * |    D   |   R     | R      |
+/* Combines rpc auth's allow and reject options.
+ * Order of checks is important.
+ * First,              REJECT if either rejects.
+ * If neither rejects, ACCEPT if either accepts.
+ * If neither accepts, DONTCARE
  */
 int
 rpcsvc_combine_allow_reject_volume_check (int allow, int reject)
 {
-        int     final = RPCSVC_AUTH_REJECT;
-
-        /* If allowed rule allows but reject rule rejects, we stay cautious
-         * and reject. */
-        if ((allow == RPCSVC_AUTH_ACCEPT) && (reject == RPCSVC_AUTH_REJECT))
-                final = RPCSVC_AUTH_REJECT;
-        /* if both are dont care, that is user did not specify for either allow
-         * or reject, we leave it up to the general rule to apply, in the hope
-         * that there is one.
-         */
-        else if ((allow == RPCSVC_AUTH_DONTCARE) &&
-                 (reject == RPCSVC_AUTH_DONTCARE))
-                final = RPCSVC_AUTH_DONTCARE;
-        /* If one is dont care, the other one applies. */
-        else if ((allow == RPCSVC_AUTH_ACCEPT) &&
-                 (reject == RPCSVC_AUTH_DONTCARE))
-                final = RPCSVC_AUTH_ACCEPT;
-        else if ((allow == RPCSVC_AUTH_DONTCARE) &&
-                 (reject == RPCSVC_AUTH_REJECT))
-                final = RPCSVC_AUTH_REJECT;
-
-        return final;
-}
-
-
-/* Combines the result of the general rule test against, the specific rule
- * to determine final permission for the client's address.
- *
- * | @gen   | @spec   | Result |
- * |    A   |   A     | A      |
- * |    A   |   R     | R      |
- * |    A   |   D     | A      |
- * |    D   |   A     | A      |
- * |    D   |   R     | R      |
- * |    D   |   D     | D      |
- * |    R   |   A     | A      |
- * |    R   |   D     | R      |
- * |    R   |   R     | R      |
- */
-int
-rpcsvc_combine_gen_spec_addr_checks (int gen, int spec)
-{
-        int     final = RPCSVC_AUTH_REJECT;
-
-        if ((gen == RPCSVC_AUTH_ACCEPT) && (spec == RPCSVC_AUTH_ACCEPT))
-                final = RPCSVC_AUTH_ACCEPT;
-        else if ((gen == RPCSVC_AUTH_ACCEPT) && (spec == RPCSVC_AUTH_REJECT))
-                final = RPCSVC_AUTH_REJECT;
-        else if ((gen == RPCSVC_AUTH_ACCEPT) && (spec == RPCSVC_AUTH_DONTCARE))
-                final = RPCSVC_AUTH_ACCEPT;
-        else if ((gen == RPCSVC_AUTH_DONTCARE) && (spec == RPCSVC_AUTH_ACCEPT))
-                final = RPCSVC_AUTH_ACCEPT;
-        else if ((gen == RPCSVC_AUTH_DONTCARE) && (spec == RPCSVC_AUTH_REJECT))
-                final = RPCSVC_AUTH_REJECT;
-        else if ((gen == RPCSVC_AUTH_DONTCARE) && (spec== RPCSVC_AUTH_DONTCARE))
-                final = RPCSVC_AUTH_DONTCARE;
-        else if ((gen == RPCSVC_AUTH_REJECT) && (spec == RPCSVC_AUTH_ACCEPT))
-                final = RPCSVC_AUTH_ACCEPT;
-        else if ((gen == RPCSVC_AUTH_REJECT) && (spec == RPCSVC_AUTH_DONTCARE))
-                final = RPCSVC_AUTH_REJECT;
-        else if ((gen == RPCSVC_AUTH_REJECT) && (spec == RPCSVC_AUTH_REJECT))
-                final = RPCSVC_AUTH_REJECT;
-
-        return final;
-}
-
+        if (allow == RPCSVC_AUTH_REJECT ||
+            reject == RPCSVC_AUTH_REJECT)
+                return RPCSVC_AUTH_REJECT;
 
+        if (allow == RPCSVC_AUTH_ACCEPT ||
+            reject == RPCSVC_AUTH_ACCEPT)
+                return RPCSVC_AUTH_ACCEPT;
 
-/* Combines the result of the general rule test against, the specific rule
- * to determine final test for the connection coming in for a given volume.
- *
- * | @gen   | @spec   | Result |
- * |    A   |   A     | A      |
- * |    A   |   R     | R      |
- * |    A   |   D     | A      |
- * |    D   |   A     | A      |
- * |    D   |   R     | R      |
- * |    D   |   D     | R      |, special case, we intentionally disallow this.
- * |    R   |   A     | A      |
- * |    R   |   D     | R      |
- * |    R   |   R     | R      |
- */
-int
-rpcsvc_combine_gen_spec_volume_checks (int gen, int spec)
-{
-        int     final = RPCSVC_AUTH_REJECT;
-
-        if ((gen == RPCSVC_AUTH_ACCEPT) && (spec == RPCSVC_AUTH_ACCEPT))
-                final = RPCSVC_AUTH_ACCEPT;
-        else if ((gen == RPCSVC_AUTH_ACCEPT) && (spec == RPCSVC_AUTH_REJECT))
-                final = RPCSVC_AUTH_REJECT;
-        else if ((gen == RPCSVC_AUTH_ACCEPT) && (spec == RPCSVC_AUTH_DONTCARE))
-                final = RPCSVC_AUTH_ACCEPT;
-        else if ((gen == RPCSVC_AUTH_DONTCARE) && (spec == RPCSVC_AUTH_ACCEPT))
-                final = RPCSVC_AUTH_ACCEPT;
-        else if ((gen == RPCSVC_AUTH_DONTCARE) && (spec == RPCSVC_AUTH_REJECT))
-                final = RPCSVC_AUTH_REJECT;
-        /* On no rule, we reject. */
-        else if ((gen == RPCSVC_AUTH_DONTCARE) && (spec== RPCSVC_AUTH_DONTCARE))
-                final = RPCSVC_AUTH_REJECT;
-        else if ((gen == RPCSVC_AUTH_REJECT) && (spec == RPCSVC_AUTH_ACCEPT))
-                final = RPCSVC_AUTH_ACCEPT;
-        else if ((gen == RPCSVC_AUTH_REJECT) && (spec == RPCSVC_AUTH_DONTCARE))
-                final = RPCSVC_AUTH_REJECT;
-        else if ((gen == RPCSVC_AUTH_REJECT) && (spec == RPCSVC_AUTH_REJECT))
-                final = RPCSVC_AUTH_REJECT;
-
-        return final;
+        return RPCSVC_AUTH_DONTCARE;
 }
 
-
 int
-rpcsvc_transport_peer_check_name (dict_t *options, char *volname,
-                                  rpc_transport_t *trans)
+rpcsvc_auth_check (rpcsvc_t *svc, char *volname,
+                   rpc_transport_t *trans)
 {
-        int     ret = RPCSVC_AUTH_REJECT;
-        int     aret = RPCSVC_AUTH_REJECT;
-        int     rjret = RPCSVC_AUTH_REJECT;
-        char    clstr[RPCSVC_PEER_STRLEN];
-
-        if (!trans)
+        int     ret                            = RPCSVC_AUTH_REJECT;
+        int     accept                         = RPCSVC_AUTH_REJECT;
+        int     reject                         = RPCSVC_AUTH_REJECT;
+        char   *hostname                       = NULL;
+        char   *ip                             = NULL;
+        char    client_ip[RPCSVC_PEER_STRLEN]  = {0};
+        char   *allow_str                      = NULL;
+        char   *reject_str                     = NULL;
+        char   *srchstr                        = NULL;
+        dict_t *options                        = NULL;
+
+        if (!svc || !volname || !trans)
                 return ret;
 
-        ret = rpcsvc_transport_peername (trans, clstr, RPCSVC_PEER_STRLEN);
-        if (ret != 0) {
-                gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to get remote addr: "
-                        "%s", gai_strerror (ret));
-                ret = RPCSVC_AUTH_REJECT;
-                goto err;
-        }
-
-        aret = rpcsvc_transport_peer_check_allow (options, volname, clstr);
-        rjret = rpcsvc_transport_peer_check_reject (options, volname, clstr);
-
-        ret = rpcsvc_combine_allow_reject_volume_check (aret, rjret);
-
-err:
-        return ret;
-}
-
-
-int
-rpcsvc_transport_peer_check_addr (dict_t *options, char *volname,
-                                  rpc_transport_t *trans)
-{
-        int     ret = RPCSVC_AUTH_REJECT;
-        int     aret = RPCSVC_AUTH_DONTCARE;
-        int     rjret = RPCSVC_AUTH_REJECT;
-        char    clstr[RPCSVC_PEER_STRLEN];
-        struct sockaddr_storage sastorage = {0,};
-
-        if (!trans)
+        /* Fetch the options from svc struct and validate */
+        options = svc->options;
+        if (!options)
                 return ret;
 
-        ret = rpcsvc_transport_peeraddr (trans, clstr, RPCSVC_PEER_STRLEN,
-                                         &sastorage, sizeof (sastorage));
+        ret = rpcsvc_transport_peername (trans, client_ip, RPCSVC_PEER_STRLEN);
         if (ret != 0) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to get remote addr: "
                         "%s", gai_strerror (ret));
-                ret = RPCSVC_AUTH_REJECT;
-                goto err;
-        }
-
-        aret = rpcsvc_transport_peer_check_allow (options, volname, clstr);
-        rjret = rpcsvc_transport_peer_check_reject (options, volname, clstr);
-
-        ret = rpcsvc_combine_allow_reject_volume_check (aret, rjret);
-err:
-        return ret;
-}
-
-
-int
-rpcsvc_transport_check_volume_specific (dict_t *options, char *volname,
-                                        rpc_transport_t *trans)
-{
-        int             namechk = RPCSVC_AUTH_REJECT;
-        int             addrchk = RPCSVC_AUTH_REJECT;
-        gf_boolean_t    namelookup = _gf_false;
-        char            *namestr = NULL;
-        int             ret = 0;
-
-        if ((!options) || (!volname) || (!trans))
                 return RPCSVC_AUTH_REJECT;
-
-        /* Disabled by default */
-        if ((dict_get (options, "rpc-auth.addr.namelookup"))) {
-                ret = dict_get_str (options, "rpc-auth.addr.namelookup"
-                                    , &namestr);
-                if (ret == 0)
-                        ret = gf_string2boolean (namestr, &namelookup);
         }
 
-        /* We need two separate checks because the rules with addresses in them
-         * can be network addresses which can be general and names can be
-         * specific which will over-ride the network address rules.
+        /* Accept if its the default case: Allow all, Reject none
+         * The default volfile always contains a 'allow *' rule
+         * for each volume. If allow rule is missing (which implies
+         * there is some bad volfile generating code doing this), we
+         * assume no one is allowed mounts, and thus, we reject mounts.
          */
-        if (namelookup)
-                namechk = rpcsvc_transport_peer_check_name (options, volname,
-                                                            trans);
-        addrchk = rpcsvc_transport_peer_check_addr (options, volname, trans);
-
-        if (namelookup)
-                ret = rpcsvc_combine_gen_spec_addr_checks (addrchk,
-                                                               namechk);
-        else
-                ret = addrchk;
-
-        return ret;
-}
-
-
-int
-rpcsvc_transport_check_volume_general (dict_t *options, rpc_transport_t *trans)
-{
-        int             addrchk = RPCSVC_AUTH_REJECT;
-        int             namechk = RPCSVC_AUTH_REJECT;
-        gf_boolean_t    namelookup = _gf_false;
-        char            *namestr = NULL;
-        int             ret = 0;
-
-        if ((!options) || (!trans))
+        ret = gf_asprintf (&srchstr, "rpc-auth.addr.%s.allow", volname);
+        if (ret == -1) {
+                gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
                 return RPCSVC_AUTH_REJECT;
-
-        /* Disabled by default */
-        if ((dict_get (options, "rpc-auth.addr.namelookup"))) {
-                ret = dict_get_str (options, "rpc-auth.addr.namelookup"
-                                    , &namestr);
-                if (ret == 0)
-                        ret = gf_string2boolean (namestr, &namelookup);
         }
 
-        /* We need two separate checks because the rules with addresses in them
-         * can be network addresses which can be general and names can be
-         * specific which will over-ride the network address rules.
-         */
-        if (namelookup)
-                namechk = rpcsvc_transport_peer_check_name (options, NULL, trans);
-        addrchk = rpcsvc_transport_peer_check_addr (options, NULL, trans);
-
-        if (namelookup)
-                ret = rpcsvc_combine_gen_spec_addr_checks (addrchk,
-                                                               namechk);
-        else
-                ret = addrchk;
+        ret = dict_get_str (options, srchstr, &allow_str);
+        GF_FREE (srchstr);
+        if (ret < 0)
+                return RPCSVC_AUTH_REJECT;
 
-        return ret;
-}
+        ret = gf_asprintf (&srchstr, "rpc-auth.addr.%s.reject", volname);
+        if (ret == -1) {
+                gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
+                return RPCSVC_AUTH_REJECT;
+        }
 
-int
-rpcsvc_transport_peer_check (dict_t *options, char *volname,
-                             rpc_transport_t *trans)
-{
-        int     general_chk = RPCSVC_AUTH_REJECT;
-        int     specific_chk = RPCSVC_AUTH_REJECT;
+        ret = dict_get_str (options, srchstr, &reject_str);
+        GF_FREE (srchstr);
+        if (reject_str == NULL && !strcmp ("*", allow_str))
+                return RPCSVC_AUTH_ACCEPT;
+
+        /* Non-default rule, authenticate */
+        if (!get_host_name (client_ip, &ip))
+                ip = client_ip;
+
+        /* addr-namelookup check */
+        if (svc->addr_namelookup == _gf_true) {
+                ret = gf_get_hostname_from_ip (ip, &hostname);
+                if (ret) {
+                        if (hostname)
+                                GF_FREE (hostname);
+                        /* failed to get hostname, but hostname auth
+                         * is enabled, so authentication will not be
+                         * 100% correct. reject mounts
+                         */
+                        return RPCSVC_AUTH_REJECT;
+                }
+        }
 
-        if ((!options) || (!volname) || (!trans))
-                return RPCSVC_AUTH_REJECT;
+        accept = rpcsvc_transport_peer_check_allow (options, volname,
+                                                    ip, hostname);
 
-        general_chk = rpcsvc_transport_check_volume_general (options, trans);
-        specific_chk = rpcsvc_transport_check_volume_specific (options, volname,
-                                                               trans);
+        reject = rpcsvc_transport_peer_check_reject (options, volname,
+                                                     ip, hostname);
 
-        return rpcsvc_combine_gen_spec_volume_checks (general_chk,
-                                                      specific_chk);
+        if (hostname)
+                GF_FREE (hostname);
+        return rpcsvc_combine_allow_reject_volume_check (accept, reject);
 }
 
-
 int
 rpcsvc_transport_privport_check (rpcsvc_t *svc, char *volname,
                                  rpc_transport_t *trans)
 {
-        struct sockaddr_storage sastorage = {0,};
-        struct sockaddr_in     *sa = NULL;
+        union gf_sock_union     sock_union;
         int                     ret = RPCSVC_AUTH_REJECT;
-        socklen_t               sasize = sizeof (sa);
+        socklen_t               sinsize = sizeof (&sock_union.sin);
         char                    *srchstr = NULL;
         char                    *valstr = NULL;
-        int                     globalinsecure = RPCSVC_AUTH_REJECT;
-        int                     exportinsecure = RPCSVC_AUTH_DONTCARE;
         uint16_t                port = 0;
         gf_boolean_t            insecure = _gf_false;
 
+        memset (&sock_union, 0, sizeof (sock_union));
+
         if ((!svc) || (!volname) || (!trans))
                 return ret;
 
-        sa = (struct sockaddr_in*) &sastorage;
-        ret = rpcsvc_transport_peeraddr (trans, NULL, 0, &sastorage,
-                                         sasize);
+        ret = rpcsvc_transport_peeraddr (trans, NULL, 0, &sock_union.storage,
+                                         sinsize);
         if (ret != 0) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to get peer addr: %s",
                         gai_strerror (ret));
@@ -2290,7 +2387,7 @@ rpcsvc_transport_privport_check (rpcsvc_t *svc, char *volname,
                 goto err;
         }
 
-        port = ntohs (sa->sin_port);
+        port = ntohs (sock_union.sin.sin_port);
         gf_log (GF_RPCSVC, GF_LOG_TRACE, "Client port: %d", (int)port);
         /* If the port is already a privileged one, dont bother with checking
          * options.
@@ -2301,23 +2398,6 @@ rpcsvc_transport_privport_check (rpcsvc_t *svc, char *volname,
         }
 
         /* Disabled by default */
-        if ((dict_get (svc->options, "rpc-auth.ports.insecure"))) {
-                ret = dict_get_str (svc->options, "rpc-auth.ports.insecure"
-                                    , &srchstr);
-                if (ret == 0) {
-                        ret = gf_string2boolean (srchstr, &insecure);
-                        if (ret == 0) {
-                                if (insecure == _gf_true)
-                                        globalinsecure = RPCSVC_AUTH_ACCEPT;
-                        } else
-                                gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to"
-                                        " read rpc-auth.ports.insecure value");
-                } else
-                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to"
-                                " read rpc-auth.ports.insecure value");
-        }
-
-        /* Disabled by default */
         ret = gf_asprintf (&srchstr, "rpc-auth.ports.%s.insecure", volname);
         if (ret == -1) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
@@ -2325,25 +2405,22 @@ rpcsvc_transport_privport_check (rpcsvc_t *svc, char *volname,
                 goto err;
         }
 
-        if (dict_get (svc->options, srchstr)) {
-                ret = dict_get_str (svc->options, srchstr, &valstr);
-                if (ret == 0) {
-                        ret = gf_string2boolean (valstr, &insecure);
-                        if (ret == 0) {
-                                if (insecure == _gf_true)
-                                       exportinsecure = RPCSVC_AUTH_ACCEPT;
-                                else
-                                        exportinsecure = RPCSVC_AUTH_REJECT;
-                        } else
-                                gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to"
-                                        " read rpc-auth.ports.insecure value");
-                } else
-                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to"
-                                " read rpc-auth.ports.insecure value");
-        }
-
-        ret = rpcsvc_combine_gen_spec_volume_checks (globalinsecure,
-                                                         exportinsecure);
+        ret = dict_get_str (svc->options, srchstr, &valstr);
+        if (ret) {
+                gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to"
+                        " read rpc-auth.ports.insecure value");
+                goto err;
+        }
+
+        ret = gf_string2boolean (valstr, &insecure);
+        if (ret) {
+                gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to"
+                        " convert rpc-auth.ports.insecure value");
+                goto err;
+        }
+
+        ret = insecure ? RPCSVC_AUTH_ACCEPT : RPCSVC_AUTH_REJECT;
+
         if (ret == RPCSVC_AUTH_ACCEPT)
                 gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Unprivileged port allowed");
         else
@@ -2351,6 +2428,9 @@ rpcsvc_transport_privport_check (rpcsvc_t *svc, char *volname,
                         " allowed");
 
 err:
+        if (srchstr)
+                GF_FREE (srchstr);
+
         return ret;
 }
 
@@ -2372,22 +2452,22 @@ rpcsvc_volume_allowed (dict_t *options, char *volname)
                 goto out;
         }
 
-        if (!dict_get (options, srchstr)) {
-                GF_FREE (srchstr);
-                srchstr = globalrule;
-                ret = dict_get_str (options, srchstr, &addrstr);
-        } else
+        if (!dict_get (options, srchstr))
+                ret = dict_get_str (options, globalrule, &addrstr);
+        else
                 ret = dict_get_str (options, srchstr, &addrstr);
 
 out:
+        GF_FREE (srchstr);
+
         return addrstr;
 }
 
 
 rpcsvc_actor_t gluster_dump_actors[] = {
-        [GF_DUMP_NULL] = {"NULL", GF_DUMP_NULL, NULL, NULL, NULL, 0},
-        [GF_DUMP_DUMP] = {"DUMP", GF_DUMP_DUMP, rpcsvc_dump, NULL, NULL, 0},
-        [GF_DUMP_MAXVALUE] = {"MAXVALUE", GF_DUMP_MAXVALUE, NULL, NULL, NULL, 0},
+        [GF_DUMP_NULL]      = {"NULL",     GF_DUMP_NULL,     NULL,        NULL, 0, DRC_NA},
+        [GF_DUMP_DUMP]      = {"DUMP",     GF_DUMP_DUMP,     rpcsvc_dump, NULL, 0, DRC_NA},
+        [GF_DUMP_MAXVALUE]  = {"MAXVALUE", GF_DUMP_MAXVALUE, NULL,        NULL, 0, DRC_NA},
 };
 
 
diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h
index b15476704..cbc1f4226 100644
--- a/rpc/rpc-lib/src/rpcsvc.h
+++ b/rpc/rpc-lib/src/rpcsvc.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _RPCSVC_H
@@ -47,16 +38,24 @@
 #define MAX_IOVEC 16
 #endif
 
+#define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64
+#define RPCSVC_MAX_OUTSTANDING_RPC_LIMIT 65536
+#define RPCSVC_MIN_OUTSTANDING_RPC_LIMIT 0 /* No limit i.e. Unlimited */
+
 #define GF_RPCSVC       "rpc-service"
 #define RPCSVC_THREAD_STACK_SIZE ((size_t)(1024 * GF_UNIT_KB))
 
 #define RPCSVC_FRAGHDR_SIZE  4       /* 4-byte RPC fragment header size */
 #define RPCSVC_DEFAULT_LISTEN_PORT      GF_DEFAULT_BASE_PORT
-#define RPCSVC_DEFAULT_MEMFACTOR        15
+#define RPCSVC_DEFAULT_MEMFACTOR        8
 #define RPCSVC_EVENTPOOL_SIZE_MULT      1024
-#define RPCSVC_POOLCOUNT_MULT           35
+#define RPCSVC_POOLCOUNT_MULT           64
 #define RPCSVC_CONN_READ        (128 * GF_UNIT_KB)
 #define RPCSVC_PAGE_SIZE        (128 * GF_UNIT_KB)
+#define RPC_ROOT_UID             0
+#define RPC_ROOT_GID             0
+#define RPC_NOBODY_UID           65534
+#define RPC_NOBODY_GID           65534
 
 /* RPC Record States */
 #define RPCSVC_READ_FRAGHDR     1
@@ -145,6 +144,9 @@ typedef struct rpcsvc_auth_data {
 
 #define rpcsvc_auth_flavour(au)    ((au).flavour)
 
+typedef struct drc_client drc_client_t;
+typedef struct drc_cached_op drc_cached_op_t;
+
 /* The container for the RPC call handed up to an actor.
  * Dynamically allocated. Lives till the call reply is completely
  * transmitted.
@@ -183,7 +185,9 @@ struct rpcsvc_request {
         /* Might want to move this to AUTH_UNIX specific state since this array
          * is not available for every authentication scheme.
          */
-        gid_t                   auxgids[GF_MAX_AUX_GROUPS];
+        gid_t                   *auxgids;
+        gid_t                   auxgidsmall[SMALL_GROUP_COUNT];
+        gid_t                   *auxgidlarge;
         int                     auxgidcount;
 
 
@@ -233,6 +237,9 @@ struct rpcsvc_request {
          */
         rpcsvc_auth_data_t      verf;
 
+	/* Execute this request's actor function as a synctask? */
+	gf_boolean_t            synctask;
+
         /* Container for a RPC program wanting to store a temp
          * request-specific item.
          */
@@ -240,6 +247,12 @@ struct rpcsvc_request {
 
         /* Container for transport to store request-specific item */
         void                    *trans_private;
+
+        /* we need to ref the 'iobuf' in case of 'synctasking' it */
+        struct iobuf            *hdr_iobuf;
+
+        /* pointer to cached reply for use in DRC */
+        drc_cached_op_t         *reply;
 };
 
 #define rpcsvc_request_program(req) ((rpcsvc_program_t *)((req)->prog))
@@ -247,8 +260,6 @@ struct rpcsvc_request {
 #define rpcsvc_request_program_private(req) (((rpcsvc_program_t *)((req)->prog))->private)
 #define rpcsvc_request_accepted(req)    ((req)->rpc_status == MSG_ACCEPTED)
 #define rpcsvc_request_accepted_success(req) ((req)->rpc_err == SUCCESS)
-#define rpcsvc_request_uid(req)         ((req)->uid)
-#define rpcsvc_request_gid(req)         ((req)->gid)
 #define rpcsvc_request_prog_minauth(req) (rpcsvc_request_program(req)->min_auth)
 #define rpcsvc_request_cred_flavour(req) (rpcsvc_auth_flavour(req->cred))
 #define rpcsvc_request_verf_flavour(req) (rpcsvc_auth_flavour(req->verf))
@@ -265,7 +276,23 @@ struct rpcsvc_request {
 #define rpcsvc_request_set_vecstate(req, state)  ((req)->vecstate = state)
 #define rpcsvc_request_vecstate(req) ((req)->vecstate)
 #define rpcsvc_request_transport(req) ((req)->trans)
-
+#define rpcsvc_request_transport_ref(req) (rpc_transport_ref((req)->trans))
+#define RPC_AUTH_ROOT_SQUASH(req)                                       \
+        do {                                                            \
+                int gidcount = 0;                                       \
+                if (req->svc->root_squash) {                            \
+                        if (req->uid == RPC_ROOT_UID)                   \
+                                req->uid = RPC_NOBODY_UID;              \
+                        if (req->gid == RPC_ROOT_GID)                   \
+                                req->gid = RPC_NOBODY_GID;              \
+                        for (gidcount = 0; gidcount < req->auxgidcount; \
+                             ++gidcount) {                              \
+                                if (!req->auxgids[gidcount])            \
+                                        req->auxgids[gidcount] =        \
+                                                RPC_NOBODY_GID;         \
+                        }                                               \
+                }                                                       \
+        } while (0);
 
 #define RPCSVC_ACTOR_SUCCESS    0
 #define RPCSVC_ACTOR_ERROR      (-1)
@@ -284,9 +311,8 @@ struct rpcsvc_request {
  *
  */
 typedef int (*rpcsvc_actor) (rpcsvc_request_t *req);
-typedef int (*rpcsvc_vector_actor) (rpcsvc_request_t *req, struct iovec *vec,
-                                    int count, struct iobref *iobref);
-typedef int (*rpcsvc_vector_sizer) (int state, ssize_t *readsize, char *addr);
+typedef int (*rpcsvc_vector_sizer) (int state, ssize_t *readsize,
+                                    char *base_addr, char *curr_addr);
 
 /* Every protocol actor will also need to specify the function the RPC layer
  * will use to serialize or encode the message into XDR format just before
@@ -300,7 +326,6 @@ typedef void *(*rpcsvc_encode_reply) (void *msg);
  */
 typedef void (*rpcsvc_deallocate_reply) (void *msg);
 
-
 #define RPCSVC_NAME_MAX            32
 /* The descriptor for each procedure/actor that runs
  * over the RPC service.
@@ -318,11 +343,11 @@ typedef struct rpcsvc_actor_desc {
          * handler for letting the RPC program read the data from the network
          * directly into its aligned buffers.
          */
-        rpcsvc_vector_actor     vector_actor;
         rpcsvc_vector_sizer     vector_sizer;
 
         /* Can actor be ran on behalf an unprivileged requestor? */
         gf_boolean_t            unprivileged;
+        drc_op_type_t           op_type;
 } rpcsvc_actor_t;
 
 /* Describes a program and its version along with the function pointers
@@ -377,6 +402,9 @@ struct rpcsvc_program {
          */
         int                     min_auth;
 
+	/* Execute actor function as a synctask? */
+	gf_boolean_t            synctask;
+
         /* list member to link to list of registered services with rpcsvc */
         struct list_head        program;
 };
@@ -413,13 +441,20 @@ extern int
 rpcsvc_program_register_portmap (rpcsvc_program_t *newprog, uint32_t port);
 
 extern int
+rpcsvc_program_unregister_portmap (rpcsvc_program_t *newprog);
+
+extern int
 rpcsvc_register_portmap_enabled (rpcsvc_t *svc);
 
 /* Inits the global RPC service data structures.
  * Called in main.
  */
 extern rpcsvc_t *
-rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options);
+rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options,
+             uint32_t poolcount);
+
+extern int
+rpcsvc_reconfigure_options (rpcsvc_t *svc, dict_t *options);
 
 int
 rpcsvc_register_notify (rpcsvc_t *svc, rpcsvc_notify_t notify, void *mydata);
@@ -431,6 +466,13 @@ int
 rpcsvc_unregister_notify (rpcsvc_t *svc, rpcsvc_notify_t notify, void *mydata);
 
 int
+rpcsvc_transport_submit (rpc_transport_t *trans, struct iovec *rpchdr,
+                         int rpchdrcount, struct iovec *proghdr,
+                         int proghdrcount, struct iovec *progpayload,
+                         int progpayloadcount, struct iobref *iobref,
+                         void *priv);
+
+int
 rpcsvc_submit_message (rpcsvc_request_t *req, struct iovec *proghdr,
                        int hdrcount, struct iovec *payload, int payloadcount,
                        struct iobref *iobref);
@@ -451,17 +493,17 @@ rpcsvc_error_reply (rpcsvc_request_t *req);
 extern int
 rpcsvc_transport_peername (rpc_transport_t *trans, char *hostname, int hostlen);
 
-extern inline int
+extern int
 rpcsvc_transport_peeraddr (rpc_transport_t *trans, char *addrstr, int addrlen,
                            struct sockaddr_storage *returnsa, socklen_t sasize);
 
 extern int
-rpcsvc_transport_peer_check (dict_t *options, char *volname,
-                             rpc_transport_t *trans);
+rpcsvc_auth_check (rpcsvc_t *svc, char *volname, rpc_transport_t *trans);
 
 extern int
 rpcsvc_transport_privport_check (rpcsvc_t *svc, char *volname,
                                  rpc_transport_t *trans);
+
 #define rpcsvc_request_seterr(req, err)                 (req)->rpc_err = err
 #define rpcsvc_request_set_autherr(req, err)            (req)->auth_err = err
 
@@ -517,6 +559,9 @@ extern int
 rpcsvc_auth_init (rpcsvc_t *svc, dict_t *options);
 
 extern int
+rpcsvc_auth_reconf (rpcsvc_t *svc, dict_t *options);
+
+extern int
 rpcsvc_auth_transport_init (rpc_transport_t *xprt);
 
 extern int
@@ -533,9 +578,6 @@ rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen);
 extern gid_t *
 rpcsvc_auth_unix_auxgids (rpcsvc_request_t *req, int *arrlen);
 
-extern int
-rpcsvc_combine_gen_spec_volume_checks (int gen, int spec);
-
 extern char *
 rpcsvc_volume_allowed (dict_t *options, char *volname);
 
@@ -543,16 +585,22 @@ int rpcsvc_callback_submit (rpcsvc_t *rpc, rpc_transport_t *trans,
                             rpcsvc_cbk_program_t *prog, int procnum,
                             struct iovec *proghdr, int proghdrcount);
 
+rpcsvc_actor_t *
+rpcsvc_program_actor (rpcsvc_request_t *req);
+
 int
 rpcsvc_transport_unix_options_build (dict_t **options, char *filepath);
 int
 rpcsvc_set_allow_insecure (rpcsvc_t *svc, dict_t *options);
 int
+rpcsvc_set_addr_namelookup (rpcsvc_t *svc, dict_t *options);
+int
+rpcsvc_set_root_squash (rpcsvc_t *svc, dict_t *options);
+int
+rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options);
+int
 rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen);
-char *
-rpcsvc_volume_allowed (dict_t *options, char *volname);
 rpcsvc_vector_sizer
 rpcsvc_get_program_vector_sizer (rpcsvc_t *svc, uint32_t prognum,
                                  uint32_t progver, uint32_t procnum);
-
 #endif
diff --git a/rpc/rpc-lib/src/xdr-common.h b/rpc/rpc-lib/src/xdr-common.h
index d5198d20f..34dc9c6a2 100644
--- a/rpc/rpc-lib/src/xdr-common.h
+++ b/rpc/rpc-lib/src/xdr-common.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _XDR_COMMON_H_
@@ -56,6 +47,7 @@ enum gf_dump_procnum {
 #define xdr_u_quad_t xdr_u_int64_t
 #define xdr_quad_t   xdr_int64_t
 #define xdr_uint32_t xdr_u_int32_t
+#define xdr_uint64_t xdr_u_int64_t
 #endif
 
 
diff --git a/rpc/rpc-lib/src/xdr-rpc.c b/rpc/rpc-lib/src/xdr-rpc.c
index e4e5b1087..adb48a531 100644
--- a/rpc/rpc-lib/src/xdr-rpc.c
+++ b/rpc/rpc-lib/src/xdr-rpc.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -43,7 +34,7 @@ xdr_to_rpc_call (char *msgbuf, size_t len, struct rpc_msg *call,
                  struct iovec *payload, char *credbytes, char *verfbytes)
 {
         XDR                     xdr;
-        char                    opaquebytes[MAX_AUTH_BYTES];
+        char                    opaquebytes[GF_MAX_AUTH_BYTES];
         struct opaque_auth      *oa = NULL;
         int ret = -1;
 
diff --git a/rpc/rpc-lib/src/xdr-rpc.h b/rpc/rpc-lib/src/xdr-rpc.h
index 226f8e8b3..f5f4a941e 100644
--- a/rpc/rpc-lib/src/xdr-rpc.h
+++ b/rpc/rpc-lib/src/xdr-rpc.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _XDR_RPC_H_
@@ -43,7 +34,8 @@
 
 typedef enum {
         AUTH_GLUSTERFS = 5,
-        AUTH_GLUSTERFS_v2 = 6,
+        AUTH_GLUSTERFS_v2 = 390039, /* using a number from  'unused' range,
+                                       from the list available in RFC5531 */
 } gf_rpc_authtype_t;
 
 /* Converts a given network buffer from its XDR format to a structure
diff --git a/rpc/rpc-lib/src/xdr-rpcclnt.c b/rpc/rpc-lib/src/xdr-rpcclnt.c
index 69daa98cb..810d1961b 100644
--- a/rpc/rpc-lib/src/xdr-rpcclnt.c
+++ b/rpc/rpc-lib/src/xdr-rpcclnt.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
diff --git a/rpc/rpc-lib/src/xdr-rpcclnt.h b/rpc/rpc-lib/src/xdr-rpcclnt.h
index 3bcf5b54b..c08d872f8 100644
--- a/rpc/rpc-lib/src/xdr-rpcclnt.h
+++ b/rpc/rpc-lib/src/xdr-rpcclnt.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _XDR_RPCCLNT_H
diff --git a/rpc/rpc-transport/rdma/src/Makefile.am b/rpc/rpc-transport/rdma/src/Makefile.am
index b4b940bca..2bf7cf238 100644
--- a/rpc/rpc-transport/rdma/src/Makefile.am
+++ b/rpc/rpc-transport/rdma/src/Makefile.am
@@ -3,18 +3,20 @@
 transport_LTLIBRARIES = rdma.la
 transportdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/rpc-transport
 
-rdma_la_LDFLAGS = -module -avoidversion
+rdma_la_LDFLAGS = -module -avoid-version
 
 rdma_la_SOURCES = rdma.c name.c
 rdma_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la     \
-	-libverbs
+	-libverbs -lrdmacm
 
 noinst_HEADERS = rdma.h name.h
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src/ \
 	-I$(top_srcdir)/xlators/protocol/lib/src/ -shared -nostartfiles $(GF_CFLAGS)
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)  \
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src/ \
-	-I$(top_srcdir)/rpc/xdr/src -shared -nostartfiles $(GF_CFLAGS)
+	-I$(top_srcdir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = *~
diff --git a/rpc/rpc-transport/rdma/src/name.c b/rpc/rpc-transport/rdma/src/name.c
index 702c8a86a..c57428ad6 100644
--- a/rpc/rpc-transport/rdma/src/name.c
+++ b/rpc/rpc-transport/rdma/src/name.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <sys/types.h>
@@ -22,12 +13,7 @@
 #include <errno.h>
 #include <netdb.h>
 #include <string.h>
-
-#ifdef CLIENT_PORT_CEILING
-#undef CLIENT_PORT_CEILING
-#endif
-
-#define CLIENT_PORT_CEILING 1024
+#include <rdma/rdma_cma.h>
 
 #ifndef AF_INET_SDP
 #define AF_INET_SDP 27
@@ -35,6 +21,8 @@
 
 #include "rpc-transport.h"
 #include "rdma.h"
+#include "common-utils.h"
+
 
 int32_t
 gf_resolve_ip6 (const char *hostname,
@@ -44,28 +32,43 @@ gf_resolve_ip6 (const char *hostname,
                 struct addrinfo **addr_info);
 
 static int32_t
-af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
+af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id,
+                                 struct sockaddr *sockaddr,
                                  socklen_t sockaddr_len, int ceiling)
 {
-        int32_t ret = -1;
-        /*  struct sockaddr_in sin = {0, }; */
-        uint16_t port = ceiling - 1;
+        int32_t        ret        = -1;
+        uint16_t      port        = ceiling - 1;
+        // by default assume none of the ports are blocked and all are available
+        gf_boolean_t  ports[1024] = {_gf_false,};
+        int           i           = 0;
+
+        ret = gf_process_reserved_ports (ports);
+        if (ret != 0) {
+                for (i = 0; i < 1024; i++)
+                        ports[i] = _gf_false;
+        }
 
         while (port)
         {
                 switch (sockaddr->sa_family)
                 {
                 case AF_INET6:
-                        ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
+                        ((struct sockaddr_in6 *)sockaddr)->sin6_port
+                                = htons (port);
                         break;
 
                 case AF_INET_SDP:
                 case AF_INET:
-                        ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
+                        ((struct sockaddr_in *)sockaddr)->sin_port
+                                = htons (port);
                         break;
                 }
-
-                ret = bind (fd, sockaddr, sockaddr_len);
+                // ignore the reserved ports
+                if (ports[port] == _gf_true) {
+                        port--;
+                        continue;
+                }
+                ret = rdma_bind_addr (cm_id, sockaddr);
 
                 if (ret == 0)
                         break;
@@ -79,11 +82,10 @@ af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
         return ret;
 }
 
+#if 0
 static int32_t
-af_unix_client_bind (rpc_transport_t *this,
-                     struct sockaddr *sockaddr,
-                     socklen_t sockaddr_len,
-                     int sock)
+af_unix_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr,
+                     socklen_t sockaddr_len, struct rdma_cm_id *cm_id)
 {
         data_t *path_data = NULL;
         struct sockaddr_un *addr = NULL;
@@ -115,6 +117,7 @@ af_unix_client_bind (rpc_transport_t *this,
 err:
         return ret;
 }
+#endif
 
 static int32_t
 client_fill_address_family (rpc_transport_t *this, struct sockaddr *sockaddr)
@@ -162,13 +165,11 @@ client_fill_address_family (rpc_transport_t *this, struct sockaddr *sockaddr)
                         sockaddr->sa_family = AF_INET6;
                 } else if (!strcasecmp (address_family, "inet-sdp")) {
                         sockaddr->sa_family = AF_INET_SDP;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        sockaddr->sa_family = AF_UNSPEC;
                 } else {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "unknown address-family (%s) specified",
                                 address_family);
+                        sockaddr->sa_family = AF_UNSPEC;
                         return -1;
                 }
         }
@@ -363,6 +364,8 @@ af_inet_server_get_local_sockaddr (rpc_transport_t *this,
         if (listen_port_data) {
                 listen_port = data_to_uint16 (listen_port_data);
         } else {
+                listen_port = GF_DEFAULT_RDMA_LISTEN_PORT;
+
                 if (addr->sa_family == AF_INET6) {
                         struct sockaddr_in6 *in = (struct sockaddr_in6 *) addr;
                         in->sin6_addr = in6addr_any;
@@ -413,10 +416,8 @@ out:
 }
 
 int32_t
-gf_rdma_client_bind (rpc_transport_t *this,
-                     struct sockaddr *sockaddr,
-                     socklen_t *sockaddr_len,
-                     int sock)
+gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr,
+                     socklen_t *sockaddr_len, struct rdma_cm_id *cm_id)
 {
         int ret = 0;
 
@@ -428,22 +429,24 @@ gf_rdma_client_bind (rpc_transport_t *this,
                 *sockaddr_len = sizeof (struct sockaddr_in);
 
         case AF_INET6:
-                ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr,
+                ret = af_inet_bind_to_port_lt_ceiling (cm_id, sockaddr,
                                                        *sockaddr_len,
-                                                       CLIENT_PORT_CEILING);
+                                                       GF_CLIENT_PORT_CEILING);
                 if (ret == -1) {
                         gf_log (this->name, GF_LOG_WARNING,
-                                "cannot bind inet socket (%d) to port "
-                                "less than %d (%s)",
-                                sock, CLIENT_PORT_CEILING, strerror (errno));
+                                "cannot bind rdma_cm_id to port "
+                                "less than %d (%s)", GF_CLIENT_PORT_CEILING,
+                                strerror (errno));
                         ret = 0;
                 }
                 break;
 
         case AF_UNIX:
                 *sockaddr_len = sizeof (struct sockaddr_un);
+#if 0
                 ret = af_unix_client_bind (this, (struct sockaddr *)sockaddr,
                                            *sockaddr_len, sock);
+#endif
                 break;
 
         default:
@@ -530,21 +533,19 @@ gf_rdma_server_get_local_sockaddr (rpc_transport_t *this,
                         addr->sa_family = AF_INET_SDP;
                 } else if (!strcasecmp (address_family, "unix")) {
                         addr->sa_family = AF_UNIX;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        addr->sa_family = AF_UNSPEC;
                 } else {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "unknown address family (%s) specified",
                                 address_family);
+                        addr->sa_family = AF_UNSPEC;
                         ret = -1;
                         goto err;
                 }
         } else {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "option address-family not specified, defaulting "
-                        "to inet/inet6");
-                addr->sa_family = AF_UNSPEC;
+                        "to inet");
+                addr->sa_family = AF_INET;
         }
 
         switch (addr->sa_family)
diff --git a/rpc/rpc-transport/rdma/src/name.h b/rpc/rpc-transport/rdma/src/name.h
index 1f6fc89fd..742fc5fc3 100644
--- a/rpc/rpc-transport/rdma/src/name.h
+++ b/rpc/rpc-transport/rdma/src/name.h
@@ -1,35 +1,23 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _IB_VERBS_NAME_H
 #define _IB_VERBS_NAME_H
 
-#include <sys/socket.h>
-#include <sys/un.h>
+#include <rdma/rdma_cma.h>
 
 #include "compat.h"
 
 int32_t
-gf_rdma_client_bind (rpc_transport_t *this,
-                     struct sockaddr *sockaddr,
-                     socklen_t *sockaddr_len,
-                     int sock);
+gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr,
+                     socklen_t *sockaddr_len, struct rdma_cm_id *cm_id);
 
 int32_t
 gf_rdma_client_get_remote_sockaddr (rpc_transport_t *this,
diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c
index d3a8e9c8d..6e6099a98 100644
--- a/rpc/rpc-transport/rdma/src/rdma.c
+++ b/rpc/rpc-transport/rdma/src/rdma.c
@@ -1,23 +1,13 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -30,6 +20,7 @@
 #include "name.h"
 #include "byte-order.h"
 #include "xlator.h"
+#include "xdr-rpc.h"
 #include <signal.h>
 
 #define GF_RDMA_LOG_NAME "rpc-transport/rdma"
@@ -43,99 +34,29 @@ gf_rdma_post_ref (gf_rdma_post_t *post);
 int
 gf_rdma_post_unref (gf_rdma_post_t *post);
 
-int32_t
-gf_resolve_ip6 (const char *hostname,
-                uint16_t port,
-                int family,
-                void **dnscache,
-                struct addrinfo **addr_info);
-
-static uint16_t
-gf_rdma_get_local_lid (struct ibv_context *context,
-                       int32_t port)
-{
-        struct ibv_port_attr attr;
-
-        if (ibv_query_port (context, port, &attr))
-                return 0;
+static void *
+gf_rdma_send_completion_proc (void *data);
 
-        return attr.lid;
-}
+static void *
+gf_rdma_recv_completion_proc (void *data);
 
-static const char *
-get_port_state_str(enum ibv_port_state pstate)
-{
-        switch (pstate) {
-        case IBV_PORT_DOWN:          return "PORT_DOWN";
-        case IBV_PORT_INIT:          return "PORT_INIT";
-        case IBV_PORT_ARMED:         return "PORT_ARMED";
-        case IBV_PORT_ACTIVE:        return "PORT_ACTIVE";
-        case IBV_PORT_ACTIVE_DEFER:  return "PORT_ACTIVE_DEFER";
-        default:                     return "invalid state";
-        }
-}
+void *
+gf_rdma_async_event_thread (void *context);
 
 static int32_t
-ib_check_active_port (struct ibv_context *ctx, uint8_t port)
-{
-        struct ibv_port_attr  port_attr = {0, };
-        int32_t               ret       = 0;
-        const char           *state_str = NULL;
-
-        if (!ctx) {
-                gf_log_callingfn (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                  "Error in supplied context");
-                return -1;
-        }
-
-        ret = ibv_query_port (ctx, port, &port_attr);
-
-        if (ret) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                        "Failed to query port %u properties", port);
-                return -1;
-        }
-
-        state_str = get_port_state_str (port_attr.state);
-        gf_log (GF_RDMA_LOG_NAME, GF_LOG_TRACE,
-                "Infiniband PORT: (%u) STATE: (%s)",
-                port, state_str);
-
-        if (port_attr.state == IBV_PORT_ACTIVE)
-                return 0;
-
-        return -1;
-}
+gf_rdma_create_qp (rpc_transport_t *this);
 
 static int32_t
-ib_get_active_port (struct ibv_context *ib_ctx)
-{
-        struct ibv_device_attr ib_device_attr = {{0, }, };
-        int32_t                ret            = -1;
-        uint8_t                ib_port        = 0;
+__gf_rdma_teardown (rpc_transport_t *this);
 
-        if (!ib_ctx) {
-                gf_log_callingfn (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                  "Error in supplied context");
-                return -1;
-        }
-        if (ibv_query_device (ib_ctx, &ib_device_attr)) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                        "Failed to query device properties");
-                return -1;
-        }
+static int32_t
+gf_rdma_teardown (rpc_transport_t *this);
 
-        for (ib_port = 1; ib_port <= ib_device_attr.phys_port_cnt; ++ib_port) {
-                ret = ib_check_active_port (ib_ctx, ib_port);
-                if (ret == 0)
-                        return ib_port;
+static int32_t
+gf_rdma_disconnect (rpc_transport_t *this);
 
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_TRACE,
-                        "Port:(%u) not active", ib_port);
-                continue;
-        }
-        return ret;
-}
+static void
+gf_rdma_cm_handle_disconnect (rpc_transport_t *this);
 
 
 static void
@@ -165,7 +86,7 @@ gf_rdma_put_post (gf_rdma_queue_t *queue, gf_rdma_post_t *post)
 
 
 static gf_rdma_post_t *
-gf_rdma_new_post (gf_rdma_device_t *device, int32_t len,
+gf_rdma_new_post (rpc_transport_t *this, gf_rdma_device_t *device, int32_t len,
                   gf_rdma_post_type_t type)
 {
         gf_rdma_post_t *post = NULL;
@@ -192,8 +113,9 @@ gf_rdma_new_post (gf_rdma_device_t *device, int32_t len,
                                post->buf_size,
                                IBV_ACCESS_LOCAL_WRITE);
         if (!post->mr) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                        "memory registration failed");
+                gf_log (this->name, GF_LOG_WARNING,
+                        "memory registration failed (%s)",
+                        strerror (errno));
                 goto out;
         }
 
@@ -203,9 +125,7 @@ gf_rdma_new_post (gf_rdma_device_t *device, int32_t len,
         ret = 0;
 out:
         if (ret != 0) {
-                if (post->buf != NULL) {
-                        free (post->buf);
-                }
+                free (post->buf);
 
                 GF_FREE (post);
                 post = NULL;
@@ -268,22 +188,6 @@ __gf_rdma_quota_get (gf_rdma_peer_t *peer)
         return ret;
 }
 
-/*
-  static int32_t
-  gf_rdma_quota_get (gf_rdma_peer_t *peer)
-  {
-  int32_t ret = -1;
-  gf_rdma_private_t *priv = peer->trans->private;
-
-  pthread_mutex_lock (&priv->write_mutex);
-  {
-  ret = __gf_rdma_quota_get (peer);
-  }
-  pthread_mutex_unlock (&priv->write_mutex);
-
-  return ret;
-  }
-*/
 
 static void
 __gf_rdma_ioq_entry_free (gf_rdma_ioq_t *entry)
@@ -299,6 +203,7 @@ __gf_rdma_ioq_entry_free (gf_rdma_ioq_t *entry)
                 iobref_unref (entry->msg.request.rsp_iobref);
                 entry->msg.request.rsp_iobref = NULL;
         }
+
         mem_put (entry);
 }
 
@@ -318,26 +223,898 @@ static int32_t
 __gf_rdma_disconnect (rpc_transport_t *this)
 {
         gf_rdma_private_t *priv = NULL;
-        int32_t            ret  = 0;
 
         priv = this->private;
 
-        if (priv->connected || priv->tcp_connected) {
-                fcntl (priv->sock, F_SETFL, O_NONBLOCK);
-                if (shutdown (priv->sock, SHUT_RDWR) != 0) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
-                                "shutdown () - error: %s",
-                                strerror (errno));
-                        ret = -errno;
-                        priv->tcp_connected = 0;
+        if (priv->connected) {
+                rdma_disconnect (priv->peer.cm_id);
+        }
+
+        return 0;
+}
+
+
+static void
+gf_rdma_queue_init (gf_rdma_queue_t *queue)
+{
+        pthread_mutex_init (&queue->lock, NULL);
+
+        queue->active_posts.next = &queue->active_posts;
+        queue->active_posts.prev = &queue->active_posts;
+        queue->passive_posts.next = &queue->passive_posts;
+        queue->passive_posts.prev = &queue->passive_posts;
+}
+
+
+static void
+__gf_rdma_destroy_queue (gf_rdma_post_t *post)
+{
+        gf_rdma_post_t *tmp = NULL;
+
+        while (post->next != post) {
+                tmp = post->next;
+
+                post->next = post->next->next;
+                post->next->prev = post;
+
+                gf_rdma_destroy_post (tmp);
+        }
+}
+
+
+static void
+gf_rdma_destroy_queue (gf_rdma_queue_t *queue)
+{
+        if (queue == NULL) {
+                goto out;
+        }
+
+        pthread_mutex_lock (&queue->lock);
+        {
+                if (queue->passive_count > 0) {
+                        __gf_rdma_destroy_queue (&queue->passive_posts);
+                        queue->passive_count = 0;
+                }
+
+                if (queue->active_count > 0) {
+                        __gf_rdma_destroy_queue (&queue->active_posts);
+                        queue->active_count = 0;
+                }
+        }
+        pthread_mutex_unlock (&queue->lock);
+
+out:
+        return;
+}
+
+
+static void
+gf_rdma_destroy_posts (rpc_transport_t *this)
+{
+        gf_rdma_device_t  *device = NULL;
+        gf_rdma_private_t *priv   = NULL;
+
+        if (this == NULL) {
+                goto out;
+        }
+
+        priv = this->private;
+        device = priv->device;
+
+        gf_rdma_destroy_queue (&device->sendq);
+        gf_rdma_destroy_queue (&device->recvq);
+
+out:
+        return;
+}
+
+
+static int32_t
+__gf_rdma_create_posts (rpc_transport_t *this, int32_t count, int32_t size,
+                        gf_rdma_queue_t *q, gf_rdma_post_type_t type)
+{
+        int32_t            i      = 0;
+        int32_t            ret    = 0;
+        gf_rdma_private_t *priv   = NULL;
+        gf_rdma_device_t  *device = NULL;
+
+        priv = this->private;
+        device = priv->device;
+
+        for (i=0 ; i<count ; i++) {
+                gf_rdma_post_t *post = NULL;
+
+                post = gf_rdma_new_post (this, device, size + 2048, type);
+                if (!post) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "post creation failed");
+                        ret = -1;
+                        break;
+                }
+
+                gf_rdma_put_post (q, post);
+        }
+        return ret;
+}
+
+
+static int32_t
+gf_rdma_post_recv (struct ibv_srq *srq,
+                   gf_rdma_post_t *post)
+{
+        struct ibv_sge list = {
+                .addr   = (unsigned long) post->buf,
+                .length = post->buf_size,
+                .lkey   = post->mr->lkey
+        };
+
+        struct ibv_recv_wr wr = {
+                .wr_id  = (unsigned long) post,
+                .sg_list = &list,
+                .num_sge = 1,
+        }, *bad_wr;
+
+        gf_rdma_post_ref (post);
+
+        return ibv_post_srq_recv (srq, &wr, &bad_wr);
+}
+
+
+static int32_t
+gf_rdma_create_posts (rpc_transport_t *this)
+{
+        int32_t            i       = 0, ret = 0;
+        gf_rdma_post_t    *post    = NULL;
+        gf_rdma_private_t *priv    = NULL;
+        gf_rdma_options_t *options = NULL;
+        gf_rdma_device_t  *device  = NULL;
+
+        priv = this->private;
+        options = &priv->options;
+        device = priv->device;
+
+        ret =  __gf_rdma_create_posts (this, options->send_count,
+                                       options->send_size,
+                                       &device->sendq, GF_RDMA_SEND_POST);
+        if (!ret)
+                ret =  __gf_rdma_create_posts (this, options->recv_count,
+                                               options->recv_size,
+                                               &device->recvq,
+                                               GF_RDMA_RECV_POST);
+
+        if (!ret) {
+                for (i=0 ; i<options->recv_count ; i++) {
+                        post = gf_rdma_get_post (&device->recvq);
+                        if (gf_rdma_post_recv (device->srq, post) != 0) {
+                                ret = -1;
+                                break;
+                        }
+                }
+        }
+
+        if (ret)
+                gf_rdma_destroy_posts (this);
+
+        return ret;
+}
+
+
+static void
+gf_rdma_destroy_cq (rpc_transport_t *this)
+{
+        gf_rdma_private_t *priv   = NULL;
+        gf_rdma_device_t  *device = NULL;
+
+        priv = this->private;
+        device = priv->device;
+
+        if (device->recv_cq)
+                ibv_destroy_cq (device->recv_cq);
+        device->recv_cq = NULL;
+
+        if (device->send_cq)
+                ibv_destroy_cq (device->send_cq);
+        device->send_cq = NULL;
+
+        return;
+}
+
+
+static int32_t
+gf_rdma_create_cq (rpc_transport_t *this)
+{
+        gf_rdma_private_t      *priv        = NULL;
+        gf_rdma_options_t      *options     = NULL;
+        gf_rdma_device_t       *device      = NULL;
+        uint64_t                send_cqe    = 0;
+        int32_t                 ret         = 0;
+        struct ibv_device_attr  device_attr = {{0}, };
+
+        priv = this->private;
+        options = &priv->options;
+        device = priv->device;
+
+        device->recv_cq = ibv_create_cq (priv->device->context,
+                                         options->recv_count * 2,
+                                         device,
+                                         device->recv_chan,
+                                         0);
+        if (!device->recv_cq) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "creation of CQ for device %s failed",
+                        device->device_name);
+                ret = -1;
+                goto out;
+        } else if (ibv_req_notify_cq (device->recv_cq, 0)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "ibv_req_notify_cq on recv CQ of device %s failed",
+                        device->device_name);
+                ret = -1;
+                goto out;
+        }
+
+        do {
+                ret = ibv_query_device (priv->device->context, &device_attr);
+                if (ret != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "ibv_query_device on %s returned %d (%s)",
+                                priv->device->device_name, ret,
+                                (ret > 0) ? strerror (ret) : "");
+                        ret = -1;
+                        goto out;
+                }
+
+                send_cqe = options->send_count * 128;
+                send_cqe = (send_cqe > device_attr.max_cqe)
+                        ? device_attr.max_cqe : send_cqe;
+
+                /* TODO: make send_cq size dynamically adaptive */
+                device->send_cq = ibv_create_cq (priv->device->context,
+                                                 send_cqe, device,
+                                                 device->send_chan, 0);
+                if (!device->send_cq) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "creation of send_cq for device %s failed",
+                                device->device_name);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (ibv_req_notify_cq (device->send_cq, 0)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "ibv_req_notify_cq on send_cq for device %s"
+                                " failed",  device->device_name);
+                        ret = -1;
+                        goto out;
+                }
+        } while (0);
+
+out:
+        if (ret != 0)
+                gf_rdma_destroy_cq (this);
+
+        return ret;
+}
+
+
+static gf_rdma_device_t *
+gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx,
+                    char *device_name)
+{
+        glusterfs_ctx_t   *ctx      = NULL;
+        gf_rdma_private_t *priv     = NULL;
+        gf_rdma_options_t *options  = NULL;
+        int32_t            ret      = 0;
+        int32_t            i        = 0;
+        gf_rdma_device_t  *trav     = NULL, *device = NULL;
+        gf_rdma_ctx_t     *rdma_ctx = NULL;
+
+        priv        = this->private;
+        options     = &priv->options;
+        ctx         = this->ctx;
+        rdma_ctx    = ctx->ib;
+
+        trav = rdma_ctx->device;
+
+        while (trav) {
+                if (!strcmp (trav->device_name, device_name))
+                        break;
+                trav = trav->next;
+        }
+
+        if (!trav) {
+                trav = GF_CALLOC (1, sizeof (*trav),
+                                  gf_common_mt_rdma_device_t);
+                if (trav == NULL) {
+                        goto out;
+                }
+
+                priv->device = trav;
+                trav->context = ibctx;
+
+                trav->request_ctx_pool
+                        = mem_pool_new (gf_rdma_request_context_t,
+                                        GF_RDMA_POOL_SIZE);
+                if (trav->request_ctx_pool == NULL) {
+                        goto out;
+                }
+
+                trav->ioq_pool
+                        = mem_pool_new (gf_rdma_ioq_t, GF_RDMA_POOL_SIZE);
+                if (trav->ioq_pool == NULL) {
+                        goto out;
+                }
+
+                trav->reply_info_pool = mem_pool_new (gf_rdma_reply_info_t,
+                                                      GF_RDMA_POOL_SIZE);
+                if (trav->reply_info_pool == NULL) {
+                        goto out;
+                }
+
+                trav->device_name = gf_strdup (device_name);
+
+                trav->next = rdma_ctx->device;
+                rdma_ctx->device = trav;
+
+                trav->send_chan = ibv_create_comp_channel (trav->context);
+                if (!trav->send_chan) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not create send completion channel for "
+                                "device (%s)", device_name);
+                        goto out;
+                }
+
+                trav->recv_chan = ibv_create_comp_channel (trav->context);
+                if (!trav->recv_chan) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not create recv completion channel for "
+                                "device (%s)", device_name);
+
+                        /* TODO: cleanup current mess */
+                        goto out;
+                }
+
+                if (gf_rdma_create_cq (this) < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not create CQ for device (%s)",
+                                device_name);
+                        goto out;
+                }
+
+                /* protection domain */
+                trav->pd = ibv_alloc_pd (trav->context);
+
+                if (!trav->pd) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not allocate protection domain for "
+                                "device (%s)", device_name);
+                        goto out;
+                }
+
+                struct ibv_srq_init_attr attr = {
+                        .attr = {
+                                .max_wr = options->recv_count,
+                                .max_sge = 1,
+                                .srq_limit = 10
+                        }
+                };
+                trav->srq = ibv_create_srq (trav->pd, &attr);
+
+                if (!trav->srq) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not create SRQ for device (%s)",
+                                device_name);
+                        goto out;
+                }
+
+                /* queue init */
+                gf_rdma_queue_init (&trav->sendq);
+                gf_rdma_queue_init (&trav->recvq);
+
+                if (gf_rdma_create_posts (this) < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not allocate posts for device (%s)",
+                                device_name);
+                        goto out;
+                }
+
+                /* completion threads */
+                ret = gf_thread_create (&trav->send_thread, NULL,
+					gf_rdma_send_completion_proc,
+					trav->send_chan);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not create send completion thread for "
+                                "device (%s)", device_name);
+                        goto out;
+                }
+
+                ret = gf_thread_create (&trav->recv_thread, NULL,
+					 gf_rdma_recv_completion_proc,
+					 trav->recv_chan);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not create recv completion thread "
+                                "for device (%s)", device_name);
+                        return NULL;
+                }
+
+                ret = gf_thread_create (&trav->async_event_thread, NULL,
+					 gf_rdma_async_event_thread,
+					 ibctx);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not create async_event_thread");
+                        return NULL;
+                }
+
+                /* qpreg */
+                pthread_mutex_init (&trav->qpreg.lock, NULL);
+                for (i=0; i<42; i++) {
+                        trav->qpreg.ents[i].next = &trav->qpreg.ents[i];
+                        trav->qpreg.ents[i].prev = &trav->qpreg.ents[i];
+                }
+        }
+
+        device = trav;
+        trav = NULL;
+out:
+
+        if (trav != NULL) {
+                gf_rdma_destroy_posts (this);
+                mem_pool_destroy (trav->ioq_pool);
+                mem_pool_destroy (trav->request_ctx_pool);
+                mem_pool_destroy (trav->reply_info_pool);
+                ibv_dealloc_pd (trav->pd);
+                gf_rdma_destroy_cq (this);
+                ibv_destroy_comp_channel (trav->recv_chan);
+                ibv_destroy_comp_channel (trav->send_chan);
+                GF_FREE ((char *)trav->device_name);
+                GF_FREE (trav);
+        }
+
+        return device;
+}
+
+
+static rpc_transport_t *
+gf_rdma_transport_new (rpc_transport_t *listener, struct rdma_cm_id *cm_id)
+{
+        gf_rdma_private_t *listener_priv = NULL, *priv = NULL;
+        rpc_transport_t   *this          = NULL, *new = NULL;
+        gf_rdma_options_t *options       = NULL;
+        char              *device_name   = NULL;
+
+        listener_priv = listener->private;
+
+        this = GF_CALLOC (1, sizeof (rpc_transport_t),
+                          gf_common_mt_rpc_transport_t);
+        if (this == NULL) {
+                goto out;
+        }
+
+        this->listener = listener;
+
+        priv = GF_CALLOC (1, sizeof (gf_rdma_private_t),
+                          gf_common_mt_rdma_private_t);
+        if (priv == NULL) {
+                goto out;
+        }
+
+        this->private = priv;
+        priv->options = listener_priv->options;
+
+        priv->listener = listener;
+        priv->entity = GF_RDMA_SERVER;
+
+        options = &priv->options;
+
+        this->ops = listener->ops;
+        this->init = listener->init;
+        this->fini = listener->fini;
+        this->ctx = listener->ctx;
+        this->name = gf_strdup (listener->name);
+        this->notify = listener->notify;
+        this->mydata = listener->mydata;
+
+        this->myinfo.sockaddr_len = sizeof (cm_id->route.addr.src_addr);
+        memcpy (&this->myinfo.sockaddr, &cm_id->route.addr.src_addr,
+                this->myinfo.sockaddr_len);
+
+        this->peerinfo.sockaddr_len = sizeof (cm_id->route.addr.dst_addr);
+        memcpy (&this->peerinfo.sockaddr, &cm_id->route.addr.dst_addr,
+                this->peerinfo.sockaddr_len);
+
+        priv->peer.trans = this;
+        gf_rdma_get_transport_identifiers (this);
+
+        device_name = (char *)ibv_get_device_name (cm_id->verbs->device);
+        if (device_name == NULL) {
+                gf_log (listener->name, GF_LOG_WARNING,
+                        "cannot get device name (peer:%s me:%s)",
+                        this->peerinfo.identifier, this->myinfo.identifier);
+                goto out;
+        }
+
+        priv->device = gf_rdma_get_device (this, cm_id->verbs,
+                                           device_name);
+        if (priv->device == NULL) {
+                gf_log (listener->name, GF_LOG_WARNING,
+                        "cannot get infiniband device %s (peer:%s me:%s)",
+                        device_name, this->peerinfo.identifier,
+                        this->myinfo.identifier);
+                goto out;
+        }
+
+        priv->peer.send_count = options->send_count;
+        priv->peer.recv_count = options->recv_count;
+        priv->peer.send_size = options->send_size;
+        priv->peer.recv_size = options->recv_size;
+        priv->peer.cm_id = cm_id;
+        INIT_LIST_HEAD (&priv->peer.ioq);
+
+        pthread_mutex_init (&priv->write_mutex, NULL);
+        pthread_mutex_init (&priv->recv_mutex, NULL);
+
+        cm_id->context = this;
+
+        new = rpc_transport_ref (this);
+        this = NULL;
+out:
+        if (this != NULL) {
+                if (this->private != NULL) {
+                        GF_FREE (this->private);
+                }
+
+                if (this->name != NULL) {
+                        GF_FREE (this->name);
+                }
+
+                GF_FREE (this);
+        }
+
+        return new;
+}
+
+
+static int
+gf_rdma_cm_handle_connect_request (struct rdma_cm_event *event)
+{
+        int                     ret         = -1;
+        rpc_transport_t        *this        = NULL, *listener = NULL;
+        struct rdma_cm_id      *child_cm_id = NULL, *listener_cm_id = NULL;
+	struct rdma_conn_param  conn_param  = {0, };
+        gf_rdma_private_t      *priv        = NULL;
+        gf_rdma_options_t      *options     = NULL;
+
+        child_cm_id = event->id;
+        listener_cm_id = event->listen_id;
+
+        listener = listener_cm_id->context;
+        priv = listener->private;
+        options = &priv->options;
+
+        this = gf_rdma_transport_new (listener, child_cm_id);
+        if (this == NULL) {
+                gf_log (listener->name, GF_LOG_WARNING,
+                        "could not create a transport for incoming connection"
+                        " (me.name:%s me.identifier:%s)", listener->name,
+                        listener->myinfo.identifier);
+                rdma_destroy_id (child_cm_id);
+                goto out;
+        }
+
+        gf_log (listener->name, GF_LOG_TRACE,
+                "got a connect request (me:%s peer:%s)",
+                listener->myinfo.identifier, this->peerinfo.identifier);
+
+        ret = gf_rdma_create_qp (this);
+        if (ret < 0) {
+                gf_log (listener->name, GF_LOG_WARNING,
+                        "could not create QP (peer:%s me:%s)",
+                        this->peerinfo.identifier, this->myinfo.identifier);
+                gf_rdma_cm_handle_disconnect (this);
+                goto out;
+        }
+
+	conn_param.responder_resources = 1;
+	conn_param.initiator_depth = 1;
+        conn_param.retry_count = options->attr_retry_cnt;
+        conn_param.rnr_retry_count = options->attr_rnr_retry;
+
+	ret = rdma_accept(child_cm_id, &conn_param);
+	if (ret < 0) {
+                gf_log (listener->name, GF_LOG_WARNING, "rdma_accept failed "
+                        "peer:%s me:%s (%s)", this->peerinfo.identifier,
+                        this->myinfo.identifier, strerror (errno));
+                gf_rdma_cm_handle_disconnect (this);
+                goto out;
+	}
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+static int
+gf_rdma_cm_handle_route_resolved (struct rdma_cm_event *event)
+{
+	struct rdma_conn_param  conn_param = {0, };
+	int                     ret        = 0;
+        rpc_transport_t        *this       = NULL;
+        gf_rdma_private_t      *priv       = NULL;
+        gf_rdma_peer_t         *peer       = NULL;
+        gf_rdma_options_t      *options    = NULL;
+
+        if (event == NULL) {
+                goto out;
+        }
+
+        this = event->id->context;
+
+        priv = this->private;
+        peer = &priv->peer;
+        options = &priv->options;
+
+        ret = gf_rdma_create_qp (this);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "could not create QP (peer:%s me:%s)",
+                        this->peerinfo.identifier, this->myinfo.identifier);
+                gf_rdma_cm_handle_disconnect (this);
+                goto out;
+        }
+
+	memset(&conn_param, 0, sizeof conn_param);
+	conn_param.responder_resources = 1;
+	conn_param.initiator_depth = 1;
+	conn_param.retry_count = options->attr_retry_cnt;
+        conn_param.rnr_retry_count = options->attr_rnr_retry;
+
+	ret = rdma_connect(peer->cm_id, &conn_param);
+	if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "rdma_connect failed (%s)", strerror (errno));
+                gf_rdma_cm_handle_disconnect (this);
+                goto out;
+	}
+
+        gf_log (this->name, GF_LOG_TRACE, "route resolved (me:%s peer:%s)",
+                this->myinfo.identifier, this->peerinfo.identifier);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+static int
+gf_rdma_cm_handle_addr_resolved (struct rdma_cm_event *event)
+{
+        rpc_transport_t   *this = NULL;
+        gf_rdma_peer_t    *peer = NULL;
+        gf_rdma_private_t *priv = NULL;
+        int                ret  = 0;
+
+        this = event->id->context;
+
+        priv = this->private;
+        peer = &priv->peer;
+
+        GF_ASSERT (peer->cm_id == event->id);
+
+        this->myinfo.sockaddr_len = sizeof (peer->cm_id->route.addr.src_addr);
+        memcpy (&this->myinfo.sockaddr, &peer->cm_id->route.addr.src_addr,
+                this->myinfo.sockaddr_len);
+
+        this->peerinfo.sockaddr_len = sizeof (peer->cm_id->route.addr.dst_addr);
+        memcpy (&this->peerinfo.sockaddr, &peer->cm_id->route.addr.dst_addr,
+                this->peerinfo.sockaddr_len);
+
+        gf_rdma_get_transport_identifiers (this);
+
+        ret = rdma_resolve_route(peer->cm_id, 2000);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "rdma_resolve_route failed (me:%s peer:%s) (%s)",
+                        this->myinfo.identifier, this->peerinfo.identifier,
+                        strerror (errno));
+                gf_rdma_cm_handle_disconnect (this);
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Address resolved (me:%s peer:%s)",
+                this->myinfo.identifier, this->peerinfo.identifier);
+
+        return ret;
+}
+
+
+static void
+gf_rdma_cm_handle_disconnect (rpc_transport_t *this)
+{
+        gf_rdma_private_t *priv       = NULL;
+        char               need_unref = 0, connected = 0;
+
+        priv = this->private;
+        gf_log (this->name, GF_LOG_DEBUG,
+                "peer disconnected, cleaning up");
+
+        pthread_mutex_lock (&priv->write_mutex);
+        {
+                if (priv->peer.cm_id != NULL) {
+                        need_unref = 1;
+                        connected = priv->connected;
                         priv->connected = 0;
                 }
+
+                __gf_rdma_teardown (this);
+        }
+        pthread_mutex_unlock (&priv->write_mutex);
+
+        if (connected) {
+                rpc_transport_notify (this, RPC_TRANSPORT_DISCONNECT, this);
         }
 
+        if (need_unref)
+                rpc_transport_unref (this);
+
+}
+
+
+static int
+gf_rdma_cm_handle_event_established (struct rdma_cm_event *event)
+{
+        rpc_transport_t   *this  = NULL;
+        gf_rdma_private_t *priv  = NULL;
+        struct rdma_cm_id *cm_id = NULL;
+        int                ret   = 0;
+
+        cm_id = event->id;
+        this = cm_id->context;
+        priv = this->private;
+
+        priv->connected = 1;
+
+        pthread_mutex_lock (&priv->write_mutex);
+        {
+                priv->peer.quota = 1;
+                priv->peer.quota_set = 0;
+        }
+        pthread_mutex_unlock (&priv->write_mutex);
+
+        if (priv->entity == GF_RDMA_CLIENT) {
+                ret = rpc_transport_notify (this, RPC_TRANSPORT_CONNECT, this);
+
+        } else if (priv->entity == GF_RDMA_SERVER) {
+                ret = rpc_transport_notify (priv->listener,
+                                            RPC_TRANSPORT_ACCEPT, this);
+        }
+
+        if (ret < 0) {
+                gf_rdma_disconnect (this);
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "recieved event RDMA_CM_EVENT_ESTABLISHED (me:%s peer:%s)",
+                this->myinfo.identifier, this->peerinfo.identifier);
+
         return ret;
 }
 
 
+static int
+gf_rdma_cm_handle_event_error (rpc_transport_t *this)
+{
+        gf_rdma_private_t *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->entity != GF_RDMA_SERVER_LISTENER) {
+                gf_rdma_cm_handle_disconnect (this);
+        }
+
+        return 0;
+}
+
+
+static int
+gf_rdma_cm_handle_device_removal (struct rdma_cm_event *event)
+{
+        return 0;
+}
+
+
+static void *
+gf_rdma_cm_event_handler (void *data)
+{
+        struct rdma_cm_event      *event         = NULL;
+        int                        ret           = 0;
+        rpc_transport_t           *this          = NULL;
+        struct rdma_event_channel *event_channel = NULL;
+
+        event_channel = data;
+
+        while (1) {
+                ret = rdma_get_cm_event (event_channel, &event);
+                if (ret != 0) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "rdma_cm_get_event failed (%s)",
+                                strerror (errno));
+                        break;
+                }
+
+                switch (event->event) {
+                case RDMA_CM_EVENT_ADDR_RESOLVED:
+                        gf_rdma_cm_handle_addr_resolved (event);
+                        break;
+
+                case RDMA_CM_EVENT_ROUTE_RESOLVED:
+                        gf_rdma_cm_handle_route_resolved (event);
+                        break;
+
+                case RDMA_CM_EVENT_CONNECT_REQUEST:
+                        gf_rdma_cm_handle_connect_request (event);
+                        break;
+
+                case RDMA_CM_EVENT_ESTABLISHED:
+                        gf_rdma_cm_handle_event_established (event);
+                        break;
+
+                case RDMA_CM_EVENT_ADDR_ERROR:
+                case RDMA_CM_EVENT_ROUTE_ERROR:
+                case RDMA_CM_EVENT_CONNECT_ERROR:
+                case RDMA_CM_EVENT_UNREACHABLE:
+                case RDMA_CM_EVENT_REJECTED:
+                        this = event->id->context;
+
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "cma event %s, error %d (me:%s peer:%s)\n",
+                                rdma_event_str(event->event), event->status,
+                                this->myinfo.identifier,
+                                this->peerinfo.identifier);
+
+                        rdma_ack_cm_event (event);
+                        event = NULL;
+
+                        gf_rdma_cm_handle_event_error (this);
+                        continue;
+
+                case RDMA_CM_EVENT_DISCONNECTED:
+                        this = event->id->context;
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "recieved disconnect (me:%s peer:%s)\n",
+                                this->myinfo.identifier,
+                                this->peerinfo.identifier);
+
+                        rdma_ack_cm_event (event);
+                        event = NULL;
+
+                        gf_rdma_cm_handle_disconnect (this);
+                        continue;
+
+                case RDMA_CM_EVENT_DEVICE_REMOVAL:
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "device removed");
+                        gf_rdma_cm_handle_device_removal (event);
+                        break;
+
+                default:
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "unhandled event: %s, ignoring",
+                                rdma_event_str(event->event));
+                        break;
+                }
+
+                rdma_ack_cm_event (event);
+        }
+
+        return NULL;
+}
+
+
 static int32_t
 gf_rdma_post_send (struct ibv_qp *qp, gf_rdma_post_t *post, int32_t len)
 {
@@ -363,33 +1140,33 @@ gf_rdma_post_send (struct ibv_qp *qp, gf_rdma_post_t *post, int32_t len)
 
 int
 __gf_rdma_encode_error(gf_rdma_peer_t *peer, gf_rdma_reply_info_t *reply_info,
-                       struct iovec *rpchdr, uint32_t *ptr,
+                       struct iovec *rpchdr, gf_rdma_header_t *hdr,
                        gf_rdma_errcode_t err)
 {
-        uint32_t       *startp  = NULL;
         struct rpc_msg *rpc_msg = NULL;
 
-        startp = ptr;
         if (reply_info != NULL) {
-                *ptr++ = hton32(reply_info->rm_xid);
+                hdr->rm_xid = hton32(reply_info->rm_xid);
         } else {
                 rpc_msg = rpchdr[0].iov_base; /* assume rpchdr contains
                                                * only one vector.
                                                * (which is true)
                                                */
-                *ptr++ = rpc_msg->rm_xid;
+                hdr->rm_xid = rpc_msg->rm_xid;
         }
 
-        *ptr++ = hton32(GF_RDMA_VERSION);
-        *ptr++ = hton32(peer->send_count);
-        *ptr++ = hton32(GF_RDMA_ERROR);
-        *ptr++ = hton32(err);
+        hdr->rm_vers = hton32(GF_RDMA_VERSION);
+        hdr->rm_credit = hton32(peer->send_count);
+        hdr->rm_type = hton32(GF_RDMA_ERROR);
+        hdr->rm_body.rm_error.rm_type = hton32(err);
         if (err == ERR_VERS) {
-                *ptr++ = hton32(GF_RDMA_VERSION);
-                *ptr++ = hton32(GF_RDMA_VERSION);
+                hdr->rm_body.rm_error.rm_version.gf_rdma_vers_low
+                        = hton32(GF_RDMA_VERSION);
+                hdr->rm_body.rm_error.rm_version.gf_rdma_vers_high
+                        = hton32(GF_RDMA_VERSION);
         }
 
-        return (int)((unsigned long)ptr - (unsigned long)startp);
+        return sizeof (*hdr);
 }
 
 
@@ -401,7 +1178,7 @@ __gf_rdma_send_error (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
         int32_t  ret = -1, len = 0;
 
         len = __gf_rdma_encode_error (peer, reply_info, entry->rpchdr,
-                                      (uint32_t *)post->buf, err);
+                                      (gf_rdma_header_t *)post->buf, err);
         if (len == -1) {
                 gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
                         "encode error returned -1");
@@ -460,8 +1237,10 @@ __gf_rdma_create_read_chunks_from_vector (gf_rdma_peer_t *peer,
                                  vector[i].iov_len,
                                  IBV_ACCESS_REMOTE_READ);
                 if (!mr) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "memory registration failed");
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "memory registration failed (%s) (peer:%s)",
+                                strerror (errno),
+                                peer->trans->peerinfo.identifier);
                         goto out;
                 }
 
@@ -510,8 +1289,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
                                                                 entry->rpchdr_count,
                                                                 request_ctx);
                 if (ret == -1) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
-                                "cannot create read chunks from vector, "
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "cannot create read chunks from vector "
                                 "entry->rpchdr");
                         goto out;
                 }
@@ -523,8 +1302,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
                                                                 entry->proghdr_count,
                                                                 request_ctx);
                 if (ret == -1) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
-                                "cannot create read chunks from vector, "
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "cannot create read chunks from vector "
                                 "entry->proghdr");
                 }
 
@@ -536,8 +1315,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
                                                                         entry->prog_payload_count,
                                                                         request_ctx);
                         if (ret == -1) {
-                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
-                                        "cannot create read chunks from vector,"
+                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                        "cannot create read chunks from vector"
                                         " entry->prog_payload");
                         }
                 }
@@ -550,8 +1329,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
                                                                 entry->prog_payload_count,
                                                                 request_ctx);
                 if (ret == -1) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
-                                "cannot create read chunks from vector, "
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "cannot create read chunks from vector "
                                 "entry->prog_payload");
                 }
         }
@@ -594,8 +1373,10 @@ __gf_rdma_create_write_chunks_from_vector (gf_rdma_peer_t *peer,
                                  IBV_ACCESS_REMOTE_WRITE
                                  | IBV_ACCESS_LOCAL_WRITE);
                 if (!mr) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "memory registration failed");
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "memory registration failed (%s) (peer:%s)",
+                                strerror (errno),
+                                peer->trans->peerinfo.identifier);
                         goto out;
                 }
 
@@ -670,7 +1451,7 @@ __gf_rdma_create_write_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
                                                                  entry->msg.request.rsp_payload_count,
                                                                  request_ctx);
                 if (ret == -1) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                                 "cannot create write chunks from vector "
                                 "entry->rpc_payload");
                         goto out;
@@ -700,7 +1481,7 @@ __gf_rdma_create_write_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
                                                                  entry->msg.request.rsphdr_count,
                                                                  request_ctx);
                 if (ret == -1) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                                 "cannot create write chunks from vector "
                                 "entry->rpchdr");
                         goto out;
@@ -716,7 +1497,7 @@ out:
 }
 
 
-inline void
+static inline void
 __gf_rdma_deregister_mr (struct ibv_mr **mr, int count)
 {
         int i = 0;
@@ -837,28 +1618,6 @@ out:
 }
 
 
-static int32_t
-gf_rdma_post_recv (struct ibv_srq *srq,
-                   gf_rdma_post_t *post)
-{
-        struct ibv_sge list = {
-                .addr   = (unsigned long) post->buf,
-                .length = post->buf_size,
-                .lkey   = post->mr->lkey
-        };
-
-        struct ibv_recv_wr wr = {
-                .wr_id  = (unsigned long) post,
-                .sg_list = &list,
-                .num_sge = 1,
-        }, *bad_wr;
-
-        gf_rdma_post_ref (post);
-
-        return ibv_post_srq_recv (srq, &wr, &bad_wr);
-}
-
-
 int
 gf_rdma_post_unref (gf_rdma_post_t *post)
 {
@@ -1044,10 +1803,11 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
 
         chunkptr = &hdr->rm_body.rm_chunks[0];
         if (rtype != gf_rdma_noch) {
-                ret = __gf_rdma_create_read_chunks (peer, entry, rtype, &chunkptr,
+                ret = __gf_rdma_create_read_chunks (peer, entry, rtype,
+                                                    &chunkptr,
                                                     request_ctx);
                 if (ret != 0) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                                 "creation of read chunks failed");
                         goto out;
                 }
@@ -1056,10 +1816,11 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
         }
 
         if (wtype != gf_rdma_noch) {
-                ret = __gf_rdma_create_write_chunks (peer, entry, wtype, &chunkptr,
+                ret = __gf_rdma_create_write_chunks (peer, entry, wtype,
+                                                     &chunkptr,
                                                      request_ctx);
                 if (ret != 0) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                                 "creation of write/reply chunk failed");
                         goto out;
                 }
@@ -1117,7 +1878,7 @@ out:
 }
 
 
-inline void
+static inline void
 __gf_rdma_fill_reply_header (gf_rdma_header_t *header, struct iovec *rpchdr,
                              gf_rdma_reply_info_t *reply_info, int credits)
 {
@@ -1147,11 +1908,12 @@ __gf_rdma_fill_reply_header (gf_rdma_header_t *header, struct iovec *rpchdr,
 
 int32_t
 __gf_rdma_send_reply_inline (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
-                             gf_rdma_post_t *post, gf_rdma_reply_info_t *reply_info)
+                             gf_rdma_post_t *post,
+                             gf_rdma_reply_info_t *reply_info)
 {
-        gf_rdma_header_t  *header    = NULL;
-        int32_t         send_size = 0, ret = 0;
-        char           *buf       = NULL;
+        gf_rdma_header_t *header    = NULL;
+        int32_t           send_size = 0, ret = 0;
+        char             *buf       = NULL;
 
         send_size = iov_length (entry->rpchdr, entry->rpchdr_count)
                 + iov_length (entry->proghdr, entry->proghdr_count)
@@ -1164,6 +1926,10 @@ __gf_rdma_send_reply_inline (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
         if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) {
                 ret = __gf_rdma_send_error (peer, entry, post, reply_info,
                                             ERR_CHUNK);
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "msg size (%d) is greater than maximum size "
+                        "of msg that can be sent inlined (%d)",
+                        send_size, GLUSTERFS_RDMA_INLINE_THRESHOLD);
                 goto out;
         }
 
@@ -1198,7 +1964,7 @@ __gf_rdma_send_reply_inline (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
                 ret = send_size;
         } else {
                 gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
-                        "gf_rdma_post_send (to %s) failed with ret = %d (%s)",
+                        "posting send (to %s) failed with ret = %d (%s)",
                         peer->trans->peerinfo.identifier, ret,
                         (ret > 0) ? strerror (ret) : "");
                 gf_rdma_post_unref (post);
@@ -1271,9 +2037,8 @@ __gf_rdma_register_local_mr_for_rdma (gf_rdma_peer_t *peer,
         gf_rdma_private_t *priv   = NULL;
         gf_rdma_device_t  *device = NULL;
 
-        if ((ctx == NULL) || (vector == NULL)) {
-                goto out;
-        }
+        GF_VALIDATE_OR_GOTO (GF_RDMA_LOG_NAME, ctx, out);
+        GF_VALIDATE_OR_GOTO (GF_RDMA_LOG_NAME, vector, out);
 
         priv = peer->trans->private;
         device = priv->device;
@@ -1294,6 +2059,9 @@ __gf_rdma_register_local_mr_for_rdma (gf_rdma_peer_t *peer,
                                                      vector[i].iov_len,
                                                      IBV_ACCESS_LOCAL_WRITE);
                 if (ctx->mr[ctx->mr_count] == NULL) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "registering memory for IBV_ACCESS_LOCAL_WRITE "
+                                "failed (%s)", strerror (errno));
                         goto out;
                 }
 
@@ -1364,7 +2132,8 @@ __gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post, struct iovec *vec,
 
         ret = ibv_post_send(peer->qp, &wr, &bad_wr);
         if (ret) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, "rdma write to "
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "rdma write to "
                         "client (%s) failed with ret = %d (%s)",
                         peer->trans->peerinfo.identifier, ret,
                         (ret > 0) ? strerror (ret) : "");
@@ -1399,6 +2168,8 @@ __gf_rdma_do_gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
         ret = __gf_rdma_register_local_mr_for_rdma (peer, vector, count,
                                                     &post->ctx);
         if (ret == -1) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "registering memory region for rdma failed");
                 goto out;
         }
 
@@ -1410,9 +2181,13 @@ __gf_rdma_do_gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
                 xfer_len = min (payload_size,
                                 reply_info->wc_array->wc_array[i].wc_target.rs_length);
 
-                ret = __gf_rdma_write (peer, post, vector, xfer_len, &payload_idx,
+                ret = __gf_rdma_write (peer, post, vector, xfer_len,
+                                       &payload_idx,
                                        &reply_info->wc_array->wc_array[i]);
                 if (ret == -1) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "rdma write to client (%s) failed",
+                                peer->trans->peerinfo.identifier);
                         goto out;
                 }
 
@@ -1431,12 +2206,12 @@ __gf_rdma_send_reply_type_nomsg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
                                  gf_rdma_post_t *post,
                                  gf_rdma_reply_info_t *reply_info)
 {
-        gf_rdma_header_t      *header       = NULL;
-        char               *buf          = NULL;
-        uint32_t            payload_size = 0;
-        int                 count        = 0, i = 0;
-        int32_t             ret          = 0;
-        struct iovec        vector[MAX_IOVEC];
+        gf_rdma_header_t *header       = NULL;
+        char             *buf          = NULL;
+        uint32_t          payload_size = 0;
+        int               count        = 0, i = 0;
+        int32_t           ret          = 0;
+        struct iovec      vector[MAX_IOVEC];
 
         header = (gf_rdma_header_t *)post->buf;
 
@@ -1451,9 +2226,10 @@ __gf_rdma_send_reply_type_nomsg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
         /* encode reply chunklist */
         buf = (char *)&header->rm_body.rm_chunks[2];
         ret = __gf_rdma_reply_encode_write_chunks (peer, payload_size, post,
-                                                   reply_info, (uint32_t **)&buf);
+                                                   reply_info,
+                                                   (uint32_t **)&buf);
         if (ret == -1) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                         "encoding write chunks failed");
                 ret = __gf_rdma_send_error (peer, entry, post, reply_info,
                                             ERR_CHUNK);
@@ -1473,6 +2249,9 @@ __gf_rdma_send_reply_type_nomsg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
         ret = __gf_rdma_do_gf_rdma_write (peer, post, vector, count,
                                           entry->iobref, reply_info);
         if (ret == -1) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "rdma write to peer (%s) failed",
+                        peer->trans->peerinfo.identifier);
                 gf_rdma_post_unref (post);
                 goto out;
         }
@@ -1480,7 +2259,7 @@ __gf_rdma_send_reply_type_nomsg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
         ret = gf_rdma_post_send (peer->qp, post, (buf - post->buf));
         if (ret) {
                 gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
-                        "gf_rdma_post_send to client (%s) failed with "
+                        "posting a send request to client (%s) failed with "
                         "ret = %d (%s)", peer->trans->peerinfo.identifier, ret,
                         (ret > 0) ? strerror (ret) : "");
                 ret = -1;
@@ -1496,12 +2275,13 @@ out:
 
 int32_t
 __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
-                               gf_rdma_post_t *post, gf_rdma_reply_info_t *reply_info)
+                               gf_rdma_post_t *post,
+                               gf_rdma_reply_info_t *reply_info)
 {
-        gf_rdma_header_t      *header       = NULL;
-        int32_t             send_size    = 0, ret = 0;
-        char               *ptr          = NULL;
-        uint32_t            payload_size = 0;
+        gf_rdma_header_t *header       = NULL;
+        int32_t           send_size    = 0, ret = 0;
+        char             *ptr          = NULL;
+        uint32_t          payload_size = 0;
 
         send_size = iov_length (entry->rpchdr, entry->rpchdr_count)
                 + iov_length (entry->proghdr, entry->proghdr_count)
@@ -1533,7 +2313,7 @@ __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
                                                    reply_info,
                                                    (uint32_t **)&ptr);
         if (ret == -1) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                         "encoding write chunks failed");
                 ret = __gf_rdma_send_error (peer, entry, post, reply_info,
                                             ERR_CHUNK);
@@ -1549,6 +2329,8 @@ __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
                                           entry->prog_payload_count,
                                           entry->iobref, reply_info);
         if (ret == -1) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, "rdma write to peer "
+                        "(%s) failed", peer->trans->peerinfo.identifier);
                 gf_rdma_post_unref (post);
                 goto out;
         }
@@ -1623,9 +2405,9 @@ __gf_rdma_ioq_churn_reply (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
         int32_t               ret        = -1;
         gf_rdma_chunktype_t   type       = gf_rdma_noch;
 
-        if ((peer == NULL) || (entry == NULL) || (post == NULL)) {
-                goto out;
-        }
+        GF_VALIDATE_OR_GOTO (GF_RDMA_LOG_NAME, peer, out);
+        GF_VALIDATE_OR_GOTO (GF_RDMA_LOG_NAME, entry, out);
+        GF_VALIDATE_OR_GOTO (GF_RDMA_LOG_NAME, post, out);
 
         reply_info = entry->msg.reply_info;
         if (reply_info != NULL) {
@@ -1636,22 +2418,39 @@ __gf_rdma_ioq_churn_reply (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
         case gf_rdma_noch:
                 ret = __gf_rdma_send_reply_inline (peer, entry, post,
                                                    reply_info);
+                if (ret < 0) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "failed to send reply to peer (%s) as an "
+                                "inlined rdma msg",
+                                peer->trans->peerinfo.identifier);
+                }
                 break;
 
         case gf_rdma_replych:
                 ret = __gf_rdma_send_reply_type_nomsg (peer, entry, post,
                                                        reply_info);
+                if (ret < 0) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "failed to send reply to peer (%s) as "
+                                "RDMA_NOMSG", peer->trans->peerinfo.identifier);
+                }
                 break;
 
         case gf_rdma_writech:
                 ret = __gf_rdma_send_reply_type_msg (peer, entry, post,
                                                      reply_info);
+                if (ret < 0) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "failed to send reply with write chunks "
+                                "to peer (%s)",
+                                peer->trans->peerinfo.identifier);
+                }
                 break;
 
         default:
                 gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
-                        "invalid chunktype (%d) specified for sending reply",
-                        type);
+                        "invalid chunktype (%d) specified for sending reply "
+                        " (peer:%s)", type, peer->trans->peerinfo.identifier);
                 break;
         }
 
@@ -1680,20 +2479,34 @@ __gf_rdma_ioq_churn_entry (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry)
         if (quota > 0) {
                 post = gf_rdma_get_post (&device->sendq);
                 if (post == NULL) {
-                        post = gf_rdma_new_post (device,
+                        post = gf_rdma_new_post (peer->trans, device,
                                                  (options->send_size + 2048),
                                                  GF_RDMA_SEND_POST);
                 }
 
                 if (post == NULL) {
                         ret = -1;
+                        gf_log_callingfn (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                          "not able to get a post to send msg");
                         goto out;
                 }
 
                 if (entry->is_request) {
                         ret = __gf_rdma_ioq_churn_request (peer, entry, post);
+                        if (ret < 0) {
+                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                        "failed to process request ioq entry "
+                                        "to peer(%s)",
+                                        peer->trans->peerinfo.identifier);
+                        }
                 } else {
                         ret = __gf_rdma_ioq_churn_reply (peer, entry, post);
+                        if (ret < 0) {
+                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                        "failed to process reply ioq entry "
+                                        "to peer (%s)",
+                                        peer->trans->peerinfo.identifier);
+                        }
                 }
 
                 if (ret != 0) {
@@ -1749,9 +2562,9 @@ gf_rdma_writev (rpc_transport_t *this, gf_rdma_ioq_t *entry)
         pthread_mutex_lock (&priv->write_mutex);
         {
                 if (!priv->connected) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "rdma is not connected to post a "
-                                "send request");
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "rdma is not connected to peer (%s)",
+                                this->peerinfo.identifier);
                         ret = -1;
                         goto unlock;
                 }
@@ -1761,6 +2574,13 @@ gf_rdma_writev (rpc_transport_t *this, gf_rdma_ioq_t *entry)
                         ret = __gf_rdma_ioq_churn_entry (peer, entry);
                         if (ret != 0) {
                                 need_append = 0;
+
+                                if (ret < 0) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "processing ioq entry destined "
+                                                "to (%s) failed",
+                                                this->peerinfo.identifier);
+                                }
                         }
                 }
 
@@ -1880,6 +2700,9 @@ gf_rdma_submit_request (rpc_transport_t *this, rpc_transport_req_t *req)
 
         entry = gf_rdma_ioq_new (this, &data);
         if (entry == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "getting a new ioq entry failed (peer:%s)",
+                        this->peerinfo.identifier);
                 goto out;
         }
 
@@ -1888,6 +2711,9 @@ gf_rdma_submit_request (rpc_transport_t *this, rpc_transport_req_t *req)
         if (ret > 0) {
                 ret = 0;
         } else if (ret < 0) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "sending request to peer (%s) failed",
+                        this->peerinfo.identifier);
                 rpc_transport_disconnect (this);
         }
 
@@ -1910,6 +2736,9 @@ gf_rdma_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply)
 
         entry = gf_rdma_ioq_new (this, &data);
         if (entry == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "getting a new ioq entry failed (peer:%s)",
+                        this->peerinfo.identifier);
                 goto out;
         }
 
@@ -1917,6 +2746,9 @@ gf_rdma_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply)
         if (ret > 0) {
                 ret = 0;
         } else if (ret < 0) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "sending request to peer (%s) failed",
+                        this->peerinfo.identifier);
                 rpc_transport_disconnect (this);
         }
 
@@ -1924,189 +2756,6 @@ out:
         return ret;
 }
 
-#if 0
-static int
-gf_rdma_receive (rpc_transport_t *this, char **hdr_p, size_t *hdrlen_p,
-                 struct iobuf **iobuf_p)
-{
-        gf_rdma_private_t *priv                   = this->private;
-        /* TODO: return error if !priv->connected, check with locks */
-        /* TODO: boundry checks for data_ptr/offset */
-        char              *copy_from              = NULL;
-        gf_rdma_header_t  *header                 = NULL;
-        uint32_t           size1, size2, data_len = 0;
-        char              *hdr                    = NULL;
-        struct iobuf      *iobuf                  = NULL;
-        int32_t            ret                    = 0;
-
-        pthread_mutex_lock (&priv->recv_mutex);
-        {
-/*
-  while (!priv->data_ptr)
-  pthread_cond_wait (&priv->recv_cond, &priv->recv_mutex);
-*/
-
-                copy_from = priv->data_ptr + priv->data_offset;
-
-                priv->data_ptr = NULL;
-                data_len = priv->data_len;
-                pthread_cond_broadcast (&priv->recv_cond);
-        }
-        pthread_mutex_unlock (&priv->recv_mutex);
-
-        header = (gf_rdma_header_t *)copy_from;
-        if (strcmp (header->colonO, ":O")) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
-                        "%s: corrupt header received", this->name);
-                ret = -1;
-                goto err;
-        }
-
-        size1 = ntoh32 (header->size1);
-        size2 = ntoh32 (header->size2);
-
-        if (data_len != (size1 + size2 + sizeof (*header))) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
-                        "%s: sizeof data read from transport is not equal "
-                        "to the size specified in the header",
-                        this->name);
-                ret = -1;
-                goto err;
-        }
-
-        copy_from += sizeof (*header);
-
-        if (size1) {
-                hdr = GF_CALLOC (1, size1, gf_common_mt_char);
-                if (!hdr) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "unable to allocate header for peer %s",
-                                this->peerinfo.identifier);
-                        ret = -ENOMEM;
-                        goto err;
-                }
-                memcpy (hdr, copy_from, size1);
-                copy_from += size1;
-                *hdr_p = hdr;
-        }
-        *hdrlen_p = size1;
-
-        if (size2) {
-                iobuf = iobuf_get (this->ctx->iobuf_pool);
-                if (!iobuf) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "unable to allocate IO buffer for peer %s",
-                                this->peerinfo.identifier);
-                        ret = -ENOMEM;
-                        goto err;
-                }
-                memcpy (iobuf->ptr, copy_from, size2);
-                *iobuf_p = iobuf;
-        }
-
-err:
-        return ret;
-}
-#endif
-
-
-static void
-gf_rdma_destroy_cq (rpc_transport_t *this)
-{
-        gf_rdma_private_t *priv   = NULL;
-        gf_rdma_device_t  *device = NULL;
-
-        priv = this->private;
-        device = priv->device;
-
-        if (device->recv_cq)
-                ibv_destroy_cq (device->recv_cq);
-        device->recv_cq = NULL;
-
-        if (device->send_cq)
-                ibv_destroy_cq (device->send_cq);
-        device->send_cq = NULL;
-
-        return;
-}
-
-
-static int32_t
-gf_rdma_create_cq (rpc_transport_t *this)
-{
-        gf_rdma_private_t      *priv        = NULL;
-        gf_rdma_options_t      *options     = NULL;
-        gf_rdma_device_t       *device      = NULL;
-        uint64_t                send_cqe    = 0;
-        int32_t                 ret         = 0;
-        struct ibv_device_attr  device_attr = {{0}, };
-
-        priv = this->private;
-        options = &priv->options;
-        device = priv->device;
-
-        device->recv_cq = ibv_create_cq (priv->device->context,
-                                         options->recv_count * 2,
-                                         device,
-                                         device->recv_chan,
-                                         0);
-        if (!device->recv_cq) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                        "%s: creation of CQ for device %s failed",
-                        this->name, device->device_name);
-                ret = -1;
-                goto out;
-        } else if (ibv_req_notify_cq (device->recv_cq, 0)) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                        "%s: ibv_req_notify_cq on recv CQ of device %s failed",
-                        this->name, device->device_name);
-                ret = -1;
-                goto out;
-        }
-
-        do {
-                ret = ibv_query_device (priv->device->context, &device_attr);
-                if (ret != 0) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "%s: ibv_query_device on %s returned %d (%s)",
-                                this->name, priv->device->device_name, ret,
-                                (ret > 0) ? strerror (ret) : "");
-                        ret = -1;
-                        goto out;
-                }
-
-                send_cqe = options->send_count * 128;
-                send_cqe = (send_cqe > device_attr.max_cqe)
-                        ? device_attr.max_cqe : send_cqe;
-
-                /* TODO: make send_cq size dynamically adaptive */
-                device->send_cq = ibv_create_cq (priv->device->context,
-                                                 send_cqe, device,
-                                                 device->send_chan, 0);
-                if (!device->send_cq) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "%s: creation of send_cq for device %s failed",
-                                this->name, device->device_name);
-                        ret = -1;
-                        goto out;
-                }
-
-                if (ibv_req_notify_cq (device->send_cq, 0)) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "%s: ibv_req_notify_cq on send_cq for device %s"
-                                " failed", this->name, device->device_name);
-                        ret = -1;
-                        goto out;
-                }
-        } while (0);
-
-out:
-        if (ret != 0)
-                gf_rdma_destroy_cq (this);
-
-        return ret;
-}
-
 
 static int
 gf_rdma_register_peer (gf_rdma_device_t *device, int32_t qp_num,
@@ -2205,25 +2854,6 @@ __gf_rdma_lookup_peer (gf_rdma_device_t *device, int32_t qp_num)
         return peer;
 }
 
-/*
-  static gf_rdma_peer_t *
-  gf_rdma_lookup_peer (gf_rdma_device_t *device,
-  int32_t qp_num)
-  {
-  gf_rdma_qpreg_t *qpreg = NULL;
-  gf_rdma_peer_t  *peer  = NULL;
-
-  qpreg = &device->qpreg;
-  pthread_mutex_lock (&qpreg->lock);
-  {
-  peer = __gf_rdma_lookup_peer (device, qp_num);
-  }
-  pthread_mutex_unlock (&qpreg->lock);
-
-  return peer;
-  }
-*/
-
 
 static void
 __gf_rdma_destroy_qp (rpc_transport_t *this)
@@ -2233,7 +2863,7 @@ __gf_rdma_destroy_qp (rpc_transport_t *this)
         priv = this->private;
         if (priv->peer.qp) {
                 gf_rdma_unregister_peer (priv->device, priv->peer.qp->qp_num);
-                ibv_destroy_qp (priv->peer.qp);
+                rdma_destroy_qp (priv->peer.cm_id);
         }
         priv->peer.qp = NULL;
 
@@ -2244,18 +2874,36 @@ __gf_rdma_destroy_qp (rpc_transport_t *this)
 static int32_t
 gf_rdma_create_qp (rpc_transport_t *this)
 {
-        gf_rdma_private_t *priv    = NULL;
-        gf_rdma_options_t *options = NULL;
-        gf_rdma_device_t  *device  = NULL;
-        int32_t            ret     = 0;
-        gf_rdma_peer_t    *peer    = NULL;
+        gf_rdma_private_t *priv        = NULL;
+        gf_rdma_device_t  *device      = NULL;
+        int32_t            ret         = 0;
+        gf_rdma_peer_t    *peer        = NULL;
+        char              *device_name = NULL;
 
         priv = this->private;
-        options = &priv->options;
-        device = priv->device;
 
         peer = &priv->peer;
 
+        device_name = (char *)ibv_get_device_name (peer->cm_id->verbs->device);
+        if (device_name == NULL) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_WARNING, "cannot get device_name");
+                goto out;
+        }
+
+        device = gf_rdma_get_device (this, peer->cm_id->verbs,
+                                     device_name);
+        if (device == NULL) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_WARNING, "cannot get device for "
+                        "device %s", device_name);
+                goto out;
+        }
+
+        if (priv->device == NULL) {
+                priv->device = device;
+        }
+
         struct ibv_qp_init_attr init_attr = {
                 .send_cq        = device->send_cq,
                 .recv_cq        = device->recv_cq,
@@ -2269,39 +2917,16 @@ gf_rdma_create_qp (rpc_transport_t *this)
                 .qp_type = IBV_QPT_RC
         };
 
-        struct ibv_qp_attr attr = {
-                .qp_state        = IBV_QPS_INIT,
-                .pkey_index      = 0,
-                .port_num        = options->port,
-                .qp_access_flags
-                = IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE
-        };
-
-        peer->qp = ibv_create_qp (device->pd, &init_attr);
-        if (!peer->qp) {
-                gf_log (GF_RDMA_LOG_NAME,
-                        GF_LOG_CRITICAL,
-                        "%s: could not create QP",
-                        this->name);
-                ret = -1;
-                goto out;
-        } else if (ibv_modify_qp (peer->qp, &attr,
-                                  IBV_QP_STATE              |
-                                  IBV_QP_PKEY_INDEX         |
-                                  IBV_QP_PORT               |
-                                  IBV_QP_ACCESS_FLAGS)) {
-                gf_log (GF_RDMA_LOG_NAME,
-                        GF_LOG_ERROR,
-                        "%s: failed to modify QP to INIT state",
-                        this->name);
+        ret = rdma_create_qp(peer->cm_id, device->pd, &init_attr);
+        if (ret != 0) {
+                gf_log (peer->trans->name, GF_LOG_CRITICAL,
+                        "%s: could not create QP (%s)", this->name,
+                        strerror (errno));
                 ret = -1;
                 goto out;
         }
 
-        peer->local_lid = gf_rdma_get_local_lid (device->context,
-                                                 options->port);
-        peer->local_qpn = peer->qp->qp_num;
-        peer->local_psn = lrand48 () & 0xffffff;
+        peer->qp = peer->cm_id->qp;
 
         ret = gf_rdma_register_peer (device, peer->qp->qp_num, peer);
 
@@ -2313,300 +2938,52 @@ out:
 }
 
 
-static void
-gf_rdma_destroy_posts (rpc_transport_t *this)
-{
-
-}
-
-
-static int32_t
-__gf_rdma_create_posts (rpc_transport_t *this, int32_t count, int32_t size,
-                        gf_rdma_queue_t *q, gf_rdma_post_type_t type)
-{
-        int32_t            i      = 0;
-        int32_t            ret    = 0;
-        gf_rdma_private_t *priv   = NULL;
-        gf_rdma_device_t  *device = NULL;
-
-        priv = this->private;
-        device = priv->device;
-
-        for (i=0 ; i<count ; i++) {
-                gf_rdma_post_t *post = NULL;
-
-                post = gf_rdma_new_post (device, size + 2048, type);
-                if (!post) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "%s: post creation failed",
-                                this->name);
-                        ret = -1;
-                        break;
-                }
-
-                gf_rdma_put_post (q, post);
-        }
-        return ret;
-}
-
-
 static int32_t
-gf_rdma_create_posts (rpc_transport_t *this)
+__gf_rdma_teardown (rpc_transport_t *this)
 {
-        int32_t            i       = 0, ret = 0;
-        gf_rdma_post_t    *post    = NULL;
-        gf_rdma_private_t *priv    = NULL;
-        gf_rdma_options_t *options = NULL;
-        gf_rdma_device_t  *device  = NULL;
+        gf_rdma_private_t *priv = NULL;
+        gf_rdma_peer_t    *peer = NULL;
 
         priv = this->private;
-        options = &priv->options;
-        device = priv->device;
-
-        ret =  __gf_rdma_create_posts (this, options->send_count,
-                                       options->send_size,
-                                       &device->sendq, GF_RDMA_SEND_POST);
-        if (!ret)
-                ret =  __gf_rdma_create_posts (this, options->recv_count,
-                                               options->recv_size,
-                                               &device->recvq,
-                                               GF_RDMA_RECV_POST);
+        peer = &priv->peer;
 
-        if (!ret) {
-                for (i=0 ; i<options->recv_count ; i++) {
-                        post = gf_rdma_get_post (&device->recvq);
-                        if (gf_rdma_post_recv (device->srq, post) != 0) {
-                                ret = -1;
-                                break;
-                        }
-                }
+        if (peer->cm_id->qp != NULL) {
+                __gf_rdma_destroy_qp (this);
         }
 
-        if (ret)
-                gf_rdma_destroy_posts (this);
-
-        return ret;
-}
-
-
-static int32_t
-gf_rdma_connect_qp (rpc_transport_t *this)
-{
-        gf_rdma_private_t *priv = this->private;
-        gf_rdma_options_t *options = &priv->options;
-        struct ibv_qp_attr attr = {
-                .qp_state               = IBV_QPS_RTR,
-                .path_mtu               = options->mtu,
-                .dest_qp_num            = priv->peer.remote_qpn,
-                .rq_psn                 = priv->peer.remote_psn,
-                .max_dest_rd_atomic     = 1,
-                .min_rnr_timer          = 12,
-                .qp_access_flags
-                = IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE,
-                .ah_attr                = {
-                        .is_global      = 0,
-                        .dlid           = priv->peer.remote_lid,
-                        .sl             = 0,
-                        .src_path_bits  = 0,
-                        .port_num       = options->port
-                }
-        };
-        if (ibv_modify_qp (priv->peer.qp, &attr,
-                           IBV_QP_STATE              |
-                           IBV_QP_AV                 |
-                           IBV_QP_PATH_MTU           |
-                           IBV_QP_DEST_QPN           |
-                           IBV_QP_RQ_PSN             |
-                           IBV_QP_MAX_DEST_RD_ATOMIC |
-                           IBV_QP_MIN_RNR_TIMER)) {
-                gf_log (GF_RDMA_LOG_NAME,
-                        GF_LOG_CRITICAL,
-                        "Failed to modify QP to RTR\n");
-                return -1;
+        if (!list_empty (&priv->peer.ioq)) {
+                __gf_rdma_ioq_flush (peer);
         }
 
-        attr.qp_state       = IBV_QPS_RTS;
-        attr.timeout        = options->attr_timeout;
-        attr.retry_cnt      = options->attr_retry_cnt;
-        attr.rnr_retry      = options->attr_rnr_retry;
-        attr.sq_psn         = priv->peer.local_psn;
-        attr.max_rd_atomic  = 1;
-        if (ibv_modify_qp (priv->peer.qp, &attr,
-                           IBV_QP_STATE              |
-                           IBV_QP_TIMEOUT            |
-                           IBV_QP_RETRY_CNT          |
-                           IBV_QP_RNR_RETRY          |
-                           IBV_QP_SQ_PSN             |
-                           IBV_QP_MAX_QP_RD_ATOMIC)) {
-                gf_log (GF_RDMA_LOG_NAME,
-                        GF_LOG_CRITICAL,
-                        "Failed to modify QP to RTS\n");
-                return -1;
+        if (peer->cm_id != NULL) {
+                rdma_destroy_id (peer->cm_id);
+                peer->cm_id = NULL;
         }
 
+        /* TODO: decrement cq size */
         return 0;
 }
 
+
 static int32_t
-__gf_rdma_teardown (rpc_transport_t *this)
+gf_rdma_teardown (rpc_transport_t *this)
 {
+        int32_t ret = 0;
         gf_rdma_private_t *priv = NULL;
 
-        priv = this->private;
-        __gf_rdma_destroy_qp (this);
-
-        if (!list_empty (&priv->peer.ioq)) {
-                __gf_rdma_ioq_flush (&priv->peer);
+        if (this == NULL) {
+                goto out;
         }
 
-        /* TODO: decrement cq size */
-        return 0;
-}
-
-/*
- * return value:
- *   0 = success (completed)
- *  -1 = error
- * > 0 = incomplete
- */
-
-static int
-__tcp_rwv (rpc_transport_t *this, struct iovec *vector, int count,
-           struct iovec **pending_vector, int *pending_count,
-           int write)
-{
-        gf_rdma_private_t *priv     = NULL;
-        int                sock     = -1;
-        int                ret      = -1;
-        struct iovec      *opvector = NULL;
-        int                opcount  = 0;
-        int                moved    = 0;
-
         priv = this->private;
-        sock = priv->sock;
-        opvector = vector;
-        opcount = count;
 
-        while (opcount)
+        pthread_mutex_lock (&priv->write_mutex);
         {
-                if (write)
-                {
-                        ret = writev (sock, opvector, opcount);
-
-                        if (ret == 0 || (ret == -1 && errno == EAGAIN))
-                        {
-                                /* done for now */
-                                break;
-                        }
-                }
-                else
-                {
-                        ret = readv (sock, opvector, opcount);
-
-                        if (ret == -1 && errno == EAGAIN)
-                        {
-                                /* done for now */
-                                break;
-                        }
-                }
-
-                if (ret == 0)
-                {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "EOF from peer %s", this->peerinfo.identifier);
-                        opcount = -1;
-                        errno = ENOTCONN;
-                        break;
-                }
-
-                if (ret == -1)
-                {
-                        if (errno == EINTR)
-                                continue;
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s failed (%s)", write ? "writev" : "readv",
-                                strerror (errno));
-                        if (write && !priv->connected &&
-                            (errno == ECONNREFUSED))
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "possible mismatch of 'rpc-transport-type'"
-                                        " in protocol server and client. "
-                                        "check volume file");
-                        opcount = -1;
-                        break;
-                }
-
-                moved = 0;
-
-                while (moved < ret)
-                {
-                        if ((ret - moved) >= opvector[0].iov_len)
-                        {
-                                moved += opvector[0].iov_len;
-                                opvector++;
-                                opcount--;
-                        }
-                        else
-                        {
-                                opvector[0].iov_len -= (ret - moved);
-                                opvector[0].iov_base += (ret - moved);
-                                moved += (ret - moved);
-                        }
-                        while (opcount && !opvector[0].iov_len)
-                        {
-                                opvector++;
-                                opcount--;
-                        }
-                }
-        }
-
-        if (pending_vector)
-                *pending_vector = opvector;
-
-        if (pending_count)
-                *pending_count = opcount;
-
-        return opcount;
-}
-
-
-static int
-__tcp_readv (rpc_transport_t *this, struct iovec *vector, int count,
-             struct iovec **pending_vector, int *pending_count)
-{
-        int ret = -1;
-
-        ret = __tcp_rwv (this, vector, count,
-                         pending_vector, pending_count, 0);
-
-        return ret;
-}
-
-
-static int
-__tcp_writev (rpc_transport_t *this, struct iovec *vector, int count,
-              struct iovec **pending_vector, int *pending_count)
-{
-        int                ret  = -1;
-        gf_rdma_private_t *priv = NULL;
-
-        priv = this->private;
-
-        ret = __tcp_rwv (this, vector, count, pending_vector,
-                         pending_count, 1);
-
-        if (ret > 0) {
-                /* TODO: Avoid multiple calls when socket is already
-                   registered for POLLOUT */
-                priv->idx = event_select_on (this->ctx->event_pool,
-                                             priv->sock, priv->idx, -1, 1);
-        } else if (ret == 0) {
-                priv->idx = event_select_on (this->ctx->event_pool,
-                                             priv->sock,
-                                             priv->idx, -1, 0);
+                ret = __gf_rdma_teardown (this);
         }
+        pthread_mutex_unlock (&priv->write_mutex);
 
+out:
         return ret;
 }
 
@@ -2701,10 +3078,11 @@ inline int32_t
 gf_rdma_decode_error_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
                           size_t bytes_in_post)
 {
-        gf_rdma_header_t *header = NULL;
-        struct iobuf     *iobuf  = NULL;
-        struct iobref    *iobref = NULL;
-        int32_t           ret    = -1;
+        gf_rdma_header_t *header  = NULL;
+        struct iobuf     *iobuf   = NULL;
+        struct iobref    *iobref  = NULL;
+        int32_t           ret     = -1;
+        struct rpc_msg    rpc_msg = {0, };
 
         header = (gf_rdma_header_t *)post->buf;
         header->rm_body.rm_error.rm_type
@@ -2716,7 +3094,11 @@ gf_rdma_decode_error_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
                         ntoh32 (header->rm_body.rm_error.rm_version.gf_rdma_vers_high);
         }
 
-        iobuf = iobuf_get (peer->trans->ctx->iobuf_pool);
+        rpc_msg.rm_xid = header->rm_xid;
+        rpc_msg.rm_direction = REPLY;
+        rpc_msg.rm_reply.rp_stat = MSG_DENIED;
+
+        iobuf = iobuf_get2 (peer->trans->ctx->iobuf_pool, bytes_in_post);
         if (iobuf == NULL) {
                 ret = -1;
                 goto out;
@@ -2730,15 +3112,15 @@ gf_rdma_decode_error_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
 
         iobref_add (iobref, iobuf);
         iobuf_unref (iobuf);
-        /*
-         * FIXME: construct an appropriate rpc-msg here, what is being sent
-         * to rpc is not correct.
-         */
-        post->ctx.vector[0].iov_base = iobuf_ptr (iobuf);
-        post->ctx.vector[0].iov_len = bytes_in_post;
-
-        memcpy (post->ctx.vector[0].iov_base, (char *)post->buf,
-                post->ctx.vector[0].iov_len);
+
+        ret = rpc_reply_to_xdr (&rpc_msg, iobuf_ptr (iobuf),
+                                iobuf_pagesize (iobuf), &post->ctx.vector[0]);
+        if (ret == -1) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "Failed to create RPC reply");
+                goto out;
+        }
+
         post->ctx.count = 1;
 
         iobuf = NULL;
@@ -2776,6 +3158,8 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
 
         ret = gf_rdma_get_read_chunklist (&ptr, readch);
         if (ret == -1) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "cannot get read chunklist from msg");
                 goto out;
         }
 
@@ -2784,6 +3168,8 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
 
         ret = gf_rdma_get_write_chunklist (&ptr, &write_ary);
         if (ret == -1) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "cannot get write chunklist from msg");
                 goto out;
         }
 
@@ -2793,6 +3179,8 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
         if (write_ary != NULL) {
                 reply_info = gf_rdma_reply_info_alloc (peer);
                 if (reply_info == NULL) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "reply_info_alloc failed");
                         ret = -1;
                         goto out;
                 }
@@ -2803,12 +3191,16 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
         } else {
                 ret = gf_rdma_get_write_chunklist (&ptr, &write_ary);
                 if (ret == -1) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "cannot get reply chunklist from msg");
                         goto out;
                 }
 
                 if (write_ary != NULL) {
                         reply_info = gf_rdma_reply_info_alloc (peer);
                         if (reply_info == NULL) {
+                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                        "reply_info_alloc_failed");
                                 ret = -1;
                                 goto out;
                         }
@@ -2822,15 +3214,17 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
         /* skip terminator of reply chunk */
         ptr = ptr + sizeof (uint32_t);
         if (header->rm_type != GF_RDMA_NOMSG) {
-                post->ctx.hdr_iobuf = iobuf_get (peer->trans->ctx->iobuf_pool);
+                header_len = (long)ptr - (long)post->buf;
+                post->ctx.vector[0].iov_len = (bytes_in_post - header_len);
+
+                post->ctx.hdr_iobuf = iobuf_get2 (peer->trans->ctx->iobuf_pool,
+                                                  (bytes_in_post - header_len));
                 if (post->ctx.hdr_iobuf == NULL) {
                         ret = -1;
                         goto out;
                 }
 
-                header_len = (long)ptr - (long)post->buf;
                 post->ctx.vector[0].iov_base = iobuf_ptr (post->ctx.hdr_iobuf);
-                post->ctx.vector[0].iov_len = bytes_in_post - header_len;
                 memcpy (post->ctx.vector[0].iov_base, ptr,
                         post->ctx.vector[0].iov_len);
                 post->ctx.count = 1;
@@ -2844,9 +3238,7 @@ out:
                         *readch = NULL;
                 }
 
-                if (write_ary != NULL) {
-                        GF_FREE (write_ary);
-                }
+                GF_FREE (write_ary);
         }
 
         return ret;
@@ -2872,28 +3264,36 @@ gf_rdma_decode_header (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
         case GF_RDMA_MSG:
         case GF_RDMA_NOMSG:
                 ret = gf_rdma_decode_msg (peer, post, readch, bytes_in_post);
+                if (ret < 0) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "cannot decode msg of type (%d)",
+                                header->rm_type);
+                }
+
                 break;
 
         case GF_RDMA_MSGP:
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                         "rdma msg of msg-type GF_RDMA_MSGP should not have "
                         "been received");
                 ret = -1;
                 break;
 
         case GF_RDMA_DONE:
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                         "rdma msg of msg-type GF_RDMA_DONE should not have "
                         "been received");
                 ret = -1;
                 break;
 
         case GF_RDMA_ERROR:
-                /* ret = gf_rdma_decode_error_msg (peer, post, bytes_in_post); */
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "received a msg of type RDMA_ERROR");
+                ret = gf_rdma_decode_error_msg (peer, post, bytes_in_post);
                 break;
 
         default:
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                         "unknown rdma msg-type (%d)", header->rm_type);
         }
 
@@ -2911,6 +3311,8 @@ __gf_rdma_read (gf_rdma_peer_t *peer, gf_rdma_post_t *post, struct iovec *to,
 
         ret = __gf_rdma_register_local_mr_for_rdma (peer, to, 1, &post->ctx);
         if (ret == -1) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "registering local memory for rdma read failed");
                 goto out;
         }
 
@@ -2928,7 +3330,8 @@ __gf_rdma_read (gf_rdma_peer_t *peer, gf_rdma_post_t *post, struct iovec *to,
 
         ret = ibv_post_send (peer->qp, &wr, &bad_wr);
         if (ret) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG, "rdma read from client "
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "rdma read from client "
                         "(%s) failed with ret = %d (%s)",
                         peer->trans->peerinfo.identifier,
                         ret, (ret > 0) ? strerror (ret) : "");
@@ -2957,7 +3360,7 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
         }
 
         if (i == 0) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                         "message type specified as rdma-read but there are no "
                         "rdma read-chunks present");
                 goto out;
@@ -2965,16 +3368,7 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
 
         post->ctx.gf_rdma_reads = i;
 
-        if (size > peer->trans->ctx->page_size) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                        "total size of rdma-read (%lu) is greater than "
-                        "page-size (%lu). This is not supported till variable "
-                        "sized iobufs are implemented", (unsigned long)size,
-                        (unsigned long)peer->trans->ctx->page_size);
-                goto out;
-        }
-
-        iobuf = iobuf_get (peer->trans->ctx->iobuf_pool);
+        iobuf = iobuf_get2 (peer->trans->ctx->iobuf_pool, size);
         if (iobuf == NULL) {
                 goto out;
         }
@@ -2996,6 +3390,10 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
         pthread_mutex_lock (&priv->write_mutex);
         {
                 if (!priv->connected) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "transport not connected to peer (%s), "
+                                "not doing rdma reads",
+                                peer->trans->peerinfo.identifier);
                         goto unlock;
                 }
 
@@ -3009,6 +3407,9 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
                                               &post->ctx.vector[count],
                                               &readch[i]);
                         if (ret == -1) {
+                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                        "rdma read from peer (%s) failed",
+                                        peer->trans->peerinfo.identifier);
                                 goto unlock;
                         }
 
@@ -3115,6 +3516,10 @@ gf_rdma_pollin_notify (gf_rdma_peer_t *peer, gf_rdma_post_t *post)
 
         ret = rpc_transport_notify (peer->trans, RPC_TRANSPORT_MSG_RECEIVED,
                                     pollin);
+        if (ret < 0) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "transport_notify failed");
+        }
 
 out:
         if (pollin != NULL) {
@@ -3175,15 +3580,15 @@ gf_rdma_recv_reply (gf_rdma_peer_t *peer, gf_rdma_post_t *post)
                                     RPC_TRANSPORT_MAP_XID_REQUEST,
                                     &request_info);
         if (ret == -1) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
-                        "cannot get request information from rpc "
-                        "layer");
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "cannot get request information (peer:%s) from rpc "
+                        "layer", peer->trans->peerinfo.identifier);
                 goto out;
         }
 
         rpc_req = request_info.rpc_req;
         if (rpc_req == NULL) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                         "rpc request structure not found");
                 ret = -1;
                 goto out;
@@ -3201,6 +3606,10 @@ gf_rdma_recv_reply (gf_rdma_peer_t *peer, gf_rdma_post_t *post)
 out:
         if (ret == 0) {
                 ret = gf_rdma_pollin_notify (peer, post);
+                if (ret < 0) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "pollin notify failed");
+                }
         }
 
         return ret;
@@ -3215,10 +3624,15 @@ gf_rdma_recv_request (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
 
         if (readch != NULL) {
                 ret = gf_rdma_do_reads (peer, post, readch);
+                if (ret < 0) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "rdma read from peer (%s) failed",
+                                peer->trans->peerinfo.identifier);
+                }
         } else {
                 ret = gf_rdma_pollin_notify (peer, post);
                 if (ret == -1) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                                 "pollin notification failed");
                 }
         }
@@ -3235,23 +3649,44 @@ gf_rdma_process_recv (gf_rdma_peer_t *peer, struct ibv_wc *wc)
         uint32_t             *ptr      = NULL;
         enum msg_type         msg_type = 0;
         gf_rdma_header_t     *header   = NULL;
+        gf_rdma_private_t    *priv     = NULL;
 
         post = (gf_rdma_post_t *) (long) wc->wr_id;
         if (post == NULL) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                         "no post found in successful work completion element");
                 goto out;
         }
 
         ret = gf_rdma_decode_header (peer, post, &readch, wc->byte_len);
         if (ret == -1) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                         "decoding of header failed");
                 goto out;
         }
 
         header = (gf_rdma_header_t *)post->buf;
 
+	priv = peer->trans->private;
+
+        pthread_mutex_lock (&priv->write_mutex);
+        {
+                if (!priv->peer.quota_set) {
+                        priv->peer.quota_set = 1;
+
+                        /* Initially peer.quota is set to 1 as per RFC 5666. We
+                         * have to account for the quota used while sending
+                         * first msg (which may or may not be returned to pool
+                         * at this point) while deriving peer.quota from
+                         * header->rm_credit. Hence the arithmatic below,
+                         * instead of directly setting it to header->rm_credit.
+                         */
+                        priv->peer.quota = header->rm_credit
+                                - ( 1 - priv->peer.quota);
+                }
+        }
+        pthread_mutex_unlock (&priv->write_mutex);
+
         switch (header->rm_type) {
         case GF_RDMA_MSG:
                 ptr = (uint32_t *)post->ctx.vector[0].iov_base;
@@ -3267,30 +3702,48 @@ gf_rdma_process_recv (gf_rdma_peer_t *peer, struct ibv_wc *wc)
                 break;
 
         case GF_RDMA_ERROR:
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                        "an error has happened while transmission of msg, "
-                        "disconnecting the transport");
-                rpc_transport_disconnect (peer->trans);
-                goto out;
-
-/*                ret = gf_rdma_pollin_notify (peer, post);
-                  if (ret == -1) {
-                  gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
-                  "pollin notification failed");
-                  }
-                  goto out;
-*/
+                if (header->rm_body.rm_error.rm_type == ERR_CHUNK) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "peer (%s), couldn't encode or decode the msg "
+                                "properly or write chunks were not provided "
+                                "for replies that were bigger than "
+                                "RDMA_INLINE_THRESHOLD (%d)",
+                                peer->trans->peerinfo.identifier,
+                                GLUSTERFS_RDMA_INLINE_THRESHOLD);
+                        ret = gf_rdma_pollin_notify (peer, post);
+                        if (ret == -1) {
+                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                                        "pollin notification failed");
+                        }
+                        goto out;
+                } else {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
+                                "an error has happened while transmission of "
+                                "msg, disconnecting the transport");
+                        ret = -1;
+                        goto out;
+                }
 
         default:
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                         "invalid rdma msg-type (%d)", header->rm_type);
-                break;
+                goto out;
         }
 
         if (msg_type == CALL) {
                 ret = gf_rdma_recv_request (peer, post, readch);
+                if (ret < 0) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "receiving a request from peer (%s) failed",
+                                peer->trans->peerinfo.identifier);
+                }
         } else {
                 ret = gf_rdma_recv_reply (peer, post);
+                if (ret < 0) {
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "receiving a reply from peer (%s) failed",
+                                peer->trans->peerinfo.identifier);
+                }
         }
 
 out:
@@ -3301,6 +3754,42 @@ out:
         return;
 }
 
+void *
+gf_rdma_async_event_thread (void *context)
+{
+        struct ibv_async_event event;
+        int ret;
+
+        while (1) {
+                do {
+                        ret = ibv_get_async_event((struct ibv_context *)context,
+                                                  &event);
+
+                        if (ret && errno != EINTR) {
+                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                        "Error getting event (%s)",
+                                        strerror (errno));
+                        }
+                } while(ret && errno == EINTR);
+
+                switch (event.event_type) {
+                case IBV_EVENT_SRQ_LIMIT_REACHED:
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                                "recieved srq_limit reached");
+                        break;
+
+                default:
+                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+                                "event (%d) recieved", event.event_type);
+                        break;
+                }
+
+                ibv_ack_async_event(&event);
+        }
+
+        return 0;
+}
+
 
 static void *
 gf_rdma_recv_completion_proc (void *data)
@@ -3419,7 +3908,7 @@ gf_rdma_handle_failed_send_completion (gf_rdma_peer_t *peer, struct ibv_wc *wc)
 
         post = (gf_rdma_post_t *) (long) wc->wr_id;
 
-        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
+        gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
                 "send work request on `%s' returned error "
                 "wc.status = %d, wc.vendor_err = %d, post->buf = %p, "
                 "wc.byte_len = %d, post->reused = %d",
@@ -3686,364 +4175,87 @@ gf_rdma_options_init (rpc_transport_t *this)
         return;
 }
 
-static void
-gf_rdma_queue_init (gf_rdma_queue_t *queue)
-{
-        pthread_mutex_init (&queue->lock, NULL);
-
-        queue->active_posts.next = &queue->active_posts;
-        queue->active_posts.prev = &queue->active_posts;
-        queue->passive_posts.next = &queue->passive_posts;
-        queue->passive_posts.prev = &queue->passive_posts;
-}
-
 
-static gf_rdma_device_t *
-gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx)
+gf_rdma_ctx_t *
+__gf_rdma_ctx_create (void)
 {
-        glusterfs_ctx_t   *ctx         = NULL;
-        gf_rdma_private_t *priv        = NULL;
-        gf_rdma_options_t *options     = NULL;
-        char              *device_name = NULL;
-        uint32_t           port        = 0;
-        uint8_t            active_port = 0;
-        int32_t            ret         = 0;
-        int32_t            i           = 0;
-        gf_rdma_device_t  *trav        = NULL;
+        gf_rdma_ctx_t *rdma_ctx = NULL;
+        int            ret      = -1;
 
-        priv        = this->private;
-        options     = &priv->options;
-        device_name = priv->options.device_name;
-        ctx         = this->ctx;
-        trav        = ctx->ib;
-        port        = priv->options.port;
-
-        while (trav) {
-                if ((!strcmp (trav->device_name, device_name)) &&
-                    (trav->port == port))
-                        break;
-                trav = trav->next;
+        rdma_ctx = GF_CALLOC (1, sizeof (*rdma_ctx), gf_common_mt_char);
+        if (rdma_ctx == NULL) {
+                goto out;
         }
 
-        if (!trav) {
-
-                trav = GF_CALLOC (1, sizeof (*trav),
-                                  gf_common_mt_rdma_device_t);
-                if (trav == NULL) {
-                        return NULL;
-                }
-
-                priv->device = trav;
-
-                trav->context = ibctx;
-
-                ret = ib_get_active_port (trav->context);
-
-                if (ret < 0) {
-                        if (!port) {
-                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                        "Failed to find any active ports and "
-                                        "none specified in volume file,"
-                                        " exiting");
-                                GF_FREE (trav);
-                                return NULL;
-                        }
-                }
-
-                trav->request_ctx_pool = mem_pool_new (gf_rdma_request_context_t,
-                                                       GF_RDMA_POOL_SIZE);
-                if (trav->request_ctx_pool == NULL) {
-                        return NULL;
-                }
-
-                trav->ioq_pool = mem_pool_new (gf_rdma_ioq_t, GF_RDMA_POOL_SIZE);
-                if (trav->ioq_pool == NULL) {
-                        mem_pool_destroy (trav->request_ctx_pool);
-                        return NULL;
-                }
-
-                trav->reply_info_pool = mem_pool_new (gf_rdma_reply_info_t,
-                                                      GF_RDMA_POOL_SIZE);
-                if (trav->reply_info_pool == NULL) {
-                        mem_pool_destroy (trav->request_ctx_pool);
-                        mem_pool_destroy (trav->ioq_pool);
-                        return NULL;
-                }
-
-
-                active_port = ret;
-
-                if (port) {
-                        ret = ib_check_active_port (trav->context, port);
-                        if (ret < 0) {
-                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
-                                        "On device %s: provided port:%u is "
-                                        "found to be offline, continuing to "
-                                        "use the same port", device_name, port);
-                        }
-                } else {
-                        priv->options.port = active_port;
-                        port = active_port;
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_TRACE,
-                                "Port unspecified in volume file using active "
-                                "port: %u", port);
-                }
-
-                trav->device_name = gf_strdup (device_name);
-                trav->port = port;
-
-                trav->next = ctx->ib;
-                ctx->ib = trav;
-
-                trav->send_chan = ibv_create_comp_channel (trav->context);
-                if (!trav->send_chan) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "%s: could not create send completion channel",
-                                device_name);
-                        mem_pool_destroy (trav->ioq_pool);
-                        mem_pool_destroy (trav->request_ctx_pool);
-                        mem_pool_destroy (trav->reply_info_pool);
-                        GF_FREE ((char *)trav->device_name);
-                        GF_FREE (trav);
-                        return NULL;
-                }
-
-                trav->recv_chan = ibv_create_comp_channel (trav->context);
-                if (!trav->recv_chan) {
-                        mem_pool_destroy (trav->ioq_pool);
-                        mem_pool_destroy (trav->request_ctx_pool);
-                        mem_pool_destroy (trav->reply_info_pool);
-                        ibv_destroy_comp_channel (trav->send_chan);
-                        GF_FREE ((char *)trav->device_name);
-                        GF_FREE (trav);
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "could not create recv completion channel");
-                        /* TODO: cleanup current mess */
-                        return NULL;
-                }
-
-                if (gf_rdma_create_cq (this) < 0) {
-                        mem_pool_destroy (trav->ioq_pool);
-                        mem_pool_destroy (trav->request_ctx_pool);
-                        mem_pool_destroy (trav->reply_info_pool);
-                        ibv_destroy_comp_channel (trav->recv_chan);
-                        ibv_destroy_comp_channel (trav->send_chan);
-                        GF_FREE ((char *)trav->device_name);
-                        GF_FREE (trav);
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "%s: could not create CQ",
-                                this->name);
-                        return NULL;
-                }
-
-                /* protection domain */
-                trav->pd = ibv_alloc_pd (trav->context);
-
-                if (!trav->pd) {
-                        mem_pool_destroy (trav->ioq_pool);
-                        mem_pool_destroy (trav->request_ctx_pool);
-                        mem_pool_destroy (trav->reply_info_pool);
-                        gf_rdma_destroy_cq (this);
-                        ibv_destroy_comp_channel (trav->recv_chan);
-                        ibv_destroy_comp_channel (trav->send_chan);
-                        GF_FREE ((char *)trav->device_name);
-                        GF_FREE (trav);
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "%s: could not allocate protection domain",
-                                this->name);
-                        return NULL;
-                }
-
-                struct ibv_srq_init_attr attr = {
-                        .attr = {
-                                .max_wr = options->recv_count,
-                                .max_sge = 1
-                        }
-                };
-                trav->srq = ibv_create_srq (trav->pd, &attr);
-
-                if (!trav->srq) {
-                        mem_pool_destroy (trav->ioq_pool);
-                        mem_pool_destroy (trav->request_ctx_pool);
-                        mem_pool_destroy (trav->reply_info_pool);
-                        ibv_dealloc_pd (trav->pd);
-                        gf_rdma_destroy_cq (this);
-                        ibv_destroy_comp_channel (trav->recv_chan);
-                        ibv_destroy_comp_channel (trav->send_chan);
-                        GF_FREE ((char *)trav->device_name);
-                        GF_FREE (trav);
-
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "%s: could not create SRQ",
-                                this->name);
-                        return NULL;
-                }
-
-                /* queue init */
-                gf_rdma_queue_init (&trav->sendq);
-                gf_rdma_queue_init (&trav->recvq);
-
-                if (gf_rdma_create_posts (this) < 0) {
-                        mem_pool_destroy (trav->ioq_pool);
-                        mem_pool_destroy (trav->request_ctx_pool);
-                        mem_pool_destroy (trav->reply_info_pool);
-                        ibv_dealloc_pd (trav->pd);
-                        gf_rdma_destroy_cq (this);
-                        ibv_destroy_comp_channel (trav->recv_chan);
-                        ibv_destroy_comp_channel (trav->send_chan);
-                        GF_FREE ((char *)trav->device_name);
-                        GF_FREE (trav);
-
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "%s: could not allocate posts",
-                                this->name);
-                        return NULL;
-                }
-
-                /* completion threads */
-                ret = pthread_create (&trav->send_thread,
-                                      NULL,
-                                      gf_rdma_send_completion_proc,
-                                      trav->send_chan);
-                if (ret) {
-                        gf_rdma_destroy_posts (this);
-                        mem_pool_destroy (trav->ioq_pool);
-                        mem_pool_destroy (trav->request_ctx_pool);
-                        mem_pool_destroy (trav->reply_info_pool);
-                        ibv_dealloc_pd (trav->pd);
-                        gf_rdma_destroy_cq (this);
-                        ibv_destroy_comp_channel (trav->recv_chan);
-                        ibv_destroy_comp_channel (trav->send_chan);
-                        GF_FREE ((char *)trav->device_name);
-                        GF_FREE (trav);
+        rdma_ctx->rdma_cm_event_channel = rdma_create_event_channel ();
+        if (rdma_ctx->rdma_cm_event_channel == NULL) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "rdma_cm event channel creation failed (%s)",
+                        strerror (errno));
+                goto out;
+        }
 
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "could not create send completion thread");
-                        return NULL;
-                }
+        ret = gf_thread_create (&rdma_ctx->rdma_cm_thread, NULL,
+				gf_rdma_cm_event_handler,
+				rdma_ctx->rdma_cm_event_channel);
+        if (ret != 0) {
+                gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+                        "creation of thread to handle rdma-cm events "
+                        "failed (%s)", strerror (ret));
+                goto out;
+        }
 
-                ret = pthread_create (&trav->recv_thread,
-                                      NULL,
-                                      gf_rdma_recv_completion_proc,
-                                      trav->recv_chan);
-                if (ret) {
-                        gf_rdma_destroy_posts (this);
-                        mem_pool_destroy (trav->ioq_pool);
-                        mem_pool_destroy (trav->request_ctx_pool);
-                        mem_pool_destroy (trav->reply_info_pool);
-                        ibv_dealloc_pd (trav->pd);
-                        gf_rdma_destroy_cq (this);
-                        ibv_destroy_comp_channel (trav->recv_chan);
-                        ibv_destroy_comp_channel (trav->send_chan);
-                        GF_FREE ((char *)trav->device_name);
-                        GF_FREE (trav);
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "could not create recv completion thread");
-                        return NULL;
+out:
+        if (ret < 0) {
+                if (rdma_ctx->rdma_cm_event_channel != NULL) {
+                        rdma_destroy_event_channel (rdma_ctx->rdma_cm_event_channel);
                 }
 
-                /* qpreg */
-                pthread_mutex_init (&trav->qpreg.lock, NULL);
-                for (i=0; i<42; i++) {
-                        trav->qpreg.ents[i].next = &trav->qpreg.ents[i];
-                        trav->qpreg.ents[i].prev = &trav->qpreg.ents[i];
-                }
+                GF_FREE (rdma_ctx);
+                rdma_ctx = NULL;
         }
-        return trav;
+
+        return rdma_ctx;
 }
 
 static int32_t
 gf_rdma_init (rpc_transport_t *this)
 {
         gf_rdma_private_t   *priv    = NULL;
-        gf_rdma_options_t   *options = NULL;
-        struct ibv_device  **dev_list;
-        struct ibv_context  *ib_ctx  = NULL;
         int32_t              ret     = 0;
+        glusterfs_ctx_t     *ctx     = NULL;
+        gf_rdma_options_t   *options = NULL;
+
+        ctx= this->ctx;
 
         priv = this->private;
-        options = &priv->options;
 
         ibv_fork_init ();
         gf_rdma_options_init (this);
 
-        {
-                dev_list = ibv_get_device_list (NULL);
-
-                if (!dev_list) {
-                        gf_log (GF_RDMA_LOG_NAME,
-                                GF_LOG_CRITICAL,
-                                "Failed to get IB devices");
-                        ret = -1;
-                        goto cleanup;
-                }
-
-                if (!*dev_list) {
-                        gf_log (GF_RDMA_LOG_NAME,
-                                GF_LOG_CRITICAL,
-                                "No IB devices found");
-                        ret = -1;
-                        goto cleanup;
-                }
-
-                if (!options->device_name) {
-                        if (*dev_list) {
-                                options->device_name =
-                                        gf_strdup (ibv_get_device_name (*dev_list));
-                        } else {
-                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_CRITICAL,
-                                        "IB device list is empty. Check for "
-                                        "'ib_uverbs' module");
-                                return -1;
-                                goto cleanup;
-                        }
-                }
-
-                while (*dev_list) {
-                        if (!strcmp (ibv_get_device_name (*dev_list),
-                                     options->device_name)) {
-                                ib_ctx = ibv_open_device (*dev_list);
-
-                                if (!ib_ctx) {
-                                        gf_log (GF_RDMA_LOG_NAME,
-                                                GF_LOG_ERROR,
-                                                "Failed to get infiniband"
-                                                "device context");
-                                        ret = -1;
-                                        goto cleanup;
-                                }
-                                break;
-                        }
-                        ++dev_list;
-                }
-
-                priv->device = gf_rdma_get_device (this, ib_ctx);
-
-                if (!priv->device) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "could not create rdma device for %s",
-                                options->device_name);
-                        ret = -1;
-                        goto cleanup;
-                }
-        }
+        options = &priv->options;
+        priv->peer.send_count = options->send_count;
+        priv->peer.recv_count = options->recv_count;
+        priv->peer.send_size = options->send_size;
+        priv->peer.recv_size = options->recv_size;
 
         priv->peer.trans = this;
         INIT_LIST_HEAD (&priv->peer.ioq);
 
-        pthread_mutex_init (&priv->read_mutex, NULL);
         pthread_mutex_init (&priv->write_mutex, NULL);
         pthread_mutex_init (&priv->recv_mutex, NULL);
         pthread_cond_init (&priv->recv_cond, NULL);
 
-cleanup:
-        if (-1 == ret) {
-                if (ib_ctx)
-                        ibv_close_device (ib_ctx);
+        pthread_mutex_lock (&ctx->lock);
+        {
+                if (ctx->ib == NULL) {
+                        ctx->ib = __gf_rdma_ctx_create ();
+                        if (ctx->ib == NULL) {
+                                ret = -1;
+                        }
+                }
         }
-
-        if (dev_list)
-                ibv_free_device_list (dev_list);
+        pthread_mutex_unlock (&ctx->lock);
 
         return ret;
 }
@@ -4056,543 +4268,69 @@ gf_rdma_disconnect (rpc_transport_t *this)
         int32_t            ret  = 0;
 
         priv = this->private;
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                ret = __gf_rdma_disconnect (this);
-        }
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        return ret;
-}
-
-
-static int32_t
-__tcp_connect_finish (int fd)
-{
-        int       ret    = -1;
-        int       optval = 0;
-        socklen_t optlen = sizeof (int);
-
-        ret = getsockopt (fd, SOL_SOCKET, SO_ERROR,
-                          (void *)&optval, &optlen);
-
-        if (ret == 0 && optval)
-        {
-                errno = optval;
-                ret = -1;
-        }
-
-        return ret;
-}
-
-static inline void
-gf_rdma_fill_handshake_data (char *buf, struct gf_rdma_nbio *nbio,
-                             gf_rdma_private_t *priv)
-{
-        sprintf (buf,
-                 "QP1:RECV_BLKSIZE=%08x:SEND_BLKSIZE=%08x\n"
-                 "QP1:LID=%04x:QPN=%06x:PSN=%06x\n",
-                 priv->peer.recv_size,
-                 priv->peer.send_size,
-                 priv->peer.local_lid,
-                 priv->peer.local_qpn,
-                 priv->peer.local_psn);
-
-        nbio->vector.iov_base = buf;
-        nbio->vector.iov_len = strlen (buf) + 1;
-        nbio->count = 1;
-        return;
-}
-
-static inline void
-gf_rdma_fill_handshake_ack (char *buf, struct gf_rdma_nbio *nbio)
-{
-        sprintf (buf, "DONE\n");
-        nbio->vector.iov_base = buf;
-        nbio->vector.iov_len = strlen (buf) + 1;
-        nbio->count = 1;
-        return;
-}
-
-static int
-gf_rdma_handshake_pollin (rpc_transport_t *this)
-{
-        int                ret           = 0;
-        gf_rdma_private_t *priv          = NULL;
-        char              *buf           = NULL;
-        int32_t            recv_buf_size = 0, send_buf_size;
-        socklen_t          sock_len      = 0;
-
-        priv = this->private;
-	buf = priv->handshake.incoming.buf;
-
-        if (priv->handshake.incoming.state == GF_RDMA_HANDSHAKE_COMPLETE) {
-                return -1;
-        }
+        gf_log_callingfn (this->name, GF_LOG_WARNING,
+                          "disconnect called (peer:%s)",
+                          this->peerinfo.identifier);
 
         pthread_mutex_lock (&priv->write_mutex);
         {
-                while (priv->handshake.incoming.state != GF_RDMA_HANDSHAKE_COMPLETE)
-                {
-                        switch (priv->handshake.incoming.state)
-                        {
-                        case GF_RDMA_HANDSHAKE_START:
-                                buf = priv->handshake.incoming.buf = GF_CALLOC (1, 256, gf_common_mt_char);
-                                gf_rdma_fill_handshake_data (buf, &priv->handshake.incoming, priv);
-                                buf[0] = 0;
-                                priv->handshake.incoming.state = GF_RDMA_HANDSHAKE_RECEIVING_DATA;
-                                break;
-
-                        case GF_RDMA_HANDSHAKE_RECEIVING_DATA:
-                                ret = __tcp_readv (this,
-                                                   &priv->handshake.incoming.vector,
-                                                   priv->handshake.incoming.count,
-                                                   &priv->handshake.incoming.pending_vector,
-                                                   &priv->handshake.incoming.pending_count);
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->name, GF_LOG_TRACE,
-                                                "partial header read on NB socket. continue later");
-                                        goto unlock;
-                                }
-
-                                if (!ret) {
-                                        priv->handshake.incoming.state = GF_RDMA_HANDSHAKE_RECEIVED_DATA;
-                                }
-                                break;
-
-                        case GF_RDMA_HANDSHAKE_RECEIVED_DATA:
-                                ret = sscanf (buf,
-                                              "QP1:RECV_BLKSIZE=%08x:SEND_BLKSIZE=%08x\n"
-                                              "QP1:LID=%04x:QPN=%06x:PSN=%06x\n",
-                                              &recv_buf_size,
-                                              &send_buf_size,
-                                              &priv->peer.remote_lid,
-                                              &priv->peer.remote_qpn,
-                                              &priv->peer.remote_psn);
-
-                                if ((ret != 5) && (strncmp (buf, "QP1:", 4))) {
-                                        gf_log (GF_RDMA_LOG_NAME,
-                                                GF_LOG_CRITICAL,
-                                                "%s: remote-host(%s)'s "
-                                                "transport type is different",
-                                                this->name,
-                                                this->peerinfo.identifier);
-                                        ret = -1;
-                                        goto unlock;
-                                }
-
-                                if (recv_buf_size < priv->peer.recv_size)
-                                        priv->peer.recv_size = recv_buf_size;
-                                if (send_buf_size < priv->peer.send_size)
-                                        priv->peer.send_size = send_buf_size;
-
-                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_TRACE,
-                                        "%s: transacted recv_size=%d "
-                                        "send_size=%d",
-                                        this->name, priv->peer.recv_size,
-                                        priv->peer.send_size);
-
-                                priv->peer.quota = priv->peer.send_count;
-
-                                if (gf_rdma_connect_qp (this)) {
-                                        gf_log (GF_RDMA_LOG_NAME,
-                                                GF_LOG_ERROR,
-                                                "%s: failed to connect with "
-                                                "remote QP", this->name);
-                                        ret = -1;
-                                        goto unlock;
-                                }
-                                gf_rdma_fill_handshake_ack (buf, &priv->handshake.incoming);
-                                buf[0] = 0;
-                                priv->handshake.incoming.state = GF_RDMA_HANDSHAKE_RECEIVING_ACK;
-                                break;
-
-                        case GF_RDMA_HANDSHAKE_RECEIVING_ACK:
-                                ret = __tcp_readv (this,
-                                                   &priv->handshake.incoming.vector,
-                                                   priv->handshake.incoming.count,
-                                                   &priv->handshake.incoming.pending_vector,
-                                                   &priv->handshake.incoming.pending_count);
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->name, GF_LOG_TRACE,
-                                                "partial header read on NB "
-                                                "socket. continue later");
-                                        goto unlock;
-                                }
-
-                                if (!ret) {
-                                        priv->handshake.incoming.state = GF_RDMA_HANDSHAKE_RECEIVED_ACK;
-                                }
-                                break;
-
-                        case GF_RDMA_HANDSHAKE_RECEIVED_ACK:
-                                if (strncmp (buf, "DONE", 4)) {
-                                        gf_log (GF_RDMA_LOG_NAME,
-                                                GF_LOG_DEBUG,
-                                                "%s: handshake-3 did not "
-                                                "return 'DONE' (%s)",
-                                                this->name, buf);
-                                        ret = -1;
-                                        goto unlock;
-                                }
-                                ret = 0;
-                                priv->connected = 1;
-                                sock_len = sizeof (struct sockaddr_storage);
-                                getpeername (priv->sock,
-                                             (struct sockaddr *) &this->peerinfo.sockaddr,
-                                             &sock_len);
-
-                                GF_FREE (priv->handshake.incoming.buf);
-                                priv->handshake.incoming.buf = NULL;
-                                priv->handshake.incoming.state = GF_RDMA_HANDSHAKE_COMPLETE;
-                        }
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        if (ret == -1) {
-                rpc_transport_disconnect (this);
-        } else {
-                ret = 0;
-        }
-
-
-        if (!ret && priv->connected) {
-                if (priv->is_server) {
-                        ret = rpc_transport_notify (priv->listener,
-                                                    RPC_TRANSPORT_ACCEPT,
-                                                    this);
-                } else {
-                        ret = rpc_transport_notify (this, RPC_TRANSPORT_CONNECT,
-                                                    this);
-                }
-        }
-
-        return ret;
-}
-
-static int
-gf_rdma_handshake_pollout (rpc_transport_t *this)
-{
-        gf_rdma_private_t *priv = NULL;
-        char              *buf  = NULL;
-        int32_t            ret  = 0;
-
-        priv = this->private;
-        buf = priv->handshake.outgoing.buf;
-
-        if (priv->handshake.outgoing.state == GF_RDMA_HANDSHAKE_COMPLETE) {
-                return 0;
-        }
-
-        pthread_mutex_unlock (&priv->write_mutex);
-        {
-                while (priv->handshake.outgoing.state
-                       != GF_RDMA_HANDSHAKE_COMPLETE)
-                {
-                        switch (priv->handshake.outgoing.state)
-                        {
-                        case GF_RDMA_HANDSHAKE_START:
-                                buf = priv->handshake.outgoing.buf
-                                        = GF_CALLOC (1, 256, gf_common_mt_char);
-                                gf_rdma_fill_handshake_data (buf,
-                                                             &priv->handshake.outgoing, priv);
-                                priv->handshake.outgoing.state
-                                        = GF_RDMA_HANDSHAKE_SENDING_DATA;
-                                break;
-
-                        case GF_RDMA_HANDSHAKE_SENDING_DATA:
-                                ret = __tcp_writev (this,
-                                                    &priv->handshake.outgoing.vector,
-                                                    priv->handshake.outgoing.count,
-                                                    &priv->handshake.outgoing.pending_vector,
-                                                    &priv->handshake.outgoing.pending_count);
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->name, GF_LOG_TRACE,
-                                                "partial header read on NB "
-                                                "socket. continue later");
-                                        goto unlock;
-                                }
-
-                                if (!ret) {
-                                        priv->handshake.outgoing.state
-                                                = GF_RDMA_HANDSHAKE_SENT_DATA;
-                                }
-                                break;
-
-                        case GF_RDMA_HANDSHAKE_SENT_DATA:
-                                gf_rdma_fill_handshake_ack (buf,
-                                                            &priv->handshake.outgoing);
-                                priv->handshake.outgoing.state
-                                        = GF_RDMA_HANDSHAKE_SENDING_ACK;
-                                break;
-
-                        case GF_RDMA_HANDSHAKE_SENDING_ACK:
-                                ret = __tcp_writev (this,
-                                                    &priv->handshake.outgoing.vector,
-                                                    priv->handshake.outgoing.count,
-                                                    &priv->handshake.outgoing.pending_vector,
-                                                    &priv->handshake.outgoing.pending_count);
-
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->name, GF_LOG_TRACE,
-                                                "partial header read on NB "
-                                                "socket. continue later");
-                                        goto unlock;
-                                }
-
-                                if (!ret) {
-                                        GF_FREE (priv->handshake.outgoing.buf);
-                                        priv->handshake.outgoing.buf = NULL;
-                                        priv->handshake.outgoing.state
-                                                = GF_RDMA_HANDSHAKE_COMPLETE;
-                                }
-                                break;
-                        }
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        if (ret == -1) {
-                rpc_transport_disconnect (this);
-        } else {
-                ret = 0;
-        }
-
-        return ret;
-}
-
-static int
-gf_rdma_handshake_pollerr (rpc_transport_t *this)
-{
-        gf_rdma_private_t *priv = this->private;
-        char need_unref = 0, connected = 0;
-
-        gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
-                "%s: peer disconnected, cleaning up",
-                this->name);
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                __gf_rdma_teardown (this);
-
-                connected = priv->connected;
-                if (priv->sock != -1) {
-                        event_unregister (this->ctx->event_pool,
-                                          priv->sock, priv->idx);
-                        need_unref = 1;
-
-                        if (close (priv->sock) != 0) {
-                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                        "close () - error: %s",
-                                        strerror (errno));
-                        }
-                        priv->tcp_connected = priv->connected = 0;
-                        priv->sock = -1;
-                }
-
-                if (priv->handshake.incoming.buf) {
-                        GF_FREE (priv->handshake.incoming.buf);
-                        priv->handshake.incoming.buf = NULL;
-                }
-
-                priv->handshake.incoming.state = GF_RDMA_HANDSHAKE_START;
-
-                if (priv->handshake.outgoing.buf) {
-                        GF_FREE (priv->handshake.outgoing.buf);
-                        priv->handshake.outgoing.buf = NULL;
-                }
-
-                priv->handshake.outgoing.state = GF_RDMA_HANDSHAKE_START;
-        }
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        if (connected) {
-                rpc_transport_notify (this, RPC_TRANSPORT_DISCONNECT, this);
-        }
-
-        if (need_unref)
-                rpc_transport_unref (this);
-
-        return 0;
-}
-
-
-static int
-tcp_connect_finish (rpc_transport_t *this)
-{
-        gf_rdma_private_t *priv  = NULL;
-        int                error = 0, ret = 0;
-
-        priv = this->private;
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                ret = __tcp_connect_finish (priv->sock);
-
-                if (!ret) {
-                        this->myinfo.sockaddr_len =
-                                sizeof (this->myinfo.sockaddr);
-                        ret = getsockname (priv->sock,
-                                           (struct sockaddr *)&this->myinfo.sockaddr,
-                                           &this->myinfo.sockaddr_len);
-                        if (ret == -1)
-                        {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "getsockname on new client-socket %d "
-                                        "failed (%s)",
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                error = 1;
-                                goto unlock;
-                        }
-
-                        gf_rdma_get_transport_identifiers (this);
-                        priv->tcp_connected = 1;
-                }
-
-                if (ret == -1 && errno != EINPROGRESS) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "tcp connect to %s failed (%s)",
-                                this->peerinfo.identifier, strerror (errno));
-                        error = 1;
-                }
+                ret = __gf_rdma_disconnect (this);
         }
-unlock:
         pthread_mutex_unlock (&priv->write_mutex);
 
-        if (error) {
-                rpc_transport_disconnect (this);
-        }
-
         return ret;
 }
 
-static int
-gf_rdma_event_handler (int fd, int idx, void *data,
-                       int poll_in, int poll_out, int poll_err)
-{
-        rpc_transport_t   *this    = NULL;
-        gf_rdma_private_t *priv    = NULL;
-        gf_rdma_options_t *options = NULL;
-        int                ret     = 0;
-
-        this = data;
-        priv = this->private;
-        if (!priv->tcp_connected) {
-                ret = tcp_connect_finish (this);
-                if (priv->tcp_connected) {
-                        options = &priv->options;
-
-                        priv->peer.send_count = options->send_count;
-                        priv->peer.recv_count = options->recv_count;
-                        priv->peer.send_size = options->send_size;
-                        priv->peer.recv_size = options->recv_size;
-
-                        if ((ret = gf_rdma_create_qp (this)) < 0) {
-                                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                        "%s: could not create QP",
-                                        this->name);
-                                rpc_transport_disconnect (this);
-                        }
-                }
-        }
-
-        if (!ret && poll_out && priv->tcp_connected) {
-                ret = gf_rdma_handshake_pollout (this);
-        }
-
-        if (!ret && !poll_err && poll_in && priv->tcp_connected) {
-                if (priv->handshake.incoming.state
-                    == GF_RDMA_HANDSHAKE_COMPLETE) {
-                        gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                                "%s: pollin received on tcp socket (peer: %s) "
-                                "after handshake is complete",
-                                this->name, this->peerinfo.identifier);
-                        gf_rdma_handshake_pollerr (this);
-                        return 0;
-                }
-                ret = gf_rdma_handshake_pollin (this);
-        }
-
-        if (ret < 0 || poll_err) {
-                ret = gf_rdma_handshake_pollerr (this);
-        }
-
-        return 0;
-}
-
-static int
-__tcp_nonblock (int fd)
-{
-        int flags = 0;
-        int ret   = -1;
-
-        flags = fcntl (fd, F_GETFL);
-
-        if (flags != -1)
-                ret = fcntl (fd, F_SETFL, flags | O_NONBLOCK);
-
-        return ret;
-}
 
 static int32_t
 gf_rdma_connect (struct rpc_transport *this, int port)
 {
         gf_rdma_private_t   *priv         = NULL;
         int32_t              ret          = 0;
-        gf_boolean_t         non_blocking = 1;
         union gf_sock_union  sock_union   = {{0, }, };
         socklen_t            sockaddr_len = 0;
+        gf_rdma_peer_t      *peer         = NULL;
+        gf_rdma_ctx_t       *rdma_ctx     = NULL;
+        gf_boolean_t         connected    = _gf_false;
 
         priv = this->private;
 
+        peer = &priv->peer;
+
+        rpc_transport_ref (this);
+
         ret = gf_rdma_client_get_remote_sockaddr (this,
                                                   &sock_union.sa,
                                                   &sockaddr_len, port);
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "cannot get remote address to connect");
-                return ret;
+                goto out;
         }
 
+        rdma_ctx = this->ctx->ib;
 
         pthread_mutex_lock (&priv->write_mutex);
         {
-                if (priv->sock != -1) {
-                        ret = 0;
+                if (peer->cm_id != NULL) {
+                        ret = -1;
+                        errno = EINPROGRESS;
+                        connected = _gf_true;
                         goto unlock;
                 }
 
-                priv->sock = socket (sock_union.sa.sa_family, SOCK_STREAM, 0);
+                priv->entity = GF_RDMA_CLIENT;
 
-                if (priv->sock == -1) {
+                ret = rdma_create_id (rdma_ctx->rdma_cm_event_channel,
+                                      &peer->cm_id, this, RDMA_PS_TCP);
+                if (ret != 0) {
                         gf_log (this->name, GF_LOG_ERROR,
-                                "socket () - error: %s", strerror (errno));
+                                "creation of rdma_cm_id failed (%s)",
+                                strerror (errno));
                         ret = -errno;
                         goto unlock;
                 }
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "socket fd = %d", priv->sock);
-
                 memcpy (&this->peerinfo.sockaddr, &sock_union.storage,
                         sockaddr_len);
                 this->peerinfo.sockaddr_len = sockaddr_len;
@@ -4603,201 +4341,84 @@ gf_rdma_connect (struct rpc_transport *this, int port)
                 ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family =
                         ((struct sockaddr *)&this->peerinfo.sockaddr)->sa_family;
 
-                if (non_blocking)
-                {
-                        ret = __tcp_nonblock (priv->sock);
-
-                        if (ret == -1)
-                        {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not set socket %d to non "
-                                        "blocking mode (%s)",
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                priv->sock = -1;
-                                goto unlock;
-                        }
-                }
-
                 ret = gf_rdma_client_bind (this,
                                            (struct sockaddr *)&this->myinfo.sockaddr,
                                            &this->myinfo.sockaddr_len,
-                                           priv->sock);
-                if (ret == -1)
-                {
+                                           peer->cm_id);
+                if (ret != 0) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "client bind failed: %s", strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
                         goto unlock;
                 }
 
-                ret = connect (priv->sock,
-                               (struct sockaddr *)&this->peerinfo.sockaddr,
-                               this->peerinfo.sockaddr_len);
-                if (ret == -1 && errno != EINPROGRESS)
-                {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "connection attempt failed (%s)",
+                ret = rdma_resolve_addr (peer->cm_id, NULL, &sock_union.sa,
+                                         2000);
+                if (ret != 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "rdma_resolve_addr failed (%s)",
                                 strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
                         goto unlock;
                 }
 
-                priv->tcp_connected = priv->connected = 0;
-
-                rpc_transport_ref (this);
-
-                priv->handshake.incoming.state = GF_RDMA_HANDSHAKE_START;
-                priv->handshake.outgoing.state = GF_RDMA_HANDSHAKE_START;
-
-                priv->idx = event_register (this->ctx->event_pool,
-                                            priv->sock, gf_rdma_event_handler,
-                                            this, 1, 1);
+                priv->connected = 0;
         }
 unlock:
         pthread_mutex_unlock (&priv->write_mutex);
 
-        return ret;
-}
-
-static int
-gf_rdma_server_event_handler (int fd, int idx, void *data,
-                              int poll_in, int poll_out, int poll_err)
-{
-        int32_t            main_sock    = -1;
-        rpc_transport_t   *this         = NULL, *trans = NULL;
-        gf_rdma_private_t *priv         = NULL;
-        gf_rdma_private_t *trans_priv   = NULL;
-        gf_rdma_options_t *options      = NULL;
-
-        if (!poll_in) {
-                return 0;
-        }
-
-        trans = data;
-        trans_priv = (gf_rdma_private_t *) trans->private;
-
-        this = GF_CALLOC (1, sizeof (rpc_transport_t),
-                          gf_common_mt_rpc_transport_t);
-        if (this == NULL) {
-                return -1;
-        }
-
-        this->listener = trans;
-
-        priv = GF_CALLOC (1, sizeof (gf_rdma_private_t),
-                          gf_common_mt_rdma_private_t);
-        if (priv == NULL) {
-                GF_FREE (priv);
-                return -1;
-        }
-        this->private = priv;
-        /* Copy all the rdma related values in priv, from trans_priv
-           as other than QP, all the values remain same */
-        priv->device = trans_priv->device;
-        priv->options = trans_priv->options;
-        priv->is_server = 1;
-        priv->listener = trans;
-
-        options = &priv->options;
-
-        this->ops = trans->ops;
-        this->init = trans->init;
-        this->fini = trans->fini;
-        this->ctx = trans->ctx;
-        this->name = gf_strdup (trans->name);
-        this->notify = trans->notify;
-        this->mydata = trans->mydata;
-
-        memcpy (&this->myinfo.sockaddr, &trans->myinfo.sockaddr,
-                trans->myinfo.sockaddr_len);
-        this->myinfo.sockaddr_len = trans->myinfo.sockaddr_len;
-
-        main_sock = (trans_priv)->sock;
-        this->peerinfo.sockaddr_len = sizeof (this->peerinfo.sockaddr);
-        priv->sock = accept (main_sock,
-                             (struct sockaddr *)&this->peerinfo.sockaddr,
-                             &this->peerinfo.sockaddr_len);
-        if (priv->sock == -1) {
-                gf_log ("rdma/server", GF_LOG_ERROR,
-                        "accept() failed: %s",
-                        strerror (errno));
-                GF_FREE (this->private);
-                GF_FREE (this);
-                return -1;
-        }
-
-        priv->peer.trans = this;
-        rpc_transport_ref (this);
-
-        gf_rdma_get_transport_identifiers (this);
-
-        priv->tcp_connected = 1;
-        priv->handshake.incoming.state = GF_RDMA_HANDSHAKE_START;
-        priv->handshake.outgoing.state = GF_RDMA_HANDSHAKE_START;
-
-        priv->peer.send_count = options->send_count;
-        priv->peer.recv_count = options->recv_count;
-        priv->peer.send_size = options->send_size;
-        priv->peer.recv_size = options->recv_size;
-        INIT_LIST_HEAD (&priv->peer.ioq);
+out:
+        if (ret != 0) {
+                if (!connected) {
+                        gf_rdma_teardown (this);
+                }
 
-        if (gf_rdma_create_qp (this) < 0) {
-                gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
-                        "%s: could not create QP",
-                        this->name);
-                rpc_transport_disconnect (this);
-                return -1;
+                rpc_transport_unref (this);
         }
 
-        priv->idx = event_register (this->ctx->event_pool, priv->sock,
-                                    gf_rdma_event_handler, this, 1, 1);
-
-        pthread_mutex_init (&priv->read_mutex, NULL);
-        pthread_mutex_init (&priv->write_mutex, NULL);
-        pthread_mutex_init (&priv->recv_mutex, NULL);
-        /*  pthread_cond_init (&priv->recv_cond, NULL); */
-        return 0;
+        return ret;
 }
 
+
 static int32_t
 gf_rdma_listen (rpc_transport_t *this)
 {
         union gf_sock_union  sock_union   = {{0, }, };
         socklen_t            sockaddr_len = 0;
         gf_rdma_private_t   *priv         = NULL;
-        int                  opt          = 1, ret = 0;
+        gf_rdma_peer_t      *peer         = NULL;
+        int                  ret          = 0;
+        gf_rdma_ctx_t       *rdma_ctx     = NULL;
         char                 service[NI_MAXSERV], host[NI_MAXHOST];
 
         priv = this->private;
-        memset (&sock_union, 0, sizeof (sock_union));
-        ret = gf_rdma_server_get_local_sockaddr (this,
-                                                 &sock_union.sa,
+        peer = &priv->peer;
+
+        priv->entity = GF_RDMA_SERVER_LISTENER;
+
+        rdma_ctx = this->ctx->ib;
+
+        ret = gf_rdma_server_get_local_sockaddr (this, &sock_union.sa,
                                                  &sockaddr_len);
         if (ret != 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_WARNING,
                         "cannot find network address of server to bind to");
                 goto err;
         }
 
-        priv->sock = socket (sock_union.sa.sa_family, SOCK_STREAM, 0);
-        if (priv->sock == -1) {
-                gf_log ("rdma/server", GF_LOG_CRITICAL,
-                        "init: failed to create socket, error: %s",
+        ret = rdma_create_id (rdma_ctx->rdma_cm_event_channel,
+                              &peer->cm_id, this, RDMA_PS_TCP);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "creation of rdma_cm_id failed (%s)",
                         strerror (errno));
-                GF_FREE (this->private);
-                ret = -1;
                 goto err;
         }
 
-        memcpy (&this->myinfo.sockaddr, &sock_union.storage, sockaddr_len);
+        memcpy (&this->myinfo.sockaddr, &sock_union.storage,
+                sockaddr_len);
         this->myinfo.sockaddr_len = sockaddr_len;
 
         ret = getnameinfo ((struct sockaddr *)&this->myinfo.sockaddr,
-                           this->myinfo.sockaddr_len,
-                           host, sizeof (host),
+                           this->myinfo.sockaddr_len, host, sizeof (host),
                            service, sizeof (service),
                            NI_NUMERICHOST);
         if (ret != 0) {
@@ -4805,34 +4426,38 @@ gf_rdma_listen (rpc_transport_t *this)
                         "getnameinfo failed (%s)", gai_strerror (ret));
                 goto err;
         }
+
         sprintf (this->myinfo.identifier, "%s:%s", host, service);
 
-        setsockopt (priv->sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof (opt));
-        if (bind (priv->sock, &sock_union.sa, sockaddr_len) != 0) {
-                ret = -1;
-                gf_log ("rdma/server", GF_LOG_ERROR,
-                        "init: failed to bind to socket for %s (%s)",
-                        this->myinfo.identifier, strerror (errno));
+        ret = rdma_bind_addr (peer->cm_id, &sock_union.sa);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "rdma_bind_addr failed (%s)", strerror (errno));
                 goto err;
         }
 
-        if (listen (priv->sock, 10) != 0) {
-                gf_log ("rdma/server", GF_LOG_ERROR,
-                        "init: listen () failed on socket for %s (%s)",
-                        this->myinfo.identifier, strerror (errno));
-                ret = -1;
+        ret = rdma_listen (peer->cm_id, 10);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "rdma_listen failed (%s)", strerror (errno));
                 goto err;
         }
 
-        /* Register the main socket */
-        priv->idx = event_register (this->ctx->event_pool, priv->sock,
-                                    gf_rdma_server_event_handler,
-                                    rpc_transport_ref (this), 1, 0);
+        rpc_transport_ref (this);
 
+        ret = 0;
 err:
+        if (ret < 0) {
+                if (peer->cm_id != NULL) {
+                        rdma_destroy_id (peer->cm_id);
+                        peer->cm_id = NULL;
+                }
+        }
+
         return ret;
 }
 
+
 struct rpc_transport_ops tops = {
         .submit_request = gf_rdma_submit_request,
         .submit_reply   = gf_rdma_submit_reply,
@@ -4851,7 +4476,6 @@ init (rpc_transport_t *this)
                 return -1;
 
         this->private = priv;
-        priv->sock = -1;
 
         if (gf_rdma_init (this)) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -4875,13 +4499,6 @@ fini (struct rpc_transport *this)
         if (priv) {
                 pthread_mutex_destroy (&priv->recv_mutex);
                 pthread_mutex_destroy (&priv->write_mutex);
-                pthread_mutex_destroy (&priv->read_mutex);
-
-                /*  pthread_cond_destroy (&priv->recv_cond); */
-                if (priv->sock != -1) {
-                        event_unregister (this->ctx->event_pool,
-                                          priv->sock, priv->idx);
-                }
 
                 gf_log (this->name, GF_LOG_TRACE,
                         "called fini on transport: %p", this);
diff --git a/rpc/rpc-transport/rdma/src/rdma.h b/rpc/rpc-transport/rdma/src/rdma.h
index 342938591..7f76244f0 100644
--- a/rpc/rpc-transport/rdma/src/rdma.h
+++ b/rpc/rpc-transport/rdma/src/rdma.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _XPORT_RDMA_H
@@ -38,6 +29,7 @@
 #include <list.h>
 #include <arpa/inet.h>
 #include <infiniband/verbs.h>
+#include <rdma/rdma_cma.h>
 
 /* FIXME: give appropriate values to these macros */
 #define GF_DEFAULT_RDMA_LISTEN_PORT (GF_DEFAULT_BASE_PORT + 1)
@@ -239,30 +231,33 @@ typedef enum __gf_rdma_send_post_type {
 
 /* represents one communication peer, two per transport_t */
 struct __gf_rdma_peer {
-        rpc_transport_t *trans;
-        struct ibv_qp *qp;
+        rpc_transport_t   *trans;
+        struct rdma_cm_id *cm_id;
+        struct ibv_qp     *qp;
+        pthread_t          rdma_event_thread;
+        char               quota_set;
 
         int32_t recv_count;
         int32_t send_count;
         int32_t recv_size;
         int32_t send_size;
 
-        int32_t quota;
+        int32_t                        quota;
         union {
-                struct list_head     ioq;
+                struct list_head       ioq;
                 struct {
-                        gf_rdma_ioq_t        *ioq_next;
-                        gf_rdma_ioq_t        *ioq_prev;
+                        gf_rdma_ioq_t *ioq_next;
+                        gf_rdma_ioq_t *ioq_prev;
                 };
         };
 
         /* QP attributes, needed to connect with remote QP */
-        int32_t local_lid;
-        int32_t local_psn;
-        int32_t local_qpn;
-        int32_t remote_lid;
-        int32_t remote_psn;
-        int32_t remote_qpn;
+        int32_t               local_lid;
+        int32_t               local_psn;
+        int32_t               local_qpn;
+        int32_t               remote_lid;
+        int32_t               remote_psn;
+        int32_t               remote_qpn;
 };
 typedef struct __gf_rdma_peer gf_rdma_peer_t;
 
@@ -329,33 +324,19 @@ struct __gf_rdma_device {
         struct ibv_comp_channel *send_chan, *recv_chan;
         struct ibv_cq *send_cq, *recv_cq;
         gf_rdma_queue_t sendq, recvq;
-        pthread_t send_thread, recv_thread;
+        pthread_t send_thread, recv_thread, async_event_thread;
         struct mem_pool *request_ctx_pool;
         struct mem_pool *ioq_pool;
         struct mem_pool *reply_info_pool;
 };
 typedef struct __gf_rdma_device gf_rdma_device_t;
 
-typedef enum {
-        GF_RDMA_HANDSHAKE_START = 0,
-        GF_RDMA_HANDSHAKE_SENDING_DATA,
-        GF_RDMA_HANDSHAKE_RECEIVING_DATA,
-        GF_RDMA_HANDSHAKE_SENT_DATA,
-        GF_RDMA_HANDSHAKE_RECEIVED_DATA,
-        GF_RDMA_HANDSHAKE_SENDING_ACK,
-        GF_RDMA_HANDSHAKE_RECEIVING_ACK,
-        GF_RDMA_HANDSHAKE_RECEIVED_ACK,
-        GF_RDMA_HANDSHAKE_COMPLETE,
-} gf_rdma_handshake_state_t;
-
-struct gf_rdma_nbio {
-        int state;
-        char *buf;
-        int count;
-        struct iovec vector;
-        struct iovec *pending_vector;
-        int pending_count;
+struct __gf_rdma_ctx {
+        gf_rdma_device_t          *device;
+        struct rdma_event_channel *rdma_cm_event_channel;
+        pthread_t                  rdma_cm_thread;
 };
+typedef struct __gf_rdma_ctx gf_rdma_ctx_t;
 
 struct __gf_rdma_request_context {
         struct ibv_mr   *mr[GF_RDMA_MAX_SEGMENTS];
@@ -367,46 +348,35 @@ struct __gf_rdma_request_context {
 };
 typedef struct __gf_rdma_request_context gf_rdma_request_context_t;
 
+typedef enum {
+        GF_RDMA_SERVER_LISTENER,
+        GF_RDMA_SERVER,
+        GF_RDMA_CLIENT,
+} gf_rdma_transport_entity_t;
+
 struct __gf_rdma_private {
-        int32_t sock;
-        int32_t idx;
-        unsigned char connected;
-        unsigned char tcp_connected;
-        unsigned char ib_connected;
-        in_addr_t addr;
+        int32_t        idx;
+        unsigned char  connected;
+        in_addr_t      addr;
         unsigned short port;
 
         /* IB Verbs Driver specific variables, pointers */
-        gf_rdma_peer_t peer;
+        gf_rdma_peer_t           peer;
         struct __gf_rdma_device *device;
-        gf_rdma_options_t options;
+        gf_rdma_options_t        options;
 
         /* Used by trans->op->receive */
-        char *data_ptr;
-        int32_t data_offset;
-        int32_t data_len;
+        char    *data_ptr;
+        int32_t  data_offset;
+        int32_t  data_len;
 
         /* Mutex */
-        pthread_mutex_t read_mutex;
-        pthread_mutex_t write_mutex;
-        pthread_barrier_t handshake_barrier;
-        char handshake_ret;
-        char is_server;
-        rpc_transport_t *listener;
-
-        pthread_mutex_t recv_mutex;
-        pthread_cond_t  recv_cond;
-
-        /* used during gf_rdma_handshake */
-        struct {
-                struct gf_rdma_nbio incoming;
-                struct gf_rdma_nbio outgoing;
-                int               state;
-                gf_rdma_header_t header;
-                char *buf;
-                size_t size;
-        } handshake;
+        pthread_mutex_t             write_mutex;
+        rpc_transport_t            *listener;
+        pthread_mutex_t             recv_mutex;
+        pthread_cond_t              recv_cond;
+        gf_rdma_transport_entity_t  entity;
 };
-typedef struct __gf_rdma_private gf_rdma_private_t;
+typedef struct __gf_rdma_private    gf_rdma_private_t;
 
 #endif /* _XPORT_GF_RDMA_H */
diff --git a/rpc/rpc-transport/socket/src/Makefile.am b/rpc/rpc-transport/socket/src/Makefile.am
index 2c918c7e3..71e6ed6ff 100644
--- a/rpc/rpc-transport/socket/src/Makefile.am
+++ b/rpc/rpc-transport/socket/src/Makefile.am
@@ -3,13 +3,15 @@ noinst_HEADERS = socket.h name.h
 rpctransport_LTLIBRARIES = socket.la
 rpctransportdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/rpc-transport
 
-socket_la_LDFLAGS = -module -avoidversion
+socket_la_LDFLAGS = -module -avoid-version
 
 socket_la_SOURCES = socket.c name.c
-socket_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+socket_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la -lssl
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src/ \
-	-I$(top_srcdir)/rpc/xdr/src/ -shared -nostartfiles $(GF_CFLAGS)
+	-I$(top_srcdir)/rpc/xdr/src/
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = *~
diff --git a/rpc/rpc-transport/socket/src/name.c b/rpc/rpc-transport/socket/src/name.c
index bed59d58b..1647d5b6b 100644
--- a/rpc/rpc-transport/socket/src/name.c
+++ b/rpc/rpc-transport/socket/src/name.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <sys/types.h>
@@ -24,12 +15,6 @@
 #include <netdb.h>
 #include <string.h>
 
-#ifdef CLIENT_PORT_CEILING
-#undef CLIENT_PORT_CEILING
-#endif
-
-#define CLIENT_PORT_CEILING 1024
-
 #ifndef AF_INET_SDP
 #define AF_INET_SDP 27
 #endif
@@ -49,9 +34,17 @@ static int32_t
 af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
                                  socklen_t sockaddr_len, int ceiling)
 {
-        int32_t ret = -1;
-        /*  struct sockaddr_in sin = {0, }; */
-        uint16_t port = ceiling - 1;
+        int32_t        ret        = -1;
+        uint16_t      port        = ceiling - 1;
+        // by default assume none of the ports are blocked and all are available
+        gf_boolean_t  ports[1024] = {_gf_false,};
+        int           i           = 0;
+
+        ret = gf_process_reserved_ports (ports);
+        if (ret != 0) {
+                for (i = 0; i < 1024; i++)
+                        ports[i] = _gf_false;
+        }
 
         while (port)
         {
@@ -66,7 +59,11 @@ af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
                         ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
                         break;
                 }
-
+                // ignore the reserved ports
+                if (ports[port] == _gf_true) {
+                        port--;
+                        continue;
+                }
                 ret = bind (fd, sockaddr, sockaddr_len);
 
                 if (ret == 0)
@@ -143,24 +140,24 @@ client_fill_address_family (rpc_transport_t *this, sa_family_t *sa_family)
                 if (!(remote_host_data || connect_path_data) ||
                     (remote_host_data && connect_path_data)) {
                         gf_log (this->name, GF_LOG_ERROR,
-                                "transport.address-family not specified and "
-                                "not able to determine the "
-                                "same from other options (remote-host:%s and "
-                                "transport.unix.connect-path:%s)",
+                                "transport.address-family not specified. "
+                                "Could not guess default value from (remote-host:%s or "
+                                "transport.unix.connect-path:%s) options",
                                 data_to_str (remote_host_data),
                                 data_to_str (connect_path_data));
+                        *sa_family = AF_UNSPEC;
                         goto out;
                 }
 
                 if (remote_host_data) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "address-family not specified, guessing it "
-                                "to be inet/inet6");
-                        *sa_family = AF_UNSPEC;
+                                "to be inet from (remote-host: %s)", data_to_str (remote_host_data));
+                        *sa_family = AF_INET;
                 } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "address-family not specified, guessing it "
-                                "to be unix");
+                                "to be unix from (transport.unix.connect-path: %s)", data_to_str (connect_path_data));
                         *sa_family = AF_UNIX;
                 }
 
@@ -174,13 +171,11 @@ client_fill_address_family (rpc_transport_t *this, sa_family_t *sa_family)
                         *sa_family = AF_INET6;
                 } else if (!strcasecmp (address_family, "inet-sdp")) {
                         *sa_family = AF_INET_SDP;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        *sa_family = AF_UNSPEC;
                 } else {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "unknown address-family (%s) specified",
                                 address_family);
+                        *sa_family = AF_UNSPEC;
                         goto out;
                 }
         }
@@ -451,12 +446,12 @@ client_bind (rpc_transport_t *this,
         case AF_INET6:
                 if (!this->bind_insecure) {
                         ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr,
-                                                       *sockaddr_len, CLIENT_PORT_CEILING);
+                                                       *sockaddr_len, GF_CLIENT_PORT_CEILING);
                 }
                 if (ret == -1) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "cannot bind inet socket (%d) to port less than %d (%s)",
-                                sock, CLIENT_PORT_CEILING, strerror (errno));
+                                sock, GF_CLIENT_PORT_CEILING, strerror (errno));
                         ret = 0;
                 }
                 break;
@@ -551,18 +546,16 @@ server_fill_address_family (rpc_transport_t *this, sa_family_t *sa_family)
                         *sa_family = AF_INET_SDP;
                 } else if (!strcasecmp (address_family, "unix")) {
                         *sa_family = AF_UNIX;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        *sa_family = AF_UNSPEC;
                 } else {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "unknown address family (%s) specified", address_family);
+                        *sa_family = AF_UNSPEC;
                         goto out;
                 }
         } else {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "option address-family not specified, defaulting to inet/inet6");
-                *sa_family = AF_UNSPEC;
+                        "option address-family not specified, defaulting to inet");
+                *sa_family = AF_INET;
         }
 
         ret = 0;
diff --git a/rpc/rpc-transport/socket/src/name.h b/rpc/rpc-transport/socket/src/name.h
index 0ca67d2e4..0a13d8a96 100644
--- a/rpc/rpc-transport/socket/src/name.h
+++ b/rpc/rpc-transport/socket/src/name.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _SOCKET_NAME_H
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index 3b2d05c45..93da3f296 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -44,97 +35,401 @@
 #include <errno.h>
 #include <netinet/tcp.h>
 #include <rpc/xdr.h>
+#include <sys/ioctl.h>
 #define GF_LOG_ERRNO(errno) ((errno == ENOTCONN) ? GF_LOG_DEBUG : GF_LOG_ERROR)
 #define SA(ptr) ((struct sockaddr *)ptr)
 
+#define SSL_ENABLED_OPT     "transport.socket.ssl-enabled"
+#define SSL_OWN_CERT_OPT    "transport.socket.ssl-own-cert"
+#define SSL_PRIVATE_KEY_OPT "transport.socket.ssl-private-key"
+#define SSL_CA_LIST_OPT     "transport.socket.ssl-ca-list"
+#define OWN_THREAD_OPT      "transport.socket.own-thread"
+
+/* TBD: do automake substitutions etc. (ick) to set these. */
+#if !defined(DEFAULT_CERT_PATH)
+#define DEFAULT_CERT_PATH   "/etc/ssl/glusterfs.pem"
+#endif
+#if !defined(DEFAULT_KEY_PATH)
+#define DEFAULT_KEY_PATH    "/etc/ssl/glusterfs.key"
+#endif
+#if !defined(DEFAULT_CA_PATH)
+#define DEFAULT_CA_PATH     "/etc/ssl/glusterfs.ca"
+#endif
+
+#define POLL_MASK_INPUT  (POLLIN | POLLPRI)
+#define POLL_MASK_OUTPUT (POLLOUT)
+#define POLL_MASK_ERROR  (POLLERR | POLLHUP | POLLNVAL)
 
-#define __socket_proto_reset_pending(priv) do {                 \
-                memset (&priv->incoming.frag.vector, 0,         \
-                        sizeof (priv->incoming.frag.vector));   \
-                priv->incoming.frag.pending_vector =            \
-                        &priv->incoming.frag.vector;            \
-                priv->incoming.frag.pending_vector->iov_base =  \
-                        priv->incoming.frag.fragcurrent;        \
-                priv->incoming.pending_vector =                 \
-                        priv->incoming.frag.pending_vector;     \
-        } while (0);
+typedef int SSL_unary_func (SSL *);
+typedef int SSL_trinary_func (SSL *, void *, int);
+
+#define __socket_proto_reset_pending(priv) do {                         \
+                struct gf_sock_incoming_frag *frag;                     \
+                frag = &priv->incoming.frag;                            \
+                                                                        \
+                memset (&frag->vector, 0, sizeof (frag->vector));       \
+                frag->pending_vector = &frag->vector;                   \
+                frag->pending_vector->iov_base = frag->fragcurrent;     \
+                priv->incoming.pending_vector =  frag->pending_vector;  \
+        } while (0)
 
 
 #define __socket_proto_update_pending(priv)                             \
         do {                                                            \
-                uint32_t remaining_fragsize = 0;                        \
-                if (priv->incoming.frag.pending_vector->iov_len == 0) { \
-                        remaining_fragsize = RPC_FRAGSIZE (priv->incoming.fraghdr) \
-                                - priv->incoming.frag.bytes_read;       \
+                uint32_t remaining;                                     \
+                struct gf_sock_incoming_frag *frag;                     \
+                frag = &priv->incoming.frag;                            \
+                if (frag->pending_vector->iov_len == 0) {               \
+                        remaining = (RPC_FRAGSIZE (priv->incoming.fraghdr) \
+                                     - frag->bytes_read);               \
                                                                         \
-                        priv->incoming.frag.pending_vector->iov_len =   \
-                                remaining_fragsize > priv->incoming.frag.remaining_size \
-                                ? priv->incoming.frag.remaining_size : remaining_fragsize; \
+                        frag->pending_vector->iov_len =                 \
+                                (remaining > frag->remaining_size)      \
+                                ? frag->remaining_size : remaining;     \
                                                                         \
-                        priv->incoming.frag.remaining_size -=           \
-                                priv->incoming.frag.pending_vector->iov_len; \
+                        frag->remaining_size -=                         \
+                                frag->pending_vector->iov_len;          \
                 }                                                       \
-        } while (0);
+        } while (0)
 
 #define __socket_proto_update_priv_after_read(priv, ret, bytes_read)    \
         {                                                               \
-                priv->incoming.frag.fragcurrent += bytes_read;          \
-                priv->incoming.frag.bytes_read += bytes_read;           \
+                struct gf_sock_incoming_frag *frag;                     \
+                frag = &priv->incoming.frag;                            \
                                                                         \
-                if ((ret > 0) || (priv->incoming.frag.remaining_size != 0)) { \
-                        if (priv->incoming.frag.remaining_size != 0 && ret == 0) {  \
+                frag->fragcurrent += bytes_read;                        \
+                frag->bytes_read += bytes_read;                         \
+                                                                        \
+                if ((ret > 0) || (frag->remaining_size != 0)) {         \
+                        if (frag->remaining_size != 0 && ret == 0) {    \
                                 __socket_proto_reset_pending (priv);    \
                         }                                               \
                                                                         \
-                        gf_log (this->name, GF_LOG_TRACE, "partial read on non-blocking socket"); \
+                        gf_log (this->name, GF_LOG_TRACE,               \
+                                "partial read on non-blocking socket"); \
                                                                         \
                         break;                                          \
                 }                                                       \
         }
 
-#define __socket_proto_init_pending(priv, size)                         \
+#define __socket_proto_init_pending(priv,size)                          \
         do {                                                            \
-                uint32_t remaining_fragsize = 0;                        \
-                remaining_fragsize = RPC_FRAGSIZE (priv->incoming.fraghdr) \
-                        - priv->incoming.frag.bytes_read;               \
+            uint32_t remaining = 0;                                     \
+            struct gf_sock_incoming_frag *frag;                         \
+            frag = &priv->incoming.frag;                                \
+                                                                        \
+            remaining = (RPC_FRAGSIZE (priv->incoming.fraghdr)          \
+                         - frag->bytes_read);                           \
                                                                         \
-                __socket_proto_reset_pending (priv);                    \
+            __socket_proto_reset_pending (priv);                        \
                                                                         \
-                priv->incoming.frag.pending_vector->iov_len =           \
-                        remaining_fragsize > size ? size : remaining_fragsize; \
+            frag->pending_vector->iov_len =                             \
+                    (remaining > size) ? size : remaining;              \
                                                                         \
-                priv->incoming.frag.remaining_size =                    \
-                        size - priv->incoming.frag.pending_vector->iov_len; \
+            frag->remaining_size = (size - frag->pending_vector->iov_len); \
                                                                         \
-        } while (0);
+            } while(0)
 
 
 /* This will be used in a switch case and breaks from the switch case if all
  * the pending data is not read.
  */
 #define __socket_proto_read(priv, ret)                                  \
-        {                                                               \
+                {                                                       \
                 size_t bytes_read = 0;                                  \
+                struct gf_sock_incoming *in;                            \
+                in = &priv->incoming;                                   \
                                                                         \
                 __socket_proto_update_pending (priv);                   \
                                                                         \
                 ret = __socket_readv (this,                             \
-                                      priv->incoming.pending_vector, 1, \
-                                      &priv->incoming.pending_vector,   \
-                                      &priv->incoming.pending_count,    \
+                                      in->pending_vector, 1,            \
+                                      &in->pending_vector,              \
+                                      &in->pending_count,               \
                                       &bytes_read);                     \
-                if (ret == -1) {                                        \
-                        gf_log (this->name, GF_LOG_WARNING,             \
-                                "reading from socket failed. Error (%s), " \
-                                "peer (%s)", strerror (errno),          \
-                                this->peerinfo.identifier);             \
+                if (ret == -1)                                          \
                         break;                                          \
-                }                                                       \
                 __socket_proto_update_priv_after_read (priv, ret, bytes_read); \
         }
 
+static int socket_init (rpc_transport_t *this);
+
+static void
+ssl_dump_error_stack (const char *caller)
+{
+	unsigned long  errnum = 0;
+	char           errbuf[120] = {0,};
+
+	/* OpenSSL docs explicitly give 120 as the error-string length. */
+
+	while ((errnum = ERR_get_error())) {
+		ERR_error_string(errnum,errbuf);
+		gf_log(caller,GF_LOG_ERROR,"  %s",errbuf);
+	}
+}
+
+static int
+ssl_do (rpc_transport_t *this, void *buf, size_t len, SSL_trinary_func *func)
+{
+	int               r = (-1);
+	struct pollfd     pfd = {-1,};
+	socket_private_t *priv = NULL;
+
+	GF_VALIDATE_OR_GOTO(this->name,this->private,out);
+	priv = this->private;
+
+	for (;;) {
+		if (buf) {
+                        if (priv->connected == -1) {
+                                /*
+                                 * Fields in the SSL structure (especially
+                                 * the BIO pointers) are not valid at this
+                                 * point, so we'll segfault if we pass them
+                                 * to SSL_read/SSL_write.
+                                 */
+                                gf_log(this->name,GF_LOG_INFO,
+                                       "lost connection in %s", __func__);
+                                break;
+                        }
+			r = func(priv->ssl_ssl,buf,len);
+		}
+		else {
+                        /*
+                         * We actually need these functions to get to
+                         * priv->connected == 1.
+                         */
+			r = ((SSL_unary_func *)func)(priv->ssl_ssl);
+		}
+		switch (SSL_get_error(priv->ssl_ssl,r)) {
+		case SSL_ERROR_NONE:
+			return r;
+		case SSL_ERROR_WANT_READ:
+			pfd.fd = priv->sock;
+			pfd.events = POLLIN;
+			if (poll(&pfd,1,-1) < 0) {
+				gf_log(this->name,GF_LOG_ERROR,"poll error %d",
+				       errno);
+			}
+			break;
+		case SSL_ERROR_WANT_WRITE:
+			pfd.fd = priv->sock;
+			pfd.events = POLLOUT;
+			if (poll(&pfd,1,-1) < 0) {
+				gf_log(this->name,GF_LOG_ERROR,"poll error %d",
+				       errno);
+			}
+			break;
+		case SSL_ERROR_SYSCALL:
+			/* This is what we get when remote disconnects. */
+			gf_log(this->name,GF_LOG_DEBUG,
+			       "syscall error (probably remote disconnect)");
+			errno = ENODATA;
+			goto out;
+		default:
+			errno = EIO;
+			goto out;	/* "break" would just loop again */
+		}
+	}
+out:
+	return -1;
+}
+
+#define ssl_connect_one(t)   ssl_do((t),NULL,0,(SSL_trinary_func *)SSL_connect)
+#define ssl_accept_one(t)    ssl_do((t),NULL,0,(SSL_trinary_func *)SSL_accept)
+#define ssl_read_one(t,b,l)  ssl_do((t),(b),(l),(SSL_trinary_func *)SSL_read)
+#define ssl_write_one(t,b,l) ssl_do((t),(b),(l),(SSL_trinary_func *)SSL_write)
+
+static int
+ssl_setup_connection (rpc_transport_t *this, int server)
+{
+	X509             *peer = NULL;
+	char              peer_CN[256] = "";
+	int               ret = -1;
+	socket_private_t *priv = NULL;
+
+	GF_VALIDATE_OR_GOTO(this->name,this->private,done);
+	priv = this->private;
+
+	priv->ssl_ssl = SSL_new(priv->ssl_ctx);
+	if (!priv->ssl_ssl) {
+		gf_log(this->name,GF_LOG_ERROR,"SSL_new failed");
+		ssl_dump_error_stack(this->name);
+		goto done;
+	}
+	priv->ssl_sbio = BIO_new_socket(priv->sock,BIO_NOCLOSE);
+	if (!priv->ssl_sbio) {
+		gf_log(this->name,GF_LOG_ERROR,"BIO_new_socket failed");
+		ssl_dump_error_stack(this->name);
+		goto free_ssl;
+	}
+	SSL_set_bio(priv->ssl_ssl,priv->ssl_sbio,priv->ssl_sbio);
+
+	if (server) {
+		ret = ssl_accept_one(this);
+	}
+	else {
+		ret = ssl_connect_one(this);
+	}
+
+	/* Make sure _the call_ succeeded. */
+	if (ret < 0) {
+		goto ssl_error;
+	}
+
+	/* Make sure _SSL verification_ succeeded, yielding an identity. */
+	if (SSL_get_verify_result(priv->ssl_ssl) != X509_V_OK) {
+		goto ssl_error;
+	}
+	peer = SSL_get_peer_certificate(priv->ssl_ssl);
+	if (!peer) {
+		goto ssl_error;
+	}
+
+	/* Finally, everything seems OK. */
+	X509_NAME_get_text_by_NID(X509_get_subject_name(peer),
+		NID_commonName, peer_CN, sizeof(peer_CN)-1);
+	peer_CN[sizeof(peer_CN)-1] = '\0';
+	gf_log(this->name,GF_LOG_INFO,"peer CN = %s", peer_CN);
+	return 0;
+
+	/* Error paths. */
+ssl_error:
+	gf_log(this->name,GF_LOG_ERROR,"SSL connect error");
+	ssl_dump_error_stack(this->name);
+free_ssl:
+	SSL_free(priv->ssl_ssl);
+        priv->ssl_ssl = NULL;
+done:
+	return ret;
+}
+
+
+static void
+ssl_teardown_connection (socket_private_t *priv)
+{
+        SSL_shutdown(priv->ssl_ssl);
+        SSL_clear(priv->ssl_ssl);
+        SSL_free(priv->ssl_ssl);
+        priv->ssl_ssl = NULL;
+}
+
+
+static ssize_t
+__socket_ssl_readv (rpc_transport_t *this, struct iovec *opvector, int opcount)
+{
+	socket_private_t    *priv = NULL;
+	int                  sock = -1;
+	int                  ret = -1;
+
+	priv = this->private;
+	sock = priv->sock;
+
+	if (priv->use_ssl) {
+		ret = ssl_read_one (this, opvector->iov_base, opvector->iov_len);
+	} else {
+		ret = readv (sock, opvector, opcount);
+	}
+
+	return ret;
+}
+
+
+static ssize_t
+__socket_ssl_read (rpc_transport_t *this, void *buf, size_t count)
+{
+	struct iovec iov = {0, };
+	int          ret = -1;
+
+	iov.iov_base = buf;
+	iov.iov_len = count;
+
+	ret = __socket_ssl_readv (this, &iov, 1);
+
+	return ret;
+}
+
+
+static int
+__socket_cached_read (rpc_transport_t *this, struct iovec *opvector, int opcount)
+{
+	socket_private_t   *priv = NULL;
+	int                 sock = -1;
+	struct gf_sock_incoming *in = NULL;
+	int                 req_len = -1;
+	int                 ret = -1;
+
+	priv = this->private;
+	sock = priv->sock;
+	in = &priv->incoming;
+	req_len = iov_length (opvector, opcount);
+
+	if (in->record_state == SP_STATE_READING_FRAGHDR) {
+		in->ra_read = 0;
+		in->ra_served = 0;
+		in->ra_max = 0;
+		in->ra_buf = NULL;
+		goto uncached;
+	}
+
+	if (!in->ra_max) {
+		/* first call after passing SP_STATE_READING_FRAGHDR */
+		in->ra_max = min (RPC_FRAGSIZE (in->fraghdr), GF_SOCKET_RA_MAX);
+		/* Note that the in->iobuf is the primary iobuf into which
+		   headers are read into. By using this itself as our
+		   read-ahead cache, we can avoid memory copies in iov_load
+		*/
+		in->ra_buf = iobuf_ptr (in->iobuf);
+	}
+
+	/* fill read-ahead */
+	if (in->ra_read < in->ra_max) {
+		ret = __socket_ssl_read (this, &in->ra_buf[in->ra_read],
+					 (in->ra_max - in->ra_read));
+		if (ret > 0)
+			in->ra_read += ret;
+
+		/* we proceed to test if there is still cached data to
+		   be served even if readahead could not progress */
+	}
+
+	/* serve cached */
+	if (in->ra_served < in->ra_read) {
+		ret = iov_load (opvector, opcount, &in->ra_buf[in->ra_served],
+				min (req_len, (in->ra_read - in->ra_served)));
+
+		in->ra_served += ret;
+		/* Do not read uncached and cached in the same call */
+		goto out;
+	}
+
+	if (in->ra_read < in->ra_max)
+		/* If there was no cached data to be served, (and we are
+		   guaranteed to have already performed an attempt to progress
+		   readahead above), and we have not yet read out the full
+		   readahead capacity, then bail out for now without doing
+		   the uncached read below (as that will overtake future cached
+		   read)
+		*/
+		goto out;
+uncached:
+	ret = __socket_ssl_readv (this, opvector, opcount);
+out:
+	return ret;
+}
+
+static gf_boolean_t
+__does_socket_rwv_error_need_logging (socket_private_t *priv, int write)
+{
+        int read = !write;
+
+        if (priv->connected == -1) /* Didn't even connect, of course it fails */
+                return _gf_false;
 
-int socket_init (rpc_transport_t *this);
+        if (read && (priv->read_fail_log == _gf_false))
+                return _gf_false;
+
+        return _gf_true;
+}
 
 /*
  * return value:
@@ -143,7 +438,7 @@ int socket_init (rpc_transport_t *this);
  * > 0 = incomplete
  */
 
-int
+static int
 __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
               struct iovec **pending_vector, int *pending_count, size_t *bytes,
               int write)
@@ -168,9 +463,22 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                 *bytes = 0;
         }
 
-        while (opcount) {
+        while (opcount > 0) {
+                if (opvector->iov_len == 0) {
+                        gf_log(this->name,GF_LOG_DEBUG,
+                               "would have passed zero length to read/write");
+                        ++opvector;
+                        --opcount;
+                        continue;
+                }
                 if (write) {
-                        ret = writev (sock, opvector, opcount);
+			if (priv->use_ssl) {
+				ret = ssl_write_one(this,
+					opvector->iov_base, opvector->iov_len);
+			}
+			else {
+				ret = writev (sock, opvector, opcount);
+			}
 
                         if (ret == 0 || (ret == -1 && errno == EAGAIN)) {
                                 /* done for now */
@@ -178,7 +486,13 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                         }
                         this->total_bytes_write += ret;
                 } else {
-                        ret = readv (sock, opvector, opcount);
+			ret = __socket_cached_read (this, opvector, opcount);
+
+			if (ret == 0) {
+				gf_log(this->name,GF_LOG_DEBUG,"EOF on socket");
+				errno = ENODATA;
+				ret = -1;
+			}
                         if (ret == -1 && errno == EAGAIN) {
                                 /* done for now */
                                 break;
@@ -199,9 +513,18 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                         if (errno == EINTR)
                                 continue;
 
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s failed (%s)", write ? "writev" : "readv",
-                                strerror (errno));
+                        if (__does_socket_rwv_error_need_logging (priv,
+                                                                  write)) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "%s on %s failed (%s)",
+                                        write ? "writev":"readv",
+                                        this->peerinfo.identifier,
+                                        strerror (errno));
+                        }
+
+			if (priv->use_ssl) {
+				ssl_dump_error_stack(this->name);
+			}
                         opcount = -1;
                         break;
                 }
@@ -213,6 +536,17 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                 moved = 0;
 
                 while (moved < ret) {
+                        if (!opcount) {
+                                gf_log(this->name,GF_LOG_DEBUG,
+                                       "ran out of iov, moved %d/%d",
+                                       moved, ret);
+                                goto ran_out;
+                        }
+                        if (!opvector[0].iov_len) {
+                                opvector++;
+                                opcount--;
+                                continue;
+                        }
                         if ((ret - moved) >= opvector[0].iov_len) {
                                 moved += opvector[0].iov_len;
                                 opvector++;
@@ -222,13 +556,11 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                                 opvector[0].iov_base += (ret - moved);
                                 moved += (ret - moved);
                         }
-                        while (opcount && !opvector[0].iov_len) {
-                                opvector++;
-                                opcount--;
-                        }
                 }
         }
 
+ran_out:
+
         if (pending_vector)
                 *pending_vector = opvector;
 
@@ -240,7 +572,7 @@ out:
 }
 
 
-int
+static int
 __socket_readv (rpc_transport_t *this, struct iovec *vector, int count,
                 struct iovec **pending_vector, int *pending_count,
                 size_t *bytes)
@@ -254,7 +586,7 @@ __socket_readv (rpc_transport_t *this, struct iovec *vector, int count,
 }
 
 
-int
+static int
 __socket_writev (rpc_transport_t *this, struct iovec *vector, int count,
                  struct iovec **pending_vector, int *pending_count)
 {
@@ -267,26 +599,55 @@ __socket_writev (rpc_transport_t *this, struct iovec *vector, int count,
 }
 
 
-int
+static int
+__socket_shutdown (rpc_transport_t *this)
+{
+        int               ret = -1;
+        socket_private_t *priv = this->private;
+
+        priv->connected = -1;
+        ret = shutdown (priv->sock, SHUT_RDWR);
+        if (ret) {
+                /* its already disconnected.. no need to understand
+                   why it failed to shutdown in normal cases */
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "shutdown() returned %d. %s",
+                        ret, strerror (errno));
+        }
+
+        return ret;
+}
+
+static int
 __socket_disconnect (rpc_transport_t *this)
 {
-        socket_private_t *priv = NULL;
         int               ret = -1;
+        socket_private_t *priv = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
 
         priv = this->private;
 
+        gf_log (this->name, GF_LOG_TRACE,
+                "disconnecting %p, state=%u gen=%u sock=%d", this,
+                priv->ot_state, priv->ot_gen, priv->sock);
+
         if (priv->sock != -1) {
-                priv->connected = -1;
-                ret = shutdown (priv->sock, SHUT_RDWR);
-                if (ret) {
-                        /* its already disconnected.. no need to understand
-                           why it failed to shutdown in normal cases */
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "shutdown() returned %d. %s",
-                                ret, strerror (errno));
+                ret = __socket_shutdown(this);
+		if (priv->own_thread) {
+                        /*
+                         * Without this, reconnect (= disconnect + connect)
+                         * won't work except by accident.
+                         */
+                        close(priv->sock);
+                        priv->sock = -1;
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "OT_PLEASE_DIE on %p", this);
+                        priv->ot_state = OT_PLEASE_DIE;
+                }
+                else if (priv->use_ssl) {
+                        ssl_teardown_connection(priv);
                 }
         }
 
@@ -295,7 +656,7 @@ out:
 }
 
 
-int
+static int
 __socket_server_bind (rpc_transport_t *this)
 {
         socket_private_t *priv = NULL;
@@ -323,7 +684,7 @@ __socket_server_bind (rpc_transport_t *this)
                 memcpy (&unix_addr, SA (&this->myinfo.sockaddr),
                         this->myinfo.sockaddr_len);
                 reuse_check_sock = socket (AF_UNIX, SOCK_STREAM, 0);
-                if (reuse_check_sock > 0) {
+                if (reuse_check_sock >= 0) {
                         ret = connect (reuse_check_sock, SA (&unix_addr),
                                        this->myinfo.sockaddr_len);
                         if ((ret == -1) && (ECONNREFUSED == errno)) {
@@ -351,7 +712,7 @@ out:
 }
 
 
-int
+static int
 __socket_nonblock (int fd)
 {
         int flags = 0;
@@ -365,8 +726,7 @@ __socket_nonblock (int fd)
         return ret;
 }
 
-
-int
+static int
 __socket_nodelay (int fd)
 {
         int     on = 1;
@@ -383,7 +743,7 @@ __socket_nodelay (int fd)
 
 
 static int
-__socket_keepalive (int fd, int keepalive_intvl, int keepalive_idle)
+__socket_keepalive (int fd, int family, int keepalive_intvl, int keepalive_idle)
 {
         int     on = 1;
         int     ret = -1;
@@ -412,18 +772,23 @@ __socket_keepalive (int fd, int keepalive_intvl, int keepalive_idle)
                 goto err;
         }
 #else
+        if (family != AF_INET)
+                goto done;
+
         ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepalive_idle,
                           sizeof (keepalive_intvl));
         if (ret == -1) {
                 gf_log ("socket", GF_LOG_WARNING,
-                        "failed to set keep idle on socket %d", fd);
+                        "failed to set keep idle %d on socket %d, %s",
+                        keepalive_idle, fd, strerror(errno));
                 goto err;
         }
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPINTVL, &keepalive_intvl,
+        ret = setsockopt (fd, IPPROTO_TCP , TCP_KEEPINTVL, &keepalive_intvl,
                           sizeof (keepalive_intvl));
         if (ret == -1) {
                 gf_log ("socket", GF_LOG_WARNING,
-                        "failed to set keep alive interval on socket %d", fd);
+                        "failed to set keep interval %d on socket %d, %s",
+                        keepalive_intvl, fd, strerror(errno));
                 goto err;
         }
 #endif
@@ -437,7 +802,7 @@ err:
 }
 
 
-int
+static int
 __socket_connect_finish (int fd)
 {
         int       ret = -1;
@@ -455,7 +820,7 @@ __socket_connect_finish (int fd)
 }
 
 
-void
+static void
 __socket_reset (rpc_transport_t *this)
 {
         socket_private_t *priv = NULL;
@@ -476,9 +841,7 @@ __socket_reset (rpc_transport_t *this)
                 iobuf_unref (priv->incoming.iobuf);
         }
 
-        if (priv->incoming.request_info != NULL) {
-                GF_FREE (priv->incoming.request_info);
-        }
+        GF_FREE (priv->incoming.request_info);
 
         memset (&priv->incoming, 0, sizeof (priv->incoming));
 
@@ -494,13 +857,13 @@ out:
 }
 
 
-void
+static void
 socket_set_lastfrag (uint32_t *fragsize) {
         (*fragsize) |= 0x80000000U;
 }
 
 
-void
+static void
 socket_set_frag_header_size (uint32_t size, char *haddr)
 {
         size = htonl (size);
@@ -508,14 +871,14 @@ socket_set_frag_header_size (uint32_t size, char *haddr)
 }
 
 
-void
+static void
 socket_set_last_frag_header_size (uint32_t size, char *haddr)
 {
         socket_set_lastfrag (&size);
         socket_set_frag_header_size (size, haddr);
 }
 
-struct ioq *
+static struct ioq *
 __socket_ioq_new (rpc_transport_t *this, rpc_transport_msg_t *msg)
 {
         struct ioq       *entry = NULL;
@@ -582,7 +945,7 @@ out:
 }
 
 
-void
+static void
 __socket_ioq_entry_free (struct ioq *entry)
 {
         GF_VALIDATE_OR_GOTO ("socket", entry, out);
@@ -599,7 +962,7 @@ out:
 }
 
 
-void
+static void
 __socket_ioq_flush (rpc_transport_t *this)
 {
         socket_private_t *priv = NULL;
@@ -620,10 +983,12 @@ out:
 }
 
 
-int
-__socket_ioq_churn_entry (rpc_transport_t *this, struct ioq *entry)
+static int
+__socket_ioq_churn_entry (rpc_transport_t *this, struct ioq *entry, int direct)
 {
-        int ret = -1;
+        int               ret = -1;
+	socket_private_t *priv = NULL;
+	char              a_byte = 0;
 
         ret = __socket_writev (this, entry->pending_vector,
                                entry->pending_count,
@@ -634,13 +999,25 @@ __socket_ioq_churn_entry (rpc_transport_t *this, struct ioq *entry)
                 /* current entry was completely written */
                 GF_ASSERT (entry->pending_count == 0);
                 __socket_ioq_entry_free (entry);
+		priv = this->private;
+		if (priv->own_thread) {
+			/*
+			 * The pipe should only remain readable if there are
+			 * more entries after this, so drain the byte
+			 * representing this entry.
+			 */
+			if (!direct && read(priv->pipe[0],&a_byte,1) < 1) {
+				gf_log(this->name,GF_LOG_WARNING,
+				       "read error on pipe");
+			}
+		}
         }
 
         return ret;
 }
 
 
-int
+static int
 __socket_ioq_churn (rpc_transport_t *this)
 {
         socket_private_t *priv = NULL;
@@ -656,13 +1033,13 @@ __socket_ioq_churn (rpc_transport_t *this)
                 /* pick next entry */
                 entry = priv->ioq_next;
 
-                ret = __socket_ioq_churn_entry (this, entry);
+                ret = __socket_ioq_churn_entry (this, entry, 0);
 
                 if (ret != 0)
                         break;
         }
 
-        if (list_empty (&priv->ioq)) {
+        if (!priv->own_thread && list_empty (&priv->ioq)) {
                 /* all pending writes done, not interested in POLLOUT */
                 priv->idx = event_select_on (this->ctx->event_pool,
                                              priv->sock, priv->idx, -1, 0);
@@ -673,7 +1050,7 @@ out:
 }
 
 
-int
+static int
 socket_event_poll_err (rpc_transport_t *this)
 {
         socket_private_t *priv = NULL;
@@ -698,7 +1075,7 @@ out:
 }
 
 
-int
+static int
 socket_event_poll_out (rpc_transport_t *this)
 {
         socket_private_t *priv = NULL;
@@ -728,43 +1105,45 @@ out:
 }
 
 
-inline int
+static inline int
 __socket_read_simple_msg (rpc_transport_t *this)
 {
-        socket_private_t *priv           = NULL;
-        int               ret            = 0;
-        uint32_t          remaining_size = 0;
-        size_t            bytes_read     = 0;
+        int                           ret            = 0;
+        uint32_t                      remaining_size = 0;
+        size_t                        bytes_read     = 0;
+        socket_private_t             *priv           = NULL;
+        struct gf_sock_incoming      *in             = NULL;
+        struct gf_sock_incoming_frag *frag           = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
 
         priv = this->private;
 
-        switch (priv->incoming.frag.simple_state) {
+        in = &priv->incoming;
+        frag = &in->frag;
+
+        switch (frag->simple_state) {
 
         case SP_STATE_SIMPLE_MSG_INIT:
-                remaining_size = RPC_FRAGSIZE (priv->incoming.fraghdr)
-                        - priv->incoming.frag.bytes_read;
+                remaining_size = RPC_FRAGSIZE (in->fraghdr) - frag->bytes_read;
 
                 __socket_proto_init_pending (priv, remaining_size);
 
-                priv->incoming.frag.simple_state =
-                        SP_STATE_READING_SIMPLE_MSG;
+                frag->simple_state = SP_STATE_READING_SIMPLE_MSG;
 
                 /* fall through */
 
         case SP_STATE_READING_SIMPLE_MSG:
                 ret = 0;
 
-                remaining_size = RPC_FRAGSIZE (priv->incoming.fraghdr)
-                        - priv->incoming.frag.bytes_read;
+                remaining_size = RPC_FRAGSIZE (in->fraghdr) - frag->bytes_read;
 
                 if (remaining_size > 0) {
                         ret = __socket_readv (this,
-                                              priv->incoming.pending_vector, 1,
-                                              &priv->incoming.pending_vector,
-                                              &priv->incoming.pending_count,
+                                              in->pending_vector, 1,
+                                              &in->pending_vector,
+                                              &in->pending_count,
                                               &bytes_read);
                 }
 
@@ -776,8 +1155,8 @@ __socket_read_simple_msg (rpc_transport_t *this)
                         break;
                 }
 
-                priv->incoming.frag.bytes_read += bytes_read;
-                priv->incoming.frag.fragcurrent += bytes_read;
+                frag->bytes_read += bytes_read;
+                frag->fragcurrent += bytes_read;
 
                 if (ret > 0) {
                         gf_log (this->name, GF_LOG_TRACE,
@@ -786,8 +1165,7 @@ __socket_read_simple_msg (rpc_transport_t *this)
                 }
 
                 if (ret == 0) {
-                        priv->incoming.frag.simple_state
-                                =  SP_STATE_SIMPLE_MSG_INIT;
+                        frag->simple_state =  SP_STATE_SIMPLE_MSG_INIT;
                 }
         }
 
@@ -796,7 +1174,7 @@ out:
 }
 
 
-inline int
+static inline int
 __socket_read_simple_request (rpc_transport_t *this)
 {
         return __socket_read_simple_msg (this);
@@ -813,7 +1191,7 @@ __socket_read_simple_request (rpc_transport_t *this)
 #define rpc_progver_addr(buf) (buf + RPC_MSGTYPE_SIZE + 8)
 #define rpc_procnum_addr(buf) (buf + RPC_MSGTYPE_SIZE + 12)
 
-inline int
+static inline int
 __socket_read_vectored_request (rpc_transport_t *this, rpcsvc_vector_sizer vector_sizer)
 {
         socket_private_t *priv                   = NULL;
@@ -823,16 +1201,26 @@ __socket_read_vectored_request (rpc_transport_t *this, rpcsvc_vector_sizer vecto
         struct iobuf     *iobuf                  = NULL;
         uint32_t          remaining_size         = 0;
         ssize_t           readsize               = 0;
+        size_t            size                   = 0;
+        struct gf_sock_incoming      *in         = NULL;
+        struct gf_sock_incoming_frag *frag       = NULL;
+        sp_rpcfrag_request_state_t   *request    = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
 
         priv = this->private;
 
-        switch (priv->incoming.frag.call_body.request.vector_state) {
+        /* used to reduce the indirection */
+        in = &priv->incoming;
+        frag = &in->frag;
+        request = &frag->call_body.request;
+
+        switch (request->vector_state) {
         case SP_STATE_VECTORED_REQUEST_INIT:
-                priv->incoming.frag.call_body.request.vector_sizer_state = 0;
-                addr = rpc_cred_addr (iobuf_ptr (priv->incoming.iobuf));
+                request->vector_sizer_state = 0;
+
+                addr = rpc_cred_addr (iobuf_ptr (in->iobuf));
 
                 /* also read verf flavour and verflen */
                 credlen = ntoh32 (*((uint32_t *)addr))
@@ -840,99 +1228,114 @@ __socket_read_vectored_request (rpc_transport_t *this, rpcsvc_vector_sizer vecto
 
                 __socket_proto_init_pending (priv, credlen);
 
-                priv->incoming.frag.call_body.request.vector_state =
-                        SP_STATE_READING_CREDBYTES;
+                request->vector_state = SP_STATE_READING_CREDBYTES;
 
                 /* fall through */
 
         case SP_STATE_READING_CREDBYTES:
                 __socket_proto_read (priv, ret);
 
-                priv->incoming.frag.call_body.request.vector_state =
-                        SP_STATE_READ_CREDBYTES;
+                request->vector_state = SP_STATE_READ_CREDBYTES;
 
                 /* fall through */
 
         case SP_STATE_READ_CREDBYTES:
-                addr = rpc_verf_addr (priv->incoming.frag.fragcurrent);
+                addr = rpc_verf_addr (frag->fragcurrent);
                 verflen = ntoh32 (*((uint32_t *)addr));
 
                 if (verflen == 0) {
-                        priv->incoming.frag.call_body.request.vector_state
-                                = SP_STATE_READ_VERFBYTES;
+                        request->vector_state = SP_STATE_READ_VERFBYTES;
                         goto sp_state_read_verfbytes;
                 }
                 __socket_proto_init_pending (priv, verflen);
 
-                priv->incoming.frag.call_body.request.vector_state
-                        = SP_STATE_READING_VERFBYTES;
+                request->vector_state = SP_STATE_READING_VERFBYTES;
 
                 /* fall through */
 
         case SP_STATE_READING_VERFBYTES:
                 __socket_proto_read (priv, ret);
 
-                priv->incoming.frag.call_body.request.vector_state =
-                        SP_STATE_READ_VERFBYTES;
+                request->vector_state = SP_STATE_READ_VERFBYTES;
 
                 /* fall through */
 
         case SP_STATE_READ_VERFBYTES:
 sp_state_read_verfbytes:
-                priv->incoming.frag.call_body.request.vector_sizer_state =
-                        vector_sizer (priv->incoming.frag.call_body.request.vector_sizer_state,
-                                      &readsize,
-                                      priv->incoming.frag.fragcurrent);
+		/* set the base_addr 'persistently' across multiple calls
+		   into the state machine */
+                in->proghdr_base_addr = frag->fragcurrent;
+
+                request->vector_sizer_state =
+                        vector_sizer (request->vector_sizer_state,
+                                      &readsize, in->proghdr_base_addr,
+                                      frag->fragcurrent);
                 __socket_proto_init_pending (priv, readsize);
-                priv->incoming.frag.call_body.request.vector_state
-                        = SP_STATE_READING_PROGHDR;
+
+                request->vector_state = SP_STATE_READING_PROGHDR;
 
                 /* fall through */
 
         case SP_STATE_READING_PROGHDR:
                 __socket_proto_read (priv, ret);
-sp_state_reading_proghdr:
-                priv->incoming.frag.call_body.request.vector_sizer_state =
-                        vector_sizer (priv->incoming.frag.call_body.request.vector_sizer_state,
-                                      &readsize,
-                                      priv->incoming.frag.fragcurrent);
+
+		request->vector_state =	SP_STATE_READ_PROGHDR;
+
+		/* fall through */
+
+	case SP_STATE_READ_PROGHDR:
+sp_state_read_proghdr:
+                request->vector_sizer_state =
+                        vector_sizer (request->vector_sizer_state,
+                                      &readsize, in->proghdr_base_addr,
+                                      frag->fragcurrent);
                 if (readsize == 0) {
-                        priv->incoming.frag.call_body.request.vector_state =
-                                SP_STATE_READ_PROGHDR;
-                } else {
-                        __socket_proto_init_pending (priv, readsize);
-                        __socket_proto_read (priv, ret);
-                        goto sp_state_reading_proghdr;
+                        request->vector_state = SP_STATE_READ_PROGHDR_XDATA;
+			goto sp_state_read_proghdr_xdata;
                 }
 
-        case SP_STATE_READ_PROGHDR:
-                if (priv->incoming.payload_vector.iov_base == NULL) {
-                        iobuf = iobuf_get (this->ctx->iobuf_pool);
+		__socket_proto_init_pending (priv, readsize);
+
+                request->vector_state =	SP_STATE_READING_PROGHDR_XDATA;
+
+		/* fall through */
+
+	case SP_STATE_READING_PROGHDR_XDATA:
+		__socket_proto_read (priv, ret);
+
+		request->vector_state =	SP_STATE_READ_PROGHDR;
+		/* check if the vector_sizer() has more to say */
+		goto sp_state_read_proghdr;
+
+        case SP_STATE_READ_PROGHDR_XDATA:
+sp_state_read_proghdr_xdata:
+                if (in->payload_vector.iov_base == NULL) {
+
+                        size = RPC_FRAGSIZE (in->fraghdr) - frag->bytes_read;
+                        iobuf = iobuf_get2 (this->ctx->iobuf_pool, size);
                         if (!iobuf) {
                                 ret = -1;
                                 break;
                         }
 
-                        if (priv->incoming.iobref == NULL) {
-                                priv->incoming.iobref = iobref_new ();
-                                if (priv->incoming.iobref == NULL) {
+                        if (in->iobref == NULL) {
+                                in->iobref = iobref_new ();
+                                if (in->iobref == NULL) {
                                         ret = -1;
                                         iobuf_unref (iobuf);
                                         break;
                                 }
                         }
 
-                        iobref_add (priv->incoming.iobref, iobuf);
+                        iobref_add (in->iobref, iobuf);
                         iobuf_unref (iobuf);
 
-                        priv->incoming.payload_vector.iov_base
-                                = iobuf_ptr (iobuf);
+                        in->payload_vector.iov_base = iobuf_ptr (iobuf);
 
-                        priv->incoming.frag.fragcurrent = iobuf_ptr (iobuf);
+                        frag->fragcurrent = iobuf_ptr (iobuf);
                 }
 
-                priv->incoming.frag.call_body.request.vector_state =
-                        SP_STATE_READING_PROG;
+                request->vector_state = SP_STATE_READING_PROG;
 
                 /* fall through */
 
@@ -943,19 +1346,15 @@ sp_state_reading_proghdr:
 
                 ret = __socket_read_simple_msg (this);
 
-                remaining_size = RPC_FRAGSIZE (priv->incoming.fraghdr)
-                        - priv->incoming.frag.bytes_read;
+                remaining_size = RPC_FRAGSIZE (in->fraghdr) - frag->bytes_read;
 
-                if ((ret == -1)
-                    || ((ret == 0)
-                        && (remaining_size == 0)
-                        && RPC_LASTFRAG (priv->incoming.fraghdr))) {
-                        priv->incoming.frag.call_body.request.vector_state
-                                = SP_STATE_VECTORED_REQUEST_INIT;
-                        priv->incoming.payload_vector.iov_len
-                                = (unsigned long)priv->incoming.frag.fragcurrent
-                                - (unsigned long)
-                                priv->incoming.payload_vector.iov_base;
+                if ((ret == -1) ||
+                    ((ret == 0) && (remaining_size == 0)
+                     && RPC_LASTFRAG (in->fraghdr))) {
+                        request->vector_state = SP_STATE_VECTORED_REQUEST_INIT;
+                        in->payload_vector.iov_len
+                                = ((unsigned long)frag->fragcurrent
+                                   - (unsigned long)in->payload_vector.iov_base);
                 }
                 break;
         }
@@ -964,7 +1363,7 @@ out:
         return ret;
 }
 
-inline int
+static inline int
 __socket_read_request (rpc_transport_t *this)
 {
         socket_private_t *priv               = NULL;
@@ -973,46 +1372,53 @@ __socket_read_request (rpc_transport_t *this)
         int               ret                = -1;
         char             *buf                = NULL;
         rpcsvc_vector_sizer     vector_sizer = NULL;
+        struct gf_sock_incoming      *in         = NULL;
+        struct gf_sock_incoming_frag *frag       = NULL;
+        sp_rpcfrag_request_state_t   *request    = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
 
         priv = this->private;
 
-        switch (priv->incoming.frag.call_body.request.header_state) {
+        /* used to reduce the indirection */
+        in = &priv->incoming;
+        frag = &in->frag;
+        request = &frag->call_body.request;
+
+        switch (request->header_state) {
 
         case SP_STATE_REQUEST_HEADER_INIT:
 
                 __socket_proto_init_pending (priv, RPC_CALL_BODY_SIZE);
 
-                priv->incoming.frag.call_body.request.header_state
-                        = SP_STATE_READING_RPCHDR1;
+                request->header_state = SP_STATE_READING_RPCHDR1;
 
                 /* fall through */
 
         case SP_STATE_READING_RPCHDR1:
                 __socket_proto_read (priv, ret);
 
-                priv->incoming.frag.call_body.request.header_state =
-                        SP_STATE_READ_RPCHDR1;
+                request->header_state = SP_STATE_READ_RPCHDR1;
 
                 /* fall through */
 
         case SP_STATE_READ_RPCHDR1:
-                buf = rpc_prognum_addr (iobuf_ptr (priv->incoming.iobuf));
+                buf = rpc_prognum_addr (iobuf_ptr (in->iobuf));
                 prognum = ntoh32 (*((uint32_t *)buf));
 
-                buf = rpc_progver_addr (iobuf_ptr (priv->incoming.iobuf));
+                buf = rpc_progver_addr (iobuf_ptr (in->iobuf));
                 progver = ntoh32 (*((uint32_t *)buf));
 
-                buf = rpc_procnum_addr (iobuf_ptr (priv->incoming.iobuf));
+                buf = rpc_procnum_addr (iobuf_ptr (in->iobuf));
                 procnum = ntoh32 (*((uint32_t *)buf));
 
-                if (this->listener) {
-                        /* this check is needed as rpcsvc and rpc-clnt actor structures are
-                         * not same */
-                        vector_sizer = rpcsvc_get_program_vector_sizer ((rpcsvc_t *)this->mydata,
-                                                                        prognum, progver, procnum);
+                if (priv->is_server) {
+                        /* this check is needed as rpcsvc and rpc-clnt
+                         * actor structures are not same */
+                        vector_sizer =
+                                rpcsvc_get_program_vector_sizer ((rpcsvc_t *)this->mydata,
+                                                                 prognum, progver, procnum);
                 }
 
                 if (vector_sizer) {
@@ -1021,15 +1427,13 @@ __socket_read_request (rpc_transport_t *this)
                         ret = __socket_read_simple_request (this);
                 }
 
-                remaining_size = RPC_FRAGSIZE (priv->incoming.fraghdr)
-                        - priv->incoming.frag.bytes_read;
+                remaining_size = RPC_FRAGSIZE (in->fraghdr) - frag->bytes_read;
 
                 if ((ret == -1)
                     || ((ret == 0)
                         && (remaining_size == 0)
-                        && (RPC_LASTFRAG (priv->incoming.fraghdr)))) {
-                        priv->incoming.frag.call_body.request.header_state =
-                                SP_STATE_REQUEST_HEADER_INIT;
+                        && (RPC_LASTFRAG (in->fraghdr)))) {
+                        request->header_state = SP_STATE_REQUEST_HEADER_INIT;
                 }
 
                 break;
@@ -1040,35 +1444,40 @@ out:
 }
 
 
-inline int
+static inline int
 __socket_read_accepted_successful_reply (rpc_transport_t *this)
 {
-        socket_private_t *priv                     = NULL;
-        int               ret                      = 0;
-        struct iobuf     *iobuf                    = NULL;
-        uint32_t          gluster_read_rsp_hdr_len = 0;
-        gfs3_read_rsp     read_rsp                 = {0, };
+        socket_private_t *priv              = NULL;
+        int               ret               = 0;
+        struct iobuf     *iobuf             = NULL;
+        gfs3_read_rsp     read_rsp          = {0, };
+        ssize_t           size              = 0;
+        ssize_t           default_read_size = 0;
+        char             *proghdr_buf       = NULL;
+        XDR               xdr;
+        struct gf_sock_incoming      *in         = NULL;
+        struct gf_sock_incoming_frag *frag       = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
 
         priv = this->private;
 
-        switch (priv->incoming.frag.call_body.reply.accepted_success_state) {
+        /* used to reduce the indirection */
+        in = &priv->incoming;
+        frag = &in->frag;
+
+        switch (frag->call_body.reply.accepted_success_state) {
 
         case SP_STATE_ACCEPTED_SUCCESS_REPLY_INIT:
-                gluster_read_rsp_hdr_len = xdr_sizeof ((xdrproc_t) xdr_gfs3_read_rsp,
-                                                       &read_rsp);
+                default_read_size = xdr_sizeof ((xdrproc_t) xdr_gfs3_read_rsp,
+                                                &read_rsp);
 
-                if (gluster_read_rsp_hdr_len == 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "xdr_sizeof on gfs3_read_rsp failed");
-                        ret = -1;
-                        goto out;
-                }
-                __socket_proto_init_pending (priv, gluster_read_rsp_hdr_len);
+                proghdr_buf = frag->fragcurrent;
+
+                __socket_proto_init_pending (priv, default_read_size);
 
-                priv->incoming.frag.call_body.reply.accepted_success_state
+                frag->call_body.reply.accepted_success_state
                         = SP_STATE_READING_PROC_HEADER;
 
                 /* fall through */
@@ -1076,34 +1485,70 @@ __socket_read_accepted_successful_reply (rpc_transport_t *this)
         case SP_STATE_READING_PROC_HEADER:
                 __socket_proto_read (priv, ret);
 
-                priv->incoming.frag.call_body.reply.accepted_success_state
-                        = SP_STATE_READ_PROC_HEADER;
+                /* there can be 'xdata' in read response, figure it out */
+                xdrmem_create (&xdr, proghdr_buf, default_read_size,
+                               XDR_DECODE);
+
+                /* This will fail if there is xdata sent from server, if not,
+                   well and good, we don't need to worry about  */
+                xdr_gfs3_read_rsp (&xdr, &read_rsp);
+
+                free (read_rsp.xdata.xdata_val);
+
+                /* need to round off to proper roof (%4), as XDR packing pads
+                   the end of opaque object with '0' */
+                size = roof (read_rsp.xdata.xdata_len, 4);
 
-                if (priv->incoming.payload_vector.iov_base == NULL) {
-                        iobuf = iobuf_get (this->ctx->iobuf_pool);
+                if (!size) {
+                        frag->call_body.reply.accepted_success_state
+                                = SP_STATE_READ_PROC_OPAQUE;
+                        goto read_proc_opaque;
+                }
+
+                __socket_proto_init_pending (priv, size);
+
+                frag->call_body.reply.accepted_success_state
+                        = SP_STATE_READING_PROC_OPAQUE;
+
+        case SP_STATE_READING_PROC_OPAQUE:
+                __socket_proto_read (priv, ret);
+
+                frag->call_body.reply.accepted_success_state
+                        = SP_STATE_READ_PROC_OPAQUE;
+
+        case SP_STATE_READ_PROC_OPAQUE:
+        read_proc_opaque:
+                if (in->payload_vector.iov_base == NULL) {
+
+                        size = (RPC_FRAGSIZE (in->fraghdr) - frag->bytes_read);
+
+                        iobuf = iobuf_get2 (this->ctx->iobuf_pool, size);
                         if (iobuf == NULL) {
                                 ret = -1;
                                 goto out;
                         }
 
-                        if (priv->incoming.iobref == NULL) {
-                                priv->incoming.iobref = iobref_new ();
-                                if (priv->incoming.iobref == NULL) {
+                        if (in->iobref == NULL) {
+                                in->iobref = iobref_new ();
+                                if (in->iobref == NULL) {
                                         ret = -1;
                                         iobuf_unref (iobuf);
                                         goto out;
                                 }
                         }
 
-                        iobref_add (priv->incoming.iobref, iobuf);
+                        iobref_add (in->iobref, iobuf);
                         iobuf_unref (iobuf);
 
-                        priv->incoming.payload_vector.iov_base
-                                = iobuf_ptr (iobuf);
+                        in->payload_vector.iov_base = iobuf_ptr (iobuf);
+
+                        in->payload_vector.iov_len = size;
                 }
 
-                priv->incoming.frag.fragcurrent
-                        = priv->incoming.payload_vector.iov_base;
+                frag->fragcurrent = in->payload_vector.iov_base;
+
+                frag->call_body.reply.accepted_success_state
+                        = SP_STATE_READ_PROC_HEADER;
 
                 /* fall through */
 
@@ -1111,9 +1556,8 @@ __socket_read_accepted_successful_reply (rpc_transport_t *this)
                 /* now read the entire remaining msg into new iobuf */
                 ret = __socket_read_simple_msg (this);
                 if ((ret == -1)
-                    || ((ret == 0)
-                        && RPC_LASTFRAG (priv->incoming.fraghdr))) {
-                        priv->incoming.frag.call_body.reply.accepted_success_state
+                    || ((ret == 0) && RPC_LASTFRAG (in->fraghdr))) {
+                        frag->call_body.reply.accepted_success_state
                                 = SP_STATE_ACCEPTED_SUCCESS_REPLY_INIT;
                 }
 
@@ -1127,7 +1571,7 @@ out:
 #define rpc_reply_verflen_addr(fragcurrent) ((char *)fragcurrent - 4)
 #define rpc_reply_accept_status_addr(fragcurrent) ((char *)fragcurrent - 4)
 
-inline int
+static inline int
 __socket_read_accepted_reply (rpc_transport_t *this)
 {
         socket_private_t *priv           = NULL;
@@ -1135,19 +1579,24 @@ __socket_read_accepted_reply (rpc_transport_t *this)
         char             *buf            = NULL;
         uint32_t          verflen        = 0, len = 0;
         uint32_t          remaining_size = 0;
+        struct gf_sock_incoming      *in         = NULL;
+        struct gf_sock_incoming_frag *frag       = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
 
         priv = this->private;
+        /* used to reduce the indirection */
+        in = &priv->incoming;
+        frag = &in->frag;
 
-        switch (priv->incoming.frag.call_body.reply.accepted_state) {
+        switch (frag->call_body.reply.accepted_state) {
 
         case SP_STATE_ACCEPTED_REPLY_INIT:
                 __socket_proto_init_pending (priv,
                                              RPC_AUTH_FLAVOUR_N_LENGTH_SIZE);
 
-                priv->incoming.frag.call_body.reply.accepted_state
+                frag->call_body.reply.accepted_state
                         = SP_STATE_READING_REPLY_VERFLEN;
 
                 /* fall through */
@@ -1155,13 +1604,13 @@ __socket_read_accepted_reply (rpc_transport_t *this)
         case SP_STATE_READING_REPLY_VERFLEN:
                 __socket_proto_read (priv, ret);
 
-                priv->incoming.frag.call_body.reply.accepted_state
+                frag->call_body.reply.accepted_state
                         = SP_STATE_READ_REPLY_VERFLEN;
 
                 /* fall through */
 
         case SP_STATE_READ_REPLY_VERFLEN:
-                buf = rpc_reply_verflen_addr (priv->incoming.frag.fragcurrent);
+                buf = rpc_reply_verflen_addr (frag->fragcurrent);
 
                 verflen = ntoh32 (*((uint32_t *) buf));
 
@@ -1170,7 +1619,7 @@ __socket_read_accepted_reply (rpc_transport_t *this)
 
                 __socket_proto_init_pending (priv, len);
 
-                priv->incoming.frag.call_body.reply.accepted_state
+                frag->call_body.reply.accepted_state
                         = SP_STATE_READING_REPLY_VERFBYTES;
 
                 /* fall through */
@@ -1178,19 +1627,19 @@ __socket_read_accepted_reply (rpc_transport_t *this)
         case SP_STATE_READING_REPLY_VERFBYTES:
                 __socket_proto_read (priv, ret);
 
-                priv->incoming.frag.call_body.reply.accepted_state
+                frag->call_body.reply.accepted_state
                         = SP_STATE_READ_REPLY_VERFBYTES;
 
-                buf = rpc_reply_accept_status_addr (priv->incoming.frag.fragcurrent);
+                buf = rpc_reply_accept_status_addr (frag->fragcurrent);
 
-                priv->incoming.frag.call_body.reply.accept_status
+                frag->call_body.reply.accept_status
                         = ntoh32 (*(uint32_t *) buf);
 
                 /* fall through */
 
         case SP_STATE_READ_REPLY_VERFBYTES:
 
-                if (priv->incoming.frag.call_body.reply.accept_status
+                if (frag->call_body.reply.accept_status
                     == SUCCESS) {
                         ret = __socket_read_accepted_successful_reply (this);
                 } else {
@@ -1200,14 +1649,13 @@ __socket_read_accepted_reply (rpc_transport_t *this)
                         ret = __socket_read_simple_msg (this);
                 }
 
-                remaining_size = RPC_FRAGSIZE (priv->incoming.fraghdr)
-                        - priv->incoming.frag.bytes_read;
+                remaining_size = RPC_FRAGSIZE (in->fraghdr)
+                        - frag->bytes_read;
 
                 if ((ret == -1)
-                    || ((ret == 0)
-                        && (remaining_size == 0)
-                        && (RPC_LASTFRAG (priv->incoming.fraghdr)))) {
-                        priv->incoming.frag.call_body.reply.accepted_state
+                    || ((ret == 0) && (remaining_size == 0)
+                        && (RPC_LASTFRAG (in->fraghdr)))) {
+                        frag->call_body.reply.accepted_state
                                 = SP_STATE_ACCEPTED_REPLY_INIT;
                 }
 
@@ -1219,7 +1667,7 @@ out:
 }
 
 
-inline int
+static inline int
 __socket_read_denied_reply (rpc_transport_t *this)
 {
         return __socket_read_simple_msg (this);
@@ -1229,25 +1677,29 @@ __socket_read_denied_reply (rpc_transport_t *this)
 #define rpc_reply_status_addr(fragcurrent) ((char *)fragcurrent - 4)
 
 
-inline int
+static inline int
 __socket_read_vectored_reply (rpc_transport_t *this)
 {
         socket_private_t *priv           = NULL;
         int               ret            = 0;
         char             *buf            = NULL;
         uint32_t          remaining_size = 0;
+        struct gf_sock_incoming      *in         = NULL;
+        struct gf_sock_incoming_frag *frag       = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
 
         priv = this->private;
+        in = &priv->incoming;
+        frag = &in->frag;
 
-        switch (priv->incoming.frag.call_body.reply.status_state) {
+        switch (frag->call_body.reply.status_state) {
 
         case SP_STATE_ACCEPTED_REPLY_INIT:
                 __socket_proto_init_pending (priv, RPC_REPLY_STATUS_SIZE);
 
-                priv->incoming.frag.call_body.reply.status_state
+                frag->call_body.reply.status_state
                         = SP_STATE_READING_REPLY_STATUS;
 
                 /* fall through */
@@ -1255,37 +1707,33 @@ __socket_read_vectored_reply (rpc_transport_t *this)
         case SP_STATE_READING_REPLY_STATUS:
                 __socket_proto_read (priv, ret);
 
-                buf = rpc_reply_status_addr (priv->incoming.frag.fragcurrent);
+                buf = rpc_reply_status_addr (frag->fragcurrent);
 
-                priv->incoming.frag.call_body.reply.accept_status
+                frag->call_body.reply.accept_status
                         = ntoh32 (*((uint32_t *) buf));
 
-                priv->incoming.frag.call_body.reply.status_state
+                frag->call_body.reply.status_state
                         = SP_STATE_READ_REPLY_STATUS;
 
                 /* fall through */
 
         case SP_STATE_READ_REPLY_STATUS:
-                if (priv->incoming.frag.call_body.reply.accept_status
-                    == MSG_ACCEPTED) {
+                if (frag->call_body.reply.accept_status == MSG_ACCEPTED) {
                         ret = __socket_read_accepted_reply (this);
                 } else {
                         ret = __socket_read_denied_reply (this);
                 }
 
-                remaining_size = RPC_FRAGSIZE (priv->incoming.fraghdr)
-                        - priv->incoming.frag.bytes_read;
+                remaining_size = RPC_FRAGSIZE (in->fraghdr) - frag->bytes_read;
 
                 if ((ret == -1)
-                    || ((ret == 0)
-                        && (remaining_size == 0)
-                        && (RPC_LASTFRAG (priv->incoming.fraghdr)))) {
-                        priv->incoming.frag.call_body.reply.status_state
+                    || ((ret == 0) && (remaining_size == 0)
+                        && (RPC_LASTFRAG (in->fraghdr)))) {
+                        frag->call_body.reply.status_state
                                 = SP_STATE_ACCEPTED_REPLY_INIT;
-                        priv->incoming.payload_vector.iov_len
-                                = (unsigned long)priv->incoming.frag.fragcurrent
-                                - (unsigned long)
-                                priv->incoming.payload_vector.iov_base;
+                        in->payload_vector.iov_len
+                                = (unsigned long)frag->fragcurrent
+                                - (unsigned long)in->payload_vector.iov_base;
                 }
                 break;
         }
@@ -1295,7 +1743,7 @@ out:
 }
 
 
-inline int
+static inline int
 __socket_read_simple_reply (rpc_transport_t *this)
 {
         return __socket_read_simple_msg (this);
@@ -1303,7 +1751,7 @@ __socket_read_simple_reply (rpc_transport_t *this)
 
 #define rpc_xid_addr(buf) (buf)
 
-inline int
+static inline int
 __socket_read_reply (rpc_transport_t *this)
 {
         socket_private_t   *priv         = NULL;
@@ -1311,26 +1759,29 @@ __socket_read_reply (rpc_transport_t *this)
         int32_t             ret          = -1;
         rpc_request_info_t *request_info = NULL;
         char                map_xid      = 0;
+        struct gf_sock_incoming      *in         = NULL;
+        struct gf_sock_incoming_frag *frag       = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
 
         priv = this->private;
+        in = &priv->incoming;
+        frag = &in->frag;
 
-        buf = rpc_xid_addr (iobuf_ptr (priv->incoming.iobuf));
+        buf = rpc_xid_addr (iobuf_ptr (in->iobuf));
 
-        if (priv->incoming.request_info == NULL) {
-                priv->incoming.request_info = GF_CALLOC (1,
-                                                         sizeof (*request_info),
-                                                         gf_common_mt_rpc_trans_reqinfo_t);
-                if (priv->incoming.request_info == NULL) {
+        if (in->request_info == NULL) {
+                in->request_info = GF_CALLOC (1, sizeof (*request_info),
+                                              gf_common_mt_rpc_trans_reqinfo_t);
+                if (in->request_info == NULL) {
                         goto out;
                 }
 
                 map_xid = 1;
         }
 
-        request_info = priv->incoming.request_info;
+        request_info = in->request_info;
 
         if (map_xid) {
                 request_info->xid = ntoh32 (*((uint32_t *) buf));
@@ -1342,7 +1793,7 @@ __socket_read_reply (rpc_transport_t *this)
                 {
                         ret = rpc_transport_notify (this,
                                                     RPC_TRANSPORT_MAP_XID_REQUEST,
-                                                    priv->incoming.request_info);
+                                                    in->request_info);
                 }
                 pthread_mutex_lock (&priv->lock);
 
@@ -1353,13 +1804,11 @@ __socket_read_reply (rpc_transport_t *this)
                 }
         }
 
-        if ((request_info->prognum == GLUSTER3_1_FOP_PROGRAM)
+        if ((request_info->prognum == GLUSTER_FOP_PROGRAM)
             && (request_info->procnum == GF_FOP_READ)) {
                 if (map_xid && request_info->rsp.rsp_payload_count != 0) {
-                        priv->incoming.iobref
-                                = iobref_ref (request_info->rsp.rsp_iobref);
-                        priv->incoming.payload_vector
-                                = *request_info->rsp.rsp_payload;
+                        in->iobref = iobref_ref (request_info->rsp.rsp_iobref);
+                        in->payload_vector = *request_info->rsp.rsp_payload;
                 }
 
                 ret = __socket_read_vectored_reply (this);
@@ -1372,42 +1821,47 @@ out:
 
 
 /* returns the number of bytes yet to be read in a fragment */
-inline int
+static inline int
 __socket_read_frag (rpc_transport_t *this)
 {
         socket_private_t *priv           = NULL;
         int32_t           ret            = 0;
         char             *buf            = NULL;
         uint32_t          remaining_size = 0;
+        struct gf_sock_incoming      *in         = NULL;
+        struct gf_sock_incoming_frag *frag       = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
 
         priv = this->private;
+        /* used to reduce the indirection */
+        in = &priv->incoming;
+        frag = &in->frag;
 
-        switch (priv->incoming.frag.state) {
+        switch (frag->state) {
         case SP_STATE_NADA:
                 __socket_proto_init_pending (priv, RPC_MSGTYPE_SIZE);
 
-                priv->incoming.frag.state = SP_STATE_READING_MSGTYPE;
+                frag->state = SP_STATE_READING_MSGTYPE;
 
                 /* fall through */
 
         case SP_STATE_READING_MSGTYPE:
                 __socket_proto_read (priv, ret);
 
-                priv->incoming.frag.state = SP_STATE_READ_MSGTYPE;
+                frag->state = SP_STATE_READ_MSGTYPE;
                 /* fall through */
 
         case SP_STATE_READ_MSGTYPE:
-                buf = rpc_msgtype_addr (iobuf_ptr (priv->incoming.iobuf));
-                priv->incoming.msg_type = ntoh32 (*((uint32_t *)buf));
+                buf = rpc_msgtype_addr (iobuf_ptr (in->iobuf));
+                in->msg_type = ntoh32 (*((uint32_t *)buf));
 
-                if (priv->incoming.msg_type == CALL) {
+                if (in->msg_type == CALL) {
                         ret = __socket_read_request (this);
-                } else if (priv->incoming.msg_type == REPLY) {
+                } else if (in->msg_type == REPLY) {
                         ret = __socket_read_reply (this);
-                } else if (priv->incoming.msg_type == GF_UNIVERSAL_ANSWER) {
+                } else if (in->msg_type == GF_UNIVERSAL_ANSWER) {
                         gf_log ("rpc", GF_LOG_ERROR,
                                 "older version of protocol/process trying to "
                                 "connect from %s. use newer version on that node",
@@ -1415,19 +1869,17 @@ __socket_read_frag (rpc_transport_t *this)
                 } else {
                         gf_log ("rpc", GF_LOG_ERROR,
                                 "wrong MSG-TYPE (%d) received from %s",
-                                priv->incoming.msg_type,
+                                in->msg_type,
                                 this->peerinfo.identifier);
                         ret = -1;
                 }
 
-                remaining_size = RPC_FRAGSIZE (priv->incoming.fraghdr)
-                        - priv->incoming.frag.bytes_read;
+                remaining_size = RPC_FRAGSIZE (in->fraghdr) - frag->bytes_read;
 
                 if ((ret == -1)
-                    || ((ret == 0)
-                        && (remaining_size == 0)
-                        && (RPC_LASTFRAG (priv->incoming.fraghdr)))) {
-                        priv->incoming.frag.state = SP_STATE_NADA;
+                    || ((ret == 0) && (remaining_size == 0)
+                        && (RPC_LASTFRAG (in->fraghdr)))) {
+                        frag->state = SP_STATE_NADA;
                 }
 
                 break;
@@ -1438,31 +1890,36 @@ out:
 }
 
 
-inline
+static inline
 void __socket_reset_priv (socket_private_t *priv)
 {
-        if (priv->incoming.iobref) {
-                iobref_unref (priv->incoming.iobref);
-                priv->incoming.iobref = NULL;
+        struct gf_sock_incoming      *in         = NULL;
+
+        /* used to reduce the indirection */
+        in = &priv->incoming;
+
+        if (in->iobref) {
+                iobref_unref (in->iobref);
+                in->iobref = NULL;
         }
 
-        if (priv->incoming.iobuf) {
-                iobuf_unref (priv->incoming.iobuf);
+        if (in->iobuf) {
+                iobuf_unref (in->iobuf);
         }
 
-        if (priv->incoming.request_info != NULL) {
-                GF_FREE (priv->incoming.request_info);
-                priv->incoming.request_info = NULL;
+        if (in->request_info != NULL) {
+                GF_FREE (in->request_info);
+                in->request_info = NULL;
         }
 
-        memset (&priv->incoming.payload_vector, 0,
-                sizeof (priv->incoming.payload_vector));
+        memset (&in->payload_vector, 0,
+                sizeof (in->payload_vector));
 
-        priv->incoming.iobuf = NULL;
+        in->iobuf = NULL;
 }
 
 
-int
+static int
 __socket_proto_state_machine (rpc_transport_t *this,
                               rpc_transport_pollin_t **pollin)
 {
@@ -1471,46 +1928,40 @@ __socket_proto_state_machine (rpc_transport_t *this,
         struct iobuf     *iobuf  = NULL;
         struct iobref    *iobref = NULL;
         struct iovec      vector[2];
+        struct gf_sock_incoming      *in         = NULL;
+        struct gf_sock_incoming_frag *frag       = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
 
         priv = this->private;
-        while (priv->incoming.record_state != SP_STATE_COMPLETE) {
-                switch (priv->incoming.record_state) {
+        /* used to reduce the indirection */
+        in = &priv->incoming;
+        frag = &in->frag;
+
+        while (in->record_state != SP_STATE_COMPLETE) {
+                switch (in->record_state) {
 
                 case SP_STATE_NADA:
-                        priv->incoming.total_bytes_read = 0;
-                        priv->incoming.payload_vector.iov_len = 0;
+                        in->total_bytes_read = 0;
+                        in->payload_vector.iov_len = 0;
 
-                        priv->incoming.pending_vector = priv->incoming.vector;
-                        priv->incoming.pending_vector->iov_base =
-                                &priv->incoming.fraghdr;
+                        in->pending_vector = in->vector;
+                        in->pending_vector->iov_base =  &in->fraghdr;
 
-                        priv->incoming.pending_vector->iov_len  =
-                                sizeof (priv->incoming.fraghdr);
+                        in->pending_vector->iov_len  = sizeof (in->fraghdr);
 
-                        priv->incoming.record_state = SP_STATE_READING_FRAGHDR;
+                        in->record_state = SP_STATE_READING_FRAGHDR;
 
                         /* fall through */
 
                 case SP_STATE_READING_FRAGHDR:
-                        ret = __socket_readv (this,
-                                              priv->incoming.pending_vector, 1,
-                                              &priv->incoming.pending_vector,
-                                              &priv->incoming.pending_count,
+                        ret = __socket_readv (this, in->pending_vector, 1,
+                                              &in->pending_vector,
+                                              &in->pending_count,
                                               NULL);
-                        if (ret == -1) {
-                                if (priv->read_fail_log == 1) {
-                                        gf_log (this->name,
-                                                ((priv->connected == 1) ?
-                                                 GF_LOG_WARNING : GF_LOG_DEBUG),
-                                                "reading from socket failed. Error (%s)"
-                                                ", peer (%s)", strerror (errno),
-                                                this->peerinfo.identifier);
-                                }
+                        if (ret == -1)
                                 goto out;
-                        }
 
                         if (ret > 0) {
                                 gf_log (this->name, GF_LOG_TRACE, "partial "
@@ -1519,44 +1970,40 @@ __socket_proto_state_machine (rpc_transport_t *this,
                         }
 
                         if (ret == 0) {
-                                priv->incoming.record_state =
-                                        SP_STATE_READ_FRAGHDR;
+                                in->record_state = SP_STATE_READ_FRAGHDR;
                         }
                         /* fall through */
 
                 case SP_STATE_READ_FRAGHDR:
 
-                        priv->incoming.fraghdr = ntoh32 (priv->incoming.fraghdr);
-                        priv->incoming.record_state = SP_STATE_READING_FRAG;
-                        priv->incoming.total_bytes_read
-                                += RPC_FRAGSIZE(priv->incoming.fraghdr);
+                        in->fraghdr = ntoh32 (in->fraghdr);
+                        in->total_bytes_read += RPC_FRAGSIZE(in->fraghdr);
                         iobuf = iobuf_get2 (this->ctx->iobuf_pool,
-                                            priv->incoming.total_bytes_read +
-                                            sizeof (priv->incoming.fraghdr));
+                                            (in->total_bytes_read +
+                                             sizeof (in->fraghdr)));
                         if (!iobuf) {
                                 ret = -ENOMEM;
                                 goto out;
                         }
 
-                        priv->incoming.iobuf = iobuf;
-                        priv->incoming.iobuf_size = 0;
-                        priv->incoming.frag.fragcurrent = iobuf_ptr (iobuf);
+                        in->iobuf = iobuf;
+                        in->iobuf_size = 0;
+                        frag->fragcurrent = iobuf_ptr (iobuf);
+                        in->record_state = SP_STATE_READING_FRAG;
                         /* fall through */
 
                 case SP_STATE_READING_FRAG:
                         ret = __socket_read_frag (this);
 
-                        if ((ret == -1)
-                            || (priv->incoming.frag.bytes_read !=
-                                RPC_FRAGSIZE (priv->incoming.fraghdr))) {
+                        if ((ret == -1) ||
+                            (frag->bytes_read != RPC_FRAGSIZE (in->fraghdr))) {
                                 goto out;
                         }
 
-                        priv->incoming.frag.bytes_read = 0;
+                        frag->bytes_read = 0;
 
-                        if (!RPC_LASTFRAG (priv->incoming.fraghdr)) {
-                                priv->incoming.record_state =
-                                        SP_STATE_READING_FRAGHDR;
+                        if (!RPC_LASTFRAG (in->fraghdr)) {
+                                in->record_state = SP_STATE_READING_FRAGHDR;
                                 break;
                         }
 
@@ -1565,44 +2012,39 @@ __socket_proto_state_machine (rpc_transport_t *this,
                          */
                         if (pollin != NULL) {
                                 int count = 0;
-                                priv->incoming.iobuf_size
-                                        = priv->incoming.total_bytes_read
-                                        - priv->incoming.payload_vector.iov_len;
+                                in->iobuf_size = (in->total_bytes_read -
+                                                  in->payload_vector.iov_len);
 
                                 memset (vector, 0, sizeof (vector));
 
-                                if (priv->incoming.iobref == NULL) {
-                                        priv->incoming.iobref = iobref_new ();
-                                        if (priv->incoming.iobref == NULL) {
+                                if (in->iobref == NULL) {
+                                        in->iobref = iobref_new ();
+                                        if (in->iobref == NULL) {
                                                 ret = -1;
                                                 goto out;
                                         }
                                 }
 
-                                vector[count].iov_base
-                                        = iobuf_ptr (priv->incoming.iobuf);
-                                vector[count].iov_len
-                                        = priv->incoming.iobuf_size;
+                                vector[count].iov_base = iobuf_ptr (in->iobuf);
+                                vector[count].iov_len = in->iobuf_size;
 
-                                iobref = priv->incoming.iobref;
+                                iobref = in->iobref;
 
                                 count++;
 
-                                if (priv->incoming.payload_vector.iov_base
-                                    != NULL) {
-                                        vector[count]
-                                                = priv->incoming.payload_vector;
+                                if (in->payload_vector.iov_base != NULL) {
+                                        vector[count] = in->payload_vector;
                                         count++;
                                 }
 
                                 *pollin = rpc_transport_pollin_alloc (this,
                                                                       vector,
                                                                       count,
-                                                                      priv->incoming.iobuf,
+                                                                      in->iobuf,
                                                                       iobref,
-                                                                      priv->incoming.request_info);
-                                iobuf_unref (priv->incoming.iobuf);
-                                priv->incoming.iobuf = NULL;
+                                                                      in->request_info);
+                                iobuf_unref (in->iobuf);
+                                in->iobuf = NULL;
 
                                 if (*pollin == NULL) {
                                         gf_log (this->name, GF_LOG_WARNING,
@@ -1610,12 +2052,12 @@ __socket_proto_state_machine (rpc_transport_t *this,
                                         ret = -1;
                                         goto out;
                                 }
-                                if (priv->incoming.msg_type == REPLY)
+                                if (in->msg_type == REPLY)
                                         (*pollin)->is_reply = 1;
 
-                                priv->incoming.request_info = NULL;
+                                in->request_info = NULL;
                         }
-                        priv->incoming.record_state = SP_STATE_COMPLETE;
+                        in->record_state = SP_STATE_COMPLETE;
                         break;
 
                 case SP_STATE_COMPLETE:
@@ -1627,8 +2069,8 @@ __socket_proto_state_machine (rpc_transport_t *this,
                 }
         }
 
-        if (priv->incoming.record_state == SP_STATE_COMPLETE) {
-                priv->incoming.record_state = SP_STATE_NADA;
+        if (in->record_state == SP_STATE_COMPLETE) {
+                in->record_state = SP_STATE_NADA;
                 __socket_reset_priv (priv);
         }
 
@@ -1640,7 +2082,7 @@ out:
 }
 
 
-int
+static int
 socket_proto_state_machine (rpc_transport_t *this,
                             rpc_transport_pollin_t **pollin)
 {
@@ -1663,18 +2105,22 @@ out:
 }
 
 
-int
+static int
 socket_event_poll_in (rpc_transport_t *this)
 {
         int                     ret    = -1;
         rpc_transport_pollin_t *pollin = NULL;
+        socket_private_t       *priv = this->private;
 
         ret = socket_proto_state_machine (this, &pollin);
 
         if (pollin != NULL) {
+                priv->ot_state = OT_CALLBACK;
                 ret = rpc_transport_notify (this, RPC_TRANSPORT_MSG_RECEIVED,
                                             pollin);
-
+                if (priv->ot_state == OT_CALLBACK) {
+                        priv->ot_state = OT_RUNNING;
+                }
                 rpc_transport_pollin_destroy (pollin);
         }
 
@@ -1682,7 +2128,7 @@ socket_event_poll_in (rpc_transport_t *this)
 }
 
 
-int
+static int
 socket_connect_finish (rpc_transport_t *this)
 {
         int                   ret        = -1;
@@ -1697,9 +2143,11 @@ socket_connect_finish (rpc_transport_t *this)
 
         pthread_mutex_lock (&priv->lock);
         {
-                if (priv->connected)
+                if (priv->connected != 0)
                         goto unlock;
 
+                get_transport_identifiers (this);
+
                 ret = __socket_connect_finish (priv->sock);
 
                 if (ret == -1 && errno == EINPROGRESS)
@@ -1714,8 +2162,6 @@ socket_connect_finish (rpc_transport_t *this)
                                 priv->connect_finish_log = 1;
                         }
                         __socket_disconnect (this);
-                        notify_rpc = 1;
-                        event = RPC_TRANSPORT_DISCONNECT;
                         goto unlock;
                 }
 
@@ -1740,7 +2186,6 @@ socket_connect_finish (rpc_transport_t *this)
                         priv->connected = 1;
                         priv->connect_finish_log = 0;
                         event = RPC_TRANSPORT_CONNECT;
-                        get_transport_identifiers (this);
                 }
         }
 unlock:
@@ -1755,13 +2200,13 @@ out:
 
 
 /* reads rpc_requests during pollin */
-int
+static int
 socket_event_handler (int fd, int idx, void *data,
                       int poll_in, int poll_out, int poll_err)
 {
-        rpc_transport_t      *this = NULL;
+        rpc_transport_t  *this = NULL;
         socket_private_t *priv = NULL;
-        int               ret = 0;
+	int               ret = -1;
 
         this = data;
         GF_VALIDATE_OR_GOTO ("socket", this, out);
@@ -1771,16 +2216,13 @@ socket_event_handler (int fd, int idx, void *data,
         THIS = this->xl;
         priv = this->private;
 
-
         pthread_mutex_lock (&priv->lock);
         {
                 priv->idx = idx;
         }
         pthread_mutex_unlock (&priv->lock);
 
-        if (!priv->connected) {
-                ret = socket_connect_finish (this);
-        }
+	ret = (priv->connected == 1) ? 0 : socket_connect_finish(this);
 
         if (!ret && poll_out) {
                 ret = socket_event_poll_out (this);
@@ -1796,14 +2238,198 @@ socket_event_handler (int fd, int idx, void *data,
                         "disconnecting now");
                 socket_event_poll_err (this);
                 rpc_transport_unref (this);
-        }
+	}
 
 out:
-        return 0;
+	return ret;
 }
 
 
-int
+static void *
+socket_poller (void *ctx)
+{
+        rpc_transport_t  *this = ctx;
+        socket_private_t *priv = this->private;
+	struct pollfd     pfd[2] = {{0,},};
+	gf_boolean_t      to_write = _gf_false;
+	int               ret = 0;
+        uint32_t          gen = 0;
+
+        priv->ot_state = OT_RUNNING;
+
+        if (priv->use_ssl) {
+                if (ssl_setup_connection(this,priv->connected) < 0) {
+                        gf_log (this->name,GF_LOG_ERROR, "%s setup failed",
+                                priv->connected ? "server" : "client");
+                        goto err;
+                }
+        }
+
+        if (!priv->bio) {
+                ret = __socket_nonblock (priv->sock);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "NBIO on %d failed (%s)",
+                                priv->sock, strerror (errno));
+                        goto err;
+                }
+        }
+
+        if (priv->connected == 0) {
+		THIS = this->xl;
+                ret = socket_connect_finish (this);
+                if (ret != 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "asynchronous socket_connect_finish failed");
+                }
+        }
+
+        ret = rpc_transport_notify (this->listener,
+                                    RPC_TRANSPORT_ACCEPT, this);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "asynchronous rpc_transport_notify failed");
+        }
+
+        gen = priv->ot_gen;
+	for (;;) {
+		pthread_mutex_lock(&priv->lock);
+		to_write = !list_empty(&priv->ioq);
+		pthread_mutex_unlock(&priv->lock);
+		pfd[0].fd = priv->pipe[0];
+		pfd[0].events = POLL_MASK_ERROR;
+		pfd[0].revents = 0;
+		pfd[1].fd = priv->sock;
+		pfd[1].events = POLL_MASK_INPUT | POLL_MASK_ERROR;
+		pfd[1].revents = 0;
+		if (to_write) {
+			pfd[1].events |= POLL_MASK_OUTPUT;
+		}
+		else {
+			pfd[0].events |= POLL_MASK_INPUT;
+		}
+		if (poll(pfd,2,-1) < 0) {
+			gf_log(this->name,GF_LOG_ERROR,"poll failed");
+			break;
+		}
+		if (pfd[0].revents & POLL_MASK_ERROR) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "poll error on pipe");
+			break;
+		}
+		/* Only glusterd actually seems to need this. */
+		THIS = this->xl;
+		if (pfd[1].revents & POLL_MASK_INPUT) {
+			ret = socket_event_poll_in(this);
+			if (ret >= 0) {
+				/* Suppress errors while making progress. */
+				pfd[1].revents &= ~POLL_MASK_ERROR;
+			}
+			else if (errno == ENOTCONN) {
+				ret = 0;
+			}
+                        if (priv->ot_state == OT_PLEASE_DIE) {
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "OT_IDLE on %p (input request)",
+                                        this);
+                                priv->ot_state = OT_IDLE;
+                                break;
+                        }
+		}
+		else if (pfd[1].revents & POLL_MASK_OUTPUT) {
+			ret = socket_event_poll_out(this);
+			if (ret >= 0) {
+				/* Suppress errors while making progress. */
+				pfd[1].revents &= ~POLL_MASK_ERROR;
+			}
+			else if (errno == ENOTCONN) {
+				ret = 0;
+			}
+                        if (priv->ot_state == OT_PLEASE_DIE) {
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "OT_IDLE on %p (output request)",
+                                        this);
+                                priv->ot_state = OT_IDLE;
+                                break;
+                        }
+		}
+		else {
+			/*
+			 * This usually means that we left poll() because
+			 * somebody pushed a byte onto our pipe.  That wakeup
+			 * is why the pipe is there, but once awake we can do
+			 * all the checking we need on the next iteration.
+			 */
+			ret = 0;
+		}
+		if (pfd[1].revents & POLL_MASK_ERROR) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "poll error on socket");
+			break;
+		}
+		if (ret < 0) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "error in polling loop");
+			break;
+		}
+                if (priv->ot_gen != gen) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "generation mismatch, my %u != %u",
+                                gen, priv->ot_gen);
+                        return NULL;
+                }
+	}
+
+err:
+	/* All (and only) I/O errors should come here. */
+        pthread_mutex_lock(&priv->lock);
+        if (priv->ssl_ssl) {
+                /*
+                 * We're always responsible for this part, but only actually
+                 * have to do it if we got far enough for ssl_ssl to be valid
+                 * (i.e. errors in ssl_setup_connection don't count).
+                 */
+                ssl_teardown_connection(priv);
+        }
+        __socket_shutdown(this);
+        close(priv->sock);
+        priv->sock = -1;
+        priv->ot_state = OT_IDLE;
+        pthread_mutex_unlock(&priv->lock);
+        rpc_transport_notify (this->listener, RPC_TRANSPORT_DISCONNECT,
+                              this);
+        rpc_transport_unref (this);
+	return NULL;
+}
+
+
+static void
+socket_spawn (rpc_transport_t *this)
+{
+        socket_private_t        *priv   = this->private;
+
+        switch (priv->ot_state) {
+        case OT_IDLE:
+        case OT_PLEASE_DIE:
+                break;
+        default:
+                gf_log (this->name, GF_LOG_WARNING,
+                        "refusing to start redundant poller");
+                return;
+        }
+
+        priv->ot_gen += 7;
+        priv->ot_state = OT_SPAWNING;
+        gf_log (this->name, GF_LOG_TRACE,
+                "spawning %p with gen %u", this, priv->ot_gen);
+
+        if (gf_thread_create(&priv->thread,NULL,socket_poller,this) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not create poll thread");
+        }
+}
+
+static int
 socket_server_event_handler (int fd, int idx, void *data,
                              int poll_in, int poll_out, int poll_err)
 {
@@ -1841,20 +2467,7 @@ socket_server_event_handler (int fd, int idx, void *data,
                                 goto unlock;
                         }
 
-                        if (!priv->bio) {
-                                ret = __socket_nonblock (new_sock);
-
-                                if (ret == -1) {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "NBIO on %d failed (%s)",
-                                                new_sock, strerror (errno));
-
-                                        close (new_sock);
-                                        goto unlock;
-                                }
-                        }
-
-                        if (priv->nodelay) {
+                        if (priv->nodelay && (new_sockaddr.ss_family != AF_UNIX)) {
                                 ret = __socket_nodelay (new_sock);
                                 if (ret == -1) {
                                         gf_log (this->name, GF_LOG_WARNING,
@@ -1864,8 +2477,10 @@ socket_server_event_handler (int fd, int idx, void *data,
                                 }
                         }
 
-                        if (priv->keepalive) {
+                        if (priv->keepalive &&
+                            new_sockaddr.ss_family != AF_UNIX) {
                                 ret = __socket_keepalive (new_sock,
+                                                          new_sockaddr.ss_family,
                                                           priv->keepaliveintvl,
                                                           priv->keepaliveidle);
                                 if (ret == -1)
@@ -1879,6 +2494,15 @@ socket_server_event_handler (int fd, int idx, void *data,
                         if (!new_trans)
                                 goto unlock;
 
+                        ret = pthread_mutex_init(&new_trans->lock, NULL);
+                        if (ret == -1) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "pthread_mutex_init() failed: %s",
+                                        strerror (errno));
+                                close (new_sock);
+                                goto unlock;
+                        }
+
                         new_trans->name = gf_strdup (this->name);
 
                         memcpy (&new_trans->peerinfo.sockaddr, &new_sockaddr,
@@ -1900,7 +2524,11 @@ socket_server_event_handler (int fd, int idx, void *data,
                         }
 
                         get_transport_identifiers (new_trans);
-                        socket_init (new_trans);
+                        ret = socket_init(new_trans);
+                        if (ret != 0) {
+                                close(new_sock);
+                                goto unlock;
+                        }
                         new_trans->ops = this->ops;
                         new_trans->init = this->init;
                         new_trans->fini = this->fini;
@@ -1911,20 +2539,62 @@ socket_server_event_handler (int fd, int idx, void *data,
                         new_trans->listener = this;
                         new_priv = new_trans->private;
 
+			new_priv->use_ssl = priv->use_ssl;
+			new_priv->sock = new_sock;
+			new_priv->own_thread = priv->own_thread;
+
+                        new_priv->ssl_ctx = priv->ssl_ctx;
+			if (priv->use_ssl && !priv->own_thread) {
+				if (ssl_setup_connection(new_trans,1) < 0) {
+					gf_log(this->name,GF_LOG_ERROR,
+					       "server setup failed");
+					close(new_sock);
+					goto unlock;
+				}
+			}
+
+                        if (!priv->bio && !priv->own_thread) {
+                                ret = __socket_nonblock (new_sock);
+
+                                if (ret == -1) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "NBIO on %d failed (%s)",
+                                                new_sock, strerror (errno));
+
+                                        close (new_sock);
+                                        goto unlock;
+                                }
+                        }
+
                         pthread_mutex_lock (&new_priv->lock);
                         {
-                                new_priv->sock = new_sock;
+                                /*
+                                 * In the own_thread case, this is used to
+                                 * indicate that we're initializing a server
+                                 * connection.
+                                 */
                                 new_priv->connected = 1;
+                                new_priv->is_server = _gf_true;
                                 rpc_transport_ref (new_trans);
 
-                                new_priv->idx =
-                                        event_register (ctx->event_pool,
-                                                        new_sock,
-                                                        socket_event_handler,
-                                                        new_trans, 1, 0);
+				if (new_priv->own_thread) {
+					if (pipe(new_priv->pipe) < 0) {
+						gf_log(this->name,GF_LOG_ERROR,
+						       "could not create pipe");
+					}
+                                        socket_spawn(new_trans);
+				}
+				else {
+					new_priv->idx =
+						event_register (ctx->event_pool,
+								new_sock,
+								socket_event_handler,
+								new_trans,
+								1, 0);
+					if (new_priv->idx == -1)
+						ret = -1;
+				}
 
-                                if (new_priv->idx == -1)
-                                        ret = -1;
                         }
                         pthread_mutex_unlock (&new_priv->lock);
                         if (ret == -1) {
@@ -1933,8 +2603,10 @@ socket_server_event_handler (int fd, int idx, void *data,
                                 goto unlock;
                         }
 
-                        ret = rpc_transport_notify (this, RPC_TRANSPORT_ACCEPT,
-                                                    new_trans);
+                        if (!priv->own_thread) {
+                                ret = rpc_transport_notify (this,
+                                        RPC_TRANSPORT_ACCEPT, new_trans);
+                        }
                 }
         }
 unlock:
@@ -1945,7 +2617,7 @@ out:
 }
 
 
-int
+static int
 socket_disconnect (rpc_transport_t *this)
 {
         socket_private_t *priv = NULL;
@@ -1967,7 +2639,7 @@ out:
 }
 
 
-int
+static int
 socket_connect (rpc_transport_t *this, int port)
 {
         int                      ret = -1;
@@ -1976,7 +2648,9 @@ socket_connect (rpc_transport_t *this, int port)
         socklen_t                sockaddr_len = 0;
         glusterfs_ctx_t         *ctx = NULL;
         sa_family_t              sa_family = {0, };
+        char                    *local_addr   = NULL;
         union gf_sock_union      sock_union;
+        struct sockaddr_in      *addr         = NULL;
 
         GF_VALIDATE_OR_GOTO ("socket", this, err);
         GF_VALIDATE_OR_GOTO ("socket", this->private, err);
@@ -2004,6 +2678,10 @@ socket_connect (rpc_transport_t *this, int port)
                 goto err;
         }
 
+        gf_log (this->name, GF_LOG_TRACE,
+                "connecting %p, state=%u gen=%u sock=%d", this,
+                priv->ot_state, priv->ot_gen, priv->sock);
+
         ret = socket_client_get_remote_sockaddr (this, &sock_union.sa,
                                                  &sockaddr_len, &sa_family);
         if (ret == -1) {
@@ -2014,6 +2692,20 @@ socket_connect (rpc_transport_t *this, int port)
         if (port > 0) {
                 sock_union.sin.sin_port = htons (port);
         }
+        if (ntohs(sock_union.sin.sin_port) == GF_DEFAULT_SOCKET_LISTEN_PORT) {
+                if (priv->use_ssl) {
+                        gf_log(this->name,GF_LOG_DEBUG,
+                               "disabling SSL for portmapper connection");
+                        priv->use_ssl = _gf_false;
+                }
+        }
+        else {
+                if (priv->ssl_enabled && !priv->use_ssl) {
+                        gf_log(this->name,GF_LOG_DEBUG,
+                               "re-enabling SSL for I/O connection");
+                        priv->use_ssl = _gf_true;
+                }
+        }
         pthread_mutex_lock (&priv->lock);
         {
                 if (priv->sock != -1) {
@@ -2037,49 +2729,41 @@ socket_connect (rpc_transport_t *this, int port)
                 /* Cant help if setting socket options fails. We can continue
                  * working nonetheless.
                  */
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "setting receive window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
-
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "setting send window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
-
+                if (priv->windowsize != 0) {
+                        if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF,
+                                        &priv->windowsize,
+                                        sizeof (priv->windowsize)) < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "setting receive window "
+                                        "size failed: %d: %d: %s",
+                                        priv->sock, priv->windowsize,
+                                        strerror (errno));
+                        }
 
-                if (priv->nodelay) {
-                        ret = __socket_nodelay (priv->sock);
-                        if (ret == -1) {
+                        if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF,
+                                        &priv->windowsize,
+                                        sizeof (priv->windowsize)) < 0) {
                                 gf_log (this->name, GF_LOG_ERROR,
-                                        "setsockopt() failed for NODELAY (%s)",
+                                        "setting send window size "
+                                        "failed: %d: %d: %s",
+                                        priv->sock, priv->windowsize,
                                         strerror (errno));
                         }
                 }
 
-                if (!priv->bio) {
-                        ret = __socket_nonblock (priv->sock);
+                if (priv->nodelay && (sa_family != AF_UNIX)) {
+                        ret = __socket_nodelay (priv->sock);
 
                         if (ret == -1) {
                                 gf_log (this->name, GF_LOG_ERROR,
-                                        "NBIO on %d failed (%s)",
+                                        "NODELAY on %d failed (%s)",
                                         priv->sock, strerror (errno));
-                                close (priv->sock);
-                                priv->sock = -1;
-                                goto unlock;
                         }
                 }
 
-                if (priv->keepalive) {
+                if (priv->keepalive && sa_family != AF_UNIX) {
                         ret = __socket_keepalive (priv->sock,
+                                                  sa_family,
                                                   priv->keepaliveintvl,
                                                   priv->keepaliveidle);
                         if (ret == -1)
@@ -2091,6 +2775,15 @@ socket_connect (rpc_transport_t *this, int port)
                 SA (&this->myinfo.sockaddr)->sa_family =
                         SA (&this->peerinfo.sockaddr)->sa_family;
 
+                /* If a source addr is explicitly specified, use it */
+                ret = dict_get_str (this->options,
+                                    "transport.socket.source-addr",
+                                    &local_addr);
+                if (!ret && SA (&this->myinfo.sockaddr)->sa_family == AF_INET) {
+                        addr = (struct sockaddr_in *)(&this->myinfo.sockaddr);
+                        ret = inet_pton (AF_INET, local_addr, &(addr->sin_addr.s_addr));
+                }
+
                 ret = client_bind (this, SA (&this->myinfo.sockaddr),
                                    &this->myinfo.sockaddr_len, priv->sock);
                 if (ret == -1) {
@@ -2101,29 +2794,86 @@ socket_connect (rpc_transport_t *this, int port)
                         goto unlock;
                 }
 
+                if (!priv->use_ssl && !priv->bio && !priv->own_thread) {
+                        ret = __socket_nonblock (priv->sock);
+                        if (ret == -1) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "NBIO on %d failed (%s)",
+                                        priv->sock, strerror (errno));
+                                close (priv->sock);
+                                priv->sock = -1;
+                                goto unlock;
+                        }
+                }
+
                 ret = connect (priv->sock, SA (&this->peerinfo.sockaddr),
                                this->peerinfo.sockaddr_len);
 
                 if (ret == -1 && ((errno != EINPROGRESS) && (errno != ENOENT))) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "connection attempt failed (%s)",
-                                strerror (errno));
+                        /* For unix path based sockets, the socket path is
+                         * cryptic (md5sum of path) and may not be useful for
+                         * the user in debugging so log it in DEBUG
+                         */
+                        gf_log (this->name, ((sa_family == AF_UNIX) ?
+                                GF_LOG_DEBUG : GF_LOG_ERROR),
+                                "connection attempt on %s failed, (%s)",
+                                this->peerinfo.identifier, strerror (errno));
                         close (priv->sock);
                         priv->sock = -1;
                         goto unlock;
                 }
 
-                priv->connected = 0;
+		if (priv->use_ssl && !priv->own_thread) {
+			ret = ssl_setup_connection(this,0);
+			if (ret < 0) {
+				gf_log(this->name,GF_LOG_ERROR,
+					"client setup failed");
+				close(priv->sock);
+				priv->sock = -1;
+				goto unlock;
+			}
+		}
 
-                rpc_transport_ref (this);
+                if (!priv->bio && !priv->own_thread) {
+                        ret = __socket_nonblock (priv->sock);
 
-                priv->idx = event_register (ctx->event_pool, priv->sock,
-                                            socket_event_handler, this, 1, 1);
-                if (priv->idx == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to register the event");
-                        ret = -1;
+                        if (ret == -1) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "NBIO on %d failed (%s)",
+                                        priv->sock, strerror (errno));
+                                close (priv->sock);
+                                priv->sock = -1;
+                                goto unlock;
+                        }
                 }
+
+                /*
+                 * In the own_thread case, this is used to indicate that we're
+                 * initializing a client connection.
+                 */
+                priv->connected = 0;
+                priv->is_server = _gf_false;
+                rpc_transport_ref (this);
+
+		if (priv->own_thread) {
+			if (pipe(priv->pipe) < 0) {
+				gf_log(this->name,GF_LOG_ERROR,
+				       "could not create pipe");
+			}
+
+                        this->listener = this;
+                        socket_spawn(this);
+		}
+		else {
+			priv->idx = event_register (ctx->event_pool, priv->sock,
+						    socket_event_handler,
+						    this, 1, 1);
+			if (priv->idx == -1) {
+				gf_log ("", GF_LOG_WARNING,
+					"failed to register the event");
+				ret = -1;
+			}
+		}
         }
 unlock:
         pthread_mutex_unlock (&priv->lock);
@@ -2133,7 +2883,7 @@ err:
 }
 
 
-int
+static int
 socket_listen (rpc_transport_t *this)
 {
         socket_private_t *       priv = NULL;
@@ -2193,25 +2943,29 @@ socket_listen (rpc_transport_t *this)
                 /* Cant help if setting socket options fails. We can continue
                  * working nonetheless.
                  */
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "setting receive window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
+                if (priv->windowsize != 0) {
+                        if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF,
+                                        &priv->windowsize,
+                                        sizeof (priv->windowsize)) < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "setting receive window size "
+                                        "failed: %d: %d: %s", priv->sock,
+                                        priv->windowsize,
+                                        strerror (errno));
+                        }
 
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "setting send window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
+                        if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF,
+                                        &priv->windowsize,
+                                        sizeof (priv->windowsize)) < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "setting send window size failed:"
+                                        " %d: %d: %s", priv->sock,
+                                        priv->windowsize,
+                                        strerror (errno));
+                        }
                 }
 
-                if (priv->nodelay) {
+                if (priv->nodelay && (sa_family != AF_UNIX)) {
                         ret = __socket_nodelay (priv->sock);
                         if (ret == -1) {
                                 gf_log (this->name, GF_LOG_ERROR,
@@ -2280,7 +3034,7 @@ out:
 }
 
 
-int32_t
+static int32_t
 socket_submit_request (rpc_transport_t *this, rpc_transport_req_t *req)
 {
         socket_private_t *priv = NULL;
@@ -2289,6 +3043,7 @@ socket_submit_request (rpc_transport_t *this, rpc_transport_req_t *req)
         char              need_append = 1;
         struct ioq       *entry = NULL;
         glusterfs_ctx_t  *ctx = NULL;
+	char              a_byte = 'j';
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
@@ -2314,21 +3069,31 @@ socket_submit_request (rpc_transport_t *this, rpc_transport_req_t *req)
                         goto unlock;
 
                 if (list_empty (&priv->ioq)) {
-                        ret = __socket_ioq_churn_entry (this, entry);
+                        ret = __socket_ioq_churn_entry (this, entry, 1);
 
-                        if (ret == 0)
+                        if (ret == 0) {
                                 need_append = 0;
-
-                        if (ret > 0)
+			}
+                        if (ret > 0) {
                                 need_poll_out = 1;
+			}
                 }
 
                 if (need_append) {
                         list_add_tail (&entry->list, &priv->ioq);
+			if (priv->own_thread) {
+				/*
+				 * Make sure the polling thread wakes up, by
+				 * writing a byte to represent this entry.
+				 */
+				if (write(priv->pipe[1],&a_byte,1) < 1) {
+					gf_log(this->name,GF_LOG_WARNING,
+					       "write error on pipe");
+				}
+			}
                         ret = 0;
                 }
-
-                if (need_poll_out) {
+                if (!priv->own_thread && need_poll_out) {
                         /* first entry to wait. continue writing on POLLOUT */
                         priv->idx = event_select_on (ctx->event_pool,
                                                      priv->sock,
@@ -2343,7 +3108,7 @@ out:
 }
 
 
-int32_t
+static int32_t
 socket_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply)
 {
         socket_private_t *priv = NULL;
@@ -2352,6 +3117,7 @@ socket_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply)
         char              need_append = 1;
         struct ioq       *entry = NULL;
         glusterfs_ctx_t  *ctx = NULL;
+	char              a_byte = 'd';
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
@@ -2370,33 +3136,44 @@ socket_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply)
                         }
                         goto unlock;
                 }
+
                 priv->submit_log = 0;
                 entry = __socket_ioq_new (this, &reply->msg);
                 if (!entry)
                         goto unlock;
+
                 if (list_empty (&priv->ioq)) {
-                        ret = __socket_ioq_churn_entry (this, entry);
+                        ret = __socket_ioq_churn_entry (this, entry, 1);
 
-                        if (ret == 0)
+                        if (ret == 0) {
                                 need_append = 0;
-
-                        if (ret > 0)
+			}
+                        if (ret > 0) {
                                 need_poll_out = 1;
+			}
                 }
 
                 if (need_append) {
                         list_add_tail (&entry->list, &priv->ioq);
+			if (priv->own_thread) {
+				/*
+				 * Make sure the polling thread wakes up, by
+				 * writing a byte to represent this entry.
+				 */
+				if (write(priv->pipe[1],&a_byte,1) < 1) {
+					gf_log(this->name,GF_LOG_WARNING,
+					       "write error on pipe");
+				}
+			}
                         ret = 0;
                 }
-
-                if (need_poll_out) {
+                if (!priv->own_thread && need_poll_out) {
                         /* first entry to wait. continue writing on POLLOUT */
                         priv->idx = event_select_on (ctx->event_pool,
                                                      priv->sock,
                                                      priv->idx, -1, 1);
                 }
         }
-
 unlock:
         pthread_mutex_unlock (&priv->lock);
 
@@ -2405,7 +3182,7 @@ out:
 }
 
 
-int32_t
+static int32_t
 socket_getpeername (rpc_transport_t *this, char *hostname, int hostlen)
 {
         int32_t ret = -1;
@@ -2424,7 +3201,7 @@ out:
 }
 
 
-int32_t
+static int32_t
 socket_getpeeraddr (rpc_transport_t *this, char *peeraddr, int addrlen,
                     struct sockaddr_storage *sa, socklen_t salen)
 {
@@ -2445,7 +3222,7 @@ out:
 }
 
 
-int32_t
+static int32_t
 socket_getmyname (rpc_transport_t *this, char *hostname, int hostlen)
 {
         int32_t ret = -1;
@@ -2464,7 +3241,7 @@ out:
 }
 
 
-int32_t
+static int32_t
 socket_getmyaddr (rpc_transport_t *this, char *myaddr, int addrlen,
                   struct sockaddr_storage *sa, socklen_t salen)
 {
@@ -2484,6 +3261,25 @@ out:
 }
 
 
+static int
+socket_throttle (rpc_transport_t *this, gf_boolean_t onoff)
+{
+        socket_private_t *priv = NULL;
+
+        priv = this->private;
+
+        /* The way we implement throttling is by taking off
+           POLLIN event from the polled flags. This way we
+           never get called with the POLLIN event and therefore
+           will never read() any more data until throttling
+           is turned off.
+        */
+        priv->idx = event_select_on (this->ctx->event_pool, priv->sock,
+                                     priv->idx, (int) !onoff, -1);
+        return 0;
+}
+
+
 struct rpc_transport_ops tops = {
         .listen             = socket_listen,
         .connect            = socket_connect,
@@ -2494,15 +3290,17 @@ struct rpc_transport_ops tops = {
         .get_peeraddr       = socket_getpeeraddr,
         .get_myname         = socket_getmyname,
         .get_myaddr         = socket_getmyaddr,
+	.throttle           = socket_throttle,
 };
 
 int
 reconfigure (rpc_transport_t *this, dict_t *options)
 {
-        socket_private_t *priv = NULL;
-        gf_boolean_t      tmp_bool = _gf_false;
-        char             *optstr = NULL;
-        int               ret = 0;
+        socket_private_t *priv          = NULL;
+        gf_boolean_t      tmp_bool      = _gf_false;
+        char             *optstr        = NULL;
+        int               ret           = 0;
+        uint64_t          windowsize    = 0;
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
@@ -2530,13 +3328,26 @@ reconfigure (rpc_transport_t *this, dict_t *options)
         }
         else
                 priv->keepalive = 1;
+
+        optstr = NULL;
+        if (dict_get_str (this->options, "tcp-window-size",
+                          &optstr) == 0) {
+                if (gf_string2bytesize (optstr, &windowsize) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid number format: %s", optstr);
+                        goto out;
+                }
+        }
+
+        priv->windowsize = (int)windowsize;
+
         ret = 0;
 out:
         return ret;
 
 }
 
-int
+static int
 socket_init (rpc_transport_t *this)
 {
         socket_private_t *priv = NULL;
@@ -2545,6 +3356,7 @@ socket_init (rpc_transport_t *this)
         char             *optstr = NULL;
         uint32_t          keepalive = 0;
         uint32_t          backlog = 0;
+	int               session_id = 0;
 
         if (this->private) {
                 gf_log_callingfn (this->name, GF_LOG_ERROR,
@@ -2556,6 +3368,7 @@ socket_init (rpc_transport_t *this)
         if (!priv) {
                 return -1;
         }
+        memset(priv,0,sizeof(*priv));
 
         pthread_mutex_init (&priv->lock, NULL);
 
@@ -2609,9 +3422,8 @@ socket_init (rpc_transport_t *this)
                 }
         }
 
-
         optstr = NULL;
-        if (dict_get_str (this->options, "transport.window-size",
+        if (dict_get_str (this->options, "tcp-window-size",
                           &optstr) == 0) {
                 if (gf_string2bytesize (optstr, &windowsize) != 0) {
                         gf_log (this->name, GF_LOG_ERROR,
@@ -2620,8 +3432,9 @@ socket_init (rpc_transport_t *this)
                 }
         }
 
-        optstr = NULL;
+        priv->windowsize = (int)windowsize;
 
+        optstr = NULL;
         /* Enable Keep-alive by default. */
         priv->keepalive = 1;
         priv->keepaliveintvl = 2;
@@ -2674,10 +3487,133 @@ socket_init (rpc_transport_t *this)
         }
 
         priv->windowsize = (int)windowsize;
+
+        priv->ssl_enabled = _gf_false;
+	if (dict_get_str(this->options,SSL_ENABLED_OPT,&optstr) == 0) {
+                if (gf_string2boolean (optstr, &priv->ssl_enabled) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+				"invalid value given for ssl-enabled boolean");
+		}
+	}
+
+        priv->ssl_own_cert = DEFAULT_CERT_PATH;
+	if (dict_get_str(this->options,SSL_OWN_CERT_OPT,&optstr) == 0) {
+                if (!priv->ssl_enabled) {
+                        gf_log(this->name,GF_LOG_WARNING,
+                               "%s specified without %s (ignored)",
+                               SSL_OWN_CERT_OPT, SSL_ENABLED_OPT);
+                }
+                priv->ssl_own_cert = optstr;
+	}
+        priv->ssl_own_cert = gf_strdup(priv->ssl_own_cert);
+
+        priv->ssl_private_key = DEFAULT_KEY_PATH;
+	if (dict_get_str(this->options,SSL_PRIVATE_KEY_OPT,&optstr) == 0) {
+                if (!priv->ssl_enabled) {
+                        gf_log(this->name,GF_LOG_WARNING,
+                               "%s specified without %s (ignored)",
+                               SSL_PRIVATE_KEY_OPT, SSL_ENABLED_OPT);
+                }
+                priv->ssl_private_key = optstr;
+	}
+        priv->ssl_private_key = gf_strdup(priv->ssl_private_key);
+
+        priv->ssl_ca_list = DEFAULT_CA_PATH;
+	if (dict_get_str(this->options,SSL_CA_LIST_OPT,&optstr) == 0) {
+                if (!priv->ssl_enabled) {
+                        gf_log(this->name,GF_LOG_WARNING,
+                               "%s specified without %s (ignored)",
+                               SSL_CA_LIST_OPT, SSL_ENABLED_OPT);
+                }
+                priv->ssl_ca_list = optstr;
+	}
+        priv->ssl_ca_list = gf_strdup(priv->ssl_ca_list);
+
+        gf_log(this->name,GF_LOG_INFO,"SSL support is %s",
+               priv->ssl_enabled ? "ENABLED" : "NOT enabled");
+        /*
+         * This might get overridden temporarily in socket_connect (q.v.)
+         * if we're using the glusterd portmapper.
+         */
+        priv->use_ssl = priv->ssl_enabled;
+
+	priv->own_thread = priv->use_ssl;
+	if (dict_get_str(this->options,OWN_THREAD_OPT,&optstr) == 0) {
+                if (gf_string2boolean (optstr, &priv->own_thread) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+				"invalid value given for own-thread boolean");
+		}
+	}
+	gf_log(this->name,GF_LOG_INFO,"using %s polling thread",
+	       priv->own_thread ? "private" : "system");
+
+	if (priv->use_ssl) {
+		SSL_library_init();
+		SSL_load_error_strings();
+		priv->ssl_meth = (SSL_METHOD *)TLSv1_method();
+		priv->ssl_ctx = SSL_CTX_new(priv->ssl_meth);
+
+                if (SSL_CTX_set_cipher_list(priv->ssl_ctx,
+                                            "HIGH:-SSLv2") == 0) {
+                        gf_log(this->name,GF_LOG_ERROR,
+                               "failed to find any valid ciphers");
+                        goto err;
+                }
+
+		if (!SSL_CTX_use_certificate_chain_file(priv->ssl_ctx,
+							priv->ssl_own_cert)) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "could not load our cert");
+			goto err;
+		}
+
+		if (!SSL_CTX_use_PrivateKey_file(priv->ssl_ctx,
+						 priv->ssl_private_key,
+						 SSL_FILETYPE_PEM)) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "could not load private key");
+			goto err;
+		}
+
+		if (!SSL_CTX_load_verify_locations(priv->ssl_ctx,
+						   priv->ssl_ca_list,0)) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "could not load CA list");
+			goto err;
+		}
+
+#if (OPENSSL_VERSION_NUMBER < 0x00905100L)
+		SSL_CTX_set_verify_depth(ctx,1);
+#endif
+
+		priv->ssl_session_id = ++session_id;
+		SSL_CTX_set_session_id_context(priv->ssl_ctx,
+					       (void *)&priv->ssl_session_id,
+					       sizeof(priv->ssl_session_id));
+
+		SSL_CTX_set_verify(priv->ssl_ctx,SSL_VERIFY_PEER,0);
+	}
+
+        if (priv->own_thread) {
+                priv->ot_state = OT_IDLE;
+        }
+
 out:
         this->private = priv;
-
         return 0;
+
+err:
+        if (priv->ssl_own_cert) {
+                GF_FREE(priv->ssl_own_cert);
+        }
+        if (priv->ssl_private_key) {
+                GF_FREE(priv->ssl_private_key);
+        }
+        if (priv->ssl_ca_list) {
+                GF_FREE(priv->ssl_ca_list);
+        }
+        GF_FREE(priv);
+        return -1;
 }
 
 
@@ -2703,6 +3639,15 @@ fini (rpc_transport_t *this)
                         "transport %p destroyed", this);
 
                 pthread_mutex_destroy (&priv->lock);
+		if (priv->ssl_private_key) {
+			GF_FREE(priv->ssl_private_key);
+		}
+		if (priv->ssl_own_cert) {
+			GF_FREE(priv->ssl_own_cert);
+		}
+		if (priv->ssl_ca_list) {
+			GF_FREE(priv->ssl_ca_list);
+		}
                 GF_FREE (priv);
         }
 
@@ -2747,18 +3692,17 @@ struct volume_options options[] = {
         },
         { .key   = { "transport.address-family",
                      "address-family" },
-          .value = {"inet", "inet6", "inet/inet6", "inet6/inet",
-                    "unix", "inet-sdp" },
+          .value = {"inet", "inet6", "unix", "inet-sdp" },
           .type  = GF_OPTION_TYPE_STR
         },
 
         { .key   = {"non-blocking-io"},
           .type  = GF_OPTION_TYPE_BOOL
         },
-        { .key   = {"transport.window-size"},
+        { .key   = {"tcp-window-size"},
           .type  = GF_OPTION_TYPE_SIZET,
           .min   = GF_MIN_SOCKET_WINDOW_SIZE,
-          .max   = GF_MAX_SOCKET_WINDOW_SIZE,
+          .max   = GF_MAX_SOCKET_WINDOW_SIZE
         },
         { .key   = {"transport.socket.nodelay"},
           .type  = GF_OPTION_TYPE_BOOL
@@ -2781,5 +3725,20 @@ struct volume_options options[] = {
         { .key   = {"transport.socket.read-fail-log"},
           .type  = GF_OPTION_TYPE_BOOL
         },
+        { .key   = {SSL_ENABLED_OPT},
+          .type  = GF_OPTION_TYPE_BOOL
+        },
+	{ .key   = {SSL_OWN_CERT_OPT},
+	  .type  = GF_OPTION_TYPE_STR
+	},
+	{ .key   = {SSL_PRIVATE_KEY_OPT},
+	  .type  = GF_OPTION_TYPE_STR
+	},
+	{ .key   = {SSL_CA_LIST_OPT},
+	  .type  = GF_OPTION_TYPE_STR
+	},
+	{ .key   = {OWN_THREAD_OPT},
+	  .type  = GF_OPTION_TYPE_BOOL
+	},
         { .key = {NULL} }
 };
diff --git a/rpc/rpc-transport/socket/src/socket.h b/rpc/rpc-transport/socket/src/socket.h
index 7f0fb6bc3..e0b412fcc 100644
--- a/rpc/rpc-transport/socket/src/socket.h
+++ b/rpc/rpc-transport/socket/src/socket.h
@@ -1,25 +1,18 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _SOCKET_H
 #define _SOCKET_H
 
+#include <openssl/ssl.h>
+#include <openssl/err.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -41,19 +34,17 @@
 
 #define RPC_MAX_FRAGMENT_SIZE 0x7fffffff
 
-/* This is the size set through setsockopt for
- * both the TCP receive window size and the
- * send buffer size.
- * Till the time iobuf size becomes configurable, this size is set to include
- * two iobufs + the GlusterFS protocol headers.
+/* The default window size will be 0, indicating not to set
+ * it to any size. Default size of Linux is found to be
+ * performance friendly.
  * Linux allows us to over-ride the max values for the system.
  * Should we over-ride them? Because if we set a value larger than the default
  * setsockopt will fail. Having larger values might be beneficial for
  * IB links.
  */
-#define GF_DEFAULT_SOCKET_WINDOW_SIZE   (512 * GF_UNIT_KB)
+#define GF_DEFAULT_SOCKET_WINDOW_SIZE   (0)
 #define GF_MAX_SOCKET_WINDOW_SIZE       (1 * GF_UNIT_MB)
-#define GF_MIN_SOCKET_WINDOW_SIZE       (128 * GF_UNIT_KB)
+#define GF_MIN_SOCKET_WINDOW_SIZE       (0)
 #define GF_USE_DEFAULT_KEEPALIVE        (-1)
 
 typedef enum {
@@ -83,6 +74,12 @@ typedef enum {
         SP_STATE_READ_VERFBYTES,        /* read verifier data */
         SP_STATE_READING_PROGHDR,
         SP_STATE_READ_PROGHDR,
+        SP_STATE_READING_PROGHDR_XDATA,
+        SP_STATE_READ_PROGHDR_XDATA,    /* It's a bad "name" in the generic
+					   RPC state machine, but greatly
+					   aids code review (and xdata is
+					   the only "consumer" of this state)
+					*/
         SP_STATE_READING_PROG,
 } sp_rpcfrag_vectored_request_state_t;
 
@@ -126,6 +123,8 @@ typedef enum {
 typedef enum {
         SP_STATE_ACCEPTED_SUCCESS_REPLY_INIT,
         SP_STATE_READING_PROC_HEADER,
+        SP_STATE_READING_PROC_OPAQUE,
+        SP_STATE_READ_PROC_OPAQUE,
         SP_STATE_READ_PROC_HEADER,
 } sp_rpcfrag_vectored_reply_accepted_success_state_t;
 
@@ -144,10 +143,60 @@ typedef struct {
         sp_rpcfrag_vectored_reply_accepted_success_state_t accepted_success_state;
 } sp_rpcfrag_vectored_reply_state_t;
 
+struct gf_sock_incoming_frag {
+        char         *fragcurrent;
+        uint32_t      bytes_read;
+        uint32_t      remaining_size;
+        struct iovec  vector;
+        struct iovec *pending_vector;
+        union {
+                sp_rpcfrag_request_state_t        request;
+                sp_rpcfrag_vectored_reply_state_t reply;
+        } call_body;
+
+        sp_rpcfrag_simple_msg_state_t     simple_state;
+        sp_rpcfrag_state_t state;
+};
+
+#define GF_SOCKET_RA_MAX 1024
+
+struct gf_sock_incoming {
+        sp_rpcrecord_state_t  record_state;
+        struct gf_sock_incoming_frag frag;
+        char                *proghdr_base_addr;
+        struct iobuf        *iobuf;
+        size_t               iobuf_size;
+        struct iovec         vector[2];
+        int                  count;
+        struct iovec         payload_vector;
+        struct iobref       *iobref;
+        rpc_request_info_t  *request_info;
+        struct iovec        *pending_vector;
+        int                  pending_count;
+        uint32_t             fraghdr;
+        char                 complete_record;
+        msg_type_t           msg_type;
+        size_t               total_bytes_read;
+
+	size_t               ra_read;
+	size_t               ra_max;
+	size_t               ra_served;
+	char                *ra_buf;
+};
+
+typedef enum {
+        OT_IDLE,        /* Uninitialized or termination complete. */
+        OT_SPAWNING,    /* Past pthread_create but not in thread yet. */
+        OT_RUNNING,     /* Poller thread running normally. */
+        OT_CALLBACK,    /* Poller thread in the middle of a callback. */
+        OT_PLEASE_DIE,  /* Poller termination requested. */
+} ot_state_t;
+
 typedef struct {
         int32_t                sock;
         int32_t                idx;
-        unsigned char          connected; // -1 = not connected. 0 = in progress. 1 = connected
+        /* -1 = not connected. 0 = in progress. 1 = connected */
+        char                   connected;
         char                   bio;
         char                   connect_finish_log;
         char                   submit_log;
@@ -158,36 +207,7 @@ typedef struct {
                         struct ioq        *ioq_prev;
                 };
         };
-        struct {
-                sp_rpcrecord_state_t  record_state;
-                struct {
-                        char         *fragcurrent;
-                        uint32_t      bytes_read;
-                        uint32_t      remaining_size;
-                        struct iovec  vector;
-                        struct iovec *pending_vector;
-                        union {
-                                sp_rpcfrag_request_state_t        request;
-                                sp_rpcfrag_vectored_reply_state_t reply;
-                        } call_body;
-
-                        sp_rpcfrag_simple_msg_state_t     simple_state;
-                        sp_rpcfrag_state_t state;
-                } frag;
-                struct iobuf        *iobuf;
-                size_t               iobuf_size;
-                struct iovec         vector[2];
-                int                  count;
-                struct iovec         payload_vector;
-                struct iobref       *iobref;
-                rpc_request_info_t  *request_info;
-                struct iovec        *pending_vector;
-                int                  pending_count;
-                uint32_t             fraghdr;
-                char                 complete_record;
-                msg_type_t           msg_type;
-                size_t               total_bytes_read;
-        } incoming;
+        struct gf_sock_incoming incoming;
         pthread_mutex_t        lock;
         int                    windowsize;
         char                   lowlat;
@@ -197,6 +217,22 @@ typedef struct {
         int                    keepaliveintvl;
         uint32_t               backlog;
         gf_boolean_t           read_fail_log;
+        gf_boolean_t           ssl_enabled;
+	gf_boolean_t           use_ssl;
+	SSL_METHOD            *ssl_meth;
+	SSL_CTX               *ssl_ctx;
+	int                    ssl_session_id;
+	BIO                   *ssl_sbio;
+	SSL                   *ssl_ssl;
+	char                  *ssl_own_cert;
+	char                  *ssl_private_key;
+	char                  *ssl_ca_list;
+	pthread_t              thread;
+	int                    pipe[2];
+	gf_boolean_t           own_thread;
+        ot_state_t             ot_state;
+        uint32_t               ot_gen;
+        gf_boolean_t           is_server;
 } socket_private_t;
 
 
diff --git a/rpc/xdr/src/Makefile.am b/rpc/xdr/src/Makefile.am
index 57c193387..949e75e8d 100644
--- a/rpc/xdr/src/Makefile.am
+++ b/rpc/xdr/src/Makefile.am
@@ -1,9 +1,8 @@
 lib_LTLIBRARIES = libgfxdr.la
 
-libgfxdr_la_CFLAGS = -fPIC  -Wall -g -shared -nostartfiles $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS)
+libgfxdr_la_CFLAGS = -Wall $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS)
 
-libgfxdr_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 \
-			-D_GNU_SOURCE -D$(GF_HOST_OS) \
+libgfxdr_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
 			-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src
 
 libgfxdr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
@@ -14,11 +13,13 @@ libgfxdr_la_SOURCES =  xdr-generic.c rpc-common-xdr.c \
 			cli1-xdr.c \
 			glusterd1-xdr.c \
 			portmap-xdr.c \
-			xdr-nfs3.c msg-nfs3.c
+			nlm4-xdr.c xdr-nfs3.c msg-nfs3.c nsm-xdr.c \
+			nlmcbk-xdr.c acl3-xdr.c
 
 noinst_HEADERS = xdr-generic.h rpc-common-xdr.h \
 		glusterfs3-xdr.h glusterfs3.h \
 		cli1-xdr.h \
 		glusterd1-xdr.h \
 		portmap-xdr.h \
-		xdr-nfs3.h msg-nfs3.h
+		nlm4-xdr.h xdr-nfs3.h msg-nfs3.h nsm-xdr.h \
+		nlmcbk-xdr.h acl3-xdr.h
diff --git a/rpc/xdr/src/acl.x b/rpc/xdr/src/acl.x
new file mode 100644
index 000000000..6cf4f1d3b
--- /dev/null
+++ b/rpc/xdr/src/acl.x
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+
+struct aclentry {
+	int type;
+	int uid;
+	int perm;
+};
+
+struct getaclargs {
+	netobj	fh;
+	int		mask;
+};
+
+struct getaclreply {
+	int status;
+	int attr_follows;
+	struct fattr3 attr;
+	int	mask;
+	int aclcount;
+	struct aclentry aclentry<>;
+	int daclcount;
+	struct aclentry daclentry<>;
+};
+
+struct setaclargs {
+	netobj fh;
+	int mask;
+	int aclcount;
+	struct aclentry aclentry<>;
+	int daclcount;
+	struct aclentry daclentry<>;
+};
+
+struct setaclreply {
+	int status;
+	int attr_follows;
+	struct fattr3 attr;
+};
+
diff --git a/rpc/xdr/src/acl3-xdr.c b/rpc/xdr/src/acl3-xdr.c
new file mode 100644
index 000000000..8fbaeff16
--- /dev/null
+++ b/rpc/xdr/src/acl3-xdr.c
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#include "acl3-xdr.h"
+
+bool_t
+xdr_aclentry (XDR *xdrs, aclentry *objp)
+{
+	 if (!xdr_int (xdrs, &objp->type))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->uid))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->perm))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_getaclargs (XDR *xdrs, getaclargs *objp)
+{
+	 if (!xdr_netobj (xdrs, &objp->fh))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->mask))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_getaclreply (XDR *xdrs, getaclreply *objp)
+{
+	 if (!xdr_int (xdrs, &objp->status))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->attr_follows))
+		 return FALSE;
+	 if (!xdr_fattr3 (xdrs, &objp->attr))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->mask))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->aclcount))
+		 return FALSE;
+	 if (!xdr_array (xdrs, (char **)&objp->aclentry.aclentry_val, (u_int *) &objp->aclentry.aclentry_len, ~0,
+		sizeof (aclentry), (xdrproc_t) xdr_aclentry))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->daclcount))
+		 return FALSE;
+	 if (!xdr_array (xdrs, (char **)&objp->daclentry.daclentry_val, (u_int *) &objp->daclentry.daclentry_len, ~0,
+		sizeof (aclentry), (xdrproc_t) xdr_aclentry))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_setaclargs (XDR *xdrs, setaclargs *objp)
+{
+	 if (!xdr_netobj (xdrs, &objp->fh))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->mask))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->aclcount))
+		 return FALSE;
+	 if (!xdr_array (xdrs, (char **)&objp->aclentry.aclentry_val, (u_int *) &objp->aclentry.aclentry_len, ~0,
+		sizeof (aclentry), (xdrproc_t) xdr_aclentry))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->daclcount))
+		 return FALSE;
+	 if (!xdr_array (xdrs, (char **)&objp->daclentry.daclentry_val, (u_int *) &objp->daclentry.daclentry_len, ~0,
+		sizeof (aclentry), (xdrproc_t) xdr_aclentry))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_setaclreply (XDR *xdrs, setaclreply *objp)
+{
+	 if (!xdr_int (xdrs, &objp->status))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->attr_follows))
+		 return FALSE;
+	 if (!xdr_fattr3 (xdrs, &objp->attr))
+		 return FALSE;
+	return TRUE;
+}
diff --git a/rpc/xdr/src/acl3-xdr.h b/rpc/xdr/src/acl3-xdr.h
new file mode 100644
index 000000000..7cebaed69
--- /dev/null
+++ b/rpc/xdr/src/acl3-xdr.h
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#ifndef _ACL_H_RPCGEN
+#define _ACL_H_RPCGEN
+
+#include <rpc/rpc.h>
+#include "xdr-nfs3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+struct aclentry {
+	int type;
+	int uid;
+	int perm;
+};
+typedef struct aclentry aclentry;
+
+struct getaclargs {
+	netobj fh;
+	int mask;
+};
+typedef struct getaclargs getaclargs;
+
+struct getaclreply {
+	int status;
+	int attr_follows;
+	struct fattr3 attr;
+	int mask;
+	int aclcount;
+	struct {
+		u_int aclentry_len;
+		struct aclentry *aclentry_val;
+	} aclentry;
+	int daclcount;
+	struct {
+		u_int daclentry_len;
+		struct aclentry *daclentry_val;
+	} daclentry;
+};
+typedef struct getaclreply getaclreply;
+
+struct setaclargs {
+	netobj fh;
+	int mask;
+	int aclcount;
+	struct {
+		u_int aclentry_len;
+		struct aclentry *aclentry_val;
+	} aclentry;
+	int daclcount;
+	struct {
+		u_int daclentry_len;
+		struct aclentry *daclentry_val;
+	} daclentry;
+};
+typedef struct setaclargs setaclargs;
+
+struct setaclreply {
+	int status;
+	int attr_follows;
+	struct fattr3 attr;
+};
+typedef struct setaclreply setaclreply;
+
+#define ACL3_NULL 0
+#define ACL3_GETACL 1
+#define ACL3_SETACL 2
+#define ACL3_PROC_COUNT 3
+/* the xdr functions */
+
+#if defined(__STDC__) || defined(__cplusplus)
+extern  bool_t xdr_aclentry (XDR *, aclentry*);
+extern  bool_t xdr_getaclargs (XDR *, getaclargs*);
+extern  bool_t xdr_getaclreply (XDR *, getaclreply*);
+extern  bool_t xdr_setaclargs (XDR *, setaclargs*);
+extern  bool_t xdr_setaclreply (XDR *, setaclreply*);
+
+#else /* K&R C */
+extern bool_t xdr_aclentry ();
+extern bool_t xdr_getaclargs ();
+extern bool_t xdr_getaclreply ();
+extern bool_t xdr_setaclargs ();
+extern bool_t xdr_setaclreply ();
+
+#endif /* K&R C */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_ACL_H_RPCGEN */
diff --git a/rpc/xdr/src/cli1-xdr.c b/rpc/xdr/src/cli1-xdr.c
index 56562ee3d..97b210e14 100644
--- a/rpc/xdr/src/cli1-xdr.c
+++ b/rpc/xdr/src/cli1-xdr.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
@@ -188,128 +179,51 @@ xdr_gf_cli_status_type (XDR *xdrs, gf_cli_status_type *objp)
 }
 
 bool_t
-xdr_gf_cli_req (XDR *xdrs, gf_cli_req *objp)
+xdr_gf1_cli_snapshot (XDR *xdrs, gf1_cli_snapshot *objp)
 {
 	register int32_t *buf;
         buf = NULL;
 
-	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+	 if (!xdr_enum (xdrs, (enum_t *) objp))
 		 return FALSE;
 	return TRUE;
 }
 
 bool_t
-xdr_gf_cli_rsp (XDR *xdrs, gf_cli_rsp *objp)
+xdr_gf1_cli_snapshot_config (XDR *xdrs, gf1_cli_snapshot_config *objp)
 {
 	register int32_t *buf;
         buf = NULL;
 
-	 if (!xdr_int (xdrs, &objp->op_ret))
-		 return FALSE;
-	 if (!xdr_int (xdrs, &objp->op_errno))
-		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
-		 return FALSE;
-	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+	 if (!xdr_enum (xdrs, (enum_t *) objp))
 		 return FALSE;
 	return TRUE;
 }
 
 bool_t
-xdr_gf1_cli_probe_req (XDR *xdrs, gf1_cli_probe_req *objp)
+xdr_gf_cli_req (XDR *xdrs, gf_cli_req *objp)
 {
 	register int32_t *buf;
         buf = NULL;
 
-	 if (!xdr_string (xdrs, &objp->hostname, ~0))
-		 return FALSE;
-	 if (!xdr_int (xdrs, &objp->port))
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
 		 return FALSE;
 	return TRUE;
 }
 
 bool_t
-xdr_gf1_cli_probe_rsp (XDR *xdrs, gf1_cli_probe_rsp *objp)
+xdr_gf_cli_rsp (XDR *xdrs, gf_cli_rsp *objp)
 {
 	register int32_t *buf;
         buf = NULL;
 
-
-	if (xdrs->x_op == XDR_ENCODE) {
-		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
-		if (buf == NULL) {
-			 if (!xdr_int (xdrs, &objp->op_ret))
-				 return FALSE;
-			 if (!xdr_int (xdrs, &objp->op_errno))
-				 return FALSE;
-			 if (!xdr_int (xdrs, &objp->port))
-				 return FALSE;
-
-		} else {
-		IXDR_PUT_LONG(buf, objp->op_ret);
-		IXDR_PUT_LONG(buf, objp->op_errno);
-		IXDR_PUT_LONG(buf, objp->port);
-		}
-		 if (!xdr_string (xdrs, &objp->hostname, ~0))
-			 return FALSE;
-		return TRUE;
-	} else if (xdrs->x_op == XDR_DECODE) {
-		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
-		if (buf == NULL) {
-			 if (!xdr_int (xdrs, &objp->op_ret))
-				 return FALSE;
-			 if (!xdr_int (xdrs, &objp->op_errno))
-				 return FALSE;
-			 if (!xdr_int (xdrs, &objp->port))
-				 return FALSE;
-
-		} else {
-		objp->op_ret = IXDR_GET_LONG(buf);
-		objp->op_errno = IXDR_GET_LONG(buf);
-		objp->port = IXDR_GET_LONG(buf);
-		}
-		 if (!xdr_string (xdrs, &objp->hostname, ~0))
-			 return FALSE;
-	 return TRUE;
-	}
-
 	 if (!xdr_int (xdrs, &objp->op_ret))
 		 return FALSE;
 	 if (!xdr_int (xdrs, &objp->op_errno))
 		 return FALSE;
-	 if (!xdr_int (xdrs, &objp->port))
-		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->hostname, ~0))
-		 return FALSE;
-	return TRUE;
-}
-
-bool_t
-xdr_gf1_cli_deprobe_req (XDR *xdrs, gf1_cli_deprobe_req *objp)
-{
-	register int32_t *buf;
-        buf = NULL;
-
-	 if (!xdr_string (xdrs, &objp->hostname, ~0))
-		 return FALSE;
-	 if (!xdr_int (xdrs, &objp->port))
-		 return FALSE;
-	 if (!xdr_int (xdrs, &objp->flags))
-		 return FALSE;
-	return TRUE;
-}
-
-bool_t
-xdr_gf1_cli_deprobe_rsp (XDR *xdrs, gf1_cli_deprobe_rsp *objp)
-{
-	register int32_t *buf;
-        buf = NULL;
-
-	 if (!xdr_int (xdrs, &objp->op_ret))
-		 return FALSE;
-	 if (!xdr_int (xdrs, &objp->op_errno))
+	 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
 		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->hostname, ~0))
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
 		 return FALSE;
 	return TRUE;
 }
diff --git a/rpc/xdr/src/cli1-xdr.h b/rpc/xdr/src/cli1-xdr.h
index ea8ab02b6..5e8c29fbb 100644
--- a/rpc/xdr/src/cli1-xdr.h
+++ b/rpc/xdr/src/cli1-xdr.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
@@ -47,22 +38,20 @@ enum gf_cli_defrag_type {
 	GF_DEFRAG_CMD_STOP = 1 + 1,
 	GF_DEFRAG_CMD_STATUS = 1 + 2,
 	GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
-	GF_DEFRAG_CMD_START_MIGRATE_DATA = 1 + 4,
-	GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE = 1 + 5,
-	GF_DEFRAG_CMD_START_FORCE = 1 + 6,
+	GF_DEFRAG_CMD_START_FORCE = 1 + 4,
 };
 typedef enum gf_cli_defrag_type gf_cli_defrag_type;
 
 enum gf_defrag_status_t {
 	GF_DEFRAG_STATUS_NOT_STARTED = 0,
-	GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED = 1,
-	GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED = 2,
-	GF_DEFRAG_STATUS_STOPPED = 3,
-	GF_DEFRAG_STATUS_COMPLETE = 4,
-	GF_DEFRAG_STATUS_FAILED = 5,
-	GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE = 6,
-	GF_DEFRAG_STATUS_MIGRATE_DATA_COMPLETE = 7,
-	GF_DEFRAG_STATUS_PAUSED = 8,
+	GF_DEFRAG_STATUS_STARTED = 1,
+	GF_DEFRAG_STATUS_STOPPED = 2,
+	GF_DEFRAG_STATUS_COMPLETE = 3,
+	GF_DEFRAG_STATUS_FAILED = 4,
+	GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED = 5,
+	GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED = 6,
+	GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE = 7,
+	GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED = 8,
 };
 typedef enum gf_defrag_status_t gf_defrag_status_t;
 
@@ -89,10 +78,9 @@ enum gf1_op_commands {
 	GF_OP_CMD_NONE = 0,
 	GF_OP_CMD_START = 0 + 1,
 	GF_OP_CMD_COMMIT = 0 + 2,
-	GF_OP_CMD_PAUSE = 0 + 3,
-	GF_OP_CMD_ABORT = 0 + 4,
-	GF_OP_CMD_STATUS = 0 + 5,
-	GF_OP_CMD_COMMIT_FORCE = 0 + 6,
+	GF_OP_CMD_STOP = 0 + 3,
+	GF_OP_CMD_STATUS = 0 + 4,
+	GF_OP_CMD_COMMIT_FORCE = 0 + 5,
 };
 typedef enum gf1_op_commands gf1_op_commands;
 
@@ -108,7 +96,8 @@ enum gf_quota_type {
 typedef enum gf_quota_type gf_quota_type;
 
 enum gf1_cli_friends_list {
-	GF_CLI_LIST_ALL = 1,
+	GF_CLI_LIST_PEERS = 1,
+	GF_CLI_LIST_POOL_NODES = 2,
 };
 typedef enum gf1_cli_friends_list gf1_cli_friends_list;
 
@@ -136,6 +125,8 @@ enum gf1_cli_gsync_set {
 	GF_GSYNC_OPTION_TYPE_CONFIG = 3,
 	GF_GSYNC_OPTION_TYPE_STATUS = 4,
 	GF_GSYNC_OPTION_TYPE_ROTATE = 5,
+	GF_GSYNC_OPTION_TYPE_CREATE = 6,
+	GF_GSYNC_OPTION_TYPE_DELETE = 7,
 };
 typedef enum gf1_cli_gsync_set gf1_cli_gsync_set;
 
@@ -161,21 +152,58 @@ enum gf1_cli_top_op {
 typedef enum gf1_cli_top_op gf1_cli_top_op;
 
 enum gf_cli_status_type {
-	GF_CLI_STATUS_INVAL = -1,
-	GF_CLI_STATUS_NONE = 0x000,
-	GF_CLI_STATUS_MEM = 0x001,
-	GF_CLI_STATUS_CLIENTS = 0x002,
-	GF_CLI_STATUS_INODE = 0x004,
-	GF_CLI_STATUS_FD = 0x008,
-	GF_CLI_STATUS_CALLPOOL = 0x010,
-	GF_CLI_STATUS_DETAIL = 0x020,
-	GF_CLI_STATUS_MASK = 0x0FF,
-	GF_CLI_STATUS_VOL = 0x100,
-	GF_CLI_STATUS_ALL = 0x200,
-	GF_CLI_STATUS_BRICK = 0x400,
+	GF_CLI_STATUS_NONE = 0x0000,
+	GF_CLI_STATUS_MEM = 0x0001,
+	GF_CLI_STATUS_CLIENTS = 0x0002,
+	GF_CLI_STATUS_INODE = 0x0004,
+	GF_CLI_STATUS_FD = 0x0008,
+	GF_CLI_STATUS_CALLPOOL = 0x0010,
+	GF_CLI_STATUS_DETAIL = 0x0020,
+	GF_CLI_STATUS_TASKS = 0x0040,
+	GF_CLI_STATUS_MASK = 0x00FF,
+	GF_CLI_STATUS_VOL = 0x0100,
+	GF_CLI_STATUS_ALL = 0x0200,
+	GF_CLI_STATUS_BRICK = 0x0400,
+	GF_CLI_STATUS_NFS = 0x0800,
+	GF_CLI_STATUS_SHD = 0x1000,
 };
 typedef enum gf_cli_status_type gf_cli_status_type;
 
+enum gf1_cli_snapshot {
+	GF_SNAP_OPTION_TYPE_NONE = 0,
+	GF_SNAP_OPTION_TYPE_CREATE = 1,
+	GF_SNAP_OPTION_TYPE_DELETE = 2,
+	GF_SNAP_OPTION_TYPE_RESTORE = 3,
+	GF_SNAP_OPTION_TYPE_START = 4,
+	GF_SNAP_OPTION_TYPE_STOP = 5,
+	GF_SNAP_OPTION_TYPE_LIST = 6,
+	GF_SNAP_OPTION_TYPE_STATUS = 7,
+	GF_SNAP_OPTION_TYPE_CONFIG = 8,
+        GF_SNAP_OPTION_TYPE_INFO = 9,
+};
+typedef enum gf1_cli_snapshot gf1_cli_snapshot;
+
+enum gf1_cli_snapshot_info {
+        GF_SNAP_INFO_TYPE_ALL = 0,
+        GF_SNAP_INFO_TYPE_SNAP = 1,
+        GF_SNAP_INFO_TYPE_VOL = 2,
+};
+typedef enum gf1_cli_snapshot_info gf1_cli_snapshot_info;
+
+enum gf1_cli_snapshot_config {
+        GF_SNAP_CONFIG_TYPE_NONE = 0,
+        GF_SNAP_CONFIG_TYPE_SET = 1,
+	GF_SNAP_CONFIG_DISPLAY  = 2,
+};
+typedef enum gf1_cli_snapshot_config gf1_cli_snapshot_config;
+
+enum  gf1_cli_snapshot_status {
+        GF_SNAP_STATUS_TYPE_ALL  = 0,
+        GF_SNAP_STATUS_TYPE_SNAP = 1,
+        GF_SNAP_STATUS_TYPE_VOL = 2,
+};
+typedef enum gf1_cli_snapshot_status gf1_cli_snapshot_status;
+
 struct gf_cli_req {
 	struct {
 		u_int dict_len;
@@ -195,34 +223,6 @@ struct gf_cli_rsp {
 };
 typedef struct gf_cli_rsp gf_cli_rsp;
 
-struct gf1_cli_probe_req {
-	char *hostname;
-	int port;
-};
-typedef struct gf1_cli_probe_req gf1_cli_probe_req;
-
-struct gf1_cli_probe_rsp {
-	int op_ret;
-	int op_errno;
-	int port;
-	char *hostname;
-};
-typedef struct gf1_cli_probe_rsp gf1_cli_probe_rsp;
-
-struct gf1_cli_deprobe_req {
-	char *hostname;
-	int port;
-	int flags;
-};
-typedef struct gf1_cli_deprobe_req gf1_cli_deprobe_req;
-
-struct gf1_cli_deprobe_rsp {
-	int op_ret;
-	int op_errno;
-	char *hostname;
-};
-typedef struct gf1_cli_deprobe_rsp gf1_cli_deprobe_rsp;
-
 struct gf1_cli_peer_list_req {
 	int flags;
 	struct {
@@ -315,12 +315,10 @@ extern  bool_t xdr_gf1_cli_gsync_set (XDR *, gf1_cli_gsync_set*);
 extern  bool_t xdr_gf1_cli_stats_op (XDR *, gf1_cli_stats_op*);
 extern  bool_t xdr_gf1_cli_top_op (XDR *, gf1_cli_top_op*);
 extern  bool_t xdr_gf_cli_status_type (XDR *, gf_cli_status_type*);
+extern  bool_t xdr_gf1_cli_snapshot (XDR *, gf1_cli_snapshot*);
+extern  bool_t xdr_gf1_cli_snapshot_config (XDR *, gf1_cli_snapshot_config*);
 extern  bool_t xdr_gf_cli_req (XDR *, gf_cli_req*);
 extern  bool_t xdr_gf_cli_rsp (XDR *, gf_cli_rsp*);
-extern  bool_t xdr_gf1_cli_probe_req (XDR *, gf1_cli_probe_req*);
-extern  bool_t xdr_gf1_cli_probe_rsp (XDR *, gf1_cli_probe_rsp*);
-extern  bool_t xdr_gf1_cli_deprobe_req (XDR *, gf1_cli_deprobe_req*);
-extern  bool_t xdr_gf1_cli_deprobe_rsp (XDR *, gf1_cli_deprobe_rsp*);
 extern  bool_t xdr_gf1_cli_peer_list_req (XDR *, gf1_cli_peer_list_req*);
 extern  bool_t xdr_gf1_cli_peer_list_rsp (XDR *, gf1_cli_peer_list_rsp*);
 extern  bool_t xdr_gf1_cli_fsm_log_req (XDR *, gf1_cli_fsm_log_req*);
@@ -347,12 +345,10 @@ extern bool_t xdr_gf1_cli_gsync_set ();
 extern bool_t xdr_gf1_cli_stats_op ();
 extern bool_t xdr_gf1_cli_top_op ();
 extern bool_t xdr_gf_cli_status_type ();
+extern bool_t xdr_gf1_cli_snapshot ();
+extern bool_t xdr_gf1_cli_snapshot_config ();
 extern bool_t xdr_gf_cli_req ();
 extern bool_t xdr_gf_cli_rsp ();
-extern bool_t xdr_gf1_cli_probe_req ();
-extern bool_t xdr_gf1_cli_probe_rsp ();
-extern bool_t xdr_gf1_cli_deprobe_req ();
-extern bool_t xdr_gf1_cli_deprobe_rsp ();
 extern bool_t xdr_gf1_cli_peer_list_req ();
 extern bool_t xdr_gf1_cli_peer_list_rsp ();
 extern bool_t xdr_gf1_cli_fsm_log_req ();
diff --git a/rpc/xdr/src/cli1-xdr.x b/rpc/xdr/src/cli1-xdr.x
index a2edc57eb..f9d29b7e1 100644
--- a/rpc/xdr/src/cli1-xdr.x
+++ b/rpc/xdr/src/cli1-xdr.x
@@ -3,21 +3,19 @@
         GF_DEFRAG_CMD_STOP,
         GF_DEFRAG_CMD_STATUS,
         GF_DEFRAG_CMD_START_LAYOUT_FIX,
-        GF_DEFRAG_CMD_START_MIGRATE_DATA,
-        GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE,
         GF_DEFRAG_CMD_START_FORCE /* used by remove-brick data migration */
 } ;
 
  enum gf_defrag_status_t {
         GF_DEFRAG_STATUS_NOT_STARTED,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
-        GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED,
+        GF_DEFRAG_STATUS_STARTED,
         GF_DEFRAG_STATUS_STOPPED,
         GF_DEFRAG_STATUS_COMPLETE,
         GF_DEFRAG_STATUS_FAILED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
         GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
-        GF_DEFRAG_STATUS_MIGRATE_DATA_COMPLETE,
-        GF_DEFRAG_STATUS_PAUSED
+        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED
 } ;
 
  enum gf1_cluster_type {
@@ -41,8 +39,7 @@
         GF_OP_CMD_NONE = 0,
         GF_OP_CMD_START,
         GF_OP_CMD_COMMIT,
-        GF_OP_CMD_PAUSE,
-        GF_OP_CMD_ABORT,
+        GF_OP_CMD_STOP,
         GF_OP_CMD_STATUS,
         GF_OP_CMD_COMMIT_FORCE
 } ;
@@ -58,7 +55,8 @@ enum gf_quota_type {
 };
 
 enum gf1_cli_friends_list {
-        GF_CLI_LIST_ALL = 1
+        GF_CLI_LIST_PEERS = 1,
+        GF_CLI_LIST_POOL_NODES = 2
 } ;
 
 enum gf1_cli_get_volume {
@@ -81,7 +79,9 @@ enum gf1_cli_gsync_set {
         GF_GSYNC_OPTION_TYPE_STOP,
         GF_GSYNC_OPTION_TYPE_CONFIG,
         GF_GSYNC_OPTION_TYPE_STATUS,
-        GF_GSYNC_OPTION_TYPE_ROTATE
+        GF_GSYNC_OPTION_TYPE_ROTATE,
+        GF_GSYNC_OPTION_TYPE_CREATE,
+        GF_GSYNC_OPTION_TYPE_DELETE
 };
 
 enum gf1_cli_stats_op {
@@ -106,18 +106,40 @@ enum gf1_cli_top_op {
 /* The unconventional hex numbers help us perform
    bit-wise operations which reduces complexity */
 enum gf_cli_status_type {
-        GF_CLI_STATUS_INVAL        = -1,
-        GF_CLI_STATUS_NONE         = 0x000,
-        GF_CLI_STATUS_MEM          = 0x001,     /*000000000001*/
-        GF_CLI_STATUS_CLIENTS      = 0x002,     /*000000000010*/
-        GF_CLI_STATUS_INODE        = 0x004,     /*000000000100*/
-        GF_CLI_STATUS_FD           = 0x008,     /*000000001000*/
-        GF_CLI_STATUS_CALLPOOL     = 0x010,     /*000000010000*/
-        GF_CLI_STATUS_DETAIL       = 0x020,     /*000000100000*/
-        GF_CLI_STATUS_MASK         = 0x0FF,     /*000011111111 Used to get the op*/
-        GF_CLI_STATUS_VOL          = 0x100,     /*000100000000*/
-        GF_CLI_STATUS_ALL          = 0x200,     /*001000000000*/
-        GF_CLI_STATUS_BRICK        = 0x400      /*010000000000*/
+        GF_CLI_STATUS_NONE         = 0x0000,
+        GF_CLI_STATUS_MEM          = 0x0001,    /*0000000000001*/
+        GF_CLI_STATUS_CLIENTS      = 0x0002,    /*0000000000010*/
+        GF_CLI_STATUS_INODE        = 0x0004,    /*0000000000100*/
+        GF_CLI_STATUS_FD           = 0x0008,    /*0000000001000*/
+        GF_CLI_STATUS_CALLPOOL     = 0x0010,    /*0000000010000*/
+        GF_CLI_STATUS_DETAIL       = 0x0020,    /*0000000100000*/
+        GF_CLI_STATUS_TASKS        = 0x0040,    /*0000001000000*/
+        GF_CLI_STATUS_MASK         = 0x00FF,    /*0000011111111 Used to get the op*/
+        GF_CLI_STATUS_VOL          = 0x0100,    /*0000100000000*/
+        GF_CLI_STATUS_ALL          = 0x0200,    /*0001000000000*/
+        GF_CLI_STATUS_BRICK        = 0x0400,    /*0010000000000*/
+        GF_CLI_STATUS_NFS          = 0x0800,    /*0100000000000*/
+        GF_CLI_STATUS_SHD          = 0x1000     /*1000000000000*/
+};
+
+/* Identifiers for snapshot clis */
+enum gf1_cli_snapshot {
+        GF_SNAP_OPTION_TYPE_NONE = 0,
+        GF_SNAP_OPTION_TYPE_CREATE,
+        GF_SNAP_OPTION_TYPE_DELETE,
+        GF_SNAP_OPTION_TYPE_RESTORE,
+        GF_SNAP_OPTION_TYPE_START,
+        GF_SNAP_OPTION_TYPE_STOP,
+        GF_SNAP_OPTION_TYPE_LIST,
+        GF_SNAP_OPTION_TYPE_STATUS,
+        GF_SNAP_OPTION_TYPE_CONFIG
+};
+
+enum gf1_cli_snapshot_config {
+        GF_SNAP_CONFIG_TYPE_NONE = 0,
+        GF_SNAP_CONFIG_TYPE_SET,
+        GF_SNAP_CONFIG_DISPLAY,
+
 };
 
  struct gf_cli_req {
@@ -131,30 +153,6 @@ enum gf_cli_status_type {
         opaque  dict<>;
 }  ;
 
- struct gf1_cli_probe_req {
-        string  hostname<>;
-	int	port;
-}  ;
-
- struct gf1_cli_probe_rsp {
-        int     op_ret;
-        int     op_errno;
-	int	port;
-        string  hostname<>;
-}  ;
-
- struct gf1_cli_deprobe_req {
-        string  hostname<>;
-	int	port;
-        int     flags;
-}  ;
-
- struct gf1_cli_deprobe_rsp {
-        int     op_ret;
-        int     op_errno;
-        string  hostname<>;
-}  ;
-
 struct gf1_cli_peer_list_req {
         int     flags;
         opaque  dict<>;
diff --git a/rpc/xdr/src/glusterd1-xdr.c b/rpc/xdr/src/glusterd1-xdr.c
index a5438d23e..7fa98aaeb 100644
--- a/rpc/xdr/src/glusterd1-xdr.c
+++ b/rpc/xdr/src/glusterd1-xdr.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
@@ -66,6 +57,55 @@ xdr_gd1_mgmt_probe_rsp (XDR *xdrs, gd1_mgmt_probe_rsp *objp)
 	register int32_t *buf;
         buf = NULL;
 
+
+	if (xdrs->x_op == XDR_ENCODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		 if (!xdr_string (xdrs, &objp->hostname, ~0))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->port))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		IXDR_PUT_LONG(buf, objp->port);
+		IXDR_PUT_LONG(buf, objp->op_ret);
+		IXDR_PUT_LONG(buf, objp->op_errno);
+		}
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+		return TRUE;
+	} else if (xdrs->x_op == XDR_DECODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		 if (!xdr_string (xdrs, &objp->hostname, ~0))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->port))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		objp->port = IXDR_GET_LONG(buf);
+		objp->op_ret = IXDR_GET_LONG(buf);
+		objp->op_errno = IXDR_GET_LONG(buf);
+		}
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+	 return TRUE;
+	}
+
 	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
 		sizeof (u_char), (xdrproc_t) xdr_u_char))
 		 return FALSE;
@@ -77,6 +117,8 @@ xdr_gd1_mgmt_probe_rsp (XDR *xdrs, gd1_mgmt_probe_rsp *objp)
 		 return FALSE;
 	 if (!xdr_int (xdrs, &objp->op_errno))
 		 return FALSE;
+	 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+		 return FALSE;
 	return TRUE;
 }
 
@@ -449,3 +491,433 @@ xdr_gd1_mgmt_brick_op_rsp (XDR *xdrs, gd1_mgmt_brick_op_rsp *objp)
 		 return FALSE;
 	return TRUE;
 }
+
+bool_t
+xdr_gd1_mgmt_v3_lock_req (XDR *xdrs, gd1_mgmt_v3_lock_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_vector (xdrs, (char *)objp->txn_id, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_lock_rsp (XDR *xdrs, gd1_mgmt_v3_lock_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_vector (xdrs, (char *)objp->txn_id, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_pre_val_req (XDR *xdrs, gd1_mgmt_v3_pre_val_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_pre_val_rsp (XDR *xdrs, gd1_mgmt_v3_pre_val_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+
+	if (xdrs->x_op == XDR_ENCODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->op))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		IXDR_PUT_LONG(buf, objp->op);
+		IXDR_PUT_LONG(buf, objp->op_ret);
+		IXDR_PUT_LONG(buf, objp->op_errno);
+		}
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+		 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+			 return FALSE;
+		return TRUE;
+	} else if (xdrs->x_op == XDR_DECODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->op))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		objp->op = IXDR_GET_LONG(buf);
+		objp->op_ret = IXDR_GET_LONG(buf);
+		objp->op_errno = IXDR_GET_LONG(buf);
+		}
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+		 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+			 return FALSE;
+	 return TRUE;
+	}
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_brick_op_req (XDR *xdrs, gd1_mgmt_v3_brick_op_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_brick_op_rsp (XDR *xdrs, gd1_mgmt_v3_brick_op_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+
+	if (xdrs->x_op == XDR_ENCODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->op))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		IXDR_PUT_LONG(buf, objp->op);
+		IXDR_PUT_LONG(buf, objp->op_ret);
+		IXDR_PUT_LONG(buf, objp->op_errno);
+		}
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+		 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+			 return FALSE;
+		return TRUE;
+	} else if (xdrs->x_op == XDR_DECODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->op))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		objp->op = IXDR_GET_LONG(buf);
+		objp->op_ret = IXDR_GET_LONG(buf);
+		objp->op_errno = IXDR_GET_LONG(buf);
+		}
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+		 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+			 return FALSE;
+	 return TRUE;
+	}
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_commit_req (XDR *xdrs, gd1_mgmt_v3_commit_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_commit_rsp (XDR *xdrs, gd1_mgmt_v3_commit_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+
+	if (xdrs->x_op == XDR_ENCODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->op))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		IXDR_PUT_LONG(buf, objp->op);
+		IXDR_PUT_LONG(buf, objp->op_ret);
+		IXDR_PUT_LONG(buf, objp->op_errno);
+		}
+		 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+			 return FALSE;
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+		return TRUE;
+	} else if (xdrs->x_op == XDR_DECODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->op))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		objp->op = IXDR_GET_LONG(buf);
+		objp->op_ret = IXDR_GET_LONG(buf);
+		objp->op_errno = IXDR_GET_LONG(buf);
+		}
+		 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+			 return FALSE;
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+	 return TRUE;
+	}
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_post_val_req (XDR *xdrs, gd1_mgmt_v3_post_val_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_post_val_rsp (XDR *xdrs, gd1_mgmt_v3_post_val_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+
+	if (xdrs->x_op == XDR_ENCODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->op))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		IXDR_PUT_LONG(buf, objp->op);
+		IXDR_PUT_LONG(buf, objp->op_ret);
+		IXDR_PUT_LONG(buf, objp->op_errno);
+		}
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+		 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+			 return FALSE;
+		return TRUE;
+	} else if (xdrs->x_op == XDR_DECODE) {
+		 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+			sizeof (u_char), (xdrproc_t) xdr_u_char))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_int (xdrs, &objp->op))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_ret))
+				 return FALSE;
+			 if (!xdr_int (xdrs, &objp->op_errno))
+				 return FALSE;
+
+		} else {
+		objp->op = IXDR_GET_LONG(buf);
+		objp->op_ret = IXDR_GET_LONG(buf);
+		objp->op_errno = IXDR_GET_LONG(buf);
+		}
+		 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+			 return FALSE;
+		 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+			 return FALSE;
+	 return TRUE;
+	}
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_string (xdrs, &objp->op_errstr, ~0))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_unlock_req (XDR *xdrs, gd1_mgmt_v3_unlock_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_vector (xdrs, (char *)objp->txn_id, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gd1_mgmt_v3_unlock_rsp (XDR *xdrs, gd1_mgmt_v3_unlock_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_vector (xdrs, (char *)objp->uuid, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_vector (xdrs, (char *)objp->txn_id, 16,
+		sizeof (u_char), (xdrproc_t) xdr_u_char))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	return TRUE;
+}
diff --git a/rpc/xdr/src/glusterd1-xdr.h b/rpc/xdr/src/glusterd1-xdr.h
index 14c41baff..b6be23d06 100644
--- a/rpc/xdr/src/glusterd1-xdr.h
+++ b/rpc/xdr/src/glusterd1-xdr.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
@@ -62,6 +53,7 @@ struct gd1_mgmt_probe_rsp {
 	int port;
 	int op_ret;
 	int op_errno;
+	char *op_errstr;
 };
 typedef struct gd1_mgmt_probe_rsp gd1_mgmt_probe_rsp;
 
@@ -210,6 +202,145 @@ struct gd1_mgmt_brick_op_rsp {
 };
 typedef struct gd1_mgmt_brick_op_rsp gd1_mgmt_brick_op_rsp;
 
+struct gd1_mgmt_v3_lock_req {
+	u_char uuid[16];
+	u_char txn_id[16];
+	int op;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gd1_mgmt_v3_lock_req gd1_mgmt_v3_lock_req;
+
+struct gd1_mgmt_v3_lock_rsp {
+	u_char uuid[16];
+	u_char txn_id[16];
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+	int op_ret;
+	int op_errno;
+};
+typedef struct gd1_mgmt_v3_lock_rsp gd1_mgmt_v3_lock_rsp;
+
+struct gd1_mgmt_v3_pre_val_req {
+	u_char uuid[16];
+	int op;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gd1_mgmt_v3_pre_val_req gd1_mgmt_v3_pre_val_req;
+
+struct gd1_mgmt_v3_pre_val_rsp {
+	u_char uuid[16];
+	int op;
+	int op_ret;
+	int op_errno;
+	char *op_errstr;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gd1_mgmt_v3_pre_val_rsp gd1_mgmt_v3_pre_val_rsp;
+
+struct gd1_mgmt_v3_brick_op_req {
+	u_char uuid[16];
+	int op;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gd1_mgmt_v3_brick_op_req gd1_mgmt_v3_brick_op_req;
+
+struct gd1_mgmt_v3_brick_op_rsp {
+	u_char uuid[16];
+	int op;
+	int op_ret;
+	int op_errno;
+	char *op_errstr;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gd1_mgmt_v3_brick_op_rsp gd1_mgmt_v3_brick_op_rsp;
+
+struct gd1_mgmt_v3_commit_req {
+	u_char uuid[16];
+	int op;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gd1_mgmt_v3_commit_req gd1_mgmt_v3_commit_req;
+
+struct gd1_mgmt_v3_commit_rsp {
+	u_char uuid[16];
+	int op;
+	int op_ret;
+	int op_errno;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+	char *op_errstr;
+};
+typedef struct gd1_mgmt_v3_commit_rsp gd1_mgmt_v3_commit_rsp;
+
+struct gd1_mgmt_v3_post_val_req {
+	u_char uuid[16];
+	int op;
+	int op_ret;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gd1_mgmt_v3_post_val_req gd1_mgmt_v3_post_val_req;
+
+struct gd1_mgmt_v3_post_val_rsp {
+	u_char uuid[16];
+	int op;
+	int op_ret;
+	int op_errno;
+	char *op_errstr;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gd1_mgmt_v3_post_val_rsp gd1_mgmt_v3_post_val_rsp;
+
+struct gd1_mgmt_v3_unlock_req {
+	u_char uuid[16];
+	u_char txn_id[16];
+	int op;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gd1_mgmt_v3_unlock_req gd1_mgmt_v3_unlock_req;
+
+struct gd1_mgmt_v3_unlock_rsp {
+	u_char uuid[16];
+	u_char txn_id[16];
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+	int op_ret;
+	int op_errno;
+};
+typedef struct gd1_mgmt_v3_unlock_rsp gd1_mgmt_v3_unlock_rsp;
+
 /* the xdr functions */
 
 #if defined(__STDC__) || defined(__cplusplus)
@@ -232,6 +363,18 @@ extern  bool_t xdr_gd1_mgmt_friend_update (XDR *, gd1_mgmt_friend_update*);
 extern  bool_t xdr_gd1_mgmt_friend_update_rsp (XDR *, gd1_mgmt_friend_update_rsp*);
 extern  bool_t xdr_gd1_mgmt_brick_op_req (XDR *, gd1_mgmt_brick_op_req*);
 extern  bool_t xdr_gd1_mgmt_brick_op_rsp (XDR *, gd1_mgmt_brick_op_rsp*);
+extern  bool_t xdr_gd1_mgmt_v3_lock_req (XDR *, gd1_mgmt_v3_lock_req*);
+extern  bool_t xdr_gd1_mgmt_v3_lock_rsp (XDR *, gd1_mgmt_v3_lock_rsp*);
+extern  bool_t xdr_gd1_mgmt_v3_pre_val_req (XDR *, gd1_mgmt_v3_pre_val_req*);
+extern  bool_t xdr_gd1_mgmt_v3_pre_val_rsp (XDR *, gd1_mgmt_v3_pre_val_rsp*);
+extern  bool_t xdr_gd1_mgmt_v3_brick_op_req (XDR *, gd1_mgmt_v3_brick_op_req*);
+extern  bool_t xdr_gd1_mgmt_v3_brick_op_rsp (XDR *, gd1_mgmt_v3_brick_op_rsp*);
+extern  bool_t xdr_gd1_mgmt_v3_commit_req (XDR *, gd1_mgmt_v3_commit_req*);
+extern  bool_t xdr_gd1_mgmt_v3_commit_rsp (XDR *, gd1_mgmt_v3_commit_rsp*);
+extern  bool_t xdr_gd1_mgmt_v3_post_val_req (XDR *, gd1_mgmt_v3_post_val_req*);
+extern  bool_t xdr_gd1_mgmt_v3_post_val_rsp (XDR *, gd1_mgmt_v3_post_val_rsp*);
+extern  bool_t xdr_gd1_mgmt_v3_unlock_req (XDR *, gd1_mgmt_v3_unlock_req*);
+extern  bool_t xdr_gd1_mgmt_v3_unlock_rsp (XDR *, gd1_mgmt_v3_unlock_rsp*);
 
 #else /* K&R C */
 extern bool_t xdr_glusterd_volume_status ();
@@ -253,6 +396,18 @@ extern bool_t xdr_gd1_mgmt_friend_update ();
 extern bool_t xdr_gd1_mgmt_friend_update_rsp ();
 extern bool_t xdr_gd1_mgmt_brick_op_req ();
 extern bool_t xdr_gd1_mgmt_brick_op_rsp ();
+extern bool_t xdr_gd1_mgmt_v3_lock_req ();
+extern bool_t xdr_gd1_mgmt_v3_lock_rsp ();
+extern bool_t xdr_gd1_mgmt_v3_pre_val_req ();
+extern bool_t xdr_gd1_mgmt_v3_pre_val_rsp ();
+extern bool_t xdr_gd1_mgmt_v3_brick_op_req ();
+extern bool_t xdr_gd1_mgmt_v3_brick_op_rsp ();
+extern bool_t xdr_gd1_mgmt_v3_commit_req ();
+extern bool_t xdr_gd1_mgmt_v3_commit_rsp ();
+extern bool_t xdr_gd1_mgmt_v3_post_val_req ();
+extern bool_t xdr_gd1_mgmt_v3_post_val_rsp ();
+extern bool_t xdr_gd1_mgmt_v3_unlock_req ();
+extern bool_t xdr_gd1_mgmt_v3_unlock_rsp ();
 
 #endif /* K&R C */
 
diff --git a/rpc/xdr/src/glusterd1-xdr.x b/rpc/xdr/src/glusterd1-xdr.x
index c30c71e02..f5c45c9e4 100644
--- a/rpc/xdr/src/glusterd1-xdr.x
+++ b/rpc/xdr/src/glusterd1-xdr.x
@@ -16,6 +16,7 @@
         int     port;
         int     op_ret;
         int     op_errno;
+        string op_errstr<>;
 }  ;
 
 struct gd1_mgmt_friend_req {
@@ -124,3 +125,94 @@ struct gd1_mgmt_brick_op_rsp {
         opaque  output<>;
         string  op_errstr<>;
 } ;
+
+struct gd1_mgmt_v3_lock_req {
+        unsigned char  uuid[16];
+        unsigned char  txn_id[16];
+        int            op;
+        opaque         dict<>;
+}  ;
+
+struct gd1_mgmt_v3_lock_rsp {
+        unsigned char  uuid[16];
+        unsigned char  txn_id[16];
+        opaque         dict<>;
+        int            op_ret;
+        int            op_errno;
+}  ;
+
+struct gd1_mgmt_v3_pre_val_req {
+        unsigned char  uuid[16];
+        int     op;
+        opaque  dict<>;
+}  ;
+
+struct gd1_mgmt_v3_pre_val_rsp {
+        unsigned char  uuid[16];
+        int     op;
+        int     op_ret;
+        int     op_errno;
+        string  op_errstr<>;
+        opaque  dict<>;
+}  ;
+
+struct gd1_mgmt_v3_brick_op_req {
+        unsigned char  uuid[16];
+        int     op;
+        opaque  dict<>;
+}  ;
+
+struct gd1_mgmt_v3_brick_op_rsp {
+        unsigned char  uuid[16];
+        int     op;
+        int     op_ret;
+        int     op_errno;
+        string  op_errstr<>;
+        opaque  dict<>;
+}  ;
+
+struct gd1_mgmt_v3_commit_req {
+        unsigned char  uuid[16];
+        int     op;
+        opaque  dict<>;
+}  ;
+
+struct gd1_mgmt_v3_commit_rsp {
+        unsigned char  uuid[16];
+        int     op;
+        int     op_ret;
+        int     op_errno;
+        opaque  dict<>;
+        string  op_errstr<>;
+}  ;
+
+struct gd1_mgmt_v3_post_val_req {
+        unsigned char  uuid[16];
+        int     op;
+        int     op_ret;
+        opaque  dict<>;
+}  ;
+
+struct gd1_mgmt_v3_post_val_rsp {
+        unsigned char  uuid[16];
+        int     op;
+        int     op_ret;
+        int     op_errno;
+        string  op_errstr<>;
+        opaque  dict<>;
+}  ;
+
+struct gd1_mgmt_v3_unlock_req {
+        unsigned char  uuid[16];
+        unsigned char  txn_id[16];
+        int            op;
+        opaque         dict<>;
+}  ;
+
+struct gd1_mgmt_v3_unlock_rsp {
+        unsigned char  uuid[16];
+        unsigned char  txn_id[16];
+        opaque         dict<>;
+        int            op_ret;
+        int            op_errno;
+}  ;
diff --git a/rpc/xdr/src/glusterfs3-xdr.c b/rpc/xdr/src/glusterfs3-xdr.c
index 75ef044a0..3205c551e 100644
--- a/rpc/xdr/src/glusterfs3-xdr.c
+++ b/rpc/xdr/src/glusterfs3-xdr.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
@@ -317,9 +308,9 @@ xdr_gfs3_mknod_req (XDR *xdrs, gfs3_mknod_req *objp)
 		 return FALSE;
 	 if (!xdr_u_int (xdrs, &objp->mode))
 		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->bname, ~0))
+	 if (!xdr_u_int (xdrs, &objp->umask))
 		 return FALSE;
-	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+	 if (!xdr_string (xdrs, &objp->bname, ~0))
 		 return FALSE;
 	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
 		 return FALSE;
@@ -357,9 +348,9 @@ xdr_gfs3_mkdir_req (XDR *xdrs, gfs3_mkdir_req *objp)
 		 return FALSE;
 	 if (!xdr_u_int (xdrs, &objp->mode))
 		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->bname, ~0))
+	 if (!xdr_u_int (xdrs, &objp->umask))
 		 return FALSE;
-	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+	 if (!xdr_string (xdrs, &objp->bname, ~0))
 		 return FALSE;
 	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
 		 return FALSE;
@@ -397,6 +388,8 @@ xdr_gfs3_unlink_req (XDR *xdrs, gfs3_unlink_req *objp)
 		 return FALSE;
 	 if (!xdr_string (xdrs, &objp->bname, ~0))
 		 return FALSE;
+	 if (!xdr_u_int (xdrs, &objp->xflags))
+		 return FALSE;
 	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
 		 return FALSE;
 	return TRUE;
@@ -429,7 +422,7 @@ xdr_gfs3_rmdir_req (XDR *xdrs, gfs3_rmdir_req *objp)
 
 	 if (!xdr_opaque (xdrs, objp->pargfid, 16))
 		 return FALSE;
-	 if (!xdr_int (xdrs, &objp->flags))
+	 if (!xdr_int (xdrs, &objp->xflags))
 		 return FALSE;
 	 if (!xdr_string (xdrs, &objp->bname, ~0))
 		 return FALSE;
@@ -467,9 +460,9 @@ xdr_gfs3_symlink_req (XDR *xdrs, gfs3_symlink_req *objp)
 		 return FALSE;
 	 if (!xdr_string (xdrs, &objp->bname, ~0))
 		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->linkname, ~0))
+	 if (!xdr_u_int (xdrs, &objp->umask))
 		 return FALSE;
-	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+	 if (!xdr_string (xdrs, &objp->linkname, ~0))
 		 return FALSE;
 	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
 		 return FALSE;
@@ -507,12 +500,8 @@ xdr_gfs3_rename_req (XDR *xdrs, gfs3_rename_req *objp)
 		 return FALSE;
 	 if (!xdr_opaque (xdrs, objp->newgfid, 16))
 		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->oldpath, ~0))
-		 return FALSE;
 	 if (!xdr_string (xdrs, &objp->oldbname, ~0))
 		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->newpath, ~0))
-		 return FALSE;
 	 if (!xdr_string (xdrs, &objp->newbname, ~0))
 		 return FALSE;
 	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
@@ -555,10 +544,6 @@ xdr_gfs3_link_req (XDR *xdrs, gfs3_link_req *objp)
 		 return FALSE;
 	 if (!xdr_opaque (xdrs, objp->newgfid, 16))
 		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->oldpath, ~0))
-		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->newpath, ~0))
-		 return FALSE;
 	 if (!xdr_string (xdrs, &objp->newbname, ~0))
 		 return FALSE;
 	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
@@ -631,8 +616,6 @@ xdr_gfs3_open_req (XDR *xdrs, gfs3_open_req *objp)
 		 return FALSE;
 	 if (!xdr_u_int (xdrs, &objp->flags))
 		 return FALSE;
-	 if (!xdr_u_int (xdrs, &objp->wbflags))
-		 return FALSE;
 	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
 		 return FALSE;
 	return TRUE;
@@ -709,8 +692,6 @@ xdr_gfs3_lookup_req (XDR *xdrs, gfs3_lookup_req *objp)
 		 return FALSE;
 	 if (!xdr_string (xdrs, &objp->bname, ~0))
 		 return FALSE;
-	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
-		 return FALSE;
 	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
 		 return FALSE;
 	return TRUE;
@@ -730,8 +711,6 @@ xdr_gfs3_lookup_rsp (XDR *xdrs, gfs3_lookup_rsp *objp)
 		 return FALSE;
 	 if (!xdr_gf_iatt (xdrs, &objp->postparent))
 		 return FALSE;
-	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
-		 return FALSE;
 	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
 		 return FALSE;
 	return TRUE;
@@ -1234,32 +1213,6 @@ xdr_gfs3_readdirp_req (XDR *xdrs, gfs3_readdirp_req *objp)
 }
 
 bool_t
-xdr_gf_setvolume_req (XDR *xdrs, gf_setvolume_req *objp)
-{
-	register int32_t *buf;
-        buf = NULL;
-
-	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
-		 return FALSE;
-	return TRUE;
-}
-
-bool_t
-xdr_gf_setvolume_rsp (XDR *xdrs, gf_setvolume_rsp *objp)
-{
-	register int32_t *buf;
-        buf = NULL;
-
-	 if (!xdr_int (xdrs, &objp->op_ret))
-		 return FALSE;
-	 if (!xdr_int (xdrs, &objp->op_errno))
-		 return FALSE;
-	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
-		 return FALSE;
-	return TRUE;
-}
-
-bool_t
 xdr_gfs3_access_req (XDR *xdrs, gfs3_access_req *objp)
 {
 	register int32_t *buf;
@@ -1280,15 +1233,62 @@ xdr_gfs3_create_req (XDR *xdrs, gfs3_create_req *objp)
 	register int32_t *buf;
         buf = NULL;
 
+
+	if (xdrs->x_op == XDR_ENCODE) {
+		 if (!xdr_opaque (xdrs, objp->pargfid, 16))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_u_int (xdrs, &objp->flags))
+				 return FALSE;
+			 if (!xdr_u_int (xdrs, &objp->mode))
+				 return FALSE;
+			 if (!xdr_u_int (xdrs, &objp->umask))
+				 return FALSE;
+
+		} else {
+		IXDR_PUT_U_LONG(buf, objp->flags);
+		IXDR_PUT_U_LONG(buf, objp->mode);
+		IXDR_PUT_U_LONG(buf, objp->umask);
+		}
+		 if (!xdr_string (xdrs, &objp->bname, ~0))
+			 return FALSE;
+		 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
+			 return FALSE;
+		return TRUE;
+	} else if (xdrs->x_op == XDR_DECODE) {
+		 if (!xdr_opaque (xdrs, objp->pargfid, 16))
+			 return FALSE;
+		buf = XDR_INLINE (xdrs, 3 * BYTES_PER_XDR_UNIT);
+		if (buf == NULL) {
+			 if (!xdr_u_int (xdrs, &objp->flags))
+				 return FALSE;
+			 if (!xdr_u_int (xdrs, &objp->mode))
+				 return FALSE;
+			 if (!xdr_u_int (xdrs, &objp->umask))
+				 return FALSE;
+
+		} else {
+		objp->flags = IXDR_GET_U_LONG(buf);
+		objp->mode = IXDR_GET_U_LONG(buf);
+		objp->umask = IXDR_GET_U_LONG(buf);
+		}
+		 if (!xdr_string (xdrs, &objp->bname, ~0))
+			 return FALSE;
+		 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
+			 return FALSE;
+	 return TRUE;
+	}
+
 	 if (!xdr_opaque (xdrs, objp->pargfid, 16))
 		 return FALSE;
 	 if (!xdr_u_int (xdrs, &objp->flags))
 		 return FALSE;
 	 if (!xdr_u_int (xdrs, &objp->mode))
 		 return FALSE;
-	 if (!xdr_string (xdrs, &objp->bname, ~0))
+	 if (!xdr_u_int (xdrs, &objp->umask))
 		 return FALSE;
-	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+	 if (!xdr_string (xdrs, &objp->bname, ~0))
 		 return FALSE;
 	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
 		 return FALSE;
@@ -1507,6 +1507,125 @@ xdr_gfs3_fsetattr_rsp (XDR *xdrs, gfs3_fsetattr_rsp *objp)
 }
 
 bool_t
+xdr_gfs3_fallocate_req (XDR *xdrs, gfs3_fallocate_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_opaque (xdrs, objp->gfid, 16))
+		 return FALSE;
+	 if (!xdr_quad_t (xdrs, &objp->fd))
+		 return FALSE;
+	 if (!xdr_u_int (xdrs, &objp->flags))
+		 return FALSE;
+	 if (!xdr_u_quad_t (xdrs, &objp->offset))
+		 return FALSE;
+	 if (!xdr_u_quad_t (xdrs, &objp->size))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gfs3_fallocate_rsp (XDR *xdrs, gfs3_fallocate_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_gf_iatt (xdrs, &objp->statpre))
+		 return FALSE;
+	 if (!xdr_gf_iatt (xdrs, &objp->statpost))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gfs3_discard_req (XDR *xdrs, gfs3_discard_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_opaque (xdrs, objp->gfid, 16))
+		 return FALSE;
+	 if (!xdr_quad_t (xdrs, &objp->fd))
+		 return FALSE;
+	 if (!xdr_u_quad_t (xdrs, &objp->offset))
+		 return FALSE;
+	 if (!xdr_u_quad_t (xdrs, &objp->size))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gfs3_discard_rsp (XDR *xdrs, gfs3_discard_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_gf_iatt (xdrs, &objp->statpre))
+		 return FALSE;
+	 if (!xdr_gf_iatt (xdrs, &objp->statpost))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gfs3_zerofill_req (XDR *xdrs, gfs3_zerofill_req *objp)
+{
+        register int32_t *buf;
+        buf = NULL;
+
+        if (!xdr_opaque (xdrs, objp->gfid, 16))
+                return FALSE;
+        if (!xdr_quad_t (xdrs, &objp->fd))
+                return FALSE;
+        if (!xdr_u_quad_t (xdrs, &objp->offset))
+                return FALSE;
+        if (!xdr_u_quad_t (xdrs, &objp->size))
+                return FALSE;
+        if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val,
+             (u_int *) &objp->xdata.xdata_len, ~0))
+                return FALSE;
+        return TRUE;
+}
+
+bool_t
+xdr_gfs3_zerofill_rsp (XDR *xdrs, gfs3_zerofill_rsp *objp)
+{
+        register int32_t *buf;
+        buf = NULL;
+
+        if (!xdr_int (xdrs, &objp->op_ret))
+                return FALSE;
+        if (!xdr_int (xdrs, &objp->op_errno))
+                return FALSE;
+        if (!xdr_gf_iatt (xdrs, &objp->statpre))
+                return FALSE;
+        if (!xdr_gf_iatt (xdrs, &objp->statpost))
+                return FALSE;
+        if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val,
+             (u_int *) &objp->xdata.xdata_len, ~0))
+                return FALSE;
+        return TRUE;
+}
+
+
+bool_t
 xdr_gfs3_rchecksum_req (XDR *xdrs, gfs3_rchecksum_req *objp)
 {
 	register int32_t *buf;
@@ -1586,6 +1705,32 @@ xdr_gfs3_rchecksum_rsp (XDR *xdrs, gfs3_rchecksum_rsp *objp)
 }
 
 bool_t
+xdr_gf_setvolume_req (XDR *xdrs, gf_setvolume_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gf_setvolume_rsp (XDR *xdrs, gf_setvolume_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
 xdr_gf_getspec_req (XDR *xdrs, gf_getspec_req *objp)
 {
 	register int32_t *buf;
@@ -1618,6 +1763,32 @@ xdr_gf_getspec_rsp (XDR *xdrs, gf_getspec_rsp *objp)
 }
 
 bool_t
+xdr_gf_mgmt_hndsk_req (XDR *xdrs, gf_mgmt_hndsk_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_bytes (xdrs, (char **)&objp->hndsk.hndsk_val, (u_int *) &objp->hndsk.hndsk_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gf_mgmt_hndsk_rsp (XDR *xdrs, gf_mgmt_hndsk_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->hndsk.hndsk_val, (u_int *) &objp->hndsk.hndsk_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
 xdr_gf_log_req (XDR *xdrs, gf_log_req *objp)
 {
 	register int32_t *buf;
@@ -1829,3 +2000,59 @@ xdr_gfs3_readdirp_rsp (XDR *xdrs, gfs3_readdirp_rsp *objp)
 		 return FALSE;
 	return TRUE;
 }
+
+bool_t
+xdr_gf_set_lk_ver_rsp (XDR *xdrs, gf_set_lk_ver_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->lk_ver))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gf_set_lk_ver_req (XDR *xdrs, gf_set_lk_ver_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_string (xdrs, &objp->uid, ~0))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->lk_ver))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gf_event_notify_req (XDR *xdrs, gf_event_notify_req *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_int (xdrs, &objp->op))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_gf_event_notify_rsp (XDR *xdrs, gf_event_notify_rsp *objp)
+{
+	register int32_t *buf;
+        buf = NULL;
+
+	 if (!xdr_int (xdrs, &objp->op_ret))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->op_errno))
+		 return FALSE;
+	 if (!xdr_bytes (xdrs, (char **)&objp->dict.dict_val, (u_int *) &objp->dict.dict_len, ~0))
+		 return FALSE;
+	return TRUE;
+}
diff --git a/rpc/xdr/src/glusterfs3-xdr.h b/rpc/xdr/src/glusterfs3-xdr.h
index 17628056a..13566e694 100644
--- a/rpc/xdr/src/glusterfs3-xdr.h
+++ b/rpc/xdr/src/glusterfs3-xdr.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
@@ -137,12 +128,9 @@ struct gfs3_mknod_req {
 	char pargfid[16];
 	u_quad_t dev;
 	u_int mode;
+	u_int umask;
 	char *bname;
 	struct {
-		u_int dict_len;
-		char *dict_val;
-	} dict;
-	struct {
 		u_int xdata_len;
 		char *xdata_val;
 	} xdata;
@@ -165,12 +153,9 @@ typedef struct gfs3_mknod_rsp gfs3_mknod_rsp;
 struct gfs3_mkdir_req {
 	char pargfid[16];
 	u_int mode;
+	u_int umask;
 	char *bname;
 	struct {
-		u_int dict_len;
-		char *dict_val;
-	} dict;
-	struct {
 		u_int xdata_len;
 		char *xdata_val;
 	} xdata;
@@ -193,6 +178,7 @@ typedef struct gfs3_mkdir_rsp gfs3_mkdir_rsp;
 struct gfs3_unlink_req {
 	char pargfid[16];
 	char *bname;
+	u_int xflags;
 	struct {
 		u_int xdata_len;
 		char *xdata_val;
@@ -214,7 +200,7 @@ typedef struct gfs3_unlink_rsp gfs3_unlink_rsp;
 
 struct gfs3_rmdir_req {
 	char pargfid[16];
-	int flags;
+	int xflags;
 	char *bname;
 	struct {
 		u_int xdata_len;
@@ -238,12 +224,9 @@ typedef struct gfs3_rmdir_rsp gfs3_rmdir_rsp;
 struct gfs3_symlink_req {
 	char pargfid[16];
 	char *bname;
+	u_int umask;
 	char *linkname;
 	struct {
-		u_int dict_len;
-		char *dict_val;
-	} dict;
-	struct {
 		u_int xdata_len;
 		char *xdata_val;
 	} xdata;
@@ -266,9 +249,7 @@ typedef struct gfs3_symlink_rsp gfs3_symlink_rsp;
 struct gfs3_rename_req {
 	char oldgfid[16];
 	char newgfid[16];
-	char *oldpath;
 	char *oldbname;
-	char *newpath;
 	char *newbname;
 	struct {
 		u_int xdata_len;
@@ -295,8 +276,6 @@ typedef struct gfs3_rename_rsp gfs3_rename_rsp;
 struct gfs3_link_req {
 	char oldgfid[16];
 	char newgfid[16];
-	char *oldpath;
-	char *newpath;
 	char *newbname;
 	struct {
 		u_int xdata_len;
@@ -343,7 +322,6 @@ typedef struct gfs3_truncate_rsp gfs3_truncate_rsp;
 struct gfs3_open_req {
 	char gfid[16];
 	u_int flags;
-	u_int wbflags;
 	struct {
 		u_int xdata_len;
 		char *xdata_val;
@@ -393,10 +371,6 @@ struct gfs3_lookup_req {
 	u_int flags;
 	char *bname;
 	struct {
-		u_int dict_len;
-		char *dict_val;
-	} dict;
-	struct {
 		u_int xdata_len;
 		char *xdata_val;
 	} xdata;
@@ -409,10 +383,6 @@ struct gfs3_lookup_rsp {
 	struct gf_iatt stat;
 	struct gf_iatt postparent;
 	struct {
-		u_int dict_len;
-		char *dict_val;
-	} dict;
-	struct {
 		u_int xdata_len;
 		char *xdata_val;
 	} xdata;
@@ -761,24 +731,6 @@ struct gfs3_readdirp_req {
 };
 typedef struct gfs3_readdirp_req gfs3_readdirp_req;
 
-struct gf_setvolume_req {
-	struct {
-		u_int dict_len;
-		char *dict_val;
-	} dict;
-};
-typedef struct gf_setvolume_req gf_setvolume_req;
-
-struct gf_setvolume_rsp {
-	int op_ret;
-	int op_errno;
-	struct {
-		u_int dict_len;
-		char *dict_val;
-	} dict;
-};
-typedef struct gf_setvolume_rsp gf_setvolume_rsp;
-
 struct gfs3_access_req {
 	char gfid[16];
 	u_int mask;
@@ -793,12 +745,9 @@ struct gfs3_create_req {
 	char pargfid[16];
 	u_int flags;
 	u_int mode;
+	u_int umask;
 	char *bname;
 	struct {
-		u_int dict_len;
-		char *dict_val;
-	} dict;
-	struct {
 		u_int xdata_len;
 		char *xdata_val;
 	} xdata;
@@ -938,6 +887,80 @@ struct gfs3_fsetattr_rsp {
 };
 typedef struct gfs3_fsetattr_rsp gfs3_fsetattr_rsp;
 
+struct gfs3_fallocate_req {
+	char gfid[16];
+	quad_t fd;
+	u_int flags;
+	u_quad_t offset;
+	u_quad_t size;
+	struct {
+		u_int xdata_len;
+		char *xdata_val;
+	} xdata;
+};
+typedef struct gfs3_fallocate_req gfs3_fallocate_req;
+
+struct gfs3_fallocate_rsp {
+	int op_ret;
+	int op_errno;
+	struct gf_iatt statpre;
+	struct gf_iatt statpost;
+	struct {
+		u_int xdata_len;
+		char *xdata_val;
+	} xdata;
+};
+typedef struct gfs3_fallocate_rsp gfs3_fallocate_rsp;
+
+struct gfs3_discard_req {
+	char gfid[16];
+	quad_t fd;
+	u_quad_t offset;
+	u_quad_t size;
+	struct {
+		u_int xdata_len;
+		char *xdata_val;
+	} xdata;
+};
+typedef struct gfs3_discard_req gfs3_discard_req;
+
+struct gfs3_discard_rsp {
+	int op_ret;
+	int op_errno;
+	struct gf_iatt statpre;
+	struct gf_iatt statpost;
+	struct {
+		u_int xdata_len;
+		char *xdata_val;
+	} xdata;
+};
+typedef struct gfs3_discard_rsp gfs3_discard_rsp;
+
+struct gfs3_zerofill_req {
+        char gfid[16];
+        quad_t fd;
+        u_quad_t offset;
+        u_quad_t size;
+        struct {
+                u_int xdata_len;
+                char *xdata_val;
+        } xdata;
+};
+typedef struct gfs3_zerofill_req gfs3_zerofill_req;
+
+struct gfs3_zerofill_rsp {
+        int op_ret;
+        int op_errno;
+        struct gf_iatt statpre;
+        struct gf_iatt statpost;
+        struct {
+                u_int xdata_len;
+                char *xdata_val;
+        } xdata;
+};
+typedef struct gfs3_zerofill_rsp gfs3_zerofill_rsp;
+
+
 struct gfs3_rchecksum_req {
 	quad_t fd;
 	u_quad_t offset;
@@ -964,6 +987,24 @@ struct gfs3_rchecksum_rsp {
 };
 typedef struct gfs3_rchecksum_rsp gfs3_rchecksum_rsp;
 
+struct gf_setvolume_req {
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gf_setvolume_req gf_setvolume_req;
+
+struct gf_setvolume_rsp {
+	int op_ret;
+	int op_errno;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gf_setvolume_rsp gf_setvolume_rsp;
+
 struct gf_getspec_req {
 	u_int flags;
 	char *key;
@@ -985,6 +1026,24 @@ struct gf_getspec_rsp {
 };
 typedef struct gf_getspec_rsp gf_getspec_rsp;
 
+struct gf_mgmt_hndsk_req {
+	struct {
+		u_int hndsk_len;
+		char *hndsk_val;
+	} hndsk;
+};
+typedef struct gf_mgmt_hndsk_req gf_mgmt_hndsk_req;
+
+struct gf_mgmt_hndsk_rsp {
+	int op_ret;
+	int op_errno;
+	struct {
+		u_int hndsk_len;
+		char *hndsk_val;
+	} hndsk;
+};
+typedef struct gf_mgmt_hndsk_rsp gf_mgmt_hndsk_rsp;
+
 struct gf_log_req {
 	struct {
 		u_int msg_len;
@@ -1092,6 +1151,38 @@ struct gfs3_readdirp_rsp {
 };
 typedef struct gfs3_readdirp_rsp gfs3_readdirp_rsp;
 
+struct gf_set_lk_ver_rsp {
+	int op_ret;
+	int op_errno;
+	int lk_ver;
+};
+typedef struct gf_set_lk_ver_rsp gf_set_lk_ver_rsp;
+
+struct gf_set_lk_ver_req {
+	char *uid;
+	int lk_ver;
+};
+typedef struct gf_set_lk_ver_req gf_set_lk_ver_req;
+
+struct gf_event_notify_req {
+	int op;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gf_event_notify_req gf_event_notify_req;
+
+struct gf_event_notify_rsp {
+	int op_ret;
+	int op_errno;
+	struct {
+		u_int dict_len;
+		char *dict_val;
+	} dict;
+};
+typedef struct gf_event_notify_rsp gf_event_notify_rsp;
+
 /* the xdr functions */
 
 #if defined(__STDC__) || defined(__cplusplus)
@@ -1152,8 +1243,6 @@ extern  bool_t xdr_gfs3_opendir_rsp (XDR *, gfs3_opendir_rsp*);
 extern  bool_t xdr_gfs3_fsyncdir_req (XDR *, gfs3_fsyncdir_req*);
 extern  bool_t xdr_gfs3_readdir_req (XDR *, gfs3_readdir_req*);
 extern  bool_t xdr_gfs3_readdirp_req (XDR *, gfs3_readdirp_req*);
-extern  bool_t xdr_gf_setvolume_req (XDR *, gf_setvolume_req*);
-extern  bool_t xdr_gf_setvolume_rsp (XDR *, gf_setvolume_rsp*);
 extern  bool_t xdr_gfs3_access_req (XDR *, gfs3_access_req*);
 extern  bool_t xdr_gfs3_create_req (XDR *, gfs3_create_req*);
 extern  bool_t xdr_gfs3_create_rsp (XDR *, gfs3_create_rsp*);
@@ -1167,10 +1256,20 @@ extern  bool_t xdr_gfs3_setattr_req (XDR *, gfs3_setattr_req*);
 extern  bool_t xdr_gfs3_setattr_rsp (XDR *, gfs3_setattr_rsp*);
 extern  bool_t xdr_gfs3_fsetattr_req (XDR *, gfs3_fsetattr_req*);
 extern  bool_t xdr_gfs3_fsetattr_rsp (XDR *, gfs3_fsetattr_rsp*);
+extern  bool_t xdr_gfs3_fallocate_req (XDR *, gfs3_fallocate_req*);
+extern  bool_t xdr_gfs3_fallocate_rsp (XDR *, gfs3_fallocate_rsp*);
+extern  bool_t xdr_gfs3_discard_req (XDR *, gfs3_discard_req*);
+extern  bool_t xdr_gfs3_discard_rsp (XDR *, gfs3_discard_rsp*);
+extern  bool_t xdr_gfs3_zerofill_req (XDR *, gfs3_zerofill_req*);
+extern  bool_t xdr_gfs3_zerofill_rsp (XDR *, gfs3_zerofill_rsp*);
 extern  bool_t xdr_gfs3_rchecksum_req (XDR *, gfs3_rchecksum_req*);
 extern  bool_t xdr_gfs3_rchecksum_rsp (XDR *, gfs3_rchecksum_rsp*);
+extern  bool_t xdr_gf_setvolume_req (XDR *, gf_setvolume_req*);
+extern  bool_t xdr_gf_setvolume_rsp (XDR *, gf_setvolume_rsp*);
 extern  bool_t xdr_gf_getspec_req (XDR *, gf_getspec_req*);
 extern  bool_t xdr_gf_getspec_rsp (XDR *, gf_getspec_rsp*);
+extern  bool_t xdr_gf_mgmt_hndsk_req (XDR *, gf_mgmt_hndsk_req*);
+extern  bool_t xdr_gf_mgmt_hndsk_rsp (XDR *, gf_mgmt_hndsk_rsp*);
 extern  bool_t xdr_gf_log_req (XDR *, gf_log_req*);
 extern  bool_t xdr_gf_notify_req (XDR *, gf_notify_req*);
 extern  bool_t xdr_gf_notify_rsp (XDR *, gf_notify_rsp*);
@@ -1181,6 +1280,10 @@ extern  bool_t xdr_gfs3_dirlist (XDR *, gfs3_dirlist*);
 extern  bool_t xdr_gfs3_readdir_rsp (XDR *, gfs3_readdir_rsp*);
 extern  bool_t xdr_gfs3_dirplist (XDR *, gfs3_dirplist*);
 extern  bool_t xdr_gfs3_readdirp_rsp (XDR *, gfs3_readdirp_rsp*);
+extern  bool_t xdr_gf_set_lk_ver_rsp (XDR *, gf_set_lk_ver_rsp*);
+extern  bool_t xdr_gf_set_lk_ver_req (XDR *, gf_set_lk_ver_req*);
+extern  bool_t xdr_gf_event_notify_req (XDR *, gf_event_notify_req*);
+extern  bool_t xdr_gf_event_notify_rsp (XDR *, gf_event_notify_rsp*);
 
 #else /* K&R C */
 extern bool_t xdr_gf_statfs ();
@@ -1240,8 +1343,6 @@ extern bool_t xdr_gfs3_opendir_rsp ();
 extern bool_t xdr_gfs3_fsyncdir_req ();
 extern bool_t xdr_gfs3_readdir_req ();
 extern bool_t xdr_gfs3_readdirp_req ();
-extern bool_t xdr_gf_setvolume_req ();
-extern bool_t xdr_gf_setvolume_rsp ();
 extern bool_t xdr_gfs3_access_req ();
 extern bool_t xdr_gfs3_create_req ();
 extern bool_t xdr_gfs3_create_rsp ();
@@ -1255,10 +1356,20 @@ extern bool_t xdr_gfs3_setattr_req ();
 extern bool_t xdr_gfs3_setattr_rsp ();
 extern bool_t xdr_gfs3_fsetattr_req ();
 extern bool_t xdr_gfs3_fsetattr_rsp ();
+extern bool_t xdr_gfs3_fallocate_req ();
+extern bool_t xdr_gfs3_fallocate_rsp ();
+extern bool_t xdr_gfs3_discard_req ();
+extern bool_t xdr_gfs3_discard_rsp ();
+extern bool_t xdr_gfs3_zerofill_req ();
+extern bool_t xdr_gfs3_zerofill_rsp ();
 extern bool_t xdr_gfs3_rchecksum_req ();
 extern bool_t xdr_gfs3_rchecksum_rsp ();
+extern bool_t xdr_gf_setvolume_req ();
+extern bool_t xdr_gf_setvolume_rsp ();
 extern bool_t xdr_gf_getspec_req ();
 extern bool_t xdr_gf_getspec_rsp ();
+extern bool_t xdr_gf_mgmt_hndsk_req ();
+extern bool_t xdr_gf_mgmt_hndsk_rsp ();
 extern bool_t xdr_gf_log_req ();
 extern bool_t xdr_gf_notify_req ();
 extern bool_t xdr_gf_notify_rsp ();
@@ -1269,6 +1380,10 @@ extern bool_t xdr_gfs3_dirlist ();
 extern bool_t xdr_gfs3_readdir_rsp ();
 extern bool_t xdr_gfs3_dirplist ();
 extern bool_t xdr_gfs3_readdirp_rsp ();
+extern bool_t xdr_gf_set_lk_ver_rsp ();
+extern bool_t xdr_gf_set_lk_ver_req ();
+extern bool_t xdr_gf_event_notify_req ();
+extern bool_t xdr_gf_event_notify_rsp ();
 
 #endif /* K&R C */
 
diff --git a/rpc/xdr/src/glusterfs3-xdr.x b/rpc/xdr/src/glusterfs3-xdr.x
index 651190c19..1edbda3ad 100644
--- a/rpc/xdr/src/glusterfs3-xdr.x
+++ b/rpc/xdr/src/glusterfs3-xdr.x
@@ -73,8 +73,8 @@ struct gfs3_readlink_req {
         opaque  pargfid[16];
 	unsigned hyper dev;
 	unsigned int mode;
+        unsigned int umask;
 	string     bname<>; /* NULL terminated */
-        opaque     dict<>;
         opaque   xdata<>; /* Extra data */
 } ;
  struct gfs3_mknod_rsp {
@@ -90,9 +90,9 @@ struct gfs3_readlink_req {
  struct  gfs3_mkdir_req {
         opaque  pargfid[16];
 	unsigned int mode;
+        unsigned int umask;
 	string     bname<>; /* NULL terminated */
-        opaque     dict<>;
-        opaque   xdata<>; /* Extra data */
+        opaque     xdata<>; /* Extra data */
 } ;
  struct  gfs3_mkdir_rsp {
         int    op_ret;
@@ -107,6 +107,7 @@ struct gfs3_readlink_req {
  struct   gfs3_unlink_req {
         opaque  pargfid[16];
 	string     bname<>; /* NULL terminated */
+        unsigned int xflags;
         opaque   xdata<>; /* Extra data */
 };
  struct   gfs3_unlink_rsp {
@@ -120,9 +121,9 @@ struct gfs3_readlink_req {
 
  struct   gfs3_rmdir_req {
         opaque  pargfid[16];
-        int        flags;
+        int        xflags;
 	string     bname<>; /* NULL terminated */
-        opaque   xdata<>; /* Extra data */
+        opaque     xdata<>; /* Extra data */
 };
  struct   gfs3_rmdir_rsp {
         int    op_ret;
@@ -136,8 +137,8 @@ struct gfs3_readlink_req {
  struct   gfs3_symlink_req {
         opaque  pargfid[16];
 	string     bname<>;
+        unsigned int umask;
 	string     linkname<>;
-        opaque     dict<>;
         opaque   xdata<>; /* Extra data */
 };
  struct  gfs3_symlink_rsp {
@@ -153,9 +154,7 @@ struct gfs3_readlink_req {
  struct   gfs3_rename_req {
         opaque  oldgfid[16];
         opaque  newgfid[16];
-	string       oldpath<>;
 	string       oldbname<>; /* NULL terminated */
-	string       newpath<>;
 	string       newbname<>; /* NULL terminated */
         opaque   xdata<>; /* Extra data */
 };
@@ -174,8 +173,6 @@ struct gfs3_readlink_req {
  struct  gfs3_link_req {
         opaque  oldgfid[16];
         opaque  newgfid[16];
-	string       oldpath<>;
-	string       newpath<>;
 	string       newbname<>;
         opaque   xdata<>; /* Extra data */
 };
@@ -205,7 +202,6 @@ struct gfs3_readlink_req {
  struct   gfs3_open_req {
         opaque gfid[16];
 	unsigned int flags;
-        unsigned int wbflags;
         opaque   xdata<>; /* Extra data */
 };
  struct   gfs3_open_rsp {
@@ -237,7 +233,6 @@ struct   gfs3_lookup_req {
         opaque  pargfid[16];
 	unsigned int flags;
 	string     bname<>;
-        opaque     dict<>;
         opaque   xdata<>; /* Extra data */
 };
  struct   gfs3_lookup_rsp {
@@ -245,7 +240,6 @@ struct   gfs3_lookup_req {
         int    op_errno;
 	struct gf_iatt stat;
         struct gf_iatt postparent;
-	opaque             dict<>;
         opaque   xdata<>; /* Extra data */
 } ;
 
@@ -466,15 +460,6 @@ struct   gfs3_finodelk_req {
 }  ;
 
 
- struct gf_setvolume_req {
-        opaque dict<>;
-}  ;
- struct  gf_setvolume_rsp {
-        int    op_ret;
-        int    op_errno;
-        opaque dict<>;
-} ;
-
 struct gfs3_access_req  {
         opaque gfid[16];
 	unsigned int mask;
@@ -486,8 +471,8 @@ struct gfs3_create_req {
         opaque  pargfid[16];
 	unsigned int flags;
 	unsigned int mode;
+	unsigned int umask;
 	string     bname<>;
-        opaque     dict<>;
         opaque   xdata<>; /* Extra data */
 }  ;
 struct  gfs3_create_rsp {
@@ -581,6 +566,56 @@ struct gfs3_fstat_req {
         opaque   xdata<>; /* Extra data */
 }  ;
 
+ struct gfs3_fallocate_req {
+	opaque		gfid[16];
+        hyper		fd;
+	unsigned int	flags;
+	unsigned hyper	offset;
+	unsigned hyper	size;
+        opaque   xdata<>; /* Extra data */
+}  ;
+
+ struct gfs3_fallocate_rsp {
+        int    op_ret;
+        int    op_errno;
+        struct gf_iatt statpre;
+        struct gf_iatt statpost;
+        opaque   xdata<>; /* Extra data */
+}  ;
+
+ struct gfs3_discard_req {
+	opaque		gfid[16];
+        hyper		fd;
+	unsigned hyper	offset;
+	unsigned hyper	size;
+        opaque   xdata<>; /* Extra data */
+}  ;
+
+ struct gfs3_discard_rsp {
+        int    op_ret;
+        int    op_errno;
+        struct gf_iatt statpre;
+        struct gf_iatt statpost;
+        opaque   xdata<>; /* Extra data */
+}  ;
+
+ struct gfs3_zerofill_req {
+        opaque          gfid[16];
+        hyper           fd;
+        unsigned hyper  offset;
+        unsigned hyper  size;
+        opaque   xdata<>;
+}  ;
+
+ struct gfs3_zerofill_rsp {
+        int    op_ret;
+        int    op_errno;
+        struct gf_iatt statpre;
+        struct gf_iatt statpost;
+        opaque   xdata<>;
+}  ;
+
+
  struct gfs3_rchecksum_req {
         hyper   fd;
         unsigned hyper  offset;
@@ -596,6 +631,16 @@ struct gfs3_fstat_req {
 }  ;
 
 
+ struct gf_setvolume_req {
+        opaque dict<>;
+}  ;
+ struct  gf_setvolume_rsp {
+        int    op_ret;
+        int    op_errno;
+        opaque dict<>;
+} ;
+
+
  struct gf_getspec_req {
 	unsigned int flags;
 	string     key<>;
@@ -608,10 +653,19 @@ struct gfs3_fstat_req {
         opaque   xdata<>; /* Extra data */
 } ;
 
+ struct gf_mgmt_hndsk_req {
+        opaque   hndsk<>;
+}  ;
+
+ struct  gf_mgmt_hndsk_rsp {
+        int    op_ret;
+        int    op_errno;
+        opaque   hndsk<>;
+} ;
 
  struct   gf_log_req {
-	opaque     msg<>;
-};
+        opaque    msg<>;
+} ;
 
  struct gf_notify_req {
 	unsigned int  flags;
@@ -679,3 +733,24 @@ struct gfs3_readdirp_rsp {
         opaque   xdata<>; /* Extra data */
 };
 
+struct gf_set_lk_ver_rsp {
+       int op_ret;
+       int op_errno;
+       int lk_ver;
+};
+
+struct gf_set_lk_ver_req {
+       string uid<>;
+       int lk_ver;
+};
+
+struct gf_event_notify_req {
+	int op;
+	opaque dict<>;
+};
+
+struct gf_event_notify_rsp {
+	int op_ret;
+	int op_errno;
+	opaque dict<>;
+};
diff --git a/rpc/xdr/src/glusterfs3.h b/rpc/xdr/src/glusterfs3.h
index 82a9e2001..798413e31 100644
--- a/rpc/xdr/src/glusterfs3.h
+++ b/rpc/xdr/src/glusterfs3.h
@@ -1,23 +1,13 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLUSTERFS3_H
 #define _GLUSTERFS3_H
 
@@ -54,6 +44,8 @@
 
 #define GF_O_LARGEFILE     0100000
 
+#define GF_O_FMODE_EXEC        040
+
 #define XLATE_BIT(from, to, bit)    do {                \
                 if (from & bit)                         \
                         to = to | GF_##bit;             \
@@ -112,6 +104,7 @@ gf_flags_from_flags (uint32_t flags)
         XLATE_BIT (flags, gf_flags, O_CLOEXEC);
 #endif
         XLATE_BIT (flags, gf_flags, O_LARGEFILE);
+        XLATE_BIT (flags, gf_flags, O_FMODE_EXEC);
 
         return gf_flags;
 }
@@ -142,6 +135,7 @@ gf_flags_to_flags (uint32_t gf_flags)
         UNXLATE_BIT (gf_flags, flags, O_CLOEXEC);
 #endif
         UNXLATE_BIT (gf_flags, flags, O_LARGEFILE);
+        UNXLATE_BIT (gf_flags, flags, O_FMODE_EXEC);
 
         return flags;
 }
diff --git a/rpc/xdr/src/mount3udp.x b/rpc/xdr/src/mount3udp.x
new file mode 100644
index 000000000..888c53120
--- /dev/null
+++ b/rpc/xdr/src/mount3udp.x
@@ -0,0 +1,25 @@
+/*
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* This is used by rpcgen to auto generate the rpc stubs.
+ * mount3udp_svc.c is heavily modified though
+ */
+
+const MNTUDPPATHLEN = 1024;
+
+typedef string mntudpdirpath<MNTPATHLEN>;
+
+program MOUNTUDP_PROGRAM {
+        version MOUNTUDP_V3 {
+                void MOUNTUDPPROC3_NULL(void) = 0;
+                mountres3 MOUNTUDPPROC3_MNT (mntudpdirpath) = 1;
+                mountstat3 MOUNTUDPPROC3_UMNT (mntudpdirpath) = 3;
+        } = 3;
+} = 100005;
diff --git a/rpc/xdr/src/msg-nfs3.c b/rpc/xdr/src/msg-nfs3.c
index 5fc92123c..6cdb5d37e 100644
--- a/rpc/xdr/src/msg-nfs3.c
+++ b/rpc/xdr/src/msg-nfs3.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -474,4 +465,108 @@ xdr_serialize_nfsstat3 (struct iovec outmsg, nfsstat3 *s)
                                           (xdrproc_t)xdr_nfsstat3);
 }
 
+ssize_t
+xdr_to_nlm4_testargs (struct iovec inmsg, nlm4_testargs *args)
+{
+        return xdr_to_generic (inmsg, (void*)args,
+                               (xdrproc_t)xdr_nlm4_testargs);
+}
+
+ssize_t
+xdr_serialize_nlm4_testres (struct iovec outmsg, nlm4_testres *res)
+{
+        return xdr_serialize_generic (outmsg, (void*)res,
+                                      (xdrproc_t)xdr_nlm4_testres);
+}
+
+ssize_t
+xdr_to_nlm4_lockargs (struct iovec inmsg, nlm4_lockargs *args)
+{
+        return xdr_to_generic (inmsg, (void*)args,
+                               (xdrproc_t)xdr_nlm4_lockargs);
+}
+
+ssize_t
+xdr_serialize_nlm4_res (struct iovec outmsg, nlm4_res *res)
+{
+        return xdr_serialize_generic (outmsg, (void*)res,
+                                      (xdrproc_t)xdr_nlm4_res);
+}
+
+ssize_t
+xdr_to_nlm4_cancelargs (struct iovec inmsg, nlm4_cancargs *args)
+{
+        return xdr_to_generic (inmsg, (void*)args,
+                               (xdrproc_t)xdr_nlm4_cancargs);
+}
+
+ssize_t
+xdr_to_nlm4_unlockargs (struct iovec inmsg, nlm4_unlockargs *args)
+{
+        return xdr_to_generic (inmsg, (void*)args,
+                               (xdrproc_t)xdr_nlm4_unlockargs);
+}
+
+ssize_t
+xdr_to_nlm4_shareargs (struct iovec inmsg, nlm4_shareargs *args)
+{
+        return xdr_to_generic (inmsg, (void*)args,
+                               (xdrproc_t)xdr_nlm4_shareargs);
+}
+
+ssize_t
+xdr_serialize_nlm4_shareres (struct iovec outmsg, nlm4_shareres *res)
+{
+        return xdr_serialize_generic (outmsg, (void *)res,
+                                      (xdrproc_t)xdr_nlm4_shareres);
+}
+
+ssize_t
+xdr_serialize_nlm4_testargs (struct iovec outmsg, nlm4_testargs *args)
+{
+        return xdr_serialize_generic (outmsg, (void*)args,
+                                      (xdrproc_t)xdr_nlm4_testargs);
+}
+
+ssize_t
+xdr_to_nlm4_res (struct iovec inmsg, nlm4_res *args)
+{
+        return xdr_to_generic (inmsg, (void*)args,
+                               (xdrproc_t)xdr_nlm4_res);
+}
+
+ssize_t
+xdr_to_nlm4_freeallargs (struct iovec inmsg, nlm4_freeallargs *args)
+{
+        return xdr_to_generic (inmsg, (void*)args,
+                               (xdrproc_t)xdr_nlm4_freeallargs);
+}
+
+ssize_t
+xdr_to_getaclargs (struct iovec inmsg, getaclargs *args)
+{
+        return xdr_to_generic (inmsg, (void *) args,
+                               (xdrproc_t)xdr_getaclargs);
+}
+
+ssize_t
+xdr_to_setaclargs (struct iovec inmsg, setaclargs *args)
+{
+        return xdr_to_generic (inmsg, (void *) args,
+                               (xdrproc_t)xdr_setaclargs);
+}
+
+ssize_t
+xdr_serialize_getaclreply (struct iovec inmsg, getaclreply *res)
+{
+        return xdr_serialize_generic (inmsg, (void *) res,
+                                      (xdrproc_t)xdr_getaclreply);
+}
+
+ssize_t
+xdr_serialize_setaclreply (struct iovec inmsg, setaclreply *res)
+{
+        return xdr_serialize_generic (inmsg, (void *) res,
+                                      (xdrproc_t)xdr_setaclreply);
+}
 
diff --git a/rpc/xdr/src/msg-nfs3.h b/rpc/xdr/src/msg-nfs3.h
index f828979e7..b8e2c9694 100644
--- a/rpc/xdr/src/msg-nfs3.h
+++ b/rpc/xdr/src/msg-nfs3.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _MSG_NFS3_H_
@@ -26,7 +17,8 @@
 #endif
 
 #include "xdr-nfs3.h"
-
+#include "nlm4-xdr.h"
+#include "acl3-xdr.h"
 #include <sys/types.h>
 #include <sys/uio.h>
 
@@ -183,4 +175,50 @@ xdr_serialize_mountstat3 (struct iovec outmsg, mountstat3 *m);
 
 extern ssize_t
 xdr_serialize_nfsstat3 (struct iovec outmsg, nfsstat3 *s);
+
+extern ssize_t
+xdr_to_nlm4_testargs (struct iovec inmsg, nlm4_testargs *args);
+
+extern ssize_t
+xdr_serialize_nlm4_testres (struct iovec outmsg, nlm4_testres *res);
+
+extern ssize_t
+xdr_to_nlm4_lockargs (struct iovec inmsg, nlm4_lockargs *args);
+
+extern ssize_t
+xdr_serialize_nlm4_res (struct iovec outmsg, nlm4_res *res);
+
+extern ssize_t
+xdr_to_nlm4_cancelargs (struct iovec inmsg, nlm4_cancargs *args);
+
+extern ssize_t
+xdr_to_nlm4_unlockargs (struct iovec inmsg, nlm4_unlockargs *args);
+
+extern ssize_t
+xdr_to_nlm4_shareargs (struct iovec inmsg, nlm4_shareargs *args);
+
+extern ssize_t
+xdr_serialize_nlm4_shareres (struct iovec outmsg, nlm4_shareres *res);
+
+extern ssize_t
+xdr_serialize_nlm4_testargs (struct iovec outmsg, nlm4_testargs *args);
+
+extern ssize_t
+xdr_to_nlm4_res (struct iovec inmsg, nlm4_res *args);
+
+extern ssize_t
+xdr_to_nlm4_freeallargs (struct iovec inmsg, nlm4_freeallargs *args);
+
+extern ssize_t
+xdr_to_getaclargs (struct iovec inmsg, getaclargs *args);
+
+extern ssize_t
+xdr_to_setaclargs (struct iovec inmsg, setaclargs *args);
+
+extern ssize_t
+xdr_serialize_getaclreply (struct iovec inmsg, getaclreply *res);
+
+extern ssize_t
+xdr_serialize_setaclreply (struct iovec inmsg, setaclreply *res);
+
 #endif
diff --git a/rpc/xdr/src/nlm4-xdr.c b/rpc/xdr/src/nlm4-xdr.c
new file mode 100644
index 000000000..caba05f58
--- /dev/null
+++ b/rpc/xdr/src/nlm4-xdr.c
@@ -0,0 +1,245 @@
+/*
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#include "nlm4-xdr.h"
+
+bool_t
+xdr_netobj (XDR *xdrs, netobj *objp)
+{
+	 if (!xdr_bytes (xdrs, (char **)&objp->n_bytes, (u_int *) &objp->n_len, MAXNETOBJ_SZ))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_fsh_mode (XDR *xdrs, fsh_mode *objp)
+{
+	 if (!xdr_enum (xdrs, (enum_t *) objp))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_fsh_access (XDR *xdrs, fsh_access *objp)
+{
+	 if (!xdr_enum (xdrs, (enum_t *) objp))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_stats (XDR *xdrs, nlm4_stats *objp)
+{
+	 if (!xdr_enum (xdrs, (enum_t *) objp))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_stat (XDR *xdrs, nlm4_stat *objp)
+{
+	 if (!xdr_nlm4_stats (xdrs, &objp->stat))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_holder (XDR *xdrs, nlm4_holder *objp)
+{
+	 if (!xdr_bool (xdrs, &objp->exclusive))
+		 return FALSE;
+	 if (!xdr_uint32_t (xdrs, &objp->svid))
+		 return FALSE;
+	 if (!xdr_netobj (xdrs, &objp->oh))
+		 return FALSE;
+	 if (!xdr_uint64_t (xdrs, &objp->l_offset))
+		 return FALSE;
+	 if (!xdr_uint64_t (xdrs, &objp->l_len))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_lock (XDR *xdrs, nlm4_lock *objp)
+{
+	 if (!xdr_string (xdrs, &objp->caller_name, MAXNAMELEN))
+		 return FALSE;
+	 if (!xdr_netobj (xdrs, &objp->fh))
+		 return FALSE;
+	 if (!xdr_netobj (xdrs, &objp->oh))
+		 return FALSE;
+	 if (!xdr_uint32_t (xdrs, &objp->svid))
+		 return FALSE;
+	 if (!xdr_uint64_t (xdrs, &objp->l_offset))
+		 return FALSE;
+	 if (!xdr_uint64_t (xdrs, &objp->l_len))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_share (XDR *xdrs, nlm4_share *objp)
+{
+	 if (!xdr_string (xdrs, &objp->caller_name, MAXNAMELEN))
+		 return FALSE;
+	 if (!xdr_netobj (xdrs, &objp->fh))
+		 return FALSE;
+	 if (!xdr_netobj (xdrs, &objp->oh))
+		 return FALSE;
+	 if (!xdr_fsh_mode (xdrs, &objp->mode))
+		 return FALSE;
+	 if (!xdr_fsh_access (xdrs, &objp->access))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_testrply (XDR *xdrs, nlm4_testrply *objp)
+{
+	 if (!xdr_nlm4_stats (xdrs, &objp->stat))
+		 return FALSE;
+	switch (objp->stat) {
+	case nlm4_denied:
+		 if (!xdr_nlm4_holder (xdrs, &objp->nlm4_testrply_u.holder))
+			 return FALSE;
+		break;
+	default:
+		break;
+	}
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_testres (XDR *xdrs, nlm4_testres *objp)
+{
+	 if (!xdr_netobj (xdrs, &objp->cookie))
+		 return FALSE;
+	 if (!xdr_nlm4_testrply (xdrs, &objp->stat))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_testargs (XDR *xdrs, nlm4_testargs *objp)
+{
+	 if (!xdr_netobj (xdrs, &objp->cookie))
+		 return FALSE;
+	 if (!xdr_bool (xdrs, &objp->exclusive))
+		 return FALSE;
+	 if (!xdr_nlm4_lock (xdrs, &objp->alock))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_res (XDR *xdrs, nlm4_res *objp)
+{
+	 if (!xdr_netobj (xdrs, &objp->cookie))
+		 return FALSE;
+	 if (!xdr_nlm4_stat (xdrs, &objp->stat))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_lockargs (XDR *xdrs, nlm4_lockargs *objp)
+{
+	 if (!xdr_netobj (xdrs, &objp->cookie))
+		 return FALSE;
+	 if (!xdr_bool (xdrs, &objp->block))
+		 return FALSE;
+	 if (!xdr_bool (xdrs, &objp->exclusive))
+		 return FALSE;
+	 if (!xdr_nlm4_lock (xdrs, &objp->alock))
+		 return FALSE;
+	 if (!xdr_bool (xdrs, &objp->reclaim))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->state))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_cancargs (XDR *xdrs, nlm4_cancargs *objp)
+{
+	 if (!xdr_netobj (xdrs, &objp->cookie))
+		 return FALSE;
+	 if (!xdr_bool (xdrs, &objp->block))
+		 return FALSE;
+	 if (!xdr_bool (xdrs, &objp->exclusive))
+		 return FALSE;
+	 if (!xdr_nlm4_lock (xdrs, &objp->alock))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_unlockargs (XDR *xdrs, nlm4_unlockargs *objp)
+{
+	 if (!xdr_netobj (xdrs, &objp->cookie))
+		 return FALSE;
+	 if (!xdr_nlm4_lock (xdrs, &objp->alock))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_shareargs (XDR *xdrs, nlm4_shareargs *objp)
+{
+	 if (!xdr_netobj (xdrs, &objp->cookie))
+		 return FALSE;
+	 if (!xdr_nlm4_share (xdrs, &objp->share))
+		 return FALSE;
+	 if (!xdr_bool (xdrs, &objp->reclaim))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_shareres (XDR *xdrs, nlm4_shareres *objp)
+{
+	 if (!xdr_netobj (xdrs, &objp->cookie))
+		 return FALSE;
+	 if (!xdr_nlm4_stats (xdrs, &objp->stat))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->sequence))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nlm4_freeallargs (XDR *xdrs, nlm4_freeallargs *objp)
+{
+        if (!xdr_string (xdrs, &objp->name, LM_MAXSTRLEN))
+                return FALSE;
+        if (!xdr_uint32_t (xdrs, &objp->state))
+                return FALSE;
+        return TRUE;
+}
+
+
+/*
+bool_t
+xdr_nlm_sm_status (XDR *xdrs, nlm_sm_status *objp)
+{
+	 if (!xdr_string (xdrs, &objp->mon_name, LM_MAXSTRLEN))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->state))
+		 return FALSE;
+	 if (!xdr_opaque (xdrs, objp->priv, 16))
+		 return FALSE;
+	return TRUE;
+}
+*/
diff --git a/rpc/xdr/src/nlm4-xdr.h b/rpc/xdr/src/nlm4-xdr.h
new file mode 100644
index 000000000..4391a4790
--- /dev/null
+++ b/rpc/xdr/src/nlm4-xdr.h
@@ -0,0 +1,258 @@
+/*
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#ifndef _NLM_H_RPCGEN
+#define _NLM_H_RPCGEN
+
+#include <rpc/rpc.h>
+
+#if defined(__NetBSD__)
+#define xdr_u_quad_t xdr_u_int64_t
+#define xdr_quad_t   xdr_int64_t
+#define xdr_uint32_t xdr_u_int32_t
+#define xdr_uint64_t xdr_u_int64_t
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MAXNETOBJ_SZ 1024
+#define LM_MAXSTRLEN 1024
+#define MAXNAMELEN 1025
+
+#if defined(__NetBSD__)
+#define xdr_u_quad_t xdr_u_int64_t
+#define xdr_quad_t   xdr_int64_t
+#define xdr_uint32_t xdr_u_int32_t
+#define xdr_uint64_t xdr_u_int64_t
+#endif
+
+/*
+ * The following enums are actually bit encoded for efficient
+ * boolean algebra.... DON'T change them.....
+ */
+
+enum fsh_mode {
+	fsm_DN = 0,
+	fsm_DR = 1,
+	fsm_DW = 2,
+	fsm_DRW = 3,
+};
+typedef enum fsh_mode fsh_mode;
+
+enum fsh_access {
+	fsa_NONE = 0,
+	fsa_R = 1,
+	fsa_W = 2,
+	fsa_RW = 3,
+};
+typedef enum fsh_access fsh_access;
+/* definitions for NLM version 4 */
+
+enum nlm4_stats {
+	nlm4_granted = 0,
+	nlm4_denied = 1,
+	nlm4_denied_nolock = 2,
+	nlm4_blocked = 3,
+	nlm4_denied_grace_period = 4,
+	nlm4_deadlck = 5,
+	nlm4_rofs = 6,
+	nlm4_stale_fh = 7,
+	nlm4_fbig = 8,
+	nlm4_failed = 9,
+};
+typedef enum nlm4_stats nlm4_stats;
+
+struct nlm4_stat {
+	nlm4_stats stat;
+};
+typedef struct nlm4_stat nlm4_stat;
+
+struct nlm4_holder {
+	bool_t exclusive;
+	u_int32_t svid;
+	netobj oh;
+	u_int64_t l_offset;
+	u_int64_t l_len;
+};
+typedef struct nlm4_holder nlm4_holder;
+
+struct nlm4_lock {
+	char *caller_name;
+	netobj fh;
+	netobj oh;
+	u_int32_t svid;
+	u_int64_t l_offset;
+	u_int64_t l_len;
+};
+typedef struct nlm4_lock nlm4_lock;
+
+struct nlm4_share {
+	char *caller_name;
+	netobj fh;
+	netobj oh;
+	fsh_mode mode;
+	fsh_access access;
+};
+typedef struct nlm4_share nlm4_share;
+
+struct nlm4_testrply {
+	nlm4_stats stat;
+	union {
+		struct nlm4_holder holder;
+	} nlm4_testrply_u;
+};
+typedef struct nlm4_testrply nlm4_testrply;
+
+struct nlm4_testres {
+	netobj cookie;
+	nlm4_testrply stat;
+};
+typedef struct nlm4_testres nlm4_testres;
+
+struct nlm4_testargs {
+	netobj cookie;
+	bool_t exclusive;
+	struct nlm4_lock alock;
+};
+typedef struct nlm4_testargs nlm4_testargs;
+
+struct nlm4_res {
+	netobj cookie;
+	nlm4_stat stat;
+};
+typedef struct nlm4_res nlm4_res;
+
+struct nlm4_lockargs {
+	netobj cookie;
+	bool_t block;
+	bool_t exclusive;
+	struct nlm4_lock alock;
+	bool_t reclaim;
+	int state;
+};
+typedef struct nlm4_lockargs nlm4_lockargs;
+
+struct nlm4_cancargs {
+	netobj cookie;
+	bool_t block;
+	bool_t exclusive;
+	struct nlm4_lock alock;
+};
+typedef struct nlm4_cancargs nlm4_cancargs;
+
+struct nlm4_unlockargs {
+	netobj cookie;
+	struct nlm4_lock alock;
+};
+typedef struct nlm4_unlockargs nlm4_unlockargs;
+
+struct nlm4_shareargs {
+	netobj cookie;
+	nlm4_share share;
+	bool_t reclaim;
+};
+typedef struct nlm4_shareargs nlm4_shareargs;
+
+struct nlm4_shareres {
+	netobj cookie;
+	nlm4_stats stat;
+	int sequence;
+};
+typedef struct nlm4_shareres nlm4_shareres;
+
+struct nlm4_freeallargs {
+        char *name;
+        u_int32_t state;
+};
+typedef struct nlm4_freeallargs nlm4_freeallargs;
+
+
+#define NLM4_NULL          0
+#define NLM4_TEST          1
+#define NLM4_LOCK          2
+#define NLM4_CANCEL        3
+#define NLM4_UNLOCK        4
+#define NLM4_GRANTED       5
+#define NLM4_TEST_MSG      6
+#define NLM4_LOCK_MSG      7
+#define NLM4_CANCEL_MSG    8
+#define NLM4_UNLOCK_MSG    9
+#define NLM4_GRANTED_MSG   10
+#define NLM4_TEST_RES      11
+#define NLM4_LOCK_RES      12
+#define NLM4_CANCEL_RES    13
+#define NLM4_UNLOCK_RES    14
+#define NLM4_GRANTED_RES   15
+#define NLM4_SM_NOTIFY     16
+#define NLM4_SEVENTEEN     17
+#define NLM4_EIGHTEEN      18
+#define NLM4_NINETEEN      19
+#define NLM4_SHARE         20
+#define NLM4_UNSHARE       21
+#define NLM4_NM_LOCK       22
+#define NLM4_FREE_ALL      23
+#define NLM4_PROC_COUNT    24
+
+/* the xdr functions */
+
+#if defined(__STDC__) || defined(__cplusplus)
+extern  bool_t xdr_netobj (XDR *, netobj*);
+extern  bool_t xdr_fsh_mode (XDR *, fsh_mode*);
+extern  bool_t xdr_fsh_access (XDR *, fsh_access*);
+extern  bool_t xdr_nlm4_stats (XDR *, nlm4_stats*);
+extern  bool_t xdr_nlm4_stat (XDR *, nlm4_stat*);
+extern  bool_t xdr_nlm4_holder (XDR *, nlm4_holder*);
+extern  bool_t xdr_nlm4_lock (XDR *, nlm4_lock*);
+extern  bool_t xdr_nlm4_share (XDR *, nlm4_share*);
+extern  bool_t xdr_nlm4_testrply (XDR *, nlm4_testrply*);
+extern  bool_t xdr_nlm4_testres (XDR *, nlm4_testres*);
+extern  bool_t xdr_nlm4_testargs (XDR *, nlm4_testargs*);
+extern  bool_t xdr_nlm4_res (XDR *, nlm4_res*);
+extern  bool_t xdr_nlm4_lockargs (XDR *, nlm4_lockargs*);
+extern  bool_t xdr_nlm4_cancargs (XDR *, nlm4_cancargs*);
+extern  bool_t xdr_nlm4_unlockargs (XDR *, nlm4_unlockargs*);
+extern  bool_t xdr_nlm4_shareargs (XDR *, nlm4_shareargs*);
+extern  bool_t xdr_nlm4_shareres (XDR *, nlm4_shareres*);
+extern  bool_t xdr_nlm4_freeallargs (XDR *, nlm4_freeallargs*);
+
+#else /* K&R C */
+extern bool_t xdr_netobj ();
+extern bool_t xdr_fsh_mode ();
+extern bool_t xdr_fsh_access ();
+extern bool_t xdr_nlm4_stats ();
+extern bool_t xdr_nlm4_stat ();
+extern bool_t xdr_nlm4_holder ();
+extern bool_t xdr_nlm4_lock ();
+extern bool_t xdr_nlm4_share ();
+extern bool_t xdr_nlm4_testrply ();
+extern bool_t xdr_nlm4_testres ();
+extern bool_t xdr_nlm4_testargs ();
+extern bool_t xdr_nlm4_res ();
+extern bool_t xdr_nlm4_lockargs ();
+extern bool_t xdr_nlm4_cancargs ();
+extern bool_t xdr_nlm4_unlockargs ();
+extern bool_t xdr_nlm4_shareargs ();
+extern bool_t xdr_nlm4_shareres ();
+extern bool_t xdr_nlm4_freeallargs ();
+
+#endif /* K&R C */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_NLM_H_RPCGEN */
diff --git a/rpc/xdr/src/nlm4.x b/rpc/xdr/src/nlm4.x
new file mode 100644
index 000000000..e22ac99f2
--- /dev/null
+++ b/rpc/xdr/src/nlm4.x
@@ -0,0 +1,154 @@
+/*
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* .x file defined as according to the RFC */
+
+const MAXNETOBJ_SZ = 1024;
+const LM_MAXSTRLEN = 1024;
+const MAXNAMELEN = 1025;
+
+typedef opaque netobj<MAXNETOBJ_SZ>;
+
+#ifdef RPC_HDR
+%/*
+% * The following enums are actually bit encoded for efficient
+% * boolean algebra.... DON'T change them.....
+% */
+#endif
+enum	fsh_mode {
+	fsm_DN  = 0,	/* deny none */
+	fsm_DR  = 1,	/* deny read */
+	fsm_DW  = 2,	/* deny write */
+	fsm_DRW = 3	/* deny read/write */
+};
+
+enum	fsh_access {
+	fsa_NONE = 0,	/* for completeness */
+	fsa_R    = 1,	/* read only */
+	fsa_W    = 2,	/* write only */
+	fsa_RW   = 3	/* read/write */
+};
+
+#ifdef RPC_HDR
+%/* definitions for NLM version 4 */
+#endif
+enum nlm4_stats {
+	nlm4_granted			= 0,
+	nlm4_denied			= 1,
+	nlm4_denied_nolock		= 2,
+	nlm4_blocked			= 3,
+	nlm4_denied_grace_period	= 4,
+	nlm4_deadlck			= 5,
+	nlm4_rofs			= 6,
+	nlm4_stale_fh			= 7,
+	nlm4_fbig			= 8,
+	nlm4_failed			= 9
+};
+
+struct nlm4_stat {
+	nlm4_stats stat;
+};
+
+struct nlm4_holder {
+	bool exclusive;
+	u_int32_t svid;
+	netobj oh;
+	u_int64_t l_offset;
+	u_int64_t l_len;
+};
+
+struct nlm4_lock {
+	string caller_name<LM_MAXSTRLEN>;
+	netobj fh;
+	netobj oh;
+	u_int32_t svid;
+	u_int64_t l_offset;
+	u_int64_t l_len;
+};
+
+struct nlm4_share {
+	string caller_name<LM_MAXSTRLEN>;
+	netobj fh;
+	netobj oh;
+	fsh_mode mode;
+	fsh_access access;
+};
+
+union nlm4_testrply switch (nlm4_stats stat) {
+	case nlm_denied:
+		struct nlm4_holder holder;
+	default:
+		void;
+};
+
+struct nlm4_testres {
+	netobj cookie;
+	nlm4_testrply stat;
+};
+
+struct nlm4_testargs {
+	netobj cookie;
+	bool exclusive;
+	struct nlm4_lock alock;
+};
+
+struct nlm4_res {
+	netobj cookie;
+	nlm4_stat stat;
+};
+
+struct nlm4_lockargs {
+	netobj cookie;
+	bool block;
+	bool exclusive;
+	struct nlm4_lock alock;
+	bool reclaim;		/* used for recovering locks */
+	int state;		/* specify local status monitor state */
+};
+
+struct nlm4_cancargs {
+	netobj cookie;
+	bool block;
+	bool exclusive;
+	struct nlm4_lock alock;
+};
+
+struct nlm4_unlockargs {
+	netobj cookie;
+	struct nlm4_lock alock;
+};
+
+struct	nlm4_shareargs {
+	netobj	cookie;
+	nlm4_share	share;
+	bool	reclaim;
+};
+
+struct	nlm4_shareres {
+	netobj	cookie;
+	nlm4_stats	stat;
+	int	sequence;
+};
+
+struct  nlm4_freeallargs {
+        string       name<LM_MAXSTRLEN>;   /* client hostname */
+        uint32       state;                /* unused */
+};
+
+/*
+ * argument for the procedure called by rpc.statd when a monitored host
+ * status change.
+ * XXX assumes LM_MAXSTRLEN == SM_MAXSTRLEN
+ */
+struct nlm_sm_status {
+	string mon_name<LM_MAXSTRLEN>; /* name of host */
+	int state;			/* new state */
+	opaque priv[16];		/* private data */
+};
diff --git a/rpc/xdr/src/nlmcbk-xdr.c b/rpc/xdr/src/nlmcbk-xdr.c
new file mode 100644
index 000000000..3d75acc55
--- /dev/null
+++ b/rpc/xdr/src/nlmcbk-xdr.c
@@ -0,0 +1,28 @@
+/*
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#include "nlmcbk-xdr.h"
+
+bool_t
+xdr_nlm_sm_status (XDR *xdrs, nlm_sm_status *objp)
+{
+	 if (!xdr_string (xdrs, &objp->mon_name, LM_MAXSTRLEN))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->state))
+		 return FALSE;
+	 if (!xdr_opaque (xdrs, objp->priv, 16))
+		 return FALSE;
+	return TRUE;
+}
diff --git a/rpc/xdr/src/nlmcbk-xdr.h b/rpc/xdr/src/nlmcbk-xdr.h
new file mode 100644
index 000000000..ad8421857
--- /dev/null
+++ b/rpc/xdr/src/nlmcbk-xdr.h
@@ -0,0 +1,65 @@
+/*
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#ifndef _NLMCBK_H_RPCGEN
+#define _NLMCBK_H_RPCGEN
+
+#include <rpc/rpc.h>
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define LM_MAXSTRLEN 1024
+
+struct nlm_sm_status {
+	char *mon_name;
+	int state;
+	char priv[16];
+};
+typedef struct nlm_sm_status nlm_sm_status;
+
+#define NLMCBK_PROGRAM 100021
+#define NLMCBK_V1 1
+
+#if defined(__STDC__) || defined(__cplusplus)
+#define NLMCBK_SM_NOTIFY 16
+extern  void * nlmcbk_sm_notify_0(struct nlm_sm_status *, CLIENT *);
+extern  void * nlmcbk_sm_notify_0_svc(struct nlm_sm_status *, struct svc_req *);
+extern int nlmcbk_program_0_freeresult (SVCXPRT *, xdrproc_t, caddr_t);
+
+#else /* K&R C */
+#define NLMCBK_SM_NOTIFY 16
+extern  void * nlmcbk_sm_notify_0();
+extern  void * nlmcbk_sm_notify_0_svc();
+extern int nlmcbk_program_0_freeresult ();
+#endif /* K&R C */
+
+/* the xdr functions */
+
+#if defined(__STDC__) || defined(__cplusplus)
+extern  bool_t xdr_nlm_sm_status (XDR *, nlm_sm_status*);
+
+#else /* K&R C */
+extern bool_t xdr_nlm_sm_status ();
+
+#endif /* K&R C */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_NLMCBK_H_RPCGEN */
diff --git a/rpc/xdr/src/nlmcbk.x b/rpc/xdr/src/nlmcbk.x
new file mode 100644
index 000000000..1d3746c9a
--- /dev/null
+++ b/rpc/xdr/src/nlmcbk.x
@@ -0,0 +1,24 @@
+/*
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+const LM_MAXSTRLEN = 1024;
+
+struct nlm_sm_status {
+        string mon_name<LM_MAXSTRLEN>; /* name of host */
+        int state;                      /* new state */
+        opaque priv[16];                /* private data */
+};
+
+program NLMCBK_PROGRAM {
+	version NLMCBK_V0 {
+		void NLMCBK_SM_NOTIFY(struct nlm_sm_status) = 1;
+	} = 0;
+} = 1238477;
+
diff --git a/rpc/xdr/src/nsm-xdr.c b/rpc/xdr/src/nsm-xdr.c
new file mode 100644
index 000000000..58712737b
--- /dev/null
+++ b/rpc/xdr/src/nsm-xdr.c
@@ -0,0 +1,96 @@
+/*
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#include "nsm-xdr.h"
+
+bool_t
+xdr_sm_name (XDR *xdrs, sm_name *objp)
+{
+	 if (!xdr_string (xdrs, &objp->mon_name, SM_MAXSTRLEN))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_res (XDR *xdrs, res *objp)
+{
+	 if (!xdr_enum (xdrs, (enum_t *) objp))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_sm_stat_res (XDR *xdrs, sm_stat_res *objp)
+{
+	 if (!xdr_res (xdrs, &objp->res_stat))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->state))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_sm_stat (XDR *xdrs, sm_stat *objp)
+{
+	 if (!xdr_int (xdrs, &objp->state))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_my_id (XDR *xdrs, my_id *objp)
+{
+	 if (!xdr_string (xdrs, &objp->my_name, SM_MAXSTRLEN))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->my_prog))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->my_vers))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->my_proc))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_mon_id (XDR *xdrs, mon_id *objp)
+{
+	 if (!xdr_string (xdrs, &objp->mon_name, SM_MAXSTRLEN))
+		 return FALSE;
+	 if (!xdr_my_id (xdrs, &objp->my_id))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_mon (XDR *xdrs, mon *objp)
+{
+	 if (!xdr_mon_id (xdrs, &objp->mon_id))
+		 return FALSE;
+	 if (!xdr_opaque (xdrs, objp->priv, 16))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_nsm_callback_status (XDR *xdrs, nsm_callback_status *objp)
+{
+	 if (!xdr_string (xdrs, &objp->mon_name, SM_MAXSTRLEN))
+		 return FALSE;
+	 if (!xdr_int (xdrs, &objp->state))
+		 return FALSE;
+	 if (!xdr_opaque (xdrs, objp->priv, 16))
+		 return FALSE;
+	return TRUE;
+}
diff --git a/rpc/xdr/src/nsm-xdr.h b/rpc/xdr/src/nsm-xdr.h
new file mode 100644
index 000000000..9de642c15
--- /dev/null
+++ b/rpc/xdr/src/nsm-xdr.h
@@ -0,0 +1,95 @@
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#ifndef _NSM_H_RPCGEN
+#define _NSM_H_RPCGEN
+
+#include <rpc/rpc.h>
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SM_MAXSTRLEN 1024
+
+struct sm_name {
+	char *mon_name;
+};
+typedef struct sm_name sm_name;
+
+enum res {
+	STAT_SUCC = 0,
+	STAT_FAIL = 1,
+};
+typedef enum res res;
+
+struct sm_stat_res {
+	res res_stat;
+	int state;
+};
+typedef struct sm_stat_res sm_stat_res;
+
+struct sm_stat {
+	int state;
+};
+typedef struct sm_stat sm_stat;
+
+struct my_id {
+	char *my_name;
+	int my_prog;
+	int my_vers;
+	int my_proc;
+};
+typedef struct my_id my_id;
+
+struct mon_id {
+	char *mon_name;
+	struct my_id my_id;
+};
+typedef struct mon_id mon_id;
+
+struct mon {
+	struct mon_id mon_id;
+	char priv[16];
+};
+typedef struct mon mon;
+
+struct nsm_callback_status {
+	char *mon_name;
+	int state;
+	char priv[16];
+};
+typedef struct nsm_callback_status nsm_callback_status;
+
+/* the xdr functions */
+
+#if defined(__STDC__) || defined(__cplusplus)
+extern  bool_t xdr_sm_name (XDR *, sm_name*);
+extern  bool_t xdr_res (XDR *, res*);
+extern  bool_t xdr_sm_stat_res (XDR *, sm_stat_res*);
+extern  bool_t xdr_sm_stat (XDR *, sm_stat*);
+extern  bool_t xdr_my_id (XDR *, my_id*);
+extern  bool_t xdr_mon_id (XDR *, mon_id*);
+extern  bool_t xdr_mon (XDR *, mon*);
+extern  bool_t xdr_nsm_callback_status (XDR *, nsm_callback_status*);
+
+#else /* K&R C */
+extern bool_t xdr_sm_name ();
+extern bool_t xdr_res ();
+extern bool_t xdr_sm_stat_res ();
+extern bool_t xdr_sm_stat ();
+extern bool_t xdr_my_id ();
+extern bool_t xdr_mon_id ();
+extern bool_t xdr_mon ();
+extern bool_t xdr_nsm_callback_status ();
+
+#endif /* K&R C */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_NSM_H_RPCGEN */
diff --git a/rpc/xdr/src/nsm.x b/rpc/xdr/src/nsm.x
new file mode 100644
index 000000000..8f97b1aaa
--- /dev/null
+++ b/rpc/xdr/src/nsm.x
@@ -0,0 +1,47 @@
+/*
+ * This defines the maximum length of the string
+ * identifying the caller.
+ */
+const SM_MAXSTRLEN = 1024;
+
+struct sm_name {
+    string mon_name<SM_MAXSTRLEN>;
+};
+
+enum res {
+    STAT_SUCC = 0,   /*  NSM agrees to monitor.  */
+    STAT_FAIL = 1    /*  NSM cannot monitor.  */
+};
+
+struct sm_stat_res {
+    res    res_stat;
+    int    state;
+};
+
+struct sm_stat {
+    int    state;    /*  state number of NSM  */
+};
+
+struct my_id {
+    string my_name<SM_MAXSTRLEN>;  /*  hostname  */
+    int    my_prog;                /*  RPC program number  */
+    int    my_vers;                /*  program version number  */
+    int    my_proc;                /*  procedure number  */
+};
+
+struct mon_id {
+    string mon_name<SM_MAXSTRLEN>; /* name of the host to be monitored */
+    struct my_id my_id;
+};
+
+struct mon {
+    struct mon_id mon_id;
+    opaque    priv[16];        /*  private information  */
+};
+
+struct nsm_callback_status {
+    string mon_name<SM_MAXSTRLEN>;
+    int    state;
+    opaque priv[16];        /*  for private information  */
+};
+
diff --git a/rpc/xdr/src/portmap-xdr.c b/rpc/xdr/src/portmap-xdr.c
index 7033213c0..4766122e6 100644
--- a/rpc/xdr/src/portmap-xdr.c
+++ b/rpc/xdr/src/portmap-xdr.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
diff --git a/rpc/xdr/src/portmap-xdr.h b/rpc/xdr/src/portmap-xdr.h
index 2686da287..8e4ff4f45 100644
--- a/rpc/xdr/src/portmap-xdr.h
+++ b/rpc/xdr/src/portmap-xdr.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
diff --git a/rpc/xdr/src/rpc-common-xdr.c b/rpc/xdr/src/rpc-common-xdr.c
index 14ddea715..6cb48a923 100644
--- a/rpc/xdr/src/rpc-common-xdr.c
+++ b/rpc/xdr/src/rpc-common-xdr.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
diff --git a/rpc/xdr/src/rpc-common-xdr.h b/rpc/xdr/src/rpc-common-xdr.h
index 66d9c3772..c43eab315 100644
--- a/rpc/xdr/src/rpc-common-xdr.h
+++ b/rpc/xdr/src/rpc-common-xdr.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-common.h"
diff --git a/rpc/xdr/src/xdr-generic.c b/rpc/xdr/src/xdr-generic.c
index 5d5cf7197..58d1ee77e 100644
--- a/rpc/xdr/src/xdr-generic.c
+++ b/rpc/xdr/src/xdr-generic.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -127,7 +118,7 @@ xdr_vector_round_up (struct iovec *vec, int vcount, uint32_t count)
 
         round_count = xdr_length_round_up (count, 1048576);
         round_count -= count;
-        if (round_count == 0)
+        if (round_count == 0 || vcount <= 0)
                 return;
 
         vec[vcount-1].iov_len += round_count;
diff --git a/rpc/xdr/src/xdr-generic.h b/rpc/xdr/src/xdr-generic.h
index 24054e11c..bb3759bbe 100644
--- a/rpc/xdr/src/xdr-generic.h
+++ b/rpc/xdr/src/xdr-generic.h
@@ -1,23 +1,13 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _XDR_GENERIC_H
 #define _XDR_GENERIC_H
 
diff --git a/rpc/xdr/src/xdr-nfs3.c b/rpc/xdr/src/xdr-nfs3.c
index 2b2b1049c..a497e9f54 100644
--- a/rpc/xdr/src/xdr-nfs3.c
+++ b/rpc/xdr/src/xdr-nfs3.c
@@ -1,24 +1,16 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "xdr-nfs3.h"
 #include "mem-pool.h"
+#include "xdr-common.h"
 
 #if GF_DARWIN_HOST_OS
 #define xdr_u_quad_t xdr_u_int64_t
@@ -1847,12 +1839,10 @@ xdr_free_exports_list (struct exportnode *first)
 
         while (first) {
                 elist = first->ex_next;
-                if (first->ex_dir)
-                        GF_FREE (first->ex_dir);
+                GF_FREE (first->ex_dir);
 
                 if (first->ex_groups) {
-                        if (first->ex_groups->gr_name)
-                                GF_FREE (first->ex_groups->gr_name);
+                        GF_FREE (first->ex_groups->gr_name);
                         GF_FREE (first->ex_groups);
                 }
 
diff --git a/rpc/xdr/src/xdr-nfs3.h b/rpc/xdr/src/xdr-nfs3.h
index 9e5ccd186..6f6b0e1f9 100644
--- a/rpc/xdr/src/xdr-nfs3.h
+++ b/rpc/xdr/src/xdr-nfs3.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _XDR_NFS3_H
@@ -1048,8 +1039,10 @@ typedef struct exportnode exportnode;
 #define MOUNT3_PROC_COUNT       6
 
 #define MOUNT1_NULL             0
+#define MOUNT1_MNT              1
 #define MOUNT1_DUMP             2
 #define MOUNT1_UMNT             3
+#define MOUNT1_UMNTALL          4
 #define MOUNT1_EXPORT           5
 #define MOUNT1_PROC_COUNT       6
 /* the xdr functions */
diff --git a/run-tests.sh b/run-tests.sh
new file mode 100755
index 000000000..128404ed5
--- /dev/null
+++ b/run-tests.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+#
+
+function _init()
+{
+    regression_testsdir=$(dirname $0);
+
+    if [ ! -f ${regression_testsdir}/tests/include.rc ]; then
+        echo "Seems like GlusterFS quality tests are corrupted..aborting!!"
+        exit 1
+    fi
+}
+
+function main()
+{
+    if [ $# -lt 1 ]; then
+        echo "Running all the regression test cases"
+        prove -rf --timer ${regression_testsdir}/tests;
+    else
+        ## TODO
+        echo "Running single regression test.."
+        echo "WARNING: yet to be implemented.. exiting safely"
+        exit 0
+        #export DEBUG=1;
+        #echo "Automatically setting up DEBUG=1 for this test $1";
+    fi
+}
+
+_init "$@" && main "$@"
diff --git a/scheduler/Makefile.am b/scheduler/Makefile.am
deleted file mode 100644
index 618fa7dd8..000000000
--- a/scheduler/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = alu random nufa rr switch
-
-CLEANFILES = 
diff --git a/scheduler/alu/src/Makefile.am b/scheduler/alu/src/Makefile.am
deleted file mode 100644
index eb7d0db07..000000000
--- a/scheduler/alu/src/Makefile.am
+++ /dev/null
@@ -1,14 +0,0 @@
-sched_LTLIBRARIES = alu.la
-scheddir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/scheduler
-
-alu_la_LDFLAGS = -module -avoidversion
-
-alu_la_SOURCES = alu.c
-alu_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = alu.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
diff --git a/scheduler/alu/src/alu-mem-types.h b/scheduler/alu/src/alu-mem-types.h
deleted file mode 100644
index 92702f286..000000000
--- a/scheduler/alu/src/alu-mem-types.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __ALU_MEM_TYPES_H__
-#define __ALU_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_alu_mem_types_ {
-        gf_alu_mt_alu_threshold = gf_common_mt_end + 1,
-        gf_alu_mt_alu_sched,
-        gf_alu_mt_alu_limits,
-        gf_alu_mt_alu_sched_struct,
-        gf_alu_mt_alu_sched_node,
-        gf_alu_mt_end
-};
-#endif
-
diff --git a/scheduler/alu/src/alu.c b/scheduler/alu/src/alu.c
deleted file mode 100644
index 58bef60ae..000000000
--- a/scheduler/alu/src/alu.c
+++ /dev/null
@@ -1,1019 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-
-/* ALU code needs a complete re-write. This is one of the most important 
- * part of GlusterFS and so needs more and more reviews and testing 
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <sys/time.h>
-#include <stdint.h>
-#include "stack.h"
-#include "alu.h"
-#include "alu-mem-types.h"
-
-#define ALU_DISK_USAGE_ENTRY_THRESHOLD_DEFAULT         (1 * GF_UNIT_GB)
-#define ALU_DISK_USAGE_EXIT_THRESHOLD_DEFAULT          (512 * GF_UNIT_MB)
-
-#define ALU_WRITE_USAGE_ENTRY_THRESHOLD_DEFAULT        25
-#define ALU_WRITE_USAGE_EXIT_THRESHOLD_DEFAULT         5
-
-#define ALU_READ_USAGE_ENTRY_THRESHOLD_DEFAULT         25
-#define ALU_READ_USAGE_EXIT_THRESHOLD_DEFAULT          5
-
-#define ALU_OPEN_FILES_USAGE_ENTRY_THRESHOLD_DEFAULT   1000
-#define ALU_OPEN_FILES_USAGE_EXIT_THRESHOLD_DEFAULT    100
-
-#define ALU_LIMITS_TOTAL_DISK_SIZE_DEFAULT             100
-
-#define ALU_REFRESH_INTERVAL_DEFAULT                   5
-#define ALU_REFRESH_CREATE_COUNT_DEFAULT               5
-
-
-static int64_t 
-get_stats_disk_usage (struct xlator_stats *this)
-{
-	return this->disk_usage;
-}
-
-static int64_t 
-get_stats_write_usage (struct xlator_stats *this)
-{
-	return this->write_usage;
-}
-
-static int64_t 
-get_stats_read_usage (struct xlator_stats *this)
-{
-	return this->read_usage;
-}
-
-static int64_t 
-get_stats_disk_speed (struct xlator_stats *this)
-{
-	return this->disk_speed;
-}
-
-static int64_t 
-get_stats_file_usage (struct xlator_stats *this)
-{
-	/* Avoid warning "defined but not used" */
-	(void) &get_stats_file_usage;    
-
-	return this->nr_files;
-}
-
-static int64_t 
-get_stats_free_disk (struct xlator_stats *this)
-{
-	if (this->total_disk_size > 0)
-		return (this->free_disk * 100) / this->total_disk_size;
-	return 0;
-}
-
-static int64_t 
-get_max_diff_write_usage (struct xlator_stats *max, struct xlator_stats *min)
-{
-	return (max->write_usage - min->write_usage);
-}
-
-static int64_t 
-get_max_diff_read_usage (struct xlator_stats *max, struct xlator_stats *min)
-{
-	return (max->read_usage - min->read_usage);
-}
-
-static int64_t 
-get_max_diff_disk_usage (struct xlator_stats *max, struct xlator_stats *min)
-{
-	return (max->disk_usage - min->disk_usage);
-}
-
-static int64_t 
-get_max_diff_disk_speed (struct xlator_stats *max, struct xlator_stats *min)
-{
-	return (max->disk_speed - min->disk_speed);
-}
-
-static int64_t 
-get_max_diff_file_usage (struct xlator_stats *max, struct xlator_stats *min)
-{
-	return (max->nr_files - min->nr_files);
-}
-
-
-int 
-alu_parse_options (xlator_t *xl, struct alu_sched *alu_sched)
-{
-	data_t *order = dict_get (xl->options, "scheduler.alu.order");
-	if (!order) {
-		gf_log (xl->name, GF_LOG_ERROR,
-			"option 'scheduler.alu.order' not specified");
-		return -1;
-	}
-	struct alu_threshold *_threshold_fn;
-	struct alu_threshold *tmp_threshold;
-	data_t *entry_fn = NULL;
-	data_t *exit_fn = NULL;
-	char *tmp_str = NULL;
-	char *order_str = strtok_r (order->data, ":", &tmp_str);
-	/* Get the scheduling priority order, specified by the user. */
-	while (order_str) {
-		gf_log ("alu", GF_LOG_DEBUG,
-			"alu_init: order string: %s",
-			order_str);
-		if (strcmp (order_str, "disk-usage") == 0) {
-			/* Disk usage */
-			_threshold_fn = 
-				GF_CALLOC (1,
-                                           sizeof (struct alu_threshold),
-                                           gf_alu_mt_alu_threshold);
-			ERR_ABORT (_threshold_fn);
-			_threshold_fn->diff_value = get_max_diff_disk_usage;
-			_threshold_fn->sched_value = get_stats_disk_usage;
-			entry_fn = 
-				dict_get (xl->options, 
-					  "scheduler.alu.disk-usage.entry-threshold");
-			if (entry_fn) {
-				if (gf_string2bytesize (entry_fn->data, 
-							&alu_sched->entry_limit.disk_usage) != 0) {
-					gf_log (xl->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\" "
-						"of \"option scheduler.alu."
-						"disk-usage.entry-threshold\"",
-						entry_fn->data);
-					return -1;
-				}
-			} else {
-				alu_sched->entry_limit.disk_usage = ALU_DISK_USAGE_ENTRY_THRESHOLD_DEFAULT;
-			}
-			_threshold_fn->entry_value = get_stats_disk_usage;
-			exit_fn = dict_get (xl->options, 
-					    "scheduler.alu.disk-usage.exit-threshold");
-			if (exit_fn) {
-				if (gf_string2bytesize (exit_fn->data, &alu_sched->exit_limit.disk_usage) != 0)	{
-					gf_log (xl->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\" "
-						"of \"option scheduler.alu."
-						"disk-usage.exit-threshold\"", 
-						exit_fn->data);
-					return -1;
-				}
-			} else {
-				alu_sched->exit_limit.disk_usage = ALU_DISK_USAGE_EXIT_THRESHOLD_DEFAULT;
-			}
-			_threshold_fn->exit_value = get_stats_disk_usage;
-			tmp_threshold = alu_sched->threshold_fn;
-			if (!tmp_threshold) {
-				alu_sched->threshold_fn = _threshold_fn;
-			} else {
-				while (tmp_threshold->next) {
-					tmp_threshold = tmp_threshold->next;
-				}
-				tmp_threshold->next = _threshold_fn;
-			}
-			gf_log ("alu",
-				GF_LOG_DEBUG, "alu_init: = %"PRId64",%"PRId64"", 
-				alu_sched->entry_limit.disk_usage, 
-				alu_sched->exit_limit.disk_usage);
-			
-		} else if (strcmp (order_str, "write-usage") == 0) {
-			/* Handle "write-usage" */
-			
-			_threshold_fn = GF_CALLOC (1, sizeof (struct alu_threshold), gf_alu_mt_alu_threshold);
-			ERR_ABORT (_threshold_fn);
-			_threshold_fn->diff_value = get_max_diff_write_usage;
-			_threshold_fn->sched_value = get_stats_write_usage;
-			entry_fn = dict_get (xl->options, 
-					     "scheduler.alu.write-usage.entry-threshold");
-			if (entry_fn) {
-				if (gf_string2bytesize (entry_fn->data, 
-							&alu_sched->entry_limit.write_usage) != 0) {
-					gf_log (xl->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\" "
-						"of option scheduler.alu."
-						"write-usage.entry-threshold", 
-						entry_fn->data);
-					return -1;
-				}
-			} else {
-				alu_sched->entry_limit.write_usage = ALU_WRITE_USAGE_ENTRY_THRESHOLD_DEFAULT;
-			}
-			_threshold_fn->entry_value = get_stats_write_usage;
-			exit_fn = dict_get (xl->options, 
-					    "scheduler.alu.write-usage.exit-threshold");
-			if (exit_fn) {
-				if (gf_string2bytesize (exit_fn->data, 
-							&alu_sched->exit_limit.write_usage) != 0) {
-					gf_log (xl->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\""
-						" of \"option scheduler.alu."
-						"write-usage.exit-threshold\"",
-						exit_fn->data);
-					return -1;
-				}
-			} else {
-				alu_sched->exit_limit.write_usage = ALU_WRITE_USAGE_EXIT_THRESHOLD_DEFAULT;
-			}
-			_threshold_fn->exit_value = get_stats_write_usage;
-			tmp_threshold = alu_sched->threshold_fn;
-			if (!tmp_threshold) {
-				alu_sched->threshold_fn = _threshold_fn;
-			} else {
-				while (tmp_threshold->next) {
-					tmp_threshold = tmp_threshold->next;
-				}
-				tmp_threshold->next = _threshold_fn;
-			}
-			gf_log (xl->name, GF_LOG_DEBUG, 
-				"alu_init: = %"PRId64",%"PRId64"", 
-				alu_sched->entry_limit.write_usage, 
-				alu_sched->exit_limit.write_usage);
-			
-		} else if (strcmp (order_str, "read-usage") == 0) {
-			/* Read usage */
-			
-			_threshold_fn = GF_CALLOC (1, sizeof (struct alu_threshold), gf_alu_mt_alu_threshold);
-			ERR_ABORT (_threshold_fn);
-			_threshold_fn->diff_value = get_max_diff_read_usage;
-			_threshold_fn->sched_value = get_stats_read_usage;
-			entry_fn = dict_get (xl->options, 
-					     "scheduler.alu.read-usage.entry-threshold");
-			if (entry_fn) {
-				if (gf_string2bytesize (entry_fn->data, 
-							&alu_sched->entry_limit.read_usage) != 0) {
-					gf_log (xl->name, 
-						GF_LOG_ERROR, 
-						"invalid number format \"%s\" "
-						"of \"option scheduler.alu."
-						"read-usage.entry-threshold\"",
-						entry_fn->data);
-					return -1;
-				}
-			} else {
-				alu_sched->entry_limit.read_usage = ALU_READ_USAGE_ENTRY_THRESHOLD_DEFAULT;
-			}
-			_threshold_fn->entry_value = get_stats_read_usage;
-			exit_fn = dict_get (xl->options, 
-					    "scheduler.alu.read-usage.exit-threshold");
-			if (exit_fn)
-			{
-				if (gf_string2bytesize (exit_fn->data, 
-							&alu_sched->exit_limit.read_usage) != 0)
-				{
-					gf_log ("alu", GF_LOG_ERROR, 
-						"invalid number format \"%s\" "
-						"of \"option scheduler.alu."
-						"read-usage.exit-threshold\"", 
-						exit_fn->data);
-					return -1;
-				}
-			}
-			else
-			{
-				alu_sched->exit_limit.read_usage = ALU_READ_USAGE_EXIT_THRESHOLD_DEFAULT;
-			}
-			_threshold_fn->exit_value = get_stats_read_usage;
-			tmp_threshold = alu_sched->threshold_fn;
-			if (!tmp_threshold) {
-				alu_sched->threshold_fn = _threshold_fn;
-			}
-			else {
-				while (tmp_threshold->next) {
-					tmp_threshold = tmp_threshold->next;
-				}
-				tmp_threshold->next = _threshold_fn;
-			}
-			gf_log ("alu", GF_LOG_DEBUG, 
-				"alu_init: = %"PRId64",%"PRId64"", 
-				alu_sched->entry_limit.read_usage, 
-				alu_sched->exit_limit.read_usage);
-			
-		} else if (strcmp (order_str, "open-files-usage") == 0) {
-			/* Open files counter */
-			
-			_threshold_fn = GF_CALLOC (1, sizeof (struct alu_threshold), gf_alu_mt_alu_threshold);
-			ERR_ABORT (_threshold_fn);
-			_threshold_fn->diff_value = get_max_diff_file_usage;
-			_threshold_fn->sched_value = get_stats_file_usage;
-			entry_fn = dict_get (xl->options, 
-					     "scheduler.alu.open-files-usage.entry-threshold");
-			if (entry_fn) {
-				if (gf_string2uint64 (entry_fn->data, 
-						      &alu_sched->entry_limit.nr_files) != 0)
-				{
-					gf_log ("alu", GF_LOG_ERROR, 
-						"invalid number format \"%s\" "
-						"of \"option scheduler.alu."
-						"open-files-usage.entry-"
-						"threshold\"", entry_fn->data);
-					return -1;
-				}
-			}
-			else
-			{
-				alu_sched->entry_limit.nr_files = ALU_OPEN_FILES_USAGE_ENTRY_THRESHOLD_DEFAULT;
-			}
-			_threshold_fn->entry_value = get_stats_file_usage;
-			exit_fn = dict_get (xl->options, 
-					    "scheduler.alu.open-files-usage.exit-threshold");
-			if (exit_fn)
-			{
-				if (gf_string2uint64 (exit_fn->data, 
-						      &alu_sched->exit_limit.nr_files) != 0)
-				{
-					gf_log ("alu", GF_LOG_ERROR, 
-						"invalid number format \"%s\" "
-						"of \"option scheduler.alu."
-						"open-files-usage.exit-"
-						"threshold\"", exit_fn->data);
-					return -1;
-				}
-			}
-			else
-			{
-				alu_sched->exit_limit.nr_files = ALU_OPEN_FILES_USAGE_EXIT_THRESHOLD_DEFAULT;
-			}
-			_threshold_fn->exit_value = get_stats_file_usage;
-			tmp_threshold = alu_sched->threshold_fn;
-			if (!tmp_threshold) {
-				alu_sched->threshold_fn = _threshold_fn;
-			}
-			else {
-				while (tmp_threshold->next) {
-					tmp_threshold = tmp_threshold->next;
-				}
-				tmp_threshold->next = _threshold_fn;
-			}
-			gf_log ("alu", GF_LOG_DEBUG, 
-				"alu.c->alu_init: = %"PRIu64",%"PRIu64"", 
-				alu_sched->entry_limit.nr_files, 
-				alu_sched->exit_limit.nr_files);
-			
-		} else if (strcmp (order_str, "disk-speed-usage") == 0) {
-			/* Disk speed */
-			
-			_threshold_fn = GF_CALLOC (1, sizeof (struct alu_threshold), gf_alu_mt_alu_threshold);
-			ERR_ABORT (_threshold_fn);
-			_threshold_fn->diff_value = get_max_diff_disk_speed;
-			_threshold_fn->sched_value = get_stats_disk_speed;
-			entry_fn = dict_get (xl->options, 
-					     "scheduler.alu.disk-speed-usage.entry-threshold");
-			if (entry_fn) {
-				gf_log ("alu", GF_LOG_DEBUG,
-					"entry-threshold is given, "
-					"value is constant");
-			}
-			_threshold_fn->entry_value = NULL;
-			exit_fn = dict_get (xl->options, 
-					    "scheduler.alu.disk-speed-usage.exit-threshold");
-			if (exit_fn) {
-				gf_log ("alu", GF_LOG_DEBUG,
-					"exit-threshold is given, "
-					"value is constant");
-			}
-			_threshold_fn->exit_value = NULL;
-			tmp_threshold = alu_sched->threshold_fn;
-			if (!tmp_threshold) {
-				alu_sched->threshold_fn = _threshold_fn;
-			}
-			else {
-				while (tmp_threshold->next) {
-					tmp_threshold = tmp_threshold->next;
-				}
-				tmp_threshold->next = _threshold_fn;
-			}
-			
-		} else {
-			gf_log ("alu", GF_LOG_DEBUG,
-				"%s, unknown option provided to scheduler",
-				order_str);
-		}
-		order_str = strtok_r (NULL, ":", &tmp_str);
-	}
-	
-	return 0;
-}
-
-static int32_t
-alu_init (xlator_t *xl)
-{
-	struct alu_sched *alu_sched = NULL;
-	struct alu_limits *_limit_fn = NULL;
-	struct alu_limits *tmp_limits = NULL;
-	uint32_t min_free_disk = 0;
-	data_t *limits = NULL;
-  
-	alu_sched = GF_CALLOC (1, sizeof (struct alu_sched),
-                               gf_alu_mt_alu_sched);
-	ERR_ABORT (alu_sched);
-
-	{
-		alu_parse_options (xl, alu_sched);
-	}
-
-	/* Get the limits */
-	
-	limits = dict_get (xl->options, 
-			   "scheduler.limits.min-free-disk");
-	if (limits) {
-		_limit_fn = GF_CALLOC (1, sizeof (struct alu_limits),
-                                       gf_alu_mt_alu_limits);
-		ERR_ABORT (_limit_fn);
-		_limit_fn->min_value = get_stats_free_disk;
-		_limit_fn->cur_value = get_stats_free_disk;
-		tmp_limits = alu_sched->limits_fn ;
-		_limit_fn->next = tmp_limits;
-		alu_sched->limits_fn = _limit_fn;
-		
-		if (gf_string2percent (limits->data, 
-				       &min_free_disk) != 0) {
-			gf_log ("alu", GF_LOG_ERROR, 
-				"invalid number format \"%s\" "
-				"of \"option scheduler.limits."
-				"min-free-disk\"", limits->data);
-			return -1;
-		}
-		alu_sched->spec_limit.free_disk = min_free_disk;
-		
-		if (alu_sched->spec_limit.free_disk >= 100) {
-			gf_log ("alu", GF_LOG_ERROR,
-				"check the \"option scheduler."
-				"limits.min-free-disk\", it should "
-				"be percentage value");
-			return -1;
-		}
-		alu_sched->spec_limit.total_disk_size = ALU_LIMITS_TOTAL_DISK_SIZE_DEFAULT; /* Its in % */
-		gf_log ("alu", GF_LOG_DEBUG,
-			"alu.limit.min-disk-free = %"PRId64"", 
-			_limit_fn->cur_value (&(alu_sched->spec_limit)));
-	}
-	
-	limits = dict_get (xl->options, 
-			   "scheduler.limits.max-open-files");
-	if (limits) {
-		// Update alu_sched->priority properly
-		_limit_fn = GF_CALLOC (1, sizeof (struct alu_limits),
-                                       gf_alu_mt_alu_limits);
-		ERR_ABORT (_limit_fn);
-		_limit_fn->max_value = get_stats_file_usage;
-		_limit_fn->cur_value = get_stats_file_usage;
-		tmp_limits = alu_sched->limits_fn ;
-		_limit_fn->next = tmp_limits;
-		alu_sched->limits_fn = _limit_fn;
-		if (gf_string2uint64_base10 (limits->data, 
-					     &alu_sched->spec_limit.nr_files) != 0)
-		{
-			gf_log ("alu", GF_LOG_ERROR, 
-				"invalid number format '%s' of option "
-				"scheduler.limits.max-open-files", 
-				limits->data);
-			return -1;
-		}
-		
-		gf_log ("alu", GF_LOG_DEBUG,
-			"alu_init: limit.max-open-files = %"PRId64"",
-			_limit_fn->cur_value (&(alu_sched->spec_limit)));
-	}
-
-
-	/* Stats refresh options */
-	limits = dict_get (xl->options, 
-			   "scheduler.refresh-interval");
-	if (limits) {
-		if (gf_string2time (limits->data, 
-				    &alu_sched->refresh_interval) != 0)	{
-			gf_log ("alu", GF_LOG_ERROR, 
-				"invalid number format \"%s\" of "
-				"option scheduler.refresh-interval", 
-				limits->data);
-			return -1;
-		}
-	} else {
-		alu_sched->refresh_interval = ALU_REFRESH_INTERVAL_DEFAULT;
-	}
-	gettimeofday (&(alu_sched->last_stat_fetch), NULL);
-	
-	
-	limits = dict_get (xl->options, 
-			   "scheduler.alu.stat-refresh.num-file-create");
-	if (limits) {
-		if (gf_string2uint32 (limits->data, 
-				      &alu_sched->refresh_create_count) != 0)
-		{
-			gf_log ("alu", GF_LOG_ERROR, 
-				"invalid number format \"%s\" of \"option "
-				"alu.stat-refresh.num-file-create\"", 
-				limits->data);
-			return -1;
-		}
-	} else {
-		alu_sched->refresh_create_count = ALU_REFRESH_CREATE_COUNT_DEFAULT;
-	}
-
-	{
-		/* Build an array of child_nodes */
-		struct alu_sched_struct *sched_array = NULL;
-		xlator_list_t *trav_xl = xl->children;
-		data_t *data = NULL;
-		int32_t index = 0;
-		
-		while (trav_xl) {
-			index++;
-			trav_xl = trav_xl->next;
-		}
-		alu_sched->child_count = index;
-		sched_array = GF_CALLOC (index, sizeof (struct alu_sched_struct), gf_alu_mt_alu_sched_struct);
-		ERR_ABORT (sched_array);
-		trav_xl = xl->children;
-		index = 0;
-		while (trav_xl) {
-			sched_array[index].xl = trav_xl->xlator;
-			sched_array[index].eligible = 1;
-			index++;
-			trav_xl = trav_xl->next;
-		}
-		alu_sched->array = sched_array;
-
-		data = dict_get (xl->options, 
-				 "scheduler.read-only-subvolumes");
-		if (data) {
-			char *child = NULL;
-			char *tmp = NULL;
-			char *childs_data = gf_strdup (data->data);
-      
-			child = strtok_r (childs_data, ",", &tmp);
-			while (child) {
-				for (index = 1; index < alu_sched->child_count; index++) {
-					if (strcmp (alu_sched->array[index -1].xl->name, child) == 0) {
-						memcpy (&(alu_sched->array[index -1]), 
-							&(alu_sched->array[alu_sched->child_count -1]), 
-							sizeof (struct alu_sched_struct));
-						alu_sched->child_count--;
-						break;
-					}
-				}
-				child = strtok_r (NULL, ",", &tmp);
-			}
-		}
-	}
-
-	*((long *)xl->private) = (long)alu_sched;
-
-	/* Initialize all the alu_sched structure's elements */
-	{
-		alu_sched->sched_nodes_pending = 0;
-
-		alu_sched->min_limit.free_disk = 0x00FFFFFF;
-		alu_sched->min_limit.disk_usage = 0xFFFFFFFF;
-		alu_sched->min_limit.total_disk_size = 0xFFFFFFFF;
-		alu_sched->min_limit.disk_speed = 0xFFFFFFFF;
-		alu_sched->min_limit.write_usage = 0xFFFFFFFF;
-		alu_sched->min_limit.read_usage = 0xFFFFFFFF;
-		alu_sched->min_limit.nr_files = 0xFFFFFFFF;
-		alu_sched->min_limit.nr_clients = 0xFFFFFFFF;
-	}
-
-	pthread_mutex_init (&alu_sched->alu_mutex, NULL);
-	return 0;
-}
-
-static void
-alu_fini (xlator_t *xl)
-{
-	if (!xl)
-		return;
-	struct alu_sched *alu_sched = (struct alu_sched *)*((long *)xl->private);
-	struct alu_limits *limit = alu_sched->limits_fn;
-	struct alu_threshold *threshold = alu_sched->threshold_fn;
-	void *tmp = NULL;
-	pthread_mutex_destroy (&alu_sched->alu_mutex);
-	GF_FREE (alu_sched->array);
-	while (limit) {
-		tmp = limit;
-		limit = limit->next;
-		GF_FREE (tmp);
-	}
-	while (threshold) {
-		tmp = threshold;
-		threshold = threshold->next;
-		GF_FREE (tmp);
-	}
-	GF_FREE (alu_sched);
-}
-
-static int32_t 
-update_stat_array_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *xl,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct xlator_stats *trav_stats)
-{
-	struct alu_sched *alu_sched = (struct alu_sched *)*((long *)xl->private);
-	struct alu_limits *limits_fn = alu_sched->limits_fn;
-	int32_t idx = 0;
-  
-	pthread_mutex_lock (&alu_sched->alu_mutex);
-	for (idx = 0; idx < alu_sched->child_count; idx++) {
-		if (alu_sched->array[idx].xl == (xlator_t *)cookie)
-			break;
-	}
-	pthread_mutex_unlock (&alu_sched->alu_mutex);
-
-	if (op_ret == -1) {
-		alu_sched->array[idx].eligible = 0;
-	} else {
-		memcpy (&(alu_sched->array[idx].stats), trav_stats, sizeof (struct xlator_stats));
-    
-		/* Get stats from all the child node */
-		/* Here check the limits specified by the user to 
-		   consider the nodes to be used by scheduler */
-		alu_sched->array[idx].eligible = 1;
-		limits_fn = alu_sched->limits_fn;
-		while (limits_fn){
-			if (limits_fn->max_value && 
-			    (limits_fn->cur_value (trav_stats) > 
-			     limits_fn->max_value (&(alu_sched->spec_limit)))) {
-				alu_sched->array[idx].eligible = 0;
-			}
-			if (limits_fn->min_value && 
-			    (limits_fn->cur_value (trav_stats) < 
-			     limits_fn->min_value (&(alu_sched->spec_limit)))) {
-				alu_sched->array[idx].eligible = 0;
-			}
-			limits_fn = limits_fn->next;
-		}
-
-		/* Select minimum and maximum disk_usage */
-		if (trav_stats->disk_usage > alu_sched->max_limit.disk_usage) {
-			alu_sched->max_limit.disk_usage = trav_stats->disk_usage;
-		}
-		if (trav_stats->disk_usage < alu_sched->min_limit.disk_usage) {
-			alu_sched->min_limit.disk_usage = trav_stats->disk_usage;
-		}
-
-		/* Select minimum and maximum disk_speed */
-		if (trav_stats->disk_speed > alu_sched->max_limit.disk_speed) {
-			alu_sched->max_limit.disk_speed = trav_stats->disk_speed;
-		}
-		if (trav_stats->disk_speed < alu_sched->min_limit.disk_speed) {
-			alu_sched->min_limit.disk_speed = trav_stats->disk_speed;
-		}
-
-		/* Select minimum and maximum number of open files */
-		if (trav_stats->nr_files > alu_sched->max_limit.nr_files) {
-			alu_sched->max_limit.nr_files = trav_stats->nr_files;
-		}
-		if (trav_stats->nr_files < alu_sched->min_limit.nr_files) {
-			alu_sched->min_limit.nr_files = trav_stats->nr_files;
-		}
-
-		/* Select minimum and maximum write-usage */
-		if (trav_stats->write_usage > alu_sched->max_limit.write_usage) {
-			alu_sched->max_limit.write_usage = trav_stats->write_usage;
-		}
-		if (trav_stats->write_usage < alu_sched->min_limit.write_usage) {
-			alu_sched->min_limit.write_usage = trav_stats->write_usage;
-		}
-
-		/* Select minimum and maximum read-usage */
-		if (trav_stats->read_usage > alu_sched->max_limit.read_usage) {
-			alu_sched->max_limit.read_usage = trav_stats->read_usage;
-		}
-		if (trav_stats->read_usage < alu_sched->min_limit.read_usage) {
-			alu_sched->min_limit.read_usage = trav_stats->read_usage;
-		}
-
-		/* Select minimum and maximum free-disk */
-		if (trav_stats->free_disk > alu_sched->max_limit.free_disk) {
-			alu_sched->max_limit.free_disk = trav_stats->free_disk;
-		}
-		if (trav_stats->free_disk < alu_sched->min_limit.free_disk) {
-			alu_sched->min_limit.free_disk = trav_stats->free_disk;
-		}
-	}
-
-	STACK_DESTROY (frame->root);
-
-	return 0;
-}
-
-static void 
-update_stat_array (xlator_t *xl)
-{
-	/* This function schedules the file in one of the child nodes */
-	struct alu_sched *alu_sched = (struct alu_sched *)*((long *)xl->private);
-	int32_t idx = 0;
-	call_frame_t *frame = NULL;
-	call_pool_t *pool = xl->ctx->pool;
-
-	for (idx = 0 ; idx < alu_sched->child_count; idx++) {
-		frame = create_frame (xl, pool);
-
-		STACK_WIND_COOKIE (frame,
-				   update_stat_array_cbk, 
-				   alu_sched->array[idx].xl, //cookie
-				   alu_sched->array[idx].xl, 
-				   (alu_sched->array[idx].xl)->mops->stats,
-				   0); //flag
-	}
-	return;
-}
-
-static void 
-alu_update (xlator_t *xl)
-{
-	struct timeval tv;
-	struct alu_sched *alu_sched = (struct alu_sched *)*((long *)xl->private);
-
-	gettimeofday (&tv, NULL);
-	if (tv.tv_sec > (alu_sched->refresh_interval + alu_sched->last_stat_fetch.tv_sec)) {
-		/* Update the stats from all the server */
-		update_stat_array (xl);
-		alu_sched->last_stat_fetch.tv_sec = tv.tv_sec;
-	}
-}
-
-static xlator_t *
-alu_scheduler (xlator_t *xl, const void *path)
-{
-	/* This function schedules the file in one of the child nodes */
-	struct alu_sched *alu_sched = (struct alu_sched *)*((long *)xl->private);
-	int32_t sched_index = 0;
-	int32_t sched_index_orig = 0;
-	int32_t idx = 0;
-
-	alu_update (xl);
-
-	/* Now check each threshold one by one if some nodes are classified */
-	{
-		struct alu_threshold *trav_threshold = alu_sched->threshold_fn;
-		struct alu_threshold *tmp_threshold = alu_sched->sched_method;
-		struct alu_sched_node *tmp_sched_node;   
-
-		/* This pointer 'trav_threshold' contains function pointers according to spec file
-		   give by user, */
-		while (trav_threshold) {
-			/* This check is needed for seeing if already there are nodes in this criteria 
-			   to be scheduled */
-			if (!alu_sched->sched_nodes_pending) {
-				for (idx = 0; idx < alu_sched->child_count; idx++) {
-					if (!alu_sched->array[idx].eligible) {
-						continue;
-					}
-					if (trav_threshold->entry_value) {
-						if (trav_threshold->diff_value (&(alu_sched->max_limit),
-										&(alu_sched->array[idx].stats)) <
-						    trav_threshold->entry_value (&(alu_sched->entry_limit))) {
-							continue;
-						}
-					}
-					tmp_sched_node = GF_CALLOC (1, sizeof (struct alu_sched_node), gf_alu_mt_alu_sched_node);
-					ERR_ABORT (tmp_sched_node);
-					tmp_sched_node->index = idx;
-					if (!alu_sched->sched_node) {
-						alu_sched->sched_node = tmp_sched_node;
-					} else {
-						pthread_mutex_lock (&alu_sched->alu_mutex);
-						tmp_sched_node->next = alu_sched->sched_node;
-						alu_sched->sched_node = tmp_sched_node;
-						pthread_mutex_unlock (&alu_sched->alu_mutex);
-					}
-					alu_sched->sched_nodes_pending++;
-				}
-			} /* end of if (sched_nodes_pending) */
-
-			/* This loop is required to check the eligible nodes */
-			struct alu_sched_node *trav_sched_node;
-			while (alu_sched->sched_nodes_pending) {
-				trav_sched_node = alu_sched->sched_node;
-				sched_index = trav_sched_node->index;
-				if (alu_sched->array[sched_index].eligible)
-					break;
-				alu_sched->sched_node = trav_sched_node->next;
-				GF_FREE (trav_sched_node);
-				alu_sched->sched_nodes_pending--;
-			}
-			if (alu_sched->sched_nodes_pending) {
-				/* There are some node in this criteria to be scheduled, no need 
-				 * to sort and check other methods 
-				 */
-				if (tmp_threshold && tmp_threshold->exit_value) {
-					/* verify the exit value && whether node is eligible or not */
-					if (tmp_threshold->diff_value (&(alu_sched->max_limit),
-								       &(alu_sched->array[sched_index].stats)) >
-					    tmp_threshold->exit_value (&(alu_sched->exit_limit))) {
-						/* Free the allocated info for the node :) */
-						pthread_mutex_lock (&alu_sched->alu_mutex);
-						alu_sched->sched_node = trav_sched_node->next;
-						GF_FREE (trav_sched_node);
-						trav_sched_node = alu_sched->sched_node;
-						alu_sched->sched_nodes_pending--;
-						pthread_mutex_unlock (&alu_sched->alu_mutex);
-					}
-				} else {
-					/* if there is no exit value, then exit after scheduling once */
-					pthread_mutex_lock (&alu_sched->alu_mutex);
-					alu_sched->sched_node = trav_sched_node->next;
-					GF_FREE (trav_sched_node);
-					trav_sched_node = alu_sched->sched_node;
-					alu_sched->sched_nodes_pending--;
-					pthread_mutex_unlock (&alu_sched->alu_mutex);
-				}
-	
-				alu_sched->sched_method = tmp_threshold; /* this is the method used for selecting */
-
-				/* */
-				if (trav_sched_node) {
-					tmp_sched_node = trav_sched_node;
-					while (trav_sched_node->next) {
-						trav_sched_node = trav_sched_node->next;
-					}
-					if (tmp_sched_node->next) {
-						pthread_mutex_lock (&alu_sched->alu_mutex);
-						alu_sched->sched_node = tmp_sched_node->next;
-						tmp_sched_node->next = NULL;
-						trav_sched_node->next = tmp_sched_node;
-						pthread_mutex_unlock (&alu_sched->alu_mutex);
-					}
-				}
-				/* return the scheduled node */
-				return alu_sched->array[sched_index].xl;
-			} /* end of if (pending_nodes) */
-      
-			tmp_threshold = trav_threshold;
-			trav_threshold = trav_threshold->next;
-		}
-	}
-  
-	/* This is used only when there is everything seems ok, or no eligible nodes */
-	sched_index_orig = alu_sched->sched_index;
-	alu_sched->sched_method = NULL;
-	while (1) {
-		//lock
-		pthread_mutex_lock (&alu_sched->alu_mutex);
-		sched_index = alu_sched->sched_index++;
-		alu_sched->sched_index = alu_sched->sched_index % alu_sched->child_count;
-		pthread_mutex_unlock (&alu_sched->alu_mutex);
-		//unlock
-		if (alu_sched->array[sched_index].eligible)
-			break;
-		if (sched_index_orig == (sched_index + 1) % alu_sched->child_count) {
-			gf_log ("alu", GF_LOG_WARNING, "No node is eligible to schedule");
-			//lock
-			pthread_mutex_lock (&alu_sched->alu_mutex);
-			alu_sched->sched_index++;
-			alu_sched->sched_index = alu_sched->sched_index % alu_sched->child_count;
-			pthread_mutex_unlock (&alu_sched->alu_mutex);
-			//unlock
-			break;
-		}
-	}
-	return alu_sched->array[sched_index].xl;
-}
-
-/**
- * notify
- */
-void
-alu_notify (xlator_t *xl, int32_t event, void *data)
-{
-	struct alu_sched *alu_sched = NULL; 
-	int32_t idx = 0;
-  
-	alu_sched = (struct alu_sched *)*((long *)xl->private);
-	if (!alu_sched)
-		return;
-
-	for (idx = 0; idx < alu_sched->child_count; idx++) {
-		if (alu_sched->array[idx].xl == (xlator_t *)data)
-			break;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_UP:
-	{
-		//alu_sched->array[idx].eligible = 1;
-	}
-	break;
-	case GF_EVENT_CHILD_DOWN:
-	{
-		alu_sched->array[idx].eligible = 0;
-	}
-	break;
-	default:
-	{
-		;
-	}
-	break;
-	}
-
-}
-
-int32_t
-alu_mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_alu_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-struct sched_ops sched = {
-	.init     = alu_init,
-	.fini     = alu_fini,
-	.update   = alu_update,
-	.schedule = alu_scheduler,
-	.notify   = alu_notify,
-        .mem_acct_init = alu_mem_acct_init,
-};
-
-struct volume_options options[] = {
-	{ .key   = { "scheduler.alu.order", "alu.order" },  
-	  .type  = GF_OPTION_TYPE_ANY 
-	},
-	{ .key   = { "scheduler.alu.disk-usage.entry-threshold", 
-		     "alu.disk-usage.entry-threshold" },  
-	  .type  = GF_OPTION_TYPE_SIZET
-	},
-	{ .key   = { "scheduler.alu.disk-usage.exit-threshold", 
-		     "alu.disk-usage.exit-threshold" },  
-	  .type  = GF_OPTION_TYPE_SIZET
-	},
-	{ .key   = { "scheduler.alu.write-usage.entry-threshold", 
-		     "alu.write-usage.entry-threshold" },  
-	  .type  = GF_OPTION_TYPE_SIZET
-	},
-	{ .key   = { "scheduler.alu.write-usage.exit-threshold", 
-		     "alu.write-usage.exit-threshold" },  
-	  .type  = GF_OPTION_TYPE_SIZET 
-	},
-	{ .key   = { "scheduler.alu.read-usage.entry-threshold", 
-		     "alu.read-usage.entry-threshold" },  
-	  .type  = GF_OPTION_TYPE_SIZET
-	},
-	{ .key   = { "scheduler.alu.read-usage.exit-threshold", 
-		     "alu.read-usage.exit-threshold" },  
-	  .type  = GF_OPTION_TYPE_SIZET 
-	},
-	{ .key   = { "scheduler.alu.open-files-usage.entry-threshold", 
-		     "alu.open-files-usage.entry-threshold" },  
-	  .type  = GF_OPTION_TYPE_INT
-	},
-	{ .key   = { "scheduler.alu.open-files-usage.exit-threshold", 
-		     "alu.open-files-usage.exit-threshold" },  
-	  .type  = GF_OPTION_TYPE_INT 
-	},
-	{ .key   = { "scheduler.read-only-subvolumes",
-		     "alu.read-only-subvolumes" },  
-	  .type  = GF_OPTION_TYPE_ANY 
-	},
-	{ .key   = { "scheduler.refresh-interval", 
-		     "alu.refresh-interval",
-		     "alu.stat-refresh.interval" },
-	  .type  = GF_OPTION_TYPE_TIME
-	},
-	{ .key   = { "scheduler.limits.min-free-disk", 
-		     "alu.limits.min-free-disk" },  
-	  .type  = GF_OPTION_TYPE_PERCENT
-	},
-	{ .key   = { "scheduler.alu.stat-refresh.num-file-create"
-		     "alu.stat-refresh.num-file-create"},  
-	  .type  = GF_OPTION_TYPE_INT
-	},
-	{ .key  =  {NULL}, }
-};
diff --git a/scheduler/alu/src/alu.h b/scheduler/alu/src/alu.h
deleted file mode 100644
index c716ad8e5..000000000
--- a/scheduler/alu/src/alu.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _ALU_H
-#define _ALU_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "scheduler.h"
-
-struct alu_sched;
-
-struct alu_sched_struct {
-  xlator_t *xl;
-  struct xlator_stats stats;
-  unsigned char eligible;
-};
-
-// Write better name for these functions
-struct alu_limits {
-  struct alu_limits *next;
-  int64_t (*max_value) (struct xlator_stats *); /* Max limit, specified by the user */
-  int64_t (*min_value) (struct xlator_stats *); /* Min limit, specified by the user */
-  int64_t (*cur_value) (struct xlator_stats *); /* Current values of variables got from stats call */
-};
-
-struct alu_threshold {
-  struct alu_threshold *next;
-  int64_t (*diff_value) (struct xlator_stats *max, struct xlator_stats *min); /* Diff b/w max and min */
-  int64_t (*entry_value) (struct xlator_stats *); /* Limit specified user */
-  int64_t (*exit_value) (struct xlator_stats *); /* Exit point for the limit */
-  int64_t (*sched_value) (struct xlator_stats *); /* This will return the index of the child area */
-};
-
-struct alu_sched_node {
-  struct alu_sched_node *next;
-  int32_t index;
-};
-
-struct alu_sched {
-  struct alu_limits *limits_fn;
-  struct alu_threshold *threshold_fn;
-  struct alu_sched_struct *array;
-  struct alu_sched_node *sched_node;
-  struct alu_threshold *sched_method;
-  struct xlator_stats max_limit;
-  struct xlator_stats min_limit;
-  struct xlator_stats entry_limit;
-  struct xlator_stats exit_limit;
-  struct xlator_stats spec_limit;     /* User given limit */
-
-  pthread_mutex_t alu_mutex;
-  struct timeval last_stat_fetch;
-  uint32_t refresh_interval;      /* in seconds */
-  uint32_t refresh_create_count;  /* num-file-create */
-
-  int32_t sched_nodes_pending;
-
-  int32_t sched_index;  /* used for round robin scheduling in case of many nodes getting the criteria match. */
-  int32_t child_count;
-
-};
-
-struct _alu_local_t {
-  int32_t call_count;
-};
-
-typedef struct _alu_local_t alu_local_t;
-
-#endif /* _ALU_H */
diff --git a/scheduler/nufa/src/Makefile.am b/scheduler/nufa/src/Makefile.am
deleted file mode 100644
index 6eb3d39f1..000000000
--- a/scheduler/nufa/src/Makefile.am
+++ /dev/null
@@ -1,12 +0,0 @@
-sched_LTLIBRARIES = nufa.la
-scheddir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/scheduler
-
-nufa_la_LDFLAGS = -module -avoidversion
-
-nufa_la_SOURCES = nufa.c
-nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
diff --git a/scheduler/nufa/src/nufa-mem-types.h b/scheduler/nufa/src/nufa-mem-types.h
deleted file mode 100644
index 945dafa7c..000000000
--- a/scheduler/nufa/src/nufa-mem-types.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __NUFA_MEM_TYPES_H__
-#define __NUFA_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_locks_mem_types_ {
-        gf_nufa_mt_nufa_struct = gf_common_mt_end + 1,
-        gf_nufa_mt_nufa_sched_struct,
-        gf_nufa_mt_int32_t,
-        gf_nufa_mt_end
-};
-#endif
-
diff --git a/scheduler/nufa/src/nufa.c b/scheduler/nufa/src/nufa.c
deleted file mode 100644
index 1bf477bdc..000000000
--- a/scheduler/nufa/src/nufa.c
+++ /dev/null
@@ -1,429 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <sys/time.h>
-
-#include "scheduler.h"
-#include "common-utils.h"
-#include "nufa-mem-types.h"
-
-struct nufa_sched_struct {
-	xlator_t *xl;
-	struct timeval last_stat_fetch;
-	int64_t free_disk;
-	int32_t refresh_interval;
-	unsigned char eligible;
-};
-
-struct nufa_struct {
-	struct nufa_sched_struct *array;
-	struct timeval last_stat_fetch;
-
-	int32_t *local_array; /* Used to keep the index of the local xlators */
-	int32_t local_xl_index; /* index in the above array */
-	int32_t local_xl_count; /* Count of the local subvolumes */
-
-	uint32_t refresh_interval;
-	uint32_t min_free_disk;
-  
-	gf_lock_t nufa_lock;
-	int32_t child_count;
-	int32_t sched_index;  
-};
-
-#define NUFA_LIMITS_MIN_FREE_DISK_DEFAULT    15
-#define NUFA_REFRESH_INTERVAL_DEFAULT        30
-
-static int32_t
-nufa_init (xlator_t *xl)
-{
-	int32_t index = 0;
-	data_t *local_name = NULL;
-	data_t *data = NULL;
-	xlator_list_t *trav_xl = xl->children;
-	struct nufa_struct *nufa_buf = NULL;
-
-	nufa_buf = GF_CALLOC (1, sizeof (struct nufa_struct),
-                              gf_nufa_mt_nufa_struct);
-	ERR_ABORT (nufa_buf);
-
-	data = dict_get (xl->options, "scheduler.limits.min-free-disk");
-	if (data) {
-		if (gf_string2percent (data->data, 
-				       &nufa_buf->min_free_disk) != 0) {
-			gf_log ("nufa", GF_LOG_ERROR, 
-				"invalid number format \"%s\" of "
-				"\"option scheduler.limits.min-free-disk\"", 
-				data->data);
-			return -1;
-		}
-		if (nufa_buf->min_free_disk >= 100) {
-			gf_log ("nufa", GF_LOG_ERROR,
-				"check \"option scheduler.limits.min-free-disk"
-				"\", it should be percentage value");
-			return -1;
-		}
-	} else {
-		gf_log ("nufa", GF_LOG_WARNING, 
-			"No option for limit min-free-disk given, "
-			"defaulting it to 15%%");
-		nufa_buf->min_free_disk = NUFA_LIMITS_MIN_FREE_DISK_DEFAULT;
-	}
-	data = dict_get (xl->options, "scheduler.refresh-interval");
-	if (data != NULL) {
-                if (gf_string2time (data->data, 
-				    &nufa_buf->refresh_interval) != 0) {
-                        gf_log ("nufa", GF_LOG_ERROR, 
-                                "invalid number format \"%s\" of "
-                                "\"option scheduler.refresh-interval\"", 
-                                data->data);
-                        return -1;
-                }
-        } else {
-		gf_log ("nufa", GF_LOG_WARNING, 
-			"No option for scheduler.refresh-interval given, "
-			"defaulting it to 30");
-		nufa_buf->refresh_interval = NUFA_REFRESH_INTERVAL_DEFAULT;
-	}
-  
-	/* Get the array built */
-	while (trav_xl) {
-		index++;
-		trav_xl = trav_xl->next;
-	}
-	nufa_buf->child_count = index;
-	nufa_buf->sched_index = 0;
-	nufa_buf->array = GF_CALLOC (index, sizeof (struct nufa_sched_struct),
-                                     gf_nufa_mt_nufa_sched_struct);
-	ERR_ABORT (nufa_buf->array);
-	nufa_buf->local_array = GF_CALLOC (index, sizeof (int32_t),
-                                           gf_nufa_mt_int32_t);
-	ERR_ABORT (nufa_buf->array);
-	trav_xl = xl->children;
-	
-	local_name = dict_get (xl->options, "scheduler.local-volume-name");
-	if (!local_name) {
-		/* Error */
-		gf_log ("nufa", GF_LOG_ERROR, 
-			"No 'local-volume-name' option given in volume file");
-		GF_FREE (nufa_buf->array);
-		GF_FREE (nufa_buf->local_array);
-		GF_FREE (nufa_buf);
-		return -1;
-	}
-
-	/* Get the array properly */
-	index = 0;
-	trav_xl = xl->children;
-	while (trav_xl) {
-		nufa_buf->array[index].xl = trav_xl->xlator;
-		nufa_buf->array[index].eligible = 1;
-		nufa_buf->array[index].free_disk = nufa_buf->min_free_disk;
-		nufa_buf->array[index].refresh_interval = 
-			nufa_buf->refresh_interval;
-
-		trav_xl = trav_xl->next;
-		index++;
-	}
-  
-	{ 
-		int32_t array_index = 0;
-		char *child = NULL;
-		char *tmp = NULL;
-		char *childs_data = gf_strdup (local_name->data);
-		
-		child = strtok_r (childs_data, ",", &tmp);
-		while (child) {
-			/* Check if the local_volume specified is proper 
-			   subvolume of unify */
-			trav_xl = xl->children;
-			index=0;
-			while (trav_xl) {
-				if (strcmp (child, trav_xl->xlator->name) == 0)
-					break;
-				trav_xl = trav_xl->next;
-				index++;
-			}
-
-			if (!trav_xl) {
-				/* entry for 'local-volume-name' is wrong, 
-				   not present in subvolumes */
-				gf_log ("nufa", GF_LOG_ERROR, 
-					"option 'scheduler.local-volume-name' "
-					"%s is wrong", child);
-				GF_FREE (nufa_buf->array);
-				GF_FREE (nufa_buf->local_array);
-				GF_FREE (nufa_buf);
-				return -1;
-			} else {
-				nufa_buf->local_array[array_index++] = index;
-				nufa_buf->local_xl_count++;
-			}
-			child = strtok_r (NULL, ",", &tmp);
-		}
-		GF_FREE (childs_data);
-	}
-
-	LOCK_INIT (&nufa_buf->nufa_lock);
-	*((long *)xl->private) = (long)nufa_buf; // put it at the proper place
-	return 0;
-}
-
-static void
-nufa_fini (xlator_t *xl)
-{
-	struct nufa_struct *nufa_buf = 
-		(struct nufa_struct *)*((long *)xl->private);
-
-	LOCK_DESTROY (&nufa_buf->nufa_lock);
-	GF_FREE (nufa_buf->local_array);
-	GF_FREE (nufa_buf->array);
-	GF_FREE (nufa_buf);
-}
-
-static int32_t 
-update_stat_array_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *xl,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct xlator_stats *trav_stats)
-{
-	struct nufa_struct *nufa_struct = NULL; 
-	int32_t idx = 0;
-	int32_t percent = 0;
-
-	nufa_struct = (struct nufa_struct *)*((long *)xl->private);
-	LOCK (&nufa_struct->nufa_lock);
-	for (idx = 0; idx < nufa_struct->child_count; idx++) {
-		if (nufa_struct->array[idx].xl->name == (char *)cookie)
-			break;
-	}
-	UNLOCK (&nufa_struct->nufa_lock);
-
-	if (op_ret == 0) {
-		percent = ((trav_stats->free_disk * 100) / 
-			   trav_stats->total_disk_size);
-		if (nufa_struct->array[idx].free_disk > percent) {
-			if (nufa_struct->array[idx].eligible)
-				gf_log ("nufa", GF_LOG_CRITICAL,
-					"node \"%s\" is _almost_ (%d %%) full",
-					nufa_struct->array[idx].xl->name, 
-					100 - percent);
-			nufa_struct->array[idx].eligible = 0;
-		} else {
-			nufa_struct->array[idx].eligible = 1;
-		}
-	} else {
-		nufa_struct->array[idx].eligible = 0;
-	}
-
-	STACK_DESTROY (frame->root);
-	return 0;
-}
-
-static void 
-update_stat_array (xlator_t *xl)
-{
-	/* This function schedules the file in one of the child nodes */
-	int32_t idx;
-	struct nufa_struct *nufa_buf = 
-		(struct nufa_struct *)*((long *)xl->private);
-	call_frame_t *frame = NULL;
-	call_pool_t *pool = xl->ctx->pool;
-
-	for (idx = 0; idx < nufa_buf->child_count; idx++) {
-		frame = create_frame (xl, pool);
-
-		STACK_WIND_COOKIE (frame,
-				   update_stat_array_cbk, 
-				   nufa_buf->array[idx].xl->name,
-				   nufa_buf->array[idx].xl, 
-				   (nufa_buf->array[idx].xl)->mops->stats,
-				   0); //flag
-	}
-
-	return;
-}
-
-static void 
-nufa_update (xlator_t *xl)
-{
-	struct nufa_struct *nufa_buf = 
-		(struct nufa_struct *)*((long *)xl->private);
-	struct timeval tv;
-	gettimeofday (&tv, NULL);
-	if (tv.tv_sec > (nufa_buf->refresh_interval 
-			 + nufa_buf->last_stat_fetch.tv_sec)) {
-		/* Update the stats from all the server */
-		update_stat_array (xl);
-		nufa_buf->last_stat_fetch.tv_sec = tv.tv_sec;
-	}
-}
-
-static xlator_t *
-nufa_schedule (xlator_t *xl, const void *path)
-{
-	struct nufa_struct *nufa_buf = 
-		(struct nufa_struct *)*((long *)xl->private);
-	int32_t nufa_orig = nufa_buf->local_xl_index;  
-	int32_t rr;
-  
-	nufa_update (xl);
-  
-	while (1) {
-		LOCK (&nufa_buf->nufa_lock);
-		rr = nufa_buf->local_xl_index++;
-		nufa_buf->local_xl_index %= nufa_buf->local_xl_count;
-		UNLOCK (&nufa_buf->nufa_lock);
-    
-		/* if 'eligible' or there are _no_ eligible nodes */
-		if (nufa_buf->array[nufa_buf->local_array[rr]].eligible) {
-			/* Return the local node */
-			return nufa_buf->array[nufa_buf->local_array[rr]].xl;
-		}
-		if ((rr + 1) % nufa_buf->local_xl_count == nufa_orig) {
-			gf_log ("nufa", GF_LOG_CRITICAL, 
-				"No free space available on any local "
-				"volumes, using RR scheduler");
-			LOCK (&nufa_buf->nufa_lock);
-			nufa_buf->local_xl_index++;
-			nufa_buf->local_xl_index %= nufa_buf->local_xl_count;
-			UNLOCK (&nufa_buf->nufa_lock);
-			break;
-		}
-	}
-
-	nufa_orig = nufa_buf->sched_index;  
-	while (1) {
-		LOCK (&nufa_buf->nufa_lock);
-		rr = nufa_buf->sched_index++;
-		nufa_buf->sched_index = (nufa_buf->sched_index % 
-					 nufa_buf->child_count);
-		UNLOCK (&nufa_buf->nufa_lock);
-    
-		/* if 'eligible' or there are _no_ eligible nodes */
-		if (nufa_buf->array[rr].eligible) {
-			break;
-		}
-		if ((rr + 1) % nufa_buf->child_count == nufa_orig) {
-			gf_log ("nufa", GF_LOG_CRITICAL, 
-				"No free space available on any server, "
-				"using RR scheduler.");
-			LOCK (&nufa_buf->nufa_lock);
-			nufa_buf->sched_index++;
-			nufa_buf->sched_index = (nufa_buf->sched_index % 
-						 nufa_buf->child_count);
-			UNLOCK (&nufa_buf->nufa_lock);
-			break;
-		}
-	}
-	return nufa_buf->array[rr].xl;
-}
-
-
-/**
- * notify
- */
-void
-nufa_notify (xlator_t *xl, int32_t event, void *data)
-{
-	struct nufa_struct *nufa_buf = 
-		(struct nufa_struct *)*((long *)xl->private);
-	int32_t idx = 0;
-  
-	if (!nufa_buf)
-		return;
-
-	for (idx = 0; idx < nufa_buf->child_count; idx++) {
-		if (strcmp (nufa_buf->array[idx].xl->name, 
-			    ((xlator_t *)data)->name) == 0)
-			break;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_UP:
-	{
-		//nufa_buf->array[idx].eligible = 1;
-	}
-	break;
-	case GF_EVENT_CHILD_DOWN:
-	{
-		nufa_buf->array[idx].eligible = 0;
-	}
-	break;
-	default:
-	{
-		;
-	}
-	break;
-	}
-
-}
-
-int32_t
-nufa_mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_nufa_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-struct sched_ops sched = {
-	.init     = nufa_init,
-	.fini     = nufa_fini,
-	.update   = nufa_update,
-	.schedule = nufa_schedule,
-	.notify   = nufa_notify,
-        .mem_acct_init = nufa_mem_acct_init,
-};
-
-struct volume_options options[] = {
-	{ .key   = { "scheduler.refresh-interval", 
-		     "nufa.refresh-interval" },  
-	  .type  = GF_OPTION_TYPE_TIME
-	},
-	{ .key   = { "scheduler.limits.min-free-disk", 
-		     "nufa.limits.min-free-disk" },  
-	  .type  = GF_OPTION_TYPE_PERCENT
-	},
-	{ .key   = { "scheduler.local-volume-name",
-		     "nufa.local-volume-name" },
-	  .type  = GF_OPTION_TYPE_XLATOR
-	},	
-	{ .key = {NULL} }
-};
-
diff --git a/scheduler/random/src/Makefile.am b/scheduler/random/src/Makefile.am
deleted file mode 100644
index 572181336..000000000
--- a/scheduler/random/src/Makefile.am
+++ /dev/null
@@ -1,14 +0,0 @@
-sched_LTLIBRARIES = random.la
-scheddir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/scheduler
-
-random_la_LDFLAGS = -module -avoidversion
-
-random_la_SOURCES = random.c
-random_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = random.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
diff --git a/scheduler/random/src/random-mem-types.h b/scheduler/random/src/random-mem-types.h
deleted file mode 100644
index ff30d9244..000000000
--- a/scheduler/random/src/random-mem-types.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __RANDOM_MEM_TYPES_H__
-#define __RANDOM_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_random_mem_types_ {
-        gf_random_mt_random_struct = gf_common_mt_end + 1,
-        gf_random_mt_random_sched_struct,
-        gf_random_mt_end
-};
-#endif
-
diff --git a/scheduler/random/src/random.c b/scheduler/random/src/random.c
deleted file mode 100644
index 03a284b66..000000000
--- a/scheduler/random/src/random.c
+++ /dev/null
@@ -1,305 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <stdlib.h>
-#include <sys/time.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "random.h"
-#include "random-mem-types.h"
-
-#define RANDOM_LIMITS_MIN_FREE_DISK_DEFAULT    15
-#define RANDOM_REFRESH_INTERVAL_DEFAULT        10
-
-
-static int32_t
-random_init (xlator_t *xl)
-{
-	struct random_struct *random_buf = NULL;
-	xlator_list_t *trav_xl = xl->children;
-	data_t *limit = NULL;
-	int32_t index = 0;
-
-	random_buf = GF_CALLOC (1, sizeof (struct random_struct),
-                                gf_random_mt_random_struct);
-	ERR_ABORT (random_buf);
-  
-	/* Set the seed for the 'random' function */
-	srandom ((uint32_t) time (NULL));
-  
-	limit = dict_get (xl->options, "scheduler.limits.min-free-disk");
-	if (limit) {
-		if (gf_string2percent (data_to_str (limit),
-				       &random_buf->min_free_disk) != 0) {
-			gf_log ("random", GF_LOG_ERROR, 
-				"invalid number format \"%s\" of \"option "
-				"scheduler.limits.min-free-disk\"", 
-				limit->data);
-			return -1;
-		}
-		if (random_buf->min_free_disk >= 100) {
-			gf_log ("random", GF_LOG_ERROR,
-				"check the \"option random.limits.min-free"
-				"-disk\", it should be percentage value");
-			return -1;
-		}
-      
-	} else {
-		gf_log ("random", GF_LOG_WARNING, 
-			"No option for limit min-free-disk given, "
-			"defaulting it to 5%%");
-		random_buf->min_free_disk = 
-			RANDOM_LIMITS_MIN_FREE_DISK_DEFAULT;
-	}
-  
-	limit = dict_get (xl->options, "scheduler.refresh-interval");
-	if (limit) {
-		if (gf_string2time (data_to_str (limit),
-				    &random_buf->refresh_interval) != 0) {
-			gf_log ("random", GF_LOG_ERROR, 
-				"invalid number format \"%s\" of "
-				"\"option random.refresh-interval\"", 
-				limit->data);
-			return -1;
-		}
-	} else {
-		random_buf->refresh_interval = RANDOM_REFRESH_INTERVAL_DEFAULT;
-	}
-  
-	while (trav_xl) {
-		index++;
-		trav_xl = trav_xl->next;
-	}
-	random_buf->child_count = index;
-	random_buf->array = GF_CALLOC (index,
-                                       sizeof (struct random_sched_struct),
-                                       gf_random_mt_random_sched_struct);
-	ERR_ABORT (random_buf->array);
-	trav_xl = xl->children;
-	index = 0;
-
-	while (trav_xl) {
-		random_buf->array[index].xl = trav_xl->xlator;
-		random_buf->array[index].eligible = 1;
-		trav_xl = trav_xl->next;
-		index++;
-	}
-	pthread_mutex_init (&random_buf->random_mutex, NULL);
-
-	// put it at the proper place  
-	*((long *)xl->private) = (long)random_buf; 
-	return 0;
-}
-
-static void
-random_fini (xlator_t *xl)
-{
-	struct random_struct *random_buf = NULL;
-
-	random_buf = (struct random_struct *)*((long *)xl->private);
-	pthread_mutex_destroy (&random_buf->random_mutex);
-	GF_FREE (random_buf->array);
-	GF_FREE (random_buf);
-}
-
-
-static int32_t 
-update_stat_array_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *xl,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct xlator_stats *trav_stats)
-{
-	int32_t idx = 0;
-	int32_t percent = 0;
-	struct random_struct *random_buf = NULL;
-	
-	random_buf = (struct random_struct *)*((long *)xl->private);
-
-	pthread_mutex_lock (&random_buf->random_mutex);
-	for (idx = 0; idx < random_buf->child_count; idx++) {
-		if (strcmp (random_buf->array[idx].xl->name, 
-			    (char *)cookie) == 0)
-			break;
-	}
-	pthread_mutex_unlock (&random_buf->random_mutex);
-
-	if (op_ret == 0) {
-		percent = ((trav_stats->free_disk *100) / 
-			   trav_stats->total_disk_size);
-		if (random_buf->min_free_disk > percent) {
-			random_buf->array[idx].eligible = 0;
-		} else {
-			random_buf->array[idx].eligible = 1;
-		}
-	} else {
-		random_buf->array[idx].eligible = 0;
-	}    
-
-	STACK_DESTROY (frame->root);
-	return 0;
-}
-
-static void 
-update_stat_array (xlator_t *xl)
-{
-	int32_t idx;
-	struct random_struct *random_buf = NULL;
-	call_frame_t *frame = NULL;
-	call_pool_t *pool = xl->ctx->pool;
-
-	random_buf = (struct random_struct *)*((long *)xl->private);
-	for (idx = 0; idx < random_buf->child_count; idx++) {
-		frame = create_frame (xl, pool);
-
-		STACK_WIND_COOKIE (frame,
-				   update_stat_array_cbk,
-				   random_buf->array[idx].xl->name,
-				   random_buf->array[idx].xl,
-				   (random_buf->array[idx].xl)->mops->stats,
-				   0);
-	}
-	return ;
-}
-
-static void 
-random_update (xlator_t *xl)
-{
-	struct timeval tv;
-	struct random_struct *random_buf = NULL;
-
-	random_buf = (struct random_struct *)*((long *)xl->private);
-
-	gettimeofday(&tv, NULL);
-	if (tv.tv_sec > (random_buf->refresh_interval + 
-			 random_buf->last_stat_entry.tv_sec)) {
-		update_stat_array (xl);
-		random_buf->last_stat_entry.tv_sec = tv.tv_sec;
-	}
-}
-
-static xlator_t *
-random_schedule (xlator_t *xl, const void *path)
-{
-	struct random_struct *random_buf = NULL;       
-	int32_t rand = 0;
-	int32_t try = 0;
-	
-	random_buf = (struct random_struct *)*((long *)xl->private);
-
-	rand = (int32_t) (1.0*random_buf->child_count * 
-			  (random() / (RAND_MAX + 1.0)));
-
-	random_update (xl);
-
-	while (!random_buf->array[rand].eligible) {
-		if (try++ > 100) {
-			/* there is a chance of this becoming a 
-			   infinite loop otherwise. */
-			break; 
-		}
-		rand = (int32_t) (1.0*random_buf->child_count * 
-				  (random() / (RAND_MAX + 1.0)));
-	}
-	return random_buf->array[rand].xl;
-}
-
-int32_t
-random_mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_random_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-/**
- * notify
- */
-void
-random_notify (xlator_t *xl, int32_t event, void *data)
-{
-	struct random_struct *random_buf = NULL;
-	int32_t idx = 0;
-  
-	random_buf = (struct random_struct *)*((long *)xl->private);
-	if (!random_buf)
-		return;
-
-	for (idx = 0; idx < random_buf->child_count; idx++) {
-		if (random_buf->array[idx].xl == (xlator_t *)data)
-			break;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_UP:
-	{
-		//random_buf->array[idx].eligible = 1;
-	}
-	break;
-	case GF_EVENT_CHILD_DOWN:
-	{
-		random_buf->array[idx].eligible = 0;
-	}
-	break;
-	default:
-	{
-		;
-	}
-	break;
-	}
-
-}
-
-struct sched_ops sched = {
-	.init     = random_init,
-	.fini     = random_fini,
-	.update   = random_update,
-	.schedule = random_schedule,
-	.notify   = random_notify,
-        .mem_acct_init = random_mem_acct_init,
-};
-
-struct volume_options options[] = {
-	{ .key   = { "scheduler.refresh-interval", 
-		     "random.refresh-interval" },  
-	  .type  = GF_OPTION_TYPE_TIME
-	},
-	{ .key   = { "scheduler.limits.min-free-disk", 
-		     "random.limits.min-free-disk" },  
-	  .type  = GF_OPTION_TYPE_PERCENT
-	},
-	{ .key = {NULL} }
-};
diff --git a/scheduler/random/src/random.h b/scheduler/random/src/random.h
deleted file mode 100644
index 154b1bdfb..000000000
--- a/scheduler/random/src/random.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _RANDOM_H
-#define _RANDOM_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include <sys/time.h>
-#include "scheduler.h"
-
-struct random_sched_struct {
-  xlator_t *xl;
-  unsigned char eligible;
-};
-
-struct random_struct {
-  int32_t child_count;
-  uint32_t refresh_interval;
-  uint32_t min_free_disk;
-  struct timeval last_stat_entry;
-  struct random_sched_struct *array;
-  pthread_mutex_t random_mutex;
-};
-
-#endif /* _RANDOM_H */
diff --git a/scheduler/rr/Makefile.am b/scheduler/rr/Makefile.am
deleted file mode 100644
index d471a3f92..000000000
--- a/scheduler/rr/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/scheduler/rr/src/Makefile.am b/scheduler/rr/src/Makefile.am
deleted file mode 100644
index 7e911c0ed..000000000
--- a/scheduler/rr/src/Makefile.am
+++ /dev/null
@@ -1,13 +0,0 @@
-sched_LTLIBRARIES = rr.la
-scheddir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/scheduler
-
-rr_la_LDFLAGS = -module -avoidversion
-
-rr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-rr_la_SOURCES = rr.c rr-options.c
-noinst_HEADERS = rr.h rr-options.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
diff --git a/scheduler/rr/src/rr-mem-types.h b/scheduler/rr/src/rr-mem-types.h
deleted file mode 100644
index cb05323cd..000000000
--- a/scheduler/rr/src/rr-mem-types.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __RR_MEM_TYPES_H__
-#define __RR_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_rr_mem_types_ {
-        gf_rr_mt_rr_t = gf_common_mt_end + 1,
-        gf_rr_mt_rr_subvolume_t,
-        gf_rr_mt_end
-};
-#endif
-
diff --git a/scheduler/rr/src/rr-options.c b/scheduler/rr/src/rr-options.c
deleted file mode 100644
index 505b1713e..000000000
--- a/scheduler/rr/src/rr-options.c
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include "scheduler.h"
-#include "rr-options.h"
-
-#define RR_LIMITS_MIN_FREE_DISK_OPTION_STRING  "scheduler.limits.min-free-disk"
-#define RR_LIMITS_MIN_FREE_DISK_VALUE_DEFAULT  15
-#define RR_LIMITS_MIN_FREE_DISK_VALUE_MIN      0
-#define RR_LIMITS_MIN_FREE_DISK_VALUE_MAX      100
-
-#define RR_REFRESH_INTERVAL_OPTION_STRING      "scheduler.refresh-interval"
-#define RR_REFRESH_INTERVAL_VALUE_DEFAULT      10
-
-#define RR_READ_ONLY_SUBVOLUMES_OPTION_STRING  "scheduler.read-only-subvolumes"
-
-#define LOG_ERROR(args...)      gf_log ("rr-options", GF_LOG_ERROR, ##args)
-#define LOG_WARNING(args...)    gf_log ("rr-options", GF_LOG_WARNING, ##args)
-
-static int 
-_rr_options_min_free_disk_validate (const char *value_string, uint32_t *n)
-{
-	uint32_t value = 0;
-  
-	if (value_string == NULL)
-	{
-		return -1;
-	}
-  
-	if (gf_string2percent (value_string, &value) != 0)
-	{
-		gf_log ("rr", 
-			GF_LOG_ERROR, 
-			"invalid number format [%s] of option [%s]", 
-			value_string, 
-			RR_LIMITS_MIN_FREE_DISK_OPTION_STRING);
-		return -1;
-	}
-  
-	if ((value <= RR_LIMITS_MIN_FREE_DISK_VALUE_MIN) || 
-	    (value >= RR_LIMITS_MIN_FREE_DISK_VALUE_MAX))
-	{
-		gf_log ("rr", 
-			GF_LOG_ERROR, 
-			"out of range [%d] of option [%s].  Allowed range is 0 to 100.", 
-			value, 
-			RR_LIMITS_MIN_FREE_DISK_OPTION_STRING);
-		return -1;
-	}
-  
-	*n = value;
-  
-	return 0;
-}
-
-static int 
-_rr_options_refresh_interval_validate (const char *value_string, uint32_t *n)
-{
-	uint32_t value = 0;
-  
-	if (value_string == NULL)
-	{
-		return -1;
-	}
-  
-	if (gf_string2time (value_string, &value) != 0)
-	{
-		gf_log ("rr", 
-			GF_LOG_ERROR, 
-			"invalid number format [%s] of option [%s]", 
-			value_string, 
-			RR_REFRESH_INTERVAL_OPTION_STRING);
-		return -1;
-	}
-  
-	*n = value;
-  
-	return 0;
-}
-
-static int 
-_rr_options_read_only_subvolumes_validate (const char *value_string, 
-					   char ***volume_list, 
-					   uint64_t *volume_count)
-{
-	char **vlist = NULL;
-	int vcount = 0;
-	int i = 0;
-  
-	if (value_string == NULL || volume_list == NULL || volume_count)
-	{
-		return -1;
-	}
-  
-	if (gf_strsplit (value_string, 
-			 ", ", 
-			 &vlist, 
-			 &vcount) != 0)
-	{
-		gf_log ("rr", 
-			GF_LOG_ERROR, 
-			"invalid subvolume list [%s] of option [%s]", 
-			value_string, 
-			RR_READ_ONLY_SUBVOLUMES_OPTION_STRING);
-		return -1;
-	}
-  
-	for (i = 0; i < vcount; i++)
-	{
-		if (gf_volume_name_validate (vlist[i]) != 0)
-		{
-			gf_log ("rr", 
-				GF_LOG_ERROR, 
-				"invalid subvolume name [%s] in [%s] of option [%s]", 
-				vlist[i], 
-				value_string, 
-				RR_READ_ONLY_SUBVOLUMES_OPTION_STRING);
-			goto free_exit;
-		}
-	}
-  
-	*volume_list = vlist;
-	*volume_count = vcount;
-  
-	return 0;
-  
- free_exit:
-	for (i = 0; i < vcount; i++)
-	{
-		GF_FREE (vlist[i]);
-	}
-	GF_FREE (vlist);
-  
-	return -1;
-}
-
-int 
-rr_options_validate (dict_t *options, rr_options_t *rr_options)
-{
-	char *value_string = NULL;
-  
-	if (options == NULL || rr_options == NULL)
-	{
-		return -1;
-	}
-  
-	if (dict_get (options, RR_LIMITS_MIN_FREE_DISK_OPTION_STRING))
-		if (data_to_str (dict_get (options, RR_LIMITS_MIN_FREE_DISK_OPTION_STRING)))
-			value_string = data_to_str (dict_get (options, 
-							      RR_LIMITS_MIN_FREE_DISK_OPTION_STRING));
-	if (value_string != NULL)
-	{
-		if (_rr_options_min_free_disk_validate (value_string, 
-							&rr_options->min_free_disk) != 0)
-		{
-			return -1;
-		}
-      
-		gf_log ("rr", 
-			GF_LOG_WARNING, 
-			"using %s = %d", 
-			RR_LIMITS_MIN_FREE_DISK_OPTION_STRING, 
-			rr_options->min_free_disk);
-	}
-	else 
-	{
-		rr_options->min_free_disk = RR_LIMITS_MIN_FREE_DISK_VALUE_DEFAULT;
-      
-		gf_log ("rr", GF_LOG_DEBUG, 
-			"using %s = %d [default]", 
-			RR_LIMITS_MIN_FREE_DISK_OPTION_STRING, 
-			rr_options->min_free_disk);
-	}
-  
-	value_string = NULL;
-	if (dict_get (options, RR_REFRESH_INTERVAL_OPTION_STRING))
-		value_string = data_to_str (dict_get (options, 
-						      RR_REFRESH_INTERVAL_OPTION_STRING));
-	if (value_string != NULL)
-	{
-		if (_rr_options_refresh_interval_validate (value_string, 
-							   &rr_options->refresh_interval) != 0)
-		{
-			return -1;
-		}
-      
-		gf_log ("rr", 
-			GF_LOG_WARNING, 
-			"using %s = %d", 
-			RR_REFRESH_INTERVAL_OPTION_STRING, 
-			rr_options->refresh_interval);
-	}
-	else 
-	{
-		rr_options->refresh_interval = RR_REFRESH_INTERVAL_VALUE_DEFAULT;
-      
-		gf_log ("rr", GF_LOG_DEBUG, 
-			"using %s = %d [default]", 
-			RR_REFRESH_INTERVAL_OPTION_STRING, 
-			rr_options->refresh_interval);
-	}
-  
-	value_string = NULL;
-	if (dict_get (options, RR_READ_ONLY_SUBVOLUMES_OPTION_STRING))
-		value_string = data_to_str (dict_get (options, 
-						      RR_READ_ONLY_SUBVOLUMES_OPTION_STRING));
-	if (value_string != NULL)
-	{
-		if (_rr_options_read_only_subvolumes_validate (value_string, 
-							       &rr_options->read_only_subvolume_list, 
-							       &rr_options->read_only_subvolume_count) != 0)
-		{
-			return -1;
-		}
-      
-		gf_log ("rr", 
-			GF_LOG_WARNING, 
-			"using %s = [%s]", 
-			RR_READ_ONLY_SUBVOLUMES_OPTION_STRING, 
-			value_string);
-	}
-  
-	return 0;
-}
-
-struct volume_options options[] = {
-	{ .key   = { "scheduler.refresh-interval", 
-		     "rr.refresh-interval" },  
-	  .type  = GF_OPTION_TYPE_TIME
-	},
-	{ .key   = { "scheduler.limits.min-free-disk", 
-		     "rr.limits.min-free-disk" },  
-	  .type  = GF_OPTION_TYPE_PERCENT
-	},
-	{ .key   = { "scheduler.read-only-subvolumes", 
-		     "rr.read-only-subvolumes" },  
-	  .type  = GF_OPTION_TYPE_ANY
-	},
-	{ .key = {NULL} }
-};
diff --git a/scheduler/rr/src/rr-options.h b/scheduler/rr/src/rr-options.h
deleted file mode 100644
index 1b0a1ef3d..000000000
--- a/scheduler/rr/src/rr-options.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _RR_OPTIONS_H
-#define _RR_OPTIONS_H
-
-struct rr_options
-{
-  uint32_t min_free_disk;
-  uint32_t refresh_interval;
-  char     **read_only_subvolume_list;
-  uint64_t read_only_subvolume_count;
-};
-typedef struct rr_options rr_options_t;
-
-int rr_options_validate (dict_t *options, rr_options_t *rr_options);
-
-#endif
diff --git a/scheduler/rr/src/rr.c b/scheduler/rr/src/rr.c
deleted file mode 100644
index e7b556e67..000000000
--- a/scheduler/rr/src/rr.c
+++ /dev/null
@@ -1,567 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <sys/time.h>
-#include <stdlib.h>
-
-#include <stdint.h>
-
-#include "scheduler.h"
-
-#include "rr-options.h"
-#include "rr.h"
-#include "rr-mem-types.h"
-
-#define RR_MIN_FREE_DISK_NOT_REACHED    0
-#define RR_MIN_FREE_DISK_REACHED        1
-
-#define RR_SUBVOLUME_OFFLINE    0
-#define RR_SUBVOLUME_ONLINE     1
-
-#define LOG_ERROR(args...)      gf_log ("rr", GF_LOG_ERROR, ##args)
-#define LOG_WARNING(args...)    gf_log ("rr", GF_LOG_WARNING, ##args)
-#define LOG_CRITICAL(args...)    gf_log ("rr", GF_LOG_CRITICAL, ##args)
-
-#define ROUND_ROBIN(index, count)    ((index + 1) % count)
-
-static int 
-_cleanup_rr (rr_t *rr)
-{
-	int i;
-  
-	if (rr == NULL)
-	{
-		return -1;
-	}
-  
-	if (rr->options.read_only_subvolume_list != NULL)
-	{
-		for (i = 0; i < rr->options.read_only_subvolume_count; i++)
-		{
-			GF_FREE (rr->options.read_only_subvolume_list[i]);
-		}
-		GF_FREE (rr->options.read_only_subvolume_list);
-	}
-  
-	GF_FREE (rr->subvolume_list);
-  
-	GF_FREE (rr);
-  
-	return 0;
-}
-
-int 
-rr_init (xlator_t *this_xl)
-{
-	rr_t *rr = NULL;
-	dict_t *options = NULL;
-	xlator_list_t *children = NULL;
-	uint64_t children_count = 0;
-	int i = 0;
-	int j = 0;
-  
-	if (this_xl == NULL)
-	{
-		return -1;
-	}
-  
-	if ((options = this_xl->options) == NULL)
-	{
-		return -1;
-	}
-  
-	if ((children = this_xl->children) == NULL)
-	{
-		return -1;
-	}
-  
-	if ((rr = GF_CALLOC (1, sizeof (rr_t), gf_rr_mt_rr_t)) == NULL)
-	{
-		return -1;
-	}
-  
-	if (rr_options_validate (options, &rr->options) != 0)
-	{
-		GF_FREE (rr);
-		return -1;
-	}
-  
-	for (i = 0; i < rr->options.read_only_subvolume_count; i++)
-	{
-		char found = 0;
-      
-		for (children = this_xl->children; 
-		     children != NULL; 
-		     children = children->next)
-		{
-			if (strcmp (rr->options.read_only_subvolume_list[i], 
-				    children->xlator->name) == 0)
-			{
-				found = 1;
-				break;
-			}
-		}
-      
-		if (!found)
-		{
-			LOG_ERROR ("read-only subvolume [%s] not found in volume list", 
-				   rr->options.read_only_subvolume_list[i]);
-			_cleanup_rr (rr);
-			return -1;
-		}
-	}
-  
-	for (children = this_xl->children; 
-	     children != NULL; 
-	     children = children->next)
-	{
-		children_count++;
-	}
-  
-	/* bala: excluding read_only_subvolumes */
-	if ((rr->subvolume_count = children_count - 
-	     rr->options.read_only_subvolume_count) == 0)
-	{
-		LOG_ERROR ("no writable volumes found for scheduling");
-		_cleanup_rr (rr);
-		return -1;
-	}
-  
-	if ((rr->subvolume_list = GF_CALLOC (rr->subvolume_count, 
-                                             sizeof (rr_subvolume_t),
-                                             gf_rr_mt_rr_subvolume_t)) == NULL)
-	{
-		_cleanup_rr (rr);
-		return -1;
-	}
-  
-	i = 0;
-	j = 0;
-	for (children = this_xl->children; 
-	     children != NULL; 
-	     children = children->next)
-	{
-		char found = 0;
-      
-		for (j = 0; j < rr->options.read_only_subvolume_count; j++)
-		{
-			if (strcmp (rr->options.read_only_subvolume_list[i], 
-				    children->xlator->name) == 0)
-			{
-				found = 1;
-				break;
-			}
-		}
-      
-		if (!found)
-		{
-			rr_subvolume_t *subvolume = NULL;
-	  
-			subvolume = &rr->subvolume_list[i];
-	  
-			subvolume->xl = children->xlator;
-			subvolume->free_disk_status = RR_MIN_FREE_DISK_NOT_REACHED;
-			subvolume->status = RR_SUBVOLUME_ONLINE;
-	  
-			i++;
-		}
-	}
-  
-	rr->schedule_index = UINT64_MAX;
-	rr->last_stat_fetched_time.tv_sec = 0;
-	rr->last_stat_fetched_time.tv_usec = 0;
-	pthread_mutex_init (&rr->mutex, NULL);
-  
-	*((long *)this_xl->private) = (long)rr;
-  
-	return 0;
-}
-
-void 
-rr_fini (xlator_t *this_xl)
-{
-	rr_t *rr = NULL;
-  
-	if (this_xl == NULL)
-	{
-		return;
-	}
-  
-	if ((rr = (rr_t *) *((long *)this_xl->private)) != NULL)
-	{
-		pthread_mutex_destroy (&rr->mutex);
-		_cleanup_rr (rr);
-		this_xl->private = NULL;
-	}
-  
-	return;
-}
-
-xlator_t *
-rr_schedule (xlator_t *this_xl, const void *path)
-{
-	rr_t *rr = NULL;
-	uint64_t next_schedule_index = 0;
-	int i = 0;
-  
-	if (this_xl == NULL || path == NULL)
-	{
-		return NULL;
-	}
-  
-	rr = (rr_t *) *((long *)this_xl->private);
-	next_schedule_index = ROUND_ROBIN (rr->schedule_index, 
-					   rr->subvolume_count);
-  
-	rr_update (this_xl);
-  
-	for (i = next_schedule_index; i < rr->subvolume_count; i++)
-	{
-		if (rr->subvolume_list[i].status == RR_SUBVOLUME_ONLINE && 
-		    rr->subvolume_list[i].status == RR_MIN_FREE_DISK_NOT_REACHED)
-		{
-			pthread_mutex_lock (&rr->mutex);
-			rr->schedule_index = i;
-			pthread_mutex_unlock (&rr->mutex);
-			return rr->subvolume_list[i].xl;
-		}
-	}
-  
-	for (i = 0; i < next_schedule_index; i++)
-	{
-		if (rr->subvolume_list[i].status == RR_SUBVOLUME_ONLINE && 
-		    rr->subvolume_list[i].status == RR_MIN_FREE_DISK_NOT_REACHED)
-		{
-			pthread_mutex_lock (&rr->mutex);
-			rr->schedule_index = i;
-			pthread_mutex_unlock (&rr->mutex);
-			return rr->subvolume_list[i].xl;
-		}
-	}
-  
-	for (i = next_schedule_index; i < rr->subvolume_count; i++)
-	{
-		if (rr->subvolume_list[i].status == RR_SUBVOLUME_ONLINE)
-		{
-			pthread_mutex_lock (&rr->mutex);
-			rr->schedule_index = i;
-			pthread_mutex_unlock (&rr->mutex);
-			return rr->subvolume_list[i].xl;
-		}
-	}
-  
-	for (i = 0; i < next_schedule_index; i++)
-	{
-		if (rr->subvolume_list[i].status == RR_SUBVOLUME_ONLINE)
-		{
-			pthread_mutex_lock (&rr->mutex);
-			rr->schedule_index = i;
-			pthread_mutex_unlock (&rr->mutex);
-			return rr->subvolume_list[i].xl;
-		}
-	}
-  
-	return NULL;
-}
-
-void
-rr_update (xlator_t *this_xl)
-{
-	rr_t *rr = NULL;
-	struct timeval ctime = {0, 0};
-	int i = 0;
-  
-	if (this_xl == NULL)
-	{
-		return ;
-	}
-  
-	if ((rr = (rr_t *) *((long *)this_xl->private)) == NULL)
-	{
-		return ;
-	}
-  
-	if (gettimeofday (&ctime, NULL) != 0)
-	{
-		return ;
-	}
-  
-	if (ctime.tv_sec > (rr->options.refresh_interval + 
-			    rr->last_stat_fetched_time.tv_sec))
-	{
-		pthread_mutex_lock (&rr->mutex);
-		rr->last_stat_fetched_time = ctime;
-		pthread_mutex_unlock (&rr->mutex);
-      
-		for (i = 0; i < rr->subvolume_count; i++)
-		{
-			xlator_t *subvolume_xl = NULL;
-			call_frame_t *frame = NULL;
-			call_pool_t *pool = NULL;
-	  
-			subvolume_xl = rr->subvolume_list[i].xl;
-	  
-			pool = this_xl->ctx->pool;
-	  
-			frame = create_frame (this_xl, pool);
-
-			STACK_WIND_COOKIE (frame,
-					   rr_update_cbk, 
-					   subvolume_xl->name, 
-					   subvolume_xl, 
-					   subvolume_xl->mops->stats, 
-					   0);
-		}
-	}
-  
-	return ;
-}
-
-int 
-rr_update_cbk (call_frame_t *frame, 
-	       void *cookie, 
-	       xlator_t *this_xl, 
-	       int32_t op_ret, 
-	       int32_t op_errno, 
-	       struct xlator_stats *stats)
-{
-	rr_t *rr = NULL;
-	rr_subvolume_t *subvolume = NULL;
-	uint8_t free_disk_percent = 0;
-	int i = 0;
-  
-	if (frame == NULL)
-	{
-		return -1;
-	}
-  
-	if (cookie == NULL || this_xl == NULL)
-	{
-		STACK_DESTROY (frame->root);
-		return -1;
-	}
-  
-	if (op_ret == 0 && stats == NULL)
-	{
-		LOG_CRITICAL ("fatal! op_ret is 0 and stats is NULL.  "
-			      "Please report this to <gluster-devel@nongnu.org>");
-		STACK_DESTROY (frame->root);
-		return -1;
-	}
-  
-	if ((rr = (rr_t *) *((long *)this_xl->private)) == NULL)
-	{
-		STACK_DESTROY (frame->root);
-		return -1;
-	}
-  
-	for (i = 0; i < rr->subvolume_count; i++)
-	{
-		if (rr->subvolume_list[i].xl->name == (char *) cookie)
-		{
-			subvolume = &rr->subvolume_list[i];
-			break;
-		}
-	}
-  
-	if (subvolume == NULL)
-	{
-		LOG_ERROR ("unknown cookie [%s]", (char *) cookie);
-		STACK_DESTROY (frame->root);
-		return -1;
-	}
-  
-	if (op_ret == 0)
-	{
-		free_disk_percent = (stats->free_disk * 100) / stats->total_disk_size;
-		if (free_disk_percent > rr->options.min_free_disk)
-		{
-			if (subvolume->free_disk_status != RR_MIN_FREE_DISK_NOT_REACHED)
-			{
-				pthread_mutex_lock (&rr->mutex);
-				subvolume->free_disk_status = RR_MIN_FREE_DISK_NOT_REACHED;
-				pthread_mutex_unlock (&rr->mutex);
-				LOG_WARNING ("subvolume [%s] is available with free space for scheduling", 
-					     subvolume->xl->name);
-			}
-		}
-		else
-		{
-			if (subvolume->free_disk_status != RR_MIN_FREE_DISK_REACHED)
-			{
-				pthread_mutex_lock (&rr->mutex);
-				subvolume->free_disk_status = RR_MIN_FREE_DISK_REACHED;
-				pthread_mutex_unlock (&rr->mutex);
-				LOG_WARNING ("subvolume [%s] reached minimum disk space requirement", 
-					     subvolume->xl->name);
-			}
-		}
-	}
-	else 
-	{
-		pthread_mutex_lock (&rr->mutex);
-		subvolume->status = RR_SUBVOLUME_OFFLINE;
-		pthread_mutex_unlock (&rr->mutex);
-		LOG_ERROR ("unable to get subvolume [%s] status information and "
-			   "scheduling is disabled", 
-			   subvolume->xl->name);
-	}
-  
-	STACK_DESTROY (frame->root);
-	return 0;
-}
-
-void
-rr_notify (xlator_t *this_xl, int32_t event, void *data)
-{
-	rr_t *rr = NULL;
-	rr_subvolume_t *subvolume = NULL;
-	xlator_t *subvolume_xl = NULL;
-	int i = 0, ret = 0;
-	call_frame_t *frame = NULL;
-	call_pool_t *pool = NULL;
-	dict_t *xattr = get_new_dict ();
-	int32_t version[1] = {1};
-
-	if (this_xl == NULL || data == NULL) {
-		return ;
-	}
-  
-	if ((rr = (rr_t *) *((long *)this_xl->private)) == NULL) {
-		return ;
-	}
-  
-	subvolume_xl = (xlator_t *) data;
-  
-	for (i = 0; i < rr->subvolume_count; i++) {
-		if (rr->subvolume_list[i].xl == subvolume_xl) {
-			subvolume = &rr->subvolume_list[i];
-			break;
-		}
-	}
-  
-	switch (event) {
-	case GF_EVENT_CHILD_UP:
-		/* Seeding, to be done only once */
-		if (rr->first_time && (i == rr->subvolume_count)) {
-			loc_t loc = {0,};
-			xlator_t *trav = NULL;
-
-			pool = this_xl->ctx->pool;
-			frame = create_frame (this_xl, pool);
-			ret = dict_set_bin (xattr, "trusted.glusterfs.scheduler.rr",
-					    version, sizeof (int32_t));
-			if (-1 == ret) {
-				gf_log (this_xl->name, GF_LOG_ERROR, "rr seed setting failed");
-			}
-			if (xattr)
-				dict_ref (xattr);
-			
-			loc.path = gf_strdup ("/");
-			for (trav = this_xl->parents->xlator; trav; trav = trav->parents->xlator) {
-				if (trav->itable) {
-					loc.inode = trav->itable->root;
-					break;
-				}
-			}
-			STACK_WIND (frame,
-				    rr_notify_cbk,
-				    (xlator_t *)data,
-				    ((xlator_t *)data)->fops->xattrop,
-				    &loc,
-				    GF_XATTROP_ADD_ARRAY,
-				    xattr);
-	  
-			if (xattr)
-				dict_unref (xattr);
-
-			rr->first_time = 0;
-		}
-		if (subvolume) {
-			pthread_mutex_lock (&rr->mutex);
-			subvolume->status = RR_SUBVOLUME_ONLINE;
-			pthread_mutex_unlock (&rr->mutex);
-		}
-		break;
-	case GF_EVENT_CHILD_DOWN:
-		if (subvolume) {
-			pthread_mutex_lock (&rr->mutex);
-			subvolume->status = RR_SUBVOLUME_OFFLINE;
-			pthread_mutex_unlock (&rr->mutex);
-		}
-		break;
-	}
-  
-	return ;
-}
-
-int 
-rr_notify_cbk (call_frame_t *frame, 
-	       void *cookie, 
-	       xlator_t *this_xl, 
-	       int32_t op_ret, 
-	       int32_t op_errno,
-	       dict_t *xattr)
-{
-	rr_t *rr = NULL;
-	int32_t *index = NULL;
-	int32_t ret = -1;
-	void *tmp_index_ptr = NULL;
-
-	if (frame == NULL) 
-	{
-		return -1;
-	}
-  
-	if ((this_xl == NULL) || (op_ret == -1))
-	{
-		STACK_DESTROY (frame->root);
-		return -1;
-	}
-  
-	if ((rr = (rr_t *) *((long *)this_xl->private)) == NULL)
-	{
-		STACK_DESTROY (frame->root);
-		return -1;
-	}
-  
-	ret = dict_get_bin (xattr, "trusted.glusterfs.scheduler.rr", &tmp_index_ptr);
-	index = tmp_index_ptr;
-	if (ret == 0)
-		rr->schedule_index = (index[0] % rr->subvolume_count);
-	else
-		rr->schedule_index = 0;
-
-	STACK_DESTROY (frame->root);
-	return 0;
-}
-
-struct sched_ops sched = {
-	.init     = rr_init,
-	.fini     = rr_fini,
-	.update   = rr_update,
-	.schedule = rr_schedule,
-	.notify   = rr_notify
-};
-
diff --git a/scheduler/rr/src/rr.h b/scheduler/rr/src/rr.h
deleted file mode 100644
index 3bd95929b..000000000
--- a/scheduler/rr/src/rr.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _RR_H
-#define _RR_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "scheduler.h"
-#include <stdint.h>
-#include <sys/time.h>
-
-struct rr_subvolume
-{
-  xlator_t  *xl;
-  uint8_t   free_disk_status;
-  uint8_t   status;
-};
-typedef struct rr_subvolume rr_subvolume_t;
-
-struct rr
-{
-  rr_options_t    options;
-  rr_subvolume_t  *subvolume_list;
-  uint64_t        subvolume_count;
-  uint64_t        schedule_index;
-  struct timeval  last_stat_fetched_time;
-  pthread_mutex_t mutex;
-  char            first_time;
-};
-typedef struct rr rr_t;
-
-int rr_init (xlator_t *this_xl);
-void rr_fini (xlator_t *this_xl);
-xlator_t *rr_schedule (xlator_t *this_xl, const void *path);
-void rr_update (xlator_t *this_xl);
-int rr_update_cbk (call_frame_t *frame, 
-		   void *cookie, 
-		   xlator_t *this_xl, 
-		   int32_t op_ret, 
-		   int32_t op_errno, 
-		   struct xlator_stats *stats);
-void rr_notify (xlator_t *this_xl, int32_t event, void *data);
-int rr_notify_cbk (call_frame_t *frame, 
-		   void *cookie, 
-		   xlator_t *this_xl, 
-		   int32_t op_ret, 
-		   int32_t op_errno,
-		   dict_t *xattr);
-
-#endif /* _RR_H */
diff --git a/scheduler/switch/Makefile.am b/scheduler/switch/Makefile.am
deleted file mode 100644
index d471a3f92..000000000
--- a/scheduler/switch/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/scheduler/switch/src/Makefile.am b/scheduler/switch/src/Makefile.am
deleted file mode 100644
index dc7d16d40..000000000
--- a/scheduler/switch/src/Makefile.am
+++ /dev/null
@@ -1,12 +0,0 @@
-sched_LTLIBRARIES = switch.la
-scheddir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/scheduler
-
-switch_la_LDFLAGS = -module -avoidversion
-
-switch_la_SOURCES = switch.c
-switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
diff --git a/scheduler/switch/src/switch-mem-types.h b/scheduler/switch/src/switch-mem-types.h
deleted file mode 100644
index 7039b035d..000000000
--- a/scheduler/switch/src/switch-mem-types.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __SWITCH_MEM_TYPES_H__
-#define __SWITCH_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_switch_mem_types_ {
-        gf_switch_mt_switch_struct = gf_common_mt_end + 1,
-        gf_switch_mt_switch_sched_struct,
-        gf_switch_mt_switch_sched_array,
-        gf_switch_mt_end
-};
-#endif
-
diff --git a/scheduler/switch/src/switch.c b/scheduler/switch/src/switch.c
deleted file mode 100644
index 1362e5cc9..000000000
--- a/scheduler/switch/src/switch.c
+++ /dev/null
@@ -1,451 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <sys/time.h>
-#include <stdlib.h>
-#include <fnmatch.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "scheduler.h"
-#include "switch-mem-types.h"
-
-struct switch_sched_array {
-	xlator_t *xl;
-	int32_t   eligible;
-	int32_t   considered;
-};
-
-/* Select one of this struct based on the path's pattern match */
-struct switch_sched_struct {
-	struct switch_sched_struct *next;
-	struct switch_sched_array  *array;
-	char                        path_pattern[256];
-	int32_t                     node_index; /* Index of the node in 
-						   this pattern. */
-	int32_t                     num_child;  /* Total num of child nodes 
-						   with this pattern. */
-};
-
-struct switch_struct {
-	struct switch_sched_struct *cond;
-	struct switch_sched_array  *array;
-	pthread_mutex_t             switch_mutex;
-	int32_t                     child_count;
-};
-
-/* This function should return child node as '*:subvolumes' is inserterd */
-static xlator_t *
-switch_get_matching_xl (const char *path, struct switch_sched_struct *cond)
-{
-	struct switch_sched_struct *trav      = cond;
-	char                       *pathname  = gf_strdup (path);
-	int                         index     = 0;
-
-	while (trav) {
-		if (fnmatch (trav->path_pattern, 
-			     pathname, FNM_NOESCAPE) == 0) {
-			GF_FREE (pathname);
-			trav->node_index %= trav->num_child;
-			index = (trav->node_index++) % trav->num_child;
-			return trav->array[index].xl;
-		}
-		trav = trav->next;
-	}
-	GF_FREE (pathname);
-	return NULL;
-}
-
-static int32_t
-gf_unify_valid_child (const char *child, 
-                      xlator_t *xl)
-{
-        xlator_list_t *children = NULL, *prev = NULL;
-        int32_t  ret = 0;
-
-        children = xl->children;
-        do {
-                if (!strcmp (child, children->xlator->name)) {
-                        ret = 1;
-                        break;
-                }
-                
-                prev = children;
-                children = prev->next;
-        } while (children);
-
-        return ret;
-}
-
-static int32_t
-switch_init (xlator_t *xl)
-{
-	int32_t index = 0;
-	data_t *data = NULL;
-	char *child = NULL;
-	char *tmp = NULL;
-	char *childs_data = NULL;
-	xlator_list_t *trav_xl = xl->children;
-	struct switch_struct *switch_buf = NULL;
-  
-	switch_buf = GF_CALLOC (1, sizeof (struct switch_struct),
-                                gf_switch_mt_switch_struct);
-	ERR_ABORT (switch_buf);
-
-	while (trav_xl) {
-		index++;
-		trav_xl = trav_xl->next;
-	}
-	switch_buf->child_count = index;
-	switch_buf->array = GF_CALLOC (index + 1,
-                                       sizeof (struct switch_sched_struct),
-                                       gf_switch_mt_switch_sched_struct);
-	ERR_ABORT (switch_buf->array);
-	trav_xl = xl->children;
-	index = 0;
-
-	while (trav_xl) {
-		switch_buf->array[index].xl = trav_xl->xlator;
-		switch_buf->array[index].eligible = 1;
-		trav_xl = trav_xl->next;
-		index++;
-	}
-
-	data = dict_get (xl->options, "scheduler.read-only-subvolumes");
-	if (data) {
-		childs_data = gf_strdup (data->data);
-		child = strtok_r (childs_data, ",", &tmp);
-		while (child) {
-			for (index = 1; 
-			     index < switch_buf->child_count; index++) {
-				if (strcmp (switch_buf->array[index - 1].xl->name, child) == 0) {
-					gf_log ("switch", GF_LOG_DEBUG, 
-						"Child '%s' is read-only", 
-						child);
-					memcpy (&(switch_buf->array[index-1]),
-						&(switch_buf->array[switch_buf->child_count - 1]), 
-						sizeof (struct switch_sched_array));
-					switch_buf->child_count--;
-					break;
-				}
-			}
-			child = strtok_r (NULL, ",", &tmp);
-		}
-		GF_FREE (childs_data);
-	}
-
-	data = dict_get (xl->options, "scheduler.local-volume-name");
-	if (data) {
-		/* Means, give preference to that node first */
-		gf_log ("switch", GF_LOG_DEBUG, 
-			"local volume defined as %s", data->data);
-
-		/* TODO: parse it properly, have an extra index to 
-		   specify that first */
-	}
-
-	/*  *jpg:child1,child2;*mpg:child3;*:child4,child5,child6 */
-	data = dict_get (xl->options, "scheduler.switch.case");
-	if (data) {
-		char *tmp_str = NULL; 
-		char *tmp_str1 = NULL;
-		char *dup_str = NULL;
-		char *switch_str = NULL;
-		char *pattern = NULL;
-		char *childs = NULL;
-		struct switch_sched_struct *switch_opt = NULL;
-		struct switch_sched_struct *trav = NULL;
-		/* Get the pattern for considering switch case. 
-		   "option block-size *avi:10MB" etc */
-		switch_str = strtok_r (data->data, ";", &tmp_str);
-		while (switch_str) {
-			dup_str = gf_strdup (switch_str);
-			switch_opt = 
-				GF_CALLOC (1, sizeof (struct switch_sched_struct), gf_switch_mt_switch_sched_struct);
-			ERR_ABORT (switch_opt);
-
-			/* Link it to the main structure */
-			if (switch_buf->cond) {
-				/* there are already few entries */
-				trav = switch_buf->cond;
-				while (trav->next)
-					trav = trav->next;
-				trav->next = switch_opt;
-			} else {
-				/* First entry */
-				switch_buf->cond = switch_opt;
-			}
-			pattern = strtok_r (dup_str, ":", &tmp_str1);
-			childs = strtok_r (NULL, ":", &tmp_str1);
-			if (strncmp (pattern, "*", 2) == 0) {
-				gf_log ("switch", GF_LOG_WARNING,
-					"'*' pattern will be taken by default "
-					"for all the unconfigured child nodes,"
-					" hence neglecting current option");
-				switch_str = strtok_r (NULL, ";", &tmp_str);
-				GF_FREE (dup_str);
-				continue;
-			}
-			memcpy (switch_opt->path_pattern, 
-				pattern, strlen (pattern));
-			if (childs) {
-				int32_t idx = 0;
-				char *tmp1 = NULL;
-				char *dup_childs = NULL;
-				/* TODO: get the list of child nodes for 
-				   the given pattern */
-				dup_childs = gf_strdup (childs);
-				child = strtok_r (dup_childs, ",", &tmp);
-				while (child) {
-                                        if (gf_unify_valid_child (child, xl)) {
-                                                idx++;
-                                                child = strtok_r (NULL, ",", &tmp);
-                                        } else {
-                                                gf_log ("switch", GF_LOG_ERROR,
-                                                        "%s is not a subvolume "
-                                                        "of %s. pattern can "
-                                                        "only be scheduled only"
-                                                        " to a subvolume of %s",
-                                                        child, xl->name,
-                                                        xl->name);
-                                                return -1;
-                                        }
-				}
-				GF_FREE (dup_childs);
-				child = strtok_r (childs, ",", &tmp1);
-				switch_opt->num_child = idx;
-				switch_opt->array = 
-					GF_CALLOC (1, idx * sizeof (struct switch_sched_array), gf_switch_mt_switch_sched_array);
-				ERR_ABORT (switch_opt->array);
-				idx = 0;
-				child = strtok_r (childs, ",", &tmp);
-				while (child) {
-					for (index = 1; 
-					     index < switch_buf->child_count; 
-					     index++) {
-						if (strcmp (switch_buf->array[index - 1].xl->name, 
-							    child) == 0) {
-							gf_log ("switch", 
-								GF_LOG_DEBUG,
-								"'%s' pattern will be scheduled to \"%s\"",
-								switch_opt->path_pattern, child);
-							/*
-							  if (switch_buf->array[index-1].considered) {
-							  gf_log ("switch", GF_LOG_DEBUG, 
-							  "ambiguity found, exiting");
-							  return -1;
-							  }
-							*/
-							switch_opt->array[idx].xl = switch_buf->array[index-1].xl;
-							switch_buf->array[index-1].considered = 1;
-							idx++;
-							break;
-						}
-					}
-					child = strtok_r (NULL, ",", &tmp1);
-				}
-			} else {
-				/* error */
-				gf_log ("switch", GF_LOG_ERROR, 
-					"Check \"scheduler.switch.case\" "
-					"option in unify volume. Exiting");
-				GF_FREE (switch_buf->array);
-				GF_FREE (switch_buf);
-				return -1;
-			}
-			GF_FREE (dup_str);
-			switch_str = strtok_r (NULL, ";", &tmp_str);
-		}
-	}
-	/* Now, all the pattern based considerations done, so for all the 
-	 * remaining pattern, '*' to all the remaining child nodes
-	 */
-	{
-		struct switch_sched_struct *switch_opt = NULL;
-		int32_t flag = 0;
-		int32_t index = 0;
-		for (index=0; index < switch_buf->child_count; index++) {
-			/* check for considered flag */
-			if (switch_buf->array[index].considered)
-				continue;
-			flag++;
-		}
-		if (!flag) {
-			gf_log ("switch", GF_LOG_ERROR,
-				"No nodes left for pattern '*'. Exiting.");
-			return -1;
-		}
-		switch_opt = GF_CALLOC (1, sizeof (struct switch_sched_struct), gf_switch_mt_switch_sched_struct);
-		ERR_ABORT (switch_opt);
-		if (switch_buf->cond) {
-			/* there are already few entries */
-			struct switch_sched_struct *trav = switch_buf->cond;
-			while (trav->next)
-				trav = trav->next;
-			trav->next = switch_opt;
-		} else {
-			/* First entry */
-			switch_buf->cond = switch_opt;
-		}
-		/* Add the '*' pattern to the array */
-		memcpy (switch_opt->path_pattern, "*", 2);
-		switch_opt->num_child = flag;
-		switch_opt->array = 
-			GF_CALLOC (1, flag * sizeof (struct switch_sched_array), gf_switch_mt_switch_sched_array);
-		ERR_ABORT (switch_opt->array);
-		flag = 0;
-		for (index=0; index < switch_buf->child_count; index++) {
-			/* check for considered flag */
-			if (switch_buf->array[index].considered)
-				continue;
-			gf_log ("switch", GF_LOG_DEBUG,
-				"'%s' pattern will be scheduled to \"%s\"",
-				switch_opt->path_pattern, 
-				switch_buf->array[index].xl->name);
-			switch_opt->array[flag].xl = 
-				switch_buf->array[index].xl;
-			switch_buf->array[index].considered = 1;
-			flag++;
-		}
-	}
-
-	pthread_mutex_init (&switch_buf->switch_mutex, NULL);
-
-	// put it at the proper place
-	*((long *)xl->private) = (long)switch_buf; 
-
-	return 0;
-}
-
-static void
-switch_fini (xlator_t *xl)
-{
-	/* TODO: free all the allocated entries */
-	struct switch_struct *switch_buf = NULL;
-	switch_buf = (struct switch_struct *)*((long *)xl->private);
-
-	pthread_mutex_destroy (&switch_buf->switch_mutex);
-	GF_FREE (switch_buf->array);
-	GF_FREE (switch_buf);
-}
-
-static xlator_t *
-switch_schedule (xlator_t *xl, const void *path)
-{
-	struct switch_struct *switch_buf = NULL;
-	switch_buf = (struct switch_struct *)*((long *)xl->private);
-
-	return switch_get_matching_xl (path, switch_buf->cond);
-}
-
-
-/**
- * notify
- */
-void
-switch_notify (xlator_t *xl, int32_t event, void *data)
-{
-	/* TODO: This should be checking in switch_sched_struct */
-#if 0
-	struct switch_struct *switch_buf = NULL;
-	int32_t idx = 0;
-
-	switch_buf = (struct switch_struct *)*((long *)xl->private);
-	if (!switch_buf)
-		return;
-
-	for (idx = 0; idx < switch_buf->child_count; idx++) {
-		if (switch_buf->array[idx].xl == (xlator_t *)data)
-			break;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_UP:
-	{
-		switch_buf->array[idx].eligible = 1;
-	}
-	break;
-	case GF_EVENT_CHILD_DOWN:
-	{
-		switch_buf->array[idx].eligible = 0;
-	}
-	break;
-	default:
-	{
-		;
-	}
-	break;
-	}
-#endif
-}
-
-static void 
-switch_update (xlator_t *xl)
-{
-	return;
-}
-        
-int32_t
-switch_mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_switch_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-struct sched_ops sched = {
-	.init     = switch_init,
-	.fini     = switch_fini,
-	.update   = switch_update,
-	.schedule = switch_schedule,
-	.notify   = switch_notify,
-        .mem_acct_init = switch_mem_acct_init,
-};
-
-struct volume_options options[] = {
-	{ .key   = { "scheduler.read-only-subvolumes" ,
-		     "switch.read-only-subvolumes"},  
-	  .type  = GF_OPTION_TYPE_ANY
-	},
-	{ .key   = { "scheduler.local-volume-name",
-		     "switch.nufa.local-volume-name" },
-	  .type  = GF_OPTION_TYPE_XLATOR
-	},
-	{ .key   = { "scheduler.switch.case",
-		     "switch.case" },
-	  .type  = GF_OPTION_TYPE_ANY
-	},
-	{ .key = {NULL} }
-};
diff --git a/smoke.sh b/smoke.sh
deleted file mode 100755
index 5080f59ca..000000000
--- a/smoke.sh
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/bin/bash
-
-set -e;
-
-M=/mnt;
-P=/build;
-H=$(hostname);
-T=600;
-V=patchy;
-
-
-function cleanup()
-{
-    killall -15 glusterfs glusterfsd glusterd glusterd 2>&1 || true;
-    killall -9 glusterfs glusterfsd glusterd glusterd 2>&1 || true;
-    umount -l $M 2>&1 || true;
-    rm -rf /etc/glusterd $P/export;
-}
-
-function start_fs()
-{
-    mkdir -p $P/export;
-    chmod 0755 $P/export;
-
-    glusterd;
-    gluster --mode=script volume create $V replica 2 $H:$P/export/export{1,2,3,4};
-    gluster volume start $V;
-    glusterfs -s $H --volfile-id $V $M;
-#    mount -t glusterfs $H:/$V $M;
-}
-
-
-function run_tests()
-{
-    cd $M;
-
-    (sleep 1; dbench -s -t 60 10 >/dev/null) &
-
-    (sleep 1; /opt/qa/tools/posix_compliance.sh) &
-
-    wait %2
-    wait %3
-
-    rm -rf clients;
-
-    cd -;
-}
-
-
-function watchdog ()
-{
-    # insurance against hangs during the test
-
-    sleep $1;
-
-    echo "Kicking in watchdog after $1 secs";
-
-    cleanup;
-}
-
-
-function finish ()
-{
-    cleanup;
-    kill %1;
-}
-
-function main ()
-{
-    cleanup;
-
-    watchdog $T &
-
-    trap finish EXIT;
-
-    set -x;
-
-    start_fs;
-
-    run_tests;
-}
-
-main "$@";
diff --git a/tests/README.md b/tests/README.md
new file mode 100644
index 000000000..2b5ed8dcd
--- /dev/null
+++ b/tests/README.md
@@ -0,0 +1,27 @@
+Regression tests framework for GlusterFS
+========================================
+
+## Prereq
+- Build and install the version of glusterfs with your changes. Make
+  sure the installed version is accessible from $PATH.
+
+## How-To
+- To mount glusterfs, NEVER use 'mount -t glusterfs', instead use
+  'glusterfs -s ' method. This is because with the patch build setup
+  doesnot install the /sbin/mount.glusterfs necessary, where as the
+  glusterfs binary will be accessible with $PATH, and will pick the
+  right version.
+- (optional) Set environment variables to specify location of
+  export directories and mount points. Unless you have special
+  requirements, the defaults should just work. The variables
+  themselves can be found at the top of tests/include.rc. All
+  of them can be overriden with environment variables.
+
+## Usage
+- Execute `/usr/share/glusterfs/run-tests.sh` as root.
+
+- If some test cases fail, report to GlusterFS community at
+  `gluster-devel@nongnu.org`.
+
+## Reminder
+- BE WARNED THAT THE TEST CASES DELETE /var/lib/glusterd/* !!!
+\ No newline at end of file
diff --git a/tests/afr.rc b/tests/afr.rc
new file mode 100644
index 000000000..60ea1b7f1
--- /dev/null
+++ b/tests/afr.rc
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+#count the number of entries marked for self-heal
+#in brick $1's index
+
+function count_sh_entries()
+{
+    val1=0
+    for g in `ls $1/.glusterfs/indices/xattrop`
+    do
+	val1=$(( val1 + 1 ))
+    done
+
+    echo $val1;
+}
diff --git a/tests/basic/bd.t b/tests/basic/bd.t
new file mode 100755
index 000000000..eb6305414
--- /dev/null
+++ b/tests/basic/bd.t
@@ -0,0 +1,131 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+function execute()
+{
+        cmd=$1
+        shift
+        ${cmd} $@ >/dev/null 2>&1
+}
+
+function bd_cleanup()
+{
+        execute vgremove -f ${V0}
+        execute pvremove ${ld}
+        execute losetup -d ${ld}
+        execute rm ${BD_DISK}
+        cleanup
+}
+
+function check()
+{
+        if [ $? -ne 0 ]; then
+                echo prerequsite $@ failed
+                bd_cleanup
+                exit
+        fi
+}
+
+SIZE=256 #in MB
+
+bd_cleanup;
+
+## Configure environment needed for BD backend volumes
+## Create a file with configured size and
+## set it as a temporary loop device to create
+## physical volume & VG. These are basic things needed
+## for testing BD xlator if anyone of these steps fail,
+## test script exits
+function configure()
+{
+    GLDIR=`$CLI system:: getwd`
+    BD_DISK=${GLDIR}/bd_disk
+
+    execute truncate -s${SIZE}M ${BD_DISK}
+    check ${BD_DISK} creation
+
+    execute losetup -f
+    check losetup
+    ld=`losetup -f`
+
+    execute losetup ${ld} ${BD_DISK}
+    check losetup ${BD_DISK}
+    execute pvcreate -f ${ld}
+    check pvcreate ${ld}
+    execute vgcreate ${V0} ${ld}
+    check vgcreate ${V0}
+    execute lvcreate --thin ${V0}/pool --size 128M
+}
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+function volume_type()
+{
+        getfattr -n volume.type $M0/. --only-values --absolute-names -e text
+}
+
+TEST glusterd
+TEST pidof glusterd
+configure
+
+TEST $CLI volume create $V0 ${H0}:/$B0/$V0?${V0}
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status'
+
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0
+EXPECT '1' volume_type
+
+## Create posix file
+TEST touch $M0/posix
+
+TEST touch $M0/lv
+gfid=`getfattr -n glusterfs.gfid.string $M0/lv --only-values --absolute-names`
+TEST setfattr -n user.glusterfs.bd -v "lv:4MB" $M0/lv
+# Check if LV is created
+TEST stat /dev/$V0/${gfid}
+
+## Create filesystem
+sleep 1
+TEST mkfs.ext4 -qF $M0/lv
+# Cloning
+TEST touch $M0/lv_clone
+gfid=`getfattr -n glusterfs.gfid.string $M0/lv_clone --only-values --absolute-names`
+TEST setfattr -n clone -v ${gfid} $M0/lv
+TEST stat /dev/$V0/${gfid}
+
+sleep 1
+## Check mounting
+TEST mount -o loop $M0/lv $M1
+umount $M1
+
+# Snapshot
+TEST touch $M0/lv_sn
+gfid=`getfattr -n glusterfs.gfid.string $M0/lv_sn --only-values --absolute-names`
+TEST setfattr -n snapshot -v ${gfid} $M0/lv
+TEST stat /dev/$V0/${gfid}
+
+# Merge
+sleep 1
+TEST setfattr -n merge -v "$M0/lv_sn" $M0/lv_sn
+TEST ! stat $M0/lv_sn
+TEST ! stat /dev/$V0/${gfid}
+
+
+rm $M0/* -f
+
+TEST umount $M0
+TEST $CLI volume stop ${V0}
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+TEST $CLI volume delete ${V0}
+
+bd_cleanup
diff --git a/tests/basic/cdc.t b/tests/basic/cdc.t
new file mode 100755
index 000000000..4cd915aa9
--- /dev/null
+++ b/tests/basic/cdc.t
@@ -0,0 +1,135 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+## Create a volume with one brick
+TEST $CLI volume create $V0 $H0:$B0/${V0}1;
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT '1' brick_count $V0
+
+## Turn off performance translators
+## This is required for testing readv calls
+TEST $CLI volume set $V0 performance.io-cache off
+EXPECT 'off' volinfo_field $V0 'performance.io-cache'
+TEST $CLI volume set $V0 performance.quick-read off
+EXPECT 'off' volinfo_field $V0 'performance.quick-read'
+
+TEST $CLI volume set $V0 strict-write-ordering on
+EXPECT 'on' volinfo_field $V0 'performance.strict-write-ordering'
+
+## Turn on cdc xlator by setting features.compress to on
+TEST $CLI volume set $V0 compress on
+EXPECT 'on' volinfo_field $V0 'features.compress'
+EXPECT 'server' volinfo_field $V0 'compress.mode'
+
+## Make sure that user cannot change compress.mode
+## This would break the cdc xlator if allowed!
+TEST $CLI volume set $V0 compress.mode client
+EXPECT 'server' volinfo_field $V0 'compress.mode'
+
+## Turn on compress.debug option
+## This will dump compressed data onto disk as gzip file
+## This is used to check if compression actually happened
+TEST $CLI volume set $V0 compress.debug on
+EXPECT 'on' volinfo_field $V0 'compress.debug'
+
+## Start the volume
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+####################
+## Testing writev ##
+####################
+
+## Create a 1K file locally and find the md5sum
+TEST dd if=/dev/zero of=/tmp/cdc-orig count=1 bs=1K 2>/dev/null
+checksum[original-file]=`md5sum /tmp/cdc-orig | cut -d' ' -f1`
+
+## Copy the file to mountpoint and find its md5sum on brick
+TEST dd if=/tmp/cdc-orig of=$M0/cdc-server count=1 bs=1K 2>/dev/null
+checksum[brick-file]=`md5sum $B0/${V0}1/cdc-server | cut -d' ' -f1`
+
+## Uncompress the gzip dump file and find its md5sum
+EXPECT '/tmp/cdcdump.gz: application/x-gzip; charset=binary' file -i /tmp/cdcdump.gz
+TEST gunzip -f /tmp/cdcdump.gz
+checksum[dump-file-writev]=`md5sum /tmp/cdcdump | cut -d' ' -f1`
+
+## Check if all 3 checksums are same
+TEST test ${checksum[original-file]} = ${checksum[brick-file]}
+TEST test ${checksum[brick-file]} = ${checksum[dump-file-writev]}
+
+## Cleanup files
+TEST rm -f /tmp/cdcdump.gz
+
+###################
+## Testing readv ##
+###################
+
+## Copy file from mount point to client and find checksum
+TEST dd if=$M0/cdc-server of=/tmp/cdc-client count=1 bs=1K 2>/dev/null
+checksum[client-file]=`md5sum /tmp/cdc-client | cut -d' ' -f1`
+
+## Uncompress the gzip dump file and find its md5sum
+EXPECT '/tmp/cdcdump.gz: application/x-gzip; charset=binary' file -i /tmp/cdcdump.gz
+TEST gunzip -f /tmp/cdcdump.gz
+checksum[dump-file-readv]=`md5sum /tmp/cdcdump | cut -d' ' -f1`
+
+## Check if all 3 checksums are same
+TEST test ${checksum[brick-file]} = ${checksum[client-file]}
+TEST test ${checksum[client-file]} = ${checksum[dump-file-readv]}
+
+## Cleanup files and unmount
+TEST rm -f /tmp/cdc* $M0/cdc*
+TEST umount $M0
+
+## Stop the volume
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+## Turn on compress.min-size and set it to 100 bytes
+## Compression should not take place if file size
+## is less than 100 bytes
+TEST $CLI volume set $V0 compress.min-size 100
+EXPECT '100' volinfo_field $V0 'compress.min-size'
+
+## Start the volume
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+## Create a file of size 99 bytes on mountpoint
+## This is should not be compressed
+TEST dd if=/dev/zero of=$M0/cdc-small count=1 bs=99 2>/dev/null
+TEST ! test -e /tmp/cdcdump.gz
+
+## Cleanup files and unmount
+TEST rm -f /tmp/cdc* $M0/cdc*
+TEST umount $M0
+
+## Reset the compress options
+TEST $CLI volume reset $V0 compress.debug
+TEST $CLI volume reset $V0 compress.min-size
+TEST $CLI volume reset $V0 compress.mode
+TEST $CLI volume reset $V0 features.compress
+
+## Stop the volume
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+## Delete the volume
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/basic/file-snapshot.t b/tests/basic/file-snapshot.t
new file mode 100755
index 000000000..36908192b
--- /dev/null
+++ b/tests/basic/file-snapshot.t
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/$V0;
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST $CLI volume set $V0 features.file-snapshot on;
+
+TEST $CLI volume set $V0 performance.quick-read off;
+TEST $CLI volume set $V0 performance.io-cache off;
+TEST glusterfs -s $H0 --volfile-id $V0 $M0 --attribute-timeout=0;
+
+TEST touch $M0/big-file;
+
+TEST setfattr -n trusted.glusterfs.block-format -v qcow2:10GB $M0/big-file;
+
+TEST ls -al $M0 # test readdirplus
+TEST [ `stat -c '%s' $M0/big-file` = 10737418240 ]
+
+echo 'ABCDEFGHIJ' > $M0/data-file1
+TEST dd if=$M0/data-file1 of=$M0/big-file conv=notrunc;
+TEST setfattr -n trusted.glusterfs.block-snapshot-create -v image1 $M0/big-file;
+
+echo '1234567890' > $M0/data-file2
+TEST dd if=$M0/data-file2 of=$M0/big-file conv=notrunc;
+TEST setfattr -n trusted.glusterfs.block-snapshot-create -v image2 $M0/big-file;
+
+TEST setfattr -n trusted.glusterfs.block-snapshot-goto -v image1 $M0/big-file;
+TEST dd if=$M0/big-file of=$M0/out-file1 bs=11 count=1;
+
+TEST setfattr -n trusted.glusterfs.block-snapshot-goto -v image2 $M0/big-file;
+TEST dd if=$M0/big-file of=$M0/out-file2 bs=11 count=1;
+
+TEST cmp $M0/data-file1 $M0/out-file1;
+TEST cmp $M0/data-file2 $M0/out-file2;
+
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/basic/mgmt_v3-locks.t b/tests/basic/mgmt_v3-locks.t
new file mode 100755
index 000000000..22ca27b9f
--- /dev/null
+++ b/tests/basic/mgmt_v3-locks.t
@@ -0,0 +1,121 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../cluster.rc
+
+function check_peers {
+        $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+
+function volume_count {
+        local cli=$1;
+        if [ $cli -eq '1' ] ; then
+                $CLI_1 volume info | grep 'Volume Name' | wc -l;
+        else
+                $CLI_2 volume info | grep 'Volume Name' | wc -l;
+        fi
+}
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI_1 volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+function two_diff_vols_create {
+        # Both volume creates should be successful
+        $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0 $H3:$B3/$V0 &
+        PID_1=$!
+
+        $CLI_2 volume create $V1 $H1:$B1/$V1 $H2:$B2/$V1 $H3:$B3/$V1 &
+        PID_2=$!
+
+        wait $PID_1 $PID_2
+}
+
+function two_diff_vols_start {
+        # Both volume starts should be successful
+        $CLI_1 volume start $V0 &
+        PID_1=$!
+
+        $CLI_2 volume start $V1 &
+        PID_2=$!
+
+        wait $PID_1 $PID_2
+}
+
+function two_diff_vols_stop_force {
+        # Force stop, so that if rebalance from the
+        # remove bricks is in progress, stop can
+        # still go ahead. Both volume stops should
+        # be successful
+        $CLI_1 volume stop $V0 force &
+        PID_1=$!
+
+        $CLI_2 volume stop $V1 force &
+        PID_2=$!
+
+        wait $PID_1 $PID_2
+}
+
+function same_vol_remove_brick {
+
+        # Running two same vol commands at the same time can result in
+        # two success', two failures, or one success and one failure, all
+        # of which are valid. The only thing that shouldn't happen is a
+        # glusterd crash.
+
+        local vol=$1
+        local brick=$2
+        $CLI_1 volume remove-brick $1 $2 start &
+        $CLI_2 volume remove-brick $1 $2 start
+}
+
+cleanup;
+
+TEST launch_cluster 3;
+TEST $CLI_1 peer probe $H2;
+TEST $CLI_1 peer probe $H3;
+
+EXPECT_WITHIN 20 2 check_peers
+
+two_diff_vols_create
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'Created' volinfo_field $V1 'Status';
+
+two_diff_vols_start
+EXPECT 'Started' volinfo_field $V0 'Status';
+EXPECT 'Started' volinfo_field $V1 'Status';
+
+same_vol_remove_brick $V0 $H2:$B2/$V0
+# Checking glusterd crashed or not after same volume remove brick
+# on both nodes.
+EXPECT_WITHIN 20 2 check_peers
+
+same_vol_remove_brick $V1 $H2:$B2/$V1
+# Checking glusterd crashed or not after same volume remove brick
+# on both nodes.
+EXPECT_WITHIN 20 2 check_peers
+
+$CLI_1 volume set $V0 diagnostics.client-log-level DEBUG &
+$CLI_1 volume set $V1 diagnostics.client-log-level DEBUG
+kill_glusterd 3
+$CLI_1 volume status $V0
+$CLI_2 volume status $V1
+$CLI_1 peer status
+EXPECT_WITHIN 20 1 check_peers
+EXPECT 'Started' volinfo_field $V0 'Status';
+EXPECT 'Started' volinfo_field $V1 'Status';
+
+TEST $glusterd_3
+$CLI_1 volume status $V0
+$CLI_2 volume status $V1
+$CLI_1 peer status
+#EXPECT_WITHIN 20 2 check_peers
+#EXPECT 'Started' volinfo_field $V0 'Status';
+#EXPECT 'Started' volinfo_field $V1 'Status';
+#two_diff_vols_stop_force
+#EXPECT_WITHIN 20 2 check_peers
+cleanup;
diff --git a/tests/basic/mount.t b/tests/basic/mount.t
new file mode 100755
index 000000000..90e522c5e
--- /dev/null
+++ b/tests/basic/mount.t
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+
+## Start and create a volume
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+
+## Make volume tightly consistent for metdata
+TEST $CLI volume set $V0 performance.stat-prefetch off;
+
+## Mount FUSE with caching disabled (read-write)
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+## Check consistent "rw" option
+TEST 'mount -t fuse.glusterfs | grep -E "^$H0:$V0 .+ \(rw,"';
+TEST 'grep -E "^$H0:$V0 .+ ,?rw," /proc/mounts';
+
+## Mount FUSE with caching disabled (read-only)
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 --read-only -s $H0 --volfile-id $V0 $M1;
+
+## Check consistent "ro" option
+TEST 'mount -t fuse.glusterfs | grep -E "^$H0:$V0 .+ \(ro,"';
+TEST 'grep -E "^$H0:$V0 .+ ,?ro,.+" /proc/mounts';
+
+## Wait for volume to register with rpc.mountd
+sleep 5;
+
+## Mount NFS
+TEST mount -t nfs -o nolock,soft,intr $H0:/$V0 $N0;
+
+
+## Test for consistent views between NFS and FUSE mounts
+## write access to $M1 should fail
+TEST ! stat $M0/newfile;
+TEST ! touch $M1/newfile;
+TEST touch $M0/newfile;
+TEST stat $M1/newfile;
+TEST stat $N0/newfile;
+TEST ! rm $M1/newfile;
+TEST rm $N0/newfile;
+TEST ! stat $M0/newfile;
+TEST ! stat $M1/newfile;
+
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/basic/nufa.t b/tests/basic/nufa.t
new file mode 100644
index 000000000..0d4c229a0
--- /dev/null
+++ b/tests/basic/nufa.t
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT '8' brick_count $V0
+
+TEST $CLI volume set $V0 nufa on;
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Mount FUSE with caching disabled (read-only)
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 --read-only -s $H0 --volfile-id $V0 $M1;
+
+## Wait for volume to register with rpc.mountd
+sleep 5;
+
+## Mount NFS
+TEST mount -t nfs -o nolock,soft,intr $H0:/$V0 $N0;
+
+cleanup;
diff --git a/tests/basic/posixonly.t b/tests/basic/posixonly.t
new file mode 100755
index 000000000..b9de317a4
--- /dev/null
+++ b/tests/basic/posixonly.t
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST mkdir -p $B0/posixonly
+cat > $B0/posixonly.vol <<EOF
+volume poisxonly
+  type storage/posix
+  option directory $B0/posixonly
+end-volume
+EOF
+
+TEST glusterfs -f $B0/posixonly.vol $M0;
+
+TEST touch $M0/filename;
+TEST stat $M0/filename;
+TEST mkdir $M0/dirname;
+TEST stat $M0/dirname;
+TEST touch $M0/dirname/filename;
+TEST stat $M0/dirname/filename;
+TEST ln $M0/dirname/filename $M0/dirname/linkname;
+TEST chown 100:100 $M0/dirname/filename;
+TEST chown 100:100 $M0/dirname;
+TEST rm -rf $M0/filename $M0/dirname;
+
+TEST umount $M0;
+
+cleanup;
diff --git a/tests/basic/pump.t b/tests/basic/pump.t
new file mode 100644
index 000000000..3faf06f05
--- /dev/null
+++ b/tests/basic/pump.t
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/${V0}0
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+cd $M0
+for i in {1..3}
+do
+        for j in {1..10}
+        do
+                dd if=/dev/urandom of=file$j bs=128K count=10 2>/dev/null 1>/dev/null
+        done
+        mkdir dir$i && cd dir$i
+done
+cd
+TEST umount $M0
+TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 start
+EXPECT_WITHIN 60 "Y" gd_is_replace_brick_completed $H0 $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 commit
+TEST $CLI volume stop $V0
+TEST diff -r --exclude=.glusterfs $B0/${V0}0 $B0/${V0}1
+
+files=""
+
+cd $B0/${V0}0
+for f in `find . -path ./.glusterfs -prune  -o -print`;
+do
+        if [ -d $f ]; then continue; fi
+        cmp $f $B0/${V0}1/$f
+        if [ $? -ne 0 ]; then
+                files="$files $f"
+        fi
+done
+
+EXPECT "" echo $files
+
+cleanup
diff --git a/tests/basic/quota.t b/tests/basic/quota.t
new file mode 100755
index 000000000..ef015a30d
--- /dev/null
+++ b/tests/basic/quota.t
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+function limit_on()
+{
+        local QUOTA_PATH=$1;
+        $CLI volume quota $V0 list | grep "$QUOTA_PATH" | awk '{print $2}'
+}
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT '8' brick_count $V0
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## ------------------------------
+## Verify quota commands
+## ------------------------------
+TEST $CLI volume quota $V0 enable
+
+TEST $CLI volume quota $V0 limit-usage /test_dir 100MB
+
+TEST $CLI volume quota $V0 limit-usage /test_dir/in_test_dir 150MB
+
+EXPECT "150MB" limit_on "/test_dir/in_test_dir";
+
+TEST $CLI volume quota $V0 remove /test_dir/in_test_dir
+
+EXPECT "100MB" limit_on "/test_dir";
+
+TEST $CLI volume quota $V0 disable
+## ------------------------------
+
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/basic/rpm.t b/tests/basic/rpm.t
new file mode 100755
index 000000000..a577726a8
--- /dev/null
+++ b/tests/basic/rpm.t
@@ -0,0 +1,109 @@
+#!/bin/bash
+#
+# This test will run mock and rebuild the srpm for the latest two EPEL version.
+# By default, the results and the chroots are deleted.
+#
+# When debugging is needed, make sure to set DEBUG=1 in the environment or this
+# script. When debugging is enabled, the resulting log files and chroots are
+# kept. With debugging enabled, this test will fail the regression test, and
+# all output is saved to rpmbuild-mock.log. Tests are run in parallel, so the
+# logfile may be difficult to read.
+#
+# chroots are configured in /etc/mock/*.cfg, with site-defaults.cfg as main
+# configuration file. The default for chroots is /var/lib/mock, but this
+# depends on the 'basedir' configuration option set in the mentioned files.
+#
+
+. $(dirname $0)/../include.rc
+
+# enable some extra debugging
+if [ -n "${DEBUG}" -a "${DEBUG}" != "0" ]
+then
+	exec &> rpmbuild-mock.log
+	set -x
+        MOCK_CLEANUP='--no-cleanup-after'
+else
+        MOCK_CLEANUP='--cleanup-after'
+fi
+
+# detect the branch we're based off
+if [ -n "${BRANCH}" ] ; then
+        # $BRANCH is set in the environment (by Jenkins or other)
+        GIT_PARENT="origin/${BRANCH}"
+else
+        # get a reference to the latest clean tree
+        GIT_PARENT=$(git describe --abbrev=0)
+fi
+
+# Filter out everything and what remains needs to be built
+BUILD_FILES=$(git diff --name-status ${GIT_PARENT} | grep -Ev '^M.*\.(c|h|py)' | awk {'print $2'})
+SELFTEST=$(grep -e 'tests/basic/rpm.t' <<< "${BUILD_FILES}")
+BUILD_FILES=$(grep -Ev '^tests/' <<< "${BUILD_FILES}")
+if [ -z "${BUILD_FILES}" -a -z "${SELFTEST}" ]
+then
+        # nothing affecting packaging changed, no need to retest rpmbuild
+        SKIP_TESTS
+        cleanup
+        exit 0
+fi
+
+# checkout the sources to a new directory to execute ./configure and all
+REPO=${PWD}
+COMMIT=$(git describe)
+mkdir rpmbuild-mock.d
+pushd rpmbuild-mock.d 2>/dev/null
+git clone -q -s file://${REPO} .
+git checkout -q -b rpm-test ${COMMIT}
+
+# build the glusterfs-*.tar.gz
+[ -e configure ] || ./autogen.sh 2>&1 > /dev/null
+TEST ./configure --enable-fusermount
+TEST make dist
+
+# build the glusterfs src.rpm
+ls extras
+TEST make -C extras/LinuxRPM testsrpm
+
+# build for the last two Fedora EPEL releases (x86_64 only)
+for MOCK_CONF in $(ls -x1 /etc/mock/*.cfg | egrep -e 'epel-[0-9]+-x86_64.cfg$' | tail -n2)
+do
+	EPEL_RELEASE=$(basename ${MOCK_CONF} .cfg)
+	# expand the mock command line
+	MOCK_CMD="/usr/bin/mock ${MOCK_CLEANUP} \
+		-r ${EPEL_RELEASE} --rebuild ${PWD}/*.src.rpm"
+
+	# write the mock command to a file, so that its easier to execute
+	cat << EOF > mock-${EPEL_RELEASE}.sh
+#!/bin/sh
+${MOCK_CMD}
+EOF
+	chmod +x mock-${EPEL_RELEASE}.sh
+
+	# root can not run 'mock', it needs to drop priviledges
+	if (groups | grep -q mock)
+	then
+		# the current user is in group 'mock'
+		${PWD}/mock-${EPEL_RELEASE}.sh &
+	else
+		# "su" might not work, using sudo instead
+		sudo -u mock -E ${PWD}/mock-${EPEL_RELEASE}.sh &
+	fi
+	sleep 5
+done
+
+# TAP and Prove aren't smart about loops
+TESTS_EXPECTED_IN_LOOP=2
+for mockjob in $(jobs -p)
+do
+	TEST_IN_LOOP wait ${mockjob}
+done
+
+# we could build for the last two Fedora releases too, but that is not
+# possible on EPEL-5/6 installations, Fedora 17 and newer have unmet
+# dependencies on the build-server :-/
+
+popd 2>/dev/null
+# only remove rpmbuild-mock.d if we're not debugging
+[ "${DEBUG}" = "0" ] && rm -rf rpmbuild-mock.d
+
+cleanup
diff --git a/tests/basic/self-heald.t b/tests/basic/self-heald.t
new file mode 100644
index 000000000..4468c881b
--- /dev/null
+++ b/tests/basic/self-heald.t
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1,2,3,4,5}
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+TEST $CLI volume set $V0 cluster.eager-lock off
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+TEST kill_brick $V0 $H0 $B0/${V0}0
+TEST kill_brick $V0 $H0 $B0/${V0}2
+TEST kill_brick $V0 $H0 $B0/${V0}4
+cd $M0
+HEAL_FILES=0
+for i in {1..10}
+do
+        dd if=/dev/urandom of=f bs=1M count=10 2>/dev/null
+        HEAL_FILES=$(($HEAL_FILES+1))
+        mkdir a; cd a;
+        HEAL_FILES=$(($HEAL_FILES+3)) #As many times as distribute subvols
+done
+HEAL_FILES=$(($HEAL_FILES + 3)) #Count the brick root dir
+
+cd ~
+EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0
+TEST ! $CLI volume heal $V0
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST ! $CLI volume heal $V0 info
+TEST ! $CLI volume heal $V0
+TEST $CLI volume start $V0 force
+TEST $CLI volume set $V0 cluster.self-heal-daemon on
+EXPECT_WITHIN 20 "Y" glustershd_up_status
+EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 0
+EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 2
+EXPECT_WITHIN 20 "1" afr_child_up_status_in_shd $V0 4
+TEST $CLI volume heal $V0
+sleep 5 #Until the heal-statistics command implementation
+#check that this heals the contents partially
+TEST [ $HEAL_FILES -gt $(afr_get_pending_heal_count $V0) ]
+
+TEST $CLI volume heal $V0 full
+EXPECT_WITHIN 30 "0" afr_get_pending_heal_count $V0
+cleanup
diff --git a/tests/basic/volume-snapshot.t b/tests/basic/volume-snapshot.t
new file mode 100755
index 000000000..35c748372
--- /dev/null
+++ b/tests/basic/volume-snapshot.t
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../cluster.rc
+. $(dirname $0)/../snapshot.rc
+
+V1="patchy2"
+
+function create_volumes() {
+        $CLI_1 volume create $V0 $H1:$L1 &
+        PID_1=$!
+
+        $CLI_2 volume create $V1 $H2:$L2 $H3:$L3 &
+        PID_2=$!
+
+        wait $PID_1 $PID_2
+}
+
+function create_snapshots() {
+        $CLI_1 snapshot create ${V0}_snap ${V0}&
+        PID_1=$!
+
+        $CLI_1 snapshot create ${V1}_snap ${V1}&
+        PID_2=$!
+
+        wait $PID_1 $PID_2
+}
+
+function delete_snapshots() {
+        $CLI_1 snapshot delete ${V0}_snap &
+        PID_1=$!
+
+        $CLI_1 snapshot delete ${V1}_snap &
+        PID_2=$!
+
+        wait $PID_1 $PID_2
+}
+cleanup;
+
+#Create cluster with 3 nodes
+TEST launch_cluster 3;
+TEST setup_lvm 3
+
+TEST $CLI_1 peer probe $H2;
+TEST $CLI_1 peer probe $H3;
+EXPECT_WITHIN 20 2 peer_count;
+
+create_volumes
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'Created' volinfo_field $V1 'Status';
+
+start_volumes 2
+EXPECT 'Started' volinfo_field $V0 'Status';
+EXPECT 'Started' volinfo_field $V1 'Status';
+
+#Snapshot Operations
+create_snapshots
+TEST snapshot_exists $V0 ${V0}_snap
+TEST snapshot_exists $V1 ${V1}_snap
+
+TEST $CLI_1 snapshot config $V0 snap-max-hard-limit 100
+TEST $CLI_1 snapshot config $V1 snap-max-hard-limit 100
+
+TEST mount -t glusterfs $H1:/snaps/${V0}_snap/${V0} $M0
+TEST umount -f $M0
+TEST mount -t glusterfs $H2:/snaps/${V1}_snap/${V1} $M0
+TEST umount -f $M0
+
+#Clean up
+delete_snapshots
+TEST ! snapshot_exists $V0 ${V0}_snap
+TEST ! snapshot_exists $V1 ${V1}_snap
+
+stop_force_volumes 2
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+EXPECT 'Stopped' volinfo_field $V1 'Status';
+
+delete_volumes 2
+TEST ! volume_exists $V0
+TEST ! volume_exists $V1
+
+cleanup;
diff --git a/tests/basic/volume-status.t b/tests/basic/volume-status.t
new file mode 100644
index 000000000..f4196ac30
--- /dev/null
+++ b/tests/basic/volume-status.t
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+TEST $CLI volume start $V0;
+
+sleep 2
+
+## Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+
+## Mount NFS
+TEST mount -t nfs -o vers=3,nolock,soft,intr $H0:/$V0 $N0;
+
+TEST $CLI volume status all
+TEST $CLI volume status $V0
+
+EXPECT_WITHIN 10 'Y' nfs_up_status
+EXPECT_WITHIN 10 'Y' glustershd_up_status
+function test_nfs_cmds () {
+    local ret=0
+    declare -a nfs_cmds=("clients" "mem" "inode" "callpool")
+    for cmd in ${nfs_cmds[@]}; do
+        $CLI volume status $V0 nfs $cmd
+        (( ret += $? ))
+    done
+    return $ret
+}
+
+function test_shd_cmds () {
+    local ret=0
+    declare -a shd_cmds=("mem" "inode" "callpool")
+    for cmd in ${shd_cmds[@]}; do
+        $CLI volume status $V0 shd $cmd
+        (( ret += $? ))
+    done
+    return $ret
+}
+
+function test_brick_cmds () {
+    local ret=0
+    declare -a cmds=("detail" "clients" "mem" "inode" "fd" "callpool")
+    for cmd in ${cmds[@]}; do
+        for i in {1..2}; do
+            $CLI volume status $V0 $H0:$B0/${V0}$i $cmd
+            (( ret += $? ))
+        done
+    done
+    return $ret
+}
+
+TEST test_shd_cmds;
+TEST test_nfs_cmds;
+TEST test_brick_cmds;
+
+cleanup;
+
diff --git a/tests/basic/volume.t b/tests/basic/volume.t
new file mode 100755
index 000000000..2f9096055
--- /dev/null
+++ b/tests/basic/volume.t
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT '8' brick_count $V0
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST $CLI volume add-brick $V0 $H0:$B0/${V0}{9,10,11,12};
+EXPECT '12' brick_count $V0
+
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}{1,2,3,4};
+EXPECT '8' brick_count $V0
+
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/859927/repl.t b/tests/bugs/859927/repl.t
new file mode 100755
index 000000000..73c86e7be
--- /dev/null
+++ b/tests/bugs/859927/repl.t
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+cleanup;
+
+TEST glusterd;
+TEST pidof glusterd
+
+#Tests for data-self-heal-algorithm option
+function create_setup_for_self_heal {
+        file=$1
+        kill_brick $V0 $H0 $B0/${V0}1
+        dd of=$file if=/dev/urandom bs=1M count=1 2>&1 > /dev/null
+        $CLI volume start $V0 force
+}
+
+function test_write {
+        dd of=$M0/a if=/dev/urandom bs=1k count=1 2>&1 > /dev/null
+}
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2};
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST $CLI volume set $V0 performance.stat-prefetch off
+TEST $CLI volume set $V0 client-log-level DEBUG
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+TEST $CLI volume start $V0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0;
+
+touch $M0/a
+
+TEST $CLI volume set $V0 cluster.data-self-heal-algorithm full
+EXPECT full volume_option $V0 cluster.data-self-heal-algorithm
+create_setup_for_self_heal $M0/a
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+ls -l $file 2>&1 > /dev/null
+TEST cmp $B0/${V0}1/a $B0/${V0}2/a
+
+TEST $CLI volume set $V0 cluster.data-self-heal-algorithm diff
+EXPECT diff volume_option $V0 cluster.data-self-heal-algorithm
+create_setup_for_self_heal $M0/a
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+ls -l $file 2>&1 > /dev/null
+TEST cmp $B0/${V0}1/a $B0/${V0}2/a
+
+TEST $CLI volume reset $V0 cluster.data-self-heal-algorithm
+create_setup_for_self_heal $M0/a
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+ls -l $file 2>&1 > /dev/null
+TEST cmp $B0/${V0}1/a $B0/${V0}2/a
+
+TEST ! $CLI volume set $V0 cluster.data-self-heal-algorithm ""
+
+#Tests for quorum-type option
+TEST ! $CLI volume set $V0 cluster.quorum-type ""
+TEST $CLI volume set $V0 cluster.quorum-type fixed
+EXPECT fixed volume_option $V0 cluster.quorum-type
+TEST $CLI volume set $V0 cluster.quorum-count 2
+kill_brick $V0 $H0 $B0/${V0}1
+TEST ! test_write
+TEST $CLI volume set $V0 cluster.quorum-type auto
+EXPECT auto volume_option $V0 cluster.quorum-type
+TEST ! test_write
+TEST $CLI volume set $V0 cluster.quorum-type none
+EXPECT none volume_option $V0 cluster.quorum-type
+TEST test_write
+TEST $CLI volume reset $V0 cluster.quorum-type
+TEST test_write
+cleanup;
diff --git a/tests/bugs/886998/strict-readdir.t b/tests/bugs/886998/strict-readdir.t
new file mode 100644
index 000000000..0de953e8a
--- /dev/null
+++ b/tests/bugs/886998/strict-readdir.t
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+function num_files_in_dir {
+        d=$1
+        ls $d | sort | uniq | wc -l
+}
+
+#Basic sanity tests for readdir functionality
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/r2d2_0 $H0:$B0/r2d2_1 $H0:$B0/r2d2_2 $H0:$B0/r2d2_3
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-server=$H0 --volfile-id=/$V0 $M0
+
+TEST touch $M0/{1..100}
+EXPECT "100" num_files_in_dir $M0
+
+TEST kill_brick $V0 $H0 $B0/r2d2_0
+TEST kill_brick $V0 $H0 $B0/r2d2_2
+EXPECT "100" num_files_in_dir $M0
+
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 2
+
+TEST kill_brick $V0 $H0 $B0/r2d2_1
+TEST kill_brick $V0 $H0 $B0/r2d2_3
+EXPECT "100" num_files_in_dir $M0
+
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 3
+
+TEST $CLI volume set $V0 cluster.strict-readdir on
+EXPECT "on" volinfo_field $V0 cluster.strict-readdir
+TEST kill_brick $V0 $H0 $B0/r2d2_0
+TEST kill_brick $V0 $H0 $B0/r2d2_2
+EXPECT "100" num_files_in_dir $M0
+
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 2
+
+TEST kill_brick $V0 $H0 $B0/r2d2_1
+TEST kill_brick $V0 $H0 $B0/r2d2_3
+EXPECT "100" num_files_in_dir $M0
+cleanup;
diff --git a/tests/bugs/949327.t b/tests/bugs/949327.t
new file mode 100644
index 000000000..7b0c5c51a
--- /dev/null
+++ b/tests/bugs/949327.t
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+function tmp_file_count()
+{
+echo $(ls -lh /tmp/tmp.* |  wc -l)
+}
+
+
+old_count=$(tmp_file_count);
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+TEST $CLI volume start $V0
+new_count=$(tmp_file_count);
+
+TEST [ "$old_count" -eq "$new_count" ]
+
+cleanup
diff --git a/tests/bugs/bug-000000.t b/tests/bugs/bug-000000.t
new file mode 100755
index 000000000..7f3d15c9d
--- /dev/null
+++ b/tests/bugs/bug-000000.t
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+
+cleanup;
diff --git a/tests/bugs/bug-1002207.t b/tests/bugs/bug-1002207.t
new file mode 100644
index 000000000..50b8c7d31
--- /dev/null
+++ b/tests/bugs/bug-1002207.t
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+dd if=/dev/zero of=$M0/file$i.data bs=1024 count=1024 1>/dev/null 2>&1
+
+function xattr_query_check()
+{
+    local path=$1
+    local xa_name=$2
+
+    local ret=`getfattr -m . -n $xa_name $path 2>&1 | grep -o "$xa_name: No such attribute" | wc -l`
+    echo $ret
+}
+
+function set_xattr()
+{
+    local path=$1
+    local xa_name=$2
+    local xa_val=$3
+
+    setfattr -n $xa_name -v $xa_val $path
+    echo $?
+}
+
+EXPECT 0 set_xattr $M0/file$i.data "trusted.name" "testofafairlylongxattrstringthatbutnotlongenoughtofailmemoryallocation"
+
+EXPECT 0 xattr_query_check $M0/file$i.data "trusted.name"
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
+
diff --git a/tests/bugs/bug-1002556.t b/tests/bugs/bug-1002556.t
new file mode 100755
index 000000000..a57f455d4
--- /dev/null
+++ b/tests/bugs/bug-1002556.t
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume start $V0
+EXPECT '1 x 2 = 2' volinfo_field $V0 'Number of Bricks';
+
+TEST $CLI volume add-brick $V0 replica 3 $H0:$B0/${V0}2
+EXPECT '1 x 3 = 3' volinfo_field $V0 'Number of Bricks';
+
+TEST $CLI volume remove-brick $V0 replica 2 $H0:$B0/${V0}1 force
+EXPECT '1 x 2 = 2' volinfo_field $V0 'Number of Bricks';
+
+TEST killall glusterd
+TEST glusterd
+
+EXPECT '1 x 2 = 2' volinfo_field $V0 'Number of Bricks';
+cleanup
diff --git a/tests/bugs/bug-1004218.t b/tests/bugs/bug-1004218.t
new file mode 100644
index 000000000..17eb3c65b
--- /dev/null
+++ b/tests/bugs/bug-1004218.t
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Test if only a single xml document is generated by 'status all'
+# when a volume is not started
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+
+TEST $CLI volume create ${V0}1 $H0:$B0/${V0}1{1,2}
+TEST $CLI volume create ${V0}2 $H0:$B0/${V0}2{1,2}
+
+TEST $CLI volume start ${V0}1
+
+function test_status_all ()
+{
+        $CLI volume status all --xml | xmllint -format -
+}
+
+TEST test_status_all
+
+TEST $CLI volume stop ${V0}1
+
+cleanup
diff --git a/tests/bugs/bug-1004744.t b/tests/bugs/bug-1004744.t
new file mode 100644
index 000000000..0290119ef
--- /dev/null
+++ b/tests/bugs/bug-1004744.t
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+#Test case: After a rebalance fix-layout, check if the rebalance status command
+#displays the appropriate message at the CLI.
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+#Create a 2x1 distributed volume
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+TEST $CLI volume start $V0
+
+# Mount FUSE and create file/directory
+TEST glusterfs -s $H0 --volfile-id $V0 $M0
+for i in `seq 1 10`;
+do
+       mkdir $M0/dir_$i
+       echo file>$M0/dir_$i/file_$i
+       for j in `seq 1 100`;
+       do
+                mkdir $M0/dir_$i/dir_$j
+                echo file>$M0/dir_$i/dir_$j/file_$j
+       done
+done
+
+#add 2 bricks
+TEST $CLI volume add-brick $V0 $H0:$B0/${V0}{3,4};
+
+#perform rebalance fix-layout
+TEST $CLI volume rebalance $V0 fix-layout start
+
+EXPECT_WITHIN 1 "fix-layout in progress" rebalance_status_field $V0;
+
+EXPECT_WITHIN 20 "fix-layout completed" rebalance_status_field $V0;
+
+TEST umount $M0
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-1015990-rep.t b/tests/bugs/bug-1015990-rep.t
new file mode 100755
index 000000000..f59bb2f75
--- /dev/null
+++ b/tests/bugs/bug-1015990-rep.t
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../afr.rc
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+
+
+
+TEST kill_brick $V0 $H0 $B0/$V0"1"
+sleep 5
+TEST kill_brick $V0 $H0 $B0/$V0"3"
+sleep 5
+
+for  i in  {1..100}; do echo "STRING" > $M0/File$i; done
+
+brick_2_sh_entries=$(count_sh_entries $B0/$V0"2")
+brick_4_sh_entries=$(count_sh_entries $B0/$V0"4")
+
+
+command_output=$(gluster volume heal $V0 statistics heal-count replica $H0:$B0/$V0"1")
+
+
+substring="Number of entries:"
+count=0
+while read -r line;
+do
+        if [[ "$line" == *$substring* ]]
+                then
+                        value=$(echo $line | cut -f 2 -d :)
+                        count=$(($count + $value))
+        fi
+
+done <<< "$command_output"
+
+brick_2_entries_count=$(($count-$value))
+
+EXPECT "0" echo $brick_2_entries_count
+
+brick_2_entries_count=$count
+
+
+xattrop_count_brick_2=$(count_sh_entries $B0/$V0"2")
+##Remove the count of the xattrop-gfid entry count as it does not contribute
+##to the number of files to be healed
+
+sub_val=1
+xattrop_count_brick_2=$(($xattrop_count_brick_2-$sub_val))
+
+ret=0
+if [ "$xattrop_count_brick_2" -eq "$brick_2_entries_count" ]
+        then
+                ret=$(($ret + $sub_val))
+fi
+
+EXPECT "1" echo $ret
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0
+
+cleanup;
diff --git a/tests/bugs/bug-1015990.t b/tests/bugs/bug-1015990.t
new file mode 100755
index 000000000..165af5168
--- /dev/null
+++ b/tests/bugs/bug-1015990.t
@@ -0,0 +1,95 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../afr.rc
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+
+
+
+TEST kill_brick $V0 $H0 $B0/$V0"1"
+sleep 5
+TEST kill_brick $V0 $H0 $B0/$V0"3"
+sleep 5
+
+for  i in  {1..100}; do echo "STRING" > $M0/File$i; done
+
+brick_2_sh_entries=$(count_sh_entries $B0/$V0"2")
+brick_4_sh_entries=$(count_sh_entries $B0/$V0"4")
+
+
+command_output=$(gluster volume heal $V0 statistics heal-count)
+
+
+substring="Number of entries:"
+count=0
+while read -r line;
+do
+        if [[ "$line" == *$substring* ]]
+                then
+                        value=$(echo $line | cut -f 2 -d :)
+                        count=$(($count + $value))
+        fi
+
+done <<< "$command_output"
+
+brick_2_entries_count=$(($count-$value))
+brick_4_entries_count=$value
+
+
+xattrop_count_brick_2=$(count_sh_entries $B0/$V0"2")
+##Remove the count of the xattrop-gfid entry count as it does not contribute
+##to the number of files to be healed
+
+sub_val=1
+xattrop_count_brick_2=$(($xattrop_count_brick_2-$sub_val))
+
+xattrop_count_brick_4=$(count_sh_entries $B0/$V0"4")
+##Remove xattrop-gfid entry count
+
+xattrop_count_brick_4=$(($xattrop_count_brick_4-$sub_val))
+
+
+ret=0
+if [ "$xattrop_count_brick_2" -eq "$brick_2_entries_count" ]
+        then
+                ret=$(($ret + $sub_val))
+fi
+
+EXPECT "1" echo $ret
+
+
+ret=0
+if [ "$xattrop_count_brick_4" -eq "$brick_4_entries_count" ]
+        then
+                ret=$(($ret + $sub_val))
+fi
+
+EXPECT "1" echo $ret
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0
+
+cleanup;
+
diff --git a/tests/bugs/bug-1022055.t b/tests/bugs/bug-1022055.t
new file mode 100755
index 000000000..c2f4218bb
--- /dev/null
+++ b/tests/bugs/bug-1022055.t
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../cluster.rc
+
+function check_peers {
+        $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+
+cleanup;
+
+TEST launch_cluster 2;
+
+TEST $CLI_1 peer probe $H2;
+
+EXPECT_WITHIN 20 1 check_peers;
+
+TEST $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0;
+
+TEST $CLI_1 volume start $V0;
+
+TEST $CLI_1 volume log rotate $V0;
+
+TEST $CLI_1 volume status;
+
+cleanup;
diff --git a/tests/bugs/bug-1022905.t b/tests/bugs/bug-1022905.t
new file mode 100644
index 000000000..aef3395dd
--- /dev/null
+++ b/tests/bugs/bug-1022905.t
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+## Create a volume
+TEST glusterd;
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1};
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Volume start
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Enable a protected and a resettable/unprotected option
+TEST $CLI volume quota $V0 enable
+TEST $CLI volume set $V0 diagnostics.client-log-level DEBUG
+
+## Reset cmd resets only unprotected option(s), succeeds.
+TEST $CLI volume reset $V0;
+
+## Reset should fail
+TEST ! $CLI volume reset $V0;
+
+## Set an unprotected option
+TEST $CLI volume set $V0 diagnostics.client-log-level DEBUG
+
+## Now 1 protected and 1 unprotected options are set
+## Reset force should succeed
+TEST $CLI volume reset $V0 force;
+
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup;
diff --git a/tests/bugs/bug-1030208.t b/tests/bugs/bug-1030208.t
new file mode 100644
index 000000000..866999692
--- /dev/null
+++ b/tests/bugs/bug-1030208.t
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+#Test case: Hardlink test
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+#Create a distributed volume
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1..2};
+TEST $CLI volume start $V0
+
+# Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0
+
+#Create a file and perform fop on a DIR
+TEST touch $M0/foo
+TEST ls $M0/
+
+#Create hardlink
+TEST ln $M0/foo $M0/bar
+
+
+TEST umount $M0
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-1040934.t b/tests/bugs/bug-1040934.t
new file mode 100644
index 000000000..3089d7ce1
--- /dev/null
+++ b/tests/bugs/bug-1040934.t
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../cluster.rc
+. $(dirname $0)/../snapshot.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST launch_cluster 2
+TEST setup_lvm 2
+
+TEST $CLI_1 peer probe $H2
+EXPECT_WITHIN 20 1 peer_count
+
+TEST $CLI_1 volume create $V0 replica 2 $H1:$L1 $H2:$L2
+EXPECT 'Created' volinfo_field $V0 'Status'
+
+TEST $CLI_1 volume start $V0
+EXPECT 'Started' volinfo_field $V0 'Status'
+
+TEST $CLI_1 snapshot create ${V0}_snap ${V0}
+PID_1=$!
+wait $PID_1
+
+TEST snapshot_exists ${V0}_snap
+TEST mount -t glusterfs $H1:/snaps/${V0}_snap/$V0 $M0
+cd $M0
+TEST ! touch a
+
+TEST $CLI_1 snapshot delete ${V0}_snap
+PID_1=$!
+wait $PID_1
+
+TEST ! snapshot_exists ${V0}_snap
+
+cleanup;
diff --git a/tests/bugs/bug-1045333.t b/tests/bugs/bug-1045333.t
new file mode 100644
index 000000000..d1f8069e8
--- /dev/null
+++ b/tests/bugs/bug-1045333.t
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../snapshot.rc
+
+cleanup;
+TEST glusterd;
+TEST pidof glusterd;
+
+TEST setup_lvm 1
+
+TEST $CLI volume create $V0 $H0:$L1
+TEST $CLI volume start $V0
+
+
+S1="${V0}-snap1"    #Create snapshot with name contains hyphen(-)
+S2="-${V0}-snap2"   #Create snapshot with name starts with hyphen(-)
+#Create snapshot with a long name
+S3="${V0}_single_gluster_volume_is_accessible_by_multiple_clients_offline_snapshot_is_a_long_name"
+
+TEST $CLI snapshot create $S1 $V0
+TEST snapshot_exists $S1
+
+TEST $CLI snapshot create $S2 $V0
+TEST snapshot_exists $S2
+
+TEST $CLI snapshot create $S3 $V0
+TEST snapshot_exists $S3
+
+
+TEST mount -t glusterfs $H0:/snaps/$S1/$V0 $M0
+TEST umount -f $M0
+
+TEST mount -t glusterfs $H0:/snaps/$S2/$V0 $M0
+TEST umount -f $M0
+
+TEST mount -t glusterfs $H0:/snaps/$S3/$V0 $M0
+TEST umount -f $M0
+
+#Clean up
+TEST $CLI snapshot delete $S1
+TEST $CLI snapshot delete $S2
+TEST $CLI snapshot delete $S3
+
+TEST $CLI volume stop $V0 force
+TEST $CLI volume delete $V0
+
+cleanup;
diff --git a/tests/bugs/bug-1049834.t b/tests/bugs/bug-1049834.t
new file mode 100755
index 000000000..6019a561c
--- /dev/null
+++ b/tests/bugs/bug-1049834.t
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../cluster.rc
+. $(dirname $0)/../snapshot.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST launch_cluster 2
+TEST setup_lvm 2
+
+TEST $CLI_1 peer probe $H2
+EXPECT_WITHIN 20 1 peer_count
+
+TEST $CLI_1 volume create $V0 $H1:$L1 $H2:$L2
+EXPECT 'Created' volinfo_field $V0 'Status'
+
+TEST $CLI_1 volume start $V0
+EXPECT 'Started' volinfo_field $V0 'Status'
+
+#Setting the snap-max-hard-limit to 4
+TEST $CLI_1 snapshot config $V0 snap-max-hard-limit 4
+PID_1=$!
+wait $PID_1
+
+#Creating 4 snapshots on the volume
+TEST create_n_snapshots $V0 4 $V0_snap
+TEST snapshot_n_exists $V0 4 $V0_snap
+
+#Creating the 5th snapshots on the volume and expecting it not to be created.
+TEST ! $CLI_1 snapshot create ${V0}_snap5 ${V0}
+TEST ! snapshot_exists ${V0}_snap5
+TEST ! $CLI_1 snapshot delete ${V0}_snap5
+
+#Deleting the 4 snaps
+TEST delete_n_snapshots $V0 4 $V0_snap
+TEST ! snapshot_n_exists $V0 4 $V0_snap
+
+cleanup;
diff --git a/tests/bugs/bug-1064768.t b/tests/bugs/bug-1064768.t
new file mode 100755
index 000000000..b87168150
--- /dev/null
+++ b/tests/bugs/bug-1064768.t
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/brick0 $H0:$B0/brick1
+TEST $CLI volume start $V0
+EXPECT_WITHIN 15 'Started' volinfo_field $V0 'Status';
+
+TEST $CLI volume profile $V0 start
+TEST $CLI volume profile $V0 info
+TEST $CLI volume profile $V0 stop
+
+TEST $CLI volume status
+TEST $CLI volume stop $V0
+EXPECT_WITHIN 15 'Stopped' volinfo_field $V0 'Status';
+cleanup;
diff --git a/tests/bugs/bug-762989.t b/tests/bugs/bug-762989.t
new file mode 100755
index 000000000..1794693cc
--- /dev/null
+++ b/tests/bugs/bug-762989.t
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+## reserve port 1023
+older_ports=$(cat /proc/sys/net/ipv4/ip_local_reserved_ports);
+echo "1023" > /proc/sys/net/ipv4/ip_local_reserved_ports;
+
+## Start and create a volume
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+TEST $CLI volume start $V0;
+
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 \
+$M0;
+
+## Wait for volume to register with rpc.mountd
+sleep 6;
+## check if port 1023 (which has been reserved) is used by the gluster processes
+op=$(netstat -ntp | grep gluster | grep -w 1023);
+EXPECT "" echo $op;
+
+#set the reserved ports to the older values
+echo $older_ports > /proc/sys/net/ipv4/ip_local_reserved_ports
+
+cleanup;
diff --git a/tests/bugs/bug-764638.t b/tests/bugs/bug-764638.t
new file mode 100644
index 000000000..816546524
--- /dev/null
+++ b/tests/bugs/bug-764638.t
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI pool list;
+TEST $CLI pool list --xml;
+
+cleanup;
diff --git a/tests/bugs/bug-765230.t b/tests/bugs/bug-765230.t
new file mode 100755
index 000000000..2012be5ad
--- /dev/null
+++ b/tests/bugs/bug-765230.t
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting quota-timeout as 20
+TEST ! $CLI volume set $V0 features.quota-timeout 20
+EXPECT '' volinfo_field $V0 'features.quota-timeout';
+
+## Enabling features.quota-deem-statfs
+TEST ! $CLI volume set $V0 features.quota-deem-statfs on
+EXPECT '' volinfo_field $V0 'features.quota-deem-statfs'
+
+## Enabling quota
+TEST $CLI volume quota $V0 enable
+EXPECT 'on' volinfo_field $V0 'features.quota'
+
+## Setting quota-timeout as 20
+TEST $CLI volume set $V0 features.quota-timeout 20
+EXPECT '20' volinfo_field $V0 'features.quota-timeout';
+
+## Enabling features.quota-deem-statfs
+TEST $CLI volume set $V0 features.quota-deem-statfs on
+EXPECT 'on' volinfo_field $V0 'features.quota-deem-statfs'
+
+## Disabling quota
+TEST $CLI volume quota $V0 disable
+EXPECT 'off' volinfo_field $V0 'features.quota'
+
+## Setting quota-timeout as 30
+TEST ! $CLI volume set $V0 features.quota-timeout 30
+EXPECT '20' volinfo_field $V0 'features.quota-timeout';
+
+## Disabling features.quota-deem-statfs
+TEST ! $CLI volume set $V0 features.quota-deem-statfs off
+EXPECT 'on' volinfo_field $V0 'features.quota-deem-statfs'
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-765380.t b/tests/bugs/bug-765380.t
new file mode 100644
index 000000000..a9784b93d
--- /dev/null
+++ b/tests/bugs/bug-765380.t
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+REPLICA=2
+
+TEST $CLI volume create $V0 replica $REPLICA $H0:$B0/${V0}00 $H0:$B0/${V0}01 $H0:$B0/${V0}10 $H0:$B0/${V0}11
+TEST $CLI volume start $V0
+
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+function count_hostname_or_uuid_from_pathinfo()
+{
+    pathinfo=`getfattr -m . -n trusted.glusterfs.pathinfo $M0/f00f`
+    echo $pathinfo | grep -o $1 | wc -l
+}
+
+touch $M0/f00f
+
+EXPECT $REPLICA count_hostname_or_uuid_from_pathinfo $H0
+
+# turn on node-uuid-pathinfo option
+TEST $CLI volume set $V0 node-uuid-pathinfo on
+
+# do not expext hostname as part of the pathinfo string
+EXPECT 0 count_hostname_or_uuid_from_pathinfo $H0
+
+uuid=`grep UUID /var/lib/glusterd/glusterd.info | cut -f2 -d=`
+
+# ... but expect the uuid $REPLICA times
+EXPECT $REPLICA count_hostname_or_uuid_from_pathinfo $uuid
+
+cleanup;
diff --git a/tests/bugs/bug-765473.t b/tests/bugs/bug-765473.t
new file mode 100755
index 000000000..5fc0ec9d7
--- /dev/null
+++ b/tests/bugs/bug-765473.t
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../fileio.rc
+
+cleanup;
+
+function clients_connected()
+{
+    volname=$1
+    gluster volume status $volname clients | grep -i 'Clients connected' | sed -e 's/[^0-9]*\(.*\)/\1/g'
+}
+        
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1}
+TEST $CLI volume start $V0;
+
+TEST glusterfs --direct-io-mode=yes --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+TEST fd=`fd_available`
+TEST fd_open $fd 'w' "$M0/testfile"
+TEST fd_write $fd "content"
+TEST $CLI volume stop $V0
+# write some content which will result in marking fd bad
+fd_write $fd "more content"
+TEST $CLI volume start $V0
+EXPECT_WITHIN 30 2 clients_connected $V0
+TEST ! fd_write $fd "still more content"
+
+cleanup
diff --git a/tests/bugs/bug-765564.t b/tests/bugs/bug-765564.t
new file mode 100644
index 000000000..0b8b8cd4f
--- /dev/null
+++ b/tests/bugs/bug-765564.t
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+TEST glusterd
+TEST pidof glusterd
+
+## Start and create a volume
+mkdir -p ${B0}/${V0}-0
+mkdir -p ${B0}/${V0}-1
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}-{0,1}
+
+TEST $CLI volume set $V0 performance.io-cache off;
+TEST $CLI volume set $V0 performance.write-behind off;
+TEST $CLI volume set $V0 performance.stat-prefetch off
+
+TEST $CLI volume start $V0;
+
+## Mount native
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+
+#returns success if 'olddir' is absent
+#'olddir' must be absent in both replicas
+function rm_succeeded () {
+    local dir1=$1
+    [[ -d $H0:$B0/${V0}-0/$dir1 || -d $H0:$B0/${V0}-1/$dir1 ]] && return 0
+    return 1
+}
+
+# returns successes if 'newdir' is present
+#'newdir' must be present in both replicas
+function mv_succeeded () {
+    local dir1=$1
+    [[ -d $H0:$B0/${V0}-0/$dir1 && -d $H0:$B0/${V0}-1/$dir1 ]] && return 1
+    return 0
+}
+
+# returns zero on success
+# Only one of rm and mv can succeed. This is captured by the XOR below
+
+function chk_backend_consistency(){
+    local dir1=$1
+    local dir2=$2
+    local rm_status=rm_succeeded $dir1
+    local mv_status=mv_succeeded $dir2
+    [[ ( $rm_status && ! $mv_status ) || ( ! $rm_status && $mv_status ) ]] && return 0
+    return 1
+}
+
+#concurrent removal/rename of dirs
+function rm_mv_correctness () {
+    ret=0
+    for i in {1..100}; do
+        mkdir $M0/"dir"$i
+        rmdir $M0/"dir"$i &
+        mv $M0/"dir"$i $M0/"adir"$i &
+        wait
+        tmp_ret=$(chk_backend_consistency "dir"$i "adir"$i)
+        (( ret += tmp_ret ))
+        rm -rf $M0/"dir"$i
+        rm -rf $M0/"adir"$i
+    done
+    return $ret
+}
+
+TEST touch $M0/a;
+TEST mv $M0/a $M0/b;
+
+#test rename fop when one of the bricks is down
+kill_brick ${V0} ${H0} ${B0}/${V0}-1;
+TEST touch $M0/h;
+TEST mv $M0/h $M0/1;
+
+TEST $CLI volume start $V0 force;
+
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1;
+find $M0 | xargs stat 2>/dev/null 1>/dev/null;
+
+TEST rm_mv_correctness;
+TEST umount $M0;
+cleanup;
+
diff --git a/tests/bugs/bug-767095.t b/tests/bugs/bug-767095.t
new file mode 100755
index 000000000..a8842bd54
--- /dev/null
+++ b/tests/bugs/bug-767095.t
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+dump_dir='/tmp/gerrit_glusterfs'
+TEST mkdir -p $dump_dir;
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+TEST $CLI volume set $V0 error-gen posix;
+TEST $CLI volume set $V0 server.statedump-path $dump_dir;
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST PID=`gluster volume status $V0 | grep patchy1 | awk {'print $5'}`;
+TEST kill -USR1 $PID;
+sleep 2;
+for file_name in $(ls $dump_dir)
+do
+    TEST grep "error-gen.priv" $dump_dir/$file_name;
+done
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+TEST rm -rf $dump_dir;
+
+cleanup;
diff --git a/tests/bugs/bug-767585-gfid.t b/tests/bugs/bug-767585-gfid.t
new file mode 100755
index 000000000..49cf7423f
--- /dev/null
+++ b/tests/bugs/bug-767585-gfid.t
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+#Test cases to perform gfid-self-heal
+#file 'a' should be assigned a fresh gfid
+#file 'b' should be healed with gfid1 from brick1
+#file 'c' should be healed with gfid2 from brick2
+
+gfid1="0x8428b7193a764bf8be8046fb860b8993"
+gfid2="0x85ad91afa2f74694bf52c3326d048209"
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 --direct-io-mode=enable
+touch $B0/${V0}0/a $B0/${V0}1/a
+touch $B0/${V0}0/b $B0/${V0}1/b
+touch $B0/${V0}0/c $B0/${V0}1/c
+
+TEST setfattr -n trusted.gfid -v $gfid1 $B0/${V0}0/b
+TEST setfattr -n trusted.gfid -v $gfid2 $B0/${V0}1/c
+
+sleep 2
+
+cd $M0
+TEST ls -l a
+TEST ls -l b
+TEST ls -l c
+
+TEST gf_get_gfid_xattr $B0/${V0}0/a
+TEST gf_get_gfid_xattr $B0/${V0}1/a
+
+EXPECT "$gfid1" gf_get_gfid_xattr $B0/${V0}0/b
+EXPECT "$gfid1" gf_get_gfid_xattr $B0/${V0}1/b
+
+EXPECT "$gfid2" gf_get_gfid_xattr $B0/${V0}0/c
+EXPECT "$gfid2" gf_get_gfid_xattr $B0/${V0}1/c
+
+cleanup;
diff --git a/tests/bugs/bug-770655.t b/tests/bugs/bug-770655.t
new file mode 100755
index 000000000..945e323bb
--- /dev/null
+++ b/tests/bugs/bug-770655.t
@@ -0,0 +1,168 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a distribute-replicate volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'Distributed-Replicate' volinfo_field $V0 'Type';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting stripe-block-size as 10MB
+TEST ! $CLI volume set $V0 stripe-block-size 10MB
+EXPECT '' volinfo_field $V0 'cluster.stripe-block-size';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
+
+## Start and create a replicate volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 replica 8 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'Replicate' volinfo_field $V0 'Type';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting stripe-block-size as 10MB
+TEST ! $CLI volume set $V0 stripe-block-size 10MB
+EXPECT '' volinfo_field $V0 'cluster.stripe-block-size';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
+
+## Start and create a distribute volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'Distribute' volinfo_field $V0 'Type';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting stripe-block-size as 10MB
+TEST ! $CLI volume set $V0 stripe-block-size 10MB
+EXPECT '' volinfo_field $V0 'cluster.stripe-block-size';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
+
+## Start and create a stripe volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 stripe 8 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'Stripe' volinfo_field $V0 'Type';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting stripe-block-size as 10MB
+TEST $CLI volume set $V0 stripe-block-size 10MB
+EXPECT '10MB' volinfo_field $V0 'cluster.stripe-block-size';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
+
+## Start and create a distributed stripe volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 stripe 4 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'Distributed-Stripe' volinfo_field $V0 'Type';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting stripe-block-size as 10MB
+TEST $CLI volume set $V0 stripe-block-size 10MB
+EXPECT '10MB' volinfo_field $V0 'cluster.stripe-block-size';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
+
+## Start and create a distributed stripe replicate volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 stripe 2 replica 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'Distributed-Striped-Replicate' volinfo_field $V0 'Type';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting stripe-block-size as 10MB
+TEST $CLI volume set $V0 stripe-block-size 10MB
+EXPECT '10MB' volinfo_field $V0 'cluster.stripe-block-size';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-782095.t b/tests/bugs/bug-782095.t
new file mode 100755
index 000000000..a0cea14ee
--- /dev/null
+++ b/tests/bugs/bug-782095.t
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting performance cache min size as 2MB
+TEST $CLI volume set $V0 performance.cache-min-file-size 2MB
+EXPECT '2MB' volinfo_field $V0 'performance.cache-min-file-size';
+
+## Setting performance cache max size as 20MB
+TEST $CLI volume set $V0 performance.cache-max-file-size 20MB
+EXPECT '20MB' volinfo_field $V0 'performance.cache-max-file-size';
+
+## Trying to set performance cache min size as 25MB
+TEST ! $CLI volume set $V0 performance.cache-min-file-size 25MB
+EXPECT '2MB' volinfo_field $V0 'performance.cache-min-file-size';
+
+## Able to set performance cache min size as long as its lesser than max size
+TEST $CLI volume set $V0 performance.cache-min-file-size 15MB
+EXPECT '15MB' volinfo_field $V0 'performance.cache-min-file-size';
+
+## Trying it out with only cache-max-file-size in CLI as 10MB
+TEST ! $CLI volume set $V0 cache-max-file-size 10MB
+EXPECT '20MB' volinfo_field $V0 'performance.cache-max-file-size';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-797171.t b/tests/bugs/bug-797171.t
new file mode 100755
index 000000000..a1b28d9ff
--- /dev/null
+++ b/tests/bugs/bug-797171.t
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/brick1;
+TEST $CLI volume set $V0 debug.trace marker;
+TEST $CLI volume set $V0 debug.log-history on
+
+TEST $CLI volume start $V0;
+
+sleep 1;
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 \
+$M0;
+
+sleep 5;
+
+touch $M0/{1..22};
+rm -f $M0/*;
+
+pid_file=$(ls /var/lib/glusterd/vols/$V0/run);
+brick_pid=$(cat /var/lib/glusterd/vols/$V0/run/$pid_file);
+
+mkdir $statedumpdir/statedump_tmp/;
+echo "path=$statedumpdir/statedump_tmp" > $statedumpdir/glusterdump.options;
+echo "all=yes" >> $statedumpdir/glusterdump.options;
+
+TEST $CLI volume statedump $V0 history;
+
+file_name=$(ls $statedumpdir/statedump_tmp);
+TEST grep "xlator.debug.trace.history" $statedumpdir/statedump_tmp/$file_name;
+
+TEST umount $M0
+
+rm -rf $statedumpdir/statedump_tmp;
+rm -f $statedumpdir/glusterdump.options;
+
+cleanup;
diff --git a/tests/bugs/bug-802417.t b/tests/bugs/bug-802417.t
new file mode 100755
index 000000000..314141f6b
--- /dev/null
+++ b/tests/bugs/bug-802417.t
@@ -0,0 +1,108 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+function write_file()
+{
+	path="$1"; shift
+	echo "$*" > "$path"
+}
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+## Start and create a volume
+mkdir -p ${B0}/${V0}-0
+mkdir -p ${B0}/${V0}-1
+mkdir -p ${B0}/${V0}-2
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}-{0,1,2}
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Make sure io-cache and write-behind don't interfere.
+TEST $CLI volume set $V0 performance.io-cache off;
+TEST $CLI volume set $V0 performance.write-behind off;
+TEST $CLI volume set $V0 performance.stat-prefetch off
+
+## Make sure automatic self-heal doesn't perturb our results.
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST $CLI volume set $V0 cluster.data-self-heal on
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Mount native
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+
+## Create a file with some recognizably stale data.
+TEST write_file $M0/a_file "old_data"
+
+## Kill two of the bricks and write some newer data.
+TEST kill_brick ${V0} ${H0} ${B0}/${V0}-1
+TEST kill_brick ${V0} ${H0} ${B0}/${V0}-2
+TEST write_file $M0/a_file "new_data"
+
+## Bring all the bricks up and kill one so we do a partial self-heal.
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 2
+TEST kill_brick ${V0} ${H0} ${B0}/${V0}-2
+TEST ls -l ${M0}/a_file
+
+
+obs_path_0=${B0}/${V0}-0/a_file
+obs_path_1=${B0}/${V0}-1/a_file
+obs_path_2=${B0}/${V0}-2/a_file
+
+tgt_xattr_0="trusted.afr.${V0}-client-0"
+tgt_xattr_1="trusted.afr.${V0}-client-1"
+tgt_xattr_2="trusted.afr.${V0}-client-2"
+
+actual=$(afr_get_changelog_xattr $obs_path_0 $tgt_xattr_0)
+EXPECT "0x000000000000000000000000" echo $actual
+
+actual=$(afr_get_changelog_xattr $obs_path_0 $tgt_xattr_1)
+EXPECT "0x000000000000000000000000" echo $actual
+
+actual=$(afr_get_changelog_xattr $obs_path_0 $tgt_xattr_2)
+EXPECT "0x000000020000000000000000" echo $actual
+
+actual=$(afr_get_changelog_xattr $obs_path_1 $tgt_xattr_0)
+EXPECT "0x000000000000000000000000" echo $actual
+
+actual=$(afr_get_changelog_xattr $obs_path_1 $tgt_xattr_1)
+EXPECT "0x000000000000000000000000" echo $actual
+
+actual=$(afr_get_changelog_xattr $obs_path_1 $tgt_xattr_2)
+EXPECT "0x000000020000000000000000" echo $actual
+
+actual=$(afr_get_changelog_xattr $obs_path_2 $tgt_xattr_0)
+EXPECT "0x000000000000000000000000" echo $actual
+
+actual=$(afr_get_changelog_xattr $obs_path_2 $tgt_xattr_1)
+EXPECT "0x000000000000000000000000" echo $actual
+
+actual=$(afr_get_changelog_xattr $obs_path_2 $tgt_xattr_2)
+EXPECT "0x000000000000000000000000" echo $actual
+
+if [ "$EXIT_EARLY" = "1" ]; then
+	exit 0;
+fi
+
+## Finish up
+TEST umount $M0;
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-808400-dist.t b/tests/bugs/bug-808400-dist.t
new file mode 100755
index 000000000..6a29eb626
--- /dev/null
+++ b/tests/bugs/bug-808400-dist.t
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/brick1 $H0:$B0/brick2;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+MOUNTDIR=$M0;
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 --volfile-server=$H0 --volfile-id=$V0 $MOUNTDIR;
+
+build_tester $(dirname $0)/bug-808400-flock.c
+build_tester $(dirname $0)/bug-808400-fcntl.c
+
+TEST $(dirname $0)/bug-808400-flock $MOUNTDIR/testfile \'gluster volume set $V0 performance.write-behind off\'
+TEST $(dirname $0)/bug-808400-fcntl $MOUNTDIR/testfile \'gluster volume set $V0 performance.write-behind on\'
+
+TEST rm -rf $MOUNTDIR/*
+TEST rm -rf $(dirname $0)/bug-808400-flock $(dirname $0)/bug-808400-fcntl $(dirname $0)/glusterfs.log
+
+TEST   umount $MOUNTDIR -l
+
+cleanup;
+\ No newline at end of file
diff --git a/tests/bugs/bug-808400-fcntl.c b/tests/bugs/bug-808400-fcntl.c
new file mode 100644
index 000000000..4deef34a5
--- /dev/null
+++ b/tests/bugs/bug-808400-fcntl.c
@@ -0,0 +1,113 @@
+#include <sys/file.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+
+int
+run_child (char *filename)
+{
+        int          fd   = -1, ret = -1;
+        struct flock lock = {0, };
+        int          ppid = 0;
+
+        fd = open (filename, O_RDWR);
+        if (fd < 0) {
+                fprintf (stderr, "open failed (%s)\n", strerror (errno));
+                goto out;
+        }
+
+        ppid = getppid ();
+
+        lock.l_type = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start = 0;
+        lock.l_len = 0;
+
+        ret = fcntl (fd, F_GETLK, &lock);
+        if (ret < 0) {
+                fprintf (stderr, "GETLK failed (%s)\n", strerror (errno));
+                goto out;
+        }
+
+        if ((lock.l_type == F_UNLCK) ||
+            (ppid != lock.l_pid)) {
+                fprintf (stderr, "no locks present, though parent has held "
+                         "one\n");
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+main (int argc, char *argv[])
+{
+        int          fd       = -1, ret = -1, status = 0;
+        char        *filename = NULL, *cmd = NULL;
+        struct stat  stbuf    = {0, };
+        struct flock lock     = {0, };
+
+        if (argc != 3) {
+                fprintf (stderr, "Usage: %s <filename> "
+                         "<gluster-cmd-to-trigger-graph-switch>\n", argv[0]);
+                goto out;
+        }
+
+        filename = argv[1];
+        cmd = argv[2];
+
+        fd = open (filename, O_RDWR | O_CREAT, 0);
+        if (fd < 0) {
+                fprintf (stderr, "open (%s) failed (%s)\n", filename,
+                         strerror (errno));
+                goto out;
+        }
+
+        lock.l_type = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start = 0;
+        lock.l_len = 0;
+
+        ret = fcntl (fd, F_SETLK, &lock);
+        if (ret < 0) {
+                fprintf (stderr, "fcntl failed (%s)\n", strerror (errno));
+                goto out;
+        }
+
+        system (cmd);
+
+        /* wait till graph switch completes */
+        ret = fstat64 (fd, &stbuf);
+        if (ret < 0) {
+                fprintf (stderr, "fstat64 failure (%s)\n", strerror (errno));
+                goto out;
+        }
+
+        sleep (10);
+
+        /* By now old-graph would be disconnected and locks should be cleaned
+         * up if they are not migrated. Check that by trying to acquire a lock
+         * on a new fd opened by another process on same file.
+         */
+        ret = fork ();
+        if (ret == 0) {
+                ret = run_child (filename);
+        } else {
+                wait (&status);
+                if (WIFEXITED(status)) {
+                        ret = WEXITSTATUS(status);
+                } else {
+                        ret = 0;
+                }
+        }
+
+out:
+        return ret;
+}
diff --git a/tests/bugs/bug-808400-flock.c b/tests/bugs/bug-808400-flock.c
new file mode 100644
index 000000000..4770c81dc
--- /dev/null
+++ b/tests/bugs/bug-808400-flock.c
@@ -0,0 +1,92 @@
+#include <sys/file.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+
+int
+run_child (char *filename)
+{
+        int fd = -1, ret = -1;
+
+        fd = open (filename, O_RDWR);
+        if (fd < 0) {
+                fprintf (stderr, "open failed (%s)\n", strerror (errno));
+                goto out;
+        }
+
+        ret = flock (fd, LOCK_EX | LOCK_NB);
+        if ((ret == 0) || (errno != EWOULDBLOCK)) {
+                fprintf (stderr, "no locks present, though parent has held "
+                         "one\n");
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+main (int argc, char *argv[])
+{
+        int          fd       = -1, ret = -1, status = 0;
+        char        *filename = NULL, *cmd = NULL;
+        struct stat  stbuf    = {0, };
+
+        if (argc != 3) {
+                fprintf (stderr, "Usage: %s <filename> "
+                         "<gluster-cmd-to-trigger-graph-switch>\n", argv[0]);
+                goto out;
+        }
+
+        filename = argv[1];
+        cmd = argv[2];
+
+        fd = open (filename, O_RDWR | O_CREAT, 0);
+        if (fd < 0) {
+                fprintf (stderr, "open (%s) failed (%s)\n", filename,
+                         strerror (errno));
+                goto out;
+        }
+
+        ret = flock (fd, LOCK_EX);
+        if (ret < 0) {
+                fprintf (stderr, "flock failed (%s)\n", strerror (errno));
+                goto out;
+        }
+
+        system (cmd);
+
+        /* wait till graph switch completes */
+        ret = fstat64 (fd, &stbuf);
+        if (ret < 0) {
+                fprintf (stderr, "fstat64 failure (%s)\n", strerror (errno));
+                goto out;
+        }
+
+        sleep (10);
+
+        /* By now old-graph would be disconnected and locks should be cleaned
+         * up if they are not migrated. Check that by trying to acquire a lock
+         * on a new fd opened by another process on same file
+         */
+        ret = fork ();
+        if (ret == 0) {
+                ret = run_child (filename);
+        } else {
+                wait (&status);
+                if (WIFEXITED(status)) {
+                        ret = WEXITSTATUS(status);
+                } else {
+                        ret = 0;
+                }
+        }
+
+out:
+        return ret;
+}
diff --git a/tests/bugs/bug-808400-repl.t b/tests/bugs/bug-808400-repl.t
new file mode 100755
index 000000000..69cd9379b
--- /dev/null
+++ b/tests/bugs/bug-808400-repl.t
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+TEST $CLI volume create $V0 replica 2 $H0:$B0/brick1 $H0:$B0/brick2;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+MOUNTDIR=$M0;
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 --volfile-server=$H0 --volfile-id=$V0 $MOUNTDIR;
+
+build_tester $(dirname $0)/bug-808400-flock.c
+build_tester $(dirname $0)/bug-808400-fcntl.c
+
+TEST $(dirname $0)/bug-808400-flock $MOUNTDIR/testfile \'gluster volume set $V0 performance.write-behind off\'
+TEST $(dirname $0)/bug-808400-fcntl $MOUNTDIR/testfile \'gluster volume set $V0 performance.write-behind on\'
+
+TEST rm -rf $MOUNTDIR/*
+TEST rm -rf $(dirname $0)/bug-808400-flock $(dirname $0)/bug-808400-fcntl $(dirname $0)/glusterfs.log
+
+TEST   umount $MOUNTDIR -l
+
+cleanup;
+\ No newline at end of file
diff --git a/tests/bugs/bug-808400-stripe.t b/tests/bugs/bug-808400-stripe.t
new file mode 100755
index 000000000..3ab6f738e
--- /dev/null
+++ b/tests/bugs/bug-808400-stripe.t
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 stripe 2 $H0:$B0/brick1 $H0:$B0/brick2;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+MOUNTDIR=$M0;
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 --volfile-server=$H0 --volfile-id=$V0 $MOUNTDIR;
+
+build_tester $(dirname $0)/bug-808400-flock.c
+build_tester $(dirname $0)/bug-808400-fcntl.c
+
+TEST $(dirname $0)/bug-808400-flock $MOUNTDIR/testfile \'gluster volume set $V0 performance.write-behind off\'
+TEST $(dirname $0)/bug-808400-fcntl $MOUNTDIR/testfile \'gluster volume set $V0 performance.write-behind on\'
+
+TEST rm -rf $MOUNTDIR/*
+TEST rm -rf $(dirname $0)/bug-808400-flock $(dirname $0)/bug-808400-fcntl $(dirname $0)/glusterfs.log
+
+TEST   umount $MOUNTDIR -l
+
+cleanup;
+\ No newline at end of file
diff --git a/tests/bugs/bug-808400.t b/tests/bugs/bug-808400.t
new file mode 100755
index 000000000..49d88afd6
--- /dev/null
+++ b/tests/bugs/bug-808400.t
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/brick1;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+#mount on a random dir
+TEST MOUNTDIR="/tmp/$RANDOM"
+TEST mkdir $MOUNTDIR
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 --volfile-server=$H0 --volfile-id=$V0 $MOUNTDIR;
+
+build_tester $(dirname $0)/bug-808400-flock.c
+build_tester $(dirname $0)/bug-808400-fcntl.c
+
+TEST $(dirname $0)/bug-808400-flock $MOUNTDIR/testfile \'gluster volume set $V0 performance.write-behind off\'
+TEST $(dirname $0)/bug-808400-fcntl $MOUNTDIR/testfile \'gluster volume set $V0 performance.write-behind on\'
+
+TEST rm -rf $MOUNTDIR/*
+TEST rm -rf $(dirname $0)/bug-808400-flock $(dirname $0)/bug-808400-fcntl $(dirname $0)/glusterfs.log
+
+TEST   umount $MOUNTDIR -l
+TEST   rm -rf $MOUNTDIR
+
+cleanup;
+\ No newline at end of file
diff --git a/tests/bugs/bug-811493.t b/tests/bugs/bug-811493.t
new file mode 100755
index 000000000..13e99af57
--- /dev/null
+++ b/tests/bugs/bug-811493.t
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI system uuid reset;
+
+uuid1=$(grep UUID /var/lib/glusterd/glusterd.info | cut -f 2 -d "=");
+
+TEST $CLI system uuid reset;
+uuid2=$(grep UUID /var/lib/glusterd/glusterd.info | cut -f 2 -d "=");
+
+TEST [ $uuid1 != $uuid2 ]
+
+cleanup
diff --git a/tests/bugs/bug-821056.t b/tests/bugs/bug-821056.t
new file mode 100644
index 000000000..5e81541ac
--- /dev/null
+++ b/tests/bugs/bug-821056.t
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume set $V0 eager-lock off
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST $CLI volume set $V0 performance.quick-read off
+TEST $CLI volume set $V0 performance.open-behind off
+TEST $CLI volume set $V0 performance.io-cache off
+TEST $CLI volume set $V0 performance.write-behind on
+TEST $CLI volume set $V0 performance.stat-prefetch off
+TEST $CLI volume set $V0 performance.read-ahead off
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+TEST $CLI volume start $V0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable
+touch $M0/a
+
+#Open file with fd as 5
+exec 5>$M0/a
+realpath=$(gf_get_gfid_backend_file_path $B0/${V0}0 "a")
+
+kill_brick $V0 $H0 $B0/${V0}0
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+
+EXPECT "Y" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 "$realpath"
+
+kill_brick $V0 $H0 $B0/${V0}0
+TEST gf_rm_file_and_gfid_link $B0/${V0}0 "a"
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+ls -l $M0/a 2>&1 > /dev/null #Make sure the file is re-created
+EXPECT "N" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 "$realpath"
+EXPECT "N" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 $B0/${V0}0/a
+
+for i in {1..1024}; do
+        echo "open sesame" >&5
+done
+
+EXPECT_WITHIN 20 "Y" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 $B0/${V0}0/a
+#close the fd
+exec 5>&-
+
+#Check that anon-fd based file is not leaking.
+EXPECT_WITHIN 20 "N" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 "$realpath"
+cleanup;
diff --git a/tests/bugs/bug-822830.t b/tests/bugs/bug-822830.t
new file mode 100755
index 000000000..000d99f03
--- /dev/null
+++ b/tests/bugs/bug-822830.t
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting nfs.rpc-auth-reject as 192.*..*
+TEST ! $CLI volume set $V0 nfs.rpc-auth-reject 192.*..*
+EXPECT '' volinfo_field $V0 'nfs.rpc-auth-reject';
+
+# Setting nfs.rpc-auth-allow as a.a.
+TEST ! $CLI volume set $V0 nfs.rpc-auth-allow a.a.
+EXPECT '' volinfo_field $V0 'nfs.rpc-auth-allow';
+
+# Setting nfs.rpc-auth-allow as a.a
+TEST $CLI volume set $V0 nfs.rpc-auth-allow a.a
+EXPECT 'a.a' volinfo_field $V0 'nfs.rpc-auth-allow';
+
+## Setting nfs.rpc-auth-reject as 192.*.*
+TEST $CLI volume set $V0 nfs.rpc-auth-reject 192.*.*
+EXPECT '192.*.*' volinfo_field $V0 'nfs.rpc-auth-reject';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-823081.t b/tests/bugs/bug-823081.t
new file mode 100755
index 000000000..760d9e2b6
--- /dev/null
+++ b/tests/bugs/bug-823081.t
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+V1=patchy2
+
+TEST glusterd
+TEST pidof glusterd
+
+logdir=`gluster --print-logdir`
+function set_tail ()
+{
+        vol=$1;
+        tail_success="volume create $vol $H0:$B0/${vol}1 $H0:$B0/${vol}2 : SUCCESS"
+        tail_failure="volume create $vol $H0:$B0/${vol}1 $H0:$B0/${vol}2 : FAILED : Volume $vol already exists"
+        tail_success_force="volume create $vol $H0:$B0/${vol}1 $H0:$B0/${vol}2 force : SUCCESS"
+        tail_failure_force="volume create $vol $H0:$B0/${vol}1 $H0:$B0/${vol}2 force : FAILED : Volume $vol already exists"
+}
+
+set_tail $V0;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+tail=`tail --lines=1 $logdir/.cmd_log_history | cut -d " " -f 5-`
+TEST [[ \"$tail\" == \"$tail_success\" ]]
+
+TEST ! $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+tail=`tail --lines=1 $logdir/.cmd_log_history | cut -d " " -f 5-`
+TEST [[ \"$tail\" == \"$tail_failure\" ]]
+
+set_tail $V1;
+TEST gluster volume create $V1 $H0:$B0/${V1}{1,2} force;
+tail=`tail --lines=1 $logdir/.cmd_log_history | cut -d " " -f 5-`
+TEST [[ \"$tail\" == \"$tail_success_force\" ]]
+
+TEST ! gluster volume create $V1 $H0:$B0/${V1}{1,2} force;
+tail=`tail --lines=1 $logdir/.cmd_log_history | cut -d " " -f 5-`
+TEST [[ \"$tail\" == \"$tail_failure_force\" ]]
+
+cleanup;
diff --git a/tests/bugs/bug-824753-file-locker.c b/tests/bugs/bug-824753-file-locker.c
new file mode 100644
index 000000000..903e23e0a
--- /dev/null
+++ b/tests/bugs/bug-824753-file-locker.c
@@ -0,0 +1,42 @@
+#include <stdio.h>
+#include <string.h>
+#include <fcntl.h>
+
+int main (int argc, char *argv[])
+{
+    int              fd            = -1;
+    int              ret           = -1;
+    char             command[2048] = "";
+    char             filepath[255]     = "";
+    struct flock fl;
+
+    fl.l_type = F_WRLCK;
+    fl.l_whence = SEEK_SET;
+    fl.l_start = 7;
+    fl.l_len = 1;
+    fl.l_pid = getpid();
+
+    snprintf(filepath, 255, "%s/%s", argv[4], argv[5]);
+
+    fd = open(filepath, O_RDWR);
+
+    if (fd == -1)
+        return -1;
+
+    if (fcntl(fd, F_SETLKW, &fl) == -1) {
+        return -1;
+    }
+
+    snprintf(command, sizeof(command),
+             "gluster volume clear-locks %s /%s kind all posix 0,7-1 |"
+             " grep %s | awk -F'..: ' '{print $1}' | grep %s:%s/%s",
+             argv[1], argv[5], argv[2], argv[2], argv[3], argv[1]);
+
+    ret = system (command);
+    close(fd);
+
+    if (ret)
+            return -1;
+    else
+            return 0;
+}
diff --git a/tests/bugs/bug-824753.t b/tests/bugs/bug-824753.t
new file mode 100755
index 000000000..38f6bf696
--- /dev/null
+++ b/tests/bugs/bug-824753.t
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST glusterfs -s $H0 --volfile-id=$V0 $M0
+touch $M0/file1;
+
+TEST gcc -g $(dirname $0)/bug-824753-file-locker.c -o $(dirname $0)/file-locker
+
+TEST $(dirname $0)/file-locker $V0 $H0 $B0 $M0 file1
+
+## Finish up
+TEST rm -f $(dirname $0)/file-locker
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-830665.t b/tests/bugs/bug-830665.t
new file mode 100755
index 000000000..0073ff1d9
--- /dev/null
+++ b/tests/bugs/bug-830665.t
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+function recreate {
+	rm -rf $1 && mkdir -p $1
+}
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+## Start and create a volume
+recreate ${B0}/${V0}-0
+recreate ${B0}/${V0}-1
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}-{0,1}
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Make sure stat-prefetch doesn't prevent self-heal checks.
+TEST $CLI volume set $V0 performance.stat-prefetch off;
+
+## Make sure automatic self-heal doesn't perturb our results.
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+
+## Wait for volume to register with rpc.mountd
+sleep 5;
+
+## Mount NFS
+TEST mount -t nfs -o vers=3,nolock,soft,intr $H0:/$V0 $N0;
+
+## Create some files and directories
+echo "test_data" > $N0/a_file;
+mkdir $N0/a_dir;
+echo "more_test_data" > $N0/a_dir/another_file;
+
+## Unmount and stop the volume.
+TEST umount $N0;
+TEST $CLI volume stop $V0;
+
+# Recreate the brick. Note that because of http://review.gluster.org/#change,4202
+# we need to preserve and restore the volume ID or else the brick (and thus the
+# entire not-very-HA-any-more volume) won't start. When that bug is fixed, we can
+# remove the [gs]etxattr calls.
+volid=$(getfattr -e hex -n trusted.glusterfs.volume-id $B0/${V0}-0 2> /dev/null \
+	| grep = | cut -d= -f2)
+rm -rf $B0/${V0}-0;
+mkdir $B0/${V0}-0;
+setfattr -n trusted.glusterfs.volume-id -v $volid $B0/${V0}-0
+
+## Restart and remount. Note that we use actimeo=0 so that the stat calls
+## we need for self-heal don't get blocked by the NFS client.
+TEST $CLI volume start $V0;
+sleep 5
+TEST mount -t nfs -o vers=3,nolock,soft,intr,actimeo=0 $H0:/$V0 $N0;
+
+## The Linux NFS client has a really charming habit of caching stuff right
+## after mount, even though we set actimeo=0 above. Life would be much easier
+## if NFS developers cared as much about correctness as they do about shaving
+## a few seconds off of benchmarks.
+ls -l $N0 &> /dev/null;
+sleep 5;
+
+## Force entry self-heal.
+find $N0 | xargs stat > /dev/null;
+#ls -lR $N0 > /dev/null;
+
+## Do NOT check through the NFS mount here. That will force a new self-heal
+## check, but we want to test whether self-heal already happened.
+
+## Make sure everything's in order on the recreated brick.
+EXPECT 'test_data' cat $B0/${V0}-0/a_file;
+EXPECT 'more_test_data' cat $B0/${V0}-0/a_dir/another_file;
+
+if [ "$EXIT_EARLY" = "1" ]; then
+	exit 0;
+fi
+
+## Finish up
+TEST umount $N0;
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-834465.c b/tests/bugs/bug-834465.c
new file mode 100644
index 000000000..61d3deac0
--- /dev/null
+++ b/tests/bugs/bug-834465.c
@@ -0,0 +1,61 @@
+#include <sys/file.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+int
+main (int argc, char *argv[])
+{
+        int          fd       = -1;
+        char        *filename = NULL;
+        struct flock lock     = {0, };
+        int          i        = 0;
+        int          ret      = -1;
+
+        if (argc != 2) {
+                fprintf (stderr, "Usage: %s <filename> ", argv[0]);
+                goto out;
+        }
+
+        filename = argv[1];
+
+        fd = open (filename, O_RDWR | O_CREAT, 0);
+        if (fd < 0) {
+                fprintf (stderr, "open (%s) failed (%s)\n", filename,
+                         strerror (errno));
+                goto out;
+        }
+
+        lock.l_type = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start = 1;
+        lock.l_len = 1;
+
+        while (i < 100) {
+                lock.l_type = F_WRLCK;
+                ret = fcntl (fd, F_SETLK, &lock);
+                if (ret < 0) {
+                        fprintf (stderr, "fcntl setlk failed (%s)\n",
+                                        strerror (errno));
+                        goto out;
+                }
+
+                lock.l_type = F_UNLCK;
+                ret = fcntl (fd, F_SETLK, &lock);
+                if (ret < 0) {
+                        fprintf (stderr, "fcntl setlk failed (%s)\n",
+                                        strerror (errno));
+                        goto out;
+                }
+
+                i++;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
diff --git a/tests/bugs/bug-834465.t b/tests/bugs/bug-834465.t
new file mode 100755
index 000000000..af7f4bd12
--- /dev/null
+++ b/tests/bugs/bug-834465.t
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/brick1 $H0:$B0/brick2;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+MOUNTDIR=$M0;
+TEST glusterfs --mem-accounting --volfile-server=$H0 --volfile-id=$V0 $MOUNTDIR;
+
+sdump1=$(generate_mount_statedump $V0);
+nalloc1=0
+grep -A2 "fuse - usage-type 85" $sdump1
+if [ $? -eq '0' ]
+then
+        nalloc1=`grep -A2 "fuse - usage-type 85" $sdump1 | grep num_allocs | cut -d '=' -f2`
+fi
+
+build_tester $(dirname $0)/bug-834465.c
+
+TEST $(dirname $0)/bug-834465 $M0/testfile
+
+sdump2=$(generate_mount_statedump $V0);
+nalloc2=`grep -A2 "fuse - usage-type 85" $sdump2 | grep num_allocs | cut -d '=' -f2`
+
+TEST [ $nalloc1 -eq $nalloc2 ];
+
+TEST rm -rf $MOUNTDIR/*
+TEST rm -rf $(dirname $0)/bug-834465
+cleanup_mount_statedump $V0
+
+TEST   umount $MOUNTDIR -l
+
+cleanup;
diff --git a/tests/bugs/bug-839595.t b/tests/bugs/bug-839595.t
new file mode 100644
index 000000000..979827fa7
--- /dev/null
+++ b/tests/bugs/bug-839595.t
@@ -0,0 +1,31 @@
+#!/bin/bash
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}1
+TEST $CLI volume set $V0 cluster.server-quorum-type server
+EXPECT "server" volume_option $V0 cluster.server-quorum-type
+TEST $CLI volume set $V0 cluster.server-quorum-type none
+EXPECT "none" volume_option $V0 cluster.server-quorum-type
+TEST $CLI volume reset $V0 cluster.server-quorum-type
+TEST ! $CLI volume set $V0 cluster.server-quorum-type abc
+TEST ! $CLI volume set all cluster.server-quorum-type none
+TEST ! $CLI volume set $V0 cluster.server-quorum-ratio 100
+
+TEST ! $CLI volume set all cluster.server-quorum-ratio abc
+TEST ! $CLI volume set all cluster.server-quorum-ratio -1
+TEST ! $CLI volume set all cluster.server-quorum-ratio 100.0000005
+TEST $CLI volume set all cluster.server-quorum-ratio 0
+EXPECT "0" volume_option $V0 cluster.server-quorum-ratio
+TEST $CLI volume set all cluster.server-quorum-ratio 100
+EXPECT "100" volume_option $V0 cluster.server-quorum-ratio
+TEST $CLI volume set all cluster.server-quorum-ratio 0.0000005
+EXPECT "0.0000005" volume_option $V0 cluster.server-quorum-ratio
+TEST $CLI volume set all cluster.server-quorum-ratio 100%
+EXPECT "100%" volume_option $V0 cluster.server-quorum-ratio
+cleanup;
diff --git a/tests/bugs/bug-844688.t b/tests/bugs/bug-844688.t
new file mode 100755
index 000000000..154d35e48
--- /dev/null
+++ b/tests/bugs/bug-844688.t
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/brick0
+TEST $CLI volume start $V0
+
+sleep 5
+
+TEST glusterfs -s $H0 --volfile-id $V0 $M0
+
+mount_pid=$(get_mount_process_pid $V0);
+# enable dumping of call stack creation and frame creation times in statedump
+kill -USR2 $mount_pid;
+
+TEST touch $M0/touchfile;
+(dd if=/dev/urandom of=$M0/file bs=5K 2>/dev/null 1>/dev/null)&
+back_pid=$!;
+statedump_file=$(generate_mount_statedump $V0);
+grep "callstack-creation-time" $statedump_file 2>/dev/null 1>/dev/null;
+TEST [ $? -eq 0 ];
+grep "frame-creation-time" $statedump_file 2>/dev/null 1>/dev/null;
+TEST [ $? -eq 0 ];
+
+kill -SIGTERM $back_pid;
+wait >/dev/null 2>&1;
+
+TEST rm -f $M0/touchfile $M0/file;
+TEST umount $M0;
+
+rm -f $statedumpdir/glusterdump.$mount_pid.*;
+cleanup
diff --git a/tests/bugs/bug-845213.t b/tests/bugs/bug-845213.t
new file mode 100644
index 000000000..e79b37109
--- /dev/null
+++ b/tests/bugs/bug-845213.t
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+## Create and start a volume with aio enabled
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+TEST $CLI volume set $V0 remote-dio enable;
+TEST $CLI volume set $V0 network.remote-dio disable;
+
+cleanup;
+
diff --git a/tests/bugs/bug-846240.t b/tests/bugs/bug-846240.t
new file mode 100644
index 000000000..12e4949ef
--- /dev/null
+++ b/tests/bugs/bug-846240.t
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../fileio.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+TEST $CLI volume create $V0 $H0:$B0/brick1 $H0:$B0/brick2;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+MOUNTDIR=$M0;
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $MOUNTDIR;
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M1;
+
+TEST touch $M0/testfile;
+
+# open the file with the fd as 4
+TEST fd=`fd_available`;
+TEST fd_open $fd 'w' "$M0/testfile";
+
+# remove the file from the other mount point. If unlink is sent from
+# $M0 itself, then the file will be actually opened by open-behind which
+# we dont want for this testcase
+TEST rm -f $M1/testfile;
+
+# below command opens the file and writes to the file.
+# upon open, open-behind unwinds the open call with success.
+# now when write comes, open-behind actually opens the file
+# and then sends write on the fd. But before sending open itself,
+# the file would have been removed from the mount $M1. open() gets error
+# and the write call which is put into a stub (open had to be sent first)
+# should unwind with the error received in the open call.
+echo "data" >> $M0/testfile 2>/dev/null 1>/dev/null;
+TEST [ $? -ne 0 ]
+
+TEST fd_close $fd;
+
+TEST rm -rf $MOUNTDIR/*
+
+TEST   umount $MOUNTDIR -l
+
+cleanup;
diff --git a/tests/bugs/bug-847622.t b/tests/bugs/bug-847622.t
new file mode 100755
index 000000000..138499527
--- /dev/null
+++ b/tests/bugs/bug-847622.t
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/brick0
+TEST $CLI volume start $V0
+
+sleep 5
+
+TEST mount -t nfs -o vers=3,nolock $H0:/$V0 $N0
+cd $N0
+
+# simple getfacl setfacl commands
+TEST touch testfile
+TEST setfacl -m u:14:r testfile
+TEST getfacl testfile
+
+cd
+TEST umount $N0
+cleanup
+
diff --git a/tests/bugs/bug-847624.t b/tests/bugs/bug-847624.t
new file mode 100755
index 000000000..f4e9942e9
--- /dev/null
+++ b/tests/bugs/bug-847624.t
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+cleanup
+
+#1
+TEST glusterd
+TEST pidof glusterd
+#3
+TEST $CLI volume create $V0 $H0:$B0/$V0
+TEST $CLI volume set $V0 nfs.drc on
+TEST $CLI volume start $V0
+sleep 5
+TEST mount -t nfs -o vers=3,nolock,soft,intr $H0:/$V0 $N0
+cd $N0
+#7
+TEST dbench -t 10 10
+TEST rm -rf $N0/*
+cd
+TEST umount $N0
+#10
+TEST $CLI volume set $V0 nfs.drc-size 10000
+cleanup
diff --git a/tests/bugs/bug-848251.t b/tests/bugs/bug-848251.t
new file mode 100644
index 000000000..dda393272
--- /dev/null
+++ b/tests/bugs/bug-848251.t
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/brick1;
+
+TEST $CLI volume start $V0;
+
+#enable quota
+TEST $CLI volume quota $V0 enable;
+
+#mount on a random dir
+TEST MOUNTDIR="/tmp/$RANDOM"
+TEST mkdir $MOUNTDIR
+TEST glusterfs -s $H0 --volfile-id=$V0 $MOUNTDIR
+
+function set_quota(){
+        mkdir "$MOUNTDIR/$name"
+        $CLI volume quota $V0 limit-usage /$name 50KB
+}
+
+function quota_list(){
+        $CLI volume quota $V0 list | grep -- /$name | awk '{print $3}'
+}
+
+TEST   name=":d1"
+#file name containing ':' in the start
+TEST   set_quota
+EXPECT "0Bytes" quota_list
+
+TEST   name=":d1/d:1"
+#file name containing ':' in between
+TEST   set_quota
+EXPECT "0Bytes" quota_list
+
+TEST   name=":d1/d:1/d1:"
+#file name containing ':' in the end
+TEST   set_quota
+EXPECT "0Bytes" quota_list
+
+TEST   umount $MOUNTDIR
+TEST   rm -rf $MOUNTDIR
+
+cleanup;
diff --git a/tests/bugs/bug-852147.t b/tests/bugs/bug-852147.t
new file mode 100755
index 000000000..0e7923086
--- /dev/null
+++ b/tests/bugs/bug-852147.t
@@ -0,0 +1,85 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+logdir=`gluster --print-logdir`"/bricks"
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST glusterfs -s $H0 --volfile-id=$V0 $M0
+touch $M0/file1;
+
+TEST $CLI volume set $V0 performance.cache-max-file-size 20MB
+TEST $CLI volume set $V0 performance.cache-min-file-size 10MB
+
+EXPECT "20MB" volinfo_field $V0 'performance.cache-max-file-size';
+EXPECT "10MB" volinfo_field $V0 'performance.cache-min-file-size';
+
+#Performing volume reset and verifying.
+TEST $CLI volume reset $V0
+EXPECT "" volinfo_field $V0 'performance.cache-max-file-size';
+EXPECT "" volinfo_field $V0 'performance.cache-min-file-size';
+
+#Verifying vlolume-profile start, info and stop
+EXPECT "Starting volume profile on $V0 has been successful " $CLI volume profile $V0 start
+
+function vol_prof_info()
+{
+    $CLI volume profile $V0 info | grep Brick | wc -l
+}
+EXPECT "8" vol_prof_info
+
+EXPECT "Stopping volume profile on $V0 has been successful " $CLI volume profile $V0 stop
+
+function log-file-name()
+{
+    logfilename=$B0"/"$V0"1.log"
+    echo ${logfilename:1} | tr / -
+}
+
+function file-size()
+{
+    ls -lrt $1 | awk '{print $5}'
+}
+
+#Finding the current log file's size
+log_file=$logdir"/"`log-file-name`
+log_file_size=`file-size $log_file`
+
+#Removing the old backup log files
+ren_file=$log_file".*"
+rm -rf $ren_file
+
+#Initiating log rotate
+TEST $CLI volume log rotate $V0
+
+#Capturing new log file's size
+new_file_size=`file-size $log_file`
+
+#Verifying the size of the new log file and the creation of the backup log file
+TEST ! [ $new_file_size -eq $log_file_size ]
+TEST ls -lrt $ren_file
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-853258.t b/tests/bugs/bug-853258.t
new file mode 100755
index 000000000..faa9d4465
--- /dev/null
+++ b/tests/bugs/bug-853258.t
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+mkdir -p $H0:$B0/${V0}0
+mkdir -p $H0:$B0/${V0}1
+mkdir -p $H0:$B0/${V0}2
+mkdir -p $H0:$B0/${V0}3
+
+# Create and start a volume.
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2
+TEST $CLI volume start $V0
+EXPECT_WITHIN 15 'Started' volinfo_field $V0 'Status';
+
+# Force assignment of initial ranges.
+TEST $CLI volume rebalance $V0 fix-layout start
+EXPECT_WITHIN 15 "fix-layout completed" rebalance_status_field $V0
+
+# Get the original values.
+xattrs=""
+for i in $(seq 0 2); do
+	xattrs="$xattrs $(dht_get_layout $B0/${V0}$i)"
+done
+
+# Expand the volume and force assignment of new ranges.
+TEST $CLI volume add-brick $V0 $H0:$B0/${V0}3
+# Force assignment of initial ranges.
+TEST $CLI volume rebalance $V0 fix-layout start
+EXPECT_WITHIN 15 "fix-layout completed" rebalance_status_field $V0
+
+for i in $(seq 0 3); do
+	xattrs="$xattrs $(dht_get_layout $B0/${V0}$i)"
+done
+
+overlap=$(python2 $(dirname $0)/overlap.py $xattrs)
+# 2863311531 = 0xaaaaaaab = 2/3 overlap
+TEST [ "$overlap" -ge 2863311531 ]
+
+cleanup
diff --git a/tests/bugs/bug-853680.t b/tests/bugs/bug-853680.t
new file mode 100755
index 000000000..72d53ae6c
--- /dev/null
+++ b/tests/bugs/bug-853680.t
@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Bug 853680
+#
+# Test that io-threads least-rate-limit throttling functions as expected. Set
+# a limit, perform a few operations with a least-priority mount and verify
+# said operations take a minimum amount of time according to the limit.
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}1
+TEST $CLI volume start $V0
+
+#Accept min val
+TEST $CLI volume set $V0 performance.least-rate-limit 0
+#Accept some value in between
+TEST $CLI volume set $V0 performance.least-rate-limit 1035
+#Accept max val INT_MAX
+TEST $CLI volume set $V0 performance.least-rate-limit 2147483647
+
+#Reject other values
+TEST ! $CLI volume set $V0 performance.least-rate-limit 2147483648
+TEST ! $CLI volume set $V0 performace.least-rate-limit -8
+TEST ! $CLI volume set $V0 performance.least-rate-limit abc
+TEST ! $CLI volume set $V0 performance.least-rate-limit 0.0
+TEST ! $CLI volume set $V0 performance.least-rate-limit -10.0
+TEST ! $CLI volume set $V0 performance.least-rate-limit 1%
+
+# set rate limit to 1 operation/sec
+TEST $CLI volume set $V0 performance.least-rate-limit 1
+
+# use client-pid=-1 for least priority mount
+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 --client-pid=-1
+
+# create a few files and verify this takes more than a few seconds
+date1=`date +%s`
+TEST touch $M0/file{0..2}
+date2=`date +%s`
+
+optime=$(($date2 - $date1))
+TEST [ $optime -ge 3 ]
+
+TEST umount $M0
+
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup;
diff --git a/tests/bugs/bug-853690.t b/tests/bugs/bug-853690.t
new file mode 100755
index 000000000..77a581f54
--- /dev/null
+++ b/tests/bugs/bug-853690.t
@@ -0,0 +1,94 @@
+#!/bin/bash
+#
+# Bug 853690 - Test that short writes do not lead to corruption.
+#
+# Mismanagement of short writes in AFR leads to corruption and immediately
+# detectable split-brain. Write a file to a replica volume using error-gen
+# to cause short writes on one replica.
+#
+# Short writes are also possible during heal. If ignored, the files are marked
+# consistent and silently differ. After reading the file, cause a lookup, wait
+# for self-heal and verify that the afr xattrs do not match.
+#
+########
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST mkdir -p $B0/test{1,2}
+
+# Our graph is a two brick replica with 100% frequency of short writes on one
+# side of the replica. This guarantees a single write fop leads to an out-of-sync
+# situation.
+cat > $B0/test.vol <<EOF
+volume test-posix-0
+    type storage/posix
+    option directory $B0/test1
+end-volume
+
+volume test-error-0
+    type debug/error-gen
+    option failure 100
+    option enable writev
+    option error-no GF_ERROR_SHORT_WRITE
+    subvolumes test-posix-0
+end-volume
+
+volume test-locks-0
+    type features/locks
+    subvolumes test-error-0
+end-volume
+
+volume test-posix-1
+    type storage/posix
+    option directory $B0/test2
+end-volume
+
+volume test-locks-1
+    type features/locks
+    subvolumes test-posix-1
+end-volume
+
+volume test-replicate-0
+    type cluster/replicate
+    option background-self-heal-count 0
+    subvolumes test-locks-0 test-locks-1
+end-volume
+EOF
+
+TEST glusterd
+
+TEST glusterfs --volfile=$B0/test.vol --attribute-timeout=0 --entry-timeout=0 $M0
+
+# Send a single write, guaranteed to be short on one replica, and attempt to
+# read the data back. Failure to detect the short write results in different
+# file sizes and immediate split-brain (EIO).
+TEST dd if=/dev/zero of=$M0/file bs=128k count=1
+TEST dd if=$M0/file of=/dev/null bs=128k count=1
+
+########
+#
+# Test self-heal with short writes...
+#
+########
+
+# Cause a lookup and wait a few seconds for posterity. This self-heal also fails
+# due to a short write.
+TEST ls $M0/file
+
+# Verify the attributes on the healthy replica do not reflect consistency with
+# the other replica.
+TEST "getfattr -n trusted.afr.test-locks-0 $B0/test2/file --only-values > $B0/out1 2> /dev/null"
+TEST "getfattr -n trusted.afr.test-locks-1 $B0/test2/file --only-values > $B0/out2 2> /dev/null"
+TEST ! cmp $B0/out1 $B0/out2
+
+TEST rm -f $B0/out1 $B0/out2
+TEST rm -f $M0/file
+TEST umount $M0
+
+rm -f $B0/test.vol
+rm -rf $B0/test1 $B0/test2
+
+cleanup;
+
diff --git a/tests/bugs/bug-856455.t b/tests/bugs/bug-856455.t
new file mode 100644
index 000000000..becb20222
--- /dev/null
+++ b/tests/bugs/bug-856455.t
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+BRICK_COUNT=3
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2
+TEST $CLI volume start $V0
+
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+function query_pathinfo()
+{
+    local path=$1;
+    local retval;
+
+    local pathinfo=`getfattr -m . -n trusted.glusterfs.pathinfo $path`;
+    retval=`echo $pathinfo | grep -o 'POSIX' | wc -l`;
+    echo $retval
+}
+
+touch $M0/f00f;
+mkdir $M0/f00d;
+
+# verify pathinfo for a file and directory
+EXPECT 1 query_pathinfo $M0/f00f;
+EXPECT $BRICK_COUNT query_pathinfo $M0/f00d;
+
+# Kill a brick process and then query for pathinfo
+# for directories pathinfo should list backend patch from available (up) subvolumes
+
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0-d-backends-${V0}1.pid`;
+
+EXPECT `expr $BRICK_COUNT - 1` query_pathinfo $M0/f00d;
+
+cleanup;
diff --git a/tests/bugs/bug-857330/common.rc b/tests/bugs/bug-857330/common.rc
new file mode 100644
index 000000000..e5a7cd79a
--- /dev/null
+++ b/tests/bugs/bug-857330/common.rc
@@ -0,0 +1,55 @@
+. $(dirname $0)/../../include.rc
+
+UUID_REGEX='[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}'
+
+TASK_ID=""
+COMMAND=""
+PATTERN=""
+
+function check-and-store-task-id()
+{
+        TASK_ID=""
+
+        local task_id=$($CLI $COMMAND | grep $PATTERN | grep -o -E "$UUID_REGEX")
+
+        if [ -z "$task_id" ] && [ "${task_id+asdf}" = "asdf" ]; then
+                return 1
+        fi
+
+        TASK_ID=$task_id
+        return 0;
+}
+
+function get-task-id()
+{
+        $CLI $COMMAND | grep $PATTERN | grep -o -E "$UUID_REGEX" | tail -n1
+
+}
+
+function check-and-store-task-id-xml()
+{
+        TASK_ID=""
+
+        local task_id=$($CLI $COMMAND --xml | xmllint --format - | grep $PATTERN | grep -o -E "$UUID_REGEX")
+
+        if [ -z "$task_id" ] && [ "${task_id+asdf}" = "asdf" ]; then
+                return 1
+        fi
+
+        TASK_ID=$task_id
+        return 0;
+}
+
+function get-task-id-xml()
+{
+        $CLI $COMMAND --xml | xmllint --format - | grep $PATTERN | grep -o -E "$UUID_REGEX"
+}
+
+function get-task-status()
+{
+        $CLI $COMMAND | grep -o $PATTERN
+        if [ ${PIPESTATUS[0]} -ne 0 ]; then
+                return 1
+        fi
+        return 0
+}
diff --git a/tests/bugs/bug-857330/normal.t b/tests/bugs/bug-857330/normal.t
new file mode 100755
index 000000000..24dfe52c4
--- /dev/null
+++ b/tests/bugs/bug-857330/normal.t
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+. $(dirname $0)/common.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}1;
+TEST $CLI volume info $V0;
+TEST $CLI volume start $V0;
+
+TEST glusterfs -s $H0 --volfile-id=$V0 $M0;
+
+TEST python2 $(dirname $0)/../../utils/create-files.py --multi -b 10 -d 10  -n 10 $M0;
+
+TEST umount $M0;
+
+###############
+## Rebalance ##
+###############
+TEST $CLI volume add-brick $V0 $H0:$B0/${V0}2;
+
+COMMAND="volume rebalance $V0 start"
+PATTERN="ID:"
+TEST check-and-store-task-id
+
+COMMAND="volume status $V0"
+PATTERN="ID"
+EXPECT $TASK_ID get-task-id
+
+COMMAND="volume rebalance $V0 status"
+PATTERN="completed"
+EXPECT_WITHIN 300 $PATTERN get-task-status
+
+###################
+## Replace-brick ##
+###################
+REP_BRICK_PAIR="$H0:$B0/${V0}2 $H0:$B0/${V0}3"
+
+COMMAND="volume replace-brick $V0 $REP_BRICK_PAIR start"
+PATTERN="ID:"
+TEST check-and-store-task-id
+
+COMMAND="volume status $V0"
+PATTERN="ID"
+EXPECT $TASK_ID get-task-id
+
+COMMAND="volume replace-brick $V0 $REP_BRICK_PAIR status"
+PATTERN="complete"
+EXPECT_WITHIN 300 $PATTERN get-task-status
+
+TEST $CLI volume replace-brick $V0 $REP_BRICK_PAIR commit;
+
+##################
+## Remove-brick ##
+##################
+COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 start"
+PATTERN="ID:"
+TEST check-and-store-task-id
+
+COMMAND="volume status $V0"
+PATTERN="ID"
+EXPECT $TASK_ID get-task-id
+
+COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 status"
+PATTERN="completed"
+EXPECT_WITHIN 300 $PATTERN get-task-status
+
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}3 commit
+
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-857330/xml.t b/tests/bugs/bug-857330/xml.t
new file mode 100755
index 000000000..688f46619
--- /dev/null
+++ b/tests/bugs/bug-857330/xml.t
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+. $(dirname $0)/common.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}1;
+TEST $CLI volume info $V0;
+TEST $CLI volume start $V0;
+
+TEST glusterfs -s $H0 --volfile-id=$V0 $M0;
+
+TEST python2 $(dirname $0)/../../utils/create-files.py --multi -b 10 -d 10  -n 10 $M0;
+
+TEST umount $M0;
+
+
+###############
+## Rebalance ##
+###############
+TEST $CLI volume add-brick $V0 $H0:$B0/${V0}2;
+
+COMMAND="volume rebalance $V0 start"
+PATTERN="task-id"
+TEST check-and-store-task-id-xml
+
+COMMAND="volume status $V0"
+PATTERN="id"
+EXPECT $TASK_ID get-task-id-xml
+
+COMMAND="volume rebalance $V0 status"
+PATTERN="task-id"
+EXPECT $TASK_ID get-task-id-xml
+
+## TODO: Add tests for rebalance stop
+
+COMMAND="volume rebalance $V0 status"
+PATTERN="completed"
+EXPECT_WITHIN 300 $PATTERN get-task-status
+
+###################
+## Replace-brick ##
+###################
+REP_BRICK_PAIR="$H0:$B0/${V0}2 $H0:$B0/${V0}3"
+
+COMMAND="volume replace-brick $V0 $REP_BRICK_PAIR start"
+PATTERN="task-id"
+TEST check-and-store-task-id-xml
+
+COMMAND="volume status $V0"
+PATTERN="id"
+EXPECT $TASK_ID get-task-id-xml
+
+COMMAND="volume replace-brick $V0 $REP_BRICK_PAIR status"
+PATTERN="task-id"
+EXPECT $TASK_ID get-task-id-xml
+
+## TODO: Add more tests for replace-brick pause|abort
+
+COMMAND="volume replace-brick $V0 $REP_BRICK_PAIR status"
+PATTERN="complete"
+EXPECT_WITHIN 300 $PATTERN get-task-status
+
+COMMAND="volume replace-brick $V0 $REP_BRICK_PAIR commit"
+PATTERN="task-id"
+EXPECT $TASK_ID get-task-id-xml
+
+##################
+## Remove-brick ##
+##################
+COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 start"
+PATTERN="task-id"
+TEST check-and-store-task-id-xml
+
+COMMAND="volume status $V0"
+PATTERN="id"
+EXPECT $TASK_ID get-task-id-xml
+
+COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 status"
+PATTERN="task-id"
+EXPECT $TASK_ID get-task-id-xml
+
+COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 status"
+PATTERN="completed"
+EXPECT_WITHIN 300 $PATTERN get-task-status
+
+## TODO: Add tests for remove-brick stop
+
+COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 commit"
+PATTERN="task-id"
+EXPECT $TASK_ID get-task-id-xml
+
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-858215.t b/tests/bugs/bug-858215.t
new file mode 100755
index 000000000..aee7d5fcb
--- /dev/null
+++ b/tests/bugs/bug-858215.t
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+## Wait for volume to register with rpc.mountd
+sleep 5;
+
+## Test for checking whether the fops have been saved in the event-history
+TEST ! stat $M0/newfile;
+TEST touch $M0/newfile;
+TEST stat $M0/newfile;
+TEST rm $M0/newfile;
+
+nfs_pid=$(cat /var/lib/glusterd/nfs/run/nfs.pid);
+glustershd_pid=$(cat /var/lib/glusterd/glustershd/run/glustershd.pid);
+
+pids=$(pidof glusterfs);
+for i in $pids
+do
+        if [ $i -ne $nfs_pid ] && [ $i -ne $glustershd_pid ]; then
+                mount_pid=$i;
+                break;
+        fi
+done
+
+dump_dir='/tmp/gerrit_glusterfs'
+cat >$statedumpdir/glusterdump.options <<EOF
+all=yes
+path=$dump_dir
+EOF
+
+TEST mkdir -p $dump_dir;
+TEST kill -USR1 $mount_pid;
+sleep 2;
+for file_name in $(ls $dump_dir)
+do
+    TEST grep "xlator.mount.fuse.history" $dump_dir/$file_name;
+done
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+TEST rm -rf $dump_dir;
+TEST rm $statedumpdir/glusterdump.options;
+
+cleanup;
diff --git a/tests/bugs/bug-858242.c b/tests/bugs/bug-858242.c
new file mode 100644
index 000000000..00a3a2d5f
--- /dev/null
+++ b/tests/bugs/bug-858242.c
@@ -0,0 +1,77 @@
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+int
+main (int argc, char *argv[])
+{
+        char        *filename = NULL, *volname = NULL, *cmd = NULL;
+        char  buffer[1024]    = {0, };
+        int          fd       = -1;
+        int          ret      = -1;
+        struct stat  statbuf  = {0, };
+
+        if (argc != 3) {
+                fprintf (stderr, "usage: %s <file-name> <volname>\n", argv[0]);
+                goto out;
+        }
+
+        filename = argv[1];
+        volname = argv[2];
+
+        fd = open (filename, O_RDWR | O_CREAT, 0);
+        if (fd < 0) {
+                fprintf (stderr, "open (%s) failed (%s)\n", filename,
+                         strerror (errno));
+                goto out;
+        }
+
+        ret = write (fd, "test-content", 12);
+        if (ret < 0) {
+                fprintf (stderr, "write failed (%s)", strerror (errno));
+                goto out;
+        }
+
+        ret = fsync (fd);
+        if (ret < 0) {
+                fprintf (stderr, "fsync failed (%s)", strerror (errno));
+                goto out;
+        }
+
+        ret = fstat64 (fd, &statbuf);
+        if (ret < 0) {
+                fprintf (stderr, "fstat64 failed (%s)", strerror (errno));
+                goto out;
+        }
+
+        ret = asprintf (&cmd, "gluster --mode=script volume stop %s force",
+                        volname);
+        if (ret < 0) {
+                fprintf (stderr, "cannot construct cli command string (%s)",
+                         strerror (errno));
+                goto out;
+        }
+
+        ret = system (cmd);
+        if (ret < 0) {
+                fprintf (stderr, "stopping volume (%s) failed", volname);
+                goto out;
+        }
+
+        ret = read (fd, buffer, 1024);
+        if (ret >= 0) {
+                fprintf (stderr, "read should've returned error, "
+                         "but is successful\n");
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/tests/bugs/bug-858242.t b/tests/bugs/bug-858242.t
new file mode 100755
index 000000000..e93c2d244
--- /dev/null
+++ b/tests/bugs/bug-858242.t
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/brick1;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST $CLI volume set $V0 performance.quick-read off
+
+#mount on a random dir
+TEST glusterfs --entry-timeout=3600 --attribute-timeout=3600 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=yes
+
+build_tester $(dirname $0)/bug-858242.c
+
+TEST $(dirname $0)/bug-858242 $M0/testfile $V0
+
+TEST rm -rf $(dirname $0)/858242
+cleanup;
+
diff --git a/tests/bugs/bug-858488-min-free-disk.t b/tests/bugs/bug-858488-min-free-disk.t
new file mode 100644
index 000000000..ae5ac3bde
--- /dev/null
+++ b/tests/bugs/bug-858488-min-free-disk.t
@@ -0,0 +1,114 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+function pidgrep()
+{
+    ps ax | grep "$1" | grep -v grep | awk '{print $1}' | head -1
+}
+
+## Start glusterd
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+## Lets create partitions for bricks
+TEST truncate -s 100M $B0/brick1
+TEST truncate -s 200M $B0/brick2
+TEST LO1=`losetup --find --show $B0/brick1`
+TEST mkfs.xfs $LO1
+TEST LO2=`losetup --find --show $B0/brick2`
+TEST mkfs.xfs $LO2
+TEST mkdir -p $B0/${V0}1 $B0/${V0}2
+TEST mount -t xfs $LO1 $B0/${V0}1
+TEST mount -t xfs $LO2 $B0/${V0}2
+
+
+## Lets create volume
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+TEST glusterfs -s $H0 --volfile-id=$V0 --acl $M0
+MOUNT_PID=`ps ax |grep "glusterfs -s $H0 --volfile-id=$V0 --acl $M0" | awk '{print $1}' | head -1`
+## Real test starts here
+## ----------------------------------------------------------------------------
+
+MINFREEDISKVALUE=90
+
+## Set min free disk to MINFREEDISKVALUE percent
+TEST $CLI volume set $V0 cluster.min-free-disk $MINFREEDISKVALUE
+
+## We need to have file name to brick map based on hash.
+## We will use this info in test case 0.
+i=1
+CONTINUE=2
+BRICK1FILE=0
+BRICK2FILE=0
+while [[ $CONTINUE -ne 0 ]]
+do
+        dd if=/dev/zero of=$M0/file$i.data bs=1024 count=1024 1>/dev/null 2>&1
+
+        if  [[ -e  $B0/${V0}1/file$i.data &&  $BRICK1FILE = "0" ]]
+        then
+                BRICK1FILE=file$i.data
+                CONTINUE=$CONTINUE-1
+        fi
+
+        if [[ -e  $B0/${V0}2/file$i.data &&  $BRICK2FILE = "0" ]]
+        then
+                BRICK2FILE=file$i.data
+                CONTINUE=$CONTINUE-1
+        fi
+
+        rm $M0/file$i.data
+        let i++
+done
+
+
+## Bring free space on one of the bricks to less than minfree value by
+## creating one big file.
+dd if=/dev/zero of=$M0/fillonebrick.data bs=1024 count=25600 1>/dev/null 2>&1
+
+#Lets find out where it was created
+if [ -f $B0/${V0}1/fillonebrick.data ]
+then
+        FILETOCREATE=$BRICK1FILE
+        OTHERBRICK=$B0/${V0}2
+else
+        FILETOCREATE=$BRICK2FILE
+        OTHERBRICK=$B0/${V0}1
+fi
+
+##--------------------------------TEST CASE 0-----------------------------------
+## If we try to create a file which should go into full brick as per hash, it
+## should go into the other brick instead.
+
+## Before that let us create files just to make gluster refresh the stat
+## Using touch so it should not change the disk usage stats
+for k in {1..20};
+do
+        touch $M0/dummyfile$k
+done
+
+dd if=/dev/zero of=$M0/$FILETOCREATE bs=1024 count=2048 1>/dev/null 2>&1
+TEST [ -e $OTHERBRICK/$FILETOCREATE ]
+
+## Done testing, lets clean up
+EXPECT "$MOUNT_PID" pidgrep $MOUNT_PID
+TEST rm -rf $M0/*
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+$CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-859927.t b/tests/bugs/bug-859927.t
new file mode 100755
index 000000000..ed74d3eb8
--- /dev/null
+++ b/tests/bugs/bug-859927.t
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+cleanup;
+
+glusterd;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+TEST ! $CLI volume set $V0 statedump-path ""
+TEST ! $CLI volume set $V0 statedump-path "     "
+TEST   $CLI volume set $V0 statedump-path "/home/"
+EXPECT "/home/" volume_option $V0 server.statedump-path
+
+TEST ! $CLI volume set $V0 background-self-heal-count ""
+TEST ! $CLI volume set $V0 background-self-heal-count "      "
+TEST   $CLI volume set $V0 background-self-heal-count 10
+EXPECT "10" volume_option $V0 cluster.background-self-heal-count
+
+TEST ! $CLI volume set $V0 cache-size ""
+TEST ! $CLI volume set $V0 cache-size "    "
+TEST   $CLI volume set $V0 cache-size 512MB
+EXPECT "512MB" volume_option $V0 performance.cache-size
+
+TEST ! $CLI volume set $V0 self-heal-daemon ""
+TEST ! $CLI volume set $V0 self-heal-daemon "    "
+TEST   $CLI volume set $V0 self-heal-daemon on
+EXPECT "on" volume_option $V0 cluster.self-heal-daemon
+
+TEST ! $CLI volume set $V0 read-subvolume ""
+TEST ! $CLI volume set $V0 read-subvolume "    "
+TEST   $CLI volume set $V0 read-subvolume $V0-client-0
+EXPECT "$V0-client-0" volume_option  $V0 cluster.read-subvolume
+
+TEST ! $CLI volume set $V0 data-self-heal-algorithm ""
+TEST ! $CLI volume set $V0 data-self-heal-algorithm "     "
+TEST ! $CLI volume set $V0 data-self-heal-algorithm on
+TEST   $CLI volume set $V0 data-self-heal-algorithm full
+EXPECT "full" volume_option $V0 cluster.data-self-heal-algorithm
+
+TEST ! $CLI volume set $V0 min-free-inodes ""
+TEST ! $CLI volume set $V0 min-free-inodes "     "
+TEST   $CLI volume set $V0 min-free-inodes 60%
+EXPECT "60%" volume_option $V0 cluster.min-free-inodes
+
+TEST ! $CLI volume set $V0 min-free-disk ""
+TEST ! $CLI volume set $V0 min-free-disk "     "
+TEST   $CLI volume set $V0 min-free-disk 60%
+EXPECT "60%" volume_option $V0 cluster.min-free-disk
+
+TEST   $CLI volume set $V0 min-free-disk 120
+EXPECT "120" volume_option $V0 cluster.min-free-disk
+
+TEST ! $CLI volume set $V0 frame-timeout ""
+TEST ! $CLI volume set $V0 frame-timeout "      "
+TEST   $CLI volume set $V0 frame-timeout 0
+EXPECT "0" volume_option $V0 network.frame-timeout
+
+TEST ! $CLI volume set $V0 auth.allow ""
+TEST ! $CLI volume set $V0 auth.allow "       "
+TEST   $CLI volume set $V0 auth.allow 192.168.122.1
+EXPECT "192.168.122.1" volume_option $V0 auth.allow
+
+TEST ! $CLI volume set $V0 stripe-block-size ""
+TEST ! $CLI volume set $V0 stripe-block-size "      "
+TEST   $CLI volume set $V0 stripe-block-size 512MB
+EXPECT "512MB" volume_option $V0 cluster.stripe-block-size
+
+cleanup;
diff --git a/tests/bugs/bug-860297.t b/tests/bugs/bug-860297.t
new file mode 100644
index 000000000..2a3ca7a7a
--- /dev/null
+++ b/tests/bugs/bug-860297.t
@@ -0,0 +1,13 @@
+#!/bin/bash
+. $(dirname $0)/../include.rc
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+TEST $CLI volume create $V0 $H0:$B0/brick1
+setfattr -x trusted.glusterfs.volume-id $B0/brick1
+## If Extended attribute trusted.glusterfs.volume-id is not present
+## then volume should not be able to start
+TEST ! $CLI volume start $V0;
+cleanup;
diff --git a/tests/bugs/bug-860663.t b/tests/bugs/bug-860663.t
new file mode 100644
index 000000000..05dea5fbc
--- /dev/null
+++ b/tests/bugs/bug-860663.t
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+function file_count()
+{
+        val=1
+
+        if [ "$1" == "$2" ]
+        then
+                val=0
+        fi
+        echo $val
+}
+
+BRICK_COUNT=3
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2
+TEST $CLI volume start $V0
+
+## Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+sleep 5;
+
+TEST touch $M0/files{1..10000};
+
+ORIG_FILE_COUNT=`ls -l $M0 | wc -l`;
+
+# Kill a brick process
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0-d-backends-${V0}1.pid`;
+
+TEST $CLI volume rebalance $V0 fix-layout start
+
+sleep 30;
+
+TEST ! touch $M0/files{1..10000};
+
+TEST $CLI volume start $V0 force
+
+sleep 5;
+
+NEW_FILE_COUNT=`ls -l $M0 | wc -l`;
+
+EXPECT "0" file_count $ORIG_FILE_COUNT $NEW_FILE_COUNT
+
+cleanup;
diff --git a/tests/bugs/bug-861015-index.t b/tests/bugs/bug-861015-index.t
new file mode 100644
index 000000000..4b148e6cc
--- /dev/null
+++ b/tests/bugs/bug-861015-index.t
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1,2,3,4,5}
+TEST $CLI volume set $V0 ensure-durability off
+TEST $CLI volume start $V0
+EXPECT_WITHIN 20 "Y" glustershd_up_status
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+TEST kill_brick $V0 $H0 $B0/${V0}0
+TEST kill_brick $V0 $H0 $B0/${V0}2
+TEST kill_brick $V0 $H0 $B0/${V0}4
+cd $M0
+HEAL_FILES=0
+for i in {1..10}
+do
+        echo "abc" > $i
+        HEAL_FILES=$(($HEAL_FILES+1))
+done
+HEAL_FILES=$(($HEAL_FILES+3)) #count brick root distribute-subvol num of times
+
+cd ~
+EXPECT "$HEAL_FILES" afr_get_pending_heal_count $V0
+TEST rm -f $M0/*
+TEST umount $M0
+TEST $CLI volume heal $V0 info
+#Only root dir should be present now in the indices
+EXPECT "1" afr_get_num_indices_in_brick $B0/${V0}1
+EXPECT "1" afr_get_num_indices_in_brick $B0/${V0}3
+EXPECT "1" afr_get_num_indices_in_brick $B0/${V0}5
+cleanup
diff --git a/tests/bugs/bug-861015-log.t b/tests/bugs/bug-861015-log.t
new file mode 100644
index 000000000..032032470
--- /dev/null
+++ b/tests/bugs/bug-861015-log.t
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+log_wd=$(gluster --print-logdir)
+TEST glusterd
+TEST pidof glusterd
+rm -f $log_wd/glustershd.log
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+TEST kill_brick $V0 $H0 $B0/${V0}0
+cd $M0
+for i in {1..10}
+do
+        dd if=/dev/urandom of=f bs=1M count=10 2>/dev/null
+done
+
+cd ~
+TEST $CLI volume heal $V0 info
+function count_inode_link_failures {
+        logfile=$1
+        grep "inode link failed on the inode" $logfile | wc -l
+}
+EXPECT "0" count_inode_link_failures $log_wd/glustershd.log
+cleanup
diff --git a/tests/bugs/bug-861542.t b/tests/bugs/bug-861542.t
new file mode 100755
index 000000000..5fd08f12d
--- /dev/null
+++ b/tests/bugs/bug-861542.t
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+# Distributed volume with a single brick was chosen solely for the ease of
+#implementing the test case (to be precise, for the ease of extracting the port number).
+TEST $CLI volume create $V0 $H0:$B0/brick0;
+
+TEST $CLI volume start $V0;
+
+function port_field()
+{
+    local vol=$1;
+    local opt=$2;
+    if [ $opt -eq '0' ]; then
+        $CLI volume status $vol | grep "brick0" | awk '{print $3}';
+    else
+        $CLI volume status $vol detail | grep "^Port " | awk '{print $3}';
+    fi
+}
+
+function xml_port_field()
+{
+    local vol=$1;
+    local opt=$2;
+        $CLI --xml volume status $vol $opt | tr -d '\n' |\
+#Find the first occurrence of the string between <port> and </port>
+        sed -r 's/<port>/&\n/;s/<\/port>/\n&/;s/^.*\n(.*)\n.*$/\1/'| \
+        grep -v xml | tr -d '\n';
+}
+
+TEST $CLI volume status $V0;
+TEST $CLI volume status $V0 detail;
+TEST $CLI --xml volume status $V0;
+TEST $CLI --xml volume status $V0 detail;
+
+# Kill the brick process. After this, port number for the killed (in this case brick) process must be "N/A".
+kill `cat /var/lib/glusterd/vols/$V0/run/$H0-d-backends-brick0.pid`
+
+EXPECT "N/A" port_field $V0 '0'; # volume status
+EXPECT "N/A" port_field $V0 '1'; # volume status detail
+
+EXPECT "N/A" xml_port_field $V0 '';
+EXPECT "N/A" xml_port_field $V0 'detail';
+
+cleanup;
diff --git a/tests/bugs/bug-862834.t b/tests/bugs/bug-862834.t
new file mode 100755
index 000000000..33aaea1a8
--- /dev/null
+++ b/tests/bugs/bug-862834.t
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+V1="patchy2"
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+
+function check_brick()
+{
+        vol=$1;
+        num=$2
+        $CLI volume info $V0 | grep "Brick$num" | awk '{print $2}';
+}
+
+function volinfo_field()
+{
+        local vol=$1;
+        local field=$2;
+
+        $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+function brick_count()
+{
+        local vol=$1;
+
+        $CLI volume info $vol | egrep "^Brick[0-9]+: " | wc -l;
+}
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT '2' brick_count $V0
+
+
+EXPECT "$H0:$B0/${V0}1" check_brick $V0 '1';
+EXPECT "$H0:$B0/${V0}2" check_brick $V0 '2';
+
+TEST ! $CLI volume create $V1 $H0:$B0/${V1}0 $H0:$B0/${V0}1;
+
+cleanup;
diff --git a/tests/bugs/bug-862967.t b/tests/bugs/bug-862967.t
new file mode 100644
index 000000000..00fa88440
--- /dev/null
+++ b/tests/bugs/bug-862967.t
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+function uid_gid_compare()
+{
+        val=1
+
+        if [ "$1" == "$3" ]
+        then
+                if [ "$2" == "$4" ]
+                        then
+                                val=0
+                fi
+        fi
+        echo "$val"
+}
+
+BRICK_COUNT=3
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2
+TEST $CLI volume set $V0 stat-prefetch off
+TEST $CLI volume start $V0
+
+## Mount FUSE
+TEST glusterfs --attribute-timeout=0 --entry-timeout=0 --gid-timeout=-1 -s $H0 --volfile-id $V0 $M0;
+
+# change dir permissions
+mkdir $M0/dir;
+chown 1:1 $M0/dir;
+
+# Kill a brick process
+
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0-d-backends-${V0}1.pid`;
+# change dir ownership
+NEW_UID=36;
+NEW_GID=36;
+chown $NEW_UID:$NEW_GID $M0/dir;
+
+# bring the brick back up
+TEST $CLI volume start $V0 force
+
+sleep 10;
+
+ls -l $M0/dir;
+
+# check if uid/gid is healed on backend brick which was taken down
+BACKEND_UID=`stat --printf=%u $B0/${V0}1/dir`;
+BACKEND_GID=`stat --printf=%g $B0/${V0}1/dir`;
+
+
+EXPECT "0" uid_gid_compare $NEW_UID $NEW_GID $BACKEND_UID $BACKEND_GID
+
+cleanup;
diff --git a/tests/bugs/bug-863068.t b/tests/bugs/bug-863068.t
new file mode 100644
index 000000000..931aad623
--- /dev/null
+++ b/tests/bugs/bug-863068.t
@@ -0,0 +1,76 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+## This function get the No. of entries for
+## gluster volume heal volnmae info healed command for brick1 and brick2
+## and compare the initial value (Before volume heal full) and final value
+## (After gluster volume heal vol full) and compare.
+
+function getdiff()
+{
+	val=10
+	if [ "$1" == "$3" ]
+	then
+		if [ "$2" == "$4" ]
+			then
+				val=0
+		else
+				val=20
+		fi
+      fi
+
+      echo $val
+}
+
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+TEST $CLI volume create $V0 replica 2  $H0:$B0/brick1 $H0:$B0/brick2;
+TEST $CLI volume start $V0;
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0;
+B0_hiphenated=`echo $B0 | tr '/' '-'`
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0$B0_hiphenated-brick1.pid` ;
+
+mkdir $M0/{a,b,c};
+echo "GLUSTERFS" >> $M0/a/file;
+
+TEST $CLI volume start $V0 force;
+sleep 5
+TEST $CLI volume heal $V0 full;
+sleep 5
+
+##First Brick Initial(Before full type self heal) value
+FBI=`gluster volume heal $V0 info healed | grep entries | awk '{print $4}' | head -n 1`
+
+##Second Brick Initial Value
+SBI=`gluster volume heal $V0 info healed | grep entries | awk '{print $4}' | tail -n 1`
+TEST $CLI volume heal $V0 full;
+
+sleep 5
+
+##First Brick Final value
+##Number of entries from output of <gluster volume heal volname info healed>
+
+FBF=`gluster volume heal $V0 info healed | grep entries | awk '{print $4}' | head -n 1`
+
+##Second Brick Final Value
+SBF=`gluster volume heal $V0 info healed | grep entries | awk '{print $4}' | tail -n 1`
+
+##get the difference of values
+EXPECT "0"  getdiff $FBI $SBI $FBF $SBF;
+
+## Tests after this comment checks for the background self heal
+
+TEST mkdir $M0/d
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0$B0_hiphenated-brick1.pid` ;
+TEST $CLI volume set $V0 self-heal-daemon off
+dd if=/dev/random of=$M0/d/file1 bs=100M count=1 2>/dev/null;
+TEST $CLI volume start $V0 force
+sleep 3
+TEST ls -l $M0/d
+
+cleanup;
diff --git a/tests/bugs/bug-864222.t b/tests/bugs/bug-864222.t
new file mode 100755
index 000000000..6e02ab60b
--- /dev/null
+++ b/tests/bugs/bug-864222.t
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/brick0
+TEST $CLI volume start $V0
+
+sleep 5
+
+TEST mount -t nfs -o vers=3,nolock $H0:/$V0 $N0
+cd $N0
+
+TEST ls
+
+TEST $CLI volume set $V0 nfs.enable-ino32 on
+# Main test. This should pass.
+TEST ls
+
+cd
+TEST umount $N0
+cleanup
+
diff --git a/tests/bugs/bug-865825.t b/tests/bugs/bug-865825.t
new file mode 100755
index 000000000..6bb1c2348
--- /dev/null
+++ b/tests/bugs/bug-865825.t
@@ -0,0 +1,76 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+## Start and create a volume
+mkdir -p ${B0}/${V0}-0
+mkdir -p ${B0}/${V0}-1
+mkdir -p ${B0}/${V0}-2
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}-{0,1,2}
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Make sure io-cache and write-behind don't interfere.
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+TEST $CLI volume set $V0 performance.io-cache off;
+TEST $CLI volume set $V0 performance.write-behind off;
+TEST $CLI volume set $V0 performance.stat-prefetch off
+
+## Make sure automatic self-heal doesn't perturb our results.
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Mount native
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+
+## Create a file with some recognizable contents.
+echo "test_data" > $M0/a_file;
+
+## Unmount.
+TEST umount $M0;
+
+## Mess with the flags as though brick-0 accuses brick-2 while brick-1 is
+## missing its brick-2 changelog altogether.
+value=0x000000010000000000000000
+setfattr -n trusted.afr.${V0}-client-2 -v $value $B0/${V0}-0/a_file
+setfattr -x trusted.afr.${V0}-client-2 $B0/${V0}-1/a_file
+echo "wrong_data" > $B0/${V0}-2/a_file
+
+## Remount and force a self-heal.
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+stat ${M0}/a_file > /dev/null
+
+## Make sure brick 2 now has the correct contents.
+EXPECT "test_data" cat $B0/${V0}-2/a_file
+
+if [ "$EXIT_EARLY" = "1" ]; then
+	exit 0;
+fi
+
+## Finish up
+TEST umount $M0;
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-866459.t b/tests/bugs/bug-866459.t
new file mode 100644
index 000000000..d66f70c69
--- /dev/null
+++ b/tests/bugs/bug-866459.t
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+
+## Start and create a volume
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+## Create and start a volume with aio enabled
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2};
+TEST $CLI volume set $V0 linux-aio on
+TEST $CLI volume set $V0 background-self-heal-count 0
+TEST $CLI volume set $V0 performance.stat-prefetch off;
+TEST $CLI volume start $V0
+
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+dd of=$M0/a if=/dev/urandom bs=1M count=1 2>&1 > /dev/null
+B0_hiphenated=`echo $B0 | tr '/' '-'`
+## Bring a brick down
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0$B0_hiphenated-${V0}1.pid`
+EXPECT '1' echo `pgrep glusterfsd | wc -l`
+## Rewrite the file
+dd of=$M0/a if=/dev/urandom bs=1M count=1 2>&1 > /dev/null
+TEST $CLI volume start $V0 force
+## Wait for the brick to give CHILD_UP in client protocol
+sleep 5
+md5offile2=`md5sum $B0/${V0}2/a | awk '{print $1}'`
+
+##trigger self-heal
+ls -l $M0/a
+
+EXPECT "$md5offile2" echo `md5sum $B0/${V0}1/a | awk '{print $1}'`
+
+## Finish up
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-867252.t b/tests/bugs/bug-867252.t
new file mode 100644
index 000000000..8309ed9b9
--- /dev/null
+++ b/tests/bugs/bug-867252.t
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}1;
+
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+
+function brick_count()
+{
+    local vol=$1;
+
+    $CLI volume info $vol | egrep "^Brick[0-9]+: " | wc -l;
+}
+
+
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT '1' brick_count $V0
+
+TEST $CLI volume add-brick $V0 $H0:$B0/${V0}2;
+EXPECT '2' brick_count $V0
+
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}2;
+EXPECT '1' brick_count $V0
+
+cleanup;
diff --git a/tests/bugs/bug-867253.t b/tests/bugs/bug-867253.t
new file mode 100644
index 000000000..ae4e243af
--- /dev/null
+++ b/tests/bugs/bug-867253.t
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+function file_count()
+{
+        val=1
+
+        if [ "$1" == "0" ]
+        then
+                if [ "$2" == "0" ]
+                then
+                        val=0
+                fi
+        fi
+        echo $val
+}
+
+BRICK_COUNT=2
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+TEST $CLI volume start $V0
+
+sleep 5;
+## Mount nfs, with nocache option
+TEST mount -o vers=3,nolock,noac -t nfs $H0:/$V0 $M0;
+
+touch $M0/files{1..1000};
+
+# Kill a brick process
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0-d-backends-${V0}0.pid`;
+
+echo 3 >/proc/sys/vm/drop_caches;
+
+ls -l $M0 >/dev/null;
+
+NEW_FILE_COUNT=`echo $?`;
+
+TEST $CLI volume start $V0 force
+
+# Kill a brick process
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0-d-backends-${V0}1.pid`;
+
+echo 3 >/proc/sys/vm/drop_caches;
+
+ls -l $M0 >/dev/null;
+
+NEW_FILE_COUNT1=`echo $?`;
+
+EXPECT "0" file_count $NEW_FILE_COUNT $NEW_FILE_COUNT1
+
+TEST umount -l $M0
+
+cleanup
diff --git a/tests/bugs/bug-869724.t b/tests/bugs/bug-869724.t
new file mode 100644
index 000000000..eec5d344c
--- /dev/null
+++ b/tests/bugs/bug-869724.t
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+
+## Start and create a volume
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}1;
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+
+## Make volume tightly consistent for metdata
+TEST $CLI volume set $V0 performance.stat-prefetch off;
+
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+touch $M0/test;
+build_tester $(dirname $0)/getlk_owner.c
+
+TEST $(dirname $0)/getlk_owner $M0/test;
+
+rm -f $(dirname $0)/getlk_owner
+cleanup;
+
diff --git a/tests/bugs/bug-872923.t b/tests/bugs/bug-872923.t
new file mode 100755
index 000000000..6757846dc
--- /dev/null
+++ b/tests/bugs/bug-872923.t
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+TEST $CLI volume create $V0 replica 2 $H0:$B0/brick0 $H0:$B0/brick1
+TEST $CLI volume start $V0
+sleep 5
+
+mount -t nfs -o vers=3,nolock `hostname`:/$V0 $N0
+
+cd $N0
+mkdir test_hardlink_self_heal;
+cd test_hardlink_self_heal;
+
+for i in `seq 1 5`;
+do
+    mkdir dir.$i;
+    for j in `seq 1 10`;
+    do
+        dd if=/dev/zero of=dir.$i/file.$j bs=1k count=$j > /dev/null 2>&1;
+    done;
+done;
+
+cd ..
+kill  `cat /var/lib/glusterd/vols/$V0/run/$H0-d-backends-brick0.pid`
+sleep 2
+
+
+cd test_hardlink_self_heal;
+
+RET=0
+for i in `seq 1 5`;
+do
+    for j in `seq 1 10`;
+    do
+        ln dir.$i/file.$j dir.$i/link_file.$j > /dev/null 2>&1;
+        RET=$?
+        if [ $RET -ne 0 ]; then
+           break;
+        fi
+    done ;
+    if [ $RET -ne 0 ]; then
+        break;
+    fi
+done;
+
+cd
+umount $N0
+
+EXPECT "0" echo $RET;
+
+cleanup;
diff --git a/tests/bugs/bug-873367.t b/tests/bugs/bug-873367.t
new file mode 100755
index 000000000..cfbbc98d0
--- /dev/null
+++ b/tests/bugs/bug-873367.t
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+SSL_BASE=/etc/ssl
+SSL_KEY=$SSL_BASE/glusterfs.key
+SSL_CERT=$SSL_BASE/glusterfs.pem
+SSL_CA=$SSL_BASE/glusterfs.ca
+
+cleanup;
+rm -f $SSL_BASE/glusterfs.*
+mkdir -p $B0/1
+mkdir -p $M0
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST openssl genrsa -out $SSL_KEY 1024
+TEST openssl req -new -x509 -key $SSL_KEY -subj /CN=Anyone -out $SSL_CERT
+ln $SSL_CERT $SSL_CA
+
+TEST $CLI volume create $V0 $H0:$B0/1
+TEST $CLI volume set $V0 server.ssl on
+TEST $CLI volume set $V0 client.ssl on
+TEST $CLI volume start $V0
+
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+echo some_data > $M0/data_file
+TEST umount $M0
+
+# If the bug is not fixed, the next mount will fail.
+
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+EXPECT some_data cat $M0/data_file
+
+TEST umount $M0
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup;
diff --git a/tests/bugs/bug-873549.t b/tests/bugs/bug-873549.t
new file mode 100644
index 000000000..5b541de6c
--- /dev/null
+++ b/tests/bugs/bug-873549.t
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd -LDEBUG;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+
+TEST $CLI volume set $V0 performance.cache-size 512MB
+TEST $CLI volume start $V0
+TEST $CLI volume statedump $V0 all
+
+cleanup;
diff --git a/tests/bugs/bug-873962-spb.t b/tests/bugs/bug-873962-spb.t
new file mode 100644
index 000000000..62a8318ed
--- /dev/null
+++ b/tests/bugs/bug-873962-spb.t
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST $CLI volume set $V0 performance.quick-read off
+TEST $CLI volume set $V0 performance.io-cache off
+TEST $CLI volume set $V0 performance.write-behind off
+TEST $CLI volume set $V0 performance.stat-prefetch off
+TEST $CLI volume set $V0 performance.read-ahead off
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+TEST $CLI volume start $V0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable
+touch $M0/a
+
+exec 5<$M0/a
+
+kill_brick $V0 $H0 $B0/${V0}0
+echo "hi" > $M0/a
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+
+kill_brick $V0 $H0 $B0/${V0}1
+echo "bye" > $M0/a
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
+
+TEST ! cat $M0/a #To mark split-brain
+
+TEST ! read -u 5 line
+exec 5<&-
+
+cleanup;
diff --git a/tests/bugs/bug-873962.t b/tests/bugs/bug-873962.t
new file mode 100755
index 000000000..b245cc3da
--- /dev/null
+++ b/tests/bugs/bug-873962.t
@@ -0,0 +1,108 @@
+#!/bin/bash
+
+#AFR TEST-IDENTIFIER SPLIT-BRAIN
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+B0_hiphenated=`echo $B0 | tr '/' '-'`
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2}
+
+# If we allow self-heal to happen in the background, we'll get spurious
+# failures - especially at the point labeled "FAIL HERE" but
+# occasionally elsewhere.  This behavior is very timing-dependent.  It
+# doesn't show up in Jenkins, but it does on JD's and KP's machines, and
+# it got sharply worse because of an unrelated fsync change (6ae6f3d)
+# which changed timing.  Putting anything at the FAIL HERE marker tends
+# to make it go away most of the time on affected machines, even if the
+# "anything" is unrelated.
+#
+# What's going on is that the I/O on the first mountpoint is allowed to
+# complete even though self-heal is still in progress and the state on
+# disk does not reflect its result.  In fact, the state changes during
+# self-heal create the appearance of split brain when the second I/O
+# comes in, so that fails even though we haven't actually been in split
+# brain since the manual xattr operations.  By disallowing background
+# self-heal, we ensure that the second I/O can't happen before self-heal
+# is complete, because it has to follow the first I/O which now has to
+# follow self-heal.
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+
+#Make sure self-heal is not triggered when the bricks are re-started
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST $CLI volume set $V0 performance.stat-prefetch off
+TEST $CLI volume start $V0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable
+TEST touch $M0/a
+TEST touch $M0/b
+TEST touch $M0/c
+TEST touch $M0/d
+echo "1" > $M0/b
+echo "1" > $M0/d
+TEST kill_brick $V0 $H0 $B0/${V0}2
+echo "1" > $M0/a
+echo "1" > $M0/c
+TEST setfattr -n trusted.mdata -v abc $M0/b
+TEST setfattr -n trusted.mdata -v abc $M0/d
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
+TEST kill_brick $V0 $H0 $B0/${V0}1
+echo "2" > $M0/a
+echo "2" > $M0/c
+TEST setfattr -n trusted.mdata -v def $M0/b
+TEST setfattr -n trusted.mdata -v def $M0/d
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
+
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M1 --direct-io-mode=enable
+#Files are in split-brain, so open should fail
+TEST ! cat $M0/a;
+TEST ! cat $M1/a;
+TEST ! cat $M0/b;
+TEST ! cat $M1/b;
+
+#Reset split-brain status
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/a;
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/b;
+
+#The operations should do self-heal and give correct output
+EXPECT "2" cat $M0/a;
+# FAIL HERE - see comment about cluster.self-heal-background-count above.
+EXPECT "2" cat $M1/a;
+EXPECT "def" getfattr -n trusted.mdata --only-values $M0/b 2>/dev/null
+EXPECT "def" getfattr -n trusted.mdata --only-values $M1/b 2>/dev/null
+
+TEST umount $M0
+TEST umount $M1
+
+TEST $CLI volume set $V0 cluster.data-self-heal off
+TEST $CLI volume set $V0 cluster.metadata-self-heal off
+
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M1 --direct-io-mode=enable
+
+#Files are in split-brain, so open should fail
+TEST ! cat $M0/c
+TEST ! cat $M1/c
+TEST ! cat $M0/d
+TEST ! cat $M1/d
+
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/c
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/d
+
+#The operations should NOT do self-heal but give correct output
+EXPECT "2" cat $M0/c
+EXPECT "2" cat $M1/c
+EXPECT "1" cat $M0/d
+EXPECT "1" cat $M1/d
+
+#Check that the self-heal is not triggered.
+EXPECT "1" cat $B0/${V0}1/c
+EXPECT "abc" getfattr -n trusted.mdata --only-values $B0/${V0}1/d 2>/dev/null
+cleanup;
diff --git a/tests/bugs/bug-874498.t b/tests/bugs/bug-874498.t
new file mode 100644
index 000000000..0b5991011
--- /dev/null
+++ b/tests/bugs/bug-874498.t
@@ -0,0 +1,61 @@
+#!/bin/bash
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../afr.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+TEST $CLI volume create $V0 replica 2  $H0:$B0/brick1 $H0:$B0/brick2;
+TEST $CLI volume start $V0;
+
+
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0;
+B0_hiphenated=`echo $B0 | tr '/' '-'`
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0$B0_hiphenated-brick1.pid` ;
+
+echo "GLUSTER FILE SYSTEM" > $M0/FILE1
+echo "GLUSTER FILE SYSTEM" > $M0/FILE2
+
+FILEN=$B0"/brick2"
+XATTROP=$FILEN/.glusterfs/indices/xattrop
+
+function get_gfid()
+{
+path_of_file=$1
+
+gfid_value=`getfattr -d -m . $path_of_file -e hex 2>/dev/null |  grep trusted.gfid | cut --complement -c -15 | sed 's/\([a-f0-9]\{8\}\)\([a-f0-9]\{4\}\)\([a-f0-9]\{4\}\)\([a-f0-9]\{4\}\)/\1-\2-\3-\4-/'`
+
+echo $gfid_value
+}
+
+GFID_ROOT=`get_gfid $B0/brick2`
+GFID_FILE1=`get_gfid $B0/brick2/FILE1`
+GFID_FILE2=`get_gfid $B0/brick2/FILE2`
+
+
+count=0
+for i in `ls $XATTROP`
+do
+ if [ "$i" == "$GFID_ROOT" ] || [ "$i" == "$GFID_FILE1" ] || [ "$i" == "$GFID_FILE2" ]
+        then
+ count=$(( count + 1 ))
+ fi
+done
+
+EXPECT "3" echo $count
+
+
+TEST $CLI volume start $V0 force
+sleep 5
+TEST $CLI volume heal $V0
+
+
+##Expected number of entries are 0 in the .glusterfs/indices/xattrop directory
+EXPECT_WITHIN 60 '0' count_sh_entries $FILEN;
+
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-877293.t b/tests/bugs/bug-877293.t
new file mode 100755
index 000000000..774c2a0cc
--- /dev/null
+++ b/tests/bugs/bug-877293.t
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+TEST glusterd
+TEST pidof glusterd
+
+## Start and create a replicated volume
+mkdir -p ${B0}/${V0}-0
+mkdir -p ${B0}/${V0}-1
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}-{0,1}
+
+TEST $CLI volume set $V0 indexing on
+
+TEST $CLI volume start $V0;
+
+## Mount native
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+
+## Mount client-pid=-1
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 --client-pid=-1 $M1
+
+TEST touch $M0
+
+vol_uuid=`getfattr -n trusted.glusterfs.volume-mark -ehex $M1 | sed -n 's/^trusted.glusterfs.volume-mark=0x//p' | cut -b5-36 | sed 's/\([a-f0-9]\{8\}\)\([a-f0-9]\{4\}\)\([a-f0-9]\{4\}\)\([a-f0-9]\{4\}\)/\1-\2-\3-\4-/'`
+xtime=trusted.glusterfs.$vol_uuid.xtime
+
+TEST "getfattr -n $xtime $M1 | grep -q ${xtime}="
+
+TEST kill_brick $V0 $H0 $B0/${V0}-0
+
+TEST "getfattr -n $xtime $M1 | grep -q ${xtime}="
+
+TEST umount $M0
+TEST umount $M1
+
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+
+cleanup
diff --git a/tests/bugs/bug-877885.t b/tests/bugs/bug-877885.t
new file mode 100755
index 000000000..0d4620b00
--- /dev/null
+++ b/tests/bugs/bug-877885.t
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/brick0 $H0:$B0/brick1
+TEST $CLI volume start $V0
+
+sleep 5
+
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 \
+$M0;
+
+TEST touch $M0/file
+TEST mkdir $M0/dir
+
+TEST mount -t nfs -o vers=3,nolock $H0:/$V0 $N0
+cd $N0
+
+rm -rf * &
+
+TEST mount -t nfs -o retry=0,nolock,vers=3 $H0:/$V0 $N1;
+
+cd;
+
+kill %1;
+
+TEST umount $N0
+TEST umount $N1;
+
+cleanup
diff --git a/tests/bugs/bug-877992.t b/tests/bugs/bug-877992.t
new file mode 100755
index 000000000..932ecc77b
--- /dev/null
+++ b/tests/bugs/bug-877992.t
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+
+## Start and create a volume
+TEST glusterd -LDEBUG
+TEST pidof glusterd
+
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+
+function hooks_prep ()
+{
+    local event=$1
+    touch /tmp/pre.out /tmp/post.out
+    touch /var/lib/glusterd/hooks/1/"$event"/pre/Spre.sh
+    touch /var/lib/glusterd/hooks/1/"$event"/post/Spost.sh
+
+    printf "#! /bin/bash\necho "$event"Pre > /tmp/pre.out\n" > /var/lib/glusterd/hooks/1/"$event"/pre/Spre.sh
+    printf "#! /bin/bash\necho "$event"Post > /tmp/post.out\n" > /var/lib/glusterd/hooks/1/"$event"/post/Spost.sh
+    chmod a+x /var/lib/glusterd/hooks/1/"$event"/pre/Spre.sh
+    chmod a+x /var/lib/glusterd/hooks/1/"$event"/post/Spost.sh
+}
+
+function hooks_cleanup ()
+{
+    local event=$1
+    rm /tmp/pre.out /tmp/post.out
+    rm /var/lib/glusterd/hooks/1/"$event"/pre/Spre.sh
+    rm /var/lib/glusterd/hooks/1/"$event"/post/Spost.sh
+}
+
+## Verify volume is created and its hooks script ran
+hooks_prep 'create'
+TEST $CLI volume create $V0 $H0:$B0/${V0}1;
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'createPre' cat /tmp/pre.out;
+EXPECT 'createPost' cat /tmp/post.out;
+hooks_cleanup 'create'
+
+
+## Start volume and verify that its hooks script ran
+hooks_prep 'start'
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+EXPECT 'startPre' cat /tmp/pre.out;
+EXPECT 'startPost' cat /tmp/post.out;
+hooks_cleanup 'start'
+
+cleanup;
diff --git a/tests/bugs/bug-878004.t b/tests/bugs/bug-878004.t
new file mode 100644
index 000000000..5bee4c62f
--- /dev/null
+++ b/tests/bugs/bug-878004.t
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}2 $H0:$B0/${V0}3;
+
+function brick_count()
+{
+    local vol=$1;
+
+    $CLI volume info $vol | egrep "^Brick[0-9]+: " | wc -l;
+}
+
+
+TEST $CLI volume start $V0
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}2;
+EXPECT '2' brick_count $V0
+
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}3;
+EXPECT '1' brick_count $V0
+
+cleanup;
+
diff --git a/tests/bugs/bug-879490.t b/tests/bugs/bug-879490.t
new file mode 100755
index 000000000..5b9ae7bb9
--- /dev/null
+++ b/tests/bugs/bug-879490.t
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+function peer_probe()
+{
+    $CLI peer probe a.b.c.d --xml | xmllint --format - | grep "<opErrstr>"
+}
+
+EXPECT "  <opErrstr>Probe returned with unknown errno 107</opErrstr>" peer_probe
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-879494.t b/tests/bugs/bug-879494.t
new file mode 100755
index 000000000..5caca7922
--- /dev/null
+++ b/tests/bugs/bug-879494.t
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+function peer_probe()
+{
+    $CLI peer detach a.b.c.d --xml | xmllint --format - | grep "<opErrstr>"
+}
+
+EXPECT "  <opErrstr>a.b.c.d is not part of cluster</opErrstr>" peer_probe
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-880898.t b/tests/bugs/bug-880898.t
new file mode 100644
index 000000000..a069d4a8a
--- /dev/null
+++ b/tests/bugs/bug-880898.t
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/brick1 $H0:$B0/brick2
+TEST $CLI volume start $V0
+pkill glusterfs
+uuid=""
+for line in $(cat /var/lib/glusterd/glusterd.info)
+do
+	if [[ $line == UUID* ]]
+	then
+		uuid=`echo $line | sed -r 's/^.{5}//'`
+	fi
+done
+
+gluster volume heal $V0 info | grep "Status: self-heal-daemon is not running on $uuid";
+EXPECT "0" echo $?
+
+cleanup;
diff --git a/tests/bugs/bug-882278.t b/tests/bugs/bug-882278.t
new file mode 100755
index 000000000..7933e1863
--- /dev/null
+++ b/tests/bugs/bug-882278.t
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+cleanup
+
+# Is there a good reason to require --fqdn elsewhere?  It's worse than useless
+# here.
+H0=$(hostname -s)
+
+function recreate {
+	# The rm is necessary so we don't get fooled by leftovers from old runs.
+	rm -rf $1 && mkdir -p $1
+}
+
+function count_lines {
+	grep "$1" $2/* | wc -l
+}
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+## Start and create a volume
+TEST recreate ${B0}/${V0}-0
+TEST recreate ${B0}/${V0}-1
+TEST $CLI volume create $V0 $H0:$B0/${V0}-{0,1}
+TEST $CLI volume set $V0 cluster.nufa on
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Mount native
+special_option="--xlator-option ${V0}-dht.local-volume-name=${V0}-client-1"
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $special_option $M0
+
+## Create a bunch of test files.
+for i in $(seq 0 99); do
+	echo hello > $(printf $M0/file%02d $i)
+done
+
+## Make sure the files went to the right place.  There might be link files in
+## the other brick, but they won't have any contents.
+EXPECT "0" count_lines hello ${B0}/${V0}-0
+EXPECT "100" count_lines hello ${B0}/${V0}-1
+
+if [ "$EXIT_EARLY" = "1" ]; then
+	exit 0;
+fi
+
+## Finish up
+TEST umount $M0;
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-884328.t b/tests/bugs/bug-884328.t
new file mode 100644
index 000000000..ee5509bbc
--- /dev/null
+++ b/tests/bugs/bug-884328.t
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+TEST check_option_help_presence "cluster.quorum-type"
+TEST check_option_help_presence "cluster.quorum-count"
+cleanup;
diff --git a/tests/bugs/bug-884452.t b/tests/bugs/bug-884452.t
new file mode 100644
index 000000000..d07651e46
--- /dev/null
+++ b/tests/bugs/bug-884452.t
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/$V0
+TEST $CLI volume start $V0
+
+TEST glusterfs -s $H0 --volfile-id $V0 $M0
+TEST touch $M0/{1..10000}
+
+RUN_LS_LOOP_FILE="$M0/run-ls-loop"
+function ls-loop
+{
+  while [ -f $RUN_LS_LOOP_FILE ]; do
+    ls -lR $M0 1>/dev/null 2>&1
+  done;
+}
+
+touch $RUN_LS_LOOP_FILE
+ls-loop &
+
+function vol-status-loop
+{
+  for i in {1..1000}; do
+    $CLI volume status $V0 clients >/dev/null 2>&1
+    if [ $? -ne 0 ]; then
+      return 1
+    fi
+  done;
+
+  return 0
+}
+
+TEST vol-status-loop
+
+rm -f $RUN_LS_LOOP_FILE
+wait
+
+TEST umount $M0
+
+cleanup;
diff --git a/tests/bugs/bug-884455.t b/tests/bugs/bug-884455.t
new file mode 100755
index 000000000..3b3a2241e
--- /dev/null
+++ b/tests/bugs/bug-884455.t
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../dht.rc
+
+cleanup;
+
+function layout_compare()
+{
+        res=0
+
+        if [ "$1" == "$2" ]
+        then
+                res=1
+        fi
+        if [ "$1" == "$3" ]
+        then
+                res=1
+        fi
+        if [ "$2" == "$3" ]
+        then
+                res=1
+        fi
+
+        echo $res
+}
+
+function get_layout()
+{
+        layout1=`getfattr -n trusted.glusterfs.dht -e hex $1 2>&1|grep dht |cut -d = -f2`
+        layout2=`getfattr -n trusted.glusterfs.dht -e hex $2 2>&1|grep dht |cut -d = -f2`
+        layout3=`getfattr -n trusted.glusterfs.dht -e hex $3 2>&1|grep dht |cut -d = -f2`
+
+        ret=$(layout_compare $layout1 $layout2 $layout3)
+
+        if [ $ret -ne 0 ]
+        then
+                echo 1
+        else
+                echo 0
+        fi
+
+}
+
+BRICK_COUNT=3
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+## set subvols-per-dir option
+TEST $CLI volume set $V0 subvols-per-directory 2
+TEST $CLI volume start $V0
+
+## Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+
+TEST mkdir $M0/dir{1..10} 2>/dev/null;
+
+## Add-brick n run rebalance to force re-write of layout
+TEST $CLI volume add-brick $V0 $H0:$B0/${V0}2
+sleep 5;
+
+## trigger dir self heal on client
+TEST ls -l $M0 2>/dev/null;
+
+TEST $CLI volume rebalance $V0 start force
+
+EXPECT_WITHIN 30 "0" rebalance_completed
+
+## check for layout overlaps.
+EXPECT "0" get_layout $B0/${V0}0 $B0/${V0}1 $B0/${V0}2
+EXPECT "0" get_layout $B0/${V0}0/dir1 $B0/${V0}1/dir1 $B0/${V0}2/dir1
+EXPECT "0" get_layout $B0/${V0}0/dir2 $B0/${V0}1/dir2 $B0/${V0}2/dir2
+EXPECT "0" get_layout $B0/${V0}0/dir3 $B0/${V0}1/dir3 $B0/${V0}2/dir3
+EXPECT "0" get_layout $B0/${V0}0/dir4 $B0/${V0}1/dir4 $B0/${V0}2/dir4
+EXPECT "0" get_layout $B0/${V0}0/dir5 $B0/${V0}1/dir5 $B0/${V0}2/dir5
+EXPECT "0" get_layout $B0/${V0}0/dir6 $B0/${V0}1/dir6 $B0/${V0}2/dir6
+EXPECT "0" get_layout $B0/${V0}0/dir7 $B0/${V0}1/dir7 $B0/${V0}2/dir7
+EXPECT "0" get_layout $B0/${V0}0/dir8 $B0/${V0}1/dir8 $B0/${V0}2/dir8
+EXPECT "0" get_layout $B0/${V0}0/dir9 $B0/${V0}1/dir9 $B0/${V0}2/dir9
+EXPECT "0" get_layout $B0/${V0}0/dir10 $B0/${V0}1/dir10 $B0/${V0}2/dir10
+
+cleanup;
diff --git a/tests/bugs/bug-884597.t b/tests/bugs/bug-884597.t
new file mode 100755
index 000000000..8eb1f330b
--- /dev/null
+++ b/tests/bugs/bug-884597.t
@@ -0,0 +1,152 @@
+#!/bin/bash
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../dht.rc
+
+cleanup;
+BRICK_COUNT=3
+function uid_gid_compare()
+{
+        val=1
+
+        if [ "$1" == "$3" ]
+        then
+                if [ "$2" == "$4" ]
+                        then
+                                val=0
+                fi
+        fi
+        echo "$val"
+}
+
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2
+TEST $CLI volume start $V0
+
+## Mount FUSE
+TEST glusterfs --attribute-timeout=0 --entry-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+i=1
+NEW_UID=36
+NEW_GID=36
+
+TEST touch $M0/$i
+
+chown $NEW_UID:$NEW_GID $M0/$i
+## rename till file gets a linkfile
+
+while [ $i -ne 0 ]
+do
+        TEST mv $M0/$i $M0/$(( $i+1 ))
+        let i++
+        file_has_linkfile $i
+        has_link=$?
+        if [ $has_link -eq 2 ]
+        then
+                break;
+        fi
+done
+
+get_hashed_brick $i
+cached=$?
+
+# check if uid/gid on linkfile is created with correct uid/gid
+BACKEND_UID=`stat --printf=%u $B0/${V0}$cached/$i`;
+BACKEND_GID=`stat --printf=%g $B0/${V0}$cached/$i`;
+
+EXPECT "0" uid_gid_compare $NEW_UID $NEW_GID $BACKEND_UID $BACKEND_GID
+
+# remove linkfile from backend, and trigger a lookup heal. uid/gid should match
+rm -rf $B0/${V0}$cached/$i
+
+# without a unmount, we are not able to trigger a lookup based heal
+
+TEST umount $M0
+
+## Mount FUSE
+TEST glusterfs --attribute-timeout=0 --entry-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+lookup=`ls -l $M0/$i 2>/dev/null`
+
+# check if uid/gid on linkfile is created with correct uid/gid
+BACKEND_UID=`stat --printf=%u $B0/${V0}$cached/$i`;
+BACKEND_GID=`stat --printf=%g $B0/${V0}$cached/$i`;
+
+EXPECT "0" uid_gid_compare $NEW_UID $NEW_GID $BACKEND_UID $BACKEND_GID
+# create hardlinks. Make sure a linkfile gets created
+
+i=1
+NEW_UID=36
+NEW_GID=36
+
+TEST touch $M0/file
+chown $NEW_UID:$NEW_GID $M0/file;
+
+## ln till file gets a linkfile
+
+while [ $i -ne 0 ]
+do
+        TEST ln $M0/file $M0/link$i
+
+        file_has_linkfile link$i
+        has_link=$?
+        if [ $has_link -eq 2 ]
+        then
+                break;
+        fi
+        let i++
+done
+
+get_hashed_brick link$i
+cached=$?
+
+# check if uid/gid on linkfile is created with correct uid/gid
+BACKEND_UID=`stat --printf=%u $B0/${V0}$cached/link$i`;
+BACKEND_GID=`stat --printf=%g $B0/${V0}$cached/link$i`;
+
+EXPECT "0" uid_gid_compare $NEW_UID $NEW_GID $BACKEND_UID $BACKEND_GID
+
+## UID/GID creation as different user
+i=1
+NEW_UID=36
+NEW_GID=36
+
+TEST touch $M0/user_file1
+TEST chown $NEW_UID:$NEW_GID $M0/user_file1;
+
+## Give permission on volume, so that different users can perform rename
+
+TEST chmod 0777 $M0
+
+## Add a user known as ABC and perform renames
+TEST `useradd -M ABC 2>/dev/null`
+
+TEST cd $M0
+## rename as different user till file gets a linkfile
+
+while [ $i -ne 0 ]
+do
+        su -c "mv $M0/user_file$i $M0/user_file$(( $i+1 ))" ABC
+        let i++
+        file_has_linkfile user_file$i
+        has_link=$?
+        if [ $has_link -eq 2 ]
+        then
+                break;
+        fi
+done
+
+## del user ABC
+TEST userdel ABC
+
+get_hashed_brick user_file$i
+cached=$?
+
+# check if uid/gid on linkfile is created with correct uid/gid
+BACKEND_UID=`stat --printf=%u $B0/${V0}$cached/user_file$i`;
+BACKEND_GID=`stat --printf=%g $B0/${V0}$cached/user_file$i`;
+
+EXPECT "0" uid_gid_compare $NEW_UID $NEW_GID $BACKEND_UID $BACKEND_GID
+cleanup;
diff --git a/tests/bugs/bug-886998.t b/tests/bugs/bug-886998.t
new file mode 100644
index 000000000..7a905a113
--- /dev/null
+++ b/tests/bugs/bug-886998.t
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+# This tests that the replicate trash directory(.landfill) has following
+# properties.
+# Note: This is to have backward compatibility with 3.3 glusterfs
+#       In the latest releases this dir is present inside .glusterfs of brick.
+# 1) lookup of trash dir fails
+# 2) readdir does not show this directory
+# 3) Self-heal does not do any self-heal of these directories.
+gfid1="0xc2e75dde97f346e7842d1076a8e699f8"
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 --direct-io-mode=enable
+
+TEST mkdir $B0/${V0}1/.landfill
+TEST setfattr -n trusted.gfid -v $gfid1 $B0/${V0}1/.landfill
+TEST mkdir $B0/${V0}0/.landfill
+TEST setfattr -n trusted.gfid -v $gfid1 $B0/${V0}0/.landfill
+
+TEST ! stat $M0/.landfill
+EXPECT "" echo $(ls -a $M0 | grep ".landfill")
+
+TEST rmdir $B0/${V0}0/.landfill
+#Force a conservative merge and it should not create .landfill
+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000000 $B0/${V0}0/
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/${V0}0/
+
+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000001 $B0/${V0}1/
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/
+
+EXPECT "" echo $(ls -a $M0 | grep ".landfill")
+TEST ! stat $B0/${V0}0/.landfill
+TEST stat $B0/${V0}1/.landfill
+
+#TEST that the dir is not deleted even when xattrs suggest to delete
+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000000 $B0/${V0}0/
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000001 $B0/${V0}0/
+
+TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000000000000000000000 $B0/${V0}1/
+TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000000000000000000000 $B0/${V0}1/
+
+EXPECT "" echo $(ls -a $M0 | grep ".landfill")
+TEST ! stat $B0/${V0}0/.landfill
+TEST stat $B0/${V0}1/.landfill
+cleanup;
diff --git a/tests/bugs/bug-887098-gmount-crash.t b/tests/bugs/bug-887098-gmount-crash.t
new file mode 100644
index 000000000..1998b4062
--- /dev/null
+++ b/tests/bugs/bug-887098-gmount-crash.t
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+function pidgrep()
+{
+    ps ax | grep "$1" | grep -v grep | awk '{print $1}' | head -1
+}
+
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST glusterfs -s $H0 --volfile-id=$V0 --acl $M0
+MOUNT_PID=`ps ax |grep "glusterfs -s $H0 --volfile-id=$V0 --acl $M0" | grep -v grep | awk '{print $1}' | head -1`
+
+for i in {1..25};
+do
+    mkdir $M0/tmp_$i && cat /etc/hosts > $M0/tmp_$i/file
+    cp -RPp $M0/tmp_$i $M0/newtmp_$i && cat /etc/hosts > $M0/newtmp_$i/newfile
+done
+
+EXPECT "$MOUNT_PID" pidgrep $MOUNT_PID
+TEST rm -rf $M0/*
+umount $M0
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-887145.t b/tests/bugs/bug-887145.t
new file mode 100755
index 000000000..e2013e50b
--- /dev/null
+++ b/tests/bugs/bug-887145.t
@@ -0,0 +1,89 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../nfs.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2};
+TEST $CLI volume set $V0 performance.open-behind off;
+TEST $CLI volume start $V0
+
+sleep 2;
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+EXPECT_WITHIN 20 "1" is_nfs_export_available;
+
+
+useradd tmp_user 2>/dev/null 1>/dev/null;
+mkdir $M0/dir;
+mkdir $M0/other;
+cp /etc/passwd $M0/;
+cp $M0/passwd $M0/file;
+chmod 600 $M0/file;
+
+TEST mount -t nfs -o vers=3,nolock $H0:/$V0 $N0;
+
+chown -R nfsnobody:nfsnobody $M0/dir;
+chown -R tmp_user:tmp_user $M0/other;
+
+TEST $CLI volume set $V0 server.root-squash on;
+
+sleep 2;
+
+EXPECT_WITHIN 20 "1" is_nfs_export_available;
+
+# create files and directories in the root of the glusterfs and nfs mount
+# which is owned by root and hence the right behavior is getting EACCESS
+# as the fops are executed as nfsnobody.
+touch $M0/foo 2>/dev/null;
+TEST [ $? -ne 0 ]
+touch $N0/foo 2>/dev/null;
+TEST [ $? -ne 0 ]
+mkdir $M0/new 2>/dev/null;
+TEST [ $? -ne 0 ]
+mkdir $N0/new 2>/dev/null;
+TEST [ $? -ne 0 ]
+cp $M0/file $M0/tmp_file 2>/dev/null;
+TEST [ $? -ne 0 ]
+cp $N0/file $N0/tmp_file 2>/dev/null;
+TEST [ $? -ne 0 ]
+cat $M0/file 2>/dev/null;
+TEST [ $? -ne 0 ]
+# here read should be allowed because eventhough file "passwd" is owned
+# by root, the permissions if the file allow other users to read it.
+cat $M0/passwd 1>/dev/null;
+TEST [ $? -eq 0 ]
+cat $N0/passwd 1>/dev/null;
+TEST [ $? -eq 0 ]
+
+# create files and directories should succeed as the fops are being executed
+# inside the directory owned by nfsnobody
+TEST touch $M0/dir/file;
+TEST touch $N0/dir/foo;
+TEST mkdir $M0/dir/new;
+TEST mkdir $N0/dir/other;
+TEST rm -f $M0/dir/file $M0/dir/foo;
+TEST rmdir $N0/dir/*;
+
+# create files and directories here should fail as other directory is owned
+# by tmp_user.
+touch $M0/other/foo 2>/dev/null;
+TEST [ $? -ne 0 ]
+touch $N0/other/foo 2>/dev/null;
+TEST [ $? -ne 0 ]
+mkdir $M0/other/new 2>/dev/null;
+TEST [ $? -ne 0 ]
+mkdir $N0/other/new 2>/dev/null;
+TEST [ $? -ne 0 ]
+
+userdel tmp_user;
+rm -rf /home/tmp_user;
+
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-888174.t b/tests/bugs/bug-888174.t
new file mode 100644
index 000000000..4ea34645b
--- /dev/null
+++ b/tests/bugs/bug-888174.t
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+#This tests if flush, fsync wakes up the delayed post-op or not.
+#If it is not woken up, INODELK from the next command waits
+#for post-op-delay secs. There would be pending changelog even after the command
+#completes.
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/r2_0 $H0:$B0/r2_1
+
+TEST $CLI volume set $V0 cluster.eager-lock on
+
+TEST $CLI volume set $V0 performance.flush-behind off
+EXPECT "off" volume_option $V0 performance.flush-behind
+
+TEST $CLI volume set $V0 cluster.post-op-delay-secs 3
+EXPECT "3" volume_option $V0 cluster.post-op-delay-secs
+
+TEST $CLI volume start $V0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0
+
+#Check that INODELK MAX latency is not in the order of seconds
+TEST gluster volume profile $V0 start
+for i in {1..5}
+do
+        echo hi > $M0/a
+done
+#Test if the MAX INODELK fop latency is of the order of seconds.
+inodelk_max_latency=$($CLI volume profile $V0 info | grep INODELK | awk 'BEGIN {max = 0} {if ($6 > max) max=$6;} END {print max}' | cut -d. -f 1 | egrep "[0-9]{7,}")
+
+TEST [ -z $inodelk_max_latency ]
+
+TEST dd of=$M0/a if=/dev/urandom bs=1M count=10 conv=fsync
+#Check for no trace of pending changelog. Flush should make sure of it.
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_0/a trusted.afr.$V0-client-0
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_0/a trusted.afr.$V0-client-1
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_1/a trusted.afr.$V0-client-0
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_1/a trusted.afr.$V0-client-1
+
+dd of=$M0/a if=/dev/urandom bs=1M count=1024 2>/dev/null &
+p=$!
+#trigger graph switches, tests for fsync not leaving any pending flags
+TEST $CLI volume set $V0 performance.quick-read off
+TEST $CLI volume set $V0 performance.io-cache off
+TEST $CLI volume set $V0 performance.stat-prefetch off
+TEST $CLI volume set $V0 performance.read-ahead off
+
+kill -SIGTERM $p
+#wait for dd to exit
+wait  > /dev/null 2>&1
+
+#Goal is to check if there is permanent FOOL changelog
+sleep 5
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_0/a trusted.afr.$V0-client-0
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_0/a trusted.afr.$V0-client-1
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_1/a trusted.afr.$V0-client-0
+EXPECT "0x000000000000000000000000" afr_get_changelog_xattr $B0/r2_1/a trusted.afr.$V0-client-1
+
+cleanup;
diff --git a/tests/bugs/bug-888752.t b/tests/bugs/bug-888752.t
new file mode 100644
index 000000000..56d3f9ffb
--- /dev/null
+++ b/tests/bugs/bug-888752.t
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../cluster.rc
+
+# Check if xml output is generated correctly for volume status for a single brick
+# present on another peer and no async tasks are running.
+
+function get_peer_count {
+        $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+cleanup
+
+TEST launch_cluster 2;
+TEST $CLI_1 peer probe $H2;
+EXPECT_WITHIN 5 1 get_peer_count
+TEST $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0
+TEST $CLI_1 volume start $V0
+
+TEST $CLI_1 volume status $V0 $H2:$B2/$V0 --xml
+
+TEST $CLI_1 volume stop $V0
+
+cleanup
diff --git a/tests/bugs/bug-889630.t b/tests/bugs/bug-889630.t
new file mode 100755
index 000000000..b04eb3407
--- /dev/null
+++ b/tests/bugs/bug-889630.t
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../cluster.rc
+
+function check_peers {
+        $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+
+function volume_count {
+        local cli=$1;
+        if [ $cli -eq '1' ] ; then
+                $CLI_1 volume info | grep 'Volume Name' | wc -l;
+        else
+                $CLI_2 volume info | grep 'Volume Name' | wc -l;
+        fi
+}
+
+cleanup;
+
+TEST launch_cluster 2;
+TEST $CLI_1 peer probe $H2;
+
+EXPECT_WITHIN 20 1 check_peers
+
+TEST $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0
+TEST $CLI_1 volume start $V0
+
+b="B1";
+
+#Create an extra file in the originator's volume store
+touch ${!b}/glusterd/vols/$V0/run/file
+
+TEST $CLI_1 volume stop $V0
+#Test for self-commit failure
+TEST $CLI_1 volume delete $V0
+
+#Check whether delete succeeded on both the nodes
+EXPECT "0" volume_count '1'
+EXPECT "0" volume_count '2'
+
+#Check whether the volume name can be reused after deletion
+TEST $CLI_1 volume create $V0 $H1:$B1/${V0}1 $H2:$B2/${V0}1
+TEST $CLI_1 volume start $V0
+
+#Create an extra file in the peer's volume store
+touch ${!b}/glusterd/vols/$V0/run/file
+
+TEST $CLI_1 volume stop $V0
+#Test for commit failure on the other node
+TEST $CLI_2 volume delete $V0
+
+EXPECT "0" volume_count '1';
+EXPECT "0" volume_count '2';
+
+cleanup;
diff --git a/tests/bugs/bug-889996.t b/tests/bugs/bug-889996.t
new file mode 100644
index 000000000..6b07d8918
--- /dev/null
+++ b/tests/bugs/bug-889996.t
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+
+rm -rf $B0/${V0}1;
+
+TEST ! $CLI volume start $V0;
+EXPECT 0 online_brick_count;
+
+cleanup;
diff --git a/tests/bugs/bug-892730.t b/tests/bugs/bug-892730.t
new file mode 100755
index 000000000..0a677069e
--- /dev/null
+++ b/tests/bugs/bug-892730.t
@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# Bug 892730 - Verify that afr handles EIO errors from the brick properly.
+#
+# The associated bug describes a problem where EIO errors returned from the
+# local filesystem of a brick that is part of a replica volume are exposed to
+# the user. This test simulates such failures and verifies that the volume
+# operates as expected.
+#
+########
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST mkdir -p $B0/test{1,2}
+
+# The graph is a two brick replica with error-gen enabled on the second brick
+# and configured to return EIO lookup errors 100% of the time. This simulates
+# a brick with a crashed or shut down local filesystem. Note that the order in
+# which errors occur is a factor in reproducing the original bug (error-gen
+# must be enabled in the second brick for this test to be effective).
+
+cat > $B0/test.vol <<EOF
+volume test-posix-0
+    type storage/posix
+    option directory $B0/test1
+end-volume
+
+volume test-locks-0
+    type features/locks
+    subvolumes test-posix-0
+end-volume
+
+volume test-posix-1
+    type storage/posix
+    option directory $B0/test2
+end-volume
+
+volume test-error-1
+    type debug/error-gen
+    option failure 100
+    option enable lookup
+    option error-no EIO
+    subvolumes test-posix-1
+end-volume
+
+volume test-locks-1
+    type features/locks
+    subvolumes test-error-1
+end-volume
+
+volume test-replicate-0
+    type cluster/replicate
+    option background-self-heal-count 0
+    subvolumes test-locks-0 test-locks-1
+end-volume
+EOF
+
+TEST glusterd
+
+TEST glusterfs --volfile=$B0/test.vol --attribute-timeout=0 --entry-timeout=0 $M0
+
+# We should be able to create and remove a file without interference from the
+# "broken" brick.
+
+TEST touch $M0/file
+TEST rm $M0/file
+
+TEST umount $M0
+
+rm -f $B0/test.vol
+rm -rf $B0/test1 $B0/test2
+
+cleanup;
+
diff --git a/tests/bugs/bug-893338.t b/tests/bugs/bug-893338.t
new file mode 100644
index 000000000..cc39f28e3
--- /dev/null
+++ b/tests/bugs/bug-893338.t
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 stripe 2 $H0:$B0/${V0}{1,2,3,4};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+
+TEST glusterfs -s $H0 --volfile-id=$V0 $M0
+
+## Test for symlink success
+TEST touch $M0/reg_file
+TEST ln -s $M0/reg_file $M0/symlink
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-893378.t b/tests/bugs/bug-893378.t
new file mode 100755
index 000000000..fd8b9a7ce
--- /dev/null
+++ b/tests/bugs/bug-893378.t
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+cleanup;
+BRICK_COUNT=3
+
+function file_has_linkfile()
+{
+        i=0
+        j=0
+        while [ $i -lt $BRICK_COUNT ]
+        do
+                stat=`stat $B0/${V0}$i/$1 2>/dev/null`
+                if [ $? -eq 0 ]
+                then
+                        let j++
+                        let "BRICK${j}=$i"
+
+                fi
+                let i++
+        done
+        return $j
+}
+
+function get_cached_brick()
+{
+        i=1
+        while [ $i -lt 3 ]
+        do
+                test=`getfattr -n trusted.glusterfs.dht.linkto -e text $B0/${V0}$BRICK$i 2>&1`
+                if [ $? -eq 1 ]
+                then
+                        cached=$BRICK"$i"
+                        i=$(( $i+3 ))
+                fi
+                let i++
+        done
+
+        return $cached
+}
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2
+TEST $CLI volume start $V0
+
+## Mount FUSE
+TEST glusterfs --attribute-timeout=0 --entry-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+## create a linkfile on subvolume 0
+TEST touch $M0/1
+TEST mv $M0/1 $M0/2
+
+file_has_linkfile 2
+has_link=$?
+if [ $has_link -eq 2 ]
+then
+        get_cached_brick
+        CACHED=$?
+        # Kill a brick process
+        kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0-d-backends-${V0}$CACHED.pid`;
+fi
+
+## trigger a lookup
+ls -l $M0/2 2>/dev/null
+
+## fail dd if file exists.
+
+dd if=/dev/zero of=$M0/2 bs=1 count=1 conv=excl 2>/dev/null
+EXPECT "1" echo $?
+
+cleanup;
diff --git a/tests/bugs/bug-895235.t b/tests/bugs/bug-895235.t
new file mode 100644
index 000000000..0764b50d4
--- /dev/null
+++ b/tests/bugs/bug-895235.t
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume set $V0 ensure-durability off
+TEST $CLI volume set $V0 performance.write-behind off
+TEST $CLI volume set $V0 cluster.eager-lock off
+TEST $CLI volume start $V0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable
+
+TEST gluster volume profile $V0 start
+TEST dd of=$M0/a if=/dev/zero bs=1M count=1 oflag=append
+finodelk_max_latency=$($CLI volume profile $V0 info | grep FINODELK | awk 'BEGIN {max = 0} {if ($6 > max) max=$6;} END {print max}' | cut -d. -f 1 | egrep "[0-9]{7,}")
+
+TEST [ -z $finodelk_max_latency ]
+
+cleanup;
diff --git a/tests/bugs/bug-896431.t b/tests/bugs/bug-896431.t
new file mode 100755
index 000000000..f968e59c1
--- /dev/null
+++ b/tests/bugs/bug-896431.t
@@ -0,0 +1,124 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting cluster.subvols-per-directory as -5
+TEST ! $CLI volume set $V0 cluster.subvols-per-directory -5
+EXPECT '' volinfo_field $V0 'cluster.subvols-per-directory';
+TEST ! $CLI volume set $V0 subvols-per-directory -5
+EXPECT '' volinfo_field $V0 'cluster.subvols-per-directory';
+
+## Setting cluster.subvols-per-directory as 0
+TEST ! $CLI volume set $V0 cluster.subvols-per-directory 0
+EXPECT '' volinfo_field $V0 'cluster.subvols-per-directory';
+TEST ! $CLI volume set $V0 subvols-per-directory 0
+EXPECT '' volinfo_field $V0 'cluster.subvols-per-directory';
+
+## Setting cluster.subvols-per-directory as 4 (the total number of bricks)
+TEST ! $CLI volume set $V0 cluster.subvols-per-directory 4
+EXPECT '' volinfo_field $V0 'cluster.subvols-per-directory';
+TEST ! $CLI volume set $V0 subvols-per-directory 4
+EXPECT '' volinfo_field $V0 'cluster.subvols-per-directory';
+
+## Setting cluster.subvols-per-directory as 2 (the total number of subvolumes)
+TEST $CLI volume set $V0 cluster.subvols-per-directory 2
+EXPECT '2' volinfo_field $V0 'cluster.subvols-per-directory';
+
+## Setting cluster.subvols-per-directory as 1
+TEST $CLI volume set $V0 subvols-per-directory 1
+EXPECT '1' volinfo_field $V0 'cluster.subvols-per-directory';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
+
+## Start and create a pure replicate volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 replica 8 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'Replicate' volinfo_field $V0 'Type';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting cluster.subvols-per-directory as 8 for a replicate volume
+TEST ! $CLI volume set $V0 cluster.subvols-per-directory 8
+EXPECT '' volinfo_field $V0 'cluster.subvols-per-directory';
+TEST ! $CLI volume set $V0 subvols-per-directory 8
+EXPECT '' volinfo_field $V0 'cluster.subvols-per-directory';
+
+## Setting cluster.subvols-per-directory as 1 for a replicate volume
+TEST $CLI volume set $V0 cluster.subvols-per-directory 1
+EXPECT '1' volinfo_field $V0 'cluster.subvols-per-directory';
+TEST $CLI volume set $V0 subvols-per-directory 1
+EXPECT '1' volinfo_field $V0 'cluster.subvols-per-directory';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
+
+## Start and create a pure stripe volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 stripe 8 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+EXPECT 'Stripe' volinfo_field $V0 'Type';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting cluster.subvols-per-directory as 8 for a stripe volume
+TEST ! $CLI volume set $V0 cluster.subvols-per-directory 8
+EXPECT '' volinfo_field $V0 'cluster.subvols-per-directory';
+TEST ! $CLI volume set $V0 subvols-per-directory 8
+EXPECT '' volinfo_field $V0 'cluster.subvols-per-directory';
+
+## Setting cluster.subvols-per-directory as 1 for a stripe volume
+TEST $CLI volume set $V0 cluster.subvols-per-directory 1
+EXPECT '1' volinfo_field $V0 'cluster.subvols-per-directory';
+TEST $CLI volume set $V0 subvols-per-directory 1
+EXPECT '1' volinfo_field $V0 'cluster.subvols-per-directory';
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-902610.t b/tests/bugs/bug-902610.t
new file mode 100755
index 000000000..00ba03adf
--- /dev/null
+++ b/tests/bugs/bug-902610.t
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+## Layout-spread set to 3, but subvols up are 2. So layout should split 50-50
+function get_layout()
+{
+        layout1=`getfattr -n trusted.glusterfs.dht -e hex $1 2>&1|grep dht |cut -d = -f2`
+        layout2=`getfattr -n trusted.glusterfs.dht -e hex $2 2>&1|grep dht |cut -d = -f2`
+
+        if [ $layout1 == "0x0000000100000000000000007ffffffe" ]
+        then
+                if [ $layout2 == "0x00000001000000007fffffffffffffff" ]
+		then
+			return 0
+		else
+			return 1
+		fi
+        fi
+
+	if [ $layout2 == "0x0000000100000000000000007ffffffe" ]
+        then
+                if [ $layout1 == "0x00000001000000007fffffffffffffff" ]
+		then
+			return 0
+		else
+			return 1
+		fi
+        fi
+	return 1
+}
+
+BRICK_COUNT=4
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2 $H0:$B0/${V0}3
+## set subvols-per-dir option
+TEST $CLI volume set $V0 subvols-per-directory 3
+TEST $CLI volume start $V0
+
+## Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0 --entry-timeout=0 --attribute-timeout=0;
+
+TEST ls -l $M0
+
+## kill 2 bricks to bring down available subvol < spread count
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0-d-backends-${V0}2.pid`;
+kill -9 `cat /var/lib/glusterd/vols/$V0/run/$H0-d-backends-${V0}3.pid`;
+
+mkdir $M0/dir1 2>/dev/null
+
+get_layout $B0/${V0}0/dir1 $B0/${V0}1/dir1
+EXPECT "0" echo $?
+
+cleanup;
diff --git a/tests/bugs/bug-903336.t b/tests/bugs/bug-903336.t
new file mode 100644
index 000000000..c1f91312a
--- /dev/null
+++ b/tests/bugs/bug-903336.t
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/${V0}
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+TEST setfattr -n trusted.io-stats-dump -v /tmp $M0
+cleanup
diff --git a/tests/bugs/bug-904065.t b/tests/bugs/bug-904065.t
new file mode 100755
index 000000000..505854d9b
--- /dev/null
+++ b/tests/bugs/bug-904065.t
@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# This test does not use 'showmount' from the nfs-utils package, it would
+# require setting up a portmapper (either rpcbind or portmap, depending on the
+# Linux distribution used for testing). The persistancy of the rmtab should not
+# affect the current showmount outputs, so existing regression tests should be
+# sufficient.
+#
+
+# count the lines of a file, return 0 if the file does not exist
+function count_lines()
+{
+        if [ -e "$1" ]
+        then
+                wc -l < $1
+        else
+                echo 0
+        fi
+}
+
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../nfs.rc
+
+cleanup
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/brick1
+EXPECT 'Created' volinfo_field $V0 'Status'
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status'
+
+# glusterfs/nfs needs some time to start up in the background
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+# before mounting the rmtab should be empty
+EXPECT '0' count_lines /var/lib/glusterd/nfs/rmtab
+
+TEST mount -t nfs -o vers=3,nolock $H0:/$V0 $N0
+# the output would looks similar to:
+#
+#   hostname-0=172.31.122.104
+#   mountpoint-0=/ufo
+#
+EXPECT '2' count_lines /var/lib/glusterd/nfs/rmtab
+
+# duplicate mounts should not be recorded (client could have crashed)
+TEST mount -t nfs -o vers=3,nolock $H0:/$V0 $N1
+EXPECT '2' count_lines /var/lib/glusterd/nfs/rmtab
+
+# removing a mount should (even if there are two) should remove the entry
+TEST umount $N1
+EXPECT '0' count_lines /var/lib/glusterd/nfs/rmtab
+
+# unmounting the other mount should work flawlessly
+TEST umount $N0
+EXPECT '0' count_lines /var/lib/glusterd/nfs/rmtab
+
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 --volfile-server=$H0 --volfile-id=$V0 $M0
+
+# we'll create a fake rmtab here, similar to how an other storage server would do
+# using an invalid IP address to prevent (unlikely) collisions on the test-machine
+cat << EOF > $M0/rmtab
+hostname-0=127.0.0.256
+mountpoint-0=/ufo
+EOF
+EXPECT '2' count_lines $M0/rmtab
+
+# reconfigure merges the rmtab with the one on the volume
+TEST gluster volume set $V0 nfs.mount-rmtab $M0/rmtab
+
+# glusterfs/nfs needs some time to restart
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+# a new mount should be added to the rmtab, not overwrite exiting ones
+TEST mount -t nfs -o vers=3,nolock $H0:/$V0 $N0
+EXPECT '4' count_lines $M0/rmtab
+
+TEST umount $N0
+EXPECT '2' count_lines $M0/rmtab
+
+# TODO: nfs/reconfigure() is never called and is therefor disabled. When the
+# NFS-server supports reloading and does not get restarted anymore, we should
+# add a test that includes the merging of entries in the old rmtab with the new
+# rmtab.
+
+cleanup
diff --git a/tests/bugs/bug-904300.t b/tests/bugs/bug-904300.t
new file mode 100755
index 000000000..4276ee229
--- /dev/null
+++ b/tests/bugs/bug-904300.t
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../nfs.rc
+
+cleanup;
+
+# 1-8
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/$V0;
+TEST $CLI volume start $V0
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+TEST mount -t nfs -o vers=3,nolock,soft,intr $H0:/$V0 $N0
+TEST mkdir $N0/dir1
+TEST umount $N0
+
+#
+# Case 1: Allow "dir1" to be mounted only from 127.0.0.1
+# 9-12
+TEST $CLI volume set $V0 export-dir \""/dir1(127.0.0.1)"\"
+EXPECT_WITHIN 20 2 is_nfs_export_available
+
+TEST mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0/dir1 $N0
+TEST umount $N0
+
+#
+# Case 2: Allow "dir1" to be mounted only from 8.8.8.8. This is
+#         a negative test case therefore the mount should fail.
+# 13-16
+TEST $CLI volume set $V0 export-dir \""/dir1(8.8.8.8)"\"
+EXPECT_WITHIN 20 2 is_nfs_export_available
+
+TEST ! mount -t nfs -o vers=3,nolock,soft,intr $H0:/$V0/dir1 $N0
+TEST ! umount $N0
+
+
+# Case 3: Variation of test case1. Here we are checking with hostname
+#         instead of ip address.
+# 17-20
+TEST $CLI volume set $V0 export-dir \""/dir1($H0)"\"
+EXPECT_WITHIN 20 2 is_nfs_export_available
+
+TEST mount -t nfs -o vers=3,nolock,soft,intr $H0:/$V0/dir1 $N0
+TEST umount $N0
+
+# Case 4: Variation of test case1. Here we are checking with IP range
+# 21-24
+TEST $CLI volume set $V0 export-dir \""/dir1(127.0.0.0/24)"\"
+EXPECT_WITHIN 20 2 is_nfs_export_available
+
+TEST mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0/dir1 $N0
+TEST umount $N0
+
+## Finish up
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-905307.t b/tests/bugs/bug-905307.t
new file mode 100644
index 000000000..d81d81c9f
--- /dev/null
+++ b/tests/bugs/bug-905307.t
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+#test functionality of post-op-delay-secs
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+
+#Strings should not be accepted.
+TEST ! $CLI volume set $V0 cluster.post-op-delay-secs abc
+
+#-ve ints should not be accepted.
+TEST ! $CLI volume set $V0 cluster.post-op-delay-secs -1
+
+#INT_MAX+1 should not be accepted.
+TEST ! $CLI volume set $V0 cluster.post-op-delay-secs 2147483648
+
+#floats should not be accepted.
+TEST ! $CLI volume set $V0 cluster.post-op-delay-secs 1.25
+
+#min val 0 should be accepted
+TEST $CLI volume set $V0 cluster.post-op-delay-secs 0
+EXPECT "0" volume_option $V0 cluster.post-op-delay-secs
+
+#max val 2147483647 should be accepted
+TEST $CLI volume set $V0 cluster.post-op-delay-secs 2147483647
+EXPECT "2147483647" volume_option $V0 cluster.post-op-delay-secs
+
+#some middle val in range 2147 should be accepted
+TEST $CLI volume set $V0 cluster.post-op-delay-secs 2147
+EXPECT "2147" volume_option $V0 cluster.post-op-delay-secs
+cleanup;
diff --git a/tests/bugs/bug-905864.c b/tests/bugs/bug-905864.c
new file mode 100644
index 000000000..ed09b6e2b
--- /dev/null
+++ b/tests/bugs/bug-905864.c
@@ -0,0 +1,82 @@
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <pthread.h>
+
+
+pthread_t   th[5] = {0};
+void
+flock_init (struct flock *f, short int type, off_t start, off_t len)
+{
+    f->l_type = type;
+    f->l_start = start;
+    f->l_len = len;
+}
+
+int
+flock_range_in_steps (int fd, int is_set, short l_type,
+                      int start, int end, int step)
+{
+        int         ret = 0;
+        int         i = 0;
+        struct      flock f = {0,};
+
+        for (i = start; i+step < end; i += step) {
+                flock_init (&f, l_type, i, step);
+                ret = fcntl (fd, (is_set)? F_SETLKW:F_GETLK, &f);
+                if (ret) {
+                        perror ("fcntl");
+                        goto out;
+                }
+        }
+out:
+        return ret;
+}
+
+void *
+random_locker (void *arg)
+{
+        int         fd = *(int *)arg;
+        int         i = 0;
+        int         is_set = 0;
+
+        /* use thread id to choose GETLK or SETLK operation*/
+        is_set = pthread_self () % 2;
+        (void)flock_range_in_steps (fd, is_set, F_WRLCK, 0, 400, 1);
+
+        return NULL;
+}
+
+
+int main (int argc, char **argv)
+{
+        int         fd = -1;
+        int         ret = 1;
+        int         i = 0;
+        char        *fname = NULL;
+
+        if (argc < 2)
+                goto out;
+
+        fname = argv[1];
+        fd = open (fname, O_RDWR);
+        if (fd == -1) {
+                perror ("open");
+                goto out;
+        }
+
+        ret = flock_range_in_steps (fd, 1, F_WRLCK, 0, 2000, 2);
+        for (i = 0; i < 5; i++) {
+                pthread_create (&th[i], NULL, random_locker, (void *) &fd);
+        }
+        ret = flock_range_in_steps (fd, 1, F_WRLCK, 0, 2000, 2);
+        for (i = 0; i < 5; i++) {
+                pthread_join (th[i], NULL);
+        }
+out:
+        if (fd != -1)
+                close (fd);
+
+        return ret;
+}
diff --git a/tests/bugs/bug-905864.t b/tests/bugs/bug-905864.t
new file mode 100644
index 000000000..44bb469f2
--- /dev/null
+++ b/tests/bugs/bug-905864.t
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4};
+TEST $CLI volume start $V0;
+
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M1;
+
+TEST touch $M0/file1;
+
+#following C program tries open up race(s) if any, in F_GETLK/F_SETLKW codepaths
+#of locks xlator
+gcc -lpthread -g3 $(dirname $0)/bug-905864.c -o $(dirname $0)/bug-905864
+$(dirname $0)/bug-905864 $M0/file1 &
+$(dirname $0)/bug-905864 $M1/file1;
+wait
+rm -f $(dirname $0)/bug-905864
+
+EXPECT $(brick_count $V0) online_brick_count
+
+cleanup
+
diff --git a/tests/bugs/bug-906646.t b/tests/bugs/bug-906646.t
new file mode 100644
index 000000000..0e6a3bcb6
--- /dev/null
+++ b/tests/bugs/bug-906646.t
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+REPLICA=2
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 replica $REPLICA $H0:$B0/${V0}-00 $H0:$B0/${V0}-01 $H0:$B0/${V0}-10 $H0:$B0/${V0}-11
+TEST $CLI volume start $V0
+
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+function xattr_query_check()
+{
+    local path=$1
+    local xa_name=$2
+
+    local ret=`getfattr -m . -n $xa_name $path 2>&1 | grep -o "$xa_name: No such attribute" | wc -l`
+    echo $ret
+}
+
+function set_xattr()
+{
+    local path=$1
+    local xa_name=$2
+    local xa_val=$3
+
+    setfattr -n $xa_name -v $xa_val $path
+    echo $?
+}
+
+function remove_xattr()
+{
+    local path=$1
+    local xa_name=$2
+
+    setfattr -x $xa_name $path
+    echo $?
+}
+
+f=f00f
+pth=$M0/$f
+
+touch $pth
+
+# fetch backend paths
+backend_paths=`get_backend_paths $pth`
+
+# convert it into and array
+backend_paths_array=($backend_paths)
+
+# setxattr xattr for this file
+EXPECT 0 set_xattr $pth "trusted.name" "test"
+
+# confirm the set on backend
+EXPECT 0 xattr_query_check ${backend_paths_array[0]} "trusted.name"
+EXPECT 0 xattr_query_check ${backend_paths_array[1]} "trusted.name"
+
+brick_path=`echo ${backend_paths_array[0]} | sed -n 's/\(.*\)\/'$f'/\1/p'`
+brick_id=`$CLI volume info $V0 | grep "Brick[[:digit:]]" | grep -n $brick_path  | cut -f1 -d:`
+
+# Kill a brick process
+TEST kill_brick $V0 $H0 $brick_path
+
+# remove the xattr from the mount point
+EXPECT 0 remove_xattr $pth "trusted.name"
+
+# we killed ${backend_paths[0]} - so expect the xattr to be there
+# on the backend there
+EXPECT 0 xattr_query_check ${backend_paths_array[0]} "trusted.name"
+EXPECT 1 xattr_query_check ${backend_paths_array[1]} "trusted.name"
+
+# restart the brick process
+TEST $CLI volume start $V0 force
+
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 `expr $brick_id - 1`
+
+stat $pth
+
+# check backends - xattr should not be present anywhere
+EXPECT 1 xattr_query_check ${backend_paths_array[0]} "trusted.name"
+EXPECT 1 xattr_query_check ${backend_paths_array[1]} "trusted.name"
+
+cleanup;
diff --git a/tests/bugs/bug-907072.t b/tests/bugs/bug-907072.t
new file mode 100755
index 000000000..49b477767
--- /dev/null
+++ b/tests/bugs/bug-907072.t
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../fileio.rc
+. $(dirname $0)/../dht.rc
+
+cleanup;
+
+TEST glusterd;
+TEST pidof glusterd;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1,2,3};
+TEST $CLI volume start $V0;
+
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+
+TEST mkdir $M0/test;
+
+OLD_LAYOUT0=`get_layout $B0/${V0}0/test`;
+OLD_LAYOUT1=`get_layout $B0/${V0}1/test`;
+OLD_LAYOUT2=`get_layout $B0/${V0}2/test`;
+OLD_LAYOUT3=`get_layout $B0/${V0}3/test`;
+
+TEST killall glusterfsd;
+
+# Delete directory on one brick
+TEST rm -rf $B0/${V}1/test;
+
+# And only layout xattr on another brick
+TEST setfattr -x trusted.glusterfs.dht $B0/${V0}2/test;
+
+TEST $CLI volume start $V0 force;
+
+TEST umount $M0;
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+TEST stat $M0/test;
+
+NEW_LAYOUT0=`get_layout $B0/${V0}0/test`;
+NEW_LAYOUT1=`get_layout $B0/${V0}1/test`;
+NEW_LAYOUT2=`get_layout $B0/${V0}2/test`;
+NEW_LAYOUT3=`get_layout $B0/${V0}3/test`;
+
+EXPECT $OLD_LAYOUT0 echo $NEW_LAYOUT0;
+EXPECT $OLD_LAYOUT1 echo $NEW_LAYOUT1;
+EXPECT $OLD_LAYOUT2 echo $NEW_LAYOUT2;
+EXPECT $OLD_LAYOUT3 echo $NEW_LAYOUT3;
diff --git a/tests/bugs/bug-908146.t b/tests/bugs/bug-908146.t
new file mode 100755
index 000000000..87b456e6e
--- /dev/null
+++ b/tests/bugs/bug-908146.t
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+function get_fd_count {
+        local vol=$1
+        local host=$2
+        local brick=$3
+        local fname=$4
+        local gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $brick/$fname))
+        local statedump=$(generate_brick_statedump $vol $host $brick)
+        local count=$(grep "gfid=$gfid_str" $statedump -A2 | grep fd-count | cut -f2 -d'=' | tail -1)
+        rm -f $statedump
+        echo $count
+}
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/${V0}0
+TEST $CLI volume set $V0 performance.open-behind off
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 --direct-io-mode=enable
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M1 --attribute-timeout=0 --entry-timeout=0 --direct-io-mode=enable
+
+TEST touch $M0/a
+
+exec 4>"$M0/a"
+exec 5>"$M1/a"
+EXPECT "2" get_fd_count $V0 $H0 $B0/${V0}0 a
+
+exec 4>&-
+EXPECT "1" get_fd_count $V0 $H0 $B0/${V0}0 a
+
+exec 5>&-
+EXPECT "0" get_fd_count $V0 $H0 $B0/${V0}0 a
+
+cleanup
diff --git a/tests/bugs/bug-912297.t b/tests/bugs/bug-912297.t
new file mode 100755
index 000000000..f5a5babf5
--- /dev/null
+++ b/tests/bugs/bug-912297.t
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume create $V0 replica 2 stripe 2 $H0:$B0/${V0}{1,2,3,4,5,6,7,8};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Setting owner-uid as -12
+TEST ! $CLI volume set $V0 owner-uid -12
+EXPECT '' volinfo_field $V0 'storage.owner-uid'
+
+## Setting owner-gid as -5
+TEST ! $CLI volume set $V0 owner-gid -5
+EXPECT '' volinfo_field $V0 'storage.owner-gid'
+
+## Setting owner-uid as 36
+TEST $CLI volume set $V0 owner-uid 36
+EXPECT '36' volinfo_field $V0 'storage.owner-uid'
+
+## Setting owner-gid as 36
+TEST $CLI volume set $V0 owner-gid 36
+EXPECT '36' volinfo_field $V0 'storage.owner-gid'
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-912564.t b/tests/bugs/bug-912564.t
new file mode 100755
index 000000000..b24268fbc
--- /dev/null
+++ b/tests/bugs/bug-912564.t
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+# Test that the rsync and "extra" regexes cause rename-in-place without
+# creating linkfiles, when they're supposed to.  Without the regex we'd have a
+# 1/4 chance of each file being assigned to the right place, so with 16 files
+# we have a 1/2^32 chance of getting the correct result by accident.
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+function count_linkfiles {
+	local i
+	local count=0
+	for i in $(seq $2 $3); do
+		x=$(find $1$i -perm -1000 | wc -l)
+		# Divide by two because of the .glusterfs links.
+		count=$((count+x/2))
+	done
+	echo $count
+}
+
+# This function only exists to get around quoting difficulties in TEST.
+function set_regex {
+	$CLI volume set $1 cluster.extra-hash-regex '^foo(.+)bar$'
+}
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+mkdir -p $H0:$B0/${V0}0
+mkdir -p $H0:$B0/${V0}1
+mkdir -p $H0:$B0/${V0}2
+mkdir -p $H0:$B0/${V0}3
+
+# Create and start a volume.
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 \
+			    $H0:$B0/${V0}2 $H0:$B0/${V0}3
+TEST $CLI volume start $V0
+EXPECT_WITHIN 15 'Started' volinfo_field $V0 'Status';
+
+# Mount it.
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+
+# Make sure the rsync regex works, by verifying that no linkfiles are
+# created.
+rm -f $M0/file*
+for i in $(seq 0 15); do
+	fn=$(printf file%x $i)
+	tmp_fn=$(printf .%s.%d $fn $RANDOM)
+	echo testing > $M0/$tmp_fn
+	mv $M0/$tmp_fn $M0/$fn
+done
+lf=$(count_linkfiles $B0/$V0 0 3)
+TEST [ "$lf" -eq "0" ]
+
+# Make sure that linkfiles *are* created for normal files.
+rm -f $M0/file*
+for i in $(seq 0 15); do
+	fn=$(printf file%x $i)
+	tmp_fn=$(printf foo%sbar $fn)
+	echo testing > $M0/$tmp_fn
+	mv $M0/$tmp_fn $M0/$fn
+done
+lf=$(count_linkfiles $B0/$V0 0 3)
+TEST [ "$lf" -ne "0" ]
+
+# Make sure that setting an extra regex suppresses the linkfiles.
+TEST set_regex $V0
+rm -f $M0/file*
+for i in $(seq 0 15); do
+	fn=$(printf file%x $i)
+	tmp_fn=$(printf foo%sbar $fn)
+	echo testing > $M0/$tmp_fn
+	mv $M0/$tmp_fn $M0/$fn
+done
+lf=$(count_linkfiles $B0/$V0 0 3)
+TEST [ "$lf" -eq "0" ]
+
+# Re-test the rsync regex, to make sure the extra one didn't break it.
+rm -f $M0/file*
+for i in $(seq 0 15); do
+	fn=$(printf file%x $i)
+	tmp_fn=$(printf .%s.%d $fn $RANDOM)
+	echo testing > $M0/$tmp_fn
+	mv $M0/$tmp_fn $M0/$fn
+done
+lf=$(count_linkfiles $B0/$V0 0 3)
+TEST [ "$lf" -eq "0" ]
+
+cleanup
diff --git a/tests/bugs/bug-913051.t b/tests/bugs/bug-913051.t
new file mode 100644
index 000000000..69e90cf66
--- /dev/null
+++ b/tests/bugs/bug-913051.t
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../fileio.rc
+
+cleanup;
+
+#Test that afr opens the file on the bricks that were offline at the time of
+# open after the brick comes online. This tests for writev, readv triggering
+# open-fd-fix in afr.
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST $CLI volume set $V0 performance.quick-read off
+TEST $CLI volume set $V0 performance.open-behind off
+TEST $CLI volume set $V0 performance.io-cache off
+TEST $CLI volume set $V0 performance.write-behind off
+TEST $CLI volume set $V0 performance.stat-prefetch off
+TEST $CLI volume set $V0 performance.read-ahead off
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+TEST $CLI volume start $V0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable
+TEST kill_brick $V0 $H0 $B0/${V0}0
+
+TEST mkdir $M0/dir
+TEST touch $M0/dir/a
+TEST touch $M0/dir/b
+echo abc > $M0/dir/b
+
+TEST wfd=`fd_available`
+TEST fd_open $wfd "w" $M0/dir/a
+TEST rfd=`fd_available`
+TEST fd_open $rfd "r" $M0/dir/b
+
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+
+#check that the files are not opned on brick-0
+realpatha=$(gf_get_gfid_backend_file_path $B0/${V0}0 "dir/a")
+EXPECT "N" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 "$realpatha"
+EXPECT "N" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 $B0/${V0}0/dir/a
+
+realpathb=$(gf_get_gfid_backend_file_path $B0/${V0}0 "dir/b")
+EXPECT "N" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 "$realpathb"
+EXPECT "N" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 $B0/${V0}0/dir/b
+
+#attempt self-heal so that the files are created on brick-0
+
+TEST ls -l $M0/dir/a
+TEST ls -l $M0/dir/b
+
+#trigger writev for attempting open-fd-fix in afr
+TEST fd_write $wfd "open sesame"
+
+#trigger readv for attempting open-fd-fix in afr
+TEST fd_cat $rfd
+
+EXPECT_WITHIN 20 "Y" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 $B0/${V0}0/dir/a
+EXPECT_WITHIN 20 "Y" gf_check_file_opened_in_brick $V0 $H0 $B0/${V0}0 $B0/${V0}0/dir/b
+
+TEST fd_close $wfd
+TEST fd_close $rfd
+cleanup;
diff --git a/tests/bugs/bug-913487.t b/tests/bugs/bug-913487.t
new file mode 100644
index 000000000..2095903d9
--- /dev/null
+++ b/tests/bugs/bug-913487.t
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd;
+TEST pidof glusterd;
+
+TEST ! $CLI volume set $V0 performance.open-behind off;
+
+TEST pidof glusterd;
+
+cleanup;
diff --git a/tests/bugs/bug-913544.t b/tests/bugs/bug-913544.t
new file mode 100644
index 000000000..790bc0898
--- /dev/null
+++ b/tests/bugs/bug-913544.t
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+#simulate a split-brain of a file and do truncate. This should not crash the mount point
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+TEST $CLI volume set $V0 stat-prefetch off
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+cd $M0
+TEST touch a
+#simulate no-changelog data split-brain
+echo "abc" > $B0/${V0}1/a
+echo "abcd" > $B0/${V0}0/a
+TEST ! truncate -s 0 a
+TEST ls
+cd
+
+cleanup
diff --git a/tests/bugs/bug-913555.t b/tests/bugs/bug-913555.t
new file mode 100755
index 000000000..f58d7bd6d
--- /dev/null
+++ b/tests/bugs/bug-913555.t
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Test that a volume becomes unwritable when the cluster loses quorum.
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../cluster.rc
+
+
+function check_fs {
+	df $1 &> /dev/null
+	echo $?
+}
+
+function check_peers {
+	$CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+
+function glusterfsd_count {
+    pidof glusterfsd | wc -w;
+}
+
+cleanup;
+
+TEST launch_cluster 3; # start 3-node virtual cluster
+TEST $CLI_1 peer probe $H2; # peer probe server 2 from server 1 cli
+TEST $CLI_1 peer probe $H3; # peer probe server 3 from server 1 cli
+
+EXPECT_WITHIN 20 2 check_peers
+
+TEST $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0 $H3:$B3/$V0
+TEST $CLI_1 volume set $V0 cluster.server-quorum-type server
+TEST $CLI_1 volume start $V0
+TEST glusterfs --volfile-server=$H1 --volfile-id=$V0 $M0
+
+# Kill one pseudo-node, make sure the others survive and volume stays up.
+TEST kill_node 3;
+EXPECT_WITHIN 20 1 check_peers;
+EXPECT 0 check_fs $M0;
+EXPECT 2 glusterfsd_count;
+
+# Kill another pseudo-node, make sure the last one dies and volume goes down.
+TEST kill_node 2;
+EXPECT_WITHIN 20 0 check_peers
+EXPECT 1 check_fs $M0;
+EXPECT 0 glusterfsd_count; # the two glusterfsds of the other two glusterds
+                           # must be dead
+
+TEST $glusterd_2;
+TEST $glusterd_3;
+EXPECT_WITHIN 20 3 glusterfsd_count; # restore quorum, all ok
+EXPECT_WITHIN 5 0 check_fs $M0;
+
+cleanup
diff --git a/tests/bugs/bug-915280.t b/tests/bugs/bug-915280.t
new file mode 100755
index 000000000..a1f92f201
--- /dev/null
+++ b/tests/bugs/bug-915280.t
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+TEST $CLI volume create $V0 $H0:$B0/brick1 $H0:$B0/brick2;
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+sleep 3
+
+MOUNTDIR=$N0;
+TEST mount -t nfs -o vers=3,nolock,soft,timeo=30,retrans=1 $H0:/$V0 $N0
+TEST touch $N0/testfile
+
+TEST $CLI volume set $V0 debug.error-gen client
+TEST $CLI volume set $V0 debug.error-fops stat
+TEST $CLI volume set $V0 debug.error-failure 100
+
+sleep 1
+
+pid_file=$(read_nfs_pidfile);
+
+getfacl $N0/testfile 2>/dev/null
+
+nfs_pid=$(get_nfs_pid);
+if [ ! $nfs_pid ]
+then
+  nfs_pid=0;
+fi
+
+TEST [ $nfs_pid -eq $pid_file ]
+
+TEST   umount $MOUNTDIR -l
+
+cleanup;
diff --git a/tests/bugs/bug-915554.t b/tests/bugs/bug-915554.t
new file mode 100755
index 000000000..beb669f8c
--- /dev/null
+++ b/tests/bugs/bug-915554.t
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Bug <915554>
+#
+# This test checks for a condition where a rebalance migrates a file and does
+# not preserve the original file size. This can occur due to hole preservation
+# logic in the file migration code. If a file size is aligned to a disk sector
+# boundary (512b) and the tail portion of the file is zero-filled, the file
+# may end up truncated to the end of the last data region in the file.
+#
+###
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../dht.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+BRICK_COUNT=3
+# create, start and mount a two brick DHT volume
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2
+TEST $CLI volume start $V0
+
+TEST glusterfs --attribute-timeout=0 --entry-timeout=0 --gid-timeout=-1 -s $H0 --volfile-id $V0 $M0;
+
+i=1
+# Write some data to a file and extend such that the file is sparse to a sector
+# aligned boundary.
+echo test > $M0/$i
+TEST truncate --size=1m $M0/$i
+
+# cache the original size
+SIZE1=`stat -c %s $M0/$i`
+
+# rename till file gets a linkfile
+
+while [ $i -ne 0 ]
+do
+        test=`mv $M0/$i $M0/$(( $i+1 )) 2>/dev/null`
+        if [ $? -ne 0 ]
+        then
+                echo "rename failed"
+                break
+        fi
+        let i++
+        file_has_linkfile $i
+        has_link=$?
+        if [ $has_link -eq 2 ]
+        then
+                break;
+        fi
+done
+
+# start a rebalance (force option to overide checks) to trigger migration of
+# file
+
+TEST $CLI volume rebalance $V0 start force
+
+# check if rebalance has completed for upto 15 secs
+
+EXPECT_WITHIN 30 "0" rebalance_completed
+
+# validate the file size after the migration
+SIZE2=`stat -c %s $M0/$i`
+
+TEST [ $SIZE1 -eq $SIZE2 ]
+
+TEST rm -f $M0/$i
+TEST umount $M0
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup;
diff --git a/tests/bugs/bug-916226.t b/tests/bugs/bug-916226.t
new file mode 100644
index 000000000..2abfa1fc6
--- /dev/null
+++ b/tests/bugs/bug-916226.t
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2 $H0:$B0/${V0}3
+TEST $CLI volume set $V0 cluster.eager-lock on
+TEST $CLI volume start $V0
+
+## Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+
+TEST mkdir $M0/dir{1..10};
+TEST touch $M0/dir{1..10}/files{1..10};
+
+TEST $CLI volume add-brick $V0 $H0:$B0/${V0}4 $H0:/$B0/${V0}5
+
+TEST $CLI volume rebalance $V0 start force
+EXPECT_WITHIN 60 "completed" rebalance_status_field $V0
+
+cleanup;
diff --git a/tests/bugs/bug-916549.t b/tests/bugs/bug-916549.t
new file mode 100755
index 000000000..d6a45b827
--- /dev/null
+++ b/tests/bugs/bug-916549.t
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd;
+TEST $CLI volume create $V0 $H0:$B0/${V0}1;
+TEST $CLI volume start $V0;
+
+pid_file=$(ls /var/lib/glusterd/vols/$V0/run);
+brick_pid=$(cat /var/lib/glusterd/vols/$V0/run/$pid_file);
+
+
+kill -SIGKILL $brick_pid;
+TEST $CLI volume start $V0 force;
+TEST process_leak_count $(pidof glusterd);
+
+cleanup;
diff --git a/tests/bugs/bug-918437-sh-mtime.t b/tests/bugs/bug-918437-sh-mtime.t
new file mode 100644
index 000000000..080956f51
--- /dev/null
+++ b/tests/bugs/bug-918437-sh-mtime.t
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+function get_mtime {
+        local f=$1
+        stat $f | grep Modify | awk '{print $2 $3}' | cut -f1 -d'.'
+}
+cleanup;
+
+## Tests if mtime is correct after self-heal.
+TEST glusterd
+TEST pidof glusterd
+TEST mkdir -p $B0/gfs0/brick0{1,2}
+TEST $CLI volume create $V0 replica 2 transport tcp $H0:$B0/gfs0/brick01 $H0:$B0/gfs0/brick02
+TEST $CLI volume set $V0 nfs.disable on
+TEST $CLI volume set $V0 performance.stat-prefetch off
+TEST $CLI volume set $V0 cluster.background-self-heal-count 0
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --direct-io-mode=enable
+# file 'a' is healed from brick02 to brick01 where as file 'b' is healed from
+# brick01 to brick02
+
+TEST cp -p /etc/passwd $M0/a
+TEST cp -p /etc/passwd $M0/b
+
+#Store mtimes before self-heals
+TEST modify_atstamp=$(get_mtime $B0/gfs0/brick02/a)
+TEST modify_btstamp=$(get_mtime $B0/gfs0/brick02/b)
+
+TEST $CLI volume stop $V0
+TEST gf_rm_file_and_gfid_link $B0/gfs0/brick01 a
+TEST gf_rm_file_and_gfid_link $B0/gfs0/brick02 b
+
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1
+
+find $M0 | xargs stat 1>/dev/null
+
+TEST modify_atstamp1=$(get_mtime $B0/gfs0/brick01/a)
+TEST modify_atstamp2=$(get_mtime $B0/gfs0/brick02/a)
+EXPECT $modify_atstamp echo $modify_atstamp1
+EXPECT $modify_atstamp echo $modify_atstamp2
+
+TEST modify_btstamp1=$(get_mtime $B0/gfs0/brick01/b)
+TEST modify_btstamp2=$(get_mtime $B0/gfs0/brick02/b)
+EXPECT $modify_btstamp echo $modify_btstamp1
+EXPECT $modify_btstamp echo $modify_btstamp2
+cleanup;
diff --git a/tests/bugs/bug-921072.t b/tests/bugs/bug-921072.t
new file mode 100755
index 000000000..e101d5b46
--- /dev/null
+++ b/tests/bugs/bug-921072.t
@@ -0,0 +1,118 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../nfs.rc
+
+cleanup;
+
+#1
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/$V0
+TEST $CLI volume start $V0
+EXPECT_WITHIN 20 1 is_nfs_export_available
+TEST mount -t nfs -o vers=3,nolock,soft,intr $H0:/$V0 $N0
+TEST umount $N0
+
+# based on ip addresses (1-4)
+# case 1: allow only localhost ip
+TEST $CLI volume set $V0 nfs.rpc-auth-allow 127.0.0.1
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+TEST mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+TEST umount $N0
+
+# case 2: allow only non-localhost ip
+TEST $CLI volume set $V0 nfs.rpc-auth-allow 192.168.1.1
+EXPECT_WITHIN 20 1 is_nfs_export_available
+#11
+TEST ! mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+TEST $CLI volume reset --mode=script $V0
+# case 3: reject only localhost ip
+TEST $CLI volume set $V0 nfs.rpc-auth-reject 127.0.0.1
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+TEST ! mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+
+# case 4: reject only non-localhost ip
+TEST $CLI volume set $V0 nfs.rpc-auth-reject 192.168.1.1
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+TEST mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+TEST umount $N0
+
+
+
+# NEED TO CHECK BOTH IP AND NAME BASED AUTH.
+# CASES WITH NFS.ADDR-NAMELOOKUP ON (5-12)
+TEST $CLI volume reset --mode=script $V0
+TEST $CLI volume set $V0 nfs.addr-namelookup on
+EXPECT_WITHIN 20 1 is_nfs_export_available
+#20
+TEST mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+TEST umount $N0
+
+# case 5: allow only localhost
+TEST $CLI volume set $V0 nfs.rpc-auth-allow localhost
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+TEST mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+TEST umount $N0
+
+# case 6: allow only somehost
+TEST $CLI volume set $V0 nfs.rpc-auth-allow somehost
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+TEST ! mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+
+# case 7: reject only localhost
+TEST $CLI volume reset --mode=script $V0
+TEST $CLI volume set $V0 nfs.addr-namelookup on
+TEST $CLI volume set $V0 nfs.rpc-auth-reject localhost
+EXPECT_WITHIN 20 1 is_nfs_export_available
+#30
+TEST ! mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+
+# case 8: reject only somehost
+TEST $CLI volume set $V0 nfs.rpc-auth-reject somehost
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+TEST mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+TEST umount $N0
+
+# based on ip addresses: repeat of cases 1-4
+# case 9: allow only localhost ip
+TEST $CLI volume reset --mode=script $V0
+TEST $CLI volume set $V0 nfs.addr-namelookup on
+TEST $CLI volume set $V0 nfs.rpc-auth-allow 127.0.0.1
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+TEST mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+TEST umount $N0
+
+# case 10: allow a non-localhost ip
+TEST $CLI volume set $V0 nfs.rpc-auth-allow 192.168.1.1
+EXPECT_WITHIN 20 1 is_nfs_export_available
+#40
+TEST ! mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+
+# case 11: reject only localhost ip
+TEST $CLI volume reset --mode=script $V0
+TEST $CLI volume set $V0 nfs.addr-namelookup on
+TEST $CLI volume set $V0 nfs.rpc-auth-reject 127.0.0.1
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+TEST ! mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+
+# case 12: reject only non-localhost ip
+TEST $CLI volume set $V0 nfs.rpc-auth-reject 192.168.1.1
+EXPECT_WITHIN 20 1 is_nfs_export_available
+
+TEST mount -t nfs -o vers=3,nolock,soft,intr localhost:/$V0 $N0
+TEST umount $N0
+
+TEST $CLI volume stop --mode=script $V0
+#49
+TEST $CLI volume delete --mode=script $V0
+cleanup
diff --git a/tests/bugs/bug-921231.t b/tests/bugs/bug-921231.t
new file mode 100644
index 000000000..db9cf3b6f
--- /dev/null
+++ b/tests/bugs/bug-921231.t
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+# This test writes to same file with 2 fds and tests that eager-lock is not
+# causing extra delay because of post-op-delay-secs
+cleanup;
+
+function write_to_file {
+        dd of=$M0/1 if=/dev/zero bs=1M count=128 oflag=append 2>&1 >/dev/null
+}
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+TEST $CLI volume set $V0 eager-lock on
+TEST $CLI volume set $V0 post-op-delay-secs 3
+TEST $CLI volume set $V0 client-log-level DEBUG
+TEST $CLI volume start $V0
+TEST $CLI volume profile $V0 start
+TEST $CLI volume set $V0 ensure-durability off
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+write_to_file &
+write_to_file &
+wait
+#Test if the MAX [F]INODELK fop latency is of the order of seconds.
+inodelk_max_latency=$($CLI volume profile $V0 info | grep INODELK | awk 'BEGIN {max = 0} {if ($6 > max) max=$6;} END {print max}' | cut -d. -f 1 | egrep "[0-9]{7,}")
+TEST [ -z $inodelk_max_latency ]
+
+cleanup;
diff --git a/tests/bugs/bug-921408.t b/tests/bugs/bug-921408.t
new file mode 100755
index 000000000..ef2b4fb21
--- /dev/null
+++ b/tests/bugs/bug-921408.t
@@ -0,0 +1,89 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../fileio.rc
+. $(dirname $0)/../dht.rc
+
+cleanup;
+wait_check_status ()
+{
+        n=0
+        while [ $n -lt $1 ]
+        do
+                ret=$(rebalance_completed)
+                if [ $ret == "0" ]
+                then
+                        return 0;
+                else
+                        sleep 1
+                        n=`expr $n + 1`;
+                fi
+       done
+       return 1;
+}
+
+addbr_rebal_till_layout_change()
+{
+        val=1
+        l=$1
+        i=1
+        while [ $i -lt 5 ]
+        do
+                $CLI volume add-brick $V0 $H0:$B0/${V0}$l &>/dev/null
+                $CLI volume rebalance $V0 fix-layout start &>/dev/null
+                wait_check_status 15
+                if [ $? -eq 1 ]
+                then
+                        break
+                fi
+                NEW_LAYOUT=`get_layout $B0/${V0}0`
+                if [ $OLD_LAYOUT == $NEW_LAYOUT ]
+                then
+                        i=`expr $i + 1`;
+                        l=`expr $l + 1`;
+                else
+                        val=0
+                        break
+                fi
+        done
+        return $val
+}
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0
+TEST $CLI volume set $V0 subvols-per-directory 1
+TEST $CLI volume start $V0
+
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+
+TEST mkdir $M0/test
+TEST touch $M0/test/test
+
+fd=`fd_available`
+TEST fd_open $fd "rw" $M0/test/test
+
+OLD_LAYOUT=`get_layout $B0/${V0}0`
+
+addbr_rebal_till_layout_change 1
+
+TEST [ $? -eq 0 ]
+
+for i in $(seq 1 1000)
+do
+	ls -l $M0/ >/dev/null
+	ret=$?
+	if [ $ret != 0 ]
+	then
+		break
+	fi
+done
+
+TEST [ $ret == 0 ];
+TEST fd_close $fd;
+
+TEST umount $M0
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup
diff --git a/tests/bugs/bug-924075.t b/tests/bugs/bug-924075.t
new file mode 100755
index 000000000..f4e03e33a
--- /dev/null
+++ b/tests/bugs/bug-924075.t
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+#FIXME: there is another patch which moves the following function into
+#include.rc
+function process_leak_count ()
+{
+    local pid=$1;
+    return $(ls -lh /proc/$pid/fd | grep "(deleted)" | wc -l)
+}
+
+TEST glusterd;
+TEST $CLI volume create $V0 $H0:$B0/${V0}1;
+TEST $CLI volume start $V0;
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+mount_pid=$(get_mount_process_pid $V0);
+TEST process_leak_count $mount_pid;
+
+cleanup;
diff --git a/tests/bugs/bug-924265.t b/tests/bugs/bug-924265.t
new file mode 100755
index 000000000..13491356d
--- /dev/null
+++ b/tests/bugs/bug-924265.t
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Test that setting cluster.dht-xattr-name works, and that DHT consistently
+# uses the specified name instead of the default.
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+# We only care about the exit code, so keep it quiet.
+function silent_getfattr {
+	getfattr $* &> /dev/null
+}
+
+cleanup
+
+TEST glusterd
+TEST pidof glusterd
+
+mkdir -p $H0:$B0/${V0}0
+
+# Create a volume and set the option.
+TEST $CLI volume create $V0 $H0:$B0/${V0}0
+TEST $CLI volume set $V0 cluster.dht-xattr-name trusted.foo.bar
+
+# Start and mount the volume.
+TEST $CLI volume start $V0
+EXPECT_WITHIN 15 'Started' volinfo_field $V0 'Status';
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+
+# Create a directory and make sure it has the right xattr.
+mkdir $M0/test
+TEST ! silent_getfattr -n trusted.glusterfs.dht $B0/${V0}0/test
+TEST silent_getfattr -n trusted.foo.bar $B0/${V0}0/test
+
+cleanup
diff --git a/tests/bugs/bug-927616.t b/tests/bugs/bug-927616.t
new file mode 100755
index 000000000..22b20aff2
--- /dev/null
+++ b/tests/bugs/bug-927616.t
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2};
+TEST $CLI volume set $V0 performance.open-behind off;
+TEST $CLI volume start $V0
+
+sleep 1;
+## Mount FUSE with caching disabled
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
+
+sleep 1;
+
+TEST mount -t nfs -o vers=3,nolock $H0:/$V0 $N0;
+
+TEST mkdir $M0/dir;
+
+mkdir $M0/other;
+cp /etc/passwd $M0/;
+cp $M0/passwd $M0/file;
+chmod 600 $M0/file;
+
+chown -R nfsnobody:nfsnobody $M0/dir;
+
+TEST $CLI volume set $V0 server.root-squash on;
+
+sleep 1;
+
+# tests should fail.
+touch $M0/foo 2>/dev/null;
+TEST [ $? -ne 0 ]
+touch $N0/foo 2>/dev/null;
+TEST [ $? -ne 0 ]
+mkdir $M0/new 2>/dev/null;
+TEST [ $? -ne 0 ]
+mkdir $N0/new 2>/dev/null;
+TEST [ $? -ne 0 ]
+
+TEST $CLI volume set $V0 server.root-squash off;
+
+sleep 1;
+
+# tests should pass.
+touch $M0/foo 2>/dev/null;
+TEST [ $? -eq 0 ]
+touch $N0/bar 2>/dev/null;
+TEST [ $? -eq 0 ]
+mkdir $M0/new 2>/dev/null;
+TEST [ $? -eq 0 ]
+mkdir $N0/old 2>/dev/null;
+TEST [ $? -eq 0 ]
+
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-948686.t b/tests/bugs/bug-948686.t
new file mode 100644
index 000000000..db9c198a9
--- /dev/null
+++ b/tests/bugs/bug-948686.t
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../cluster.rc
+
+function check_peers {
+    $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+cleanup;
+#setup cluster and test volume
+TEST launch_cluster 3; # start 3-node virtual cluster
+TEST $CLI_1 peer probe $H2; # peer probe server 2 from server 1 cli
+TEST $CLI_1 peer probe $H3; # peer probe server 3 from server 1 cli
+
+EXPECT_WITHIN 20 2 check_peers;
+
+TEST $CLI_1 volume create $V0 replica 2 $H1:$B1/$V0 $H1:$B1/${V0}_1 $H2:$B2/$V0 $H3:$B3/$V0
+TEST $CLI_1 volume start $V0
+TEST glusterfs --volfile-server=$H1 --volfile-id=$V0 $M0
+
+#kill a node
+TEST kill_node 3
+
+#modify volume config to see change in volume-sync
+TEST $CLI_1 volume set $V0 write-behind off
+#add some files to the volume to see effect of volume-heal cmd
+TEST touch $M0/{1..100};
+TEST $CLI_1 volume stop $V0;
+TEST $glusterd_3;
+sleep 3;
+TEST $CLI_3 volume start $V0;
+TEST $CLI_2 volume stop $V0;
+TEST $CLI_2 volume delete $V0;
+
+cleanup;
+
+TEST glusterd;
+TEST $CLI volume create $V0 $H0:$B0/$V0
+TEST $CLI volume start $V0
+pkill glusterd;
+pkill glusterfsd;
+TEST glusterd
+TEST $CLI volume status $V0
+
+cleanup;
diff --git a/tests/bugs/bug-948729/bug-948729-force.t b/tests/bugs/bug-948729/bug-948729-force.t
new file mode 100644
index 000000000..d14e94061
--- /dev/null
+++ b/tests/bugs/bug-948729/bug-948729-force.t
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../cluster.rc
+
+function check_peers {
+        $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+
+cleanup;
+uuid1=`uuidgen`;
+uuid2=`uuidgen`;
+uuid3=`uuidgen`;
+
+V1=patchy1
+V2=patchy2
+
+TEST launch_cluster 2;
+
+TEST $CLI_1 peer probe $H2;
+
+EXPECT_WITHIN 20 1 check_peers;
+
+B3=/d/backends/3
+B4=/d/backends/4
+B5=/d/backends/5
+B6=/d/backends/6
+
+mkdir -p $B3 $B4 $B5 $B6
+
+TEST truncate -s 16M $B1/brick1
+TEST truncate -s 16M $B2/brick2
+TEST truncate -s 16M $B3/brick3
+TEST truncate -s 16M $B4/brick4
+TEST truncate -s 16M $B5/brick5
+TEST truncate -s 16M $B6/brick6
+
+TEST LD1=`losetup --find --show $B1/brick1`
+TEST mkfs.xfs $LD1
+TEST LD2=`losetup --find --show $B2/brick2`
+TEST mkfs.xfs $LD2
+TEST LD3=`losetup --find --show $B3/brick3`
+TEST mkfs.xfs $LD3
+TEST LD4=`losetup --find --show $B4/brick4`
+TEST mkfs.xfs $LD4
+TEST LD5=`losetup --find --show $B5/brick5`
+TEST mkfs.xfs $LD5
+TEST LD6=`losetup --find --show $B6/brick6`
+TEST mkfs.xfs $LD6
+
+mkdir -p $B1/$V0 $B2/$V0 $B3/$V0 $B4/$V0 $B5/$V0 $B6/$V0
+
+TEST mount -t xfs $LD1 $B1/$V0
+TEST mount -t xfs $LD2 $B2/$V0
+TEST mount -t xfs $LD3 $B3/$V0
+TEST mount -t xfs $LD4 $B4/$V0
+TEST mount -t xfs $LD5 $B5/$V0
+TEST mount -t xfs $LD6 $B6/$V0
+
+#Case 0: Parent directory of the brick is absent
+TEST ! $CLI1 volume create $V0 $H1:$B1/$V0/nonexistent/b1 $H2:$B2/$V0/nonexistent/b2 force
+
+#Case 1: File system root is being used as brick directory
+TEST   $CLI1 volume create $V0 $H1:$B5/$V0 $H2:$B6/$V0 force
+
+#Case 2: Brick directory contains only one component
+TEST   $CLI1 volume create $V1 $H1:/$uuid1 $H2:/$uuid2 force
+
+#Case 3: Sub-directories of the backend FS being used as brick directory
+TEST   $CLI1 volume create $V2 $H1:$B1/$V0/brick1 $H2:$B2/$V0/brick2 force
+
+#add-brick tests
+TEST ! $CLI1 volume add-brick $V0 $H1:$B3/$V0/nonexistent/brick3 force
+TEST   $CLI1 volume add-brick $V0 $H1:$B3/$V0 force
+TEST   $CLI1 volume add-brick $V1 $H1:/$uuid3 force
+TEST   $CLI1 volume add-brick $V2 $H1:$B4/$V0/brick3 force
+
+#####replace-brick tests
+#FIX-ME: replace-brick does not work with the newly introduced cluster test
+#####framework
+
+rmdir /$uuid1 /$uuid2 /$uuid3;
+
+cleanup;
diff --git a/tests/bugs/bug-948729/bug-948729-mode-script.t b/tests/bugs/bug-948729/bug-948729-mode-script.t
new file mode 100644
index 000000000..541ca897d
--- /dev/null
+++ b/tests/bugs/bug-948729/bug-948729-mode-script.t
@@ -0,0 +1,85 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../cluster.rc
+
+function check_peers {
+        $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+
+cleanup;
+
+uuid1=`uuidgen`;
+uuid2=`uuidgen`;
+uuid3=`uuidgen`;
+
+V1=patchy1
+V2=patchy2
+V3=patchy3
+
+TEST launch_cluster 2;
+
+TEST $CLI_1 peer probe $H2;
+
+EXPECT_WITHIN 20 1 check_peers;
+
+B3=/d/backends/3
+B4=/d/backends/4
+B5=/d/backends/5
+B6=/d/backends/6
+mkdir -p $B3 $B4 $B5 $B6
+
+TEST truncate -s 16M $B1/brick1
+TEST truncate -s 16M $B2/brick2
+TEST truncate -s 16M $B3/brick3
+TEST truncate -s 16M $B4/brick4
+TEST truncate -s 16M $B5/brick5
+TEST truncate -s 16M $B6/brick6
+
+TEST LD1=`losetup --find --show $B1/brick1`
+TEST mkfs.xfs $LD1
+TEST LD2=`losetup --find --show $B2/brick2`
+TEST mkfs.xfs $LD2
+TEST LD3=`losetup --find --show $B3/brick3`
+TEST mkfs.xfs $LD3
+TEST LD4=`losetup --find --show $B4/brick4`
+TEST mkfs.xfs $LD4
+TEST LD5=`losetup --find --show $B5/brick5`
+TEST mkfs.xfs $LD5
+TEST LD6=`losetup --find --show $B6/brick6`
+TEST mkfs.xfs $LD6
+
+mkdir -p $B1/$V0 $B2/$V0 $B3/$V0 $B4/$V0 $B5/$V0 $B6/$V0
+
+TEST mount -t xfs $LD1 $B1/$V0
+TEST mount -t xfs $LD2 $B2/$V0
+TEST mount -t xfs $LD3 $B3/$V0
+TEST mount -t xfs $LD4 $B4/$V0
+TEST mount -t xfs $LD5 $B5/$V0
+TEST mount -t xfs $LD6 $B6/$V0
+
+#Case 0: Parent directory of the brick is absent
+TEST ! $CLI_1 volume create $V0 $H1:$B1/$V0/nonexistent/b1 $H2:$B2/$V0/nonexistent/b2
+
+#Case 1: File system root being used as brick directory
+TEST   $CLI_1 volume create $V0 $H1:$B5/$V0 $H2:$B6/$V0
+
+#Case 2: Brick directory contains only one component
+TEST   $CLI_1 volume create $V1 $H1:/$uuid1 $H2:/$uuid2
+
+#Case 3: Sub-directories of the backend FS being used as brick directory
+TEST   $CLI_1 volume create $V2 $H1:$B1/$V0/brick1 $H2:$B2/$V0/brick2
+
+#add-brick tests
+TEST ! $CLI_1 volume add-brick $V0 $H1:$B3/$V0/nonexistent/brick3
+TEST   $CLI_1 volume add-brick $V0 $H1:$B3/$V0
+TEST   $CLI_1 volume add-brick $V1 $H1:/$uuid3
+TEST   $CLI_1 volume add-brick $V2 $H1:$B4/$V0/brick3
+
+#####replace-brick tests
+#FIX-ME : replace-brick does not currently work in the newly introduced
+#####cluster test framework
+
+rmdir /$uuid1 /$uuid2 /$uuid3
+
+cleanup;
diff --git a/tests/bugs/bug-948729/bug-948729.t b/tests/bugs/bug-948729/bug-948729.t
new file mode 100644
index 000000000..f94db1ea0
--- /dev/null
+++ b/tests/bugs/bug-948729/bug-948729.t
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../cluster.rc
+
+function check_peers {
+        $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+
+cleanup;
+
+uuid1=`uuidgen`;
+uuid2=`uuidgen`;
+uuid3=`uuidgen`;
+
+TEST launch_cluster 2;
+
+TEST $CLI_1 peer probe $H2;
+
+EXPECT_WITHIN 20 1 check_peers;
+
+B3=/d/backends/3
+
+mkdir -p $B3
+
+TEST truncate -s 16M $B1/brick1
+TEST truncate -s 16M $B2/brick2
+TEST truncate -s 16M $B3/brick3
+
+TEST LD1=`losetup --find --show $B1/brick1`
+TEST mkfs.xfs $LD1
+TEST LD2=`losetup --find --show $B2/brick2`
+TEST mkfs.xfs $LD2
+TEST LD3=`losetup --find --show $B3/brick3`
+TEST mkfs.xfs $LD3
+
+mkdir -p $B1/$V0 $B2/$V0 $B3/$V0
+
+TEST mount -t xfs $LD1 $B1/$V0
+TEST mount -t xfs $LD2 $B2/$V0
+TEST mount -t xfs $LD3 $B3/$V0
+
+#Tests without --mode=script option
+cli1=$(echo $CLI1 | sed 's/ --mode=script//')
+#Case 0: Parent directory of the brick is absent
+TEST ! $cli1 volume create $V0 $H1:$B1/$V0/nonexistent/b1 $H2:$B2/$V0/nonexistent/b2
+
+#Case 1: File system root being used as brick directory
+TEST ! $cli1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0
+
+#Case 2: Brick directory contains only one component
+TEST ! $cli1 volume create $V0 $H1:/$uuid1 $H2:/$uuid2
+
+#Case 3: Sub-directories of the backend FS being used as brick directory
+TEST   $cli1 volume create $V0 $H1:$B1/$V0/brick1 $H2:$B2/$V0/brick2
+
+#add-brick tests
+TEST ! $cli1 volume add-brick $V0 $H1:$B3/$V0/nonexistent/b3
+TEST ! $cli1 volume add-brick $V0 $H1:$B3/$V0
+TEST ! $cli1 volume add-brick $V0 $H1:/$uuid3
+TEST   $cli1 volume add-brick $V0 $H1:$B3/$V0/brick3
+
+#####replace-brick tests
+#FIX-ME: Replace-brick does not work currently in the newly introduced cluster
+#####test framework.
+
+cleanup;
diff --git a/tests/bugs/bug-949242.t b/tests/bugs/bug-949242.t
new file mode 100644
index 000000000..717084673
--- /dev/null
+++ b/tests/bugs/bug-949242.t
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Bug 949242 - Test basic fallocate functionality.
+#
+# Run several commands to verify basic fallocate functionality. We verify that
+# fallocate creates and allocates blocks to a file. We also verify that the keep
+# size option does not modify the file size.
+###
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../fallocate.rc
+
+cleanup;
+
+TEST glusterd
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4}
+TEST $CLI volume start $V0
+
+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+
+# check for fallocate support before continuing the test
+require_fallocate -l 1m -n $M0/file && rm -f $M0/file
+
+# fallocate a file and verify blocks are allocated
+TEST fallocate -l 1m $M0/file
+blksz=`stat --printf=%b $M0/file`
+nblks=`stat --printf=%B $M0/file`
+TEST [ $(($blksz * $nblks)) -eq 1048576 ]
+
+TEST unlink $M0/file
+
+# truncate a file to a fixed size, fallocate and verify that the size does not
+# change
+TEST truncate --size=1m $M0/file
+TEST fallocate -l 2m -n $M0/file
+blksz=`stat --printf=%b $M0/file`
+nblks=`stat --printf=%B $M0/file`
+sz=`stat --printf=%s $M0/file`
+TEST [ $sz -eq 1048576 ]
+# Note that gluster currently incorporates a hack to limit the number of blocks
+# reported as allocated to the file by the file size. We have allocated beyond the
+# file size here. Just check for non-zero allocation to avoid setting a land mine
+# for if/when that behavior might change.
+TEST [ ! $(($blksz * $nblks)) -eq 0 ]
+
+TEST unlink $M0/file
+
+TEST umount $M0
+
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup;
diff --git a/tests/bugs/bug-949298.t b/tests/bugs/bug-949298.t
new file mode 100644
index 000000000..1394127ec
--- /dev/null
+++ b/tests/bugs/bug-949298.t
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI --xml volume info $V0
+
+cleanup;
diff --git a/tests/bugs/bug-949930.t b/tests/bugs/bug-949930.t
new file mode 100644
index 000000000..4a738befa
--- /dev/null
+++ b/tests/bugs/bug-949930.t
@@ -0,0 +1,27 @@
+#!/bin/bash
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+V1=patchy2
+
+cleanup;
+
+TEST glusterd;
+TEST pidof glusterd;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+TEST $CLI volume start $V0;
+
+TEST $CLI volume create $V1 $H0:$B0/${V1}{1,2};
+TEST $CLI volume start $V1;
+
+TEST ! $CLI volume set $V0 performance.nfs.read-ahead blah
+EXPECT '' volume_option $V0 performance.nfs.read-ahead
+
+TEST $CLI volume set $V0 performance.nfs.read-ahead on
+EXPECT "on" volume_option $V0 performance.nfs.read-ahead
+
+EXPECT '' volume_option $V1 performance.nfs.read-ahead
+
+cleanup;
+
diff --git a/tests/bugs/bug-955588.t b/tests/bugs/bug-955588.t
new file mode 100755
index 000000000..3f0361167
--- /dev/null
+++ b/tests/bugs/bug-955588.t
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+function get_brick_host_uuid()
+{
+    local vol=$1;
+    local uuid_regex='[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}'
+    local host_uuid_list=$($CLI volume info $vol --xml | grep "brick.uuid" | grep -o -E "$uuid_regex");
+
+    echo $host_uuid_list | awk '{print $1}'
+}
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+
+uuid=`grep UUID /var/lib/glusterd/glusterd.info | cut -f2 -d=`
+EXPECT $uuid get_brick_host_uuid $V0
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-957877.t b/tests/bugs/bug-957877.t
new file mode 100644
index 000000000..23aefea25
--- /dev/null
+++ b/tests/bugs/bug-957877.t
@@ -0,0 +1,31 @@
+#!/bin/bash
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../afr.rc
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume start $V0;
+
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0;
+kill_brick $V0 $H0 $B0/${V0}0
+TEST touch $M0/f1
+TEST setfattr -n "user.foo" -v "test" $M0/f1
+
+BRICK=$B0"/${V0}1"
+
+TEST $CLI volume start $V0 force
+sleep 5
+TEST $CLI volume heal $V0
+
+# Wait for self-heal to complete
+EXPECT_WITHIN 30 '0' count_sh_entries $BRICK;
+
+TEST getfattr -n "user.foo" $B0/${V0}0/f1;
+
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-958691.t b/tests/bugs/bug-958691.t
new file mode 100644
index 000000000..a5ac406c9
--- /dev/null
+++ b/tests/bugs/bug-958691.t
@@ -0,0 +1,50 @@
+#!/bin/bash
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1}
+TEST $CLI volume start $V0;
+
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0;
+sleep 1;
+TEST mount -t nfs -o vers=3,nolock $H0:/$V0 $N0;
+
+sleep 2;
+
+# Tests for the fuse mount
+TEST mkdir $M0/dir;
+TEST chmod 1777 $M0/dir;
+TEST touch $M0/dir/file{1,2};
+
+TEST $CLI volume set $V0 server.root-squash enable;
+
+mv $M0/dir/file1 $M0/dir/file11 2>/dev/null;
+TEST [ $? -ne 0 ];
+
+TEST $CLI volume set $V0 server.root-squash disable;
+TEST rm -rf $M0/dir;
+
+sleep 1;
+
+# tests for nfs mount
+TEST mkdir $N0/dir;
+TEST chmod 1777 $N0/dir;
+TEST touch $N0/dir/file{1,2};
+
+TEST $CLI volume set $V0 server.root-squash enable;
+
+mv $N0/dir/file1 $N0/dir/file11 2>/dev/null;
+TEST [ $? -ne 0 ];
+
+TEST $CLI volume set $V0 server.root-squash disable;
+TEST rm -rf $N0/dir;
+TEST umount $N0;
+
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-958790.t b/tests/bugs/bug-958790.t
new file mode 100644
index 000000000..6cc799c25
--- /dev/null
+++ b/tests/bugs/bug-958790.t
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+touch /var/lib/glusterd/groups/test
+echo "read-ahead=off" > /var/lib/glusterd/groups/test
+echo "open-behind=off" >> /var/lib/glusterd/groups/test
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+TEST $CLI volume set $V0 group test
+EXPECT "off" volume_option $V0 performance.read-ahead
+EXPECT "off" volume_option $V0 performance.open-behind
+
+cleanup;
diff --git a/tests/bugs/bug-961307.t b/tests/bugs/bug-961307.t
new file mode 100644
index 000000000..96e93a56f
--- /dev/null
+++ b/tests/bugs/bug-961307.t
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+REPLICA=2
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 replica $REPLICA $H0:$B0/${V0}-00 $H0:$B0/${V0}-01 $H0:$B0/${V0}-10 $H0:$B0/${V0}-11
+TEST $CLI volume start $V0
+
+var1=$(gluster volume remove-brick $H0:$B0/${V0}-00 $H0:$B0/${V0}-01 start 2>&1)
+var2="volume remove-brick start: failed: Volume $H0:$B0/${V0}-00 does not exist"
+
+
+function compare_string()
+{
+        val="-1"
+        if [ "$1" == "$2" ]; then
+                val="0"
+        else
+                val="1"
+        fi
+        echo $val
+}
+
+EXPECT 0 compare_string "$var1" "$var2"
+cleanup;
diff --git a/tests/bugs/bug-961615.t b/tests/bugs/bug-961615.t
new file mode 100644
index 000000000..d183e6c52
--- /dev/null
+++ b/tests/bugs/bug-961615.t
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+#This test tests that an extra fd_unref does not happen in rebalance
+#migration completion check code path in dht
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1
+TEST $CLI volume set $V0 performance.quick-read off
+TEST $CLI volume set $V0 performance.io-cache off
+TEST $CLI volume set $V0 performance.write-behind off
+TEST $CLI volume set $V0 performance.stat-prefetch off
+TEST $CLI volume set $V0 performance.read-ahead off
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+TEST touch $M0/1
+#This rename creates a link file for 10 in the other volume.
+TEST mv $M0/1 $M0/10
+#Lets keep writing to the file which will trigger rebalance completion check
+dd if=/dev/zero of=$M0/10 bs=1k &
+bg_pid=$!
+#Now rebalance force will migrate file '10'
+TEST $CLI volume rebalance $V0 start force
+EXPECT_WITHIN 60 "completed" rebalance_status_field $V0
+#If the bug exists mount would have crashed by now
+TEST ls $M0
+kill -9 $bg_pid > /dev/null 2>&1
+wait > /dev/null 2>&1
+cleanup
diff --git a/tests/bugs/bug-961669.t b/tests/bugs/bug-961669.t
new file mode 100644
index 000000000..751a63df2
--- /dev/null
+++ b/tests/bugs/bug-961669.t
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+#Test case: Fail remove-brick 'start' variant when reducing the replica count of a volume.
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+#Create a 3x3 dist-rep volume
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2,3,4,5,6,7,8};
+TEST $CLI volume start $V0
+
+# Mount FUSE and create file/directory
+TEST glusterfs -s $H0 --volfile-id $V0 $M0
+TEST touch $M0/zerobytefile.txt
+TEST mkdir $M0/test_dir
+TEST dd if=/dev/zero of=$M0/file bs=1024 count=1024
+
+function remove_brick_start {
+        $CLI volume remove-brick $V0 replica 2 $H0:$B0/${V0}{1,4,7} start 2>&1|grep -oE 'success|failed'
+}
+
+function remove_brick {
+        $CLI volume remove-brick $V0 replica 2 $H0:$B0/${V0}{1,4,7} 2>&1|grep -oE 'success|failed'
+}
+
+#remove-brick start variant
+#Actual message displayed at cli is:
+#"volume remove-brick start: failed: Rebalancing not needed when reducing replica count. Try without the 'start' option"
+EXPECT "failed" remove_brick_start;
+
+#remove-brick commit-force
+#Actual message displayed at cli is:
+#"volume remove-brick commit force: success"
+EXPECT "success" remove_brick
+
+TEST umount $M0
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-963541.t b/tests/bugs/bug-963541.t
new file mode 100755
index 000000000..950c7db54
--- /dev/null
+++ b/tests/bugs/bug-963541.t
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1..3};
+TEST $CLI volume start $V0;
+
+# Start a remove-brick and try to start a rebalance/remove-brick without committing
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}1 start
+
+TEST ! $CLI volume rebalance $V0 start
+TEST ! $CLI volume remove-brick $V0 $H0:$B0/${V0}2 start
+
+#Try to start rebalance/remove-brick again after commit
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}1 commit
+
+gluster volume status
+
+TEST $CLI volume rebalance $V0 start
+TEST $CLI volume rebalance $V0 stop
+
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}2 start
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}2 stop
+
+TEST $CLI volume stop $V0
+
+cleanup;
+
diff --git a/tests/bugs/bug-963678.t b/tests/bugs/bug-963678.t
new file mode 100644
index 000000000..14d566579
--- /dev/null
+++ b/tests/bugs/bug-963678.t
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Bug 963678 - Test discard functionality
+#
+# Test that basic discard (hole punch) functionality works via the fallocate
+# command line tool. Hole punch deallocates a region of a file, creating a hole
+# and a zero-filled data region. We verify that hole punch works, frees blocks
+# and that subsequent reads do not read stale data (caches are invalidated).
+#
+# NOTE: fuse fallocate is known to be broken with regard to cache invalidation
+# 	up to 3.9.0 kernels. Therefore, FOPEN_KEEP_CACHE is not used in this
+#	test (opens will invalidate the fuse cache).
+###
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../fallocate.rc
+
+cleanup;
+
+TEST glusterd
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2}
+TEST $CLI volume start $V0
+
+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+
+# check for fallocate and hole punch support
+require_fallocate -l 1m $M0/file
+require_fallocate -p -l 512k $M0/file && rm -f $M0/file
+
+# allocate some blocks, punch a hole and verify block allocation
+TEST fallocate -l 1m $M0/file
+blksz=`stat --printf=%B $M0/file`
+nblks=`stat --printf=%b $M0/file`
+TEST [ $(($blksz * $nblks)) -ge 1048576 ]
+TEST fallocate -p -o 512k -l 128k $M0/file
+
+nblks=`stat --printf=%b $M0/file`
+# allow some room for xattr blocks
+TEST [ $(($blksz * $nblks)) -lt $((917504 + 16384)) ]
+TEST unlink $M0/file
+
+# write some data, punch a hole and verify the file content changes
+TEST dd if=/dev/urandom of=$M0/file bs=1M count=1
+TEST cp $M0/file $M0/file.copy.pre
+TEST fallocate -p -o 512k -l 128k $M0/file
+TEST cp $M0/file $M0/file.copy.post
+TEST ! cmp $M0/file.copy.pre $M0/file.copy.post
+TEST unlink $M0/file
+
+TEST umount $M0
+
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup;
diff --git a/tests/bugs/bug-964059.t b/tests/bugs/bug-964059.t
new file mode 100755
index 000000000..df07f95ee
--- /dev/null
+++ b/tests/bugs/bug-964059.t
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../cluster.rc
+
+function check_peers {
+        $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+
+function volume_count {
+        local cli=$1;
+        if [ $cli -eq '1' ] ; then
+                $CLI_1 volume info | grep 'Volume Name' | wc -l;
+        else
+                $CLI_2 volume info | grep 'Volume Name' | wc -l;
+        fi
+}
+
+cleanup;
+
+TEST launch_cluster 2;
+TEST $CLI_1 peer probe $H2;
+
+EXPECT_WITHIN 20 1 check_peers
+
+TEST $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0
+TEST $CLI_1 volume start $V0
+TEST $CLI_1 volume remove-brick $V0 $H2:$B2/$V0 start
+TEST $CLI_1 volume status
+cleanup;
diff --git a/tests/bugs/bug-966018.t b/tests/bugs/bug-966018.t
new file mode 100644
index 000000000..2a4697241
--- /dev/null
+++ b/tests/bugs/bug-966018.t
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+#This tests if eager-lock blocks metadata operations on nfs/fuse mounts.
+#If it is not woken up, INODELK from the next command waits
+#for post-op-delay secs.
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/r2_0 $H0:$B0/r2_1
+TEST $CLI volume set $V0 ensure-durability off
+TEST $CLI volume set $V0 cluster.eager-lock on
+TEST $CLI volume set $V0 cluster.post-op-delay-secs 3
+
+TEST $CLI volume start $V0
+TEST $CLI volume profile $V0 start
+sleep 5
+TEST mount -t nfs -o vers=3,nolock $H0:/$V0 $N0;
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0
+echo 1 > $N0/1 && chmod +x $N0/1
+echo 1 > $M0/1 && chmod +x $M0/1
+
+#Check that INODELK MAX latency is not in the order of seconds
+#Test if the MAX INODELK fop latency is of the order of seconds.
+inodelk_max_latency=$($CLI volume profile $V0 info | grep INODELK | awk 'BEGIN {max = 0} {if ($6 > max) max=$6;} END {print max}' | cut -d. -f 1 | egrep "[0-9]{7,}")
+
+TEST [ -z $inodelk_max_latency ]
+TEST umount $N0
+
+cleanup;
diff --git a/tests/bugs/bug-969193.t b/tests/bugs/bug-969193.t
new file mode 100755
index 000000000..e78a2980e
--- /dev/null
+++ b/tests/bugs/bug-969193.t
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Test that "system getspec" works without op_version problems.
+
+. $(dirname $0)/../include.rc
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+TEST $CLI volume create $V0 $H0:$B0/brick1
+TEST $CLI system getspec $V0
+cleanup;
diff --git a/tests/bugs/bug-970070.t b/tests/bugs/bug-970070.t
new file mode 100755
index 000000000..da28b1ed7
--- /dev/null
+++ b/tests/bugs/bug-970070.t
@@ -0,0 +1,14 @@
+#!/bin/bash
+# TEST the nfs.acl option
+. $(dirname $0)/../include.rc
+
+cleanup
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/$V0
+TEST $CLI volume start $V0
+sleep 5
+TEST $CLI volume set $V0 nfs.acl off
+TEST $CLI volume set $V0 nfs.acl on
+cleanup
diff --git a/tests/bugs/bug-973073.t b/tests/bugs/bug-973073.t
new file mode 100755
index 000000000..83e2839c6
--- /dev/null
+++ b/tests/bugs/bug-973073.t
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../dht.rc
+
+## Steps followed are one descibed in bugzilla
+
+cleanup;
+
+function get_layout()
+{
+        layout1=`getfattr -n trusted.glusterfs.dht -e hex $1 2>&1`
+
+        if [ $? -ne 0 ]
+        then
+                echo 1
+        else
+                echo 0
+        fi
+
+}
+
+BRICK_COUNT=3
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}0 $H0:$B0/${V0}1 $H0:$B0/${V0}2
+TEST $CLI volume start $V0
+
+## Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}2 start
+
+## remove-brick status == rebalance_status
+EXPECT_WITHIN 30 "0" rebalance_completed
+
+TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}2 stop
+
+TEST $CLI volume rebalance $V0 fix-layout start
+
+EXPECT_WITHIN 30 "0" rebalance_completed
+
+TEST mkdir $M0/dir 2>/dev/null;
+
+EXPECT "0" get_layout $B0/${V0}2/dir
+cleanup;
diff --git a/tests/bugs/bug-974007.t b/tests/bugs/bug-974007.t
new file mode 100644
index 000000000..c8c1c862b
--- /dev/null
+++ b/tests/bugs/bug-974007.t
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+#Test case: Create a distributed replicate volume, and remove multiple
+#replica pairs in a single remove-brick command.
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+#Create a 3X2 distributed-replicate volume
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1..6};
+TEST $CLI volume start $V0
+
+# Mount FUSE and create files
+TEST glusterfs -s $H0 --volfile-id $V0 $M0
+TEST touch $M0/file{1..10}
+
+# Remove bricks from two sub-volumes to make it a 1x2 vol.
+# Bricks in question are given in a random order but from the same subvols.
+function remove_brick_start_status {
+        $CLI volume remove-brick $V0 \
+        $H0:$B0/${V0}6  $H0:$B0/${V0}1 \
+        $H0:$B0/${V0}2  $H0:$B0/${V0}5 start 2>&1 |grep -oE "success|failed"
+}
+EXPECT "success"  remove_brick_start_status;
+
+# Wait for rebalance to complete
+EXPECT_WITHIN 10 "completed" remove_brick_status_completed_field "$V0" "$H0:$B0/${V0}6 $H0:$B0/${V0}1 $H0:$B0/${V0}2 $H0:$B0/${V0}5"
+
+# Check commit status
+function remove_brick_commit_status {
+        $CLI volume remove-brick $V0 \
+         $H0:$B0/${V0}6  $H0:$B0/${V0}1 \
+         $H0:$B0/${V0}2  $H0:$B0/${V0}5 commit 2>&1 |grep -oE "success|failed"
+}
+EXPECT "success" remove_brick_commit_status;
+
+# Check the volume type
+EXPECT "Replicate" echo `$CLI volume info |grep Type |awk '{print $2}'`
+
+TEST umount $M0
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-974972.t b/tests/bugs/bug-974972.t
new file mode 100755
index 000000000..15deac090
--- /dev/null
+++ b/tests/bugs/bug-974972.t
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+#This script checks that nfs mount does not fail lookup on files with split-brain
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume set $V0 self-heal-daemon off
+TEST $CLI volume start $V0
+sleep 5
+TEST mount -t nfs -o vers=3 $H0:/$V0 $N0
+TEST touch $N0/1
+TEST kill_brick ${V0} ${H0} ${B0}/${V0}1
+echo abc > $N0/1
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "Y" nfs_up_status
+EXPECT_WITHIN 20 "1" afr_child_up_status_in_nfs $V0 0
+EXPECT_WITHIN 20 "1" afr_child_up_status_in_nfs $V0 1
+
+TEST kill_brick ${V0} ${H0} ${B0}/${V0}0
+echo def > $N0/1
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "Y" nfs_up_status
+EXPECT_WITHIN 20 "1" afr_child_up_status_in_nfs $V0 0
+EXPECT_WITHIN 20 "1" afr_child_up_status_in_nfs $V0 1
+
+#Lookup should not fail
+TEST ls $N0/1
+TEST ! cat $N0/1
+
+TEST umount $N0
+cleanup
diff --git a/tests/bugs/bug-976800.t b/tests/bugs/bug-976800.t
new file mode 100644
index 000000000..2aee8cc11
--- /dev/null
+++ b/tests/bugs/bug-976800.t
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+# This test checks if there are any open fds on the brick
+# even after the file is closed on the mount. This particular
+# test tests dd with "fsync" to check afr's fsync codepath
+cleanup;
+
+function is_fd_open {
+        local v=$1
+        local h=$2
+        local b=$3
+        local bpid=$(get_brick_pid $v $h $b)
+        ls -l /proc/$bpid/fd | grep -w "\-> $b/1"
+}
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume set $V0 ensure-durability off
+TEST $CLI volume set $V0 eager-lock off
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+TEST dd of=$M0/1 if=/dev/zero bs=1k count=1 conv=fsync
+TEST ! is_fd_open $V0 $H0 $B0/${V0}0
+cleanup;
diff --git a/tests/bugs/bug-977246.t b/tests/bugs/bug-977246.t
new file mode 100644
index 000000000..e07ee1919
--- /dev/null
+++ b/tests/bugs/bug-977246.t
@@ -0,0 +1,21 @@
+#! /bin/bash
+
+# This test checks if address validation, correctly catches hostnames
+# with consective dots, such as 'example..org', as invalid
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}1
+TEST $CLI volume info $V0
+TEST $CLI volume start $V0
+
+TEST ! $CLI volume set $V0 auth.allow example..org
+
+TEST $CLI volume stop $V0
+
+cleanup;
diff --git a/tests/bugs/bug-977797.t b/tests/bugs/bug-977797.t
new file mode 100755
index 000000000..08cdbe8f1
--- /dev/null
+++ b/tests/bugs/bug-977797.t
@@ -0,0 +1,114 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2};
+
+## Verify volume is is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+TEST $CLI volume set $V0 self-heal-daemon off
+TEST $CLI volume set $V0 open-behind off
+TEST $CLI volume set $V0 quick-read off
+TEST $CLI volume set $V0 read-ahead off
+TEST $CLI volume set $V0 write-behind off
+TEST $CLI volume set $V0 io-cache off
+TEST $CLI volume set $V0 background-self-heal-count 0
+
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+
+
+TEST mkdir -p $M0/a
+TEST `echo "GLUSTERFS" > $M0/a/file`
+
+TEST kill_brick $V0 $H0 $B0/$V0"1"
+
+TEST chown root $M0/a
+TEST chown root $M0/a/file
+TEST `echo "GLUSTER-FILE-SYSTEM" > $M0/a/file`
+TEST mkdir $M0/a/b
+
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0;
+
+
+
+TEST kill_brick $V0 $H0 $B0/$V0"2"
+
+TEST chmod 757 $M0/a
+TEST chmod 757 $M0/a/file
+
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 1;
+
+TEST ls   -l $M0/a/file
+
+b1c0dir=$(afr_get_specific_changelog_xattr $B0/$V0"1"/a \
+          trusted.afr.$V0-client-0 "entry")
+b1c1dir=$(afr_get_specific_changelog_xattr $B0/$V0"1"/a \
+          trusted.afr.$V0-client-1 "entry")
+b2c0dir=$(afr_get_specific_changelog_xattr \
+          $B0/$V0"2"/a trusted.afr.$V0-client-0 "entry")
+b2c1dir=$(afr_get_specific_changelog_xattr \
+          $B0/$V0"2"/a trusted.afr.$V0-client-1 "entry")
+
+
+b1c0f=$(afr_get_specific_changelog_xattr $B0/$V0"1"/a/file \
+        trusted.afr.$V0-client-0 "data")
+b1c1f=$(afr_get_specific_changelog_xattr $B0/$V0"1"/a/file \
+        trusted.afr.$V0-client-1 "data")
+b2c0f=$(afr_get_specific_changelog_xattr $B0/$V0"2"/a/file \
+        trusted.afr.$V0-client-0 "data")
+b2c1f=$(afr_get_specific_changelog_xattr $B0/$V0"2"/a/file \
+        trusted.afr.$V0-client-1 "data")
+
+EXPECT "00000000" echo $b1c0f
+EXPECT "00000000" echo $b1c1f
+EXPECT "00000000" echo $b2c0f
+EXPECT "00000000" echo $b2c1f
+
+EXPECT "00000000" echo $b1c0dir
+EXPECT "00000000" echo $b1c1dir
+EXPECT "00000000" echo $b2c0dir
+EXPECT "00000000" echo $b2c1dir
+
+contains() {
+    string="$1"
+    substring="$2"
+    var="-1"
+    if test "${string#*$substring}" != "$string"
+    then
+        var="0"    # $substring is in $string
+    else
+        var="1"    # $substring is not in $string
+    fi
+    echo $var
+}
+
+var1=$(cat $M0/a/file 2>&1)
+var2="Input/output error"
+
+
+EXPECT "0" contains "$var1" "$var2"
+
+## Finish up
+TEST $CLI volume stop $V0;
+EXPECT 'Stopped' volinfo_field $V0 'Status';
+
+TEST $CLI volume delete $V0;
+TEST ! $CLI volume info $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-978794.t b/tests/bugs/bug-978794.t
new file mode 100644
index 000000000..d22d3cde3
--- /dev/null
+++ b/tests/bugs/bug-978794.t
@@ -0,0 +1,29 @@
+#!/bin/bash
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../fileio.rc
+
+
+# This test opens 100 fds and triggers graph switches to check if fsync
+# as part of graph-switch causes crash or not.
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0
+TEST touch $M0/{1..100}
+for i in {1..100}; do fd[$i]=`fd_available`; fd_open ${fd[$i]} 'w' $M0/$i; done
+TEST $CLI volume add-brick $V0 $H0:$B0/${V0}{2,3}
+TEST $CLI volume rebalance $V0 start force
+EXPECT_WITHIN 120 "completed" rebalance_status_field $V0
+TEST cat $M0/{1..100}
+for i in {1..100}; do fd_write ${fd[$i]} 'abc'; done
+TEST $CLI volume add-brick $V0 $H0:$B0/${V0}{4,5}
+TEST $CLI volume rebalance $V0 start force
+EXPECT_WITHIN 120 "completed" rebalance_status_field $V0
+for i in {1..100}; do fd_write ${fd[$i]} 'abc'; done
+TEST cat $M0/{1..100}
+cleanup
diff --git a/tests/bugs/bug-979365.t b/tests/bugs/bug-979365.t
new file mode 100755
index 000000000..e94dc9aa8
--- /dev/null
+++ b/tests/bugs/bug-979365.t
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+#This script checks that ensure-durability option enables/disables afr
+#sending fsyncs
+cleanup;
+
+function num_fsyncs {
+        $CLI volume profile $V0 info | grep -w FSYNC | wc -l
+}
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume set $V0 ensure-durability on
+TEST $CLI volume set $V0 eager-lock off
+TEST $CLI volume start $V0
+TEST $CLI volume profile $V0 start
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0
+TEST kill_brick $V0 $H0 $B0/${V0}0
+TEST dd of=$M0/a if=/dev/zero bs=1M count=10
+#fsyncs take a while to complete.
+sleep 5
+
+# There can be zero or more fsyncs, depending on the order
+# in which the writes reached the server, in turn deciding
+# whether they were treated as "appending" writes or not.
+
+TEST [[ $(num_fsyncs) -ge 0 ]]
+#Stop the volume to erase the profile info of old operations
+TEST $CLI volume profile $V0 stop
+TEST $CLI volume stop $V0
+umount $M0
+#Disable ensure-durability now to disable fsyncs in afr.
+TEST $CLI volume set $V0 ensure-durability off
+TEST $CLI volume start $V0
+TEST kill_brick $V0 $H0 $B0/${V0}0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0
+TEST $CLI volume profile $V0 start
+TEST dd of=$M0/a if=/dev/zero bs=1M count=10
+#fsyncs take a while to complete.
+sleep 5
+TEST [[ $(num_fsyncs) -eq 0 ]]
+
+cleanup
diff --git a/tests/bugs/bug-982174.t b/tests/bugs/bug-982174.t
new file mode 100644
index 000000000..460af7511
--- /dev/null
+++ b/tests/bugs/bug-982174.t
@@ -0,0 +1,36 @@
+#!/bin/bash
+# Test to check
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+#Check if incorrect log-level keywords does not crash the CLI
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/brick1 $H0:$B0/brick2
+TEST $CLI volume start $V0
+
+function set_log_level_status {
+        local level=$1
+        $CLI volume set $V0 diagnostics.client-log-level $level 2>&1 |grep -oE 'success|failed'
+}
+
+
+LOG_LEVEL="trace"
+EXPECT "failed" set_log_level_status $LOG_LEVEL
+
+
+LOG_LEVEL="error-gen"
+EXPECT "failed" set_log_level_status $LOG_LEVEL
+
+
+LOG_LEVEL="TRACE"
+EXPECT "success" set_log_level_status $LOG_LEVEL
+
+EXPECT "$LOG_LEVEL" echo `$CLI volume info | grep diagnostics | awk '{print $2}'`
+
+TEST $CLI volume stop $V0;
+TEST $CLI volume delete $V0;
+
+cleanup;
diff --git a/tests/bugs/bug-983477.t b/tests/bugs/bug-983477.t
new file mode 100755
index 000000000..c19fa96c8
--- /dev/null
+++ b/tests/bugs/bug-983477.t
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+#This script checks if use-readdirp option works as accepted in mount options
+
+function get_use_readdirp_value {
+        local vol=$1
+        local statedump=$(generate_mount_statedump $vol)
+        local val=$(grep "use_readdirp=" $statedump | cut -f2 -d'=' | tail -1)
+        rm -f $statedump
+        echo $val
+}
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/${V0}
+TEST $CLI volume start $V0
+#If readdirp is enabled statedump should reflect it
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 --use-readdirp=yes
+TEST cd $M0
+EXPECT_WITHIN 20 "1" get_use_readdirp_value $V0
+TEST cd -
+TEST umount $M0
+
+#If readdirp is enabled statedump should reflect it
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 --use-readdirp=no
+TEST cd $M0
+EXPECT_WITHIN 20 "0" get_use_readdirp_value $V0
+TEST cd -
+TEST umount $M0
+
+#Since args are optional on this argument just specifying "--use-readdirp" should also turn it `on` not `off`
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 --use-readdirp
+TEST cd $M0
+EXPECT_WITHIN 20 "1" get_use_readdirp_value $V0
+TEST cd -
+TEST umount $M0
+
+#By default it is enabled.
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+TEST cd $M0
+EXPECT_WITHIN 20 "1" get_use_readdirp_value $V0
+TEST cd -
+TEST umount $M0
+
+#Invalid values for use-readdirp should not be accepted
+TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 --use-readdirp=please-fail
+
+cleanup
diff --git a/tests/bugs/bug-985074.t b/tests/bugs/bug-985074.t
new file mode 100644
index 000000000..80052129e
--- /dev/null
+++ b/tests/bugs/bug-985074.t
@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Bug 985074 - Verify stale inode/dentry mappings are cleaned out.
+#
+# This test verifies that an inode/dentry mapping for a file removed via a
+# separate mount point is cleaned up appropriately. We create a file and hard
+# link from client 1. Next we remove the link via client 2. Finally, from client
+# 1 we attempt to rename the original filename to the name of the just removed
+# hard link.
+#
+# If the inode is not unlinked properly, the removed directory entry can resolve
+# to an inode (on the client that never saw the rm) that ends up passed down
+# through the lookup call. If md-cache holds valid metadata on the inode (due to
+# a large timeout value or recent lookup on the valid name), it is tricked into
+# returning a successful lookup that should have returned ENOENT. This manifests
+# as an error from the mv command in the following test sequence because file
+# and file.link resolve to the same file:
+#
+# # mv /mnt/glusterfs/0/file /mnt/glusterfs/0/file.link
+# mv: `/mnt/glusterfs/0/file' and `/mnt/glusterfs/0/file.link' are the same file
+#
+###
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/$V0
+TEST $CLI volume start $V0
+TEST $CLI volume set $V0 md-cache-timeout 3
+
+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 --entry-timeout=0 --attribute-timeout=0
+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M1 --entry-timeout=0 --attribute-timeout=0
+
+TEST touch $M0/file
+TEST ln $M0/file $M0/file.link
+TEST ls -ali $M0 $M1
+TEST rm -f $M1/file.link
+TEST ls -ali $M0 $M1
+# expire the md-cache timeout
+sleep 3
+TEST mv $M0/file $M0/file.link
+TEST stat $M0/file.link
+TEST ! stat $M0/file
+
+TEST umount $M1
+TEST umount $M0
+
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup;
diff --git a/tests/bugs/bug-986429.t b/tests/bugs/bug-986429.t
new file mode 100644
index 000000000..6e43f72b7
--- /dev/null
+++ b/tests/bugs/bug-986429.t
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+## This tests failover achieved by providing multiple
+## servers from the trusted pool for fetching volume
+## specification
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/$V0
+TEST $CLI volume start $V0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s non-existent -s $H0 --volfile-id=/$V0 $M0
+
+cleanup;
diff --git a/tests/bugs/bug-986905.t b/tests/bugs/bug-986905.t
new file mode 100755
index 000000000..0fac40fb4
--- /dev/null
+++ b/tests/bugs/bug-986905.t
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+#This script checks if hardlinks that are created while a brick is down are
+#healed properly.
+
+cleanup;
+function get_inum {
+        ls -i $1 | awk '{print $1}'
+}
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1}
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+TEST kill_brick $V0 $H0 $B0/${V0}0
+TEST touch $M0/a
+TEST ln $M0/a $M0/link_a
+TEST $CLI volume start $V0 force
+EXPECT_WITHIN 20 "1" afr_child_up_status $V0 0
+TEST ls -l $M0
+inum=$(get_inum $B0/${V0}0/a)
+EXPECT "$inum" get_inum $B0/${V0}0/link_a
+cleanup
diff --git a/tests/bugs/bug-991622.t b/tests/bugs/bug-991622.t
new file mode 100644
index 000000000..5c3243465
--- /dev/null
+++ b/tests/bugs/bug-991622.t
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+. $(dirname $0)/../fileio.rc
+
+#This tests that no fd leaks are observed in unlink/rename in open-behind
+function leaked_fds {
+        ls -l /proc/$(get_brick_pid $V0 $H0 $B0/$V0)/fd | grep deleted
+}
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/$V0
+TEST $CLI volume set $V0 open-behind on
+TEST $CLI volume start $V0
+TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id=$V0 $M0 --direct-io-mode=enable
+
+TEST fd1=`fd_available`
+TEST fd_open $fd1 'w' "$M0/testfile1"
+TEST fd_write $fd1 "content"
+
+TEST fd2=`fd_available`
+TEST fd_open $fd2 'w' "$M0/testfile2"
+TEST fd_write $fd2 "content"
+
+TEST touch $M0/a
+TEST rm $M0/testfile1
+TEST mv $M0/a $M0/testfile2
+TEST fd_close $fd1
+TEST fd_close $fd2
+TEST ! leaked_fds
+cleanup;
diff --git a/tests/bugs/getlk_owner.c b/tests/bugs/getlk_owner.c
new file mode 100644
index 000000000..619c2e32d
--- /dev/null
+++ b/tests/bugs/getlk_owner.c
@@ -0,0 +1,120 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+
+#define GETLK_OWNER_CHECK(f, cp, label)                     \
+    do {                                                    \
+        switch (f.l_type) {                                 \
+            case F_RDLCK:                                   \
+            case F_WRLCK:                                   \
+                ret = 1;                                    \
+                goto label;                                 \
+            case F_UNLCK:                                   \
+                if (!are_flocks_sane (&f, &cp)) {           \
+                    ret = 1;                                \
+                    goto label;                             \
+                }                                           \
+                break;                                      \
+        }                                                   \
+    } while (0)
+
+void
+flock_init (struct flock *f, short int type, off_t start, off_t len)
+{
+    f->l_type = type;
+    f->l_start = start;
+    f->l_len = len;
+}
+
+int
+flock_cp (struct flock *dst, struct flock *src)
+{
+    memcpy ((void *) dst, (void *) src, sizeof (struct flock));
+}
+
+int
+are_flocks_sane (struct flock *src, struct flock *cpy)
+{
+    return ((src->l_whence == cpy->l_whence) &&
+            (src->l_start == cpy->l_start) &&
+            (src->l_len == cpy->l_len));
+}
+
+/*
+ * Test description:
+ * SETLK (0,3), F_WRLCK
+ * SETLK (3,3), F_WRLCK
+ *
+ * the following GETLK requests must return flock struct unmodified
+ * except for l_type to F_UNLCK
+ * GETLK (3,3), F_WRLCK
+ * GETLK (3,3), F_RDLCK
+ *
+ * */
+
+int main (int argc, char **argv)
+{
+    int fd = -1;
+    int ret = 1;
+    char *fname = NULL;
+    struct flock f = {0,};
+    struct flock cp = {0,};
+
+    if (argc < 2)
+        goto out;
+
+    fname = argv[1];
+    fd = open (fname, O_RDWR);
+    if (fd == -1) {
+        perror ("open");
+        goto out;
+    }
+
+    flock_init (&f, F_WRLCK, 0, 3);
+    flock_cp (&cp, &f);
+    ret = fcntl (fd, F_SETLK, &f);
+    if (ret) {
+        perror ("fcntl");
+        goto out;
+    }
+    if (!are_flocks_sane (&f, &cp)) {
+        ret = 1;
+        goto out;
+    }
+
+    flock_init (&f, F_WRLCK, 3, 3);
+    flock_cp (&cp, &f);
+    ret = fcntl (fd, F_SETLK, &f);
+    if (ret) {
+        perror ("fcntl");
+        goto out;
+    }
+    if (!are_flocks_sane (&f, &cp)) {
+        ret = 1;
+        goto out;
+    }
+
+    flock_init (&f, F_WRLCK, 3, 3);
+    flock_cp (&cp, &f);
+    ret = fcntl (fd, F_GETLK, &f);
+    if (ret) {
+        perror ("fcntl");
+        return 1;
+    }
+    GETLK_OWNER_CHECK (f, cp, out);
+
+    flock_init (&f, F_RDLCK, 3, 3);
+    flock_cp (&cp, &f);
+    ret = fcntl (fd, F_GETLK, &f);
+    if (ret) {
+        perror ("fcntl");
+        return 1;
+    }
+    GETLK_OWNER_CHECK (f, cp, out);
+
+out:
+    if (fd != -1)
+        close (fd);
+    return ret;
+}
diff --git a/tests/bugs/overlap.py b/tests/bugs/overlap.py
new file mode 100755
index 000000000..15f2da473
--- /dev/null
+++ b/tests/bugs/overlap.py
@@ -0,0 +1,59 @@
+#!/usr/bin/python
+
+import sys
+
+def calculate_one (ov, nv):
+    old_start = int(ov[18:26],16)
+    old_end = int(ov[26:34],16)
+    new_start = int(nv[18:26],16)
+    new_end = int(nv[26:34],16)
+    if (new_end < old_start) or (new_start > old_end):
+        #print '%s, %s -> ZERO' % (ov, nv)
+        return 0
+    all_start = max(old_start,new_start)
+    all_end = min(old_end,new_end)
+    #print '%s, %s -> %08x' % (ov, nv, all_end - all_start + 1)
+    return all_end - all_start + 1
+
+def calculate_all (values):
+    total = 0
+    nv_index = len(values) / 2
+    for old_val in values[:nv_index]:
+        new_val = values[nv_index]
+        nv_index += 1
+        total += calculate_one(old_val,new_val)
+    return total
+
+"""
+test1_vals = [
+    '0x0000000000000000000000003fffffff',   # first quarter
+    '0x0000000000000000400000007fffffff',   # second quarter
+    '0x000000000000000080000000ffffffff',   # second half
+    '0x00000000000000000000000055555554',   # first third
+    '0x000000000000000055555555aaaaaaa9',   # second third
+    '0x0000000000000000aaaaaaaaffffffff',   # last third
+]
+
+test2_vals = [
+    '0x0000000000000000000000003fffffff',   # first quarter
+    '0x0000000000000000400000007fffffff',   # second quarter
+    '0x000000000000000080000000ffffffff',   # second half
+    '0x00000000000000000000000055555554',   # first third
+    # Next two are (incorrectly) swapped.
+    '0x0000000000000000aaaaaaaaffffffff',   # last third
+    '0x000000000000000055555555aaaaaaa9',   # second third
+]
+
+print '%08x' % calculate_one(test1_vals[0],test1_vals[3])
+print '%08x' % calculate_one(test1_vals[1],test1_vals[4])
+print '%08x' % calculate_one(test1_vals[2],test1_vals[5])
+print '= %08x' % calculate_all(test1_vals)
+print '%08x' % calculate_one(test2_vals[0],test2_vals[3])
+print '%08x' % calculate_one(test2_vals[1],test2_vals[4])
+print '%08x' % calculate_one(test2_vals[2],test2_vals[5])
+print '= %08x' % calculate_all(test2_vals)
+"""
+
+if __name__ == '__main__':
+    # Return decimal so bash can reason about it.
+    print '%d' % calculate_all(sys.argv[1:])
diff --git a/tests/cluster.rc b/tests/cluster.rc
new file mode 100755
index 000000000..3b10d19f7
--- /dev/null
+++ b/tests/cluster.rc
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+CLUSTER_PFX="127.1.1"; # ".x" for each glusterd
+CLUSTER_COUNT=1; # Just initial definition
+
+function launch_cluster() {
+    local count=$1;
+
+    CLUSTER_COUNT=$count;
+
+    define_backends $count;
+    define_hosts $count;
+    define_glusterds $count;
+    define_clis $count;
+
+    start_glusterds;
+}
+
+
+function define_backends() {
+    local b;
+
+    for i in `seq 1 $count`; do
+	eval "B$i=$B0/$i";
+    done
+
+    for i in `seq 1 $count`; do
+	b="B$i";
+	mkdir -pv ${!b}/glusterd;
+    done
+}
+
+
+function define_glusterds() {
+    local count=$1;
+    local h;
+    local b;
+    local wopt;
+    local bopt;
+    local popt;
+
+    for i in `seq 1 $count`; do
+	b="B$i";
+	h="H$i";
+	wopt="management.working-directory=${!b}/glusterd";
+	bopt="management.transport.socket.bind-address=${!h}";
+	popt="--pid-file=${!b}/glusterd.pid";
+        sopt="management.glusterd-sockfile=${!b}/glusterd/gd.sock"
+        lopt="--log-file=${!b}/glusterd.log"
+	eval "glusterd_$i='glusterd --xlator-option $wopt --xlator-option $bopt --xlator-option $sopt $lopt $popt'";
+	eval "glusterd$i='glusterd --xlator-option $wopt --xlator-option $bopt --xlator-option $sopt $lopt $popt'";
+    done
+}
+
+
+function start_glusterds() {
+    local g;
+
+    for i in `seq 1 $CLUSTER_COUNT`; do
+	g="glusterd_$i";
+	${!g};
+    done
+}
+
+
+function kill_glusterd() {
+    local index=$1;
+    local b;
+    local pidfile;
+
+    b="B$index";
+    pidfile="${!b}/glusterd.pid";
+
+    kill `cat $pidfile`;
+}
+
+
+function kill_node() {
+    local index=$1;
+    local h;
+
+    h="H$index";
+
+    kill -9 $(ps -ef | grep gluster | grep ${!h} | awk '{print $2}');
+}
+
+
+function define_hosts() {
+    local count=$1;
+
+    for i in `seq 1 $count`; do
+	eval "H_$i=${CLUSTER_PFX}.$i"
+	eval "H$i=${CLUSTER_PFX}.$i";
+    done
+}
+
+
+function define_clis() {
+    local count=$1;
+    local h;
+
+    for i in `seq 1 $count`; do
+	b="B$i";
+	eval "CLI_$i='$CLI --glusterd-sock=${!b}/glusterd/gd.sock'";
+	eval "CLI$i='$CLI --glusterd-sock=${!b}/glusterd/gd.sock'";
+    done
+    CLI="$CLI_1"
+}
+
+function peer_count() {
+    $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
diff --git a/tests/dht.rc b/tests/dht.rc
new file mode 100644
index 000000000..663ea5431
--- /dev/null
+++ b/tests/dht.rc
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+function get_layout()
+{
+        getfattr -n trusted.glusterfs.dht -e hex $1 2>&1|grep dht |cut -d = -f2
+
+}
+
+## populates $BRICK1 and $BRICK2 with hashed/cached subvolume. These will be
+## used by get_cached_brick and get_hashed_brick
+
+function file_has_linkfile()
+{
+        k=0
+        l=0
+        while [ $k -lt $BRICK_COUNT ]
+        do
+                stat=`stat $B0/${V0}$k/$1 2>/dev/null`
+                if [ $? -eq 0 ]
+                then
+                        let l++
+                        let "BRICK${l}=$k"
+
+                fi
+                let k++
+        done
+        return $l
+}
+
+function get_cached_brick()
+{
+        i=1
+        brick=$BRICK1
+        while [ $i -lt 3 ]
+        do
+                test=`getfattr -n trusted.glusterfs.dht.linkto -e text $B0/${V0}$brick/$1 2>&1`
+                if [ $? -eq 1 ]
+                then
+                        cached=$brick
+                        i=$(( $i+3 ))
+                fi
+                brick=$BRICK1
+                let i++
+        done
+
+        return $cached
+}
+
+function get_hashed_brick()
+{
+        j=1
+        brick=$BRICK1
+        while [ $j -lt 3 ]
+        do
+                test=`getfattr -n trusted.glusterfs.dht.linkto -e text $B0/${V0}$brick/$1 2>&1`
+                if [ $? -eq 0 ]
+                then
+                        hashed=$brick
+                        j=$(( $j+3 ))
+                fi
+                brick=$BRICK2
+                let j++
+        done
+
+        return $hashed
+}
+
+
+function rebalance_completed()
+{
+       val=1
+       test=`gluster volume rebalance $V0 status |grep localhost|grep -v "in progress" 2>&1`
+       if [ $? -eq 0 ]
+       then
+                val=0
+       fi
+
+       echo $val
+}
diff --git a/tests/fallocate.rc b/tests/fallocate.rc
new file mode 100644
index 000000000..3756bb949
--- /dev/null
+++ b/tests/fallocate.rc
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Helper to verify a given fallocate command is supported and skip a test
+# otherwise. Older versions of the fallocate utility might not support all modes
+# (i.e., discard) and older versions of fuse might not support the associated
+# fallocate requests.
+
+function require_fallocate()
+{
+	output=`fallocate $* 2>&1`
+	ret=$?
+	if [ ! $ret -eq 0 ] && ([[ $output == *unsupported* ]] ||
+				[[ $output == *invalid* ]] ||
+				[[ $output == *"not supported"* ]])
+	then
+		SKIP_TESTS
+		exit
+	fi
+}
diff --git a/tests/features/glupy.t b/tests/features/glupy.t
new file mode 100755
index 000000000..49bf11df5
--- /dev/null
+++ b/tests/features/glupy.t
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST mkdir -p $B0/glupytest
+cat > $B0/glupytest.vol <<EOF
+volume vol-posix
+    type storage/posix
+    option directory $B0/glupytest
+end-volume
+
+volume vol-glupy
+    type features/glupy
+    option module-name helloworld
+    subvolumes vol-posix
+end-volume
+EOF
+
+TEST glusterfs -f $B0/glupytest.vol $M0;
+
+TEST touch $M0/filename;
+EXPECT "filename" ls $M0
+TEST rm -f $M0/filename;
+
+TEST umount -l $M0;
+
+cleanup;
diff --git a/tests/features/readdir-ahead.t b/tests/features/readdir-ahead.t
new file mode 100755
index 000000000..c7ee637f0
--- /dev/null
+++ b/tests/features/readdir-ahead.t
@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Test basic readdir-ahead functionality. Verify that readdir-ahead can be
+# enabled, create a set of files and run some ls tests.
+#
+###
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+
+TEST $CLI volume create $V0 $H0:$B0/$V0
+TEST $CLI volume start $V0
+
+TEST $CLI volume set $V0 readdir-ahead on
+
+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0
+
+TEST mkdir $M0/test
+for i in $(seq 0 99)
+do
+	touch $M0/test/$i
+done
+
+count=`ls -1 $M0/test | wc -l`
+TEST [ $count -eq 100 ]
+
+count=`ls -1 $M0/test | wc -l`
+TEST [ $count -eq 100 ]
+
+TEST rm -rf $M0/test/*
+
+count=`ls -1 $M0/test | wc -l`
+TEST [ $count -eq 0 ]
+
+TEST rmdir $M0/test
+
+TEST umount -l $M0;
+TEST $CLI volume stop $V0
+TEST $CLI volume delete $V0
+
+cleanup;
diff --git a/tests/fileio.rc b/tests/fileio.rc
new file mode 100644
index 000000000..58871b3b9
--- /dev/null
+++ b/tests/fileio.rc
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+function fd_available() {
+         for i in {1..65536}; do
+	     if [ ! -e /proc/$$/fd/$i ]; then
+		 echo $i;
+		 return 0;
+	     fi
+	 done
+
+	 return 1;
+}
+
+function fd_open() {
+	 local fd=$1;
+	 local mode=$2
+	 local path=$3;
+
+	 case $mode in
+	     r)
+		 eval "exec $fd<$path";;
+	     w)
+		 eval "exec $fd>$path";;
+	     rw)
+		 eval "exec $fd<>$path";;
+	     *)
+		 false;;
+	 esac
+}
+
+
+function fd_cat() {
+	 local fd=$1;
+
+	 eval "cat <&$fd";
+}
+
+
+function fd_write() {
+	 local fd=$1;
+	 shift;
+	 local msg="$@";
+
+	 eval "echo $@ >&$fd";
+}
+
+
+function fd_close() {
+	 local fd=$1;
+
+	 eval "exec $fd>&-";
+}
+
+
+function fd_based_example() {
+    TEST fd=`fd_available`;
+    TEST fd_open $fd "rw" $M0/filename;
+    TEST fd_cat $fd; # print existing stuff
+    TEST fd_write $fd "new stuff"; # append
+    TEST fd_close $fd;
+}
diff --git a/tests/include.rc b/tests/include.rc
new file mode 100644
index 000000000..80457f124
--- /dev/null
+++ b/tests/include.rc
@@ -0,0 +1,248 @@
+M0=${M0:=/mnt/glusterfs/0};   # 0th mount point for FUSE
+M1=${M1:=/mnt/glusterfs/1};   # 1st mount point for FUSE
+N0=${N0:=/mnt/nfs/0};         # 0th mount point for NFS
+N1=${N1:=/mnt/nfs/1};         # 1st mount point for NFS
+V0=${V0:=patchy};             # volume name to use in tests
+V1=${V1:=patchy1};            # volume name to use in tests
+B0=${B0:=/d/backends};        # top level of brick directories
+H0=${H0:=`hostname --fqdn`};  # hostname
+DEBUG=${DEBUG:=0}             # turn on debugging?
+statedumpdir=`gluster --print-statedumpdir`; # Default directory for statedump
+
+CLI="gluster --mode=script";
+
+mkdir -p $B0;
+mkdir -p $M0 $M1;
+mkdir -p $N0 $N1;
+
+testcnt=`egrep '^[[:space:]]*(EXPECT|TEST|EXPECT_WITHIN|EXPECT_KEYWORD)[[:space:]]' $0 | wc -l`
+expect_tests=`egrep '^[[:space:]]*TESTS_EXPECTED_IN_LOOP[[:space:]]*' $0`
+
+x_ifs=$IFS
+IFS=$'\n'
+for line in $expect_tests; do
+	expect_tests=`echo $line | cut -f 2 -d =`
+	testcnt=`expr $testcnt + $expect_tests`
+done
+IFS=$x_ifs
+
+echo 1..$testcnt
+
+t=1
+
+function dbg()
+{
+	[ "x$DEBUG" = "x0" ] || echo "$*" >&2;
+}
+
+
+function test_header()
+{
+	dbg "=========================";
+	dbg "TEST $t (line $TESTLINE): $*";
+}
+
+
+function test_footer()
+{
+	RET=$?
+        local err=$1
+
+	if [ $RET -eq 0 ]; then
+	     	echo "ok $t";
+	else
+                echo "not ok $t $err";
+		if [ "$EXIT_EARLY" = "1" ]; then
+			exit $RET
+		fi
+	fi
+
+	dbg "RESULT $t: $RET";
+
+	t=`expr $t + 1`;
+}
+
+function test_expect_footer()
+{
+        local e=$1
+        local a=$2
+        local err=""
+
+        if [ "x${e}" != "x${a}" ]; then
+                err="Got \"$a\" instead of \"$e\""
+        fi
+        [[ "x${e}" == "x${a}" ]];
+        test_footer "$err";
+}
+
+function _EXPECT()
+{
+	TESTLINE=$1;
+	shift;
+        local a=""
+
+	test_header "$@";
+
+	e="$1";
+	shift;
+	a=$("$@" | tail -1)
+
+	test_expect_footer "$e" "$a";
+}
+
+function _EXPECT_KEYWORD()
+{
+	TESTLINE=$1;
+	shift;
+
+	test_header "$@";
+
+	e="$1";
+	shift;
+	"$@" | tail -1 | grep -q "$e"
+
+	test_footer;
+}
+
+
+function _TEST()
+{
+	TESTLINE=$1;
+	shift;
+        local redirect=""
+
+	test_header "$@";
+
+        if [ "$1" = "!" ]; then
+                redirect="2>&1"
+        fi
+
+	eval "$@" >/dev/null $redirect
+
+	test_footer;
+}
+
+function _EXPECT_WITHIN()
+{
+        TESTLINE=$1
+        shift;
+
+        local timeout=$1
+        shift;
+
+        test_header "$@"
+
+        e=$1;
+        shift;
+
+        local endtime=$(( ${timeout}+`date +%s` ))
+
+        local success=0
+        while [ `date +%s` -lt $endtime ]; do
+                ("$@") | tail -1 | egrep -q "^${e}\$"
+
+                local pipestatus=(${PIPESTATUS[@]})
+
+                ## Check command success
+                if [ ${pipestatus[0]} -ne 0 ]; then
+                        break;
+                fi
+
+                ## Check match success
+                if [ ${pipestatus[2]} -eq 0 ]; then
+                        success=1;
+                        break;
+                fi
+                sleep 1;
+        done
+
+        if [ $success -eq 1 ]; then
+                true;
+        else
+                false;
+        fi
+
+        test_footer;
+}
+
+
+function SKIP_TESTS()
+{
+        dbg "Skipping tests $t-$testcnt";
+        while [ $t -le $testcnt ]; do
+                true ; test_footer;
+        done
+}
+
+
+function _TEST_IN_LOOP()
+{
+	testcnt=`expr $testcnt + 1`;
+	_TEST $@
+}
+
+
+function cleanup()
+{
+	killall -15 glusterfs glusterfsd glusterd 2>/dev/null || true;
+	killall -9 glusterfs glusterfsd glusterd 2>/dev/null || true;
+
+        type cleanup_lvm &>/dev/null && cleanup_lvm
+
+        MOUNTPOINTS=`mount | grep "$B0/" | awk '{print $3}'`
+        for m in $MOUNTPOINTS;
+        do
+                umount $m
+        done
+
+        LOOPDEVICES=`losetup -a | grep "$B0/" | awk '{print $1}' | tr -d :`
+        for l in $LOOPDEVICES;
+        do
+                losetup -d $l
+        done
+
+
+	rm -rf /var/lib/glusterd/* $B0/* /etc/glusterd/*;
+
+	umount -l $M0 2>/dev/null || true;
+	umount -l $M1 2>/dev/null || true;
+	umount -l $N0 2>/dev/null || true;
+	umount -l $N1 2>/dev/null || true;
+
+}
+
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+function cleanup_tester ()
+{
+    local exe=$1
+    rm -f $exe
+}
+
+function build_tester ()
+{
+    local cfile=$1
+    local fname=$(basename "$cfile")
+    local ext="${fname##*.}"
+    local execname="${fname%.*}"
+    gcc -g -o $(dirname $cfile)/$execname $cfile
+}
+
+function process_leak_count ()
+{
+    local pid=$1;
+    return $(ls -lh /proc/$pid/fd | grep "(deleted)"| wc -l)
+}
+
+alias EXPECT='_EXPECT $LINENO'
+alias TEST='_TEST $LINENO'
+alias EXPECT_WITHIN='_EXPECT_WITHIN $LINENO'
+alias EXPECT_KEYWORD='_EXPECT_KEYWORD $LINENO'
+alias TEST_IN_LOOP='_TEST_IN_LOOP $LINENO'
+shopt -s expand_aliases
diff --git a/tests/nfs.rc b/tests/nfs.rc
new file mode 100644
index 000000000..f3abee842
--- /dev/null
+++ b/tests/nfs.rc
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+
+# Due to portmap registration NFS takes some time to
+# export all volumes. Therefore tests should start only
+# after exports are visible by showmount command. This
+# routine will check if showmount shows the exports or not
+#
+function is_nfs_export_available ()
+{
+        vol=$1
+
+        if [ "$vol" == "" ]; then
+                vol=$V0
+        fi
+
+        exp=$(showmount -e 2> /dev/null | grep $vol | wc -l)
+        echo "$exp"
+}
+
+
diff --git a/tests/performance/open-behind.t b/tests/performance/open-behind.t
new file mode 100755
index 000000000..032154a20
--- /dev/null
+++ b/tests/performance/open-behind.t
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+
+TEST $CLI volume start $V0;
+
+## Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+
+TEST glusterfs -s $H0 --volfile-id $V0 $M1;
+
+D0="hello-this-is-a-test-message0";
+F0="test-file0";
+
+function write_to()
+{
+    local file="$1";
+    local data="$2";
+
+    echo "$data" > "$file";
+}
+
+
+TEST write_to "$M0/$F0" "$D0";
+EXPECT "$D0" cat $M1/$F0;
+
+# open-behind delays open and uses anonymous fds for fops like
+# fstat and readv. So after creating the file, if volume is restarted
+# then later when the file is read, because of the use of anonymous fds
+# volume top open will show number of files opened as 0.
+TEST $CLI volume stop $V0;
+sleep 1;
+TEST $CLI volume start $V0;
+
+sleep 2;
+cat $M1/$F0 >/dev/null;
+
+string=$(gluster volume top $V0 open | grep -w "$F0");
+
+EXPECT "" echo $string;
+
+TEST $CLI volume set $V0 performance.open-behind off;
+
+D1="hello-this-is-a-test-message1";
+F1="test-file1";
+
+TEST write_to "$M0/$F1" "$D1";
+EXPECT "$D1" cat $M1/$F1;
+
+EXPECT "$D0" cat $M1/$F0;
+
+gluster volume top $V0 open | grep -w "$F0" >/dev/null 2>&1
+TEST [ $? -eq 0 ];
+
+cleanup;
diff --git a/tests/performance/quick-read.t b/tests/performance/quick-read.t
new file mode 100644
index 000000000..082998e43
--- /dev/null
+++ b/tests/performance/quick-read.t
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+
+TEST $CLI volume start $V0;
+
+## Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0;
+
+TEST glusterfs -s $H0 --volfile-id $V0 $M1;
+
+D0="hello-this-is-a-test-message0";
+F0="test-file0";
+
+function write_to()
+{
+    local file="$1";
+    local data="$2";
+
+    echo "$data" > "$file";
+}
+
+
+TEST write_to "$M0/$F0" "$D0";
+EXPECT "$D0" cat $M1/$F0;
+EXPECT "$D0" cat $M0/$F0;
+EXPECT "$D0" cat $M1/$F0;
+EXPECT "$D0" cat $M0/$F0;
+
+sleep 1;
+
+EXPECT "$D0" cat $M1/$F0;
+EXPECT "$D0" cat $M0/$F0;
+EXPECT "$D0" cat $M1/$F0;
+EXPECT "$D0" cat $M0/$F0;
+
+TEST $CLI volume set $V0 performance.quick-read off;
+
+D1="hello-this-is-a-test-message1";
+F1="test-file1";
+
+TEST write_to "$M0/$F1" "$D1";
+EXPECT "$D1" cat $M0/$F1;
+
+EXPECT "$D0" cat $M1/$F0;
+
+cleanup;
diff --git a/tests/snapshot.rc b/tests/snapshot.rc
new file mode 100755
index 000000000..a5b86f674
--- /dev/null
+++ b/tests/snapshot.rc
@@ -0,0 +1,251 @@
+#!/bin/bash
+
+LVM_DEFINED=0
+LVM_PREFIX="patchy_snap"
+LVM_COUNT=0
+VHD_SIZE="1G"
+
+function init_lvm() {
+    if [ "$1" == "" ]; then
+        echo "Error: Invalid argument supplied"
+        return 1
+    fi
+    LVM_COUNT=$1
+
+    if [ "$2" != "" ]; then
+        VHD_SIZE=$2
+    fi
+
+    local b
+    local i
+
+    if [ "$B1" = "" ]; then
+        B1=$B0
+    fi
+
+    for i in `seq 1 $LVM_COUNT`; do
+        b="B$i"
+        if [ "${!b}" = "" ]; then
+            echo "Error: $b not defined."
+            echo "Please run launch_cluster with atleast $LVM_COUNT nodes"
+            return 1
+        fi
+
+        eval "L$i=${!b}/${LVM_PREFIX}_mnt"
+        l="L$i"
+        mkdir -p ${!l}
+        if [ $? -ne 0 ]; then
+            echo "Error: failed to create dir ${!l}"
+            return 1
+        fi
+
+        eval "VG$i=${LVM_PREFIX}_vg_${i}"
+    done
+
+    LVM_DEFINED=1
+    return 0
+}
+
+function setup_lvm() {
+    init_lvm $@ || return 1
+    _setup_lvm
+    return 0
+}
+
+function cleanup_lvm() {
+    pkill gluster
+    sleep 2
+
+    if [ "$LVM_DEFINED" = "1" ]; then
+        _cleanup_lvm
+    fi
+
+    _cleanup_lvm_again >/dev/null 2>&1
+    return 0
+}
+
+########################################################
+# Private Functions
+########################################################
+function _setup_lvm() {
+    local count=$LVM_COUNT
+    local b
+    local i
+
+    for i in `seq 1 $count`; do
+        b="B$i"
+
+        _create_vhd ${!b} $i
+        _create_lv ${!b} $i
+        _mount_lv $i
+    done
+}
+
+function _cleanup_lvm() {
+    local count=$LVM_COUNT
+    local b
+    local i
+
+    for i in `seq 1 $count`; do
+        b="B$i"
+        _umount_lv $i
+        _remove_lv $i
+        _remove_vhd ${!b}
+    done
+}
+
+function _cleanup_lvm_again() {
+    local file
+
+    mount | grep $LVM_PREFIX | awk '{print $3}' | xargs -r umount -f
+
+    /sbin/vgs | grep $LVM_PREFIX | awk '{print $1}' | xargs -r vgremove -f
+
+    find $B0 -name "${LVM_PREFIX}_loop" | xargs -r losetup -d
+
+    find $B0 -name "${LVM_PREFIX}*" | xargs -r rm -rf
+
+    find /run/gluster/snaps -name "*${LVM_PREFIX}*" | xargs -r rm -rf
+
+    for file in `ls /run/gluster/snaps`; do
+        find /run/gluster/snaps/$file -mmin -2 | xargs -r rm -rf
+    done
+}
+
+########################################################
+########################################################
+function _create_vhd() {
+    local dir=$1
+    local num=$2
+    local loop_num=`expr $2 + 8`
+
+    fallocate -l${VHD_SIZE} $dir/${LVM_PREFIX}_vhd
+    mknod -m660 $dir/${LVM_PREFIX}_loop b 7 $loop_num
+    /sbin/losetup $dir/${LVM_PREFIX}_loop $dir/${LVM_PREFIX}_vhd
+}
+
+function _create_lv() {
+    local dir=$1
+    local num=$2
+    local vg="VG$num"
+
+    /sbin/pvcreate $dir/${LVM_PREFIX}_loop
+    /sbin/vgcreate ${!vg} $dir/${LVM_PREFIX}_loop
+
+    /sbin/lvcreate -l 100%FREE -T /dev/${!vg}/thinpool
+    /sbin/lvcreate -V $VHD_SIZE -T /dev/${!vg}/thinpool -n brick_lvm
+
+    mkfs.xfs -f /dev/${!vg}/brick_lvm
+}
+
+function _mount_lv() {
+    local num=$1
+    local vg="VG$num"
+    local l="L$num"
+
+    mount -t xfs -o nouuid /dev/${!vg}/brick_lvm ${!l}
+}
+
+function _umount_lv() {
+    local num=$1
+    local l="L$num"
+
+    umount -f ${!l} 2>/dev/null || true
+    rmdir ${!l} 2>/dev/null || true
+}
+
+function _remove_lv() {
+    local num=$1
+    local vg="VG$num"
+
+    vgremove -f ${!vg}
+}
+
+function _remove_vhd() {
+    local dir=$1
+
+    losetup -d $dir/${LVM_PREFIX}_loop
+    rm -f $dir/${LVM_PREFIX}_loop
+    rm -f $dir/${LVM_PREFIX}_vhd
+}
+
+########################################################
+# Utility Functions
+########################################################
+function snapshot_exists() {
+        local volname=$1
+        local snapname=$2
+        $CLI snapshot list $volname | egrep -q "^$snapname\$"
+        return $?
+}
+
+#Create N number of snaps in a given volume
+#Arg1 : <Volume Name>
+#Arg2 : <Count of snaps to be created>
+#Arg3 : <Snap Name Pattern>
+#Return: Returns 0 if all snaps are created ,
+#        if not will return exit code of last failed
+#        snap create command.
+function create_n_snapshots() {
+        local vol=$1
+        local snap_count=$2
+        local snap_name=$3
+        local ret=0
+        for i in `seq 1 $snap_count`; do
+                $CLI_1 snapshot create $snap_name$i ${vol}&
+                PID_1=$!
+                wait $PID_1
+                ret=$?
+                if [ "$ret" != "0" ]; then
+                        break
+                fi
+         done
+         return $ret
+}
+
+
+#Delete N number of snaps in a given volume
+#Arg1 : <Volume Name>
+#Arg2 : <Count of snaps to be deleted>
+#Arg3 : <Snap Name Pattern>
+#Return: Returns 0 if all snaps are Delete,
+#        if not will return exit code of last failed
+#        snap delete command.
+function delete_n_snapshots() {
+        local vol=$1
+        local snap_count=$2
+        local snap_name=$3
+        local ret=0
+        for i in `seq 1 $snap_count`; do
+                $CLI_1 snapshot delete $snap_name$i &
+                PID_1=$!
+                wait $PID_1
+                temp=$?
+                if [ "$temp" != "0" ]; then
+                        ret=$temp
+                fi
+         done
+         return $ret
+}
+
+#Check for the existance of N number of snaps in a given volume
+#Arg1 : <Volume Name>
+#Arg2 : <Count of snaps to be checked>
+#Arg3 : <Snap Name Pattern>
+#Return: Returns 0 if all snaps exists,
+#        if not will return exit code of last failed
+#        snapshot_exists().
+function snapshot_n_exists() {
+        local vol=$1
+        local snap_count=$2
+        local snap_name=$3
+        local ret=0
+        for i in `seq 1 $snap_count`; do
+                snapshot_exists $snap_name$i
+                ret=$?
+                if [ "$ret" != "0" ]; then
+                        break
+                fi
+         done
+         return $ret
+}
diff --git a/tests/utils/create-files.py b/tests/utils/create-files.py
new file mode 100755
index 000000000..0d937eff9
--- /dev/null
+++ b/tests/utils/create-files.py
@@ -0,0 +1,207 @@
+#!/usr/bin/python
+
+# This script was developed by Vijaykumar Koppad (vkoppad@redhat.com)
+# The latest version of this script can found at
+# http://github.com/vijaykumar-koppad/crefi
+
+from __future__ import with_statement
+import sys
+import os
+import re
+import random
+from optparse import OptionParser
+import time
+import string
+import errno
+
+def os_rd(src, size):
+    fd = os.open(src,os.O_RDONLY)
+    data = os.read(fd, size)
+    os.close(fd)
+    return data
+
+def os_wr(dest, data):
+    fd = os.open(dest,os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0644)
+    os.write(fd, data)
+    os.close(fd)
+    return
+
+def create_sparse_file(fil):
+    if option.size:
+        option.random = False
+        size = option.size
+    else:
+        size = random.randint(option.min, option.max)
+    data = os_rd("/dev/zero", size)
+    os_wr(fil, data)
+    return
+
+def create_binary_file(fil):
+    if option.size:
+        option.random = False
+        size = option.size
+    else:
+        size = random.randint(option.min, option.max)
+    data = os_rd("/dev/urandom", size)
+    os_wr(fil, data)
+    return
+
+def create_txt_file(fil):
+    if option.size:
+        option.random = False
+        size = option.size
+    else:
+        size = random.randint(option.min, option.max)
+    if size < 500*1024:
+        data = os_rd("/etc/services", size)
+        os_wr(fil, data)
+    else:
+        data = os_rd("/etc/services", 500*1024)
+        file_size = 0
+        fd = os.open(fil,os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0644)
+        while file_size < size:
+            os.write(fd, data)
+            file_size += 500*1024
+        os.close(fd)
+    return
+
+def get_filename():
+    size = option.flen
+    char = string.uppercase+string.digits
+    st = ''.join(random.choice(char) for i in range(size))
+    ti = str((hex(int(str(time.time()).split('.')[0])))[2:])
+    return ti+"~~"+st
+
+def text_files(files, file_count):
+    for k in range(files):
+        if not file_count%option.inter:
+            print file_count
+        fil = get_filename()
+        create_txt_file(fil)
+        file_count += 1
+    return file_count
+
+def sparse_files(files, file_count):
+    for k in range(files):
+        if not file_count%option.inter:
+            print file_count
+        fil = get_filename()
+        create_sparse_file(fil)
+        file_count += 1
+    return file_count
+
+def binary_files(files, file_count):
+    for k in range(files):
+        if not file_count%option.inter:
+            print file_count
+        fil = get_filename()
+        create_binary_file(fil)
+        file_count += 1
+    return file_count
+
+def human2bytes(size):
+    size_short = {
+        1024 : ['K','KB','KiB','k','kB','kiB'],
+        1024*1024 : ['M','MB','MiB'],
+        1024*1024*1024 : ['G','GB','GiB']
+}
+    num = re.search('(\d+)',size).group()
+    ext = size[len(num):]
+    num = int(num)
+    if ext == '':
+        return num
+    for value, keys in size_short.items():
+        if ext in keys:
+            size = num*value
+            return size
+
+def multipledir(mnt_pnt,brdth,depth,files):
+    files_count = 1
+    for i in range(brdth):
+        breadth = mnt_pnt+"/"+str(i)
+        try:
+           os.makedirs(breadth)
+        except OSError as ex:
+            if not ex.errno is errno.EEXIST:
+                raise
+        os.chdir(breadth)
+        dir_depth = breadth
+        print breadth
+        for j in range(depth):
+            dir_depth = dir_depth+"/"+str(j)
+            try:
+                os.makedirs(dir_depth)
+            except OSError as ex:
+                if not ex.errno is errno.EEXIST:
+                    raise
+            os.chdir(dir_depth)
+            if option.file_type == "text":
+                files_count = text_files(files, files_count)
+            elif option.file_type == "sparse":
+                files_count = sparse_files(files, files_count)
+            elif option.file_type == "binary":
+                files_count = binary_files(files, files_count)
+            else:
+                print "Not a valid file type"
+                sys.exit(1)
+
+def singledir(mnt_pnt, files):
+    files_count = 1
+    os.chdir(mnt_pnt)
+    if option.file_type == "text":
+        files_count = text_files(files, files_count)
+    elif option.file_type == "sparse":
+        files_count = sparse_files(files, files_count)
+    elif option.file_type == "binary":
+        files_count = binary_files(files, files_count)
+    else:
+        print "Not a valid file type"
+        sys.exit(1)
+
+if __name__ == '__main__':
+    usage = "usage: %prog [option] <MNT_PT>"
+    parser = OptionParser(usage=usage)
+    parser.add_option("-n", dest="files",type="int" ,default=100,
+                      help="number of files in each level [default: %default]")
+    parser.add_option("--size", action = "store",type="string",
+                      help="size of the files to be used")
+    parser.add_option("--random",  action="store_true", default=True,
+                      help="random size of the file between --min and --max "
+                      "[default: %default]")
+    parser.add_option("--max", action = "store",type="string", default="500K",
+                      help="maximum size of the files, if random is True "
+                      "[default: %default]")
+    parser.add_option("--min", action = "store",type="string", default="10K",
+                      help="minimum size of the files, if random is True "
+                      "[default: %default]" )
+    parser.add_option("--single", action="store_true", dest="dir",default=True,
+                      help="create files in single directory [default: %default]" )
+    parser.add_option("--multi", action="store_false", dest="dir",
+                      help="create files in multiple directories")
+    parser.add_option("-b", dest="brdth",type="int",default=5,
+                      help="number of directories in one level(works with --multi)[default: %default]")
+    parser.add_option("-d", dest="depth",type="int",default=5,
+                      help="number of levels of directories(works with --multi)[default: %default]")
+    parser.add_option("-l", dest="flen",type="int" ,default=10,
+                      help="number of bytes for filename "
+                      "[default: %default]")
+    parser.add_option("-t","--type", action="store", type="string" , dest="file_type",default="text",
+                      help="type of the file to be created (text, sparse, binary) [default: %default]" )
+    parser.add_option("-I", dest="inter", type="int", default=100,
+                      help="print number files created of interval [defailt: %dafault]")
+    (option,args) = parser.parse_args()
+    if not args:
+        print "usage: <script> [option] <MNT_PT>"
+        print ""
+        sys.exit(1)
+    args[0] = os.path.abspath(args[0])
+    if option.size:
+        option.size = human2bytes(option.size)
+    else:
+        option.max = human2bytes(option.max)
+        option.min = human2bytes(option.min)
+    if option.dir:
+        singledir(args[0], option.files)
+    else:
+        multipledir(args[0], option.brdth, option.depth, option.files)
+    print "creation of files completed.\n"
diff --git a/tests/volume.rc b/tests/volume.rc
new file mode 100644
index 000000000..171f8d709
--- /dev/null
+++ b/tests/volume.rc
@@ -0,0 +1,325 @@
+function volinfo_field()
+{
+    local vol=$1;
+    local field=$2;
+
+    $CLI volume info $vol | grep "^$field: " | sed 's/.*: //';
+}
+
+
+function brick_count()
+{
+    local vol=$1;
+
+    $CLI volume info $vol | egrep "^Brick[0-9]+: " | wc -l;
+}
+
+function online_brick_count ()
+{
+    pidof glusterfsd | wc -w
+}
+
+function volume_option()
+{
+        local vol=$1
+        local key=$2
+        $CLI volume info $vol | egrep "^$key: " | cut -f2 -d' ';
+}
+
+function rebalance_status_field {
+        #The rebalance status can be upto 3 words, (ex:'fix-layout in progress'), hence the awk-print $7 thru $9.
+        #But if the status is less than 3 words, it also prints the next field i.e the run_time_in_secs.(ex:'completed 3.00').
+        #So we trim the numbers out with  `tr`. Finally remove the trailing white spaces with sed. What we get is one of the
+        #strings in the 'cli_vol_task_status_str' char array of cli-rpc-ops.c
+
+        $CLI volume rebalance $1 status | awk '{print $7,$8,$9}' |sed -n 3p |tr -d '[^0-9+\.]'|sed 's/ *$//g'
+}
+
+function remove_brick_status_completed_field {
+        local vol=$1
+        local brick_list=$2
+        $CLI volume remove-brick $vol $brick_list status | awk '{print $7}' | sed -n 3p
+}
+
+function get_mount_process_pid {
+        local vol=$1
+        ps aux | grep glusterfs | grep -E "volfile-id[ =]/?$vol " | awk '{print $2}' | head -1
+}
+
+function get_nfs_pid ()
+{
+        ps aux | grep "volfile-id\ gluster\/nfs" | awk '{print $2}' | head -1
+}
+
+function read_nfs_pidfile ()
+{
+        echo `cat /var/lib/glusterd/nfs/run/nfs.pid`
+}
+
+function cleanup_statedump {
+       pid=$1
+       rm -f $statedumpdir/*$pid.dump.*
+       #.vimrc friendly comment */
+}
+
+function generate_statedump {
+        local fpath=""
+        pid=$1
+        #remove old stale statedumps
+        cleanup_statedump $pid
+        kill -USR1 $pid
+        #Wait till the statedump is generated
+        sleep 1
+        fname=$(ls $statedumpdir | grep -E "\.$pid\.dump\.")
+        echo $statedumpdir/$fname
+}
+
+function generate_mount_statedump {
+        local vol=$1
+        generate_statedump $(get_mount_process_pid $vol)
+}
+
+function cleanup_mount_statedump {
+        local vol=$1
+        cleanup_statedump $(get_mount_process_pid $vol)
+}
+
+function _afr_child_up_status {
+        local vol=$1
+        #brick_id is (brick-num in volume info - 1)
+        local brick_id=$2
+        local gen_state_dump=$3
+        local fpath=$($gen_state_dump $vol)
+        up=$(grep -B1 trusted.afr.$vol-client-$brick_id $fpath | head -1 | cut -f2 -d'=')
+        rm -f $fpath
+        echo "$up"
+}
+
+function afr_child_up_status {
+        local vol=$1
+        #brick_id is (brick-num in volume info - 1)
+        local brick_id=$2
+        _afr_child_up_status $vol $brick_id generate_mount_statedump
+}
+
+function get_shd_process_pid {
+        local vol=$1
+        ps aux | grep glusterfs | grep -E "glustershd/run/glustershd.pid" | awk '{print $2}' | head -1
+}
+
+function generate_shd_statedump {
+        local vol=$1
+        generate_statedump $(get_shd_process_pid $vol)
+}
+
+function generate_nfs_statedump {
+        local vol=$1
+        generate_statedump $(get_nfs_pid $vol)
+}
+
+function generate_brick_statedump {
+        local vol=$1
+        local host=$2
+        local brick=$3
+        generate_statedump $(get_brick_pid $vol $host $brick)
+}
+
+function afr_child_up_status_in_shd {
+        local vol=$1
+        #brick_id is (brick-num in volume info - 1)
+        local brick_id=$2
+        _afr_child_up_status $vol $brick_id generate_shd_statedump
+}
+
+function afr_child_up_status_in_nfs {
+        local vol=$1
+        #brick_id is (brick-num in volume info - 1)
+        local brick_id=$2
+        _afr_child_up_status $vol $brick_id generate_nfs_statedump
+}
+
+function nfs_up_status {
+        gluster volume status | grep "NFS Server" | awk '{print $6}'
+}
+
+function glustershd_up_status {
+        gluster volume status | grep "Self-heal Daemon" | awk '{print $6}'
+}
+
+function get_brick_pid {
+        local vol=$1
+        local host=$2
+        local brick=$3
+        local brick_hiphenated=$(echo $brick | tr '/' '-')
+        echo `cat /var/lib/glusterd/vols/$vol/run/${host}${brick_hiphenated}.pid`
+}
+
+function kill_brick {
+        local vol=$1
+        local host=$2
+        local brick=$3
+        kill -9 $(get_brick_pid $vol $host $brick)
+}
+
+function check_option_help_presence {
+        local option=$1
+        $CLI volume set help | grep "^Option:" | grep -w $option
+}
+
+function afr_get_changelog_xattr {
+        local file=$1
+        local xkey=$2
+        getfattr -n $xkey -e hex $file 2>/dev/null | grep "client-" | cut -f2 -d'='
+}
+
+function afr_get_pending_heal_count {
+        local vol=$1
+        gluster volume heal $vol info | grep "Number of entries" | awk '{ sum+=$4} END {print sum}'
+}
+
+function afr_get_index_path {
+        local brick_path=$1
+        echo "$brick_path/.glusterfs/indices/xattrop"
+}
+
+function afr_get_num_indices_in_brick {
+        local brick_path=$1
+        echo $(ls $(afr_get_index_path $brick_path) | grep -v xattrop | wc -l)
+}
+
+function gf_get_gfid_xattr {
+        file=$1
+        getfattr -n trusted.gfid -e hex $file 2>/dev/null | grep "trusted.gfid" | cut -f2 -d'='
+}
+
+function gf_gfid_xattr_to_str {
+        xval=$1
+        echo "${xval:2:8}-${xval:10:4}-${xval:14:4}-${xval:18:4}-${xval:22:12}"
+}
+
+function gf_check_file_opened_in_brick {
+        vol=$1
+        host=$2
+        brick=$3
+        realpath=$4
+        ls -l /proc/$(get_brick_pid $vol $host $brick)/fd | grep "${realpath}$" 2>&1 > /dev/null
+        if [ $? -eq 0 ]; then
+                echo "Y"
+        else
+                echo "N"
+        fi
+}
+
+function gf_get_gfid_backend_file_path {
+        brickpath=$1
+        filepath_in_brick=$2
+        gfid=$(gf_get_gfid_xattr "$brickpath/$filepath_in_brick")
+        gfidstr=$(gf_gfid_xattr_to_str $gfid)
+        echo "$brickpath/.glusterfs/${gfidstr:0:2}/${gfidstr:2:2}/$gfidstr"
+}
+
+function gf_rm_file_and_gfid_link {
+        brickpath=$1
+        filepath_in_brick=$2
+        rm -f $(gf_get_gfid_backend_file_path $brickpath $filepath_in_brick)
+        rm -f "$brickpath/$filepath_in_brick"
+}
+
+
+function gd_is_replace_brick_completed {
+        local host=$1
+        local vol=$2
+        local src_brick=$3
+        local dst_brick=$4
+        $CLI volume replace-brick $vol $src_brick $dst_brick status | grep -i "Migration complete"
+        if [ $? -eq 0 ]; then
+                echo "Y"
+        else
+                echo "N"
+        fi
+}
+
+function dht_get_layout {
+	local my_xa=trusted.glusterfs.dht
+	getfattr -d -e hex -n $my_xa $1 2> /dev/null | grep "$my_xa=" | cut -d= -f2
+}
+
+function afr_get_specific_changelog_xattr ()
+{
+        local path=$1
+        local key=$2
+        local type=$3
+        local specific_changelog=""
+
+        changelog_xattr=$(afr_get_changelog_xattr "$path" "$key")
+        if [ "$type" == "data" ]; then
+                specific_changelog=${changelog_xattr:2:8}
+        elif [ "$type" == "metadata" ]; then
+                specific_changelog=${changelog_xattr:10:8}
+        elif [ "$type" == "entry" ]; then
+                specific_changelog=${changelog_xattr:18:8}
+        else
+                specific_changlog="error"
+        fi
+
+        echo $specific_changelog
+}
+##
+ # query pathinfo xattr and extract POSIX pathname(s)
+ ##
+function get_backend_paths {
+       local path=$1
+
+       getfattr -m . -n trusted.glusterfs.pathinfo $path | tr ' ' '\n' | sed -n 's/<POSIX.*:.*:\(.*\)>.*/\1/p'
+}
+
+function do_volume_operations() {
+        local operation=$1
+        local count=$2
+        local force=$3
+
+        local pids=()
+        local cli
+        local v
+
+         for i in `seq 1 $count`; do
+                cli="CLI_$i"
+                v="V`expr $i - 1`"
+                ${!cli} volume $operation ${!v} $force &
+                pids[$i]=$!
+        done
+
+        for i in `seq 1 $count`; do
+                wait ${pids[$i]}
+        done
+}
+
+function start_volumes() {
+        do_volume_operations start $1
+}
+
+function stop_volumes() {
+        do_volume_operations stop $1
+}
+
+function start_force_volumes() {
+        do_volume_operations start $1 force
+}
+
+function stop_force_volumes() {
+        do_volume_operations stop $1 force
+}
+
+function delete_volumes() {
+        do_volume_operations delete $1
+}
+
+function volume_exists() {
+        local volname=$1
+        $CLI volume info $volname 2>&1 | grep -q 'does not exist'
+        if [ $? -eq 0 ]; then
+                return 1
+        else
+                return 0
+        fi
+}
diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index b1643d26c..f60fa85ce 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system
+SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system \
+          playground
 
 CLEANFILES =
diff --git a/xlators/bindings/python/src/Makefile.am b/xlators/bindings/python/src/Makefile.am
index c0b9141c6..90370d861 100644
--- a/xlators/bindings/python/src/Makefile.am
+++ b/xlators/bindings/python/src/Makefile.am
@@ -9,7 +9,7 @@ pythondir = $(xlatordir)/python
 
 python_so_SOURCES = python.c
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
+AM_CFLAGS = -fPIC $(GF_CPPFLAGS) -Wall \
 	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
 	$(PYTHON_CPPLAGS) -DGLUSTER_PYTHON_PATH=\"$(pythondir)\"
 
diff --git a/xlators/bindings/python/src/gluster.py b/xlators/bindings/python/src/gluster.py
index ee0eb1310..337c983ec 100644
--- a/xlators/bindings/python/src/gluster.py
+++ b/xlators/bindings/python/src/gluster.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
+
+#  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
 from ctypes import *
 from glustertypes import *
 from glusterstack import *
diff --git a/xlators/bindings/python/src/glusterstack.py b/xlators/bindings/python/src/glusterstack.py
index ba24c8165..0c071ae98 100644
--- a/xlators/bindings/python/src/glusterstack.py
+++ b/xlators/bindings/python/src/glusterstack.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
+
+#  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
 from ctypes import *
 from glustertypes import *
 
diff --git a/xlators/bindings/python/src/glustertypes.py b/xlators/bindings/python/src/glustertypes.py
index e9069d07c..98437d22e 100644
--- a/xlators/bindings/python/src/glustertypes.py
+++ b/xlators/bindings/python/src/glustertypes.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
+
+#  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
 from ctypes import *
 import collections
 
diff --git a/xlators/bindings/python/src/python.c b/xlators/bindings/python/src/python.c
index 3310a2115..9b96790de 100644
--- a/xlators/bindings/python/src/python.c
+++ b/xlators/bindings/python/src/python.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2007-2010 Chris AtLee <chris@atlee.ca>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #include <Python.h>
 
 #ifndef _CONFIG_H
@@ -45,7 +35,7 @@ python_writev (call_frame_t *frame,
               xlator_t *this,
               fd_t *fd,
               struct iovec *vector,
-              int32_t count, 
+              int32_t count,
               off_t offset)
 {
   python_private_t *priv = (python_private_t *)this->private;
@@ -148,7 +138,7 @@ init (xlator_t *this)
   Py_InitializeEx(0);
 
   if (!this->children) {
-    gf_log ("python", GF_LOG_ERROR, 
+    gf_log ("python", GF_LOG_ERROR,
             "FATAL: python should have exactly one child");
     return -1;
   }
@@ -166,7 +156,7 @@ init (xlator_t *this)
   }
 
   priv->pInterp = Py_NewInterpreter();
-    
+
   // Adjust python's path
   PyObject *syspath = PySys_GetObject("path");
   PyObject *path = PyString_FromString(GLUSTER_PYTHON_PATH);
@@ -188,7 +178,7 @@ init (xlator_t *this)
   priv->pVectorType = PyObject_GetAttrString(priv->pGlusterModule, "iovec");
 
   gf_log("python", GF_LOG_DEBUG, "Loading script...%s", priv->scriptname);
-  
+
   priv->pScriptModule = AnonModule_FromFile(priv->scriptname);
   if (!priv->pScriptModule || PyErr_Occurred())
   {
@@ -217,7 +207,7 @@ init (xlator_t *this)
   return 0;
 }
 
-void 
+void
 fini (xlator_t *this)
 {
   python_private_t *priv = (python_private_t*)(this->private);
diff --git a/xlators/bindings/python/src/testxlator.py b/xlators/bindings/python/src/testxlator.py
index 507455c85..59a991dca 100644
--- a/xlators/bindings/python/src/testxlator.py
+++ b/xlators/bindings/python/src/testxlator.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+"""
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+"""
 
 """
 This is a test translator written in python.
diff --git a/xlators/cluster/afr/src/Makefile.am b/xlators/cluster/afr/src/Makefile.am
index 16ed25af1..35d18a6c0 100644
--- a/xlators/cluster/afr/src/Makefile.am
+++ b/xlators/cluster/afr/src/Makefile.am
@@ -1,21 +1,31 @@
 xlator_LTLIBRARIES = afr.la pump.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c afr-inode-write.c afr-open.c afr-transaction.c afr-self-heal-data.c afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c afr-self-heal-algorithm.c afr-lk-common.c afr-self-heald.c $(top_builddir)/xlators/lib/src/libxlator.c
+afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c \
+	afr-inode-write.c afr-open.c afr-transaction.c afr-self-heal-data.c \
+	afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c \
+	afr-self-heal-algorithm.c afr-lk-common.c afr-self-heald.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-afr_la_LDFLAGS = -module -avoidversion
+afr_la_LDFLAGS = -module -avoid-version
 afr_la_SOURCES = $(afr_common_source) afr.c
 afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-pump_la_LDFLAGS = -module -avoidversion
+pump_la_LDFLAGS = -module -avoid-version
 pump_la_SOURCES =  $(afr_common_source) pump.c
 pump_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h afr-self-heal-algorithm.h pump.h afr-mem-types.h afr-common.c afr-self-heald.h $(top_builddir)/xlators/lib/src/libxlator.h $(top_builddir)/glusterfsd/src/glusterfsd.h
+noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h \
+	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h \
+	afr-self-heal-algorithm.h pump.h afr-mem-types.h afr-common.c \
+	afr-self-heald.h $(top_builddir)/xlators/lib/src/libxlator.h \
+	$(top_builddir)/glusterfsd/src/glusterfsd.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/contrib/md5 -shared -nostartfiles $(GF_CFLAGS) \
-	    -I$(top_srcdir)/xlators/lib/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
 
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index a5dee65f6..af01f2ef2 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -58,10 +49,9 @@
 #include "afr-self-heald.h"
 #include "pump.h"
 
-#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000200000000ULL
-#define AFR_ICTX_SPLIT_BRAIN_MASK      0x0000000100000000ULL
+#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000100000000ULL
 #define AFR_ICTX_READ_CHILD_MASK       0x00000000FFFFFFFFULL
-
+#define AFR_STATISTICS_HISTORY_SIZE    50
 int
 afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
                                 gf_boolean_t fail_conflict);
@@ -92,6 +82,11 @@ afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req, const char *path)
                                 path, priv->pending_key[i]);
                 /* 3 = data+metadata+entry */
         }
+        ret = dict_set_int32 (xattr_req, GF_GFIDLESS_LOOKUP, 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s: failed to set gfidless "
+                        "lookup", path);
+        }
 }
 
 int
@@ -123,6 +118,13 @@ afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
                         loc->path, GLUSTERFS_ENTRYLK_COUNT);
         }
 
+        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
+        }
+
         ret = dict_get_ptr (local->xattr_req, "gfid-req", gfid_req);
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -200,60 +202,86 @@ out:
         return ret;
 }
 
-afr_inode_ctx_t*
-afr_inode_ctx_get_from_addr (uint64_t addr, int32_t child_count)
+void
+afr_inode_ctx_destroy (afr_inode_ctx_t *ctx)
 {
-        int             ret  = -1;
-        afr_inode_ctx_t *ctx = NULL;
-        size_t          size = 0;
+        if (!ctx)
+                return;
+        GF_FREE (ctx->fresh_children);
+        GF_FREE (ctx);
+}
 
-        GF_ASSERT (child_count > 0);
+afr_inode_ctx_t*
+__afr_inode_ctx_get (inode_t *inode, xlator_t *this)
+{
+        int             ret      = 0;
+        uint64_t        ctx_addr = 0;
+        afr_inode_ctx_t *ctx     = NULL;
+        afr_private_t   *priv    = NULL;
 
-        if (!addr) {
-                ctx = GF_CALLOC (1, sizeof (*ctx),
-                                 gf_afr_mt_inode_ctx_t);
-                if (!ctx)
-                        goto out;
-                size = sizeof (*ctx->fresh_children);
-                ctx->fresh_children = GF_CALLOC (child_count, size,
-                                                 gf_afr_mt_int32_t);
-                if (!ctx->fresh_children)
-                        goto out;
-        } else {
-                ctx = (afr_inode_ctx_t*) (long) addr;
+        priv = this->private;
+        ret = __inode_ctx_get (inode, this, &ctx_addr);
+        if (ret < 0)
+                ctx_addr = 0;
+        if (ctx_addr != 0) {
+                ctx = (afr_inode_ctx_t*) (long) ctx_addr;
+                goto out;
         }
-        ret = 0;
+        ctx = GF_CALLOC (1, sizeof (*ctx),
+                         gf_afr_mt_inode_ctx_t);
+        if (!ctx)
+                goto fail;
+        ctx->fresh_children = GF_CALLOC (priv->child_count,
+                                         sizeof (*ctx->fresh_children),
+                                         gf_afr_mt_int32_t);
+        if (!ctx->fresh_children)
+                goto fail;
+        ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
+        if (ret) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
+                                  "set the inode ctx (%s)",
+                                  uuid_utoa (inode->gfid));
+                goto fail;
+        }
+
 out:
-        if (ret && ctx) {
-                if (ctx->fresh_children)
-                        GF_FREE (ctx->fresh_children);
-                GF_FREE (ctx);
-                ctx = NULL;
+        return ctx;
+
+fail:
+        afr_inode_ctx_destroy (ctx);
+        return NULL;
+}
+
+afr_inode_ctx_t*
+afr_inode_ctx_get (inode_t *inode, xlator_t *this)
+{
+        afr_inode_ctx_t *ctx = NULL;
+
+        LOCK (&inode->lock);
+        {
+                ctx = __afr_inode_ctx_get (inode, this);
         }
+        UNLOCK (&inode->lock);
         return ctx;
 }
 
 void
-afr_inode_get_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
+afr_inode_get_ctx_params (xlator_t *this, inode_t *inode,
+                          afr_inode_params_t *params)
 {
         GF_ASSERT (inode);
         GF_ASSERT (params);
 
-        int             ret = 0;
         afr_inode_ctx_t *ctx = NULL;
         afr_private_t   *priv = NULL;
         int             i = 0;
-        uint64_t        ctx_addr   = 0;
         int32_t         read_child = -1;
         int32_t         *fresh_children = NULL;
 
         priv = this->private;
         LOCK (&inode->lock);
         {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        goto unlock;
-                ctx = afr_inode_ctx_get_from_addr (ctx_addr, priv->child_count);
+                ctx = __afr_inode_ctx_get (inode, this);
                 if (!ctx)
                         goto unlock;
                 switch (params->op) {
@@ -272,12 +300,6 @@ afr_inode_get_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
                         if (ctx->masks & AFR_ICTX_OPENDIR_DONE_MASK)
                                 params->u.value = _gf_true;
                         break;
-                case AFR_INODE_GET_SPLIT_BRAIN:
-                        params->u.value = _gf_false;
-                        if (ctx->masks & AFR_ICTX_SPLIT_BRAIN_MASK)
-                                params->u.value = _gf_true;
-                        ;
-                        break;
                 default:
                         GF_ASSERT (0);
                         break;
@@ -290,11 +312,16 @@ unlock:
 gf_boolean_t
 afr_is_split_brain (xlator_t *this, inode_t *inode)
 {
-        afr_inode_params_t params = {0};
+        afr_inode_ctx_t *ctx = NULL;
+        gf_boolean_t    spb  = _gf_false;
 
-        params.op = AFR_INODE_GET_SPLIT_BRAIN;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.value;
+        ctx = afr_inode_ctx_get (inode, this);
+        if (!ctx)
+                goto out;
+        if ((ctx->mdata_spb == SPB) || (ctx->data_spb == SPB))
+                spb = _gf_true;
+out:
+        return spb;
 }
 
 gf_boolean_t
@@ -303,11 +330,10 @@ afr_is_opendir_done (xlator_t *this, inode_t *inode)
         afr_inode_params_t params = {0};
 
         params.op = AFR_INODE_GET_OPENDIR_DONE;
-        afr_inode_get_ctx (this, inode, &params);
+        afr_inode_get_ctx_params (this, inode, &params);
         return params.u.value;
 }
 
-
 int32_t
 afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children)
 {
@@ -315,7 +341,7 @@ afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children)
 
         params.op = AFR_INODE_GET_READ_CTX;
         params.u.read_ctx.children = fresh_children;
-        afr_inode_get_ctx (this, inode, &params);
+        afr_inode_get_ctx_params (this, inode, &params);
         return params.u.read_ctx.read_child;
 }
 
@@ -377,31 +403,14 @@ afr_inode_ctx_set_opendir_done (afr_inode_ctx_t *ctx)
 }
 
 void
-afr_inode_ctx_set_splitbrain (afr_inode_ctx_t *ctx, gf_boolean_t set)
-{
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask = 0;
-
-        if (set) {
-                remaining_mask = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx->masks);
-                mask = (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_SPLIT_BRAIN_MASK);
-                ctx->masks = remaining_mask | mask;
-        } else {
-                ctx->masks = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx->masks);
-        }
-}
-
-void
-afr_inode_set_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
+afr_inode_set_ctx_params (xlator_t *this, inode_t *inode,
+                          afr_inode_params_t *params)
 {
         GF_ASSERT (inode);
         GF_ASSERT (params);
 
-        int             ret = 0;
         afr_inode_ctx_t *ctx            = NULL;
         afr_private_t   *priv           = NULL;
-        uint64_t        ctx_addr        = 0;
-        gf_boolean_t    set             = _gf_false;
         int32_t         read_child      = -1;
         int32_t         *fresh_children = NULL;
         int32_t         *stale_children = NULL;
@@ -409,10 +418,7 @@ afr_inode_set_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
         priv = this->private;
         LOCK (&inode->lock);
         {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        ctx_addr = 0;
-                ctx = afr_inode_ctx_get_from_addr (ctx_addr, priv->child_count);
+                ctx = __afr_inode_ctx_get (inode, this);
                 if (!ctx)
                         goto unlock;
                 switch (params->op) {
@@ -432,33 +438,26 @@ afr_inode_set_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
                 case AFR_INODE_SET_OPENDIR_DONE:
                         afr_inode_ctx_set_opendir_done (ctx);
                         break;
-                case AFR_INODE_SET_SPLIT_BRAIN:
-                        set = params->u.value;
-                        afr_inode_ctx_set_splitbrain (ctx, set);
-                        break;
                 default:
                         GF_ASSERT (0);
                         break;
                 }
-                ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
-                if (ret) {
-                        gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
-                                          "set the inode ctx (%s)",
-                                          uuid_utoa (inode->gfid));
-                }
         }
 unlock:
         UNLOCK (&inode->lock);
 }
 
 void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set)
+afr_set_split_brain (xlator_t *this, inode_t *inode, afr_spb_state_t mdata_spb,
+                     afr_spb_state_t data_spb)
 {
-        afr_inode_params_t      params = {0};
+        afr_inode_ctx_t *ctx = NULL;
 
-        params.op = AFR_INODE_SET_SPLIT_BRAIN;
-        params.u.value          = set;
-        afr_inode_set_ctx (this, inode, &params);
+        ctx = afr_inode_ctx_get (inode, this);
+        if (mdata_spb != DONT_KNOW)
+                ctx->mdata_spb = mdata_spb;
+        if (data_spb != DONT_KNOW)
+                ctx->data_spb = data_spb;
 }
 
 void
@@ -467,7 +466,7 @@ afr_set_opendir_done (xlator_t *this, inode_t *inode)
         afr_inode_params_t params = {0};
 
         params.op = AFR_INODE_SET_OPENDIR_DONE;
-        afr_inode_set_ctx (this, inode, &params);
+        afr_inode_set_ctx_params (this, inode, &params);
 }
 
 void
@@ -486,7 +485,7 @@ afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
         params.op = AFR_INODE_SET_READ_CTX;
         params.u.read_ctx.read_child     = read_child;
         params.u.read_ctx.children = fresh_children;
-        afr_inode_set_ctx (this, inode, &params);
+        afr_inode_set_ctx_params (this, inode, &params);
 }
 
 void
@@ -499,7 +498,7 @@ afr_inode_rm_stale_children (xlator_t *this, inode_t *inode,
 
         params.op = AFR_INODE_RM_STALE_CHILDREN;
         params.u.read_ctx.children = stale_children;
-        afr_inode_set_ctx (this, inode, &params);
+        afr_inode_set_ctx_params (this, inode, &params);
 }
 
 gf_boolean_t
@@ -543,6 +542,10 @@ afr_is_read_child (int32_t *success_children, int32_t *sources,
         gf_boolean_t             success_child = _gf_false;
         gf_boolean_t             source        = _gf_false;
 
+        if (child < 0) {
+                return _gf_false;
+        }
+
         GF_ASSERT (success_children);
         GF_ASSERT (child_count > 0);
 
@@ -559,29 +562,69 @@ out:
         return (success_child && source);
 }
 
+int32_t
+afr_hash_child (int32_t *success_children, int32_t child_count,
+                unsigned int hmode, uuid_t gfid)
+{
+        uuid_t  gfid_copy = {0,};
+        pid_t pid;
+
+        if (!hmode) {
+                return -1;
+        }
+
+        if (gfid) {
+               uuid_copy(gfid_copy,gfid);
+        }
+        if (hmode > 1) {
+                /*
+                 * Why getpid?  Because it's one of the cheapest calls
+                 * available - faster than gethostname etc. - and returns a
+                 * constant-length value that's sure to be shorter than a UUID.
+                 * It's still very unlikely to be the same across clients, so
+                 * it still provides good mixing.  We're not trying for
+                 * perfection here.  All we need is a low probability that
+                 * multiple clients won't converge on the same subvolume.
+                 */
+                pid = getpid();
+                memcpy (gfid_copy, &pid, sizeof(pid));
+        }
+
+        return SuperFastHash((char *)gfid_copy,
+                             sizeof(gfid_copy)) % child_count;
+}
+
 /* If sources is NULL the xattrs are assumed to be of source for all
  * success_children.
  */
 int
-afr_select_read_child_from_policy (int32_t *success_children, int32_t child_count,
-                                   int32_t prev_read_child,
-                                   int32_t config_read_child, int32_t *sources)
+afr_select_read_child_from_policy (int32_t *success_children,
+                                   int32_t child_count, int32_t prev_read_child,
+                                   int32_t config_read_child, int32_t *sources,
+                                   unsigned int hmode, uuid_t gfid)
 {
         int32_t                  read_child   = -1;
         int                      i            = 0;
 
         GF_ASSERT (success_children);
 
-        read_child = prev_read_child;
+        read_child = config_read_child;
         if (afr_is_read_child (success_children, sources, child_count,
                                read_child))
                 goto out;
 
-        read_child = config_read_child;
+        read_child = prev_read_child;
         if (afr_is_read_child (success_children, sources, child_count,
                                read_child))
                 goto out;
 
+        read_child = afr_hash_child (success_children, child_count,
+                                     hmode, gfid);
+        if (afr_is_read_child (success_children, sources, child_count,
+                               read_child)) {
+                goto out;
+        }
+
         for (i = 0; i < child_count; i++) {
                 read_child = success_children[i];
                 if (read_child < 0)
@@ -601,7 +644,7 @@ out:
 void
 afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
                               int32_t *fresh_children, int32_t prev_read_child,
-                              int32_t config_read_child)
+                              int32_t config_read_child, uuid_t gfid)
 {
         int                      read_child = -1;
         afr_private_t            *priv = NULL;
@@ -611,7 +654,8 @@ afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
                                                         priv->child_count,
                                                         prev_read_child,
                                                         config_read_child,
-                                                        NULL);
+                                                        NULL,
+                                                        priv->hash_mode, gfid);
         if (read_child >= 0)
                 afr_inode_set_read_ctx (this, inode, read_child,
                                         fresh_children);
@@ -667,8 +711,11 @@ afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
         GF_ASSERT (call_child);
         GF_ASSERT (last_index);
         GF_ASSERT (fresh_children);
-        GF_ASSERT (read_child >= 0);
 
+        if (read_child < 0) {
+                ret = -EIO;
+                goto out;
+        }
         priv = this->private;
         *call_child = -1;
         *last_index = -1;
@@ -717,6 +764,13 @@ out:
 }
 
 void
+afr_xattr_array_destroy (dict_t **xattr, unsigned int child_count)
+{
+        afr_reset_xattr (xattr, child_count);
+        GF_FREE (xattr);
+}
+
+void
 afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
 {
         afr_self_heal_t *sh = NULL;
@@ -725,60 +779,51 @@ afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
         sh = &local->self_heal;
         priv = this->private;
 
-        if (sh->buf)
-                GF_FREE (sh->buf);
+        if (sh->data_sh_info && strcmp (sh->data_sh_info, ""))
+                GF_FREE (sh->data_sh_info);
+
+        if (sh->metadata_sh_info && strcmp (sh->metadata_sh_info, ""))
+                GF_FREE (sh->metadata_sh_info);
+
+        GF_FREE (sh->buf);
 
-        if (sh->parentbufs)
-                GF_FREE (sh->parentbufs);
+        GF_FREE (sh->parentbufs);
 
         if (sh->inode)
                 inode_unref (sh->inode);
 
-        if (sh->xattr) {
-                afr_reset_xattr (sh->xattr, priv->child_count);
-                GF_FREE (sh->xattr);
-        }
+        afr_xattr_array_destroy (sh->xattr, priv->child_count);
 
-        if (sh->child_errno)
-                GF_FREE (sh->child_errno);
+        GF_FREE (sh->child_errno);
 
         afr_matrix_cleanup (sh->pending_matrix, priv->child_count);
         afr_matrix_cleanup (sh->delta_matrix, priv->child_count);
 
-        if (sh->sources)
-                GF_FREE (sh->sources);
+        GF_FREE (sh->sources);
 
-        if (sh->success)
-                GF_FREE (sh->success);
+        GF_FREE (sh->success);
 
-        if (sh->locked_nodes)
-                GF_FREE (sh->locked_nodes);
+        GF_FREE (sh->locked_nodes);
 
         if (sh->healing_fd) {
                 fd_unref (sh->healing_fd);
                 sh->healing_fd = NULL;
         }
 
-        if (sh->linkname)
-                GF_FREE ((char *)sh->linkname);
+        GF_FREE ((char *)sh->linkname);
 
-        if (sh->success_children)
-                GF_FREE (sh->success_children);
+        GF_FREE (sh->success_children);
 
-        if (sh->fresh_children)
-                GF_FREE (sh->fresh_children);
+        GF_FREE (sh->fresh_children);
 
-        if (sh->fresh_parent_dirs)
-                GF_FREE (sh->fresh_parent_dirs);
+        GF_FREE (sh->fresh_parent_dirs);
 
         loc_wipe (&sh->parent_loc);
         loc_wipe (&sh->lookup_loc);
 
-        if (sh->checksum)
-                GF_FREE (sh->checksum);
+        GF_FREE (sh->checksum);
 
-        if (sh->write_needed)
-                GF_FREE (sh->write_needed);
+        GF_FREE (sh->write_needed);
         if (sh->healing_fd)
                 fd_unref (sh->healing_fd);
 }
@@ -787,27 +832,26 @@ afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
 void
 afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
 {
-        afr_private_t * priv = NULL;
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
 
         priv = this->private;
 
         afr_matrix_cleanup (local->pending, priv->child_count);
+        afr_matrix_cleanup (local->transaction.txn_changelog,
+                            priv->child_count);
 
-        if (local->internal_lock.locked_nodes)
-                GF_FREE (local->internal_lock.locked_nodes);
-
-        if (local->internal_lock.inode_locked_nodes)
-                GF_FREE (local->internal_lock.inode_locked_nodes);
+        GF_FREE (local->internal_lock.locked_nodes);
 
-        if (local->internal_lock.entry_locked_nodes)
-                GF_FREE (local->internal_lock.entry_locked_nodes);
+        for (i = 0; local->internal_lock.inodelk[i].domain; i++) {
+                GF_FREE (local->internal_lock.inodelk[i].locked_nodes);
+        }
 
-        if (local->internal_lock.lower_locked_nodes)
-                GF_FREE (local->internal_lock.lower_locked_nodes);
+        GF_FREE (local->internal_lock.lower_locked_nodes);
 
+        afr_entry_lockee_cleanup (&local->internal_lock);
 
         GF_FREE (local->transaction.pre_op);
-        GF_FREE (local->transaction.child_errno);
         GF_FREE (local->transaction.eager_lock);
 
         GF_FREE (local->transaction.basename);
@@ -815,6 +859,8 @@ afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
 
         loc_wipe (&local->transaction.parent_loc);
         loc_wipe (&local->transaction.new_parent_loc);
+
+        GF_FREE (local->transaction.postop_piggybacked);
 }
 
 
@@ -841,17 +887,16 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         if (local->xattr_req)
                 dict_unref (local->xattr_req);
 
-        if (local->child_up)
-                GF_FREE (local->child_up);
+        if (local->dict)
+                dict_unref (local->dict);
 
-        if (local->child_errno)
-                GF_FREE (local->child_errno);
+	GF_FREE(local->replies);
 
-        if (local->fresh_children)
-                GF_FREE (local->fresh_children);
+        GF_FREE (local->child_up);
 
-        if (local->fd_open_on)
-                GF_FREE (local->fd_open_on);
+        GF_FREE (local->child_errno);
+
+        GF_FREE (local->fresh_children);
 
         { /* lookup */
                 if (local->cont.lookup.xattrs) {
@@ -869,27 +914,23 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
                         inode_unref (local->cont.lookup.inode);
                 }
 
-                if (local->cont.lookup.postparents)
-                        GF_FREE (local->cont.lookup.postparents);
+                GF_FREE (local->cont.lookup.postparents);
 
-                if (local->cont.lookup.bufs)
-                        GF_FREE (local->cont.lookup.bufs);
+                GF_FREE (local->cont.lookup.bufs);
 
-                if (local->cont.lookup.success_children)
-                        GF_FREE (local->cont.lookup.success_children);
+                GF_FREE (local->cont.lookup.success_children);
 
-                if (local->cont.lookup.sources)
-                        GF_FREE (local->cont.lookup.sources);
+                GF_FREE (local->cont.lookup.sources);
+                afr_matrix_cleanup (local->cont.lookup.pending_matrix,
+                                    priv->child_count);
         }
 
         { /* getxattr */
-                if (local->cont.getxattr.name)
-                        GF_FREE (local->cont.getxattr.name);
+                GF_FREE (local->cont.getxattr.name);
         }
 
         { /* lk */
-                if (local->cont.lk.locked_nodes)
-                        GF_FREE (local->cont.lk.locked_nodes);
+                GF_FREE (local->cont.lk.locked_nodes);
         }
 
         { /* create */
@@ -931,20 +972,32 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         { /* removexattr */
                 GF_FREE (local->cont.removexattr.name);
         }
-
+        { /* xattrop */
+                if (local->cont.xattrop.xattr)
+                        dict_unref (local->cont.xattrop.xattr);
+        }
+        { /* fxattrop */
+                if (local->cont.fxattrop.xattr)
+                        dict_unref (local->cont.fxattrop.xattr);
+        }
         { /* symlink */
                 GF_FREE (local->cont.symlink.linkpath);
         }
 
         { /* opendir */
-                if (local->cont.opendir.checksum)
-                        GF_FREE (local->cont.opendir.checksum);
+                GF_FREE (local->cont.opendir.checksum);
         }
 
         { /* readdirp */
                 if (local->cont.readdir.dict)
                         dict_unref (local->cont.readdir.dict);
         }
+
+        if (local->xdata_req)
+                dict_unref (local->xdata_req);
+
+        if (local->xdata_rsp)
+                dict_unref (local->xdata_rsp);
 }
 
 
@@ -1027,34 +1080,144 @@ afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
                 uuid_copy (loc->pargfid, postparent->ia_gfid);
 }
 
+/*
+ * Quota size xattrs are not maintained by afr. There is a
+ * possibility that they differ even when both the directory changelog xattrs
+ * suggest everything is fine. So if there is at least one 'source' check among
+ * the sources which has the maximum quota size. Otherwise check among all the
+ * available ones for maximum quota size. This way if there is a source and
+ * stale copies it always votes for the 'source'.
+ * */
+
+static void
+afr_handle_quota_size (afr_local_t *local, xlator_t *this,
+                       dict_t *rsp_dict)
+{
+        int32_t       *sources       = NULL;
+        dict_t        *xattr         = NULL;
+        data_t        *max_data      = NULL;
+        int64_t       max_quota_size = -1;
+        data_t        *data          = NULL;
+        int64_t       *size          = NULL;
+        int64_t       quota_size     = -1;
+        afr_private_t *priv          = NULL;
+        int           i              = 0;
+        int           ret            = -1;
+        gf_boolean_t  source_present = _gf_false;
+
+        priv    = this->private;
+        sources = local->cont.lookup.sources;
+
+        if (rsp_dict == NULL) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "%s: Invalid "
+                                  "response dictionary", local->loc.path);
+                return;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sources[i]) {
+                        source_present = _gf_true;
+                        break;
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                /*
+                 * If there is at least one source lets check
+                 * for maximum quota sizes among sources, otherwise take the
+                 * maximum of the ones present to be on the safer side.
+                 */
+                if (source_present && !sources[i])
+                        continue;
+
+                xattr = local->cont.lookup.xattrs[i];
+                if (!xattr)
+                        continue;
+
+                data = dict_get (xattr, QUOTA_SIZE_KEY);
+                if (!data)
+                        continue;
+
+                size = (int64_t*)data->data;
+                quota_size = ntoh64(*size);
+                gf_log (this->name, GF_LOG_DEBUG, "%s: %d, size: %"PRId64,
+                        local->loc.path, i, quota_size);
+                if (quota_size > max_quota_size) {
+                        if (max_data)
+                                data_unref (max_data);
+
+                        max_quota_size = quota_size;
+                        max_data = data_ref (data);
+                }
+        }
+
+        if (max_data) {
+                ret = dict_set (rsp_dict, QUOTA_SIZE_KEY, max_data);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
+                                "quota size", local->loc.path);
+                }
+
+                data_unref (max_data);
+        }
+}
+
 int
 afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
 {
-        int32_t         read_child = -1;
         struct iatt     *buf = NULL;
         struct iatt     *postparent = NULL;
         dict_t          **xattr = NULL;
+        int32_t         *success_children = NULL;
+        int32_t         *sources = NULL;
+        afr_private_t   *priv = NULL;
+        int32_t         read_child = -1;
         int             ret = 0;
+        int             i = 0;
 
         GF_ASSERT (local);
 
         buf = &local->cont.lookup.buf;
         postparent = &local->cont.lookup.postparent;
         xattr = &local->cont.lookup.xattr;
+        priv = this->private;
 
         read_child = afr_inode_get_read_ctx (this, local->cont.lookup.inode,
-                                             NULL);
+                                             local->fresh_children);
+        if (read_child < 0) {
+                ret = -1;
+                goto out;
+        }
+        success_children = local->cont.lookup.success_children;
+        sources = local->cont.lookup.sources;
+        memset (sources, 0, sizeof (*sources) * priv->child_count);
+        afr_children_intersection_get (local->fresh_children, success_children,
+                                       sources, priv->child_count);
+        if (!sources[read_child]) {
+                read_child = -1;
+                for (i = 0; i < priv->child_count; i++) {
+                        if (sources[i]) {
+                                read_child = i;
+                                break;
+                        }
+                }
+        }
         if (read_child < 0) {
                 ret = -1;
                 goto out;
         }
+
         gf_log (this->name, GF_LOG_DEBUG, "Building lookup response from %d",
                 read_child);
         if (!*xattr)
                 *xattr = dict_ref (local->cont.lookup.xattrs[read_child]);
+
         *buf = local->cont.lookup.bufs[read_child];
         *postparent = local->cont.lookup.postparents[read_child];
 
+        if (dict_get (local->xattr_req, QUOTA_SIZE_KEY))
+                afr_handle_quota_size (local, this, *xattr);
+
         if (IA_INVAL == local->cont.lookup.inode->ia_type) {
                 /* fix for RT #602 */
                 local->cont.lookup.inode->ia_type = buf->ia_type;
@@ -1070,6 +1233,7 @@ afr_lookup_update_lk_counts (afr_local_t *local, xlator_t *this,
         uint32_t inodelk_count = 0;
         uint32_t entrylk_count = 0;
         int      ret           = -1;
+        uint32_t parent_entrylk = 0;
 
         GF_ASSERT (local);
         GF_ASSERT (this);
@@ -1085,43 +1249,103 @@ afr_lookup_update_lk_counts (afr_local_t *local, xlator_t *this,
                                &entrylk_count);
         if (ret == 0)
                 local->entrylk_count += entrylk_count;
+        ret = dict_get_uint32 (xattr, GLUSTERFS_PARENT_ENTRYLK,
+                               &parent_entrylk);
+        if (!ret)
+                local->cont.lookup.parent_entrylk += parent_entrylk;
 }
 
+/*
+ * It's important to maintain a commutative property on do_*_self_heal and
+ * found*; once set, they must not be cleared by a subsequent iteration or
+ * call, so that they represent a logical OR of all iterations and calls
+ * regardless of child/key order.  That allows the caller to call us multiple
+ * times without having to use a separate variable as a "reduce" accumulator.
+ */
 static void
 afr_lookup_set_self_heal_params_by_xattr (afr_local_t *local, xlator_t *this,
                                           dict_t *xattr)
 {
+        afr_private_t *priv        = NULL;
+        int            i           = 0;
+        int            ret         = -1;
+        void          *pending_raw = NULL;
+        int32_t       *pending     = NULL;
+
         GF_ASSERT (local);
         GF_ASSERT (this);
         GF_ASSERT (xattr);
 
-        if (afr_sh_has_metadata_pending (xattr, this)) {
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "metadata self-heal is pending for %s.",
-                       local->loc.path);
-        }
+        priv = this->private;
 
-        if (afr_sh_has_entry_pending (xattr, this)) {
-                local->self_heal.do_entry_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "entry self-heal is pending for %s.", local->loc.path);
-        }
+        for (i = 0; i < priv->child_count; i++) {
+                ret = dict_get_ptr (xattr, priv->pending_key[i],
+                                    &pending_raw);
+                if (ret != 0) {
+                        continue;
+                }
+                pending = pending_raw;
 
-        if (afr_sh_has_data_pending (xattr, this)) {
-                local->self_heal.do_data_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "data self-heal is pending for %s.", local->loc.path);
+                if (pending[AFR_METADATA_TRANSACTION]) {
+                        gf_log(this->name, GF_LOG_DEBUG,
+                               "metadata self-heal is pending for %s.",
+                               local->loc.path);
+                        local->self_heal.do_metadata_self_heal = _gf_true;
+                }
+
+                if (pending[AFR_ENTRY_TRANSACTION]) {
+                        gf_log(this->name, GF_LOG_DEBUG,
+                               "entry self-heal is pending for %s.",
+                               local->loc.path);
+                        local->self_heal.do_entry_self_heal = _gf_true;
+                }
+
+                if (pending[AFR_DATA_TRANSACTION]) {
+                        gf_log(this->name, GF_LOG_DEBUG,
+                               "data self-heal is pending for %s.",
+                               local->loc.path);
+                        local->self_heal.do_data_self_heal = _gf_true;
+                }
         }
 }
 
+void
+afr_lookup_check_set_metadata_split_brain (afr_local_t *local, xlator_t *this)
+{
+        int32_t                  *sources = NULL;
+        afr_private_t            *priv = NULL;
+        int32_t                  subvol_status = 0;
+        int32_t                  *success_children   = NULL;
+        dict_t                   **xattrs = NULL;
+        struct iatt              *bufs = NULL;
+        int32_t                  **pending_matrix = NULL;
+
+        priv = this->private;
+
+        sources = GF_CALLOC (priv->child_count, sizeof (*sources),
+                             gf_afr_mt_int32_t);
+        if (NULL == sources)
+                goto out;
+        success_children = local->cont.lookup.success_children;
+        xattrs = local->cont.lookup.xattrs;
+        bufs = local->cont.lookup.bufs;
+        pending_matrix = local->cont.lookup.pending_matrix;
+        afr_build_sources (this, xattrs, bufs, pending_matrix,
+                           sources, success_children, AFR_METADATA_TRANSACTION,
+                           &subvol_status, _gf_false);
+        if (subvol_status & SPLIT_BRAIN)
+                local->cont.lookup.possible_spb = _gf_true;
+out:
+        GF_FREE (sources);
+}
+
 static void
 afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
                             struct iatt *buf, struct iatt *lookup_buf)
 {
         if (PERMISSION_DIFFERS (buf, lookup_buf)) {
                 /* mismatching permissions */
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "permissions differ for %s ", local->loc.path);
                 local->self_heal.do_metadata_self_heal = _gf_true;
         }
@@ -1129,27 +1353,45 @@ afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
         if (OWNERSHIP_DIFFERS (buf, lookup_buf)) {
                 /* mismatching permissions */
                 local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "ownership differs for %s ", local->loc.path);
         }
 
         if (SIZE_DIFFERS (buf, lookup_buf)
             && IA_ISREG (buf->ia_type)) {
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "size differs for %s ", local->loc.path);
                 local->self_heal.do_data_self_heal = _gf_true;
         }
 
         if (uuid_compare (buf->ia_gfid, lookup_buf->ia_gfid)) {
                 /* mismatching gfid */
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "%s: gfid different on subvolume", local->loc.path);
         }
 }
 
 static void
-afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this,
-                                       gf_boolean_t split_brain)
+afr_detect_self_heal_by_split_brain_status (afr_local_t *local, xlator_t *this)
+{
+        gf_boolean_t split_brain = _gf_false;
+        afr_self_heal_t *sh = NULL;
+
+        sh = &local->self_heal;
+
+        split_brain = afr_is_split_brain (this, local->cont.lookup.inode);
+        split_brain = split_brain || local->cont.lookup.possible_spb;
+        if ((local->success_count > 0) && split_brain &&
+            IA_ISREG (local->cont.lookup.inode->ia_type)) {
+                sh->force_confirm_spb = _gf_true;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "split brain detected during lookup of %s.",
+                        local->loc.path);
+        }
+}
+
+static void
+afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this)
 {
         GF_ASSERT (local);
         GF_ASSERT (this);
@@ -1160,24 +1402,11 @@ afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this,
                 local->self_heal.do_entry_self_heal    = _gf_true;
                 local->self_heal.do_gfid_self_heal    = _gf_true;
                 local->self_heal.do_missing_entry_self_heal    = _gf_true;
-                gf_log(this->name, GF_LOG_INFO,
+                gf_log(this->name, GF_LOG_DEBUG,
                        "entries are missing in lookup of %s.",
                        local->loc.path);
-                //If all self-heals are needed no need to check for other rules
-                goto out;
-        }
-
-        if ((local->success_count > 0) && split_brain &&
-            IA_ISREG (local->cont.lookup.inode->ia_type)) {
-                local->self_heal.do_data_self_heal = _gf_true;
-                local->self_heal.do_gfid_self_heal    = _gf_true;
-                local->self_heal.do_missing_entry_self_heal    = _gf_true;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "split brain detected during lookup of %s.",
-                        local->loc.path);
         }
 
-out:
         return;
 }
 
@@ -1187,6 +1416,8 @@ afr_can_self_heal_proceed (afr_self_heal_t *sh, afr_private_t *priv)
         GF_ASSERT (sh);
         GF_ASSERT (priv);
 
+        if (sh->force_confirm_spb)
+                return _gf_true;
         return (sh->do_gfid_self_heal
                 || sh->do_missing_entry_self_heal
                 || (afr_data_self_heal_enabled (priv->data_self_heal) &&
@@ -1220,6 +1451,7 @@ afr_lookup_select_read_child (afr_local_t *local, xlator_t *this,
         dict_t                  **xattrs       = NULL;
         int32_t                 *success_children = NULL;
         afr_transaction_type    type           = AFR_METADATA_TRANSACTION;
+        uuid_t                  *gfid          = NULL;
 
         GF_ASSERT (local);
         GF_ASSERT (this);
@@ -1233,8 +1465,9 @@ afr_lookup_select_read_child (afr_local_t *local, xlator_t *this,
         ia_type = local->cont.lookup.bufs[success_children[0]].ia_type;
         type = afr_transaction_type_get (ia_type);
         xattrs = local->cont.lookup.xattrs;
+        gfid = &local->cont.lookup.buf.ia_gfid;
         source = afr_lookup_select_read_child_by_txn_type (this, local, xattrs,
-                                                           type);
+                                                           type, *gfid);
         if (source < 0) {
                 gf_log (this->name, GF_LOG_DEBUG, "failed to select source "
                         "for %s", local->loc.path);
@@ -1262,7 +1495,8 @@ afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
                       void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
                                                    xlator_t *this),
                       int (*unwind) (call_frame_t *frame, xlator_t *this,
-                                     int32_t op_ret, int32_t op_errno))
+                                     int32_t op_ret, int32_t op_errno,
+                                     int32_t sh_failed))
 {
         afr_local_t             *local = NULL;
         char                    sh_type_str[256] = {0,};
@@ -1285,7 +1519,7 @@ afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
 
         if (background)
                 bg = "background";
-        gf_log (this->name, GF_LOG_INFO,
+        gf_log (this->name, GF_LOG_DEBUG,
                 "%s %s self-heal triggered. path: %s, reason: %s", bg,
                 sh_type_str, local->loc.path, reason);
 
@@ -1356,7 +1590,7 @@ afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
 
                 child2 = &bufs[success_children[i-1]];
                 if (FILETYPE_DIFFERS (child1, child2)) {
-                        gf_log (xlator_name, GF_LOG_WARNING, "%s: filetype "
+                        gf_log (xlator_name, GF_LOG_DEBUG, "%s: filetype "
                                 "differs on subvolumes (%d, %d)", path,
                                 success_children[i-1], success_children[i]);
                         conflicting = _gf_true;
@@ -1365,7 +1599,7 @@ afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
                 if (!gfid || uuid_is_null (child1->ia_gfid))
                         continue;
                 if (uuid_compare (*gfid, child1->ia_gfid)) {
-                       gf_log (xlator_name, GF_LOG_WARNING, "%s: gfid differs"
+                       gf_log (xlator_name, GF_LOG_DEBUG, "%s: gfid differs"
                                " on subvolume %d", path, success_children[i]);
                        conflicting = _gf_true;
                        goto out;
@@ -1448,13 +1682,11 @@ afr_lookup_set_self_heal_params (afr_local_t *local, xlator_t *this)
         int32_t                 child1 = -1;
         int32_t                 child2 = -1;
         afr_self_heal_t         *sh = NULL;
-        gf_boolean_t            split_brain = _gf_false;
 
         priv  = this->private;
         sh = &local->self_heal;
 
-        split_brain = afr_is_split_brain (this, local->cont.lookup.inode);
-        afr_detect_self_heal_by_lookup_status (local, this, split_brain);
+        afr_detect_self_heal_by_lookup_status (local, this);
 
         if (afr_lookup_gfid_missing_count (local, this))
                 local->self_heal.do_gfid_self_heal    = _gf_true;
@@ -1481,23 +1713,28 @@ afr_lookup_set_self_heal_params (afr_local_t *local, xlator_t *this)
                 afr_lookup_set_self_heal_params_by_xattr (local, this,
                                                           xattr[child1]);
         }
-        if (afr_open_only_data_self_heal (priv->data_self_heal)
-            && !split_brain)
+        if (afr_open_only_data_self_heal (priv->data_self_heal))
                 sh->do_data_self_heal = _gf_false;
+        if (sh->do_metadata_self_heal)
+                afr_lookup_check_set_metadata_split_brain (local, this);
+        afr_detect_self_heal_by_split_brain_status (local, this);
 }
 
 int
 afr_self_heal_lookup_unwind (call_frame_t *frame, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+                             int32_t op_ret, int32_t op_errno,
+                             int32_t sh_failed)
 {
         afr_local_t *local = NULL;
+        int         ret    = -1;
+        dict_t      *xattr = NULL;
 
         local = frame->local;
 
         if (op_ret == -1) {
                 local->op_ret = -1;
-                if (afr_error_more_important (local->op_errno, op_errno))
-                        local->op_errno = op_errno;
+		local->op_errno = afr_most_important_error(local->op_errno,
+							   op_errno, _gf_true);
 
                 goto out;
         } else {
@@ -1505,6 +1742,23 @@ afr_self_heal_lookup_unwind (call_frame_t *frame, xlator_t *this,
         }
 
         afr_lookup_done_success_action (frame, this, _gf_true);
+        xattr = local->cont.lookup.xattr;
+        if (xattr) {
+                ret = dict_set_int32 (xattr, "sh-failed", sh_failed);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
+                                "sh-failed to %d", local->loc.path, sh_failed);
+
+                if (local->self_heal.actual_sh_started == _gf_true &&
+                    sh_failed == 0) {
+                        ret = dict_set_int32 (xattr, "actual-sh-done", 1);
+                        if (ret)
+                                gf_log(this->name, GF_LOG_ERROR, "%s: Failed to"
+                                       " set actual-sh-done to %d",
+                                       local->loc.path,
+                                       local->self_heal.actual_sh_started);
+                }
+        }
 out:
         AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
                           local->cont.lookup.inode, &local->cont.lookup.buf,
@@ -1578,7 +1832,8 @@ afr_lookup_perform_self_heal (call_frame_t *frame, xlator_t *this,
 
         afr_lookup_set_self_heal_params (local, this);
         if (afr_can_self_heal_proceed (&local->self_heal, priv)) {
-                if  (afr_is_transaction_running (local))
+                if  (afr_is_transaction_running (local) &&
+                     (!local->allow_sh_for_running_transaction))
                         goto out;
 
                 reason = "lookup detected pending operations";
@@ -1639,26 +1894,23 @@ afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
         int32_t             read_child = -1;
         int32_t             ret        = -1;
         afr_local_t         *local     = NULL;
-        afr_private_t       *priv      = NULL;
+        gf_boolean_t        fresh_lookup = _gf_false;
 
         local   = frame->local;
-        priv    = this->private;
+        fresh_lookup = local->cont.lookup.fresh_lookup;
 
         if (local->loc.parent == NULL)
                 fail_conflict = _gf_true;
 
-        if (afr_conflicting_iattrs (local->cont.lookup.bufs,
-                                     local->cont.lookup.success_children,
-                                     priv->child_count, local->loc.path,
-                                     this->name)) {
+        if (afr_lookup_conflicting_entries (local, this)) {
                 if (fail_conflict == _gf_false)
                         ret = 0;
                 goto out;
         }
 
-        if (!afr_is_transaction_running (local)) {
-                ret = afr_lookup_select_read_child (local, this, &read_child);
-                if (ret)
+        ret = afr_lookup_select_read_child (local, this, &read_child);
+        if (!afr_is_transaction_running (local) || fresh_lookup) {
+                if (read_child < 0)
                         goto out;
 
                 ret = afr_lookup_set_read_ctx (local, this, read_child);
@@ -1669,11 +1921,9 @@ afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
         ret = afr_lookup_build_response_params (local, this);
         if (ret)
                 goto out;
-        if (afr_is_fresh_lookup (&local->loc, this)) {
-                afr_update_loc_gfids (&local->loc,
-                                      &local->cont.lookup.buf,
-                                      &local->cont.lookup.postparent);
-        }
+        afr_update_loc_gfids (&local->loc,
+                              &local->cont.lookup.buf,
+                              &local->cont.lookup.postparent);
 
         ret = 0;
 out:
@@ -1684,6 +1934,135 @@ out:
         return ret;
 }
 
+int
+afr_lookup_get_latest_subvol (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        int32_t       *success_children = NULL;
+        struct iatt   *bufs = NULL;
+        int           i = 0;
+        int           child = 0;
+        int           lsubvol = -1;
+
+        priv = this->private;
+        success_children = local->cont.lookup.success_children;
+        bufs = local->cont.lookup.bufs;
+        for (i = 0; i < priv->child_count; i++) {
+                child = success_children[i];
+                if (child == -1)
+                        break;
+                if (uuid_is_null (bufs[child].ia_gfid))
+                        continue;
+                if (lsubvol < 0) {
+                        lsubvol = child;
+                } else if (bufs[lsubvol].ia_ctime < bufs[child].ia_ctime) {
+                        lsubvol = child;
+                } else if ((bufs[lsubvol].ia_ctime == bufs[child].ia_ctime) &&
+                  (bufs[lsubvol].ia_ctime_nsec < bufs[child].ia_ctime_nsec)) {
+                        lsubvol = child;
+                }
+        }
+        return lsubvol;
+}
+
+void
+afr_lookup_mark_other_entries_stale (afr_local_t *local, xlator_t *this,
+                                     int subvol)
+{
+        afr_private_t *priv = NULL;
+        int32_t       *success_children = NULL;
+        struct iatt   *bufs = NULL;
+        int           i = 0;
+        int           child = 0;
+
+        priv = this->private;
+        success_children = local->cont.lookup.success_children;
+        bufs = local->cont.lookup.bufs;
+        memcpy (local->fresh_children, success_children,
+                sizeof (*success_children) * priv->child_count);
+        for (i = 0; i < priv->child_count; i++) {
+                child = local->fresh_children[i];
+                if (child == -1)
+                        break;
+                if (child == subvol)
+                        continue;
+                if (uuid_is_null (bufs[child].ia_gfid) &&
+                    (bufs[child].ia_type == bufs[subvol].ia_type))
+                        continue;
+                afr_children_rm_child (success_children, child,
+                                       priv->child_count);
+                local->success_count--;
+        }
+        afr_reset_children (local->fresh_children, priv->child_count);
+}
+
+void
+afr_succeed_lookup_on_latest_iatt (afr_local_t *local, xlator_t *this)
+{
+        int    lsubvol = 0;
+
+        if (!afr_lookup_conflicting_entries (local, this))
+                goto out;
+
+        lsubvol = afr_lookup_get_latest_subvol (local, this);
+        if (lsubvol < 0)
+                goto out;
+        afr_lookup_mark_other_entries_stale (local, this, lsubvol);
+out:
+        return;
+}
+
+gf_boolean_t
+afr_is_entry_possibly_under_creation (afr_local_t *local, xlator_t *this)
+{
+        /*
+         * We need to perform this test in lookup done and treat on going
+         * create/DELETE as ENOENT.
+         * Reason:
+        Multiple clients A, B and C are attempting 'mkdir -p /mnt/a/b/c'
+
+        1 Client A is in the middle of mkdir(/a). It has acquired lock.
+          It has performed mkdir(/a) on one subvol, and second one is still
+          in progress
+        2 Client B performs a lookup, sees directory /a on one,
+          ENOENT on the other, succeeds lookup.
+        3 Client B performs lookup on /a/b on both subvols, both return ENOENT
+          (one subvol because /a/b does not exist, another because /a
+          itself does not exist)
+        4 Client B proceeds to mkdir /a/b. It obtains entrylk on inode=/a with
+          basename=b on one subvol, but fails on other subvol as /a is yet to
+          be created by Client A.
+        5 Client A finishes mkdir of /a on other subvol
+        6 Client C also attempts to create /a/b, lookup returns ENOENT on
+          both subvols.
+        7 Client C tries to obtain entrylk on on inode=/a with basename=b,
+          obtains on one subvol (where B had failed), and waits for B to unlock
+          on other subvol.
+        8 Client B finishes mkdir() on one subvol with GFID-1 and completes
+          transaction and unlocks
+        9 Client C gets the lock on the second subvol, At this stage second
+          subvol already has /a/b created from Client B, but Client C does not
+          check that in the middle of mkdir transaction
+        10 Client C attempts mkdir /a/b on both subvols. It succeeds on
+           ONLY ONE (where Client B could not get lock because of
+           missing parent /a dir) with GFID-2, and gets EEXIST from ONE subvol.
+        This way we have /a/b in GFID mismatch. One subvol got GFID-1 because
+        Client B performed transaction on only one subvol (because entrylk()
+        could not be obtained on second subvol because of missing parent dir --
+        caused by premature/speculative succeeding of lookup() on /a when locks
+        are detected). Other subvol gets GFID-2 from Client C because while
+        it was waiting for entrylk() on both subvols, Client B was in the
+        middle of creating mkdir() on only one subvol, and Client C does not
+        "expect" this when it is between lock() and pre-op()/op() phase of the
+        transaction.
+         */
+	if (local->cont.lookup.parent_entrylk && local->enoent_count)
+		return _gf_true;
+
+	return _gf_false;
+}
+
+
 static void
 afr_lookup_done (call_frame_t *frame, xlator_t *this)
 {
@@ -1700,8 +2079,18 @@ afr_lookup_done (call_frame_t *frame, xlator_t *this)
         priv  = this->private;
         local = frame->local;
 
+	if (afr_is_entry_possibly_under_creation (local, this)) {
+		local->op_ret = -1;
+		local->op_errno = ENOENT;
+		goto unwind;
+	}
+
         if (local->op_ret < 0)
                 goto unwind;
+
+        if (local->cont.lookup.parent_entrylk && local->success_count > 1)
+                afr_succeed_lookup_on_latest_iatt (local, this);
+
         gfid_miss_count = afr_lookup_gfid_missing_count (local, this);
         up_children_count = afr_up_children_count (local->child_up,
                                                    priv->child_count);
@@ -1753,25 +2142,20 @@ afr_lookup_done (call_frame_t *frame, xlator_t *this)
  * others in that they must be given higher priority while
  * returning to the user.
  *
- * The hierarchy is ESTALE > ENOENT > others
- *
+ * The hierarchy is ESTALE > EIO > ENOENT > others
  */
-
-gf_boolean_t
-afr_error_more_important (int32_t old_errno, int32_t new_errno)
+int32_t
+afr_most_important_error(int32_t old_errno, int32_t new_errno,
+			 gf_boolean_t eio)
 {
-        gf_boolean_t ret = _gf_true;
-
-        /* Nothing should ever overwrite ESTALE */
-        if (old_errno == ESTALE)
-                ret = _gf_false;
-
-        /* Nothing should overwrite ENOENT, except ESTALE/EIO*/
-        else if ((old_errno == ENOENT) && (new_errno != ESTALE)
-                 && (new_errno != EIO))
-                ret = _gf_false;
-
-        return ret;
+	if (old_errno == ESTALE || new_errno == ESTALE)
+		return ESTALE;
+	if (eio && (old_errno == EIO || new_errno == EIO))
+		return EIO;
+	if (old_errno == ENOENT || new_errno == ENOENT)
+		return ENOENT;
+
+	return new_errno;
 }
 
 int32_t
@@ -1790,8 +2174,9 @@ afr_resultant_errno_get (int32_t *children,
                 } else {
                         child = i;
                 }
-                if (afr_error_more_important (op_errno, child_errno[child]))
-                                op_errno = child_errno[child];
+		op_errno = afr_most_important_error(op_errno,
+						    child_errno[child],
+						    _gf_false);
         }
         return op_errno;
 }
@@ -1803,8 +2188,8 @@ afr_lookup_handle_error (afr_local_t *local, int32_t op_ret,  int32_t op_errno)
         if (op_errno == ENOENT)
                 local->enoent_count++;
 
-        if (afr_error_more_important (local->op_errno, op_errno))
-                local->op_errno = op_errno;
+	local->op_errno = afr_most_important_error(local->op_errno, op_errno,
+						   _gf_false);
 
         if (local->op_errno == ESTALE) {
                 local->op_ret = -1;
@@ -1851,12 +2236,79 @@ afr_lookup_handle_first_success (afr_local_t *local, xlator_t *this,
         afr_set_root_inode_on_first_lookup (local, this, inode);
 }
 
+static int32_t
+afr_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
+{
+        int              ret            = 0;
+        char            *pathinfo       = NULL;
+        gf_boolean_t     is_local        = _gf_false;
+        afr_private_t   *priv           = NULL;
+        int32_t          child_index    = -1;
+
+        if (op_ret != 0) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+        if (ret != 0) {
+                goto out;
+        }
+
+        ret = afr_local_pathinfo (pathinfo, &is_local);
+        if (ret) {
+                goto out;
+        }
+
+        priv = this->private;
+        /*
+         * Note that one local subvolume will override another here.  The only
+         * way to avoid that would be to retain extra information about whether
+         * the previous read_child is local, and it's just not worth it.  Even
+         * the slowest local subvolume is far preferable to a remote one.
+         */
+        if (is_local) {
+                child_index = (int32_t)(long)cookie;
+                gf_log (this->name, GF_LOG_INFO,
+                        "selecting local read_child %s",
+                        priv->children[child_index]->name);
+                priv->read_child = child_index;
+        }
+
+out:
+        STACK_DESTROY(frame->root);
+        return 0;
+}
+
+static void
+afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
+{
+        call_frame_t    *newframe = NULL;
+        loc_t            tmploc = {0,};
+        afr_private_t   *priv = this->private;
+
+        newframe = create_frame(this,this->ctx->pool);
+        if (!newframe) {
+                return;
+        }
+
+        tmploc.gfid[sizeof(tmploc.gfid)-1] = 1;
+        STACK_WIND_COOKIE (newframe, afr_discovery_cbk,
+                           (void *)(long)child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->getxattr,
+                           &tmploc, GF_XATTR_PATHINFO_KEY, NULL);
+}
+
 static void
 afr_lookup_handle_success (afr_local_t *local, xlator_t *this, int32_t child_index,
                            int32_t op_ret, int32_t op_errno, inode_t *inode,
                            struct iatt *buf, dict_t *xattr,
                            struct iatt *postparent)
 {
+        afr_private_t   *priv   = this->private;
+
         if (local->success_count == 0) {
                 if (local->op_errno != ESTALE) {
                         local->op_ret = op_ret;
@@ -1869,6 +2321,11 @@ afr_lookup_handle_success (afr_local_t *local, xlator_t *this, int32_t child_ind
 
         afr_lookup_cache_args (local, child_index, xattr,
                                buf, postparent);
+
+        if (local->do_discovery && (priv->read_child == (-1))) {
+                afr_attempt_local_discovery(this,child_index);
+        }
+
         local->cont.lookup.success_children[local->success_count] = child_index;
         local->success_count++;
 }
@@ -1916,6 +2373,7 @@ afr_lookup_cont_init (afr_local_t *local, unsigned int child_count)
         struct iatt       *iatts         = NULL;
         int32_t           *success_children = NULL;
         int32_t           *sources       = NULL;
+        int32_t           **pending_matrix = NULL;
 
         GF_ASSERT (local);
         local->cont.lookup.xattrs = GF_CALLOC (child_count,
@@ -1948,6 +2406,11 @@ afr_lookup_cont_init (afr_local_t *local, unsigned int child_count)
                 goto out;
         local->cont.lookup.sources = sources;
 
+        pending_matrix = afr_matrix_create (child_count, child_count);
+        if (NULL == pending_matrix)
+                goto out;
+        local->cont.lookup.pending_matrix = pending_matrix;
+
         ret = 0;
 out:
         return ret;
@@ -1965,37 +2428,51 @@ afr_lookup (call_frame_t *frame, xlator_t *this,
         int               call_count     = 0;
         uint64_t          ctx            = 0;
         int32_t           op_errno       = 0;
-
+        int               allow_sh       = 0;
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
 
         local->op_ret = -1;
 
         frame->local = local;
         local->fop = GF_FOP_LOOKUP;
 
-        if (!strcmp (loc->path, "/" GF_REPLICATE_TRASH_DIR)) {
-                op_errno = ENOENT;
+        loc_copy (&local->loc, loc);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0) {
+                op_errno = EINVAL;
                 goto out;
         }
 
-        loc_copy (&local->loc, loc);
+        if (local->loc.path &&
+            (strcmp (local->loc.path, "/" GF_REPLICATE_TRASH_DIR) == 0)) {
+                op_errno = EPERM;
+                ret = -1;
+                goto out;
+        }
 
-        ret = inode_ctx_get (loc->inode, this, &ctx);
+        ret = inode_ctx_get (local->loc.inode, this, &ctx);
         if (ret == 0) {
                 /* lookup is a revalidate */
 
                 local->read_child_index = afr_inode_get_read_ctx (this,
-                                                                  loc->inode,
-                                                                  NULL);
+                                                               local->loc.inode,
+                                                               NULL);
         } else {
                 LOCK (&priv->read_child_lock);
                 {
-                        local->read_child_index = (++priv->read_child_rr)
-                                % (priv->child_count);
+                        if (priv->hash_mode) {
+                                local->read_child_index = -1;
+                        }
+                        else {
+                                local->read_child_index =
+                                        (++priv->read_child_rr) %
+                                        (priv->child_count);
+                        }
                 }
                 UNLOCK (&priv->read_child_lock);
+                local->cont.lookup.fresh_lookup = _gf_true;
         }
 
         local->child_up = memdup (priv->child_up,
@@ -2023,24 +2500,33 @@ afr_lookup (call_frame_t *frame, xlator_t *this,
         /* By default assume ENOTCONN. On success it will be set to 0. */
         local->op_errno = ENOTCONN;
 
-        local->call_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        ret = afr_lookup_xattr_req_prepare (local, this, xattr_req, loc,
+        ret = dict_get_int32 (xattr_req, "allow-sh-for-running-transaction",
+                              &allow_sh);
+        dict_del (xattr_req, "allow-sh-for-running-transaction");
+        local->allow_sh_for_running_transaction = allow_sh;
+
+        ret = afr_lookup_xattr_req_prepare (local, this, xattr_req, &local->loc,
                                             &gfid_req);
         if (ret) {
                 local->op_errno = -ret;
                 goto out;
         }
         afr_lookup_save_gfid (local->cont.lookup.gfid_req, gfid_req,
-                              loc);
+                              &local->loc);
         local->fop = GF_FOP_LOOKUP;
+        if (priv->choose_local && !priv->did_discovery) {
+                if (gfid_req && __is_root_gfid(gfid_req)) {
+                        local->do_discovery = _gf_true;
+                        priv->did_discovery = _gf_true;
+                }
+        }
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND_COOKIE (frame, afr_lookup_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->lookup,
-                                           loc, local->xattr_req);
+                                           &local->loc, local->xattr_req);
                         if (!--call_count)
                                 break;
                 }
@@ -2134,8 +2620,11 @@ __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
                 goto out;
         }
 
-        INIT_LIST_HEAD (&fd_ctx->paused_calls);
+	pthread_mutex_init (&fd_ctx->delay_lock, NULL);
         INIT_LIST_HEAD (&fd_ctx->entries);
+        fd_ctx->call_child = -1;
+
+        INIT_LIST_HEAD (&fd_ctx->eager_locked);
 
         ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
         if (ret)
@@ -2163,134 +2652,70 @@ afr_fd_ctx_set (xlator_t *this, fd_t *fd)
 /* {{{ flush */
 
 int
-afr_flush_unwind (call_frame_t *frame, xlator_t *this)
+afr_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (flush, main_frame,
-                                  local->op_ret, local->op_errno);
-        }
-
-        return 0;
-}
-
-
-int
-afr_flush_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
         int call_count  = -1;
-        int child_index = (long) cookie;
-        int need_unwind = 0;
 
         local = frame->local;
-        priv  = this->private;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
                 if (op_ret != -1) {
                         if (local->success_count == 0) {
                                 local->op_ret = op_ret;
                         }
                         local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
                 }
 
                 local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                afr_flush_unwind (frame, this);
+	call_count = afr_frame_return (frame);
 
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+	if (call_count == 0)
+		AFR_STACK_UNWIND(flush, frame, local->op_ret,
+				 local->op_errno, NULL);
 
         return 0;
 }
 
-
-int
-afr_flush_wind (call_frame_t *frame, xlator_t *this)
+static int
+afr_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int call_count = -1;
+        int           i      = 0;
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int call_count       = -1;
 
-        local = frame->local;
         priv = this->private;
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
+        local = frame->local;
+        call_count = local->call_count;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_flush_wind_cbk,
+                        STACK_WIND_COOKIE (frame, afr_flush_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->flush,
-                                           local->fd);
-
+                                           local->fd, NULL);
                         if (!--call_count)
                                 break;
+
                 }
         }
 
         return 0;
 }
 
-
-int
-afr_flush_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
 int
-afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
         afr_local_t   *local = NULL;
-        call_frame_t  *transaction_frame = NULL;
+        call_stub_t   *stub = NULL;
         int            ret        = -1;
         int            op_errno   = 0;
 
@@ -2300,47 +2725,27 @@ afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
         priv = this->private;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+	local = frame->local;
 
-        local->op = GF_FOP_FLUSH;
+	ret = afr_local_init(local, priv, &op_errno);
+	if (ret < 0)
+		goto out;
 
-        local->transaction.fop    = afr_flush_wind;
-        local->transaction.done   = afr_flush_done;
-        local->transaction.unwind = afr_flush_unwind;
-
-        local->fd                 = fd_ref (fd);
-
-        local->transaction.main_frame = frame;
-        local->transaction.start  = 0;
-        local->transaction.len    = 0;
-
-        ret = afr_open_fd_fix (transaction_frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
+	local->fd = fd_ref(fd);
+        stub = fop_flush_stub (frame, afr_flush_wrapper, fd, xdata);
+        if (!stub) {
+                ret = -1;
+                op_errno = ENOMEM;
                 goto out;
         }
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
 
+        afr_delayed_changelog_wake_resume (this, fd, stub);
+	ret = 0;
 
-        ret = 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-
-                AFR_STACK_UNWIND (flush, frame, -1, op_errno);
-        }
+	if (ret < 0)
+		AFR_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2354,8 +2759,6 @@ afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
         uint64_t        ctx = 0;
         afr_fd_ctx_t    *fd_ctx = NULL;
         int             ret = 0;
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
 
         ret = fd_ctx_get (fd, this, &ctx);
         if (ret < 0)
@@ -2364,28 +2767,18 @@ afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
         fd_ctx = (afr_fd_ctx_t *)(long) ctx;
 
         if (fd_ctx) {
-                if (fd_ctx->pre_op_done)
-                        GF_FREE (fd_ctx->pre_op_done);
+                GF_FREE (fd_ctx->pre_op_done);
 
-                if (fd_ctx->opened_on)
-                        GF_FREE (fd_ctx->opened_on);
+                GF_FREE (fd_ctx->opened_on);
 
-                if (fd_ctx->locked_on)
-                        GF_FREE (fd_ctx->locked_on);
+                GF_FREE (fd_ctx->locked_on);
 
-                if (fd_ctx->pre_op_piggyback)
-                        GF_FREE (fd_ctx->pre_op_piggyback);
-                list_for_each_entry_safe (paused_call, tmp, &fd_ctx->paused_calls,
-                                          call_list) {
-                        list_del_init (&paused_call->call_list);
-                        GF_FREE (paused_call);
-                }
+                GF_FREE (fd_ctx->pre_op_piggyback);
+                GF_FREE (fd_ctx->lock_piggyback);
 
-                if (fd_ctx->lock_piggyback)
-                        GF_FREE (fd_ctx->lock_piggyback);
+                GF_FREE (fd_ctx->lock_acquired);
 
-                if (fd_ctx->lock_acquired)
-                        GF_FREE (fd_ctx->lock_acquired);
+		pthread_mutex_destroy (&fd_ctx->delay_lock);
 
                 GF_FREE (fd_ctx);
         }
@@ -2423,14 +2816,25 @@ afr_release (xlator_t *this, fd_t *fd)
 /* {{{ fsync */
 
 int
+afr_fsync_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        AFR_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+        return 0;
+}
+
+int
 afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+               struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
         int child_index = (long) cookie;
         int read_child  = 0;
+	call_stub_t *stub = NULL;
 
         local = frame->local;
 
@@ -2446,13 +2850,13 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = 0;
 
                         if (local->success_count == 0) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
                         }
 
                         if (child_index == read_child) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
                         }
 
                         local->success_count++;
@@ -2465,9 +2869,32 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                AFR_STACK_UNWIND (fsync, frame, local->op_ret, local->op_errno,
-                                  &local->cont.fsync.prebuf,
-                                  &local->cont.fsync.postbuf);
+		/* Make a stub out of the frame, and register it
+		   with the waking up post-op. When the call-stub resumes,
+		   we are guaranteed that there was no post-op pending
+		   (i.e changelogs were unset in the server). This is an
+		   essential "guarantee", that fsync() returns only after
+		   completely finishing EVERYTHING, including the delayed
+		   post-op. This guarantee is expected by FUSE graph switching
+		   for example.
+		*/
+		stub = fop_fsync_cbk_stub (frame, afr_fsync_unwind_cbk,
+                                           local->op_ret, local->op_errno,
+                                           &local->cont.inode_wfop.prebuf,
+                                           &local->cont.inode_wfop.postbuf,
+                                           xdata);
+		if (!stub) {
+			AFR_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
+			return 0;
+		}
+
+		/* If no new unstable writes happened between the
+		   time we cleared the unstable write witness flag in afr_fsync
+		   and now, calling afr_delayed_changelog_wake_up() should
+		   wake up and skip over the fsync phase and go straight to
+		   afr_changelog_post_op_now()
+		*/
+		afr_delayed_changelog_wake_resume (this, local->fd, stub);
         }
 
         return 0;
@@ -2476,7 +2903,7 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           int32_t datasync)
+           int32_t datasync, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
@@ -2491,7 +2918,7 @@ afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2502,13 +2929,17 @@ afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         local->fd             = fd_ref (fd);
 
+	if (afr_fd_has_witnessed_unstable_write (this, fd)) {
+		/* don't care. we only wanted to CLEAR the bit */
+	}
+
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND_COOKIE (frame, afr_fsync_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->fsync,
-                                           fd, datasync);
+                                           fd, datasync, xdata);
                         if (!--call_count)
                                 break;
                 }
@@ -2517,7 +2948,7 @@ afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2527,7 +2958,8 @@ out:
 
 int32_t
 afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
@@ -2547,7 +2979,7 @@ afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -2555,7 +2987,7 @@ afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t datasync)
+              int32_t datasync, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
@@ -2570,7 +3002,7 @@ afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2584,7 +3016,7 @@ afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
                         STACK_WIND (frame, afr_fsyncdir_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fsyncdir,
-                                    fd, datasync);
+                                    fd, datasync, xdata);
                         if (!--call_count)
                                 break;
                 }
@@ -2593,7 +3025,7 @@ afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno);
+                AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -2604,7 +3036,7 @@ out:
 int32_t
 afr_xattrop_cbk (call_frame_t *frame, void *cookie,
                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 dict_t *xattr)
+                 dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
@@ -2613,8 +3045,11 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == 0)
+                if (op_ret == 0) {
+                        if (!local->cont.xattrop.xattr)
+                                local->cont.xattrop.xattr = dict_ref (xattr);
                         local->op_ret = 0;
+                }
 
                 local->op_errno = op_errno;
         }
@@ -2624,7 +3059,7 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
-                                  xattr);
+                local->cont.xattrop.xattr, xdata);
 
         return 0;
 }
@@ -2632,7 +3067,7 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr)
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -2647,7 +3082,7 @@ afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2661,7 +3096,7 @@ afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
                         STACK_WIND (frame, afr_xattrop_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->xattrop,
-                                    loc, optype, xattr);
+                                    loc, optype, xattr, xdata);
                         if (!--call_count)
                                 break;
                 }
@@ -2670,7 +3105,7 @@ afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -2681,7 +3116,7 @@ out:
 int32_t
 afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xattr)
+                  dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
 
@@ -2691,8 +3126,12 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == 0)
+                if (op_ret == 0) {
+                        if (!local->cont.fxattrop.xattr)
+                                local->cont.fxattrop.xattr = dict_ref (xattr);
+
                         local->op_ret = 0;
+                }
 
                 local->op_errno = op_errno;
         }
@@ -2702,7 +3141,7 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
-                                  xattr);
+                                  local->cont.fxattrop.xattr, xdata);
 
         return 0;
 }
@@ -2710,7 +3149,7 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr)
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -2725,7 +3164,7 @@ afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2739,7 +3178,7 @@ afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
                         STACK_WIND (frame, afr_fxattrop_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fxattrop,
-                                    fd, optype, xattr);
+                                    fd, optype, xattr, xdata);
                         if (!--call_count)
                                 break;
                 }
@@ -2748,7 +3187,7 @@ afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -2757,7 +3196,7 @@ out:
 
 int32_t
 afr_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -2778,7 +3217,7 @@ afr_inodelk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -2786,7 +3225,8 @@ afr_inodelk_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+             const char *volume, loc_t *loc, int32_t cmd,
+             struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -2801,7 +3241,7 @@ afr_inodelk (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2815,7 +3255,7 @@ afr_inodelk (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, afr_inodelk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->inodelk,
-                                    volume, loc, cmd, flock);
+                                    volume, loc, cmd, flock, xdata);
 
                         if (!--call_count)
                                 break;
@@ -2825,14 +3265,15 @@ afr_inodelk (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (inodelk, frame, -1, op_errno);
+                AFR_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int32_t
 afr_finodelk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -2853,7 +3294,7 @@ afr_finodelk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (finodelk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -2861,7 +3302,8 @@ afr_finodelk_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+              dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -2876,7 +3318,7 @@ afr_finodelk (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2890,7 +3332,7 @@ afr_finodelk (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, afr_finodelk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->finodelk,
-                                    volume, fd, cmd, flock);
+                                    volume, fd, cmd, flock, xdata);
 
                         if (!--call_count)
                                 break;
@@ -2900,15 +3342,14 @@ afr_finodelk (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (finodelk, frame, -1, op_errno);
+                AFR_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int32_t
-afr_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+afr_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
@@ -2928,7 +3369,7 @@ afr_entrylk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (entrylk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -2937,7 +3378,8 @@ afr_entrylk_cbk (call_frame_t *frame, void *cookie,
 int32_t
 afr_entrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, loc_t *loc,
-             const char *basename, entrylk_cmd cmd, entrylk_type type)
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -2952,7 +3394,7 @@ afr_entrylk (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -2966,7 +3408,7 @@ afr_entrylk (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, afr_entrylk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->entrylk,
-                                    volume, loc, basename, cmd, type);
+                                    volume, loc, basename, cmd, type, xdata);
 
                         if (!--call_count)
                                 break;
@@ -2976,7 +3418,7 @@ afr_entrylk (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (entrylk, frame, -1, op_errno);
+                AFR_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -2984,7 +3426,7 @@ out:
 
 int32_t
 afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -3005,7 +3447,7 @@ afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fentrylk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -3014,7 +3456,8 @@ afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
 int32_t
 afr_fentrylk (call_frame_t *frame, xlator_t *this,
               const char *volume, fd_t *fd,
-              const char *basename, entrylk_cmd cmd, entrylk_type type)
+              const char *basename, entrylk_cmd cmd,
+              entrylk_type type, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -3029,7 +3472,7 @@ afr_fentrylk (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -3043,7 +3486,7 @@ afr_fentrylk (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, afr_fentrylk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fentrylk,
-                                    volume, fd, basename, cmd, type);
+                                    volume, fd, basename, cmd, type, xdata);
 
                         if (!--call_count)
                                 break;
@@ -3053,14 +3496,14 @@ afr_fentrylk (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno);
+                AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
         return 0;
 }
 
 int32_t
 afr_statfs_cbk (call_frame_t *frame, void *cookie,
                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                struct statvfs *statvfs)
+                struct statvfs *statvfs, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = 0;
@@ -3091,7 +3534,7 @@ afr_statfs_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->cont.statfs.buf);
+                                  &local->cont.statfs.buf, xdata);
 
         return 0;
 }
@@ -3099,7 +3542,7 @@ afr_statfs_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_statfs (call_frame_t *frame, xlator_t *this,
-            loc_t *loc)
+            loc_t *loc, dict_t *xdata)
 {
         afr_private_t *  priv        = NULL;
         int              child_count = 0;
@@ -3116,7 +3559,7 @@ afr_statfs (call_frame_t *frame, xlator_t *this,
         priv = this->private;
         child_count = priv->child_count;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -3130,7 +3573,7 @@ afr_statfs (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, afr_statfs_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->statfs,
-                                    loc);
+                                    loc, xdata);
                         if (!--call_count)
                                 break;
                 }
@@ -3139,14 +3582,15 @@ afr_statfs (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                   dict_t *xdata)
 {
         afr_local_t * local = NULL;
         int call_count = -1;
@@ -3156,7 +3600,7 @@ afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  lock);
+                                  lock, xdata);
 
         return 0;
 }
@@ -3178,7 +3622,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
 
         if (call_count == 0) {
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
                 return 0;
         }
 
@@ -3192,7 +3636,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
                                     priv->children[i],
                                     priv->children[i]->fops->lk,
                                     local->fd, F_SETLK,
-                                    &local->cont.lk.user_flock);
+                                    &local->cont.lk.user_flock, NULL);
 
                         if (!--call_count)
                                 break;
@@ -3205,7 +3649,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
 
 int32_t
 afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+            int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
@@ -3240,12 +3684,12 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                    priv->children[child_index],
                                    priv->children[child_index]->fops->lk,
                                    local->fd, local->cont.lk.cmd,
-                                   &local->cont.lk.user_flock);
+                                   &local->cont.lk.user_flock, xdata);
         } else if (local->op_ret == -1) {
                 /* all nodes have gone down */
 
                 AFR_STACK_UNWIND (lk, frame, -1, ENOTCONN,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
         } else {
                 /* locking has succeeded on all nodes that are up */
 
@@ -3263,7 +3707,7 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 */
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
         }
 
         return 0;
@@ -3272,7 +3716,7 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 afr_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int32_t cmd, struct gf_flock *flock)
+        fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
@@ -3286,7 +3730,7 @@ afr_lk (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -3310,12 +3754,12 @@ afr_lk (call_frame_t *frame, xlator_t *this,
         STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
                            priv->children[i],
                            priv->children[i]->fops->lk,
-                           fd, cmd, flock);
+                           fd, cmd, flock, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -3331,8 +3775,7 @@ afr_forget (xlator_t *this, inode_t *inode)
                 goto out;
 
         ctx = (afr_inode_ctx_t *)(long)ctx_addr;
-        if (ctx->fresh_children)
-                GF_FREE (ctx->fresh_children);
+        GF_FREE (ctx->fresh_children);
         GF_FREE (ctx);
 out:
         return 0;
@@ -3399,7 +3842,7 @@ find_child_index (xlator_t *this, xlator_t *child)
 
 int32_t
 afr_notify (xlator_t *this, int32_t event,
-            void *data, ...)
+            void *data, void *data2)
 {
         afr_private_t   *priv               = NULL;
         int             i                   = -1;
@@ -3412,12 +3855,22 @@ afr_notify (xlator_t *this, int32_t event,
         int             ret                 = -1;
         int             call_psh            = 0;
         int             up_child            = AFR_ALL_CHILDREN;
+        dict_t          *input              = NULL;
+        dict_t          *output             = NULL;
 
         priv = this->private;
 
         if (!priv)
                 return 0;
 
+        /*
+         * We need to reset this in case children come up in "staggered"
+         * fashion, so that we discover a late-arriving local subvolume.  Note
+         * that we could end up issuing N lookups to the first subvolume, and
+         * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+         */
+        priv->did_discovery = _gf_false;
+
         had_heard_from_all = 1;
         for (i = 0; i < priv->child_count; i++) {
                 if (!priv->last_event[i]) {
@@ -3445,9 +3898,19 @@ afr_notify (xlator_t *this, int32_t event,
         case GF_EVENT_CHILD_UP:
                 LOCK (&priv->lock);
                 {
+                        /*
+                         * This only really counts if the child was never up
+                         * (value = -1) or had been down (value = 0).  See
+                         * comment at GF_EVENT_CHILD_DOWN for a more detailed
+                         * explanation.
+                         */
+                        if (priv->child_up[idx] != 1) {
+                                priv->up_count++;
+                        }
                         priv->child_up[idx] = 1;
-                        priv->up_count++;
 
+                        call_psh = 1;
+                        up_child = idx;
                         for (i = 0; i < priv->child_count; i++)
                                 if (priv->child_up[i] == 1)
                                         up_children++;
@@ -3457,12 +3920,6 @@ afr_notify (xlator_t *this, int32_t event,
                                         "going online.", ((xlator_t *)data)->name);
                         } else {
                                 event = GF_EVENT_CHILD_MODIFIED;
-                                gf_log (this->name, GF_LOG_INFO, "subvol %d came up, "
-                                        "start crawl", idx);
-                                if (had_heard_from_all) {
-                                        call_psh = 1;
-                                        up_child = idx;
-                                }
                         }
 
                         priv->last_event[idx] = event;
@@ -3474,8 +3931,22 @@ afr_notify (xlator_t *this, int32_t event,
         case GF_EVENT_CHILD_DOWN:
                 LOCK (&priv->lock);
                 {
+                        /*
+                         * If a brick is down when we start, we'll get a
+                         * CHILD_DOWN to indicate its initial state.  There
+                         * was never a CHILD_UP in this case, so if we
+                         * increment "down_count" the difference between than
+                         * and "up_count" will no longer be the number of
+                         * children that are currently up.  This has serious
+                         * implications e.g. for quorum enforcement, so we
+                         * don't increment these values unless the event
+                         * represents an actual state transition between "up"
+                         * (value = 1) and anything else.
+                         */
+                        if (priv->child_up[idx] == 1) {
+                                priv->down_count++;
+                        }
                         priv->child_up[idx] = 0;
-                        priv->down_count++;
 
                         for (i = 0; i < priv->child_count; i++)
                                 if (priv->child_up[i] == 0)
@@ -3503,10 +3974,11 @@ afr_notify (xlator_t *this, int32_t event,
 
                 break;
 
-        case GF_EVENT_TRIGGER_HEAL:
-                gf_log (this->name, GF_LOG_INFO, "Self-heal was triggered"
-                        " manually. Start crawling");
-                call_psh = 1;
+        case GF_EVENT_TRANSLATOR_OP:
+                input = data;
+                output = data2;
+                ret = afr_xl_op (this, input, output);
+                goto out;
                 break;
 
         default:
@@ -3551,18 +4023,13 @@ afr_notify (xlator_t *this, int32_t event,
                         }
                 }
                 UNLOCK (&priv->lock);
-                if (up_children > 1) {
-                        gf_log (this->name, GF_LOG_INFO, "All subvolumes came "
-                                "up, start crawl");
-                        call_psh = 1;
-                }
         }
 
         ret = 0;
         if (propagate)
                 ret = default_notify (this, event, data);
-        if (call_psh)
-                afr_proactive_self_heal (this, up_child);
+        if (call_psh && priv->shd.iamshd)
+                afr_proactive_self_heal ((void*) (long) up_child);
 
 out:
         return ret;
@@ -3623,6 +4090,17 @@ afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
                 goto out;
         }
 
+        local->transaction.postop_piggybacked = GF_CALLOC (priv->child_count,
+							   sizeof (int),
+							   gf_afr_mt_int32_t);
+        if (!local->transaction.postop_piggybacked) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+
+	local->append_write = _gf_false;
+
         ret = 0;
 out:
         return ret;
@@ -3634,16 +4112,6 @@ afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
 {
         int             ret = -ENOMEM;
 
-        lk->inode_locked_nodes = GF_CALLOC (sizeof (*lk->inode_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->inode_locked_nodes)
-                goto out;
-
-        lk->entry_locked_nodes = GF_CALLOC (sizeof (*lk->entry_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->entry_locked_nodes)
-                goto out;
-
         lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
                                       child_count, gf_afr_mt_char);
         if (NULL == lk->locked_nodes)
@@ -3702,6 +4170,21 @@ out:
 }
 
 int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count)
+{
+        int             ret = -ENOMEM;
+
+        lk->domain = dom;
+        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
+                                      child_count, gf_afr_mt_char);
+        if (NULL == lk->locked_nodes)
+                goto out;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
 afr_transaction_local_init (afr_local_t *local, xlator_t *this)
 {
         int            child_up_count = 0;
@@ -3714,6 +4197,14 @@ afr_transaction_local_init (afr_local_t *local, xlator_t *this)
         if (ret < 0)
                 goto out;
 
+        if ((local->transaction.type == AFR_DATA_TRANSACTION) ||
+            (local->transaction.type == AFR_METADATA_TRANSACTION)) {
+                ret = afr_inodelk_init (&local->internal_lock.inodelk[0],
+                                        this->name, priv->child_count);
+                if (ret < 0)
+                        goto out;
+        }
+
         ret = -ENOMEM;
         child_up_count = afr_up_children_count (local->child_up,
                                                 priv->child_count);
@@ -3735,14 +4226,6 @@ afr_transaction_local_init (afr_local_t *local, xlator_t *this)
         if (!local->fresh_children)
                 goto out;
 
-        if (local->fd) {
-                local->fd_open_on = GF_CALLOC (sizeof (*local->fd_open_on),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-                if (!local->fd_open_on)
-                        goto out;
-        }
-
         local->transaction.pre_op = GF_CALLOC (sizeof (*local->transaction.pre_op),
                                                priv->child_count,
                                                gf_afr_mt_char);
@@ -3754,11 +4237,12 @@ afr_transaction_local_init (afr_local_t *local, xlator_t *this)
         if (!local->pending)
                 goto out;
 
-        local->transaction.child_errno =
-                GF_CALLOC (sizeof (*local->transaction.child_errno),
-                           priv->child_count,
-                           gf_afr_mt_int32_t);
-        local->transaction.erase_pending = 1;
+        local->transaction.txn_changelog = afr_matrix_create (priv->child_count,
+                                                           AFR_NUM_CHANGE_LOGS);
+        if (!local->transaction.txn_changelog)
+                goto out;
+
+	INIT_LIST_HEAD (&local->transaction.eager_locked);
 
         ret = 0;
 out:
@@ -3854,7 +4338,7 @@ afr_set_low_priority (call_frame_t *frame)
 
 int
 afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags, int32_t wbflags)
+                      int flags)
 {
         int             ret = 0;
         uint64_t        ctx = 0;
@@ -3879,7 +4363,6 @@ afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
         fd_ctx->opened_on[child] = AFR_FD_OPENED;
         if (!IA_ISDIR (fd->inode->ia_type)) {
                 fd_ctx->flags            = flags;
-                fd_ctx->wbflags          = wbflags;
         }
         ret = 0;
 out:
@@ -3932,6 +4415,32 @@ afr_priv_destroy (afr_private_t *priv)
                 goto out;
         inode_unref (priv->root_inode);
         GF_FREE (priv->shd.pos);
+        GF_FREE (priv->shd.pending);
+        GF_FREE (priv->shd.inprogress);
+//        for (i = 0; i < priv->child_count; i++)
+//                if (priv->shd.timer && priv->shd.timer[i])
+//                        gf_timer_call_cancel (this->ctx, priv->shd.timer[i]);
+        GF_FREE (priv->shd.timer);
+
+        if (priv->shd.healed)
+                eh_destroy (priv->shd.healed);
+
+        if (priv->shd.heal_failed)
+                eh_destroy (priv->shd.heal_failed);
+
+        if (priv->shd.split_brain)
+                eh_destroy (priv->shd.split_brain);
+
+        for (i = 0; i < priv->child_count; i++)
+        {
+                if (priv->shd.statistics[i])
+                        eh_destroy (priv->shd.statistics[i]);
+        }
+
+        GF_FREE (priv->shd.statistics);
+
+        GF_FREE (priv->shd.crawl_events);
+
         GF_FREE (priv->last_event);
         if (priv->pending_key) {
                 for (i = 0; i < priv->child_count; i++)
@@ -3958,3 +4467,125 @@ xlator_subvolume_count (xlator_t *this)
                 i++;
         return i;
 }
+
+inline gf_boolean_t
+afr_is_errno_set (int *child_errno, int child)
+{
+        return child_errno[child];
+}
+
+inline gf_boolean_t
+afr_is_errno_unset (int *child_errno, int child)
+{
+        return !afr_is_errno_set (child_errno, child);
+}
+
+void
+afr_prepare_new_entry_pending_matrix (int32_t **pending,
+                                      gf_boolean_t (*is_pending) (int *, int),
+                                      int *ctx, struct iatt *buf,
+                                      unsigned int child_count)
+{
+        int midx = 0;
+        int idx  = 0;
+        int i    = 0;
+
+        midx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
+        if (IA_ISDIR (buf->ia_type))
+                idx = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
+        else if (IA_ISREG (buf->ia_type))
+                idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+        else
+                idx = -1;
+        for (i = 0; i < child_count; i++) {
+                if (is_pending (ctx, i)) {
+                        pending[i][midx] = hton32 (1);
+                        if (idx == -1)
+                                continue;
+                        pending[i][idx] = hton32 (1);
+                }
+        }
+}
+
+gf_boolean_t
+afr_is_fd_fixable (fd_t *fd)
+{
+        if (!fd || !fd->inode)
+                return _gf_false;
+        else if (fd_is_anonymous (fd))
+                return _gf_false;
+        else if (uuid_is_null (fd->inode->gfid))
+                return _gf_false;
+
+        return _gf_true;
+}
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        inode_t         *inode = NULL;
+        afr_inode_ctx_t *ctx   = NULL;
+
+        local = frame->local;
+
+        if (local->fd)
+                inode = local->fd->inode;
+        else
+                inode = local->loc.inode;
+
+        if (!inode)
+                return;
+
+        LOCK (&inode->lock);
+        {
+                ctx = __afr_inode_ctx_get (inode, this);
+                ctx->open_fd_count = local->open_fd_count;
+        }
+        UNLOCK (&inode->lock);
+}
+
+int
+afr_initialise_statistics (xlator_t *this)
+{
+        afr_private_t       *priv = NULL;
+        int                 ret = -1;
+        int                 i = 0;
+        int                 child_count = 0;
+        eh_t                *stats_per_brick = NULL;
+        shd_crawl_event_t   ***shd_crawl_events = NULL;
+        priv = this->private;
+
+        priv->shd.statistics = GF_CALLOC (sizeof(eh_t *), priv->child_count,
+                                          gf_common_mt_eh_t);
+        if (!priv->shd.statistics) {
+                ret = -1;
+                goto out;
+        }
+        child_count = priv->child_count;
+        for (i=0; i < child_count ; i++) {
+                stats_per_brick = eh_new (AFR_STATISTICS_HISTORY_SIZE,
+                                          _gf_false,
+                                          _destroy_crawl_event_data);
+                if (!stats_per_brick) {
+                        ret = -1;
+                        goto out;
+                }
+                priv->shd.statistics[i] = stats_per_brick;
+
+        }
+
+        shd_crawl_events = (shd_crawl_event_t***)(&priv->shd.crawl_events);
+        *shd_crawl_events  = GF_CALLOC (sizeof(shd_crawl_event_t*),
+                                        priv->child_count,
+                                        gf_afr_mt_shd_crawl_event_t);
+
+        if (!priv->shd.crawl_events) {
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+
+}
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index 389515e3c..689dd84e6 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -51,7 +42,7 @@
 
 int
 afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this, int32_t op_ret,
-                           int32_t op_errno)
+                           int32_t op_errno, int32_t sh_failed)
 {
         afr_local_t *local  = NULL;
 
@@ -60,7 +51,7 @@ afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this, int32_t op_ret,
         afr_set_opendir_done (this, local->fd->inode);
 
         AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd);
+                          local->op_errno, local->fd, NULL);
 
         return 0;
 }
@@ -99,7 +90,7 @@ __checksums_differ (uint32_t *checksum, int child_count,
 int32_t
 afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
                              xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             gf_dirent_t *entries)
+                             gf_dirent_t *entries, dict_t *xdata)
 {
         afr_private_t *   priv        = NULL;
         afr_local_t *     local       = NULL;
@@ -137,7 +128,7 @@ afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
         }
 
         list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                entry_cksum = gf_rsync_weak_checksum (entry->d_name,
+                entry_cksum = gf_rsync_weak_checksum ((unsigned char *)entry->d_name,
                                                       strlen (entry->d_name));
                 local->cont.opendir.checksum[child_index] ^= entry_cksum;
         }
@@ -152,7 +143,7 @@ afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
                            (void *) (long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->readdir,
-                           local->fd, 131072, last_offset);
+                           local->fd, 131072, last_offset, NULL);
 
         return 0;
 
@@ -175,7 +166,7 @@ out:
                         afr_set_opendir_done (this, inode);
 
                         AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                          local->op_errno, local->fd);
+                                          local->op_errno, local->fd, NULL);
                 }
         }
 
@@ -208,7 +199,7 @@ afr_examine_dir (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->readdir,
-                                           local->fd, 131072, 0);
+                                           local->fd, 131072, 0, NULL);
 
                         if (!--call_count)
                                 break;
@@ -222,7 +213,7 @@ afr_examine_dir (call_frame_t *frame, xlator_t *this)
 int32_t
 afr_opendir_cbk (call_frame_t *frame, void *cookie,
                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 fd_t *fd)
+                 fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv              = NULL;
         afr_local_t   *local             = NULL;
@@ -242,8 +233,7 @@ afr_opendir_cbk (call_frame_t *frame, void *cookie,
         {
                 if (op_ret >= 0) {
                         local->op_ret = op_ret;
-                        ret = afr_child_fd_ctx_set (this, fd, child_index,
-                                                    0, 0);
+                        ret = afr_child_fd_ctx_set (this, fd, child_index, 0);
                         if (ret) {
                                 local->op_ret = -1;
                                 local->op_errno = -ret;
@@ -263,7 +253,7 @@ unlock:
                         goto out;
 
                 if (!afr_is_opendir_done (this, local->fd->inode) &&
-                    up_children_count > 1) {
+                    up_children_count > 1 && priv->entry_self_heal) {
 
                         /*
                          * This is the first opendir on this inode. We need
@@ -291,7 +281,7 @@ unlock:
 
 out:
         AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd);
+                          local->op_errno, local->fd, NULL);
 
         return 0;
 }
@@ -317,7 +307,7 @@ afr_opendir (call_frame_t *frame, xlator_t *this,
 
         child_count = priv->child_count;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -336,7 +326,7 @@ afr_opendir (call_frame_t *frame, xlator_t *this,
                                            (void*) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->opendir,
-                                           loc, fd);
+                                           loc, fd, NULL);
 
                         if (!--call_count)
                                 break;
@@ -346,7 +336,7 @@ afr_opendir (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd);
+                AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
 
         return 0;
 }
@@ -368,85 +358,6 @@ struct entry_name {
         struct list_head list;
 };
 
-
-static gf_boolean_t
-remembered_name (const char *name, struct list_head *entries)
-{
-        struct entry_name *e   = NULL;
-        gf_boolean_t       ret = _gf_false;
-
-        list_for_each_entry (e, entries, list) {
-                if (!strcmp (name, e->name)) {
-                        ret = _gf_true;
-                        goto out;
-                }
-        }
-
-out:
-        return ret;
-}
-
-
-static void
-afr_remember_entries (gf_dirent_t *entries, fd_t *fd)
-{
-        struct entry_name *n      = NULL;
-        gf_dirent_t       *entry  = NULL;
-        int                ret    = 0;
-        uint64_t           ctx    = 0;
-        afr_fd_ctx_t      *fd_ctx = NULL;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return;
-        }
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                n = GF_CALLOC (1, sizeof (*n), gf_afr_mt_entry_name);
-                n->name = gf_strdup (entry->d_name);
-                INIT_LIST_HEAD (&n->list);
-
-                list_add (&n->list, &fd_ctx->entries);
-        }
-}
-
-
-static off_t
-afr_filter_entries (gf_dirent_t *entries, fd_t *fd)
-{
-        gf_dirent_t  *entry  = NULL;
-        gf_dirent_t  *tmp    = NULL;
-        int           ret    = 0;
-        uint64_t      ctx    = 0;
-        afr_fd_ctx_t *fd_ctx = NULL;
-        off_t         offset = 0;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return -1;
-        }
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                offset = entry->d_off;
-
-                if (remembered_name (entry->d_name, &fd_ctx->entries)) {
-                        list_del (&entry->list);
-                        GF_FREE (entry);
-                }
-        }
-
-        return offset;
-}
-
-
 static void
 afr_forget_entries (fd_t *fd)
 {
@@ -472,159 +383,55 @@ afr_forget_entries (fd_t *fd)
         }
 }
 
-
-int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 gf_dirent_t *entries)
+static void
+afr_readdir_filter_trash_dir (gf_dirent_t *entries, fd_t *fd)
 {
-        afr_local_t *   local       = NULL;
         gf_dirent_t *   entry       = NULL;
         gf_dirent_t *   tmp         = NULL;
 
-        local = frame->local;
-
-        if (op_ret == -1)
-                goto out;
-
         list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                if ((local->fd->inode == local->fd->inode->table->root)
-                    && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
+                if (__is_root_gfid (fd->inode->gfid) &&
+                    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
                         list_del_init (&entry->list);
                         GF_FREE (entry);
                 }
         }
-
-out:
-        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries);
-
-        return 0;
 }
 
-
 int32_t
-afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+afr_readdir_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno,
+                 gf_dirent_t *entries, dict_t *xdata)
 {
-        afr_private_t *  priv            = NULL;
-        afr_local_t *    local           = NULL;
-        xlator_t **      children        = NULL;
-        int32_t          next_call_child = -1;
-        int              ret             = 0;
-        gf_dirent_t *    entry           = NULL;
-        gf_dirent_t *    tmp             = NULL;
-        int32_t          *last_index     = NULL;
-        int32_t          read_child      = -1;
-        int32_t         *fresh_children   = NULL;
-        uint64_t         ctx             = 0;
-        afr_fd_ctx_t    *fd_ctx          = NULL;
-        off_t            offset          = 0;
-        int32_t         call_child       = -1;
+        afr_local_t     *local = NULL;
 
-        priv     = this->private;
-        children = priv->children;
+        if (op_ret == -1)
+                goto out;
 
         local = frame->local;
+        afr_readdir_filter_trash_dir (entries, local->fd);
 
-        read_child = (long) cookie;
-        last_index = &local->cont.readdir.last_index;
-        fresh_children = local->fresh_children;
-
-        /* the value of the last_index changes if afr_next_call_child is
-         * called. So to find the call_child of this callback use last_index
-         * before the next_call_child call.
-         */
-        if (*last_index == -1)
-                call_child = read_child;
-        else
-                call_child = fresh_children[*last_index];
-
-        if (priv->strict_readdir) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not get fd ctx for fd=%p", local->fd);
-                        op_ret   = -1;
-                        op_errno = -ret;
-                        goto out;
-                }
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                if (op_ret == -1) {
-                        next_call_child = afr_next_call_child (fresh_children,
-                                                               local->child_up,
-                                                               priv->child_count,
-                                                               last_index,
-                                                               read_child);
-                        if (next_call_child < 0)
-                                goto out;
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "starting readdir afresh on child %d, offset %"PRId64,
-                                next_call_child, (uint64_t) 0);
-
-                        fd_ctx->failed_over = _gf_true;
-
-                        STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                           (void *) (long) read_child,
-                                           children[next_call_child],
-                                           children[next_call_child]->fops->readdirp,
-                                           local->fd,
-                                           local->cont.readdir.size, 0,
-                                           local->cont.readdir.dict);
-                        return 0;
-                }
-        }
-
-        if (op_ret != -1) {
-                list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                        if ((local->fd->inode == local->fd->inode->table->root)
-                            && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-                                list_del_init (&entry->list);
-                                GF_FREE (entry);
-                        }
-                }
-        }
-
-        if (priv->strict_readdir) {
-                if (fd_ctx->failed_over) {
-                        if (list_empty (&entries->list)) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "no entries found");
-                                goto out;
-                        }
-
-                        offset = afr_filter_entries (entries, local->fd);
+out:
+        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries, NULL);
+        return 0;
+}
 
-                        afr_remember_entries (entries, local->fd);
 
-                        if (list_empty (&entries->list)) {
-                                /* All the entries we got were duplicate. We
-                                   shouldn't send an empty list now, because
-                                   that will make the application stop reading. So
-                                   try to get more entries */
+int32_t
+afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                  dict_t *xdata)
+{
+        afr_local_t     *local = NULL;
 
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "trying to fetch non-duplicate entries "
-                                        "from offset %"PRId64", child %s",
-                                        offset, children[call_child]->name);
+        if (op_ret == -1)
+                goto out;
 
-                                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                                   (void *) (long) read_child,
-                                                   children[call_child],
-                                                   children[call_child]->fops->readdirp,
-                                                   local->fd, local->cont.readdir.size, offset,
-                                                   local->cont.readdir.dict);
-                                return 0;
-                        }
-                } else {
-                        afr_remember_entries (entries, local->fd);
-                }
-        }
+        local = frame->local;
+        afr_readdir_filter_trash_dir (entries, local->fd);
 
 out:
-        AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries);
-
+        AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, NULL);
         return 0;
 }
 
@@ -632,15 +439,14 @@ int32_t
 afr_do_readdir (call_frame_t *frame, xlator_t *this,
                 fd_t *fd, size_t size, off_t offset, int whichop, dict_t *dict)
 {
-        afr_private_t *  priv       = NULL;
-        xlator_t **      children   = NULL;
-        int              call_child = 0;
-        afr_local_t     *local      = NULL;
-        uint64_t         ctx        = 0;
-        afr_fd_ctx_t    *fd_ctx     = NULL;
-        int              ret        = -1;
-        int32_t          op_errno   = 0;
-        uint64_t         read_child = 0;
+        afr_private_t *priv      = NULL;
+        xlator_t      **children = NULL;
+        int           call_child = 0;
+        afr_local_t   *local     = NULL;
+        afr_fd_ctx_t  *fd_ctx    = NULL;
+        int           ret        = -1;
+        int32_t       op_errno   = 0;
+        uint64_t      read_child = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -649,7 +455,7 @@ afr_do_readdir (call_frame_t *frame, xlator_t *this,
         priv     = this->private;
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -665,48 +471,38 @@ afr_do_readdir (call_frame_t *frame, xlator_t *this,
         read_child = afr_inode_get_read_ctx (this, fd->inode,
                                              local->fresh_children);
         ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readdir.last_index);
+                                  local->fresh_children,
+                                  &call_child,
+                                  &local->cont.readdir.last_index);
         if (ret < 0) {
                 op_errno = -ret;
                 goto out;
         }
 
+        fd_ctx  = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx) {
+                op_errno = EBADF;
+                goto out;
+        }
+
+        if ((offset == 0) || (fd_ctx->call_child == -1)) {
+                fd_ctx->call_child = call_child;
+        } else if ((priv->readdir_failover == _gf_false) &&
+                   (call_child != fd_ctx->call_child)) {
+                op_errno = EBADF;
+                goto out;
+        }
+
         local->fd                  = fd_ref (fd);
         local->cont.readdir.size   = size;
         local->cont.readdir.dict   = (dict)? dict_ref (dict) : NULL;
 
-        if (priv->strict_readdir) {
-                ret = fd_ctx_get (fd, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not get fd ctx for fd=%p", fd);
-                        op_errno = -ret;
-                        goto out;
-                }
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                if (fd_ctx->last_tried != call_child) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "first up child has changed from %d to %d, "
-                                "restarting readdir from offset 0",
-                                fd_ctx->last_tried, call_child);
-
-                        fd_ctx->failed_over = _gf_true;
-                        offset = 0;
-                }
-
-                fd_ctx->last_tried = call_child;
-        }
-
         if (whichop == GF_FOP_READDIR)
                 STACK_WIND_COOKIE (frame, afr_readdir_cbk,
                                    (void *) (long) call_child,
                                    children[call_child],
                                    children[call_child]->fops->readdir, fd,
-                                   size, offset);
+                                   size, offset, dict);
         else
                 STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
                                    (void *) (long) call_child,
@@ -714,19 +510,18 @@ afr_do_readdir (call_frame_t *frame, xlator_t *this,
                                    children[call_child]->fops->readdirp, fd,
                                    size, offset, dict);
 
-        ret = 0;
+        return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL);
+        AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset)
+             off_t offset, dict_t *xdata)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, NULL);
+        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
         return 0;
 }
 
diff --git a/xlators/cluster/afr/src/afr-dir-read.h b/xlators/cluster/afr/src/afr-dir-read.h
index 7e50a1c8c..09456d159 100644
--- a/xlators/cluster/afr/src/afr-dir-read.h
+++ b/xlators/cluster/afr/src/afr-dir-read.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_READ_H__
@@ -23,14 +14,14 @@
 
 int32_t
 afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd);
+	     loc_t *loc, fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_releasedir (xlator_t *this, fd_t *fd);
 
 int32_t
 afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset);
+	     fd_t *fd, size_t size, off_t offset, dict_t *xdata);
 
 
 int32_t
@@ -39,7 +30,7 @@ afr_readdirp (call_frame_t *frame, xlator_t *this,
 
 int32_t
 afr_checksum (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags);
+	      loc_t *loc, int32_t flags, dict_t *xdata);
 
 
 #endif /* __DIR_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index 91aa2a9e7..1943b719b 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -47,31 +38,222 @@
 #include "afr.h"
 #include "afr-transaction.h"
 
-void
-afr_build_parent_loc (loc_t *parent, loc_t *child)
+int
+afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
 {
-        char *tmp = NULL;
+        int     ret = -1;
+        char    *child_path = NULL;
 
         if (!child->parent) {
-                //this should never be called with root as the child
-                GF_ASSERT (0);
-                loc_copy (parent, child);
-                return;
+                if (op_errno)
+                        *op_errno = EINVAL;
+                goto out;
         }
 
-        tmp = gf_strdup (child->path);
-        parent->path   = gf_strdup (dirname (tmp));
-        GF_FREE (tmp);
+        child_path = gf_strdup (child->path);
+        if (!child_path) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+        parent->path = gf_strdup( dirname (child_path) );
+	if (!parent->path) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+        parent->inode  = inode_ref (child->parent);
+        uuid_copy (parent->gfid, child->pargfid);
 
-        parent->name   = strrchr (parent->path, '/');
-        if (parent->name)
-                parent->name++;
+        ret = 0;
+out:
+	GF_FREE(child_path);
 
-        parent->inode  = inode_ref (child->parent);
-        parent->parent = inode_parent (parent->inode, 0, NULL);
+        return ret;
+}
 
-        if (!uuid_is_null (child->pargfid))
-                uuid_copy (parent->gfid, child->pargfid);
+void
+__dir_entry_fop_common_cbk (call_frame_t *frame, int child_index,
+                            xlator_t *this, int32_t op_ret,
+                            int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, struct iatt *preparent,
+                            struct iatt *postparent, struct iatt *prenewparent,
+                            struct iatt *postnewparent)
+{
+        afr_local_t     *local          = NULL;
+
+        local = frame->local;
+
+        if (afr_fop_failed (op_ret, op_errno))
+                afr_transaction_fop_failed (frame, this, child_index);
+
+        if (op_ret > -1) {
+                local->op_ret = op_ret;
+
+                if ((local->success_count == 0) ||
+                    (child_index == local->read_child_index)) {
+                        local->cont.dir_fop.preparent      = *preparent;
+                        local->cont.dir_fop.postparent     = *postparent;
+                        if (buf)
+                                local->cont.dir_fop.buf            = *buf;
+                        if (prenewparent)
+                             local->cont.dir_fop.prenewparent  = *prenewparent;
+                        if (postnewparent)
+                             local->cont.dir_fop.postnewparent = *postnewparent;
+                }
+
+                local->cont.dir_fop.inode = inode;
+
+                local->fresh_children[local->success_count] = child_index;
+                local->success_count++;
+                local->child_errno[child_index] = 0;
+        } else {
+                local->child_errno[child_index] = op_errno;
+        }
+
+        local->op_errno = op_errno;
+}
+
+int
+afr_mark_new_entry_changelog_cbk (call_frame_t *frame, void *cookie,
+                                  xlator_t *this,
+                                  int32_t op_ret, int32_t op_errno,
+                                  dict_t *xattr, dict_t *xdata)
+{
+        int     call_count = 0;
+
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+                AFR_STACK_DESTROY (frame);
+        }
+        return 0;
+}
+
+void
+afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t  *new_frame  = NULL;
+        afr_local_t   *local      = NULL;
+        afr_local_t   *new_local  = NULL;
+        afr_private_t *priv       = NULL;
+        dict_t        **xattr     = NULL;
+        int32_t       **changelog = NULL;
+        int           i           = 0;
+        GF_UNUSED int op_errno    = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame) {
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (new_frame->local, out);
+        new_local = new_frame->local;
+        changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
+        if (!changelog)
+                goto out;
+
+        xattr = GF_CALLOC (priv->child_count, sizeof (*xattr),
+                           gf_afr_mt_dict_t);
+        if (!xattr)
+                goto out;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_errno[i])
+                        continue;
+                xattr[i] = dict_new ();
+                if (!xattr[i])
+                        goto out;
+        }
+
+        afr_prepare_new_entry_pending_matrix (changelog,
+                                              afr_is_errno_set,
+                                              local->child_errno,
+                                              &local->cont.dir_fop.buf,
+                                              priv->child_count);
+
+        new_local->pending = changelog;
+        uuid_copy (new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+        new_local->loc.inode = inode_ref (local->cont.dir_fop.inode);
+        new_local->call_count = local->success_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_errno[i])
+                        continue;
+
+                afr_set_pending_dict (priv, xattr[i], changelog, i, LOCAL_LAST);
+                STACK_WIND_COOKIE (new_frame, afr_mark_new_entry_changelog_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->xattrop,
+                                   &new_local->loc, GF_XATTROP_ADD_ARRAY,
+                                   xattr[i], NULL);
+        }
+        new_frame = NULL;
+out:
+        if (new_frame)
+                AFR_STACK_DESTROY (new_frame);
+        afr_xattr_array_destroy (xattr, priv->child_count);
+        return;
+}
+
+gf_boolean_t
+afr_is_new_entry_changelog_needed (glusterfs_fop_t fop)
+{
+        glusterfs_fop_t fops[]   = {GF_FOP_CREATE, GF_FOP_MKNOD, GF_FOP_NULL};
+        int             i        = 0;
+
+        for (i = 0; fops[i] != GF_FOP_NULL; i++) {
+                if (fop == fops[i])
+                        return _gf_true;
+        }
+        return _gf_false;
+}
+
+void
+afr_dir_fop_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+
+        if (local->op_ret < 0)
+                goto out;
+
+        if (local->success_count == priv->child_count)
+                goto out;
+
+        if (!afr_is_new_entry_changelog_needed (local->op))
+                goto out;
+
+        afr_mark_new_entry_changelog (frame, this);
+
+out:
+        return;
+}
+
+void
+afr_dir_fop_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local          = NULL;
+        afr_private_t   *priv           = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+
+        if (local->cont.dir_fop.inode == NULL)
+                goto done;
+        afr_set_read_ctx_from_policy (this, local->cont.dir_fop.inode,
+                                      local->fresh_children,
+                                      local->read_child_index,
+                                      priv->read_child,
+                                      local->cont.dir_fop.buf.ia_gfid);
+done:
+        local->transaction.unwind (frame, this);
+        afr_dir_fop_mark_entry_pending_changelog (frame, this);
+        local->transaction.resume (frame, this);
 }
 
 /* {{{ create */
@@ -81,7 +263,6 @@ afr_create_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt  *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -95,18 +276,14 @@ afr_create_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.create.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.create.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.create.buf;
-                }
-
                 AFR_STACK_UNWIND (create, main_frame,
                                   local->op_ret, local->op_errno,
                                   local->cont.create.fd,
-                                  local->cont.create.inode,
-                                  unwind_buf, &local->cont.create.preparent,
-                                  &local->cont.create.postparent);
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  local->xdata_rsp);
         }
 
         return 0;
@@ -117,32 +294,24 @@ int
 afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
                      fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
         afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
         uint64_t        ctx = 0;
         afr_fd_ctx_t    *fd_ctx = NULL;
         int             ret = 0;
         int             call_count = -1;
         int             child_index = -1;
-        int32_t         *fresh_children = NULL;
 
         local = frame->local;
-        priv  = this->private;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret = op_ret;
-
+                if (op_ret > -1) {
                         ret = afr_fd_ctx_set (this, fd);
-
                         if (ret < 0) {
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "could not set ctx on fd=%p", fd);
@@ -153,7 +322,6 @@ afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         }
 
                         ret = fd_ctx_get (fd, this, &ctx);
-
                         if (ret < 0) {
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "could not get fd ctx for fd=%p", fd);
@@ -167,23 +335,14 @@ afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
                         fd_ctx->flags                  = local->cont.create.flags;
 
-                        if (local->success_count == 0)
-                                local->cont.create.buf        = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.create.read_child_buf = *buf;
-                                local->cont.create.preparent      = *preparent;
-                                local->cont.create.postparent     = *postparent;
-                        }
-
-                        local->cont.create.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
+                        if (local->success_count == 0) {
+				if (xdata)
+					local->xdata_rsp = dict_ref(xdata);
+			}
                 }
-
-                local->op_errno = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
         }
 
 unlock:
@@ -191,15 +350,8 @@ unlock:
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -235,8 +387,9 @@ afr_create_wind (call_frame_t *frame, xlator_t *this)
                                            &local->loc,
                                            local->cont.create.flags,
                                            local->cont.create.mode,
+                                           local->umask,
                                            local->cont.create.fd,
-                                           local->cont.create.params);
+                                           local->xdata_req);
                         if (!--call_count)
                                 break;
                 }
@@ -264,13 +417,14 @@ afr_create_done (call_frame_t *frame, xlator_t *this)
 int
 afr_create (call_frame_t *frame, xlator_t *this,
             loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params)
+            mode_t umask, fd_t *fd, dict_t *params)
 {
-        afr_private_t  *priv  = NULL;
-        afr_local_t    *local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int             ret = -1;
-        int             op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -286,7 +440,7 @@ afr_create (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -302,22 +456,41 @@ afr_create (call_frame_t *frame, xlator_t *this,
         }
         UNLOCK (&priv->read_child_lock);
 
+        local->op                = GF_FOP_CREATE;
         local->cont.create.flags = flags;
         local->cont.create.mode  = mode;
         local->cont.create.fd    = fd_ref (fd);
+        local->umask  = umask;
         if (params)
-                local->cont.create.params = dict_ref (params);
+                local->xdata_req = dict_ref (params);
 
         local->transaction.fop    = afr_create_wind;
         local->transaction.done   = afr_create_done;
         local->transaction.unwind = afr_create_unwind;
 
-        afr_build_parent_loc (&local->transaction.parent_loc, loc);
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -325,7 +498,7 @@ out:
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
                 AFR_STACK_UNWIND (create, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -340,7 +513,6 @@ afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -354,17 +526,13 @@ afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.mknod.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mknod.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mknod.buf;
-                }
-
                 AFR_STACK_UNWIND (mknod, main_frame,
                                   local->op_ret, local->op_errno,
-                                  local->cont.mknod.inode,
-                                  unwind_buf, &local->cont.mknod.preparent,
-                                  &local->cont.mknod.postparent);
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -375,58 +543,25 @@ int
 afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
         int             call_count      = -1;
         int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.mknod.buf   = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mknod.read_child_buf = *buf;
-                                local->cont.mknod.preparent      = *preparent;
-                                local->cont.mknod.postparent     = *postparent;
-                        }
-
-                        local->cont.mknod.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -460,7 +595,8 @@ afr_mknod_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->mknod,
                                            &local->loc, local->cont.mknod.mode,
                                            local->cont.mknod.dev,
-                                           local->cont.mknod.params);
+                                           local->umask,
+                                           local->xdata_req);
                         if (!--call_count)
                                 break;
                 }
@@ -485,14 +621,15 @@ afr_mknod_done (call_frame_t *frame, xlator_t *this)
 
 
 int
-afr_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t dev, dict_t *params)
+afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t dev, mode_t umask, dict_t *params)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -508,7 +645,7 @@ afr_mknod (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -524,21 +661,40 @@ afr_mknod (call_frame_t *frame, xlator_t *this,
         }
         UNLOCK (&priv->read_child_lock);
 
+        local->op               = GF_FOP_MKNOD;
         local->cont.mknod.mode  = mode;
         local->cont.mknod.dev   = dev;
+        local->umask = umask;
         if (params)
-                local->cont.mknod.params = dict_ref (params);
+                local->xdata_req = dict_ref (params);
 
         local->transaction.fop    = afr_mknod_wind;
         local->transaction.done   = afr_mknod_done;
         local->transaction.unwind = afr_mknod_unwind;
 
-        afr_build_parent_loc (&local->transaction.parent_loc, loc);
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -546,7 +702,7 @@ out:
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
                 AFR_STACK_UNWIND (mknod, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -562,7 +718,6 @@ afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -576,17 +731,13 @@ afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.mkdir.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mkdir.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mkdir.buf;
-                }
-
                 AFR_STACK_UNWIND (mkdir, main_frame,
                                   local->op_ret, local->op_errno,
-                                  local->cont.mkdir.inode,
-                                  unwind_buf, &local->cont.mkdir.preparent,
-                                  &local->cont.mkdir.postparent);
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -597,58 +748,25 @@ int
 afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
         int             call_count      = -1;
         int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret           = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.mkdir.buf   = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mkdir.read_child_buf = *buf;
-                                local->cont.mkdir.preparent      = *preparent;
-                                local->cont.mkdir.postparent     = *postparent;
-                        }
-
-                        local->cont.mkdir.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno         = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -682,7 +800,8 @@ afr_mkdir_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i],
                                            priv->children[i]->fops->mkdir,
                                            &local->loc, local->cont.mkdir.mode,
-                                           local->cont.mkdir.params);
+                                           local->umask,
+                                           local->xdata_req);
                         if (!--call_count)
                                 break;
                 }
@@ -706,16 +825,16 @@ afr_mkdir_done (call_frame_t *frame, xlator_t *this)
         return 0;
 }
 
-
 int
 afr_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dict_t *params)
+           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -731,7 +850,7 @@ afr_mkdir (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -748,19 +867,38 @@ afr_mkdir (call_frame_t *frame, xlator_t *this,
         UNLOCK (&priv->read_child_lock);
 
         local->cont.mkdir.mode  = mode;
+        local->umask = umask;
         if (params)
-                local->cont.mkdir.params = dict_ref (params);
+                local->xdata_req = dict_ref (params);
 
+        local->op = GF_FOP_MKDIR;
         local->transaction.fop    = afr_mkdir_wind;
         local->transaction.done   = afr_mkdir_done;
         local->transaction.unwind = afr_mkdir_unwind;
 
-        afr_build_parent_loc (&local->transaction.parent_loc, loc);
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -769,7 +907,7 @@ out:
                         AFR_STACK_DESTROY (transaction_frame);
 
                 AFR_STACK_UNWIND (mkdir, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -785,7 +923,6 @@ afr_link_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -799,17 +936,13 @@ afr_link_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.link.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.link.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.link.buf;
-                }
-
                 AFR_STACK_UNWIND (link, main_frame,
                                   local->op_ret, local->op_errno,
-                                  local->cont.link.inode,
-                                  unwind_buf, &local->cont.link.preparent,
-                                  &local->cont.link.postparent);
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -820,59 +953,25 @@ int
 afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
         int             call_count      = -1;
         int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret   = op_ret;
-
-                        if (local->success_count == 0) {
-                                local->cont.link.buf        = *buf;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.link.read_child_buf = *buf;
-                                local->cont.link.preparent      = *preparent;
-                                local->cont.link.postparent     = *postparent;
-                        }
-
-                        local->cont.link.inode    = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -901,11 +1000,12 @@ afr_link_wind (call_frame_t *frame, xlator_t *this)
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) i,
+                        STACK_WIND_COOKIE (frame, afr_link_wind_cbk,
+                                           (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->link,
                                            &local->loc,
-                                           &local->newloc);
+                                           &local->newloc, local->xdata_req);
 
                         if (!--call_count)
                                 break;
@@ -931,13 +1031,14 @@ afr_link_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc)
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -953,7 +1054,7 @@ afr_link (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -962,6 +1063,8 @@ afr_link (call_frame_t *frame, xlator_t *this,
 
         loc_copy (&local->loc,    oldloc);
         loc_copy (&local->newloc, newloc);
+        if (xdata)
+                local->xdata_req = dict_ref (xdata);
 
         LOCK (&priv->read_child_lock);
         {
@@ -970,24 +1073,41 @@ afr_link (call_frame_t *frame, xlator_t *this,
         }
         UNLOCK (&priv->read_child_lock);
 
+        local->op = GF_FOP_LINK;
         local->transaction.fop    = afr_link_wind;
         local->transaction.done   = afr_link_done;
         local->transaction.unwind = afr_link_unwind;
 
-        afr_build_parent_loc (&local->transaction.parent_loc, newloc);
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, newloc,
+                                    &op_errno);
+        if (ret)
+                goto out;
 
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
                 AFR_STACK_UNWIND (link, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1003,7 +1123,6 @@ afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -1017,17 +1136,13 @@ afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.symlink.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.symlink.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.symlink.buf;
-                }
-
                 AFR_STACK_UNWIND (symlink, main_frame,
                                   local->op_ret, local->op_errno,
-                                  local->cont.symlink.inode,
-                                  unwind_buf, &local->cont.symlink.preparent,
-                                  &local->cont.symlink.postparent);
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -1038,58 +1153,25 @@ int
 afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, inode_t *inode,
                       struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent)
+                      struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
         int             call_count      = -1;
         int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret   = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.symlink.buf        = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.symlink.read_child_buf = *buf;
-                                local->cont.symlink.preparent      = *preparent;
-                                local->cont.symlink.postparent     = *postparent;
-                        }
-
-                        local->cont.symlink.inode    = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -1124,7 +1206,8 @@ afr_symlink_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->symlink,
                                            local->cont.symlink.linkpath,
                                            &local->loc,
-                                           local->cont.symlink.params);
+                                           local->umask,
+                                           local->xdata_req);
 
                         if (!--call_count)
                                 break;
@@ -1151,13 +1234,14 @@ afr_symlink_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkpath, loc_t *loc, dict_t *params)
+             const char *linkpath, loc_t *loc, mode_t umask, dict_t *params)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -1173,7 +1257,7 @@ afr_symlink (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1190,19 +1274,38 @@ afr_symlink (call_frame_t *frame, xlator_t *this,
         UNLOCK (&priv->read_child_lock);
 
         local->cont.symlink.linkpath = gf_strdup (linkpath);
+        local->umask = umask;
         if (params)
-                local->cont.symlink.params = dict_ref (params);
+                local->xdata_req = dict_ref (params);
 
+        local->op = GF_FOP_SYMLINK;
         local->transaction.fop    = afr_symlink_wind;
         local->transaction.done   = afr_symlink_done;
         local->transaction.unwind = afr_symlink_unwind;
 
-        afr_build_parent_loc (&local->transaction.parent_loc, loc);
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
 
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -1210,7 +1313,7 @@ out:
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
                 AFR_STACK_UNWIND (symlink, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1225,7 +1328,6 @@ afr_rename_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -1239,19 +1341,14 @@ afr_rename_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.rename.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.rename.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.rename.buf;
-                }
-
                 AFR_STACK_UNWIND (rename, main_frame,
                                   local->op_ret, local->op_errno,
-                                  unwind_buf,
-                                  &local->cont.rename.preoldparent,
-                                  &local->cont.rename.postoldparent,
-                                  &local->cont.rename.prenewparent,
-                                  &local->cont.rename.postnewparent);
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  &local->cont.dir_fop.prenewparent,
+                                  &local->cont.dir_fop.postnewparent,
+                                  NULL);
         }
 
         return 0;
@@ -1262,7 +1359,8 @@ int
 afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
                      struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent)
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         int call_count = -1;
@@ -1276,38 +1374,22 @@ afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         {
                 if (afr_fop_failed (op_ret, op_errno) && op_errno != ENOTEMPTY)
                         afr_transaction_fop_failed (frame, this, child_index);
+                local->op_errno = op_errno;
+                local->child_errno[child_index] = op_errno;
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-
-                                if (buf) {
-                                        local->cont.rename.buf = *buf;
-                                }
-
-                                local->success_count++;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.rename.read_child_buf = *buf;
+                if (op_ret > -1)
+                        __dir_entry_fop_common_cbk (frame, child_index, this,
+                                                   op_ret, op_errno, NULL, buf,
+                                                   preoldparent, postoldparent,
+                                                   prenewparent, postnewparent);
 
-                                local->cont.rename.preoldparent  = *preoldparent;
-                                local->cont.rename.postoldparent = *postoldparent;
-                                local->cont.rename.prenewparent  = *prenewparent;
-                                local->cont.rename.postnewparent = *postnewparent;
-                        }
-                }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -1341,7 +1423,7 @@ afr_rename_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i],
                                            priv->children[i]->fops->rename,
                                            &local->loc,
-                                           &local->newloc);
+                                           &local->newloc, NULL);
                         if (!--call_count)
                                 break;
                 }
@@ -1366,13 +1448,15 @@ afr_rename_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc)
+            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+        int                     nlockee                 = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -1388,7 +1472,7 @@ afr_rename (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1400,18 +1484,61 @@ afr_rename (call_frame_t *frame, xlator_t *this,
 
         local->read_child_index = afr_inode_get_read_ctx (this, oldloc->inode, NULL);
 
+        local->op = GF_FOP_RENAME;
         local->transaction.fop    = afr_rename_wind;
         local->transaction.done   = afr_rename_done;
         local->transaction.unwind = afr_rename_unwind;
 
-        afr_build_parent_loc (&local->transaction.parent_loc, oldloc);
-        afr_build_parent_loc (&local->transaction.new_parent_loc, newloc);
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, oldloc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+        ret = afr_build_parent_loc (&local->transaction.new_parent_loc, newloc,
+                                    &op_errno);
+        if (ret)
+                goto out;
 
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (oldloc->path);
         local->transaction.new_basename = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.new_parent_loc,
+                                     local->transaction.new_basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        if (local->newloc.inode && IA_ISDIR (local->newloc.inode->ia_type)) {
+                ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                             &local->newloc,
+                                             NULL,
+                                             priv->child_count);
+                if (ret)
+                        goto out;
+
+                nlockee++;
+        }
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -1420,7 +1547,7 @@ out:
                         AFR_STACK_DESTROY (transaction_frame);
 
                 AFR_STACK_UNWIND (rename, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1450,8 +1577,9 @@ afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (unlink, main_frame,
                                   local->op_ret, local->op_errno,
-                                  &local->cont.unlink.preparent,
-                                  &local->cont.unlink.postparent);
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -1461,7 +1589,7 @@ afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         int call_count  = -1;
@@ -1474,36 +1602,15 @@ afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (child_index == local->read_child_index) {
                         local->read_child_returned = _gf_true;
                 }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret   = op_ret;
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, NULL, NULL,
+                                            preparent, postparent, NULL, NULL);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -1536,7 +1643,8 @@ afr_unlink_wind (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->unlink,
-                                           &local->loc);
+                                           &local->loc, local->xflag,
+                                           local->xdata_req);
 
                         if (!--call_count)
                                 break;
@@ -1562,13 +1670,14 @@ afr_unlink_done (call_frame_t *frame, xlator_t *this)
 
 int32_t
 afr_unlink (call_frame_t *frame, xlator_t *this,
-            loc_t *loc)
+            loc_t *loc, int xflag, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -1584,7 +1693,7 @@ afr_unlink (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1592,17 +1701,38 @@ afr_unlink (call_frame_t *frame, xlator_t *this,
                 goto out;
 
         loc_copy (&local->loc, loc);
+        local->xflag = xflag;
+        if (xdata)
+                local->xdata_req = dict_ref (xdata);
 
+        local->op = GF_FOP_UNLINK;
         local->transaction.fop    = afr_unlink_wind;
         local->transaction.done   = afr_unlink_done;
         local->transaction.unwind = afr_unlink_unwind;
 
-        afr_build_parent_loc (&local->transaction.parent_loc, loc);
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -1610,7 +1740,7 @@ out:
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
                 AFR_STACK_UNWIND (unlink, frame, -1, op_errno,
-                                  NULL, NULL);
+                                  NULL, NULL, NULL);
         }
 
         return 0;
@@ -1642,8 +1772,9 @@ afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (rmdir, main_frame,
                                   local->op_ret, local->op_errno,
-                                  &local->cont.rmdir.preparent,
-                                  &local->cont.rmdir.postparent);
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -1653,7 +1784,7 @@ afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         int call_count  = -1;
@@ -1667,36 +1798,22 @@ afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (child_index == read_child) {
                         local->read_child_returned = _gf_true;
                 }
-
                 if (afr_fop_failed (op_ret, op_errno) && (op_errno != ENOTEMPTY))
                         afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-                        }
-
-                        local->success_count++;
-                }
-
                 local->op_errno = op_errno;
+                local->child_errno[child_index] = op_errno;
+                if (op_ret > -1)
+                        __dir_entry_fop_common_cbk (frame, child_index, this,
+                                                   op_ret, op_errno, NULL, NULL,
+                                                   preparent, postparent, NULL,
+                                                   NULL);
+
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -1729,7 +1846,8 @@ afr_rmdir_wind (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->rmdir,
-                                           &local->loc, local->cont.rmdir.flags);
+                                           &local->loc, local->cont.rmdir.flags,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1755,13 +1873,15 @@ afr_rmdir_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_rmdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int flags)
+           loc_t *loc, int flags, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+        int                     nlockee                 = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -1777,7 +1897,7 @@ afr_rmdir (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1787,23 +1907,53 @@ afr_rmdir (call_frame_t *frame, xlator_t *this,
         local->cont.rmdir.flags = flags;
         loc_copy (&local->loc, loc);
 
+        local->op = GF_FOP_RMDIR;
         local->transaction.fop    = afr_rmdir_wind;
         local->transaction.done   = afr_rmdir_done;
         local->transaction.unwind = afr_rmdir_unwind;
 
-        afr_build_parent_loc (&local->transaction.parent_loc, loc);
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->loc,
+                                     NULL,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
+
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
diff --git a/xlators/cluster/afr/src/afr-dir-write.h b/xlators/cluster/afr/src/afr-dir-write.h
index 0290c6350..02f0a3682 100644
--- a/xlators/cluster/afr/src/afr-dir-write.h
+++ b/xlators/cluster/afr/src/afr-dir-write.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_WRITE_H__
@@ -23,38 +14,34 @@
 int32_t
 afr_create (call_frame_t *frame, xlator_t *this,
             loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params);
+            mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, dict_t *params);
+	   loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata);
 
 int32_t
 afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *params);
+	   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
 
 int32_t
 afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc);
+	    loc_t *loc, int xflag, dict_t *xdata);
 
 int32_t
 afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags);
+	   loc_t *loc, int flags, dict_t *xdata);
 
 int32_t
 afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc);
+	  loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int32_t
 afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc);
+	    loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int
 afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *oldloc, dict_t *params);
-
-int32_t
-afr_setdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, int32_t flags, dir_entry_t *entries, int32_t count);
+	     const char *linkpath, loc_t *oldloc, mode_t umask, dict_t *params);
 
 #endif /* __DIR_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index ec0acbd3b..e06e3b2f2 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -58,7 +49,7 @@
 
 int32_t
 afr_access_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno)
+                xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_private_t * priv            = NULL;
         afr_local_t *   local           = NULL;
@@ -92,12 +83,13 @@ afr_access_cbk (call_frame_t *frame, void *cookie,
                                    (void *) (long) read_child,
                                    children[next_call_child],
                                    children[next_call_child]->fops->access,
-                                   &local->loc, local->cont.access.mask);
+                                   &local->loc, local->cont.access.mask,
+                                   NULL);
         }
 
 out:
         if (unwind) {
-                AFR_STACK_UNWIND (access, frame, op_ret, op_errno);
+                AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
         }
 
         return 0;
@@ -105,7 +97,8 @@ out:
 
 
 int32_t
-afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+            dict_t *xdata)
 {
         afr_private_t   *priv      = NULL;
         xlator_t        **children = NULL;
@@ -124,7 +117,9 @@ afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
 
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -156,12 +151,12 @@ afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
                            (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->access,
-                           loc, mask);
+                           loc, mask, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (access, frame, -1, op_errno);
+                AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -173,7 +168,7 @@ out:
 int32_t
 afr_stat_cbk (call_frame_t *frame, void *cookie,
               xlator_t *this, int32_t op_ret, int32_t op_errno,
-              struct iatt *buf)
+              struct iatt *buf, dict_t *xdata)
 {
         afr_private_t * priv            = NULL;
         afr_local_t *   local           = NULL;
@@ -207,12 +202,12 @@ afr_stat_cbk (call_frame_t *frame, void *cookie,
                                    (void *) (long) read_child,
                                    children[next_call_child],
                                    children[next_call_child]->fops->stat,
-                                   &local->loc);
+                                   &local->loc, NULL);
         }
 
 out:
         if (unwind) {
-                AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf);
+                AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
         }
 
         return 0;
@@ -220,7 +215,7 @@ out:
 
 
 int32_t
-afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         afr_private_t   *priv      = NULL;
         afr_local_t     *local     = NULL;
@@ -239,7 +234,9 @@ afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -267,12 +264,12 @@ afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
         STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->stat,
-                           loc);
+                           loc, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -284,7 +281,8 @@ out:
 
 int32_t
 afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf)
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
 {
         afr_private_t   *priv           = NULL;
         afr_local_t     *local          = NULL;
@@ -318,12 +316,12 @@ afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                    (void *) (long) read_child,
                                    children[next_call_child],
                                    children[next_call_child]->fops->fstat,
-                                   local->fd);
+                                   local->fd, NULL);
         }
 
 out:
         if (unwind) {
-                AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf);
+                AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
         }
 
         return 0;
@@ -332,7 +330,7 @@ out:
 
 int32_t
 afr_fstat (call_frame_t *frame, xlator_t *this,
-           fd_t *fd)
+           fd_t *fd, dict_t *xdata)
 {
         afr_private_t   *priv      = NULL;
         afr_local_t     *local     = NULL;
@@ -354,7 +352,9 @@ afr_fstat (call_frame_t *frame, xlator_t *this,
 
         VALIDATE_OR_GOTO (fd->inode, out);
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_SBRAIN_CHECK_FD (fd, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -383,20 +383,17 @@ afr_fstat (call_frame_t *frame, xlator_t *this,
 
         local->fd = fd_ref (fd);
 
-        ret = afr_open_fd_fix (frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+        afr_open_fd_fix (fd, this);
+
         STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->fstat,
-                           fd);
+                           fd, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -408,7 +405,7 @@ out:
 int32_t
 afr_readlink_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  const char *buf, struct iatt *sbuf)
+                  const char *buf, struct iatt *sbuf, dict_t *xdata)
 {
         afr_private_t * priv                  = NULL;
         afr_local_t *   local                 = NULL;
@@ -442,12 +439,13 @@ afr_readlink_cbk (call_frame_t *frame, void *cookie,
                                    children[next_call_child],
                                    children[next_call_child]->fops->readlink,
                                    &local->loc,
-                                   local->cont.readlink.size);
+                                   local->cont.readlink.size, NULL);
         }
 
 out:
         if (unwind) {
-                AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, sbuf);
+                AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, sbuf,
+                                  xdata);
         }
 
         return 0;
@@ -456,7 +454,7 @@ out:
 
 int32_t
 afr_readlink (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, size_t size)
+              loc_t *loc, size_t size, dict_t *xdata)
 {
         afr_private_t   *priv      = NULL;
         xlator_t        **children = NULL;
@@ -475,7 +473,9 @@ afr_readlink (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -506,12 +506,12 @@ afr_readlink (call_frame_t *frame, xlator_t *this,
                            (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->readlink,
-                           loc, size);
+                           loc, size, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -526,7 +526,7 @@ struct _xattr_key {
 };
 
 
-void
+int
 __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
                      void *data)
 {
@@ -538,13 +538,14 @@ __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
 
                 xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
                 if (!xkey)
-                        return;
+                        return -1;
 
                 xkey->key = key;
                 INIT_LIST_HEAD (&xkey->list);
 
                 list_add_tail (&xkey->list, list);
         }
+        return 0;
 }
 
 
@@ -574,7 +575,7 @@ __filter_xattrs (dict_t *dict)
 int32_t
 afr_getxattr_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *dict)
+                  dict_t *dict, dict_t *xdata)
 {
         afr_private_t * priv            = NULL;
         afr_local_t *   local           = NULL;
@@ -608,7 +609,8 @@ afr_getxattr_cbk (call_frame_t *frame, void *cookie,
                                    children[next_call_child],
                                    children[next_call_child]->fops->getxattr,
                                    &local->loc,
-                                   local->cont.getxattr.name);
+                                   local->cont.getxattr.name,
+                                   NULL);
         }
 
 out:
@@ -616,25 +618,25 @@ out:
                 if (op_ret >= 0 && dict)
                         __filter_xattrs (dict);
 
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
         }
 
         return 0;
 }
 
 int32_t
-afr_getxattr_unwind (call_frame_t *frame,
-                     int op_ret, int op_errno, dict_t *dict)
+afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
+                     dict_t *dict, dict_t *xdata)
 
 {
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 int32_t
-afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this, int32_t op_ret, int32_t op_errno,
-                        dict_t *dict)
+afr_fgetxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *dict, dict_t *xdata)
 {
         afr_local_t     *local                  = NULL;
         afr_private_t   *priv                   = NULL;
@@ -656,6 +658,8 @@ afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
         LOCK (&frame->lock);
         {
                 callcnt = --local->call_count;
+                if (op_ret == -1)
+                        local->child_errno[cky] = op_errno;
 
                 if (!local->dict)
                         local->dict = dict_new ();
@@ -664,14 +668,12 @@ afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
                                             &tmp_report);
                         if (ret)
                                 goto unlock;
-                        ret = dict_set_str (local->dict,
-                                            children[cky]->name,
-                                            tmp_report);
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
                         if (ret)
                                 goto unlock;
                 }
-                if (op_ret == -1)
-                        local->child_errno[cky] = op_errno;
         }
 unlock:
         UNLOCK (&frame->lock);
@@ -711,13 +713,97 @@ unlock:
                 local->child_errno[cky] = op_errno;
                 op_errno = afr_resultant_errno_get (NULL, local->child_errno,
                                                     priv->child_count);
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr,
+                                  xdata);
+
+                if (xattr)
+                        dict_unref (xattr);
+        }
+
+        return ret;
+}
+
+int32_t
+afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
+                        xlator_t *this, int32_t op_ret, int32_t op_errno,
+                        dict_t *dict, dict_t *xdata)
+{
+        afr_local_t     *local                  = NULL;
+        afr_private_t   *priv                   = NULL;
+        xlator_t        **children              = NULL;
+        dict_t          *xattr                  = NULL;
+        char            *tmp_report             = NULL;
+        char            lk_summary[1024]        = {0,};
+        int             serz_len                = 0;
+        int32_t         callcnt                 = 0;
+        long int        cky                     = 0;
+        int             ret                     = 0;
 
-                if (local->dict)
-                        dict_unref (local->dict);
+        priv     = this->private;
+        children = priv->children;
 
-                if (local->child_errno)
-                        GF_FREE (local->child_errno);
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+                if (op_ret == -1)
+                        local->child_errno[cky] = op_errno;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+                if (local->dict) {
+                        ret = dict_get_str (dict, local->cont.getxattr.name,
+                                            &tmp_report);
+                        if (ret)
+                                goto unlock;
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
+                        if (ret)
+                                goto unlock;
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                xattr = dict_new ();
+                if (!xattr) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       lk_summary,
+                                                       &serz_len, '\n');
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error serializing dictionary");
+                        goto unwind;
+                }
+                if (serz_len == -1)
+                        snprintf (lk_summary, sizeof (lk_summary),
+                                  "No locks cleared.");
+                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
+                                       gf_strdup (lk_summary));
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting dictionary");
+                        goto unwind;
+                }
+
+        unwind:
+                // Updating child_errno with more recent 'events'
+                local->child_errno[cky] = op_errno;
+                op_errno = afr_resultant_errno_get (NULL, local->child_errno,
+                                                    priv->child_count);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
                 if (xattr)
                         dict_unref (xattr);
@@ -726,24 +812,426 @@ unlock:
         return ret;
 }
 
+/**
+ * node-uuid cbk uses next child querying mechanism
+ */
+int32_t
+afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        afr_private_t  *priv            = NULL;
+        afr_local_t    *local           = NULL;
+        xlator_t      **children        = NULL;
+        int             unwind          = 1;
+        int             curr_call_child = 0;
+
+        priv = this->private;
+        children = priv->children;
+
+        local = frame->local;
+
+        if (op_ret == -1) { /** query the _next_ child */
+
+                /**
+                 * _current_ becomes _next_
+                 * If done with all childs and yet no success; give up !
+                 */
+                curr_call_child = (int) ((long)cookie);
+                if (++curr_call_child == priv->child_count)
+                        goto unwind;
+
+                gf_log (this->name, GF_LOG_WARNING,
+                        "op_ret (-1): Re-querying afr-child (%d/%d)",
+                        curr_call_child, priv->child_count);
+
+                unwind = 0;
+                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
+                                   (void *) (long) curr_call_child,
+                                   children[curr_call_child],
+                                   children[curr_call_child]->fops->getxattr,
+                                   &local->loc,
+                                   local->cont.getxattr.name,
+                                   NULL);
+        }
+
+ unwind:
+        if (unwind)
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
+                                  NULL);
+
+        return 0;
+}
+
+int32_t
+afr_getxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                           xlator_t *this, int32_t op_ret, int32_t op_errno,
+                           dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                call_cnt = --local->call_count;
+
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
+
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
+
+                if (!dict) {
+                        goto unlock;
+                }
+
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
+
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
+
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
+
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
+
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                len = dict_serialized_length (local->dict);
+                if (len == 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (getxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
+
+        return 0;
+}
+
+int32_t
+afr_fgetxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                call_cnt = --local->call_count;
+
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
+
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
+
+                if (!dict) {
+                        goto unlock;
+                }
+
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
+
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
+
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
+
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
+
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                len = dict_serialized_length (local->dict);
+                if (len <= 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
+
+        return 0;
+}
+
+int32_t
+afr_fgetxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+        int          ret            = 0;
+        char        *xattr          = NULL;
+        char        *xattr_serz     = NULL;
+        char        xattr_cky[1024] = {0,};
+        dict_t      *nxattr         = NULL;
+        long         cky            = 0;
+        int32_t      padding        = 0;
+        int32_t      tlen           = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (!dict || (op_ret < 0))
+                        goto out;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+
+                if (local->dict) {
+                        ret = dict_get_str (dict,
+                                            local->cont.getxattr.name,
+                                            &xattr);
+                        if (ret)
+                                goto out;
+
+                        xattr = gf_strdup (xattr);
+
+                        (void)snprintf (xattr_cky, 1024, "%s-%ld",
+                                        local->cont.getxattr.name, cky);
+                        ret = dict_set_dynstr (local->dict,
+                                               xattr_cky, xattr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Cannot set xattr cookie key");
+                                goto out;
+                        }
+
+                        local->cont.getxattr.xattr_len
+                                += strlen (xattr) + 1;
+                }
+        }
+out:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (!local->cont.getxattr.xattr_len)
+                        goto unwind;
+
+                nxattr = dict_new ();
+                if (!nxattr)
+                        goto unwind;
+
+                /* extra bytes for decorations (brackets and <>'s) */
+                padding += strlen (this->name)
+                        + strlen (AFR_PATHINFO_HEADER) + 4;
+                local->cont.getxattr.xattr_len += (padding + 2);
+
+                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
+                                        sizeof (char), gf_common_mt_char);
+
+                if (!xattr_serz)
+                        goto unwind;
+
+                /* the xlator info */
+                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
+                                this->name);
+
+                /* actual series of pathinfo */
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       xattr_serz
+                                                       + strlen (xattr_serz),
+                                                       &tlen, ' ');
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
+                                " dictionary");
+                        goto unwind;
+                }
+
+                /* closing part */
+                *(xattr_serz + padding + tlen) = ')';
+                *(xattr_serz + padding + tlen + 1) = '\0';
+
+                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
+                                       xattr_serz);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
+                                " key in dict");
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, nxattr,
+                                  xdata);
+
+                if (nxattr)
+                        dict_unref (nxattr);
+        }
+
+        return ret;
+}
+
 int32_t
 afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           dict_t *dict)
+                           dict_t *dict, dict_t *xdata)
 {
-        afr_local_t *local             = NULL;
-        int32_t      callcnt           = 0;
-        int          ret               = 0;
-        char        *pathinfo          = NULL;
-        char        *pathinfo_serz     = NULL;
-        char        pathinfo_cky[1024] = {0,};
-        dict_t      *xattr             = NULL;
-        long         cky               = 0;
-        int32_t      padding           = 0;
-        int32_t      tlen              = 0;
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+        int          ret            = 0;
+        char        *xattr          = NULL;
+        char        *xattr_serz     = NULL;
+        char        xattr_cky[1024] = {0,};
+        dict_t      *nxattr         = NULL;
+        long         cky            = 0;
+        int32_t      padding        = 0;
+        int32_t      tlen           = 0;
 
         if (!frame || !frame->local || !this) {
-                gf_log (this->name, GF_LOG_ERROR, "possible NULL deref");
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
                 goto out;
         }
 
@@ -761,77 +1249,142 @@ afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                                 local->dict = dict_new ();
 
                         if (local->dict) {
-                                ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+                                ret = dict_get_str (dict,
+                                                    local->cont.getxattr.name,
+                                                    &xattr);
                                 if (ret)
                                         goto out;
 
-                                pathinfo = gf_strdup (pathinfo);
+                                xattr = gf_strdup (xattr);
 
-                                snprintf (pathinfo_cky, 1024, "%s-%ld", GF_XATTR_PATHINFO_KEY, cky);
-                                ret = dict_set_dynstr (local->dict, pathinfo_cky, pathinfo);
+                                (void)snprintf (xattr_cky, 1024, "%s-%ld",
+                                                local->cont.getxattr.name, cky);
+                                ret = dict_set_dynstr (local->dict,
+                                                       xattr_cky, xattr);
                                 if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo cookie key");
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Cannot set xattr cookie key");
                                         goto out;
                                 }
 
-                                local->cont.getxattr.pathinfo_len += strlen (pathinfo) + 1;
+                                local->cont.getxattr.xattr_len += strlen (xattr) + 1;
                         }
                 }
  out:
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
-                if (!local->cont.getxattr.pathinfo_len)
+                if (!local->cont.getxattr.xattr_len)
                         goto unwind;
 
-                xattr = dict_new ();
-                if (!xattr)
+                nxattr = dict_new ();
+                if (!nxattr)
                         goto unwind;
 
                 /* extra bytes for decorations (brackets and <>'s) */
-                padding = strlen (this->name) + strlen (AFR_PATHINFO_HEADER) + 4;
-                local->cont.getxattr.pathinfo_len += (padding + 2);
+                padding += strlen (this->name) + strlen (AFR_PATHINFO_HEADER) + 4;
+                local->cont.getxattr.xattr_len += (padding + 2);
 
-                pathinfo_serz = GF_CALLOC (local->cont.getxattr.pathinfo_len, sizeof (char),
-                                           gf_common_mt_char);
+                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
+                                        sizeof (char), gf_common_mt_char);
 
-                if (!pathinfo_serz)
+                if (!xattr_serz)
                         goto unwind;
 
                 /* the xlator info */
-                sprintf (pathinfo_serz, "(<"AFR_PATHINFO_HEADER"%s> ", this->name);
+                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
+                                this->name);
 
                 /* actual series of pathinfo */
-                ret = dict_serialize_value_with_delim (local->dict, pathinfo_serz + strlen (pathinfo_serz),
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       xattr_serz + strlen (xattr_serz),
                                                        &tlen, ' ');
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Error serializing dictionary");
+                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
+                                " dictionary");
                         goto unwind;
                 }
 
                 /* closing part */
-                *(pathinfo_serz + padding + tlen) = ')';
-                *(pathinfo_serz + padding + tlen + 1) = '\0';
+                *(xattr_serz + padding + tlen) = ')';
+                *(xattr_serz + padding + tlen + 1) = '\0';
 
-                ret = dict_set_dynstr (xattr, GF_XATTR_PATHINFO_KEY, pathinfo_serz);
+                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
+                                       xattr_serz);
                 if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo key in dict");
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
+                                " key in dict");
 
         unwind:
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
-
-                if (local->dict)
-                        dict_unref (local->dict);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, nxattr,
+                                  xdata);
 
-                if (xattr)
-                        dict_unref (xattr);
+                if (nxattr)
+                        dict_unref (nxattr);
         }
 
         return ret;
 }
 
+static int
+afr_aggregate_stime_xattr (dict_t *this, char *key, data_t *value, void *data)
+{
+        int ret = 0;
+
+        if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+                ret = gf_get_min_stime (THIS, data, key, value);
+
+        return ret;
+}
+
+int32_t
+afr_common_getxattr_stime_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (!dict || (op_ret < 0)) {
+                        local->op_errno = op_errno;
+                        goto cleanup;
+                }
+
+                if (!local->dict)
+                        local->dict = dict_copy_with_ref (dict, NULL);
+                else
+                        dict_foreach (dict, afr_aggregate_stime_xattr,
+                                      local->dict);
+                local->op_ret = 0;
+        }
+
+cleanup:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+                                  local->op_errno, local->dict, xdata);
+        }
+
+out:
+        return 0;
+}
+
+
 static gf_boolean_t
-afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk)
+afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk,
+                      gf_boolean_t is_fgetxattr)
 {
         gf_boolean_t    is_spl = _gf_true;
 
@@ -841,13 +1394,31 @@ afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk)
                 goto out;
         }
 
-        if (!strcmp (name, GF_XATTR_PATHINFO_KEY))
-                *cbk = afr_getxattr_pathinfo_cbk;
-
-        else if (!strcmp (name, GF_XATTR_CLRLK_CMD))
-                *cbk = afr_getxattr_clrlk_cbk;
-        else
+        if (!strcmp (name, GF_XATTR_PATHINFO_KEY)) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_pathinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_pathinfo_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
+                             strlen (GF_XATTR_CLRLK_CMD))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_clrlk_cbk;
+                } else {
+                        *cbk = afr_getxattr_clrlk_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                             strlen (GF_XATTR_LOCKINFO_KEY))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_lockinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_lockinfo_cbk;
+                }
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+                *cbk = afr_common_getxattr_stime_cbk;
+        } else {
                 is_spl = _gf_false;
+        }
 
 out:
         return is_spl;
@@ -873,14 +1444,14 @@ afr_getxattr_frm_all_children (xlator_t *this, call_frame_t *frame,
                 STACK_WIND_COOKIE (frame, cbk,
                                    (void *) (long) i,
                                    children[i], children[i]->fops->getxattr,
-                                   loc, name);
+                                   loc, name, NULL);
         }
         return;
 }
 
 int32_t
 afr_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *name)
+              loc_t *loc, const char *name, dict_t *xdata)
 {
         afr_private_t           *priv         = NULL;
         xlator_t                **children    = NULL;
@@ -893,7 +1464,7 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
         int32_t                 read_child    = -1;
         int                     ret           = -1;
         fop_getxattr_cbk_t      cbk           = NULL;
-
+        int                     afr_xtime_gauge[MCNT_MAX] = {0,};
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -904,7 +1475,9 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -912,38 +1485,98 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
                 goto out;
 
         loc_copy (&local->loc, loc);
-        if (name)
-                local->cont.getxattr.name = gf_strdup (name);
+        if (!name)
+                goto no_name;
+
+        local->cont.getxattr.name = gf_strdup (name);
+
+        if (!strncmp (name, AFR_XATTR_PREFIX,
+                      strlen (AFR_XATTR_PREFIX))) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: no data present for key %s",
+                        loc->path, name);
+                op_errno = ENODATA;
+                goto out;
+        }
+        if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+
+                local->marker.call_count = priv->child_count;
+
+                sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
+                for (i = 0, trav = this->children; trav ;
+                     trav = trav->next, i++) {
 
+                        *(sub_volumes + i)  = trav->xlator;
+                }
+
+                if (cluster_getmarkerattr (frame, this, loc, name,
+                                           local, afr_getxattr_unwind,
+                                           sub_volumes,
+                                           priv->child_count,
+                                           MARKER_UUID_TYPE,
+                                           marker_uuid_default_gauge,
+                                           priv->vol_uuid)) {
 
-        if (name) {
-                if (!strncmp (name, AFR_XATTR_PREFIX,
-                              strlen (AFR_XATTR_PREFIX))) {
                         gf_log (this->name, GF_LOG_INFO,
-                                "%s: no data present for key %s",
+                                "%s: failed to get marker attr (%s)",
                                 loc->path, name);
-                        op_errno = ENODATA;
+                        op_errno = EINVAL;
                         goto out;
                 }
-                if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
-                    && (-1 == frame->root->pid)) {
 
+                return 0;
+        }
+
+        /*
+         * if we are doing getxattr with pathinfo as the key then we
+         * collect information from all childs
+         */
+        if (afr_is_special_xattr (name, &cbk, 0)) {
+                afr_getxattr_frm_all_children (this, frame, name,
+                                               loc, cbk);
+                return 0;
+        }
+
+        if (XATTR_IS_NODE_UUID (name)) {
+                i = 0;
+                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
+                                   (void *) (long) i,
+                                   children[i],
+                                   children[i]->fops->getxattr,
+                                   loc, name, xdata);
+                return 0;
+        }
+
+        if (*priv->vol_uuid) {
+                if ((match_uuid_local (name, priv->vol_uuid) == 0)
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
                         local->marker.call_count = priv->child_count;
 
-                        sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
+                        sub_volumes = alloca ( priv->child_count
+                                               * sizeof (xlator_t *));
                         for (i = 0, trav = this->children; trav ;
                              trav = trav->next, i++) {
 
                                 *(sub_volumes + i)  = trav->xlator;
+
                         }
 
-                        if (cluster_getmarkerattr (frame, this, loc, name,
-                                                   local, afr_getxattr_unwind,
+                        /* don't err out on getting ENOTCONN (brick down)
+                         * from a subset of the bricks
+                         */
+                        memcpy (afr_xtime_gauge, marker_xtime_default_gauge,
+                                sizeof (afr_xtime_gauge));
+                        afr_xtime_gauge[MCNT_NOTFOUND] = 0;
+                        afr_xtime_gauge[MCNT_ENOTCONN] = 0;
+                        if (cluster_getmarkerattr (frame, this, loc,
+                                                   name, local,
+                                                   afr_getxattr_unwind,
                                                    sub_volumes,
                                                    priv->child_count,
-                                                   MARKER_UUID_TYPE,
+                                                   MARKER_XTIME_TYPE,
+                                                   afr_xtime_gauge,
                                                    priv->vol_uuid)) {
-
                                 gf_log (this->name, GF_LOG_INFO,
                                         "%s: failed to get marker attr (%s)",
                                         loc->path, name);
@@ -953,57 +1586,17 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 
                         return 0;
                 }
-
-                /*
-                 * if we are doing getxattr with pathinfo as the key then we
-                 * collect information from all childs
-                 */
-                if (afr_is_special_xattr (name, &cbk)) {
-                        afr_getxattr_frm_all_children (this, frame, name,
-                                                       loc, cbk);
-                        return 0;
-                }
-
-                if (*priv->vol_uuid) {
-                        if ((match_uuid_local (name, priv->vol_uuid) == 0)
-                            && (-1 == frame->root->pid)) {
-
-                                local->marker.call_count = priv->child_count;
-
-                                sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
-                                for (i = 0, trav = this->children; trav ;
-                                     trav = trav->next, i++) {
-
-                                        *(sub_volumes + i)  = trav->xlator;
-
-                                }
-
-                                if (cluster_getmarkerattr (frame, this, loc,
-                                                           name, local,
-                                                           afr_getxattr_unwind,
-                                                           sub_volumes,
-                                                           priv->child_count,
-                                                           MARKER_XTIME_TYPE,
-                                                           priv->vol_uuid)) {
-                                        gf_log (this->name, GF_LOG_INFO,
-                                                "%s: failed to get marker attr (%s)",
-                                                loc->path, name);
-                                        op_errno = EINVAL;
-                                        goto out;
-                                }
-
-                                return 0;
-                        }
-                }
         }
 
+no_name:
         local->fresh_children = afr_children_create (priv->child_count);
         if (!local->fresh_children) {
                 op_errno = ENOMEM;
                 goto out;
         }
 
-        read_child = afr_inode_get_read_ctx (this, loc->inode, local->fresh_children);
+        read_child = afr_inode_get_read_ctx (this, loc->inode,
+                                             local->fresh_children);
         ret = afr_get_call_child (this, local->child_up, read_child,
                                      local->fresh_children,
                                      &call_child,
@@ -1017,12 +1610,12 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
                            (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->getxattr,
-                           loc, name);
+                           loc, name, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -1032,7 +1625,7 @@ out:
 int32_t
 afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
                    xlator_t *this, int32_t op_ret, int32_t op_errno,
-                   dict_t *dict)
+                   dict_t *dict, dict_t *xdata)
 {
         afr_private_t * priv            = NULL;
         afr_local_t *   local           = NULL;
@@ -1066,7 +1659,8 @@ afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
                                    children[next_call_child],
                                    children[next_call_child]->fops->fgetxattr,
                                    local->fd,
-                                   local->cont.getxattr.name);
+                                   local->cont.getxattr.name,
+                                   NULL);
         }
 
 out:
@@ -1074,7 +1668,8 @@ out:
                 if (op_ret >= 0 && dict)
                         __filter_xattrs (dict);
 
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict);
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict,
+                                  xdata);
         }
 
         return 0;
@@ -1082,25 +1677,51 @@ out:
 
 int32_t
 afr_fgetxattr_unwind (call_frame_t *frame,
-                      int op_ret, int op_errno, dict_t *dict)
+                      int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 
 {
-        AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict);
+        AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
+static void
+afr_fgetxattr_frm_all_children (xlator_t *this, call_frame_t *frame,
+                                const char *name, fd_t *fd,
+                                fop_fgetxattr_cbk_t cbk)
+{
+        afr_private_t   *priv           = NULL;
+        afr_local_t     *local          = NULL;
+        xlator_t        **children      = NULL;
+        int             i               = 0;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+        local->call_count = priv->child_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                STACK_WIND_COOKIE (frame, cbk,
+                                   (void *) (long) i,
+                                   children[i], children[i]->fops->fgetxattr,
+                                   fd, name, NULL);
+        }
+
+        return;
+}
+
 int32_t
 afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name)
+               fd_t *fd, const char *name, dict_t *xdata)
 {
-        afr_private_t   *priv         = NULL;
-        xlator_t        **children    = NULL;
-        int             call_child    = 0;
-        afr_local_t     *local        = NULL;
-        int32_t         op_ret        = -1;
-        int32_t         op_errno      = 0;
-        int32_t         read_child    = -1;
-
+        afr_private_t        *priv       = NULL;
+        xlator_t            **children   = NULL;
+        int                   call_child = 0;
+        afr_local_t          *local      = NULL;
+        int32_t               op_ret     = -1;
+        int32_t               op_errno   = 0;
+        int32_t               read_child = -1;
+        fop_fgetxattr_cbk_t   cbk        = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -1111,7 +1732,9 @@ afr_fgetxattr (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        AFR_SBRAIN_CHECK_FD (fd, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
         frame->local = local;
 
         op_ret = afr_local_init (local, priv, &op_errno);
@@ -1124,7 +1747,17 @@ afr_fgetxattr (call_frame_t *frame, xlator_t *this,
         if (name)
                 local->cont.getxattr.name = gf_strdup (name);
 
-        /* pathinfo gets handled only in getxattr() */
+        /* pathinfo gets handled only in getxattr(), but we need to handle
+         * lockinfo.
+         * If we are doing fgetxattr with lockinfo as the key then we
+         * collect information from all children.
+         */
+        if (afr_is_special_xattr (name, &cbk, 1)) {
+                afr_fgetxattr_frm_all_children (this, frame, name,
+                                                fd, cbk);
+                return 0;
+        }
+
 
         local->fresh_children = afr_children_create (priv->child_count);
         if (!local->fresh_children) {
@@ -1132,7 +1765,8 @@ afr_fgetxattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode, local->fresh_children);
+        read_child = afr_inode_get_read_ctx (this, fd->inode,
+                                             local->fresh_children);
         op_ret = afr_get_call_child (this, local->child_up, read_child,
                                      local->fresh_children,
                                      &call_child,
@@ -1147,12 +1781,13 @@ afr_fgetxattr (call_frame_t *frame, xlator_t *this,
                            (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->fgetxattr,
-                           fd, name);
+                           fd, name, xdata);
 
         op_ret = 0;
 out:
         if (op_ret == -1) {
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, NULL);
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, NULL,
+                                  NULL);
         }
         return 0;
 }
@@ -1178,7 +1813,7 @@ int32_t
 afr_readv_cbk (call_frame_t *frame, void *cookie,
                xlator_t *this, int32_t op_ret, int32_t op_errno,
                struct iovec *vector, int32_t count, struct iatt *buf,
-               struct iobref *iobref)
+               struct iobref *iobref, dict_t *xdata)
 {
         afr_private_t * priv            = NULL;
         afr_local_t *   local           = NULL;
@@ -1220,13 +1855,14 @@ afr_readv_cbk (call_frame_t *frame, void *cookie,
                                    children[next_call_child]->fops->readv,
                                    local->fd, local->cont.readv.size,
                                    local->cont.readv.offset,
-                                   local->cont.readv.flags);
+                                   local->cont.readv.flags,
+                                   NULL);
         }
 
 out:
         if (unwind) {
                 AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
-                                  vector, count, buf, iobref);
+                                  vector, count, buf, iobref, xdata);
         }
 
         return 0;
@@ -1235,7 +1871,7 @@ out:
 
 int32_t
 afr_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t offset, uint32_t flags)
+           fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         afr_private_t * priv       = NULL;
         afr_local_t   * local      = NULL;
@@ -1253,7 +1889,9 @@ afr_readv (call_frame_t *frame, xlator_t *this,
         priv     = this->private;
         children = priv->children;
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_SBRAIN_CHECK_FD (fd, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1282,22 +1920,19 @@ afr_readv (call_frame_t *frame, xlator_t *this,
         local->cont.readv.offset     = offset;
         local->cont.readv.flags      = flags;
 
-        ret = afr_open_fd_fix (frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+        afr_open_fd_fix (fd, this);
+
         STACK_WIND_COOKIE (frame, afr_readv_cbk,
                            (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->readv,
-                           fd, size, offset, flags);
+                           fd, size, offset, flags, xdata);
 
         ret = 0;
 out:
         if (ret < 0) {
                 AFR_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL,
-                                  NULL);
+                                  NULL, NULL);
         }
         return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-inode-read.h b/xlators/cluster/afr/src/afr-inode-read.h
index 5ec7411b1..e4091a793 100644
--- a/xlators/cluster/afr/src/afr-inode-read.h
+++ b/xlators/cluster/afr/src/afr-inode-read.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_READ_H__
@@ -22,30 +13,30 @@
 
 int32_t
 afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask);
+	    loc_t *loc, int32_t mask, dict_t *xdata);
 
 int32_t
 afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc);
+	  loc_t *loc, dict_t *xdata);
 
 int32_t
 afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd);
+	   fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size);
+	      loc_t *loc, size_t size, dict_t *xdata);
 
 int32_t
 afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset, uint32_t flags);
+	   fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t
 afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name);
+	      loc_t *loc, const char *name, dict_t *xdata);
 
 int32_t
 afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name);
+               fd_t *fd, const char *name, dict_t *xdata);
 
 #endif /* __INODE_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 72dcdc478..c1ec69a55 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -48,45 +39,151 @@
 #include "afr-transaction.h"
 #include "afr-self-heal-common.h"
 
+void
+__inode_write_fop_cbk (call_frame_t *frame, int child_index, int read_child,
+                       xlator_t *this, int32_t *op_ret, int32_t *op_errno,
+                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t     *local = NULL;
+
+        local = frame->local;
+
+        if (afr_fop_failed (*op_ret, *op_errno)) {
+                local->child_errno[child_index] = *op_errno;
+
+                switch (local->op) {
+                case GF_FOP_TRUNCATE:
+                case GF_FOP_FTRUNCATE:
+                        if (*op_errno != EFBIG)
+                                afr_transaction_fop_failed (frame, this,
+                                                            child_index);
+                break;
+                default:
+                        afr_transaction_fop_failed (frame, this, child_index);
+                break;
+                }
+                local->op_errno = *op_errno;
+                goto out;
+        }
+
+        if ((local->success_count == 0) || (read_child == child_index)) {
+                local->op_ret              = *op_ret;
+                if (prebuf)
+                        local->cont.inode_wfop.prebuf  = *prebuf;
+                if (postbuf)
+                        local->cont.inode_wfop.postbuf = *postbuf;
+        }
+
+        local->success_count++;
+out:
+        return;
+}
+
 /* {{{ writev */
 
-int
+void
+afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame)
+{
+        afr_local_t *src_local = NULL;
+        afr_local_t *dst_local = NULL;
+
+        src_local = src_frame->local;
+        dst_local = dst_frame->local;
+
+        dst_local->op_ret = src_local->op_ret;
+        dst_local->op_errno = src_local->op_errno;
+        dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+        dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+}
+
+void
 afr_writev_unwind (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+        local = frame->local;
+
+        AFR_STACK_UNWIND (writev, frame,
+                          local->op_ret, local->op_errno,
+                          &local->cont.inode_wfop.prebuf,
+                          &local->cont.inode_wfop.postbuf,
+                          NULL);
+}
+
+call_frame_t*
+afr_transaction_detach_fop_frame (call_frame_t *frame)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *fop_frame = NULL;
 
         local = frame->local;
 
         LOCK (&frame->lock);
         {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
+                fop_frame = local->transaction.main_frame;
                 local->transaction.main_frame = NULL;
         }
         UNLOCK (&frame->lock);
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (writev, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.writev.prebuf,
-                                  &local->cont.writev.postbuf);
+        return fop_frame;
+}
+
+int
+afr_transaction_writev_unwind (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t *fop_frame = NULL;
+
+        fop_frame = afr_transaction_detach_fop_frame (frame);
+
+        if (fop_frame) {
+                afr_writev_copy_outvars (frame, fop_frame);
+                afr_writev_unwind (fop_frame, this);
         }
         return 0;
 }
 
+static void
+afr_writev_handle_short_writes (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int           i      = 0;
+
+        local = frame->local;
+        priv = this->private;
+        /*
+         * We already have the best case result of the writev calls staged
+         * as the return value. Any writev that returns some value less
+         * than the best case is now out of sync, so mark the fop as
+         * failed. Note that fops that have returned with errors have
+         * already been marked as failed.
+         */
+        for (i = 0; i < priv->child_count; i++) {
+                if ((!local->replies[i].valid) ||
+                    (local->replies[i].op_ret == -1))
+                        continue;
+
+                if (local->replies[i].op_ret < local->op_ret)
+                        afr_transaction_fop_failed(frame, this, i);
+        }
+}
 
 int
 afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
+        afr_private_t  *priv  = NULL;
+        call_frame_t    *fop_frame = NULL;
         int child_index = (long) cookie;
         int call_count  = -1;
         int read_child  = 0;
+        int      ret = 0;
+        uint32_t open_fd_count = 0;
+        uint32_t write_is_append = 0;
 
         local = frame->local;
+        priv  = this->private;
 
         read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
 
@@ -96,32 +193,81 @@ afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret              = op_ret;
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+
+		local->replies[child_index].valid = 1;
+		local->replies[child_index].op_ret = op_ret;
+		local->replies[child_index].op_errno = op_errno;
+
+
+		/* stage the best case return value for unwind */
+                if ((local->success_count == 0) || (op_ret > local->op_ret)) {
+                        local->op_ret              = op_ret;
+			local->op_errno		   = op_errno;
+		}
+
+		if (op_ret != -1) {
+                        if (xdata) {
+                                ret = dict_get_uint32 (xdata,
+                                                       GLUSTERFS_OPEN_FD_COUNT,
+                                                       &open_fd_count);
+                                if ((ret == 0) &&
+                                    (open_fd_count > local->open_fd_count)) {
+                                        local->open_fd_count = open_fd_count;
+                                        local->update_open_fd_count = _gf_true;
+                                }
+
+				write_is_append = 0;
+                                ret = dict_get_uint32 (xdata,
+                                                       GLUSTERFS_WRITE_IS_APPEND,
+                                                       &write_is_append);
+                                if (ret || !write_is_append)
+					local->append_write = _gf_false;
                         }
-                }
 
-                local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                local->transaction.unwind (frame, this);
 
-                local->transaction.resume (frame, this);
+                if (local->update_open_fd_count)
+                        afr_handle_open_fd_count (frame, this);
+
+                if (!local->stable_write && !local->append_write)
+			/* An appended write removes the necessity to
+			   fsync() the file. This is because self-heal
+			   has the logic to check for larger file when
+			   the xattrs are not reliably pointing at
+			   a stale file.
+			*/
+                        afr_fd_report_unstable_write (this, local->fd);
+
+                afr_writev_handle_short_writes (frame, this);
+                if (afr_any_fops_failed (local, priv)) {
+                        //Don't unwind until post-op is complete
+                        local->transaction.resume (frame, this);
+                } else {
+                /*
+                 * Generally inode-write fops do transaction.unwind then
+                 * transaction.resume, but writev needs to make sure that
+                 * delayed post-op frame is placed in fdctx before unwind
+                 * happens. This prevents the race of flush doing the
+                 * changelog wakeup first in fuse thread and then this
+                 * writev placing its delayed post-op frame in fdctx.
+                 * This helps flush make sure all the delayed post-ops are
+                 * completed.
+                 */
+
+                        fop_frame = afr_transaction_detach_fop_frame (frame);
+                        afr_writev_copy_outvars (frame, fop_frame);
+                        local->transaction.resume (frame, this);
+                        afr_writev_unwind (fop_frame, this);
+                }
         }
         return 0;
 }
@@ -133,6 +279,8 @@ afr_writev_wind (call_frame_t *frame, xlator_t *this)
         afr_private_t *priv = NULL;
         int i = 0;
         int call_count = -1;
+        dict_t *xdata = NULL;
+        GF_UNUSED int     ret = 0;
 
         local = frame->local;
         priv = this->private;
@@ -146,6 +294,28 @@ afr_writev_wind (call_frame_t *frame, xlator_t *this)
         }
 
         local->call_count = call_count;
+	local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+				   gf_afr_mt_reply_t);
+	if (!local->replies) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		local->transaction.unwind(frame, this);
+		local->transaction.resume(frame, this);
+		return 0;
+	}
+
+        xdata = dict_new ();
+        if (xdata) {
+                ret = dict_set_uint32 (xdata, GLUSTERFS_OPEN_FD_COUNT,
+                                       sizeof (uint32_t));
+		ret = dict_set_uint32 (xdata, GLUSTERFS_WRITE_IS_APPEND,
+				       0);
+		/* Set append_write to be true speculatively. If on any
+		   server it turns not be true, we unset it in the
+		   callback.
+		*/
+		local->append_write = _gf_true;
+        }
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->transaction.pre_op[i]) {
@@ -158,13 +328,17 @@ afr_writev_wind (call_frame_t *frame, xlator_t *this)
                                            local->cont.writev.count,
                                            local->cont.writev.offset,
                                            local->cont.writev.flags,
-                                           local->cont.writev.iobref);
+                                           local->cont.writev.iobref,
+                                           xdata);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
+        if (xdata)
+                dict_unref (xdata);
+
         return 0;
 }
 
@@ -204,7 +378,7 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
         }
 
         transaction_frame->local = local;
-        frame->local = NULL;
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
 
         local->op = GF_FOP_WRITE;
 
@@ -212,10 +386,17 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
 
         local->transaction.fop    = afr_writev_wind;
         local->transaction.done   = afr_writev_done;
-        local->transaction.unwind = afr_writev_unwind;
+        local->transaction.unwind = afr_transaction_writev_unwind;
 
         local->transaction.main_frame = frame;
         if (local->fd->flags & O_APPEND) {
+               /*
+                * Backend vfs ignores the 'offset' for append mode fd so
+                * locking just the region provided for the writev does not
+                * give consistency gurantee. The actual write may happen at a
+                * completely different range than the one provided by the
+                * offset, len in the fop. So lock the entire file.
+                */
                 local->transaction.start   = 0;
                 local->transaction.len     = 0;
         } else {
@@ -224,162 +405,91 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
                                                          local->cont.writev.count);
         }
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
 
         op_ret = 0;
 out:
-        if (op_ret == -1) {
+        if (op_ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
 }
 
-static int
-afr_prepare_loc (call_frame_t *frame, fd_t *fd)
+static void
+afr_trigger_open_fd_self_heal (fd_t *fd, xlator_t *this)
 {
-        afr_local_t    *local = NULL;
-        char           *name = NULL;
-        char           *path = NULL;
-        int             ret = 0;
-
-        if ((!fd) || (!fd->inode))
-                return -1;
-
-        local = frame->local;
-        ret = inode_path (fd->inode, NULL, (char **)&path);
-        if (ret <= 0) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "Unable to get path for gfid: %s",
-                        uuid_utoa (fd->inode->gfid));
-                return -1;
-        }
-
-        if (local->loc.path) {
-                if (strcmp (path, local->loc.path))
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "overwriting old loc->path %s with %s",
-                                local->loc.path, path);
-                GF_FREE ((char *)local->loc.path);
-        }
-        local->loc.path = path;
-
-        name = strrchr (local->loc.path, '/');
-        if (name)
-                name++;
-        local->loc.name = name;
-
-        if (local->loc.inode) {
-                inode_unref (local->loc.inode);
-        }
-        local->loc.inode = inode_ref (fd->inode);
-
-        if (local->loc.parent) {
-                inode_unref (local->loc.parent);
+        call_frame_t    *frame   = NULL;
+        afr_local_t     *local   = NULL;
+        afr_self_heal_t *sh      = NULL;
+        char            *reason  = NULL;
+        int32_t         op_errno = 0;
+        int             ret      = 0;
+
+        if (!fd || !fd->inode || uuid_is_null (fd->inode->gfid)) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid args: "
+                                  "fd: %p, inode: %p", fd,
+                                  fd ? fd->inode : NULL);
+                goto out;
         }
 
-        local->loc.parent = inode_parent (local->loc.inode, 0, NULL);
-
-        return 0;
-}
-
-afr_fd_paused_call_t*
-afr_paused_call_create (call_frame_t *frame)
-{
-        afr_local_t             *local = NULL;
-        afr_fd_paused_call_t   *paused_call = NULL;
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
 
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
-        GF_ASSERT (local->fop_call_continue);
-
-        paused_call = GF_CALLOC (1, sizeof (*paused_call),
-                                  gf_afr_fd_paused_call_t);
-        if (paused_call) {
-                INIT_LIST_HEAD (&paused_call->call_list);
-                paused_call->frame = frame;
-        }
-
-        return paused_call;
-}
-
-static int
-afr_pause_fd_fop (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        int                    ret = 0;
-
-        paused_call = afr_paused_call_create (frame);
-        if (paused_call)
-                list_add (&paused_call->call_list, &fd_ctx->paused_calls);
-        else
-                ret = -ENOMEM;
-
-        return ret;
-}
+        ret = afr_local_init (local, this->private, &op_errno);
+        if (ret < 0)
+                goto out;
 
-static void
-afr_trigger_open_fd_self_heal (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t             *local = NULL;
-        afr_self_heal_t         *sh = NULL;
-        inode_t                 *inode = NULL;
-        char                    *reason = NULL;
+        local->loc.inode = inode_ref (fd->inode);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
 
-        local = frame->local;
         sh    = &local->self_heal;
-        inode = local->fd->inode;
-
-        sh->do_missing_entry_self_heal = _gf_true;
-        sh->do_gfid_self_heal = _gf_true;
-        sh->do_data_self_heal = _gf_true;
+        sh->do_metadata_self_heal = _gf_true;
+        if (fd->inode->ia_type == IA_IFREG)
+                sh->do_data_self_heal = _gf_true;
+        else if (fd->inode->ia_type == IA_IFDIR)
+                sh->do_entry_self_heal = _gf_true;
 
         reason = "subvolume came online";
-        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
-                              reason, NULL, NULL);
+        afr_launch_self_heal (frame, this, fd->inode, _gf_true,
+                              fd->inode->ia_type, reason, NULL, NULL);
+        return;
+out:
+        AFR_STACK_DESTROY (frame);
 }
 
-int
-afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop)
-{
-        int                     ret = 0;
-        int                     i   = 0;
-        afr_fd_ctx_t            *fd_ctx = NULL;
-        gf_boolean_t            need_self_heal = _gf_false;
-        int                     *need_open = NULL;
-        int                     need_open_count = 0;
-        afr_local_t             *local = NULL;
-        afr_private_t           *priv = NULL;
-        gf_boolean_t            fop_continue = _gf_true;
+void
+afr_open_fd_fix (fd_t *fd, xlator_t *this)
+{
+        int           ret             = 0;
+        int           i               = 0;
+        afr_fd_ctx_t  *fd_ctx         = NULL;
+        gf_boolean_t  need_self_heal  = _gf_false;
+        int           *need_open      = NULL;
+        size_t        need_open_count = 0;
+        afr_private_t *priv           = NULL;
 
-        local = frame->local;
         priv  = this->private;
 
-        GF_ASSERT (local->fd);
-
-        if (fd_is_anonymous (local->fd)) {
-                fop_continue = _gf_true;
-                goto out;
-        }
-
-        if (pause_fop)
-                GF_ASSERT (local->fop_call_continue);
-
-        ret = afr_prepare_loc (frame, local->fd);
-        if (ret < 0) {
-                //File does not exist we cant open it.
-                ret = 0;
+        if (!afr_is_fd_fixable (fd))
                 goto out;
-        }
 
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-        if (!fd_ctx) {
-                ret = -EINVAL;
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx)
                 goto out;
-        }
 
-        LOCK (&local->fd->lock);
+        LOCK (&fd->lock);
         {
                 if (fd_ctx->up_count < priv->up_count) {
                         need_self_heal = _gf_true;
@@ -387,62 +497,40 @@ afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop)
                         fd_ctx->down_count = priv->down_count;
                 }
 
+                need_open = alloca (priv->child_count * sizeof (*need_open));
                 for (i = 0; i < priv->child_count; i++) {
-                        if ((fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED) &&
-                            local->child_up[i]) {
-                                fd_ctx->opened_on[i] = AFR_FD_OPENING;
-                                if (!need_open)
-                                        need_open = GF_CALLOC (priv->child_count,
-                                                               sizeof (*need_open),
-                                                               gf_afr_mt_int32_t);
-                                need_open[i] = 1;
-                                need_open_count++;
-                        } else if (pause_fop && local->child_up[i] &&
-                                   (fd_ctx->opened_on[i] == AFR_FD_OPENING)) {
-                                local->fop_paused = _gf_true;
-                        }
-                }
+                        need_open[i] = 0;
+                        if (fd_ctx->opened_on[i] != AFR_FD_NOT_OPENED)
+                                continue;
 
-                if (local->fop_paused) {
-                        GF_ASSERT (pause_fop);
-                        gf_log (this->name, GF_LOG_INFO, "Pause fd %p",
-                                local->fd);
-                        ret = afr_pause_fd_fop (frame, this, fd_ctx);
-                        if (ret)
-                                goto unlock;
-                        fop_continue = _gf_false;
+                        if (!priv->child_up[i])
+                                continue;
+
+                        fd_ctx->opened_on[i] = AFR_FD_OPENING;
+
+                        need_open[i] = 1;
+                        need_open_count++;
                 }
         }
-unlock:
-        UNLOCK (&local->fd->lock);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to fix fd for %s",
-                        local->loc.path);
-                fop_continue = _gf_false;
+        UNLOCK (&fd->lock);
+        if (ret)
                 goto out;
-        }
 
         if (need_self_heal)
-                afr_trigger_open_fd_self_heal (frame, this);
+                afr_trigger_open_fd_self_heal (fd, this);
 
         if (!need_open_count)
                 goto out;
 
-        gf_log (this->name, GF_LOG_INFO, "Opening fd %p", local->fd);
-        afr_fix_open (frame, this, fd_ctx, need_open_count, need_open);
-        fop_continue = _gf_false;
+        afr_fix_open (this, fd, need_open_count, need_open);
 out:
-        if (need_open)
-                GF_FREE (need_open);
-        if (fop_continue && local->fop_call_continue)
-                local->fop_call_continue (frame, this);
-        return ret;
+        return;
 }
 
 int
 afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref)
+            uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -455,9 +543,14 @@ afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         priv = this->private;
 
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+
         QUORUM_CHECK(writev,out);
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -471,18 +564,20 @@ afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         local->cont.writev.iobref     = iobref_ref (iobref);
 
         local->fd                = fd_ref (fd);
-        local->fop_call_continue = afr_do_writev;
 
-        ret = afr_open_fd_fix (frame, this, _gf_true);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+	/* detect here, but set it in writev_wind_cbk *after* the unstable
+	   write is performed
+	*/
+	local->stable_write = !!((fd->flags|flags)&(O_SYNC|O_DSYNC));
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_writev (frame, this);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -511,8 +606,9 @@ afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (truncate, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.truncate.prebuf,
-                                  &local->cont.truncate.postbuf);
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
         }
 
         return 0;
@@ -522,17 +618,14 @@ afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf)
+                       struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
         int child_index = (long) cookie;
         int read_child  = 0;
         int call_count  = -1;
-        int need_unwind = 0;
 
         local = frame->local;
-        priv  = this->private;
 
         read_child = afr_inode_get_read_ctx (this, local->loc.inode, NULL);
 
@@ -542,38 +635,22 @@ afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != EFBIG)
-                        afr_transaction_fop_failed (frame, this, child_index);
-
                 if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
+			if (prebuf->ia_size != postbuf->ia_size)
+				local->stable_write = _gf_false;
                 }
-                local->op_errno = op_errno;
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
         }
         UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
+		if (local->stable_write && afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
+
                 local->transaction.resume (frame, this);
         }
 
@@ -601,6 +678,7 @@ afr_truncate_wind (call_frame_t *frame, xlator_t *this)
         }
 
         local->call_count = call_count;
+	local->stable_write = _gf_true;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->transaction.pre_op[i]) {
@@ -609,7 +687,8 @@ afr_truncate_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i],
                                            priv->children[i]->fops->truncate,
                                            &local->loc,
-                                           local->cont.truncate.offset);
+                                           local->cont.truncate.offset,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -637,7 +716,7 @@ afr_truncate_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_truncate (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, off_t offset)
+              loc_t *loc, off_t offset, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -659,7 +738,7 @@ afr_truncate (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -678,14 +757,18 @@ afr_truncate (call_frame_t *frame, xlator_t *this,
         local->transaction.start   = offset;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
@@ -716,8 +799,9 @@ afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.ftruncate.prebuf,
-                                  &local->cont.ftruncate.postbuf);
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
         }
         return 0;
 }
@@ -726,17 +810,14 @@ afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
+                        struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
         int child_index = (long) cookie;
         int call_count  = -1;
-        int need_unwind = 0;
         int read_child  = 0;
 
         local = frame->local;
-        priv  = this->private;
 
         read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
 
@@ -746,38 +827,22 @@ afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
                 if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
+			if (prebuf->ia_size != postbuf->ia_size)
+				local->stable_write = _gf_false;
                 }
-                local->op_errno = op_errno;
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
         }
         UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
+		if (local->stable_write && afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
+
                 local->transaction.resume (frame, this);
         }
 
@@ -805,6 +870,7 @@ afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
         }
 
         local->call_count = call_count;
+	local->stable_write = _gf_true;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->transaction.pre_op[i]) {
@@ -812,7 +878,9 @@ afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->ftruncate,
-                                           local->fd, local->cont.ftruncate.offset);
+                                           local->fd,
+                                           local->cont.ftruncate.offset,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -867,14 +935,19 @@ afr_do_ftruncate (call_frame_t *frame, xlator_t *this)
         local->transaction.start   = local->cont.ftruncate.offset;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
 
         op_ret = 0;
 out:
-        if (op_ret == -1) {
+        if (op_ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
         }
 
         return 0;
@@ -883,7 +956,7 @@ out:
 
 int
 afr_ftruncate (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, off_t offset)
+               fd_t *fd, off_t offset, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -897,9 +970,13 @@ afr_ftruncate (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
         QUORUM_CHECK(ftruncate,out);
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -909,20 +986,17 @@ afr_ftruncate (call_frame_t *frame, xlator_t *this,
         local->cont.ftruncate.offset  = offset;
 
         local->fd = fd_ref (fd);
-        local->fop_call_continue = afr_do_ftruncate;
 
-        ret = afr_open_fd_fix (frame, this, _gf_true);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+        afr_open_fd_fix (fd, this);
+
+        afr_do_ftruncate (frame, this);
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
@@ -951,8 +1025,9 @@ afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (setattr, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.setattr.preop_buf,
-                                  &local->cont.setattr.postop_buf);
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
         }
 
         return 0;
@@ -962,7 +1037,7 @@ afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop)
+                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = NULL;
@@ -982,29 +1057,14 @@ afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, preop, postop,
+                                       xdata);
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
                 }
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1050,7 +1110,8 @@ afr_setattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->setattr,
                                            &local->loc,
                                            &local->cont.setattr.in_buf,
-                                           local->cont.setattr.valid);
+                                           local->cont.setattr.valid,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1078,7 +1139,7 @@ afr_setattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid)
+             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -1100,7 +1161,7 @@ afr_setattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1120,14 +1181,18 @@ afr_setattr (call_frame_t *frame, xlator_t *this,
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1154,8 +1219,9 @@ afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.fsetattr.preop_buf,
-                                  &local->cont.fsetattr.postop_buf);
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
         }
 
         return 0;
@@ -1165,7 +1231,7 @@ afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop)
+                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = NULL;
@@ -1185,29 +1251,14 @@ afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, preop, postop,
+                                       xdata);
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
                 }
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1253,7 +1304,8 @@ afr_fsetattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->fsetattr,
                                            local->fd,
                                            &local->cont.fsetattr.in_buf,
-                                           local->cont.fsetattr.valid);
+                                           local->cont.fsetattr.valid,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1280,7 +1332,7 @@ afr_fsetattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid)
+              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -1294,6 +1346,11 @@ afr_fsetattr (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+
         QUORUM_CHECK(fsetattr,out);
 
         transaction_frame = copy_frame (frame);
@@ -1302,7 +1359,7 @@ afr_fsetattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1318,24 +1375,24 @@ afr_fsetattr (call_frame_t *frame, xlator_t *this,
 
         local->fd                 = fd_ref (fd);
 
-        ret = afr_open_fd_fix (transaction_frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+        afr_open_fd_fix (fd, this);
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1363,7 +1420,8 @@ afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
 
         if (main_frame) {
                 AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno);
+                                  local->op_ret, local->op_errno,
+                                  NULL);
         }
         return 0;
 }
@@ -1371,30 +1429,25 @@ afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
 
 int
 afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t      *local         = NULL;
-        afr_private_t    *priv          = NULL;
-        int               call_count    = -1;
-        int               need_unwind   = 0;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->child_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1440,7 +1493,8 @@ afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->setxattr,
                                            &local->loc,
                                            local->cont.setxattr.dict,
-                                           local->cont.setxattr.flags);
+                                           local->cont.setxattr.flags,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1465,24 +1519,24 @@ afr_setxattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags)
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
 {
         afr_private_t  *priv              = NULL;
         afr_local_t    *local             = NULL;
         call_frame_t   *transaction_frame = NULL;
-        data_pair_t    *trav              = NULL;
         int             ret               = -1;
         int             op_errno          = EINVAL;
 
-        VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
         priv = this->private;
 
@@ -1493,7 +1547,7 @@ afr_setxattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1513,14 +1567,18 @@ afr_setxattr (call_frame_t *frame, xlator_t *this,
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno);
+                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
         }
 
         return 0;
@@ -1547,7 +1605,8 @@ afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
 
         if (main_frame) {
                 AFR_STACK_UNWIND (fsetxattr, main_frame,
-                                  local->op_ret, local->op_errno);
+                                  local->op_ret, local->op_errno,
+                                  NULL);
         }
         return 0;
 }
@@ -1555,30 +1614,26 @@ afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
 
 int
 afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t     *local       = NULL;
-        afr_private_t   *priv        = NULL;
-        int              call_count  = -1;
-        int              need_unwind = 0;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
 
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->child_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1624,7 +1679,8 @@ afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->fsetxattr,
                                            local->fd,
                                            local->cont.fsetxattr.dict,
-                                           local->cont.fsetxattr.flags);
+                                           local->cont.fsetxattr.flags,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1649,33 +1705,34 @@ afr_fsetxattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags)
+               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
 {
         afr_private_t    *priv              = NULL;
         afr_local_t      *local             = NULL;
         call_frame_t     *transaction_frame = NULL;
         int               ret               = -1;
         int               op_errno          = EINVAL;
-        data_pair_t      *trav              = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
         VALIDATE_OR_GOTO (this->private, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
-
-        if (ret)
-                goto out;
+                                   op_errno, out);
 
         priv = this->private;
 
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+
         QUORUM_CHECK(fsetxattr,out);
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
 
         ret = afr_local_init (local, priv, &op_errno);
         if (ret < 0)
@@ -1703,14 +1760,18 @@ afr_fsetxattr (call_frame_t *frame, xlator_t *this,
         local->transaction.start  = LLONG_MAX - 1;
         local->transaction.len    = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno);
+                AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
         }
 
         return 0;
@@ -1740,7 +1801,8 @@ afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
 
         if (main_frame) {
                 AFR_STACK_UNWIND (removexattr, main_frame,
-                                  local->op_ret, local->op_errno);
+                                  local->op_ret, local->op_errno,
+                                  NULL);
         }
         return 0;
 }
@@ -1748,30 +1810,25 @@ afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
 
 int
 afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->wait_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1816,7 +1873,8 @@ afr_removexattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i],
                                            priv->children[i]->fops->removexattr,
                                            &local->loc,
-                                           local->cont.removexattr.name);
+                                           local->cont.removexattr.name,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1842,16 +1900,23 @@ afr_removexattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name)
+                 loc_t *loc, const char *name, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t   *priv              = NULL;
+        afr_local_t     *local             = NULL;
+        call_frame_t    *transaction_frame = NULL;
+        int              ret               = -1;
+        int              op_errno          = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
+                                 name, op_errno, out);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
+                                 name, op_errno, out);
+
+        VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this->private, out);
         VALIDATE_OR_GOTO (loc, out);
 
@@ -1865,7 +1930,7 @@ afr_removexattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (transaction_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
         local = transaction_frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1884,14 +1949,18 @@ afr_removexattr (call_frame_t *frame, xlator_t *this,
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno);
+                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
         }
 
         return 0;
@@ -1916,7 +1985,8 @@ afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
 
         if (main_frame) {
                 AFR_STACK_UNWIND (fremovexattr, main_frame,
-                                  local->op_ret, local->op_errno);
+                                  local->op_ret, local->op_errno,
+                                  NULL);
         }
         return 0;
 }
@@ -1924,30 +1994,26 @@ afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
 
 int
 afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
+        afr_local_t *   local       = NULL;
+        afr_private_t * priv        = NULL;
+        int             call_count  = -1;
+        int             need_unwind = 0;
+        int             child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
 
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
+                if (local->success_count == priv->wait_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1992,7 +2058,8 @@ afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i],
                                            priv->children[i]->fops->fremovexattr,
                                            local->fd,
-                                           local->cont.removexattr.name);
+                                           local->cont.removexattr.name,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -2018,7 +2085,7 @@ afr_fremovexattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name)
+                  fd_t *fd, const char *name, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -2027,11 +2094,22 @@ afr_fremovexattr (call_frame_t *frame, xlator_t *this,
         int op_ret   = -1;
         int op_errno = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
+                                 name, op_errno, out);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
+                                 name, op_errno, out);
+
+        VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this->private, out);
 
         priv = this->private;
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
 
         QUORUM_CHECK(fremovexattr, out);
 
@@ -2040,7 +2118,7 @@ afr_fremovexattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
 
         ret = afr_local_init (local, priv, &op_errno);
         if (ret < 0) {
@@ -2064,15 +2142,720 @@ afr_fremovexattr (call_frame_t *frame, xlator_t *this,
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        op_ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
+        }
+
+        return 0;
+}
+
+static int
+afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
+        return 0;
+}
+
+static int
+afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int need_unwind = 0;
+        int read_child  = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
+
+static int
+afr_fallocate_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fallocate,
+                                           local->fd,
+                                           local->cont.fallocate.mode,
+                                           local->cont.fallocate.offset,
+                                           local->cont.fallocate.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+static int
+afr_fallocate_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->transaction.unwind (frame, this);
+
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+static int
+afr_do_fallocate (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t * transaction_frame = NULL;
+        afr_local_t *  local             = NULL;
+        int op_ret   = -1;
+        int op_errno = 0;
+
+        local = frame->local;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
+
+        transaction_frame->local = local;
+        frame->local = NULL;
+
+        local->op = GF_FOP_FALLOCATE;
+
+        local->transaction.fop    = afr_fallocate_wind;
+        local->transaction.done   = afr_fallocate_done;
+        local->transaction.unwind = afr_fallocate_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.fallocate.offset;
+        local->transaction.len     = 0;
+
+        /* fallocate can modify the file size */
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
 
         op_ret = 0;
 out:
-        if (op_ret == -1) {
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fallocate, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata)
+{
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(fallocate,out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->cont.fallocate.mode = mode;
+        local->cont.fallocate.offset  = offset;
+        local->cont.fallocate.len = len;
+
+        local->fd = fd_ref (fd);
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_fallocate (frame, this);
+
+        ret = 0;
+out:
+        if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno);
+                AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        }
+
+        return 0;
+}
+
+/* }}} */
+
+/* {{{ discard */
+
+static int
+afr_discard_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
         }
+        UNLOCK (&frame->lock);
 
+        if (main_frame) {
+                AFR_STACK_UNWIND (discard, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
         return 0;
 }
+
+static int
+afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int need_unwind = 0;
+        int read_child  = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
+
+static int
+afr_discard_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_discard_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->discard,
+                                           local->fd,
+                                           local->cont.discard.offset,
+                                           local->cont.discard.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+static int
+afr_discard_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->transaction.unwind (frame, this);
+
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+static int
+afr_do_discard (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t * transaction_frame = NULL;
+        afr_local_t *  local             = NULL;
+        int op_ret   = -1;
+        int op_errno = 0;
+
+        local = frame->local;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
+
+        transaction_frame->local = local;
+        frame->local = NULL;
+
+        local->op = GF_FOP_DISCARD;
+
+        local->transaction.fop    = afr_discard_wind;
+        local->transaction.done   = afr_discard_done;
+        local->transaction.unwind = afr_discard_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.discard.offset;
+        local->transaction.len     = 0;
+
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (discard, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(discard, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->cont.discard.offset  = offset;
+        local->cont.discard.len = len;
+
+        local->fd = fd_ref (fd);
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_discard(frame, this);
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        }
+
+        return 0;
+}
+
+
+/* {{{ zerofill */
+
+static int
+afr_zerofill_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local            = NULL;
+        call_frame_t    *main_frame       = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (zerofill, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.zerofill.prebuf,
+                                  &local->cont.zerofill.postbuf,
+                                  NULL);
+        }
+        return 0;
+}
+
+static int
+afr_zerofill_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t       *local             = NULL;
+        afr_private_t     *priv              = NULL;
+        int                child_index       = (long) cookie;
+        int                call_count        = -1;
+        int                need_unwind       = 0;
+        int                read_child        = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                if (afr_fop_failed (op_ret, op_errno)) {
+                        afr_transaction_fop_failed (frame, this, child_index);
+                }
+
+                if (op_ret != -1) {
+                        if (local->success_count == 0) {
+                                local->op_ret = op_ret;
+                                local->cont.zerofill.prebuf  = *prebuf;
+                                local->cont.zerofill.postbuf = *postbuf;
+                        }
+
+                        if (child_index == read_child) {
+                                local->cont.zerofill.prebuf  = *prebuf;
+                                local->cont.zerofill.postbuf = *postbuf;
+                        }
+
+                        local->success_count++;
+
+                        if ((local->success_count >= priv->wait_count)
+                            && local->read_child_returned) {
+                                need_unwind = 1;
+                        }
+                }
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind) {
+                local->transaction.unwind (frame, this);
+        }
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
+
+static int
+afr_zerofill_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t    *local         = NULL;
+        afr_private_t  *priv          = NULL;
+        int             call_count    = -1;
+        int             i             = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_zerofill_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->zerofill,
+                                           local->fd,
+                                           local->cont.zerofill.offset,
+                                           local->cont.zerofill.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+static int
+afr_zerofill_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->transaction.unwind (frame, this);
+
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+static int
+afr_do_zerofill(call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t  *transaction_frame = NULL;
+        afr_local_t   *local             = NULL;
+        int            op_ret            = -1;
+        int            op_errno          = 0;
+
+        local = frame->local;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
+
+        transaction_frame->local = local;
+        frame->local = NULL;
+
+        local->op = GF_FOP_ZEROFILL;
+
+        local->transaction.fop    = afr_zerofill_wind;
+        local->transaction.done   = afr_zerofill_done;
+        local->transaction.unwind = afr_zerofill_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.zerofill.offset;
+        local->transaction.len     = 0;
+
+        op_ret = afr_transaction (transaction_frame, this,
+                                  AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+                op_errno = -op_ret;
+                goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame) {
+                        AFR_STACK_DESTROY (transaction_frame);
+                }
+                AFR_STACK_UNWIND (zerofill, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+int
+afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              size_t len, dict_t *xdata)
+{
+        afr_private_t   *priv               = NULL;
+        afr_local_t     *local              = NULL;
+        call_frame_t    *transaction_frame  = NULL;
+        int              ret                = -1;
+        int              op_errno           = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(zerofill, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0) {
+                goto out;
+        }
+        local->cont.zerofill.offset  = offset;
+        local->cont.zerofill.len = len;
+
+        local->fd = fd_ref (fd);
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_zerofill(frame, this);
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame) {
+                        AFR_STACK_DESTROY (transaction_frame);
+                }
+                AFR_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+/* }}} */
+
+
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index 729a490d5..8e93ca44a 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_WRITE_H__
@@ -22,59 +13,70 @@
 
 int32_t
 afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode);
+	   loc_t *loc, mode_t mode, dict_t *xdata);
 
 int32_t
 afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid);
+	   loc_t *loc, uid_t uid, gid_t gid, dict_t *xdata);
 
 int
 afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid);
+	    fd_t *fd, uid_t uid, gid_t gid, dict_t *xdata);
 
 int32_t
 afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode);
+	    fd_t *fd, mode_t mode, dict_t *xdata);
 
 int32_t
 afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	    struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref);
+            uint32_t flags, struct iobref *iobref, dict_t *xdata);
 
 int32_t
 afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset);
+	      loc_t *loc, off_t offset, dict_t *xdata);
 
 int32_t
 afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset);
+	       fd_t *fd, off_t offset, dict_t *xdata);
 
 int32_t
 afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2]);
+	     loc_t *loc, struct timespec tv[2], dict_t *xdata);
 
 int
 afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid);
+             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata);
 
 int
 afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid);
+              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata);
 
 int32_t
 afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags);
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata);
 
 int32_t
 afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags);
+               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata);
 
 int32_t
 afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name);
+		 loc_t *loc, const char *name, dict_t *xdata);
 
 int32_t
 afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name);
+                  fd_t *fd, const char *name, dict_t *xdata);
+
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
 
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata);
+
+int
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index 2fe134990..060d78f35 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "dict.h"
@@ -31,8 +22,69 @@
 #define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path */
 #define LOCKED_LOWER    0x2        /* for lower path */
 
+#define AFR_TRACE_INODELK_IN(frame, this, params ...)           \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->inodelk_trace)                      \
+                        break;                                  \
+                afr_trace_inodelk_in (frame, this, params);     \
+        } while (0);
+
+#define AFR_TRACE_INODELK_OUT(frame, this, params ...)          \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->inodelk_trace)                      \
+                        break;                                  \
+                afr_trace_inodelk_out (frame, this, params);    \
+        } while (0);
+
+#define AFR_TRACE_ENTRYLK_IN(frame, this, params ...)           \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->entrylk_trace)                      \
+                        break;                                  \
+                afr_trace_entrylk_in (frame, this, params);     \
+        } while (0);
+
+#define AFR_TRACE_ENTRYLK_OUT(frame, this, params ...)          \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->entrylk_trace)                      \
+                        break;                                  \
+                afr_trace_entrylk_out (frame, this, params);    \
+        } while (0);
+
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+afr_entry_lockee_cmp (const void *l1, const void *l2)
+{
+        const afr_entry_lockee_t       *r1 = l1;
+        const afr_entry_lockee_t       *r2 = l2;
+        int                            ret = 0;
+        uuid_t                         gfid1 = {0};
+        uuid_t                         gfid2 = {0};
+
+        loc_gfid ((loc_t*)&r1->loc, gfid1);
+        loc_gfid ((loc_t*)&r2->loc, gfid2);
+        ret = uuid_compare (gfid1, gfid2);
+        /*Entrylks with NULL basename are the 'smallest'*/
+        if (ret == 0) {
+                if (!r1->basename)
+                        return -1;
+                if (!r2->basename)
+                        return 1;
+                ret = strcmp (r1->basename, r2->basename);
+        }
+
+        if (ret <= 0)
+                return -1;
+        else
+                return 1;
+}
+
+int afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+
+static int
+afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this);
 
 static uint64_t afr_lock_number = 1;
 
@@ -57,13 +109,13 @@ afr_set_lock_number (call_frame_t *frame, xlator_t *this)
 }
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this)
+afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner)
 {
         gf_log (this->name, GF_LOG_TRACE,
                 "Setting lk-owner=%llu",
-                (unsigned long long) (unsigned long)frame->root);
+                (unsigned long long) (unsigned long)lk_owner);
 
-        set_lk_owner_from_ptr (&frame->root->lk_owner, frame->root);
+        set_lk_owner_from_ptr (&frame->root->lk_owner, lk_owner);
 }
 
 static int
@@ -99,16 +151,9 @@ internal_lock_count (call_frame_t *frame, xlator_t *this)
         local = frame->local;
         priv  = this->private;
 
-        if (local->fd) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && local->fd_open_on[i])
-                                ++call_count;
-                }
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i])
-                                ++call_count;
-                }
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i])
+                        ++call_count;
         }
 
         return call_count;
@@ -242,27 +287,20 @@ afr_set_lock_call_type (afr_lock_call_type_t lock_call_type,
 }
 
 static void
-afr_trace_inodelk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
+afr_trace_inodelk_out (call_frame_t *frame, xlator_t *this,
+                       afr_lock_call_type_t lock_call_type,
                        afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
                        int op_ret, int op_errno, int32_t child_index)
 {
-        xlator_t            *this     = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
 
         char lockee[256];
         char lock_call_type_str[256];
         char verdict[16];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->inodelk_trace) {
-                return;
-        }
 
         afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
 
@@ -271,37 +309,29 @@ afr_trace_inodelk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
         afr_print_verdict (op_ret, op_errno, verdict);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] Lockee={%s} Number={%llu}",
+                "[%s %s] [%s] lk-owner=%s Lockee={%s} Number={%llu}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict,
-                lockee,
+                verdict, lkowner_utoa (&frame->root->lk_owner), lockee,
                 (unsigned long long) int_lock->lock_number);
 
 }
 
 static void
-afr_trace_inodelk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
+afr_trace_inodelk_in (call_frame_t *frame, xlator_t *this,
+                      afr_lock_call_type_t lock_call_type,
                       afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
                       int32_t cmd, int32_t child_index)
 {
-        xlator_t            *this     = NULL;
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
 
         char lock[256];
         char lockee[256];
         char lock_call_type_str[256];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->inodelk_trace) {
-                return;
-        }
 
         afr_print_inodelk (lock, 256, cmd, flock, &frame->root->lk_owner);
         afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
@@ -318,20 +348,21 @@ afr_trace_inodelk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
 }
 
 static void
-afr_trace_entrylk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
+afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
+                      afr_lock_call_type_t lock_call_type,
                       afr_lock_op_type_t lk_op_type, const char *basename,
-                      int32_t child_index)
+                      int32_t cookie)
 {
-        xlator_t            *this     = NULL;
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_private_t       *priv     = NULL;
+        int                 child_index = 0;
+        int                 lockee_no = 0;
 
         char lock[256];
         char lockee[256];
         char lock_call_type_str[256];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
@@ -339,36 +370,41 @@ afr_trace_entrylk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
         if (!priv->entrylk_trace) {
                 return;
         }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
 
         afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
+                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}, Cookie={%d}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
                 lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+                (unsigned long long) int_lock->lock_number,
+                cookie);
 }
 
 static void
-afr_trace_entrylk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, const char *basename, int op_ret,
-                       int op_errno, int32_t child_index)
+afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
+                       afr_lock_call_type_t lock_call_type,
+                       afr_lock_op_type_t lk_op_type, const char *basename,
+                       int op_ret, int op_errno, int32_t cookie)
 {
-        xlator_t            *this     = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
+        int                 lockee_no = 0;
+        int                 child_index = 0;
 
         char lock[256];
         char lockee[256];
         char lock_call_type_str[256];
         char verdict[16];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
@@ -376,20 +412,25 @@ afr_trace_entrylk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
         if (!priv->entrylk_trace) {
                 return;
         }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
 
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
         afr_print_verdict (op_ret, op_errno, verdict);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu}",
+                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu} Cookie={%d}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
                 verdict,
                 lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+                (unsigned long long) int_lock->lock_number,
+                cookie);
 
 }
 
@@ -442,6 +483,47 @@ is_afr_lock_transaction (afr_local_t *local)
         return ret;
 }
 
+int
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count)
+{
+        int     ret     = -1;
+
+        loc_copy (&lockee->loc, loc);
+        lockee->basename        = (basename)? gf_strdup (basename): NULL;
+        if (basename && !lockee->basename)
+                goto out;
+
+        lockee->locked_count    = 0;
+        lockee->locked_nodes    = GF_CALLOC (child_count,
+                                             sizeof (*lockee->locked_nodes),
+                                             gf_afr_mt_afr_node_character);
+
+        if (!lockee->locked_nodes)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock)
+{
+        int     i   = 0;
+
+        for (i = 0; i < int_lock->lockee_count; i++) {
+                loc_wipe (&int_lock->lockee[i].loc);
+                if (int_lock->lockee[i].basename)
+                        GF_FREE (int_lock->lockee[i].basename);
+                if (int_lock->lockee[i].locked_nodes)
+                       GF_FREE (int_lock->lockee[i].locked_nodes);
+        }
+
+        return;
+}
+
 static int
 initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
 {
@@ -459,8 +541,13 @@ initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
         int_lock->lock_op_ret        = -1;
         int_lock->lock_op_errno      = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->entry_locked_nodes[i] = 0;
+        for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+                if (!int_lock->lockee[i].locked_nodes)
+                        break;
+                int_lock->lockee[i].locked_count = 0;
+                memset (int_lock->lockee[i].locked_nodes, 0,
+                        sizeof (*int_lock->lockee[i].locked_nodes) *
+                        priv->child_count);
         }
 
         return 0;
@@ -472,19 +559,23 @@ initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_private_t       *priv     = NULL;
-        int i = 0;
+        afr_inodelk_t       *inodelk  = NULL;
 
         priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
-        int_lock->inodelk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->inode_locked_nodes[i] = 0;
-        }
+        inodelk->lock_count    = 0;
+        int_lock->lk_attempted_count = 0;
+        int_lock->lock_op_ret   = -1;
+        int_lock->lock_op_errno = 0;
+
+        memset (inodelk->locked_nodes, 0,
+                sizeof (*inodelk->locked_nodes) * priv->child_count);
+        memset (int_lock->locked_nodes, 0,
+                sizeof (*int_lock->locked_nodes) * priv->child_count);
 
         return 0;
 }
@@ -494,7 +585,7 @@ lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2)
 {
         int ret = 0;
 
-        ret = strcmp (l1->path, l2->path);
+        ret = uuid_compare (l1->inode->gfid, l2->inode->gfid);
 
         if (ret == 0)
                 ret = strcmp (b1, b2);
@@ -506,6 +597,18 @@ lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2)
 }
 
 int
+afr_lockee_locked_nodes_count (afr_internal_lock_t *int_lock)
+{
+        int call_count  = 0;
+        int i           = 0;
+
+        for (i = 0; i < int_lock->lockee_count; i++)
+                call_count += int_lock->lockee[i].locked_count;
+
+        return call_count;
+}
+
+int
 afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
 
 {
@@ -523,7 +626,7 @@ afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
 /* FIXME: What if UNLOCK fails */
 static int32_t
 afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
@@ -549,33 +652,37 @@ afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 static int32_t
 afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t         *local = NULL;
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk = NULL;
         int32_t             child_index = (long)cookie;
+        afr_private_t       *priv = NULL;
 
         local = frame->local;
         int_lock = &local->internal_lock;
 
-        afr_trace_inodelk_out (frame, AFR_INODELK_TRANSACTION,
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
                                AFR_UNLOCK_OP, NULL, op_ret,
                                op_errno, child_index);
 
+        priv = this->private;
+
         if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: unlock failed on %d, reason: %s",
-                        local->loc.path, child_index, strerror (op_errno));
+                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on subvolume %s "
+                        "with lock owner %s", local->loc.path,
+                        priv->children[child_index]->name,
+                        lkowner_utoa (&frame->root->lk_owner));
         }
 
 
-        int_lock->inode_locked_nodes[child_index] &= LOCKED_NO;
-
-        if (op_ret == 1) {
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+        inodelk->locked_nodes[child_index] &= LOCKED_NO;
+        if (local->transaction.eager_lock)
                 local->transaction.eager_lock[child_index] = 0;
-        }
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         return 0;
 
@@ -585,9 +692,12 @@ static int
 afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
         struct gf_flock flock = {0,};
+        struct gf_flock full_flock = {0,};
+        struct gf_flock *flock_use = NULL;
         int call_count = 0;
         int i = 0;
         int piggyback = 0;
@@ -598,15 +708,14 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
         priv     = this->private;
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = F_UNLCK;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        gf_log (this->name, GF_LOG_DEBUG, "attempting data unlock range %"PRIu64
-                " %"PRIu64" by %s", flock.l_start, flock.l_len,
-                lkowner_utoa (&frame->root->lk_owner));
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
+        flock.l_type  = F_UNLCK;
 
-        call_count = afr_locked_nodes_count (int_lock->inode_locked_nodes,
+        full_flock.l_type = F_UNLCK;
+        call_count = afr_locked_nodes_count (inodelk->locked_nodes,
                                              priv->child_count);
 
         int_lock->lk_call_count = call_count;
@@ -622,11 +731,11 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                 fd_ctx = afr_fd_ctx_get (local->fd, this);
 
         for (i = 0; i < priv->child_count; i++) {
-                if ((int_lock->inode_locked_nodes[i] & LOCKED_YES)
-                    != LOCKED_YES)
+                if ((inodelk->locked_nodes[i] & LOCKED_YES) != LOCKED_YES)
                         continue;
 
                 if (local->fd) {
+                        flock_use = &flock;
                         if (!local->transaction.eager_lock[i]) {
                                 goto wind;
                         }
@@ -638,43 +747,48 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                                 if (fd_ctx->lock_piggyback[i]) {
                                         fd_ctx->lock_piggyback[i]--;
                                         piggyback = 1;
+                                } else {
+                                        fd_ctx->lock_acquired[i]--;
                                 }
                         }
                         UNLOCK (&local->fd->lock);
 
                         if (piggyback) {
                                 afr_unlock_inodelk_cbk (frame, (void *) (long) i,
-                                                        this, 1, 0);
+                                                        this, 1, 0, NULL);
                                 if (!--call_count)
                                         break;
                                 continue;
                         }
 
-                        fd_ctx->lock_acquired[i]--;
+                        flock_use = &full_flock;
                 wind:
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
-                                              AFR_UNLOCK_OP, &flock, F_SETLK, i);
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
+                                              AFR_UNLOCK_OP, flock_use, F_SETLK,
+                                              i);
 
                         STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
                                            (void *) (long)i,
                                            priv->children[i],
                                            priv->children[i]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLK, &flock);
+                                           int_lock->domain, local->fd,
+                                           F_SETLK, flock_use, NULL);
 
                         if (!--call_count)
                                 break;
 
                 } else {
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
                                               AFR_UNLOCK_OP, &flock, F_SETLK, i);
 
                         STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
                                            (void *) (long)i,
                                            priv->children[i],
                                            priv->children[i]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLK, &flock);
+                                           int_lock->domain, &local->loc,
+                                           F_SETLK, &flock, NULL);
 
                         if (!--call_count)
                                 break;
@@ -686,24 +800,34 @@ out:
 
 static int32_t
 afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t         *local = NULL;
-        int32_t             child_index = (long)cookie;
+        afr_private_t       *priv  = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        int32_t             child_index = 0;
+        int                 lockee_no   = 0;
+
+        priv = this->private;
+        lockee_no = (int)((long) cookie) / priv->child_count;
+        child_index = (int) ((long) cookie) % priv->child_count;
 
         local = frame->local;
+        int_lock = &local->internal_lock;
 
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
-                               AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, child_index);
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
+                               AFR_UNLOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
+                               op_errno, (int) ((long)cookie));
 
-        if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        if (op_ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: unlock failed on %d, reason: %s",
                         local->loc.path, child_index, strerror (op_errno));
         }
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno);
+        int_lock->lockee[lockee_no].locked_nodes[child_index] &= LOCKED_NO;
+        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, NULL);
 
         return 0;
 }
@@ -711,24 +835,22 @@ afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 static int
 afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int call_count = 0;
-        int i = -1;
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     call_count      = 0;
+        int                     index           = 0;
+        int                     lockee_no       = 0;
+        int                     copies          = 0;
+        int                     i               = -1;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
+        copies   = priv->child_count;
 
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
+        call_count = afr_lockee_locked_nodes_count (int_lock);
 
-        call_count = afr_locked_nodes_count (int_lock->entry_locked_nodes,
-                                             priv->child_count);
         int_lock->lk_call_count = call_count;
 
         if (!call_count){
@@ -738,18 +860,23 @@ afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
                 goto out;
         }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->entry_locked_nodes[i] & LOCKED_YES) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
+        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+                lockee_no = i / copies;
+                index     = i % copies;
+                if (int_lock->lockee[lockee_no].locked_nodes[index] & LOCKED_YES) {
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                              AFR_UNLOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              i);
 
                         STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
                                            (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
+                                           priv->children[index],
+                                           priv->children[index]->fops->entrylk,
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
 
                         if (!--call_count)
                                 break;
@@ -763,15 +890,22 @@ out:
 
 static int32_t
 afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        int child_index = (long) cookie;
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     cky             = (long) cookie;
+        int                     child_index     = 0;
+        int                     lockee_no       = 0;
 
+        priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
+        child_index = ((int)cky) % priv->child_count;
+        lockee_no   = ((int)cky) / priv->child_count;
+
         LOCK (&frame->lock);
         {
                 if (op_ret == -1) {
@@ -787,6 +921,8 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_errno              = op_errno;
                         int_lock->lock_op_errno      = op_errno;
                 }
+
+		int_lock->lk_attempted_count++;
         }
         UNLOCK (&frame->lock);
 
@@ -795,10 +931,17 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 afr_unlock (frame, this);
         } else {
                 if (op_ret == 0) {
-                        int_lock->locked_nodes[child_index] |= LOCKED_YES;
-                        int_lock->lock_count++;
+                        if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+                            local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+                                int_lock->lockee[lockee_no].locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lockee[lockee_no].locked_count++;
+                                int_lock->entrylk_lock_count++;
+                        } else {
+                                int_lock->locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lock_count++;
+                        }
                 }
-                afr_lock_blocking (frame, this, child_index + 1);
+                afr_lock_blocking (frame, this, cky + 1);
         }
 
         return 0;
@@ -806,98 +949,26 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 static int32_t
 afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_trace_inodelk_out (frame, AFR_INODELK_TRANSACTION,
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long) cookie);
 
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
         return 0;
 
 }
 
 static int32_t
-afr_lock_lower_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_private_t       *priv        = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *higher_name = NULL;
-        int child_index = (long) cookie;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOSYS) {
-                                /* return ENOTSUP */
-
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "subvolume does not support locking. "
-                                        "please load features/locks xlator on server");
-
-                                local->op_ret   = op_ret;
-                        }
-
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret != 0) {
-                afr_unlock (frame, this);
-                goto out;
-        } else {
-                int_lock->lower_locked_nodes[child_index] |= LOCKED_LOWER;
-                int_lock->lock_count++;
-        }
-
-        /* The lower path has been locked. Now lock the higher path */
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                              AFR_LOCK_OP, higher_name, child_index);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->entrylk,
-                           this->name, higher, higher_name,
-                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-out:
-        return 0;
-}
-
-static int32_t
 afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long)cookie);
 
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -905,6 +976,7 @@ static int
 afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
 
@@ -915,18 +987,16 @@ afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                memcpy (int_lock->inode_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->inodelk_lock_count = int_lock->lock_count;
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                memcpy (inodelk->locked_nodes, int_lock->locked_nodes,
+                        sizeof (*inodelk->locked_nodes) * priv->child_count);
+                inodelk->lock_count = int_lock->lock_count;
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
         case AFR_ENTRY_TRANSACTION:
-                memcpy (int_lock->entry_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->entrylk_lock_count = int_lock->lock_count;
+                /*entrylk_count is being used in both non-blocking and blocking
+                 * modes */
                 break;
         }
 
@@ -934,25 +1004,67 @@ afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
 
 }
 
+static inline gf_boolean_t
+afr_is_entrylk (afr_internal_lock_t *int_lock,
+                afr_transaction_type trans_type)
+{
+        gf_boolean_t is_entrylk = _gf_false;
+
+        if ((int_lock->transaction_lk_type == AFR_SELFHEAL_LK) &&
+            int_lock->selfheal_lk_type == AFR_ENTRY_SELF_HEAL_LK) {
+
+                is_entrylk = _gf_true;
+
+        } else if ((int_lock->transaction_lk_type == AFR_TRANSACTION_LK) &&
+                 (trans_type == AFR_ENTRY_TRANSACTION ||
+                  trans_type == AFR_ENTRY_RENAME_TRANSACTION)) {
+
+                is_entrylk = _gf_true;
+
+        } else {
+                is_entrylk = _gf_false;
+        }
+
+        return is_entrylk;
+}
+
+static gf_boolean_t
+_is_lock_wind_needed (afr_local_t *local, int child_index)
+{
+        if (!local->child_up[child_index])
+                return _gf_false;
+
+        return _gf_true;
+}
+
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
+afr_lock_blocking (call_frame_t *frame, xlator_t *this, int cookie)
 {
         afr_internal_lock_t *int_lock    = NULL;
+        afr_inodelk_t       *inodelk     = NULL;
         afr_local_t         *local       = NULL;
         afr_private_t       *priv        = NULL;
-        loc_t               *lower       = NULL;
-        const char          *lower_name  = NULL;
         struct gf_flock flock = {0,};
         uint64_t ctx = 0;
         int ret = 0;
+        int child_index = 0;
+        int lockee_no   = 0;
+        gf_boolean_t is_entrylk = _gf_false;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
+        local         = frame->local;
+        int_lock      = &local->internal_lock;
+        priv          = this->private;
+        child_index   = cookie % priv->child_count;
+        lockee_no     = cookie / priv->child_count;
+        is_entrylk    = afr_is_entrylk (int_lock, local->transaction.type);
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = int_lock->lk_flock.l_type;
+
+        if (!is_entrylk) {
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                flock.l_start = inodelk->flock.l_start;
+                flock.l_len   = inodelk->flock.l_len;
+                flock.l_type  = inodelk->flock.l_type;
+        }
 
         if (local->fd) {
                 ret = fd_ctx_get (local->fd, this, &ctx);
@@ -971,42 +1083,26 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
 
                         return 0;
                 }
-
-                /* skip over children that or down
-                   or don't have the fd open */
-
-                while ((child_index < priv->child_count)
-                       && (!local->child_up[child_index] ||
-                          !local->fd_open_on[child_index]))
-
-                        child_index++;
-        } else {
-                /* skip over children that are down */
-                while ((child_index < priv->child_count)
-                       && !local->child_up[child_index])
-                        child_index++;
         }
 
-        if ((child_index == priv->child_count) &&
-            int_lock->lock_count == 0) {
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "unable to lock on even one child");
-
-                local->op_ret           = -1;
-                int_lock->lock_op_ret   = -1;
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+                if ((is_entrylk && int_lock->entrylk_lock_count == 0) ||
+                    (!is_entrylk && int_lock->lock_count == 0)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "unable to lock on even one child");
 
-                afr_copy_locked_nodes (frame, this);
+                        local->op_ret           = -1;
+                        int_lock->lock_op_ret   = -1;
 
-                afr_unlock(frame, this);
+                        afr_copy_locked_nodes (frame, this);
 
-                return 0;
+                        afr_unlock(frame, this);
 
+                        return 0;
+                }
         }
 
-        if ((child_index == priv->child_count)
-            || (int_lock->lock_count == int_lock->lk_expected_count)) {
-
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
                 /* we're done locking */
 
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -1019,12 +1115,18 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                 return 0;
         }
 
+        if (!_is_lock_wind_needed (local, child_index)) {
+                afr_lock_blocking (frame, this, cookie + 1);
+                return 0;
+        }
+
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
 
                 if (local->fd) {
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
                                               AFR_LOCK_OP, &flock, F_SETLKW,
                                               child_index);
 
@@ -1032,11 +1134,12 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                                            (void *) (long) child_index,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLKW, &flock);
+                                           int_lock->domain, local->fd,
+                                           F_SETLKW, &flock, NULL);
 
                 } else {
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
                                               AFR_LOCK_OP, &flock, F_SETLKW,
                                               child_index);
 
@@ -1044,63 +1147,44 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                                            (void *) (long) child_index,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLKW, &flock);
+                                           int_lock->domain, &local->loc,
+                                           F_SETLKW, &flock, NULL);
                 }
 
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
-        {
-                lower = lower_path (&local->transaction.parent_loc,
-                                    local->transaction.basename,
-                                    &local->transaction.new_parent_loc,
-                                    local->transaction.new_basename);
-
-                lower_name = (lower == &local->transaction.parent_loc ?
-                              local->transaction.basename :
-                              local->transaction.new_basename);
-
-                afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                                      AFR_LOCK_OP, lower_name, child_index);
-
-
-                STACK_WIND_COOKIE (frame, afr_lock_lower_cbk,
-                                   (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->entrylk,
-                                   this->name, lower, lower_name,
-                                   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-                break;
-        }
-
         case AFR_ENTRY_TRANSACTION:
+                /*Accounting for child_index increments on 'down'
+                 *and 'fd-less' children */
+
                 if (local->fd) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
+                                              AFR_LOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              cookie);
 
                         STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) child_index,
+                                           (void *) (long) cookie,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->fentrylk,
-                                           this->name, local->fd,
-                                           local->transaction.basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
+                                           int_lock->domain, local->fd,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
                 } else {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
+                        AFR_TRACE_ENTRYLK_IN (frame, this,
+                                              AFR_ENTRYLK_TRANSACTION,
                                               AFR_LOCK_OP, local->transaction.basename,
                                               child_index);
 
                         STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) child_index,
+                                           (void *) (long) cookie,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->entrylk,
-                                           this->name,
-                                           &local->transaction.parent_loc,
-                                           local->transaction.basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
                 }
 
                 break;
@@ -1128,11 +1212,12 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this)
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
                 up_count = afr_up_children_count (local->child_up,
                                                   priv->child_count);
-                int_lock->lk_expected_count = 2 * up_count;
-                //fallthrough
-        case AFR_ENTRY_TRANSACTION:
+                int_lock->lk_call_count = int_lock->lk_expected_count
+                                        = (int_lock->lockee_count *
+                                           up_count);
                 initialize_entrylk_variables (frame, this);
                 break;
         }
@@ -1144,42 +1229,55 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this)
 
 static int32_t
 afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
         int call_count          = 0;
         int child_index         = (long) cookie;
+        int copies              = 0;
+        int index               = 0;
+        int lockee_no           = 0;
+        afr_private_t       *priv = NULL;
+
+        priv = this->private;
+
+        copies = priv->child_count;
+        index = child_index % copies;
+        lockee_no = child_index / copies;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
 
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
+                               AFR_LOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
                                op_errno, (long) cookie);
 
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret < 0 ) {
-                if (op_errno == ENOSYS) {
+	LOCK (&frame->lock);
+	{
+		if (op_ret < 0 ) {
+			if (op_errno == ENOSYS) {
                         /* return ENOTSUP */
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "subvolume does not support locking. "
-                                "please load features/locks xlator on server");
-                        local->op_ret         = op_ret;
-                        int_lock->lock_op_ret = op_ret;
+				gf_log (this->name, GF_LOG_ERROR,
+					"subvolume does not support locking. "
+					"please load features/locks xlator on server");
+				local->op_ret         = op_ret;
+				int_lock->lock_op_ret = op_ret;
+
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+		} else if (op_ret == 0) {
+			int_lock->lockee[lockee_no].locked_nodes[index] |= \
+				LOCKED_YES;
+			int_lock->lockee[lockee_no].locked_count++;
+			int_lock->entrylk_lock_count++;
+		}
 
-                        int_lock->lock_op_errno      = op_errno;
-                        local->op_errno              = op_errno;
-                }
-        } else if (op_ret == 0) {
-                int_lock->entry_locked_nodes[child_index] |= LOCKED_YES;
-                int_lock->entrylk_lock_count++;
+                call_count = --int_lock->lk_call_count;
         }
+        UNLOCK (&frame->lock);
 
         if (call_count == 0) {
                 gf_log (this->name, GF_LOG_TRACE,
@@ -1206,42 +1304,26 @@ afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-void
-afr_mark_fd_open_on (afr_local_t *local, afr_fd_ctx_t *fd_ctx,
-                       size_t child_count)
-{
-        int             i = 0;
-
-        GF_ASSERT (local->fd_open_on);
-
-        memset (local->fd_open_on, 0, sizeof (*local->fd_open_on)*child_count);
-        for (i = 0; i < child_count; i++)
-                if (fd_ctx->opened_on[i] == AFR_FD_OPENED)
-                        local->fd_open_on[i] = 1;
-}
-
 int
 afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int32_t call_count = 0;
+        afr_internal_lock_t *int_lock   = NULL;
+        afr_local_t         *local      = NULL;
+        afr_private_t       *priv       = NULL;
+        afr_fd_ctx_t        *fd_ctx     = NULL;
+        int                 copies      = 0;
+        int                 index       = 0;
+        int                 lockee_no   = 0;
+        int32_t             call_count  = 0;
         int i = 0;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
+        copies = priv->child_count;
         initialize_entrylk_variables (frame, this);
 
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
         if (local->fd) {
                 fd_ctx = afr_fd_ctx_get (local->fd, this);
                 if (!fd_ctx) {
@@ -1254,11 +1336,11 @@ afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
                         local->op_errno         = EINVAL;
                         int_lock->lock_op_errno = EINVAL;
 
+			afr_unlock (frame, this);
                         return -1;
                 }
 
-                afr_mark_fd_open_on (local, fd_ctx, priv->child_count);
-                call_count = internal_lock_count (frame, this);
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
 
@@ -1271,42 +1353,52 @@ afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
 
                 /* Send non-blocking entrylk calls only on up children
                    and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && local->fd_open_on[i]) {
-                                afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
 
                                 STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
                                                    (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fentrylk,
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->fentrylk,
                                                    this->name, local->fd,
-                                                   basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
+                                                   int_lock->lockee[lockee_no].basename,
+                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
+                                                   NULL);
+                                if (!--call_count)
+                                        break;
                         }
                 }
         } else {
-                GF_ASSERT (loc);
-
-                call_count = internal_lock_count (frame, this);
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
 
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i]) {
-                                afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
 
                                 STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
                                                    (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->entrylk,
-                                                   this->name, loc, basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->entrylk,
+                                                   this->name, &int_lock->lockee[lockee_no].loc,
+                                                   int_lock->lockee[lockee_no].basename,
+                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
+                                                   NULL);
 
                                 if (!--call_count)
                                         break;
-
                         }
                 }
         }
@@ -1316,70 +1408,69 @@ out:
 
 int32_t
 afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         int call_count  = 0;
         int child_index = (long) cookie;
         afr_fd_ctx_t        *fd_ctx = NULL;
-        afr_private_t       *priv     = NULL;
 
 
-        priv = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        afr_trace_inodelk_out (frame, AFR_INODELK_NB_TRANSACTION,
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_NB_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long) cookie);
 
+	if (local->fd)
+		fd_ctx = afr_fd_ctx_get (local->fd, this);
+
         LOCK (&frame->lock);
         {
+		if (op_ret < 0) {
+			if (op_errno == ENOSYS) {
+				/* return ENOTSUP */
+				gf_log (this->name, GF_LOG_ERROR,
+					"subvolume does not support locking. "
+					"please load features/locks xlator on "
+					"server");
+				local->op_ret                = op_ret;
+				int_lock->lock_op_ret        = op_ret;
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+			if (local->transaction.eager_lock)
+				local->transaction.eager_lock[child_index] = 0;
+		} else {
+			inodelk->locked_nodes[child_index] |= LOCKED_YES;
+			inodelk->lock_count++;
+
+			if (local->transaction.eager_lock &&
+			    local->transaction.eager_lock[child_index] &&
+			    local->fd) {
+				/* piggybacked */
+				if (op_ret == 1) {
+					/* piggybacked */
+				} else if (op_ret == 0) {
+					/* lock acquired from server */
+                                        fd_ctx->lock_acquired[child_index]++;
+				}
+			}
+		}
+
                 call_count = --int_lock->lk_call_count;
         }
         UNLOCK (&frame->lock);
 
-        if (op_ret < 0) {
-                if (op_errno == ENOSYS) {
-                        /* return ENOTSUP */
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "subvolume does not support locking. "
-                                "please load features/locks xlator on server");
-                        local->op_ret                = op_ret;
-                        int_lock->lock_op_ret        = op_ret;
-                        int_lock->lock_op_errno      = op_errno;
-                        local->op_errno              = op_errno;
-                }
-        } else {
-                int_lock->inode_locked_nodes[child_index]
-                        |= LOCKED_YES;
-                int_lock->inodelk_lock_count++;
-
-                if (priv->eager_lock && local->fd) {
-                        fd_ctx = afr_fd_ctx_get (local->fd, this);
-                        local->transaction.eager_lock[child_index] = 1;
-                        /* piggybacked */
-
-                        if (op_ret == 1) {
-                                /* piggybacked */
-                        } else if (op_ret == 0) {
-                                /* lock acquired from server */
-                                LOCK (&local->fd->lock);
-                                {
-                                        fd_ctx->lock_acquired[child_index]++;
-                                }
-                                UNLOCK (&local->fd->lock);
-                        }
-                }
-        }
-
         if (call_count == 0) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "Last inode locking reply received");
                 /* all locks successful. Proceed to call FOP */
-                if (int_lock->inodelk_lock_count ==
-                                int_lock->lk_expected_count) {
+                if (inodelk->lock_count == int_lock->lk_expected_count) {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "All servers locked. Calling the cbk");
                         int_lock->lock_op_ret = 0;
@@ -1403,30 +1494,29 @@ int
 afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
         afr_fd_ctx_t        *fd_ctx   = NULL;
-        int32_t  call_count = 0;
-        int      i          = 0;
-        int      ret        = 0;
-        struct gf_flock flock = {0,};
-        struct gf_flock full_flock = {0,};
-        struct gf_flock *flock_use = &flock;
-        int     piggyback = 0;
+        int32_t             call_count = 0;
+        int                 i          = 0;
+        int                 ret        = 0;
+        struct              gf_flock flock = {0,};
+        struct              gf_flock full_flock = {0,};
+        struct              gf_flock *flock_use = NULL;
+        int                 piggyback = 0;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = int_lock->lk_flock.l_type;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        gf_log (this->name, GF_LOG_DEBUG, "attempting data lock range %"PRIu64
-                " %"PRIu64" by %s", flock.l_start, flock.l_len,
-                lkowner_utoa (&frame->root->lk_owner));
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
+        flock.l_type  = inodelk->flock.l_type;
 
-        full_flock.l_type = int_lock->lk_flock.l_type;
+        full_flock.l_type = inodelk->flock.l_type;
 
         initialize_inodelk_variables (frame, this);
 
@@ -1442,11 +1532,11 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                         local->op_errno         = EINVAL;
                         int_lock->lock_op_errno = EINVAL;
 
+			afr_unlock (frame, this);
                         ret = -1;
                         goto out;
                 }
 
-                afr_mark_fd_open_on (local, fd_ctx, priv->child_count);
                 call_count = internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
@@ -1461,14 +1551,18 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                 /* Send non-blocking inodelk calls only on up children
                    and where the fd has been opened */
                 for (i = 0; i < priv->child_count; i++) {
-                        if (!local->child_up[i] || !local->fd_open_on[i])
+                        if (!local->child_up[i])
                                 continue;
 
-                        if (!priv->eager_lock)
+                        flock_use = &flock;
+                        if (!local->transaction.eager_lock_on) {
                                 goto wind;
+                        }
 
-                        flock_use = &full_flock;
                         piggyback = 0;
+                        local->transaction.eager_lock[i] = 1;
+
+			afr_set_delayed_post_op (frame, this);
 
                         LOCK (&local->fd->lock);
                         {
@@ -1482,21 +1576,23 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                         if (piggyback) {
                                 /* (op_ret == 1) => indicate piggybacked lock */
                                 afr_nonblocking_inodelk_cbk (frame, (void *) (long) i,
-                                                             this, 1, 0);
+                                                             this, 1, 0, NULL);
                                 if (!--call_count)
                                         break;
                                 continue;
                         }
+                        flock_use = &full_flock;
                 wind:
-                        afr_trace_inodelk_in (frame, AFR_INODELK_NB_TRANSACTION,
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_NB_TRANSACTION,
                                               AFR_LOCK_OP, flock_use, F_SETLK, i);
 
                         STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLK, flock_use);
+                                           int_lock->domain, local->fd,
+                                           F_SETLK, flock_use, NULL);
 
                         if (!--call_count)
                                 break;
@@ -1509,15 +1605,16 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                 for (i = 0; i < priv->child_count; i++) {
                         if (!local->child_up[i])
                                 continue;
-                        afr_trace_inodelk_in (frame, AFR_INODELK_NB_TRANSACTION,
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_NB_TRANSACTION,
                                               AFR_LOCK_OP, &flock, F_SETLK, i);
 
                         STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLK, &flock);
+                                           int_lock->domain, &local->loc,
+                                           F_SETLK, &flock, NULL);
 
                         if (!--call_count)
                                 break;
@@ -1527,200 +1624,6 @@ out:
         return ret;
 }
 
-static int
-__is_lower_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int count = 0;
-        int i     = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER)
-                        count++;
-        }
-
-        return count;
-
-}
-
-static int
-__is_higher_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int count = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->locked_nodes[i] & LOCKED_YES)
-                        count++;
-        }
-
-        return count;
-
-}
-
-static int
-afr_unlock_lower_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int call_count = 0;
-        int i = -1;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-        priv  = this->private;
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        call_count = __is_lower_locked (frame, this);
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-
-                        if (!--call_count)
-                                break;
-
-                }
-        }
-
-out:
-        return 0;
-
-}
-
-
-static int
-afr_post_unlock_higher_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.done (frame, this);
-        return 0;
-}
-
-static int
-afr_post_unlock_lower_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *higher_name = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        if (__is_higher_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking higher");
-                int_lock->lk_basename = higher_name;
-                int_lock->lk_loc      = higher;
-                int_lock->lock_cbk    = afr_post_unlock_higher_cbk;
-
-                afr_unlock_entrylk (frame, this);
-        } else
-                local->transaction.done (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        const char          *lower_name  = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        if (__is_lower_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking lower");
-                int_lock->lk_basename = lower_name;
-                int_lock->lk_loc      = lower;
-                int_lock->lock_cbk    = afr_post_unlock_lower_cbk;
-
-                afr_unlock_lower_entrylk (frame, this);
-        } else
-                afr_post_unlock_lower_cbk (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_transaction (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        return (local->transaction.type ==
-                AFR_ENTRY_RENAME_TRANSACTION);
-
-}
-
 int32_t
 afr_unlock (call_frame_t *frame, xlator_t *this)
 {
@@ -1732,10 +1635,8 @@ afr_unlock (call_frame_t *frame, xlator_t *this)
                 if (is_afr_lock_transaction (local))
                         afr_unlock_inodelk (frame, this);
                 else
-                        if (!afr_rename_transaction (frame, this))
-                                afr_unlock_entrylk (frame, this);
-                        else
-                                afr_rename_unlock (frame, this);
+                        afr_unlock_entrylk (frame, this);
+
         } else {
                 if (is_afr_lock_selfheal (local))
                         afr_unlock_inodelk (frame, this);
@@ -1904,10 +1805,12 @@ out:
 
 int32_t
 afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock);
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata);
 int32_t
 afr_recover_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
 {
         afr_local_t   *local = NULL;
         afr_private_t *priv  = NULL;
@@ -1931,7 +1834,7 @@ afr_recover_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            (void *) (long) source_child,
                            priv->children[source_child],
                            priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock);
+                           local->fd, F_GETLK_FD, &flock, NULL);
 
         return 0;
 
@@ -1959,7 +1862,7 @@ afr_recover_lock (call_frame_t *frame, xlator_t *this,
                            (void *) (long) lock_recovery_child,
                            priv->children[lock_recovery_child],
                            priv->children[lock_recovery_child]->fops->lk,
-                           local->fd, F_SETLK, flock);
+                           local->fd, F_SETLK, flock, NULL);
 
         return 0;
 }
@@ -1977,7 +1880,8 @@ is_afr_lock_eol (struct gf_flock *lock)
 
 int32_t
 afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
 {
         if (op_ret) {
                 gf_log (this->name, GF_LOG_INFO,
@@ -2037,7 +1941,7 @@ afr_lock_recovery (call_frame_t *frame, xlator_t *this)
                            (void *) (long) source_child,
                            priv->children[source_child],
                            priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock);
+                           local->fd, F_GETLK_FD, &flock, NULL);
 
 out:
         return ret;
@@ -2065,7 +1969,8 @@ out:
 
 int32_t
 afr_lock_recovery_preopen_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int32_t op_ret, int32_t op_errno, fd_t *fd)
+                               int32_t op_ret, int32_t op_errno, fd_t *fd,
+                               dict_t *xdata)
 {
         int32_t child_index = (long )cookie;
         int ret = 0;
@@ -2137,8 +2042,7 @@ afr_lock_recovery_preopen (call_frame_t *frame, xlator_t *this)
                            (void *)(long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->open,
-                           &loc, fdctx->flags, local->fd,
-                           fdctx->wbflags);
+                           &loc, fdctx->flags, local->fd, NULL);
 
         return 0;
 }
@@ -2187,7 +2091,7 @@ afr_attempt_lock_recovery (xlator_t *this, int32_t child_index)
                 goto out;
         }
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
         ret = afr_local_init (local, priv, &op_errno);
         if (ret < 0) {
@@ -2233,27 +2137,38 @@ out:
         return ret;
 }
 
-void
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src,
+int
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
                           unsigned int child_count)
 {
-        afr_local_t *dst_local = NULL;
-        afr_local_t *src_local = NULL;
-        afr_internal_lock_t *dst_lock = NULL;
-        afr_internal_lock_t *src_lock = NULL;
+        afr_local_t         *dst_local   = NULL;
+        afr_local_t         *src_local   = NULL;
+        afr_internal_lock_t *dst_lock    = NULL;
+        afr_internal_lock_t *src_lock    = NULL;
+        afr_inodelk_t       *dst_inodelk = NULL;
+        afr_inodelk_t       *src_inodelk = NULL;
+        int                 ret = -1;
 
-        dst_local = dst->local;
-        dst_lock  = &dst_local->internal_lock;
         src_local = src->local;
         src_lock  = &src_local->internal_lock;
-        if (src_lock->inode_locked_nodes) {
-                memcpy (dst_lock->inode_locked_nodes,
-                        src_lock->inode_locked_nodes,
-                        sizeof (*dst_lock->inode_locked_nodes) * child_count);
-                memset (src_lock->inode_locked_nodes, 0,
-                        sizeof (*src_lock->inode_locked_nodes) * child_count);
-        }
-
-        dst_lock->inodelk_lock_count = src_lock->inodelk_lock_count;
-        src_lock->inodelk_lock_count = 0;
+        src_inodelk = afr_get_inodelk (src_lock, dom);
+        dst_local = dst->local;
+        dst_lock  = &dst_local->internal_lock;
+        dst_inodelk = afr_get_inodelk (dst_lock, dom);
+        if (!dst_inodelk || !src_inodelk)
+                goto out;
+        if (src_inodelk->locked_nodes) {
+                memcpy (dst_inodelk->locked_nodes, src_inodelk->locked_nodes,
+                        sizeof (*dst_inodelk->locked_nodes) * child_count);
+                memset (src_inodelk->locked_nodes, 0,
+                        sizeof (*src_inodelk->locked_nodes) * child_count);
+        }
+
+        dst_lock->transaction_lk_type = src_lock->transaction_lk_type;
+        dst_lock->selfheal_lk_type    = src_lock->selfheal_lk_type;
+        dst_inodelk->lock_count = src_inodelk->lock_count;
+        src_inodelk->lock_count = 0;
+        ret = 0;
+out:
+        return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index ebe189c35..73594f265 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -26,7 +17,6 @@
 enum gf_afr_mem_types_ {
         gf_afr_mt_iovec  = gf_common_mt_end + 1,
         gf_afr_mt_afr_fd_ctx_t,
-        gf_afr_mt_afr_local_t,
         gf_afr_mt_afr_private_t,
         gf_afr_mt_int32_t,
         gf_afr_mt_char,
@@ -44,8 +34,17 @@ enum gf_afr_mem_types_ {
         gf_afr_mt_locked_fd,
         gf_afr_mt_inode_ctx_t,
         gf_afr_fd_paused_call_t,
-        gf_afr_mt_afr_crawl_data_t,
-        gf_afr_mt_afr_brick_pos_t,
+        gf_afr_mt_crawl_data_t,
+        gf_afr_mt_brick_pos_t,
+        gf_afr_mt_shd_bool_t,
+        gf_afr_mt_shd_timer_t,
+        gf_afr_mt_shd_event_t,
+        gf_afr_mt_time_t,
+        gf_afr_mt_pos_data_t,
+        gf_afr_mt_reply_t,
+        gf_afr_mt_stats_t,
+        gf_afr_mt_shd_crawl_event_t,
+        gf_afr_mt_uint64_t,
         gf_afr_mt_end
 };
 #endif
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index 083e78a2a..643a5d692 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -123,7 +114,7 @@ out:
 int
 afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
+                        struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t * local = frame->local;
         afr_private_t *priv = NULL;
@@ -132,7 +123,7 @@ afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (afr_open_only_data_self_heal (priv->data_self_heal))
                 afr_perform_data_self_heal (frame, this);
         AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                          local->fd);
+                          local->fd, xdata);
         return 0;
 }
 
@@ -140,7 +131,7 @@ afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 afr_open_cbk (call_frame_t *frame, void *cookie,
               xlator_t *this, int32_t op_ret, int32_t op_errno,
-              fd_t *fd)
+              fd_t *fd, dict_t *xdata)
 {
         afr_local_t *  local       = NULL;
         int            ret         = 0;
@@ -162,8 +153,7 @@ afr_open_cbk (call_frame_t *frame, void *cookie,
                         local->success_count++;
 
                         ret = afr_child_fd_ctx_set (this, fd, child_index,
-                                                    local->cont.open.flags,
-                                                    local->cont.open.wbflags);
+                                                    local->cont.open.flags);
                         if (ret) {
                                 local->op_ret = -1;
                                 local->op_errno = -ret;
@@ -181,12 +171,12 @@ unlock:
                     && (local->op_ret >= 0)) {
                         STACK_WIND (frame, afr_open_ftruncate_cbk,
                                     this, this->fops->ftruncate,
-                                    fd, 0);
+                                    fd, 0, NULL);
                 } else {
                         if (afr_open_only_data_self_heal (priv->data_self_heal))
                                 afr_perform_data_self_heal (frame, this);
                         AFR_STACK_UNWIND (open, frame, local->op_ret,
-                                          local->op_errno, local->fd);
+                                          local->op_errno, local->fd, xdata);
                 }
         }
 
@@ -195,7 +185,7 @@ unlock:
 
 int
 afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags)
+          fd_t *fd, dict_t *xdata)
 {
         afr_private_t * priv       = NULL;
         afr_local_t *   local      = NULL;
@@ -225,7 +215,7 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -236,7 +226,6 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         loc_copy (&local->loc, loc);
 
         local->cont.open.flags   = flags;
-        local->cont.open.wbflags = wbflags;
 
         local->fd = fd_ref (fd);
 
@@ -245,7 +234,7 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
                         STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->open,
-                                           loc, wind_flags, fd, wbflags);
+                                           loc, wind_flags, fd, xdata);
 
                         if (!--call_count)
                                 break;
@@ -255,189 +244,131 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (open, frame, -1, op_errno, fd);
+                AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, xdata);
 
         return 0;
 }
 
-//NOTE: this function should be called with holding the lock on
-//fd to which fd_ctx belongs
-void
-afr_get_resumable_calls (xlator_t *this, afr_fd_ctx_t *fd_ctx,
-                         struct list_head *list)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-        afr_local_t           *call_local  = NULL;
-        afr_private_t         *priv        = NULL;
-        int                    i = 0;
-        gf_boolean_t           call = _gf_false;
-
-        priv = this->private;
-        list_for_each_entry_safe (paused_call, tmp, &fd_ctx->paused_calls,
-                                  call_list) {
-                call = _gf_true;
-                call_local = paused_call->frame->local;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (call_local->child_up[i] &&
-                            (fd_ctx->opened_on[i] == AFR_FD_OPENING))
-                                call = _gf_false;
-                }
-
-                if (call) {
-                        list_del_init (&paused_call->call_list);
-                        list_add (&paused_call->call_list, list);
-                }
-        }
-}
-
-void
-afr_resume_calls (xlator_t *this, struct list_head *list)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-        afr_local_t           *call_local  = NULL;
-
-        list_for_each_entry_safe (paused_call, tmp, list, call_list) {
-                list_del_init (&paused_call->call_list);
-                call_local = paused_call->frame->local;
-                call_local->fop_call_continue (paused_call->frame, this);
-                GF_FREE (paused_call);
-        }
-}
-
 int
 afr_openfd_fix_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, fd_t *fd)
+                         int32_t op_ret, int32_t op_errno, fd_t *fd,
+                         dict_t *xdata)
 {
-        afr_local_t           *local       = NULL;
-        afr_private_t         *priv        = NULL;
-        afr_fd_ctx_t          *fd_ctx      = NULL;
-        int                    call_count  = 0;
-        int                    child_index = (long) cookie;
-        struct list_head       paused_calls = {0};
-        gf_boolean_t           fop_paused = _gf_false;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        afr_fd_ctx_t  *fd_ctx     = NULL;
+        int           call_count  = 0;
+        int           child_index = (long) cookie;
 
         priv     = this->private;
         local    = frame->local;
-        fop_paused = local->fop_paused;
 
         if (op_ret >= 0) {
-                gf_log (this->name, GF_LOG_INFO, "fd for %s opened "
+                gf_log (this->name, GF_LOG_DEBUG, "fd for %s opened "
                         "successfully on subvolume %s", local->loc.path,
                         priv->children[child_index]->name);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open %s "
+                        "on subvolume %s", local->loc.path,
+                        priv->children[child_index]->name);
         }
 
-        call_count = afr_frame_return (frame);
-        //Note: Do not access any thing using the frame outside call_count 0
-
-        //Note: No frame locking needed for this block of code
-        fd_ctx = afr_fd_ctx_get (fd, this);
+        fd_ctx = afr_fd_ctx_get (local->fd, this);
         if (!fd_ctx) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "failed to get fd context, %p", fd);
+                        "failed to get fd context, %p", local->fd);
                 goto out;
         }
 
-        LOCK (&fd->lock);
+        LOCK (&local->fd->lock);
         {
                 if (op_ret >= 0) {
                         fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
                 } else {
                         fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
                 }
-                if (call_count == 0) {
-                        INIT_LIST_HEAD (&paused_calls);
-                        afr_get_resumable_calls (this, fd_ctx, &paused_calls);
-                }
         }
-        UNLOCK (&fd->lock);
+        UNLOCK (&local->fd->lock);
 out:
-        if (call_count == 0) {
-                afr_resume_calls (this, &paused_calls);
-                //If the fop is paused then resume_calls will continue the fop
-                if (fop_paused)
-                        goto done;
-
-                if (local->fop_call_continue)
-                        local->fop_call_continue (frame, this);
-                else
-                        AFR_STACK_DESTROY (frame);
-        }
+        call_count = afr_frame_return (frame);
+        if (call_count == 0)
+                AFR_STACK_DESTROY (frame);
 
-done:
         return 0;
 }
 
-int
-afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
-              int need_open_count, int *need_open)
+void
+afr_fix_open (xlator_t *this, fd_t *fd, size_t need_open_count, int *need_open)
 {
-        afr_local_t       *local = NULL;
-        afr_private_t     *priv  = NULL;
-        int               i      = 0;
-        call_frame_t      *open_frame = NULL;
-        afr_local_t      *open_local = NULL;
-        int               ret    = -1;
-        ia_type_t         ia_type = IA_INVAL;
-        int32_t           op_errno = 0;
-
-        GF_ASSERT (fd_ctx);
-        GF_ASSERT (need_open_count > 0);
-        GF_ASSERT (need_open);
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
+        call_frame_t  *frame   = NULL;
+        afr_local_t   *local   = NULL;
+        int           ret      = -1;
+        int32_t       op_errno = 0;
+        afr_fd_ctx_t  *fd_ctx  = NULL;
 
-        local = frame->local;
         priv  = this->private;
-        if (!local->fop_call_continue) {
-                open_frame = copy_frame (frame);
-                if (!open_frame) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-                ALLOC_OR_GOTO (open_frame->local, afr_local_t, out);
-                open_local = open_frame->local;
-                ret = afr_local_init (open_local, priv, &op_errno);
-                if (ret < 0)
-                        goto out;
-                loc_copy (&open_local->loc, &local->loc);
-                open_local->fd = fd_ref (local->fd);
-        } else {
-                ret = 0;
-                open_frame = frame;
-                open_local = local;
+
+        if (!afr_is_fd_fixable (fd) || !need_open || !need_open_count)
+                goto out;
+
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx) {
+                ret = -1;
+                goto out;
         }
 
-        open_local->call_count = need_open_count;
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-        gf_log (this->name, GF_LOG_DEBUG, "need open count: %d",
+        local->loc.inode = inode_ref (fd->inode);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
+
+        local->fd = fd_ref (fd);
+        local->call_count = need_open_count;
+
+        gf_log (this->name, GF_LOG_DEBUG, "need open count: %zd",
                 need_open_count);
 
-        ia_type = open_local->fd->inode->ia_type;
-        GF_ASSERT (ia_type != IA_INVAL);
         for (i = 0; i < priv->child_count; i++) {
                 if (!need_open[i])
                         continue;
-                if (IA_IFDIR == ia_type) {
+
+                if (IA_IFDIR == fd->inode->ia_type) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "opening fd for dir %s on subvolume %s",
                                 local->loc.path, priv->children[i]->name);
 
-                        STACK_WIND_COOKIE (open_frame, afr_openfd_fix_open_cbk,
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
                                            (void*) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->opendir,
-                                           &open_local->loc, open_local->fd);
+                                           &local->loc, local->fd,
+                                           NULL);
                 } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "opening fd for file %s on subvolume %s",
                                 local->loc.path, priv->children[i]->name);
 
-                        STACK_WIND_COOKIE (open_frame, afr_openfd_fix_open_cbk,
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
                                            (void *)(long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->open,
-                                           &open_local->loc, fd_ctx->flags,
-                                           open_local->fd, fd_ctx->wbflags);
+                                           &local->loc,
+                                           fd_ctx->flags & (~O_TRUNC),
+                                           local->fd, NULL);
                 }
 
         }
@@ -445,8 +376,7 @@ afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
         ret = 0;
 out:
         if (op_errno)
-                ret = -op_errno;
-        if (ret && open_frame)
-                AFR_STACK_DESTROY (open_frame);
-        return ret;
+                ret = -1; //For handling ALLOC_OR_GOTO
+        if (ret && frame)
+                AFR_STACK_DESTROY (frame);
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.c b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
index 629822a8e..83846f152 100644
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.c
+++ b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
@@ -1,23 +1,15 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
+#include <openssl/md5.h>
 #include "glusterfs.h"
 #include "afr.h"
 #include "xlator.h"
@@ -33,7 +25,6 @@
 #include "compat-errno.h"
 #include "compat.h"
 #include "byte-order.h"
-#include "md5.h"
 
 #include "afr-transaction.h"
 #include "afr-self-heal.h"
@@ -72,8 +63,7 @@ sh_private_cleanup (call_frame_t *frame, xlator_t *this)
         sh    = &local->self_heal;
 
         sh_priv = sh->private;
-        if (sh_priv)
-                GF_FREE (sh_priv);
+        GF_FREE (sh_priv);
 }
 
 static int
@@ -110,10 +100,10 @@ sh_loop_driver_done (call_frame_t *sh_frame, xlator_t *this,
         }
 
         sh_private_cleanup (sh_frame, this);
-        if (sh->op_failed) {
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                 GF_ASSERT (!last_loop_frame);
                 //loop_finish should have happened and the old_loop should be NULL
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "self-heal aborting on %s",
                         local->loc.path);
 
@@ -121,10 +111,10 @@ sh_loop_driver_done (call_frame_t *sh_frame, xlator_t *this,
         } else {
                 GF_ASSERT (last_loop_frame);
                 if (diff_blocks == total_blocks) {
-                        gf_log (this->name, GF_LOG_INFO, "full self-heal "
+                        gf_log (this->name, GF_LOG_DEBUG, "full self-heal "
                                 "completed on %s",local->loc.path);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "diff self-heal on %s: completed. "
                                 "(%d blocks of %d were different (%.2f%%))",
                                 local->loc.path, diff_blocks, total_blocks,
@@ -153,7 +143,7 @@ sh_loop_finish (call_frame_t *loop_frame, xlator_t *this)
         }
 
         if (loop_sh && loop_sh->data_lock_held) {
-                afr_sh_data_unlock (loop_frame, this,
+                afr_sh_data_unlock (loop_frame, this, this->name,
                                     sh_destroy_frame);
         } else {
                 sh_destroy_frame (loop_frame, this);
@@ -174,7 +164,7 @@ sh_loop_lock_success (call_frame_t *loop_frame, xlator_t *this)
         sh_loop_finish (loop_sh->old_loop_frame, this);
         loop_sh->old_loop_frame = NULL;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Aquired lock for range %"PRIu64
+        gf_log (this->name, GF_LOG_DEBUG, "Acquired lock for range %"PRIu64
                 " %"PRIu64, loop_sh->offset, loop_sh->block_size);
         loop_sh->data_lock_held = _gf_true;
         loop_sh->sh_data_algo_start (loop_frame, this);
@@ -224,7 +214,7 @@ sh_loop_frame_create (call_frame_t *sh_frame, xlator_t *this,
                 goto out;
         //We want the frame to have same lk_owner as sh_frame
         //so that locks translator allows conflicting locks
-        new_loop_local = afr_local_copy (local, this);
+        new_loop_local = afr_self_heal_local_init (local, this);
         if (!new_loop_local)
                 goto out;
         new_loop_frame->local = new_loop_local;
@@ -239,7 +229,7 @@ sh_loop_frame_create (call_frame_t *sh_frame, xlator_t *this,
                                                gf_afr_mt_char);
         if (!new_loop_sh->write_needed)
                 goto out;
-        new_loop_sh->checksum = GF_CALLOC (priv->child_count, MD5_DIGEST_LEN,
+        new_loop_sh->checksum = GF_CALLOC (priv->child_count, MD5_DIGEST_LENGTH,
                                            gf_afr_mt_uint8_t);
         if (!new_loop_sh->checksum)
                 goto out;
@@ -283,10 +273,10 @@ sh_loop_start (call_frame_t *sh_frame, xlator_t *this, off_t offset,
         new_loop_sh->offset = offset;
         new_loop_sh->block_size = sh->block_size;
         afr_sh_data_lock (new_loop_frame, this, offset, new_loop_sh->block_size,
-                          sh_loop_lock_success, sh_loop_lock_failure);
+                          _gf_true, this->name, sh_loop_lock_success, sh_loop_lock_failure);
         return 0;
 out:
-        sh->op_failed = 1;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         if (old_loop_frame)
                 sh_loop_finish (old_loop_frame, this);
         sh_loop_return (sh_frame, this, new_loop_frame, -1, ENOMEM);
@@ -317,8 +307,9 @@ sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
                         sh_priv->loops_running--;
                 offset = sh_priv->offset;
                 block_size = sh->block_size;
-                while ((!sh->eof_reached) && (0 == sh->op_failed) &&
-                       (sh_priv->loops_running < priv->data_self_heal_window_size)
+                while ((!sh->eof_reached) &&
+                       (!is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) &&
+                     (sh_priv->loops_running < priv->data_self_heal_window_size)
                        && (sh_priv->offset < sh->file_size)) {
 
                         loop++;
@@ -337,7 +328,8 @@ sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
         if (0 == loop) {
                 //loop finish does unlock, but the erasing of the pending
                 //xattrs needs to happen before that so do not finish the loop
-                if (is_driver_done && !sh->op_failed)
+                if (is_driver_done &&
+                    !is_self_heal_failed (sh, AFR_CHECK_SPECIFIC))
                         goto driver_done;
                 if (old_loop_frame) {
                         sh_loop_finish (old_loop_frame, this);
@@ -348,7 +340,7 @@ sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
         //If we have more loops to form we should finish previous loop after
         //the next loop lock
         while (loop--) {
-                if (sh->op_failed) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                         // op failed in other loop, stop spawning more loops
                         if (old_loop_frame) {
                                 sh_loop_finish (old_loop_frame, this);
@@ -394,7 +386,7 @@ sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame
         }
 
         if (op_ret == -1) {
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_set_error (sh, op_errno);
                 if (loop_frame) {
                         sh_loop_finish (loop_frame, this);
@@ -410,7 +402,7 @@ sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame
 static int
 sh_loop_write_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         afr_private_t *             priv        = NULL;
         afr_local_t *               loop_local    = NULL;
@@ -442,13 +434,22 @@ sh_loop_write_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                         priv->children[child_index]->name,
                         strerror (op_errno));
 
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_set_error (loop_sh, op_errno);
+        } else if (op_ret < loop_local->cont.writev.vector->iov_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "incomplete write to %s on subvolume %s "
+                        "(expected %lu, returned %d)", sh_local->loc.path,
+                        priv->children[child_index]->name,
+                        loop_local->cont.writev.vector->iov_len, op_ret);
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         }
 
         call_count = afr_frame_return (loop_frame);
 
         if (call_count == 0) {
+		iobref_unref(loop_local->cont.writev.iobref);
+
                 sh_loop_return (sh_frame, this, loop_frame,
                                 loop_sh->op_ret, loop_sh->op_errno);
         }
@@ -456,12 +457,41 @@ sh_loop_write_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
         return 0;
 }
 
+static void
+sh_prune_writes_needed (call_frame_t *sh_frame, call_frame_t *loop_frame,
+                        afr_private_t *priv)
+{
+        afr_local_t     *sh_local     = NULL;
+        afr_self_heal_t *sh           = NULL;
+        afr_local_t     *loop_local   = NULL;
+        afr_self_heal_t *loop_sh      = NULL;
+        int             i             = 0;
+
+        sh_local   = sh_frame->local;
+        sh         = &sh_local->self_heal;
+
+        if (!strcmp (sh->algo->name, "diff"))
+                return;
+
+        loop_local = loop_frame->local;
+        loop_sh    = &loop_local->self_heal;
+
+        /* full self-heal guarantees there exists atleast 1 file with size 0
+         * That means for other files we can preserve holes that come after
+         * its size before 'trim'
+         */
+        for (i = 0; i < priv->child_count; i++) {
+                if (loop_sh->write_needed[i] &&
+                    ((loop_sh->offset + 1) > sh->buf[i].ia_size))
+                        loop_sh->write_needed[i] = 0;
+        }
+}
 
 static int
 sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
                   struct iovec *vector, int32_t count, struct iatt *buf,
-                  struct iobref *iobref)
+                  struct iobref *iobref, dict_t *xdata)
 {
         afr_private_t *               priv       = NULL;
         afr_local_t *                 loop_local   = NULL;
@@ -486,7 +516,7 @@ sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
 
         if (op_ret <= 0) {
                 if (op_ret < 0) {
-                        sh->op_failed = 1;
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                         gf_log (this->name, GF_LOG_ERROR, "read failed on %d "
                                 "for %s reason :%s", sh->source,
                                 sh_local->loc.path, strerror (errno));
@@ -499,18 +529,26 @@ sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
                 goto out;
         }
 
-        if (loop_sh->file_has_holes && iov_0filled (vector, count) == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG, "0 filled block");
-                        sh_loop_return (sh_frame, this, loop_frame,
-                                        op_ret, op_errno);
-                        goto out;
-        }
+        if (loop_sh->file_has_holes && iov_0filled (vector, count) == 0)
+                sh_prune_writes_needed (sh_frame, loop_frame, priv);
 
         call_count = sh_number_of_writes_needed (loop_sh->write_needed,
                                                  priv->child_count);
-        GF_ASSERT (call_count > 0);
+        if (call_count == 0) {
+                sh_loop_return (sh_frame, this, loop_frame, 0, 0);
+                goto out;
+        }
+
         loop_local->call_count = call_count;
 
+	/*
+	 * We only really need the request size at the moment, but the buffer
+	 * is required if we want to issue a retry in the event of a short write.
+	 * Therefore, we duplicate the vector and ref the iobref here...
+	 */
+	loop_local->cont.writev.vector = iov_dup(vector, count);
+	loop_local->cont.writev.iobref = iobref_ref(iobref);
+
         for (i = 0; i < priv->child_count; i++) {
                 if (!loop_sh->write_needed[i])
                         continue;
@@ -519,7 +557,7 @@ sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
                                    priv->children[i],
                                    priv->children[i]->fops->writev,
                                    loop_sh->healing_fd, vector, count,
-                                   loop_sh->offset, 0, iobref);
+                                   loop_sh->offset, 0, iobref, NULL);
 
                 if (!--call_count)
                         break;
@@ -546,7 +584,7 @@ sh_loop_read (call_frame_t *loop_frame, xlator_t *this)
                            priv->children[loop_sh->source],
                            priv->children[loop_sh->source]->fops->readv,
                            loop_sh->healing_fd, loop_sh->block_size,
-                           loop_sh->offset, 0);
+                           loop_sh->offset, 0, NULL);
 
         return 0;
 }
@@ -555,7 +593,8 @@ sh_loop_read (call_frame_t *loop_frame, xlator_t *this)
 static int
 sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum)
+                      uint32_t weak_checksum, uint8_t *strong_checksum,
+                      dict_t *xdata)
 {
         afr_private_t                 *priv         = NULL;
         afr_local_t                   *loop_local   = NULL;
@@ -587,10 +626,10 @@ sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                         "checksum on %s failed on subvolume %s (%s)",
                         sh_local->loc.path, priv->children[child_index]->name,
                         strerror (op_errno));
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         } else {
-                memcpy (loop_sh->checksum + child_index * MD5_DIGEST_LEN,
-                        strong_checksum, MD5_DIGEST_LEN);
+                memcpy (loop_sh->checksum + child_index * MD5_DIGEST_LENGTH,
+                        strong_checksum, MD5_DIGEST_LENGTH);
         }
 
         call_count = afr_frame_return (loop_frame);
@@ -600,9 +639,9 @@ sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                         if (sh->sources[i] || !sh_local->child_up[i])
                                 continue;
 
-                        if (memcmp (loop_sh->checksum + (i * MD5_DIGEST_LEN),
-                                    loop_sh->checksum + (sh->source * MD5_DIGEST_LEN),
-                                    MD5_DIGEST_LEN)) {
+                        if (memcmp (loop_sh->checksum + (i * MD5_DIGEST_LENGTH),
+                                    loop_sh->checksum + (sh->source * MD5_DIGEST_LENGTH),
+                                    MD5_DIGEST_LENGTH)) {
                                 /*
                                   Checksums differ, so this block
                                   must be written to this sink
@@ -625,7 +664,8 @@ sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                 }
                 UNLOCK (&sh_priv->lock);
 
-                if (write_needed && !sh->op_failed) {
+                if (write_needed &&
+                    !is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                         sh_loop_read (loop_frame, this);
                 } else {
                         sh_loop_return (sh_frame, this, loop_frame,
@@ -658,7 +698,7 @@ sh_diff_checksum (call_frame_t *loop_frame, xlator_t *this)
                            priv->children[loop_sh->source],
                            priv->children[loop_sh->source]->fops->rchecksum,
                            loop_sh->healing_fd,
-                           loop_sh->offset, loop_sh->block_size);
+                           loop_sh->offset, loop_sh->block_size, NULL);
 
         for (i = 0; i < priv->child_count; i++) {
                 if (loop_sh->sources[i] || !loop_local->child_up[i])
@@ -669,7 +709,7 @@ sh_diff_checksum (call_frame_t *loop_frame, xlator_t *this)
                                    priv->children[i],
                                    priv->children[i]->fops->rchecksum,
                                    loop_sh->healing_fd,
-                                   loop_sh->offset, loop_sh->block_size);
+                                   loop_sh->offset, loop_sh->block_size, NULL);
 
                 if (!--call_count)
                         break;
@@ -714,14 +754,15 @@ out:
         return sh_priv;
 }
 
-void
-afr_sh_transfer_lock (call_frame_t *dst, call_frame_t *src,
+int
+afr_sh_transfer_lock (call_frame_t *dst, call_frame_t *src, char *dom,
                       unsigned int child_count)
 {
         afr_local_t             *dst_local   = NULL;
         afr_self_heal_t         *dst_sh      = NULL;
         afr_local_t             *src_local   = NULL;
         afr_self_heal_t         *src_sh      = NULL;
+        int                     ret          = -1;
 
         dst_local = dst->local;
         dst_sh = &dst_local->self_heal;
@@ -729,9 +770,12 @@ afr_sh_transfer_lock (call_frame_t *dst, call_frame_t *src,
         src_sh = &src_local->self_heal;
         GF_ASSERT (src_sh->data_lock_held);
         GF_ASSERT (!dst_sh->data_lock_held);
-        afr_lk_transfer_datalock (dst, src, child_count);
+        ret = afr_lk_transfer_datalock (dst, src, dom, child_count);
+        if (ret)
+                return ret;
         src_sh->data_lock_held = _gf_false;
         dst_sh->data_lock_held = _gf_true;
+        return 0;
 }
 
 int
@@ -753,7 +797,10 @@ afr_sh_start_loops (call_frame_t *sh_frame, xlator_t *this,
         ret = sh_loop_frame_create (sh_frame, this, NULL, &first_loop_frame);
         if (ret)
                 goto out;
-        afr_sh_transfer_lock (first_loop_frame, sh_frame, priv->child_count);
+        ret = afr_sh_transfer_lock (first_loop_frame, sh_frame, this->name,
+                                    priv->child_count);
+        if (ret)
+                goto out;
         sh->private = afr_sh_priv_init ();
         if (!sh->private) {
                 ret = -1;
@@ -763,7 +810,7 @@ afr_sh_start_loops (call_frame_t *sh_frame, xlator_t *this,
         ret = 0;
 out:
         if (ret) {
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 sh_loop_driver_done (sh_frame, this, NULL);
         }
         return 0;
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.h b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
index 04d8e8a6c..6b20789b1 100644
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.h
+++ b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
@@ -1,26 +1,16 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __AFR_SELF_HEAL_ALGORITHM_H__
 #define __AFR_SELF_HEAL_ALGORITHM_H__
 
-
 typedef int (*afr_sh_algo_fn) (call_frame_t *frame,
                                xlator_t *this);
 
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 8fbea8c9d..ef92b4205 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "glusterfs.h"
@@ -27,6 +18,52 @@
 #include "afr-self-heal.h"
 #include "pump.h"
 
+#define ADD_FMT_STRING(msg, off, sh_str, status, print_log)                 \
+        do {                                                                \
+                if (AFR_SELF_HEAL_NOT_ATTEMPTED != status) {                \
+                        off += snprintf (msg + off, sizeof (msg) - off,     \
+                                         " "sh_str" self heal %s,",         \
+                                         get_sh_completion_status (status));\
+                        print_log = 1;                                      \
+                }                                                           \
+        } while (0)
+
+#define ADD_FMT_STRING_SYNC(msg, off, sh_str, status, print_log)            \
+        do {                                                                \
+                if (AFR_SELF_HEAL_SYNC_BEGIN == status ||                   \
+                    AFR_SELF_HEAL_FAILED == status)  {                      \
+                        off += snprintf (msg + off, sizeof (msg) - off,     \
+                                         " "sh_str" self heal %s,",         \
+                                         get_sh_completion_status (status));\
+                        print_log = 1;                                      \
+                }                                                           \
+        } while (0)
+
+
+void
+afr_sh_reset (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        memset (sh->child_errno, 0,
+                sizeof (*sh->child_errno) * priv->child_count);
+        memset (sh->buf, 0, sizeof (*sh->buf) * priv->child_count);
+        memset (sh->parentbufs, 0,
+                sizeof (*sh->parentbufs) * priv->child_count);
+        memset (sh->success, 0, sizeof (*sh->success) * priv->child_count);
+        memset (sh->locked_nodes, 0,
+                sizeof (*sh->locked_nodes) * priv->child_count);
+        sh->active_sinks = 0;
+
+        afr_reset_xattr (sh->xattr, priv->child_count);
+}
+
 //Intersection[child]=1 if child is part of intersection
 void
 afr_children_intersection_get (int32_t *set1, int32_t *set2,
@@ -97,8 +134,8 @@ void
 afr_sh_set_error (afr_self_heal_t *sh, int32_t op_errno)
 {
         sh->op_ret = -1;
-        if (afr_error_more_important (sh->op_errno, op_errno))
-                sh->op_errno = op_errno;
+	sh->op_errno = afr_most_important_error(sh->op_errno, op_errno,
+						_gf_false);
 }
 
 void
@@ -120,13 +157,85 @@ afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this)
                         ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
                 }
                 sprintf (ptr, "]");
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "pending_matrix: %s", buf);
+                gf_log (this->name, GF_LOG_DEBUG, "pending_matrix: %s", buf);
         }
 
         GF_FREE (buf);
 }
 
+char*
+afr_get_pending_matrix_str (int32_t *pending_matrix[], xlator_t *this)
+{
+        afr_private_t *  priv = this->private;
+        char            *buf  = NULL;
+        char            *ptr  = NULL;
+        int              i    = 0;
+        int              j    = 0;
+        int             child_count = priv->child_count;
+        char            *matrix_begin = "[ [ ";
+        char            *matrix_end = "] ]";
+        char            *seperator = "] [ ";
+        int             pending_entry_strlen = 12; //Including space after entry
+        int             matrix_begin_strlen = 0;
+        int             matrix_end_strlen = 0;
+        int             seperator_strlen = 0;
+        int             string_length = 0;
+        char            *msg = "- Pending matrix:  ";
+
+        /*
+         *  for a list of lists of [ [ a b ] [ c d ] ]
+         * */
+
+        matrix_begin_strlen = strlen (matrix_begin);
+        matrix_end_strlen = strlen (matrix_end);
+        seperator_strlen = strlen (seperator);
+        string_length = matrix_begin_strlen + matrix_end_strlen
+                        + (child_count -1) * seperator_strlen
+                        + (child_count * child_count * pending_entry_strlen);
+
+        buf = GF_CALLOC (1, 1 + strlen (msg) + string_length , gf_afr_mt_char);
+        if (!buf)
+                goto out;
+
+        ptr = buf;
+        ptr += sprintf (ptr, "%s", msg);
+        ptr += sprintf (ptr, "%s", matrix_begin);
+        for (i = 0; i < priv->child_count; i++) {
+                for (j = 0; j < priv->child_count; j++) {
+                        ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
+                }
+                if (i < priv->child_count -1)
+                        ptr += sprintf (ptr, "%s", seperator);
+        }
+
+        ptr += sprintf (ptr, "%s", matrix_end);
+
+out:
+        return buf;
+}
+
+void
+afr_sh_print_split_brain_log (int32_t *pending_matrix[], xlator_t *this,
+                              const char *loc)
+{
+        char *buf      = NULL;
+        char *free_ptr = NULL;
+
+        buf = afr_get_pending_matrix_str (pending_matrix, this);
+        if (buf)
+                free_ptr = buf;
+        else
+                buf = "";
+
+
+        gf_log (this->name, GF_LOG_ERROR, "Unable to self-heal contents of '%s'"
+                " (possible split-brain). Please delete the file from all but "
+                "the preferred subvolume.%s", loc, buf);
+        GF_FREE (free_ptr);
+        return;
+}
+
+
 void
 afr_init_pending_matrix (int32_t **pending_matrix, size_t child_count)
 {
@@ -392,6 +501,8 @@ afr_find_biggest_witness_among_fools (int32_t *witnesses,
 {
         int i               = 0;
         int biggest_witness = -1;
+        int biggest_witness_idx = -1;
+        int biggest_witness_cnt = -1;
 
         GF_ASSERT (witnesses);
         GF_ASSERT (characters);
@@ -401,10 +512,21 @@ afr_find_biggest_witness_among_fools (int32_t *witnesses,
                 if (characters[i].type != AFR_NODE_FOOL)
                         continue;
 
-                if (biggest_witness < witnesses[i])
+                if (biggest_witness < witnesses[i]) {
                         biggest_witness = witnesses[i];
+			biggest_witness_idx = i;
+			biggest_witness_cnt = 1;
+			continue;
+		}
+
+		if (biggest_witness == witnesses[i])
+			biggest_witness_cnt++;
         }
-        return biggest_witness;
+
+	if (biggest_witness_cnt != 1)
+		return -1;
+
+        return biggest_witness_idx;
 }
 
 int
@@ -432,10 +554,84 @@ afr_mark_fool_as_source_by_witness (int32_t *sources, int32_t *witnesses,
         return nsources;
 }
 
+
+int
+afr_mark_fool_as_source_by_idx (int32_t *sources, int child_count, int idx)
+{
+	if (idx >= 0 && idx < child_count) {
+		sources[idx] = 1;
+		return 1;
+	}
+	return 0;
+}
+
+
+static int
+afr_find_largest_file_size (struct iatt *bufs, int32_t *success_children,
+			    int child_count)
+{
+	int idx = -1;
+	int i = -1;
+	int child = -1;
+	uint64_t max_size = 0;
+        uint64_t min_size = 0;
+        int      num_children = 0;
+
+	for (i = 0; i < child_count; i++) {
+		if (success_children[i] == -1)
+			break;
+
+		child = success_children[i];
+		if (bufs[child].ia_size > max_size) {
+			max_size = bufs[child].ia_size;
+			idx = child;
+		}
+
+                if ((num_children == 0) || (bufs[child].ia_size < min_size)) {
+                        min_size = bufs[child].ia_size;
+                }
+
+                num_children++;
+	}
+
+        /* If sizes are same for all of them, finding sources will have to
+         * happen with pending changelog. So return -1
+         */
+        if ((num_children > 1) && (min_size == max_size))
+                return -1;
+	return idx;
+}
+
+
+static int
+afr_find_newest_file (struct iatt *bufs, int32_t *success_children,
+		      int child_count)
+{
+	int idx = -1;
+	int i = -1;
+	int child = -1;
+	uint64_t max_ctime = 0;
+
+	for (i = 0; i < child_count; i++) {
+		if (success_children[i] == -1)
+			break;
+
+		child = success_children[i];
+		if (bufs[child].ia_ctime > max_ctime) {
+			max_ctime = bufs[child].ia_ctime;
+			idx = child;
+		}
+	}
+
+	return idx;
+}
+
+
 static int
 afr_mark_biggest_of_fools_as_source (int32_t *sources, int32_t **pending_matrix,
                                      afr_node_character *characters,
-                                     int child_count)
+				     int32_t *success_children,
+                                     int child_count, struct iatt *bufs)
 {
         int32_t       biggest_witness = 0;
         int           nsources        = 0;
@@ -443,6 +639,11 @@ afr_mark_biggest_of_fools_as_source (int32_t *sources, int32_t **pending_matrix,
 
         GF_ASSERT (child_count > 0);
 
+	biggest_witness = afr_find_largest_file_size (bufs, success_children,
+						      child_count);
+	if (biggest_witness != -1)
+		goto found;
+
         witnesses = GF_CALLOC (child_count, sizeof (*witnesses),
                                gf_afr_mt_int32_t);
         if (NULL == witnesses) {
@@ -455,12 +656,17 @@ afr_mark_biggest_of_fools_as_source (int32_t *sources, int32_t **pending_matrix,
         biggest_witness = afr_find_biggest_witness_among_fools (witnesses,
                                                                 characters,
                                                                 child_count);
-        nsources = afr_mark_fool_as_source_by_witness (sources, witnesses,
-                                                       characters, child_count,
-                                                       biggest_witness);
+	if (biggest_witness != -1)
+		goto found;
+
+	biggest_witness = afr_find_newest_file (bufs, success_children,
+						child_count);
+
+found:
+	nsources = afr_mark_fool_as_source_by_idx (sources, child_count,
+						   biggest_witness);
 out:
-        if (witnesses)
-                GF_FREE (witnesses);
+        GF_FREE (witnesses);
         return nsources;
 }
 
@@ -526,6 +732,78 @@ out:
         return nsources;
 }
 
+int
+afr_get_no_xattr_dir_read_child (xlator_t *this, int32_t *success_children,
+                                 struct iatt *bufs)
+{
+        afr_private_t *priv = NULL;
+        int            i = 0;
+        int            child = -1;
+        int            read_child = -1;
+
+        priv = this->private;
+        for (i = 0; i < priv->child_count; i++) {
+                child = success_children[i];
+                if (child < 0)
+                        break;
+                if (read_child < 0)
+                        read_child = child;
+                else if (bufs[read_child].ia_size < bufs[child].ia_size)
+                        read_child = child;
+        }
+        return read_child;
+}
+
+int
+afr_sh_mark_zero_size_file_as_sink (struct iatt *bufs, int32_t *success_children,
+                                    int child_count, int32_t *sources)
+{
+        int             nsources = 0;
+        int             i = 0;
+        int             child = 0;
+        gf_boolean_t    sink_exists = _gf_false;
+        gf_boolean_t    source_exists = _gf_false;
+        int             source = -1;
+
+        for (i = 0; i < child_count; i++) {
+                child = success_children[i];
+                if (child < 0)
+                        break;
+                if (!bufs[child].ia_size) {
+                        sink_exists = _gf_true;
+                        continue;
+                }
+                if (!source_exists) {
+                        source_exists = _gf_true;
+                        source = child;
+                        continue;
+                }
+                if (bufs[source].ia_size != bufs[child].ia_size) {
+                        nsources = -1;
+                        goto out;
+                }
+        }
+        if (!source_exists && !sink_exists) {
+                nsources = -1;
+                goto out;
+        }
+
+        if (!source_exists || !sink_exists)
+                goto out;
+
+        for (i = 0; i < child_count; i++) {
+                child = success_children[i];
+                if (child < 0)
+                        break;
+                if (bufs[child].ia_size) {
+                        sources[child] = 1;
+                        nsources++;
+                }
+        }
+out:
+        return nsources;
+}
+
 char *
 afr_get_character_str (afr_node_type type)
 {
@@ -691,11 +969,24 @@ afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
         afr_find_character_types (characters, pending_matrix, success_children,
                                   child_count);
         if (afr_sh_all_nodes_innocent (characters, child_count)) {
-                if (type == AFR_SELF_HEAL_METADATA)
+                switch (type) {
+                case AFR_SELF_HEAL_METADATA:
                         nsources = afr_sh_mark_lowest_uid_as_source (bufs,
                                                              success_children,
                                                              child_count,
                                                              sources);
+                        break;
+                case AFR_SELF_HEAL_DATA:
+                        nsources = afr_sh_mark_zero_size_file_as_sink (bufs,
+                                                             success_children,
+                                                             child_count,
+                                                             sources);
+                        if ((nsources < 0) && subvol_status)
+                                *subvol_status |= SPLIT_BRAIN;
+                        break;
+                default:
+                        break;
+                }
                 goto out;
         }
 
@@ -717,7 +1008,8 @@ afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
                 nsources = afr_mark_biggest_of_fools_as_source (sources,
                                                                 pending_matrix,
                                                                 characters,
-                                                                child_count);
+								success_children,
+                                                                child_count, bufs);
         }
 
 out:
@@ -735,50 +1027,108 @@ afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
                          int32_t *delta_matrix[], unsigned char success[],
                          int child_count, afr_transaction_type type)
 {
-        int i = 0;
-        int j = 0;
+        int     tgt     = 0;
+        int     src     = 0;
+        int     value   = 0;
 
         afr_build_pending_matrix (priv->pending_key, delta_matrix, NULL,
                                   xattr, type, priv->child_count);
-        for (i = 0; i < priv->child_count; i++)
-                for (j = 0; j < priv->child_count; j++)
-                        delta_matrix[i][j] = -delta_matrix[i][j];
+
+        /*
+         * The algorithm here has two parts.  First, for each subvol indexed
+         * as tgt, we try to figure out what count everyone should have for it.
+         * If the self-heal succeeded, that's easy; the value is zero.
+         * Otherwise, the value is the maximum of the succeeding nodes' counts.
+         * Once we know the value, we loop through (possibly for a second time)
+         * setting each count to the difference so that when we're done all
+         * succeeding nodes will have the same count for tgt.
+         */
+        for (tgt = 0; tgt < priv->child_count; ++tgt) {
+                value = 0;
+                if (!success[tgt]) {
+                        /* Find the maximum. */
+                        for (src = 0; src < priv->child_count; ++src) {
+                                if (!success[src]) {
+                                        continue;
+                                }
+                                if (delta_matrix[src][tgt] > value) {
+                                        value = delta_matrix[src][tgt];
+                                }
+                        }
+                }
+                /* Force everyone who succeeded to the chosen value. */
+                for (src = 0; src < priv->child_count; ++src) {
+                        if (success[src]) {
+                                delta_matrix[src][tgt] = value
+                                                       - delta_matrix[src][tgt];
+                        }
+                        else {
+                                delta_matrix[src][tgt] = 0;
+                        }
+                }
+        }
 }
 
 
 int
-afr_sh_delta_to_xattr (afr_private_t *priv,
+afr_sh_delta_to_xattr (xlator_t *this,
                        int32_t *delta_matrix[], dict_t *xattr[],
                        int child_count, afr_transaction_type type)
 {
-        int      i       = 0;
-        int      j       = 0;
-        int      k       = 0;
-        int      ret     = 0;
-        int32_t *pending = NULL;
+        int              i       = 0;
+        int              j       = 0;
+        int              k       = 0;
+        int              ret     = 0;
+        int32_t         *pending = NULL;
+        int32_t         *local_pending = NULL;
+        afr_private_t   *priv = NULL;
 
+        priv = this->private;
         for (i = 0; i < child_count; i++) {
                 if (!xattr[i])
                         continue;
 
+                local_pending = NULL;
                 for (j = 0; j < child_count; j++) {
                         pending = GF_CALLOC (sizeof (int32_t), 3,
                                              gf_afr_mt_int32_t);
 
-                        if (!pending)
+                        if (!pending) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to allocate pending entry "
+                                        "for %s[%d] on %s",
+                                        priv->pending_key[j], type,
+                                        priv->children[i]->name);
                                 continue;
+                        }
                         /* 3 = data+metadata+entry */
 
                         k = afr_index_for_transaction_type (type);
 
                         pending[k] = hton32 (delta_matrix[i][j]);
 
+                        if (j == i) {
+                                local_pending = pending;
+                                continue;
+                        }
                         ret = dict_set_bin (xattr[i], priv->pending_key[j],
                                             pending,
-                                            3 * sizeof (int32_t));
-                        if (ret < 0)
-                                gf_log (THIS->name, GF_LOG_WARNING,
+                                        AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
                                         "Unable to set dict value.");
+                                GF_FREE (pending);
+                        }
+                }
+                if (local_pending) {
+                        ret = dict_set_bin (xattr[i], priv->pending_key[i],
+                                            local_pending,
+                                        AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Unable to set dict value.");
+                                GF_FREE (local_pending);
+                        }
                 }
         }
         return 0;
@@ -786,127 +1136,23 @@ afr_sh_delta_to_xattr (afr_private_t *priv,
 
 
 int
-afr_sh_has_metadata_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
-
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-
-                if (pending[j])
-                        return 1;
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_has_data_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
-
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-
-                if (pending[j])
-                        return 1;
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_has_entry_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
-
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
-
-                if (pending[j])
-                        return 1;
-        }
-
-        return 0;
-}
-
-int
 afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              i = 0;
 
         local = frame->local;
         sh = &local->self_heal;
-        priv = this->private;
 
-//      memset (sh->child_errno, 0, sizeof (int) * priv->child_count);
-        memset (sh->buf, 0, sizeof (struct iatt) * priv->child_count);
+        afr_sh_reset (frame, this);
 
-        for (i = 0; i < priv->child_count; i++) {
-                sh->locked_nodes[i] = 0;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i])
-                        dict_unref (sh->xattr[i]);
-                sh->xattr[i] = NULL;
-        }
-
-        if (local->govinda_gOvinda || sh->op_failed) {
-                gf_log (this->name, GF_LOG_INFO,
+        if (local->unhealable) {
+                gf_log (this->name, GF_LOG_DEBUG,
                         "split brain found, aborting selfheal of %s",
                         local->loc.path);
-                sh->op_failed = 1;
+        }
+
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                 sh->completion_cbk (frame, this);
         } else {
                 gf_log (this->name, GF_LOG_TRACE,
@@ -992,7 +1238,7 @@ afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
                         sh->success_count++;
                         sh->xattr[child_index] = dict_ref (xattr);
                 } else {
-                        gf_log (this->name, GF_LOG_ERROR, "path %s on subvolume"
+                        gf_log (this->name, GF_LOG_DEBUG, "path %s on subvolume"
                                 " %s => -1 (%s)", loc->path,
                                 priv->children[child_index]->name,
                                 strerror (op_errno));
@@ -1040,7 +1286,7 @@ afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (impunge_local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (impunge_local, out);
 
         local = frame->local;
         new_frame->local = impunge_local;
@@ -1086,6 +1332,7 @@ afr_sh_missing_entry_call_impunge_recreate (call_frame_t *frame, xlator_t *this,
         unsigned int    enoent_count = 0;
         afr_private_t   *priv = NULL;
         int             i = 0;
+        int32_t         op_errno = 0;
 
         local = frame->local;
         sh    = &local->self_heal;
@@ -1106,7 +1353,12 @@ afr_sh_missing_entry_call_impunge_recreate (call_frame_t *frame, xlator_t *this,
         impunge_local = impunge_frame->local;
         impunge_sh    = &impunge_local->self_heal;
         loc_copy (&impunge_local->loc, &local->loc);
-        afr_build_parent_loc (&impunge_sh->parent_loc, &impunge_local->loc);
+        ret = afr_build_parent_loc (&impunge_sh->parent_loc,
+                                    &impunge_local->loc, &op_errno);
+        if (ret) {
+                ret = -op_errno;
+                goto out;
+        }
         impunge_local->call_count = enoent_count;
         impunge_sh->entrybuf = sh->buf[sh->source];
         impunge_sh->parentbuf = sh->parentbufs[sh->source];
@@ -1132,7 +1384,7 @@ out:
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "impunge of %s failed, "
                         "reason: %s", local->loc.path, strerror (-ret));
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         }
         afr_sh_missing_entries_finish (frame, this);
 }
@@ -1147,7 +1399,7 @@ afr_sh_create_entry_cbk (call_frame_t *frame, xlator_t *this,
         local = frame->local;
         sh = &local->self_heal;
         if (op_ret < 0)
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         afr_sh_missing_entries_finish (frame, this);
         return 0;
 }
@@ -1171,7 +1423,7 @@ sh_missing_entries_create (call_frame_t *frame, xlator_t *this)
         if (!afr_valid_ia_type (type)) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: unknown file type: 0%o", local->loc.path, type);
-                local->govinda_gOvinda = 1;
+                afr_set_local_for_unhealable (local);
                 afr_sh_missing_entries_finish (frame, this);
                 goto out;
         }
@@ -1193,6 +1445,10 @@ afr_sh_missing_entries_lookup_done (call_frame_t *frame, xlator_t *this,
         ia_type_t       ia_type = IA_INVAL;
         int32_t         nsources = 0;
         loc_t           *loc = NULL;
+        int32_t         subvol_status = 0;
+        afr_transaction_type txn_type = AFR_DATA_TRANSACTION;
+        gf_boolean_t    split_brain = _gf_false;
+        int             read_child = -1;
 
         local = frame->local;
         sh = &local->self_heal;
@@ -1200,25 +1456,47 @@ afr_sh_missing_entries_lookup_done (call_frame_t *frame, xlator_t *this,
         loc = &local->loc;
 
         if (op_ret < 0) {
-                if (op_errno == EIO)
-                        local->govinda_gOvinda = 1;
+                if (op_errno == EIO) {
+                        afr_set_local_for_unhealable (local);
+                }
                 // EIO can happen if finding the fresh parent dir failed
                 goto out;
         }
 
         //now No chance for the ia_type to conflict
         ia_type = sh->buf[sh->success_children[0]].ia_type;
+        txn_type = afr_transaction_type_get (ia_type);
         nsources = afr_build_sources (this, sh->xattr, sh->buf,
                                       sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      afr_transaction_type_get (ia_type),
-                                      NULL, _gf_false);
+                                      sh->success_children, txn_type,
+                                      &subvol_status, _gf_false);
         if (nsources < 0) {
                 gf_log (this->name, GF_LOG_INFO, "No sources for dir of %s,"
                         " in missing entry self-heal, continuing with the rest"
                         " of the self-heals", local->loc.path);
-                op_errno = EIO;
-                goto out;
+                if (subvol_status & SPLIT_BRAIN) {
+                        split_brain = _gf_true;
+                        switch (txn_type) {
+                        case AFR_DATA_TRANSACTION:
+                                nsources = 1;
+                                sh->sources[sh->success_children[0]] = 1;
+                                break;
+                        case AFR_ENTRY_TRANSACTION:
+                                read_child = afr_get_no_xattr_dir_read_child
+                                                          (this,
+                                                           sh->success_children,
+                                                           sh->buf);
+                                sh->sources[read_child] = 1;
+                                nsources = 1;
+                                break;
+                        default:
+                                op_errno = EIO;
+                                goto out;
+                        }
+                } else {
+                        op_errno = EIO;
+                        goto out;
+                }
         }
 
         afr_get_fresh_children (sh->success_children, sh->sources,
@@ -1235,10 +1513,14 @@ afr_sh_missing_entries_lookup_done (call_frame_t *frame, xlator_t *this,
         sh->type = sh->buf[sh->source].ia_type;
         if (uuid_is_null (loc->inode->gfid))
                 uuid_copy (loc->gfid, sh->buf[sh->source].ia_gfid);
-        sh_missing_entries_create (frame, this);
+        if (split_brain) {
+                afr_sh_missing_entries_finish (frame, this);
+        } else {
+                sh_missing_entries_create (frame, this);
+        }
         return;
 out:
-        sh->op_failed = 1;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         afr_sh_set_error (sh, op_errno);
         afr_sh_missing_entries_finish (frame, this);
         return;
@@ -1322,7 +1604,7 @@ afr_sh_remove_entry_cbk (call_frame_t *frame, xlator_t *this, int child,
                 LOCK (&frame->lock);
                 {
                         afr_sh_set_error (sh, EIO);
-                        sh->op_failed = 1;
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 }
                 UNLOCK (&frame->lock);
         }
@@ -1344,13 +1626,14 @@ afr_sh_call_entry_expunge_remove (call_frame_t *frame, xlator_t *this,
         afr_self_heal_t *sh = NULL;
         afr_self_heal_t *expunge_sh = NULL;
         int32_t         op_errno = 0;
+        int             ret = 0;
 
         expunge_frame = copy_frame (frame);
         if (!expunge_frame) {
                 goto out;
         }
 
-        ALLOC_OR_GOTO (expunge_local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
 
         local = frame->local;
         sh = &local->self_heal;
@@ -1358,6 +1641,12 @@ afr_sh_call_entry_expunge_remove (call_frame_t *frame, xlator_t *this,
         expunge_sh = &expunge_local->self_heal;
         expunge_sh->sh_frame = frame;
         loc_copy (&expunge_local->loc, &local->loc);
+        ret = afr_build_parent_loc (&expunge_sh->parent_loc,
+                                    &expunge_local->loc, &op_errno);
+        if (ret) {
+                ret = -op_errno;
+                goto out;
+        }
         sh->expunge_done = expunge_done;
         afr_sh_entry_expunge_remove (expunge_frame, this, child_index, buf,
                                      parentbuf);
@@ -1397,7 +1686,7 @@ afr_sh_purge_stale_entries_done (call_frame_t *frame, xlator_t *this)
         sh       = &local->self_heal;
         priv     = this->private;
 
-        if (sh->op_failed) {
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                 afr_sh_missing_entries_finish (frame, this);
         } else {
                 if (afr_gfid_missing_count (this->name, sh->fresh_children,
@@ -1407,7 +1696,8 @@ afr_sh_purge_stale_entries_done (call_frame_t *frame, xlator_t *this)
                                               afr_sh_missing_entries_lookup_done,
                                               sh->sh_gfid_req,
                                               AFR_LOOKUP_FAIL_CONFLICTS|
-                                              AFR_LOOKUP_FAIL_MISSING_GFIDS);
+                                              AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                              NULL);
                 } else {
                         //No need to set gfid so goto missing entries lookup done
                         //Behave as if you have done the lookup
@@ -1489,7 +1779,7 @@ afr_sh_purge_entry_common (call_frame_t *frame, xlator_t *this,
                 if (!purge_condition (local, priv, i))
                         continue;
                 gf_log (this->name, GF_LOG_INFO, "purging the stale entry %s "
-                        "on %d", local->loc.path, i);
+                        "on %s", local->loc.path, priv->children[i]->name);
                 afr_sh_call_entry_expunge_remove (frame, this,
                                                   (long) i, &sh->buf[i],
                                                   &sh->parentbufs[i],
@@ -1609,10 +1899,8 @@ afr_sh_children_lookup_done (call_frame_t *frame, xlator_t *this,
                                                sh->child_errno,
                                                priv->child_count, ENOENT);
         if (fresh_child_enoents == fresh_parent_count) {
-                gf_log (this->name, GF_LOG_INFO, "Deleting stale file %s",
-                        local->loc.path);
                 afr_sh_set_error (sh, ENOENT);
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_purge_entry (frame, this);
         } else if (!afr_conflicting_iattrs (sh->buf, sh->fresh_children,
                                             priv->child_count, local->loc.path,
@@ -1626,14 +1914,14 @@ afr_sh_children_lookup_done (call_frame_t *frame, xlator_t *this,
                 afr_sh_purge_stale_entry (frame, this);
         } else {
                 op_errno = EIO;
-                local->govinda_gOvinda = 1;
+                afr_set_local_for_unhealable (local);
                 goto fail;
         }
 
         return;
 
 fail:
-        sh->op_failed = 1;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         afr_sh_set_error (sh, op_errno);
         afr_sh_missing_entries_finish (frame, this);
         return;
@@ -1655,25 +1943,17 @@ afr_sh_find_fresh_parents (call_frame_t *frame, xlator_t *this,
         sh = &local->self_heal;
         priv = this->private;
 
-        /* If We can't find a fresh parent directory here,
-         * we wont know which subvol is correct without finding a parent dir
-         * upwards which has correct xattrs, for that we may have to
-         * do lookups till root, we dont wanna do that,
-         * instead make sure that if there are conflicting gfid
-         * parent dirs, self-heal thus lookup is failed with EIO.
-         * if there are missing entries we dont know whether to delete or
-         * create so fail with EIO,
-         * If there are conflicting xattr fail with EIO.
-         */
         if (op_ret < 0)
                 goto out;
         enoent_count = afr_errno_count (NULL, sh->child_errno,
                                         priv->child_count, ENOENT);
         if (enoent_count > 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Parent dir missing for %s,"
-                        " in missing entry self-heal, aborting self-heal",
+                gf_log (this->name, GF_LOG_INFO, "Parent dir missing for %s,"
+                        " in missing entry self-heal, aborting missing-entry "
+                        "self-heal",
                         local->loc.path);
-                goto out;
+                afr_sh_missing_entries_finish (frame, this);
+                return;
         }
 
         nsources = afr_build_sources (this, sh->xattr, sh->buf,
@@ -1692,6 +1972,7 @@ afr_sh_find_fresh_parents (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_ERROR, "No sources for dir "
                         "of %s, in missing entry self-heal, aborting "
                         "self-heal", local->loc.path);
+                op_errno = EIO;
                 goto out;
         }
 
@@ -1699,18 +1980,20 @@ afr_sh_find_fresh_parents (call_frame_t *frame, xlator_t *this,
         if (source == -1) {
                 GF_ASSERT (0);
                 gf_log (this->name, GF_LOG_DEBUG, "No active sources found.");
+                op_errno = EIO;
                 goto out;
         }
         afr_get_fresh_children (sh->success_children, sh->sources,
                                 sh->fresh_parent_dirs, priv->child_count);
         afr_sh_common_lookup (frame, this, &local->loc,
-                              afr_sh_children_lookup_done, NULL, 0);
+                              afr_sh_children_lookup_done, NULL, 0,
+                              NULL);
         return;
 
 out:
-        afr_sh_set_error (sh, EIO);
-        sh->op_failed = 1;
-        afr_sh_missing_entries_finish (frame, this);
+        afr_sh_set_error (sh, op_errno);
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+	afr_sh_missing_entries_finish (frame, this);
         return;
 }
 
@@ -1738,7 +2021,7 @@ afr_sh_common_reset (afr_self_heal_t *sh, unsigned int child_count)
 int
 afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
                       afr_lookup_done_cbk_t lookup_done , uuid_t gfid,
-                      int32_t flags)
+                      int32_t flags, dict_t *xdata)
 {
         afr_local_t    *local = NULL;
         int             i = 0;
@@ -1799,7 +2082,8 @@ afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 
 int
-afr_sh_post_nb_entrylk_conflicting_sh_cbk (call_frame_t *frame, xlator_t *this)
+afr_sh_post_nb_entrylk_missing_entry_sh_cbk (call_frame_t *frame,
+                                             xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
@@ -1812,7 +2096,7 @@ afr_sh_post_nb_entrylk_conflicting_sh_cbk (call_frame_t *frame, xlator_t *this)
         if (int_lock->lock_op_ret < 0) {
                 gf_log (this->name, GF_LOG_INFO,
                         "Non blocking entrylks failed.");
-                sh->op_failed = -1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_missing_entries_done (frame, this);
         } else {
 
@@ -1820,34 +2104,8 @@ afr_sh_post_nb_entrylk_conflicting_sh_cbk (call_frame_t *frame, xlator_t *this)
                         "Non blocking entrylks done. Proceeding to FOP");
                 afr_sh_common_lookup (frame, this, &sh->parent_loc,
                                       afr_sh_find_fresh_parents,
-                                      NULL, AFR_LOOKUP_FAIL_CONFLICTS);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_post_nb_entrylk_gfid_sh_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        sh       = &local->self_heal;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Non blocking entrylks failed.");
-                afr_sh_missing_entries_done (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_missing_entries_lookup_done,
-                                      sh->sh_gfid_req, AFR_LOOKUP_FAIL_CONFLICTS|
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
+                                      NULL, AFR_LOOKUP_FAIL_CONFLICTS,
+                                      NULL);
         }
 
         return 0;
@@ -1859,7 +2117,9 @@ afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
 
+        priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
@@ -1871,7 +2131,12 @@ afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
         int_lock->lk_basename = base_name;
         int_lock->lk_loc      = loc;
         int_lock->lock_cbk    = lock_cbk;
+        int_lock->domain      = this->name;
 
+        int_lock->lockee_count = 0;
+        afr_init_entry_lockee (&int_lock->lockee[0], local, loc,
+                               base_name, priv->child_count);
+        int_lock->lockee_count++;
         afr_nonblocking_entrylk (frame, this);
 
         return 0;
@@ -1883,6 +2148,9 @@ afr_self_heal_parent_entrylk (call_frame_t *frame, xlator_t *this,
 {
         afr_local_t         *local    = NULL;
         afr_self_heal_t     *sh       = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        int                 ret       = -1;
+        int32_t             op_errno  = 0;
 
         local    = frame->local;
         sh       = &local->self_heal;
@@ -1891,43 +2159,52 @@ afr_self_heal_parent_entrylk (call_frame_t *frame, xlator_t *this,
                 "attempting to recreate missing entries for path=%s",
                 local->loc.path);
 
-        GF_ASSERT (local->loc.parent);
-        afr_build_parent_loc (&sh->parent_loc, &local->loc);
+        ret = afr_build_parent_loc (&sh->parent_loc, &local->loc, &op_errno);
+        if (ret)
+                goto out;
 
         afr_sh_entrylk (frame, this, &sh->parent_loc, NULL,
                         lock_cbk);
         return 0;
-}
-
-static int
-afr_self_heal_conflicting_entries (call_frame_t *frame, xlator_t *this)
-{
-        afr_self_heal_parent_entrylk (frame, this,
-                                      afr_sh_post_nb_entrylk_conflicting_sh_cbk);
+out:
+        int_lock = &local->internal_lock;
+        int_lock->lock_op_ret = -1;
+        lock_cbk (frame, this);
         return 0;
 }
 
 static int
-afr_self_heal_gfids (call_frame_t *frame, xlator_t *this)
+afr_self_heal_missing_entries (call_frame_t *frame, xlator_t *this)
 {
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        sh->sh_type_in_action  = AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY;
+
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
+
         afr_self_heal_parent_entrylk (frame, this,
-                                      afr_sh_post_nb_entrylk_gfid_sh_cbk);
+                                      afr_sh_post_nb_entrylk_missing_entry_sh_cbk);
         return 0;
 }
 
-afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
+afr_local_t*
+afr_self_heal_local_init (afr_local_t *l, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t   *lc     = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *shc = NULL;
+        afr_private_t   *priv  = NULL;
+        afr_local_t     *lc    = NULL;
+        afr_self_heal_t *sh    = NULL;
+        afr_self_heal_t *shc   = NULL;
+        int             ret    = 0;
 
         priv = this->private;
 
         sh = &l->self_heal;
 
-        lc = GF_CALLOC (1, sizeof (afr_local_t),
-                        gf_afr_mt_afr_local_t);
+        lc = mem_get0 (this->local_pool);
         if (!lc)
                 goto out;
 
@@ -1940,16 +2217,27 @@ afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
         shc->do_data_self_heal = sh->do_data_self_heal;
         shc->do_metadata_self_heal = sh->do_metadata_self_heal;
         shc->do_entry_self_heal = sh->do_entry_self_heal;
+        shc->force_confirm_spb = sh->force_confirm_spb;
         shc->forced_merge = sh->forced_merge;
         shc->background = sh->background;
         shc->type = sh->type;
+        shc->data_sh_info = "";
+        shc->metadata_sh_info =  "";
 
         uuid_copy (shc->sh_gfid_req, sh->sh_gfid_req);
-        if (l->loc.path)
-                loc_copy (&lc->loc, &l->loc);
+        if (l->loc.path) {
+                ret = loc_copy (&lc->loc, &l->loc);
+                if (ret < 0)
+                        goto out;
+        }
 
         lc->child_up  = memdup (l->child_up,
                                 sizeof (*lc->child_up) * priv->child_count);
+        if (!lc->child_up) {
+                ret = -1;
+                goto out;
+        }
+
         if (l->xattr_req)
                 lc->xattr_req = dict_ref (l->xattr_req);
 
@@ -1957,40 +2245,25 @@ afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
                 lc->cont.lookup.inode = inode_ref (l->cont.lookup.inode);
         if (l->cont.lookup.xattr)
                 lc->cont.lookup.xattr = dict_ref (l->cont.lookup.xattr);
-        if (l->internal_lock.inode_locked_nodes)
-                lc->internal_lock.inode_locked_nodes =
-                        memdup (l->internal_lock.inode_locked_nodes,
-                                sizeof (*lc->internal_lock.inode_locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.inode_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.inode_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.entry_locked_nodes)
-                lc->internal_lock.entry_locked_nodes =
-                        memdup (l->internal_lock.entry_locked_nodes,
-                                sizeof (*lc->internal_lock.entry_locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.entry_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.entry_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.locked_nodes)
-                lc->internal_lock.locked_nodes =
-                        memdup (l->internal_lock.locked_nodes,
-                                sizeof (*lc->internal_lock.locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
 
-        lc->internal_lock.inodelk_lock_count =
-                l->internal_lock.inodelk_lock_count;
-        lc->internal_lock.entrylk_lock_count =
-                l->internal_lock.entrylk_lock_count;
+        lc->internal_lock.locked_nodes =
+                             GF_CALLOC (sizeof (*l->internal_lock.locked_nodes),
+                                        priv->child_count, gf_afr_mt_char);
+        if (!lc->internal_lock.locked_nodes) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = afr_inodelk_init (&lc->internal_lock.inodelk[0],
+                                this->name, priv->child_count);
+        if (ret)
+                goto out;
 
 out:
+        if (ret) {
+                afr_local_cleanup (lc, this);
+                lc = NULL;
+        }
         return lc;
 }
 
@@ -2000,32 +2273,39 @@ afr_self_heal_completion_cbk (call_frame_t *bgsh_frame, xlator_t *this)
         afr_private_t *   priv  = NULL;
         afr_local_t *     local = NULL;
         afr_self_heal_t * sh    = NULL;
+        afr_local_t *     orig_frame_local = NULL;
+        afr_self_heal_t * orig_frame_sh = NULL;
         char              sh_type_str[256] = {0,};
-        gf_boolean_t      split_brain = _gf_false;
+        gf_loglevel_t     loglevel = 0;
 
         priv  = this->private;
         local = bgsh_frame->local;
         sh    = &local->self_heal;
 
-        if (local->govinda_gOvinda)
-                split_brain = _gf_true;
-
-        afr_set_split_brain (this, sh->inode, split_brain);
+        if (local->unhealable) {
+                afr_set_split_brain (this, sh->inode, SPB, SPB);
+        }
 
         afr_self_heal_type_str_get (sh, sh_type_str,
                                     sizeof(sh_type_str));
-        if (sh->op_failed) {
-                gf_log (this->name, GF_LOG_ERROR, "background %s self-heal "
-                        "failed on %s", sh_type_str, local->loc.path);
+        if (is_self_heal_failed (sh, AFR_CHECK_ALL) && !priv->shd.iamshd) {
+                loglevel = GF_LOG_ERROR;
+        } else if (!is_self_heal_failed (sh, AFR_CHECK_ALL)) {
+                loglevel = GF_LOG_INFO;
         } else {
-                gf_log (this->name, GF_LOG_INFO, "background %s self-heal "
-                        "completed on %s", sh_type_str, local->loc.path);
+                loglevel = GF_LOG_DEBUG;
         }
 
+        afr_log_self_heal_completion_status (local, loglevel);
+
         FRAME_SU_UNDO (bgsh_frame, afr_local_t);
 
         if (!sh->unwound && sh->unwind) {
-                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno);
+                orig_frame_local = sh->orig_frame->local;
+                orig_frame_sh = &orig_frame_local->self_heal;
+                orig_frame_sh->actual_sh_started = _gf_true;
+                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno,
+                            is_self_heal_failed (sh, AFR_CHECK_ALL));
         }
 
         if (sh->background) {
@@ -2071,17 +2351,16 @@ afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
         sh_frame        = copy_frame (frame);
         if (!sh_frame)
                 goto out;
-        afr_set_lk_owner (sh_frame, this);
+        afr_set_lk_owner (sh_frame, this, sh_frame->root);
         afr_set_low_priority (sh_frame);
 
-        sh_local        = afr_local_copy (local, this);
+        sh_local        = afr_self_heal_local_init (local, this);
         if (!sh_local)
                 goto out;
         sh_frame->local = sh_local;
         sh              = &sh_local->self_heal;
 
         sh->inode       = inode_ref (inode);
-
         sh->orig_frame  = frame;
 
         sh->completion_cbk = afr_self_heal_completion_cbk;
@@ -2135,18 +2414,16 @@ afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
                 UNLOCK (&priv->lock);
         }
 
-        if (!local->loc.name) {
-                /* nameless lookup */
+        if (!local->loc.parent) {
                 sh->do_missing_entry_self_heal = _gf_false;
                 sh->do_gfid_self_heal = _gf_false;
         }
 
+        sh->sh_type_in_action = AFR_SELF_HEAL_INVALID;
+
         FRAME_SU_DO (sh_frame, afr_local_t);
-        if (sh->do_missing_entry_self_heal) {
-                afr_self_heal_conflicting_entries (sh_frame, this);
-        } else if (sh->do_gfid_self_heal) {
-                GF_ASSERT (!uuid_is_null (sh->sh_gfid_req));
-                afr_self_heal_gfids (sh_frame, this);
+        if (sh->do_missing_entry_self_heal || sh->do_gfid_self_heal) {
+                afr_self_heal_missing_entries (sh_frame, this);
         } else {
                 loc = &sh_local->loc;
                 if (uuid_is_null (loc->inode->gfid) && uuid_is_null (loc->gfid)) {
@@ -2165,7 +2442,7 @@ afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
 
 out:
         if (op_errno) {
-                orig_sh->unwind (frame, this, -1, op_errno);
+                orig_sh->unwind (frame, this, -1, op_errno, 1);
                 if (sh_frame)
                         AFR_STACK_DESTROY (sh_frame);
         }
@@ -2224,13 +2501,21 @@ afr_self_heal_type_for_transaction (afr_transaction_type type)
 }
 
 int
-afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name, uuid_t gfid)
+afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
 {
         int   ret = -1;
+        uuid_t pargfid = {0};
 
-        if (!child) {
+        if (!child)
+                goto out;
+
+        if (!uuid_is_null (parent->inode->gfid))
+                uuid_copy (pargfid, parent->inode->gfid);
+        else if (!uuid_is_null (parent->gfid))
+                uuid_copy (pargfid, parent->gfid);
+
+        if (uuid_is_null (pargfid))
                 goto out;
-        }
 
         if (strcmp (parent->path, "/") == 0)
                 ret = gf_asprintf ((char **)&child->path, "/%s", name);
@@ -2243,27 +2528,285 @@ afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name, uu
                         "asprintf failed while setting child path");
         }
 
-        if (!child->path) {
-                goto out;
-        }
-
         child->name = strrchr (child->path, '/');
         if (child->name)
                 child->name++;
 
         child->parent = inode_ref (parent->inode);
         child->inode = inode_new (parent->inode->table);
+        uuid_copy (child->pargfid, pargfid);
 
         if (!child->inode) {
                 ret = -1;
                 goto out;
         }
-        uuid_copy (child->gfid, gfid);
 
         ret = 0;
 out:
-        if (ret == -1)
+        if ((ret == -1) && child)
                 loc_wipe (child);
 
         return ret;
 }
+
+int
+afr_sh_erase_pending (call_frame_t *frame, xlator_t *this,
+                      afr_transaction_type type, afr_fxattrop_cbk_t cbk,
+                      int (*finish)(call_frame_t *frame, xlator_t *this))
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              call_count = 0;
+        int              i = 0;
+        dict_t          **erase_xattr = NULL;
+        int             ret = -1;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix,
+                                 sh->success, priv->child_count, type);
+
+        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
+                                 gf_afr_mt_dict_t);
+        if (!erase_xattr)
+                goto out;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sh->xattr[i]) {
+                        call_count++;
+                        erase_xattr[i] = dict_new ();
+                        if (!erase_xattr[i])
+                                goto out;
+                }
+        }
+
+        afr_sh_delta_to_xattr (this, sh->delta_matrix, erase_xattr,
+                               priv->child_count, type);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Delta matrix for: %s",
+                lkowner_utoa (&frame->root->lk_owner));
+        afr_sh_print_pending_matrix (sh->delta_matrix, this);
+        local->call_count = call_count;
+        if (call_count == 0) {
+                ret = 0;
+                finish (frame, this);
+                goto out;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!erase_xattr[i])
+                        continue;
+
+                if (sh->healing_fd) {//true for ENTRY, reg file DATA transaction
+                        STACK_WIND_COOKIE (frame, cbk, (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fxattrop,
+                                           sh->healing_fd,
+                                           GF_XATTROP_ADD_ARRAY, erase_xattr[i],
+                                           NULL);
+                } else {
+                        STACK_WIND_COOKIE (frame, cbk, (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->xattrop,
+                                           &local->loc,
+                                           GF_XATTROP_ADD_ARRAY, erase_xattr[i],
+                                           NULL);
+                }
+        }
+
+        ret = 0;
+out:
+        if (erase_xattr) {
+                for (i = 0; i < priv->child_count; i++) {
+                        if (erase_xattr[i]) {
+                                dict_unref (erase_xattr[i]);
+                        }
+                }
+        }
+
+        GF_FREE (erase_xattr);
+
+        if (ret < 0) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                finish (frame, this);
+        }
+
+        return 0;
+}
+
+void
+afr_set_self_heal_status(afr_self_heal_t *sh, afr_self_heal_status status)
+{
+        xlator_t                *this = NULL;
+        afr_sh_status_for_all_type *sh_status = &(sh->afr_all_sh_status);
+        afr_self_heal_type  sh_type_in_action = sh->sh_type_in_action;
+        this = THIS;
+
+        if (!sh) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Null self heal"
+                                  "Structure");
+                goto out;
+        }
+
+        switch (sh_type_in_action) {
+                case AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY:
+                       sh_status->gfid_or_missing_entry_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_METADATA:
+                        sh_status->metadata_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_DATA:
+                        sh_status->data_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_ENTRY:
+                        sh_status->entry_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_INVALID:
+                        gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid"
+                                          "self heal type in action");
+                        break;
+        }
+out:
+        return;
+}
+
+void
+afr_set_local_for_unhealable (afr_local_t *local)
+{
+        afr_self_heal_t  *sh = NULL;
+
+        sh = &local->self_heal;
+
+        local->unhealable = 1;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+}
+
+int
+is_self_heal_failed (afr_self_heal_t *sh, afr_sh_fail_check_type type)
+{
+        afr_sh_status_for_all_type      sh_status = sh->afr_all_sh_status;
+        afr_self_heal_type   sh_type_in_action =  AFR_SELF_HEAL_INVALID;
+        afr_self_heal_status    status = AFR_SELF_HEAL_FAILED;
+        xlator_t                *this = NULL;
+        int                     sh_failed = 0;
+
+        this = THIS;
+
+        if (!sh) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Null self heal "
+                                  "structure");
+                sh_failed = 1;
+                goto out;
+        }
+
+        if (type == AFR_CHECK_ALL) {
+                if ((sh_status.gfid_or_missing_entry_self_heal == AFR_SELF_HEAL_FAILED)
+                    || (sh_status.metadata_self_heal == AFR_SELF_HEAL_FAILED)
+                    || (sh_status.data_self_heal == AFR_SELF_HEAL_FAILED)
+                    || (sh_status.entry_self_heal == AFR_SELF_HEAL_FAILED))
+                sh_failed = 1;
+        } else if (type == AFR_CHECK_SPECIFIC) {
+                sh_type_in_action = sh->sh_type_in_action;
+                switch (sh_type_in_action) {
+                        case AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY:
+                             status = sh_status.gfid_or_missing_entry_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_METADATA:
+                                status = sh_status.metadata_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_ENTRY:
+                                status = sh_status.entry_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_DATA:
+                                status = sh_status.data_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_INVALID:
+                                status = AFR_SELF_HEAL_NOT_ATTEMPTED;
+                                break;
+                }
+                if (status == AFR_SELF_HEAL_FAILED)
+                        sh_failed = 1;
+
+        }
+
+out:
+        return sh_failed;
+}
+
+char *
+get_sh_completion_status (afr_self_heal_status status)
+{
+
+        char *not_attempted       = " is not attempted";
+        char *failed              = " failed";
+        char *started             = " is started";
+        char *sync_begin          = " is successfully completed";
+        char *result              = " has unknown status";
+
+        switch (status)
+        {
+                case AFR_SELF_HEAL_NOT_ATTEMPTED:
+                        result = not_attempted;
+                        break;
+                case AFR_SELF_HEAL_FAILED:
+                        result = failed;
+                        break;
+                case AFR_SELF_HEAL_STARTED:
+                        result = started;
+                        break;
+                case AFR_SELF_HEAL_SYNC_BEGIN:
+                        result = sync_begin;
+                        break;
+        }
+
+        return result;
+
+}
+
+void
+afr_log_self_heal_completion_status (afr_local_t *local, gf_loglevel_t loglvl)
+{
+
+        char sh_log[4096]              = {0};
+        afr_self_heal_t *sh            = &local->self_heal;
+        afr_sh_status_for_all_type   all_status = sh->afr_all_sh_status;
+        xlator_t      *this            = NULL;
+        size_t        off              = 0;
+        int           data_sh          = 0;
+        int           metadata_sh      = 0;
+        int           print_log        = 0;
+
+        this = THIS;
+
+        ADD_FMT_STRING (sh_log, off, "gfid or missing entry",
+                        all_status.gfid_or_missing_entry_self_heal, print_log);
+        ADD_FMT_STRING_SYNC (sh_log, off, "metadata",
+                             all_status.metadata_self_heal, print_log);
+        if (sh->background) {
+                ADD_FMT_STRING_SYNC (sh_log, off, "backgroung data",
+                                all_status.data_self_heal, print_log);
+        } else {
+                ADD_FMT_STRING_SYNC (sh_log, off, "foreground data",
+                                all_status.data_self_heal, print_log);
+        }
+        ADD_FMT_STRING_SYNC (sh_log, off, "entry", all_status.entry_self_heal,
+                             print_log);
+
+        if (AFR_SELF_HEAL_SYNC_BEGIN == all_status.data_self_heal &&
+	    strcmp (sh->data_sh_info, "") && sh->data_sh_info )
+                data_sh = 1;
+        if (AFR_SELF_HEAL_SYNC_BEGIN == all_status.metadata_self_heal &&
+	    strcmp (sh->metadata_sh_info, "") && sh->metadata_sh_info)
+                metadata_sh = 1;
+
+        if (!print_log)
+                return;
+
+        gf_log (this->name, loglvl, "%s %s %s on %s", sh_log,
+                ((data_sh == 1) ? sh->data_sh_info : ""),
+                ((metadata_sh == 1) ? sh->metadata_sh_info : ""),
+                local->loc.path);
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.h b/xlators/cluster/afr/src/afr-self-heal-common.h
index 114c17777..473264776 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.h
+++ b/xlators/cluster/afr/src/afr-self-heal-common.h
@@ -1,33 +1,18 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __AFR_SELF_HEAL_COMMON_H__
 #define __AFR_SELF_HEAL_COMMON_H__
 
 #define FILE_HAS_HOLES(buf) (((buf)->ia_size) > ((buf)->ia_blocks * 512))
-
-typedef enum {
-        AFR_SELF_HEAL_ENTRY,
-        AFR_SELF_HEAL_METADATA,
-        AFR_SELF_HEAL_DATA,
-        AFR_SELF_HEAL_INVALID = -1,
-} afr_self_heal_type;
+#define AFR_SH_MIN_PARTICIPANTS 2
 
 typedef enum {
         AFR_LOOKUP_FAIL_CONFLICTS = 1,
@@ -43,6 +28,10 @@ afr_sh_source_count (int sources[], int child_count);
 void
 afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this);
 
+void
+afr_sh_print_split_brain_log (int32_t *pending_matrix[], xlator_t *this,
+                              const char *loc);
+
 int
 afr_build_pending_matrix (char **pending_key, int32_t **pending_matrix,
                           unsigned char *ignorant_subvols,
@@ -60,7 +49,7 @@ afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
                   int32_t *success_children, int32_t *subvol_status);
 
 int
-afr_sh_delta_to_xattr (afr_private_t *priv,
+afr_sh_delta_to_xattr (xlator_t *this,
                        int32_t *delta_matrix[], dict_t *xattr[],
 		       int child_count, afr_transaction_type type);
 
@@ -90,7 +79,7 @@ afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
 int
 afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
                       afr_lookup_done_cbk_t lookup_cbk, uuid_t uuid,
-                      int32_t flags);
+                      int32_t flags, dict_t *xdata);
 int
 afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
                              int active_src, struct iatt *buf,
@@ -102,13 +91,13 @@ int
 afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
                              int child_index);
 int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this,
+afr_sh_data_unlock (call_frame_t *frame, xlator_t *this, char *dom,
                     afr_lock_cbk_t lock_cbk);
 afr_local_t *
-afr_local_copy (afr_local_t *l, xlator_t *this);
+afr_self_heal_local_init (afr_local_t *l, xlator_t *this);
 int
 afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
-                  off_t start, off_t len,
+                  off_t start, off_t len, gf_boolean_t block, char *dom,
                   afr_lock_cbk_t success_handler,
                   afr_lock_cbk_t failure_handler);
 void
@@ -118,11 +107,38 @@ afr_sh_mark_source_sinks (call_frame_t *frame, xlator_t *this);
 typedef int
 (*afr_fxattrop_cbk_t) (call_frame_t *frame, void *cookie,
                        xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xattr);
+                       dict_t *xattr, dict_t *xdata);
 int
-afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name,
-                     uuid_t gfid);
+afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
 int
 afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
                           int active_source, call_frame_t **impunge_frame);
+void
+afr_sh_reset (call_frame_t *frame, xlator_t *this);
+
+void
+afr_children_intersection_get (int32_t *set1, int32_t *set2,
+                               int *intersection, unsigned int child_count);
+int
+afr_get_no_xattr_dir_read_child (xlator_t *this, int32_t *success_children,
+                                 struct iatt *bufs);
+int
+afr_sh_erase_pending (call_frame_t *frame, xlator_t *this,
+                      afr_transaction_type type, afr_fxattrop_cbk_t cbk,
+                      int (*finish)(call_frame_t *frame, xlator_t *this));
+
+void
+afr_set_local_for_unhealable (afr_local_t *local);
+
+int
+is_self_heal_failed (afr_self_heal_t *sh, afr_sh_fail_check_type type);
+
+void
+afr_set_self_heal_status (afr_self_heal_t *sh, afr_self_heal_status status);
+
+void
+afr_log_self_heal_completion_status (afr_local_t *local, gf_loglevel_t  logl);
+
+char*
+afr_get_pending_matrix_str (int32_t *pending_matrix[], xlator_t *this);
 #endif /* __AFR_SELF_HEAL_COMMON_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 765edd277..9de26ee56 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -49,6 +40,14 @@
 #include "afr-self-heal-common.h"
 #include "afr-self-heal-algorithm.h"
 
+int
+afr_sh_data_fail (call_frame_t *frame, xlator_t *this);
+
+static inline gf_boolean_t
+afr_sh_data_proceed (unsigned int success_count)
+{
+        return (success_count >= AFR_SH_MIN_PARTICIPANTS);
+}
 
 extern int
 sh_loop_finish (call_frame_t *loop_frame, xlator_t *this);
@@ -63,15 +62,6 @@ int
 afr_sh_data_finish (call_frame_t *frame, xlator_t *this);
 
 int
-afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this,
-                      afr_fxattrop_cbk_t fxattrop_cbk);
-
-int
-afr_post_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *xattr);
-
-int
 afr_sh_data_done (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t     *local = NULL;
@@ -88,7 +78,7 @@ afr_sh_data_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t   *local       = NULL;
         afr_private_t *priv        = NULL;
@@ -101,7 +91,7 @@ afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         LOCK (&frame->lock);
         {
                 if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "flush failed on %s on subvolume %s: %s",
                                 local->loc.path, priv->children[child_index]->name,
                                 strerror (op_errno));
@@ -131,6 +121,11 @@ afr_sh_data_close (call_frame_t *frame, xlator_t *this)
         sh    = &local->self_heal;
         priv  = this->private;
 
+        if (!sh->healing_fd) {
+                //This happens when file is non-reg
+                afr_sh_data_done (frame, this);
+                return 0;
+        }
         call_count        = afr_set_elem_count_get (sh->success,
                                                     priv->child_count);
         local->call_count = call_count;
@@ -151,7 +146,7 @@ afr_sh_data_close (call_frame_t *frame, xlator_t *this)
                                    (void *) (long) i,
                                    priv->children[i],
                                    priv->children[i]->fops->flush,
-                                   sh->healing_fd);
+                                   sh->healing_fd, NULL);
 
                 if (!--call_count)
                         break;
@@ -161,9 +156,28 @@ afr_sh_data_close (call_frame_t *frame, xlator_t *this)
 }
 
 int
+afr_sh_dom_unlock (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh    = NULL;
+        afr_private_t   *priv  = NULL;
+
+        local = frame->local;
+        sh    = &local->self_heal;
+        priv  = this->private;
+
+        if (sh->sh_dom_lock_held)
+                afr_sh_data_unlock (frame, this, priv->sh_domain,
+                                    afr_sh_data_close);
+        else
+                afr_sh_data_close (frame, this);
+        return 0;
+}
+
+int
 afr_sh_data_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                         struct iatt *statpost)
+                         struct iatt *statpost, dict_t *xdata)
 {
 
         afr_local_t   *local       = NULL;
@@ -195,29 +209,20 @@ afr_sh_data_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 int
-afr_sh_data_setattr (call_frame_t *frame, xlator_t *this)
+afr_sh_data_setattr (call_frame_t *frame, xlator_t *this, struct iatt* stbuf)
 {
         afr_local_t     *local      = NULL;
         afr_private_t   *priv       = NULL;
         afr_self_heal_t *sh         = NULL;
         int              i          = 0;
         int              call_count = 0;
-        int              source     = 0;
         int32_t          valid      = 0;
-        struct iatt      stbuf      = {0,};
 
         local = frame->local;
         sh    = &local->self_heal;
         priv  = this->private;
 
-        source = sh->source;
-
-        valid |= (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
-
-        stbuf.ia_atime = sh->buf[source].ia_atime;
-        stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
-        stbuf.ia_mtime = sh->buf[source].ia_mtime;
-        stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
+        valid = (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
 
         call_count        = afr_set_elem_count_get (sh->success,
                                                     priv->child_count);
@@ -237,7 +242,7 @@ afr_sh_data_setattr (call_frame_t *frame, xlator_t *this)
                                    (void *) (long) i,
                                    priv->children[i],
                                    priv->children[i]->fops->setattr,
-                                   &local->loc, &stbuf, valid);
+                                   &local->loc, stbuf, valid, NULL);
 
                 if (!--call_count)
                         break;
@@ -249,7 +254,7 @@ afr_sh_data_setattr (call_frame_t *frame, xlator_t *this)
 int
 afr_sh_data_setattr_fstat_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               struct iatt *buf)
+                               struct iatt *buf, dict_t *xdata)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
@@ -259,9 +264,14 @@ afr_sh_data_setattr_fstat_cbk (call_frame_t *frame, void *cookie,
         sh = &local->self_heal;
 
         GF_ASSERT (sh->source == child_index);
-        if (op_ret != -1)
+        if (op_ret != -1) {
                 sh->buf[child_index] = *buf;
-        afr_sh_data_setattr (frame, this);
+                afr_sh_data_setattr (frame, this, buf);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
+                        "time-stamps after self-heal", local->loc.path);
+                afr_sh_data_fail (frame, this);
+        }
 
         return 0;
 }
@@ -286,37 +296,51 @@ afr_sh_set_timestamps (call_frame_t *frame, xlator_t *this)
                            (void *) (long) sh->source,
                            priv->children[sh->source],
                            priv->children[sh->source]->fops->fstat,
-                           sh->healing_fd);
+                           sh->healing_fd, NULL);
         return 0;
 }
 
 //Fun fact, lock_cbk is being used for both lock & unlock
 int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this,
+afr_sh_data_unlock (call_frame_t *frame, xlator_t *this, char *dom,
                     afr_lock_cbk_t lock_cbk)
 {
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_self_heal_t     *sh       = NULL;
+        afr_private_t       *priv     = NULL;
+        int                 ret       = 0;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         sh       = &local->self_heal;
+        priv     = this->private;
 
-        GF_ASSERT (sh->data_lock_held);
-
-        sh->data_lock_held = _gf_false;
+        if (strcmp (dom, this->name) == 0) {
+                sh->data_lock_held = _gf_false;
+        } else if (strcmp (dom, priv->sh_domain) == 0) {
+                sh->sh_dom_lock_held = _gf_false;
+        } else {
+                ret = -1;
+                goto out;
+        }
         int_lock->lock_cbk = lock_cbk;
+        int_lock->domain = dom;
         afr_unlock (frame, this);
 
+out:
+        if (ret) {
+                int_lock->lock_op_ret = -1;
+                int_lock->lock_cbk (frame, this);
+        }
         return 0;
 }
 
 int
 afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh    = NULL;
 
         local = frame->local;
         sh = &local->self_heal;
@@ -325,9 +349,9 @@ afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
                 "finishing data selfheal of %s", local->loc.path);
 
         if (sh->data_lock_held)
-                afr_sh_data_unlock (frame, this, afr_sh_data_close);
+                afr_sh_data_unlock (frame, this, this->name, afr_sh_dom_unlock);
         else
-                afr_sh_data_close (frame, this);
+                afr_sh_dom_unlock (frame, this);
 
         return 0;
 }
@@ -344,34 +368,49 @@ afr_sh_data_fail (call_frame_t *frame, xlator_t *this)
         gf_log (this->name, GF_LOG_DEBUG,
                 "finishing failed data selfheal of %s", local->loc.path);
 
-        sh->op_failed = 1;
-        if (sh->data_lock_held)
-                afr_sh_data_unlock (frame, this, afr_sh_data_close);
-        else
-                afr_sh_data_close (frame, this);
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        afr_sh_data_finish (frame, this);
         return 0;
 }
 
 int
 afr_sh_data_erase_pending_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this, int32_t op_ret,
-                               int32_t op_errno, dict_t *xattr)
+                               int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
         int             call_count = 0;
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int32_t         child_index = (long) cookie;
+
+        priv  = this->private;
+        local = frame->local;
+        sh    = &local->self_heal;
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Erasing of pending change "
+                        "log failed on %s for subvol %s, reason: %s",
+                        local->loc.path, priv->children[child_index]->name,
+                        strerror (op_errno));
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        }
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                local = frame->local;
-                sh = &local->self_heal;
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                        if (sh->old_loop_frame)
+                                sh_loop_finish (sh->old_loop_frame, this);
+                        sh->old_loop_frame = NULL;
+                        afr_sh_data_fail (frame, this);
+                        goto out;
+                }
                 if (!IA_ISREG (sh->type)) {
                         afr_sh_data_finish (frame, this);
                         goto out;
                 }
                 GF_ASSERT (sh->old_loop_frame);
-                afr_sh_data_lock (frame, this, 0, 0,
+                afr_sh_data_lock (frame, this, 0, 0, _gf_true, this->name,
                                   afr_post_sh_big_lock_success,
                                   afr_post_sh_big_lock_failure);
         }
@@ -382,74 +421,95 @@ out:
 int
 afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
+        afr_sh_erase_pending (frame, this, AFR_DATA_TRANSACTION,
+                              afr_sh_data_erase_pending_cbk,
+                              afr_sh_data_finish);
+        return 0;
+}
+
+int
+afr_sh_data_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata)
+{
+        afr_local_t     *local      = NULL;
+        afr_private_t   *priv       = NULL;
+        afr_self_heal_t *sh         = NULL;
+        int             call_count  = 0;
+        int             child_index = (long) cookie;
 
         local = frame->local;
-        sh = &local->self_heal;
         priv = this->private;
+        sh   = &local->self_heal;
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to fsync on "
+                        "%s - %s", local->loc.path,
+                        priv->children[child_index]->name, strerror (op_errno));
+                LOCK (&frame->lock);
+                {
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                }
+                UNLOCK (&frame->lock);
+                if (sh->old_loop_frame)
+                        sh_loop_finish (sh->old_loop_frame, this);
+                sh->old_loop_frame = NULL;
+        }
 
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_DATA_TRANSACTION);
-        gf_log (this->name, GF_LOG_DEBUG, "Delta matrix for: %s",
-                lkowner_utoa (&frame->root->lk_owner));
-        afr_sh_print_pending_matrix (sh->delta_matrix, this);
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC))
+                        afr_sh_data_fail (frame, this);
+                else
+                        afr_sh_data_erase_pending (frame, this);
+        }
+        return 0;
+}
 
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
+/*
+ * Before erasing xattrs, make sure the data is written to disk
+ */
+int
+afr_sh_data_fsync (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local     = NULL;
+        afr_private_t   *priv      = NULL;
+        afr_self_heal_t *sh        = NULL;
+        int             i          = 0;
+        int             call_count = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
+        local = frame->local;
+        priv = this->private;
+        sh   = &local->self_heal;
 
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
+        call_count        = sh->active_sinks;
+        if (call_count == 0) {
+                afr_sh_data_erase_pending (frame, this);
+                return 0;
         }
 
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_DATA_TRANSACTION);
-
-        GF_ASSERT (call_count);
         local->call_count = call_count;
         for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
+                if (!sh->success[i] || sh->sources[i])
                         continue;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->fxattrop,
-                                   sh->healing_fd,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-                if (!--call_count)
-                        break;
+                STACK_WIND_COOKIE (frame, afr_sh_data_fsync_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->fsync,
+                                   sh->healing_fd, 1, NULL);
         }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
-
         return 0;
 }
 
-
 static struct afr_sh_algorithm *
 sh_algo_from_name (xlator_t *this, char *name)
 {
         int i = 0;
 
+        if (name == NULL)
+                goto out;
+
         while (afr_self_heal_algorithms[i].name) {
                 if (!strcmp (name, afr_self_heal_algorithms[i].name)) {
                         return &afr_self_heal_algorithms[i];
@@ -458,17 +518,22 @@ sh_algo_from_name (xlator_t *this, char *name)
                 i++;
         }
 
+out:
         return NULL;
 }
 
 
 static int
-sh_zero_byte_files_exist (afr_self_heal_t *sh, int child_count)
+sh_zero_byte_files_exist (afr_local_t *local, int child_count)
 {
-        int i;
-        int ret = 0;
+        int             i = 0;
+        int             ret = 0;
+        afr_self_heal_t *sh = NULL;
 
+        sh = &local->self_heal;
         for (i = 0; i < child_count; i++) {
+                if (!local->child_up[i] || sh->child_errno[i])
+                        continue;
                 if (sh->buf[i].ia_size == 0) {
                         ret = 1;
                         break;
@@ -495,8 +560,7 @@ afr_sh_data_pick_algo (call_frame_t *frame, xlator_t *this)
         if (algo == NULL) {
                 /* option not set, so fall back on heuristics */
 
-                if ((local->enoent_count != 0)
-                    || sh_zero_byte_files_exist (sh, priv->child_count)
+                if (sh_zero_byte_files_exist (local, priv->child_count)
                     || (sh->file_size <= (priv->data_self_heal_window_size *
                                           this->ctx->page_size))) {
 
@@ -534,11 +598,12 @@ afr_sh_data_sync_prepare (call_frame_t *frame, xlator_t *this)
         local = frame->local;
         sh = &local->self_heal;
 
-        sh->algo_completion_cbk = afr_sh_data_erase_pending;
+        sh->algo_completion_cbk = afr_sh_data_fsync;
         sh->algo_abort_cbk      = afr_sh_data_fail;
 
         sh_algo = afr_sh_data_pick_algo (frame, this);
 
+        sh->algo = sh_algo;
         sh_algo->fn (frame, this);
 
         return 0;
@@ -547,38 +612,46 @@ afr_sh_data_sync_prepare (call_frame_t *frame, xlator_t *this)
 int
 afr_sh_data_trim_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        afr_private_t * priv = NULL;
-        afr_local_t * local  = NULL;
         int              call_count = 0;
         int              child_index = 0;
+        afr_private_t    *priv = NULL;
+        afr_local_t      *local  = NULL;
+        afr_self_heal_t  *sh = NULL;
 
-        priv = this->private;
+        priv  = this->private;
         local = frame->local;
+        sh    = &local->self_heal;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (op_ret == -1)
-                        gf_log (this->name, GF_LOG_INFO,
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "ftruncate of %s on subvolume %s failed (%s)",
                                 local->loc.path,
                                 priv->children[child_index]->name,
                                 strerror (op_errno));
-                else
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "ftruncate of %s on subvolume %s completed",
                                 local->loc.path,
                                 priv->children[child_index]->name);
+                }
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0)
-                afr_sh_data_sync_prepare (frame, this);
+        if (call_count == 0) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC))
+                        afr_sh_data_fail (frame, this);
+                else
+                        afr_sh_data_sync_prepare (frame, this);
+        }
 
         return 0;
 }
@@ -612,7 +685,8 @@ afr_sh_data_trim_sinks (call_frame_t *frame, xlator_t *this)
                                    (void *) (long) i,
                                    priv->children[i],
                                    priv->children[i]->fops->ftruncate,
-                                   sh->healing_fd, sh->file_size);
+                                   sh->healing_fd, sh->file_size,
+                                   NULL);
 
                 if (!--call_count)
                         break;
@@ -626,6 +700,7 @@ afr_sh_inode_set_read_ctx (afr_self_heal_t *sh, xlator_t *this)
 {
         afr_private_t   *priv = NULL;
         int             ret = 0;
+        int             i = 0;
 
         priv = this->private;
         sh->source = afr_sh_select_source (sh->sources, priv->child_count);
@@ -634,6 +709,15 @@ afr_sh_inode_set_read_ctx (afr_self_heal_t *sh, xlator_t *this)
                 goto out;
         }
 
+        /* detect changes not visible through pending flags -- JIC */
+        for (i = 0; i < priv->child_count; i++) {
+                if (i == sh->source || sh->child_errno[i])
+                        continue;
+
+                if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[sh->source]))
+                        sh->sources[i] = 0;
+        }
+
         afr_reset_children (sh->fresh_children, priv->child_count);
         afr_get_fresh_children (sh->success_children, sh->sources,
                                 sh->fresh_children, priv->child_count);
@@ -643,16 +727,254 @@ out:
         return ret;
 }
 
-int
+char*
+afr_get_sizes_str (afr_local_t *local, struct iatt *bufs, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        int           i     = 0;
+        char          num[1024] = {0};
+        size_t        len = 0;
+        char          *sizes_str = NULL;
+        size_t        off = 0;
+        char          *fmt_str = "%llu bytes on %s, ";
+        char          *child_down =  " %s,";
+        char          *child_unknown = " %s,";
+        int           down_child_present = 0;
+        int           down_count = 0;
+        int           unknown_count = 0;
+        int           unknown_child_present = 0;
+        char          *down_subvol_1 = " down subvolume is ";
+        char          *unknown_subvol_1 = " unknown subvolume is ";
+        char          *down_subvol_2 = " down subvolumes are ";
+        char          *unknown_subvol_2 = " unknown subvolumes are ";
+
+        priv = this->private;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 1) {
+                        len += snprintf (num, sizeof (num), fmt_str,
+                                         (unsigned long long) bufs[i].ia_size,
+                                         priv->children[i]->name);
+                } else if (local->child_up[i] == 0) {
+                        len += snprintf (num, sizeof (num), child_down,
+                                         priv->children[i]->name);
+                        if (!down_child_present)
+                                down_child_present = 1;
+                        down_count ++;
+                } else if (local->child_up[i] == -1) {
+                        len += snprintf (num, sizeof (num), child_unknown,
+                                         priv->children[i]->name);
+                        if (!unknown_child_present)
+                                unknown_child_present = 1;
+                        unknown_count++;
+                }
+
+        }
+
+        if (down_child_present) {
+                if (down_count > 1)
+                        len += snprintf (num, sizeof (num), "%s",
+                                         down_subvol_2);
+                else
+                        len += snprintf (num, sizeof (num), "%s",
+                                        down_subvol_1);
+        }
+        if (unknown_child_present) {
+                if (unknown_count > 1)
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_2);
+                else
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_1);
+        }
+
+        len++;//for '\0'
+
+        sizes_str = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+
+        if (!sizes_str)
+                return NULL;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 1) {
+                        off += snprintf (sizes_str + off, len - off, fmt_str,
+                                         (unsigned long long) bufs[i].ia_size,
+                                         priv->children[i]->name);
+                }
+        }
+
+        if (down_child_present) {
+                if (down_count > 1) {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                         down_subvol_2);
+                } else {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                         down_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 0) {
+                        off += snprintf (sizes_str + off, len - off, child_down,
+                                         priv->children[i]->name);
+                }
+        }
+
+        if (unknown_child_present) {
+                if (unknown_count > 1) {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                        unknown_subvol_2);
+                } else {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                         unknown_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == -1) {
+                        off += snprintf (sizes_str + off, len - off,
+                                         child_unknown,
+                                         priv->children[i]->name);
+
+                }
+        }
+
+        return sizes_str;
+}
+
+char*
+afr_get_sinks_str (xlator_t *this, afr_local_t *local, afr_self_heal_t *sh)
+{
+        afr_private_t   *priv = NULL;
+        int             i = 0;
+        char            num[1024] = {0};
+        size_t          len = 0;
+        char            *sinks_str = NULL;
+        char            *temp_str = " to sinks ";
+        char            *str_format = " %s,";
+        char            off = 0;
+
+        priv = this->private;
+
+        len += snprintf (num, sizeof (num), "%s", temp_str);
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->sources[i] == 0) && (local->child_up[i] == 1)) {
+                        len += snprintf (num, sizeof (num), str_format,
+                                         priv->children[i]->name);
+                }
+        }
+
+        len ++;
+
+        sinks_str = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+
+        if (!sinks_str)
+                return NULL;
+
+        off += snprintf (sinks_str + off, len - off, "%s", temp_str);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->sources[i] == 0) && (local->child_up[i] == 1)) {
+                        off += snprintf (sinks_str + off, len - off,
+                                         str_format,
+                                         priv->children[i]->name);
+                }
+        }
+
+        return sinks_str;
+
+}
+
+
+void
+afr_set_data_sh_info_str (afr_local_t *local, afr_self_heal_t *sh, xlator_t *this)
+{
+        char            *pending_matrix_str = NULL;
+        char            *sizes_str = NULL;
+        char            *sinks_str = NULL;
+        afr_private_t   *priv = NULL;
+
+        priv = this->private;
+
+        pending_matrix_str = afr_get_pending_matrix_str (sh->pending_matrix,
+                                                         this);
+        if (!pending_matrix_str)
+                pending_matrix_str = "";
+
+        sizes_str = afr_get_sizes_str (local, sh->buf, this);
+        if (!sizes_str)
+                sizes_str = "";
+
+        sinks_str = afr_get_sinks_str (this, local, sh);
+        if (!sinks_str)
+                sinks_str = "";
+
+        gf_asprintf (&sh->data_sh_info, " data self heal from %s %s with "
+                     "%s data %s", priv->children[sh->source]->name, sinks_str,
+                     sizes_str, pending_matrix_str);
+
+        if (pending_matrix_str && strcmp (pending_matrix_str, ""))
+                GF_FREE (pending_matrix_str);
+
+        if (sizes_str && strcmp (sizes_str, ""))
+                GF_FREE (sizes_str);
+}
+
+void
 afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
 {
+        int              source = 0;
+        afr_local_t     *local      = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        source     = sh->source;
+        sh->block_size = this->ctx->page_size;
+        sh->file_size  = sh->buf[source].ia_size;
+
+        if (FILE_HAS_HOLES (&sh->buf[source]))
+                sh->file_has_holes = 1;
+
+        if (sh->background && sh->unwind && !sh->unwound) {
+                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno,
+                            is_self_heal_failed (sh, AFR_CHECK_SPECIFIC));
+                sh->unwound = _gf_true;
+        }
+
+        afr_sh_mark_source_sinks (frame, this);
+        if (sh->active_sinks == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no active sinks for performing self-heal on file %s",
+                        local->loc.path);
+                afr_sh_data_finish (frame, this);
+                return;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "self-healing file %s from subvolume %s to %d other",
+                local->loc.path, priv->children[sh->source]->name,
+                sh->active_sinks);
+
+        sh->actual_sh_started = _gf_true;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_SYNC_BEGIN);
+        afr_sh_data_trim_sinks (frame, this);
+}
+
+int
+afr_sh_data_fxattrop_fstat_done (call_frame_t *frame, xlator_t *this)
+{
         afr_local_t     *local      = NULL;
         afr_self_heal_t *sh = NULL;
         afr_private_t   *priv = NULL;
         int              nsources = 0;
-        int              source = 0;
-        int              i = 0;
         int              ret = 0;
+        int             *old_sources = NULL;
+        int             tstamp_source = 0;
+        int             i = 0;
 
         local = frame->local;
         sh = &local->self_heal;
@@ -660,19 +982,17 @@ afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
 
         gf_log (this->name, GF_LOG_DEBUG, "Pending matrix for: %s",
                 lkowner_utoa (&frame->root->lk_owner));
+        if (sh->sync_done) {
+                //store sources before sync so that mtime can be set using the
+                //iatt buf from one of them.
+                old_sources = alloca (priv->child_count*sizeof (*old_sources));
+                memcpy (old_sources, sh->sources,
+                        priv->child_count * sizeof (*old_sources));
+        }
 
         nsources = afr_build_sources (this, sh->xattr, sh->buf, sh->pending_matrix,
                                       sh->sources, sh->success_children,
-                                      AFR_DATA_TRANSACTION, NULL, _gf_false);
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_data_finish (frame, this);
-                return 0;
-        }
-
+                                      AFR_DATA_TRANSACTION, NULL, _gf_true);
         if ((nsources == -1)
             && (priv->favorite_child != -1)
             && (sh->child_errno[priv->favorite_child] == 0)) {
@@ -690,17 +1010,16 @@ afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
         }
 
         if (nsources == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to self-heal contents of '%s' (possible "
-                        "split-brain). Please delete the file from all but "
-                        "the preferred subvolume.", local->loc.path);
-
-                local->govinda_gOvinda = 1;
+                afr_sh_print_split_brain_log (sh->pending_matrix, this,
+                                              local->loc.path);
+                afr_set_split_brain (this, sh->inode, DONT_KNOW, SPB);
 
                 afr_sh_data_fail (frame, this);
                 return 0;
         }
 
+        afr_set_split_brain (this, sh->inode, DONT_KNOW, NO_SPB);
+
         ret = afr_sh_inode_set_read_ctx (sh, this);
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -710,49 +1029,42 @@ afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
                 return 0;
         }
 
-        source     = sh->source;
-        sh->block_size = this->ctx->page_size;
-        sh->file_size  = sh->buf[source].ia_size;
-
-        if (FILE_HAS_HOLES (&sh->buf[source]))
-                sh->file_has_holes = 1;
-
-        /* detect changes not visible through pending flags -- JIC */
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == source || sh->child_errno[i])
-                        continue;
-
-                if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[source]))
-                        sh->sources[i] = 0;
-        }
+        if (sh->sync_done) {
+                /* Perform setattr from one of the old_sources if possible
+                 * Because only they have the correct mtime, the new sources
+                 * (i.e. old sinks) have mtime from last writev in sync.
+                 */
+                tstamp_source = sh->source;
+                for (i = 0; i < priv->child_count; i++) {
+                        if (old_sources[i] && sh->sources[i])
+                                tstamp_source = i;
+                }
+                afr_sh_data_setattr (frame, this, &sh->buf[tstamp_source]);
+        } else {
+                afr_set_data_sh_info_str (local, sh, this);
+                if (nsources == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "No self-heal needed for %s",
+                                local->loc.path);
 
-        if (sh->background && sh->unwind) {
-                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno);
-                sh->unwound = _gf_true;
-        }
+                        afr_sh_data_finish (frame, this);
+                        return 0;
+                }
 
-        afr_sh_mark_source_sinks (frame, this);
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no active sinks for performing self-heal on file %s",
-                        local->loc.path);
-                afr_sh_data_finish (frame, this);
-                return 0;
+                if (sh->do_data_self_heal &&
+                    afr_data_self_heal_enabled (priv->data_self_heal))
+                        afr_sh_data_fix (frame, this);
+                else
+                        afr_sh_data_finish (frame, this);
         }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "self-healing file %s from subvolume %s to %d other",
-                local->loc.path, priv->children[sh->source]->name,
-                sh->active_sinks);
-        afr_sh_data_trim_sinks (frame, this);
-
         return 0;
 }
 
 int
 afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
                                           dict_t **xattr,
-                                          afr_transaction_type txn_type)
+                                          afr_transaction_type txn_type,
+                                          uuid_t gfid)
 {
         afr_private_t            *priv      = NULL;
         int                      read_child = -1;
@@ -769,20 +1081,33 @@ afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
         bufs = local->cont.lookup.bufs;
         success_children = local->cont.lookup.success_children;
 
-        pending_matrix = afr_matrix_create (priv->child_count,
-                                            priv->child_count);
-        if (NULL == pending_matrix)
-                goto out;
-
+        pending_matrix = local->cont.lookup.pending_matrix;
         sources = local->cont.lookup.sources;
         memset (sources, 0, sizeof (*sources) * priv->child_count);
 
         nsources = afr_build_sources (this, xattr, bufs, pending_matrix,
                                       sources, success_children, txn_type,
                                       &subvol_status, _gf_false);
-        if (subvol_status & SPLIT_BRAIN)
-                gf_log (this->name, GF_LOG_WARNING, "%s: Possible split-brain",
+        if (subvol_status & SPLIT_BRAIN) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s: Possible split-brain",
                         local->loc.path);
+                switch (txn_type) {
+                case AFR_DATA_TRANSACTION:
+                        local->cont.lookup.possible_spb = _gf_true;
+                        nsources = 1;
+                        sources[success_children[0]] = 1;
+                        break;
+                case AFR_ENTRY_TRANSACTION:
+                        read_child = afr_get_no_xattr_dir_read_child (this,
+                                                             success_children,
+                                                             bufs);
+                        sources[read_child] = 1;
+                        nsources = 1;
+                        break;
+                default:
+                        break;
+                }
+        }
         if (nsources < 0)
                 goto out;
 
@@ -792,39 +1117,18 @@ afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
                                                         priv->child_count,
                                                         prev_read_child,
                                                         config_read_child,
-                                                        sources);
+                                                        sources,
+                                                        priv->hash_mode, gfid);
 out:
-        afr_matrix_cleanup (pending_matrix, priv->child_count);
         gf_log (this->name, GF_LOG_DEBUG, "returning read_child: %d",
                 read_child);
         return read_child;
 }
 
 int
-afr_sh_data_special_file_fix (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_local_t     *local = NULL;
-        int             i = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count ; i++)
-                if (1 == local->child_up[i])
-                        sh->success[i] = 1;
-
-        afr_sh_data_erase_pending (frame, this);
-
-        return 0;
-}
-
-int
 afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
                        xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       struct iatt *buf)
+                       struct iatt *buf, dict_t *xdata)
 {
         afr_private_t   *priv  = NULL;
         afr_local_t     *local = NULL;
@@ -847,6 +1151,12 @@ afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
                         sh->buf[child_index] = *buf;
                         sh->success_children[sh->success_count] = child_index;
                         sh->success_count++;
+                } else {
+                        gf_log (this->name, GF_LOG_ERROR, "%s: fstat failed "
+                                "on %s, reason %s", local->loc.path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+                        sh->child_errno[child_index] = op_errno;
                 }
         }
         UNLOCK (&frame->lock);
@@ -857,13 +1167,17 @@ afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
                 /* Previous versions of glusterfs might have set
                  * the pending data xattrs which need to be erased
                  */
-                if (IA_ISREG (buf->ia_type))
-                        afr_sh_data_fix (frame, this);
-                else
-                        afr_sh_data_special_file_fix (frame, this);
-
+                if (!afr_sh_data_proceed (sh->success_count)) {
+                        gf_log (this->name, GF_LOG_ERROR, "inspecting metadata "
+                                "succeeded on < %d children, aborting "
+                                "self-heal for %s", AFR_SH_MIN_PARTICIPANTS,
+                                local->loc.path);
+                        afr_sh_data_fail (frame, this);
+                        goto out;
+                }
+                afr_sh_data_fxattrop_fstat_done (frame, this);
         }
-
+out:
         return 0;
 }
 
@@ -874,33 +1188,41 @@ afr_sh_data_fstat (call_frame_t *frame, xlator_t *this)
         afr_self_heal_t *sh    = NULL;
         afr_local_t     *local = NULL;
         afr_private_t   *priv  = NULL;
-        int call_count = 0;
-        int i = 0;
+        int             call_count = 0;
+        int             i = 0;
+        int             child = 0;
+        int32_t         *fstat_children = NULL;
 
         priv  = this->private;
         local = frame->local;
         sh    = &local->self_heal;
 
-        call_count = afr_up_children_count (local->child_up,
-                                            priv->child_count);
-
+        fstat_children = memdup (sh->success_children,
+                                 sizeof (*fstat_children) * priv->child_count);
+        if (!fstat_children) {
+                afr_sh_data_fail (frame, this);
+                goto out;
+        }
+        call_count = sh->success_count;
         local->call_count = call_count;
 
+        memset (sh->buf, 0, sizeof (*sh->buf) * priv->child_count);
         afr_reset_children (sh->success_children, priv->child_count);
         sh->success_count = 0;
         for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_sh_data_fstat_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fstat,
-                                           sh->healing_fd);
-
-                        if (!--call_count)
-                                break;
-                }
+                child = fstat_children[i];
+                if (child == -1)
+                        break;
+                STACK_WIND_COOKIE (frame, afr_sh_data_fstat_cbk,
+                                   (void *) (long) child,
+                                   priv->children[child],
+                                   priv->children[child]->fops->fstat,
+                                   sh->healing_fd, NULL);
+                --call_count;
         }
-
+        GF_ASSERT (!call_count);
+out:
+        GF_FREE (fstat_children);
         return 0;
 }
 
@@ -929,73 +1251,60 @@ afr_sh_common_fxattrop_resp_handler (call_frame_t *frame, void *cookie,
                         sh->xattr[child_index] = dict_ref (xattr);
                         sh->success_children[sh->success_count] = child_index;
                         sh->success_count++;
+                } else {
+                        gf_log (this->name, GF_LOG_ERROR, "fxattrop of %s "
+                                "failed on %s, reason %s", local->loc.path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+                        sh->child_errno[child_index] = op_errno;
                 }
         }
         UNLOCK (&frame->lock);
 }
 
 int
-afr_post_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *xattr)
+afr_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
+                          xlator_t *this, int32_t op_ret, int32_t op_errno,
+                          dict_t *xattr, dict_t *xdata)
 {
         int             call_count  = -1;
-        int             ret = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        afr_sh_common_fxattrop_resp_handler (frame, cookie, this, op_ret,
-                                             op_errno, xattr);
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh     = NULL;
 
         local = frame->local;
-        sh = &local->self_heal;
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                (void) afr_build_sources (this, sh->xattr, NULL,
-                                          sh->pending_matrix,
-                                          sh->sources, sh->success_children,
-                                          AFR_DATA_TRANSACTION, NULL, _gf_false);
-                ret = afr_sh_inode_set_read_ctx (sh, this);
-                if (ret)
-                        afr_sh_data_fail (frame, this);
-                else
-                        afr_sh_set_timestamps (frame, this);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                          xlator_t *this, int32_t op_ret, int32_t op_errno,
-                          dict_t *xattr)
-{
-        int call_count  = -1;
+        sh    = &local->self_heal;
 
         afr_sh_common_fxattrop_resp_handler (frame, cookie, this, op_ret,
                                              op_errno, xattr);
 
         call_count = afr_frame_return (frame);
         if (call_count == 0) {
+                if (!afr_sh_data_proceed (sh->success_count)) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s, inspecting "
+                                "change log succeeded on < %d children",
+                                local->loc.path, AFR_SH_MIN_PARTICIPANTS);
+                        afr_sh_data_fail (frame, this);
+                        goto out;
+                }
                 afr_sh_data_fstat (frame, this);
         }
-
+out:
         return 0;
 }
 
 
 int
-afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this,
-                      afr_fxattrop_cbk_t fxattrop_cbk)
+afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this)
 {
         afr_self_heal_t *sh    = NULL;
         afr_local_t     *local = NULL;
         afr_private_t   *priv  = NULL;
-        dict_t          *xattr_req = NULL;
+        dict_t          **xattr_req;
         int32_t         *zero_pending = NULL;
         int call_count = 0;
         int i = 0;
         int ret = 0;
+	int j;
 
         priv  = this->private;
         local = frame->local;
@@ -1006,42 +1315,53 @@ afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this,
 
         local->call_count = call_count;
 
-        xattr_req = dict_new();
-        if (!xattr_req) {
-                ret = -1;
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                zero_pending = GF_CALLOC (3, sizeof (*zero_pending),
-                                          gf_afr_mt_int32_t);
-                if (!zero_pending) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynptr (xattr_req, priv->pending_key[i],
-                                       zero_pending,
-                                       3 * sizeof (*zero_pending));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unable to set dict value");
-                        goto out;
-                } else {
-                        zero_pending = NULL;
-                }
-        }
+	xattr_req = GF_CALLOC(priv->child_count, sizeof(struct dict_t *),
+			      gf_afr_mt_dict_t);
+	if (!xattr_req)
+		goto out;
+
+	for (i = 0; i < priv->child_count; i++) {
+		xattr_req[i] = dict_new();
+		if (!xattr_req[i]) {
+			ret = -1;
+			goto out;
+		}
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		for (j = 0; j < priv->child_count; j++) {
+			zero_pending = GF_CALLOC (3, sizeof (*zero_pending),
+						  gf_afr_mt_int32_t);
+			if (!zero_pending) {
+				ret = -1;
+				goto out;
+			}
+			ret = dict_set_dynptr (xattr_req[i], priv->pending_key[j],
+					       zero_pending,
+					       3 * sizeof (*zero_pending));
+			if (ret < 0) {
+				gf_log (this->name, GF_LOG_WARNING,
+					"Unable to set dict value");
+				goto out;
+			} else {
+				zero_pending = NULL;
+			}
+		}
+	}
 
         afr_reset_xattr (sh->xattr, priv->child_count);
         afr_reset_children (sh->success_children, priv->child_count);
+        memset (sh->child_errno, 0,
+                sizeof (*sh->child_errno) * priv->child_count);
         sh->success_count = 0;
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, fxattrop_cbk,
+                        STACK_WIND_COOKIE (frame, afr_sh_data_fxattrop_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->fxattrop,
                                            sh->healing_fd, GF_XATTROP_ADD_ARRAY,
-                                           xattr_req);
+                                           xattr_req[i], NULL);
 
                         if (!--call_count)
                                 break;
@@ -1049,14 +1369,16 @@ afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this,
         }
 
 out:
-        if (xattr_req)
-                dict_unref (xattr_req);
+	if (xattr_req) {
+		for (i = 0; i < priv->child_count; i++)
+			if (xattr_req[i])
+				dict_unref(xattr_req[i]);
+		GF_FREE(xattr_req);
+	}
 
         if (ret) {
-                if (zero_pending)
-                        GF_FREE (zero_pending);
-                sh->op_failed = 1;
-                afr_sh_data_done (frame, this);
+                GF_FREE (zero_pending);
+                afr_sh_data_fail (frame, this);
         }
 
         return 0;
@@ -1072,7 +1394,23 @@ afr_sh_data_big_lock_success (call_frame_t *frame, xlator_t *this)
         sh = &local->self_heal;
 
         sh->data_lock_held = _gf_true;
-        afr_sh_data_fxattrop (frame, this, afr_sh_data_fxattrop_cbk);
+        afr_sh_data_fxattrop (frame, this);
+        return 0;
+}
+
+int
+afr_sh_dom_lock_success (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        sh->sh_dom_lock_held = _gf_true;
+        afr_sh_data_lock (frame, this, 0, 0, _gf_true, this->name,
+                          afr_sh_data_big_lock_success,
+                          afr_sh_data_fail);
         return 0;
 }
 
@@ -1121,8 +1459,13 @@ afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
                         "failed for %s. by %s",
                         local->loc.path, lkowner_utoa (&frame->root->lk_owner));
 
-                int_lock->lock_cbk = afr_sh_data_post_blocking_inodelk_cbk;
-                afr_blocking_lock (frame, this);
+		if (!sh->data_lock_block) {
+			sh->data_lock_failure_handler(frame, this);
+		} else {
+			int_lock->lock_cbk =
+				afr_sh_data_post_blocking_inodelk_cbk;
+			afr_blocking_lock (frame, this);
+		}
         } else {
 
                 gf_log (this->name, GF_LOG_DEBUG, "Non Blocking data inodelks "
@@ -1135,9 +1478,11 @@ afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
 }
 
 int
-afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, off_t start, off_t len)
+afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, char *dom,
+                      off_t start, off_t len)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
 
         local    = frame->local;
@@ -1148,11 +1493,14 @@ afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, off_t start, off_t le
 
         afr_set_lock_number (frame, this);
 
-        int_lock->lk_flock.l_start = start;
-        int_lock->lk_flock.l_len   = len;
-        int_lock->lk_flock.l_type  = F_WRLCK;
         int_lock->lock_cbk         = afr_sh_data_post_nonblocking_inodelk_cbk;
 
+        int_lock->domain = dom;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+        inodelk->flock.l_start = start;
+        inodelk->flock.l_len   = len;
+        inodelk->flock.l_type  = F_WRLCK;
+
         afr_nonblocking_inodelk (frame, this);
 
         return 0;
@@ -1171,7 +1519,8 @@ afr_post_sh_big_lock_success (call_frame_t *frame, xlator_t *this)
         sh_loop_finish (sh->old_loop_frame, this);
         sh->old_loop_frame = NULL;
         sh->data_lock_held = _gf_true;
-        afr_sh_data_fxattrop (frame, this, afr_post_sh_data_fxattrop_cbk);
+        sh->sync_done = _gf_true;
+        afr_sh_data_fxattrop (frame, this);
         return 0;
 }
 
@@ -1194,8 +1543,8 @@ afr_post_sh_big_lock_failure (call_frame_t *frame, xlator_t *this)
 
 int
 afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
-                  off_t start, off_t len,
-                  afr_lock_cbk_t success_handler,
+                  off_t start, off_t len, gf_boolean_t block,
+                  char *dom, afr_lock_cbk_t success_handler,
                   afr_lock_cbk_t failure_handler)
 {
         afr_local_t *   local = NULL;
@@ -1206,12 +1555,13 @@ afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
 
         sh->data_lock_success_handler = success_handler;
         sh->data_lock_failure_handler = failure_handler;
-        return afr_sh_data_lock_rec (frame, this, start, len);
+	sh->data_lock_block = block;
+        return afr_sh_data_lock_rec (frame, this, dom, start, len);
 }
 
 int
 afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd)
+                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
@@ -1237,20 +1587,20 @@ afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 local->loc.path,
                                 priv->children[child_index]->name,
                                 strerror (op_errno));
-                        sh->op_failed = 1;
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                } else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "open of %s succeeded on child %s",
+                                local->loc.path,
+                                priv->children[child_index]->name);
                 }
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "open of %s succeeded on child %s",
-                        local->loc.path,
-                        priv->children[child_index]->name);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                if (sh->op_failed) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                         afr_sh_data_fail (frame, this);
                         return 0;
                 }
@@ -1259,9 +1609,8 @@ afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "fd for %s opened, commencing sync",
                         local->loc.path);
 
-                afr_sh_data_lock (frame, this, 0, 0,
-                                  afr_sh_data_big_lock_success,
-                                  afr_sh_data_fail);
+                afr_sh_data_lock (frame, this, 0, 0, _gf_true, priv->sh_domain,
+                                  afr_sh_dom_lock_success, afr_sh_data_fail);
         }
 
         return 0;
@@ -1298,7 +1647,7 @@ afr_sh_data_open (call_frame_t *frame, xlator_t *this)
                                    priv->children[i],
                                    priv->children[i]->fops->open,
                                    &local->loc,
-                                   O_RDWR|O_LARGEFILE, fd, 0);
+                                   O_RDWR|O_LARGEFILE, fd, NULL);
 
                 if (!--call_count)
                         break;
@@ -1307,20 +1656,93 @@ afr_sh_data_open (call_frame_t *frame, xlator_t *this)
         return 0;
 }
 
+void
+afr_sh_non_reg_fix (call_frame_t *frame, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno)
+{
+        afr_private_t   *priv = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_local_t     *local = NULL;
+        int             i = 0;
+
+        if (op_ret < 0) {
+                afr_sh_data_fail (frame, this);
+                return;
+        }
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        for (i = 0; i < priv->child_count ; i++) {
+                if (1 == local->child_up[i])
+                        sh->success[i] = 1;
+        }
+
+        afr_sh_erase_pending (frame, this, AFR_DATA_TRANSACTION,
+                              afr_sh_data_erase_pending_cbk,
+                              afr_sh_data_finish);
+}
 
 int
-afr_self_heal_data (call_frame_t *frame, xlator_t *this)
+afr_sh_non_reg_lock_success (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t   *local = NULL;
         afr_self_heal_t *sh = NULL;
-        afr_private_t *priv = this->private;
 
         local = frame->local;
         sh = &local->self_heal;
+        sh->data_lock_held = _gf_true;
+        afr_sh_common_lookup (frame, this, &local->loc,
+                              afr_sh_non_reg_fix, NULL,
+                              AFR_LOOKUP_FAIL_CONFLICTS |
+                              AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                              NULL);
+        return 0;
+}
 
+gf_boolean_t
+afr_can_start_data_self_heal (afr_self_heal_t *sh, afr_private_t *priv)
+{
+        if (sh->force_confirm_spb)
+                return _gf_true;
         if (sh->do_data_self_heal &&
-            afr_data_self_heal_enabled (priv->data_self_heal)) {
-                afr_sh_data_open (frame, this);
+            afr_data_self_heal_enabled (priv->data_self_heal))
+                return _gf_true;
+        return _gf_false;
+}
+
+int
+afr_self_heal_data (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh    = NULL;
+        afr_private_t   *priv  = this->private;
+        int             ret    = -1;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        sh->sh_type_in_action = AFR_SELF_HEAL_DATA;
+
+        if (afr_can_start_data_self_heal (sh, priv)) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
+                ret = afr_inodelk_init (&local->internal_lock.inodelk[1],
+                                        priv->sh_domain, priv->child_count);
+                if (ret < 0) {
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                        afr_sh_data_done (frame, this);
+                        return 0;
+                }
+
+                if (IA_ISREG (sh->type)) {
+                        afr_sh_data_open (frame, this);
+                } else {
+                        afr_sh_data_lock (frame, this, 0, 0, _gf_true,
+                                          this->name,
+                                          afr_sh_non_reg_lock_success,
+                                          afr_sh_data_fail);
+                }
         } else {
                 gf_log (this->name, GF_LOG_TRACE,
                         "not doing data self heal on %s",
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 570c7080f..53491a1d7 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -59,6 +50,9 @@
         } while (0);
 
 int
+afr_sh_entry_impunge_create_file (call_frame_t *impunge_frame, xlator_t *this,
+                                  int child_index);
+int
 afr_sh_entry_done (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t     *local = NULL;
@@ -67,10 +61,6 @@ afr_sh_entry_done (call_frame_t *frame, xlator_t *this)
         local = frame->local;
         sh = &local->self_heal;
 
-        if (sh->healing_fd)
-                fd_unref (sh->healing_fd);
-        sh->healing_fd = NULL;
-
         sh->completion_cbk (frame, this);
 
         return 0;
@@ -112,7 +102,7 @@ afr_sh_entry_finish (call_frame_t *frame, xlator_t *this)
 int
 afr_sh_entry_erase_pending_cbk (call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, dict_t *xattr)
+                                int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
         long                 i          = 0;
         int                  call_count = 0;
@@ -167,72 +157,20 @@ afr_sh_entry_erase_pending (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-        int              need_unwind = 0;
 
         local = frame->local;
         sh = &local->self_heal;
-        priv = this->private;
 
         if (sh->entries_skipped) {
-                need_unwind = 1;
-                sh->op_failed = _gf_true;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 goto out;
         }
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_ENTRY_TRANSACTION);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
-        }
-
-        if (call_count == 0)
-                need_unwind = 1;
-
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_ENTRY_TRANSACTION);
-
-        local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_entry_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &local->loc,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-                if (!--call_count)
-                        break;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
-
+        afr_sh_erase_pending (frame, this, AFR_ENTRY_TRANSACTION,
+                              afr_sh_entry_erase_pending_cbk,
+                              afr_sh_entry_finish);
+        return 0;
 out:
-        if (need_unwind)
-                afr_sh_entry_finish (frame, this);
-
+        afr_sh_entry_finish (frame, this);
         return 0;
 }
 
@@ -345,7 +283,8 @@ int
 afr_sh_entry_expunge_parent_setattr_cbk (call_frame_t *expunge_frame,
                                          void *cookie, xlator_t *this,
                                          int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop)
+                                         struct iatt *preop, struct iatt *postop,
+                                         dict_t *xdata)
 {
         afr_private_t   *priv          = NULL;
         afr_local_t     *expunge_local = NULL;
@@ -381,7 +320,7 @@ afr_sh_entry_expunge_remove_cbk (call_frame_t *expunge_frame, void *cookie,
                                  xlator_t *this,
                                  int32_t op_ret, int32_t op_errno,
                                  struct iatt *preparent,
-                                 struct iatt *postparent)
+                                 struct iatt *postparent, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *expunge_local = NULL;
@@ -409,7 +348,6 @@ afr_sh_entry_expunge_remove_cbk (call_frame_t *expunge_frame, void *cookie,
         }
 
         valid = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-        afr_build_parent_loc (&expunge_sh->parent_loc, &expunge_local->loc);
 
         STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_parent_setattr_cbk,
                            (void *) (long) active_src,
@@ -417,7 +355,7 @@ afr_sh_entry_expunge_remove_cbk (call_frame_t *expunge_frame, void *cookie,
                            priv->children[active_src]->fops->setattr,
                            &expunge_sh->parent_loc,
                            &expunge_sh->parentbuf,
-                           valid);
+                           valid, NULL);
 
         return 0;
 }
@@ -441,7 +379,7 @@ afr_sh_entry_expunge_unlink (call_frame_t *expunge_frame, xlator_t *this,
                            (void *) (long) active_src,
                            priv->children[active_src],
                            priv->children[active_src]->fops->unlink,
-                           &expunge_local->loc);
+                           &expunge_local->loc, 0, NULL);
 
         return 0;
 }
@@ -466,7 +404,7 @@ afr_sh_entry_expunge_rmdir (call_frame_t *expunge_frame, xlator_t *this,
                            (void *) (long) active_src,
                            priv->children[active_src],
                            priv->children[active_src]->fops->rmdir,
-                           &expunge_local->loc, 1);
+                           &expunge_local->loc, 1, NULL);
 
         return 0;
 }
@@ -590,7 +528,7 @@ afr_sh_entry_expunge_purge (call_frame_t *expunge_frame, xlator_t *this,
                            (void *) (long) active_src,
                            priv->children[active_src],
                            priv->children[active_src]->fops->lookup,
-                           &expunge_local->loc, 0);
+                           &expunge_local->loc, NULL);
 
         return 0;
 }
@@ -642,7 +580,8 @@ afr_sh_entry_expunge_entry_cbk (call_frame_t *expunge_frame, void *cookie,
 
         if (need_expunge) {
                 gf_log (this->name, GF_LOG_INFO,
-                        "missing entry %s on %s",
+                        "Entry %s is missing on %s and deleting from "
+                        "replica's other bricks",
                         expunge_local->loc.path,
                         priv->children[source]->name);
 
@@ -674,6 +613,19 @@ out:
         return 0;
 }
 
+static gf_boolean_t
+can_skip_entry_self_heal (char *name, loc_t *parent_loc)
+{
+        if (strcmp (name, ".") == 0) {
+                return _gf_true;
+        } else if (strcmp (name, "..") == 0) {
+                return _gf_true;
+        } else if (loc_is_root (parent_loc) &&
+                   (strcmp (name, GF_REPLICATE_TRASH_DIR) == 0)) {
+                return _gf_true;
+        }
+        return _gf_false;
+}
 
 int
 afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
@@ -701,15 +653,7 @@ afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
         sh->expunge_done = afr_sh_entry_expunge_entry_done;
 
         name = entry->d_name;
-
-        if ((strcmp (name, ".") == 0)
-            || (strcmp (name, "..") == 0)
-            || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (name, GF_REPLICATE_TRASH_DIR) == 0))) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "skipping inspection of %s under %s",
-                        name, local->loc.path);
+        if (can_skip_entry_self_heal (name, &local->loc)) {
                 op_ret = 0;
                 goto out;
         }
@@ -724,16 +668,17 @@ afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        ALLOC_OR_GOTO (expunge_local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
 
         expunge_frame->local = expunge_local;
         expunge_sh = &expunge_local->self_heal;
         expunge_sh->sh_frame = frame;
         expunge_sh->active_source = active_src;
         expunge_sh->entrybuf = entry->d_stat;
+        loc_copy (&expunge_sh->parent_loc, &local->loc);
 
         ret = afr_build_child_loc (this, &expunge_local->loc, &local->loc,
-                                   name, entry->d_stat.ia_gfid);
+                                   name);
         if (ret != 0) {
                 op_errno = EINVAL;
                 goto out;
@@ -748,7 +693,7 @@ afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
                            (void *) (long) source,
                            priv->children[source],
                            priv->children[source]->fops->lookup,
-                           &expunge_local->loc, 0);
+                           &expunge_local->loc, NULL);
 
         ret = 0;
 out:
@@ -763,7 +708,7 @@ int
 afr_sh_entry_expunge_readdir_cbk (call_frame_t *frame, void *cookie,
                                   xlator_t *this,
                                   int32_t op_ret, int32_t op_errno,
-                                  gf_dirent_t *entries)
+                                  gf_dirent_t *entries, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *local  = NULL;
@@ -861,7 +806,7 @@ afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this)
         active_src = next_active_sink (frame, this, sh->active_source);
         sh->active_source = active_src;
 
-        if (sh->op_failed) {
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                 goto out;
         }
 
@@ -924,7 +869,8 @@ int
 afr_sh_entry_impunge_setattr_cbk (call_frame_t *impunge_frame, void *cookie,
                                   xlator_t *this,
                                   int32_t op_ret, int32_t op_errno,
-                                  struct iatt *preop, struct iatt *postop)
+                                  struct iatt *preop, struct iatt *postop,
+                                  dict_t *xdata)
 {
         int              call_count = 0;
         afr_private_t   *priv = NULL;
@@ -961,7 +907,8 @@ int
 afr_sh_entry_impunge_parent_setattr_cbk (call_frame_t *setattr_frame,
                                          void *cookie, xlator_t *this,
                                          int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop)
+                                         struct iatt *preop, struct iatt *postop,
+                                         dict_t *xdata)
 {
         int             call_count = 0;
         afr_local_t     *setattr_local = NULL;
@@ -1009,7 +956,7 @@ afr_sh_entry_impunge_setattr (call_frame_t *impunge_frame, xlator_t *this)
                 op_errno = ENOMEM;
                 goto out;
         }
-        ALLOC_OR_GOTO (setattr_frame->local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (setattr_frame->local, out);
         setattr_local = setattr_frame->local;
         call_count = afr_errno_count (NULL, impunge_sh->child_errno,
                                       priv->child_count, 0);
@@ -1025,7 +972,7 @@ afr_sh_entry_impunge_setattr (call_frame_t *impunge_frame, xlator_t *this)
                                    (void *) (long) i, priv->children[i],
                                    priv->children[i]->fops->setattr,
                                    &setattr_local->loc,
-                                   &impunge_sh->parentbuf, valid);
+                                   &impunge_sh->parentbuf, valid, NULL);
 
                 valid = GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
                         GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
@@ -1034,7 +981,7 @@ afr_sh_entry_impunge_setattr (call_frame_t *impunge_frame, xlator_t *this)
                                    (void *) (long) i, priv->children[i],
                                    priv->children[i]->fops->setattr,
                                    &impunge_local->loc,
-                                   &impunge_sh->entrybuf, valid);
+                                   &impunge_sh->entrybuf, valid, NULL);
                 call_count--;
         }
         GF_ASSERT (!call_count);
@@ -1050,7 +997,7 @@ int
 afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
                                   xlator_t *this,
                                   int32_t op_ret, int32_t op_errno,
-                                  dict_t *xattr)
+                                  dict_t *xattr, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *impunge_local = NULL;
@@ -1078,33 +1025,6 @@ out:
         return 0;
 }
 
-void
-afr_sh_prepare_new_entry_pending_matrix (int32_t **pending,
-                                         int *child_errno,
-                                         struct iatt *buf,
-                                         unsigned int child_count)
-{
-        int              midx             = 0;
-        int              idx              = 0;
-        int              i                = 0;
-
-        midx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-        if (IA_ISDIR (buf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
-        else if (IA_ISREG (buf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-        else
-                idx = -1;
-        for (i = 0; i < child_count; i++) {
-                if (child_errno[i])
-                        continue;
-                pending[i][midx] = hton32 (1);
-                if (idx == -1)
-                        continue;
-                pending[i][idx] = hton32 (1);
-        }
-}
-
 int
 afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
                                       xlator_t *this)
@@ -1121,23 +1041,25 @@ afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
         impunge_sh = &impunge_local->self_heal;
         active_src = impunge_sh->active_source;
 
-        afr_sh_prepare_new_entry_pending_matrix (impunge_local->pending,
-                                                 impunge_sh->child_errno,
-                                                 &impunge_sh->entrybuf,
-                                                 priv->child_count);
+        afr_prepare_new_entry_pending_matrix (impunge_local->pending,
+                                              afr_is_errno_unset,
+                                              impunge_sh->child_errno,
+                                              &impunge_sh->entrybuf,
+                                              priv->child_count);
         xattr = dict_new ();
         if (!xattr) {
                 op_errno = ENOMEM;
                 goto out;
         }
 
-        afr_set_pending_dict (priv, xattr, impunge_local->pending);
+        afr_set_pending_dict (priv, xattr, impunge_local->pending, active_src,
+                              LOCAL_LAST);
 
         STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_xattrop_cbk,
                            (void *) (long) active_src,
                            priv->children[active_src],
                            priv->children[active_src]->fops->xattrop,
-                           &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr);
+                           &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr, NULL);
 
         if (xattr)
                 dict_unref (xattr);
@@ -1154,7 +1076,7 @@ afr_sh_entry_impunge_newfile_cbk (call_frame_t *impunge_frame, void *cookie,
                                   int32_t op_ret, int32_t op_errno,
                                   inode_t *inode, struct iatt *stbuf,
                                   struct iatt *preparent,
-                                  struct iatt *postparent)
+                                  struct iatt *postparent, dict_t *xdata)
 {
         int              call_count       = 0;
         afr_private_t   *priv             = NULL;
@@ -1194,6 +1116,125 @@ out:
         return 0;
 }
 
+int
+afr_sh_entry_impunge_hardlink_cbk (call_frame_t *impunge_frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, inode_t *inode,
+                                   struct iatt *buf, struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata)
+{
+        int              call_count        = 0;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh  = NULL;
+
+        impunge_local = impunge_frame->local;
+        impunge_sh = &impunge_local->self_heal;
+
+        if (IA_IFLNK == impunge_sh->entrybuf.ia_type) {
+                //For symlinks impunge is attempted un-conditionally
+                //So the file can already exist.
+                if ((op_ret < 0) && (op_errno == EEXIST))
+                        op_ret = 0;
+        }
+
+        call_count = afr_frame_return (impunge_frame);
+        if (call_count == 0)
+                afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                op_ret, op_errno);
+
+        return 0;
+}
+
+int
+afr_sh_entry_impunge_hardlink (call_frame_t *impunge_frame, xlator_t *this,
+                               int child_index)
+{
+        afr_private_t   *priv          = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh  = NULL;
+        loc_t           *loc           = NULL;
+        struct iatt     *buf           = NULL;
+        loc_t            oldloc        = {0};
+
+        priv = this->private;
+        impunge_local = impunge_frame->local;
+        impunge_sh = &impunge_local->self_heal;
+        loc = &impunge_local->loc;
+        buf = &impunge_sh->entrybuf;
+
+        oldloc.inode = inode_ref (loc->inode);
+        uuid_copy (oldloc.gfid, buf->ia_gfid);
+        gf_log (this->name, GF_LOG_DEBUG, "linking missing file %s on %s",
+                loc->path, priv->children[child_index]->name);
+
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_hardlink_cbk,
+                           (void *) (long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->link,
+                           &oldloc, loc, NULL);
+        loc_wipe (&oldloc);
+
+        return 0;
+}
+
+int
+afr_sh_nameless_lookup_cbk (call_frame_t *impunge_frame, void *cookie,
+                            xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, dict_t *xattr,
+                            struct iatt *postparent)
+{
+        if (op_ret < 0) {
+                 afr_sh_entry_impunge_create_file (impunge_frame, this,
+                                                   (long)cookie);
+        } else {
+                afr_sh_entry_impunge_hardlink (impunge_frame, this,
+                                               (long)cookie);
+        }
+        return 0;
+}
+
+int
+afr_sh_entry_impunge_check_hardlink (call_frame_t *impunge_frame,
+                                     xlator_t *this,
+                                     int child_index, struct iatt *stbuf)
+{
+        afr_private_t   *priv          = NULL;
+        call_frame_t    *frame             = NULL;
+        afr_local_t     *impunge_local     = NULL;
+        afr_local_t     *local             = NULL;
+        afr_self_heal_t *impunge_sh        = NULL;
+        afr_self_heal_t *sh                = NULL;
+        loc_t           *loc           = NULL;
+        dict_t          *xattr_req     = NULL;
+        loc_t            oldloc        = {0};
+        int              ret           = -1;
+
+        priv = this->private;
+        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
+                                frame, local, sh);
+        loc = &impunge_local->loc;
+
+        xattr_req = dict_new ();
+        if (!xattr_req)
+                goto out;
+        oldloc.inode = inode_ref (loc->inode);
+        uuid_copy (oldloc.gfid, stbuf->ia_gfid);
+
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_nameless_lookup_cbk,
+                           (void *) (long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->lookup,
+                           &oldloc, xattr_req);
+        ret = 0;
+out:
+        if (xattr_req)
+                dict_unref (xattr_req);
+        loc_wipe (&oldloc);
+        if (ret)
+                sh->impunge_done (frame, this, -1, ENOMEM);
+        return 0;
+}
 
 int
 afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
@@ -1222,6 +1263,35 @@ afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_INFO, "%s: gfid set failed",
                         impunge_local->loc.path);
 
+        /*
+         * Reason for adding GLUSTERFS_INTERNAL_FOP_KEY :
+         *
+         * Problem:
+         * While a brick is down in a replica pair, lets say the user creates
+         * one file(file-A) and a hard link to that file(h-file-A). After the
+         * brick comes back up, entry self-heal is attempted on parent dir of
+         * these two files. As part of readdir in self-heal it reads both the
+         * entries file-A and h-file-A for both of them it does name less lookup
+         * to check if there are any hardlinks already present in the
+         * destination brick. It finds that there are no hard links already
+         * present for files file-A, h-file-A. Self-heal does mknods for both
+         * file-A and h-file-A. This leads to file-A and h-file-A not being
+         * hardlinks anymore.
+         *
+         * Fix: (More like shrinking of race-window, the race itself is still
+         * present in posix-mknod).
+         * If mknod comes with the presence of GLUSTERFS_INTERNAL_FOP_KEY then
+         * posix_mknod checks if there are already any gfid-links and does
+         * link() instead of mknod. There still can be a race where two
+         * posix_mknods same gfid see that
+         * gfid-link file is not present and proceeds with mknods and result in
+         * two different files with same gfid.
+         */
+        ret = dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+        if (ret)
+                gf_log (this->name, GF_LOG_INFO, "%s: %s set failed",
+                        impunge_local->loc.path, GLUSTERFS_INTERNAL_FOP_KEY);
+
         STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
                            (void *) (long) child_index,
                            priv->children[child_index],
@@ -1229,7 +1299,7 @@ afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
                            &impunge_local->loc,
                            st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
                            makedev (ia_major (stbuf->ia_rdev),
-                                    ia_minor (stbuf->ia_rdev)), dict);
+                                    ia_minor (stbuf->ia_rdev)), 0, dict);
 
         if (dict)
                 dict_unref (dict);
@@ -1276,7 +1346,7 @@ afr_sh_entry_impunge_mkdir (call_frame_t *impunge_frame, xlator_t *this,
                            priv->children[child_index]->fops->mkdir,
                            &impunge_local->loc,
                            st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                           dict);
+                           0, dict);
 
         if (dict)
                 dict_unref (dict);
@@ -1298,7 +1368,7 @@ afr_sh_entry_impunge_symlink (call_frame_t *impunge_frame, xlator_t *this,
         priv = this->private;
         impunge_local = impunge_frame->local;
 
-        buf = &impunge_local->cont.symlink.buf;
+        buf = &impunge_local->cont.dir_fop.buf;
 
         dict = dict_new ();
         if (!dict) {
@@ -1323,7 +1393,7 @@ afr_sh_entry_impunge_symlink (call_frame_t *impunge_frame, xlator_t *this,
                            (void *) (long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->symlink,
-                           linkname, &impunge_local->loc, dict);
+                           linkname, &impunge_local->loc, 0, dict);
 
         if (dict)
                 dict_unref (dict);
@@ -1337,7 +1407,7 @@ afr_sh_entry_impunge_symlink_unlink_cbk (call_frame_t *impunge_frame,
                                          void *cookie, xlator_t *this,
                                          int32_t op_ret, int32_t op_errno,
                                          struct iatt *preparent,
-                                         struct iatt *postparent)
+                                         struct iatt *postparent, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *impunge_local = NULL;
@@ -1398,7 +1468,7 @@ afr_sh_entry_impunge_symlink_unlink (call_frame_t *impunge_frame, xlator_t *this
                            (void *) (long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->unlink,
-                           &impunge_local->loc);
+                           &impunge_local->loc, 0, NULL);
 
         return 0;
 }
@@ -1408,7 +1478,7 @@ int
 afr_sh_entry_impunge_readlink_sink_cbk (call_frame_t *impunge_frame, void *cookie,
                                         xlator_t *this,
                                         int32_t op_ret, int32_t op_errno,
-                                        const char *linkname, struct iatt *sbuf)
+                                        const char *linkname, struct iatt *sbuf, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *impunge_local = NULL;
@@ -1490,7 +1560,7 @@ afr_sh_entry_impunge_readlink_sink (call_frame_t *impunge_frame, xlator_t *this,
                            (void *) (long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->readlink,
-                           &impunge_local->loc, 4096);
+                           &impunge_local->loc, 4096, NULL);
 
         return 0;
 }
@@ -1500,7 +1570,7 @@ int
 afr_sh_entry_impunge_readlink_cbk (call_frame_t *impunge_frame, void *cookie,
                                    xlator_t *this,
                                    int32_t op_ret, int32_t op_errno,
-                                   const char *linkname, struct iatt *sbuf)
+                                   const char *linkname, struct iatt *sbuf, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *impunge_local = NULL;
@@ -1558,13 +1628,13 @@ afr_sh_entry_impunge_readlink (call_frame_t *impunge_frame, xlator_t *this,
         impunge_local = impunge_frame->local;
         impunge_sh = &impunge_local->self_heal;
         active_src = impunge_sh->active_source;
-        impunge_local->cont.symlink.buf = *stbuf;
+        impunge_local->cont.dir_fop.buf = *stbuf;
 
         STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_cbk,
                            (void *) (long) child_index,
                            priv->children[active_src],
                            priv->children[active_src]->fops->readlink,
-                           &impunge_local->loc, 4096);
+                           &impunge_local->loc, 4096, NULL);
 
         return 0;
 }
@@ -1573,16 +1643,18 @@ int
 afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
                              int child_index)
 {
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
+        call_frame_t    *frame             = NULL;
+        afr_local_t     *impunge_local     = NULL;
+        afr_local_t     *local             = NULL;
+        afr_self_heal_t *impunge_sh        = NULL;
+        afr_self_heal_t *sh                = NULL;
         afr_private_t   *priv = NULL;
         ia_type_t       type = IA_INVAL;
-        int             ret = 0;
         int             active_src = 0;
         struct iatt     *buf = NULL;
 
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
+        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
+                                frame, local, sh);
         active_src = impunge_sh->active_source;
         afr_update_loc_gfids (&impunge_local->loc, &impunge_sh->entrybuf,
                               &impunge_sh->parentbuf);
@@ -1596,6 +1668,52 @@ afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
         case IA_IFBLK:
         case IA_IFCHR:
         case IA_IFIFO:
+        case IA_IFLNK:
+                afr_sh_entry_impunge_check_hardlink (impunge_frame, this,
+                                                     child_index, buf);
+                break;
+        case IA_IFDIR:
+                afr_sh_entry_impunge_mkdir (impunge_frame, this,
+                                            child_index, buf);
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s has unknown file type on %s: 0%o",
+                        impunge_local->loc.path,
+                        priv->children[active_src]->name, type);
+                sh->impunge_done (frame, this, -1, EINVAL);
+                break;
+        }
+
+        return 0;
+}
+
+int
+afr_sh_entry_impunge_create_file (call_frame_t *impunge_frame, xlator_t *this,
+                                  int child_index)
+{
+        call_frame_t    *frame             = NULL;
+        afr_local_t     *impunge_local     = NULL;
+        afr_local_t     *local             = NULL;
+        afr_self_heal_t *impunge_sh        = NULL;
+        afr_self_heal_t *sh                = NULL;
+        afr_private_t   *priv = NULL;
+        ia_type_t       type = IA_INVAL;
+        int             active_src = 0;
+        struct iatt     *buf = NULL;
+
+        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
+                                frame, local, sh);
+        active_src = impunge_sh->active_source;
+        buf = &impunge_sh->entrybuf;
+        type = buf->ia_type;
+
+        switch (type) {
+        case IA_IFSOCK:
+        case IA_IFREG:
+        case IA_IFBLK:
+        case IA_IFCHR:
+        case IA_IFIFO:
                 afr_sh_entry_impunge_mknod (impunge_frame, this,
                                             child_index, buf);
                 break;
@@ -1603,20 +1721,16 @@ afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
                 afr_sh_entry_impunge_readlink (impunge_frame, this,
                                                child_index, buf);
                 break;
-        case IA_IFDIR:
-                afr_sh_entry_impunge_mkdir (impunge_frame, this,
-                                            child_index, buf);
-                break;
         default:
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s has unknown file type on %s: 0%o",
                         impunge_local->loc.path,
                         priv->children[active_src]->name, type);
-                ret = -1;
+                sh->impunge_done (frame, this, -1, EINVAL);
                 break;
         }
 
-        return ret;
+        return 0;
 }
 
 gf_boolean_t
@@ -1762,7 +1876,8 @@ afr_sh_entry_common_lookup_done (call_frame_t *impunge_frame, xlator_t *this,
                 afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
                                       afr_sh_entry_common_lookup_done, gfid,
                                       AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
+                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                      NULL);
         } else {
                 afr_sh_entry_call_impunge_recreate (impunge_frame, this);
         }
@@ -1793,14 +1908,7 @@ afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
         active_src = sh->active_source;
         sh->impunge_done = afr_sh_entry_impunge_entry_done;
 
-        if ((strcmp (entry->d_name, ".") == 0)
-            || (strcmp (entry->d_name, "..") == 0)
-            || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR) == 0))) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "skipping inspection of %s under %s",
-                        entry->d_name, local->loc.path);
+        if (can_skip_entry_self_heal (entry->d_name, &local->loc)) {
                 op_ret = 0;
                 goto out;
         }
@@ -1819,7 +1927,7 @@ afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
         impunge_local = impunge_frame->local;
         impunge_sh = &impunge_local->self_heal;
         ret = afr_build_child_loc (this, &impunge_local->loc, &local->loc,
-                                   entry->d_name, entry->d_stat.ia_gfid);
+                                   entry->d_name);
         loc_copy (&impunge_sh->parent_loc, &local->loc);
         if (ret != 0) {
                 op_errno = ENOMEM;
@@ -1828,7 +1936,7 @@ afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
 
         afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
                               afr_sh_entry_common_lookup_done, NULL,
-                              AFR_LOOKUP_FAIL_CONFLICTS);
+                              AFR_LOOKUP_FAIL_CONFLICTS, NULL);
 
         op_ret = 0;
 out:
@@ -1846,7 +1954,7 @@ int
 afr_sh_entry_impunge_readdir_cbk (call_frame_t *frame, void *cookie,
                                   xlator_t *this,
                                   int32_t op_ret, int32_t op_errno,
-                                  gf_dirent_t *entries)
+                                  gf_dirent_t *entries, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *local  = NULL;
@@ -1869,7 +1977,7 @@ afr_sh_entry_impunge_readdir_cbk (call_frame_t *frame, void *cookie,
                                 local->loc.path,
                                 priv->children[active_src]->name,
                                 strerror (op_errno));
-                        sh->op_failed = 1;
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 } else {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "readdir of %s on subvolume %s complete",
@@ -1919,7 +2027,7 @@ afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this)
         STACK_WIND (frame, afr_sh_entry_impunge_readdir_cbk,
                     priv->children[active_src],
                     priv->children[active_src]->fops->readdirp,
-                    sh->healing_fd, sh->block_size, sh->offset, 0);
+                    sh->healing_fd, sh->block_size, sh->offset, NULL);
 
         return 0;
 }
@@ -1942,7 +2050,7 @@ afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this)
         active_src = next_active_source (frame, this, sh->active_source);
         sh->active_source = active_src;
 
-        if (sh->op_failed) {
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                 afr_sh_entry_finish (frame, this);
                 return 0;
         }
@@ -1965,7 +2073,7 @@ afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this)
 
 int
 afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, fd_t *fd)
+                          int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
@@ -1991,7 +2099,7 @@ afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 local->loc.path,
                                 priv->children[child_index]->name,
                                 strerror (op_errno));
-                        sh->op_failed = 1;
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 }
         }
         UNLOCK (&frame->lock);
@@ -1999,7 +2107,7 @@ afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                if (sh->op_failed) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                         afr_sh_entry_finish (frame, this);
                         return 0;
                 }
@@ -2037,7 +2145,7 @@ afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
         source  = local->self_heal.source;
         sources = local->self_heal.sources;
 
-        sh->block_size = 65536; //131072
+        sh->block_size = priv->sh_readdir_size;
         sh->offset = 0;
 
         call_count = sh->active_sinks;
@@ -2059,7 +2167,7 @@ afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
                                    (void *) (long) source,
                                    priv->children[source],
                                    priv->children[source]->fops->opendir,
-                                   &local->loc, fd);
+                                   &local->loc, fd, NULL);
                 call_count--;
         }
 
@@ -2076,7 +2184,7 @@ afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
                                    (void *) (long) i,
                                    priv->children[i],
                                    priv->children[i]->fops->opendir,
-                                   &local->loc, fd);
+                                   &local->loc, fd, NULL);
 
                 if (!--call_count)
                         break;
@@ -2131,6 +2239,8 @@ afr_sh_entry_sync_prepare (call_frame_t *frame, xlator_t *this)
                         "merging all entries as a conservative decision",
                         local->loc.path);
 
+        sh->actual_sh_started = _gf_true;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_SYNC_BEGIN);
         afr_sh_entry_open (frame, this);
 
         return 0;
@@ -2153,7 +2263,7 @@ afr_sh_entry_fix (call_frame_t *frame, xlator_t *this,
         priv = this->private;
 
         if (op_ret < 0) {
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_set_error (sh, op_errno);
                 afr_sh_entry_finish (frame, this);
                 goto out;
@@ -2216,7 +2326,7 @@ afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
         if (int_lock->lock_op_ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR, "Non Blocking entrylks "
                         "failed for %s.", local->loc.path);
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_entry_done (frame, this);
         } else {
 
@@ -2225,7 +2335,8 @@ afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
                 afr_sh_common_lookup (frame, this, &local->loc,
                                       afr_sh_entry_fix, NULL,
                                       AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
+                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                      NULL);
         }
 
         return 0;
@@ -2234,14 +2345,18 @@ afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
 int
 afr_self_heal_entry (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
+        afr_local_t     *local = NULL;
         afr_private_t   *priv = NULL;
-
+        afr_self_heal_t *sh = NULL;
 
         priv = this->private;
         local = frame->local;
+        sh = &local->self_heal;
+
+        sh->sh_type_in_action = AFR_SELF_HEAL_ENTRY;
 
         if (local->self_heal.do_entry_self_heal && priv->entry_self_heal) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
                 afr_sh_entrylk (frame, this, &local->loc, NULL,
                                 afr_sh_post_nonblocking_entry_cbk);
         } else {
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index 148e858bd..fd5da6cfd 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -54,31 +45,17 @@ afr_sh_metadata_done (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
 
         local = frame->local;
         sh = &local->self_heal;
-        priv = this->private;
-
-//      memset (sh->child_errno, 0, sizeof (int) * priv->child_count);
-        memset (sh->buf, 0, sizeof (struct iatt) * priv->child_count);
-        memset (sh->success, 0, sizeof (*sh->success) * priv->child_count);
 
-        afr_reset_xattr (sh->xattr, priv->child_count);
-        if (local->govinda_gOvinda) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "split-brain detected, aborting selfheal of %s",
+        afr_sh_reset (frame, this);
+        if (IA_ISDIR (sh->type)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "proceeding to entry check on %s",
                         local->loc.path);
-                sh->op_failed = 1;
-                sh->completion_cbk (frame, this);
+                afr_self_heal_entry (frame, this);
         } else {
-                if (IA_ISDIR (sh->type)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "proceeding to entry check on %s",
-                                local->loc.path);
-                        afr_self_heal_entry (frame, this);
-                        return 0;
-                }
                 gf_log (this->name, GF_LOG_DEBUG,
                         "proceeding to data check on %s",
                         local->loc.path);
@@ -111,11 +88,24 @@ afr_sh_metadata_finish (call_frame_t *frame, xlator_t *this)
         return 0;
 }
 
+int
+afr_sh_metadata_fail (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t         *local    = NULL;
+        afr_self_heal_t     *sh       = NULL;
+
+        local    = frame->local;
+        sh       = &local->self_heal;
+
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        afr_sh_metadata_finish (frame, this);
+        return 0;
+}
 
 int
 afr_sh_metadata_erase_pending_cbk (call_frame_t *frame, void *cookie,
                                    xlator_t *this, int32_t op_ret,
-                                   int32_t op_errno, dict_t *xattr)
+                                   int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
         afr_local_t     *local     = NULL;
         int             call_count = 0;
@@ -147,85 +137,19 @@ afr_sh_metadata_erase_pending_cbk (call_frame_t *frame, void *cookie,
         return 0;
 }
 
-
 int
 afr_sh_metadata_erase_pending (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix,
-                                 sh->success, priv->child_count,
-                                 AFR_METADATA_TRANSACTION);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-        if (!erase_xattr)
-                return -ENOMEM;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
-        }
-
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_METADATA_TRANSACTION);
-
-        local->call_count = call_count;
-
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "metadata of %s not healed on any subvolume",
-                        local->loc.path);
-
-                afr_sh_metadata_finish (frame, this);
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &local->loc,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-                if (!--call_count)
-                        break;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
-
-        return 0;
+         afr_sh_erase_pending (frame, this, AFR_METADATA_TRANSACTION,
+                               afr_sh_metadata_erase_pending_cbk,
+                               afr_sh_metadata_finish);
+         return 0;
 }
 
 
 int
 afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
@@ -256,8 +180,13 @@ afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0)
+        if (call_count == 0) {
+                if (local->xattr_req) {
+                        dict_unref (local->xattr_req);
+                        local->xattr_req = NULL;
+                }
                 afr_sh_metadata_erase_pending (frame, this);
+        }
 
         return 0;
 }
@@ -266,9 +195,9 @@ afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 afr_sh_metadata_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                              int32_t op_ret, int32_t op_errno,
-                             struct iatt *preop, struct iatt *postop)
+                             struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -276,13 +205,93 @@ afr_sh_metadata_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 afr_sh_metadata_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno)
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int
+afr_sh_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        dict_t *xdata)
 {
-        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno);
+        int            i     = 0;
+        afr_private_t *priv  = NULL;
+        afr_local_t   *local = NULL;
 
+        priv = this->private;
+        local = frame->local;
+
+        if (op_ret < 0) {
+                afr_sh_metadata_sync_cbk (frame, cookie,
+                                          this, -1, op_errno, xdata);
+                goto out;
+        }
+
+        i = (long) cookie;
+
+        STACK_WIND_COOKIE (frame, afr_sh_metadata_xattr_cbk,
+                           (void *) (long) i,
+                           priv->children[i],
+                           priv->children[i]->fops->setxattr,
+                           &local->loc, local->xattr_req, 0, NULL);
+
+ out:
         return 0;
 }
 
+inline void
+afr_prune_special_keys (dict_t *xattr_dict)
+{
+        dict_del (xattr_dict, GF_SELINUX_XATTR_KEY);
+}
+
+inline void
+afr_prune_pending_keys (dict_t *xattr_dict, afr_private_t *priv)
+{
+        int i = 0;
+
+        for (; i < priv->child_count; i++) {
+                dict_del (xattr_dict, priv->pending_key[i]);
+        }
+}
+
+int
+afr_sh_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                     dict_t *xdata)
+{
+        int            i     = 0;
+        afr_private_t *priv  = NULL;
+        afr_local_t   *local = NULL;
+
+        priv = this->private;
+        local = frame->local;
+
+        if (op_ret < 0) {
+                afr_sh_metadata_sync_cbk (frame, cookie,
+                                          this, -1, op_errno, xdata);
+                goto out;
+        }
+
+        afr_prune_pending_keys (xattr, priv);
+
+        afr_prune_special_keys (xattr);
+
+        i = (long) cookie;
+
+        /* send removexattr in bulk via xdata */
+        STACK_WIND_COOKIE (frame, afr_sh_removexattr_cbk,
+                           cookie,
+                           priv->children[i],
+                           priv->children[i]->fops->removexattr,
+                           &local->loc, "", xattr);
+
+ out:
+        return 0;
+}
 
 int
 afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
@@ -308,9 +317,10 @@ afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
         /*
          * 2 calls per sink - setattr, setxattr
          */
-        if (xattr)
+        if (xattr) {
                 call_count = active_sinks * 2;
-        else
+                local->xattr_req = dict_ref (xattr);
+        } else
                 call_count = active_sinks;
 
         local->call_count = call_count;
@@ -346,18 +356,18 @@ afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
                                    (void *) (long) i,
                                    priv->children[i],
                                    priv->children[i]->fops->setattr,
-                                   &local->loc, &stbuf, valid);
+                                   &local->loc, &stbuf, valid, NULL);
 
                 call_count--;
 
                 if (!xattr)
                         continue;
 
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_xattr_cbk,
+                STACK_WIND_COOKIE (frame, afr_sh_getxattr_cbk,
                                    (void *) (long) i,
                                    priv->children[i],
-                                   priv->children[i]->fops->setxattr,
-                                   &local->loc, xattr, 0);
+                                   priv->children[i]->fops->getxattr,
+                                   &local->loc, NULL, NULL);
                 call_count--;
         }
 
@@ -366,17 +376,15 @@ afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
 
 
 int
-afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie,
-                              xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xattr)
+afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                              dict_t *xdata)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
         afr_private_t   *priv = NULL;
         int              source = 0;
 
-        int i;
-
         local = frame->local;
         sh = &local->self_heal;
         priv = this->private;
@@ -391,16 +399,147 @@ afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie,
 
                 afr_sh_metadata_sync (frame, this, NULL);
         } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        dict_del (xattr, priv->pending_key[i]);
-                }
-
+                afr_prune_pending_keys (xattr, priv);
                 afr_sh_metadata_sync (frame, this, xattr);
         }
 
         return 0;
 }
 
+static void
+afr_set_metadata_sh_info_str (afr_local_t *local, afr_self_heal_t *sh,
+                              xlator_t *this)
+{
+        afr_private_t    *priv = NULL;
+        int              i = 0;
+        char             num[1024] = {0};
+        size_t           len = 0;
+        char             *string = NULL;
+        size_t           off = 0;
+        char             *source_child =  " from source %s to";
+        char             *format = " %s, ";
+        char             *string_msg = " metadata self heal";
+        char             *pending_matrix_str = NULL;
+        int              down_child_present = 0;
+        int              unknown_child_present = 0;
+        char             *down_subvol_1 = " down subvolume is ";
+        char             *unknown_subvol_1 = " unknown subvolume is";
+        char             *down_subvol_2 = " down subvolumes are ";
+        char             *unknown_subvol_2 = " unknown subvolumes are ";
+        int              down_count = 0;
+        int              unknown_count = 0;
+
+        priv = this->private;
+
+        pending_matrix_str = afr_get_pending_matrix_str (sh->pending_matrix,
+                                                         this);
+
+        if (!pending_matrix_str)
+                pending_matrix_str = "";
+
+        len += snprintf (num, sizeof (num), "%s", string_msg);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->source == i) && (local->child_up[i] == 1)) {
+                        len += snprintf (num, sizeof (num), source_child,
+                                         priv->children[i]->name);
+                } else if ((local->child_up[i] == 1) && (sh->sources[i] == 0)) {
+                        len += snprintf (num, sizeof (num), format,
+                                         priv->children[i]->name);
+                } else if (local->child_up[i] == 0) {
+                        len += snprintf (num, sizeof (num), format,
+                                         priv->children[i]->name);
+                        if (!down_child_present)
+                                down_child_present = 1;
+                        down_count++;
+                } else if (local->child_up[i] == -1) {
+                        len += snprintf (num, sizeof (num), format,
+                                         priv->children[i]->name);
+                        if (!unknown_child_present)
+                                unknown_child_present = 1;
+                        unknown_count++;
+                }
+        }
+
+        if (down_child_present) {
+                if (down_count > 1) {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         down_subvol_2);
+                } else {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         down_subvol_1);
+                }
+        }
+        if (unknown_child_present) {
+                if (unknown_count > 1) {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_2);
+                } else {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_1);
+                }
+        }
+
+        len ++;
+
+        string = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+        if (!string)
+                return;
+
+        off += snprintf (string + off, len - off, "%s", string_msg);
+        for (i=0; i < priv->child_count; i++) {
+                if ((sh->source == i) && (local->child_up[i] == 1))
+                        off += snprintf (string + off, len - off, source_child,
+                                         priv->children[i]->name);
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((local->child_up[i] == 1)&& (sh->sources[i] == 0))
+                        off += snprintf (string + off, len - off, format,
+                                         priv->children[i]->name);
+        }
+
+        if (down_child_present) {
+                if (down_count > 1) {
+                        off += snprintf (string + off, len - off, "%s",
+                                         down_subvol_2);
+                } else {
+                        off += snprintf (string + off, len - off, "%s",
+                                         down_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 0)
+                        off += snprintf (string + off, len - off, format,
+                                         priv->children[i]->name);
+        }
+
+        if (unknown_child_present) {
+                if (unknown_count > 1) {
+                        off += snprintf (string + off, len - off, "%s",
+                                 unknown_subvol_2);
+                } else {
+                        off += snprintf (string + off, len - off, "%s",
+                                         unknown_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == -1)
+                        off += snprintf (string + off, len - off, format,
+                                         priv->children[i]->name);
+        }
+
+        gf_asprintf (&sh->metadata_sh_info, "%s metadata %s,", string,
+                     pending_matrix_str);
+
+        if (pending_matrix_str && strcmp (pending_matrix_str, ""))
+                GF_FREE (pending_matrix_str);
+
+        if (string && strcmp (string, ""))
+                GF_FREE (string);
+}
 
 int
 afr_sh_metadata_sync_prepare (call_frame_t *frame, xlator_t *this)
@@ -430,10 +569,13 @@ afr_sh_metadata_sync_prepare (call_frame_t *frame, xlator_t *this)
                 local->loc.path, priv->children[source]->name,
                 sh->active_sinks);
 
+        sh->actual_sh_started = _gf_true;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_SYNC_BEGIN);
+        afr_set_metadata_sh_info_str (local, sh, this);
         STACK_WIND (frame, afr_sh_metadata_getxattr_cbk,
                     priv->children[source],
                     priv->children[source]->fops->getxattr,
-                    &local->loc, NULL);
+                    &local->loc, NULL, NULL);
 
         return 0;
 }
@@ -455,7 +597,7 @@ afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
         priv = this->private;
 
         if (op_ret < 0) {
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_set_error (sh, op_errno);
                 afr_sh_metadata_finish (frame, this);
                 goto out;
@@ -464,15 +606,6 @@ afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
                                       sh->pending_matrix, sh->sources,
                                       sh->success_children,
                                       AFR_METADATA_TRANSACTION, NULL, _gf_false);
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-
         if ((nsources == -1)
             && (priv->favorite_child != -1)
             && (sh->child_errno[priv->favorite_child] == 0)) {
@@ -489,12 +622,18 @@ afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
         }
 
         if (nsources == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to self-heal permissions/ownership of '%s' "
-                        "(possible split-brain). Please fix the file on "
-                        "all backend volumes", local->loc.path);
+                afr_sh_print_split_brain_log (sh->pending_matrix, this,
+                                              local->loc.path);
+                afr_set_split_brain (this, sh->inode, SPB, DONT_KNOW);
+                afr_sh_metadata_fail (frame, this);
+                goto out;
+        }
 
-                local->govinda_gOvinda = 1;
+        afr_set_split_brain (this, sh->inode, NO_SPB, DONT_KNOW);
+        if (nsources == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "No self-heal needed for %s",
+                        local->loc.path);
 
                 afr_sh_metadata_finish (frame, this);
                 goto out;
@@ -533,7 +672,10 @@ afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
                                         sh->fresh_children);
         }
 
-        afr_sh_metadata_sync_prepare (frame, this);
+        if (sh->do_metadata_self_heal && priv->metadata_self_heal)
+                afr_sh_metadata_sync_prepare (frame, this);
+        else
+                afr_sh_metadata_finish (frame, this);
 out:
         return;
 }
@@ -549,9 +691,9 @@ afr_sh_metadata_post_nonblocking_inodelk_cbk (call_frame_t *frame,
         int_lock = &local->internal_lock;
 
         if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Non Blocking metadata "
+                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking metadata "
                         "inodelks failed for %s.", local->loc.path);
-                gf_log (this->name, GF_LOG_ERROR, "Metadata self-heal "
+                gf_log (this->name, GF_LOG_DEBUG, "Metadata self-heal "
                         "failed for %s.", local->loc.path);
                 afr_sh_metadata_done (frame, this);
         } else {
@@ -562,7 +704,8 @@ afr_sh_metadata_post_nonblocking_inodelk_cbk (call_frame_t *frame,
                 afr_sh_common_lookup (frame, this, &local->loc,
                                       afr_sh_metadata_fix, NULL,
                                       AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
+                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                      NULL);
         }
 
         return 0;
@@ -572,19 +715,22 @@ int
 afr_sh_metadata_lock (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
 
+        int_lock->domain           = this->name;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
         int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
         int_lock->selfheal_lk_type    = AFR_METADATA_SELF_HEAL_LK;
 
         afr_set_lock_number (frame, this);
 
-        int_lock->lk_flock.l_start = LLONG_MAX - 1;
-        int_lock->lk_flock.l_len   = 0;
-        int_lock->lk_flock.l_type  = F_WRLCK;
+        inodelk->flock.l_start = LLONG_MAX - 1;
+        inodelk->flock.l_len   = 0;
+        inodelk->flock.l_type  = F_WRLCK;
         int_lock->lock_cbk         = afr_sh_metadata_post_nonblocking_inodelk_cbk;
 
         afr_nonblocking_inodelk (frame, this);
@@ -592,17 +738,29 @@ afr_sh_metadata_lock (call_frame_t *frame, xlator_t *this)
         return 0;
 }
 
+gf_boolean_t
+afr_can_start_metadata_self_heal (afr_self_heal_t *sh, afr_private_t *priv)
+{
+        if (sh->force_confirm_spb)
+                return _gf_true;
+        if (sh->do_metadata_self_heal && priv->metadata_self_heal)
+                return _gf_true;
+        return _gf_false;
+}
 
 int
 afr_self_heal_metadata (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t   *local = NULL;
         afr_private_t *priv = this->private;
-
+        afr_self_heal_t *sh = &local->self_heal;
 
         local = frame->local;
+        sh = &local->self_heal;
+        sh->sh_type_in_action = AFR_SELF_HEAL_METADATA;
 
-        if (local->self_heal.do_metadata_self_heal && priv->metadata_self_heal) {
+        if (afr_can_start_metadata_self_heal (sh, priv)) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
                 afr_sh_metadata_lock (frame, this);
         } else {
                 afr_sh_metadata_done (frame, this);
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index f40c06faa..7c9bc8111 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __AFR_SELF_HEAL_H__
@@ -30,13 +21,6 @@
 #define SIZE_GREATER(buf1,buf2) ((buf1)->ia_size > (buf2)->ia_size)
 
 int
-afr_sh_has_metadata_pending (dict_t *xattr, xlator_t *this);
-int
-afr_sh_has_entry_pending (dict_t *xattr, xlator_t *this);
-int
-afr_sh_has_data_pending (dict_t *xattr, xlator_t *this);
-
-int
 afr_self_heal_entry (call_frame_t *frame, xlator_t *this);
 
 int
@@ -54,5 +38,6 @@ afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode);
 int
 afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
                                           dict_t **xattr,
-                                          afr_transaction_type txn_type);
+                                          afr_transaction_type txn_type,
+                                          uuid_t gfid);
 #endif /* __AFR_SELF_HEAL_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index 186d7dd26..1b48a1bca 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -25,9 +16,1076 @@
 #include "syncop.h"
 #include "afr-self-heald.h"
 #include "afr-self-heal-common.h"
+#include "protocol-common.h"
+#include "event-history.h"
+
+typedef enum {
+        STOP_CRAWL_ON_SINGLE_SUBVOL = 1
+} afr_crawl_flags_t;
+
+typedef enum {
+        HEAL = 1,
+        INFO,
+        STATISTICS_TO_BE_HEALED,
+} shd_crawl_op;
+
+typedef struct shd_dump {
+        dict_t   *dict;
+        xlator_t *this;
+        int      child;
+} shd_dump_t;
+
+typedef struct shd_event_ {
+        int     child;
+        char    *path;
+} shd_event_t;
+
+typedef struct shd_pos_ {
+        int     child;
+        xlator_t *this;
+        afr_child_pos_t pos;
+} shd_pos_t;
+
+typedef int
+(*afr_crawl_done_cbk_t)  (int ret, call_frame_t *sync_frame, void *crawl_data);
+
+void
+afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
+                 process_entry_cbk_t process_entry, void *op_data,
+                 gf_boolean_t exclusive, int crawl_flags,
+                 afr_crawl_done_cbk_t crawl_done);
 
 static int
-_crawl_directory (loc_t *loc, pid_t pid);
+_crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data);
+
+/* For calling straight through (e.g. already in a synctask). */
+int
+afr_find_child_position (xlator_t *this, int child, afr_child_pos_t *pos);
+
+/* For deferring through a new synctask. */
+int
+afr_syncop_find_child_position (void *data);
+
+static int
+_loc_assign_gfid_path (loc_t *loc)
+{
+        int  ret = -1;
+        char gfid_path[64] = {0};
+
+        if (loc->inode && !uuid_is_null (loc->inode->gfid)) {
+                ret = inode_path (loc->inode, NULL, (char**)&loc->path);
+        } else if (!uuid_is_null (loc->gfid)) {
+                snprintf (gfid_path, sizeof (gfid_path), "<gfid:%s>",
+                          uuid_utoa (loc->gfid));
+                loc->path = gf_strdup (gfid_path);
+                if (loc->path)
+                        ret = 0;
+        }
+        return ret;
+}
+
+void
+_destroy_crawl_event_data (void *data)
+{
+        shd_crawl_event_t        *crawl_event = NULL;
+
+        if (!data)
+                goto out;
+
+        crawl_event = (shd_crawl_event_t *)data;
+        GF_FREE (crawl_event->start_time_str);
+        GF_FREE (crawl_event->end_time_str);
+
+out:
+        return;
+}
+
+void
+_destroy_shd_event_data (void *data)
+{
+        shd_event_t             *event = NULL;
+        if (!data)
+                goto out;
+        event = (shd_event_t*)data;
+        GF_FREE (event->path);
+out:
+        return;
+}
+void
+shd_cleanup_event (void *event)
+{
+        shd_event_t *shd_event = event;
+
+        if (!shd_event)
+                goto out;
+        GF_FREE (shd_event->path);
+        GF_FREE (shd_event);
+out:
+        return;
+}
+
+int
+afr_get_local_child (afr_self_heald_t *shd, unsigned int child_count)
+{
+        int i = 0;
+        int ret = -1;
+        for (i = 0; i < child_count; i++) {
+                if (shd->pos[i] == AFR_POS_LOCAL) {
+                        ret = i;
+                        break;
+                }
+        }
+        return ret;
+}
+
+static int
+_build_index_loc (xlator_t *this, loc_t *loc, char *name, loc_t *parent)
+{
+        int             ret = 0;
+
+        uuid_copy (loc->pargfid, parent->inode->gfid);
+        loc->path = "";
+        loc->name = name;
+        loc->parent = inode_ref (parent->inode);
+        if (!loc->parent) {
+                loc->path = NULL;
+                loc_wipe (loc);
+                ret = -1;
+        }
+        return ret;
+}
+
+int
+_add_crawl_stats_to_dict (xlator_t *this, dict_t *output, int child,
+                          shd_crawl_event_t *shd_event, struct timeval *tv)
+{
+        int             ret = 0;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+        uint64_t        healed_count = 0;
+        uint64_t        split_brain_count = 0;
+        uint64_t        heal_failed_count = 0;
+        char            *start_time_str = NULL;
+        char            *end_time_str = NULL;
+        char            *crawl_type = NULL;
+        int             progress = -1;
+
+        healed_count = shd_event->healed_count;
+        split_brain_count = shd_event->split_brain_count;
+        heal_failed_count = shd_event->heal_failed_count;
+        start_time_str = shd_event->start_time_str;
+        end_time_str = shd_event->end_time_str;
+        crawl_type = shd_event->crawl_type;
+
+        if (!start_time_str) {
+                ret = -1;
+                goto out;
+        }
+
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
+
+        snprintf (key, sizeof (key), "statistics_healed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64(output, key, healed_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "healed_count to outout");
+                goto out;
+         }
+        snprintf (key, sizeof (key), "statistics_sb_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, split_brain_count);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "split_brain_count to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_crawl_type-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, gf_strdup (crawl_type));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_type to output");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_heal_failed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, heal_failed_count);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "healed_failed_count to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_strt_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, gf_strdup(start_time_str));
+
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_start_time to outout");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics_end_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+
+        if (!end_time_str)
+                end_time_str = "Could not determine the end time";
+        ret = dict_set_dynstr (output, key, gf_strdup(end_time_str));
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_end_time to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_inprogress-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+
+        if (shd_event->crawl_inprogress == _gf_true)
+                progress = 1;
+        else
+                progress = 0;
+
+        ret = dict_set_int32 (output, key, progress);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "inprogress to outout");
+                goto out;
+        }
+
+         snprintf (key, sizeof (key), "statistics-%d-%d-count",xl_id, child);
+         ret = dict_set_uint64 (output, key, count + 1);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not increment the "
+                        "counter.");
+                goto out;
+         }
+out:
+        return ret;
+}
+
+int
+_add_path_to_dict (xlator_t *this, dict_t *output, int child, char *path,
+                   struct timeval *tv, gf_boolean_t dyn)
+{
+        //subkey not used for now
+        int             ret = -1;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
+
+        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
+        if (dyn)
+                ret = dict_set_dynstr (output, key, path);
+        else
+                ret = dict_set_str (output, key, path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Could not add to output",
+                        path);
+                goto out;
+        }
+
+        if (!tv)
+                goto inc_count;
+        snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
+                  child, count);
+        ret = dict_set_uint32 (output, key, tv->tv_sec);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Could not set time",
+                        path);
+                goto out;
+        }
+
+inc_count:
+        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+        ret = dict_set_uint64 (output, key, count + 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not increment count");
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+_get_path_from_gfid_loc (xlator_t *this, xlator_t *readdir_xl, loc_t *child,
+                         char **fpath, gf_boolean_t *missing)
+{
+        dict_t          *xattr = NULL;
+        char            *path = NULL;
+        int             ret = -1;
+
+        ret = syncop_getxattr (readdir_xl, child, &xattr, GFID_TO_PATH_KEY);
+        if (ret < 0) {
+                if ((errno == ENOENT) && missing)
+                        *missing = _gf_true;
+                goto out;
+        }
+        ret = dict_get_str (xattr, GFID_TO_PATH_KEY, &path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get path for "
+                        "gfid %s", uuid_utoa (child->gfid));
+                goto out;
+        }
+        path = gf_strdup (path);
+        if (!path) {
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        if (!ret)
+                *fpath = path;
+        if (xattr)
+                dict_unref (xattr);
+        return ret;
+}
+
+int
+_add_event_to_dict (circular_buffer_t *cb, void *data)
+{
+        int               ret = 0;
+        shd_dump_t        *dump_data = NULL;
+        shd_event_t       *shd_event = NULL;
+
+        dump_data = data;
+        shd_event = cb->data;
+        if (shd_event->child != dump_data->child)
+                goto out;
+        ret = _add_path_to_dict (dump_data->this, dump_data->dict,
+                                 dump_data->child, shd_event->path, &cb->tv,
+                                 _gf_false);
+out:
+        return ret;
+}
+
+int
+_add_crawl_event_statistics_to_dict (circular_buffer_t *cb, void *data)
+{
+        int               ret = 0;
+        shd_dump_t        *dump_data = NULL;
+        shd_crawl_event_t *shd_event = NULL;
+
+        dump_data = data;
+        shd_event = cb->data;
+        ret = _add_crawl_stats_to_dict (dump_data->this, dump_data->dict,
+                                        dump_data->child, shd_event, &cb->tv);
+        return ret;
+}
+
+int
+_add_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict, int child)
+{
+        shd_dump_t dump_data = {0};
+
+        dump_data.this = this;
+        dump_data.dict = dict;
+        dump_data.child = child;
+        eh_dump (eh, &dump_data, _add_event_to_dict);
+        return 0;
+}
+
+
+int
+_add_statistics_to_dict (xlator_t *this, dict_t *dict, int child)
+{
+        shd_dump_t              dump_data = {0};
+        afr_private_t           *priv  = NULL;
+        afr_self_heald_t        *shd = NULL;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        dump_data.this = this;
+        dump_data.dict = dict;
+        dump_data.child = child;
+        eh_dump (shd->statistics[child], &dump_data,
+                 _add_crawl_event_statistics_to_dict);
+        return 0;
+
+}
+
+void
+_remove_stale_index (xlator_t *this, xlator_t *readdir_xl,
+                     loc_t *parent, char *fname)
+{
+        int              ret = 0;
+        loc_t            index_loc = {0};
+
+        ret = _build_index_loc (this, &index_loc, fname, parent);
+        if (ret)
+                goto out;
+        gf_log (this->name, GF_LOG_DEBUG, "Removing stale index "
+                "for %s on %s", index_loc.name, readdir_xl->name);
+        ret = syncop_unlink (readdir_xl, &index_loc);
+        if(ret && (errno != ENOENT)) {
+                gf_log(this->name, GF_LOG_ERROR, "%s: Failed to remove index "
+                       "on %s - %s",index_loc.name, readdir_xl->name,
+                       strerror (errno));
+        }
+        index_loc.path = NULL;
+        loc_wipe (&index_loc);
+out:
+        return;
+}
+
+int
+_count_hard_links_under_base_indices_dir (xlator_t *this,
+                                           afr_crawl_data_t *crawl_data,
+                                           gf_dirent_t *entry, loc_t *childloc,
+                                           loc_t *parentloc, struct iatt *iattr)
+{
+        xlator_t                *readdir_xl = crawl_data->readdir_xl;
+        struct iatt             parent = {0};
+        int                     ret = 0;
+        dict_t                  *output = NULL;
+        int                     xl_id =  0;
+        char                    key[256] = {0};
+        int                     child  = -1;
+        uint64_t                     hardlinks = 0;
+
+        output = crawl_data->op_data;
+        child = crawl_data->child;
+
+        ret = syncop_lookup (readdir_xl, childloc, NULL, iattr, NULL, &parent);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret)
+                goto out;
+
+        snprintf (key, sizeof (key), "%d-%d-hardlinks", xl_id, child);
+        ret =  dict_get_uint64 (output, key, &hardlinks);
+
+        /*Removing the count of base_entry under indices/base_indicies and
+         * entry under indices/xattrop */
+        hardlinks = hardlinks + iattr->ia_nlink - 2;
+        ret = dict_set_uint64 (output, key, hardlinks);
+        if (ret)
+                goto out;
+
+out:
+        return ret;
+}
+
+int
+_add_summary_to_dict (xlator_t *this, afr_crawl_data_t *crawl_data,
+                      gf_dirent_t *entry,
+                      loc_t *childloc, loc_t *parentloc, struct iatt *iattr)
+{
+        dict_t          *output = NULL;
+        xlator_t        *readdir_xl = NULL;
+        int             ret = -1;
+        char            *path = NULL;
+        gf_boolean_t    missing = _gf_false;
+        char            gfid_str[64] = {0};
+
+        if (uuid_is_null (childloc->gfid))
+                goto out;
+
+        output = crawl_data->op_data;
+        readdir_xl = crawl_data->readdir_xl;
+
+        ret = _get_path_from_gfid_loc (this, readdir_xl, childloc, &path,
+                                       &missing);
+        if (ret == 0) {
+                ret = _add_path_to_dict (this, output, crawl_data->child, path,
+                                         NULL, _gf_true);
+        } else if (missing) {
+                _remove_stale_index (this, readdir_xl, parentloc,
+                                     uuid_utoa_r (childloc->gfid, gfid_str));
+        }
+
+out:
+        if (ret && path)
+                GF_FREE (path);
+        return ret;
+}
+
+void
+_crawl_post_sh_action (xlator_t *this, loc_t *parent, loc_t *child,
+                       int32_t op_ret, int32_t op_errno, dict_t *xattr_rsp,
+                       afr_crawl_data_t *crawl_data)
+{
+        int                ret = 0;
+        afr_private_t      *priv = NULL;
+        afr_self_heald_t   *shd = NULL;
+        eh_t               *eh = NULL;
+        char               *path = NULL;
+        char               gfid_str[64] = {0};
+        shd_event_t        *event = NULL;
+        int32_t            sh_failed = 0;
+        gf_boolean_t       split_brain = 0;
+        int32_t            actual_sh_done = 0;
+        shd_crawl_event_t  **shd_crawl_event = NULL;
+
+        priv = this->private;
+        shd  = &priv->shd;
+        if (crawl_data->crawl == INDEX) {
+                if ((op_ret < 0) && (op_errno == ENOENT)) {
+                        _remove_stale_index (this, crawl_data->readdir_xl,
+                                             parent, uuid_utoa_r (child->gfid,
+                                                                  gfid_str));
+                        goto out;
+                }
+                ret = _get_path_from_gfid_loc (this, crawl_data->readdir_xl,
+                                               child, &path, NULL);
+                if (ret)
+                        goto out;
+        } else {
+                path = gf_strdup (child->path);
+                if (!path) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (xattr_rsp) {
+                ret = dict_get_int32 (xattr_rsp, "sh-failed", &sh_failed);
+                ret = dict_get_int32 (xattr_rsp, "actual-sh-done", &actual_sh_done);
+        }
+
+        shd_crawl_event = (shd_crawl_event_t**)(shd->crawl_events);
+
+        split_brain = afr_is_split_brain (this, child->inode);
+        if ((op_ret < 0 && op_errno == EIO) || split_brain) {
+                eh = shd->split_brain;
+                shd_crawl_event[crawl_data->child]->split_brain_count += 1;
+        } else if ((op_ret < 0) || sh_failed) {
+                eh = shd->heal_failed;
+                shd_crawl_event[crawl_data->child]->heal_failed_count += 1;
+        } else if (actual_sh_done == 1) {
+                eh = shd->healed;
+                shd_crawl_event[crawl_data->child]->healed_count += 1;
+        }
+        ret = -1;
+
+        if (eh != NULL) {
+                event = GF_CALLOC (1, sizeof (*event), gf_afr_mt_shd_event_t);
+                if (!event)
+                        goto out;
+                event->child = crawl_data->child;
+                event->path = path;
+
+                ret = eh_save_history (eh, event);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s:Failed to save "
+                                "to event history, (%d, %s)", path, op_ret,
+                                strerror (op_errno));
+
+                        goto out;
+                }
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG, "%s:Self heal already done ",
+                        path);
+
+        }
+        ret = 0;
+out:
+        if (ret && path)
+                GF_FREE (path);
+        return;
+}
+
+int
+_link_inode_update_loc (xlator_t *this, loc_t *loc, struct iatt *iattr)
+{
+        inode_t       *link_inode = NULL;
+        int           ret = -1;
+
+        link_inode = inode_link (loc->inode, NULL, NULL, iattr);
+        if (link_inode == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "inode link failed "
+                        "on the inode (%s)", uuid_utoa (iattr->ia_gfid));
+                goto out;
+        }
+        inode_unref (loc->inode);
+        loc->inode = link_inode;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+_self_heal_entry (xlator_t *this, afr_crawl_data_t *crawl_data, gf_dirent_t *entry,
+                  loc_t *child, loc_t *parent, struct iatt *iattr)
+{
+        struct iatt      parentbuf = {0};
+        int              ret = 0;
+        dict_t           *xattr_rsp = NULL;
+        dict_t           *xattr_req = NULL;
+
+        xattr_req = dict_new ();
+        if (!xattr_req) {
+                errno = ENOMEM;
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_int32 (xattr_req, "allow-sh-for-running-transaction", 1);
+
+        gf_log (this->name, GF_LOG_DEBUG, "lookup %s", child->path);
+
+        ret = syncop_lookup (this, child, xattr_req,
+                             iattr, &xattr_rsp, &parentbuf);
+        _crawl_post_sh_action (this, parent, child, ret, errno, xattr_rsp,
+                               crawl_data);
+        if (xattr_rsp)
+                dict_unref (xattr_rsp);
+        if (ret == 0)
+                ret = _link_inode_update_loc (this, child, iattr);
+
+out:
+        if (xattr_req)
+                dict_unref(xattr_req);
+        return ret;
+}
+
+static int
+afr_crawl_done  (int ret, call_frame_t *sync_frame, void *data)
+{
+        GF_FREE (data);
+        STACK_DESTROY (sync_frame->root);
+        return 0;
+}
+
+void
+_do_self_heal_on_subvol (xlator_t *this, int child, afr_crawl_type_t crawl)
+{
+        afr_start_crawl (this, child, crawl, _self_heal_entry,
+                         NULL, _gf_true, STOP_CRAWL_ON_SINGLE_SUBVOL,
+                         afr_crawl_done);
+}
+
+gf_boolean_t
+_crawl_proceed (xlator_t *this, int child, int crawl_flags, char **reason)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        gf_boolean_t            proceed = _gf_false;
+        char                    *msg = NULL;
+
+        priv = this->private;
+        shd  = &priv->shd;
+        if (!shd->enabled) {
+                msg = "Self-heal daemon is not enabled";
+                gf_log (this->name, GF_LOG_DEBUG, "%s", msg);
+                goto out;
+        }
+        if (!priv->child_up[child]) {
+                gf_log (this->name, GF_LOG_DEBUG, "Stopping crawl for %s , "
+                        "subvol went down", priv->children[child]->name);
+                msg = "Brick is Not connected";
+                goto out;
+        }
+
+        if (crawl_flags & STOP_CRAWL_ON_SINGLE_SUBVOL) {
+                if (afr_up_children_count (priv->child_up,
+                                           priv->child_count) < 2) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Stopping crawl as "
+                                "< 2 children are up");
+                        msg = "< 2 bricks in replica are running";
+                        goto out;
+                }
+        }
+        proceed = _gf_true;
+out:
+        if (reason)
+                *reason = msg;
+        return proceed;
+}
+
+int
+_do_crawl_op_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
+                               shd_crawl_op op, dict_t *output)
+{
+        afr_private_t       *priv = NULL;
+        char                *status = NULL;
+        char                *subkey = NULL;
+        char                key[256] = {0};
+        shd_pos_t           pos_data = {0};
+        int                 op_ret = -1;
+        int                 xl_id = -1;
+        int                 i = 0;
+        int                 ret = 0;
+        int                 crawl_flags = 0;
+
+        priv = this->private;
+        if (op == HEAL)
+                crawl_flags |= STOP_CRAWL_ON_SINGLE_SUBVOL;
+
+        if (output) {
+                ret = dict_get_int32 (output, this->name, &xl_id);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid input, "
+                                "translator-id is not available");
+                        goto out;
+                }
+        }
+        pos_data.this = this;
+        subkey = "status";
+        for (i = 0; i < priv->child_count; i++) {
+                if (_crawl_proceed (this, i, crawl_flags, &status)) {
+                        pos_data.child = i;
+                        /*
+                         * We're already in a synctask in this case, so we
+                         * don't need to defer through a second (and in fact
+                         * that can cause deadlock).  Just call straight
+                         * through instead.
+                         */
+                        ret = afr_find_child_position(pos_data.this,
+                                                      pos_data.child,
+                                                      &pos_data.pos);
+                        if (ret) {
+                                status = "Not able to find brick location";
+                        } else if (pos_data.pos == AFR_POS_REMOTE) {
+                                status = "brick is remote";
+                        } else {
+                                op_ret = 0;
+                                if (op == HEAL) {
+                                        status = "Started self-heal";
+                                        _do_self_heal_on_subvol (this, i,
+                                                                 crawl);
+                                } else if (output && (op == INFO)) {
+                                        status = "";
+                                        afr_start_crawl (this, i, INDEX,
+                                                         _add_summary_to_dict,
+                                                         output, _gf_false, 0,
+                                                         NULL);
+                                } else if (output &&
+                                           (op == STATISTICS_TO_BE_HEALED)) {
+                                            status = "";
+                                            afr_start_crawl (this, i,
+                                                             INDEX_TO_BE_HEALED,
+                                       _count_hard_links_under_base_indices_dir,
+                                                             output, _gf_false,
+                                                             0, NULL);
+                                }
+                        }
+                        if (output) {
+                                snprintf (key, sizeof (key), "%d-%d-%s", xl_id,
+                                          i, subkey);
+                                ret = dict_set_str (output, key, status);
+                        }
+                        if (!op_ret && (crawl == FULL))
+                                break;
+                }
+                if (output) {
+                        snprintf (key, sizeof (key), "%d-%d-%s", xl_id, i,
+                                  subkey);
+                        ret = dict_set_str (output, key, status);
+                }
+        }
+out:
+        return op_ret;
+}
+
+int
+_do_self_heal_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
+                                dict_t *output)
+{
+        return _do_crawl_op_on_local_subvols (this, crawl, HEAL, output);
+}
+
+int
+_get_index_summary_on_local_subvols (xlator_t *this, dict_t *output)
+{
+        return _do_crawl_op_on_local_subvols (this, INDEX, INFO, output);
+}
+
+void
+afr_fill_completed_crawl_statistics_to_dict (xlator_t *this, dict_t *dict)
+{
+        afr_private_t           *priv  = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     i = 0;
+        priv = this->private;
+        shd= &priv->shd;
+        for (i = 0; i < priv->child_count; i++) {
+                if (shd->pos[i] != AFR_POS_LOCAL)
+                        continue;
+                _add_statistics_to_dict (this, dict, i);
+        }
+
+        return ;
+}
+
+static void
+reset_crawl_event (shd_crawl_event_t *crawl_event)
+{
+    crawl_event->healed_count = 0;
+    crawl_event->split_brain_count = 0;
+    crawl_event->heal_failed_count = 0;
+    GF_FREE (crawl_event->start_time_str);
+    crawl_event->start_time_str = NULL;
+    crawl_event->end_time_str = NULL;
+    crawl_event->crawl_type = NULL;
+    crawl_event->crawl_inprogress = _gf_false;
+    return;
+}
+
+static void
+afr_copy_crawl_event_struct (shd_crawl_event_t *src, shd_crawl_event_t *dst)
+{
+        dst->healed_count = src->healed_count;
+        dst->split_brain_count = src->split_brain_count;
+        dst->heal_failed_count = src->heal_failed_count;
+        dst->start_time_str = gf_strdup (src->start_time_str);
+        dst->end_time_str = "Crawl is already in progress";
+        dst->crawl_type = src->crawl_type;
+        dst->crawl_inprogress = _gf_true;
+        return;
+}
+
+static int
+afr_fill_crawl_statistics_of_running_crawl(xlator_t *this, dict_t *dict)
+{
+        shd_crawl_event_t       *evnt = NULL;
+        int                     ret = 0;
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     i = 0;
+        priv = this->private;
+        shd = &priv->shd;
+
+        evnt = GF_CALLOC (1, sizeof (shd_crawl_event_t),
+                          gf_afr_mt_shd_crawl_event_t);
+        if (!evnt) {
+                ret = -1;
+                goto out;
+        }
+        LOCK (&priv->lock);
+        {
+                for (i = 0; i < priv->child_count; i++) {
+                        if (shd->pos[i] != AFR_POS_LOCAL)
+                                continue;
+
+                        reset_crawl_event (evnt);
+
+                        if (!shd->crawl_events[i]) {
+                                continue;
+                        }
+
+                        afr_copy_crawl_event_struct (shd->crawl_events[i],
+                                                     evnt);
+                        _add_crawl_stats_to_dict (this, dict, i, evnt, NULL);
+
+                }
+        }
+        UNLOCK (&priv->lock);
+        reset_crawl_event (evnt);
+        GF_FREE (evnt);
+
+out:
+        return ret;
+}
+
+static int
+_add_local_subvols_crawl_statistics_to_dict (xlator_t *this, dict_t *dict)
+{
+        int ret = 0;
+        afr_fill_completed_crawl_statistics_to_dict (this, dict);
+        ret = afr_fill_crawl_statistics_of_running_crawl (this, dict);
+        return ret;
+}
+int
+_add_local_subvols_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     i = 0;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (shd->pos[i] != AFR_POS_LOCAL)
+                        continue;
+                _add_eh_to_dict (this, eh, dict, i);
+        }
+        return 0;
+}
+
+int
+afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+{
+        gf_xl_afr_op_t   op = GF_AFR_OP_INVALID;
+        int              ret = 0;
+        afr_private_t    *priv = NULL;
+        afr_self_heald_t *shd = NULL;
+        int              xl_id = 0;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (input, this->name, &xl_id);
+        if (ret)
+                goto out;
+        ret = dict_set_int32 (output, this->name, xl_id);
+        if (ret)
+                goto out;
+        switch (op) {
+        case GF_AFR_OP_HEAL_INDEX:
+                ret = _do_self_heal_on_local_subvols (this, INDEX, output);
+                break;
+        case GF_AFR_OP_HEAL_FULL:
+                ret = _do_self_heal_on_local_subvols (this, FULL, output);
+                break;
+        case GF_AFR_OP_INDEX_SUMMARY:
+                (void)_get_index_summary_on_local_subvols (this, output);
+                ret = 0;
+                break;
+        case GF_AFR_OP_HEALED_FILES:
+                ret = _add_local_subvols_eh_to_dict (this, shd->healed, output);
+                break;
+        case GF_AFR_OP_HEAL_FAILED_FILES:
+                ret = _add_local_subvols_eh_to_dict (this, shd->heal_failed,
+                                                   output);
+                break;
+        case GF_AFR_OP_SPLIT_BRAIN_FILES:
+                ret = _add_local_subvols_eh_to_dict (this, shd->split_brain,
+                                                   output);
+                break;
+        case GF_AFR_OP_STATISTICS:
+                ret = _add_local_subvols_crawl_statistics_to_dict (this, output);
+                break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                ret = _do_crawl_op_on_local_subvols (this, INDEX_TO_BE_HEALED,
+                                                     STATISTICS_TO_BE_HEALED,
+                                                     output);
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
+                break;
+        }
+out:
+        dict_del (output, this->name);
+        return ret;
+}
+
+void
+afr_poll_self_heal (void *data)
+{
+        afr_private_t    *priv = NULL;
+        afr_self_heald_t *shd = NULL;
+        struct timespec  timeout = {0};
+        xlator_t         *this = NULL;
+        long             child = (long)data;
+        gf_timer_t       *old_timer = NULL;
+        gf_timer_t       *new_timer = NULL;
+        shd_pos_t        pos_data = {0};
+        int              ret = 0;
+
+        this = THIS;
+        priv = this->private;
+        shd = &priv->shd;
+
+        if (shd->pos[child] == AFR_POS_UNKNOWN) {
+                pos_data.this = this;
+                pos_data.child = child;
+                ret = synctask_new (this->ctx->env,
+                                    afr_syncop_find_child_position,
+                                    NULL, NULL, &pos_data);
+                if (!ret)
+                        shd->pos[child] = pos_data.pos;
+        }
+        if (shd->enabled && (shd->pos[child] == AFR_POS_LOCAL))
+                _do_self_heal_on_subvol (this, child, INDEX);
+        timeout.tv_sec = shd->timeout;
+        timeout.tv_nsec = 0;
+        //notify and previous timer should be synchronized.
+        LOCK (&priv->lock);
+        {
+                old_timer = shd->timer[child];
+                if (shd->pos[child] == AFR_POS_REMOTE)
+                        goto unlock;
+                shd->timer[child] = gf_timer_call_after (this->ctx, timeout,
+                                                         afr_poll_self_heal,
+                                                         data);
+                new_timer = shd->timer[child];
+        }
+unlock:
+        UNLOCK (&priv->lock);
+
+        if (old_timer)
+                gf_timer_call_cancel (this->ctx, old_timer);
+        if (!new_timer && (shd->pos[child] != AFR_POS_REMOTE)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Could not create self-heal polling timer for %s",
+                        priv->children[child]->name);
+        }
+        return;
+}
+
+static int
+afr_handle_child_up  (int ret, call_frame_t *sync_frame, void *data)
+{
+        afr_self_heald_t *shd = NULL;
+        shd_pos_t        *pos_data = data;
+        afr_private_t    *priv = NULL;
+
+        if (ret)
+                goto out;
+
+        priv = pos_data->this->private;
+        shd = &priv->shd;
+        shd->pos[pos_data->child] = pos_data->pos;
+        if (pos_data->pos != AFR_POS_REMOTE)
+                afr_poll_self_heal ((void*)(long)pos_data->child);
+        _do_self_heal_on_local_subvols (THIS, INDEX, NULL);
+out:
+        GF_FREE (data);
+        return 0;
+}
+
+void
+afr_proactive_self_heal (void *data)
+{
+        xlator_t         *this = NULL;
+        long             child = (long)data;
+        shd_pos_t        *pos_data = NULL;
+        int              ret = 0;
+
+        this = THIS;
+
+        //Position of brick could have changed and it could be local now.
+        //Compute the position again
+        pos_data = GF_CALLOC (1, sizeof (*pos_data), gf_afr_mt_pos_data_t);
+        if (!pos_data)
+                goto out;
+        pos_data->this = this;
+        pos_data->child = child;
+        ret = synctask_new (this->ctx->env, afr_syncop_find_child_position,
+                            afr_handle_child_up, NULL, pos_data);
+        if (ret)
+                goto out;
+out:
+        return;
+}
+
 static int
 get_pathinfo_host (char *pathinfo, char *hostname, size_t size)
 {
@@ -84,30 +1142,206 @@ out:
         return ret;
 }
 
-inline void
-afr_fill_loc_info (loc_t *loc, struct iatt *iatt, struct iatt *parent)
+int
+afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
+                           loc_t *dirloc)
+{
+        afr_private_t *priv = NULL;
+        dict_t        *xattr = NULL;
+        void          *index_gfid = NULL;
+        void          *base_indices_holder_vgfid = NULL;
+        loc_t         rootloc = {0};
+        struct iatt   iattr = {0};
+        struct iatt   parent = {0};
+        int           ret = 0;
+        xlator_t      *readdir_xl = crawl_data->readdir_xl;
+
+        priv = this->private;
+        if (crawl_data->crawl == FULL) {
+                afr_build_root_loc (this, dirloc);
+        } else if (crawl_data->crawl == INDEX) {
+                afr_build_root_loc (this, &rootloc);
+                ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
+                                       GF_XATTROP_INDEX_GFID);
+                if (ret < 0)
+                        goto out;
+                ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to get index "
+                                "dir gfid on %s", readdir_xl->name);
+                        goto out;
+                }
+                if (!index_gfid) {
+                        gf_log (this->name, GF_LOG_ERROR, "index gfid empty "
+                                "on %s", readdir_xl->name);
+                        ret = -1;
+                        goto out;
+                }
+                uuid_copy (dirloc->gfid, index_gfid);
+                dirloc->path = "";
+                dirloc->inode = inode_new (priv->root_inode->table);
+                ret = syncop_lookup (readdir_xl, dirloc, NULL,
+                                     &iattr, NULL, &parent);
+                if (ret < 0) {
+                        if (errno != ENOENT) {
+                                gf_log (this->name, GF_LOG_ERROR, "lookup "
+                                        "failed on index dir on %s - (%s)",
+                                        readdir_xl->name, strerror (errno));
+                        }
+                        goto out;
+                }
+                ret = _link_inode_update_loc (this, dirloc, &iattr);
+                if (ret)
+                        goto out;
+        } else if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                afr_build_root_loc (this, &rootloc);
+                ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
+                                       GF_BASE_INDICES_HOLDER_GFID);
+                if (ret < 0)
+                        goto out;
+                ret = dict_get_ptr (xattr, GF_BASE_INDICES_HOLDER_GFID,
+                                    &base_indices_holder_vgfid);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "index gfid empty "
+                                "on %s", readdir_xl->name);
+                        ret = -1;
+                        goto out;
+                }
+                if (!base_indices_holder_vgfid) {
+                        gf_log (this->name, GF_LOG_ERROR, "Base indices holder"
+                                "virtual gfid is null on %s", readdir_xl->name);
+                        ret = -1;
+                        goto out;
+                }
+                uuid_copy (dirloc->gfid,  base_indices_holder_vgfid);
+                dirloc->path = "";
+                dirloc->inode = inode_new (priv->root_inode->table);
+                ret = syncop_lookup (readdir_xl, dirloc, NULL, &iattr, NULL,
+                                     &parent);
+                if (ret < 0) {
+                        if (errno != ENOENT) {
+                                gf_log (this->name, GF_LOG_ERROR, "lookup "
+                                        "failed for base_indices_holder dir"
+                                        " on %s - (%s)", readdir_xl->name,
+                                        strerror (errno));
+
+                        } else {
+                                gf_log (this->name, GF_LOG_ERROR, "base_indices"
+                                        "_holder is not yet created.");
+                        }
+                        goto out;
+                }
+                ret = _link_inode_update_loc (this, dirloc, &iattr);
+                if (ret)
+                        goto out;
+        }
+        ret = 0;
+out:
+        if (xattr)
+                dict_unref (xattr);
+        loc_wipe (&rootloc);
+        return ret;
+}
+
+int
+afr_crawl_opendir (xlator_t *this, afr_crawl_data_t *crawl_data, fd_t **dirfd,
+                   loc_t *dirloc)
 {
-        afr_update_loc_gfids (loc, iatt, parent);
-        uuid_copy (loc->inode->gfid, iatt->ia_gfid);
+        fd_t          *fd   = NULL;
+        int           ret = 0;
+
+        if (crawl_data->crawl == FULL) {
+                fd = fd_create (dirloc->inode, crawl_data->pid);
+                if (!fd) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to create fd for %s", dirloc->path);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = syncop_opendir (crawl_data->readdir_xl, dirloc, fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "opendir failed on %s", dirloc->path);
+                        goto out;
+                }
+        } else {
+                fd = fd_anonymous (dirloc->inode);
+        }
+        ret = 0;
+out:
+        if (!ret)
+                *dirfd = fd;
+        return ret;
+}
+
+xlator_t*
+afr_crawl_readdir_xl_get (xlator_t *this, afr_crawl_data_t *crawl_data)
+{
+        afr_private_t *priv = this->private;
+
+        if (crawl_data->crawl == FULL) {
+                return this;
+        } else {
+                return priv->children[crawl_data->child];
+        }
+        return NULL;
+}
+
+int
+afr_crawl_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent,
+                           gf_dirent_t *entry, afr_crawl_data_t *crawl_data)
+{
+        int           ret = -1;
+        afr_private_t *priv = NULL;
+
+        priv = this->private;
+        if (crawl_data->crawl == FULL) {
+                ret = afr_build_child_loc (this, child, parent, entry->d_name);
+        } else if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                ret = _build_index_loc (this, child, entry->d_name, parent);
+                if (ret)
+                        goto out;
+                child->inode = inode_new (priv->root_inode->table);
+                if (!child->inode) {
+                        ret = -1;
+                        goto out;
+                }
+                child->path = NULL;
+        } else {
+                child->inode = inode_new (priv->root_inode->table);
+                if (!child->inode)
+                        goto out;
+                uuid_parse (entry->d_name, child->gfid);
+                ret = _loc_assign_gfid_path (child);
+        }
+out:
+        return ret;
 }
 
 static int
-_perform_self_heal (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
-                    off_t *offset, pid_t pid)
+_process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
+                  off_t *offset, afr_crawl_data_t *crawl_data)
 {
         gf_dirent_t      *entry = NULL;
         gf_dirent_t      *tmp = NULL;
-        struct iatt      iatt = {0};
-        struct iatt      parent = {0};;
         int              ret = 0;
         loc_t            entry_loc = {0};
+        fd_t             *fd = NULL;
+        struct iatt      iattr = {0};
 
         list_for_each_entry_safe (entry, tmp, &entries->list, list) {
+                if (!_crawl_proceed (this, crawl_data->child,
+                                     crawl_data->crawl_flags, NULL)) {
+                        ret = -1;
+                        goto out;
+                }
                 *offset = entry->d_off;
                 if (IS_ENTRY_CWD (entry->d_name) ||
                     IS_ENTRY_PARENT (entry->d_name))
                         continue;
-                if (uuid_is_null (entry->d_stat.ia_gfid)) {
+                if ((crawl_data->crawl == FULL) &&
+                     uuid_is_null (entry->d_stat.ia_gfid)) {
                         gf_log (this->name, GF_LOG_WARNING, "%s/%s: No "
                                 "gfid present skipping",
                                 parentloc->path, entry->d_name);
@@ -115,88 +1349,93 @@ _perform_self_heal (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
                 }
 
                 loc_wipe (&entry_loc);
-                ret = afr_build_child_loc (this, &entry_loc, parentloc,
-                                           entry->d_name, entry->d_stat.ia_gfid);
+                ret = afr_crawl_build_child_loc (this, &entry_loc, parentloc,
+                                                 entry, crawl_data);
                 if (ret)
                         goto out;
 
-                gf_log (this->name, GF_LOG_DEBUG, "lookup %s", entry_loc.path);
+                ret = crawl_data->process_entry (this, crawl_data, entry,
+                                                 &entry_loc, parentloc, &iattr);
 
-                ret = syncop_lookup (this, &entry_loc, NULL,
-                                     &iatt, NULL, &parent);
-                //Don't fail the crawl if lookup fails as it
-                //could be because of split-brain
-                if (ret || (!IA_ISDIR (iatt.ia_type)))
+                if (crawl_data->crawl == INDEX_TO_BE_HEALED && ret) {
+                       goto out;
+                } else if (ret) {
                         continue;
-                afr_fill_loc_info (&entry_loc, &iatt, &parent);
-                ret = _crawl_directory (&entry_loc, pid);
+                }
+
+                if ((crawl_data->crawl == INDEX) ||
+                    (crawl_data->crawl == INDEX_TO_BE_HEALED))
+                        continue;
+
+                if (!IA_ISDIR (iattr.ia_type))
+                        continue;
+                fd = NULL;
+                ret = afr_crawl_opendir (this, crawl_data, &fd, &entry_loc);
+                if (ret)
+                        continue;
+                ret = _crawl_directory (fd, &entry_loc, crawl_data);
+                if (fd)
+                        fd_unref (fd);
         }
         ret = 0;
 out:
-        if (entry_loc.path)
-                loc_wipe (&entry_loc);
+        if ((crawl_data->crawl == INDEX_TO_BE_HEALED)  && ret) {
+                gf_log (this->name, GF_LOG_ERROR,"Failed to get the hardlink "
+                        "count");
+        }
+        loc_wipe (&entry_loc);
         return ret;
 }
 
 static int
-_crawl_directory (loc_t *loc, pid_t pid)
+_crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data)
 {
         xlator_t        *this = NULL;
-        afr_private_t   *priv = NULL;
-        fd_t            *fd   = NULL;
         off_t           offset   = 0;
         gf_dirent_t     entries;
-        struct iatt     iatt = {0};
-        struct iatt     parent = {0};;
         int             ret = 0;
         gf_boolean_t    free_entries = _gf_false;
+        xlator_t        *readdir_xl = crawl_data->readdir_xl;
 
         INIT_LIST_HEAD (&entries.list);
         this = THIS;
-        priv = this->private;
 
         GF_ASSERT (loc->inode);
 
-        gf_log (this->name, GF_LOG_DEBUG, "crawling %s", loc->path);
-        fd = fd_create (loc->inode, pid);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create fd for %s", loc->path);
-                goto out;
-        }
-
-        if (!loc->parent) {
-                ret = syncop_lookup (this, loc, NULL,
-                                     &iatt, NULL, &parent);
-        }
-
-        ret = syncop_opendir (this, loc, fd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "opendir failed on %s", loc->path);
-                goto out;
-        }
+        if (crawl_data->crawl == FULL)
+                gf_log (this->name, GF_LOG_DEBUG, "crawling %s", loc->path);
+        else
+                gf_log (this->name, GF_LOG_DEBUG, "crawling INDEX %s",
+                        uuid_utoa (loc->gfid));
 
-        while (syncop_readdirp (this, fd, 131072, offset, NULL, &entries)) {
+        while (1) {
+                if (crawl_data->crawl == FULL)
+                        ret = syncop_readdirp (readdir_xl, fd, 131072, offset,
+                                               NULL, &entries);
+                else
+                        ret = syncop_readdir (readdir_xl, fd, 131072, offset,
+                                              &entries);
+                if (ret <= 0)
+                        break;
                 ret = 0;
                 free_entries = _gf_true;
-                if (afr_up_children_count (priv->child_up,
-                                           priv->child_count) < 2) {
-                        gf_log (this->name, GF_LOG_ERROR, "Stopping crawl as "
-                                "< 2 children are up");
+
+                if (!_crawl_proceed (this, crawl_data->child,
+                                     crawl_data->crawl_flags, NULL)) {
                         ret = -1;
                         goto out;
                 }
-
                 if (list_empty (&entries.list))
                         goto out;
 
-                ret = _perform_self_heal (this, loc, &entries, &offset, pid);
+                ret = _process_entries (this, loc, &entries, &offset,
+                                        crawl_data);
+                if ((ret < 0) && (crawl_data->crawl == INDEX_TO_BE_HEALED)) {
+                        goto out;
+                }
                 gf_dirent_free (&entries);
                 free_entries = _gf_false;
         }
-        if (fd)
-                fd_unref (fd);
         ret = 0;
 out:
         if (free_entries)
@@ -204,279 +1443,332 @@ out:
         return ret;
 }
 
+static char*
+position_str_get (afr_child_pos_t pos)
+{
+        switch (pos) {
+        case AFR_POS_UNKNOWN:
+                return "unknown";
+        case AFR_POS_LOCAL:
+                return "local";
+        case AFR_POS_REMOTE:
+                return "remote";
+        }
+        return NULL;
+}
+
 int
-afr_find_child_position (xlator_t *this, int child)
+afr_find_child_position (xlator_t *this, int child, afr_child_pos_t *pos)
 {
         afr_private_t    *priv = NULL;
+        afr_self_heald_t *shd  = NULL;
         dict_t           *xattr_rsp = NULL;
         loc_t            loc = {0};
         int              ret = 0;
-        gf_boolean_t     local = _gf_false;
-        char             *pathinfo = NULL;
-        afr_child_pos_t  *pos = NULL;
-        inode_table_t    *itable = NULL;
+        char             *node_uuid = NULL;
 
         priv = this->private;
-        pos = &priv->shd.pos[child];
+        shd  = &priv->shd;
 
-        if (*pos != AFR_POS_UNKNOWN) {
-                goto out;
-        }
-
-        //TODO: Hack to make the root_loc hack work
-        LOCK (&priv->lock);
-        {
-                if (!priv->root_inode) {
-                        itable = inode_table_new (0, this);
-                        if (!itable)
-                                goto unlock;
-                        priv->root_inode = inode_new (itable);
-                        if (!priv->root_inode)
-                                goto unlock;
-                }
-        }
-unlock:
-        UNLOCK (&priv->lock);
-
-        if (!priv->root_inode) {
-                ret = -1;
-                goto out;
-        }
-        afr_build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
 
         ret = syncop_getxattr (priv->children[child], &loc, &xattr_rsp,
-                               GF_XATTR_PATHINFO_KEY);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "getxattr failed on child "
-                        "%d", child);
+                               GF_XATTR_NODE_UUID_KEY);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "getxattr failed on %s - "
+                        "(%s)", priv->children[child]->name, strerror (errno));
                 goto out;
         }
 
-        ret = dict_get_str (xattr_rsp, GF_XATTR_PATHINFO_KEY, &pathinfo);
+        ret = dict_get_str (xattr_rsp, GF_XATTR_NODE_UUID_KEY, &node_uuid);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Pathinfo key not found on "
-                        "child %d", child);
+                gf_log (this->name, GF_LOG_ERROR, "node-uuid key not found on "
+                        "child %s", priv->children[child]->name);
                 goto out;
         }
 
-        ret = afr_local_pathinfo (pathinfo, &local);
-        if (ret)
-                goto out;
-        if (local)
+        if (!strcmp (node_uuid, shd->node_uuid))
                 *pos = AFR_POS_LOCAL;
         else
                 *pos = AFR_POS_REMOTE;
 
-        gf_log (this->name, GF_LOG_INFO, "child %d is %d", child, *pos);
+        gf_log (this->name, GF_LOG_DEBUG, "child %s is %s",
+                priv->children[child]->name, position_str_get (*pos));
 out:
+        if (ret)
+                *pos = AFR_POS_UNKNOWN;
+        loc_wipe (&loc);
         return ret;
 }
 
-static int
-afr_crawl_done  (int ret, call_frame_t *sync_frame, void *data)
+int
+afr_syncop_find_child_position (void *data)
 {
-        GF_FREE (data);
-        STACK_DESTROY (sync_frame->root);
-        return 0;
+        shd_pos_t *pos_data = data;
+        int       ret = 0;
+
+        ret = afr_find_child_position (pos_data->this, pos_data->child,
+                                       &pos_data->pos);
+        return ret;
 }
 
 static int
-afr_find_all_children_postions (xlator_t *this)
+afr_dir_crawl (void *data)
 {
-        int              ret = -1;
-        int              i = 0;
-        gf_boolean_t     succeeded = _gf_false;
-        afr_private_t    *priv = NULL;
+        xlator_t            *this = NULL;
+        int                 ret = -1;
+        xlator_t            *readdir_xl = NULL;
+        fd_t                *fd = NULL;
+        loc_t               dirloc = {0};
+        afr_crawl_data_t    *crawl_data = data;
 
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++) {
-                if (priv->child_up[i] != 1)
-                        continue;
-                ret = afr_find_child_position (this, i);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to determine if the "
-                                "child %s is local.",
-                                priv->children[i]->name);
-                        continue;
+        this = THIS;
+
+        if (!_crawl_proceed (this, crawl_data->child, crawl_data->crawl_flags,
+                             NULL))
+                goto out;
+
+        readdir_xl = afr_crawl_readdir_xl_get (this, crawl_data);
+        if (!readdir_xl)
+                goto out;
+        crawl_data->readdir_xl = readdir_xl;
+
+        ret = afr_crawl_build_start_loc (this, crawl_data, &dirloc);
+        if (ret)
+                goto out;
+
+        ret = afr_crawl_opendir (this, crawl_data, &fd, &dirloc);
+        if (ret) {
+                if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to open base_"
+                                "indices_holder");
                 }
-                succeeded = _gf_true;
+                goto out;
         }
-        if (succeeded)
-                ret = 0;
+
+        ret = _crawl_directory (fd, &dirloc, crawl_data);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Crawl failed on %s",
+                        readdir_xl->name);
+        else
+                gf_log (this->name, GF_LOG_DEBUG, "Crawl completed "
+                        "on %s", readdir_xl->name);
+        if (crawl_data->crawl == INDEX)
+                dirloc.path = NULL;
+out:
+        if (fd)
+                fd_unref (fd);
+        if ((crawl_data->crawl == INDEX) ||
+            (crawl_data->crawl == INDEX_TO_BE_HEALED ))
+                dirloc.path = NULL;
+        loc_wipe (&dirloc);
         return ret;
 }
 
-static gf_boolean_t
-afr_local_child_exists (afr_child_pos_t *pos, unsigned int child_count)
+char *
+get_crawl_type_in_string (afr_crawl_type_t crawl)
 {
-        int             i = 0;
-        gf_boolean_t    local = _gf_false;
-
-        for (i = 0; i < child_count; i++, pos++) {
-                if (*pos == AFR_POS_LOCAL) {
-                        local = _gf_true;
-                        break;
-                }
+        char    *index = "INDEX";
+        char    *full  = "FULL";
+        char    *crawl_type = NULL;
+
+        if (crawl == INDEX){
+                crawl_type = index;
+        } else if (crawl == FULL) {
+                crawl_type = full;
         }
-        return local;
+
+        return  crawl_type;
 }
 
-int
-afr_init_child_position (xlator_t *this, int child)
+static int
+afr_allocate_crawl_event (xlator_t *this, int child, afr_crawl_type_t crawl)
 {
-        int     ret = 0;
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     ret = 0;
+        shd_crawl_event_t       *crawl_event = NULL;
+        time_t                  get_time = 0;
 
-        if (child == AFR_ALL_CHILDREN) {
-                ret = afr_find_all_children_postions (this);
-        } else {
-                ret = afr_find_child_position (this, child);
+        priv = this->private;
+        shd = &priv->shd;
+
+        crawl_event = GF_CALLOC (sizeof (shd_crawl_event_t), 1,
+                                 gf_afr_mt_shd_crawl_event_t);
+        if (!crawl_event) {
+                ret = -1;
+                goto out;
+        }
+
+        get_time =  time(NULL);
+        if (get_time == ((time_t)-1)) {
+                 ret = -1;
+                goto out;
         }
+
+        crawl_event->start_time_str = gf_strdup (ctime(&get_time));
+
+        crawl_event->crawl_type = get_crawl_type_in_string (crawl);
+        if (!crawl_event->crawl_type) {
+                ret = -1;
+                goto out;
+        }
+        LOCK (&priv->lock);
+        {
+                shd->crawl_events[child] = crawl_event;
+        }
+        UNLOCK (&priv->lock);
+        ret = 0;
+out:
         return ret;
+
 }
 
-int
-afr_is_local_child (afr_self_heald_t *shd, int child, unsigned int child_count)
+static int
+afr_put_crawl_event_in_eh (xlator_t *this, int child)
 {
-        gf_boolean_t local = _gf_false;
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     ret = 0;
+        time_t                  get_time = 0;
+        shd_crawl_event_t       **crawl_event = NULL;
 
-        if (child == AFR_ALL_CHILDREN)
-                local = afr_local_child_exists (shd->pos, child_count);
-        else
-                local = (shd->pos[child] == AFR_POS_LOCAL);
+        priv = this->private;
+        shd = &priv->shd;
 
-        return local;
+        get_time = time(NULL);
+        if (get_time == ((time_t)-1)) {
+                ret = -1;
+                goto out;
+        }
+        crawl_event = (shd_crawl_event_t**)shd->crawl_events;
+        LOCK (&priv->lock);
+        {
+                crawl_event[child]->end_time_str = gf_strdup (ctime(&get_time));
+                ret = eh_save_history (shd->statistics[child],
+                                       crawl_event[child]);
+                crawl_event[child] = NULL;
+        }
+        UNLOCK (&priv->lock);
+out:
+        return ret;
 }
 
 static int
-afr_crawl_directory (xlator_t *this, pid_t pid)
+afr_dir_exclusive_crawl (void *data)
 {
         afr_private_t    *priv = NULL;
         afr_self_heald_t *shd = NULL;
-        loc_t            loc = {0};
         gf_boolean_t     crawl = _gf_false;
-        int             ret = 0;
+        int              ret = 0;
+        int              child = -1;
+        xlator_t         *this = NULL;
+        afr_crawl_data_t *crawl_data = data;
 
+        this = THIS;
         priv = this->private;
         shd = &priv->shd;
-
+        child = crawl_data->child;
 
         LOCK (&priv->lock);
         {
-                if (shd->inprogress) {
-                        shd->pending = _gf_true;
+                if (shd->inprogress[child]) {
+                        if (shd->pending[child] != FULL)
+                                shd->pending[child] = crawl_data->crawl;
                 } else {
-                        shd->inprogress = _gf_true;
+                        shd->inprogress[child] = _gf_true;
                         crawl = _gf_true;
                 }
         }
         UNLOCK (&priv->lock);
 
-        if (!priv->root_inode) {
-                ret = -1;
+        if (!crawl) {
+                gf_log (this->name, GF_LOG_INFO, "Another crawl is in progress "
+                        "for %s", priv->children[child]->name);
                 goto out;
         }
 
-        if (!crawl)
-                goto out;
-
-        afr_build_root_loc (priv->root_inode, &loc);
-        while (crawl) {
-                ret = _crawl_directory (&loc, pid);
+        do {
+                ret = afr_allocate_crawl_event (this, child, crawl_data->crawl);
                 if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Crawl failed");
-                else
-                        gf_log (this->name, GF_LOG_INFO, "Crawl completed");
+                        goto out;
+                afr_dir_crawl (data);
+
+                ret = afr_put_crawl_event_in_eh (this, child);
+                if (ret < 0)
+                        goto out;
+
                 LOCK (&priv->lock);
                 {
-                        if (shd->pending) {
-                                shd->pending = _gf_false;
+                        if (shd->pending[child] != NONE) {
+                                crawl_data->crawl = shd->pending[child];
+                                shd->pending[child] = NONE;
                         } else {
-                                shd->inprogress = _gf_false;
+                                shd->inprogress[child] = _gf_false;
                                 crawl = _gf_false;
                         }
                 }
                 UNLOCK (&priv->lock);
-        }
-out:
-        return ret;
-}
-
-static int
-afr_crawl (void *data)
-{
-        xlator_t         *this = NULL;
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        int              ret = -1;
-        afr_crawl_data_t *crawl_data = data;
-
-        this = THIS;
-        priv = this->private;
-        shd = &priv->shd;
-
-        ret = afr_init_child_position (this, crawl_data->child);
-        if (ret)
-                goto out;
-
-        if (!afr_is_local_child (shd, crawl_data->child, priv->child_count))
-                goto out;
-
-        ret = afr_crawl_directory (this, crawl_data->pid);
+        } while (crawl);
 out:
         return ret;
 }
 
 void
-afr_proactive_self_heal (xlator_t *this, int idx)
+afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
+                 process_entry_cbk_t process_entry, void *op_data,
+                 gf_boolean_t exclusive, int crawl_flags,
+                 afr_crawl_done_cbk_t crawl_done)
 {
         afr_private_t              *priv = NULL;
-        afr_self_heald_t           *shd = NULL;
         call_frame_t               *frame = NULL;
         afr_crawl_data_t           *crawl_data = NULL;
         int                        ret = 0;
+        int (*crawler) (void*) = NULL;
 
         priv = this->private;
-        shd = &priv->shd;
-        if (!shd->enabled)
-                goto out;
-
-        if ((idx != AFR_ALL_CHILDREN) &&
-            (shd->pos[idx] == AFR_POS_REMOTE))
-                goto out;
 
         frame = create_frame (this, this->ctx->pool);
         if (!frame)
                 goto out;
 
-        afr_set_lk_owner (frame, this);
+        afr_set_lk_owner (frame, this, frame->root);
         afr_set_low_priority (frame);
         crawl_data = GF_CALLOC (1, sizeof (*crawl_data),
-                                gf_afr_mt_afr_crawl_data_t);
+                                gf_afr_mt_crawl_data_t);
         if (!crawl_data)
                 goto out;
+        crawl_data->process_entry = process_entry;
         crawl_data->child = idx;
         crawl_data->pid = frame->root->pid;
-        gf_log (this->name, GF_LOG_INFO, "starting crawl for %d", idx);
-        ret = synctask_new (this->ctx->env, afr_crawl,
-                            afr_crawl_done, frame, crawl_data);
+        crawl_data->crawl = crawl;
+        crawl_data->op_data = op_data;
+        crawl_data->crawl_flags = crawl_flags;
+        gf_log (this->name, GF_LOG_DEBUG, "starting crawl %d for %s",
+                crawl_data->crawl, priv->children[idx]->name);
+
+        if (exclusive)
+                crawler = afr_dir_exclusive_crawl;
+        else
+                crawler = afr_dir_crawl;
+        ret = synctask_new (this->ctx->env, crawler,
+                            crawl_done, frame, crawl_data);
         if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Could not create the "
-                        "task for %d ret %d", idx, ret);
+                gf_log (this->name, GF_LOG_ERROR, "afr crawl failed for child"
+                        " %d with ret %d", idx, ret);
 out:
         return;
 }
 
-//TODO: This is a hack
 void
-afr_build_root_loc (inode_t *inode, loc_t *loc)
+afr_build_root_loc (xlator_t *this, loc_t *loc)
 {
-        loc->path = "/";
+        afr_private_t   *priv = NULL;
+
+        priv = this->private;
+        loc->path = gf_strdup ("/");
         loc->name = "";
-        loc->inode = inode;
-        loc->inode->ia_type = IA_IFDIR;
-        memset (loc->inode->gfid, 0, 16);
-        loc->inode->gfid[15] = 1;
+        loc->inode = inode_ref (priv->root_inode);
         uuid_copy (loc->gfid, loc->inode->gfid);
 }
 
@@ -493,4 +1785,3 @@ afr_set_root_gfid (dict_t *dict)
 
         return ret;
 }
-
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
index 6eb119b07..e0c083754 100644
--- a/xlators/cluster/afr/src/afr-self-heald.h
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __AFR_SELF_HEALD_H__
@@ -27,17 +18,48 @@
 #define AFR_ALL_CHILDREN -1
 
 typedef struct afr_crawl_data_ {
-        int     child;
-        pid_t   pid;
+        int                 child;
+        pid_t               pid;
+        afr_crawl_type_t    crawl;
+        xlator_t            *readdir_xl;
+        void                *op_data;
+        int                 crawl_flags;
+        int (*process_entry) (xlator_t *this, struct afr_crawl_data_ *crawl_data,
+                              gf_dirent_t *entry, loc_t *child, loc_t *parent,
+                              struct iatt *iattr);
 } afr_crawl_data_t;
 
-void afr_proactive_self_heal (xlator_t *this, int idx);
+typedef struct crawl_event_stats_ {
+        uint64_t healed_count;
+        uint64_t split_brain_count;
+        uint64_t heal_failed_count;
+        char     *start_time_str;
+        char     *end_time_str;
+        char     *crawl_type;
+        gf_boolean_t crawl_inprogress;
+} shd_crawl_event_t;
+
+void _destroy_crawl_event_data (void *data);
+void _destroy_shd_event_data (void *data);
+
+typedef int (*process_entry_cbk_t) (xlator_t *this, afr_crawl_data_t *crawl_data,
+                              gf_dirent_t *entry, loc_t *child, loc_t *parent,
+                              struct iatt *iattr);
 
-void afr_build_root_loc (inode_t *inode, loc_t *loc);
+void afr_build_root_loc (xlator_t *this, loc_t *loc);
 
 int afr_set_root_gfid (dict_t *dict);
 
-inline void
-afr_fill_loc_info (loc_t *loc, struct iatt *iatt, struct iatt *parent);
+void
+afr_proactive_self_heal (void *data);
 
+int
+afr_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+
+/*
+ * In addition to its self-heal use, this is used to find a local default
+ * read_child.
+ */
+int
+afr_local_pathinfo (char *pathinfo, gf_boolean_t *local);
 #endif /* __AFR_SELF_HEALD_H__ */
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index 36e2812f9..20306e469 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -1,25 +1,17 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "dict.h"
 #include "byte-order.h"
 #include "common-utils.h"
+#include "timer.h"
 
 #include "afr.h"
 #include "afr-transaction.h"
@@ -32,7 +24,6 @@
                                       of RENAME */
 #define LOCKED_LOWER    0x2        /* for lower_path of RENAME */
 
-
 afr_fd_ctx_t *
 __afr_fd_ctx_get (fd_t *fd, xlator_t *this)
 {
@@ -82,27 +73,26 @@ afr_fd_ctx_get (fd_t *fd, xlator_t *this)
 
 
 static void
-afr_pid_save (call_frame_t *frame)
+afr_save_lk_owner (call_frame_t *frame)
 {
         afr_local_t * local = NULL;
 
         local = frame->local;
 
-        local->saved_pid = frame->root->pid;
+        local->saved_lk_owner = frame->root->lk_owner;
 }
 
 
 static void
-afr_pid_restore (call_frame_t *frame)
+afr_restore_lk_owner (call_frame_t *frame)
 {
         afr_local_t * local = NULL;
 
         local = frame->local;
 
-        frame->root->pid = local->saved_pid;
+        frame->root->lk_owner = local->saved_lk_owner;
 }
 
-
 static void
 __mark_all_pending (int32_t *pending[], int child_count,
                     afr_transaction_type type)
@@ -155,51 +145,23 @@ out:
         return;
 }
 
-
-static void
-__mark_pre_op_undone_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
-{
-        afr_local_t   *local = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
-
-        local = frame->local;
-
-        if (!local->fd)
-                return;
-
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        if (!fd_ctx)
-                goto out;
-
-        LOCK (&local->fd->lock);
-        {
-                if (local->transaction.type == AFR_DATA_TRANSACTION)
-                        fd_ctx->pre_op_done[child_index]--;
-        }
-        UNLOCK (&local->fd->lock);
-out:
-        return;
-}
-
-
 static void
-__mark_down_children (int32_t *pending[], int child_count,
-                      unsigned char *child_up, afr_transaction_type type)
+__mark_non_participant_children (int32_t *pending[], int child_count,
+                                 unsigned char *participants,
+                                 afr_transaction_type type)
 {
         int i = 0;
         int j = 0;
 
+        j = afr_index_for_transaction_type (type);
         for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
-
-                if (!child_up[i])
+                if (!participants[i])
                         pending[i][j] = 0;
         }
 }
 
 
-static void
+void
 __mark_all_success (int32_t *pending[], int child_count,
                     afr_transaction_type type)
 {
@@ -212,6 +174,54 @@ __mark_all_success (int32_t *pending[], int child_count,
         }
 }
 
+void
+_set_all_child_errno (int *child_errno, unsigned int child_count)
+{
+        int     i = 0;
+
+        for (i = 0; i < child_count; i++)
+                if (child_errno[i] == 0)
+                        child_errno[i] = ENOTCONN;
+}
+
+void
+afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv = NULL;
+        fd_t            *fd   = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+        fd    = local->fd;
+
+        __mark_all_success (local->pending, priv->child_count,
+                            local->transaction.type);
+
+        _set_all_child_errno (local->child_errno, priv->child_count);
+
+        /*  Perform fops with the lk-owner from top xlator.
+         *  Eg: lk-owner of posix-lk and flush should be same,
+         *  flush cant clear the  posix-lks without that lk-owner.
+         */
+        afr_save_lk_owner (frame);
+        frame->root->lk_owner =
+                local->transaction.main_frame->root->lk_owner;
+
+
+        /* The wake up needs to happen independent of
+           what type of fop arrives here. If it was
+           a write, then it has already inherited the
+           lock and changelog. If it was not a write,
+           then the presumption of the optimization (of
+           optimizing for successive write operations)
+           fails.
+        */
+        if (fd)
+                afr_delayed_changelog_wake_up (this, fd);
+        local->transaction.fop (frame, this);
+}
+
 
 static int
 __changelog_enabled (afr_private_t *priv, afr_transaction_type type)
@@ -276,63 +286,41 @@ __fop_changelog_needed (call_frame_t *frame, xlator_t *this)
 }
 
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending)
+afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
+                      int child, afr_xattrop_type_t op)
 {
         int i = 0;
         int ret = 0;
 
+        if (op == LOCAL_FIRST) {
+                ret = dict_set_static_bin (xattr, priv->pending_key[child],
+                                           pending[child],
+                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                if (ret)
+                        goto out;
+        }
         for (i = 0; i < priv->child_count; i++) {
+                if (i == child)
+                        continue;
                 ret = dict_set_static_bin (xattr, priv->pending_key[i],
                                            pending[i],
-                                AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
                 /* 3 = data+metadata+entry */
 
                 if (ret < 0)
                         goto out;
         }
-
-out:
-        return ret;
-}
-
-
-static int
-afr_set_piggyback_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
-                        afr_transaction_type type)
-{
-        int i = 0;
-        int ret = 0;
-        int *arr = NULL;
-        int index = 0;
-        size_t pending_xattr_size = 3 * sizeof (int32_t);
-        /* 3 = data+metadata+entry */
-
-        index = afr_index_for_transaction_type (type);
-
-        for (i = 0; i < priv->child_count; i++) {
-                arr = GF_CALLOC (1, pending_xattr_size,
-                                 gf_afr_mt_char);
-                if (!arr) {
-                        ret = -1;
-                        goto out;
-                }
-
-                memcpy (arr, pending[i], pending_xattr_size);
-
-                arr[index] = hton32 (ntoh32(arr[index]) + 1);
-
-                ret = dict_set_bin (xattr, priv->pending_key[i],
-                                    arr, pending_xattr_size);
-
-                if (ret < 0)
+        if (op == LOCAL_LAST) {
+                ret = dict_set_static_bin (xattr, priv->pending_key[child],
+                                           pending[child],
+                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                if (ret)
                         goto out;
         }
-
 out:
         return ret;
 }
 
-
 int
 afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
 {
@@ -360,7 +348,8 @@ afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
 
 int32_t
 afr_changelog_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xattr)
+                           int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_private_t       *priv     = NULL;
@@ -378,6 +367,11 @@ afr_changelog_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&frame->lock);
 
         if (call_count == 0) {
+                if (local->transaction.resume_stub) {
+			call_resume (local->transaction.resume_stub);
+                        local->transaction.resume_stub = NULL;
+                }
+
                 if (afr_lock_server_count (priv, local->transaction.type) == 0) {
                         local->transaction.done (frame, this);
                 } else {
@@ -444,24 +438,41 @@ afr_transaction_rm_stale_children (call_frame_t *frame, xlator_t *this,
 
         afr_inode_rm_stale_children (this, inode, stale_children);
 out:
-        if (stale_children)
-                GF_FREE (stale_children);
+        GF_FREE (stale_children);
         return;
 }
 
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom)
+{
+        afr_inodelk_t *inodelk = NULL;
+        int           i = 0;
+
+        for (i = 0; int_lock->inodelk[i].domain; i++) {
+                inodelk = &int_lock->inodelk[i];
+                if (strcmp (dom, inodelk->domain) == 0)
+                        return inodelk;
+        }
+        return NULL;
+}
+
 unsigned char*
 afr_locked_nodes_get (afr_transaction_type type, afr_internal_lock_t *int_lock)
 {
         unsigned char *locked_nodes = NULL;
+        afr_inodelk_t *inodelk = NULL;
         switch (type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                locked_nodes = int_lock->inode_locked_nodes;
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                locked_nodes = inodelk->locked_nodes;
         break;
 
         case AFR_ENTRY_TRANSACTION:
         case AFR_ENTRY_RENAME_TRANSACTION:
-                locked_nodes = int_lock->entry_locked_nodes;
+                /*Because same set of subvols participate in all lockee
+                 * entities*/
+                locked_nodes = int_lock->lockee[0].locked_nodes;
         break;
         }
         return locked_nodes;
@@ -499,12 +510,141 @@ afr_changelog_post_op_call_count (afr_transaction_type type,
         return call_count;
 }
 
+void
+afr_compute_txn_changelog (afr_local_t *local, afr_private_t *priv)
+{
+        int             i = 0;
+        int             index = 0;
+        int32_t         postop = 0;
+        int32_t         preop = 1;
+        int32_t         **txn_changelog = NULL;
+
+        txn_changelog = local->transaction.txn_changelog;
+        index = afr_index_for_transaction_type (local->transaction.type);
+        for (i = 0; i < priv->child_count; i++) {
+                postop = ntoh32 (local->pending[i][index]);
+                txn_changelog[i][index] = hton32 (postop + preop);
+        }
+}
+
+afr_xattrop_type_t
+afr_get_postop_xattrop_type (int32_t **pending, int optimized, int child,
+                             afr_transaction_type type)
+{
+        int                     index = 0;
+        afr_xattrop_type_t      op = LOCAL_LAST;
+
+        index = afr_index_for_transaction_type (type);
+        if (optimized && !pending[child][index])
+                op = LOCAL_FIRST;
+        return op;
+}
+
+void
+afr_set_postop_dict (afr_local_t *local, xlator_t *this, dict_t *xattr,
+                     int optimized, int child)
+{
+        int32_t                 **txn_changelog = NULL;
+        int32_t                 **changelog = NULL;
+        afr_private_t           *priv = NULL;
+        int                     ret = 0;
+        afr_xattrop_type_t      op = LOCAL_LAST;
+
+        priv = this->private;
+        txn_changelog = local->transaction.txn_changelog;
+        op = afr_get_postop_xattrop_type (local->pending, optimized, child,
+                                          local->transaction.type);
+        if (optimized)
+                changelog = txn_changelog;
+        else
+                changelog = local->pending;
+        ret = afr_set_pending_dict (priv, xattr, changelog, child, op);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_INFO,
+                        "failed to set pending entry");
+}
+
+
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int index = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        index = afr_index_for_transaction_type (local->transaction.type);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->pending[i][index] == 0)
+                        return _gf_false;
+        }
+
+        return _gf_true;
+}
+
+static void
+afr_dir_fop_handle_all_fop_failures (call_frame_t *frame)
+{
+        xlator_t        *this = NULL;
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv = NULL;
+
+        this = frame->this;
+        local = frame->local;
+        priv = this->private;
+
+        if ((local->transaction.type != AFR_ENTRY_TRANSACTION) &&
+            (local->transaction.type != AFR_ENTRY_RENAME_TRANSACTION))
+                return;
+
+        if (local->op_ret >= 0)
+                goto out;
+
+        __mark_all_success (local->pending, priv->child_count,
+                            local->transaction.type);
+out:
+        return;
+}
+
+static void
+afr_data_handle_quota_errors (call_frame_t *frame, xlator_t *this)
+{
+        int     i = 0;
+        afr_private_t *priv = NULL;
+        afr_local_t   *local = NULL;
+        gf_boolean_t  all_quota_failures = _gf_false;
+
+        local = frame->local;
+        priv  = this->private;
+        if (local->transaction.type != AFR_DATA_TRANSACTION)
+                return;
+        /*
+         * Idea is to not leave the file in FOOL-FOOL scenario in case on
+         * all the bricks data transaction failed with EDQUOT to avoid
+         * increasing un-necessary load of self-heals in the system.
+         */
+        all_quota_failures = _gf_true;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i] &&
+                    (local->child_errno[i] != EDQUOT)) {
+                        all_quota_failures = _gf_false;
+                        break;
+                }
+        }
+        if (all_quota_failures)
+                __mark_all_success (local->pending, priv->child_count,
+                                    local->transaction.type);
+}
+
 int
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
 {
         afr_private_t * priv = this->private;
         afr_internal_lock_t *int_lock = NULL;
-        int ret        = 0;
         int i          = 0;
         int call_count = 0;
 
@@ -512,14 +652,17 @@ afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
         afr_fd_ctx_t  *fdctx = NULL;
         dict_t        **xattr = NULL;
         int            piggyback = 0;
-        int            index = 0;
         int            nothing_failed = 1;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
 
-        __mark_down_children (local->pending, priv->child_count,
-                              local->child_up, local->transaction.type);
+        __mark_non_participant_children (local->pending, priv->child_count,
+                                         local->transaction.pre_op,
+                                         local->transaction.type);
+
+        afr_data_handle_quota_errors (frame, this);
+        afr_dir_fop_handle_all_fop_failures (frame);
 
         if (local->fd)
                 afr_transaction_rm_stale_children (frame, this,
@@ -547,81 +690,65 @@ afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
                 goto out;
         }
 
-        /* check if something has failed, to handle piggybacking */
-        nothing_failed = 1;
-        index = afr_index_for_transaction_type (local->transaction.type);
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->pending[i][index] == 0) {
-                        nothing_failed = 0;
-                        break;
-                }
-        }
+        nothing_failed = afr_txn_nothing_failed (frame, this);
 
-        index = afr_index_for_transaction_type (local->transaction.type);
-        if (local->optimistic_change_log &&
-            local->transaction.type != AFR_DATA_TRANSACTION) {
-                /* if nothing_failed, then local->pending[..] == {0 .. 0} */
-                for (i = 0; i < priv->child_count; i++)
-                        local->pending[i][index]++;
-        }
+        afr_compute_txn_changelog (local , priv);
 
         for (i = 0; i < priv->child_count; i++) {
                 if (!local->transaction.pre_op[i])
                         continue;
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
-
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
-
 
+                if (local->transaction.type != AFR_DATA_TRANSACTION)
+                        afr_set_postop_dict (local, this, xattr[i],
+                                             local->optimistic_change_log, i);
                 switch (local->transaction.type) {
                 case AFR_DATA_TRANSACTION:
                 {
                         if (!fdctx) {
+                                afr_set_postop_dict (local, this, xattr[i],
+                                                     0, i);
                                 STACK_WIND (frame, afr_changelog_post_op_cbk,
                                             priv->children[i],
                                             priv->children[i]->fops->xattrop,
                                             &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
                                 break;
                         }
 
-                        LOCK (&local->fd->lock);
-                        {
-                                piggyback = 0;
-                                if (fdctx->pre_op_piggyback[i]) {
-                                        fdctx->pre_op_piggyback[i]--;
-                                        piggyback = 1;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
+                        /* local->transaction.postop_piggybacked[] was
+                           precomputed in is_piggyback_postop() when called from
+                           afr_changelog_post_op_safe()
+                        */
+
+                        piggyback = 0;
+                        if (local->transaction.postop_piggybacked[i])
+                                piggyback = 1;
 
-                        if (piggyback && !nothing_failed)
-                                ret = afr_set_piggyback_dict (priv, xattr[i],
-                                                              local->pending,
-                                                              local->transaction.type);
+                        afr_set_postop_dict (local, this, xattr[i],
+                                             piggyback, i);
 
                         if (nothing_failed && piggyback) {
                                 afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
+                                                           this, 1, 0, xattr[i], NULL);
                         } else {
-                                __mark_pre_op_undone_on_fd (frame, this, i);
                                 STACK_WIND_COOKIE (frame,
                                                    afr_changelog_post_op_cbk,
                                                    (void *) (long) i,
                                                    priv->children[i],
                                                    priv->children[i]->fops->fxattrop,
                                                    local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                         }
                 }
                 break;
                 case AFR_METADATA_TRANSACTION:
                 {
-                        if (nothing_failed) {
+                        if (nothing_failed && local->optimistic_change_log) {
                                 afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
+                                                           this, 1, 0, xattr[i],
+                                                           NULL);
                                 break;
                         }
 
@@ -630,28 +757,32 @@ afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
                                             priv->children[i],
                                             priv->children[i]->fops->fxattrop,
                                             local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
                         else
                                 STACK_WIND (frame, afr_changelog_post_op_cbk,
                                             priv->children[i],
                                             priv->children[i]->fops->xattrop,
                                             &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
                 }
                 break;
 
                 case AFR_ENTRY_RENAME_TRANSACTION:
                 {
-                        if (nothing_failed) {
+                        if (nothing_failed && local->optimistic_change_log) {
                                 afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
+                                                           this, 1, 0, xattr[i],
+                                                           NULL);
                         } else {
                                 STACK_WIND_COOKIE (frame, afr_changelog_post_op_cbk,
                                                    (void *) (long) i,
                                                    priv->children[i],
                                                    priv->children[i]->fops->xattrop,
                                                    &local->transaction.new_parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                         }
                         call_count--;
                 }
@@ -664,19 +795,17 @@ afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
                   value
                 */
 
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
-
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
+                afr_set_postop_dict (local, this, xattr[i],
+                                     local->optimistic_change_log, i);
 
                 /* fall through */
 
                 case AFR_ENTRY_TRANSACTION:
                 {
-                        if (nothing_failed) {
+                        if (nothing_failed && local->optimistic_change_log) {
                                 afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
+                                                           this, 1, 0, xattr[i],
+                                                           NULL);
                                 break;
                         }
 
@@ -685,13 +814,15 @@ afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
                                             priv->children[i],
                                             priv->children[i]->fops->fxattrop,
                                             local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
                         else
                                 STACK_WIND (frame, afr_changelog_post_op_cbk,
                                             priv->children[i],
                                             priv->children[i]->fops->xattrop,
                                             &local->transaction.parent_loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
                 }
                 break;
                 }
@@ -711,7 +842,8 @@ out:
 
 int32_t
 afr_changelog_pre_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xattr)
+                          int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                          dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = this->private;
@@ -756,12 +888,7 @@ afr_changelog_pre_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     (local->op_errno == ENOTSUP)) {
                         local->transaction.resume (frame, this);
                 } else {
-                        __mark_all_success (local->pending, priv->child_count,
-                                            local->transaction.type);
-
-                        afr_pid_restore (frame);
-
-                        local->transaction.fop (frame, this);
+                        afr_transaction_perform_fop (frame, this);
                 }
         }
 
@@ -814,7 +941,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
         for (i = 0; i < priv->child_count; i++) {
                 if (!locked_nodes[i])
                         continue;
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
+                ret = afr_set_pending_dict (priv, xattr[i], local->pending,
+                                            i, LOCAL_FIRST);
 
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_INFO,
@@ -831,7 +959,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->xattrop,
                                                    &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                                 break;
                         }
 
@@ -848,9 +977,12 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                         }
                         UNLOCK (&local->fd->lock);
 
+                        afr_set_delayed_post_op (frame, this);
+
                         if (piggyback)
                                 afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
                         else
                                 STACK_WIND_COOKIE (frame,
                                                    afr_changelog_pre_op_cbk,
@@ -858,14 +990,16 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->fxattrop,
                                                    local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                 }
                 break;
                 case AFR_METADATA_TRANSACTION:
                 {
                         if (local->optimistic_change_log) {
                                 afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
                                 break;
                         }
 
@@ -876,7 +1010,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->fxattrop,
                                                    local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                         else
                                 STACK_WIND_COOKIE (frame,
                                                    afr_changelog_pre_op_cbk,
@@ -884,7 +1019,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->xattrop,
                                                    &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                 }
                 break;
 
@@ -892,7 +1028,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                 {
                         if (local->optimistic_change_log) {
                                 afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
                         } else {
                                 STACK_WIND_COOKIE (frame,
                                                    afr_changelog_pre_op_cbk,
@@ -900,7 +1037,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->xattrop,
                                                    &local->transaction.new_parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                         }
 
                         call_count--;
@@ -915,7 +1053,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                   value
                 */
 
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
+                ret = afr_set_pending_dict (priv, xattr[i], local->pending,
+                                            i, LOCAL_FIRST);
 
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_INFO,
@@ -927,7 +1066,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                 {
                         if (local->optimistic_change_log) {
                                 afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
                                 break;
                         }
 
@@ -938,7 +1078,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->fxattrop,
                                                    local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                         else
                                 STACK_WIND_COOKIE (frame,
                                                    afr_changelog_pre_op_cbk,
@@ -946,7 +1087,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->xattrop,
                                                    &local->transaction.parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                 }
                 break;
                 }
@@ -1108,12 +1250,14 @@ int
 afr_set_transaction_flock (afr_local_t *local)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
 
         int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        int_lock->lk_flock.l_len   = local->transaction.len;
-        int_lock->lk_flock.l_start = local->transaction.start;
-        int_lock->lk_flock.l_type  = F_WRLCK;
+        inodelk->flock.l_len   = local->transaction.len;
+        inodelk->flock.l_start = local->transaction.start;
+        inodelk->flock.l_type  = F_WRLCK;
 
         return 0;
 }
@@ -1128,6 +1272,7 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
 
         int_lock->transaction_lk_type = AFR_TRANSACTION_LK;
+        int_lock->domain = this->name;
 
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
@@ -1141,8 +1286,8 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
 
         case AFR_ENTRY_RENAME_TRANSACTION:
 
-                int_lock->lock_cbk = afr_post_blocking_rename_cbk;
-                afr_blocking_lock (frame, this);
+                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
+                afr_nonblocking_entrylk (frame, this);
                 break;
 
         case AFR_ENTRY_TRANSACTION:
@@ -1164,12 +1309,6 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
 int
 afr_lock (call_frame_t *frame, xlator_t *this)
 {
-        afr_pid_save (frame);
-
-        frame->root->pid = (long) frame->root;
-
-        afr_set_lk_owner (frame, this);
-
         afr_set_lock_number (frame, this);
 
         return afr_lock_rec (frame, this);
@@ -1181,28 +1320,463 @@ afr_lock (call_frame_t *frame, xlator_t *this)
 int
 afr_internal_lock_finish (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
+        if (__fop_changelog_needed (frame, this)) {
+                afr_changelog_pre_op (frame, this);
+        } else {
+                afr_transaction_perform_fop (frame, this);
+        }
+
+        return 0;
+}
+
+
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+        /* call this function from any of the related optimizations
+           which benefit from delaying post op are enabled, namely:
+
+           - changelog piggybacking
+           - eager locking
+        */
+
+        priv = this->private;
+        if (!priv)
+                return;
+
+        if (!priv->post_op_delay_secs)
+                return;
+
+        local = frame->local;
+        if (!local->transaction.eager_lock_on)
+                return;
+
+        if (!local)
+                return;
+
+        if (!local->fd)
+                return;
+
+        if (local->op == GF_FOP_WRITE)
+                local->delayed_post_op = _gf_true;
+}
+
+gf_boolean_t
+afr_are_multiple_fds_opened (inode_t *inode, xlator_t *this)
+{
+        afr_inode_ctx_t *ictx = NULL;
+
+        if (!inode) {
+                /* If false is returned, it may keep on taking eager-lock
+                 * which may lead to starvation, so return true to avoid that.
+                 */
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid inode");
+                return _gf_true;
+        }
+        /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
+         * is taken mount2 opened the same file, it won't be able to
+         * perform any data operations until mount1 releases eager-lock.
+         * To avoid such scenario do not enable eager-lock for this transaction
+         * if open-fd-count is > 1
+         */
+
+        ictx = afr_inode_ctx_get (inode, this);
+        if (!ictx)
+                return _gf_true;
+
+        if (ictx->open_fd_count > 1)
+                return _gf_true;
+
+        return _gf_false;
+}
+
+gf_boolean_t
+afr_any_fops_failed (afr_local_t *local, afr_private_t *priv)
+{
+        if (local->success_count != priv->child_count)
+                return _gf_true;
+        return _gf_false;
+}
+
+gf_boolean_t
+is_afr_delayed_changelog_post_op_needed (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t      *local = NULL;
+        gf_boolean_t      res = _gf_false;
+        afr_private_t    *priv  = NULL;
 
         priv  = this->private;
+
         local = frame->local;
+        if (!local)
+                goto out;
 
-        if (__fop_changelog_needed (frame, this)) {
-                afr_changelog_pre_op (frame, this);
-        } else {
-                __mark_all_success (local->pending, priv->child_count,
-                                    local->transaction.type);
+        if (!local->delayed_post_op)
+                goto out;
+
+        //Mark pending changelog ASAP
+        if (afr_any_fops_failed (local, priv))
+                goto out;
+
+        if (local->fd && afr_are_multiple_fds_opened (local->fd->inode, this))
+                goto out;
+
+        res = _gf_true;
+out:
+        return res;
+}
+
+
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub);
 
-                afr_pid_restore (frame);
+void
+afr_delayed_changelog_wake_up_cbk (void *data)
+{
+        fd_t           *fd = NULL;
+
+        fd = data;
+
+        afr_delayed_changelog_wake_up (THIS, fd);
+}
+
+
+/*
+  Check if the frame is destined to get optimized away
+  with changelog piggybacking
+*/
+static gf_boolean_t
+is_piggyback_post_op (call_frame_t *frame, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        afr_local_t *local = NULL;
+        gf_boolean_t piggyback = _gf_true;
+        afr_private_t *priv = NULL;
+        int i = 0;
 
-                local->transaction.fop (frame, this);
+        priv = frame->this->private;
+        local = frame->local;
+        fdctx = afr_fd_ctx_get (fd, frame->this);
+
+        LOCK(&fd->lock);
+        {
+                piggyback = _gf_true;
+
+                for (i = 0; i < priv->child_count; i++) {
+                        if (!local->transaction.pre_op[i])
+                                continue;
+                        if (fdctx->pre_op_piggyback[i]) {
+                                fdctx->pre_op_piggyback[i]--;
+                                local->transaction.postop_piggybacked[i] = 1;
+                        } else {
+                                /* For at least _one_ subvolume we cannot
+                                   piggyback on the changelog, and have to
+                                   perform a hard POST-OP and therefore fsync
+                                   if necesssary
+                                */
+                                piggyback = _gf_false;
+                                GF_ASSERT (fdctx->pre_op_done[i]);
+                                fdctx->pre_op_done[i]--;
+                        }
+                }
         }
+        UNLOCK(&fd->lock);
+
+        if (!afr_txn_nothing_failed (frame, frame->this)) {
+                /* something failed in this transaction,
+                   we will be performing a hard post-op
+                */
+                return _gf_false;
+        }
+
+        return piggyback;
+}
+
+
+/* SET operation */
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+
+        fdctx = afr_fd_ctx_get (fd, this);
+
+        LOCK(&fd->lock);
+        {
+                fdctx->witnessed_unstable_write = _gf_true;
+        }
+        UNLOCK(&fd->lock);
 
         return 0;
 }
 
+/* TEST and CLEAR operation */
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        gf_boolean_t witness = _gf_false;
+
+	fdctx = afr_fd_ctx_get (fd, this);
+        if (!fdctx)
+                return _gf_true;
+
+        LOCK(&fd->lock);
+        {
+                if (fdctx->witnessed_unstable_write) {
+                        witness = _gf_true;
+                        fdctx->witnessed_unstable_write = _gf_false;
+                }
+        }
+        UNLOCK (&fd->lock);
+
+        return witness;
+}
+
 
 int
+afr_changelog_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *pre,
+                         struct iatt *post, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
+        afr_local_t *local = NULL;
+
+        priv = this->private;
+        local = frame->local;
+
+        if (afr_fop_failed (op_ret, op_errno)) {
+                /* Failure of fsync() is as good as failure of previous
+                   write(). So treat it like one.
+                */
+                gf_log (this->name, GF_LOG_WARNING,
+                        "fsync(%s) failed on subvolume %s. Transaction was %s",
+                        uuid_utoa (local->fd->inode->gfid),
+                        priv->children[child_index]->name,
+                        gf_fop_list[local->op]);
+
+                afr_transaction_fop_failed (frame, this, child_index);
+        }
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                afr_changelog_post_op_now (frame, this);
+
+        return 0;
+}
+
+
+int
+afr_changelog_fsync (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        int i = 0;
+        int call_count = 0;
+        afr_private_t *priv = NULL;
+ 	dict_t *xdata = NULL;
+ 	GF_UNUSED int ret = -1;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (!call_count) {
+                /* will go straight to unlock */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+	xdata = dict_new();
+	if (xdata)
+		ret = dict_set_int32 (xdata, "batch-fsync", 1);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op[i])
+                        continue;
+
+                STACK_WIND_COOKIE (frame, afr_changelog_fsync_cbk,
+                                (void *) (long) i, priv->children[i],
+                                priv->children[i]->fops->fsync, local->fd,
+                                1, xdata);
+                if (!--call_count)
+                        break;
+        }
+
+	if (xdata)
+		dict_unref (xdata);
+
+        return 0;
+}
+
+
+        int
+afr_changelog_post_op_safe (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+	local = frame->local;
+        priv = this->private;
+
+        if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        if (is_piggyback_post_op (frame, local->fd)) {
+                /* just detected that this post-op is about to
+                   be optimized away as a new write() has
+                   already piggybacked on this frame's changelog.
+                   */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Calling afr_changelog_post_op_now() now will result in
+           issuing ->[f]xattrop().
+
+           Performing a hard POST-OP (->[f]xattrop() FOP) is a more
+           responsible operation that what it might appear on the surface.
+
+           The changelog of a file (in the xattr of the file on the server)
+           stores information (pending count) about the state of the file
+           on the OTHER server. This changelog is blindly trusted, and must
+           therefore be updated in such a way it remains trustworthy. This
+           implies that decrementing the pending count (essentially "clearing
+           the dirty flag") must be done STRICTLY after we are sure that the
+           operation on the other server has reached stable storage.
+
+           While the backend filesystem on that server will eventually flush
+           it to stable storage, we (being in userspace) have no mechanism
+           to get notified when the write became "stable".
+
+           This means we need take matter into our own hands and issue an
+           fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
+           and get an acknowledgement for it. And we need to wait for the
+           fsync() acknowledgement before initiating the hard POST-OP.
+
+           However if the FD itself was opened in O_SYNC or O_DSYNC then
+           we are already guaranteed that the writes were made stable as
+           part of the FOP itself. The same holds true for NFS stable
+           writes which happen on an anonymous FD with O_DSYNC or O_SYNC
+           flag set in the writev() @flags param. For all other write types,
+           mark a flag in the fdctx whenever an unstable write is witnessed.
+           */
+
+        if (!afr_fd_has_witnessed_unstable_write (this, local->fd)) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Check whether users want durability and perform fsync/post-op
+         * accordingly.
+         */
+        if (priv->ensure_durability) {
+                /* Time to fsync() */
+                afr_changelog_fsync (frame, this);
+        } else {
+                afr_changelog_post_op_now (frame, this);
+        }
+
+        return 0;
+}
+
+
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub)
+{
+	afr_fd_ctx_t      *fd_ctx = NULL;
+	call_frame_t      *prev_frame = NULL;
+	struct timespec    delta = {0, };
+	afr_private_t     *priv = NULL;
+	afr_local_t       *local = NULL;
+
+	priv = this->private;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+                goto out;
+
+	delta.tv_sec = priv->post_op_delay_secs;
+	delta.tv_nsec = 0;
+
+	pthread_mutex_lock (&fd_ctx->delay_lock);
+	{
+		prev_frame = fd_ctx->delay_frame;
+		fd_ctx->delay_frame = NULL;
+		if (fd_ctx->delay_timer)
+			gf_timer_call_cancel (this->ctx, fd_ctx->delay_timer);
+		fd_ctx->delay_timer = NULL;
+		if (!frame)
+			goto unlock;
+		fd_ctx->delay_timer = gf_timer_call_after (this->ctx, delta,
+							   afr_delayed_changelog_wake_up_cbk,
+							   fd);
+		fd_ctx->delay_frame = frame;
+	}
+unlock:
+	pthread_mutex_unlock (&fd_ctx->delay_lock);
+
+out:
+	if (prev_frame) {
+		local = prev_frame->local;
+		local->transaction.resume_stub = stub;
+		afr_changelog_post_op_safe (prev_frame, this);
+	} else if (stub) {
+		call_resume (stub);
+	}
+}
+
+
+void
+afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (is_afr_delayed_changelog_post_op_needed (frame, this))
+                afr_delayed_changelog_post_op (this, frame, local->fd, NULL);
+        else
+                afr_changelog_post_op_safe (frame, this);
+}
+
+
+
+/* Wake up the sleeping/delayed post-op, and also register
+   a stub to have it resumed after this transaction
+   completely finishes.
+
+   The @stub gets saved in @local and gets resumed in
+   afr_local_cleanup()
+   */
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, stub);
+}
+
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, NULL);
+}
+
+
+        int
 afr_transaction_resume (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
@@ -1213,6 +1787,19 @@ afr_transaction_resume (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
         priv     = this->private;
 
+        if (local->transaction.eager_lock_on) {
+                /* We don't need to retain "local" in the
+                   fd list anymore, writes to all subvols
+                   are finished by now */
+                LOCK (&local->fd->lock);
+                {
+                        list_del_init (&local->transaction.eager_locked);
+                }
+                UNLOCK (&local->fd->lock);
+        }
+
+        afr_restore_lk_owner (frame);
+
         if (__fop_changelog_needed (frame, this)) {
                 afr_changelog_post_op (frame, this);
         } else {
@@ -1233,7 +1820,8 @@ afr_transaction_resume (call_frame_t *frame, xlator_t *this)
  */
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this, int child_index)
+afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
+                            int child_index)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = NULL;
@@ -1242,7 +1830,89 @@ afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this, int child_index
         priv  = this->private;
 
         __mark_child_dead (local->pending, priv->child_count,
-                           child_index, local->transaction.type);
+                        child_index, local->transaction.type);
+}
+
+
+
+        static gf_boolean_t
+afr_locals_overlap (afr_local_t *local1, afr_local_t *local2)
+{
+        uint64_t start1 = local1->transaction.start;
+        uint64_t start2 = local2->transaction.start;
+        uint64_t end1 = 0;
+        uint64_t end2 = 0;
+
+        if (local1->transaction.len)
+                end1 = start1 + local1->transaction.len - 1;
+        else
+                end1 = ULLONG_MAX;
+
+        if (local2->transaction.len)
+                end2 = start2 + local2->transaction.len - 1;
+        else
+                end2 = ULLONG_MAX;
+
+        return ((end1 >= start2) && (end2 >= start1));
+}
+
+void
+afr_transaction_eager_lock_init (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        afr_fd_ctx_t  *fdctx = NULL;
+        afr_local_t   *each = NULL;
+
+        priv = this->private;
+
+        if (!local->fd)
+                return;
+
+        if (local->transaction.type != AFR_DATA_TRANSACTION)
+                return;
+
+        if (!priv->eager_lock)
+                return;
+
+        fdctx = afr_fd_ctx_get (local->fd, this);
+        if (!fdctx)
+                return;
+
+        if (afr_are_multiple_fds_opened (local->fd->inode, this))
+                return;
+        /*
+         * Once full file lock is acquired in eager-lock phase, overlapping
+         * writes do not compete for inode-locks, instead are transferred to the
+         * next writes. Because of this overlapping writes are not ordered.
+         * This can cause inconsistencies in replication.
+         * Example:
+         * Two overlapping writes w1, w2 are sent in parallel on same fd
+         * in two threads t1, t2.
+         * Both threads can execute afr_writev_wind in the following manner.
+         * t1 winds w1 on brick-0
+         * t2 winds w2 on brick-0
+         * t2 winds w2 on brick-1
+         * t1 winds w1 on brick-1
+         *
+         * This check makes sure the locks are not transferred for
+         * overlapping writes.
+         */
+        LOCK (&local->fd->lock);
+        {
+                list_for_each_entry (each, &fdctx->eager_locked,
+                                     transaction.eager_locked) {
+                        if (afr_locals_overlap (each, local)) {
+                                local->transaction.eager_lock_on = _gf_false;
+                                goto unlock;
+                        }
+                }
+
+                local->transaction.eager_lock_on = _gf_true;
+                list_add_tail (&local->transaction.eager_locked,
+                               &fdctx->eager_locked);
+        }
+unlock:
+        UNLOCK (&local->fd->lock);
 }
 
 
@@ -1251,20 +1921,43 @@ afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = NULL;
+        fd_t            *fd   = NULL;
+        int             ret   = -1;
 
         local = frame->local;
         priv  = this->private;
 
-        afr_transaction_local_init (local, this);
-
         local->transaction.resume = afr_transaction_resume;
         local->transaction.type   = type;
 
+        ret = afr_transaction_local_init (local, this);
+        if (ret < 0)
+            goto out;
+
+        afr_transaction_eager_lock_init (local, this);
+
+        if (local->fd && local->transaction.eager_lock_on)
+                afr_set_lk_owner (frame, this, local->fd);
+        else
+                afr_set_lk_owner (frame, this, frame->root);
+
+        if (!local->transaction.eager_lock_on && local->loc.inode) {
+                fd = fd_lookup (local->loc.inode, frame->root->pid);
+                if (fd == NULL)
+                        fd = fd_lookup_anonymous (local->loc.inode);
+
+                if (fd) {
+                        afr_delayed_changelog_wake_up (this, fd);
+                        fd_unref (fd);
+                }
+        }
+
         if (afr_lock_server_count (priv, local->transaction.type) == 0) {
                 afr_internal_lock_finish (frame, this);
         } else {
                 afr_lock (frame, this);
         }
-
-        return 0;
+        ret = 0;
+out:
+        return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index f470f2697..fa626fd0d 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -1,25 +1,21 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __TRANSACTION_H__
 #define __TRANSACTION_H__
 
+typedef enum {
+        LOCAL_FIRST = 1,
+        LOCAL_LAST = 2
+} afr_xattrop_type_t;
+
 void
 afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 			    int child_index);
@@ -27,11 +23,29 @@ afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 int
 afr_lock_server_count (afr_private_t *priv, afr_transaction_type type);
 
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom);
+
 int32_t
 afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type);
 
 afr_fd_ctx_t *
 afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending);
+afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
+                      int child, afr_xattrop_type_t op);
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this);
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd);
+
+void
+__mark_all_success (int32_t *pending[], int child_count,
+                    afr_transaction_type type);
+gf_boolean_t
+afr_any_fops_failed (afr_local_t *local, afr_private_t *priv);
+
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this);
 #endif /* __TRANSACTION_H__ */
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 18cd030f1..c724eb2ae 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -30,6 +21,11 @@
 #endif
 #include "afr-common.c"
 
+#define SHD_INODE_LRU_LIMIT          2048
+#define AFR_EH_HEALED_LIMIT          1024
+#define AFR_EH_HEAL_FAIL_LIMIT       1024
+#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
+
 struct volume_options options[];
 
 int32_t
@@ -37,8 +33,13 @@ notify (xlator_t *this, int32_t event,
         void *data, ...)
 {
         int ret = -1;
+        va_list         ap;
+        void *data2 = NULL;
 
-        ret = afr_notify (this, event, data);
+        va_start (ap, data);
+        data2 = va_arg (ap, dict_t*);
+        va_end (ap);
+        ret = afr_notify (this, event, data, data2);
 
         return ret;
 }
@@ -106,6 +107,7 @@ reconfigure (xlator_t *this, dict_t *options)
 {
         afr_private_t *priv        = NULL;
         xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
         int            ret         = -1;
         int            index       = -1;
         char          *qtype       = NULL;
@@ -148,6 +150,9 @@ reconfigure (xlator_t *this, dict_t *options)
 
         GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);
 
+        GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode,
+                          options, uint32, out);
+
         if (read_subvol) {
                 index = xlator_subvolume_index (this, read_subvol);
                 if (index == -1) {
@@ -158,10 +163,37 @@ reconfigure (xlator_t *this, dict_t *options)
                 priv->read_child = index;
         }
 
+        GF_OPTION_RECONF ("read-subvolume-index",read_subvol_index, options,int32,out);
+
+        if (read_subvol_index >-1) {
+                index=read_subvol_index;
+                if (index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                index);
+                        goto out;
+                }
+                priv->read_child = index;
+        }
+
+        GF_OPTION_RECONF ("eager-lock", priv->eager_lock, options, bool, out);
         GF_OPTION_RECONF ("quorum-type", qtype, options, str, out);
         GF_OPTION_RECONF ("quorum-count", priv->quorum_count, options,
                           uint32, out);
         fix_quorum_options(this,priv,qtype);
+        GF_OPTION_RECONF ("heal-timeout", priv->shd.timeout, options,
+                          int32, out);
+
+	GF_OPTION_RECONF ("post-op-delay-secs", priv->post_op_delay_secs, options,
+			  uint32, out);
+
+        GF_OPTION_RECONF (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size,
+                          options, size, out);
+        /* Reset this so we re-discover in case the topology changed.  */
+        GF_OPTION_RECONF ("readdir-failover", priv->readdir_failover, options,
+                          bool, out);
+        GF_OPTION_RECONF ("ensure-durability", priv->ensure_durability, options,
+                          bool, out);
+        priv->did_discovery = _gf_false;
 
         ret = 0;
 out:
@@ -189,6 +221,7 @@ init (xlator_t *this)
         int            ret         = -1;
         GF_UNUSED int  op_errno    = 0;
         xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
         xlator_t      *fav_child   = NULL;
         char          *qtype       = NULL;
 
@@ -204,7 +237,10 @@ init (xlator_t *this)
                         "Volume is dangling.");
         }
 
-        ALLOC_OR_GOTO (this->private, afr_private_t, out);
+	this->private = GF_CALLOC (1, sizeof (afr_private_t),
+                                   gf_afr_mt_afr_private_t);
+        if (!this->private)
+                goto out;
 
         priv = this->private;
         LOCK_INIT (&priv->lock);
@@ -217,7 +253,6 @@ init (xlator_t *this)
 
         priv->child_count = child_count;
 
-
         priv->read_child = -1;
 
         GF_OPTION_INIT ("read-subvolume", read_subvol, xlator, out);
@@ -229,6 +264,18 @@ init (xlator_t *this)
                         goto out;
                 }
         }
+        GF_OPTION_INIT ("read-subvolume-index",read_subvol_index,int32,out);
+        if (read_subvol_index > -1) {
+                if (read_subvol_index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                read_subvol_index);
+                        goto out;
+                }
+                priv->read_child = read_subvol_index;
+        }
+        GF_OPTION_INIT ("choose-local", priv->choose_local, bool, out);
+
+        GF_OPTION_INIT ("read-hash-mode", priv->hash_mode, uint32, out);
 
         priv->favorite_child = -1;
         GF_OPTION_INIT ("favorite-child", fav_child, xlator, out);
@@ -263,6 +310,8 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
 
+        GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
+
         GF_OPTION_INIT ("data-change-log", priv->data_change_log, bool, out);
 
         GF_OPTION_INIT ("metadata-change-log", priv->metadata_change_log, bool,
@@ -279,10 +328,18 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("strict-readdir", priv->strict_readdir, bool, out);
 
+        GF_OPTION_INIT ("eager-lock", priv->eager_lock, bool, out);
         GF_OPTION_INIT ("quorum-type", qtype, str, out);
         GF_OPTION_INIT ("quorum-count", priv->quorum_count, uint32, out);
+        GF_OPTION_INIT (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size,
+                        out);
         fix_quorum_options(this,priv,qtype);
 
+	GF_OPTION_INIT ("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+        GF_OPTION_INIT ("readdir-failover", priv->readdir_failover, bool, out);
+        GF_OPTION_INIT ("ensure-durability", priv->ensure_durability, bool,
+                        out);
+
         priv->wait_count = 1;
 
         priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
@@ -323,8 +380,6 @@ init (xlator_t *this)
                                    AFR_XATTR_PREFIX,
                                    trav->xlator->name);
                 if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "asprintf failed to set pending key");
                         ret = -ENOMEM;
                         goto out;
                 }
@@ -333,6 +388,13 @@ init (xlator_t *this)
                 i++;
         }
 
+        ret = gf_asprintf (&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT,
+                           this->name);
+        if (-1 == ret) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
         priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
                                       gf_afr_mt_int32_t);
         if (!priv->last_event) {
@@ -340,16 +402,67 @@ init (xlator_t *this)
                 goto out;
         }
 
-        priv->shd.pos = GF_CALLOC (sizeof (*priv->shd.pos), child_count,
-                                   gf_afr_mt_afr_brick_pos_t);
-        if (!priv->shd.pos) {
-                ret = -ENOMEM;
+        /* keep more local here as we may need them for self-heal etc */
+        this->local_pool = mem_pool_new (afr_local_t, 512);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
                 goto out;
         }
 
         priv->first_lookup = 1;
         priv->root_inode = NULL;
 
+        if (!priv->shd.iamshd) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = -ENOMEM;
+        priv->shd.pos = GF_CALLOC (sizeof (*priv->shd.pos), child_count,
+                                   gf_afr_mt_brick_pos_t);
+        if (!priv->shd.pos)
+                goto out;
+
+        priv->shd.pending = GF_CALLOC (sizeof (*priv->shd.pending), child_count,
+                                       gf_afr_mt_int32_t);
+        if (!priv->shd.pending)
+                goto out;
+
+        priv->shd.inprogress = GF_CALLOC (sizeof (*priv->shd.inprogress),
+                                          child_count, gf_afr_mt_shd_bool_t);
+        if (!priv->shd.inprogress)
+                goto out;
+        priv->shd.timer = GF_CALLOC (sizeof (*priv->shd.timer), child_count,
+                                     gf_afr_mt_shd_timer_t);
+        if (!priv->shd.timer)
+                goto out;
+
+        priv->shd.healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false,
+                                   _destroy_shd_event_data);
+        if (!priv->shd.healed)
+                goto out;
+
+        priv->shd.heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false,
+                                        _destroy_shd_event_data);
+        if (!priv->shd.heal_failed)
+                goto out;
+
+        priv->shd.split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+                                        _destroy_shd_event_data);
+        if (!priv->shd.split_brain)
+                goto out;
+
+        this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
+        if (!this->itable)
+                goto out;
+        priv->root_inode = inode_ref (this->itable->root);
+        GF_OPTION_INIT ("node-uuid", priv->shd.node_uuid, str, out);
+        GF_OPTION_INIT ("heal-timeout", priv->shd.timeout, int32, out);
+        ret = afr_initialise_statistics (this);
+        if (ret)
+                goto out;
         ret = 0;
 out:
         return ret;
@@ -364,6 +477,8 @@ fini (xlator_t *this)
         priv = this->private;
         this->private = NULL;
         afr_priv_destroy (priv);
+        if (this->itable);//I dont see any destroy func
+
         return 0;
 }
 
@@ -382,6 +497,9 @@ struct xlator_fops fops = {
         .finodelk    = afr_finodelk,
         .entrylk     = afr_entrylk,
         .fentrylk    = afr_fentrylk,
+	.fallocate   = afr_fallocate,
+	.discard     = afr_discard,
+        .zerofill    = afr_zerofill,
 
         /* inode read */
         .access      = afr_access,
@@ -434,33 +552,79 @@ struct xlator_cbks cbks = {
 
 struct volume_options options[] = {
         { .key  = {"read-subvolume" },
-          .type = GF_OPTION_TYPE_XLATOR
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. Afr will prefer the one specified using "
+                         "this option if it is not stale. Option value must be "
+                         "one of the xlator names of the children. "
+                         "Ex: <volname>-client-0 till "
+                         "<volname>-client-<number-of-bricks - 1>"
+        },
+        { .key  = {"read-subvolume-index" },
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "-1",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one specified using "
+                         "this option if it is not stale. allowed options"
+                         " include -1 till replica-count - 1"
+        },
+        { .key = {"read-hash-mode" },
+          .type = GF_OPTION_TYPE_INT,
+          .min = 0,
+          .max = 2,
+          .default_value = "0",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one computed using "
+                         "the method specified using this option"
+                         "0 = first responder, "
+                         "1 = hash by GFID of file (all clients use "
+                                                    "same subvolume), "
+                         "2 = hash by GFID of file and client PID",
+        },
+        { .key  = {"choose-local" },
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "true",
+          .description = "Choose a local subvolume(i.e. Brick) to read from if "
+                         "read-subvolume is not explicitly set.",
         },
         { .key  = {"favorite-child"},
-          .type = GF_OPTION_TYPE_XLATOR
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "If a split-brain happens choose subvol/brick set by "
+                         "this option as source."
         },
         { .key  = {"background-self-heal-count"},
           .type = GF_OPTION_TYPE_INT,
           .min  = 0,
           .default_value = "16",
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "This specifies the number of self-heals that can be "
+                         " performed in background without blocking the fop"
         },
         { .key  = {"data-self-heal"},
           .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
           .value = {"1", "on", "yes", "true", "enable",
                     "0", "off", "no", "false", "disable",
                     "open"},
           .default_value = "on",
+          .description   = "Using this option we can enable/disable data "
+                           "self-heal on the file. \"open\" means data "
+                           "self-heal action will only be triggered by file "
+                           "open operations."
         },
         { .key  = {"data-self-heal-algorithm"},
           .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
           .description   = "Select between \"full\", \"diff\". The "
                            "\"full\" algorithm copies the entire file from "
                            "source to sink. The \"diff\" algorithm copies to "
                            "sink only those blocks whose checksums don't match "
-                           "with those of source.",
-          .value = { "diff", "full", "" }
+                           "with those of source. If no option is configured "
+                           "the option is chosen dynamically as follows: "
+                           "If the file does not exist on one of the sinks "
+                           "or empty file exists or if the source file size is "
+                           "about the same as page size the entire file will "
+                           "be read and written i.e \"full\" algo, "
+                           "otherwise \"diff\" algo is chosen.",
+          .value = { "diff", "full"}
         },
         { .key  = {"data-self-heal-window-size"},
           .type = GF_OPTION_TYPE_INT,
@@ -473,26 +637,43 @@ struct volume_options options[] = {
         { .key  = {"metadata-self-heal"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Using this option we can enable/disable metadata "
+                         "i.e. Permissions, ownerships, xattrs self-heal on "
+                         "the file/directory."
         },
         { .key  = {"entry-self-heal"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Using this option we can enable/disable entry "
+                         "self-heal on the directory."
         },
         { .key  = {"data-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Data fops like write/truncate will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"metadata-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Metadata fops like setattr/setxattr will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"entry-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Entry fops like create/unlink will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"optimistic-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Entry/Metadata fops will not perform "
+                         "pre fop changelog operations in afr transaction "
+                         "if this option is enabled."
         },
         { .key  = {"strict-readdir"},
           .type = GF_OPTION_TYPE_BOOL,
@@ -501,18 +682,55 @@ struct volume_options options[] = {
         { .key = {"inodelk-trace"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "Enabling this option logs inode lock/unlocks"
         },
         { .key = {"entrylk-trace"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "Enabling this option logs entry lock/unlocks"
+        },
+        { .key = {"eager-lock"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Lock phase of a transaction has two sub-phases. "
+                         "First is an attempt to acquire locks in parallel by "
+                         "broadcasting non-blocking lock requests. If lock "
+                         "aquistion fails on any server, then the held locks "
+                         "are unlocked and revert to a blocking locked mode "
+                         "sequentially on one server after another.  If this "
+                         "option is enabled the initial broadcasting lock "
+                         "request attempt to acquire lock on the entire file. "
+                         "If this fails, we revert back to the sequential "
+                         "\"regional\" blocking lock as before. In the case "
+                         "where such an \"eager\" lock is granted in the "
+                         "non-blocking phase, it gives rise to an opportunity "
+                         "for optimization. i.e, if the next write transaction "
+                         "on the same FD arrives before the unlock phase of "
+                         "the first transaction, it \"takes over\" the full "
+                         "file lock. Similarly if yet another data transaction "
+                         "arrives before the unlock phase of the \"optimized\" "
+                         "transaction, that in turn \"takes over\" the lock as "
+                         "well. The actual unlock now happens at the end of "
+                         "the last \"optimzed\" transaction."
+
         },
         { .key = {"self-heal-daemon"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "This option applies to only self-heal-daemon. "
+                         "Index directory crawl and automatic healing of files"
+                         "will not be performed if this option is turned off."
+        },
+        { .key = {"iam-self-heal-daemon"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option differentiates if the replicate "
+                         "translator is running as part of self-heal-daemon "
+                         "or not."
         },
         { .key = {"quorum-type"},
           .type = GF_OPTION_TYPE_STR,
-          .value = { "none", "auto", "fixed", "" },
+          .value = { "none", "auto", "fixed"},
           .default_value = "none",
           .description = "If value is \"fixed\" only allow writes if "
                          "quorum-count bricks are present.  If value is "
@@ -529,5 +747,47 @@ struct volume_options options[] = {
                          "this many bricks or present.  Other quorum types "
                          "will OVERWRITE this value.",
         },
+        { .key  = {"node-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Local glusterd uuid string, used in starting "
+                         "self-heal-daemon so that it can crawl only on "
+                         "local index directories.",
+        },
+        { .key  = {"heal-timeout"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 60,
+          .max  = INT_MAX,
+          .default_value = "600",
+          .description = "time interval for checking the need to self-heal "
+                         "in self-heal-daemon"
+        },
+        { .key  = {"post-op-delay-secs"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 0,
+          .max  = INT_MAX,
+          .default_value = "1",
+          .description = "Time interval induced artificially before "
+	                 "post-operation phase of the transaction to "
+                         "enhance overlap of adjacent write operations.",
+        },
+        { .key = {AFR_SH_READDIR_SIZE_KEY},
+          .type = GF_OPTION_TYPE_SIZET,
+          .description = "readdirp size for performing entry self-heal",
+          .min = 1024,
+          .max = 131072,
+          .default_value = "1KB",
+        },
+        { .key = {"readdir-failover"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "readdir(p) will not failover if this option is off",
+          .default_value = "on",
+        },
+        { .key = {"ensure-durability"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "Afr performs fsyncs for transactions if this "
+                         "option is on to make sure the changelogs/data is "
+                         "written to the disk",
+          .default_value = "on",
+        },
         { .key  = {NULL} },
 };
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 003d666e0..21064db58 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -32,9 +23,15 @@
 #include "afr-self-heal-algorithm.h"
 
 #include "libxlator.h"
+#include "timer.h"
 
 #define AFR_XATTR_PREFIX "trusted.afr"
 #define AFR_PATHINFO_HEADER "REPLICATE:"
+#define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
+#define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
+
+#define AFR_LOCKEE_COUNT_MAX    3
+#define AFR_DOM_COUNT_MAX    3
 
 struct _pump_private;
 
@@ -65,10 +62,8 @@ typedef enum {
         AFR_INODE_SET_READ_CTX = 1,
         AFR_INODE_RM_STALE_CHILDREN,
         AFR_INODE_SET_OPENDIR_DONE,
-        AFR_INODE_SET_SPLIT_BRAIN,
         AFR_INODE_GET_READ_CTX,
         AFR_INODE_GET_OPENDIR_DONE,
-        AFR_INODE_GET_SPLIT_BRAIN,
 } afr_inode_op_t;
 
 typedef struct afr_inode_params_ {
@@ -82,16 +77,41 @@ typedef struct afr_inode_params_ {
         } u;
 } afr_inode_params_t;
 
+typedef enum afr_spb_state {
+        DONT_KNOW,
+        SPB,
+        NO_SPB
+} afr_spb_state_t;
+
 typedef struct afr_inode_ctx_ {
         uint64_t masks;
         int32_t  *fresh_children;//increasing order of latency
+        afr_spb_state_t mdata_spb;
+        afr_spb_state_t data_spb;
+        uint32_t        open_fd_count;
 } afr_inode_ctx_t;
 
+typedef enum {
+        NONE,
+        INDEX,
+        INDEX_TO_BE_HEALED,
+        FULL,
+} afr_crawl_type_t;
+
 typedef struct afr_self_heald_ {
-        gf_boolean_t    enabled;
-        gf_boolean_t    pending;
-        gf_boolean_t    inprogress;
-        afr_child_pos_t *pos;
+        gf_boolean_t            enabled;
+        gf_boolean_t            iamshd;
+        afr_crawl_type_t        *pending;
+        gf_boolean_t            *inprogress;
+        afr_child_pos_t         *pos;
+        gf_timer_t              **timer;
+        eh_t                    *healed;
+        eh_t                    *heal_failed;
+        eh_t                    *split_brain;
+        eh_t                    **statistics;
+        void                    **crawl_events;
+        char                    *node_uuid;
+        int                     timeout;
 } afr_self_heald_t;
 
 typedef struct _afr_private {
@@ -125,6 +145,7 @@ typedef struct _afr_private {
         gf_boolean_t entry_change_log;      /* on/off */
 
         int read_child;               /* read-subvolume */
+        unsigned int hash_mode;       /* for when read_child is not set */
         int favorite_child;  /* subvolume to be preferred in resolving
                                          split-brain cases */
 
@@ -145,14 +166,48 @@ typedef struct _afr_private {
         struct list_head saved_fds;   /* list of fds on which locks have succeeded */
         gf_boolean_t      optimistic_change_log;
         gf_boolean_t      eager_lock;
+	uint32_t          post_op_delay_secs;
         unsigned int      quorum_count;
 
         char                   vol_uuid[UUID_SIZE + 1];
         int32_t                *last_event;
         afr_self_heald_t       shd;
+        gf_boolean_t           choose_local;
+        gf_boolean_t           did_discovery;
+        gf_boolean_t           readdir_failover;
+        uint64_t               sh_readdir_size;
+        gf_boolean_t           ensure_durability;
+        char                   *sh_domain;
 } afr_private_t;
 
+typedef enum {
+        AFR_SELF_HEAL_NOT_ATTEMPTED,
+        AFR_SELF_HEAL_STARTED,
+        AFR_SELF_HEAL_FAILED,
+        AFR_SELF_HEAL_SYNC_BEGIN,
+} afr_self_heal_status;
+
 typedef struct {
+        afr_self_heal_status gfid_or_missing_entry_self_heal;
+        afr_self_heal_status metadata_self_heal;
+        afr_self_heal_status data_self_heal;
+        afr_self_heal_status entry_self_heal;
+} afr_sh_status_for_all_type;
+
+typedef enum {
+        AFR_SELF_HEAL_ENTRY,
+        AFR_SELF_HEAL_METADATA,
+        AFR_SELF_HEAL_DATA,
+        AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY,
+        AFR_SELF_HEAL_INVALID = -1,
+} afr_self_heal_type;
+
+typedef enum {
+        AFR_CHECK_ALL,
+        AFR_CHECK_SPECIFIC,
+} afr_sh_fail_check_type;
+
+struct afr_self_heal_ {
         /* External interface: These are variables (some optional) that
            are set by whoever has triggered self-heal */
 
@@ -161,6 +216,8 @@ typedef struct {
         gf_boolean_t do_entry_self_heal;
         gf_boolean_t do_gfid_self_heal;
         gf_boolean_t do_missing_entry_self_heal;
+        gf_boolean_t force_confirm_spb; /* Check for split-brains even when
+                                           self-heal is turned off */
 
         gf_boolean_t forced_merge;        /* Is this a self-heal triggered to
                                              forcibly merge the directories? */
@@ -178,7 +235,7 @@ typedef struct {
            background, this function will be called as soon as possible. */
 
         int (*unwind) (call_frame_t *frame, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno);
+                       int32_t op_errno, int32_t sh_failed);
 
         /* End of external interface members */
 
@@ -227,9 +284,10 @@ typedef struct {
         const char *linkname;
         gf_boolean_t entries_skipped;
 
-        int   op_failed;
-
+        gf_boolean_t actual_sh_started;
+        gf_boolean_t sync_done;
         gf_boolean_t data_lock_held;
+        gf_boolean_t sh_dom_lock_held;
         gf_boolean_t eof_reached;
         fd_t  *healing_fd;
         int   file_has_holes;
@@ -240,16 +298,22 @@ typedef struct {
         uint8_t *checksum;
         afr_post_remove_call_t post_remove_call;
 
-        loc_t parent_loc;
+        char    *data_sh_info;
+        char    *metadata_sh_info;
 
+        loc_t parent_loc;
         call_frame_t *orig_frame;
         call_frame_t *old_loop_frame;
         gf_boolean_t unwound;
 
         afr_sh_algo_private_t *private;
+        afr_sh_status_for_all_type  afr_all_sh_status;
+        afr_self_heal_type       sh_type_in_action;
 
+        struct afr_sh_algorithm  *algo;
         afr_lock_cbk_t data_lock_success_handler;
         afr_lock_cbk_t data_lock_failure_handler;
+	gf_boolean_t data_lock_block;
         int (*completion_cbk) (call_frame_t *frame, xlator_t *this);
         int (*sh_data_algo_start) (call_frame_t *frame, xlator_t *this);
         int (*algo_completion_cbk) (call_frame_t *frame, xlator_t *this);
@@ -257,7 +321,9 @@ typedef struct {
         void (*gfid_sh_success_cbk) (call_frame_t *sh_frame, xlator_t *this);
 
         call_frame_t *sh_frame;
-} afr_self_heal_t;
+};
+
+typedef struct afr_self_heal_ afr_self_heal_t;
 
 typedef enum {
         AFR_DATA_TRANSACTION,          /* truncate, write, ... */
@@ -319,11 +385,31 @@ afr_index_for_transaction_type (afr_transaction_type type)
         return -1;  /* make gcc happy */
 }
 
+typedef struct {
+        loc_t                   loc;
+        char                    *basename;
+        unsigned char           *locked_nodes;
+        int                     locked_count;
+
+} afr_entry_lockee_t;
+
+int
+afr_entry_lockee_cmp (const void *l1, const void *l2);
+
+typedef struct {
+        char    *domain; /* Domain on which inodelk is taken */
+        struct gf_flock flock;
+        unsigned char *locked_nodes;
+        int32_t lock_count;
+} afr_inodelk_t;
 
 typedef struct {
         loc_t *lk_loc;
-        struct gf_flock lk_flock;
 
+        int                     lockee_count;
+        afr_entry_lockee_t      lockee[AFR_LOCKEE_COUNT_MAX];
+
+        afr_inodelk_t       inodelk[AFR_DOM_COUNT_MAX];
         const char *lk_basename;
         const char *lower_basename;
         const char *higher_basename;
@@ -332,23 +418,22 @@ typedef struct {
 
         unsigned char *locked_nodes;
         unsigned char *lower_locked_nodes;
-        unsigned char *inode_locked_nodes;
-        unsigned char *entry_locked_nodes;
 
         selfheal_lk_type_t selfheal_lk_type;
         transaction_lk_type_t transaction_lk_type;
 
         int32_t lock_count;
-        int32_t inodelk_lock_count;
         int32_t entrylk_lock_count;
 
         uint64_t lock_number;
         int32_t lk_call_count;
         int32_t lk_expected_count;
+        int32_t lk_attempted_count;
 
         int32_t lock_op_ret;
         int32_t lock_op_errno;
         afr_lock_cbk_t lock_cbk;
+        char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
 } afr_internal_lock_t;
 
 typedef struct _afr_locked_fd {
@@ -356,21 +441,29 @@ typedef struct _afr_locked_fd {
         struct list_head list;
 } afr_locked_fd_t;
 
+struct afr_reply {
+	int	valid;
+	int32_t	op_ret;
+	int32_t	op_errno;
+};
+
 typedef struct _afr_local {
         int     uid;
         int     gid;
         unsigned int call_count;
         unsigned int success_count;
         unsigned int enoent_count;
+        uint32_t     open_fd_count;
+        gf_boolean_t update_open_fd_count;
 
 
-        unsigned int govinda_gOvinda;
+        unsigned int unhealable;
 
         unsigned int read_child_index;
         unsigned char read_child_returned;
         unsigned int first_up_child;
 
-        pid_t saved_pid;
+	gf_lkowner_t  saved_lk_owner;
 
         int32_t op_ret;
         int32_t op_errno;
@@ -381,7 +474,6 @@ typedef struct _afr_local {
         loc_t newloc;
 
         fd_t *fd;
-        unsigned char *fd_open_on;
 
         glusterfs_fop_t fop;
 
@@ -403,13 +495,25 @@ typedef struct _afr_local {
 
         dict_t  *dict;
         int      optimistic_change_log;
+	gf_boolean_t      delayed_post_op;
+
 
-        gf_boolean_t    fop_paused;
-        int (*fop_call_continue) (call_frame_t *frame, xlator_t *this);
+	/* Is the current writev() going to perform a stable write?
+	   i.e, is fd->flags or @flags writev param have O_SYNC or
+	   O_DSYNC?
+	*/
+        gf_boolean_t      stable_write;
 
-        /*
-          This struct contains the arguments for the "continuation"
-          (scheme-like) of fops
+        /* This write appended to the file. Nnot necessarily O_APPEND,
+           just means the offset of write was at the end of file.
+        */
+        gf_boolean_t      append_write;
+
+        int allow_sh_for_running_transaction;
+
+
+        /* This struct contains the arguments for the "continuation"
+           (scheme-like) of fops
         */
 
         int   op;
@@ -420,6 +524,7 @@ typedef struct _afr_local {
                 } statfs;
 
                 struct {
+                        uint32_t parent_entrylk;
                         uuid_t  gfid_req;
                         inode_t *inode;
                         struct iatt buf;
@@ -431,11 +536,13 @@ typedef struct _afr_local {
                         int32_t read_child;
                         int32_t *sources;
                         int32_t *success_children;
+                        int32_t **pending_matrix;
+                        gf_boolean_t fresh_lookup;
+                        gf_boolean_t possible_spb;
                 } lookup;
 
                 struct {
                         int32_t flags;
-                        int32_t wbflags;
                 } open;
 
                 struct {
@@ -468,7 +575,7 @@ typedef struct _afr_local {
                 struct {
                         char *name;
                         int last_index;
-                        long pathinfo_len;
+                        long xattr_len;
                 } getxattr;
 
                 struct {
@@ -502,7 +609,9 @@ typedef struct _afr_local {
                 struct {
                         struct iatt prebuf;
                         struct iatt postbuf;
+                } inode_wfop; //common structure for all inode-write-fops
 
+                struct {
                         int32_t op_ret;
 
                         struct iovec *vector;
@@ -513,34 +622,21 @@ typedef struct _afr_local {
                 } writev;
 
                 struct {
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } fsync;
-
-                struct {
                         off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
                 } truncate;
 
                 struct {
                         off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
                 } ftruncate;
 
                 struct {
                         struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
                 } setattr;
 
                 struct {
                         struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
                 } fsetattr;
 
                 struct {
@@ -557,93 +653,77 @@ typedef struct _afr_local {
                         char *name;
                 } removexattr;
 
+                struct {
+                        dict_t *xattr;
+                } xattrop;
+
+                struct {
+                        dict_t *xattr;
+                } fxattrop;
+
                 /* dir write */
 
                 struct {
-                        fd_t *fd;
-                        dict_t *params;
-                        int32_t flags;
-                        mode_t mode;
                         inode_t *inode;
                         struct iatt buf;
                         struct iatt preparent;
                         struct iatt postparent;
-                        struct iatt read_child_buf;
+                        struct iatt prenewparent;
+                        struct iatt postnewparent;
+                } dir_fop; //common structure for all dir fops
+
+                struct {
+                        fd_t *fd;
+                        dict_t *params;
+                        int32_t flags;
+                        mode_t mode;
                 } create;
 
                 struct {
                         dev_t dev;
                         mode_t mode;
                         dict_t *params;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                        struct iatt read_child_buf;
                 } mknod;
 
                 struct {
                         int32_t mode;
                         dict_t *params;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
                 } mkdir;
 
                 struct {
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } unlink;
-
-                struct {
-                        int   flags;
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
+                        int flags;
                 } rmdir;
 
                 struct {
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preoldparent;
-                        struct iatt prenewparent;
-                        struct iatt postoldparent;
-                        struct iatt postnewparent;
-                } rename;
-
-                struct {
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } link;
-
-                struct {
-                        inode_t *inode;
                         dict_t *params;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
                         char *linkpath;
-                        struct iatt preparent;
-                        struct iatt postparent;
                 } symlink;
 
+		struct {
+			int32_t mode;
+			off_t offset;
+			size_t len;
+		} fallocate;
+
+		struct {
+			off_t offset;
+			size_t len;
+		} discard;
+
                 struct {
-                        int32_t flags;
-                        dir_entry_t *entries;
-                        int32_t count;
-                } setdents;
+                        off_t offset;
+                        size_t len;
+                        struct iatt prebuf;
+                        struct iatt postbuf;
+                } zerofill;
+
+
         } cont;
 
         struct {
                 off_t start, len;
 
+                gf_boolean_t    eager_lock_on;
                 int *eager_lock;
 
                 char *basename;
@@ -654,12 +734,18 @@ typedef struct _afr_local {
 
                 afr_transaction_type type;
 
-                int success_count;
-                int erase_pending;
-                int failure_count;
+		/* pre-compute the post piggyback status before
+		   entering POST-OP phase
+		*/
+		int              *postop_piggybacked;
+
+		/* stub to resume on destruction
+		   of the transaction frame */
+		call_stub_t      *resume_stub;
+
+		struct list_head  eager_locked;
 
-                int last_tried;
-                int32_t *child_errno;
+                int32_t         **txn_changelog;//changelog after pre+post ops
                 unsigned char   *pre_op;
 
                 call_frame_t *main_frame;
@@ -678,6 +764,15 @@ typedef struct _afr_local {
         afr_self_heal_t self_heal;
 
         struct marker_str     marker;
+
+        /* extra data for fops */
+        dict_t         *xdata_req;
+        dict_t         *xdata_rsp;
+
+        mode_t          umask;
+        int             xflag;
+        gf_boolean_t    do_discovery;
+	struct afr_reply *replies;
 } afr_local_t;
 
 typedef enum {
@@ -687,11 +782,6 @@ typedef enum {
 } afr_fd_open_status_t;
 
 typedef struct {
-        struct list_head call_list;
-        call_frame_t    *frame;
-} afr_fd_paused_call_t;
-
-typedef struct {
         unsigned int *pre_op_done;
         afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
         unsigned int *pre_op_piggyback;
@@ -700,7 +790,6 @@ typedef struct {
         unsigned int *lock_acquired;
 
         int flags;
-        int32_t wbflags;
         uint64_t up_count;   /* number of CHILD_UPs this fd has seen */
         uint64_t down_count; /* number of CHILD_DOWNs this fd has seen */
 
@@ -711,20 +800,32 @@ typedef struct {
         struct list_head entries; /* needed for readdir failover */
 
         unsigned char *locked_on; /* which subvolumes locks have been successful */
-	struct list_head  paused_calls; /* queued calls while fix_open happens  */
+
+	/* used for delayed-post-op optimization */
+	pthread_mutex_t    delay_lock;
+	gf_timer_t        *delay_timer;
+	call_frame_t      *delay_frame;
+        int               call_child;
+
+	/* set if any write on this fd was a non stable write
+	   (i.e, without O_SYNC or O_DSYNC)
+	*/
+	gf_boolean_t      witnessed_unstable_write;
+
+	/* list of frames currently in progress */
+	struct list_head  eager_locked;
 } afr_fd_ctx_t;
 
 
 /* try alloc and if it fails, goto label */
-#define ALLOC_OR_GOTO(var, type, label) do {                    \
-                var = GF_CALLOC (sizeof (type), 1,              \
-                                 gf_afr_mt_##type);             \
-                if (!var) {                                     \
-                        gf_log (this->name, GF_LOG_ERROR,       \
-                                "out of memory :(");            \
-                        op_errno = ENOMEM;                      \
-                        goto label;                             \
-                }                                               \
+#define AFR_LOCAL_ALLOC_OR_GOTO(var, label) do {                    \
+                var = mem_get0 (THIS->local_pool);                  \
+                if (!var) {                                         \
+                        gf_log (this->name, GF_LOG_ERROR,           \
+                                "out of memory :(");                \
+                        op_errno = ENOMEM;                          \
+                        goto label;                                 \
+                }                                                   \
         } while (0);
 
 
@@ -745,8 +846,14 @@ int
 pump_command_reply (call_frame_t *frame, xlator_t *this);
 
 int32_t
-afr_notify (xlator_t *this, int32_t event,
-            void *data, ...);
+afr_notify (xlator_t *this, int32_t event, void *data, void *data2);
+
+int
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count);
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock);
 
 int
 afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
@@ -759,7 +866,7 @@ afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
                        unsigned char *locked_nodes);
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this);
+afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner);
 
 int
 afr_set_lock_number (call_frame_t *frame, xlator_t *this);
@@ -783,8 +890,8 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this);
 int
 afr_internal_lock_finish (call_frame_t *frame, xlator_t *this);
 
-void
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src,
+int
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
                           unsigned int child_count);
 
 int pump_start (call_frame_t *frame, xlator_t *this);
@@ -802,8 +909,8 @@ void
 afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
                         int32_t *fresh_children);
 
-void
-afr_build_parent_loc (loc_t *parent, loc_t *child);
+int
+afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno);
 
 unsigned int
 afr_up_children_count (unsigned char *child_up, unsigned int child_count);
@@ -834,11 +941,12 @@ gf_boolean_t
 afr_is_split_brain (xlator_t *this, inode_t *inode);
 
 void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set);
+afr_set_split_brain (xlator_t *this, inode_t *inode, afr_spb_state_t mdata_spb,
+                     afr_spb_state_t data_spb);
 
 int
 afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags);
+          fd_t *fd, dict_t *xdata);
 
 void
 afr_set_opendir_done (xlator_t *this, inode_t *inode);
@@ -865,20 +973,24 @@ afr_launch_openfd_self_heal (call_frame_t *frame, xlator_t *this, fd_t *fd);
                         frame->local = NULL;                    \
                 }                                               \
                 STACK_UNWIND_STRICT (fop, frame, params);       \
-                afr_local_cleanup (__local, __this);            \
-                GF_FREE (__local);                              \
+                if (__local) {                                  \
+                        afr_local_cleanup (__local, __this);    \
+                        mem_put (__local);                      \
+                }                                               \
         } while (0)
 
-#define AFR_STACK_DESTROY(frame)                        \
-        do {                                            \
-                afr_local_t *__local = NULL;            \
-                xlator_t    *__this = NULL;             \
-                __local = frame->local;                 \
-                __this = frame->this;                   \
-                frame->local = NULL;                    \
-                STACK_DESTROY (frame->root);            \
-                afr_local_cleanup (__local, __this);    \
-                GF_FREE (__local);                      \
+#define AFR_STACK_DESTROY(frame)                                \
+        do {                                                    \
+                afr_local_t *__local = NULL;                    \
+                xlator_t    *__this = NULL;                     \
+                __local = frame->local;                         \
+                __this = frame->this;                           \
+                frame->local = NULL;                            \
+                STACK_DESTROY (frame->root);                    \
+                if (__local) {                                  \
+                        afr_local_cleanup (__local, __this);    \
+                        mem_put (__local);                      \
+                }                                               \
         } while (0);
 
 #define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
@@ -917,12 +1029,13 @@ afr_first_up_child (unsigned char *child_up, size_t child_count);
 int
 afr_select_read_child_from_policy (int32_t *fresh_children, int32_t child_count,
                                    int32_t prev_read_child,
-                                   int32_t config_read_child, int32_t *sources);
+                                   int32_t config_read_child, int32_t *sources,
+                                   unsigned int hmode, uuid_t gfid);
 
 void
 afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
                               int32_t *fresh_children, int32_t prev_read_child,
-                              int32_t config_read_child);
+                              int32_t config_read_child, uuid_t gfid);
 
 int32_t
 afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
@@ -944,8 +1057,9 @@ afr_children_rm_child (int32_t *children, int32_t child,
                              int32_t child_count);
 void
 afr_reset_children (int32_t *children, int32_t child_count);
-gf_boolean_t
-afr_error_more_important (int32_t old_errno, int32_t new_errno);
+int32_t
+afr_most_important_error(int32_t old_errno, int32_t new_errno,
+			 gf_boolean_t eio);
 int
 afr_errno_count (int32_t *children, int *child_errno,
                  unsigned int child_count, int32_t op_errno);
@@ -986,12 +1100,13 @@ afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
                       void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
                                                    xlator_t *this),
                       int (*unwind) (call_frame_t *frame, xlator_t *this,
-                                     int32_t op_ret, int32_t op_errno));
-int
-afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
-              int need_open_count, int *need_open);
-int
-afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop);
+                                     int32_t op_ret, int32_t op_errno,
+                                     int32_t sh_failed));
+void
+afr_fix_open (xlator_t *this, fd_t *fd, size_t need_open_count, int *need_open);
+
+void
+afr_open_fd_fix (fd_t *fd, xlator_t *this);
 int
 afr_set_elem_count_get (unsigned char *elems, int child_count);
 
@@ -1008,7 +1123,7 @@ void
 afr_set_low_priority (call_frame_t *frame);
 int
 afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags, int32_t wb_flags);
+                      int flags);
 
 gf_boolean_t
 afr_have_quorum (char *logname, afr_private_t *priv);
@@ -1018,6 +1133,23 @@ afr_matrix_cleanup (int32_t **pending, unsigned int m);
 
 int32_t**
 afr_matrix_create (unsigned int m, unsigned int n);
+
+gf_boolean_t
+afr_is_errno_set (int *child_errno, int child);
+
+gf_boolean_t
+afr_is_errno_unset (int *child_errno, int child);
+
+gf_boolean_t
+afr_is_fd_fixable (fd_t *fd);
+
+void
+afr_prepare_new_entry_pending_matrix (int32_t **pending,
+                                      gf_boolean_t (*is_pending) (int *, int),
+                                      int *ctx, struct iatt *buf,
+                                      unsigned int child_count);
+void
+afr_xattr_array_destroy (dict_t **xattr, unsigned int child_count);
 /*
  * Special value indicating we should use the "auto" quorum method instead of
  * a fixed value (including zero to turn off quorum enforcement).
@@ -1037,4 +1169,44 @@ afr_matrix_create (unsigned int m, unsigned int n);
         }                                                                \
 } while (0);
 
+
+#define AFR_SBRAIN_MSG "Failed on %s as split-brain is seen. Returning EIO."
+
+#define AFR_SBRAIN_CHECK_FD(fd, label) do {                              \
+        if (fd->inode && afr_is_split_brain (this, fd->inode)) {        \
+                op_errno = EIO;                                         \
+                gf_log (this->name, GF_LOG_WARNING,                     \
+                        AFR_SBRAIN_MSG ,uuid_utoa (fd->inode->gfid));   \
+                goto label;                                             \
+        }                                                               \
+} while (0)
+
+#define AFR_SBRAIN_CHECK_LOC(loc, label) do {                           \
+        if (loc->inode && afr_is_split_brain (this, loc->inode)) {      \
+                op_errno = EIO;                                         \
+                loc_path (loc, NULL);                                   \
+                gf_log (this->name, GF_LOG_WARNING,                     \
+                        AFR_SBRAIN_MSG , loc->path);                    \
+                goto label;                                             \
+        }                                                               \
+} while (0)
+
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd);
+
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd);
+
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub);
+
+int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count);
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this);
+
+afr_inode_ctx_t*
+afr_inode_ctx_get (inode_t *inode, xlator_t *this);
+
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
index df0b31166..a7f72fb30 100644
--- a/xlators/cluster/afr/src/pump.c
+++ b/xlators/cluster/afr/src/pump.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <unistd.h>
@@ -32,6 +23,13 @@
 #include "glusterfs.h"
 
 static uint64_t pump_pid = 0;
+static inline void
+pump_fill_loc_info (loc_t *loc, struct iatt *iatt, struct iatt *parent)
+{
+        afr_update_loc_gfids (loc, iatt, parent);
+        uuid_copy (loc->inode->gfid, iatt->ia_gfid);
+}
+
 static int
 pump_mark_start_pending (xlator_t *this)
 {
@@ -167,7 +165,7 @@ pump_save_path (xlator_t *this, const char *path)
 
         GF_ASSERT (priv->root_inode);
 
-        afr_build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
 
         dict = dict_new ();
         dict_ret = dict_set_str (dict, PUMP_PATH, (char *)path);
@@ -187,6 +185,7 @@ pump_save_path (xlator_t *this, const char *path)
 
         dict_unref (dict);
 
+        loc_wipe (&loc);
         return 0;
 }
 
@@ -384,8 +383,7 @@ gf_pump_traverse_directory (loc_t *loc)
                         }
                         loc_wipe (&entry_loc);
                         ret = afr_build_child_loc (this, &entry_loc, loc,
-                                                   entry->d_name,
-                                                   entry->d_stat.ia_gfid);
+                                                   entry->d_name);
                         if (ret)
                                 goto out;
 
@@ -407,7 +405,7 @@ gf_pump_traverse_directory (loc_t *loc)
                                                     entry_loc.path);
                                             continue;
                                     }
-                                    afr_fill_loc_info (&entry_loc, &iatt,
+                                    pump_fill_loc_info (&entry_loc, &iatt,
                                                        &parent);
 
                                     pump_update_resume_state (this, entry_loc.path);
@@ -430,7 +428,7 @@ gf_pump_traverse_directory (loc_t *loc)
                                                     gf_pump_traverse_directory (&entry_loc);
                                             }
                                     }
-                            }
+                        }
                 }
 
                 gf_dirent_free (&entries);
@@ -441,6 +439,10 @@ gf_pump_traverse_directory (loc_t *loc)
 
         }
 
+        ret = syncop_close (fd);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_DEBUG, "closing the fd failed");
+
         if (is_directory_empty && IS_ROOT_PATH (loc->path)) {
                pump_change_state (this, PUMP_STATE_RUNNING);
                gf_log (this->name, GF_LOG_INFO, "Empty source brick. "
@@ -480,7 +482,7 @@ pump_update_resume_path (xlator_t *this)
 
 static int32_t
 pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_private_t  *priv      = NULL;
         loc_t           loc       = {0};
@@ -491,7 +493,7 @@ pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
 
         priv      = this->private;
 
-        afr_build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
 
         ret = syncop_removexattr (priv->children[source], &loc,
                                           PUMP_PATH);
@@ -507,6 +509,7 @@ pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
                                 "failed with %s", strerror (errno));
         }
 
+        loc_wipe (&loc);
         return pump_command_reply (frame, this);
 }
 
@@ -526,7 +529,7 @@ pump_complete_migration (xlator_t *this)
 
         GF_ASSERT (priv->root_inode);
 
-        afr_build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
 
         dict = dict_new ();
 
@@ -568,6 +571,7 @@ pump_complete_migration (xlator_t *this)
                 call_resume (pump_priv->cleaner);
         }
 
+        loc_wipe (&loc);
         return 0;
 }
 
@@ -623,7 +627,7 @@ pump_task (void *data)
 
         GF_ASSERT (priv->root_inode);
 
-        afr_build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
         xattr_req = dict_new ();
         if (!xattr_req) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -661,6 +665,7 @@ out:
         if (xattr_req)
                 dict_unref (xattr_req);
 
+        loc_wipe (&loc);
 	return 0;
 }
 
@@ -694,7 +699,7 @@ pump_start (call_frame_t *pump_frame, xlator_t *this)
 	priv = this->private;
         pump_priv = priv->pump_private;
 
-        afr_set_lk_owner (pump_frame, this);
+        afr_set_lk_owner (pump_frame, this, pump_frame->root);
 	pump_pid = (uint64_t) (unsigned long)pump_frame->root;
 
 	ret = synctask_new (pump_priv->env, pump_task,
@@ -743,7 +748,7 @@ pump_cmd_start_setxattr_cbk (call_frame_t *frame,
                              void *cookie,
                              xlator_t *this,
                              int32_t op_ret,
-                             int32_t op_errno)
+                             int32_t op_errno, dict_t *xdata)
 
 {
         call_frame_t *prev = NULL;
@@ -795,7 +800,7 @@ pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
 
         GF_ASSERT (priv->root_inode);
 
-        afr_build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
 
         data = data_ref (dict_get (local->dict, RB_PUMP_CMD_START));
         if (!data) {
@@ -836,7 +841,7 @@ pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
 		    PUMP_SINK_CHILD(this)->fops->setxattr,
 		    &loc,
 		    dict,
-		    0);
+		    0, NULL);
 
         ret = 0;
 
@@ -850,6 +855,7 @@ out:
         if (ret && clnt_cmd)
                 GF_FREE (clnt_cmd);
 
+        loc_wipe (&loc);
         return ret;
 }
 
@@ -869,7 +875,7 @@ pump_cmd_start_getxattr_cbk (call_frame_t *frame,
                              xlator_t *this,
                              int32_t op_ret,
                              int32_t op_errno,
-                             dict_t *dict)
+                             dict_t *dict, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         char *path = NULL;
@@ -936,6 +942,7 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
         uint64_t number_files = 0;
 
         char filename[PATH_MAX];
+        char summary[PATH_MAX+256];
         char *dict_str = NULL;
 
         int32_t op_ret = 0;
@@ -964,12 +971,15 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
         }
 
         if (pump_priv->pump_finished) {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"        Migration complete ",
-                  number_files);
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64, number_files);
         } else {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"       Current file= %s ",
-                  number_files, filename);
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64":current_file=%s",
+                          number_files, filename);
         }
+        snprintf (dict_str, PATH_MAX+256, "status=%d:%s",
+                  (pump_priv->pump_finished)?1:0, summary);
 
         dict = dict_new ();
 
@@ -985,13 +995,12 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
 
 out:
 
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
 
         if (dict)
                 dict_unref (dict);
 
-        if (dict_str)
-                GF_FREE (dict_str);
+        GF_FREE (dict_str);
 
         return 0;
 }
@@ -1033,14 +1042,14 @@ pump_execute_start (call_frame_t *frame, xlator_t *this)
 
         GF_ASSERT (priv->root_inode);
 
-        afr_build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
 
 	STACK_WIND (frame,
 		    pump_cmd_start_getxattr_cbk,
 		    PUMP_SOURCE_CHILD(this),
 		    PUMP_SOURCE_CHILD(this)->fops->getxattr,
 		    &loc,
-		    PUMP_PATH);
+		    PUMP_PATH, NULL);
 
         ret = 0;
 
@@ -1050,6 +1059,7 @@ out:
                 pump_command_reply (frame, this);
         }
 
+        loc_wipe (&loc);
 	return 0;
 }
 
@@ -1057,7 +1067,7 @@ static int
 pump_cleanup_helper (void *data) {
         call_frame_t *frame = data;
 
-        pump_xattr_cleaner (frame, 0, frame->this, 0, 0);
+        pump_xattr_cleaner (frame, 0, frame->this, 0, 0, NULL);
 
         return 0;
 }
@@ -1139,7 +1149,7 @@ pump_execute_abort (call_frame_t *frame, xlator_t *this)
         } else {
                 pump_priv->cleaner = fop_setxattr_cbk_stub (frame,
                                                             pump_xattr_cleaner,
-                                                            0, 0);
+                                                            0, 0, NULL);
         }
 
         return 0;
@@ -1270,7 +1280,7 @@ struct _xattr_key {
         struct list_head list;
 };
 
-static void
+static int
 __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
                      void *data)
 {
@@ -1282,13 +1292,14 @@ __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
 
                 xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
                 if (!xkey)
-                        return;
+                        return -1;
 
                 xkey->key = key;
                 INIT_LIST_HEAD (&xkey->list);
 
                 list_add_tail (&xkey->list, list);
         }
+        return 0;
 }
 
 static void
@@ -1316,7 +1327,7 @@ __filter_xattrs (dict_t *dict)
 int32_t
 pump_getxattr_cbk (call_frame_t *frame, void *cookie,
 		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dict_t *dict)
+		  dict_t *dict, dict_t *xdata)
 {
 	afr_private_t   *priv           = NULL;
 	afr_local_t     *local          = NULL;
@@ -1351,7 +1362,7 @@ pump_getxattr_cbk (call_frame_t *frame, void *cookie,
 				   children[next_call_child],
 				   children[next_call_child]->fops->getxattr,
 				   &local->loc,
-				   local->cont.getxattr.name);
+				   local->cont.getxattr.name, NULL);
 	}
 
 out:
@@ -1359,7 +1370,7 @@ out:
                 if (op_ret >= 0 && dict)
                         __filter_xattrs (dict);
 
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
 	}
 
 	return 0;
@@ -1367,7 +1378,7 @@ out:
 
 int32_t
 pump_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name)
+	      loc_t *loc, const char *name, dict_t *xdata)
 {
 	afr_private_t *   priv       = NULL;
 	xlator_t **       children   = NULL;
@@ -1390,12 +1401,12 @@ pump_getxattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_getxattr_cbk,
                             FIRST_CHILD (this),
                             (FIRST_CHILD (this))->fops->getxattr,
-                            loc, name);
+                            loc, name, xdata);
                 return 0;
         }
 
 
-	ALLOC_OR_GOTO (frame->local, afr_local_t, out);
+	AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
 	local = frame->local;
 
         ret = afr_local_init (local, priv, &op_errno);
@@ -1444,12 +1455,12 @@ pump_getxattr (call_frame_t *frame, xlator_t *this,
 	STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
 			   (void *) (long) call_child,
 			   children[call_child], children[call_child]->fops->getxattr,
-			   loc, name);
+			   loc, name, xdata);
 
 	ret = 0;
 out:
 	if (ret < 0)
-		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
 	return 0;
 }
 
@@ -1471,14 +1482,14 @@ afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
 
 	if (main_frame) {
 		AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno);
+                                  local->op_ret, local->op_errno, NULL);
 	}
 	return 0;
 }
 
 static int
 afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
+		       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
 	afr_local_t *   local = NULL;
 	afr_private_t * priv  = NULL;
@@ -1547,7 +1558,7 @@ afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
 					   priv->children[i]->fops->setxattr,
 					   &local->loc,
 					   local->cont.setxattr.dict,
-					   local->cont.setxattr.flags);
+					   local->cont.setxattr.flags, NULL);
 
 			if (!--call_count)
 				break;
@@ -1575,11 +1586,9 @@ pump_setxattr_cbk (call_frame_t *frame,
 		      void *cookie,
 		      xlator_t *this,
 		      int32_t op_ret,
-		      int32_t op_errno)
+		      int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
+	AFR_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
 	return 0;
 }
 
@@ -1597,12 +1606,10 @@ pump_command_reply (call_frame_t *frame, xlator_t *this)
                 gf_log (this->name, GF_LOG_INFO,
                         "Command succeeded");
 
-        dict_unref (local->dict);
-
         AFR_STACK_UNWIND (setxattr,
                           frame,
                           local->op_ret,
-                          local->op_errno);
+                          local->op_errno, NULL);
 
         return 0;
 }
@@ -1639,12 +1646,11 @@ pump_parse_command (call_frame_t *frame, xlator_t *this,
 
 int
 pump_setxattr (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *dict, int32_t flags)
+               loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
 {
 	afr_private_t * priv  = NULL;
 	afr_local_t   * local = NULL;
 	call_frame_t   *transaction_frame = NULL;
-        data_pair_t   * trav  = NULL;
 	int ret = -1;
 	int op_errno = 0;
 
@@ -1653,19 +1659,19 @@ pump_setxattr (call_frame_t *frame, xlator_t *this,
 	VALIDATE_OR_GOTO (this->private, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
 	priv = this->private;
         if (!priv->use_afr_in_pump) {
                 STACK_WIND (frame, default_setxattr_cbk,
                             FIRST_CHILD (this),
                             (FIRST_CHILD (this))->fops->setxattr,
-                            loc, dict, flags);
+                            loc, dict, flags, xdata);
                 return 0;
         }
 
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+	AFR_LOCAL_ALLOC_OR_GOTO (local, out);
 
 	ret = afr_local_init (local, priv, &op_errno);
 	if (ret < 0) {
@@ -1714,7 +1720,7 @@ out:
 	if (ret < 0) {
 		if (transaction_frame)
 			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno);
+		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
 	}
 
 	return 0;
@@ -1748,7 +1754,7 @@ static int32_t
 pump_truncate (call_frame_t *frame,
                xlator_t *this,
                loc_t *loc,
-               off_t offset)
+               off_t offset, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1758,11 +1764,11 @@ pump_truncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->truncate,
                             loc,
-                            offset);
+                            offset, xdata);
                 return 0;
         }
 
-        afr_truncate (frame, this, loc, offset);
+        afr_truncate (frame, this, loc, offset, xdata);
         return 0;
 }
 
@@ -1771,7 +1777,7 @@ static int32_t
 pump_ftruncate (call_frame_t *frame,
                 xlator_t *this,
                 fd_t *fd,
-                off_t offset)
+                off_t offset, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1781,11 +1787,11 @@ pump_ftruncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->ftruncate,
                             fd,
-                            offset);
+                            offset, xdata);
                 return 0;
         }
 
-        afr_ftruncate (frame, this, fd, offset);
+        afr_ftruncate (frame, this, fd, offset, xdata);
         return 0;
 }
 
@@ -1794,7 +1800,7 @@ pump_ftruncate (call_frame_t *frame,
 
 int
 pump_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, dict_t *parms)
+            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1802,10 +1808,10 @@ pump_mknod (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mknod_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, parms);
+                            loc, mode, rdev, umask, xdata);
                 return 0;
         }
-        afr_mknod (frame, this, loc, mode, rdev, parms);
+        afr_mknod (frame, this, loc, mode, rdev, umask, xdata);
         return 0;
 
 }
@@ -1814,7 +1820,7 @@ pump_mknod (call_frame_t *frame, xlator_t *this,
 
 int
 pump_mkdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dict_t *params)
+            loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1822,10 +1828,10 @@ pump_mkdir (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mkdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, params);
+                            loc, mode, umask, xdata);
                 return 0;
         }
-        afr_mkdir (frame, this, loc, mode, params);
+        afr_mkdir (frame, this, loc, mode, umask, xdata);
         return 0;
 
 }
@@ -1834,7 +1840,7 @@ pump_mkdir (call_frame_t *frame, xlator_t *this,
 static int32_t
 pump_unlink (call_frame_t *frame,
              xlator_t *this,
-             loc_t *loc)
+             loc_t *loc, int xflag, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1843,10 +1849,10 @@ pump_unlink (call_frame_t *frame,
                             default_unlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->unlink,
-                            loc);
+                            loc, xflag, xdata);
                 return 0;
         }
-        afr_unlink (frame, this, loc);
+        afr_unlink (frame, this, loc, xflag, xdata);
         return 0;
 
 }
@@ -1854,7 +1860,7 @@ pump_unlink (call_frame_t *frame,
 
 static int
 pump_rmdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int flags)
+            loc_t *loc, int flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 
@@ -1864,11 +1870,11 @@ pump_rmdir (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_rmdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags);
+                            loc, flags, xdata);
                 return 0;
         }
 
-        afr_rmdir (frame, this, loc, flags);
+        afr_rmdir (frame, this, loc, flags, xdata);
         return 0;
 
 }
@@ -1877,7 +1883,7 @@ pump_rmdir (call_frame_t *frame, xlator_t *this,
 
 int
 pump_symlink (call_frame_t *frame, xlator_t *this,
-              const char *linkpath, loc_t *loc, dict_t *params)
+              const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1885,10 +1891,10 @@ pump_symlink (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_symlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, params);
+                            linkpath, loc, umask, xdata);
                 return 0;
         }
-        afr_symlink (frame, this, linkpath, loc, params);
+        afr_symlink (frame, this, linkpath, loc, umask, xdata);
         return 0;
 
 }
@@ -1898,7 +1904,7 @@ static int32_t
 pump_rename (call_frame_t *frame,
              xlator_t *this,
              loc_t *oldloc,
-             loc_t *newloc)
+             loc_t *newloc, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1907,10 +1913,10 @@ pump_rename (call_frame_t *frame,
                             default_rename_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
                 return 0;
         }
-        afr_rename (frame, this, oldloc, newloc);
+        afr_rename (frame, this, oldloc, newloc, xdata);
         return 0;
 
 }
@@ -1920,7 +1926,7 @@ static int32_t
 pump_link (call_frame_t *frame,
            xlator_t *this,
            loc_t *oldloc,
-           loc_t *newloc)
+           loc_t *newloc, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1929,10 +1935,10 @@ pump_link (call_frame_t *frame,
                             default_link_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
                 return 0;
         }
-        afr_link (frame, this, oldloc, newloc);
+        afr_link (frame, this, oldloc, newloc, xdata);
         return 0;
 
 }
@@ -1941,7 +1947,7 @@ pump_link (call_frame_t *frame,
 static int32_t
 pump_create (call_frame_t *frame, xlator_t *this,
              loc_t *loc, int32_t flags, mode_t mode,
-             fd_t *fd, dict_t *params)
+             mode_t umask, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1949,10 +1955,10 @@ pump_create (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_create_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->create,
-                            loc, flags, mode, fd, params);
+                            loc, flags, mode, umask, fd, xdata);
                 return 0;
         }
-        afr_create (frame, this, loc, flags, mode, fd, params);
+        afr_create (frame, this, loc, flags, mode, umask, fd, xdata);
         return 0;
 
 }
@@ -1962,8 +1968,7 @@ static int32_t
 pump_open (call_frame_t *frame,
            xlator_t *this,
            loc_t *loc,
-           int32_t flags, fd_t *fd,
-           int32_t wbflags)
+           int32_t flags, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1972,10 +1977,10 @@ pump_open (call_frame_t *frame,
                             default_open_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->open,
-                            loc, flags, fd, wbflags);
+                            loc, flags, fd, xdata);
                 return 0;
         }
-        afr_open (frame, this, loc, flags, fd, wbflags);
+        afr_open (frame, this, loc, flags, fd, xdata);
         return 0;
 
 }
@@ -1988,7 +1993,7 @@ pump_writev (call_frame_t *frame,
              struct iovec *vector,
              int32_t count,
              off_t off, uint32_t flags,
-             struct iobref *iobref)
+             struct iobref *iobref, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2001,19 +2006,19 @@ pump_writev (call_frame_t *frame,
                             vector,
                             count,
                             off, flags,
-                            iobref);
+                            iobref, xdata);
                 return 0;
         }
-        afr_writev (frame, this, fd, vector, count, off, flags, iobref);
-        return 0;
 
+        afr_writev (frame, this, fd, vector, count, off, flags, iobref, xdata);
+        return 0;
 }
 
 
 static int32_t
 pump_flush (call_frame_t *frame,
             xlator_t *this,
-            fd_t *fd)
+            fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2022,10 +2027,10 @@ pump_flush (call_frame_t *frame,
                             default_flush_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->flush,
-                            fd);
+                            fd, xdata);
                 return 0;
         }
-        afr_flush (frame, this, fd);
+        afr_flush (frame, this, fd, xdata);
         return 0;
 
 }
@@ -2035,7 +2040,7 @@ static int32_t
 pump_fsync (call_frame_t *frame,
             xlator_t *this,
             fd_t *fd,
-            int32_t flags)
+            int32_t flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2045,10 +2050,10 @@ pump_fsync (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsync,
                             fd,
-                            flags);
+                            flags, xdata);
                 return 0;
         }
-        afr_fsync (frame, this, fd, flags);
+        afr_fsync (frame, this, fd, flags, xdata);
         return 0;
 
 }
@@ -2057,7 +2062,7 @@ pump_fsync (call_frame_t *frame,
 static int32_t
 pump_opendir (call_frame_t *frame,
               xlator_t *this,
-              loc_t *loc, fd_t *fd)
+              loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2066,10 +2071,10 @@ pump_opendir (call_frame_t *frame,
                             default_opendir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->opendir,
-                            loc, fd);
+                            loc, fd, xdata);
                 return 0;
         }
-        afr_opendir (frame, this, loc, fd);
+        afr_opendir (frame, this, loc, fd, xdata);
         return 0;
 
 }
@@ -2079,7 +2084,7 @@ static int32_t
 pump_fsyncdir (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
-               int32_t flags)
+               int32_t flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2089,10 +2094,10 @@ pump_fsyncdir (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsyncdir,
                             fd,
-                            flags);
+                            flags, xdata);
                 return 0;
         }
-        afr_fsyncdir (frame, this, fd, flags);
+        afr_fsyncdir (frame, this, fd, flags, xdata);
         return 0;
 
 }
@@ -2103,7 +2108,7 @@ pump_xattrop (call_frame_t *frame,
               xlator_t *this,
               loc_t *loc,
               gf_xattrop_flags_t flags,
-              dict_t *dict)
+              dict_t *dict, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2114,10 +2119,10 @@ pump_xattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->xattrop,
                             loc,
                             flags,
-                            dict);
+                            dict, xdata);
                 return 0;
         }
-        afr_xattrop (frame, this, loc, flags, dict);
+        afr_xattrop (frame, this, loc, flags, dict, xdata);
         return 0;
 
 }
@@ -2127,7 +2132,7 @@ pump_fxattrop (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
                gf_xattrop_flags_t flags,
-               dict_t *dict)
+               dict_t *dict, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2138,10 +2143,10 @@ pump_fxattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->fxattrop,
                             fd,
                             flags,
-                            dict);
+                            dict, xdata);
                 return 0;
         }
-        afr_fxattrop (frame, this, fd, flags, dict);
+        afr_fxattrop (frame, this, fd, flags, dict, xdata);
         return 0;
 
 }
@@ -2151,9 +2156,17 @@ static int32_t
 pump_removexattr (call_frame_t *frame,
                   xlator_t *this,
                   loc_t *loc,
-                  const char *name)
+                  const char *name, dict_t *xdata)
 {
-        afr_private_t *priv  = NULL;
+        afr_private_t *priv     = NULL;
+        int            op_errno = -1;
+
+        VALIDATE_OR_GOTO (this, out);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.pump*",
+                                 name, op_errno, out);
+
+        op_errno = 0;
 	priv = this->private;
         if (!priv->use_afr_in_pump) {
                 STACK_WIND (frame,
@@ -2161,10 +2174,14 @@ pump_removexattr (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->removexattr,
                             loc,
-                            name);
+                            name, xdata);
                 return 0;
         }
-        afr_removexattr (frame, this, loc, name);
+        afr_removexattr (frame, this, loc, name, xdata);
+
+ out:
+        if (op_errno)
+                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
         return 0;
 
 }
@@ -2176,7 +2193,7 @@ pump_readdir (call_frame_t *frame,
               xlator_t *this,
               fd_t *fd,
               size_t size,
-              off_t off)
+              off_t off, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2185,10 +2202,10 @@ pump_readdir (call_frame_t *frame,
                             default_readdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off);
+                            fd, size, off, xdata);
                 return 0;
         }
-        afr_readdir (frame, this, fd, size, off);
+        afr_readdir (frame, this, fd, size, off, xdata);
         return 0;
 
 }
@@ -2239,13 +2256,24 @@ pump_release (xlator_t *this,
 
 }
 
+static int32_t
+pump_forget (xlator_t *this, inode_t *inode)
+{
+        afr_private_t  *priv  = NULL;
+
+        priv = this->private;
+        if (priv->use_afr_in_pump)
+                afr_forget (this, inode);
+
+        return 0;
+}
 
 static int32_t
 pump_setattr (call_frame_t *frame,
               xlator_t *this,
               loc_t *loc,
               struct iatt *stbuf,
-              int32_t valid)
+              int32_t valid, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2254,10 +2282,10 @@ pump_setattr (call_frame_t *frame,
                             default_setattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, xdata);
                 return 0;
         }
-        afr_setattr (frame, this, loc, stbuf, valid);
+        afr_setattr (frame, this, loc, stbuf, valid, xdata);
         return 0;
 
 }
@@ -2268,7 +2296,7 @@ pump_fsetattr (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
                struct iatt *stbuf,
-               int32_t valid)
+               int32_t valid, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2277,10 +2305,10 @@ pump_fsetattr (call_frame_t *frame,
                             default_fsetattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid);
+                            fd, stbuf, valid, xdata);
                 return 0;
         }
-        afr_fsetattr (frame, this, fd, stbuf, valid);
+        afr_fsetattr (frame, this, fd, stbuf, valid, xdata);
         return 0;
 
 }
@@ -2329,7 +2357,7 @@ notify (xlator_t *this, int32_t event,
 
         child_xl = (xlator_t *) data;
 
-        ret = afr_notify (this, event, data);
+        ret = afr_notify (this, event, data, NULL);
 
 	switch (event) {
 	case GF_EVENT_CHILD_DOWN:
@@ -2380,7 +2408,10 @@ init (xlator_t *this)
 			"Volume is dangling.");
 	}
 
-	ALLOC_OR_GOTO (this->private, afr_private_t, out);
+	this->private = GF_CALLOC (1, sizeof (afr_private_t),
+                                   gf_afr_mt_afr_private_t);
+        if (!this->private)
+                goto out;
 
 	priv = this->private;
         LOCK_INIT (&priv->lock);
@@ -2406,14 +2437,13 @@ init (xlator_t *this)
 	priv->metadata_self_heal = 1;
 	priv->entry_self_heal    = 1;
 
-        priv->data_self_heal_algorithm = "";
-
         priv->data_self_heal_window_size = 16;
 
 	priv->data_change_log     = 1;
 	priv->metadata_change_log = 1;
 	priv->entry_change_log    = 1;
         priv->use_afr_in_pump = 1;
+        priv->sh_readdir_size = 65536;
 
 	/* Locking options */
 
@@ -2471,6 +2501,12 @@ init (xlator_t *this)
 		i++;
 	}
 
+        ret = gf_asprintf (&priv->sh_domain, "%s-self-heal", this->name);
+        if (-1 == ret) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
         priv->first_lookup = 1;
         priv->root_inode = NULL;
 
@@ -2502,7 +2538,7 @@ init (xlator_t *this)
                 goto out;
         }
 
-	pump_priv->env = syncenv_new (0);
+	pump_priv->env = this->ctx->env;
         if (!pump_priv->env) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Could not create new sync-environment");
@@ -2510,6 +2546,15 @@ init (xlator_t *this)
                 goto out;
         }
 
+        /* keep more local here as we may need them for self-heal etc */
+        this->local_pool = mem_pool_new (afr_local_t, 128);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
 	priv->pump_private = pump_priv;
 
         pump_change_state (this, PUMP_STATE_ABORT);
@@ -2534,9 +2579,6 @@ fini (xlator_t *this)
         if (!pump_priv)
                 goto afr_priv;
 
-        if (pump_priv->env)
-                syncenv_destroy (pump_priv->env);
-
         GF_FREE (pump_priv->resume_path);
         LOCK_DESTROY (&pump_priv->resume_path_lock);
         LOCK_DESTROY (&pump_priv->pump_state_lock);
@@ -2591,6 +2633,7 @@ struct xlator_dumpops dumpops = {
 struct xlator_cbks cbks = {
 	.release     = pump_release,
 	.releasedir  = pump_releasedir,
+        .forget      = pump_forget,
 };
 
 struct volume_options options[] = {
diff --git a/xlators/cluster/afr/src/pump.h b/xlators/cluster/afr/src/pump.h
index 4213a635e..bc4c31a78 100644
--- a/xlators/cluster/afr/src/pump.h
+++ b/xlators/cluster/afr/src/pump.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __PUMP_H__
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index e35058d65..174bea841 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -4,7 +4,7 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
 dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c dht-rebalance.c \
 	dht-selfheal.c dht-rename.c dht-hashfn.c dht-diskusage.c \
-	dht-common.c dht-inode-write.c dht-inode-read.c \
+	dht-common.c dht-inode-write.c dht-inode-read.c dht-shared.c \
 	$(top_builddir)/xlators/lib/src/libxlator.c
 
 dht_la_SOURCES = $(dht_common_source) dht.c
@@ -12,22 +12,23 @@ dht_la_SOURCES = $(dht_common_source) dht.c
 nufa_la_SOURCES = $(dht_common_source) nufa.c
 switch_la_SOURCES = $(dht_common_source) switch.c
 
-dht_la_LDFLAGS = -module -avoidversion
+dht_la_LDFLAGS = -module -avoid-version
 dht_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-nufa_la_LDFLAGS = -module -avoidversion
+nufa_la_LDFLAGS = -module -avoid-version
 nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-switch_la_LDFLAGS = -module -avoidversion
+switch_la_LDFLAGS = -module -avoid-version
 switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = dht-common.h dht-mem-types.h \
 	$(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/lib/src
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
 CLEANFILES =
 
 uninstall-local:
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 29b3dca83..8f61339e6 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -31,17 +22,18 @@
 #include "dht-common.h"
 #include "defaults.h"
 #include "byte-order.h"
+#include "glusterfs-acl.h"
 
 #include <sys/time.h>
 #include <libgen.h>
 
-void
+int
 dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t  *dst  = NULL;
         int64_t *ptr  = 0, *size = NULL;
         int32_t  ret  = -1;
-        data_pair_t  *data_pair = NULL;
+        data_t  *dict_data = NULL;
 
         dst = data;
 
@@ -53,32 +45,37 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
                         if (size == NULL) {
                                 gf_log ("dht", GF_LOG_WARNING,
                                         "memory allocation failed");
-                                return;
+                                return -1;
                         }
                         ret = dict_set_bin (dst, key, size, sizeof (int64_t));
                         if (ret < 0) {
                                 gf_log ("dht", GF_LOG_WARNING,
                                         "dht aggregate dict set failed");
                                 GF_FREE (size);
-                                return;
+                                return -1;
                         }
                 }
 
                 ptr = data_to_bin (value);
                 if (ptr == NULL) {
                         gf_log ("dht", GF_LOG_WARNING, "data to bin failed");
-                        return;
+                        return -1;
                 }
 
                 *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
+                ret = gf_get_min_stime (THIS, dst, key, value);
+                if (ret < 0)
+                        return ret;
         } else {
                 /* compare user xattrs only */
                 if (!strncmp (key, "user.", strlen ("user."))) {
-                        ret = dict_lookup (dst, key, &data_pair); 
-                        if (!ret && data_pair && value) {
-                                ret = is_data_equal (data_pair->value, value);
+                        ret = dict_lookup (dst, key, &dict_data);
+                        if (!ret && dict_data && value) {
+                                ret = is_data_equal (dict_data, value);
                                 if (!ret)
-                                        gf_log ("dht", GF_LOG_WARNING,
+                                        gf_log ("dht", GF_LOG_DEBUG,
                                                 "xattr mismatch for %s", key);
                         }
                 }
@@ -87,7 +84,7 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
                         gf_log ("dht", GF_LOG_WARNING, "xattr dict set failed");
         }
 
-        return;
+        return 0;
 }
 
 
@@ -114,7 +111,7 @@ out:
 int
 dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
                          xlator_t *this,
-                         int op_ret, int op_errno)
+                         int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
@@ -134,7 +131,10 @@ dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
                 ret = dht_layout_set (this, local->inode, layout);
         }
 
-        WIPE (&local->postparent);
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           &local->postparent, 1);
+        }
 
         DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
 
@@ -154,9 +154,11 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
         int              op_errno = 0;
         int              ret = -1;
         dht_layout_t    *layout = NULL;
+        dht_conf_t      *conf = NULL;
 
         local = discover_frame->local;
         layout = local->layout;
+        conf = this->private;
 
         LOCK(&discover_frame->lock);
         {
@@ -190,16 +192,23 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
                 }
         } else {
                 ret = dht_layout_normalize (this, &local->loc, layout);
-
-                if (ret != 0) {
+                if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
+                        /* either the layout is incorrect or the directory is
+                         * not found even in one subvolume.
+                         */
                         gf_log (this->name, GF_LOG_DEBUG,
-                                "normalizing failed on %s",
-                                local->loc.path);
-                        op_errno = EINVAL;
-                        goto out;
+                                "normalizing failed on %s "
+                                "(overlaps/holes present: %s, "
+                                "ENOENT errors: %d)", local->loc.path,
+                                (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
+                        if ((ret > 0) && (ret == conf->subvolume_cnt)) {
+                                op_errno = ESTALE;
+                                goto out;
+                        }
                 }
 
-                dht_layout_set (this, local->inode, layout);
+                if (local->inode)
+                        dht_layout_set (this, local->inode, layout);
         }
 
         DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
@@ -228,6 +237,7 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int           is_dir                  = 0;
         int           is_linkfile             = 0;
         int           attempt_unwind          = 0;
+        dht_conf_t   *conf                    = 0;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, out);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -237,6 +247,7 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local = frame->local;
         prev  = cookie;
+        conf = this->private;
 
         layout = local->layout;
 
@@ -271,7 +282,8 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto unlock;
                 }
 
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
                 is_dir = check_is_dir (inode, stbuf, xattr);
 
                 if (is_dir) {
@@ -330,23 +342,20 @@ dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
         int          i = 0;
         call_frame_t *discover_frame = NULL;
 
-
         conf = this->private;
         local = frame->local;
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht", 4 * 4);
+        ret = dict_set_uint32 (local->xattr_req, conf->xattr_name, 4 * 4);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht' key",
-                        loc->path);
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->xattr_name);
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht.linkto", 256);
+        ret = dict_set_uint32 (local->xattr_req, conf->link_xattr_name, 256);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht.linkto' key",
-                        loc->path);
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->link_xattr_name);
 
         call_cnt        = conf->subvolume_cnt;
         local->call_cnt = call_cnt;
@@ -380,7 +389,8 @@ dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
         return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
 
         return 0;
 }
@@ -431,7 +441,7 @@ dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         op_ret, op_errno, xattr);
 
                 if (op_ret == -1) {
-                        local->op_errno = ENOENT;
+                        local->op_errno = op_errno;
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "lookup of %s on %s returned error (%s)",
                                 local->loc.path, prev->this->name,
@@ -490,6 +500,11 @@ unlock:
                         dht_layout_set (this, local->inode, layout);
                 }
 
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
                                   local->inode, &local->stbuf, local->xattr,
@@ -521,6 +536,8 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int           ret  = -1;
         int           is_dir = 0;
         int           is_linkfile = 0;
+        call_frame_t *copy          = NULL;
+        dht_local_t  *copy_local    = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, err);
@@ -579,7 +596,8 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 layout = local->layout;
 
                 is_dir = check_is_dir (inode, stbuf, xattr);
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
 
                 if (is_linkfile) {
                         gf_log (this->name, GF_LOG_INFO,
@@ -591,6 +609,23 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
 
                 if (is_dir) {
+                        ret = dht_dir_has_layout (xattr, conf->xattr_name);
+                        if (ret >= 0) {
+                                if (is_greater_time(local->stbuf.ia_ctime,
+                                                    local->stbuf.ia_ctime_nsec,
+                                                    stbuf->ia_ctime,
+                                                    stbuf->ia_ctime_nsec)) {
+                                        local->prebuf.ia_gid = stbuf->ia_gid;
+                                        local->prebuf.ia_uid = stbuf->ia_uid;
+                                }
+                        }
+                        if (local->stbuf.ia_type != IA_INVAL)
+                        {
+                                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                                    (local->stbuf.ia_uid != stbuf->ia_uid)) {
+                                        local->need_selfheal = 1;
+                                }
+                        }
                         ret = dht_layout_dir_mismatch (this, layout,
                                                        prev->this, &local->loc,
                                                        xattr);
@@ -629,7 +664,28 @@ out:
                     && (conf && conf->unhashed_sticky_bit)) {
                         local->stbuf.ia_prot.sticky = 1;
                 }
-                if (local->layout_mismatch) {
+                if (local->need_selfheal) {
+                        local->need_selfheal = 0;
+                        uuid_copy (local->gfid, local->stbuf.ia_gfid);
+                        local->stbuf.ia_gid = local->prebuf.ia_gid;
+                        local->stbuf.ia_uid = local->prebuf.ia_uid;
+                        copy = create_frame (this, this->ctx->pool);
+                        if (copy) {
+                                copy_local = dht_local_init (copy, &local->loc,
+                                                             NULL, 0);
+                                if (!copy_local)
+                                        goto cont;
+                                copy_local->stbuf = local->stbuf;
+                                copy->local = copy_local;
+                                FRAME_SU_DO (copy, dht_local_t);
+                                ret = synctask_new (this->ctx->env,
+                                                    dht_dir_attr_heal,
+                                                    dht_dir_attr_heal_done,
+                                                    copy, copy);
+                        }
+                }
+cont:
+		if (local->layout_mismatch) {
                         /* Found layout mismatch in the directory, need to
                            fix this in the inode context */
                         dht_layout_unref (this, local->layout);
@@ -655,7 +711,10 @@ out:
                         local->op_errno = ESTALE;
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
@@ -673,7 +732,8 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                 xlator_t *this,
                                 int32_t op_ret, int32_t op_errno,
                                 inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent)
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -706,8 +766,14 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                 local->stbuf.ia_prot.sticky = 1;
         }
 
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+
 unwind:
-        WIPE (&local->postparent);
+        if (local->linked == _gf_true)
+                dht_linkfile_attr_heal (frame, this);
 
         DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
         DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
@@ -780,7 +846,10 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                                                   "<nil>"));
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret,
@@ -810,7 +879,10 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                         local->op_errno = EINVAL;
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret,
@@ -826,7 +898,7 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                 hashed_subvol->name);
 
         ret = dht_linkfile_create (frame,
-                                   dht_lookup_linkfile_create_cbk,
+                                   dht_lookup_linkfile_create_cbk, this,
                                    cached_subvol, hashed_subvol, &local->loc);
 
         return ret;
@@ -836,7 +908,8 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
 int
 dht_lookup_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno,
-                       struct iatt *preparent, struct iatt *postparent)
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
 {
         int  this_call_cnt = 0;
 
@@ -863,8 +936,9 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         xlator_t     *subvol        = NULL;
         loc_t        *loc           = NULL;
         xlator_t     *link_subvol   = NULL;
-        int           ret = -1;
-        int32_t       fd_count = 0;
+        int           ret           = -1;
+        int32_t       fd_count      = 0;
+        dht_conf_t   *conf          = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, out);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -874,6 +948,7 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local  = frame->local;
         loc    = &local->loc;
+        conf   = this->private;
 
         prev   = cookie;
         subvol = prev->this;
@@ -895,7 +970,8 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 loc->path, prev->this->name);
                 }
 
-                is_linkfile = check_is_linkfile (inode, buf, xattr);
+                is_linkfile = check_is_linkfile (inode, buf, xattr,
+                                                 conf->link_xattr_name);
                 is_dir = check_is_dir (inode, buf, xattr);
 
                 if (is_linkfile) {
@@ -955,7 +1031,7 @@ unlock:
                                 "deleting stale linkfile %s on %s",
                                 loc->path, subvol->name);
                         STACK_WIND (frame, dht_lookup_unlink_cbk,
-                                    subvol, subvol->fops->unlink, loc);
+                                    subvol, subvol->fops->unlink, loc, 0, NULL);
                         return 0;
                 }
         }
@@ -1037,7 +1113,16 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 gf_log (this->name, GF_LOG_INFO,
                         "lookup of %s on %s (following linkfile) failed (%s)",
                         local->loc.path, subvol->name, strerror (op_errno));
-                goto err;
+
+                /* If cached subvol returned ENOTCONN, do not do
+                lookup_everywhere. We need to make sure linkfile does not get
+                removed, which can take away the namespace, and subvol is
+                anyways down. */
+
+                if (op_errno != ENOTCONN)
+                        goto err;
+                else
+                        goto unwind;
         }
 
         if (check_is_dir (inode, stbuf, xattr)) {
@@ -1047,7 +1132,7 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 goto err;
         }
 
-        if (check_is_linkfile (inode, stbuf, xattr)) {
+        if (check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
                 gf_log (this->name, GF_LOG_INFO,
                         "lookup of %s on %s (following linkfile) reached link",
                         local->loc.path, subvol->name);
@@ -1075,9 +1160,12 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 op_errno = EINVAL;
         }
 
-unwind:
-        WIPE (postparent);
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
+unwind:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
                           postparent);
@@ -1159,7 +1247,6 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         loc_t        *loc           = NULL;
         call_frame_t *prev          = NULL;
         int           ret           = 0;
-        uint64_t      tmp_layout    = 0;
         dht_layout_t *parent_layout = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
@@ -1190,8 +1277,10 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
                 if ((conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) &&
                     (loc->parent)) {
-                        ret = inode_ctx_get (loc->parent, this, &tmp_layout);
-                        parent_layout = (dht_layout_t *)(long)tmp_layout;
+                        ret = dht_inode_ctx_layout_get (loc->parent, this,
+                                                        &parent_layout);
+                        if (ret || !parent_layout)
+                                goto out;
                         if (parent_layout->search_unhashed) {
                                 local->op_errno = ENOENT;
                                 dht_lookup_everywhere (frame, this, loc);
@@ -1220,7 +1309,8 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
 
         if (!is_linkfile) {
                 /* non-directory and not a linkfile */
@@ -1260,7 +1350,10 @@ out:
          * from each of the subvolume. See dht_iatt_merge for reference.
          */
 
-        WIPE (postparent);
+        if (!op_ret && local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
@@ -1269,6 +1362,39 @@ err:
         return 0;
 }
 
+/* For directories, check if acl xattrs have been requested (by the acl xlator),
+ * if not, request for them. These xattrs are needed for dht dir self-heal to
+ * perform proper self-healing of dirs
+ */
+void
+dht_check_and_set_acl_xattr_req (inode_t *inode, dict_t *xattr_req)
+{
+        int     ret = 0;
+
+        GF_ASSERT (inode);
+        GF_ASSERT (xattr_req);
+
+        if (inode->ia_type != IA_IFDIR)
+                return;
+
+        if (!dict_get (xattr_req, POSIX_ACL_ACCESS_XATTR)) {
+                ret = dict_set_int8 (xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set key %s",
+                                POSIX_ACL_ACCESS_XATTR);
+        }
+
+        if (!dict_get (xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
+                ret = dict_set_int8 (xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set key %s",
+                                POSIX_ACL_DEFAULT_XATTR);
+        }
+
+        return;
+}
 
 int
 dht_lookup (call_frame_t *frame, xlator_t *this,
@@ -1289,7 +1415,6 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         conf = this->private;
         if (!conf)
@@ -1364,7 +1489,7 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                  *       revalidates directly go to the cached-subvolume.
                  */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
 
                 if (IA_ISDIR (local->inode->ia_type)) {
                         local->call_cnt = call_cnt = conf->subvolume_cnt;
@@ -1384,6 +1509,9 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                 ret = dict_set_uint32 (local->xattr_req,
                                        GLUSTERFS_OPEN_FD_COUNT, 4);
 
+                /* need it for dir self-heal */
+                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+
 		for (i = 0; i < call_cnt; i++) {
 			subvol = layout->list[i].xlator;
 
@@ -1396,16 +1524,19 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         do_fresh_lookup:
                 /* TODO: remove the hard-coding */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
 
                 ret = dict_set_uint32 (local->xattr_req,
-                                       DHT_LINKFILE_KEY, 256);
+                                       conf->link_xattr_name, 256);
 
                 /* need it for self-healing linkfiles which is
                    'in-migration' state */
                 ret = dict_set_uint32 (local->xattr_req,
                                        GLUSTERFS_OPEN_FD_COUNT, 4);
 
+                /* need it for dir self-heal */
+                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+
                 if (!hashed_subvol) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "no subvolume in layout for path=%s, "
@@ -1439,7 +1570,8 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
         return 0;
 }
 
@@ -1447,7 +1579,7 @@ err:
 int
 dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev  = NULL;
@@ -1471,14 +1603,18 @@ dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->postparent = *postparent;
                 local->preparent = *preparent;
 
-                WIPE (&local->postparent);
-                WIPE (&local->preparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
         }
 unlock:
         UNLOCK (&frame->lock);
 
         DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
 
         return 0;
 }
@@ -1487,7 +1623,7 @@ unlock:
 int
 dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -1499,7 +1635,8 @@ dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == -1) {
+                if ((op_ret == -1) && !((op_errno == ENOENT) ||
+                                        (op_errno == ENOTCONN))) {
                         local->op_errno = op_errno;
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "subvolume %s returned -1 (%s)",
@@ -1512,7 +1649,7 @@ dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unlock:
         UNLOCK (&frame->lock);
 
-        if (op_ret == -1)
+        if (local->op_ret == -1)
                 goto err;
 
         cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
@@ -1526,53 +1663,19 @@ unlock:
 
         STACK_WIND (frame, dht_unlink_cbk,
                     cached_subvol, cached_subvol->fops->unlink,
-                    &local->loc);
+                    &local->loc, local->flags, NULL);
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL);
-        return 0;
-}
-
-static int
-dht_ufo_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, local->op_errno);
-        }
-
+                          NULL, NULL, NULL);
         return 0;
 }
 
-
 int
 dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int op_ret, int op_errno)
+             int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -1598,7 +1701,8 @@ unlock:
 
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, local->op_errno);
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
+                                  local->op_errno, NULL);
         }
 
         return 0;
@@ -1621,114 +1725,249 @@ fill_layout_info (dht_layout_t *layout, char *buf)
         }
 }
 
+void
+dht_fill_pathinfo_xattr (xlator_t *this, dht_local_t *local,
+                         char *xattr_buf, int32_t alloc_len,
+                         int flag, char *layout_buf)
+{
+        if (flag && local->xattr_val)
+                snprintf (xattr_buf, alloc_len,
+                          "((<"DHT_PATHINFO_HEADER"%s> %s) (%s-layout %s))",
+                          this->name, local->xattr_val, this->name,
+                          layout_buf);
+        else if (local->xattr_val)
+                snprintf (xattr_buf, alloc_len,
+                          "(<"DHT_PATHINFO_HEADER"%s> %s)",
+                          this->name, local->xattr_val);
+        else if (flag)
+                snprintf (xattr_buf, alloc_len, "(%s-layout %s)",
+                          this->name, layout_buf);
+}
+
 int
-dht_pathinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+dht_vgetxattr_alloc_and_fill (dht_local_t *local, dict_t *xattr, xlator_t *this,
+                              int op_errno)
+{
+        int      ret       = -1;
+        char    *value     = NULL;
+        int32_t  plen      = 0;
+
+        ret = dict_get_str (xattr, local->xsel, &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Subvolume %s returned -1 (%s)", this->name,
+                        strerror (op_errno));
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                goto out;
+        }
+
+        local->alloc_len += strlen(value);
+
+        if (!local->xattr_val) {
+                local->alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
+                local->xattr_val = GF_CALLOC (local->alloc_len, sizeof (char),
+                                              gf_common_mt_char);
+                if (!local->xattr_val) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (local->xattr_val) {
+                plen = strlen (local->xattr_val);
+                if (plen) {
+                        /* extra byte(s) for \0 to be safe */
+                        local->alloc_len += (plen + 2);
+                        local->xattr_val = GF_REALLOC (local->xattr_val,
+                                                       local->alloc_len);
+                        if (!local->xattr_val) {
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                (void) strcat (local->xattr_val, value);
+                (void) strcat (local->xattr_val, " ");
+                local->op_ret = 0;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int
+dht_vgetxattr_fill_and_set (dht_local_t *local, dict_t **dict, xlator_t *this,
+                            gf_boolean_t flag)
+{
+        int   ret             = -1;
+        char *xattr_buf       = NULL;
+        char layout_buf[8192] = {0,};
+
+        if (flag)
+                fill_layout_info (local->layout, layout_buf);
+
+        *dict = dict_new ();
+        if (!*dict)
+                goto out;
+
+        local->xattr_val[strlen (local->xattr_val) - 1] = '\0';
+
+        /* we would need max this many bytes to create xattr string
+         * extra 40 bytes is just an estimated amount of additional
+         * space required as we include translator name and some
+         * spaces, brackets etc. when forming the pathinfo string.
+         *
+         * For node-uuid we just don't have all the pretty formatting,
+         * but since this is a generic routine for pathinfo & node-uuid
+         * we dont have conditional space allocation and try to be
+         * generic
+         */
+        local->alloc_len += (2 * strlen (this->name))
+                + strlen (layout_buf)
+                + 40;
+        xattr_buf = GF_CALLOC (local->alloc_len, sizeof (char),
+                               gf_common_mt_char);
+        if (!xattr_buf)
+                goto out;
+
+        if (XATTR_IS_PATHINFO (local->xsel)) {
+                (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
+                                                local->alloc_len, flag,
+                                                layout_buf);
+        } else if (XATTR_IS_NODE_UUID (local->xsel)) {
+                (void) snprintf (xattr_buf, local->alloc_len, "%s",
+                                 local->xattr_val);
+        } else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unknown local->xsel (%s)", local->xsel);
+                goto out;
+        }
+
+        ret = dict_set_dynstr (*dict, local->xsel, xattr_buf);
+        GF_FREE (local->xattr_val);
+
+ out:
+        return ret;
+}
+
+int
+dht_vgetxattr_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        dht_local_t *local         = NULL;
         int          ret           = 0;
-        int          flag          = 0;
+        dht_local_t *local         = NULL;
         int          this_call_cnt = 0;
-        char        *value_got     = NULL;
-        char  layout_buf[8192]     = {0,};
-        char        *xattr_buf     = NULL;
         dict_t      *dict          = NULL;
-        int32_t      alloc_len     = 0;
-        int32_t      plen          = 0;
 
-        local = frame->local;
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
 
-        if (op_ret != -1) {
-                ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value_got);
-                if (!ret) {
-                        alloc_len = strlen (value_got);
+        local = frame->local;
 
-                        /**
-                         * allocate the buffer:- we allocate 10 bytes extra in case we need to
-                         * append ' Link: ' in the buffer for another STACK_WIND
-                         */
-                        if (!local->pathinfo) {
-                                alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
-                                local->pathinfo = GF_CALLOC (alloc_len, sizeof (char), gf_common_mt_char);
+        LOCK (&frame->lock);
+        {
+                this_call_cnt = --local->call_cnt;
+                if (op_ret < 0) {
+                        if (op_errno != ENOTCONN) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "getxattr err (%s) for dir",
+                                        strerror (op_errno));
+				local->op_ret = -1;
+				local->op_errno = op_errno;
                         }
 
-                        if (local->pathinfo) {
-                                plen = strlen (local->pathinfo);
-                                if (plen) {
-                                        /* extra byte(s) for \0 to be safe */
-                                        alloc_len += (plen + 2);
-                                        local->pathinfo = GF_REALLOC (local->pathinfo,
-                                                                      alloc_len);
-                                        if (!local->pathinfo)
-                                                goto out;
-                                }
-
-                                strcat (local->pathinfo, value_got);
-                        }
+                        goto unlock;
                 }
+
+                ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                                    op_errno);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "alloc or fill failure");
         }
+ unlock:
+        UNLOCK (&frame->lock);
 
- out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                if (local->layout->cnt > 1) {
-                        /* Set it for directory */
-                        fill_layout_info (local->layout, layout_buf);
-                        flag = 1;
-                }
+        if (!is_last_call (this_call_cnt))
+                goto out;
 
-                dict = dict_new ();
+        /* -- last call: do patch ups -- */
 
-                /* we would need max-to-max this many bytes to create pathinfo string */
-                alloc_len += (2 * strlen (this->name)) + strlen (layout_buf) + 40;
-                xattr_buf = GF_CALLOC (alloc_len, sizeof (char), gf_common_mt_char);
+        if (local->op_ret == -1) {
+                goto unwind;
+        }
 
-                if (flag && local->pathinfo)
-                        snprintf (xattr_buf, alloc_len, "((<"DHT_PATHINFO_HEADER"%s> %s) (%s-layout %s))",
-                                  this->name, local->pathinfo, this->name,
-                                  layout_buf);
-                else if (local->pathinfo)
-                        snprintf (xattr_buf, alloc_len, "(<"DHT_PATHINFO_HEADER"%s> %s)",
-                                  this->name, local->pathinfo);
-                else if (flag)
-                        snprintf (xattr_buf, alloc_len, "(%s-layout %s)",
-                                  this->name, layout_buf);
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, _gf_true);
+        if (ret)
+                goto unwind;
 
-                ret = dict_set_dynstr (dict, GF_XATTR_PATHINFO_KEY,
-                                       xattr_buf);
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
 
-                if (local->pathinfo)
-                        GF_FREE (local->pathinfo);
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
+ out:
+        return 0;
+}
 
-                DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+int
+dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        dht_local_t  *local         = NULL;
+        int           ret           = 0;
+        dict_t       *dict          = NULL;
+        call_frame_t *prev          = NULL;
+        gf_boolean_t  flag          = _gf_true;
 
-                if (dict)
-                        dict_unref (dict);
+        local = frame->local;
+        prev = cookie;
 
-                return 0;
+        if (op_ret < 0) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                gf_log (this->name, GF_LOG_ERROR, "Subvolume %s returned -1 "
+                        "(%s)", prev->this->name, strerror (op_errno));
+                goto unwind;
         }
 
-        if (local->pathinfo)
-                strcat (local->pathinfo, " Link: ");
-        if (local->hashed_subvol) {
-                /* This will happen if there pending */
-                STACK_WIND (frame, dht_pathinfo_getxattr_cbk, local->hashed_subvol,
-                            local->hashed_subvol->fops->getxattr,
-                            &local->loc, local->key);
-
-                return 0;
+        ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                            op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "alloc or fill failure");
+                goto unwind;
         }
 
-        gf_log ("this->name", GF_LOG_ERROR, "Unable to find hashed_subvol for path"
-                " %s", local->pathinfo);
+        flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
+
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, flag);
+        if (ret)
+                goto unwind;
+
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
+
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno,
+                          NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
 
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, dict);
         return 0;
 }
 
 int
 dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+                           int op_ret, int op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
         int   ret   = 0;
         char *value = NULL;
@@ -1743,21 +1982,24 @@ dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
         }
 
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
+        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
         return 0;
 }
 
 int
 dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xattr)
+                  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
         int             this_call_cnt = 0;
         dht_local_t     *local = NULL;
+        dht_conf_t      *conf = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (frame->local, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
+        conf = this->private;
         local = frame->local;
 
         this_call_cnt = dht_frame_return (frame);
@@ -1765,8 +2007,8 @@ dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!xattr || (op_ret == -1))
                 goto out;
 
-        if (dict_get (xattr, "trusted.glusterfs.dht")) {
-                dict_del (xattr, "trusted.glusterfs.dht");
+        if (dict_get (xattr, conf->xattr_name)) {
+                dict_del (xattr, conf->xattr_name);
         }
         local->op_ret = 0;
 
@@ -1783,24 +2025,88 @@ dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 out:
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno, local->xattr);
+                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+                                  local->xattr, NULL);
         }
         return 0;
 }
 
 int32_t
 dht_getxattr_unwind (call_frame_t *frame,
-                     int op_ret, int op_errno, dict_t *dict)
+                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
+dht_getxattr_get_real_filename_cbk (call_frame_t *frame, void *cookie,
+				    xlator_t *this, int op_ret, int op_errno,
+				    dict_t *xattr, dict_t *xdata)
+{
+        int             this_call_cnt = 0;
+        dht_local_t     *local = NULL;
+
+
+        local = frame->local;
+
+	if (op_ret != -1) {
+		if (local->xattr)
+			dict_unref (local->xattr);
+		local->xattr = dict_ref (xattr);
+
+		if (local->xattr_req)
+			dict_unref (local->xattr_req);
+		local->xattr_req = dict_ref (xdata);
+	}
+
+	this_call_cnt = dht_frame_return (frame);
+	if (is_last_call (this_call_cnt)) {
+		DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+				  local->xattr, local->xattr_req);
+	}
+
+	return 0;
+}
+
+
+int
+dht_getxattr_get_real_filename (call_frame_t *frame, xlator_t *this,
+				loc_t *loc, const char *key, dict_t *xdata)
+{
+	dht_local_t     *local = NULL;
+	int              i = 0;
+	dht_layout_t    *layout = NULL;
+	int              cnt = 0;
+	xlator_t        *subvol = NULL;
+
+
+	local = frame->local;
+	layout = local->layout;
+
+	cnt = local->call_cnt = layout->cnt;
+
+	local->op_ret = -1;
+	local->op_errno = ENODATA;
+
+	for (i = 0; i < cnt; i++) {
+		subvol = layout->list[i].xlator;
+		STACK_WIND (frame, dht_getxattr_get_real_filename_cbk,
+			    subvol, subvol->fops->getxattr,
+			    loc, key, xdata);
+	}
+
+	return 0;
+}
+
+
+int
 dht_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *key)
+              loc_t *loc, const char *key, dict_t *xdata)
+#define DHT_IS_DIR(layout)  (layout->cnt > 1)
 {
+
         xlator_t     *subvol        = NULL;
         xlator_t     *hashed_subvol = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -1816,7 +2122,6 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf   = this->private;
@@ -1844,24 +2149,67 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (key && (strcmp (key, GF_XATTR_PATHINFO_KEY) == 0)) {
-                hashed_subvol = dht_subvol_get_hashed (this, loc);
+	if (key &&
+	    (strncmp (key, GF_XATTR_GET_REAL_FILENAME_KEY,
+		      strlen (GF_XATTR_GET_REAL_FILENAME_KEY)) == 0)
+	    && DHT_IS_DIR(layout)) {
+		dht_getxattr_get_real_filename (frame, this, loc, key, xdata);
+		return 0;
+	}
+
+        /* for file use cached subvolume (obviously!): see if {}
+         * below
+         * for directory:
+         *  wind to all subvolumes and exclude subvolumes which
+         *  return ENOTCONN (in callback)
+         *
+         * NOTE: Don't trust inode here, as that may not be valid
+         *       (until inode_link() happens)
+         */
+        if (key && DHT_IS_DIR(layout) &&
+            ((strcmp (key, GF_XATTR_PATHINFO_KEY) == 0)
+             || (strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0))) {
+                (void) strncpy (local->xsel, key, 256);
+                cnt = local->call_cnt = layout->cnt;
+                for (i = 0; i < cnt; i++) {
+                        subvol = layout->list[i].xlator;
+                        STACK_WIND (frame, dht_vgetxattr_dir_cbk,
+                                    subvol, subvol->fops->getxattr,
+                                    loc, key, NULL);
+                }
+                return 0;
+        }
+
+        /* node-uuid or pathinfo for files */
+        if (key && ((strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)
+                    || (strcmp (key, GF_XATTR_PATHINFO_KEY) == 0))) {
                 cached_subvol = local->cached_subvol;
+                (void) strncpy (local->xsel, key, 256);
 
                 local->call_cnt = 1;
-                if (hashed_subvol != cached_subvol) {
-                        local->call_cnt = 2;
-                        local->hashed_subvol = hashed_subvol;
-                }
-
-                STACK_WIND (frame, dht_pathinfo_getxattr_cbk, cached_subvol,
-                            cached_subvol->fops->getxattr, loc, key);
+                STACK_WIND (frame, dht_vgetxattr_cbk, cached_subvol,
+                            cached_subvol->fops->getxattr, loc, key, NULL);
 
                 return 0;
         }
+
         if (key && (strcmp (key, GF_XATTR_LINKINFO_KEY) == 0)) {
                 hashed_subvol = dht_subvol_get_hashed (this, loc);
+                if (!hashed_subvol) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get"
+                                "hashed subvol for %s", loc->path);
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
                 cached_subvol = dht_subvol_get_cached (this, loc->inode);
+                if (!cached_subvol) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get"
+                                "cached subvol for %s", loc->path);
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
                 if (hashed_subvol == cached_subvol) {
                         op_errno = ENODATA;
                         goto err;
@@ -1869,7 +2217,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 if (hashed_subvol) {
                         STACK_WIND (frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
                                     hashed_subvol->fops->getxattr, loc,
-                                    GF_XATTR_PATHINFO_KEY);
+                                    GF_XATTR_PATHINFO_KEY, NULL);
                         return 0;
                 }
                 op_errno = ENODATA;
@@ -1877,13 +2225,13 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
         }
 
         if (key && (!strcmp (GF_XATTR_MARKER_KEY, key))
-            && (-1 == frame->root->pid)) {
-
-                if (loc->inode-> ia_type == IA_IFDIR) {
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                if (DHT_IS_DIR(layout)) {
                         cnt = layout->cnt;
                 } else {
                         cnt = 1;
                 }
+
                 sub_volumes = alloca ( cnt * sizeof (xlator_t *));
                 for (i = 0; i < cnt; i++)
                         *(sub_volumes + i) = layout->list[i].xlator;
@@ -1891,7 +2239,8 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 if (cluster_getmarkerattr (frame, this, loc, key,
                                            local, dht_getxattr_unwind,
                                            sub_volumes, cnt,
-                                           MARKER_UUID_TYPE, conf->vol_uuid)) {
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           conf->vol_uuid)) {
                         op_errno = EINVAL;
                         goto err;
                 }
@@ -1901,8 +2250,8 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
 
         if (key && *conf->vol_uuid) {
                 if ((match_uuid_local (key, conf->vol_uuid) == 0) &&
-                    (-1 == frame->root->pid)) {
-                        if (loc->inode-> ia_type == IA_IFDIR) {
+                    (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                        if (DHT_IS_DIR(layout)) {
                                 cnt = layout->cnt;
                         } else {
                                 cnt = 1;
@@ -1915,6 +2264,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                                                    local, dht_getxattr_unwind,
                                                    sub_volumes, cnt,
                                                    MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
                                                    conf->vol_uuid)) {
                                 op_errno = EINVAL;
                                 goto err;
@@ -1924,7 +2274,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (loc->inode-> ia_type == IA_IFDIR) {
+        if (DHT_IS_DIR(layout)) {
                 cnt = local->call_cnt = layout->cnt;
         } else {
                 cnt = local->call_cnt  = 1;
@@ -1934,20 +2284,21 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 subvol = layout->list[i].xlator;
                 STACK_WIND (frame, dht_getxattr_cbk,
                             subvol, subvol->fops->getxattr,
-                            loc, key);
+                            loc, key, NULL);
         }
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
+#undef DHT_IS_DIR
 
 int
 dht_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *key)
+               fd_t *fd, const char *key, dict_t *xdata)
 {
         xlator_t     *subvol        = NULL;
         dht_local_t  *local         = NULL;
@@ -1985,7 +2336,9 @@ dht_fgetxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (fd->inode->ia_type == IA_IFDIR) {
+        if ((fd->inode->ia_type == IA_IFDIR)
+            && (strncmp (key, GF_XATTR_LOCKINFO_KEY,
+                         strlen (GF_XATTR_LOCKINFO_KEY) != 0))) {
                 cnt = local->call_cnt = layout->cnt;
         } else {
                 cnt = local->call_cnt  = 1;
@@ -1995,33 +2348,36 @@ dht_fgetxattr (call_frame_t *frame, xlator_t *this,
                 subvol = layout->list[i].xlator;
                 STACK_WIND (frame, dht_getxattr_cbk,
                             subvol, subvol->fops->fgetxattr,
-                            fd, key);
+                            fd, key, NULL);
         }
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *xattr, int flags)
+               fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
 {
         xlator_t     *subvol   = NULL;
         dht_local_t  *local    = NULL;
         int           op_errno = EINVAL;
-        data_pair_t  *trav     = NULL;
+        dht_conf_t   *conf     = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
         VALIDATE_OR_GOTO (fd->inode, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
 
         local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
         if (!local) {
@@ -2040,13 +2396,13 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_err_cbk, subvol, subvol->fops->fsetxattr,
-                    fd, xattr, flags);
+                    fd, xattr, flags, NULL);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2054,16 +2410,18 @@ err:
 
 static int
 dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno)
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *xdata)
 {
-        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
 
 int
 dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+                           int op_ret, int op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
         int           i     = -1;
         int           ret   = -1;
@@ -2095,7 +2453,7 @@ dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP);
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP, NULL);
         }
         return 0;
 
@@ -2103,7 +2461,7 @@ out:
 
 int
 dht_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr, int flags)
+              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
 {
         xlator_t     *subvol   = NULL;
         dht_local_t  *local    = NULL;
@@ -2115,21 +2473,19 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
         data_t       *tmp      = NULL;
         uint32_t      dir_spread = 0;
         char          value[4096] = {0,};
-        int           forced_rebalance = 0;
+        gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
         int           call_cnt = 0;
-        data_pair_t  *trav     = NULL;
-
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
 
         conf   = this->private;
+
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
+
         local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
         if (!local) {
                 op_errno = ENOMEM;
@@ -2154,25 +2510,6 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
 
         local->call_cnt = call_cnt = layout->cnt;
 
-        /*  This key is sent by Unified File and Object storage
-         *  to test xattr support in backend.
-         */
-        tmp = dict_get (xattr, "user.ufo-test");
-        if (tmp) {
-                if (IA_ISREG (loc->inode->ia_type)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
-                local->op_ret = 0;
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_ufo_xattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->setxattr,
-                                    loc, xattr, flags);
-                }
-                return 0;
-        }
-
         tmp = dict_get (xattr, "distribute.migrate-data");
         if (tmp) {
                 if (IA_ISDIR (loc->inode->ia_type)) {
@@ -2184,9 +2521,20 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                    (ie, 'target' subvolume given there, etc) */
                 memcpy (value, tmp->data, tmp->len);
                 if (strcmp (value, "force") == 0)
-                        forced_rebalance = 1;
+                       forced_rebalance =
+                                GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+
+                if (conf->decommission_in_progress)
+                        forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
 
                 local->rebalance.target_node = dht_subvol_get_hashed (this, loc);
+                if (!local->rebalance.target_node) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "hashed subvol for %s", loc->path);
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
                 local->rebalance.from_subvol = local->cached_subvol;
 
                 if (local->rebalance.target_node == local->rebalance.from_subvol) {
@@ -2226,7 +2574,7 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, dht_checking_pathinfo_cbk,
                                     conf->subvolumes[i],
                                     conf->subvolumes[i]->fops->getxattr,
-                                    loc, GF_XATTR_PATHINFO_KEY);
+                                    loc, GF_XATTR_PATHINFO_KEY, NULL);
                 }
                 return 0;
         }
@@ -2236,9 +2584,13 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_INFO,
                         "fixing the layout of %s", loc->path);
 
-                dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
-                                          layout);
-                return 0;
+                ret = dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
+                                                layout);
+                if (ret) {
+                        op_errno = ENOTCONN;
+                        goto err;
+                }
+                return ret;
         }
 
         tmp = dict_get (xattr, "distribute.directory-spread-count");
@@ -2250,10 +2602,14 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                              (dir_spread > 0))) {
                         layout->spread_cnt = dir_spread;
 
-                        dht_fix_directory_layout (frame,
-                                                  dht_common_setxattr_cbk,
-                                                  layout);
-                        return 0;
+                        ret = dht_fix_directory_layout (frame,
+                                                        dht_common_setxattr_cbk,
+                                                        layout);
+                        if (ret) {
+                                op_errno = ENOTCONN;
+                                goto err;
+                        }
+                        return ret;
                 }
                 gf_log (this->name, GF_LOG_ERROR,
                         "wrong 'directory-spread-count' value (%s)", value);
@@ -2265,14 +2621,14 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, dht_err_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->setxattr,
-                            loc, xattr, flags);
+                            loc, xattr, flags, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2280,7 +2636,7 @@ err:
 
 int
 dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno)
+                     int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -2306,7 +2662,8 @@ unlock:
 
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (removexattr, frame, local->op_ret, local->op_errno);
+                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
+                                  local->op_errno, NULL);
         }
 
         return 0;
@@ -2315,21 +2672,27 @@ unlock:
 
 int
 dht_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *key)
+                 loc_t *loc, const char *key, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
         int           call_cnt = 0;
+        dht_conf_t   *conf = NULL;
 
         int i;
 
-        VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
+
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
+
+        VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
         if (!local) {
@@ -2360,32 +2723,39 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, dht_removexattr_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->removexattr,
-                            loc, key);
+                            loc, key, NULL);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
 
 int
 dht_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *key)
+                  fd_t *fd, const char *key, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
         int           call_cnt = 0;
+        dht_conf_t   *conf = 0;
 
         int i;
 
-        VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
+
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
+
+        VALIDATE_OR_GOTO (frame, err);
 
         local = dht_local_init (frame, NULL, fd, GF_FOP_FREMOVEXATTR);
         if (!local) {
@@ -2417,14 +2787,14 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, dht_removexattr_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->fremovexattr,
-                            fd, key);
+                            fd, key, NULL);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2432,7 +2802,7 @@ err:
 
 int
 dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, fd_t *fd)
+            int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -2459,7 +2829,7 @@ unlock:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
                 DHT_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                                  local->fd);
+                                  local->fd, NULL);
 
         return 0;
 }
@@ -2489,7 +2859,7 @@ dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
 
 int
 dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct statvfs *statvfs)
+                int op_ret, int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
         dht_local_t *local         = NULL;
         int          this_call_cnt = 0;
@@ -2535,14 +2905,14 @@ unlock:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
                 DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->statvfs);
+                                  &local->statvfs, xdata);
 
         return 0;
 }
 
 
 int
-dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         dht_local_t  *local  = NULL;
@@ -2554,7 +2924,6 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf = this->private;
@@ -2571,7 +2940,8 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 for (i = 0; i < conf->subvolume_cnt; i++) {
                         STACK_WIND (frame, dht_statfs_cbk,
                                     conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->statfs, loc);
+                                    conf->subvolumes[i]->fops->statfs, loc,
+                                    xdata);
                 }
                 return 0;
         }
@@ -2587,20 +2957,21 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_statfs_cbk,
-                    subvol, subvol->fops->statfs, loc);
+                    subvol, subvol->fops->statfs, loc, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+             dict_t *xdata)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -2627,14 +2998,14 @@ dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
                 STACK_WIND (frame, dht_fd_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->opendir,
-                            loc, fd);
+                            loc, fd, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -2642,7 +3013,7 @@ err:
 
 int
 dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-                  int op_errno, gf_dirent_t *orig_entries)
+                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         gf_dirent_t   entries;
@@ -2655,6 +3026,7 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
         dht_layout_t *layout = 0;
         dht_conf_t   *conf   = NULL;
         xlator_t     *subvol = 0;
+        int           ret    = 0;
 
         INIT_LIST_HEAD (&entries.list);
         prev = cookie;
@@ -2671,10 +3043,13 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
 
         list_for_each_entry (orig_entry, (&orig_entries->list), list) {
                 next_offset = orig_entry->d_off;
-
-                if (check_is_linkfile_wo_dict (NULL, (&orig_entry->d_stat))
-                    || (check_is_dir (NULL, (&orig_entry->d_stat), NULL)
-                        && (prev->this != dht_first_up_subvol (this)))) {
+                if (check_is_dir (NULL, (&orig_entry->d_stat), NULL) &&
+                    (prev->this != local->first_up_subvol)) {
+                        continue;
+                }
+                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
+                                       orig_entry->dict,
+                                       conf->link_xattr_name)) {
                         continue;
                 }
 
@@ -2703,6 +3078,24 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 entry->d_type = orig_entry->d_type;
                 entry->d_len  = orig_entry->d_len;
 
+                if (orig_entry->dict)
+                        entry->dict = dict_ref (orig_entry->dict);
+
+                /* making sure we set the inode ctx right with layout,
+                   currently possible only for non-directories, so for
+                   directories don't set entry inodes */
+                if (!IA_ISDIR(entry->d_stat.ia_type)) {
+                        ret = dht_layout_preset (this, prev->this,
+                                                 orig_entry->inode);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to link the layout in inode");
+                        entry->inode = inode_ref (orig_entry->inode);
+                } else if (orig_entry->inode) {
+                        dht_inode_ctx_time_update (orig_entry->inode, this,
+                                                   &entry->d_stat, 1);
+                }
+
                 list_add_tail (&entry->list, &entries.list);
                 count++;
         }
@@ -2732,10 +3125,23 @@ done:
                         goto unwind;
                 }
 
+		if (conf->readdir_optimize == _gf_true) {
+                        if (next_subvol != local->first_up_subvol) {
+                                ret = dict_set_int32 (local->xattr,
+                                                      GF_READDIR_SKIP_DIRS, 1);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+					         "dict set failed");
+		        } else {
+                                 dict_del (local->xattr,
+                                           GF_READDIR_SKIP_DIRS);
+                        }
+                }
+
                 STACK_WIND (frame, dht_readdirp_cbk,
                             next_subvol, next_subvol->fops->readdirp,
                             local->fd, local->size, next_offset,
-                            local->xattr_req);
+                            local->xattr);
                 return 0;
         }
 
@@ -2743,7 +3149,7 @@ unwind:
         if (op_ret < 0)
                 op_ret = 0;
 
-        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries);
+        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries, NULL);
 
         gf_dirent_free (&entries);
 
@@ -2754,7 +3160,8 @@ unwind:
 
 int
 dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries)
+                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
+                 dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         gf_dirent_t   entries;
@@ -2831,7 +3238,7 @@ done:
 
                 STACK_WIND (frame, dht_readdir_cbk,
                             next_subvol, next_subvol->fops->readdir,
-                            local->fd, local->size, next_offset);
+                            local->fd, local->size, next_offset, NULL);
                 return 0;
         }
 
@@ -2839,7 +3246,7 @@ unwind:
         if (op_ret < 0)
                 op_ret = 0;
 
-        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries);
+        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, NULL);
 
         gf_dirent_free (&entries);
 
@@ -2855,11 +3262,15 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         int           op_errno = -1;
         xlator_t     *xvol = NULL;
         off_t         xoff = 0;
-
+        int           ret = 0;
+        dht_conf_t   *conf = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
         local = dht_local_init (frame, NULL, NULL, whichop);
         if (!local) {
@@ -2870,22 +3281,51 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         local->fd = fd_ref (fd);
         local->size = size;
         local->xattr_req = (dict)? dict_ref (dict) : NULL;
+        local->first_up_subvol = dht_first_up_subvol (this);
 
         dht_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
 
         /* TODO: do proper readdir */
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND (frame, dht_readdir_cbk, xvol, xvol->fops->readdir,
-                            fd, size, xoff);
-        else
+        if (whichop == GF_FOP_READDIRP) {
+                if (dict)
+                        local->xattr = dict_ref (dict);
+                else
+                        local->xattr = dict_new ();
+
+                if (local->xattr) {
+                        ret = dict_set_uint32 (local->xattr,
+                                               conf->link_xattr_name, 256);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to set '%s' key",
+                                        conf->link_xattr_name);
+			if (conf->readdir_optimize == _gf_true) {
+                                if (xvol != local->first_up_subvol) {
+				        ret = dict_set_int32 (local->xattr,
+			                               GF_READDIR_SKIP_DIRS, 1);
+				        if (ret)
+					        gf_log (this->name,
+                                                        GF_LOG_ERROR,
+						        "Dict set failed");
+                                } else {
+                                        dict_del (local->xattr,
+                                                  GF_READDIR_SKIP_DIRS);
+                                }
+			}
+                }
+
                 STACK_WIND (frame, dht_readdirp_cbk, xvol, xvol->fops->readdirp,
-                            fd, size, xoff, dict);
+                            fd, size, xoff, local->xattr);
+        } else {
+                STACK_WIND (frame, dht_readdir_cbk, xvol, xvol->fops->readdir,
+                            fd, size, xoff, local->xattr);
+        }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -2893,7 +3333,7 @@ err:
 
 int
 dht_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff)
+             off_t yoff, dict_t *xdata)
 {
         int          op = GF_FOP_READDIR;
         dht_conf_t  *conf = NULL;
@@ -2930,7 +3370,7 @@ dht_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
 int
 dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno)
+                  int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -2950,14 +3390,16 @@ dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret, local->op_errno);
+                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret,
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
 int
-dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              int datasync, dict_t *xdata)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -2984,14 +3426,14 @@ dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
                 STACK_WIND (frame, dht_fsyncdir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->fsyncdir,
-                            fd, datasync);
+                            fd, datasync, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -3001,9 +3443,9 @@ int
 dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int op_ret, int op_errno,
                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
-        call_frame_t *prev = NULL;
+        xlator_t     *prev = NULL;
         int           ret = -1;
         dht_local_t  *local = NULL;
 
@@ -3021,19 +3463,24 @@ dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         if (local->loc.parent) {
-                WIPE (preparent);
-                WIPE (postparent);
+
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
         }
 
-        ret = dht_layout_preset (this, prev->this, inode);
+        ret = dht_layout_preset (this, prev, inode);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "could not set pre-set layout for subvolume %s",
-                        prev->this->name);
+                        prev? prev->name: NULL);
                 op_ret   = -1;
                 op_errno = EINVAL;
                 goto out;
         }
+        if (local->linked == _gf_true)
+                dht_linkfile_attr_heal (frame, this);
 out:
         /*
          * FIXME: ia_size and st_blocks of preparent and postparent do not have
@@ -3042,10 +3489,9 @@ out:
          * corresponding values from each of the subvolume.
          * See dht_iatt_merge for reference.
          */
-
         DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf,
+                          preparent, postparent, xdata);
         return 0;
 }
 
@@ -3054,7 +3500,8 @@ dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this,
                                int32_t op_ret, int32_t op_errno,
                                inode_t *inode, struct iatt *stbuf,
-                               struct iatt *preparent, struct iatt *postparent)
+                               struct iatt *preparent, struct iatt *postparent,
+                               dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -3063,22 +3510,28 @@ dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                 goto err;
 
         local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
         cached_subvol = local->cached_subvol;
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    cached_subvol, cached_subvol->fops->mknod,
-                    &local->loc, local->mode, local->rdev,
-                    local->params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)cached_subvol,
+                           cached_subvol, cached_subvol->fops->mknod,
+                           &local->loc, local->mode, local->rdev, local->umask,
+                           local->params);
 
         return 0;
 err:
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL);
         return 0;
 }
 
 int
 dht_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
 {
         xlator_t    *subvol = NULL;
         int          op_errno = -1;
@@ -3110,11 +3563,13 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating %s on %s", loc->path, subvol->name);
 
-                STACK_WIND (frame, dht_newfile_cbk,
-                            subvol, subvol->fops->mknod,
-                            loc, mode, rdev, params);
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol,
+                                   subvol, subvol->fops->mknod, loc, mode,
+                                   rdev, umask, params);
         } else {
-                avail_subvol = dht_free_disk_available_subvol (this, subvol);
+
+                avail_subvol = dht_free_disk_available_subvol (this, subvol,
+                                                               local);
                 if (avail_subvol != subvol) {
                         /* Choose the minimum filled volume, and create the
                            files there */
@@ -3123,17 +3578,18 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
                         local->cached_subvol = avail_subvol;
                         local->mode = mode;
                         local->rdev = rdev;
-
+                        local->umask = umask;
                         dht_linkfile_create (frame,
                                              dht_mknod_linkfile_create_cbk,
-                                             avail_subvol, subvol, loc);
+                                             this, avail_subvol, subvol, loc);
                 } else {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "creating %s on %s", loc->path, subvol->name);
 
-                        STACK_WIND (frame, dht_newfile_cbk,
-                                    subvol, subvol->fops->mknod,
-                                    loc, mode, rdev, params);
+                        STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                                           (void *)subvol, subvol,
+                                           subvol->fops->mknod, loc, mode,
+                                           rdev, umask, params);
                 }
         }
 
@@ -3142,7 +3598,7 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -3150,7 +3606,7 @@ err:
 
 int
 dht_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkname, loc_t *loc, dict_t *params)
+             const char *linkname, loc_t *loc, mode_t umask, dict_t *params)
 {
         xlator_t    *subvol = NULL;
         int          op_errno = -1;
@@ -3178,23 +3634,24 @@ dht_symlink (call_frame_t *frame, xlator_t *this,
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->symlink,
-                    linkname, loc, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->symlink, linkname, loc, umask,
+                           params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (link, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+            dict_t *xdata)
 {
         xlator_t    *cached_subvol = NULL;
         xlator_t    *hashed_subvol = NULL;
@@ -3212,7 +3669,7 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                         local->loc.path, cached_subvol->name, loc->path);
                 STACK_WIND (frame, dht_unlink_cbk,
                             cached_subvol, cached_subvol->fops->unlink,
-                            &local->loc);
+                            &local->loc, xflag, xdata);
                 goto done;
         }
 
@@ -3240,18 +3697,21 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto err;
         }
 
+        local->flags = xflag;
         if (hashed_subvol != cached_subvol) {
                 STACK_WIND (frame, dht_unlink_linkfile_cbk,
-                            hashed_subvol, hashed_subvol->fops->unlink, loc);
+                            hashed_subvol, hashed_subvol->fops->unlink, loc,
+                            xflag, xdata);
         } else {
                 STACK_WIND (frame, dht_unlink_cbk,
-                            cached_subvol, cached_subvol->fops->unlink, loc);
+                            cached_subvol, cached_subvol->fops->unlink, loc,
+                            xflag, xdata);
         }
 done:
         return 0;
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -3261,13 +3721,16 @@ int
 dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int op_ret, int op_errno,
               inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent)
+              struct iatt *postparent, dict_t *xdata)
 {
         call_frame_t *prev = NULL;
         dht_layout_t *layout = NULL;
+        dht_local_t  *local = NULL;
 
         prev = cookie;
 
+        local = frame->local;
+
         if (op_ret == -1)
                 goto out;
 
@@ -3281,13 +3744,20 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        WIPE (preparent);
-        WIPE (postparent);
-
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+                          postparent, NULL);
 
         return 0;
 }
@@ -3297,7 +3767,8 @@ int
 dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno,
                        inode_t *inode, struct iatt *stbuf,
-                       struct iatt *preparent, struct iatt *postparent)
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *srcvol = NULL;
@@ -3309,14 +3780,14 @@ dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         srcvol = local->linkfile.srcvol;
 
         STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link,
-                    &local->loc, &local->loc2);
+                    &local->loc, &local->loc2, xdata);
 
         return 0;
 
 err:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+                          postparent, NULL);
 
         return 0;
 }
@@ -3324,7 +3795,7 @@ err:
 
 int
 dht_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc)
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         xlator_t    *cached_subvol = NULL;
         xlator_t    *hashed_subvol = NULL;
@@ -3369,19 +3840,19 @@ dht_link (call_frame_t *frame, xlator_t *this,
 
         if (hashed_subvol != cached_subvol) {
                 uuid_copy (local->gfid, oldloc->inode->gfid);
-                dht_linkfile_create (frame, dht_link_linkfile_cbk,
+                dht_linkfile_create (frame, dht_link_linkfile_cbk, this,
                                      cached_subvol, hashed_subvol, newloc);
         } else {
                 STACK_WIND (frame, dht_link_cbk,
                             cached_subvol, cached_subvol->fops->link,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -3391,7 +3862,7 @@ int
 dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno,
                 fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent)
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         call_frame_t *prev = NULL;
         int           ret = -1;
@@ -3410,8 +3881,11 @@ dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         if (local->loc.parent) {
-                WIPE (preparent);
-                WIPE (postparent);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
         }
 
         ret = dht_layout_preset (this, prev->this, inode);
@@ -3423,11 +3897,14 @@ dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 op_errno = EINVAL;
                 goto out;
         }
-
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf, preparent,
-                          postparent);
+                          postparent, NULL);
         return 0;
 }
 
@@ -3437,7 +3914,8 @@ dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                 xlator_t *this,
                                 int32_t op_ret, int32_t op_errno,
                                 inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent)
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -3451,18 +3929,19 @@ dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
         STACK_WIND (frame, dht_create_cbk,
                     cached_subvol, cached_subvol->fops->create,
                     &local->loc, local->flags, local->mode,
-                    local->fd, local->params);
+                    local->umask, local->fd, local->params);
 
         return 0;
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL, NULL);
         return 0;
 }
 
 int
 dht_create (call_frame_t *frame, xlator_t *this,
             loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params)
+            mode_t umask, fd_t *fd, dict_t *params)
 {
         int          op_errno = -1;
         xlator_t    *subvol = NULL;
@@ -3488,7 +3967,7 @@ dht_create (call_frame_t *frame, xlator_t *this,
                         local->loc.path, subvol->name, loc->path);
                 STACK_WIND (frame, dht_create_cbk,
                             subvol, subvol->fops->create,
-                            &local->loc, flags, mode, fd, params);
+                            &local->loc, flags, mode, umask, fd, params);
                 goto done;
         }
 
@@ -3506,38 +3985,38 @@ dht_create (call_frame_t *frame, xlator_t *this,
                         "creating %s on %s", loc->path, subvol->name);
                 STACK_WIND (frame, dht_create_cbk,
                             subvol, subvol->fops->create,
-                            loc, flags, mode, fd, params);
+                            loc, flags, mode, umask, fd, params);
                 goto done;
         }
         /* Choose the minimum filled volume, and create the
            files there */
-        avail_subvol = dht_free_disk_available_subvol (this, subvol);
+        avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
         if (avail_subvol != subvol) {
                 local->params = dict_ref (params);
                 local->flags = flags;
                 local->mode = mode;
-
+                local->umask = umask;
                 local->cached_subvol = avail_subvol;
                 local->hashed_subvol = subvol;
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating %s on %s (link at %s)", loc->path,
                         avail_subvol->name, subvol->name);
-                dht_linkfile_create (frame,
-                                     dht_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, dht_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 goto done;
         }
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
         STACK_WIND (frame, dht_create_cbk,
                     subvol, subvol->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, params);
 done:
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL, NULL);
 
         return 0;
 }
@@ -3546,7 +4025,7 @@ err:
 int
 dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
                         xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         dht_local_t   *local = NULL;
         dht_layout_t  *layout = NULL;
@@ -3557,14 +4036,17 @@ dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
         if (op_ret == 0) {
                 dht_layout_set (this, local->inode, layout);
                 if (local->loc.parent) {
-                        WIPE (&local->preparent);
-                        WIPE (&local->postparent);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
                 }
         }
 
         DHT_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
                           local->inode, &local->stbuf, &local->preparent,
-                          &local->postparent);
+                          &local->postparent, NULL);
 
         return 0;
 }
@@ -3572,7 +4054,7 @@ dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
 int
 dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-               struct iatt *preparent, struct iatt *postparent)
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -3593,6 +4075,15 @@ dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         ret = dht_layout_merge (this, layout, prev->this,
                                                 -1, ENOSPC, NULL);
                 } else {
+			if (op_ret == -1 && op_errno == EEXIST)
+				/* Very likely just a race between mkdir and
+				   self-heal (from lookup of a concurrent mkdir
+				   attempt).
+				   Ignore error for now. layout setting will
+				   anyways fail if this was a different (old)
+				   pre-existing different directory.
+				*/
+				op_ret = 0;
                         ret = dht_layout_merge (this, layout, prev->this,
                                                 op_ret, op_errno, NULL);
                 }
@@ -3625,7 +4116,8 @@ int
 dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
                       xlator_t *this, int op_ret, int op_errno,
                       inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent,
+                      dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           ret = -1;
@@ -3682,19 +4174,20 @@ dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
                         continue;
                 STACK_WIND (frame, dht_mkdir_cbk,
                             conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->mkdir,
-                            &local->loc, local->mode, local->params);
+                            conf->subvolumes[i]->fops->mkdir, &local->loc,
+                            local->mode, local->umask, local->params);
         }
         return 0;
 err:
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL);
         return 0;
 }
 
 
-int
+ int
 dht_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dict_t *params)
+           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -3730,6 +4223,7 @@ dht_mkdir (call_frame_t *frame, xlator_t *this,
 
         local->hashed_subvol = hashed_subvol;
         local->mode = mode;
+        local->umask = umask;
         local->params = dict_ref (params);
         local->inode  = inode_ref (loc->inode);
 
@@ -3742,13 +4236,14 @@ dht_mkdir (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, dht_mkdir_hashed_cbk,
                     hashed_subvol,
                     hashed_subvol->fops->mkdir,
-                    loc, mode, params);
+                    loc, mode, umask, params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL);
 
         return 0;
 }
@@ -3756,14 +4251,87 @@ err:
 
 int
 dht_rmdir_selfheal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno)
+                        int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
         local = frame->local;
 
         DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
+
+        return 0;
+}
+
+
+int
+dht_rmdir_hashed_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+
+        local = frame->local;
+        prev  = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        local->op_ret   = -1;
+                        if (op_errno != ENOENT && op_errno != EACCES) {
+                                local->need_selfheal = 1;
+                        }
+
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "rmdir on %s for %s failed (%s)",
+                                prev->this->name, local->loc.path,
+                                strerror (op_errno));
+                        goto unlock;
+                }
+
+                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
+
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+               if (local->need_selfheal) {
+                        local->layout =
+                                dht_layout_get (this, local->loc.inode);
+
+                        /* TODO: neater interface needed below */
+                        local->stbuf.ia_type = local->loc.inode->ia_type;
+
+                        uuid_copy (local->gfid, local->loc.inode->gfid);
+                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
+                                              &local->loc, local->layout);
+               } else {
+
+                        if (local->loc.parent) {
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+                        }
+
+                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
+                                          local->op_errno, &local->preparent,
+                                          &local->postparent, NULL);
+               }
+        }
 
         return 0;
 }
@@ -3772,11 +4340,12 @@ dht_rmdir_selfheal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 dht_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent)
+               struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
         call_frame_t *prev = NULL;
+        int           done = 0;
 
         local = frame->local;
         prev  = cookie;
@@ -3809,7 +4378,16 @@ unlock:
 
 
         this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
+
+        /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
+        if (local->hashed_subvol && (this_call_cnt == 1)) {
+                done = 1;
+        } else if (!local->hashed_subvol && !this_call_cnt) {
+                done = 1;
+        }
+
+
+        if (done) {
                 if (local->need_selfheal && local->fop_succeeded) {
                         local->layout =
                                 dht_layout_get (this, local->loc.inode);
@@ -3820,15 +4398,34 @@ unlock:
                         uuid_copy (local->gfid, local->loc.inode->gfid);
                         dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
                                               &local->loc, local->layout);
-                } else {
+                } else if (this_call_cnt) {
+                        /* If non-hashed subvol's have responded, proceed */
+
+                        local->need_selfheal = 0;
+                        STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
+                                    local->hashed_subvol,
+                                    local->hashed_subvol->fops->rmdir,
+                                    &local->loc, local->flags, NULL);
+                } else if (!this_call_cnt) {
+                        /* All subvol's have responded, proceed */
+
                         if (local->loc.parent) {
-                                WIPE (&local->preparent);
-                                WIPE (&local->postparent);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+
                         }
 
                         DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
                                           local->op_errno, &local->preparent,
-                                          &local->postparent);
+                                          &local->postparent, NULL);
                 }
         }
 
@@ -3842,6 +4439,7 @@ dht_rmdir_do (call_frame_t *frame, xlator_t *this)
         dht_local_t  *local = NULL;
         dht_conf_t   *conf = NULL;
         int           i = 0;
+        xlator_t     *hashed_subvol = NULL;
 
         VALIDATE_OR_GOTO (this->private, err);
 
@@ -3853,18 +4451,41 @@ dht_rmdir_do (call_frame_t *frame, xlator_t *this)
 
         local->call_cnt = conf->subvolume_cnt;
 
+        /* first remove from non-hashed_subvol */
+        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
+
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to get hashed "
+                        "subvol for %s",local->loc.path);
+        } else {
+                local->hashed_subvol = hashed_subvol;
+        }
+
+        /* When DHT has only 1 child */
+        if (conf->subvolume_cnt == 1) {
+                STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
+                            conf->subvolumes[0],
+                            conf->subvolumes[0]->fops->rmdir,
+                            &local->loc, local->flags, NULL);
+                return 0;
+        }
+
         for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (hashed_subvol &&
+                    (hashed_subvol == conf->subvolumes[i]))
+                        continue;
+
                 STACK_WIND (frame, dht_rmdir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->rmdir,
-                            &local->loc, local->flags);
+                            &local->loc, local->flags, NULL);
         }
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
         return 0;
 }
 
@@ -3872,7 +4493,7 @@ err:
 int
 dht_rmdir_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                int op_ret, int op_errno, struct iatt *preparent,
-                               struct iatt *postparent)
+                               struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t    *local = NULL;
         call_frame_t   *prev = NULL;
@@ -3920,6 +4541,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_frame_t   *main_frame = NULL;
         dht_local_t    *main_local = NULL;
         int             this_call_cnt = 0;
+        dht_conf_t     *conf = this->private;
 
         local = frame->local;
         prev  = cookie;
@@ -3931,7 +4553,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret != 0)
                 goto err;
 
-        if (check_is_linkfile (inode, stbuf, xattr) == 0) {
+        if (!check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
                 main_local->op_ret  = -1;
                 main_local->op_errno = ENOTEMPTY;
 
@@ -3942,7 +4564,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_WIND (frame, dht_rmdir_linkfile_unlink_cbk,
-                    src, src->fops->unlink, &local->loc);
+                    src, src->fops->unlink, &local->loc, 0, NULL);
         return 0;
 err:
 
@@ -3965,6 +4587,8 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
         call_frame_t       *lookup_frame = NULL;
         dht_local_t        *lookup_local = NULL;
         dht_local_t        *local = NULL;
+        dict_t             *xattrs = NULL;
+        dht_conf_t         *conf = this->private;
 
         local = frame->local;
 
@@ -3973,7 +4597,8 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                         continue;
                 if (strcmp (trav->d_name, "..") == 0)
                         continue;
-                if (check_is_linkfile (NULL, (&trav->d_stat), NULL) == 1) {
+                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict,
+                                              conf->link_xattr_name)) {
                         ret++;
                         continue;
                 }
@@ -3985,6 +4610,21 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
+        xattrs = dict_new ();
+        if (!xattrs) {
+                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
+                return -1;
+        }
+
+        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set linkto key"
+                        " in dict");
+                if (xattrs)
+                        dict_unref (xattrs);
+                return -1;
+        }
+
         list_for_each_entry (trav, &entries->list, list) {
                 if (strcmp (trav->d_name, ".") == 0)
                         continue;
@@ -4001,8 +4641,7 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                         goto err;
                 }
 
-                lookup_local = GF_CALLOC (sizeof (*local), 1,
-                                          gf_dht_mt_dht_local_t);
+                lookup_local = mem_get0 (this->local_pool);
                 if (!lookup_local) {
                         goto err;
                 }
@@ -4029,12 +4668,18 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
 
                 STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
                             src, src->fops->lookup,
-                            &lookup_local->loc, NULL);
+                            &lookup_local->loc, xattrs);
                 ret++;
         }
 
+        if (xattrs)
+                dict_unref (xattrs);
+
         return ret;
 err:
+        if (xattrs)
+                dict_unref (xattrs);
+
         DHT_STACK_DESTROY (lookup_frame);
         return 0;
 }
@@ -4042,7 +4687,8 @@ err:
 
 int
 dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries)
+                        int op_ret, int op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = -1;
@@ -4086,12 +4732,14 @@ dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, fd_t *fd)
+                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = -1;
         call_frame_t *prev = NULL;
-
+        dict_t       *dict = NULL;
+        int           ret = 0;
+        dht_conf_t   *conf = this->private;
 
         local = frame->local;
         prev  = cookie;
@@ -4101,14 +4749,32 @@ dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "opendir on %s for %s failed (%s)",
                         prev->this->name, local->loc.path,
                         strerror (op_errno));
+                if (op_errno != ENOENT) {
+                        local->op_ret = -1;
+                        local->op_errno = op_errno;
+                }
+                goto err;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
                 local->op_ret = -1;
-                local->op_errno = op_errno;
+                local->op_errno = ENOMEM;
                 goto err;
         }
 
+        ret = dict_set_uint32 (dict, conf->link_xattr_name, 256);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set '%s' key",
+                        local->loc.path, conf->link_xattr_name);
+
         STACK_WIND (frame, dht_rmdir_readdirp_cbk,
                     prev->this, prev->this->fops->readdirp,
-                    local->fd, 4096, 0, NULL);
+                    local->fd, 4096, 0, dict);
+
+        if (dict)
+                dict_unref (dict);
 
         return 0;
 
@@ -4124,7 +4790,8 @@ err:
 
 
 int
-dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           dict_t *xdata)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -4163,7 +4830,7 @@ dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
                 STACK_WIND (frame, dht_rmdir_opendir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->opendir,
-                            loc, local->fd);
+                            loc, local->fd, NULL);
         }
 
         return 0;
@@ -4171,17 +4838,17 @@ dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (rmdir, frame, -1, op_errno,
-                          NULL, NULL);
+                          NULL, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -4189,7 +4856,7 @@ dht_entrylk_cbk (call_frame_t *frame, void *cookie,
 int
 dht_entrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, loc_t *loc, const char *basename,
-             entrylk_cmd cmd, entrylk_type type)
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -4199,7 +4866,6 @@ dht_entrylk (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
         if (!local) {
@@ -4219,13 +4885,13 @@ dht_entrylk (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_entrylk_cbk,
                     subvol, subvol->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -4233,10 +4899,10 @@ err:
 
 int
 dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, NULL);
         return 0;
 }
 
@@ -4244,7 +4910,7 @@ dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
 int
 dht_fentrylk (call_frame_t *frame, xlator_t *this,
               const char *volume, fd_t *fd, const char *basename,
-              entrylk_cmd cmd, entrylk_type type)
+              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -4263,13 +4929,13 @@ dht_fentrylk (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_fentrylk_cbk,
                     subvol, subvol->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
+                    volume, fd, basename, cmd, type, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -4278,16 +4944,21 @@ err:
 int
 dht_forget (xlator_t *this, inode_t *inode)
 {
-        uint64_t      tmp_layout = 0;
+        uint64_t        ctx_int = 0;
+        dht_inode_ctx_t *ctx    = NULL;
         dht_layout_t *layout = NULL;
 
-        inode_ctx_del (inode, this, &tmp_layout);
+        inode_ctx_del (inode, this, &ctx_int);
 
-        if (!tmp_layout)
+        if (!ctx_int)
                 return 0;
 
-        layout = (dht_layout_t *)(long)tmp_layout;
+        ctx = (dht_inode_ctx_t *) (long) ctx_int;
+
+        layout = ctx->layout;
+        ctx->layout = NULL;
         dht_layout_unref (this, layout);
+        GF_FREE (ctx);
 
         return 0;
 }
@@ -4296,16 +4967,22 @@ dht_forget (xlator_t *this, inode_t *inode)
 int
 dht_notify (xlator_t *this, int event, void *data, ...)
 {
-        xlator_t   *subvol = NULL;
-        int         cnt    = -1;
-        int         i      = -1;
-        dht_conf_t *conf   = NULL;
-        int         ret    = -1;
-        int         propagate = 0;
+        xlator_t                *subvol = NULL;
+        int                      cnt    = -1;
+        int                      i      = -1;
+        dht_conf_t              *conf   = NULL;
+        int                      ret    = -1;
+        int                      propagate = 0;
+
+        int                      had_heard_from_all = 0;
+        int                      have_heard_from_all = 0;
+        struct timeval           time = {0,};
+        gf_defrag_info_t        *defrag = NULL;
+        dict_t                  *dict   = NULL;
+        gf_defrag_type           cmd    = 0;
+        dict_t                  *output = NULL;
+        va_list                  ap;
 
-        int         had_heard_from_all = 0;
-        int         have_heard_from_all = 0;
-        struct timeval  time = {0,};
 
         conf = this->private;
         if (!conf)
@@ -4367,7 +5044,11 @@ dht_notify (xlator_t *this, int event, void *data, ...)
                 if (conf->assert_no_child_down) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "Received CHILD_DOWN. Exiting");
-                        exit(0);
+                        if (conf->defrag) {
+                                gf_defrag_stop (conf->defrag, NULL);
+                        } else {
+                                kill (getpid(), SIGTERM);
+                        }
                 }
 
                 for (i = 0; i < conf->subvolume_cnt; i++) {
@@ -4418,6 +5099,36 @@ dht_notify (xlator_t *this, int event, void *data, ...)
                 UNLOCK (&conf->subvolume_lock);
 
                 break;
+        case GF_EVENT_VOLUME_DEFRAG:
+        {
+                if (!conf->defrag) {
+                        return ret;
+                }
+                defrag = conf->defrag;
+
+                dict = data;
+                va_start (ap, data);
+                output = va_arg (ap, dict_t*);
+
+                ret = dict_get_int32 (dict, "rebalance-command",
+                                      (int32_t*)&cmd);
+                if (ret)
+                        return ret;
+                LOCK (&defrag->lock);
+                {
+                        if (defrag->is_exiting)
+                                goto unlock;
+                        if (cmd == GF_DEFRAG_CMD_STATUS)
+                                gf_defrag_status_get (defrag, output);
+                        else if (cmd == GF_DEFRAG_CMD_STOP)
+                                gf_defrag_stop (defrag, output);
+                }
+unlock:
+                UNLOCK (&defrag->lock);
+                return 0;
+                break;
+        }
+
         default:
                 propagate = 1;
                 break;
@@ -4433,9 +5144,12 @@ dht_notify (xlator_t *this, int event, void *data, ...)
 
         /* if all subvols have reported status, no need to hide anything
            or wait for anything else. Just propagate blindly */
-        if (have_heard_from_all)
+        if (have_heard_from_all) {
                 propagate = 1;
 
+        }
+
+
         if (!had_heard_from_all && have_heard_from_all) {
                 /* This is the first event which completes aggregation
                    of events from all subvolumes. If at least one subvol
@@ -4454,6 +5168,19 @@ dht_notify (xlator_t *this, int event, void *data, ...)
                                 /* continue to check other events for CHILD_UP */
                         }
                 }
+
+                /* rebalance is started with assert_no_child_down. So we do
+                 * not need to handle CHILD_DOWN event here.
+                 */
+                if (conf->defrag) {
+                        ret = gf_thread_create (&conf->defrag->th, NULL,
+						gf_defrag_start, this);
+                        if (ret) {
+                                conf->defrag = NULL;
+                                GF_FREE (conf->defrag);
+                                kill (getpid(), SIGTERM);
+                        }
+                }
         }
 
         ret = 0;
@@ -4462,3 +5189,24 @@ dht_notify (xlator_t *this, int event, void *data, ...)
 
         return ret;
 }
+
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (!ret && ctx) {
+                if (ctx->layout) {
+                        if (layout)
+                                *layout = ctx->layout;
+                        ret = 0;
+                } else {
+                        ret = -1;
+                }
+        }
+
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 749abe538..5ccd66799 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -22,6 +13,8 @@
 #include "config.h"
 #endif
 
+#include <regex.h>
+
 #include "dht-mem-types.h"
 #include "libxlator.h"
 #include "syncop.h"
@@ -38,7 +31,8 @@
 
 typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
                                        xlator_t     *this,
-                                       int32_t       op_ret, int32_t op_errno);
+                                       int32_t       op_ret, int32_t op_errno,
+                                       dict_t *xdata);
 typedef int (*dht_defrag_cbk_fn_t) (xlator_t        *this, call_frame_t *frame,
                                     int              ret);
 
@@ -61,20 +55,38 @@ struct dht_layout {
                 uint32_t   start;
                 uint32_t   stop;
                 xlator_t  *xlator;
-        } list[0];
+        } list[];
 };
 typedef struct dht_layout  dht_layout_t;
 
+struct dht_stat_time {
+        uint32_t        atime;
+        uint32_t        atime_nsec;
+        uint32_t        ctime;
+        uint32_t        ctime_nsec;
+        uint32_t        mtime;
+        uint32_t        mtime_nsec;
+};
+
+typedef struct dht_stat_time dht_stat_time_t;
+
+struct dht_inode_ctx {
+        dht_layout_t    *layout;
+        dht_stat_time_t  time;
+};
+
+typedef struct dht_inode_ctx dht_inode_ctx_t;
+
 
 typedef enum {
         DHT_HASH_TYPE_DM,
+        DHT_HASH_TYPE_DM_USER,
 } dht_hashfn_type_t;
 
 /* rebalance related */
 struct dht_rebalance_ {
         xlator_t            *from_subvol;
         xlator_t            *target_node;
-        int32_t              wbflags;
         off_t                offset;
         size_t               size;
         int32_t              flags;
@@ -83,6 +95,7 @@ struct dht_rebalance_ {
         struct iovec        *vector;
         struct iatt          stbuf;
         dht_defrag_cbk_fn_t  target_op_fn;
+        dict_t              *xdata;
 };
 
 struct dht_local {
@@ -129,7 +142,6 @@ struct dht_local {
         struct {
                 uint32_t         hole_cnt;
                 uint32_t         overlaps_cnt;
-                uint32_t         missing;
                 uint32_t         down;
                 uint32_t         misc;
                 dht_selfheal_dir_cbk_t   dir_cbk;
@@ -142,11 +154,16 @@ struct dht_local {
         int32_t flags;
         mode_t  mode;
         dev_t   rdev;
+        mode_t  umask;
 
         /* need for file-info */
-        char   *pathinfo;
+        char   *xattr_val;
         char   *key;
 
+        /* which xattr request? */
+        char xsel[256];
+        int32_t alloc_len;
+
         char   *newpath;
 
         /* gfid related */
@@ -162,7 +179,12 @@ struct dht_local {
 
         glusterfs_fop_t      fop;
 
+        gf_boolean_t     linked;
+        xlator_t        *link_subvol;
+
         struct dht_rebalance_ rebalance;
+        xlator_t        *first_up_subvol;
+
 };
 typedef struct dht_local dht_local_t;
 
@@ -175,6 +197,59 @@ struct dht_du {
 };
 typedef struct dht_du dht_du_t;
 
+enum gf_defrag_type {
+        GF_DEFRAG_CMD_START = 1,
+        GF_DEFRAG_CMD_STOP = 1 + 1,
+        GF_DEFRAG_CMD_STATUS = 1 + 2,
+        GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
+        GF_DEFRAG_CMD_START_FORCE = 1 + 4,
+};
+typedef enum gf_defrag_type gf_defrag_type;
+
+enum gf_defrag_status_t {
+        GF_DEFRAG_STATUS_NOT_STARTED,
+        GF_DEFRAG_STATUS_STARTED,
+        GF_DEFRAG_STATUS_STOPPED,
+        GF_DEFRAG_STATUS_COMPLETE,
+        GF_DEFRAG_STATUS_FAILED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
+};
+typedef enum gf_defrag_status_t gf_defrag_status_t;
+
+typedef struct gf_defrag_pattern_list gf_defrag_pattern_list_t;
+
+struct gf_defrag_pattern_list {
+        char                       path_pattern[256];
+        uint64_t                   size;
+        gf_defrag_pattern_list_t  *next;
+};
+
+struct gf_defrag_info_ {
+        uint64_t                     total_files;
+        uint64_t                     total_data;
+        uint64_t                     num_files_lookedup;
+        uint64_t                     total_failures;
+        uint64_t                     skipped;
+        gf_lock_t                    lock;
+        int                          cmd;
+        pthread_t                    th;
+        gf_defrag_status_t           defrag_status;
+        struct rpc_clnt             *rpc;
+        uint32_t                     connected;
+        uint32_t                     is_exiting;
+        pid_t                        pid;
+        inode_t                     *root_inode;
+        uuid_t                       node_uuid;
+        struct timeval               start_time;
+        gf_boolean_t                 stats;
+        gf_defrag_pattern_list_t    *defrag_pattern;
+};
+
+typedef struct gf_defrag_info_ gf_defrag_info_t;
+
 struct dht_conf {
         gf_lock_t      subvolume_lock;
         int            subvolume_cnt;
@@ -186,8 +261,8 @@ struct dht_conf {
         gf_boolean_t   search_unhashed;
         int            gen;
         dht_du_t      *du_stats;
-        uint64_t       min_free_disk;
-	uint32_t       min_free_inodes;
+        double         min_free_disk;
+	double         min_free_inodes;
         char           disk_unit;
         int32_t        refresh_interval;
         gf_boolean_t   unhashed_sticky_bit;
@@ -204,10 +279,28 @@ struct dht_conf {
         /* Will be a global flag to control the layout spread count */
         uint32_t       dir_spread_cnt;
 
-	struct syncenv *env; /* The env pointer to the rebalance synctask */
-
         /* to keep track of nodes which are decomissioned */
         xlator_t     **decommissioned_bricks;
+        int            decommission_in_progress;
+        int            decommission_subvols_cnt;
+
+        /* defrag related */
+        gf_defrag_info_t *defrag;
+
+        /* Request to filter directory entries in readdir request */
+
+        gf_boolean_t    readdir_optimize;
+
+        /* Support regex-based name reinterpretation. */
+        regex_t         rsync_regex;
+        gf_boolean_t    rsync_regex_valid;
+        regex_t         extra_regex;
+        gf_boolean_t    extra_regex_valid;
+
+        /* Support variable xattr names. */
+        char            *xattr_name;
+        char            *link_xattr_name;
+        char            *wild_xattr_name;
 };
 typedef struct dht_conf dht_conf_t;
 
@@ -222,29 +315,28 @@ struct dht_disk_layout {
 };
 typedef struct dht_disk_layout dht_disk_layout_t;
 
+typedef enum {
+        GF_DHT_MIGRATE_DATA,
+        GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
+        GF_DHT_MIGRATE_HARDLINK,
+        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
+} gf_dht_migrate_data_type_t;
 
 #define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_fs_root(loc) (strcmp (loc->path, "/") == 0)
-
-#define is_revalidate(loc) (inode_ctx_get (loc->inode, this, NULL) == 0)
+#define is_revalidate(loc) (dht_inode_ctx_layout_get (loc->inode, this, NULL) == 0)
 
 #define is_last_call(cnt) (cnt == 0)
 
 #define DHT_MIGRATION_IN_PROGRESS 1
 #define DHT_MIGRATION_COMPLETED   2
 
-#define DHT_LINKFILE_KEY         "trusted.glusterfs.dht.linkto"
 #define DHT_LINKFILE_MODE        (S_ISVTX)
 
-#define check_is_linkfile(i,s,x) (                                      \
+#define check_is_linkfile(i,s,x,n) (                                      \
                 ((st_mode_from_ia ((s)->ia_prot, (s)->ia_type) & ~S_IFMT) \
-                 == DHT_LINKFILE_MODE) &&                               \
-                dict_get (x, DHT_LINKFILE_KEY))
-
-#define check_is_linkfile_wo_dict(i,s) (                                \
-                ((st_mode_from_ia ((s)->ia_prot, (s)->ia_type) & ~S_IFMT) \
-                 == DHT_LINKFILE_MODE))
+                 == DHT_LINKFILE_MODE) &&                                 \
+                dict_get (x, n))
 
 #define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
                 IA_ISREG ((buf)->ia_type) &&                            \
@@ -289,6 +381,25 @@ typedef struct dht_disk_layout dht_disk_layout_t;
                 dht_local_wipe (__xl, __local);         \
         } while (0)
 
+#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, inode, post) do {\
+                int32_t sec = 0;                                        \
+                sec = new_sec;                                          \
+                LOCK (&inode->lock);                                    \
+                {                                                       \
+                        new_sec = max(new_sec, ctx_sec);                \
+                        if (sec < new_sec)                              \
+                                new_nsec = ctx_nsec;                    \
+                        if (sec == new_sec)                             \
+                                new_nsec = max (new_nsec, ctx_nsec);    \
+                        if (post) {                                     \
+                                ctx_sec = new_sec;                      \
+                                ctx_nsec = new_nsec;                    \
+                        }                                               \
+                }                                                       \
+                UNLOCK (&inode->lock);                                  \
+        } while (0)
+
+#define is_greater_time(a, an, b, bn) (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
 dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
 dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
 dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
@@ -298,7 +409,7 @@ int                                      dht_layout_normalize (xlator_t *this, l
 int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
                           uint32_t      *holes_p, uint32_t *overlaps_p,
                           uint32_t      *missing_p, uint32_t *down_p,
-                          uint32_t      *misc_p);
+                          uint32_t      *misc_p, uint32_t *no_space_p);
 int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
                              xlator_t   *subvol, loc_t *loc, dict_t *xattr);
 
@@ -332,12 +443,14 @@ int dht_iatt_merge (xlator_t                 *this, struct iatt *to, struct iatt
 xlator_t *dht_subvol_get_hashed (xlator_t *this, loc_t *loc);
 xlator_t *dht_subvol_get_cached (xlator_t *this, inode_t *inode);
 xlator_t *dht_subvol_next (xlator_t *this, xlator_t *prev);
+xlator_t *dht_subvol_next_available (xlator_t *this, xlator_t *prev);
 int       dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
 
-int dht_hash_compute (int type, const char *name, uint32_t *hash_p);
+int dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p);
 
 int dht_linkfile_create (call_frame_t    *frame, fop_mknod_cbk_t linkfile_cbk,
-                         xlator_t        *tovol, xlator_t *fromvol, loc_t *loc);
+                         xlator_t        *this, xlator_t *tovol,
+                         xlator_t        *fromvol, loc_t *loc);
 int                                       dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int                                       dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int
@@ -352,17 +465,15 @@ dht_selfheal_restore (call_frame_t       *frame, dht_selfheal_dir_cbk_t cbk,
 int
 dht_layout_sort_volname (dht_layout_t *layout);
 
-int dht_rename (call_frame_t *frame, xlator_t *this,
-                loc_t *oldloc, loc_t *newloc);
-
 int dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
 
 gf_boolean_t dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                          dht_local_t *layout);
 int       dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
 
 int dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode);
-int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);
+int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);;
 void          dht_layout_unref (xlator_t *this, dht_layout_t *layout);
 dht_layout_t *dht_layout_ref (xlator_t *this, dht_layout_t *layout);
 xlator_t     *dht_first_up_subvol (xlator_t *this);
@@ -377,7 +488,8 @@ int                                     dht_rename_cleanup (call_frame_t *frame)
 int dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t           op_ret, int32_t op_errno,
                           inode_t          *inode, struct iatt *stbuf,
-                          struct iatt      *preparent, struct iatt *postparent);
+                          struct iatt      *preparent, struct iatt *postparent,
+                          dict_t *xdata);
 
 int dht_fix_directory_layout (call_frame_t *frame,
                               dht_selfheal_dir_cbk_t  dir_cbk,
@@ -400,73 +512,73 @@ int32_t dht_lookup (call_frame_t *frame,
 
 int32_t dht_stat (call_frame_t *frame,
                   xlator_t *this,
-                  loc_t    *loc);
+                  loc_t    *loc, dict_t *xdata);
 
 int32_t dht_fstat (call_frame_t *frame,
                    xlator_t *this,
-                   fd_t     *fd);
+                   fd_t     *fd, dict_t *xdata);
 
 int32_t dht_truncate (call_frame_t *frame,
                       xlator_t *this,
                       loc_t    *loc,
-                      off_t     offset);
+                      off_t     offset, dict_t *xdata);
 
 int32_t dht_ftruncate (call_frame_t *frame,
                        xlator_t *this,
                        fd_t     *fd,
-                       off_t     offset);
+                       off_t     offset, dict_t *xdata);
 
 int32_t dht_access (call_frame_t *frame,
                     xlator_t *this,
                     loc_t    *loc,
-                    int32_t   mask);
+                    int32_t   mask, dict_t *xdata);
 
 int32_t dht_readlink (call_frame_t *frame,
                       xlator_t *this,
                       loc_t    *loc,
-                      size_t    size);
+                      size_t    size, dict_t *xdata);
 
-int32_t dht_mknod (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, mode_t mode, dev_t rdev, dict_t *params);
+int32_t dht_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
 
 int32_t dht_mkdir (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, mode_t mode, dict_t *params);
+                   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
 
 int32_t dht_unlink (call_frame_t *frame,
                     xlator_t *this,
-                    loc_t    *loc);
+                    loc_t    *loc, int xflag, dict_t *xdata);
 
 int32_t dht_rmdir (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, int flags);
+                   loc_t    *loc, int flags, dict_t *xdata);
 
 int32_t dht_symlink (call_frame_t   *frame, xlator_t *this,
-                     const char *linkpath, loc_t *loc, dict_t *params);
+                     const char *linkpath, loc_t *loc, mode_t umask,
+                     dict_t *xdata);
 
 int32_t dht_rename (call_frame_t *frame,
                     xlator_t *this,
                     loc_t    *oldloc,
-                    loc_t    *newloc);
+                    loc_t    *newloc, dict_t *xdata);
 
 int32_t dht_link (call_frame_t *frame,
                   xlator_t *this,
                   loc_t    *oldloc,
-                  loc_t    *newloc);
+                  loc_t    *newloc, dict_t *xdata);
 
 int32_t dht_create (call_frame_t *frame, xlator_t *this,
                     loc_t    *loc, int32_t flags, mode_t mode,
-                    fd_t     *fd, dict_t *params);
+                    mode_t umask, fd_t     *fd, dict_t *params);
 
 int32_t dht_open (call_frame_t *frame,
                   xlator_t *this,
                   loc_t    *loc,
-                  int32_t   flags, fd_t *fd,
-                  int32_t   wbflags);
+                  int32_t   flags, fd_t *fd, dict_t *xdata);
 
 int32_t dht_readv (call_frame_t *frame,
                    xlator_t *this,
                    fd_t     *fd,
                    size_t    size,
-                   off_t     offset, uint32_t flags);
+                   off_t     offset, uint32_t flags, dict_t *xdata);
 
 int32_t dht_writev (call_frame_t      *frame,
                     xlator_t      *this,
@@ -475,87 +587,87 @@ int32_t dht_writev (call_frame_t      *frame,
                     int32_t        count,
                     off_t          offset,
                     uint32_t       flags,
-                    struct iobref *iobref);
+                    struct iobref *iobref, dict_t *xdata);
 
 int32_t dht_flush (call_frame_t *frame,
                    xlator_t *this,
-                   fd_t     *fd);
+                   fd_t     *fd, dict_t *xdata);
 
 int32_t dht_fsync (call_frame_t *frame,
                    xlator_t *this,
                    fd_t     *fd,
-                   int32_t   datasync);
+                   int32_t   datasync, dict_t *xdata);
 
 int32_t dht_opendir (call_frame_t *frame,
                      xlator_t *this,
-                     loc_t    *loc, fd_t *fd);
+                     loc_t    *loc, fd_t *fd, dict_t *xdata);
 
 int32_t dht_fsyncdir (call_frame_t *frame,
                       xlator_t *this,
                       fd_t     *fd,
-                      int32_t   datasync);
+                      int32_t   datasync, dict_t *xdata);
 
 int32_t dht_statfs (call_frame_t *frame,
                     xlator_t *this,
-                    loc_t    *loc);
+                    loc_t    *loc, dict_t *xdata);
 
 int32_t dht_setxattr (call_frame_t *frame,
                       xlator_t *this,
                       loc_t    *loc,
                       dict_t   *dict,
-                      int32_t   flags);
+                      int32_t   flags, dict_t *xdata);
 
 int32_t dht_getxattr (call_frame_t   *frame,
                       xlator_t   *this,
                       loc_t      *loc,
-                      const char *name);
+                      const char *name, dict_t *xdata);
 
 int32_t dht_fsetxattr (call_frame_t *frame,
                        xlator_t *this,
                        fd_t     *fd,
                        dict_t   *dict,
-                       int32_t   flags);
+                       int32_t   flags, dict_t *xdata);
 
 int32_t dht_fgetxattr (call_frame_t   *frame,
                        xlator_t   *this,
                        fd_t       *fd,
-                       const char *name);
+                       const char *name, dict_t *xdata);
 
 int32_t dht_removexattr (call_frame_t   *frame,
                          xlator_t   *this,
                          loc_t      *loc,
-                         const char *name);
+                         const char *name, dict_t *xdata);
 int32_t dht_fremovexattr (call_frame_t   *frame,
                           xlator_t   *this,
                           fd_t      *fd,
-                          const char *name);
+                          const char *name, dict_t *xdata);
 
 int32_t dht_lk (call_frame_t        *frame,
                 xlator_t        *this,
                 fd_t            *fd,
                 int32_t          cmd,
-                struct gf_flock *flock);
+                struct gf_flock *flock, dict_t *xdata);
 
 int32_t dht_inodelk (call_frame_t *frame, xlator_t *this,
                      const char      *volume, loc_t *loc, int32_t cmd,
-                     struct gf_flock *flock);
+                     struct gf_flock *flock, dict_t *xdata);
 
 int32_t dht_finodelk (call_frame_t        *frame, xlator_t *this,
                       const char      *volume, fd_t *fd, int32_t cmd,
-                      struct gf_flock *flock);
+                      struct gf_flock *flock, dict_t *xdata);
 
 int32_t dht_entrylk (call_frame_t    *frame, xlator_t *this,
                      const char  *volume, loc_t *loc, const char *basename,
-                     entrylk_cmd  cmd, entrylk_type type);
+                     entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
 
 int32_t dht_fentrylk (call_frame_t    *frame, xlator_t *this,
                       const char  *volume, fd_t *fd, const char *basename,
-                      entrylk_cmd  cmd, entrylk_type type);
+                      entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
 
 int32_t dht_readdir (call_frame_t  *frame,
                      xlator_t *this,
                      fd_t     *fd,
-                     size_t    size, off_t off);
+                     size_t    size, off_t off, dict_t *xdata);
 
 int32_t dht_readdirp (call_frame_t *frame,
                       xlator_t *this,
@@ -566,20 +678,29 @@ int32_t dht_xattrop (call_frame_t           *frame,
                      xlator_t           *this,
                      loc_t              *loc,
                      gf_xattrop_flags_t  flags,
-                     dict_t             *dict);
+                     dict_t             *dict, dict_t *xdata);
 
 int32_t dht_fxattrop (call_frame_t *frame,
                       xlator_t           *this,
                       fd_t               *fd,
                       gf_xattrop_flags_t  flags,
-                      dict_t             *dict);
+                      dict_t             *dict, dict_t *xdata);
 
 int32_t dht_forget (xlator_t *this, inode_t *inode);
 int32_t dht_setattr (call_frame_t  *frame, xlator_t *this, loc_t *loc,
-                     struct iatt   *stbuf, int32_t valid);
+                     struct iatt   *stbuf, int32_t valid, dict_t *xdata);
 int32_t dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      struct iatt  *stbuf, int32_t valid);
-
+                      struct iatt  *stbuf, int32_t valid, dict_t *xdata);
+int32_t dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      int32_t mode, off_t offset, size_t len, dict_t *xdata);
+int32_t dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		    off_t offset, size_t len, dict_t *xdata);
+int32_t dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    off_t offset, size_t len, dict_t *xdata);
+
+int32_t dht_init (xlator_t *this);
+void    dht_fini (xlator_t *this);
+int     dht_reconfigure (xlator_t *this, dict_t *options);
 int32_t dht_notify (xlator_t *this, int32_t event, void *data, ...);
 
 /* definitions for nufa/switch */
@@ -602,12 +723,65 @@ int dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int op_ret, int op_errno,
                     fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                    struct iatt *preparent, struct iatt *postparent);
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t *xdata);
 int dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int op_ret, int op_errno,
                      inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                     struct iatt *postparent);
+                     struct iatt *postparent, dict_t *xdata);
+
+int
+gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict);
+
+int
+gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output);
 
+void*
+gf_defrag_start (void *this);
 
+int32_t
+gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
+                           struct iatt *stbuf);
+int
+dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                 int flag);
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this,
+                          dht_layout_t **layout_int);
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t* layout_int);
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t update_ctx);
+
+int dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
+int
+dht_dir_attr_heal (void *data);
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data);
+int
+dht_dir_has_layout (dict_t *xattr, char *name);
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator);
+xlator_t *
+dht_subvol_with_free_space_inodes (xlator_t *this, xlator_t *subvol,
+                                   dht_layout_t *layout);
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout);
+int
+dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this);
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix);
+int32_t
+dht_priv_dump (xlator_t *this);
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode);
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol);
 
 #endif/* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c
index 5453e3b10..fe3955ecb 100644
--- a/xlators/cluster/dht/src/dht-diskusage.c
+++ b/xlators/cluster/dht/src/dht-diskusage.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -35,7 +26,8 @@
 
 int
 dht_du_info_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, struct statvfs *statvfs)
+		 int op_ret, int op_errno, struct statvfs *statvfs,
+                 dict_t *xdata)
 {
 	dht_conf_t    *conf         = NULL;
 	call_frame_t  *prev          = NULL;
@@ -105,6 +97,7 @@ dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx)
 	call_frame_t  *statfs_frame = NULL;
 	dht_local_t   *statfs_local = NULL;
 	call_pool_t   *pool         = NULL;
+	loc_t          tmp_loc      = {0,};
 
 	conf = this->private;
 	pool = this->ctx->pool;
@@ -121,15 +114,14 @@ dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx)
 		goto err;
 	}
 
-	loc_t tmp_loc = { .inode = NULL,
-			  .path = "/",
-	};
+        /* make it root gfid, should be enough to get the proper info back */
+        tmp_loc.gfid[15] = 1;
 
 	statfs_local->call_cnt = 1;
 	STACK_WIND (statfs_frame, dht_du_info_cbk,
 		    conf->subvolumes[subvol_idx],
 		    conf->subvolumes[subvol_idx]->fops->statfs,
-		    &tmp_loc);
+		    &tmp_loc, NULL);
 
 	return 0;
 err:
@@ -142,15 +134,21 @@ err:
 int
 dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-	int            i = 0;
+	int            i            = 0;
 	dht_conf_t    *conf         = NULL;
 	call_frame_t  *statfs_frame = NULL;
 	dht_local_t   *statfs_local = NULL;
-	struct timeval tv = {0,};
+	struct timeval tv           = {0,};
+        loc_t          tmp_loc      = {0,};
 
 	conf  = this->private;
 
 	gettimeofday (&tv, NULL);
+
+        /* make it root gfid, should be enough to get the proper
+           info back */
+        tmp_loc.gfid[15] = 1;
+
 	if (tv.tv_sec > (conf->refresh_interval
 			 + conf->last_stat_fetch.tv_sec)) {
 
@@ -166,16 +164,12 @@ dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
 			goto err;
 		}
 
-		loc_t tmp_loc = { .inode = NULL,
-				  .path = "/",
-		};
-
 		statfs_local->call_cnt = conf->subvolume_cnt;
 		for (i = 0; i < conf->subvolume_cnt; i++) {
 			STACK_WIND (statfs_frame, dht_du_info_cbk,
 				    conf->subvolumes[i],
 				    conf->subvolumes[i]->fops->statfs,
-				    &tmp_loc);
+				    &tmp_loc, NULL);
 		}
 
 		conf->last_stat_fetch.tv_sec = tv.tv_sec;
@@ -254,50 +248,167 @@ dht_is_subvol_filled (xlator_t *this, xlator_t *subvol)
 	return is_subvol_filled;
 }
 
+
+/*Get the best subvolume to create the file in*/
 xlator_t *
-dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol)
+dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                dht_local_t *local)
 {
-	int         i = 0;
-	double      max = 0;
-	double      max_inodes = 0;
 	xlator_t   *avail_subvol = NULL;
 	dht_conf_t *conf = NULL;
+        dht_layout_t *layout = NULL;
+        loc_t      *loc = NULL;
 
 	conf = this->private;
-
-	LOCK (&conf->subvolume_lock);
+        if (!local)
+                goto out;
+        loc = &local->loc;
+        if (!local->layout) {
+                layout = dht_layout_get (this, loc->parent);
+
+                if (!layout) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "layout missing path=%s parent=%s",
+                                loc->path, uuid_utoa (loc->parent->gfid));
+                        goto out;
+                }
+        } else {
+                layout = dht_layout_ref (this, local->layout);
+        }
+
+        LOCK (&conf->subvolume_lock);
 	{
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (conf->disk_unit == 'p') {
-				if ((conf->du_stats[i].avail_percent > max)
-				    && (conf->du_stats[i].avail_inodes > max_inodes)) {
-					max = conf->du_stats[i].avail_percent;
-					max_inodes = conf->du_stats[i].avail_inodes;
-					avail_subvol = conf->subvolumes[i];
-				}
-			} else {
-				if ((conf->du_stats[i].avail_space > max)
-				    && (conf->du_stats[i].avail_inodes > max_inodes)) {
-					max = conf->du_stats[i].avail_space;
-					max_inodes = conf->du_stats[i].avail_inodes;
-					avail_subvol = conf->subvolumes[i];
-				}
+                avail_subvol = dht_subvol_with_free_space_inodes(this, subvol,
+                                                                 layout);
+                if(!avail_subvol)
+                {
+                        avail_subvol = dht_subvol_maxspace_nonzeroinode(this,
+                                                                        subvol,
+                                                                        layout);
+                }
 
-			}
-		}
 	}
 	UNLOCK (&conf->subvolume_lock);
-
+out:
 	if (!avail_subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume has enough free space and inodes to create");
+		gf_log (this->name,
+                        GF_LOG_DEBUG,
+			"no subvolume has enough free space and/or inodes\
+                         to create");
+                avail_subvol = subvol;
 	}
 
-	if ((max < conf->min_free_disk) && (max_inodes < conf->min_free_inodes))
-		avail_subvol = subvol;
+        if (layout)
+                dht_layout_unref (this, layout);
+	return avail_subvol;
+}
 
-	if (!avail_subvol)
-		avail_subvol = subvol;
+static inline
+int32_t dht_subvol_has_err (xlator_t *this, dht_layout_t *layout)
+{
+        int ret = -1;
+        int i   = 0;
+
+        if (!this || !layout)
+                goto out;
+
+        /* check if subvol has layout errors, before selecting it */
+        for (i = 0; i < layout->cnt; i++) {
+                if (!strcmp (layout->list[i].xlator->name, this->name) &&
+                     (layout->list[i].err != 0)) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
 
-	return avail_subvol;
+/*Get subvolume which has both space and inodes more than the min criteria*/
+xlator_t *
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
+{
+        int i = 0;
+        double max = 0;
+        double max_inodes = 0;
+        int    ignore_subvol = 0;
+
+        xlator_t *avail_subvol = NULL;
+        dht_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        for(i=0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
+                if ((conf->disk_unit == 'p') &&
+                    (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
+                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
+                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                            (conf->du_stats[i].avail_percent > max)) {
+                                max = conf->du_stats[i].avail_percent;
+                                max_inodes = conf->du_stats[i].avail_inodes;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+                }
+
+                if ((conf->disk_unit != 'p') &&
+                    (conf->du_stats[i].avail_space > conf->min_free_disk) &&
+                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
+                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                            (conf->du_stats[i].avail_space > max)) {
+                                max = conf->du_stats[i].avail_space;
+                                max_inodes = conf->du_stats[i].avail_inodes;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+                }
+        }
+
+        return avail_subvol;
+}
+
+
+/* Get subvol which has atleast one inode and maximum space */
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
+{
+        int         i = 0;
+        double      max = 0;
+        int         ignore_subvol = 0;
+
+        xlator_t   *avail_subvol = NULL;
+        dht_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
+                if (conf->disk_unit == 'p') {
+                        if ((conf->du_stats[i].avail_percent > max)
+                            && (conf->du_stats[i].avail_inodes > 0 )) {
+                                max = conf->du_stats[i].avail_percent;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+               } else {
+                         if ((conf->du_stats[i].avail_space > max)
+                            && (conf->du_stats[i].avail_inodes > 0)) {
+                                 max = conf->du_stats[i].avail_space;
+                                 avail_subvol = conf->subvolumes[i];
+                         }
+               }
+        }
+
+        return avail_subvol;
 }
diff --git a/xlators/cluster/dht/src/dht-hashfn.c b/xlators/cluster/dht/src/dht-hashfn.c
index c8ae74172..656cf23a0 100644
--- a/xlators/cluster/dht/src/dht-hashfn.c
+++ b/xlators/cluster/dht/src/dht-hashfn.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -37,6 +28,7 @@ dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
 
         switch (type) {
         case DHT_HASH_TYPE_DM:
+        case DHT_HASH_TYPE_DM_USER:
                 hash = gf_dm_hashfn (name, strlen (name));
                 break;
         default:
@@ -52,30 +44,68 @@ dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
 }
 
 
-#define MAKE_RSYNC_FRIENDLY_NAME(rsync_frndly_name, name) do {          \
-                rsync_frndly_name = (char *) name;                      \
-                if (name[0] == '.') {                                   \
-                        char *dot   = 0;                                \
-                        int namelen = 0;                                \
-                                                                        \
-                        dot = strrchr (name, '.');                      \
-                        if (dot && dot > (name + 1) && *(dot + 1)) {    \
-                                namelen = (dot - name);                 \
-                                rsync_frndly_name = alloca (namelen);   \
-                                strncpy (rsync_frndly_name, name + 1,   \
-                                         namelen);                      \
-                                rsync_frndly_name[namelen - 1] = 0;     \
-                        }                                               \
-                }                                                       \
-        } while (0);
+static inline
+gf_boolean_t
+dht_munge_name (const char *original, char *modified, size_t len, regex_t *re)
+{
+        regmatch_t      matches[2];
+        size_t          new_len;
+
+        if (regexec(re,original,2,matches,0) != REG_NOMATCH) {
+                if (matches[1].rm_so != -1) {
+                        new_len = matches[1].rm_eo - matches[1].rm_so;
+                        /* Equal would fail due to the NUL at the end. */
+                        if (new_len < len) {
+                                memcpy (modified,original+matches[1].rm_so,
+                                        new_len);
+                                modified[new_len] = '\0';
+                                return _gf_true;
+                        }
+                }
+        }
 
+        /* This is guaranteed safe because of how the dest was allocated. */
+        strcpy(modified,original);
+        return _gf_false;
+}
 
 int
-dht_hash_compute (int type, const char *name, uint32_t *hash_p)
+dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
-        char     *rsync_friendly_name = NULL;
+        char            *rsync_friendly_name    = NULL;
+        dht_conf_t      *priv                   = this->private;
+        size_t           len                    = 0;
+        gf_boolean_t     munged                 = _gf_false;
+
+        /*
+         * It wouldn't be safe to use alloca in an inline function that doesn't
+         * actually get inlined, and it wouldn't be efficient to do a real
+         * allocation, so we use alloca here (if needed) and pass that to the
+         * inline.
+         */
+
+        if (priv->extra_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->extra_regex);
+        }
+
+        if (!munged && priv->rsync_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                gf_log (this->name, GF_LOG_TRACE, "trying regex for %s", name);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->rsync_regex);
+                if (munged) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "munged down to %s", rsync_friendly_name);
+                }
+        }
 
-        MAKE_RSYNC_FRIENDLY_NAME (rsync_friendly_name, name);
+        if (!munged) {
+                rsync_friendly_name = (char *)name;
+        }
 
         return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
 }
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index 80de4d738..311a48112 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -27,6 +18,28 @@
 #include "xlator.h"
 #include "dht-common.h"
 
+static inline int
+dht_inode_ctx_set1 (xlator_t *this, inode_t *inode, xlator_t *subvol)
+{
+        uint64_t tmp_subvol = 0;
+
+        tmp_subvol = (long)subvol;
+        return inode_ctx_set1 (inode, this, &tmp_subvol);
+}
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol)
+{
+        int ret = -1;
+        uint64_t tmp_subvol = 0;
+
+        ret =  inode_ctx_get1 (inode, this, &tmp_subvol);
+        if (tmp_subvol && subvol)
+                *subvol = (xlator_t *)tmp_subvol;
+
+        return ret;
+}
+
 
 int
 dht_frame_return (call_frame_t *frame)
@@ -49,6 +62,43 @@ dht_frame_return (call_frame_t *frame)
 }
 
 
+static uint64_t
+dht_bits_for (uint64_t num)
+{
+	uint64_t bits = 0, ctrl = 1;
+
+	while (ctrl < num) {
+		ctrl *= 2;
+		bits ++;
+	}
+
+	return bits;
+}
+
+/*
+ * A slightly "updated" version of the algorithm described in the commit log
+ * is used here.
+ *
+ * The only enhancement is that:
+ *
+ * - The number of bits used by the backend filesystem for HUGE d_off which
+ *   is described as 63, and
+ * - The number of bits used by the d_off presented by the transformation
+ *   upwards which is described as 64, are both made "configurable."
+ */
+
+
+#define BACKEND_D_OFF_BITS 63
+#define PRESENT_D_OFF_BITS 63
+
+#define ONE 1ULL
+#define MASK (~0ULL)
+#define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS))
+#define BACKEND_MASK (MASK >> (64 - BACKEND_D_OFF_BITS))
+
+#define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1))
+#define SHIFT_BITS (max (0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
+
 int
 dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
 {
@@ -56,6 +106,9 @@ dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
         int         cnt = 0;
         int         max = 0;
         uint64_t    y = 0;
+        uint64_t    hi_mask = 0;
+        uint64_t    off_mask = 0;
+        int         max_bits = 0;
 
         if (x == ((uint64_t) -1)) {
                 y = (uint64_t) -1;
@@ -69,7 +122,23 @@ dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
         max = conf->subvolume_cnt;
         cnt = dht_subvol_cnt (this, subvol);
 
-        y = ((x * max) + cnt);
+	if (max == 1) {
+		y = x;
+		goto out;
+	}
+
+        max_bits = dht_bits_for (max);
+
+        hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
+
+        if (x & hi_mask) {
+                /* HUGE d_off */
+                off_mask = MASK << max_bits;
+                y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | cnt;
+        } else {
+                /* small d_off */
+                y = ((x * max) + cnt);
+        }
 
 out:
         if (y_p)
@@ -89,7 +158,7 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
         int            ret       = 0; /* not found */
 
         /* Why do other tasks if first required 'char' itself is not there */
-        if (!loc->name || !strchr (loc->name, '@'))
+        if (!new_loc || !loc || !loc->name || !strchr (loc->name, '@'))
                 goto out;
 
         trav = this->children;
@@ -129,10 +198,8 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
 out:
         if (!ret) {
                 /* !success */
-                if (new_path)
-                        GF_FREE (new_path);
-                if (new_name)
-                        GF_FREE (new_name);
+                GF_FREE (new_path);
+                GF_FREE (new_name);
         }
         return ret;
 }
@@ -146,16 +213,38 @@ dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
         int         max = 0;
         uint64_t    x = 0;
         xlator_t   *subvol = 0;
+        int         max_bits = 0;
+        uint64_t    off_mask = 0;
+        uint64_t    host_mask = 0;
 
         if (!this->private)
-                goto out;
+                return -1;
 
         conf = this->private;
         max = conf->subvolume_cnt;
 
-        cnt = y % max;
-        x   = y / max;
+	if (max == 1) {
+		x = y;
+		cnt = 0;
+		goto out;
+	}
+
+        if (y & TOP_BIT) {
+                /* HUGE d_off */
+                max_bits = dht_bits_for (max);
+                off_mask = (MASK << max_bits);
+                host_mask = ~(off_mask);
+
+                x = ((y & ~TOP_BIT) & off_mask) << SHIFT_BITS;
+
+                cnt = y & host_mask;
+	} else {
+                /* small d_off */
+                cnt = y % max;
+                x = y / max;
+        }
 
+out:
         subvol = conf->subvolumes[cnt];
 
         if (subvol_p)
@@ -164,7 +253,6 @@ dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
         if (x_p)
                 *x_p = x;
 
-out:
         return 0;
 }
 
@@ -215,21 +303,16 @@ dht_local_wipe (xlator_t *this, dht_local_t *local)
                 local->selfheal.layout = NULL;
         }
 
-        if (local->newpath) {
-                GF_FREE (local->newpath);
-        }
+        GF_FREE (local->newpath);
 
-        if (local->key) {
-                GF_FREE (local->key);
-        }
+        GF_FREE (local->key);
 
-        if (local->rebalance.vector)
-                GF_FREE (local->rebalance.vector);
+        GF_FREE (local->rebalance.vector);
 
         if (local->rebalance.iobref)
                 iobref_unref (local->rebalance.iobref);
 
-        GF_FREE (local);
+        mem_put (local);
 }
 
 
@@ -240,8 +323,7 @@ dht_local_init (call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
         inode_t     *inode = NULL;
         int          ret   = 0;
 
-        /* TODO: use mem-pool */
-        local = GF_CALLOC (1, sizeof (*local), gf_dht_mt_dht_local_t);
+        local = mem_get0 (THIS->local_pool);
         if (!local)
                 goto out;
 
@@ -274,26 +356,12 @@ dht_local_init (call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
 out:
         if (ret) {
                 if (local)
-                        GF_FREE (local);
+                        mem_put (local);
                 local = NULL;
         }
         return local;
 }
 
-
-char *
-basestr (const char *str)
-{
-        char *basestr = NULL;
-
-        basestr = strrchr (str, '/');
-        if (basestr)
-                basestr ++;
-
-        return basestr;
-}
-
-
 xlator_t *
 dht_first_up_subvol (xlator_t *this)
 {
@@ -358,11 +426,17 @@ dht_subvol_get_hashed (xlator_t *this, loc_t *loc)
         dht_layout_t *layout = NULL;
         xlator_t     *subvol = NULL;
 
-        if (is_fs_root (loc)) {
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, loc, out);
+
+        if (__is_root_gfid (loc->gfid)) {
                 subvol = dht_first_up_subvol (this);
                 goto out;
         }
 
+        GF_VALIDATE_OR_GOTO (this->name, loc->parent, out);
+        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
+
         layout = dht_layout_get (this, loc->parent);
 
         if (!layout) {
@@ -396,6 +470,8 @@ dht_subvol_get_cached (xlator_t *this, inode_t *inode)
         dht_layout_t *layout = NULL;
         xlator_t     *subvol = NULL;
 
+        GF_VALIDATE_OR_GOTO (this->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
 
         layout = dht_layout_get (this, inode);
 
@@ -437,7 +513,36 @@ out:
         return next;
 }
 
+/* This func wraps around, if prev is actually the last subvol.
+ */
+xlator_t *
+dht_subvol_next_available (xlator_t *this, xlator_t *prev)
+{
+        dht_conf_t *conf = NULL;
+        int         i = 0;
+        xlator_t   *next = NULL;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == prev) {
+                        /* if prev is last in conf->subvolumes, then wrap
+                         * around.
+                         */
+                        if ((i + 1) < conf->subvolume_cnt) {
+                                next = conf->subvolumes[i + 1];
+                        } else {
+                                next = conf->subvolumes[0];
+                        }
+                        break;
+                }
+        }
 
+out:
+        return next;
+}
 int
 dht_subvol_cnt (xlator_t *this, xlator_t *subvol)
 {
@@ -629,20 +734,36 @@ dht_migration_complete_check_task (void *data)
         call_frame_t *frame    = NULL;
         loc_t         tmp_loc  = {0,};
         char         *path     = NULL;
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        uint64_t      tmp_subvol = 0;
+        int           open_failed = 0;
 
         this  = THIS;
         frame = data;
         local = frame->local;
+        conf = this->private;
 
         src_node = local->cached_subvol;
 
-        /* getxattr on cached_subvol for 'linkto' value */
-        if (!local->loc.inode)
+        if (!local->loc.inode && !local->fd)
+                goto out;
+
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+
+        if (!local->loc.inode) {
                 ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        DHT_LINKFILE_KEY);
-        else
+                                        conf->link_xattr_name);
+        } else {
+                SYNCTASK_SETID (0, 0);
                 ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       DHT_LINKFILE_KEY);
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        }
 
         if (!ret)
                 dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
@@ -693,10 +814,7 @@ dht_migration_complete_check_task (void *data)
         /* update inode ctx (the layout) */
         dht_layout_unref (this, local->layout);
 
-        if (!local->loc.inode)
-                ret = dht_layout_preset (this, dst_node, local->fd->inode);
-        else
-                ret = dht_layout_preset (this, dst_node, local->loc.inode);
+        ret = dht_layout_preset (this, dst_node, inode);
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "%s: could not set preset layout for subvol %s",
@@ -714,10 +832,7 @@ dht_migration_complete_check_task (void *data)
                 goto out;
         }
 
-        if (!local->loc.inode)
-                ret = dht_layout_set (this, local->fd->inode, layout);
-        else
-                ret = dht_layout_set (this, local->loc.inode, layout);
+        ret = dht_layout_set (this, inode, layout);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set the new layout",
@@ -728,43 +843,46 @@ dht_migration_complete_check_task (void *data)
         local->cached_subvol = dst_node;
         ret = 0;
 
-        if (!local->fd)
+        /* once we detect the migration complete, the inode-ctx2 is no more
+           required.. delete the ctx and also, it means, open() already
+           done on all the fd of inode */
+        ret = inode_ctx_reset1 (inode, this, &tmp_subvol);
+        if (tmp_subvol)
                 goto out;
 
-        /* once we detect the migration complete, the fd-ctx is no more
-           required.. delete the ctx, and do one extra 'fd_unref' for open fd */
-        ret = fd_ctx_del (local->fd, this, NULL);
-        if (!ret) {
-                fd_unref (local->fd);
-                ret = 0;
+        if (list_empty (&inode->fd_list))
                 goto out;
-        }
 
-        /* if 'local->fd' (ie, fd based operation), send a 'open()' on
-           destination if not already done */
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
-                ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                if (path)
-                        GF_FREE (path);
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID(0, 0);
 
+        /* perform 'open()' on all the fd's present on the inode */
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
+                ret = syncop_open (dst_node, &tmp_loc,
+                                   iter_fd->flags, iter_fd);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                }
         }
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
+        GF_FREE (path);
+
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        if (open_failed) {
+                ret = -1;
                 goto out;
         }
-
-        /* need this unref for the fd on src_node */
-        fd_unref (local->fd);
         ret = 0;
 out:
 
@@ -775,11 +893,8 @@ int
 dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
 {
         int         ret     = -1;
-        dht_conf_t *conf    = NULL;
-
-        conf = this->private;
 
-        ret = synctask_new (conf->env, dht_migration_complete_check_task,
+        ret = synctask_new (this->ctx->env, dht_migration_complete_check_task,
                             dht_migration_complete_check_done,
                             frame, frame);
         return ret;
@@ -811,20 +926,34 @@ dht_rebalance_inprogress_task (void *data)
         char         *path     = NULL;
         struct iatt   stbuf    = {0,};
         loc_t         tmp_loc  = {0,};
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        int           open_failed = 0;
 
         this  = THIS;
         frame = data;
         local = frame->local;
+        conf = this->private;
 
         src_node = local->cached_subvol;
 
-        /* getxattr on cached_subvol for 'linkto' value */
-        if (local->loc.inode)
+        if (!local->loc.inode && !local->fd)
+                goto out;
+
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+        if (local->loc.inode) {
+                SYNCTASK_SETID (0, 0);
                 ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       DHT_LINKFILE_KEY);
-        else
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        } else {
                 ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        DHT_LINKFILE_KEY);
+                                        conf->link_xattr_name);
+        }
 
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -866,34 +995,46 @@ dht_rebalance_inprogress_task (void *data)
 
         ret = 0;
 
-        if (!local->fd)
-                goto out;
+        if (list_empty (&inode->fd_list))
+                goto done;
+
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID (0, 0);
+
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
 
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
                 ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                if (path)
-                        GF_FREE (path);
+                                   iter_fd->flags, iter_fd);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to send open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                }
         }
+        GF_FREE (path);
 
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+
+        if (open_failed) {
+                ret = -1;
                 goto out;
         }
 
-        ret = fd_ctx_set (local->fd, this, (uint64_t)(long)dst_node);
+done:
+        ret = dht_inode_ctx_set1 (this, inode, dst_node);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set fd-ctx target file at %s",
+                        "%s: failed to set inode-ctx target file at %s",
                         local->loc.path, dst_node->name);
                 goto out;
         }
@@ -908,12 +1049,99 @@ dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame)
 {
 
         int         ret     = -1;
-        dht_conf_t *conf    = NULL;
 
-        conf = this->private;
-
-        ret = synctask_new (conf->env, dht_rebalance_inprogress_task,
+        ret = synctask_new (this->ctx->env, dht_rebalance_inprogress_task,
                             dht_inprogress_check_done,
                             frame, frame);
         return ret;
 }
+
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t *layout_int)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+        if (!ret && ctx) {
+                ctx->layout = layout_int;
+        } else {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return ret;
+                ctx->layout = layout_int;
+        }
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+
+        return ret;
+}
+
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t post)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        dht_stat_time_t         *time           = 0;
+        int                      ret            = -1;
+
+        GF_VALIDATE_OR_GOTO (this->name, stat, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (ret) {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return -1;
+        }
+
+        time = &ctx->time;
+
+        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec,
+                        stat->ia_mtime, stat->ia_mtime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec,
+                        stat->ia_ctime, stat->ia_ctime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->atime, time->atime_nsec,
+                        stat->ia_atime, stat->ia_atime_nsec, inode, post);
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+out:
+        return 0;
+}
+
+int
+dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
+{
+        int             ret     = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = inode_ctx_get (inode, this, &ctx_int);
+
+        if (ret)
+                return ret;
+
+        if (ctx)
+                *ctx = (dht_inode_ctx_t *) ctx_int;
+out:
+        return ret;
+}
+
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
+{
+        int             ret = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+
+        ctx_int = (long)ctx;
+        ret = inode_ctx_set (inode, this, &ctx_int);
+out:
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 3ec6f4ba2..ece84151a 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -34,7 +25,7 @@ int dht_fsync2 (xlator_t *this, call_frame_t *frame, int ret);
 
 int
 dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, fd_t *fd)
+              int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -61,7 +52,7 @@ dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 return 0;
 
 out:
-        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd);
+        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd, xdata);
 
         return 0;
 }
@@ -86,17 +77,17 @@ dht_open2 (xlator_t *this, call_frame_t *frame, int op_ret)
 
         STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
                     &local->loc, local->rebalance.flags, local->fd,
-                    local->rebalance.wbflags);
+                    NULL);
         return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 int
 dht_open (call_frame_t *frame, xlator_t *this,
-          loc_t *loc, int flags, fd_t *fd, int wbflags)
+          loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -120,30 +111,30 @@ dht_open (call_frame_t *frame, xlator_t *this,
                 goto err;
         }
 
-        local->rebalance.wbflags = wbflags;
         local->rebalance.flags = flags;
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *stbuf)
+                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        uint64_t      tmp_subvol = 0;
+        xlator_t     *subvol = 0;
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        inode_t      *inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -164,26 +155,28 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (local->call_cnt != 1)
                 goto out;
 
+        local->op_errno = op_errno;
         /* Check if the rebalance phase2 is true */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                if (local->fd)
-                        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-                if (ret) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
                         /* Phase 2 of migration */
                         local->rebalance.target_op_fn = dht_attr2;
                         ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_attr2 (this, frame, 0);
-                }
-                if (!ret)
                         return 0;
+                }
         }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf);
+        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf, xdata);
 err:
         return 0;
 }
@@ -208,21 +201,21 @@ dht_attr2 (xlator_t *this, call_frame_t *frame, int op_ret)
 
         if (local->fop == GF_FOP_FSTAT) {
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, local->fd);
+                            subvol->fops->fstat, local->fd, NULL);
         } else {
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, &local->loc);
+                            subvol->fops->stat, &local->loc, NULL);
         }
         return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 int
 dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, struct iatt *stbuf)
+              int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -257,14 +250,14 @@ out:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
                 DHT_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf);
+                                  &local->stbuf, xdata);
         }
 err:
         return 0;
 }
 
 int
-dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -300,7 +293,7 @@ dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 subvol = local->cached_subvol;
 
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, loc);
+                            subvol->fops->stat, loc, xdata);
 
                 return 0;
         }
@@ -312,21 +305,21 @@ dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
                 STACK_WIND (frame, dht_attr_cbk,
                             subvol, subvol->fops->stat,
-                            loc);
+                            loc, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -360,7 +353,7 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 subvol = local->cached_subvol;
 
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, fd);
+                            subvol->fops->fstat, fd, xdata);
 
                 return 0;
         }
@@ -371,14 +364,14 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 subvol = layout->list[i].xlator;
                 STACK_WIND (frame, dht_attr_cbk,
                             subvol, subvol->fops->fstat,
-                            fd);
+                            fd, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -387,10 +380,12 @@ int
 dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno,
                struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref)
+               struct iobref *iobref, dict_t *xdata)
 {
         dht_local_t *local      = NULL;
         int          ret        = 0;
+        inode_t     *inode      = NULL;
+        xlator_t    *subvol = 0;
 
         local = frame->local;
         if (!local) {
@@ -406,25 +401,27 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno != ENOENT))
                 goto out;
 
+        local->op_errno = op_errno;
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
                 /* File would be migrated to other node */
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (ret) {
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
                         local->rebalance.target_op_fn = dht_readv2;
                         ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_readv2 (this, frame, 0);
-                }
-                if (!ret)
                         return 0;
+                }
         }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
-                          iobref);
+                          iobref, xdata);
 
         return 0;
 }
@@ -449,18 +446,18 @@ dht_readv2 (xlator_t *this, call_frame_t *frame, int op_ret)
 
         STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
                     local->fd, local->rebalance.size, local->rebalance.offset,
-                    local->rebalance.flags);
+                    local->rebalance.flags, NULL);
 
         return 0;
 
 out:
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
         return 0;
 }
 
 int
 dht_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t off, uint32_t flags)
+           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -491,31 +488,52 @@ dht_readv (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_readv_cbk,
                     subvol, subvol->fops->readv,
-                    fd, size, off, flags);
+                    fd, size, off, flags, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno)
+                int op_ret, int op_errno, dict_t *xdata)
 {
         int          ret = -1;
         dht_local_t *local = NULL;
+        xlator_t    *subvol = NULL;
+        call_frame_t *prev = NULL;
 
         local = frame->local;
+        prev = cookie;
 
+        if (!prev || !prev->this)
+                goto out;
         if (local->call_cnt != 1)
                 goto out;
+        if ((op_ret == -1) && (op_errno == ENOTCONN) &&
+            IA_ISDIR(local->loc.inode->ia_type)) {
+
+                subvol = dht_subvol_next_available (this, prev->this);
+                if (!subvol)
+                        goto out;
 
+                /* check if we are done with visiting every node */
+                if (subvol == local->cached_subvol) {
+                        goto out;
+                }
+
+                STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
+                            &local->loc, local->rebalance.flags, NULL);
+                return 0;
+        }
         if ((op_ret == -1) && (op_errno == ENOENT)) {
                 /* File would be migrated to other node */
+                local->op_errno = op_errno;
                 local->rebalance.target_op_fn = dht_access2;
                 ret = dht_rebalance_complete_check (frame->this, frame);
                 if (!ret)
@@ -523,7 +541,7 @@ dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 out:
-        DHT_STACK_UNWIND (access, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -546,18 +564,19 @@ dht_access2 (xlator_t *this, call_frame_t *frame, int op_ret)
         subvol = local->cached_subvol;
 
         STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    &local->loc, local->rebalance.flags);
+                    &local->loc, local->rebalance.flags, NULL);
 
         return 0;
 
 out:
-        DHT_STACK_UNWIND (access, frame, -1, op_errno);
+        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int
-dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+            dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -586,13 +605,13 @@ dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
         }
 
         STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    loc, mask);
+                    loc, mask, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (access, frame, -1, op_errno);
+        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -600,10 +619,11 @@ err:
 
 int
 dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno)
+               int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           ret = -1;
+        dht_local_t  *local  = NULL;
+        inode_t      *inode  = NULL;
+        xlator_t     *subvol = 0;
 
         local = frame->local;
 
@@ -613,14 +633,14 @@ dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
 
         /* If context is set, then send flush() it to the destination */
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (!ret) {
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (subvol) {
                 dht_flush2 (this, frame, 0);
                 return 0;
         }
 
 out:
-        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -630,14 +650,10 @@ dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -645,20 +661,19 @@ dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
         local->call_cnt = 2; /* This is the second attempt */
 
         STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, local->fd);
+                    subvol, subvol->fops->flush, local->fd, NULL);
 
         return 0;
 }
 
 
 int
-dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
 
-
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
@@ -680,13 +695,13 @@ dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, fd);
+                    subvol, subvol->fops->flush, fd, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (flush, frame, -1, op_errno);
+        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -694,17 +709,20 @@ err:
 
 int
 dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-               int op_errno, struct iatt *prebuf, struct iatt *postbuf)
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        inode_t      *inode = NULL;
+        xlator_t     *subvol = 0;
 
         local = frame->local;
         prev = cookie;
 
         local->op_errno = op_errno;
-        if (op_ret == -1) {
+        if (op_ret == -1 && (op_errno != ENOENT)) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "subvolume %s returned -1 (%s)",
                         prev->this->name, strerror (op_errno));
@@ -719,8 +737,9 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 goto out;
         }
 
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (ret) {
+        local->op_errno = op_errno;
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (!subvol) {
                 local->rebalance.target_op_fn = dht_fsync2;
 
                 /* Check if the rebalance phase1 is true */
@@ -735,17 +754,18 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                         ret = dht_rebalance_complete_check (this, frame);
                 }
+                if (!ret)
+                        return 0;
         } else {
                 dht_fsync2 (this, frame, 0);
-        }
-        if (!ret)
                 return 0;
+        }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
-                          prebuf, postbuf);
+                          prebuf, postbuf, xdata);
 
         return 0;
 }
@@ -755,34 +775,29 @@ dht_fsync2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
-
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
         if (!subvol)
                 subvol = local->cached_subvol;
 
         local->call_cnt = 2; /* This is the second attempt */
 
         STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    local->fd, local->rebalance.flags);
+                    local->fd, local->rebalance.flags, NULL);
 
         return 0;
 }
 
 int
-dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+           dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
 
-
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
@@ -800,13 +815,13 @@ dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
         subvol = local->cached_subvol;
 
         STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    fd, datasync);
+                    fd, datasync, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -817,9 +832,9 @@ err:
    phase 2 of migration */
 int
 dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, struct gf_flock *flock)
+            int op_ret, int op_errno, struct gf_flock *flock, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock);
+        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock, xdata);
 
         return 0;
 }
@@ -827,7 +842,7 @@ dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int cmd, struct gf_flock *flock)
+        fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -847,13 +862,13 @@ dht_lk (call_frame_t *frame, xlator_t *this,
 
         /* TODO: for rebalance, we need to preserve the fop arguments */
         STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, fd,
-                    cmd, flock);
+                    cmd, flock, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -861,7 +876,8 @@ err:
 /* Symlinks are currently not migrated, so no need for any check here */
 int
 dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, const char *path, struct iatt *stbuf)
+                  int op_ret, int op_errno, const char *path,
+                  struct iatt *stbuf, dict_t *xdata)
 {
         dht_local_t *local = NULL;
 
@@ -876,14 +892,15 @@ dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 err:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf);
+        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf, xdata);
 
         return 0;
 }
 
 
 int
-dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+              dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -911,13 +928,13 @@ dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
 
         STACK_WIND (frame, dht_readlink_cbk,
                     subvol, subvol->fops->readlink,
-                    loc, size);
+                    loc, size, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -928,16 +945,16 @@ err:
 
 int
 dht_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict)
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict);
+        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t flags, dict_t *dict)
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -968,13 +985,13 @@ dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
         STACK_WIND (frame,
                     dht_xattrop_cbk,
                     subvol, subvol->fops->xattrop,
-                    loc, flags, dict);
+                    loc, flags, dict, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -982,16 +999,16 @@ err:
 
 int
 dht_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict)
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict);
+        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 dht_fxattrop (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -1011,13 +1028,13 @@ dht_fxattrop (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame,
                     dht_fxattrop_cbk,
                     subvol, subvol->fops->fxattrop,
-                    fd, flags, dict);
+                    fd, flags, dict, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -1025,17 +1042,17 @@ err:
 
 int
 dht_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int32_t
-dht_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
+dht_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+             loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -1067,31 +1084,31 @@ dht_inodelk (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame,
                     dht_inodelk_cbk,
                     subvol, subvol->fops->inodelk,
-                    volume, loc, cmd, lock);
+                    volume, loc, cmd, lock, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
 
         return 0;
 }
 
 
 int
-dht_finodelk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+dht_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
-dht_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
+dht_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+              fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -1109,16 +1126,14 @@ dht_finodelk (call_frame_t *frame, xlator_t *this,
         }
 
 
-        STACK_WIND (frame,
-                    dht_finodelk_cbk,
-                    subvol, subvol->fops->finodelk,
-                    volume, fd, cmd, lock);
+        STACK_WIND (frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
+                    volume, fd, cmd, lock, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
 
         return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index 2f59e7c94..4b3f3a049 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -28,16 +19,20 @@
 int dht_writev2 (xlator_t *this, call_frame_t *frame, int ret);
 int dht_truncate2 (xlator_t *this, call_frame_t *frame, int ret);
 int dht_setattr2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret);
 
 int
 dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf)
+                struct iatt *postbuf, dict_t *xdata)
 {
         dht_local_t *local = NULL;
         int          ret   = -1;
+        xlator_t    *subvol = NULL;
 
-        if (op_ret == -1) {
+        if (op_ret == -1 && (op_errno != ENOENT)) {
                 goto out;
         }
 
@@ -59,6 +54,7 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local->rebalance.target_op_fn = dht_writev2;
 
+        local->op_errno = op_errno;
         /* Phase 2 of migration */
         if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -71,8 +67,8 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
 
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                ret = dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
                         dht_writev2 (this, frame, 0);
                         return 0;
                 }
@@ -85,7 +81,8 @@ out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
 
-        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 
         return 0;
 }
@@ -95,14 +92,10 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -113,7 +106,7 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
                     subvol, subvol->fops->writev,
                     local->fd, local->rebalance.vector, local->rebalance.count,
                     local->rebalance.offset, local->rebalance.flags,
-                    local->rebalance.iobref);
+                    local->rebalance.iobref, NULL);
 
         return 0;
 }
@@ -121,7 +114,7 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
 int
 dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int count, off_t off, uint32_t flags,
-            struct iobref *iobref)
+            struct iobref *iobref, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -156,13 +149,13 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         STACK_WIND (frame, dht_writev_cbk,
                     subvol, subvol->fops->writev,
-                    fd, vector, count, off, flags, iobref);
+                    fd, vector, count, off, flags, iobref, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -172,11 +165,13 @@ err:
 int
 dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        xlator_t    *subvol = NULL;
+        inode_t      *inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -206,6 +201,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local->rebalance.target_op_fn = dht_truncate2;
 
+        local->op_errno = op_errno;
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -217,8 +213,9 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                dht_inode_ctx_get1 (this, inode, &subvol);
+                if (subvol) {
                         dht_truncate2 (this, frame, 0);
                         return 0;
                 }
@@ -231,7 +228,7 @@ out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
-                          prebuf, postbuf);
+                          prebuf, postbuf, xdata);
 err:
         return 0;
 }
@@ -242,16 +239,13 @@ dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+        inode_t      *inode = NULL;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        inode = local->fd ? local->fd->inode : local->loc.inode;
 
+        dht_inode_ctx_get1 (this, inode, &subvol);
         if (!subvol)
                 subvol = local->cached_subvol;
 
@@ -260,18 +254,19 @@ dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
         if (local->fop == GF_FOP_TRUNCATE) {
                 STACK_WIND (frame, dht_truncate_cbk, subvol,
                             subvol->fops->truncate, &local->loc,
-                            local->rebalance.offset);
+                            local->rebalance.offset, NULL);
         } else {
                 STACK_WIND (frame, dht_truncate_cbk, subvol,
                             subvol->fops->ftruncate, local->fd,
-                            local->rebalance.offset);
+                            local->rebalance.offset, NULL);
         }
 
         return 0;
 }
 
 int
-dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -301,19 +296,20 @@ dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
 
         STACK_WIND (frame, dht_truncate_cbk,
                     subvol, subvol->fops->truncate,
-                    loc, offset);
+                    loc, offset, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
 
 int
-dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -341,22 +337,423 @@ dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
 
         STACK_WIND (frame, dht_truncate_cbk,
                     subvol, subvol->fops->ftruncate,
-                    fd, offset);
+                    fd, offset, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
 
+
+int
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_fallocate2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_fallocate2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (fallocate, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+	STACK_WIND(frame, dht_fallocate_cbk, subvol, subvol->fops->fallocate,
+		   local->fd, local->rebalance.flags, local->rebalance.offset,
+		   local->rebalance.size, NULL);
+
+        return 0;
+}
+
+int
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+	      off_t offset, size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FALLOCATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+	local->rebalance.flags = mode;
+        local->rebalance.offset = offset;
+	local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_fallocate_cbk,
+                    subvol, subvol->fops->fallocate,
+                    fd, mode, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_discard2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_discard2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (discard, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+	STACK_WIND(frame, dht_discard_cbk, subvol, subvol->fops->discard,
+		   local->fd, local->rebalance.offset, local->rebalance.size,
+		   NULL);
+
+        return 0;
+}
+
+int
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	    size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_DISCARD);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+	local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_discard_cbk, subvol, subvol->fops->discard,
+                    fd, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev  = NULL;
+        int           ret   = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_zerofill2;
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                ret = fd_ctx_get (local->fd, this, NULL);
+                if (!ret) {
+                        dht_zerofill2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local          = NULL;
+        xlator_t     *subvol         = NULL;
+        uint64_t      tmp_subvol     = 0;
+        int           ret            = -1;
+
+        local = frame->local;
+
+        if (local->fd)
+                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
+        if (!ret)
+                subvol = (xlator_t *)(long)tmp_subvol;
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND(frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                   local->fd, local->rebalance.offset, local->rebalance.size,
+                   NULL);
+
+        return 0;
+}
+
+int
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol       = NULL;
+        int           op_errno     = -1;
+        dht_local_t  *local        = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_ZEROFILL);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+        local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                    fd, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+
 /* handle cases of migration here for 'setattr()' calls */
 int
 dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int op_ret, int op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -393,7 +790,7 @@ out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                          prebuf, postbuf);
+                          prebuf, postbuf, xdata);
 
         return 0;
 }
@@ -403,15 +800,13 @@ dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+        inode_t      *inode = NULL;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get1 (this, inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -421,11 +816,13 @@ dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
         if (local->fop == GF_FOP_SETATTR) {
                 STACK_WIND (frame, dht_file_setattr_cbk, subvol,
                             subvol->fops->setattr, &local->loc,
-                            &local->rebalance.stbuf, local->rebalance.flags);
+                            &local->rebalance.stbuf, local->rebalance.flags,
+                            NULL);
         } else {
                 STACK_WIND (frame, dht_file_setattr_cbk, subvol,
                             subvol->fops->fsetattr, local->fd,
-                            &local->rebalance.stbuf, local->rebalance.flags);
+                            &local->rebalance.stbuf, local->rebalance.flags,
+                            NULL);
         }
 
         return 0;
@@ -436,7 +833,7 @@ dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
 int
 dht_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int op_ret, int op_errno, struct iatt *statpre,
-                 struct iatt *statpost)
+                 struct iatt *statpost, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -467,7 +864,7 @@ unlock:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
                 DHT_STACK_UNWIND (setattr, frame, local->op_ret, local->op_errno,
-                                  &local->prebuf, &local->stbuf);
+                                  &local->prebuf, &local->stbuf, xdata);
 
         return 0;
 }
@@ -475,7 +872,7 @@ unlock:
 
 int
 dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid)
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         dht_layout_t *layout = NULL;
@@ -521,7 +918,7 @@ dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
                 STACK_WIND (frame, dht_file_setattr_cbk, subvol,
                             subvol->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, xdata);
 
                 return 0;
         }
@@ -532,14 +929,14 @@ dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 STACK_WIND (frame, dht_setattr_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -547,7 +944,7 @@ err:
 
 int
 dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
-              int32_t valid)
+              int32_t valid, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         dht_layout_t *layout = NULL;
@@ -592,7 +989,7 @@ dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
 
                 STACK_WIND (frame, dht_file_setattr_cbk, subvol,
                             subvol->fops->fsetattr,
-                            fd, stbuf, valid);
+                            fd, stbuf, valid, xdata);
 
                 return 0;
         }
@@ -603,14 +1000,14 @@ dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
                 STACK_WIND (frame, dht_setattr_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->fsetattr,
-                            fd, stbuf, valid);
+                            fd, stbuf, valid, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index 3c8f63f5b..38e9970a7 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -68,9 +59,7 @@ dht_layout_t *
 dht_layout_get (xlator_t *this, inode_t *inode)
 {
         dht_conf_t   *conf = NULL;
-        uint64_t      layout_int = 0;
         dht_layout_t *layout = NULL;
-        int           ret    = -1;
 
         conf = this->private;
         if (!conf)
@@ -78,9 +67,8 @@ dht_layout_get (xlator_t *this, inode_t *inode)
 
         LOCK (&conf->layout_lock);
         {
-                ret = inode_ctx_get (inode, this, &layout_int);
-                if (ret == 0) {
-                        layout = (dht_layout_t *) (unsigned long) layout_int;
+                dht_inode_ctx_layout_get (inode, this, &layout);
+                if (layout) {
                         layout->ref++;
                 }
         }
@@ -98,7 +86,6 @@ dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
         int           oldret = -1;
         int           ret = 0;
         dht_layout_t *old_layout;
-        uint64_t      old_layout_int;
 
         conf = this->private;
         if (!conf)
@@ -106,16 +93,13 @@ dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
 
         LOCK (&conf->layout_lock);
         {
-                oldret = inode_ctx_get (inode, this, &old_layout_int);
-
+                oldret = dht_inode_ctx_layout_get (inode, this, &old_layout);
                 layout->ref++;
-                ret = inode_ctx_put (inode, this, (uint64_t) (unsigned long)
-                                     layout);
+                dht_inode_ctx_layout_set (inode, this, layout);
         }
         UNLOCK (&conf->layout_lock);
 
-        if (oldret == 0) {
-                old_layout = (dht_layout_t *) (unsigned long) old_layout_int;
+        if (!oldret) {
                 dht_layout_unref (this, old_layout);
         }
 
@@ -130,7 +114,7 @@ dht_layout_unref (xlator_t *this, dht_layout_t *layout)
         dht_conf_t  *conf = NULL;
         int          ref = 0;
 
-        if (layout->preset || !this->private)
+        if (!layout || layout->preset || !this->private)
                 return;
 
         conf = this->private;
@@ -174,9 +158,9 @@ dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
         int        ret = 0;
 
 
-        ret = dht_hash_compute (layout->type, name, &hash);
+        ret = dht_hash_compute (this, layout->type, name, &hash);
         if (ret != 0) {
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_WARNING,
                         "hash computation failed for type=%d name=%s",
                         layout->type, name);
                 goto out;
@@ -191,7 +175,7 @@ dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
         }
 
         if (!subvol) {
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_WARNING,
                         "no subvolume for hash (value) = %u", hash);
         }
 
@@ -280,6 +264,9 @@ dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
 
         if (disk_layout_p)
                 *disk_layout_p = disk_layout;
+        else
+                GF_FREE (disk_layout);
+
         ret = 0;
 
 out:
@@ -309,14 +296,18 @@ dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
 
         cnt  = ntoh32 (disk_layout[0]);
         if (cnt != 1) {
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_ERROR,
                         "disk layout has invalid count %d", cnt);
                 return -1;
         }
 
-	switch (disk_layout[1]) {
+        type = ntoh32 (disk_layout[1]);
+	switch (type) {
+        case DHT_HASH_TYPE_DM_USER:
+                gf_log (this->name, GF_LOG_DEBUG, "found user-set layout");
+                layout->type = type;
+                /* Fall through. */
 	case DHT_HASH_TYPE_DM:
-		type = ntoh32 (disk_layout[1]);
 		break;
         default:
 		gf_log (this->name, GF_LOG_CRITICAL,
@@ -344,11 +335,12 @@ int
 dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
                   int op_ret, int op_errno, dict_t *xattr)
 {
-        int      i     = 0;
-        int      ret   = -1;
-        int      err   = -1;
-        void    *disk_layout_raw = NULL;
-        int      disk_layout_len = 0;
+        int         i     = 0;
+        int         ret   = -1;
+        int         err   = -1;
+        void       *disk_layout_raw = NULL;
+        int         disk_layout_len = 0;
+        dht_conf_t *conf = this->private;
 
         if (op_ret != 0) {
                 err = op_errno;
@@ -369,12 +361,12 @@ dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
 
         if (xattr) {
                 /* during lookup and not mkdir */
-                ret = dict_get_ptr_and_len (xattr, "trusted.glusterfs.dht",
+                ret = dict_get_ptr_and_len (xattr, conf->xattr_name,
 					    &disk_layout_raw, &disk_layout_len);
         }
 
         if (ret != 0) {
-                layout->list[i].err = -1;
+                layout->list[i].err = 0;
                 gf_log (this->name, GF_LOG_TRACE,
                         "missing disk layout on %s. err = %d",
                         subvol->name, err);
@@ -385,7 +377,7 @@ dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
         ret = dht_disk_layout_merge (this, layout, i, disk_layout_raw,
 				     disk_layout_len);
         if (ret != 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_WARNING,
                         "layout merge from subvolume %s failed",
                         subvol->name);
                 goto out;
@@ -421,6 +413,22 @@ dht_layout_entry_swap (dht_layout_t *layout, int i, int j)
         layout->list[j].err    = err_swap;
 }
 
+void
+dht_layout_range_swap (dht_layout_t *layout, int i, int j)
+{
+        uint32_t  start_swap = 0;
+        uint32_t  stop_swap = 0;
+
+        start_swap  = layout->list[i].start;
+        stop_swap   = layout->list[i].stop;
+
+        layout->list[i].start  = layout->list[j].start;
+        layout->list[i].stop   = layout->list[j].stop;
+
+        layout->list[j].start  = start_swap;
+        layout->list[j].stop   = stop_swap;
+}
+
 int64_t
 dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
 {
@@ -428,17 +436,37 @@ dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
                         layout->list[j].xlator->name));
 }
 
+
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator)
+{
+        int i = 0;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (!strcmp (layout->list[i].xlator->name, xlator->name))
+                        return _gf_true;
+        }
+        return _gf_false;
+}
+
 int64_t
 dht_layout_entry_cmp (dht_layout_t *layout, int i, int j)
 {
         int64_t diff = 0;
 
+        /* swap zero'ed out layouts to front, if needed */
+        if (!layout->list[j].start && !layout->list[j].stop) {
+                diff = (int64_t) layout->list[i].stop
+                       - (int64_t) layout->list[j].stop;
+                       goto out;
+        }
         if (layout->list[i].err || layout->list[j].err)
                 diff = layout->list[i].err - layout->list[j].err;
         else
                 diff = (int64_t) layout->list[i].start
                         - (int64_t) layout->list[j].start;
 
+out:
         return diff;
 }
 
@@ -487,7 +515,8 @@ dht_layout_sort_volname (dht_layout_t *layout)
 int
 dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
                       uint32_t *holes_p, uint32_t *overlaps_p,
-                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p)
+                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                      uint32_t *no_space_p)
 {
         uint32_t    overlaps = 0;
         uint32_t    missing  = 0;
@@ -500,6 +529,7 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
         uint32_t    prev_stop = 0;
         uint32_t    last_stop = 0;
         char        is_virgin = 1;
+        uint32_t    no_space  = 0;
 
         /* TODO: explain what is happening */
 
@@ -507,23 +537,30 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
         prev_stop = last_stop;
 
         for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err) {
-                        switch (layout->list[i].err) {
-                        case -1:
-                        case ENOENT:
-                                missing++;
-                                break;
-                        case ENOTCONN:
-                                down++;
-                                break;
-                        case ENOSPC:
-                                down++;
-                                break;
-                        default:
-                                misc++;
+                switch (layout->list[i].err) {
+                case -1:
+                case ENOENT:
+                        missing++;
+                        continue;
+                case ENOTCONN:
+                        down++;
+                        continue;
+                case ENOSPC:
+                        no_space++;
+                        continue;
+                case 0:
+                        /* if err == 0 and start == stop, then it is a non misc++;
+                         * participating subvolume(spread-cnt). Then, do not
+                         * check for anomalies. If start != stop, then treat it
+                         * as misc err */
+                        if (layout->list[i].start == layout->list[i].stop) {
+                                continue;
                         }
+                        break;
+                default:
+                        misc++;
                         continue;
-                }
+                 }
 
                 is_virgin = 0;
 
@@ -556,6 +593,9 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
         if (misc_p)
                 *misc_p = misc;
 
+        if (no_space_p)
+                *no_space_p = no_space;
+
         return ret;
 }
 
@@ -571,7 +611,6 @@ dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
         uint32_t     down = 0;
         uint32_t     misc = 0;
 
-
         ret = dht_layout_sort (layout);
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
@@ -581,7 +620,7 @@ dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
 
         ret = dht_layout_anomalies (this, loc, layout,
                                     &holes, &overlaps,
-                                    &missing, &down, &misc);
+                                    &missing, &down, &misc, NULL);
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "error while finding anomalies in %s -- not good news",
@@ -599,43 +638,56 @@ dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
                                 "found anomalies in %s. holes=%d overlaps=%d",
                                 loc->path, holes, overlaps);
                 }
-                ret = 1;
+                ret = -1;
         }
 
         for (i = 0; i < layout->cnt; i++) {
-                /* TODO During DHT selfheal rewrite (almost) find a better place to
-                 * detect this - probably in dht_layout_anomalies()
+                /* TODO During DHT selfheal rewrite (almost) find a better place
+                 * to detect this - probably in dht_layout_anomalies()
                  */
                 if (layout->list[i].err > 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "path=%s err=%s on subvol=%s",
-                                loc->path, strerror (layout->list[i].err),
-                                (layout->list[i].xlator ?
-                                 layout->list[i].xlator->name : "<>"));
-                        if (layout->list[i].err == ENOENT)
-                                ret = 1;
+                        gf_log_callingfn (this->name, GF_LOG_DEBUG,
+                                          "path=%s err=%s on subvol=%s",
+                                          loc->path,
+                                          strerror (layout->list[i].err),
+                                          (layout->list[i].xlator ?
+                                           layout->list[i].xlator->name
+                                           : "<>"));
+                        if ((layout->list[i].err == ENOENT) && (ret >= 0)) {
+                                ret++;
+                        }
                 }
         }
 
+
 out:
         return ret;
 }
 
+int
+dht_dir_has_layout (dict_t *xattr, char *name)
+{
+
+        void     *disk_layout_raw = NULL;
+
+        return dict_get_ptr (xattr, name, &disk_layout_raw);
+}
 
 int
 dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
                          loc_t *loc, dict_t *xattr)
 {
-        int       idx = 0;
-        int       pos = -1;
-        int       ret = 0;
-        int       err = 0;
-        int       dict_ret = 0;
-        int32_t   disk_layout[4];
-        void     *disk_layout_raw = NULL;
-        int32_t   count = -1;
-        uint32_t  start_off = -1;
-        uint32_t  stop_off = -1;
+        int         idx = 0;
+        int         pos = -1;
+        int         ret = 0;
+        int         err = 0;
+        int         dict_ret = 0;
+        int32_t     disk_layout[4];
+        void       *disk_layout_raw = NULL;
+        int32_t     count = -1;
+        uint32_t    start_off = -1;
+        uint32_t    stop_off = -1;
+        dht_conf_t *conf = this->private;
 
 
         for (idx = 0; idx < layout->cnt; idx++) {
@@ -665,7 +717,7 @@ dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
                 goto out;
         }
 
-        dict_ret = dict_get_ptr (xattr, "trusted.glusterfs.dht",
+        dict_ret = dict_get_ptr (xattr, conf->xattr_name,
                                  &disk_layout_raw);
 
         if (dict_ret < 0) {
@@ -681,7 +733,7 @@ dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
 
         count  = ntoh32 (disk_layout[0]);
         if (count != 1) {
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_ERROR,
                         "%s - disk layout has invalid count %d",
                         loc->path, count);
                 ret = -1;
@@ -730,7 +782,7 @@ dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode)
 
         LOCK (&conf->layout_lock);
         {
-                inode_ctx_put (inode, this, (uint64_t)(long)layout);
+                dht_inode_ctx_layout_set (inode, this, layout);
         }
         UNLOCK (&conf->layout_lock);
 
diff --git a/xlators/cluster/dht/src/dht-linkfile.c b/xlators/cluster/dht/src/dht-linkfile.c
index 2186b064a..dbc9d0b3c 100644
--- a/xlators/cluster/dht/src/dht-linkfile.c
+++ b/xlators/cluster/dht/src/dht-linkfile.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -28,37 +19,106 @@
 #include "compat.h"
 #include "dht-common.h"
 
+int
+dht_linkfile_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno,
+                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                         struct iatt *postparent)
+{
+        char          is_linkfile   = 0;
+        dht_conf_t   *conf          = NULL;
+        dht_local_t  *local         = NULL;
+        call_frame_t *prev          = NULL;
 
+        local = frame->local;
+        prev = cookie;
+        conf = this->private;
+
+        if (op_ret)
+                goto out;
+
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
+        if (!is_linkfile)
+                gf_log (this->name, GF_LOG_WARNING, "got non-linkfile %s:%s",
+                        prev->this->name, local->loc.path);
+out:
+        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
+                                      inode, stbuf, postparent, postparent,
+                                      xattr);
+        return 0;
+}
 
+#define is_equal(a, b) (a == b)
 int
 dht_linkfile_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int op_ret, int op_errno, inode_t *inode,
                          struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
+        xlator_t     *subvol = NULL;
+        call_frame_t *prev = NULL;
+        dict_t       *xattrs = NULL;
+        dht_conf_t   *conf = NULL;
+        int           ret = -1;
 
         local = frame->local;
 
+        if (!op_ret)
+                local->linked = _gf_true;
+
+        FRAME_SU_UNDO (frame, dht_local_t);
+
+        if (op_ret && (op_errno == EEXIST)) {
+                conf = this->private;
+                prev = cookie;
+                subvol = prev->this;
+                if (!subvol)
+                        goto out;
+                xattrs = dict_new ();
+                if (!xattrs)
+                        goto out;
+                ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set linkto key");
+                        goto out;
+                }
+
+                STACK_WIND (frame, dht_linkfile_lookup_cbk, subvol,
+                            subvol->fops->lookup, &local->loc, xattrs);
+                if (xattrs)
+                        dict_unref (xattrs);
+                return 0;
+        }
+out:
         local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-                                      inode, stbuf, preparent, postparent);
+                                      inode, stbuf, preparent, postparent,
+                                      xdata);
+        if (xattrs)
+                dict_unref (xattrs);
         return 0;
 }
 
 
 int
 dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                     xlator_t *this,
                      xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
 {
         dht_local_t *local = NULL;
         dict_t      *dict = NULL;
         int          need_unref = 0;
         int          ret = 0;
+        dht_conf_t  *conf = this->private;
 
         local = frame->local;
         local->linkfile.linkfile_cbk = linkfile_cbk;
         local->linkfile.srcvol = tovol;
 
+        local->linked = _gf_false;
+
         dict = local->params;
         if (!dict) {
                 dict = dict_new ();
@@ -74,8 +134,12 @@ dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
                                 "%s: gfid set failed", loc->path);
         }
 
-        ret = dict_set_str (dict, "trusted.glusterfs.dht.linkto",
-                                   tovol->name);
+        ret = dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+        if (ret)
+                gf_log ("dht-linkfile", GF_LOG_INFO,
+                        "%s: internal-fop set failed", loc->path);
+
+        ret = dict_set_str (dict, conf->link_xattr_name, tovol->name);
 
         if (ret < 0) {
                 gf_log (frame->this->name, GF_LOG_INFO,
@@ -84,9 +148,13 @@ dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
                 goto out;
         }
 
+        local->link_subvol = fromvol;
+        /* Always create as root:root. dht_linkfile_attr_heal fixes the
+         * ownsership */
+        FRAME_SU_DO (frame, dht_local_t);
         STACK_WIND (frame, dht_linkfile_create_cbk,
                     fromvol, fromvol->fops->mknod, loc,
-                    S_IFREG | DHT_LINKFILE_MODE, 0, dict);
+                    S_IFREG | DHT_LINKFILE_MODE, 0, 0, dict);
 
         if (need_unref && dict)
                 dict_unref (dict);
@@ -94,7 +162,7 @@ dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
         return 0;
 out:
         local->linkfile.linkfile_cbk (frame, NULL, frame->this, -1, ENOMEM,
-                                      loc->inode, NULL, NULL, NULL);
+                                      loc->inode, NULL, NULL, NULL, NULL);
 
         if (need_unref && dict)
                 dict_unref (dict);
@@ -106,7 +174,8 @@ out:
 int
 dht_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno,
-                         struct iatt *preparent, struct iatt *postparent)
+                         struct iatt *preparent, struct iatt *postparent,
+                         dict_t *xdata)
 {
         dht_local_t   *local = NULL;
         call_frame_t  *prev = NULL;
@@ -150,7 +219,7 @@ dht_linkfile_unlink (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (unlink_frame, dht_linkfile_unlink_cbk,
                     subvol, subvol->fops->unlink,
-                    &unlink_local->loc);
+                    &unlink_local->loc, 0, NULL);
 
         return 0;
 err:
@@ -175,7 +244,7 @@ dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
         if (!xattr)
                 goto out;
 
-        ret = dict_get_ptr (xattr, "trusted.glusterfs.dht.linkto", &volname);
+        ret = dict_get_ptr (xattr, conf->link_xattr_name, &volname);
 
         if ((-1 == ret) || !volname)
                 goto out;
@@ -190,3 +259,70 @@ dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
 out:
         return subvol;
 }
+
+int
+dht_linkfile_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, struct iatt *statpre,
+                          struct iatt *statpost, dict_t *xdata)
+{
+        dht_local_t *local = NULL;
+        loc_t *loc = NULL;
+
+        local = frame->local;
+        loc = &local->loc;
+
+        if (op_ret)
+                gf_log (this->name, GF_LOG_ERROR, "setattr of uid/gid on %s"
+                        " :<gfid:%s> failed (%s)",
+                        (loc->path? loc->path: "NULL"),
+                        uuid_utoa(local->gfid), strerror(op_errno));
+
+        DHT_STACK_DESTROY (frame);
+
+       return 0;
+}
+
+int
+dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this)
+{
+        int     ret = -1;
+        call_frame_t *copy = NULL;
+        dht_local_t  *local = NULL;
+        dht_local_t  *copy_local = NULL;
+        xlator_t     *subvol = NULL;
+        struct iatt   stbuf = {0,};
+
+        local = frame->local;
+
+        GF_VALIDATE_OR_GOTO ("dht", local, out);
+        GF_VALIDATE_OR_GOTO ("dht", local->link_subvol, out);
+
+        if (local->stbuf.ia_type == IA_INVAL)
+                return 0;
+
+        uuid_copy (local->loc.gfid, local->stbuf.ia_gfid);
+
+        copy = copy_frame (frame);
+
+        if (!copy)
+                goto out;
+
+       copy_local = dht_local_init (copy, &local->loc, NULL, 0);
+
+       if (!copy_local)
+               goto out;
+
+        stbuf = local->stbuf;
+        subvol = local->link_subvol;
+
+        copy->local = copy_local;
+
+        FRAME_SU_DO (copy, dht_local_t);
+
+        STACK_WIND (copy, dht_linkfile_setattr_cbk, subvol,
+                    subvol->fops->setattr, &copy_local->loc,
+                    &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-mem-types.h b/xlators/cluster/dht/src/dht-mem-types.h
index 21fb5a7ca..e893eb48f 100644
--- a/xlators/cluster/dht/src/dht-mem-types.h
+++ b/xlators/cluster/dht/src/dht-mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -28,7 +19,6 @@ enum gf_dht_mem_types_ {
         gf_dht_mt_dht_conf_t,
         gf_dht_mt_char,
         gf_dht_mt_int32_t,
-        gf_dht_mt_dht_local_t,
         gf_dht_mt_xlator_t,
         gf_dht_mt_dht_layout_t,
         gf_switch_mt_dht_conf_t,
@@ -37,6 +27,9 @@ enum gf_dht_mem_types_ {
         gf_switch_mt_switch_struct,
         gf_dht_mt_subvol_time,
         gf_dht_mt_loc_t,
+        gf_defrag_info_mt,
+        gf_dht_mt_inode_ctx_t,
+        gf_dht_mt_ctx_stat_time_t,
         gf_dht_mt_end
 };
 #endif
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index dfd6f3b6e..bcb19f23e 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -24,11 +15,12 @@
 #endif
 
 #include "dht-common.h"
+#include "xlator.h"
+#include <fnmatch.h>
 
 #define GF_DISK_SECTOR_SIZE             512
 #define DHT_REBALANCE_PID               4242 /* Change it if required */
 #define DHT_REBALANCE_BLKSIZE           (128 * 1024)
-#define DHT_MIGRATE_EVEN_IF_LINK_EXISTS 1
 
 static int
 dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
@@ -99,8 +91,118 @@ out:
 
 }
 
+int32_t
+gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
+                           struct iatt *stbuf)
+{
+        int32_t                 ret             = -1;
+        xlator_t               *cached_subvol   = NULL;
+        xlator_t               *hashed_subvol   = NULL;
+        xlator_t               *linkto_subvol   = NULL;
+        data_t                 *data            = NULL;
+        struct iatt             iatt            = {0,};
+        int32_t                 op_errno        = 0;
+        dht_conf_t             *conf            = NULL;
+
+        GF_VALIDATE_OR_GOTO ("defrag", loc, out);
+        GF_VALIDATE_OR_GOTO ("defrag", loc->name, out);
+        GF_VALIDATE_OR_GOTO ("defrag", stbuf, out);
+        GF_VALIDATE_OR_GOTO ("defrag", this, out);
+        GF_VALIDATE_OR_GOTO ("defrag", xattrs, out);
+        GF_VALIDATE_OR_GOTO ("defrag", this->private, out);
+
+        conf = this->private;
+
+        if (uuid_is_null (loc->pargfid)) {
+                gf_log ("", GF_LOG_ERROR, "loc->pargfid is NULL for "
+                        "%s", loc->path);
+                goto out;
+        }
+
+        if (uuid_is_null (loc->gfid)) {
+                gf_log ("", GF_LOG_ERROR, "loc->gfid is NULL for "
+                        "%s", loc->path);
+                goto out;
+        }
+
+        cached_subvol = dht_subvol_get_cached (this, loc->inode);
+        if (!cached_subvol) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get cached subvol"
+                        " for %s on %s", loc->name, this->name);
+                goto out;
+        }
+
+        hashed_subvol = dht_subvol_get_hashed (this, loc);
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get hashed subvol"
+                        " for %s on %s", loc->name, this->name);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Attempting to migrate hardlink %s "
+                "with gfid %s from %s -> %s", loc->name, uuid_utoa (loc->gfid),
+                cached_subvol->name, hashed_subvol->name);
+        data = dict_get (xattrs, conf->link_xattr_name);
+        /* set linkto on cached -> hashed if not present, else link it */
+        if (!data) {
+                ret = dict_set_str (xattrs, conf->link_xattr_name,
+                                    hashed_subvol->name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "linkto xattr in dict for %s", loc->name);
+                        goto out;
+                }
+
+                ret = syncop_setxattr (cached_subvol, loc, xattrs, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Linkto setxattr "
+                                "failed %s -> %s (%s)", cached_subvol->name,
+                                loc->name, strerror (errno));
+                        goto out;
+                }
+                goto out;
+        } else {
+                linkto_subvol = dht_linkfile_subvol (this, NULL, NULL, xattrs);
+                if (!linkto_subvol) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "linkto subvol for %s", loc->name);
+                } else {
+                        hashed_subvol = linkto_subvol;
+                }
+
+                ret = syncop_link (hashed_subvol, loc, loc);
+                if  (ret) {
+                        op_errno = errno;
+                        gf_log (this->name, GF_LOG_ERROR, "link of %s -> %s"
+                                " failed on  subvol %s (%s)", loc->name,
+                                uuid_utoa(loc->gfid),
+                                hashed_subvol->name, strerror (op_errno));
+                        if (op_errno != EEXIST)
+                                goto out;
+                }
+        }
+        ret = syncop_lookup (hashed_subvol, loc, NULL, &iatt, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed lookup %s on %s (%s)"
+                        , loc->name, hashed_subvol->name, strerror (errno));
+                goto out;
+        }
+
+        if (iatt.ia_nlink == stbuf->ia_nlink) {
+                ret = dht_migrate_file (this, loc, cached_subvol, hashed_subvol,
+                                        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS);
+                if (ret)
+                        goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+
 static inline int
-__is_file_migratable (xlator_t *this, loc_t *loc, struct iatt *stbuf)
+__is_file_migratable (xlator_t *this, loc_t *loc,
+                      struct iatt *stbuf, dict_t *xattrs, int flags)
 {
         int ret = -1;
 
@@ -111,11 +213,25 @@ __is_file_migratable (xlator_t *this, loc_t *loc, struct iatt *stbuf)
                 goto out;
         }
 
+        if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
+                ret = 0;
+                goto out;
+        }
         if (stbuf->ia_nlink > 1) {
-                /* TODO : support migrating hardlinks */
-                gf_log (this->name, GF_LOG_WARNING, "%s: file has hardlinks",
-                        loc->path);
-                ret = -ENOTSUP;
+                /* support for decomission */
+                if (flags == GF_DHT_MIGRATE_HARDLINK) {
+                        ret = gf_defrag_handle_hardlink (this, loc,
+                                                         xattrs, stbuf);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "%s: failed to migrate file with link",
+                                        loc->path);
+                        }
+                } else {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: file has hardlinks", loc->path);
+                }
+                ret = ENOTSUP;
                 goto out;
         }
 
@@ -127,14 +243,16 @@ out:
 
 static inline int
 __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf,
-                                 dict_t *dict, fd_t **dst_fd)
+                                 dict_t *dict, fd_t **dst_fd, dict_t *xattr)
 {
-        xlator_t *this = NULL;
-        int       ret  = -1;
-        fd_t     *fd   = NULL;
-        struct iatt new_stbuf = {0,};
+        xlator_t    *this = NULL;
+        int          ret  = -1;
+        fd_t        *fd   = NULL;
+        struct iatt  new_stbuf = {0,};
+        dht_conf_t  *conf = NULL;
 
         this = THIS;
+        conf = this->private;
 
         ret = dict_set_static_bin (dict, "gfid-req", stbuf->ia_gfid, 16);
         if (ret) {
@@ -143,7 +261,7 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc
                 goto out;
         }
 
-        ret = dict_set_str (dict, DHT_LINKFILE_KEY, from->name);
+        ret = dict_set_str (dict, conf->link_xattr_name, from->name);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set gfid in dict for create", loc->path);
@@ -181,7 +299,7 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc
         /* Create the destination with LINKFILE mode, and linkto xattr,
            if the linkfile already exists, it will just open the file */
         ret = syncop_create (to, loc, O_RDWR, DHT_LINKFILE_MODE, fd,
-                             dict);
+                             dict, &new_stbuf);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to create %s on %s (%s)",
@@ -189,6 +307,26 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc
                 goto out;
         }
 
+        ret = syncop_fsetxattr (to, fd, xattr, 0);
+        if (ret == -1)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set xattr on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+
+        ret = syncop_ftruncate (to, fd, stbuf->ia_size);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "ftruncate failed for %s on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+
+        ret = syncop_fsetattr (to, fd, stbuf,
+                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                                NULL, NULL);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "chown failed for %s on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+
         if (dst_fd)
                 *dst_fd = fd;
 
@@ -201,13 +339,16 @@ out:
 
 static inline int
 __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
-                        struct iatt *stbuf)
+                        struct iatt *stbuf, int flag)
 {
         struct statvfs  src_statfs = {0,};
         struct statvfs  dst_statfs = {0,};
         int             ret        = -1;
         xlator_t       *this       = NULL;
 
+        uint64_t        src_statfs_blocks = 1;
+        uint64_t        dst_statfs_blocks = 1;
+
         this = THIS;
 
         ret = syncop_statfs (from, loc, &src_statfs);
@@ -225,18 +366,47 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
                         loc->path, to->name, strerror (errno));
                 goto out;
         }
-        if (((dst_statfs.f_bavail *
-              dst_statfs.f_bsize) / GF_DISK_SECTOR_SIZE) >
-            (((src_statfs.f_bavail * src_statfs.f_bsize) /
-              GF_DISK_SECTOR_SIZE) - stbuf->ia_blocks)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "data movement attempted from node (%s) with"
-                        " higher disk space to a node (%s) with "
-                        "lesser disk space (%s)", from->name,
-                        to->name, loc->path);
 
-                /* this is not a 'failure', but we don't want to
-                   consider this as 'success' too :-/ */
+        /* if force option is given, do not check for space @ dst.
+         * Check only if space is avail for the file */
+        if (flag != GF_DHT_MIGRATE_DATA)
+                goto check_avail_space;
+
+        /* Check:
+           During rebalance `migrate-data` - Destination subvol experiences
+           a `reduction` in 'blocks' of free space, at the same time source
+           subvol gains certain 'blocks' of free space. A valid check is
+           necessary here to avoid errorneous move to destination where
+           the space could be scantily available.
+         */
+        if (stbuf) {
+                dst_statfs_blocks = ((dst_statfs.f_bavail *
+                                      dst_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                src_statfs_blocks = ((src_statfs.f_bavail *
+                                      src_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                if ((dst_statfs_blocks - stbuf->ia_blocks) <
+                    (src_statfs_blocks + stbuf->ia_blocks)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "data movement attempted from node (%s) with"
+                                " higher disk space to a node (%s) with "
+                                "lesser disk space (%s)", from->name,
+                                to->name, loc->path);
+
+                        /* this is not a 'failure', but we don't want to
+                           consider this as 'success' too :-/ */
+                        ret = 1;
+                        goto out;
+                }
+        }
+check_avail_space:
+        if (((dst_statfs.f_bavail * dst_statfs.f_bsize) /
+              GF_DISK_SECTOR_SIZE) < stbuf->ia_blocks) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "data movement attempted from node (%s) with "
+                        "to node (%s) which does not have required free space"
+                        " for %s", from->name, to->name, loc->path);
                 ret = 1;
                 goto out;
         }
@@ -280,8 +450,7 @@ __dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst
                 offset += ret;
                 total += ret;
 
-                if (vector)
-                        GF_FREE (vector);
+                GF_FREE (vector);
                 if (iobref)
                         iobref_unref (iobref);
                 iobref = NULL;
@@ -289,8 +458,7 @@ __dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst
         }
         if (iobref)
                 iobref_unref (iobref);
-        if (vector)
-                GF_FREE (vector);
+        GF_FREE (vector);
 
         if (ret >= 0)
                 ret = 0;
@@ -308,8 +476,10 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         dict_t      *dict = NULL;
         xlator_t    *this = NULL;
         struct iatt  iatt = {0,};
+        dht_conf_t  *conf = NULL;
 
         this = THIS;
+        conf = this->private;
 
         fd = fd_create (loc->inode, DHT_REBALANCE_PID);
         if (!fd) {
@@ -332,7 +502,7 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         if (!dict)
                 goto out;
 
-        ret = dict_set_str (dict, DHT_LINKFILE_KEY, to->name);
+        ret = dict_set_str (dict, conf->link_xattr_name, to->name);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to set xattr in dict for %s (linkto:%s)",
@@ -385,12 +555,13 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
         dict_t      *dict     = NULL;
         char        *link     = NULL;
         struct iatt  stbuf    = {0,};
+        dht_conf_t  *conf     = this->private;
 
         dict = dict_new ();
         if (!dict)
                 goto out;
 
-        ret = dict_set_int32 (dict, DHT_LINKFILE_KEY, 256);
+        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set 'linkto' key in dict", loc->path);
@@ -406,12 +577,13 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
         }
 
         /* we no more require this key */
-        dict_del (dict, DHT_LINKFILE_KEY);
+        dict_del (dict, conf->link_xattr_name);
 
         /* file exists in target node, only if it is 'linkfile' its valid,
            otherwise, error out */
         if (!ret) {
-                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict)) {
+                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict,
+                                        conf->link_xattr_name)) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: file exists in destination", loc->path);
                         ret = -1;
@@ -447,7 +619,7 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
                         goto out;
                 }
 
-                ret = syncop_symlink (to, loc, link, dict);
+                ret = syncop_symlink (to, loc, link, dict, 0);
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: creating symlink failed (%s)",
@@ -461,7 +633,7 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
         ret = syncop_mknod (to, loc, st_mode_from_ia (buf->ia_prot,
                                                       buf->ia_type),
                             makedev (ia_major (buf->ia_rdev),
-                                     ia_minor (buf->ia_rdev)), dict);
+                                     ia_minor (buf->ia_rdev)), dict, 0);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING, "%s: mknod failed (%s)",
                         loc->path, strerror (errno));
@@ -469,6 +641,15 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
         }
 
 done:
+        ret = syncop_setattr (to, loc, buf,
+                              (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
+                               GF_SET_ATTR_MODE), NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+        }
+
         ret = syncop_unlink (from, loc);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING, "%s: unlink failed (%s)",
@@ -504,7 +685,9 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         fd_t           *dst_fd         = NULL;
         dict_t         *dict           = NULL;
         dict_t         *xattr          = NULL;
+        dict_t         *xattr_rsp      = NULL;
         int             file_has_holes = 0;
+        dht_conf_t     *conf           = this->private;
 
         gf_log (this->name, GF_LOG_INFO, "%s: attempting to move from %s to %s",
                 loc->path, from->name, to->name);
@@ -513,19 +696,29 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         if (!dict)
                 goto out;
 
+        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set 'linkto' key in dict", loc->path);
+                goto out;
+        }
+
         /* Phase 1 - Data migration is in progress from now on */
-        ret = syncop_lookup (from, loc, NULL, &stbuf, NULL, NULL);
+        ret = syncop_lookup (from, loc, dict, &stbuf, &xattr_rsp, NULL);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "%s: lookup failed on %s (%s)",
                         loc->path, from->name, strerror (errno));
                 goto out;
         }
 
+        /* we no more require this key */
+        dict_del (dict, conf->link_xattr_name);
+
         /* preserve source mode, so set the same to the destination */
         src_ia_prot = stbuf.ia_prot;
 
         /* Check if file can be migrated */
-        ret = __is_file_migratable (this, loc, &stbuf);
+        ret = __is_file_migratable (this, loc, &stbuf, xattr_rsp, flag);
         if (ret)
                 goto out;
 
@@ -536,18 +729,22 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 goto out;
         }
 
+        /* TODO: move all xattr related operations to fd based operations */
+        ret = syncop_listxattr (from, loc, &xattr);
+        if (ret == -1)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get xattr from %s (%s)",
+                        loc->path, from->name, strerror (errno));
+
         /* create the destination, with required modes/xattr */
         ret = __dht_rebalance_create_dst_file (to, from, loc, &stbuf,
-                                               dict, &dst_fd);
+                                               dict, &dst_fd, xattr);
         if (ret)
                 goto out;
 
-        /* Should happen on all files when 'force' option is not given */
-        if (flag != DHT_MIGRATE_EVEN_IF_LINK_EXISTS) {
-                ret = __dht_check_free_space (to, from, loc, &stbuf);
-                if (ret) {
-                        goto out;
-                }
+        ret = __dht_check_free_space (to, from, loc, &stbuf, flag);
+        if (ret) {
+                goto out;
         }
 
         /* Open the source, and also update mode/xattr */
@@ -558,6 +755,7 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 goto out;
         }
 
+
         ret = syncop_fstat (from, src_fd, &stbuf);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "failed to lookup %s on %s (%s)",
@@ -587,22 +785,9 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 goto out;
         }
 
-        /* TODO: move all xattr related operations to fd based operations */
-        ret = syncop_listxattr (from, loc, &xattr);
-        if (ret == -1)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to get xattr from %s (%s)",
-                        loc->path, from->name, strerror (errno));
-
-        ret = syncop_setxattr (to, loc, xattr, 0);
-        if (ret == -1)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set xattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
-
         /* TODO: Sync the locks */
 
-        ret = syncop_fsync (to, dst_fd);
+        ret = syncop_fsync (to, dst_fd, 0);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to fsync on %s (%s)",
@@ -639,6 +824,7 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to perform setattr on %s (%s)",
                         loc->path, to->name, strerror (errno));
+                goto out;
         }
 
         /* Because 'futimes' is not portable */
@@ -659,6 +845,24 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 gf_log (this->name, GF_LOG_WARNING,             \
                         "%s: failed to perform setattr on %s (%s)",
                         loc->path, from->name, strerror (errno));
+                goto out;
+        }
+
+       /* Free up the data blocks on the source node, as the whole
+           file is migrated */
+        ret = syncop_ftruncate (from, src_fd, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform truncate on %s (%s)",
+                        loc->path, from->name, strerror (errno));
+        }
+
+        /* remove the 'linkto' xattr from the destination */
+        ret = syncop_fremovexattr (to, dst_fd, conf->link_xattr_name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform removexattr on %s (%s)",
+                        loc->path, to->name, strerror (errno));
         }
 
         /* Do a stat and check the gfid before unlink */
@@ -667,38 +871,23 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to do a stat on %s (%s)",
                         loc->path, from->name, strerror (errno));
+                goto out;
         }
 
-        if (uuid_compare (empty_iatt.ia_gfid, loc->inode->gfid) == 0) {
+        if (uuid_compare (empty_iatt.ia_gfid, loc->gfid) == 0) {
                 /* take out the source from namespace */
                 ret = syncop_unlink (from, loc);
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: failed to perform unlink on %s (%s)",
                                 loc->path, from->name, strerror (errno));
+                        goto out;
                 }
         }
 
-        /* Free up the data blocks on the source node, as the whole
-           file is migrated */
-        ret = syncop_ftruncate (from, src_fd, 0);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform truncate on %s (%s)",
-                        loc->path, from->name, strerror (errno));
-        }
-
-        /* remove the 'linkto' xattr from the destination */
-        ret = syncop_fremovexattr (to, dst_fd, DHT_LINKFILE_KEY);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform removexattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
-        }
-
         ret = syncop_lookup (this, loc, NULL, NULL, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "%s: failed to lookup the file on subvolumes (%s)",
                         loc->path, strerror (errno));
         }
@@ -714,6 +903,8 @@ out:
 
         if (xattr)
                 dict_unref (xattr);
+        if (xattr_rsp)
+                dict_unref (xattr_rsp);
 
         if (dst_fd)
                 syncop_close (dst_fd);
@@ -788,7 +979,7 @@ rebalance_task_completion (int op_ret, call_frame_t *sync_frame, void *data)
                 op_errno = EPERM;
         }
 
-        DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno, NULL);
         return 0;
 }
 
@@ -796,12 +987,829 @@ int
 dht_start_rebalance_task (xlator_t *this, call_frame_t *frame)
 {
         int         ret     = -1;
-        dht_conf_t *conf    = NULL;
 
-        conf = this->private;
-
-        ret = synctask_new (conf->env, rebalance_task,
+        ret = synctask_new (this->ctx->env, rebalance_task,
                             rebalance_task_completion,
                             frame, frame);
         return ret;
 }
+
+int
+gf_listener_stop (xlator_t *this)
+{
+        glusterfs_ctx_t  *ctx = NULL;
+        cmd_args_t       *cmd_args = NULL;
+        int              ret = 0;
+
+        ctx = this->ctx;
+        GF_ASSERT (ctx);
+        cmd_args = &ctx->cmd_args;
+        if (cmd_args->sock_file) {
+                ret = unlink (cmd_args->sock_file);
+                if (ret && (ENOENT == errno)) {
+                        ret = 0;
+                }
+        }
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to unlink listener "
+                        "socket %s, error: %s", cmd_args->sock_file,
+                        strerror (errno));
+        }
+        return ret;
+}
+
+void
+dht_build_root_inode (xlator_t *this, inode_t **inode)
+{
+        inode_table_t    *itable        = NULL;
+        uuid_t            root_gfid     = {0, };
+
+        itable = inode_table_new (0, this);
+        if (!itable)
+                return;
+
+        root_gfid[15] = 1;
+        *inode = inode_find (itable, root_gfid);
+}
+
+void
+dht_build_root_loc (inode_t *inode, loc_t *loc)
+{
+        loc->path = "/";
+        loc->inode = inode;
+        loc->inode->ia_type = IA_IFDIR;
+        memset (loc->gfid, 0, 16);
+        loc->gfid[15] = 1;
+}
+
+
+/* return values: 1 -> error, bug ignore and continue
+                  0 -> proceed
+                 -1 -> error, handle it */
+int32_t
+gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
+{
+        /* if errno is not ENOSPC or ENOTCONN, we can still continue
+           with rebalance process */
+        if ((errno != ENOSPC) || (errno != ENOTCONN))
+                return 1;
+
+        if (errno == ENOTCONN) {
+                /* Most probably mount point went missing (mostly due
+                   to a brick down), say rebalance failure to user,
+                   let him restart it if everything is fine */
+                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+                return -1;
+        }
+
+        if (errno == ENOSPC) {
+                /* rebalance process itself failed, may be
+                   remote brick went down, or write failed due to
+                   disk full etc etc.. */
+                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+                return -1;
+        }
+
+        return 0;
+}
+
+static gf_boolean_t
+gf_defrag_pattern_match (gf_defrag_info_t *defrag, char *name, uint64_t size)
+{
+        gf_defrag_pattern_list_t *trav = NULL;
+        gf_boolean_t               match = _gf_false;
+        gf_boolean_t               ret = _gf_false;
+
+        GF_VALIDATE_OR_GOTO ("dht", defrag, out);
+
+        trav = defrag->defrag_pattern;
+        while (trav) {
+                if (!fnmatch (trav->path_pattern, name, FNM_NOESCAPE)) {
+                        match = _gf_true;
+                        break;
+                }
+                trav = trav->next;
+        }
+
+        if ((match == _gf_true) && (size >= trav->size))
+                ret = _gf_true;
+
+ out:
+        return ret;
+}
+
+/* We do a depth first traversal of directories. But before we move into
+ * subdirs, we complete the data migration of those directories whose layouts
+ * have been fixed
+ */
+
+int
+gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                        dict_t *migrate_data)
+{
+        int                      ret            = -1;
+        loc_t                    entry_loc      = {0,};
+        fd_t                    *fd             = NULL;
+        gf_dirent_t              entries;
+        gf_dirent_t             *tmp            = NULL;
+        gf_dirent_t             *entry          = NULL;
+        gf_boolean_t             free_entries   = _gf_false;
+        off_t                    offset         = 0;
+        dict_t                  *dict           = NULL;
+        struct iatt              iatt           = {0,};
+        int32_t                  op_errno       = 0;
+        char                    *uuid_str       = NULL;
+        uuid_t                   node_uuid      = {0,};
+        int                      readdir_operrno = 0;
+        struct timeval           dir_start      = {0,};
+        struct timeval           end            = {0,};
+        double                   elapsed        = {0,};
+        struct timeval           start          = {0,};
+        int32_t                  err            = 0;
+
+        gf_log (this->name, GF_LOG_INFO, "migrate data called on %s",
+                loc->path);
+        gettimeofday (&dir_start, NULL);
+
+        fd = fd_create (loc->inode, defrag->pid);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
+                goto out;
+        }
+
+        ret = syncop_opendir (this, loc, fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
+                        loc->path);
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&entries.list);
+
+        while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
+                                       &entries)) != 0) {
+
+                if (ret < 0) {
+
+                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s."
+                                " Aborting migrate-data",
+                                strerror(readdir_operrno));
+                        goto out;
+                }
+
+                /* Need to keep track of ENOENT errno, that means, there is no
+                   need to send more readdirp() */
+                readdir_operrno = errno;
+
+                if (list_empty (&entries.list))
+                        break;
+
+                free_entries = _gf_true;
+
+                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
+                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+                                ret = 1;
+                                goto out;
+                        }
+
+                        offset = entry->d_off;
+
+                        if (!strcmp (entry->d_name, ".") ||
+                            !strcmp (entry->d_name, ".."))
+                                continue;
+
+                        if (IA_ISDIR (entry->d_stat.ia_type))
+                                continue;
+
+                        defrag->num_files_lookedup++;
+                        if (defrag->stats == _gf_true) {
+                                gettimeofday (&start, NULL);
+                        }
+                        if (defrag->defrag_pattern &&
+                            (gf_defrag_pattern_match (defrag, entry->d_name,
+                                                      entry->d_stat.ia_size)
+                             == _gf_false)) {
+                                continue;
+                        }
+                        loc_wipe (&entry_loc);
+                        ret =dht_build_child_loc (this, &entry_loc, loc,
+                                                  entry->d_name);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
+                                        " build failed");
+                                goto out;
+                        }
+
+                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+
+                        if (uuid_is_null (loc->gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        uuid_copy (entry_loc.pargfid, loc->gfid);
+
+                        entry_loc.inode->ia_type = entry->d_stat.ia_type;
+
+                        ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
+                                             NULL, NULL);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s"
+                                        " lookup failed", entry_loc.path);
+                                continue;
+                        }
+
+                        ret = syncop_getxattr (this, &entry_loc, &dict,
+                                               GF_XATTR_NODE_UUID_KEY);
+                        if(ret < 0) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "get node-uuid for %s", entry_loc.path);
+                                continue;
+                        }
+
+                        ret = dict_get_str (dict, GF_XATTR_NODE_UUID_KEY,
+                                            &uuid_str);
+                        if(ret < 0) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "get node-uuid from dict for %s",
+                                        entry_loc.path);
+                                continue;
+                        }
+
+                        if (uuid_parse (uuid_str, node_uuid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "uuid_parse "
+                                        "failed for %s", entry_loc.path);
+                                continue;
+                        }
+
+                        /* if file belongs to different node, skip migration
+                         * the other node will take responsibility of migration
+                         */
+                        if (uuid_compare (node_uuid, defrag->node_uuid)) {
+                                gf_log (this->name, GF_LOG_TRACE, "%s does not"
+                                        "belong to this node", entry_loc.path);
+                                continue;
+                        }
+
+                        uuid_str = NULL;
+
+                        dict_del (dict, GF_XATTR_NODE_UUID_KEY);
+
+
+                        /* if distribute is present, it will honor this key.
+                         * -1 is returned if distribute is not present or file
+                         * doesn't have a link-file. If file has link-file, the
+                         * path of link-file will be the value, and also that
+                         * guarantees that file has to be mostly migrated */
+
+                        ret = syncop_getxattr (this, &entry_loc, &dict,
+                                               GF_XATTR_LINKINFO_KEY);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_TRACE, "failed to "
+                                        "get link-to key for %s",
+                                        entry_loc.path);
+                                continue;
+                        }
+
+                        ret = syncop_setxattr (this, &entry_loc, migrate_data,
+                                               0);
+                        if (ret) {
+                                err = op_errno;
+                                /* errno is overloaded. See
+                                 * rebalance_task_completion () */
+                                if (err != ENOSPC) {
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "migrate-data skipped for %s"
+                                                " due to space constraints",
+                                                entry_loc.path);
+                                        defrag->skipped +=1;
+                                } else{
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "migrate-data failed for %s",
+                                                entry_loc.path);
+                                        defrag->total_failures +=1;
+                                }
+                        }
+
+                        if (ret == -1) {
+                                op_errno = errno;
+                                ret = gf_defrag_handle_migrate_error (op_errno,
+                                                                      defrag);
+
+                                if (!ret)
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "migrate-data on %s failed: %s",
+                                                entry_loc.path,
+                                                strerror (op_errno));
+                                else if (ret == 1)
+                                        continue;
+                                else if (ret == -1)
+                                        goto out;
+                        }
+
+                        LOCK (&defrag->lock);
+                        {
+                                defrag->total_files += 1;
+                                defrag->total_data += iatt.ia_size;
+                        }
+                        UNLOCK (&defrag->lock);
+                        if (defrag->stats == _gf_true) {
+                                gettimeofday (&end, NULL);
+                                elapsed = (end.tv_sec - start.tv_sec) * 1e6 +
+                                          (end.tv_usec - start.tv_usec);
+                                gf_log (this->name, GF_LOG_INFO, "Migration of "
+                                        "file:%s size:%"PRIu64" bytes took %.2f"
+                                        "secs", entry_loc.path, iatt.ia_size,
+                                         elapsed/1e6);
+                        }
+                }
+
+                gf_dirent_free (&entries);
+                free_entries = _gf_false;
+                INIT_LIST_HEAD (&entries.list);
+
+                if (readdir_operrno == ENOENT)
+                        break;
+        }
+
+        gettimeofday (&end, NULL);
+        elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 +
+                  (end.tv_usec - dir_start.tv_usec);
+        gf_log (this->name, GF_LOG_INFO, "Migration operation on dir %s took "
+                "%.2f secs", loc->path, elapsed/1e6);
+        ret = 0;
+out:
+        if (free_entries)
+                gf_dirent_free (&entries);
+
+        loc_wipe (&entry_loc);
+
+        if (dict)
+                dict_unref(dict);
+
+        if (fd)
+                fd_unref (fd);
+        return ret;
+
+}
+
+
+int
+gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                  dict_t *fix_layout, dict_t *migrate_data)
+{
+        int                      ret            = -1;
+        loc_t                    entry_loc      = {0,};
+        fd_t                    *fd             = NULL;
+        gf_dirent_t              entries;
+        gf_dirent_t             *tmp            = NULL;
+        gf_dirent_t             *entry          = NULL;
+        gf_boolean_t             free_entries   = _gf_false;
+        dict_t                  *dict           = NULL;
+        off_t                    offset         = 0;
+        struct iatt              iatt           = {0,};
+        int                      readdirp_errno = 0;
+
+        ret = syncop_lookup (this, loc, NULL, &iatt, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Lookup failed on %s",
+                        loc->path);
+                goto out;
+        }
+
+        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                ret = gf_defrag_migrate_data (this, defrag, loc, migrate_data);
+                if (ret)
+                        goto out;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "fix layout called on %s", loc->path);
+
+        fd = fd_create (loc->inode, defrag->pid);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
+                ret = -1;
+                goto out;
+        }
+
+        ret = syncop_opendir (this, loc, fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
+                        loc->path);
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&entries.list);
+        while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
+                &entries)) != 0)
+        {
+
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s"
+                                ". Aborting fix-layout",strerror(errno));
+                        goto out;
+                }
+
+                /* Need to keep track of ENOENT errno, that means, there is no
+                   need to send more readdirp() */
+                readdirp_errno = errno;
+
+                if (list_empty (&entries.list))
+                        break;
+
+                free_entries = _gf_true;
+
+                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
+                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+                                ret = 1;
+                                goto out;
+                        }
+
+                        offset = entry->d_off;
+
+                        if (!strcmp (entry->d_name, ".") ||
+                            !strcmp (entry->d_name, ".."))
+                                continue;
+
+                        if (!IA_ISDIR (entry->d_stat.ia_type))
+                                continue;
+
+                        loc_wipe (&entry_loc);
+                        ret =dht_build_child_loc (this, &entry_loc, loc,
+                                                  entry->d_name);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
+                                        " build failed");
+                                goto out;
+                        }
+
+                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        entry_loc.inode->ia_type = entry->d_stat.ia_type;
+
+                        uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+                        if (uuid_is_null (loc->gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        uuid_copy (entry_loc.pargfid, loc->gfid);
+
+                        ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
+                                             NULL, NULL);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s"
+                                        " lookup failed", entry_loc.path);
+                                continue;
+                        }
+
+                        ret = syncop_setxattr (this, &entry_loc, fix_layout,
+                                               0);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Setxattr "
+                                        "failed for %s", entry_loc.path);
+                                defrag->defrag_status =
+                                GF_DEFRAG_STATUS_FAILED;
+                                defrag->total_failures ++;
+                                goto out;
+                        }
+                        ret = gf_defrag_fix_layout (this, defrag, &entry_loc,
+                                                    fix_layout, migrate_data);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Fix layout "
+                                        "failed for %s", entry_loc.path);
+                                defrag->total_failures++;
+                                goto out;
+                        }
+
+                }
+                gf_dirent_free (&entries);
+                free_entries = _gf_false;
+                INIT_LIST_HEAD (&entries.list);
+                if (readdirp_errno == ENOENT)
+                        break;
+        }
+
+        ret = 0;
+out:
+        if (free_entries)
+                gf_dirent_free (&entries);
+
+        loc_wipe (&entry_loc);
+
+        if (dict)
+                dict_unref(dict);
+
+        if (fd)
+                fd_unref (fd);
+
+        return ret;
+
+}
+
+
+int
+gf_defrag_start_crawl (void *data)
+{
+        xlator_t                *this   = NULL;
+        dht_conf_t              *conf   = NULL;
+        gf_defrag_info_t        *defrag = NULL;
+        int                      ret    = -1;
+        loc_t                    loc    = {0,};
+        struct iatt              iatt   = {0,};
+        struct iatt              parent = {0,};
+        dict_t                  *fix_layout = NULL;
+        dict_t                  *migrate_data = NULL;
+        dict_t                  *status = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
+
+        this = data;
+        if (!this)
+                goto out;
+
+        ctx = this->ctx;
+        if (!ctx)
+                goto out;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        defrag = conf->defrag;
+        if (!defrag)
+                goto out;
+
+        gettimeofday (&defrag->start_time, NULL);
+        dht_build_root_inode (this, &defrag->root_inode);
+        if (!defrag->root_inode)
+                goto out;
+
+        dht_build_root_loc (defrag->root_inode, &loc);
+
+        /* fix-layout on '/' first */
+
+        ret = syncop_lookup (this, &loc, NULL, &iatt, NULL, &parent);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "look up on / failed");
+                goto out;
+        }
+
+        fix_layout = dict_new ();
+        if (!fix_layout) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set dict str");
+                goto out;
+        }
+
+        ret = syncop_setxattr (this, &loc, fix_layout, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "fix layout on %s failed",
+                        loc.path);
+                defrag->total_failures++;
+                goto out;
+        }
+
+        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                migrate_data = dict_new ();
+                if (!migrate_data) {
+                        ret = -1;
+                        goto out;
+                }
+                if (defrag->cmd == GF_DEFRAG_CMD_START_FORCE)
+                        ret = dict_set_str (migrate_data,
+                                            "distribute.migrate-data", "force");
+                else
+                        ret = dict_set_str (migrate_data,
+                                            "distribute.migrate-data",
+                                            "non-force");
+                if (ret)
+                        goto out;
+        }
+        ret = gf_defrag_fix_layout (this, defrag, &loc, fix_layout,
+                                    migrate_data);
+        if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
+            (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
+                defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+        }
+
+
+
+out:
+        LOCK (&defrag->lock);
+        {
+                status = dict_new ();
+                gf_defrag_status_get (defrag, status);
+                if (ctx->notify)
+                        ctx->notify (GF_EN_DEFRAG_STATUS, status);
+                if (status)
+                        dict_unref (status);
+                defrag->is_exiting = 1;
+        }
+        UNLOCK (&defrag->lock);
+
+        if (defrag) {
+                GF_FREE (defrag);
+                conf->defrag = NULL;
+        }
+
+        return ret;
+}
+
+
+static int
+gf_defrag_done  (int ret, call_frame_t *sync_frame, void *data)
+{
+        gf_listener_stop (sync_frame->this);
+
+        STACK_DESTROY (sync_frame->root);
+        kill (getpid(), SIGTERM);
+        return 0;
+}
+
+void *
+gf_defrag_start (void *data)
+{
+        int                      ret    = -1;
+        call_frame_t            *frame  = NULL;
+        dht_conf_t              *conf   = NULL;
+        gf_defrag_info_t        *defrag = NULL;
+        xlator_t                *this  = NULL;
+
+        this = data;
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        defrag = conf->defrag;
+        if (!defrag)
+                goto out;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        frame->root->pid = GF_CLIENT_PID_DEFRAG;
+
+        defrag->pid = frame->root->pid;
+
+        defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
+
+        ret = synctask_new (this->ctx->env, gf_defrag_start_crawl,
+                            gf_defrag_done, frame, this);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Could not create"
+                        " task for rebalance");
+out:
+        return NULL;
+}
+
+int
+gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
+{
+        int      ret    = 0;
+        uint64_t files  = 0;
+        uint64_t size   = 0;
+        uint64_t lookup = 0;
+        uint64_t failures = 0;
+        uint64_t skipped = 0;
+        char     *status = "";
+        double   elapsed = 0;
+        struct timeval end = {0,};
+
+
+        if (!defrag)
+                goto out;
+
+        ret = 0;
+        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
+                goto out;
+
+        files  = defrag->total_files;
+        size   = defrag->total_data;
+        lookup = defrag->num_files_lookedup;
+        failures = defrag->total_failures;
+        skipped = defrag->skipped;
+
+        gettimeofday (&end, NULL);
+
+        elapsed = end.tv_sec - defrag->start_time.tv_sec;
+
+        if (!dict)
+                goto log;
+
+        ret = dict_set_uint64 (dict, "files", files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set file count");
+
+        ret = dict_set_uint64 (dict, "size", size);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set size of xfer");
+
+        ret = dict_set_uint64 (dict, "lookups", lookup);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set lookedup file count");
+
+
+        ret = dict_set_int32 (dict, "status", defrag->defrag_status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set status");
+        if (elapsed) {
+                ret = dict_set_double (dict, "run-time", elapsed);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set run-time");
+        }
+
+        ret = dict_set_uint64 (dict, "failures", failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set failure count");
+
+        ret = dict_set_uint64 (dict, "skipped", skipped);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set skipped file count");
+log:
+        switch (defrag->defrag_status) {
+        case GF_DEFRAG_STATUS_NOT_STARTED:
+                status = "not started";
+                break;
+        case GF_DEFRAG_STATUS_STARTED:
+                status = "in progress";
+                break;
+        case GF_DEFRAG_STATUS_STOPPED:
+                status = "stopped";
+                break;
+        case GF_DEFRAG_STATUS_COMPLETE:
+                status = "completed";
+                break;
+        case GF_DEFRAG_STATUS_FAILED:
+                status = "failed";
+                break;
+        default:
+                break;
+        }
+
+        gf_log (THIS->name, GF_LOG_INFO, "Rebalance is %s. Time taken is %.2f "
+                "secs", status, elapsed);
+        gf_log (THIS->name, GF_LOG_INFO, "Files migrated: %"PRIu64", size: %"
+                PRIu64", lookups: %"PRIu64", failures: %"PRIu64", skipped: "
+                "%"PRIu64, files, size, lookup, failures, skipped);
+
+
+out:
+        return 0;
+}
+
+int
+gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output)
+{
+        /* TODO: set a variable 'stop_defrag' here, it should be checked
+           in defrag loop */
+        int     ret = -1;
+        GF_ASSERT (defrag);
+
+        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
+                goto out;
+        }
+
+        gf_log ("", GF_LOG_INFO, "Received stop command on rebalance");
+        defrag->defrag_status = GF_DEFRAG_STATUS_STOPPED;
+
+        if (output)
+                gf_defrag_status_get (defrag, output);
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index fa06de711..5d6f4f232 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /* TODO: link(oldpath, newpath) fails if newpath already exists. DHT should
@@ -35,7 +26,8 @@ int
 dht_rename_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
                     struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent)
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -84,7 +76,7 @@ unwind:
                 DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                                   &local->stbuf, &local->preoldparent,
                                   &local->postoldparent,
-                                  &local->preparent, &local->postparent);
+                                  &local->preparent, &local->postparent, xdata);
         }
 
         return 0;
@@ -97,7 +89,7 @@ dht_rename_hashed_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            struct iatt *preoldparent,
                            struct iatt *postoldparent,
                            struct iatt *prenewparent,
-                           struct iatt *postnewparent)
+                           struct iatt *postnewparent, dict_t *xdata)
 {
         dht_conf_t   *conf = NULL;
         dht_local_t  *local = NULL;
@@ -147,7 +139,7 @@ dht_rename_hashed_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 STACK_WIND (frame, dht_rename_dir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->rename,
-                            &local->loc, &local->loc2);
+                            &local->loc, &local->loc2, NULL);
                 if (!--call_cnt)
                         break;
         }
@@ -164,7 +156,7 @@ unwind:
         DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                           &local->stbuf, &local->preoldparent,
                           &local->postoldparent,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
 
         return 0;
 }
@@ -185,19 +177,20 @@ dht_rename_dir_do (call_frame_t *frame, xlator_t *this)
         STACK_WIND (frame, dht_rename_hashed_dir_cbk,
                     local->dst_hashed,
                     local->dst_hashed->fops->rename,
-                    &local->loc, &local->loc2);
+                    &local->loc, &local->loc2, NULL);
         return 0;
 
 err:
         DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno, NULL, NULL,
-                          NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL);
         return 0;
 }
 
 
 int
 dht_rename_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries)
+                        int op_ret, int op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = -1;
@@ -226,7 +219,7 @@ dht_rename_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_rename_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, fd_t *fd)
+                        int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = -1;
@@ -246,7 +239,7 @@ dht_rename_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         STACK_WIND (frame, dht_rename_readdir_cbk,
                     prev->this, prev->this->fops->readdir,
-                    local->fd, 4096, 0);
+                    local->fd, 4096, 0, NULL);
 
         return 0;
 
@@ -302,22 +295,54 @@ dht_rename_dir (call_frame_t *frame, xlator_t *this)
                 STACK_WIND (frame, dht_rename_opendir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->opendir,
-                            &local->loc2, local->fd);
+                            &local->loc2, local->fd, NULL);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL, NULL);
         return 0;
 }
+#define DHT_MARK_FOP_INTERNAL(xattr) do {                                      \
+                int tmp = -1;                                                  \
+                if (!xattr) {                                                  \
+                        xattr = dict_new ();                                   \
+                        if (!xattr)                                            \
+                                break;                                         \
+                }                                                              \
+                tmp = dict_set_str (xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes"); \
+                if (tmp) {                                                     \
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set"      \
+                                " internal dict key for %s", local->loc.path); \
+                }                                                              \
+        }while (0)
+int
+dht_rename_done (call_frame_t *frame, xlator_t *this)
+{
+        dht_local_t     *local = NULL;
+
+        local = frame->local;
+
+        if (local->linked == _gf_true) {
+                local->linked = _gf_false;
+                dht_linkfile_attr_heal (frame, this);
+        }
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                          &local->stbuf, &local->preoldparent,
+                          &local->postoldparent, &local->preparent,
+                          &local->postparent, NULL);
 
+        return 0;
+}
 
 int
 dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                       struct iatt *postparent)
+                       struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -346,11 +371,7 @@ dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         WIPE (&local->postparent);
 
         if (is_last_call (this_call_cnt)) {
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf, &local->preoldparent,
-                                  &local->postoldparent, &local->preparent,
-                                  &local->postparent);
+                dht_rename_done (frame, this);
         }
 
 out:
@@ -368,7 +389,7 @@ dht_rename_cleanup (call_frame_t *frame)
         xlator_t    *dst_hashed = NULL;
         xlator_t    *dst_cached = NULL;
         int          call_cnt = 0;
-
+        dict_t      *xattr      = NULL;
 
         local = frame->local;
         this  = frame->this;
@@ -392,13 +413,15 @@ dht_rename_cleanup (call_frame_t *frame)
         if (!call_cnt)
                 goto nolinks;
 
+        DHT_MARK_FOP_INTERNAL (xattr);
+
         if (dst_hashed != src_hashed && dst_hashed != src_cached) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "unlinking linkfile %s @ %s => %s",
                         local->loc.path, dst_hashed->name, src_cached->name);
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             dst_hashed, dst_hashed->fops->unlink,
-                            &local->loc);
+                            &local->loc, 0, xattr);
         }
 
         if (src_cached != dst_hashed) {
@@ -407,9 +430,12 @@ dht_rename_cleanup (call_frame_t *frame)
                         local->loc2.path, src_cached->name);
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             src_cached, src_cached->fops->unlink,
-                            &local->loc2);
+                            &local->loc2, 0, xattr);
         }
 
+        if (xattr)
+                dict_unref (xattr);
+
         return 0;
 
 nolinks:
@@ -422,7 +448,7 @@ nolinks:
         DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                           &local->stbuf, &local->preoldparent,
                           &local->postoldparent, &local->preparent,
-                          &local->postparent);
+                          &local->postparent, NULL);
 
         return 0;
 }
@@ -430,9 +456,10 @@ nolinks:
 
 int
 dht_rename_links_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent)
+                             int32_t op_ret, int32_t op_errno,
+                             inode_t *inode, struct iatt *stbuf,
+                             struct iatt *preparent, struct iatt *postparent,
+                             dict_t *xdata)
 {
         call_frame_t *prev = NULL;
         dht_local_t  *local = NULL;
@@ -446,6 +473,10 @@ dht_rename_links_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->loc.path, prev->this->name, strerror (op_errno));
         }
 
+        if (local->linked == _gf_true) {
+                local->linked = _gf_false;
+                dht_linkfile_attr_heal (frame, this);
+        }
         DHT_STACK_DESTROY (frame);
 
         return 0;
@@ -456,7 +487,8 @@ int
 dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
                 struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent)
+                struct iatt *prenewparent, struct iatt *postnewparent,
+                dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -467,6 +499,7 @@ dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         xlator_t     *rename_subvol = NULL;
         call_frame_t *link_frame = NULL;
         dht_local_t *link_local = NULL;
+        dict_t       *xattr     = NULL;
 
         local = frame->local;
         prev = cookie;
@@ -476,6 +509,8 @@ dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         dst_hashed = local->dst_hashed;
         dst_cached = local->dst_cached;
 
+        if (local->linked == _gf_true)
+                FRAME_SU_UNDO (frame, dht_local_t);
         if (op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: rename on %s failed (%s)", local->loc.path,
@@ -505,15 +540,25 @@ dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 uuid_copy (link_local->gfid, local->loc.inode->gfid);
 
                 dht_linkfile_create (link_frame, dht_rename_links_create_cbk,
-                                     src_cached, dst_hashed, &link_local->loc);
+                                     this, src_cached, dst_hashed,
+                                     &link_local->loc);
         }
 
 err:
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent, prev->this);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent, prev->this);
-        dht_iatt_merge (this, &local->preparent, prenewparent, prev->this);
-        dht_iatt_merge (this, &local->postparent, postnewparent, prev->this);
+        /* Merge attrs only from src_cached. In case there of src_cached !=
+         * dst_hashed, this ignores linkfile attrs. */
+        if (prev->this == src_cached) {
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                dht_iatt_merge (this, &local->preoldparent, preoldparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->postoldparent, postoldparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->preparent, prenewparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->postparent, postnewparent,
+                                prev->this);
+        }
+
 
         /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
          *       is called. since rename has already happened on rename_subvol,
@@ -538,6 +583,8 @@ err:
         if (local->call_cnt == 0)
                 goto unwind;
 
+        DHT_MARK_FOP_INTERNAL (xattr);
+
         if (src_cached != dst_hashed && src_cached != dst_cached) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "deleting old src datafile %s @ %s",
@@ -545,7 +592,7 @@ err:
 
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             src_cached, src_cached->fops->unlink,
-                            &local->loc);
+                            &local->loc, 0, xattr);
         }
 
         if (src_hashed != rename_subvol && src_hashed != src_cached) {
@@ -555,7 +602,7 @@ err:
 
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             src_hashed, src_hashed->fops->unlink,
-                            &local->loc);
+                            &local->loc, 0, xattr);
         }
 
         if (dst_cached
@@ -567,8 +614,10 @@ err:
 
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             dst_cached, dst_cached->fops->unlink,
-                            &local->loc2);
+                            &local->loc2, 0, xattr);
         }
+        if (xattr)
+                dict_unref (xattr);
         return 0;
 
 unwind:
@@ -576,16 +625,16 @@ unwind:
         WIPE (&local->postoldparent);
         WIPE (&local->preparent);
         WIPE (&local->postparent);
+        if (xattr)
+                dict_unref (xattr);
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                          &local->stbuf, &local->preoldparent,
-                          &local->postoldparent, &local->preparent,
-                          &local->postparent);
+        dht_rename_done (frame, this);
 
         return 0;
 
 cleanup:
+        if (xattr)
+                dict_unref (xattr);
         dht_rename_cleanup (frame);
 
         return 0;
@@ -619,9 +668,11 @@ dht_do_rename (call_frame_t *frame)
                 "renaming %s => %s (%s)",
                 local->loc.path, local->loc2.path, rename_subvol->name);
 
+        if (local->linked == _gf_true)
+                FRAME_SU_DO (frame, dht_local_t);
         STACK_WIND (frame, dht_rename_cbk,
                     rename_subvol, rename_subvol->fops->rename,
-                    &local->loc, &local->loc2);
+                    &local->loc, &local->loc2, NULL);
 
         return 0;
 }
@@ -631,7 +682,8 @@ int
 dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
                       inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent,
+                      dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -648,6 +700,9 @@ dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->op_ret   = -1;
                 if (op_errno != ENOENT)
                         local->op_errno = op_errno;
+        } else if (local->src_cached == prev->this) {
+                /* merge of attr returned only from linkfile creation */
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
         }
 
         this_call_cnt = dht_frame_return (frame);
@@ -670,7 +725,8 @@ cleanup:
 int
 dht_rename_unlink_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                              int32_t op_ret, int32_t op_errno,
-                             struct iatt *preparent, struct iatt *postparent)
+                             struct iatt *preparent, struct iatt *postparent,
+                             dict_t *xdata)
 {
 	dht_local_t  *local = NULL;
 	call_frame_t *prev = NULL;
@@ -712,6 +768,7 @@ dht_rename_create_links (call_frame_t *frame)
         xlator_t    *dst_hashed = NULL;
         xlator_t    *dst_cached = NULL;
         int          call_cnt = 0;
+        dict_t      *xattr = NULL;
 
 
         local = frame->local;
@@ -722,6 +779,7 @@ dht_rename_create_links (call_frame_t *frame)
         dst_hashed = local->dst_hashed;
         dst_cached = local->dst_cached;
 
+        DHT_MARK_FOP_INTERNAL (xattr);
 
         if (src_cached == dst_cached) {
                 if (dst_hashed == dst_cached)
@@ -733,7 +791,7 @@ dht_rename_create_links (call_frame_t *frame)
 
 		STACK_WIND (frame, dht_rename_unlink_links_cbk,
 			    dst_hashed, dst_hashed->fops->unlink,
-			    &local->loc2);
+			    &local->loc2, 0, xattr);
                 return 0;
         }
 
@@ -750,7 +808,7 @@ dht_rename_create_links (call_frame_t *frame)
                         "linkfile %s @ %s => %s",
                         local->loc.path, dst_hashed->name, src_cached->name);
                 memcpy (local->gfid, local->loc.inode->gfid, 16);
-		dht_linkfile_create (frame, dht_rename_links_cbk,
+		dht_linkfile_create (frame, dht_rename_links_cbk, this,
 				     src_cached, dst_hashed, &local->loc);
 	}
 
@@ -760,7 +818,7 @@ dht_rename_create_links (call_frame_t *frame)
 			local->loc2.path, src_cached->name);
 		STACK_WIND (frame, dht_rename_links_cbk,
 			    src_cached, src_cached->fops->link,
-			    &local->loc, &local->loc2);
+			    &local->loc, &local->loc2, xattr);
 	}
 
 nolinks:
@@ -768,6 +826,8 @@ nolinks:
                 /* skip to next step */
                 dht_do_rename (frame);
         }
+        if (xattr)
+                dict_unref (xattr);
 
         return 0;
 }
@@ -775,7 +835,7 @@ nolinks:
 
 int
 dht_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc)
+            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         xlator_t    *src_cached = NULL;
         xlator_t    *src_hashed = NULL;
@@ -857,7 +917,8 @@ dht_rename (call_frame_t *frame, xlator_t *this,
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL, NULL);
 
         return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index b87e0ab48..3fe96b1c7 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -26,7 +17,7 @@
 #include "glusterfs.h"
 #include "xlator.h"
 #include "dht-common.h"
-
+#include "glusterfs-acl.h"
 
 #define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,cnt,path)    do {       \
                 layout->list[i].start = srt;                            \
@@ -38,43 +29,40 @@
                         layout->list[i].xlator->name, path);            \
         } while (0)
 
+#define DHT_RESET_LAYOUT_RANGE(layout)    do {                          \
+                int cnt = 0;                                            \
+                for (cnt = 0; cnt < layout->cnt; cnt++ ) {              \
+                        layout->list[cnt].start = 0;                    \
+                        layout->list[cnt].stop  = 0;                    \
+                }                                                       \
+        } while (0)
 
-static inline uint32_t
-dht_find_overlap (int idx, int cnk_idx, uint32_t start, uint32_t stop,
-                  uint32_t chunk_size)
+static uint32_t
+dht_overlap_calc (dht_layout_t *old, int o, dht_layout_t *new, int n)
 {
-        uint32_t overlap = 0;
-        uint32_t chunk_begin = 0;
+	if (o >= old->cnt || n >= new->cnt)
+		return 0;
 
-        chunk_begin = cnk_idx * chunk_size;
+	if (old->list[o].err > 0 || new->list[n].err > 0)
+		return 0;
 
-        /* There is no chance of overlap */
-        if ((chunk_begin > stop) ||
-            ((chunk_begin + chunk_size) < start))
-                goto out;
-
-        if ((chunk_begin <= start) &&
-            ((chunk_begin + chunk_size) <= stop)) {
-                overlap = ((chunk_begin + chunk_size) - start);
-                goto out;
+        if (old->list[o].start == old->list[o].stop) {
+                return 0;
         }
 
-        if ((chunk_begin <= start) &&
-            ((chunk_begin + chunk_size) >= stop)) {
-                overlap = (stop - start);
-                goto out;
+        if (new->list[n].start == new->list[n].stop) {
+                return 0;
         }
 
-        if ((chunk_begin < stop) &&
-            ((chunk_begin + chunk_size) >= stop)) {
-                overlap = (stop - chunk_begin);
-                goto out;
-        }
+	if ((old->list[o].start > new->list[n].stop) ||
+	    (old->list[o].stop < new->list[n].start))
+		return 0;
 
-out:
-        return overlap;
+	return min (old->list[o].stop, new->list[n].stop) -
+	        max (old->list[o].start, new->list[n].start) + 1;
 }
 
+
 int
 dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
 {
@@ -82,7 +70,7 @@ dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
 
         local = frame->local;
         local->selfheal.dir_cbk (frame, NULL, frame->this, ret,
-                                 local->op_errno);
+                                 local->op_errno, NULL);
 
         return 0;
 }
@@ -90,7 +78,7 @@ dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
 
 int
 dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int op_ret, int op_errno)
+                            int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -129,7 +117,8 @@ dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
-                                  dht_layout_t *layout, int i)
+                                  dht_layout_t *layout, int i,
+                                  xlator_t *req_subvol)
 {
         xlator_t          *subvol = NULL;
         dict_t            *xattr = NULL;
@@ -137,12 +126,23 @@ dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
         xlator_t          *this = NULL;
         int32_t           *disk_layout = NULL;
         dht_local_t       *local = NULL;
-
+        dht_conf_t        *conf = NULL;
 
         local = frame->local;
-        subvol = layout->list[i].xlator;
+        if (req_subvol)
+                subvol = req_subvol;
+        else
+                subvol = layout->list[i].xlator;
         this = frame->this;
 
+        GF_VALIDATE_OR_GOTO ("", this, err);
+        GF_VALIDATE_OR_GOTO (this->name, layout, err);
+        GF_VALIDATE_OR_GOTO (this->name, local, err);
+        GF_VALIDATE_OR_GOTO (this->name, subvol, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
+
         xattr = get_new_dict ();
         if (!xattr) {
                 goto err;
@@ -156,8 +156,7 @@ dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
                 goto err;
         }
 
-        ret = dict_set_bin (xattr, "trusted.glusterfs.dht",
-                            disk_layout, 4 * 4);
+        ret = dict_set_bin (xattr, conf->xattr_name, disk_layout, 4 * 4);
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: (subvol %s) failed to set xattr dictionary",
@@ -178,7 +177,7 @@ dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
 
         STACK_WIND (frame, dht_selfheal_dir_xattr_cbk,
                     subvol, subvol->fops->setxattr,
-                    loc, xattr, 0);
+                    loc, xattr, 0, NULL);
 
         dict_unref (xattr);
 
@@ -188,11 +187,10 @@ err:
         if (xattr)
                 dict_destroy (xattr);
 
-        if (disk_layout)
-                GF_FREE (disk_layout);
+        GF_FREE (disk_layout);
 
         dht_selfheal_dir_xattr_cbk (frame, subvol, frame->this,
-                                    -1, ENOMEM);
+                                    -1, ENOMEM, NULL);
         return 0;
 }
 
@@ -203,21 +201,42 @@ dht_fix_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
         int          i = 0;
         int          count = 0;
         xlator_t    *this = NULL;
+        dht_conf_t  *conf = NULL;
+        dht_layout_t *dummy = NULL;
 
         local = frame->local;
         this = frame->this;
+        conf = this->private;
 
         gf_log (this->name, GF_LOG_DEBUG,
                 "writing the new range for all subvolumes");
 
-        local->call_cnt = count = layout->cnt;
+        local->call_cnt = count = conf->subvolume_cnt;
 
         for (i = 0; i < layout->cnt; i++) {
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i);
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
 
                 if (--count == 0)
-                        break;
+                        goto out;
+        }
+        /* if we are here, subvolcount > layout_count. subvols-per-directory
+         * option might be set here. We need to clear out layout from the
+         * non-participating subvolumes, else it will result in overlaps */
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                        if (--count == 0)
+                                break;
+                }
         }
+
+        dht_layout_unref (this, dummy);
+out:
         return 0;
 }
 
@@ -228,9 +247,12 @@ dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
         int          missing_xattr = 0;
         int          i = 0;
         xlator_t    *this = NULL;
+        dht_conf_t   *conf = NULL;
+        dht_layout_t *dummy = NULL;
 
         local = frame->local;
         this = frame->this;
+        conf = this->private;
 
         for (i = 0; i < layout->cnt; i++) {
                 if (layout->list[i].err != -1 || !layout->list[i].stop) {
@@ -259,18 +281,30 @@ dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
                 if (layout->list[i].err != -1 || !layout->list[i].stop)
                         continue;
 
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i);
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
 
                 if (--missing_xattr == 0)
                         break;
         }
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                }
+        }
+        dht_layout_unref (this, dummy);
+out:
         return 0;
 }
 
 int
 dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                               int op_ret, int op_errno, struct iatt *statpre,
-                              struct iatt *statpost)
+                              struct iatt *statpost, dict_t *xdata)
 {
         dht_local_t   *local = NULL;
         dht_layout_t  *layout = NULL;
@@ -324,7 +358,7 @@ dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
                         STACK_WIND (frame, dht_selfheal_dir_setattr_cbk,
                                     layout->list[i].xlator,
                                     layout->list[i].xlator->fops->setattr,
-                                    loc, stbuf, valid);
+                                    loc, stbuf, valid, NULL);
                 }
         }
 
@@ -335,7 +369,8 @@ int
 dht_selfheal_dir_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                             int op_ret, int op_errno,
                             inode_t *inode, struct iatt *stbuf,
-                            struct iatt *preparent, struct iatt *postparent)
+                            struct iatt *preparent, struct iatt *postparent,
+                            dict_t *xdata)
 {
         dht_local_t   *local = NULL;
         dht_layout_t  *layout = NULL;
@@ -381,6 +416,46 @@ out:
         return 0;
 }
 
+void
+dht_selfheal_dir_mkdir_setacl (dict_t *xattr, dict_t *dict)
+{
+        data_t          *acl_default = NULL;
+        data_t          *acl_access = NULL;
+        xlator_t        *this = NULL;
+        int     ret = -1;
+
+        GF_ASSERT (xattr);
+        GF_ASSERT (dict);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        acl_default = dict_get (xattr, POSIX_ACL_DEFAULT_XATTR);
+
+        if (!acl_default) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "ACL_DEFAULT xattr not present");
+                goto cont;
+        }
+        ret = dict_set (dict, POSIX_ACL_DEFAULT_XATTR, acl_default);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Could not set ACL_DEFAULT xattr");
+cont:
+        acl_access = dict_get (xattr, POSIX_ACL_ACCESS_XATTR);
+        if (!acl_access) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "ACL_ACCESS xattr not present");
+                goto out;
+        }
+        ret = dict_set (dict, POSIX_ACL_ACCESS_XATTR, acl_access);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Could not set ACL_ACCESS xattr");
+
+out:
+        return;
+}
 
 int
 dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
@@ -414,16 +489,19 @@ dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
 
                 ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
                 if (ret)
-                        gf_log (this->name, GF_LOG_INFO,
+                        gf_log (this->name, GF_LOG_WARNING,
                                 "%s: failed to set gfid in dict", loc->path);
         } else if (local->params) {
                 /* Send the dictionary from higher layers directly */
                 dict = dict_ref (local->params);
         }
+        /* Set acls */
+        if (local->xattr && dict)
+                dht_selfheal_dir_mkdir_setacl (local->xattr, dict);
 
         if (!dict)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "dict is NULL, need to make sure gfid's are same");
+                        "dict is NULL, need to make sure gfids are same");
 
         for (i = 0; i < layout->cnt; i++) {
                 if (layout->list[i].err == ENOENT || force) {
@@ -437,7 +515,7 @@ dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
                                     loc,
                                     st_mode_from_ia (local->stbuf.ia_prot,
                                                      local->stbuf.ia_type),
-                                    dict);
+                                    0, dict);
                 }
         }
 
@@ -456,7 +534,7 @@ dht_selfheal_layout_alloc_start (xlator_t *this, loc_t *loc,
         uint32_t      hashval = 0;
         int           ret = 0;
 
-        ret = dht_hash_compute (layout->type, loc->path, &hashval);
+        ret = dht_hash_compute (this, layout->type, loc->path, &hashval);
         if (ret == 0) {
                 start = (hashval % layout->cnt);
         }
@@ -479,7 +557,7 @@ dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
                 for (j = 0; j < conf->subvolume_cnt; j++) {
                         if (conf->decommissioned_bricks[j] &&
                             conf->decommissioned_bricks[j] == layout->list[i].xlator) {
-                                layout->list[i].err = -EINVAL;
+                                layout->list[i].err = EINVAL;
                                 break;
                         }
                 }
@@ -487,9 +565,33 @@ dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
 
         for (i = 0; i < layout->cnt; i++) {
                 err = layout->list[i].err;
-                if (err == -1 || err == 0) {
-                        layout->list[i].err = -1;
+                if (err == -1 || err == 0 || err == ENOENT) {
+			/* Setting list[i].err = -1 is an indication for
+			   dht_selfheal_layout_new_directory() to assign
+			   a range. We set it to -1 based on any one of
+			   the three criteria:
+
+			   - err == -1 already, which means directory
+			     existed but layout was not set on it.
+
+			   - err == 0, which means directory exists and
+			     has an old layout piece which will be
+			     overwritten now.
+
+			   - err == ENOENT, which means directory does
+			     not exist (possibly racing with mkdir or
+			     finishing half done mkdir). The missing
+			     directory will be attempted to be recreated.
+
+			     It is important to note that it is safe
+			     to race with mkdir() as self-heal and
+			     mkdir are idempotent operations. Both will
+			     strive to set the directory and layouts to
+			     the same final state.
+			*/
                         count++;
+			if (!err)
+				layout->list[i].err = -1;
                 }
         }
 
@@ -504,49 +606,126 @@ dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
                 }
         }
 
-        count = ((layout->spread_cnt) ? layout->spread_cnt :
-                 ((count) ? count : 1));
+        /* if layout->spread_cnt is set, check if it is <= available
+         * subvolumes (down brick and decommissioned bricks are considered
+         * un-availbale). Else return count (available up bricks) */
+        count = ((layout->spread_cnt &&
+                 (layout->spread_cnt <= count)) ?
+                 layout->spread_cnt : ((count) ? count : 1));
 
         return count;
 }
 
 
+void dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
+					dht_layout_t *new_layout);
+
+void dht_layout_entry_swap (dht_layout_t *layout, int i, int j);
+void dht_layout_range_swap (dht_layout_t *layout, int i, int j);
+
+/*
+ * It's a bit icky using local variables in a macro, but it makes the rest
+ * of the code a lot clearer.
+ */
+#define OV_ENTRY(x,y)      table[x*new->cnt+y]
+
+void
+dht_selfheal_layout_maximize_overlap (call_frame_t *frame, loc_t *loc,
+				      dht_layout_t *new, dht_layout_t *old)
+{
+	int           i            = 0;
+	int           j            = 0;
+	uint32_t      curr_overlap = 0;
+	uint32_t      max_overlap  = 0;
+	int           max_overlap_idx = -1;
+	uint32_t      overlap      = 0;
+        uint32_t     *table = NULL;
+
+	dht_layout_sort_volname (old);
+	/* Now both old_layout->list[] and new_layout->list[]
+	   are match the same xlators/subvolumes. i.e,
+	   old_layout->[i] and new_layout->[i] are referring
+	   to the same subvolumes
+	*/
+
+        /* Build a table of overlaps between new[i] and old[j]. */
+        table = alloca(sizeof(overlap)*old->cnt*new->cnt);
+        if (!table) {
+                return;
+        }
+        memset(table,0,sizeof(overlap)*old->cnt*new->cnt);
+        for (i = 0; i < new->cnt; ++i) {
+                for (j = 0; j < old->cnt; ++j) {
+                        OV_ENTRY(i,j) = dht_overlap_calc(old,j,new,i);
+                }
+        }
+
+	for (i = 0; i < new->cnt; i++) {
+		if (new->list[i].err > 0) {
+			/* Subvol might be marked for decommission
+			   with EINVAL, or some other serious error
+			   marked with positive errno.
+			*/
+			continue;
+                }
+
+                max_overlap = 0;
+                max_overlap_idx = i;
+                for (j = (i + 1); j < new->cnt; ++j) {
+                        if (new->list[j].err > 0) {
+			        /* Subvol might be marked for decommission
+			        with EINVAL, or some other serious error
+			        marked with positive errno.
+			        */
+			        continue;
+                        }
+                        /* Calculate the overlap now. */
+                        curr_overlap = OV_ENTRY(i,i) + OV_ENTRY(j,j);
+                        /* Calculate the overlap after the proposed swap. */
+                        overlap = OV_ENTRY(i,j) + OV_ENTRY(j,i);
+                        /* Are we better than status quo? */
+                        if (overlap > curr_overlap) {
+                                overlap -= curr_overlap;
+                                /* Are we better than the previous choice? */
+                                if (overlap > max_overlap) {
+                                        max_overlap = overlap;
+                                        max_overlap_idx = j;
+                                }
+                        }
+                }
+
+		if (max_overlap_idx != i) {
+ 			dht_layout_range_swap (new, i, max_overlap_idx);
+                        /* Need to swap the table values too. */
+                        for (j = 0; j < old->cnt; ++j) {
+                                overlap = OV_ENTRY(i,j);
+                                OV_ENTRY(i,j) = OV_ENTRY(max_overlap_idx,j);
+                                OV_ENTRY(max_overlap_idx,j) = overlap;
+                        }
+                }
+	}
+}
+
+
 dht_layout_t *
 dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
                              dht_layout_t *layout)
 {
-        uint32_t      chunk        = 0;
-        uint32_t      start        = 0;
-        uint32_t      stop         = 0;
-        uint32_t      overlap      = 0;
-        uint32_t      max_overlap  = 0;
-        uint32_t      chunk_begin  = 0;
-        int           count        = 0;
-        int           cnt          = 0;
         int           i            = 0;
-        int           j            = 0;
-        int           k            = 0;
-        int           loop_cnt     = 0;
-        int           start_subvol = 0;
-        int          *fix_array    = NULL;
         xlator_t     *this         = NULL;
         dht_layout_t *new_layout   = NULL;
         dht_conf_t   *priv         = NULL;
         dht_local_t  *local        = NULL;
+        uint32_t      subvol_down  = 0;
+        int           ret          = 0;
 
         this  = frame->this;
         priv  = this->private;
         local = frame->local;
 
-        count = cnt = dht_get_layout_count (this, layout, 0);
-
-        chunk = ((unsigned long) 0xffffffff) / ((cnt) ? cnt : 1);
-
-        start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
-
-        fix_array = GF_CALLOC (sizeof (int), layout->cnt, gf_common_mt_char);
-        if (!fix_array) {
-                /* No fix, use the existing layout itself */
+        if (layout->type == DHT_HASH_TYPE_DM_USER) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "leaving %s alone",
+                        loc->path);
                 goto done;
         }
 
@@ -554,98 +733,33 @@ dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
         if (!new_layout)
                 goto done;
 
-        for (i = 0; i < new_layout->cnt; i++) {
-                /* TODO: fix this in layout_alloc() itself */
-                new_layout->list[i].err = -ENOENT;
-                if (i < layout->cnt)
-                        new_layout->list[i].xlator = layout->list[i].xlator;
-        }
-
-        /* Check if there are any overlap in layout, and give the proper fix */
-        for (i = 0; i < layout->cnt; i++) {
-                /* No need to fix if 'err' is not '-1' */
-                if (layout->list[i].err != -1)
-                        continue;
-
-                /* If already existing layout is having no range, skip it */
-                start = layout->list[i].start;
-                stop  = layout->list[i].stop;
-                if ((stop - start) == 0)
-                        continue;
-
-                max_overlap = 0;
-
-                /* 'j' is used as starting point of each chunk */
-                for (j = 1; j <= count; j++) {
-                        /* if chunk is already used, don't use it again */
-                        for (k = 0; k < i; k++)
-                                if (j == fix_array[k])
-                                        break;
-                        if (k < i)
-                                continue;
-
-                        overlap = dht_find_overlap (i, (j-1), start, stop, chunk);
-                        if (max_overlap < overlap) {
-                                max_overlap = overlap;
-                                fix_array[i] = j;
-                        }
-                }
-
-                /* If we have any overlap, then use that itself as new
-                   layout for the subvolume */
-                if (fix_array[i]) {
-                        chunk_begin = chunk * (fix_array[i] - 1);
-                        new_layout->list[i].err = -1;
-                        DHT_SET_LAYOUT_RANGE (new_layout, i, chunk_begin,
-                                              chunk, cnt, loc->path);
-                        /* make sure to give (max - 1) as 'stop' range,
-                           if it is last chunk */
-                        if (fix_array[i] == count)
-                                new_layout->list[i].stop = 0xffffffff;
-                        if (--cnt == 0)
-                                goto done;
+        /* If a subvolume is down, do not re-write the layout. */
+        ret = dht_layout_anomalies (this, loc, layout, NULL, NULL, NULL,
+                                    &subvol_down, NULL, NULL);
 
-                }
+        if (subvol_down || (ret == -1)) {
+                gf_log (this->name, GF_LOG_WARNING, "%u subvolume(s) are down"
+                        ". Skipping fix layout.", subvol_down);
+                GF_FREE (new_layout);
+                return NULL;
         }
 
-        /* Now, look for layouts which are not having any overlaps
-           and give it a fix */
-        for (loop_cnt = 0, i = start_subvol; loop_cnt < new_layout->cnt;
-             i++, loop_cnt++) {
-                if (i == new_layout->cnt)
-                        i = 0;
-
-                /* If 'fix_array[i]' is set, the layout is already fixed. */
-                if (fix_array[i])
-                        continue;
+        for (i = 0; i < new_layout->cnt; i++) {
+		if (layout->list[i].err != ENOSPC)
+			new_layout->list[i].err = layout->list[i].err;
+		else
+			new_layout->list[i].err = -1;
 
-                if (layout->list[i].err != -1) {
-                        new_layout->list[i].err = layout->list[i].err;
-                        continue;
-                }
+		new_layout->list[i].xlator = layout->list[i].xlator;
+        }
 
-                for (k = 1; k <= count; k++) {
-                        for (j = 0; j < new_layout->cnt; j++) {
-                                if (k == fix_array[j])
-                                        break;
-                        }
-                        /* Didn't find any of the list begining with 'k' */
-                        if (j == new_layout->cnt)
-                                break;
-                }
+	/* First give it a layout as though it is a new directory. This
+	   ensures rotation to kick in */
+        dht_layout_sort_volname (new_layout);
+	dht_selfheal_layout_new_directory (frame, loc, new_layout);
 
-                fix_array[i] = k;
-                chunk_begin = (k - 1) * chunk;
-                new_layout->list[i].err = -1;
-                DHT_SET_LAYOUT_RANGE (new_layout, i, chunk_begin, chunk, cnt,
-                                      loc->path);
-                /* make sure to give (max - 1) as 'stop' range,
-                   if it is last chunk */
-                if (k == count)
-                        new_layout->list[i].stop = 0xffffffff;
-                if (--cnt == 0)
-                        goto done;
-        }
+	/* Now selectively re-assign ranges only when it helps */
+	dht_selfheal_layout_maximize_overlap (frame, loc, new_layout, layout);
 
 done:
         if (new_layout) {
@@ -659,7 +773,7 @@ done:
                 local->layout = new_layout;
         }
 
-        return new_layout;
+        return local->layout;
 }
 
 
@@ -683,9 +797,11 @@ dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
 
         start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
 
+        /* clear out the range, as we are re-computing here */
+        DHT_RESET_LAYOUT_RANGE (layout);
         for (i = start_subvol; i < layout->cnt; i++) {
                 err = layout->list[i].err;
-                if (err == -1) {
+                if (err == -1 || err == ENOENT) {
                         DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
                                              cnt, loc->path);
                         if (--cnt == 0) {
@@ -698,7 +814,7 @@ dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
 
         for (i = 0; i < start_subvol; i++) {
                 err = layout->list[i].err;
-                if (err == -1) {
+                if (err == -1 || err == ENOENT) {
                         DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
                                              cnt, loc->path);
                         if (--cnt == 0) {
@@ -717,35 +833,17 @@ int
 dht_selfheal_dir_getafix (call_frame_t *frame, loc_t *loc,
                           dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-        xlator_t    *this = NULL;
         dht_local_t *local = NULL;
-        int          missing = -1;
-        int          down = -1;
-        int          holes = -1;
+        uint32_t     holes = 0;
         int          ret = -1;
         int          i = -1;
-        int          overlaps = -1;
+        uint32_t     overlaps = 0;
 
-        this = frame->this;
-        conf = this->private;
         local = frame->local;
 
-        missing = local->selfheal.missing;
-        down = local->selfheal.down;
         holes = local->selfheal.hole_cnt;
         overlaps = local->selfheal.overlaps_cnt;
 
-        if ((missing + down) == conf->subvolume_cnt) {
-                dht_selfheal_layout_new_directory (frame, loc, layout);
-                ret = 0;
-        }
-
-        if (holes <= down) {
-                /* the down subvol might fill up the holes */
-                ret = 0;
-        }
-
         if (holes || overlaps) {
                 dht_selfheal_layout_new_directory (frame, loc, layout);
                 ret = 0;
@@ -797,6 +895,9 @@ dht_fix_directory_layout (call_frame_t *frame,
 
         /* No layout sorting required here */
         tmp_layout = dht_fix_layout_of_directory (frame, &local->loc, layout);
+        if (!tmp_layout) {
+                return -1;
+        }
         dht_fix_dir_xattr (frame, &local->loc, tmp_layout);
 
         return 0;
@@ -819,9 +920,8 @@ dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
         dht_layout_anomalies (this, loc, layout,
                               &local->selfheal.hole_cnt,
                               &local->selfheal.overlaps_cnt,
-                              &local->selfheal.missing,
-                              &local->selfheal.down,
-                              &local->selfheal.misc);
+                              NULL, &local->selfheal.down,
+                              &local->selfheal.misc, NULL);
 
         down     = local->selfheal.down;
         misc     = local->selfheal.misc;
@@ -830,14 +930,14 @@ dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
         local->selfheal.layout = dht_layout_ref (this, layout);
 
         if (down) {
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_WARNING,
                         "%d subvolumes down -- not fixing", down);
                 ret = 0;
                 goto sorry_no_fix;
         }
 
         if (misc) {
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_WARNING,
                         "%d subvolumes have unrecoverable errors", misc);
                 ret = 0;
                 goto sorry_no_fix;
@@ -847,7 +947,7 @@ dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
         ret = dht_selfheal_dir_getafix (frame, loc, layout);
 
         if (ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_WARNING,
                         "not able to form layout for the directory");
                 goto sorry_no_fix;
         }
@@ -880,3 +980,50 @@ dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
 
         return ret;
 }
+
+int
+dht_dir_attr_heal (void *data)
+{
+        call_frame_t    *frame = NULL;
+        dht_local_t     *local = NULL;
+        xlator_t        *subvol = NULL;
+        xlator_t        *this  = NULL;
+        dht_conf_t      *conf  = NULL;
+        int              call_cnt = 0;
+        int              ret   = -1;
+        int              i     = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", data, out);
+
+        frame = data;
+        local = frame->local;
+        this = frame->this;
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", local, out);
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO ("dht", conf, out);
+
+        call_cnt = conf->subvolume_cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                subvol = conf->subvolumes[i];
+                if (!subvol || (subvol == dht_first_up_subvol (this)))
+                        continue;
+                ret = syncop_setattr (subvol, &local->loc, &local->stbuf,
+                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                                      NULL, NULL);
+                if (ret)
+                        gf_log ("dht", GF_LOG_ERROR, "Failed to set uid/gid on"
+                                " %s on %s subvol (%s)", local->loc.path,
+                                subvol->name, strerror (errno));
+        }
+out:
+        return 0;
+}
+
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data)
+{
+        DHT_STACK_DESTROY (sync_frame);
+        return 0;
+}
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
new file mode 100644
index 000000000..70aac7710
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -0,0 +1,758 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+/* TODO: add NS locking */
+
+#include "statedump.h"
+#include "dht-common.h"
+
+/* TODO:
+   - use volumename in xattr instead of "dht"
+   - use NS locks
+   - handle all cases in self heal layout reconstruction
+   - complete linkfile selfheal
+*/
+struct volume_options options[];
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix)
+{
+
+        char    key[GF_DUMP_MAX_BUF_LEN];
+        int     i = 0;
+
+        if (!layout)
+                goto out;
+        if (!prefix)
+                goto out;
+
+        gf_proc_dump_build_key(key, prefix, "cnt");
+        gf_proc_dump_write(key, "%d", layout->cnt);
+        gf_proc_dump_build_key(key, prefix, "preset");
+        gf_proc_dump_write(key, "%d", layout->preset);
+        gf_proc_dump_build_key(key, prefix, "gen");
+        gf_proc_dump_write(key, "%d", layout->gen);
+        if (layout->type != IA_INVAL) {
+                gf_proc_dump_build_key(key, prefix, "inode type");
+                gf_proc_dump_write(key, "%d", layout->type);
+        }
+
+        if  (!IA_ISDIR (layout->type))
+                goto out;
+
+        for (i = 0; i < layout->cnt; i++) {
+                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
+                gf_proc_dump_write(key, "%d", layout->list[i].err);
+                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
+                gf_proc_dump_write(key, "%u", layout->list[i].start);
+                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
+                gf_proc_dump_write(key, "%u", layout->list[i].stop);
+                if (layout->list[i].xlator) {
+                        gf_proc_dump_build_key(key, prefix,
+                                               "list[%d].xlator.type", i);
+                        gf_proc_dump_write(key, "%s",
+                                           layout->list[i].xlator->type);
+                        gf_proc_dump_build_key(key, prefix,
+                                               "list[%d].xlator.name", i);
+                        gf_proc_dump_write(key, "%s",
+                                           layout->list[i].xlator->name);
+                }
+        }
+
+out:
+        return;
+}
+
+
+int32_t
+dht_priv_dump (xlator_t *this)
+{
+        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
+        char            key[GF_DUMP_MAX_BUF_LEN];
+        int             i = 0;
+        dht_conf_t      *conf = NULL;
+        int             ret = -1;
+
+        if (!this)
+                goto out;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        ret = TRY_LOCK(&conf->subvolume_lock);
+        if (ret != 0) {
+                return ret;
+        }
+
+        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
+        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
+                               this->name);
+        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                sprintf (key, "subvolumes[%d]", i);
+                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
+                                   conf->subvolumes[i]->name);
+                if (conf->file_layouts && conf->file_layouts[i]){
+                        sprintf (key, "file_layouts[%d]", i);
+                        dht_layout_dump(conf->file_layouts[i], key);
+                }
+                if (conf->dir_layouts && conf->dir_layouts[i]) {
+                        sprintf (key, "dir_layouts[%d]", i);
+                        dht_layout_dump(conf->dir_layouts[i], key);
+                }
+                if (conf->subvolume_status) {
+
+                        sprintf (key, "subvolume_status[%d]", i);
+                        gf_proc_dump_write(key, "%d",
+                                           (int)conf->subvolume_status[i]);
+                }
+
+        }
+
+        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
+        gf_proc_dump_write("gen", "%d", conf->gen);
+        gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
+	gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
+        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
+        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
+        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
+        if (conf ->du_stats) {
+                gf_proc_dump_write("du_stats.avail_percent", "%lf",
+                                   conf->du_stats->avail_percent);
+                gf_proc_dump_write("du_stats.avail_space", "%lu",
+                                   conf->du_stats->avail_space);
+		gf_proc_dump_write("du_stats.avail_inodes", "%lf",
+                                   conf->du_stats->avail_inodes);
+                gf_proc_dump_write("du_stats.log", "%lu", conf->du_stats->log);
+        }
+
+        if (conf->last_stat_fetch.tv_sec)
+                gf_proc_dump_write("last_stat_fetch", "%s",
+                                    ctime(&conf->last_stat_fetch.tv_sec));
+
+        UNLOCK(&conf->subvolume_lock);
+
+out:
+        return ret;
+}
+
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode)
+{
+        int             ret = -1;
+        dht_layout_t    *layout = NULL;
+
+        if (!this)
+                goto out;
+        if (!inode)
+                goto out;
+
+        ret = dht_inode_ctx_layout_get (inode, this, &layout);
+
+        if ((ret != 0) || !layout)
+                return ret;
+
+        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
+        dht_layout_dump(layout, "layout");
+
+out:
+        return ret;
+}
+
+void
+dht_fini (xlator_t *this)
+{
+        int         i = 0;
+        dht_conf_t *conf = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+        conf = this->private;
+        this->private = NULL;
+        if (conf) {
+                if (conf->file_layouts) {
+                        for (i = 0; i < conf->subvolume_cnt; i++) {
+                                GF_FREE (conf->file_layouts[i]);
+                        }
+                        GF_FREE (conf->file_layouts);
+                }
+
+                GF_FREE (conf->subvolumes);
+
+                GF_FREE (conf->subvolume_status);
+
+                GF_FREE (conf);
+        }
+out:
+        return;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        "failed");
+                return ret;
+        }
+out:
+        return ret;
+}
+
+
+int
+dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
+                                 const char *bricks)
+{
+        int         i  = 0;
+        int         ret  = -1;
+        char       *tmpstr = NULL;
+        char       *dup_brick = NULL;
+        char       *node = NULL;
+
+        if (!conf || !bricks)
+                goto out;
+
+        dup_brick = gf_strdup (bricks);
+        node = strtok_r (dup_brick, ",", &tmpstr);
+        while (node) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (!strcmp (conf->subvolumes[i]->name, node)) {
+                                conf->decommissioned_bricks[i] =
+                                        conf->subvolumes[i];
+                                        conf->decommission_subvols_cnt++;
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "decommissioning subvolume %s",
+                                        conf->subvolumes[i]->name);
+                                break;
+                        }
+                }
+                if (i == conf->subvolume_cnt) {
+                        /* Wrong node given. */
+                        goto out;
+                }
+                node = strtok_r (NULL, ",", &tmpstr);
+        }
+
+        ret = 0;
+        conf->decommission_in_progress = 1;
+out:
+        GF_FREE (dup_brick);
+
+        return ret;
+}
+
+
+int
+dht_decommissioned_remove (xlator_t *this, dht_conf_t *conf)
+{
+        int         i  = 0;
+        int         ret  = -1;
+
+        if (!conf)
+                goto out;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->decommissioned_bricks[i]) {
+                        conf->decommissioned_bricks[i] = NULL;
+                        conf->decommission_subvols_cnt--;
+                }
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+void
+dht_init_regex (xlator_t *this, dict_t *odict, char *name,
+                regex_t *re, gf_boolean_t *re_valid)
+{
+        char    *temp_str;
+
+        if (dict_get_str (odict, name, &temp_str) != 0) {
+                if (strcmp(name,"rsync-hash-regex")) {
+                        return;
+                }
+                temp_str = "^\\.(.+)\\.[^.]+$";
+        }
+
+        if (*re_valid) {
+                regfree(re);
+                *re_valid = _gf_false;
+        }
+
+        if (!strcmp(temp_str,"none")) {
+                return;
+        }
+
+        if (regcomp(re,temp_str,REG_EXTENDED) == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "using regex %s = %s", name, temp_str);
+                *re_valid = _gf_true;
+        }
+        else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "compiling regex %s failed", temp_str);
+        }
+}
+
+int
+dht_reconfigure (xlator_t *this, dict_t *options)
+{
+        dht_conf_t      *conf = NULL;
+        char            *temp_str = NULL;
+        gf_boolean_t     search_unhashed;
+        int              ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", options, out);
+
+        conf = this->private;
+        if (!conf)
+                return 0;
+
+        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
+                /* If option is not "auto", other options _should_ be boolean*/
+                if (strcasecmp (temp_str, "auto")) {
+                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
+                                gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
+                                       " lookup-unhashed reconfigured (%s)",
+                                       temp_str);
+                                conf->search_unhashed = search_unhashed;
+                        } else {
+                                gf_log(this->name, GF_LOG_ERROR, "Reconfigure:"
+                                       " lookup-unhashed should be boolean,"
+                                       " not (%s), defaulting to (%d)",
+                                       temp_str, conf->search_unhashed);
+                                //return -1;
+                                ret = -1;
+                                goto out;
+                        }
+                } else {
+                        gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
+                               " lookup-unhashed reconfigured auto ");
+                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+                }
+        }
+
+	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
+                          percent_or_size, out);
+        /* option can be any one of percent or bytes */
+        conf->disk_unit = 0;
+        if (conf->min_free_disk < 100.0)
+                conf->disk_unit = 'p';
+
+	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
+                          percent, out);
+
+        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
+                          options, uint32, out);
+
+        GF_OPTION_RECONF ("readdir-optimize", conf->readdir_optimize, options,
+                          bool, out);
+        if (conf->defrag) {
+                GF_OPTION_RECONF ("rebalance-stats", conf->defrag->stats,
+                                  options, bool, out);
+        }
+
+        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
+                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
+                if (ret == -1)
+                        goto out;
+        } else {
+                ret = dht_decommissioned_remove (this, conf);
+                if (ret == -1)
+                        goto out;
+        }
+
+        dht_init_regex (this, options, "rsync-hash-regex",
+                        &conf->rsync_regex, &conf->rsync_regex_valid);
+        dht_init_regex (this, options, "extra-hash-regex",
+                        &conf->extra_regex, &conf->extra_regex_valid);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+gf_defrag_pattern_list_fill (xlator_t *this, gf_defrag_info_t *defrag, char *data)
+{
+        int                    ret = -1;
+        char                  *tmp_str = NULL;
+        char                  *tmp_str1 = NULL;
+        char                  *dup_str = NULL;
+        char                  *num = NULL;
+        char                  *pattern_str = NULL;
+        char                  *pattern = NULL;
+        gf_defrag_pattern_list_t *temp_list = NULL;
+        gf_defrag_pattern_list_t *pattern_list = NULL;
+
+        if (!this || !defrag || !data)
+                goto out;
+
+        /* Get the pattern for pattern list. "pattern:<optional-size>"
+         * eg: *avi, *pdf:10MB, *:1TB
+         */
+        pattern_str = strtok_r (data, ",", &tmp_str);
+        while (pattern_str) {
+                dup_str = gf_strdup (pattern_str);
+                pattern_list = GF_CALLOC (1, sizeof (gf_defrag_pattern_list_t),
+                                        1);
+                if (!pattern_list) {
+                        goto out;
+                }
+                pattern = strtok_r (dup_str, ":", &tmp_str1);
+                num = strtok_r (NULL, ":", &tmp_str1);
+                if (!pattern)
+                        goto out;
+                if (!num) {
+                        if (gf_string2bytesize(pattern, &pattern_list->size)
+                             == 0) {
+                                pattern = "*";
+                        }
+                } else if (gf_string2bytesize (num, &pattern_list->size) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid number format \"%s\"", num);
+                        goto out;
+                }
+                memcpy (pattern_list->path_pattern, pattern, strlen (dup_str));
+
+                if (!defrag->defrag_pattern)
+                        temp_list = NULL;
+                else
+                        temp_list = defrag->defrag_pattern;
+
+                pattern_list->next = temp_list;
+
+                defrag->defrag_pattern = pattern_list;
+                pattern_list = NULL;
+
+                GF_FREE (dup_str);
+                dup_str = NULL;
+
+                pattern_str = strtok_r (NULL, ",", &tmp_str);
+        }
+
+        ret = 0;
+out:
+        if (ret)
+                GF_FREE (pattern_list);
+        GF_FREE (dup_str);
+
+        return ret;
+}
+
+int
+dht_init (xlator_t *this)
+{
+        dht_conf_t                      *conf           = NULL;
+        char                            *temp_str       = NULL;
+        int                              ret            = -1;
+        int                              i              = 0;
+        gf_defrag_info_t                *defrag         = NULL;
+        int                              cmd            = 0;
+        char                            *node_uuid      = NULL;
+
+
+        GF_VALIDATE_OR_GOTO ("dht", this, err);
+
+        if (!this->children) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Distribute needs more than one subvolume");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile");
+        }
+
+        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
+        if (!conf) {
+                goto err;
+        }
+
+        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
+
+        if (cmd) {
+                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
+                                    gf_defrag_info_mt);
+
+                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
+
+                LOCK_INIT (&defrag->lock);
+
+                defrag->is_exiting = 0;
+
+                conf->defrag = defrag;
+
+                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "node-uuid not "
+                                "specified");
+                        goto err;
+                }
+
+                if (uuid_parse (node_uuid, defrag->node_uuid)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot parse "
+                                "glusterd node uuid");
+                        goto err;
+                }
+
+                defrag->cmd = cmd;
+
+                defrag->stats = _gf_false;
+        }
+
+        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
+        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
+                /* If option is not "auto", other options _should_ be boolean */
+                if (strcasecmp (temp_str, "auto"))
+                        gf_string2boolean (temp_str, &conf->search_unhashed);
+                else
+                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        }
+
+        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
+                        err);
+
+        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
+
+	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
+			err);
+
+        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
+                        err);
+
+        conf->dir_spread_cnt = conf->subvolume_cnt;
+        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
+                        uint32, err);
+
+        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
+                        bool, err);
+
+        GF_OPTION_INIT ("readdir-optimize", conf->readdir_optimize, bool, err);
+
+        if (defrag) {
+                GF_OPTION_INIT ("rebalance-stats", defrag->stats, bool, err);
+                if (dict_get_str (this->options, "rebalance-filter", &temp_str)
+                    == 0) {
+                        if (gf_defrag_pattern_list_fill (this, defrag, temp_str)
+                            == -1) {
+                                gf_log (this->name, GF_LOG_ERROR, "Cannot parse"
+                                        " rebalance-filter (%s)", temp_str);
+                                goto err;
+                        }
+                }
+        }
+
+        /* option can be any one of percent or bytes */
+        conf->disk_unit = 0;
+        if (conf->min_free_disk < 100)
+                conf->disk_unit = 'p';
+
+        ret = dht_init_subvolumes (this, conf);
+        if (ret == -1) {
+                goto err;
+        }
+
+        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
+                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
+                if (ret == -1)
+                        goto err;
+        }
+
+        dht_init_regex (this, this->options, "rsync-hash-regex",
+                        &conf->rsync_regex, &conf->rsync_regex_valid);
+        dht_init_regex (this, this->options, "extra-hash-regex",
+                        &conf->extra_regex, &conf->extra_regex_valid);
+
+        ret = dht_layouts_init (this, conf);
+        if (ret == -1) {
+                goto err;
+        }
+
+        LOCK_INIT (&conf->subvolume_lock);
+        LOCK_INIT (&conf->layout_lock);
+
+        conf->gen = 1;
+
+        this->local_pool = mem_pool_new (dht_local_t, 512);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto err;
+        }
+
+        GF_OPTION_INIT ("xattr-name", conf->xattr_name, str, err);
+        gf_asprintf (&conf->link_xattr_name, "%s.linkto", conf->xattr_name);
+        gf_asprintf (&conf->wild_xattr_name, "%s*", conf->xattr_name);
+        if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+                goto err;
+        }
+
+        this->private = conf;
+
+        return 0;
+
+err:
+        if (conf) {
+                if (conf->file_layouts) {
+                        for (i = 0; i < conf->subvolume_cnt; i++) {
+                                GF_FREE (conf->file_layouts[i]);
+                        }
+                        GF_FREE (conf->file_layouts);
+                }
+
+                GF_FREE (conf->subvolumes);
+
+                GF_FREE (conf->subvolume_status);
+
+                GF_FREE (conf->du_stats);
+
+                GF_FREE (conf->defrag);
+
+                GF_FREE (conf->xattr_name);
+                GF_FREE (conf->link_xattr_name);
+                GF_FREE (conf->wild_xattr_name);
+
+                GF_FREE (conf);
+        }
+
+        return -1;
+}
+
+
+struct volume_options options[] = {
+        { .key  = {"lookup-unhashed"},
+          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
+                    "on", "off"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "on",
+          .description = "This option if set to ON, does a lookup through "
+          "all the sub-volumes, in case a lookup didn't return any result "
+          "from the hash subvolume. If set to OFF, it does not do a lookup "
+          "on the remaining subvolumes."
+        },
+        { .key  = {"min-free-disk"},
+          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
+          .default_value = "10%",
+          .description = "Percentage/Size of disk space, after which the "
+          "process starts balancing out the cluster, and logs will appear "
+          "in log files",
+        },
+	{ .key  = {"min-free-inodes"},
+          .type = GF_OPTION_TYPE_PERCENT,
+          .default_value = "5%",
+          .description = "after system has only N% of inodes, warnings "
+          "starts to appear in log files",
+        },
+        { .key = {"unhashed-sticky-bit"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+        { .key = {"use-readdirp"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "This option if set to ON, forces the use of "
+          "readdirp, and hence also displays the stats of the files."
+        },
+        { .key = {"assert-no-child-down"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON, in the event of "
+          "CHILD_DOWN, will call exit."
+        },
+        { .key  = {"directory-layout-spread"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 1,
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "Specifies the directory layout spread."
+        },
+        { .key  = {"decommissioned-bricks"},
+          .type = GF_OPTION_TYPE_ANY,
+          .description = "This option if set to ON, decommissions "
+          "the brick, so that no new data is allowed to be created "
+          "on that brick."
+        },
+        { .key  = {"rebalance-cmd"},
+          .type = GF_OPTION_TYPE_INT,
+        },
+        { .key = {"node-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+        { .key = {"rebalance-stats"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON displays and logs the "
+          " time taken for migration of each file, during the rebalance "
+          "process. If set to OFF, the rebalance logs will only display the "
+          "time spent in each directory."
+        },
+        { .key = {"readdir-optimize"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON enables the optimization "
+          "that allows DHT to requests non-first subvolumes to filter out "
+          "directory entries."
+        },
+        { .key = {"rsync-hash-regex"},
+          .type = GF_OPTION_TYPE_STR,
+          /* Setting a default here doesn't work.  See dht_init_regex. */
+          .description = "Regular expression for stripping temporary-file "
+          "suffix and prefix used by rsync, to prevent relocation when the "
+          "file is renamed."
+        },
+        { .key = {"extra-hash-regex"},
+          .type = GF_OPTION_TYPE_STR,
+          /* Setting a default here doesn't work.  See dht_init_regex. */
+          .description = "Regular expression for stripping temporary-file "
+          "suffix and prefix used by an application, to prevent relocation when "
+          "the file is renamed."
+        },
+        { .key = {"rebalance-filter"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
+        { .key = {"xattr-name"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "trusted.glusterfs.dht",
+          .description = "Base for extended attributes used by this "
+          "translator instance, to avoid conflicts with others above or "
+          "below it."
+        },
+
+        /* NUFA option */
+        { .key  = {"local-volume-name"},
+          .type = GF_OPTION_TYPE_XLATOR
+        },
+
+        /* switch option */
+        { .key  = {"pattern.switch.case"},
+          .type = GF_OPTION_TYPE_ANY
+        },
+
+        { .key  = {NULL} },
+};
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index 18fee7cd3..fc0ca2f77 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -23,434 +14,15 @@
 #include "config.h"
 #endif
 
-/* TODO: add NS locking */
-
 #include "statedump.h"
 #include "dht-common.h"
 
-/* TODO:
-   - use volumename in xattr instead of "dht"
-   - use NS locks
-   - handle all cases in self heal layout reconstruction
-   - complete linkfile selfheal
-*/
-struct volume_options options[];
-
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix)
-{
-
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     i = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", layout, out);
-        GF_VALIDATE_OR_GOTO ("dht", prefix, out);
-
-        gf_proc_dump_build_key(key, prefix, "cnt");
-        gf_proc_dump_write(key, "%d", layout->cnt);
-        gf_proc_dump_build_key(key, prefix, "preset");
-        gf_proc_dump_write(key, "%d", layout->preset);
-        gf_proc_dump_build_key(key, prefix, "gen");
-        gf_proc_dump_write(key, "%d", layout->gen);
-        gf_proc_dump_build_key(key, prefix, "type");
-        gf_proc_dump_write(key, "%d", layout->type);
-
-        for (i = 0; i < layout->cnt; i++) {
-                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
-                gf_proc_dump_write(key, "%d", layout->list[i].err);
-                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].start);
-                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].stop);
-                if (layout->list[i].xlator) {
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.type", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->type);
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.name", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->name);
-                }
-        }
-
-out:
-        return;
-}
-
-
-int32_t
-dht_priv_dump (xlator_t *this)
-{
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char            key[GF_DUMP_MAX_BUF_LEN];
-        int             i = 0;
-        dht_conf_t      *conf = NULL;
-        int             ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        conf = this->private;
-
-        if (!conf)
-                return -1;
-
-        ret = TRY_LOCK(&conf->subvolume_lock);
-
-        if (ret != 0) {
-                gf_log("", GF_LOG_WARNING, "Unable to lock dht subvolume %s",
-                       this->name);
-                return ret;
-        }
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
-                               this->name);
-        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                sprintf (key, "subvolumes[%d]", i);
-                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
-                                   conf->subvolumes[i]->name);
-                if (conf->file_layouts && conf->file_layouts[i]){
-                        sprintf (key, "file_layouts[%d]", i);
-                        dht_layout_dump(conf->file_layouts[i], key);
-                }
-                if (conf->dir_layouts && conf->dir_layouts[i]) {
-                        sprintf (key, "dir_layouts[%d]", i);
-                        dht_layout_dump(conf->dir_layouts[i], key);
-                }
-                if (conf->subvolume_status) {
-
-                        sprintf (key, "subvolume_status[%d]", i);
-                        gf_proc_dump_write(key, "%d",
-                                           (int)conf->subvolume_status[i]);
-                }
-
-        }
-
-        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
-        gf_proc_dump_write("gen", "%d", conf->gen);
-        gf_proc_dump_write("min_free_disk", "%lu", conf->min_free_disk);
-	gf_proc_dump_write("min_free_inodes", "%lu", conf->min_free_inodes);
-        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
-        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
-        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
-        if (conf ->du_stats) {
-                gf_proc_dump_write("du_stats.avail_percent", "%lf",
-                                   conf->du_stats->avail_percent);
-                gf_proc_dump_write("du_stats.avail_space", "%lu",
-                                   conf->du_stats->avail_space);
-		gf_proc_dump_write("du_stats.avail_inodes", "%lf",
-                                   conf->du_stats->avail_inodes);
-                gf_proc_dump_write("du_stats.log", "%lu", conf->du_stats->log);
-        }
-        gf_proc_dump_write("last_stat_fetch", "%s", ctime(&conf->last_stat_fetch.tv_sec));
-
-        UNLOCK(&conf->subvolume_lock);
-
-out:
-        return ret;
-}
-
-int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode)
-{
-        int             ret = -1;
-        dht_layout_t    *layout = NULL;
-        uint64_t        tmp_layout = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", inode, out);
-
-        ret = inode_ctx_get (inode, this, &tmp_layout);
-
-        if (ret != 0)
-                return ret;
-
-        layout = (dht_layout_t *)(long)tmp_layout;
-
-        if (!layout)
-                return -1;
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
-        dht_layout_dump(layout, "layout");
-
-out:
-        return ret;
-}
-
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        int ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        ret = dht_notify (this, event, data);
-
-out:
-        return ret;
-}
-
-void
-fini (xlator_t *this)
-{
-        int         i = 0;
-        dht_conf_t *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        conf = this->private;
-        this->private = NULL;
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf);
-        }
-out:
-        return;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-int
-dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
-                                 const char *bricks)
-{
-        int         i  = 0;
-        int         ret  = -1;
-        char       *tmpstr = NULL;
-        char       *dup_brick = NULL;
-        char       *node = NULL;
-
-        if (!conf || !bricks)
-                goto out;
-
-        dup_brick = gf_strdup (bricks);
-        node = strtok_r (dup_brick, ",", &tmpstr);
-        while (node) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (!strcmp (conf->subvolumes[i]->name, node)) {
-                                conf->decommissioned_bricks[i] =
-                                        conf->subvolumes[i];
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "decommissioning subvolume %s",
-                                        conf->subvolumes[i]->name);
-                                break;
-                        }
-                }
-                if (i == conf->subvolume_cnt) {
-                        /* Wrong node given. */
-                        goto out;
-                }
-                node = strtok_r (NULL, ",", &tmpstr);
-        }
-
-        ret = 0;
-out:
-        if (dup_brick)
-                GF_FREE (dup_brick);
-
-        return ret;
-}
-
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        dht_conf_t      *conf = NULL;
-        char            *temp_str = NULL;
-        gf_boolean_t     search_unhashed;
-        int              ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", options, out);
-
-        conf = this->private;
-        if (!conf)
-                return 0;
-
-        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean*/
-                if (strcasecmp (temp_str, "auto")) {
-                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
-                                gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-                                       " lookup-unhashed reconfigured (%s)",
-                                       temp_str);
-                                conf->search_unhashed = search_unhashed;
-                        } else {
-                                gf_log(this->name, GF_LOG_ERROR, "Reconfigure:"
-                                       " lookup-unhashed should be boolean,"
-                                       " not (%s), defaulting to (%d)",
-                                       temp_str, conf->search_unhashed);
-                                //return -1;
-                                ret = -1;
-                                goto out;
-                        }
-                } else {
-                        gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-                               " lookup-unhashed reconfigured auto ");
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-                }
-        }
-
-	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
-                          percent_or_size, out);
-	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
-                          percent, out);
-        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
-                          options, uint32, out);
-
-        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-int
-init (xlator_t *this)
-{
-        dht_conf_t    *conf = NULL;
-        char          *temp_str = NULL;
-        int            ret = -1;
-        int            i = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Distribute needs more than one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
-
-        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
-                        err);
-
-        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
-
-	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
-			err);
-
-        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
-                        err);
-
-        conf->dir_spread_cnt = conf->subvolume_cnt;
-        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
-                        uint32, err);
-
-        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
-                        bool, err);
-
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
-
-        conf->gen = 1;
-
-        /* Create 'syncop' environment */
-	conf->env = syncenv_new (0);
-        if (!conf->env) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create sync environment %s",
-                        strerror (errno));
-                goto err;
-        }
-
-        this->private = conf;
-
-        return 0;
-
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
-
-        return -1;
-}
-
+class_methods_t class_methods = {
+        .init           = dht_init,
+        .fini           = dht_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
 
 struct xlator_fops fops = {
         .lookup      = dht_lookup,
@@ -498,6 +70,9 @@ struct xlator_fops fops = {
         .fxattrop    = dht_fxattrop,
         .setattr     = dht_setattr,
         .fsetattr    = dht_fsetattr,
+	.fallocate   = dht_fallocate,
+	.discard     = dht_discard,
+        .zerofill    = dht_zerofill,
 };
 
 struct xlator_dumpops dumpops = {
@@ -511,44 +86,4 @@ struct xlator_cbks cbks = {
 //      .releasedir = dht_releasedir,
         .forget     = dht_forget
 };
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "on",
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-          .default_value = "10%",
-          .description = "Percentage/Size of disk space that must be "
-                         "kept free."
-        },
-	{ .key  = {"min-free-inodes"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "5%",
-          .description = "Percentage inodes that must be "
-                         "kept free."
-        },
-        { .key = {"unhashed-sticky-bit"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"use-readdirp"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key = {"assert-no-child-down"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key  = {"directory-layout-spread"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key  = {"decommissioned-bricks"},
-          .type = GF_OPTION_TYPE_ANY,
-        },
-        { .key  = {NULL} },
-};
+;
diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c
index 2f196951a..e934acdf0 100644
--- a/xlators/cluster/dht/src/nufa.c
+++ b/xlators/cluster/dht/src/nufa.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -27,6 +18,8 @@
 
 /* TODO: all 'TODO's in dht.c holds good */
 
+extern struct volume_options options[];
+
 int
 nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno,
@@ -44,7 +37,6 @@ nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int           call_cnt    = 0;
         int           ret         = 0;
 
-
         conf  = this->private;
 
         prev  = cookie;
@@ -62,7 +54,8 @@ nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret == -1)
                 goto out;
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
         is_dir      = check_is_dir (inode, stbuf, xattr);
 
         if (!is_dir && !is_linkfile) {
@@ -141,7 +134,7 @@ out:
 
 err:
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, NULL);
+                          inode, stbuf, xattr, postparent);
         return 0;
 }
 
@@ -211,7 +204,7 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
                  *       revalidates directly go to the cached-subvolume.
                  */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -232,7 +225,7 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
         } else {
         do_fresh_lookup:
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -241,7 +234,7 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
                 }
 
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht.linkto", 256);
+                                       conf->link_xattr_name, 256);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -260,7 +253,8 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
         return 0;
 }
 
@@ -269,7 +263,7 @@ nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int op_ret, int op_errno,
                                  inode_t *inode, struct iatt *stbuf,
                                  struct iatt *preparent,
-                                 struct iatt *postparent)
+                                 struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
@@ -280,21 +274,21 @@ nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
 
         STACK_WIND (frame, dht_create_cbk,
                     local->cached_subvol, local->cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode, local->fd,
-                    local->params);
+                    &local->loc, local->flags, local->mode, local->umask,
+                    local->fd, local->params);
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
 int
 nufa_create (call_frame_t *frame, xlator_t *this,
              loc_t *loc, int32_t flags, mode_t mode,
-             fd_t *fd, dict_t *params)
+             mode_t umask, fd_t *fd, dict_t *params)
 {
         dht_local_t *local = NULL;
         dht_conf_t  *conf  = NULL;
@@ -329,7 +323,8 @@ nufa_create (call_frame_t *frame, xlator_t *this,
         if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
                 avail_subvol =
                         dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private);
+                                                        (xlator_t *)conf->private,
+                                                        local);
         }
 
         if (subvol != avail_subvol) {
@@ -337,11 +332,10 @@ nufa_create (call_frame_t *frame, xlator_t *this,
                 local->params = dict_ref (params);
                 local->mode = mode;
                 local->flags = flags;
-
+                local->umask = umask;
                 local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame,
-                                     nufa_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, nufa_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
@@ -350,14 +344,14 @@ nufa_create (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_create_cbk,
                     subvol, subvol->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -366,34 +360,39 @@ int
 nufa_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int op_ret, int op_errno, inode_t *inode,
                          struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
         local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto err;
+        }
 
         if (op_ret >= 0) {
-                STACK_WIND (frame, dht_newfile_cbk,
-                            local->cached_subvol,
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                            (void *)local->cached_subvol, local->cached_subvol,
                             local->cached_subvol->fops->mknod,
                             &local->loc, local->mode, local->rdev,
-                            local->params);
+                            local->umask, local->params);
 
                 return 0;
         }
-
+err:
         WIPE (postparent);
         WIPE (preparent);
 
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent);
+                          inode, stbuf, preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 nufa_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
 {
         dht_local_t *local = NULL;
         dht_conf_t  *conf  = NULL;
@@ -429,7 +428,8 @@ nufa_mknod (call_frame_t *frame, xlator_t *this,
         if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
                 avail_subvol =
                         dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private);
+                                                        (xlator_t *)conf->private,
+                                                        local);
         }
 
         if (avail_subvol != subvol) {
@@ -437,10 +437,11 @@ nufa_mknod (call_frame_t *frame, xlator_t *this,
 
                 local->params = dict_ref (params);
                 local->mode = mode;
+                local->umask = umask;
                 local->rdev = rdev;
                 local->cached_subvol = avail_subvol;
 
-                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk,
+                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk, this,
                                      avail_subvol, subvol, loc);
                 return 0;
         }
@@ -448,211 +449,185 @@ nufa_mknod (call_frame_t *frame, xlator_t *this,
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->mknod,
-                    loc, mode, rdev, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->mknod, loc, mode, rdev, umask,
+                           params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
 
 
-int
-notify (xlator_t *this, int event, void *data, ...)
+gf_boolean_t
+same_first_part (char *str1, char term1, char *str2, char term2)
 {
-        int ret = -1;
-
-        ret = dht_notify (this, event, data);
-
-        return ret;
+        gf_boolean_t    ended1;
+        gf_boolean_t    ended2;
+
+        for (;;) {
+                ended1 = ((*str1 == '\0') || (*str1 == term1));
+                ended2 = ((*str2 == '\0') || (*str2 == term2));
+                if (ended1 && ended2) {
+                        return _gf_true;
+                }
+                if (ended1 || ended2 || (*str1 != *str2)) {
+                        return _gf_false;
+                }
+                ++str1;
+                ++str2;
+        }
 }
 
-void
-fini (xlator_t *this)
-{
-        int         i = 0;
-        dht_conf_t *conf = NULL;
+typedef struct nufa_args {
+        xlator_t    *this;
+        char        *volname;
+        gf_boolean_t addr_match;
+} nufa_args_t;
 
-        conf = this->private;
+static void
+nufa_find_local_brick (xlator_t *xl, void *data)
+{
+        nufa_args_t     *args = data;
+        xlator_t        *this = args->this;
+        char            *local_volname = args->volname;
+        gf_boolean_t    addr_match = args->addr_match;
+        char            *brick_host = NULL;
+        dht_conf_t      *conf = this->private;
+        int             ret = -1;
+
+        /*This means a local subvol was already found. We pick the first brick
+         * that is local*/
+        if (conf->private)
+                return;
+
+        if (strcmp (xl->name, local_volname) == 0) {
+                conf->private = xl;
+                gf_log (this->name, GF_LOG_INFO, "Using specified subvol %s",
+                        local_volname);
+                return;
+        }
 
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
+        if (!addr_match)
+                return;
 
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
+        ret = dict_get_str (xl->options, "remote-host", &brick_host);
+        if ((ret == 0) &&
+            (gf_is_same_address (local_volname, brick_host) ||
+             gf_is_local_addr (brick_host))) {
+                conf->private = xl;
+                gf_log (this->name, GF_LOG_INFO, "Using the first local "
+                        "subvol %s", xl->name);
+                return;
+        }
 
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
+}
 
-                GF_FREE (conf);
-        }
+static void
+nufa_to_dht (xlator_t *this)
+{
+        GF_ASSERT (this);
+        GF_ASSERT (this->fops);
 
-        return;
+        this->fops->lookup = dht_lookup;
+        this->fops->create = dht_create;
+        this->fops->mknod  = dht_mknod;
 }
 
 int
-init (xlator_t *this)
+nufa_find_local_subvol (xlator_t *this,
+                        void (*fn) (xlator_t *each, void* data), void *data)
 {
-        dht_conf_t    *conf = NULL;
-        xlator_list_t *trav = NULL;
-        data_t        *data = NULL;
-        char          *local_volname = NULL;
-        char          *temp_str = NULL;
-        int            ret = -1;
-        int            i = 0;
-        char           my_hostname[256];
-        uint32_t       temp_free_disk = 0;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "NUFA needs more than one subvolume");
+        int             ret = -1;
+        dht_conf_t      *conf = this->private;
+        xlator_list_t   *trav = NULL;
+        xlator_t        *parent = NULL;
+        xlator_t        *candidate = NULL;
+
+        xlator_foreach_depth_first (this, fn, data);
+        if (!conf->private) {
+                gf_log (this->name, GF_LOG_ERROR, "Couldn't find a local "
+                        "brick");
                 return -1;
         }
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf),
-                          gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
+        candidate = conf->private;
+        trav = candidate->parents;
+        while (trav) {
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
+                parent = trav->xlator;
+                if (strcmp (parent->type, "cluster/nufa") == 0) {
+                        gf_log (this->name, GF_LOG_INFO, "Found local subvol, "
+                                "%s", candidate->name);
+                        ret = 0;
+                        conf->private = candidate;
+                        break;
+                }
 
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
+                candidate = parent;
+                trav = parent->parents;
         }
 
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
+        return ret;
+}
 
-        conf->gen = 1;
+int
+nufa_init (xlator_t *this)
+{
+        data_t        *data = NULL;
+        char          *local_volname = NULL;
+        int            ret = -1;
+        char           my_hostname[256];
+        gf_boolean_t   addr_match = _gf_false;
+        nufa_args_t    args = {0, };
 
-        local_volname = "localhost";
-        ret = gethostname (my_hostname, 256);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "could not find hostname (%s)",
-                        strerror (errno));
+        ret = dht_init(this);
+        if (ret) {
+                return ret;
         }
 
-        if (ret == 0)
-                local_volname = my_hostname;
-
-        data = dict_get (this->options, "local-volume-name");
-        if (data) {
+        if ((data = dict_get (this->options, "local-volume-name"))) {
                 local_volname = data->data;
-        }
 
-        trav = this->children;
-        while (trav) {
-                if (strcmp (trav->xlator->name, local_volname) == 0)
-                        break;
-                trav = trav->next;
-        }
+        } else {
+                addr_match = _gf_true;
+                local_volname = "localhost";
+                ret = gethostname (my_hostname, 256);
+                if (ret == 0)
+                        local_volname = my_hostname;
 
-        if (!trav) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not find subvolume named '%s'. "
-                        "Please define volume with the name as the hostname "
-                        "or override it with 'option local-volume-name'",
-                        local_volname);
-                goto err;
-        }
-        /* The volume specified exists */
-        conf->private = trav->xlator;
-
-        conf->min_free_disk = 10;
-        conf->disk_unit = 'p';
-
-        if (dict_get_str (this->options, "min-free-disk",
-                          &temp_str) == 0) {
-                if (gf_string2percent (temp_str,
-                                       &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str,
-                                                    &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                                conf->disk_unit = 'p';
-                        }
-                } else {
-                        gf_string2bytesize (temp_str,
-                                            &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
-        }
+                else
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "could not find hostname (%s)",
+                                strerror (errno));
 
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_dht_mt_dht_du_t);
-        if (!conf->du_stats) {
-                goto err;
         }
 
-        /* Create 'syncop' environment */
-	conf->env = syncenv_new (0);
-        if (!conf->env) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create sync environment %s",
-                        strerror (errno));
-                goto err;
+        args.this = this;
+        args.volname = local_volname;
+        args.addr_match = addr_match;
+        ret = nufa_find_local_subvol (this, nufa_find_local_brick, &args);
+        if (ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Unable to find local subvolume, switching "
+                        "to dht mode");
+                nufa_to_dht (this);
         }
-
-        this->private = conf;
-
         return 0;
+}
 
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
 
-        return -1;
-}
+class_methods_t class_methods = {
+        .init           = nufa_init,
+        .fini           = dht_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
 
 
 struct xlator_fops fops = {
@@ -699,19 +674,3 @@ struct xlator_fops fops = {
 struct xlator_cbks cbks = {
         .forget     = dht_forget
 };
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key  = {"local-volume-name"},
-          .type = GF_OPTION_TYPE_XLATOR
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-        },
-        { .key  = {NULL} },
-};
diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c
index fd3f22ea0..d3ea90ba8 100644
--- a/xlators/cluster/dht/src/switch.c
+++ b/xlators/cluster/dht/src/switch.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -31,6 +22,8 @@
 #include <fnmatch.h>
 #include <string.h>
 
+extern struct volume_options options[];
+
 struct switch_sched_array {
         xlator_t *xl;
         int32_t   eligible;
@@ -76,29 +69,37 @@ get_switch_matching_subvol (const char *path, dht_conf_t *conf,
         struct switch_struct *cond      = NULL;
         struct switch_struct *trav      = NULL;
         char                 *pathname  = NULL;
-        int                   idx     = 0;
+        int                   idx       = 0;
+        xlator_t             *subvol    = NULL;
 
         cond = conf->private;
+        subvol = hashed_subvol;
         if (!cond)
-                return hashed_subvol;
+                goto out;
 
-        trav = cond;
         pathname = gf_strdup (path);
+        if (!pathname)
+                goto out;
+
+        trav = cond;
         while (trav) {
                 if (fnmatch (trav->path_pattern,
                              pathname, FNM_NOESCAPE) == 0) {
                         for (idx = 0; idx < trav->num_child; idx++) {
                                 if (trav->array[idx].xl == hashed_subvol)
-                                        return hashed_subvol;
+                                        goto out;
                         }
                         idx = trav->node_index++;
                         trav->node_index %= trav->num_child;
-                        return trav->array[idx].xl;
+                        subvol = trav->array[idx].xl;
+                        goto out;
                 }
                 trav = trav->next;
         }
+out:
         GF_FREE (pathname);
-        return hashed_subvol;
+
+        return subvol;
 }
 
 
@@ -136,7 +137,8 @@ switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret == -1)
                 goto out;
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
         is_dir      = check_is_dir (inode, stbuf, xattr);
 
         if (!is_dir && !is_linkfile) {
@@ -290,11 +292,11 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
                  * attribute, revalidates directly go to the cached-subvolume.
                  */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht");
+                                "failed to set dict value for %s",
+                                conf->xattr_name);
 
                 for (i = 0; i < layout->cnt; i++) {
                         subvol = layout->list[i].xlator;
@@ -309,18 +311,18 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
         } else {
         do_fresh_lookup:
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht");
+                                "failed to set dict value for %s",
+                                conf->xattr_name);
 
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht.linkto", 256);
+                                       conf->link_xattr_name, 256);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht.linkto");
+                                "failed to set dict value for %s",
+                                conf->link_xattr_name);
 
                 if (!hashed_subvol) {
                         gf_log (this->name, GF_LOG_DEBUG,
@@ -366,7 +368,8 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -375,7 +378,7 @@ switch_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                    xlator_t *this, int op_ret, int op_errno,
                                    inode_t *inode, struct iatt *stbuf,
                                    struct iatt *preparent,
-                                   struct iatt *postparent)
+                                   struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
@@ -386,21 +389,21 @@ switch_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
 
         STACK_WIND (frame, dht_create_cbk,
                     local->cached_subvol, local->cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode, local->fd,
-                    local->params);
+                    &local->loc, local->flags, local->mode, local->umask,
+                    local->fd, local->params);
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
 int
 switch_create (call_frame_t *frame, xlator_t *this,
                loc_t *loc, int32_t flags, mode_t mode,
-               fd_t *fd, dict_t *params)
+               mode_t umask, fd_t *fd, dict_t *params)
 {
         dht_local_t *local = NULL;
         dht_conf_t  *conf  = NULL;
@@ -434,18 +437,18 @@ switch_create (call_frame_t *frame, xlator_t *this,
         avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
         if (dht_is_subvol_filled (this, avail_subvol)) {
                 avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol);
+                        dht_free_disk_available_subvol (this, avail_subvol,
+                                                        local);
         }
 
         if (subvol != avail_subvol) {
                 /* create a link file instead of actual file */
                 local->mode = mode;
                 local->flags = flags;
-
+                local->umask = umask;
                 local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame,
-                                     switch_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, switch_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
@@ -454,14 +457,14 @@ switch_create (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_create_cbk,
                     subvol, subvol->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -470,31 +473,36 @@ int
 switch_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            int op_ret, int op_errno, inode_t *inode,
                            struct iatt *stbuf, struct iatt *preparent,
-                           struct iatt *postparent)
+                           struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
         local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto err;
+        }
 
         if (op_ret >= 0) {
-                STACK_WIND (frame, dht_newfile_cbk,
-                            local->cached_subvol,
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                            (void *)local->cached_subvol, local->cached_subvol,
                             local->cached_subvol->fops->mknod,
                             &local->loc, local->mode, local->rdev,
-                            local->params);
+                            local->umask, local->params);
 
                 return 0;
         }
-
+err:
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent);
+                          inode, stbuf, preparent, postparent, xdata);
         return 0;
 }
 
 
 int
-switch_mknod (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *params)
 {
         dht_local_t *local = NULL;
         dht_conf_t  *conf  = NULL;
@@ -529,7 +537,8 @@ switch_mknod (call_frame_t *frame, xlator_t *this,
         avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
         if (dht_is_subvol_filled (this, avail_subvol)) {
                 avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol);
+                        dht_free_disk_available_subvol (this, avail_subvol,
+                                                        local);
         }
 
         if (avail_subvol != subvol) {
@@ -537,46 +546,36 @@ switch_mknod (call_frame_t *frame, xlator_t *this,
 
                 local->params = dict_ref (params);
                 local->mode = mode;
+                local->umask = umask;
                 local->rdev = rdev;
                 local->cached_subvol = avail_subvol;
 
                 dht_linkfile_create (frame, switch_mknod_linkfile_cbk,
-                                     avail_subvol, subvol, loc);
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->mknod,
-                    loc, mode, rdev, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->mknod, loc, mode, rdev, umask,
+                           params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
 
 
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        int ret = -1;
-
-        ret = dht_notify (this, event, data);
-
-        return ret;
-}
-
 void
-fini (xlator_t *this)
+switch_fini (xlator_t *this)
 {
-        int                   i = 0;
         dht_conf_t           *conf = NULL;
         struct switch_struct *trav = NULL;
         struct switch_struct *prev = NULL;
@@ -587,30 +586,14 @@ fini (xlator_t *this)
                 trav = (struct switch_struct *)conf->private;
                 conf->private = NULL;
                 while (trav) {
-                        if (trav->array)
-                                GF_FREE (trav->array);
+                        GF_FREE (trav->array);
                         prev = trav;
                         trav = trav->next;
                         GF_FREE (prev);
                 }
-
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf);
         }
 
-        return;
+        dht_fini(this);
 }
 
 int
@@ -670,8 +653,10 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                 dup_str = gf_strdup (switch_str);
                 switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
                                         gf_switch_mt_switch_struct);
-                if (!switch_opt)
+                if (!switch_opt) {
+                        GF_FREE (dup_str);
                         goto err;
+                }
 
                 pattern = strtok_r (dup_str, ":", &tmp_str1);
                 childs = strtok_r (NULL, ":", &tmp_str1);
@@ -681,6 +666,7 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                                 "for all the unconfigured child nodes,"
                                 " hence neglecting current option");
                         switch_str = strtok_r (NULL, ";", &tmp_str);
+                        GF_FREE (switch_opt);
                         GF_FREE (dup_str);
                         continue;
                 }
@@ -753,6 +739,7 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                         /* First entry */
                         switch_buf = switch_opt;
                 }
+                switch_opt = NULL;
                 switch_str = strtok_r (NULL, ";", &tmp_str);
         }
 
@@ -809,19 +796,20 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                         /* First entry */
                         switch_buf = switch_opt;
                 }
+                switch_opt = NULL;
         }
         /* */
         conf->private = switch_buf;
 
         return 0;
 err:
+        GF_FREE (switch_buf_array);
+        GF_FREE (switch_opt);
+
         if (switch_buf) {
-                if (switch_buf_array)
-                        GF_FREE (switch_buf_array);
                 trav = switch_buf;
                 while (trav) {
-                        if (trav->array)
-                                GF_FREE (trav->array);
+                        GF_FREE (trav->array);
                         switch_opt = trav;
                         trav = trav->next;
                         GF_FREE (switch_opt);
@@ -831,68 +819,18 @@ err:
 }
 
 
-int
-init (xlator_t *this)
+int32_t
+switch_init (xlator_t *this)
 {
         dht_conf_t            *conf = NULL;
         data_t                *data = NULL;
-        char                  *temp_str = NULL;
         int                    ret = -1;
-        int                    i = 0;
-        uint32_t               temp_free_disk = 0;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "SWITCH needs more than one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_switch_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
 
-        conf->unhashed_sticky_bit = 0;
-        if (dict_get_str (this->options, "unhashed-sticky-bit",
-                          &temp_str) == 0) {
-                gf_string2boolean (temp_str, &conf->unhashed_sticky_bit);
-        }
-
-        conf->min_free_disk = 10;
-        conf->disk_unit = 'p';
-
-        if (dict_get_str (this->options, "min-free-disk",
-                          &temp_str) == 0) {
-                if (gf_string2percent (temp_str,
-                                       &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str,
-                                                    &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                                conf->disk_unit = 'p';
-                        }
-                } else {
-                        gf_string2bytesize (temp_str,
-                                            &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
+        ret = dht_init(this);
+        if (ret) {
+                return ret;
         }
+        conf = this->private;
 
         data = dict_get (this->options, "pattern.switch.case");
         if (data) {
@@ -903,65 +841,23 @@ init (xlator_t *this)
                 }
         }
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
-
-        conf->gen = 1;
-
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_switch_mt_dht_du_t);
-        if (!conf->du_stats) {
-                goto err;
-        }
-
-        /* Create 'syncop' environment */
-	conf->env = syncenv_new (0);
-        if (!conf->env) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create sync environment %s",
-                        strerror (errno));
-                goto err;
-        }
-
         this->private = conf;
-
         return 0;
 
 err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
-
+        dht_fini(this);
         return -1;
 }
 
 
+class_methods_t class_methods = {
+        .init           = switch_init,
+        .fini           = switch_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
+
+
 struct xlator_fops fops = {
         .lookup      = switch_lookup,
         .create      = switch_create,
@@ -1006,19 +902,3 @@ struct xlator_fops fops = {
 struct xlator_cbks cbks = {
         .forget     = dht_forget
 };
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key  = {"pattern.switch.case"},
-          .type = GF_OPTION_TYPE_ANY
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-        },
-        { .key  = {NULL} },
-};
diff --git a/xlators/cluster/ha/src/Makefile.am b/xlators/cluster/ha/src/Makefile.am
index 5f78a2965..5c1364b7f 100644
--- a/xlators/cluster/ha/src/Makefile.am
+++ b/xlators/cluster/ha/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = ha.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
 
-ha_la_LDFLAGS = -module -avoidversion 
+ha_la_LDFLAGS = -module -avoid-version 
 
 ha_la_SOURCES = ha-helpers.c ha.c
 ha_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = ha.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/ha/src/ha-helpers.c b/xlators/cluster/ha/src/ha-helpers.c
index 1e4af1b62..19be1ed27 100644
--- a/xlators/cluster/ha/src/ha-helpers.c
+++ b/xlators/cluster/ha/src/ha-helpers.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include "xlator.h"
 #include "call-stub.h"
 #include "defaults.h"
diff --git a/xlators/cluster/ha/src/ha-mem-types.h b/xlators/cluster/ha/src/ha-mem-types.h
index 9bfb3972b..e5e97d237 100644
--- a/xlators/cluster/ha/src/ha-mem-types.h
+++ b/xlators/cluster/ha/src/ha-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __HA_MEM_TYPES_H__
 #define __HA_MEM_TYPES_H__
 
diff --git a/xlators/cluster/ha/src/ha.c b/xlators/cluster/ha/src/ha.c
index 38d4229d3..3eccb516b 100644
--- a/xlators/cluster/ha/src/ha.c
+++ b/xlators/cluster/ha/src/ha.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 /* generate errors randomly, code is simple now, better alogorithm
  * can be written to decide what error to be returned and when
  */
@@ -1876,13 +1866,9 @@ err:
         }
 
         if (hafdp) {
-                if (hafdp->fdstate) {
-                        GF_FREE (hafdp->fdstate);
-                }
+                GF_FREE (hafdp->fdstate);
 
-                if (hafdp->path) {
-                        GF_FREE (hafdp->path);
-                }
+                GF_FREE (hafdp->path);
 
                 GF_FREE (hafdp);
         }
diff --git a/xlators/cluster/ha/src/ha.h b/xlators/cluster/ha/src/ha.h
index 39b6851e7..e2ed7eaa6 100644
--- a/xlators/cluster/ha/src/ha.h
+++ b/xlators/cluster/ha/src/ha.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __HA_H_
 #define __HA_H_
 
diff --git a/xlators/cluster/map/src/Makefile.am b/xlators/cluster/map/src/Makefile.am
index 26e19137a..a278b05e2 100644
--- a/xlators/cluster/map/src/Makefile.am
+++ b/xlators/cluster/map/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = map.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
 
-map_la_LDFLAGS = -module -avoidversion 
+map_la_LDFLAGS = -module -avoid-version 
 
 map_la_SOURCES = map.c map-helper.c
 map_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = map.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/map/src/map-helper.c b/xlators/cluster/map/src/map-helper.c
index 81212fcfd..851397b68 100644
--- a/xlators/cluster/map/src/map-helper.c
+++ b/xlators/cluster/map/src/map-helper.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2009-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/cluster/map/src/map-mem-types.h b/xlators/cluster/map/src/map-mem-types.h
index 669b93dc2..3e89f4736 100644
--- a/xlators/cluster/map/src/map-mem-types.h
+++ b/xlators/cluster/map/src/map-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __MAP_MEM_TYPES_H__
 #define __MAP_MEM_TYPES_H__
 
diff --git a/xlators/cluster/map/src/map.c b/xlators/cluster/map/src/map.c
index ead9da0b9..6150a33ce 100644
--- a/xlators/cluster/map/src/map.c
+++ b/xlators/cluster/map/src/map.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -2375,8 +2365,7 @@ fini (xlator_t *this)
 	priv = this->private;
 
 	if (priv) {
-		if (priv->xlarray)
-			GF_FREE (priv->xlarray);
+		GF_FREE (priv->xlarray);
 
 		trav_map = priv->map;
 		while (trav_map) {
diff --git a/xlators/cluster/map/src/map.h b/xlators/cluster/map/src/map.h
index bccac437c..7703a543e 100644
--- a/xlators/cluster/map/src/map.h
+++ b/xlators/cluster/map/src/map.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __MAP_H__
 #define __MAP_H__
 
diff --git a/xlators/cluster/stripe/src/Makefile.am b/xlators/cluster/stripe/src/Makefile.am
index 0db3c9eeb..2d151422a 100644
--- a/xlators/cluster/stripe/src/Makefile.am
+++ b/xlators/cluster/stripe/src/Makefile.am
@@ -2,16 +2,19 @@
 xlator_LTLIBRARIES = stripe.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-stripe_la_LDFLAGS = -module -avoidversion
+stripe_la_LDFLAGS = -module -avoid-version
+
+stripe_la_SOURCES = stripe.c stripe-helpers.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-stripe_la_SOURCES = stripe.c $(top_builddir)/xlators/lib/src/libxlator.c
 stripe_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = stripe.h stripe-mem-types.h $(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/lib/src
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
 CLEANFILES = 
 
diff --git a/xlators/cluster/stripe/src/stripe-helpers.c b/xlators/cluster/stripe/src/stripe-helpers.c
new file mode 100644
index 000000000..a83abdc72
--- /dev/null
+++ b/xlators/cluster/stripe/src/stripe-helpers.c
@@ -0,0 +1,675 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <fnmatch.h>
+
+#include "stripe.h"
+#include "byte-order.h"
+#include "mem-types.h"
+
+void
+stripe_local_wipe (stripe_local_t *local)
+{
+        if (!local)
+                goto out;
+
+        loc_wipe (&local->loc);
+        loc_wipe (&local->loc2);
+
+        if (local->fd)
+                fd_unref (local->fd);
+
+        if (local->inode)
+                inode_unref (local->inode);
+
+        if (local->xattr)
+                dict_unref (local->xattr);
+
+        if (local->xdata)
+                dict_unref (local->xdata);
+
+out:
+        return;
+}
+
+
+
+int
+stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
+{
+        dict_t  *dst  = NULL;
+        int64_t *ptr  = 0, *size = NULL;
+        int32_t  ret  = -1;
+
+        dst = data;
+
+        if (strcmp (key, GF_XATTR_QUOTA_SIZE_KEY) == 0) {
+                ret = dict_get_bin (dst, key, (void **)&size);
+                if (ret < 0) {
+                        size = GF_CALLOC (1, sizeof (int64_t),
+                                          gf_common_mt_char);
+                        if (size == NULL) {
+                                gf_log ("stripe", GF_LOG_WARNING,
+                                        "memory allocation failed");
+                                goto out;
+                        }
+                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
+                        if (ret < 0) {
+                                gf_log ("stripe", GF_LOG_WARNING,
+                                        "stripe aggregate dict set failed");
+                                GF_FREE (size);
+                                goto out;
+                        }
+                }
+
+                ptr = data_to_bin (value);
+                if (ptr == NULL) {
+                        gf_log ("stripe", GF_LOG_WARNING, "data to bin failed");
+                        goto out;
+                }
+
+                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+        } else if (strcmp (key, GF_CONTENT_KEY)) {
+                /* No need to aggregate 'CONTENT' data */
+                ret = dict_set (dst, key, value);
+                if (ret)
+                        gf_log ("stripe", GF_LOG_WARNING, "xattr dict set failed");
+        }
+
+out:
+        return 0;
+}
+
+
+void
+stripe_aggregate_xattr (dict_t *dst, dict_t *src)
+{
+        if ((dst == NULL) || (src == NULL)) {
+                goto out;
+        }
+
+        dict_foreach (src, stripe_aggregate, dst);
+out:
+        return;
+}
+
+
+int32_t
+stripe_xattr_aggregate (char *buffer, stripe_local_t *local, int32_t *total)
+{
+        int32_t              i     = 0;
+        int32_t              ret   = -1;
+        int32_t              len   = 0;
+        char                *sbuf  = NULL;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (!buffer || !local || !local->xattr_list)
+                goto out;
+
+        sbuf = buffer;
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+                len = xattr->xattr_len;
+
+                if (len && xattr && xattr->xattr_value) {
+                        memcpy (buffer, xattr->xattr_value, len);
+                        buffer += len;
+                        *buffer++ = ' ';
+                }
+        }
+
+        *--buffer = '\0';
+        if (total)
+                *total = buffer - sbuf;
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int32_t
+stripe_free_xattr_str (stripe_local_t *local)
+{
+        int32_t              i     = 0;
+        int32_t              ret   = -1;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (!local || !local->xattr_list)
+                goto out;
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+
+                if (xattr && xattr->xattr_value)
+                        GF_FREE (xattr->xattr_value);
+        }
+
+        ret = 0;
+ out:
+        return ret;
+}
+
+
+int32_t
+stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            void **xattr_serz)
+{
+        int32_t              ret   = -1, i = 0, len = 0;
+        dict_t              *tmp1  = NULL, *tmp2 = NULL;
+        char                *buf   = NULL;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (xattr_serz == NULL) {
+                goto out;
+        }
+
+        tmp2 = dict_new ();
+
+        if (tmp2 == NULL) {
+                goto out;
+        }
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+                len = xattr->xattr_len;
+
+                if (len && xattr && xattr->xattr_value) {
+                        ret = dict_reset (tmp2);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "dict_reset failed (%s)",
+                                        strerror (-ret));
+                        }
+
+                        ret = dict_unserialize (xattr->xattr_value,
+                                                xattr->xattr_len,
+                                                &tmp2);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "dict_unserialize failed (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+
+                        tmp1 = dict_copy (tmp2, tmp1);
+                        if (tmp1 == NULL) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "dict_copy failed (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
+        len = dict_serialized_length (tmp1);
+        if (len > 0) {
+                buf = GF_CALLOC (1, len, gf_common_mt_dict_t);
+                if (buf == NULL) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_serialize (tmp1, buf);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "dict_serialize failed (%s)", strerror (-ret));
+                        ret = -1;
+                        goto out;
+                }
+
+                *xattr_serz = buf;
+        }
+
+        ret = 0;
+out:
+        if (tmp1 != NULL) {
+                dict_unref (tmp1);
+        }
+
+        if (tmp2 != NULL) {
+                dict_unref (tmp2);
+        }
+
+        return ret;
+}
+
+
+int32_t
+stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            char **xattr_serz)
+{
+        int      ret             = -1;
+        int32_t  padding         = 0;
+        int32_t  tlen            = 0;
+        char stripe_size_str[20] = {0,};
+        char    *pathinfo_serz   = NULL;
+
+        if (!local) {
+                gf_log (this->name, GF_LOG_ERROR, "Possible NULL deref");
+                goto out;
+        }
+
+        (void) snprintf (stripe_size_str, 20, "%ld",
+                         (local->fctx) ? local->fctx->stripe_size : 0);
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding = strlen (this->name) + strlen (STRIPE_PATHINFO_HEADER)
+                + strlen (stripe_size_str) + 7;
+        local->xattr_total_len += (padding + 2);
+
+        pathinfo_serz = GF_CALLOC (local->xattr_total_len, sizeof (char),
+                                   gf_common_mt_char);
+        if (!pathinfo_serz)
+                goto out;
+
+        /* xlator info */
+        (void) sprintf (pathinfo_serz, "(<"STRIPE_PATHINFO_HEADER"%s:[%s]> ",
+                        this->name, stripe_size_str);
+
+        ret = stripe_xattr_aggregate (pathinfo_serz + padding, local, &tlen);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot aggregate pathinfo list");
+                goto out;
+        }
+
+        *(pathinfo_serz + padding + tlen) = ')';
+        *(pathinfo_serz + padding + tlen + 1) = '\0';
+
+        *xattr_serz = pathinfo_serz;
+
+        ret = 0;
+ out:
+        return ret;
+}
+
+/**
+ * stripe_get_matching_bs - Get the matching block size for the given path.
+ */
+int32_t
+stripe_get_matching_bs (const char *path, stripe_private_t *priv)
+{
+        struct stripe_options *trav       = NULL;
+        uint64_t               block_size = 0;
+
+        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
+        GF_VALIDATE_OR_GOTO ("stripe", path, out);
+
+        LOCK (&priv->lock);
+        {
+                block_size = priv->block_size;
+                trav = priv->pattern;
+                while (trav) {
+                        if (!fnmatch (trav->path_pattern, path, FNM_NOESCAPE)) {
+                                block_size = trav->block_size;
+                                break;
+                        }
+                        trav = trav->next;
+                }
+        }
+        UNLOCK (&priv->lock);
+
+out:
+        return block_size;
+}
+
+int32_t
+stripe_ctx_handle (xlator_t *this, call_frame_t *prev, stripe_local_t *local,
+                   dict_t *dict)
+{
+        char            key[256]       = {0,};
+        data_t         *data            = NULL;
+        int32_t         index           = 0;
+        stripe_private_t *priv          = NULL;
+
+        priv = this->private;
+
+
+        if (!local->fctx) {
+                local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
+                                         gf_stripe_mt_stripe_fd_ctx_t);
+                if (!local->fctx) {
+                        local->op_errno = ENOMEM;
+                        local->op_ret = -1;
+                        goto out;
+                }
+
+                local->fctx->static_array = 0;
+        }
+        /* Stripe block size */
+        sprintf (key, "trusted.%s.stripe-size", this->name);
+        data = dict_get (dict, key);
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-size");
+                goto out;
+        } else {
+                if (!local->fctx->stripe_size) {
+                        local->fctx->stripe_size =
+                                     data_to_int64 (data);
+                }
+
+                if (local->fctx->stripe_size != data_to_int64 (data)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "stripe-size mismatch in blocks");
+                        local->xattr_self_heal_needed = 1;
+                }
+        }
+
+        /* Stripe count */
+        sprintf (key, "trusted.%s.stripe-count", this->name);
+        data = dict_get (dict, key);
+
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-count");
+                goto out;
+        }
+        if (!local->fctx->xl_array) {
+                local->fctx->stripe_count = data_to_int32 (data);
+                if (!local->fctx->stripe_count) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error with stripe-count xattr");
+                        local->op_ret   = -1;
+                        local->op_errno = EIO;
+                        goto out;
+                }
+
+                local->fctx->xl_array = GF_CALLOC (local->fctx->stripe_count,
+                                                   sizeof (xlator_t *),
+                                                   gf_stripe_mt_xlator_t);
+
+                if (!local->fctx->xl_array) {
+                        local->op_errno = ENOMEM;
+                        local->op_ret   = -1;
+                        goto out;
+                }
+        }
+        if (local->fctx->stripe_count != data_to_int32 (data)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error with stripe-count xattr (%d != %d)",
+                        local->fctx->stripe_count, data_to_int32 (data));
+                local->op_ret   = -1;
+                local->op_errno = EIO;
+                goto out;
+        }
+
+        /* index */
+        sprintf (key, "trusted.%s.stripe-index", this->name);
+        data = dict_get (dict, key);
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-index");
+                goto out;
+        }
+        index = data_to_int32 (data);
+        if (index > priv->child_count) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error with stripe-index xattr (%d)", index);
+                local->op_ret   = -1;
+                local->op_errno = EIO;
+                goto out;
+        }
+        if (local->fctx->xl_array) {
+                if (!local->fctx->xl_array[index])
+                        local->fctx->xl_array[index] = prev->this;
+        }
+
+	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
+	data = dict_get(dict, key);
+	if (!data) {
+		/*
+		 * The file was probably created prior to coalesce support.
+		 * Assume non-coalesce mode for this file to maintain backwards
+		 * compatibility.
+		 */
+		gf_log(this->name, GF_LOG_DEBUG, "missing stripe-coalesce "
+			"attr, assume non-coalesce mode");
+		local->fctx->stripe_coalesce = 0;
+	} else {
+		local->fctx->stripe_coalesce = data_to_int32(data);
+	}
+
+
+out:
+        return 0;
+}
+
+int32_t
+stripe_xattr_request_build (xlator_t *this, dict_t *dict, uint64_t stripe_size,
+                            uint32_t stripe_count, uint32_t stripe_index,
+			    uint32_t stripe_coalesce)
+{
+        char            key[256]       = {0,};
+        int32_t         ret             = -1;
+
+        sprintf (key, "trusted.%s.stripe-size", this->name);
+        ret = dict_set_int64 (dict, key, stripe_size);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+        sprintf (key, "trusted.%s.stripe-count", this->name);
+        ret = dict_set_int32 (dict, key, stripe_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+        sprintf (key, "trusted.%s.stripe-index", this->name);
+        ret = dict_set_int32 (dict, key, stripe_index);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
+	ret = dict_set_int32(dict, key, stripe_coalesce);
+	if (ret) {
+		gf_log(this->name, GF_LOG_WARNING,
+			"failed to set %s in xattr_req_dict", key);
+		goto out;
+	}
+out:
+        return ret;
+}
+
+
+static int
+set_default_block_size (stripe_private_t *priv, char *num)
+{
+
+        int             ret = -1;
+        GF_VALIDATE_OR_GOTO ("stripe", THIS, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, num, out);
+
+
+        if (gf_string2bytesize (num, &priv->block_size) != 0) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "invalid number format \"%s\"", num);
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+
+}
+
+
+int
+set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
+{
+        int                    ret = -1;
+        char                  *tmp_str = NULL;
+        char                  *tmp_str1 = NULL;
+        char                  *dup_str = NULL;
+        char                  *stripe_str = NULL;
+        char                  *pattern = NULL;
+        char                  *num = NULL;
+        struct stripe_options *temp_stripeopt = NULL;
+        struct stripe_options *stripe_opt = NULL;
+
+        if (!this || !priv || !data)
+                goto out;
+
+        /* Get the pattern for striping.
+           "option block-size *avi:10MB" etc */
+        stripe_str = strtok_r (data, ",", &tmp_str);
+        while (stripe_str) {
+                dup_str = gf_strdup (stripe_str);
+                stripe_opt = GF_CALLOC (1, sizeof (struct stripe_options),
+                                        gf_stripe_mt_stripe_options);
+                if (!stripe_opt) {
+                        goto out;
+                }
+
+                pattern = strtok_r (dup_str, ":", &tmp_str1);
+                num = strtok_r (NULL, ":", &tmp_str1);
+                if (!num) {
+                        num = pattern;
+                        pattern = "*";
+                        ret = set_default_block_size (priv, num);
+                        if (ret)
+                                goto out;
+                }
+                if (gf_string2bytesize (num, &stripe_opt->block_size) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid number format \"%s\"", num);
+                        goto out;
+                }
+
+                if (stripe_opt->block_size < STRIPE_MIN_BLOCK_SIZE) {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid Block-size: "
+                                "%s. Should be atleast %llu bytes", num,
+                                STRIPE_MIN_BLOCK_SIZE);
+                        goto out;
+                }
+                if (stripe_opt->block_size % 512) {
+                        gf_log (this->name, GF_LOG_ERROR, "Block-size: %s should"
+                                " be a multiple of 512 bytes", num);
+                        goto out;
+                }
+
+                memcpy (stripe_opt->path_pattern, pattern, strlen (pattern));
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "block-size : pattern %s : size %"PRId64,
+                        stripe_opt->path_pattern, stripe_opt->block_size);
+
+                if (priv->pattern)
+                        temp_stripeopt = NULL;
+                else
+                        temp_stripeopt = priv->pattern;
+
+                stripe_opt->next = temp_stripeopt;
+
+                priv->pattern = stripe_opt;
+                stripe_opt = NULL;
+
+                GF_FREE (dup_str);
+                dup_str = NULL;
+
+                stripe_str = strtok_r (NULL, ",", &tmp_str);
+        }
+
+        ret = 0;
+out:
+
+        GF_FREE (dup_str);
+
+        GF_FREE (stripe_opt);
+
+        return ret;
+}
+
+int32_t
+stripe_iatt_merge (struct iatt *from, struct iatt *to)
+{
+        if (to->ia_size < from->ia_size)
+                to->ia_size = from->ia_size;
+        if (to->ia_mtime < from->ia_mtime)
+                to->ia_mtime = from->ia_mtime;
+        if (to->ia_ctime < from->ia_ctime)
+                to->ia_ctime = from->ia_ctime;
+        if (to->ia_atime < from->ia_atime)
+                to->ia_atime = from->ia_atime;
+        return 0;
+}
+
+off_t
+coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count)
+{
+	size_t line_size = 0;
+	uint64_t stripe_num = 0;
+	off_t coalesced_offset = 0;
+
+	line_size = stripe_size * stripe_count;
+	stripe_num = offset / line_size;
+
+	coalesced_offset = (stripe_num * stripe_size) +
+		(offset % stripe_size);
+
+	return coalesced_offset;
+}
+
+off_t
+uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
+		int stripe_index)
+{
+        uint64_t nr_full_stripe_chunks = 0, mod = 0;
+
+        if (!size)
+                return size;
+
+	/*
+	 * Estimate the number of fully written stripes from the
+	 * local file size. Each stripe_size chunk corresponds to
+	 * a stripe.
+	 */
+        nr_full_stripe_chunks = (size / stripe_size) * stripe_count;
+        mod = size % stripe_size;
+
+        if (!mod) {
+                /*
+		 * There is no remainder, thus we could have overestimated
+		 * the size of the file in terms of chunks. Trim the number
+		 * of chunks by the following stripe members and leave it
+		 * up to those nodes to respond with a larger size (if
+		 * necessary).
+		 */
+                nr_full_stripe_chunks -= stripe_count -
+                        (stripe_index + 1);
+                size = nr_full_stripe_chunks * stripe_size;
+        } else {
+		/*
+		 * There is a remainder and thus we own the last chunk of the
+		 * file. Add the preceding stripe members of the final stripe
+		 * along with the remainder to calculate the exact size.
+		 */
+                nr_full_stripe_chunks += stripe_index;
+                size = nr_full_stripe_chunks * stripe_size + mod;
+        }
+
+        return size;
+}
+
diff --git a/xlators/cluster/stripe/src/stripe-mem-types.h b/xlators/cluster/stripe/src/stripe-mem-types.h
index 29c95c257..e9ac9cf46 100644
--- a/xlators/cluster/stripe/src/stripe-mem-types.h
+++ b/xlators/cluster/stripe/src/stripe-mem-types.h
@@ -1,21 +1,11 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -25,12 +15,12 @@
 #include "mem-types.h"
 
 enum gf_stripe_mem_types_ {
-        gf_stripe_mt_stripe_local_t = gf_common_mt_end + 1,
-        gf_stripe_mt_iovec,
-        gf_stripe_mt_readv_replies,
+        gf_stripe_mt_iovec = gf_common_mt_end + 1,
+        gf_stripe_mt_stripe_replies,
         gf_stripe_mt_stripe_fd_ctx_t,
         gf_stripe_mt_char,
         gf_stripe_mt_int8_t,
+        gf_stripe_mt_int32_t,
         gf_stripe_mt_xlator_t,
         gf_stripe_mt_stripe_private_t,
         gf_stripe_mt_stripe_options,
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
index 9c86060c3..69b510e23 100644
--- a/xlators/cluster/stripe/src/stripe.c
+++ b/xlators/cluster/stripe/src/stripe.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /**
@@ -41,217 +32,10 @@
 
 struct volume_options options[];
 
-void
-stripe_local_wipe (stripe_local_t *local)
-{
-        if (!local)
-                goto out;
-
-        loc_wipe (&local->loc);
-        loc_wipe (&local->loc2);
-
-        if (local->fd)
-                fd_unref (local->fd);
-
-        if (local->inode)
-                inode_unref (local->inode);
-
-        if (local->xattr)
-                dict_unref (local->xattr);
-
-        if (local->dict)
-                dict_unref (local->dict);
-
-out:
-        return;
-}
-
-/**
- * stripe_get_matching_bs - Get the matching block size for the given path.
- */
-int32_t
-stripe_get_matching_bs (const char *path, struct stripe_options *opts,
-                        uint64_t default_bs)
-{
-        struct stripe_options *trav       = NULL;
-        char                  *pathname   = NULL;
-        uint64_t               block_size = 0;
-
-        block_size = default_bs;
-
-        if (!path || !opts)
-                goto out;
-
-        /* FIXME: is a strdup really necessary? */
-        pathname = gf_strdup (path);
-        if (!pathname)
-                goto out;
-
-        trav = opts;
-        while (trav) {
-                if (!fnmatch (trav->path_pattern, pathname, FNM_NOESCAPE)) {
-                        block_size = trav->block_size;
-                        break;
-                }
-                trav = trav->next;
-        }
-
-        GF_FREE (pathname);
-
-out:
-        return block_size;
-}
-
-
-
-int32_t
-stripe_ctx_handle (xlator_t *this, call_frame_t *prev, stripe_local_t *local,
-                   dict_t *dict)
-{
-        char            key[256]       = {0,};
-        data_t         *data            = NULL;
-        int32_t         index           = 0;
-        stripe_private_t *priv          = NULL;
-        int32_t         ret             = -1;
-
-        priv = this->private;
-
-
-        if (!local->fctx) {
-                local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                         gf_stripe_mt_stripe_fd_ctx_t);
-                if (!local->fctx) {
-                        local->op_errno = ENOMEM;
-                        local->op_ret = -1;
-                        goto out;
-                }
-
-                local->fctx->static_array = 0;
-        }
-        /* Stripe block size */
-        sprintf (key, "trusted.%s.stripe-size", this->name);
-        data = dict_get (dict, key);
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-size");
-                goto out;
-        } else {
-                if (!local->fctx->stripe_size) {
-                        local->fctx->stripe_size =
-                                     data_to_int64 (data);
-                }
-
-                if (local->fctx->stripe_size != data_to_int64 (data)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "stripe-size mismatch in blocks");
-                        local->xattr_self_heal_needed = 1;
-                }
-        }
-
-        /* Stripe count */
-        sprintf (key, "trusted.%s.stripe-count", this->name);
-        data = dict_get (dict, key);
-
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-count");
-                goto out;
-        }
-        if (!local->fctx->xl_array) {
-                local->fctx->stripe_count = data_to_int32 (data);
-                if (!local->fctx->stripe_count) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error with stripe-count xattr");
-                        local->op_ret   = -1;
-                        local->op_errno = EIO;
-                        goto out;
-                }
-
-                local->fctx->xl_array = GF_CALLOC (local->fctx->stripe_count,
-                                                   sizeof (xlator_t *),
-                                                   gf_stripe_mt_xlator_t);
-
-                if (!local->fctx->xl_array) {
-                        local->op_errno = ENOMEM;
-                        local->op_ret   = -1;
-                        goto out;
-                }
-        }
-        if (local->fctx->stripe_count != data_to_int32 (data)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error with stripe-count xattr (%d != %d)",
-                        local->fctx->stripe_count, data_to_int32 (data));
-                local->op_ret   = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-
-        /* index */
-        sprintf (key, "trusted.%s.stripe-index", this->name);
-        data = dict_get (dict, key);
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-index");
-                goto out;
-        }
-        index = data_to_int32 (data);
-        if (index > priv->child_count) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error with stripe-index xattr (%d)", index);
-                local->op_ret   = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-        if (local->fctx->xl_array) {
-                if (!local->fctx->xl_array[index])
-                        local->fctx->xl_array[index] = prev->this;
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-stripe_xattr_request_build (xlator_t *this, dict_t *dict, uint64_t stripe_size,
-                            uint32_t stripe_count, uint32_t stripe_index)
-{
-        char            key[256]       = {0,};
-        int32_t         ret             = -1;
-
-        sprintf (key, "trusted.%s.stripe-size", this->name);
-        ret = dict_set_int64 (dict, key, stripe_size);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-        sprintf (key, "trusted.%s.stripe-count", this->name);
-        ret = dict_set_int32 (dict, key, stripe_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-        sprintf (key, "trusted.%s.stripe-index", this->name);
-        ret = dict_set_int32 (dict, key, stripe_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-out:
-        return ret;
-}
-
 int32_t
 stripe_sh_chown_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *preop, struct iatt *postop)
+                     struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         int             callcnt = -1;
         stripe_local_t *local   = NULL;
@@ -280,7 +64,7 @@ int32_t
 stripe_sh_make_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, inode_t *inode,
                           struct iatt *buf, struct iatt *preparent,
-                          struct iatt *postparent)
+                          struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         call_frame_t    *prev = NULL;
@@ -295,7 +79,7 @@ stripe_sh_make_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         STACK_WIND (frame, stripe_sh_chown_cbk, prev->this,
                     prev->this->fops->setattr, &local->loc,
-                    &local->stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID));
+                    &local->stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
 
 out:
         return 0;
@@ -309,7 +93,7 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
         call_frame_t     *rframe = NULL;
         stripe_local_t   *rlocal = NULL;
         stripe_private_t *priv   = NULL;
-        dict_t           *dict   = NULL;
+        dict_t           *xdata  = NULL;
         int               ret    = 0;
 
         if (!local || !this || !frame) {
@@ -327,8 +111,7 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
         if (!rframe) {
                 goto out;
         }
-        rlocal = GF_CALLOC (1, sizeof (stripe_local_t),
-                            gf_stripe_mt_stripe_local_t);
+        rlocal = mem_get0 (this->local_pool);
         if (!rlocal) {
                 goto out;
         }
@@ -337,11 +120,11 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
         loc_copy (&rlocal->loc, &local->loc);
         memcpy (&rlocal->stbuf, &local->stbuf, sizeof (struct iatt));
 
-        dict = dict_new ();
-        if (!dict)
+        xdata = dict_new ();
+        if (!xdata)
                 goto out;
 
-        ret = dict_set_static_bin (dict, "gfid-req", local->stbuf.ia_gfid, 16);
+        ret = dict_set_static_bin (xdata, "gfid-req", local->stbuf.ia_gfid, 16);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to set gfid-req", local->loc.path);
@@ -352,97 +135,38 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
                                     trav->xlator, trav->xlator->fops->mknod,
                                     &local->loc,
                                     st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type), 0,
-                                    dict);
+                                                     local->stbuf.ia_type),
+                                    0, 0, xdata);
                 }
                 if (IA_ISDIR (local->stbuf.ia_type)) {
                         STACK_WIND (rframe, stripe_sh_make_entry_cbk,
                                     trav->xlator, trav->xlator->fops->mkdir,
-                                    &local->loc, st_mode_from_ia (local->stbuf.ia_prot,
-                                                                  local->stbuf.ia_type),
-                                    dict);
+                                    &local->loc,
+                                    st_mode_from_ia (local->stbuf.ia_prot,
+                                                     local->stbuf.ia_type),
+                                    0, xdata);
                 }
                 trav = trav->next;
         }
 
-        if (dict)
-                dict_unref (dict);
+        if (xdata)
+                dict_unref (xdata);
         return 0;
 
 out:
         if (rframe)
                 STRIPE_STACK_DESTROY (rframe);
-        if (dict)
-                dict_unref (dict);
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
-void
-stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
-{
-        dict_t  *dst  = NULL;
-        int64_t *ptr  = 0, *size = NULL;
-        int32_t  ret  = -1;
-
-        dst = data;
-
-        if (strcmp (key, GF_XATTR_QUOTA_SIZE_KEY) == 0) {
-                ret = dict_get_bin (dst, key, (void **)&size);
-                if (ret < 0) {
-                        size = GF_CALLOC (1, sizeof (int64_t),
-                                          gf_common_mt_char);
-                        if (size == NULL) {
-                                gf_log ("stripe", GF_LOG_WARNING,
-                                        "memory allocation failed");
-                                goto out;
-                        }
-                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
-                        if (ret < 0) {
-                                gf_log ("stripe", GF_LOG_WARNING,
-                                        "stripe aggregate dict set failed");
-                                GF_FREE (size);
-                                goto out;
-                        }
-                }
-
-                ptr = data_to_bin (value);
-                if (ptr == NULL) {
-                        gf_log ("stripe", GF_LOG_WARNING, "data to bin failed");
-                        goto out;
-                }
-
-                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
-        } else if (strcmp (key, GF_CONTENT_KEY)) {
-                /* No need to aggregate 'CONTENT' data */
-                ret = dict_set (dst, key, value);
-                if (ret)
-                        gf_log ("stripe", GF_LOG_WARNING, "xattr dict set failed");
-        }
-
-out:
-        return;
-}
-
-
-void
-stripe_aggregate_xattr (dict_t *dst, dict_t *src)
-{
-        if ((dst == NULL) || (src == NULL)) {
-                goto out;
-        }
-
-        dict_foreach (src, stripe_aggregate, dst);
-out:
-        return;
-}
-
-
 int32_t
 stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
         int32_t         callcnt     = 0;
         stripe_local_t *local       = NULL;
@@ -480,7 +204,7 @@ stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = 0;
                         if (IA_ISREG (buf->ia_type)) {
                                 ret = stripe_ctx_handle (this, prev, local,
-                                                         dict);
+                                                         xdata);
                                 if (ret)
                                         gf_log (this->name, GF_LOG_ERROR,
                                                  "Error getting fctx info from"
@@ -491,26 +215,29 @@ stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 local->stbuf      = *buf;
                                 local->postparent = *postparent;
                                 local->inode = inode_ref (inode);
-                                local->dict = dict_ref (dict);
+                                if (xdata)
+                                        local->xdata = dict_ref (xdata);
                                 if (local->xattr) {
-                                        stripe_aggregate_xattr (local->dict,
+                                        stripe_aggregate_xattr (local->xdata,
                                                                 local->xattr);
                                         dict_unref (local->xattr);
                                         local->xattr = NULL;
                                 }
                         }
 
-                        if (!local->dict && !local->xattr) {
-                                local->xattr = dict_ref (dict);
-                        } else if (local->dict) {
-                                stripe_aggregate_xattr (local->dict, dict);
+                        if (!local->xdata && !local->xattr) {
+                                local->xattr = dict_ref (xdata);
+                        } else if (local->xdata) {
+                                stripe_aggregate_xattr (local->xdata, xdata);
                         } else if (local->xattr) {
-                                stripe_aggregate_xattr (local->xattr, dict);
+                                stripe_aggregate_xattr (local->xattr, xdata);
                         }
 
                         local->stbuf_blocks      += buf->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->postparent_size < postparent->ia_size)
@@ -548,7 +275,7 @@ stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STRIPE_STACK_UNWIND (lookup, frame, local->op_ret,
                                      local->op_errno, local->inode,
-                                     &local->stbuf, local->dict,
+                                     &local->stbuf, local->xdata,
                                      &local->postparent);
         }
 out:
@@ -557,7 +284,7 @@ out:
 
 int32_t
 stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xattr_req)
+               dict_t *xdata)
 {
         stripe_local_t   *local    = NULL;
         xlator_list_t    *trav     = NULL;
@@ -577,8 +304,7 @@ stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -591,15 +317,29 @@ stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         if (tmpctx)
                 local->fctx = (stripe_fd_ctx_t*) (long)tmpctx;
 
-        if (xattr_req && dict_get (xattr_req, GF_CONTENT_KEY)) {
-                ret = dict_get_int64 (xattr_req, GF_CONTENT_KEY, &filesize);
+        /* quick-read friendly changes */
+        if (xdata && dict_get (xdata, GF_CONTENT_KEY)) {
+                ret = dict_get_int64 (xdata, GF_CONTENT_KEY, &filesize);
                 if (!ret && (filesize > priv->block_size))
-                        dict_del (xattr_req, GF_CONTENT_KEY);
+                        dict_del (xdata, GF_CONTENT_KEY);
         }
 
-        /* get stripe-size xattr on lookup for pathinfo string */
-        if (xattr_req && IA_ISREG (loc->inode->ia_type)) {
-                ret = stripe_xattr_request_build (this, xattr_req, 8, 4, 4);
+        /* get stripe-size xattr on lookup. This would be required for
+         * open/read/write/pathinfo calls. Hence we send down the request
+         * even when type == IA_INVAL */
+
+	/*
+	 * We aren't guaranteed to have xdata here. We need the format info for
+	 * the file, so allocate xdata if necessary.
+	 */
+	if (!xdata)
+		xdata = dict_new();
+	else
+		xdata = dict_ref(xdata);
+
+        if (xdata && (IA_ISREG (loc->inode->ia_type) ||
+            (loc->inode->ia_type == IA_INVAL))) {
+                ret = stripe_xattr_request_build (this, xdata, 8, 4, 4, 0);
                 if (ret)
                         gf_log (this->name , GF_LOG_ERROR, "Failed to build"
                                 " xattr request for %s", loc->path);
@@ -611,11 +351,12 @@ stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         local->call_count = priv->child_count;
         while (trav) {
                 STACK_WIND (frame, stripe_lookup_cbk, trav->xlator,
-                            trav->xlator->fops->lookup,
-                            loc, xattr_req);
+                            trav->xlator->fops->lookup, loc, xdata);
                 trav = trav->next;
         }
 
+	dict_unref(xdata);
+
         return 0;
 err:
         STRIPE_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
@@ -625,7 +366,7 @@ err:
 
 int32_t
 stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -660,6 +401,9 @@ stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         }
 
                         local->stbuf_blocks += buf->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                 }
@@ -676,18 +420,19 @@ stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
 
                 STRIPE_STACK_UNWIND (stat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf);
+                                     local->op_errno, &local->stbuf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = EINVAL;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -705,8 +450,7 @@ stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -715,23 +459,30 @@ stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
         frame->local = local;
         local->call_count = priv->child_count;
 
+	if (IA_ISREG(loc->inode->ia_type)) {
+		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
+
         while (trav) {
                 STACK_WIND (frame, stripe_stat_cbk, trav->xlator,
-                            trav->xlator->fops->stat, loc);
+                            trav->xlator->fops->stat, loc, NULL);
                 trav = trav->next;
         }
 
         return 0;
 
 err:
-        STRIPE_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf)
+                   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf, dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         int32_t         callcnt = 0;
@@ -769,14 +520,14 @@ stripe_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (!callcnt) {
                 STRIPE_STACK_UNWIND (statfs, frame, local->op_ret,
-                                     local->op_errno, &local->statvfs_buf);
+                                     local->op_errno, &local->statvfs_buf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         xlator_list_t    *trav = NULL;
@@ -791,8 +542,7 @@ stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         priv = this->private;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -804,13 +554,13 @@ stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         local->call_count = priv->child_count;
         while (trav) {
                 STACK_WIND (frame, stripe_statfs_cbk, trav->xlator,
-                            trav->xlator->fops->statfs, loc);
+                            trav->xlator->fops->statfs, loc, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (statfs, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -819,7 +569,7 @@ err:
 int32_t
 stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -857,6 +607,9 @@ stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->prebuf_blocks  += prebuf->ia_blocks;
                         local->postbuf_blocks += postbuf->ia_blocks;
 
+			correct_file_size(prebuf, local->fctx, prev);
+			correct_file_size(postbuf, local->fctx, prev);
+
                         if (local->prebuf_size < prebuf->ia_size)
                                 local->prebuf_size = prebuf->ia_size;
 
@@ -879,19 +632,21 @@ stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STRIPE_STACK_UNWIND (truncate, frame, local->op_ret,
                                      local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
 {
-        xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = EINVAL;
+	int		  i, eof_idx;
+	off_t		  dest_offset, tmp_offset;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -900,7 +655,6 @@ stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         VALIDATE_OR_GOTO (loc->inode, err);
 
         priv = this->private;
-        trav = this->children;
 
         if (priv->first_child_down) {
                 op_errno = ENOTCONN;
@@ -908,8 +662,7 @@ stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -918,15 +671,55 @@ stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         frame->local = local;
         local->call_count = priv->child_count;
 
-        while (trav) {
-                STACK_WIND (frame, stripe_truncate_cbk, trav->xlator,
-                            trav->xlator->fops->truncate, loc, offset);
-                trav = trav->next;
-        }
+	inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
+		op_errno = EINVAL;
+		goto err;
+	}
+
+	local->fctx = fctx;
+	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
+
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (!fctx->xl_array[i]) {
+			gf_log(this->name, GF_LOG_ERROR,
+				"no xlator at index %d", i);
+			op_errno = EINVAL;
+			goto err;
+		}
+
+		if (fctx->stripe_coalesce) {
+			/*
+	 		 * The node that owns EOF is truncated to the exact
+	 		 * coalesced offset. Nodes prior to this index should
+	 		 * be rounded up to the size of the complete stripe,
+	 		 * while nodes after this index should be rounded down
+			 * to the size of the previous stripe.
+			 */
+			if (i < eof_idx)
+				tmp_offset = roof(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else if (i > eof_idx)
+				tmp_offset = floor(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else
+				tmp_offset = offset;
+
+			dest_offset = coalesced_offset(tmp_offset,
+					fctx->stripe_size, fctx->stripe_count);
+		} else {
+			dest_offset = offset;
+		}
+
+		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
+			fctx->xl_array[i]->fops->truncate, loc, dest_offset,
+			NULL);
+	}
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -934,7 +727,7 @@ err:
 int32_t
 stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *preop, struct iatt *postop)
+                    struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -973,6 +766,9 @@ stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->prebuf_blocks  += preop->ia_blocks;
                         local->postbuf_blocks += postop->ia_blocks;
 
+			correct_file_size(preop, local->fctx, prev);
+			correct_file_size(postop, local->fctx, prev);
+
                         if (local->prebuf_size < preop->ia_size)
                                 local->prebuf_size = preop->ia_size;
                         if (local->postbuf_size < postop->ia_size)
@@ -994,7 +790,7 @@ stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STRIPE_STACK_UNWIND (setattr, frame, local->op_ret,
                                      local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
@@ -1003,11 +799,12 @@ out:
 
 int32_t
 stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t	 *fctx = NULL;
         int32_t           op_errno = EINVAL;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -1025,8 +822,7 @@ stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -1038,28 +834,35 @@ stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 local->call_count = 1;
                 STACK_WIND (frame, stripe_setattr_cbk, FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, NULL);
                 return 0;
         }
 
+	if (IA_ISREG(loc->inode->ia_type)) {
+		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
+
         local->call_count = priv->child_count;
         while (trav) {
                 STACK_WIND (frame, stripe_setattr_cbk,
                             trav->xlator, trav->xlator->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid)
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -1075,8 +878,7 @@ stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -1087,13 +889,13 @@ stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         while (trav) {
                 STACK_WIND (frame, stripe_setattr_cbk, trav->xlator,
-                            trav->xlator->fops->fsetattr, fd, stbuf, valid);
+                            trav->xlator->fops->fsetattr, fd, stbuf, valid, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1101,7 +903,8 @@ int32_t
 stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
                          struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent)
+                         struct iatt *prenewparent, struct iatt *postnewparent,
+                         dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -1138,6 +941,8 @@ stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->pre_buf.ia_blocks    += prenewparent->ia_blocks;
                         local->post_buf.ia_blocks   += postnewparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf.ia_size < buf->ia_size)
                                 local->stbuf.ia_size =  buf->ia_size;
 
@@ -1163,7 +968,7 @@ stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 STRIPE_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                                      &local->stbuf, &local->preparent,
                                      &local->postparent,  &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
@@ -1173,7 +978,8 @@ int32_t
 stripe_first_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
                          struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent)
+                         struct iatt *prenewparent, struct iatt *postnewparent,
+                         dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         xlator_list_t  *trav = NULL;
@@ -1204,24 +1010,25 @@ stripe_first_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         while (trav) {
                 STACK_WIND (frame, stripe_stack_rename_cbk,
                             trav->xlator, trav->xlator->fops->rename,
-                            &local->loc, &local->loc2);
+                            &local->loc, &local->loc2, NULL);
                 trav = trav->next;
         }
         return 0;
 
 unwind:
         STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent);
+                             postoldparent, prenewparent, postnewparent, NULL);
         return 0;
 }
 
 int32_t
 stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc)
+               loc_t *newloc, dict_t *xdata)
 {
         stripe_private_t *priv = NULL;
         stripe_local_t   *local = NULL;
         xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t	 *fctx = NULL;
         int32_t           op_errno = EINVAL;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -1241,8 +1048,7 @@ stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -1253,21 +1059,28 @@ stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 
         local->call_count = priv->child_count;
 
+	if (IA_ISREG(oldloc->inode->ia_type)) {
+		inode_ctx_get(oldloc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
+
         frame->local = local;
 
         STACK_WIND (frame, stripe_first_rename_cbk, trav->xlator,
-                    trav->xlator->fops->rename, oldloc, newloc);
+                    trav->xlator->fops->rename, oldloc, newloc, NULL);
 
         return 0;
 err:
         STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
+                             NULL, NULL, NULL);
         return 0;
 }
 int32_t
 stripe_first_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                         int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t *local   = NULL;
         call_frame_t   *prev = NULL;
@@ -1292,10 +1105,10 @@ stripe_first_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->postparent_blocks += postparent->ia_blocks;
 
         STRIPE_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
-                            &local->preparent, &local->postparent);
+                            &local->preparent, &local->postparent, xdata);
         return 0;
 out:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -1306,7 +1119,7 @@ out:
 int32_t
 stripe_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -1342,17 +1155,19 @@ stripe_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto out;
                 }
                 STACK_WIND(frame, stripe_first_unlink_cbk, FIRST_CHILD (this),
-                           FIRST_CHILD (this)->fops->unlink, &local->loc);
+                           FIRST_CHILD (this)->fops->unlink, &local->loc,
+                           local->xflag, local->xdata);
         }
         return 0;
 out:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
 
 int32_t
-stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int xflag, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -1380,14 +1195,18 @@ stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
         local->op_ret = -1;
         loc_copy (&local->loc, loc);
+        local->xflag = xflag;
+
+        if (xdata)
+            local->xdata = dict_ref (xdata);
+
         frame->local = local;
         local->call_count = priv->child_count;
         trav = trav->next; /* Skip the first child */
@@ -1395,13 +1214,13 @@ stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         while (trav) {
                 STACK_WIND (frame, stripe_unlink_cbk,
                             trav->xlator, trav->xlator->fops->unlink,
-                            loc);
+                            loc, xflag, xdata);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1409,8 +1228,7 @@ err:
 int32_t
 stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno,struct iatt *preparent,
-                        struct iatt *postparent)
-
+                        struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t *local = NULL;
 
@@ -1437,10 +1255,10 @@ stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->postparent_blocks += postparent->ia_blocks;
 
         STRIPE_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                             &local->preparent, &local->postparent);
+                             &local->preparent, &local->postparent, xdata);
         return 0;
 err:
-        STRIPE_STACK_UNWIND (rmdir, frame, op_ret, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (rmdir, frame, op_ret, op_errno, NULL, NULL, NULL);
         return 0;
 
 }
@@ -1448,7 +1266,7 @@ err:
 int32_t
 stripe_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -1480,16 +1298,16 @@ stripe_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto out;
                 STACK_WIND (frame, stripe_first_rmdir_cbk, FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->rmdir, &local->loc,
-                            local->flags);
+                            local->flags, NULL);
         }
         return 0;
 out:
-        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 int32_t
-stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -1512,8 +1330,7 @@ stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -1527,13 +1344,13 @@ stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 
         while (trav) {
                 STACK_WIND (frame, stripe_rmdir_cbk,  trav->xlator,
-                            trav->xlator->fops->rmdir, loc, flags);
+                            trav->xlator->fops->rmdir, loc, flags, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1542,7 +1359,7 @@ int32_t
 stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame, void *cookie,
                                     xlator_t *this, int32_t op_ret,
                                     int32_t op_errno, struct iatt *preparent,
-                                    struct iatt *postparent)
+                                    struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -1563,7 +1380,7 @@ stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame, void *cookie,
         if (!callcnt) {
                 STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
                                      local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -1575,7 +1392,7 @@ out:
 int32_t
 stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno)
+                                 int32_t op_errno, dict_t *xdata)
 {
         int32_t           callcnt = 0;
         stripe_local_t   *local = NULL;
@@ -1614,7 +1431,7 @@ stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
                                             stripe_mknod_ifreg_fail_unlink_cbk,
                                             trav->xlator,
                                             trav->xlator->fops->unlink,
-                                            &local->loc);
+                                            &local->loc, 0, NULL);
                                 trav = trav->next;
                         }
                         return 0;
@@ -1622,7 +1439,7 @@ stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
 
                 STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
                                      local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -1632,14 +1449,13 @@ int32_t
 stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, inode_t *inode,
                         struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent)
+                        struct iatt *postparent, dict_t *xdata)
 {
         int32_t           callcnt = 0;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
         call_frame_t     *prev = NULL;
         xlator_list_t    *trav = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -1671,10 +1487,16 @@ stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         if (uuid_is_null (local->ia_gfid))
                                 uuid_copy (local->ia_gfid, buf->ia_gfid);
 
+			if (stripe_ctx_handle(this, prev, local, xdata))
+				gf_log(this->name, GF_LOG_ERROR,
+					"Error getting fctx info from dict");
+
                         local->stbuf_blocks += buf->ia_blocks;
                         local->preparent_blocks  += preparent->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->preparent_size < preparent->ia_size)
@@ -1699,7 +1521,7 @@ stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                             stripe_mknod_ifreg_fail_unlink_cbk,
                                             trav->xlator,
                                             trav->xlator->fops->unlink,
-                                            &local->loc);
+                                            &local->loc, 0, NULL);
                                 trav = trav->next;
                         }
                         return 0;
@@ -1713,26 +1535,13 @@ stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->postparent.ia_size   = local->postparent_size;
                         local->stbuf.ia_size        = local->stbuf_size;
                         local->stbuf.ia_blocks      = local->stbuf_blocks;
-                        fctx = GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                          gf_stripe_mt_stripe_fd_ctx_t);
-                        if (!fctx) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unwind;
-                        }
-
-                        fctx->stripe_size  = local->stripe_size;
-                        fctx->stripe_count = priv->child_count;
-                        fctx->static_array = 1;
-                        fctx->xl_array = priv->xl_array;
                         inode_ctx_put (local->inode, this,
-                                       (uint64_t)(long)fctx);
+                                       (uint64_t)(long) local->fctx);
 
                 }
-unwind:
                 STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
                                      local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -1743,7 +1552,7 @@ int32_t
 stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, inode_t *inode,
                         struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent)
+                        struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -1803,7 +1612,8 @@ stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                         ret = stripe_xattr_request_build (this, dict,
                                                           local->stripe_size,
-                                                          priv->child_count, i);
+                                                          priv->child_count, i,
+							  priv->coalesce);
                         if (ret)
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "Failed to build xattr request");
@@ -1814,7 +1624,7 @@ stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STACK_WIND (frame, stripe_mknod_ifreg_cbk,
                             trav->xlator, trav->xlator->fops->mknod,
-                            &local->loc, local->mode, local->rdev, dict);
+                            &local->loc, local->mode, local->rdev, 0, dict);
                 trav = trav->next;
                 i++;
 
@@ -1826,7 +1636,7 @@ stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
 
-       STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL);
+       STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL, NULL);
        return 0;
 }
 
@@ -1835,25 +1645,22 @@ int32_t
 stripe_single_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, inode_t *inode,
                          struct iatt *buf, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
         STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, dict_t *params)
+              dev_t rdev, mode_t umask, dict_t *xdata)
 {
         stripe_private_t *priv           = NULL;
         stripe_local_t   *local          = NULL;
         int32_t           op_errno       = EINVAL;
         int32_t           i              = 0;
-        char              size_key[256]  = {0,};
-        char              index_key[256] = {0,};
-        char              count_key[256] = {0,};
         dict_t           *dict           = NULL;
         int               ret            = 0;
         int               need_unref     = 0;
@@ -1883,37 +1690,26 @@ stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 }
 
                 /* Initialization */
-                local = GF_CALLOC (1, sizeof (stripe_local_t),
-                                   gf_stripe_mt_stripe_local_t);
+                local = mem_get0 (this->local_pool);
                 if (!local) {
                         op_errno = ENOMEM;
                         goto err;
                 }
                 local->op_ret = -1;
                 local->op_errno = ENOTCONN;
-                local->stripe_size = stripe_get_matching_bs (loc->path,
-                                                             priv->pattern,
-                                                             priv->block_size);
+                local->stripe_size = stripe_get_matching_bs (loc->path, priv);
                 frame->local = local;
                 local->inode = inode_ref (loc->inode);
                 loc_copy (&local->loc, loc);
-                local->xattr = dict_copy_with_ref (params, NULL);
+                local->xattr = dict_copy_with_ref (xdata, NULL);
                 local->mode = mode;
+                local->umask = umask;
                 local->rdev = rdev;
 
                 /* Everytime in stripe lookup, all child nodes should
                    be looked up */
                 local->call_count = priv->child_count;
 
-                /* Send a setxattr request to nodes where the
-                   files are created */
-                sprintf (size_key,
-                         "trusted.%s.stripe-size", this->name);
-                sprintf (count_key,
-                         "trusted.%s.stripe-count", this->name);
-                sprintf (index_key,
-                         "trusted.%s.stripe-index", this->name);
-
                 if (priv->xattr_supported) {
                         dict = dict_new ();
                         if (!dict) {
@@ -1922,22 +1718,22 @@ stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                         }
                         need_unref = 1;
 
-                        dict_copy (params, dict);
+                        dict_copy (xdata, dict);
 
                         ret = stripe_xattr_request_build (this, dict,
                                                           local->stripe_size,
                                                           priv->child_count,
-                                                          i);
+                                                          i, priv->coalesce);
                         if (ret)
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "failed to build xattr request");
                 } else {
-                        dict = params;
+                        dict = xdata;
                 }
 
                 STACK_WIND (frame, stripe_mknod_first_ifreg_cbk,
                             FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                            loc, mode, rdev, dict);
+                            loc, mode, rdev, umask, dict);
 
                         if (dict && need_unref)
                                 dict_unref (dict);
@@ -1946,11 +1742,11 @@ stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
 
         STACK_WIND (frame, stripe_single_mknod_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, params);
+                    loc, mode, rdev, umask, xdata);
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        STRIPE_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1959,7 +1755,7 @@ int32_t
 stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t  *local   = NULL;
@@ -2016,7 +1812,7 @@ stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 STRIPE_STACK_UNWIND (mkdir, frame, local->op_ret,
                                      local->op_errno, local->inode,
                                      &local->stbuf, &local->preparent,
-                                     &local->postparent);
+                                     &local->postparent, NULL);
         }
 out:
         return 0;
@@ -2025,9 +1821,9 @@ out:
 
 int32_t
 stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *buf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t  *local   = NULL;
         call_frame_t    *prev = NULL;
@@ -2058,7 +1854,7 @@ stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->stbuf      = *buf;
         local->postparent = *postparent;
         local->preparent  = *preparent;
-         
+
         local->stbuf_blocks      += buf->ia_blocks;
         local->preparent_blocks  += preparent->ia_blocks;
         local->postparent_blocks += postparent->ia_blocks;
@@ -2070,13 +1866,13 @@ stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         while (trav) {
                 STACK_WIND (frame, stripe_mkdir_cbk, trav->xlator,
                             trav->xlator->fops->mkdir, &local->loc, local->mode,
-                            local->dict);
+                            local->umask, local->xdata);
                 trav = trav->next;
         }
         return 0;
 out:
         STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                      NULL);
+                             NULL, NULL);
 
         return 0;
 
@@ -2085,7 +1881,7 @@ out:
 
 int
 stripe_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dict_t *params)
+              mode_t umask, dict_t *xdata)
 {
         stripe_private_t *priv = NULL;
         stripe_local_t   *local = NULL;
@@ -2107,26 +1903,27 @@ stripe_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
         local->op_ret = -1;
         local->call_count = priv->child_count;
-        local->dict = dict_ref (params);
-        local->mode = mode;
+        if (xdata)
+                local->xdata = dict_ref (xdata);
+        local->mode  = mode;
+        local->umask = umask;
         loc_copy (&local->loc, loc);
         frame->local = local;
 
         /* Everytime in stripe lookup, all child nodes should be looked up */
         STACK_WIND (frame, stripe_first_mkdir_cbk, trav->xlator,
-                    trav->xlator->fops->mkdir, loc, mode, params);
+                    trav->xlator->fops->mkdir, loc, mode, umask, xdata);
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2135,11 +1932,12 @@ int32_t
 stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
                  struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t  *local   = NULL;
         call_frame_t    *prev = NULL;
+	stripe_fd_ctx_t *fctx = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -2166,6 +1964,16 @@ stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (op_ret >= 0) {
                         local->op_ret = 0;
 
+			if (IA_ISREG(inode->ia_type)) {
+				inode_ctx_get(inode, this, (uint64_t *) &fctx);
+				if (!fctx) {
+					gf_log(this->name, GF_LOG_ERROR,
+						"failed to get stripe context");
+					op_ret = -1;
+					op_errno = EINVAL;
+				}
+			}
+
                         if (FIRST_CHILD(this) == prev->this) {
                                 local->inode      = inode_ref (inode);
                                 local->stbuf      = *buf;
@@ -2176,6 +1984,8 @@ stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->preparent_blocks  += preparent->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->preparent_size < preparent->ia_size)
@@ -2201,14 +2011,14 @@ stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 STRIPE_STACK_UNWIND (link, frame, local->op_ret,
                                      local->op_errno, local->inode,
                                      &local->stbuf, &local->preparent,
-                                     &local->postparent);
+                                     &local->postparent, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -2231,8 +2041,7 @@ stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2246,13 +2055,13 @@ stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
         while (trav) {
                 STACK_WIND (frame, stripe_link_cbk,
                             trav->xlator, trav->xlator->fops->link,
-                            oldloc, newloc);
+                            oldloc, newloc, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        STRIPE_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2260,7 +2069,7 @@ int32_t
 stripe_create_fail_unlink_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this, int32_t op_ret,
                                int32_t op_errno, struct iatt *preparent,
-                               struct iatt *postparent)
+                               struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -2281,7 +2090,7 @@ stripe_create_fail_unlink_cbk (call_frame_t *frame, void *cookie,
         if (!callcnt) {
                 STRIPE_STACK_UNWIND (create, frame, local->op_ret, local->op_errno,
                                      local->fd, local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -2292,12 +2101,11 @@ int32_t
 stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, fd_t *fd,
                    inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         int32_t           callcnt = 0;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
         call_frame_t     *prev = NULL;
         xlator_list_t    *trav = NULL;
 
@@ -2323,12 +2131,21 @@ stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
 
                 if (op_ret >= 0) {
+			if (IA_ISREG(buf->ia_type)) {
+				if (stripe_ctx_handle(this, prev, local, xdata))
+					gf_log(this->name, GF_LOG_ERROR,
+						"Error getting fctx info from "
+						"dict");
+			}
+
                         local->op_ret = op_ret;
 
                         local->stbuf_blocks += buf->ia_blocks;
                         local->preparent_blocks  += preparent->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->preparent_size < preparent->ia_size)
@@ -2351,7 +2168,7 @@ stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                             stripe_create_fail_unlink_cbk,
                                             trav->xlator,
                                             trav->xlator->fops->unlink,
-                                            &local->loc);
+                                            &local->loc, 0, NULL);
                                 trav = trav->next;
                         }
 
@@ -2366,29 +2183,19 @@ stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->stbuf.ia_size        = local->stbuf_size;
                         local->stbuf.ia_blocks      = local->stbuf_blocks;
 
-                        fctx = GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                          gf_stripe_mt_stripe_fd_ctx_t);
-                        if (!fctx) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unwind;
-                        }
-
-                        fctx->stripe_size  = local->stripe_size;
-                        fctx->stripe_count = priv->child_count;
-                        fctx->static_array = 1;
-                        fctx->xl_array = priv->xl_array;
-                        inode_ctx_put (local->inode, this,
-                                    (uint64_t)(long)fctx);
+			stripe_copy_xl_array(local->fctx->xl_array,
+				             priv->xl_array,
+					     local->fctx->stripe_count);
+			inode_ctx_put(local->inode, this,
+					(uint64_t) local->fctx);
                 }
 
-        unwind:
                 /* Create itself has failed.. so return
                    without setxattring */
                 STRIPE_STACK_UNWIND (create, frame, local->op_ret,
                                      local->op_errno, local->fd,
                                      local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 
 out:
@@ -2401,7 +2208,7 @@ int32_t
 stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, fd_t *fd,
                    inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -2458,7 +2265,7 @@ stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->call_count = 1;
                 STACK_WIND (frame, stripe_create_fail_unlink_cbk,
                             FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                            &local->loc);
+                            &local->loc, 0, NULL);
                 return 0;
         }
 
@@ -2488,17 +2295,17 @@ stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         ret = stripe_xattr_request_build (this, dict,
                                                           local->stripe_size,
                                                           priv->child_count,
-                                                          i);
+                                                          i, priv->coalesce);
                         if (ret)
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "failed to build xattr request");
                 } else {
                         dict = local->xattr;
                 }
- 
+
                 STACK_WIND (frame, stripe_create_cbk, trav->xlator,
                             trav->xlator->fops->create, &local->loc,
-                            local->flags, local->mode, local->fd,
+                            local->flags, local->mode, local->umask, local->fd,
                             dict);
                 trav = trav->next;
                 if (need_unref && dict)
@@ -2520,7 +2327,7 @@ out:
  */
 int32_t
 stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+               int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         stripe_private_t *priv = NULL;
         stripe_local_t   *local = NULL;
@@ -2549,24 +2356,23 @@ stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
         local->op_ret = -1;
         local->op_errno = ENOTCONN;
-        local->stripe_size = stripe_get_matching_bs (loc->path,
-                                                     priv->pattern,
-                                                     priv->block_size);
+        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
         frame->local = local;
         local->inode = inode_ref (loc->inode);
         loc_copy (&local->loc, loc);
         local->fd = fd_ref (fd);
         local->flags = flags;
         local->mode = mode;
-        local->xattr = dict_copy_with_ref (params, NULL);
+        local->umask = umask;
+        if (xdata)
+                local->xattr = dict_ref (xdata);
 
         local->call_count = priv->child_count;
         /* Send a setxattr request to nodes where the
@@ -2580,23 +2386,23 @@ stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 }
                 need_unref = 1;
 
-                dict_copy (params, dict);
+                dict_copy (xdata, dict);
 
                 ret = stripe_xattr_request_build (this, dict,
                                                   local->stripe_size,
                                                   priv->child_count,
-                                                  i);
+                                                  i, priv->coalesce);
                 if (ret)
                         gf_log (this->name, GF_LOG_ERROR,
                                 "failed to build xattr request");
         } else {
-                        dict = params;
+                        dict = xdata;
         }
 
 
         STACK_WIND (frame, stripe_first_create_cbk, FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->create, loc, flags, mode,
-                    fd, dict);
+                    umask, fd, dict);
 
         if (need_unref && dict)
                 dict_unref (dict);
@@ -2605,13 +2411,13 @@ stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
         return 0;
 err:
         STRIPE_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
+                             NULL, NULL, xdata);
         return 0;
 }
 
 int32_t
 stripe_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd)
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -2650,7 +2456,7 @@ stripe_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = -1;
 
                 STRIPE_STACK_UNWIND (open, frame, local->op_ret,
-                                     local->op_errno, local->fd);
+                                     local->op_errno, local->fd, xdata);
         }
 out:
         return 0;
@@ -2662,7 +2468,7 @@ out:
  */
 int32_t
 stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int32_t flags, fd_t *fd, int32_t wbflags)
+             int32_t flags, fd_t *fd, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -2684,8 +2490,7 @@ stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2701,27 +2506,25 @@ stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
         /* Striped files */
         local->flags = flags;
         local->call_count = priv->child_count;
-        local->stripe_size = stripe_get_matching_bs (loc->path,
-                                                     priv->pattern,
-                                                     priv->block_size);
+        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
 
         while (trav) {
                 STACK_WIND (frame, stripe_open_cbk, trav->xlator,
                             trav->xlator->fops->open,
                             &local->loc, local->flags, local->fd,
-                            wbflags);
+                            xdata);
                 trav = trav->next;
         }
         return 0;
 err:
-        STRIPE_STACK_UNWIND (open, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd)
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -2754,7 +2557,7 @@ stripe_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (!callcnt) {
                 STRIPE_STACK_UNWIND (opendir, frame, local->op_ret,
-                                     local->op_errno, local->fd);
+                                     local->op_errno, local->fd, NULL);
         }
 out:
         return 0;
@@ -2762,7 +2565,7 @@ out:
 
 
 int32_t
-stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -2784,8 +2587,7 @@ stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2796,19 +2598,19 @@ stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 
         while (trav) {
                 STACK_WIND (frame, stripe_opendir_cbk, trav->xlator,
-                            trav->xlator->fops->opendir, loc, fd);
+                            trav->xlator->fops->opendir, loc, fd, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (opendir, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 int32_t
 stripe_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -2848,7 +2650,7 @@ stripe_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (local->failed)
                         local->op_ret = -1;
                 STRIPE_STACK_UNWIND (lk, frame, local->op_ret,
-                                     local->op_errno, &local->lock);
+                                     local->op_errno, &local->lock, NULL);
         }
 out:
         return 0;
@@ -2856,7 +2658,7 @@ out:
 
 int32_t
 stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *lock)
+           struct gf_flock *lock, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         xlator_list_t    *trav = NULL;
@@ -2872,8 +2674,7 @@ stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
         priv = this->private;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2884,20 +2685,20 @@ stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
 
         while (trav) {
                 STACK_WIND (frame, stripe_lk_cbk, trav->xlator,
-                            trav->xlator->fops->lk, fd, cmd, lock);
+                            trav->xlator->fops->lk, fd, cmd, lock, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -2934,14 +2735,14 @@ stripe_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = -1;
 
                 STRIPE_STACK_UNWIND (flush, frame, local->op_ret,
-                                     local->op_errno);
+                                     local->op_errno, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -2961,8 +2762,7 @@ stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 goto err;
         }
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2973,13 +2773,13 @@ stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
         while (trav) {
                 STACK_WIND (frame, stripe_flush_cbk,  trav->xlator,
-                            trav->xlator->fops->flush, fd);
+                            trav->xlator->fops->flush, fd, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (flush, frame, -1, op_errno);
+        STRIPE_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -2988,7 +2788,7 @@ err:
 int32_t
 stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -3024,6 +2824,9 @@ stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->prebuf_blocks  += prebuf->ia_blocks;
                         local->postbuf_blocks += postbuf->ia_blocks;
 
+			correct_file_size(prebuf, local->fctx, prev);
+			correct_file_size(postbuf, local->fctx, prev);
+
                         if (local->prebuf_size < prebuf->ia_size)
                                 local->prebuf_size = prebuf->ia_size;
 
@@ -3046,18 +2849,19 @@ stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STRIPE_STACK_UNWIND (fsync, frame, local->op_ret,
                                      local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
         xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = 1;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -3069,31 +2873,38 @@ stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
+
+	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		op_errno = EINVAL;
+		goto err;
+	}
+	local->fctx = fctx;
+
         local->op_ret = -1;
         frame->local = local;
         local->call_count = priv->child_count;
 
         while (trav) {
                 STACK_WIND (frame, stripe_fsync_cbk, trav->xlator,
-                            trav->xlator->fops->fsync, fd, flags);
+                            trav->xlator->fops->fsync, fd, flags, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 int32_t
 stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -3128,6 +2939,9 @@ stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 local->stbuf = *buf;
 
                         local->stbuf_blocks += buf->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                 }
@@ -3144,7 +2958,7 @@ stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
 
                 STRIPE_STACK_UNWIND (fstat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf);
+                                     local->op_errno, &local->stbuf, NULL);
         }
 
 out:
@@ -3154,11 +2968,12 @@ out:
 int32_t
 stripe_fstat (call_frame_t *frame,
               xlator_t *this,
-              fd_t *fd)
+              fd_t *fd, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
         xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = 1;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -3170,8 +2985,7 @@ stripe_fstat (call_frame_t *frame,
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -3180,26 +2994,35 @@ stripe_fstat (call_frame_t *frame,
         frame->local = local;
         local->call_count = priv->child_count;
 
+	if (IA_ISREG(fd->inode->ia_type)) {
+		inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
+
         while (trav) {
                 STACK_WIND (frame, stripe_fstat_cbk, trav->xlator,
-                            trav->xlator->fops->fstat, fd);
+                            trav->xlator->fops->fstat, fd, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
-stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
+	stripe_fd_ctx_t  *fctx = NULL;
+	int		  i, eof_idx;
+	off_t		  dest_offset, tmp_offset;
+        int32_t		  op_errno = 1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -3207,11 +3030,9 @@ stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         VALIDATE_OR_GOTO (fd->inode, err);
 
         priv = this->private;
-        trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -3220,22 +3041,60 @@ stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         frame->local = local;
         local->call_count = priv->child_count;
 
-        while (trav) {
-                STACK_WIND (frame, stripe_truncate_cbk, trav->xlator,
-                            trav->xlator->fops->ftruncate, fd, offset);
-                trav = trav->next;
-        }
+	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
+		op_errno = EINVAL;
+		goto err;
+	}
+	if (!fctx->stripe_count) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe count");
+		op_errno = EINVAL;
+		goto err;
+	}
+
+	local->fctx = fctx;
+	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
+
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (!fctx->xl_array[i]) {
+			gf_log(this->name, GF_LOG_ERROR, "no xlator at index "
+				"%d", i);
+			op_errno = EINVAL;
+			goto err;
+		}
+
+		if (fctx->stripe_coalesce) {
+			if (i < eof_idx)
+				tmp_offset = roof(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else if (i > eof_idx)
+				tmp_offset = floor(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else
+				tmp_offset = offset;
+
+			dest_offset = coalesced_offset(tmp_offset,
+				fctx->stripe_size, fctx->stripe_count);
+		} else {
+			dest_offset = offset;
+		}
+
+		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
+			fctx->xl_array[i]->fops->ftruncate, fd, dest_offset,
+			NULL);
+	}
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -3272,14 +3131,14 @@ stripe_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = -1;
 
                 STRIPE_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                     local->op_errno);
+                                     local->op_errno, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -3295,8 +3154,7 @@ stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -3307,20 +3165,20 @@ stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
 
         while (trav) {
                 STACK_WIND (frame, stripe_fsyncdir_cbk, trav->xlator,
-                            trav->xlator->fops->fsyncdir, fd, flags);
+                            trav->xlator->fops->fsyncdir, fd, flags, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (fsyncdir, frame, -1, op_errno);
+        STRIPE_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         int32_t         i = 0;
         int32_t         callcnt = 0;
@@ -3330,6 +3188,7 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         struct iatt     tmp_stbuf = {0,};
         struct iobref  *tmp_iobref = NULL;
         struct iobuf   *iobuf = NULL;
+	call_frame_t   *prev = NULL;
 
         if (!this || !frame || !frame->local) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -3337,13 +3196,16 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         local = frame->local;
+	prev = cookie;
 
         LOCK (&frame->lock);
         {
                 callcnt = --local->call_count;
-                if (op_ret != -1)
+                if (op_ret != -1) {
+			correct_file_size(buf, local->fctx, prev);
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
+		}
         }
         UNLOCK (&frame->lock);
 
@@ -3372,7 +3234,8 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 vec[count].iov_len  =
                                         (local->replies[i].requested_size -
                                          local->replies[i].op_ret);
-                                iobuf = iobuf_get (this->ctx->iobuf_pool);
+                                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
+                                                    vec[count].iov_len);
                                 if (!iobuf) {
                                         gf_log (this->name, GF_LOG_ERROR,
                                                 "Out of memory.");
@@ -3381,9 +3244,11 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         goto done;
                                 }
                                 memset (iobuf->ptr, 0, vec[count].iov_len);
-                                iobref_add (local->iobref, iobuf);
                                 vec[count].iov_base = iobuf->ptr;
 
+                                iobref_add (local->iobref, iobuf);
+                                iobuf_unref(iobuf);
+
                                 op_ret += vec[count].iov_len;
                                 count++;
                         }
@@ -3401,11 +3266,10 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 GF_FREE (local->replies);
                 tmp_iobref = local->iobref;
                 STRIPE_STACK_UNWIND (readv, frame, op_ret, op_errno, vec,
-                                     count, &tmp_stbuf, tmp_iobref);
+                                     count, &tmp_stbuf, tmp_iobref, NULL);
 
                 iobref_unref (tmp_iobref);
-                if (vec)
-                        GF_FREE (vec);
+                GF_FREE (vec);
         }
 out:
         return 0;
@@ -3418,7 +3282,7 @@ out:
 int32_t
 stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                  int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
         int32_t         index = 0;
         int32_t         callcnt = 0;
@@ -3432,6 +3296,7 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         struct iatt    *tmp_stbuf_p = NULL; //need it for a warning
         struct iobref  *tmp_iobref = NULL;
         stripe_fd_ctx_t  *fctx = NULL;
+	call_frame_t	*prev = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -3440,6 +3305,7 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local  = frame->local;
         index  = local->node_index;
+	prev = cookie;
         mframe = local->orig_frame;
         if (!mframe)
                 goto out;
@@ -3459,6 +3325,9 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         mlocal->replies[index].stbuf  = *stbuf;
                         mlocal->replies[index].count  = count;
                         mlocal->replies[index].vector = iov_dup (vector, count);
+
+			correct_file_size(stbuf, fctx, prev);
+
                         if (local->stbuf_size < stbuf->ia_size)
                                 local->stbuf_size = stbuf->ia_size;
                         local->stbuf_blocks += stbuf->ia_blocks;
@@ -3530,11 +3399,10 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 tmp_stbuf_p = &tmp_stbuf;
                 WIPE (tmp_stbuf_p);
                 STRIPE_STACK_UNWIND (readv, mframe, op_ret, op_errno, final_vec,
-                                     final_count, &tmp_stbuf, tmp_iobref);
+                                     final_count, &tmp_stbuf, tmp_iobref, NULL);
 
                 iobref_unref (tmp_iobref);
-                if (final_vec)
-                        GF_FREE (final_vec);
+                GF_FREE (final_vec);
         }
 
         goto out;
@@ -3546,7 +3414,7 @@ check_size:
                 STACK_WIND (mframe, stripe_readv_fstat_cbk,
                             (fctx->xl_array[index]),
                             (fctx->xl_array[index])->fops->fstat,
-                            mlocal->fd);
+                            mlocal->fd, NULL);
         }
 
 out:
@@ -3558,7 +3426,7 @@ end:
 
 int32_t
 stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              size_t size, off_t offset, uint32_t flags)
+              size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         int32_t           op_errno = EINVAL;
         int32_t           idx = 0;
@@ -3571,6 +3439,7 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         uint64_t          stripe_size = 0;
         off_t             rounded_start = 0;
         off_t             frame_offset = offset;
+	off_t		  dest_offset = 0;
         stripe_local_t   *local = NULL;
         call_frame_t     *rframe = NULL;
         stripe_local_t   *rlocal = NULL;
@@ -3589,6 +3458,8 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
         stripe_size = fctx->stripe_size;
 
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
         if (!stripe_size) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Wrong stripe size for the file");
@@ -3603,8 +3474,7 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         rounded_end = roof (offset+size, stripe_size);
         num_stripe = (rounded_end- rounded_start)/stripe_size;
 
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -3612,8 +3482,8 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         frame->local = local;
 
         /* This is where all the vectors should be copied. */
-        local->replies = GF_CALLOC (num_stripe, sizeof (struct readv_replies),
-                                    gf_stripe_mt_readv_replies);
+        local->replies = GF_CALLOC (num_stripe, sizeof (struct stripe_replies),
+                                    gf_stripe_mt_stripe_replies);
         if (!local->replies) {
                 op_errno = ENOMEM;
                 goto err;
@@ -3628,8 +3498,7 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         for (index = off_index; index < (num_stripe + off_index); index++) {
                 rframe = copy_frame (frame);
-                rlocal = GF_CALLOC (1, sizeof (stripe_local_t),
-                                    gf_stripe_mt_stripe_local_t);
+                rlocal = mem_get0 (this->local_pool);
                 if (!rlocal) {
                         op_errno = ENOMEM;
                         goto err;
@@ -3643,9 +3512,16 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 rlocal->readv_size = frame_size;
                 rframe->local = rlocal;
                 idx = (index % fctx->stripe_count);
+
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(frame_offset,
+				stripe_size, fctx->stripe_count);
+		else
+			dest_offset = frame_offset;
+
                 STACK_WIND (rframe, stripe_readv_cbk, fctx->xl_array[idx],
                             fctx->xl_array[idx]->fops->readv,
-                            fd, frame_size, frame_offset, flags);
+                            fd, frame_size, dest_offset, flags, xdata);
 
                 frame_offset += frame_size;
         }
@@ -3655,7 +3531,7 @@ err:
         if (rframe)
                 STRIPE_STACK_DESTROY (rframe);
 
-        STRIPE_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        STRIPE_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
         return 0;
 }
 
@@ -3663,11 +3539,15 @@ err:
 int32_t
 stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
         call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+	struct stripe_replies *reply = NULL;
+	int32_t		i = 0;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -3676,39 +3556,82 @@ stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         prev  = cookie;
         local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
 
         LOCK(&frame->lock);
         {
-                callcnt = ++local->call_count;
+                callcnt = ++mlocal->call_count;
+
+		mlocal->replies[local->node_index].op_ret = op_ret;
+		mlocal->replies[local->node_index].op_errno = op_errno;
 
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        local->op_ret = -1;
-                }
                 if (op_ret >= 0) {
-                        local->op_ret += op_ret;
-                        local->post_buf = *postbuf;
-                        local->pre_buf = *prebuf;
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
                 }
         }
         UNLOCK (&frame->lock);
 
-        if ((callcnt == local->wind_count) && local->unwind) {
-                STRIPE_STACK_UNWIND (writev, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+		/*
+		 * Only return the number of consecutively written bytes up until
+		 * the first error. Only return an error if it occurs first.
+		 *
+		 * When a short write occurs, the application should retry at the
+		 * appropriate offset, at which point we'll potentially pass back
+		 * the error.
+		 */
+		for (i = 0, reply = mlocal->replies; i < mlocal->wind_count;
+			i++, reply++) {
+			if (reply->op_ret == -1) {
+				gf_log(this->name, GF_LOG_DEBUG, "reply %d "
+					"returned error %s", i,
+					strerror(reply->op_errno));
+				if (!mlocal->op_ret) {
+					mlocal->op_ret = -1;
+					mlocal->op_errno = reply->op_errno;
+				}
+				break;
+			}
+
+			mlocal->op_ret += reply->op_ret;
+
+			if (reply->op_ret < reply->requested_size)
+				break;
+		}
+
+		GF_FREE(mlocal->replies);
+
+                STRIPE_STACK_UNWIND (writev, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
         }
 out:
+	STRIPE_STACK_DESTROY(frame);
         return 0;
 }
 
 int32_t
 stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                struct iovec *vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref *iobref)
+               uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         struct iovec     *tmp_vec = NULL;
         stripe_local_t   *local = NULL;
@@ -3722,6 +3645,12 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         off_t             fill_size = 0;
         uint64_t          stripe_size = 0;
         uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	off_t		  rounded_start = 0;
+	off_t		  rounded_end = 0;
+	int32_t		  total_chunks = 0;
+	call_frame_t	  *wframe = NULL;
+	stripe_local_t	  *wlocal = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -3736,22 +3665,51 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
         stripe_size = fctx->stripe_size;
 
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
         /* File has to be stripped across the child nodes */
         for (idx = 0; idx< count; idx ++) {
                 total_size += vector[idx].iov_len;
         }
         remaining_size = total_size;
 
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
         frame->local = local;
         local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
 
+	rounded_start = floor(offset, stripe_size);
+	rounded_end = roof(offset + total_size, stripe_size);
+	total_chunks = (rounded_end - rounded_start) / stripe_size;
+	local->replies = GF_CALLOC(total_chunks, sizeof(struct stripe_replies),
+				gf_stripe_mt_stripe_replies);
+	if (!local->replies) {
+		op_errno = ENOMEM;
+		goto err;
+	}
+
+	total_chunks = 0;
         while (1) {
+		wframe = copy_frame(frame);
+		wlocal = mem_get0(this->local_pool);
+		if (!wlocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		wlocal->orig_frame = frame;
+		wframe->local = wlocal;
+
                 /* Send striped chunk of the vector to child
                    nodes appropriately. */
                 idx = (((offset + offset_offset) /
@@ -3779,23 +3737,561 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 if (remaining_size == 0)
                         local->unwind = 1;
 
-                STACK_WIND (frame, stripe_writev_cbk, fctx->xl_array[idx],
+		/*
+		 * Store off the request index (with respect to the chunk of the
+		 * initial offset) and the size of the request. This is required
+		 * in the callback to calculate an appropriate return value in
+		 * the event of a write failure in one or more requests.
+		 */
+		wlocal->node_index = total_chunks;
+		local->replies[total_chunks].requested_size = fill_size;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+                STACK_WIND (wframe, stripe_writev_cbk, fctx->xl_array[idx],
                             fctx->xl_array[idx]->fops->writev, fd, tmp_vec,
-                            tmp_count, offset + offset_offset, flags, iobref);
+                            tmp_count, dest_offset, flags, iobref,
+                            xdata);
+
                 GF_FREE (tmp_vec);
                 offset_offset += fill_size;
+		total_chunks++;
                 if (remaining_size == 0)
                         break;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+	if (wframe)
+		STRIPE_STACK_DESTROY(wframe);
+
+        STRIPE_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
+stripe_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+		/* return the first failure */
+		if (mlocal->op_ret == 0) {
+			mlocal->op_ret = op_ret;
+			mlocal->op_errno = op_errno;
+		}
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (fallocate, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		 off_t offset, size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
+        uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	call_frame_t	  *fframe = NULL;
+	stripe_local_t	  *flocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+		fframe = copy_frame(frame);
+		flocal = mem_get0(this->local_pool);
+		if (!flocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		flocal->orig_frame = frame;
+		fframe->local = flocal;
+
+		/* send fallocate request to the associated child node */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+		/*
+		 * TODO: Create a separate handler for coalesce mode that sends a
+		 * single fallocate per-child (since the ranges are linear).
+		 */
+		STACK_WIND(fframe, stripe_fallocate_cbk, fctx->xl_array[idx],
+			   fctx->xl_array[idx]->fops->fallocate, fd, mode,
+			   dest_offset, fill_size, xdata);
+
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (fframe)
+		STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+stripe_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+		/* return the first failure */
+		if (mlocal->op_ret == 0) {
+			mlocal->op_ret = op_ret;
+			mlocal->op_errno = op_errno;
+		}
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (discard, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
+        uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	call_frame_t	  *fframe = NULL;
+	stripe_local_t	  *flocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+		fframe = copy_frame(frame);
+		flocal = mem_get0(this->local_pool);
+		if (!flocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		flocal->orig_frame = frame;
+		fframe->local = flocal;
+
+		/* send discard request to the associated child node */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+		/*
+		 * TODO: Create a separate handler for coalesce mode that sends a
+		 * single discard per-child (since the ranges are linear).
+		 */
+		STACK_WIND(fframe, stripe_discard_cbk, fctx->xl_array[idx],
+			   fctx->xl_array[idx]->fops->discard, fd, dest_offset,
+			   fill_size, xdata);
+
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (fframe)
+		STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+stripe_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt    = 0;
+        stripe_local_t *local      = NULL;
+        stripe_local_t *mlocal     = NULL;
+        call_frame_t   *prev       = NULL;
+        call_frame_t   *mframe     = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+        mframe = local->orig_frame;
+        mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+                        mlocal->prebuf_blocks  += prebuf->ia_blocks;
+                        mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+                        correct_file_size(prebuf, mlocal->fctx, prev);
+                        correct_file_size(postbuf, mlocal->fctx, prev);
+
+                        if (mlocal->prebuf_size < prebuf->ia_size)
+                                mlocal->prebuf_size = prebuf->ia_size;
+                        if (mlocal->postbuf_size < postbuf->ia_size)
+                                mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+                /* return the first failure */
+                if (mlocal->op_ret == 0) {
+                        mlocal->op_ret = op_ret;
+                        mlocal->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+                mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+                mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+                mlocal->post_buf.ia_size = mlocal->postbuf_size;
+                mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (zerofill, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+        STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local            = NULL;
+        stripe_fd_ctx_t  *fctx             = NULL;
+        int32_t           op_errno         = 1;
+        int32_t           idx              = 0;
+        int32_t           offset_offset    = 0;
+        int32_t           remaining_size   = 0;
+        off_t             fill_size        = 0;
+        uint64_t          stripe_size      = 0;
+        uint64_t          tmp_fctx         = 0;
+        off_t             dest_offset      = 0;
+        call_frame_t     *fframe           = NULL;
+        stripe_local_t   *flocal           = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+        local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+                fframe = copy_frame(frame);
+                flocal = mem_get0(this->local_pool);
+                if (!flocal) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+                flocal->orig_frame = frame;
+                fframe->local = flocal;
+
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+                dest_offset = offset + offset_offset;
+                if (fctx->stripe_coalesce)
+                        dest_offset = coalesced_offset(dest_offset,
+                                                       local->stripe_size,
+                                                       fctx->stripe_count);
+
+                STACK_WIND(fframe, stripe_zerofill_cbk, fctx->xl_array[idx],
+                           fctx->xl_array[idx]->fops->zerofill, fd,
+                           dest_offset, fill_size, xdata);
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+        if (fframe)
+                STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
 stripe_release (xlator_t *this, fd_t *fd)
 {
 	return 0;
@@ -3831,6 +4327,7 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         stripe_private_t *priv = NULL;
         int               down_client = 0;
         int               i = 0;
+        gf_boolean_t      heard_from_all_children = _gf_false;
 
         if (!this)
                 return 0;
@@ -3842,30 +4339,34 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         switch (event)
         {
         case GF_EVENT_CHILD_UP:
-        case GF_EVENT_CHILD_CONNECTING:
         {
                 /* get an index number to set */
                 for (i = 0; i < priv->child_count; i++) {
                         if (data == priv->xl_array[i])
                                 break;
                 }
-                priv->state[i] = 1;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!priv->state[i])
-                                down_client++;
+
+                if (priv->child_count == i) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "got GF_EVENT_CHILD_UP bad subvolume %s",
+                                data? ((xlator_t *)data)->name: NULL);
+                        break;
                 }
 
                 LOCK (&priv->lock);
                 {
-                        priv->nodes_down = down_client;
                         if (data == FIRST_CHILD (this))
                                 priv->first_child_down = 0;
-                        if (!priv->nodes_down)
-                                default_notify (this, event, data);
+                        priv->last_event[i] = event;
                 }
                 UNLOCK (&priv->lock);
         }
         break;
+        case GF_EVENT_CHILD_CONNECTING:
+        {
+                // 'CONNECTING' doesn't ensure its CHILD_UP, so do nothing
+                goto out;
+        }
         case GF_EVENT_CHILD_DOWN:
         {
                 /* get an index number to set */
@@ -3873,20 +4374,19 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                         if (data == priv->xl_array[i])
                                 break;
                 }
-                priv->state[i] = 0;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!priv->state[i])
-                                down_client++;
+
+                if (priv->child_count == i) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
+                                data? ((xlator_t *)data)->name: NULL);
+                        break;
                 }
 
                 LOCK (&priv->lock);
                 {
-                        priv->nodes_down = down_client;
-
                         if (data == FIRST_CHILD (this))
                                 priv->first_child_down = 1;
-                        if (priv->nodes_down)
-                                default_notify (this, event, data);
+                        priv->last_event[i] = event;
                 }
                 UNLOCK (&priv->lock);
         }
@@ -3896,190 +4396,356 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         {
                 /* */
                 default_notify (this, event, data);
+                goto out;
         }
         break;
         }
 
+        // Consider child as down if it's last_event is not CHILD_UP
+        for (i = 0, down_client = 0; i < priv->child_count; i++)
+                if (priv->last_event[i] != GF_EVENT_CHILD_UP)
+                        down_client++;
+
+        LOCK (&priv->lock);
+        {
+                priv->nodes_down = down_client;
+        }
+        UNLOCK (&priv->lock);
+
+        heard_from_all_children = _gf_true;
+        for (i = 0; i < priv->child_count; i++)
+                if (!priv->last_event[i])
+                        heard_from_all_children = _gf_false;
+
+        if (heard_from_all_children)
+                default_notify (this, event, data);
+out:
         return 0;
 }
 
 int
-set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
+stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
+                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
 {
-        int                    ret = -1;
-        char                  *tmp_str = NULL;
-        char                  *tmp_str1 = NULL;
-        char                  *dup_str = NULL;
-        char                  *stripe_str = NULL;
-        char                  *pattern = NULL;
-        char                  *num = NULL;
-        struct stripe_options *temp_stripeopt = NULL;
-        struct stripe_options *stripe_opt = NULL;
-
-        if (!this || !priv || !data)
-                goto out;
+        int ret = -1;
+        int call_cnt = 0;
+        stripe_local_t *local = NULL;
 
-        /* Get the pattern for striping.
-           "option block-size *avi:10MB" etc */
-        stripe_str = strtok_r (data, ",", &tmp_str);
-        while (stripe_str) {
-                dup_str = gf_strdup (stripe_str);
-                stripe_opt = CALLOC (1, sizeof (struct stripe_options));
-                if (!stripe_opt) {
-                        GF_FREE (dup_str);
-                        goto out;
-                }
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
+                return ret;
+        }
 
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                num = strtok_r (NULL, ":", &tmp_str1);
-                if (!num) {
-                        num = pattern;
-                        pattern = "*";
-                }
-                if (gf_string2bytesize (num, &stripe_opt->block_size) != 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid number format \"%s\"", num);
-                        goto out;
-                }
+        local = frame->local;
 
-                if (stripe_opt->block_size < 512) {
-                        gf_log (this->name, GF_LOG_ERROR, "Invalid Block-size: "
-                                "%s. Should be atleast 512 bytes", num);
-                        goto out;
-                }
-                if (stripe_opt->block_size % 512) {
-                        gf_log (this->name, GF_LOG_ERROR, "Block-size: %s should"
-                                " be a multiple of 512 bytes", num);
-                        goto out;
+        LOCK (&frame->lock);
+        {
+                call_cnt = --local->wind_count;
+
+                /**
+                 * We overwrite ->op_* values here for subsequent faliure
+                 * conditions, hence we propogate the last errno down the
+                 * stack.
+                 */
+                if (op_ret < 0) {
+                        local->op_ret = op_ret;
+                        local->op_errno = op_errno;
+                        goto unlock;
                 }
+        }
 
-                memcpy (stripe_opt->path_pattern, pattern, strlen (pattern));
+ unlock:
+        UNLOCK (&frame->lock);
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "block-size : pattern %s : size %"PRId64,
-                        stripe_opt->path_pattern, stripe_opt->block_size);
+        if (!call_cnt) {
+                STRIPE_STACK_UNWIND (setxattr, frame, local->op_ret,
+                                     local->op_errno, xdata);
+        }
 
-                if (!priv->pattern) {
-                        priv->pattern = stripe_opt;
-                } else {
-                        temp_stripeopt = priv->pattern;
-                        while (temp_stripeopt->next)
-                                temp_stripeopt = temp_stripeopt->next;
-                        temp_stripeopt->next = stripe_opt;
+        return 0;
+}
+
+#ifdef HAVE_BD_XLATOR
+int
+stripe_is_bd (dict_t *this, char *key, data_t *value, void *data)
+{
+        gf_boolean_t *is_bd = data;
+
+        if (data == NULL)
+                return 0;
+
+        if (XATTR_IS_BD (key))
+            *is_bd = _gf_true;
+
+        return 0;
+}
+
+inline gf_boolean_t
+stripe_setxattr_is_bd (dict_t *dict)
+{
+        gf_boolean_t is_bd = _gf_false;
+
+        if (dict == NULL)
+                goto out;
+
+        dict_foreach (dict, stripe_is_bd, &is_bd);
+out:
+        return is_bd;
+}
+#else
+#define stripe_setxattr_is_bd(dict) _gf_false
+#endif
+
+int
+stripe_setxattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
+{
+        int32_t           op_errno = EINVAL;
+        xlator_list_t    *trav     = NULL;
+        stripe_private_t *priv     = NULL;
+        stripe_local_t   *local    = NULL;
+        int               i        = 0;
+        gf_boolean_t      is_bd    = _gf_false;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
+                                   op_errno, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        frame->local = local;
+        local->wind_count = priv->child_count;
+        local->op_ret = local->op_errno = 0;
+
+        is_bd = stripe_setxattr_is_bd (dict);
+
+        /**
+         * Set xattrs for directories on all subvolumes. Additionally
+         * this power is only given to a special client. Bd xlator
+         * also needs xattrs for regular files (ie LVs)
+         */
+        if (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&
+             IA_ISDIR (loc->inode->ia_type)) || is_bd) {
+                for (i = 0; i < priv->child_count; i++, trav = trav->next) {
+                        STACK_WIND (frame, stripe_setxattr_cbk,
+                                    trav->xlator, trav->xlator->fops->setxattr,
+                                    loc, dict, flags, xdata);
                 }
-                stripe_str = strtok_r (NULL, ",", &tmp_str);
-                GF_FREE (dup_str);
+        } else {
+                local->wind_count = 1;
+                STACK_WIND (frame, stripe_setxattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr,
+                            loc, dict, flags, xdata);
         }
 
-        ret = 0;
-out:
-        return ret;
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (setxattr, frame, -1,  op_errno, NULL);
+        return 0;
 }
 
-int32_t
-stripe_iatt_merge (struct iatt *from, struct iatt *to)
+
+int
+stripe_fsetxattr_cbk (call_frame_t *frame, void *cookie,
+                      xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
 {
-        if (to->ia_size < from->ia_size)
-                to->ia_size = from->ia_size;
-        if (to->ia_mtime < from->ia_mtime)
-                to->ia_mtime = from->ia_mtime;
-        if (to->ia_ctime < from->ia_ctime)
-                to->ia_ctime = from->ia_ctime;
-        if (to->ia_atime < from->ia_atime)
-                to->ia_atime = from->ia_atime;
+        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-int32_t
-stripe_readdirp_entry_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno, struct iatt *buf)
+
+int
+stripe_is_special_key (dict_t *this,
+                       char *key,
+                       data_t *value,
+                       void *data)
 {
-        gf_dirent_t    *entry = NULL;
-        stripe_local_t *local = NULL;
-        int32_t        done = 0;
+        gf_boolean_t *is_special = NULL;
 
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log (this->name, GF_LOG_DEBUG, "possible NULL deref");
+        if (data == NULL) {
                 goto out;
         }
-        entry = cookie;
+
+        is_special = data;
+
+        if (XATTR_IS_LOCKINFO (key) || XATTR_IS_BD (key))
+                *is_special = _gf_true;
+
+out:
+        return 0;
+}
+
+int32_t
+stripe_fsetxattr_everyone_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *xdata)
+{
+        int             call_count = 0;
+        stripe_local_t *local      = NULL;
+
         local = frame->local;
+
         LOCK (&frame->lock);
         {
+                call_count = --local->wind_count;
 
-                local->wind_count--;
-                if (!local->wind_count)
-                        done = 1;
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
+                if (op_ret < 0) {
                         local->op_ret = op_ret;
-                        goto unlock;
+                        local->op_errno = op_errno;
                 }
-                stripe_iatt_merge (buf, &entry->d_stat);
         }
-unlock:
-        UNLOCK(&frame->lock);
+        UNLOCK (&frame->lock);
 
-        if (done) {
-                frame->local = NULL;
-                STRIPE_STACK_UNWIND (readdir, frame, local->op_ret,
-                                     local->op_errno, &local->entries);
+        if (call_count == 0) {
+                STRIPE_STACK_UNWIND (fsetxattr, frame, local->op_ret,
+                                     local->op_errno, NULL);
+        }
+        return 0;
+}
 
-                gf_dirent_free (&local->entries);
-                stripe_local_wipe (local);
-                GF_FREE (local);
+int
+stripe_fsetxattr_to_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              dict_t *dict, int flags, dict_t *xdata)
+{
+        xlator_list_t    *trav  = NULL;
+        stripe_private_t *priv  = NULL;
+        int               ret   = -1;
+        stripe_local_t   *local = NULL;
+
+        priv = this->private;
+
+        local = mem_get0 (this->local_pool);
+        if (local == NULL) {
+                goto out;
+        }
+
+        frame->local = local;
+
+        local->wind_count = priv->child_count;
+
+        trav = this->children;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_fsetxattr_everyone_cbk,
+                            trav->xlator, trav->xlator->fops->fsetxattr,
+                            fd, dict, flags, xdata);
+                trav = trav->next;
         }
+
+        ret = 0;
 out:
-        return 0;
+        return ret;
+}
+
+inline gf_boolean_t
+stripe_fsetxattr_is_special (dict_t *dict)
+{
+        gf_boolean_t is_spl = _gf_false;
+
+        if (dict == NULL) {
+                goto out;
+        }
+
+        dict_foreach (dict, stripe_is_special_key, &is_spl);
 
+out:
+        return is_spl;
 }
 
 int
-stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
-                     xlator_t *this, int op_ret, int op_errno)
+stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  dict_t *dict, int flags, dict_t *xdata)
 {
-        STRIPE_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+        int32_t      op_ret = -1, ret = -1, op_errno = EINVAL;
+        gf_boolean_t is_spl = _gf_false;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
+                                   op_errno, err);
+
+        is_spl = stripe_fsetxattr_is_special (dict);
+        if (is_spl) {
+                ret = stripe_fsetxattr_to_everyone (frame, this, fd, dict,
+                                                    flags, xdata);
+                if (ret < 0) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+
+                goto out;
+        }
+
+        STACK_WIND (frame, stripe_fsetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+out:
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
         return 0;
 }
 
 int
-stripe_setxattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *dict, int flags)
+stripe_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        STRIPE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+stripe_removexattr (call_frame_t *frame, xlator_t *this,
+                    loc_t *loc, const char *name, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
         int32_t         op_errno = EINVAL;
 
-        VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   trav, op_errno, err);
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
+                                 name, op_errno, err);
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-        STACK_WIND (frame, stripe_setxattr_cbk,
+        STACK_WIND (frame, stripe_removexattr_cbk,
                     FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    FIRST_CHILD(this)->fops->removexattr,
+                    loc, name, xdata);
         return 0;
 err:
-        STRIPE_STACK_UNWIND (setxattr, frame, -1,  op_errno);
+        STRIPE_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
         return 0;
 }
 
 
 int
-stripe_fsetxattr_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno)
+stripe_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
+        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
-stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  dict_t *dict, int flags)
+stripe_fremovexattr (call_frame_t *frame, xlator_t *this,
+                     fd_t *fd, const char *name, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
         int32_t         op_ret   = -1;
         int32_t         op_errno = EINVAL;
 
@@ -4087,35 +4753,116 @@ stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   trav, op_errno, err);
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
+                                 name, op_errno, err);
 
-        STACK_WIND (frame, stripe_fsetxattr_cbk,
+        STACK_WIND (frame, stripe_fremovexattr_cbk,
                     FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, name, xdata);
         return 0;
  err:
-        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
+        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+stripe_readdirp_lookup_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int op_ret, int op_errno,
+                            inode_t *inode, struct iatt *stbuf,
+                            dict_t *xattr, struct iatt *parent)
+{
+        stripe_local_t          *local          = NULL;
+        call_frame_t            *main_frame     = NULL;
+        stripe_local_t          *main_local     = NULL;
+        gf_dirent_t             *entry          = NULL;
+        call_frame_t            *prev           = NULL;
+        int                      done           = 0;
+
+        local = frame->local;
+        prev = cookie;
+
+        entry = local->dirent;
+
+        main_frame = local->orig_frame;
+        main_local = main_frame->local;
+        LOCK (&frame->lock);
+        {
+
+                local->call_count--;
+                if (!local->call_count)
+                        done = 1;
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        local->op_ret = op_ret;
+                        goto unlock;
+                }
+
+		if (stripe_ctx_handle(this, prev, local, xattr))
+			gf_log(this->name, GF_LOG_ERROR,
+				"Error getting fctx info from dict.");
+
+		correct_file_size(stbuf, local->fctx, prev);
+
+                stripe_iatt_merge (stbuf, &entry->d_stat);
+                local->stbuf_blocks += stbuf->ia_blocks;
+        }
+unlock:
+        UNLOCK(&frame->lock);
+
+        if (done) {
+                inode_ctx_put (entry->inode, this,
+                               (uint64_t) (long)local->fctx);
+
+                done = 0;
+                LOCK (&main_frame->lock);
+                {
+                        main_local->wind_count--;
+                        if (!main_local->wind_count)
+                                done = 1;
+                        if (local->op_ret == -1) {
+                                main_local->op_errno = local->op_errno;
+                                main_local->op_ret = local->op_ret;
+                        }
+                        entry->d_stat.ia_blocks = local->stbuf_blocks;
+                }
+                UNLOCK (&main_frame->lock);
+                if (done) {
+                        main_frame->local = NULL;
+                        STRIPE_STACK_UNWIND (readdir, main_frame,
+                                             main_local->op_ret,
+                                             main_local->op_errno,
+                                             &main_local->entries, NULL);
+                        gf_dirent_free (&main_local->entries);
+                        stripe_local_wipe (main_local);
+                        mem_put (main_local);
+                }
+                frame->local = NULL;
+                stripe_local_wipe (local);
+                mem_put (local);
+                STRIPE_STACK_DESTROY (frame);
+        }
+
         return 0;
 }
 
 int32_t
 stripe_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *orig_entries)
+                     int32_t op_ret, int32_t op_errno,
+                     gf_dirent_t *orig_entries, dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         call_frame_t   *prev = NULL;
         gf_dirent_t    *local_entry = NULL;
-        int32_t        ret = -1;
         gf_dirent_t    *tmp_entry = NULL;
         xlator_list_t  *trav = NULL;
         loc_t          loc = {0, };
-        inode_t        *inode = NULL;
-        char           *path;
         int32_t        count = 0;
         stripe_private_t *priv = NULL;
         int32_t        subvols = 0;
+        dict_t         *xattrs = NULL;
+        call_frame_t   *local_frame = NULL;
+        stripe_local_t *local_ent = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -4141,8 +4888,9 @@ stripe_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = op_ret;
                         list_splice_init (&orig_entries->list,
                                           &local->entries.list);
-                        local->wind_count = op_ret * subvols;
+                        local->wind_count = op_ret;
                 }
+
         }
 unlock:
         UNLOCK (&frame->lock);
@@ -4150,8 +4898,10 @@ unlock:
         if (op_ret == -1)
                 goto out;
 
+        xattrs = dict_new ();
+        if (xattrs)
+                (void) stripe_xattr_request_build (this, xattrs, 0, 0, 0, 0);
         count = op_ret;
-        ret = 0;
         list_for_each_entry_safe (local_entry, tmp_entry,
                                   (&local->entries.list), list) {
 
@@ -4160,68 +4910,73 @@ unlock:
                 if (!IA_ISREG (local_entry->d_stat.ia_type)) {
                         LOCK (&frame->lock);
                         {
-                                local->wind_count -= subvols;
+                                local->wind_count--;
                                 count = local->wind_count;
                         }
                         UNLOCK (&frame->lock);
                         continue;
                 }
 
-                inode = inode_new (local->fd->inode->table);
-                if (!inode)
+                local_frame = copy_frame (frame);
+
+                if (!local_frame) {
+                        op_errno = ENOMEM;
+                        op_ret = -1;
                         goto out;
+                }
 
-                loc.inode = inode;
-                loc.parent = local->fd->inode;
-                ret = inode_path (local->fd->inode, local_entry->d_name, &path);
-                if (ret != -1) {
-                        loc.path = path;
-                } else  if (inode) {
-                        ret = inode_path (inode, NULL, &path);
-                        if (ret != -1) {
-                                loc.path = path;
-                        } else {
-                                goto out;
-                        }
+                local_ent = mem_get0 (this->local_pool);
+                if (!local_ent) {
+                        op_errno = ENOMEM;
+                        op_ret = -1;
+                        goto out;
                 }
 
-                loc.name = strrchr (loc.path, '/');
-                loc.name++;
+                loc.inode = inode_ref (local_entry->inode);
+
                 uuid_copy (loc.gfid, local_entry->d_stat.ia_gfid);
 
+                local_ent->orig_frame = frame;
+
+                local_ent->call_count = subvols;
+
+                local_ent->dirent = local_entry;
+
+                local_frame->local = local_ent;
+
                 trav = this->children;
                 while (trav) {
-                        STACK_WIND_COOKIE (frame, stripe_readdirp_entry_stat_cbk,
-                                           local_entry, trav->xlator,
-                                           trav->xlator->fops->stat, &loc);
+                        STACK_WIND (local_frame, stripe_readdirp_lookup_cbk,
+                                    trav->xlator, trav->xlator->fops->lookup,
+                                    &loc, xattrs);
                         trav = trav->next;
                 }
-                inode_unref (loc.inode);
+                loc_wipe (&loc);
         }
 out:
         if (!count) {
                 /* all entries are directories */
                 frame->local = NULL;
                 STRIPE_STACK_UNWIND (readdir, frame, local->op_ret,
-                                     local->op_errno, &local->entries);
+                                     local->op_errno, &local->entries, NULL);
                 gf_dirent_free (&local->entries);
                 stripe_local_wipe (local);
-                GF_FREE (local);
+                mem_put (local);
         }
-
+        if (xattrs)
+                dict_unref (xattrs);
         return 0;
 
 }
 int32_t
 stripe_readdirp (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, size_t size, off_t off, dict_t *dict)
+                 fd_t *fd, size_t size, off_t off, dict_t *xdata)
 {
         stripe_local_t  *local  = NULL;
         stripe_private_t *priv = NULL;
         xlator_list_t   *trav = NULL;
         int             op_errno = -1;
 
-
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
@@ -4235,8 +4990,7 @@ stripe_readdirp (call_frame_t *frame, xlator_t *this,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -4256,15 +5010,16 @@ stripe_readdirp (call_frame_t *frame, xlator_t *this,
                 goto err;
 
         STACK_WIND (frame, stripe_readdirp_cbk, trav->xlator,
-                    trav->xlator->fops->readdirp, fd, size, off, dict);
+                    trav->xlator->fops->readdirp, fd, size, off, xdata);
         return 0;
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        STRIPE_STACK_UNWIND (readdir, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 
 }
+
 int32_t
 mem_acct_init (xlator_t *this)
 {
@@ -4285,21 +5040,86 @@ out:
         return ret;
 }
 
+static int
+clear_pattern_list (stripe_private_t *priv)
+{
+        struct stripe_options *prev = NULL;
+        struct stripe_options *trav = NULL;
+        int                    ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
+
+        trav = priv->pattern;
+        priv->pattern = NULL;
+        while (trav) {
+                prev = trav;
+                trav = trav->next;
+                GF_FREE (prev);
+        }
+
+        ret = 0;
+ out:
+        return ret;
+
+
+}
+
 
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
 
-	stripe_private_t *priv = NULL;
-	int		  ret = -1;
+        stripe_private_t *priv = NULL;
+        data_t           *data = NULL;
+        int               ret = -1;
+        volume_option_t  *opt = NULL;
+
+        GF_ASSERT (this);
+        GF_ASSERT (this->private);
 
-	priv = this->private;
+        priv = this->private;
 
-        GF_OPTION_RECONF ("block-size", priv->block_size, options, size, out);
 
         ret = 0;
-out:
-	return ret;
+        LOCK (&priv->lock);
+        {
+                ret = clear_pattern_list (priv);
+                if (ret)
+                         goto unlock;
+
+                data = dict_get (options, "block-size");
+                if (data) {
+                        ret = set_stripe_block_size (this, priv, data->data);
+                        if (ret)
+                                goto unlock;
+                } else {
+                        opt = xlator_volume_option_get (this, "block-size");
+                        if (!opt) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "option 'block-size' not found");
+                                ret = -1;
+                                goto unlock;
+                        }
+
+                        if (gf_string2bytesize (opt->default_value, &priv->block_size)){
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to set default block-size ");
+                                ret = -1;
+                                goto unlock;
+                        }
+                }
+
+		GF_OPTION_RECONF("coalesce", priv->coalesce, options, bool,
+				unlock);
+        }
+ unlock:
+        UNLOCK (&priv->lock);
+        if (ret)
+                goto out;
+
+        ret = 0;
+ out:
+        return ret;
 
 }
 
@@ -4312,6 +5132,7 @@ int32_t
 init (xlator_t *this)
 {
         stripe_private_t *priv = NULL;
+        volume_option_t  *opt = NULL;
         xlator_list_t    *trav = NULL;
         data_t           *data = NULL;
         int32_t           count = 0;
@@ -4355,9 +5176,9 @@ init (xlator_t *this)
         if (!priv->xl_array)
                 goto out;
 
-        priv->state = GF_CALLOC (count, sizeof (int8_t),
-                                 gf_stripe_mt_int8_t);
-        if (!priv->state)
+        priv->last_event = GF_CALLOC (count, sizeof (int),
+                                      gf_stripe_mt_int32_t);
+        if (!priv->last_event)
                 goto out;
 
         priv->child_count = count;
@@ -4377,30 +5198,56 @@ init (xlator_t *this)
                 goto out;
         }
 
-
-        GF_OPTION_INIT ("block-size", priv->block_size, size, out);
-
-        /* option stripe-pattern *avi:1GB,*pdf:4096 */
-        data = dict_get (this->options, "block-size");
-        if (data) {
-                ret = set_stripe_block_size (this, priv, data->data);
-                if (ret)
-                        goto out;
+        ret = 0;
+        LOCK (&priv->lock);
+        {
+                opt = xlator_volume_option_get (this, "block-size");
+                if (!opt) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "option 'block-size' not found");
+                        ret = -1;
+                        goto unlock;
+                }
+                if (gf_string2bytesize (opt->default_value, &priv->block_size)){
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set default block-size ");
+                        ret = -1;
+                        goto unlock;
+                }
+                /* option stripe-pattern *avi:1GB,*pdf:16K */
+                data = dict_get (this->options, "block-size");
+                if (data) {
+                        ret = set_stripe_block_size (this, priv, data->data);
+                        if (ret)
+                                goto unlock;
+                }
         }
+ unlock:
+        UNLOCK (&priv->lock);
+        if (ret)
+                goto out;
 
         GF_OPTION_INIT ("use-xattr", priv->xattr_supported, bool, out);
-
         /* notify related */
         priv->nodes_down = priv->child_count;
-        this->private = priv;
 
+	GF_OPTION_INIT("coalesce", priv->coalesce, bool, out);
+
+        this->local_pool = mem_pool_new (stripe_local_t, 128);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
+        this->private = priv;
 
         ret = 0;
 out:
         if (ret) {
                 if (priv) {
-                        if (priv->xl_array)
-                                GF_FREE (priv->xl_array);
+                        GF_FREE (priv->xl_array);
                         GF_FREE (priv);
                 }
         }
@@ -4424,15 +5271,15 @@ fini (xlator_t *this)
         priv = this->private;
         if (priv) {
                 this->private = NULL;
-                if (priv->xl_array)
-                        GF_FREE (priv->xl_array);
+                GF_FREE (priv->xl_array);
 
                 trav = priv->pattern;
                 while (trav) {
                         prev = trav;
                         trav = trav->next;
-                        FREE (prev);
+                        GF_FREE (prev);
                 }
+                GF_FREE (priv->last_event);
                 LOCK_DESTROY (&priv->lock);
                 GF_FREE (priv);
         }
@@ -4443,17 +5290,50 @@ out:
 
 int32_t
 stripe_getxattr_unwind (call_frame_t *frame,
-                        int op_ret, int op_errno, dict_t *dict)
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 
 {
-        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
+int
+stripe_internal_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int op_ret, int op_errno, dict_t *xattr,
+                              dict_t *xdata)
+{
+
+        char        size_key[256]  = {0,};
+        char        index_key[256] = {0,};
+        char        count_key[256] = {0,};
+        char        coalesce_key[256] = {0,};
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
+
+        if (!xattr || (op_ret == -1))
+            goto out;
+
+        sprintf (size_key, "trusted.%s.stripe-size", this->name);
+        sprintf (count_key, "trusted.%s.stripe-count", this->name);
+        sprintf (index_key, "trusted.%s.stripe-index", this->name);
+	sprintf (coalesce_key, "trusted.%s.stripe-coalesce", this->name);
+
+        dict_del (xattr, size_key);
+        dict_del (xattr, count_key);
+        dict_del (xattr, index_key);
+        dict_del (xattr, coalesce_key);
+
+out:
+        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+
+        return 0;
+
+}
 
 int
 stripe_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xattr)
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
         int                     call_cnt = 0;
         stripe_local_t         *local = NULL;
@@ -4483,92 +5363,39 @@ stripe_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         if (!call_cnt) {
                 STRIPE_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
-                                     local->xattr);
+                                     local->xattr, xdata);
         }
 
         return 0;
 }
 
 int32_t
-stripe_pathinfo_aggregate (char *buffer, stripe_local_t *local, int32_t *total)
+stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
+                      xlator_t *this, int32_t op_ret, int32_t op_errno,
+                      dict_t *dict, dict_t *xdata)
 {
-        int32_t              i     = 0;
-        int32_t              ret   = -1;
-        int32_t              len   = 0;
-        char                *sbuf  = NULL;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (!buffer || !local || !local->xattr_list)
-                goto out;
-
-        sbuf = buffer;
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-                len = xattr->pathinfo_len;
-
-                if (len && xattr && xattr->pathinfo) {
-                        memcpy (buffer, xattr->pathinfo, len);
-                        buffer += len;
-                        *buffer++ = ' ';
-                }
-        }
-
-        *--buffer = '\0';
-        if (total)
-                *total = buffer - sbuf;
-        ret = 0;
-
- out:
-        return ret;
-}
-
-int32_t
-stripe_free_pathinfo_str (stripe_local_t *local)
-{
-        int32_t              i     = 0;
-        int32_t              ret   = -1;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (!local || !local->xattr_list)
-                goto out;
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-
-                if (xattr && xattr->pathinfo)
-                        GF_FREE (xattr->pathinfo);
-        }
-
-        ret = 0;
- out:
-        return ret;
-}
-
-int32_t
-stripe_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             dict_t *dict) {
         stripe_local_t      *local         = NULL;
         int32_t              callcnt       = 0;
         int32_t              ret           = -1;
         long                 cky           = 0;
-        char                *pathinfo      = NULL;
-        char                *pathinfo_serz = NULL;
-        int32_t              padding       = 0;
-        int32_t              tlen          = 0;
-        char stripe_size_str[20]           = {0,};
+        void                *xattr_val     = NULL;
+        void                *xattr_serz    = NULL;
         stripe_xattr_sort_t *xattr         = NULL;
         dict_t              *stripe_xattr  = NULL;
 
         if (!frame || !frame->local || !this) {
-                gf_log (this->name, GF_LOG_ERROR, "Possible NULL deref");
+                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
                 return ret;
         }
 
         local = frame->local;
         cky = (long) cookie;
 
+        if (local->xsel[0] == '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "Empty xattr in cbk");
+                return ret;
+        }
+
         LOCK (&frame->lock);
         {
                 callcnt = --local->wind_count;
@@ -4577,23 +5404,26 @@ stripe_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                         goto out;
 
                 if (!local->xattr_list)
-                        local->xattr_list = (stripe_xattr_sort_t *) GF_CALLOC (local->nallocs,
-                                                                               sizeof (stripe_xattr_sort_t),
-                                                                               gf_stripe_mt_xattr_sort_t);
+                        local->xattr_list = (stripe_xattr_sort_t *)
+                                GF_CALLOC (local->nallocs,
+                                           sizeof (stripe_xattr_sort_t),
+                                           gf_stripe_mt_xattr_sort_t);
 
                 if (local->xattr_list) {
-                        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
-                        if (ret)
-                                goto out;
-
                         xattr = local->xattr_list + (int32_t) cky;
 
-                        pathinfo = gf_strdup (pathinfo);
+                        ret = dict_get_ptr_and_len (dict, local->xsel,
+                                                    &xattr_val,
+                                                    &xattr->xattr_len);
+                        if (xattr->xattr_len == 0)
+                                goto out;
+
                         xattr->pos = cky;
-                        xattr->pathinfo = pathinfo;
-                        xattr->pathinfo_len = strlen (pathinfo);
+                        xattr->xattr_value = gf_memdup (xattr_val,
+                                                        xattr->xattr_len);
 
-                        local->xattr_total_len += strlen (pathinfo) + 1;
+                        if (xattr->xattr_value != NULL)
+                                local->xattr_total_len += xattr->xattr_len + 1;
                 }
         }
  out:
@@ -4607,41 +5437,36 @@ stripe_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                 if (!stripe_xattr)
                         goto unwind;
 
-                snprintf (stripe_size_str, 20, "%ld", local->stripe_size);
-
-                /* extra bytes for decorations (brackets and <>'s) */
-                padding = strlen (this->name) + strlen (STRIPE_PATHINFO_HEADER)
-                        + strlen (stripe_size_str) + 7;
-                local->xattr_total_len += (padding + 2);
-
-                pathinfo_serz = GF_CALLOC (local->xattr_total_len, sizeof (char),
-                                           gf_common_mt_char);
-                if (!pathinfo_serz)
-                        goto unwind;
-
-                /* xlator info */
-                sprintf (pathinfo_serz, "(<"STRIPE_PATHINFO_HEADER"%s:[%s]> ", this->name, stripe_size_str);
-
-                ret = stripe_pathinfo_aggregate (pathinfo_serz + padding, local, &tlen);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot aggregate pathinfo list");
+                /* select filler based on ->xsel */
+                if (XATTR_IS_PATHINFO (local->xsel))
+                        ret = stripe_fill_pathinfo_xattr (this, local,
+                                                          (char **)&xattr_serz);
+                else if (XATTR_IS_LOCKINFO (local->xsel)) {
+                        ret = stripe_fill_lockinfo_xattr (this, local,
+                                                          &xattr_serz);
+                } else {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Unknown xattr in xattr request");
                         goto unwind;
                 }
 
-                *(pathinfo_serz + padding + tlen) = ')';
-                *(pathinfo_serz + padding + tlen + 1) = '\0';
-
-                ret = dict_set_dynstr (stripe_xattr, GF_XATTR_PATHINFO_KEY, pathinfo_serz);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo key in dict");
+                if (!ret) {
+                        ret = dict_set_dynptr (stripe_xattr, local->xsel,
+                                               xattr_serz,
+                                               local->xattr_total_len);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Can't set %s key in dict",
+                                        local->xsel);
+                }
 
         unwind:
-                STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, stripe_xattr);
+                STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno,
+                                     stripe_xattr, NULL);
 
-                ret = stripe_free_pathinfo_str (local);
+                ret = stripe_free_xattr_str (local);
 
-                if (local->xattr_list)
-                        GF_FREE (local->xattr_list);
+                GF_FREE (local->xattr_list);
 
                 if (stripe_xattr)
                         dict_unref (stripe_xattr);
@@ -4652,7 +5477,7 @@ stripe_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 stripe_getxattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name)
+                 loc_t *loc, const char *name, dict_t *xdata)
 {
         stripe_local_t    *local    = NULL;
         xlator_list_t     *trav     = NULL;
@@ -4672,8 +5497,7 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -4684,7 +5508,7 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
 
 
         if (name && (strcmp (GF_XATTR_MARKER_KEY, name) == 0)
-            && (-1 == frame->root->pid)) {
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
                 local->marker.call_count = priv->child_count;
 
                 sub_volumes = alloca ( priv->child_count *
@@ -4699,7 +5523,8 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
                 if (cluster_getmarkerattr (frame, this, loc, name,
                                            local, stripe_getxattr_unwind,
                                            sub_volumes, priv->child_count,
-                                           MARKER_UUID_TYPE, priv->vol_uuid)) {
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           priv->vol_uuid)) {
                         op_errno = EINVAL;
                         goto err;
                 }
@@ -4715,28 +5540,39 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
                              trav = trav->next) {
                         STACK_WIND (frame, stripe_getxattr_cbk,
                                     trav->xlator, trav->xlator->fops->getxattr,
-                                    loc, name);
+                                    loc, name, xdata);
                 }
 
                 return 0;
         }
 
-        if (name && (strncmp (name, GF_XATTR_PATHINFO_KEY,
-                              strlen (GF_XATTR_PATHINFO_KEY)) == 0)) {
-                ret = inode_ctx_get (loc->inode, this,
-                                     (uint64_t *) &local->stripe_size);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "stripe size unavailable from inode ctx - relying"
-                                " on pathinfo could lead to wrong results");
+        if (name &&
+            ((strncmp (name, GF_XATTR_PATHINFO_KEY,
+                       strlen (GF_XATTR_PATHINFO_KEY)) == 0))) {
+                if (IA_ISREG (loc->inode->ia_type)) {
+                        ret = inode_ctx_get (loc->inode, this,
+                                             (uint64_t *) &local->fctx);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "stripe size unavailable from fctx"
+                                        " relying on pathinfo could lead to"
+                                        " wrong results");
+                }
+
                 local->nallocs = local->wind_count = priv->child_count;
+                (void) strncpy (local->xsel, name, strlen (name));
 
+                /**
+                 * for xattrs that need info from all childs, fill ->xsel
+                 * as above and call the filler function in cbk based on
+                 * it
+                 */
                 for (i = 0, trav = this->children; i < priv->child_count; i++,
                      trav = trav->next) {
-                        STACK_WIND_COOKIE (frame, stripe_getxattr_pathinfo_cbk,
+                        STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
                                            (void *) (long) i, trav->xlator,
                                            trav->xlator->fops->getxattr,
-                                           loc, name);
+                                           loc, name, xdata);
                 }
 
                 return 0;
@@ -4744,42 +5580,125 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
 
         if (name &&(*priv->vol_uuid)) {
                 if ((match_uuid_local (name, priv->vol_uuid) == 0)
-                    && (-1 == frame->root->pid)) {
-                        local->marker.call_count = priv->child_count;
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+
+                        if (!IA_FILE_OR_DIR (loc->inode->ia_type))
+                                local->marker.call_count = 1;
+                        else
+                                local->marker.call_count = priv->child_count;
 
-                        sub_volumes = alloca ( priv->child_count *
-                                               sizeof (xlator_t *));
-                        for (i = 0, trav = this->children; trav ;
-                             trav = trav->next, i++) {
+                        sub_volumes = alloca (local->marker.call_count *
+                                              sizeof (xlator_t *));
 
+                        for (i = 0, trav = this->children;
+                             i < local->marker.call_count;
+                             i++, trav = trav->next) {
                                 *(sub_volumes + i) = trav->xlator;
 
                         }
 
                         if (cluster_getmarkerattr (frame, this, loc, name,
-                                                   local, stripe_getxattr_unwind,
+                                                   local,
+                                                   stripe_getxattr_unwind,
                                                    sub_volumes,
-                                                   priv->child_count,
+                                                   local->marker.call_count,
                                                    MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
                                                    priv->vol_uuid)) {
                                 op_errno = EINVAL;
                                 goto err;
                         }
+
                         return 0;
                 }
         }
 
 
-        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
+        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
 
         return 0;
 
 err:
-        STRIPE_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
+inline gf_boolean_t
+stripe_is_special_xattr (const char *name)
+{
+        gf_boolean_t    is_spl = _gf_false;
+
+        if (!name) {
+                goto out;
+        }
+
+        if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                      strlen (GF_XATTR_LOCKINFO_KEY))
+            || !strncmp (name, GF_XATTR_PATHINFO_KEY,
+                         strlen (GF_XATTR_PATHINFO_KEY)))
+                is_spl = _gf_true;
+out:
+        return is_spl;
+}
+
+int32_t
+stripe_fgetxattr_from_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                const char *name, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv  = NULL;
+        int32_t           ret   = -1, op_errno = 0;
+        int               i     = 0;
+        xlator_list_t    *trav  = NULL;
+
+        priv = this->private;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->op_ret = -1;
+        frame->local = local;
+
+        strncpy (local->xsel, name, strlen (name));
+        local->nallocs = local->wind_count = priv->child_count;
+
+        for (i = 0, trav = this->children; i < priv->child_count; i++,
+                     trav = trav->next) {
+                STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
+                                   (void *) (long) i, trav->xlator,
+                                   trav->xlator->fops->fgetxattr,
+                                   fd, name, xdata);
+        }
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, NULL);
+        return ret;
+}
+
+int32_t
+stripe_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
+{
+        if (stripe_is_special_xattr (name)) {
+                stripe_fgetxattr_from_everyone (frame, this, fd, name, xdata);
+                goto out;
+        }
+
+        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+
+out:
+        return 0;
+}
+
+
+
 int32_t
 stripe_priv_dump (xlator_t *this)
 {
@@ -4828,33 +5747,39 @@ out:
 }
 
 struct xlator_fops fops = {
-        .stat        = stripe_stat,
-        .unlink      = stripe_unlink,
-        .rename      = stripe_rename,
-        .link        = stripe_link,
-        .truncate    = stripe_truncate,
-        .create      = stripe_create,
-        .open        = stripe_open,
-        .readv       = stripe_readv,
-        .writev      = stripe_writev,
-        .statfs      = stripe_statfs,
-        .flush       = stripe_flush,
-        .fsync       = stripe_fsync,
-        .ftruncate   = stripe_ftruncate,
-        .fstat       = stripe_fstat,
-        .mkdir       = stripe_mkdir,
-        .rmdir       = stripe_rmdir,
-        .lk          = stripe_lk,
-        .opendir     = stripe_opendir,
-        .fsyncdir    = stripe_fsyncdir,
-        .setattr     = stripe_setattr,
-        .fsetattr    = stripe_fsetattr,
-        .lookup      = stripe_lookup,
-        .mknod       = stripe_mknod,
-        .setxattr    = stripe_setxattr,
-        .fsetxattr   = stripe_fsetxattr,
-        .getxattr    = stripe_getxattr,
-        .readdirp    = stripe_readdirp,
+        .stat           = stripe_stat,
+        .unlink         = stripe_unlink,
+        .rename         = stripe_rename,
+        .link           = stripe_link,
+        .truncate       = stripe_truncate,
+        .create         = stripe_create,
+        .open           = stripe_open,
+        .readv          = stripe_readv,
+        .writev         = stripe_writev,
+        .statfs         = stripe_statfs,
+        .flush          = stripe_flush,
+        .fsync          = stripe_fsync,
+        .ftruncate      = stripe_ftruncate,
+        .fstat          = stripe_fstat,
+        .mkdir          = stripe_mkdir,
+        .rmdir          = stripe_rmdir,
+        .lk             = stripe_lk,
+        .opendir        = stripe_opendir,
+        .fsyncdir       = stripe_fsyncdir,
+        .setattr        = stripe_setattr,
+        .fsetattr       = stripe_fsetattr,
+        .lookup         = stripe_lookup,
+        .mknod          = stripe_mknod,
+        .setxattr       = stripe_setxattr,
+        .fsetxattr      = stripe_fsetxattr,
+        .getxattr       = stripe_getxattr,
+        .fgetxattr      = stripe_fgetxattr,
+        .removexattr    = stripe_removexattr,
+        .fremovexattr   = stripe_fremovexattr,
+        .readdirp       = stripe_readdirp,
+	.fallocate	= stripe_fallocate,
+	.discard	= stripe_discard,
+        .zerofill       = stripe_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -4868,8 +5793,9 @@ struct xlator_dumpops dumpops = {
 
 struct volume_options options[] = {
         { .key  = {"block-size"},
-          .type = GF_OPTION_TYPE_ANY,
+          .type = GF_OPTION_TYPE_SIZE_LIST,
           .default_value = "128KB",
+          .min = STRIPE_MIN_BLOCK_SIZE,
           .description = "Size of the stripe unit that would be read "
                          "from or written to the striped servers."
         },
@@ -4877,5 +5803,12 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "true"
         },
+	{ .key = {"coalesce"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "true",
+	  .description = "Enable/Disable coalesce mode to flatten striped "
+			 "files as stored on the server (i.e., eliminate holes "
+			 "caused by the traditional format)."
+	},
         { .key  = {NULL} },
 };
diff --git a/xlators/cluster/stripe/src/stripe.h b/xlators/cluster/stripe/src/stripe.h
index 3ab67d621..5673d18f3 100644
--- a/xlators/cluster/stripe/src/stripe.h
+++ b/xlators/cluster/stripe/src/stripe.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -38,36 +29,53 @@
 #include <signal.h>
 
 #define STRIPE_PATHINFO_HEADER "STRIPE:"
-
+#define STRIPE_MIN_BLOCK_SIZE  (16*GF_UNIT_KB)
 
 #define STRIPE_STACK_UNWIND(fop, frame, params ...) do {           \
                 stripe_local_t *__local = NULL;                    \
-                if (frame) {                                    \
-                        __local = frame->local;                 \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
+                if (frame) {                                       \
+                        __local = frame->local;                    \
+                        frame->local = NULL;                       \
+                }                                                  \
+                STACK_UNWIND_STRICT (fop, frame, params);          \
+                if (__local) {                                     \
+                        stripe_local_wipe(__local);                \
+                        mem_put (__local);       \
+                }                                                  \
+        } while (0)
+
+#define STRIPE_STACK_DESTROY(frame) do {                        \
+                stripe_local_t *__local = NULL;                 \
+                __local = frame->local;                         \
+                frame->local = NULL;                            \
+                STACK_DESTROY (frame->root);                    \
                 if (__local) {                                  \
-                        stripe_local_wipe(__local);             \
-                        GF_FREE (__local);                      \
+                        stripe_local_wipe (__local);            \
+                        mem_put (__local);    \
                 }                                               \
         } while (0)
 
-#define STRIPE_STACK_DESTROY(frame) do {                  \
-                stripe_local_t *__local = NULL;           \
-                __local = frame->local;                   \
-                frame->local = NULL;                      \
-                STACK_DESTROY (frame->root);              \
-                if (__local) {                            \
-                        stripe_local_wipe (__local);      \
-                        GF_FREE (__local);                \
-                }                                         \
-        } while (0)
+#define STRIPE_VALIDATE_FCTX(fctx, label) do {                  \
+        int     idx = 0;                                        \
+        if (!fctx) {                                            \
+                op_errno = EINVAL;                              \
+                goto label;                                     \
+        }                                                       \
+        for (idx = 0; idx < fctx->stripe_count; idx++) {        \
+                if (!fctx->xl_array[idx]) {                     \
+                        gf_log (this->name, GF_LOG_ERROR,       \
+                                "fctx->xl_array[%d] is NULL",   \
+                                idx);                           \
+                        op_errno = ESTALE;                      \
+                        goto label;                             \
+                }                                               \
+        }                                                       \
+       } while (0)
 
 typedef struct stripe_xattr_sort {
-        int32_t  pos;
-        int32_t  pathinfo_len;
-        char    *pathinfo;
+        int   pos;
+        int   xattr_len;
+        char *xattr_value;
 } stripe_xattr_sort_t;
 
 /**
@@ -90,16 +98,17 @@ struct stripe_private {
         gf_lock_t               lock;
         uint8_t                 nodes_down;
         int8_t                  first_child_down;
+        int                    *last_event;
         int8_t                  child_count;
-        int8_t                 *state; /* Current state of child node */
         gf_boolean_t            xattr_supported;  /* default yes */
+	gf_boolean_t		coalesce;
         char                    vol_uuid[UUID_SIZE + 1];
 };
 
 /**
- * Used to keep info about the replies received from fops->readv calls
+ * Used to keep info about the replies received from readv/writev calls
  */
-struct readv_replies {
+struct stripe_replies {
         struct iovec *vector;
         int32_t       count;    //count of vector
         int32_t       op_ret;   //op_ret of readv
@@ -111,6 +120,7 @@ struct readv_replies {
 typedef struct _stripe_fd_ctx {
         off_t      stripe_size;
         int        stripe_count;
+	int	   stripe_coalesce;
         int        static_array;
         xlator_t **xl_array;
 } stripe_fd_ctx_t;
@@ -146,7 +156,7 @@ struct stripe_local {
         blkcnt_t             preparent_blocks;
         blkcnt_t             postparent_blocks;
 
-        struct readv_replies *replies;
+        struct stripe_replies *replies;
         struct statvfs        statvfs_buf;
         dir_entry_t          *entry;
 
@@ -173,11 +183,12 @@ struct stripe_local {
         mode_t               mode;
         dev_t                rdev;
         /* For File I/O fops */
-        dict_t              *dict;
+        dict_t              *xdata;
 
         stripe_xattr_sort_t *xattr_list;
         int32_t              xattr_total_len;
         int32_t              nallocs;
+        char xsel[256];
 
         struct marker_str    marker;
 
@@ -194,12 +205,84 @@ struct stripe_local {
         void                *value;
         struct iobref       *iobref;
         gf_dirent_t          entries;
+        gf_dirent_t         *dirent;
         dict_t              *xattr;
         uuid_t               ia_gfid;
+
+        int                  xflag;
+        mode_t               umask;
 };
 
 typedef struct stripe_local   stripe_local_t;
 typedef struct stripe_private stripe_private_t;
 
+/*
+ * Determine the stripe index of a particular frame based on the translator.
+ */
+static inline int32_t stripe_get_frame_index(stripe_fd_ctx_t *fctx,
+					     call_frame_t *prev)
+{
+	int32_t i, idx = -1;
+
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (fctx->xl_array[i] == prev->this) {
+			idx = i;
+			break;
+		}
+	}
+
+	return idx;
+}
+
+static inline void stripe_copy_xl_array(xlator_t **dst, xlator_t **src,
+					int count)
+{
+	int i;
+
+	for (i = 0; i < count; i++)
+		dst[i] = src[i];
+}
+
+void stripe_local_wipe (stripe_local_t *local);
+int32_t stripe_ctx_handle (xlator_t *this, call_frame_t *prev,
+                           stripe_local_t *local, dict_t *dict);
+void stripe_aggregate_xattr (dict_t *dst, dict_t *src);
+int32_t stripe_xattr_request_build (xlator_t *this, dict_t *dict,
+                                    uint64_t stripe_size, uint32_t stripe_count,
+                                    uint32_t stripe_index,
+				    uint32_t stripe_coalesce);
+int32_t stripe_get_matching_bs (const char *path, stripe_private_t *priv);
+int set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data);
+int32_t stripe_iatt_merge (struct iatt *from, struct iatt *to);
+int32_t stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
+                                    char **xattr_serz);
+int32_t stripe_free_xattr_str (stripe_local_t *local);
+int32_t stripe_xattr_aggregate (char *buffer, stripe_local_t *local,
+                                int32_t *total);
+off_t coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count);
+off_t uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
+			int stripe_index);
+int32_t
+stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            void **xattr_serz);
+
+/*
+ * Adjust the size attribute for files if coalesce is enabled.
+ */
+static inline void correct_file_size(struct iatt *buf, stripe_fd_ctx_t *fctx,
+	call_frame_t *prev)
+{
+	int index;
+
+	if (!IA_ISREG(buf->ia_type))
+		return;
+
+	if (!fctx || !fctx->stripe_coalesce)
+		return;
+
+	index = stripe_get_frame_index(fctx, prev);
+	buf->ia_size = uncoalesced_size(buf->ia_size, fctx->stripe_size,
+		fctx->stripe_count, index);
+}
 
 #endif /* _STRIPE_H_ */
diff --git a/xlators/cluster/unify/Makefile.am b/xlators/cluster/unify/Makefile.am
deleted file mode 100644
index d471a3f92..000000000
--- a/xlators/cluster/unify/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/cluster/unify/src/Makefile.am b/xlators/cluster/unify/src/Makefile.am
deleted file mode 100644
index 2a1fe8372..000000000
--- a/xlators/cluster/unify/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-
-xlator_LTLIBRARIES = unify.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/legacy/cluster
-
-unify_la_LDFLAGS = -module -avoidversion
-
-unify_la_SOURCES = unify.c unify-self-heal.c
-unify_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = unify.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/unify/src/unify-mem-types.h b/xlators/cluster/unify/src/unify-mem-types.h
deleted file mode 100644
index 13c9cc1f7..000000000
--- a/xlators/cluster/unify/src/unify-mem-types.h
+++ /dev/null
@@ -1,41 +0,0 @@
-
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __UNIFY_MEM_TYPES_H__
-#define __UNIFY_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_unify_mem_types_ {
-        gf_unify_mt_char = gf_common_mt_end + 1,
-        gf_unify_mt_int16_t,
-        gf_unify_mt_xlator_t,
-        gf_unify_mt_unify_private_t,
-        gf_unify_mt_xlator_list_t,
-        gf_unify_mt_dir_entry_t,
-        gf_unify_mt_off_t,
-        gf_unify_mt_int,
-        gf_unify_mt_unify_self_heal_struct,
-        gf_unify_mt_unify_local_t,
-        gf_unify_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/unify/src/unify-self-heal.c b/xlators/cluster/unify/src/unify-self-heal.c
deleted file mode 100644
index f99e4c7c3..000000000
--- a/xlators/cluster/unify/src/unify-self-heal.c
+++ /dev/null
@@ -1,1239 +0,0 @@
-/*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * unify-self-heal.c : 
- *   This file implements few functions which enables 'unify' translator 
- *  to be consistent in its behaviour when 
- *     > a node fails, 
- *     > a node gets added, 
- *     > a failed node comes back
- *     > a new namespace server is added (ie, an fresh namespace server).
- * 
- *  This functionality of 'unify' will enable glusterfs to support storage
- *  system failure, and maintain consistancy. This works both ways, ie, when
- *  an entry (either file or directory) is found on namespace server, and not
- *  on storage nodes, its created in storage nodes and vica-versa.
- * 
- *  The two fops, where it can be implemented are 'getdents ()' and 'lookup ()'
- *
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "unify.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "common-utils.h"
-
-int32_t
-unify_sh_getdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dir_entry_t *entry,
-		       int32_t count);
-
-int32_t
-unify_sh_ns_getdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dir_entry_t *entry,
-			  int32_t count);
-
-int32_t
-unify_bgsh_getdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dir_entry_t *entry,
-			 int32_t count);
-
-int32_t
-unify_bgsh_ns_getdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dir_entry_t *entry,
-			    int32_t count);
-
-/**
- * unify_local_wipe - free all the extra allocation of local->* here.
- */
-static void 
-unify_local_wipe (unify_local_t *local)
-{
-	/* Free the strdup'd variables in the local structure */
-	if (local->name) {
-		GF_FREE (local->name);
-	}
-
-	if (local->sh_struct) {
-		if (local->sh_struct->offset_list)
-			GF_FREE (local->sh_struct->offset_list);
-
-		if (local->sh_struct->entry_list)
-			GF_FREE (local->sh_struct->entry_list);
-
-		if (local->sh_struct->count_list)
-			GF_FREE (local->sh_struct->count_list);
-
-		GF_FREE (local->sh_struct);
-	}
-
-	loc_wipe (&local->loc1);
-	loc_wipe (&local->loc2);
-}
-
-int32_t 
-unify_sh_setdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		/* if local->call_count == 0, that means, setdents on 
-		 * storagenodes is still pending.
-		 */
-		if (local->call_count)
-			callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		if (local->sh_struct->entry_list[0]) {
-			prev = entry = local->sh_struct->entry_list[0];
-			if (!entry)
-				return 0;
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-
-		if (!local->flags) {
-			if (local->sh_struct->count_list[0] >= 
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-				/* count == size, that means, there are more entries
-				   to read from */
-				//local->call_count = 0;
-				local->sh_struct->offset_list[0] += 
-					UNIFY_SELF_HEAL_GETDENTS_COUNT;
-				STACK_WIND (frame,
-					    unify_sh_ns_getdents_cbk,
-					    NS(this),
-					    NS(this)->fops->getdents,
-					    local->fd,
-					    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-					    local->sh_struct->offset_list[0],
-					    GF_GET_DIR_ONLY);
-			}		
-		} else {
-			inode = local->loc1.inode;
-			fd_unref (local->fd);
-			tmp_dict = local->dict;
-
-			unify_local_wipe (local);
-			
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      inode, &local->stbuf, local->dict,
-                                      &local->oldpostparent);
-			if (tmp_dict)
-				dict_unref (tmp_dict);
-		}
-	}
-  
-	return 0;
-}
-
-
-int32_t
-unify_sh_ns_getdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dir_entry_t *entry,
-			  int32_t count)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = 0;
-	unsigned long final = 0;
-	dir_entry_t *tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                      gf_unify_mt_dir_entry_t);
-	
-	local->sh_struct->entry_list[0] = tmp;
-	local->sh_struct->count_list[0] = count;
-	if (entry) {
-		tmp->next = entry->next;
-		entry->next = NULL;
-	}
-
-	if ((count < UNIFY_SELF_HEAL_GETDENTS_COUNT) || !entry) {
-		final = 1;
-	}
-
-	LOCK (&frame->lock);
-	{
-		/* local->call_count will be '0' till now. make it 1 so, it 
-		   can be UNWIND'ed for the last call. */
-		local->call_count = priv->child_count;
-		if (final)
-			local->flags = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	for (index = 0; index < priv->child_count; index++) 
-	{
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_setdents_cbk, 
-				   (void *)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->setdents,
-				   local->fd, GF_SET_DIR_ONLY,
-				   local->sh_struct->entry_list[0], count);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_sh_ns_setdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		if (local->sh_struct->entry_list[index]) {
-			prev = entry = local->sh_struct->entry_list[index];
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (local->sh_struct->count_list[index] < 
-	    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_sh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_sh_getdents_cbk -
- */
-int32_t
-unify_sh_getdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dir_entry_t *entry,
-		       int32_t count)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *tmp = NULL; 
-
-	if (op_ret >= 0 && count > 0) {
-		/* There is some dentry found, just send the dentry to NS */
-		tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                 gf_unify_mt_dir_entry_t);
-		local->sh_struct->entry_list[index] = tmp;
-		local->sh_struct->count_list[index] = count;
-		if (entry) {
-			tmp->next = entry->next;
-			entry->next = NULL;
-		}
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_ns_setdents_cbk,
-				   cookie,
-				   NS(this),
-				   NS(this)->fops->setdents,
-				   local->fd,
-				   GF_SET_IF_NOT_PRESENT,
-				   local->sh_struct->entry_list[index],
-				   count);
-		return 0;
-	}
-  
-	if (count < UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_sh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-/**
- * unify_sh_opendir_cbk -
- *
- * @cookie: 
- */
-int32_t 
-unify_sh_opendir_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      fd_t *fd)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		} else {
-			gf_log (this->name, GF_LOG_WARNING, "failed");
-			local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local->call_count = priv->child_count + 1;
-    
-		if (!local->failed) {
-			/* send getdents() namespace after finishing
-			   storage nodes */
-			local->call_count--; 
-      
-			fd_bind (fd);
-
-			if (local->call_count) {
-				/* Used as the offset index. This list keeps
-				 * track of offset sent to each node during
-				 * STACK_WIND.
-				 */
-				local->sh_struct->offset_list = 
-					GF_CALLOC (priv->child_count, 
-					           sizeof (off_t),
-                                                   gf_unify_mt_off_t);
-				ERR_ABORT (local->sh_struct->offset_list);
-	
-				local->sh_struct->entry_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (dir_entry_t *),
-                                                   gf_unify_mt_dir_entry_t);
-				ERR_ABORT (local->sh_struct->entry_list);
-
-				local->sh_struct->count_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (int),
-                                                   gf_unify_mt_int);
-				ERR_ABORT (local->sh_struct->count_list);
-
-				/* Send getdents on all the fds */
-				for (index = 0; 
-				     index < priv->child_count; index++) {
-					STACK_WIND_COOKIE (frame,
-							   unify_sh_getdents_cbk,
-							   (void *)(long)index,
-							   priv->xl_array[index],
-							   priv->xl_array[index]->fops->getdents,
-							   local->fd,
-							   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-							   0, /* In this call, do send '0' as offset */
-							   GF_GET_ALL);
-				}
-
-				/* did stack wind, so no need to unwind here */
-				return 0;
-			} /* (local->call_count) */
-		} /* (!local->failed) */
-
-		/* Opendir failed on one node. */ 
-		inode = local->loc1.inode;
-		fd_unref (local->fd);
-		tmp_dict = local->dict;
-
-		unify_local_wipe (local);
-		/* Only 'self-heal' failed, lookup() was successful. */
-		local->op_ret = 0;
-
-		/* This is lookup_cbk ()'s UNWIND. */
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, inode,
-			      &local->stbuf, local->dict, &local->oldpostparent);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-/**
- * gf_sh_checksum_cbk - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-unify_sh_checksum_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       uint8_t *file_checksum,
-		       uint8_t *dir_checksum)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int32_t callcnt = 0;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (NS(this) == (xlator_t *)cookie) {
-				memcpy (local->sh_struct->ns_file_checksum, 
-					file_checksum, NAME_MAX);
-				memcpy (local->sh_struct->ns_dir_checksum, 
-					dir_checksum, NAME_MAX);
-			} else {
-				if (local->entry_count == 0) {
-					/* Initialize the dir_checksum to be 
-					 * used for comparision with other
-					 * storage nodes. Should be done for
-					 * the first successful call *only*. 
-					 */
-                                        /* Using 'entry_count' as a flag */
-					local->entry_count = 1;
-					memcpy (local->sh_struct->dir_checksum,
-						dir_checksum, NAME_MAX);
-				}
-
-				/* Reply from the storage nodes */
-				for (index = 0; 
-				     index < NAME_MAX; index++) {
-					/* Files should be present in
-					   only one node */
-					local->sh_struct->file_checksum[index] ^= file_checksum[index];
-	  
-					/* directory structure should be
-					   same accross */
-					if (local->sh_struct->dir_checksum[index] != dir_checksum[index])
-						local->failed = 1;
-				}
-			}
-		} 
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		for (index = 0; index < NAME_MAX ; index++) {
-			if (local->sh_struct->file_checksum[index] != 
-			    local->sh_struct->ns_file_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-			if (local->sh_struct->dir_checksum[index] != 
-			    local->sh_struct->ns_dir_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-		}
-	
-		if (local->failed) {
-			/* Log it, it should be a rare event */
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Self-heal triggered on directory %s", 
-				local->loc1.path);
-
-			/* Any self heal will be done at directory level */
-			local->call_count = 0;
-			local->op_ret = -1;
-			local->failed = 0;
-      
-			local->fd = fd_create (local->loc1.inode, 
-					       frame->root->pid);
-
-			local->call_count = priv->child_count + 1;
-	
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_sh_opendir_cbk,
-						   priv->xl_array[index]->name,
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->opendir,
-						   &local->loc1,
-						   local->fd);
-			}
-			/* opendir can be done on the directory */
-			return 0;
-		}
-
-		/* no mismatch */
-		inode = local->loc1.inode;
-		tmp_dict = local->dict;
-
-		unify_local_wipe (local);
-
-		/* This is lookup_cbk ()'s UNWIND. */
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      inode,
-			      &local->stbuf,
-			      local->dict, &local->oldpostparent);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-/* Foreground self-heal part over */
-
-/* Background self-heal part */
-
-int32_t 
-unify_bgsh_setdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		/* if local->call_count == 0, that means, setdents 
-		   on storagenodes is still pending. */
-		if (local->call_count)
-			callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-
-	if (callcnt == 0) {
-		if (local->sh_struct->entry_list[0]) {
-			prev = entry = local->sh_struct->entry_list[0];
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-
-		if (!local->flags) {
-			if (local->sh_struct->count_list[0] >= 
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-				/* count == size, that means, there are more
-				   entries to read from */
-				//local->call_count = 0;
-				local->sh_struct->offset_list[0] += 
-					UNIFY_SELF_HEAL_GETDENTS_COUNT;
-				STACK_WIND (frame,
-					    unify_bgsh_ns_getdents_cbk,
-					    NS(this),
-					    NS(this)->fops->getdents,
-					    local->fd,
-					    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-					    local->sh_struct->offset_list[0],
-					    GF_GET_DIR_ONLY);
-			}		
-		} else {
-			fd_unref (local->fd);
-			unify_local_wipe (local);
-			STACK_DESTROY (frame->root);
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_bgsh_ns_getdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dir_entry_t *entry,
-			    int32_t count)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = 0;
-	unsigned long final = 0;
-	dir_entry_t *tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                      gf_unify_mt_dir_entry_t);
-
-	local->sh_struct->entry_list[0] = tmp;
-	local->sh_struct->count_list[0] = count;
-	if (entry) {
-		tmp->next = entry->next;
-		entry->next = NULL;
-	}
-
-	if ((count < UNIFY_SELF_HEAL_GETDENTS_COUNT) || !entry) {
-		final = 1;
-	}
-
-	LOCK (&frame->lock);
-	{
-		/* local->call_count will be '0' till now. make it 1 so, 
-		   it can be UNWIND'ed for the last call. */
-		local->call_count = priv->child_count;
-		if (final)
-			local->flags = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	for (index = 0; index < priv->child_count; index++) 
-	{
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_setdents_cbk, 
-				   (void *)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->setdents,
-				   local->fd, GF_SET_DIR_ONLY,
-				   local->sh_struct->entry_list[0], count);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_bgsh_ns_setdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *prev, *entry, *trav;
-
-	if (local->sh_struct->entry_list[index]) {
-		prev = entry = local->sh_struct->entry_list[index];
-		if (!entry)
-			return 0;
-		trav = entry->next;
-		while (trav) {
-			prev->next = trav->next;
-			GF_FREE (trav->name);
-			if (IA_ISLNK (trav->buf.ia_type))
-				GF_FREE (trav->link);
-			GF_FREE (trav);
-			trav = prev->next;
-		}
-		GF_FREE (entry);
-	}
-
-	if (local->sh_struct->count_list[index] < 
-	    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_bgsh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_bgsh_getdents_cbk -
- */
-int32_t
-unify_bgsh_getdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dir_entry_t *entry,
-			 int32_t count)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *tmp = NULL; 
-
-	if (op_ret >= 0 && count > 0) {
-		/* There is some dentry found, just send the dentry to NS */
-		tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                 gf_unify_mt_dir_entry_t);
-		local->sh_struct->entry_list[index] = tmp;
-		local->sh_struct->count_list[index] = count;
-		if (entry) {
-			tmp->next = entry->next;
-			entry->next = NULL;
-		}
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_ns_setdents_cbk,
-				   cookie,
-				   NS(this),
-				   NS(this)->fops->setdents,
-				   local->fd,
-				   GF_SET_IF_NOT_PRESENT,
-				   local->sh_struct->entry_list[index],
-				   count);
-		return 0;
-	}
-  
-	if (count < UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for 
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_bgsh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-/**
- * unify_bgsh_opendir_cbk -
- *
- * @cookie: 
- */
-int32_t 
-unify_bgsh_opendir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			fd_t *fd)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int32_t callcnt = 0;
-	int16_t index = 0;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		} else {
-			local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local->call_count = priv->child_count + 1;
-    
-		if (!local->failed) {
-			/* send getdents() namespace after finishing 
-			   storage nodes */
-			local->call_count--; 
-			callcnt = local->call_count;
-      
-			fd_bind (fd);
-
-			if (local->call_count) {
-				/* Used as the offset index. This list keeps 
-				   track of offset sent to each node during 
-				   STACK_WIND. */
-				local->sh_struct->offset_list = 
-					GF_CALLOC (priv->child_count, 
-					           sizeof (off_t),
-                                                   gf_unify_mt_off_t);
-				ERR_ABORT (local->sh_struct->offset_list);
-	
-				local->sh_struct->entry_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (dir_entry_t *),
-                                                   gf_unify_mt_dir_entry_t);
-				ERR_ABORT (local->sh_struct->entry_list);
-
-				local->sh_struct->count_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (int),
-                                                   gf_unify_mt_int);
-				ERR_ABORT (local->sh_struct->count_list);
-
-				/* Send getdents on all the fds */
-				for (index = 0; 
-				     index < priv->child_count; index++) {
-					STACK_WIND_COOKIE (frame,
-							   unify_bgsh_getdents_cbk,
-							   (void *)(long)index,
-							   priv->xl_array[index],
-							   priv->xl_array[index]->fops->getdents,
-							   local->fd,
-							   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-							   0, /* In this call, do send '0' as offset */
-							   GF_GET_ALL);
-				}
-				/* did a stack wind, so no need to unwind here */
-				return 0;
-			} /* (local->call_count) */
-		} /* (!local->failed) */
-
-		/* Opendir failed on one node. 	 */
-		fd_unref (local->fd);
-		
-		unify_local_wipe (local);
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-/**
- * gf_bgsh_checksum_cbk - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-unify_bgsh_checksum_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 uint8_t *file_checksum,
-			 uint8_t *dir_checksum)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int32_t callcnt = 0;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (NS(this) == (xlator_t *)cookie) {
-				memcpy (local->sh_struct->ns_file_checksum, 
-					file_checksum, NAME_MAX);
-				memcpy (local->sh_struct->ns_dir_checksum, 
-					dir_checksum, NAME_MAX);
-			} else {
-				if (local->entry_count == 0) {
-					/* Initialize the dir_checksum to be 
-					 * used for comparision with other 
-					 * storage nodes. Should be done for
-					 * the first successful call *only*. 
-					 */
-					/* Using 'entry_count' as a flag */
-					local->entry_count = 1; 
-					memcpy (local->sh_struct->dir_checksum,
-						dir_checksum, NAME_MAX);
-				}
-
-				/* Reply from the storage nodes */
-				for (index = 0; 
-				     index < NAME_MAX; index++) {
-					/* Files should be present in only 
-					   one node */
-					local->sh_struct->file_checksum[index] ^= file_checksum[index];
-	  
-					/* directory structure should be same 
-					   accross */
-					if (local->sh_struct->dir_checksum[index] != dir_checksum[index])
-						local->failed = 1;
-				}
-			}
-		} 
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		for (index = 0; index < NAME_MAX ; index++) {
-			if (local->sh_struct->file_checksum[index] != 
-			    local->sh_struct->ns_file_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-			if (local->sh_struct->dir_checksum[index] != 
-			    local->sh_struct->ns_dir_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-		}
-	
-		if (local->failed) {
-			/* Log it, it should be a rare event */
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Self-heal triggered on directory %s", 
-				local->loc1.path);
-
-			/* Any self heal will be done at the directory level */
-			local->op_ret = -1;
-			local->failed = 0;
-      
-			local->fd = fd_create (local->loc1.inode, 
-					       frame->root->pid);
-			local->call_count = priv->child_count + 1;
-	
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_bgsh_opendir_cbk,
-						   priv->xl_array[index]->name,
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->opendir,
-						   &local->loc1,
-						   local->fd);
-			}
-      
-			/* opendir can be done on the directory */
-			return 0;
-		}
-
-		/* no mismatch */
-		unify_local_wipe (local);
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-/* Background self-heal part over */
-
-
-
-
-/**
- * zr_unify_self_heal - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-zr_unify_self_heal (call_frame_t *frame,
-		    xlator_t *this,
-		    unify_local_t *local)
-{
-	unify_private_t *priv = this->private;
-	call_frame_t *bg_frame = NULL;
-	unify_local_t *bg_local = NULL;
-	inode_t *tmp_inode = NULL;
-	dict_t *tmp_dict = NULL;
-	int16_t index = 0;
-  
-	if (local->inode_generation < priv->inode_generation) {
-		/* Any self heal will be done at the directory level */
-		/* Update the inode's generation to the current generation
-		   value. */
-		local->inode_generation = priv->inode_generation;
-		inode_ctx_put (local->loc1.inode, this, 
-			  (uint64_t)(long)local->inode_generation);
-
-		if (priv->self_heal == ZR_UNIFY_FG_SELF_HEAL) {
-			local->op_ret = 0;
-			local->failed = 0;
-			local->call_count = priv->child_count + 1;
-			local->sh_struct = 
-	         	     GF_CALLOC (1, sizeof (struct unify_self_heal_struct),
-                                        gf_unify_mt_unify_self_heal_struct);
-      
-			/* +1 is for NS */
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_sh_checksum_cbk,
-						   priv->xl_array[index],
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->checksum,
-						   &local->loc1,
-						   0);
-			}
-
-			/* Self-heal in foreground, hence no need 
-			   to UNWIND here */
-			return 0;
-		}
-
-		/* Self Heal done in background */
-		bg_frame = copy_frame (frame);
-		INIT_LOCAL (bg_frame, bg_local);
-		loc_copy (&bg_local->loc1, &local->loc1);
-		bg_local->op_ret = 0;
-		bg_local->failed = 0;
-		bg_local->call_count = priv->child_count + 1;
-		bg_local->sh_struct = 
-			GF_CALLOC (1, sizeof (struct unify_self_heal_struct),
-                                   gf_unify_mt_unify_self_heal_struct);
-    
-		/* +1 is for NS */
-		for (index = 0; index < (priv->child_count + 1); index++) {
-			STACK_WIND_COOKIE (bg_frame,
-					   unify_bgsh_checksum_cbk,
-					   priv->xl_array[index],
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->checksum,
-					   &bg_local->loc1,
-					   0);
-		}
-	}
-
-	/* generation number matches, self heal already done or
-	 * self heal done in background: just do STACK_UNWIND 
-	 */
-	tmp_inode = local->loc1.inode;
-	tmp_dict = local->dict;
-
-	unify_local_wipe (local);
-
-	/* This is lookup_cbk ()'s UNWIND. */
-	STACK_UNWIND (frame,
-		      local->op_ret,
-		      local->op_errno,
-		      tmp_inode,
-		      &local->stbuf,
-		      local->dict,
-                      &local->oldpostparent);
-
-	if (tmp_dict)
-		dict_unref (tmp_dict);
-
-	return 0;
-}
-
diff --git a/xlators/cluster/unify/src/unify.c b/xlators/cluster/unify/src/unify.c
deleted file mode 100644
index 6dc93083d..000000000
--- a/xlators/cluster/unify/src/unify.c
+++ /dev/null
@@ -1,4589 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * xlators/cluster/unify:
- *     - This xlator is one of the main translator in GlusterFS, which
- *   actually does the clustering work of the file system. One need to 
- *   understand that, unify assumes file to be existing in only one of 
- *   the child node, and directories to be present on all the nodes. 
- *
- * NOTE:
- *   Now, unify has support for global namespace, which is used to keep a 
- * global view of fs's namespace tree. The stat for directories are taken
- * just from the namespace, where as for files, just 'ia_ino' is taken from
- * Namespace node, and other stat info is taken from the actual storage node.
- * Also Namespace node helps to keep consistant inode for files across 
- * glusterfs (re-)mounts.
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "unify.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include <signal.h>
-#include <libgen.h>
-#include "compat-errno.h"
-#include "compat.h"
-
-#define UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR(_loc) do { \
-  if (!(_loc && _loc->inode)) {                            \
-    STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL);    \
-    return 0;                                              \
-  }                                                        \
-} while(0)
-
-
-#define UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(_fd) do { \
-  if (!(_fd && !fd_ctx_get (_fd, this, NULL))) {       \
-    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);      \
-    return 0;                                          \
-  }                                                    \
-} while(0)
-
-#define UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(_fd) do { \
-  if (!_fd) {                                      \
-    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);  \
-    return 0;                                      \
-  }                                                \
-} while(0)
-
-/**
- * unify_local_wipe - free all the extra allocation of local->* here.
- */
-static void 
-unify_local_wipe (unify_local_t *local)
-{
-	/* Free the strdup'd variables in the local structure */
-	if (local->name) {
-		GF_FREE (local->name);
-	}
-	loc_wipe (&local->loc1);
-	loc_wipe (&local->loc2);
-}
-
-
-
-/*
- * unify_normalize_stats -
- */
-void
-unify_normalize_stats (struct statvfs *buf,
-		       unsigned long bsize,
-		       unsigned long frsize)
-{
-	double factor;
-
-	if (buf->f_bsize != bsize) {
-		factor = ((double) buf->f_bsize) / bsize;
-		buf->f_bsize  = bsize;
-		buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-		buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
-	}
-  
-	if (buf->f_frsize != frsize) {
-		factor = ((double) buf->f_frsize) / frsize;
-		buf->f_frsize = frsize;
-		buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-	}
-}
-
-
-xlator_t *
-unify_loc_subvol (loc_t *loc, xlator_t *this)
-{
-	unify_private_t *priv = NULL;
-	xlator_t        *subvol = NULL;
-	int16_t         *list = NULL;
-	long             index = 0;
-	xlator_t        *subvol_i = NULL;
-	int              ret = 0;
-	uint64_t         tmp_list = 0;
-
-	priv   = this->private;
-	subvol = NS (this);
-
-	if (!IA_ISDIR (loc->inode->ia_type)) {
-		ret = inode_ctx_get (loc->inode, this, &tmp_list);
-		list = (int16_t *)(long)tmp_list;
-		if (!list)
-			goto out;
-
-		for (index = 0; list[index] != -1; index++) {
-			subvol_i = priv->xl_array[list[index]];
-			if (subvol_i != NS (this)) {
-				subvol = subvol_i;
-				break;
-			}
-		}
-	}
-out:
-	return subvol;
-}
-
-
-
-/**
- * unify_statfs_cbk -
- */
-int32_t
-unify_statfs_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct statvfs *stbuf)
-{
-	int32_t callcnt = 0;
-	struct statvfs *dict_buf = NULL;
-	unsigned long bsize;
-	unsigned long frsize;
-	unify_local_t *local = (unify_local_t *)frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			/* when a call is successfull, add it to local->dict */
-			dict_buf = &local->statvfs_buf;
-
-			if (dict_buf->f_bsize != 0) {
-				bsize  = max (dict_buf->f_bsize, 
-					      stbuf->f_bsize);
-
-				frsize = max (dict_buf->f_frsize, 
-					      stbuf->f_frsize);
-				unify_normalize_stats(dict_buf, bsize, frsize);
-				unify_normalize_stats(stbuf, bsize, frsize);
-			} else {
-				dict_buf->f_bsize   = stbuf->f_bsize;
-				dict_buf->f_frsize  = stbuf->f_frsize;
-			}
-      
-			dict_buf->f_blocks += stbuf->f_blocks;
-			dict_buf->f_bfree  += stbuf->f_bfree;
-			dict_buf->f_bavail += stbuf->f_bavail;
-			dict_buf->f_files  += stbuf->f_files;
-			dict_buf->f_ffree  += stbuf->f_ffree;
-			dict_buf->f_favail += stbuf->f_favail;
-			dict_buf->f_fsid    = stbuf->f_fsid;
-			dict_buf->f_flag    = stbuf->f_flag;
-			dict_buf->f_namemax = stbuf->f_namemax;
-			local->op_ret = op_ret;
-		} else {
-			/* fop on storage node has failed due to some error */
-			if (op_errno != ENOTCONN) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"child(%s): %s", 
-					prev_frame->this->name, 
-					strerror (op_errno));
-			}
-			local->op_errno = op_errno;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->statvfs_buf);
-	}
-
-	return 0;
-}
-
-/**
- * unify_statfs -
- */
-int32_t
-unify_statfs (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	unify_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
-
-	INIT_LOCAL (frame, local);
-	local->call_count = ((unify_private_t *)this->private)->child_count;
-
-	while(trav) {
-		STACK_WIND (frame,
-			    unify_statfs_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->statfs,
-			    loc);
-		trav = trav->next;
-	}
-
-	return 0;
-}
-
-/**
- * unify_buf_cbk - 
- */
-int32_t
-unify_buf_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct iatt *buf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s(): child(%s): path(%s): %s", 
-				gf_fop_list[frame->root->op],
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-
-			local->op_errno = op_errno;
-			if ((op_errno == ENOENT) && priv->optimist) 
-				local->op_ret = 0;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = buf->ia_ino;
-				/* If the entry is directory, get the stat
-				   from NS node */
-				if (IA_ISDIR (buf->ia_type) || 
-				    !local->stbuf.ia_blksize) {
-					local->stbuf = *buf;
-				}
-			}
-
-			if ((!IA_ISDIR (buf->ia_type)) && 
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from Storage 
-				   node. */
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		/* If the inode number is not filled, operation should
-		   fail */
-		if (!local->ia_ino)
-			local->op_ret = -1;
-
-		local->stbuf.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-
-	return 0;
-}
-
-#define check_if_dht_linkfile(s) \
-        ((st_mode_from_ia (s->ia_prot, s->ia_type) & ~S_IFMT) == S_ISVTX)
-
-/**
- * unify_lookup_cbk - 
- */
-int32_t 
-unify_lookup_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-		  struct iatt *buf,
-		  dict_t *dict,
-                  struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	inode_t *tmp_inode = NULL;
-	dict_t *local_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
- 
-		if (op_ret == -1) {
-                        if (local->revalidate &&
-                            (op_errno == ESTALE)) {
-                                /* ESTALE takes priority */
-                                local->op_errno = op_errno;
-                                local->failed = 1;
-                        }
-
-			if ((op_errno != ENOTCONN) 
-                            && (op_errno != ENOENT)
-                            && (local->op_errno != ESTALE)) {
-                                /* if local->op_errno is already ESTALE, then
-                                 * ESTALE has to propogated to the parent first.
-                                 * do not enter here.
-                                 */
-				gf_log (this->name, GF_LOG_ERROR,
-					"child(%s): path(%s): %s", 
-					priv->xl_array[(long)cookie]->name, 
-					local->loc1.path, strerror (op_errno));
-                                local->op_errno = op_errno;
-				local->failed = 1;
-
-			} else if (local->revalidate && 
-                                   (local->op_errno != ESTALE) &&
-				   !(priv->optimist && (op_errno == ENOENT))) {
-
-				gf_log (this->name, 
-					(op_errno == ENOTCONN) ? 
-					GF_LOG_DEBUG:GF_LOG_ERROR,
-					"child(%s): path(%s): %s", 
-					priv->xl_array[(long)cookie]->name, 
-					local->loc1.path, strerror (op_errno));
-				local->op_errno = op_errno;
-				local->failed = 1;
-			}
-		}
-
-		if (op_ret == 0) {
-			local->op_ret = 0; 
-			
-			if (check_if_dht_linkfile(buf)) {
-				gf_log (this->name, GF_LOG_CRITICAL,
-					"file %s may be DHT link file on %s, "
-					"make sure the backend is not shared "
-					"between unify and DHT", 
-					local->loc1.path, 
-					priv->xl_array[(long)cookie]->name);
-			}
-
-			if (local->stbuf.ia_type && local->stbuf.ia_blksize) {
-				/* make sure we already have a stbuf
-				   stored in local->stbuf */
-				if (IA_ISDIR (local->stbuf.ia_type) && 
-				    !IA_ISDIR (buf->ia_type)) {
-					gf_log (this->name, GF_LOG_CRITICAL, 
-						"[CRITICAL] '%s' is directory "
-						"on namespace, non-directory "
-						"on node '%s', returning EIO",
-						local->loc1.path, 
-						priv->xl_array[(long)cookie]->name);
-					local->return_eio = 1;
-				}
-				if (!IA_ISDIR (local->stbuf.ia_type) && 
-				    IA_ISDIR (buf->ia_type)) {
-					gf_log (this->name, GF_LOG_CRITICAL, 
-						"[CRITICAL] '%s' is directory "
-						"on node '%s', non-directory "
-						"on namespace, returning EIO",
-						local->loc1.path, 
-						priv->xl_array[(long)cookie]->name);
-					local->return_eio = 1;
-				}
-			}
-	
-			if (!local->revalidate && !IA_ISDIR (buf->ia_type)) {
-				/* This is the first time lookup on file*/
-				if (!local->list) {
-					/* list is not allocated, allocate 
-					   the max possible range */
-					local->list = GF_CALLOC (1, 2 * (priv->child_count + 2),
-                                                                 gf_unify_mt_int16_t);
-					if (!local->list) {
-						gf_log (this->name, 
-							GF_LOG_CRITICAL, 
-							"Not enough memory");
-						STACK_UNWIND (frame, -1, 
-							      ENOMEM, inode, 
-							      NULL, NULL, NULL);
-						return 0;
-					}
-				}
-				/* update the index of the list */
-				local->list [local->index++] = 
-					(int16_t)(long)cookie;
-			}
-                        
-                        if (!local->revalidate && IA_ISDIR (buf->ia_type)) {
-                                /* fresh lookup of a directory */
-                                inode_ctx_put (local->loc1.inode, this, 
-                                               priv->inode_generation);
-                        }
-
-			if ((!local->dict) && dict &&
-			    (priv->xl_array[(long)cookie] != NS(this)))	{
-				local->dict = dict_ref (dict);
-			}
-
-			/* index of NS node is == total child count */
-			if (priv->child_count == (int16_t)(long)cookie) {
-				/* Take the inode number from namespace */
-				local->ia_ino = buf->ia_ino;
-				if (IA_ISDIR (buf->ia_type) || 
-				    !(local->stbuf.ia_blksize)) {
-					local->stbuf = *buf;
-                                        local->oldpostparent = *postparent;
-				}
-			} else if (!IA_ISDIR (buf->ia_type)) {
-				/* If file, then get the stat from 
-				   storage node */
-				local->stbuf = *buf;
-			}
-
-			if (local->ia_nlink < buf->ia_nlink) {
-				local->ia_nlink = buf->ia_nlink;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local_dict = local->dict;
-		if (local->return_eio) {
-			gf_log (this->name, GF_LOG_CRITICAL, 
-				"[CRITICAL] Unable to fix the path (%s) with "
-				"self-heal, try manual verification. "
-				"returning EIO.", local->loc1.path);
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, -1, EIO, inode, NULL, NULL);
-			if (local_dict)	{
-				dict_unref (local_dict);
-			}
-			return 0;
-		}
-
-		if (!local->stbuf.ia_blksize) {
-			/* Inode not present */
-			local->op_ret = -1;
-		} else {
-			if (!local->revalidate && 
-			    !IA_ISDIR (local->stbuf.ia_type)) { 
-				/* If its a file, big array is useless, 
-				   allocate the smaller one */
-				int16_t *list = NULL;
-				list = GF_CALLOC (1, 2 * (local->index + 1),
-                                                  gf_unify_mt_int16_t);
-				ERR_ABORT (list);
-				memcpy (list, local->list, 2 * local->index);
-				/* Make the end of the list as -1 */
-				GF_FREE (local->list);
-				local->list = list;
-				local->list [local->index] = -1;
-				/* Update the inode's ctx with proper array */
-				/* TODO: log on failure */
-				inode_ctx_put (local->loc1.inode, this, 
-					       (uint64_t)(long)local->list);
-			}
-
-			if (IA_ISDIR(local->loc1.inode->ia_type)) {
-				/* lookup is done for directory */
-				if (local->failed && priv->self_heal) {
-					/* Triggering self-heal */
-                                        /* means, self-heal required for this 
-					   inode */
-					local->inode_generation = 0; 
-					priv->inode_generation++;
-				}
-			} else {
-				local->stbuf.ia_ino = local->ia_ino;
-			}
-	  
-			local->stbuf.ia_nlink = local->ia_nlink;
-		}
-		if (local->op_ret == -1) {
-			if (!local->revalidate && local->list)
-				GF_FREE (local->list);
-		}
-
-		if ((local->op_ret >= 0) && local->failed && 
-		    local->revalidate) {
-			/* Done revalidate, but it failed */
-			if ((op_errno != ENOTCONN) 
-                            && (local->op_errno != ESTALE)) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Revalidate failed for path(%s): %s", 
-					local->loc1.path, strerror (op_errno));
-			}
-			local->op_ret = -1;
-		}
-
-		if ((priv->self_heal && !priv->optimist) && 
-		    (!local->revalidate && (local->op_ret == 0) && 
-		     IA_ISDIR(local->stbuf.ia_type))) {
-			/* Let the self heal be done here */
-			zr_unify_self_heal (frame, this, local);
-			local_dict = NULL;
-		} else {
-                        if (local->failed) {
-                                /* NOTE: directory lookup is sent to all 
-                                 * subvolumes and success from a subvolume
-                                 * might set local->op_ret to 0 (zero) */
-                                local->op_ret = -1;
-                        }
-
-			/* either no self heal, or op_ret == -1 (failure) */
-			tmp_inode = local->loc1.inode;
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      tmp_inode, &local->stbuf, local->dict,
-                                      &local->oldpostparent);
-		}
-		if (local_dict) {
-			dict_unref (local_dict);
-		}
-	}
-  
-	return 0;
-}
-
-/**
- * unify_lookup - 
- */
-int32_t 
-unify_lookup (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      dict_t *xattr_req)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	long index = 0;
-
-	if (!(loc && loc->inode)) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: Argument not right", loc?loc->path:"(null)");
-		STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL, NULL, NULL);
-		return 0;
-	}
-        
-        if (inode_ctx_get (loc->inode, this, NULL)
-            && IA_ISDIR (loc->inode->ia_type)) {
-                local->revalidate = 1;
-        }
-
-	if (!inode_ctx_get (loc->inode, this, NULL) && 
-	    loc->inode->ia_type && 
-	    !IA_ISDIR (loc->inode->ia_type)) {
-		uint64_t tmp_list = 0;
-		/* check if revalidate or fresh lookup */
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-	}
-
-	if (local->list) {
-		list = local->list;
-		for (index = 0; list[index] != -1; index++);
-		if (index != 2) {
-			if (index < 2) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning ESTALE for %s: file "
-					"count is %ld", loc->path, index);
-				/* Print where all the file is present */
-				for (index = 0; 
-				     local->list[index] != -1; index++) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"%s: found on %s", loc->path, 
-						priv->xl_array[list[index]]->name);
-				}
-				unify_local_wipe (local);
-				STACK_UNWIND (frame, -1, ESTALE, 
-					      NULL, NULL, NULL, NULL);
-				return 0;  
-			} else {
-				/* There are more than 2 presences */
-				/* Just log and continue */
-				gf_log (this->name, GF_LOG_ERROR,
-					"%s: file count is %ld", 
-					loc->path, index);
-				/* Print where all the file is present */
-				for (index = 0; 
-				     local->list[index] != -1; index++) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"%s: found on %s", loc->path, 
-						priv->xl_array[list[index]]->name);
-				}
-			}
-		}
-      
-		/* is revalidate */
-		local->revalidate = 1;
-      
-		for (index = 0; list[index] != -1; index++)
-			local->call_count++;
-      
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_lookup_cbk,
-					   (void *)(long)list[index], //cookie
-					   priv->xl_array [list[index]],
-					   priv->xl_array [list[index]]->fops->lookup,
-					   loc,
-					   xattr_req);
-			if (need_break)
-				break;
-		}
-	} else {
-		if (loc->inode->ia_type) {
-			if (inode_ctx_get (loc->inode, this, NULL)) {
-				inode_ctx_get (loc->inode, this, 
-					       &local->inode_generation);
-			}
-		}
-		/* This is first call, there is no list */
-		/* call count should be all child + 1 namespace */
-		local->call_count = priv->child_count + 1;
-      
-		for (index = 0; index <= priv->child_count; index++) {
-			STACK_WIND_COOKIE (frame,
-					   unify_lookup_cbk,
-					   (void *)index, //cookie
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->lookup,
-					   loc,
-					   xattr_req);
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_stat - if directory, get the stat directly from NameSpace child.
- *     if file, check for a hint and send it only there (also to NS).
- *     if its a fresh stat, then do it on all the nodes.
- *
- * NOTE: for all the call, sending cookie as xlator pointer, which will be 
- *       used in cbk.
- */
-int32_t
-unify_stat (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int16_t *list = NULL;
-	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL);
-		return 0;
-	}
-	local->ia_ino = loc->inode->ino;
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		/* Directory */
-		local->call_count = 1;
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->stat, loc);
-	} else {
-		/* File */
-		inode_ctx_get (loc->inode, this, &tmp_list);
-    		list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; list[index] != -1; index++)
-			local->call_count++;
-    
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->stat,
-				    loc);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_access_cbk -
- */
-int32_t
-unify_access_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-/**
- * unify_access - Send request to only namespace, which has all the 
- *      attributes set for the file.
- */
-int32_t
-unify_access (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t mask)
-{
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	STACK_WIND (frame,
-		    unify_access_cbk,
-		    NS(this),
-		    NS(this)->fops->access,
-		    loc,
-		    mask);
-
-	return 0;
-}
-
-int32_t
-unify_mkdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-                 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	inode_t *tmp_inode = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-  
-		if ((op_ret == -1) && !(priv->optimist && 
-					(op_errno == ENOENT || 
-					 op_errno == EEXIST))) {
-			/* TODO: Decrement the inode_generation of 
-			 * this->inode's parent inode, hence the missing 
-			 * directory is created properly by self-heal. 
-			 * Currently, there is no way to get the parent 
-			 * inode directly.
-			 */
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			if (op_errno != EEXIST)
-				local->failed = 1;
-			local->op_errno = op_errno;
-		}
-  
-		if (op_ret >= 0)
-			local->op_ret = 0;
-
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (!local->failed) {
-			inode_ctx_put (local->loc1.inode, this, 
-				       priv->inode_generation);
-		}
-		
-		tmp_inode = local->loc1.inode;
-		unify_local_wipe (local);
-
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      tmp_inode, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-/**
- * unify_ns_mkdir_cbk -
- */
-int32_t
-unify_ns_mkdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	long index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send mkdir request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR,
-			"namespace: path(%s): %s", 
-			local->name, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->stbuf = *buf;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	local->call_count = priv->child_count;
-
-	/* Send mkdir request to all the nodes now */
-	for (index = 0; index < priv->child_count; index++) {
-		STACK_WIND_COOKIE (frame,
-				   unify_mkdir_cbk,
-				   (void *)index, //cookie
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->mkdir,
-				   &local->loc1,
-				   local->mode);
-	}
-  
-	return 0;
-}
-
-
-/**
- * unify_mkdir -
- */
-int32_t
-unify_mkdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode)
-{
-	unify_local_t *local = NULL;
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-
-	loc_copy (&local->loc1, loc);
-
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_mkdir_cbk,
-		    NS(this),
-		    NS(this)->fops->mkdir,
-		    loc,
-		    mode);
-	return 0;
-}
-
-/**
- * unify_rmdir_cbk -
- */
-int32_t
-unify_rmdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == 0 || (priv->optimist && (op_errno == ENOENT)))
-			local->op_ret = 0;
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-/**
- * unify_ns_rmdir_cbk -
- */
-int32_t
-unify_ns_rmdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-  
-	if (op_ret == -1) {
-		/* No need to send rmdir request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, 
-			((op_errno != ENOTEMPTY) ? 
-			 GF_LOG_ERROR : GF_LOG_DEBUG),
-			"namespace: path(%s): %s", 
-			local->loc1.path, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, NULL, NULL);
-		return 0;
-	}
-
-	local->call_count = priv->child_count;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	for (index = 0; index < priv->child_count; index++) {
-		STACK_WIND (frame,
-			    unify_rmdir_cbk,
-			    priv->xl_array[index],
-			    priv->xl_array[index]->fops->rmdir,
-			    &local->loc1);
-	}
-
-	return 0;
-}
-
-/**
- * unify_rmdir -
- */
-int32_t
-unify_rmdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{
-	unify_local_t *local = NULL;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_rmdir_cbk,
-		    NS(this),
-		    NS(this)->fops->rmdir,
-		    loc);
-
-	return 0;
-}
-
-/**
- * unify_open_cbk -
- */
-int32_t
-unify_open_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			if (NS(this) != (xlator_t *)cookie) {
-				/* Store child node's ptr, used in 
-				   all the f*** / FileIO calls */
-				fd_ctx_set (fd, this, (uint64_t)(long)cookie);
-			}
-		}
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if ((local->failed == 1) && (local->op_ret >= 0)) {
-			local->call_count = 1;
-			/* return -1 to user */
-			local->op_ret = -1;
-			//local->op_errno = EIO; 
-      
-			if (!fd_ctx_get (local->fd, this, NULL)) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Open success on child node, "
-					"failed on namespace");
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Open success on namespace, "
-					"failed on child node");
-			}
-		}
-
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local->fd);
-	}
-
-	return 0;
-}
-
-#ifdef GF_DARWIN_HOST_OS
-/**
- * unify_create_lookup_cbk - 
- */
-int32_t 
-unify_open_lookup_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct iatt *buf,
-		       dict_t *dict,
-		       struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if ((op_ret == -1) && (op_errno != ENOENT)) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-		}
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret; 
-			local->index++;
-			if (NS(this) == priv->xl_array[(long)cookie]) {
-				local->list[0] = (int16_t)(long)cookie;
-			} else {
-				local->list[1] = (int16_t)(long)cookie;
-			}
-			if (IA_ISDIR (buf->ia_type))
-				local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		int16_t file_list[3] = {0,};
-		local->op_ret = -1;
-
-		file_list[0] = local->list[0];
-		file_list[1] = local->list[1];
-		file_list[2] = -1;
-
-		if (local->index != 2) {
-			/* Lookup failed, can't do open */
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: present on %d nodes", 
-				local->name, local->index);
-
-			if (local->index < 2) {
-				unify_local_wipe (local);
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning as file found on less "
-					"than 2 nodes");
-				STACK_UNWIND (frame, local->op_ret, 
-					      local->op_errno, local->fd);
-				return 0;
-			}
-		}
-
-		if (local->failed) {
-			/* Open on directory, return EISDIR */
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, -1, EISDIR, local->fd);
-			return 0;
-		}
-
-		/* Everything is perfect :) */    
-		local->call_count = 2;
-    
-		for (index = 0; file_list[index] != -1; index++) {
-			char need_break = (file_list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_open_cbk,
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]]->fops->open,
-					   &local->loc1,
-					   local->flags,
-					   local->fd, local->wbflags);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_open_readlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 const char *path,
-			 struct iatt *sbuf)
-{
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		STACK_UNWIND (frame, -1, ENOENT);
-		return 0;
-	}
-
-	if (path[0] == '/') {
-		local->name = gf_strdup (path);
-		ERR_ABORT (local->name);
-	} else {
-		char *tmp_str = gf_strdup (local->loc1.path);
-		char *tmp_base = dirname (tmp_str);
-		local->name = GF_CALLOC (1, ZR_PATH_MAX, gf_unify_mt_char);
-		strcpy (local->name, tmp_base);
-		strncat (local->name, "/", 1);
-		strcat (local->name, path);
-		GF_FREE (tmp_str);
-	}
-  
-	local->list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                 gf_unify_mt_int16_t);
-	ERR_ABORT (local->list);
-	local->call_count = priv->child_count + 1;
-	local->op_ret = -1;
-	for (index = 0; index <= priv->child_count; index++) {
-		/* Send the lookup to all the nodes including namespace */
-		STACK_WIND_COOKIE (frame,
-				   unify_open_lookup_cbk,
-				   (void *)(long)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->lookup,
-				   &local->loc1,
-				   NULL);
-	}
-
-	return 0;
-}
-#endif /* GF_DARWIN_HOST_OS */
-
-/**
- * unify_open - 
- */
-int32_t
-unify_open (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    int32_t flags,
-	    fd_t *fd,
-            int32_t wbflags)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int16_t file_list[3] = {0,};
-	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Init */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->fd    = fd;
-	local->flags = flags;
-        local->wbflags = wbflags;
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	local->list = list;
-	file_list[0] = priv->child_count; /* Thats namespace */
-	file_list[2] = -1;
-	for (index = 0; list[index] != -1; index++) {
-		local->call_count++;
-		if (list[index] != priv->child_count)
-			file_list[1] = list[index];
-	}
-
-	if (local->call_count != 2) {
-		/* If the lookup was done for file */
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: entry_count is %d",
-			loc->path, local->call_count);
-		for (index = 0; local->list[index] != -1; index++)
-			gf_log (this->name, GF_LOG_ERROR, "%s: found on %s",
-				loc->path, priv->xl_array[list[index]]->name);
-
-		if (local->call_count < 2) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"returning EIO as file found on onlyone node");
-			STACK_UNWIND (frame, -1, EIO, fd);
-			return 0;
-		}
-	}
-
-#ifdef GF_DARWIN_HOST_OS
-	/* Handle symlink here */
-	if (IA_ISLNK (loc->inode->ia_type)) {
-		/* Callcount doesn't matter here */
-		STACK_WIND (frame,
-			    unify_open_readlink_cbk,
-			    NS(this),
-			    NS(this)->fops->readlink,
-			    loc, ZR_PATH_MAX);
-		return 0;
-	}
-#endif /* GF_DARWIN_HOST_OS */
-
-	local->call_count = 2;
-	for (index = 0; file_list[index] != -1; index++) {
-		char need_break = (file_list[index+1] == -1);
-		STACK_WIND_COOKIE (frame,
-				   unify_open_cbk,
-				   priv->xl_array[file_list[index]], //cookie
-				   priv->xl_array[file_list[index]],
-				   priv->xl_array[file_list[index]]->fops->open,
-				   loc,
-				   flags,
-				   fd, wbflags);
-		if (need_break)
-			break;
-	}
-
-	return 0;
-}
-
-
-int32_t 
-unify_create_unlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-                         struct iatt *preparent,
-                         struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-	inode_t *inode = local->loc1.inode;
-
-	unify_local_wipe (local);
-
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, local->fd, 
-		      inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-  
-	return 0;
-}
-
-/**
- * unify_create_open_cbk -
- */
-int32_t
-unify_create_open_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       fd_t *fd)
-{
-	int ret = 0;
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	inode_t *inode = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_value = 0;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			if (NS(this) != (xlator_t *)cookie) {
-				/* Store child node's ptr, used in all 
-				   the f*** / FileIO calls */
-				/* TODO: log on failure */
-				ret = fd_ctx_get (fd, this, &tmp_value);
-				cookie = (void *)(long)tmp_value;
-			} else {
-				/* NOTE: open successful on namespace.
-				 *       fd's ctx can be used to identify open 
-				 *       failure on storage subvolume. cool 
-				 *       ide ;) */
-				local->failed = 0;
-			}
-		} else {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				((xlator_t *)cookie)->name,
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed == 1 && (local->op_ret >= 0)) {
-			local->call_count = 1;
-			/* return -1 to user */
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			local->fd = fd;
-			local->call_count = 1;
-
-			if (!fd_ctx_get (local->fd, this, &tmp_value)) {
-				child = (xlator_t *)(long)tmp_value;
-
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Create success on child node, "
-					"failed on namespace");
-
-				STACK_WIND (frame,
-					    unify_create_unlink_cbk,
-					    child,
-					    child->fops->unlink,
-					    &local->loc1);
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Create success on namespace, "
-					"failed on child node");
-
-				STACK_WIND (frame,
-					    unify_create_unlink_cbk,
-					    NS(this),
-					    NS(this)->fops->unlink,
-					    &local->loc1);
-			}
-			return 0;
-		}
-		inode = local->loc1.inode;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, fd,
-			      inode, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-	return 0;
-}
-
-/**
- * unify_create_lookup_cbk - 
- */
-int32_t 
-unify_create_lookup_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 inode_t *inode,
-			 struct iatt *buf,
-			 dict_t *dict,
-                         struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = op_ret; 
-			local->list[local->index++] = (int16_t)(long)cookie;
-			if (NS(this) == priv->xl_array[(long)cookie]) {
-				local->ia_ino = buf->ia_ino;
-			} else {
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		int16_t *list = local->list;
-		int16_t file_list[3] = {0,};
-		local->op_ret = -1;
-
-		local->list [local->index] = -1;
-		file_list[0] = list[0];
-		file_list[1] = list[1];
-		file_list[2] = -1;
-
-		local->stbuf.ia_ino = local->ia_ino;
-		/* TODO: log on failure */
-		inode_ctx_put (local->loc1.inode, this, 
-			       (uint64_t)(long)local->list);
-
-		if (local->index != 2) {
-			/* Lookup failed, can't do open */
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: present on %d nodes", 
-				local->loc1.path, local->index);
-			file_list[0] = priv->child_count;
-			for (index = 0; list[index] != -1; index++) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"%s: found on %s", local->loc1.path, 
-					priv->xl_array[list[index]]->name);
-				if (list[index] != priv->child_count)
-					file_list[1] = list[index];
-			}
-
-			if (local->index < 2) {
-				unify_local_wipe (local);
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning EIO as file found on "
-					"only one node");
-				STACK_UNWIND (frame, -1, EIO, 
-					      local->fd, inode, NULL,
-                                              NULL, NULL);
-				return 0;
-			}
-		}
-		/* Everything is perfect :) */    
-		local->call_count = 2;
-    
-		for (index = 0; file_list[index] != -1; index++) {
-			char need_break = (file_list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_create_open_cbk,
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]]->fops->open,
-					   &local->loc1,
-					   local->flags,
-					   local->fd, 0);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_create_cbk -
- */
-int32_t
-unify_create_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  fd_t *fd,
-		  inode_t *inode,
-		  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int ret = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-	inode_t *tmp_inode = NULL;
-
-	if (op_ret == -1) {
-		/* send unlink () on Namespace */
-		local->op_errno = op_errno;
-		local->op_ret = -1;
-		local->call_count = 1;
-		gf_log (this->name, GF_LOG_ERROR,
-			"create failed on %s (file %s, error %s), "
-			"sending unlink to namespace", 
-			prev_frame->this->name, 
-			local->loc1.path, strerror (op_errno));
-
-		STACK_WIND (frame,
-			    unify_create_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-
-		return 0;
-	}
-
-	if (op_ret >= 0) {
-		local->op_ret = op_ret;
-		local->stbuf = *buf;
-		/* Just inode number should be from NS node */
-		local->stbuf.ia_ino = local->ia_ino;
-
-		/* TODO: log on failure */
-		ret = fd_ctx_set (fd, this, (uint64_t)(long)prev_frame->this);
-	}
-  
-	tmp_inode = local->loc1.inode;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, local->fd, 
-		      tmp_inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_create_cbk -
- * 
- */
-int32_t
-unify_ns_create_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     fd_t *fd,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     struct iatt *preparent,
-                     struct iatt *postparent)
-{
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send create request to other servers, as 
-		   namespace action failed. Handle exclusive create here. */
-		if ((op_errno != EEXIST) || 
-		    ((op_errno == EEXIST) && 
-		     ((local->flags & O_EXCL) == O_EXCL))) {
-			/* If its just a create call without O_EXCL, 
-			   don't do this */
-			gf_log (this->name, GF_LOG_ERROR,
-				"namespace: path(%s): %s", 
-				local->loc1.path, strerror (op_errno));
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf,
-                                      preparent, postparent);
-			return 0;
-		}
-	}
-  
-	if (op_ret >= 0) {
-		/* Get the inode number from the NS node */
-		local->ia_ino = buf->ia_ino;
-  
-                local->oldpreparent = *preparent;
-                local->oldpostparent = *postparent;
-
-		local->op_ret = -1;
-
-		/* Start the mapping list */
-		list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                  gf_unify_mt_int16_t);
-		ERR_ABORT (list);
-		inode_ctx_put (inode, this, (uint64_t)(long)list);
-		list[0] = priv->child_count;
-		list[2] = -1;
-
-		/* This means, file doesn't exist anywhere in the Filesystem */
-		sched_ops = priv->sched_ops;
-
-		/* Send create request to the scheduled node now */
-		sched_xl = sched_ops->schedule (this, local->loc1.path);
-		if (sched_xl == NULL)
-		{
-			/* send unlink () on Namespace */
-			local->op_errno = ENOTCONN;
-			local->op_ret = -1;
-			local->call_count = 1;
-			gf_log (this->name, GF_LOG_ERROR,
-				"no node online to schedule create:(file %s) "
-				"sending unlink to namespace", 
-				(local->loc1.path)?local->loc1.path:"");
-
-			STACK_WIND (frame,
-				    unify_create_unlink_cbk,
-				    NS(this),
-				    NS(this)->fops->unlink,
-				    &local->loc1);
-	
-			return 0;
-		}
-
-		for (index = 0; index < priv->child_count; index++)
-			if (sched_xl == priv->xl_array[index])
-				break;
-		list[1] = index;
-
-		STACK_WIND (frame, unify_create_cbk,
-			    sched_xl, sched_xl->fops->create,
-			    &local->loc1, local->flags, local->mode, fd);
-	} else {
-		/* File already exists, and there is no O_EXCL flag */
-
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"File(%s) already exists on namespace, sending "
-			"open instead", local->loc1.path);
-
-		local->list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                         gf_unify_mt_int16_t);
-		ERR_ABORT (local->list);
-		local->call_count = priv->child_count + 1;
-		local->op_ret = -1;
-		for (index = 0; index <= priv->child_count; index++) {
-			/* Send lookup() to all nodes including namespace */
-			STACK_WIND_COOKIE (frame,
-					   unify_create_lookup_cbk,
-					   (void *)(long)index,
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->lookup,
-					   &local->loc1,
-					   NULL);
-		}
-	}
-	return 0;
-}
-
-/**
- * unify_create - create a file in global namespace first, so other 
- *    clients can see them. Create the file in storage nodes in background.
- */
-int32_t
-unify_create (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t flags,
-	      mode_t mode,
-	      fd_t *fd)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-	local->flags = flags;
-	local->fd = fd;
-
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, fd, loc->inode, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_create_cbk,
-		    NS(this),
-		    NS(this)->fops->create,
-		    loc,
-		    flags | O_EXCL,
-		    mode,
-		    fd);
-  
-	return 0;
-}
-
-
-/**
- * unify_opendir_cbk - 
- */
-int32_t
-unify_opendir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-
-	return 0;
-}
-
-/** 
- * unify_opendir -
- */
-int32_t
-unify_opendir (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       fd_t *fd)
-{
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	STACK_WIND (frame, unify_opendir_cbk,
-		    NS(this), NS(this)->fops->opendir, loc, fd);
-
-	return 0;
-}
-
-
-int32_t
-unify_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                   struct iatt *statpost)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s(): child(%s): path(%s): %s", 
-				gf_fop_list[frame->root->op],
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-
-			local->op_errno = op_errno;
-			if ((op_errno == ENOENT) && priv->optimist) 
-				local->op_ret = 0;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = statpost->ia_ino;
-				/* If the entry is directory, get the stat
-				   from NS node */
-				if (IA_ISDIR (statpost->ia_type) || 
-				    !local->stpost.ia_blksize) {
-					local->stpre = *statpre;
-                                        local->stpost = *statpost;
-				}
-			}
-
-			if ((!IA_ISDIR (statpost->ia_type)) && 
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from Storage 
-				   node. */
-				local->stpre = *statpre;
-                                local->stpost = *statpost;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		/* If the inode number is not filled, operation should
-		   fail */
-		if (!local->ia_ino)
-			local->op_ret = -1;
-
-                local->stpre.ia_ino = local->ia_ino;
-		local->stpost.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stpre, &local->stpost);
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-
-        if (!(loc && loc->inode)) {                     
-                STACK_UNWIND (frame, -1, EINVAL, NULL, NULL);
-                return 0;
-        }                                                    
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = 1;
-      
-                STACK_WIND (frame,
-                            unify_setattr_cbk,
-                            NS (this),
-                            NS (this)->fops->setattr,
-                            loc, stbuf, valid);
-        } else {
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		for (index = 0; local->list[index] != -1; index++) {
-			STACK_WIND (frame,
-				    unify_setattr_cbk,
-				    priv->xl_array[local->list[index]],
-				    priv->xl_array[local->list[index]]->fops->setattr,
-				    loc, stbuf, valid);
-
-			if (!--callcnt)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_setattr_cbk, child,
-			    child->fops->fsetattr, fd, stbuf, valid);
-
-		STACK_WIND (frame, unify_setattr_cbk, NS(this),
-			    NS(this)->fops->fsetattr, fd, stbuf, valid);
-	} else {
-		local->call_count = 1;
-    
-		STACK_WIND (frame, unify_setattr_cbk,
-			    NS(this), NS(this)->fops->fsetattr,
-			    fd, stbuf, valid);
-	}
-  
-	return 0;
-}
-
-
-/**
- * unify_truncate_cbk - 
- */
-int32_t
-unify_truncate_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iatt *prebuf,
-                    struct iatt *postbuf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-			if (!((op_errno == ENOENT) && priv->optimist))
-				local->op_ret = -1;
-		}
-
-		if (op_ret >= 0) {
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = postbuf->ia_ino;
-				/* If the entry is directory, get the 
-				   stat from NS node */
-				if (IA_ISDIR (postbuf->ia_type) ||
-				    !local->stbuf.ia_blksize) {
-					local->stbuf = *prebuf;
-                                        local->poststbuf = *postbuf;
-				}
-			}
-
-			if ((!IA_ISDIR (postbuf->ia_type)) &&
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from 
-				   Storage node. */
-				local->stbuf = *prebuf;
-                                local->poststbuf = *postbuf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		if (local->ia_ino) {
-			local->stbuf.ia_ino = local->ia_ino;
-                        local->poststbuf.ia_ino = local->ia_ino;
-                } else {
-			local->op_ret = -1;
-                }
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf, &local->poststbuf);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_truncate - 
- */
-int32_t
-unify_truncate (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		off_t offset)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->ia_ino = loc->inode->ino;
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = 1;
-      
-		STACK_WIND (frame,
-			    unify_truncate_cbk,
-			    NS(this),
-			    NS(this)->fops->truncate,
-			    loc,
-                            0);
-	} else {
-		local->op_ret = 0;
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		/* Don't send offset to NS truncate */
-		STACK_WIND (frame, unify_truncate_cbk, NS(this),
-			    NS(this)->fops->truncate, loc, 0);
-		callcnt--;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			if (NS(this) != priv->xl_array[local->list[index]]) {
-				STACK_WIND (frame,
-					    unify_truncate_cbk,
-					    priv->xl_array[local->list[index]],
-					    priv->xl_array[local->list[index]]->fops->truncate,
-					    loc,
-					    offset);
-				if (!--callcnt)
-					break;
-			}
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_readlink_cbk - 
- */
-int32_t
-unify_readlink_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    const char *path,
-                    struct iatt *sbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, path, sbuf);
-	return 0;
-}
-
-/**
- * unify_readlink - Read the link only from the storage node.
- */
-int32_t
-unify_readlink (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		size_t size)
-{
-	unify_private_t *priv = this->private;
-	int32_t entry_count = 0;
-	int16_t *list = NULL;
-	int16_t index = 0;
-  	uint64_t tmp_list = 0;
-  
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-	
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++)
-		entry_count++;
-
-	if (entry_count >= 2) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_readlink_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->readlink,
-					    loc,
-					    size);
-				break;
-			}
-		}
-	} else {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"returning ENOENT, no softlink files found "
-			"on storage node");
-		STACK_UNWIND (frame, -1, ENOENT, NULL);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_unlink_cbk - 
- */
-int32_t
-unify_unlink_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == 0  || ((op_errno == ENOENT) && priv->optimist))
-			local->op_ret = 0;
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-
-                if (((call_frame_t *)cookie)->this == NS(this)) {
-                        local->oldpreparent = *preparent;
-                        local->oldpostparent = *postparent;
-                }
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_unlink - 
- */
-int32_t
-unify_unlink (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++)
-		local->call_count++;
-
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND (frame,
-				    unify_unlink_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->unlink,
-				    loc);
-			if (need_break)
-				break;
-		}
-	} else {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: returning ENOENT", loc->path);
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_readv_cbk - 
- */
-int32_t
-unify_readv_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct iovec *vector,
-		 int32_t count,
-		 struct iatt *stbuf,
-                 struct iobref *iobref)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf, iobref);
-	return 0;
-}
-
-/**
- * unify_readv - 
- */
-int32_t
-unify_readv (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     size_t size,
-	     off_t offset)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame,
-		    unify_readv_cbk,
-		    child,
-		    child->fops->readv,
-		    fd,
-		    size,
-		    offset);
-
-
-	return 0;
-}
-
-/**
- * unify_writev_cbk - 
- */
-int32_t
-unify_writev_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-                  struct iatt *prebuf,
-		  struct iatt *postbuf)
-{
-	unify_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->stbuf = *prebuf;
-        local->stbuf.ia_ino = local->ia_ino;
-
-        local->poststbuf = *postbuf;
-        local->poststbuf.ia_ino = local->ia_ino;
-
-	STACK_UNWIND (frame, op_ret, op_errno,
-                      &local->stbuf, &local->poststbuf);
-	return 0;
-}
-
-/**
- * unify_writev - 
- */
-int32_t
-unify_writev (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      struct iovec *vector,
-	      int32_t count,
-	      off_t off,
-              struct iobref *iobref)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-	unify_local_t *local = NULL;
-
-	INIT_LOCAL (frame, local);
-        local->ia_ino = fd->inode->ino;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame,
-		    unify_writev_cbk,
-		    child,
-		    child->fops->writev,
-		    fd,
-		    vector,
-		    count,
-		    off,
-                    iobref);
-
-	return 0;
-}
-
-/**
- * unify_ftruncate -
- */
-int32_t
-unify_ftruncate (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 off_t offset)
-{
-	xlator_t *child = NULL;
-	unify_local_t *local = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->op_ret = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	local->call_count = 2;
-  
-	STACK_WIND (frame, unify_truncate_cbk,
-		    child, child->fops->ftruncate,
-		    fd, offset);
-  
-	STACK_WIND (frame, unify_truncate_cbk,
-		    NS(this), NS(this)->fops->ftruncate,
-		    fd, 0);
-  
-	return 0;
-}
-
-
-/**
- * unify_flush_cbk - 
- */
-int32_t
-unify_flush_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_flush -
- */
-int32_t
-unify_flush (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_flush_cbk, child, 
-		    child->fops->flush, fd);
-
-	return 0;
-}
-
-
-/**
- * unify_fsync_cbk - 
- */
-int32_t
-unify_fsync_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-                 struct iatt *prebuf,
-                 struct iatt *postbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-/**
- * unify_fsync - 
- */
-int32_t
-unify_fsync (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     int32_t flags)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fsync_cbk, child,
-		    child->fops->fsync, fd, flags);
-
-	return 0;
-}
-
-/**
- * unify_fstat - Send fstat FOP to Namespace only if its directory, and to 
- *     both namespace and the storage node if its a file.
- */
-int32_t
-unify_fstat (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	INIT_LOCAL (frame, local);
-	local->ia_ino = fd->inode->ino;
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_buf_cbk, child,
-			    child->fops->fstat, fd);
-
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fstat, fd);
-
-	} else {
-		/* this is an directory */
-		local->call_count = 1;
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fstat, fd);
-	}
-
-	return 0;
-}
-
-/**
- * unify_getdents_cbk - 
- */
-int32_t
-unify_getdents_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dir_entry_t *entry,
-		    int32_t count)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, entry, count);
-	return 0;
-}
-
-/**
- * unify_getdents - send the FOP request to all the nodes.
- */
-int32_t
-unify_getdents (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		size_t size,
-		off_t offset,
-		int32_t flag)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_getdents_cbk, NS(this),
-		    NS(this)->fops->getdents, fd, size, offset, flag);
-
-	return 0;
-}
-
-
-/**
- * unify_readdir_cbk - 
- */
-int32_t
-unify_readdir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-
-	return 0;
-}
-
-/**
- * unify_readdir - send the FOP request to all the nodes.
- */
-int32_t
-unify_readdir (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       size_t size,
-	       off_t offset)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_readdir_cbk, NS(this),
-		    NS(this)->fops->readdir, fd, size, offset);
-
-	return 0;
-}
-
-
-int32_t
-unify_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-
-	return 0;
-}
-
-
-int32_t
-unify_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_readdirp_cbk, NS(this),
-		    NS(this)->fops->readdirp, fd, size, offset);
-
-	return 0;
-}
-
-
-/**
- * unify_fsyncdir_cbk - 
- */
-int32_t
-unify_fsyncdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-
-	return 0;
-}
-
-/**
- * unify_fsyncdir -
- */
-int32_t
-unify_fsyncdir (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		int32_t flags)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_fsyncdir_cbk,
-		    NS(this), NS(this)->fops->fsyncdir, fd, flags);
-
-	return 0;
-}
-
-/**
- * unify_lk_cbk - UNWIND frame with the proper return arguments.
- */
-int32_t
-unify_lk_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct gf_flock *lock)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, lock);
-	return 0;
-}
-
-/**
- * unify_lk - Send it to all the storage nodes, (should be 1) which has file.
- */
-int32_t
-unify_lk (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd,
-	  int32_t cmd,
-	  struct gf_flock *lock)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_lk_cbk, child,
-		    child->fops->lk, fd, cmd, lock);
-
-	return 0;
-}
-
-
-int32_t
-unify_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno);
-
-static int32_t
-unify_setxattr_file_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	unify_private_t *private = this->private;
-	unify_local_t *local = frame->local;
-	xlator_t *sched_xl = NULL;
-	struct sched_ops *sched_ops = NULL;
-
-	if (op_ret == -1) {
-		if (!ENOTSUP)
-			gf_log (this->name, GF_LOG_ERROR,
-				"setxattr with XATTR_CREATE on ns: "
-				"path(%s) key(%s): %s",
-				local->loc1.path, local->name, 
-				strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno);
-		return 0;
-	} 
-
-	LOCK (&frame->lock);
-	{
-		local->failed = 0;
-		local->op_ret = 0;
-		local->op_errno = 0;
-		local->call_count = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	/* schedule XATTR_CREATE on one of the child node */
-	sched_ops = private->sched_ops;
-    
-	/* Send create request to the scheduled node now */
-	sched_xl = sched_ops->schedule (this, local->name); 
-	if (!sched_xl) {
-		STACK_UNWIND (frame, -1, ENOTCONN);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_setxattr_cbk,
-		    sched_xl,
-		    sched_xl->fops->setxattr,
-		    &local->loc1,
-		    local->dict,
-		    local->flags);
-	return 0;
-}
-
-/**
- * unify_setxattr_cbk - When all the child nodes return, UNWIND frame.
- */
-int32_t
-unify_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-	dict_t *dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, (((op_errno == ENOENT) || 
-					      (op_errno == ENOTSUP))? 
-					     GF_LOG_DEBUG : GF_LOG_ERROR), 
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-			if (local->failed == -1) {
-				local->failed = 1;
-			}
-			local->op_errno = op_errno;
-		} else {
-			local->failed = 0;
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed && local->name && 
-		    ZR_FILE_CONTENT_REQUEST(local->name)) {      
-			dict = get_new_dict ();
-			dict_set (dict, local->dict->members_list->key, 
-				  data_from_dynptr(NULL, 0));
-			dict_ref (dict);
-
-			local->call_count = 1;
-
-			STACK_WIND (frame,
-				    unify_setxattr_file_cbk,
-				    NS(this),
-				    NS(this)->fops->setxattr,
-				    &local->loc1,
-				    dict,
-				    XATTR_CREATE);
-
-			dict_unref (dict);
-			return 0;
-		}
-    
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_sexattr - This function should be sent to all the storage nodes, 
- *       which contains the file, (excluding namespace).
- */
-int32_t
-unify_setxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		dict_t *dict,
-		int32_t flags)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int32_t call_count = 0;
-  	uint64_t tmp_list = 0;
-	data_pair_t *trav = dict->members_list;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->failed = -1;
-	loc_copy (&local->loc1, loc);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-
-		if (trav && trav->key && ZR_FILE_CONTENT_REQUEST(trav->key)) {
-			/* direct the storage xlators to change file 
-			   content only if file exists */
-			local->flags = flags;
-			local->dict = dict;
-			local->name = gf_strdup (trav->key);
-			flags |= XATTR_REPLACE;
-		}
-
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++) {
-			STACK_WIND (frame,
-				    unify_setxattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->setxattr,
-				    loc, dict, flags);
-		}
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			call_count++;
-		}
-	}
-  
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_setxattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->setxattr,
-					    loc,
-					    dict,
-					    flags);
-				if (!--call_count)
-					break;
-			}
-		}
-		return 0;
-	}
-
-	/* No entry in storage nodes */
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"returning ENOENT, file not found on storage node.");
-	STACK_UNWIND (frame, -1, ENOENT);
-
-	return 0;
-}
-
-
-/**
- * unify_getxattr_cbk - This function is called from only one child, so, no
- *     need of any lock or anything else, just send it to above layer 
- */
-int32_t
-unify_getxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dict_t *value)
-{
-	int32_t callcnt = 0;
-	dict_t *local_value = NULL;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, 
-				(((op_errno == ENOENT) || 
-				  (op_errno == ENODATA) || 
-				  (op_errno == ENOTSUP)) ? 
-				 GF_LOG_DEBUG : GF_LOG_ERROR), 
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-		} else {
-			if (!local->dict)
-				local->dict = dict_ref (value);
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local_value = local->dict;
-		local->dict = NULL;
-      
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local_value);
-      
-		if (local_value)
-			dict_unref (local_value);
-	} 
-
-	return 0;
-}
-
-
-/** 
- * unify_getxattr - This FOP is sent to only the storage node.
- */
-int32_t
-unify_getxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		const char *name)
-{
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int16_t count = 0;
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-	INIT_LOCAL (frame, local);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++)
-			STACK_WIND (frame,
-				    unify_getxattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->getxattr,
-				    loc,
-				    name);
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			count++;
-		}
-	}
-
-	if (count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_getxattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->getxattr,
-					    loc,
-					    name);
-				if (!--count)
-					break;
-			}
-		}
-	} else {
-		dict_t *tmp_dict = get_new_dict ();
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"%s: returning ENODATA, no file found on storage node",
-			loc->path);
-		STACK_UNWIND (frame, -1, ENODATA, tmp_dict);
-		dict_destroy (tmp_dict);
-	}
-
-	return 0;
-}
-
-/**
- * unify_removexattr_cbk - Wait till all the child node returns the call
- *      and then UNWIND to above layer.
- */
-int32_t
-unify_removexattr_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{ 
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			if (op_errno != ENOTSUP)
-				gf_log (this->name, GF_LOG_ERROR, 
-					"child(%s): path(%s): %s", 
-					prev_frame->this->name, 
-					local->loc1.path, strerror (op_errno));
-		} else {
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);  
-
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_removexattr - Send it to all the child nodes which has the files.
- */
-int32_t
-unify_removexattr (call_frame_t *frame,
-		   xlator_t *this,
-		   loc_t *loc,
-		   const char *name)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int32_t call_count = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++)
-			STACK_WIND (frame, 		    
-				    unify_removexattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->removexattr,
-				    loc,
-				    name);
-
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			call_count++;
-		}
-	}
-
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_removexattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->removexattr,
-					    loc,
-					    name);
-				if (!--call_count)
-					break;
-			}
-		}
-		return 0;
-	} 
-
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"%s: returning ENOENT, not found on storage node.", loc->path);
-	STACK_UNWIND (frame, -1, ENOENT);
-
-	return 0;
-}
-
-
-int32_t 
-unify_mknod_unlink_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-                        struct iatt *preparent,
-                        struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1)
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: %s", local->loc1.path, strerror (op_errno));
-  
-	unify_local_wipe (local);
-	/* No log required here as this -1 is for mknod call */
-	STACK_UNWIND (frame, -1, local->op_errno, NULL, NULL);
-	return 0;
-}
-
-/**
- * unify_mknod_cbk - 
- */
-int32_t
-unify_mknod_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-                 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"mknod failed on storage node, sending unlink to "
-			"namespace");
-		local->op_errno = op_errno;
-		STACK_WIND (frame,
-			    unify_mknod_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-		return 0;
-	}
-  
-	local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-	return 0;
-}
-
-/**
- * unify_ns_mknod_cbk - 
- */
-int32_t
-unify_ns_mknod_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	call_frame_t *prev_frame = cookie;
-
-	if (op_ret == -1) {
-		/* No need to send mknod request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"child(%s): path(%s): %s", 
-			prev_frame->this->name, local->loc1.path, 
-			strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                              preparent, postparent);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->stbuf = *buf;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	list = GF_CALLOC (1, sizeof (int16_t) * 3, gf_unify_mt_int16_t);
-	ERR_ABORT (list);
-	list[0] = priv->child_count;
-	list[2] = -1;
-	inode_ctx_put (inode, this, (uint64_t)(long)list);
-
-	sched_ops = priv->sched_ops;
-
-	/* Send mknod request to scheduled node now */
-	sched_xl = sched_ops->schedule (this, local->loc1.path); 
-	if (!sched_xl) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"mknod failed on storage node, no node online "
-			"at the moment, sending unlink to NS");
-		local->op_errno = ENOTCONN;
-		STACK_WIND (frame,
-			    unify_mknod_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-      
-		return 0;
-	}
-
-	for (index = 0; index < priv->child_count; index++)
-		if (sched_xl == priv->xl_array[index])
-			break;
-	list[1] = index;
-  
-	STACK_WIND (frame,  unify_mknod_cbk,
-		    sched_xl,  sched_xl->fops->mknod,
-		    &local->loc1, local->mode, local->dev);
-
-	return 0;
-}
-
-/**
- * unify_mknod - Create a device on namespace first, and later create on 
- *       the storage node.
- */
-int32_t
-unify_mknod (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode,
-	     dev_t rdev)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-	local->dev = rdev;
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_mknod_cbk,
-		    NS(this),
-		    NS(this)->fops->mknod,
-		    loc,
-		    mode,
-		    rdev);
-
-	return 0;
-}
-
-int32_t 
-unify_symlink_unlink_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-                          struct iatt *preparent,
-                          struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-	if (op_ret == -1)
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: %s", local->loc1.path, strerror (op_errno));
-
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, -1, local->op_errno, NULL, NULL);
-	return 0;
-}
-
-/**
- * unify_symlink_cbk - 
- */
-int32_t
-unify_symlink_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		/* Symlink on storage node failed, hence send unlink 
-		   to the NS node */
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, 
-			"symlink on storage node failed, sending unlink "
-			"to namespace");
-
-		STACK_WIND (frame,
-			    unify_symlink_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-    
-		return 0;
-	}
-  
-	local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_symlink_cbk - 
- */
-int32_t
-unify_ns_symlink_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      inode_t *inode,
-                      struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent)
-{
-
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	int16_t *list = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send symlink request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s): %s", 
-			local->loc1.path, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, NULL, buf,
-                              preparent, postparent);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->ia_ino = buf->ia_ino;
-  
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	/* Start the mapping list */
-
-	list = GF_CALLOC (1, sizeof (int16_t) * 3, gf_unify_mt_int16_t);
-	ERR_ABORT (list);
-	list[0] = priv->child_count; //namespace's index
-	list[2] = -1;
-	inode_ctx_put (inode, this, (uint64_t)(long)list);
-
-	sched_ops = priv->sched_ops;
-
-	/* Send symlink request to all the nodes now */
-	sched_xl = sched_ops->schedule (this, local->loc1.path); 
-	if (!sched_xl) {
-		/* Symlink on storage node failed, hence send unlink 
-		   to the NS node */
-		local->op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_ERROR, 
-			"symlink on storage node failed, no node online, "
-			"sending unlink to namespace");
-      
-		STACK_WIND (frame,
-			    unify_symlink_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-      
-		return 0;
-	}
-
-	for (index = 0; index < priv->child_count; index++)
-		if (sched_xl == priv->xl_array[index])
-			break;
-	list[1] = index;
-	
-	STACK_WIND (frame,
-		    unify_symlink_cbk,
-		    sched_xl,
-		    sched_xl->fops->symlink,
-		    local->name,
-		    &local->loc1);
-
-	return 0;
-}
-
-/**
- * unify_symlink - 
- */
-int32_t
-unify_symlink (call_frame_t *frame,
-	       xlator_t *this,
-	       const char *linkpath,
-	       loc_t *loc)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->name = gf_strdup (linkpath);
-
-	if ((local->name == NULL) || 
-	    (local->loc1.path == NULL)) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_symlink_cbk,
-		    NS(this),
-		    NS(this)->fops->symlink,
-		    linkpath,
-		    loc);
-
-	return 0;
-}
-
-
-int32_t 
-unify_rename_unlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-                         struct iatt *preparent,
-                         struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"child(%s): path(%s -> %s): %s", 
-			prev_frame->this->name, 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-      
-	}
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local->stbuf.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-	return 0;
-}
-
-int32_t 
-unify_ns_rename_undo_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  struct iatt *buf,
-                          struct iatt *preoldparent,
-                          struct iatt *postoldparent,
-                          struct iatt *prenewparent,
-                          struct iatt *postnewparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-	}
-
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, &local->stbuf);
-	return 0;
-}
-
-int32_t 
-unify_rename_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iatt *buf,
-                  struct iatt *preoldparent,
-                  struct iatt *postoldparent,
-                  struct iatt *prenewparent,
-                  struct iatt *postnewparent)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	int16_t *list = NULL;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (!IA_ISDIR (buf->ia_type))
-				local->stbuf = *buf;
-			local->op_ret = op_ret;
-		} else {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"child(%s): path(%s -> %s): %s", 
-				prev_frame->this->name, 
-				local->loc1.path, local->loc2.path, 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local->stbuf.ia_ino = local->ia_ino;
-		if (IA_ISDIR (local->loc1.inode->ia_type)) {
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                                      &local->stbuf, &local->oldpreparent,
-                                      &local->oldpostparent, &local->newpreparent,
-                                      &local->newpostparent);
-			return 0;
-		}
-
-		if (local->op_ret == -1) {
-			/* TODO: check this logic */
-
-			/* Rename failed in storage node, successful on NS, 
-			 * hence, rename back the entries in NS */
-			/* NOTE: this will be done only if the destination 
-			 * doesn't exists, if  the destination exists, the 
-			 * job of correcting NS is left to self-heal
-			 */
-			if (!local->index) {
-				loc_t tmp_oldloc = {
-                                        /* its actual 'newloc->path' */
-					.path = local->loc2.path, 
-					.inode = local->loc1.inode,
-					.parent = local->loc2.parent
-				};
-	
-				loc_t tmp_newloc = {
-					/* Actual 'oldloc->path' */
-					.path = local->loc1.path,
-					.parent = local->loc1.parent
-				};
-
-				gf_log (this->name, GF_LOG_ERROR, 
-					"rename succussful on namespace, on "
-					"stroage node failed, reverting back");
-
-				STACK_WIND (frame,
-					    unify_ns_rename_undo_cbk,
-					    NS(this),
-					    NS(this)->fops->rename,
-					    &tmp_oldloc,
-					    &tmp_newloc);
-				return 0;
-			}
-		} else {
-			/* Rename successful on storage nodes */
-
-			int32_t idx = 0;
-			int16_t *tmp_list = NULL;
-			uint64_t tmp_list_int64 = 0;
-			if (local->loc2.inode) {
-				inode_ctx_get (local->loc2.inode, 
-					       this, &tmp_list_int64);
-				list = (int16_t *)(long)tmp_list_int64;
-
-			}
-
-			if (list) {				
-				for (index = 0; list[index] != -1; index++);
-				tmp_list = GF_CALLOC (1, index * 2, 
-                                                      gf_unify_mt_int16_t);
-				memcpy (tmp_list, list, index * 2);
-
-				for (index = 0; list[index] != -1; index++) {
-					/* TODO: Check this logic. */
-					/* If the destination file exists in 
-					 * the same storage node where we sent
-					 * 'rename' call, no need to send 
-					 * unlink 
-					 */
-					for (idx = 0; 
-					     local->list[idx] != -1; idx++) {
-						if (tmp_list[index] == local->list[idx]) {
-							tmp_list[index] = priv->child_count;
-							continue;
-						}
-					}
-	  
-					if (NS(this) != priv->xl_array[tmp_list[index]]) {
-						local->call_count++;
-						callcnt++;
-					}
-				}
-
-				if (local->call_count) {
-					if (callcnt > 1)
-						gf_log (this->name, 
-							GF_LOG_ERROR, 
-							"%s->%s: more (%d) "
-							"subvolumes have the "
-							"newloc entry", 
-							local->loc1.path, 
-							local->loc2.path, 
-							callcnt);
-
-					for (index=0; 
-					     tmp_list[index] != -1; index++) {
-						if (NS(this) != priv->xl_array[tmp_list[index]]) {		    
-							STACK_WIND (frame,
-								    unify_rename_unlink_cbk,
-								    priv->xl_array[tmp_list[index]],
-								    priv->xl_array[tmp_list[index]]->fops->unlink,
-								    &local->loc2);
-							if (!--callcnt)
-								break;
-						}
-					}
-
-					GF_FREE (tmp_list);
-					return 0;
-				}
-				if (tmp_list)
-					GF_FREE (tmp_list);
-			}
-		}
-    
-		/* Need not send 'unlink' to storage node */
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent,
-                              &local->newpreparent, &local->newpostparent);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_ns_rename_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf,
-                     struct iatt *preoldparent,
-                     struct iatt *postoldparent,
-                     struct iatt *prenewparent,
-                     struct iatt *postnewparent)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	int16_t *list = NULL;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		/* Free local->new_inode */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, buf,
-                              preoldparent, postoldparent,
-                              prenewparent, postnewparent);
-		return 0;
-	}
-
-	local->stbuf = *buf;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preoldparent;
-        local->oldpostparent = *postoldparent;
-        local->newpreparent = *prenewparent;
-        local->newpostparent = *postnewparent;
-
-	/* Everything is fine. */
-	if (IA_ISDIR (buf->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index=0; index < priv->child_count; index++) {
-			STACK_WIND (frame,
-				    unify_rename_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->rename,
-				    &local->loc1,
-				    &local->loc2);
-		}
-
-		return 0;
-	}
-
-	local->call_count = 0;  
-	/* send rename */
-	list = local->list;
-	for (index=0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			callcnt++;
-		}
-	}
-
-	if (local->call_count) {
-		for (index=0; list[index] != -1; index++) {
-			if (NS(this) != priv->xl_array[list[index]]) {
-				STACK_WIND (frame,
-					    unify_rename_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->rename,
-					    &local->loc1,
-					    &local->loc2);
-				if (!--callcnt)
-					break;
-			}
-		}
-	} else {
-		/* file doesn't seem to be present in storage nodes */
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"CRITICAL: source file not in storage node, "
-			"rename successful on namespace :O");
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, -1, EIO, NULL,
-                              NULL, NULL, /* preoldparent, postoldparent */
-                              NULL, NULL); /* prenewparent, postnewparent */
-	}
-	return 0;
-}
-
-
-/**
- * unify_rename - One of the tricky function. The deadliest of all :O
- */
-int32_t
-unify_rename (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *oldloc,
-	      loc_t *newloc)
-{
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, oldloc);
-	loc_copy (&local->loc2, newloc);
-
-	if ((local->loc1.path == NULL) || 
-	    (local->loc2.path == NULL)) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL,
-                              NULL, NULL, /* preoldparent, postoldparent */
-                              NULL, NULL); /* prenewparent, postnewparent */
-		return 0;
-	}
-  
-	inode_ctx_get (oldloc->inode, this, &tmp_list);
-	local->list = (int16_t *)(long)tmp_list;
-
-	STACK_WIND (frame,
-		    unify_ns_rename_cbk,
-		    NS(this),
-		    NS(this)->fops->rename,
-		    oldloc,
-		    newloc);
-	return 0;
-}
-
-/**
- * unify_link_cbk -
- */
-int32_t
-unify_link_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-                struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret >= 0) 
-		local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_link_cbk - 
- */
-int32_t
-unify_ns_link_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	int16_t *list = local->list;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send link request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                              preparent, postparent);
-		return 0;
-	}
-
-	/* Update inode for this entry */
-	local->op_ret = 0;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	/* Send link request to the node now */
-	for (index = 0; list[index] != -1; index++) {
-		char need_break = (list[index+1] == -1);
-		if (priv->xl_array[list[index]] != NS (this)) {
-			STACK_WIND (frame,
-				    unify_link_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->link,
-				    &local->loc1,
-				    &local->loc2);
-                        break;
-		}
-		if (need_break)
-			break;
-	}
-
-	return 0;
-}
-
-/**
- * unify_link - 
- */
-int32_t
-unify_link (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *oldloc,
-	    loc_t *newloc)
-{
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (oldloc);
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (newloc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, oldloc);
-	loc_copy (&local->loc2, newloc);
-
-	inode_ctx_get (oldloc->inode, this, &tmp_list);
-	local->list = (int16_t *)(long)tmp_list;
-
-	STACK_WIND (frame,
-		    unify_ns_link_cbk,
-		    NS(this),
-		    NS(this)->fops->link,
-		    oldloc,
-		    newloc);
-
-	return 0;
-}
-
-
-/**
- * unify_checksum_cbk - 
- */
-int32_t
-unify_checksum_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    uint8_t *fchecksum,
-		    uint8_t *dchecksum)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fchecksum, dchecksum);
-
-	return 0;
-}
-
-/**
- * unify_checksum - 
- */
-int32_t
-unify_checksum (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		int32_t flag)
-{
-	STACK_WIND (frame,
-		    unify_checksum_cbk,
-		    NS(this),
-		    NS(this)->fops->checksum,
-		    loc,
-		    flag);
-
-	return 0;
-}
-
-
-/**
- * unify_finodelk_cbk - 
- */
-int
-unify_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_finodelk
- */
-int
-unify_finodelk (call_frame_t *frame, xlator_t *this,
-		const char *volume, fd_t *fd, int cmd, struct gf_flock *flock)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_finodelk_cbk,
-		    child, child->fops->finodelk,
-		    volume, fd, cmd, flock);
-
-	return 0;
-}
-
-
-
-/**
- * unify_fentrylk_cbk - 
- */
-int
-unify_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_fentrylk
- */
-int
-unify_fentrylk (call_frame_t *frame, xlator_t *this,
-		const char *volume, fd_t *fd, const char *basename,
-		entrylk_cmd cmd, entrylk_type type)
-		
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fentrylk_cbk,
-		    child, child->fops->fentrylk,
-		    volume, fd, basename, cmd, type);
-
-	return 0;
-}
-
-
-
-/**
- * unify_fxattrop_cbk - 
- */
-int
-unify_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, xattr);
-	return 0;
-}
-
-/**
- * unify_fxattrop
- */
-int
-unify_fxattrop (call_frame_t *frame, xlator_t *this,
-		fd_t *fd, gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fxattrop_cbk,
-		    child, child->fops->fxattrop,
-		    fd, optype, xattr);
-
-	return 0;
-}
-
-
-/**
- * unify_inodelk_cbk - 
- */
-int
-unify_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-/**
- * unify_inodelk
- */
-int
-unify_inodelk (call_frame_t *frame, xlator_t *this,
-	       const char *volume, loc_t *loc, int cmd, struct gf_flock *flock)
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_inodelk_cbk,
-		    child, child->fops->inodelk,
-		    volume, loc, cmd, flock);
-
-	return 0;
-}
-
-
-
-/**
- * unify_entrylk_cbk - 
- */
-int
-unify_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_entrylk
- */
-int
-unify_entrylk (call_frame_t *frame, xlator_t *this,
-	       const char *volume, loc_t *loc, const char *basename,
-	       entrylk_cmd cmd, entrylk_type type)
-		
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_entrylk_cbk,
-		    child, child->fops->entrylk,
-		    volume, loc, basename, cmd, type);
-
-	return 0;
-}
-
-
-
-/**
- * unify_xattrop_cbk - 
- */
-int
-unify_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, xattr);
-	return 0;
-}
-
-/**
- * unify_xattrop
- */
-int
-unify_xattrop (call_frame_t *frame, xlator_t *this,
-		loc_t *loc, gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_xattrop_cbk,
-		    child, child->fops->xattrop,
-		    loc, optype, xattr);
-
-	return 0;
-}
-
-int
-unify_forget (xlator_t *this,
-              inode_t *inode)
-{
-        int16_t *list = NULL;
-        uint64_t tmp_list = 0;
-
-        if (inode->ia_type && (!IA_ISDIR(inode->ia_type))) {
-                inode_ctx_get (inode, this, &tmp_list);
-                if (tmp_list) {
-                        list = (int16_t *)(long)tmp_list;
-                        GF_FREE (list);
-                }
-        }
-
-        return 0;
-}
-
-/**
- * notify
- */
-int32_t
-notify (xlator_t *this,
-        int32_t event,
-        void *data,
-        ...)
-{
-	unify_private_t *priv = this->private;
-	struct sched_ops *sched = NULL;
-
-	if (!priv) {
-		return 0;
-	}
-
-	sched = priv->sched_ops;    
-	if (!sched) {
-		gf_log (this->name, GF_LOG_CRITICAL, "No scheduler :O");
-		raise (SIGTERM);
-		return 0;
-	}
-	if (priv->namespace == data) {
-		if (event == GF_EVENT_CHILD_UP) {
-			sched->notify (this, event, data);
-		}
-		return 0;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_UP:
-	{
-		/* Call scheduler's update () to enable it for scheduling */
-		sched->notify (this, event, data);
-	
-		LOCK (&priv->lock);
-		{
-			/* Increment the inode's generation, which is 
-			   used for self_heal */
-			++priv->inode_generation;
-			++priv->num_child_up;
-		}
-		UNLOCK (&priv->lock);
-
-		if (!priv->is_up) {
-			default_notify (this, event, data);
-			priv->is_up = 1;
-		}
-	}
-	break;
-	case GF_EVENT_CHILD_DOWN:
-	{
-		/* Call scheduler's update () to disable the child node 
-		 * for scheduling
-		 */
-		sched->notify (this, event, data);
-		LOCK (&priv->lock);
-		{
-			--priv->num_child_up;
-		}
-		UNLOCK (&priv->lock);
-
-		if (priv->num_child_up == 0) {
-			/* Send CHILD_DOWN to upper layer */
-			default_notify (this, event, data);
-			priv->is_up = 0;
-		}
-	}
-	break;
-
-	default:
-	{
-		default_notify (this, event, data);
-	}
-	break;
-	}
-
-	return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_unify_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-/** 
- * init - This function is called first in the xlator, while initializing.
- *   All the config file options are checked and appropriate flags are set.
- *
- * @this - 
- */
-int32_t 
-init (xlator_t *this)
-{
-	int32_t          ret       = 0;
-	int32_t          count     = 0;
-	data_t          *scheduler = NULL;
-	data_t          *data      = NULL;
-	xlator_t        *ns_xl     = NULL;
-	xlator_list_t   *trav      = NULL;
-	xlator_list_t   *xlparent  = NULL;
-	xlator_list_t   *parent    = NULL;
-	unify_private_t *_private  = NULL; 
-
-
-	/* Check for number of child nodes, if there is no child nodes, exit */
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"No child nodes specified. check \"subvolumes \" "
-			"option in volfile");
-		return -1;
-	}
-
-  	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	/* Check for 'scheduler' in volume */
-	scheduler = dict_get (this->options, "scheduler");
-	if (!scheduler) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"\"option scheduler <x>\" is missing in volfile");
-		return -1;
-	}
-
-	/* Setting "option namespace <node>" */
-	data = dict_get (this->options, "namespace");
-	if(!data) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace option not specified, Exiting");
-		return -1;
-	}
-	/* Search namespace in the child node, if found, exit */
-	trav = this->children;
-	while (trav) {
-		if (strcmp (trav->xlator->name, data->data) == 0)
-			break;
-		trav = trav->next;
-	}
-	if (trav) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace node used as a subvolume, Exiting");
-		return -1;
-	}
-	
-	/* Search for the namespace node, if found, continue */
-	ns_xl = this->next;
-	while (ns_xl) {
-		if (strcmp (ns_xl->name, data->data) == 0)
-			break;
-		ns_xl = ns_xl->next;
-	}
-	if (!ns_xl) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace node not found in volfile, Exiting");
-		return -1;
-	}
-	
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"namespace node specified as %s", data->data);
-	
-	_private = GF_CALLOC (1, sizeof (*_private), 
-                              gf_unify_mt_unify_private_t);
-	ERR_ABORT (_private);
-	_private->sched_ops = get_scheduler (this, scheduler->data);
-	if (!_private->sched_ops) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"Error while loading scheduler. Exiting");
-		GF_FREE (_private);
-		return -1;
-	}
-	
-	if (ns_xl->parents) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"Namespace node should not be a child of any other node. Exiting");
-		GF_FREE (_private);
-		return -1;
-	}
-
-	_private->namespace = ns_xl;
-	
-	/* update _private structure */
-	{
-		count = 0;
-		trav = this->children;
-		/* Get the number of child count */
-		while (trav) {
-			count++;
-			trav = trav->next;
-		}
-		
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"Child node count is %d", count);    
-
-		_private->child_count = count;
-		if (count == 1) {
-			/* TODO: Should I error out here? */
-			gf_log (this->name, GF_LOG_CRITICAL, 
-				"WARNING: You have defined only one "
-				"\"subvolumes\" for unify volume. It may not "
-				"be the desired config, review your volume "
-				"volfile. If this is how you are testing it,"
-				" you may hit some performance penalty");
-		}
-		
-		_private->xl_array = GF_CALLOC (1, 
-					        sizeof (xlator_t) * (count + 1),
-                                                gf_unify_mt_xlator_t);
-		ERR_ABORT (_private->xl_array);
-		
-		count = 0;
-		trav = this->children;
-		while (trav) {
-			_private->xl_array[count++] = trav->xlator;
-			trav = trav->next;
-		}
-		_private->xl_array[count] = _private->namespace;
-		
-		/* self-heal part, start with generation '1' */
-		_private->inode_generation = 1; 
-                /* Because, Foreground part is tested well */
-		_private->self_heal = ZR_UNIFY_FG_SELF_HEAL; 
-		data = dict_get (this->options, "self-heal");
-		if (data) {
-			if (strcasecmp (data->data, "off") == 0) 
-				_private->self_heal = ZR_UNIFY_SELF_HEAL_OFF;
-
-			if (strcasecmp (data->data, "foreground") == 0)
-				_private->self_heal = ZR_UNIFY_FG_SELF_HEAL;
-
-			if (strcasecmp (data->data, "background") == 0)
-				_private->self_heal = ZR_UNIFY_BG_SELF_HEAL;
-		}
-    
-		/* optimist - ask bulde for more about it */
-		data = dict_get (this->options, "optimist");
-		if (data) {
-			if (gf_string2boolean (data->data, 
-					       &_private->optimist) == -1) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"optimist excepts only boolean "
-					"options");
-			}
-		}
-
-		LOCK_INIT (&_private->lock);
-	}
-
-	/* Now that everything is fine. */
-	this->private = (void *)_private;
-	{
-                ret = _private->sched_ops->mem_acct_init (this);
-
-                if (ret == -1) {
-                        return -1;
-                }
-
-		/* Initialize scheduler, if everything else is successful */
-		ret = _private->sched_ops->init (this); 
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_CRITICAL,
-				"Initializing scheduler failed, Exiting");
-			GF_FREE (_private);
-			return -1;
-		}
-
-
-		ret = 0;
-
-		/* This section is required because some fops may look 
-		 * for 'xl->parent' variable 
-		 */
-		xlparent = GF_CALLOC (1, sizeof (*xlparent),
-                                      gf_unify_mt_xlator_list_t);
-		xlparent->xlator = this;
-		if (!ns_xl->parents) {
-			ns_xl->parents = xlparent;
-		} else {
-			parent = ns_xl->parents;
-			while (parent->next)
-				parent = parent->next;
-			parent->next = xlparent;
-		}
-	}
-
-	/* Tell namespace node that init is done */
-	xlator_notify (ns_xl, GF_EVENT_PARENT_UP, this);
-
-	return 0;
-}
-
-/** 
- * fini  - Free all the allocated memory 
- */
-void
-fini (xlator_t *this)
-{
-	unify_private_t *priv = this->private;
-	priv->sched_ops->fini (this);
-	this->private = NULL;
-	LOCK_DESTROY (&priv->lock);
-	GF_FREE (priv->xl_array);
-	GF_FREE (priv);
-	return;
-}
-
-
-struct xlator_fops fops = {
-	.stat        = unify_stat,
-	.readlink    = unify_readlink,
-	.mknod       = unify_mknod,
-	.mkdir       = unify_mkdir,
-	.unlink      = unify_unlink,
-	.rmdir       = unify_rmdir,
-	.symlink     = unify_symlink,
-	.rename      = unify_rename,
-	.link        = unify_link,
-	.truncate    = unify_truncate,
-	.create      = unify_create,
-	.open        = unify_open,
-	.readv       = unify_readv,
-	.writev      = unify_writev,
-	.statfs      = unify_statfs,
-	.flush       = unify_flush,
-	.fsync       = unify_fsync,
-	.setxattr    = unify_setxattr,
-	.getxattr    = unify_getxattr,
-	.removexattr = unify_removexattr,
-	.opendir     = unify_opendir,
-	.readdir     = unify_readdir,
-	.readdirp    = unify_readdirp,
-	.fsyncdir    = unify_fsyncdir,
-	.access      = unify_access,
-	.ftruncate   = unify_ftruncate,
-	.fstat       = unify_fstat,
-	.lk          = unify_lk,
-	.lookup      = unify_lookup,
-	.getdents    = unify_getdents,
-	.checksum    = unify_checksum,
-	.inodelk     = unify_inodelk,
-	.finodelk    = unify_finodelk,
-	.entrylk     = unify_entrylk,
-	.fentrylk    = unify_fentrylk,
-	.xattrop     = unify_xattrop,
-	.fxattrop    = unify_fxattrop,
-        .setattr     = unify_setattr, 
-        .fsetattr    = unify_fsetattr,
-};
-
-
-struct xlator_cbks cbks = {
-        .forget  = unify_forget,
-};
-
-struct volume_options options[] = {
-	{ .key   = { "namespace" },  
-	  .type  = GF_OPTION_TYPE_XLATOR 
-	},
-	{ .key   = { "scheduler" },  
-	  .value = { "alu", "rr", "random", "nufa", "switch" },
-	  .type  = GF_OPTION_TYPE_STR
-	},
-	{ .key   = {"self-heal"},  
-	  .value = { "foreground", "background", "off" },
-	  .type  = GF_OPTION_TYPE_STR
-	},
-	/* TODO: remove it some time later */
-	{ .key   = {"optimist"},  
-	  .type  = GF_OPTION_TYPE_BOOL 
-	},
-
-	{ .key   = {NULL} },
-};
diff --git a/xlators/cluster/unify/src/unify.h b/xlators/cluster/unify/src/unify.h
deleted file mode 100644
index dbd5e44a2..000000000
--- a/xlators/cluster/unify/src/unify.h
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef _UNIFY_H
-#define _UNIFY_H
-
-#include "scheduler.h"
-#include "list.h"
-#include "unify-mem-types.h"
-
-#define MAX_DIR_ENTRY_STRING     (32 * 1024)
-
-#define ZR_UNIFY_SELF_HEAL_OFF 0
-#define ZR_UNIFY_FG_SELF_HEAL  1
-#define ZR_UNIFY_BG_SELF_HEAL  2
-
-/* Sometimes one should use completely random numbers.. its good :p */
-#define UNIFY_SELF_HEAL_GETDENTS_COUNT 512
-
-#define NS(xl)          (((unify_private_t *)xl->private)->namespace)
-
-/* This is used to allocate memory for local structure */
-#define INIT_LOCAL(fr, loc)                   \
-do {                                          \
-  loc = GF_CALLOC (1, sizeof (unify_local_t), gf_unify_mt_unify_local_t);   \
-  ERR_ABORT (loc);			      \
-  if (!loc) {                                 \
-    STACK_UNWIND (fr, -1, ENOMEM);            \
-    return 0;                                 \
-  }                                           \
-  fr->local = loc;                            \
-  loc->op_ret = -1;                           \
-  loc->op_errno = ENOENT;                     \
-} while (0)
-
-
-
-struct unify_private {
-	/* Update this structure depending on requirement */
-	void *scheduler;               /* THIS SHOULD BE THE FIRST VARIABLE, 
-					  if xlator is using scheduler */
-	struct sched_ops *sched_ops;   /* Scheduler options  */
-	xlator_t *namespace;           /* ptr to namespace xlator */
-	xlator_t **xl_array;
-	gf_boolean_t optimist;
-	int16_t child_count;
-	int16_t num_child_up;
-	uint8_t self_heal;
-	uint8_t is_up;
-	uint64_t inode_generation;
-	gf_lock_t lock;
-};
-typedef struct unify_private unify_private_t;
-
-struct unify_self_heal_struct {
-	uint8_t dir_checksum[NAME_MAX];
-	uint8_t ns_dir_checksum[NAME_MAX];
-	uint8_t file_checksum[NAME_MAX];
-	uint8_t ns_file_checksum[NAME_MAX];
-	off_t *offset_list;
-	int   *count_list;
-	dir_entry_t **entry_list;
-};
-
-
-struct _unify_local_t {
-	int32_t call_count;
-	int32_t op_ret;
-	int32_t op_errno;
-	mode_t mode;
-	off_t offset;
-	dev_t dev;
-	uid_t uid;
-	gid_t gid;
-	int32_t flags;
-	int32_t entry_count;
-	int32_t count;    // dir_entry_t count;
-	fd_t *fd;
-	struct iatt stbuf;
-        struct iatt stpre;
-        struct iatt stpost;
-	struct statvfs statvfs_buf;
-	struct timespec tv[2];
-	char *name;
-	int32_t revalidate;
-
-	ino_t ia_ino;
-	nlink_t ia_nlink;
-  
-	dict_t *dict;
-
-	int16_t *list;
-	int16_t *new_list; /* Used only in case of rename */
-	int16_t index;
-
-	int32_t failed;
-	int32_t return_eio;  /* Used in case of different st-mode 
-				present for a given path */
-
-	uint64_t inode_generation; /* used to store the per directory 
-				    * inode_generation. Got from inode's ctx 
-				    * of directory inodes
-				    */
-
-	struct unify_self_heal_struct *sh_struct;
-	loc_t loc1, loc2;
-
-        struct iatt poststbuf;
-        /* When not used for rename, old*
-         * are used as the attrs for the current
-         * parent directory.
-         */
-        struct iatt oldpreparent;
-        struct iatt oldpostparent;
-        struct iatt newpreparent;
-        struct iatt newpostparent;
-        int32_t wbflags;
-};
-typedef struct _unify_local_t unify_local_t;
-
-int32_t zr_unify_self_heal (call_frame_t *frame,
-			    xlator_t *this,
-			    unify_local_t *local);
-
-#endif /* _UNIFY_H */
diff --git a/xlators/debug/error-gen/src/Makefile.am b/xlators/debug/error-gen/src/Makefile.am
index df9080358..5075c59a8 100644
--- a/xlators/debug/error-gen/src/Makefile.am
+++ b/xlators/debug/error-gen/src/Makefile.am
@@ -2,15 +2,16 @@
 xlator_LTLIBRARIES = error-gen.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-error_gen_la_LDFLAGS = -module -avoidversion
+error_gen_la_LDFLAGS = -module -avoid-version
 
 error_gen_la_SOURCES = error-gen.c
 error_gen_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = error-gen.h error-gen-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/debug/error-gen/src/error-gen-mem-types.h b/xlators/debug/error-gen/src/error-gen-mem-types.h
index b643dc5f7..f02280535 100644
--- a/xlators/debug/error-gen/src/error-gen-mem-types.h
+++ b/xlators/debug/error-gen/src/error-gen-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __ERROR_GEN_MEM_TYPES_H__
 #define __ERROR_GEN_MEM_TYPES_H__
 
diff --git a/xlators/debug/error-gen/src/error-gen.c b/xlators/debug/error-gen/src/error-gen.c
index e44baf5b7..ec0874b35 100644
--- a/xlators/debug/error-gen/src/error-gen.c
+++ b/xlators/debug/error-gen/src/error-gen.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -24,6 +14,7 @@
 
 #include "xlator.h"
 #include "error-gen.h"
+#include "statedump.h"
 
 sys_error_t error_no_list[] = {
         [GF_FOP_LOOKUP]            = { .error_no_count = 4,
@@ -91,9 +82,10 @@ sys_error_t error_no_list[] = {
         [GF_FOP_READ]              = { .error_no_count = 5,
                                     .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
                                                  ENAMETOOLONG}},
-        [GF_FOP_WRITE]             = { .error_no_count = 5,
+        [GF_FOP_WRITE]             = { .error_no_count = 7,
                                     .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
-                                                 ENAMETOOLONG}},
+                                                 ENAMETOOLONG,ENOSPC,
+						 GF_ERROR_SHORT_WRITE}},
         [GF_FOP_STATFS]            = {.error_no_count = 10,
                                    .error_no = {EACCES,EBADF,EFAULT,EINTR,
                                                 EIO,ENAMETOOLONG,ENOENT,
@@ -246,6 +238,8 @@ conv_errno_to_int (char **error_no)
                 return EINTR;
         else if (!strcmp ((*error_no), "EFBIG"))
                 return EFBIG;
+	else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
+		return GF_ERROR_SHORT_WRITE;
         else
                 return EAGAIN;
 }
@@ -387,17 +381,17 @@ error_gen (xlator_t *this, int op_no)
 int
 error_gen_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno, inode_t *inode,
-		      struct iatt *buf, dict_t *dict, struct iatt *postparent)
+		      struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
 	STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode,
-                             buf, dict, postparent);
-	return 0;
+                             buf, xdata, postparent);
+        return 0;
 }
 
 
 int
 error_gen_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  dict_t *xattr_req)
+                  dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -412,36 +406,28 @@ error_gen_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                              NULL);
-		return 0;
+                                     NULL);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_lookup_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->lookup,
-		    loc, xattr_req);
-	return 0;
-}
-
-
-int
-error_gen_forget (xlator_t *this, inode_t *inode)
-{
-	return 0;
+		    loc, xdata);
+        return 0;
 }
 
 
 int
 error_gen_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, struct iatt *buf)
+		    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
-
-	return 0;
+	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
 int
-error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -455,32 +441,31 @@ error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_stat_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->stat,
-		    loc);
-	return 0;
+		    loc, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop)
+                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop);
-
-	return 0;
+	STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid)
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -494,21 +479,21 @@ error_gen_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_setattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->setattr,
-		    loc, stbuf, valid);
-	return 0;
+		    loc, stbuf, valid, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid)
+                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -522,32 +507,32 @@ error_gen_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_setattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsetattr,
-		    fd, stbuf, valid);
-	return 0;
+		    fd, stbuf, valid, xdata);
+        return 0;
 }
 
 
 int
 error_gen_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			int32_t op_ret, int32_t op_errno,
-			struct iatt *prebuf, struct iatt *postbuf)
+			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
-	return 0;
+                             prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset)
+                    off_t offset, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -562,32 +547,32 @@ error_gen_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (truncate, frame, -1, op_errno,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_truncate_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->truncate,
-		    loc, offset);
-	return 0;
+		    loc, offset, xdata);
+        return 0;
 }
 
 
 int
 error_gen_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf)
+                         struct iatt *postbuf, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
-	return 0;
+                             prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     off_t offset)
+		     off_t offset, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp =NULL;
@@ -602,31 +587,30 @@ error_gen_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_ftruncate_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd, offset);
-	return 0;
+		    fd, offset, xdata);
+        return 0;
 }
 
 
 int
 error_gen_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno)
+		      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t mask)
+		  int32_t mask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -640,31 +624,31 @@ error_gen_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (access, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (access, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_access_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->access,
-		    loc, mask);
-	return 0;
+		    loc, mask, xdata);
+        return 0;
 }
 
 
 int
 error_gen_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			int32_t op_ret, int32_t op_errno,
-			const char *path, struct iatt *sbuf)
+			const char *path, struct iatt *sbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, sbuf);
-	return 0;
+	STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, sbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    size_t size)
+                    size_t size, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -678,15 +662,15 @@ error_gen_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_readlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readlink,
-		    loc, size);
-	return 0;
+		    loc, size, xdata);
+        return 0;
 }
 
 
@@ -694,18 +678,18 @@ int
 error_gen_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno,
                              inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		 mode_t mode, dev_t rdev, dict_t *params)
+		 mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -720,15 +704,15 @@ error_gen_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_mknod_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev, params);
-	return 0;
+		    loc, mode, rdev, umask, xdata);
+        return 0;
 }
 
 
@@ -736,17 +720,17 @@ int
 error_gen_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno,
                              inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 int
 error_gen_mkdir (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, mode_t mode, dict_t *params)
+		 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -761,31 +745,32 @@ error_gen_mkdir (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+                return 0;
 	}
 
 	STACK_WIND (frame, error_gen_mkdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->mkdir,
-		    loc, mode, params);
-	return 0;
+		    loc, mode, umask, xdata);
+        return 0;
 }
 
 
 int
 error_gen_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
-error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                  dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -799,31 +784,34 @@ error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL,
+                                     xdata);
+                return 0;
 	}
 
 	STACK_WIND (frame, error_gen_unlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->unlink,
-		    loc);
-	return 0;
+		    loc, xflag, xdata);
+        return 0;
 }
 
 
 int
 error_gen_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
-error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                 dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -837,15 +825,15 @@ error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_rmdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->rmdir,
-		    loc, flags);
-	return 0;
+		    loc, flags, xdata);
+        return 0;
 }
 
 
@@ -853,17 +841,17 @@ int
 error_gen_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		       int32_t op_ret, int32_t op_errno, inode_t *inode,
                        struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent)
+                       struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-		   loc_t *loc, dict_t *params)
+		   loc_t *loc, mode_t umask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -878,15 +866,15 @@ error_gen_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL); /* pre & post parent attr */
+                                     NULL, NULL, NULL); /* pre & post parent attr */
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_symlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc, params);
-	return 0;
+		    linkpath, loc, umask, xdata);
+        return 0;
 }
 
 
@@ -894,18 +882,19 @@ int
 error_gen_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno, struct iatt *buf,
                       struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent)
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
                              preoldparent, postoldparent,
-                             prenewparent, postnewparent);
-	return 0;
+                             prenewparent, postnewparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_rename (call_frame_t *frame, xlator_t *this,
-		  loc_t *oldloc, loc_t *newloc)
+		  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -920,15 +909,15 @@ error_gen_rename (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                                     NULL, NULL, NULL, NULL); /* pre & post parent attr */
+                                     NULL, NULL, NULL, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_rename_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->rename,
-		    oldloc, newloc);
-	return 0;
+		    oldloc, newloc, xdata);
+        return 0;
 }
 
 
@@ -936,17 +925,17 @@ int
 error_gen_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		    int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_link (call_frame_t *frame, xlator_t *this,
-		loc_t *oldloc, loc_t *newloc)
+		loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -961,15 +950,15 @@ error_gen_link (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL); /* pre & post parent attr */
+                                     NULL, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_link_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->link,
-		    oldloc, newloc);
-	return 0;
+		    oldloc, newloc, xdata);
+        return 0;
 }
 
 
@@ -977,17 +966,18 @@ int
 error_gen_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno,
 		      fd_t *fd, inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+		  int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                  dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -1002,30 +992,30 @@ error_gen_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL); /* pre & post attr */
+                                     NULL, NULL, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_create_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, fd, params);
-	return 0;
+		    loc, flags, mode, umask, fd, xdata);
+        return 0;
 }
 
 
 int
 error_gen_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, fd_t *fd)
+		    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
-	return 0;
+	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
 }
 
 
 int
 error_gen_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		int32_t flags, fd_t *fd, int32_t wbflags)
+		int32_t flags, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1039,15 +1029,15 @@ error_gen_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_open_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd, wbflags);
-	return 0;
+		    loc, flags, fd, xdata);
+        return 0;
 }
 
 
@@ -1055,17 +1045,17 @@ int
 error_gen_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno,
 		     struct iovec *vector, int32_t count,
-		     struct iatt *stbuf, struct iobref *iobref)
+		     struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref);
-	return 0;
+                             vector, count, stbuf, iobref, xdata);
+        return 0;
 }
 
 
 int
 error_gen_readv (call_frame_t *frame, xlator_t *this,
-		 fd_t *fd, size_t size, off_t offset, uint32_t flags)
+		 fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1080,33 +1070,33 @@ error_gen_readv (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 
 	STACK_WIND (frame, error_gen_readv_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readv,
-		    fd, size, offset, flags);
-	return 0;
+		    fd, size, offset, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno,
-                      struct iatt *prebuf, struct iatt *postbuf)
+                      struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 		  struct iovec *vector, int32_t count,
-		  off_t off, uint32_t flags, struct iobref *iobref)
+		  off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1118,31 +1108,47 @@ error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (enable)
                 op_errno = error_gen (this, GF_FOP_WRITE);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+	if (op_errno == GF_ERROR_SHORT_WRITE) {
+		struct iovec *shortvec;
+
+		/*
+		 * A short write error returns some value less than what was
+		 * requested from a write. To simulate this, replace the vector
+		 * with one half the size;
+		 */
+		shortvec = iov_dup(vector, 1);
+		shortvec->iov_len /= 2;
+
+		STACK_WIND(frame, error_gen_writev_cbk, FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev, fd, shortvec, count,
+			   off, flags, iobref, xdata);
+		GF_FREE(shortvec);
 		return 0;
+	} else if (op_errno) {
+		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
+		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, xdata);
+        	return 0;
 	}
 
 	STACK_WIND (frame, error_gen_writev_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->writev,
-		    fd, vector, count, off, flags, iobref);
-	return 0;
+		    fd, vector, count, off, flags, iobref, xdata);
+        return 0;
 }
 
 
 int
 error_gen_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno)
+		     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
-error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1156,15 +1162,15 @@ error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (flush, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_flush_cbk,
                     FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->flush,
-		    fd);
-	return 0;
+		    fd, xdata);
+        return 0;
 }
 
 
@@ -1172,15 +1178,15 @@ int
 error_gen_fsync_cbk (call_frame_t *frame, void *cookie,
 		     xlator_t *this, int32_t op_ret,
 		     int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
+	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
-error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1194,29 +1200,29 @@ error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fsync_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsync,
-		    fd, flags);
-	return 0;
+		    fd, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
-	return 0;
+	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
 
 int
-error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1230,29 +1236,29 @@ error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fstat_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fstat,
-		    fd);
-	return 0;
+		    fd, xdata);
+        return 0;
 }
 
 
 int
 error_gen_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, fd_t *fd)
+		       int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
-	return 0;
+	STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        return 0;
 }
 
 
 int
-error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1266,29 +1272,29 @@ error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_opendir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->opendir,
-		    loc, fd);
-	return 0;
+		    loc, fd, xdata);
+        return 0;
 }
 
 int
 error_gen_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno)
+			int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    int32_t flags)
+                    int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1302,30 +1308,29 @@ error_gen_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fsyncdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsyncdir,
-		    fd, flags);
-	return 0;
+		    fd, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+		      int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
-
-	return 0;
+	STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
 
 int
-error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1339,31 +1344,30 @@ error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_statfs_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->statfs,
-		    loc);
-	return 0;
+		    loc, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno)
+			int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *dict, int32_t flags)
+		    dict_t *dict, int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1377,30 +1381,30 @@ error_gen_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_setxattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->setxattr,
-		    loc, dict, flags);
-	return 0;
+		    loc, dict, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *dict)
+			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
-	return 0;
+	STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    const char *name)
+		    const char *name, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1414,30 +1418,29 @@ error_gen_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_getxattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->getxattr,
-		    loc, name);
-	return 0;
+		    loc, name, xdata);
+        return 0;
 }
 
 int
 error_gen_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno)
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     dict_t *dict, int32_t flags)
+                     dict_t *dict, int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1451,30 +1454,30 @@ error_gen_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fsetxattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsetxattr,
-		    fd, dict, flags);
-	return 0;
+		    fd, dict, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *dict)
+                         int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
-	return 0;
+	STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     const char *name)
+                     const char *name, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1488,31 +1491,30 @@ error_gen_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fgetxattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fgetxattr,
-		    fd, name);
-	return 0;
+		    fd, name, xdata);
+        return 0;
 }
 
 
 int
 error_gen_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
-
-	return 0;
+	STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		   gf_xattrop_flags_t flags, dict_t *dict)
+		   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1526,31 +1528,30 @@ error_gen_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_xattrop_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->xattrop,
-		    loc, flags, dict);
-	return 0;
+		    loc, flags, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *dict)
+			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
-
-	return 0;
+	STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    gf_xattrop_flags_t flags, dict_t *dict)
+		    gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1564,31 +1565,30 @@ error_gen_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fxattrop_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fxattrop,
-		    fd, flags, dict);
-	return 0;
+		    fd, flags, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno)
+			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       const char *name)
+		       const char *name, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1602,30 +1602,29 @@ error_gen_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_removexattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->removexattr,
-		    loc, name);
-	return 0;
+		    loc, name, xdata);
+        return 0;
 }
 
 int
 error_gen_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno)
+			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        const char *name)
+                        const char *name, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1639,30 +1638,30 @@ error_gen_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fremovexattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fremovexattr,
-		    fd, name);
-	return 0;
+		    fd, name, xdata);
+        return 0;
 }
 
 
 int
 error_gen_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+		  int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
-	return 0;
+	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
+        return 0;
 }
 
 
 int
 error_gen_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-	      struct gf_flock *lock)
+	      struct gf_flock *lock, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1676,32 +1675,31 @@ error_gen_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_lk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->lk,
-		    fd, cmd, lock);
-	return 0;
+		    fd, cmd, lock, xdata);
+        return 0;
 }
 
 
 int
-error_gen_inodelk_cbk (call_frame_t *frame, void *cookie,
-		       xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_inodelk (call_frame_t *frame, xlator_t *this,
 		   const char *volume, loc_t *loc, int32_t cmd,
-                   struct gf_flock *lock)
+                   struct gf_flock *lock, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1715,32 +1713,31 @@ error_gen_inodelk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_inodelk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->inodelk,
-		    volume, loc, cmd, lock);
-	return 0;
+		    volume, loc, cmd, lock, xdata);
+        return 0;
 }
 
 
 int
-error_gen_finodelk_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_finodelk (call_frame_t *frame, xlator_t *this,
 		    const char *volume, fd_t *fd, int32_t cmd,
-                    struct gf_flock *lock)
+                    struct gf_flock *lock, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1754,32 +1751,31 @@ error_gen_finodelk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_finodelk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->finodelk,
-		    volume, fd, cmd, lock);
-	return 0;
+		    volume, fd, cmd, lock, xdata);
+        return 0;
 }
 
 
 int
-error_gen_entrylk_cbk (call_frame_t *frame, void *cookie,
-		       xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_entrylk (call_frame_t *frame, xlator_t *this,
 		   const char *volume, loc_t *loc, const char *basename,
-		   entrylk_cmd cmd, entrylk_type type)
+		   entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1793,32 +1789,31 @@ error_gen_entrylk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_entrylk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->entrylk,
-		    volume, loc, basename, cmd, type);
-	return 0;
+		    volume, loc, basename, cmd, type, xdata);
+        return 0;
 }
 
 
 int
-error_gen_fentrylk_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fentrylk (call_frame_t *frame, xlator_t *this,
 		    const char *volume, fd_t *fd, const char *basename,
-		    entrylk_cmd cmd, entrylk_type type)
+		    entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -1832,15 +1827,15 @@ error_gen_fentrylk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno, xdata);
+                return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fentrylk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fentrylk,
-		    volume, fd, basename, cmd, type);
-	return 0;
+		    volume, fd, basename, cmd, type, xdata);
+        return 0;
 }
 
 
@@ -1852,8 +1847,7 @@ error_gen_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		       int32_t op_ret, int32_t op_errno, char *spec_data)
 {
 	STACK_UNWIND_STRICT (getspec, frame, op_ret, op_errno, spec_data);
-
-	return 0;
+        return 0;
 }
 
 
@@ -1887,16 +1881,17 @@ error_gen_getspec (call_frame_t *frame, xlator_t *this, const char *key,
 
 int
 error_gen_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+		       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries);
+	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
 	return 0;
 }
 
 
 int
 error_gen_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		   size_t size, off_t off)
+		   size_t size, off_t off, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1910,23 +1905,24 @@ error_gen_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL);
+		STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL, xdata);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_readdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readdir,
-		    fd, size, off);
+		    fd, size, off, xdata);
 	return 0;
 }
 
 
 int
 error_gen_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                        int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 	return 0;
 }
 
@@ -1947,7 +1943,7 @@ error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL);
+		STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL, NULL);
 		return 0;
 	}
 
@@ -1958,18 +1954,83 @@ error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 	return 0;
 }
 
-
-int
-error_gen_closedir (xlator_t *this, fd_t *fd)
+static void
+error_gen_set_failure (eg_t *pvt, int percent)
 {
-	return 0;
+        GF_ASSERT (pvt);
+
+        if (percent)
+                pvt->failure_iter_no = 100/percent;
+        else
+                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
 }
 
+static void
+error_gen_parse_fill_fops (eg_t *pvt, char *enable_fops)
+{
+        char            *op_no_str = NULL;
+        int              op_no = -1;
+        int              i = 0;
+        xlator_t        *this = THIS;
+        char            *saveptr = NULL;
 
-int
-error_gen_close (xlator_t *this, fd_t *fd)
+        GF_ASSERT (pvt);
+        GF_ASSERT (this);
+
+        for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                pvt->enable[i] = 0;
+
+        if (!enable_fops) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "All fops are enabled.");
+                for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                        pvt->enable[i] = 1;
+        } else {
+                op_no_str = strtok_r (enable_fops, ",", &saveptr);
+                while (op_no_str) {
+                        op_no = get_fop_int (&op_no_str);
+                        if (op_no == -1) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Wrong option value %s", op_no_str);
+                        } else
+                                pvt->enable[op_no] = 1;
+
+                        op_no_str = strtok_r (NULL, ",", &saveptr);
+                }
+        }
+}
+
+int32_t
+error_gen_priv_dump (xlator_t *this)
 {
-	return 0;
+        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
+        int             ret = -1;
+        eg_t            *conf = NULL;
+
+        if (!this)
+            goto out;
+
+        conf = this->private;
+        if (!conf)
+            goto out;
+
+        ret = TRY_LOCK(&conf->lock);
+        if (ret != 0) {
+                return ret;
+        }
+
+        gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
+        gf_proc_dump_build_key(key_prefix,"xlator.debug.error-gen","%s.priv",
+                               this->name);
+
+        gf_proc_dump_write("op_count", "%d", conf->op_count);
+        gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
+        gf_proc_dump_write("error_no", "%s", conf->error_no);
+        gf_proc_dump_write("random_failure", "%d", conf->random_failure);
+
+        UNLOCK(&conf->lock);
+out:
+        return ret;
 }
 
 int32_t
@@ -1992,18 +2053,43 @@ mem_acct_init (xlator_t *this)
 }
 
 int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        eg_t            *pvt = NULL;
+        int32_t          ret = 0;
+        char            *error_enable_fops = NULL;
+        int32_t          failure_percent_int = 0;
+
+        if (!this || !this->private)
+                goto out;
+
+        pvt = this->private;
+
+        GF_OPTION_RECONF ("error-no", pvt->error_no, options, str, out);
+
+        GF_OPTION_RECONF ("failure", failure_percent_int, options, int32,
+                          out);
+
+        GF_OPTION_RECONF ("enable", error_enable_fops, options, str, out);
+
+        GF_OPTION_RECONF ("random-failure", pvt->random_failure, options,
+                          bool, out);
+
+        error_gen_parse_fill_fops (pvt, error_enable_fops);
+        error_gen_set_failure (pvt, failure_percent_int);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "reconfigure returning %d", ret);
+        return ret;
+}
+
+int
 init (xlator_t *this)
 {
         eg_t            *pvt = NULL;
-        data_t          *error_no = NULL;
-        data_t          *failure_percent = NULL;
-        data_t          *enable = NULL;
-        gf_boolean_t    random_failure = _gf_false;
         int32_t          ret = 0;
         char            *error_enable_fops = NULL;
-        char            *op_no_str = NULL;
-        int              op_no = -1;
-        int              i = 0;
         int32_t          failure_percent_int = 0;
 
         if (!this->children || this->children->next) {
@@ -2018,79 +2104,34 @@ init (xlator_t *this)
                         "dangling volume. check volfile ");
         }
 
-        error_no = dict_get (this->options, "error-no");
-        failure_percent = dict_get (this->options, "failure");
-        enable = dict_get (this->options, "enable");
-
         pvt = GF_CALLOC (1, sizeof (eg_t), gf_error_gen_mt_eg_t);
 
         if (!pvt) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "out of memory.");
                 ret = -1;
                 goto out;
         }
 
         LOCK_INIT (&pvt->lock);
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                pvt->enable[i] = 0;
-        if (!error_no) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "error-no not specified.");
-        } else {
-                pvt->error_no = data_to_str (error_no);
-        }
+        GF_OPTION_INIT ("error-no", pvt->error_no, str, out);
 
-        if (!failure_percent) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failure percent not specified.");
-                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
-        } else {
-                failure_percent_int = data_to_int32 (failure_percent);
-                if (failure_percent_int)
-                        pvt->failure_iter_no = 100/failure_percent_int;
-                else
-                        pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
-        }
+        GF_OPTION_INIT ("failure", failure_percent_int, int32, out);
+
+        GF_OPTION_INIT ("enable", error_enable_fops, str, out);
+
+        GF_OPTION_INIT ("random-failure", pvt->random_failure, bool, out);
 
-        if (!enable) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "All fops are enabled.");
-                for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                        pvt->enable[i] = 1;
-        } else {
-                error_enable_fops = data_to_str (enable);
-                op_no_str = error_enable_fops;
-                while ((*error_enable_fops) != '\0') {
-                        error_enable_fops++;
-                        if (((*error_enable_fops) == ',') ||
-                            ((*error_enable_fops) == '\0')) {
-                                if ((*error_enable_fops) != '\0') {
-                                        (*error_enable_fops) = '\0';
-                                        error_enable_fops++;
-                                }
-                                op_no = get_fop_int (&op_no_str);
-                                if (op_no == -1) {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "Wrong option value %s",
-                                                op_no_str);
-                                } else
-                                        pvt->enable[op_no] = 1;
-                                op_no_str = error_enable_fops;
-                        }
-                }
-        }
 
-        random_failure = dict_get_str_boolean (this->options, "random-failure",
-                                               _gf_false);
-        pvt->random_failure = random_failure;
+        error_gen_parse_fill_fops (pvt, error_enable_fops);
+        error_gen_set_failure (pvt, failure_percent_int);
 
         this->private = pvt;
 
         /* Give some seed value here */
         srand (time(NULL));
 out:
+        if (ret)
+                GF_FREE (pvt);
         return ret;
 }
 
@@ -2112,6 +2153,12 @@ fini (xlator_t *this)
         return;
 }
 
+struct xlator_dumpops dumpops = {
+        .priv = error_gen_priv_dump,
+};
+
+struct xlator_fops cbks;
+
 struct xlator_fops fops = {
 	.lookup      = error_gen_lookup,
 	.stat        = error_gen_stat,
@@ -2157,24 +2204,29 @@ struct xlator_fops fops = {
 	.getspec     = error_gen_getspec,
 };
 
-struct xlator_cbks cbks = {
-	.release = error_gen_close,
-	.releasedir = error_gen_closedir,
-};
-
 struct volume_options options[] = {
         { .key  = {"failure"},
-          .type = GF_OPTION_TYPE_INT },
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Percentage failure of operations when enabled.",
+        },
+
         { .key  = {"error-no"},
           .value = {"ENOENT","ENOTDIR","ENAMETOOLONG","EACCES","EBADF",
                     "EFAULT","ENOMEM","EINVAL","EIO","EEXIST","ENOSPC",
                     "EPERM","EROFS","EBUSY","EISDIR","ENOTEMPTY","EMLINK"
                     "ENODEV","EXDEV","EMFILE","ENFILE","ENOSYS","EINTR",
-                    "EFBIG","EAGAIN"},
-          .type = GF_OPTION_TYPE_STR },
+                    "EFBIG","EAGAIN","GF_ERROR_SHORT_WRITE"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
         { .key = {"random-failure"},
-          .type = GF_OPTION_TYPE_BOOL},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+
         { .key  = {"enable"},
-          .type = GF_OPTION_TYPE_STR },
+          .type = GF_OPTION_TYPE_STR,
+        },
+
         { .key  = {NULL} }
 };
diff --git a/xlators/debug/error-gen/src/error-gen.h b/xlators/debug/error-gen/src/error-gen.h
index bb3adb2ab..d92c23062 100644
--- a/xlators/debug/error-gen/src/error-gen.h
+++ b/xlators/debug/error-gen/src/error-gen.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _ERROR_GEN_H
 #define _ERROR_GEN_H
 
@@ -29,6 +19,18 @@
 
 #define GF_FAILURE_DEFAULT 10
 
+/*
+ * Pseudo-errors refer to errors beyond the scope of traditional <-1, op_errno>
+ * returns. This facilitates the ability to return unexpected, but not -1 values
+ * and/or to inject operations that lead to implicit error conditions. The range
+ * for pseudo errors resides at a high value to avoid conflicts with the errno
+ * range.
+ */
+enum GF_PSEUDO_ERRORS {
+	GF_ERROR_SHORT_WRITE = 1000,	/* short writev return value */
+	GF_ERROR_MAX
+};
+
 typedef struct {
         int enable[GF_FOP_MAXVALUE];
         int op_count;
diff --git a/xlators/debug/io-stats/src/Makefile.am b/xlators/debug/io-stats/src/Makefile.am
index b894e79c3..332d79015 100644
--- a/xlators/debug/io-stats/src/Makefile.am
+++ b/xlators/debug/io-stats/src/Makefile.am
@@ -2,14 +2,15 @@
 xlator_LTLIBRARIES = io-stats.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-io_stats_la_LDFLAGS = -module -avoidversion
+io_stats_la_LDFLAGS = -module -avoid-version
 
 io_stats_la_SOURCES = io-stats.c
 io_stats_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = io-stats-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
diff --git a/xlators/debug/io-stats/src/io-stats-mem-types.h b/xlators/debug/io-stats/src/io-stats-mem-types.h
index 2063f6d6a..c30dfb17e 100644
--- a/xlators/debug/io-stats/src/io-stats-mem-types.h
+++ b/xlators/debug/io-stats/src/io-stats-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __IO_STATS_MEM_TYPES_H__
 #define __IO_STATS_MEM_TYPES_H__
 
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 888c36dfb..7fb697ae4 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -58,7 +48,7 @@ typedef enum {
         IOS_STATS_TYPE_READDIRP,
         IOS_STATS_TYPE_READ_THROUGHPUT,
         IOS_STATS_TYPE_WRITE_THROUGHPUT,
-        IOS_STATS_TYPE_MAX,
+        IOS_STATS_TYPE_MAX
 }ios_stats_type_t;
 
 typedef enum {
@@ -293,43 +283,43 @@ is_fop_latency_started (call_frame_t *frame)
                 }                                                               \
                 UNLOCK (&iosstat->lock);                                        \
                 ios_stat_add_to_list (&conf->list[type],                        \
-				     value, iosstat);                           \
+                                     value, iosstat);                           \
                                                                                 \
         } while (0)
 
 
 #define BUMP_THROUGHPUT(iosstat, type)						\
         do {									\
-	        struct ios_conf         *conf = NULL;				\
-		double                   elapsed;				\
-		struct timeval          *begin, *end;				\
-		double                   throughput;				\
+                struct ios_conf         *conf = NULL;				\
+                double                   elapsed;				\
+                struct timeval          *begin, *end;				\
+                double                   throughput;				\
                int                      flag = 0;                              \
-										\
-		begin = &frame->begin;						\
-		end   = &frame->end;						\
-										\
-		elapsed = (end->tv_sec - begin->tv_sec) * 1e6			\
-			+ (end->tv_usec - begin->tv_usec);			\
-		throughput = op_ret / elapsed;					\
-										\
-		conf = this->private;						\
-		LOCK(&iosstat->lock);						\
-		{								\
-			if (iosstat->thru_counters[type].throughput             \
+                                                                                \
+                begin = &frame->begin;						\
+                end   = &frame->end;						\
+                                                                                \
+                elapsed = (end->tv_sec - begin->tv_sec) * 1e6			\
+                        + (end->tv_usec - begin->tv_usec);			\
+                throughput = op_ret / elapsed;					\
+                                                                                \
+                conf = this->private;						\
+                LOCK(&iosstat->lock);						\
+                {								\
+                        if (iosstat->thru_counters[type].throughput             \
                                 <= throughput) {                                \
-				iosstat->thru_counters[type].throughput =       \
+                                iosstat->thru_counters[type].throughput =       \
                                                                 throughput;     \
-				gettimeofday (&iosstat->                        \
+                                gettimeofday (&iosstat->                        \
                                              thru_counters[type].time, NULL);   \
                                flag = 1;                                       \
                         }							\
                 }								\
-        	UNLOCK (&iosstat->lock);					\
+                UNLOCK (&iosstat->lock);					\
                if (flag)                                                       \
                        ios_stat_add_to_list (&conf->thru_list[type],           \
                                                throughput, iosstat);           \
-	} while (0)
+        } while (0)
 
 int
 ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
@@ -484,12 +474,12 @@ ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
                         new = GF_CALLOC (1, sizeof (*new),
                                         gf_io_stats_mt_ios_stat_list);
                         new->iosstat = iosstat;
-                        new->value = value; 
+                        new->value = value;
                         ios_stat_ref (iosstat);
-                        list_add_tail (&new->list, &tmp->list);  
+                        list_add_tail (&new->list, &tmp->list);
                         stat = last->iosstat;
                         last->iosstat = NULL;
-                        ios_stat_unref (stat); 
+                        ios_stat_unref (stat);
                         list_del (&last->list);
                         GF_FREE (last);
                         if (reposition == MAX_LIST_MEMBERS)
@@ -511,7 +501,7 @@ ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
                         list_head->members++;
                         if (list_head->min_cnt > value)
                                 list_head->min_cnt = value;
-                } 
+                }
         }
 out:
         UNLOCK (&list_head->lock);
@@ -568,19 +558,16 @@ ios_dump_throughput_stats (struct ios_stat_head *list_head, xlator_t *this,
                             FILE* logfp, ios_stats_type_t type)
 {
         struct ios_stat_list *entry = NULL;
-        struct timeval        time = {0, };
-        struct tm             *tm = NULL;
+        struct timeval        time  = {0, };
         char                  timestr[256] = {0, };
 
         LOCK (&list_head->lock);
         {
                 list_for_each_entry (entry, &list_head->iosstats->list, list) {
-                        time = entry->iosstat->thru_counters[type].time;
-                        tm    = localtime (&time.tv_sec);
-                        if (!tm)
-                                continue;
-                        strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                        snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                        gf_time_fmt (timestr, sizeof timestr,
+                                     entry->iosstat->thru_counters[type].time.tv_sec,
+                                     gf_timefmt_FT);
+                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
                           ".%"GF_PRI_SUSECONDS, time.tv_usec);
 
                         ios_log (this, logfp, "%s \t %-10.2f  \t  %s",
@@ -600,7 +587,6 @@ io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
         int                   index = 0;
         struct ios_stat_head *list_head = NULL;
         struct ios_conf      *conf = NULL;
-        struct tm            *tm = NULL;
         char                  timestr[256] = {0, };
         char                  str_header[128] = {0};
         char                  str_read[128] = {0};
@@ -694,9 +680,10 @@ io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
         if (interval == -1) {
                 LOCK (&conf->lock);
                 {
-                        tm = localtime (&conf->cumulative.max_openfd_time.tv_sec);
-                        strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                        snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                        gf_time_fmt (timestr, sizeof timestr,
+                                     conf->cumulative.max_openfd_time.tv_sec,
+                                     gf_timefmt_FT);
+                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
                                   ".%"GF_PRI_SUSECONDS,
                                   conf->cumulative.max_openfd_time.tv_usec);
                         ios_log (this, logfp, "Current open fd's: %"PRId64
@@ -735,13 +722,13 @@ io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
                 ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
                          "\tFILE NAME");
                 list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-		ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
+                ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
 
                 ios_log (this, logfp, "\n======Write Throughput File Stats======");
                 ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
                          "\tFILE NAME");
                 list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
-		ios_dump_throughput_stats (list_head, this, logfp, IOS_STATS_THRU_WRITE);
+                ios_dump_throughput_stats (list_head, this, logfp, IOS_STATS_THRU_WRITE);
         }
         return 0;
 }
@@ -1080,36 +1067,45 @@ io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
         struct ios_stat_list    *entry = NULL;
         int                      ret = -1;
         ios_stats_thru_t         index = IOS_STATS_THRU_MAX;
-        struct tm               *tm = NULL;
         char                     timestr[256] = {0, };
+        char                    *dict_timestr = NULL;
 
         conf = this->private;
 
         switch (flags) {
-                case IOS_STATS_TYPE_OPEN: 
+                case IOS_STATS_TYPE_OPEN:
                         list_head = &conf->list[IOS_STATS_TYPE_OPEN];
                         LOCK (&conf->lock);
                         {
                                 ret = dict_set_uint64 (resp, "current-open",
                                                      conf->cumulative.nr_opens);
                                 if (ret)
-                                        goto out;
+                                        goto unlock;
                                 ret = dict_set_uint64 (resp, "max-open",
                                                        conf->cumulative.max_nr_opens);
 
-                                tm = localtime (&conf->cumulative.max_openfd_time.tv_sec);
-                                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                                          ".%"GF_PRI_SUSECONDS,
-                                          conf->cumulative.max_openfd_time.tv_usec);
-
-                                ret = dict_set_str (resp, "max-openfd-time",
-                                                    timestr);
+                                gf_time_fmt (timestr, sizeof timestr,
+                                             conf->cumulative.max_openfd_time.tv_sec,
+                                             gf_timefmt_FT);
+                                if (conf->cumulative.max_openfd_time.tv_sec)
+                                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                                                  ".%"GF_PRI_SUSECONDS,
+                                                  conf->cumulative.max_openfd_time.tv_usec);
+
+                                dict_timestr = gf_strdup (timestr);
+                                if (!dict_timestr)
+                                        goto unlock;
+                                ret = dict_set_dynstr (resp, "max-openfd-time",
+                                                       dict_timestr);
                                 if (ret)
-                                        goto out;
+                                        goto unlock;
                         }
+        unlock:
                         UNLOCK (&conf->lock);
-
+                        /* Do not proceed if we came here because of some error
+                         * during the dict operation */
+                        if (ret)
+                                goto out;
                         break;
                 case IOS_STATS_TYPE_READ:
                         list_head = &conf->list[IOS_STATS_TYPE_READ];
@@ -1125,7 +1121,7 @@ io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
                         break;
                 case IOS_STATS_TYPE_READ_THROUGHPUT:
                         list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-                        index = IOS_STATS_THRU_READ; 
+                        index = IOS_STATS_THRU_READ;
                         break;
                 case IOS_STATS_TYPE_WRITE_THROUGHPUT:
                         list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
@@ -1146,39 +1142,44 @@ io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
                         snprintf (key, 256, "%s-%d", "filename", cnt);
                         ret = dict_set_str (resp, key, entry->iosstat->filename);
                         if (ret)
-                                goto out;
+                                goto unlock_list_head;
                          snprintf (key, 256, "%s-%d", "value",cnt);
                          ret = dict_set_uint64 (resp, key, entry->value);
                          if (ret)
-                                 goto out;
+                                 goto unlock_list_head;
                          if (index != IOS_STATS_THRU_MAX) {
                                  snprintf (key, 256, "%s-%d", "time-sec", cnt);
-                                 ret = dict_set_int32 (resp, key, 
+                                 ret = dict_set_int32 (resp, key,
                                          entry->iosstat->thru_counters[index].time.tv_sec);
                                  if (ret)
-                                         goto out;
+                                         goto unlock_list_head;
                                  snprintf (key, 256, "%s-%d", "time-usec", cnt);
-                                 ret = dict_set_int32 (resp, key, 
+                                 ret = dict_set_int32 (resp, key,
                                          entry->iosstat->thru_counters[index].time.tv_usec);
                                  if (ret)
-                                         goto out;
+                                         goto unlock_list_head;
                          }
                          if (cnt == list_cnt)
                                  break;
 
                 }
         }
+unlock_list_head:
         UNLOCK (&list_head->lock);
-
+        /* ret is !=0 if some dict operation in the above critical region
+         * failed. */
+        if (ret)
+                goto out;
         ret = dict_set_int32 (resp, "members", cnt);
  out:
         return ret;
 }
+
 int
 io_stats_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, fd_t *fd,
                      inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         struct ios_fd *iosfd = NULL;
         char          *path = NULL;
@@ -1231,21 +1232,21 @@ io_stats_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unwind:
         UPDATE_PROFILE_STATS (frame, CREATE);
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd)
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         struct ios_fd *iosfd = NULL;
         char          *path = NULL;
         struct   ios_stat *iosstat = NULL;
         struct ios_conf   *conf = NULL;
 
-	conf = this->private;
+        conf = this->private;
         path = frame->local;
         frame->local = NULL;
 
@@ -1269,6 +1270,16 @@ io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         ios_fd_ctx_set (fd, this, iosfd);
 
         ios_inode_ctx_get (fd->inode, this, &iosstat);
+        if (!iosstat) {
+                iosstat = GF_CALLOC (1, sizeof (*iosstat),
+                                     gf_io_stats_mt_ios_stat);
+                if (iosstat) {
+                        iosstat->filename = gf_strdup (path);
+                        uuid_copy (iosstat->gfid, fd->inode->gfid);
+                        LOCK_INIT (&iosstat->lock);
+                        ios_inode_ctx_set (fd->inode, this, iosstat);
+                }
+        }
 
         LOCK (&conf->lock);
         {
@@ -1286,7 +1297,7 @@ io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unwind:
         UPDATE_PROFILE_STATS (frame, OPEN);
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
         return 0;
 
 }
@@ -1294,10 +1305,10 @@ unwind:
 
 int
 io_stats_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, STAT);
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
@@ -1306,7 +1317,7 @@ int
 io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     struct iovec *vector, int32_t count,
-                    struct iatt *buf, struct iobref *iobref)
+                    struct iatt *buf, struct iobref *iobref, dict_t *xdata)
 {
         int              len = 0;
         fd_t            *fd = NULL;
@@ -1330,7 +1341,7 @@ io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, buf, iobref);
+                             vector, count, buf, iobref, xdata);
         return 0;
 
 }
@@ -1339,7 +1350,7 @@ io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf)
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         struct ios_stat *iosstat = NULL;
         inode_t         *inode   = NULL;
@@ -1351,13 +1362,13 @@ io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 ios_inode_ctx_get (inode, this, &iosstat);
                 if (iosstat) {
                         BUMP_STATS (iosstat, IOS_STATS_TYPE_WRITE);
-			BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
+                        BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
                         inode = NULL;
                         iosstat = NULL;
                 }
         }
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
         return 0;
 
 }
@@ -1367,7 +1378,7 @@ io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                       int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
 {
         struct ios_stat *iosstat = NULL;
         inode_t         *inode   = frame->local;
@@ -1383,17 +1394,17 @@ io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               iosstat = NULL;
         }
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
 io_stats_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, READDIR);
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
@@ -1401,10 +1412,10 @@ io_stats_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf)
+                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FSYNC);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
         return 0;
 }
 
@@ -1412,10 +1423,10 @@ io_stats_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop)
+                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, SETATTR);
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop);
+        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
         return 0;
 }
 
@@ -1423,11 +1434,11 @@ io_stats_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, UNLINK);
         STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 
 }
@@ -1437,12 +1448,12 @@ int
 io_stats_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
                      struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent)
+                     struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, RENAME);
         STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
                              preoldparent, postoldparent,
-                             prenewparent, postnewparent);
+                             prenewparent, postnewparent, xdata);
         return 0;
 }
 
@@ -1450,10 +1461,10 @@ io_stats_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, const char *buf,
-                       struct iatt *sbuf)
+                       struct iatt *sbuf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, READLINK);
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, sbuf);
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
         return 0;
 }
 
@@ -1462,10 +1473,10 @@ int
 io_stats_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
                      inode_t *inode, struct iatt *buf,
-                     dict_t *xattr, struct iatt *postparent)
+                     dict_t *xdata, struct iatt *postparent)
 {
         UPDATE_PROFILE_STATS (frame, LOOKUP);
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xattr,
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xdata,
                              postparent);
         return 0;
 }
@@ -1475,11 +1486,11 @@ int
 io_stats_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
                       inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, SYMLINK);
         STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -1488,11 +1499,11 @@ int
 io_stats_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent)
+                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, MKNOD);
         STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -1501,7 +1512,8 @@ int
 io_stats_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent)
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t *xdata)
 {
         struct ios_stat *iosstat = NULL;
         char   *path = frame->local;
@@ -1519,8 +1531,11 @@ io_stats_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 unwind:
+        /* local is assigned with path */
+        GF_FREE (frame->local);
+        frame->local = NULL;
         STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -1529,28 +1544,28 @@ int
 io_stats_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
                    inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent)
+                   struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, LINK);
         STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 io_stats_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FLUSH);
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd)
+                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         struct ios_stat *iosstat = NULL;
         int              ret     = -1;
@@ -1566,7 +1581,7 @@ io_stats_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 BUMP_STATS (iosstat, IOS_STATS_TYPE_OPENDIR);
 
 unwind:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
@@ -1574,13 +1589,13 @@ unwind:
 int
 io_stats_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *preparent, struct iatt *postparent)
+                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
 
         UPDATE_PROFILE_STATS (frame, RMDIR);
 
         STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -1588,100 +1603,100 @@ io_stats_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno,
-                       struct iatt *prebuf, struct iatt *postbuf)
+                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, TRUNCATE);
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
+                             prebuf, postbuf, xdata);
         return 0;
 }
 
 
 int
 io_stats_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                     int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, STATFS);
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
 io_stats_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, SETXATTR);
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, GETXATTR);
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, REMOVEXATTR);
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 io_stats_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FSETXATTR);
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *dict)
+                        int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FGETXATTR);
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno)
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FREMOVEXATTR);
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FSYNCDIR);
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, ACCESS);
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -1689,92 +1704,126 @@ io_stats_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno,
-                        struct iatt *prebuf, struct iatt *postbuf)
+                        struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FTRUNCATE);
         STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
+                             prebuf, postbuf, xdata);
         return 0;
 }
 
 
 int
 io_stats_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FSTAT);
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
+io_stats_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		       struct iatt *postbuf, dict_t *xdata)
+{
+	UPDATE_PROFILE_STATS(frame, FALLOCATE);
+	STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+			    xdata);
+	return 0;
+}
+
+
+int
+io_stats_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		     struct iatt *postbuf, dict_t *xdata)
+{
+	UPDATE_PROFILE_STATS(frame, DISCARD);
+	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+			    xdata);
+	return 0;
+}
+
+int
+io_stats_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS(frame, ZEROFILL);
+        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+        return 0;
+}
+
+int
 io_stats_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                 int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, LK);
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
         return 0;
 }
 
 
 int
 io_stats_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, ENTRYLK);
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, XATTROP);
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FXATTROP);
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, INODELK);
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 io_stats_entrylk (call_frame_t *frame, xlator_t *this,
                   const char *volume, loc_t *loc, const char *basename,
-                  entrylk_cmd cmd, entrylk_type type)
+                  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_entrylk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
         return 0;
 }
 
 
 int
 io_stats_inodelk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
 
         START_FOP_LATENCY (frame);
@@ -1782,128 +1831,122 @@ io_stats_inodelk (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_inodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock);
+                    volume, loc, cmd, flock, xdata);
         return 0;
 }
 
 
 int
 io_stats_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
 
         UPDATE_PROFILE_STATS (frame, FINODELK);
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
-io_stats_finodelk (call_frame_t *frame, xlator_t *this,
-                   const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+io_stats_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+                   fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_finodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock);
+                    volume, fd, cmd, flock, xdata);
         return 0;
 }
 
 
 int
-io_stats_xattrop (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, gf_xattrop_flags_t flags, dict_t *dict)
+io_stats_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_xattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict);
-
+                    loc, flags, dict, xdata);
         return 0;
 }
 
 
 int
-io_stats_fxattrop (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+io_stats_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fxattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict);
-
+                    fd, flags, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_lookup (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *xattr_req)
+                 loc_t *loc, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
-                    loc, xattr_req);
-
+                    loc, xdata);
         return 0;
 }
 
 
 int
-io_stats_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+io_stats_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_stat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->stat,
-                    loc);
-
+                    loc, xdata);
         return 0;
 }
 
 
 int
 io_stats_readlink (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, size_t size)
+                   loc_t *loc, size_t size, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_readlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readlink,
-                    loc, size);
-
+                    loc, size, xdata);
         return 0;
 }
 
 
 int
-io_stats_mknod (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, dev_t dev, dict_t *params)
+io_stats_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_mknod_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, params);
-
+                    loc, mode, dev, umask, xdata);
         return 0;
 }
 
 
 int
 io_stats_mkdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, dict_t *params)
+                loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
         frame->local = gf_strdup (loc->path);
 
@@ -1912,117 +1955,112 @@ io_stats_mkdir (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_mkdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, params);
+                    loc, mode, umask, xdata);
         return 0;
 }
 
 
 int
 io_stats_unlink (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc)
+                 loc_t *loc, int xflag, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_unlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->unlink,
-                    loc);
+                    loc, xflag, xdata);
         return 0;
 }
 
 
 int
 io_stats_rmdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int flags)
+                loc_t *loc, int flags, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_rmdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags);
-
+                    loc, flags, xdata);
         return 0;
 }
 
 
 int
-io_stats_symlink (call_frame_t *frame, xlator_t *this,
-                  const char *linkpath, loc_t *loc, dict_t *params)
+io_stats_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+                  loc_t *loc, mode_t umask, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_symlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, params);
-
+                    linkpath, loc, umask, xdata);
         return 0;
 }
 
 
 int
 io_stats_rename (call_frame_t *frame, xlator_t *this,
-                 loc_t *oldloc, loc_t *newloc)
+                 loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_rename_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc);
-
+                    oldloc, newloc, xdata);
         return 0;
 }
 
 
 int
 io_stats_link (call_frame_t *frame, xlator_t *this,
-               loc_t *oldloc, loc_t *newloc)
+               loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_link_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
         return 0;
 }
 
 
 int
 io_stats_setattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, struct iatt *stbuf, int32_t valid)
+                  loc_t *loc, struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_setattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid);
-
+                    loc, stbuf, valid, xdata);
         return 0;
 }
 
 
 int
 io_stats_truncate (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, off_t offset)
+                   loc_t *loc, off_t offset, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_truncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->truncate,
-                    loc, offset);
-
+                    loc, offset, xdata);
         return 0;
 }
 
 
 int
-io_stats_open (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, int32_t flags, fd_t *fd, int32_t wbflags)
+io_stats_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int32_t flags, fd_t *fd, dict_t *xdata)
 {
         frame->local = gf_strdup (loc->path);
 
@@ -2031,7 +2069,7 @@ io_stats_open (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_open_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
         return 0;
 }
 
@@ -2039,7 +2077,7 @@ io_stats_open (call_frame_t *frame, xlator_t *this,
 int
 io_stats_create (call_frame_t *frame, xlator_t *this,
                  loc_t *loc, int32_t flags, mode_t mode,
-                 fd_t *fd, dict_t *params)
+                 mode_t umask, fd_t *fd, dict_t *xdata)
 {
         frame->local = gf_strdup (loc->path);
 
@@ -2048,14 +2086,14 @@ io_stats_create (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_create_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
 
 int
 io_stats_readv (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, size_t size, off_t offset, uint32_t flags)
+                fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         frame->local = fd;
 
@@ -2064,7 +2102,7 @@ io_stats_readv (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_readv_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags);
+                    fd, size, offset, flags, xdata);
         return 0;
 }
 
@@ -2073,7 +2111,7 @@ int
 io_stats_writev (call_frame_t *frame, xlator_t *this,
                  fd_t *fd, struct iovec *vector,
                  int32_t count, off_t offset,
-                 uint32_t flags, struct iobref *iobref)
+                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         int                 len = 0;
 
@@ -2087,7 +2125,7 @@ io_stats_writev (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_writev_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
         return 0;
 
 }
@@ -2095,47 +2133,47 @@ io_stats_writev (call_frame_t *frame, xlator_t *this,
 
 int
 io_stats_statfs (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc)
+                 loc_t *loc, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_statfs_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->statfs,
-                    loc);
+                    loc, xdata);
         return 0;
 }
 
 
 int
 io_stats_flush (call_frame_t *frame, xlator_t *this,
-                fd_t *fd)
+                fd_t *fd, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_flush_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->flush,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsync (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t flags)
+                fd_t *fd, int32_t flags, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fsync_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsync,
-                    fd, flags);
+                    fd, flags, xdata);
         return 0;
 }
 
 
-void
+int
 conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
 {
         struct {
@@ -2159,27 +2197,27 @@ conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
 
                 if (!strncmp (filename, "", 1)) {
                         gf_log (this->name, GF_LOG_ERROR, "No filename given");
-                        return;
+                        return -1;
                 }
                 logfp = fopen (filename, "w+");
-                GF_ASSERT (logfp);
                 if (!logfp) {
                         gf_log (this->name, GF_LOG_ERROR, "failed to open %s "
                                 "for writing", filename);
-                        return;
+                        return -1;
                 }
                 (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_FILE,
                                            logfp);
                 io_stats_dump (this, &args);
                 fclose (logfp);
         }
+        return 0;
 }
 
 
 int
 io_stats_setxattr (call_frame_t *frame, xlator_t *this,
                    loc_t *loc, dict_t *dict,
-                   int32_t flags)
+                   int32_t flags, dict_t *xdata)
 {
         struct {
                 xlator_t     *this;
@@ -2198,36 +2236,35 @@ io_stats_setxattr (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    loc, dict, flags, xdata);
         return 0;
 }
 
 
 int
 io_stats_getxattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name)
+                   loc_t *loc, const char *name, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr,
-                    loc, name);
+                    loc, name, xdata);
         return 0;
 }
 
 
 int
 io_stats_removexattr (call_frame_t *frame, xlator_t *this,
-                      loc_t *loc, const char *name)
+                      loc_t *loc, const char *name, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_removexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->removexattr,
-                    loc, name);
-
+                    loc, name, xdata);
         return 0;
 }
 
@@ -2235,50 +2272,49 @@ io_stats_removexattr (call_frame_t *frame, xlator_t *this,
 int
 io_stats_fsetxattr (call_frame_t *frame, xlator_t *this,
                     fd_t *fd, dict_t *dict,
-                    int32_t flags)
+                    int32_t flags, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fsetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    fd, dict, flags, xdata);
         return 0;
 }
 
 
 int
 io_stats_fgetxattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, const char *name)
+                    fd_t *fd, const char *name, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fgetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name);
+                    fd, name, xdata);
         return 0;
 }
 
 
 int
 io_stats_fremovexattr (call_frame_t *frame, xlator_t *this,
-                       fd_t *fd, const char *name)
+                       fd_t *fd, const char *name, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fremovexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name);
-
+                    fd, name, xdata);
         return 0;
 }
 
 
 int
 io_stats_opendir (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, fd_t *fd)
+                  loc_t *loc, fd_t *fd, dict_t *xdata)
 {
 
         START_FOP_LATENCY (frame);
@@ -2286,7 +2322,7 @@ io_stats_opendir (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_opendir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->opendir,
-                    loc, fd);
+                    loc, fd, xdata);
         return 0;
 }
 
@@ -2301,107 +2337,143 @@ io_stats_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdirp,
                     fd, size, offset, dict);
-
         return 0;
 }
 
 
 int
 io_stats_readdir (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, size_t size, off_t offset)
+                  fd_t *fd, size_t size, off_t offset, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_readdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset);
-
+                    fd, size, offset, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsyncdir (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, int32_t datasync)
+                   fd_t *fd, int32_t datasync, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fsyncdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync);
+                    fd, datasync, xdata);
         return 0;
 }
 
 
 int
 io_stats_access (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int32_t mask)
+                 loc_t *loc, int32_t mask, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_access_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->access,
-                    loc, mask);
+                    loc, mask, xdata);
         return 0;
 }
 
 
 int
 io_stats_ftruncate (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, off_t offset)
+                    fd_t *fd, off_t offset, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_ftruncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset);
-
+                    fd, offset, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsetattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, struct iatt *stbuf, int32_t valid)
+                   fd_t *fd, struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_setattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid);
+                    fd, stbuf, valid, xdata);
         return 0;
 }
 
 
 int
 io_stats_fstat (call_frame_t *frame, xlator_t *this,
-                fd_t *fd)
+                fd_t *fd, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fstat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fstat,
-                    fd);
+                    fd, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		   off_t offset, size_t len, dict_t *xdata)
+{
+	START_FOP_LATENCY(frame);
+
+	STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		   xdata);
+
+	return 0;
+}
+
+
+int
+io_stats_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+		 size_t len, dict_t *xdata)
+{
+	START_FOP_LATENCY(frame);
+
+	STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+
+	return 0;
+}
+
+int
+io_stats_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 size_t len, dict_t *xdata)
+{
+        START_FOP_LATENCY(frame);
+
+        STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+
         return 0;
 }
 
 
 int
 io_stats_lk (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock)
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_lk_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock);
+                    fd, cmd, lock, xdata);
         return 0;
 }
 
@@ -2418,7 +2490,7 @@ io_stats_release (xlator_t *this, fd_t *fd)
 
         LOCK (&conf->lock);
         {
-		conf->cumulative.nr_opens--;
+                conf->cumulative.nr_opens--;
         }
         UNLOCK (&conf->lock);
 
@@ -2426,8 +2498,7 @@ io_stats_release (xlator_t *this, fd_t *fd)
         if (iosfd) {
                 io_stats_dump_fd (this, iosfd);
 
-                if (iosfd->filename)
-                        GF_FREE (iosfd->filename);
+                GF_FREE (iosfd->filename);
                 GF_FREE (iosfd);
         }
 
@@ -2452,6 +2523,93 @@ io_stats_forget (xlator_t *this, inode_t *inode)
         return 0;
 }
 
+static int
+ios_init_top_stats (struct ios_conf *conf)
+{
+        int     i = 0;
+
+        GF_ASSERT (conf);
+
+        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
+                conf->list[i].iosstats = GF_CALLOC (1,
+                                         sizeof(*conf->list[i].iosstats),
+                                         gf_io_stats_mt_ios_stat);
+
+                if (!conf->list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
+                LOCK_INIT (&conf->list[i].lock);
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
+                conf->thru_list[i].iosstats = GF_CALLOC (1,
+                                 sizeof (*conf->thru_list[i].iosstats),
+                                 gf_io_stats_mt_ios_stat);
+
+                if (!conf->thru_list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
+                LOCK_INIT (&conf->thru_list[i].lock);
+        }
+
+        return 0;
+}
+
+static void
+ios_destroy_top_stats (struct ios_conf *conf)
+{
+        int                     i = 0;
+        struct ios_stat_head    *list_head = NULL;
+        struct ios_stat_list    *entry     = NULL;
+        struct ios_stat_list    *tmp       = NULL;
+        struct ios_stat_list    *list      = NULL;
+        struct ios_stat         *stat      = NULL;
+
+        GF_ASSERT (conf);
+
+        LOCK (&conf->lock);
+
+        conf->cumulative.nr_opens = 0;
+        conf->cumulative.max_nr_opens = 0;
+        conf->cumulative.max_openfd_time.tv_sec = 0;
+        conf->cumulative.max_openfd_time.tv_usec = 0;
+
+        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+                list_head = &conf->list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+                list_head = &conf->thru_list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        UNLOCK (&conf->lock);
+
+        return;
+}
 
 int
 reconfigure (xlator_t *this, dict_t *options)
@@ -2519,7 +2677,6 @@ int
 init (xlator_t *this)
 {
         struct ios_conf    *conf = NULL;
-        int                 i = 0;
         char               *sys_log_str = NULL;
         int                 sys_log_level = -1;
         char               *log_str = NULL;
@@ -2556,35 +2713,9 @@ init (xlator_t *this)
         gettimeofday (&conf->cumulative.started_at, NULL);
         gettimeofday (&conf->incremental.started_at, NULL);
 
-        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
-                conf->list[i].iosstats = GF_CALLOC (1,
-                                         sizeof(*conf->list[i].iosstats),
-                                         gf_io_stats_mt_ios_stat);
-
-                if (!conf->list[i].iosstats) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                               "Out of memory");
-                        return -1;
-                }
-
-                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
-                LOCK_INIT (&conf->list[i].lock);
-        }
-
-	for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
-		conf->thru_list[i].iosstats = GF_CALLOC (1,
-		                 sizeof (*conf->thru_list[i].iosstats),
-				 gf_io_stats_mt_ios_stat);
-
-	        if (!conf->thru_list[i].iosstats) {
-        	        gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                	return -1;
-        	}
-
-		INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
-		LOCK_INIT (&conf->thru_list[i].lock);
-        }
+        ret = ios_init_top_stats (conf);
+        if (ret)
+                return -1;
 
         GF_OPTION_INIT ("dump-fd-stats", conf->dump_fd_stats, bool, out);
 
@@ -2616,12 +2747,6 @@ void
 fini (xlator_t *this)
 {
         struct ios_conf *conf = NULL;
-        struct ios_stat_head *list_head = NULL;
-        struct ios_stat_list *entry     = NULL;
-        struct ios_stat_list *tmp       = NULL;
-        struct ios_stat_list *list      = NULL;
-        struct ios_stat      *stat      = NULL;
-        int    i                        = 0;
 
         if (!this)
                 return;
@@ -2632,45 +2757,15 @@ fini (xlator_t *this)
                 return;
         this->private = NULL;
 
-        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
-                list_head = &conf->list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        if (list)
-                                GF_FREE (list);
-                }
-        }
-
-        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
-                list_head = &conf->thru_list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        if (list)
-                                GF_FREE (list);
-                }
-        }
+        ios_destroy_top_stats (conf);
 
-        if (conf)
-                GF_FREE(conf);
+        GF_FREE(conf);
 
         gf_log (this->name, GF_LOG_INFO,
                 "io-stats translator unloaded");
         return;
 }
 
-
 int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
@@ -2690,6 +2785,28 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         va_end (ap);
         switch (event) {
         case GF_EVENT_TRANSLATOR_INFO:
+                ret = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+                if (ret) {
+                         ret = dict_set_int32 (output, "top-op", top_op);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set top-op in dict");
+                                goto out;
+                        }
+                        ios_destroy_top_stats (this->private);
+                        ret = ios_init_top_stats (this->private);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to reset top stats");
+                        ret = dict_set_int32 (output, "stats-cleared",
+                                              ret ? 0 : 1);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set stats-cleared"
+                                        " in dict");
+                        goto out;
+                }
+
                 ret = dict_get_int32 (dict, "top-op", &top_op);
                 if (!ret) {
                         ret = dict_get_int32 (dict, "list-cnt", &list_cnt);
@@ -2773,6 +2890,9 @@ struct xlator_fops fops = {
         .fxattrop    = io_stats_fxattrop,
         .setattr     = io_stats_setattr,
         .fsetattr    = io_stats_fsetattr,
+	.fallocate   = io_stats_fallocate,
+	.discard     = io_stats_discard,
+        .zerofill    = io_stats_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -2816,7 +2936,7 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_STR,
           .default_value = "CRITICAL",
           .description = "Gluster's syslog log-level",
-          .value = { "WARNING", "ERROR", "CRITICAL"}
+          .value = { "WARNING", "ERROR", "INFO", "CRITICAL"}
         },
         { .key = {"brick-log-level"},
           .type = GF_OPTION_TYPE_STR,
diff --git a/xlators/debug/trace/src/Makefile.am b/xlators/debug/trace/src/Makefile.am
index 0f1679a04..7b2597b4d 100644
--- a/xlators/debug/trace/src/Makefile.am
+++ b/xlators/debug/trace/src/Makefile.am
@@ -2,13 +2,15 @@
 xlator_LTLIBRARIES = trace.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-trace_la_LDFLAGS = -module -avoidversion
+trace_la_LDFLAGS = -module -avoid-version
 
 trace_la_SOURCES = trace.c
 trace_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = trace.h trace-mem-types.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/debug/trace/src/trace-mem-types.h b/xlators/debug/trace/src/trace-mem-types.h
new file mode 100644
index 000000000..9fa7d97c2
--- /dev/null
+++ b/xlators/debug/trace/src/trace-mem-types.h
@@ -0,0 +1,21 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __TRACE_MEM_TYPES_H__
+#define __TRACE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_trace_mem_types_ {
+        gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
+        gf_trace_mt_end
+};
+#endif
diff --git a/xlators/debug/trace/src/trace.c b/xlators/debug/trace/src/trace.c
index 09c58bdfd..c9d839356 100644
--- a/xlators/debug/trace/src/trace.c
+++ b/xlators/debug/trace/src/trace.c
@@ -1,26 +1,15 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include "trace.h"
+#include "trace-mem-types.h"
 
 /**
  * xlators/debug/trace :
@@ -29,1203 +18,1388 @@
  *    Very helpful translator for debugging.
  */
 
-#include <time.h>
-#include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "common-utils.h"
-
-
-struct {
-        char *name;
-        int enabled;
-} trace_fop_names[GF_FOP_MAXVALUE];
-
-int trace_log_level = GF_LOG_INFO;
-
-static char *
-trace_stat_to_str (struct iatt *buf)
+int
+dump_history_trace (circular_buffer_t *cb, void *data)
 {
-        char    *statstr           = NULL;
-        char     atime_buf[256]    = {0,};
-        char     mtime_buf[256]    = {0,};
-        char     ctime_buf[256]    = {0,};
-        int      asprint_ret_value = 0;
-        uint64_t ia_time           = 0;
-
-        if (!buf) {
-                statstr = NULL;
-                goto out;
-        }
-
-        ia_time = buf->ia_atime;
-        strftime (atime_buf, 256, "[%b %d %H:%M:%S]",
-                  localtime ((time_t *)&ia_time));
-
-        ia_time = buf->ia_mtime;
-        strftime (mtime_buf, 256, "[%b %d %H:%M:%S]",
-                  localtime ((time_t *)&ia_time));
+        char    *string      = NULL;
+        struct  tm      *tm  = NULL;
+        char    timestr[256] = {0,};
 
-        ia_time = buf->ia_ctime;
-        strftime (ctime_buf, 256, "[%b %d %H:%M:%S]",
-                  localtime ((time_t *)&ia_time));
+        string = (char *)cb->data;
+        tm    = localtime (&cb->tv.tv_sec);
 
-        asprint_ret_value = gf_asprintf (&statstr,
-                                         "gfid=%s ino=%"PRIu64", mode=%o, "
-                                         "nlink=%"GF_PRI_NLINK", uid=%u, "
-                                         "gid=%u, size=%"PRIu64", "
-                                         "blocks=%"PRIu64", atime=%s, "
-                                         "mtime=%s, ctime=%s",
-                                         uuid_utoa (buf->ia_gfid), buf->ia_ino,
-                                         st_mode_from_ia (buf->ia_prot,
-                                                          buf->ia_type),
-                                         buf->ia_nlink, buf->ia_uid,
-                                         buf->ia_gid, buf->ia_size,
-                                         buf->ia_blocks, atime_buf,
-                                         mtime_buf, ctime_buf);
+        /* Since we are continuing with adding entries to the buffer even when
+           gettimeofday () fails, it's safe to check tm and then dump the time
+           at which the entry was added to the buffer */
+        if (tm) {
+                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
+                gf_proc_dump_write ("TIME", "%s", timestr);
+        }
 
-        if (asprint_ret_value < 0)
-                statstr = NULL;
+        gf_proc_dump_write ("FOP", "%s\n", string);
 
-out:
-        return statstr;
+        return 0;
 }
 
-
 int
 trace_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, fd_t *fd,
                   inode_t *inode, struct iatt *buf,
-                  struct iatt *preparent, struct iatt *postparent)
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_CREATE].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
-                                "*stbuf {%s}, *preparent {%s}, *postparent = "
-                                "{%s})", frame->root->unique,
-                                uuid_utoa (inode->gfid), op_ret, fd,
-                                statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
+                                  "*stbuf {%s}, *preparent {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid), op_ret, fd,
+                                  statstr, preparentstr, postparentstr);
 
                         /* for 'release' log */
                         fd_ctx_set (fd, this, 0);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd)
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t      *conf = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, *fd=%p",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno, fd);
+                char     string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, "
+                          "*fd=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          fd);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         /* for 'release' log */
         if (op_ret >= 0)
                 fd_ctx_set (fd, this, 0);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        TRACE_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
-
 int
 trace_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
 {
-        char *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STAT].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, statstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+out:
+        TRACE_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref)
+                 int32_t count, struct iatt *buf, struct iobref *iobref,
+                 dict_t *xdata)
 {
-        char  *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READ].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, statstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             buf, iobref);
+out:
+        TRACE_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
+                            buf, iobref, xdata);
         return 0;
 }
 
-
 int
 trace_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf)
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
+        char         preopstr[4096]  = {0, };
+        char         postopstr[4096] = {0, };
+        trace_conf_t *conf           = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_WRITE].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
-
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
         return 0;
 }
 
-
-
 int
 trace_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                   dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
-        }
+                char    string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (readdir, frame, op_ret, op_errno, buf, xdata);
 
         return 0;
 }
 
-
 int
 trace_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                    dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
-        }
+                char        string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf);
+                LOG_ELEMENT (conf, string);
+        }
 
+out:
+        TRACE_STACK_UNWIND (readdirp, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *prebuf, struct iatt *postbuf)
+                 struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s}",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
+        conf = this->private;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s}",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
 
         return 0;
 }
 
-
 int
 trace_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
-                   struct iatt *statpre, struct iatt *statpost)
+                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (statpre);
-                        postopstr = trace_stat_to_str (statpost);
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
+        conf = this->private;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (statpre, preopstr);
+                        trace_stat_to_str (statpost, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre, statpost);
+out:
+        TRACE_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
+                            statpost, xdata);
         return 0;
 }
 
-
 int
 trace_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *statpre, struct iatt *statpost)
+                    struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (statpre);
-                        postopstr = trace_stat_to_str (statpost);
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
+        conf = this->private;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (statpre, preopstr);
+                        trace_stat_to_str (statpost, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno,
-                             statpre, statpost);
+out:
+        TRACE_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
+                            statpre, statpost, xdata);
         return 0;
 }
 
-
 int
 trace_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *preparent, struct iatt *postparent)
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        char *preparentstr = NULL;
-        char *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                if (op_ret >= 0) {
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, preparentstr,
-                                postparentstr);
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  " *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
                   struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent)
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preoldparentstr = NULL;
-        char  *postoldparentstr = NULL;
-        char  *prenewparentstr = NULL;
-        char  *postnewparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preoldparentstr = trace_stat_to_str (preoldparent);
-                        postoldparentstr = trace_stat_to_str (postoldparent);
-
-                        prenewparentstr = trace_stat_to_str (prenewparent);
-                        postnewparentstr = trace_stat_to_str (postnewparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *stbuf = {%s}, "
-                                "*preoldparent = {%s}, *postoldparent = {%s}"
-                                " *prenewparent = {%s}, *postnewparent = {%s})",
-                                frame->root->unique, op_ret, statstr,
-                                preoldparentstr, postoldparentstr,
-                                prenewparentstr, postnewparentstr);
+        char           statstr[4096]          = {0, };
+        char           preoldparentstr[4096]  = {0, };
+        char           postoldparentstr[4096] = {0, };
+        char           prenewparentstr[4096]  = {0, };
+        char           postnewparentstr[4096] = {0, };
+        trace_conf_t   *conf                  = NULL;
 
-                        if (preoldparentstr)
-                                GF_FREE (preoldparentstr);
+        conf = this->private;
 
-                        if (postoldparentstr)
-                                GF_FREE (postoldparentstr);
-
-                        if (prenewparentstr)
-                                GF_FREE (prenewparentstr);
-
-                        if (postnewparentstr)
-                                GF_FREE (postnewparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RENAME].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preoldparent, preoldparentstr);
+                        trace_stat_to_str (postoldparent, postoldparentstr);
+                        trace_stat_to_str (prenewparent, prenewparentstr);
+                        trace_stat_to_str (postnewparent, postnewparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*stbuf = {%s}, *preoldparent = {%s},"
+                                  " *postoldparent = {%s}"
+                                  " *prenewparent = {%s}, "
+                                  "*postnewparent = {%s})",
+                                  frame->root->unique, op_ret, statstr,
+                                  preoldparentstr, postoldparentstr,
+                                  prenewparentstr, postnewparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
+
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent,
-                             prenewparent, postnewparent);
+out:
+        TRACE_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
+                            preoldparent, postoldparent,
+                            prenewparent, postnewparent, xdata);
         return 0;
 }
 
-
 int
 trace_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    const char *buf, struct iatt *stbuf)
+                    const char *buf, struct iatt *stbuf, dict_t *xdata)
 {
-        char *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
 
-        if (trace_fop_names[GF_FOP_READLINK].enabled) {
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READLINK].enabled) {
+                char string[4096] = {0,};
                 if (op_ret == 0) {
-                        statstr = trace_stat_to_str (stbuf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d, buf=%s, "
-                                "stbuf = { %s })",
-                                frame->root->unique, op_ret, op_errno, buf,
-                                statstr);
-                } else
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        trace_stat_to_str (stbuf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d,"
+                                  "buf=%s, stbuf = { %s })",
+                                  frame->root->unique, op_ret, op_errno,
+                                  buf, statstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
 
-                if (statstr)
-                        GF_FREE (statstr);
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, stbuf);
+out:
+        TRACE_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, stbuf,
+                            xdata);
         return 0;
 }
 
-
 int
 trace_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
                   inode_t *inode, struct iatt *buf,
-                  dict_t *xattr, struct iatt *postparent)
+                  dict_t *xdata, struct iatt *postparent)
 {
-        char  *statstr = NULL;
-        char  *postparentstr = NULL;
+        char          statstr[4096]       = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        postparentstr = trace_stat_to_str (postparent);
+        conf = this->private;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                "*buf {%s}, *postparent {%s}",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (postparent, postparentstr);
+                        /* print buf->ia_gfid instead of inode->gfid,
+                         * since if the inode is not yet linked to the
+                         * inode table (fresh lookup) then null gfid
+                         * will be printed.
+                         */
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*buf {%s}, *postparent {%s}",
+                                  frame->root->unique,
+                                  uuid_utoa (buf->ia_gfid),
+                                  op_ret, statstr, postparentstr);
 
                         /* For 'forget' */
                         inode_ctx_put (inode, this, 0);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xattr, postparent);
+out:
+        TRACE_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
+                            xdata, postparent);
         return 0;
 }
 
-
 int
 trace_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
                    inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent)
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                "*stbuf = {%s}, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*stbuf = {%s}, *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": op_ret=%d, op_errno=%d",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": op_ret=%d, op_errno=%d",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char *statstr = NULL;
-        char *preparentstr = NULL;
-        char *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                "*stbuf = {%s}, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char string[4096]  = {0,};
+        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*stbuf = {%s}, *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                ", *stbuf = {%s}, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  ", *stbuf = {%s}, *prebuf = {%s}, "
+                                  "*postbuf = {%s} )",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
                 inode_t *inode, struct iatt *buf,
-                struct iatt *preparent, struct iatt *postparent)
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_LINK].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *stbuf = {%s}, "
-                                " *prebuf = {%s}, *postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                statstr, preparentstr, postparentstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (statstr)
-                                GF_FREE (statstr);
+        conf = this->private;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char  string[4096]  = {0,};
+        if (trace_fop_names[GF_FOP_LINK].enabled) {
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*stbuf = {%s},  *prebuf = {%s},"
+                                  " *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  statstr, preparentstr, postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno)
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char      string[4096] = {0,};
         if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd)
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char    string[4096] = {0,};
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, fd=%p",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno, fd);
-        }
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " fd=%p",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno, fd);
 
+                LOG_ELEMENT (conf, string);
+        }
+out:
         /* for 'releasedir' log */
         if (op_ret >= 0)
                 fd_ctx_set (fd, this, 0);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        TRACE_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
-
 int
 trace_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
+        char           preparentstr[4096]  = {0, };
+        char           postparentstr[4096] = {0, };
+        trace_conf_t   *conf               = NULL;
 
-        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                if (op_ret >= 0) {
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
+        conf = this->private;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s}",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, preparentstr,  postparentstr);
-
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "*prebuf={%s},  *postbuf={%s}",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, preparentstr, postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf)
+                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
+        char           preopstr[4096]  = {0, };
+        char           postopstr[4096] = {0, };
+        trace_conf_t   *conf           = NULL;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, op_ret, preopstr,
-                                postopstr);
+        conf = this->private;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+                char   string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s} )",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
+                            postbuf, xdata);
         return 0;
 }
 
-
 int
 trace_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                  dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                if (op_ret >= 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": ({f_bsize=%lu, f_frsize=%lu, f_blocks=%"GF_PRI_FSBLK
-                                ", f_bfree=%"GF_PRI_FSBLK", f_bavail=%"GF_PRI_FSBLK", "
-                                "f_files=%"GF_PRI_FSBLK", f_ffree=%"GF_PRI_FSBLK", f_favail=%"
-                                GF_PRI_FSBLK", f_fsid=%lu, f_flag=%lu, f_namemax=%lu}) => ret=%d",
-                                frame->root->unique, buf->f_bsize, buf->f_frsize, buf->f_blocks,
-                                buf->f_bfree, buf->f_bavail, buf->f_files, buf->f_ffree,
-                                buf->f_favail, buf->f_fsid, buf->f_flag, buf->f_namemax, op_ret);
+                char   string[4096] = {0,};
+                if (op_ret == 0) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": ({f_bsize=%lu, "
+                                  "f_frsize=%lu, "
+                                  "f_blocks=%"GF_PRI_FSBLK
+                                  ", f_bfree=%"GF_PRI_FSBLK", "
+                                  "f_bavail=%"GF_PRI_FSBLK", "
+                                  "f_files=%"GF_PRI_FSBLK", "
+                                  "f_ffree=%"GF_PRI_FSBLK", "
+                                  "f_favail=%"GF_PRI_FSBLK", "
+                                  "f_fsid=%lu, f_flag=%lu, "
+                                  "f_namemax=%lu}) => ret=%d",
+                                  frame->root->unique, buf->f_bsize,
+                                  buf->f_frsize, buf->f_blocks,
+                                  buf->f_bfree, buf->f_bavail,
+                                  buf->f_files, buf->f_ffree,
+                                  buf->f_favail, buf->f_fsid,
+                                  buf->f_flag, buf->f_namemax, op_ret);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+out:
+        TRACE_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret,
+                          op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, dict=%p",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret,
-                        op_errno, dict);
-        }
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " dict=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          dict);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
 
         return 0;
 }
 
 int
 trace_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret,
-                        op_errno);
-        }
+                char    string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, dict=%p",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret,
-                        op_errno, dict);
-        }
+                char      string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " dict=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          dict);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
         return 0;
 }
 
 int
 trace_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
 
-
 int
 trace_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char       string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsyncdir, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, "
+                          "op_errno=%d)", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf)
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *prebufstr = NULL;
-        char  *postbufstr = NULL;
-
-        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                if (op_ret >= 0) {
-                        prebufstr = trace_stat_to_str (prebuf);
-                        postbufstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, op_ret,
-                                prebufstr, postbufstr);
+        char          prebufstr[4096]  = {0, };
+        char          postbufstr[4096] = {0, };
+        trace_conf_t  *conf            = NULL;
 
-                        if (prebufstr)
-                                GF_FREE (prebufstr);
-
-                        if (postbufstr)
-                                GF_FREE (postbufstr);
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, prebufstr);
+                        trace_stat_to_str (postbuf, postbufstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s} )",
+                                  frame->root->unique, op_ret,
+                                  prebufstr, postbufstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
         return 0;
 }
 
-
 int
 trace_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        char *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, statstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+                char string[4096]  = {0.};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d "
+                                  "buf=%s", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
+out:
+        TRACE_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+              int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+              dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LK].enabled) {
-                if (op_ret >= 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, {l_type=%d, l_whence=%d, "
-                                "l_start=%"PRId64", l_len=%"PRId64", l_pid=%u})",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, lock->l_type, lock->l_whence,
-                                lock->l_start, lock->l_len, lock->l_pid);
+                char      string[4096] = {0,};
+                if (op_ret == 0) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "{l_type=%d, l_whence=%d, "
+                                  "l_start=%"PRId64", "
+                                  "l_len=%"PRId64", l_pid=%u})",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, lock->l_type, lock->l_whence,
+                                  lock->l_start, lock->l_len,
+                                  lock->l_pid);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
-        }
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
         return 0;
 }
 
-
-
 int
 trace_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char   string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 trace_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *dict)
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char    string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
-
 int
 trace_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
-
 int
 trace_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_INODELK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local),op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 trace_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FINODELK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     uint32_t weak_checksum, uint8_t *strong_checksum)
+                     uint32_t weak_checksum, uint8_t *strong_checksum,
+                     dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
         }
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum);
+out:
+        TRACE_STACK_UNWIND (rchecksum, frame, op_ret, op_errno, weak_checksum,
+                            strong_checksum, xdata);
 
         return 0;
 }
@@ -1235,35 +1409,54 @@ trace_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 trace_entrylk (call_frame_t *frame, xlator_t *this,
                const char *volume, loc_t *loc, const char *basename,
-               entrylk_cmd cmd, entrylk_type type)
+               entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (path=%s basename=%s, "
-                        "cmd=%s, type=%s)",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        volume, loc->path, basename,
-                        ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
-                        ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
+                char     string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (path=%s "
+                          "basename=%s, cmd=%s, type=%s)",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid),
+                          volume, loc->path, basename,
+                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
+                           "ENTRYLK_UNLOCK"),
+                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
+                           "ENTRYLK_WRLCK"));
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_entrylk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
         return 0;
 }
 
-
 int
 trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-               loc_t *loc, int32_t cmd, struct gf_flock *flock)
+               loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
+        char         *cmd_str  = NULL;
+        char         *type_str = NULL;
+        trace_conf_t *conf     = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_INODELK].enabled) {
+                char string[4096]  = {0,};
                 switch (cmd) {
 #if F_GETLK != F_GETLK64
                 case F_GETLK64:
@@ -1306,32 +1499,43 @@ trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                         break;
                 }
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (path=%s "
-                        "cmd=%s, type=%s, start=%llu, len=%llu, pid=%llu)",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        volume, loc->path,
-                        cmd_str, type_str, (unsigned long long) flock->l_start,
-                        (unsigned long long) flock->l_len,
-                        (unsigned long long) flock->l_pid);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (path=%s "
+                          "cmd=%s, type=%s, start=%llu, len=%llu, "
+                          "pid=%llu)", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), volume,
+                          loc->path, cmd_str, type_str,
+                          (unsigned long long)flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_inodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock);
+                    volume, loc, cmd, flock, xdata);
         return 0;
 }
 
-
 int
 trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                fd_t *fd, int32_t cmd, struct gf_flock *flock)
+                fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char *cmd_str = NULL, *type_str = NULL;
+        char          *cmd_str  = NULL;
+        char          *type_str = NULL;
+        trace_conf_t  *conf     = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+                char  string[4096] = {0,};
                 switch (cmd) {
 #if F_GETLK != F_GETLK64
                 case F_GETLK64:
@@ -1374,233 +1578,351 @@ trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                         break;
                 }
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (fd =%p "
-                        "cmd=%s, type=%s, start=%llu, len=%llu, pid=%llu)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), volume, fd,
-                        cmd_str, type_str, (unsigned long long) flock->l_start,
-                        (unsigned long long) flock->l_len,
-                        (unsigned long long) flock->l_pid);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (fd =%p "
+                          "cmd=%s, type=%s, start=%llu, len=%llu, "
+                          "pid=%llu)", frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), volume, fd,
+                          cmd_str, type_str,
+                          (unsigned long long) flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid);
+
                 frame->local = fd->inode->gfid;
-        }
 
+                LOG_ELEMENT (conf, string);
+        }
+out:
         STACK_WIND (frame, trace_finodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock);
+                    volume, fd, cmd, flock, xdata);
         return 0;
 }
 
-
 int
 trace_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               gf_xattrop_flags_t flags, dict_t *dict)
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s (path=%s flags=%d)",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s (path=%s flags=%d)",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_xattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict);
+                    loc, flags, dict, xdata);
 
         return 0;
 }
 
-
 int
 trace_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                gf_xattrop_flags_t flags, dict_t *dict)
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, flags=%d",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, flags);
+                char    string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fxattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict);
+                    fd, flags, dict, xdata);
 
         return 0;
 }
 
-
 int
 trace_lookup (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr_req)
+              loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char string[4096] = {0,};
                 /* TODO: print all the keys mentioned in xattr_req */
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
-                    loc, xattr_req);
+                    loc, xdata);
 
         return 0;
 }
 
-
 int
-trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_STAT].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path);
+                char  string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_stat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->stat,
-                    loc);
+                    loc, xdata);
 
         return 0;
 }
 
-
 int
-trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READLINK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s, size=%"GF_PRI_SIZET")",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, size);
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, "
+                          "size=%"GF_PRI_SIZET")", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          size);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readlink,
-                    loc, size);
+                    loc, size, xdata);
 
         return 0;
 }
 
-
 int
 trace_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             mode_t mode, dev_t dev, dict_t *params)
+             mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s mode=%d dev=%"GF_PRI_DEV")",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, mode, dev);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mode=%d "
+                          "umask=0%o, dev=%"GF_PRI_DEV")",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          mode, umask, dev);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_mknod_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, params);
+                    loc, mode, dev, umask, xdata);
 
         return 0;
 }
 
-
 int
 trace_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dict_t *params)
+             mode_t umask, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s mode=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, mode);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mode=%d"
+                          " umask=0%o", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          mode, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_mkdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, params);
+                    loc, mode, umask, xdata);
         return 0;
 }
 
-
 int
-trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flag=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          xflag);
+
                 frame->local = loc->inode->gfid;
-        }
 
+                LOG_ELEMENT (conf, string);
+        }
+out:
         STACK_WIND (frame, trace_unlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->unlink,
-                    loc);
+                    loc, xflag, xdata);
         return 0;
 }
 
-
 int
-trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+             dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s flags=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rmdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags);
+                    loc, flags, xdata);
 
         return 0;
 }
 
-
 int
 trace_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-               loc_t *loc, dict_t *params)
+               loc_t *loc, mode_t umask, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s linkpath=%s, path=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        linkpath, loc->path);
+                char     string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s linkpath=%s, path=%s"
+                          " umask=0%o", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), linkpath,
+                          loc->path, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_symlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, params);
+                    linkpath, loc, umask, xdata);
 
         return 0;
 }
 
-
 int
-trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        char oldgfid[50] = {0,};
-        char newgfid[50] = {0,};
+        char         oldgfid[50] = {0,};
+        char         newgfid[50] = {0,};
+        trace_conf_t *conf       = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RENAME].enabled) {
+                char string[4096] = {0,};
                 if (newloc->inode)
                         uuid_utoa_r (newloc->inode->gfid, newgfid);
                 else
@@ -1608,29 +1930,40 @@ trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
 
                 uuid_utoa_r (oldloc->inode->gfid, oldgfid);
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": oldgfid=%s oldpath=%s --> newgfid=%s newpath=%s",
-                        frame->root->unique, oldgfid, oldloc->path, newgfid, newloc->path);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": oldgfid=%s oldpath=%s --> "
+                          "newgfid=%s newpath=%s",
+                          frame->root->unique, oldgfid,
+                          oldloc->path, newgfid, newloc->path);
 
                 frame->local = oldloc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rename_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
 
         return 0;
 }
 
-
 int
-trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        char oldgfid[50] = {0,};
-        char newgfid[50] = {0,};
+        char         oldgfid[50] = {0,};
+        char         newgfid[50] = {0,};
+        trace_conf_t *conf       = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LINK].enabled) {
+                char string[4096]  = {0,};
                 if (newloc->inode)
                         uuid_utoa_r (newloc->inode->gfid, newgfid);
                 else
@@ -1638,42 +1971,63 @@ trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
 
                 uuid_utoa_r (oldloc->inode->gfid, oldgfid);
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": oldgfid=%s oldpath=%s --> newgfid=%s newpath=%s",
-                        frame->root->unique, oldgfid, oldloc->path,
-                        newgfid, newloc->path);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": oldgfid=%s oldpath=%s --> "
+                          "newgfid=%s newpath=%s", frame->root->unique,
+                          oldgfid, oldloc->path, newgfid,
+                          newloc->path);
+
                 frame->local = oldloc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_link_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
         return 0;
 }
 
-
 int
 trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid)
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        uint64_t ia_time          = 0;
-        char     actime_str[256]  = {0,};
-        char     modtime_str[256] = {0,};
+        uint64_t     ia_time          = 0;
+        char         actime_str[256]  = {0,};
+        char         modtime_str[256] = {0,};
+        trace_conf_t *conf            = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+                char     string[4096]  =  {0,};
                 if (valid & GF_SET_ATTR_MODE) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s path=%s mode=%o)",
-                                frame->root->unique, uuid_utoa (loc->inode->gfid),
-                                loc->path, st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type));
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s mode=%o)",
+                                  frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path,
+                                  st_mode_from_ia (stbuf->ia_prot,
+                                                   stbuf->ia_type));
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s path=%s uid=%o, gid=%o",
-                                frame->root->unique,  uuid_utoa (loc->inode->gfid),
-                                loc->path, stbuf->ia_uid, stbuf->ia_gid);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s uid=%o,"
+                                  " gid=%o", frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, stbuf->ia_uid,
+                                  stbuf->ia_gid);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
@@ -1685,44 +2039,64 @@ trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                         strftime (modtime_str, 256, "[%b %d %H:%M:%S]",
                                   localtime ((time_t *)&ia_time));
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s path=%s ia_atime=%s, ia_mtime=%s",
-                                frame->root->unique, uuid_utoa (loc->inode->gfid),
-                                loc->path, actime_str, modtime_str);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s "
+                                  "ia_atime=%s, ia_mtime=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, actime_str, modtime_str);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
                 frame->local = loc->inode->gfid;
         }
 
+out:
         STACK_WIND (frame, trace_setattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid);
+                    loc, stbuf, valid, xdata);
 
         return 0;
 }
 
-
 int
 trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        uint64_t ia_time          = 0;
-        char     actime_str[256]  = {0,};
-        char     modtime_str[256] = {0,};
+        uint64_t       ia_time          = 0;
+        char           actime_str[256]  = {0,};
+        char           modtime_str[256] = {0,};
+        trace_conf_t   *conf            = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+                char     string[4096]  =  {0,};
                 if (valid & GF_SET_ATTR_MODE) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s fd=%p, mode=%o",
-                                frame->root->unique, uuid_utoa (fd->inode->gfid), fd,
-                                st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type));
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p, mode=%o",
+                                  frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid), fd,
+                                  st_mode_from_ia (stbuf->ia_prot,
+                                                   stbuf->ia_type));
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s fd=%p, uid=%o, gid=%o",
-                                frame->root->unique, uuid_utoa (fd->inode->gfid),
-                                fd, stbuf->ia_uid, stbuf->ia_gid);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p, uid=%o, "
+                                  "gid=%o", frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid),
+                                  fd, stbuf->ia_uid, stbuf->ia_gid);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
@@ -1734,256 +2108,394 @@ trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                         strftime (modtime_str, 256, "[%b %d %H:%M:%S]",
                                   localtime ((time_t *)&ia_time));
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s fd=%p ia_atime=%s, ia_mtime=%s",
-                                frame->root->unique, uuid_utoa (fd->inode->gfid),
-                                fd, actime_str, modtime_str);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p "
+                                  "ia_atime=%s, ia_mtime=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid),
+                                  fd, actime_str, modtime_str);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
                 frame->local = fd->inode->gfid;
         }
 
+out:
         STACK_WIND (frame, trace_fsetattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid);
+                    fd, stbuf, valid, xdata);
 
         return 0;
 }
 
-
 int
 trace_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                off_t offset)
+                off_t offset, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s, offset=%"PRId64"",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, offset);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, "
+                          "offset=%"PRId64"", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          offset);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_truncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->truncate,
-                    loc, offset);
+                    loc, offset, xdata);
 
         return 0;
 }
 
-
 int
 trace_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            int32_t flags, fd_t *fd, int32_t wbflags)
+            int32_t flags, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s flags=%d fd=%p wbflags=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags, fd, wbflags);
+                char      string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags, fd);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_open_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
         return 0;
 }
 
-
 int
 trace_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+              int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+              dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_CREATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s, fd=%p, flags=0%o mode=0%o",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, fd, flags, mode);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, fd=%p, "
+                          "flags=0%o mode=0%o umask=0%o",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          fd, flags, mode, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_create_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
-
 int
 trace_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             size_t size, off_t offset, uint32_t flags)
+             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READ].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET", "
-                        "offset=%"PRId64" flags=0%x)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, size, offset, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"
+                          GF_PRI_SIZET"offset=%"PRId64" flags=0%x)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readv_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags);
+                    fd, size, offset, flags, xdata);
         return 0;
 }
 
-
 int
 trace_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
               struct iovec *vector, int32_t count,
-              off_t offset, uint32_t flags, struct iobref *iobref)
+              off_t offset, uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_WRITE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, count=%d, offset=%"PRId64
-                        " flag=0%x)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, count, offset, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, count=%d, "
+                          " offset=%"PRId64" flags=0%x)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, count,
+                          offset, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_writev_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
         return 0;
 }
 
-
 int
-trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, (loc->inode)?
-                        uuid_utoa (loc->inode->gfid):"0", loc->path);
+                char  string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique, (loc->inode)?
+                          uuid_utoa (loc->inode->gfid):"0", loc->path);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_statfs_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->statfs,
-                    loc);
+                    loc, xdata);
         return 0;
 }
 
-
 int
-trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_flush_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->flush,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
-
 int
-trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s flags=%d fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), flags, fd);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s flags=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), flags, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsync_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsync,
-                    fd, flags);
+                    fd, flags, xdata);
         return 0;
 }
 
-
 int
 trace_setxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, dict_t *dict, int32_t flags)
+                loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s flags=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    loc, dict, flags, xdata);
         return 0;
 }
 
-
 int
 trace_getxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, const char *name)
+                loc_t *loc, const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s name=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, name);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s name=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          name);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr,
-                    loc, name);
+                    loc, name, xdata);
         return 0;
 }
 
-
 int
 trace_removexattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name)
+                   loc_t *loc, const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s name=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, name);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s name=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          name);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_removexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->removexattr,
-                    loc, name);
+                    loc, name, xdata);
 
         return 0;
 }
 
-
 int
-trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+               dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s fd=%p",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path, fd);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_opendir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->opendir,
-                    loc, fd);
+                    loc, fd, xdata);
         return 0;
 }
 
@@ -1991,15 +2503,27 @@ int
 trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                 off_t offset, dict_t *dict)
 {
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET", "
-                        "offset=%"PRId64" dict=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, size, offset, dict);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
+                          ", offset=%"PRId64" dict=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset, dict);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readdirp_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdirp,
@@ -2008,83 +2532,128 @@ trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         return 0;
 }
 
-
 int
 trace_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t offset)
+               size_t size, off_t offset, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET", offset=%"PRId64,
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, size, offset);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
+                          ", offset=%"PRId64,
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset);
+                    fd, size, offset, xdata);
 
         return 0;
 }
 
-
 int
 trace_fsyncdir (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t datasync)
+                fd_t *fd, int32_t datasync, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s datasync=%d fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        datasync, fd);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s datasync=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), datasync, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsyncdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync);
+                    fd, datasync, xdata);
         return 0;
 }
 
-
 int
-trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+              dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s mask=0%o",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, mask);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mask=0%o",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid),
+                          loc->path, mask);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_access_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->access,
-                    loc, mask);
+                    loc, mask, xdata);
         return 0;
 }
 
-
 int32_t
 trace_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 int32_t len)
+                 int32_t len, dict_t *xdata)
 {
+
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s offset=%"PRId64" len=%u fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        offset, len, fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s offset=%"PRId64
+                          "len=%u fd=%p", frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), offset, len, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rchecksum_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rchecksum,
-                    fd, offset, len);
+                    fd, offset, len, xdata);
 
         return 0;
 
@@ -2093,162 +2662,264 @@ trace_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
 int32_t
 trace_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                 fd_t *fd, const char *basename, entrylk_cmd cmd,
-                entrylk_type type)
+                entrylk_type type, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (fd=%p basename=%s, "
-                        "cmd=%s, type=%s)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        volume, fd, basename,
-                        ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
-                        ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
+                char      string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (fd=%p "
+                          "basename=%s, cmd=%s, type=%s)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), volume, fd,
+                          basename,
+                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
+                           "ENTRYLK_UNLOCK"),
+                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
+                           "ENTRYLK_WRLCK"));
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fentrylk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
+                    volume, fd, basename, cmd, type, xdata);
         return 0;
 
 }
 
 int32_t
 trace_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p name=%s",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, name);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p name=%s",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, name);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fgetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name);
+                    fd, name, xdata);
         return 0;
 }
 
 int32_t
 trace_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int32_t flags)
+                 dict_t *dict, int32_t flags, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p flags=%d",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, flags);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    fd, dict, flags, xdata);
         return 0;
 }
 
 int
 trace_ftruncate (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, off_t offset)
+                 fd_t *fd, off_t offset, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        offset, fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), offset, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_ftruncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset);
+                    fd, offset, xdata);
 
         return 0;
 }
 
-
 int
-trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fstat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fstat,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
-
 int
 trace_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-          int32_t cmd, struct gf_flock *lock)
+          int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, cmd=%d, lock {l_type=%d, l_whence=%d, "
-                        "l_start=%"PRId64", l_len=%"PRId64", l_pid=%u})",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), fd,
-                        cmd, lock->l_type, lock->l_whence,
-                        lock->l_start, lock->l_len, lock->l_pid);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, cmd=%d, "
+                          "lock {l_type=%d, "
+                          "l_whence=%d, l_start=%"PRId64", "
+                          "l_len=%"PRId64", l_pid=%u})",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, cmd,
+                          lock->l_type, lock->l_whence,
+                          lock->l_start, lock->l_len, lock->l_pid);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_lk_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock);
+                    fd, cmd, lock, xdata);
         return 0;
 }
 
 int32_t
 trace_forget (xlator_t *this, inode_t *inode)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
         /* If user want to understand when a lookup happens,
            he should know about 'forget' too */
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "gfid=%s", uuid_utoa (inode->gfid));
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s", uuid_utoa (inode->gfid));
+
+                LOG_ELEMENT (conf, string);
         }
+
+out:
         return 0;
 }
 
-
 int32_t
 trace_releasedir (xlator_t *this, fd_t *fd)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "gfid=%s fd=%p", uuid_utoa (fd->inode->gfid), fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s fd=%p",
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         return 0;
 }
 
 int32_t
 trace_release (xlator_t *this, fd_t *fd)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled ||
             trace_fop_names[GF_FOP_CREATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "gfid=%s fd=%p", uuid_utoa (fd->inode->gfid), fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s fd=%p",
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                LOG_ELEMENT (conf, string);
         }
+
+out:
         return 0;
 }
 
 
-
 void
 enable_all_calls (int enabled)
 {
@@ -2258,7 +2929,6 @@ enable_all_calls (int enabled)
                 trace_fop_names[i].enabled = enabled;
 }
 
-
 void
 enable_call (const char *name, int enabled)
 {
@@ -2286,6 +2956,105 @@ process_call_list (const char *list, int include)
         }
 }
 
+int32_t
+trace_dump_history (xlator_t *this)
+{
+        int ret = -1;
+        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
+        trace_conf_t *conf = NULL;
+
+        GF_VALIDATE_OR_GOTO ("trace", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
+
+        conf = this->private;
+        // Is it ok to return silently if log-history option his off?
+        if (conf && conf->log_history == _gf_true) {
+                gf_proc_dump_build_key (key_prefix, "xlator.debug.trace",
+                                        "history");
+                gf_proc_dump_add_section (key_prefix);
+                eh_dump (this->history, NULL, dump_history_trace);
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_trace_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        " failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int32_t       ret            = -1;
+        trace_conf_t    *conf           = NULL;
+        char            *includes = NULL, *excludes = NULL;
+
+        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+        GF_VALIDATE_OR_GOTO (this->name, options, out);
+
+        conf = this->private;
+
+        includes = data_to_str (dict_get (options, "include-ops"));
+        excludes = data_to_str (dict_get (options, "exclude-ops"));
+
+        {
+                int i;
+                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+                        if (gf_fop_list[i])
+                                strncpy (trace_fop_names[i].name,
+                                         gf_fop_list[i],
+                                         strlen (gf_fop_list[i]));
+                        else
+                                strncpy (trace_fop_names[i].name, ":O",
+                                         strlen (":O"));
+                        trace_fop_names[i].enabled = 1;
+                }
+        }
+
+        if (includes && excludes) {
+                gf_log (this->name,
+                        GF_LOG_ERROR,
+                        "must specify only one of 'include-ops' and "
+                        "'exclude-ops'");
+                goto out;
+        }
+
+        if (includes)
+                process_call_list (includes, 1);
+        if (excludes)
+                process_call_list (excludes, 0);
+
+        /* Should resizing of the event-history be allowed in reconfigure?
+         * for which a new event_history might have to be allocated and the
+         * older history has to be freed.
+         */
+        GF_OPTION_RECONF ("log-file", conf->log_file, options, bool, out);
+
+        GF_OPTION_RECONF ("log-history", conf->log_history, options, bool, out);
+
+        ret = 0;
+
+out:
+        return ret;
+}
 
 int32_t
 init (xlator_t *this)
@@ -2293,6 +3062,10 @@ init (xlator_t *this)
         dict_t *options = NULL;
         char *includes = NULL, *excludes = NULL;
         char *forced_loglevel = NULL;
+        eh_t *history = NULL;
+        int  ret = -1;
+        size_t  history_size = TRACE_DEFAULT_HISTORY_SIZE;
+        trace_conf_t    *conf = NULL;
 
         if (!this)
                 return -1;
@@ -2307,6 +3080,12 @@ init (xlator_t *this)
                         "dangling volume. check volfile ");
         }
 
+        conf = GF_CALLOC (1, sizeof (trace_conf_t), gf_trace_mt_trace_conf_t);
+        if (!conf) {
+                gf_log (this->name, GF_LOG_ERROR, "cannot allocate "
+                        "xl->private");
+                return -1;
+        }
 
         options = this->options;
         includes = data_to_str (dict_get (options, "include-ops"));
@@ -2315,8 +3094,13 @@ init (xlator_t *this)
         {
                 int i;
                 for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        trace_fop_names[i].name = (gf_fop_list[i] ?
-                                                   gf_fop_list[i] : ":O");
+                        if (gf_fop_list[i])
+                                strncpy (trace_fop_names[i].name,
+                                         gf_fop_list[i],
+                                         strlen (gf_fop_list[i]));
+                        else
+                                strncpy (trace_fop_names[i].name, ":O",
+                                         strlen (":O"));
                         trace_fop_names[i].enabled = 1;
                 }
         }
@@ -2324,14 +3108,43 @@ init (xlator_t *this)
         if (includes && excludes) {
                 gf_log (this->name,
                         GF_LOG_ERROR,
-                        "must specify only one of 'include-ops' and 'exclude-ops'");
+                        "must specify only one of 'include-ops' and "
+                        "'exclude-ops'");
                 return -1;
         }
+
         if (includes)
                 process_call_list (includes, 1);
         if (excludes)
                 process_call_list (excludes, 0);
 
+
+        GF_OPTION_INIT ("history-size", conf->history_size, size, out);
+
+        gf_log (this->name, GF_LOG_INFO, "history size %"GF_PRI_SIZET,
+                history_size);
+
+        GF_OPTION_INIT ("log-file", conf->log_file, bool, out);
+
+        gf_log (this->name, GF_LOG_INFO, "logging to file %s",
+                (conf->log_file == _gf_true)?"enabled":"disabled");
+
+        GF_OPTION_INIT ("log-history", conf->log_history, bool, out);
+
+        gf_log (this->name, GF_LOG_DEBUG, "logging to history %s",
+                (conf->log_history == _gf_true)?"enabled":"disabled");
+
+        history = eh_new (history_size, _gf_false, NULL);
+        if (!history) {
+                gf_log (this->name, GF_LOG_ERROR, "event history cannot be "
+                        "initialized");
+                return -1;
+        }
+
+        this->history = history;
+
+        conf->trace_log_level = GF_LOG_INFO;
+
         if (dict_get (options, "force-log-level")) {
                 forced_loglevel = data_to_str (dict_get (options,
                                                          "force-log-level"));
@@ -2339,25 +3152,34 @@ init (xlator_t *this)
                         goto setloglevel;
 
                 if (strcmp (forced_loglevel, "INFO") == 0)
-                        trace_log_level = GF_LOG_INFO;
+                        conf->trace_log_level = GF_LOG_INFO;
                 else if (strcmp (forced_loglevel, "TRACE") == 0)
-                        trace_log_level = GF_LOG_TRACE;
+                        conf->trace_log_level = GF_LOG_TRACE;
                 else if (strcmp (forced_loglevel, "ERROR") == 0)
-                        trace_log_level = GF_LOG_ERROR;
+                        conf->trace_log_level = GF_LOG_ERROR;
                 else if (strcmp (forced_loglevel, "DEBUG") == 0)
-                        trace_log_level = GF_LOG_DEBUG;
+                        conf->trace_log_level = GF_LOG_DEBUG;
                 else if (strcmp (forced_loglevel, "WARNING") == 0)
-                        trace_log_level = GF_LOG_WARNING;
+                        conf->trace_log_level = GF_LOG_WARNING;
                 else if (strcmp (forced_loglevel, "CRITICAL") == 0)
-                        trace_log_level = GF_LOG_CRITICAL;
+                        conf->trace_log_level = GF_LOG_CRITICAL;
                 else if (strcmp (forced_loglevel, "NONE") == 0)
-                        trace_log_level = GF_LOG_NONE;
+                        conf->trace_log_level = GF_LOG_NONE;
         }
 
 setloglevel:
-        gf_log_set_loglevel (trace_log_level);
+        gf_log_set_loglevel (conf->trace_log_level);
+        this->private = conf;
+        ret = 0;
+out:
+        if (ret == -1) {
+                if (history)
+                        GF_FREE (history);
+                if (conf)
+                        GF_FREE (conf);
+        }
 
-        return 0;
+        return ret;
 }
 
 void
@@ -2366,6 +3188,9 @@ fini (xlator_t *this)
         if (!this)
                 return;
 
+        if (this->history)
+                eh_destroy (this->history);
+
         gf_log (this->name, GF_LOG_INFO,
                 "trace translator unloaded");
         return;
@@ -2414,7 +3239,6 @@ struct xlator_fops fops = {
         .fsetattr    = trace_fsetattr,
 };
 
-
 struct xlator_cbks cbks = {
         .release     = trace_release,
         .releasedir  = trace_releasedir,
@@ -2430,5 +3254,21 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_STR
           /*.value = { ""} */
         },
+        { .key  = {"history-size"},
+          .type = GF_OPTION_TYPE_SIZET,
+          .default_value = "1024",
+        },
+        { .key  = {"log-file"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+        },
+        { .key  = {"log-history"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+        },
         { .key  = {NULL} },
 };
+
+struct xlator_dumpops dumpops = {
+        .history = trace_dump_history
+};
diff --git a/xlators/debug/trace/src/trace.h b/xlators/debug/trace/src/trace.h
new file mode 100644
index 000000000..045eefb36
--- /dev/null
+++ b/xlators/debug/trace/src/trace.h
@@ -0,0 +1,98 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <time.h>
+#include <errno.h>
+#include "glusterfs.h"
+#include "xlator.h"
+#include "common-utils.h"
+#include "event-history.h"
+#include "logging.h"
+#include "circ-buff.h"
+#include "statedump.h"
+#include "options.h"
+
+#define TRACE_DEFAULT_HISTORY_SIZE 1024
+
+typedef struct {
+        /* Since the longest fop name is fremovexattr i.e 12 characters, array size
+         * is kept 24, i.e double of the maximum.
+         */
+        char name[24];
+        int enabled;
+} trace_fop_name_t;
+
+trace_fop_name_t trace_fop_names[GF_FOP_MAXVALUE];
+
+typedef struct {
+        gf_boolean_t log_file;
+        gf_boolean_t log_history;
+        size_t       history_size;
+        int trace_log_level;
+} trace_conf_t;
+
+#define TRACE_STACK_UNWIND(op, frame, params ...)                       \
+        do {                                                            \
+                frame->local = NULL;                                    \
+                STACK_UNWIND_STRICT (op, frame, params);                \
+        } while (0);
+
+#define LOG_ELEMENT(_conf, _string)                                     \
+        do {                                                            \
+                if (_conf) {                                            \
+                        if ((_conf->log_history) == _gf_true)           \
+                                gf_log_eh ("%s", _string);              \
+                        if ((_conf->log_file) == _gf_true)              \
+                                gf_log (THIS->name, _conf->trace_log_level, \
+                                        "%s", _string);                 \
+                }                                                       \
+        } while (0);
+
+#define trace_stat_to_str(buf, statstr)                                 \
+        do {                                                            \
+                char     atime_buf[256]    = {0,};                      \
+                char     mtime_buf[256]    = {0,};                      \
+                char     ctime_buf[256]    = {0,};                      \
+                uint64_t ia_time           = 0;                         \
+                                                                        \
+                if (!buf)                                               \
+                        break;                                          \
+                                                                        \
+                ia_time = buf->ia_atime;                                \
+                strftime (atime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                ia_time = buf->ia_mtime;                                \
+                strftime (mtime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                ia_time = buf->ia_ctime;                                \
+                strftime (ctime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                snprintf (statstr, sizeof (statstr),                    \
+                          "gfid=%s ino=%"PRIu64", mode=%o, "            \
+                          "nlink=%"GF_PRI_NLINK", uid=%u, "             \
+                          "gid=%u, size=%"PRIu64", "                    \
+                          "blocks=%"PRIu64", atime=%s, "                \
+                          "mtime=%s, ctime=%s",                         \
+                          uuid_utoa (buf->ia_gfid), buf->ia_ino,        \
+                          st_mode_from_ia (buf->ia_prot,                \
+                                           buf->ia_type),               \
+                          buf->ia_nlink, buf->ia_uid,                   \
+                          buf->ia_gid, buf->ia_size,                    \
+                          buf->ia_blocks, atime_buf,                    \
+                          mtime_buf, ctime_buf);                        \
+        } while (0);
diff --git a/xlators/encryption/Makefile.am b/xlators/encryption/Makefile.am
index 2cbde680f..36efc6698 100644
--- a/xlators/encryption/Makefile.am
+++ b/xlators/encryption/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = rot-13
+SUBDIRS = rot-13 crypt
 
 CLEANFILES = 
diff --git a/libglusterfsclient/Makefile.am b/xlators/encryption/crypt/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/libglusterfsclient/Makefile.am
+++ b/xlators/encryption/crypt/Makefile.am
diff --git a/xlators/encryption/crypt/src/Makefile.am b/xlators/encryption/crypt/src/Makefile.am
new file mode 100644
index 000000000..faadd117f
--- /dev/null
+++ b/xlators/encryption/crypt/src/Makefile.am
@@ -0,0 +1,24 @@
+if ENABLE_CRYPT_XLATOR
+
+xlator_LTLIBRARIES = crypt.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
+
+crypt_la_LDFLAGS = -module -avoidversion -lssl -lcrypto
+
+crypt_la_SOURCES = keys.c data.c metadata.c atom.c crypt.c
+crypt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
+else
+
+noinst_DIST = keys.c data.c metadata.c atom.c crypt.c
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+endif
+\ No newline at end of file
diff --git a/xlators/encryption/crypt/src/atom.c b/xlators/encryption/crypt/src/atom.c
new file mode 100644
index 000000000..1ec41495c
--- /dev/null
+++ b/xlators/encryption/crypt/src/atom.c
@@ -0,0 +1,962 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/*
+ *                              Glossary
+ *
+ *
+ * cblock            (or cipher block). A logical unit in a file.
+ *                   cblock size is defined as the number of bits
+ *                   in an input (or output) block of the block
+ *                   cipher (*). Cipher block size is a property of
+ *                   cipher algorithm. E.g. cblock size is 64 bits
+ *                   for DES, 128 bits for AES, etc.
+ *
+ * atomic cipher     A cipher algorithm, which requires some chunks of
+ * algorithm         text to be padded at left and(or) right sides before
+ *                   cipher transaform.
+ *
+ *
+ * block (atom)      Minimal chunk of file's data, which doesn't require
+ *                   padding. We'll consider logical units in a file of
+ *                   block size (atom size).
+ *
+ * cipher algorithm  Atomic cipher algorithm, which requires the last
+ * with EOF issue    incomplete cblock in a file to be padded with some
+ *                   data (usually zeros).
+ *
+ *
+ * operation, which  reading/writing from offset, which is not aligned to
+ * forms a gap at    to atom size
+ * the beginning
+ *
+ *
+ * operation, which  reading/writing count bytes starting from offset off,
+ * forms a gap at    so that off+count is not aligned to atom_size
+ * the end
+ *
+ * head block        the first atom affected by an operation, which forms
+ *                   a gap at the beginning, or(and) at the end.
+ *                   Сomment. Head block has at least one gap (either at
+ *                   the beginning, or at the end)
+ *
+ *
+ * tail block        the last atom different from head, affected by an
+ *                   operation, which forms a gap at the end.
+ *                   Сomment: Tail block has exactly one gap (at the end).
+ *
+ *
+ * partial block     head or tail block
+ *
+ *
+ * full block        block without gaps.
+ *
+ *
+ * (*) Recommendation for Block Cipher Modes of Operation
+ *     Methods and Techniques
+ *     NIST Special Publication 800-38A Edition 2001
+ */
+
+/*
+ * atom->offset_at()
+ */
+static off_t offset_at_head(struct avec_config *conf)
+{
+	return conf->aligned_offset;
+}
+
+static off_t offset_at_hole_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_hole_conf(frame));
+}
+
+static off_t offset_at_data_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_data_conf(frame));
+}
+
+
+static off_t offset_at_tail(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0) +
+		(conf->nr_full_blocks << get_atom_bits(object));
+}
+
+static off_t offset_at_hole_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_hole_conf(frame), object);
+}
+
+
+static off_t offset_at_data_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_data_conf(frame), object);
+}
+
+static off_t offset_at_full(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0);
+}
+
+static off_t offset_at_data_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_data_conf(frame), object);
+}
+
+static off_t offset_at_hole_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->io_size_nopad()
+ */
+
+static uint32_t io_size_nopad_head(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	uint32_t gap_at_beg;
+	uint32_t gap_at_end;
+
+	check_head_block(conf);
+
+	gap_at_beg = conf->off_in_head;
+
+	if (has_tail_block(conf) || has_full_blocks(conf) || conf->off_in_tail == 0 )
+		gap_at_end = 0;
+	else
+		gap_at_end = get_atom_size(object) - conf->off_in_tail;
+
+	return get_atom_size(object) - (gap_at_beg + gap_at_end);
+}
+
+static uint32_t io_size_nopad_tail(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	check_tail_block(conf);
+	return conf->off_in_tail;
+}
+
+static uint32_t io_size_nopad_full(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{	
+	check_full_block(conf);
+	return get_atom_size(object);
+}
+
+static uint32_t io_size_nopad_data_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_hole_conf(frame), object);
+}
+
+static uint32_t offset_in_head(struct avec_config *conf)
+{
+	check_cursor_head(conf);
+
+	return conf->off_in_head;
+}
+
+static uint32_t offset_in_tail(call_frame_t *frame,
+			       struct object_cipher_info *object)
+{
+	return 0;
+}
+
+static uint32_t offset_in_full(struct avec_config *conf,
+			       struct object_cipher_info *object)
+{
+	check_cursor_full(conf);
+
+	if (has_head_block(conf))
+		return (conf->cursor - 1) << get_atom_bits(object);
+	else
+		return conf->cursor << get_atom_bits(object);
+}
+
+static uint32_t offset_in_data_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_data_conf(frame));
+}
+
+static uint32_t offset_in_hole_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_hole_conf(frame));
+}
+
+static uint32_t offset_in_data_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_data_conf(frame), object);
+}
+
+static uint32_t offset_in_hole_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->rmw()
+ */
+/*
+ * Pre-conditions:
+ * @vec contains plain text of the latest
+ * version.
+ *
+ * Uptodate gaps of the @partial block with
+ * this plain text, encrypt the whole block
+ * and write the result to disk.
+ */
+static int32_t rmw_partial_block(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 struct iovec *vec,
+				 int32_t count,
+				 struct iatt *stbuf,
+				 struct iobref *iobref,
+				 struct rmw_atom *atom)
+{
+	size_t was_read = 0;
+	uint64_t file_size;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *partial = atom->get_iovec(frame, 0);
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_partial_block;
+#if DEBUG_CRYPT
+	gf_boolean_t check_last_cblock = _gf_false;
+#endif
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto exit;
+
+	file_size = local->cur_file_size;
+	was_read = op_ret;
+
+	if (atom->locality == HEAD_ATOM && conf->off_in_head) {
+		/*
+		 * head atom with a non-uptodate gap
+		 * at the beginning
+		 *
+		 * fill the gap with plain text of the
+		 * latest version. Convert a part of hole
+		 * (if any) to zeros.
+		 */
+		int32_t i;
+		int32_t copied = 0;
+		int32_t to_gap; /* amount of data needed to uptodate
+				   the gap at the beginning */
+#if 0
+		int32_t hole = 0; /* The part of the hole which
+				   * got in the head block */
+#endif /* 0 */
+		to_gap = conf->off_in_head;
+
+		if (was_read < to_gap) {
+			if (file_size >
+			    offset_at_head(conf) + was_read) {
+				/*
+				 * It is impossible to uptodate
+				 * head block: too few bytes have
+				 * been read from disk, so that
+				 * partial write is impossible.
+				 *
+				 * It could happen because of many
+				 * reasons: IO errors, (meta)data
+				 * corruption in the local file system,
+				 * etc.
+				 */
+				gf_log(this->name, GF_LOG_WARNING,
+				     "Can not uptodate a gap at the beginning");
+				local->op_ret = -1;
+				local->op_errno = EIO;
+				goto exit;
+			}
+#if 0
+			hole = to_gap - was_read;
+#endif /* 0 */
+			to_gap = was_read;
+		}
+		/*
+		 * uptodate the gap at the beginning
+		 */
+		for (i = 0; i < count && copied < to_gap; i++) {
+			int32_t to_copy;
+
+			to_copy = vec[i].iov_len;
+			if (to_copy > to_gap - copied)
+				to_copy = to_gap - copied;
+
+			memcpy(partial->iov_base, vec[i].iov_base, to_copy);
+			copied += to_copy;
+		}
+#if 0
+		/*
+		 * If possible, convert part of the
+		 * hole, which got in the head block
+		 */
+		ret = TRY_LOCK(&local->hole_lock);
+		if (!ret) {
+			if (local->hole_handled)
+				/*
+				 * already converted by
+				 * crypt_writev_cbk()
+				 */
+				UNLOCK(&local->hole_lock);
+			else {
+				/*
+				 * convert the part of the hole
+				 * which got in the head block
+				 * to zeros.
+				 *
+				 * Update the orig_offset to make
+				 * sure writev_cbk() won't care
+				 * about this part of the hole.
+				 *
+				 */
+				memset(partial->iov_base + to_gap, 0, hole);
+
+				conf->orig_offset -= hole;
+				conf->orig_size += hole;
+				UNLOCK(&local->hole_lock);
+			}
+		}
+		else /*
+		      * conversion is being performed
+		      * by crypt_writev_cbk()
+		      */
+			;
+#endif /* 0 */
+	}
+	if (atom->locality == TAIL_ATOM ||
+	    (!has_tail_block(conf) && conf->off_in_tail)) {
+		/*
+		 * tail atom, or head atom with a non-uptodate
+		 * gap at the end.
+		 *
+		 * fill the gap at the end of the block
+		 * with plain text of the latest version.
+		 * Pad the result, (if needed)
+		 */
+		int32_t i;
+		int32_t to_gap;
+		int copied;
+		off_t off_in_tail;
+		int32_t to_copy;
+
+		off_in_tail = conf->off_in_tail;
+		to_gap = conf->gap_in_tail;
+
+		if (to_gap && was_read < off_in_tail + to_gap) {
+			/*
+			 * It is impossible to uptodate
+			 * the gap at the end: too few bytes
+			 * have been read from disk, so that
+			 * partial write is impossible.
+			 *
+			 * It could happen because of many
+			 * reasons: IO errors, (meta)data
+			 * corruption in the local file system,
+			 * etc.
+			 */
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Can not uptodate a gap at the end");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * uptodate the gap at the end
+		 */
+		copied = 0;
+		to_copy = to_gap;
+		for(i = count - 1; i >= 0 && to_copy > 0; i--) {
+			uint32_t from_vec, off_in_vec;
+
+			off_in_vec = 0;
+			from_vec = vec[i].iov_len;
+			if (from_vec > to_copy) {
+				off_in_vec = from_vec - to_copy;
+				from_vec = to_copy;
+			}
+			memcpy(partial->iov_base +
+			       off_in_tail + to_gap - copied - from_vec,
+			       vec[i].iov_base + off_in_vec,
+			       from_vec);
+
+			gf_log(this->name, GF_LOG_DEBUG,
+			       "uptodate %d bytes at tail. Offset at target(source): %d(%d)",
+			       (int)from_vec,
+			       (int)off_in_tail + to_gap - copied - from_vec,
+			       (int)off_in_vec);
+
+			copied += from_vec;
+			to_copy -= from_vec;
+		}
+		partial->iov_len = off_in_tail + to_gap;
+
+		if (object_alg_should_pad(object)) {
+			int32_t resid = 0;
+			resid = partial->iov_len & (object_alg_blksize(object) - 1);
+			if (resid) {
+				/*
+				 * append a new EOF padding
+				 */
+				local->eof_padding_size =
+					object_alg_blksize(object) - resid;
+
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "set padding size %d",
+				       local->eof_padding_size);
+
+				memset(partial->iov_base + partial->iov_len,
+				       1,
+				       local->eof_padding_size);
+				partial->iov_len += local->eof_padding_size;
+#if DEBUG_CRYPT
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "pad cblock with %d zeros:",
+				       local->eof_padding_size);
+				dump_cblock(this,
+					    (unsigned char *)partial->iov_base +
+					    partial->iov_len - object_alg_blksize(object));
+				check_last_cblock = _gf_true;
+#endif
+			}
+		}
+	}
+	/*
+	 * encrypt the whole block
+	 */
+	encrypt_aligned_iov(object,
+			    partial,
+			    1,
+			    atom->offset_at(frame, object));
+#if DEBUG_CRYPT
+	if (check_last_cblock == _gf_true) {
+		gf_log(this->name, GF_LOG_DEBUG,
+		       "encrypt last cblock with offset %llu",
+		       (unsigned long long)atom->offset_at(frame, object));
+		dump_cblock(this, (unsigned char *)partial->iov_base +
+			    partial->iov_len - object_alg_blksize(object));
+	}
+#endif
+	set_local_io_params_writev(frame, object, atom,
+				   atom->offset_at(frame, object),
+				   iovec_get_size(partial, 1));
+	/*
+	 * write the whole block to disk
+	 */
+	end_writeback_partial_block = dispatch_end_writeback(local->fop);
+	conf->cursor ++;
+	STACK_WIND(frame,
+		   end_writeback_partial_block,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->writev,
+		   local->fd,
+		   partial,
+		   1,
+		   atom->offset_at(frame, object),
+		   local->flags,
+		   local->iobref_data,
+		   local->xdata);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "submit partial block: %d bytes from %d offset",
+	       (int)iovec_get_size(partial, 1),
+	       (int)atom->offset_at(frame, object));
+ exit:
+	return 0;
+}
+
+/*
+ * Perform a (read-)modify-write sequence.
+ * This should be performed only after approval
+ * of upper server-side manager, i.e. the caller
+ * needs to make sure this is his turn to rmw.
+ */
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype)
+{
+	int32_t ret;
+	dict_t *dict;
+	struct rmw_atom *atom;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	atom = atom_by_types(local->active_setup, ltype);
+	/*
+	 * To perform the "read" component of the read-modify-write
+	 * sequence the crypt translator does stack_wind to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 */
+	dict = dict_new();
+	if (!dict) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		return;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		dict_unref(dict);
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   atom->rmw,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   fd,
+		   atom->count_to_uptodate(frame, object), /* count */
+		   atom->offset_at(frame, object), /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	dict_unref(dict);
+}
+
+/*
+ * submit blocks of FULL_ATOM type
+ */
+void submit_full(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	struct rmw_atom *atom = atom_by_types(local->active_setup, FULL_ATOM);
+	uint32_t count; /* total number of full blocks to submit */
+	uint32_t granularity; /* number of blocks to submit in one iteration */
+
+	uint64_t off_in_file; /* start offset in the file, bytes */
+	uint32_t off_in_atom; /* start offset in the atom, blocks */
+	uint32_t blocks_written = 0; /* blocks written for this submit */
+
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_full_block;
+	/*
+	 * Write full blocks by groups of granularity size.
+	 */
+	end_writeback_full_block = dispatch_end_writeback(local->fop);
+
+	if (is_ordered_mode(frame)) {
+		uint32_t skip = has_head_block(conf) ? 1 : 0;
+		count = 1;
+		granularity = 1;
+		/*
+		 * calculate start offset using cursor value;
+		 * here we should take into accout head block,
+		 * which corresponds to cursor value 0.
+		 */
+		off_in_file = atom->offset_at(frame, object) +
+			((conf->cursor - skip) << get_atom_bits(object));
+		off_in_atom = conf->cursor - skip;
+	}
+	else {
+		/*
+		 * in parallel mode
+		 */
+		count = conf->nr_full_blocks;
+		granularity = MAX_IOVEC;
+		off_in_file = atom->offset_at(frame, object);
+		off_in_atom = 0;
+	}
+	while (count) {
+		uint32_t blocks_to_write = count;
+
+		if (blocks_to_write > granularity)
+			blocks_to_write = granularity;
+		if (conf->type == HOLE_ATOM)
+			/*
+			 * reset iovec before encryption
+			 */
+			memset(atom->get_iovec(frame, 0)->iov_base,
+			       0,
+			       get_atom_size(object));
+		/*
+		 * encrypt the group
+		 */
+		encrypt_aligned_iov(object,
+				    atom->get_iovec(frame,
+						    off_in_atom +
+						    blocks_written),
+				    blocks_to_write,
+				    off_in_file + (blocks_written <<
+						   get_atom_bits(object)));
+		
+		set_local_io_params_writev(frame, object, atom,
+		        off_in_file + (blocks_written << get_atom_bits(object)),
+			blocks_to_write <<  get_atom_bits(object));
+
+		conf->cursor += blocks_to_write;
+
+		STACK_WIND(frame,
+			   end_writeback_full_block,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev,
+			   local->fd,
+			   atom->get_iovec(frame, off_in_atom + blocks_written),
+			   blocks_to_write,
+			   off_in_file + (blocks_written << get_atom_bits(object)),
+			   local->flags,
+			   local->iobref_data ? local->iobref_data : local->iobref,
+			   local->xdata);
+
+		gf_log("crypt", GF_LOG_DEBUG, "submit %d full blocks from %d offset",
+		       blocks_to_write,
+		       (int)(off_in_file + (blocks_written << get_atom_bits(object))));
+
+		count -= blocks_to_write;
+		blocks_written += blocks_to_write;
+	}
+	return;
+}
+
+static int32_t rmw_data_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_data_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, TAIL_ATOM));
+}
+
+static int32_t rmw_hole_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_hole_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, TAIL_ATOM));
+}
+
+/*
+ * atom->count_to_uptodate()
+ */
+static uint32_t count_to_uptodate_head(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	if (conf->acount == 1 && conf->off_in_tail)
+		return get_atom_size(object);
+	else
+		/* there is no need to read the whole head block */
+		return conf->off_in_head;
+}
+
+static uint32_t count_to_uptodate_tail(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	/* we need to read the whole tail block */
+	return get_atom_size(object);
+}
+
+static uint32_t count_to_uptodate_data_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_data_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_hole_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_hole_conf(frame), object);
+}
+
+/* atom->get_config() */
+
+static struct avec_config *get_config_data(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->data_conf;
+}
+
+static struct avec_config *get_config_hole(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->hole_conf;
+}
+
+/*
+ * atom->get_iovec()
+ */
+static struct iovec *get_iovec_hole_head(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec;
+}
+
+static struct iovec *get_iovec_hole_full(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0);
+}
+
+static inline struct iovec *get_iovec_hole_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->blocks_in_pool - 1);
+}
+
+static inline struct iovec *get_iovec_data_head(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec;
+}
+
+static inline struct iovec *get_iovec_data_full(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0) + count;
+}
+
+static inline struct iovec *get_iovec_data_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec +
+		(conf->off_in_head ? 1 : 0) +
+		conf->nr_full_blocks;
+}
+
+static struct rmw_atom atoms[LAST_DATA_TYPE][LAST_LOCALITY_TYPE] = {
+	[DATA_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_data_head,
+	  .offset_at = offset_at_data_head,
+	  .offset_in = offset_in_data_head,
+	  .get_iovec = get_iovec_data_head,
+	  .io_size_nopad = io_size_nopad_data_head,
+	  .count_to_uptodate = count_to_uptodate_data_head,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_data_tail,
+	  .offset_at = offset_at_data_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_data_tail,
+	  .io_size_nopad = io_size_nopad_data_tail,
+	  .count_to_uptodate = count_to_uptodate_data_tail,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_data_full,
+	  .offset_in = offset_in_data_full,
+	  .get_iovec = get_iovec_data_full,
+	  .io_size_nopad = io_size_nopad_data_full,
+	  .get_config = get_config_data
+	},
+	[HOLE_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_hole_head,
+	  .offset_at = offset_at_hole_head,
+	  .offset_in = offset_in_hole_head,
+	  .get_iovec = get_iovec_hole_head,
+	  .io_size_nopad = io_size_nopad_hole_head,
+	  .count_to_uptodate = count_to_uptodate_hole_head,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_hole_tail,
+	  .offset_at = offset_at_hole_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_hole_tail,
+	  .io_size_nopad = io_size_nopad_hole_tail,
+	  .count_to_uptodate = count_to_uptodate_hole_tail,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_hole_full,
+	  .offset_in = offset_in_hole_full,
+	  .get_iovec = get_iovec_hole_full,
+	  .io_size_nopad = io_size_nopad_hole_full,
+	  .get_config = get_config_hole
+	}
+};
+
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality)
+{
+	return &atoms[data][locality];
+}
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-common.h b/xlators/encryption/crypt/src/crypt-common.h
new file mode 100644
index 000000000..7c212ad5d
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-common.h
@@ -0,0 +1,141 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_COMMON_H__
+#define __CRYPT_COMMON_H__
+
+#define INVAL_SUBVERSION_NUMBER (0xff)
+#define CRYPT_INVAL_OP   (GF_FOP_NULL)
+
+#define CRYPTO_FORMAT_PREFIX  "trusted.glusterfs.crypt.att.cfmt"
+#define FSIZE_XATTR_PREFIX    "trusted.glusterfs.crypt.att.size"
+#define SUBREQ_PREFIX         "trusted.glusterfs.crypt.msg.sreq"
+#define FSIZE_MSG_PREFIX      "trusted.glusterfs.crypt.msg.size"
+#define DE_MSG_PREFIX         "trusted.glusterfs.crypt.msg.dent"
+#define REQUEST_ID_PREFIX     "trusted.glusterfs.crypt.msg.rqid"
+#define MSGFLAGS_PREFIX       "trusted.glusterfs.crypt.msg.xfgs"
+
+
+/* messages for crypt_open() */
+#define MSGFLAGS_REQUEST_MTD_RLOCK   1 /* take read lock and don't unlock */
+#define MSGFLAGS_REQUEST_MTD_WLOCK   2 /* take write lock and don't unlock */
+
+#define AES_BLOCK_BITS (4) /* AES_BLOCK_SIZE == 1 << AES_BLOCK_BITS */
+
+#define noop   do {; } while (0)
+#define cassert(cond) ({ switch (-1) { case (cond): case 0: break; } })
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+
+/*
+ * Format of file's metadata
+ */
+struct crypt_format {
+	uint8_t loader_id;  /* version of metadata loader */
+	uint8_t versioned[0]; /* file's metadata of specific version */
+} __attribute__((packed));
+
+typedef enum {
+	AES_CIPHER_ALG,
+	LAST_CIPHER_ALG
+} cipher_alg_t;
+
+typedef enum {
+	XTS_CIPHER_MODE,
+	LAST_CIPHER_MODE
+} cipher_mode_t;
+
+typedef enum {
+	MTD_LOADER_V1,
+	LAST_MTD_LOADER
+} mtd_loader_id;
+
+static inline void msgflags_set_mtd_rlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline void msgflags_set_mtd_wlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_rlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_wlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_lock(uint32_t *flags)
+{
+	return  msgflags_check_mtd_rlock(flags) ||
+		msgflags_check_mtd_wlock(flags);
+}
+
+/*
+ * returns number of logical blocks occupied
+ * (maybe partially) by @count bytes
+ * at offset @start.
+ */
+static inline off_t logical_blocks_occupied(uint64_t start, off_t count,
+					    int blkbits)
+{
+	return ((start + count - 1) >> blkbits) - (start >> blkbits) + 1;
+}
+
+/*
+ * are two bytes (represented by offsets @off1
+ * and @off2 respectively) in the same logical
+ * block.
+ */
+static inline int in_same_lblock(uint64_t off1, uint64_t off2,
+				 int blkbits)
+{
+	return off1 >> blkbits == off2 >> blkbits;
+}
+
+static inline void dump_cblock(xlator_t *this, unsigned char *buf)
+{
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "dump cblock: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
+	       (buf)[0],
+	       (buf)[1],
+	       (buf)[2],
+	       (buf)[3],
+	       (buf)[4],
+	       (buf)[5],
+	       (buf)[6],
+	       (buf)[7],
+	       (buf)[8],
+	       (buf)[9],
+	       (buf)[10],
+	       (buf)[11],
+	       (buf)[12],
+	       (buf)[13],
+	       (buf)[14],
+	       (buf)[15]);
+}
+
+#endif /* __CRYPT_COMMON_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-mem-types.h b/xlators/encryption/crypt/src/crypt-mem-types.h
new file mode 100644
index 000000000..799727573
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-mem-types.h
@@ -0,0 +1,43 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __CRYPT_MEM_TYPES_H__
+#define __CRYPT_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_crypt_mem_types_ {
+	gf_crypt_mt_priv = gf_common_mt_end + 1,
+	gf_crypt_mt_inode,
+	gf_crypt_mt_data,
+	gf_crypt_mt_mtd,
+	gf_crypt_mt_loc,
+	gf_crypt_mt_iatt,
+	gf_crypt_mt_key,
+	gf_crypt_mt_iovec,
+	gf_crypt_mt_char,
+};
+
+#endif /* __CRYPT_MEM_TYPES_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
+
+
+
diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c
new file mode 100644
index 000000000..db2e6d83c
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.c
@@ -0,0 +1,4498 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#include <ctype.h>
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd);
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master);
+static int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				       uint64_t from, off_t size);
+static int32_t load_file_size(call_frame_t *frame, void *cookie,
+			      xlator_t *this, int32_t op_ret, int32_t op_errno,
+			      dict_t *dict, dict_t *xdata);
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype);
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype);
+static void put_one_call_open(call_frame_t *frame);
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this);
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this);
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this);
+static void free_avec(struct iovec *avec, char **pool, int blocks_in_pool);
+static void free_avec_data(crypt_local_t *local);
+static void free_avec_hole(crypt_local_t *local);
+
+static crypt_local_t *crypt_alloc_local(call_frame_t *frame, xlator_t *this,
+					glusterfs_fop_t fop)
+{
+	crypt_local_t *local = NULL;
+
+	local = mem_get0(this->local_pool);
+	if (!local) {
+		gf_log(this->name, GF_LOG_ERROR, "out of memory");
+		return NULL;
+	}
+	local->fop = fop;
+	LOCK_INIT(&local->hole_lock);
+	LOCK_INIT(&local->call_lock);
+	LOCK_INIT(&local->rw_count_lock);
+
+	frame->local = local;
+	return local;
+}
+
+struct crypt_inode_info *get_crypt_inode_info(inode_t *inode, xlator_t *this)
+{
+	int ret;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+
+	ret = inode_ctx_get(inode, this, &value);
+	if (ret == -1) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not get inode info");
+		return NULL;
+	}
+	info = (struct crypt_inode_info *)(long)value;
+	if (info == NULL) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not obtain inode info");
+		return NULL;
+	}
+	return info;
+}
+
+static struct crypt_inode_info *local_get_inode_info(crypt_local_t *local,
+						      xlator_t *this)
+{
+	if (local->info)
+		return local->info;
+	local->info = get_crypt_inode_info(local->fd->inode, this);
+	return local->info;
+}
+
+static struct crypt_inode_info *alloc_inode_info(crypt_local_t *local,
+						 loc_t *loc)
+{
+	struct crypt_inode_info *info;
+
+	info = GF_CALLOC(1, sizeof(*info), gf_crypt_mt_inode);
+	if (!info) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		gf_log ("crypt", GF_LOG_WARNING,
+			"Can not allocate inode info");
+		return NULL;
+	}
+	memset(info, 0, sizeof(*info));
+#if DEBUG_CRYPT
+	info->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!info->loc) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not allocate loc");
+		GF_FREE(info);
+		return NULL;
+	}
+	if (loc_copy(info->loc, loc)){
+		GF_FREE(info->loc);
+		GF_FREE(info);
+		return NULL;
+	}
+#endif /* DEBUG_CRYPT */
+
+	local->info = info;
+	return info;
+}
+
+static void free_inode_info(struct crypt_inode_info *info)
+{
+#if DEBUG_CRYPT
+	loc_wipe(info->loc);
+	GF_FREE(info->loc);
+#endif
+	memset(info, 0, sizeof(*info));
+	GF_FREE(info);
+}
+
+int crypt_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t ctx_addr = 0;
+        if (!inode_ctx_del (inode, this, &ctx_addr))
+                free_inode_info((struct crypt_inode_info *)(long)ctx_addr);
+        return 0;
+}
+
+#if DEBUG_CRYPT
+static void check_read(call_frame_t *frame, xlator_t *this, int32_t read,
+		       struct iovec *vec, int32_t count, struct iatt *stbuf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = get_object_cinfo(local->info);
+	struct avec_config *conf = &local->data_conf;
+	uint32_t resid = stbuf->ia_size & (object_alg_blksize(object) - 1);
+
+	if (read <= 0)
+		return;
+	if (read != iovec_get_size(vec, count))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"op_ret differs from amount of read bytes");
+
+	if (object_alg_should_pad(object) && (read & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (!= 0 mod(cblock size))");
+
+	if (conf->aligned_offset + read >
+	    stbuf->ia_size + (resid ? object_alg_blksize(object) - resid : 0))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (too large))");
+
+}
+
+#define PT_BYTES_TO_DUMP (32)
+static void dump_plain_text(crypt_local_t *local, struct iovec *avec)
+{
+	int32_t to_dump;
+	char str[PT_BYTES_TO_DUMP + 1];
+
+	if (!avec)
+		return;
+	to_dump = avec->iov_len;
+	if (to_dump > PT_BYTES_TO_DUMP)
+		to_dump = PT_BYTES_TO_DUMP;
+	memcpy(str, avec->iov_base, to_dump);
+	memset(str + to_dump, '0', 1);
+	gf_log("crypt", GF_LOG_DEBUG, "Read file: %s", str);
+}
+
+static int32_t data_conf_invariant(struct avec_config *conf)
+{
+	return conf->acount ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		conf->nr_full_blocks;
+}
+
+static int32_t hole_conf_invariant(struct avec_config *conf)
+{
+	return conf->blocks_in_pool ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		!!has_full_blocks(conf);
+}
+
+static void crypt_check_conf(struct avec_config *conf)
+{
+	int32_t ret = 0;
+	const char *msg;
+
+	switch (conf->type) {
+	case DATA_ATOM:
+		msg = "data";
+		ret = data_conf_invariant(conf);
+		break;
+	case HOLE_ATOM:
+		msg = "hole";
+		ret = hole_conf_invariant(conf);
+		break;
+	default:
+		msg = "unknown";
+	}
+	if (!ret)
+		gf_log("crypt", GF_LOG_DEBUG, "bad %s conf", msg);
+}
+
+static void check_buf(call_frame_t *frame, xlator_t *this, struct iatt *buf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	uint64_t local_file_size;
+
+	switch(local->fop) {
+	case GF_FOP_FTRUNCATE:
+		return;
+	case GF_FOP_WRITE:
+		local_file_size = local->new_file_size;
+		break;
+	case GF_FOP_READ:
+		if (parent_is_crypt_xlator(frame, this))
+			return;
+		local_file_size = local->cur_file_size;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad file operation");
+		return;
+	}
+	if (buf->ia_size != round_up(local_file_size,
+					 object_alg_blksize(object)))
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "bad ia_size in buf (%llu), should be %llu",
+		       (unsigned long long)buf->ia_size,
+		       (unsigned long long)round_up(local_file_size,
+						    object_alg_blksize(object)));
+}
+
+#else
+#define check_read(frame, this, op_ret, vec, count, stbuf) noop
+#define dump_plain_text(local, avec) noop
+#define crypt_check_conf(conf) noop
+#define check_buf(frame, this, buf) noop
+#endif /* DEBUG_CRYPT */
+
+/*
+ * Pre-conditions:
+ * @vec represents a ciphertext of expanded size and
+ * aligned offset.
+ *
+ * Compound a temporal vector @avec with block-aligned
+ * components, decrypt and fix it up to represent a chunk
+ * of data corresponding to the original size and offset.
+ * Pass the result to the next translator.
+ */
+int32_t crypt_readv_cbk(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			struct iovec *vec,
+			int32_t count,
+			struct iatt *stbuf,
+			struct iobref *iobref,
+			dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *avec;
+	uint32_t i;
+	uint32_t to_vec;
+	uint32_t to_user;
+
+	check_buf(frame, this, stbuf);
+	check_read(frame, this, op_ret, vec, count, stbuf);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->iobref = iobref_ref(iobref);
+
+	local->buf = *stbuf;
+	local->buf.ia_size = local->cur_file_size;
+
+	if (op_ret <= 0 || count == 0 || vec[0].iov_len == 0)
+		goto put_one_call;
+
+	if (conf->orig_offset >= local->cur_file_size) {
+		local->op_ret = 0;
+		goto put_one_call;
+	}
+	/* 
+	 * correct config params with real file size
+	 * and actual amount of bytes read
+	 */
+	set_config_offsets(frame, this,
+			   conf->orig_offset, op_ret, DATA_ATOM, 0);
+
+	if (conf->orig_offset + conf->orig_size > local->cur_file_size)
+		conf->orig_size = local->cur_file_size - conf->orig_offset;
+	/*
+	 * calculate amount of data to be returned
+	 * to user.
+	 */
+	to_user = op_ret;
+	if (conf->aligned_offset + to_user <= conf->orig_offset) {
+		gf_log(this->name, GF_LOG_WARNING, "Incomplete read");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	to_user -= (conf->aligned_offset - conf->orig_offset);
+
+	if (to_user > conf->orig_size)
+		to_user = conf->orig_size;
+	local->rw_count = to_user;
+
+	op_errno = set_config_avec_data(this, local,
+					conf, object, vec, count);
+	if (op_errno) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
+		goto put_one_call;
+	}
+	avec = conf->avec;
+#if DEBUG_CRYPT
+	if (conf->off_in_tail != 0 &&
+	    conf->off_in_tail < object_alg_blksize(object) &&
+	    object_alg_should_pad(object))
+		gf_log(this->name, GF_LOG_DEBUG, "Bad offset in tail %d",
+		       conf->off_in_tail);
+	if (iovec_get_size(vec, count) != 0 &&
+	    in_same_lblock(conf->orig_offset + iovec_get_size(vec, count) - 1,
+			   local->cur_file_size - 1,
+			   object_alg_blkbits(object))) {
+		gf_log(this->name, GF_LOG_DEBUG, "Compound last cblock");
+		dump_cblock(this,
+			    (unsigned char *)(avec[conf->acount - 1].iov_base) +
+			    avec[conf->acount - 1].iov_len - object_alg_blksize(object));
+		dump_cblock(this,
+			    (unsigned char *)(vec[count - 1].iov_base) +
+			    vec[count - 1].iov_len - object_alg_blksize(object));
+	}
+#endif
+	decrypt_aligned_iov(object, avec,
+			    conf->acount, conf->aligned_offset);
+	/*
+	 * pass proper plain data to user
+	 */
+	avec[0].iov_base += (conf->aligned_offset - conf->orig_offset);
+	avec[0].iov_len  -= (conf->aligned_offset - conf->orig_offset);
+
+	to_vec = to_user;
+	for (i = 0; i < conf->acount; i++) {
+		if (avec[i].iov_len > to_vec)
+			avec[i].iov_len = to_vec;
+		to_vec -= avec[i].iov_len;
+	}
+ put_one_call:
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t do_readv(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			dict_t *dict,
+			dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->cur_file_size = data_to_uint64(data);
+	
+	get_one_call(frame);
+	STACK_WIND(frame,
+		   crypt_readv_cbk,
+		   FIRST_CHILD (this),
+		   FIRST_CHILD (this)->fops->readv,
+		   local->fd,
+		   /*
+		    * FIXME: read amount can be reduced
+		    */
+		   local->data_conf.expanded_size,
+		   local->data_conf.aligned_offset,
+		   local->flags,
+		   local->xdata);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call(frame);
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t crypt_readv_finodelk_cbk(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size
+	 */
+	STACK_WIND(frame,
+		   do_readv,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return  0;
+ error:
+	fd_unref(local->fd);
+	if (local->xdata)
+		dict_unref(local->xdata);
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    op_errno,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t readv_trivial_completion(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					struct iatt *buf,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "stat failed (%d)", op_errno);
+		goto error;
+	}
+	local->buf = *buf;
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;	
+ error:
+	STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno,
+			    NULL, 0, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t crypt_readv(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    size_t size,
+		    off_t offset,
+		    uint32_t flags, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "reading %d bytes from offset %llu",
+	       (int)size, (long long)offset);
+	if (parent_is_crypt_xlator(frame, this))
+		gf_log("crypt", GF_LOG_DEBUG, "parent is crypt");
+#endif	
+	local = crypt_alloc_local(frame, this, GF_FOP_READ);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	if (size == 0)
+		goto trivial;
+
+	local->fd = fd_ref(fd);
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	set_config_offsets(frame, this, offset, size,
+			   DATA_ATOM, 0);
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * We are called by crypt_writev (or cypt_ftruncate)
+		 * to perform the "read" component of the read-modify-write
+		 * (or read-prune-write) sequence for some atom;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 *
+		 * Retrieve current file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		get_one_call(frame);
+		STACK_WIND(frame,
+			   crypt_readv_cbk,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->readv,
+			   local->fd,
+			   /*
+			    * FIXME: read amount can be reduced
+			    */
+			   local->data_conf.expanded_size,
+			   local->data_conf.aligned_offset,
+			   flags,
+			   NULL);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_readv_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   readv_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom,
+				off_t io_offset,
+				uint32_t io_size)
+{
+	crypt_local_t *local = frame->local;
+
+	local->io_offset = io_offset;
+	local->io_size = io_size;
+
+	local->io_offset_nopad =
+		atom->offset_at(frame, object) + atom->offset_in(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad offset to %llu",
+	       (unsigned long long)local->io_offset_nopad);
+
+	local->io_size_nopad = atom->io_size_nopad(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad size to %llu",
+	       (unsigned long long)local->io_size_nopad);
+
+	local->update_disk_file_size = 0;
+	/*
+	 * NOTE: eof_padding_size is 0 for all full atoms;
+	 * For head and tail atoms it will be set up at rmw_partial block()
+	 */
+	local->new_file_size = local->cur_file_size;
+
+	if (local->io_offset_nopad + local->io_size_nopad > local->cur_file_size) {
+
+		local->new_file_size = local->io_offset_nopad + local->io_size_nopad;
+
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "set new file size to %llu",
+		       (unsigned long long)local->new_file_size);
+
+		local->update_disk_file_size = 1;
+	}
+}
+
+void set_local_io_params_ftruncate(call_frame_t *frame,
+				   struct object_cipher_info *object)
+{
+	uint32_t resid;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	resid = conf->orig_offset & (object_alg_blksize(object) - 1);
+	if (resid) {
+		local->eof_padding_size =
+			object_alg_blksize(object) - resid;
+		local->new_file_size = conf->aligned_offset;
+		local->update_disk_file_size = 0;
+		/*
+		 * file size will be updated
+		 * in the ->writev() stack,
+		 * when submitting file tail
+		 */
+	}
+	else {	
+		local->eof_padding_size = 0;
+		local->new_file_size = conf->orig_offset;
+		local->update_disk_file_size = 1;
+		/*
+		 * file size will be updated
+		 * in this ->ftruncate stack
+		 */
+	}
+}
+
+static inline void submit_head(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, HEAD_ATOM);
+}
+
+static inline void submit_tail(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, TAIL_ATOM);
+}
+
+static void submit_hole(call_frame_t *frame, xlator_t *this)
+{
+	/*
+	 * hole conversion always means
+	 * appended write and goes in ordered fashion
+	 */
+	do_ordered_submit(frame, this, HOLE_ATOM);
+}
+
+static void submit_data(call_frame_t *frame, xlator_t *this)
+{
+	if (is_ordered_mode(frame)) {
+		do_ordered_submit(frame, this, DATA_ATOM);
+		return;
+	}
+	gf_log("crypt", GF_LOG_WARNING, "Bad submit mode");
+	get_nr_calls(frame, nr_calls_data(frame));
+	do_parallel_submit(frame, this, DATA_ATOM);
+	return;
+}
+
+/*
+ * heplers called by writev_cbk, fruncate_cbk in ordered mode
+ */
+
+static inline int32_t should_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	return conf->avec != NULL;
+}
+
+static inline int32_t should_resume_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	if (local->fop == GF_FOP_WRITE && has_tail_block(conf))
+		/*
+		 * Don't submit a part of hole, which
+		 * fits into a data block:
+ 		 * this part of hole will be converted
+		 * as a gap filled by zeros in data head
+		 * block.
+		 */
+		return conf->cursor < conf->acount - 1;
+	else
+		return conf->cursor < conf->acount;
+}
+
+static inline int32_t should_resume_submit_data(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	if (is_ordered_mode(frame))
+		return conf->cursor < conf->acount;
+	/*
+	 * parallel writes
+	 */
+	return 0;
+}
+
+static inline int32_t should_submit_data_after_hole(crypt_local_t *local)
+{
+	return local->data_conf.avec != NULL;
+}
+
+static void update_local_file_params(call_frame_t *frame,
+				     xlator_t *this,
+				     struct iatt *prebuf,
+				     struct iatt *postbuf)
+{
+	crypt_local_t *local = frame->local;
+
+	check_buf(frame, this, postbuf);
+
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	local->prebuf.ia_size  = local->cur_file_size;
+	local->postbuf.ia_size = local->new_file_size;
+
+	local->cur_file_size = local->new_file_size;	
+}
+
+static int32_t end_writeback_writev(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    struct iatt *prebuf,
+				    struct iatt *postbuf,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret <= 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "writev iteration failed");
+		goto put_one_call;
+	}
+	/*
+	 * op_ret includes paddings (atom's head, atom's tail and EOF)
+	 */
+	if (op_ret < local->io_size) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Incomplete writev iteration");
+		goto put_one_call;
+	}
+	op_ret -= local->eof_padding_size;
+	local->op_ret = op_ret;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local)) {
+
+		LOCK(&local->rw_count_lock);
+		local->rw_count += op_ret;
+		UNLOCK(&local->rw_count_lock);
+
+		if (should_resume_submit_data(frame))
+			submit_data(frame, this);
+	}
+	else {
+		/*
+		 * hole conversion is going on;
+		 * don't take into account written zeros
+		 */
+		if (should_resume_submit_hole(local))
+			submit_hole(frame, this);
+
+		else if (should_submit_data_after_hole(local))
+			submit_data(frame, this);
+	}
+ put_one_call:
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+#define crypt_writev_cbk end_writeback_writev
+
+#define HOLE_WRITE_CHUNK_BITS 12
+#define HOLE_WRITE_CHUNK_SIZE (1 << HOLE_WRITE_CHUNK_BITS)
+
+/*
+ * Convert hole of size @size at offset @off to
+ * zeros and prepare respective iovecs for submit.
+ * The hole lock should be held.
+ *
+ * Pre-conditions:
+ * @local->file_size is set and valid.
+ */
+int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				uint64_t off, off_t size)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, off, size, HOLE_ATOM, 1);
+
+	ret = set_config_avec_hole(this, local,
+				   &local->hole_conf, object, local->fop);
+	crypt_check_conf(&local->hole_conf);
+
+	return ret;
+}
+
+/*
+ * prepare for submit @count bytes at offset @from
+ */
+int32_t prepare_for_submit_data(call_frame_t *frame, xlator_t *this,
+				off_t from, int32_t size, struct iovec *vec,
+				int32_t vec_count, int32_t setup_gap)
+{
+	uint32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, from, size,
+			   DATA_ATOM, setup_gap);
+
+	ret = set_config_avec_data(this, local,
+				   &local->data_conf, object, vec, vec_count);
+	crypt_check_conf(&local->data_conf);
+
+	return ret;
+}
+
+static void free_avec(struct iovec *avec,
+		      char **pool, int blocks_in_pool)
+{
+	if (!avec)
+		return;
+	GF_FREE(pool);
+	GF_FREE(avec);
+}
+
+static void free_avec_data(crypt_local_t *local)
+{
+	return free_avec(local->data_conf.avec,
+			 local->data_conf.pool,
+			 local->data_conf.blocks_in_pool);
+}
+
+static void free_avec_hole(crypt_local_t *local)
+{
+	return free_avec(local->hole_conf.avec,
+			 local->hole_conf.pool,
+			 local->hole_conf.blocks_in_pool);
+}
+
+
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (has_head_block(conf))
+		submit_head(frame, this);
+
+	if (has_full_blocks(conf))
+		submit_full(frame, this);
+
+	if (has_tail_block(conf))
+		submit_tail(frame, this);
+	return;
+}
+
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (should_submit_head_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_head(frame, this);
+	}
+	else if (should_submit_full_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_full(frame, this);
+	}
+	else if (should_submit_tail_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_tail(frame, this);
+	}
+	else
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "nothing has been submitted in ordered mode");
+	return;
+}
+
+static int32_t do_writev(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	set_gap_at_end(frame, object, &local->data_conf, DATA_ATOM);
+
+	if (local->cur_file_size < local->data_conf.orig_offset) {
+		/*
+		 * Set up hole config
+		 */
+		op_errno = prepare_for_submit_hole(frame,
+			       this,
+			       local->cur_file_size,
+			       local->data_conf.orig_offset - local->cur_file_size);
+		if (op_errno) {
+			local->op_ret = -1;
+			local->op_errno = op_errno;
+			goto error;
+		}
+	}
+	if (should_submit_hole(local))
+		submit_hole(frame, this);
+	else 
+		submit_data(frame, this);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+static int32_t crypt_writev_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_writev,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;	
+}
+
+static int32_t writev_trivial_completion(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 struct iatt *buf,
+					 dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+	
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+int crypt_writev(call_frame_t *frame,
+		 xlator_t *this,
+		 fd_t *fd,
+		 struct iovec *vec,
+		 int32_t count,
+		 off_t offset,
+		 uint32_t flags,
+		 struct iobref *iobref,
+		 dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+#if DEBUG_CRYPT
+	gf_log ("crypt", GF_LOG_DEBUG, "writing %d bytes from offset %llu",
+		(int)iovec_get_size(vec, count), (long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_WRITE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	if (iobref)
+		local->iobref = iobref_ref(iobref);
+	/*
+	 * to update real file size on the server
+	 */
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (iovec_get_size(vec, count) == 0)
+		goto trivial;
+
+	ret = prepare_for_submit_data(frame, this, offset,
+				      iovec_get_size(vec, count),
+				      vec, count, 0 /* don't setup gup
+						       in tail: we don't
+						       know file size yet */);
+	if (ret)
+		goto error;
+
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * we are called by shinking crypt_ftruncate(),
+		 * which doesn't perform hole conversion;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 */
+
+		/*
+		 * extract file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		submit_data(frame, this);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	/*
+	 * lock the file and retrieve its size
+	 */
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_writev_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   writev_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->iobref)
+		iobref_unref(iobref);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(writev, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t prepare_for_prune(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	set_config_offsets(frame, this,
+			   offset,
+			   0, /* count */
+			   DATA_ATOM,
+			   0 /* since we prune, there is no
+				gap in tail to uptodate */);
+	return 0;
+}
+
+/*
+ * Finish the read-prune-modify sequence
+ *
+ * Can be invoked as
+ * 1) ->ftruncate_cbk() for cblock-aligned, or trivial prune
+ * 2) ->writev_cbk() for non-cblock-aligned prune
+ */
+
+static int32_t prune_complete(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      struct iatt *prebuf,
+			      struct iatt *postbuf,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as ->ftruncate_cbk()
+ *
+ * Perform the "write" component of the
+ * read-prune-write sequence.
+ *
+ * submuit the rest of the file
+ */
+static int32_t prune_submit_file_tail(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno,
+				      struct iatt *prebuf,
+				      struct iatt *postbuf,
+				      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	dict_t *dict;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	dict = dict_new();
+	if (!dict) {
+		op_errno = ENOMEM;
+		goto error;
+	}
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+	local->new_file_size = conf->orig_offset;
+
+	/*
+	 * The rest of the file is a partial block and, hence,
+	 * should be written via RMW sequence, so the crypt xlator
+	 * does STACK_WIND to itself.
+	 *
+	 * Pass current file size to crypt_writev()
+	 */
+	op_errno = dict_set(dict,
+			    FSIZE_XATTR_PREFIX,
+			    data_from_uint64(local->cur_file_size));
+	if (op_errno) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "can not set key to update file size");
+		dict_unref(dict);
+		goto error;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "passing current file size (%llu) to crypt_writev",
+	       (unsigned long long)local->cur_file_size);
+	/*
+	 * Padding will be filled with
+	 * zeros by rmw_partial_block()
+	 */
+	STACK_WIND(frame,
+		   prune_complete,
+		   this,
+		   this->fops->writev, /* crypt_writev */
+		   local->fd,
+		   &local->vec,
+		   1,
+		   conf->aligned_offset, /* offset to write from */
+		   0,
+		   local->iobref,
+		   dict);
+
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as a callback of ->writev() invoked in behalf
+ * of ftruncate(): it can be
+ * 1) ordered writes issued by hole conversion in the case of
+ *    expanded truncate, or
+ * 2) an rmw partial data block issued by non-cblock-aligned
+ * prune.
+ */
+int32_t end_writeback_ftruncate(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				struct iatt *prebuf,
+				struct iatt *postbuf,
+				dict_t *xdata)
+{
+	crypt_local_t  *local = frame->local;
+	/*
+	 * if nothing has been written,
+	 * then it must be an error
+	 */
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local))
+		/* case (2) */
+		goto put_one_call;
+	/* case (1) */
+	if (should_resume_submit_hole(local))
+		submit_hole(frame, this);
+	/*
+	 * case of hole, when we should't resume
+	 */
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform prune and write components of the
+ * read-prune-write sequence.
+ *
+ * Called as ->readv_cbk()
+ *
+ * Pre-conditions:
+ * @vec contains the latest atom of the file
+ * (plain text)
+ */
+static int32_t prune_write(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno,
+			   struct iovec *vec,
+			   int32_t count,
+			   struct iatt *stbuf,
+			   struct iobref *iobref,
+			   dict_t *xdata)
+{
+	int32_t i;
+	size_t to_copy;
+	size_t copied = 0;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret == -1)
+		goto put_one_call;
+
+	/*
+	 * At first, uptodate head block
+	 */
+	if (iovec_get_size(vec, count) < conf->off_in_head) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Failed to uptodate head block for prune");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	local->vec.iov_len = conf->off_in_head;
+	local->vec.iov_base = GF_CALLOC(1, local->vec.iov_len,
+					gf_crypt_mt_data);
+
+	if (local->vec.iov_base == NULL) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+	}
+	for (i = 0; i < count; i++) {
+		to_copy = vec[i].iov_len;
+		if (to_copy > local->vec.iov_len - copied)
+			to_copy = local->vec.iov_len - copied;
+
+		memcpy((char *)local->vec.iov_base + copied,
+		       vec[i].iov_base,
+		       to_copy);
+		copied += to_copy;
+		if (copied == local->vec.iov_len)
+			break;
+	}
+	/*
+	 * perform prune with aligned offset
+	 * (i.e. at this step we prune a bit
+	 * more then it is needed
+	 */
+	STACK_WIND(frame,
+		   prune_submit_file_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->ftruncate,
+		   local->fd,
+		   conf->aligned_offset,
+		   local->xdata);
+	return 0;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform a read-prune-write sequence
+ */
+int32_t read_prune_write(call_frame_t *frame, xlator_t *this)
+{
+	int32_t ret = 0;
+	dict_t *dict = NULL;
+	crypt_local_t  *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_local_io_params_ftruncate(frame, object);
+	get_one_call_nolock(frame);
+
+	if ((conf->orig_offset & (object_alg_blksize(object) - 1)) == 0) {
+		/*
+		 * cblock-aligned prune:
+		 * we don't need read and write components,
+		 * just cut file body
+		 */
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "prune without RMW (at offset %llu",
+		       (unsigned long long)conf->orig_offset);
+
+		STACK_WIND(frame,
+			   prune_complete,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->ftruncate,
+			   local->fd,
+			   conf->orig_offset,
+			   local->xdata);
+		return 0;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "prune with RMW (at offset %llu",
+	       (unsigned long long)conf->orig_offset);	
+	/*
+	 * We are about to perform the "read" component of the
+	 * read-prune-write sequence. It means that we need to
+	 * read encrypted data from disk and decrypt it.
+	 * So, the crypt translator does STACK_WIND to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 
+	 */
+	dict = dict_new();
+	if (!dict) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		ret = ENOMEM;
+		goto exit;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   prune_write,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   local->fd,
+		   get_atom_size(object), /* bytes to read */
+		   conf->aligned_offset, /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	if (dict)
+		dict_unref(dict);
+	return ret;
+}
+
+/*
+ * File prune is more complicated than expand.
+ * First we need to read the latest atom to not lose info
+ * needed for proper update. Also we need to make sure that
+ * every component of read-prune-write sequence leaves data
+ * consistent
+ *
+ * Non-cblock aligned prune is performed as read-prune-write
+ * sequence:
+ *
+ * 1) read the latest atom;
+ * 2) perform cblock-aligned prune
+ * 3) issue a write request for the end-of-file
+ */
+int32_t prune_file(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	int32_t ret;
+
+	ret = prepare_for_prune(frame, this, offset);
+	if (ret)
+		return ret;
+	return read_prune_write(frame, this);
+}
+
+int32_t expand_file(call_frame_t *frame, xlator_t *this,
+		    uint64_t offset)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+
+	ret = prepare_for_submit_hole(frame, this,
+				      local->old_file_size,
+				      offset - local->old_file_size);
+	if (ret)
+		return ret;
+	submit_hole(frame, this);
+	return 0;
+}
+
+static int32_t ftruncate_trivial_completion(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    struct iatt *buf,
+					    dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t do_ftruncate(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    dict_t *dict,
+			    dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	if (local->data_conf.orig_offset == local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "trivial ftruncate (current file size %llu)",
+		       (unsigned long long)local->cur_file_size);
+#endif
+		goto trivial;
+	}
+	else if (local->data_conf.orig_offset < local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "prune from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = prune_file(frame,
+				      this,
+				      local->data_conf.orig_offset);
+	}
+	else {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "expand from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = expand_file(frame,
+				       this,
+				       local->data_conf.orig_offset);
+	}
+	if (op_errno)
+		goto error;
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   ftruncate_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	/*
+	 * finish with ftruncate
+	 */
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_finodelk_cbk(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_ftruncate,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * ftruncate is performed in 2 steps:
+ * . recieve file size;
+ * . expand or prune file.
+ */
+static int32_t crypt_ftruncate(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       off_t offset,
+			       dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FTRUNCATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	local->data_conf.orig_offset = offset;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_ftruncate_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(ftruncate, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+/* ->flush_cbk() */
+int32_t truncate_end(call_frame_t *frame,
+		     void *cookie,
+		     xlator_t *this,
+		     int32_t op_ret,
+		     int32_t op_errno,
+		     dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_UNWIND_STRICT(truncate,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local->xdata);
+	return 0;
+}
+
+/* ftruncate_cbk() */
+int32_t truncate_flush(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iatt *prebuf,
+		       struct iatt *postbuf,
+		       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *fd = local->fd;
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	STACK_WIND(frame,
+		   truncate_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   fd,
+		   NULL);
+	fd_unref(fd);
+	return 0;
+}
+
+/*
+ * is called as ->open_cbk()
+ */
+static int32_t truncate_begin(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0) {
+		fd_unref(fd);
+		STACK_UNWIND_STRICT(truncate,
+				    frame,
+				    op_ret,
+				    op_errno, NULL, NULL, NULL);
+		return 0;
+	}
+	/*
+	 * crypt_truncate() is implemented via crypt_ftruncate(),
+	 * so the crypt xlator does STACK_WIND to itself here
+	 */
+	STACK_WIND(frame,
+		   truncate_flush,
+		   this,
+		   this->fops->ftruncate, /* crypt_ftruncate */
+		   fd,
+		   local->offset,
+		   NULL);
+	return 0;
+}
+
+/*
+ * crypt_truncate() is implemented via crypt_ftruncate() as a
+ * sequence crypt_open() - crypt_ftruncate() - truncate_flush()
+ */
+int32_t crypt_truncate(call_frame_t *frame,
+		       xlator_t *this,
+		       loc_t *loc,
+		       off_t offset,
+		       dict_t *xdata)
+{
+	fd_t *fd;
+	crypt_local_t *local;
+
+#if DEBUG_CRYPT
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "truncate file %s at offset %llu",
+	       loc->path, (unsigned long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_TRUNCATE);
+	if (!local)
+		goto error;
+
+	fd = fd_create(loc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->offset = offset;
+	local->xdata = xdata;
+	STACK_WIND(frame,
+		   truncate_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   loc,
+		   O_RDWR,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(truncate, frame, -1, EINVAL, NULL, NULL, NULL);
+	return 0;
+}
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_WRITE:
+		return end_writeback_writev;
+	case GF_FOP_FTRUNCATE:
+		return end_writeback_ftruncate;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad wb operation %d", fop);
+		return NULL;
+	}
+}
+
+/*
+ * true, if the caller needs metadata string
+ */
+static int32_t is_custom_mtd(dict_t *xdata)
+{
+	data_t *data;
+	uint32_t flags;
+
+	if (!xdata)
+		return 0;
+
+	data = dict_get(xdata, MSGFLAGS_PREFIX);
+	if (!data)
+		return 0;
+	if (data->len != sizeof(uint32_t)) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "Bad msgflags size (%d)", data->len);
+		return -1;
+	}
+	flags = *((uint32_t *)data->data);
+	return msgflags_check_mtd_lock(&flags);
+}
+
+static int32_t crypt_open_done(call_frame_t *frame,
+			       void *cookie,
+			       xlator_t *this,
+			       int32_t op_ret,
+			       int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret < 0)
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+	put_one_call_open(frame);
+	return 0;
+}
+
+static void crypt_open_tail(call_frame_t *frame, xlator_t *this)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_open_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+}
+
+/*
+ * load private inode info at open time
+ * called as ->fgetxattr_cbk()
+ */
+static int load_mtd_open(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	int32_t ret;
+	gf_boolean_t upload_info;
+	data_t *mtd;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	/*
+	 * first, check for cached info
+	 */
+	ret = inode_ctx_get(local->fd->inode, this, &value);
+	if (ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Inode info expected, but not found");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * info has been found in the cache
+		 */
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info hasn't been found in the cache.
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			goto exit;
+		}
+		init_inode_info_head(info, local->fd);
+		upload_info = _gf_true;
+	}
+	/*
+	 * extract metadata
+	 */
+	mtd = dict_get(dict, CRYPTO_FORMAT_PREFIX);
+	if (!mtd) {
+		local->op_ret = -1;
+		local->op_errno = ENOENT;
+		gf_log (this->name, GF_LOG_WARNING,
+			"Format string wasn't found");
+		goto exit;
+	}
+	/*
+	 * authenticate metadata against the path
+	 */	
+	ret = open_format((unsigned char *)mtd->data,
+			  mtd->len,
+			  local->loc,
+			  info,
+			  get_master_cinfo(priv),
+			  local,
+			  upload_info);
+	if (ret) {
+		local->op_ret = -1;
+		local->op_errno = ret;
+		goto exit;
+	}
+	if (upload_info) {
+		ret = init_inode_info_tail(info, get_master_cinfo(priv));
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;	
+		}
+		ret = inode_ctx_put(local->fd->inode,
+				    this, (uint64_t)(long)info);
+		if (ret == -1) {
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+	}
+	if (local->custom_mtd) {
+		/*
+		 * pass the metadata string to the customer
+		 */
+		ret = dict_set_static_bin(local->xdata,
+					  CRYPTO_FORMAT_PREFIX,
+					  mtd->data,
+					  mtd->len);
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;
+		}
+	}
+ exit:
+	if (!local->custom_mtd)
+		crypt_open_tail(frame, this);
+	else
+		put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open_finodelk_cbk(call_frame_t *frame,
+				       void *cookie,
+				       xlator_t *this,
+				       int32_t op_ret,
+				       int32_t op_errno,
+				       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING, "finodelk (LOCK) failed");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   load_mtd_open,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   CRYPTO_FORMAT_PREFIX,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+/*
+ * verify metadata against the specified pathname
+ */
+static int32_t crypt_open_cbk(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	else if (local->custom_mtd){
+		local->xdata = dict_new();
+		if (!local->xdata) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			gf_log ("crypt", GF_LOG_ERROR,
+				"Can not get new dict for mtd string");
+			goto exit;
+		}
+	}
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = local->custom_mtd ? F_WRLCK : F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_open_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc,
+			  int32_t flags,
+			  fd_t *fd,
+			  dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_OPEN);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc) {
+		ret = ENOMEM;
+		goto error;
+	}
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	ret = is_custom_mtd(xdata);
+	if (ret < 0) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+		ret = EINVAL;
+		goto error;
+	}
+	local->custom_mtd = ret;
+
+	if ((flags & O_ACCMODE) == O_WRONLY)
+		/*
+		 * we can't open O_WRONLY, because
+		 * we need to do read-modify-write
+		 */
+		flags = (flags & ~O_ACCMODE) | O_RDWR;
+	/*
+	 * Make sure that out translated offsets
+	 * and counts won't be ignored
+	 */
+	flags &= ~O_APPEND;
+	get_one_call_nolock(frame);
+	STACK_WIND(frame,
+		   crypt_open_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->open,
+		   loc,
+		   flags,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(open,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct object_cipher_info *object = &info->cinfo;
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "Init inode info for object %s",
+	       uuid_utoa(info->oid));
+#endif
+	ret = data_cipher_algs[object->o_alg][object->o_mode].set_private(info,
+									master);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set private info failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * Init inode info at ->create() time
+ */
+static void init_inode_info_create(struct crypt_inode_info *info,
+				   struct master_cipher_info *master,
+				   data_t *data)
+{
+	struct object_cipher_info *object;
+
+	info->nr_minor = CRYPT_XLATOR_ID;
+	memcpy(info->oid, data->data, data->len);
+
+	object = &info->cinfo;
+
+	object->o_alg        = master->m_alg;
+	object->o_mode       = master->m_mode;
+	object->o_block_bits = master->m_block_bits;
+	object->o_dkey_size  = master->m_dkey_size;
+}
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd)
+{
+	memcpy(info->oid, fd->inode->gfid, sizeof(uuid_t));
+}
+
+static int32_t crypt_create_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno, dict_t *xdata)
+{
+	crypt_private_t *priv = this->private;
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0) {
+		free_inode_info(info);
+		goto unwind;
+	}
+	op_errno = init_inode_info_tail(info, get_master_cinfo(priv));
+	if (op_errno) {
+		op_ret = -1;
+		free_inode_info(info);
+		goto unwind;
+	}
+	/*
+	 * FIXME: drop major subversion number
+	 */
+	op_ret = inode_ctx_put(local->fd->inode, this, (uint64_t)(long)info);
+	if (op_ret == -1) {
+		op_errno = EIO;
+		free_inode_info(info);
+		goto unwind;
+	}
+ unwind:
+	free_format(local);
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int crypt_create_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	dict_unref(local->xattr);
+
+	if (op_ret < 0)
+		goto error;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_create_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	free_inode_info(local->info);
+	free_format(local);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_create_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+
+	STACK_WIND(frame,
+		   crypt_create_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   local->fd,
+		   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+		   0,
+		   NULL);
+	return 0;
+ error: 
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+	if (local->xdata)
+		dict_unref(local->xdata);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * Create and store crypt-specific format on disk;
+ * Populate cache with private inode info
+ */
+static int32_t crypt_create_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				fd_t *fd,
+				inode_t *inode,
+				struct iatt *buf,
+				struct iatt *preparent,
+				struct iatt *postparent,
+				dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);	
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        STACK_WIND(frame,
+		   crypt_create_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);	
+	return 0;
+ error:
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_create(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t flags,
+			    mode_t mode,
+			    mode_t umask,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	int ret;
+	data_t *data;
+	crypt_local_t *local;
+	crypt_private_t *priv;
+	struct master_cipher_info *master;
+	struct crypt_inode_info *info;
+
+	priv = this->private;
+	master = get_master_cinfo(priv);
+
+	if (master_alg_atomic(master)) {
+		/*
+		 * We can't open O_WRONLY, because we
+		 * need to do read-modify-write.
+		 */
+		if ((flags & O_ACCMODE) == O_WRONLY)
+			flags = (flags & ~O_ACCMODE) | O_RDWR;
+		/*
+		 * Make sure that out translated offsets
+		 * and counts won't be ignored
+		 */
+		flags &= ~O_APPEND;
+	}
+	local = crypt_alloc_local(frame, this, GF_FOP_CREATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	data = dict_get(xdata, "gfid-req");
+	if (!data) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING, "gfid not found");
+		goto error;
+	}
+	if (data->len != sizeof(uuid_t)) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad gfid size (%d), should be %d",
+		       (int)data->len, (int)sizeof(uuid_t));
+		goto error;
+	}
+	info = alloc_inode_info(local, loc);
+	if (!info){
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * NOTE:
+	 * format has to be created BEFORE
+	 * proceeding to the untrusted server
+	 */
+	ret = alloc_format_create(local);
+	if (ret) {
+		free_inode_info(info);
+		goto error;
+	}
+	init_inode_info_create(info, master, data);
+
+	ret = create_format(local->format,
+			    loc,
+			    info,
+			    master);
+	if (ret) {
+		free_inode_info(info);
+		goto error;		
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		free_inode_info(info);
+		free_format(local);	
+		goto error;
+	}
+	ret = dict_set_static_bin(local->xattr,
+				  CRYPTO_FORMAT_PREFIX,
+				  local->format,
+				  new_format_size());
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	ret = dict_set(local->xattr, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	STACK_WIND(frame,
+		   crypt_create_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->create,
+		   loc,
+		   flags,
+		   mode,
+		   umask,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	gf_log("crypt", GF_LOG_WARNING, "can not create file");
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    -1,
+			    ret,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+/*
+ * FIXME: this should depends on the version of format string
+ */
+static int32_t filter_crypt_xattr(dict_t *dict,
+				  char *key, data_t *value, void *data)
+{
+        dict_del(dict, key);
+        return 0;
+}
+
+static int32_t crypt_fsetxattr(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       dict_t *dict,
+			       int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_fsetxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   fd,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * TBD: verify file metadata before wind
+ */
+static int32_t crypt_setxattr(call_frame_t *frame,
+			      xlator_t *this,
+			      loc_t *loc,
+			      dict_t *dict,
+			      int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_setxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   loc,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t linkop_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0 &&
+	    op_errno == ENOENT &&
+	    local->loc->inode->ia_type == IA_IFLNK) {
+		local->op_ret = 0;
+		local->op_errno = 0;
+	}
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * unpin inode on the server
+ */
+static int32_t link_flush(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  inode_t *inode,
+			  struct iatt *buf,
+			  struct iatt *preparent,
+			  struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	link_unwind(frame);
+	return 0;
+}
+
+void link_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	inode_t *inode;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(link,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	inode = local->inode;
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc) {
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(link,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (inode)
+		inode_unref(inode);
+}
+
+void link_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   link_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->link,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+/*
+ * unlink()
+ */
+static int32_t unlink_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *preparent,
+			    struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unlink_unwind(frame);
+	return 0;
+}
+
+void unlink_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(unlink,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(unlink,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+}
+
+void unlink_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   unlink_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->unlink,
+		   local->loc,
+		   local->flags,
+		   local->xdata);
+}
+
+void rename_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(rename,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);	
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	prenewparent = local->prenewparent;
+	postnewparent = local->postnewparent;	
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc){
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(rename,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    prenewparent,
+			    postnewparent,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (prenewparent)
+		GF_FREE(prenewparent);
+	if (postnewparent)
+		GF_FREE(postnewparent);
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t rename_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	rename_unwind(frame);
+	return 0;
+}
+
+static int32_t rename_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *buf,
+			    struct iatt *preoldparent,
+			    struct iatt *postoldparent,
+			    struct iatt *prenewparent,
+			    struct iatt *postnewparent,
+			    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	dict_unref(local->xdata);
+	local->xdata = NULL;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	local->buf = *buf;
+	local->prebuf = *preoldparent;
+	local->postbuf = *postoldparent;
+	if (prenewparent) {
+		local->prenewparent = GF_CALLOC(1, sizeof(*prenewparent),
+						gf_crypt_mt_iatt);
+		if (!local->prenewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->prenewparent = *prenewparent;
+	}
+	if (postnewparent) {
+		local->postnewparent = GF_CALLOC(1, sizeof(*postnewparent),
+						 gf_crypt_mt_iatt);
+		if (!local->postnewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->postnewparent = *postnewparent;
+	}
+	STACK_WIND(frame,
+		   rename_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	rename_unwind(frame);
+	return 0;
+}
+
+void rename_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   rename_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->rename,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+static int32_t __do_linkop(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_wind_handler_t wind_fn;
+	linkop_unwind_handler_t unwind_fn;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret >= 0)
+		wind_fn(frame, this);
+	else {
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+		unwind_fn(frame);
+	}
+	return 0;
+}
+
+static int32_t do_linkop(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if(op_ret < 0)
+		goto error;
+
+	lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:		
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * Update the metadata string (against the new pathname);
+ * submit the result
+ */
+static int32_t linkop_begin(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	gf_boolean_t upload_info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+	struct crypt_inode_info *info;
+	data_t *old_mtd;
+	uint32_t new_mtd_size;
+	uint64_t value = 0;
+	void (*unwind_fn)(call_frame_t *frame);
+	void (*wind_fn)(call_frame_t *frame, xlator_t *this);
+	mtd_op_t mop;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	mop = linkop_mtdop_dispatch(local->fop);
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto wind;
+	if (op_ret < 0)
+		/*
+		 * verification failed
+		 */
+		goto error;
+
+	old_mtd = dict_get(xdata, CRYPTO_FORMAT_PREFIX);
+	if (!old_mtd) {
+		op_errno = EIO;
+		gf_log (this->name, GF_LOG_DEBUG,
+			"Metadata string wasn't found");
+		goto error;
+	}
+	new_mtd_size = format_size(mop, old_mtd->len);
+	op_errno = alloc_format(local, new_mtd_size);
+	if (op_errno)
+		goto error;
+	/*
+	 * check for cached info 
+	 */
+	op_ret = inode_ctx_get(fd->inode, this, &value);
+	if (op_ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log (this->name, GF_LOG_WARNING,
+				"Inode info was not found");
+			op_errno = EINVAL;
+			goto error;
+		}
+		/*
+		 * info was found in the cache
+		 */
+		local->info = info;
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info wasn't found in the cache;
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info)
+			goto error;
+		init_inode_info_head(info, fd);
+		local->info = info;
+		upload_info = _gf_true;
+	}
+	op_errno = open_format((unsigned char *)old_mtd->data,
+			       old_mtd->len,
+			       local->loc,
+			       info,
+			       get_master_cinfo(priv),
+			       local,
+			       upload_info);
+	if (op_errno)
+		goto error;
+	if (upload_info == _gf_true) {
+		op_errno = init_inode_info_tail(info,
+						get_master_cinfo(priv));
+		if (op_errno)
+			goto error;
+		op_errno = inode_ctx_put(fd->inode, this,
+					 (uint64_t)(long)(info));
+		if (op_errno == -1) {
+			op_errno = EIO;
+			goto error;
+		}
+	}
+	/*
+	 * update the format string (append/update/cup a MAC)
+	 */
+	op_errno = update_format(local->format,
+				 (unsigned char *)old_mtd->data,
+				 old_mtd->len,
+				 local->mac_idx,
+				 mop,
+				 local->newloc,
+				 info,
+				 get_master_cinfo(priv),
+				 local);
+	if (op_errno)
+		goto error;
+	/*
+	 * store the new format string on the server
+	 */
+	if (new_mtd_size) {
+		op_errno = dict_set_static_bin(local->xattr,
+					       CRYPTO_FORMAT_PREFIX,
+					       local->format,
+					       new_mtd_size);
+		if (op_errno)
+			goto error;
+	}
+	STACK_WIND(frame,
+		   do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   local->loc,
+		   local->xattr,
+		   0,
+		   NULL);
+	return 0;
+ wind:
+	wind_fn(frame, this);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t linkop_grab_local(call_frame_t *frame,
+				 xlator_t *this,
+				 loc_t *oldloc,
+				 loc_t *newloc,
+				 int flags, dict_t *xdata,
+				 glusterfs_fop_t op)
+{
+	int32_t ret = ENOMEM;
+	fd_t *fd;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, op);
+	if (!local)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	fd = fd_create(oldloc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->flags = flags;
+	local->loc = GF_CALLOC(1, sizeof(*oldloc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, oldloc);
+	if (ret) {
+		GF_FREE(local->loc);
+		local->loc = NULL;
+		goto error;
+	}
+	if (newloc) {
+		local->newloc = GF_CALLOC(1, sizeof(*newloc), gf_crypt_mt_loc);
+		if (!local->newloc) {
+			GF_FREE(local->loc);
+			loc_wipe(local->loc);
+			goto error;
+		}
+		memset(local->newloc, 0, sizeof(*local->newloc));
+		ret = loc_copy(local->newloc, newloc);
+		if (ret) {
+			GF_FREE(local->loc);
+			loc_wipe(local->loc);
+			GF_FREE(local->newloc);
+			goto error;
+		}
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	return 0;
+ error:
+	if (local->xdata)
+		dict_unref(local->xdata);
+	if (local->fd)
+		fd_unref(local->fd);
+	local->fd = 0;
+	local->loc = NULL;
+	local->newloc = NULL;
+
+	local->op_ret = -1;
+	local->op_errno = ret;
+
+	return ret;
+}
+
+/*
+ * read and verify locked metadata against the old pathname (via open);
+ * update the metadata string in accordance with the new pathname;
+ * submit modified metadata;
+ * wind;
+ */
+static int32_t linkop(call_frame_t *frame,
+		      xlator_t *this,
+		      loc_t *oldloc,
+		      loc_t *newloc,
+		      int flags,
+		      dict_t *xdata,
+		      glusterfs_fop_t op)
+{
+	int32_t ret;
+	dict_t *dict;
+	crypt_local_t *local;
+	void (*unwind_fn)(call_frame_t *frame);
+
+	unwind_fn = linkop_unwind_dispatch(op);
+
+	ret = linkop_grab_local(frame, this, oldloc, newloc, flags, xdata, op);
+	local = frame->local;
+	if (ret)
+		goto error;
+	dict = dict_new();
+	if (!dict) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * Set a message to crypt_open() that we need
+	 * locked metadata string.
+	 * All link operations (link, unlink, rename)
+	 * need write lock
+	 */
+	msgflags_set_mtd_wlock(&local->msgflags);
+	ret = dict_set_static_bin(dict,
+				  MSGFLAGS_PREFIX,
+				  &local->msgflags,
+				  sizeof(local->msgflags));
+	if (ret) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not set dict");
+		dict_unref(dict);
+		goto error;
+	}
+	/*
+	 * verify metadata against the old pathname
+	 * and retrieve locked metadata string
+	 */
+	STACK_WIND(frame,
+		   linkop_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   oldloc,
+		   O_RDWR,
+		   local->fd,
+		   dict);
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = ret;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t crypt_link(call_frame_t *frame, xlator_t *this,
+			  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_LINK);
+}
+
+static int32_t crypt_unlink(call_frame_t *frame, xlator_t *this,
+			    loc_t *loc, int flags, dict_t *xdata)
+{
+	return linkop(frame, this, loc, NULL, flags, xdata, GF_FOP_UNLINK);
+}
+
+static int32_t crypt_rename(call_frame_t *frame, xlator_t *this,
+			    loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_RENAME);
+}
+
+static void put_one_call_open(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		fd_t *fd = local->fd;
+		loc_t *loc = local->loc;
+		dict_t *xdata = local->xdata;
+
+		STACK_UNWIND_STRICT(open,
+				    frame,
+				    local->op_ret,
+				    local->op_errno,
+				    fd,
+				    xdata);
+		fd_unref(fd);
+		if (xdata)
+			dict_unref(xdata);
+		loc_wipe(loc);
+		GF_FREE(loc);
+	}
+}
+
+static int32_t __crypt_readv_done(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	/* read deals with data configs only */
+	struct iovec *avec = local->data_conf.avec;
+	char **pool = local->data_conf.pool;
+	int blocks_in_pool = local->data_conf.blocks_in_pool;
+	struct iobref *iobref = local->iobref;
+	struct iobref *iobref_data = local->iobref_data;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "readv unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	dump_plain_text(local, avec);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "readv: ret_to_user: %d, iovec len: %d, ia_size: %llu",
+	       (int)(local->rw_count > 0 ? local->rw_count : local->op_ret),
+	       (int)(local->rw_count > 0 ? iovec_get_size(avec, local->data_conf.acount) : 0),
+	       (unsigned long long)local->buf.ia_size);
+
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    local->rw_count > 0 ? local->rw_count : local->op_ret,
+			    local->op_errno,
+			    avec,
+			    avec ? local->data_conf.acount : 0,
+			    &local->buf,
+			    local->iobref,
+			    local_xdata);
+
+	free_avec(avec, pool, blocks_in_pool);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobref)
+		iobref_unref(iobref);
+	if (iobref_data)
+		iobref_unref(iobref_data);
+	return 0;
+}
+
+static void crypt_readv_done(call_frame_t *frame, xlator_t *this)
+{
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 *  don't unlock (it will be done by the parent)
+		 */
+		__crypt_readv_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		crypt_local_t *local = frame->local;
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_readv_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+}
+
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local))
+		crypt_readv_done(frame, this);
+}
+
+static int32_t __crypt_writev_done(call_frame_t *frame,
+				   void *cookie,
+				   xlator_t *this,
+				   int32_t op_ret,
+				   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	int32_t ret_to_user;
+
+	if (local->xattr)
+		dict_unref(local->xattr);
+	/*
+	 * Calculate amout of butes to be returned
+	 * to user. We need to subtract paddings that
+	 * have been written as a part of atom.
+	 */
+	/*
+	 * subtract head padding
+	 */
+	if (local->rw_count == 0)
+		/*
+		 * Nothing has been written, it must be an error
+		 */
+		ret_to_user = local->op_ret;
+	else if (local->rw_count <= local->data_conf.off_in_head) {
+		gf_log("crypt", GF_LOG_WARNING, "Incomplete write");
+		ret_to_user = 0;
+	}
+	else
+		ret_to_user = local->rw_count -
+			local->data_conf.off_in_head;
+	/*
+	 * subtract tail padding
+	 */
+	if (ret_to_user > local->data_conf.orig_size)
+		ret_to_user = local->data_conf.orig_size;
+
+	if (local->iobref)
+		iobref_unref(local->iobref);
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "writev: ret_to_user: %d", ret_to_user);
+
+	STACK_UNWIND_STRICT(writev,
+			    frame,
+			    ret_to_user,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_writev_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 * don't unlock (it will be done by the parent)
+		 */
+		__crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_writev_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+	return 0;
+}
+
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_writev_done(frame, NULL,
+						  this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_writev_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+static int32_t __crypt_ftruncate_done(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	char *iobase = local->vec.iov_base;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "ftruncate unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "ftruncate, return to user: presize=%llu, postsize=%llu",
+	       (unsigned long long)local->prebuf.ia_size,
+	       (unsigned long long)local->postbuf.ia_size);
+
+	STACK_UNWIND_STRICT(ftruncate,
+			    frame,
+			    local->op_ret < 0 ? -1 : 0,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobase)
+		GF_FREE(iobase);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_done(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct gf_flock  lock  = {0, };
+
+	dict_unref(local->xattr);
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	lock.l_type   = F_UNLCK;
+	lock.l_whence = SEEK_SET;
+	lock.l_start  = 0;
+	lock.l_len    = 0;
+	lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __crypt_ftruncate_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+}
+
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_ftruncate_done(frame, NULL,
+						     this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_ftruncate_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_ftruncate_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+/*
+ * load regular file size for some FOPs
+ */
+static int32_t load_file_size(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      dict_t *dict,
+			      dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0)
+		goto unwind;
+	/*
+	 * load regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		if (local->xdata)
+			dict_unref(local->xdata);
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto unwind;
+	}
+	local->buf.ia_size = data_to_uint64(data);
+
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "FOP %d: Translate regular file to %llu",
+	       local->fop,
+	       (unsigned long long)local->buf.ia_size);
+ unwind:	
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_LOOKUP:
+		STACK_UNWIND_STRICT(lookup,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? local->inode : NULL,
+				    op_ret >= 0 ? &local->buf : NULL,				    
+				    local->xdata,
+				    op_ret >= 0 ? &local->postbuf : NULL);
+		break;
+	case GF_FOP_READ:
+		STACK_UNWIND_STRICT(readv,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    NULL,
+				    0,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    NULL,
+				    NULL);
+		break;
+	default:
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Improper file operation %d", local->fop);
+	}
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (local_inode)
+		inode_unref(local_inode);
+	return 0;
+}
+
+static int32_t crypt_stat_common_cbk(call_frame_t *frame,
+				     void *cookie,
+				     xlator_t *this,
+				     int32_t op_ret,
+				     int32_t op_errno,
+				     struct iatt *buf, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->buf = *buf;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->fgetxattr,
+			   local->fd,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->getxattr,
+			   local->loc,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+ unwind:
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+}
+
+static int32_t crypt_fstat(call_frame_t *frame,
+			   xlator_t *this,
+			   fd_t *fd, dict_t *xdata)
+{
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FSTAT);
+	if (!local)
+		goto error;
+	local->fd = fd_ref(fd);
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(fstat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_stat(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_STAT);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->stat,
+		   loc,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(stat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_lookup_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				inode_t *inode,
+				struct iatt *buf, dict_t *xdata,
+				struct iatt *postparent)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->postbuf = *postparent;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	uuid_copy(local->loc->gfid, buf->ia_gfid);
+
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ unwind:
+	loc_wipe(local->loc);
+	GF_FREE(local->loc);
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    inode,
+			    buf,
+			    xdata,
+			    postparent);
+	return 0;
+}
+
+static int32_t crypt_lookup(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_LOOKUP);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "Lookup %s", loc->path);
+	STACK_WIND(frame,
+		   crypt_lookup_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->lookup,
+		   loc,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * for every regular directory entry find its real file size
+ * and update stat's buf properly
+ */
+static int32_t crypt_readdirp_cbk(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno,
+				  gf_dirent_t *entries, dict_t *xdata)
+{
+	gf_dirent_t *entry = NULL;
+
+	if (op_ret < 0)
+		goto unwind;
+
+	list_for_each_entry (entry, (&entries->list), list) {
+		data_t *data;
+
+		if (!IA_ISREG(entry->d_stat.ia_type))
+			continue;
+		data = dict_get(entry->dict, FSIZE_XATTR_PREFIX);
+		if (!data){
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size of direntry not found");
+			op_errno = EIO;
+			op_ret = -1;
+			break;
+		}
+		entry->d_stat.ia_size = data_to_uint64(data);
+	}
+ unwind:
+	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+/*
+ * ->readdirp() fills in-core inodes, so we need to set proper
+ * file sizes for all directory entries of the parent @fd.
+ * Actual updates take place in ->crypt_readdirp_cbk()
+ */
+static int32_t crypt_readdirp(call_frame_t *frame, xlator_t *this,
+			      fd_t *fd, size_t size, off_t offset,
+			      dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+
+	if (!xdata) {
+		xdata = dict_new();
+		if (!xdata)
+			goto error;
+	}
+	else
+		dict_ref(xdata);
+	/*
+	 * make sure that we'll have real file sizes at ->readdirp_cbk()
+	 */
+	ret = dict_set(xdata, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(xdata);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_readdirp_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp,
+		   fd,
+		   size,
+		   offset,
+		   xdata);
+	dict_unref(xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readdirp, frame, -1, ret, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_access(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t mask, dict_t *xdata)
+{
+	gf_log(this->name, GF_LOG_WARNING,
+	       "NFS mounts of encrypted volumes are unsupported");
+	STACK_UNWIND_STRICT(access, frame, -1, EPERM, NULL);
+	return 0;
+}
+
+int32_t master_set_block_size (xlator_t *this, crypt_private_t *priv,
+			       dict_t *options)
+{
+	uint64_t block_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("block-size", block_size, options,
+				 size, error);
+	else
+		GF_OPTION_INIT("block-size", block_size, size, error);
+
+	switch (block_size) {
+	case 512:
+		master->m_block_bits = 9;
+		break;
+	case 1024:
+		master->m_block_bits = 10;
+		break;
+	case 2048:
+		master->m_block_bits = 11;
+		break;
+	case 4096:
+		master->m_block_bits = 12;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: unsupported block size %llu",
+		       (unsigned long long)block_size);
+		goto error;
+	}
+	return 0;
+ error:
+	return -1;
+}
+
+int32_t master_set_alg(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_alg = AES_CIPHER_ALG;
+	return 0;
+}
+
+int32_t master_set_mode(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_mode = XTS_CIPHER_MODE;
+	return 0;
+}
+
+/*
+ * set key size in bits to the master info
+ * Pre-conditions: cipher mode in the master info is uptodate.
+ */
+static int master_set_data_key_size (xlator_t *this, crypt_private_t *priv,
+				     dict_t *options)
+{
+	int32_t ret;
+	uint64_t key_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("data-key-size", key_size, options,
+				 size, error);
+	else
+		GF_OPTION_INIT("data-key-size", key_size, size, error);
+
+	ret = data_cipher_algs[master->m_alg][master->m_mode].check_key(key_size);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: wrong bin key size %llu for alg %d mode %d",
+		       (unsigned long long)key_size,
+		       (int)master->m_alg,
+		       (int)master->m_mode);
+		goto error;
+	}
+	master->m_dkey_size = key_size;
+	return 0;
+ error:
+	return -1;
+}
+
+static int is_hex(char *s) {
+	return  ('0' <= *s && *s <= '9') || ('a' <= *s && *s <= 'f');
+}
+
+static int parse_hex_buf(xlator_t *this, char *src, unsigned char *dst,
+			 int hex_size)
+{
+	int i;
+	int hex_byte = 0;
+
+	for (i = 0; i < (hex_size / 2); i++) {
+		if (!is_hex(src + i*2) || !is_hex(src + i*2 + 1)) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "FATAL: not hex symbol in key");
+			return -1;
+		}
+		if (sscanf(src + i*2, "%2x", &hex_byte) != 1) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "FATAL: can not parse hex key");
+			return -1;
+		}
+		dst[i] = hex_byte & 0xff;
+	}
+	return 0;
+}
+
+/*
+ * Parse options;
+ * install master volume key
+ */
+int32_t master_set_master_vol_key(xlator_t *this, crypt_private_t *priv)
+{
+	int32_t ret;
+	FILE *file = NULL;
+
+	int32_t key_size;
+	char *opt_key_file_pathname = NULL;
+
+	unsigned char bin_buf[MASTER_VOL_KEY_SIZE];
+	char hex_buf[2 * MASTER_VOL_KEY_SIZE];
+
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	/*
+	 * extract master key passed via option
+	 */
+	GF_OPTION_INIT("master-key", opt_key_file_pathname, path, bad_key);
+
+	if (!opt_key_file_pathname) {
+		gf_log(this->name, GF_LOG_ERROR, "FATAL: missing master key");
+		return -1;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "handling file key %s",
+	       opt_key_file_pathname);
+
+	file = fopen(opt_key_file_pathname, "r");
+	if (file == NULL) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "FATAL: can not open file with master key");
+		return -1;
+	}
+	/*
+	 * extract hex key
+	 */
+	key_size = fread(hex_buf, 1, sizeof(hex_buf), file);
+	if (key_size < sizeof(hex_buf)) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "FATAL: master key is too short");
+		goto bad_key;
+	}
+	ret = parse_hex_buf(this, hex_buf, bin_buf, key_size);
+	if (ret)
+		goto bad_key;
+	memcpy(master->m_key, bin_buf, MASTER_VOL_KEY_SIZE);
+	memset(hex_buf, 0, sizeof(hex_buf));
+	fclose(file);
+
+ 	memset(bin_buf, 0, sizeof(bin_buf));
+	return 0;
+ bad_key:
+	gf_log(this->name, GF_LOG_ERROR, "FATAL: bad master key");
+	if (file)
+		fclose(file);
+	memset(bin_buf, 0, sizeof(bin_buf));
+	return -1;
+}
+
+/*
+ * Derive volume key for object-id authentication
+ */
+int32_t master_set_nmtd_vol_key(xlator_t *this, crypt_private_t *priv)
+{
+	return get_nmtd_vol_key(get_master_cinfo(priv));
+}
+
+int32_t crypt_init_xlator(xlator_t *this)
+{
+	int32_t ret;
+	crypt_private_t *priv = this->private; 
+
+	ret = master_set_alg(this, priv);
+	if (ret)
+		return ret;
+	ret = master_set_mode(this, priv);
+	if (ret)
+		return ret;
+	ret = master_set_block_size(this, priv, NULL);
+	if (ret)
+		return ret;
+	ret = master_set_data_key_size(this, priv, NULL);
+	if (ret)
+		return ret;
+	ret = master_set_master_vol_key(this, priv);
+	if (ret)
+		return ret;
+	return master_set_nmtd_vol_key(this, priv);
+}
+
+static int32_t crypt_alloc_private(xlator_t *this)
+{
+	this->private = GF_CALLOC(1, sizeof(crypt_private_t), gf_crypt_mt_priv);
+	if (!this->private) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Can not allocate memory for private data");
+		return ENOMEM;
+	}
+	return 0;
+}
+
+static void crypt_free_private(xlator_t *this)
+{
+	crypt_private_t *priv = this->private;
+	if (priv) {
+		memset(priv, 0, sizeof(*priv));
+		GF_FREE(priv);
+	}
+}
+
+int32_t reconfigure (xlator_t *this, dict_t *options)
+{
+	int32_t ret = -1;
+	crypt_private_t *priv = NULL;
+
+	GF_VALIDATE_OR_GOTO ("crypt", this, error);
+	GF_VALIDATE_OR_GOTO (this->name, this->private, error);
+	GF_VALIDATE_OR_GOTO (this->name, options, error);
+
+	priv = this->private;
+
+	ret = master_set_block_size(this, priv, options);
+	if (ret) {
+		gf_log("this->name", GF_LOG_ERROR,
+		       "Failed to reconfure block size");
+		goto error;
+	}
+	ret = master_set_data_key_size(this, priv, options);
+	if (ret) {
+		gf_log("this->name", GF_LOG_ERROR,
+		       "Failed to reconfure data key size");
+		goto error;
+	}
+	return 0;
+ error:
+	return ret;
+}
+
+int32_t init(xlator_t *this)
+{
+	int32_t ret;
+
+	if (!this->children || this->children->next) {
+		gf_log ("crypt", GF_LOG_ERROR,
+			"FATAL: crypt should have exactly one child");
+		return EINVAL;
+	}
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+	ret = crypt_alloc_private(this);
+	if (ret)
+		return ret;
+	ret = crypt_init_xlator(this);
+	if (ret)
+		goto error;
+	this->local_pool = mem_pool_new(crypt_local_t, 64);
+        if (!this->local_pool) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "failed to create local_t's memory pool");
+		ret = ENOMEM;
+                goto error;
+        }
+	gf_log ("crypt", GF_LOG_INFO, "crypt xlator loaded");
+	return 0;
+ error:
+	crypt_free_private(this);
+	return ret;
+}
+
+void fini (xlator_t *this)
+{
+	crypt_free_private(this);
+}
+
+struct xlator_fops fops = {
+	.readv        = crypt_readv,
+	.writev       = crypt_writev,
+	.truncate     = crypt_truncate,
+	.ftruncate    = crypt_ftruncate,
+	.setxattr     = crypt_setxattr,
+	.fsetxattr    = crypt_fsetxattr,
+	.link         = crypt_link,
+	.unlink       = crypt_unlink,
+	.rename       = crypt_rename,
+	.open         = crypt_open,
+	.create       = crypt_create,
+	.stat         = crypt_stat,
+	.fstat        = crypt_fstat,
+	.lookup       = crypt_lookup,
+	.readdirp     = crypt_readdirp,
+	.access       = crypt_access   
+};
+
+struct xlator_cbks cbks = {
+	.forget       = crypt_forget
+};
+
+struct volume_options options[] = {
+	{ .key = {"master-key"},
+	  .type = GF_OPTION_TYPE_PATH,
+	  .description = "Pathname of regular file which contains master volume key"
+	},
+	{ .key = {"data-key-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .description = "Data key size (bits)",
+	  .min = 256,
+	  .max = 512,
+	  .default_value = "256",
+	},
+	{ .key = {"block-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .description = "Atom size (bits)",
+	  .min = 512,
+	  .max = 4096,
+	  .default_value = "4096"
+	},
+	{ .key  = {NULL} },
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt.h b/xlators/encryption/crypt/src/crypt.h
new file mode 100644
index 000000000..01a8542ab
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.h
@@ -0,0 +1,899 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_H__
+#define __CRYPT_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <openssl/aes.h>
+#include <openssl/evp.h>
+#include <openssl/sha.h>
+#include <openssl/hmac.h>
+#include <openssl/cmac.h>
+#include <openssl/modes.h>
+#include "crypt-mem-types.h"
+
+#define CRYPT_XLATOR_ID  (0)
+
+#define MAX_IOVEC_BITS (3)
+#define MAX_IOVEC (1 << MAX_IOVEC_BITS)
+#define KEY_FACTOR_BITS  (6)
+
+#define DEBUG_CRYPT (0)
+#define TRIVIAL_TFM (0)
+
+#define CRYPT_MIN_BLOCK_BITS (9)
+#define CRYPT_MAX_BLOCK_BITS (12)
+
+#define MASTER_VOL_KEY_SIZE (32)
+#define NMTD_VOL_KEY_SIZE (16)
+
+struct crypt_key {
+	uint32_t len;
+	const char *label;	
+};
+
+/*
+ * Add new key types to the end of this
+ * enumeration but before LAST_KEY_TYPE
+ */
+typedef enum {
+	MASTER_VOL_KEY,
+	NMTD_VOL_KEY,
+	NMTD_LINK_KEY,
+	EMTD_FILE_KEY,
+	DATA_FILE_KEY_256,
+	DATA_FILE_KEY_512,
+	LAST_KEY_TYPE
+}crypt_key_type;
+
+struct kderive_context {
+	const unsigned char *pkey;/* parent key */
+	uint32_t pkey_len;        /* parent key size, bits */
+	uint32_t ckey_len;        /* child key size, bits */
+	unsigned char *fid;       /* fixed input data, NIST 800-108, 5.1 */
+	uint32_t fid_len;         /* fid len, bytes */
+	unsigned char *out;       /* contains child keying material */
+	uint32_t out_len;         /* out len, bytes */
+};
+
+typedef enum {
+	DATA_ATOM,
+	HOLE_ATOM,
+	LAST_DATA_TYPE
+}atom_data_type;
+
+typedef enum {
+	HEAD_ATOM,
+	TAIL_ATOM,
+	FULL_ATOM,
+	LAST_LOCALITY_TYPE
+}atom_locality_type;
+
+typedef enum {
+	MTD_CREATE,
+	MTD_APPEND,
+	MTD_OVERWRITE,
+	MTD_CUT,
+	MTD_LAST_OP
+} mtd_op_t;
+
+struct xts128_context {
+	void      *key1, *key2;
+	block128_f block1,block2;
+};
+
+struct object_cipher_info {
+	cipher_alg_t  o_alg;
+        cipher_mode_t o_mode;
+	uint32_t      o_block_bits;
+        uint32_t      o_dkey_size; /* raw data key size in bits */
+	union {
+		struct {
+			unsigned char ivec[16];
+			AES_KEY dkey[2];
+			AES_KEY tkey;    /* key used for tweaking */
+			XTS128_CONTEXT xts;
+		} aes_xts;
+	} u;
+};
+
+struct master_cipher_info {
+	/*
+	 * attributes inherited by newly created regular files
+	 */
+	cipher_alg_t  m_alg;
+        cipher_mode_t m_mode;
+	uint32_t      m_block_bits;
+        uint32_t      m_dkey_size; /* raw key size in bits */
+	/*
+	 * master key
+	 */
+	unsigned char m_key[MASTER_VOL_KEY_SIZE];
+	/* 
+	 * volume key for oid authentication
+	 */
+	unsigned char m_nmtd_key[NMTD_VOL_KEY_SIZE];
+};
+
+/*
+* This info is not changed during file's life
+ */
+struct crypt_inode_info {
+#if DEBUG_CRYPT
+	loc_t *loc; /* pathname that the file has been
+		       opened, or created with */
+#endif
+	uint16_t nr_minor;
+	uuid_t oid;
+	struct object_cipher_info cinfo;
+};
+
+/*
+ * this should locate in secure memory
+ */
+typedef struct {
+	struct master_cipher_info master;
+} crypt_private_t;
+
+static inline struct master_cipher_info *get_master_cinfo(crypt_private_t *priv)
+{
+	return &priv->master;
+}
+
+static inline struct object_cipher_info *get_object_cinfo(struct crypt_inode_info
+							 *info)
+{
+	return &info->cinfo;
+}
+
+/*
+ * this describes layouts and properties
+ * of atoms in an aligned vector
+ */
+struct avec_config {
+	uint32_t atom_size;
+	atom_data_type type;
+	size_t orig_size;
+	off_t orig_offset;
+	size_t expanded_size;
+	off_t aligned_offset;
+
+	uint32_t off_in_head;
+	uint32_t off_in_tail;
+	uint32_t gap_in_tail;
+	uint32_t nr_full_blocks;
+
+	struct iovec *avec; /* aligned vector */
+	uint32_t acount; /* number of avec components. The same
+			  * as number of occupied logical blocks */
+	char **pool;
+	uint32_t blocks_in_pool;
+	uint32_t cursor; /* makes sense only for ordered writes,
+			  * so there is no races on this counter.
+			  *
+			  * Cursor is per-config object, we don't
+			  * reset cursor for atoms of different
+			  * localities (head, tail, full)
+			  */
+};
+
+
+typedef struct {
+	glusterfs_fop_t fop; /* code of FOP this local info built for */
+	fd_t *fd;
+	inode_t *inode;
+	loc_t *loc;
+	int32_t mac_idx;
+	loc_t *newloc;
+	int32_t flags;
+	int32_t wbflags;
+	struct crypt_inode_info *info;
+	struct iobref *iobref;
+	struct iobref *iobref_data;
+	off_t offset;
+
+	uint64_t old_file_size; /* per FOP, retrieved under lock held */
+	uint64_t cur_file_size; /* per iteration, before issuing IOs */
+	uint64_t new_file_size; /* per iteration, after issuing IOs */
+
+	uint64_t io_offset; /* offset of IOs issued per iteration */
+	uint64_t io_offset_nopad; /* offset of user's data in the atom */
+	uint32_t io_size; /* size of IOs issued per iteration */
+	uint32_t io_size_nopad; /* size of user's data in the IOs */
+	uint32_t eof_padding_size; /* size od EOF padding in the IOs */
+
+	gf_lock_t call_lock; /* protect nr_calls from many cbks */
+	int32_t nr_calls;
+
+	atom_data_type active_setup; /* which setup (hole or date)
+					is currently active */
+	/* data setup */
+	struct avec_config data_conf;
+
+	/* hole setup */
+	int hole_conv_in_proggress;
+	gf_lock_t hole_lock; /* protect hole config from many cbks */
+	int hole_handled;
+	struct avec_config hole_conf;
+	struct iatt buf;
+	struct iatt prebuf;
+	struct iatt postbuf;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+	int32_t op_ret;
+	int32_t op_errno;
+	int32_t rw_count; /* total read or written */
+	gf_lock_t rw_count_lock; /* protect the counter above */
+	unsigned char *format; /* for create, update format string */
+	uint32_t format_size;
+	uint32_t msgflags; /* messages for crypt_open() */
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iovec vec; /* contains last file's atom for
+			     read-prune-write sequence */
+	gf_boolean_t custom_mtd;
+	/*
+	 * the next 3 fields are used by readdir and friends
+	 */
+	gf_dirent_t *de; /* directory entry */
+	char *de_path; /* pathname of directory entry */
+	uint32_t de_prefix_len; /* lenght of the parent's pathname */
+	gf_dirent_t *entries;
+
+	uint32_t update_disk_file_size:1;
+} crypt_local_t;
+
+/* This represents a (read)modify-write atom */
+struct rmw_atom {
+	atom_locality_type locality;
+	/*
+	 * read-modify-write sequence of the atom
+	 */
+	int32_t (*rmw)(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iovec *vec,
+		       int32_t count,
+		       struct iatt *stbuf,
+		       struct iobref *iobref,
+		       dict_t *xdata);
+	/*
+	 * offset of the logical block in a file
+	 */
+	loff_t (*offset_at)(call_frame_t *frame,
+			    struct object_cipher_info *object);
+	/*
+	 * IO offset in an atom
+	 */
+	uint32_t (*offset_in)(call_frame_t *frame,
+			      struct object_cipher_info *object);
+	/*
+	 * number of bytes of plain text of this atom that user
+	 * wants to read/write.
+	 * It can be smaller than atom_size in the case of head
+	 * or tail atoms.
+	 */
+	uint32_t (*io_size_nopad)(call_frame_t *frame,
+				  struct object_cipher_info *object);
+	/*
+	 * which iovec represents the atom
+	 */
+	struct iovec *(*get_iovec)(call_frame_t *frame, uint32_t count);
+	/*
+	 * how many bytes of partial block should be uptodated by
+	 * reading from disk.
+	 * This is used to perform a read component of RMW (read-modify-write).
+	 */
+	uint32_t (*count_to_uptodate)(call_frame_t *frame, struct object_cipher_info *object);
+	struct avec_config *(*get_config)(call_frame_t *frame);
+};
+
+struct data_cipher_alg {
+	gf_boolean_t atomic; /* true means that algorithm requires
+				to pad data before cipher transform */
+	gf_boolean_t should_pad; /* true means that algorithm requires
+				    to pad the end of file with extra-data */
+	uint32_t blkbits; /* blksize = 1 << blkbits */
+	/*
+	 * any preliminary sanity checks goes here
+	 */
+	int32_t (*init)(void);
+	/*
+	 * set alg-mode specific inode info
+	 */
+	int32_t (*set_private)(struct crypt_inode_info *info,
+			       struct master_cipher_info *master);
+	/*
+	 * check alg-mode specific data key
+	 */
+	int32_t (*check_key)(uint32_t key_size);
+	void (*set_iv)(off_t offset, struct object_cipher_info *object);
+	int32_t (*encrypt)(const unsigned char *from, unsigned char *to,
+			   size_t length, off_t offset, const int enc,
+			   struct object_cipher_info *object);
+};
+
+/*
+ * version-dependent metadata loader
+ */
+struct crypt_mtd_loader {
+	/*
+	 * return core format size
+	 */
+	size_t (*format_size)(mtd_op_t op, size_t old_size);
+	/*
+	 * pack version-specific metadata of an object
+	 * at ->create()
+	 */
+	int32_t (*create_format)(unsigned char *wire,
+				 loc_t *loc,
+				 struct crypt_inode_info *info,
+				 struct master_cipher_info *master);
+	/*
+	 * extract version-specific metadata of an object
+	 * at ->open() time
+	 */
+	int32_t (*open_format)(unsigned char *wire,
+			       int32_t len,
+			       loc_t *loc,
+			       struct crypt_inode_info *info,
+			       struct master_cipher_info *master,
+			       crypt_local_t *local,
+			       gf_boolean_t load_info);
+	int32_t (*update_format)(unsigned char *new,
+				 unsigned char *old,
+				 size_t old_len,
+				 int32_t mac_idx,
+				 mtd_op_t op,
+				 loc_t *loc,
+				 struct crypt_inode_info *info,
+				 struct master_cipher_info *master,
+				 crypt_local_t *local);
+};
+
+typedef int32_t (*end_writeback_handler_t)(call_frame_t *frame,
+					   void *cookie,
+					   xlator_t *this,
+					   int32_t op_ret,
+					   int32_t op_errno,
+					   struct iatt *prebuf,
+					   struct iatt *postbuf,
+					   dict_t *xdata);
+typedef void (*linkop_wind_handler_t)(call_frame_t *frame, xlator_t *this);
+typedef void (*linkop_unwind_handler_t)(call_frame_t *frame);
+
+
+/* Declarations */
+
+/* keys.c */
+extern struct crypt_key crypt_keys[LAST_KEY_TYPE];
+int32_t get_nmtd_vol_key(struct master_cipher_info *master);
+int32_t get_nmtd_link_key(loc_t *loc,
+			  struct master_cipher_info *master,
+			  unsigned char *result);
+int32_t get_emtd_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  unsigned char *result);
+int32_t get_data_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  uint32_t keysize,
+			  unsigned char *key);
+/* data.c */
+extern struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE];
+void encrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off);
+void decrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off);
+int32_t align_iov_by_atoms(xlator_t *this,
+			   crypt_local_t *local,
+			   struct object_cipher_info *object,
+			   struct iovec *vec /* input vector */,
+			   int32_t count /* number of vec components */,
+			   struct iovec *avec /* aligned vector */,
+			   char **blocks /* pool of blocks */,
+			   uint32_t *blocks_allocated,
+			   struct avec_config *conf);
+int32_t set_config_avec_data(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     struct iovec *vec,
+			     int32_t vec_count);
+int32_t set_config_avec_hole(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     glusterfs_fop_t fop);
+void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
+		    struct avec_config *conf, atom_data_type dtype);
+void set_config_offsets(call_frame_t *frame,
+			xlator_t *this,
+			uint64_t offset,
+			uint64_t count,
+			atom_data_type dtype,
+			int32_t setup_gap_in_tail);
+
+/* metadata.c */
+extern struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER];
+
+int32_t alloc_format(crypt_local_t *local, size_t size);
+int32_t alloc_format_create(crypt_local_t *local);
+void free_format(crypt_local_t *local);
+size_t format_size(mtd_op_t op, size_t old_size);
+size_t new_format_size(void);
+int32_t open_format(unsigned char *str, int32_t len, loc_t *loc,
+		    struct crypt_inode_info *info,
+		    struct master_cipher_info *master, crypt_local_t *local,
+		    gf_boolean_t load_info);
+int32_t update_format(unsigned char *new, unsigned char *old,
+		      size_t old_len, int32_t mac_idx, mtd_op_t op, loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master,
+		      crypt_local_t *local);
+int32_t create_format(unsigned char *wire,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master);
+
+/* atom.c */
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality);
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype);
+void submit_full(call_frame_t *frame, xlator_t *this);
+
+/* crypt.c */
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop);
+static size_t iovec_get_size(struct iovec *vec, uint32_t count);
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom, off_t io_offset,
+				uint32_t io_size);
+void link_wind(call_frame_t *frame, xlator_t *this);
+void unlink_wind(call_frame_t *frame, xlator_t *this);
+void link_unwind(call_frame_t *frame);
+void unlink_unwind(call_frame_t *frame);
+void rename_wind(call_frame_t *frame, xlator_t *this);
+void rename_unwind(call_frame_t *frame);
+
+/* Inline functions */
+
+static inline size_t iovec_get_size(struct iovec *vec, uint32_t count)
+{
+	int i;
+	size_t size = 0;
+	for (i = 0; i < count; i++)
+		size += vec[i].iov_len;
+	return size;
+}
+
+static inline int32_t crypt_xlator_id(void)
+{
+	return CRYPT_XLATOR_ID;
+}
+
+static inline mtd_loader_id current_mtd_loader(void)
+{
+	return MTD_LOADER_V1;
+}
+
+static inline uint32_t master_key_size (void)
+{
+	return crypt_keys[MASTER_VOL_KEY].len >> 3;
+}
+
+static inline uint32_t nmtd_vol_key_size (void)
+{
+	return crypt_keys[NMTD_VOL_KEY].len >> 3;
+}
+
+static inline uint32_t alg_mode_blkbits(cipher_alg_t alg,
+					cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].blkbits;
+}
+
+static inline uint32_t alg_mode_blksize(cipher_alg_t alg,
+					cipher_mode_t mode)
+{
+	return 1 << alg_mode_blkbits(alg, mode);
+}
+
+static inline gf_boolean_t alg_mode_atomic(cipher_alg_t alg,
+					   cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].atomic;
+}
+
+static inline gf_boolean_t alg_mode_should_pad(cipher_alg_t alg,
+					       cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].should_pad;
+}
+
+static inline uint32_t master_alg_blksize(struct master_cipher_info *mr)
+{
+	return alg_mode_blksize(mr->m_alg, mr->m_mode);
+}
+
+static inline uint32_t master_alg_blkbits(struct master_cipher_info *mr)
+{
+	return alg_mode_blkbits(mr->m_alg, mr->m_mode);
+}
+
+static inline gf_boolean_t master_alg_atomic(struct master_cipher_info *mr)
+{
+	return alg_mode_atomic(mr->m_alg, mr->m_mode);
+}
+
+static inline gf_boolean_t master_alg_should_pad(struct master_cipher_info *mr)
+{
+	return alg_mode_should_pad(mr->m_alg, mr->m_mode);
+}
+
+static inline uint32_t object_alg_blksize(struct object_cipher_info *ob)
+{
+	return alg_mode_blksize(ob->o_alg, ob->o_mode);
+}
+
+static inline uint32_t object_alg_blkbits(struct object_cipher_info *ob)
+{
+	return alg_mode_blkbits(ob->o_alg, ob->o_mode);
+}
+
+static inline gf_boolean_t object_alg_atomic(struct object_cipher_info *ob)
+{
+	return alg_mode_atomic(ob->o_alg, ob->o_mode);
+}
+
+static inline gf_boolean_t object_alg_should_pad(struct object_cipher_info *ob)
+{
+	return alg_mode_should_pad(ob->o_alg, ob->o_mode);
+}
+
+static inline uint32_t aes_raw_key_size(struct master_cipher_info *master)
+{
+	return master->m_dkey_size >> 3;
+}
+
+static inline struct avec_config *get_hole_conf(call_frame_t *frame)
+{
+	return &(((crypt_local_t *)frame->local)->hole_conf);
+}
+
+static inline struct avec_config *get_data_conf(call_frame_t *frame)
+{
+	return &(((crypt_local_t *)frame->local)->data_conf);
+}
+
+static inline int32_t get_atom_bits (struct object_cipher_info *object)
+{
+	return object->o_block_bits;
+}
+
+static inline int32_t get_atom_size (struct object_cipher_info *object)
+{
+	return 1 << get_atom_bits(object);
+}
+
+static inline int32_t has_head_block(struct avec_config *conf)
+{
+	return conf->off_in_head ||
+		(conf->acount == 1 && conf->off_in_tail);
+}
+
+static inline int32_t has_tail_block(struct avec_config *conf)
+{
+	return conf->off_in_tail && conf->acount > 1;
+}
+
+static inline int32_t has_full_blocks(struct avec_config *conf)
+{
+	return conf->nr_full_blocks;
+}
+
+static inline int32_t should_submit_head_block(struct avec_config *conf)
+{
+	return has_head_block(conf) && (conf->cursor == 0);
+}
+
+static inline int32_t should_submit_tail_block(struct avec_config *conf)
+{
+	return has_tail_block(conf) && (conf->cursor == conf->acount - 1);
+}
+
+static inline int32_t should_submit_full_block(struct avec_config *conf)
+{
+	uint32_t start = has_head_block(conf) ? 1 : 0;
+
+	return has_full_blocks(conf) &&
+		conf->cursor >= start &&
+		conf->cursor < start + conf->nr_full_blocks;
+}
+
+#if DEBUG_CRYPT
+static inline void crypt_check_input_len(size_t len,
+					 struct object_cipher_info *object)
+{
+	if (object_alg_should_pad(object) && (len & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG, "bad input len: %d", (int)len);
+}
+
+static inline void check_head_block(struct avec_config *conf)
+{
+	if (!has_head_block(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a head atom");
+}
+
+static inline void check_tail_block(struct avec_config *conf)
+{
+	if (!has_tail_block(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a tail atom");
+}
+
+static inline void check_full_block(struct avec_config *conf)
+{
+	if (!has_full_blocks(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a full atom");
+}
+
+static inline void check_cursor_head(struct avec_config *conf)
+{
+	if (!has_head_block(conf))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Illegal call of head atom method");
+	else if (conf->cursor != 0)
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Cursor (%d) is not at head atom",
+		       conf->cursor);
+}
+
+static inline void check_cursor_full(struct avec_config *conf)
+{
+	if (!has_full_blocks(conf))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Illegal call of full atom method");
+	if (has_head_block(conf) && (conf->cursor == 0))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Cursor is not at full atom");
+}
+
+/*
+ * FIXME: use avec->iov_len to check setup
+ */
+static inline int data_local_invariant(crypt_local_t *local)
+{
+	return 0;
+}
+
+#else
+#define crypt_check_input_len(len, object) noop
+#define check_head_block(conf) noop
+#define check_tail_block(conf) noop
+#define check_full_block(conf) noop
+#define check_cursor_head(conf) noop
+#define check_cursor_full(conf) noop
+
+#endif /* DEBUG_CRYPT */
+
+static inline struct avec_config *conf_by_type(call_frame_t *frame,
+					       atom_data_type dtype)
+{
+	struct avec_config *conf = NULL;
+
+	switch (dtype) {
+	case HOLE_ATOM:
+		conf = get_hole_conf(frame);
+		break;
+	case DATA_ATOM:
+		conf = get_data_conf(frame);
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad atom type");
+	}
+	return conf;
+}
+
+static inline uint32_t nr_calls_head(struct avec_config *conf)
+{
+	return has_head_block(conf) ? 1 : 0;
+}
+
+static inline uint32_t nr_calls_tail(struct avec_config *conf)
+{
+	return has_tail_block(conf) ? 1 : 0;
+}
+
+static inline uint32_t nr_calls_full(struct avec_config *conf)
+{
+	switch(conf->type) {
+	case HOLE_ATOM:
+		return has_full_blocks(conf);
+	case DATA_ATOM:
+		return has_full_blocks(conf) ?
+			logical_blocks_occupied(0,
+						conf->nr_full_blocks,
+						MAX_IOVEC_BITS)	: 0;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad atom data type");
+		return 0;
+	}
+}
+
+static inline uint32_t nr_calls(struct avec_config *conf)
+{
+	return nr_calls_head(conf) + nr_calls_tail(conf) + nr_calls_full(conf);
+}
+
+static inline uint32_t nr_calls_data(call_frame_t *frame)
+{
+	return nr_calls(get_data_conf(frame));
+}
+
+static inline uint32_t nr_calls_hole(call_frame_t *frame)
+{
+	return nr_calls(get_hole_conf(frame));
+}
+
+static inline void get_one_call_nolock(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+
+	++local->nr_calls;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", 1);
+}
+
+static inline void get_one_call(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+
+	LOCK(&local->call_lock);
+	get_one_call_nolock(frame);
+	UNLOCK(&local->call_lock);
+}
+
+static inline void get_nr_calls_nolock(call_frame_t *frame, int32_t nr)
+{
+	crypt_local_t *local = frame->local;
+
+	local->nr_calls += nr;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", nr);
+}
+
+static inline void get_nr_calls(call_frame_t *frame, int32_t nr)
+{
+	crypt_local_t *local = frame->local;
+
+	LOCK(&local->call_lock);
+	get_nr_calls_nolock(frame, nr);
+	UNLOCK(&local->call_lock);
+}
+
+static inline int put_one_call(crypt_local_t *local)
+{
+	uint32_t last = 0;
+
+	LOCK(&local->call_lock);
+	if (--local->nr_calls == 0)
+		last = 1;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "put %d calls", 1);
+
+	UNLOCK(&local->call_lock);
+	return last;
+}
+
+static inline int is_appended_write(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->orig_offset + conf->orig_size > local->old_file_size;
+}
+
+static inline int is_ordered_mode(call_frame_t *frame)
+{
+#if 0
+	crypt_local_t *local = frame->local;
+	return local->fop == GF_FOP_FTRUNCATE ||
+		(local->fop == GF_FOP_WRITE && is_appended_write(frame));
+#endif
+	return 1;
+}
+
+static inline int32_t hole_conv_completed(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+	return conf->cursor == conf->acount;
+}
+
+static inline int32_t data_write_in_progress(crypt_local_t *local)
+{
+	return local->active_setup == DATA_ATOM;
+}
+
+static inline int32_t parent_is_crypt_xlator(call_frame_t *frame,
+					     xlator_t *this)
+{
+	return frame->parent->this == this;
+}
+
+static inline linkop_wind_handler_t linkop_wind_dispatch(glusterfs_fop_t fop)
+{
+	switch(fop){
+	case GF_FOP_LINK:
+		return link_wind;
+	case GF_FOP_UNLINK:
+		return unlink_wind;
+	case GF_FOP_RENAME:
+		return rename_wind;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
+		return NULL;
+	}
+}
+
+static inline linkop_unwind_handler_t linkop_unwind_dispatch(glusterfs_fop_t fop)
+{
+	switch(fop){
+	case GF_FOP_LINK:
+		return link_unwind;
+	case GF_FOP_UNLINK:
+		return unlink_unwind;
+	case GF_FOP_RENAME:
+		return rename_unwind;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
+		return NULL;		
+	}	
+}
+
+static inline mtd_op_t linkop_mtdop_dispatch(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_LINK:
+		return MTD_APPEND;
+	case GF_FOP_UNLINK:
+		return MTD_CUT;
+	case GF_FOP_RENAME:
+		return MTD_OVERWRITE;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad link operation %d", fop);
+		return MTD_LAST_OP;
+	}
+}
+
+#endif /* __CRYPT_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/data.c b/xlators/encryption/crypt/src/data.c
new file mode 100644
index 000000000..762fa554a
--- /dev/null
+++ b/xlators/encryption/crypt/src/data.c
@@ -0,0 +1,769 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void set_iv_aes_xts(off_t offset, struct object_cipher_info *object)
+{
+	unsigned char *ivec;
+
+	ivec = object->u.aes_xts.ivec;
+
+	/* convert the tweak into a little-endian byte
+	 * array (IEEE P1619/D16, May 2007, section 5.1)
+	 */
+
+	*((uint64_t *)ivec) = htole64(offset);
+
+	/* ivec is padded with zeroes */
+}
+
+static int32_t aes_set_keys_common(unsigned char *raw_key, uint32_t key_size,
+				   AES_KEY *keys)
+{
+	int32_t ret;
+
+	ret = AES_set_encrypt_key(raw_key,
+				  key_size,
+				  &keys[AES_ENCRYPT]);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set encrypt key failed");
+		return ret;
+	}
+	ret = AES_set_decrypt_key(raw_key,
+				  key_size,
+				  &keys[AES_DECRYPT]);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set decrypt key failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * set private cipher info for xts mode
+ */
+static int32_t set_private_aes_xts(struct crypt_inode_info *info,
+				   struct master_cipher_info *master)
+{
+	int ret;
+	struct object_cipher_info *object = get_object_cinfo(info);
+	unsigned char *data_key;
+	uint32_t subkey_size;
+
+	/* init tweak value */
+	memset(object->u.aes_xts.ivec, 0, 16);
+
+	data_key = GF_CALLOC(1, object->o_dkey_size, gf_crypt_mt_key);
+	if (!data_key)
+		return ENOMEM;
+
+	/*
+	 * retrieve data keying meterial
+	 */
+	ret = get_data_file_key(info, master, object->o_dkey_size, data_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Failed to retrieve data key");
+		GF_FREE(data_key);
+		return ret;
+	}
+	/*
+	 * parse compound xts key
+	 */
+	subkey_size = object->o_dkey_size >> 4; /* (xts-key-size-in-bytes / 2) */
+	/*
+	 * install key for data encryption
+	 */
+	ret = aes_set_keys_common(data_key,
+				  subkey_size << 3, object->u.aes_xts.dkey);
+	if (ret) {
+		GF_FREE(data_key);
+		return ret;
+	}
+	/*
+	 * set up key used to encrypt tweaks
+	 */
+	ret = AES_set_encrypt_key(data_key + subkey_size,
+				  object->o_dkey_size / 2,
+				  &object->u.aes_xts.tkey);
+	if (ret < 0)
+		gf_log("crypt", GF_LOG_ERROR, "Set tweak key failed");
+
+	GF_FREE(data_key);
+	return ret;
+}
+
+static int32_t aes_xts_init(void)
+{
+	cassert(AES_BLOCK_SIZE == (1 << AES_BLOCK_BITS));
+	return 0;
+}
+
+static int32_t check_key_aes_xts(uint32_t keysize)
+{
+	switch(keysize) {
+	case 256:
+	case 512:
+		return 0;
+	default:
+		break;
+	}
+	return -1;
+}
+
+static int32_t encrypt_aes_xts(const unsigned char *from,
+			       unsigned char *to, size_t length,
+			       off_t offset, const int enc,
+			       struct object_cipher_info *object)
+{
+	XTS128_CONTEXT ctx;
+	if (enc) {
+		ctx.key1 = &object->u.aes_xts.dkey[AES_ENCRYPT];
+		ctx.block1 = (block128_f)AES_encrypt;
+	}
+	else {
+		ctx.key1 = &object->u.aes_xts.dkey[AES_DECRYPT];
+		ctx.block1 = (block128_f)AES_decrypt;
+	}
+	ctx.key2 = &object->u.aes_xts.tkey;
+	ctx.block2 = (block128_f)AES_encrypt;
+
+	return CRYPTO_xts128_encrypt(&ctx,
+				     object->u.aes_xts.ivec,
+				     from,
+				     to,
+				     length, enc);
+}
+
+/*
+ * Cipher input chunk @from of length @len;
+ * @to: result of cipher transform;
+ * @off: offset in a file (must be cblock-aligned);
+ */
+static void cipher_data(struct object_cipher_info *object,
+			char *from,
+			char *to,
+			off_t off,
+			size_t len,
+			const int enc)
+{
+	crypt_check_input_len(len, object);
+
+#if TRIVIAL_TFM && DEBUG_CRYPT
+	return;
+#endif
+	data_cipher_algs[object->o_alg][object->o_mode].set_iv(off, object);
+	data_cipher_algs[object->o_alg][object->o_mode].encrypt
+		((const unsigned char *)from,
+		 (unsigned char *)to,
+		 len,
+		 off,
+		 enc,
+		 object);
+}
+
+#define MAX_CIPHER_CHUNK (1 << 30)
+
+/*
+ * Do cipher (encryption/decryption) transform of a
+ * continuous region of memory.
+ *
+ * @len: a number of bytes to transform;
+ * @buf: data to transform;
+ * @off: offset in a file, should be block-aligned
+ *       for atomic cipher modes and ksize-aligned
+ *       for other modes).
+ * @dir: direction of transform (encrypt/decrypt).
+ */
+static void cipher_region(struct object_cipher_info *object,
+			  char *from,
+			  char *to,
+			  off_t off,
+			  size_t len,
+			  int dir)
+{
+	while (len > 0) {
+		size_t to_cipher;
+
+		to_cipher = len;
+		if (to_cipher > MAX_CIPHER_CHUNK)
+			to_cipher = MAX_CIPHER_CHUNK;
+
+		/* this will reset IV */
+		cipher_data(object,
+			    from,
+			    to,
+			    off,
+			    to_cipher,
+			    dir);
+		from += to_cipher;
+		to   += to_cipher;
+		off  += to_cipher;
+		len  -= to_cipher;
+	}
+}
+
+/*
+ * Do cipher transform (encryption/decryption) of
+ * plaintext/ciphertext represented by @vec.
+ *
+ * Pre-conditions: @vec represents a continuous piece
+ * of data in a file at offset @off to be ciphered
+ * (encrypted/decrypted).
+ * @count is the number of vec's components. All the
+ * components must be block-aligned, the caller is
+ * responsible for this. @dir is "direction" of
+ * transform (encrypt/decrypt).
+ */
+static void cipher_aligned_iov(struct object_cipher_info *object,
+			       struct iovec *vec,
+			       int count,
+			       off_t off,
+			       int32_t dir)
+{
+	int i;
+	int len = 0;
+
+	for (i = 0; i < count; i++) {
+		cipher_region(object,
+			      vec[i].iov_base,
+			      vec[i].iov_base,
+			      off + len,
+			      vec[i].iov_len,
+			      dir);
+		len += vec[i].iov_len;
+	}
+}
+
+void encrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off)
+{
+	cipher_aligned_iov(object, vec, count, off, 1);
+}
+
+void decrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off)
+{
+	cipher_aligned_iov(object, vec, count, off, 0);
+}
+
+#if DEBUG_CRYPT
+static void compound_stream(struct iovec *vec, int count, char *buf, off_t skip)
+{
+	int i;
+	int off = 0;
+	for (i = 0; i < count; i++) {
+		memcpy(buf + off,
+		       vec[i].iov_base + skip,
+		       vec[i].iov_len - skip);
+
+		off += (vec[i].iov_len - skip);
+		skip = 0;
+	}
+}
+
+static void check_iovecs(struct iovec *vec, int cnt,
+			 struct iovec *avec, int acnt, uint32_t off_in_head)
+{
+	char *s1, *s2;
+	uint32_t size, asize;
+
+	size = iovec_get_size(vec, cnt);
+	asize = iovec_get_size(avec, acnt) - off_in_head;
+	if (size != asize) {
+		gf_log("crypt", GF_LOG_DEBUG, "size %d is not eq asize %d",
+		       size, asize);
+		return;
+	}
+	s1 = GF_CALLOC(1, size, gf_crypt_mt_data);
+	if (!s1) {
+		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
+		return;
+	}
+	s2 = GF_CALLOC(1, asize, gf_crypt_mt_data);
+	if (!s2) {
+		GF_FREE(s1);
+		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
+		return;
+	}
+	compound_stream(vec, cnt, s1, 0);
+	compound_stream(avec, acnt, s2, off_in_head);
+	if (memcmp(s1, s2, size))
+		gf_log("crypt", GF_LOG_DEBUG, "chunks of different data");
+	GF_FREE(s1);
+	GF_FREE(s2);
+}
+
+#else
+#define check_iovecs(vec, count, avec, avecn, off) noop
+#endif /* DEBUG_CRYPT */
+
+static char *data_alloc_block(xlator_t *this, crypt_local_t *local,
+			      int32_t block_size)
+{
+	struct iobuf *iobuf = NULL;
+
+	iobuf = iobuf_get2(this->ctx->iobuf_pool, block_size);
+	if (!iobuf) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Failed to get iobuf");
+		return NULL;
+	}
+	if (!local->iobref_data) {
+		local->iobref_data = iobref_new();
+		if (!local->iobref_data) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "Failed to get iobref");
+			iobuf_unref(iobuf);
+			return NULL;
+		}
+	}
+	iobref_add(local->iobref_data, iobuf);
+	return iobuf->ptr;
+}
+
+/*
+ * Compound @avec, which represent the same data
+ * chunk as @vec, but has aligned components of
+ * specified block size. Alloc blocks, if needed.
+ * In particular, incomplete head and tail blocks
+ * must be allocated.
+ * Put number of allocated blocks to @num_blocks.
+ *
+ * Example:
+ *
+ * input: data chunk represented by 4 components
+ * [AB],[BC],[CD],[DE];
+ * output: 5 logical blocks (0, 1, 2, 3, 4).
+ *
+ *   A     B       C     D               E
+ *   *-----*+------*-+---*----+--------+-*
+ *   |     ||      | |   |    |        | |
+ * *-+-----+*------+-*---+----*--------*-+------*
+ * 0        1        2        3        4
+ *
+ * 0 - incomplete compound (head);
+ * 1, 2 - full compound;
+ * 3 - full non-compound (the case of reuse);
+ * 4 - incomplete non-compound (tail).
+ */
+int32_t align_iov_by_atoms(xlator_t *this,
+			   crypt_local_t *local,
+			   struct object_cipher_info *object,
+			   struct iovec *vec /* input vector */,
+			   int32_t count /* number of vec components */,
+			   struct iovec *avec /* aligned vector */,
+			   char **blocks /* pool of blocks */,
+			   uint32_t *blocks_allocated,
+			   struct avec_config *conf)
+{
+	int vecn = 0;      /* number of the current component in vec */
+	int avecn = 0;     /* number of the current component in avec */
+	off_t vec_off = 0; /* offset in the current vec component,
+			    * i.e. the number of bytes have already
+			    * been copied */
+	int32_t block_size = get_atom_size(object);
+	size_t to_process; /* number of vec's bytes to copy and(or) re-use */
+	int32_t off_in_head = conf->off_in_head;
+
+	to_process = iovec_get_size(vec, count);
+
+	while (to_process > 0) {
+		if (off_in_head ||
+		    vec[vecn].iov_len - vec_off < block_size) {
+			/*
+			 * less than block_size:
+			 * the case of incomplete (head or tail),
+			 * or compound block
+			 */
+			size_t copied = 0;
+			/*
+			 * populate the pool with a new block
+			 */
+			blocks[*blocks_allocated] = data_alloc_block(this,
+								    local,
+								    block_size);
+			if (!blocks[*blocks_allocated])
+				return -ENOMEM;
+			memset(blocks[*blocks_allocated], 0, off_in_head);
+			/*
+			 * fill the block with vec components
+			 */
+			do {
+				size_t to_copy;
+
+				to_copy = vec[vecn].iov_len - vec_off;
+				if (to_copy > block_size - off_in_head)
+					to_copy = block_size - off_in_head;
+
+				memcpy(blocks[*blocks_allocated] + off_in_head + copied,
+				       vec[vecn].iov_base + vec_off,
+				       to_copy);
+
+				copied += to_copy;
+				to_process -= to_copy;
+
+				vec_off += to_copy;
+				if (vec_off == vec[vecn].iov_len) {
+					/* finished with this vecn */
+					vec_off = 0;
+					vecn++;
+				}
+			} while (copied < (block_size - off_in_head) && to_process > 0);
+			/*
+			 * update avec
+			 */
+			avec[avecn].iov_len = off_in_head + copied;
+			avec[avecn].iov_base = blocks[*blocks_allocated];
+
+			(*blocks_allocated)++;
+			off_in_head = 0;
+		} else {
+			/*
+			 * the rest of the current vec component
+			 * is not less than block_size, so reuse
+			 * the memory buffer of the component.
+			 */
+			size_t to_reuse;
+			to_reuse = (to_process > block_size ?
+				    block_size :
+				    to_process);
+			avec[avecn].iov_len = to_reuse;
+			avec[avecn].iov_base = vec[vecn].iov_base + vec_off;
+
+			vec_off += to_reuse;
+			if (vec_off == vec[vecn].iov_len) {
+				/* finished with this vecn */
+				vec_off = 0;
+				vecn++;
+			}
+			to_process -= to_reuse;
+		}
+		avecn++;
+	}
+	check_iovecs(vec, count, avec, avecn, conf->off_in_head);
+	return 0;
+}
+
+/*
+ * allocate and setup aligned vector for data submission
+ * Pre-condition: @conf is set.
+ */
+int32_t set_config_avec_data(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     struct iovec *vec,
+			     int32_t vec_count)
+{
+	int32_t ret = ENOMEM;
+	struct iovec *avec;
+	char **pool;
+	uint32_t blocks_in_pool = 0;
+
+	conf->type = DATA_ATOM;
+
+	avec = GF_CALLOC(conf->acount, sizeof(*avec), gf_crypt_mt_iovec);
+	if (!avec)
+		return ret;
+	pool = GF_CALLOC(conf->acount, sizeof(pool), gf_crypt_mt_char);
+	if (!pool) {
+		GF_FREE(avec);
+		return ret;
+	}
+	if (!vec) {
+		/*
+		 * degenerated case: no data
+		 */
+		pool[0] = data_alloc_block(this, local, get_atom_size(object));
+		if (!pool[0])
+			goto free;
+		blocks_in_pool = 1;
+		avec->iov_base = pool[0];
+		avec->iov_len = conf->off_in_tail;
+	}
+	else {
+		ret = align_iov_by_atoms(this, local, object, vec, vec_count,
+					 avec, pool, &blocks_in_pool, conf);
+		if (ret)
+			goto free;
+	}
+	conf->avec = avec;
+	conf->pool = pool;
+	conf->blocks_in_pool = blocks_in_pool;
+	return 0;
+ free:
+	GF_FREE(avec);
+	GF_FREE(pool);
+	return ret;
+}
+
+/*
+ * allocate and setup aligned vector for hole submission
+ */
+int32_t set_config_avec_hole(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     glusterfs_fop_t fop)
+{
+	uint32_t i, idx;
+	struct iovec *avec;
+	char **pool;
+	uint32_t num_blocks;
+	uint32_t blocks_in_pool = 0;
+
+	conf->type = HOLE_ATOM;
+
+	num_blocks = conf->acount -
+		(conf->nr_full_blocks ? conf->nr_full_blocks - 1 : 0);
+
+	switch (fop) {
+	case GF_FOP_WRITE:
+		/*
+		 * hole goes before data
+		 */
+		if (num_blocks == 1 && conf->off_in_tail != 0)
+		/*
+		 * we won't submit a hole which fits into
+		 * a data atom: this part of hole will be
+		 * submitted with data write
+		 */
+			return 0;
+		break;
+	case GF_FOP_FTRUNCATE:
+		/*
+		 * expanding truncate, hole goes after data,
+		 * and will be submited in any case.
+ 		 */
+		break;
+	default:
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad file operation %d", fop);
+		return 0;
+	}
+	avec = GF_CALLOC(num_blocks, sizeof(*avec), gf_crypt_mt_iovec);
+	if (!avec)
+		return ENOMEM;
+	pool = GF_CALLOC(num_blocks, sizeof(pool), gf_crypt_mt_char);
+	if (!pool) {
+		GF_FREE(avec);
+		return ENOMEM;
+	}
+	for (i = 0; i < num_blocks; i++) {
+		pool[i] = data_alloc_block(this, local, get_atom_size(object));
+		if (pool[i] == NULL)
+			goto free;
+		blocks_in_pool++;
+	}
+	if (has_head_block(conf)) {
+		/* set head block */
+		idx = 0;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		memset(avec[idx].iov_base + conf->off_in_head,
+		       0,
+		       get_atom_size(object) - conf->off_in_head);
+	}
+	if (has_tail_block(conf)) {
+		/* set tail block */
+		idx = num_blocks - 1;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		memset(avec[idx].iov_base, 0, conf->off_in_tail);
+	}
+	if (has_full_blocks(conf)) {
+		/* set full block */
+		idx = conf->off_in_head ? 1 : 0;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		/*
+		 * since we re-use the buffer,
+		 * zeroes will be set every time
+		 * before encryption, see submit_full()
+		 */
+	}
+	conf->avec = avec;
+	conf->pool = pool;
+	conf->blocks_in_pool = blocks_in_pool;
+	return 0;
+ free:
+	GF_FREE(avec);
+	GF_FREE(pool);
+	return ENOMEM;
+}
+
+/* A helper for setting up config of partial atoms (which
+ * participate in read-modify-write sequence).
+ *
+ * Calculate and setup precise amount of "extra-bytes"
+ * that should be uptodated at the end of partial (not
+ * necessarily tail!) block.
+ *
+ * Pre-condition: local->old_file_size is valid!
+ * @conf contains setup, which is enough for correct calculation
+ * of has_tail_block(), ->get_offset().
+ */
+void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
+		    struct avec_config *conf, atom_data_type dtype)
+{
+	uint32_t to_block;
+	crypt_local_t *local = frame->local;
+	uint64_t old_file_size = local->old_file_size;
+	struct rmw_atom *partial = atom_by_types(dtype,
+						 has_tail_block(conf) ?
+						 TAIL_ATOM : HEAD_ATOM);
+
+	if (old_file_size <= partial->offset_at(frame, object))
+		to_block = 0;
+	else {
+		to_block = old_file_size - partial->offset_at(frame, object);
+		if (to_block > get_atom_size(object))
+			to_block = get_atom_size(object);
+	}
+	if (to_block > conf->off_in_tail)
+		conf->gap_in_tail = to_block - conf->off_in_tail;
+	else
+		/*
+		 * nothing to uptodate
+		 */
+		conf->gap_in_tail = 0;
+}
+
+/*
+ * fill struct avec_config with offsets layouts
+ */
+void set_config_offsets(call_frame_t *frame,
+			xlator_t *this,
+			uint64_t offset,
+			uint64_t count,
+			atom_data_type dtype,
+			int32_t set_gap)
+{
+	crypt_local_t *local;
+	struct object_cipher_info *object;
+	struct avec_config *conf;
+	uint32_t resid;
+
+	uint32_t atom_size;
+	uint32_t atom_bits;
+
+	size_t orig_size;
+	off_t orig_offset;
+	size_t expanded_size;
+	off_t aligned_offset;
+
+	uint32_t off_in_head = 0;
+	uint32_t off_in_tail = 0;
+	uint32_t nr_full_blocks;
+	int32_t size_full_blocks;
+
+	uint32_t acount; /* number of alifned components to write.
+			  * The same as number of occupied logical
+			  * blocks (atoms)
+			  */
+	local = frame->local;
+	object = &local->info->cinfo;
+	conf = (dtype == DATA_ATOM ?
+		get_data_conf(frame) : get_hole_conf(frame));
+
+	orig_offset = offset;
+	orig_size = count;
+
+	atom_size = get_atom_size(object);
+	atom_bits = get_atom_bits(object);
+
+	/*
+	 * Round-down the start,
+	 * round-up the end.
+	 */
+	resid = offset & (uint64_t)(atom_size - 1);
+
+	if (resid)
+		off_in_head = resid;
+	aligned_offset = offset - off_in_head;
+	expanded_size = orig_size + off_in_head;
+
+	/* calculate tail,
+	   expand size forward  */
+	resid = (offset + orig_size) & (uint64_t)(atom_size - 1);
+
+	if (resid) {
+		off_in_tail = resid;
+		expanded_size += (atom_size - off_in_tail);
+	}
+	/*
+	 * calculate number of occupied blocks
+	 */
+	acount = expanded_size >> atom_bits;
+	/*
+	 * calculate number of full blocks
+	 */
+	size_full_blocks = expanded_size;
+	if (off_in_head)
+		size_full_blocks -= atom_size;
+	if (off_in_tail && size_full_blocks > 0)
+		size_full_blocks -= atom_size;
+	nr_full_blocks = size_full_blocks >> atom_bits;
+
+	conf->atom_size = atom_size;
+	conf->orig_size = orig_size;
+	conf->orig_offset = orig_offset;
+	conf->expanded_size = expanded_size;
+	conf->aligned_offset = aligned_offset;
+
+	conf->off_in_head = off_in_head;
+	conf->off_in_tail = off_in_tail;
+	conf->nr_full_blocks = nr_full_blocks;
+	conf->acount = acount;
+	/*
+	 * Finally, calculate precise amount of
+	 * "extra-bytes" that should be uptodated
+	 * at the end.
+	 * Only if RMW is expected.
+	 */
+	if (off_in_tail && set_gap)
+		set_gap_at_end(frame, object, conf, dtype);
+}
+
+struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE] = {
+	[AES_CIPHER_ALG][XTS_CIPHER_MODE] =
+	{ .atomic = _gf_true,
+	  .should_pad = _gf_true,
+	  .blkbits = AES_BLOCK_BITS,
+	  .init = aes_xts_init,
+	  .set_private = set_private_aes_xts,
+	  .check_key = check_key_aes_xts,
+	  .set_iv = set_iv_aes_xts,
+	  .encrypt = encrypt_aes_xts
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/keys.c b/xlators/encryption/crypt/src/keys.c
new file mode 100644
index 000000000..4a1d3bb5a
--- /dev/null
+++ b/xlators/encryption/crypt/src/keys.c
@@ -0,0 +1,302 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/* Key hierarchy
+                  
+                   +----------------+
+                   | MASTER_VOL_KEY |
+                   +-------+--------+
+                           |
+                           |
+          +----------------+----------------+
+          |                |                |
+          |                |                |
+  +-------+------+ +-------+-------+ +------+--------+
+  | NMTD_VOL_KEY | | EMTD_FILE_KEY | | DATA_FILE_KEY |
+  +-------+------+ +---------------+ +---------------+
+          |
+          |
+  +-------+-------+
+  | NMTD_LINK_KEY |
+  +---------------+
+
+ */
+
+#if DEBUG_CRYPT
+static void check_prf_iters(uint32_t num_iters)
+{
+	if (num_iters == 0)
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad number of prf iterations : %d", num_iters);
+}
+#else
+#define check_prf_iters(num_iters) noop
+#endif /* DEBUG_CRYPT */
+
+unsigned char crypt_fake_oid[16] =
+	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+/*
+ * derive key in the counter mode using
+ * sha256-based HMAC as PRF, see
+ * NIST Special Publication 800-108, 5.1)
+ */
+
+#define PRF_OUTPUT_SIZE  SHA256_DIGEST_LENGTH
+
+static int32_t kderive_init(struct kderive_context *ctx,
+			    const unsigned char *pkey, /* parent key */
+			    uint32_t pkey_size,  /* parent key size */
+			    const unsigned char *idctx, /* id-context */
+			    uint32_t idctx_size,
+			    crypt_key_type type  /* type of child key */)
+{
+	unsigned char *pos;
+	uint32_t llen = strlen(crypt_keys[type].label);	
+	/*
+	 * Compoud the fixed input data for KDF:
+	 * [i]_2 || Label || 0x00 || Id-Context || [L]_2),
+	 * NIST SP 800-108, 5.1
+	 */
+	ctx->fid_len =
+		sizeof(uint32_t) +
+		llen +
+		1 +
+		idctx_size +
+		sizeof(uint32_t);
+
+	ctx->fid = GF_CALLOC(ctx->fid_len, 1, gf_crypt_mt_key);
+	if (!ctx->fid)
+		return ENOMEM;
+	ctx->out_len = round_up(crypt_keys[type].len >> 3,
+				PRF_OUTPUT_SIZE);
+	ctx->out = GF_CALLOC(ctx->out_len, 1, gf_crypt_mt_key);
+	if (!ctx->out) {
+		GF_FREE(ctx->fid);
+		return ENOMEM;
+	}
+	ctx->pkey = pkey;
+	ctx->pkey_len = pkey_size;
+	ctx->ckey_len = crypt_keys[type].len;
+
+	pos = ctx->fid;
+
+	/* counter will be set up in kderive_rfn() */
+	pos += sizeof(uint32_t);
+
+	memcpy(pos, crypt_keys[type].label, llen);
+	pos += llen;
+
+	/* set up zero octet */
+	*pos = 0;
+	pos += 1;
+
+	memcpy(pos, idctx, idctx_size);
+	pos += idctx_size;
+
+	*((uint32_t *)pos) = htobe32(ctx->ckey_len);
+
+	return 0;
+}
+
+static void kderive_update(struct kderive_context *ctx)
+{	
+	uint32_t i;
+	HMAC_CTX hctx;
+	unsigned char *pos = ctx->out;
+	uint32_t *p_iter = (uint32_t *)ctx->fid;
+	uint32_t num_iters = ctx->out_len / PRF_OUTPUT_SIZE;
+
+	check_prf_iters(num_iters);
+
+	HMAC_CTX_init(&hctx);
+	for (i = 0; i < num_iters; i++) {
+		/*
+		 * update the iteration number in the fid
+		 */
+		*p_iter = htobe32(i);
+		HMAC_Init_ex(&hctx,
+			     ctx->pkey, ctx->pkey_len >> 3,
+			     EVP_sha256(),
+			     NULL);
+		HMAC_Update(&hctx, ctx->fid, ctx->fid_len);
+		HMAC_Final(&hctx, pos, NULL);
+
+		pos += PRF_OUTPUT_SIZE;
+	}
+	HMAC_CTX_cleanup(&hctx);
+}
+
+static void kderive_final(struct kderive_context *ctx, unsigned char *child)
+{
+	memcpy(child, ctx->out, ctx->ckey_len >> 3);
+	GF_FREE(ctx->fid);
+	GF_FREE(ctx->out);
+	memset(ctx, 0, sizeof(*ctx));
+}
+
+/*
+ * derive per-volume key for object ids aithentication
+ */
+int32_t get_nmtd_vol_key(struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   crypt_fake_oid, sizeof(uuid_t), NMTD_VOL_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, master->m_nmtd_key);
+	return 0;
+}
+
+/*
+ * derive per-link key for aithentication of non-encrypted
+ * meta-data (nmtd)
+ */
+int32_t get_nmtd_link_key(loc_t *loc,
+			  struct master_cipher_info *master,
+			  unsigned char *result)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_nmtd_key,
+			   nmtd_vol_key_size(),
+			   (const unsigned char *)loc->path,
+			   strlen(loc->path), NMTD_LINK_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, result);
+	return 0;
+}
+
+/*
+ * derive per-file key for encryption and authentication
+ * of encrypted part of metadata (emtd)
+ */
+int32_t get_emtd_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  unsigned char *result)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   info->oid, sizeof(uuid_t), EMTD_FILE_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, result);
+	return 0;
+}
+
+static int32_t data_key_type_by_size(uint32_t keysize, crypt_key_type *type)
+{
+	int32_t ret = 0;
+	switch (keysize) {
+	case 256:
+		*type = DATA_FILE_KEY_256;
+		break;
+	case 512:
+		*type = DATA_FILE_KEY_512;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Unsupported data key size %d",
+		       keysize);
+		ret = ENOTSUP;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * derive per-file key for data encryption
+ */
+int32_t get_data_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  uint32_t keysize,
+			  unsigned char *key)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+	crypt_key_type type;
+
+	ret = data_key_type_by_size(keysize, &type);
+	if (ret)
+		return ret;
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   info->oid, sizeof(uuid_t), type);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, key);
+	return 0;
+}
+
+/*
+ * NOTE: Don't change existing keys: it will break compatibility;
+ */
+struct crypt_key crypt_keys[LAST_KEY_TYPE] = {
+	[MASTER_VOL_KEY] =
+	{ .len = MASTER_VOL_KEY_SIZE << 3,
+	  .label = "volume-master",
+	},
+	[NMTD_VOL_KEY] =
+	{ .len = NMTD_VOL_KEY_SIZE << 3,
+	  .label = "volume-nmtd-key-generation"
+	},
+	[NMTD_LINK_KEY] =
+	{ .len = 128,
+	  .label = "link-nmtd-authentication"
+	},
+	[EMTD_FILE_KEY] =
+	{ .len = 128,
+	  .label = "file-emtd-encryption-and-auth"
+	},
+	[DATA_FILE_KEY_256] =
+	{ .len = 256,
+	  .label = "file-data-encryption-256"
+	},
+	[DATA_FILE_KEY_512] =
+	{ .len = 512,
+	  .label = "file-data-encryption-512"
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/metadata.c b/xlators/encryption/crypt/src/metadata.c
new file mode 100644
index 000000000..36b14c055
--- /dev/null
+++ b/xlators/encryption/crypt/src/metadata.c
@@ -0,0 +1,605 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+#include "metadata.h"
+
+int32_t alloc_format(crypt_local_t *local, size_t size)
+{
+	if (size > 0) {
+		local->format = GF_CALLOC(1, size, gf_crypt_mt_mtd);
+		if (!local->format)
+			return ENOMEM;
+	}
+	local->format_size = size;
+	return 0;
+}
+
+int32_t alloc_format_create(crypt_local_t *local)
+{
+	return alloc_format(local, new_format_size());
+}
+
+void free_format(crypt_local_t *local)
+{
+	GF_FREE(local->format);
+}
+
+/*
+ * Check compatibility with extracted metadata
+ */
+static int32_t check_file_metadata(struct crypt_inode_info *info)
+{
+	struct object_cipher_info *object = &info->cinfo;
+
+	if (info->nr_minor != CRYPT_XLATOR_ID) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported minor subversion %d", info->nr_minor);
+		return EINVAL;
+	}
+	if (object->o_alg > LAST_CIPHER_ALG) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported cipher algorithm %d",
+		       object->o_alg);
+		return EINVAL;
+	}
+	if (object->o_mode > LAST_CIPHER_MODE) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported cipher mode %d",
+		       object->o_mode);
+		return EINVAL;
+	}
+	if (object->o_block_bits < CRYPT_MIN_BLOCK_BITS ||
+	    object->o_block_bits > CRYPT_MAX_BLOCK_BITS) {
+		gf_log("crypt", GF_LOG_WARNING, "unsupported block bits %d",
+		       object->o_block_bits);
+		return EINVAL;
+	}
+	/* TBD: check data key size */
+	return 0;
+}
+
+static size_t format_size_v1(mtd_op_t op, size_t old_size)
+{
+	
+	switch (op) {
+	case MTD_CREATE:
+		return sizeof(struct mtd_format_v1);
+	case MTD_OVERWRITE:
+		return old_size;
+	case MTD_APPEND:
+		return old_size + NMTD_8_MAC_SIZE;
+	case MTD_CUT:
+		if (old_size > sizeof(struct mtd_format_v1))
+			return old_size - NMTD_8_MAC_SIZE;
+		else
+			return 0;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad mtd operation");
+		return 0;
+	}
+}
+
+/*
+ * Calculate size of the updated format string.
+ * Returned zero means that we don't need to update the format string.
+ */
+size_t format_size(mtd_op_t op, size_t old_size)
+{
+	size_t versioned;
+
+	versioned = mtd_loaders[current_mtd_loader()].format_size(op,
+				 old_size - sizeof(struct crypt_format));
+	if (versioned != 0)
+		return versioned + sizeof(struct crypt_format);
+	return 0;
+}
+
+/*
+ * size of the format string of newly created file (nr_links = 1)
+ */
+size_t new_format_size(void)
+{
+	return format_size(MTD_CREATE, 0);
+}
+
+/*
+ * Calculate per-link MAC by pathname
+ */
+static int32_t calc_link_mac_v1(struct mtd_format_v1 *fmt,
+				loc_t *loc,
+				unsigned char *result,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master)
+{
+	int32_t ret;	
+	unsigned char nmtd_link_key[16];
+	CMAC_CTX *cctx;
+	size_t len;
+
+	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not get nmtd link key");
+		return -1;
+	}
+	cctx = CMAC_CTX_new();
+	if (!cctx) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_CTX_new failed");
+		return -1; 
+	}
+	ret = CMAC_Init(cctx, nmtd_link_key, sizeof(nmtd_link_key),
+			EVP_aes_128_cbc(), 0);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Init failed");
+		CMAC_CTX_free(cctx);
+		return -1;
+	}
+	ret = CMAC_Update(cctx, get_NMTD_V1(info), SIZE_OF_NMTD_V1);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Update failed");
+		CMAC_CTX_free(cctx);
+		return -1;
+	}
+	ret = CMAC_Final(cctx, result, &len);
+	CMAC_CTX_free(cctx);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Final failed");
+		return -1;
+	}
+	return 0;
+}
+
+/*
+ * Create per-link MAC of index @idx by pathname
+ */
+static int32_t create_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t idx,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	unsigned char *mac;
+	unsigned char cmac[16];
+
+	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
+
+	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
+	if (ret)
+		return -1;
+	memcpy(mac, cmac, SIZE_OF_NMTD_V1_MAC);
+	return 0;
+}
+
+static int32_t create_format_v1(unsigned char *wire,
+				loc_t *loc,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct mtd_format_v1 *fmt;
+	unsigned char mtd_key[16];
+	AES_KEY EMTD_KEY;
+	unsigned char nmtd_link_key[16];
+	uint32_t ad;
+	GCM128_CONTEXT *gctx;
+
+	fmt = (struct mtd_format_v1 *)wire;
+
+	fmt->minor_id = info->nr_minor;
+	fmt->alg_id = AES_CIPHER_ALG;
+	fmt->dkey_factor = master->m_dkey_size >> KEY_FACTOR_BITS;
+	fmt->block_bits = master->m_block_bits;
+	fmt->mode_id = master->m_mode;
+	/*
+	 * retrieve keys for the parts of metadata
+	 */
+	ret = get_emtd_file_key(info, master, mtd_key);
+	if (ret)
+		return ret;
+	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
+	if (ret)
+		return ret;
+	
+	AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
+
+	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
+
+	/* TBD: Check return values */
+
+	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
+
+	ad = htole32(MTD_LOADER_V1);
+	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	ret = CRYPTO_gcm128_encrypt(gctx,
+				    get_EMTD_V1(fmt),
+				    get_EMTD_V1(fmt),
+				    SIZE_OF_EMTD_V1);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_encrypt failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	/*
+	 * set MAC of encrypted part of metadata
+	 */
+	CRYPTO_gcm128_tag(gctx, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC);
+	CRYPTO_gcm128_release(gctx);
+	/*
+	 * set the first MAC of non-encrypted part of metadata 
+	 */
+	return create_link_mac_v1(fmt, 0, loc, info, master);
+}
+
+/*
+ * Called by fops:
+ * ->create();
+ * ->link();
+ *
+ * Pack common and version-specific parts of file's metadata
+ * Pre-conditions: @info contains valid object-id.
+ */
+int32_t create_format(unsigned char *wire,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master)
+{
+	struct crypt_format *fmt = (struct crypt_format *)wire;
+
+	fmt->loader_id = current_mtd_loader();
+
+	wire += sizeof(struct crypt_format);
+	return mtd_loaders[current_mtd_loader()].create_format(wire, loc,
+							       info, master);
+}
+
+/*
+ * Append or overwrite per-link mac of @mac_idx index
+ * in accordance with the new pathname
+ */
+int32_t appov_link_mac_v1(unsigned char *new,
+			  unsigned char *old,
+			  uint32_t old_size,
+			  int32_t mac_idx,
+			  loc_t *loc,
+			  struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  crypt_local_t *local)
+{
+	memcpy(new, old, old_size);
+	return create_link_mac_v1((struct mtd_format_v1 *)new, mac_idx,
+				  loc, info, master);
+}
+
+/*
+ * Cut per-link mac of @mac_idx index
+ */
+static int32_t cut_link_mac_v1(unsigned char *new,
+			       unsigned char *old,			       
+			       uint32_t old_size,
+			       int32_t mac_idx,
+			       loc_t *loc,
+			       struct crypt_inode_info *info,
+			       struct master_cipher_info *master,
+			       crypt_local_t *local)
+{
+	memcpy(new,
+	       old,
+	       sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1));
+
+	memcpy(new + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1),
+	       old + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx,
+	       old_size - (sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx));
+	return 0;
+}
+
+int32_t update_format_v1(unsigned char *new,
+			 unsigned char *old,
+				size_t old_len,
+				int32_t mac_idx, /* of old name */
+				mtd_op_t op,
+				loc_t *loc,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master,
+				crypt_local_t *local)
+{
+	switch (op) {
+	case MTD_APPEND:
+		mac_idx = 1 + (old_len - sizeof(struct mtd_format_v1))/8;
+	case MTD_OVERWRITE:
+		return appov_link_mac_v1(new, old, old_len, mac_idx,
+					 loc, info, master, local);
+	case MTD_CUT:
+		return cut_link_mac_v1(new, old, old_len, mac_idx,
+				       loc, info, master, local);
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad  mtd operation %d", op);
+		return -1;
+	}
+}
+
+/*
+ * Called by fops:
+ *
+ * ->link()
+ * ->unlink()
+ * ->rename()
+ *
+ */
+int32_t update_format(unsigned char *new,
+		      unsigned char *old,
+		      size_t old_len,
+		      int32_t mac_idx,
+		      mtd_op_t op,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master,
+		      crypt_local_t *local)
+{
+	if (!new)
+		return 0;
+	memcpy(new, old, sizeof(struct crypt_format));
+
+	old += sizeof(struct crypt_format);
+	new += sizeof(struct crypt_format);
+	old_len -= sizeof(struct crypt_format);
+
+	return mtd_loaders[current_mtd_loader()].update_format(new, old,
+							       old_len,
+							       mac_idx, op,
+							       loc, info,
+							       master, local);
+}
+
+/*
+ * Perform preliminary checks of found metadata
+ * Return < 0 on errors;
+ * Return number of object-id MACs (>= 1) on success
+ */
+int32_t check_format_v1(uint32_t len, unsigned char *wire)
+{
+	uint32_t nr_links;
+
+	if (len < sizeof(struct mtd_format_v1)) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "v1-loader: bad metadata size %d", len);
+		goto error;
+	}
+	len -= sizeof(struct mtd_format_v1);
+	if (len % sizeof(nmtd_8_mac_t)) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "v1-loader: bad metadata format");
+		goto error;
+	}
+	nr_links = 1 + len / sizeof(nmtd_8_mac_t);
+	if (nr_links > _POSIX_LINK_MAX)
+		goto error;
+	return nr_links;
+ error:
+	return EIO;
+}
+
+/*
+ * Verify per-link MAC specified by index @idx
+ *
+ * return:
+ * -1 on errors;
+ *  0 on failed verification;
+ *  1 on sucessful verification
+ */
+static int32_t verify_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t idx /* index of the mac to verify */,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	unsigned char *mac;
+	unsigned char cmac[16];
+
+	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
+
+	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
+	if (ret)
+		return -1;
+	if (memcmp(cmac, mac, SIZE_OF_NMTD_V1_MAC))
+		return 0;
+	return 1;
+}
+
+/*
+ * Lookup per-link MAC by pathname.
+ *
+ * return index of the MAC, if it was found;
+ * return < 0 on errors, or if the MAC wasn't found
+ */
+static int32_t lookup_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t nr_macs,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	uint32_t idx;
+
+	for (idx = 0; idx < nr_macs; idx++) {
+		ret = verify_link_mac_v1(fmt, idx, loc, info, master);
+		if (ret < 0)
+			return ret;
+		if (ret > 0)
+			return idx;
+	}
+	return -ENOENT;
+}
+
+/*
+ * Extract version-specific part of metadata
+ */
+static int32_t open_format_v1(unsigned char *wire,
+			      int32_t len,
+			      loc_t *loc,
+			      struct crypt_inode_info *info,
+			      struct master_cipher_info *master,
+			      crypt_local_t *local,
+			      gf_boolean_t load_info)
+{
+	int32_t ret;
+	int32_t num_nmtd_macs;
+	struct mtd_format_v1 *fmt;
+	unsigned char mtd_key[16];
+	AES_KEY EMTD_KEY;
+	GCM128_CONTEXT *gctx;
+	uint32_t ad;
+	emtd_8_mac_t gmac;
+	struct object_cipher_info *object;
+
+	num_nmtd_macs = check_format_v1(len, wire);
+	if (num_nmtd_macs <= 0)
+		return EIO;
+	fmt = (struct mtd_format_v1 *)wire;
+
+	ret = lookup_link_mac_v1(fmt, num_nmtd_macs, loc, info, master);
+	if (ret < 0) {
+		gf_log("crypt", GF_LOG_ERROR, "NMTD verification failed");
+		return EINVAL;
+	}
+	local->mac_idx = ret;
+	if (load_info == _gf_false)
+		/* the case of partial open */
+		return 0;
+
+	object = &info->cinfo;
+
+	ret = get_emtd_file_key(info, master, mtd_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not retrieve metadata key");
+		return ret;
+	}
+	/*
+	 * decrypt encrypted meta-data
+	 */
+	ret = AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
+	if (ret < 0) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not set encrypt key");
+		return ret;
+	}
+	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
+	if (!gctx) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not alloc gcm context");
+		return ENOMEM;
+	}
+	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
+
+	ad = htole32(MTD_LOADER_V1);
+	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;	
+	}
+	ret = CRYPTO_gcm128_decrypt(gctx,
+				    get_EMTD_V1(fmt),
+				    get_EMTD_V1(fmt),
+				    SIZE_OF_EMTD_V1);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_decrypt failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	/*
+	 * verify metadata
+	 */
+	CRYPTO_gcm128_tag(gctx, gmac, sizeof(gmac));
+	CRYPTO_gcm128_release(gctx);
+	if (memcmp(gmac, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC)) {
+		gf_log("crypt", GF_LOG_ERROR, "EMTD verification failed");
+		return EINVAL;
+	}
+	/*
+	 * load verified metadata to the private part of inode
+	 */
+	info->nr_minor = fmt->minor_id;
+
+	object->o_alg = fmt->alg_id;
+	object->o_dkey_size = fmt->dkey_factor << KEY_FACTOR_BITS;
+	object->o_block_bits = fmt->block_bits;
+	object->o_mode = fmt->mode_id;
+
+	return check_file_metadata(info);
+}
+
+/*
+ * perform metadata authentication against @loc->path;
+ * extract crypt-specific attribtes and populate @info
+ * with them (optional)
+ */
+int32_t open_format(unsigned char *str,
+		    int32_t len,
+		    loc_t *loc,
+		    struct crypt_inode_info *info,
+		    struct master_cipher_info *master,
+		    crypt_local_t *local,
+		    gf_boolean_t load_info)
+{
+	struct crypt_format *fmt;
+	if (len < sizeof(*fmt)) {
+		gf_log("crypt", GF_LOG_ERROR, "Bad core format");
+		return EIO;
+	}
+	fmt = (struct crypt_format *)str;
+
+	if (fmt->loader_id >= LAST_MTD_LOADER) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Unsupported loader id %d", fmt->loader_id);
+		return EINVAL;
+	}
+	str += sizeof(*fmt);
+	len -= sizeof(*fmt);
+
+	return mtd_loaders[fmt->loader_id].open_format(str,
+						       len,
+						       loc,
+						       info,
+						       master,
+						       local,
+						       load_info);
+}
+
+struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER] = {
+	[MTD_LOADER_V1] = 
+	{.format_size = format_size_v1,
+	 .create_format = create_format_v1,
+	 .open_format = open_format_v1,
+	 .update_format = update_format_v1
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/metadata.h b/xlators/encryption/crypt/src/metadata.h
new file mode 100644
index 000000000..a92f149ef
--- /dev/null
+++ b/xlators/encryption/crypt/src/metadata.h
@@ -0,0 +1,74 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __METADATA_H__
+#define __METADATA_H__
+
+#define NMTD_8_MAC_SIZE   (8)
+#define EMTD_8_MAC_SIZE   (8)
+
+typedef uint8_t nmtd_8_mac_t[NMTD_8_MAC_SIZE];
+typedef uint8_t emtd_8_mac_t[EMTD_8_MAC_SIZE] ;
+
+/*
+ * Version "v1" of file's metadata.
+ * Metadata of this version has 4 components:
+ * 
+ * 1) EMTD (Encrypted part of MeTaData);
+ * 2) NMTD (Non-encrypted part of MeTaData);
+ * 3) EMTD_MAC; (EMTD Message Authentication Code);
+ * 4) Array of per-link NMTD MACs (for every (hard)link it includes
+ *    exactly one MAC)
+ */
+struct mtd_format_v1 {
+	/* EMTD, encrypted part of meta-data */
+	uint8_t            alg_id;  /* cipher algorithm id (only AES for now) */
+        uint8_t            mode_id; /* cipher mode id; (only XTS for now) */
+	uint8_t            block_bits; /* encoded block size */
+	uint8_t            minor_id; /* client translator id */
+	uint8_t            dkey_factor; /* encoded size of the data key */
+	/* MACs */
+	emtd_8_mac_t       gmac; /* MAC of the encrypted meta-data, 8 bytes */
+	nmtd_8_mac_t       omac; /* per-link MACs of the non-encrypted
+				  * meta-data: at least one such MAC is always
+				  * present */
+} __attribute__((packed));
+
+/*
+ * NMTD, the non-encrypted part of metadata of version "v1"
+ * is file's gfid, which is generated on trusted machines.
+ */
+#define SIZE_OF_NMTD_V1 (sizeof(uuid_t))
+#define SIZE_OF_EMTD_V1 (offsetof(struct mtd_format_v1, gmac) -		\
+			 offsetof(struct mtd_format_v1, alg_id))
+#define SIZE_OF_NMTD_V1_MAC (NMTD_8_MAC_SIZE)
+#define SIZE_OF_EMTD_V1_MAC (EMTD_8_MAC_SIZE)
+
+static inline unsigned char *get_EMTD_V1(struct mtd_format_v1 *format)
+{
+        return &format->alg_id;
+}
+
+static inline unsigned char *get_NMTD_V1(struct crypt_inode_info *info)
+{
+        return info->oid;
+}
+
+static inline unsigned char *get_EMTD_V1_MAC(struct mtd_format_v1 *format)
+{
+        return format->gmac;
+}
+
+static inline unsigned char *get_NMTD_V1_MAC(struct mtd_format_v1 *format)
+{
+        return format->omac;
+}
+
+#endif /* __METADATA_H__ */
diff --git a/xlators/encryption/rot-13/src/Makefile.am b/xlators/encryption/rot-13/src/Makefile.am
index ba5e623d8..94e8d18e7 100644
--- a/xlators/encryption/rot-13/src/Makefile.am
+++ b/xlators/encryption/rot-13/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = rot-13.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
 
-rot_13_la_LDFLAGS = -module -avoidversion
+rot_13_la_LDFLAGS = -module -avoid-version
 
 rot_13_la_SOURCES = rot-13.c
 rot_13_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = rot-13.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/encryption/rot-13/src/rot-13.c b/xlators/encryption/rot-13/src/rot-13.c
index 5e6820206..b9ac29a72 100644
--- a/xlators/encryption/rot-13/src/rot-13.c
+++ b/xlators/encryption/rot-13/src/rot-13.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <ctype.h>
 #include <sys/uio.h>
 
@@ -32,13 +22,13 @@
 #include "rot-13.h"
 
 /*
- * This is a rot13 ``encryption'' xlator. It rot13's data when 
- * writing to disk and rot13's it back when reading it. 
+ * This is a rot13 ``encryption'' xlator. It rot13's data when
+ * writing to disk and rot13's it back when reading it.
  * This xlator is meant as an example, NOT FOR PRODUCTION
  * USE ;) (hence no error-checking)
  */
 
-void 
+void
 rot13 (char *buf, int len)
 {
 	int i;
@@ -68,14 +58,15 @@ rot13_readv_cbk (call_frame_t *frame,
                  struct iovec *vector,
                  int32_t count,
 		 struct iatt *stbuf,
-                 struct iobref *iobref)
+                 struct iobref *iobref, dict_t *xdata)
 {
 	rot_13_private_t *priv = (rot_13_private_t *)this->private;
-  
+
 	if (priv->decrypt_read)
 		rot13_iovec (vector, count);
 
-	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count, stbuf, iobref);
+	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
+                             stbuf, iobref, xdata);
 	return 0;
 }
 
@@ -84,13 +75,13 @@ rot13_readv (call_frame_t *frame,
              xlator_t *this,
              fd_t *fd,
              size_t size,
-             off_t offset, uint32_t flags)
+             off_t offset, uint32_t flags, dict_t *xdata)
 {
 	STACK_WIND (frame,
 		    rot13_readv_cbk,
 		    FIRST_CHILD (this),
 		    FIRST_CHILD (this)->fops->readv,
-		    fd, size, offset, flags);
+		    fd, size, offset, flags, xdata);
 	return 0;
 }
 
@@ -101,9 +92,10 @@ rot13_writev_cbk (call_frame_t *frame,
                   int32_t op_ret,
                   int32_t op_errno,
                   struct iatt *prebuf,
-		  struct iatt *postbuf)
+		  struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 	return 0;
 }
 
@@ -114,7 +106,7 @@ rot13_writev (call_frame_t *frame,
               struct iovec *vector,
               int32_t count,
               off_t offset, uint32_t flags,
-              struct iobref *iobref)
+              struct iobref *iobref, dict_t *xdata)
 {
 	rot_13_private_t *priv = (rot_13_private_t *)this->private;
 	if (priv->encrypt_write)
@@ -125,7 +117,7 @@ rot13_writev (call_frame_t *frame,
 		    FIRST_CHILD (this),
 		    FIRST_CHILD (this)->fops->writev,
 		    fd, vector, count, offset, flags,
-                    iobref);
+                    iobref, xdata);
 	return 0;
 }
 
@@ -136,7 +128,7 @@ init (xlator_t *this)
 	rot_13_private_t *priv = NULL;
 
 	if (!this->children || this->children->next) {
-		gf_log ("rot13", GF_LOG_ERROR, 
+		gf_log ("rot13", GF_LOG_ERROR,
 			"FATAL: rot13 should have exactly one child");
 		return -1;
 	}
@@ -194,15 +186,14 @@ struct xlator_fops fops = {
 	.writev       = rot13_writev
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
-	{ .key  = {"encrypt-write"}, 
+	{ .key  = {"encrypt-write"},
 	  .type = GF_OPTION_TYPE_BOOL
 	},
-	{ .key  = {"decrypt-read"}, 
-	  .type = GF_OPTION_TYPE_BOOL 
+	{ .key  = {"decrypt-read"},
+	  .type = GF_OPTION_TYPE_BOOL
 	},
 	{ .key  = {NULL} },
 };
diff --git a/xlators/encryption/rot-13/src/rot-13.h b/xlators/encryption/rot-13/src/rot-13.h
index 8ef8162ae..3e9fc19c7 100644
--- a/xlators/encryption/rot-13/src/rot-13.h
+++ b/xlators/encryption/rot-13/src/rot-13.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __ROT_13_H__
 #define __ROT_13_H__
 
diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am
index 809bbe510..d2f5ef192 100644
--- a/xlators/features/Makefile.am
+++ b/xlators/features/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = locks trash quota read-only mac-compat quiesce marker#path-converter # filter
+SUBDIRS = locks quota read-only mac-compat quiesce marker index \
+	  protect compress changelog gfid-access $(GLUPY_SUBDIR) qemu-block # trash path-converter # filter
 
 CLEANFILES =
diff --git a/xlators/features/changelog/Makefile.am b/xlators/features/changelog/Makefile.am
new file mode 100644
index 000000000..153bb6850
--- /dev/null
+++ b/xlators/features/changelog/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src lib
+
+CLEANFILES =
diff --git a/scheduler/nufa/Makefile.am b/xlators/features/changelog/lib/Makefile.am
index d471a3f92..a985f42a8 100644
--- a/scheduler/nufa/Makefile.am
+++ b/xlators/features/changelog/lib/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/changelog/lib/examples/c/get-changes.c b/xlators/features/changelog/lib/examples/c/get-changes.c
new file mode 100644
index 000000000..14562585a
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/c/get-changes.c
@@ -0,0 +1,87 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+/**
+ * get set of new changes every 10 seconds (just print the file names)
+ *
+ * Compile it using:
+ *  gcc -o getchanges `pkg-config --cflags libgfchangelog` get-changes.c \
+ *  `pkg-config --libs libgfchangelog`
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/un.h>
+#include <limits.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <errno.h>
+
+#include "changelog.h"
+
+#define handle_error(fn)                                \
+        printf ("%s (reason: %s)\n", fn, strerror (errno))
+
+int
+main (int argc, char ** argv)
+{
+        int     i           = 0;
+        int     ret         = 0;
+        ssize_t nr_changes  = 0;
+        ssize_t changes     = 0;
+        char fbuf[PATH_MAX] = {0,};
+
+        /* get changes for brick "/home/vshankar/export/yow/yow-1" */
+        ret = gf_changelog_register ("/home/vshankar/export/yow/yow-1",
+                                     "/tmp/scratch", "/tmp/change.log", 9, 5);
+        if (ret) {
+                handle_error ("register failed");
+                goto out;
+        }
+
+        while (1) {
+                i = 0;
+                nr_changes = gf_changelog_scan ();
+                if (nr_changes < 0) {
+                        handle_error ("scan(): ");
+                        break;
+                }
+
+                if (nr_changes == 0)
+                        goto next;
+
+                printf ("Got %ld changelog files\n", nr_changes);
+
+                while ( (changes =
+                         gf_changelog_next_change (fbuf, PATH_MAX)) > 0) {
+                        printf ("changelog file [%d]: %s\n", ++i, fbuf);
+
+                        /* process changelog */
+                        /* ... */
+                        /* ... */
+                        /* ... */
+                        /* done processing */
+
+                        ret = gf_changelog_done (fbuf);
+                        if (ret)
+                                handle_error ("gf_changelog_done");
+                }
+
+                if (changes == -1)
+                        handle_error ("gf_changelog_next_change");
+
+        next:
+                sleep (10);
+        }
+
+ out:
+        return ret;
+}
diff --git a/xlators/features/changelog/lib/examples/python/changes.py b/xlators/features/changelog/lib/examples/python/changes.py
new file mode 100644
index 000000000..d21db8eab
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/python/changes.py
@@ -0,0 +1,32 @@
+#!/usr/bin/python
+
+import os
+import sys
+import time
+import libgfchangelog
+
+cl = libgfchangelog.Changes()
+
+def get_changes(brick, scratch_dir, log_file, log_level, interval):
+    change_list = []
+    try:
+        cl.cl_register(brick, scratch_dir, log_file, log_level)
+        while True:
+            cl.cl_scan()
+            change_list = cl.cl_getchanges()
+            if change_list:
+                print change_list
+            for change in change_list:
+                print('done with %s' % (change))
+                cl.cl_done(change)
+            time.sleep(interval)
+    except OSError:
+        ex = sys.exc_info()[1]
+        print ex
+
+if __name__ == '__main__':
+    if len(sys.argv) != 5:
+        print("usage: %s <brick> <scratch-dir> <log-file> <fetch-interval>"
+              % (sys.argv[0]))
+        sys.exit(1)
+    get_changes(sys.argv[1], sys.argv[2], sys.argv[3], 9, int(sys.argv[4]))
diff --git a/xlators/features/changelog/lib/examples/python/libgfchangelog.py b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
new file mode 100644
index 000000000..68ec3baf1
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
@@ -0,0 +1,64 @@
+import os
+from ctypes import *
+from ctypes.util import find_library
+
+class Changes(object):
+    libgfc = CDLL(find_library("gfchangelog"), use_errno=True)
+
+    @classmethod
+    def geterrno(cls):
+        return get_errno()
+
+    @classmethod
+    def raise_oserr(cls):
+        errn = cls.geterrno()
+        raise OSError(errn, os.strerror(errn))
+
+    @classmethod
+    def _get_api(cls, call):
+        return getattr(cls.libgfc, call)
+
+    @classmethod
+    def cl_register(cls, brick, path, log_file, log_level, retries = 0):
+        ret = cls._get_api('gf_changelog_register')(brick, path,
+                                                    log_file, log_level, retries)
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_scan(cls):
+        ret = cls._get_api('gf_changelog_scan')()
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_startfresh(cls):
+        ret = cls._get_api('gf_changelog_start_fresh')()
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_getchanges(cls):
+        """ remove hardcoding for path name length """
+        def clsort(f):
+            return f.split('.')[-1]
+        changes = []
+        buf = create_string_buffer('\0', 4096)
+        call = cls._get_api('gf_changelog_next_change')
+
+        while True:
+            ret = call(buf, 4096)
+            if ret in (0, -1):
+                break;
+            changes.append(buf.raw[:ret-1])
+        if ret == -1:
+            cls.raise_oserr()
+        # cleanup tracker
+        cls.cl_startfresh()
+        return sorted(changes, key=clsort)
+
+    @classmethod
+    def cl_done(cls, clfile):
+        ret = cls._get_api('gf_changelog_done')(clfile)
+        if ret == -1:
+            cls.raise_oserr()
diff --git a/xlators/features/changelog/lib/src/Makefile.am b/xlators/features/changelog/lib/src/Makefile.am
new file mode 100644
index 000000000..fbaaea628
--- /dev/null
+++ b/xlators/features/changelog/lib/src/Makefile.am
@@ -0,0 +1,37 @@
+libgfchangelog_la_CFLAGS = -Wall $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
+			-DDATADIR=\"$(localstatedir)\"
+
+libgfchangelog_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -fpic \
+				-I../../../src/ -I$(top_srcdir)/libglusterfs/src \
+				-I$(top_srcdir)/xlators/features/changelog/src \
+				-DDATADIR=\"$(localstatedir)\"
+
+libgfchangelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+			$(GF_GLUSTERFS_LIBS)
+
+libgfchangelog_la_LDFLAGS = $(GF_LDFLAGS)
+
+libgfchangelogdir = $(includedir)/glusterfs/gfchangelog
+lib_LTLIBRARIES = libgfchangelog.la
+
+CONTRIB_BUILDDIR = $(top_builddir)/contrib
+
+libgfchangelog_la_SOURCES = gf-changelog.c gf-changelog-process.c \
+		gf-changelog-helpers.c $(CONTRIBDIR)/uuid/clear.c \
+		$(CONTRIBDIR)/uuid/copy.c $(CONTRIBDIR)/uuid/gen_uuid.c \
+		$(CONTRIBDIR)/uuid/pack.c $(CONTRIBDIR)/uuid/parse.c \
+		$(CONTRIBDIR)/uuid/unparse.c $(CONTRIBDIR)/uuid/uuid_time.c \
+		$(CONTRIBDIR)/uuid/compare.c $(CONTRIBDIR)/uuid/isnull.c \
+		$(CONTRIBDIR)/uuid/unpack.c
+
+noinst_HEADERS = gf-changelog-helpers.h $(CONTRIBDIR)/uuid/uuidd.h \
+		$(CONTRIBDIR)/uuid/uuid.h $(CONTRIBDIR)/uuid/uuidP.h \
+		$(CONTRIB_BUILDDIR)/uuid/uuid_types.h
+
+libgfchangelog_HEADERS = changelog.h
+
+CLEANFILES =
+CONFIG_CLEAN_FILES = $(CONTRIB_BUILDDIR)/uuid/uuid_types.h
+
+$(top_builddir)/libglusterfs/src/libglusterfs.la:
+	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
diff --git a/xlators/features/changelog/lib/src/changelog.h b/xlators/features/changelog/lib/src/changelog.h
new file mode 100644
index 000000000..5cddfb583
--- /dev/null
+++ b/xlators/features/changelog/lib/src/changelog.h
@@ -0,0 +1,31 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_CHANGELOG_H
+#define _GF_CHANGELOG_H
+
+/* API set */
+
+int
+gf_changelog_register (char *brick_path, char *scratch_dir,
+                       char *log_file, int log_levl, int max_reconnects);
+ssize_t
+gf_changelog_scan ();
+
+int
+gf_changelog_start_fresh ();
+
+ssize_t
+gf_changelog_next_change (char *bufptr, size_t maxlen);
+
+int
+gf_changelog_done (char *file);
+
+#endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.c b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
new file mode 100644
index 000000000..1eef8bf04
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
@@ -0,0 +1,180 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "changelog-mem-types.h"
+#include "gf-changelog-helpers.h"
+
+ssize_t gf_changelog_read_path (int fd, char *buffer, size_t bufsize)
+{
+        return read (fd, buffer, bufsize);
+}
+
+size_t
+gf_changelog_write (int fd, char *buffer, size_t len)
+{
+        ssize_t size = 0;
+        size_t writen = 0;
+
+        while (writen < len) {
+                size = write (fd,
+                              buffer + writen, len - writen);
+                if (size <= 0)
+                        break;
+
+                writen += size;
+        }
+
+        return writen;
+}
+
+void
+gf_rfc3986_encode (unsigned char *s, char *enc, char *estr)
+{
+        for (; *s; s++) {
+                if (estr[*s])
+                        sprintf(enc, "%c", estr[*s]);
+                else
+                        sprintf(enc, "%%%02X", *s);
+                while (*++enc);
+        }
+}
+
+/**
+ * thread safe version of readline with buffering
+ * (taken from Unix Network Programming Volume I, W.R. Stevens)
+ *
+ * This is favoured over fgets() as we'd need to ftruncate()
+ * (see gf_changelog_scan() API) to record new changelog files.
+ * stream open functions does have a truncate like api (although
+ * that can be done via @fflush(fp), @ftruncate(fd) and @fseek(fp),
+ * but this involves mixing POSIX file descriptors and stream FILE *).
+ *
+ * NOTE: This implmentation still does work with more than one fd's
+ *       used to perform gf_readline(). For this very reason it's not
+ *       made a part of libglusterfs.
+ */
+
+static pthread_key_t rl_key;
+static pthread_once_t rl_once = PTHREAD_ONCE_INIT;
+
+static void
+readline_destructor (void *ptr)
+{
+        GF_FREE (ptr);
+}
+
+static void
+readline_once (void)
+{
+        pthread_key_create (&rl_key, readline_destructor);
+}
+
+static ssize_t
+my_read (read_line_t *tsd, int fd, char *ptr)
+{
+        if (tsd->rl_cnt <= 0) {
+                if ( (tsd->rl_cnt = read (fd, tsd->rl_buf, MAXLINE)) < 0 )
+                        return -1;
+                else if (tsd->rl_cnt == 0)
+                        return 0;
+                tsd->rl_bufptr = tsd->rl_buf;
+        }
+
+        tsd->rl_cnt--;
+        *ptr = *tsd->rl_bufptr++;
+        return 1;
+}
+
+static int
+gf_readline_init_once (read_line_t **tsd)
+{
+        if (pthread_once (&rl_once, readline_once) != 0)
+                return -1;
+
+        *tsd = pthread_getspecific (rl_key);
+        if (*tsd)
+                goto out;
+
+        *tsd = GF_CALLOC (1, sizeof (**tsd),
+                          gf_changelog_mt_libgfchangelog_rl_t);
+        if (!*tsd)
+                return -1;
+
+        if (pthread_setspecific (rl_key, *tsd) != 0)
+                return -1;
+
+ out:
+        return 0;
+}
+
+ssize_t
+gf_readline (int fd, void *vptr, size_t maxlen)
+{
+        size_t       n   = 0;
+        size_t       rc  = 0;
+        char         c   = ' ';
+        char        *ptr = NULL;
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        ptr = vptr;
+        for (n = 1; n < maxlen; n++) {
+                if ( (rc = my_read (tsd, fd, &c)) == 1 ) {
+                        *ptr++ = c;
+                        if (c == '\n')
+                                break;
+                } else if (rc == 0) {
+                        *ptr = '\0';
+                        return (n - 1);
+                } else
+                        return -1;
+        }
+
+        *ptr = '\0';
+        return n;
+
+}
+
+off_t
+gf_lseek (int fd, off_t offset, int whence)
+{
+        off_t        off = 0;
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        if ( (off = lseek (fd, offset, whence)) == -1)
+                return -1;
+
+        tsd->rl_cnt = 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+
+        return off;
+}
+
+int
+gf_ftruncate (int fd, off_t length)
+{
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        if (ftruncate (fd, 0))
+                return -1;
+
+        tsd->rl_cnt = 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+
+        return 0;
+}
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.h b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
new file mode 100644
index 000000000..3aa6ed7b8
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
@@ -0,0 +1,97 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_CHANGELOG_HELPERS_H
+#define _GF_CHANGELOG_HELPERS_H
+
+#include <unistd.h>
+#include <dirent.h>
+#include <limits.h>
+#include <pthread.h>
+
+#include <xlator.h>
+
+#define GF_CHANGELOG_TRACKER  "tracker"
+
+#define GF_CHANGELOG_CURRENT_DIR    ".current"
+#define GF_CHANGELOG_PROCESSED_DIR  ".processed"
+#define GF_CHANGELOG_PROCESSING_DIR ".processing"
+
+#ifndef MAXLINE
+#define MAXLINE 4096
+#endif
+
+#define GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, len) do {     \
+                memcpy (ascii + off, ptr, len);                 \
+                off += len;                                     \
+        } while (0)
+
+typedef struct read_line {
+        int rl_cnt;
+        char *rl_bufptr;
+        char rl_buf[MAXLINE];
+} read_line_t;
+
+typedef struct gf_changelog {
+        xlator_t *this;
+
+        /* 'processing' directory stream */
+        DIR *gfc_dir;
+
+        /* fd to the tracker file */
+        int gfc_fd;
+
+        /* connection retries */
+        int gfc_connretries;
+
+        char gfc_sockpath[PATH_MAX];
+
+        char gfc_brickpath[PATH_MAX];
+
+        /* socket for recieving notifications */
+        int gfc_sockfd;
+
+        char *gfc_working_dir;
+
+        /* RFC 3986 string encoding */
+        char rfc3986[256];
+
+        char gfc_current_dir[PATH_MAX];
+        char gfc_processed_dir[PATH_MAX];
+        char gfc_processing_dir[PATH_MAX];
+
+        pthread_t gfc_changelog_processor;
+} gf_changelog_t;
+
+int
+gf_changelog_notification_init (xlator_t *this, gf_changelog_t *gfc);
+
+void *
+gf_changelog_process (void *data);
+
+ssize_t
+gf_changelog_read_path (int fd, char *buffer, size_t bufsize);
+
+void
+gf_rfc3986_encode (unsigned char *s, char *enc, char *estr);
+
+size_t
+gf_changelog_write (int fd, char *buffer, size_t len);
+
+ssize_t
+gf_readline (int fd, void *vptr, size_t maxlen);
+
+int
+gf_ftruncate (int fd, off_t length);
+
+off_t
+gf_lseek (int fd, off_t offset, int whence);
+
+#endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-process.c b/xlators/features/changelog/lib/src/gf-changelog-process.c
new file mode 100644
index 000000000..df7204931
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-process.c
@@ -0,0 +1,571 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <pthread.h>
+
+#include "uuid.h"
+#include "globals.h"
+#include "glusterfs.h"
+
+#include "gf-changelog-helpers.h"
+
+/* from the changelog translator */
+#include "changelog-misc.h"
+
+extern int byebye;
+
+/**
+ * number of gfid records after fop number
+ */
+int nr_gfids[] = {
+        [GF_FOP_MKNOD]   = 1,
+        [GF_FOP_MKDIR]   = 1,
+        [GF_FOP_UNLINK]  = 1,
+        [GF_FOP_RMDIR]   = 1,
+        [GF_FOP_SYMLINK] = 1,
+        [GF_FOP_RENAME]  = 2,
+        [GF_FOP_LINK]    = 1,
+        [GF_FOP_CREATE]  = 1,
+};
+
+static char *
+binary_to_ascii (uuid_t uuid)
+{
+        return uuid_utoa (uuid);
+}
+
+static char *
+conv_noop (char *ptr) { return ptr; }
+
+#define VERIFY_SEPARATOR(ptr, plen, perr)       \
+        {                                       \
+                if (*(ptr + plen) != '\0') {    \
+                        perr = 1;               \
+                        break;                  \
+                }                               \
+        }
+
+#define MOVER_MOVE(mover, nleft, bytes)         \
+        {                                       \
+                mover += bytes;                 \
+                nleft -= bytes;                 \
+        }                                       \
+
+#define PARSE_GFID(mov, ptr, le, fn, perr)                      \
+        {                                                       \
+                VERIFY_SEPARATOR (mov, le, perr);               \
+                ptr = fn (mov);                                 \
+                if (!ptr) {                                     \
+                        perr = 1;                               \
+                        break;                                  \
+                }                                               \
+        }
+
+#define FILL_AND_MOVE(pt, buf, of, mo, nl, le)                          \
+        {                                                               \
+                GF_CHANGELOG_FILL_BUFFER (pt, buf, of, strlen (pt));    \
+                MOVER_MOVE (mo, nl, le);                                \
+        }
+
+
+#define PARSE_GFID_MOVE(ptr, uuid, mover, nleft, perr)          \
+        {                                                       \
+                memcpy (uuid, mover, sizeof (uuid_t));          \
+                ptr = binary_to_ascii (uuid);                   \
+                if (!ptr) {                                     \
+                        perr = 1;                               \
+                        break;                                  \
+                }                                               \
+                MOVER_MOVE (mover, nleft, sizeof (uuid_t));     \
+        }                                                       \
+
+#define LINE_BUFSIZE  3*PATH_MAX /* enough buffer for extra chars too */
+
+/**
+ * using mmap() makes parsing easy. fgets() cannot be used here as
+ * the binary gfid could contain a line-feed (0x0A), in that case fgets()
+ * would read an incomplete line and parsing would fail. using POSIX fds
+ * would result is additional code to maintain state in case of partial
+ * reads of data (where multiple entries do not fit extirely in the buffer).
+ *
+ * mmap() gives the flexibility of pointing to an offset in the file
+ * without us worrying about reading it in memory (VM does that for us for
+ * free).
+ */
+
+static int
+gf_changelog_parse_binary (xlator_t *this,
+                           gf_changelog_t *gfc, int from_fd, int to_fd,
+                           size_t start_offset, struct stat *stbuf)
+
+{
+        int     ret              = -1;
+        off_t   off              = 0;
+        off_t   nleft            = 0;
+        uuid_t  uuid             = {0,};
+        char   *ptr              = NULL;
+        char   *bname_start      = NULL;
+        char   *bname_end        = NULL;
+        char   *mover            = NULL;
+        char   *start            = NULL;
+        char    current_mover    = ' ';
+        size_t  blen             = 0;
+        int     parse_err        = 0;
+        char ascii[LINE_BUFSIZE] = {0,};
+
+        nleft = stbuf->st_size;
+
+        start = (char *) mmap (NULL, nleft,
+                               PROT_READ, MAP_PRIVATE, from_fd, 0);
+        if (!start) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "mmap() error (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        mover = start;
+
+        MOVER_MOVE (mover, nleft, start_offset);
+
+        while (nleft > 0) {
+
+                off = blen = 0;
+                ptr = bname_start = bname_end = NULL;
+
+                current_mover = *mover;
+
+                switch (current_mover) {
+                case 'D':
+                case 'M':
+                        MOVER_MOVE (mover, nleft, 1);
+                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+
+                        break;
+
+                case 'E':
+                        MOVER_MOVE (mover, nleft, 1);
+                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+
+                        bname_start = mover;
+                        if ( (bname_end = strchr (mover, '\n')) == NULL ) {
+                                parse_err = 1;
+                                break;
+                        }
+
+                        blen = bname_end - bname_start;
+                        MOVER_MOVE (mover, nleft, blen);
+
+                        break;
+
+                default:
+                        parse_err = 1;
+                }
+
+                if (parse_err)
+                        break;
+
+                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (ptr, ascii, off, strlen (ptr));
+                if (blen)
+                        GF_CHANGELOG_FILL_BUFFER (bname_start,
+                                                  ascii, off, blen);
+                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
+
+                if (gf_changelog_write (to_fd, ascii, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "processing binary changelog failed due to "
+                                " error in writing ascii change (reason: %s)",
+                                strerror (errno));
+                        break;
+                }
+
+                MOVER_MOVE (mover, nleft, 1);
+        }
+
+        if ( (nleft == 0) && (!parse_err))
+                ret = 0;
+
+        if (munmap (start, stbuf->st_size))
+                gf_log (this->name, GF_LOG_ERROR,
+                        "munmap() error (reason: %s)", strerror (errno));
+ out:
+        return ret;
+}
+
+/**
+ * ascii decoder:
+ *  - separate out one entry from another
+ *  - use fop name rather than fop number
+ */
+static int
+gf_changelog_parse_ascii (xlator_t *this,
+                          gf_changelog_t *gfc, int from_fd, int to_fd,
+                          size_t start_offset, struct stat *stbuf)
+{
+        int         ng            = 0;
+        int         ret           = -1;
+        int         fop           = 0;
+        int         len           = 0;
+        off_t       off           = 0;
+        off_t       nleft         = 0;
+        char       *ptr           = NULL;
+        char       *eptr          = NULL;
+        char       *start         = NULL;
+        char       *mover         = NULL;
+        int         parse_err     = 0;
+        char        current_mover = ' ';
+        char ascii[LINE_BUFSIZE]  = {0,};
+        const char *fopname       = NULL;
+
+        nleft = stbuf->st_size;
+
+        start = (char *) mmap (NULL, nleft,
+                               PROT_READ, MAP_PRIVATE, from_fd, 0);
+        if (!start) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "mmap() error (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        mover = start;
+
+        MOVER_MOVE (mover, nleft, start_offset);
+
+        while (nleft > 0) {
+                off = 0;
+                current_mover = *mover;
+
+                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+
+                switch (current_mover) {
+                case 'D':
+                case 'M':
+                        MOVER_MOVE (mover, nleft, 1);
+
+                        /* target gfid */
+                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
+                                    conv_noop, parse_err);
+                        FILL_AND_MOVE(ptr, ascii, off,
+                                      mover, nleft, UUID_CANONICAL_FORM_LEN);
+                        break;
+
+                case 'E':
+                        MOVER_MOVE (mover, nleft, 1);
+
+                        /* target gfid */
+                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
+                                    conv_noop, parse_err);
+                        FILL_AND_MOVE (ptr, ascii, off,
+                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
+                        FILL_AND_MOVE (" ", ascii, off,
+                                       mover, nleft, 1);
+
+                        /* fop */
+                        len = strlen (mover);
+                        VERIFY_SEPARATOR (mover, len, parse_err);
+
+                        fop = atoi (mover);
+                        if ( (fopname = gf_fop_list[fop]) == NULL) {
+                                parse_err = 1;
+                                break;
+                        }
+
+                        MOVER_MOVE (mover, nleft, len);
+
+                        len = strlen (fopname);
+                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
+
+                        /* pargfid + bname */
+                        ng = nr_gfids[fop];
+                        while (ng-- > 0) {
+                                MOVER_MOVE (mover, nleft, 1);
+                                len = strlen (mover);
+                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+
+                                PARSE_GFID (mover, ptr, len,
+                                            conv_noop, parse_err);
+                                eptr = calloc (3, strlen (ptr));
+                                if (!eptr) {
+                                        parse_err = 1;
+                                        break;
+                                }
+
+                                gf_rfc3986_encode ((unsigned char *) ptr,
+                                                   eptr, gfc->rfc3986);
+                                FILL_AND_MOVE (eptr, ascii, off,
+                                               mover, nleft, len);
+                                free (eptr);
+                        }
+
+                        break;
+                default:
+                        parse_err = 1;
+                }
+
+                if (parse_err)
+                        break;
+
+                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
+
+                if (gf_changelog_write (to_fd, ascii, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "processing ascii changelog failed due to "
+                                " wrror in writing change (reason: %s)",
+                                strerror (errno));
+                        break;
+                }
+
+                MOVER_MOVE (mover, nleft, 1);
+
+        }
+
+        if ( (nleft == 0) && (!parse_err))
+                ret = 0;
+
+        if (munmap (start, stbuf->st_size))
+                gf_log (this->name, GF_LOG_ERROR,
+                        "munmap() error (reason: %s)", strerror (errno));
+
+ out:
+        return ret;
+}
+
+#define COPY_BUFSIZE  8192
+static int
+gf_changelog_copy (xlator_t *this, int from_fd, int to_fd)
+{
+        ssize_t size                  = 0;
+        char   buffer[COPY_BUFSIZE+1] = {0,};
+
+        while (1) {
+                size = read (from_fd, buffer, COPY_BUFSIZE);
+                if (size <= 0)
+                        break;
+
+                if (gf_changelog_write (to_fd,
+                                        buffer, size) != size) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error processing ascii changlog");
+                        size = -1;
+                        break;
+                }
+        }
+
+        return (size < 0 ? -1 : 0);
+}
+
+static int
+gf_changelog_decode (xlator_t *this, gf_changelog_t *gfc, int from_fd,
+                     int to_fd, struct stat *stbuf, int *zerob)
+{
+        int    ret        = -1;
+        int    encoding   = -1;
+        size_t elen       = 0;
+        char buffer[1024] = {0,};
+
+        CHANGELOG_GET_ENCODING (from_fd, buffer, 1024, encoding, elen);
+        if (encoding == -1) /* unknown encoding */
+                goto out;
+
+        if (!CHANGELOG_VALID_ENCODING (encoding))
+                goto out;
+
+        if (elen == stbuf->st_size) {
+                *zerob = 1;
+                goto out;
+        }
+
+        /**
+         * start processing after the header
+         */
+        lseek (from_fd, elen, SEEK_SET);
+
+        switch (encoding) {
+        case CHANGELOG_ENCODE_BINARY:
+                /**
+                 * this ideally should have been a part of changelog-encoders.c
+                 * (ie. part of the changelog translator).
+                 */
+                ret = gf_changelog_parse_binary (this, gfc, from_fd,
+                                                 to_fd, elen, stbuf);
+                break;
+
+        case CHANGELOG_ENCODE_ASCII:
+                ret = gf_changelog_parse_ascii (this, gfc, from_fd,
+                                                to_fd, elen, stbuf);
+                break;
+        default:
+                ret = gf_changelog_copy (this, from_fd, to_fd);
+        }
+
+ out:
+        return ret;
+}
+
+static int
+gf_changelog_consume (xlator_t *this, gf_changelog_t *gfc, char *from_path)
+{
+        int         ret        = -1;
+        int         fd1        = 0;
+        int         fd2        = 0;
+        int         zerob      = 0;
+        struct stat stbuf      = {0,};
+        char dest[PATH_MAX]    = {0,};
+        char to_path[PATH_MAX] = {0,};
+
+        ret = stat (from_path, &stbuf);
+        if (ret || !S_ISREG(stbuf.st_mode)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat failed on changelog file: %s", from_path);
+                goto out;
+        }
+
+        fd1 = open (from_path, O_RDONLY);
+        if (fd1 < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot open changelog file: %s (reason: %s)",
+                        from_path, strerror (errno));
+                goto out;
+        }
+
+        (void) snprintf (to_path, PATH_MAX, "%s%s",
+                         gfc->gfc_current_dir, basename (from_path));
+        (void) snprintf (dest, PATH_MAX, "%s%s",
+                         gfc->gfc_processing_dir, basename (from_path));
+
+        fd2 = open (to_path, O_CREAT | O_TRUNC | O_RDWR,
+                    S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (fd2 < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot create ascii changelog file %s (reason %s)",
+                        to_path, strerror (errno));
+                goto close_fd;
+        } else {
+                ret = gf_changelog_decode (this, gfc, fd1,
+                                           fd2, &stbuf, &zerob);
+
+                close (fd2);
+
+                if (!ret) {
+                        /* move it to processing on a successfull
+                           decode */
+                        ret = rename (to_path, dest);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "error moving %s to processing dir"
+                                        " (reason: %s)", to_path,
+                                        strerror (errno));
+                }
+
+                /* remove it from .current if it's an empty file */
+                if (zerob) {
+                        ret = unlink (to_path);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "could not unlink %s (reason: %s",
+                                        to_path, strerror (errno));
+                }
+        }
+
+ close_fd:
+        close (fd1);
+
+ out:
+        return ret;
+}
+
+static char *
+gf_changelog_ext_change (xlator_t *this,
+                         gf_changelog_t *gfc, char *path, size_t readlen)
+{
+        int     alo = 0;
+        int     ret = 0;
+        size_t  len = 0;
+        char   *buf = NULL;
+
+        buf = path;
+        while (len < readlen) {
+                if (*buf == '\0') {
+                        alo = 1;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "processing changelog: %s", path);
+                        ret = gf_changelog_consume (this, gfc, path);
+                }
+
+                if (ret)
+                        break;
+
+                len++; buf++;
+                if (alo) {
+                        alo = 0;
+                        path = buf;
+                }
+        }
+
+        return (ret) ? NULL : path;
+}
+
+void *
+gf_changelog_process (void *data)
+{
+        ssize_t         len      = 0;
+        ssize_t         offlen   = 0;
+        xlator_t       *this     = NULL;
+        char           *sbuf     = NULL;
+        gf_changelog_t *gfc      = NULL;
+        char from_path[PATH_MAX] = {0,};
+
+        gfc = (gf_changelog_t *) data;
+        this = gfc->this;
+
+        pthread_detach (pthread_self());
+
+        for (;;) {
+                len = gf_changelog_read_path (gfc->gfc_sockfd,
+                                              from_path + offlen,
+                                              PATH_MAX - offlen);
+                if (len < 0)
+                        continue; /* ignore it for now */
+
+                if (len == 0) { /* close() from the changelog translator */
+                        gf_log (this->name, GF_LOG_INFO, "close from changelog"
+                                " notification translator.");
+
+                        if (gfc->gfc_connretries != 1) {
+                                if (!gf_changelog_notification_init(this, gfc))
+                                        continue;
+                        }
+
+                        byebye = 1;
+                        break;
+                }
+
+                len += offlen;
+                sbuf = gf_changelog_ext_change (this, gfc, from_path, len);
+                if (!sbuf) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not extract changelog filename");
+                        continue;
+                }
+
+                offlen = 0;
+                if (sbuf != (from_path + len)) {
+                        offlen = from_path + len - sbuf;
+                        memmove (from_path, sbuf, offlen);
+                }
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "byebye (%d) from processing thread...", byebye);
+        return NULL;
+}
diff --git a/xlators/features/changelog/lib/src/gf-changelog.c b/xlators/features/changelog/lib/src/gf-changelog.c
new file mode 100644
index 000000000..ca8e373e7
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog.c
@@ -0,0 +1,515 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <errno.h>
+#include <dirent.h>
+#include <stddef.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "gf-changelog-helpers.h"
+
+/* from the changelog translator */
+#include "changelog-misc.h"
+#include "changelog-mem-types.h"
+
+int byebye = 0;
+
+static void
+gf_changelog_cleanup (gf_changelog_t *gfc)
+{
+        /* socket */
+        if (gfc->gfc_sockfd != -1)
+                close (gfc->gfc_sockfd);
+        /* tracker fd */
+        if (gfc->gfc_fd != -1)
+                close (gfc->gfc_fd);
+        /* processing dir */
+        if (gfc->gfc_dir)
+                closedir (gfc->gfc_dir);
+
+        if (gfc->gfc_working_dir)
+                free (gfc->gfc_working_dir); /* allocated by realpath */
+}
+
+void
+__attribute__ ((constructor)) gf_changelog_ctor (void)
+{
+        glusterfs_ctx_t *ctx = NULL;
+
+        ctx = glusterfs_ctx_new ();
+        if (!ctx)
+                return;
+
+        if (glusterfs_globals_init (ctx)) {
+                free (ctx);
+                ctx = NULL;
+                return;
+        }
+
+        THIS->ctx = ctx;
+}
+
+void
+__attribute__ ((destructor)) gf_changelog_dtor (void)
+{
+        xlator_t        *this = NULL;
+        glusterfs_ctx_t *ctx  = NULL;
+        gf_changelog_t  *gfc  = NULL;
+
+        this = THIS;
+        if (!this)
+                return;
+
+        ctx = this->ctx;
+        gfc = this->private;
+
+        if (gfc) {
+                gf_changelog_cleanup (gfc);
+                GF_FREE (gfc);
+        }
+
+        if (ctx) {
+                pthread_mutex_destroy (&ctx->lock);
+                free (ctx);
+                ctx = NULL;
+        }
+}
+
+
+static int
+gf_changelog_open_dirs (gf_changelog_t *gfc)
+{
+        int  ret                    = -1;
+        DIR *dir                    = NULL;
+        int  tracker_fd             = 0;
+        char tracker_path[PATH_MAX] = {0,};
+
+        (void) snprintf (gfc->gfc_current_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_CURRENT_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_current_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        (void) snprintf (gfc->gfc_processed_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_PROCESSED_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_processed_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        (void) snprintf (gfc->gfc_processing_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_PROCESSING_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_processing_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        dir = opendir (gfc->gfc_processing_dir);
+        if (!dir) {
+                gf_log ("", GF_LOG_ERROR,
+                        "opendir() error [reason: %s]", strerror (errno));
+                goto out;
+        }
+
+        gfc->gfc_dir = dir;
+
+        (void) snprintf (tracker_path, PATH_MAX,
+                         "%s/"GF_CHANGELOG_TRACKER, gfc->gfc_working_dir);
+
+        tracker_fd = open (tracker_path, O_CREAT | O_APPEND | O_RDWR,
+                           S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (tracker_fd < 0) {
+                closedir (gfc->gfc_dir);
+                ret = -1;
+                goto out;
+        }
+
+        gfc->gfc_fd = tracker_fd;
+        ret = 0;
+ out:
+        return ret;
+}
+
+int
+gf_changelog_notification_init (xlator_t *this, gf_changelog_t *gfc)
+{
+        int                ret    = 0;
+        int                len    = 0;
+        int                tries  = 0;
+        int                sockfd = 0;
+        struct sockaddr_un remote;
+
+        this = gfc->this;
+
+        if (gfc->gfc_sockfd != -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Reconnecting...");
+                close (gfc->gfc_sockfd);
+        }
+
+        sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
+        if (sockfd < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        CHANGELOG_MAKE_SOCKET_PATH (gfc->gfc_brickpath,
+                                    gfc->gfc_sockpath, PATH_MAX);
+        gf_log (this->name, GF_LOG_INFO,
+                "connecting to changelog socket: %s (brick: %s)",
+                gfc->gfc_sockpath, gfc->gfc_brickpath);
+
+        remote.sun_family = AF_UNIX;
+        strcpy (remote.sun_path, gfc->gfc_sockpath);
+
+        len = strlen (remote.sun_path) + sizeof (remote.sun_family);
+
+        while (tries < gfc->gfc_connretries) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "connection attempt %d/%d...",
+                        tries + 1, gfc->gfc_connretries);
+
+                /* initiate a connect */
+                if (connect (sockfd, (struct sockaddr *) &remote, len) == 0) {
+                        gfc->gfc_sockfd = sockfd;
+                        break;
+                }
+
+                tries++;
+                sleep (2);
+        }
+
+        if (tries == gfc->gfc_connretries) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not connect to changelog socket!"
+                        " bailing out...");
+                ret = -1;
+        } else
+                gf_log (this->name, GF_LOG_INFO,
+                        "connection successful");
+
+ out:
+        return ret;
+}
+
+int
+gf_changelog_done (char *file)
+{
+        int             ret    = -1;
+        char           *buffer = NULL;
+        xlator_t       *this   = NULL;
+        gf_changelog_t *gfc    = NULL;
+        char to_path[PATH_MAX] = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        if (!file || !strlen (file))
+                goto out;
+
+        /* make sure 'file' is inside ->gfc_working_dir */
+        buffer = realpath (file, NULL);
+        if (!buffer)
+                goto out;
+
+        if (strncmp (gfc->gfc_working_dir,
+                     buffer, strlen (gfc->gfc_working_dir)))
+                goto out;
+
+        (void) snprintf (to_path, PATH_MAX, "%s%s",
+                         gfc->gfc_processed_dir, basename (buffer));
+        gf_log (this->name, GF_LOG_DEBUG,
+                "moving %s to processed directory", file);
+        ret = rename (buffer, to_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot move %s to %s (reason: %s)",
+                        file, to_path, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        if (buffer)
+                free (buffer); /* allocated by realpath() */
+        return ret;
+}
+
+/**
+ * @API
+ *  for a set of changelogs, start from the begining
+ */
+int
+gf_changelog_start_fresh ()
+{
+        xlator_t *this = NULL;
+        gf_changelog_t *gfc = NULL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        errno = EINVAL;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        if (gf_ftruncate (gfc->gfc_fd, 0))
+                goto out;
+
+        return 0;
+
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ * return the next changelog file entry. zero means all chanelogs
+ * consumed.
+ */
+ssize_t
+gf_changelog_next_change (char *bufptr, size_t maxlen)
+{
+        ssize_t         size       = 0;
+        int             tracker_fd = 0;
+        xlator_t       *this       = NULL;
+        gf_changelog_t *gfc        = NULL;
+        char buffer[PATH_MAX]      = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        tracker_fd = gfc->gfc_fd;
+
+        size = gf_readline (tracker_fd, buffer, maxlen);
+        if (size < 0)
+                goto out;
+        if (size == 0)
+                return 0;
+
+        memcpy (bufptr, buffer, size - 1);
+        *(buffer + size) = '\0';
+
+        return size;
+
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ *  gf_changelog_scan() - scan and generate a list of change entries
+ *
+ * calling this api multiple times (without calling gf_changlog_done())
+ * would result new changelogs(s) being refreshed in the tracker file.
+ * This call also acts as a cancellation point for the consumer.
+ */
+ssize_t
+gf_changelog_scan ()
+{
+        int             ret        = 0;
+        int             tracker_fd = 0;
+        size_t          len        = 0;
+        size_t          off        = 0;
+        xlator_t       *this       = NULL;
+        size_t          nr_entries = 0;
+        gf_changelog_t *gfc        = NULL;
+        struct dirent  *entryp     = NULL;
+        struct dirent  *result     = NULL;
+        char buffer[PATH_MAX]      = {0,};
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        /**
+         * do we need to protect 'byebye' with locks? worst, the
+         * consumer would get notified during next scan().
+         */
+        if (byebye) {
+                errno = ECONNREFUSED;
+                goto out;
+        }
+
+        errno = EINVAL;
+
+        tracker_fd = gfc->gfc_fd;
+
+        if (gf_ftruncate (tracker_fd, 0))
+                goto out;
+
+        len = offsetof(struct dirent, d_name)
+                + pathconf(gfc->gfc_processing_dir, _PC_NAME_MAX) + 1;
+        entryp = GF_CALLOC (1, len,
+                            gf_changelog_mt_libgfchangelog_dirent_t);
+        if (!entryp)
+                goto out;
+
+        rewinddir (gfc->gfc_dir);
+        while (1) {
+                ret = readdir_r (gfc->gfc_dir, entryp, &result);
+                if (ret || !result)
+                        break;
+
+                if ( !strcmp (basename (entryp->d_name), ".")
+                     || !strcmp (basename (entryp->d_name), "..") )
+                        continue;
+
+                nr_entries++;
+
+                GF_CHANGELOG_FILL_BUFFER (gfc->gfc_processing_dir,
+                                          buffer, off,
+                                          strlen (gfc->gfc_processing_dir));
+                GF_CHANGELOG_FILL_BUFFER (entryp->d_name, buffer,
+                                          off, strlen (entryp->d_name));
+                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
+
+                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error writing changelog filename"
+                                " to tracker file");
+                        break;
+                }
+                off = 0;
+        }
+
+        GF_FREE (entryp);
+
+        if (!result) {
+                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1)
+                        return nr_entries;
+        }
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ *  gf_changelog_register() - register a client for updates.
+ */
+int
+gf_changelog_register (char *brick_path, char *scratch_dir,
+                       char *log_file, int log_level, int max_reconnects)
+{
+        int             i    = 0;
+        int             ret  = -1;
+        int             errn = 0;
+        xlator_t       *this = NULL;
+        gf_changelog_t *gfc  = NULL;
+
+        this = THIS;
+        if (!this->ctx)
+                goto out;
+
+        errno = ENOMEM;
+
+        gfc = GF_CALLOC (1, sizeof (*gfc),
+                         gf_changelog_mt_libgfchangelog_t);
+        if (!gfc)
+                goto out;
+
+        gfc->this = this;
+
+        gfc->gfc_dir = NULL;
+        gfc->gfc_fd = gfc->gfc_sockfd = -1;
+
+        gfc->gfc_working_dir = realpath (scratch_dir, NULL);
+        if (!gfc->gfc_working_dir) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        ret = gf_changelog_open_dirs (gfc);
+        if (ret) {
+                errn = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not create entries in scratch dir");
+                goto cleanup;
+        }
+
+        /* passing ident as NULL means to use default ident for syslog */
+        if (gf_log_init (this->ctx, log_file, NULL))
+                goto cleanup;
+
+        gf_log_set_loglevel ((log_level == -1) ? GF_LOG_INFO :
+                             log_level);
+
+        gfc->gfc_connretries = (max_reconnects <= 0) ? 1 : max_reconnects;
+        (void) strncpy (gfc->gfc_brickpath, brick_path, PATH_MAX);
+
+        ret = gf_changelog_notification_init (this, gfc);
+        if (ret) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        ret = gf_thread_create (&gfc->gfc_changelog_processor,
+				NULL, gf_changelog_process, gfc);
+        if (ret) {
+                errn = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error creating changelog processor thread"
+                        " new changes won't be recorded!!!");
+                goto cleanup;
+        }
+
+        for (; i < 256; i++) {
+                gfc->rfc3986[i] =
+                        (isalnum(i) || i == '~' ||
+                        i == '-' || i == '.' || i == '_') ? i : 0;
+        }
+
+        ret = 0;
+        this->private = gfc;
+
+        goto out;
+
+ cleanup:
+        gf_changelog_cleanup (gfc);
+        GF_FREE (gfc);
+        this->private = NULL;
+        errno = errn;
+
+ out:
+        return ret;
+}
diff --git a/xlators/features/changelog/src/Makefile.am b/xlators/features/changelog/src/Makefile.am
new file mode 100644
index 000000000..e85031ad4
--- /dev/null
+++ b/xlators/features/changelog/src/Makefile.am
@@ -0,0 +1,19 @@
+xlator_LTLIBRARIES = changelog.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+noinst_HEADERS = changelog-helpers.h changelog-mem-types.h changelog-rt.h \
+	changelog-misc.h changelog-encoders.h changelog-notifier.h
+
+changelog_la_LDFLAGS = -module -avoidversion
+
+changelog_la_SOURCES = changelog.c changelog-rt.c changelog-helpers.c \
+	changelog-encoders.c changelog-notifier.c
+changelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -fPIC -D_FILE_OFFSET_BITS=64 \
+	-D_GNU_SOURCE -D$(GF_HOST_OS) -shared -nostartfiles -DDATADIR=\"$(localstatedir)\"
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/changelog/src/changelog-encoders.c b/xlators/features/changelog/src/changelog-encoders.c
new file mode 100644
index 000000000..553eec85c
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-encoders.c
@@ -0,0 +1,176 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "changelog-encoders.h"
+
+size_t
+entry_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        char    *tmpbuf = NULL;
+        size_t  bufsz  = 0;
+        struct changelog_entry_fields *ce = NULL;
+
+        ce = (struct changelog_entry_fields *) data;
+
+        if (encode) {
+                tmpbuf = uuid_utoa (ce->cef_uuid);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, tmpbuf, strlen (tmpbuf));
+        } else {
+                CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                                       ce->cef_uuid, sizeof (uuid_t));
+        }
+
+        CHANGELOG_FILL_BUFFER (buffer, bufsz, "/", 1);
+        CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                               ce->cef_bname, strlen (ce->cef_bname));
+        return bufsz;
+}
+
+size_t
+fop_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        char buf[10]          = {0,};
+        size_t         bufsz = 0;
+        glusterfs_fop_t fop   = 0;
+
+        fop = *(glusterfs_fop_t *) data;
+
+        if (encode) {
+                (void) snprintf (buf, sizeof (buf), "%d", fop);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
+        } else
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, &fop, sizeof (fop));
+
+        return bufsz;
+}
+
+void
+entry_free_fn (void *data)
+{
+        changelog_opt_t *co = data;
+
+        if (!co)
+                return;
+
+        GF_FREE (co->co_entry.cef_bname);
+}
+
+/**
+ * try to write all data in one shot
+ */
+
+static inline void
+changelog_encode_write_xtra (changelog_log_data_t *cld,
+                             char *buffer, size_t *off, gf_boolean_t encode)
+{
+        int              i      = 0;
+        size_t           offset = 0;
+        void            *data   = NULL;
+        changelog_opt_t *co     = NULL;
+
+        offset = *off;
+
+        co = (changelog_opt_t *) cld->cld_ptr;
+
+        for (; i < cld->cld_xtra_records; i++, co++) {
+                CHANGELOG_FILL_BUFFER (buffer, offset, "\0", 1);
+
+                switch (co->co_type) {
+                case CHANGELOG_OPT_REC_FOP:
+                        data = &co->co_fop;
+                        break;
+                case CHANGELOG_OPT_REC_ENTRY:
+                        data = &co->co_entry;
+                        break;
+                }
+
+                if (co->co_convert)
+                        offset += co->co_convert (data,
+                                                  buffer + offset, encode);
+                else /* no coversion: write it out as it is */
+                        CHANGELOG_FILL_BUFFER (buffer, offset,
+                                               data, co->co_len);
+        }
+
+        *off = offset;
+}
+
+int
+changelog_encode_ascii (xlator_t *this, changelog_log_data_t *cld)
+{
+        size_t            off      = 0;
+        size_t            gfid_len = 0;
+        char             *gfid_str = NULL;
+        char             *buffer   = NULL;
+        changelog_priv_t *priv     = NULL;
+
+        priv = this->private;
+
+        gfid_str = uuid_utoa (cld->cld_gfid);
+        gfid_len = strlen (gfid_str);
+
+        /* extra bytes for decorations */
+        buffer = alloca (gfid_len + cld->cld_ptr_len + 10);
+        CHANGELOG_STORE_ASCII (priv, buffer,
+                               off, gfid_str, gfid_len, cld);
+
+        if (cld->cld_xtra_records)
+                changelog_encode_write_xtra (cld, buffer, &off, _gf_true);
+
+        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+
+        return changelog_write_change (priv, buffer, off);
+}
+
+int
+changelog_encode_binary (xlator_t *this, changelog_log_data_t *cld)
+{
+        size_t            off    = 0;
+        char             *buffer = NULL;
+        changelog_priv_t *priv   = NULL;
+
+        priv = this->private;
+
+        /* extra bytes for decorations */
+        buffer = alloca (sizeof (uuid_t) + cld->cld_ptr_len + 10);
+        CHANGELOG_STORE_BINARY (priv, buffer, off, cld->cld_gfid, cld);
+
+        if (cld->cld_xtra_records)
+                changelog_encode_write_xtra (cld, buffer, &off, _gf_false);
+
+        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+
+        return changelog_write_change (priv, buffer, off);
+}
+
+static struct changelog_encoder
+cb_encoder[] = {
+        [CHANGELOG_ENCODE_BINARY] =
+        {
+                .encoder = CHANGELOG_ENCODE_BINARY,
+                .encode = changelog_encode_binary,
+        },
+        [CHANGELOG_ENCODE_ASCII] =
+        {
+                .encoder = CHANGELOG_ENCODE_ASCII,
+                .encode = changelog_encode_ascii,
+        },
+};
+
+void
+changelog_encode_change( changelog_priv_t * priv)
+{
+        priv->ce = &cb_encoder[priv->encode_mode];
+}
diff --git a/xlators/features/changelog/src/changelog-encoders.h b/xlators/features/changelog/src/changelog-encoders.h
new file mode 100644
index 000000000..a3efbee05
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-encoders.h
@@ -0,0 +1,46 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_ENCODERS_H
+#define _CHANGELOG_ENCODERS_H
+
+#include "xlator.h"
+#include "defaults.h"
+
+#include "changelog-helpers.h"
+
+#define CHANGELOG_STORE_ASCII(priv, buf, off, gfid, gfid_len, cld) do { \
+                CHANGELOG_FILL_BUFFER (buffer, off,                     \
+                                       priv->maps[cld->cld_type], 1);   \
+                CHANGELOG_FILL_BUFFER (buffer,                          \
+                                       off, gfid, gfid_len);            \
+        } while (0)
+
+#define CHANGELOG_STORE_BINARY(priv, buf, off, gfid, cld) do {          \
+                CHANGELOG_FILL_BUFFER (buffer, off,                     \
+                                       priv->maps[cld->cld_type], 1);   \
+                CHANGELOG_FILL_BUFFER (buffer,                          \
+                                       off, gfid, sizeof (uuid_t));     \
+        } while (0)
+
+size_t
+entry_fn (void *data, char *buffer, gf_boolean_t encode);
+size_t
+fop_fn (void *data, char *buffer, gf_boolean_t encode);
+void
+entry_free_fn (void *data);
+int
+changelog_encode_binary (xlator_t *, changelog_log_data_t *);
+int
+changelog_encode_ascii (xlator_t *, changelog_log_data_t *);
+void
+changelog_encode_change(changelog_priv_t *);
+
+#endif /* _CHANGELOG_ENCODERS_H */
diff --git a/xlators/features/changelog/src/changelog-helpers.c b/xlators/features/changelog/src/changelog-helpers.c
new file mode 100644
index 000000000..7ab0091b5
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-helpers.c
@@ -0,0 +1,693 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+#include "iobuf.h"
+
+#include "changelog-helpers.h"
+#include "changelog-mem-types.h"
+
+#include "changelog-encoders.h"
+#include <pthread.h>
+
+void
+changelog_thread_cleanup (xlator_t *this, pthread_t thr_id)
+{
+        int   ret    = 0;
+        void *retval = NULL;
+
+        /* send a cancel request to the thread */
+        ret = pthread_cancel (thr_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not cancel thread (reason: %s)",
+                        strerror (errno));
+                goto out;
+        }
+
+        ret = pthread_join (thr_id, &retval);
+        if (ret || (retval != PTHREAD_CANCELED)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cancel request not adhered as expected"
+                        " (reason: %s)", strerror (errno));
+        }
+
+ out:
+        return;
+}
+
+inline void *
+changelog_get_usable_buffer (changelog_local_t *local)
+{
+        changelog_log_data_t *cld = NULL;
+
+        cld = &local->cld;
+        if (!cld->cld_iobuf)
+                return NULL;
+
+        return cld->cld_iobuf->ptr;
+}
+
+inline void
+changelog_set_usable_record_and_length (changelog_local_t *local,
+                                        size_t len, int xr)
+{
+        changelog_log_data_t *cld = NULL;
+
+        cld = &local->cld;
+
+        cld->cld_ptr_len = len;
+        cld->cld_xtra_records = xr;
+}
+
+void
+changelog_local_cleanup (xlator_t *xl, changelog_local_t *local)
+{
+        int                   i   = 0;
+        changelog_opt_t      *co  = NULL;
+        changelog_log_data_t *cld = NULL;
+
+        if (!local)
+                return;
+
+        cld = &local->cld;
+
+        /* cleanup dynamic allocation for extra records */
+        if (cld->cld_xtra_records) {
+                co = (changelog_opt_t *) cld->cld_ptr;
+                for (; i < cld->cld_xtra_records; i++, co++)
+                        if (co->co_free)
+                                co->co_free (co);
+        }
+
+        CHANGELOG_IOBUF_UNREF (cld->cld_iobuf);
+
+        if (local->inode)
+                inode_unref (local->inode);
+
+        mem_put (local);
+}
+
+inline int
+changelog_write (int fd, char *buffer, size_t len)
+{
+        ssize_t size = 0;
+        size_t writen = 0;
+
+        while (writen < len) {
+                size = write (fd,
+                              buffer + writen, len - writen);
+                if (size <= 0)
+                        break;
+
+                writen += size;
+        }
+
+        return (writen != len);
+}
+
+static int
+changelog_rollover_changelog (xlator_t *this,
+                              changelog_priv_t *priv, unsigned long ts)
+{
+        int   ret            = -1;
+        int   notify         = 0;
+        char *bname          = NULL;
+        char ofile[PATH_MAX] = {0,};
+        char nfile[PATH_MAX] = {0,};
+
+        if (priv->changelog_fd != -1) {
+                close (priv->changelog_fd);
+                priv->changelog_fd = -1;
+        }
+
+        (void) snprintf (ofile, PATH_MAX,
+                         "%s/"CHANGELOG_FILE_NAME, priv->changelog_dir);
+        (void) snprintf (nfile, PATH_MAX,
+                         "%s/"CHANGELOG_FILE_NAME".%lu",
+                         priv->changelog_dir, ts);
+
+        ret = rename (ofile, nfile);
+        if (!ret)
+                notify = 1;
+
+        if (ret && (errno == ENOENT)) {
+                ret = 0;
+        }
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error renaming %s -> %s (reason %s)",
+                        ofile, nfile, strerror (errno));
+        }
+
+        if (notify) {
+                bname = basename (nfile);
+                gf_log (this->name, GF_LOG_DEBUG, "notifying: %s", bname);
+                ret = changelog_write (priv->wfd, bname, strlen (bname) + 1);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to send file name to notify thread"
+                                " (reason: %s)", strerror (errno));
+                }
+        }
+
+        return ret;
+}
+
+int
+changelog_open (xlator_t *this,
+                changelog_priv_t *priv)
+{
+        int fd                        = 0;
+        int ret                       = -1;
+        int flags                     = 0;
+        char buffer[1024]             = {0,};
+        char changelog_path[PATH_MAX] = {0,};
+
+        (void) snprintf (changelog_path, PATH_MAX,
+                         "%s/"CHANGELOG_FILE_NAME,
+                         priv->changelog_dir);
+
+        flags |= (O_CREAT | O_RDWR);
+        if (priv->fsync_interval == 0)
+                flags |= O_SYNC;
+
+        fd = open (changelog_path, flags,
+                   S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to open/create changelog file %s"
+                        " (reason: %s). change-logging will be"
+                        " inactive", changelog_path, strerror (errno));
+                goto out;
+        }
+
+        priv->changelog_fd = fd;
+
+        (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
+                         CHANGELOG_VERSION_MAJOR,
+                         CHANGELOG_VERSION_MINOR,
+                         priv->ce->encoder);
+        ret = changelog_write_change (priv, buffer, strlen (buffer));
+        if (ret) {
+                close (priv->changelog_fd);
+                priv->changelog_fd = -1;
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int
+changelog_start_next_change (xlator_t *this,
+                             changelog_priv_t *priv,
+                             unsigned long ts, gf_boolean_t finale)
+{
+        int ret = -1;
+
+        ret = changelog_rollover_changelog (this, priv, ts);
+
+        if (!ret && !finale)
+                ret = changelog_open (this, priv);
+
+        return ret;
+}
+
+/**
+ * return the length of entry
+ */
+inline size_t
+changelog_entry_length ()
+{
+        return sizeof (changelog_log_data_t);
+}
+
+int
+changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last)
+{
+        struct timeval tv = {0,};
+
+        cld->cld_type = CHANGELOG_TYPE_ROLLOVER;
+
+        if (gettimeofday (&tv, NULL))
+                return -1;
+
+        cld->cld_roll_time = (unsigned long) tv.tv_sec;
+        cld->cld_finale = is_last;
+        return 0;
+}
+
+int
+changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len)
+{
+        return changelog_write (priv->changelog_fd, buffer, len);
+}
+
+inline int
+changelog_handle_change (xlator_t *this,
+                         changelog_priv_t *priv, changelog_log_data_t *cld)
+{
+        int ret = 0;
+
+        if (CHANGELOG_TYPE_IS_ROLLOVER (cld->cld_type)) {
+                changelog_encode_change(priv);
+                ret = changelog_start_next_change (this, priv,
+                                                   cld->cld_roll_time,
+                                                   cld->cld_finale);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Problem rolling over changelog(s)");
+                goto out;
+        }
+
+        /**
+         * case when there is reconfigure done (disabling changelog) and there
+         * are still fops that have updates in prgress.
+         */
+        if (priv->changelog_fd == -1)
+                return 0;
+
+        if (CHANGELOG_TYPE_IS_FSYNC (cld->cld_type)) {
+                ret = fsync (priv->changelog_fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "fsync failed (reason: %s)",
+                                strerror (errno));
+                }
+                goto out;
+        }
+
+        ret = priv->ce->encode (this, cld);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error writing changelog to disk");
+        }
+
+ out:
+        return ret;
+}
+
+changelog_local_t *
+changelog_local_init (xlator_t *this, inode_t *inode,
+                      uuid_t gfid, int xtra_records,
+                      gf_boolean_t update_flag)
+{
+        changelog_local_t *local = NULL;
+        struct iobuf      *iobuf = NULL;
+
+        /**
+         * We relax the presence of inode if @update_flag is true.
+         * The caller (implmentation of the fop) needs to be careful to
+         * not blindly use local->inode.
+         */
+        if (!update_flag && !inode) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                                  "inode needed for version checking !!!");
+                goto out;
+        }
+
+        if (xtra_records) {
+                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
+                                    xtra_records * CHANGELOG_OPT_RECORD_LEN);
+                if (!iobuf)
+                        goto out;
+        }
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                CHANGELOG_IOBUF_UNREF (iobuf);
+                goto out;
+        }
+
+        local->update_no_check = update_flag;
+
+        uuid_copy (local->cld.cld_gfid, gfid);
+
+        local->cld.cld_iobuf = iobuf;
+        local->cld.cld_xtra_records = 0; /* set by the caller */
+
+        if (inode)
+                local->inode = inode_ref (inode);
+
+ out:
+        return local;
+}
+
+int
+changelog_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t ctx_addr = 0;
+        changelog_inode_ctx_t *ctx = NULL;
+
+        inode_ctx_del (inode, this, &ctx_addr);
+        if (!ctx_addr)
+                return 0;
+
+        ctx = (changelog_inode_ctx_t *) (long) ctx_addr;
+        GF_FREE (ctx);
+
+        return 0;
+}
+
+int
+changelog_inject_single_event (xlator_t *this,
+                               changelog_priv_t *priv,
+                               changelog_log_data_t *cld)
+{
+        return priv->cd.dispatchfn (this, priv, priv->cd.cd_data, cld, NULL);
+}
+
+/**
+ * TODO: these threads have many thing in common (wake up after
+ * a certain time etc..). move them into separate routine.
+ */
+void *
+changelog_rollover (void *data)
+{
+        int                     ret   = 0;
+        xlator_t               *this  = NULL;
+        struct timeval          tv    = {0,};
+        changelog_log_data_t    cld   = {0,};
+        changelog_time_slice_t *slice = NULL;
+        changelog_priv_t       *priv  = data;
+
+        this = priv->cr.this;
+        slice = &priv->slice;
+
+        while (1) {
+                tv.tv_sec  = priv->rollover_time;
+                tv.tv_usec = 0;
+
+                ret = select (0, NULL, NULL, NULL, &tv);
+                if (ret)
+                        continue;
+
+                ret = changelog_fill_rollover_data (&cld, _gf_false);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to fill rollover data");
+                        continue;
+                }
+
+                LOCK (&priv->lock);
+                {
+                        ret = changelog_inject_single_event (this, priv, &cld);
+                        if (!ret)
+                                SLICE_VERSION_UPDATE (slice);
+                }
+                UNLOCK (&priv->lock);
+        }
+
+        return NULL;
+}
+
+void *
+changelog_fsync_thread (void *data)
+{
+        int                   ret  = 0;
+        xlator_t             *this = NULL;
+        struct timeval        tv   = {0,};
+        changelog_log_data_t  cld  = {0,};
+        changelog_priv_t     *priv = data;
+
+        this = priv->cf.this;
+        cld.cld_type = CHANGELOG_TYPE_FSYNC;
+
+        while (1) {
+                tv.tv_sec  = priv->fsync_interval;
+                tv.tv_usec = 0;
+
+                ret = select (0, NULL, NULL, NULL, &tv);
+                if (ret)
+                        continue;
+
+                ret = changelog_inject_single_event (this, priv, &cld);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to inject fsync event");
+        }
+
+        return NULL;
+}
+
+/* macros for inode/changelog version checks */
+
+#define INODE_VERSION_UPDATE(priv, inode, iver, slice, type) do {       \
+                LOCK (&inode->lock);                                    \
+                {                                                       \
+                        LOCK (&priv->lock);                             \
+                        {                                               \
+                                *iver = slice->changelog_version[type]; \
+                        }                                               \
+                        UNLOCK (&priv->lock);                           \
+                }                                                       \
+                UNLOCK (&inode->lock);                                  \
+        } while (0)
+
+#define INODE_VERSION_EQUALS_SLICE(priv, ver, slice, type, upd) do {    \
+                LOCK (&priv->lock);                                     \
+                {                                                       \
+                        upd = (ver == slice->changelog_version[type])   \
+                                ? _gf_false : _gf_true;                 \
+                }                                                       \
+                UNLOCK (&priv->lock);                                   \
+        } while (0)
+
+static int
+__changelog_inode_ctx_set (xlator_t *this,
+                           inode_t *inode, changelog_inode_ctx_t *ctx)
+{
+        uint64_t ctx_addr = (uint64_t) ctx;
+        return __inode_ctx_set (inode, this, &ctx_addr);
+}
+
+/**
+ * one shot routine to get the address and the value of a inode version
+ * for a particular type.
+ */
+static changelog_inode_ctx_t *
+__changelog_inode_ctx_get (xlator_t *this,
+                           inode_t *inode, unsigned long **iver,
+                           unsigned long *version, changelog_log_type type)
+{
+        int                    ret      = 0;
+        uint64_t               ctx_addr = 0;
+        changelog_inode_ctx_t *ctx      = NULL;
+
+        ret = __inode_ctx_get (inode, this, &ctx_addr);
+        if (ret < 0)
+                ctx_addr = 0;
+        if (ctx_addr != 0) {
+                ctx = (changelog_inode_ctx_t *) (long)ctx_addr;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_changelog_mt_inode_ctx_t);
+        if (!ctx)
+                goto out;
+
+        ret = __changelog_inode_ctx_set (this, inode, ctx);
+        if (ret) {
+                GF_FREE (ctx);
+                ctx = NULL;
+        }
+
+ out:
+        if (ctx && iver && version) {
+                *iver = CHANGELOG_INODE_VERSION_TYPE (ctx, type);
+                *version = **iver;
+        }
+
+        return ctx;
+}
+
+static changelog_inode_ctx_t *
+changelog_inode_ctx_get (xlator_t *this,
+                         inode_t *inode, unsigned long **iver,
+                         unsigned long *version, changelog_log_type type)
+{
+        changelog_inode_ctx_t *ctx = NULL;
+
+        LOCK (&inode->lock);
+        {
+                ctx = __changelog_inode_ctx_get (this,
+                                                 inode, iver, version, type);
+        }
+        UNLOCK (&inode->lock);
+
+        return ctx;
+}
+
+/**
+ * This is the main update routine. Locking has been made granular so as to
+ * maximize parallelism of fops - I'll try to explain it below using execution
+ * timelines.
+ *
+ * Basically, the contention is between multiple execution threads of this
+ * routine and the roll-over thread. So, instead of having a big lock, we hold
+ * granular locks: inode->lock and priv->lock. Now I'll explain what happens
+ * when there is an update and a roll-over at just about the same time.
+ * NOTE:
+ *  - the dispatcher itself synchronizes updates via it's own lock
+ *  - the slice version in incremented by the roll-over thread
+ *
+ * Case 1: When the rollover thread wins before the inode version can be
+ * compared with the slice version.
+ *
+ *          [updater]                 |             [rollover]
+ *                                    |
+ *                                    |           <SLICE: 1, 1, 1>
+ * <changelog_update>                 |
+ *   <changelog_inode_ctx_get>        |
+ *      <CTX: 1, 1, 1>                |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 2, 2, 2>
+ *                                    |         UNLOCK (&priv->lock)
+ *                                    |
+ * LOCK (&priv->lock)                 |
+ *   <INODE_VERSION_EQUALS_SLICE>     |
+ *    I: 1 <-> S: 2                   |
+ *    update: true                    |
+ * UNLOCK (&priv->lock)               |
+ *                                    |
+ * <if update == true>                |
+ *  <dispath-update-event>            |
+ *  <INODE_VERSION_UPDATE>            |
+ *   LOCK (&inode->lock)              |
+ *    LOCK (&priv->lock)              |
+ *     <CTX: 2, 1, 1>                 |
+ *    UNLOCK (&priv->lock)            |
+ *   UNLOCK (&inode->lock)            |
+ *
+ * Therefore, the change gets recorded in the next change (no lost change). If
+ * the slice version was ahead of the inode version (say I:1, S: 2), then
+ * anyway the comparison would result in a update (I: 1, S: 3).
+ *
+ * If the rollover time is too less, then there is another contention when the
+ * updater tries to bring up inode version to the slice version (this is also
+ * the case when the roll-over thread wakes up during INODE_VERSION_UPDATE.
+ *
+ *   <CTX: 1, 1, 1>                   |       <SLICE: 2, 2, 2>
+ *                                    |
+ *                                    |
+ * <dispath-update-event>             |
+ * <INODE_VERSION_UPDATE>             |
+ *  LOCK (&inode->lock)               |
+ *   LOCK (&priv->lock)               |
+ *    <CTX: 2, 1, 1>                  |
+ *   UNLOCK (&priv->lock)             |
+ *  UNLOCK (&inode->lock)             |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 3, 3, 3>
+ *                                    |         UNLOCK (&priv->lock)
+ *
+ *
+ * Case 2: When the fop thread wins
+ *
+ *          [updater]                 |             [rollover]
+ *                                    |
+ *                                    |           <SLICE: 1, 1, 1>
+ * <changelog_update>                 |
+ *   <changelog_inode_ctx_get>        |
+ *      <CTX: 0, 0, 0>                |
+ *                                    |
+ * LOCK (&priv->lock)                 |
+ *   <INODE_VERSION_EQUALS_SLICE>     |
+ *    I: 0 <-> S: 1                   |
+ *    update: true                    |
+ * UNLOCK (&priv->lock)               |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 2, 2, 2>
+ *                                    |         UNLOCK (&priv->lock)
+ * <if update == true>                |
+ *  <dispath-update-event>            |
+ *  <INODE_VERSION_UPDATE>            |
+ *   LOCK (&inode->lock)              |
+ *    LOCK (&priv->lock)              |
+ *     <CTX: 2, 0, 0>                 |
+ *    UNLOCK (&priv->lock)            |
+ *   UNLOCK (&inode->lock)            |
+ *
+ * Here again, if the inode version was equal to the slice version (I: 1, S: 1)
+ * then there is no need to record an update (as the equality of the two version
+ * signifies an update was recorded in the current time slice).
+ */
+inline void
+changelog_update (xlator_t *this, changelog_priv_t *priv,
+                  changelog_local_t *local, changelog_log_type type)
+{
+        int                     ret        = 0;
+        unsigned long          *iver       = NULL;
+        unsigned long           version    = 0;
+        inode_t                *inode      = NULL;
+        changelog_time_slice_t *slice      = NULL;
+        changelog_inode_ctx_t  *ctx        = NULL;
+        changelog_log_data_t   *cld_0      = NULL;
+        changelog_log_data_t   *cld_1      = NULL;
+        changelog_local_t      *next_local = NULL;
+        gf_boolean_t            need_upd   = _gf_true;
+
+        slice = &priv->slice;
+
+        /**
+         * for fops that do not require inode version checking
+         */
+        if (local->update_no_check)
+                goto update;
+
+        inode = local->inode;
+
+        ctx = changelog_inode_ctx_get (this,
+                                       inode, &iver, &version, type);
+        if (!ctx)
+                goto update;
+
+        INODE_VERSION_EQUALS_SLICE (priv, version, slice, type, need_upd);
+
+ update:
+        if (need_upd) {
+                cld_0 = &local->cld;
+                cld_0->cld_type = type;
+
+                if ( (next_local = local->prev_entry) != NULL ) {
+                        cld_1 = &next_local->cld;
+                        cld_1->cld_type = type;
+                }
+
+                ret = priv->cd.dispatchfn (this, priv,
+                                           priv->cd.cd_data, cld_0, cld_1);
+
+                /**
+                 * update after the dispatcher has successfully done
+                 * it's job.
+                 */
+                if (!local->update_no_check && iver && !ret)
+                        INODE_VERSION_UPDATE (priv, inode, iver, slice, type);
+        }
+
+        return;
+}
diff --git a/xlators/features/changelog/src/changelog-helpers.h b/xlators/features/changelog/src/changelog-helpers.h
new file mode 100644
index 000000000..ad79636b0
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-helpers.h
@@ -0,0 +1,395 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_HELPERS_H
+#define _CHANGELOG_HELPERS_H
+
+#include "locking.h"
+#include "timer.h"
+#include "pthread.h"
+#include "iobuf.h"
+
+#include "changelog-misc.h"
+
+/**
+ * the changelog entry
+ */
+typedef struct changelog_log_data {
+        /* rollover related */
+        unsigned long cld_roll_time;
+
+        /* reopen changelog? */
+        gf_boolean_t cld_finale;
+
+        changelog_log_type cld_type;
+
+        /**
+         * sincd gfid is _always_ a necessity, it's not a part
+         * of the iobuf. by doing this we do not add any overhead
+         * for data and metadata related fops.
+         */
+        uuid_t        cld_gfid;
+
+        /**
+         * iobufs are used for optionals records: pargfid, path,
+         * write offsets etc.. It's the fop implementers job
+         * to allocate (iobuf_get() in the fop) and get unref'ed
+         * in the callback (CHANGELOG_STACK_UNWIND).
+         */
+        struct iobuf *cld_iobuf;
+
+#define cld_ptr cld_iobuf->ptr
+
+        /**
+         * after allocation you can point this to the length of
+         * usable data, but make sure it does not exceed the
+         * the size of the requested iobuf.
+         */
+        size_t        cld_iobuf_len;
+
+#define cld_ptr_len cld_iobuf_len
+
+        /**
+         * number of optional records
+         */
+        int cld_xtra_records;
+} changelog_log_data_t;
+
+/**
+ * holder for dispatch function and private data
+ */
+
+typedef struct changelog_priv changelog_priv_t;
+
+typedef struct changelog_dispatcher {
+        void *cd_data;
+        int (*dispatchfn) (xlator_t *, changelog_priv_t *, void *,
+                           changelog_log_data_t *, changelog_log_data_t *);
+} changelog_dispatcher_t;
+
+struct changelog_bootstrap {
+        changelog_mode_t mode;
+        int (*ctor) (xlator_t *, changelog_dispatcher_t *);
+        int (*dtor) (xlator_t *, changelog_dispatcher_t *);
+};
+
+struct changelog_encoder {
+        changelog_encoder_t encoder;
+        int (*encode) (xlator_t *, changelog_log_data_t *);
+};
+
+
+/* xlator private */
+
+typedef struct changelog_time_slice {
+        /**
+         * just in case we need nanosecond granularity some day.
+         * field is unused as of now (maybe we'd need it later).
+         */
+        struct timeval tv_start;
+
+        /**
+         * version of changelog file, incremented each time changes
+         * rollover.
+         */
+        unsigned long changelog_version[CHANGELOG_MAX_TYPE];
+} changelog_time_slice_t;
+
+typedef struct changelog_rollover {
+        /* rollover thread */
+        pthread_t rollover_th;
+
+        xlator_t *this;
+} changelog_rollover_t;
+
+typedef struct changelog_fsync {
+        /* fsync() thread */
+        pthread_t fsync_th;
+
+        xlator_t *this;
+} changelog_fsync_t;
+
+# define CHANGELOG_MAX_CLIENTS  5
+typedef struct changelog_notify {
+        /* reader end of the pipe */
+        int rfd;
+
+        /* notifier thread */
+        pthread_t notify_th;
+
+        /* unique socket path */
+        char sockpath[PATH_MAX];
+
+        int socket_fd;
+
+        /**
+         * simple array of accept()'ed fds. Not scalable at all
+         * for large number of clients, but it's okay as we have
+         * a ahrd limit in this version (@CHANGELOG_MAX_CLIENTS).
+         */
+        int client_fd[CHANGELOG_MAX_CLIENTS];
+
+        xlator_t *this;
+} changelog_notify_t;
+
+struct changelog_priv {
+        gf_boolean_t active;
+
+        /* to generate unique socket file per brick */
+        char *changelog_brick;
+
+        /* logging directory */
+        char *changelog_dir;
+
+        /* one file for all changelog types */
+        int changelog_fd;
+
+        gf_lock_t lock;
+
+        /* writen end of the pipe */
+        int wfd;
+
+        /* rollover time */
+        int32_t rollover_time;
+
+        /* fsync() interval */
+        int32_t fsync_interval;
+
+        /* changelog type maps */
+        const char *maps[CHANGELOG_MAX_TYPE];
+
+        /* time slicer */
+        changelog_time_slice_t slice;
+
+        /* context of the updater */
+        changelog_dispatcher_t cd;
+
+        /* context of the rollover thread */
+        changelog_rollover_t cr;
+
+        /* context of fsync thread */
+        changelog_fsync_t cf;
+
+        /* context of the notifier thread */
+        changelog_notify_t cn;
+
+        /* operation mode */
+        changelog_mode_t op_mode;
+
+        /* bootstrap routine for 'current' logger */
+        struct changelog_bootstrap *cb;
+
+        /* encoder mode */
+        changelog_encoder_t encode_mode;
+
+        /* encoder */
+        struct changelog_encoder *ce;
+};
+
+struct changelog_local {
+        inode_t              *inode;
+        gf_boolean_t          update_no_check;
+
+        changelog_log_data_t  cld;
+
+        /**
+         * ->prev_entry is used in cases when there needs to be
+         * additional changelog entry for the parent (eg. rename)
+         * It's analogous to ->next in single linked list world,
+         * but we call it as ->prev_entry... ha ha ha
+         */
+        struct changelog_local *prev_entry;
+};
+
+typedef struct changelog_local changelog_local_t;
+
+/* inode version is stored in inode ctx */
+typedef struct changelog_inode_ctx {
+        unsigned long iversion[CHANGELOG_MAX_TYPE];
+} changelog_inode_ctx_t;
+
+#define CHANGELOG_INODE_VERSION_TYPE(ctx, type)  &(ctx->iversion[type])
+
+/**
+ * Optional Records:
+ *  fops that need to save additional information request a array of
+ *  @changelog_opt_t struct. The array is allocated via @iobufs.
+ */
+typedef enum {
+        CHANGELOG_OPT_REC_FOP,
+        CHANGELOG_OPT_REC_ENTRY,
+} changelog_optional_rec_type_t;
+
+struct changelog_entry_fields {
+        uuid_t  cef_uuid;
+        char   *cef_bname;
+};
+
+typedef struct {
+        /**
+         * @co_covert can be used to do post-processing of the record before
+         * it's persisted to the CHANGELOG. If this is NULL, then the record
+         * is persisted as per it's in memory format.
+         */
+        size_t (*co_convert) (void *data, char *buffer, gf_boolean_t encode);
+
+        /* release routines */
+        void (*co_free) (void *data);
+
+        /* type of the field */
+        changelog_optional_rec_type_t co_type;
+
+        /**
+         * sizeof of the 'valid' field in the union. This field is not used if
+         * @co_convert is specified.
+         */
+        size_t co_len;
+
+        union {
+                glusterfs_fop_t co_fop;
+                struct changelog_entry_fields co_entry;
+        };
+} changelog_opt_t;
+
+#define CHANGELOG_OPT_RECORD_LEN  sizeof (changelog_opt_t)
+
+/**
+ * helpers routines
+ */
+
+void
+changelog_thread_cleanup (xlator_t *this, pthread_t thr_id);
+inline void *
+changelog_get_usable_buffer (changelog_local_t *local);
+inline void
+changelog_set_usable_record_and_length (changelog_local_t *local,
+                                        size_t len, int xr);
+void
+changelog_local_cleanup (xlator_t *xl, changelog_local_t *local);
+changelog_local_t *
+changelog_local_init (xlator_t *this, inode_t *inode, uuid_t gfid,
+                      int xtra_records, gf_boolean_t update_flag);
+int
+changelog_start_next_change (xlator_t *this,
+                             changelog_priv_t *priv,
+                             unsigned long ts, gf_boolean_t finale);
+int
+changelog_open (xlator_t *this, changelog_priv_t *priv);
+int
+changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last);
+int
+changelog_inject_single_event (xlator_t *this,
+                               changelog_priv_t *priv,
+                               changelog_log_data_t *cld);
+inline size_t
+changelog_entry_length ();
+inline int
+changelog_write (int fd, char *buffer, size_t len);
+int
+changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len);
+inline int
+changelog_handle_change (xlator_t *this,
+                         changelog_priv_t *priv, changelog_log_data_t *cld);
+inline void
+changelog_update (xlator_t *this, changelog_priv_t *priv,
+                  changelog_local_t *local, changelog_log_type type);
+void *
+changelog_rollover (void *data);
+void *
+changelog_fsync_thread (void *data);
+int
+changelog_forget (xlator_t *this, inode_t *inode);
+
+/* macros */
+
+#define CHANGELOG_STACK_UNWIND(fop, frame, params ...) do {             \
+                changelog_local_t *__local = NULL;                      \
+                xlator_t          *__xl    = NULL;                      \
+                if (frame) {                                            \
+                        __local      = frame->local;                    \
+                        __xl         = frame->this;                     \
+                        frame->local = NULL;                            \
+                }                                                       \
+                STACK_UNWIND_STRICT (fop, frame, params);               \
+                changelog_local_cleanup (__xl, __local);                \
+                if (__local && __local->prev_entry)                     \
+                        changelog_local_cleanup (__xl,                  \
+                                                 __local->prev_entry);  \
+        } while (0)
+
+#define CHANGELOG_IOBUF_REF(iobuf) do {         \
+                if (iobuf)                      \
+                        iobuf_ref (iobuf);      \
+        } while (0)
+
+#define CHANGELOG_IOBUF_UNREF(iobuf) do {       \
+                if (iobuf)                      \
+                        iobuf_unref (iobuf);    \
+        } while (0)
+
+#define CHANGELOG_FILL_BUFFER(buffer, off, val, len) do {       \
+                memcpy (buffer + off, val, len);                \
+                off += len;                                     \
+        } while (0)
+
+#define SLICE_VERSION_UPDATE(slice) do {                \
+                int i = 0;                              \
+                for (; i < CHANGELOG_MAX_TYPE; i++) {   \
+                        slice->changelog_version[i]++;  \
+                }                                       \
+        } while (0)
+
+#define CHANGLOG_FILL_FOP_NUMBER(co, fop, converter, xlen) do { \
+                co->co_convert = converter;                     \
+                co->co_free = NULL;                             \
+                co->co_type = CHANGELOG_OPT_REC_FOP;            \
+                co->co_fop = fop;                               \
+                xlen += sizeof (fop);                           \
+        } while (0)
+
+#define CHANGELOG_FILL_ENTRY(co, pargfid, bname,                        \
+                             converter, freefn, xlen, label)            \
+        do {                                                            \
+                co->co_convert = converter;                             \
+                co->co_free = freefn;                                   \
+                co->co_type = CHANGELOG_OPT_REC_ENTRY;                  \
+                uuid_copy (co->co_entry.cef_uuid, pargfid);             \
+                co->co_entry.cef_bname = gf_strdup(bname);              \
+                if (!co->co_entry.cef_bname)                            \
+                        goto label;                                     \
+                xlen += (UUID_CANONICAL_FORM_LEN + strlen (bname));     \
+        } while (0)
+
+#define CHANGELOG_INIT(this, local, inode, gfid, xrec)                  \
+        local = changelog_local_init (this, inode, gfid, xrec, _gf_false)
+
+#define CHANGELOG_INIT_NOCHECK(this, local, inode, gfid, xrec)          \
+        local = changelog_local_init (this, inode, gfid, xrec, _gf_true)
+
+#define CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, label) do { \
+                if (!priv->active)                              \
+                        goto label;                             \
+                /* ignore rebalance process's activity. */      \
+                if (frame->root->pid == GF_CLIENT_PID_DEFRAG)   \
+                        goto label;                             \
+        } while (0)
+
+/* ignore internal fops */
+#define CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(dict, label) do {           \
+                if (dict && dict_get (dict, GLUSTERFS_INTERNAL_FOP_KEY)) \
+                        goto label;                                     \
+        } while (0)
+
+#define CHANGELOG_COND_GOTO(priv, cond, label) do {                    \
+                if (!priv->active || cond)                             \
+                        goto label;                                    \
+        } while (0)
+
+#endif /* _CHANGELOG_HELPERS_H */
diff --git a/xlators/features/changelog/src/changelog-mem-types.h b/xlators/features/changelog/src/changelog-mem-types.h
new file mode 100644
index 000000000..d72464eab
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-mem-types.h
@@ -0,0 +1,29 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_MEM_TYPES_H
+#define _CHANGELOG_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_changelog_mem_types {
+        gf_changelog_mt_priv_t                  = gf_common_mt_end + 1,
+        gf_changelog_mt_str_t                   = gf_common_mt_end + 2,
+        gf_changelog_mt_batch_t                 = gf_common_mt_end + 3,
+        gf_changelog_mt_rt_t                    = gf_common_mt_end + 4,
+        gf_changelog_mt_inode_ctx_t             = gf_common_mt_end + 5,
+        gf_changelog_mt_libgfchangelog_t        = gf_common_mt_end + 6,
+        gf_changelog_mt_libgfchangelog_rl_t     = gf_common_mt_end + 7,
+        gf_changelog_mt_libgfchangelog_dirent_t = gf_common_mt_end + 8,
+        gf_changelog_mt_changelog_buffer_t      = gf_common_mt_end + 9,
+        gf_changelog_mt_end
+};
+
+#endif
diff --git a/xlators/features/changelog/src/changelog-misc.h b/xlators/features/changelog/src/changelog-misc.h
new file mode 100644
index 000000000..0712a3771
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-misc.h
@@ -0,0 +1,101 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_MISC_H
+#define _CHANGELOG_MISC_H
+
+#include "glusterfs.h"
+#include "common-utils.h"
+
+#define CHANGELOG_MAX_TYPE  3
+#define CHANGELOG_FILE_NAME "CHANGELOG"
+
+#define CHANGELOG_VERSION_MAJOR  1
+#define CHANGELOG_VERSION_MINOR  0
+
+#define CHANGELOG_UNIX_SOCK  DEFAULT_VAR_RUN_DIRECTORY"/changelog-%s.sock"
+
+/**
+ * header starts with the version and the format of the changelog.
+ * 'version' not much of a use now.
+ */
+#define CHANGELOG_HEADER                                                \
+        "GlusterFS Changelog | version: v%d.%d | encoding : %d\n"
+
+#define CHANGELOG_MAKE_SOCKET_PATH(brick_path, sockpath, len) do {      \
+                char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};             \
+                md5_wrapper((unsigned char *) brick_path,               \
+                            strlen(brick_path),                         \
+                            md5_sum);                                   \
+                (void) snprintf (sockpath, len,                         \
+                                 CHANGELOG_UNIX_SOCK, md5_sum);         \
+        } while (0)
+
+/**
+ * ... used by libgfchangelog.
+ */
+#define CHANGELOG_GET_ENCODING(fd, buffer, len, enc, enc_len) do {      \
+                FILE *fp;                                               \
+                int fd_dup, maj, min;                                   \
+                                                                        \
+                enc = -1;                                               \
+                fd_dup = dup (fd);                                      \
+                                                                        \
+                if (fd_dup != -1) {                                     \
+                        fp = fdopen (fd_dup, "r");                      \
+                        if (fp) {                                       \
+                                if (fgets (buffer, len, fp)) {          \
+                                        elen = strlen (buffer);         \
+                                        sscanf (buffer,                 \
+                                                CHANGELOG_HEADER,       \
+                                                &maj, &min, &enc);      \
+                                }                                       \
+                                fclose (fp);                            \
+                        } else {                                        \
+                                close (fd_dup);                         \
+                        }                                               \
+                }                                                       \
+        } while (0)
+
+/**
+ * everything after 'CHANGELOG_TYPE_ENTRY' are internal types
+ * (ie. none of the fops trigger this type of event), hence
+ * CHANGELOG_MAX_TYPE = 3
+ */
+typedef enum {
+        CHANGELOG_TYPE_DATA = 0,
+        CHANGELOG_TYPE_METADATA,
+        CHANGELOG_TYPE_ENTRY,
+        CHANGELOG_TYPE_ROLLOVER,
+        CHANGELOG_TYPE_FSYNC,
+} changelog_log_type;
+
+/* operation modes - RT for now */
+typedef enum {
+        CHANGELOG_MODE_RT = 0,
+} changelog_mode_t;
+
+/* encoder types */
+
+typedef enum {
+        CHANGELOG_ENCODE_MIN = 0,
+        CHANGELOG_ENCODE_BINARY,
+        CHANGELOG_ENCODE_ASCII,
+        CHANGELOG_ENCODE_MAX,
+} changelog_encoder_t;
+
+#define CHANGELOG_VALID_ENCODING(enc)                                   \
+        (enc > CHANGELOG_ENCODE_MIN && enc < CHANGELOG_ENCODE_MAX)
+
+#define CHANGELOG_TYPE_IS_ENTRY(type)  (type == CHANGELOG_TYPE_ENTRY)
+#define CHANGELOG_TYPE_IS_ROLLOVER(type)  (type == CHANGELOG_TYPE_ROLLOVER)
+#define CHANGELOG_TYPE_IS_FSYNC(type)  (type == CHANGELOG_TYPE_FSYNC)
+
+#endif /* _CHANGELOG_MISC_H */
diff --git a/xlators/features/changelog/src/changelog-notifier.c b/xlators/features/changelog/src/changelog-notifier.c
new file mode 100644
index 000000000..1f8b31253
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-notifier.c
@@ -0,0 +1,314 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "changelog-notifier.h"
+
+#include <pthread.h>
+
+inline static void
+changelog_notify_clear_fd (changelog_notify_t *cn, int i)
+{
+        cn->client_fd[i] = -1;
+}
+
+inline static void
+changelog_notify_save_fd (changelog_notify_t *cn, int i, int fd)
+{
+        cn->client_fd[i] = fd;
+}
+
+static int
+changelog_notify_insert_fd (xlator_t *this, changelog_notify_t *cn, int fd)
+{
+        int i   = 0;
+        int ret = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        break;
+        }
+
+        if (i == CHANGELOG_MAX_CLIENTS) {
+                /**
+                 * this case should not be hit as listen() would limit
+                 * the number of completely established connections.
+                 */
+                gf_log (this->name, GF_LOG_WARNING,
+                        "hit max client limit (%d)", CHANGELOG_MAX_CLIENTS);
+                ret = -1;
+        }
+        else
+                changelog_notify_save_fd (cn, i, fd);
+
+        return ret;
+}
+
+static void
+changelog_notify_fill_rset (changelog_notify_t *cn, fd_set *rset, int *maxfd)
+{
+        int i = 0;
+
+        FD_ZERO (rset);
+
+        FD_SET (cn->socket_fd, rset);
+        *maxfd = cn->socket_fd;
+
+        FD_SET (cn->rfd, rset);
+        *maxfd = max (*maxfd, cn->rfd);
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] != -1) {
+                        FD_SET (cn->client_fd[i], rset);
+                        *maxfd = max (*maxfd, cn->client_fd[i]);
+                }
+        }
+
+        *maxfd = *maxfd + 1;
+}
+
+static int
+changelog_notify_client (changelog_notify_t *cn, char *path, ssize_t len)
+{
+        int i = 0;
+        int ret = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        continue;
+
+                if (changelog_write (cn->client_fd[i],
+                                     path, len)) {
+                        ret = -1;
+
+                        close (cn->client_fd[i]);
+                        changelog_notify_clear_fd (cn, i);
+                }
+        }
+
+        return ret;
+}
+
+static void
+changelog_notifier_init (changelog_notify_t *cn)
+{
+        int i = 0;
+
+        cn->socket_fd = -1;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                changelog_notify_clear_fd (cn, i);
+        }
+}
+
+static void
+changelog_close_client_conn (changelog_notify_t *cn)
+{
+        int i = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        continue;
+
+                close (cn->client_fd[i]);
+                changelog_notify_clear_fd (cn, i);
+        }
+}
+
+static void
+changelog_notifier_cleanup (void *arg)
+{
+        changelog_notify_t *cn = NULL;
+
+        cn = (changelog_notify_t *) arg;
+
+        changelog_close_client_conn (cn);
+
+        if (cn->socket_fd != -1)
+                close (cn->socket_fd);
+
+        if (cn->rfd)
+                close (cn->rfd);
+
+        if (unlink (cn->sockpath))
+                gf_log ("", GF_LOG_WARNING,
+                        "could not unlink changelog socket file"
+                        " %s (reason: %s", cn->sockpath, strerror (errno));
+}
+
+void *
+changelog_notifier (void *data)
+{
+        int                 i         = 0;
+        int                 fd        = 0;
+        int                 max_fd    = 0;
+        int                 len       = 0;
+        ssize_t             readlen   = 0;
+        xlator_t           *this      = NULL;
+        changelog_priv_t   *priv      = NULL;
+        changelog_notify_t *cn        = NULL;
+        struct sockaddr_un  local     = {0,};
+        char path[PATH_MAX]           = {0,};
+        char abspath[PATH_MAX]        = {0,};
+
+        char buffer;
+        fd_set rset;
+
+        priv = (changelog_priv_t *) data;
+
+        cn = &priv->cn;
+        this = cn->this;
+
+        pthread_cleanup_push (changelog_notifier_cleanup, cn);
+
+        changelog_notifier_init (cn);
+
+        cn->socket_fd = socket (AF_UNIX, SOCK_STREAM, 0);
+        if (cn->socket_fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "changelog socket error (reason: %s)",
+                        strerror (errno));
+                goto out;
+        }
+
+        CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
+                                    cn->sockpath, PATH_MAX);
+        if (unlink (cn->sockpath) < 0) {
+                if (errno != ENOENT) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not unlink changelog socket file (%s)"
+                                " (reason: %s)",
+                                CHANGELOG_UNIX_SOCK, strerror (errno));
+                        goto cleanup;
+                }
+        }
+
+        local.sun_family = AF_UNIX;
+        strcpy (local.sun_path, cn->sockpath);
+
+        len = strlen (local.sun_path) + sizeof (local.sun_family);
+
+        /* bind to the unix domain socket */
+        if (bind (cn->socket_fd, (struct sockaddr *) &local, len) < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not bind to changelog socket (reason: %s)",
+                        strerror (errno));
+                goto cleanup;
+        }
+
+        /* listen for incoming connections */
+        if (listen (cn->socket_fd, CHANGELOG_MAX_CLIENTS) < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "listen() error on changelog socket (reason: %s)",
+                        strerror (errno));
+                goto cleanup;
+        }
+
+        /**
+         * simple select() on all to-be-read file descriptors. This method
+         * though old school works pretty well when you have a handfull of
+         * fd's to be watched (clients).
+         *
+         * Future TODO: move this to epoll based notification facility if
+         *              number of clients increase.
+         */
+        for (;;) {
+                changelog_notify_fill_rset (cn, &rset, &max_fd);
+
+                if (select (max_fd, &rset, NULL, NULL, NULL) < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "select() returned -1 (reason: %s)",
+                                strerror (errno));
+                        sleep (2);
+                        continue;
+                }
+
+                if (FD_ISSET (cn->socket_fd, &rset)) {
+                        fd = accept (cn->socket_fd, NULL, NULL);
+                        if (fd < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "accept error on changelog socket"
+                                        " (reason: %s)", strerror (errno));
+                        } else if (changelog_notify_insert_fd (this, cn, fd)) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "hit max client limit");
+                        }
+                }
+
+                if (FD_ISSET (cn->rfd, &rset)) {
+                        /**
+                         * read changelog filename and notify all connected
+                         * clients.
+                         */
+                        readlen = 0;
+                        while (readlen < PATH_MAX) {
+                                len = read (cn->rfd, &path[readlen++], 1);
+                                if (len == -1) {
+                                        break;
+                                }
+
+                                if (len == 0) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "rollover thread sent EOF"
+                                                " on pipe - possibly a crash.");
+                                        /* be blunt and close all connections */
+                                        pthread_exit(NULL);
+                                }
+
+                                if (path[readlen - 1] == '\0')
+                                        break;
+                        }
+
+                        /* should we close all client connections here too? */
+                        if (len < 0 || readlen == PATH_MAX) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not get pathname from rollover"
+                                        " thread or pathname too long");
+                                goto process_rest;
+                        }
+
+                        (void) snprintf (abspath, PATH_MAX,
+                                         "%s/%s", priv->changelog_dir, path);
+                        if (changelog_notify_client (cn, abspath,
+                                                     strlen (abspath) + 1))
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "could not notify some clients with new"
+                                        " changelogs");
+                }
+
+        process_rest:
+                for (i = 0; i < CHANGELOG_MAX_CLIENTS; i++) {
+                        if ( (fd = cn->client_fd[i]) == -1 )
+                                continue;
+
+                        if (FD_ISSET (fd, &rset)) {
+                                /**
+                                 * the only data we accept from the client is a
+                                 * disconnect. Anything else is treated as bogus
+                                 * and is silently discarded (also warned!!!).
+                                 */
+                                if ( (readlen = read (fd, &buffer, 1)) <= 0 ) {
+                                        close (fd);
+                                        changelog_notify_clear_fd (cn, i);
+                                } else {
+                                        /* silently discard data and log */
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "misbehaving changelog client");
+                                }
+                        }
+                }
+
+        }
+
+ cleanup:;
+        pthread_cleanup_pop (1);
+
+ out:
+        return NULL;
+}
diff --git a/xlators/features/changelog/src/changelog-notifier.h b/xlators/features/changelog/src/changelog-notifier.h
new file mode 100644
index 000000000..55e728356
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-notifier.h
@@ -0,0 +1,19 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_NOTIFIER_H
+#define _CHANGELOG_NOTIFIER_H
+
+#include "changelog-helpers.h"
+
+void *
+changelog_notifier (void *data);
+
+#endif
diff --git a/xlators/features/changelog/src/changelog-rt.c b/xlators/features/changelog/src/changelog-rt.c
new file mode 100644
index 000000000..c147f68ca
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-rt.c
@@ -0,0 +1,72 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "changelog-rt.h"
+#include "changelog-mem-types.h"
+
+int
+changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd)
+{
+        changelog_rt_t *crt = NULL;
+
+        crt = GF_CALLOC (1, sizeof (*crt),
+                         gf_changelog_mt_rt_t);
+        if (!crt)
+                return -1;
+
+        LOCK_INIT (&crt->lock);
+
+        cd->cd_data = crt;
+        cd->dispatchfn = &changelog_rt_enqueue;
+
+        return 0;
+}
+
+int
+changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd)
+{
+        changelog_rt_t *crt = NULL;
+
+        crt = cd->cd_data;
+
+        LOCK_DESTROY (&crt->lock);
+        GF_FREE (crt);
+
+        return 0;
+}
+
+int
+changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1)
+{
+        int             ret = 0;
+        changelog_rt_t *crt = NULL;
+
+        crt = (changelog_rt_t *) cbatch;
+
+        LOCK (&crt->lock);
+        {
+                ret = changelog_handle_change (this, priv, cld_0);
+                if (!ret && cld_1)
+                        ret = changelog_handle_change (this, priv, cld_1);
+        }
+        UNLOCK (&crt->lock);
+
+        return ret;
+}
diff --git a/xlators/features/changelog/src/changelog-rt.h b/xlators/features/changelog/src/changelog-rt.h
new file mode 100644
index 000000000..1fc2bbc5b
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-rt.h
@@ -0,0 +1,33 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_RT_H
+#define _CHANGELOG_RT_H
+
+#include "locking.h"
+#include "timer.h"
+#include "pthread.h"
+
+#include "changelog-helpers.h"
+
+/* unused as of now - may be you would need it later */
+typedef struct changelog_rt {
+        gf_lock_t lock;
+} changelog_rt_t;
+
+int
+changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd);
+int
+changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd);
+int
+changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1);
+
+#endif /* _CHANGELOG_RT_H */
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
new file mode 100644
index 000000000..cea0e8c70
--- /dev/null
+++ b/xlators/features/changelog/src/changelog.c
@@ -0,0 +1,1477 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+#include "iobuf.h"
+
+#include "changelog-rt.h"
+
+#include "changelog-encoders.h"
+#include "changelog-mem-types.h"
+
+#include <pthread.h>
+
+#include "changelog-notifier.h"
+
+static struct changelog_bootstrap
+cb_bootstrap[] = {
+        {
+                .mode = CHANGELOG_MODE_RT,
+                .ctor = changelog_rt_init,
+                .dtor = changelog_rt_fini,
+        },
+};
+
+/* Entry operations - TYPE III */
+
+/**
+ * entry operations do not undergo inode version checking.
+ */
+
+/* {{{ */
+
+/* rmdir */
+
+int32_t
+changelog_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                                preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_rmdir (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, int xflags, dict_t *xdata)
+{
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local,
+                                NULL, loc->inode->gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_rmdir_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rmdir,
+                    loc, xflags, xdata);
+        return 0;
+}
+
+/* unlink */
+
+int32_t
+changelog_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                                preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_unlink (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, int xflags, dict_t *xdata)
+{
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (xdata, wind);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, loc->inode->gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_unlink_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
+                    loc, xflags, xdata);
+        return 0;
+}
+
+/* rename */
+
+int32_t
+changelog_rename_cbk (call_frame_t *frame,
+                      void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      struct iatt *buf, struct iatt *preoldparent,
+                      struct iatt *postoldparent, struct iatt *prenewparent,
+                      struct iatt *postnewparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (rename, frame, op_ret, op_errno,
+                                buf, preoldparent, postoldparent,
+                                prenewparent, postnewparent, xdata);
+        return 0;
+}
+
+
+int32_t
+changelog_rename (call_frame_t *frame, xlator_t *this,
+                  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        size_t            xtra_len  = 0;
+        changelog_priv_t *priv      = NULL;
+        changelog_opt_t  *co        = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        /* 3 == fop + oldloc + newloc */
+        CHANGELOG_INIT_NOCHECK (this, frame->local,
+                                NULL, oldloc->inode->gfid, 3);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, oldloc->pargfid, oldloc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 3);
+
+ wind:
+        STACK_WIND (frame, changelog_rename_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rename,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+/* link */
+
+int32_t
+changelog_link_cbk (call_frame_t *frame,
+                    void *cookie, xlator_t *this, int32_t op_ret,
+                    int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (link, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_link (call_frame_t *frame,
+                xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
+{
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (xdata, wind);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, oldloc->gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_link_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->link,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+/* mkdir */
+
+int32_t
+changelog_mkdir_cbk (call_frame_t *frame,
+                     void *cookie, xlator_t *this, int32_t op_ret,
+                     int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_mkdir (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+{
+        int               ret      = -1;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_mkdir_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mkdir,
+                    loc, mode, umask, xdata);
+        return 0;
+}
+
+/* symlink */
+
+int32_t
+changelog_symlink_cbk (call_frame_t *frame,
+                       void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (symlink, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_symlink (call_frame_t *frame, xlator_t *this,
+                   const char *linkname, loc_t *loc,
+                   mode_t umask, dict_t *xdata)
+{
+        int               ret      = -1;
+        size_t            xtra_len = 0;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_symlink_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->symlink,
+                    linkname, loc, umask, xdata);
+        return 0;
+}
+
+/* mknod */
+
+int32_t
+changelog_mknod_cbk (call_frame_t *frame,
+                     void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (mknod, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_mknod (call_frame_t *frame,
+                 xlator_t *this, loc_t *loc,
+                 mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+{
+        int               ret      = -1;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_mknod_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
+                    loc, mode, dev, umask, xdata);
+        return 0;
+}
+
+/* creat */
+
+int32_t
+changelog_create_cbk (call_frame_t *frame,
+                      void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      fd_t *fd, inode_t *inode, struct iatt *buf,
+                      struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (create, frame,
+                                op_ret, op_errno, fd, inode,
+                                buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_create (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, int32_t flags, mode_t mode,
+                  mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int               ret      = -1;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        changelog_opt_t  *co       = NULL;
+        changelog_priv_t *priv     = NULL;
+        size_t            xtra_len = 0;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        /* init with two extra records */
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
+        if (!frame->local)
+                goto wind;
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_create_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+}
+
+/* }}} */
+
+
+/* Metadata modification fops - TYPE II */
+
+/* {{{ */
+
+/* {f}setattr */
+
+int32_t
+changelog_fsetattr_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this, int32_t op_ret,
+                        int32_t op_errno, struct iatt *preop_stbuf,
+                        struct iatt *postop_stbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
+                                preop_stbuf, postop_stbuf, xdata);
+
+        return 0;
+
+
+}
+
+int32_t
+changelog_fsetattr (call_frame_t *frame,
+                    xlator_t *this, fd_t *fd,
+                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_fsetattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+        return 0;
+
+
+}
+
+int32_t
+changelog_setattr_cbk (call_frame_t *frame,
+                       void *cookie, xlator_t *this, int32_t op_ret,
+                       int32_t op_errno, struct iatt *preop_stbuf,
+                       struct iatt *postop_stbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (setattr, frame, op_ret, op_errno,
+                                preop_stbuf, postop_stbuf, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_setattr (call_frame_t *frame,
+                   xlator_t *this, loc_t *loc,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_setattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+        return 0;
+}
+
+/* {f}removexattr */
+
+int32_t
+changelog_fremovexattr_cbk (call_frame_t *frame,
+                            void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_fremovexattr (call_frame_t *frame, xlator_t *this,
+                        fd_t *fd, const char *name, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_fremovexattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+int32_t
+changelog_removexattr_cbk (call_frame_t *frame,
+                           void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_removexattr (call_frame_t *frame, xlator_t *this,
+                       loc_t *loc, const char *name, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_removexattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+/* {f}setxattr */
+
+int32_t
+changelog_setxattr_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_setxattr (call_frame_t *frame,
+                    xlator_t *this, loc_t *loc,
+                    dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_setxattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+changelog_fsetxattr_cbk (call_frame_t *frame,
+                         void *cookie, xlator_t *this, int32_t op_ret,
+                         int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_fsetxattr (call_frame_t *frame,
+                     xlator_t *this, fd_t *fd, dict_t *dict,
+                     int32_t flags, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_fsetxattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+        return 0;
+}
+
+/* }}} */
+
+
+/* Data modification fops - TYPE I */
+
+/* {{{ */
+
+/* {f}truncate() */
+
+int32_t
+changelog_truncate_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this, int32_t op_ret,
+                        int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (truncate, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_truncate (call_frame_t *frame,
+                    xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_truncate_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
+                    loc, offset, xdata);
+        return 0;
+}
+
+int32_t
+changelog_ftruncate_cbk (call_frame_t *frame,
+                         void *cookie, xlator_t *this, int32_t op_ret,
+                         int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (ftruncate, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_ftruncate (call_frame_t *frame,
+                     xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_ftruncate_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
+                    fd, offset, xdata);
+        return 0;
+}
+
+/* writev() */
+
+int32_t
+changelog_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf,
+                    dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret <= 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (writev, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_writev (call_frame_t *frame,
+                  xlator_t *this, fd_t *fd, struct iovec *vector,
+                  int32_t count, off_t offset, uint32_t flags,
+                  struct iobref *iobref, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_writev_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev, fd, vector,
+                    count, offset, flags, iobref, xdata);
+        return 0;
+}
+
+/* }}} */
+
+/**
+ * The
+ *   - @init ()
+ *   - @fini ()
+ *   - @reconfigure ()
+ *   ... and helper routines
+ */
+
+/**
+ * needed if there are more operation modes in the future.
+ */
+static void
+changelog_assign_opmode (changelog_priv_t *priv, char *mode)
+{
+        if ( strncmp (mode, "realtime", 8) == 0 ) {
+                priv->op_mode = CHANGELOG_MODE_RT;
+        }
+}
+
+static void
+changelog_assign_encoding (changelog_priv_t *priv, char *enc)
+{
+        if ( strncmp (enc, "binary", 6) == 0 ) {
+                priv->encode_mode = CHANGELOG_ENCODE_BINARY;
+        } else if ( strncmp (enc, "ascii", 5) == 0 ) {
+                priv->encode_mode = CHANGELOG_ENCODE_ASCII;
+        }
+}
+
+/* cleanup any helper threads that are running */
+static void
+changelog_cleanup_helper_threads (xlator_t *this, changelog_priv_t *priv)
+{
+        if (priv->cr.rollover_th) {
+                changelog_thread_cleanup (this, priv->cr.rollover_th);
+                priv->cr.rollover_th = 0;
+        }
+
+        if (priv->cf.fsync_th) {
+                changelog_thread_cleanup (this, priv->cf.fsync_th);
+                priv->cf.fsync_th = 0;
+        }
+}
+
+/* spawn helper thread; cleaning up in case of errors */
+static int
+changelog_spawn_helper_threads (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret = 0;
+
+        priv->cr.this = this;
+        ret = gf_thread_create (&priv->cr.rollover_th,
+				NULL, changelog_rollover, priv);
+        if (ret)
+                goto out;
+
+        if (priv->fsync_interval) {
+                priv->cf.this = this;
+                ret = gf_thread_create (&priv->cf.fsync_th,
+					NULL, changelog_fsync_thread, priv);
+        }
+
+        if (ret)
+                changelog_cleanup_helper_threads (this, priv);
+
+ out:
+        return ret;
+}
+
+/* cleanup the notifier thread */
+static int
+changelog_cleanup_notifier (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret = 0;
+
+        if (priv->cn.notify_th) {
+                changelog_thread_cleanup (this, priv->cn.notify_th);
+                priv->cn.notify_th = 0;
+
+                ret = close (priv->wfd);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error closing writer end of notifier pipe"
+                                " (reason: %s)", strerror (errno));
+        }
+
+        return ret;
+}
+
+/* spawn the notifier thread - nop if already running */
+static int
+changelog_spawn_notifier (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret        = 0;
+        int flags      = 0;
+        int pipe_fd[2] = {0, 0};
+
+        if (priv->cn.notify_th)
+                goto out; /* notifier thread already running */
+
+        ret = pipe (pipe_fd);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot create pipe (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        /* writer is non-blocking */
+        flags = fcntl (pipe_fd[1], F_GETFL);
+        flags |= O_NONBLOCK;
+
+        ret = fcntl (pipe_fd[1], F_SETFL, flags);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set O_NONBLOCK flag");
+                goto out;
+        }
+
+        priv->wfd = pipe_fd[1];
+
+        priv->cn.this = this;
+        priv->cn.rfd  = pipe_fd[0];
+
+        ret = gf_thread_create (&priv->cn.notify_th,
+				NULL, changelog_notifier, priv);
+
+ out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_changelog_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
+                        " init failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+static int
+changelog_init (xlator_t *this, changelog_priv_t *priv)
+{
+        int                  i   = 0;
+        int                  ret = -1;
+        struct timeval       tv  = {0,};
+        changelog_log_data_t cld = {0,};
+
+        ret = gettimeofday (&tv, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gettimeofday() failure");
+                goto out;
+        }
+
+        priv->slice.tv_start = tv;
+
+        priv->maps[CHANGELOG_TYPE_DATA]     = "D ";
+        priv->maps[CHANGELOG_TYPE_METADATA] = "M ";
+        priv->maps[CHANGELOG_TYPE_ENTRY]    = "E ";
+
+        for (; i < CHANGELOG_MAX_TYPE; i++) {
+                /* start with version 1 */
+                priv->slice.changelog_version[i] = 1;
+        }
+
+        if (!priv->active)
+                return ret;
+
+        /* spawn the notifier thread */
+        ret = changelog_spawn_notifier (this, priv);
+        if (ret)
+                goto out;
+
+        /**
+         * start with a fresh changelog file every time. this is done
+         * in case there was an encoding change. so... things are kept
+         * simple here.
+         */
+        ret = changelog_fill_rollover_data (&cld, _gf_false);
+        if (ret)
+                goto out;
+
+        LOCK (&priv->lock);
+        {
+                ret = changelog_inject_single_event (this, priv, &cld);
+        }
+        UNLOCK (&priv->lock);
+
+        /* ... and finally spawn the helpers threads */
+        ret = changelog_spawn_helper_threads (this, priv);
+
+ out:
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int                     ret            = 0;
+        char                   *tmp            = NULL;
+        changelog_priv_t       *priv           = NULL;
+        gf_boolean_t            active_earlier = _gf_true;
+        gf_boolean_t            active_now     = _gf_true;
+        changelog_time_slice_t *slice          = NULL;
+        changelog_log_data_t    cld            = {0,};
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+
+        ret = -1;
+        active_earlier = priv->active;
+
+        /* first stop the rollover and the fsync thread */
+        changelog_cleanup_helper_threads (this, priv);
+
+        GF_OPTION_RECONF ("changelog-dir", tmp, options, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-dir\" option is not set");
+                goto out;
+        }
+
+        GF_FREE (priv->changelog_dir);
+        priv->changelog_dir = gf_strdup (tmp);
+        if (!priv->changelog_dir)
+                goto out;
+
+        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+        if (ret)
+                goto out;
+
+        GF_OPTION_RECONF ("changelog", active_now, options, bool, out);
+
+        /**
+         * changelog_handle_change() handles changes that could possibly
+         * have been submit changes before changelog deactivation.
+         */
+        if (!active_now)
+                priv->active = _gf_false;
+
+        GF_OPTION_RECONF ("op-mode", tmp, options, str, out);
+        changelog_assign_opmode (priv, tmp);
+
+        tmp = NULL;
+
+        GF_OPTION_RECONF ("encoding", tmp, options, str, out);
+        changelog_assign_encoding (priv, tmp);
+
+        GF_OPTION_RECONF ("rollover-time",
+                          priv->rollover_time, options, int32, out);
+        GF_OPTION_RECONF ("fsync-interval",
+                          priv->fsync_interval, options, int32, out);
+
+        if (active_now || active_earlier) {
+                ret = changelog_fill_rollover_data (&cld, !active_now);
+                if (ret)
+                        goto out;
+
+                slice = &priv->slice;
+
+                LOCK (&priv->lock);
+                {
+                        ret = changelog_inject_single_event (this, priv, &cld);
+                        if (!ret && active_now)
+                                SLICE_VERSION_UPDATE (slice);
+                }
+                UNLOCK (&priv->lock);
+
+                if (ret)
+                        goto out;
+
+                if (active_now) {
+                        ret = changelog_spawn_notifier (this, priv);
+                        if (!ret)
+                                ret = changelog_spawn_helper_threads (this,
+                                                                      priv);
+                } else
+                        ret = changelog_cleanup_notifier (this, priv);
+        }
+
+ out:
+        if (ret) {
+                ret = changelog_cleanup_notifier (this, priv);
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "changelog reconfigured");
+                if (active_now)
+                        priv->active = _gf_true;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        int               ret  = -1;
+        char             *tmp  = NULL;
+        changelog_priv_t *priv = NULL;
+
+        GF_VALIDATE_OR_GOTO ("changelog", this, out);
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "translator needs a single subvolume");
+                goto out;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "dangling volume. please check volfile");
+                goto out;
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_changelog_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        this->local_pool = mem_pool_new (changelog_local_t, 64);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local memory pool");
+                goto out;
+        }
+
+        LOCK_INIT (&priv->lock);
+
+        GF_OPTION_INIT ("changelog-brick", tmp, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-brick\" option is not set");
+                goto out;
+        }
+
+        priv->changelog_brick = gf_strdup (tmp);
+        if (!priv->changelog_brick)
+                goto out;
+        tmp = NULL;
+
+        GF_OPTION_INIT ("changelog-dir", tmp, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-dir\" option is not set");
+                goto out;
+        }
+
+        priv->changelog_dir = gf_strdup (tmp);
+        if (!priv->changelog_dir)
+                goto out;
+        tmp = NULL;
+
+        /**
+         * create the directory even if change-logging would be inactive
+         * so that consumers can _look_ into it (finding nothing...)
+         */
+        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+        if (ret)
+                goto out;
+
+        GF_OPTION_INIT ("changelog", priv->active, bool, out);
+
+        GF_OPTION_INIT ("op-mode", tmp, str, out);
+        changelog_assign_opmode (priv, tmp);
+
+        tmp = NULL;
+
+        GF_OPTION_INIT ("encoding", tmp, str, out);
+        changelog_assign_encoding (priv, tmp);
+
+        GF_OPTION_INIT ("rollover-time", priv->rollover_time, int32, out);
+
+        GF_OPTION_INIT ("fsync-interval", priv->fsync_interval, int32, out);
+
+        changelog_encode_change(priv);
+
+        GF_ASSERT (cb_bootstrap[priv->op_mode].mode == priv->op_mode);
+        priv->cb = &cb_bootstrap[priv->op_mode];
+
+        /* ... now bootstrap the logger */
+        ret = priv->cb->ctor (this, &priv->cd);
+        if (ret)
+                goto out;
+
+        priv->changelog_fd = -1;
+        ret = changelog_init (this, priv);
+        if (ret)
+                goto out;
+
+        gf_log (this->name, GF_LOG_DEBUG, "changelog translator loaded");
+
+ out:
+        if (ret) {
+                if (this->local_pool)
+                        mem_pool_destroy (this->local_pool);
+                if (priv->cb) {
+                        ret = priv->cb->dtor (this, &priv->cd);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "error in cleanup during init()");
+                }
+                GF_FREE (priv->changelog_brick);
+                GF_FREE (priv->changelog_dir);
+                GF_FREE (priv);
+                this->private = NULL;
+        } else
+                this->private = priv;
+
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        int               ret  = -1;
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+
+        if (priv) {
+                ret = priv->cb->dtor (this, &priv->cd);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error in fini");
+                mem_pool_destroy (this->local_pool);
+                GF_FREE (priv->changelog_brick);
+                GF_FREE (priv->changelog_dir);
+                GF_FREE (priv);
+        }
+
+        this->private = NULL;
+
+        return;
+}
+
+struct xlator_fops fops = {
+        .mknod        = changelog_mknod,
+        .mkdir        = changelog_mkdir,
+        .create       = changelog_create,
+        .symlink      = changelog_symlink,
+        .writev       = changelog_writev,
+        .truncate     = changelog_truncate,
+        .ftruncate    = changelog_ftruncate,
+        .link         = changelog_link,
+        .rename       = changelog_rename,
+        .unlink       = changelog_unlink,
+        .rmdir        = changelog_rmdir,
+        .setattr      = changelog_setattr,
+        .fsetattr     = changelog_fsetattr,
+        .setxattr     = changelog_setxattr,
+        .fsetxattr    = changelog_fsetxattr,
+        .removexattr  = changelog_removexattr,
+        .fremovexattr = changelog_fremovexattr,
+};
+
+struct xlator_cbks cbks = {
+        .forget = changelog_forget,
+};
+
+struct volume_options options[] = {
+        {.key = {"changelog"},
+         .type = GF_OPTION_TYPE_BOOL,
+         .default_value = "off",
+         .description = "enable/disable change-logging"
+        },
+        {.key = {"changelog-brick"},
+         .type = GF_OPTION_TYPE_PATH,
+         .description = "brick path to generate unique socket file name."
+                       " should be the export directory of the volume strictly."
+        },
+        {.key = {"changelog-dir"},
+         .type = GF_OPTION_TYPE_PATH,
+         .description = "directory for the changelog files"
+        },
+        {.key = {"op-mode"},
+         .type = GF_OPTION_TYPE_STR,
+         .default_value = "realtime",
+         .value = {"realtime"},
+         .description = "operation mode - futuristic operation modes"
+        },
+        {.key = {"encoding"},
+         .type = GF_OPTION_TYPE_STR,
+         .default_value = "ascii",
+         .value = {"binary", "ascii"},
+         .description = "encoding type for changelogs"
+        },
+        {.key = {"rollover-time"},
+         .default_value = "60",
+         .type = GF_OPTION_TYPE_TIME,
+         .description = "time to switch to a new changelog file (in seconds)"
+        },
+        {.key = {"fsync-interval"},
+         .type = GF_OPTION_TYPE_TIME,
+         .default_value = "0",
+         .description = "do not open CHANGELOG file with O_SYNC mode."
+                        " instead perform fsync() at specified intervals"
+        },
+        {.key = {NULL}
+        },
+};
diff --git a/mod_glusterfs/apache/2.2/Makefile.am b/xlators/features/compress/Makefile.am
index d471a3f92..a985f42a8 100644
--- a/mod_glusterfs/apache/2.2/Makefile.am
+++ b/xlators/features/compress/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/compress/src/Makefile.am b/xlators/features/compress/src/Makefile.am
new file mode 100644
index 000000000..4a64b52a9
--- /dev/null
+++ b/xlators/features/compress/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = cdc.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+noinst_HEADERS = cdc.h cdc-mem-types.h
+
+cdc_la_LDFLAGS = -module -avoidversion $(LIBZ_LIBS)
+
+cdc_la_SOURCES = cdc.c cdc-helper.c
+cdc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
+-shared -nostartfiles  $(LIBZ_CFLAGS)
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/compress/src/cdc-helper.c b/xlators/features/compress/src/cdc-helper.c
new file mode 100644
index 000000000..54432ff45
--- /dev/null
+++ b/xlators/features/compress/src/cdc-helper.c
@@ -0,0 +1,547 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "cdc.h"
+#include "cdc-mem-types.h"
+
+#ifdef HAVE_LIB_Z
+#include "zlib.h"
+#endif
+
+#ifdef HAVE_LIB_Z
+/* gzip header looks something like this
+ * (RFC 1950)
+ *
+ * +---+---+---+---+---+---+---+---+---+---+
+ * |ID1|ID2|CM |FLG|     MTIME     |XFL|OS |
+ * +---+---+---+---+---+---+---+---+---+---+
+ *
+ * Data is usually sent without this header i.e
+ * Data sent = <compressed-data> + trailer(8)
+ * The trailer contains the checksum.
+ *
+ * gzip_header is added only during debugging.
+ * Refer to the function cdc_dump_iovec_to_disk
+ */
+static const char gzip_header[10] =
+        {
+                '\037', '\213', Z_DEFLATED, 0,
+                0,      0,      0,          0,
+                0,      GF_CDC_OS_ID
+        };
+
+static int32_t
+cdc_next_iovec (xlator_t *this, cdc_info_t *ci)
+{
+        int ret = -1;
+
+        ci->ncount++;
+        /* check for iovec overflow -- should not happen */
+        if (ci->ncount == MAX_IOVEC) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Zlib output buffer overflow"
+                        " ->ncount (%d) | ->MAX_IOVEC (%d)",
+                        ci->ncount, MAX_IOVEC);
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static void
+cdc_put_long (unsigned char *string, unsigned long x)
+{
+        string[0] = (unsigned char) (x & 0xff);
+        string[1] = (unsigned char) ((x & 0xff00) >> 8);
+        string[2] = (unsigned char) ((x & 0xff0000) >> 16);
+        string[3] = (unsigned char) ((x & 0xff000000) >> 24);
+}
+
+static unsigned long
+cdc_get_long (unsigned char *buf)
+{
+        return ((unsigned long) buf[0])
+                | (((unsigned long) buf[1]) << 8)
+                | (((unsigned long) buf[2]) << 16)
+                | (((unsigned long) buf[3]) << 24);
+}
+
+static int32_t
+cdc_init_gzip_trailer (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+{
+        int   ret = -1;
+        char *buf = NULL;
+
+        ret = cdc_next_iovec (this, ci);
+        if (ret)
+                goto out;
+
+        buf = CURR_VEC(ci).iov_base =
+                (char *) GF_CALLOC (1, GF_CDC_VALIDATION_SIZE,
+                                    gf_cdc_mt_gzip_trailer_t);
+
+        if (!CURR_VEC(ci).iov_base)
+                goto out;
+
+        CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
+
+        cdc_put_long ((unsigned char *)&buf[0], ci->crc);
+        cdc_put_long ((unsigned char *)&buf[4], ci->stream.total_in);
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static int32_t
+cdc_alloc_iobuf_and_init_vec (xlator_t *this,
+                              cdc_priv_t *priv, cdc_info_t *ci,
+                              int size)
+{
+        int           ret       = -1;
+        int           alloc_len = 0;
+        struct iobuf *iobuf     = NULL;
+
+        ret = cdc_next_iovec (this, ci);
+        if (ret)
+                goto out;
+
+        alloc_len = size ? size : ci->buffer_size;
+
+        iobuf = iobuf_get2 (this->ctx->iobuf_pool, alloc_len);
+        if (!iobuf)
+                goto out;
+
+        ret = iobref_add (ci->iobref, iobuf);
+        if (ret)
+                goto out;
+
+        /* Initialize this iovec */
+        CURR_VEC(ci).iov_base = iobuf->ptr;
+        CURR_VEC(ci).iov_len  = alloc_len;
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static void
+cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
+{
+        ci->stream.next_out  = (unsigned char *) CURR_VEC(ci).iov_base;
+        ci->stream.avail_out = size ? size : ci->buffer_size;
+}
+
+/* This routine is for testing and debugging only.
+ * Data written = header(10) + <compressed-data> + trailer(8)
+ * So each gzip dump file is at least 18 bytes in size.
+ */
+void
+cdc_dump_iovec_to_disk (xlator_t *this, cdc_info_t *ci, const char *file)
+{
+        int    i      = 0;
+        int    fd     = 0;
+        size_t writen = 0;
+        size_t total_writen = 0;
+
+        fd = open (file, O_WRONLY|O_CREAT|O_TRUNC, 0777 );
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot open file: %s", file);
+                return;
+        }
+
+        writen = write (fd, (char *) gzip_header, 10);
+        total_writen += writen;
+        for (i = 0; i < ci->ncount; i++) {
+                writen = write (fd, (char *) ci->vec[i].iov_base, ci->vec[i].iov_len);
+                total_writen += writen;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                        "dump'd %zu bytes to %s", total_writen, GF_CDC_DEBUG_DUMP_FILE );
+
+        close (fd);
+}
+
+static int32_t
+cdc_flush_libz_buffer (cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
+                       int (*libz_func)(z_streamp, int),
+                       int flush)
+{
+        int32_t ret = Z_OK;
+        int done = 0;
+        unsigned int deflate_len = 0;
+
+        for (;;) {
+                deflate_len = ci->buffer_size - ci->stream.avail_out;
+
+                if (deflate_len != 0) {
+                        CURR_VEC(ci).iov_len = deflate_len;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret) {
+                                ret = Z_MEM_ERROR;
+                                break;
+                        }
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                if (done) {
+                        ci->ncount--;
+                        break;
+                }
+
+                ret = libz_func (&ci->stream, flush);
+
+                if (ret == Z_BUF_ERROR) {
+                        ret = Z_OK;
+                        ci->ncount--;
+                        break;
+                }
+
+                done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
+
+                if (ret != Z_OK && ret != Z_STREAM_END)
+                        break;
+        }
+
+        return ret;
+}
+
+static int32_t
+do_cdc_compress (struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
+                 cdc_info_t *ci)
+{
+        int ret = -1;
+
+        /* Initialize defalte */
+        ret = deflateInit2 (&ci->stream, priv->cdc_level, Z_DEFLATED,
+                            priv->window_size, priv->mem_level,
+                            Z_DEFAULT_STRATEGY);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to init Zlib (retval: %d)", ret);
+                goto out;
+        }
+
+        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+        if (ret)
+                goto out;
+
+        /* setup output buffer */
+        cdc_init_zlib_output_stream (priv, ci, 0);
+
+        /* setup input buffer */
+        ci->stream.next_in  = (unsigned char *) vec->iov_base;
+        ci->stream.avail_in = vec->iov_len;
+
+        ci->crc = crc32 (ci->crc, (const Bytef *) vec->iov_base, vec->iov_len);
+
+        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d",
+                ci->crc, ci->stream.avail_in, ci->buffer_size);
+
+        /* compress !! */
+        while (ci->stream.avail_in != 0) {
+                if (ci->stream.avail_out == 0) {
+
+                        CURR_VEC(ci).iov_len = ci->buffer_size;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret)
+                                break;
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                ret = deflate (&ci->stream, Z_NO_FLUSH);
+                if (ret != Z_OK)
+                        break;
+        }
+
+ out:
+        return ret;
+}
+
+int32_t
+cdc_compress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+              dict_t **xdata)
+{
+        int ret = -1;
+        int i   = 0;
+
+        ci->iobref = iobref_new ();
+        if (!ci->iobref)
+                goto out;
+
+        if (!*xdata) {
+                *xdata = dict_new ();
+                if (!*xdata) {
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot allocate xdata"
+                                " dict");
+                        goto out;
+                }
+        }
+
+        /* data */
+        for (i = 0; i < ci->count; i++) {
+                ret = do_cdc_compress (&ci->vector[i], this, priv, ci);
+                if (ret != Z_OK)
+                        goto deflate_cleanup_out;
+        }
+
+        /* flush zlib buffer */
+        ret = cdc_flush_libz_buffer (priv, this, ci, deflate, Z_FINISH);
+        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Compression Error: ret (%d)", ret);
+                ret = -1;
+                goto deflate_cleanup_out;
+        }
+
+        /* trailer */
+        ret = cdc_init_gzip_trailer (this, priv, ci);
+        if (ret)
+                goto deflate_cleanup_out;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Compressed %ld to %ld bytes",
+                ci->stream.total_in, ci->stream.total_out);
+
+        ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
+
+        /* set deflated canary value for identification */
+        ret = dict_set_int32 (*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
+        if (ret) {
+                /* Send uncompressed data if we can't _tell_ the client
+                 * that deflated data is on it's way. So, we just log
+                 * the faliure and continue as usual.
+                 */
+                 gf_log (this->name, GF_LOG_ERROR,
+                 "Data deflated, but could not set canary"
+                 " value in dict for identification");
+        }
+
+        /* This is to be used in testing */
+        if ( priv->debug ) {
+                cdc_dump_iovec_to_disk (this, ci, GF_CDC_DEBUG_DUMP_FILE );
+        }
+
+ deflate_cleanup_out:
+        (void) deflateEnd(&ci->stream);
+
+ out:
+        return ret;
+}
+
+
+/* deflate content is checked by the presence of a canary
+ * value in the dict as the key
+ */
+static int32_t
+cdc_check_content_for_deflate (dict_t *xdata)
+{
+        return dict_get (xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
+}
+
+static unsigned long
+cdc_extract_crc (char *trailer)
+{
+        return cdc_get_long ((unsigned char *) &trailer[0]);
+}
+
+static unsigned long
+cdc_extract_size (char *trailer)
+{
+        return cdc_get_long ((unsigned char *) &trailer[4]);
+}
+
+static int32_t
+cdc_validate_inflate (cdc_info_t *ci, unsigned long crc,
+                      unsigned long len)
+{
+        return !((crc == ci->crc)
+                 /* inflated length is hidden inside
+                  * Zlib stream struct */
+                 && (len == ci->stream.total_out));
+}
+
+static int32_t
+do_cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+{
+        int            ret          = -1;
+        int            i            = 0;
+        int            len          = 0;
+        char          *inflte       = NULL;
+        char          *trailer      = NULL;
+        struct iovec   vec          = {0,};
+        unsigned long  computed_crc = 0;
+        unsigned long  computed_len = 0;
+
+        ret = inflateInit2 (&ci->stream, priv->window_size);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Zlib: Unable to initialize inflate");
+                goto out;
+        }
+
+        vec = THIS_VEC(ci, 0);
+
+        trailer = (char *) (((char *) vec.iov_base) + vec.iov_len
+                - GF_CDC_VALIDATION_SIZE);
+
+        /* CRC of uncompressed data */
+        computed_crc = cdc_extract_crc (trailer);
+
+        /* size of uncomrpessed data */
+        computed_len = cdc_extract_size (trailer);
+
+        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
+                computed_crc, computed_len, ci->buffer_size);
+
+        inflte = vec.iov_base ;
+        len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
+
+        /* allocate buffer of the original length of the data */
+        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+        if (ret)
+                goto out;
+
+        /* setup output buffer */
+        cdc_init_zlib_output_stream (priv, ci, 0);
+
+        /* setup input buffer */
+        ci->stream.next_in = (unsigned char *) inflte;
+        ci->stream.avail_in = len;
+
+        while (ci->stream.avail_in != 0) {
+                if (ci->stream.avail_out == 0) {
+                        CURR_VEC(ci).iov_len = ci->buffer_size;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret)
+                                break;
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                ret = inflate (&ci->stream, Z_NO_FLUSH);
+                if (ret == Z_STREAM_ERROR)
+                        break;
+        }
+
+        /* flush zlib buffer */
+        ret = cdc_flush_libz_buffer (priv, this, ci, inflate, Z_SYNC_FLUSH);
+        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Decompression Error: ret (%d)", ret);
+                ret = -1;
+                goto out;
+        }
+
+        /* compute CRC of the uncompresses data to check for
+         * correctness */
+
+        for (i = 0; i < ci->ncount; i++) {
+                ci->crc = crc32 (ci->crc,
+                                 (const Bytef *) ci->vec[i].iov_base,
+                                 ci->vec[i].iov_len);
+        }
+
+        /* validate inflated data */
+        ret = cdc_validate_inflate (ci, computed_crc, computed_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Checksum or length mismatched in inflated data");
+        }
+
+ out:
+        return ret;
+}
+
+int32_t
+cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+                dict_t *xdata)
+{
+        int32_t ret = -1;
+
+        /* check for deflate content */
+        if (!cdc_check_content_for_deflate (xdata)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Content not deflated, passing through ...");
+                goto passthrough_out;
+        }
+
+        ci->iobref = iobref_new ();
+        if (!ci->iobref)
+                goto passthrough_out;
+
+        /* do we need to do this? can we assume that one iovec
+         * will hold per request data everytime?
+         *
+         * server/client protocol seems to deal with a single
+         * iovec even if op_ret > 1M. So, it looks ok to
+         * assume that a single iovec will contain all the
+         * data (This saves us a lot from finding the trailer
+         * and the data since it could have been split-up onto
+         * two adjacent iovec's.
+         *
+         * But, in case this translator is loaded above quick-read
+         * for some reason, then it's entirely possible that we get
+         * multiple iovec's...
+         *
+         * This case (handled below) is not tested. (by loading the
+         * xlator below quick-read)
+         */
+
+        /* @@ I_HOPE_THIS_IS_NEVER_HIT */
+        if (ci->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING, "unable to handle"
+                        " multiple iovecs (%d in number)", ci->count);
+                goto inflate_cleanup_out;
+                /* TODO: coallate all iovecs in one */
+        }
+
+        ret = do_cdc_decompress (this, priv, ci);
+        if (ret)
+                goto inflate_cleanup_out;
+
+        ci->nbytes = ci->stream.total_out;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Inflated %ld to %ld bytes",
+                ci->stream.total_in, ci->stream.total_out);
+
+ inflate_cleanup_out:
+        (void) inflateEnd (&ci->stream);
+
+ passthrough_out:
+        return ret;
+}
+
+#endif
diff --git a/xlators/features/compress/src/cdc-mem-types.h b/xlators/features/compress/src/cdc-mem-types.h
new file mode 100644
index 000000000..efa008059
--- /dev/null
+++ b/xlators/features/compress/src/cdc-mem-types.h
@@ -0,0 +1,22 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CDC_MEM_TYPES_H
+#define __CDC_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_cdc_mem_types {
+        gf_cdc_mt_priv_t         = gf_common_mt_end + 1,
+        gf_cdc_mt_vec_t          = gf_common_mt_end + 2,
+        gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
+};
+
+#endif
diff --git a/xlators/features/compress/src/cdc.c b/xlators/features/compress/src/cdc.c
new file mode 100644
index 000000000..eb7d87c56
--- /dev/null
+++ b/xlators/features/compress/src/cdc.c
@@ -0,0 +1,342 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "cdc.h"
+#include "cdc-mem-types.h"
+
+static void
+cdc_cleanup_iobref (cdc_info_t *ci)
+{
+        assert(ci->iobref != NULL);
+        iobref_clear (ci->iobref);
+}
+
+int32_t
+cdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iovec *vector, int32_t count,
+               struct iatt *stbuf, struct iobref *iobref,
+               dict_t *xdata)
+{
+        int         ret   = -1;
+        cdc_priv_t *priv  = NULL;
+        cdc_info_t  ci    = {0,};
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
+        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
+
+        priv = this->private;
+
+        if (op_ret <= 0)
+                goto default_out;
+
+        if ( (priv->min_file_size != 0)
+             && (op_ret < priv->min_file_size) )
+                goto default_out;
+
+        ci.count       = count;
+        ci.ibytes      = op_ret;
+        ci.vector      = vector;
+        ci.buf         = NULL;
+        ci.iobref      = NULL;
+        ci.ncount      = 0;
+        ci.crc         = 0;
+        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+/* A readv compresses on the server side and decompresses on the client side
+ */
+        if (priv->op_mode == GF_CDC_MODE_SERVER) {
+                ret = cdc_compress (this, priv, &ci, &xdata);
+        } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+                ret = cdc_decompress (this, priv, &ci, xdata);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid operation mode (%d)", priv->op_mode);
+        }
+
+        if (ret)
+                goto default_out;
+
+        STACK_UNWIND_STRICT (readv, frame, ci.nbytes, op_errno,
+                             ci.vec, ci.ncount, stbuf, iobref,
+                             xdata);
+        cdc_cleanup_iobref (&ci);
+        return 0;
+
+ default_out:
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
+                             vector, count, stbuf, iobref, xdata);
+        return 0;
+}
+
+int32_t
+cdc_readv (call_frame_t *frame, xlator_t *this,
+           fd_t *fd, size_t size, off_t offset, uint32_t flags,
+           dict_t *xdata)
+{
+        fop_readv_cbk_t cbk = NULL;
+
+#ifdef HAVE_LIB_Z
+        cbk = cdc_readv_cbk;
+#else
+        cbk = default_readv_cbk;
+#endif
+        STACK_WIND (frame, cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+        return 0;
+}
+
+int32_t
+cdc_writev_cbk (call_frame_t *frame,
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+	return 0;
+}
+
+int32_t
+cdc_writev (call_frame_t *frame,
+            xlator_t *this,
+            fd_t *fd,
+            struct iovec *vector,
+            int32_t count,
+            off_t offset,
+            uint32_t flags,
+            struct iobref *iobref, dict_t *xdata)
+{
+	int	     ret   = -1;
+	cdc_priv_t  *priv  = NULL;
+	cdc_info_t   ci    = {0,};
+	size_t       isize = 0;
+
+	GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
+	GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
+
+	priv = this->private;
+
+	isize =  iov_length(vector, count);
+
+	if (isize <= 0)
+                goto default_out;
+
+        if ( (priv->min_file_size != 0)
+             && (isize < priv->min_file_size) )
+                goto default_out;
+
+        ci.count       = count;
+        ci.ibytes      = isize;
+        ci.vector      = vector;
+        ci.buf         = NULL;
+        ci.iobref      = NULL;
+        ci.ncount      = 0;
+        ci.crc         = 0;
+        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+/* A writev compresses on the client side and decompresses on the server side
+ */
+	if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+		ret = cdc_compress (this, priv, &ci, &xdata);
+	} else if (priv->op_mode == GF_CDC_MODE_SERVER) {
+		ret = cdc_decompress (this, priv, &ci, xdata);
+	} else {
+		gf_log (this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ", priv->op_mode);
+	}
+
+	if (ret)
+		goto default_out;
+
+	STACK_WIND (frame,
+                    cdc_writev_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev,
+                    fd, ci.vec, ci.ncount, offset, flags,
+                    iobref, xdata);
+
+        cdc_cleanup_iobref (&ci);
+        return 0;
+
+ default_out:
+	STACK_WIND (frame,
+                    cdc_writev_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev,
+                    fd, vector, count, offset, flags,
+                    iobref, xdata);
+	return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        int         ret      = -1;
+        char       *temp_str = NULL;
+        cdc_priv_t *priv     = NULL;
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, err);
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Need subvolume == 1");
+                goto err;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Dangling volume. Check volfile");
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_cdc_mt_priv_t);
+        if (!priv) {
+                goto err;
+        }
+
+        /* Check if debug mode is turned on */
+        GF_OPTION_INIT ("debug", priv->debug, bool, err);
+        if( priv->debug ) {
+                gf_log (this->name, GF_LOG_DEBUG, "CDC debug option turned on");
+        }
+
+        /* Set Gzip Window Size */
+        GF_OPTION_INIT ("window-size", priv->window_size, int32, err);
+        if ( (priv->window_size > GF_CDC_MAX_WINDOWSIZE)
+             || (priv->window_size < GF_CDC_DEF_WINDOWSIZE) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip window size (%d), using default",
+                        priv->window_size);
+                priv->window_size = GF_CDC_DEF_WINDOWSIZE;
+        }
+
+        /* Set Gzip (De)Compression Level */
+        GF_OPTION_INIT ("compression-level", priv->cdc_level, int32, err);
+        if ( ((priv->cdc_level < 1) || (priv->cdc_level > 9))
+             && (priv->cdc_level != GF_CDC_DEF_COMPRESSION) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip (de)compression level (%d),"
+                        " using default", priv->cdc_level);
+                priv->cdc_level = GF_CDC_DEF_COMPRESSION;
+        }
+
+        /* Set Gzip Memory Level */
+        GF_OPTION_INIT ("mem-level", priv->mem_level, int32, err);
+        if ( (priv->mem_level < 1) || (priv->mem_level > 9) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip memory level, using the default");
+                priv->mem_level = GF_CDC_DEF_MEMLEVEL;
+        }
+
+        /* Set min file size to enable compression */
+        GF_OPTION_INIT ("min-size", priv->min_file_size, int32, err);
+
+        /* Mode of operation - Server/Client */
+        ret = dict_get_str (this->options, "mode", &temp_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Operation mode not specified !!");
+                goto err;
+        }
+
+        if (GF_CDC_MODE_IS_CLIENT (temp_str)) {
+                priv->op_mode = GF_CDC_MODE_CLIENT;
+        } else if (GF_CDC_MODE_IS_SERVER (temp_str)) {
+                priv->op_mode = GF_CDC_MODE_SERVER;
+        } else {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Bogus operation mode (%s) specified", temp_str);
+                goto err;
+        }
+
+        this->private = priv;
+        gf_log (this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",temp_str);
+        return 0;
+
+ err:
+        if (priv)
+                GF_FREE (priv);
+
+        return -1;
+}
+
+void
+fini (xlator_t *this)
+{
+        cdc_priv_t *priv = this->private;
+
+        if (priv)
+                GF_FREE (priv);
+        this->private = NULL;
+        return;
+}
+
+struct xlator_fops fops = {
+        .readv  = cdc_readv,
+        .writev = cdc_writev,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {"window-size"},
+          .default_value = "-15",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Size of the zlib history buffer."
+        },
+        { .key  = {"mem-level"},
+          .default_value = "8",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Memory allocated for internal compression state.\
+                          1 uses minimum memory but is slow and reduces \
+                          compression ratio; memLevel=9 uses maximum memory \
+                          for optimal speed. The default value is 8."
+        },
+        { .key  = {"compression-level"},
+          .default_value = "-1",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Compression levels \
+                          0 : no compression, 1 : best speed, \
+                          9 : best compression, -1 : default compression "
+        },
+        { .key  = {"min-size"},
+          .default_value = "0",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Data is compressed only when its size exceeds this."
+        },
+        { .key  = {"mode"},
+          .value = {"server", "client"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Set on the basis of where the xlator is loaded."
+        },
+        { .key = {"debug"},
+          .default_value = "false",
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "This is used in testing. Will dump compressed data \
+                          to disk as a gzip file."
+        },
+        { .key  = {NULL}
+        },
+};
diff --git a/xlators/features/compress/src/cdc.h b/xlators/features/compress/src/cdc.h
new file mode 100644
index 000000000..71f4d2317
--- /dev/null
+++ b/xlators/features/compress/src/cdc.h
@@ -0,0 +1,107 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CDC_H
+#define __CDC_H
+
+#ifdef HAVE_LIB_Z
+#include "zlib.h"
+#endif
+
+#include "xlator.h"
+
+#ifndef MAX_IOVEC
+#define MAX_IOVEC 16
+#endif
+
+typedef struct cdc_priv {
+        int window_size;
+        int mem_level;
+        int cdc_level;
+        int min_file_size;
+        int op_mode;
+        gf_boolean_t debug;
+        gf_lock_t lock;
+} cdc_priv_t;
+
+typedef struct cdc_info {
+        /* input bits */
+        int            count;
+        int32_t        ibytes;
+        struct iovec  *vector;
+        struct iatt   *buf;
+
+        /* output bits */
+        int            ncount;
+        int            nbytes;
+        int            buffer_size;
+        struct iovec   vec[MAX_IOVEC];
+        struct iobref *iobref;
+
+        /* zlib bits */
+#ifdef HAVE_LIB_Z
+        z_stream      stream;
+#endif
+        unsigned long crc;
+} cdc_info_t;
+
+#define NVEC(ci) (ci->ncount - 1)
+#define CURR_VEC(ci) ci->vec[ci->ncount - 1]
+#define THIS_VEC(ci, i) ci->vector[i]
+
+/* Gzip defaults */
+#define GF_CDC_DEF_WINDOWSIZE  -15 /* default value */
+#define GF_CDC_MAX_WINDOWSIZE  -8  /* max value     */
+
+#ifdef HAVE_LIB_Z
+#define GF_CDC_DEF_COMPRESSION Z_DEFAULT_COMPRESSION
+#else
+#define GF_CDC_DEF_COMPRESSION -1
+#endif
+
+#define GF_CDC_DEF_MEMLEVEL    8
+#define GF_CDC_DEF_BUFFERSIZE  262144 // 256K - default compression buffer size
+
+/* Operation mode
+ * If xlator is loaded on client, readv decompresses and writev compresses
+ * If xlator is loaded on server, readv compresses and writev decompresses
+ */
+#define GF_CDC_MODE_CLIENT   0
+#define GF_CDC_MODE_SERVER   1
+
+/* min size of data to do cmpression
+ * 0 == compress even 1byte
+ */
+#define GF_CDC_MIN_CHUNK_SIZE 0
+
+#define GF_CDC_VALIDATION_SIZE 8
+
+#define GF_CDC_OS_ID 0xFF
+#define GF_CDC_DEFLATE_CANARY_VAL "deflate"
+#define GF_CDC_DEBUG_DUMP_FILE "/tmp/cdcdump.gz"
+
+#define GF_CDC_MODE_IS_CLIENT(m) \
+        (strcmp (m, "client") == 0)
+
+#define GF_CDC_MODE_IS_SERVER(m) \
+        (strcmp (m, "server") == 0)
+
+int32_t
+cdc_compress (xlator_t *this,
+              cdc_priv_t *priv,
+              cdc_info_t *ci,
+              dict_t **xdata);
+int32_t
+cdc_decompress (xlator_t *this,
+                cdc_priv_t *priv,
+                cdc_info_t *ci,
+                dict_t *xdata);
+
+#endif
diff --git a/xlators/features/filter/src/Makefile.am b/xlators/features/filter/src/Makefile.am
index d473b9ea1..d1fda8b0a 100644
--- a/xlators/features/filter/src/Makefile.am
+++ b/xlators/features/filter/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = filter.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
 
-filter_la_LDFLAGS = -module -avoidversion
+filter_la_LDFLAGS = -module -avoid-version
 
 filter_la_SOURCES = filter.c
 filter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = filter-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/filter/src/filter-mem-types.h b/xlators/features/filter/src/filter-mem-types.h
index de2cb9665..47a17249b 100644
--- a/xlators/features/filter/src/filter-mem-types.h
+++ b/xlators/features/filter/src/filter-mem-types.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __FILTER_MEM_TYPES_H__
 #define __FILTER_MEM_TYPES_H__
 
diff --git a/xlators/features/filter/src/filter.c b/xlators/features/filter/src/filter.c
index eda042f35..1d4887b71 100644
--- a/xlators/features/filter/src/filter.c
+++ b/xlators/features/filter/src/filter.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/performance/stat-prefetch/Makefile.am b/xlators/features/gfid-access/Makefile.am
index af437a64d..af437a64d 100644
--- a/xlators/performance/stat-prefetch/Makefile.am
+++ b/xlators/features/gfid-access/Makefile.am
diff --git a/xlators/features/gfid-access/src/Makefile.am b/xlators/features/gfid-access/src/Makefile.am
new file mode 100644
index 000000000..db53affaa
--- /dev/null
+++ b/xlators/features/gfid-access/src/Makefile.am
@@ -0,0 +1,15 @@
+xlator_LTLIBRARIES = gfid-access.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+gfid_access_la_LDFLAGS = -module -avoid-version
+
+gfid_access_la_SOURCES = gfid-access.c
+gfid_access_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = gfid-access.h gfid-access-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/gfid-access/src/gfid-access-mem-types.h b/xlators/features/gfid-access/src/gfid-access-mem-types.h
new file mode 100644
index 000000000..168d67b43
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access-mem-types.h
@@ -0,0 +1,23 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GFID_ACCESS_MEM_TYPES_H
+#define _GFID_ACCESS_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_changelog_mem_types {
+        gf_gfid_access_mt_priv_t = gf_common_mt_end + 1,
+        gf_gfid_access_mt_gfid_t,
+        gf_gfid_access_mt_end
+};
+
+#endif
+
diff --git a/xlators/features/gfid-access/src/gfid-access.c b/xlators/features/gfid-access/src/gfid-access.c
new file mode 100644
index 000000000..da0ba7e50
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access.c
@@ -0,0 +1,1172 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "gfid-access.h"
+#include "inode.h"
+#include "byte-order.h"
+
+
+
+void
+ga_newfile_args_free (ga_newfile_args_t *args)
+{
+        if (!args)
+                goto out;
+
+        GF_FREE (args->bname);
+
+        if (S_ISLNK (args->st_mode) && args->args.symlink.linkpath) {
+                GF_FREE (args->args.symlink.linkpath);
+                args->args.symlink.linkpath = NULL;
+        }
+
+        mem_put (args);
+out:
+        return;
+}
+
+
+void
+ga_heal_args_free (ga_heal_args_t *args)
+{
+        if (!args)
+                goto out;
+
+        GF_FREE (args->bname);
+
+        mem_put (args);
+out:
+        return;
+}
+
+
+ga_newfile_args_t *
+ga_newfile_parse_args (xlator_t *this, data_t *data)
+{
+        ga_newfile_args_t *args     = NULL;
+        ga_private_t      *priv     = NULL;
+        int                len      = 0;
+        int                blob_len = 0;
+        int                min_len  = 0;
+        void              *blob     = NULL;
+
+        priv = this->private;
+
+        blob = data->data;
+        blob_len = data->len;
+
+        min_len = sizeof (args->uid) + sizeof (args->gid) + sizeof (args->gfid)
+                + sizeof (args->st_mode) + 2 + 2;
+        if (blob_len < min_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid length: Total length is less "
+                        "than minimum length.");
+                goto err;
+        }
+
+        args = mem_get0 (priv->newfile_args_pool);
+        if (args == NULL)
+                goto err;
+
+        args->uid = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        args->gid = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        memcpy (args->gfid, blob, sizeof (args->gfid));
+        blob += sizeof (args->gfid);
+        blob_len -= sizeof (args->gfid);
+
+        args->st_mode = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        len = strnlen (blob, blob_len);
+        if (len == blob_len)
+        if (len == blob_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gfid: %s. No null byte present.",
+                        args->gfid);
+                goto err;
+        }
+
+        args->bname = GF_CALLOC (1, (len + 1), gf_common_mt_char);
+        if (args->bname == NULL)
+                goto err;
+
+        memcpy (args->bname, blob, (len + 1));
+        blob += (len + 1);
+        blob_len -= (len + 1);
+
+        if (S_ISDIR (args->st_mode)) {
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mkdir.mode = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mkdir.umask = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+                if (blob_len < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+        } else if (S_ISLNK (args->st_mode)) {
+                len = strnlen (blob, blob_len);
+                if (len == blob_len) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.symlink.linkpath = GF_CALLOC (1, len + 1,
+                                                         gf_common_mt_char);
+                if (args->args.symlink.linkpath == NULL)
+                        goto err;
+
+                memcpy (args->args.symlink.linkpath, blob, (len + 1));
+                blob += (len + 1);
+                blob_len -= (len + 1);
+        } else {
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.mode = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.rdev = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.umask = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+        }
+
+        if (blob_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gfid: %s. Invalid length",
+                        args->gfid);
+                goto err;
+        }
+
+        return args;
+
+err:
+        if (args)
+                ga_newfile_args_free (args);
+
+        return NULL;
+}
+
+ga_heal_args_t *
+ga_heal_parse_args (xlator_t *this, data_t *data)
+{
+        ga_heal_args_t *args     = NULL;
+        ga_private_t   *priv     = NULL;
+        void           *blob     = NULL;
+        int             len      = 0;
+        int             blob_len = 0;
+
+        blob = data->data;
+        blob_len = data->len;
+
+        priv = this->private;
+
+        /* bname should at least contain a character */
+        if (blob_len < (sizeof (args->gfid) + 2))
+                goto err;
+
+        args = mem_get0 (priv->heal_args_pool);
+        if (!args)
+                goto err;
+
+        memcpy (args->gfid, blob, sizeof (args->gfid));
+        blob += sizeof (args->gfid);
+        blob_len -= sizeof (args->gfid);
+
+        len = strnlen (blob, blob_len);
+        if (len == blob_len)
+                goto err;
+
+        args->bname = GF_CALLOC (1, len + 1, gf_common_mt_char);
+        if (!args->bname)
+                goto err;
+
+        memcpy (args->bname, blob, len);
+        blob_len -= (len + 1);
+
+        if (blob_len)
+                goto err;
+
+        return args;
+
+err:
+        if (args)
+                ga_heal_args_free (args);
+
+        return NULL;
+}
+
+static int32_t
+ga_fill_tmp_loc (loc_t *loc, xlator_t *this, char *gfid,
+                 char *bname, dict_t *xdata, loc_t *new_loc)
+{
+        int       ret    = -1;
+        uint64_t  value  = 0;
+        inode_t  *parent = NULL;
+
+        parent = loc->inode;
+        ret = inode_ctx_get (loc->inode, this, &value);
+        if (!ret) {
+                parent = (void *)value;
+        }
+
+        /* parent itself should be looked up */
+        uuid_copy (new_loc->pargfid, parent->gfid);
+        new_loc->parent = inode_ref (parent);
+
+        new_loc->inode = inode_grep (parent->table, parent, bname);
+        if (!new_loc->inode)
+                new_loc->inode = inode_new (parent->table);
+
+        loc_path (new_loc, bname);
+        new_loc->name = basename (new_loc->path);
+
+        /* As GFID would not be set on the entry yet, lets not send entry
+           gfid in the request */
+        /*uuid_copy (new_loc->gfid, (const unsigned char *)gfid); */
+
+        ret = dict_set_static_bin (xdata, "gfid-req", gfid, 16);
+        if (ret < 0)
+                goto out;
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+
+static gf_boolean_t
+__is_gfid_access_dir (uuid_t gfid)
+{
+        uuid_t  aux_gfid;
+
+        memset (aux_gfid, 0, 16);
+        aux_gfid[15] = GF_AUX_GFID;
+
+        if (uuid_compare (gfid, aux_gfid) == 0)
+                return _gf_true;
+
+        return _gf_false;
+}
+
+int32_t
+ga_forget (xlator_t *this, inode_t *inode)
+{
+        int       ret = -1;
+        uint64_t  value = 0;
+        inode_t  *tmp_inode = NULL;
+
+        ret = inode_ctx_del (inode, this, &value);
+        if (ret)
+                goto out;
+
+        tmp_inode = (void *)value;
+        inode_unref (tmp_inode);
+
+out:
+        return 0;
+}
+
+
+static int
+ga_heal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno,
+             inode_t *inode, struct iatt *stat, dict_t *dict,
+             struct iatt *postparent)
+{
+        call_frame_t *orig_frame = NULL;
+
+        orig_frame = frame->local;
+        frame->local = NULL;
+
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        STACK_DESTROY (frame->root);
+
+        STACK_UNWIND_STRICT (setxattr, orig_frame, op_ret, op_errno, dict);
+
+        return 0;
+}
+
+static int
+ga_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 inode_t *inode, struct iatt *buf,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t *xdata)
+{
+        call_frame_t *orig_frame = NULL;
+
+        orig_frame = frame->local;
+        frame->local = NULL;
+
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        STACK_DESTROY (frame->root);
+
+        STACK_UNWIND_STRICT (setxattr, orig_frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+ga_new_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+              dict_t *xdata)
+{
+        int                ret       = -1;
+        ga_newfile_args_t *args      = NULL;
+        loc_t              tmp_loc   = {0,};
+        call_frame_t      *new_frame = NULL;
+        mode_t             mode      = 0;
+
+        args = ga_newfile_parse_args (this, data);
+        if (!args)
+                goto out;
+
+        if (!xdata)
+                xdata = dict_new ();
+
+        ret = ga_fill_tmp_loc (loc, this, args->gfid,
+                               args->bname, xdata, &tmp_loc);
+        if (ret)
+                goto out;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
+        new_frame->local = (void *)frame;
+
+        new_frame->root->uid = args->uid;
+        new_frame->root->gid = args->gid;
+
+        if (S_ISDIR (args->st_mode)) {
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                            &tmp_loc, args->args.mkdir.mode,
+                            args->args.mkdir.umask, xdata);
+        } else if (S_ISLNK (args->st_mode)) {
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                            args->args.symlink.linkpath,
+                            &tmp_loc, 0, xdata);
+        } else {
+                /* use 07777 (4 7s) for considering the Sticky bits etc) */
+                mode = (S_IFMT & args->st_mode) |
+                        (07777 | args->args.mknod.mode);;
+
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                            &tmp_loc, mode,
+                            args->args.mknod.rdev, args->args.mknod.umask,
+                            xdata);
+        }
+
+        ret = 0;
+out:
+        ga_newfile_args_free (args);
+
+        return ret;
+}
+
+int32_t
+ga_heal_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+               dict_t *xdata)
+{
+        int             ret       = -1;
+        ga_heal_args_t *args      = NULL;
+        loc_t           tmp_loc   = {0,};
+        call_frame_t   *new_frame = NULL;
+
+        args = ga_heal_parse_args (this, data);
+        if (!args)
+                goto out;
+
+        if (!xdata)
+                xdata = dict_new ();
+
+        ret = ga_fill_tmp_loc (loc, this, args->gfid, args->bname,
+                               xdata, &tmp_loc);
+        if (ret)
+                goto out;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
+        new_frame->local = (void *)frame;
+
+        STACK_WIND (new_frame, ga_heal_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD(this)->fops->lookup,
+                    &tmp_loc, xdata);
+
+        ret = 0;
+out:
+        if (args)
+                ga_heal_args_free (args);
+
+        return ret;
+}
+
+int32_t
+ga_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+ga_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+
+        data_t  *data     = NULL;
+        int      op_errno = ENOMEM;
+        int      ret      = 0;
+        inode_t *unref    = NULL;
+
+        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
+            ((loc->parent &&
+              __is_root_gfid (loc->parent->gfid)) ||
+             __is_root_gfid (loc->pargfid))) {
+                op_errno = EPERM;
+                goto err;
+        }
+
+        data = dict_get (dict, GF_FUSE_AUX_GFID_NEWFILE);
+        if (data) {
+                ret = ga_new_entry (frame, this, loc, data, xdata);
+                if (ret)
+                        goto err;
+                return 0;
+        }
+
+        data = dict_get (dict, GF_FUSE_AUX_GFID_HEAL);
+        if (data) {
+                ret = ga_heal_entry (frame, this, loc, data, xdata);
+                if (ret)
+                        goto err;
+                return 0;
+        }
+
+        //If the inode is a virtual inode change the inode otherwise perform
+        //the operation on same inode
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, ga_setxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+        return 0;
+}
+
+
+int32_t
+ga_virtual_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        int       j           = 0;
+        int       i           = 0;
+        int       ret         = 0;
+        uint64_t  temp_ino    = 0;
+        inode_t  *cbk_inode   = NULL;
+        inode_t  *true_inode  = NULL;
+        uuid_t    random_gfid = {0,};
+
+        if (frame->local)
+                cbk_inode = frame->local;
+        else
+                cbk_inode = inode;
+
+        frame->local = NULL;
+        if (op_ret)
+                goto unwind;
+
+        if (!IA_ISDIR (buf->ia_type))
+                goto unwind;
+
+        /* need to send back a different inode for linking in itable */
+        if (cbk_inode == inode) {
+                /* check if the inode is in the 'itable' or
+                   if its just previously discover()'d inode */
+                true_inode = inode_find (inode->table, buf->ia_gfid);
+                if (!true_inode) {
+                        cbk_inode = inode_new (inode->table);
+
+                        if (!cbk_inode) {
+                                op_ret = -1;
+                                op_errno = ENOMEM;
+                                goto unwind;
+                        }
+                        /* the inode is not present in itable, ie, the actual
+                           path is not yet looked up. Use the current inode
+                           itself for now */
+                        inode_ref (inode);
+                } else {
+                        /* 'inode_ref()' has been done in inode_find() */
+                        inode = true_inode;
+                }
+
+                ret = inode_ctx_put (cbk_inode, this, (uint64_t)inode);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to set the inode ctx with"
+                                "the actual inode");
+                        if (inode)
+                                inode_unref (inode);
+                }
+                inode = NULL;
+        }
+
+        if (!uuid_is_null (cbk_inode->gfid)) {
+                /* if the previous linked inode is used, use the
+                   same gfid */
+                uuid_copy (random_gfid, cbk_inode->gfid);
+        } else {
+                /* replace the buf->ia_gfid to a random gfid
+                   for directory, for files, what we received is fine */
+                uuid_generate (random_gfid);
+        }
+
+        uuid_copy (buf->ia_gfid, random_gfid);
+
+        for (i = 15; i > (15 - 8); i--) {
+                temp_ino += (uint64_t)(buf->ia_gfid[i]) << j;
+                j += 8;
+        }
+        buf->ia_ino = temp_ino;
+
+unwind:
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, cbk_inode, buf,
+                             xdata, postparent);
+
+        return 0;
+}
+
+int32_t
+ga_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        ga_private_t *priv = NULL;
+
+        /* if the entry in question is not 'root',
+           then follow the normal path */
+        if (op_ret || !__is_root_gfid(buf->ia_gfid))
+                goto unwind;
+
+        priv = this->private;
+
+        /* do we need to copy root stbuf everytime? */
+        /* mostly yes, as we want to have the 'stat' info show latest
+           in every _cbk() */
+
+        /* keep the reference for root stat buf */
+        priv->root_stbuf = *buf;
+        priv->gfiddir_stbuf = priv->root_stbuf;
+        priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
+        priv->gfiddir_stbuf.ia_ino = GF_AUX_GFID;
+
+unwind:
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
+        return 0;
+}
+
+int32_t
+ga_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        ga_private_t *priv     = NULL;
+        int           ret      = -1;
+        uuid_t        tmp_gfid = {0,};
+        loc_t         tmp_loc  = {0,};
+        uint64_t      value    = 0;
+        inode_t      *inode    = NULL;
+        inode_t      *true_inode    = NULL;
+        int32_t       op_errno = ENOENT;
+
+        /* if its discover(), no need for any action here */
+        if (!loc->name)
+                goto wind;
+
+        /* if its revalidate, and inode is not of type directory,
+           proceed with 'wind' */
+        if (loc->inode && loc->inode->ia_type &&
+            !IA_ISDIR (loc->inode->ia_type))
+                goto wind;
+
+        priv = this->private;
+
+        /* need to check if the lookup is on virtual dir */
+        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
+            ((loc->parent && __is_root_gfid (loc->parent->gfid)) ||
+             __is_root_gfid (loc->pargfid))) {
+                /* this means, the query is on '/.gfid', return the fake stat,
+                   and say success */
+
+                STACK_UNWIND_STRICT (lookup, frame, 0, 0, loc->inode,
+                                     &priv->gfiddir_stbuf, xdata,
+                                     &priv->root_stbuf);
+                return 0;
+        }
+
+        /* now, check if the lookup() is on an existing entry,
+           but on gfid-path */
+        if (!((loc->parent && __is_gfid_access_dir (loc->parent->gfid)) ||
+              __is_gfid_access_dir (loc->pargfid)))
+                goto wind;
+
+        /* make sure the 'basename' is actually a 'canonical-gfid',
+           otherwise, return error */
+        ret = uuid_parse (loc->name, tmp_gfid);
+        if (ret)
+                goto err;
+
+        /* if its fresh lookup, go ahead and send it down, if not,
+           for directory, we need indirection to actual dir inode */
+        if (!(loc->inode && loc->inode->ia_type))
+                goto discover;
+
+        /* revalidate on directory */
+        ret = inode_ctx_get (loc->inode, this, &value);
+        if (ret)
+                goto err;
+
+        inode = (void *)value;
+
+        /* valid inode, already looked up, work on that */
+        if (inode->ia_type)
+                goto discover;
+
+        /* check if the inode is in the 'itable' or
+           if its just previously discover()'d inode */
+        true_inode = inode_find (loc->inode->table, tmp_gfid);
+        if (true_inode) {
+                /* time do another lookup and update the context
+                   with proper inode */
+                op_errno = ESTALE;
+                goto err;
+        }
+
+discover:
+        /* for the virtual entries, we don't need to send 'gfid-req' key, as
+           for these entries, we don't want to 'set' a new gfid */
+        if (xdata)
+                dict_del (xdata, "gfid-req");
+
+        uuid_copy (tmp_loc.gfid, tmp_gfid);
+
+        /* if revalidate, then we need to have the proper reference */
+        if (inode) {
+                tmp_loc.inode = inode_ref (inode);
+                frame->local = loc->inode;
+        } else {
+                tmp_loc.inode = inode_ref (loc->inode);
+        }
+
+        STACK_WIND (frame, ga_virtual_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
+
+        inode_unref (tmp_loc.inode);
+
+        return 0;
+
+wind:
+        /* used for all the normal lookup path */
+        STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, loc->inode,
+                             &priv->gfiddir_stbuf, xdata,
+                             &priv->root_stbuf);
+        return 0;
+}
+
+int
+ga_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                    xdata);
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, loc->inode,
+                             NULL, NULL, NULL, xdata);
+        return 0;
+}
+
+
+int
+ga_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_create_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL, xdata);
+
+        return 0;
+
+}
+
+int
+ga_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
+            loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_symlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkname, loc, umask, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev,
+                    umask, xdata);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+          dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_rmdir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flag, xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL,
+                             NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+           dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_unlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL,
+                             NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_rename (call_frame_t *frame, xlator_t *this,
+           loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *oldloc_unref = NULL;
+        inode_t *newloc_unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
+        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, oldloc, oldloc_unref,
+                                         handle_newloc);
+
+handle_newloc:
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, newloc, newloc_unref, wind);
+
+wind:
+        STACK_WIND (frame, default_rename_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+
+        if (oldloc_unref)
+                inode_unref (oldloc_unref);
+
+        if (newloc_unref)
+                inode_unref (newloc_unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+
+int
+ga_link (call_frame_t *frame, xlator_t *this,
+         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *oldloc_unref = NULL;
+        inode_t *newloc_unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
+        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, oldloc, oldloc_unref,
+                                         handle_newloc);
+
+handle_newloc:
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, newloc, newloc_unref, wind);
+
+wind:
+        STACK_WIND (frame, default_link_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+
+        if (oldloc_unref)
+                inode_unref (oldloc_unref);
+
+        if (newloc_unref)
+                inode_unref (newloc_unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int32_t
+ga_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            fd_t *fd, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        /* also check if the loc->inode itself is virtual
+           inode, if yes, return with failure, mainly because we
+           can't handle all the readdirp and other things on it. */
+        if (inode_ctx_get (loc->inode, this, NULL) == 0) {
+                op_errno = ENOTSUP;
+                goto err;
+        }
+
+        STACK_WIND (frame, default_opendir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+
+        return 0;
+}
+
+int32_t
+ga_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             const char *name, dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
+         dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct iatt *stbuf, int32_t valid,
+            dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_gfid_access_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
+                        " init failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        ga_private_t *priv = NULL;
+        int ret = -1;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "not configured with exactly one child. exiting");
+                goto out;
+        }
+
+        /* This can be the top of graph in certain cases */
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dangling volume. check volfile ");
+        }
+
+        /* TODO: define a mem-type structure */
+        priv = GF_CALLOC (1, sizeof (*priv), gf_gfid_access_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        priv->newfile_args_pool = mem_pool_new (ga_newfile_args_t, 512);
+        if (!priv->newfile_args_pool)
+                goto out;
+
+        priv->heal_args_pool = mem_pool_new (ga_heal_args_t, 512);
+        if (!priv->heal_args_pool)
+                goto out;
+
+        this->private = priv;
+
+        ret = 0;
+out:
+        if (ret && priv) {
+                if (priv->newfile_args_pool)
+                        mem_pool_destroy (priv->newfile_args_pool);
+                GF_FREE (priv);
+        }
+
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        ga_private_t *priv = NULL;
+        priv = this->private;
+        this->private = NULL;
+
+        if (priv) {
+                if (priv->newfile_args_pool)
+                        mem_pool_destroy (priv->newfile_args_pool);
+                if (priv->heal_args_pool)
+                        mem_pool_destroy (priv->heal_args_pool);
+                GF_FREE (priv);
+        }
+
+        return;
+}
+
+
+struct xlator_fops fops = {
+        .lookup = ga_lookup,
+
+        /* entry fops */
+        .mkdir   = ga_mkdir,
+        .mknod   = ga_mknod,
+        .create  = ga_create,
+        .symlink = ga_symlink,
+        .link    = ga_link,
+        .unlink  = ga_unlink,
+        .rmdir   = ga_rmdir,
+        .rename  = ga_rename,
+
+        /* handle any other directory operations here */
+        .opendir  = ga_opendir,
+        .stat     = ga_stat,
+        .setattr  = ga_setattr,
+        .getxattr = ga_getxattr,
+        .removexattr = ga_removexattr,
+
+        /* special fop to handle more entry creations */
+        .setxattr = ga_setxattr,
+};
+
+struct xlator_cbks cbks = {
+        .forget = ga_forget,
+};
+
+struct volume_options options[] = {
+        /* This translator doesn't take any options, or provide any options */
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/gfid-access/src/gfid-access.h b/xlators/features/gfid-access/src/gfid-access.h
new file mode 100644
index 000000000..e13c9b724
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access.h
@@ -0,0 +1,128 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __GFID_ACCESS_H__
+#define __GFID_ACCESS_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "defaults.h"
+#include "gfid-access-mem-types.h"
+
+#define UUID_CANONICAL_FORM_LEN 36
+
+#define GF_FUSE_AUX_GFID_NEWFILE "glusterfs.gfid.newfile"
+#define GF_FUSE_AUX_GFID_HEAL    "glusterfs.gfid.heal"
+
+#define GF_GFID_KEY "GLUSTERFS_GFID"
+#define GF_GFID_DIR ".gfid"
+#define GF_AUX_GFID 0xd
+
+#define GFID_ACCESS_GET_VALID_DIR_INODE(x,l,unref,lbl) do {             \
+                int       ret       = 0;                                \
+                uint64_t  value     = 0;                                \
+                inode_t  *tmp_inode = NULL;                             \
+                                                                        \
+                /* if its an entry operation, on the virtual */         \
+                /* directory inode as parent, we need to handle */      \
+                /* it properly */                                       \
+                if (l->parent) {                                        \
+                        ret = inode_ctx_get (l->parent, x, &value);     \
+                        if (ret)                                        \
+                                goto lbl;                               \
+                        tmp_inode = (inode_t *)value;                   \
+                        unref = inode_ref (tmp_inode);                  \
+                        l->parent = tmp_inode;                          \
+                        /* if parent is virtual, no need to handle */   \
+                        /* loc->inode */                                \
+                        break;                                          \
+                }                                                       \
+                                                                        \
+                /* if its an inode operation, on the virtual */         \
+                /* directory inode itself, we need to handle */         \
+                /* it properly */                                       \
+                if (l->inode) {                                         \
+                        ret = inode_ctx_get (l->inode, x, &value);      \
+                        if (ret)                                        \
+                                goto lbl;                               \
+                        tmp_inode = (inode_t *)value;                   \
+                        unref = inode_ref (tmp_inode);                  \
+                        l->inode = tmp_inode;                           \
+                }                                                       \
+                                                                        \
+        } while (0)
+
+#define GFID_ACCESS_ENTRY_OP_CHECK(loc,err,lbl)    do {                 \
+                /* need to check if the lookup is on virtual dir */     \
+                if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&  \
+                    ((loc->parent &&                                    \
+                      __is_root_gfid (loc->parent->gfid)) ||            \
+                      __is_root_gfid (loc->pargfid))) {                 \
+                        err = EEXIST;                                   \
+                        goto lbl;                                       \
+                }                                                       \
+                                                                        \
+                /* now, check if the lookup() is on an existing */      \
+                /* entry, but on gfid-path */                           \
+                if ((loc->parent &&                                     \
+                     __is_gfid_access_dir (loc->parent->gfid)) ||       \
+                    __is_gfid_access_dir (loc->pargfid)) {              \
+                        err = EPERM;                                    \
+                        goto lbl;                                       \
+                }                                                       \
+        } while (0)
+
+
+typedef struct {
+        unsigned int  uid;
+        unsigned int  gid;
+        char          gfid[UUID_CANONICAL_FORM_LEN + 1];
+        unsigned int  st_mode;
+        char         *bname;
+
+        union {
+                struct _symlink_in {
+                        char     *linkpath;
+                } __attribute__ ((__packed__)) symlink;
+
+                struct _mknod_in {
+                        unsigned int mode;
+                        unsigned int rdev;
+                        unsigned int umask;
+                } __attribute__ ((__packed__)) mknod;
+
+                struct _mkdir_in {
+                        unsigned int mode;
+                        unsigned int umask;
+                } __attribute__ ((__packed__)) mkdir;
+        } __attribute__ ((__packed__)) args;
+} __attribute__((__packed__)) ga_newfile_args_t;
+
+typedef struct {
+        char      gfid[UUID_CANONICAL_FORM_LEN + 1];
+        char     *bname; /* a null terminated basename */
+} __attribute__((__packed__)) ga_heal_args_t;
+
+struct ga_private {
+        /* root inode's stbuf */
+        struct iatt root_stbuf;
+        struct iatt gfiddir_stbuf;
+        struct mem_pool *newfile_args_pool;
+        struct mem_pool *heal_args_pool;
+};
+typedef struct ga_private ga_private_t;
+
+#endif /* __GFID_ACCESS_H__ */
diff --git a/scheduler/random/Makefile.am b/xlators/features/glupy/Makefile.am
index d471a3f92..a985f42a8 100644
--- a/scheduler/random/Makefile.am
+++ b/xlators/features/glupy/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/glupy/doc/README.md b/xlators/features/glupy/doc/README.md
new file mode 100644
index 000000000..2d7b30ef6
--- /dev/null
+++ b/xlators/features/glupy/doc/README.md
@@ -0,0 +1,44 @@
+This is just the very start for a GlusterFS[1] meta-translator that will
+allow translator code to be written in Python.  It's based on the standard
+Python embedding (not extending) techniques, plus a dash of the ctypes module.
+The interface is a pretty minimal adaptation of the dispatches and callbacks
+from the C API[2] to Python, as follows:
+
+* Dispatch functions and callbacks must be defined on an "xlator" class
+  derived from gluster.Translator so that they'll be auto-registered with
+  the C translator during initialization.
+
+* For each dispatch or callback function you want to intercept, you define a
+  Python function using the xxx\_fop\_t or xxx\_cbk\_t decorator.
+
+* The arguments for each operation are different, so you'll need to refer to
+  the C API.  GlusterFS-specific types are used (though only loc\_t is fully
+  defined so far) and type correctness is enforced by ctypes.
+
+* If you do intercept a dispatch function, it is your responsibility to call
+  xxx\_wind (like STACK\_WIND in the C API but operation-specific) to pass
+  the request to the next translator.  If you do not intercept a function, it
+  will default the same way as for C (pass through to the same operation with
+  the same arguments on the first child translator).
+
+* If you intercept a callback function, it is your responsibility to call
+  xxx\_unwind (like STACK\_UNWIND\_STRICT in the C API) to pass the request back
+  to the caller.
+
+So far only the lookup and create operations are handled this way, to support
+the "negative lookup" example.  Now that the basic infrastructure is in place,
+adding more functions should be very quick, though with that much boilerplate I
+might pause to write a code generator.  I also plan to add structure
+definitions and interfaces for some of the utility functions in libglusterfs
+(especially those having to do with inode and fd context) in the fairly near
+future.  Note that you can also use ctypes to get at anything not explicitly
+exposed to Python already.
+
+_If you're coming here because of the Linux Journal article, please note that
+the code has evolved since that was written. The version that matches the
+article is here:_
+
+https://github.com/jdarcy/glupy/tree/4bbae91ba459ea46ef32f2966562492e4ca9187a
+
+[1] http://www.gluster.org
+[2] http://hekafs.org/dist/xlator_api_2.html
diff --git a/xlators/features/glupy/doc/TESTING b/xlators/features/glupy/doc/TESTING
new file mode 100644
index 000000000..e05f17f49
--- /dev/null
+++ b/xlators/features/glupy/doc/TESTING
@@ -0,0 +1,9 @@
+Loading a translator written in Python using the glupy meta translator
+-------------------------------------------------------------------------------
+'test.vol' is a simple volfile with the debug-trace Python translator on top
+of a brick. The volfile can be mounted using the following command.
+
+$ glusterfs --debug -f test.vol /path/to/mntpt
+
+If then file operations are performed on the newly mounted file system, log
+output would be printed by the Python translator on the standard output.
diff --git a/xlators/features/glupy/doc/test.vol b/xlators/features/glupy/doc/test.vol
new file mode 100644
index 000000000..0751a488c
--- /dev/null
+++ b/xlators/features/glupy/doc/test.vol
@@ -0,0 +1,10 @@
+volume vol-posix
+    type storage/posix
+    option directory /path/to/brick
+end-volume
+
+volume vol-glupy
+    type features/glupy
+    option module-name debug-trace
+    subvolumes vol-posix
+end-volume
diff --git a/xlators/features/glupy/src/Makefile.am b/xlators/features/glupy/src/Makefile.am
new file mode 100644
index 000000000..960862839
--- /dev/null
+++ b/xlators/features/glupy/src/Makefile.am
@@ -0,0 +1,20 @@
+xlator_LTLIBRARIES = glupy.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+glupydir = $(xlatordir)/glupy
+
+glupy_PYTHON = gluster.py negative.py helloworld.py debug-trace.py
+
+glupy_la_LDFLAGS = -module -avoid-version -shared -nostartfiles
+glupy_la_SOURCES = glupy.c
+glupy_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	 -lpthread -l$(BUILD_PYTHON_LIB)
+
+noinst_HEADERS = glupy.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -isystem $(BUILD_PYTHON_INC)
+
+AM_CFLAGS = -Wall -fno-strict-aliasing -DGLUSTER_PYTHON_PATH=\"$(glupydir)\" $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/glupy/src/debug-trace.py b/xlators/features/glupy/src/debug-trace.py
new file mode 100644
index 000000000..53e76546b
--- /dev/null
+++ b/xlators/features/glupy/src/debug-trace.py
@@ -0,0 +1,774 @@
+import sys
+import stat
+from uuid import UUID
+from time import strftime, localtime
+from gluster import *
+# This translator was written primarily to test the fop entry point definitions
+# and structure definitions in 'gluster.py'.
+# It is similar to the debug-trace translator, one of the already available
+# translator types written in C, that logs the arguments passed to the fops and
+# their corresponding cbk functions.
+
+dl.get_id.restype = c_long
+dl.get_id.argtypes = [ POINTER(call_frame_t) ]
+
+dl.get_rootunique.restype = c_uint64
+dl.get_rootunique.argtypes = [ POINTER(call_frame_t) ]
+
+def uuid2str (gfid):
+        return str(UUID(''.join(map("{0:02x}".format, gfid))))
+
+
+def st_mode_from_ia (prot, filetype):
+        st_mode = 0
+        type_bit = 0
+        prot_bit = 0
+
+        if filetype == IA_IFREG:
+                type_bit = stat.S_IFREG
+        elif filetype == IA_IFDIR:
+                type_bit = stat.S_IFDIR
+        elif filetype == IA_IFLNK:
+                type_bit = stat.S_IFLNK
+        elif filetype == IA_IFBLK:
+                type_bit = stat.S_IFBLK
+        elif filetype == IA_IFCHR:
+                type_bit = stat.S_IFCHR
+        elif filetype == IA_IFIFO:
+                type_bit = stat.S_IFIFO
+        elif filetype == IA_IFSOCK:
+                type_bit = stat.S_IFSOCK
+        elif filetype == IA_INVAL:
+                pass
+
+
+        if prot.suid:
+                prot_bit |= stat.S_ISUID
+        if prot.sgid:
+                prot_bit |= stat.S_ISGID
+        if prot.sticky:
+                prot_bit |= stat.S_ISVTX
+
+        if prot.owner.read:
+                prot_bit |= stat.S_IRUSR
+        if prot.owner.write:
+                prot_bit |= stat.S_IWUSR
+        if prot.owner.execn:
+                prot_bit |= stat.S_IXUSR
+
+        if prot.group.read:
+                prot_bit |= stat.S_IRGRP
+        if prot.group.write:
+                prot_bit |= stat.S_IWGRP
+        if prot.group.execn:
+                prot_bit |= stat.S_IXGRP
+
+        if prot.other.read:
+                prot_bit |= stat.S_IROTH
+        if prot.other.write:
+                prot_bit |= stat.S_IWOTH
+        if prot.other.execn:
+                prot_bit |= stat.S_IXOTH
+
+        st_mode = (type_bit | prot_bit)
+
+        return st_mode
+
+
+def trace_stat2str (buf):
+        gfid = uuid2str(buf.contents.ia_gfid)
+        mode = st_mode_from_ia(buf.contents.ia_prot, buf.contents.ia_type)
+        atime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_atime))
+        mtime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_mtime))
+        ctime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_ctime))
+        return ("(gfid={0:s}, ino={1:d}, mode={2:o}, nlink={3:d}, uid ={4:d}, "+
+                "gid ={5:d}, size={6:d}, blocks={7:d}, atime={8:s}, mtime={9:s}, "+
+                "ctime={10:s})").format(gfid, buf.contents.ia_no, mode,
+                                        buf.contents.ia_nlink,
+                                        buf.contents.ia_uid,
+                                        buf.contents.ia_gid,
+                                        buf.contents.ia_size,
+                                        buf.contents.ia_blocks,
+                                        atime_buf, mtime_buf,
+                                        ctime_buf)
+
+class xlator(Translator):
+
+        def __init__(self, c_this):
+                Translator.__init__(self, c_this)
+                self.gfids = {}
+
+        def lookup_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.gfid)
+                print("GLUPY TRACE LOOKUP FOP- {0:d}: gfid={1:s}; " +
+                      "path={2:s}").format(unique, gfid, loc.contents.path)
+                self.gfids[key] = gfid
+                dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def lookup_cbk(self, frame, cookie, this, op_ret, op_errno,
+                       inode, buf, xdata, postparent):
+                unique =dl.get_rootunique(frame)
+                key =dl.get_id(frame)
+                if op_ret == 0:
+                        gfid = uuid2str(buf.contents.ia_gfid)
+                        statstr = trace_stat2str(buf)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE LOOKUP CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *buf={3:s}; " +
+                              "*postparent={4:s}").format(unique, gfid,
+                                                          op_ret, statstr,
+                                                          postparentstr)
+                else:
+                        gfid = self.gfids[key]
+                        print("GLUPY TRACE LOOKUP CBK - {0:d}: gfid={1:s};" +
+                              " op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                      gfid,
+                                                                      op_ret,
+                                                                      op_errno)
+                del self.gfids[key]
+                dl.unwind_lookup(frame, cookie, this, op_ret, op_errno,
+                                 inode, buf, xdata, postparent)
+                return 0
+
+        def create_fop(self, frame, this, loc, flags, mode, umask, fd,
+                       xdata):
+                unique = dl.get_rootunique(frame)
+                gfid = uuid2str(loc.contents.gfid)
+                print("GLUPY TRACE CREATE FOP- {0:d}: gfid={1:s}; path={2:s}; " +
+                      "fd={3:s}; flags=0{4:o}; mode=0{5:o}; " +
+                      "umask=0{6:o}").format(unique, gfid, loc.contents.path,
+                                             fd, flags, mode, umask)
+                dl.wind_create(frame, POINTER(xlator_t)(), loc, flags,mode,
+                               umask, fd, xdata)
+                return 0
+
+        def create_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
+                       inode, buf, preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret >= 0:
+                        gfid = uuid2str(inode.contents.gfid)
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE CREATE CBK- {0:d}: gfid={1:s};" +
+                              " op_ret={2:d}; fd={3:s}; *stbuf={4:s}; " +
+                              "*preparent={5:s};" +
+                              " *postparent={6:s}").format(unique, gfid, op_ret,
+                                                           fd, statstr,
+                                                           preparentstr,
+                                                           postparentstr)
+                else:
+                        print ("GLUPY TRACE CREATE CBK- {0:d}: op_ret={1:d}; " +
+                              "op_errno={2:d}").format(unique, op_ret, op_errno)
+                dl.unwind_create(frame, cookie, this, op_ret, op_errno, fd,
+                                 inode, buf, preparent, postparent, xdata)
+                return 0
+
+        def open_fop(self, frame, this, loc, flags, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE OPEN FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "flags={3:d}; fd={4:s}").format(unique, gfid,
+                                                      loc.contents.path, flags,
+                                                      fd)
+                self.gfids[key] = gfid
+                dl.wind_open(frame, POINTER(xlator_t)(), loc, flags, fd, xdata)
+                return 0
+
+        def open_cbk(self, frame, cookie, this, op_ret, op_errno, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE OPEN CBK- {0:d}: gfid={1:s}; op_ret={2:d}; "
+                      "op_errno={3:d}; *fd={4:s}").format(unique, gfid,
+                                                          op_ret, op_errno, fd)
+                del self.gfids[key]
+                dl.unwind_open(frame, cookie, this, op_ret, op_errno, fd,
+                               xdata)
+                return 0
+
+        def readv_fop(self, frame, this, fd, size, offset, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READV FOP- {0:d}: gfid={1:s}; "+
+                      "fd={2:s}; size ={3:d}; offset={4:d}; " +
+                      "flags=0{5:x}").format(unique, gfid, fd, size, offset,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_readv (frame, POINTER(xlator_t)(), fd, size, offset,
+                               flags, xdata)
+                return 0
+
+        def readv_cbk(self, frame, cookie, this, op_ret, op_errno, vector,
+                      count, buf, iobref, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret >= 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
+                              "op_ret={2:d}; *buf={3:s};").format(unique, gfid,
+                                                                  op_ret,
+                                                                  statstr)
+
+                else:
+                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_readv (frame, cookie, this, op_ret, op_errno,
+                                 vector, count, buf, iobref, xdata)
+                return 0
+
+        def writev_fop(self, frame, this, fd, vector, count, offset, flags,
+                       iobref, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE  WRITEV FOP- {0:d}: gfid={1:s}; " +
+                      "fd={2:s}; count={3:d}; offset={4:d}; " +
+                      "flags=0{5:x}").format(unique, gfid, fd, count, offset,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_writev(frame, POINTER(xlator_t)(), fd, vector, count,
+                               offset, flags, iobref, xdata)
+                return 0
+
+        def writev_cbk(self, frame, cookie, this, op_ret, op_errno, prebuf,
+                       postbuf, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                if op_ret >= 0:
+                        preopstr = trace_stat2str(prebuf)
+                        postopstr = trace_stat2str(postbuf)
+                        print("GLUPY TRACE WRITEV CBK- {0:d}: op_ret={1:d}; " +
+                              "*prebuf={2:s}; " +
+                              "*postbuf={3:s}").format(unique, op_ret, preopstr,
+                                                       postopstr)
+                else:
+                        gfid = self.gfids[key]
+                        print("GLUPY TRACE WRITEV CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_writev (frame, cookie, this, op_ret, op_errno,
+                                  prebuf, postbuf, xdata)
+                return 0
+
+        def opendir_fop(self, frame, this, loc, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE OPENDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "fd={3:s}").format(unique, gfid, loc.contents.path, fd)
+                self.gfids[key] = gfid
+                dl.wind_opendir(frame, POINTER(xlator_t)(), loc, fd, xdata)
+                return 0
+
+        def opendir_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
+                        xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE OPENDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
+                      " op_errno={3:d}; fd={4:s}").format(unique, gfid, op_ret,
+                                                          op_errno, fd)
+                del self.gfids[key]
+                dl.unwind_opendir(frame, cookie, this, op_ret, op_errno,
+                                  fd, xdata)
+                return 0
+
+        def readdir_fop(self, frame, this, fd, size, offset, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READDIR FOP- {0:d}:  gfid={1:s}; fd={2:s}; " +
+                      "size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
+                                                         offset)
+                self.gfids[key] = gfid
+                dl.wind_readdir(frame, POINTER(xlator_t)(), fd, size, offset,
+                                xdata)
+                return 0
+
+        def readdir_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                        xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE READDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
+                      " op_errno={3:d}").format(unique, gfid, op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_readdir(frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata)
+                return 0
+
+        def readdirp_fop(self, frame, this, fd, size, offset, dictionary):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READDIRP FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
+                      " size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
+                                                          offset)
+                self.gfids[key] = gfid
+                dl.wind_readdirp(frame, POINTER(xlator_t)(), fd, size, offset,
+                                 dictionary)
+                return 0
+
+        def readdirp_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                         xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE READDIRP CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_readdirp(frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata)
+                return 0
+
+        def mkdir_fop(self, frame, this, loc, mode, umask, xdata):
+                unique = dl.get_rootunique(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE MKDIR FOP- {0:d}: gfid={1:s}; path={2:s}; " +
+                      "mode={3:d}; umask=0{4:o}").format(unique, gfid,
+                                                         loc.contents.path, mode,
+                                                         umask)
+                dl.wind_mkdir(frame, POINTER(xlator_t)(), loc, mode, umask,
+                              xdata)
+                return 0
+
+        def mkdir_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                      preparent, postparent,  xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret == 0:
+                        gfid = uuid2str(inode.contents.gfid)
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE MKDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *stbuf={3:s}; *prebuf={4:s}; "+
+                              "*postbuf={5:s} ").format(unique, gfid, op_ret,
+                                                        statstr,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE MKDIR CBK- {0:d}:  op_ret={1:d}; "+
+                              "op_errno={2:d}").format(unique, op_ret, op_errno)
+                dl.unwind_mkdir(frame, cookie, this, op_ret, op_errno, inode,
+                                buf, preparent, postparent, xdata)
+                return 0
+
+        def rmdir_fop(self, frame, this, loc, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE RMDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "flags={3:d}").format(unique, gfid, loc.contents.path,
+                                            flags)
+                self.gfids[key] = gfid
+                dl.wind_rmdir(frame, POINTER(xlator_t)(), loc, flags, xdata)
+                return 0
+
+        def rmdir_cbk(self, frame, cookie, this, op_ret, op_errno, preparent,
+                      postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *prebuf={3:s}; "+
+                              "*postbuf={4:s}").format(unique, gfid, op_ret,
+                                                       preparentstr,
+                                                       postparentstr)
+                else:
+                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_rmdir(frame, cookie, this, op_ret, op_errno,
+                                preparent, postparent, xdata)
+                return 0
+
+        def stat_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE STAT FOP- {0:d}: gfid={1:s}; " +
+                      " path={2:s}").format(unique, gfid, loc.contents.path)
+                self.gfids[key] = gfid
+                dl.wind_stat(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def stat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                     xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d};  *buf={3:s};").format(unique,
+                                                                   gfid,
+                                                                   op_ret,
+                                                                   statstr)
+                else:
+                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_stat(frame, cookie, this, op_ret, op_errno,
+                               buf, xdata)
+                return 0
+
+        def fstat_fop(self, frame, this, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FSTAT FOP- {0:d}:  gfid={1:s}; " +
+                      "fd={2:s}").format(unique, gfid, fd)
+                self.gfids[key] = gfid
+                dl.wind_fstat(frame, POINTER(xlator_t)(), fd, xdata)
+                return 0
+
+        def fstat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                      xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; *buf={3:s}").format(unique,
+                                                                  gfid,
+                                                                  op_ret,
+                                                                  statstr)
+                else:
+                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique.
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_fstat(frame, cookie, this, op_ret, op_errno,
+                                buf, xdata)
+                return 0
+
+        def statfs_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                if loc.contents.inode:
+                        gfid = uuid2str(loc.contents.inode.contents.gfid)
+                else:
+                        gfid = "0"
+                print("GLUPY TRACE STATFS FOP- {0:d}: gfid={1:s}; "+
+                      "path={2:s}").format(unique, gfid, loc.contents.path)
+                dl.wind_statfs(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def statfs_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                       xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret == 0:
+                        #TBD: print buf (pointer to an iovec type object)
+                        print("GLUPY TRACE STATFS CBK {0:d}: "+
+                              "op_ret={1:d}").format(unique, op_ret)
+                else:
+                        print("GLUPY TRACE STATFS CBK-  {0:d}"+
+                              "op_ret={1:d}; op_errno={2:d}").format(unique,
+                                                                     op_ret,
+                                                                     op_errno)
+                dl.unwind_statfs(frame, cookie, this, op_ret, op_errno,
+                                 buf, xdata)
+                return 0
+
+        def getxattr_fop(self, frame, this, loc, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE GETXATTR FOP- {0:d}: gfid={1:s}; path={2:s};"+
+                      " name={3:s}").format(unique, gfid, loc.contents.path,
+                                            name)
+                self.gfids[key]=gfid
+                dl.wind_getxattr(frame, POINTER(xlator_t)(), loc, name, xdata)
+                return 0
+
+        def getxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                         dictionary, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE GETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}; "+
+                      " dictionary={4:s}").format(unique, gfid, op_ret, op_errno,
+                                                  dictionary)
+                del self.gfids[key]
+                dl.unwind_getxattr(frame, cookie, this, op_ret, op_errno,
+                                   dictionary, xdata)
+                return 0
+
+        def fgetxattr_fop(self, frame, this, fd, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FGETXATTR FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
+                      "name={3:s}").format(unique, gfid, fd, name)
+                self.gfids[key] = gfid
+                dl.wind_fgetxattr(frame, POINTER(xlator_t)(), fd, name, xdata)
+                return 0
+
+        def fgetxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                          dictionary, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE FGETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d};"+
+                      " dictionary={4:s}").format(unique, gfid, op_ret,
+                                                  op_errno, dictionary)
+                del self.gfids[key]
+                dl.unwind_fgetxattr(frame, cookie, this, op_ret, op_errno,
+                                    dictionary, xdata)
+                return 0
+
+        def setxattr_fop(self, frame, this, loc, dictionary, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE SETXATTR FOP- {0:d}:  gfid={1:s}; path={2:s};"+
+                      " flags={3:d}").format(unique, gfid, loc.contents.path,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_setxattr(frame, POINTER(xlator_t)(), loc, dictionary,
+                                 flags, xdata)
+                return 0
+
+        def setxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE SETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_setxattr(frame, cookie, this, op_ret, op_errno,
+                                   xdata)
+                return 0
+
+        def fsetxattr_fop(self, frame, this, fd, dictionary, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FSETXATTR FOP- {0:d}: gfid={1:s}; fd={2:p}; "+
+                      "flags={3:d}").format(unique, gfid, fd, flags)
+                self.gfids[key] = gfid
+                dl.wind_fsetxattr(frame, POINTER(xlator_t)(), fd, dictionary,
+                                  flags, xdata)
+                return 0
+
+        def fsetxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE FSETXATTR CBK- {0:d}: gfid={1:s};  "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_fsetxattr(frame, cookie, this, op_ret, op_errno,
+                                   xdata)
+                return 0
+
+        def removexattr_fop(self, frame, this, loc, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE REMOVEXATTR FOP- {0:d}:  gfid={1:s}; "+
+                      "path={2:s}; name={3:s}").format(unique, gfid,
+                                                       loc.contents.path,
+                                                       name)
+                self.gfids[key] = gfid
+                dl.wind_removexattr(frame, POINTER(xlator_t)(), loc, name,
+                                    xdata)
+                return 0
+
+        def removexattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                            xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE REMOVEXATTR CBK- {0:d}: gfid={1:s} "+
+                      " op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                              op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_removexattr(frame, cookie, this, op_ret, op_errno,
+                                      xdata)
+                return 0
+
+        def link_fop(self, frame, this, oldloc, newloc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                if (newloc.contents.inode):
+                        newgfid = uuid2str(newloc.contents.inode.contents.gfid)
+                else:
+                        newgfid = "0"
+                oldgfid = uuid2str(oldloc.contents.inode.contents.gfid)
+                print("GLUPY TRACE LINK FOP-{0:d}: oldgfid={1:s}; oldpath={2:s};"+
+                      "newgfid={3:s};"+
+                      "newpath={4:s}").format(unique, oldgfid,
+                                              oldloc.contents.path,
+                                              newgfid,
+                                              newloc.contents.path)
+                self.gfids[key] =  oldgfid
+                dl.wind_link(frame, POINTER(xlator_t)(), oldloc, newloc,
+                             xdata)
+                return 0
+
+        def link_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                     preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE LINK CBK- {0:d}: op_ret={1:d} "+
+                              "*stbuf={2:s}; *prebuf={3:s}; "+
+                              "*postbuf={4:s} ").format(unique, op_ret, statstr,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE LINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; "+
+                              "op_errno={3:d}").format(unique, gfid,
+                                                       op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_link(frame, cookie, this, op_ret, op_errno, inode,
+                               buf, preparent, postparent, xdata)
+                return 0
+
+        def unlink_fop(self, frame, this, loc, xflag, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE UNLINK FOP- {0:d}; gfid={1:s}; path={2:s}; "+
+                      "flag={3:d}").format(unique, gfid, loc.contents.path,
+                                           xflag)
+                self.gfids[key] = gfid
+                dl.wind_unlink(frame, POINTER(xlator_t)(), loc, xflag,
+                               xdata)
+                return 0
+
+        def unlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                       preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE UNLINK CBK- {0:d}: gfid ={1:s}; "+
+                              "op_ret={2:d}; *prebuf={3:s}; "+
+                              "*postbuf={4:s} ").format(unique, gfid, op_ret,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE UNLINK CBK: {0:d}: gfid ={1:s}; "+
+                              "op_ret={2:d}; "+
+                              "op_errno={3:d}").format(unique, gfid, op_ret,
+                                                       op_errno)
+                del self.gfids[key]
+                dl.unwind_unlink(frame, cookie, this, op_ret, op_errno,
+                                 preparent, postparent, xdata)
+                return 0
+
+        def readlink_fop(self, frame, this, loc, size, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE READLINK FOP- {0:d}:  gfid={1:s}; path={2:s};"+
+                      " size={3:d}").format(unique, gfid, loc.contents.path,
+                                            size)
+                self.gfids[key] = gfid
+                dl.wind_readlink(frame, POINTER(xlator_t)(), loc, size,
+                               xdata)
+                return 0
+
+        def readlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                         buf, stbuf, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid  = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(stbuf)
+                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; op_errno={3:d}; *prebuf={4:s}; "+
+                              "*postbuf={5:s} ").format(unique, gfid,
+                                                        op_ret, op_errno,
+                                                        buf, statstr)
+                else:
+                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                      gfid,
+                                                                      op_ret,
+                                                                      op_errno)
+                del self.gfids[key]
+                dl.unwind_readlink(frame, cookie, this, op_ret, op_errno, buf,
+                                   stbuf, xdata)
+                return 0
+
+        def symlink_fop(self, frame, this, linkpath, loc, umask, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE SYMLINK FOP- {0:d}: gfid={1:s}; "+
+                      "linkpath={2:s}; path={3:s};"+
+                      "umask=0{4:o}").format(unique, gfid, linkpath,
+                                             loc.contents.path, umask)
+                self.gfids[key] = gfid
+                dl.wind_symlink(frame, POINTER(xlator_t)(), linkpath, loc,
+                                umask, xdata)
+                return 0
+
+        def symlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                        inode, buf, preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *stbuf={3:s}; *preparent={4:s}; "+
+                              "*postparent={5:s}").format(unique, gfid,
+                                                          op_ret, statstr,
+                                                          preparentstr,
+                                                          postparentstr)
+                else:
+                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_symlink(frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata)
+                return 0
diff --git a/xlators/features/glupy/src/glupy.c b/xlators/features/glupy/src/glupy.c
new file mode 100644
index 000000000..dc86c0071
--- /dev/null
+++ b/xlators/features/glupy/src/glupy.c
@@ -0,0 +1,2470 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  GlusterFS is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published
+  by the Free Software Foundation; either version 3 of the License,
+  or (at your option) any later version.
+
+  GlusterFS is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see
+  <http://www.gnu.org/licenses/>.
+*/
+
+#include <ctype.h>
+#include <sys/uio.h>
+#include <Python.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+#include "glupy.h"
+
+/* UTILITY FUNCTIONS FOR FOP-SPECIFIC CODE */
+
+pthread_key_t gil_init_key;
+
+PyGILState_STATE
+glupy_enter (void)
+{
+#if 0
+        if (!pthread_getspecific(gil_init_key)) {
+                PyEval_ReleaseLock();
+                (void)pthread_setspecific(gil_init_key,(void *)1);
+        }
+#endif
+
+        return PyGILState_Ensure();
+}
+
+void
+glupy_leave (PyGILState_STATE gstate)
+{
+        PyGILState_Release(gstate);
+}
+
+/* FOP: LOOKUP */
+
+int32_t
+glupy_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_LOOKUP]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_lookup_cbk_t)(priv->cbks[GLUPY_LOOKUP]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, xdata, postparent);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
+        return 0;
+}
+
+int32_t
+glupy_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_LOOKUP]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_lookup_t)(priv->fops[GLUPY_LOOKUP]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+        return 0;
+}
+
+void
+wind_lookup (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_lookup_cbk,xl,xl->fops->lookup,loc,xdata);
+}
+
+void
+unwind_lookup (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(lookup,frame,op_ret,op_errno,
+                            inode,buf,xdata,postparent);
+}
+
+void
+set_lookup_fop (long py_this, fop_lookup_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_LOOKUP] = (long)fop;
+}
+
+void
+set_lookup_cbk (long py_this, fop_lookup_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_LOOKUP] = (long)cbk;
+}
+
+/* FOP: CREATE */
+
+int32_t
+glupy_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_CREATE]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_create_cbk_t)(priv->cbks[GLUPY_CREATE]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_CREATE]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_create_t)(priv->fops[GLUPY_CREATE]))(
+                frame, this, loc, flags, mode, umask, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_create_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
+                    fd, xdata);
+        return 0;
+}
+
+void
+wind_create (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_create_cbk,xl, xl->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+}
+
+void
+unwind_create (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_create_fop (long py_this, fop_create_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_CREATE] = (long)fop;
+}
+
+void
+set_create_cbk (long py_this, fop_create_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_CREATE] = (long)cbk;
+}
+
+/* FOP: OPEN */
+
+int32_t
+glupy_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_OPEN]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_open_cbk_t)(priv->cbks[GLUPY_OPEN]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int32_t
+glupy_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            int32_t flags, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_OPEN]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_open_t)(priv->fops[GLUPY_OPEN]))(
+                frame, this, loc, flags, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_open_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+        return 0;
+}
+
+void
+wind_open (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_open_cbk, xl, xl->fops->open, loc, flags,
+                    fd, xdata);
+}
+
+void
+unwind_open (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+}
+
+void
+set_open_fop (long py_this, fop_open_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_OPEN] = (long)fop;
+}
+
+void
+set_open_cbk (long py_this, fop_open_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_OPEN] = (long)cbk;
+}
+
+/* FOP: READV */
+
+int32_t
+glupy_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                 int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                 dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READV]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readv_cbk_t)(priv->cbks[GLUPY_READV]))(
+                frame, cookie, this, op_ret, op_errno,
+                vector, count, stbuf, iobref, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
+                             count, stbuf, iobref, xdata);
+        return 0;
+}
+
+int32_t
+glupy_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READV]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readv_t)(priv->fops[GLUPY_READV]))(
+                frame, this, fd, size, offset, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readv_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_readv (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_readv_cbk, xl, xl->fops->readv, fd, size,
+                    offset, flags, xdata);
+}
+
+void
+unwind_readv (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, struct iovec *vector,
+              int32_t count, struct iatt *stbuf, struct iobref *iobref,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
+                             count, stbuf, iobref, xdata);
+}
+
+void
+set_readv_fop (long py_this, fop_readv_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_READV] = (long)fop;
+}
+
+void
+set_readv_cbk (long py_this, fop_readv_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_READV] = (long)cbk;
+}
+
+/* FOP: WRITEV */
+
+int32_t
+glupy_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_WRITEV]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_writev_cbk_t)(priv->cbks[GLUPY_WRITEV]))(
+                frame, cookie, this, op_ret, op_errno,
+                prebuf, postbuf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int32_t count, off_t offset,
+              uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_WRITEV]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_writev_t)(priv->fops[GLUPY_WRITEV]))(
+                frame, this, fd, vector, count, offset, flags,
+                iobref, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_writev_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
+                    offset, flags, iobref, xdata);
+        return 0;
+}
+
+void
+wind_writev (call_frame_t *frame, xlator_t *xl, fd_t *fd, struct iovec *vector,
+             int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+             dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_writev_cbk, xl, xl->fops->writev, fd, vector,
+                    count, offset, flags, iobref, xdata);
+}
+
+void
+unwind_writev (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+}
+
+void
+set_writev_fop (long py_this, fop_writev_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_WRITEV] = (long)fop;
+}
+
+void
+set_writev_cbk (long py_this, fop_writev_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_WRITEV] = (long)cbk;
+}
+
+
+/* FOP: OPENDIR */
+
+int32_t
+glupy_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd,
+                   dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_OPENDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_opendir_cbk_t)(priv->cbks[GLUPY_OPENDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int32_t
+glupy_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_OPENDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_opendir_t)(priv->fops[GLUPY_OPENDIR]))(
+                frame, this, loc, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_opendir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+        return 0;
+}
+
+void
+wind_opendir (call_frame_t *frame, xlator_t *xl, loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_opendir_cbk,xl,xl->fops->opendir,loc,fd,xdata);
+}
+
+void
+unwind_opendir (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(opendir,frame,op_ret,op_errno,
+                            fd,xdata);
+}
+
+void
+set_opendir_fop (long py_this, fop_opendir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_OPENDIR] = (long)fop;
+}
+
+void
+set_opendir_cbk (long py_this, fop_opendir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_OPENDIR] = (long)cbk;
+}
+
+/* FOP: READDIR */
+
+int32_t
+glupy_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                   dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readdir_cbk_t)(priv->cbks[GLUPY_READDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                entries,  xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               size_t size, off_t offset, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readdir_t)(priv->fops[GLUPY_READDIR]))(
+                frame, this, fd, size, offset, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir,fd, size, offset, xdata);
+        return 0;
+}
+
+void
+wind_readdir(call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+             off_t offset, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_readdir_cbk,xl,xl->fops->readdir,fd,size,offset,xdata);
+}
+
+void
+unwind_readdir (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(readdir,frame,op_ret,op_errno,
+                            entries, xdata);
+}
+
+void
+set_readdir_fop (long py_this, fop_readdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READDIR] = (long)fop;
+}
+
+void
+set_readdir_cbk (long py_this, fop_readdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READDIR] = (long)cbk;
+}
+
+
+/* FOP: READDIRP */
+
+int32_t
+glupy_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READDIRP]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readdirp_cbk_t)(priv->cbks[GLUPY_READDIRP]))(
+                frame, cookie, this, op_ret, op_errno,
+                entries,  xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                size_t size, off_t offset, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READDIRP]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readdirp_t)(priv->fops[GLUPY_READDIRP]))(
+                frame, this, fd, size, offset, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readdirp_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdirp,fd, size, offset, xdata);
+        return 0;
+}
+
+void
+wind_readdirp (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+               off_t offset, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_readdirp_cbk,xl,xl->fops->readdirp,fd,size,offset,xdata);
+}
+
+void
+unwind_readdirp (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                 dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(readdirp,frame,op_ret,op_errno,
+                            entries, xdata);
+}
+
+void
+set_readdirp_fop (long py_this, fop_readdirp_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READDIRP] = (long)fop;
+}
+
+void
+set_readdirp_cbk (long py_this, fop_readdirp_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READDIRP] = (long)cbk;
+}
+
+
+/* FOP:STAT */
+
+int32_t
+glupy_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_STAT]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_stat_cbk_t)(priv->cbks[GLUPY_STAT]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_STAT]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_stat_t)(priv->fops[GLUPY_STAT]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+}
+
+void
+wind_stat (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_stat_cbk,xl,xl->fops->stat,loc,xdata);
+}
+
+void
+unwind_stat (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, struct iatt *buf,
+             dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(stat,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_stat_fop (long py_this, fop_stat_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_STAT] = (long)fop;
+}
+
+void
+set_stat_cbk (long py_this, fop_stat_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_STAT] = (long)cbk;
+}
+
+
+/* FOP: FSTAT */
+
+int32_t
+glupy_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FSTAT]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fstat_cbk_t)(priv->cbks[GLUPY_FSTAT]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FSTAT]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fstat_t)(priv->fops[GLUPY_FSTAT]))(
+                frame, this, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fstat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+}
+
+void
+wind_fstat (call_frame_t *frame, xlator_t *xl, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_fstat_cbk,xl,xl->fops->fstat,fd,xdata);
+}
+
+void
+unwind_fstat (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, struct iatt *buf,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(fstat,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_fstat_fop (long py_this, fop_fstat_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FSTAT] = (long)fop;
+}
+
+void
+set_fstat_cbk (long py_this, fop_fstat_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FSTAT] = (long)cbk;
+}
+
+/* FOP:STATFS */
+
+int32_t
+glupy_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_STATFS]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_statfs_cbk_t)(priv->cbks[GLUPY_STATFS]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_STATFS]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_statfs_t)(priv->fops[GLUPY_STATFS]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_statfs_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->statfs, loc, xdata);
+        return 0;
+}
+
+void
+wind_statfs (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_statfs_cbk,xl,xl->fops->statfs,loc,xdata);
+}
+
+void
+unwind_statfs (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+               dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(statfs,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_statfs_fop (long py_this, fop_statfs_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_STATFS] = (long)fop;
+}
+
+void
+set_statfs_cbk (long py_this, fop_statfs_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_STATFS] = (long)cbk;
+}
+
+
+/* FOP: SETXATTR */
+
+int32_t
+glupy_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_SETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_setxattr_cbk_t)(priv->cbks[GLUPY_SETXATTR]))(
+                frame, cookie, this, op_ret, op_errno,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_SETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_setxattr_t)(priv->fops[GLUPY_SETXATTR]))(
+                frame, this, loc, dict, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_setxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_setxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_setxattr_cbk, xl, xl->fops->setxattr,
+                    loc, dict, flags, xdata);
+}
+
+
+void
+unwind_setxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_setxattr_fop (long py_this, fop_setxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_SETXATTR] = (long)fop;
+}
+
+void
+set_setxattr_cbk (long py_this, fop_setxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_SETXATTR] = (long)cbk;
+}
+
+/* FOP: GETXATTR */
+
+int32_t
+glupy_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_GETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_getxattr_cbk_t)(priv->cbks[GLUPY_GETXATTR]))(
+                frame, cookie, this, op_ret, op_errno, dict,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_GETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_getxattr_t)(priv->fops[GLUPY_GETXATTR]))(
+                frame, this, loc, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_getxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_getxattr_cbk, xl, xl->fops->getxattr,
+                    loc, name, xdata);
+}
+
+
+void
+unwind_getxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict,
+                 dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+
+}
+
+
+void
+set_getxattr_fop (long py_this, fop_getxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_GETXATTR] = (long)fop;
+}
+
+
+void
+set_getxattr_cbk (long py_this, fop_getxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_GETXATTR] = (long)cbk;
+}
+
+/* FOP: FSETXATTR */
+
+int32_t
+glupy_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FSETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fsetxattr_cbk_t)(priv->cbks[GLUPY_FSETXATTR]))(
+                frame, cookie, this, op_ret, op_errno,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FSETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fsetxattr_t)(priv->fops[GLUPY_FSETXATTR]))(
+                frame, this, fd, dict, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fsetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_fsetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fsetxattr_cbk, xl, xl->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+}
+
+
+void
+unwind_fsetxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_fsetxattr_fop (long py_this, fop_fsetxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FSETXATTR] = (long)fop;
+}
+
+void
+set_fsetxattr_cbk (long py_this, fop_fsetxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FSETXATTR] = (long)cbk;
+}
+
+/* FOP: FGETXATTR */
+
+int32_t
+glupy_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FGETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fgetxattr_cbk_t)(priv->cbks[GLUPY_FGETXATTR]))(
+                frame, cookie, this, op_ret, op_errno, dict,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FGETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fgetxattr_t)(priv->fops[GLUPY_FGETXATTR]))(
+                frame, this, fd, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fgetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_fgetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fgetxattr_cbk, xl, xl->fops->fgetxattr,
+                    fd, name, xdata);
+}
+
+
+void
+unwind_fgetxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict,
+                  dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+
+}
+
+
+void
+set_fgetxattr_fop (long py_this, fop_fgetxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FGETXATTR] = (long)fop;
+}
+
+
+void
+set_fgetxattr_cbk (long py_this, fop_fgetxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FGETXATTR] = (long)cbk;
+}
+
+/* FOP:REMOVEXATTR */
+
+int32_t
+glupy_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_REMOVEXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_removexattr_cbk_t)(priv->cbks[GLUPY_REMOVEXATTR]))(
+                frame, cookie, this, op_ret, op_errno, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_REMOVEXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_removexattr_t)(priv->fops[GLUPY_REMOVEXATTR]))(
+                frame, this, loc, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_removexattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+                  const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_removexattr_cbk, xl, xl->fops->removexattr,
+                    loc, name, xdata);
+}
+
+
+void
+unwind_removexattr (call_frame_t *frame, long cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_removexattr_fop (long py_this, fop_removexattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_REMOVEXATTR] = (long)fop;
+}
+
+void
+set_removexattr_cbk (long py_this, fop_removexattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_REMOVEXATTR] = (long)cbk;
+}
+
+
+/* FOP:FREMOVEXATTR */
+
+int32_t
+glupy_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FREMOVEXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fremovexattr_cbk_t)(priv->cbks[GLUPY_FREMOVEXATTR]))(
+                frame, cookie, this, op_ret, op_errno, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FREMOVEXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fremovexattr_t)(priv->fops[GLUPY_FREMOVEXATTR]))(
+                frame, this, fd, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fremovexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_fremovexattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fremovexattr_cbk, xl, xl->fops->fremovexattr,
+                    fd, name, xdata);
+}
+
+
+void
+unwind_fremovexattr (call_frame_t *frame, long cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_fremovexattr_fop (long py_this, fop_fremovexattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FREMOVEXATTR] = (long)fop;
+}
+
+void
+set_fremovexattr_cbk (long py_this, fop_fremovexattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FREMOVEXATTR] = (long)cbk;
+}
+
+
+/* FOP: LINK*/
+int32_t
+glupy_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_LINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_link_cbk_t)(priv->cbks[GLUPY_LINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_LINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_link_t)(priv->fops[GLUPY_LINK]))(
+                frame, this, oldloc, newloc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_link_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc,
+                    xdata);
+        return 0;
+}
+
+void
+wind_link (call_frame_t *frame, xlator_t *xl, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_link_cbk, xl, xl->fops->link,
+                    oldloc, newloc, xdata);
+}
+
+void
+unwind_link (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, inode_t *inode,
+             struct iatt *buf, struct iatt *preparent,
+             struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_link_fop (long py_this, fop_link_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_LINK] = (long)fop;
+}
+
+void
+set_link_cbk (long py_this, fop_link_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_LINK] = (long)cbk;
+}
+
+/* FOP: SYMLINK*/
+int32_t
+glupy_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_SYMLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_symlink_cbk_t)(priv->cbks[GLUPY_SYMLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_SYMLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_symlink_t)(priv->fops[GLUPY_SYMLINK]))(
+                frame, this, linkname, loc, umask, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_symlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->symlink, linkname, loc,
+                    umask, xdata);
+        return 0;
+}
+
+void
+wind_symlink (call_frame_t *frame, xlator_t *xl, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_symlink_cbk, xl, xl->fops->symlink,
+                    linkname, loc, umask, xdata);
+}
+
+void
+unwind_symlink (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_symlink_fop (long py_this, fop_symlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_SYMLINK] = (long)fop;
+}
+
+void
+set_symlink_cbk (long py_this, fop_symlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_SYMLINK] = (long)cbk;
+}
+
+
+/* FOP: READLINK */
+int32_t
+glupy_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, const char *path,
+                    struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readlink_cbk_t)(priv->cbks[GLUPY_READLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                path, buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path,
+                             buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                size_t size, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readlink_t)(priv->fops[GLUPY_READLINK]))(
+                frame, this, loc, size, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readlink, loc,
+                    size, xdata);
+        return 0;
+}
+
+void
+wind_readlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               size_t size, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_readlink_cbk, xl, xl->fops->readlink,
+                    loc, size, xdata);
+}
+
+void
+unwind_readlink (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, const char *path,
+                 struct iatt *buf, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf,
+                             xdata);
+}
+
+void
+set_readlink_fop (long py_this, fop_readlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READLINK] = (long)fop;
+}
+
+void
+set_readlink_cbk (long py_this, fop_readlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READLINK] = (long)cbk;
+}
+
+
+/* FOP: UNLINK */
+
+int32_t
+glupy_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_UNLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_unlink_cbk_t)(priv->cbks[GLUPY_UNLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
+                             postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              int xflags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_UNLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_unlink_t)(priv->fops[GLUPY_UNLINK]))(
+                frame, this, loc, xflags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink, loc,
+                    xflags, xdata);
+        return 0;
+}
+
+void
+wind_unlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+             int xflags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_unlink_cbk, xl, xl->fops->unlink,
+                    loc, xflags, xdata);
+}
+
+void
+unwind_unlink (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iatt *preparent, struct iatt *postparent,
+               dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+}
+
+void
+set_unlink_fop (long py_this, fop_unlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_UNLINK] = (long)fop;
+}
+
+void
+set_unlink_cbk (long py_this, fop_unlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_UNLINK] = (long)cbk;
+}
+
+
+/* FOP: MKDIR */
+
+int32_t
+glupy_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_MKDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_mkdir_cbk_t)(priv->cbks[GLUPY_MKDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_MKDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_mkdir_t)(priv->fops[GLUPY_MKDIR]))(
+                frame, this, loc, mode, umask, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_mkdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                    xdata);
+        return 0;
+}
+
+void
+wind_mkdir (call_frame_t *frame, xlator_t *xl, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
+{
+
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_mkdir_cbk, xl, xl->fops->mkdir,
+                    loc, mode, umask, xdata);
+}
+
+void
+unwind_mkdir (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, inode_t *inode,
+              struct iatt *buf, struct iatt *preparent,
+              struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_mkdir_fop (long py_this, fop_mkdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_MKDIR] = (long)fop;
+}
+
+void
+set_mkdir_cbk (long py_this, fop_mkdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_MKDIR] = (long)cbk;
+}
+
+
+/* FOP: RMDIR */
+
+int32_t
+glupy_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_RMDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_rmdir_cbk_t)(priv->cbks[GLUPY_RMDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
+                             postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             int xflags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_RMDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_rmdir_t)(priv->fops[GLUPY_RMDIR]))(
+                frame, this, loc, xflags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_rmdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rmdir, loc,
+                    xflags, xdata);
+        return 0;
+}
+
+void
+wind_rmdir (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+            int xflags, dict_t *xdata)
+{
+
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_rmdir_cbk, xl, xl->fops->rmdir,
+                    loc, xflags, xdata);
+}
+
+void
+unwind_rmdir (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno,
+              struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+}
+
+void
+set_rmdir_fop (long py_this, fop_rmdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_RMDIR] = (long)fop;
+}
+
+void
+set_rmdir_cbk (long py_this, fop_rmdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_RMDIR] = (long)cbk;
+}
+
+
+/* NON-FOP-SPECIFIC CODE */
+
+
+long
+get_id (call_frame_t *frame)
+{
+        return (long)(frame->local);
+}
+
+uint64_t
+get_rootunique (call_frame_t *frame)
+{
+        return frame->root->unique;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        glupy_private_t         *priv           = NULL;
+        char                    *module_name    = NULL;
+        PyObject                *py_mod_name    = NULL;
+        PyObject                *py_init_func   = NULL;
+        PyObject                *py_args        = NULL;
+        PyObject                *syspath        = NULL;
+	PyObject                *path           = NULL;
+	static gf_boolean_t      py_inited      = _gf_false;
+        void *                   err_cleanup    = &&err_return;
+
+        if (dict_get_str(this->options,"module-name",&module_name) != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "missing module-name");
+                return -1;
+        }
+
+	priv = GF_CALLOC (1, sizeof (glupy_private_t), gf_glupy_mt_priv);
+        if (!priv) {
+                goto *err_cleanup;
+        }
+        this->private = priv;
+        err_cleanup = &&err_free_priv;
+
+	if (!py_inited) {
+                Py_Initialize();
+                PyEval_InitThreads();
+#if 0
+                (void)pthread_key_create(&gil_init_key,NULL);
+                (void)pthread_setspecific(gil_init_key,(void *)1);
+#endif
+                /* PyEval_InitThreads takes this "for" us.  No thanks. */
+                PyEval_ReleaseLock();
+                py_inited = _gf_true;
+        }
+
+	/* Adjust python's path */
+	syspath = PySys_GetObject("path");
+	path = PyString_FromString(GLUSTER_PYTHON_PATH);
+	PyList_Append(syspath, path);
+	Py_DECREF(path);
+
+	py_mod_name = PyString_FromString(module_name);
+        if (!py_mod_name) {
+                gf_log (this->name, GF_LOG_ERROR, "could not create name");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+
+        gf_log (this->name, GF_LOG_ERROR, "py_mod_name = %s", module_name);
+        priv->py_module = PyImport_Import(py_mod_name);
+        Py_DECREF(py_mod_name);
+        if (!priv->py_module) {
+                gf_log (this->name, GF_LOG_ERROR, "Python import failed");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        err_cleanup = &&err_deref_module;
+
+        py_init_func = PyObject_GetAttrString(priv->py_module, "xlator");
+        if (!py_init_func || !PyCallable_Check(py_init_func)) {
+                gf_log (this->name, GF_LOG_ERROR, "missing init func");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        err_cleanup = &&err_deref_init;
+
+        py_args = PyTuple_New(1);
+        if (!py_args) {
+                gf_log (this->name, GF_LOG_ERROR, "could not create args");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        PyTuple_SetItem(py_args,0,PyLong_FromLong((long)this));
+
+        /* TBD: pass in list of children */
+        priv->py_xlator = PyObject_CallObject(py_init_func, py_args);
+        Py_DECREF(py_args);
+        if (!priv->py_xlator) {
+                gf_log (this->name, GF_LOG_ERROR, "Python init failed");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        gf_log (this->name, GF_LOG_INFO, "init returned %p", priv->py_xlator);
+
+        return 0;
+
+err_deref_init:
+        Py_DECREF(py_init_func);
+err_deref_module:
+        Py_DECREF(priv->py_module);
+err_free_priv:
+        GF_FREE(priv);
+err_return:
+        return -1;
+}
+
+void
+fini (xlator_t *this)
+{
+        glupy_private_t *priv = this->private;
+
+        if (!priv)
+                return;
+        Py_DECREF(priv->py_xlator);
+        Py_DECREF(priv->py_module);
+        this->private = NULL;
+        GF_FREE (priv);
+
+        return;
+}
+
+struct xlator_fops fops = {
+        .lookup       = glupy_lookup,
+        .create       = glupy_create,
+        .open         = glupy_open,
+        .readv        = glupy_readv,
+        .writev       = glupy_writev,
+        .opendir      = glupy_opendir,
+        .readdir      = glupy_readdir,
+        .stat         = glupy_stat,
+        .fstat        = glupy_fstat,
+        .setxattr     = glupy_setxattr,
+        .getxattr     = glupy_getxattr,
+        .fsetxattr    = glupy_fsetxattr,
+        .fgetxattr    = glupy_fgetxattr,
+        .removexattr  = glupy_removexattr,
+        .fremovexattr = glupy_fremovexattr,
+        .link         = glupy_link,
+        .unlink       = glupy_unlink,
+        .readlink     = glupy_readlink,
+        .symlink      = glupy_symlink,
+        .mkdir        = glupy_mkdir,
+        .rmdir        = glupy_rmdir,
+        .statfs       = glupy_statfs,
+        .readdirp     = glupy_readdirp
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/glupy/src/glupy.h b/xlators/features/glupy/src/glupy.h
new file mode 100644
index 000000000..8661fce88
--- /dev/null
+++ b/xlators/features/glupy/src/glupy.h
@@ -0,0 +1,69 @@
+/*
+   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   This file is part of GlusterFS.
+
+   GlusterFS is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published
+   by the Free Software Foundation; either version 3 of the License,
+   or (at your option) any later version.
+
+   GlusterFS is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef __GLUPY_H__
+#define __GLUPY_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include "mem-types.h"
+
+enum {
+        GLUPY_LOOKUP = 0,
+        GLUPY_CREATE,
+        GLUPY_OPEN,
+        GLUPY_READV,
+        GLUPY_WRITEV,
+        GLUPY_OPENDIR,
+        GLUPY_READDIR,
+        GLUPY_READDIRP,
+        GLUPY_STAT,
+        GLUPY_FSTAT,
+        GLUPY_STATFS,
+        GLUPY_SETXATTR,
+        GLUPY_GETXATTR,
+        GLUPY_FSETXATTR,
+        GLUPY_FGETXATTR,
+        GLUPY_REMOVEXATTR,
+        GLUPY_FREMOVEXATTR,
+        GLUPY_LINK,
+        GLUPY_UNLINK,
+        GLUPY_READLINK,
+        GLUPY_SYMLINK,
+        GLUPY_MKNOD,
+        GLUPY_MKDIR,
+        GLUPY_RMDIR,
+        GLUPY_N_FUNCS
+};
+
+typedef struct {
+        PyObject        *py_module;
+        PyObject        *py_xlator;
+        long            fops[GLUPY_N_FUNCS];
+        long            cbks[GLUPY_N_FUNCS];
+} glupy_private_t;
+
+enum gf_glupy_mem_types_ {
+        gf_glupy_mt_priv = gf_common_mt_end + 1,
+        gf_glupy_mt_end
+};
+
+#endif /* __GLUPY_H__ */
diff --git a/xlators/features/glupy/src/gluster.py b/xlators/features/glupy/src/gluster.py
new file mode 100644
index 000000000..a5daa77d3
--- /dev/null
+++ b/xlators/features/glupy/src/gluster.py
@@ -0,0 +1,841 @@
+import sys
+from ctypes import *
+
+dl = CDLL("",RTLD_GLOBAL)
+
+
+class call_frame_t (Structure):
+        pass
+
+class dev_t (Structure):
+        pass
+
+
+class dict_t (Structure):
+        pass
+
+
+class gf_dirent_t (Structure):
+        pass
+
+
+class iobref_t (Structure):
+	pass
+
+
+class iovec_t (Structure):
+        pass
+
+
+class list_head (Structure):
+        pass
+
+list_head._fields_ = [
+                ("next", POINTER(list_head)),
+                ("prev", POINTER(list_head))
+        ]
+
+
+class rwxperm_t (Structure):
+        _fields_ = [
+                ("read", c_uint8, 1),
+                ("write", c_uint8, 1),
+                ("execn", c_uint8, 1)
+        ]
+
+
+class statvfs_t (Structure):
+        pass
+
+
+class xlator_t (Structure):
+        pass
+
+
+class ia_prot_t (Structure):
+        _fields_ = [
+                ("suid", c_uint8, 1),
+                ("sgid", c_uint8, 1),
+                ("sticky", c_uint8, 1),
+                ("owner", rwxperm_t),
+                ("group", rwxperm_t),
+                ("other", rwxperm_t)
+        ]
+
+# For checking file type.
+(IA_INVAL, IA_IFREG, IA_IFDIR, IA_IFLNK, IA_IFBLK, IA_IFCHR, IA_IFIFO,
+ IA_IFSOCK) = xrange(8)
+
+
+class iatt_t (Structure):
+        _fields_ = [
+                ("ia_no", c_uint64),
+                ("ia_gfid", c_ubyte * 16),
+                ("ia_dev", c_uint64),
+                ("ia_type", c_uint),
+                ("ia_prot", ia_prot_t),
+                ("ia_nlink", c_uint32),
+                ("ia_uid", c_uint32),
+                ("ia_gid", c_uint32),
+                ("ia_rdev", c_uint64),
+                ("ia_size", c_uint64),
+                ("ia_blksize", c_uint32),
+                ("ia_blocks", c_uint64),
+                ("ia_atime", c_uint32 ),
+                ("ia_atime_nsec", c_uint32),
+                ("ia_mtime", c_uint32),
+                ("ia_mtime_nsec", c_uint32),
+                ("ia_ctime", c_uint32),
+                ("ia_ctime_nsec", c_uint32)
+        ]
+
+
+class mem_pool (Structure):
+        _fields_ = [
+                ("list", list_head),
+                ("hot_count", c_int),
+                ("cold_count", c_int),
+                ("lock", c_void_p),
+                ("padded_sizeof_type", c_ulong),
+                ("pool", c_void_p),
+                ("pool_end", c_void_p),
+                ("real_sizeof_type", c_int),
+                ("alloc_count", c_uint64),
+                ("pool_misses", c_uint64),
+                ("max_alloc", c_int),
+                ("curr_stdalloc", c_int),
+                ("max_stdalloc", c_int),
+                ("name", c_char_p),
+                ("global_list", list_head)
+        ]
+
+
+class U_ctx_key_inode (Union):
+        _fields_ = [
+                ("key", c_uint64),
+                ("xl_key", POINTER(xlator_t))
+        ]
+
+
+class U_ctx_value1 (Union):
+        _fields_ = [
+                ("value1", c_uint64),
+                ("ptr1", c_void_p)
+        ]
+
+
+class U_ctx_value2 (Union):
+        _fields_ = [
+                ("value2", c_uint64),
+                ("ptr2", c_void_p)
+        ]
+
+class inode_ctx (Structure):
+        _anonymous_ = ("u_key","u_value1","u_value2",)
+        _fields_ = [
+                ("u_key", U_ctx_key_inode),
+                ("u_value1", U_ctx_value1),
+                ("u_value2", U_ctx_value2)
+        ]
+
+class inode_t (Structure):
+        pass
+
+class inode_table_t (Structure):
+        _fields_ = [
+                ("lock", c_void_p),
+                ("hashsize", c_size_t),
+                ("name", c_char_p),
+                ("root", POINTER(inode_t)),
+                ("xl", POINTER(xlator_t)),
+                ("lru_limit", c_uint32),
+                ("inode_hash", POINTER(list_head)),
+                ("name_hash", POINTER(list_head)),
+                ("active", list_head),
+                ("active_size", c_uint32),
+                ("lru", list_head),
+                ("lru_size", c_uint32),
+                ("purge", list_head),
+                ("purge_size", c_uint32),
+                ("inode_pool", POINTER(mem_pool)),
+                ("dentry_pool", POINTER(mem_pool)),
+                ("fd_mem_pool", POINTER(mem_pool))
+        ]
+
+inode_t._fields_ = [
+                ("table", POINTER(inode_table_t)),
+                ("gfid", c_ubyte * 16),
+                ("lock", c_void_p),
+                ("nlookup", c_uint64),
+                ("fd_count", c_uint32),
+                ("ref", c_uint32),
+                ("ia_type", c_uint),
+                ("fd_list", list_head),
+                ("dentry_list", list_head),
+                ("hashv", list_head),
+                ("listv", list_head),
+                ("ctx", POINTER(inode_ctx))
+        ]
+
+
+
+class U_ctx_key_fd (Union):
+        _fields_ = [
+                ("key", c_uint64),
+                ("xl_key", c_void_p)
+        ]
+
+class fd_lk_ctx (Structure):
+        _fields_ = [
+                ("lk_list", list_head),
+                ("ref", c_int),
+                ("lock", c_void_p)
+        ]
+
+class fd_ctx (Structure):
+        _anonymous_ = ("u_key","u_value1")
+        _fields_ = [
+                ("u_key", U_ctx_key_fd),
+                ("u_value1", U_ctx_value1)
+        ]
+
+class fd_t (Structure):
+        _fields_ = [
+                ("pid", c_uint64),
+                ("flags", c_int32),
+                ("refcount", c_int32),
+                ("inode_list", list_head),
+                ("inode", POINTER(inode_t)),
+                ("lock", c_void_p),
+                ("ctx", POINTER(fd_ctx)),
+                ("xl_count", c_int),
+                ("lk_ctx", POINTER(fd_lk_ctx)),
+                ("anonymous", c_uint)
+        ]
+
+class loc_t (Structure):
+        _fields_ = [
+                ("path", c_char_p),
+                ("name", c_char_p),
+                ("inode", POINTER(inode_t)),
+                ("parent", POINTER(inode_t)),
+                ("gfid", c_ubyte * 16),
+                ("pargfid", c_ubyte * 16),
+        ]
+
+
+
+def _init_op (a_class, fop, cbk, wind, unwind):
+        # Decorators, used by translators. We could pass the signatures as
+        # parameters, but it's actually kind of nice to keep them around for
+        # inspection.
+        a_class.fop_type = apply(CFUNCTYPE,a_class.fop_sig)
+        a_class.cbk_type = apply(CFUNCTYPE,a_class.cbk_sig)
+        # Dispatch-function registration.
+        fop.restype = None
+        fop.argtypes = [ c_long, a_class.fop_type ]
+        # Callback-function registration.
+        cbk.restype = None
+        cbk.argtypes = [ c_long, a_class.cbk_type ]
+        # STACK_WIND function.
+        wind.restype = None
+        wind.argtypes = list(a_class.fop_sig[1:])
+        # STACK_UNWIND function.
+        unwind.restype = None
+        unwind.argtypes = list(a_class.cbk_sig[1:])
+
+class OpLookup:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(dict_t), POINTER(iatt_t))
+_init_op (OpLookup, dl.set_lookup_fop, dl.set_lookup_cbk,
+                    dl.wind_lookup,    dl.unwind_lookup)
+
+class OpCreate:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, c_uint, c_uint, POINTER(fd_t),
+                   POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(inode_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpCreate, dl.set_create_fop, dl.set_create_cbk,
+                    dl.wind_create,    dl.unwind_create)
+
+class OpOpen:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(fd_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
+_init_op (OpOpen, dl.set_open_fop, dl.set_open_cbk,
+                  dl.wind_open,    dl.unwind_open)
+
+class OpReadv:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, c_uint32, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iovec_t), c_int, POINTER(iatt_t),
+                   POINTER(iobref_t), POINTER(dict_t))
+_init_op (OpReadv, dl.set_readv_fop, dl.set_readv_cbk,
+                   dl.wind_readv,    dl.unwind_readv)
+class OpWritev:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(iovec_t), c_int, c_long, c_uint32,
+                   POINTER(iobref_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpWritev, dl.set_writev_fop, dl.set_writev_cbk,
+                    dl.wind_writev,    dl.unwind_writev)
+
+class OpOpendir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(fd_t) ,POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
+_init_op (OpOpendir, dl.set_opendir_fop, dl.set_opendir_cbk,
+                     dl.wind_opendir,    dl.unwind_opendir)
+
+class OpReaddir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
+_init_op (OpReaddir, dl.set_readdir_fop, dl.set_readdir_cbk,
+                     dl.wind_readdir,    dl.unwind_readdir)
+
+class OpReaddirp:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
+_init_op (OpReaddirp, dl.set_readdirp_fop, dl.set_readdirp_cbk,
+                      dl.wind_readdirp,    dl.unwind_readdirp)
+
+class OpStat:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpStat, dl.set_stat_fop, dl.set_stat_cbk,
+                  dl.wind_stat,    dl.unwind_stat)
+
+class OpFstat:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpFstat, dl.set_fstat_fop, dl.set_fstat_cbk,
+                   dl.wind_fstat,    dl.unwind_fstat)
+
+class OpStatfs:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(statvfs_t), POINTER(dict_t))
+_init_op (OpStatfs, dl.set_statfs_fop, dl.set_statfs_cbk,
+                    dl.wind_statfs,    dl.unwind_statfs)
+
+
+class OpSetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t), c_int32,
+                   POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpSetxattr, dl.set_setxattr_fop, dl.set_setxattr_cbk,
+                      dl.wind_setxattr,    dl.unwind_setxattr)
+
+class OpGetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_char_p, POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
+_init_op (OpGetxattr, dl.set_getxattr_fop, dl.set_getxattr_cbk,
+                      dl.wind_getxattr,    dl.unwind_getxattr)
+
+class OpFsetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(dict_t), c_int32,
+                   POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpFsetxattr, dl.set_fsetxattr_fop, dl.set_fsetxattr_cbk,
+                       dl.wind_fsetxattr,    dl.unwind_fsetxattr)
+
+class OpFgetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_char_p, POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
+_init_op (OpFgetxattr, dl.set_fgetxattr_fop, dl.set_fgetxattr_cbk,
+                       dl.wind_fgetxattr,    dl.unwind_fgetxattr)
+
+class OpRemovexattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_char_p, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpRemovexattr, dl.set_removexattr_fop, dl.set_removexattr_cbk,
+                         dl.wind_removexattr,    dl.unwind_removexattr)
+
+
+class OpFremovexattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_char_p, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpFremovexattr, dl.set_fremovexattr_fop, dl.set_fremovexattr_cbk,
+                          dl.wind_fremovexattr,    dl.unwind_fremovexattr)
+
+class OpLink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpLink, dl.set_link_fop, dl.set_link_cbk,
+                  dl.wind_link,    dl.unwind_link)
+
+class OpSymlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   c_char_p, POINTER(loc_t), c_uint, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpSymlink, dl.set_symlink_fop, dl.set_symlink_cbk,
+                     dl.wind_symlink,    dl.unwind_symlink)
+
+class OpUnlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpUnlink, dl.set_unlink_fop, dl.set_unlink_cbk,
+                    dl.wind_unlink,    dl.unwind_unlink)
+
+class OpReadlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_size_t, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, c_char_p, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpReadlink, dl.set_readlink_fop, dl.set_readlink_cbk,
+                      dl.wind_readlink,    dl.unwind_readlink)
+
+class OpMkdir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_uint, c_uint, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpMkdir, dl.set_mkdir_fop, dl.set_mkdir_cbk,
+                   dl.wind_mkdir,    dl.unwind_mkdir)
+
+class OpRmdir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpRmdir, dl.set_rmdir_fop, dl.set_rmdir_cbk,
+                   dl.wind_rmdir,    dl.unwind_rmdir)
+
+
+class Translator:
+        def __init__ (self, c_this):
+                # This is only here to keep references to the stubs we create,
+                # because ctypes doesn't and glupy.so can't because it doesn't
+                # get a pointer to the actual Python object. It's a dictionary
+                # instead of a list in case we ever allow changing fops/cbks
+                # after initialization and need to look them up.
+                self.stub_refs = {}
+                funcs = dir(self.__class__)
+                if "lookup_fop" in funcs:
+                        @OpLookup.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.lookup_fop (frame, this, loc, xdata)
+                        self.stub_refs["lookup_fop"] = stub
+                        dl.set_lookup_fop(c_this,stub)
+                if "lookup_cbk" in funcs:
+                        @OpLookup.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, inode,
+                                  buf, xdata, postparent, s=self):
+                                return s.lookup_cbk(frame, cookie, this, op_ret,
+                                                    op_errno, inode, buf, xdata,
+                                                    postparent)
+                        self.stub_refs["lookup_cbk"] = stub
+                        dl.set_lookup_cbk(c_this,stub)
+                if "create_fop" in funcs:
+                        @OpCreate.fop_type
+                        def stub (frame, this, loc, flags, mode, umask, fd,
+                                  xdata, s=self):
+                                return s.create_fop (frame, this, loc, flags,
+                                                     mode, umask, fd, xdata)
+                        self.stub_refs["create_fop"] = stub
+                        dl.set_create_fop(c_this,stub)
+                if "create_cbk" in funcs:
+                        @OpCreate.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.create_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, fd,
+                                                     inode, buf, preparent,
+                                                     postparent, xdata)
+                        self.stub_refs["create_cbk"] = stub
+                        dl.set_create_cbk(c_this,stub)
+                if "open_fop" in funcs:
+                        @OpOpen.fop_type
+                        def stub (frame, this, loc, flags, fd,
+                                  xdata, s=self):
+                                return s.open_fop (frame, this, loc, flags,
+                                                   fd, xdata)
+                        self.stub_refs["open_fop"] = stub
+                        dl.set_open_fop(c_this,stub)
+                if "open_cbk" in funcs:
+                        @OpOpen.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  xdata, s=self):
+                                return s.open_cbk (frame, cookie, this,
+                                                   op_ret, op_errno, fd,
+                                                   xdata)
+                        self.stub_refs["open_cbk"] = stub
+                        dl.set_open_cbk(c_this,stub)
+                if "readv_fop" in funcs:
+                        @OpReadv.fop_type
+                        def stub (frame, this, fd, size, offset, flags,
+                                  xdata, s=self):
+                                return s.readv_fop (frame, this, fd, size,
+                                                    offset, flags, xdata)
+                        self.stub_refs["readv_fop"] = stub
+                        dl.set_readv_fop(c_this,stub)
+                if "readv_cbk" in funcs:
+                        @OpReadv.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  vector, count, stbuf, iobref, xdata,
+                                  s=self):
+                                return s.readv_cbk (frame, cookie, this,
+                                                    op_ret, op_errno, vector,
+                                                    count, stbuf, iobref,
+                                                    xdata)
+                        self.stub_refs["readv_cbk"] = stub
+                        dl.set_readv_cbk(c_this,stub)
+                if "writev_fop" in funcs:
+                        @OpWritev.fop_type
+                        def stub (frame, this, fd, vector, count,
+                                  offset, flags, iobref, xdata, s=self):
+                                return s.writev_fop (frame, this, fd, vector,
+                                                     count, offset, flags,
+                                                     iobref, xdata)
+                        self.stub_refs["writev_fop"] = stub
+                        dl.set_writev_fop(c_this,stub)
+                if "writev_cbk" in funcs:
+                        @OpWritev.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  prebuf, postbuf, xdata, s=self):
+                                return s.writev_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, prebuf,
+                                                     postbuf, xdata)
+                        self.stub_refs["writev_cbk"] = stub
+                        dl.set_writev_cbk(c_this,stub)
+                if "opendir_fop" in funcs:
+                        @OpOpendir.fop_type
+                        def stub (frame, this, loc, fd, xdata, s=self):
+                                return s.opendir_fop (frame, this, loc, fd,
+                                                      xdata)
+                        self.stub_refs["opendir_fop"] = stub
+                        dl.set_opendir_fop(c_this,stub)
+                if "opendir_cbk" in funcs:
+                        @OpOpendir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  xdata, s=self):
+                                return s.opendir_cbk(frame, cookie, this,
+                                                     op_ret, op_errno, fd,
+                                                     xdata)
+                        self.stub_refs["opendir_cbk"] = stub
+                        dl.set_opendir_cbk(c_this,stub)
+                if "readdir_fop" in funcs:
+                        @OpReaddir.fop_type
+                        def stub (frame, this, fd, size, offset, xdata, s=self):
+                                return s.readdir_fop (frame, this, fd, size,
+                                                      offset, xdata)
+                        self.stub_refs["readdir_fop"] = stub
+                        dl.set_readdir_fop(c_this,stub)
+                if "readdir_cbk" in funcs:
+                        @OpReaddir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  entries, xdata, s=self):
+                                return s.readdir_cbk(frame, cookie, this,
+                                                     op_ret, op_errno, entries,
+                                                     xdata)
+                        self.stub_refs["readdir_cbk"] = stub
+                        dl.set_readdir_cbk(c_this,stub)
+                if "readdirp_fop" in funcs:
+                        @OpReaddirp.fop_type
+                        def stub (frame, this, fd, size, offset, xdata, s=self):
+                                return s.readdirp_fop (frame, this, fd, size,
+                                                       offset, xdata)
+                        self.stub_refs["readdirp_fop"] = stub
+                        dl.set_readdirp_fop(c_this,stub)
+                if "readdirp_cbk" in funcs:
+                        @OpReaddirp.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  entries, xdata, s=self):
+                                return s.readdirp_cbk (frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       entries, xdata)
+                        self.stub_refs["readdirp_cbk"] = stub
+                        dl.set_readdirp_cbk(c_this,stub)
+                if "stat_fop" in funcs:
+                        @OpStat.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.stat_fop (frame, this, loc, xdata)
+                        self.stub_refs["stat_fop"] = stub
+                        dl.set_stat_fop(c_this,stub)
+                if "stat_cbk" in funcs:
+                        @OpStat.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.stat_cbk(frame, cookie, this, op_ret,
+                                                  op_errno, buf, xdata)
+                        self.stub_refs["stat_cbk"] = stub
+                        dl.set_stat_cbk(c_this,stub)
+                if "fstat_fop" in funcs:
+                        @OpFstat.fop_type
+                        def stub (frame, this, fd, xdata, s=self):
+                                return s.fstat_fop (frame, this, fd, xdata)
+                        self.stub_refs["fstat_fop"] = stub
+                        dl.set_fstat_fop(c_this,stub)
+                if "fstat_cbk" in funcs:
+                        @OpFstat.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.fstat_cbk(frame, cookie, this, op_ret,
+                                                   op_errno, buf, xdata)
+                        self.stub_refs["fstat_cbk"] = stub
+                        dl.set_fstat_cbk(c_this,stub)
+                if "statfs_fop" in funcs:
+                        @OpStatfs.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.statfs_fop (frame, this, loc, xdata)
+                        self.stub_refs["statfs_fop"] = stub
+                        dl.set_statfs_fop(c_this,stub)
+                if "statfs_cbk" in funcs:
+                        @OpStatfs.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.statfs_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, buf,
+                                                     xdata)
+                        self.stub_refs["statfs_cbk"] = stub
+                        dl.set_statfs_cbk(c_this,stub)
+                if "setxattr_fop" in funcs:
+                        @OpSetxattr.fop_type
+                        def stub (frame, this, loc, dictionary, flags, xdata,
+                                  s=self):
+                                return s.setxattr_fop (frame, this, loc,
+                                                       dictionary, flags,
+                                                       xdata)
+                        self.stub_refs["setxattr_fop"] = stub
+                        dl.set_setxattr_fop(c_this,stub)
+                if "setxattr_cbk" in funcs:
+                        @OpSetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
+                                  s=self):
+                                return s.setxattr_cbk(frame, cookie, this,
+                                                      op_ret, op_errno, xdata)
+                        self.stub_refs["setxattr_cbk"] = stub
+                        dl.set_setxattr_cbk(c_this,stub)
+                if "getxattr_fop" in funcs:
+                        @OpGetxattr.fop_type
+                        def stub (frame, this, loc, name, xdata, s=self):
+                                return s.getxattr_fop (frame, this, loc, name,
+                                                       xdata)
+                        self.stub_refs["getxattr_fop"] = stub
+                        dl.set_getxattr_fop(c_this,stub)
+                if "getxattr_cbk" in funcs:
+                        @OpGetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  dictionary, xdata, s=self):
+                                return s.getxattr_cbk(frame, cookie, this,
+                                                      op_ret, op_errno,
+                                                      dictionary, xdata)
+                        self.stub_refs["getxattr_cbk"] = stub
+                        dl.set_getxattr_cbk(c_this,stub)
+                if "fsetxattr_fop" in funcs:
+                        @OpFsetxattr.fop_type
+                        def stub (frame, this, fd, dictionary, flags, xdata,
+                                  s=self):
+                                return s.fsetxattr_fop (frame, this, fd,
+                                                        dictionary, flags,
+                                                        xdata)
+                        self.stub_refs["fsetxattr_fop"] = stub
+                        dl.set_fsetxattr_fop(c_this,stub)
+                if "fsetxattr_cbk" in funcs:
+                        @OpFsetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
+                                  s=self):
+                                return s.fsetxattr_cbk(frame, cookie, this,
+                                                       op_ret, op_errno, xdata)
+                        self.stub_refs["fsetxattr_cbk"] = stub
+                        dl.set_fsetxattr_cbk(c_this,stub)
+                if "fgetxattr_fop" in funcs:
+                        @OpFgetxattr.fop_type
+                        def stub (frame, this, fd, name, xdata, s=self):
+                                return s.fgetxattr_fop (frame, this, fd, name,
+                                                        xdata)
+                        self.stub_refs["fgetxattr_fop"] = stub
+                        dl.set_fgetxattr_fop(c_this,stub)
+                if "fgetxattr_cbk" in funcs:
+                        @OpFgetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  dictionary, xdata, s=self):
+                                return s.fgetxattr_cbk(frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       dictionary, xdata)
+                        self.stub_refs["fgetxattr_cbk"] = stub
+                        dl.set_fgetxattr_cbk(c_this,stub)
+                if "removexattr_fop" in funcs:
+                        @OpRemovexattr.fop_type
+                        def stub (frame, this, loc, name, xdata, s=self):
+                                return s.removexattr_fop (frame, this, loc,
+                                                          name, xdata)
+                        self.stub_refs["removexattr_fop"] = stub
+                        dl.set_removexattr_fop(c_this,stub)
+                if "removexattr_cbk" in funcs:
+                        @OpRemovexattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  xdata, s=self):
+                                return s.removexattr_cbk(frame, cookie, this,
+                                                         op_ret, op_errno,
+                                                         xdata)
+                        self.stub_refs["removexattr_cbk"] = stub
+                        dl.set_removexattr_cbk(c_this,stub)
+                if "fremovexattr_fop" in funcs:
+                        @OpFremovexattr.fop_type
+                        def stub (frame, this, fd, name, xdata, s=self):
+                                return s.fremovexattr_fop (frame, this, fd,
+                                                           name, xdata)
+                        self.stub_refs["fremovexattr_fop"] = stub
+                        dl.set_fremovexattr_fop(c_this,stub)
+                if "fremovexattr_cbk" in funcs:
+                        @OpFremovexattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  xdata, s=self):
+                                return s.fremovexattr_cbk(frame, cookie, this,
+                                                          op_ret, op_errno,
+                                                          xdata)
+                        self.stub_refs["fremovexattr_cbk"] = stub
+                        dl.set_fremovexattr_cbk(c_this,stub)
+                if "link_fop" in funcs:
+                        @OpLink.fop_type
+                        def stub (frame, this, oldloc, newloc,
+                                  xdata, s=self):
+                                return s.link_fop (frame, this, oldloc,
+                                                   newloc, xdata)
+                        self.stub_refs["link_fop"] = stub
+                        dl.set_link_fop(c_this,stub)
+                if "link_cbk" in funcs:
+                        @OpLink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.link_cbk (frame, cookie, this,
+                                                   op_ret, op_errno, inode,
+                                                   buf, preparent,
+                                                   postparent, xdata)
+                        self.stub_refs["link_cbk"] = stub
+                        dl.set_link_cbk(c_this,stub)
+                if "symlink_fop" in funcs:
+                        @OpSymlink.fop_type
+                        def stub (frame, this, linkname, loc,
+                                  umask, xdata, s=self):
+                                return s.symlink_fop (frame, this, linkname,
+                                                      loc, umask, xdata)
+                        self.stub_refs["symlink_fop"] = stub
+                        dl.set_symlink_fop(c_this,stub)
+                if "symlink_cbk" in funcs:
+                        @OpSymlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.symlink_cbk (frame, cookie, this,
+                                                      op_ret, op_errno, inode,
+                                                      buf, preparent,
+                                                      postparent, xdata)
+                        self.stub_refs["symlink_cbk"] = stub
+                        dl.set_symlink_cbk(c_this,stub)
+                if "unlink_fop" in funcs:
+                        @OpUnlink.fop_type
+                        def stub (frame, this, loc, xflags,
+                                  xdata, s=self):
+                                return s.unlink_fop (frame, this, loc,
+                                                     xflags, xdata)
+                        self.stub_refs["unlink_fop"] = stub
+                        dl.set_unlink_fop(c_this,stub)
+                if "unlink_cbk" in funcs:
+                        @OpUnlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  preparent, postparent, xdata, s=self):
+                                return s.unlink_cbk (frame, cookie, this,
+                                                     op_ret, op_errno,
+                                                     preparent, postparent,
+                                                     xdata)
+                        self.stub_refs["unlink_cbk"] = stub
+                        dl.set_unlink_cbk(c_this,stub)
+                if "readlink_fop" in funcs:
+                        @OpReadlink.fop_type
+                        def stub (frame, this, loc, size,
+                                  xdata, s=self):
+                                return s.readlink_fop (frame, this, loc,
+                                                       size, xdata)
+                        self.stub_refs["readlink_fop"] = stub
+                        dl.set_readlink_fop(c_this,stub)
+                if "readlink_cbk" in funcs:
+                        @OpReadlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  path, buf, xdata, s=self):
+                                return s.readlink_cbk (frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       path, buf, xdata)
+                        self.stub_refs["readlink_cbk"] = stub
+                        dl.set_readlink_cbk(c_this,stub)
+                if "mkdir_fop" in funcs:
+                        @OpMkdir.fop_type
+                        def stub (frame, this, loc, mode, umask, xdata,
+                                  s=self):
+                                return s.mkdir_fop (frame, this, loc, mode,
+                                                    umask, xdata)
+                        self.stub_refs["mkdir_fop"] = stub
+                        dl.set_mkdir_fop(c_this,stub)
+                if "mkdir_cbk" in funcs:
+                        @OpMkdir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, inode,
+                                  buf, preparent, postparent, xdata, s=self):
+                                return s.mkdir_cbk (frame, cookie, this,
+                                                    op_ret, op_errno, inode,
+                                                    buf, preparent,
+                                                    postparent, xdata)
+                        self.stub_refs["mkdir_cbk"] = stub
+                        dl.set_mkdir_cbk(c_this,stub)
+                if "rmdir_fop" in funcs:
+                        @OpRmdir.fop_type
+                        def stub (frame, this, loc, xflags,
+                                  xdata, s=self):
+                                return s.rmdir_fop (frame, this, loc,
+                                                    xflags, xdata)
+                        self.stub_refs["rmdir_fop"] = stub
+                        dl.set_rmdir_fop(c_this,stub)
+                if "rmdir_cbk" in funcs:
+                        @OpRmdir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  preparent, postparent, xdata, s=self):
+                                return s.rmdir_cbk (frame, cookie, this,
+                                                    op_ret, op_errno,
+                                                    preparent, postparent,
+                                                    xdata)
+                        self.stub_refs["rmdir_cbk"] = stub
+                        dl.set_rmdir_cbk(c_this,stub)
diff --git a/xlators/features/glupy/src/helloworld.py b/xlators/features/glupy/src/helloworld.py
new file mode 100644
index 000000000..8fe403711
--- /dev/null
+++ b/xlators/features/glupy/src/helloworld.py
@@ -0,0 +1,19 @@
+import sys
+from gluster import *
+
+class xlator (Translator):
+
+    def __init__(self, c_this):
+        Translator.__init__(self, c_this)
+
+    def lookup_fop(self, frame, this, loc, xdata):
+        print "Python xlator: Hello!"
+        dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
+        return 0
+
+    def lookup_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                   xdata, postparent):
+        print "Python xlator: Hello again!"
+        dl.unwind_lookup(frame, cookie, this, op_ret, op_errno, inode, buf,
+                         xdata, postparent)
+        return 0
diff --git a/xlators/features/glupy/src/negative.py b/xlators/features/glupy/src/negative.py
new file mode 100644
index 000000000..1023602b9
--- /dev/null
+++ b/xlators/features/glupy/src/negative.py
@@ -0,0 +1,92 @@
+import sys
+from uuid import UUID
+from gluster import *
+
+# Negative-lookup-caching example.  If a file wasn't there the last time we
+# looked, it's probably still not there.  This translator keeps track of
+# those failed lookups for us, and returns ENOENT without needing to pass the
+# call any further for repeated requests.
+
+# If we were doing this for real, we'd need separate caches for each xlator
+# instance.  The easiest way to do this would be to have xlator.__init__
+# "register" each instance in a module-global dict, with the key as the C
+# translator address and the value as the xlator object itself.  For testing
+# and teaching, it's sufficient just to have one cache.  The keys are parent
+# GFIDs, and the entries are lists of names within that parent that we know
+# don't exist.
+cache = {}
+
+# TBD: we need a better way of handling per-request data (frame->local in C).
+dl.get_id.restype = c_long
+dl.get_id.argtypes = [ POINTER(call_frame_t) ]
+
+def uuid2str (gfid):
+    return str(UUID(''.join(map("{0:02x}".format, gfid))))
+
+class xlator (Translator):
+
+    def __init__ (self, c_this):
+        self.requests = {}
+        Translator.__init__(self,c_this)
+
+    def lookup_fop (self, frame, this, loc, xdata):
+        pargfid = uuid2str(loc.contents.pargfid)
+        print "lookup FOP: %s:%s" % (pargfid, loc.contents.name)
+        # Check the cache.
+        if cache.has_key(pargfid):
+            if loc.contents.name in cache[pargfid]:
+                print "short-circuiting for %s:%s" % (pargfid,
+                    loc.contents.name)
+                dl.unwind_lookup(frame,0,this,-1,2,None,None,None,None)
+                return 0
+        key = dl.get_id(frame)
+        self.requests[key] = (pargfid, loc.contents.name[:])
+        # TBD: get real child xl from init, pass it here
+        dl.wind_lookup(frame,POINTER(xlator_t)(),loc,xdata)
+        return 0
+
+    def lookup_cbk (self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                    xdata, postparent):
+        print "lookup CBK: %d (%d)" % (op_ret, op_errno)
+        key = dl.get_id(frame)
+        pargfid, name = self.requests[key]
+        # Update the cache.
+        if op_ret == 0:
+            print "found %s, removing from cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].discard(name)
+        elif op_errno == 2:    # ENOENT
+            print "failed to find %s, adding to cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].add(name)
+            else:
+                cache[pargfid] = set([name])
+        del self.requests[key]
+        dl.unwind_lookup(frame,cookie,this,op_ret,op_errno,
+                         inode,buf,xdata,postparent)
+        return 0
+
+    def create_fop (self, frame, this, loc, flags, mode, umask, fd, xdata):
+        pargfid = uuid2str(loc.contents.pargfid)
+        print "create FOP: %s:%s" % (pargfid, loc.contents.name)
+        key = dl.get_id(frame)
+        self.requests[key] = (pargfid, loc.contents.name[:])
+        # TBD: get real child xl from init, pass it here
+        dl.wind_create(frame,POINTER(xlator_t)(),loc,flags,mode,umask,fd,xdata)
+        return 0
+
+    def create_cbk (self, frame, cookie, this, op_ret, op_errno, fd, inode,
+                    buf, preparent, postparent, xdata):
+        print "create CBK: %d (%d)" % (op_ret, op_errno)
+        key = dl.get_id(frame)
+        pargfid, name = self.requests[key]
+        # Update the cache.
+        if op_ret == 0:
+            print "created %s, removing from cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].discard(name)
+        del self.requests[key]
+        dl.unwind_create(frame,cookie,this,op_ret,op_errno,fd,inode,buf,
+                         preparent,postparent,xdata)
+        return 0
+
diff --git a/scheduler/alu/Makefile.am b/xlators/features/index/Makefile.am
index d471a3f92..a985f42a8 100644
--- a/scheduler/alu/Makefile.am
+++ b/xlators/features/index/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/index/src/Makefile.am b/xlators/features/index/src/Makefile.am
new file mode 100644
index 000000000..73bb8972e
--- /dev/null
+++ b/xlators/features/index/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = index.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+index_la_LDFLAGS = -module -avoid-version
+
+index_la_SOURCES = index.c
+index_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = index.h index-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/index/src/index-mem-types.h b/xlators/features/index/src/index-mem-types.h
new file mode 100644
index 000000000..553d492df
--- /dev/null
+++ b/xlators/features/index/src/index-mem-types.h
@@ -0,0 +1,22 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QUIESCE_MEM_TYPES_H__
+#define __QUIESCE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_index_mem_types_ {
+        gf_index_mt_priv_t = gf_common_mt_end + 1,
+        gf_index_inode_ctx_t = gf_common_mt_end + 2,
+        gf_index_fd_ctx_t = gf_common_mt_end + 3,
+        gf_index_mt_end
+};
+#endif
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
new file mode 100644
index 000000000..9253120f3
--- /dev/null
+++ b/xlators/features/index/src/index.c
@@ -0,0 +1,1489 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "index.h"
+#include "options.h"
+#include "glusterfs3-xdr.h"
+#include "syncop.h"
+
+#define XATTROP_SUBDIR "xattrop"
+#define BASE_INDICES_HOLDER_SUBDIR "base_indices_holder"
+
+call_stub_t *
+__index_dequeue (struct list_head *callstubs)
+{
+        call_stub_t *stub = NULL;
+
+        if (!list_empty (callstubs)) {
+                stub = list_entry (callstubs->next, call_stub_t, list);
+                list_del_init (&stub->list);
+        }
+
+        return stub;
+}
+
+inline static void
+__index_enqueue (struct list_head *callstubs, call_stub_t *stub)
+{
+        list_add_tail (&stub->list, callstubs);
+}
+
+static void
+worker_enqueue (xlator_t *this, call_stub_t *stub)
+{
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        pthread_mutex_lock (&priv->mutex);
+        {
+                __index_enqueue (&priv->callstubs, stub);
+                pthread_cond_signal (&priv->cond);
+        }
+        pthread_mutex_unlock (&priv->mutex);
+}
+
+void *
+index_worker (void *data)
+{
+        index_priv_t     *priv = NULL;
+        xlator_t         *this = NULL;
+        call_stub_t      *stub = NULL;
+        int               ret = 0;
+
+        THIS = data;
+        this = data;
+        priv = this->private;
+
+        for (;;) {
+                pthread_mutex_lock (&priv->mutex);
+                {
+                        while (list_empty (&priv->callstubs)) {
+                                ret = pthread_cond_wait (&priv->cond,
+                                                         &priv->mutex);
+                        }
+
+                        stub = __index_dequeue (&priv->callstubs);
+                }
+                pthread_mutex_unlock (&priv->mutex);
+
+                if (stub) /* guard against spurious wakeups */
+                        call_resume (stub);
+        }
+
+        return NULL;
+}
+int
+__index_inode_ctx_get (inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
+{
+        int               ret = 0;
+        index_inode_ctx_t *ictx = NULL;
+        uint64_t          tmpctx = 0;
+
+        ret = __inode_ctx_get (inode, this, &tmpctx);
+        if (!ret) {
+                ictx = (index_inode_ctx_t*) (long) tmpctx;
+                goto out;
+        }
+        ictx = GF_CALLOC (1, sizeof (*ictx), gf_index_inode_ctx_t);
+        if (!ictx) {
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&ictx->callstubs);
+        ret = __inode_ctx_put (inode, this, (uint64_t)ictx);
+        if (ret) {
+                GF_FREE (ictx);
+                ictx = NULL;
+                goto out;
+        }
+out:
+        if (ictx)
+                *ctx = ictx;
+        return ret;
+}
+
+int
+index_inode_ctx_get (inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
+{
+        int               ret = 0;
+
+        LOCK (&inode->lock);
+        {
+                ret = __index_inode_ctx_get (inode, this, ctx);
+        }
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+
+static void
+make_index_dir_path (char *base, const char *subdir,
+                     char *index_dir, size_t len)
+{
+        snprintf (index_dir, len, "%s/%s", base, subdir);
+}
+
+int
+index_dir_create (xlator_t *this, const char *subdir)
+{
+        int          ret = 0;
+        struct       stat st = {0};
+        char         fullpath[PATH_MAX] = {0};
+        char         path[PATH_MAX] = {0};
+        char         *dir = NULL;
+        index_priv_t *priv = NULL;
+        size_t       len = 0;
+        size_t       pathlen = 0;
+
+        priv = this->private;
+        make_index_dir_path (priv->index_basepath, subdir, fullpath,
+                             sizeof (fullpath));
+        ret = stat (fullpath, &st);
+        if (!ret) {
+                if (!S_ISDIR (st.st_mode))
+                        ret = -2;
+                goto out;
+        }
+
+        pathlen = strlen (fullpath);
+        if ((pathlen > 1) && fullpath[pathlen - 1] == '/')
+                fullpath[pathlen - 1] = '\0';
+        dir = strchr (fullpath, '/');
+        while (dir) {
+                dir = strchr (dir + 1, '/');
+                if (dir)
+                        len = pathlen - strlen (dir);
+                else
+                        len = pathlen;
+                strncpy (path, fullpath, len);
+                path[len] = '\0';
+                ret = mkdir (path, 0600);
+                if (ret && (errno != EEXIST))
+                        goto out;
+        }
+        ret = 0;
+out:
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "%s/%s: Failed to "
+                        "create (%s)", priv->index_basepath, subdir,
+                        strerror (errno));
+        } else if (ret == -2) {
+                gf_log (this->name, GF_LOG_ERROR, "%s/%s: Failed to create, "
+                        "path exists, not a directory ", priv->index_basepath,
+                        subdir);
+        }
+        return ret;
+}
+
+void
+index_get_index (index_priv_t *priv, uuid_t index)
+{
+        LOCK (&priv->lock);
+        {
+                uuid_copy (index, priv->index);
+        }
+        UNLOCK (&priv->lock);
+}
+
+void
+index_generate_index (index_priv_t *priv, uuid_t index)
+{
+        LOCK (&priv->lock);
+        {
+                //To prevent duplicate generates.
+                //This method fails if number of contending threads is greater
+                //than MAX_LINK count of the fs
+                if (!uuid_compare (priv->index, index))
+                        uuid_generate (priv->index);
+                uuid_copy (index, priv->index);
+        }
+        UNLOCK (&priv->lock);
+}
+
+static void
+make_index_path (char *base, const char *subdir, uuid_t index,
+                 char *index_path, size_t len)
+{
+        make_index_dir_path (base, subdir, index_path, len);
+        snprintf (index_path + strlen (index_path), len - strlen (index_path),
+                  "/%s-%s", subdir, uuid_utoa (index));
+}
+
+static void
+make_gfid_path (char *base, const char *subdir, uuid_t gfid,
+                char *gfid_path, size_t len)
+{
+        make_index_dir_path (base, subdir, gfid_path, len);
+        snprintf (gfid_path + strlen (gfid_path), len - strlen (gfid_path),
+                  "/%s", uuid_utoa (gfid));
+}
+
+static void
+make_file_path (char *base, const char *subdir, const char *filename,
+                char *file_path, size_t len)
+{
+        make_index_dir_path (base, subdir, file_path, len);
+        snprintf (file_path + strlen (file_path), len - strlen (file_path),
+                  "/%s", filename);
+}
+
+static void
+check_delete_stale_index_file (xlator_t *this, char *filename)
+{
+        int             ret = 0;
+        struct stat     st = {0};
+        struct stat     base_index_st = {0};
+        char            filepath[PATH_MAX] = {0};
+        char            filepath_under_base_indices_holder[PATH_MAX] = {0};
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        if (priv->to_be_healed_states != synced_state)
+                return;
+
+        make_file_path (priv->index_basepath, XATTROP_SUBDIR,
+                        filename, filepath, sizeof (filepath));
+
+        make_file_path (priv->index_basepath, BASE_INDICES_HOLDER_SUBDIR,
+                        filename, filepath_under_base_indices_holder,
+                        sizeof (filepath_under_base_indices_holder));
+
+
+        ret = stat (filepath_under_base_indices_holder, &base_index_st);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Base index is not created"
+                        "under index/base_indices_holder");
+                return;
+        }
+
+        ret = stat (filepath, &st);
+        if (!ret && st.st_nlink == 2) {
+                unlink (filepath);
+                unlink (filepath_under_base_indices_holder);
+        }
+}
+
+static int
+index_fill_readdir (fd_t *fd, DIR *dir, off_t off,
+                    size_t size, gf_dirent_t *entries, readdir_directory type)
+{
+        off_t     in_case = -1;
+        size_t    filled = 0;
+        int       count = 0;
+        char      entrybuf[sizeof(struct dirent) + 256 + 8];
+        struct dirent  *entry          = NULL;
+        int32_t              this_size      = -1;
+        gf_dirent_t          *this_entry     = NULL;
+        xlator_t             *this = NULL;
+
+        this = THIS;
+        if (!off) {
+                rewinddir (dir);
+        } else {
+                seekdir (dir, off);
+        }
+
+        while (filled <= size) {
+                in_case = telldir (dir);
+
+                if (in_case == -1) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "telldir failed on dir=%p: %s",
+                                dir, strerror (errno));
+                        goto out;
+                }
+
+                errno = 0;
+                entry = NULL;
+                readdir_r (dir, (struct dirent *)entrybuf, &entry);
+
+                if (!entry) {
+                        if (errno == EBADF) {
+                                gf_log (THIS->name, GF_LOG_WARNING,
+                                        "readdir failed on dir=%p: %s",
+                                        dir, strerror (errno));
+                                goto out;
+                        }
+                        break;
+                }
+
+                if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
+                              strlen (XATTROP_SUBDIR"-")) &&
+                              (type == INDEX_XATTROP)) {
+                        check_delete_stale_index_file (this, entry->d_name);
+                        continue;
+                }
+
+                this_size = max (sizeof (gf_dirent_t),
+                                 sizeof (gfs3_dirplist))
+                        + strlen (entry->d_name) + 1;
+
+                if (this_size + filled > size) {
+                        seekdir (dir, in_case);
+                        break;
+                }
+
+                this_entry = gf_dirent_for_name (entry->d_name);
+
+                if (!this_entry) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "could not create gf_dirent for entry %s: (%s)",
+                                entry->d_name, strerror (errno));
+                        goto out;
+                }
+                this_entry->d_off = telldir (dir);
+                this_entry->d_ino = entry->d_ino;
+
+                list_add_tail (&this_entry->list, &entries->list);
+
+                filled += this_size;
+                count ++;
+        }
+
+        if ((!readdir (dir) && (errno == 0)))
+                /* Indicate EOF */
+                errno = ENOENT;
+out:
+        return count;
+}
+
+int
+sync_base_indices (void *index_priv)
+{
+        index_priv_t    *priv = NULL;
+        DIR             *dir_base_holder  = NULL;
+        DIR             *xattrop_dir = NULL;
+        struct dirent   *entry = NULL;
+        char            base_indices_holder[PATH_MAX] = {0};
+        char            xattrop_directory[PATH_MAX] = {0};
+        char            base_index_path[PATH_MAX] = {0};
+        char            xattrop_index_path[PATH_MAX] = {0};
+        int             ret = 0;
+
+        priv = index_priv;
+
+        snprintf (base_indices_holder, PATH_MAX, "%s/%s", priv->index_basepath,
+                  BASE_INDICES_HOLDER_SUBDIR);
+        snprintf (xattrop_directory, PATH_MAX, "%s/%s", priv->index_basepath,
+                  XATTROP_SUBDIR);
+
+        if ((dir_base_holder = opendir(base_indices_holder)) == NULL) {
+                ret = -1;
+                goto out;
+        }
+        if ((xattrop_dir = opendir (xattrop_directory)) == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        priv->to_be_healed_states = sync_started;
+        while ((entry = readdir(xattrop_dir)) != NULL) {
+                if (!strcmp (entry->d_name, ".") ||
+                    !strcmp (entry->d_name, "..")) {
+                        continue;
+                }
+                if (strncmp (entry->d_name, XATTROP_SUBDIR"-",
+                    strlen (XATTROP_SUBDIR"-"))) {
+                        continue;
+                }
+                if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
+                    strlen (XATTROP_SUBDIR"-"))) {
+
+                    snprintf (xattrop_index_path, PATH_MAX, "%s/%s",
+                              xattrop_directory, entry->d_name);
+
+                    snprintf (base_index_path, PATH_MAX, "%s/%s",
+                              base_indices_holder, entry->d_name);
+
+                    ret = link (xattrop_index_path, base_index_path);
+                    if (ret && errno != EEXIST)
+                        goto out;
+
+                }
+        }
+        ret = closedir (xattrop_dir);
+        if (ret)
+                goto out;
+        ret = closedir (dir_base_holder);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+int
+base_indices_syncing_done (int ret, call_frame_t *frame, void *data)
+{
+        index_priv_t         *priv = NULL;
+        priv = data;
+
+        if (!priv)
+                goto out;
+
+        if (ret) {
+                priv->to_be_healed_states = sync_not_started;
+        } else {
+                priv->to_be_healed_states = synced_state;
+        }
+
+        STACK_DESTROY (frame->root);
+
+out:
+        return 0;
+}
+
+int
+sync_base_indices_from_xattrop (xlator_t *this)
+{
+
+        index_priv_t          *priv = NULL;
+        char                  base_indices_holder[PATH_MAX] =  {0};
+        int                   ret = 0;
+        struct stat           st = {0};
+        DIR                   *dir = NULL;
+        struct dirent         *entry = NULL;
+        call_frame_t          *frame = NULL;
+
+        priv = this->private;
+
+        if (priv->to_be_healed_states != sync_not_started) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (base_indices_holder, PATH_MAX, "%s/%s", priv->index_basepath,
+                  BASE_INDICES_HOLDER_SUBDIR);
+
+        ret = stat (base_indices_holder, &st);
+
+        if (ret && (errno != ENOENT)) {
+                goto out;
+        } else if (errno == ENOENT) {
+                ret = index_dir_create (this, BASE_INDICES_HOLDER_SUBDIR);
+                if (ret)
+                        goto out;
+        } else {
+                if ((dir = opendir (base_indices_holder)) == NULL) {
+                        ret = -1;
+                        goto out;
+                }
+                while ((entry = readdir (dir)) != NULL) {
+                        if (!strcmp (entry->d_name, ".") ||
+                            !strcmp (entry->d_name,"..")) {
+                                continue;
+                        }
+                        ret = unlink (entry->d_name);
+                        if (ret)
+                                goto out;
+                }
+                closedir (dir);
+        }
+
+        /*At this point of time we have index/base_indicies_holder directory
+         *is with no entries*/
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto out;
+        }
+        set_lk_owner_from_ptr (&frame->root->lk_owner, frame->root);
+
+        frame->root->pid = LOW_PRIO_PROC_PID;
+
+        ret = synctask_new (this->ctx->env, sync_base_indices,
+                            base_indices_syncing_done,frame, priv);
+
+
+
+out:
+        return ret;
+
+}
+
+int
+index_add (xlator_t *this, uuid_t gfid, const char *subdir)
+{
+        int32_t           op_errno = 0;
+        char              gfid_path[PATH_MAX] = {0};
+        char              index_path[PATH_MAX] = {0};
+        char              base_path[PATH_MAX] = {0};
+        int               ret = 0;
+        uuid_t            index = {0};
+        index_priv_t      *priv = NULL;
+        struct stat       st = {0};
+        int               fd = 0;
+        int               index_created = 0;
+
+        priv = this->private;
+        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !uuid_is_null (gfid),
+                                       out, op_errno, EINVAL);
+
+        make_gfid_path (priv->index_basepath, subdir, gfid,
+                        gfid_path, sizeof (gfid_path));
+
+        ret = stat (gfid_path, &st);
+        if (!ret)
+                goto out;
+        index_get_index (priv, index);
+        make_index_path (priv->index_basepath, subdir,
+                         index, index_path, sizeof (index_path));
+        ret = link (index_path, gfid_path);
+        if (!ret || (errno == EEXIST))  {
+                ret = 0;
+                index_created = 1;
+                goto out;
+        }
+
+
+        op_errno = errno;
+        if (op_errno == ENOENT) {
+                ret = index_dir_create (this, subdir);
+                if (ret)
+                        goto out;
+        } else if (op_errno == EMLINK) {
+                index_generate_index (priv, index);
+                make_index_path (priv->index_basepath, subdir,
+                                 index, index_path, sizeof (index_path));
+        } else {
+                goto out;
+        }
+
+        fd = creat (index_path, 0);
+        if ((fd < 0) && (errno != EEXIST)) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
+                        "create index (%s)", uuid_utoa (gfid),
+                        strerror (errno));
+                goto out;
+        }
+
+        if (fd >= 0)
+                close (fd);
+
+        ret = link (index_path, gfid_path);
+        if (ret && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
+                        "add to index (%s)", uuid_utoa (gfid),
+                        strerror (errno));
+                goto out;
+        } else {
+                index_created = 1;
+        }
+
+        if (priv->to_be_healed_states != sync_not_started) {
+                 make_index_path (priv->index_basepath,
+                                  GF_BASE_INDICES_HOLDER_GFID,
+                                  index, base_path, sizeof (base_path));
+                 ret = link (index_path, base_path);
+                 if (ret)
+                         goto out;
+        }
+
+        ret = 0;
+out:
+        /*If base_indices_holder is not created: create and sync
+         *If directory is present: delete contents and start syncing
+         *If syncing is in progress :No need to do any thing
+         *If syncing is done: No need to do anything*/
+        if (!ret) {
+                switch (priv->to_be_healed_states) {
+                        case sync_not_started:
+                                ret = sync_base_indices_from_xattrop (this);
+                                break;
+                        case sync_started:
+                        case synced_state:
+                                /*No need to do anything*/
+                                break;
+                }
+        }
+        return ret;
+}
+
+int
+index_del (xlator_t *this, uuid_t gfid, const char *subdir)
+{
+        int32_t      op_errno __attribute__((unused)) = 0;
+        index_priv_t *priv = NULL;
+        int          ret = 0;
+        char         gfid_path[PATH_MAX] = {0};
+
+        priv = this->private;
+        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !uuid_is_null (gfid),
+                                       out, op_errno, EINVAL);
+        make_gfid_path (priv->index_basepath, subdir, gfid,
+                        gfid_path, sizeof (gfid_path));
+        ret = unlink (gfid_path);
+        if (ret && (errno != ENOENT)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to delete from index (%s)",
+                        gfid_path, strerror (errno));
+                ret = -errno;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+_check_key_is_zero_filled (dict_t *d, char *k, data_t *v,
+                           void *tmp)
+{
+        if (mem_0filled ((const char*)v->data, v->len)) {
+                /* -1 means, no more iterations, treat as 'break' */
+                return -1;
+        }
+        return 0;
+}
+
+
+void
+_xattrop_index_action (xlator_t *this, inode_t *inode,  dict_t *xattr)
+{
+        gf_boolean_t      zero_xattr = _gf_true;
+        index_inode_ctx_t *ctx = NULL;
+        int               ret = 0;
+
+        ret = dict_foreach (xattr, _check_key_is_zero_filled, NULL);
+        if (ret == -1)
+                zero_xattr = _gf_false;
+
+        ret = index_inode_ctx_get (inode, this, &ctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Not able to %s %s -> index",
+                        zero_xattr?"add":"del", uuid_utoa (inode->gfid));
+                goto out;
+        }
+        if (zero_xattr) {
+                if (ctx->state == NOTIN)
+                        goto out;
+                ret = index_del (this, inode->gfid, XATTROP_SUBDIR);
+                if (!ret)
+                        ctx->state = NOTIN;
+        } else {
+                if (ctx->state == IN)
+                        goto out;
+                ret = index_add (this, inode->gfid, XATTROP_SUBDIR);
+                if (!ret)
+                        ctx->state = IN;
+        }
+out:
+        return;
+}
+
+void
+fop_xattrop_index_action (xlator_t *this, inode_t *inode, dict_t *xattr)
+{
+        _xattrop_index_action (this, inode, xattr);
+}
+
+void
+fop_fxattrop_index_action (xlator_t *this, inode_t *inode, dict_t *xattr)
+{
+        _xattrop_index_action (this, inode, xattr);
+}
+
+inline gf_boolean_t
+index_xattrop_track (loc_t *loc, gf_xattrop_flags_t flags, dict_t *dict)
+{
+        return (flags == GF_XATTROP_ADD_ARRAY);
+}
+
+inline gf_boolean_t
+index_fxattrop_track (fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+{
+        return (flags == GF_XATTROP_ADD_ARRAY);
+}
+
+int
+__index_fd_ctx_get (fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+{
+        int               ret = 0;
+        index_fd_ctx_t    *fctx = NULL;
+        uint64_t          tmpctx = 0;
+        char              index_dir[PATH_MAX] = {0};
+        index_priv_t      *priv = NULL;
+
+        priv = this->private;
+        if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid)) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        ret = __fd_ctx_get (fd, this, &tmpctx);
+        if (!ret) {
+                fctx = (index_fd_ctx_t*) (long) tmpctx;
+                goto out;
+        }
+
+        fctx = GF_CALLOC (1, sizeof (*fctx), gf_index_fd_ctx_t);
+        if (!fctx) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
+                             index_dir, sizeof (index_dir));
+        fctx->dir = opendir (index_dir);
+        if (!fctx->dir) {
+                ret = -errno;
+                GF_FREE (fctx);
+                fctx = NULL;
+                goto out;
+        }
+
+        ret = __fd_ctx_set (fd, this, (uint64_t)(long)fctx);
+        if (ret) {
+                GF_FREE (fctx);
+                fctx = NULL;
+                ret = -EINVAL;
+                goto out;
+        }
+out:
+        if (fctx)
+                *ctx = fctx;
+        return ret;
+}
+
+int
+index_fd_ctx_get (fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+{
+        int     ret = 0;
+        LOCK (&fd->lock);
+        {
+                ret = __index_fd_ctx_get (fd, this, ctx);
+        }
+        UNLOCK (&fd->lock);
+        return ret;
+}
+
+//new - Not NULL means start a fop
+//new - NULL means done processing the fop
+void
+index_queue_process (xlator_t *this, inode_t *inode, call_stub_t *new)
+{
+        call_stub_t       *stub = NULL;
+        index_inode_ctx_t *ctx = NULL;
+        int               ret = 0;
+        call_frame_t      *frame = NULL;
+
+        LOCK (&inode->lock);
+        {
+                ret = __index_inode_ctx_get (inode, this, &ctx);
+                if (ret)
+                        goto unlock;
+
+                if (new) {
+                        __index_enqueue (&ctx->callstubs, new);
+                        new = NULL;
+                } else {
+                        ctx->processing = _gf_false;
+                }
+
+                if (!ctx->processing) {
+                        stub = __index_dequeue (&ctx->callstubs);
+                        if (stub)
+                                ctx->processing = _gf_true;
+                        else
+                                ctx->processing = _gf_false;
+                }
+        }
+unlock:
+        UNLOCK (&inode->lock);
+
+        if (ret && new) {
+                frame = new->frame;
+                if (new->fop == GF_FOP_XATTROP) {
+                        INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM,
+                                            NULL, NULL);
+                } else if (new->fop == GF_FOP_FXATTROP) {
+                        INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM,
+                                            NULL, NULL);
+                }
+                call_stub_destroy (new);
+        } else if (stub) {
+                call_resume (stub);
+        }
+        return;
+}
+
+int32_t
+index_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata)
+{
+        inode_t *inode = NULL;
+
+        inode = inode_ref (frame->local);
+        if (op_ret < 0)
+                goto out;
+        fop_xattrop_index_action (this, frame->local, xattr);
+out:
+        INDEX_STACK_UNWIND (xattrop, frame, op_ret, op_errno, xattr, xdata);
+        index_queue_process (this, inode, NULL);
+        inode_unref (inode);
+
+        return 0;
+}
+
+int32_t
+index_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                    dict_t *xdata)
+{
+        inode_t *inode = NULL;
+
+        inode = inode_ref (frame->local);
+        if (op_ret < 0)
+                goto out;
+
+        fop_fxattrop_index_action (this, frame->local, xattr);
+out:
+        INDEX_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, xattr, xdata);
+        index_queue_process (this, inode, NULL);
+        inode_unref (inode);
+
+        return 0;
+}
+
+int
+index_xattrop_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        STACK_WIND (frame, index_xattrop_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->xattrop, loc, optype, xattr,
+                    xdata);
+        return 0;
+}
+
+int
+index_fxattrop_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        STACK_WIND (frame, index_fxattrop_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->fxattrop, fd, optype, xattr,
+                    xdata);
+        return 0;
+}
+
+int32_t
+index_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+	       gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+
+        if (!index_xattrop_track (loc, flags, dict))
+                goto out;
+
+        frame->local = inode_ref (loc->inode);
+        stub = fop_xattrop_stub (frame, index_xattrop_wrapper,
+                                 loc, flags, dict, xdata);
+        if (!stub) {
+                INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        index_queue_process (this, loc->inode, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_xattrop_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+        return 0;
+}
+
+int32_t
+index_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+
+        if (!index_fxattrop_track (fd, flags, dict))
+                goto out;
+
+        frame->local = inode_ref (fd->inode);
+        stub = fop_fxattrop_stub (frame, index_fxattrop_wrapper,
+                                  fd, flags, dict, xdata);
+        if (!stub) {
+                INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM, NULL, xdata);
+                return 0;
+        }
+
+        index_queue_process (this, fd->inode, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_fxattrop_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+        return 0;
+}
+
+int32_t
+index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
+                        loc_t *loc, const char *name, dict_t *xdata)
+{
+        index_priv_t    *priv = NULL;
+        dict_t          *xattr = NULL;
+        int             ret = 0;
+
+        priv = this->private;
+
+        xattr = dict_new ();
+        if (!xattr) {
+                ret = -ENOMEM;
+                goto done;
+        }
+
+        if (!strcmp (name, GF_XATTROP_INDEX_GFID)) {
+
+                ret = dict_set_static_bin (xattr, (char*)name,
+                                           priv->xattrop_vgfid,
+                                           sizeof (priv->xattrop_vgfid));
+
+        } else if (!strcmp (name, GF_BASE_INDICES_HOLDER_GFID)) {
+
+                ret = dict_set_static_bin (xattr, (char*)name,
+                                           priv->base_indices_holder_vgfid,
+                                      sizeof (priv->base_indices_holder_vgfid));
+        }
+        if (ret) {
+                ret = -ENOMEM;
+                gf_log (THIS->name, GF_LOG_ERROR, "xattrop index "
+                        "gfid set failed");
+                goto done;
+        }
+done:
+        if (ret)
+                STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, xattr, xdata);
+        else
+                STACK_UNWIND_STRICT (getxattr, frame, 0, 0, xattr, xdata);
+
+        if (xattr)
+                dict_unref (xattr);
+
+        return 0;
+}
+
+int32_t
+index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
+                      loc_t *loc, dict_t *xattr_req)
+{
+        index_priv_t    *priv = NULL;
+        struct stat     lstatbuf = {0};
+        int             ret = 0;
+        int32_t         op_errno = EINVAL;
+        int32_t         op_ret = -1;
+        char            path[PATH_MAX] = {0};
+        struct iatt     stbuf        = {0, };
+        struct iatt     postparent = {0,};
+        dict_t          *xattr = NULL;
+        gf_boolean_t    is_dir = _gf_false;
+
+        priv = this->private;
+
+        VALIDATE_OR_GOTO (loc, done);
+        if (!uuid_compare (loc->gfid, priv->xattrop_vgfid)) {
+                make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
+                                     path, sizeof (path));
+                is_dir = _gf_true;
+        } else if (!uuid_compare (loc->pargfid, priv->xattrop_vgfid)) {
+                make_file_path (priv->index_basepath, XATTROP_SUBDIR,
+                                loc->name, path, sizeof (path));
+        } else if (!uuid_compare (loc->gfid,priv->base_indices_holder_vgfid)){
+                make_index_dir_path (priv->index_basepath,
+                                     BASE_INDICES_HOLDER_SUBDIR, path,
+                                    sizeof (path));
+                is_dir = _gf_true;
+        } else if (!uuid_compare (loc->pargfid, priv->base_indices_holder_vgfid)) {
+                make_file_path (priv->index_basepath,
+                                BASE_INDICES_HOLDER_SUBDIR,loc->name, path,
+                                sizeof (path));
+        }
+
+        ret = lstat (path, &lstatbuf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Stat failed on index dir "
+                        "(%s)", strerror (errno));
+                op_errno = errno;
+                goto done;
+        } else if (!S_ISDIR (lstatbuf.st_mode) && is_dir) {
+                gf_log (this->name, GF_LOG_DEBUG, "Stat failed on index dir, "
+                        "not a directory");
+                op_errno = ENOENT;
+                goto done;
+        }
+        xattr = dict_new ();
+        if (!xattr) {
+                op_errno = ENOMEM;
+                goto done;
+        }
+
+        iatt_from_stat (&stbuf, &lstatbuf);
+        if (is_dir && !uuid_compare (loc->gfid, priv->xattrop_vgfid)) {
+                uuid_copy (stbuf.ia_gfid, priv->xattrop_vgfid);
+        } else if (is_dir &&
+                !uuid_compare (loc->gfid, priv->base_indices_holder_vgfid)) {
+                uuid_copy (stbuf.ia_gfid, priv->base_indices_holder_vgfid);
+        } else {
+                uuid_generate (stbuf.ia_gfid);
+        }
+        stbuf.ia_ino = -1;
+        op_ret = 0;
+done:
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno,
+                             loc->inode, &stbuf, xattr, &postparent);
+        if (xattr)
+                dict_unref (xattr);
+        return 0;
+}
+
+int32_t
+base_indices_readdir_wrapper (call_frame_t *frame, xlator_t *this,
+                              fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        index_priv_t    *priv = NULL;
+        char            base_indices_holder[PATH_MAX] = {0};
+        DIR             *dir = NULL;
+        int32_t         op_ret = -1;
+        int32_t         op_errno = 0;
+        int             count = 0;
+        gf_dirent_t     entries;
+
+        priv = this->private;
+
+        make_index_dir_path (priv->index_basepath, BASE_INDICES_HOLDER_SUBDIR,
+                             base_indices_holder, sizeof (base_indices_holder));
+
+        dir = opendir (base_indices_holder);
+        if (!dir) {
+                op_errno = EINVAL;
+                goto done;
+        }
+
+
+        INIT_LIST_HEAD (&entries.list);
+
+        count = index_fill_readdir (fd, dir, off, size, &entries,
+                                    BASE_INDICES_HOLDER);
+        /* pick ENOENT to indicate EOF */
+        op_errno = errno;
+        op_ret = count;
+        closedir (dir);
+done:
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
+        gf_dirent_free (&entries);
+        return 0;
+}
+
+int32_t
+index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
+                       fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        index_fd_ctx_t       *fctx           = NULL;
+        DIR                  *dir            = NULL;
+        int                   ret            = -1;
+        int32_t               op_ret         = -1;
+        int32_t               op_errno       = 0;
+        int                   count          = 0;
+        gf_dirent_t           entries;
+
+        INIT_LIST_HEAD (&entries.list);
+
+        ret = index_fd_ctx_get (fd, this, &fctx);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "pfd is NULL, fd=%p", fd);
+                op_errno = -ret;
+                goto done;
+        }
+
+        dir = fctx->dir;
+
+        if (!dir) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dir is NULL for fd=%p", fd);
+                op_errno = EINVAL;
+                goto done;
+        }
+
+        count = index_fill_readdir (fd, dir, off, size, &entries,
+                                    INDEX_XATTROP);
+
+        /* pick ENOENT to indicate EOF */
+        op_errno = errno;
+        op_ret = count;
+done:
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
+        gf_dirent_free (&entries);
+        return 0;
+}
+
+int
+index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+                      dict_t *xdata)
+{
+        index_priv_t    *priv = NULL;
+        int32_t         op_ret = 0;
+        int32_t         op_errno = 0;
+        int             ret = 0;
+        struct  iatt    preparent = {0};
+        struct  iatt    postparent = {0};
+        char            index_dir[PATH_MAX] = {0};
+        struct  stat    lstatbuf = {0};
+        uuid_t          gfid = {0};
+
+        priv = this->private;
+        make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
+                             index_dir, sizeof (index_dir));
+        ret = lstat (index_dir, &lstatbuf);
+        if (ret < 0) {
+                op_ret = -1;
+                op_errno = errno;
+                goto done;
+        }
+
+        iatt_from_stat (&preparent, &lstatbuf);
+        uuid_copy (preparent.ia_gfid, priv->xattrop_vgfid);
+        preparent.ia_ino = -1;
+        uuid_parse (loc->name, gfid);
+        ret = index_del (this, gfid, XATTROP_SUBDIR);
+        if (ret < 0) {
+                op_ret = -1;
+                op_errno = -ret;
+                goto done;
+        }
+        memset (&lstatbuf, 0, sizeof (lstatbuf));
+        ret = lstat (index_dir, &lstatbuf);
+        if (ret < 0) {
+                op_ret = -1;
+                op_errno = errno;
+                goto done;
+        }
+        iatt_from_stat (&postparent, &lstatbuf);
+        uuid_copy (postparent.ia_gfid, priv->xattrop_vgfid);
+        postparent.ia_ino = -1;
+done:
+        INDEX_STACK_UNWIND (unlink, frame, op_ret, op_errno, &preparent,
+                            &postparent, xdata);
+        return 0;
+}
+
+int32_t
+index_getxattr (call_frame_t *frame, xlator_t *this,
+                loc_t *loc, const char *name, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+
+        if (!name)
+                goto out;
+        if (strcmp (GF_XATTROP_INDEX_GFID, name) &&
+            strcmp (GF_BASE_INDICES_HOLDER_GFID, name))
+                goto out;
+
+        stub = fop_getxattr_stub (frame, index_getxattr_wrapper, loc, name,
+                                  xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+        return 0;
+}
+
+int32_t
+index_lookup (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, dict_t *xattr_req)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+
+        if (uuid_compare (loc->gfid, priv->xattrop_vgfid) &&
+            uuid_compare (loc->pargfid, priv->xattrop_vgfid) &&
+            uuid_compare (loc->gfid, priv->base_indices_holder_vgfid) &&
+            uuid_compare (loc->pargfid, priv->base_indices_holder_vgfid))
+                goto normal;
+
+        stub = fop_lookup_stub (frame, index_lookup_wrapper, loc, xattr_req);
+        if (!stub) {
+                STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM, loc->inode,
+                                     NULL, NULL, NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+normal:
+        STACK_WIND (frame, default_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+
+        return 0;
+}
+
+int32_t
+index_readdir (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid) &&
+            uuid_compare (fd->inode->gfid, priv->base_indices_holder_vgfid))
+                goto out;
+
+        if (!uuid_compare (fd->inode->gfid, priv->xattrop_vgfid)) {
+                stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, 
+                                         off, xdata);
+        } else if (!uuid_compare (fd->inode->gfid,
+                                  priv->base_indices_holder_vgfid)) {
+                stub = fop_readdir_stub (frame, base_indices_readdir_wrapper,
+                                         fd, size, off, xdata);
+        }
+
+        if (!stub) {
+                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+        return 0;
+}
+
+int
+index_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        if (uuid_compare (loc->pargfid, priv->xattrop_vgfid))
+                goto out;
+
+        stub = fop_unlink_stub (frame, index_unlink_wrapper, loc, xflag, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL,
+                                     NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_index_mt_end + 1);
+
+        return ret;
+}
+
+int
+init (xlator_t *this)
+{
+        int ret = -1;
+        index_priv_t *priv = NULL;
+        pthread_t thread;
+        pthread_attr_t  w_attr;
+        gf_boolean_t    mutex_inited = _gf_false;
+        gf_boolean_t    cond_inited  = _gf_false;
+        gf_boolean_t    attr_inited  = _gf_false;
+
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"'index' not configured with exactly one child");
+                goto out;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_index_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        LOCK_INIT (&priv->lock);
+        if ((ret = pthread_cond_init(&priv->cond, NULL)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_cond_init failed (%d)", ret);
+                goto out;
+        }
+        cond_inited = _gf_true;
+
+        if ((ret = pthread_mutex_init(&priv->mutex, NULL)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_mutex_init failed (%d)", ret);
+                goto out;
+        }
+        mutex_inited = _gf_true;
+
+        if ((ret = pthread_attr_init (&w_attr)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_attr_init failed (%d)", ret);
+                goto out;
+        }
+        attr_inited = _gf_true;
+
+        ret = pthread_attr_setstacksize (&w_attr, INDEX_THREAD_STACK_SIZE);
+        if (ret == EINVAL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Using default thread stack size");
+        }
+        GF_OPTION_INIT ("index-base", priv->index_basepath, path, out);
+        uuid_generate (priv->index);
+        uuid_generate (priv->xattrop_vgfid);
+        /*base_indices_holder is a directory which contains hard links to
+         * all base indices inside indices/xattrop directory*/
+        uuid_generate (priv->base_indices_holder_vgfid);
+        INIT_LIST_HEAD (&priv->callstubs);
+
+        this->private = priv;
+        ret = gf_thread_create (&thread, &w_attr, index_worker, this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create "
+                        "worker thread, aborting");
+                goto out;
+        }
+        ret = 0;
+out:
+        if (ret) {
+                if (cond_inited)
+                        pthread_cond_destroy (&priv->cond);
+                if (mutex_inited)
+                        pthread_mutex_destroy (&priv->mutex);
+                if (priv)
+                        GF_FREE (priv);
+                this->private = NULL;
+        }
+        if (attr_inited)
+                pthread_attr_destroy (&w_attr);
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        index_priv_t *priv = NULL;
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+        this->private = NULL;
+        LOCK_DESTROY (&priv->lock);
+        pthread_cond_destroy (&priv->cond);
+        pthread_mutex_destroy (&priv->mutex);
+        GF_FREE (priv);
+out:
+        return;
+}
+
+int
+index_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t tmp_cache = 0;
+        if (!inode_ctx_del (inode, this, &tmp_cache))
+                GF_FREE ((index_inode_ctx_t*) (long)tmp_cache);
+
+        return 0;
+}
+
+int32_t
+index_releasedir (xlator_t *this, fd_t *fd)
+{
+        index_fd_ctx_t *fctx = NULL;
+        uint64_t        ctx = 0;
+        int             ret = 0;
+
+        ret = fd_ctx_del (fd, this, &ctx);
+        if (ret < 0)
+                goto out;
+
+        fctx = (index_fd_ctx_t*) (long) ctx;
+        if (fctx->dir)
+                closedir (fctx->dir);
+
+        GF_FREE (fctx);
+out:
+        return 0;
+}
+
+int32_t
+index_release (xlator_t *this, fd_t *fd)
+{
+        index_fd_ctx_t *fctx = NULL;
+        uint64_t        ctx = 0;
+        int             ret = 0;
+
+        ret = fd_ctx_del (fd, this, &ctx);
+        if (ret < 0)
+                goto out;
+
+        fctx = (index_fd_ctx_t*) (long) ctx;
+        GF_FREE (fctx);
+out:
+        return 0;
+}
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+        int     ret = 0;
+        ret = default_notify (this, event, data);
+        return ret;
+}
+
+struct xlator_fops fops = {
+	.xattrop     = index_xattrop,
+	.fxattrop    = index_fxattrop,
+
+        //interface functions follow
+        .getxattr    = index_getxattr,
+        .lookup      = index_lookup,
+        .readdir     = index_readdir,
+        .unlink      = index_unlink
+};
+
+struct xlator_dumpops dumpops;
+
+struct xlator_cbks cbks = {
+        .forget         = index_forget,
+        .release        = index_release,
+        .releasedir     = index_releasedir
+};
+
+struct volume_options options[] = {
+        { .key  = {"index-base" },
+          .type = GF_OPTION_TYPE_PATH,
+          .description = "path where the index files need to be stored",
+        },
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/index/src/index.h b/xlators/features/index/src/index.h
new file mode 100644
index 000000000..d6dcb1c23
--- /dev/null
+++ b/xlators/features/index/src/index.h
@@ -0,0 +1,73 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __INDEX_H__
+#define __INDEX_H__
+
+#include "xlator.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "byte-order.h"
+#include "common-utils.h"
+#include "index-mem-types.h"
+
+#define INDEX_THREAD_STACK_SIZE   ((size_t)(1024*1024))
+
+typedef enum {
+        UNKNOWN,
+        IN,
+        NOTIN
+} index_state_t;
+
+typedef struct index_inode_ctx {
+        gf_boolean_t processing;
+        struct list_head callstubs;
+        index_state_t state;
+} index_inode_ctx_t;
+
+typedef struct index_fd_ctx {
+        DIR *dir;
+} index_fd_ctx_t;
+
+typedef enum {
+        sync_not_started,
+        sync_started,
+        synced_state,
+} to_be_healed_states_t;
+
+typedef enum {
+        INDEX_XATTROP,
+        BASE_INDICES_HOLDER,
+} readdir_directory;
+
+typedef struct index_priv {
+        char *index_basepath;
+        uuid_t index;
+        gf_lock_t lock;
+        uuid_t xattrop_vgfid;//virtual gfid of the xattrop index dir
+        uuid_t base_indices_holder_vgfid; //virtual gfid of the
+                                          //to_be_healed_xattrop directory
+        struct list_head callstubs;
+        pthread_mutex_t mutex;
+        pthread_cond_t  cond;
+        to_be_healed_states_t to_be_healed_states;
+} index_priv_t;
+
+#define INDEX_STACK_UNWIND(fop, frame, params ...)      \
+do {                                                    \
+        if (frame) {                                    \
+                inode_t *_inode = frame->local;         \
+                frame->local = NULL;                    \
+                inode_unref (_inode);                   \
+        }                                               \
+        STACK_UNWIND_STRICT (fop, frame, params);       \
+} while (0)
+
+#endif
diff --git a/xlators/features/locks/src/Makefile.am b/xlators/features/locks/src/Makefile.am
index e39676826..0f79731b4 100644
--- a/xlators/features/locks/src/Makefile.am
+++ b/xlators/features/locks/src/Makefile.am
@@ -1,7 +1,7 @@
 xlator_LTLIBRARIES = locks.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-locks_la_LDFLAGS = -module -avoidversion
+locks_la_LDFLAGS = -module -avoid-version
 
 locks_la_SOURCES = common.c posix.c entrylk.c inodelk.c reservelk.c \
 		   clear.c
@@ -9,9 +9,10 @@ locks_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = locks.h common.h locks-mem-types.h clear.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
-	-fno-strict-aliasing -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src $(GF_CFLAGS) -shared -nostartfiles
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/locks/src/clear.c b/xlators/features/locks/src/clear.c
index a7b2455e3..124b9ad0f 100644
--- a/xlators/features/locks/src/clear.c
+++ b/xlators/features/locks/src/clear.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -141,15 +131,15 @@ clrlk_parse_args (const char* cmd, clrlk_args *args)
         if ((args->type == CLRLK_TYPE_MAX) || (args->kind == CLRLK_KIND_MAX))
                 goto out;
 
-        /*optional args*/
-        tok = strtok_r (NULL, ".", &sptr);
+        /*optional args, neither range nor basename can 'legally' contain
+         * "/" in them*/
+        tok = strtok_r (NULL, "/", &sptr);
         if (tok)
                 args->opts = gf_strdup (tok);
 
         ret = 0;
 out:
-        if (free_ptr)
-                GF_FREE (free_ptr);
+        GF_FREE (free_ptr);
         return ret;
 }
 
@@ -193,8 +183,8 @@ clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
                                               F_SETLKW, &plock->user_flock,
                                               -1, EAGAIN, NULL);
 
-                                STACK_UNWIND (plock->frame, -1, EAGAIN,
-                                              &plock->user_flock);
+                                STACK_UNWIND_STRICT (lk, plock->frame, -1, EAGAIN,
+                                                     &plock->user_flock, NULL);
 
                         } else {
                                 gcount++;
@@ -204,6 +194,7 @@ clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
         }
         pthread_mutex_unlock (&pl_inode->mutex);
         grant_blocked_locks (this, pl_inode);
+        ret = 0;
 out:
         *blkd    = bcount;
         *granted = gcount;
@@ -222,7 +213,9 @@ clrlk_clear_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
         int                     bcount          = 0;
         int                     gcount          = 0;
         gf_boolean_t            chk_range       = _gf_false;
+        struct list_head        released;
 
+        INIT_LIST_HEAD (&released);
         if (clrlk_get_lock_range (args->opts, &ulock, &chk_range)) {
                 *op_errno = EINVAL;
                 goto out;
@@ -246,17 +239,23 @@ blkd:
                                 continue;
 
                         bcount++;
-                        list_del_init (&ilock->list);
-                        pl_trace_out (this, ilock->frame, NULL, NULL, F_SETLKW,
-                                      &ilock->user_flock, -1, EAGAIN,
-                                      ilock->volume);
-                        STACK_UNWIND_STRICT (inodelk, ilock->frame, -1,
-                                             EAGAIN);
-                        GF_FREE (ilock);
+                        list_del_init (&ilock->blocked_locks);
+                        list_add (&ilock->blocked_locks, &released);
                 }
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
+        list_for_each_entry_safe (ilock, tmp, &released, blocked_locks) {
+                list_del_init (&ilock->blocked_locks);
+                pl_trace_out (this, ilock->frame, NULL, NULL, F_SETLKW,
+                              &ilock->user_flock, -1, EAGAIN,
+                              ilock->volume);
+                STACK_UNWIND_STRICT (inodelk, ilock->frame, -1,
+                                     EAGAIN, NULL);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (ilock);
+        }
+
         if (!(args->kind & CLRLK_GRANTED)) {
                 ret = 0;
                 goto out;
@@ -275,14 +274,20 @@ granted:
 
                         gcount++;
                         list_del_init (&ilock->list);
-                        GF_FREE (ilock);
+                        list_add (&ilock->list, &released);
                 }
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
-        grant_blocked_inode_locks (this, pl_inode, dom);
+        list_for_each_entry_safe (ilock, tmp, &released, list) {
+                list_del_init (&ilock->list);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (ilock);
+        }
+
         ret = 0;
 out:
+        grant_blocked_inode_locks (this, pl_inode, dom);
         *blkd    = bcount;
         *granted = gcount;
         return ret;
@@ -295,11 +300,13 @@ clrlk_clear_entrylk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
 {
         pl_entry_lock_t         *elock          = NULL;
         pl_entry_lock_t         *tmp            = NULL;
-        struct list_head        removed         = {0};
         int                     bcount          = 0;
         int                     gcount          = 0;
         int                     ret             = -1;
+        struct list_head        removed;
+        struct list_head        released;
 
+        INIT_LIST_HEAD (&released);
         if (args->kind & CLRLK_BLOCKED)
                 goto blkd;
 
@@ -311,21 +318,31 @@ blkd:
         {
                 list_for_each_entry_safe (elock, tmp, &dom->blocked_entrylks,
                                           blocked_locks) {
-                        if (args->opts &&
-                            strncmp (elock->basename, args->opts,
-                                     strlen (elock->basename)))
-                                continue;
+                        if (args->opts) {
+                                if (!elock->basename ||
+                                    strcmp (elock->basename, args->opts))
+                                        continue;
+                        }
 
                         bcount++;
-                        list_del_init (&elock->domain_list);
-                        STACK_UNWIND_STRICT (entrylk, elock->frame, -1,
-                                             EAGAIN);
-                        GF_FREE ((char *) elock->basename);
-                        GF_FREE (elock);
+
+                        list_del_init (&elock->blocked_locks);
+                        list_add_tail (&elock->blocked_locks, &released);
                 }
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
+        list_for_each_entry_safe (elock, tmp, &released, blocked_locks) {
+                list_del_init (&elock->blocked_locks);
+                entrylk_trace_out (this, elock->frame, elock->volume, NULL, NULL,
+                                   elock->basename, ENTRYLK_LOCK, elock->type,
+                                   -1, EAGAIN);
+                STACK_UNWIND_STRICT (entrylk, elock->frame, -1, EAGAIN, NULL);
+                GF_FREE ((char *) elock->basename);
+                GF_FREE (elock->connection_id);
+                GF_FREE (elock);
+        }
+
         if (!(args->kind & CLRLK_GRANTED)) {
                 ret = 0;
                 goto out;
@@ -337,13 +354,11 @@ granted:
         {
                 list_for_each_entry_safe (elock, tmp, &dom->entrylk_list,
                                           domain_list) {
-                        if (!elock->basename)
-                                continue;
-
-                        if (args->opts &&
-                            strncmp (elock->basename, args->opts,
-                                     strlen (elock->basename)))
-                                continue;
+                        if (args->opts) {
+                                if (!elock->basename ||
+                                    strcmp (elock->basename, args->opts))
+                                        continue;
+                        }
 
                         gcount++;
                         list_del_init (&elock->domain_list);
@@ -365,8 +380,8 @@ out:
 
 int
 clrlk_clear_lks_in_all_domains (xlator_t *this, pl_inode_t *pl_inode,
-                          clrlk_args *args, int *blkd, int *granted,
-                          int *op_errno)
+                                clrlk_args *args, int *blkd, int *granted,
+                                int *op_errno)
 {
         pl_dom_list_t   *dom            = NULL;
         int             ret             = -1;
diff --git a/xlators/features/locks/src/clear.h b/xlators/features/locks/src/clear.h
index 95572a971..511f3f74a 100644
--- a/xlators/features/locks/src/clear.h
+++ b/xlators/features/locks/src/clear.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLEAR_H__
 #define __CLEAR_H__
 
diff --git a/xlators/features/locks/src/common.c b/xlators/features/locks/src/common.c
index 0bc7baa30..b3309580d 100644
--- a/xlators/features/locks/src/common.c
+++ b/xlators/features/locks/src/common.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -45,8 +35,9 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock);
 static int
 pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
                         posix_lock_t *old_lock);
+
 static pl_dom_list_t *
-allocate_domain (const char *volume)
+__allocate_domain (const char *volume)
 {
         pl_dom_list_t *dom = NULL;
 
@@ -85,24 +76,26 @@ get_domain (pl_inode_t *pl_inode, const char *volume)
 {
         pl_dom_list_t *dom = NULL;
 
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, pl_inode, out);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, volume, out);
-
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                if (strcmp (dom->domain, volume) == 0)
-                        goto found;
+        GF_VALIDATE_OR_GOTO ("posix-locks", pl_inode, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", volume, out);
 
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+                        if (strcmp (dom->domain, volume) == 0)
+                                goto unlock;
+                }
 
+                dom = __allocate_domain (volume);
+                if (dom)
+                        list_add (&dom->inode_list, &pl_inode->dom_list);
         }
-
-        dom = allocate_domain (volume);
-        if (dom)
-                list_add (&dom->inode_list, &pl_inode->dom_list);
-found:
+unlock:
+        pthread_mutex_unlock (&pl_inode->mutex);
         if (dom) {
-                gf_log (POSIX_LOCKS, GF_LOG_TRACE, "Domain %s found", volume);
+                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s found", volume);
         } else {
-                gf_log (POSIX_LOCKS, GF_LOG_TRACE, "Domain %s not found", volume);
+                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s not found", volume);
         }
 out:
         return dom;
@@ -143,10 +136,10 @@ __pl_inode_is_empty (pl_inode_t *pl_inode)
 void
 pl_print_locker (char *str, int size, xlator_t *this, call_frame_t *frame)
 {
-        snprintf (str, size, "Pid=%llu, lk-owner=%s, Transport=%p, Frame=%llu",
+        snprintf (str, size, "Pid=%llu, lk-owner=%s, Client=%p, Frame=%llu",
                   (unsigned long long) frame->root->pid,
                   lkowner_utoa (&frame->root->lk_owner),
-                  (void *)frame->root->trans,
+                  frame->root->client,
                   (unsigned long long) frame->root->unique);
 }
 
@@ -180,8 +173,7 @@ pl_print_lockee (char *str, int size, fd_t *fd, loc_t *loc)
                   uuid_utoa (inode->gfid), fd,
                   ipath ? ipath : "<nul>");
 
-        if (ipath)
-                GF_FREE (ipath);
+        GF_FREE (ipath);
 }
 
 
@@ -435,46 +427,50 @@ pl_inode_get (xlator_t *this, inode_t *inode)
         pl_inode_t *pl_inode = NULL;
         int         ret = 0;
 
-        ret = inode_ctx_get (inode, this,&tmp_pl_inode);
-        if (ret == 0) {
-                pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
-                goto out;
-        }
-        pl_inode = GF_CALLOC (1, sizeof (*pl_inode),
-                              gf_locks_mt_pl_inode_t);
-        if (!pl_inode) {
-                goto out;
-        }
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
+                if (ret == 0) {
+                        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+                        goto unlock;
+                }
+                pl_inode = GF_CALLOC (1, sizeof (*pl_inode),
+                                      gf_locks_mt_pl_inode_t);
+                if (!pl_inode) {
+                        goto unlock;
+                }
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "Allocating new pl inode");
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Allocating new pl inode");
 
-        pthread_mutex_init (&pl_inode->mutex, NULL);
+                pthread_mutex_init (&pl_inode->mutex, NULL);
 
-        INIT_LIST_HEAD (&pl_inode->dom_list);
-        INIT_LIST_HEAD (&pl_inode->ext_list);
-        INIT_LIST_HEAD (&pl_inode->rw_list);
-        INIT_LIST_HEAD (&pl_inode->reservelk_list);
-        INIT_LIST_HEAD (&pl_inode->blocked_reservelks);
-        INIT_LIST_HEAD (&pl_inode->blocked_calls);
+                INIT_LIST_HEAD (&pl_inode->dom_list);
+                INIT_LIST_HEAD (&pl_inode->ext_list);
+                INIT_LIST_HEAD (&pl_inode->rw_list);
+                INIT_LIST_HEAD (&pl_inode->reservelk_list);
+                INIT_LIST_HEAD (&pl_inode->blocked_reservelks);
+                INIT_LIST_HEAD (&pl_inode->blocked_calls);
 
-        inode_ctx_put (inode, this, (uint64_t)(long)(pl_inode));
+                __inode_ctx_put (inode, this, (uint64_t)(long)(pl_inode));
+        }
+unlock:
+        UNLOCK (&inode->lock);
 
-out:
         return pl_inode;
 }
 
 
 /* Create a new posix_lock_t */
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
+new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
                 gf_lkowner_t *owner, fd_t *fd)
 {
         posix_lock_t *lock = NULL;
 
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, flock, out);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, transport, out);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, fd, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", flock, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", client, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", fd, out);
 
         lock = GF_CALLOC (1, sizeof (posix_lock_t),
                           gf_locks_mt_posix_lock_t);
@@ -490,7 +486,7 @@ new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
         else
                 lock->fl_end = flock->l_start + flock->l_len - 1;
 
-        lock->transport  = transport;
+        lock->client     = client;
         lock->fd_num     = fd_to_fdnum (fd);
         lock->fd         = fd;
         lock->client_pid = client_pid;
@@ -570,7 +566,7 @@ same_owner (posix_lock_t *l1, posix_lock_t *l2)
 {
 
         return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->transport  == l2->transport));
+                (l1->client == l2->client));
 
 }
 
@@ -699,7 +695,7 @@ subtract_locks (posix_lock_t *big, posix_lock_t *small)
         }
 
         GF_ASSERT (0);
-        gf_log (POSIX_LOCKS, GF_LOG_ERROR, "Unexpected case in subtract_locks");
+        gf_log ("posix-locks", GF_LOG_ERROR, "Unexpected case in subtract_locks");
 
 out:
         if (v.locks[0]) {
@@ -719,6 +715,36 @@ done:
         return v;
 }
 
+static posix_lock_t *
+first_conflicting_overlap (pl_inode_t *pl_inode, posix_lock_t *lock)
+{
+        posix_lock_t *l = NULL;
+        posix_lock_t *conf = NULL;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (l, &pl_inode->ext_list, list) {
+                        if (l->blocked)
+                                continue;
+
+                        if (locks_overlap (l, lock)) {
+                                if (same_owner (l, lock))
+                                        continue;
+
+                                if ((l->fl_type == F_WRLCK) ||
+                                    (lock->fl_type == F_WRLCK)) {
+                                        conf = l;
+                                        goto unlock;
+                                }
+                        }
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        return conf;
+}
+
 /*
   Start searching from {begin}, and return the first lock that
   conflicts, NULL if no conflict
@@ -777,6 +803,8 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
         struct _values v = { .locks = {0, 0, 0} };
 
         list_for_each_entry_safe (conf, t, &pl_inode->ext_list, list) {
+                if (conf->blocked)
+                        continue;
                 if (!locks_overlap (conf, lock))
                         continue;
 
@@ -785,7 +813,7 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 sum = add_locks (lock, conf);
 
                                 sum->fl_type    = lock->fl_type;
-                                sum->transport  = lock->transport;
+                                sum->client     = lock->client;
                                 sum->fd_num     = lock->fd_num;
                                 sum->client_pid = lock->client_pid;
                                 sum->owner      = lock->owner;
@@ -794,6 +822,8 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 __destroy_lock (conf);
 
                                 __destroy_lock (lock);
+                                INIT_LIST_HEAD (&sum->list);
+                                posix_lock_to_flock (sum, &sum->user_flock);
                                 __insert_and_merge (pl_inode, sum);
 
                                 return;
@@ -801,7 +831,7 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 sum = add_locks (lock, conf);
 
                                 sum->fl_type    = conf->fl_type;
-                                sum->transport  = conf->transport;
+                                sum->client     = conf->client;
                                 sum->fd_num     = conf->fd_num;
                                 sum->client_pid = conf->client_pid;
                                 sum->owner      = conf->owner;
@@ -821,6 +851,8 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                                 continue;
 
                                         INIT_LIST_HEAD (&v.locks[i]->list);
+                                        posix_lock_to_flock (v.locks[i],
+                                                       &v.locks[i]->user_flock);
                                         __insert_and_merge (pl_inode,
                                                             v.locks[i]);
                                 }
@@ -927,7 +959,8 @@ grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode)
                 pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
                               &lock->user_flock, 0, 0, NULL);
 
-                STACK_UNWIND (lock->frame, 0, 0, &lock->user_flock);
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
+                                     &lock->user_flock, NULL);
 
                 GF_FREE (lock);
         }
@@ -956,7 +989,7 @@ pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
         flock.l_len    = old_lock->user_flock.l_len;
 
 
-        unlock_lock = new_posix_lock (&flock, old_lock->transport,
+        unlock_lock = new_posix_lock (&flock, old_lock->client,
                                       old_lock->client_pid, &old_lock->owner,
                                       old_lock->fd);
         GF_VALIDATE_OR_GOTO (this->name, unlock_lock, out);
@@ -972,7 +1005,8 @@ pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
                 pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
                               &lock->user_flock, 0, 0, NULL);
 
-                STACK_UNWIND (lock->frame, 0, 0, &lock->user_flock);
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
+                                     &lock->user_flock, NULL);
 
                 GF_FREE (lock);
         }
@@ -1055,7 +1089,7 @@ pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
         posix_lock_t *conf = NULL;
 
-        conf = first_overlap (pl_inode, lock);
+        conf = first_conflicting_overlap (pl_inode, lock);
 
         if (conf == NULL) {
                 lock->fl_type = F_UNLCK;
@@ -1064,3 +1098,124 @@ pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock)
 
         return conf;
 }
+
+
+struct _lock_table *
+pl_lock_table_new (void)
+{
+        struct _lock_table *new = NULL;
+
+        new = GF_CALLOC (1, sizeof (struct _lock_table), gf_common_mt_lock_table);
+        if (new == NULL) {
+                goto out;
+        }
+        INIT_LIST_HEAD (&new->entrylk_lockers);
+        INIT_LIST_HEAD (&new->inodelk_lockers);
+        LOCK_INIT (&new->lock);
+out:
+        return new;
+}
+
+
+int
+pl_add_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc, fd_t *fd, pid_t pid, gf_lkowner_t *owner,
+               glusterfs_fop_t type)
+{
+        int32_t         ret = -1;
+        struct _locker *new = NULL;
+
+        GF_VALIDATE_OR_GOTO ("lock-table", table, out);
+        GF_VALIDATE_OR_GOTO ("lock-table", volume, out);
+
+        new = GF_CALLOC (1, sizeof (struct _locker), gf_common_mt_locker);
+        if (new == NULL) {
+                goto out;
+        }
+        INIT_LIST_HEAD (&new->lockers);
+
+        new->volume = gf_strdup (volume);
+
+        if (fd == NULL) {
+                loc_copy (&new->loc, loc);
+        } else {
+                new->fd = fd_ref (fd);
+        }
+
+        new->pid   = pid;
+        new->owner = *owner;
+
+        LOCK (&table->lock);
+        {
+                if (type == GF_FOP_ENTRYLK)
+                        list_add_tail (&new->lockers, &table->entrylk_lockers);
+                else
+                        list_add_tail (&new->lockers, &table->inodelk_lockers);
+        }
+        UNLOCK (&table->lock);
+out:
+        return ret;
+}
+
+int
+pl_del_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc, fd_t *fd, gf_lkowner_t *owner, glusterfs_fop_t type)
+{
+        struct _locker    *locker = NULL;
+        struct _locker    *tmp = NULL;
+        int32_t            ret = -1;
+        struct list_head  *head = NULL;
+        struct list_head   del;
+
+        GF_VALIDATE_OR_GOTO ("lock-table", table, out);
+        GF_VALIDATE_OR_GOTO ("lock-table", volume, out);
+
+        INIT_LIST_HEAD (&del);
+
+        LOCK (&table->lock);
+        {
+                if (type == GF_FOP_ENTRYLK) {
+                        head = &table->entrylk_lockers;
+                } else {
+                        head = &table->inodelk_lockers;
+                }
+
+                list_for_each_entry_safe (locker, tmp, head, lockers) {
+                        if (!is_same_lkowner (&locker->owner, owner) ||
+                            strcmp (locker->volume, volume))
+                                continue;
+
+                        /*
+                         * It is possible for inodelk lock to come on anon-fd
+                         * and inodelk unlock to come on normal fd in case of
+                         * client re-opens. So don't check for fds to be equal.
+                         */
+                        if (locker->fd && fd)
+                                list_move_tail (&locker->lockers, &del);
+                        else if (locker->loc.inode && loc &&
+                                 (locker->loc.inode == loc->inode))
+                                list_move_tail (&locker->lockers, &del);
+                }
+        }
+        UNLOCK (&table->lock);
+
+        tmp = NULL;
+        locker = NULL;
+
+        list_for_each_entry_safe (locker, tmp, &del, lockers) {
+                list_del_init (&locker->lockers);
+                if (locker->fd)
+                        fd_unref (locker->fd);
+                else
+                        loc_wipe (&locker->loc);
+
+                GF_FREE (locker->volume);
+                GF_FREE (locker);
+        }
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
diff --git a/xlators/features/locks/src/common.h b/xlators/features/locks/src/common.h
index 0c0dd22e7..db19ec978 100644
--- a/xlators/features/locks/src/common.h
+++ b/xlators/features/locks/src/common.h
@@ -1,30 +1,54 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __COMMON_H__
 #define __COMMON_H__
 
 #include "lkowner.h"
+/*dump locks format strings */
+#define RANGE_FMT               "type=%s, whence=%hd, start=%llu, len=%llu"
+#define ENTRY_FMT               "type=%s on basename=%s"
+#define DUMP_GEN_FMT            "pid = %llu, owner=%s, client=%p"
+#define GRNTD_AT                "granted at %s"
+#define BLKD_AT                 "blocked at %s"
+#define CONN_ID                 "connection-id=%s"
+#define DUMP_BLKD_FMT           DUMP_GEN_FMT", "CONN_ID", "BLKD_AT
+#define DUMP_GRNTD_FMT          DUMP_GEN_FMT", "CONN_ID", "GRNTD_AT
+#define DUMP_BLKD_GRNTD_FMT     DUMP_GEN_FMT", "CONN_ID", "BLKD_AT", "GRNTD_AT
+
+#define ENTRY_BLKD_FMT          ENTRY_FMT", "DUMP_BLKD_FMT
+#define ENTRY_GRNTD_FMT         ENTRY_FMT", "DUMP_GRNTD_FMT
+#define ENTRY_BLKD_GRNTD_FMT    ENTRY_FMT", "DUMP_BLKD_GRNTD_FMT
+
+#define RANGE_BLKD_FMT          RANGE_FMT", "DUMP_BLKD_FMT
+#define RANGE_GRNTD_FMT         RANGE_FMT", "DUMP_GRNTD_FMT
+#define RANGE_BLKD_GRNTD_FMT    RANGE_FMT", "DUMP_BLKD_GRNTD_FMT
 
 #define SET_FLOCK_PID(flock, lock) ((flock)->l_pid = lock->client_pid)
+
+struct _locker {
+        struct list_head  lockers;
+        char             *volume;
+        loc_t             loc;
+        fd_t             *fd;
+        gf_lkowner_t      owner;
+        pid_t             pid;
+};
+
+struct _lock_table {
+        struct list_head  inodelk_lockers;
+        struct list_head  entrylk_lockers;
+        gf_lock_t         lock;
+};
+
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
+new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
                 gf_lkowner_t *owner, fd_t *fd);
 
 pl_inode_t *
@@ -57,13 +81,14 @@ pl_dom_list_t *
 get_domain (pl_inode_t *pl_inode, const char *volume);
 
 void
-grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom);
+grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
+                           pl_dom_list_t *dom);
 
 void
 __delete_inode_lock (pl_inode_lock_t *lock);
 
 void
-__destroy_inode_lock (pl_inode_lock_t *lock);
+__pl_inodelk_unref (pl_inode_lock_t *lock);
 
 void
 grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
@@ -72,9 +97,13 @@ grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
 void pl_update_refkeeper (xlator_t *this, inode_t *inode);
 
 int32_t
-get_inodelk_count (xlator_t *this, inode_t *inode);
+__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname);
+int32_t
+get_inodelk_count (xlator_t *this, inode_t *inode, char *domname);
 
 int32_t
+__get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode);
+int32_t
 get_entrylk_count (xlator_t *this, inode_t *inode);
 
 void pl_trace_in (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
@@ -133,4 +162,26 @@ pl_verify_reservelk (xlator_t *this, pl_inode_t *pl_inode,
                      posix_lock_t *lock, int can_block);
 int
 pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *reqlock);
+
+uint32_t
+check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename);
+
+int32_t
+pl_add_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc,
+               fd_t *fd,
+               pid_t pid,
+               gf_lkowner_t *owner,
+               glusterfs_fop_t type);
+
+int32_t
+pl_del_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc,
+               fd_t *fd,
+               gf_lkowner_t *owner,
+               glusterfs_fop_t type);
+
+struct _lock_table *
+pl_lock_table_new (void);
+
 #endif /* __COMMON_H__ */
diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c
index 3e91b9d94..0785dc547 100644
--- a/xlators/features/locks/src/entrylk.c
+++ b/xlators/features/locks/src/entrylk.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -35,7 +25,7 @@
 
 static pl_entry_lock_t *
 new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
-                  void *trans, pid_t client_pid, gf_lkowner_t *owner,
+                  client_t *client, pid_t client_pid, gf_lkowner_t *owner,
                   const char *volume)
 
 {
@@ -49,7 +39,7 @@ new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
 
         newlock->basename   = basename ? gf_strdup (basename) : NULL;
         newlock->type       = type;
-        newlock->trans      = trans;
+        newlock->trans      = client;
         newlock->volume     = volume;
         newlock->client_pid = client_pid;
         newlock->owner      = *owner;
@@ -315,18 +305,15 @@ __find_most_matching_lock (pl_dom_list_t *dom, const char *basename)
 
 int
 __lock_name (pl_inode_t *pinode, const char *basename, entrylk_type type,
-             call_frame_t *frame, pl_dom_list_t *dom, xlator_t *this, int nonblock)
+             call_frame_t *frame, pl_dom_list_t *dom, xlator_t *this,
+             int nonblock, char *conn_id)
 {
         pl_entry_lock_t *lock       = NULL;
         pl_entry_lock_t *conf       = NULL;
-        void            *trans      = NULL;
-        pid_t            client_pid = 0;
         int              ret        = -EINVAL;
 
-        trans = frame->root->trans;
-        client_pid = frame->root->pid;
-
-        lock = new_entrylk_lock (pinode, basename, type, trans, client_pid,
+        lock = new_entrylk_lock (pinode, basename, type,
+                                 frame->root->client, frame->root->pid,
                                  &frame->root->lk_owner, dom->domain);
         if (!lock) {
                 ret = -ENOMEM;
@@ -335,14 +322,18 @@ __lock_name (pl_inode_t *pinode, const char *basename, entrylk_type type,
 
         lock->frame   = frame;
         lock->this    = this;
-        lock->trans   = trans;
+        lock->trans   = frame->root->client;
+
+        if (conn_id) {
+                lock->connection_id = gf_strdup (conn_id);
+        }
 
         conf = __lock_grantable (dom, basename, type);
         if (conf) {
                 ret = -EAGAIN;
                 if (nonblock){
-                        if (lock->basename)
-                                GF_FREE ((char *)lock->basename);
+                        GF_FREE (lock->connection_id);
+                        GF_FREE ((char *)lock->basename);
                         GF_FREE (lock);
                         goto out;
 
@@ -361,8 +352,8 @@ __lock_name (pl_inode_t *pinode, const char *basename, entrylk_type type,
         if ( __blocked_lock_conflict (dom, basename, type) && !(__owner_has_lock (dom, lock))) {
                 ret = -EAGAIN;
                 if (nonblock) {
-                        if (lock->basename)
-                                GF_FREE ((char *) lock->basename);
+                        GF_FREE (lock->connection_id);
+                        GF_FREE ((char *) lock->basename);
                         GF_FREE (lock);
                         goto out;
 
@@ -441,6 +432,32 @@ out:
         return ret_lock;
 }
 
+uint32_t
+check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename)
+{
+        uint32_t        entrylk = 0;
+        pl_inode_t      *pinode = 0;
+        pl_dom_list_t   *dom = NULL;
+        pl_entry_lock_t *conf       = NULL;
+
+        pinode = pl_inode_get (this, parent);
+        if (!pinode)
+                goto out;
+        pthread_mutex_lock (&pinode->mutex);
+        {
+                list_for_each_entry (dom, &pinode->dom_list, inode_list) {
+                        conf = __lock_grantable (dom, basename, ENTRYLK_WRLCK);
+                        if (conf && conf->basename) {
+                                entrylk = 1;
+                                break;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pinode->mutex);
+
+out:
+        return entrylk;
+}
 
 void
 __grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
@@ -466,15 +483,16 @@ __grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
                         pl_inode, bl->basename);
 
                 bl_ret = __lock_name (pl_inode, bl->basename, bl->type,
-                                      bl->frame, dom, bl->this, 0);
+                                      bl->frame, dom, bl->this, 0,
+                                      bl->connection_id);
 
                 if (bl_ret == 0) {
                         list_add (&bl->blocked_locks, granted);
                 } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "should never happen");
-                        if (bl->basename)
-                                GF_FREE ((char *)bl->basename);
+                        GF_FREE (bl->connection_id);
+                        GF_FREE ((char *)bl->basename);
                         GF_FREE (bl);
                 }
         }
@@ -494,7 +512,8 @@ grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
 
         pthread_mutex_lock (&pl_inode->mutex);
         {
-                __grant_blocked_entry_locks (this, pl_inode, dom, &granted_list);
+                __grant_blocked_entry_locks (this, pl_inode, dom,
+                                             &granted_list);
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
@@ -505,24 +524,28 @@ grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
                                    lock->basename, ENTRYLK_LOCK, lock->type,
                                    0, 0);
 
-                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0);
+                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0, NULL);
 
+                GF_FREE (lock->connection_id);
+                GF_FREE ((char *)lock->basename);
+                GF_FREE (lock);
         }
 
         GF_FREE ((char *)unlocked->basename);
+        GF_FREE (unlocked->connection_id);
         GF_FREE (unlocked);
 
         return;
 }
 
 /**
- * release_entry_locks_for_transport: release all entry locks from this
- * transport for this loc_t
+ * release_entry_locks_for_client: release all entry locks from this
+ * client for this loc_t
  */
 
 static int
-release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
-                                   pl_dom_list_t *dom, void *trans)
+release_entry_locks_for_client (xlator_t *this, pl_inode_t *pinode,
+                                pl_dom_list_t *dom, client_t *client)
 {
         pl_entry_lock_t  *lock = NULL;
         pl_entry_lock_t  *tmp = NULL;
@@ -536,14 +559,14 @@ release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
         {
                 list_for_each_entry_safe (lock, tmp, &dom->blocked_entrylks,
                                           blocked_locks) {
-                        if (lock->trans != trans)
+                        if (lock->trans != client)
                                 continue;
 
                         list_del_init (&lock->blocked_locks);
 
                         gf_log (this->name, GF_LOG_TRACE,
                                 "releasing lock on  held by "
-                                "{transport=%p}",trans);
+                                "{client=%p}", client);
 
                         list_add (&lock->blocked_locks, &released);
 
@@ -551,16 +574,17 @@ release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
 
                 list_for_each_entry_safe (lock, tmp, &dom->entrylk_list,
                                           domain_list) {
-                        if (lock->trans != trans)
+                        if (lock->trans != client)
                                 continue;
 
                         list_del_init (&lock->domain_list);
 
                         gf_log (this->name, GF_LOG_TRACE,
                                 "releasing lock on  held by "
-                                "{transport=%p}",trans);
+                                "{client=%p}", client);
 
                         GF_FREE ((char *)lock->basename);
+                        GF_FREE (lock->connection_id);
                         GF_FREE (lock);
                 }
 
@@ -573,10 +597,10 @@ release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
         list_for_each_entry_safe (lock, tmp, &released, blocked_locks) {
                 list_del_init (&lock->blocked_locks);
 
-                STACK_UNWIND_STRICT (entrylk, lock->frame, -1, EAGAIN);
+                STACK_UNWIND_STRICT (entrylk, lock->frame, -1, EAGAIN, NULL);
 
-                if (lock->basename)
-                        GF_FREE ((char *)lock->basename);
+                GF_FREE ((char *)lock->basename);
+                GF_FREE (lock->connection_id);
                 GF_FREE (lock);
 
         }
@@ -584,10 +608,10 @@ release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
         list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
                 list_del_init (&lock->blocked_locks);
 
-                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0);
+                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0, NULL);
 
-                if (lock->basename)
-                        GF_FREE ((char *)lock->basename);
+                GF_FREE ((char *)lock->basename);
+                GF_FREE (lock->connection_id);
                 GF_FREE (lock);
         }
 
@@ -598,19 +622,23 @@ release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
 int
 pl_common_entrylk (call_frame_t *frame, xlator_t *this,
                    const char *volume, inode_t *inode, const char *basename,
-                   entrylk_cmd cmd, entrylk_type type, loc_t *loc, fd_t *fd)
-{
-        int32_t  op_ret   = -1;
-        int32_t  op_errno = 0;
+                   entrylk_cmd cmd, entrylk_type type, loc_t *loc, fd_t *fd,
+                   dict_t *xdata)
 
-        void *        transport = NULL;
-
-        pl_inode_t *     pinode   = NULL;
+{
+        int32_t          op_ret   = -1;
+        int32_t          op_errno = 0;
         int              ret      = -1;
-        pl_entry_lock_t *unlocked = NULL;
         char             unwind   = 1;
+        GF_UNUSED int    dict_ret = -1;
+        pl_inode_t      *pinode   = NULL;
+        pl_entry_lock_t *unlocked = NULL;
+        pl_dom_list_t   *dom      = NULL;
+        char            *conn_id  = NULL;
+        pl_ctx_t        *ctx      = NULL;
 
-        pl_dom_list_t          *dom = NULL;
+        if (xdata)
+                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
 
         pinode = pl_inode_get (this, inode);
         if (!pinode) {
@@ -626,18 +654,17 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
 
         entrylk_trace_in (this, frame, volume, fd, loc, basename, cmd, type);
 
-        transport = frame->root->trans;
-
         if (frame->root->lk_owner.len == 0) {
                 /*
                   this is a special case that means release
-                  all locks from this transport
+                  all locks from this client
                 */
 
                 gf_log (this->name, GF_LOG_TRACE,
-                        "Releasing locks for transport %p", transport);
+                        "Releasing locks for client %p", frame->root->client);
 
-                release_entry_locks_for_transport (this, pinode, dom, transport);
+                release_entry_locks_for_client (this, pinode, dom,
+                                                frame->root->client);
                 op_ret = 0;
 
                 goto out;
@@ -648,7 +675,7 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
                 pthread_mutex_lock (&pinode->mutex);
                 {
                         ret = __lock_name (pinode, basename, type,
-                                           frame, dom, this, 0);
+                                           frame, dom, this, 0, conn_id);
                 }
                 pthread_mutex_unlock (&pinode->mutex);
 
@@ -673,7 +700,7 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
                 pthread_mutex_lock (&pinode->mutex);
                 {
                         ret = __lock_name (pinode, basename, type,
-                                           frame, dom, this, 1);
+                                           frame, dom, this, 1, conn_id);
                 }
                 pthread_mutex_unlock (&pinode->mutex);
 
@@ -710,7 +737,25 @@ out:
                 entrylk_trace_out (this, frame, volume, fd, loc, basename,
                                    cmd, type, op_ret, op_errno);
 
-                STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+                ctx = pl_ctx_get (frame->root->client, this);
+
+                if (ctx == NULL) {
+                        gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+                        goto unwind;
+                }
+
+                if (cmd == ENTRYLK_UNLOCK)
+                        pl_del_locker (ctx->ltable, volume, loc, fd,
+                                       &frame->root->lk_owner,
+                                       GF_FOP_ENTRYLK);
+                else
+                        pl_add_locker (ctx->ltable, volume, loc, fd,
+                                       frame->root->pid,
+                                       &frame->root->lk_owner,
+                                       GF_FOP_ENTRYLK);
+
+unwind:
+                STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, NULL);
         } else {
                 entrylk_trace_block (this, frame, volume, fd, loc, basename,
                                      cmd, type);
@@ -729,10 +774,10 @@ out:
 int
 pl_entrylk (call_frame_t *frame, xlator_t *this,
             const char *volume, loc_t *loc, const char *basename,
-            entrylk_cmd cmd, entrylk_type type)
+            entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-
-        pl_common_entrylk (frame, this, volume, loc->inode, basename, cmd, type, loc, NULL);
+        pl_common_entrylk (frame, this, volume, loc->inode, basename, cmd,
+                           type, loc, NULL, xdata);
 
         return 0;
 }
@@ -747,16 +792,16 @@ pl_entrylk (call_frame_t *frame, xlator_t *this,
 int
 pl_fentrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, fd_t *fd, const char *basename,
-             entrylk_cmd cmd, entrylk_type type)
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-
-        pl_common_entrylk (frame, this, volume, fd->inode, basename, cmd, type, NULL, fd);
+        pl_common_entrylk (frame, this, volume, fd->inode, basename, cmd,
+                           type, NULL, fd, xdata);
 
         return 0;
 }
 
 
-static int32_t
+int32_t
 __get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode)
 {
         int32_t            count = 0;
@@ -765,24 +810,10 @@ __get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode)
 
         list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
                 list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                " domain: %s  %s on %s state = Active",
-                                dom->domain,
-                                lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
-                                "ENTRYLK_WRLCK", lock->basename);
                         count++;
                 }
 
                 list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                " domain: %s  %s on %s state = Blocked",
-                                dom->domain,
-                                lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
-                                "ENTRYLK_WRLCK", lock->basename);
                         count++;
                 }
 
diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c
index 5785f1f8e..508523e11 100644
--- a/xlators/features/locks/src/inodelk.c
+++ b/xlators/features/locks/src/inodelk.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -33,20 +23,30 @@
 #include "locks.h"
 #include "common.h"
 
-void
+inline void
 __delete_inode_lock (pl_inode_lock_t *lock)
 {
         list_del (&lock->list);
 }
 
-void
-__destroy_inode_lock (pl_inode_lock_t *lock)
+static inline void
+__pl_inodelk_ref (pl_inode_lock_t *lock)
+{
+        lock->ref++;
+}
+
+inline void
+__pl_inodelk_unref (pl_inode_lock_t *lock)
 {
-        GF_FREE (lock);
+        lock->ref--;
+        if (!lock->ref) {
+                GF_FREE (lock->connection_id);
+                GF_FREE (lock);
+        }
 }
 
 /* Check if 2 inodelks are conflicting on type. Only 2 shared locks don't conflict */
-static int
+static inline int
 inodelk_type_conflict (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
         if (l2->fl_type == F_WRLCK || l1->fl_type == F_WRLCK)
@@ -124,7 +124,7 @@ static inline int
 same_inodelk_owner (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
         return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->transport  == l2->transport));
+                (l1->client == l2->client));
 }
 
 /* Returns true if the 2 inodelks conflict with each other */
@@ -245,6 +245,7 @@ __lock_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
 
                 goto out;
         }
+        __pl_inodelk_ref (lock);
         gettimeofday (&lock->granted_time, NULL);
         list_add (&lock->list, &dom->inodelk_list);
 
@@ -289,20 +290,22 @@ __inode_unlock_lock (xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
 
         conf = find_matching_inodelk (lock, dom);
         if (!conf) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " Matching lock not found for unlock");
+                gf_log (this->name, GF_LOG_ERROR,
+                        " Matching lock not found for unlock %llu-%llu, by %s "
+                        "on %p", (unsigned long long)lock->fl_start,
+                        (unsigned long long)lock->fl_end,
+                        lkowner_utoa (&lock->owner), lock->client);
                 goto out;
         }
         __delete_inode_lock (conf);
         gf_log (this->name, GF_LOG_DEBUG,
-                " Matching lock found for unlock");
-        __destroy_inode_lock (lock);
-
+                " Matching lock found for unlock %llu-%llu, by %s on %p",
+                (unsigned long long)lock->fl_start,
+                (unsigned long long)lock->fl_end, lkowner_utoa (&lock->owner),
+                lock->client);
 
 out:
         return conf;
-
-
 }
 static void
 __grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
@@ -332,7 +335,8 @@ __grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
 
 /* Grant all inodelks blocked on a lock */
 void
-grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom)
+grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
+                           pl_dom_list_t *dom)
 {
         struct list_head granted;
         pl_inode_lock_t *lock;
@@ -340,11 +344,6 @@ grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *
 
         INIT_LIST_HEAD (&granted);
 
-        if (list_empty (&dom->blocked_inodelks)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No blocked locks to be granted for domain: %s", dom->domain);
-        }
-
         pthread_mutex_lock (&pl_inode->mutex);
         {
                 __grant_blocked_inode_locks (this, pl_inode, &granted, dom);
@@ -363,28 +362,34 @@ grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *
                 pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
                               &lock->user_flock, 0, 0, lock->volume);
 
-                STACK_UNWIND_STRICT (inodelk, lock->frame, 0, 0);
+                STACK_UNWIND_STRICT (inodelk, lock->frame, 0, 0, NULL);
         }
 
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
+                        list_del_init (&lock->blocked_locks);
+                        __pl_inodelk_unref (lock);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
 }
 
-/* Release all inodelks from this transport */
+/* Release all inodelks from this client */
 static int
-release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
-                                  inode_t *inode, void *trans)
+release_inode_locks_of_client (xlator_t *this, pl_dom_list_t *dom,
+                                  inode_t *inode, client_t *client)
 {
         pl_inode_lock_t *tmp = NULL;
         pl_inode_lock_t *l = NULL;
 
         pl_inode_t * pinode = NULL;
 
-        struct list_head granted;
         struct list_head released;
 
         char *path = NULL;
         char *file = NULL;
 
-        INIT_LIST_HEAD (&granted);
         INIT_LIST_HEAD (&released);
 
         pinode = pl_inode_get (this, inode);
@@ -393,7 +398,7 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
         {
 
                 list_for_each_entry_safe (l, tmp, &dom->blocked_inodelks, blocked_locks) {
-                        if (l->transport != trans)
+                        if (l->client != client)
                                 continue;
 
                         list_del_init (&l->blocked_locks);
@@ -406,8 +411,8 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
 
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "releasing blocking lock on %s held by "
-                                "{transport=%p, pid=%"PRId64" lk-owner=%s}",
-                                file, trans, (uint64_t) l->client_pid,
+                                "{client=%p, pid=%"PRId64" lk-owner=%s}",
+                                file, client, (uint64_t) l->client_pid,
                                 lkowner_utoa (&l->owner));
 
                         list_add (&l->blocked_locks, &released);
@@ -418,12 +423,9 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
                 }
 
                 list_for_each_entry_safe (l, tmp, &dom->inodelk_list, list) {
-                        if (l->transport != trans)
+                        if (l->client != client)
                                 continue;
 
-                        __delete_inode_lock (l);
-                        __destroy_inode_lock (l);
-
                         inode_path (inode, NULL, &path);
                         if (path)
                                 file = path;
@@ -432,25 +434,29 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
 
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "releasing granted lock on %s held by "
-                                "{transport=%p, pid=%"PRId64" lk-owner=%s}",
-                                file, trans, (uint64_t) l->client_pid,
+                                "{client=%p, pid=%"PRId64" lk-owner=%s}",
+                                file, client, (uint64_t) l->client_pid,
                                 lkowner_utoa (&l->owner));
+
                         if (path) {
                                 GF_FREE (path);
                                 path = NULL;
                         }
+
+                        __delete_inode_lock (l);
+                        __pl_inodelk_unref (l);
                 }
         }
-        if (path)
-                GF_FREE (path);
+        GF_FREE (path);
 
         pthread_mutex_unlock (&pinode->mutex);
 
         list_for_each_entry_safe (l, tmp, &released, blocked_locks) {
                 list_del_init (&l->blocked_locks);
 
-                STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN);
-                GF_FREE (l);
+                STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN, NULL);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (l);
         }
 
         grant_blocked_inode_locks (this, pinode, dom);
@@ -464,12 +470,13 @@ pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
 {
         int ret = -EINVAL;
         pl_inode_lock_t *retlock = NULL;
+        gf_boolean_t    unref = _gf_true;
 
         pthread_mutex_lock (&pl_inode->mutex);
         {
                 if (lock->fl_type != F_UNLCK) {
                         ret = __lock_inodelk (this, pl_inode, lock, can_block, dom);
-                        if (ret == 0)
+                        if (ret == 0) {
                                 gf_log (this->name, GF_LOG_TRACE,
                                         "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
                                         lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
@@ -477,8 +484,7 @@ pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
                                         lkowner_utoa (&lock->owner),
                                         lock->fl_start,
                                         lock->fl_end);
-
-                        if (ret == -EAGAIN)
+                        } else if (ret == -EAGAIN) {
                                 gf_log (this->name, GF_LOG_TRACE,
                                         "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => NOK",
                                         lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
@@ -486,25 +492,25 @@ pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
                                         lkowner_utoa (&lock->owner),
                                         lock->user_flock.l_start,
                                         lock->user_flock.l_len);
+                                if (can_block)
+                                        unref = _gf_false;
+                        }
+                } else {
+                        retlock = __inode_unlock_lock (this, lock, dom);
+                        if (!retlock) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "Bad Unlock issued on Inode lock");
+                                ret = -EINVAL;
+                                goto out;
+                        }
+                        __pl_inodelk_unref (retlock);
 
-                        goto out;
-                }
-
-
-                retlock = __inode_unlock_lock (this, lock, dom);
-                if (!retlock) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Bad Unlock issued on Inode lock");
-                        ret = -EINVAL;
-                        goto out;
+                        ret = 0;
                 }
-                __destroy_inode_lock (retlock);
-
-                ret = 0;
-
-
         }
 out:
+        if (unref)
+                __pl_inodelk_unref (lock);
         pthread_mutex_unlock (&pl_inode->mutex);
         grant_blocked_inode_locks (this, pl_inode, dom);
         return ret;
@@ -512,8 +518,9 @@ out:
 
 /* Create a new inode_lock_t */
 pl_inode_lock_t *
-new_inode_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
-                gf_lkowner_t *owner, const char *volume)
+new_inode_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
+                call_frame_t *frame, xlator_t *this, const char *volume,
+                char *conn_id)
 
 {
         pl_inode_lock_t *lock = NULL;
@@ -532,32 +539,76 @@ new_inode_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
         else
                 lock->fl_end = flock->l_start + flock->l_len - 1;
 
-        lock->transport  = transport;
+        lock->client     = client;
         lock->client_pid = client_pid;
         lock->volume     = volume;
-        lock->owner      = *owner;
+        lock->owner      = frame->root->lk_owner;
+        lock->frame      = frame;
+        lock->this       = this;
+
+        if (conn_id) {
+                lock->connection_id = gf_strdup (conn_id);
+        }
 
         INIT_LIST_HEAD (&lock->list);
         INIT_LIST_HEAD (&lock->blocked_locks);
+        __pl_inodelk_ref (lock);
 
         return lock;
 }
 
+int32_t
+_pl_convert_volume (const char *volume, char **res)
+{
+        char    *mdata_vol = NULL;
+        int     ret = 0;
+
+        mdata_vol = strrchr (volume, ':');
+        //if the volume already ends with :metadata don't bother
+        if (mdata_vol && (strcmp (mdata_vol, ":metadata") == 0))
+                return 0;
+
+        ret = gf_asprintf (res, "%s:metadata", volume);
+        if (ret <= 0)
+                return ENOMEM;
+        return 0;
+}
+
+int32_t
+_pl_convert_volume_for_special_range (struct gf_flock *flock,
+                                      const char *volume, char **res)
+{
+        int32_t     ret = 0;
+
+        if ((flock->l_start == LLONG_MAX -1) &&
+            (flock->l_len == 0)) {
+                ret = _pl_convert_volume (volume, res);
+        }
+
+        return ret;
+}
+
 /* Common inodelk code called from pl_inodelk and pl_finodelk */
 int
 pl_common_inodelk (call_frame_t *frame, xlator_t *this,
                    const char *volume, inode_t *inode, int32_t cmd,
-                   struct gf_flock *flock, loc_t *loc, fd_t *fd)
+                   struct gf_flock *flock, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         int32_t           op_ret     = -1;
         int32_t           op_errno   = 0;
         int               ret        = -1;
+        GF_UNUSED int     dict_ret   = -1;
         int               can_block  = 0;
-        pid_t             client_pid = -1;
-        void *            transport  = NULL;
         pl_inode_t *      pinode     = NULL;
         pl_inode_lock_t * reqlock    = NULL;
         pl_dom_list_t *   dom        = NULL;
+        char             *res        = NULL;
+        char             *res1       = NULL;
+        char             *conn_id    = NULL;
+        pl_ctx_t         *ctx        = NULL;
+
+        if (xdata)
+                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (inode, unwind);
@@ -568,10 +619,13 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        pl_trace_in (this, frame, fd, loc, cmd, flock, volume);
+        op_errno = _pl_convert_volume_for_special_range (flock, volume, &res);
+        if (op_errno)
+                goto unwind;
+        if (res)
+                volume = res;
 
-        transport  = frame->root->trans;
-        client_pid = frame->root->pid;
+        pl_trace_in (this, frame, fd, loc, cmd, flock, volume);
 
         pinode = pl_inode_get (this, inode);
         if (!pinode) {
@@ -580,23 +634,34 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
         }
 
         dom = get_domain (pinode, volume);
+        if (!dom) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
 
         if (frame->root->lk_owner.len == 0) {
                 /*
                   special case: this means release all locks
-                  from this transport
+                  from this client
                 */
                 gf_log (this->name, GF_LOG_TRACE,
-                        "Releasing all locks from transport %p", transport);
-
-                release_inode_locks_of_transport (this, dom, inode, transport);
+                        "Releasing all locks from client %p", frame->root->client);
+
+                release_inode_locks_of_client (this, dom, inode, frame->root->client);
+                _pl_convert_volume (volume, &res1);
+                if (res1) {
+                        dom = get_domain (pinode, res1);
+                        if (dom)
+                                release_inode_locks_of_client (this, dom,
+                                                        inode, frame->root->client);
+                }
 
                 op_ret = 0;
                 goto unwind;
         }
 
-        reqlock = new_inode_lock (flock, transport, client_pid,
-                                  &frame->root->lk_owner, volume);
+        reqlock = new_inode_lock (flock, frame->root->client, frame->root->pid,
+                                  frame, this, volume, conn_id);
 
         if (!reqlock) {
                 op_ret = -1;
@@ -604,14 +669,10 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        reqlock->frame = frame;
-        reqlock->this  = this;
 
         switch (cmd) {
         case F_SETLKW:
                 can_block = 1;
-                reqlock->frame = frame;
-                reqlock->this  = this;
 
                 /* fall through */
 
@@ -628,7 +689,6 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
                         }
                         gf_log (this->name, GF_LOG_TRACE, "returning EAGAIN");
                         op_errno = -ret;
-                        __destroy_inode_lock (reqlock);
                         goto unwind;
                 }
                 break;
@@ -644,86 +704,103 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
 
         op_ret = 0;
 
+        ctx = pl_ctx_get (frame->root->client, this);
+
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+                goto unwind;
+        }
+
+        if (flock->l_type == F_UNLCK)
+                pl_del_locker (ctx->ltable, volume, loc, fd,
+                               &frame->root->lk_owner,
+                               GF_FOP_INODELK);
+        else
+                pl_add_locker (ctx->ltable, volume, loc, fd,
+                               frame->root->pid,
+                               &frame->root->lk_owner,
+                               GF_FOP_INODELK);
+
 unwind:
         if ((inode != NULL) && (flock !=NULL)) {
                 pl_update_refkeeper (this, inode);
                 pl_trace_out (this, frame, fd, loc, cmd, flock, op_ret, op_errno, volume);
         }
 
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, NULL);
 out:
+        GF_FREE (res);
+        GF_FREE (res1);
         return 0;
 }
 
 int
 pl_inodelk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
+            dict_t *xdata)
 {
-
-        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock, loc, NULL);
+        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock,
+                           loc, NULL, xdata);
 
         return 0;
 }
 
 int
 pl_finodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+             dict_t *xdata)
 {
-
-        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock, NULL, fd);
+        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock,
+                           NULL, fd, xdata);
 
         return 0;
 
 }
 
+static inline int32_t
+__get_inodelk_dom_count (pl_dom_list_t *dom)
+{
+        pl_inode_lock_t     *lock   = NULL;
+        int32_t             count   = 0;
 
-static int32_t
-__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode)
+        list_for_each_entry (lock, &dom->inodelk_list, list) {
+                count++;
+        }
+        list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+                count++;
+        }
+        return count;
+}
+
+/* Returns the no. of locks (blocked/granted) held on a given domain name
+ * If @domname is NULL, returns the no. of locks in all the domains present.
+ * If @domname is non-NULL and non-existent, returns 0 */
+int32_t
+__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname)
 {
         int32_t            count  = 0;
-        pl_inode_lock_t   *lock   = NULL;
         pl_dom_list_t     *dom    = NULL;
 
         list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                list_for_each_entry (lock, &dom->inodelk_list, list) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                " domain: %s %s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" "
-                                "state = Active",
-                                dom->domain,
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len);
-
-                        count++;
-                }
+                if (domname) {
+                        if (strcmp (domname, dom->domain) == 0) {
+                                count = __get_inodelk_dom_count (dom);
+                                goto out;
+                        }
 
-                list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+                } else {
+                    /* Counting locks from all domains */
+                        count += __get_inodelk_dom_count (dom);
 
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                " domain: %s %s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" "
-                                "state = Blocked",
-                                dom->domain,
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len);
-
-                        count++;
                 }
-
         }
 
+out:
         return count;
 }
 
 int32_t
-get_inodelk_count (xlator_t *this, inode_t *inode)
+get_inodelk_count (xlator_t *this, inode_t *inode, char *domname)
 {
         pl_inode_t   *pl_inode = NULL;
         uint64_t      tmp_pl_inode = 0;
@@ -739,7 +816,7 @@ get_inodelk_count (xlator_t *this, inode_t *inode)
 
         pthread_mutex_lock (&pl_inode->mutex);
         {
-                count = __get_inodelk_count (this, pl_inode);
+                count = __get_inodelk_count (this, pl_inode, domname);
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
diff --git a/xlators/features/locks/src/locks-mem-types.h b/xlators/features/locks/src/locks-mem-types.h
index 9d44f0eba..08aeb0a79 100644
--- a/xlators/features/locks/src/locks-mem-types.h
+++ b/xlators/features/locks/src/locks-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __LOCKS_MEM_TYPES_H__
 #define __LOCKS_MEM_TYPES_H__
 
@@ -32,7 +22,6 @@ enum gf_locks_mem_types_ {
         gf_locks_mt_truncate_ops,
         gf_locks_mt_pl_rw_req_t,
         gf_locks_mt_posix_locks_private_t,
-        gf_locks_mt_pl_local_t,
         gf_locks_mt_pl_fdctx_t,
         gf_locks_mt_end
 };
diff --git a/xlators/features/locks/src/locks.h b/xlators/features/locks/src/locks.h
index 0dcbdf979..76fc941d7 100644
--- a/xlators/features/locks/src/locks.h
+++ b/xlators/features/locks/src/locks.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __POSIX_LOCKS_H__
 #define __POSIX_LOCKS_H__
 
@@ -29,10 +19,10 @@
 #include "stack.h"
 #include "call-stub.h"
 #include "locks-mem-types.h"
+#include "client_t.h"
 
 #include "lkowner.h"
 
-#define POSIX_LOCKS "posix-locks"
 struct __pl_fd;
 
 struct __posix_lock {
@@ -43,7 +33,7 @@ struct __posix_lock {
         off_t              fl_end;
 
         short              blocked;    /* waiting to acquire */
-        struct gf_flock       user_flock; /* the flock supplied by the user */
+        struct gf_flock    user_flock; /* the flock supplied by the user */
         xlator_t          *this;       /* required for blocked locks */
         unsigned long      fd_num;
 
@@ -56,7 +46,7 @@ struct __posix_lock {
         /* These two together serve to uniquely identify each process
            across nodes */
 
-        void              *transport;     /* to identify client node */
+        void              *client;     /* to identify client node */
         gf_lkowner_t       owner;
         pid_t              client_pid;    /* pid of client process */
 };
@@ -65,6 +55,7 @@ typedef struct __posix_lock posix_lock_t;
 struct __pl_inode_lock {
         struct list_head   list;
         struct list_head   blocked_locks; /* list_head pointing to blocked_inodelks */
+        int                ref;
 
         short              fl_type;
         off_t              fl_start;
@@ -72,7 +63,7 @@ struct __pl_inode_lock {
 
         const char        *volume;
 
-        struct gf_flock       user_flock; /* the flock supplied by the user */
+        struct gf_flock    user_flock; /* the flock supplied by the user */
         xlator_t          *this;       /* required for blocked locks */
         fd_t              *fd;
 
@@ -84,9 +75,11 @@ struct __pl_inode_lock {
         /* These two together serve to uniquely identify each process
            across nodes */
 
-        void              *transport;     /* to identify client node */
+        void              *client;     /* to identify client node */
         gf_lkowner_t       owner;
         pid_t              client_pid;    /* pid of client process */
+
+        char              *connection_id; /* stores the client connection id */
 };
 typedef struct __pl_inode_lock pl_inode_lock_t;
 
@@ -124,7 +117,9 @@ struct __entry_lock {
 
         void             *trans;
         gf_lkowner_t      owner;
-        pid_t             client_pid;    /* pid of client process */
+	pid_t             client_pid;    /* pid of client process */
+
+        char             *connection_id; /* stores the client connection id */
 };
 typedef struct __entry_lock pl_entry_lock_t;
 
@@ -158,16 +153,40 @@ typedef struct __pl_fd pl_fd_t;
 typedef struct {
         gf_boolean_t    mandatory;      /* if mandatory locking is enabled */
         gf_boolean_t    trace;          /* trace lock requests in and out */
+        char           *brickname;
 } posix_locks_private_t;
 
+
 typedef struct {
         gf_boolean_t   entrylk_count_req;
         gf_boolean_t   inodelk_count_req;
+        gf_boolean_t   inodelk_dom_count_req;
         gf_boolean_t   posixlk_count_req;
+        gf_boolean_t   parent_entrylk_req;
+
+        /* used by {f,}truncate */
+        loc_t  loc;
+        fd_t  *fd;
+        off_t  offset;
+        dict_t *xdata;
+        enum {TRUNCATE, FTRUNCATE} op;
 } pl_local_t;
 
+
 typedef struct {
         struct list_head locks_list;
 } pl_fdctx_t;
 
+
+typedef struct _locks_ctx {
+        gf_lock_t            ltable_lock; /* only for replace, 
+                                             ltable has its own internal 
+                                             lock for operations */
+        struct _lock_table  *ltable;
+} pl_ctx_t;
+
+
+pl_ctx_t *
+pl_ctx_get (client_t *client, xlator_t *xlator);
+
 #endif /* __POSIX_LOCKS_H__ */
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index bd30ca493..7bfb38a51 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -38,6 +28,8 @@
 #include "common.h"
 #include "statedump.h"
 #include "clear.h"
+#include "defaults.h"
+#include "syncop.h"
 
 #ifndef LLONG_MAX
 #define LLONG_MAX LONG_LONG_MAX /* compat with old gcc */
@@ -48,13 +40,9 @@
 
 void do_blocked_rw (pl_inode_t *);
 static int __rw_allowable (pl_inode_t *, posix_lock_t *, glusterfs_fop_t);
-
-struct _truncate_ops {
-        loc_t  loc;
-        fd_t  *fd;
-        off_t  offset;
-        enum {TRUNCATE, FTRUNCATE} op;
-};
+static int format_brickname(char *);
+int pl_lockinfo_get_brickname (xlator_t *, inode_t *, int32_t *);
+static int fetch_pathinfo(xlator_t *, inode_t *, int32_t *, char **);
 
 static pl_fdctx_t *
 pl_new_fdctx ()
@@ -63,7 +51,7 @@ pl_new_fdctx ()
 
         fdctx = GF_CALLOC (1, sizeof (*fdctx),
                            gf_locks_mt_pl_fdctx_t);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, fdctx, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", fdctx, out);
 
         INIT_LIST_HEAD (&fdctx->locks_list);
 
@@ -78,7 +66,7 @@ pl_check_n_create_fdctx (xlator_t *this, fd_t *fd)
         uint64_t    tmp   = 0;
         pl_fdctx_t *fdctx = NULL;
 
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, this, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", this, out);
         GF_VALIDATE_OR_GOTO (this->name, fd, out);
 
         LOCK (&fd->lock);
@@ -109,24 +97,29 @@ out:
 int
 pl_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf)
+                 struct iatt *postbuf, dict_t *xdata)
 {
-        struct _truncate_ops *local = NULL;
+        pl_local_t *local = NULL;
 
         local = frame->local;
 
         if (local->op == TRUNCATE)
                 loc_wipe (&local->loc);
 
+        if (local->xdata)
+                dict_unref (local->xdata);
+        if (local->fd)
+                fd_unref (local->fd);
+
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
+                             prebuf, postbuf, xdata);
         return 0;
 }
 
 
 static int
 truncate_allowed (pl_inode_t *pl_inode,
-                  void *transport, pid_t client_pid,
+                  client_t *client, pid_t client_pid,
                   gf_lkowner_t *owner, off_t offset)
 {
         posix_lock_t *l = NULL;
@@ -135,7 +128,7 @@ truncate_allowed (pl_inode_t *pl_inode,
 
         region.fl_start   = offset;
         region.fl_end     = LLONG_MAX;
-        region.transport  = transport;
+        region.client     = client;
         region.client_pid = client_pid;
         region.owner      = *owner;
 
@@ -146,7 +139,7 @@ truncate_allowed (pl_inode_t *pl_inode,
                             && locks_overlap (&region, l)
                             && !same_owner (&region, l)) {
                                 ret = 0;
-                                gf_log (POSIX_LOCKS, GF_LOG_TRACE, "Truncate "
+                                gf_log ("posix-locks", GF_LOG_TRACE, "Truncate "
                                         "allowed");
                                 break;
                         }
@@ -160,10 +153,11 @@ truncate_allowed (pl_inode_t *pl_inode,
 
 static int
 truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
 {
         posix_locks_private_t *priv = NULL;
-        struct _truncate_ops  *local = NULL;
+        pl_local_t            *local = NULL;
         inode_t               *inode = NULL;
         pl_inode_t            *pl_inode = NULL;
 
@@ -192,7 +186,7 @@ truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (priv->mandatory
             && pl_inode->mandatory
-            && !truncate_allowed (pl_inode, frame->root->trans,
+            && !truncate_allowed (pl_inode, frame->root->client,
                                   frame->root->pid, &frame->root->lk_owner,
                                   local->offset)) {
                 op_ret   = -1;
@@ -204,12 +198,12 @@ truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         case TRUNCATE:
                 STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->truncate,
-                            &local->loc, local->offset);
+                            &local->loc, local->offset, local->xdata);
                 break;
         case FTRUNCATE:
                 STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->ftruncate,
-                            local->fd, local->offset);
+                            local->fd, local->offset, local->xdata);
                 break;
         }
 
@@ -220,37 +214,42 @@ unwind:
                 "error: %s", op_ret, strerror (op_errno));
         if (local->op == TRUNCATE)
                 loc_wipe (&local->loc);
+        if (local->xdata)
+                dict_unref (local->xdata);
+        if (local->fd)
+                fd_unref (local->fd);
 
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, NULL, xdata);
         return 0;
 }
 
 
 int
 pl_truncate (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, off_t offset)
+             loc_t *loc, off_t offset, dict_t *xdata)
 {
-        struct _truncate_ops *local = NULL;
+        pl_local_t *local = NULL;
 
-        local = GF_CALLOC (1, sizeof (struct _truncate_ops),
-                           gf_locks_mt_truncate_ops);
+        local = mem_get0 (this->local_pool);
         GF_VALIDATE_OR_GOTO (this->name, local, unwind);
 
         local->op         = TRUNCATE;
         local->offset     = offset;
         loc_copy (&local->loc, loc);
+        if (xdata)
+                local->xdata = dict_ref (xdata);
 
         frame->local = local;
 
         STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, loc);
+                    FIRST_CHILD (this)->fops->stat, loc, NULL);
 
         return 0;
 
 unwind:
         gf_log (this->name, GF_LOG_ERROR, "truncate for %s failed with ret: %d, "
                 "error: %s", loc->path, -1, strerror (ENOMEM));
-        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -258,32 +257,54 @@ unwind:
 
 int
 pl_ftruncate (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, off_t offset)
+              fd_t *fd, off_t offset, dict_t *xdata)
 {
-        struct _truncate_ops *local = NULL;
+        pl_local_t *local = NULL;
 
-        local = GF_CALLOC (1, sizeof (struct _truncate_ops),
-                           gf_locks_mt_truncate_ops);
+        local = mem_get0 (this->local_pool);
         GF_VALIDATE_OR_GOTO (this->name, local, unwind);
 
         local->op         = FTRUNCATE;
         local->offset     = offset;
-        local->fd         = fd;
+        local->fd         = fd_ref (fd);
+        if (xdata)
+                local->xdata = dict_ref (xdata);
 
         frame->local = local;
 
         STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
         return 0;
 
 unwind:
         gf_log (this->name, GF_LOG_ERROR, "ftruncate failed with ret: %d, "
                 "error: %s", -1, strerror (ENOMEM));
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
 
+int
+pl_locks_by_fd (pl_inode_t *pl_inode, fd_t *fd)
+{
+       posix_lock_t *l = NULL;
+       int found = 0;
+
+       pthread_mutex_lock (&pl_inode->mutex);
+       {
+
+               list_for_each_entry (l, &pl_inode->ext_list, list) {
+                       if ((l->fd_num == fd_to_fdnum(fd))) {
+                               found = 1;
+                               break;
+                       }
+               }
+
+       }
+       pthread_mutex_unlock (&pl_inode->mutex);
+       return found;
+}
+
 static void
 delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
 {
@@ -313,7 +334,8 @@ delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
 
        list_for_each_entry_safe (l, tmp, &blocked_list, list) {
                list_del_init(&l->list);
-               STACK_UNWIND_STRICT (lk, l->frame, -1, EAGAIN, &l->user_flock);
+               STACK_UNWIND_STRICT (lk, l->frame, -1, EAGAIN, &l->user_flock,
+                                    NULL);
                __destroy_lock (l);
        }
 
@@ -325,7 +347,7 @@ delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
 
 static void
 __delete_locks_of_owner (pl_inode_t *pl_inode,
-                         void *transport, gf_lkowner_t *owner)
+                         client_t *client, gf_lkowner_t *owner)
 {
         posix_lock_t *tmp = NULL;
         posix_lock_t *l = NULL;
@@ -333,7 +355,9 @@ __delete_locks_of_owner (pl_inode_t *pl_inode,
         /* TODO: what if it is a blocked lock with pending l->frame */
 
         list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                if ((l->transport == transport) &&
+                if (l->blocked)
+                        continue;
+                if ((l->client == client) &&
                     is_same_lkowner (&l->owner, owner)) {
                         gf_log ("posix-locks", GF_LOG_TRACE,
                                 " Flushing lock"
@@ -356,18 +380,18 @@ __delete_locks_of_owner (pl_inode_t *pl_inode,
 
 int32_t
 pl_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict)
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 
 }
 
 int32_t
 pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             const char *name)
+             const char *name, dict_t *xdata)
 {
-        int                     op_errno        = EINVAL;
+        int32_t                 op_errno        = EINVAL;
         int                     op_ret          = -1;
         int32_t                 bcount          = 0;
         int32_t                 gcount          = 0;
@@ -375,7 +399,8 @@ pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         char                    *lk_summary     = NULL;
         pl_inode_t              *pl_inode       = NULL;
         dict_t                  *dict           = NULL;
-        clrlk_args              args           = {0,};
+        clrlk_args              args            = {0,};
+        char                    *brickname      = NULL;
 
         if (!name)
                 goto usual;
@@ -422,33 +447,51 @@ pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                         goto out;
         }
 
+        op_ret = fetch_pathinfo (this, loc->inode, &op_errno, &brickname);
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Couldn't get brickname");
+        } else {
+                op_ret = format_brickname(brickname);
+                if (op_ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Couldn't format brickname");
+                        GF_FREE(brickname);
+                        brickname = NULL;
+                }
+        }
+
         if (!gcount && !bcount) {
                 if (gf_asprintf (&lk_summary, "No locks cleared.") == -1) {
+                        op_ret = -1;
                         op_errno = ENOMEM;
                         goto out;
                 }
         } else if (gf_asprintf (&lk_summary, "%s: %s blocked locks=%d "
-                                "granted locks=%d", this->name,
+                                "granted locks=%d",
+                                (brickname == NULL)? this->name : brickname,
                                 (args.type == CLRLK_INODE)? "inode":
                                 (args.type == CLRLK_ENTRY)? "entry":
                                 (args.type == CLRLK_POSIX)? "posix": " ",
                                 bcount, gcount) == -1) {
+                op_ret = -1;
                 op_errno = ENOMEM;
                 goto out;
         }
 
         strncpy (key, name, strlen (name));
         if (dict_set_dynstr (dict, key, lk_summary)) {
+                op_ret = -1;
                 op_errno = ENOMEM;
                 goto out;
         }
 
         op_ret = 0;
 out:
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        GF_FREE(brickname);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
 
-        if (args.opts)
-                GF_FREE (args.opts);
+        GF_FREE (args.opts);
         if (op_ret && lk_summary)
                 GF_FREE (lk_summary);
         if (dict)
@@ -457,17 +500,415 @@ out:
 
 usual:
         STACK_WIND (frame, pl_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+        return 0;
+}
+
+static int
+format_brickname(char *brickname)
+{
+       int   ret             = -1;
+       char *hostname        = NULL;
+       char *volume          = NULL;
+       char *saveptr         = NULL;
+
+       if (!brickname)
+               goto out;
+
+       strtok_r(brickname, ":", &saveptr);
+       hostname = gf_strdup(strtok_r(NULL, ":", &saveptr));
+       if (hostname == NULL)
+               goto out;
+       volume = gf_strdup(strtok_r(NULL, ".", &saveptr));
+       if (volume == NULL)
+               goto out;
+
+       sprintf(brickname, "%s:%s", hostname, volume);
+
+       ret = 0;
+out:
+       GF_FREE(hostname);
+       GF_FREE(volume);
+       return ret;
+}
+
+static int
+fetch_pathinfo (xlator_t *this, inode_t *inode, int32_t *op_errno,
+                char **brickname)
+{
+        int                    ret       = -1;
+        loc_t                  loc       = {0, };
+        dict_t                *dict      = NULL;
+
+        if (!brickname)
+                goto out;
+
+        if (!op_errno)
+                goto out;
+
+        uuid_copy (loc.gfid, inode->gfid);
+        loc.inode = inode_ref (inode);
+
+        ret = syncop_getxattr (FIRST_CHILD(this), &loc, &dict,
+                               GF_XATTR_PATHINFO_KEY);
+        if (ret < 0) {
+                *op_errno = errno;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, brickname);
+        if (ret)
+                goto out;
+
+        *brickname = gf_strdup(*brickname);
+        if (*brickname == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (dict != NULL) {
+                dict_unref (dict);
+        }
+        loc_wipe(&loc);
+
+        return ret;
+}
+
+
+int
+pl_lockinfo_get_brickname (xlator_t *this, inode_t *inode, int32_t *op_errno)
+{
+        int                    ret       = -1;
+        posix_locks_private_t *priv      = NULL;
+        char                  *brickname = NULL;
+        char                  *end       = NULL;
+        char                  *tmp       = NULL;
+
+        priv = this->private;
+
+        ret = fetch_pathinfo (this, inode, op_errno, &brickname);
+        if (ret)
+                goto out;
+
+        end = strrchr (brickname, ':');
+        if (!end) {
+                GF_FREE(brickname);
+                ret = -1;
+                goto out;
+        }
+
+        tmp = brickname;
+        brickname = gf_strndup (brickname, (end - brickname));
+        if (brickname == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        priv->brickname = brickname;
+        ret = 0;
+out:
+        GF_FREE(tmp);
+        return ret;
+}
+
+char *
+pl_lockinfo_key (xlator_t *this, inode_t *inode, int32_t *op_errno)
+{
+        posix_locks_private_t *priv = NULL;
+        char                  *key  = NULL;
+        int                    ret = 0;
+
+        priv = this->private;
+
+        if (priv->brickname == NULL) {
+                ret = pl_lockinfo_get_brickname (this, inode, op_errno);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "cannot get brickname");
+                        goto out;
+                }
+        }
+
+        key = priv->brickname;
+out:
+        return key;
+}
+
+int32_t
+pl_fgetxattr_handle_lockinfo (xlator_t *this, fd_t *fd,
+                              dict_t *dict, int32_t *op_errno)
+{
+        pl_inode_t    *pl_inode = NULL;
+        char          *key      = NULL, *buf = NULL;
+        int32_t        op_ret   = 0;
+        unsigned long  fdnum    = 0, len = 0;
+        dict_t        *tmp      = NULL;
+
+        pl_inode = pl_inode_get (this, fd->inode);
+
+        if (!pl_inode) {
+                gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
+                *op_errno = EBADFD;
+                op_ret = -1;
+                goto out;
+        }
+
+        if (!pl_locks_by_fd (pl_inode, fd)) {
+                op_ret = 0;
+                goto out;
+        }
+
+        fdnum = fd_to_fdnum (fd);
+
+        key = pl_lockinfo_key (this, fd->inode, op_errno);
+        if (key == NULL) {
+                op_ret = -1;
+                goto out;
+        }
+
+        tmp = dict_new ();
+        if (tmp == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_set_uint64 (tmp, key, fdnum);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
+                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+                        fdnum, fd, uuid_utoa (fd->inode->gfid),
+                        strerror (*op_errno));
+                goto out;
+        }
+
+        len = dict_serialized_length (tmp);
+        if (len < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict_serialized_length failed (%s) while handling "
+                        "lockinfo for fd (ptr:%p inode-gfid:%s)",
+                        strerror (*op_errno), fd, uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+        buf = GF_CALLOC (1, len, gf_common_mt_char);
+        if (buf == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_serialize (tmp, buf);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict_serialize failed (%s) while handling lockinfo "
+                        "for fd (ptr: %p inode-gfid:%s)", strerror (*op_errno),
+                        fd, uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+        op_ret = dict_set_dynptr (dict, GF_XATTR_LOCKINFO_KEY, buf, len);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
+                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+                        fdnum, fd, uuid_utoa (fd->inode->gfid),
+                        strerror (*op_errno));
+                goto out;
+        }
+
+        buf = NULL;
+out:
+        if (tmp != NULL) {
+                dict_unref (tmp);
+        }
+
+        if (buf != NULL) {
+                GF_FREE (buf);
+        }
+
+        return op_ret;
+}
+
+
+int32_t
+pl_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              const char *name, dict_t *xdata)
+{
+        int32_t  op_ret = 0, op_errno = 0;
+        dict_t  *dict   = NULL;
+
+        if (!name) {
+                goto usual;
+        }
+
+        if (strcmp (name, GF_XATTR_LOCKINFO_KEY) == 0) {
+                dict = dict_new ();
+                if (dict == NULL) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = pl_fgetxattr_handle_lockinfo (this, fd, dict,
+                                                       &op_errno);
+                if (op_ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "getting lockinfo on fd (ptr:%p inode-gfid:%s) "
+                                "failed (%s)", fd, uuid_utoa (fd->inode->gfid),
+                                strerror (op_errno));
+                }
+
+                goto unwind;
+        } else {
+                goto usual;
+        }
+
+unwind:
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, NULL);
+        if (dict != NULL) {
+                dict_unref (dict);
+        }
+
+        return 0;
+
+usual:
+        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+        return 0;
+}
+
+int32_t
+pl_migrate_locks (call_frame_t *frame, fd_t *newfd, uint64_t oldfd_num,
+                  int32_t *op_errno)
+{
+        pl_inode_t   *pl_inode  = NULL;
+        uint64_t      newfd_num = 0;
+        posix_lock_t *l         = NULL;
+        int32_t       op_ret    = 0;
+
+        newfd_num = fd_to_fdnum (newfd);
+
+        pl_inode = pl_inode_get (frame->this, newfd->inode);
+        if (pl_inode == NULL) {
+                op_ret = -1;
+                *op_errno = EBADFD;
+                goto out;
+        }
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (l, &pl_inode->ext_list, list) {
+                        if (l->fd_num == oldfd_num) {
+                                l->fd_num = newfd_num;
+                                l->client = frame->root->client;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        op_ret = 0;
+out:
+        return op_ret;
+}
+
+int32_t
+pl_fsetxattr_handle_lockinfo (call_frame_t *frame, fd_t *fd, char *lockinfo_buf,
+                              int len, int32_t *op_errno)
+{
+        int32_t   op_ret    = -1;
+        dict_t   *lockinfo  = NULL;
+        uint64_t  oldfd_num = 0;
+        char     *key       = NULL;
+
+        lockinfo = dict_new ();
+        if (lockinfo == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_unserialize (lockinfo_buf, len, &lockinfo);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                goto out;
+        }
+
+        key = pl_lockinfo_key (frame->this, fd->inode, op_errno);
+        if (key == NULL) {
+                op_ret = -1;
+                goto out;
+        }
+
+        op_ret = dict_get_uint64 (lockinfo, key, &oldfd_num);
+
+        if (oldfd_num == 0) {
+                op_ret = 0;
+                goto out;
+        }
+
+        op_ret = pl_migrate_locks (frame, fd, oldfd_num, op_errno);
+        if (op_ret < 0) {
+                gf_log (frame->this->name, GF_LOG_WARNING,
+                        "migration of locks from oldfd (ptr:%p) to newfd "
+                        "(ptr:%p) (inode-gfid:%s)", (void *)oldfd_num, fd,
+                        uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+out:
+        dict_unref (lockinfo);
+
+        return op_ret;
+}
+
+int32_t
+pl_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
+{
+        int32_t  op_ret       = 0, op_errno = 0;
+        void    *lockinfo_buf = NULL;
+        int      len          = 0;
+
+        op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                       &lockinfo_buf, &len);
+        if (lockinfo_buf == NULL) {
+                goto usual;
+        }
+
+        op_ret = pl_fsetxattr_handle_lockinfo (frame, fd, lockinfo_buf, len,
+                                               &op_errno);
+        if (op_ret < 0) {
+                goto unwind;
+        }
+
+usual:
+        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+        return 0;
+
+unwind:
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
         return 0;
 }
 
 int32_t
 pl_opendir_cbk (call_frame_t *frame,
-                     void *cookie,
-                     xlator_t *this,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     fd_t *fd)
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                fd_t *fd, dict_t *xdata)
 {
         pl_fdctx_t *fdctx = NULL;
 
@@ -486,28 +927,28 @@ unwind:
                              frame,
                              op_ret,
                              op_errno,
-                             fd);
+                             fd, xdata);
         return 0;
 }
 
 int32_t
 pl_opendir (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, fd_t *fd)
+             loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         STACK_WIND (frame,
                     pl_opendir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->opendir,
-                    loc, fd);
+                    loc, fd, xdata);
         return 0;
 
 }
 
 int
 pl_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -515,7 +956,7 @@ pl_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 pl_flush (call_frame_t *frame, xlator_t *this,
-          fd_t *fd)
+          fd_t *fd, dict_t *xdata)
 {
         pl_inode_t *pl_inode = NULL;
 
@@ -523,7 +964,7 @@ pl_flush (call_frame_t *frame, xlator_t *this,
 
         if (!pl_inode) {
                 gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
-                STACK_UNWIND_STRICT (flush, frame, -1, EBADFD);
+                STACK_UNWIND_STRICT (flush, frame, -1, EBADFD, NULL);
                 return 0;
         }
 
@@ -542,7 +983,7 @@ pl_flush (call_frame_t *frame, xlator_t *this,
         }
         pthread_mutex_lock (&pl_inode->mutex);
         {
-                __delete_locks_of_owner (pl_inode, frame->root->trans,
+                __delete_locks_of_owner (pl_inode, frame->root->client,
                                          &frame->root->lk_owner);
         }
         pthread_mutex_unlock (&pl_inode->mutex);
@@ -553,14 +994,14 @@ pl_flush (call_frame_t *frame, xlator_t *this,
 
 wind:
         STACK_WIND (frame, pl_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd);
+                    FIRST_CHILD(this)->fops->flush, fd, xdata);
         return 0;
 }
 
 
 int
 pl_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd)
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         pl_fdctx_t *fdctx = NULL;
 
@@ -575,7 +1016,7 @@ pl_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 unwind:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 
         return 0;
 }
@@ -583,12 +1024,11 @@ unwind:
 
 int
 pl_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+         fd_t *fd, dict_t *xdata)
 {
-        /* why isn't O_TRUNC being handled ? */
         STACK_WIND (frame, pl_open_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
-                    loc, flags & ~O_TRUNC, fd, wbflags);
+                    loc, flags, fd, xdata);
 
         return 0;
 }
@@ -598,7 +1038,7 @@ int
 pl_create_cbk (call_frame_t *frame, void *cookie,
                xlator_t *this, int32_t op_ret, int32_t op_errno,
                fd_t *fd, inode_t *inode, struct iatt *buf,
-               struct iatt *preparent, struct iatt *postparent)
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         pl_fdctx_t *fdctx = NULL;
 
@@ -614,7 +1054,7 @@ pl_create_cbk (call_frame_t *frame, void *cookie,
 
 unwind:
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         return 0;
 }
@@ -622,11 +1062,12 @@ unwind:
 
 int
 pl_create (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+           loc_t *loc, int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+           dict_t *xdata)
 {
         STACK_WIND (frame, pl_create_cbk,
                     FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
@@ -635,10 +1076,10 @@ int
 pl_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int32_t op_ret, int32_t op_errno,
               struct iovec *vector, int32_t count, struct iatt *stbuf,
-              struct iobref *iobref)
+              struct iobref *iobref, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref);
+                             vector, count, stbuf, iobref, xdata);
 
         return 0;
 }
@@ -646,9 +1087,10 @@ pl_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 pl_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+               struct iatt *postbuf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 
         return 0;
 }
@@ -706,12 +1148,12 @@ __rw_allowable (pl_inode_t *pl_inode, posix_lock_t *region,
 
 
 int
-pl_readv_cont (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, size_t size, off_t offset, uint32_t flags)
+pl_readv_cont (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, uint32_t flags, dict_t *xdata)
 {
         STACK_WIND (frame, pl_readv_cbk,
                     FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                    fd, size, offset, flags);
+                    fd, size, offset, flags, xdata);
 
         return 0;
 }
@@ -719,7 +1161,7 @@ pl_readv_cont (call_frame_t *frame, xlator_t *this,
 
 int
 pl_readv (call_frame_t *frame, xlator_t *this,
-          fd_t *fd, size_t size, off_t offset, uint32_t flags)
+          fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         posix_locks_private_t *priv = NULL;
         pl_inode_t            *pl_inode = NULL;
@@ -736,7 +1178,7 @@ pl_readv (call_frame_t *frame, xlator_t *this,
         if (priv->mandatory && pl_inode->mandatory) {
                 region.fl_start   = offset;
                 region.fl_end     = offset + size - 1;
-                region.transport  = frame->root->trans;
+                region.client     = frame->root->client;
                 region.fd_num     = fd_to_fdnum(fd);
                 region.client_pid = frame->root->pid;
                 region.owner      = frame->root->lk_owner;
@@ -766,7 +1208,8 @@ pl_readv (call_frame_t *frame, xlator_t *this,
                         }
 
                         rw->stub = fop_readv_stub (frame, pl_readv_cont,
-                                                   fd, size, offset, flags);
+                                                   fd, size, offset, flags,
+                                                   xdata);
                         if (!rw->stub) {
                                 op_errno = ENOMEM;
                                 op_ret = -1;
@@ -786,12 +1229,12 @@ pl_readv (call_frame_t *frame, xlator_t *this,
         if (wind_needed) {
                 STACK_WIND (frame, pl_readv_cbk,
                             FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                            fd, size, offset, flags);
+                            fd, size, offset, flags, xdata);
         }
 
         if (op_ret == -1)
                 STACK_UNWIND_STRICT (readv, frame, -1, op_errno,
-                                     NULL, 0, NULL, NULL);
+                                     NULL, 0, NULL, NULL, NULL);
 
         return 0;
 }
@@ -800,11 +1243,11 @@ pl_readv (call_frame_t *frame, xlator_t *this,
 int
 pl_writev_cont (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 struct iovec *vector, int count, off_t offset,
-                uint32_t flags, struct iobref *iobref)
+                uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         STACK_WIND (frame, pl_writev_cbk,
                     FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
 
         return 0;
 }
@@ -813,7 +1256,7 @@ pl_writev_cont (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int
 pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
            struct iovec *vector, int32_t count, off_t offset,
-           uint32_t flags, struct iobref *iobref)
+           uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         posix_locks_private_t *priv = NULL;
         pl_inode_t            *pl_inode = NULL;
@@ -829,7 +1272,7 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (priv->mandatory && pl_inode->mandatory) {
                 region.fl_start   = offset;
                 region.fl_end     = offset + iov_length (vector, count) - 1;
-                region.transport  = frame->root->trans;
+                region.client     = frame->root->client;
                 region.fd_num     = fd_to_fdnum(fd);
                 region.client_pid = frame->root->pid;
                 region.owner      = frame->root->lk_owner;
@@ -860,7 +1303,7 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
                         rw->stub = fop_writev_stub (frame, pl_writev_cont,
                                                     fd, vector, count, offset,
-                                                    flags, iobref);
+                                                    flags, iobref, xdata);
                         if (!rw->stub) {
                                 op_errno = ENOMEM;
                                 op_ret = -1;
@@ -880,10 +1323,11 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (wind_needed)
                 STACK_WIND (frame, pl_writev_cbk,
                             FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-                            fd, vector, count, offset, flags, iobref);
+                            fd, vector, count, offset, flags, iobref, xdata);
 
         if (op_ret == -1)
-                STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+                STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL,
+                                     NULL);
 
         return 0;
 }
@@ -909,7 +1353,7 @@ lock_dup (posix_lock_t *lock)
 {
         posix_lock_t *new_lock = NULL;
 
-        new_lock = new_posix_lock (&lock->user_flock, lock->transport,
+        new_lock = new_posix_lock (&lock->user_flock, lock->client,
                                    lock->client_pid, &lock->owner,
                                    (fd_t *)lock->fd_num);
         return new_lock;
@@ -1067,10 +1511,8 @@ unlock:
 
 int
 pl_lk (call_frame_t *frame, xlator_t *this,
-       fd_t *fd, int32_t cmd, struct gf_flock *flock)
+       fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        void         *transport  = NULL;
-        pid_t         client_pid = 0;
         pl_inode_t   *pl_inode   = NULL;
         int           op_ret     = 0;
         int           op_errno   = 0;
@@ -1079,9 +1521,6 @@ pl_lk (call_frame_t *frame, xlator_t *this,
         posix_lock_t *conf       = NULL;
         int           ret        = 0;
 
-        transport  = frame->root->trans;
-        client_pid = frame->root->pid;
-
         if ((flock->l_start < 0) || (flock->l_len < 0)) {
                 op_ret = -1;
                 op_errno = EINVAL;
@@ -1095,7 +1534,7 @@ pl_lk (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        reqlock = new_posix_lock (flock, transport, client_pid,
+        reqlock = new_posix_lock (flock, frame->root->client, frame->root->pid,
                                   &frame->root->lk_owner, fd);
 
         if (!reqlock) {
@@ -1213,13 +1652,22 @@ pl_lk (call_frame_t *frame, xlator_t *this,
                         op_ret = -1;
                         op_errno = EAGAIN;
                         __destroy_lock (reqlock);
+
+                } else if ((0 == ret) && (F_UNLCK == flock->l_type)) {
+                        /* For NLM's last "unlock on fd" detection */
+                        if (pl_locks_by_fd (pl_inode, fd))
+                                flock->l_type = F_RDLCK;
+                        else
+                                flock->l_type = F_UNLCK;
                 }
         }
 
 unwind:
         pl_trace_out (this, frame, fd, NULL, cmd, flock, op_ret, op_errno, NULL);
         pl_update_refkeeper (this, fd->inode);
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock);
+
+
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock, xdata);
 out:
         return 0;
 }
@@ -1296,7 +1744,7 @@ pl_forget (xlator_t *this,
 
                                 list_for_each_entry_safe (ino_l, ino_tmp, &dom->inodelk_list, list) {
                                         __delete_inode_lock (ino_l);
-                                        __destroy_inode_lock (ino_l);
+                                        __pl_inodelk_unref (ino_l);
                                 }
 
                                 list_splice_init (&dom->blocked_inodelks, &inodelks_released);
@@ -1310,8 +1758,8 @@ pl_forget (xlator_t *this,
                                 list_for_each_entry_safe (entry_l, entry_tmp, &dom->entrylk_list, domain_list) {
                                         list_del_init (&entry_l->domain_list);
 
-                                        if (entry_l->basename)
-                                                GF_FREE ((char *)entry_l->basename);
+                                        GF_FREE ((char *)entry_l->basename);
+                                        GF_FREE (entry_l->connection_id);
                                         GF_FREE (entry_l);
                                 }
 
@@ -1330,21 +1778,22 @@ pl_forget (xlator_t *this,
 
         list_for_each_entry_safe (ext_l, ext_tmp, &posixlks_released, list) {
 
-                STACK_UNWIND_STRICT (lk, ext_l->frame, -1, 0, &ext_l->user_flock);
+                STACK_UNWIND_STRICT (lk, ext_l->frame, -1, 0,
+                                     &ext_l->user_flock, NULL);
                 __destroy_lock (ext_l);
         }
 
         list_for_each_entry_safe (ino_l, ino_tmp, &inodelks_released, blocked_locks) {
 
-                STACK_UNWIND_STRICT (inodelk, ino_l->frame, -1, 0);
-                __destroy_inode_lock (ino_l);
+                STACK_UNWIND_STRICT (inodelk, ino_l->frame, -1, 0, NULL);
+                __pl_inodelk_unref (ino_l);
         }
 
         list_for_each_entry_safe (entry_l, entry_tmp, &entrylks_released, blocked_locks) {
 
-                STACK_UNWIND_STRICT (entrylk, entry_l->frame, -1, 0);
-                if (entry_l->basename)
-                        GF_FREE ((char *)entry_l->basename);
+                STACK_UNWIND_STRICT (entrylk, entry_l->frame, -1, 0, NULL);
+                GF_FREE ((char *)entry_l->basename);
+                GF_FREE (entry_l->connection_id);
                 GF_FREE (entry_l);
 
         }
@@ -1420,7 +1869,7 @@ out:
         return ret;
 }
 
-static int32_t
+int32_t
 __get_posixlk_count (xlator_t *this, pl_inode_t *pl_inode)
 {
         posix_lock_t *lock   = NULL;
@@ -1428,16 +1877,6 @@ __get_posixlk_count (xlator_t *this, pl_inode_t *pl_inode)
 
         list_for_each_entry (lock, &pl_inode->ext_list, list) {
 
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" state: %s",
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len,
-                                lock->blocked == 1 ? "Blocked" : "Active");
-
                 count++;
         }
 
@@ -1470,6 +1909,24 @@ out:
 }
 
 void
+pl_parent_entrylk_xattr_fill (xlator_t *this, inode_t *parent,
+                              char *basename, dict_t *dict)
+{
+        uint32_t         entrylk = 0;
+        int             ret     = -1;
+
+        if (!parent || !basename || !strlen (basename))
+                goto out;
+        entrylk = check_entrylk_on_basename (this, parent, basename);
+out:
+        ret = dict_set_uint32 (dict, GLUSTERFS_PARENT_ENTRYLK, entrylk);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        " dict_set failed on key %s", GLUSTERFS_PARENT_ENTRYLK);
+        }
+}
+
+void
 pl_entrylk_xattr_fill (xlator_t *this, inode_t *inode,
                        dict_t *dict)
 {
@@ -1486,19 +1943,34 @@ pl_entrylk_xattr_fill (xlator_t *this, inode_t *inode,
 }
 
 void
-pl_inodelk_xattr_fill (xlator_t *this, inode_t *inode,
-                       dict_t *dict)
+pl_inodelk_xattr_fill (xlator_t *this, inode_t *inode, dict_t *dict,
+                       gf_boolean_t per_dom)
 {
         int32_t     count = 0;
         int         ret   = -1;
+        char        *domname = NULL;
+
+
+        if (per_dom){
+                ret = dict_get_str (dict, GLUSTERFS_INODELK_DOM_COUNT,
+                                    &domname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "value for key %s",GLUSTERFS_INODELK_DOM_COUNT);
+                        goto out;
+                }
+        }
+
+        count = get_inodelk_count (this, inode, domname);
 
-        count = get_inodelk_count (this, inode);
         ret = dict_set_int32 (dict, GLUSTERFS_INODELK_COUNT, count);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " dict_set failed on key %s", GLUSTERFS_INODELK_COUNT);
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to set count for "
+                        "key %s", GLUSTERFS_INODELK_COUNT);
         }
 
+out:
+        return;
 }
 
 void
@@ -1525,50 +1997,57 @@ pl_lookup_cbk (call_frame_t *frame,
                     int32_t op_errno,
                     inode_t *inode,
                     struct iatt *buf,
-                    dict_t *dict,
+                    dict_t *xdata,
                     struct iatt *postparent)
 {
         pl_local_t *local = NULL;
 
         GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
 
-        if (op_ret) {
+        if (op_ret)
                 goto out;
-        }
 
         local = frame->local;
 
+        if (local->parent_entrylk_req)
+                pl_parent_entrylk_xattr_fill (this, local->loc.parent,
+                                              (char*)local->loc.name, xdata);
         if (local->entrylk_count_req)
-                pl_entrylk_xattr_fill (this, inode, dict);
+                pl_entrylk_xattr_fill (this, inode, xdata);
         if (local->inodelk_count_req)
-                pl_inodelk_xattr_fill (this, inode, dict);
+                pl_inodelk_xattr_fill (this, inode, xdata, _gf_false);
+        if (local->inodelk_dom_count_req)
+                pl_inodelk_xattr_fill (this, inode, xdata, _gf_true);
         if (local->posixlk_count_req)
-                pl_posixlk_xattr_fill (this, inode, dict);
+                pl_posixlk_xattr_fill (this, inode, xdata);
 
 
+out:
+        local = frame->local;
         frame->local = NULL;
 
-        if (local != NULL)
-                GF_FREE (local);
+        if (local != NULL) {
+                loc_wipe (&local->loc);
+                mem_put (local);
+        }
 
-out:
         STACK_UNWIND_STRICT (
                      lookup,
                      frame,
-                      op_ret,
-                      op_errno,
-                      inode,
-                      buf,
-                      dict,
-                      postparent);
+                     op_ret,
+                     op_errno,
+                     inode,
+                     buf,
+                     xdata,
+                     postparent);
         return 0;
 }
 
 int32_t
 pl_lookup (call_frame_t *frame,
-                xlator_t *this,
-                loc_t *loc,
-                dict_t *xattr_req)
+           xlator_t *this,
+           loc_t *loc,
+           dict_t *xdata)
 {
         pl_local_t *local  = NULL;
         int         ret    = -1;
@@ -1577,36 +2056,41 @@ pl_lookup (call_frame_t *frame,
         VALIDATE_OR_GOTO (this, out);
         VALIDATE_OR_GOTO (loc, out);
 
-        local = GF_CALLOC (1, sizeof (*local), gf_locks_mt_pl_local_t);
+        local = mem_get0 (this->local_pool);
         GF_VALIDATE_OR_GOTO (this->name, local, out);
 
-        if (xattr_req) {
-                if (dict_get (xattr_req, GLUSTERFS_ENTRYLK_COUNT))
+        if (xdata) {
+                if (dict_get (xdata, GLUSTERFS_ENTRYLK_COUNT))
                         local->entrylk_count_req = 1;
-                if (dict_get (xattr_req, GLUSTERFS_INODELK_COUNT))
+                if (dict_get (xdata, GLUSTERFS_INODELK_COUNT))
                         local->inodelk_count_req = 1;
-                if (dict_get (xattr_req, GLUSTERFS_POSIXLK_COUNT))
+                if (dict_get (xdata, GLUSTERFS_INODELK_DOM_COUNT))
+                        local->inodelk_dom_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_POSIXLK_COUNT))
                         local->posixlk_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_PARENT_ENTRYLK))
+                        local->parent_entrylk_req = 1;
         }
 
         frame->local = local;
+        loc_copy (&local->loc, loc);
 
         STACK_WIND (frame,
                     pl_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
-                    loc,
-                    xattr_req);
+                    loc, xdata);
         ret = 0;
 out:
         if (ret == -1)
-                STACK_UNWIND_STRICT (lookup, frame, -1, 0, NULL, NULL, NULL, NULL);
+                STACK_UNWIND_STRICT (lookup, frame, -1, 0, NULL,
+                                     NULL, NULL, NULL);
 
         return 0;
 }
 int
 pl_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *entries)
+                 int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
         pl_local_t *local  = NULL;
         gf_dirent_t *entry = NULL;
@@ -1620,17 +2104,21 @@ pl_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (local->entrylk_count_req)
                         pl_entrylk_xattr_fill (this, entry->inode, entry->dict);
                 if (local->inodelk_count_req)
-                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict);
+                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict,
+                                               _gf_false);
+                if (local->inodelk_dom_count_req)
+                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict,
+                                               _gf_true);
                 if (local->posixlk_count_req)
                         pl_posixlk_xattr_fill (this, entry->inode, entry->dict);
         }
 
 unwind:
         frame->local = NULL;
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 
         if (local)
-                GF_FREE (local);
+                mem_put (local);
 
         return 0;
 }
@@ -1641,7 +2129,7 @@ pl_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 {
         pl_local_t *local  = NULL;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_locks_mt_pl_local_t);
+        local = mem_get0 (this->local_pool);
         GF_VALIDATE_OR_GOTO (this->name, local, out);
 
         if (dict) {
@@ -1649,6 +2137,8 @@ pl_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                         local->entrylk_count_req = 1;
                 if (dict_get (dict, GLUSTERFS_INODELK_COUNT))
                         local->inodelk_count_req = 1;
+                if (dict_get (dict, GLUSTERFS_INODELK_DOM_COUNT))
+                        local->inodelk_dom_count_req = 1;
                 if (dict_get (dict, GLUSTERFS_POSIXLK_COUNT))
                         local->posixlk_count_req = 1;
         }
@@ -1661,17 +2151,19 @@ pl_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
         return 0;
 out:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL);
+        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
         return 0;
 }
 
 
 void
 pl_dump_lock (char *str, int size, struct gf_flock *flock,
-              gf_lkowner_t *owner, void *trans, time_t *granted_time,
-              time_t *blkd_time, gf_boolean_t active)
+              gf_lkowner_t *owner, void *trans, char *conn_id,
+              time_t *granted_time, time_t *blkd_time, gf_boolean_t active)
 {
-        char *type_str = NULL;
+        char  *type_str    = NULL;
+        char   granted[32] = {0,};
+        char   blocked[32] = {0,};
 
         switch (flock->l_type) {
         case F_RDLCK:
@@ -1690,33 +2182,32 @@ pl_dump_lock (char *str, int size, struct gf_flock *flock,
 
         if (active) {
                 if (blkd_time && *blkd_time == 0) {
-                        snprintf (str, size, "type=%s, start=%llu, len=%llu, "
-                                  "pid=%llu, lk-owner=%s, transport=%p, "
-                                  "granted at %s",
-                                  type_str, (unsigned long long) flock->l_start,
+                        snprintf (str, size, RANGE_GRNTD_FMT,
+                                  type_str, flock->l_whence,
+                                  (unsigned long long) flock->l_start,
                                   (unsigned long long) flock->l_len,
                                   (unsigned long long) flock->l_pid,
-                                  lkowner_utoa (owner),
-                                  trans, ctime (granted_time));
+                                  lkowner_utoa (owner), trans, conn_id,
+                                  ctime_r (granted_time, granted));
                 } else {
-                        snprintf (str, size, "type=%s, start=%llu, len=%llu, "
-                                  "pid=%llu, lk-owner=%s, transport=%p, "
-                                  "blocked at %s, granted at  %s",
-                                  type_str, (unsigned long long) flock->l_start,
+                        snprintf (str, size, RANGE_BLKD_GRNTD_FMT,
+                                  type_str, flock->l_whence,
+                                  (unsigned long long) flock->l_start,
                                   (unsigned long long) flock->l_len,
                                   (unsigned long long) flock->l_pid,
-                                  lkowner_utoa (owner),
-                                  trans, ctime (blkd_time), ctime (granted_time));
+                                  lkowner_utoa (owner), trans, conn_id,
+                                  ctime_r (blkd_time, blocked),
+                                  ctime_r (granted_time, granted));
                 }
         }
         else {
-                snprintf (str, size, "type=%s, start=%llu, len=%llu, pid=%llu, "
-                          "lk-owner=%s, transport=%p, blocked at  %s",
-                          type_str, (unsigned long long) flock->l_start,
+                snprintf (str, size, RANGE_BLKD_FMT,
+                          type_str, flock->l_whence,
+                          (unsigned long long) flock->l_start,
                           (unsigned long long) flock->l_len,
                           (unsigned long long) flock->l_pid,
-                          lkowner_utoa (owner),
-                          trans, ctime (blkd_time));
+                          lkowner_utoa (owner), trans, conn_id,
+                          ctime_r (blkd_time, blocked));
         }
 
 }
@@ -1726,8 +2217,10 @@ __dump_entrylks (pl_inode_t *pl_inode)
 {
         pl_dom_list_t   *dom  = NULL;
         pl_entry_lock_t *lock = NULL;
-        int             count = 0;
-        char            key[GF_DUMP_MAX_BUF_LEN];
+        char             blocked[32] = {0,};
+        char             granted[32] = {0,};
+        int              count = 0;
+        char             key[GF_DUMP_MAX_BUF_LEN] = {0,};
 
         char tmp[256];
 
@@ -1746,22 +2239,22 @@ __dump_entrylks (pl_inode_t *pl_inode)
                                                "xlator.feature.locks.lock-dump.domain.entrylk",
                                                "entrylk[%d](ACTIVE)", count );
                         if (lock->blkd_time.tv_sec == 0 && lock->blkd_time.tv_usec == 0) {
-                                snprintf (tmp, 256," %s on %s pid = %llu, owner=%s, transport=%p,"
-                                          " granted at %s",
+                                snprintf (tmp, 256, ENTRY_GRNTD_FMT,
                                           lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
                                           "ENTRYLK_WRLCK", lock->basename,
                                           (unsigned long long) lock->client_pid,
                                           lkowner_utoa (&lock->owner), lock->trans,
-                                          ctime (&lock->granted_time.tv_sec));
+                                          lock->connection_id,
+                                          ctime_r (&lock->granted_time.tv_sec, granted));
                         } else {
-                                snprintf (tmp, 256," %s on %s pid = %llu, owner=%s, transport=%p,"
-                                          " blocked at %s, granted at %s",
+                                snprintf (tmp, 256, ENTRY_BLKD_GRNTD_FMT,
                                           lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
                                           "ENTRYLK_WRLCK", lock->basename,
                                           (unsigned long long) lock->client_pid,
                                           lkowner_utoa (&lock->owner), lock->trans,
-                                          ctime (&lock->blkd_time.tv_sec),
-                                          ctime (&lock->granted_time.tv_sec));
+                                          lock->connection_id,
+                                          ctime_r (&lock->blkd_time.tv_sec, blocked),
+                                          ctime_r (&lock->granted_time.tv_sec, granted));
                         }
 
                         gf_proc_dump_write(key, tmp);
@@ -1774,13 +2267,13 @@ __dump_entrylks (pl_inode_t *pl_inode)
                         gf_proc_dump_build_key(key,
                                                "xlator.feature.locks.lock-dump.domain.entrylk",
                                                "entrylk[%d](BLOCKED)", count );
-                        snprintf (tmp, 256," %s on %s pid = %llu, owner=%s, transport=%p,"
-                                  " blocked at %s",
+                        snprintf (tmp, 256, ENTRY_BLKD_FMT,
                                   lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
                                   "ENTRYLK_WRLCK", lock->basename,
                                   (unsigned long long) lock->client_pid,
                                   lkowner_utoa (&lock->owner), lock->trans,
-                                  ctime (&lock->blkd_time.tv_sec));
+                                  lock->connection_id,
+                                  ctime_r (&lock->blkd_time.tv_sec, blocked));
 
                         gf_proc_dump_write(key, tmp);
 
@@ -1830,7 +2323,7 @@ __dump_inodelks (pl_inode_t *pl_inode)
                         SET_FLOCK_PID (&lock->user_flock, lock);
                         pl_dump_lock (tmp, 256, &lock->user_flock,
                                       &lock->owner,
-                                      lock->transport,
+                                      lock->client, lock->connection_id,
                                       &lock->granted_time.tv_sec,
                                       &lock->blkd_time.tv_sec,
                                       _gf_true);
@@ -1847,7 +2340,7 @@ __dump_inodelks (pl_inode_t *pl_inode)
                         SET_FLOCK_PID (&lock->user_flock, lock);
                         pl_dump_lock (tmp, 256, &lock->user_flock,
                                       &lock->owner,
-                                      lock->transport,
+                                      lock->client, lock->connection_id,
                                       0, &lock->blkd_time.tv_sec,
                                       _gf_false);
                         gf_proc_dump_write(key, tmp);
@@ -1888,16 +2381,13 @@ __dump_posixlks (pl_inode_t *pl_inode)
                                      count,
                                      lock->blocked ? "BLOCKED" : "ACTIVE");
               pl_dump_lock (tmp, 256, &lock->user_flock,
-                            &lock->owner, lock->transport,
+                            &lock->owner, lock->client, NULL,
                             &lock->granted_time.tv_sec, &lock->blkd_time.tv_sec,
                             (lock->blocked)? _gf_false: _gf_true);
               gf_proc_dump_write(key, tmp);
 
               count++;
         }
-
-
-
 }
 
 void
@@ -1918,46 +2408,81 @@ pl_dump_inode_priv (xlator_t *this, inode_t *inode)
         int             ret = -1;
         uint64_t        tmp_pl_inode = 0;
         pl_inode_t      *pl_inode = NULL;
+        char            *pathname = NULL;
+        gf_boolean_t    section_added = _gf_false;
 
         int count      = 0;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
+        if (!inode) {
+                errno = EINVAL;
+                goto out;
+        }
 
-        if (ret != 0)
+        ret = TRY_LOCK (&inode->lock);
+        if (ret)
+                goto out;
+        {
+                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
+                if (ret)
+                        goto unlock;
+        }
+unlock:
+        UNLOCK (&inode->lock);
+        if (ret)
                 goto out;
 
         pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
-
         if (!pl_inode) {
                 ret = -1;
                 goto out;
         }
 
         gf_proc_dump_add_section("xlator.features.locks.%s.inode", this->name);
+        section_added = _gf_true;
+
+        /*We are safe to call __inode_path since we have the
+         * inode->table->lock */
+        __inode_path (inode, NULL, &pathname);
+        if (pathname)
+                gf_proc_dump_write ("path", "%s", pathname);
 
         gf_proc_dump_write("mandatory", "%d", pl_inode->mandatory);
 
-        count = get_entrylk_count (this, inode);
-        if (count) {
-                gf_proc_dump_write("entrylk-count", "%d", count);
-                dump_entrylks(pl_inode);
-        }
+        ret = pthread_mutex_trylock (&pl_inode->mutex);
+        if (ret)
+                goto out;
+        {
+                count = __get_entrylk_count (this, pl_inode);
+                if (count) {
+                        gf_proc_dump_write("entrylk-count", "%d", count);
+                        __dump_entrylks (pl_inode);
+                }
 
-        count = get_inodelk_count (this, inode);
-        if (count) {
-                gf_proc_dump_write("inodlk-count", "%d", count);
-                dump_inodelks(pl_inode);
-        }
+                count = __get_inodelk_count (this, pl_inode, NULL);
+                if (count) {
+                        gf_proc_dump_write("inodelk-count", "%d", count);
+                        __dump_inodelks (pl_inode);
+                }
 
-        count = get_posixlk_count (this, inode);
-        if (count) {
-                gf_proc_dump_write("posixlk-count", "%d", count);
-                dump_posixlks(pl_inode);
+                count = __get_posixlk_count (this, pl_inode);
+                if (count) {
+                        gf_proc_dump_write("posixlk-count", "%d", count);
+                        __dump_posixlks (pl_inode);
+                }
         }
+        pthread_mutex_unlock (&pl_inode->mutex);
 
 out:
+        GF_FREE (pathname);
+
+        if (ret && inode) {
+                if (!section_added)
+                        gf_proc_dump_add_section ("xlator.features.locks.%s."
+                                                  "inode", this->name);
+                gf_proc_dump_write ("Unable to print lock state", "(Lock "
+                                    "acquisition failure) %s",
+                                    uuid_utoa (inode->gfid));
+        }
         return ret;
 }
 
@@ -1980,6 +2505,124 @@ mem_acct_init (xlator_t *this)
         return ret;
 }
 
+
+pl_ctx_t*
+pl_ctx_get (client_t *client, xlator_t *xlator)
+{
+        void *tmp = NULL;
+        pl_ctx_t *ctx = NULL;
+
+        client_ctx_get (client, xlator, &tmp);
+
+        ctx = tmp;
+
+        if (ctx != NULL)
+                goto out;
+
+        ctx = GF_CALLOC (1, sizeof (pl_ctx_t), gf_locks_mt_posix_lock_t);
+
+        if (ctx == NULL)
+                goto out;
+
+        ctx->ltable = pl_lock_table_new();
+
+        if (ctx->ltable == NULL) {
+                GF_FREE (ctx);
+                ctx = NULL;
+                goto out;
+        }
+
+        LOCK_INIT (&ctx->ltable_lock);
+
+        if (client_ctx_set (client, xlator, ctx) != 0) {
+                LOCK_DESTROY (&ctx->ltable_lock);
+                GF_FREE (ctx->ltable);
+                GF_FREE (ctx);
+                ctx = NULL;
+        }
+out:
+        return ctx;
+}
+
+static void
+ltable_delete_locks (struct _lock_table *ltable)
+{
+        struct _locker *locker = NULL;
+        struct _locker *tmp    = NULL;
+
+        list_for_each_entry_safe (locker, tmp, &ltable->inodelk_lockers, lockers) {
+                if (locker->fd)
+                        pl_del_locker (ltable, locker->volume, &locker->loc,
+                                       locker->fd, &locker->owner,
+                                       GF_FOP_INODELK);
+                GF_FREE (locker->volume);
+                GF_FREE (locker);
+        }
+
+        list_for_each_entry_safe (locker, tmp, &ltable->entrylk_lockers, lockers) {
+                if (locker->fd)
+                        pl_del_locker (ltable, locker->volume, &locker->loc,
+                                       locker->fd, &locker->owner,
+                                       GF_FOP_ENTRYLK);
+                GF_FREE (locker->volume);
+                GF_FREE (locker);
+        }
+        GF_FREE (ltable);
+}
+
+
+static int32_t
+destroy_cbk (xlator_t *this, client_t *client)
+{
+        void     *tmp       = NULL;
+        pl_ctx_t *locks_ctx = NULL;
+
+        client_ctx_del (client, this, &tmp);
+
+        if (tmp == NULL)
+                return 0
+;
+        locks_ctx = tmp;
+        if (locks_ctx->ltable)
+                ltable_delete_locks (locks_ctx->ltable);
+
+        LOCK_DESTROY (&locks_ctx->ltable_lock);
+        GF_FREE (locks_ctx);
+
+        return 0;
+}
+
+
+static int32_t
+disconnect_cbk (xlator_t *this, client_t *client)
+{
+        int32_t              ret       = 0;
+        pl_ctx_t            *locks_ctx = NULL;
+        struct _lock_table  *ltable    = NULL;
+
+        locks_ctx = pl_ctx_get (client, this);
+        if (locks_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+                goto out;
+        }
+
+        LOCK (&locks_ctx->ltable_lock);
+        {
+                if (locks_ctx->ltable) {
+                        ltable = locks_ctx->ltable;
+                        locks_ctx->ltable = pl_lock_table_new ();
+                }
+        }
+        UNLOCK (&locks_ctx->ltable_lock);
+
+        if (ltable)
+                ltable_delete_locks (ltable);
+
+out:
+        return ret;
+}
+
+
 int
 init (xlator_t *this)
 {
@@ -2008,7 +2651,7 @@ init (xlator_t *this)
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "'locks' translator is not loaded over a storage "
                         "translator");
-                goto out;;
+                goto out;
         }
 
         priv = GF_CALLOC (1, sizeof (*priv),
@@ -2029,13 +2672,20 @@ init (xlator_t *this)
                 }
         }
 
+        this->local_pool = mem_pool_new (pl_local_t, 32);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
         this->private = priv;
         ret = 0;
 
 out:
         if (ret) {
-                if (priv)
-                        GF_FREE (priv);
+                GF_FREE (priv);
         }
         return ret;
 }
@@ -2050,6 +2700,7 @@ fini (xlator_t *this)
         if (!priv)
                 return 0;
         this->private = NULL;
+        GF_FREE (priv->brickname);
         GF_FREE (priv);
 
         return 0;
@@ -2058,21 +2709,23 @@ fini (xlator_t *this)
 
 int
 pl_inodelk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock);
+            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
+            dict_t *xdata);
 
 int
 pl_finodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock);
+             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+             dict_t *xdata);
 
 int
 pl_entrylk (call_frame_t *frame, xlator_t *this,
             const char *volume, loc_t *loc, const char *basename,
-            entrylk_cmd cmd, entrylk_type type);
+            entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 int
 pl_fentrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, fd_t *fd, const char *basename,
-             entrylk_cmd cmd, entrylk_type type);
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 struct xlator_fops fops = {
         .lookup      = pl_lookup,
@@ -2091,6 +2744,8 @@ struct xlator_fops fops = {
         .opendir     = pl_opendir,
         .readdirp    = pl_readdirp,
         .getxattr    = pl_getxattr,
+        .fgetxattr   = pl_fgetxattr,
+        .fsetxattr   = pl_fsetxattr,
 };
 
 struct xlator_dumpops dumpops = {
@@ -2098,9 +2753,11 @@ struct xlator_dumpops dumpops = {
 };
 
 struct xlator_cbks cbks = {
-        .forget      = pl_forget,
-        .release     = pl_release,
-        .releasedir  = pl_releasedir,
+        .forget            = pl_forget,
+        .release           = pl_release,
+        .releasedir        = pl_releasedir,
+        .client_destroy    = destroy_cbk,
+        .client_disconnect = disconnect_cbk,
 };
 
 
diff --git a/xlators/features/locks/src/reservelk.c b/xlators/features/locks/src/reservelk.c
index 7a75cdea1..11abd26d8 100644
--- a/xlators/features/locks/src/reservelk.c
+++ b/xlators/features/locks/src/reservelk.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -299,7 +289,8 @@ grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode)
                         lock->user_flock.l_start,
                         lock->user_flock.l_len);
 
-                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0, &lock->user_flock);
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0, &lock->user_flock,
+                                     NULL);
         }
 
 }
@@ -376,7 +367,9 @@ grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode)
                                 pl_trace_out (this, lock->frame, fd, NULL, cmd,
                                               &lock->user_flock, -1, EAGAIN, NULL);
                                 pl_update_refkeeper (this, fd->inode);
-                                STACK_UNWIND_STRICT (lk, lock->frame, -1, EAGAIN, &lock->user_flock);
+                                STACK_UNWIND_STRICT (lk, lock->frame, -1,
+                                                     EAGAIN, &lock->user_flock,
+                                                     NULL);
                                 __destroy_lock (lock);
                         }
                 }
diff --git a/xlators/features/locks/tests/unit-test.c b/xlators/features/locks/tests/unit-test.c
index e95612ad4..d2cca32de 100644
--- a/xlators/features/locks/tests/unit-test.c
+++ b/xlators/features/locks/tests/unit-test.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/features/mac-compat/src/Makefile.am b/xlators/features/mac-compat/src/Makefile.am
index 915c13e30..f8567edce 100644
--- a/xlators/features/mac-compat/src/Makefile.am
+++ b/xlators/features/mac-compat/src/Makefile.am
@@ -1,13 +1,14 @@
 xlator_LTLIBRARIES = mac-compat.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-mac_compat_la_LDFLAGS = -module -avoidversion
+mac_compat_la_LDFLAGS = -module -avoid-version
 
 mac_compat_la_SOURCES = mac-compat.c
 mac_compat_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/mac-compat/src/mac-compat.c b/xlators/features/mac-compat/src/mac-compat.c
index 188b593e4..7cb550ad5 100644
--- a/xlators/features/mac-compat/src/mac-compat.c
+++ b/xlators/features/mac-compat/src/mac-compat.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -57,7 +47,8 @@ static int32_t apple_xattr_len[] = {
 
 int32_t
 maccomp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
         intptr_t ax = (intptr_t)this->private;
         int i = 0;
@@ -80,7 +71,7 @@ maccomp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
          }
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
 
         return 0;
 }
@@ -88,7 +79,7 @@ maccomp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 maccomp_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  const char *name)
+                  const char *name, dict_t *xdata)
 {
         intptr_t ax = GF_XATTR_NONE;
         int i = 0;
@@ -109,14 +100,14 @@ maccomp_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         STACK_WIND (frame, maccomp_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr,
-                    loc, name);
+                    loc, name, xdata);
         return 0;
 }
 
 
 int32_t
 maccomp_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name)
+                   const char *name, dict_t *xdata)
 {
         intptr_t ax = GF_XATTR_NONE;
         int i = 0;
@@ -137,21 +128,21 @@ maccomp_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         STACK_WIND (frame, maccomp_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name);
+                    fd, name, xdata);
         return 0;
 }
 
 
 int32_t
 maccomp_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         intptr_t ax = (intptr_t)this->private;
 
         if (op_ret == -1 && ax != GF_XATTR_NONE)
                 op_ret = op_errno = 0;
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -159,7 +150,7 @@ maccomp_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 maccomp_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                  int32_t flags)
+                  int32_t flags, dict_t *xdata)
 {
         intptr_t ax = GF_XATTR_NONE;
         int i = 0;
@@ -177,14 +168,14 @@ maccomp_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
         STACK_WIND (frame, maccomp_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    loc, dict, flags, xdata);
         return 0;
 }
 
 
 int32_t
 maccomp_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                   int32_t flags)
+                   int32_t flags, dict_t *xdata)
 {
         intptr_t ax = GF_XATTR_NONE;
         int i = 0;
@@ -202,7 +193,7 @@ maccomp_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
         STACK_WIND (frame, maccomp_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    fd, dict, flags, xdata);
         return 0;
 }
 
@@ -239,8 +230,7 @@ struct xlator_fops fops = {
         .fsetxattr   = maccomp_fsetxattr,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
         { .key = {NULL} },
diff --git a/xlators/features/marker/Makefile.am b/xlators/features/marker/Makefile.am
index a6ba2de16..a985f42a8 100644
--- a/xlators/features/marker/Makefile.am
+++ b/xlators/features/marker/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = src @SYNCDAEMON_SUBDIR@
+SUBDIRS = src
 
 CLEANFILES =
diff --git a/xlators/features/marker/src/Makefile.am b/xlators/features/marker/src/Makefile.am
index 501586a76..a7c676472 100644
--- a/xlators/features/marker/src/Makefile.am
+++ b/xlators/features/marker/src/Makefile.am
@@ -1,15 +1,17 @@
 xlator_LTLIBRARIES = marker.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-marker_la_LDFLAGS = -module -avoidversion
+marker_la_LDFLAGS = -module -avoid-version
 
 marker_la_SOURCES = marker.c marker-quota.c marker-quota-helper.c marker-common.c
 marker_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = marker-mem-types.h marker.h marker-quota.h marker-quota-helper.h marker-common.h $(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -fno-strict-aliasing -D$(GF_HOST_OS) \
-        -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src $(GF_CFLAGS) -shared -nostartfiles
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/lib/src
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
 CLEANFILES =
 
diff --git a/xlators/features/marker/src/marker-common.c b/xlators/features/marker/src/marker-common.c
index a413781bc..84a718add 100644
--- a/xlators/features/marker/src/marker-common.c
+++ b/xlators/features/marker/src/marker-common.c
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -69,18 +60,10 @@ unlock: UNLOCK (&inode->lock);
         return ret;
 }
 
-void
+int
 marker_filter_quota_xattr (dict_t *dict, char *key,
                            data_t *value, void *data)
 {
-        int ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", dict, out);
-        GF_VALIDATE_OR_GOTO ("marker", key, out);
-
-        ret = fnmatch ("trusted.glusterfs.quota*", key, 0);
-        if (ret == 0)
-                dict_del (dict, key);
-out:
-        return;
+        dict_del (dict, key);
+        return 0;
 }
diff --git a/xlators/features/marker/src/marker-common.h b/xlators/features/marker/src/marker-common.h
index 0a7ee2619..23dd846cb 100644
--- a/xlators/features/marker/src/marker-common.h
+++ b/xlators/features/marker/src/marker-common.h
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _MARKER_COMMON_H
 #define _MARKER_COMMON_H
 
@@ -31,6 +22,6 @@
 int32_t
 marker_force_inode_ctx_get (inode_t *, xlator_t *, marker_inode_ctx_t **);
 
-void
+int
 marker_filter_quota_xattr (dict_t *, char *, data_t *, void *);
 #endif
diff --git a/xlators/features/marker/src/marker-mem-types.h b/xlators/features/marker/src/marker-mem-types.h
index f2723dc26..1f74d5048 100644
--- a/xlators/features/marker/src/marker-mem-types.h
+++ b/xlators/features/marker/src/marker-mem-types.h
@@ -1,36 +1,24 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __MARKER_MEM_TYPES_H__
 #define __MARKER_MEM_TYPES_H__
 
 #include "mem-types.h"
 
 enum gf_marker_mem_types_ {
-        gf_marker_mt_marker_local_t = gf_common_mt_end + 1,
-        gf_marker_mt_marker_conf_t,
+        gf_marker_mt_marker_conf_t = gf_common_mt_end + 1,
         gf_marker_mt_loc_t,
         gf_marker_mt_volume_mark,
         gf_marker_mt_int64_t,
         gf_marker_mt_quota_inode_ctx_t,
         gf_marker_mt_marker_inode_ctx_t,
-        gf_marker_mt_quota_local_t,
         gf_marker_mt_inode_contribution_t,
         gf_marker_mt_end
 };
diff --git a/xlators/features/marker/src/marker-quota-helper.c b/xlators/features/marker/src/marker-quota-helper.c
index 7b7c3026f..af5fed132 100644
--- a/xlators/features/marker/src/marker-quota-helper.c
+++ b/xlators/features/marker/src/marker-quota-helper.c
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -168,8 +159,16 @@ __mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx, loc_t *l
         int32_t ret = 0;
         inode_contribution_t *contribution = NULL;
 
-        if (!loc->parent)
-                goto out;
+        if (!loc->parent) {
+                if (!uuid_is_null (loc->pargfid))
+                        loc->parent = inode_find (loc->inode->table,
+                                                  loc->pargfid);
+                if (!loc->parent)
+                        loc->parent = inode_parent (loc->inode, loc->pargfid,
+                                                    loc->name);
+                if (!loc->parent)
+                        goto out;
+        }
 
         list_for_each_entry (contribution, &ctx->contribution_head, contri_list) {
                 if (loc->parent &&
@@ -328,21 +327,15 @@ mq_inode_ctx_new (inode_t * inode, xlator_t *this)
 quota_local_t *
 mq_local_new ()
 {
-        int32_t         ret     = -1;
         quota_local_t  *local   = NULL;
 
-        QUOTA_ALLOC (local, quota_local_t, ret);
-        if (ret < 0)
+        local = mem_get0 (THIS->local_pool);
+        if (!local)
                 goto out;
 
         local->ref = 1;
-        local->delta = 0;
-        local->err = 0;
         LOCK_INIT (&local->lock);
 
-        memset (&local->loc, 0, sizeof (loc_t));
-        memset (&local->parent_loc, 0, sizeof (loc_t));
-
         local->ctx = NULL;
         local->contri = NULL;
 
@@ -384,7 +377,7 @@ mq_local_unref (xlator_t *this, quota_local_t *local)
 
         LOCK_DESTROY (&local->lock);
 
-        GF_FREE (local);
+        mem_put (local);
 out:
         return 0;
 }
diff --git a/xlators/features/marker/src/marker-quota-helper.h b/xlators/features/marker/src/marker-quota-helper.h
index 56080f0dc..6cdd14881 100644
--- a/xlators/features/marker/src/marker-quota-helper.h
+++ b/xlators/features/marker/src/marker-quota-helper.h
@@ -1,20 +1,13 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
+
 #ifndef _MARKER_QUOTA_HELPER_H
 #define _MARKER_QUOTA_HELPER
 
@@ -23,7 +16,7 @@
 #include "config.h"
 #endif
 
-#include "marker-quota.h"
+#include "marker.h"
 
 #define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)     \
         do {                                            \
diff --git a/xlators/features/marker/src/marker-quota.c b/xlators/features/marker/src/marker-quota.c
index bc5fc6040..6f9af6e13 100644
--- a/xlators/features/marker/src/marker-quota.c
+++ b/xlators/features/marker/src/marker-quota.c
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -208,7 +199,7 @@ out:
 
 int32_t
 mq_dirty_inode_updation_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno)
+                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         QUOTA_STACK_DESTROY (frame, this);
 
@@ -217,7 +208,7 @@ mq_dirty_inode_updation_done (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno)
+                                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         struct gf_flock   lock  = {0, };
         quota_local_t    *local = NULL;
@@ -229,7 +220,7 @@ mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *thi
         if (op_ret == -1) {
                 local->err = -1;
 
-                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0);
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
 
                 return 0;
         }
@@ -247,7 +238,7 @@ mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *thi
 	if (ret == -1) {
                 local->err = -1;
                 frame->local = NULL;
-                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0);
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
                 return 0;
         }
 
@@ -261,20 +252,21 @@ mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *thi
                     mq_dirty_inode_updation_done,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &loc, F_SETLKW, &lock);
+                    this->name, &loc, F_SETLKW, &lock, NULL);
 
         loc_wipe (&loc);
 
         return 0;
 out:
-        mq_dirty_inode_updation_done (frame, NULL, this, -1, 0);
+        mq_dirty_inode_updation_done (frame, NULL, this, -1, 0, NULL);
 
         return 0;
 }
 
 int32_t
 mq_mark_inode_undirty (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
         int32_t        ret     = -1;
         int64_t       *size    = NULL;
@@ -315,14 +307,14 @@ wind:
         STACK_WIND (frame, mq_release_lock_on_dirty_inode,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    &local->loc, newdict, 0);
+                    &local->loc, newdict, 0, NULL);
         ret = 0;
 
 err:
         if (op_ret == -1 || ret == -1) {
                 local->err = -1;
 
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict)
@@ -379,13 +371,13 @@ mq_update_size_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto err;
 
         if (uuid_is_null (local->loc.gfid))
-                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
 
         GF_UUID_ASSERT (local->loc.gfid);
 
         STACK_WIND (frame, mq_mark_inode_undirty, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop, &local->loc,
-                    GF_XATTROP_ADD_ARRAY64, new_dict);
+                    GF_XATTROP_ADD_ARRAY64, new_dict, NULL);
 
         ret = 0;
 
@@ -393,7 +385,7 @@ err:
         if (op_ret == -1 || ret == -1) {
                 local->err = -1;
 
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
         }
 
         if (new_dict)
@@ -457,7 +449,7 @@ err:
         if (ret) {
                 local->err = -1;
 
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
         }
 
         if (dict)
@@ -495,7 +487,8 @@ mq_get_child_contribution (call_frame_t *frame,
                 val = -2;
                 if (!mq_test_and_set_local_err (local, &val) &&
                     val != -2)
-                        mq_release_lock_on_dirty_inode (local->frame, NULL, this, 0, 0);
+                        mq_release_lock_on_dirty_inode (local->frame, NULL,
+                                                        this, 0, 0, NULL);
 
                 goto exit;
         }
@@ -523,7 +516,7 @@ out:
 
         if (val == 0) {
                 mq_dirty_inode_readdir (local->frame, NULL, this,
-                                                   0, 0, NULL);
+                                        0, 0, NULL, NULL);
         }
         mq_local_unref (this, local);
 
@@ -539,7 +532,7 @@ mq_readdir_cbk (call_frame_t *frame,
                 xlator_t *this,
                 int32_t op_ret,
                 int32_t op_errno,
-                gf_dirent_t *entries)
+                gf_dirent_t *entries, dict_t *xdata)
 {
         char           contri_key [512]   = {0, };
         int32_t        ret                = 0;
@@ -559,7 +552,7 @@ mq_readdir_cbk (call_frame_t *frame,
                         "readdir failed %s", strerror (op_errno));
                 local->err = -1;
 
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
 
                 goto end;
         } else if (op_ret == 0) {
@@ -681,7 +674,7 @@ mq_readdir_cbk (call_frame_t *frame,
         }
 
         if (ret && val != -2) {
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
         }
 end:
         mq_local_unref (this, local);
@@ -695,7 +688,7 @@ mq_dirty_inode_readdir (call_frame_t *frame,
                         xlator_t *this,
                         int32_t op_ret,
                         int32_t op_errno,
-                        fd_t *fd)
+                        fd_t *fd, dict_t *xdata)
 {
         quota_local_t *local = NULL;
 
@@ -703,7 +696,7 @@ mq_dirty_inode_readdir (call_frame_t *frame,
 
         if (op_ret == -1) {
                 local->err = -1;
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
                 return 0;
         }
 
@@ -714,7 +707,7 @@ mq_dirty_inode_readdir (call_frame_t *frame,
                     mq_readdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdir,
-                    local->fd, READDIR_BUF, local->d_off);
+                    local->fd, READDIR_BUF, local->d_off, xdata);
 
         return 0;
 }
@@ -754,7 +747,7 @@ mq_check_if_still_dirty (call_frame_t *frame,
 
         //the inode is not dirty anymore
         if (dirty == 0) {
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
 
                 return 0;
         }
@@ -764,21 +757,21 @@ mq_check_if_still_dirty (call_frame_t *frame,
         local->d_off = 0;
 
         if (uuid_is_null (local->loc.gfid))
-                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
 
         GF_UUID_ASSERT (local->loc.gfid);
         STACK_WIND(frame,
                    mq_dirty_inode_readdir,
                    FIRST_CHILD(this),
                    FIRST_CHILD(this)->fops->opendir,
-                   &local->loc, fd);
+                   &local->loc, fd, NULL);
 
         ret = 0;
 
 err:
         if (op_ret == -1 || ret == -1) {
                 local->err = -1;
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
         }
 
         if (fd != NULL) {
@@ -789,15 +782,15 @@ err:
 }
 
 int32_t
-mq_get_dirty_xattr (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int32_t op_ret, int32_t op_errno)
+mq_get_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t        ret       = -1;
         dict_t        *xattr_req = NULL;
         quota_local_t *local     = NULL;
 
         if (op_ret == -1) {
-                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0);
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
                 return 0;
         }
 
@@ -829,7 +822,7 @@ mq_get_dirty_xattr (call_frame_t *frame, void *cookie,
 err:
         if (ret) {
                 local->err = -1;
-                mq_release_lock_on_dirty_inode(frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode(frame, NULL, this, 0, 0, NULL);
         }
 
         if (xattr_req)
@@ -896,7 +889,7 @@ mq_update_dirty_inode (xlator_t *this,
                     mq_get_dirty_xattr,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->loc, F_SETLKW, &lock);
+                    this->name, &local->loc, F_SETLKW, &lock, NULL);
         return 1;
 
 fr_destroy:
@@ -909,7 +902,7 @@ out:
 
 int32_t
 mq_inode_creation_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         quota_local_t *local = NULL;
 
@@ -931,7 +924,7 @@ mq_inode_creation_done (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 mq_xattr_creation_release_lock (call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
-                                int32_t op_errno)
+                                int32_t op_errno, dict_t *xdata)
 {
         struct gf_flock  lock  = {0, };
         quota_local_t   *local = NULL;
@@ -949,7 +942,7 @@ mq_xattr_creation_release_lock (call_frame_t *frame, void *cookie,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
                     this->name, &local->loc,
-                    F_SETLKW, &lock);
+                    F_SETLKW, &lock, NULL);
 
         return 0;
 }
@@ -957,7 +950,8 @@ mq_xattr_creation_release_lock (call_frame_t *frame, void *cookie,
 
 int32_t
 mq_create_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
         int32_t          ret       = -1;
         dict_t          *newdict   = NULL;
@@ -986,16 +980,16 @@ mq_create_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
                 STACK_WIND (frame, mq_xattr_creation_release_lock,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->setxattr,
-                            &local->loc, newdict, 0);
+                            &local->loc, newdict, 0, NULL);
         } else {
-                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0);
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         ret = 0;
 
 err:
         if (ret < 0) {
-                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0);
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict != NULL)
@@ -1057,12 +1051,11 @@ mq_create_xattr (xlator_t *this, call_frame_t *frame)
                         goto free_value;
         }
 
-        uuid_copy (local->loc.gfid, local->loc.inode->gfid);
         GF_UUID_ASSERT (local->loc.gfid);
 
         STACK_WIND (frame, mq_create_dirty_xattr, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop, &local->loc,
-                    GF_XATTROP_ADD_ARRAY64, dict);
+                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
         ret = 0;
 
 free_size:
@@ -1080,7 +1073,7 @@ err:
 
 out:
         if (ret < 0) {
-                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0);
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         return 0;
@@ -1125,10 +1118,14 @@ mq_check_n_set_inode_xattr (call_frame_t *frame, void *cookie,
         }
 
 out:
-        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0);
+        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
         return 0;
 
 create_xattr:
+        if (uuid_is_null (local->loc.gfid)) {
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+        }
+
         mq_create_xattr (this, frame);
         return 0;
 }
@@ -1136,7 +1133,7 @@ create_xattr:
 
 int32_t
 mq_get_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         dict_t        *xattr_req = NULL;
         quota_local_t *local     = NULL;
@@ -1172,14 +1169,14 @@ mq_get_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 
 err:
-        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0);
+        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
 
         if (xattr_req)
                 dict_unref (xattr_req);
         return 0;
 
 lock_err:
-        mq_inode_creation_done (frame, NULL, this, 0, 0);
+        mq_inode_creation_done (frame, NULL, this, 0, 0, NULL);
         return 0;
 }
 
@@ -1221,7 +1218,7 @@ mq_set_inode_xattr (xlator_t *this, loc_t *loc)
                     mq_get_xattr,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->loc, F_SETLKW, &lock);
+                    this->name, &local->loc, F_SETLKW, &lock, NULL);
 
         return 0;
 
@@ -1294,7 +1291,7 @@ mq_xattr_updation_done (call_frame_t *frame,
                         xlator_t *this,
                         int32_t op_ret,
                         int32_t op_errno,
-                        dict_t *dict)
+                        dict_t *dict, dict_t *xdata)
 {
         QUOTA_STACK_DESTROY (frame, this);
         return 0;
@@ -1303,7 +1300,7 @@ mq_xattr_updation_done (call_frame_t *frame,
 
 int32_t
 mq_inodelk_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno)
+                xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t         ret    = 0;
         gf_boolean_t    status = _gf_false;
@@ -1317,7 +1314,7 @@ mq_inodelk_cbk (call_frame_t *frame, void *cookie,
                                 "unlocking failed on path (%s)(%s)",
                                 local->parent_loc.path, strerror (op_errno));
                 }
-                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL);
+                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL, NULL);
 
                 return 0;
         }
@@ -1327,11 +1324,12 @@ mq_inodelk_cbk (call_frame_t *frame, void *cookie,
 
         if ((strcmp (local->parent_loc.path, "/") == 0)
             || (local->delta == 0)) {
-                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL);
+                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL, NULL);
         } else {
                 ret = mq_get_parent_inode_local (this, local);
                 if (ret < 0) {
-                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL);
+                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL,
+                                                NULL);
                         goto out;
                 }
                 status = _gf_true;
@@ -1340,7 +1338,8 @@ mq_inodelk_cbk (call_frame_t *frame, void *cookie,
                 if (ret == 0 && status == _gf_false) {
                         mq_get_lock_on_parent (frame, this);
                 } else {
-                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL);
+                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL,
+                                                NULL);
                 }
         }
 out:
@@ -1352,7 +1351,7 @@ out:
 int32_t
 mq_release_parent_lock (call_frame_t *frame, void *cookie,
                         xlator_t *this, int32_t op_ret,
-                        int32_t op_errno)
+                        int32_t op_errno, dict_t *xdata)
 {
         int32_t            ret      = 0;
         quota_local_t     *local    = NULL;
@@ -1381,7 +1380,7 @@ mq_release_parent_lock (call_frame_t *frame, void *cookie,
         UNLOCK (&ctx->lock);
 
         if (local->parent_loc.inode == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Invalid parent inode.");
                 goto err;
         }
@@ -1398,12 +1397,12 @@ wind:
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
                     this->name, &local->parent_loc,
-                    F_SETLKW, &lock);
+                    F_SETLKW, &lock, NULL);
 
         return 0;
 err:
         mq_xattr_updation_done (frame, NULL, this,
-                             0, 0 , NULL);
+                                0, 0 , NULL, NULL);
         return 0;
 }
 
@@ -1414,7 +1413,7 @@ mq_mark_undirty (call_frame_t *frame,
                  xlator_t *this,
                  int32_t op_ret,
                  int32_t op_errno,
-                 dict_t *dict)
+                 dict_t *dict, dict_t *xdata)
 {
         int32_t            ret          = -1;
         int64_t           *size         = NULL;
@@ -1475,14 +1474,14 @@ mq_mark_undirty (call_frame_t *frame,
         STACK_WIND (frame, mq_release_parent_lock,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    &local->parent_loc, newdict, 0);
+                    &local->parent_loc, newdict, 0, NULL);
 
         ret = 0;
 err:
         if (op_ret == -1 || ret == -1) {
                 local->err = op_errno;
 
-                mq_release_parent_lock (frame, NULL, this, 0, 0);
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict)
@@ -1498,7 +1497,7 @@ mq_update_parent_size (call_frame_t *frame,
                        xlator_t *this,
                        int32_t op_ret,
                        int32_t op_errno,
-                       dict_t *dict)
+                       dict_t *dict, dict_t *xdata)
 {
         int64_t             *size       = NULL;
         int32_t              ret        = -1;
@@ -1565,12 +1564,12 @@ mq_update_parent_size (call_frame_t *frame,
                     FIRST_CHILD(this)->fops->xattrop,
                     &local->parent_loc,
                     GF_XATTROP_ADD_ARRAY64,
-                    newdict);
+                    newdict, NULL);
         ret = 0;
 err:
         if (op_ret == -1 || ret < 0) {
                 local->err = op_errno;
-                mq_release_parent_lock (frame, NULL, this, 0, 0);
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict)
@@ -1658,7 +1657,7 @@ unlock:
         local->delta = size_int - contri_int;
 
         if (local->delta == 0) {
-                mq_mark_undirty (frame, NULL, this, 0, 0, NULL);
+                mq_mark_undirty (frame, NULL, this, 0, 0, NULL, NULL);
                 return 0;
         }
 
@@ -1681,7 +1680,7 @@ unlock:
         }
 
         if (uuid_is_null (local->loc.gfid))
-                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
 
         GF_UUID_ASSERT (local->loc.gfid);
 
@@ -1691,14 +1690,14 @@ unlock:
                     FIRST_CHILD(this)->fops->xattrop,
                     &local->loc,
                     GF_XATTROP_ADD_ARRAY64,
-                    newdict);
+                    newdict, NULL);
         ret = 0;
 
 err:
         if (op_ret == -1 || ret < 0) {
                 local->err = op_errno;
 
-                mq_release_parent_lock (frame, NULL, this, 0, 0);
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict)
@@ -1710,7 +1709,7 @@ err:
 int32_t
 mq_fetch_child_size_and_contri (call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
-                                int32_t op_errno)
+                                int32_t op_errno, dict_t *xdata)
 {
         int32_t            ret              = -1;
         char               contri_key [512] = {0, };
@@ -1792,7 +1791,7 @@ err:
 
                 mq_set_ctx_updation_status (local->ctx, _gf_false);
 
-                mq_release_parent_lock (frame, NULL, this, 0, 0);
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict)
@@ -1803,7 +1802,7 @@ err:
 
 int32_t
 mq_markdirty (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno)
+              xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t        ret      = -1;
         dict_t        *dict     = NULL;
@@ -1820,7 +1819,7 @@ mq_markdirty (call_frame_t *frame, void *cookie,
 
                 mq_set_ctx_updation_status (local->ctx, _gf_false);
 
-                mq_inodelk_cbk (frame, NULL, this, 0, 0);
+                mq_inodelk_cbk (frame, NULL, this, 0, 0, NULL);
 
                 return 0;
         }
@@ -1845,7 +1844,7 @@ mq_markdirty (call_frame_t *frame, void *cookie,
         STACK_WIND (frame, mq_fetch_child_size_and_contri,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    &local->parent_loc, dict, 0);
+                    &local->parent_loc, dict, 0, NULL);
 
         ret = 0;
 err:
@@ -1854,7 +1853,7 @@ err:
 
                 mq_set_ctx_updation_status (local->ctx, _gf_false);
 
-                mq_release_parent_lock (frame, NULL, this, 0, 0);
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (dict)
@@ -1892,7 +1891,7 @@ mq_get_lock_on_parent (call_frame_t *frame, xlator_t *this)
                     mq_markdirty,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc, F_SETLKW, &lock);
+                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
 
         return 0;
 
@@ -2035,8 +2034,9 @@ mq_inspect_directory_xattr (xlator_t *this,
         if (strcmp (loc->path, "/") != 0) {
                 contribution = mq_add_new_contribution_node (this, ctx, loc);
                 if (contribution == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot add a new contribution node");
+                        if (!uuid_is_null (loc->inode->gfid))
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "cannot add a new contribution node");
                         ret = -1;
                         goto err;
                 }
@@ -2221,7 +2221,7 @@ out:
 
 int32_t
 mq_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         QUOTA_STACK_DESTROY (frame, this);
 
@@ -2230,7 +2230,7 @@ mq_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 _mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t        ret                = 0;
         char           contri_key [512]   = {0, };
@@ -2239,7 +2239,7 @@ _mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
         local = (quota_local_t *) frame->local;
 
         if (op_ret == -1 || local->err == -1) {
-                mq_removexattr_cbk (frame, NULL, this, -1, 0);
+                mq_removexattr_cbk (frame, NULL, this, -1, 0, NULL);
                 return 0;
         }
 
@@ -2250,10 +2250,10 @@ _mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STACK_WIND (frame, mq_removexattr_cbk, FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->removexattr,
-                            &local->loc, contri_key);
+                            &local->loc, contri_key, NULL);
                 ret = 0;
         } else {
-                mq_removexattr_cbk (frame, NULL, this, 0, 0);
+                mq_removexattr_cbk (frame, NULL, this, 0, 0, NULL);
         }
 
         if (strcmp (local->parent_loc.path, "/") != 0) {
@@ -2271,7 +2271,8 @@ out:
 
 int32_t
 mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
         int32_t            ret   = -1;
         struct gf_flock    lock  = {0, };
@@ -2318,13 +2319,13 @@ mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
                     this->name, &local->parent_loc,
-                    F_SETLKW, &lock);
+                    F_SETLKW, &lock, NULL);
         return 0;
 }
 
 int32_t
-mq_reduce_parent_size_xattr (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno)
+mq_reduce_parent_size_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t                  ret               = -1;
         int64_t                 *size              = NULL;
@@ -2361,13 +2362,13 @@ mq_reduce_parent_size_xattr (call_frame_t *frame, void *cookie,
 
         STACK_WIND (frame, mq_inode_remove_done, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop, &local->parent_loc,
-                    GF_XATTROP_ADD_ARRAY64, dict);
+                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
         dict_unref (dict);
         return 0;
 
 err:
         local->err = 1;
-        mq_inode_remove_done (frame, NULL, this, -1, 0, NULL);
+        mq_inode_remove_done (frame, NULL, this, -1, 0, NULL, NULL);
         if (dict)
                 dict_unref (dict);
         return 0;
@@ -2443,7 +2444,7 @@ mq_reduce_parent_size (xlator_t *this, loc_t *loc, int64_t contri)
 
         if (local->parent_loc.inode == NULL) {
                 ret = -1;
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Inode is NULL, so can't stackwind.");
                 goto out;
         }
@@ -2452,7 +2453,7 @@ mq_reduce_parent_size (xlator_t *this, loc_t *loc, int64_t contri)
                     mq_reduce_parent_size_xattr,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc, F_SETLKW, &lock);
+                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
         local = NULL;
         ret = 0;
 
diff --git a/xlators/features/marker/src/marker-quota.h b/xlators/features/marker/src/marker-quota.h
index 7a90b28b7..385760ac4 100644
--- a/xlators/features/marker/src/marker-quota.h
+++ b/xlators/features/marker/src/marker-quota.h
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _MARKER_QUOTA_H
 #define _MARKER_QUOTA_H
 
@@ -24,7 +15,6 @@
 #include "config.h"
 #endif
 
-#include "marker.h"
 #include "xlator.h"
 #include "marker-mem-types.h"
 
@@ -104,28 +94,6 @@ struct inode_contribution {
 };
 typedef struct inode_contribution inode_contribution_t;
 
-struct quota_local {
-        int64_t delta;
-        int64_t d_off;
-        int32_t err;
-        int32_t ref;
-        int64_t sum;
-        int64_t size;
-        int32_t hl_count;
-        int32_t dentry_child_count;
-
-        fd_t         *fd;
-        call_frame_t *frame;
-        gf_lock_t     lock;
-
-        loc_t loc;
-        loc_t parent_loc;
-
-        quota_inode_ctx_t    *ctx;
-        inode_contribution_t *contri;
-};
-typedef struct quota_local quota_local_t;
-
 int32_t
 mq_get_lock_on_parent (call_frame_t *, xlator_t *);
 
@@ -146,7 +114,7 @@ mq_initiate_quota_txn (xlator_t *, loc_t *);
 
 int32_t
 mq_dirty_inode_readdir (call_frame_t *, void *, xlator_t *,
-                        int32_t, int32_t, fd_t *);
+                        int32_t, int32_t, fd_t *, dict_t *);
 
 int32_t
 mq_reduce_parent_size (xlator_t *, loc_t *, int64_t);
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index 54a7e1a28..6a2c85691 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -65,23 +56,25 @@ marker_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
 
         if (inode) {
                 loc->inode = inode_ref (inode);
-		uuid_copy (loc->gfid, loc->inode->gfid);
+                if (uuid_is_null (loc->gfid)) {
+                        uuid_copy (loc->gfid, loc->inode->gfid);
+                }
         }
 
         if (parent)
                 loc->parent = inode_ref (parent);
 
-        loc->path = gf_strdup (path);
-        if (!loc->path) {
-                gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
-                goto loc_wipe;
-        }
+        if (path) {
+                loc->path = gf_strdup (path);
+                if (!loc->path) {
+                        gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
+                        goto loc_wipe;
+                }
 
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name)
-                loc->name++;
-        else
-                goto loc_wipe;
+                loc->name = strrchr (loc->path, '/');
+                if (loc->name)
+                        loc->name++;
+        }
 
         ret = 0;
 loc_wipe:
@@ -107,7 +100,7 @@ marker_inode_loc_fill (inode_t *inode, loc_t *loc)
         if (ret < 0)
                 goto err;
 
-        ret = marker_loc_fill (loc, inode, NULL, resolvedpath);
+        ret = marker_loc_fill (loc, inode, parent, resolvedpath);
         if (ret < 0)
                 goto err;
 
@@ -115,8 +108,7 @@ err:
 	if (parent)
 		inode_unref (parent);
 
-        if (resolvedpath)
-                GF_FREE (resolvedpath);
+        GF_FREE (resolvedpath);
 
         return ret;
 }
@@ -126,8 +118,17 @@ marker_trav_parent (marker_local_t *local)
 {
         int32_t ret = 0;
         loc_t   loc = {0, };
+        inode_t *parent = NULL;
+        int8_t  need_unref = 0;
+
+        if (!local->loc.parent) {
+                parent = inode_parent (local->loc.inode, NULL, NULL);
+                if (parent)
+                        need_unref = 1;
+        } else
+                parent = local->loc.parent;
 
-        ret = marker_inode_loc_fill (local->loc.parent, &loc);
+        ret = marker_inode_loc_fill (parent, &loc);
 
         if (ret < 0) {
                 ret = -1;
@@ -138,16 +139,28 @@ marker_trav_parent (marker_local_t *local)
 
         local->loc = loc;
 out:
+        if (need_unref)
+                inode_unref (parent);
+
         return ret;
 }
 
 int32_t
-marker_error_handler (xlator_t *this)
+marker_error_handler (xlator_t *this, marker_local_t *local, int32_t op_errno)
 {
-        marker_conf_t *priv = NULL;
+         marker_conf_t *priv = NULL;
+         const char    *path = NULL;
 
-        priv = (marker_conf_t *) this->private;
+         priv = (marker_conf_t *) this->private;
+         path = local
+                 ? (local->loc.path
+                    ? local->loc.path : uuid_utoa(local->loc.gfid))
+                 : "<nul>";
 
+         gf_log (this->name, GF_LOG_CRITICAL,
+                 "Indexing gone corrupt at %s (reason: %s)."
+                 " Geo-replication slave content needs to be revalidated",
+                 path, strerror (op_errno));
         unlink (priv->timestamp_file);
 
         return 0;
@@ -172,12 +185,14 @@ marker_local_unref (marker_local_t *local)
 
         loc_wipe (&local->loc);
         loc_wipe (&local->parent_loc);
+        if (local->xdata)
+                dict_unref (local->xdata);
 
         if (local->oplocal) {
                 marker_local_unref (local->oplocal);
                 local->oplocal = NULL;
         }
-        GF_FREE (local);
+        mem_put (local);
 out:
         return 0;
 }
@@ -212,13 +227,14 @@ stat_stampfile (xlator_t *this, marker_conf_t *priv,
 
 int32_t
 marker_getxattr_stampfile_cbk (call_frame_t *frame, xlator_t *this,
-                               const char *name, struct volume_mark *vol_mark)
+                               const char *name, struct volume_mark *vol_mark,
+                               dict_t *xdata)
 {
         int32_t   ret  = -1;
         dict_t   *dict = NULL;
 
         if (vol_mark == NULL){
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
 
                 goto out;
         }
@@ -231,7 +247,7 @@ marker_getxattr_stampfile_cbk (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_WARNING, "failed to set key %s",
                         name);
 
-        STACK_UNWIND_STRICT (getxattr, frame, 0, 0, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, 0, 0, dict, xdata);
 
         dict_unref (dict);
 out:
@@ -255,28 +271,31 @@ call_from_special_client (call_frame_t *frame, xlator_t *this, const char *name)
 
         stat_stampfile (this, priv, &vol_mark);
 
-        marker_getxattr_stampfile_cbk (frame, this, name, vol_mark);
+        marker_getxattr_stampfile_cbk (frame, this, name, vol_mark, NULL);
 out:
         return ret;
 }
 
 int32_t
 marker_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
         if (cookie) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Filtering the quota extended attributes");
 
-                dict_foreach (dict, marker_filter_quota_xattr, NULL);
+                dict_foreach_fnmatch (dict, "trusted.glusterfs.quota*",
+                                      marker_filter_quota_xattr, NULL);
         }
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 int32_t
 marker_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
         gf_boolean_t   ret    = _gf_false;
         marker_conf_t *priv   = NULL;
@@ -303,7 +322,7 @@ wind:
                 STACK_WIND_COOKIE (frame, marker_getxattr_cbk, (void *)cookie,
                                    FIRST_CHILD(this),
                                    FIRST_CHILD(this)->fops->getxattr, loc,
-                                   name);
+                                   name, xdata);
         }
 
         return 0;
@@ -328,7 +347,7 @@ marker_setxattr_done (call_frame_t *frame)
 
 int
 marker_specific_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno)
+                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t         ret   = 0;
         int32_t         done  = 0;
@@ -337,14 +356,20 @@ marker_specific_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local = (marker_local_t*) frame->local;
 
         if (op_ret == -1 && op_errno == ENOSPC) {
-                marker_error_handler (this);
+                marker_error_handler (this, local, op_errno);
                 done = 1;
                 goto out;
         }
 
-        if (strcmp (local->loc.path, "/") == 0) {
-                done = 1;
-                goto out;
+        if (local) {
+                if (local->loc.path && strcmp (local->loc.path, "/") == 0) {
+                        done = 1;
+                        goto out;
+                }
+                if (__is_root_gfid (local->loc.gfid)) {
+                        done = 1;
+                        goto out;
+                }
         }
 
         ret = marker_trav_parent (local);
@@ -402,7 +427,8 @@ marker_start_setxattr (call_frame_t *frame, xlator_t *this)
         }
 
         STACK_WIND (frame, marker_specific_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, &local->loc, dict, 0);
+                    FIRST_CHILD(this)->fops->setxattr, &local->loc, dict, 0,
+                    NULL);
 
         ret = 0;
 out:
@@ -442,10 +468,16 @@ marker_create_frame (xlator_t *this, marker_local_t *local)
 int32_t
 marker_xtime_update_marks (xlator_t *this, marker_local_t *local)
 {
+        marker_conf_t *priv = NULL;
+
         GF_VALIDATE_OR_GOTO ("marker", this, out);
         GF_VALIDATE_OR_GOTO (this->name, local, out);
 
-        if (local->pid == -1)
+        priv = this->private;
+
+        if ((local->pid == GF_CLIENT_PID_GSYNCD
+             && !(priv->feature_enabled & GF_XTIME_GSYNC_FORCE))
+            || (local->pid == GF_CLIENT_PID_DEFRAG))
                 goto out;
 
         marker_gettimeofday (local);
@@ -462,7 +494,7 @@ int32_t
 marker_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         marker_conf_t      *priv    = NULL;
         marker_local_t     *local   = NULL;
@@ -477,11 +509,14 @@ marker_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent);
+                             buf, preparent, postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
 
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
         priv = this->private;
 
         if (priv->feature_enabled & GF_QUOTA)
@@ -498,7 +533,7 @@ out:
 
 int
 marker_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dict_t *params)
+              mode_t umask, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -509,7 +544,7 @@ marker_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -519,12 +554,12 @@ marker_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 goto err;
 wind:
         STACK_WIND (frame, marker_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, params);
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 
         return 0;
 err:
         STACK_UNWIND_STRICT (mkdir, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL);
+                             NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -533,7 +568,7 @@ int32_t
 marker_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -548,11 +583,14 @@ marker_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
 
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
         priv = this->private;
 
         if (priv->feature_enabled & GF_QUOTA)
@@ -569,7 +607,7 @@ out:
 
 int32_t
 marker_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-               mode_t mode, fd_t *fd, dict_t *params)
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -580,7 +618,7 @@ marker_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -590,12 +628,12 @@ marker_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
                 goto err;
 wind:
         STACK_WIND (frame, marker_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, fd,
-                    params);
+                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
+                    fd, xdata);
         return 0;
 err:
         STACK_UNWIND_STRICT (create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                             NULL);
+                             NULL, NULL);
 
         return 0;
 }
@@ -604,7 +642,7 @@ err:
 int32_t
 marker_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         marker_conf_t      *priv    = NULL;
         marker_local_t     *local   = NULL;
@@ -618,7 +656,8 @@ marker_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -644,7 +683,7 @@ marker_writev (call_frame_t *frame,
                struct iovec *vector,
                int32_t count,
                off_t offset, uint32_t flags,
-               struct iobref *iobref)
+               struct iobref *iobref, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -655,7 +694,7 @@ marker_writev (call_frame_t *frame,
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -666,10 +705,10 @@ marker_writev (call_frame_t *frame,
 wind:
         STACK_WIND (frame, marker_writev_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
-                    flags, iobref);
+                    flags, iobref, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -678,7 +717,7 @@ err:
 int32_t
 marker_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         marker_conf_t      *priv    = NULL;
         marker_local_t     *local   = NULL;
@@ -693,7 +732,7 @@ marker_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent);
+                             postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -712,7 +751,8 @@ out:
 }
 
 int32_t
-marker_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+marker_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -723,7 +763,7 @@ marker_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -733,10 +773,10 @@ marker_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
                 goto err;
 wind:
         STACK_WIND (frame, marker_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags);
+                    FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -745,7 +785,7 @@ err:
 int32_t
 marker_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         marker_conf_t      *priv    = NULL;
         marker_local_t     *local   = NULL;
@@ -760,7 +800,7 @@ marker_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent);
+                             postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -781,7 +821,8 @@ out:
 
 int32_t
 marker_unlink_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        dict_t *xdata)
 {
         marker_local_t *local = NULL;
 
@@ -798,18 +839,20 @@ marker_unlink_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->ia_nlink = buf->ia_nlink;
 
         STACK_WIND (frame, marker_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, &local->loc);
+                    FIRST_CHILD(this)->fops->unlink, &local->loc, local->xflag,
+                    local->xdata);
         return 0;
 err:
         frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL);
+        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL, NULL);
         marker_local_unref (local);
         return 0;
 }
 
 
 int32_t
-marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -820,8 +863,10 @@ marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         if (priv->feature_enabled == 0)
                 goto unlink_wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
-
+        local = mem_get0 (this->local_pool);
+        local->xflag = xflag;
+        if (xdata)
+                local->xdata = dict_ref (xdata);
         MARKER_INIT_LOCAL (frame, local);
 
         ret = loc_copy (&local->loc, loc);
@@ -833,16 +878,16 @@ marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 uuid_copy (loc->gfid, loc->inode->gfid);
 
         STACK_WIND (frame, marker_unlink_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
         return 0;
 
 unlink_wind:
         STACK_WIND (frame, marker_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
         return 0;
 err:
         frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
         marker_local_unref (local);
         return 0;
 }
@@ -852,7 +897,7 @@ int32_t
 marker_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
                  struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -867,7 +912,7 @@ marker_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -886,7 +931,8 @@ out:
 }
 
 int32_t
-marker_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+marker_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -897,7 +943,7 @@ marker_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -907,10 +953,11 @@ marker_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
                 goto err;
 wind:
         STACK_WIND (frame, marker_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc);
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+        STACK_UNWIND_STRICT (link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                             NULL);
 
         return 0;
 }
@@ -918,7 +965,7 @@ err:
 
 int32_t
 marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         marker_local_t *local  = NULL, *oplocal = NULL;
         loc_t           newloc = {0, };
@@ -948,7 +995,7 @@ marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->stub = NULL;
         } else if (local->err != 0) {
                 STACK_UNWIND_STRICT (rename, frame, -1, local->err, NULL, NULL,
-                                     NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL);
         }
 
         mq_reduce_parent_size (this, &oplocal->loc, oplocal->contribution);
@@ -984,7 +1031,7 @@ marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 marker_rename_release_newp_lock (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno)
+                                 int32_t op_errno, dict_t *xdata)
 {
         marker_local_t  *local = NULL, *oplocal = NULL;
         struct gf_flock  lock  = {0, };
@@ -1005,7 +1052,7 @@ marker_rename_release_newp_lock (call_frame_t *frame, void *cookie,
         }
 
         if (local->next_lock_on == NULL) {
-                marker_rename_done (frame, NULL, this, 0, 0);
+                marker_rename_done (frame, NULL, this, 0, 0, NULL);
                 goto out;
         }
 
@@ -1019,7 +1066,7 @@ marker_rename_release_newp_lock (call_frame_t *frame, void *cookie,
                     marker_rename_done,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc, F_SETLKW, &lock);
+                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
 
 out:
         return 0;
@@ -1029,7 +1076,7 @@ out:
 int32_t
 marker_rename_release_oldp_lock (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno)
+                                 int32_t op_errno, dict_t *xdata)
 {
         marker_local_t  *local = NULL, *oplocal = NULL;
         struct gf_flock  lock  = {0, };
@@ -1055,7 +1102,7 @@ marker_rename_release_oldp_lock (call_frame_t *frame, void *cookie,
                     marker_rename_release_newp_lock,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &oplocal->parent_loc, F_SETLKW, &lock);
+                    this->name, &oplocal->parent_loc, F_SETLKW, &lock, NULL);
         return 0;
 }
 
@@ -1064,7 +1111,8 @@ int32_t
 marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
                    struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent)
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
 {
         marker_conf_t  *priv                 = NULL;
         marker_local_t *local                = NULL;
@@ -1099,7 +1147,7 @@ marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 stub = fop_rename_cbk_stub (frame, default_rename_cbk, op_ret,
                                             op_errno, buf, preoldparent,
                                             postoldparent, prenewparent,
-                                            postnewparent);
+                                            postnewparent, xdata);
                 if (stub == NULL) {
                         local->err = ENOMEM;
                         goto quota_err;
@@ -1129,7 +1177,7 @@ marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 STACK_WIND_COOKIE (frame, marker_rename_release_oldp_lock,
                                    frame->cookie, FIRST_CHILD(this),
                                    FIRST_CHILD(this)->fops->removexattr,
-                                   &newloc, contri_key);
+                                   &newloc, contri_key, NULL);
 
                 loc_wipe (&newloc);
         } else {
@@ -1137,7 +1185,7 @@ marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
                                      preoldparent, postoldparent, prenewparent,
-                                     postnewparent);
+                                     postnewparent, xdata);
 
                 if ((op_ret < 0) || (local == NULL)) {
                         goto out;
@@ -1160,15 +1208,14 @@ out:
         return 0;
 
 quota_err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0);
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
         return 0;
 }
 
 
 int32_t
 marker_do_rename (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict)
-
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         marker_local_t       *local           = NULL, *oplocal = NULL;
         char                  contri_key[512] = {0, };
@@ -1207,12 +1254,12 @@ marker_do_rename (call_frame_t *frame, void *cookie, xlator_t *this,
 
         STACK_WIND (frame, marker_rename_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rename, &oplocal->loc,
-                    &local->loc);
+                    &local->loc, NULL);
 
         return 0;
 
 err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0);
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
         return 0;
 }
 
@@ -1220,7 +1267,7 @@ err:
 int32_t
 marker_get_newpath_contribution (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *dict)
+                                 int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         marker_local_t *local           = NULL, *oplocal = NULL;
         char            contri_key[512] = {0, };
@@ -1272,14 +1319,14 @@ marker_get_newpath_contribution (call_frame_t *frame, void *cookie,
                 STACK_WIND_COOKIE (frame, marker_do_rename,
                                    frame->cookie, FIRST_CHILD(this),
                                    FIRST_CHILD(this)->fops->getxattr,
-                                   &local->loc, contri_key);
+                                   &local->loc, contri_key, NULL);
         } else {
-                marker_do_rename (frame, NULL, this, 0, 0, NULL);
+                marker_do_rename (frame, NULL, this, 0, 0, NULL, NULL);
         }
 
         return 0;
 err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0);
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
         return 0;
 }
 
@@ -1287,7 +1334,7 @@ err:
 int32_t
 marker_get_oldpath_contribution (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno)
+                                 int32_t op_errno, dict_t *xdata)
 {
         marker_local_t *local           = NULL, *oplocal = NULL;
         char            contri_key[512] = {0, };
@@ -1326,20 +1373,20 @@ marker_get_oldpath_contribution (call_frame_t *frame, void *cookie,
         STACK_WIND_COOKIE (frame, marker_get_newpath_contribution,
                            frame->cookie, FIRST_CHILD(this),
                            FIRST_CHILD(this)->fops->getxattr,
-                           &oplocal->loc, contri_key);
+                           &oplocal->loc, contri_key, NULL);
         return 0;
 
 quota_err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0);
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
         return 0;
 
 lock_err:
         if ((local->next_lock_on == NULL)
             || (local->next_lock_on == &local->parent_loc)) {
                 local->next_lock_on = NULL;
-                marker_rename_release_oldp_lock (frame, NULL, this, 0, 0);
+                marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
         } else {
-                marker_rename_release_newp_lock (frame, NULL, this, 0, 0);
+                marker_rename_release_newp_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         return 0;
@@ -1348,7 +1395,7 @@ lock_err:
 
 int32_t
 marker_rename_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno)
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         marker_local_t  *local = NULL, *oplocal = NULL;
         loc_t           *loc   = NULL;
@@ -1383,22 +1430,22 @@ marker_rename_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->inodelk,
                             this->name, local->next_lock_on,
-                            F_SETLKW, &lock);
+                            F_SETLKW, &lock, NULL);
         } else {
-                marker_get_oldpath_contribution (frame, 0, this, 0, 0);
+                marker_get_oldpath_contribution (frame, 0, this, 0, 0, NULL);
         }
 
         return 0;
 
 err:
-        marker_rename_done (frame, NULL, this, 0, 0);
+        marker_rename_done (frame, NULL, this, 0, 0, NULL);
         return 0;
 }
 
 
 int32_t
 marker_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc)
+               loc_t *newloc, dict_t *xdata)
 {
         int32_t         ret              = 0;
         marker_local_t *local            = NULL;
@@ -1412,11 +1459,11 @@ marker_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
         if (priv->feature_enabled == 0)
                 goto rename_wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
-        ALLOCATE_OR_GOTO (oplocal, marker_local_t, err);
+        oplocal = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, oplocal);
 
@@ -1467,18 +1514,18 @@ marker_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
                     this->name, lock_on,
-                    F_SETLKW, &lock);
+                    F_SETLKW, &lock, NULL);
 
         return 0;
 
 rename_wind:
         STACK_WIND (frame, marker_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc);
+                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
 
         return 0;
 err:
         STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL, NULL);
+                             NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -1487,7 +1534,7 @@ err:
 int32_t
 marker_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1502,7 +1549,7 @@ marker_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -1522,7 +1569,8 @@ out:
 }
 
 int32_t
-marker_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+marker_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1533,7 +1581,7 @@ marker_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -1543,10 +1591,10 @@ marker_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
                 goto err;
 wind:
         STACK_WIND (frame, marker_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -1555,7 +1603,7 @@ err:
 int32_t
 marker_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1570,7 +1618,7 @@ marker_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -1589,7 +1637,8 @@ out:
 }
 
 int32_t
-marker_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+marker_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1600,7 +1649,7 @@ marker_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -1610,10 +1659,10 @@ marker_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
                 goto err;
 wind:
         STACK_WIND (frame, marker_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -1623,7 +1672,7 @@ int32_t
 marker_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         marker_conf_t      *priv    = NULL;
         marker_local_t     *local   = NULL;
@@ -1638,11 +1687,14 @@ marker_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
 
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
         priv = this->private;
 
         if (priv->feature_enabled & GF_QUOTA)
@@ -1658,7 +1710,7 @@ out:
 
 int
 marker_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                loc_t *loc, dict_t *params)
+                loc_t *loc, mode_t umask, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1669,7 +1721,7 @@ marker_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -1679,11 +1731,12 @@ marker_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
                 goto err;
 wind:
         STACK_WIND (frame, marker_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, params);
+                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                    xdata);
         return 0;
 err:
         STACK_UNWIND_STRICT (symlink, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL);
+                             NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1692,7 +1745,7 @@ int32_t
 marker_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1707,11 +1760,14 @@ marker_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent);
+                             buf, preparent, postparent, xdata);
 
         if (op_ret == -1 ||  local == NULL)
                 goto out;
 
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
         priv = this->private;
 
         if ((priv->feature_enabled & GF_QUOTA) && (S_ISREG (local->mode))) {
@@ -1728,7 +1784,7 @@ out:
 
 int
 marker_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, dict_t *parms)
+              dev_t rdev, mode_t umask, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1739,7 +1795,7 @@ marker_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -1751,11 +1807,216 @@ marker_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 goto err;
 wind:
         STACK_WIND (frame, marker_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, parms);
+                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                    xdata);
         return 0;
 err:
         STACK_UNWIND_STRICT (mknod, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL);
+                             NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+marker_fallocate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
+                        "fallocating a file ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (fallocate, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		 off_t offset, size_t len, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_fallocate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		    xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred during discard",
+                        strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       size_t len, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_discard_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (discard, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int32_t
+marker_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred during zerofill",
+                        strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               size_t len, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_zerofill_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
+
         return 0;
 }
 
@@ -1815,7 +2076,7 @@ call_from_sp_client_to_reset_tmfile (call_frame_t *frame,
                 op_errno = EINVAL;
         }
 out:
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
 
         return 0;
 }
@@ -1823,7 +2084,7 @@ out:
 
 int32_t
 marker_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1837,7 +2098,7 @@ marker_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -1854,7 +2115,7 @@ out:
 
 int32_t
 marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                 int32_t flags)
+                 int32_t flags, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1869,7 +2130,7 @@ marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
         if (ret == 0)
                 return 0;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -1879,10 +2140,10 @@ marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
                 goto err;
 wind:
         STACK_WIND (frame, marker_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags);
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM);
+        STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, NULL);
 
         return 0;
 }
@@ -1890,7 +2151,7 @@ err:
 
 int32_t
 marker_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1904,7 +2165,7 @@ marker_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -1921,7 +2182,7 @@ out:
 
 int32_t
 marker_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                  int32_t flags)
+                  int32_t flags, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1936,7 +2197,7 @@ marker_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
         if (ret == 0)
                 return 0;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -1946,10 +2207,10 @@ marker_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
                 goto err;
 wind:
         STACK_WIND (frame, marker_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags);
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM);
+        STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, NULL);
 
         return 0;
 }
@@ -1958,7 +2219,7 @@ err:
 int32_t
 marker_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost)
+                     struct iatt *statpost, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1973,7 +2234,7 @@ marker_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+                             statpost, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -1991,7 +2252,7 @@ out:
 
 int32_t
 marker_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid)
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -2002,7 +2263,7 @@ marker_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -2012,10 +2273,10 @@ marker_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto err;
 wind:
         STACK_WIND (frame, marker_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid);
+                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -2024,7 +2285,7 @@ err:
 int32_t
 marker_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                    struct iatt *statpost)
+                    struct iatt *statpost, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -2042,7 +2303,7 @@ marker_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+                             statpost, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -2059,7 +2320,7 @@ out:
 
 int32_t
 marker_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -2070,7 +2331,7 @@ marker_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -2080,10 +2341,10 @@ marker_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto err;
 wind:
         STACK_WIND (frame, marker_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid);
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -2091,7 +2352,7 @@ err:
 
 int32_t
 marker_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -2105,7 +2366,7 @@ marker_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -2122,7 +2383,7 @@ out:
 
 int32_t
 marker_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name)
+                    const char *name, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -2133,7 +2394,7 @@ marker_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -2143,10 +2404,10 @@ marker_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto err;
 wind:
         STACK_WIND (frame, marker_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name);
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM);
+        STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM, NULL);
 
         return 0;
 }
@@ -2169,15 +2430,21 @@ marker_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         frame->local = NULL;
 
-        if (!op_ret && local && uuid_is_null (local->loc.gfid))
-                        uuid_copy (local->loc.gfid, inode->gfid);
-
         STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
                              dict, postparent);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
 
+        /* copy the gfid from the stat structure instead of inode,
+         * since if the lookup is fresh lookup, then the inode
+         * would have not yet linked to the inode table which happens
+         * in protocol/server.
+         */
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+
         priv = this->private;
 
         if (priv->feature_enabled & GF_QUOTA) {
@@ -2203,7 +2470,7 @@ marker_lookup (call_frame_t *frame, xlator_t *this,
         if (priv->feature_enabled == 0)
                 goto wind;
 
-        ALLOCATE_OR_GOTO (local, marker_local_t, err);
+        local = mem_get0 (this->local_pool);
 
         MARKER_INIT_LOCAL (frame, local);
 
@@ -2225,7 +2492,8 @@ err:
 
 int
 marker_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, gf_dirent_t *entries)
+                     int op_ret, int op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
 {
         gf_dirent_t *entry = NULL;
 
@@ -2237,10 +2505,11 @@ marker_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 
         return 0;
 }
+
 int
 marker_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                  off_t offset, dict_t *dict)
@@ -2364,14 +2633,11 @@ marker_xtime_priv_cleanup (xlator_t *this)
 
         GF_VALIDATE_OR_GOTO (this->name, priv, out);
 
-        if (priv->volume_uuid != NULL)
-                GF_FREE (priv->volume_uuid);
+        GF_FREE (priv->volume_uuid);
 
-        if (priv->timestamp_file != NULL)
-                GF_FREE (priv->timestamp_file);
+        GF_FREE (priv->timestamp_file);
 
-        if (priv->marker_xattr != NULL)
-                GF_FREE (priv->marker_xattr);
+        GF_FREE (priv->marker_xattr);
 out:
         return;
 }
@@ -2399,7 +2665,7 @@ out:
 int32_t
 reconfigure (xlator_t *this, dict_t *options)
 {
-        int32_t         ret     = -1;
+        int32_t         ret     = 0;
         data_t         *data    = NULL;
         gf_boolean_t    flag    = _gf_false;
         marker_conf_t  *priv    = NULL;
@@ -2440,11 +2706,17 @@ reconfigure (xlator_t *this, dict_t *options)
                                         "xtime updation will fail");
                         } else {
                                 priv->feature_enabled |= GF_XTIME;
+                                data = dict_get (options, "gsync-force-xtime");
+                                if (!data)
+                                        goto out;
+                                ret = gf_string2boolean (data->data, &flag);
+                                if (ret == 0 && flag)
+                                        priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
                         }
                 }
         }
 out:
-        return 0;
+        return ret;
 }
 
 
@@ -2500,9 +2772,23 @@ init (xlator_t *this)
                                 goto err;
 
                         priv->feature_enabled |= GF_XTIME;
+                        data = dict_get (options, "gsync-force-xtime");
+                        if (!data)
+                                goto cont;
+                        ret = gf_string2boolean (data->data, &flag);
+                        if (ret == 0 && flag)
+                                priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
                 }
         }
 
+ cont:
+        this->local_pool = mem_pool_new (marker_local_t, 128);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto err;
+        }
+
         return 0;
 err:
         marker_priv_cleanup (this);
@@ -2557,6 +2843,9 @@ struct xlator_fops fops = {
         .removexattr = marker_removexattr,
         .getxattr    = marker_getxattr,
         .readdirp    = marker_readdirp,
+	.fallocate   = marker_fallocate,
+	.discard     = marker_discard,
+        .zerofill    = marker_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -2568,5 +2857,6 @@ struct volume_options options[] = {
         {.key = {"timestamp-file"}},
         {.key = {"quota"}},
         {.key = {"xtime"}},
+        {.key = {"gsync-force-xtime"}},
         {.key = {NULL}}
 };
diff --git a/xlators/features/marker/src/marker.h b/xlators/features/marker/src/marker.h
index f5ed9df39..1a58f8cfc 100644
--- a/xlators/features/marker/src/marker.h
+++ b/xlators/features/marker/src/marker.h
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _MARKER_H
 #define _MARKER_H
 
@@ -37,8 +28,9 @@
 #define TIMESTAMP_FILE      "timestamp-file"
 
 enum {
-        GF_QUOTA=1,
-        GF_XTIME=2
+        GF_QUOTA             = 1,
+        GF_XTIME             = 2,
+        GF_XTIME_GSYNC_FORCE = 4,
 };
 
 /*initialize the local variable*/
@@ -103,9 +95,28 @@ struct marker_local{
         call_stub_t    *stub;
         int64_t         contribution;
         struct marker_local *oplocal;
+
+        /* marker quota specific */
+        int64_t delta;
+        int64_t d_off;
+        int64_t sum;
+        int64_t size;
+        int32_t hl_count;
+        int32_t dentry_child_count;
+
+        fd_t         *fd;
+        call_frame_t *frame;
+
+        quota_inode_ctx_t    *ctx;
+        inode_contribution_t *contri;
+
+        int xflag;
+        dict_t *xdata;
 };
 typedef struct marker_local marker_local_t;
 
+#define quota_local_t marker_local_t
+
 struct marker_inode_ctx {
         struct quota_inode_ctx *quota_ctx;
 };
diff --git a/xlators/features/marker/utils/src/Makefile.am b/xlators/features/marker/utils/src/Makefile.am
deleted file mode 100644
index 73c99cb76..000000000
--- a/xlators/features/marker/utils/src/Makefile.am
+++ /dev/null
@@ -1,22 +0,0 @@
-gsyncddir = $(libexecdir)/glusterfs
-
-gsyncd_PROGRAMS = gsyncd
-
-gsyncd_SOURCES = gsyncd.c procdiggy.c
-
-gsyncd_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-gsyncd_LDFLAGS = $(GF_LDFLAGS) $(GF_GLUSTERFS_LDFLAGS)
-
-noinst_HEADERS = procdiggy.h
-
-AM_CFLAGS = -fPIC -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src\
-	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
-	-DSBIN_DIR=\"$(sbindir)\" -DPYTHON=\"$(PYTHON)\"
-
-
-CLEANFILES =
-
-$(top_builddir)/libglusterfs/src/libglusterfs.la:
-	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
diff --git a/xlators/features/marker/utils/src/procdiggy.h b/xlators/features/marker/utils/src/procdiggy.h
deleted file mode 100644
index f4586de6c..000000000
--- a/xlators/features/marker/utils/src/procdiggy.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#define PROC "/proc"
-
-pid_t pidinfo (pid_t pid, char **name);
-
-int prociter (int (*proch) (pid_t pid, pid_t ppid, char *name, void *data),
-              void *data);
-
diff --git a/xlators/features/marker/utils/syncdaemon/Makefile.am b/xlators/features/marker/utils/syncdaemon/Makefile.am
deleted file mode 100644
index cc7cee102..000000000
--- a/xlators/features/marker/utils/syncdaemon/Makefile.am
+++ /dev/null
@@ -1,6 +0,0 @@
-syncdaemondir = $(libexecdir)/glusterfs/python/syncdaemon
-
-syncdaemon_PYTHON = gconf.py gsyncd.py __init__.py master.py README.md repce.py resource.py configinterface.py syncdutils.py monitor.py libcxattr.py \
-                    $(top_builddir)/contrib/ipaddr-py/ipaddr.py
-
-CLEANFILES =
diff --git a/xlators/features/marker/utils/syncdaemon/master.py b/xlators/features/marker/utils/syncdaemon/master.py
deleted file mode 100644
index cd470f315..000000000
--- a/xlators/features/marker/utils/syncdaemon/master.py
+++ /dev/null
@@ -1,519 +0,0 @@
-import os
-import sys
-import time
-import stat
-import signal
-import logging
-import errno
-from errno import ENOENT, ENODATA
-from threading import currentThread, Condition, Lock
-
-from gconf import gconf
-from syncdutils import FreeObject, Thread, GsyncdError, boolify
-
-URXTIME = (-1, 0)
-
-class GMaster(object):
-    """class impementling master role"""
-
-    KFGN = 0
-    KNAT = 1
-
-    def get_sys_volinfo(self):
-        """query volume marks on fs root
-
-        err out on multiple foreign masters
-        """
-        fgn_vis, nat_vi = self.master.server.foreign_volume_infos(), \
-                          self.master.server.native_volume_info()
-        fgn_vi = None
-        if fgn_vis:
-            if len(fgn_vis) > 1:
-                raise GsyncdError("cannot work with multiple foreign masters")
-            fgn_vi = fgn_vis[0]
-        return fgn_vi, nat_vi
-
-    @property
-    def uuid(self):
-        if self.volinfo:
-            return self.volinfo['uuid']
-
-    @property
-    def volmark(self):
-        if self.volinfo:
-            return self.volinfo['volume_mark']
-
-    @property
-    def inter_master(self):
-        """decide if we are an intermediate master
-        in a cascading setup
-        """
-        return self.volinfo_state[self.KFGN] and True or False
-
-    def xtime(self, path, *a, **opts):
-        """get amended xtime
-
-        as of amending, we can create missing xtime, or
-        determine a valid value if what we get is expired
-        (as of the volume mark expiry); way of amendig
-        depends on @opts and on subject of query (master
-        or slave).
-        """
-        if a:
-            rsc = a[0]
-        else:
-            rsc = self.master
-        if not 'create' in opts:
-            opts['create'] = (rsc == self.master and not self.inter_master)
-        if not 'default_xtime' in opts:
-            if rsc == self.master and self.inter_master:
-                opts['default_xtime'] = ENODATA
-            else:
-                opts['default_xtime'] = URXTIME
-        xt = rsc.server.xtime(path, self.uuid)
-        if isinstance(xt, int) and xt != ENODATA:
-            return xt
-        invalid_xtime = (xt == ENODATA or xt < self.volmark)
-        if invalid_xtime:
-            if opts['create']:
-                t = time.time()
-                sec = int(t)
-                nsec = int((t - sec) * 1000000)
-                xt = (sec, nsec)
-                rsc.server.set_xtime(path, self.uuid, xt)
-            else:
-                xt = opts['default_xtime']
-        return xt
-
-    def __init__(self, master, slave):
-        self.master = master
-        self.slave = slave
-        self.jobtab = {}
-        self.syncer = Syncer(slave)
-        # crawls vs. turns:
-        # - self.crawls is simply the number of crawl() invocations on root
-        # - one turn is a maximal consecutive sequence of crawls so that each
-        #   crawl in it detects a change to be synced
-        # - self.turns is the number of turns since start
-        # - self.total_turns is a limit so that if self.turns reaches it, then
-        #   we exit (for diagnostic purposes)
-        # so, eg., if the master fs changes unceasingly, self.turns will remain 0.
-        self.crawls = 0
-        self.turns = 0
-        self.total_turns = int(gconf.turns)
-        self.lastreport = {'crawls': 0, 'turns': 0}
-        self.start = None
-        self.change_seen = None
-        # the authoritative (foreign, native) volinfo pair
-        # which lets us deduce what to do when we refetch
-        # the volinfos from system
-        uuid_preset = getattr(gconf, 'volume_id', None)
-        self.volinfo_state = (uuid_preset and {'uuid': uuid_preset}, None)
-        # the actual volinfo we make use of
-        self.volinfo = None
-        self.terminate = False
-
-    def crawl_loop(self):
-        """start the keep-alive thread and iterate .crawl"""
-        timo = int(gconf.timeout or 0)
-        if timo > 0:
-            def keep_alive():
-                while True:
-                    gap = timo * 0.5
-                    # first grab a reference as self.volinfo
-                    # can be changed in main thread
-                    vi = self.volinfo
-                    if vi:
-                        # then have a private copy which we can mod
-                        vi = vi.copy()
-                        vi['timeout'] = int(time.time()) + timo
-                    else:
-                        # send keep-alives more frequently to
-                        # avoid a delay in announcing our volume info
-                        # to slave if it becomes established in the
-                        # meantime
-                        gap = min(10, gap)
-                    self.slave.server.keep_alive(vi)
-                    time.sleep(gap)
-            t = Thread(target=keep_alive)
-            t.start()
-        self.lastreport['time'] = time.time()
-        while not self.terminate:
-            self.crawl()
-
-    def add_job(self, path, label, job, *a, **kw):
-        """insert @job function to job table at @path with @label"""
-        if self.jobtab.get(path) == None:
-            self.jobtab[path] = []
-        self.jobtab[path].append((label, a, lambda : job(*a, **kw)))
-
-    def add_failjob(self, path, label):
-        """invoke .add_job with a job that does nothing just fails"""
-        logging.debug('salvaged: ' + label)
-        self.add_job(path, label, lambda: False)
-
-    def wait(self, path, *args):
-        """perform jobs registered for @path
-
-        Reset jobtab entry for @path,
-        determine success as the conjuction of
-        success of all the jobs. In case of
-        success, call .sendmark on @path
-        """
-        jobs = self.jobtab.pop(path, [])
-        succeed = True
-        for j in jobs:
-            ret = j[-1]()
-            if not ret:
-                succeed = False
-        if succeed:
-            self.sendmark(path, *args)
-        return succeed
-
-    def sendmark(self, path, mark, adct=None):
-        """update slave side xtime for @path to master side xtime
-
-        also can send a setattr payload (see Server.setattr).
-        """
-        if adct:
-            self.slave.server.setattr(path, adct)
-        self.slave.server.set_xtime(path, self.uuid, mark)
-
-    @staticmethod
-    def volinfo_state_machine(volinfo_state, volinfo_sys):
-        """compute new volinfo_state from old one and incoming
-           as of current system state, also indicating if there was a
-           change regarding which volume mark is the authoritative one
-
-        @volinfo_state, @volinfo_sys are pairs of volume mark dicts
-        (foreign, native).
-
-        Note this method is marked as static, ie. the computation is
-        pure, without reliance on any excess implicit state. State
-        transitions which are deemed as ambiguous or banned will raise
-        an exception.
-
-        """
-        # store the value below "boxed" to emulate proper closures
-        # (variables of the enclosing scope are available inner functions
-        # provided they are no reassigned; mutation is OK).
-        param = FreeObject(relax_mismatch = False, state_change = None, index=-1)
-        def select_vi(vi0, vi):
-            param.index += 1
-            if vi and (not vi0 or vi0['uuid'] == vi['uuid']):
-                if not vi0 and not param.relax_mismatch:
-                    param.state_change = param.index
-                # valid new value found; for the rest, we are graceful about
-                # uuid mismatch
-                param.relax_mismatch = True
-                return vi
-            if vi0 and vi and vi0['uuid'] != vi['uuid'] and not param.relax_mismatch:
-                # uuid mismatch for master candidate, bail out
-                raise GsyncdError("aborting on uuid change from %s to %s" % \
-                                   (vi0['uuid'], vi['uuid']))
-            # fall back to old
-            return vi0
-        newstate = tuple(select_vi(*vip) for vip in zip(volinfo_state, volinfo_sys))
-        srep = lambda vi: vi and vi['uuid'][0:8]
-        logging.debug('(%s, %s) << (%s, %s) -> (%s, %s)' % \
-                      tuple(srep(vi) for vi in volinfo_state + volinfo_sys + newstate))
-        return newstate, param.state_change
-
-    def crawl(self, path='.', xtl=None):
-        """crawling...
-
-          Standing around
-          All the right people
-          Crawling
-          Tennis on Tuesday
-          The ladder is long
-          It is your nature
-          You've gotta suntan
-          Football on Sunday
-          Society boy
-
-        Recursively walk the master side tree and check if updates are
-        needed due to xtime differences. One invocation of crawl checks
-        children of @path and do a recursive enter only on
-        those directory children where there is an update needed.
-
-        Way of updates depend on file type:
-        - for symlinks, sync them directy and synchronously
-        - for regular children, register jobs for @path (cf. .add_job) to start
-          and wait on their rsync
-        - for directory children, register a job for @path which waits (.wait)
-          on jobs for the given child
-        (other kind of filesystem nodes are not considered)
-
-        Those slave side children which do not exist on master are simply
-        purged (see Server.purge).
-
-        Behavior is fault tolerant, synchronization is adaptive: if some action fails,
-        just go on relentlessly, adding a fail job (see .add_failjob) which will prevent
-        the .sendmark on @path, so when the next crawl will arrive to @path it will not
-        see it as up-to-date and  will try to sync it again. While this semantics can be
-        supported by funky design principles (http://c2.com/cgi/wiki?LazinessImpatienceHubris),
-        the ultimate reason which excludes other possibilities is simply transience: we cannot
-        assert that the file systems (master / slave) underneath do not change and actions
-        taken upon some condition will not lose their context by the time they are performed.
-        """
-        if path == '.':
-            if self.start:
-                self.crawls += 1
-                logging.debug("... crawl #%d done, took %.6f seconds" % \
-                              (self.crawls, time.time() - self.start))
-            time.sleep(1)
-            self.start = time.time()
-            should_display_info = self.start - self.lastreport['time'] >= 60
-            if should_display_info:
-                logging.info("completed %d crawls, %d turns",
-                             self.crawls - self.lastreport['crawls'],
-                             self.turns - self.lastreport['turns'])
-                self.lastreport.update(crawls = self.crawls,
-                                       turns = self.turns,
-                                       time = self.start)
-            volinfo_sys = self.get_sys_volinfo()
-            self.volinfo_state, state_change = self.volinfo_state_machine(self.volinfo_state,
-                                                                          volinfo_sys)
-            if self.inter_master:
-                self.volinfo = volinfo_sys[self.KFGN]
-            else:
-                self.volinfo = volinfo_sys[self.KNAT]
-            if state_change == self.KFGN or (state_change == self.KNAT and not self.inter_master):
-                logging.info('new master is %s', self.uuid)
-                if self.volinfo:
-                    logging.info("%s master with volume id %s ..." % \
-                                 (self.inter_master and "intermediate" or "primary",
-                                  self.uuid))
-            if state_change == self.KFGN:
-               gconf.configinterface.set('volume_id', self.uuid)
-            if self.volinfo:
-                if self.volinfo['retval']:
-                    raise GsyncdError ("master is corrupt")
-            else:
-                if should_display_info or self.crawls == 0:
-                    if self.inter_master:
-                        logging.info("waiting for being synced from %s ..." % \
-                                     self.volinfo_state[self.KFGN]['uuid'])
-                    else:
-                        logging.info("waiting for volume info ...")
-                return
-        logging.debug("entering " + path)
-        if not xtl:
-            xtl = self.xtime(path)
-            if isinstance(xtl, int):
-                self.add_failjob(path, 'no-local-node')
-                return
-        xtr0 = self.xtime(path, self.slave)
-        if isinstance(xtr0, int):
-            if xtr0 != ENOENT:
-                self.slave.server.purge(path)
-            try:
-                self.slave.server.mkdir(path)
-            except OSError:
-                self.add_failjob(path, 'no-remote-node')
-                return
-            xtr = URXTIME
-        else:
-            xtr = xtr0
-            if xtr > xtl:
-                raise GsyncdError("timestamp corruption for " + path)
-            if xtl == xtr:
-                if path == '.' and self.change_seen:
-                    self.turns += 1
-                    self.change_seen = False
-                    if self.total_turns:
-                        logging.info("finished turn #%s/%s" % \
-                                     (self.turns, self.total_turns))
-                        if self.turns == self.total_turns:
-                            logging.info("reached turn limit")
-                            self.terminate = True
-                return
-        if path == '.':
-            self.change_seen = True
-        try:
-            dem = self.master.server.entries(path)
-        except OSError:
-            self.add_failjob(path, 'local-entries-fail')
-            return
-        try:
-            des = self.slave.server.entries(path)
-        except OSError:
-            self.slave.server.purge(path)
-            try:
-                self.slave.server.mkdir(path)
-                des = self.slave.server.entries(path)
-            except OSError:
-                self.add_failjob(path, 'remote-entries-fail')
-                return
-        dd = set(des) - set(dem)
-        if dd and not boolify(gconf.ignore_deletes):
-            self.slave.server.purge(path, dd)
-        chld = []
-        for e in dem:
-            e = os.path.join(path, e)
-            xte = self.xtime(e)
-            if isinstance(xte, int):
-                logging.warn("irregular xtime for %s: %s" % (e, errno.errorcode[xte]))
-            elif xte > xtr:
-                chld.append((e, xte))
-        def indulgently(e, fnc, blame=None):
-            if not blame:
-                blame = path
-            try:
-                return fnc(e)
-            except (IOError, OSError):
-                ex = sys.exc_info()[1]
-                if ex.errno == ENOENT:
-                    logging.warn("salvaged ENOENT for" + e)
-                    self.add_failjob(blame, 'by-indulgently')
-                    return False
-                else:
-                    raise
-        for e, xte in chld:
-            st = indulgently(e, lambda e: os.lstat(e))
-            if st == False:
-                continue
-            mo = st.st_mode
-            adct = {'own': (st.st_uid, st.st_gid)}
-            if stat.S_ISLNK(mo):
-                if indulgently(e, lambda e: self.slave.server.symlink(os.readlink(e), e)) == False:
-                    continue
-                self.sendmark(e, xte, adct)
-            elif stat.S_ISREG(mo):
-                logging.debug("syncing %s ..." % e)
-                pb = self.syncer.add(e)
-                def regjob(e, xte, pb):
-                    if pb.wait():
-                        logging.debug("synced " + e)
-                        self.sendmark(e, xte)
-                        return True
-                    else:
-                        logging.error("failed to sync " + e)
-                self.add_job(path, 'reg', regjob, e, xte, pb)
-            elif stat.S_ISDIR(mo):
-                adct['mode'] = mo
-                if indulgently(e, lambda e: (self.add_job(path, 'cwait', self.wait, e, xte, adct),
-                                             self.crawl(e, xte),
-                                             True)[-1], blame=e) == False:
-                    continue
-            else:
-                # ignore fifos, sockets and special files
-                pass
-        if path == '.':
-            self.wait(path, xtl)
-
-class BoxClosedErr(Exception):
-    pass
-
-class PostBox(list):
-    """synchronized collection for storing things thought of as "requests" """
-
-    def __init__(self, *a):
-        list.__init__(self, *a)
-        # too bad Python stdlib does not have read/write locks...
-        # it would suffivce to grab the lock in .append as reader, in .close as writer
-        self.lever = Condition()
-        self.open = True
-        self.done = False
-
-    def wait(self):
-        """wait on requests to be processed"""
-        self.lever.acquire()
-        if not self.done:
-            self.lever.wait()
-        self.lever.release()
-        return self.result
-
-    def wakeup(self, data):
-        """wake up requestors with the result"""
-        self.result = data
-        self.lever.acquire()
-        self.done = True
-        self.lever.notifyAll()
-        self.lever.release()
-
-    def append(self, e):
-        """post a request"""
-        self.lever.acquire()
-        if not self.open:
-            raise BoxClosedErr
-        list.append(self, e)
-        self.lever.release()
-
-    def close(self):
-        """prohibit the posting of further requests"""
-        self.lever.acquire()
-        self.open = False
-        self.lever.release()
-
-class Syncer(object):
-    """a staged queue to relay rsync requests to rsync workers
-
-    By "staged queue" its meant that when a consumer comes to the
-    queue, it takes _all_ entries, leaving the queue empty.
-    (I don't know if there is an official term for this pattern.)
-
-    The queue uses a PostBox to accumulate incoming items.
-    When a consumer (rsync worker) comes, a new PostBox is
-    set up and the old one is passed on to the consumer.
-
-    Instead of the simplistic scheme of having one big lock
-    which synchronizes both the addition of new items and
-    PostBox exchanges, use a separate lock to arbitrate consumers,
-    and rely on PostBox's synchronization mechanisms take
-    care about additions.
-
-    There is a corner case racy situation, producers vs. consumers,
-    which is not handled by this scheme: namely, when the PostBox
-    exchange occurs in between being passed to the producer for posting
-    and the post placement. But that's what Postbox.close is for:
-    such a posting will find the PostBox closed, in which case
-    the producer can re-try posting against the actual PostBox of
-    the queue.
-
-    To aid accumlation of items in the PostBoxen before grabbed
-    by an rsync worker, the worker goes to sleep a bit after
-    each completed syncjob.
-    """
-
-    def __init__(self, slave):
-        """spawn worker threads"""
-        self.slave = slave
-        self.lock = Lock()
-        self.pb = PostBox()
-        for i in range(int(gconf.sync_jobs)):
-            t = Thread(target=self.syncjob)
-            t.start()
-
-    def syncjob(self):
-        """the life of a worker"""
-        while True:
-            pb = None
-            while True:
-                self.lock.acquire()
-                if self.pb:
-                    pb, self.pb = self.pb, PostBox()
-                self.lock.release()
-                if pb:
-                    break
-                time.sleep(0.5)
-            pb.close()
-            po = self.slave.rsync(pb)
-            if po.returncode == 0:
-                ret = True
-            elif po.returncode in (23, 24):
-                # partial transfer (cf. rsync(1)), that's normal
-                ret = False
-            else:
-                po.errfail()
-            pb.wakeup(ret)
-
-    def add(self, e):
-        while True:
-            pb = self.pb
-            try:
-                pb.append(e)
-                return pb
-            except BoxClosedErr:
-                pass
diff --git a/xlators/features/marker/utils/syncdaemon/monitor.py b/xlators/features/marker/utils/syncdaemon/monitor.py
deleted file mode 100644
index 9536f3e26..000000000
--- a/xlators/features/marker/utils/syncdaemon/monitor.py
+++ /dev/null
@@ -1,123 +0,0 @@
-import os
-import sys
-import time
-import signal
-import logging
-from gconf import gconf
-from syncdutils import update_file, select, waitpid
-
-class Monitor(object):
-    """class which spawns and manages gsyncd workers"""
-
-    def __init__(self):
-        self.state = None
-
-    def set_state(self, state):
-        """set the state that can be used by external agents
-           like glusterd for status reporting"""
-        if state == self.state:
-            return
-        self.state = state
-        logging.info('new state: %s' % state)
-        if getattr(gconf, 'state_file', None):
-            update_file(gconf.state_file, lambda f: f.write(state + '\n'))
-
-    def monitor(self):
-        """the monitor loop
-
-        Basic logic is a blantantly simple blunt heuristics:
-        if spawned client survives 60 secs, it's considered OK.
-        This servers us pretty well as it's not vulneralbe to
-        any kind of irregular behavior of the child...
-
-        ... well, except for one: if children is hung up on
-        waiting for some event, it can survive aeons, still
-        will be defunct. So we tweak the above logic to
-        expect the worker to send us a signal within 60 secs
-        (in the form of closing its end of a pipe). The worker
-        does this when it's done with the setup stage
-        ready to enter the service loop (note it's the setup
-        stage which is vulnerable to hangs -- the full
-        blown worker blows up on EPIPE if the net goes down,
-        due to the keep-alive thread)
-        """
-        def sigcont_handler(*a):
-            """
-            Re-init logging and send group kill signal
-            """
-            md = gconf.log_metadata
-            logging.shutdown()
-            lcls = logging.getLoggerClass()
-            lcls.setup(label=md.get('saved_label'), **md)
-            pid = os.getpid()
-            os.kill(-pid, signal.SIGUSR1)
-        signal.signal(signal.SIGUSR1, lambda *a: ())
-        signal.signal(signal.SIGCONT, sigcont_handler)
-
-        argv = sys.argv[:]
-        for o in ('-N', '--no-daemon', '--monitor'):
-            while o in argv:
-                argv.remove(o)
-        argv.extend(('-N', '-p', ''))
-        argv.insert(0, os.path.basename(sys.executable))
-
-        self.set_state('starting...')
-        ret = 0
-        def nwait(p, o=0):
-            p2, r = waitpid(p, o)
-            if not p2:
-                return
-            return r
-        def exit_signalled(s):
-            """ child teminated due to receipt of SIGUSR1 """
-            return (os.WIFSIGNALED(s) and (os.WTERMSIG(s) == signal.SIGUSR1))
-        def exit_status(s):
-            if os.WIFEXITED(s):
-                return os.WEXITSTATUS(s)
-            return 1
-        conn_timeout = 60
-        while ret in (0, 1):
-            logging.info('-' * conn_timeout)
-            logging.info('starting gsyncd worker')
-            pr, pw = os.pipe()
-            cpid = os.fork()
-            if cpid == 0:
-                os.close(pr)
-                os.execv(sys.executable, argv + ['--feedback-fd', str(pw)])
-            os.close(pw)
-            t0 = time.time()
-            so = select((pr,), (), (), conn_timeout)[0]
-            os.close(pr)
-            if so:
-                ret = nwait(cpid, os.WNOHANG)
-                if ret != None:
-                    logging.debug("worker died before establishing connection")
-                else:
-                    logging.debug("worker seems to be connected (?? racy check)")
-                    while time.time() < t0 + conn_timeout:
-                        ret = nwait(cpid, os.WNOHANG)
-                        if ret != None:
-                            logging.debug("worker died in startup phase")
-                            break
-                        time.sleep(1)
-            else:
-                logging.debug("worker not confirmed in %d sec, aborting it" % \
-                              conn_timeout)
-                os.kill(cpid, signal.SIGKILL)
-                ret = nwait(cpid)
-            if ret == None:
-                self.set_state('OK')
-                ret = nwait(cpid)
-            if exit_signalled(ret):
-                ret = 0
-            else:
-                ret = exit_status(ret)
-                if ret in (0,1):
-                    self.set_state('faulty')
-            time.sleep(10)
-        self.set_state('inconsistent')
-        return ret
-
-def monitor():
-    """oh yeah, actually Monitor is used as singleton, too"""
-    return Monitor().monitor()
diff --git a/xlators/features/path-convertor/src/Makefile.am b/xlators/features/path-convertor/src/Makefile.am
index 58cfed0f9..393a7bd08 100644
--- a/xlators/features/path-convertor/src/Makefile.am
+++ b/xlators/features/path-convertor/src/Makefile.am
@@ -2,13 +2,14 @@
 xlator_LTLIBRARIES = path-converter.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
 
-path_converter_la_LDFLAGS = -module -avoidversion 
+path_converter_la_LDFLAGS = -module -avoid-version 
 
 path_converter_la_SOURCES = path.c
 path_converter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/path-convertor/src/path-mem-types.h b/xlators/features/path-convertor/src/path-mem-types.h
index c071513b6..77ada8d53 100644
--- a/xlators/features/path-convertor/src/path-mem-types.h
+++ b/xlators/features/path-convertor/src/path-mem-types.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __PATH_MEM_TYPES_H__
 #define __PATH_MEM_TYPES_H__
 
diff --git a/xlators/features/path-convertor/src/path.c b/xlators/features/path-convertor/src/path.c
index f61630535..5c52e0a8d 100644
--- a/xlators/features/path-convertor/src/path.c
+++ b/xlators/features/path-convertor/src/path.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 /* TODO: add gf_log to all the cases returning errors */
 
 #ifndef _CONFIG_H
@@ -52,7 +42,7 @@ static char *
 name_this_to_that (xlator_t *xl, const char *path, const char *name)
 {
 	path_private_t *priv = xl->private;
-	char priv_path[ZR_PATH_MAX] = {0,};
+	char priv_path[PATH_MAX] = {0,};
 	char *tmp_name = NULL;
 	int32_t path_len = strlen (path);
 	int32_t name_len = strlen (name) - ZR_FILE_CONTENT_STRLEN;
@@ -848,8 +838,7 @@ path_setxattr (call_frame_t *frame,
 	if (tmp_path != loc_path)
 		GF_FREE (tmp_path);
 
-	if (tmp_name)
-		GF_FREE (tmp_name);
+	GF_FREE (tmp_name);
 
 	return 0;
 }
diff --git a/mod_glusterfs/apache/1.3/Makefile.am b/xlators/features/protect/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/mod_glusterfs/apache/1.3/Makefile.am
+++ b/xlators/features/protect/Makefile.am
diff --git a/xlators/features/protect/src/Makefile.am b/xlators/features/protect/src/Makefile.am
new file mode 100644
index 000000000..7eb93f32e
--- /dev/null
+++ b/xlators/features/protect/src/Makefile.am
@@ -0,0 +1,21 @@
+xlator_LTLIBRARIES = prot_dht.la prot_client.la prot_server.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+prot_dht_la_LDFLAGS = -module -avoidversion
+prot_dht_la_SOURCES = prot_dht.c
+prot_dht_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+prot_client_la_LDFLAGS = -module -avoidversion
+prot_client_la_SOURCES = prot_client.c
+prot_client_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+prot_server_la_LDFLAGS = -module -avoidversion
+prot_server_la_SOURCES = prot_server.c
+prot_server_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES = 
+
diff --git a/xlators/features/protect/src/prot_client.c b/xlators/features/protect/src/prot_client.c
new file mode 100644
index 000000000..a27216d0a
--- /dev/null
+++ b/xlators/features/protect/src/prot_client.c
@@ -0,0 +1,215 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+#include <execinfo.h>
+
+#define NUM_FRAMES 20
+
+static char PROTECT_KEY[] = "trusted.glusterfs.protect";
+
+enum {
+        PROT_ACT_NONE = 0,
+        PROT_ACT_LOG,
+        PROT_ACT_REJECT,
+};
+
+void
+pcli_print_trace (char *name, call_frame_t *frame)
+{
+        void    *frames[NUM_FRAMES];
+        char    **symbols;
+        int     size;
+        int     i;
+
+        gf_log (name, GF_LOG_INFO, "Translator stack:");
+        while (frame) {
+                gf_log (name, GF_LOG_INFO, "%s (%s)",
+                        frame->wind_from, frame->this->name);
+                frame = frame->next;
+        }
+
+        size = backtrace(frames,NUM_FRAMES);
+        if (size <= 0) {
+                return;
+        }
+        symbols = backtrace_symbols(frames,size);
+        if (!symbols) {
+                return;
+        }
+
+        gf_log(name, GF_LOG_INFO, "Processor stack:");
+        for (i = 0; i < size; ++i) {
+                gf_log (name, GF_LOG_INFO, "%s", symbols[i]);
+        }
+        free(symbols);
+}
+
+int32_t
+pcli_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
+{
+        uint64_t        value;
+
+        if (newloc->parent == oldloc->parent) {
+                gf_log (this->name, GF_LOG_DEBUG, "rename in same directory");
+                goto simple_unwind;
+        }
+        if (!oldloc->parent) {
+                goto simple_unwind;
+        }
+        if (inode_ctx_get(oldloc->parent,this,&value) != 0) {
+                goto simple_unwind;
+        }
+
+        if (value != PROT_ACT_NONE) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "got rename for protected %s", oldloc->path);
+                pcli_print_trace(this->name,frame->next);
+                if (value == PROT_ACT_REJECT) {
+                        STACK_UNWIND_STRICT (rename, frame, -1, EPERM,
+                                             NULL, NULL, NULL, NULL, NULL,
+                                             xdata);
+                        return 0;
+                }
+        }
+
+simple_unwind:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->rename, oldloc, newloc,
+                         xdata);
+        return 0;
+}
+
+int32_t
+pcli_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
+{
+        data_t          *data;
+        uint64_t        value;
+
+        /*
+         * We can't use dict_get_str and strcmp here, because the value comes
+         * directly from the user and might not be NUL-terminated (it would
+         * be if we had set it ourselves.
+         */
+
+        data = dict_get(dict,PROTECT_KEY);
+        if (!data) {
+                goto simple_wind;
+        }
+
+        if (dict->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "attempted to mix %s with other keys", PROTECT_KEY);
+                goto simple_wind;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "got %s request", PROTECT_KEY);
+        if (!strncmp(data->data,"log",data->len)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "logging removals on %s", loc->path);
+                value = PROT_ACT_LOG;
+        }
+        else if (!strncmp(data->data,"reject",data->len)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "rejecting removals on %s", loc->path);
+                value = PROT_ACT_REJECT;
+        }
+        else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "removing protection on %s", loc->path);
+                value = PROT_ACT_NONE;
+        }
+        /* Right now the value doesn't matter - just the presence. */
+        if (inode_ctx_set(loc->inode,this,&value) != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set protection status for %s", loc->path);
+        }
+        STACK_UNWIND_STRICT (setxattr, frame, 0, 0, NULL);
+        return 0;
+
+simple_wind:
+        STACK_WIND_TAIL (frame,
+                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                         loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+pcli_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
+{
+        uint64_t        value;
+
+        if (!loc->parent || (inode_ctx_get(loc->parent,this,&value) != 0)) {
+                goto simple_unwind;
+        }
+
+        if (value != PROT_ACT_NONE) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "got unlink for protected %s", loc->path);
+                pcli_print_trace(this->name,frame->next);
+                if (value == PROT_ACT_REJECT) {
+                        STACK_UNWIND_STRICT (unlink, frame, -1, EPERM,
+                                             NULL, NULL, NULL);
+                        return 0;
+                }
+        }
+
+simple_unwind:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+
+struct xlator_fops fops = {
+        .rename         = pcli_rename,
+        .setxattr       = pcli_setxattr,
+        .unlink         = pcli_unlink,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/features/protect/src/prot_dht.c b/xlators/features/protect/src/prot_dht.c
new file mode 100644
index 000000000..feec6ffd6
--- /dev/null
+++ b/xlators/features/protect/src/prot_dht.c
@@ -0,0 +1,168 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+enum gf_pdht_mem_types_ {
+        gf_pdht_mt_coord_t = gf_common_mt_end + 1,
+        gf_pdht_mt_end
+};
+
+typedef struct {
+        pthread_mutex_t         lock;
+        uint16_t                refs;
+        int32_t                 op_ret;
+        int32_t                 op_errno;
+        dict_t                  *xdata;
+} pdht_coord_t;
+
+static char PROTECT_KEY[] = "trusted.glusterfs.protect";
+
+void
+pdht_unref_and_unlock (call_frame_t *frame, xlator_t *this,
+                       pdht_coord_t *coord)
+{
+        gf_boolean_t    should_unwind;
+
+        should_unwind = (--(coord->refs) == 0);
+        pthread_mutex_unlock(&coord->lock);
+
+        if (should_unwind) {
+                STACK_UNWIND_STRICT (setxattr, frame, 
+                                     coord->op_ret, coord->op_errno,
+                                     coord->xdata);
+                if (coord->xdata) {
+                        dict_unref(coord->xdata);
+                }
+                GF_FREE(coord);
+        }
+}
+
+int32_t
+pdht_recurse_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        pdht_coord_t    *coord = cookie;
+
+        pthread_mutex_lock(&coord->lock);
+        if (op_ret) {
+                coord->op_ret = op_ret;
+                coord->op_errno = op_errno;
+        }
+        if (xdata) {
+                if (coord->xdata) {
+                        dict_unref(coord->xdata);
+                }
+                coord->xdata = dict_ref(xdata);
+        }
+        pdht_unref_and_unlock(frame,this,coord);
+                
+        return 0;
+}
+
+void
+pdht_recurse (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+              int32_t flags, dict_t *xdata, xlator_t *xl, pdht_coord_t *coord)
+{
+        xlator_list_t   *iter;
+
+        if (!strcmp(xl->type,"features/prot_client")) {
+                pthread_mutex_lock(&coord->lock);
+                ++(coord->refs);
+                pthread_mutex_unlock(&coord->lock);
+                STACK_WIND_COOKIE (frame, pdht_recurse_cbk, coord, xl,
+                                   xl->fops->setxattr, loc, dict, flags, xdata);
+        }
+
+        else for (iter = xl->children; iter; iter = iter->next) {
+                pdht_recurse (frame, this, loc, dict, flags, xdata,
+                              iter->xlator, coord);
+        }
+}
+
+int32_t
+pdht_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
+{
+        pdht_coord_t    *coord;
+
+        if (!dict_get(dict,PROTECT_KEY)) {
+                goto simple_wind;
+        }
+
+        if (dict->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "attempted to mix %s with other keys", PROTECT_KEY);
+                goto simple_wind;
+        }
+
+        coord = GF_CALLOC(1,sizeof(*coord),gf_pdht_mt_coord_t);
+        if (!coord) {
+                gf_log (this->name, GF_LOG_WARNING, "allocation failed");
+                goto simple_wind;
+        }
+
+        pthread_mutex_init(&coord->lock,NULL);
+        coord->refs = 1;
+        coord->op_ret = 0;
+        coord->xdata = NULL;
+
+        pdht_recurse(frame,this,loc,dict,flags,xdata,this,coord);
+        pthread_mutex_lock(&coord->lock);
+        pdht_unref_and_unlock(frame,this,coord);
+
+        return 0;
+
+simple_wind:
+        STACK_WIND_TAIL (frame,
+                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                         loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+struct xlator_fops fops = {
+        .setxattr = pdht_setxattr,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/features/protect/src/prot_server.c b/xlators/features/protect/src/prot_server.c
new file mode 100644
index 000000000..beaee0889
--- /dev/null
+++ b/xlators/features/protect/src/prot_server.c
@@ -0,0 +1,51 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+
+struct xlator_fops fops = {
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/features/qemu-block/Makefile.am b/xlators/features/qemu-block/Makefile.am
new file mode 100644
index 000000000..af437a64d
--- /dev/null
+++ b/xlators/features/qemu-block/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = src
diff --git a/xlators/features/qemu-block/src/Makefile.am b/xlators/features/qemu-block/src/Makefile.am
new file mode 100644
index 000000000..08a7b62a0
--- /dev/null
+++ b/xlators/features/qemu-block/src/Makefile.am
@@ -0,0 +1,155 @@
+if ENABLE_QEMU_BLOCK
+xlator_LTLIBRARIES = qemu-block.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+qemu_block_la_LDFLAGS = -module -avoid-version
+qemu_block_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GLIB_LIBS) -lz -lrt
+
+qemu_block_la_SOURCES_qemu = \
+	$(CONTRIBDIR)/qemu/qemu-coroutine.c \
+	$(CONTRIBDIR)/qemu/qemu-coroutine-lock.c \
+	$(CONTRIBDIR)/qemu/qemu-coroutine-sleep.c \
+	$(CONTRIBDIR)/qemu/coroutine-ucontext.c \
+	$(CONTRIBDIR)/qemu/block.c \
+	$(CONTRIBDIR)/qemu/nop-symbols.c
+
+qemu_block_la_SOURCES_qemu_util = \
+	$(CONTRIBDIR)/qemu/util/aes.c \
+	$(CONTRIBDIR)/qemu/util/bitmap.c \
+	$(CONTRIBDIR)/qemu/util/bitops.c \
+	$(CONTRIBDIR)/qemu/util/cutils.c \
+	$(CONTRIBDIR)/qemu/util/error.c \
+	$(CONTRIBDIR)/qemu/util/hbitmap.c \
+	$(CONTRIBDIR)/qemu/util/iov.c \
+	$(CONTRIBDIR)/qemu/util/module.c \
+	$(CONTRIBDIR)/qemu/util/oslib-posix.c \
+	$(CONTRIBDIR)/qemu/util/qemu-option.c \
+	$(CONTRIBDIR)/qemu/util/qemu-error.c \
+	$(CONTRIBDIR)/qemu/util/qemu-thread-posix.c \
+	$(CONTRIBDIR)/qemu/util/unicode.c \
+	$(CONTRIBDIR)/qemu/util/hexdump.c
+
+qemu_block_la_SOURCES_qemu_block = \
+	$(CONTRIBDIR)/qemu/block/snapshot.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-cache.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-cluster.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-refcount.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-snapshot.c \
+	$(CONTRIBDIR)/qemu/block/qcow2.c \
+	$(CONTRIBDIR)/qemu/block/qed-check.c \
+	$(CONTRIBDIR)/qemu/block/qed-cluster.c \
+	$(CONTRIBDIR)/qemu/block/qed-gencb.c \
+	$(CONTRIBDIR)/qemu/block/qed-l2-cache.c \
+	$(CONTRIBDIR)/qemu/block/qed-table.c \
+	$(CONTRIBDIR)/qemu/block/qed.c
+
+qemu_block_la_SOURCES_qemu_qobject = \
+	$(CONTRIBDIR)/qemu/qobject/json-lexer.c \
+	$(CONTRIBDIR)/qemu/qobject/json-parser.c \
+	$(CONTRIBDIR)/qemu/qobject/json-streamer.c \
+	$(CONTRIBDIR)/qemu/qobject/qbool.c \
+	$(CONTRIBDIR)/qemu/qobject/qdict.c \
+	$(CONTRIBDIR)/qemu/qobject/qerror.c \
+	$(CONTRIBDIR)/qemu/qobject/qfloat.c \
+	$(CONTRIBDIR)/qemu/qobject/qint.c \
+	$(CONTRIBDIR)/qemu/qobject/qjson.c \
+	$(CONTRIBDIR)/qemu/qobject/qlist.c \
+	$(CONTRIBDIR)/qemu/qobject/qstring.c
+
+qemu_block_la_SOURCES = \
+	$(qemu_block_la_SOURCES_qemu) \
+	$(qemu_block_la_SOURCES_qemu_util) \
+	$(qemu_block_la_SOURCES_qemu_block) \
+	$(qemu_block_la_SOURCES_qemu_qobject) \
+	bdrv-xlator.c \
+	coroutine-synctask.c \
+	bh-syncop.c \
+	monitor-logging.c \
+	clock-timer.c \
+	qemu-block.c \
+	qb-coroutines.c
+
+noinst_HEADERS_qemu = \
+	$(CONTRIBDIR)/qemu/config-host.h \
+	$(CONTRIBDIR)/qemu/qapi-types.h \
+	$(CONTRIBDIR)/qemu/qmp-commands.h \
+	$(CONTRIBDIR)/qemu/trace/generated-tracers.h \
+	$(CONTRIBDIR)/qemu/include/config.h \
+	$(CONTRIBDIR)/qemu/include/glib-compat.h \
+	$(CONTRIBDIR)/qemu/include/qemu-common.h \
+	$(CONTRIBDIR)/qemu/include/trace.h \
+	$(CONTRIBDIR)/qemu/include/block/coroutine.h \
+	$(CONTRIBDIR)/qemu/include/block/aio.h \
+	$(CONTRIBDIR)/qemu/include/block/block.h \
+	$(CONTRIBDIR)/qemu/include/block/block_int.h \
+	$(CONTRIBDIR)/qemu/include/block/blockjob.h \
+	$(CONTRIBDIR)/qemu/include/block/coroutine.h \
+	$(CONTRIBDIR)/qemu/include/block/coroutine_int.h \
+	$(CONTRIBDIR)/qemu/include/block/snapshot.h \
+	$(CONTRIBDIR)/qemu/include/exec/cpu-common.h \
+	$(CONTRIBDIR)/qemu/include/exec/hwaddr.h \
+	$(CONTRIBDIR)/qemu/include/exec/poison.h \
+	$(CONTRIBDIR)/qemu/include/fpu/softfloat.h \
+	$(CONTRIBDIR)/qemu/include/migration/migration.h \
+	$(CONTRIBDIR)/qemu/include/migration/qemu-file.h \
+	$(CONTRIBDIR)/qemu/include/migration/vmstate.h \
+	$(CONTRIBDIR)/qemu/include/monitor/monitor.h \
+	$(CONTRIBDIR)/qemu/include/monitor/readline.h \
+	$(CONTRIBDIR)/qemu/include/qapi/error.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-lexer.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-parser.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-streamer.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qbool.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qdict.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qerror.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qfloat.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qint.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qjson.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qlist.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qobject.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qstring.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/types.h \
+	$(CONTRIBDIR)/qemu/include/qemu/aes.h \
+	$(CONTRIBDIR)/qemu/include/qemu/atomic.h \
+	$(CONTRIBDIR)/qemu/include/qemu/bitmap.h \
+	$(CONTRIBDIR)/qemu/include/qemu/bitops.h \
+	$(CONTRIBDIR)/qemu/include/qemu/bswap.h \
+	$(CONTRIBDIR)/qemu/include/qemu/compiler.h \
+	$(CONTRIBDIR)/qemu/include/qemu/error-report.h \
+	$(CONTRIBDIR)/qemu/include/qemu/event_notifier.h \
+	$(CONTRIBDIR)/qemu/include/qemu/hbitmap.h \
+	$(CONTRIBDIR)/qemu/include/qemu/host-utils.h \
+	$(CONTRIBDIR)/qemu/include/qemu/iov.h \
+	$(CONTRIBDIR)/qemu/include/qemu/main-loop.h \
+	$(CONTRIBDIR)/qemu/include/qemu/module.h \
+	$(CONTRIBDIR)/qemu/include/qemu/notify.h \
+	$(CONTRIBDIR)/qemu/include/qemu/option.h \
+	$(CONTRIBDIR)/qemu/include/qemu/option_int.h \
+	$(CONTRIBDIR)/qemu/include/qemu/osdep.h \
+	$(CONTRIBDIR)/qemu/include/qemu/queue.h \
+	$(CONTRIBDIR)/qemu/include/qemu/sockets.h \
+	$(CONTRIBDIR)/qemu/include/qemu/thread-posix.h \
+	$(CONTRIBDIR)/qemu/include/qemu/thread.h \
+	$(CONTRIBDIR)/qemu/include/qemu/timer.h \
+	$(CONTRIBDIR)/qemu/include/qemu/typedefs.h \
+	$(CONTRIBDIR)/qemu/include/sysemu/sysemu.h \
+	$(CONTRIBDIR)/qemu/include/sysemu/os-posix.h \
+	$(CONTRIBDIR)/qemu/block/qcow2.h \
+	$(CONTRIBDIR)/qemu/block/qed.h
+
+noinst_HEADERS = \
+	$(noinst_HEADERS_qemu) \
+	qemu-block.h \
+	qemu-block-memory-types.h \
+	qb-coroutines.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+            -I$(CONTRIBDIR)/qemu \
+            -I$(CONTRIBDIR)/qemu/include \
+            -DGLUSTER_XLATOR
+
+AM_CFLAGS = -fno-strict-aliasing -Wall $(GF_CFLAGS) $(GLIB_CFLAGS)
+
+CLEANFILES =
+
+endif
diff --git a/xlators/features/qemu-block/src/bdrv-xlator.c b/xlators/features/qemu-block/src/bdrv-xlator.c
new file mode 100644
index 000000000..106c59775
--- /dev/null
+++ b/xlators/features/qemu-block/src/bdrv-xlator.c
@@ -0,0 +1,397 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "inode.h"
+#include "syncop.h"
+#include "qemu-block.h"
+#include "block/block_int.h"
+
+typedef struct BDRVGlusterState {
+	inode_t *inode;
+} BDRVGlusterState;
+
+static QemuOptsList runtime_opts = {
+	.name = "gluster",
+	.head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
+	.desc = {
+		{
+			.name = "filename",
+			.type = QEMU_OPT_STRING,
+			.help = "GFID of file",
+		},
+		{ /* end of list */ }
+	},
+};
+
+inode_t *
+qb_inode_from_filename (const char *filename)
+{
+	const char *iptr = NULL;
+	inode_t *inode = NULL;
+
+	iptr = filename + 17;
+	sscanf (iptr, "%p", &inode);
+
+	return inode;
+}
+
+
+int
+qb_inode_to_filename (inode_t *inode, char *filename, int size)
+{
+	return snprintf (filename, size, "gluster://inodep:%p", inode);
+}
+
+
+static fd_t *
+fd_from_bs (BlockDriverState *bs)
+{
+	BDRVGlusterState *s = bs->opaque;
+
+	return fd_anonymous (s->inode);
+}
+
+
+static int
+qemu_gluster_open (BlockDriverState *bs, QDict *options, int bdrv_flags)
+{
+	inode_t *inode = NULL;
+	BDRVGlusterState *s = bs->opaque;
+	QemuOpts *opts = NULL;
+	Error *local_err = NULL;
+	const char *filename = NULL;
+	char gfid_str[128];
+	int ret;
+	qb_conf_t *conf = THIS->private;
+
+	opts = qemu_opts_create_nofail(&runtime_opts);
+	qemu_opts_absorb_qdict(opts, options, &local_err);
+	if (error_is_set(&local_err)) {
+		qerror_report_err(local_err);
+		error_free(local_err);
+		return -EINVAL;
+	}
+
+	filename = qemu_opt_get(opts, "filename");
+
+	/*
+	 * gfid:<gfid> format means we're opening a backing image.
+	 */
+	ret = sscanf(filename, "gluster://gfid:%s", gfid_str);
+	if (ret) {
+		loc_t loc = {0,};
+		struct iatt buf = {0,};
+		uuid_t gfid;
+
+		uuid_parse(gfid_str, gfid);
+
+		loc.inode = inode_find(conf->root_inode->table, gfid);
+		if (!loc.inode) {
+			loc.inode = inode_new(conf->root_inode->table);
+			uuid_copy(loc.inode->gfid, gfid);
+		}
+
+		uuid_copy(loc.gfid, loc.inode->gfid);
+		ret = syncop_lookup(FIRST_CHILD(THIS), &loc, NULL, &buf, NULL,
+				    NULL);
+		if (ret) {
+			loc_wipe(&loc);
+			return -errno;
+		}
+
+		s->inode = inode_ref(loc.inode);
+		loc_wipe(&loc);
+	} else {
+		inode = qb_inode_from_filename (filename);
+		if (!inode)
+			return -EINVAL;
+
+		s->inode = inode_ref(inode);
+	}
+
+	return 0;
+}
+
+
+static int
+qemu_gluster_create (const char *filename, QEMUOptionParameter *options)
+{
+	uint64_t total_size = 0;
+	inode_t *inode = NULL;
+	fd_t *fd = NULL;
+	struct iatt stat = {0, };
+	int ret = 0;
+
+	inode = qb_inode_from_filename (filename);
+	if (!inode)
+		return -EINVAL;
+
+	while (options && options->name) {
+		if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
+			total_size = options->value.n / BDRV_SECTOR_SIZE;
+		}
+		options++;
+	}
+
+	fd = fd_anonymous (inode);
+	if (!fd)
+		return -ENOMEM;
+
+	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &stat);
+	if (ret) {
+		fd_unref (fd);
+		return -errno;
+	}
+
+	if (stat.ia_size) {
+		/* format ONLY if the filesize is 0 bytes */
+		fd_unref (fd);
+		return -EFBIG;
+	}
+
+	if (total_size) {
+		ret = syncop_ftruncate (FIRST_CHILD(THIS), fd, total_size);
+		if (ret) {
+			fd_unref (fd);
+			return -errno;
+		}
+	}
+
+	fd_unref (fd);
+	return 0;
+}
+
+
+static int
+qemu_gluster_co_readv (BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+		       QEMUIOVector *qiov)
+{
+	fd_t *fd = NULL;
+	off_t offset = 0;
+	size_t size = 0;
+	struct iovec *iov = NULL;
+	int count = 0;
+	struct iobref *iobref = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+	if (!fd)
+		return -EIO;
+
+	offset = sector_num * BDRV_SECTOR_SIZE;
+	size = nb_sectors * BDRV_SECTOR_SIZE;
+
+	ret = syncop_readv (FIRST_CHILD(THIS), fd, size, offset, 0,
+			    &iov, &count, &iobref);
+	if (ret < 0) {
+		ret = -errno;
+		goto out;
+	}
+
+	iov_copy (qiov->iov, qiov->niov, iov, count); /* *choke!* */
+
+out:
+	GF_FREE (iov);
+	if (iobref)
+		iobref_unref (iobref);
+	fd_unref (fd);
+	return ret;
+}
+
+
+static int
+qemu_gluster_co_writev (BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+			QEMUIOVector *qiov)
+{
+	fd_t *fd = NULL;
+	off_t offset = 0;
+	size_t size = 0;
+	struct iobref *iobref = NULL;
+	struct iobuf *iobuf = NULL;
+	struct iovec iov = {0, };
+	int ret = -ENOMEM;
+
+	fd = fd_from_bs (bs);
+	if (!fd)
+		return -EIO;
+
+	offset = sector_num * BDRV_SECTOR_SIZE;
+	size = nb_sectors * BDRV_SECTOR_SIZE;
+
+	iobuf = iobuf_get2 (THIS->ctx->iobuf_pool, size);
+	if (!iobuf)
+		goto out;
+
+	iobref = iobref_new ();
+	if (!iobref) {
+		iobuf_unref (iobuf);
+		goto out;
+	}
+
+	iobref_add (iobref, iobuf);
+
+	iov_unload (iobuf_ptr (iobuf), qiov->iov, qiov->niov); /* *choke!* */
+
+	iov.iov_base = iobuf_ptr (iobuf);
+	iov.iov_len = size;
+
+	ret = syncop_writev (FIRST_CHILD(THIS), fd, &iov, 1, offset, iobref, 0);
+	if (ret < 0)
+		ret = -errno;
+
+out:
+	if (iobuf)
+		iobuf_unref (iobuf);
+	if (iobref)
+		iobref_unref (iobref);
+	fd_unref (fd);
+	return ret;
+}
+
+
+static int
+qemu_gluster_co_flush (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_flush (FIRST_CHILD(THIS), fd);
+
+	fd_unref (fd);
+
+	return ret;
+}
+
+
+static int
+qemu_gluster_co_fsync (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_fsync (FIRST_CHILD(THIS), fd, 0);
+
+	fd_unref (fd);
+
+	return ret;
+}
+
+
+static int
+qemu_gluster_truncate (BlockDriverState *bs, int64_t offset)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_ftruncate (FIRST_CHILD(THIS), fd, offset);
+
+	fd_unref (fd);
+
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+
+static int64_t
+qemu_gluster_getlength (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+	struct iatt iatt = {0, };
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &iatt);
+	if (ret < 0)
+		return -1;
+
+	return iatt.ia_size;
+}
+
+
+static int64_t
+qemu_gluster_allocated_file_size (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+	struct iatt iatt = {0, };
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &iatt);
+	if (ret < 0)
+		return -1;
+
+	return iatt.ia_blocks * 512;
+}
+
+
+static void
+qemu_gluster_close (BlockDriverState *bs)
+{
+	BDRVGlusterState *s = NULL;
+
+	s = bs->opaque;
+
+	inode_unref (s->inode);
+
+	return;
+}
+
+
+static QEMUOptionParameter qemu_gluster_create_options[] = {
+	{
+		.name = BLOCK_OPT_SIZE,
+		.type = OPT_SIZE,
+		.help = "Virtual disk size"
+	},
+	{ NULL }
+};
+
+
+static BlockDriver bdrv_gluster = {
+	.format_name                  = "gluster",
+	.protocol_name                = "gluster",
+	.instance_size                = sizeof(BDRVGlusterState),
+	.bdrv_file_open               = qemu_gluster_open,
+	.bdrv_close                   = qemu_gluster_close,
+	.bdrv_create                  = qemu_gluster_create,
+	.bdrv_getlength               = qemu_gluster_getlength,
+	.bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+	.bdrv_co_readv                = qemu_gluster_co_readv,
+	.bdrv_co_writev               = qemu_gluster_co_writev,
+	.bdrv_co_flush_to_os          = qemu_gluster_co_flush,
+	.bdrv_co_flush_to_disk        = qemu_gluster_co_fsync,
+	.bdrv_truncate                = qemu_gluster_truncate,
+	.create_options               = qemu_gluster_create_options,
+};
+
+
+static void bdrv_gluster_init(void)
+{
+	bdrv_register(&bdrv_gluster);
+}
+
+
+block_init(bdrv_gluster_init);
diff --git a/xlators/features/qemu-block/src/bh-syncop.c b/xlators/features/qemu-block/src/bh-syncop.c
new file mode 100644
index 000000000..e8686f6d4
--- /dev/null
+++ b/xlators/features/qemu-block/src/bh-syncop.c
@@ -0,0 +1,48 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "syncop.h"
+#include "qemu-block-memory-types.h"
+
+#include "block/aio.h"
+
+void
+qemu_bh_schedule (QEMUBH *bh)
+{
+	return;
+}
+
+void
+qemu_bh_cancel (QEMUBH *bh)
+{
+	return;
+}
+
+void
+qemu_bh_delete (QEMUBH *bh)
+{
+
+}
+
+QEMUBH *
+qemu_bh_new (QEMUBHFunc *cb, void *opaque)
+{
+	return NULL;
+}
diff --git a/xlators/features/qemu-block/src/clock-timer.c b/xlators/features/qemu-block/src/clock-timer.c
new file mode 100644
index 000000000..fcbec6ad1
--- /dev/null
+++ b/xlators/features/qemu-block/src/clock-timer.c
@@ -0,0 +1,60 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "syncop.h"
+#include "qemu-block-memory-types.h"
+
+#include "qemu/timer.h"
+
+QEMUClock *vm_clock;
+int use_rt_clock = 0;
+
+QEMUTimer *qemu_new_timer (QEMUClock *clock, int scale,
+			   QEMUTimerCB *cb, void *opaque)
+{
+	return NULL;
+}
+
+int64_t qemu_get_clock_ns (QEMUClock *clock)
+{
+	return 0;
+}
+
+void qemu_mod_timer (QEMUTimer *ts, int64_t expire_time)
+{
+	return;
+}
+
+void qemu_free_timer (QEMUTimer *ts)
+{
+
+}
+
+void qemu_del_timer (QEMUTimer *ts)
+{
+
+}
+
+bool qemu_aio_wait()
+{
+	synctask_wake (synctask_get());
+	synctask_yield (synctask_get());
+	return 0;
+}
diff --git a/xlators/features/qemu-block/src/coroutine-synctask.c b/xlators/features/qemu-block/src/coroutine-synctask.c
new file mode 100644
index 000000000..e43988a95
--- /dev/null
+++ b/xlators/features/qemu-block/src/coroutine-synctask.c
@@ -0,0 +1,116 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "syncop.h"
+#include "qemu-block-memory-types.h"
+
+#include "qemu-block.h"
+
+/*
+ * This code serves as the bridge from the main glusterfs context to the qemu
+ * coroutine context via synctask. We create a single threaded syncenv with a
+ * single synctask responsible for processing a queue of coroutines. The qemu
+ * code invoked from within the synctask function handlers uses the ucontext
+ * coroutine implementation and scheduling logic internal to qemu. This
+ * effectively donates a thread of execution to qemu and its internal coroutine
+ * management.
+ *
+ * NOTE: The existence of concurrent synctasks has proven quite racy with regard
+ * to qemu coroutine management, particularly related to the lifecycle
+ * differences with top-level synctasks and internally created coroutines and
+ * interactions with qemu-internal queues (and locks, in turn). We explicitly
+ * disallow this scenario, via the queue, until it is more well supported.
+ */
+
+static struct {
+	struct list_head	queue;
+	gf_lock_t		lock;
+	struct synctask		*task;
+} qb_co;
+
+static void
+init_qbco()
+{
+	INIT_LIST_HEAD(&qb_co.queue);
+	LOCK_INIT(&qb_co.lock);
+}
+
+static int
+synctask_nop_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+	return 0;
+}
+
+static int
+qb_synctask_wrap (void *opaque)
+{
+	qb_local_t *qb_local, *tmp;
+
+	LOCK(&qb_co.lock);
+
+	while (!list_empty(&qb_co.queue)) {
+		list_for_each_entry_safe(qb_local, tmp, &qb_co.queue, list) {
+			list_del_init(&qb_local->list);
+			break;
+		}
+
+		UNLOCK(&qb_co.lock);
+
+		qb_local->synctask_fn(qb_local);
+		/* qb_local is now unwound and gone! */
+
+		LOCK(&qb_co.lock);
+	}
+
+	qb_co.task = NULL;
+
+	UNLOCK(&qb_co.lock);
+
+	return 0;
+}
+
+int
+qb_coroutine (call_frame_t *frame, synctask_fn_t fn)
+{
+	qb_local_t *qb_local = NULL;
+	qb_conf_t *qb_conf = NULL;
+	static int init = 0;
+
+	qb_local = frame->local;
+	qb_local->synctask_fn = fn;
+	qb_conf = frame->this->private;
+
+	if (!init) {
+		init = 1;
+		init_qbco();
+	}
+
+	LOCK(&qb_co.lock);
+
+	if (!qb_co.task)
+		qb_co.task = synctask_create(qb_conf->env, qb_synctask_wrap,
+					     synctask_nop_cbk, frame, NULL);
+
+	list_add_tail(&qb_local->list, &qb_co.queue);
+
+	UNLOCK(&qb_co.lock);
+
+	return 0;
+}
diff --git a/xlators/features/qemu-block/src/monitor-logging.c b/xlators/features/qemu-block/src/monitor-logging.c
new file mode 100644
index 000000000..d37c37f0f
--- /dev/null
+++ b/xlators/features/qemu-block/src/monitor-logging.c
@@ -0,0 +1,50 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "qemu-block-memory-types.h"
+
+#include "block/block_int.h"
+
+Monitor *cur_mon;
+
+int
+monitor_cur_is_qmp()
+{
+	/* No QMP support here */
+	return 0;
+}
+
+void
+monitor_set_error (Monitor *mon, QError *qerror)
+{
+	/* NOP here */
+	return;
+}
+
+
+void
+monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
+{
+	char buf[4096];
+
+	vsnprintf(buf, sizeof(buf), fmt, ap);
+
+	gf_log (THIS->name, GF_LOG_ERROR, "%s", buf);
+}
diff --git a/xlators/features/qemu-block/src/qb-coroutines.c b/xlators/features/qemu-block/src/qb-coroutines.c
new file mode 100644
index 000000000..7c52adb21
--- /dev/null
+++ b/xlators/features/qemu-block/src/qb-coroutines.c
@@ -0,0 +1,662 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "inode.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "qemu-block-memory-types.h"
+#include "qemu-block.h"
+#include "qb-coroutines.h"
+
+
+int
+qb_format_and_resume (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	char filename[64];
+	char base_filename[128];
+	int use_base = 0;
+	qb_inode_t *qb_inode = NULL;
+	Error *local_err = NULL;
+	fd_t *fd = NULL;
+	dict_t *xattr = NULL;
+	qb_conf_t *qb_conf = NULL;
+	int ret = -1;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+	qb_conf = frame->this->private;
+
+	qb_inode_to_filename (inode, filename, 64);
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+
+	/*
+	 * See if the caller specified a backing image.
+	 */
+	if (!uuid_is_null(qb_inode->backing_gfid) || qb_inode->backing_fname) {
+		loc_t loc = {0,};
+		char gfid_str[64];
+		struct iatt buf;
+
+		if (!uuid_is_null(qb_inode->backing_gfid)) {
+			loc.inode = inode_find(qb_conf->root_inode->table,
+					qb_inode->backing_gfid);
+			if (!loc.inode) {
+				loc.inode = inode_new(qb_conf->root_inode->table);
+				uuid_copy(loc.inode->gfid,
+					qb_inode->backing_gfid);
+			}
+			uuid_copy(loc.gfid, loc.inode->gfid);
+		} else if (qb_inode->backing_fname) {
+			loc.inode = inode_new(qb_conf->root_inode->table);
+			loc.name = qb_inode->backing_fname;
+			loc.parent = inode_parent(inode, NULL, NULL);
+			loc_path(&loc, loc.name);
+		}
+
+		/*
+		 * Lookup the backing image. Verify existence and/or get the
+		 * gfid if we don't already have it.
+		 */
+		ret = syncop_lookup(FIRST_CHILD(frame->this), &loc, NULL, &buf,
+				    NULL, NULL);
+		GF_FREE(qb_inode->backing_fname);
+		if (ret) {
+			loc_wipe(&loc);
+			ret = errno;
+			goto err;
+		}
+
+		uuid_copy(qb_inode->backing_gfid, buf.ia_gfid);
+		loc_wipe(&loc);
+
+		/*
+		 * We pass the filename of the backing image into the qemu block
+		 * subsystem as the associated gfid. This is embedded into the
+		 * clone image and passed along to the gluster bdrv backend when
+		 * the block subsystem needs to operate on the backing image on
+		 * behalf of the clone.
+		 */
+		uuid_unparse(qb_inode->backing_gfid, gfid_str);
+		snprintf(base_filename, sizeof(base_filename),
+			 "gluster://gfid:%s", gfid_str);
+		use_base = 1;
+	}
+
+	bdrv_img_create (filename, qb_inode->fmt,
+			 use_base ? base_filename : NULL, 0, 0, qb_inode->size,
+			 0, &local_err, true);
+
+	if (error_is_set (&local_err)) {
+		gf_log (frame->this->name, GF_LOG_ERROR, "%s",
+			error_get_pretty (local_err));
+		error_free (local_err);
+		QB_STUB_UNWIND (stub, -1, EIO);
+		return 0;
+	}
+
+	fd = fd_anonymous (inode);
+	if (!fd) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"could not create anonymous fd for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		return 0;
+	}
+
+	xattr = dict_new ();
+	if (!xattr) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"could not allocate xattr dict for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		fd_unref (fd);
+		return 0;
+	}
+
+	ret = dict_set_str (xattr, qb_conf->qb_xattr_key, local->fmt);
+	if (ret) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"could not dict_set for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		fd_unref (fd);
+		dict_unref (xattr);
+		return 0;
+	}
+
+	ret = syncop_fsetxattr (FIRST_CHILD(THIS), fd, xattr, 0);
+	if (ret) {
+		ret = errno;
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"failed to setxattr for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ret);
+		fd_unref (fd);
+		dict_unref (xattr);
+		return 0;
+	}
+
+	fd_unref (fd);
+	dict_unref (xattr);
+
+	QB_STUB_UNWIND (stub, 0, 0);
+
+	return 0;
+
+err:
+	QB_STUB_UNWIND(stub, -1, ret);
+	return 0;
+}
+
+
+static BlockDriverState *
+qb_bs_create (inode_t *inode, const char *fmt)
+{
+	char filename[64];
+	BlockDriverState *bs = NULL;
+	BlockDriver *drv = NULL;
+	int op_errno = 0;
+	int ret = 0;
+
+	bs = bdrv_new (uuid_utoa (inode->gfid));
+	if (!bs) {
+		op_errno = ENOMEM;
+		gf_log (THIS->name, GF_LOG_ERROR,
+			"could not allocate @bdrv for gfid:%s",
+			uuid_utoa (inode->gfid));
+		goto err;
+	}
+
+	drv = bdrv_find_format (fmt);
+	if (!drv) {
+		op_errno = EINVAL;
+		gf_log (THIS->name, GF_LOG_ERROR,
+			"Unknown file format: %s for gfid:%s",
+			fmt, uuid_utoa (inode->gfid));
+		goto err;
+	}
+
+	qb_inode_to_filename (inode, filename, 64);
+
+	ret = bdrv_open (bs, filename, NULL, BDRV_O_RDWR, drv);
+	if (ret < 0) {
+		op_errno = -ret;
+		gf_log (THIS->name, GF_LOG_ERROR,
+			"Unable to bdrv_open() gfid:%s (%s)",
+			uuid_utoa (inode->gfid), strerror (op_errno));
+		goto err;
+	}
+
+	return bs;
+err:
+	errno = op_errno;
+	return NULL;
+}
+
+
+int
+qb_co_open (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+	qb_inode->refcnt++;
+
+	QB_STUB_RESUME (stub);
+
+	return 0;
+}
+
+
+int
+qb_co_writev (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	QEMUIOVector qiov = {0, };
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	qemu_iovec_init_external (&qiov, stub->args.vector, stub->args.count);
+
+	ret = bdrv_pwritev (qb_inode->bs, stub->args.offset, &qiov);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_co_readv (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	struct iobuf *iobuf = NULL;
+	struct iobref *iobref = NULL;
+	struct iovec iov = {0, };
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	if (stub->args.offset >= qb_inode->size) {
+		QB_STUB_UNWIND (stub, 0, 0);
+		return 0;
+	}
+
+	iobuf = iobuf_get2 (frame->this->ctx->iobuf_pool, stub->args.size);
+	if (!iobuf) {
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		return 0;
+	}
+
+	iobref = iobref_new ();
+	if (!iobref) {
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		iobuf_unref (iobuf);
+		return 0;
+	}
+
+	if (iobref_add (iobref, iobuf) < 0) {
+		iobuf_unref (iobuf);
+		iobref_unref (iobref);
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		return 0;
+	}
+
+	ret = bdrv_pread (qb_inode->bs, stub->args.offset, iobuf_ptr (iobuf),
+			  stub->args.size);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+		iobref_unref (iobref);
+		return 0;
+	}
+
+	iov.iov_base = iobuf_ptr (iobuf);
+	iov.iov_len = ret;
+
+	stub->args_cbk.vector = iov_dup (&iov, 1);
+	stub->args_cbk.count = 1;
+	stub->args_cbk.iobref = iobref;
+
+	QB_STUB_UNWIND (stub, ret, 0);
+
+	return 0;
+}
+
+
+int
+qb_co_fsync (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	ret = bdrv_flush (qb_inode->bs);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+static void
+qb_update_size_xattr (xlator_t *this, fd_t *fd, const char *fmt, off_t offset)
+{
+	char val[QB_XATTR_VAL_MAX];
+	qb_conf_t *qb_conf = NULL;
+	dict_t *xattr = NULL;
+
+	qb_conf = this->private;
+
+	snprintf (val, QB_XATTR_VAL_MAX, "%s:%llu",
+		  fmt, (long long unsigned) offset);
+
+	xattr = dict_new ();
+	if (!xattr)
+		return;
+
+	if (dict_set_str (xattr, qb_conf->qb_xattr_key, val) != 0) {
+		dict_unref (xattr);
+		return;
+	}
+
+	syncop_fsetxattr (FIRST_CHILD(this), fd, xattr, 0);
+	dict_unref (xattr);
+}
+
+
+int
+qb_co_truncate (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+	off_t offset = 0;
+	xlator_t *this = NULL;
+
+	this = THIS;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	syncop_fstat (FIRST_CHILD(this), local->fd, &stub->args_cbk.prestat);
+	stub->args_cbk.prestat.ia_size = qb_inode->size;
+
+	ret = bdrv_truncate (qb_inode->bs, stub->args.offset);
+	if (ret < 0)
+		goto out;
+
+	offset = bdrv_getlength (qb_inode->bs);
+
+	qb_inode->size = offset;
+
+	syncop_fstat (FIRST_CHILD(this), local->fd, &stub->args_cbk.poststat);
+	stub->args_cbk.poststat.ia_size = qb_inode->size;
+
+	qb_update_size_xattr (this, local->fd, qb_inode->fmt, qb_inode->size);
+
+out:
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_co_close (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	BlockDriverState *bs = NULL;
+
+	local = opaque;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (THIS, inode);
+
+	if (!--qb_inode->refcnt) {
+		bs = qb_inode->bs;
+		qb_inode->bs = NULL;
+		bdrv_delete (bs);
+	}
+
+	frame = local->frame;
+	frame->local = NULL;
+	qb_local_free (THIS, local);
+	STACK_DESTROY (frame->root);
+
+	return 0;
+}
+
+
+int
+qb_snapshot_create (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	QEMUSnapshotInfo sn;
+	struct timeval tv = {0, };
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	memset (&sn, 0, sizeof (sn));
+        pstrcpy (sn.name, sizeof(sn.name), local->name);
+        gettimeofday (&tv, NULL);
+        sn.date_sec = tv.tv_sec;
+        sn.date_nsec = tv.tv_usec * 1000;
+
+        ret = bdrv_snapshot_create (qb_inode->bs, &sn);
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_snapshot_delete (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	ret = bdrv_snapshot_delete (qb_inode->bs, local->name);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_snapshot_goto (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	ret = bdrv_snapshot_goto (qb_inode->bs, local->name);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
diff --git a/xlators/features/qemu-block/src/qb-coroutines.h b/xlators/features/qemu-block/src/qb-coroutines.h
new file mode 100644
index 000000000..583319f3b
--- /dev/null
+++ b/xlators/features/qemu-block/src/qb-coroutines.h
@@ -0,0 +1,30 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QB_COROUTINES_H
+#define __QB_COROUTINES_H
+
+#include "syncop.h"
+#include "call-stub.h"
+#include "block/block_int.h"
+#include "monitor/monitor.h"
+
+int qb_format_and_resume (void *opaque);
+int qb_snapshot_create (void *opaque);
+int qb_snapshot_delete (void *opaque);
+int qb_snapshot_goto (void *opaque);
+int qb_co_open (void *opaque);
+int qb_co_close (void *opaque);
+int qb_co_writev (void *opaque);
+int qb_co_readv (void *opaque);
+int qb_co_fsync (void *opaque);
+int qb_co_truncate (void *opaque);
+
+#endif /* __QB_COROUTINES_H */
diff --git a/xlators/features/qemu-block/src/qemu-block-memory-types.h b/xlators/features/qemu-block/src/qemu-block-memory-types.h
new file mode 100644
index 000000000..267b3893f
--- /dev/null
+++ b/xlators/features/qemu-block/src/qemu-block-memory-types.h
@@ -0,0 +1,25 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __QB_MEM_TYPES_H__
+#define __QB_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_qb_mem_types_ {
+        gf_qb_mt_qb_conf_t   = gf_common_mt_end + 1,
+        gf_qb_mt_qb_inode_t,
+        gf_qb_mt_qb_local_t,
+	gf_qb_mt_coroutinesynctask_t,
+        gf_qb_mt_end
+};
+#endif
+
diff --git a/xlators/features/qemu-block/src/qemu-block.c b/xlators/features/qemu-block/src/qemu-block.c
new file mode 100644
index 000000000..48bbf3140
--- /dev/null
+++ b/xlators/features/qemu-block/src/qemu-block.c
@@ -0,0 +1,1140 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "inode.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "qemu-block-memory-types.h"
+#include "qemu-block.h"
+#include "qb-coroutines.h"
+
+
+qb_inode_t *
+__qb_inode_ctx_get (xlator_t *this, inode_t *inode)
+{
+        uint64_t    value    = 0;
+        qb_inode_t *qb_inode = NULL;
+
+        __inode_ctx_get (inode, this, &value);
+        qb_inode = (qb_inode_t *)(unsigned long) value;
+
+        return qb_inode;
+}
+
+
+qb_inode_t *
+qb_inode_ctx_get (xlator_t *this, inode_t *inode)
+{
+        qb_inode_t *qb_inode = NULL;
+
+        LOCK (&inode->lock);
+        {
+                qb_inode = __qb_inode_ctx_get (this, inode);
+        }
+        UNLOCK (&inode->lock);
+
+        return qb_inode;
+}
+
+
+qb_inode_t *
+qb_inode_ctx_del (xlator_t *this, inode_t *inode)
+{
+        uint64_t    value    = 0;
+        qb_inode_t *qb_inode = NULL;
+
+        inode_ctx_del (inode, this, &value);
+        qb_inode = (qb_inode_t *)(unsigned long) value;
+
+        return qb_inode;
+}
+
+
+int
+qb_inode_cleanup (xlator_t *this, inode_t *inode, int warn)
+{
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_del (this, inode);
+
+	if (!qb_inode)
+		return 0;
+
+	if (warn)
+		gf_log (this->name, GF_LOG_WARNING,
+			"inode %s no longer block formatted",
+			uuid_utoa (inode->gfid));
+
+	/* free (qb_inode->bs); */
+
+	GF_FREE (qb_inode);
+
+	return 0;
+}
+
+
+int
+qb_iatt_fixup (xlator_t *this, inode_t *inode, struct iatt *iatt)
+{
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, inode);
+	if (!qb_inode)
+		return 0;
+
+	iatt->ia_size = qb_inode->size;
+
+	return 0;
+}
+
+
+int
+qb_format_extract (xlator_t *this, char *format, inode_t *inode)
+{
+	char       *s, *save;
+	uint64_t    size = 0;
+	char        fmt[QB_XATTR_VAL_MAX+1] = {0, };
+	qb_inode_t *qb_inode = NULL;
+	char *formatstr = NULL;
+	uuid_t gfid = {0,};
+	char gfid_str[64] = {0,};
+	int ret;
+
+	strncpy(fmt, format, QB_XATTR_VAL_MAX);
+
+	s = strtok_r(fmt, ":", &save);
+	if (!s)
+		goto invalid;
+	formatstr = gf_strdup(s);
+
+	s = strtok_r(NULL, ":", &save);
+	if (!s)
+		goto invalid;
+	if (gf_string2bytesize (s, &size))
+		goto invalid;
+	if (!size)
+		goto invalid;
+
+	s = strtok_r(NULL, "\0", &save);
+	if (s && !strncmp(s, "<gfid:", strlen("<gfid:"))) {
+		/*
+		 * Check for valid gfid backing image specifier.
+		 */
+		if (strlen(s) + 1 > sizeof(gfid_str))
+			goto invalid;
+		ret = sscanf(s, "<gfid:%[^>]s", gfid_str);
+		if (ret == 1) {
+			ret = uuid_parse(gfid_str, gfid);
+			if (ret < 0)
+				goto invalid;
+		}
+	}
+
+	qb_inode = qb_inode_ctx_get (this, inode);
+	if (!qb_inode)
+		qb_inode = GF_CALLOC (1, sizeof (*qb_inode),
+				      gf_qb_mt_qb_inode_t);
+	if (!qb_inode) {
+		GF_FREE(formatstr);
+		return ENOMEM;
+	}
+
+	strncpy(qb_inode->fmt, formatstr, QB_XATTR_VAL_MAX);
+	qb_inode->size = size;
+
+	/*
+	 * If a backing gfid was not specified, interpret any remaining bytes
+	 * associated with a backing image as a filename local to the parent
+	 * directory. The format processing will validate further.
+	 */
+	if (!uuid_is_null(gfid))
+		uuid_copy(qb_inode->backing_gfid, gfid);
+	else if (s)
+		qb_inode->backing_fname = gf_strdup(s);
+
+	inode_ctx_set (inode, this, (void *)&qb_inode);
+
+	GF_FREE(formatstr);
+
+	return 0;
+
+invalid:
+	GF_FREE(formatstr);
+
+	gf_log (this->name, GF_LOG_WARNING,
+		"invalid format '%s' in inode %s", format,
+		uuid_utoa (inode->gfid));
+	return EINVAL;
+}
+
+
+void
+qb_local_free (xlator_t *this, qb_local_t *local)
+{
+	if (local->inode)
+		inode_unref (local->inode);
+	if (local->fd)
+		fd_unref (local->fd);
+	GF_FREE (local);
+}
+
+
+int
+qb_local_init (call_frame_t *frame)
+{
+	qb_local_t *qb_local = NULL;
+
+	qb_local = GF_CALLOC (1, sizeof (*qb_local), gf_qb_mt_qb_local_t);
+	if (!qb_local)
+		return -1;
+	INIT_LIST_HEAD(&qb_local->list);
+
+	qb_local->frame = frame;
+	frame->local = qb_local;
+
+	return 0;
+}
+
+
+int
+qb_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	       int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
+	       dict_t *xdata, struct iatt *postparent)
+{
+	char *format = NULL;
+	qb_conf_t *conf = NULL;
+
+	conf = this->private;
+
+	if (op_ret == -1)
+		goto out;
+
+	/*
+	 * Cache the root inode for dealing with backing images. The format
+	 * coroutine and the gluster qemu backend driver both use the root inode
+	 * table to verify and/or redirect I/O to the backing image via
+	 * anonymous fd's.
+	 */
+	if (!conf->root_inode && __is_root_gfid(inode->gfid))
+		conf->root_inode = inode_ref(inode);
+
+	if (!xdata)
+		goto out;
+
+	if (dict_get_str (xdata, conf->qb_xattr_key, &format))
+		goto out;
+
+	if (!format) {
+		qb_inode_cleanup (this, inode, 1);
+		goto out;
+	}
+
+	op_errno = qb_format_extract (this, format, inode);
+	if (op_errno)
+		op_ret = -1;
+
+	qb_iatt_fixup (this, inode, buf);
+out:
+	QB_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
+			 xdata, postparent);
+	return 0;
+}
+
+
+int
+qb_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+	qb_conf_t *conf = NULL;
+
+	conf = this->private;
+
+	xdata = xdata ? dict_ref (xdata) : dict_new ();
+
+	if (!xdata)
+		goto enomem;
+
+	if (dict_set_int32 (xdata, conf->qb_xattr_key, 0))
+		goto enomem;
+
+	STACK_WIND (frame, qb_lookup_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+	dict_unref (xdata);
+	return 0;
+enomem:
+	QB_STACK_UNWIND (lookup, frame, -1, ENOMEM, 0, 0, 0, 0);
+	if (xdata)
+		dict_unref (xdata);
+	return 0;
+}
+
+
+int
+qb_setxattr_format (call_frame_t *frame, xlator_t *this, call_stub_t *stub,
+		    dict_t *xattr, inode_t *inode)
+{
+	char *format = NULL;
+	int op_errno = 0;
+	qb_local_t *qb_local = NULL;
+	data_t *data = NULL;
+	qb_inode_t *qb_inode;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-format"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	format = alloca (data->len + 1);
+	memcpy (format, data->data, data->len);
+	format[data->len] = 0;
+
+	op_errno = qb_format_extract (this, format, inode);
+	if (op_errno) {
+		QB_STUB_UNWIND (stub, -1, op_errno);
+		return 0;
+	}
+	qb_inode = qb_inode_ctx_get(this, inode);
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+
+	snprintf(qb_local->fmt, QB_XATTR_VAL_MAX, "%s:%lu", qb_inode->fmt,
+		 qb_inode->size);
+
+	qb_coroutine (frame, qb_format_and_resume);
+
+	return 0;
+}
+
+
+int
+qb_setxattr_snapshot_create (call_frame_t *frame, xlator_t *this,
+			     call_stub_t *stub, dict_t *xattr, inode_t *inode)
+{
+	qb_local_t *qb_local = NULL;
+	char *name = NULL;
+	data_t *data = NULL;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-create"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	name = alloca (data->len + 1);
+	memcpy (name, data->data, data->len);
+	name[data->len] = 0;
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+	strncpy (qb_local->name, name, 128);
+
+	qb_coroutine (frame, qb_snapshot_create);
+
+	return 0;
+}
+
+
+int
+qb_setxattr_snapshot_delete (call_frame_t *frame, xlator_t *this,
+			     call_stub_t *stub, dict_t *xattr, inode_t *inode)
+{
+	qb_local_t *qb_local = NULL;
+	char *name = NULL;
+	data_t *data = NULL;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-delete"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	name = alloca (data->len + 1);
+	memcpy (name, data->data, data->len);
+	name[data->len] = 0;
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+	strncpy (qb_local->name, name, 128);
+
+	qb_coroutine (frame, qb_snapshot_delete);
+
+	return 0;
+}
+
+int
+qb_setxattr_snapshot_goto (call_frame_t *frame, xlator_t *this,
+			   call_stub_t *stub, dict_t *xattr, inode_t *inode)
+{
+	qb_local_t *qb_local = NULL;
+	char *name = NULL;
+	data_t *data = NULL;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-goto"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	name = alloca (data->len + 1);
+	memcpy (name, data->data, data->len);
+	name[data->len] = 0;
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+	strncpy (qb_local->name, name, 128);
+
+	qb_coroutine (frame, qb_snapshot_goto);
+
+	return 0;
+}
+
+
+int
+qb_setxattr_common (call_frame_t *frame, xlator_t *this, call_stub_t *stub,
+		    dict_t *xattr, inode_t *inode)
+{
+	data_t *data = NULL;
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-format"))) {
+		qb_setxattr_format (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-snapshot-create"))) {
+		qb_setxattr_snapshot_create (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-snapshot-delete"))) {
+		qb_setxattr_snapshot_delete (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-snapshot-goto"))) {
+		qb_setxattr_snapshot_goto (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	QB_STUB_RESUME (stub);
+
+	return 0;
+}
+
+
+int
+qb_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr,
+	     int flags, dict_t *xdata)
+{
+	call_stub_t *stub = NULL;
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	stub = fop_setxattr_stub (frame, default_setxattr_resume, loc, xattr,
+				  flags, xdata);
+	if (!stub)
+		goto enomem;
+
+	qb_setxattr_common (frame, this, stub, xattr, loc->inode);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (setxattr, frame, -1, ENOMEM, 0);
+	return 0;
+}
+
+
+int
+qb_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+	      int flags, dict_t *xdata)
+{
+	call_stub_t *stub = NULL;
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume, fd, xattr,
+				   flags, xdata);
+	if (!stub)
+		goto enomem;
+
+	qb_setxattr_common (frame, this, stub, xattr, fd->inode);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (fsetxattr, frame, -1, ENOMEM, 0);
+	return 0;
+}
+
+
+int
+qb_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	     int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+	call_stub_t *stub = NULL;
+	qb_local_t *qb_local = NULL;
+
+	qb_local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+
+	if (!qb_inode_ctx_get (this, qb_local->inode))
+		goto unwind;
+
+	stub = fop_open_cbk_stub (frame, NULL, op_ret, op_errno, fd, xdata);
+	if (!stub) {
+		op_ret = -1;
+		op_errno = ENOMEM;
+		goto unwind;
+	}
+
+	qb_local->stub = stub;
+
+	qb_coroutine (frame, qb_co_open);
+
+	return 0;
+unwind:
+	QB_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
+	return 0;
+}
+
+
+int
+qb_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+	 fd_t *fd, dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, loc->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_open_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->open, loc, flags, fd,
+			    xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (loc->inode);
+	qb_local->fd = fd_ref (fd);
+
+	STACK_WIND (frame, qb_open_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+	return 0;
+enomem:
+	QB_STACK_UNWIND (open, frame, -1, ENOMEM, 0, 0);
+	return 0;
+}
+
+
+int
+qb_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+	   int count, off_t offset, uint32_t flags, struct iobref *iobref,
+	   dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_writev_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->writev, fd, vector, count,
+			    offset, flags, iobref, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_writev_stub (frame, NULL, fd, vector, count,
+					  offset, flags, iobref, xdata);
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_writev);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (writev, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	  off_t offset, uint32_t flags, dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->readv, fd, size, offset,
+			    flags, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_readv_stub (frame, NULL, fd, size, offset,
+					  flags, xdata);
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_readv);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (readv, frame, -1, ENOMEM, 0, 0, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int dsync,
+	  dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->fsync, fd, dsync, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_fsync_stub (frame, NULL, fd, dsync, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_fsync);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->flush, fd, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_flush_stub (frame, NULL, fd, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_fsync);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (flush, frame, -1, ENOMEM, 0);
+	return 0;
+}
+
+static int32_t
+qb_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+		dict_t *xdata)
+{
+	qb_conf_t *conf = this->private;
+	gf_dirent_t *entry;
+	char *format;
+
+	list_for_each_entry(entry, &entries->list, list) {
+		if (!entry->inode || !entry->dict)
+			continue;
+
+		format = NULL;
+		if (dict_get_str(entry->dict, conf->qb_xattr_key, &format))
+			continue;
+
+		if (!format) {
+			qb_inode_cleanup(this, entry->inode, 1);
+			continue;
+		}
+
+		if (qb_format_extract(this, format, entry->inode))
+			continue;
+
+		qb_iatt_fixup(this, entry->inode, &entry->d_stat);
+	}
+
+	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+static int32_t
+qb_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	    off_t off, dict_t *xdata)
+{
+	qb_conf_t *conf = this->private;
+
+	xdata = xdata ? dict_ref(xdata) : dict_new();
+	if (!xdata)
+		goto enomem;
+
+	if (dict_set_int32 (xdata, conf->qb_xattr_key, 0))
+		goto enomem;
+
+	STACK_WIND(frame, qb_readdirp_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+
+	dict_unref(xdata);
+	return 0;
+
+enomem:
+	QB_STACK_UNWIND(readdirp, frame, -1, ENOMEM, NULL, NULL);
+	if (xdata)
+		dict_unref(xdata);
+	return 0;
+}
+
+int
+qb_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+	     dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, loc->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->truncate, loc, offset,
+			    xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (loc->inode);
+	qb_local->fd = fd_anonymous (loc->inode);
+
+	qb_local->stub = fop_truncate_stub (frame, NULL, loc, offset, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_truncate);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (truncate, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	     dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_ftruncate_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->ftruncate, fd, offset,
+			    xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_ftruncate_stub (frame, NULL, fd, offset, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_truncate);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	     int op_ret, int op_errno, struct iatt *iatt, dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, iatt);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (stat, frame, op_ret, op_errno, iatt, xdata);
+
+	return 0;
+}
+
+int
+qb_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, loc->inode))
+		frame->local = inode_ref (loc->inode);
+
+	STACK_WIND (frame, qb_stat_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->stat, loc, xdata);
+	return 0;
+}
+
+
+int
+qb_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	      int op_ret, int op_errno, struct iatt *iatt, dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, iatt);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (fstat, frame, op_ret, op_errno, iatt, xdata);
+
+	return 0;
+}
+
+
+int
+qb_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, fd->inode))
+		frame->local = inode_ref (fd->inode);
+
+	STACK_WIND (frame, qb_fstat_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+	return 0;
+}
+
+
+int
+qb_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
+		dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, pre);
+		qb_iatt_fixup (this, inode, post);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (setattr, frame, op_ret, op_errno, pre, post, xdata);
+
+	return 0;
+}
+
+
+int
+qb_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+	    int valid, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, loc->inode))
+		frame->local = inode_ref (loc->inode);
+
+	STACK_WIND (frame, qb_setattr_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->setattr, loc, buf, valid, xdata);
+	return 0;
+}
+
+
+int
+qb_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
+		 dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, pre);
+		qb_iatt_fixup (this, inode, post);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, pre, post, xdata);
+
+	return 0;
+}
+
+
+int
+qb_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+	     int valid, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, fd->inode))
+		frame->local = inode_ref (fd->inode);
+
+	STACK_WIND (frame, qb_setattr_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->fsetattr, fd, buf, valid, xdata);
+	return 0;
+}
+
+
+int
+qb_forget (xlator_t *this, inode_t *inode)
+{
+	return qb_inode_cleanup (this, inode, 0);
+}
+
+
+int
+qb_release (xlator_t *this, fd_t *fd)
+{
+	call_frame_t *frame = NULL;
+
+	frame = create_frame (this, this->ctx->pool);
+	if (!frame) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Could not allocate frame. "
+			"Leaking QEMU BlockDriverState");
+		return -1;
+	}
+
+	if (qb_local_init (frame) != 0) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Could not allocate local. "
+			"Leaking QEMU BlockDriverState");
+		STACK_DESTROY (frame->root);
+		return -1;
+	}
+
+	if (qb_coroutine (frame, qb_co_close) != 0) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Could not allocate coroutine. "
+			"Leaking QEMU BlockDriverState");
+		qb_local_free (this, frame->local);
+		frame->local = NULL;
+		STACK_DESTROY (frame->root);
+	}
+
+	return 0;
+}
+
+int
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_qb_mt_end + 1);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init "
+                        "failed");
+        return ret;
+}
+
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+	return 0;
+}
+
+
+int
+init (xlator_t *this)
+{
+        qb_conf_t *conf    = NULL;
+        int32_t    ret     = -1;
+	static int bdrv_inited = 0;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "FATAL: qemu-block (%s) not configured with exactly "
+                        "one child", this->name);
+                goto out;
+        }
+
+        conf = GF_CALLOC (1, sizeof (*conf), gf_qb_mt_qb_conf_t);
+        if (!conf)
+                goto out;
+
+        /* configure 'option window-size <size>' */
+        GF_OPTION_INIT ("default-password", conf->default_password, str, out);
+
+	/* qemu coroutines use "co_mutex" for synchronizing among themselves.
+	   However "co_mutex" itself is not threadsafe if the coroutine framework
+	   is multithreaded (which usually is not). However synctasks are
+	   fundamentally multithreaded, so for now create a syncenv which has
+	   scaling limits set to max 1 thread so that the qemu coroutines can
+	   execute "safely".
+
+	   Future work: provide an implementation of "co_mutex" which is
+	   threadsafe and use the global multithreaded ctx->env syncenv.
+	*/
+	conf->env = syncenv_new (0, 1, 1);
+
+        this->private = conf;
+
+        ret = 0;
+
+	snprintf (conf->qb_xattr_key, QB_XATTR_KEY_MAX, QB_XATTR_KEY_FMT,
+		  this->name);
+
+	cur_mon = (void *) 1;
+
+	if (!bdrv_inited) {
+		bdrv_init ();
+		bdrv_inited = 1;
+	}
+
+out:
+        if (ret)
+                GF_FREE (conf);
+
+        return ret;
+}
+
+
+void
+fini (xlator_t *this)
+{
+        qb_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        this->private = NULL;
+
+	if (conf->root_inode)
+		inode_unref(conf->root_inode);
+        GF_FREE (conf);
+
+	return;
+}
+
+
+struct xlator_fops fops = {
+	.lookup      = qb_lookup,
+	.fsetxattr   = qb_fsetxattr,
+	.setxattr    = qb_setxattr,
+	.open        = qb_open,
+        .writev      = qb_writev,
+        .readv       = qb_readv,
+        .fsync       = qb_fsync,
+        .truncate    = qb_truncate,
+        .ftruncate   = qb_ftruncate,
+        .stat        = qb_stat,
+        .fstat       = qb_fstat,
+	.setattr     = qb_setattr,
+	.fsetattr    = qb_fsetattr,
+        .flush       = qb_flush,
+/*
+	.getxattr    = qb_getxattr,
+	.fgetxattr   = qb_fgetxattr
+*/
+	.readdirp    = qb_readdirp,
+};
+
+
+struct xlator_cbks cbks = {
+        .forget   = qb_forget,
+	.release  = qb_release,
+};
+
+
+struct xlator_dumpops dumpops = {
+};
+
+
+struct volume_options options[] = {
+        { .key  = {"default-password"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "",
+          .description = "Default password for the AES encrypted block images."
+        },
+        { .key = {NULL} },
+};
diff --git a/xlators/features/qemu-block/src/qemu-block.h b/xlators/features/qemu-block/src/qemu-block.h
new file mode 100644
index 000000000..c95f2799a
--- /dev/null
+++ b/xlators/features/qemu-block/src/qemu-block.h
@@ -0,0 +1,109 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QEMU_BLOCK_H
+#define __QEMU_BLOCK_H
+
+#include "syncop.h"
+#include "call-stub.h"
+#include "block/block_int.h"
+#include "monitor/monitor.h"
+
+/* QB_XATTR_KEY_FMT is the on-disk xattr stored in the inode which
+   indicates that the file must be "interpreted" by the block format
+   logic. The value of the key is of the pattern:
+
+   "format:virtual_size"
+
+   e.g
+
+   "qcow2:20GB" or "qed:100GB"
+
+   The format and virtual size are colon separated. The format is
+   a case sensitive string which qemu recognizes. virtual_size is
+   specified as a size which glusterfs recognizes as size (i.e.,
+   value accepted by gf_string2bytesize())
+*/
+#define QB_XATTR_KEY_FMT "trusted.glusterfs.%s.format"
+
+#define QB_XATTR_KEY_MAX 64
+
+#define QB_XATTR_VAL_MAX 64
+
+
+typedef struct qb_inode {
+	char fmt[QB_XATTR_VAL_MAX]; /* this is only the format, not "format:size" */
+	size_t size; /* virtual size in bytes */
+	BlockDriverState *bs;
+	int refcnt;
+	uuid_t backing_gfid;
+	char *backing_fname;
+} qb_inode_t;
+
+
+typedef struct qb_conf {
+	Monitor *mon;
+	struct syncenv *env;
+	char qb_xattr_key[QB_XATTR_KEY_MAX];
+	char *default_password;
+	inode_t *root_inode;
+} qb_conf_t;
+
+
+typedef struct qb_local {
+	call_frame_t      *frame; /* backpointer */
+	call_stub_t       *stub;
+	inode_t           *inode;
+	fd_t              *fd;
+	char               fmt[QB_XATTR_VAL_MAX+1];
+	char               name[256];
+	synctask_fn_t      synctask_fn;
+	struct list_head   list;
+} qb_local_t;
+
+void qb_local_free (xlator_t *this, qb_local_t *local);
+int qb_coroutine (call_frame_t *frame, synctask_fn_t fn);
+inode_t *qb_inode_from_filename (const char *filename);
+int qb_inode_to_filename (inode_t *inode, char *filename, int size);
+int qb_format_extract (xlator_t *this, char *format, inode_t *inode);
+
+qb_inode_t *qb_inode_ctx_get (xlator_t *this, inode_t *inode);
+
+#define QB_STACK_UNWIND(typ, frame, args ...) do {	\
+	qb_local_t *__local = frame->local;		\
+	xlator_t *__this = frame->this;			\
+							\
+	frame->local = NULL;				\
+	STACK_UNWIND_STRICT (typ, frame, args);		\
+	if (__local)					\
+		qb_local_free (__this, __local);	\
+	} while (0)
+
+#define QB_STUB_UNWIND(stub, op_ret, op_errno) do {	\
+	qb_local_t *__local = stub->frame->local;	\
+	xlator_t *__this = stub->frame->this;		\
+							\
+	stub->frame->local = NULL;			\
+	call_unwind_error (stub, op_ret, op_errno);	\
+	if (__local)					\
+		qb_local_free (__this, __local);	\
+	} while (0)
+
+#define QB_STUB_RESUME(stub_errno) do {			\
+	qb_local_t *__local = stub->frame->local;	\
+	xlator_t *__this = stub->frame->this;		\
+							\
+	stub->frame->local = NULL;			\
+	call_resume (stub);				\
+	if (__local)					\
+		qb_local_free (__this, __local);	\
+	} while (0)
+
+#endif /* !__QEMU_BLOCK_H */
diff --git a/xlators/features/quiesce/src/Makefile.am b/xlators/features/quiesce/src/Makefile.am
index e8ab4cb24..15e46629e 100644
--- a/xlators/features/quiesce/src/Makefile.am
+++ b/xlators/features/quiesce/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = quiesce.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-quiesce_la_LDFLAGS = -module -avoidversion
+quiesce_la_LDFLAGS = -module -avoid-version
 
 quiesce_la_SOURCES = quiesce.c
 quiesce_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = quiesce.h quiesce-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
diff --git a/xlators/features/quiesce/src/quiesce-mem-types.h b/xlators/features/quiesce/src/quiesce-mem-types.h
index 00f7aa4f1..6e582f424 100644
--- a/xlators/features/quiesce/src/quiesce-mem-types.h
+++ b/xlators/features/quiesce/src/quiesce-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __QUIESCE_MEM_TYPES_H__
 #define __QUIESCE_MEM_TYPES_H__
 
diff --git a/xlators/features/quiesce/src/quiesce.c b/xlators/features/quiesce/src/quiesce.c
index 1b150b215..24c7dc6ed 100644
--- a/xlators/features/quiesce/src/quiesce.c
+++ b/xlators/features/quiesce/src/quiesce.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -41,16 +31,13 @@ gf_quiesce_local_wipe (xlator_t *this, quiesce_local_t *local)
                 loc_wipe (&local->loc);
         if (local->fd)
                 fd_unref (local->fd);
-        if (local->name)
-                GF_FREE (local->name);
-        if (local->volname)
-                GF_FREE (local->volname);
+        GF_FREE (local->name);
+        GF_FREE (local->volname);
         if (local->dict)
                 dict_unref (local->dict);
         if (local->iobref)
                 iobref_unref (local->iobref);
-        if (local->vector)
-                GF_FREE (local->vector);
+        GF_FREE (local->vector);
 
         mem_put (local);
 }
@@ -124,7 +111,7 @@ void
 gf_quiesce_enqueue (xlator_t *this, call_stub_t *stub)
 {
         quiesce_priv_t *priv    = NULL;
-        struct timeval  timeout = {0,};
+        struct timespec timeout = {0,};
 
         priv = this->private;
         if (!priv) {
@@ -142,7 +129,7 @@ gf_quiesce_enqueue (xlator_t *this, call_stub_t *stub)
 
         if (!priv->timer) {
                 timeout.tv_sec = 20;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
 
                 priv->timer = gf_timer_call_after (this->ctx,
                                                    timeout,
@@ -191,7 +178,8 @@ out:
 
 int32_t
 quiesce_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -201,10 +189,10 @@ quiesce_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_stat_stub (frame, default_stat_resume,
-                                      &local->loc);
+                                      &local->loc, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -212,7 +200,7 @@ quiesce_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -221,7 +209,7 @@ out:
 
 int32_t
 quiesce_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -231,9 +219,9 @@ quiesce_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_access_stub (frame, default_access_resume,
-                                        &local->loc, local->flag);
+                                        &local->loc, local->flag, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (access, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (access, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -241,7 +229,7 @@ quiesce_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -251,7 +239,7 @@ out:
 int32_t
 quiesce_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf)
+                      struct iatt *buf, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -261,10 +249,10 @@ quiesce_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_readlink_stub (frame, default_readlink_resume,
-                                          &local->loc, local->size);
+                                          &local->loc, local->size, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM,
-                                             NULL, NULL);
+                                             NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -272,7 +260,7 @@ quiesce_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf);
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -281,7 +269,7 @@ out:
 
 int32_t
 quiesce_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd)
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -292,10 +280,10 @@ quiesce_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_open_stub (frame, default_open_resume,
                                       &local->loc, local->flag, local->fd,
-                                      local->wbflags);
+                                      xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (open, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -303,7 +291,7 @@ quiesce_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -313,7 +301,7 @@ out:
 int32_t
 quiesce_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -324,10 +312,10 @@ quiesce_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_readv_stub (frame, default_readv_resume,
                                        local->fd, local->size, local->offset,
-                                       local->io_flag);
+                                       local->io_flag, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM,
-                                             NULL, 0, NULL, NULL);
+                                             NULL, 0, NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -336,7 +324,7 @@ quiesce_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
+                             stbuf, iobref, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -345,7 +333,7 @@ out:
 
 int32_t
 quiesce_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -355,9 +343,9 @@ quiesce_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_flush_stub (frame, default_flush_resume,
-                                       local->fd);
+                                       local->fd, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -365,7 +353,7 @@ quiesce_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -377,7 +365,7 @@ out:
 int32_t
 quiesce_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -387,10 +375,10 @@ quiesce_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fsync_stub (frame, default_fsync_resume,
-                                       local->fd, local->flag);
+                                       local->fd, local->flag, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM,
-                                             NULL, NULL);
+                                             NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -398,7 +386,7 @@ quiesce_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -407,7 +395,7 @@ out:
 
 int32_t
 quiesce_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -417,10 +405,10 @@ quiesce_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fstat_stub (frame, default_fstat_resume,
-                                       local->fd);
+                                       local->fd, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -428,7 +416,7 @@ quiesce_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -437,7 +425,7 @@ out:
 
 int32_t
 quiesce_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd)
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -447,10 +435,10 @@ quiesce_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_opendir_stub (frame, default_opendir_resume,
-                                         &local->loc, local->fd);
+                                         &local->loc, local->fd, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -458,7 +446,7 @@ quiesce_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -467,7 +455,7 @@ out:
 
 int32_t
 quiesce_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -477,9 +465,9 @@ quiesce_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume,
-                                          local->fd, local->flag);
+                                          local->fd, local->flag, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -487,7 +475,7 @@ quiesce_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -496,7 +484,7 @@ out:
 
 int32_t
 quiesce_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                    int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -506,10 +494,10 @@ quiesce_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_statfs_stub (frame, default_statfs_resume,
-                                        &local->loc);
+                                        &local->loc, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -517,7 +505,7 @@ quiesce_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -526,7 +514,7 @@ out:
 
 int32_t
 quiesce_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -536,10 +524,10 @@ quiesce_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume,
-                                           local->fd, local->name);
+                                           local->fd, local->name, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -547,7 +535,7 @@ quiesce_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -557,7 +545,7 @@ out:
 
 int32_t
 quiesce_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -567,10 +555,10 @@ quiesce_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_getxattr_stub (frame, default_getxattr_resume,
-                                          &local->loc, local->name);
+                                          &local->loc, local->name, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -578,7 +566,7 @@ quiesce_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -589,7 +577,7 @@ out:
 int32_t
 quiesce_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                       uint8_t *strong_checksum)
+                       uint8_t *strong_checksum, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -599,10 +587,10 @@ quiesce_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_rchecksum_stub (frame, default_rchecksum_resume,
-                                           local->fd, local->offset, local->flag);
+                                           local->fd, local->offset, local->flag, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM,
-                                             0, NULL);
+                                             0, NULL, NULL);
                         goto out;
                 }
 
@@ -611,7 +599,7 @@ quiesce_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum);
+                             strong_checksum, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -621,7 +609,7 @@ out:
 
 int32_t
 quiesce_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -631,10 +619,10 @@ quiesce_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_readdir_stub (frame, default_readdir_resume,
-                                         local->fd, local->size, local->offset);
+                                         local->fd, local->size, local->offset, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -642,7 +630,7 @@ quiesce_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -652,7 +640,7 @@ out:
 
 int32_t
 quiesce_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -666,7 +654,7 @@ quiesce_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                           local->dict);
                 if (!stub) {
                         STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -674,7 +662,7 @@ quiesce_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -687,7 +675,7 @@ out:
 int32_t
 quiesce_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf)
+                    struct iatt *postbuf, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -702,10 +690,10 @@ quiesce_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 stub = fop_writev_stub (frame, default_writev_resume,
                                         local->fd, local->vector, local->flag,
                                         local->offset, local->io_flags,
-                                        local->iobref);
+                                        local->iobref, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM,
-                                             NULL, NULL);
+                                             NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -713,7 +701,7 @@ quiesce_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -722,7 +710,7 @@ out:
 
 int32_t
 quiesce_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
+                     int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -736,10 +724,10 @@ quiesce_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_xattrop_stub (frame, default_xattrop_resume,
                                          &local->loc, local->xattrop_flags,
-                                         local->dict);
+                                         local->dict, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (xattrop, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -747,7 +735,7 @@ quiesce_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -756,7 +744,7 @@ out:
 
 int32_t
 quiesce_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -770,10 +758,10 @@ quiesce_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fxattrop_stub (frame, default_fxattrop_resume,
                                           local->fd, local->xattrop_flags,
-                                          local->dict);
+                                          local->dict, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -781,7 +769,7 @@ quiesce_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -790,7 +778,7 @@ out:
 
 int32_t
 quiesce_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -803,10 +791,10 @@ quiesce_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_lk_stub (frame, default_lk_resume,
-                                    local->fd, local->flag, &local->flock);
+                                    local->fd, local->flag, &local->flock, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -814,7 +802,7 @@ quiesce_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -823,7 +811,7 @@ out:
 
 int32_t
 quiesce_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -837,9 +825,9 @@ quiesce_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_inodelk_stub (frame, default_inodelk_resume,
                                          local->volname, &local->loc,
-                                         local->flag, &local->flock);
+                                         local->flag, &local->flock, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -847,7 +835,7 @@ quiesce_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -857,7 +845,7 @@ out:
 
 int32_t
 quiesce_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -871,9 +859,9 @@ quiesce_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_finodelk_stub (frame, default_finodelk_resume,
                                          local->volname, local->fd,
-                                         local->flag, &local->flock);
+                                         local->flag, &local->flock, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -881,7 +869,7 @@ quiesce_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -890,7 +878,7 @@ out:
 
 int32_t
 quiesce_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -904,9 +892,9 @@ quiesce_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_entrylk_stub (frame, default_entrylk_resume,
                                          local->volname, &local->loc,
-                                         local->name, local->cmd, local->type);
+                                         local->name, local->cmd, local->type, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -914,7 +902,7 @@ quiesce_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -923,7 +911,7 @@ out:
 
 int32_t
 quiesce_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -937,9 +925,9 @@ quiesce_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fentrylk_stub (frame, default_fentrylk_resume,
                                           local->volname, local->fd,
-                                          local->name, local->cmd, local->type);
+                                          local->name, local->cmd, local->type, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -947,7 +935,7 @@ quiesce_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -957,7 +945,7 @@ out:
 int32_t
 quiesce_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost)
+                     struct iatt *statpost, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -970,10 +958,10 @@ quiesce_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_setattr_stub (frame, default_setattr_resume,
-                                         &local->loc, &local->stbuf, local->flag);
+                                         &local->loc, &local->stbuf, local->flag, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM,
-                                             NULL, NULL);
+                                             NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -982,7 +970,7 @@ quiesce_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+                             statpost, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -992,7 +980,7 @@ out:
 int32_t
 quiesce_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                      struct iatt *statpost)
+                      struct iatt *statpost, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1006,10 +994,10 @@ quiesce_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fsetattr_stub (frame, default_fsetattr_resume,
-                                          local->fd, &local->stbuf, local->flag);
+                                          local->fd, &local->stbuf, local->flag, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM,
-                                             NULL, NULL);
+                                             NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -1018,7 +1006,7 @@ quiesce_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+                             statpost, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -1036,7 +1024,7 @@ int32_t
 quiesce_removexattr (call_frame_t *frame,
 		     xlator_t *this,
 		     loc_t *loc,
-		     const char *name)
+		     const char *name, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1049,14 +1037,14 @@ quiesce_removexattr (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->removexattr,
                             loc,
-                            name);
-	        return 0;
+                            name, xdata);
+                return 0;
         }
 
         stub = fop_removexattr_stub (frame, default_removexattr_resume,
-                                     loc, name);
+                                     loc, name, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1069,7 +1057,7 @@ int32_t
 quiesce_truncate (call_frame_t *frame,
 		  xlator_t *this,
 		  loc_t *loc,
-		  off_t offset)
+		  off_t offset, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1082,13 +1070,13 @@ quiesce_truncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->truncate,
                             loc,
-                            offset);
-	        return 0;
+                            offset, xdata);
+                return 0;
         }
 
-        stub = fop_truncate_stub (frame, default_truncate_resume, loc, offset);
+        stub = fop_truncate_stub (frame, default_truncate_resume, loc, offset, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1102,7 +1090,7 @@ quiesce_fsetxattr (call_frame_t *frame,
                    xlator_t *this,
                    fd_t *fd,
                    dict_t *dict,
-                   int32_t flags)
+                   int32_t flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1116,14 +1104,14 @@ quiesce_fsetxattr (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->fsetxattr,
                             fd,
                             dict,
-                            flags);
-	        return 0;
+                            flags, xdata);
+                return 0;
         }
 
         stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume,
-                                   fd, dict, flags);
+                                   fd, dict, flags, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1137,7 +1125,7 @@ quiesce_setxattr (call_frame_t *frame,
 		  xlator_t *this,
 		  loc_t *loc,
 		  dict_t *dict,
-		  int32_t flags)
+		  int32_t flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1151,14 +1139,14 @@ quiesce_setxattr (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->setxattr,
                             loc,
                             dict,
-                            flags);
-	        return 0;
+                            flags, xdata);
+                return 0;
         }
 
         stub = fop_setxattr_stub (frame, default_setxattr_resume,
-                                  loc, dict, flags);
+                                  loc, dict, flags, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1170,7 +1158,7 @@ quiesce_setxattr (call_frame_t *frame,
 int32_t
 quiesce_create (call_frame_t *frame, xlator_t *this,
 		loc_t *loc, int32_t flags, mode_t mode,
-                fd_t *fd, dict_t *params)
+                mode_t umask, fd_t *fd, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1183,15 +1171,15 @@ quiesce_create (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_create_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->create,
-                            loc, (flags & ~O_APPEND), mode, fd, params);
-	        return 0;
+                            loc, (flags & ~O_APPEND), mode, umask, fd, xdata);
+                return 0;
         }
 
         stub = fop_create_stub (frame, default_create_resume,
-                                loc, (flags & ~O_APPEND), mode, fd, params);
+                                loc, (flags & ~O_APPEND), mode, umask, fd, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (create, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1204,7 +1192,7 @@ int32_t
 quiesce_link (call_frame_t *frame,
 	      xlator_t *this,
 	      loc_t *oldloc,
-	      loc_t *newloc)
+	      loc_t *newloc, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1216,14 +1204,14 @@ quiesce_link (call_frame_t *frame,
                             default_link_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc);
-	        return 0;
+                            oldloc, newloc, xdata);
+                return 0;
         }
 
-        stub = fop_link_stub (frame, default_link_resume, oldloc, newloc);
+        stub = fop_link_stub (frame, default_link_resume, oldloc, newloc, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (link, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1236,7 +1224,7 @@ int32_t
 quiesce_rename (call_frame_t *frame,
 		xlator_t *this,
 		loc_t *oldloc,
-		loc_t *newloc)
+		loc_t *newloc, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1248,14 +1236,14 @@ quiesce_rename (call_frame_t *frame,
                             default_rename_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc);
-	        return 0;
+                            oldloc, newloc, xdata);
+                return 0;
         }
 
-        stub = fop_rename_stub (frame, default_rename_resume, oldloc, newloc);
+        stub = fop_rename_stub (frame, default_rename_resume, oldloc, newloc, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1267,7 +1255,7 @@ quiesce_rename (call_frame_t *frame,
 
 int
 quiesce_symlink (call_frame_t *frame, xlator_t *this,
-		 const char *linkpath, loc_t *loc, dict_t *params)
+		 const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1278,15 +1266,15 @@ quiesce_symlink (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_symlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, params);
-	        return 0;
+                            linkpath, loc, umask, xdata);
+                return 0;
         }
 
         stub = fop_symlink_stub (frame, default_symlink_resume,
-                                 linkpath, loc, params);
+                                 linkpath, loc, umask, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (symlink, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1297,7 +1285,7 @@ quiesce_symlink (call_frame_t *frame, xlator_t *this,
 
 
 int
-quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1308,13 +1296,13 @@ quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
                 STACK_WIND (frame, default_rmdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags);
-	        return 0;
+                            loc, flags, xdata);
+                return 0;
         }
 
-        stub = fop_rmdir_stub (frame, default_rmdir_resume, loc, flags);
+        stub = fop_rmdir_stub (frame, default_rmdir_resume, loc, flags, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1326,7 +1314,7 @@ quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 int32_t
 quiesce_unlink (call_frame_t *frame,
 		xlator_t *this,
-		loc_t *loc)
+		loc_t *loc, int xflag, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1338,13 +1326,13 @@ quiesce_unlink (call_frame_t *frame,
                             default_unlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->unlink,
-                            loc);
-	        return 0;
+                            loc, xflag, xdata);
+                return 0;
         }
 
-        stub = fop_unlink_stub (frame, default_unlink_resume, loc);
+        stub = fop_unlink_stub (frame, default_unlink_resume, loc, xflag, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1355,7 +1343,7 @@ quiesce_unlink (call_frame_t *frame,
 
 int
 quiesce_mkdir (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, mode_t mode, dict_t *params)
+	       loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1366,15 +1354,15 @@ quiesce_mkdir (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mkdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, params);
-	        return 0;
+                            loc, mode, umask, xdata);
+                return 0;
         }
 
         stub = fop_mkdir_stub (frame, default_mkdir_resume,
-                               loc, mode, params);
+                               loc, mode, umask, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (mkdir, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1386,7 +1374,7 @@ quiesce_mkdir (call_frame_t *frame, xlator_t *this,
 
 int
 quiesce_mknod (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, mode_t mode, dev_t rdev, dict_t *parms)
+	       loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1397,15 +1385,15 @@ quiesce_mknod (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mknod_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, parms);
-	        return 0;
+                            loc, mode, rdev, umask, xdata);
+                return 0;
         }
 
         stub = fop_mknod_stub (frame, default_mknod_resume,
-                               loc, mode, rdev, parms);
+                               loc, mode, rdev, umask, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (mknod, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1418,7 +1406,7 @@ int32_t
 quiesce_ftruncate (call_frame_t *frame,
 		   xlator_t *this,
 		   fd_t *fd,
-		   off_t offset)
+		   off_t offset, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1431,13 +1419,13 @@ quiesce_ftruncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->ftruncate,
                             fd,
-                            offset);
-	        return 0;
+                            offset, xdata);
+                return 0;
         }
 
-        stub = fop_ftruncate_stub (frame, default_ftruncate_resume, fd, offset);
+        stub = fop_ftruncate_stub (frame, default_ftruncate_resume, fd, offset, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1452,7 +1440,7 @@ int32_t
 quiesce_readlink (call_frame_t *frame,
 		  xlator_t *this,
 		  loc_t *loc,
-		  size_t size)
+		  size_t size, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1471,13 +1459,13 @@ quiesce_readlink (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readlink,
                             loc,
-                            size);
-	        return 0;
+                            size, xdata);
+                return 0;
         }
 
-        stub = fop_readlink_stub (frame, default_readlink_resume, loc, size);
+        stub = fop_readlink_stub (frame, default_readlink_resume, loc, size, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1491,7 +1479,7 @@ int32_t
 quiesce_access (call_frame_t *frame,
 		xlator_t *this,
 		loc_t *loc,
-		int32_t mask)
+		int32_t mask, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1510,13 +1498,13 @@ quiesce_access (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->access,
                             loc,
-                            mask);
-	        return 0;
+                            mask, xdata);
+                return 0;
         }
 
-        stub = fop_access_stub (frame, default_access_resume, loc, mask);
+        stub = fop_access_stub (frame, default_access_resume, loc, mask, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (access, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (access, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1529,7 +1517,7 @@ int32_t
 quiesce_fgetxattr (call_frame_t *frame,
                    xlator_t *this,
                    fd_t *fd,
-                   const char *name)
+                   const char *name, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1550,13 +1538,13 @@ quiesce_fgetxattr (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fgetxattr,
                             fd,
-                            name);
+                            name, xdata);
                 return 0;
         }
 
-        stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume, fd, name);
+        stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume, fd, name, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1568,7 +1556,7 @@ quiesce_fgetxattr (call_frame_t *frame,
 int32_t
 quiesce_statfs (call_frame_t *frame,
 		xlator_t *this,
-		loc_t *loc)
+		loc_t *loc, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1585,13 +1573,13 @@ quiesce_statfs (call_frame_t *frame,
                             quiesce_statfs_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->statfs,
-                            loc);
-	        return 0;
+                            loc, xdata);
+                return 0;
         }
 
-        stub = fop_statfs_stub (frame, default_statfs_resume, loc);
+        stub = fop_statfs_stub (frame, default_statfs_resume, loc, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1604,7 +1592,7 @@ int32_t
 quiesce_fsyncdir (call_frame_t *frame,
 		  xlator_t *this,
 		  fd_t *fd,
-		  int32_t flags)
+		  int32_t flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1623,13 +1611,13 @@ quiesce_fsyncdir (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsyncdir,
                             fd,
-                            flags);
-	        return 0;
+                            flags, xdata);
+                return 0;
         }
 
-        stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume, fd, flags);
+        stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume, fd, flags, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1641,7 +1629,7 @@ quiesce_fsyncdir (call_frame_t *frame,
 int32_t
 quiesce_opendir (call_frame_t *frame,
 		 xlator_t *this,
-		 loc_t *loc, fd_t *fd)
+		 loc_t *loc, fd_t *fd, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1659,13 +1647,13 @@ quiesce_opendir (call_frame_t *frame,
                             quiesce_opendir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->opendir,
-                            loc, fd);
-	        return 0;
+                            loc, fd, xdata);
+                return 0;
         }
 
-        stub = fop_opendir_stub (frame, default_opendir_resume, loc, fd);
+        stub = fop_opendir_stub (frame, default_opendir_resume, loc, fd, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1677,7 +1665,7 @@ quiesce_opendir (call_frame_t *frame,
 int32_t
 quiesce_fstat (call_frame_t *frame,
 	       xlator_t *this,
-	       fd_t *fd)
+	       fd_t *fd, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1694,13 +1682,13 @@ quiesce_fstat (call_frame_t *frame,
                             quiesce_fstat_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fstat,
-                            fd);
-	        return 0;
+                            fd, xdata);
+                return 0;
         }
 
-        stub = fop_fstat_stub (frame, default_fstat_resume, fd);
+        stub = fop_fstat_stub (frame, default_fstat_resume, fd, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1713,7 +1701,7 @@ int32_t
 quiesce_fsync (call_frame_t *frame,
 	       xlator_t *this,
 	       fd_t *fd,
-	       int32_t flags)
+	       int32_t flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1732,13 +1720,13 @@ quiesce_fsync (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsync,
                             fd,
-                            flags);
-	        return 0;
+                            flags, xdata);
+                return 0;
         }
 
-        stub = fop_fsync_stub (frame, default_fsync_resume, fd, flags);
+        stub = fop_fsync_stub (frame, default_fsync_resume, fd, flags, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1750,7 +1738,7 @@ quiesce_fsync (call_frame_t *frame,
 int32_t
 quiesce_flush (call_frame_t *frame,
 	       xlator_t *this,
-	       fd_t *fd)
+	       fd_t *fd, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1767,13 +1755,13 @@ quiesce_flush (call_frame_t *frame,
                             quiesce_flush_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->flush,
-                            fd);
-	        return 0;
+                            fd, xdata);
+                return 0;
         }
 
-        stub = fop_flush_stub (frame, default_flush_resume, fd);
+        stub = fop_flush_stub (frame, default_flush_resume, fd, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1789,7 +1777,7 @@ quiesce_writev (call_frame_t *frame,
 		struct iovec *vector,
 		int32_t count,
 		off_t off, uint32_t flags,
-                struct iobref *iobref)
+                struct iobref *iobref, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1805,14 +1793,14 @@ quiesce_writev (call_frame_t *frame,
                             vector,
                             count,
                             off, flags,
-                            iobref);
-	        return 0;
+                            iobref, xdata);
+                return 0;
         }
 
         stub = fop_writev_stub (frame, default_writev_resume,
-                                fd, vector, count, off, flags, iobref);
+                                fd, vector, count, off, flags, iobref, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1826,7 +1814,7 @@ quiesce_readv (call_frame_t *frame,
 	       xlator_t *this,
 	       fd_t *fd,
 	       size_t size,
-	       off_t offset, uint32_t flags)
+	       off_t offset, uint32_t flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1848,15 +1836,15 @@ quiesce_readv (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->readv,
                             fd,
                             size,
-                            offset, flags);
-	        return 0;
+                            offset, flags, xdata);
+                return 0;
         }
 
         stub = fop_readv_stub (frame, default_readv_resume, fd, size, offset,
-                               flags);
+                               flags, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM,
-                                     NULL, 0, NULL, NULL);
+                                     NULL, 0, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1871,7 +1859,7 @@ quiesce_open (call_frame_t *frame,
 	      xlator_t *this,
 	      loc_t *loc,
 	      int32_t flags, fd_t *fd,
-              int32_t wbflags)
+              dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1887,21 +1875,20 @@ quiesce_open (call_frame_t *frame,
                 /* Don't send O_APPEND below, as write() re-transmittions can
                    fail with O_APPEND */
                 local->flag = (flags & ~O_APPEND);
-                local->wbflags = wbflags;
                 frame->local = local;
 
                 STACK_WIND (frame,
                             quiesce_open_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->open,
-                            loc, (flags & ~O_APPEND), fd, wbflags);
-	        return 0;
+                            loc, (flags & ~O_APPEND), fd, xdata);
+                return 0;
         }
 
         stub = fop_open_stub (frame, default_open_resume, loc,
-                              (flags & ~O_APPEND), fd, wbflags);
+                              (flags & ~O_APPEND), fd, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1914,7 +1901,7 @@ int32_t
 quiesce_getxattr (call_frame_t *frame,
 		  xlator_t *this,
 		  loc_t *loc,
-		  const char *name)
+		  const char *name, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1935,13 +1922,13 @@ quiesce_getxattr (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->getxattr,
                             loc,
-                            name);
-	        return 0;
+                            name, xdata);
+                return 0;
         }
 
-        stub = fop_getxattr_stub (frame, default_getxattr_resume, loc, name);
+        stub = fop_getxattr_stub (frame, default_getxattr_resume, loc, name, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1956,7 +1943,7 @@ quiesce_xattrop (call_frame_t *frame,
 		 xlator_t *this,
 		 loc_t *loc,
 		 gf_xattrop_flags_t flags,
-		 dict_t *dict)
+		 dict_t *dict, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1970,14 +1957,14 @@ quiesce_xattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->xattrop,
                             loc,
                             flags,
-                            dict);
-	        return 0;
+                            dict, xdata);
+                return 0;
         }
 
         stub = fop_xattrop_stub (frame, default_xattrop_resume,
-                                 loc, flags, dict);
+                                 loc, flags, dict, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1991,7 +1978,7 @@ quiesce_fxattrop (call_frame_t *frame,
 		  xlator_t *this,
 		  fd_t *fd,
 		  gf_xattrop_flags_t flags,
-		  dict_t *dict)
+		  dict_t *dict, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2005,14 +1992,14 @@ quiesce_fxattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->fxattrop,
                             fd,
                             flags,
-                            dict);
-	        return 0;
+                            dict, xdata);
+                return 0;
         }
 
         stub = fop_fxattrop_stub (frame, default_fxattrop_resume,
-                                  fd, flags, dict);
+                                  fd, flags, dict, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -2026,7 +2013,7 @@ quiesce_lk (call_frame_t *frame,
 	    xlator_t *this,
 	    fd_t *fd,
 	    int32_t cmd,
-	    struct gf_flock *lock)
+	    struct gf_flock *lock, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2040,13 +2027,13 @@ quiesce_lk (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->lk,
                             fd,
                             cmd,
-                            lock);
-	        return 0;
+                            lock, xdata);
+                return 0;
         }
 
-        stub = fop_lk_stub (frame, default_lk_resume, fd, cmd, lock);
+        stub = fop_lk_stub (frame, default_lk_resume, fd, cmd, lock, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -2059,7 +2046,7 @@ quiesce_lk (call_frame_t *frame,
 int32_t
 quiesce_inodelk (call_frame_t *frame, xlator_t *this,
 		 const char *volume, loc_t *loc, int32_t cmd,
-                 struct gf_flock *lock)
+                 struct gf_flock *lock, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2071,14 +2058,14 @@ quiesce_inodelk (call_frame_t *frame, xlator_t *this,
                             default_inodelk_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->inodelk,
-                            volume, loc, cmd, lock);
-	        return 0;
+                            volume, loc, cmd, lock, xdata);
+                return 0;
         }
 
         stub = fop_inodelk_stub (frame, default_inodelk_resume,
-                                 volume, loc, cmd, lock);
+                                 volume, loc, cmd, lock, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -2089,7 +2076,7 @@ quiesce_inodelk (call_frame_t *frame, xlator_t *this,
 
 int32_t
 quiesce_finodelk (call_frame_t *frame, xlator_t *this,
-		  const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
+		  const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2101,14 +2088,14 @@ quiesce_finodelk (call_frame_t *frame, xlator_t *this,
                             default_finodelk_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->finodelk,
-                            volume, fd, cmd, lock);
-	        return 0;
+                            volume, fd, cmd, lock, xdata);
+                return 0;
         }
 
         stub = fop_finodelk_stub (frame, default_finodelk_resume,
-                                  volume, fd, cmd, lock);
+                                  volume, fd, cmd, lock, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -2120,7 +2107,7 @@ quiesce_finodelk (call_frame_t *frame, xlator_t *this,
 int32_t
 quiesce_entrylk (call_frame_t *frame, xlator_t *this,
 		 const char *volume, loc_t *loc, const char *basename,
-		 entrylk_cmd cmd, entrylk_type type)
+		 entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2131,14 +2118,14 @@ quiesce_entrylk (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_entrylk_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->entrylk,
-                            volume, loc, basename, cmd, type);
-	        return 0;
+                            volume, loc, basename, cmd, type, xdata);
+                return 0;
         }
 
         stub = fop_entrylk_stub (frame, default_entrylk_resume,
-                                 volume, loc, basename, cmd, type);
+                                 volume, loc, basename, cmd, type, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -2150,7 +2137,7 @@ quiesce_entrylk (call_frame_t *frame, xlator_t *this,
 int32_t
 quiesce_fentrylk (call_frame_t *frame, xlator_t *this,
 		  const char *volume, fd_t *fd, const char *basename,
-		  entrylk_cmd cmd, entrylk_type type)
+		  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2161,14 +2148,14 @@ quiesce_fentrylk (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_fentrylk_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fentrylk,
-                            volume, fd, basename, cmd, type);
-	        return 0;
+                            volume, fd, basename, cmd, type, xdata);
+                return 0;
         }
 
         stub = fop_fentrylk_stub (frame, default_fentrylk_resume,
-                                  volume, fd, basename, cmd, type);
+                                  volume, fd, basename, cmd, type, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -2181,7 +2168,7 @@ int32_t
 quiesce_rchecksum (call_frame_t *frame,
                    xlator_t *this,
                    fd_t *fd, off_t offset,
-                   int32_t len)
+                   int32_t len, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2200,14 +2187,14 @@ quiesce_rchecksum (call_frame_t *frame,
                             quiesce_rchecksum_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rchecksum,
-                            fd, offset, len);
-	        return 0;
+                            fd, offset, len, xdata);
+                return 0;
         }
 
         stub = fop_rchecksum_stub (frame, default_rchecksum_resume,
-                                   fd, offset, len);
+                                   fd, offset, len, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM, 0, NULL);
+                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM, 0, NULL, NULL);
                 return 0;
         }
 
@@ -2222,7 +2209,7 @@ quiesce_readdir (call_frame_t *frame,
 		 xlator_t *this,
 		 fd_t *fd,
 		 size_t size,
-		 off_t off)
+		 off_t off, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2241,13 +2228,13 @@ quiesce_readdir (call_frame_t *frame,
                             quiesce_readdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off);
-	        return 0;
+                            fd, size, off, xdata);
+                return 0;
         }
 
-        stub = fop_readdir_stub (frame, default_readdir_resume, fd, size, off);
+        stub = fop_readdir_stub (frame, default_readdir_resume, fd, size, off, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -2283,13 +2270,13 @@ quiesce_readdirp (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readdirp,
                             fd, size, off, dict);
-	        return 0;
+                return 0;
         }
 
         stub = fop_readdirp_stub (frame, default_readdirp_resume, fd, size,
                                   off, dict);
         if (!stub) {
-                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -2303,7 +2290,7 @@ quiesce_setattr (call_frame_t *frame,
                  xlator_t *this,
                  loc_t *loc,
                  struct iatt *stbuf,
-                 int32_t valid)
+                 int32_t valid, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2315,14 +2302,14 @@ quiesce_setattr (call_frame_t *frame,
                             default_setattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid);
-	        return 0;
+                            loc, stbuf, valid, xdata);
+                return 0;
         }
 
         stub = fop_setattr_stub (frame, default_setattr_resume,
-                                   loc, stbuf, valid);
+                                   loc, stbuf, valid, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -2335,7 +2322,7 @@ quiesce_setattr (call_frame_t *frame,
 int32_t
 quiesce_stat (call_frame_t *frame,
 	      xlator_t *this,
-	      loc_t *loc)
+	      loc_t *loc, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2352,13 +2339,13 @@ quiesce_stat (call_frame_t *frame,
                             quiesce_stat_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->stat,
-                            loc);
-	        return 0;
+                            loc, xdata);
+                return 0;
         }
 
-        stub = fop_stat_stub (frame, default_stat_resume, loc);
+        stub = fop_stat_stub (frame, default_stat_resume, loc, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -2389,9 +2376,8 @@ quiesce_lookup (call_frame_t *frame,
                             quiesce_lookup_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->lookup,
-                            loc,
-                            xattr_req);
-	        return 0;
+                            loc, xattr_req);
+                return 0;
         }
 
         stub = fop_lookup_stub (frame, default_lookup_resume, loc, xattr_req);
@@ -2411,7 +2397,7 @@ quiesce_fsetattr (call_frame_t *frame,
                   xlator_t *this,
                   fd_t *fd,
                   struct iatt *stbuf,
-                  int32_t valid)
+                  int32_t valid, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2423,14 +2409,14 @@ quiesce_fsetattr (call_frame_t *frame,
                             default_fsetattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid);
-	        return 0;
+                            fd, stbuf, valid, xdata);
+                return 0;
         }
 
         stub = fop_fsetattr_stub (frame, default_fsetattr_resume,
-                                  fd, stbuf, valid);
+                                  fd, stbuf, valid, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -2506,7 +2492,7 @@ notify (xlator_t *this, int event, void *data, ...)
 {
         int             ret     = 0;
         quiesce_priv_t *priv    = NULL;
-        struct timeval  timeout = {0,};
+        struct timespec timeout = {0,};
 
         priv = this->private;
         if (!priv)
@@ -2539,7 +2525,7 @@ notify (xlator_t *this, int event, void *data, ...)
                 if (priv->timer)
                         break;
                 timeout.tv_sec = 20;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
 
                 priv->timer = gf_timer_call_after (this->ctx,
                                                    timeout,
@@ -2613,12 +2599,10 @@ struct xlator_fops fops = {
 
 };
 
-struct xlator_dumpops dumpops = {
-};
+struct xlator_dumpops dumpops;
 
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 
 struct volume_options options[] = {
diff --git a/xlators/features/quiesce/src/quiesce.h b/xlators/features/quiesce/src/quiesce.h
index 1adb70ebe..878ed77e9 100644
--- a/xlators/features/quiesce/src/quiesce.h
+++ b/xlators/features/quiesce/src/quiesce.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __QUIESCE_H__
 #define __QUIESCE_H__
 
@@ -25,7 +15,7 @@
 #include "xlator.h"
 #include "timer.h"
 
-#define GF_FOPS_EXPECTED_IN_PARALLEL 4096
+#define GF_FOPS_EXPECTED_IN_PARALLEL 512
 
 typedef struct {
         gf_timer_t       *timer;
diff --git a/xlators/features/quota/src/Makefile.am b/xlators/features/quota/src/Makefile.am
index 4baa5f06e..9546f4276 100644
--- a/xlators/features/quota/src/Makefile.am
+++ b/xlators/features/quota/src/Makefile.am
@@ -1,16 +1,17 @@
 xlator_LTLIBRARIES = quota.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-quota_la_LDFLAGS = -module -avoidversion
+quota_la_LDFLAGS = -module -avoid-version
 
 quota_la_SOURCES = quota.c
 quota_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = quota-mem-types.h quota.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/cluster/dht/src
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
 CLEANFILES = 
 
diff --git a/xlators/features/quota/src/quota-mem-types.h b/xlators/features/quota/src/quota-mem-types.h
index da28be5b3..3082865da 100644
--- a/xlators/features/quota/src/quota-mem-types.h
+++ b/xlators/features/quota/src/quota-mem-types.h
@@ -1,30 +1,19 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __QUOTA_MEM_TYPES_H__
 #define __QUOTA_MEM_TYPES_H__
 
 #include "mem-types.h"
 
 enum gf_quota_mem_types_ {
-        gf_quota_mt_quota_local_t = gf_common_mt_end + 1,
-        gf_quota_mt_quota_priv_t,
+        gf_quota_mt_quota_priv_t = gf_common_mt_end + 1,
         gf_quota_mt_quota_inode_ctx_t,
         gf_quota_mt_loc_t,
         gf_quota_mt_char,
diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index 3710e8f90..c527e7ca7 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <fnmatch.h>
 
 #include "quota.h"
@@ -89,7 +79,7 @@ quota_inode_loc_fill (inode_t *inode, loc_t *loc)
 
         parent = inode_parent (inode, 0, NULL);
         if (!parent) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "cannot find parent for inode (gfid:%s)",
                         uuid_utoa (inode->gfid));
                 goto err;
@@ -98,7 +88,7 @@ quota_inode_loc_fill (inode_t *inode, loc_t *loc)
 ignore_parent:
         ret = inode_path (inode, NULL, &resolvedpath);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "cannot construct path for inode (gfid:%s)",
                         uuid_utoa (inode->gfid));
                 goto err;
@@ -136,19 +126,19 @@ quota_local_cleanup (xlator_t *this, quota_local_t *local)
         inode_unref (local->inode);
         LOCK_DESTROY (&local->lock);
 
+        mem_put (local);
 out:
         return 0;
 }
 
 
-quota_local_t *
+static inline quota_local_t *
 quota_local_new ()
 {
-        quota_local_t     *local  = NULL;
-        GF_UNUSED int32_t  ret    = 0;
-
-        QUOTA_LOCAL_ALLOC_OR_GOTO (local, quota_local_t, err);
-err:
+        quota_local_t *local = NULL;
+        local = mem_get0 (THIS->local_pool);
+        if (local)
+                LOCK_INIT (&local->lock);
         return local;
 }
 
@@ -195,7 +185,8 @@ out:
 
 int32_t
 quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
         quota_local_t     *local          = NULL;
         uint32_t           validate_count = 0, link_count = 0;
@@ -388,7 +379,7 @@ quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
                 }
 
                 if (parent == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "cannot find parent for inode (gfid:%s), hence "
                                 "aborting enforcing quota-limits and continuing"
                                 " with the fop", uuid_utoa (_inode->gfid));
@@ -458,7 +449,7 @@ validate:
 
         STACK_WIND (frame, quota_validate_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr, &local->validate_loc,
-                    QUOTA_SIZE_KEY);
+                    QUOTA_SIZE_KEY, NULL);
 
 loc_fill_failed:
         inode_unref (_inode);
@@ -497,6 +488,8 @@ quota_get_limit_value (inode_t *inode, xlator_t *this, int64_t *n)
         }
 
 out:
+        GF_FREE (path);
+
         return ret;
 }
 
@@ -577,16 +570,20 @@ quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
 {
-        int32_t            ret    = -1;
-        char               found  = 0;
-        quota_local_t     *local  = NULL;
-        quota_inode_ctx_t *ctx    = NULL;
-        quota_dentry_t    *dentry = NULL;
-        int64_t           *size   = 0;
-        uint64_t           value  = 0;
+        int32_t            ret        = -1;
+        char               found      = 0;
+        quota_local_t     *local      = NULL;
+        quota_inode_ctx_t *ctx        = NULL;
+        quota_dentry_t    *dentry     = NULL;
+        int64_t           *size       = 0;
+        uint64_t           value      = 0;
+        limits_t          *limit_node = NULL;
+        quota_priv_t      *priv       = NULL;
 
         local = frame->local;
 
+        priv = this->private;
+
         inode_ctx_get (inode, this, &value);
         ctx = (quota_inode_ctx_t *)(unsigned long)value;
 
@@ -597,8 +594,17 @@ quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto unwind;
         }
 
-        if (local->loc.name == NULL)
-                goto unwind;
+        LOCK (&priv->lock);
+        {
+                list_for_each_entry (limit_node, &priv->limit_head,
+                                     limit_list) {
+                        if (strcmp (local->loc.path, limit_node->path) == 0) {
+                                uuid_copy (limit_node->gfid, buf->ia_gfid);
+                                break;
+                        }
+                }
+        }
+        UNLOCK (&priv->lock);
 
         ret = quota_inode_ctx_get (local->loc.inode, local->limit, this, dict,
                                    buf, &ctx, 1);
@@ -633,6 +639,9 @@ quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto unlock;
                 }
 
+                if (local->loc.name == NULL)
+                        goto unlock;
+
                 list_for_each_entry (dentry, &ctx->parents, next) {
                         if ((strcmp (dentry->name, local->loc.name) == 0) &&
                             (uuid_compare (local->loc.parent->gfid,
@@ -774,7 +783,7 @@ quota_update_size (xlator_t *this, inode_t *inode, char *name, uuid_t par,
 
                 parent = inode_parent (_inode, trav_uuid, name);
                 if (parent == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "cannot find parent for inode (gfid:%s), hence "
                                 "aborting size updation of parents",
                                 uuid_utoa (_inode->gfid));
@@ -804,7 +813,7 @@ out:
 int32_t
 quota_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
         int32_t                  ret            = 0;
         uint64_t                 ctx_int        = 0;
@@ -848,7 +857,8 @@ quota_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 out:
-        QUOTA_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        QUOTA_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
 
         return 0;
 }
@@ -857,7 +867,7 @@ out:
 int32_t
 quota_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
                      struct iovec *vector, int32_t count, off_t off,
-                     uint32_t flags, struct iobref *iobref)
+                     uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
@@ -875,11 +885,11 @@ quota_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         STACK_WIND (frame, quota_writev_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
-                    flags, iobref);
+                    flags, iobref, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -887,7 +897,7 @@ unwind:
 int32_t
 quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
               struct iovec *vector, int32_t count, off_t off,
-              uint32_t flags, struct iobref *iobref)
+              uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         int32_t            ret     = -1, op_errno = EINVAL;
         int32_t            parents = 0;
@@ -919,7 +929,7 @@ quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         }
 
         stub = fop_writev_stub (frame, quota_writev_helper, fd, vector, count,
-                                off, flags, iobref);
+                                off, flags, iobref, xdata);
         if (stub == NULL) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -968,7 +978,7 @@ quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -977,17 +987,17 @@ int32_t
 quota_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
                  struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         QUOTA_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode,
-                            buf, preparent, postparent);
+                            buf, preparent, postparent, xdata);
         return 0;
 }
 
 
 int32_t
 quota_mkdir_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    mode_t mode, dict_t *params)
+                    mode_t mode, mode_t umask, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
@@ -1005,19 +1015,19 @@ quota_mkdir_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         STACK_WIND (frame, quota_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, params);
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL);
+                            NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dict_t *params)
+             mode_t umask, dict_t *xdata)
 {
         int32_t        ret            = 0, op_errno = 0;
         quota_local_t *local          = NULL;
@@ -1040,7 +1050,8 @@ quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 goto err;
         }
 
-        stub = fop_mkdir_stub (frame, quota_mkdir_helper, loc, mode, params);
+        stub = fop_mkdir_stub (frame, quota_mkdir_helper, loc, mode, umask,
+                               xdata);
         if (stub == NULL) {
                 op_errno = ENOMEM;
                 goto err;
@@ -1071,7 +1082,7 @@ quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         return 0;
 err:
         QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                            NULL);
+                            NULL, NULL);
 
         return 0;
 }
@@ -1081,7 +1092,7 @@ int32_t
 quota_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         int32_t            ret    = -1;
         quota_local_t     *local  = NULL;
@@ -1124,14 +1135,15 @@ unlock:
 
 unwind:
         QUOTA_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
-                            preparent, postparent);
+                            preparent, postparent, xdata);
         return 0;
 }
 
 
 int32_t
 quota_create_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+                     int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                     dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
@@ -1148,20 +1160,20 @@ quota_create_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         STACK_WIND (frame, quota_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, fd,
-                    params);
+                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
+                    fd, xdata);
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
+                            NULL, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 quota_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              mode_t mode, fd_t *fd, dict_t *params)
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         int32_t            ret            = -1;
         quota_local_t     *local          = NULL;
@@ -1181,7 +1193,7 @@ quota_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         }
 
         stub = fop_create_stub (frame, quota_create_helper, loc, flags, mode,
-                                fd, params);
+                                umask, fd, xdata);
         if (stub == NULL) {
                 goto err;
         }
@@ -1211,7 +1223,7 @@ quota_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         return 0;
 err:
         QUOTA_STACK_UNWIND (create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                            NULL);
+                            NULL, NULL);
 
         return 0;
 }
@@ -1220,7 +1232,7 @@ err:
 int32_t
 quota_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -1248,13 +1260,14 @@ quota_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (unlink, frame, op_ret, op_errno, preparent,
-                            postparent);
+                            postparent, xdata);
         return 0;
 }
 
 
 int32_t
-quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
         int32_t        ret = 0;
         quota_local_t *local = NULL;
@@ -1273,13 +1286,13 @@ quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         }
 
         STACK_WIND (frame, quota_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
 
         ret = 0;
 
 err:
         if (ret == -1) {
-                QUOTA_STACK_UNWIND (unlink, frame, -1, 0, NULL, NULL);
+                QUOTA_STACK_UNWIND (unlink, frame, -1, 0, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1290,7 +1303,7 @@ int32_t
 quota_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, inode_t *inode,
                 struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
         int32_t               ret          = -1;
         quota_local_t        *local        = NULL;
@@ -1355,7 +1368,7 @@ unlock:
 
 out:
         QUOTA_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent);
+                            preparent, postparent, xdata);
 
         return 0;
 }
@@ -1363,7 +1376,7 @@ out:
 
 int32_t
 quota_link_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                   loc_t *newloc)
+                   loc_t *newloc, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
@@ -1381,18 +1394,19 @@ quota_link_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
         }
 
         STACK_WIND (frame, quota_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc);
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL);
+                            NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
-quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
         int32_t            ret   = -1, op_errno = ENOMEM;
         quota_local_t     *local = NULL;
@@ -1412,7 +1426,7 @@ quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
                 goto err;
         }
 
-        stub = fop_link_stub (frame, quota_link_helper, oldloc, newloc);
+        stub = fop_link_stub (frame, quota_link_helper, oldloc, newloc, xdata);
         if (stub == NULL) {
                 goto err;
         }
@@ -1455,7 +1469,7 @@ quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
 err:
         if (ret < 0) {
                 QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
-                                    NULL, NULL);
+                                    NULL, NULL, NULL);
         }
 
         return 0;
@@ -1466,7 +1480,8 @@ int32_t
 quota_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
                   struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent)
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
 {
         int32_t               ret              = -1;
         quota_local_t        *local            = NULL;
@@ -1570,7 +1585,7 @@ unlock:
 
 out:
         QUOTA_STACK_UNWIND (rename, frame, op_ret, op_errno, buf, preoldparent,
-                            postoldparent, prenewparent, postnewparent);
+                            postoldparent, prenewparent, postnewparent, xdata);
 
         return 0;
 }
@@ -1578,7 +1593,7 @@ out:
 
 int32_t
 quota_rename_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                     loc_t *newloc)
+                     loc_t *newloc, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
@@ -1596,19 +1611,19 @@ quota_rename_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
         }
 
         STACK_WIND (frame, quota_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc);
+                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
+                            NULL, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-              loc_t *newloc)
+              loc_t *newloc, dict_t *xdata)
 {
         int32_t            ret            = -1, op_errno = ENOMEM;
         quota_local_t     *local          = NULL;
@@ -1634,7 +1649,8 @@ quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
                 goto err;
         }
 
-        stub = fop_rename_stub (frame, quota_rename_helper, oldloc, newloc);
+        stub = fop_rename_stub (frame, quota_rename_helper, oldloc, newloc,
+                                xdata);
         if (stub == NULL) {
                 goto err;
         }
@@ -1682,7 +1698,7 @@ quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 err:
         if (ret == -1) {
                 QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
-                                    NULL, NULL, NULL, NULL);
+                                    NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1693,7 +1709,7 @@ int32_t
 quota_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         int64_t            size   = 0;
         quota_local_t     *local  = NULL;
@@ -1737,7 +1753,7 @@ quota_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent);
+                            preparent, postparent, xdata);
 
         return 0;
 }
@@ -1745,7 +1761,7 @@ out:
 
 int
 quota_symlink_helper (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                      loc_t *loc, dict_t *params)
+                      loc_t *loc, mode_t umask, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
@@ -1762,19 +1778,20 @@ quota_symlink_helper (call_frame_t *frame, xlator_t *this, const char *linkpath,
         }
 
         STACK_WIND (frame, quota_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, params);
+                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                    xdata);
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL);
+                            NULL, NULL, NULL);
         return 0;
 }
 
 
 int
 quota_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-               loc_t *loc, dict_t *params)
+               loc_t *loc, mode_t umask, dict_t *xdata)
 {
         int32_t          ret      = -1;
         int32_t          op_errno = ENOMEM;
@@ -1797,7 +1814,7 @@ quota_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
         local->link_count = 1;
 
         stub = fop_symlink_stub (frame, quota_symlink_helper, linkpath, loc,
-                                 params);
+                                 umask, xdata);
         if (stub == NULL) {
                 goto err;
         }
@@ -1828,7 +1845,7 @@ quota_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
 
 err:
         QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
-                            NULL);
+                            NULL, NULL);
 
         return 0;
 }
@@ -1837,7 +1854,7 @@ err:
 int32_t
 quota_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf)
+                    struct iatt *postbuf, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         int64_t            delta = 0;
@@ -1874,13 +1891,14 @@ quota_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
-                            postbuf);
+                            postbuf, xdata);
         return 0;
 }
 
 
 int32_t
-quota_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+quota_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                dict_t *xdata)
 {
         int32_t          ret   = -1;
         quota_local_t   *local = NULL;
@@ -1899,11 +1917,11 @@ quota_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         }
 
         STACK_WIND (frame, quota_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
 
         return 0;
 err:
-        QUOTA_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -1912,7 +1930,7 @@ err:
 int32_t
 quota_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         int64_t            delta = 0;
@@ -1949,13 +1967,14 @@ quota_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf,
-                            postbuf);
+                            postbuf, xdata);
         return 0;
 }
 
 
 int32_t
-quota_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+quota_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 dict_t *xdata)
 {
         quota_local_t   *local = NULL;
 
@@ -1968,11 +1987,11 @@ quota_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         local->loc.inode = inode_ref (fd->inode);
 
         STACK_WIND (frame, quota_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
 
         return 0;
 err:
-        QUOTA_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -2007,7 +2026,7 @@ quota_send_dir_limit_to_cli (call_frame_t *frame, xlator_t *this,
 
         gf_log (this->name, GF_LOG_INFO, "str = %s", dir_limit);
 
-        QUOTA_STACK_UNWIND (getxattr, frame, 0, 0, dict);
+        QUOTA_STACK_UNWIND (getxattr, frame, 0, 0, dict, NULL);
 
         ret = 0;
 
@@ -2018,7 +2037,7 @@ out:
 
 int32_t
 quota_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
         int32_t ret     = 0;
 
@@ -2031,14 +2050,14 @@ quota_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         }
 
         STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name);
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
         return 0;
 }
 
 
 int32_t
 quota_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name)
+                const char *name, dict_t *xdata)
 {
         int32_t ret     = 0;
 
@@ -2050,14 +2069,14 @@ quota_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
         return 0;
 }
 
 
 int32_t
 quota_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2089,13 +2108,13 @@ quota_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (stat, frame, op_ret, op_errno, buf);
+        QUOTA_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int32_t
-quota_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+quota_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         quota_local_t *local = NULL;
         int32_t        ret   = -1;
@@ -2113,18 +2132,19 @@ quota_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
         }
 
         STACK_WIND (frame, quota_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (stat, frame, -1, ENOMEM, NULL);
+        QUOTA_STACK_UNWIND (stat, frame, -1, ENOMEM, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 quota_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2156,13 +2176,13 @@ quota_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf);
+        QUOTA_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int32_t
-quota_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+quota_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         quota_local_t *local = NULL;
 
@@ -2176,11 +2196,11 @@ quota_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
         local->loc.inode = inode_ref (fd->inode);
 
         STACK_WIND (frame, quota_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fstat, frame, -1, ENOMEM, NULL);
+        QUOTA_STACK_UNWIND (fstat, frame, -1, ENOMEM, NULL, NULL);
         return 0;
 }
 
@@ -2188,7 +2208,7 @@ unwind:
 int32_t
 quota_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, const char *path,
-                    struct iatt *buf)
+                    struct iatt *buf, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2219,13 +2239,14 @@ quota_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, buf);
+        QUOTA_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, buf, xdata);
         return 0;
 }
 
 
 int32_t
-quota_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+quota_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
 {
         quota_local_t *local = NULL;
         int32_t        ret   = -1;
@@ -2244,11 +2265,11 @@ quota_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
         }
 
         STACK_WIND (frame, quota_readlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink, loc, size);
+                    FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (readlink, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2256,7 +2277,8 @@ unwind:
 int32_t
 quota_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref)
+                 int32_t count, struct iatt *buf, struct iobref *iobref,
+                 dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2288,14 +2310,14 @@ quota_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                            buf, iobref);
+                            buf, iobref, xdata);
         return 0;
 }
 
 
 int32_t
 quota_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, uint32_t flags)
+             off_t offset, uint32_t flags, dict_t *xdata)
 {
         quota_local_t *local = NULL;
 
@@ -2309,11 +2331,12 @@ quota_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         local->loc.inode = inode_ref (fd->inode);
 
         STACK_WIND (frame, quota_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags);
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                    xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (readv, frame, -1, ENOMEM, NULL, -1, NULL, NULL);
+        QUOTA_STACK_UNWIND (readv, frame, -1, ENOMEM, NULL, -1, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2321,7 +2344,7 @@ unwind:
 int32_t
 quota_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf)
+                 struct iatt *postbuf, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2352,13 +2375,15 @@ quota_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        QUOTA_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
         return 0;
 }
 
 
 int32_t
-quota_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+quota_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
 {
         quota_local_t *local = NULL;
 
@@ -2372,11 +2397,11 @@ quota_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
         frame->local = local;
 
         STACK_WIND (frame, quota_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, flags);
+                    FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fsync, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
 
 }
@@ -2385,7 +2410,7 @@ unwind:
 int32_t
 quota_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                   struct iatt *statpost)
+                   struct iatt *statpost, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2418,14 +2443,14 @@ quota_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
-                            statpost);
+                            statpost, xdata);
         return 0;
 }
 
 
 int32_t
 quota_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid)
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         quota_local_t *local = NULL;
         int32_t        ret   = -1;
@@ -2444,11 +2469,11 @@ quota_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         STACK_WIND (frame, quota_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid);
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (setattr, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2456,7 +2481,7 @@ unwind:
 int32_t
 quota_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                    struct iatt *statpost)
+                    struct iatt *statpost, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2488,14 +2513,14 @@ quota_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, statpre,
-                            statpost);
+                            statpost, xdata);
         return 0;
 }
 
 
 int32_t
 quota_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         quota_local_t *local = NULL;
 
@@ -2509,11 +2534,11 @@ quota_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         local->loc.inode = inode_ref (fd->inode);
 
         STACK_WIND (frame, quota_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid);
+                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fsetattr, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2522,7 +2547,7 @@ int32_t
 quota_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
                  struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         int32_t            ret    = -1;
         quota_local_t     *local  = NULL;
@@ -2564,14 +2589,14 @@ unlock:
 
 unwind:
         QUOTA_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode,
-                            buf, preparent, postparent);
+                            buf, preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 quota_mknod_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    mode_t mode, dev_t rdev, dict_t *parms)
+                    mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
@@ -2588,20 +2613,21 @@ quota_mknod_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         STACK_WIND (frame, quota_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, parms);
+                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                    xdata);
 
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL);
+                            NULL, NULL, NULL);
         return 0;
 }
 
 
 int
 quota_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dev_t rdev, dict_t *parms)
+             dev_t rdev, mode_t umask, dict_t *xdata)
 {
         int32_t            ret            = -1;
         quota_local_t     *local          = NULL;
@@ -2621,7 +2647,7 @@ quota_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         }
 
         stub = fop_mknod_stub (frame, quota_mknod_helper, loc, mode, rdev,
-                               parms);
+                               umask, xdata);
         if (stub == NULL) {
                 goto err;
         }
@@ -2650,24 +2676,24 @@ quota_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
 
         return 0;
 err:
-        QUOTA_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+        QUOTA_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
 
         return 0;
 }
 
 int
 quota_setxattr_cbk (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int op_ret, int op_errno)
+                    xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 quota_setxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, dict_t *dict, int flags)
+                loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
         int             op_errno = EINVAL;
         int             op_ret   = -1;
 
@@ -2676,31 +2702,30 @@ quota_setxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
-                                   trav, op_errno, err);
+                                   op_errno, err);
 
         STACK_WIND (frame, quota_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    loc, dict, flags, xdata);
         return 0;
 err:
-        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
         return 0;
 }
 
 int
 quota_fsetxattr_cbk (call_frame_t *frame, void *cookie,
-                     xlator_t *this, int op_ret, int op_errno)
+                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
+        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 quota_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int flags)
+                 dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
         int32_t         op_ret   = -1;
         int32_t         op_errno = EINVAL;
 
@@ -2709,30 +2734,98 @@ quota_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         VALIDATE_OR_GOTO (fd, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
-                                   trav, op_errno, err);
+                                   op_errno, err);
 
         STACK_WIND (frame, quota_fsetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    fd, dict, flags, xdata);
+        return 0;
+ err:
+        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
+        return 0;
+}
+
+
+int
+quota_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        QUOTA_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+quota_removexattr (call_frame_t *frame, xlator_t *this,
+                   loc_t *loc, const char *name, dict_t *xdata)
+{
+        int32_t         op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (this, err);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.quota*",
+                                 name, op_errno, err);
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (loc, err);
+
+        STACK_WIND (frame, quota_removexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+err:
+        QUOTA_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
+        return 0;
+}
+
+
+int
+quota_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+quota_fremovexattr (call_frame_t *frame, xlator_t *this,
+                    fd_t *fd, const char *name, dict_t *xdata)
+{
+        int32_t         op_ret   = -1;
+        int32_t         op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.quota*",
+                                 name, op_errno, err);
+
+        STACK_WIND (frame, quota_fremovexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, name, xdata);
         return 0;
  err:
-        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
+        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
         return 0;
 }
 
 
 int32_t
 quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf)
-{
-	inode_t            *root_inode = NULL;
-        quota_priv_t       *priv       = NULL;
-        uint64_t            value      = 0;
-        quota_inode_ctx_t  *ctx        = NULL;
-        limits_t           *limit_node = NULL;
-	int64_t             usage      = -1;
-	int64_t             avail      = -1;
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                  dict_t *xdata)
+{
+	inode_t           *root_inode = NULL;
+        quota_priv_t      *priv       = NULL;
+        uint64_t           value      = 0;
+        quota_inode_ctx_t *ctx        = NULL;
+        limits_t          *limit_node = NULL;
+	int64_t            usage      = -1;
+	int64_t            avail      = -1;
+        int64_t            blocks     = 0;
 
 	root_inode = cookie;
 
@@ -2767,7 +2860,12 @@ quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         list_for_each_entry (limit_node, &priv->limit_head, limit_list) {
 		/* Notice that this only works for volume-level quota. */
                 if (strcmp (limit_node->path, "/") == 0) {
-			buf->f_blocks = limit_node->value / buf->f_bsize;
+                        blocks = limit_node->value / buf->f_bsize;
+                        if (usage > blocks) {
+                                break;
+                        }
+
+			buf->f_blocks = blocks;
 			avail = buf->f_blocks - usage;
 			if (buf->f_bfree > avail) {
 				buf->f_bfree = avail;
@@ -2789,22 +2887,25 @@ unwind:
 	if (root_inode) {
 		inode_unref(root_inode);
 	}
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int32_t
-quota_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+quota_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
 	inode_t *root_inode = NULL;
+        quota_priv_t    *priv = NULL;
+
+        priv = this->private;
 
-	if (loc->inode) {
+	if (priv->consider_statfs && loc->inode) {
 		root_inode = loc->inode->table->root;
 		inode_ref(root_inode);
 		STACK_WIND_COOKIE (frame, quota_statfs_cbk, root_inode,
 				   FIRST_CHILD(this),
-				   FIRST_CHILD(this)->fops->statfs, loc);
+				   FIRST_CHILD(this)->fops->statfs, loc, xdata);
 	}
 	else {
 		/*
@@ -2813,11 +2914,15 @@ quota_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
 		 * which is exactly what would happen with STACK_UNWIND using
 		 * that as a callback.  Therefore, use default_statfs_cbk in
 		 * this case instead.
+                 *
+                 * Also if the option deem-statfs is not set to "on" don't
+                 * bother calculating quota limit on / in statfs_cbk.
 		 */
-		gf_log(this->name,GF_LOG_WARNING,
-		       "missing inode, cannot adjust for quota");
+                if (priv->consider_statfs)
+                        gf_log(this->name,GF_LOG_WARNING,
+                               "missing inode, cannot adjust for quota");
 		STACK_WIND (frame, default_statfs_cbk, FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->statfs, loc);
+			    FIRST_CHILD(this)->fops->statfs, loc, xdata);
 	}
         return 0;
 }
@@ -2825,7 +2930,8 @@ quota_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 int
 quota_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, gf_dirent_t *entries)
+                    int op_ret, int op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
 {
         gf_dirent_t *entry = NULL;
 
@@ -2837,7 +2943,7 @@ quota_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 
         return 0;
 }
@@ -2859,7 +2965,178 @@ quota_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                     fd, size, offset, dict);
         return 0;
 err:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, EINVAL, NULL);
+        STACK_UNWIND_STRICT (readdirp, frame, -1, EINVAL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+quota_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t                  ret            = 0;
+        uint64_t                 ctx_int        = 0;
+        quota_inode_ctx_t       *ctx            = NULL;
+        quota_local_t           *local          = NULL;
+        quota_dentry_t          *dentry         = NULL;
+        int64_t                  delta          = 0;
+
+        local = frame->local;
+
+        if ((op_ret < 0) || (local == NULL)) {
+                goto out;
+        }
+
+        ret = inode_ctx_get (local->loc.inode, this, &ctx_int);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get the context", local->loc.path);
+                goto out;
+        }
+
+        ctx = (quota_inode_ctx_t *)(unsigned long) ctx_int;
+
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in %s (gfid:%s)",
+                        local->loc.path, uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *postbuf;
+        }
+        UNLOCK (&ctx->lock);
+
+        list_for_each_entry (dentry, &ctx->parents, next) {
+                delta = (postbuf->ia_blocks - prebuf->ia_blocks) * 512;
+                quota_update_size (this, local->loc.inode,
+                                   dentry->name, dentry->par, delta);
+        }
+
+out:
+        QUOTA_STACK_UNWIND (fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+
+        return 0;
+}
+
+int32_t
+quota_fallocate_helper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		       int32_t mode, off_t offset, size_t len, dict_t *xdata)
+{
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = EINVAL;
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto unwind;
+        }
+
+        if (local->op_ret == -1) {
+                op_errno = local->op_errno;
+                goto unwind;
+        }
+
+        STACK_WIND (frame, quota_fallocate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		    xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+quota_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		off_t offset, size_t len, dict_t *xdata)
+{
+        int32_t            ret     = -1, op_errno = EINVAL;
+        int32_t            parents = 0;
+        quota_local_t     *local   = NULL;
+        quota_inode_ctx_t *ctx     = NULL;
+        quota_priv_t      *priv    = NULL;
+        call_stub_t       *stub    = NULL;
+        quota_dentry_t    *dentry  = NULL;
+
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO ("quota", this, unwind);
+        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        frame->local = local;
+        local->loc.inode = inode_ref (fd->inode);
+
+        ret = quota_inode_ctx_get (fd->inode, -1, this, NULL, NULL, &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (fd->inode->gfid));
+                goto unwind;
+        }
+
+        stub = fop_fallocate_stub(frame, quota_fallocate_helper, fd, mode, offset, len,
+				  xdata); 
+        if (stub == NULL) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        priv = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
+
+        LOCK (&ctx->lock);
+        {
+                list_for_each_entry (dentry, &ctx->parents, next) {
+                        parents++;
+                }
+        }
+        UNLOCK (&ctx->lock);
+
+	/*
+	 * Note that by using len as the delta we're assuming the range from
+	 * offset to offset+len has not already been allocated. This can result
+	 * in ENOSPC errors attempting to allocate an already allocated range.
+	 */
+        local->delta = len;
+        local->stub = stub;
+        local->link_count = parents;
+
+        list_for_each_entry (dentry, &ctx->parents, next) {
+                ret = quota_check_limit (frame, fd->inode, this, dentry->name,
+                                         dentry->par);
+                if (ret == -1) {
+                        break;
+                }
+        }
+
+        stub = NULL;
+
+        LOCK (&local->lock);
+        {
+                local->link_count = 0;
+                if (local->validate_count == 0) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
+        }
+
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2917,22 +3194,26 @@ quota_forget (xlator_t *this, inode_t *inode)
 
 
 int
-quota_parse_limits (quota_priv_t *priv, xlator_t *this, dict_t *xl_options)
+quota_parse_limits (quota_priv_t *priv, xlator_t *this, dict_t *xl_options,
+                    struct list_head *old_list)
 {
         int32_t       ret       = -1;
         char         *str       = NULL;
         char         *str_val   = NULL;
-        char         *path      = NULL;
+        char         *path      = NULL, *saveptr = NULL;
         uint64_t      value     = 0;
-        limits_t     *quota_lim = NULL;
+        limits_t     *quota_lim = NULL, *old = NULL;
+        char         *last_colon= NULL;
 
         ret = dict_get_str (xl_options, "limit-set", &str);
 
         if (str) {
-                path = strtok (str, ":");
+                path = strtok_r (str, ",", &saveptr);
 
                 while (path) {
-                        str_val = strtok (NULL, ",");
+                        last_colon = strrchr (path, ':');
+                        *last_colon = '\0';
+                        str_val = last_colon + 1;
 
                         ret = gf_string2bytesize (str_val, &value);
                         if (ret != 0)
@@ -2947,20 +3228,40 @@ quota_parse_limits (quota_priv_t *priv, xlator_t *this, dict_t *xl_options)
                         gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64,
                                 quota_lim->path, quota_lim->value);
 
-                        list_add_tail (&quota_lim->limit_list,
-                                       &priv->limit_head);
+                        if (old_list != NULL) {
+                                list_for_each_entry (old, old_list,
+                                                     limit_list) {
+                                        if (strcmp (old->path, quota_lim->path)
+                                            == 0) {
+                                                uuid_copy (quota_lim->gfid,
+                                                           old->gfid);
+                                                break;
+                                        }
+                                }
+                        }
+
+                        LOCK (&priv->lock);
+                        {
+                                list_add_tail (&quota_lim->limit_list,
+                                               &priv->limit_head);
+                        }
+                        UNLOCK (&priv->lock);
 
-                        path = strtok (NULL, ":");
+                        path = strtok_r (NULL, ",", &saveptr);
                 }
         } else {
                 gf_log (this->name, GF_LOG_INFO,
                         "no \"limit-set\" option provided");
         }
 
-        list_for_each_entry (quota_lim, &priv->limit_head, limit_list) {
-                gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64, quota_lim->path,
-                        quota_lim->value);
+        LOCK (&priv->lock);
+        {
+                list_for_each_entry (quota_lim, &priv->limit_head, limit_list) {
+                        gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64,
+                                quota_lim->path, quota_lim->value);
+                }
         }
+        UNLOCK (&priv->lock);
 
         ret = 0;
 err:
@@ -2991,15 +3292,26 @@ init (xlator_t *this)
 
         INIT_LIST_HEAD (&priv->limit_head);
 
+        LOCK_INIT (&priv->lock);
+
         this->private = priv;
 
-        ret = quota_parse_limits (priv, this, this->options);
+        ret = quota_parse_limits (priv, this, this->options, NULL);
 
         if (ret) {
                 goto err;
         }
 
         GF_OPTION_INIT ("timeout", priv->timeout, int64, err);
+        GF_OPTION_INIT ("deem-statfs", priv->consider_statfs, bool, err);
+
+        this->local_pool = mem_pool_new (quota_local_t, 64);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto err;
+        }
 
         ret = 0;
 err:
@@ -3007,23 +3319,81 @@ err:
 }
 
 
+void
+__quota_reconfigure_inode_ctx (xlator_t *this, inode_t *inode, limits_t *limit)
+{
+        int                ret = -1;
+        quota_inode_ctx_t *ctx = NULL;
+
+        GF_VALIDATE_OR_GOTO ("quota", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, limit, out);
+
+        ret = quota_inode_ctx_get (inode, limit->value, this, NULL, NULL, &ctx,
+                                   1);
+        if ((ret == -1) || (ctx == NULL)) {
+                gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
+                        "context in inode(gfid:%s)",
+                        uuid_utoa (inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->limit = limit->value;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        return;
+}
+
+
+void
+__quota_reconfigure (xlator_t *this, inode_table_t *itable, limits_t *limit)
+{
+        inode_t *inode = NULL;
+
+        if ((this == NULL) || (itable == NULL) || (limit == NULL)) {
+                goto out;
+        }
+
+        if (!uuid_is_null (limit->gfid)) {
+                inode = inode_find (itable, limit->gfid);
+        } else {
+                inode = inode_resolve (itable, limit->path);
+        }
+
+        if (inode != NULL) {
+                __quota_reconfigure_inode_ctx (this, inode, limit);
+        }
+
+out:
+        return;
+}
+
+
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
-        int32_t          ret    = -1;
-        quota_priv_t    *priv   = NULL;
-        limits_t        *limit  = NULL;
-        limits_t        *next   = NULL;
+        int32_t           ret   = -1;
+        quota_priv_t     *priv  = NULL;
+        limits_t         *limit = NULL, *next = NULL, *new = NULL;
+        struct list_head  head  = {0, };
+        xlator_t         *top   = NULL;
+        char              found = 0;
 
         priv = this->private;
 
-        list_for_each_entry_safe (limit, next, &priv->limit_head, limit_list) {
-                list_del (&limit->limit_list);
+        INIT_LIST_HEAD (&head);
 
-                GF_FREE (limit);
+        LOCK (&priv->lock);
+        {
+                list_splice_init (&priv->limit_head, &head);
         }
+        UNLOCK (&priv->lock);
 
-        ret = quota_parse_limits (priv, this, options);
+        ret = quota_parse_limits (priv, this, options, &head);
         if (ret == -1) {
                 gf_log ("quota", GF_LOG_WARNING,
                         "quota reconfigure failed, "
@@ -3031,7 +3401,39 @@ reconfigure (xlator_t *this, dict_t *options)
                 goto out;
         }
 
+        LOCK (&priv->lock);
+        {
+                top = ((glusterfs_ctx_t *)this->ctx)->active->top;
+                GF_ASSERT (top);
+
+                list_for_each_entry (limit, &priv->limit_head, limit_list) {
+                        __quota_reconfigure (this, top->itable, limit);
+                }
+
+                list_for_each_entry_safe (limit, next, &head, limit_list) {
+                        found = 0;
+                        list_for_each_entry (new, &priv->limit_head,
+                                             limit_list) {
+                                if (strcmp (new->path, limit->path) == 0) {
+                                        found = 1;
+                                        break;
+                                }
+                        }
+
+                        if (!found) {
+                                limit->value = -1;
+                                __quota_reconfigure (this, top->itable, limit);
+                        }
+
+                        list_del_init (&limit->limit_list);
+                        GF_FREE (limit);
+                }
+        }
+        UNLOCK (&priv->lock);
+
         GF_OPTION_RECONF ("timeout", priv->timeout, options, int64, out);
+        GF_OPTION_RECONF ("deem-statfs", priv->consider_statfs, options, bool,
+                          out);
 
         ret = 0;
 out:
@@ -3047,30 +3449,33 @@ fini (xlator_t *this)
 
 
 struct xlator_fops fops = {
-	.statfs    = quota_statfs,
-        .lookup    = quota_lookup,
-        .writev    = quota_writev,
-        .create    = quota_create,
-        .mkdir     = quota_mkdir,
-        .truncate  = quota_truncate,
-        .ftruncate = quota_ftruncate,
-        .unlink    = quota_unlink,
-        .symlink   = quota_symlink,
-        .link      = quota_link,
-        .rename    = quota_rename,
-        .getxattr  = quota_getxattr,
-        .fgetxattr = quota_fgetxattr,
-        .stat      = quota_stat,
-        .fstat     = quota_fstat,
-        .readlink  = quota_readlink,
-        .readv     = quota_readv,
-        .fsync     = quota_fsync,
-        .setattr   = quota_setattr,
-        .fsetattr  = quota_fsetattr,
-        .mknod     = quota_mknod,
-        .setxattr  = quota_setxattr,
-        .fsetxattr = quota_fsetxattr,
-        .readdirp  = quota_readdirp,
+        .statfs       = quota_statfs,
+        .lookup       = quota_lookup,
+        .writev       = quota_writev,
+        .create       = quota_create,
+        .mkdir        = quota_mkdir,
+        .truncate     = quota_truncate,
+        .ftruncate    = quota_ftruncate,
+        .unlink       = quota_unlink,
+        .symlink      = quota_symlink,
+        .link         = quota_link,
+        .rename       = quota_rename,
+        .getxattr     = quota_getxattr,
+        .fgetxattr    = quota_fgetxattr,
+        .stat         = quota_stat,
+        .fstat        = quota_fstat,
+        .readlink     = quota_readlink,
+        .readv        = quota_readv,
+        .fsync        = quota_fsync,
+        .setattr      = quota_setattr,
+        .fsetattr     = quota_fsetattr,
+        .mknod        = quota_mknod,
+        .setxattr     = quota_setxattr,
+        .fsetxattr    = quota_fsetxattr,
+        .removexattr  = quota_removexattr,
+        .fremovexattr = quota_fremovexattr,
+        .readdirp     = quota_readdirp,
+	.fallocate    = quota_fallocate,
 };
 
 struct xlator_cbks cbks = {
@@ -3081,9 +3486,18 @@ struct volume_options options[] = {
         {.key = {"limit-set"}},
         {.key = {"timeout"},
          .type = GF_OPTION_TYPE_SIZET,
+         .min = 0,
+         .max = 60,
          .default_value = "0",
          .description = "quota caches the directory sizes on client. Timeout "
                         "indicates the timeout for the cache to be revalidated."
         },
+        {.key = {"deem-statfs"},
+         .type = GF_OPTION_TYPE_BOOL,
+         .default_value = "off",
+         .description = "If set to on, it takes quota limits into"
+                        "consideration while estimating fs size. (df command)"
+                        " (Default is off)."
+        },
         {.key = {NULL}}
 };
diff --git a/xlators/features/quota/src/quota.h b/xlators/features/quota/src/quota.h
index 9b94d5920..84ecbb308 100644
--- a/xlators/features/quota/src/quota.h
+++ b/xlators/features/quota/src/quota.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -36,16 +26,6 @@
 #define VAL_LENGTH              8
 #define READDIR_BUF             4096
 
-#define QUOTA_STACK_DESTROY(_frame, _this)              \
-        do {                                            \
-                quota_local_t *_local = NULL;           \
-                _local = _frame->local;                 \
-                _frame->local = NULL;                   \
-                STACK_DESTROY (_frame->root);           \
-                quota_local_cleanup (_this, _local);    \
-                GF_FREE (_local);                       \
-        } while (0)
-
 #define QUOTA_SAFE_INCREMENT(lock, var)         \
         do {                                    \
                 LOCK (lock);                    \
@@ -60,12 +40,6 @@
                 UNLOCK (lock);                  \
         } while (0)
 
-#define QUOTA_LOCAL_ALLOC_OR_GOTO(local, type, label)           \
-        do {                                                    \
-                QUOTA_ALLOC_OR_GOTO (local, type, label);       \
-                LOCK_INIT (&local->lock);                       \
-        } while (0)
-
 #define QUOTA_ALLOC_OR_GOTO(var, type, label)           \
         do {                                            \
                 var = GF_CALLOC (sizeof (type), 1,      \
@@ -89,7 +63,6 @@
                 }                                                       \
                 STACK_UNWIND_STRICT (fop, frame, params);               \
                 quota_local_cleanup (_this, _local);                    \
-                GF_FREE (_local);                                       \
         } while (0)
 
 #define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)     \
@@ -160,16 +133,19 @@ struct quota_local {
 typedef struct quota_local quota_local_t;
 
 struct quota_priv {
-        int64_t  timeout;
-        struct list_head limit_head;
+        int64_t           timeout;
+        gf_boolean_t      consider_statfs;
+        struct list_head  limit_head;
+        gf_lock_t         lock;
 };
 typedef struct quota_priv quota_priv_t;
 
 struct limits {
         struct list_head  limit_list;
         char             *path;
-        int64_t          value;
+        int64_t           value;
+        uuid_t            gfid;
 };
-typedef struct limits limits_t;
+typedef struct limits     limits_t;
 
 uint64_t cn = 1;
diff --git a/xlators/features/read-only/src/Makefile.am b/xlators/features/read-only/src/Makefile.am
index 31ae4f340..4c1462137 100644
--- a/xlators/features/read-only/src/Makefile.am
+++ b/xlators/features/read-only/src/Makefile.am
@@ -4,18 +4,19 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 noinst_HEADERS = read-only-common.h
 
-read_only_la_LDFLAGS = -module -avoidversion
+read_only_la_LDFLAGS = -module -avoid-version
 
 read_only_la_SOURCES = read-only.c read-only-common.c
 read_only_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
-worm_la_LDFLAGS = -module -avoidversion
+worm_la_LDFLAGS = -module -avoid-version
 
 worm_la_SOURCES = read-only-common.c worm.c
 worm_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/read-only/src/read-only-common.c b/xlators/features/read-only/src/read-only-common.c
index efd175081..56a7a7176 100644
--- a/xlators/features/read-only/src/read-only-common.c
+++ b/xlators/features/read-only/src/read-only-common.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -27,220 +17,223 @@
 
 int32_t
 ro_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            gf_xattrop_flags_t flags, dict_t *dict)
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (xattrop, frame, -1, EROFS, NULL);
+        STACK_UNWIND_STRICT (xattrop, frame, -1, EROFS, NULL, xdata);
         return 0;
 }
 
 int32_t
 ro_fxattrop (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fxattrop, frame, -1, EROFS, NULL);
+        STACK_UNWIND_STRICT (fxattrop, frame, -1, EROFS, NULL, xdata);
         return 0;
 }
 
 int32_t
 ro_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
             loc_t *loc, const char *basename, entrylk_cmd cmd,
-            entrylk_type type)
+            entrylk_type type, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (entrylk, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (entrylk, frame, -1, EROFS, xdata);
         return 0;
 }
 
 int32_t
 ro_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type type)
+             fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fentrylk, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (fentrylk, frame, -1, EROFS, xdata);
         return 0;
 }
 
 int32_t
 ro_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, int32_t cmd, struct gf_flock *lock)
+            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (inodelk, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (inodelk, frame, -1, EROFS, xdata);
         return 0;
 }
 
 int32_t
 ro_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock)
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (finodelk, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (finodelk, frame, -1, EROFS, xdata);
         return 0;
 }
 
 int32_t
 ro_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
-       struct gf_flock *flock)
+       struct gf_flock *flock, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (lk, frame, -1, EROFS, NULL);
+        STACK_UNWIND_STRICT (lk, frame, -1, EROFS, NULL, xdata);
         return 0;
 }
 
 int32_t
 ro_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid)
+            struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setattr, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (setattr, frame, -1, EROFS, NULL, NULL, xdata);
 	return 0;
 }
 
 int32_t
 ro_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid)
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsetattr, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (fsetattr, frame, -1, EROFS, NULL, NULL, xdata);
         return 0;
 }
 
 
 int32_t
-ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (truncate, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, -1, EROFS, NULL, NULL, xdata);
 	return 0;
 }
 
 int32_t
-ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, EROFS, NULL, NULL, xdata);
 	return 0;
 }
 
 int
 ro_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, dict_t *params)
+          dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (mknod, frame, -1, EROFS, NULL, NULL, NULL, NULL);
+        STACK_UNWIND_STRICT (mknod, frame, -1, EROFS, NULL, NULL, NULL, NULL, xdata);
         return 0;
 }
 
 
 int
 ro_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dict_t *params)
+          mode_t umask, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (mkdir, frame, -1, EROFS, NULL, NULL, NULL, NULL);
+        STACK_UNWIND_STRICT (mkdir, frame, -1, EROFS, NULL, NULL, NULL, NULL, xdata);
         return 0;
 }
 
 int32_t
-ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (unlink, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (unlink, frame, -1, EROFS, NULL, NULL, xdata);
         return 0;
 }
 
 
 int
-ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (rmdir, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (rmdir, frame, -1, EROFS, NULL, NULL, xdata);
         return 0;
 }
 
 
 int
 ro_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-            loc_t *loc, dict_t *params)
+            loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (symlink, frame, -1, EROFS, NULL, NULL, NULL, NULL);
+        STACK_UNWIND_STRICT (symlink, frame, -1, EROFS, NULL, NULL, NULL,
+                             NULL, xdata);
         return 0;
 }
 
 
 
 int32_t
-ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (rename, frame, -1, EROFS, NULL, NULL, NULL, NULL,
-                             NULL);
+                             NULL, xdata);
         return 0;
 }
 
 
 int32_t
-ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (link, frame, -1, EROFS, NULL, NULL, NULL, NULL);
+        STACK_UNWIND_STRICT (link, frame, -1, EROFS, NULL, NULL, NULL, NULL, xdata);
         return 0;
 }
 
 int32_t
 ro_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, fd_t *fd, dict_t *params)
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (create, frame, -1, EROFS, NULL, NULL, NULL,
-                             NULL, NULL);
+                             NULL, NULL, xdata);
         return 0;
 }
 
 
 static int32_t
 ro_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, fd_t *fd)
+             int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 	return 0;
 }
 
 int32_t
 ro_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+         fd_t *fd, dict_t *xdata)
 {
         if (((flags & O_ACCMODE) == O_WRONLY) ||
               ((flags & O_ACCMODE) == O_RDWR)) {
-                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL, xdata);
                 return 0;
 	}
 
 	STACK_WIND (frame, ro_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
 	return 0;
 }
 
 int32_t
 ro_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags)
+              int32_t flags, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsetxattr, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (fsetxattr, frame, -1, EROFS, xdata);
 	return 0;
 }
 
 int32_t
-ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsyncdir, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (fsyncdir, frame, -1, EROFS, xdata);
 	return 0;
 }
 
 int32_t
 ro_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t flags, struct iobref *iobref)
+           int32_t count, off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (writev, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (writev, frame, -1, EROFS, NULL, NULL, xdata);
         return 0;
 }
 
 
 int32_t
 ro_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags)
+             int32_t flags, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setxattr, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (setxattr, frame, -1, EROFS, xdata);
         return 0;
 }
 
 int32_t
 ro_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name)
+                const char *name, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (removexattr, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (removexattr, frame, -1, EROFS, xdata);
         return 0;
 }
diff --git a/xlators/features/read-only/src/read-only-common.h b/xlators/features/read-only/src/read-only-common.h
index 8c89e88ce..5d4c7e260 100644
--- a/xlators/features/read-only/src/read-only-common.h
+++ b/xlators/features/read-only/src/read-only-common.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -27,97 +17,99 @@
 
 int32_t
 ro_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            gf_xattrop_flags_t flags, dict_t *dict);
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
 
 int32_t
 ro_fxattrop (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict);
+             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
 
 int32_t
 ro_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
             loc_t *loc, const char *basename, entrylk_cmd cmd,
-            entrylk_type type);
+            entrylk_type type, dict_t *xdata);
 
 int32_t
 ro_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
              fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type
-             type);
+             type, dict_t *xdata);
 
 int32_t
 ro_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, int32_t cmd, struct gf_flock *lock);
+            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
 
 int32_t
 ro_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock);
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
 
 int32_t
 ro_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
-       struct gf_flock *flock);
+       struct gf_flock *flock, dict_t *xdata);
 
 int32_t
 ro_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid);
+            struct iatt *stbuf, int32_t valid, dict_t *xdata);
 
 int32_t
 ro_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid);
+             struct iatt *stbuf, int32_t valid, dict_t *xdata);
 
 
 int32_t
-ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset);
+ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata);
 
 int32_t
-ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset);
+ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata);
 
 int
 ro_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, dict_t *params);
+          dev_t rdev, mode_t umask, dict_t *xdata);
 
 int
 ro_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dict_t *params);
+          mode_t umask, dict_t *xdata);
 
 int32_t
-ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc);
+ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
 
 int
-ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags);
+ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
 
 
 int
 ro_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-            loc_t *loc, dict_t *params);
+            loc_t *loc, mode_t umask, dict_t *xdata);
 
 int32_t
-ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc);
+ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int32_t
-ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc);
+ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int32_t
 ro_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, fd_t *fd, dict_t *params);
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
 ro_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags);
+         fd_t *fd, dict_t *xdata);
 
 int32_t
 ro_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags);
+              int32_t flags, dict_t *xdata);
 
 int32_t
-ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags);
+ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata);
 
 int32_t
 ro_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t flags, struct iobref *iobref);
+           int32_t count, off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata);
 
 int32_t
 ro_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags);
+             int32_t flags, dict_t *xdata);
 
 int32_t
 ro_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name);
+                const char *name, dict_t *xdata);
diff --git a/xlators/features/read-only/src/read-only.c b/xlators/features/read-only/src/read-only.c
index b11e84f24..e49e54a1b 100644
--- a/xlators/features/read-only/src/read-only.c
+++ b/xlators/features/read-only/src/read-only.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/features/read-only/src/worm.c b/xlators/features/read-only/src/worm.c
index 790bc3234..16c3eb3da 100644
--- a/xlators/features/read-only/src/worm.c
+++ b/xlators/features/read-only/src/worm.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -28,25 +18,25 @@
 
 static int32_t
 worm_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, fd_t *fd)
+               int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
 int32_t
 worm_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+           fd_t *fd, dict_t *xdata)
 {
         if ((((flags & O_ACCMODE) == O_WRONLY) ||
               ((flags & O_ACCMODE) == O_RDWR)) &&
               !(flags & O_APPEND)) {
-                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL, NULL);
                 return 0;
         }
 
         STACK_WIND (frame, worm_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
         return 0;
 }
 
@@ -75,7 +65,6 @@ fini (xlator_t *this)
 }
 
 struct xlator_fops fops = {
-        
         .open        = worm_open,
 
         .unlink      = ro_unlink,
@@ -92,8 +81,7 @@ struct xlator_fops fops = {
         .lk          = ro_lk,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
         { .key = {NULL} },
diff --git a/xlators/features/trash/src/Makefile.am b/xlators/features/trash/src/Makefile.am
index 4671d06d3..5251eb082 100644
--- a/xlators/features/trash/src/Makefile.am
+++ b/xlators/features/trash/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = trash.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
 
-trash_la_LDFLAGS = -module -avoidversion 
+trash_la_LDFLAGS = -module -avoid-version 
 
 trash_la_SOURCES = trash.c
 trash_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = trash.h trash-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/trash/src/trash-mem-types.h b/xlators/features/trash/src/trash-mem-types.h
index 6608abf6a..0e6ef572f 100644
--- a/xlators/features/trash/src/trash-mem-types.h
+++ b/xlators/features/trash/src/trash-mem-types.h
@@ -1,30 +1,19 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __TRASH_MEM_TYPES_H__
 #define __TRASH_MEM_TYPES_H__
 
 #include "mem-types.h"
 
 enum gf_trash_mem_types_ {
-        gf_trash_mt_trash_local_t = gf_common_mt_end + 1,
-        gf_trash_mt_trash_private_t,
+        gf_trash_mt_trash_private_t = gf_common_mt_end + 1,
         gf_trash_mt_char,
         gf_trash_mt_trash_elim_pattern_t,
         gf_trash_mt_end
diff --git a/xlators/features/trash/src/trash.c b/xlators/features/trash/src/trash.c
index 38920db6c..addeb66a0 100644
--- a/xlators/features/trash/src/trash.c
+++ b/xlators/features/trash/src/trash.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -63,7 +53,7 @@ trash_local_wipe (trash_local_t *local)
         if (local->newfd)
                 fd_unref (local->newfd);
 
-        GF_FREE (local);
+        mem_put (local);
 out:
         return;
 }
@@ -170,8 +160,7 @@ trash_unlink_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         GF_FREE (cookie);
-        if (tmp_str)
-                GF_FREE (tmp_str);
+        GF_FREE (tmp_str);
 
         return 0;
 }
@@ -442,8 +431,7 @@ trash_rename_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         GF_FREE (cookie); /* strdup (dir_name) was sent here :) */
-        if (tmp_str)
-                GF_FREE (tmp_str);
+        GF_FREE (tmp_str);
 
         return 0;
 }
@@ -504,9 +492,7 @@ trash_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
         trash_elim_pattern_t  *trav  = NULL;
         trash_private_t *priv = NULL;
         trash_local_t   *local = NULL;
-        struct tm       *tm = NULL;
-        char             timestr[256] = {0,};
-        time_t           utime = 0;
+        char             timestr[64] = {0,};
         int32_t          match = 0;
 
         priv = this->private;
@@ -533,8 +519,7 @@ trash_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
                 return 0;
         }
 
-        local = GF_CALLOC (1, sizeof (trash_local_t), 
-                           gf_trash_mt_trash_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
                 TRASH_STACK_UNWIND (rename, frame, -1, ENOMEM,
@@ -554,9 +539,8 @@ trash_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
         {
                 /* append timestamp to file name */
                 /* TODO: can we make it optional? */
-                utime = time (NULL);
-                tm    = localtime (&utime);
-                strftime (timestr, 256, ".%Y-%m-%d-%H%M%S", tm);
+                gf_time_ftm (timestr, sizeof timestr, time (NULL),
+                             gf_timefmt_F_HMS);
                 strcat (local->newpath, timestr);
         }
 
@@ -575,9 +559,7 @@ trash_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         trash_elim_pattern_t  *trav  = NULL;
         trash_private_t *priv = NULL;
         trash_local_t   *local = NULL;
-        struct tm       *tm = NULL;
-        char             timestr[256] = {0,};
-        time_t           utime = 0;
+        char             timestr[64] = {0,};
         int32_t          match = 0;
 
         priv = this->private;
@@ -610,8 +592,7 @@ trash_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 return 0;
         }
 
-        local = GF_CALLOC (1, sizeof (trash_local_t),
-                           gf_trash_mt_trash_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 gf_log (this->name, GF_LOG_DEBUG, "out of memory");
                 TRASH_STACK_UNWIND (unlink, frame, -1, ENOMEM, NULL, NULL);
@@ -627,9 +608,8 @@ trash_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         {
                 /* append timestamp to file name */
                 /* TODO: can we make it optional? */
-                utime = time (NULL);
-                tm    = localtime (&utime);
-                strftime (timestr, 256, ".%Y-%m-%d-%H%M%S", tm);
+                gf_time_fmt (timestr, sizeof timestr, time (NULL),
+                             gf_timefmt_F_HMS);
                 strcat (local->newpath, timestr);
         }
 
@@ -932,8 +912,7 @@ trash_truncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         GF_FREE (cookie); /* strdup (dir_name) was sent here :) */
-        if (tmp_str)
-                GF_FREE (tmp_str);
+        GF_FREE (tmp_str);
 
         return 0;
 }
@@ -945,10 +924,8 @@ trash_truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         trash_private_t     *priv  = NULL;
         trash_local_t       *local = NULL;
-        struct tm           *tm = NULL;
-        char                 timestr[256] = {0,};
+        char                 timestr[64] = {0,};
         char                 loc_newname[PATH_MAX] = {0,};
-        time_t               utime = 0;
         int32_t              flags = 0;
 
         priv = this->private;
@@ -980,9 +957,8 @@ trash_truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         strcat (local->newpath, local->loc.path);
 
         {
-                utime = time (NULL);
-                tm    = localtime (&utime);
-                strftime (timestr, 256, ".%Y-%m-%d-%H%M%S", tm);
+                gf_time_fmt (timestr, sizeof timestr, time (NULL),
+                             gf_timefmt_F_HMS);
                 strcat (local->newpath, timestr);
         }
         strcpy (loc_newname,local->loc.name);
@@ -1044,8 +1020,7 @@ trash_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         LOCK_INIT (&frame->lock);
 
-        local = GF_CALLOC (1, sizeof (trash_local_t), 
-                           gf_trash_mt_trash_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 gf_log (this->name, GF_LOG_DEBUG, "out of memory");
                 TRASH_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL);
@@ -1299,8 +1274,7 @@ trash_ftruncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         GF_FREE (cookie); /* strdup (dir_name) was sent here :) */
-        if (tmp_str)
-                GF_FREE (tmp_str);
+        GF_FREE (tmp_str);
 
         return 0;
 }
@@ -1349,11 +1323,9 @@ trash_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         trash_private_t       *priv = NULL;
         trash_local_t         *local = NULL;
         dentry_t              *dir_entry = NULL;
-        struct tm             *tm = NULL;
         char                  *pathbuf = NULL;
         inode_t               *newinode = NULL;
-        time_t                 utime = 0;
-        char                   timestr[256];
+        char                   timestr[64];
         int32_t                retval = 0;
         int32_t                match = 0;
 
@@ -1385,18 +1357,14 @@ trash_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
                 return 0;
         }
 
-        local = GF_CALLOC (1, sizeof (trash_local_t), 
-                           gf_trash_mt_trash_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 gf_log (this->name, GF_LOG_DEBUG, "out of memory");
                 TRASH_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
-        utime = time (NULL);
-        tm    = localtime (&utime);
-        strftime (timestr, 256, ".%Y-%m-%d-%H%M%S", tm);
-
+        gf_time_fmt (timestr, sizeof timestr, time (NULL), gf_timefmt_F_HMS);
         strcpy (local->newpath, priv->trash_dir);
         strcat (local->newpath, pathbuf);
         strcat (local->newpath, timestr);
@@ -1522,6 +1490,14 @@ init (xlator_t *this)
                         _priv->max_trash_file_size);
         }
 
+        this->local_pool = mem_pool_new (trash_local_t, 64);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                return -1;
+        }
+
+
         this->private = (void *)_priv;
         return 0;
 }
@@ -1532,8 +1508,7 @@ fini (xlator_t *this)
         trash_private_t *priv = NULL;
 
         priv = this->private;
-        if (priv)
-                GF_FREE (priv);
+        GF_FREE (priv);
 
         return;
 }
diff --git a/xlators/features/trash/src/trash.h b/xlators/features/trash/src/trash.h
index d385ee346..9a7c03361 100644
--- a/xlators/features/trash/src/trash.h
+++ b/xlators/features/trash/src/trash.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __TRASH_H__
 #define __TRASH_H__
 
diff --git a/xlators/lib/src/libxlator.c b/xlators/lib/src/libxlator.c
index df302d11d..9e5357255 100644
--- a/xlators/lib/src/libxlator.c
+++ b/xlators/lib/src/libxlator.c
@@ -1,7 +1,44 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include "mem-types.h"
 #include "libxlator.h"
 
 
+int marker_xtime_default_gauge[] = {
+        [MCNT_FOUND]    =  1,
+        [MCNT_NOTFOUND] = -1,
+        [MCNT_ENODATA]  = -1,
+        [MCNT_ENOTCONN] = -1,
+        [MCNT_ENOENT]   = -1,
+        [MCNT_EOTHER]   = -1,
+};
+
+int marker_uuid_default_gauge[] = {
+        [MCNT_FOUND]    =  1,
+        [MCNT_NOTFOUND] =  0,
+        [MCNT_ENODATA]  =  0,
+        [MCNT_ENOTCONN] =  0,
+        [MCNT_ENOENT]   =  0,
+        [MCNT_EOTHER]   =  0,
+};
+
+static int marker_idx_errno_map[] = {
+        [MCNT_FOUND]    = EINVAL,
+        [MCNT_NOTFOUND] = EINVAL,
+        [MCNT_ENOENT]   = ENOENT,
+        [MCNT_ENOTCONN] = ENOTCONN,
+        [MCNT_ENODATA]  = ENODATA,
+        [MCNT_EOTHER]   = EINVAL,
+        [MCNT_MAX]      = 0,
+};
+
 /*Copy the contents of oldtimebuf to newtimbuf*/
 static void
 update_timebuf (uint32_t *oldtimbuf, uint32_t *newtimebuf)
@@ -35,139 +72,148 @@ match_uuid_local (const char *name, char *uuid)
         return 0;
 }
 
+static void
+marker_local_incr_errcount (xl_marker_local_t *local, int op_errno)
+{
+        marker_result_idx_t i = -1;
 
+        if (!local)
+                return;
+
+        switch (op_errno) {
+                case ENODATA:
+                        i = MCNT_ENODATA;
+                        break;
+                case ENOENT:
+                        i = MCNT_ENOENT;
+                        break;
+                case ENOTCONN:
+                        i = MCNT_ENOTCONN;
+                        break;
+                default:
+                        i = MCNT_EOTHER;
+                        break;
+        }
 
+        local->count[i]++;
+}
+
+static int
+evaluate_marker_results (int *gauge, int *count)
+{
+        int i = 0;
+        int op_errno = 0;
+        gf_boolean_t sane = _gf_true;
+
+        /* check if the policy of the gauge is violated;
+         * if yes, try to get the best errno, ie. look
+         * for the first position where there is a more
+         * specific kind of vioilation than the generic EINVAL
+         */
+        for (i = 0; i < MCNT_MAX; i++) {
+                if (sane) {
+                        if ((gauge[i] > 0 && count[i] < gauge[i]) ||
+                            (gauge[i] < 0 && count[i] >= -gauge[i])) {
+                                sane = _gf_false;
+                                /* generic action: adopt corresponding errno */
+                                op_errno = marker_idx_errno_map[i];
+                        }
+                } else {
+                        /* already insane; trying to get a more informative
+                         * errno by checking subsequent counters
+                         */
+                        if (count[i] > 0)
+                                op_errno = marker_idx_errno_map[i];
+                }
+                if (op_errno && op_errno != EINVAL)
+                        break;
+        }
+
+        return op_errno;
+}
 
 /* Aggregate all the <volid>.xtime attrs of the cluster and send the max*/
 int32_t
 cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict)
+                         int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 
 {
 
-        int32_t            callcnt         = 0;
-        int                ret             = -1;
-        uint32_t          *net_timebuf     = NULL;
-        uint32_t           host_timebuf[2] = {0,};
-        char              *marker_xattr    = NULL;
-        struct marker_str *local           = NULL;
-        char              *vol_uuid        = NULL;
+        int32_t           callcnt         = 0;
+        int               ret             = -1;
+        uint32_t          *net_timebuf    = NULL;
+        uint32_t          host_timebuf[2] = {0,};
+        char              *marker_xattr   = NULL;
+        xl_marker_local_t *local          = NULL;
+        char              *vol_uuid       = NULL;
+        char              need_unwind     = 0;
 
         if (!this || !frame || !frame->local || !cookie) {
-                gf_log (this->name, GF_LOG_DEBUG, "possible NULL deref");
+                gf_log ("", GF_LOG_DEBUG, "possible NULL deref");
+                need_unwind = 1;
                 goto out;
         }
 
         local = frame->local;
         if (!local || !local->vol_uuid) {
                 gf_log (this->name, GF_LOG_DEBUG, "possible NULL deref");
+                need_unwind = 1;
                 goto out;
         }
 
-        if (local->esomerr) {
-                 LOCK (&frame->lock);
-                {
-                        callcnt = --local->call_count;
-                }
-                goto done;
-        }
-
-        vol_uuid = local->vol_uuid;
-
-        if (op_ret && op_errno == ENODATA) {
-                LOCK (&frame->lock);
-                {
-                        callcnt = --local->call_count;
-                        local->enodata_count++;
-                }
-                goto done;
-        }
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
 
-        if (op_ret && op_errno == ENOENT) {
-                LOCK (&frame->lock);
-                {
-                        callcnt = --local->call_count;
-                        local->enoent_count++;
-                }
-                goto done;
-        }
+                vol_uuid = local->vol_uuid;
 
-        if (op_ret && op_errno == ENOTCONN) {
-                LOCK (&frame->lock);
-                {
-                        callcnt = --local->call_count;
-                        local->enotconn_count++;
+                if (op_ret) {
+                        marker_local_incr_errcount (local, op_errno);
+                        goto unlock;
                 }
-                goto done;
-        }
 
-        if (op_ret) {
-                LOCK (&frame->lock);
-                {
-                        callcnt = --local->call_count;
-                        local->esomerr = op_errno;
-                }
-                goto done;
-        }
-
-
-
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (!gf_asprintf (& marker_xattr, "%s.%s.%s",
+                if (!gf_asprintf (&marker_xattr, "%s.%s.%s",
                                 MARKER_XATTR_PREFIX, vol_uuid, XTIME)) {
                         op_errno = ENOMEM;
-                        goto done;
+                        goto unlock;
                 }
 
 
                 if (dict_get_ptr (dict, marker_xattr, (void **)&net_timebuf)) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "Unable to get <uuid>.xtime attr");
-                        local->noxtime_count++;
-                        goto done;
+                        local->count[MCNT_NOTFOUND]++;
+                        goto unlock;
                 }
 
-                if (local->has_xtime) {
-
+                if (local->count[MCNT_FOUND]) {
                         get_hosttime (net_timebuf, host_timebuf);
                         if ( (host_timebuf[0]>local->host_timebuf[0]) ||
                                 (host_timebuf[0] == local->host_timebuf[0] &&
                                  host_timebuf[1] >= local->host_timebuf[1])) {
-
                                 update_timebuf (net_timebuf, local->net_timebuf);
                                 update_timebuf (host_timebuf, local->host_timebuf);
-
                         }
 
-                }
-                else {
+                } else {
                         get_hosttime (net_timebuf, local->host_timebuf);
                         update_timebuf (net_timebuf, local->net_timebuf);
-                        local->has_xtime = _gf_true;
+                        local->count[MCNT_FOUND]++;
                 }
 
-
-
         }
-done:
+unlock:
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
-
                 op_ret = 0;
                 op_errno = 0;
-                if (local->has_xtime) {
-                        if (!dict) {
+                need_unwind = 1;
+
+                if (local->count[MCNT_FOUND]) {
+                        if (!dict)
                                 dict = dict_new();
-                                if (ret) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        goto out;
-                                }
-                        }
+
                         ret = dict_set_static_bin (dict, marker_xattr,
                                            (void *)local->net_timebuf, 8);
                         if (ret) {
@@ -175,178 +221,133 @@ done:
                                 op_errno = ENOMEM;
                                 goto out;
                         }
-                        goto out;
                 }
 
-                if (local->noxtime_count)
-                        goto out;
-
-                if (local->enodata_count) {
-                        op_ret = -1;
-                        op_errno = ENODATA;
-                        goto out;
-                }
-                if (local->enotconn_count) {
-                        op_ret = -1;
-                        op_errno = ENOTCONN;
-                        goto out;
-                }
-                if (local->enoent_count) {
+                op_errno = evaluate_marker_results (local->gauge, local->count);
+                if (op_errno)
                         op_ret = -1;
-                        op_errno = ENOENT;
-                        goto out;
-                }
-                else {
-                        op_errno = local->esomerr;
-                        goto out;
-                }
-out:
-                if (local->xl_specf_unwind) {
-                        frame->local = local->xl_local;
-                        local->xl_specf_unwind (frame, op_ret,
-                                                 op_errno, dict);
-                        return 0;
-                }
-                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        }
 
+out:
+        if (need_unwind && local && local->xl_specf_unwind) {
+                frame->local = local->xl_local;
+                local->xl_specf_unwind (frame, op_ret,
+                                        op_errno, dict, xdata);
+        } else if (need_unwind) {
+                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno,
+                                     dict, xdata);
         }
 
+        GF_FREE (marker_xattr);
         return 0;
 
 }
 
 int32_t
 cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict)
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 {
-        int32_t              callcnt = 0;
-        data_t              *data = NULL;
-        struct volume_mark  *volmark = NULL;
-        struct marker_str   *marker = NULL;
-        char                *vol_uuid;
+        int32_t             callcnt     = 0;
+        struct volume_mark  *volmark    = NULL;
+        xl_marker_local_t   *local      = NULL;
+        int32_t             ret         = -1;
+        char                need_unwind = 0;
+        char                *vol_uuid   = NULL;
 
         if (!this || !frame || !cookie) {
-                gf_log (this->name, GF_LOG_DEBUG, "possible NULL deref");
+                gf_log ("", GF_LOG_DEBUG, "possible NULL deref");
+                need_unwind = 1;
                 goto out;
         }
 
-        marker = frame->local;
+        local = frame->local;
 
-        if (!marker) {
+        if (!local) {
                 gf_log (this->name, GF_LOG_DEBUG, "possible NULL deref");
+                need_unwind = 1;
                 goto out;
         }
 
-        vol_uuid = marker->vol_uuid;
-
-        if (op_ret && (ENOENT == op_errno)) {
-                LOCK (&frame->lock);
-                {
-                        callcnt = --marker->call_count;
-                        marker->enoent_count++;
-                }
-                goto done;
-        }
-
-        if (op_ret && (ENOTCONN == op_errno)) {
-                LOCK (&frame->lock);
-                {
-                        callcnt = --marker->call_count;
-                        marker->enotconn_count++;
-                }
-                goto done;
-        }
-
-        if (!(data = dict_get (dict, GF_XATTR_MARKER_KEY))) {
-                LOCK (&frame->lock);
-                {
-                        callcnt = --marker->call_count;
-                }
-                goto done;
-        }
-
-        volmark = (struct volume_mark *)data->data;
-
         LOCK (&frame->lock);
         {
-                callcnt = --marker->call_count;
+                callcnt = --local->call_count;
+                vol_uuid = local->vol_uuid;
 
-                if (marker_has_volinfo (marker)) {
+                if (op_ret) {
+                        marker_local_incr_errcount (local, op_errno);
+                        goto unlock;
+                }
+
+                ret = dict_get_bin (dict, GF_XATTR_MARKER_KEY,
+                                    (void *)&volmark);
+                if (ret)
+                        goto unlock;
 
-                        if ((marker->volmark->major != volmark->major) ||
-                            (marker->volmark->minor != volmark->minor)) {
+                if (local->count[MCNT_FOUND]) {
+                        if ((local->volmark->major != volmark->major) ||
+                            (local->volmark->minor != volmark->minor)) {
                                 op_ret = -1;
                                 op_errno = EINVAL;
-                                goto done;
-                        }
-                        else if (volmark->retval) {
-                                data_unref ((data_t *) marker->volmark);
-                                marker->volmark = volmark;
-                                callcnt = 0;
+                                goto unlock;
                         }
-                        else if ( (volmark->sec > marker->volmark->sec) ||
-                                   ((volmark->sec == marker->volmark->sec)
-                                      && (volmark->usec >= marker->volmark->usec))) {
 
-                                GF_FREE (marker->volmark);
-                                marker->volmark = memdup (volmark, sizeof (struct volume_mark));
-                                VALIDATE_OR_GOTO (marker->volmark, done);
+                        if (local->retval)
+                                goto unlock;
+                        else if (volmark->retval) {
+                                GF_FREE (local->volmark);
+                                local->volmark =
+                                        memdup (volmark, sizeof (*volmark));
+                                local->retval = volmark->retval;
+                        } else if ((volmark->sec > local->volmark->sec) ||
+                                   ((volmark->sec == local->volmark->sec) &&
+                                    (volmark->usec >= local->volmark->usec))) {
+                                GF_FREE (local->volmark);
+                                local->volmark =
+                                      memdup (volmark, sizeof (*volmark));
                         }
 
                 } else {
-                        marker->volmark = memdup (volmark, sizeof (struct volume_mark));
-                        VALIDATE_OR_GOTO (marker->volmark, done);
-
+                        local->volmark = memdup (volmark, sizeof (*volmark));
+                        VALIDATE_OR_GOTO (local->volmark, unlock);
                         uuid_unparse (volmark->uuid, vol_uuid);
                         if (volmark->retval)
-                                callcnt = 0;
+                                local->retval = volmark->retval;
+                        local->count[MCNT_FOUND]++;
                 }
         }
-done:
+unlock:
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
                 op_ret = 0;
                 op_errno = 0;
-                if (marker_has_volinfo (marker)) {
-                        if (!dict) {
+                need_unwind = 1;
+
+                if (local->count[MCNT_FOUND]) {
+                        if (!dict)
                                 dict = dict_new();
-                                if (!dict) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        goto out;
-                                }
-                        }
+
                         if (dict_set_bin (dict, GF_XATTR_MARKER_KEY,
-                                          marker->volmark,
+                                          local->volmark,
                                           sizeof (struct volume_mark))) {
                                 op_ret = -1;
                                 op_errno = ENOMEM;
                         }
-                        goto out;
-                }
-                if (marker->enotconn_count) {
-                        op_ret = -1;
-                        op_errno = ENOTCONN;
-                        goto out;
-                }
-                if (marker->enoent_count) {
-                        op_ret = -1;
-                        op_errno = ENOENT;
                 }
-                else {
+                op_errno = evaluate_marker_results (local->gauge, local->count);
+                if (op_errno)
                         op_ret = -1;
-                        op_errno = EINVAL;
-                }
+        }
 
  out:
-                if (marker->xl_specf_unwind) {
-                        frame->local = marker->xl_local;
-                        marker->xl_specf_unwind (frame, op_ret,
-                                                 op_errno, dict);
-                        return 0;
-                }
-                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        if (need_unwind && local && local->xl_specf_unwind) {
+                frame->local = local->xl_local;
+                local->xl_specf_unwind (frame, op_ret,
+                                        op_errno, dict, xdata);
+                return 0;
+        } else if (need_unwind){
+                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno,
+                                     dict, xdata);
         }
         return 0;
 }
@@ -357,10 +358,10 @@ cluster_getmarkerattr (call_frame_t *frame,xlator_t *this, loc_t *loc,
                        const char *name, void *xl_local,
                        xlator_specf_unwind_t xl_specf_getxattr_unwind,
                        xlator_t **sub_volumes, int count, int type,
-                       char *vol_uuid)
+                       int *gauge, char *vol_uuid)
 {
-        int               i;
-        struct marker_str *local;
+        int                i     = 0;
+        xl_marker_local_t  *local = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -373,33 +374,36 @@ cluster_getmarkerattr (call_frame_t *frame,xlator_t *this, loc_t *loc,
         local = GF_CALLOC (sizeof (struct marker_str), 1,
                             gf_common_mt_libxl_marker_local);
 
-        local->xl_local = xl_local;
-        frame->local = local;
+        if (!local)
+                goto err;
 
+        local->xl_local = xl_local;
         local->call_count = count;
-
         local->xl_specf_unwind = xl_specf_getxattr_unwind;
-
         local->vol_uuid = vol_uuid;
+        memcpy (local->gauge, gauge, sizeof (local->gauge));
+
+        frame->local = local;
 
         for (i=0; i < count; i++) {
                 if (MARKER_UUID_TYPE == type)
                         STACK_WIND (frame, cluster_markeruuid_cbk,
                                     *(sub_volumes + i),
                                     (*(sub_volumes + i))->fops->getxattr,
-                                    loc, name);
+                                    loc, name, NULL);
                 else if (MARKER_XTIME_TYPE == type)
                         STACK_WIND (frame, cluster_markerxtime_cbk,
                                     *(sub_volumes + i),
                                     (*(sub_volumes + i))->fops->getxattr,
-                                    loc, name);
+                                    loc, name, NULL);
                 else {
                         gf_log (this->name, GF_LOG_WARNING,
-                                 "Unrecognized type of marker attr received");
+                                 "Unrecognized type (%d) of marker attr "
+                                 "received", type);
                         STACK_WIND (frame, default_getxattr_cbk,
                                     *(sub_volumes + i),
                                     (*(sub_volumes + i))->fops->getxattr,
-                                    loc, name);
+                                    loc, name, NULL);
                         break;
                 }
         }
@@ -409,3 +413,58 @@ err:
         return -1;
 
 }
+
+int
+gf_get_min_stime (xlator_t *this, dict_t *dst, char *key, data_t *value)
+{
+        int ret = -1;
+        uint32_t *net_timebuf = NULL;
+        uint32_t *value_timebuf = NULL;
+        uint32_t host_timebuf[2] = {0,};
+        uint32_t host_value_timebuf[2] = {0,};
+
+        /* stime should be minimum of all the other nodes */
+        ret = dict_get_bin (dst, key, (void **)&net_timebuf);
+        if (ret < 0) {
+                net_timebuf = GF_CALLOC (1, sizeof (int64_t),
+                                           gf_common_mt_char);
+                if (!net_timebuf)
+                        goto out;
+
+                ret = dict_set_bin (dst, key, net_timebuf, sizeof (int64_t));
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "key=%s: dict set failed", key);
+                        goto error;
+                }
+        }
+
+        value_timebuf = data_to_bin (value);
+        if (!value_timebuf) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "key=%s: getting value of stime failed", key);
+                ret = -1;
+                goto out;
+        }
+
+        get_hosttime (value_timebuf, host_value_timebuf);
+        get_hosttime (net_timebuf, host_timebuf);
+
+        /* can't use 'min()' macro here as we need to compare two fields
+           in the array, selectively */
+        if ((host_value_timebuf[0] > host_timebuf[0]) ||
+            ((host_value_timebuf[0] == host_timebuf[0]) &&
+             (host_value_timebuf[1] > host_timebuf[1]))) {
+                update_timebuf (value_timebuf, net_timebuf);
+        }
+
+        ret = 0;
+out:
+        return ret;
+error:
+        /* To be used only when net_timebuf is not set in the dict */
+        if (net_timebuf)
+                GF_FREE (net_timebuf);
+
+        return ret;
+}
diff --git a/xlators/lib/src/libxlator.h b/xlators/lib/src/libxlator.h
index af7eb434a..1d5e1657f 100644
--- a/xlators/lib/src/libxlator.h
+++ b/xlators/lib/src/libxlator.h
@@ -1,3 +1,12 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _LIBXLATOR_H
 #define _LIBXLATOR_H
 
@@ -26,7 +35,8 @@
 
 
 typedef int32_t (*xlator_specf_unwind_t) (call_frame_t *frame,
-                                         int op_ret, int op_errno, dict_t *dict);
+                                          int op_ret, int op_errno,
+                                          dict_t *dict, dict_t *xdata);
 
 
 struct volume_mark {
@@ -38,6 +48,69 @@ struct volume_mark {
         uint32_t usec;
 }__attribute__ ((__packed__));
 
+
+/*
+ * The enumerated type here
+ * is used to index two kind
+ * of integer arrays:
+ * - gauges
+ * - counters
+
+ * A counter is used internally,
+ * in getxattr callbacks, to count
+ * the results, categorized as
+ * the enum names suggest. So values
+ * in the counter are always non-negative.
+
+ * Gauges are part of the API.
+ * The caller passes one to the
+ * top-level aggregator function,
+ * cluster_getmarkerattr(). The gauge
+ * defines an evaluation policy for the
+ * counter. That is, at the
+ * end of the aggregation process
+ * the gauge is matched against the
+ * counter, and the policy
+ * represented by the gauge decides
+ * whether to return with success or failure,
+ * and in latter case, what particular failure
+ * case (errno).
+
+ * The rules are the following: for some index i,
+ * - if gauge[i] == 0, no requirement is set
+ *   against counter[i];
+ * - if gauge[i] > 0, counter[i] >= gauge[i]
+ *   is required;
+ * - if gauge[i] < 0, counter[i] < |gauge[i]|
+ *   is required.
+
+ * If the requirement is not met, then i is mapped
+ * to the respective errno (MCNT_ENOENT -> ENOENT),
+ * or in lack of that, EINVAL.
+
+ * Cf. evaluate_marker_results() and marker_idx_errno_map[]
+ * in libxlator.c
+
+ * We provide two default gauges, one inteded for xtime
+ * aggregation, other for volume mark aggregation. The
+ * policies they represent agree with the hard-coded
+ * one prior to gauges. Cf. marker_xtime_default_gauge
+ * and marker_uuid_default_gauge in libxlator.c
+ */
+
+typedef enum {
+        MCNT_FOUND,
+        MCNT_NOTFOUND,
+        MCNT_ENODATA,
+        MCNT_ENOTCONN,
+        MCNT_ENOENT,
+        MCNT_EOTHER,
+        MCNT_MAX
+} marker_result_idx_t;
+
+extern int marker_xtime_default_gauge[];
+extern int marker_uuid_default_gauge[];
+
 struct marker_str {
         struct volume_mark    *volmark;
         data_t                *data;
@@ -45,47 +118,36 @@ struct marker_str {
         uint32_t               host_timebuf[2];
         uint32_t               net_timebuf[2];
         int32_t                call_count;
-        unsigned               has_xtime:1;
-        int32_t                enoent_count;
-        int32_t                enotconn_count;
-        int32_t                enodata_count;
-        int32_t                noxtime_count;
-
-        int                    esomerr;
+        int                    gauge[MCNT_MAX];
+        int                    count[MCNT_MAX];
 
         xlator_specf_unwind_t  xl_specf_unwind;
         void                  *xl_local;
         char                  *vol_uuid;
+        uint8_t                retval;
 };
 
-static inline gf_boolean_t
-marker_has_volinfo (struct marker_str *marker)
-{
-       if (marker->volmark)
-                return _gf_true;
-       else
-                return _gf_false;
-}
+typedef struct marker_str xl_marker_local_t;
 
 int32_t
 cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict);
+                         int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
 
 int32_t
 cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict);
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
 
 int32_t
 cluster_getmarkerattr (call_frame_t *frame,xlator_t *this, loc_t *loc,
                        const char *name, void *xl_local,
                        xlator_specf_unwind_t xl_specf_getxattr_unwind,
                        xlator_t **sub_volumes, int count, int type,
-                       char *vol_uuid);
+                       int *gauge, char *vol_uuid);
 
 int
 match_uuid_local (const char *name, char *uuid);
 
-
-
+int
+gf_get_min_stime (xlator_t *this, dict_t *dst, char *key, data_t *value);
 
 #endif /* !_LIBXLATOR_H */
diff --git a/xlators/meta/src/Makefile.am b/xlators/meta/src/Makefile.am
index 385ff553f..f8fa7d4cb 100644
--- a/xlators/meta/src/Makefile.am
+++ b/xlators/meta/src/Makefile.am
@@ -4,7 +4,8 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/
 meta_so_SOURCES = meta.c tree.c misc.c view.c
 noinst_HEADERS = meta.h tree.h misc.h view.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall
 
 CLEANFILES = 
diff --git a/xlators/meta/src/meta-mem-types.h b/xlators/meta/src/meta-mem-types.h
index 9585b7838..62028b246 100644
--- a/xlators/meta/src/meta-mem-types.h
+++ b/xlators/meta/src/meta-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __META_MEM_TYPES_H__
 #define __META_MEM_TYPES_H__
 
diff --git a/xlators/meta/src/meta.c b/xlators/meta/src/meta.c
index 412b4a2b5..e69719f3c 100644
--- a/xlators/meta/src/meta.c
+++ b/xlators/meta/src/meta.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
diff --git a/xlators/meta/src/meta.h b/xlators/meta/src/meta.h
index 5636487c9..73e0e50db 100644
--- a/xlators/meta/src/meta.h
+++ b/xlators/meta/src/meta.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __META_H__
 #define __META_H__
 
diff --git a/xlators/meta/src/misc.c b/xlators/meta/src/misc.c
index bea07e70c..1a8dfa806 100644
--- a/xlators/meta/src/misc.c
+++ b/xlators/meta/src/misc.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #include <unistd.h>
 #include <sys/uio.h>
 
diff --git a/xlators/meta/src/misc.h b/xlators/meta/src/misc.h
index f934a6d8d..30dd10e34 100644
--- a/xlators/meta/src/misc.h
+++ b/xlators/meta/src/misc.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __MISC_H__
 #define __MISC_H__
 
diff --git a/xlators/meta/src/tree.c b/xlators/meta/src/tree.c
index cad5cbd71..dacbd665a 100644
--- a/xlators/meta/src/tree.c
+++ b/xlators/meta/src/tree.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -123,7 +113,7 @@ lookup_meta_entry (meta_dirent_t *root, const char *path,
 	    gf_asprintf (remain, "/%s/%s", *remain, piece);
 	  else
 	    gf_asprintf (remain, "/%s", piece);
-	  if (tmp) GF_FREE (tmp);
+	  GF_FREE (tmp);
 	  piece = strtok (NULL, "/");
 	}
       }
diff --git a/xlators/meta/src/tree.h b/xlators/meta/src/tree.h
index 8157148db..985df3bd7 100644
--- a/xlators/meta/src/tree.h
+++ b/xlators/meta/src/tree.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __TREE_H__
 #define __TREE_H__
 
diff --git a/xlators/meta/src/view.c b/xlators/meta/src/view.c
index ba3c30e0b..b4e2d64a2 100644
--- a/xlators/meta/src/view.c
+++ b/xlators/meta/src/view.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/meta/src/view.h b/xlators/meta/src/view.h
index 440c0d34d..2eff6126e 100644
--- a/xlators/meta/src/view.h
+++ b/xlators/meta/src/view.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __VIEW_H__
 #define __VIEW_H__
 
diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am
index 817f02735..933c44019 100644
--- a/xlators/mgmt/glusterd/src/Makefile.am
+++ b/xlators/mgmt/glusterd/src/Makefile.am
@@ -1,27 +1,50 @@
 xlator_LTLIBRARIES = glusterd.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mgmt
-glusterd_la_CPPFLAGS = "-DFILTERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/filter\""
-glusterd_la_LDFLAGS = -module -avoidversion $(LIBXML2_LIBS)
-glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c glusterd-op-sm.c \
-	glusterd-utils.c glusterd-rpc-ops.c glusterd-store.c glusterd-handshake.c \
-	glusterd-pmap.c glusterd-volgen.c glusterd-rebalance.c glusterd-quota.c \
+glusterd_la_CPPFLAGS = $(AM_CPPFLAGS) "-DFILTERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/filter\""
+glusterd_la_LDFLAGS = -module -avoid-version
+if ENABLE_BD_XLATOR
+glusterd_la_LDFLAGS += -llvm2app
+endif
+glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c \
+	glusterd-op-sm.c glusterd-utils.c glusterd-rpc-ops.c \
+	glusterd-store.c glusterd-handshake.c glusterd-pmap.c \
+	glusterd-volgen.c glusterd-rebalance.c glusterd-quota.c \
 	glusterd-geo-rep.c glusterd-replace-brick.c glusterd-log-ops.c \
-	glusterd-volume-ops.c glusterd-brick-ops.c glusterd-mountbroker.c
+	glusterd-volume-ops.c glusterd-brick-ops.c glusterd-mountbroker.c \
+	glusterd-syncop.c glusterd-hooks.c glusterd-volume-set.c \
+        glusterd-locks.c glusterd-snapshot.c glusterd-mgmt-handler.c \
+        glusterd-mgmt.c
 
 glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 		     $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-		     $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+		     $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+		     $(XML_LIBS) -lcrypto
 
-noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h glusterd-sm.h \
-	glusterd-store.h glusterd-mem-types.h glusterd-pmap.h glusterd-volgen.h \
-        glusterd-mountbroker.h
+noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h \
+	glusterd-sm.h glusterd-store.h glusterd-mem-types.h \
+	glusterd-pmap.h glusterd-volgen.h glusterd-mountbroker.h \
+	glusterd-syncop.h glusterd-hooks.h glusterd-locks.h \
+	glusterd-mgmt.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)\
-	-I$(rpclibdir) -L$(xlatordir)/ -I$(CONTRIBDIR)/rbtree -I$(top_srcdir)/rpc/xdr/src\
-	-I$(top_srcdir)/rpc/rpc-lib/src -I$(CONTRIBDIR)/uuid -I$(top_srcdir)/contrib/md5  -DSBIN_DIR=\"$(sbindir)\"\
-        -DDATADIR=\"$(localstatedir)\" -DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
-        -DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(LIBXML2_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(rpclibdir) -I$(CONTRIBDIR)/rbtree \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(CONTRIBDIR)/uuid \
+	-DSBIN_DIR=\"$(sbindir)\" -DDATADIR=\"$(localstatedir)\" \
+	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
+        -DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(XML_CPPFLAGS)
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+AM_LDFLAGS = -L$(xlatordir)
 
 CLEANFILES =
+
+install-data-hook:
+
+if GF_INSTALL_VAR_LIB_GLUSTERD
+	$(mkdir_p) $(localstatedir)/lib/
+	(stat $(sysconfdir)/glusterd && \
+	    mv $(sysconfdir)/glusterd $(localstatedir)/lib/) || true;
+	(ln -sf $(localstatedir)/lib/glusterd $(sysconfdir)/glusterd) || true;
+endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index c170972ce..596503c21 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -31,6 +21,7 @@
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "run.h"
+#include <sys/signal.h>
 
 /* misc */
 
@@ -132,6 +123,15 @@ gd_addbr_validate_stripe_count (glusterd_volinfo_t *volinfo, int stripe_count,
                                 volinfo->volname);
                         ret = 0;
                         goto out;
+                } else {
+                        snprintf (err_str, err_len, "Incorrect number of "
+                                  "bricks (%d) supplied for changing volume's "
+                                  "stripe count to %d, need at least %d bricks",
+                                  (total_bricks - volinfo->brick_count),
+                                  stripe_count,
+                                  (volinfo->replica_count * stripe_count));
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
                 }
                 break;
         case GF_CLUSTER_TYPE_STRIPE:
@@ -213,6 +213,16 @@ gd_addbr_validate_replica_count (glusterd_volinfo_t *volinfo, int replica_count,
                                 volinfo->volname);
                         ret = 0;
                         goto out;
+                } else {
+                        snprintf (err_str, err_len, "Incorrect number of "
+                                  "bricks (%d) supplied for changing volume's "
+                                  "replica count to %d, need at least %d "
+                                  "bricks",
+                                  (total_bricks - volinfo->brick_count),
+                                  replica_count, (volinfo->dist_leaf_count *
+                                                  replica_count));
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
                 }
                 break;
         case GF_CLUSTER_TYPE_REPLICATE:
@@ -255,8 +265,10 @@ out:
 }
 
 static int
-gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo, int32_t replica_count,
-                                int32_t brick_count, char *err_str)
+gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo,
+                                int32_t replica_count,
+                                int32_t brick_count, char *err_str,
+                                size_t err_len)
 {
         int ret = -1;
         int replica_nodes = 0;
@@ -264,7 +276,7 @@ gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo, int32_t replica_cou
         switch (volinfo->type) {
         case GF_CLUSTER_TYPE_NONE:
         case GF_CLUSTER_TYPE_STRIPE:
-                snprintf (err_str, 2048,
+                snprintf (err_str, err_len,
                           "replica count (%d) option given for non replicate "
                           "volume %s", replica_count, volinfo->volname);
                 gf_log (THIS->name, GF_LOG_WARNING, "%s", err_str);
@@ -274,7 +286,7 @@ gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo, int32_t replica_cou
         case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
                 /* in remove brick, you can only reduce the replica count */
                 if (replica_count > volinfo->replica_count) {
-                        snprintf (err_str, 2048,
+                        snprintf (err_str, err_len,
                                   "given replica count (%d) option is more "
                                   "than volume %s's replica count (%d)",
                                   replica_count, volinfo->volname,
@@ -283,15 +295,30 @@ gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo, int32_t replica_cou
                         goto out;
                 }
                 if (replica_count == volinfo->replica_count) {
+                        /* This means the 'replica N' option on CLI was
+                           redundant. Check if the total number of bricks given
+                           for removal is same as 'dist_leaf_count' */
+                        if (brick_count % volinfo->dist_leaf_count) {
+                                snprintf (err_str, err_len,
+                                          "number of bricks provided (%d) is "
+                                          "not valid. need at least %d "
+                                          "(or %dxN)", brick_count,
+                                          volinfo->dist_leaf_count,
+                                          volinfo->dist_leaf_count);
+                                gf_log (THIS->name, GF_LOG_WARNING, "%s",
+                                        err_str);
+                                goto out;
+                        }
                         ret = 1;
                         goto out;
                 }
 
-                replica_nodes = ((volinfo->brick_count / volinfo->replica_count) *
+                replica_nodes = ((volinfo->brick_count /
+                                  volinfo->replica_count) *
                                  (volinfo->replica_count - replica_count));
 
                 if (brick_count % replica_nodes) {
-                        snprintf (err_str, 2048,
+                        snprintf (err_str, err_len,
                                   "need %d(xN) bricks for reducing replica "
                                   "count of the volume from %d to %d",
                                   replica_nodes, volinfo->replica_count,
@@ -306,11 +333,9 @@ out:
         return ret;
 }
 
-
-
 /* Handler functions */
 int
-glusterd_handle_add_brick (rpcsvc_request_t *req)
+__glusterd_handle_add_brick (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -318,7 +343,6 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
         char                            *bricks = NULL;
         char                            *volname = NULL;
         int                             brick_count = 0;
-        char                            *brick_list = NULL;
         void                            *cli_rsp = NULL;
         char                            err_str[2048] = {0,};
         gf_cli_rsp                      rsp = {0,};
@@ -334,15 +358,16 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 snprintf (err_str, sizeof (err_str), "Garbage args received");
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received add brick req");
+        gf_log (this->name, GF_LOG_INFO, "Received add brick req");
 
         if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
@@ -352,59 +377,62 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
                         snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the buffer");
+                                  "the command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
 
-        gf_cmd_log ("Volume add-brick", "on volname: %s attempted",
-                    volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
                 snprintf (err_str, sizeof (err_str), "Unable to get volume "
                           "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         if (!(ret = glusterd_check_volume_exists (volname))) {
                 ret = -1;
-                snprintf(err_str, 2048, "Volume %s does not exist", volname);
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                snprintf (err_str, sizeof (err_str), "Volume %s does not exist",
+                          volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
                 snprintf (err_str, sizeof (err_str), "Unable to get volume "
                           "brick count");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "replica-count", &replica_count);
         if (!ret) {
-                gf_log (THIS->name, GF_LOG_INFO, "replica-count is %d",
+                gf_log (this->name, GF_LOG_INFO, "replica-count is %d",
                         replica_count);
         }
 
         ret = dict_get_int32 (dict, "stripe-count", &stripe_count);
         if (!ret) {
-                gf_log (THIS->name, GF_LOG_INFO, "stripe-count is %d",
+                gf_log (this->name, GF_LOG_INFO, "stripe-count is %d",
                         stripe_count);
         }
 
+        if (!dict_get (dict, "force")) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get flag");
+                goto out;
+        }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
                 snprintf (err_str, sizeof (err_str), "Unable to get volinfo "
                           "for volume name %s", volname);
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
 
         }
@@ -420,10 +448,10 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
                         goto brick_val;
 
                 if ((brick_count % volinfo->dist_leaf_count) != 0) {
-                        snprintf(err_str, 2048, "Incorrect number of bricks"
-                                 " supplied %d with count %d",
+                        snprintf (err_str, sizeof (err_str), "Incorrect number "
+                                  "of bricks supplied %d with count %d",
                                  brick_count, volinfo->dist_leaf_count);
-                        gf_log("glusterd", GF_LOG_ERROR, "%s", err_str);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         ret = -1;
                         goto out;
                 }
@@ -439,7 +467,7 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
                                                       err_str,
                                                       sizeof (err_str));
                 if (ret == -1) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         goto out;
                 }
 
@@ -449,7 +477,7 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
 
                 ret = dict_set_int32 (dict, "stripe-count", stripe_count);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to set the stripe-count in dict");
                         goto out;
                 }
@@ -461,7 +489,7 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
                                                &type, err_str,
                                                sizeof (err_str));
         if (ret == -1) {
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
@@ -471,7 +499,7 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
 
         ret = dict_set_int32 (dict, "replica-count", replica_count);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "failed to set the replica-count in dict");
                 goto out;
         }
@@ -481,51 +509,111 @@ brick_val:
         if (ret) {
                 snprintf (err_str, sizeof (err_str), "Unable to get volume "
                           "bricks");
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_cmd_log ("Volume add-brick", "volname: %s type %d count:%d bricks:%s"
-                    ,volname, volinfo->type, brick_count, brick_list);
-
         if (type != volinfo->type) {
                 ret = dict_set_int32 (dict, "type", type);
                 if (ret)
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to set the new type in dict");
         }
 
-        ret = glusterd_op_begin (req, GD_OP_ADD_BRICK, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
 
 out:
-        gf_cmd_log ("Volume add-brick","on volname: %s %s", volname,
-                   (ret != 0)? "FAILED" : "SUCCESS");
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
                 rsp.op_ret = -1;
                 rsp.op_errno = 0;
                 if (err_str[0] == '\0')
                         snprintf (err_str, sizeof (err_str), "Operation failed");
                 rsp.op_errstr = err_str;
                 cli_rsp = &rsp;
-                glusterd_submit_reply(req, cli_rsp, NULL, 0, NULL,
-                                      (xdrproc_t)xdr_gf_cli_rsp);
+                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
                 ret = 0; //sent error to cli, prevent second reply
         }
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        free (cli_req.dict.dict_val); //its malloced by xdr
+
+        return ret;
+}
+
+int
+glusterd_handle_add_brick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_add_brick);
+}
+
+static int
+subvol_matcher_init (int **subvols, int count)
+{
+        int ret = -1;
+
+        *subvols = GF_CALLOC (count, sizeof(int), gf_gld_mt_int);
+        if (*subvols)
+                ret = 0;
+
+        return ret;
+}
+
+static void
+subvol_matcher_update (int *subvols, glusterd_volinfo_t *volinfo,
+                       glusterd_brickinfo_t *brickinfo)
+{
+        glusterd_brickinfo_t *tmp        = NULL;
+        int32_t               sub_volume = 0;
+        int                   pos        = 0;
+
+        list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
+
+                if (strcmp (tmp->hostname, brickinfo->hostname) ||
+                    strcmp (tmp->path, brickinfo->path)) {
+                        pos++;
+                        continue;
+                }
+                gf_log (THIS->name, GF_LOG_DEBUG, LOGSTR_FOUND_BRICK,
+                                        brickinfo->hostname, brickinfo->path,
+                                        volinfo->volname);
+                sub_volume = (pos / volinfo->dist_leaf_count);
+                subvols[sub_volume]++;
+                break;
+        }
+
+}
+
+static int
+subvol_matcher_verify (int *subvols, glusterd_volinfo_t *volinfo, char *err_str,
+                       size_t err_len, char *vol_type)
+{
+        int i = 0;
+        int ret = 0;
 
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
+       do {
+
+                if (subvols[i] % volinfo->dist_leaf_count == 0) {
+                        continue;
+                } else {
+                        ret = -1;
+                        snprintf (err_str, err_len,
+                                "Bricks not from same subvol for %s", vol_type);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        break;
+                }
+        } while (++i < volinfo->subvol_count);
 
         return ret;
 }
 
+static void
+subvol_matcher_destroy (int *subvols)
+{
+        GF_FREE (subvols);
+}
 
 int
-glusterd_handle_remove_brick (rpcsvc_request_t *req)
+__glusterd_handle_remove_brick (rpcsvc_request_t *req)
 {
         int32_t                   ret              = -1;
         gf_cli_req                cli_req          = {{0,}};
@@ -537,10 +625,7 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
         int                       i                = 1;
         glusterd_volinfo_t       *volinfo          = NULL;
         glusterd_brickinfo_t     *brickinfo        = NULL;
-        int32_t                   pos              = 0;
-        int32_t                   sub_volume       = 0;
-        int32_t                   sub_volume_start = 0;
-        int32_t                   sub_volume_end   = 0;
+        int                      *subvols          = NULL;
         glusterd_brickinfo_t     *tmp              = NULL;
         char                      err_str[2048]    = {0};
         gf_cli_rsp                rsp              = {0,};
@@ -552,18 +637,23 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
         int                       found            = 0;
         int                       diff_count       = 0;
         char                     *volname          = 0;
+        xlator_t                 *this             = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
+                snprintf (err_str, sizeof (err_str), "Received garbage args");
                 goto out;
         }
 
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received rem brick req");
+        gf_log (this->name, GF_LOG_INFO, "Received rem brick req");
 
         if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
@@ -573,52 +663,61 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to get volname");
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_cmd_log ("Volume remove-brick","on volname: %s attempted", volname);
         ret = dict_get_int32 (dict, "count", &count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
+                snprintf (err_str, sizeof (err_str), "Unable to get brick "
+                          "count");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                 snprintf (err_str, 2048, "Volume %s does not exist",
-                          volname);
-                 gf_log ("", GF_LOG_ERROR, "%s", err_str);
+                 snprintf (err_str, sizeof (err_str),"Volume %s does not exist",
+                           volname);
+                 gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                  goto out;
         }
 
         ret = dict_get_int32 (dict, "replica-count", &replica_count);
         if (!ret) {
-                gf_log (THIS->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_INFO,
                         "request to change replica-count to %d", replica_count);
                 ret = gd_rmbr_validate_replica_count (volinfo, replica_count,
-                                                      count, err_str);
+                                                      count, err_str,
+                                                      sizeof (err_str));
                 if (ret < 0) {
-                        /* logging and error msg are done in above function itself */
+                        /* logging and error msg are done in above function
+                           itself */
                         goto out;
                 }
                 dict_del (dict, "replica-count");
                 if (ret) {
                         replica_count = 0;
                 } else {
-                        ret = dict_set_int32 (dict, "replica-count", replica_count);
+                        ret = dict_set_int32 (dict, "replica-count",
+                                              replica_count);
                         if (ret) {
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "failed to set the replica_count in dict");
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to set the replica_count "
+                                        "in dict");
                                 goto out;
                         }
                 }
@@ -635,41 +734,54 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
                 strcpy (vol_type, "distribute");
         }
 
-	/* Do not allow remove-brick if the volume is plain stripe */
+	/* Do not allow remove-brick if the volume is a stripe volume*/
 	if ((volinfo->type == GF_CLUSTER_TYPE_STRIPE) &&
             (volinfo->brick_count == volinfo->stripe_count)) {
-                snprintf (err_str, 2048, "Removing brick from a plain stripe is not allowed");
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                snprintf (err_str, sizeof (err_str),
+                          "Removing brick from a stripe volume is not allowed");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 ret = -1;
                 goto out;
 	}
 
-	/* Do not allow remove-brick if the bricks given is less than the replica count
-	   or stripe count */
-        if (!replica_count && (volinfo->type != GF_CLUSTER_TYPE_NONE) &&
-            !(volinfo->brick_count <= volinfo->dist_leaf_count)) {
-                if (volinfo->dist_leaf_count &&
-                    (count % volinfo->dist_leaf_count)) {
-                        snprintf (err_str, 2048, "Remove brick incorrect"
-                                  " brick count of %d for %s %d",
-                                  count, vol_type, volinfo->dist_leaf_count);
-                        gf_log ("", GF_LOG_ERROR, "%s", err_str);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
 	if (!replica_count &&
             (volinfo->type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) &&
             (volinfo->brick_count == volinfo->dist_leaf_count)) {
-                snprintf (err_str, 2048, "Removing bricks from stripe-replicate"
+                snprintf (err_str, sizeof(err_str),
+                          "Removing bricks from stripe-replicate"
                           " configuration is not allowed without reducing "
                           "replica or stripe count explicitly.");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+	if (!replica_count &&
+            (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
+            (volinfo->brick_count == volinfo->dist_leaf_count)) {
+                snprintf (err_str, sizeof (err_str),
+                          "Removing bricks from replicate configuration "
+                          "is not allowed without reducing replica count "
+                          "explicitly.");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 ret = -1;
                 goto out;
         }
 
+	/* Do not allow remove-brick if the bricks given is less than
+           the replica count or stripe count */
+        if (!replica_count && (volinfo->type != GF_CLUSTER_TYPE_NONE)) {
+                if (volinfo->dist_leaf_count &&
+                    (count % volinfo->dist_leaf_count)) {
+                        snprintf (err_str, sizeof (err_str), "Remove brick "
+                                  "incorrect brick count of %d for %s %d",
+                                  count, vol_type, volinfo->dist_leaf_count);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
         brick_list = GF_MALLOC (120000 * sizeof(*brick_list),gf_common_mt_char);
 
         if (!brick_list) {
@@ -678,23 +790,32 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
         }
 
         strcpy (brick_list, " ");
+
+        if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
+            (volinfo->subvol_count > 1)) {
+                ret = subvol_matcher_init (&subvols, volinfo->subvol_count);
+                if (ret)
+                        goto out;
+        }
+
         while ( i <= count) {
-                snprintf (key, 256, "brick%d", i);
+                snprintf (key, sizeof (key), "brick%d", i);
                 ret = dict_get_str (dict, key, &brick);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to get %s", key);
+                        snprintf (err_str, sizeof (err_str), "Unable to get %s",
+                                  key);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         goto out;
                 }
-                gf_log ("", GF_LOG_DEBUG, "Remove brick count %d brick: %s",
-                        i, brick);
+                gf_log (this->name, GF_LOG_DEBUG, "Remove brick count %d brick:"
+                        " %s", i, brick);
 
                 ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo,
-                                                             &brickinfo,
-                                                             GF_PATH_COMPLETE);
+                                                             &brickinfo);
                 if (ret) {
-                        snprintf(err_str, 2048,"Incorrect brick %s for volume"
-                                " %s", brick, volname);
-                        gf_log ("", GF_LOG_ERROR, "%s", err_str);
+                        snprintf (err_str, sizeof (err_str), "Incorrect brick "
+                                  "%s for volume %s", brick, volname);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         goto out;
                 }
                 strcat(brick_list, brick);
@@ -715,7 +836,7 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
                         found = 0;
                         list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
                                 tmp_brick_idx++;
-                                gf_log (THIS->name, GF_LOG_TRACE,
+                                gf_log (this->name, GF_LOG_TRACE,
                                         "validate brick %s:%s (%d %d %d)",
                                         tmp->hostname, tmp->path, tmp_brick_idx,
                                         brick_index, volinfo->replica_count);
@@ -733,8 +854,9 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
                         if (found)
                                 continue;
 
-                        snprintf(err_str, 2048,"Bricks are from same subvol");
-                        gf_log (THIS->name, GF_LOG_INFO,
+                        snprintf (err_str, sizeof (err_str), "Bricks are from "
+                                  "same subvol");
+                        gf_log (this->name, GF_LOG_INFO,
                                 "failed to validate brick %s:%s (%d %d %d)",
                                 tmp->hostname, tmp->path, tmp_brick_idx,
                                 brick_index, volinfo->replica_count);
@@ -743,72 +865,146 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
                         goto out;
                 }
 
-                pos = 0;
-                list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
-
-                        if (strcmp (tmp->hostname,brickinfo->hostname) ||
-                            strcmp (tmp->path, brickinfo->path)) {
-                                pos++;
-                                continue;
-                        }
+                /* Find which subvolume the brick belongs to */
+                subvol_matcher_update (subvols, volinfo, brickinfo);
+        }
 
-                        gf_log ("", GF_LOG_INFO, "Found brick");
-                        if (!sub_volume && (volinfo->dist_leaf_count > 1)) {
-                                sub_volume = (pos / volinfo->dist_leaf_count) + 1;
-                                sub_volume_start = (volinfo->dist_leaf_count *
-                                                    (sub_volume - 1));
-                                sub_volume_end = (volinfo->dist_leaf_count *
-                                                  sub_volume) - 1;
-                        } else {
-                                if (pos < sub_volume_start ||
-                                    pos >sub_volume_end) {
-                                        ret = -1;
-                                        snprintf(err_str, 2048,"Bricks not from"
-                                                 " same subvol for %s",
-                                                 vol_type);
-                                        gf_log ("", GF_LOG_ERROR,
-                                                "%s", err_str);
-                                        goto out;
-                                }
-                        }
-                        break;
-                }
+        /* Check if the bricks belong to the same subvolumes.*/
+        if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
+            (volinfo->subvol_count > 1)) {
+                ret = subvol_matcher_verify (subvols, volinfo,
+                                             err_str, sizeof(err_str),
+                                             vol_type);
+                if (ret)
+                        goto out;
         }
-        gf_cmd_log ("Volume remove-brick","volname: %s count:%d bricks:%s",
-                    volname, count, brick_list);
 
-        ret = glusterd_op_begin (req, GD_OP_REMOVE_BRICK, dict);
-        gf_cmd_log ("Volume remove-brick","on volname: %s %s", volname,
-                    (ret) ? "FAILED" : "SUCCESS");
+        ret = glusterd_op_begin_synctask (req, GD_OP_REMOVE_BRICK, dict);
 
 out:
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
                 rsp.op_ret = -1;
                 rsp.op_errno = 0;
                 if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str), "Operation failed");
-                gf_log ("", GF_LOG_ERROR, "%s", err_str);
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 rsp.op_errstr = err_str;
                 cli_rsp = &rsp;
-                glusterd_submit_reply(req, cli_rsp, NULL, 0, NULL,
-                                      (xdrproc_t)xdr_gf_cli_rsp);
+                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
 
                 ret = 0; //sent error to cli, prevent second reply
 
         }
-        if (brick_list)
-                GF_FREE (brick_list);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        GF_FREE (brick_list);
+        subvol_matcher_destroy (subvols);
+        free (cli_req.dict.dict_val); //its malloced by xdr
 
         return ret;
 }
 
+int
+glusterd_handle_remove_brick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_remove_brick);
+}
+
+static int
+_glusterd_restart_gsync_session (dict_t *this, char *key,
+                                 data_t *value, void *data)
+{
+        char                          *slave      = NULL;
+        char                          *slave_buf  = NULL;
+        char                          *path_list  = NULL;
+        char                          *slave_vol  = NULL;
+        char                          *slave_ip   = NULL;
+        char                          *conf_path  = NULL;
+        char                         **errmsg     = NULL;
+        int                            ret        = -1;
+        glusterd_gsync_status_temp_t  *param      = NULL;
+        gf_boolean_t                   is_running = _gf_false;
+
+        param = (glusterd_gsync_status_temp_t *)data;
+
+        GF_ASSERT (param);
+        GF_ASSERT (param->volinfo);
+
+        slave = strchr(value->data, ':');
+        if (slave) {
+                slave++;
+                slave_buf = gf_strdup (slave);
+                if (!slave_buf) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Failed to gf_strdup");
+                        ret = -1;
+                        goto out;
+                }
+        }
+        else
+                return 0;
+
+        ret = dict_set_dynstr (param->rsp_dict, "slave", slave_buf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store slave");
+                if (slave_buf)
+                        GF_FREE(slave_buf);
+                goto out;
+        }
+
+        ret = glusterd_get_slave_details_confpath (param->volinfo,
+                                                   param->rsp_dict,
+                                                   &slave_ip, &slave_vol,
+                                                   &conf_path, errmsg);
+        if (ret) {
+                if (*errmsg)
+                        gf_log ("", GF_LOG_ERROR, "%s", *errmsg);
+                else
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to fetch slave or confpath details.");
+                goto out;
+        }
+
+        /* In cases that gsyncd is not running, we will not invoke it
+         * because of add-brick. */
+        ret = glusterd_check_gsync_running_local (param->volinfo->volname,
+                                                  slave, conf_path,
+                                                  &is_running);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "gsync running validation failed.");
+                goto out;
+        }
+        if (_gf_false == is_running) {
+                gf_log ("", GF_LOG_DEBUG, "gsync session for %s and %s is"
+                        " not running on this node. Hence not restarting.",
+                        param->volinfo->volname, slave);
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_get_local_brickpaths (param->volinfo, &path_list);
+        if (!path_list) {
+                gf_log ("", GF_LOG_DEBUG, "This node not being part of"
+                        " volume should not be running gsyncd. Hence"
+                        " no gsyncd process to restart.");
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_check_restart_gsync_session (param->volinfo, slave,
+                                                    param->rsp_dict, path_list,
+                                                    conf_path, 0);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to restart gsync session.");
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d.", ret);
+        return ret;
+}
 
 /* op-sm */
 
@@ -816,17 +1012,21 @@ int
 glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                                 char  *bricks, dict_t *dict)
 {
-        glusterd_brickinfo_t *brickinfo     = NULL;
-        char                 *brick         = NULL;
-        int32_t               i             = 1;
-        char                 *brick_list    = NULL;
-        char                 *free_ptr1     = NULL;
-        char                 *free_ptr2     = NULL;
-        char                 *saveptr       = NULL;
-        int32_t               ret           = -1;
-        int32_t               stripe_count  = 0;
-        int32_t               replica_count = 0;
-        int32_t               type          = 0;
+        char                         *brick          = NULL;
+        int32_t                       i              = 1;
+        char                         *brick_list     = NULL;
+        char                         *free_ptr1      = NULL;
+        char                         *free_ptr2      = NULL;
+        char                         *saveptr        = NULL;
+        int32_t                       ret            = -1;
+        int32_t                       stripe_count   = 0;
+        int32_t                       replica_count  = 0;
+        int32_t                       type           = 0;
+        glusterd_brickinfo_t         *brickinfo      = NULL;
+        glusterd_gsync_status_temp_t  param          = {0, };
+        gf_boolean_t                  restart_needed = 0;
+        char                          msg[1024] __attribute__((unused)) = {0, };
+        int                           caps           = 0;
 
         GF_ASSERT (volinfo);
 
@@ -855,7 +1055,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         }
 
         while ( i <= count) {
-                ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
+                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
                 if (ret)
                         goto out;
 
@@ -884,46 +1084,94 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         if (stripe_count) {
                 volinfo->stripe_count = stripe_count;
         }
-        volinfo->dist_leaf_count = (volinfo->stripe_count *
-                                    volinfo->replica_count);
+        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
 
         /* backward compatibility */
         volinfo->sub_count = ((volinfo->dist_leaf_count == 1) ? 0:
                               volinfo->dist_leaf_count);
 
+        volinfo->subvol_count = (volinfo->brick_count /
+                                 volinfo->dist_leaf_count);
+
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        if (ret)
+                goto out;
+
+        ret = 0;
+        if (GLUSTERD_STATUS_STARTED != volinfo->status)
+                goto out;
+
         brick_list = gf_strdup (bricks);
         free_ptr2 = brick_list;
         i = 1;
 
         if (count)
                 brick = strtok_r (brick_list+1, " \n", &saveptr);
-
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret)
-                goto out;
+#ifdef HAVE_BD_XLATOR
+        if (brickinfo->vg[0])
+                caps = CAPS_BD | CAPS_THIN |
+                        CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+#endif
 
         while (i <= count) {
-
                 ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo,
-                                                              GF_PATH_PARTIAL);
+                                                              &brickinfo);
                 if (ret)
                         goto out;
+#ifdef HAVE_BD_XLATOR
+                /* Check for VG/thin pool if its BD volume */
+                if (brickinfo->vg[0]) {
+                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_CRITICAL, "%s", msg);
+                                goto out;
+                        }
+                        /* if anyone of the brick does not have thin support,
+                           disable it for entire volume */
+                        caps &= brickinfo->caps;
+                } else
+                        caps = 0;
+#endif
 
-                if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                        ret = glusterd_brick_start (volinfo, brickinfo);
-                        if (ret)
+                if (uuid_is_null (brickinfo->uuid)) {
+                        ret = glusterd_resolve_brick (brickinfo);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                        brickinfo->hostname, brickinfo->path);
                                 goto out;
+                        }
                 }
+
+                ret = glusterd_brick_start (volinfo, brickinfo,
+                                            _gf_true);
+                if (ret)
+                        goto out;
                 i++;
                 brick = strtok_r (NULL, " \n", &saveptr);
+
+                /* Check if the brick is added in this node, and set
+                 * the restart_needed flag. */
+                if ((!uuid_compare (brickinfo->uuid, MY_UUID)) &&
+                    !restart_needed) {
+                        restart_needed = 1;
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Restart gsyncd session, if it's already "
+                                "running.");
+                }
         }
 
+        /* If the restart_needed flag is set, restart gsyncd sessions for that
+         * particular master with all the slaves. */
+        if (restart_needed) {
+                param.rsp_dict = dict;
+                param.volinfo = volinfo;
+                dict_foreach (volinfo->gsync_slaves,
+                              _glusterd_restart_gsync_session, &param);
+        }
+        volinfo->caps = caps;
 out:
-        if (free_ptr1)
-                GF_FREE (free_ptr1);
-        if (free_ptr2)
-                GF_FREE (free_ptr2);
+        GF_FREE (free_ptr1);
+        GF_FREE (free_ptr2);
 
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -935,7 +1183,6 @@ glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
                                   int force, int *need_migrate)
 {
         glusterd_brickinfo_t    *brickinfo = NULL;
-        char                    *dup_brick = NULL;
         int32_t                  ret = -1;
         glusterd_conf_t         *priv = NULL;
 
@@ -945,12 +1192,8 @@ glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
         priv = THIS->private;
         GF_ASSERT (priv);
 
-        dup_brick = gf_strdup (brick);
-        if (!dup_brick)
-                goto out;
-
-        ret = glusterd_volume_brickinfo_get_by_brick (dup_brick, volinfo,
-                                                      &brickinfo, GF_PATH_COMPLETE);
+        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                      &brickinfo);
         if (ret)
                 goto out;
 
@@ -958,30 +1201,27 @@ glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
         if (ret)
                 goto out;
 
-        if (!uuid_compare (brickinfo->uuid, priv->uuid)) {
+        glusterd_volinfo_reset_defrag_stats (volinfo);
+
+        if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
                 /* Only if the brick is in this glusterd, do the rebalance */
                 if (need_migrate)
                         *need_migrate = 1;
         }
 
         if (force) {
-                if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                        ret = glusterd_brick_stop (volinfo, brickinfo);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR, "Unable to stop "
-                                        "glusterfs, ret: %d", ret);
-                                goto out;
-                        }
+                ret = glusterd_brick_stop (volinfo, brickinfo,
+                                           _gf_true);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Unable to stop "
+                                "glusterfs, ret: %d", ret);
                 }
-                glusterd_delete_brick (volinfo, brickinfo);
                 goto out;
         }
 
         brickinfo->decommissioned = 1;
+        ret = 0;
 out:
-        if (dup_brick)
-                GF_FREE (dup_brick);
-
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
@@ -1005,6 +1245,7 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
         gf_boolean_t                            brick_alloc = _gf_false;
         char                                    *all_bricks = NULL;
         char                                    *str_ret = NULL;
+        gf_boolean_t                            is_force = _gf_false;
 
         priv = THIS->private;
         if (!priv)
@@ -1024,6 +1265,10 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         if (glusterd_is_rb_ongoing (volinfo)) {
                 snprintf (msg, sizeof (msg), "Replace brick is in progress on "
                           "volume %s. Please retry after replace-brick "
@@ -1054,6 +1299,8 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
         if (bricks) {
                 brick_list = gf_strdup (bricks);
                 all_bricks = gf_strdup (bricks);
@@ -1077,24 +1324,14 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
 
                 }
 
-                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo,
-                                                              GF_PATH_PARTIAL);
-                if (!ret) {
+                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
+                if (ret) {
                         gf_log (THIS->name, GF_LOG_ERROR,
-                                "Adding duplicate brick: %s", brick);
-                        ret = -1;
+                                "Add-brick: Unable"
+                                " to get brickinfo");
                         goto out;
-                } else {
-                        ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Add-brick: Unable"
-                                        " to get brickinfo");
-                                goto out;
-                        }
-                        brick_alloc = _gf_true;
                 }
+                brick_alloc = _gf_true;
 
                 ret = glusterd_new_brick_validate (brick, brickinfo, msg,
                                                    sizeof (msg));
@@ -1104,11 +1341,22 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 }
 
-                if (!uuid_compare (brickinfo->uuid, priv->uuid)) {
-                        ret = glusterd_brick_create_path (brickinfo->hostname,
-                                                          brickinfo->path,
+                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+#ifdef HAVE_BD_XLATOR
+                        if (brickinfo->vg[0]) {
+                                ret = glusterd_is_valid_vg (brickinfo, 1, msg);
+                                if (ret) {
+                                        gf_log (THIS->name, GF_LOG_ERROR, "%s",
+                                                msg);
+                                        *op_errstr = gf_strdup (msg);
+                                        goto out;
+                                }
+                        }
+#endif
+
+                        ret = glusterd_validate_and_create_brickpath (brickinfo,
                                                           volinfo->volume_id,
-                                                          0777, op_errstr);
+                                                          op_errstr, is_force);
                         if (ret)
                                 goto out;
                 }
@@ -1121,14 +1369,11 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
         }
 
 out:
-        if (free_ptr)
-                GF_FREE (free_ptr);
+        GF_FREE (free_ptr);
         if (brick_alloc && brickinfo)
                 glusterd_brickinfo_delete (brickinfo);
-        if (str_ret)
-                GF_FREE (str_ret);
-        if (all_bricks)
-                GF_FREE (all_bricks);
+        GF_FREE (str_ret);
+        GF_FREE (all_bricks);
 
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
@@ -1146,25 +1391,34 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
         char                msg[2048]   = {0,};
         int32_t             flag        = 0;
         gf1_op_commands     cmd         = GF_OP_CMD_NONE;
+        char               *task_id_str = NULL;
+        xlator_t           *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Volume %s does not exist", volname);
+                gf_log (this->name, GF_LOG_ERROR, "Volume %s does not exist", volname);
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         if (glusterd_is_rb_ongoing (volinfo)) {
                 snprintf (msg, sizeof (msg), "Replace brick is in progress on "
                           "volume %s. Please retry after replace-brick "
                           "operation is committed or aborted", volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 *op_errstr = gf_strdup (msg);
                 ret = -1;
                 goto out;
@@ -1172,7 +1426,7 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
 
         ret = dict_get_int32 (dict, "command", &flag);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get brick count");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count");
                 goto out;
         }
         cmd = flag;
@@ -1189,35 +1443,67 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
 
         case GF_OP_CMD_START:
         {
+                if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
+                    dict_get (dict, "replica-count")) {
+                        snprintf (msg, sizeof(msg), "Migration of data is not "
+                                  "needed when reducing replica count. Use the"
+                                  " 'force' option");
+                        errstr = gf_strdup (msg);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+
                 if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                        snprintf (msg, sizeof (msg), "Volume %s needs to be started "
-                                  "before remove-brick (you can use 'force' or "
-                                  "'commit' to override this behavior)",
+                        snprintf (msg, sizeof (msg), "Volume %s needs to be "
+                                  "started before remove-brick (you can use "
+                                  "'force' or 'commit' to override this "
+                                  "behavior)", volinfo->volname);
+                        errstr = gf_strdup (msg);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+                if (!gd_is_remove_brick_committed (volinfo)) {
+                        snprintf (msg, sizeof (msg), "An earlier remove-brick "
+                                  "task exists for volume %s. Either commit it"
+                                  " or stop it before starting a new task.",
                                   volinfo->volname);
                         errstr = gf_strdup (msg);
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", errstr);
+                        gf_log (this->name, GF_LOG_ERROR, "Earlier remove-brick"
+                                " task exists for volume %s.",
+                                volinfo->volname);
                         goto out;
                 }
                 if (glusterd_is_defrag_on(volinfo)) {
-                        errstr = gf_strdup("Rebalance is in progress. Please retry"
-                                           " after completion");
-                        gf_log ("glusterd", GF_LOG_ERROR, "%s", errstr);
+                        errstr = gf_strdup("Rebalance is in progress. Please "
+                                           "retry after completion");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
                         goto out;
                 }
-                break;
-        }
 
-        case GF_OP_CMD_PAUSE:
-        case GF_OP_CMD_ABORT:
-        {
-                if (!volinfo->decommission_in_progress) {
-                        errstr = gf_strdup("remove-brick is not in progress");
-                        gf_log ("glusterd", GF_LOG_ERROR, "%s", errstr);
-                        goto out;
+                if (is_origin_glusterd (dict)) {
+                        ret = glusterd_generate_and_set_task_id
+                                (dict, GF_REMOVE_BRICK_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to generate task-id");
+                                goto out;
+                        }
+                } else {
+                        ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Missing remove-brick-id");
+                                ret = 0;
+                        }
                 }
                 break;
         }
 
+        case GF_OP_CMD_STOP:
+                ret = 0;
+                break;
+
         case GF_OP_CMD_COMMIT:
                 if (volinfo->decommission_in_progress) {
                         errstr = gf_strdup ("use 'force' option as migration "
@@ -1232,7 +1518,7 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
 
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get brick count");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count");
                 goto out;
         }
 
@@ -1245,7 +1531,7 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         if (ret && errstr) {
                 if (op_errstr)
                         *op_errstr = errstr;
@@ -1290,6 +1576,7 @@ glusterd_remove_brick_migrate_cbk (glusterd_volinfo_t *volinfo,
                                 brickinfo->path);
                         brickinfo->decommissioned = 0;
                         if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                            /*TODO: use the 'atomic' flavour of brick_stop*/
                                 ret = glusterd_brick_stop (volinfo, brickinfo);
                                 if (ret) {
                                         gf_log (THIS->name, GF_LOG_ERROR,
@@ -1380,17 +1667,6 @@ glusterd_op_add_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        /* Need to reset the defrag/rebalance status accordingly */
-        switch (volinfo->defrag_status) {
-        case GF_DEFRAG_STATUS_FAILED:
-        case GF_DEFRAG_STATUS_COMPLETE:
-        case GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE:
-        case GF_DEFRAG_STATUS_MIGRATE_DATA_COMPLETE:
-                volinfo->defrag_status = 0;
-        default:
-                break;
-        }
-
         ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret)
                 goto out;
@@ -1405,42 +1681,74 @@ out:
 int
 glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
 {
-        int                 ret            = -1;
-        char               *volname        = NULL;
-        glusterd_volinfo_t *volinfo        = NULL;
-        char               *brick          = NULL;
-        int32_t             count          = 0;
-        int32_t             i              = 1;
-        char                key[256]       = {0,};
-        int32_t             flag           = 0;
-        char                err_str[4096]  = {0,};
-        int                 need_rebalance = 0;
-        int                 force          = 0;
-        gf1_op_commands     cmd            = 0;
-        int32_t             replica_count  = 0;
-        glusterd_brickinfo_t *brickinfo    = NULL;
-        glusterd_brickinfo_t *tmp          = NULL;
+        int                     ret            = -1;
+        char                    *volname       = NULL;
+        glusterd_volinfo_t      *volinfo       = NULL;
+        char                    *brick         = NULL;
+        int32_t                 count          = 0;
+        int32_t                 i              = 1;
+        char                    key[256]       = {0,};
+        int32_t                 flag           = 0;
+        char                    err_str[4096]  = {0,};
+        int                     need_rebalance = 0;
+        int                     force          = 0;
+        gf1_op_commands         cmd            = 0;
+        int32_t                 replica_count  = 0;
+        glusterd_brickinfo_t    *brickinfo     = NULL;
+        glusterd_brickinfo_t    *tmp           = NULL;
+        char                    *task_id_str   = NULL;
+        xlator_t                *this          = NULL;
+        dict_t                  *bricks_dict   = NULL;
+        char                    *brick_tmpstr  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "command", &flag);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get brick count");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get command");
                 goto out;
         }
         cmd = flag;
 
+        /* Set task-id, if available, in ctx dict for operations other than
+         * start
+         */
+        if (is_origin_glusterd (dict) && (cmd != GF_OP_CMD_START)) {
+                if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+                        ret = glusterd_copy_uuid_to_dict
+                                (volinfo->rebal.rebalance_id, dict,
+                                 GF_REMOVE_BRICK_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set remove-brick-id");
+                                goto out;
+                        }
+                }
+        }
+
+        /* Clear task-id, rebal.op and stored bricks on commmitting/stopping
+         * remove-brick */
+        if ((cmd != GF_OP_CMD_START) || (cmd != GF_OP_CMD_STATUS)) {
+                uuid_clear (volinfo->rebal.rebalance_id);
+                volinfo->rebal.op = GD_OP_NONE;
+                dict_unref (volinfo->rebal.dict);
+                volinfo->rebal.dict = NULL;
+        }
+
         ret = -1;
         switch (cmd) {
         case GF_OP_CMD_NONE:
@@ -1450,46 +1758,44 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 ret = 0;
                 goto out;
 
-        case GF_OP_CMD_PAUSE:
+        case GF_OP_CMD_STOP:
         {
-                if (volinfo->decommission_in_progress) {
-                        if (volinfo->defrag) {
-                                LOCK (&volinfo->defrag->lock);
-
-                                volinfo->defrag_status = GF_DEFRAG_STATUS_PAUSED;
-
-                                UNLOCK (&volinfo->defrag->lock);
-                        }
-                }
-
-                /* no need to update anything */
-                ret = 0;
-                goto out;
-        }
-
-        case GF_OP_CMD_ABORT:
-        {
-                if (volinfo->decommission_in_progress) {
-                        if (volinfo->defrag) {
-                                LOCK (&volinfo->defrag->lock);
-
-                                volinfo->defrag_status = GF_DEFRAG_STATUS_STOPPED;
-
-                                UNLOCK (&volinfo->defrag->lock);
-                        }
-                }
-
                 /* Fall back to the old volume file */
-                list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks, brick_list) {
+                list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
+                                          brick_list) {
                         if (!brickinfo->decommissioned)
                                 continue;
                         brickinfo->decommissioned = 0;
                 }
+                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to create volfiles");
+                        goto out;
+                }
+
+                ret = glusterd_store_volinfo (volinfo,
+                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to store volinfo");
+                        goto out;
+                }
+
                 ret = 0;
-                break;
+                goto out;
         }
 
         case GF_OP_CMD_START:
+                ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Missing remove-brick-id");
+                        ret = 0;
+                } else {
+                        uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
+                        volinfo->rebal.op = GD_OP_REMOVE_BRICK;
+                }
                 force = 0;
                 break;
 
@@ -1500,13 +1806,14 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
         case GF_OP_CMD_COMMIT_FORCE:
 
                 if (volinfo->decommission_in_progress) {
-                        if (volinfo->defrag) {
-                                LOCK (&volinfo->defrag->lock);
+                        if (volinfo->rebal.defrag) {
+                                LOCK (&volinfo->rebal.defrag->lock);
                                 /* Fake 'rebalance-complete' so the graph change
                                    happens right away */
-                                volinfo->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+                                volinfo->rebal.defrag_status =
+                                                GF_DEFRAG_STATUS_COMPLETE;
 
-                                UNLOCK (&volinfo->defrag->lock);
+                                UNLOCK (&volinfo->rebal.defrag->lock);
                         }
                         /* Graph change happens in rebalance _cbk function,
                            no need to do anything here */
@@ -1524,30 +1831,62 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-
+        /* Save the list of bricks for later usage. Right now this is required
+         * for displaying the task parameters with task status in volume status.
+         */
+        bricks_dict = dict_new ();
+        if (!bricks_dict) {
+                ret = -1;
+                goto out;
+        }
+        ret = dict_set_int32 (bricks_dict, "count", count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to save remove-brick count");
+                goto out;
+        }
         while ( i <= count) {
                 snprintf (key, 256, "brick%d", i);
                 ret = dict_get_str (dict, key, &brick);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to get %s", key);
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to get %s",
+                                key);
                         goto out;
                 }
 
+                brick_tmpstr = gf_strdup (brick);
+                if (!brick_tmpstr) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to duplicate brick name");
+                        goto out;
+                }
+                ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add brick to dict");
+                        goto out;
+                }
+                brick_tmpstr = NULL;
+
                 ret = glusterd_op_perform_remove_brick (volinfo, brick, force,
-                                                        (i == 1) ? &need_rebalance : NULL);
+                                                        &need_rebalance);
                 if (ret)
                         goto out;
                 i++;
         }
         ret = dict_get_int32 (dict, "replica-count", &replica_count);
         if (!ret) {
-                gf_log (THIS->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_INFO,
                         "changing replica count %d to %d on volume %s",
                         volinfo->replica_count, replica_count,
                         volinfo->volname);
                 volinfo->replica_count = replica_count;
-                volinfo->dist_leaf_count = (volinfo->stripe_count *
-                                            replica_count);
+                volinfo->sub_count = replica_count;
+                volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+                volinfo->subvol_count = (volinfo->brick_count /
+                                         volinfo->dist_leaf_count);
+
                 if (replica_count == 1) {
                         if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
                                 volinfo->type = GF_CLUSTER_TYPE_NONE;
@@ -1560,39 +1899,41 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         }
                 }
         }
+        volinfo->rebal.dict = bricks_dict;
+        bricks_dict = NULL;
 
         ret = glusterd_create_volfiles_and_notify_services (volinfo);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "failed to create volfiles");
+                gf_log (this->name, GF_LOG_WARNING, "failed to create volfiles");
                 goto out;
         }
 
         ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "failed to store volinfo");
+                gf_log (this->name, GF_LOG_WARNING, "failed to store volinfo");
                 goto out;
         }
 
         /* Need to reset the defrag/rebalance status accordingly */
-        switch (volinfo->defrag_status) {
+        switch (volinfo->rebal.defrag_status) {
         case GF_DEFRAG_STATUS_FAILED:
         case GF_DEFRAG_STATUS_COMPLETE:
-        case GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE:
-        case GF_DEFRAG_STATUS_MIGRATE_DATA_COMPLETE:
-                volinfo->defrag_status = 0;
+                volinfo->rebal.defrag_status = 0;
         default:
                 break;
         }
         if (!force && need_rebalance) {
                 /* perform the rebalance operations */
-                ret = glusterd_handle_defrag_start (volinfo, err_str, 4096,
-                                                    GF_DEFRAG_CMD_START_FORCE,
-                                                    glusterd_remove_brick_migrate_cbk);
+                ret = glusterd_handle_defrag_start
+                        (volinfo, err_str, sizeof (err_str),
+                         GF_DEFRAG_CMD_START_FORCE,
+                         glusterd_remove_brick_migrate_cbk, GD_OP_REMOVE_BRICK);
+
                 if (!ret)
                         volinfo->decommission_in_progress = 1;
 
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to start the rebalance");
                 }
         } else {
@@ -1604,5 +1945,9 @@ out:
         if (ret && err_str[0] && op_errstr)
                 *op_errstr = gf_strdup (err_str);
 
+        GF_FREE (brick_tmpstr);
+        if (bricks_dict)
+                dict_unref (bricks_dict);
+
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
index db43ecb4d..5786694bd 100644
--- a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
+++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -31,19 +21,203 @@
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "run.h"
+#include "syscall.h"
 
 #include <signal.h>
 
+static int
+dict_get_param (dict_t *dict, char *key, char **param);
+
+static int
+glusterd_get_statefile_name (glusterd_volinfo_t *volinfo, char *slave,
+                             char *conf_path, char **statefile);
+
+static int
+glusterd_get_slave_info (char *slave, char **slave_ip,
+                         char **slave_vol, char **op_errstr);
+
+static int
+glusterd_gsync_read_frm_status (char *path, char *buf, size_t blen);
+
+struct gsync_config_opt_vals_ gsync_confopt_vals[] = {
+        {.op_name        = "change_detector",
+         .no_of_pos_vals = 2,
+         .case_sensitive = _gf_true,
+         .values         = {"xsync", "changelog"},
+        },
+        {.op_name        = "special_sync_mode",
+         .no_of_pos_vals = 2,
+         .case_sensitive = _gf_true,
+         .values         = {"partial", "recover"}
+        },
+        {.op_name        = "log-level",
+         .no_of_pos_vals = 5,
+         .case_sensitive = _gf_false,
+         .values         = {"critical", "error", "warning", "info", "debug"}
+        },
+        {.op_name = NULL,
+        },
+};
+
 static char *gsync_reserved_opts[] = {
-        "gluster-command",
+        "gluster-command-dir",
         "pid-file",
+        "remote-gsyncd"
         "state-file",
         "session-owner",
+        "state-socket-unencoded",
+        "socketdir",
+        "ignore-deletes",
+        "local-id",
+        "local-path",
+        "slave-id",
         NULL
 };
 
 int
-glusterd_handle_gsync_set (rpcsvc_request_t *req)
+__glusterd_handle_sys_exec (rpcsvc_request_t *req)
+{
+        int32_t                 ret     = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req              cli_req = {{0},};
+        glusterd_op_t           cli_op = GD_OP_SYS_EXEC;
+        glusterd_conf_t         *priv   = NULL;
+        char                    *host_uuid = NULL;
+        char                    err_str[2048] = {0,};
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+               } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
+                if (host_uuid == NULL) {
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        return ret;
+}
+
+int
+__glusterd_handle_copy_file (rpcsvc_request_t *req)
+{
+        int32_t                 ret     = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req              cli_req = {{0},};
+        glusterd_op_t           cli_op = GD_OP_COPY_FILE;
+        glusterd_conf_t         *priv   = NULL;
+        char                    *host_uuid = NULL;
+        char                    err_str[2048] = {0,};
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+               } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
+                if (host_uuid == NULL) {
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        return ret;
+}
+
+int
+__glusterd_handle_gsync_set (rpcsvc_request_t *req)
 {
         int32_t                 ret     = 0;
         dict_t                  *dict   = NULL;
@@ -55,15 +229,19 @@ glusterd_handle_gsync_set (rpcsvc_request_t *req)
         int                     type = 0;
         glusterd_conf_t         *priv   = NULL;
         char                    *host_uuid = NULL;
+        char                    err_str[2048] = {0,};
+        xlator_t                *this = NULL;
 
         GF_ASSERT (req);
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
 
-        priv = THIS->private;
+       this = THIS;
+       GF_ASSERT (this);
+       priv = this->private;
+       GF_ASSERT (priv);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -77,17 +255,19 @@ glusterd_handle_gsync_set (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
                 }
 
-                host_uuid = gf_strdup (uuid_utoa(priv->uuid));
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
                 if (host_uuid == NULL) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "failed to get"
-                                "the uuid of the host machine");
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
                         ret = -1;
                         goto out;
                 }
@@ -99,26 +279,31 @@ glusterd_handle_gsync_set (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "master", &master);
         if (ret < 0) {
-                gf_log ("", GF_LOG_INFO, "master not found, while handling"
-                         GEOREP" options");
+                gf_log (this->name, GF_LOG_INFO, "master not found, while "
+                        "handling "GEOREP" options");
                 master = "(No Master)";
         }
 
         ret = dict_get_str (dict, "slave", &slave);
         if (ret < 0) {
-                gf_log ("", GF_LOG_INFO, "slave not not found, while"
+                gf_log (this->name, GF_LOG_INFO, "slave not found, while "
                         "handling "GEOREP" options");
                 slave = "(No Slave)";
         }
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "command type not found, while"
-                        "handling "GEOREP" options");
+                snprintf (err_str, sizeof (err_str), "Command type not found "
+                          "while handling "GEOREP" options");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         switch (type) {
+        case GF_GSYNC_OPTION_TYPE_CREATE:
+                strncpy (operation, "create", sizeof (operation));
+                cli_op = GD_OP_GSYNC_CREATE;
+                break;
 
         case GF_GSYNC_OPTION_TYPE_START:
                 strncpy (operation, "start", sizeof (operation));
@@ -135,30 +320,38 @@ glusterd_handle_gsync_set (rpcsvc_request_t *req)
         case GF_GSYNC_OPTION_TYPE_STATUS:
                 strncpy (operation, "status", sizeof (operation));
                 break;
-        case GF_GSYNC_OPTION_TYPE_ROTATE:
-                strncpy (operation, "rotate", sizeof(operation));
-                break;
         }
 
-        gf_cmd_log ("volume "GEOREP, " %s command on %s,%s", operation, master,
-                    slave);
-        ret = glusterd_op_begin (req, GD_OP_GSYNC_SET, dict);
-        gf_cmd_log ("volume "GEOREP, " %s command on %s,%s %s  ", operation,
-                    master, slave, (ret != 0)? "FAILED" : "SUCCEEDED");
+        ret = glusterd_op_begin_synctask (req, cli_op, dict);
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
         }
         return ret;
 }
 
+int
+glusterd_handle_sys_exec (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_sys_exec);
+}
+
+int
+glusterd_handle_copy_file (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_copy_file);
+}
+
+int
+glusterd_handle_gsync_set (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_gsync_set);
+}
 
 /*****
  *
@@ -180,7 +373,7 @@ glusterd_urltransform_add (runner_t *runner, const char *url)
         runner_add_arg (runner, url);
 }
 
-static void
+static int
 _glusterd_urltransform_add_iter (dict_t *dict, char *key, data_t *value, void *data)
 {
         runner_t *runner = (runner_t *)data;
@@ -190,6 +383,8 @@ _glusterd_urltransform_add_iter (dict_t *dict, char *key, data_t *value, void *d
         GF_ASSERT (slave);
         slave++;
         runner_add_arg (runner, slave);
+
+        return 0;
 }
 
 static void
@@ -228,6 +423,7 @@ glusterd_urltransform (runner_t *runner, char ***linearrp)
 
         arr_idx = 0;
         for (;;) {
+                size_t len;
                 line = GF_MALLOC (1024, gf_gld_mt_linebuf);
                 if (!line) {
                         error = _gf_true;
@@ -238,21 +434,24 @@ glusterd_urltransform (runner_t *runner, char ***linearrp)
                     NULL)
                         break;
 
-                if (line[strlen (line) - 1] != '\n') {
+                len = strlen (line);
+                if (len == 0 || line[len - 1] != '\n') {
                         GF_FREE (line);
                         error = _gf_true;
                         goto out;
                 }
-                line[strlen (line) - 1] = '\0';
+                line[len - 1] = '\0';
 
                 if (arr_idx == arr_len) {
+                        void *p = linearr;
                         arr_len <<= 1;
-                        linearr = GF_REALLOC (linearr, arr_len);
-                        if (!linearr) {
+                        p = GF_REALLOC (linearr, arr_len);
+                        if (!p) {
                                 GF_FREE (line);
                                 error = _gf_true;
                                 goto out;
                         }
+                        linearr = p;
                 }
                 linearr[arr_idx] = line;
 
@@ -300,7 +499,7 @@ struct dictidxmark {
         char *ikey;
 };
 
-static void
+static int
 _dict_mark_atindex (dict_t *dict, char *key, data_t *value, void *data)
 {
         struct dictidxmark *dim = data;
@@ -309,6 +508,7 @@ _dict_mark_atindex (dict_t *dict, char *key, data_t *value, void *data)
                 dim->ikey = key;
 
         dim->ithis++;
+        return 0;
 }
 
 static char *
@@ -354,9 +554,9 @@ glusterd_get_slave (glusterd_volinfo_t *vol, const char *slaveurl, char **slavek
 
 
 static int
-glusterd_query_extutil (char *resbuf, runner_t *runner)
+glusterd_query_extutil_generic (char *resbuf, size_t blen, runner_t *runner, void *data,
+                                int (*fcbk)(char *resbuf, size_t blen, FILE *fp, void *data))
 {
-        char               *ptr = NULL;
         int                 ret = 0;
 
         runner_redir (runner, STDOUT_FILENO, RUN_PIPE);
@@ -366,92 +566,117 @@ glusterd_query_extutil (char *resbuf, runner_t *runner)
                 return -1;
         }
 
-        ptr = fgets(resbuf, PATH_MAX, runner_chio (runner, STDOUT_FILENO));
-        if (ptr)
-                resbuf[strlen(resbuf)-1] = '\0'; //strip off \n
+        ret = fcbk (resbuf, blen, runner_chio (runner, STDOUT_FILENO), data);
 
-        ret = runner_end (runner);
+        ret |= runner_end (runner);
         if (ret)
                 gf_log ("", GF_LOG_ERROR, "reading data from child failed");
 
         return ret ? -1 : 0;
 }
 
-int
-glusterd_gsync_get_param_file (char *prmfile, const char *param, char *master,
-                               char *slave, char *gl_workdir)
+static int
+_fcbk_singleline(char *resbuf, size_t blen, FILE *fp, void *data)
 {
-        runner_t            runner = {0,};
+        char *ptr = NULL;
+
+        errno = 0;
+        ptr = fgets (resbuf, blen, fp);
+        if (ptr) {
+                size_t len = strlen(resbuf);
+                if (len && resbuf[len-1] == '\n')
+                        resbuf[len-1] = '\0'; //strip off \n
+        }
 
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, gl_workdir);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, "--config-get", NULL);
-        runner_argprintf (&runner, "%s-file", param);
+        return errno ? -1 : 0;
+}
 
-        return glusterd_query_extutil (prmfile, &runner);
+static int
+glusterd_query_extutil (char *resbuf, runner_t *runner)
+{
+        return glusterd_query_extutil_generic (resbuf, PATH_MAX, runner, NULL,
+                                               _fcbk_singleline);
 }
 
-int
-glusterd_gsync_get_session_owner (char *master, char *slave, char *session_owner,
-                                  char *gl_workdir)
+static int
+_fcbk_conftodict (char *resbuf, size_t blen, FILE *fp, void *data)
 {
-        runner_t runner = {0,};
+        char   *ptr = NULL;
+        dict_t *dict = data;
+        char   *v = NULL;
 
-        runinit(&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, gl_workdir);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, "--config-get", "session-owner",
-                          NULL);
+        for (;;) {
+                errno = 0;
+                ptr = fgets (resbuf, blen, fp);
+                if (!ptr)
+                        break;
+                v = resbuf + strlen(resbuf) - 1;
+                while (isspace (*v))
+                        /* strip trailing space */
+                        *v-- = '\0';
+                if (v == resbuf)
+                        /* skip empty line */
+                        continue;
+                v = strchr (resbuf, ':');
+                if (!v)
+                        return -1;
+                *v++ = '\0';
+                while (isspace (*v))
+                        v++;
+                v = gf_strdup (v);
+                if (!v)
+                        return -1;
+                if (dict_set_dynstr (dict, resbuf, v) != 0) {
+                        GF_FREE (v);
+                        return -1;
+                }
+        }
 
-        return glusterd_query_extutil (session_owner, &runner);
+        return errno ? -1 : 0;
 }
 
-int
-glusterd_gsync_get_slave_log_file (char *master, char *slave, char *log_file)
+static int
+glusterd_gsync_get_config (char *master, char *slave, char *conf_path, dict_t *dict)
 {
-        int              ret        = -1;
-        runner_t         runner     = {0,};
-        char uuid_str[64]           = {0,};
-        glusterd_conf_t *priv       = NULL;
-        char            *gl_workdir = NULL;
-
-        GF_ASSERT(THIS);
-        GF_ASSERT(THIS->private);
-
-        priv = THIS->private;
+        /* key + value, where value must be able to accommodate a path */
+        char resbuf[256 + PATH_MAX] = {0,};
+        runner_t             runner = {0,};
 
-        GF_VALIDATE_OR_GOTO("gsyncd", master, out);
-        GF_VALIDATE_OR_GOTO("gsyncd", slave, out);
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, "--config-get-all", NULL);
 
-        gl_workdir = priv->workdir;
+        return glusterd_query_extutil_generic (resbuf, sizeof (resbuf),
+                                               &runner, dict, _fcbk_conftodict);
+}
 
-        /* get the session owner for the master-slave session */
-        ret = glusterd_gsync_get_session_owner (master, slave, uuid_str,
-                                                gl_workdir);
-        if (ret)
-                goto out;
+static int
+glusterd_gsync_get_param_file (char *prmfile, const char *param, char *master,
+                               char *slave, char *conf_path)
+{
+        runner_t            runner = {0,};
 
-        /* get the log file for the slave */
-        runinit(&runner);
+        runinit (&runner);
         runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, gl_workdir);
-        runner_argprintf (&runner, "--session-owner=%s", uuid_str);
-        runner_add_args  (&runner, slave, "--config-get", "log-file", NULL);
-
-        ret = glusterd_query_extutil (log_file, &runner);
+        runner_argprintf (&runner, "%s", conf_path);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, "--config-get", NULL);
+        runner_argprintf (&runner, "%s-file", param);
 
- out:
-        return ret;
+        return glusterd_query_extutil (prmfile, &runner);
 }
 
 static int
-gsyncd_getpidfile (char *master, char *slave, char *pidfile)
+gsyncd_getpidfile (char *master, char *slave, char *pidfile, char *conf_path)
 {
         int                ret             = -1;
         glusterd_conf_t    *priv  = NULL;
+        char               *confpath = NULL;
+        char                conf_buf[PATH_MAX] = "";
+        struct stat         stbuf = {0,};
+
 
         GF_ASSERT (THIS);
         GF_ASSERT (THIS->private);
@@ -461,8 +686,22 @@ gsyncd_getpidfile (char *master, char *slave, char *pidfile)
         GF_VALIDATE_OR_GOTO ("gsync", master, out);
         GF_VALIDATE_OR_GOTO ("gsync", slave, out);
 
+        ret = lstat (conf_path, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "Using passed config template(%s).",
+                        conf_path);
+                confpath = conf_path;
+        } else {
+                ret = snprintf (conf_buf, sizeof(conf_buf) - 1,
+                                "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
+                conf_buf[ret] = '\0';
+                confpath = conf_buf;
+                gf_log ("", GF_LOG_DEBUG, "Using default config template(%s).",
+                        confpath);
+        }
+
         ret = glusterd_gsync_get_param_file (pidfile, "pid", master,
-                                              slave, priv->workdir);
+                                              slave, confpath);
         if (ret == -1) {
                 ret = -2;
                 gf_log ("", GF_LOG_WARNING, "failed to create the pidfile string");
@@ -476,32 +715,6 @@ gsyncd_getpidfile (char *master, char *slave, char *pidfile)
 }
 
 static int
-glusterd_gsyncd_getlogfile (char *master, char *slave, char *log_file)
-{
-        int                ret             = -1;
-        glusterd_conf_t    *priv  = NULL;
-
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
-
-        priv = THIS->private;
-
-        GF_VALIDATE_OR_GOTO ("gsync", master, out);
-        GF_VALIDATE_OR_GOTO ("gsync", slave, out);
-
-        ret = glusterd_gsync_get_param_file (log_file, "log", master,
-                                              slave, priv->workdir);
-        if (ret == -1) {
-                ret = -2;
-                gf_log ("", GF_LOG_WARNING, "failed to gsyncd logfile");
-                goto out;
-        }
-
- out:
-        return ret;
-}
-
-static int
 gsync_status_byfd (int fd)
 {
         GF_ASSERT (fd >= -1);
@@ -518,23 +731,24 @@ gsync_status_byfd (int fd)
  * return -1 when not running
  */
 int
-gsync_status (char *master, char *slave, int *status)
+gsync_status (char *master, char *slave, char *conf_path, int *status)
 {
         char pidfile[PATH_MAX] = {0,};
         int  fd                = -1;
 
-        fd = gsyncd_getpidfile (master, slave, pidfile);
+        fd = gsyncd_getpidfile (master, slave, pidfile, conf_path);
         if (fd == -2)
                 return -1;
 
         *status = gsync_status_byfd (fd);
 
-        close (fd);
+        sys_close (fd);
+
         return 0;
 }
 
 
-int32_t
+static int32_t
 glusterd_gsync_volinfo_dict_set (glusterd_volinfo_t *volinfo,
                                  char *key, char *value)
 {
@@ -558,17 +772,49 @@ out:
         return 0;
 }
 
-int
-gsync_verify_config_options (dict_t *dict, char **op_errstr)
+static int
+glusterd_verify_gsyncd_spawn (char *master, char *slave)
+{
+        int                 ret = 0;
+        runner_t            runner = {0,};
+
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd",
+                         "--verify", "spawning", NULL);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "spawning child failed");
+                ret = -1;
+                goto out;
+        }
+
+        if (runner_end (&runner) != 0)
+                ret = -1;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+static int
+gsync_verify_config_options (dict_t *dict, char **op_errstr, char *volname)
 {
-        char  **resopt    = NULL;
-        int     i         = 0;
-        char   *subop     = NULL;
-        char   *slave     = NULL;
-        char   *op_name   = NULL;
-        char   *op_value  = NULL;
-        char   *t         = NULL;
-        gf_boolean_t banned = _gf_true;
+        char                         **resopt           = NULL;
+        int                            i                = 0;
+        int                            ret              = -1;
+        char                          *subop            = NULL;
+        char                          *slave            = NULL;
+        char                          *op_name          = NULL;
+        char                          *op_value         = NULL;
+        char                          *t                = NULL;
+        char                           errmsg[PATH_MAX] = "";
+        gf_boolean_t                   banned           = _gf_true;
+        gf_boolean_t                   op_match         = _gf_true;
+        gf_boolean_t                   val_match        = _gf_true;
+        struct gsync_config_opt_vals_ *conf_vals        = NULL;
 
         if (dict_get_str (dict, "subop", &subop) != 0) {
                 gf_log ("", GF_LOG_WARNING, "missing subop");
@@ -592,6 +838,12 @@ gsync_verify_config_options (dict_t *dict, char **op_errstr)
         }
 
         if (runcmd (GSYNCD_PREFIX"/gsyncd", "--config-check", op_name, NULL)) {
+                ret = glusterd_verify_gsyncd_spawn (volname, slave);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to spawn gsyncd");
+                        return 0;
+                }
+
                 gf_log ("", GF_LOG_WARNING, "Invalid option %s", op_name);
                 *op_errstr = gf_strdup ("Invalid option");
 
@@ -636,34 +888,113 @@ gsync_verify_config_options (dict_t *dict, char **op_errstr)
                 }
         }
 
+        /* Check options in gsync_confopt_vals for invalid values */
+        for (conf_vals = gsync_confopt_vals; conf_vals->op_name; conf_vals++) {
+                op_match = _gf_true;
+                for (i = 0; conf_vals->op_name[i] && op_name[i]; i++) {
+                        if (conf_vals->op_name[i] == op_name[i] ||
+                            (conf_vals->op_name[i] == '_' && op_name[i] == '-'))
+                                continue;
+                        op_match = _gf_false;
+                }
+
+                if (op_match) {
+                        val_match = _gf_false;
+                        for (i = 0; i < conf_vals->no_of_pos_vals; i++) {
+                                if(conf_vals->case_sensitive){
+                                        if (!strcmp (conf_vals->values[i], op_value))
+                                                val_match = _gf_true;
+                                } else {
+                                        if (!strcasecmp (conf_vals->values[i], op_value))
+                                                val_match = _gf_true;
+                                }
+                        }
+
+                        if (!val_match) {
+                                ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                                "Invalid values (%s) for"
+                                                " option %s", op_value,
+                                                op_name);
+                                errmsg[ret] = '\0';
+
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                return -1;
+                        }
+                }
+        }
+
         return 0;
 }
 
-static void
+static int
+glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
+                                   char *slave, char *conf_path,
+                                   dict_t *rsp_dict, char *node);
+
+static int
 _get_status_mst_slv (dict_t *this, char *key, data_t *value, void *data)
 {
         glusterd_gsync_status_temp_t  *param = NULL;
         char                          *slave = NULL;
-        int                           ret = 0;
+        char                          *slave_buf = NULL;
+        char                          *slave_ip  = NULL;
+        char                          *slave_vol = NULL;
+        char                          *errmsg    = NULL;
+        char                           conf_path[PATH_MAX] = "";
+        int                           ret = -1;
+        glusterd_conf_t              *priv = NULL;
 
         param = (glusterd_gsync_status_temp_t *)data;
 
         GF_ASSERT (param);
         GF_ASSERT (param->volinfo);
 
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                goto out;
+        }
+
         slave = strchr(value->data, ':');
-        if (slave)
-                slave ++;
-        else
-                return;
+        if (!slave)
+                return 0;
+        slave++;
+
+        ret = glusterd_get_slave_info (slave, &slave_ip, &slave_vol, &errmsg);
+        if (ret) {
+                if (errmsg)
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch "
+                                "slave details. Error: %s", errmsg);
+                else
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to fetch slave details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (conf_path, sizeof(conf_path) - 1,
+                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                        priv->workdir, param->volinfo->volname,
+                        slave_ip, slave_vol);
+        conf_path[ret] = '\0';
 
         ret = glusterd_get_gsync_status_mst_slv(param->volinfo,
-                                                slave, param->rsp_dict);
+                                                slave, conf_path,
+                                                param->rsp_dict,
+                                                param->node);
+out:
+
+        if (slave_buf)
+                GF_FREE(slave_buf);
 
+        gf_log ("", GF_LOG_DEBUG, "Returning %d.", ret);
+        return ret;
 }
 
 
-static void
+static int
 _get_max_gsync_slave_num (dict_t *this, char *key, data_t *value, void *data)
 {
         int  tmp_slvnum = 0;
@@ -672,25 +1003,30 @@ _get_max_gsync_slave_num (dict_t *this, char *key, data_t *value, void *data)
         sscanf (key, "slave%d", &tmp_slvnum);
         if (tmp_slvnum > *slvnum)
                 *slvnum = tmp_slvnum;
+
+        return 0;
 }
 
 static int
 glusterd_remove_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
                                char **op_errstr)
 {
+        int   zero_slave_entries = _gf_true;
         int   ret = 0;
         char *slavekey = NULL;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (slave);
 
-        ret = glusterd_get_slave (volinfo, slave, &slavekey);
-        if (ret < 0) {
-                ret++;
-                goto out;
-        }
-
-        dict_del (volinfo->gsync_slaves, slavekey);
+        do {
+                ret = glusterd_get_slave (volinfo, slave, &slavekey);
+                if (ret < 0 && zero_slave_entries) {
+                        ret++;
+                        goto out;
+                }
+                zero_slave_entries = _gf_false;
+                dict_del (volinfo->gsync_slaves, slavekey);
+        } while (ret >= 0);
 
         ret = glusterd_store_volinfo (volinfo,
                                       GLUSTERD_VOLINFO_VER_AC_INCREMENT);
@@ -743,8 +1079,9 @@ glusterd_gsync_get_uuid (char *slave, glusterd_volinfo_t *vol,
         return ret;
 }
 
-static int
+int
 glusterd_check_gsync_running_local (char *master, char *slave,
+                                    char *conf_path,
                                     gf_boolean_t *is_run)
 {
         int                 ret    = -1;
@@ -755,7 +1092,7 @@ glusterd_check_gsync_running_local (char *master, char *slave,
         GF_ASSERT (is_run);
 
         *is_run = _gf_false;
-        ret = gsync_status (master, slave, &ret_status);
+        ret = gsync_status (master, slave, conf_path, &ret_status);
         if (ret == 0 && ret_status == 0) {
                 *is_run = _gf_true;
         } else if (ret == -1) {
@@ -772,7 +1109,8 @@ glusterd_check_gsync_running_local (char *master, char *slave,
 
 static int
 glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
-                              char *host_uuid, char **op_errstr)
+                              char *host_uuid, char **op_errstr,
+                              gf_boolean_t is_force)
 {
         int    ret = 0;
         int    maxslv = 0;
@@ -795,7 +1133,8 @@ glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
         case -1:
                                 break;
         default:
-                GF_ASSERT (ret > 0);
+                if (!is_force)
+                        GF_ASSERT (ret > 0);
                 ret = dict_get_str (volinfo->gsync_slaves, slavekey, &slaveentry);
                 GF_ASSERT (ret == 0);
 
@@ -804,13 +1143,23 @@ glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
                  * assert an uuid mismatch
                  */
                 t = strtail (slaveentry, host_uuid);
-                GF_ASSERT (!t || *t != ':')
+                if (!is_force)
+                        GF_ASSERT (!t || *t != ':');
+
+                if (is_force) {
+                        gf_log ("", GF_LOG_DEBUG, GEOREP" has already been "
+                                "invoked for the %s (master) and %s (slave)."
+                                " Allowing without saving info again due to"
+                                " force command.", volinfo->volname, slave);
+                        ret = 0;
+                        goto out;
+                }
 
                 gf_log ("", GF_LOG_ERROR, GEOREP" has already been invoked for "
                                           "the %s (master) and %s (slave) "
                                           "from a different machine",
                                            volinfo->volname, slave);
-                *op_errstr = gf_strdup (GEOREP" already running in an an"
+                *op_errstr = gf_strdup (GEOREP" already running in "
                                         "another machine");
                 ret = -1;
                 goto out;
@@ -843,23 +1192,26 @@ glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
         return ret;
 }
 
-
 static int
 glusterd_op_verify_gsync_start_options (glusterd_volinfo_t *volinfo,
-                                        char *slave, char **op_errstr)
+                                        char *slave, char *conf_path,
+                                        char *statefile, char **op_errstr,
+                                        gf_boolean_t is_force)
 {
         int                     ret = -1;
         gf_boolean_t            is_running = _gf_false;
         char                    msg[2048] = {0};
         uuid_t                  uuid = {0};
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
+        glusterd_conf_t        *priv = NULL;
+        xlator_t               *this = NULL;
+        struct stat             stbuf = {0,};
 
         this = THIS;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (slave);
         GF_ASSERT (op_errstr);
+        GF_ASSERT (conf_path);
         GF_ASSERT (this && this->private);
 
         priv  = this->private;
@@ -869,26 +1221,56 @@ glusterd_op_verify_gsync_start_options (glusterd_volinfo_t *volinfo,
                           "before "GEOREP" start", volinfo->volname);
                 goto out;
         }
+
+        ret = lstat (statefile, &stbuf);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Session between %s and %s has"
+                          " not been created. Please create session and retry.",
+                          volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        /* Check if the gsync slave info is stored. If not
+         * session has not been created */
+        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Session between %s and %s has"
+                          " not been created. Please create session and retry.",
+                          volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        if (is_force) {
+                ret = 0;
+                goto out;
+        }
+
         /*Check if the gsync is already started in cmd. inited host
          * If so initiate add it into the glusterd's priv*/
-        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-        if ((ret == 0) && (uuid_compare (priv->uuid, uuid) == 0)) {
-                ret = glusterd_check_gsync_running_local (volinfo->volname,
-                                                          slave, &is_running);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), GEOREP" start option "
-                                  "validation failed ");
-                        goto out;
-                }
-                if (_gf_true == is_running) {
-                        snprintf (msg, sizeof (msg), GEOREP " session between"
-                                  " %s & %s already started", volinfo->volname,
-                                  slave);
-                        ret = -1;
-                        goto out;
-                }
+        ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                  slave, conf_path,
+                                                  &is_running);
+        if (ret) {
+                snprintf (msg, sizeof (msg), GEOREP" start option "
+                          "validation failed ");
+                goto out;
+        }
+        if (_gf_true == is_running) {
+                snprintf (msg, sizeof (msg), GEOREP " session between"
+                          " %s & %s already started", volinfo->volname,
+                          slave);
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Unable to spawn gsyncd");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
         }
-        ret = 0;
 out:
         if (ret && (msg[0] != '\0')) {
                 *op_errstr = gf_strdup (msg);
@@ -914,11 +1296,13 @@ glusterd_check_gsync_running (glusterd_volinfo_t *volinfo, gf_boolean_t *flag)
 
 static int
 glusterd_op_verify_gsync_running (glusterd_volinfo_t *volinfo,
-                                  char *slave, char **op_errstr)
+                                  char *slave, char *conf_path,
+                                  char **op_errstr)
 {
-        int                     ret = -1;
-        char                    msg[2048] = {0};
-        uuid_t                  uuid = {0};
+        int        pfd               = -1;
+        int        ret               = -1;
+        char       msg[2048]         = {0};
+        char       pidfile[PATH_MAX] = {0,};
 
         GF_ASSERT (THIS && THIS->private);
         GF_ASSERT (volinfo);
@@ -931,13 +1315,26 @@ glusterd_op_verify_gsync_running (glusterd_volinfo_t *volinfo,
 
                 goto out;
         }
-        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-        if (ret == -1) {
-                snprintf (msg, sizeof (msg), GEOREP" session between %s & %s"
-                          " not active", volinfo->volname, slave);
+
+        pfd = gsyncd_getpidfile (volinfo->volname, slave, pidfile, conf_path);
+        if (pfd == -2) {
+                gf_log ("", GF_LOG_ERROR, GEOREP" stop validation "
+                        "failed for %s & %s", volinfo->volname, slave);
+                ret = -1;
+                goto out;
+        }
+        if (gsync_status_byfd (pfd) == -1) {
+                snprintf (msg, sizeof (msg), GEOREP" session b/w %s & %s is not"
+                          " running on this node.", volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                ret = -1;
+                /* monitor gsyncd already dead */
                 goto out;
         }
 
+        if (pfd < 0)
+                goto out;
+
         ret = 0;
 out:
         if (ret && (msg[0] != '\0')) {
@@ -956,6 +1353,18 @@ glusterd_verify_gsync_status_opts (dict_t *dict, char **op_errstr)
         gf_boolean_t       exists = _gf_false;
         glusterd_volinfo_t *volinfo = NULL;
         int                ret = 0;
+        char               *conf_path = NULL;
+        char               *slave_ip = NULL;
+        char               *slave_vol = NULL;
+        glusterd_conf_t    *priv = NULL;
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
 
         ret = dict_get_str (dict, "master", &volname);
         if (ret < 0) {
@@ -980,38 +1389,447 @@ glusterd_verify_gsync_status_opts (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
- out:
+        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_ip,
+                                                   &slave_vol, &conf_path,
+                                                   op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave  or confpath details.");
+                ret = -1;
+                goto out;
+        }
+
+out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
-
 }
 
 
-static int
+int
 glusterd_op_gsync_args_get (dict_t *dict, char **op_errstr,
-                            char **master, char **slave)
+                            char **master, char **slave, char **host_uuid)
 {
 
         int             ret = -1;
         GF_ASSERT (dict);
         GF_ASSERT (op_errstr);
-        GF_ASSERT (master);
-        GF_ASSERT (slave);
 
-        ret = dict_get_str (dict, "master", master);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "master not found");
-                *op_errstr = gf_strdup ("master not found");
+        if (master) {
+                ret = dict_get_str (dict, "master", master);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_WARNING, "master not found");
+                        *op_errstr = gf_strdup ("master not found");
+                        goto out;
+                }
+        }
+
+        if (slave) {
+                ret = dict_get_str (dict, "slave", slave);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_WARNING, "slave not found");
+                        *op_errstr = gf_strdup ("slave not found");
+                        goto out;
+                }
+        }
+
+        if (host_uuid) {
+                ret = dict_get_str (dict, "host-uuid", host_uuid);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_WARNING, "host_uuid not found");
+                        *op_errstr = gf_strdup ("host_uuid not found");
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_sys_exec (dict_t *dict, char **op_errstr)
+{
+        char             errmsg[PATH_MAX]          = "";
+        char            *command                   = NULL;
+        char             command_path[PATH_MAX]    = "";
+        struct stat      st                        = {0,};
+        int              ret                       = -1;
+        glusterd_conf_t *conf                      = NULL;
+        xlator_t        *this                      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (conf->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "command", &command);
+        if (ret) {
+                strcpy (errmsg, "internal error");
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get command from dict");
+                goto out;
+        }
+
+        /* enforce local occurrence of the command */
+        if (strchr (command, '/')) {
+                strcpy (errmsg, "invalid command name");
+                ret = -1;
+                goto out;
+        }
+
+        sprintf (command_path, GSYNCD_PREFIX"/peer_%s", command);
+        /* check if it's executable */
+        ret = access (command_path, X_OK);
+        if (!ret)
+                /* check if it's a regular file */
+                ret = stat (command_path, &st);
+        if (!ret && !S_ISREG (st.st_mode))
+                ret = -1;
+
+out:
+        if (ret) {
+                if (errmsg[0] == '\0')
+                        snprintf (errmsg, sizeof (errmsg), "%s not found.",
+                                  command);
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_copy_file (dict_t *dict, char **op_errstr)
+{
+        char             abs_filename[PATH_MAX] = "";
+        char             errmsg[PATH_MAX]       = "";
+        char            *filename               = NULL;
+        char            *host_uuid              = NULL;
+        char             uuid_str [64]          = {0};
+        int              ret                    = -1;
+        glusterd_conf_t *priv                   = NULL;
+        struct stat      stbuf                  = {0,};
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
                 goto out;
         }
 
-        ret = dict_get_str (dict, "slave", slave);
+        if (priv->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "host-uuid", &host_uuid);
         if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "slave not found");
-                *op_errstr = gf_strdup ("slave not found");
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch"
+                        " host-uuid from dict.");
+                goto out;
+        }
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = dict_get_str (dict, "source", &filename);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch"
+                                " filename from dict.");
+                        *op_errstr = gf_strdup ("command unsuccessful");
+                        goto out;
+                }
+                snprintf (abs_filename, sizeof(abs_filename),
+                          "%s/%s", priv->workdir, filename);
+
+                ret = lstat (abs_filename, &stbuf);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Source file"
+                                  " does not exist in %s", priv->workdir);
+                        *op_errstr = gf_strdup (errmsg);
+                        goto out;
+                }
+
+                if (!S_ISREG(stbuf.st_mode)) {
+                        snprintf (errmsg, sizeof (errmsg), "Source file"
+                                 " is not a regular file.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_get_statefile_name (glusterd_volinfo_t *volinfo, char *slave,
+                             char *conf_path, char **statefile)
+{
+        glusterd_conf_t *priv = NULL;
+        int              ret = -1;
+        char            *master    = NULL;
+        char            *buf       = NULL;
+        dict_t          *confd = NULL;
+        char            *confpath = NULL;
+        char             conf_buf[PATH_MAX] = "";
+        struct stat      stbuf = {0,};
+
+        GF_ASSERT (THIS);
+        GF_ASSERT (THIS->private);
+        GF_ASSERT (volinfo);
+
+        master = volinfo->volname;
+
+        confd = dict_new ();
+        if (!confd) {
+                gf_log ("", GF_LOG_ERROR, "Unable to create new dict");
+                goto out;
+        }
+
+        priv = THIS->private;
+
+        ret = lstat (conf_path, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_INFO, "Using passed config template(%s).",
+                        conf_path);
+                confpath = conf_path;
+        } else {
+                ret = snprintf (conf_buf, sizeof(conf_buf) - 1,
+                                "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
+                conf_buf[ret] = '\0';
+                confpath = conf_buf;
+                gf_log ("", GF_LOG_INFO, "Using default config template(%s).",
+                        confpath);
+        }
+
+        ret = glusterd_gsync_get_config (master, slave, confpath,
+                                         confd);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data"
+                        "for %s(master), %s(slave)", master, slave);
+                goto out;
+
+        }
+
+        ret = dict_get_param (confd, "state_file", &buf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get state_file's name.");
+                goto out;
+        }
+
+        *statefile = gf_strdup(buf);
+        if (!*statefile) {
+                gf_log ("", GF_LOG_ERROR, "Unable to gf_strdup.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+ out:
+        if (confd)
+                dict_destroy (confd);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
+        return ret;
+}
+
+static int
+glusterd_create_status_file (char *master, char *slave, char *slave_ip,
+                             char *slave_vol, char *status)
+{
+        int                ret    = -1;
+        runner_t           runner = {0,};
+        glusterd_conf_t   *priv   = NULL;
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                goto out;
+        }
+
+        if (!status) {
+                gf_log ("", GF_LOG_ERROR, "Status Empty");
+                goto out;
+        }
+        gf_log ("", GF_LOG_DEBUG, "slave = %s", slave);
+
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--create",
+                         status, "-c", NULL);
+        runner_argprintf (&runner, "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                          priv->workdir, master, slave_ip, slave_vol);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, NULL);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Creating status file failed.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_verify_slave (char *volname, char *slave_ip, char *slave,
+                       char **op_errstr, gf_boolean_t *is_force_blocker)
+{
+        int32_t          ret                     = -1;
+        runner_t         runner                  = {0,};
+        char             log_file_path[PATH_MAX] = "";
+        char             buf[PATH_MAX]           = "";
+        char            *tmp                     = NULL;
+        char            *save_ptr                = NULL;
+        glusterd_conf_t *priv                    = NULL;
+
+        GF_ASSERT (volname);
+        GF_ASSERT (slave_ip);
+        GF_ASSERT (slave);
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                goto out;
+        }
+
+        snprintf (log_file_path, sizeof(log_file_path),
+                  DEFAULT_LOG_FILE_DIRECTORY"/create_verify_log");
+
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gverify.sh", NULL);
+        runner_argprintf (&runner, "%s", volname);
+        runner_argprintf (&runner, "%s", slave_ip);
+        runner_argprintf (&runner, "%s", slave);
+        runner_argprintf (&runner, "%s", log_file_path);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Not a valid slave");
+                ret = glusterd_gsync_read_frm_status (log_file_path,
+                                                      buf, sizeof(buf));
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to read from %s",
+                                log_file_path);
+                        goto out;
+                }
+
+                /* Tokenize the error message from gverify.sh to figure out
+                 * if the error is a force blocker or not. */
+                tmp = strtok_r (buf, "|", &save_ptr);
+                if (!strcmp (tmp, "FORCE_BLOCKER"))
+                        *is_force_blocker = 1;
+                else {
+                        /* No FORCE_BLOCKER flag present so all that is
+                         * present is the error message. */
+                        *is_force_blocker = 0;
+                        if (tmp)
+                                *op_errstr = gf_strdup (tmp);
+                        ret = -1;
+                        goto out;
+                }
+
+                /* Copy rest of the error message to op_errstr */
+                tmp = strtok_r (NULL, "|", &save_ptr);
+                if (tmp)
+                        *op_errstr = gf_strdup (tmp);
+                ret = -1;
                 goto out;
         }
+        ret = 0;
+out:
+        unlink (log_file_path);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mountbroker_check (char **slave_ip, char **op_errstr)
+{
+        int   ret             = -1;
+        char *tmp             = NULL;
+        char *save_ptr        = NULL;
+        char *username        = NULL;
+        char *host            = NULL;
+        char errmsg[PATH_MAX] = "";
+
+        GF_ASSERT (slave_ip);
+        GF_ASSERT (*slave_ip);
+
+        /* Checking if hostname has user specified */
+        host = strstr (*slave_ip, "@");
+        if (!host) {
+                gf_log ("", GF_LOG_DEBUG, "No username provided.");
+                ret = 0;
+                goto out;
+        } else {
+                /* Moving the host past the '@' and checking if the
+                 * actual hostname also has '@' */
+                host++;
+                if (strstr (host, "@")) {
+                        gf_log ("", GF_LOG_DEBUG, "host = %s", host);
+                        ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                        "Invalid Hostname (%s).", host);
+                        errmsg[ret] = '\0';
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        if (op_errstr)
+                                *op_errstr = gf_strdup (errmsg);
+                        goto out;
+                }
 
+                /* Fetching the username and hostname
+                 * and checking if the username is non-root */
+                username = strtok_r (*slave_ip, "@", &save_ptr);
+                tmp = strtok_r (NULL, "@", &save_ptr);
+                if (strcmp (username, "root")) {
+                        ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                        "Non-root username (%s@%s) not allowed.",
+                                        username, tmp);
+                        errmsg[ret] = '\0';
+                        if (op_errstr)
+                                *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR,
+                                "Non-Root username not allowed.");
+                        ret = -1;
+                        goto out;
+                }
+
+                *slave_ip = gf_strdup (tmp);
+                if (!*slave_ip) {
+                        gf_log ("", GF_LOG_ERROR, "Out of memory");
+                        ret = -1;
+                        goto out;
+                }
+        }
 
         ret = 0;
 out:
@@ -1020,17 +1838,257 @@ out:
 }
 
 int
+glusterd_op_stage_gsync_create (dict_t *dict, char **op_errstr)
+{
+        char               *down_peerstr              = NULL;
+        char               *slave                     = NULL;
+        char               *volname                   = NULL;
+        char               *host_uuid                 = NULL;
+        char               *statefile                 = NULL;
+        char               *slave_ip                  = NULL;
+        char               *slave_vol                 = NULL;
+        char               *conf_path                 = NULL;
+        char                errmsg[PATH_MAX]          = "";
+        char                common_pem_file[PATH_MAX] = "";
+        char                hook_script[PATH_MAX]     = "";
+        char                uuid_str [64]             = "";
+        int                 ret                       = -1;
+        int                 is_pem_push               = -1;
+        gf_boolean_t        is_force                  = -1;
+        gf_boolean_t        is_force_blocker          = -1;
+        gf_boolean_t        exists                    = _gf_false;
+        glusterd_conf_t    *conf                      = NULL;
+        glusterd_volinfo_t *volinfo                   = NULL;
+        struct stat         stbuf                     = {0,};
+        xlator_t           *this                      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = glusterd_op_gsync_args_get (dict, op_errstr, &volname,
+                                          &slave, &host_uuid);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch arguments");
+                gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+                return -1;
+        }
+
+        if (conf->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if ((ret) || (!exists)) {
+                gf_log ("", GF_LOG_WARNING, "volume name does not exist");
+                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
+                          " exist", volname);
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+                return -1;
+        }
+
+        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_ip,
+                                                   &slave_vol, &conf_path,
+                                                   op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave or confpath details.");
+                ret = -1;
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = glusterd_are_vol_all_peers_up (volinfo,
+                                                     &conf->peers,
+                                                     &down_peerstr);
+                if ((ret == _gf_false) && !is_force) {
+                        snprintf (errmsg, sizeof (errmsg), "Peer %s,"
+                                  " which is a part of %s volume, is"
+                                  " down. Please bring up the peer and"
+                                  " retry.", down_peerstr,
+                                  volinfo->volname);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        *op_errstr = gf_strdup (errmsg);
+                        GF_FREE (down_peerstr);
+                        down_peerstr = NULL;
+                        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+                        return -1;
+                } else if (ret == _gf_false) {
+                        gf_log ("", GF_LOG_INFO, "Peer %s,"
+                                " which is a part of %s volume, is"
+                                " down. Force creating geo-rep session."
+                                " On bringing up the peer, re-run"
+                                " \"gluster system:: execute"
+                                " gsec_create\" and \"gluster volume"
+                                " geo-replication %s %s create push-pem"
+                                " force\"", down_peerstr, volinfo->volname,
+                                volinfo->volname, slave);
+                }
+
+                /* Checking if slave host is pingable, has proper passwordless
+                 * ssh login setup, slave volume is created, slave vol is empty,
+                 * and if it has enough memory and bypass in case of force if
+                 * the error is not a force blocker */
+                ret = glusterd_verify_slave (volname, slave_ip, slave_vol,
+                                             op_errstr, &is_force_blocker);
+                if (ret) {
+                        if (is_force && !is_force_blocker) {
+                                gf_log ("", GF_LOG_INFO, "%s is not a valid slave"
+                                        " volume. Error: %s. Force creating geo-rep"
+                                        " session.", slave, *op_errstr);
+                        } else {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "%s is not a valid slave volume. Error: %s",
+                                        slave, *op_errstr);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                ret = dict_get_int32 (dict, "push_pem", &is_pem_push);
+                if (!ret && is_pem_push) {
+                        ret = snprintf (common_pem_file,
+                                        sizeof(common_pem_file) - 1,
+                                        "%s"GLUSTERD_COMMON_PEM_PUB_FILE,
+                                        conf->workdir);
+                        common_pem_file[ret] = '\0';
+
+                        ret = snprintf (hook_script, sizeof(hook_script) - 1,
+                                        "%s"GLUSTERD_CREATE_HOOK_SCRIPT,
+                                        conf->workdir);
+                        hook_script[ret] = '\0';
+
+                        ret = lstat (common_pem_file, &stbuf);
+                        if (ret) {
+                                snprintf (errmsg, sizeof (errmsg), "%s"
+                                          " required for push-pem is"
+                                          " not present. Please run"
+                                          " \"gluster system:: execute"
+                                          " gsec_create\"", common_pem_file);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+
+                        ret = lstat (hook_script, &stbuf);
+                        if (ret) {
+                                snprintf (errmsg, sizeof (errmsg),
+                                          "The hook-script (%s) required "
+                                          "for push-pem is not present. "
+                                          "Please install the hook-script "
+                                          "and retry", hook_script);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+
+                        if (!S_ISREG(stbuf.st_mode)) {
+                                snprintf (errmsg, sizeof (errmsg), "%s"
+                                          " required for push-pem is"
+                                          " not a regular file. Please run"
+                                          " \"gluster system:: execute"
+                                          " gsec_create\"", common_pem_file);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
+        ret = glusterd_get_statefile_name (volinfo, slave, conf_path, &statefile);
+        if (ret) {
+                if (!strstr(slave, "::"))
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "%s is not a valid slave url.", slave);
+                else
+                        snprintf (errmsg, sizeof (errmsg), "Please check gsync "
+                                  "config file. Unable to get statefile's name");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "statefile", statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store statefile path");
+                goto out;
+        }
+
+        ret = lstat (statefile, &stbuf);
+        if (!ret && !is_force) {
+                snprintf (errmsg, sizeof (errmsg), "Session between %s"
+                          " and %s is already created.",
+                          volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        } else if (!ret)
+                gf_log ("", GF_LOG_INFO, "Session between %s"
+                        " and %s is already created. Force"
+                        " creating again.", volinfo->volname, slave);
+
+        ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to spawn gsyncd.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        if (ret && errmsg[0] != '\0')
+                *op_errstr = gf_strdup (errmsg);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
 glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
 {
         int                     ret     = 0;
         int                     type    = 0;
         char                    *volname = NULL;
         char                    *slave   = NULL;
+        char                    *slave_ip = NULL;
+        char                    *slave_vol = NULL;
+        char                    *down_peerstr = NULL;
+        char                    *statefile    = NULL;
+        char                    *path_list    = NULL;
+        char                    *conf_path    = NULL;
         gf_boolean_t            exists   = _gf_false;
         glusterd_volinfo_t      *volinfo = NULL;
         char                    errmsg[PATH_MAX] = {0,};
         dict_t                  *ctx = NULL;
+        gf_boolean_t            is_force = 0;
+        gf_boolean_t            is_force_blocker = -1;
+        gf_boolean_t            is_running = _gf_false;
+        uuid_t                  uuid = {0};
+        char                    uuid_str [64] = {0};
+        char                    *host_uuid = NULL;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *conf = NULL;
+        struct stat             stbuf = {0,};
 
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret < 0) {
@@ -1039,25 +2097,26 @@ glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        switch (type) {
-        case GF_GSYNC_OPTION_TYPE_STATUS:
+        if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
                 ret = glusterd_verify_gsync_status_opts (dict, op_errstr);
-
                 goto out;
-        case GF_GSYNC_OPTION_TYPE_CONFIG:
-                ret = gsync_verify_config_options (dict, op_errstr);
+        }
 
+        ret = glusterd_op_gsync_args_get (dict, op_errstr,
+                                          &volname, &slave, &host_uuid);
+        if (ret)
                 goto out;
 
-        case GF_GSYNC_OPTION_TYPE_ROTATE:
-                /* checks same as status mode */
-                ret = glusterd_verify_gsync_status_opts(dict, op_errstr);
-                goto out;
-        }
+        uuid_utoa_r (MY_UUID, uuid_str);
 
-        ret = glusterd_op_gsync_args_get (dict, op_errstr, &volname, &slave);
-        if (ret)
+        if (conf->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+               *op_errstr = gf_strdup (errmsg);
+                ret = -1;
                 goto out;
+        }
 
         exists = glusterd_check_volume_exists (volname);
         ret = glusterd_volinfo_find (volname, &volinfo);
@@ -1070,19 +2129,115 @@ glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_ip,
+                                                   &slave_vol, &conf_path,
+                                                   op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave or confpath details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_statefile_name (volinfo, slave, conf_path, &statefile);
+        if (ret) {
+                /* Checking if slave host is pingable, has proper passwordless
+                 * ssh login setup */
+                ret = glusterd_verify_slave (volname, slave_ip, slave_vol,
+                                             op_errstr, &is_force_blocker);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "%s is not a valid slave volume. Error: %s",
+                                slave, *op_errstr);
+                        goto out;
+                }
+
+                if (!strstr(slave, "::"))
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "%s is not a valid slave url.", slave);
+                else
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "Unable to get statefile's name");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "statefile", statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store statefile path");
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        /* Allowing stop force to bypass the statefile check
+         * as this command acts as a fail safe method to stop geo-rep
+         * session. */
+        if ((type == GF_GSYNC_OPTION_TYPE_CONFIG) ||
+            ((type == GF_GSYNC_OPTION_TYPE_STOP) && !is_force) ||
+            (type == GF_GSYNC_OPTION_TYPE_DELETE)) {
+                ret = lstat (statefile, &stbuf);
+                if (ret) {
+                        snprintf (errmsg, sizeof(errmsg), "Geo-replication"
+                                  " session between %s and %s does not exist.",
+                                  volinfo->volname, slave);
+                        gf_log ("", GF_LOG_ERROR, "%s. statefile = %s",
+                                errmsg, statefile);
+                        *op_errstr = gf_strdup (errmsg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Check if all peers that are a part of the volume are up or not */
+        if ((type == GF_GSYNC_OPTION_TYPE_DELETE) ||
+            ((type == GF_GSYNC_OPTION_TYPE_STOP) && !is_force)) {
+                if (!strcmp (uuid_str, host_uuid)) {
+                        ret = glusterd_are_vol_all_peers_up (volinfo,
+                                                             &conf->peers,
+                                                             &down_peerstr);
+                        if (ret == _gf_false) {
+                                snprintf (errmsg, sizeof (errmsg), "Peer %s,"
+                                          " which is a part of %s volume, is"
+                                          " down. Please bring up the peer and"
+                                          " retry.", down_peerstr,
+                                          volinfo->volname);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                GF_FREE (down_peerstr);
+                                down_peerstr = NULL;
+                                goto out;
+                        }
+                }
+        }
+
         switch (type) {
         case GF_GSYNC_OPTION_TYPE_START:
+                /* don't attempt to start gsync if replace-brick is
+                 * in progress */
+                if (glusterd_is_rb_ongoing (volinfo)) {
+                        snprintf (errmsg, sizeof(errmsg), "replace-brick is in"
+                                   " progress, not starting geo-replication");
+                        *op_errstr = gf_strdup (errmsg);
+                        ret = -1;
+                        goto out;
+                }
+
                 ret = glusterd_op_verify_gsync_start_options (volinfo, slave,
-                                                              op_errstr);
+                                                              conf_path, statefile,
+                                                              op_errstr, is_force);
                 if (ret)
                         goto out;
                 ctx = glusterd_op_get_ctx();
                 if (ctx) {
-                        /*gsyncd does a fuse mount to start the geo-rep session*/
+                        /* gsyncd does a fuse mount to start
+                         * the geo-rep session */
                         if (!glusterd_is_fuse_available ()) {
-                                gf_log ("glusterd", GF_LOG_ERROR, "Unable to open"
-                                        " /dev/fuse (%s), geo-replication start"
-                                        " failed", strerror (errno));
+                                gf_log ("glusterd", GF_LOG_ERROR, "Unable to "
+                                        "open /dev/fuse (%s), geo-replication "
+                                        "start failed", strerror (errno));
                                 snprintf (errmsg, sizeof(errmsg),
                                           "fuse unvailable");
                                 *op_errstr = gf_strdup (errmsg);
@@ -1093,17 +2248,72 @@ glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
                 break;
 
         case GF_GSYNC_OPTION_TYPE_STOP:
-                ret = glusterd_op_verify_gsync_running (volinfo, slave,
-                                                        op_errstr);
+                if (!is_force) {
+                        ret = glusterd_op_verify_gsync_running (volinfo, slave,
+                                                                conf_path,
+                                                                op_errstr);
+                        if (ret) {
+                                ret = glusterd_get_local_brickpaths (volinfo,
+                                                                     &path_list);
+                                if (path_list)
+                                        ret = -1;
+                        }
+                }
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+                ret = gsync_verify_config_options (dict, op_errstr, volname);
+                goto out;
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_DELETE:
+                /* Check if the gsync session is still running
+                 * If so ask the user to stop geo-replication first.*/
+                ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
+                if (ret) {
+                        snprintf (errmsg, sizeof(errmsg), "Geo-replication"
+                                  " session between %s and %s does not exist.",
+                                  volinfo->volname, slave);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        *op_errstr = gf_strdup (errmsg);
+                        ret = -1;
+                        goto out;
+                } else {
+                        ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                                  slave, conf_path,
+                                                                  &is_running);
+                        if (_gf_true == is_running) {
+                                snprintf (errmsg, sizeof (errmsg), GEOREP
+                                          " session between %s & %s is "
+                                          "still active. Please stop the "
+                                          "session and retry.",
+                                          volinfo->volname, slave);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "Unable to spawn gsyncd");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                }
+
                 break;
         }
 
 out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-int
-stop_gsync (char *master, char *slave, char **msg)
+static int
+stop_gsync (char *master, char *slave, char **msg,
+            char *conf_path, gf_boolean_t is_force)
 {
         int32_t         ret     = 0;
         int             pfd     = -1;
@@ -1115,23 +2325,23 @@ stop_gsync (char *master, char *slave, char **msg)
         GF_ASSERT (THIS);
         GF_ASSERT (THIS->private);
 
-        pfd = gsyncd_getpidfile (master, slave, pidfile);
-        if (pfd == -2) {
+        pfd = gsyncd_getpidfile (master, slave, pidfile, conf_path);
+        if (pfd == -2 && !is_force) {
                 gf_log ("", GF_LOG_ERROR, GEOREP" stop validation "
                         " failed for %s & %s", master, slave);
                 ret = -1;
                 goto out;
         }
-        if (gsync_status_byfd (pfd) == -1) {
+        if (gsync_status_byfd (pfd) == -1 && !is_force) {
                 gf_log ("", GF_LOG_ERROR, "gsyncd b/w %s & %s is not"
                         " running", master, slave);
-                if (msg)
-                        *msg = gf_strdup ("Warning: "GEOREP" session was in "
-                                          "corrupt  state");
                 /* monitor gsyncd already dead */
                 goto out;
         }
 
+        if (pfd < 0)
+                goto out;
+
         ret = read (pfd, buf, 1024);
         if (ret > 0) {
                 pid = strtol (buf, NULL, 10);
@@ -1158,17 +2368,17 @@ stop_gsync (char *master, char *slave, char **msg)
         ret = 0;
 
 out:
-        close (pfd);
+        sys_close (pfd);
+
+        if (is_force)
+                ret = 0;
         return ret;
 }
 
-int
-glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
-                                      dict_t *resp_dict);
-
-int
+static int
 glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
-                          dict_t *dict, dict_t *resp_dict, char **op_errstr)
+                          char *path_list, dict_t *dict,
+                          dict_t *resp_dict, char **op_errstr)
 {
         int32_t         ret     = -1;
         char            *op_name = NULL;
@@ -1177,6 +2387,10 @@ glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
         glusterd_conf_t *priv   = NULL;
         char            *subop  = NULL;
         char            *master = NULL;
+        char            *conf_path = NULL;
+        char            *slave_ip  = NULL;
+        char            *slave_vol = NULL;
+        struct stat      stbuf     = {0, };
 
         GF_ASSERT (slave);
         GF_ASSERT (op_errstr);
@@ -1211,10 +2425,17 @@ glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
                 goto out;
         }
 
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
+
         master = "";
         runinit (&runner);
         runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, priv->workdir);
+        runner_argprintf (&runner, "%s", conf_path);
         if (volinfo) {
                 master = volinfo->volname;
                 runner_argprintf (&runner, ":%s", master);
@@ -1224,7 +2445,9 @@ glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
         runner_add_arg (&runner, op_name);
         if (op_value)
                 runner_add_arg (&runner, op_value);
+        synclock_unlock (&priv->big_lock);
         ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
         if (ret) {
                 gf_log ("", GF_LOG_WARNING, "gsyncd failed to "
                         "%s %s option for %s %s peers",
@@ -1235,13 +2458,46 @@ glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
 
                 goto out;
         }
+
+        if (!strcmp (op_name, "state_file")) {
+
+                ret = lstat (op_value, &stbuf);
+                if (ret) {
+                        ret = dict_get_str (dict, "slave_ip", &slave_ip);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to fetch slave IP.");
+                                goto out;
+                        }
+
+                        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to fetch slave volume name.");
+                                goto out;
+                        }
+
+                        ret = glusterd_create_status_file (volinfo->volname, slave,
+                                                           slave_ip, slave_vol,
+                                                           "Switching Status File");
+                        if (ret || lstat (op_value, &stbuf)) {
+                                gf_log ("", GF_LOG_ERROR, "Unable to create %s"
+                                        ". Error : %s", op_value,
+                                        strerror (errno));
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
         ret = 0;
         gf_asprintf (op_errstr, "config-%s successful", subop);
 
 out:
         if (!ret && volinfo) {
                 ret = glusterd_check_restart_gsync_session (volinfo, slave,
-                                                            resp_dict);
+                                                            resp_dict, path_list,
+                                                            conf_path, 0);
                 if (ret)
                         *op_errstr = gf_strdup ("internal error");
         }
@@ -1250,74 +2506,233 @@ out:
         return ret;
 }
 
-int
-glusterd_gsync_read_frm_status (char *path, char *data)
+static int
+glusterd_gsync_read_frm_status (char *path, char *buf, size_t blen)
 {
         int                 ret = 0;
-        FILE               *status_file = NULL;
+        int           status_fd = -1;
 
         GF_ASSERT (path);
-        GF_ASSERT (data);
-        status_file = fopen (path, "r");
-        if (status_file  == NULL) {
-                gf_log ("", GF_LOG_WARNING, "Unable to read gsyncd status"
+        GF_ASSERT (buf);
+        status_fd = open (path, O_RDONLY);
+        if (status_fd == -1) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read gsyncd status"
                         " file");
                 return -1;
         }
-        ret = fread (data, PATH_MAX, 1, status_file);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "Status file of gsyncd is corrupt");
+        ret = read (status_fd, buf, blen - 1);
+        if (ret > 0) {
+                size_t len = strnlen (buf, ret);
+                /* Ensure there is a NUL byte and that it's not the first.  */
+                if (len == 0 || len == blen - 1) {
+                        ret = -1;
+                } else {
+                        char *p = buf + len - 1;
+                        while (isspace (*p))
+                                *p-- = '\0';
+                        ret = 0;
+                }
+        } else if (ret < 0)
+                gf_log ("", GF_LOG_ERROR, "Status file of gsyncd is corrupt");
+
+        close (status_fd);
+        return ret;
+}
+
+static int
+glusterd_gsync_fetch_status_extra (char *path, char *buf, size_t blen)
+{
+        char sockpath[PATH_MAX] = {0,};
+        struct sockaddr_un   sa = {0,};
+        size_t                l = 0;
+        int                   s = -1;
+        struct pollfd       pfd = {0,};
+        int                 ret = 0;
+
+        l = strlen (buf);
+        /* seek to end of data in buf */
+        buf += l;
+        blen -= l;
+
+        glusterd_set_socket_filepath (path, sockpath, sizeof (sockpath));
+
+        strncpy(sa.sun_path, sockpath, sizeof(sa.sun_path));
+        if (sa.sun_path[sizeof (sa.sun_path) - 1])
                 return -1;
+        sa.sun_family = AF_UNIX;
+
+        s = socket(AF_UNIX, SOCK_STREAM, 0);
+        if (s == -1)
+                return -1;
+        ret = fcntl (s, F_GETFL);
+        if (ret != -1)
+                ret = fcntl (s, F_SETFL, ret | O_NONBLOCK);
+        if (ret == -1)
+                goto out;
+
+        ret = connect (s, (struct sockaddr *)&sa, sizeof (sa));
+        if (ret == -1)
+                goto out;
+        pfd.fd = s;
+        pfd.events = POLLIN;
+        /* we don't want to hang on gsyncd */
+        if (poll (&pfd, 1, 5000) < 1 ||
+            !(pfd.revents & POLLIN)) {
+                ret = -1;
+                goto out;
         }
+        ret = read(s, buf, blen);
+        /* we expect a terminating 0 byte */
+        if (ret == 0 || (ret > 0 && buf[ret - 1]))
+                ret = -1;
+        if (ret > 0)
+                ret = 0;
+
+ out:
+        close (s);
+        return ret;
+}
 
-        data[strlen(data)-1] = '\0';
+static int
+dict_get_param (dict_t *dict, char *key, char **param)
+{
+        char  *dk = NULL;
+        char   *s = NULL;
+        char    x = '\0';
+        int   ret = 0;
 
-        return 0;
+        if (dict_get_str (dict, key, param) == 0)
+                return 0;
+
+        dk = gf_strdup (key);
+        if (!key)
+                return -1;
+
+        s = strpbrk (dk, "-_");
+        if (!s)
+                return -1;
+        x = (*s == '-') ? '_' : '-';
+        *s++ = x;
+        while ((s = strpbrk (s, "-_")))
+                *s++ = x;
+
+        ret = dict_get_str (dict, dk, param);
+
+        GF_FREE (dk);
+        return ret;
 }
 
-int
-glusterd_read_status_file (char *master, char *slave,
-                           dict_t *dict)
+static int
+glusterd_read_status_file (glusterd_volinfo_t *volinfo, char *slave,
+                           char *conf_path, dict_t *dict, char *node)
 {
-        glusterd_conf_t  *priv = NULL;
+        glusterd_conf_t *priv = NULL;
         int              ret = 0;
-        char             statusfile[PATH_MAX] = {0, };
-        char             buff[PATH_MAX] = {0, };
-        char             mst[PATH_MAX] = {0, };
-        char             slv[PATH_MAX] = {0, };
-        char             sts[PATH_MAX] = {0, };
+        char            *statefile = NULL;
+        char            *master    = NULL;
+        char             buf[1024] = "defunct";
+        char             nds[1024] = {0, };
+        char             mst[1024] = {0, };
+        char             slv[1024] = {0, };
+        char             sts[1024] = {0, };
+        char            *bufp = NULL;
+        dict_t          *confd = NULL;
         int              gsync_count = 0;
         int              status = 0;
+        char *dyn_node = NULL;
+        char *path_list = NULL;
 
         GF_ASSERT (THIS);
         GF_ASSERT (THIS->private);
+        GF_ASSERT (volinfo);
+
+        master = volinfo->volname;
+
+        confd = dict_new ();
+        if (!dict) {
+                gf_log ("", GF_LOG_ERROR, "Not able to create dict.");
+                return -1;
+        }
 
         priv = THIS->private;
-        ret = glusterd_gsync_get_param_file (statusfile, "state", master,
-                                             slave, priv->workdir);
+
+        ret = glusterd_gsync_get_config (master, slave, conf_path,
+                                         confd);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Unable to get the name of status"
-                        "file for %s(master), %s(slave)", master, slave);
-                goto out;
+                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data"
+                        "for %s(master), %s(slave)", master, slave);
+                goto done;
 
         }
 
-        ret = gsync_status (master, slave, &status);
-        if (ret == 0 && status == -1) {
-                strncpy (buff, "corrupt", sizeof (buff));
+        ret = dict_get_param (confd, "state_file", &statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get state_file's name "
+                        "for %s(master), %s(slave). Please check gsync "
+                        "config file.", master, slave);
                 goto done;
-        } else if (ret == -1)
-                goto out;
-
-        ret = glusterd_gsync_read_frm_status (statusfile, buff);
+        }
+        ret = glusterd_gsync_read_frm_status (statefile, buf, sizeof (buf));
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Unable to read the status"
+                gf_log ("", GF_LOG_ERROR, "Unable to read the status"
                         "file for %s(master), %s(slave)", master, slave);
-                goto out;
+                strncpy (buf, "defunct", sizeof (buf));
+                goto done;
+        }
 
+        ret = gsync_status (master, slave, conf_path, &status);
+        if (ret == 0 && status == -1) {
+                if ((strcmp (buf, "Not Started")) &&
+                    (strcmp (buf, "Stopped")))
+                    strncpy (buf, "defunct", sizeof (buf));
+                goto done;
+        } else if (ret == -1) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get gsync status");
+                goto done;
+        }
+
+        if (strcmp (buf, "Stable") != 0)
+                goto done;
+
+        ret = dict_get_param (confd, "state_socket_unencoded", &statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get state_socket_unencoded"
+                        " filepath. Please check gsync config file.");
+                goto done;
+        }
+        ret = glusterd_gsync_fetch_status_extra (statefile, buf, sizeof (buf));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch extra status"
+                        "for %s(master), %s(slave)", master, slave);
+                /* there is a slight chance that this occurs due to race
+                 * -- in that case, the following options all seem bad:
+                 *
+                 * - suppress irregurlar behavior by just leaving status
+                 *   on "OK"
+                 * - freak out users with a misleading "defunct"
+                 * - overload the meaning of the regular error signal
+                 *   mechanism of gsyncd, that is, when status is "faulty"
+                 *
+                 * -- so we just come up with something new...
+                 */
+                strncpy (buf, "N/A", sizeof (buf));
+                goto done;
         }
 
  done:
+        if ((!strcmp (buf, "defunct")) ||
+            (!strcmp (buf, "Not Started")) ||
+            (!strcmp (buf, "Stopped"))) {
+                ret = glusterd_get_local_brickpaths (volinfo, &path_list);
+                if (!path_list) {
+                        gf_log ("", GF_LOG_DEBUG, "This node not being part of"
+                                " volume should not be running gsyncd. Hence"
+                                " shouldn't display status for this node.");
+                        ret = 0;
+                        goto out;
+                }
+        }
+
         ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
 
         if (ret)
@@ -1325,39 +2740,65 @@ glusterd_read_status_file (char *master, char *slave,
         else
                 gsync_count++;
 
+        (void) snprintf (nds, sizeof (nds), "node%d", gsync_count);
+        dyn_node = gf_strdup (node);
+        if (!dyn_node)
+                goto out;
+        ret = dict_set_dynstr (dict, nds, dyn_node);
+        if (ret) {
+                GF_FREE (dyn_node);
+                goto out;
+        }
+
         snprintf (mst, sizeof (mst), "master%d", gsync_count);
-        ret = dict_set_dynstr (dict, mst, gf_strdup (master));
-        if (ret)
+        master = gf_strdup (master);
+        if (!master)
                 goto out;
+        ret = dict_set_dynstr (dict, mst, master);
+        if (ret) {
+                GF_FREE (master);
+                goto out;
+        }
 
         snprintf (slv, sizeof (slv), "slave%d", gsync_count);
-        ret = dict_set_dynstr (dict, slv, gf_strdup (slave));
-        if (ret)
+        slave = gf_strdup (slave);
+        if (!slave)
+                goto out;
+        ret = dict_set_dynstr (dict, slv, slave);
+        if (ret) {
+                GF_FREE (slave);
                 goto out;
+        }
 
         snprintf (sts, sizeof (slv), "status%d", gsync_count);
-        ret = dict_set_dynstr (dict, sts, gf_strdup (buff));
-        if (ret)
+        bufp = gf_strdup (buf);
+        if (!bufp)
                 goto out;
+        ret = dict_set_dynstr (dict, sts, bufp);
+        if (ret) {
+                GF_FREE (bufp);
+                goto out;
+        }
         ret = dict_set_int32 (dict, "gsync-count", gsync_count);
         if (ret)
                 goto out;
 
-        ret = 0;
  out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
-        return ret;
+        dict_destroy (confd);
+
+        return 0;
 }
 
 int
 glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
-                                      dict_t *resp_dict)
+                                      dict_t *resp_dict, char *path_list,
+                                      char *conf_path, gf_boolean_t is_force)
 {
 
         int                    ret = 0;
-        uuid_t                 uuid = {0, };
         glusterd_conf_t        *priv = NULL;
         char                   *status_msg = NULL;
+        gf_boolean_t           is_running = _gf_false;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (slave);
@@ -1366,26 +2807,30 @@ glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
 
         priv = THIS->private;
 
-        if (glusterd_gsync_get_uuid (slave, volinfo, uuid))
-                /* session does not exist, nothing to do */
+        ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                  slave, conf_path,
+                                                  &is_running);
+        if (!ret && (_gf_true != is_running))
+                /* gsynd not running, nothing to do */
                 goto out;
-        if (uuid_compare (priv->uuid, uuid) == 0) {
-                ret = stop_gsync (volinfo->volname, slave, &status_msg);
-                if (ret == 0 && status_msg)
-                        ret = dict_set_str (resp_dict, "gsync-status",
-                                            status_msg);
-                if (ret == 0)
-                        ret = glusterd_start_gsync (volinfo, slave,
-                                                    uuid_utoa(priv->uuid), NULL);
-        }
+
+        ret = stop_gsync (volinfo->volname, slave, &status_msg,
+                          conf_path, is_force);
+        if (ret == 0 && status_msg)
+                ret = dict_set_str (resp_dict, "gsync-status",
+                                    status_msg);
+        if (ret == 0)
+                ret = glusterd_start_gsync (volinfo, slave, path_list,
+                                            conf_path, uuid_utoa(MY_UUID),
+                                            NULL);
 
  out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-int32_t
-glusterd_marker_create_volfile (glusterd_volinfo_t *volinfo)
+static int32_t
+glusterd_marker_changelog_create_volfile (glusterd_volinfo_t *volinfo)
 {
         int32_t          ret     = 0;
 
@@ -1408,58 +2853,83 @@ out:
         return ret;
 }
 
-int
-glusterd_set_marker_gsync (glusterd_volinfo_t *volinfo)
+static int
+glusterd_set_gsync_knob (glusterd_volinfo_t *volinfo, char *key, int *vc)
 {
-        int                      ret     = -1;
-        int                      marker_set = _gf_false;
-        char                    *gsync_status = NULL;
+        int   ret          = -1;
+        int   conf_enabled = _gf_false;
+        char *knob_on      = NULL;
 
         GF_ASSERT (THIS);
         GF_ASSERT (THIS->private);
 
-        marker_set = glusterd_volinfo_get_boolean (volinfo, VKEY_MARKER_XTIME);
-        if (marker_set == -1) {
-                gf_log ("", GF_LOG_ERROR, "failed to get the marker status");
-                ret = -1;
+        conf_enabled = glusterd_volinfo_get_boolean (volinfo, key);
+        if (conf_enabled == -1) {
+                gf_log ("", GF_LOG_ERROR,
+                        "failed to get key %s from volinfo", key);
                 goto out;
         }
 
-        if (marker_set == _gf_false) {
-                gsync_status = gf_strdup ("on");
-                if (gsync_status == NULL) {
+        ret = 0;
+        if (conf_enabled == _gf_false) {
+                *vc = 1;
+                knob_on = gf_strdup ("on");
+                if (knob_on == NULL) {
                         ret = -1;
                         goto out;
                 }
 
                 ret = glusterd_gsync_volinfo_dict_set (volinfo,
-                                                       VKEY_MARKER_XTIME, gsync_status);
-                if (ret < 0)
-                        goto out;
-
-                ret = glusterd_marker_create_volfile (volinfo);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Setting dict failed");
-                        goto out;
-                }
+                                                       key, knob_on);
         }
-        ret = 0;
 
-out:
+ out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+static int
+glusterd_set_gsync_confs (glusterd_volinfo_t *volinfo)
+{
+        int ret             = -1;
+        int volfile_changed = 0;
 
+        ret = glusterd_set_gsync_knob (volinfo,
+                                       VKEY_MARKER_XTIME, &volfile_changed);
+        if (ret)
+                goto out;
 
+        /**
+         * enable ignore-pid-check blindly as it could be needed for
+         * cascading setups.
+         */
+        ret = glusterd_set_gsync_knob (volinfo, VKEY_MARKER_XTIME_FORCE,
+                                       &volfile_changed);
+        if (ret)
+                goto out;
 
-int
-glusterd_get_gsync_status_mst_slv( glusterd_volinfo_t *volinfo,
-                                   char *slave, dict_t *rsp_dict)
+        ret = glusterd_set_gsync_knob (volinfo,
+                                       VKEY_CHANGELOG, &volfile_changed);
+        if (ret)
+                goto out;
+
+        if (volfile_changed)
+                ret = glusterd_marker_changelog_create_volfile (volinfo);
+
+ out:
+        return ret;
+}
+
+static int
+glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
+                                   char *slave, char *conf_path,
+                                   dict_t *rsp_dict, char *node)
 {
+        char              *statefile = NULL;
         uuid_t             uuid = {0, };
         glusterd_conf_t    *priv = NULL;
         int                ret = 0;
+        struct stat        stbuf = {0, };
 
         GF_ASSERT (volinfo);
         GF_ASSERT (slave);
@@ -1469,24 +2939,45 @@ glusterd_get_gsync_status_mst_slv( glusterd_volinfo_t *volinfo,
         priv = THIS->private;
 
         ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-        if ((ret == 0) && (uuid_compare (priv->uuid, uuid) != 0))
-                goto out;
-
         if (ret) {
-                ret = 0;
                 gf_log ("", GF_LOG_INFO, "geo-replication status %s %s :"
                         "session is not active", volinfo->volname, slave);
-                goto out;
+
+                ret = glusterd_get_statefile_name (volinfo, slave,
+                                                   conf_path, &statefile);
+                if (ret) {
+                        if (!strstr(slave, "::"))
+                                gf_log ("", GF_LOG_INFO,
+                                        "%s is not a valid slave url.", slave);
+                        else
+                                gf_log ("", GF_LOG_INFO, "Unable to get"
+                                        " statefile's name");
+                        ret = 0;
+                        goto out;
+                }
+
+                ret = lstat (statefile, &stbuf);
+                if (ret) {
+                        gf_log ("", GF_LOG_INFO, "%s statefile not present.",
+                                statefile);
+                        ret = 0;
+                        goto out;
+                }
         }
 
-        ret = glusterd_read_status_file (volinfo->volname, slave, rsp_dict);
- out:
+        ret = glusterd_read_status_file (volinfo, slave, conf_path,
+                                         rsp_dict, node);
+out:
+        if (statefile)
+                GF_FREE (statefile);
+
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
 static int
-glusterd_get_gsync_status_mst (glusterd_volinfo_t *volinfo, dict_t *rsp_dict)
+glusterd_get_gsync_status_mst (glusterd_volinfo_t *volinfo, dict_t *rsp_dict,
+                               char *node)
 {
         glusterd_gsync_status_temp_t  param = {0, };
 
@@ -1494,13 +2985,14 @@ glusterd_get_gsync_status_mst (glusterd_volinfo_t *volinfo, dict_t *rsp_dict)
 
         param.rsp_dict = rsp_dict;
         param.volinfo = volinfo;
+        param.node = node;
         dict_foreach (volinfo->gsync_slaves, _get_status_mst_slv, &param);
 
         return 0;
 }
 
 static int
-glusterd_get_gsync_status_all ( dict_t *rsp_dict)
+glusterd_get_gsync_status_all (dict_t *rsp_dict, char *node)
 {
 
         int32_t                 ret = 0;
@@ -1513,7 +3005,7 @@ glusterd_get_gsync_status_all ( dict_t *rsp_dict)
         GF_ASSERT (priv);
 
         list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                ret = glusterd_get_gsync_status_mst (volinfo, rsp_dict);
+                ret = glusterd_get_gsync_status_mst (volinfo, rsp_dict, node);
                 if (ret)
                         goto out;
         }
@@ -1529,15 +3021,22 @@ glusterd_get_gsync_status (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
         char               *slave  = NULL;
         char               *volname = NULL;
+        char               *conf_path = NULL;
         char               errmsg[PATH_MAX] = {0, };
         gf_boolean_t       exists = _gf_false;
         glusterd_volinfo_t *volinfo = NULL;
         int                ret = 0;
+        char my_hostname[256] = {0,};
 
+        ret = gethostname(my_hostname, 256);
+        if (ret) {
+                /* stick to N/A */
+                (void) strcpy (my_hostname, "N/A");
+        }
 
         ret = dict_get_str (dict, "master", &volname);
         if (ret < 0){
-                ret = glusterd_get_gsync_status_all (rsp_dict);
+                ret = glusterd_get_gsync_status_all (rsp_dict, my_hostname);
                 goto out;
         }
 
@@ -1555,297 +3054,427 @@ glusterd_get_gsync_status (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 
         ret = dict_get_str (dict, "slave", &slave);
         if (ret < 0) {
-                ret = glusterd_get_gsync_status_mst (volinfo, rsp_dict);
+                ret = glusterd_get_gsync_status_mst (volinfo,
+                                                     rsp_dict, my_hostname);
                 goto out;
         }
 
-        ret = glusterd_get_gsync_status_mst_slv (volinfo, slave, rsp_dict);
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
+
+        ret = glusterd_get_gsync_status_mst_slv (volinfo, slave, conf_path,
+                                                 rsp_dict, my_hostname);
 
  out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-int
-glusterd_send_sigstop (pid_t pid)
-{
-        int ret = 0;
-        ret = kill (pid, SIGSTOP);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, GEOREP"failed to send SIGSTOP signal");
-        return ret;
-}
-
-int
-glusterd_send_sigcont (pid_t pid)
-{
-        int ret = 0;
-        ret = kill (pid, SIGCONT);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, GEOREP"failed to send SIGCONT signal");
-        return ret;
-}
-
-/*
- * Log rotations flow is something like this:
- * - Send SIGSTOP to process group (this will stop monitor/worker process
- *   and also the slave if it's local)
- * - Rotate log file for monitor/worker
- * - Rotate log file for slave if it's local
- * - Send SIGCONT to the process group. Monitor wakes up, kills the worker
- *   (this is done in the SIGCONT handler), which results in the termination
- *   of the slave (local/remote). After returning from signal handler,
- *   monitor detects absence of worker and starts it again, which in-turn
- *   starts the slave.
- */
-int
-glusterd_send_log_rotate_signal (pid_t pid, char *logfile1, char *logfile2)
+static int
+glusterd_gsync_delete (glusterd_volinfo_t *volinfo, char *slave, char *slave_ip,
+                       char *slave_vol, char *path_list, dict_t *dict,
+                       dict_t *resp_dict, char **op_errstr)
 {
-        int         ret         = 0;
-        struct stat stbuf       = {0,};
-        char rlogfile[PATH_MAX] = {0,};
-        time_t      rottime     = 0;
-
-        ret = glusterd_send_sigstop (-pid);
-        rottime = time (NULL);
+        int32_t         ret     = -1;
+        runner_t        runner    = {0,};
+        glusterd_conf_t *priv   = NULL;
+        char            *master = NULL;
+        char            *gl_workdir = NULL;
+        char             geo_rep_dir[PATH_MAX] = "";
+        char            *conf_path = NULL;
 
-        snprintf (rlogfile, sizeof (rlogfile), "%s.%"PRIu64, logfile1,
-                  (uint64_t) rottime);
-        ret = rename (logfile1, rlogfile);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "rename failed for geo-rep log file");
+        GF_ASSERT (slave);
+        GF_ASSERT (slave_ip);
+        GF_ASSERT (slave_vol);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (dict);
+        GF_ASSERT (resp_dict);
 
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
 
-        snprintf (rlogfile, sizeof (rlogfile), "%s.%"PRIu64, logfile2,
-                  (uint64_t) rottime);
-        ret = stat (logfile2, &stbuf);
+        ret = dict_get_str (dict, "conf_path", &conf_path);
         if (ret) {
-                if (errno != ENOENT)
-                        gf_log("", GF_LOG_ERROR, "stat failed for slave log"
-                               " file: %s", logfile2);
-                else {
-                        gf_log ("", GF_LOG_DEBUG, "Slave is not local, skipping rotation");
-                        ret = 0;
-                }
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
                 goto out;
         }
 
-        ret = rename (logfile2, rlogfile);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "rename failed for geo-rep slave log file");
-
- out:
-        ret = glusterd_send_sigcont (-pid);
-
-        return ret;
-}
+        gl_workdir = priv->workdir;
+        master = "";
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
+                          "--delete", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
 
-int
-glusterd_get_pid_from_file (char *master, char *slave, pid_t *pid)
-{
-        int ret                = -1;
-        int pfd                = 0;
-        char pidfile[PATH_MAX] = {0,};
-        char buff[1024]        = {0,};
+        if (volinfo) {
+                master = volinfo->volname;
+                runner_argprintf (&runner, ":%s", master);
+        }
+        runner_add_arg (&runner, slave);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "gsyncd failed to "
+                        "delete session info for %s and %s peers",
+                        master, slave);
 
-        pfd = gsyncd_getpidfile (master, slave, pidfile);
+                gf_asprintf (op_errstr, "gsyncd failed to "
+                             "delete session info for %s and %s peers",
+                             master, slave);
 
-        if (pfd == -2) {
-                gf_log ("", GF_LOG_ERROR, GEOREP" log-rotate validation "
-                        " failed for %s & %s", master, slave);
-                goto out;
-        }
-        if (gsync_status_byfd (pfd) == -1) {
-                gf_log ("", GF_LOG_ERROR, "gsyncd b/w %s & %s is not"
-                        " running", master, slave);
                 goto out;
         }
 
-        ret = read (pfd, buff, 1024);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, GEOREP" cannot read pid from pid-file");
-                goto out;
-        }
+        ret = snprintf (geo_rep_dir, sizeof(geo_rep_dir) - 1,
+                        "%s/"GEOREP"/%s_%s_%s", gl_workdir,
+                        volinfo->volname, slave_ip, slave_vol);
+        geo_rep_dir[ret] = '\0';
 
-        close(pfd);
+        ret = rmdir (geo_rep_dir);
+        if (ret) {
+                if (errno == ENOENT)
+                        gf_log ("", GF_LOG_DEBUG, "Geo Rep Dir(%s) Not Present.",
+                                geo_rep_dir);
+                else {
+                        gf_log ("", GF_LOG_ERROR, "Unable to delete "
+                                "Geo Rep Dir(%s). Error: %s", geo_rep_dir,
+                                strerror (errno));
+                        goto out;
+                }
+        }
 
-        *pid = strtol (buff, NULL, 10);
         ret = 0;
 
- out:
+        gf_asprintf (op_errstr, "delete successful");
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
-glusterd_do_gsync_log_rotate (char *master, char *slave, uuid_t *uuid, char **op_errstr)
+glusterd_op_sys_exec (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int              ret     = 0;
-        glusterd_conf_t *priv    = NULL;
-        pid_t            pid     = 0;
-        char log_file1[PATH_MAX] = {0,};
-        char log_file2[PATH_MAX] = {0,};
-
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
+        char                   buf[PATH_MAX]          = "";
+        char                   cmd_arg_name[PATH_MAX] = "";
+        char                   output_name[PATH_MAX]  = "";
+        char                   errmsg[PATH_MAX]       = "";
+        char                  *ptr                    = NULL;
+        char                  *bufp                   = NULL;
+        char                  *command                = NULL;
+        char                 **cmd_args               = NULL;
+        int                    ret                    = -1;
+        int                    i                      = -1;
+        int                    cmd_args_count         = 0;
+        int                    output_count           = 0;
+        glusterd_conf_t       *priv                   = NULL;
+        runner_t               runner                 = {0,};
 
-        priv = THIS->private;
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
 
-        ret = glusterd_get_pid_from_file (master, slave, &pid);
-        if (ret)
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
                 goto out;
+        }
 
-        /* log file */
-        ret = glusterd_gsyncd_getlogfile (master, slave, log_file1);
-        if (ret)
+        ret = dict_get_str (dict, "command", &command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get command from dict");
                 goto out;
+        }
 
-        /* slave log file */
-        ret = glusterd_gsync_get_slave_log_file (master, slave, log_file2);
+        ret = dict_get_int32 (dict, "cmd_args_count", &cmd_args_count);
         if (ret)
-                goto out;
-
-        ret = glusterd_send_log_rotate_signal (pid, log_file1, log_file2);
-
- out:
-        if (ret && op_errstr)
-                *op_errstr = gf_strdup("Error rotating log file");
-        return ret;
-}
-
-int
-glusterd_do_gsync_log_rotation_mst_slv (glusterd_volinfo_t *volinfo, char *slave,
-                                        char **op_errstr)
-{
-        uuid_t           uuid = {0, };
-        glusterd_conf_t *priv = NULL;
-        int              ret  = 0;
-        char errmsg[1024] = {0,};
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
+                gf_log ("", GF_LOG_INFO, "No cmd_args_count");
+
+        if (cmd_args_count) {
+                cmd_args = GF_CALLOC (cmd_args_count, sizeof (char*),
+                                      gf_common_mt_char);
+                if (!cmd_args) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to calloc. "
+                                "Errno = %s", strerror(errno));
+                        goto out;
+                }
 
-        priv = THIS->private;
+                for (i=1; i <= cmd_args_count; i++) {
+                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
+                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
+                                  "cmd_arg_%d", i);
+                        ret = dict_get_str (dict, cmd_arg_name, &cmd_args[i-1]);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s in dict",
+                                        cmd_arg_name);
+                                goto out;
+                        }
+                }
+        }
 
-        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-        if ((ret == 0) && (uuid_compare (priv->uuid, uuid) != 0))
+        runinit (&runner);
+        runner_argprintf (&runner, GSYNCD_PREFIX"/peer_%s", command);
+        for (i=0; i < cmd_args_count; i++)
+                runner_add_arg (&runner, cmd_args[i]);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_start (&runner);
+        if (ret == -1) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to "
+                          "execute command. Error : %s",
+                          strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                synclock_lock (&priv->big_lock);
                 goto out;
+        }
 
+        ptr = fgets(buf, sizeof(buf), runner_chio (&runner, STDOUT_FILENO));
+        if (ptr) {
+                ret = dict_get_int32 (rsp_dict, "output_count", &output_count);
+                if (ret)
+                        output_count = 1;
+                else
+                        output_count++;
+                memset (output_name, '\0', sizeof (output_name));
+                snprintf (output_name, sizeof (output_name),
+                          "output_%d", output_count);
+                if (buf[strlen(buf) - 1] == '\n')
+                        buf[strlen(buf) - 1] = '\0';
+                bufp = gf_strdup (buf);
+                if (!bufp)
+                        gf_log ("", GF_LOG_ERROR, "gf_strdup failed.");
+                ret = dict_set_dynstr (rsp_dict, output_name, bufp);
+                if (ret) {
+                        GF_FREE (bufp);
+                        gf_log ("", GF_LOG_ERROR, "output set failed.");
+                }
+                ret = dict_set_int32 (rsp_dict, "output_count", output_count);
+                if (ret)
+                        gf_log ("", GF_LOG_ERROR, "output_count set failed.");
+        }
+
+        ret = runner_end (&runner);
         if (ret) {
-                snprintf(errmsg, sizeof(errmsg), "geo-replication session b/w %s %s not active",
-                         volinfo->volname, slave);
-                gf_log ("", GF_LOG_WARNING, errmsg);
-                if (op_errstr)
-                        *op_errstr = gf_strdup(errmsg);
+                snprintf (errmsg, sizeof (errmsg), "Unable to "
+                          "end. Error : %s",
+                          strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                synclock_lock (&priv->big_lock);
                 goto out;
         }
+        synclock_lock (&priv->big_lock);
 
-        ret = glusterd_do_gsync_log_rotate (volinfo->volname, slave, &uuid, op_errstr);
+        ret = 0;
+out:
+        if (cmd_args) {
+                GF_FREE (cmd_args);
+                cmd_args = NULL;
+        }
 
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-static void
-_iterate_log_rotate_mst_slv (dict_t *this, char *key, data_t *value, void *data)
+int
+glusterd_op_copy_file (dict_t *dict, char **op_errstr)
 {
-        glusterd_gsync_status_temp_t  *param = NULL;
-        char                          *slave = NULL;
+        char             abs_filename[PATH_MAX] = "";
+        char             errmsg[PATH_MAX]       = "";
+        char            *filename               = NULL;
+        char            *host_uuid              = NULL;
+        char             uuid_str [64]          = {0};
+        char            *contents               = NULL;
+        char             buf[1024]              = "";
+        int              ret                    = -1;
+        int              fd                     = -1;
+        int              bytes_writen           = 0;
+        int              bytes_read             = 0;
+        int              contents_size          = -1;
+        int              file_mode              = -1;
+        glusterd_conf_t *priv                   = NULL;
+        struct stat      stbuf                  = {0,};
 
-        param = (glusterd_gsync_status_temp_t *) data;
 
-        GF_ASSERT (param);
-        GF_ASSERT (param->volinfo);
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
 
-        slave = strchr (value->data, ':');
-        if (slave)
-                slave++;
-        else {
-                gf_log ("", GF_LOG_ERROR, "geo-replication log-rotate: slave (%s) "
-                        "not conforming to format", slave);
-                return;
+        ret = dict_get_str (dict, "host-uuid", &host_uuid);
+        if (ret < 0)
+                goto out;
+
+        ret = dict_get_str (dict, "source", &filename);
+        if (ret < 0) {
+               gf_log ("", GF_LOG_ERROR, "Unable to fetch"
+                       " filename from dict.");
+               *op_errstr = gf_strdup ("command unsuccessful");
+               goto out;
         }
+        snprintf (abs_filename, sizeof(abs_filename),
+                  "%s/%s", priv->workdir, filename);
 
-        (void) glusterd_do_gsync_log_rotation_mst_slv (param->volinfo, slave, NULL);
-}
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = lstat (abs_filename, &stbuf);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Source file"
+                                 " does not exist in %s", priv->workdir);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
-int
-glusterd_do_gsync_log_rotation_mst (glusterd_volinfo_t *volinfo)
-{
-        glusterd_gsync_status_temp_t  param = {0, };
+                contents = GF_CALLOC(1, stbuf.st_size+1, gf_common_mt_char);
+                if (!contents) {
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "Unable to allocate memory");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
 
-        GF_ASSERT (volinfo);
+                fd = open (abs_filename, O_RDONLY);
+                if (fd < 0) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to open %s",
+                                  abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
 
-        param.volinfo = volinfo;
-        dict_foreach (volinfo->gsync_slaves, _iterate_log_rotate_mst_slv, &param);
-        return 0;
-}
+                do {
+                        ret = read (fd, buf, sizeof(buf));
+                        if (ret > 0) {
+                                memcpy (contents+bytes_read, buf, ret);
+                                bytes_read += ret;
+                                memset (buf, '\0', sizeof(buf));
+                        }
+                } while (ret > 0);
 
-static int
-glusterd_rotate_gsync_all ()
-{
-        int32_t             ret     = 0;
-        glusterd_conf_t    *priv    = NULL;
-        glusterd_volinfo_t *volinfo = NULL;
+                if (bytes_read != stbuf.st_size) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to read all "
+                                  "the data from %s", abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
 
-        GF_ASSERT (THIS);
-        priv = THIS->private;
+                ret = dict_set_int32 (dict, "contents_size", stbuf.st_size);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " contents size in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
-        GF_ASSERT (priv);
+                ret = dict_set_int32 (dict, "file_mode",
+                                      (int32_t)stbuf.st_mode);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " file mode in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
-        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                ret = glusterd_do_gsync_log_rotation_mst (volinfo);
-                if (ret)
+                ret = dict_set_bin (dict, "common_pem_contents",
+                                    contents, stbuf.st_size);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " pem contents in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
                         goto out;
-        }
+                }
+                close (fd);
+        } else {
+                ret = dict_get_bin (dict, "common_pem_contents",
+                                    (void **) &contents);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to get"
+                                  " pem contents in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-}
+                ret = dict_get_int32 (dict, "contents_size", &contents_size);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " contents size in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
-static int
-glusterd_rotate_gsync_logs (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
-{
-        char               *slave   = NULL;
-        char               *volname = NULL;
-        char errmsg[1024]           = {0,};
-        gf_boolean_t        exists  = _gf_false;
-        glusterd_volinfo_t *volinfo = NULL;
-        int                 ret     = 0;
+                ret = dict_get_int32 (dict, "file_mode", &file_mode);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to get"
+                                  " file mode in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
-        ret = dict_get_str (dict, "master", &volname);
-        if (ret < 0) {
-                ret = glusterd_rotate_gsync_all ();
-                goto out;
-        }
+                fd = open (abs_filename, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+                if (fd < 0) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to open %s",
+                                  abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
 
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((ret) || (!exists)) {
-                snprintf (errmsg, sizeof(errmsg), "Volume %s does not"
-                          " exist", volname);
-                gf_log ("", GF_LOG_WARNING, errmsg);
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
+                bytes_writen = write (fd, contents, contents_size);
 
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0) {
-                ret = glusterd_do_gsync_log_rotation_mst (volinfo);
-                goto out;
-        }
+                if (bytes_writen != contents_size) {
+                        snprintf (errmsg, sizeof (errmsg), "Failed to write"
+                                  " to %s", abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
 
-        ret = glusterd_do_gsync_log_rotation_mst_slv (volinfo, slave, op_errstr);
+                fchmod (fd, file_mode);
+                close (fd);
+        }
 
- out:
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-
 int
 glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
@@ -1855,11 +3484,16 @@ glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         dict_t             *resp_dict = NULL;
         char               *host_uuid = NULL;
         char               *slave  = NULL;
+        char               *slave_ip  = NULL;
+        char               *slave_vol = NULL;
         char               *volname = NULL;
+        char               *path_list = NULL;
         glusterd_volinfo_t *volinfo = NULL;
         glusterd_conf_t    *priv = NULL;
+        gf_boolean_t        is_force = _gf_false;
         char               *status_msg = NULL;
-        uuid_t              uuid = {0, };
+        gf_boolean_t        is_running = _gf_false;
+        char               *conf_path = NULL;
 
         GF_ASSERT (THIS);
         GF_ASSERT (THIS->private);
@@ -1885,15 +3519,28 @@ glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                 goto out;
         }
 
-        if (type == GF_GSYNC_OPTION_TYPE_ROTATE) {
-                ret = glusterd_rotate_gsync_logs (dict, op_errstr, resp_dict);
+        ret = dict_get_str (dict, "slave", &slave);
+        if (ret < 0)
                 goto out;
 
+        ret = dict_get_str (dict, "slave_ip", &slave_ip);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch slave volume name.");
+                goto out;
         }
 
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0)
+        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch slave volume name.");
                 goto out;
+        }
+
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
 
         if (dict_get_str (dict, "master", &volname) == 0) {
                 ret = glusterd_volinfo_find (volname, &volinfo);
@@ -1902,11 +3549,31 @@ glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                                 volname);
                         goto out;
                 }
+
+                ret = glusterd_get_local_brickpaths (volinfo, &path_list);
         }
 
         if (type == GF_GSYNC_OPTION_TYPE_CONFIG) {
-                ret = glusterd_gsync_configure (volinfo, slave, dict, resp_dict,
-                                                op_errstr);
+                ret = glusterd_gsync_configure (volinfo, slave, path_list,
+                                                dict, resp_dict, op_errstr);
+
+                ret = dict_set_str (resp_dict, "conf_path", conf_path);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to store conf_file_path.");
+                        goto out;
+                }
+                goto out;
+        }
+
+        if (type == GF_GSYNC_OPTION_TYPE_DELETE) {
+                ret = glusterd_remove_slave_in_info(volinfo, slave, op_errstr);
+                if (ret && !is_force && path_list)
+                        goto out;
+
+                ret = glusterd_gsync_delete (volinfo, slave, slave_ip,
+                                             slave_vol, path_list, dict,
+                                             resp_dict, op_errstr);
                 goto out;
         }
 
@@ -1915,48 +3582,652 @@ glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                 goto out;
         }
 
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
         if (type == GF_GSYNC_OPTION_TYPE_START) {
 
-                ret = glusterd_set_marker_gsync (volinfo);
+                ret = glusterd_set_gsync_confs (volinfo);
                 if (ret != 0) {
-                        gf_log ("", GF_LOG_WARNING, "marker start failed");
+                        gf_log ("", GF_LOG_WARNING, "marker/changelog start failed");
                         *op_errstr = gf_strdup ("failed to initialize indexing");
                         ret = -1;
                         goto out;
                 }
-                ret = glusterd_store_slave_in_info(volinfo, slave,
-                                                   host_uuid, op_errstr);
-                if (ret)
-                        goto out;
 
-                ret = glusterd_start_gsync (volinfo, slave, host_uuid,
-                                            op_errstr);
+                ret = glusterd_start_gsync (volinfo, slave, path_list,
+                                            conf_path, host_uuid, op_errstr);
         }
 
         if (type == GF_GSYNC_OPTION_TYPE_STOP) {
-
-                ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-                if (ret) {
+                ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                          slave, conf_path,
+                                                          &is_running);
+                if (!ret && !is_force && path_list &&
+                    (_gf_true != is_running)) {
                         gf_log ("", GF_LOG_WARNING, GEOREP" is not set up for"
                                 "%s(master) and %s(slave)", volname, slave);
                         *op_errstr = strdup (GEOREP" is not set up");
                         goto out;
                 }
 
-                ret = glusterd_remove_slave_in_info(volinfo, slave, op_errstr);
-                if (ret)
+                ret = stop_gsync (volname, slave, &status_msg, conf_path, is_force);
+                if (ret == 0 && status_msg)
+                        ret = dict_set_str (resp_dict, "gsync-status",
+                                            status_msg);
+                if (ret != 0 && !is_force && path_list)
+                        *op_errstr = gf_strdup ("internal error");
+
+                if (!ret) {
+                        ret = glusterd_create_status_file (volinfo->volname,
+                                                           slave, slave_ip,
+                                                           slave_vol, "Stopped");
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR, "Unable to update"
+                                        "state_file. Error : %s",
+                                        strerror (errno));
+                        }
+                }
+        }
+
+out:
+        if (path_list) {
+                GF_FREE (path_list);
+                path_list = NULL;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_get_slave_details_confpath (glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char **slave_ip, char **slave_vol,
+                                     char **conf_path, char **op_errstr)
+{
+        int                ret                = -1;
+        char               confpath[PATH_MAX] = "";
+        glusterd_conf_t   *priv               = NULL;
+        char              *slave              = NULL;
+
+        GF_ASSERT (THIS);
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "slave", &slave);
+        if (ret || !slave) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch slave from dict");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_slave_info (slave, slave_ip, slave_vol, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "slave_ip", *slave_ip);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store slave IP.");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "slave_vol", *slave_vol);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store slave volume name.");
+                goto out;
+        }
+
+        ret = snprintf (confpath, sizeof(confpath) - 1,
+                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                        priv->workdir, volinfo->volname,
+                        *slave_ip, *slave_vol);
+        confpath[ret] = '\0';
+        *conf_path = gf_strdup (confpath);
+        if (!(*conf_path)) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to gf_strdup. Error: %s", strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "conf_path", *conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store conf_path");
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG,"Returning %d", ret);
+        return ret;
+
+}
+
+static int
+glusterd_get_slave_info (char *slave, char **slave_ip,
+                         char **slave_vol, char **op_errstr)
+{
+        char     *tmp              = NULL;
+        char     *save_ptr         = NULL;
+        char    **linearr          = NULL;
+        int32_t   ret              = -1;
+        char      errmsg[PATH_MAX] = "";
+
+        ret = glusterd_urltransform_single (slave, "normalize",
+                                            &linearr);
+        if (ret == -1) {
+                ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                "Invalid Url: %s", slave);
+                errmsg[ret] = '\0';
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "Failed to normalize url");
+                goto out;
+        }
+
+        tmp = strtok_r (linearr[0], "/", &save_ptr);
+        tmp = strtok_r (NULL, "/", &save_ptr);
+        slave = strtok_r (tmp, ":", &save_ptr);
+        if (slave) {
+                ret = glusterd_mountbroker_check (&slave, op_errstr);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Invalid slave url: %s", *op_errstr);
                         goto out;
+                }
 
-                if (uuid_compare (priv->uuid, uuid) != 0) {
+                *slave_ip = gf_strdup (slave);
+                if (!*slave_ip) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Failed to gf_strdup");
+                        ret = -1;
                         goto out;
                 }
+                gf_log ("", GF_LOG_DEBUG, "Slave IP : %s", *slave_ip);
+                ret = 0;
+        } else {
+                gf_log ("", GF_LOG_ERROR, "Invalid slave name");
+                goto out;
+        }
 
-                ret = stop_gsync (volname, slave, &status_msg);
-                if (ret == 0 && status_msg)
-                        ret = dict_set_str (resp_dict, "gsync-status",
-                                            status_msg);
-                if (ret != 0)
-                        *op_errstr = gf_strdup ("internal error");
+        slave = strtok_r (NULL, ":", &save_ptr);
+        if (slave) {
+                *slave_vol = gf_strdup (slave);
+                if (!*slave_vol) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Failed to gf_strdup");
+                        ret = -1;
+                        goto out;
+                }
+                gf_log ("", GF_LOG_DEBUG, "Slave Vol : %s", *slave_vol);
+                ret = 0;
+        } else {
+                gf_log ("", GF_LOG_ERROR, "Invalid slave name");
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static void
+runinit_gsyncd_setrx (runner_t *runner, char *conf_path)
+{
+        runinit (runner);
+        runner_add_args (runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
+        runner_argprintf (runner, "%s", conf_path);
+        runner_add_arg (runner, "--config-set-rx");
+}
+
+static int
+glusterd_check_gsync_present (int *valid_state)
+{
+        char                buff[PATH_MAX] = {0, };
+        runner_t            runner = {0,};
+        char               *ptr = NULL;
+        int                 ret = 0;
+
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        ret = runner_start (&runner);
+        if (ret == -1) {
+                if (errno == ENOENT) {
+                        gf_log ("glusterd", GF_LOG_INFO, GEOREP
+                                 " module not installed in the system");
+                        *valid_state = 0;
+                }
+                else {
+                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP
+                                  " module not working as desired");
+                        *valid_state = -1;
+                }
+                goto out;
+        }
+
+        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
+        if (ptr) {
+                if (!strstr (buff, "gsyncd")) {
+                        ret = -1;
+                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
+                                 "working as desired");
+                        *valid_state = -1;
+                        goto out;
+                }
+        } else {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
+                         "working as desired");
+                *valid_state = -1;
+                goto out;
+        }
+
+        ret = 0;
+ out:
+
+        runner_end (&runner);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int
+create_conf_file (glusterd_conf_t *conf, char *conf_path)
+#define RUN_GSYNCD_CMD do {                                                          \
+        ret = runner_run_reuse (&runner);                                            \
+        if (ret == -1) {                                                             \
+                runner_log (&runner, "glusterd", GF_LOG_ERROR, "command failed");    \
+                runner_end (&runner);                                                \
+                goto out;                                                            \
+        }                                                                            \
+        runner_end (&runner);                                                        \
+} while (0)
+{
+        int ret = 0;
+        runner_t runner = {0,};
+        char georepdir[PATH_MAX] = {0,};
+        int valid_state = 0;
+
+        valid_state = -1;
+        ret = glusterd_check_gsync_present (&valid_state);
+        if (-1 == ret) {
+                ret = valid_state;
+                goto out;
+        }
+
+        ret = snprintf (georepdir, sizeof(georepdir) - 1, "%s/"GEOREP,
+                        conf->workdir);
+        georepdir[ret] = '\0';
+
+        /************
+         * master pre-configuration
+         ************/
+
+        /* remote-gsyncd */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "remote-gsyncd", GSYNCD_PREFIX"/gsyncd", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "remote-gsyncd", "/nonexistent/gsyncd",
+                         ".", "^ssh:", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-command-dir */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-params */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-params",
+                         "aux-gfid-mount xlator-option=*-dht.assert-no-child-down=true",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* ssh-command */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "ssh-command");
+        runner_argprintf (&runner,
+                          "ssh -oPasswordAuthentication=no "
+                           "-oStrictHostKeyChecking=no "
+                           "-i %s/secret.pem", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* pid-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "pid-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.pid", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "state-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.status", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-detail-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "state-detail-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-socket */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "state-socket-unencoded");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.socket", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* socketdir */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.log",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "gluster-log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}.gluster.log",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* ignore-deletes */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "ignore-deletes", "true", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* special-sync-mode */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "special-sync-mode", "partial", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* change-detector == changelog */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg(&runner, "working-dir");
+        runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
+                         DEFAULT_VAR_RUN_DIRECTORY);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /************
+         * slave pre-configuration
+         ************/
+
+        /* gluster-command-dir */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-params */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-params",
+                         "aux-gfid-mount xlator-option=*-dht.assert-no-child-down=true",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* MountBroker log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "log-file-mbr",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr/${session_owner}:${eSlave}.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "gluster-log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.gluster.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+ out:
+        return ret ? -1 : 0;
+}
+
+static int
+glusterd_create_essential_dir_files (glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char *slave, char *slave_ip,
+                                     char *slave_vol, char **op_errstr)
+{
+        int                ret              = -1;
+        char              *conf_path        = NULL;
+        char              *statefile        = NULL;
+        char               buf[PATH_MAX]    = "";
+        char               errmsg[PATH_MAX] = "";
+        glusterd_conf_t   *conf             = NULL;
+        struct stat        stbuf            = {0,};
+
+        GF_ASSERT (THIS);
+        conf = THIS->private;
+
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch conf file path.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "statefile", &statefile);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch statefile path.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = snprintf (buf, sizeof(buf) - 1, "%s/"GEOREP"/%s_%s_%s",
+                        conf->workdir, volinfo->volname, slave_ip, slave_vol);
+        buf[ret] = '\0';
+        ret = mkdir_p (buf, 0777, _gf_true);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
+                          ". Error : %s", buf, strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = snprintf (buf, PATH_MAX, DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/%s",
+                        volinfo->volname);
+        buf[ret] = '\0';
+        ret = mkdir_p (buf, 0777, _gf_true);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
+                          ". Error : %s", buf, strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = lstat (conf_path, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "Session already running."
+                        " Not creating config file again.");
+        } else {
+                ret = create_conf_file (conf, conf_path);
+                if (ret || lstat (conf_path, &stbuf)) {
+                        snprintf (errmsg, sizeof (errmsg), "Failed to create"
+                                  " config file(%s).", conf_path);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
+        }
+
+        ret = lstat (statefile, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "Session already running."
+                        " Not creating status file again.");
+                goto out;
+        } else {
+                ret = glusterd_create_status_file (volinfo->volname, slave,
+                                                   slave_ip, slave_vol,
+                                                   "Not Started");
+                if (ret || lstat (statefile, &stbuf)) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
+                                  ". Error : %s", statefile, strerror (errno));
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG,"Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_gsync_create (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        char                common_pem_file[PATH_MAX] = "";
+        char                errmsg[PATH_MAX]          = "";
+        char                hooks_args[PATH_MAX]      = "";
+        char                uuid_str [64]             = "";
+        char               *host_uuid                 = NULL;
+        char               *slave_ip                  = NULL;
+        char               *slave_vol                 = NULL;
+        char               *arg_buf                   = NULL;
+        char               *volname                   = NULL;
+        char               *slave                     = NULL;
+        int32_t             ret                       = -1;
+        int32_t             is_pem_push               = -1;
+        gf_boolean_t        is_force                  = -1;
+        glusterd_conf_t    *conf                      = NULL;
+        glusterd_volinfo_t *volinfo                   = NULL;
+
+        GF_ASSERT (THIS);
+        conf = THIS->private;
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        ret = glusterd_op_gsync_args_get (dict, op_errstr,
+                                          &volname, &slave, &host_uuid);
+        if (ret)
+                goto out;
+
+        snprintf (common_pem_file, sizeof(common_pem_file),
+                  "%s"GLUSTERD_COMMON_PEM_PUB_FILE, conf->workdir);
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Volinfo for %s"
+                        " (master) not found", volname);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch slave volume name.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "slave_ip", &slave_ip);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch slave IP.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = dict_get_int32 (dict, "push_pem", &is_pem_push);
+                if (!ret && is_pem_push) {
+                        gf_log ("", GF_LOG_DEBUG, "Trying to setup"
+                                " pem files in slave");
+                        is_pem_push = 1;
+                } else
+                        is_pem_push = 0;
+
+                snprintf(hooks_args, sizeof(hooks_args),
+                         "is_push_pem=%d pub_file=%s slave_ip=%s",
+                         is_pem_push, common_pem_file, slave_ip);
+
+        } else
+                snprintf(hooks_args, sizeof(hooks_args),
+                         "This argument will stop the hooks script");
+
+        arg_buf = gf_strdup (hooks_args);
+        if (!arg_buf) {
+                gf_log ("", GF_LOG_ERROR, "Failed to"
+                        " gf_strdup");
+                if (is_force) {
+                        ret = 0;
+                        goto create_essentials;
+                }
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "hooks_args", arg_buf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Failed to set"
+                        " hooks_args in dict.");
+                if (is_force) {
+                        ret = 0;
+                        goto create_essentials;
+                }
+                goto out;
+        }
+
+create_essentials:
+
+        ret = glusterd_create_essential_dir_files (volinfo, dict, slave,
+                                                   slave_ip, slave_vol,
+                                                   op_errstr);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_slave_in_info (volinfo, slave,
+                                            host_uuid, op_errstr,
+                                            is_force);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to store"
+                          " slave info.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
         }
 
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index f2980e7ef..71d076624 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -43,6 +33,7 @@
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
+#include "glusterd-locks.h"
 
 #include "glusterd1-xdr.h"
 #include "cli1-xdr.h"
@@ -57,6 +48,39 @@
 #include "defaults.c"
 #include "common-utils.h"
 
+#include "globals.h"
+#include "glusterd-syncop.h"
+
+#ifdef HAVE_BD_XLATOR
+#include <lvm2app.h>
+#endif
+
+extern uuid_t global_txn_id;
+
+int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
+                                rpc_clnt_event_t event,
+                                void *data, rpc_clnt_notify_t notify_fn)
+{
+        glusterd_conf_t *priv = THIS->private;
+        int             ret   = -1;
+        synclock_lock (&priv->big_lock);
+        ret = notify_fn (rpc, mydata, event, data);
+        synclock_unlock (&priv->big_lock);
+        return ret;
+}
+
+int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn)
+{
+        glusterd_conf_t *priv = THIS->private;
+        int             ret   = -1;
+
+        synclock_lock (&priv->big_lock);
+        ret = actor_fn (req);
+        synclock_unlock (&priv->big_lock);
+
+        return ret;
+}
+
 static int
 glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
                             char *hostname, int port,
@@ -78,10 +102,12 @@ glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
         ret = glusterd_friend_find (uuid, rhost, &peerinfo);
 
         if (ret) {
-                ret = glusterd_xfer_friend_add_resp (req, rhost, port, -1,
-                                                     GF_PROBE_UNKNOWN_PEER);
-                if (friend_req->vols.vols_val)
+                ret = glusterd_xfer_friend_add_resp (req, hostname, rhost, port,
+                                                     -1, GF_PROBE_UNKNOWN_PEER);
+                if (friend_req->vols.vols_val) {
                         free (friend_req->vols.vols_val);
+                        friend_req->vols.vols_val = NULL;
+                }
                 goto out;
         }
 
@@ -139,19 +165,16 @@ out:
         if (0 != ret) {
                 if (ctx && ctx->hostname)
                         GF_FREE (ctx->hostname);
-                if (ctx)
-                        GF_FREE (ctx);
+                GF_FREE (ctx);
                 if (dict) {
                         if ((!dict->extra_stdfree) &&
                             friend_req->vols.vols_val)
                                 free (friend_req->vols.vols_val);
                         dict_unref (dict);
                 } else {
-                    if (friend_req->vols.vols_val)
-                        free (friend_req->vols.vols_val);
+                    free (friend_req->vols.vols_val);
                 }
-                if (event)
-                        GF_FREE (event);
+                GF_FREE (event);
         } else {
                 if (peerinfo && (0 == peerinfo->connected))
                         ret = GLUSTERD_CONNECTION_AWAITED;
@@ -221,8 +244,7 @@ out:
         if (0 != ret) {
                 if (ctx && ctx->hostname)
                         GF_FREE (ctx->hostname);
-                if (ctx)
-                        GF_FREE (ctx);
+                GF_FREE (ctx);
         }
 
         return ret;
@@ -235,13 +257,14 @@ glusterd_add_peer_detail_to_dict (glusterd_peerinfo_t   *peerinfo,
 
         int             ret = -1;
         char            key[256] = {0, };
+        char           *peer_uuid_str = NULL;
 
         GF_ASSERT (peerinfo);
         GF_ASSERT (friends);
 
         snprintf (key, 256, "friend%d.uuid", count);
-        uuid_utoa_r (peerinfo->uuid, peerinfo->uuid_str);
-        ret = dict_set_str (friends, key, peerinfo->uuid_str);
+        peer_uuid_str = gd_peer_uuid_str (peerinfo);
+        ret = dict_set_str (friends, key, peer_uuid_str);
         if (ret)
                 goto out;
 
@@ -255,6 +278,11 @@ glusterd_add_peer_detail_to_dict (glusterd_peerinfo_t   *peerinfo,
         if (ret)
                 goto out;
 
+        snprintf (key, 256, "friend%d.stateId", count);
+        ret = dict_set_int32 (friends, key, peerinfo->state.state);
+        if (ret)
+                goto out;
+
         snprintf (key, 256, "friend%d.state", count);
         ret = dict_set_str (friends, key,
                     glusterd_friend_sm_state_name_get(peerinfo->state.state));
@@ -270,10 +298,34 @@ out:
         return ret;
 }
 
+struct args_pack {
+    dict_t *dict;
+    int vol_count;
+    int opt_count;
+};
+
+static int
+_build_option_key (dict_t *d, char *k, data_t *v, void *tmp)
+{
+        char                    reconfig_key[256] = {0, };
+        struct args_pack        *pack             = NULL;
+        int                     ret               = -1;
+
+        pack = tmp;
+        if (strcmp (k, GLUSTERD_GLOBAL_OPT_VERSION) == 0)
+                return 0;
+        snprintf (reconfig_key, 256, "volume%d.option.%s",
+                  pack->vol_count, k);
+        ret = dict_set_str (pack->dict, reconfig_key, v->data);
+        if (0 == ret)
+                pack->opt_count++;
+
+        return 0;
+}
 
 int
 glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
-                                    dict_t  *volumes, int   count)
+                                    dict_t  *volumes, int count)
 {
 
         int                     ret = -1;
@@ -281,19 +333,20 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
         glusterd_brickinfo_t    *brickinfo = NULL;
         char                    *buf = NULL;
         int                     i = 1;
-        data_pair_t             *pairs = NULL;
-        char                    reconfig_key[256] = {0, };
         dict_t                  *dict = NULL;
-        data_t                  *value = NULL;
-        int                     opt_count = 0;
         glusterd_conf_t         *priv = NULL;
         char                    *volume_id_str  = NULL;
-
+        struct args_pack        pack = {0,};
+        xlator_t                *this = NULL;
+#ifdef HAVE_BD_XLATOR
+        int                     caps = 0;
+#endif
 
         GF_ASSERT (volinfo);
         GF_ASSERT (volumes);
 
-        priv = THIS->private;
+        this = THIS;
+        priv = this->private;
 
         GF_ASSERT (priv);
 
@@ -312,6 +365,21 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
+        /* As of now, the snap volumes are also displayed as part of
+           volume info command. So this change is to display whether
+           the volume is original volume or the snap_volume. If
+           displaying of snap volumes in volume info o/p is not needed
+           this should be removed.
+        */
+        snprintf (key, 256, "volume%d.snap_volume", count);
+        ret = dict_set_int32 (volumes, key, volinfo->is_snap_volume);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to set whether "
+                        "the volume is a snap volume or actual volume (%s)",
+                        volinfo->volname);
+                goto out;
+        }
+
         snprintf (key, 256, "volume%d.brick_count", count);
         ret = dict_set_int32 (volumes, key, volinfo->brick_count);
         if (ret)
@@ -346,8 +414,82 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
+        snprintf (key, 256, "volume%d.rebalance", count);
+        ret = dict_set_int32 (volumes, key, volinfo->rebal.defrag_cmd);
+        if (ret)
+                goto out;
+
+#ifdef HAVE_BD_XLATOR
+        if (volinfo->caps) {
+                caps = 0;
+                snprintf (key, 256, "volume%d.xlator0", count);
+                buf = GF_MALLOC (256, gf_common_mt_char);
+                if (!buf) {
+                        ret = ENOMEM;
+                        goto out;
+                }
+                if (volinfo->caps & CAPS_BD)
+                        snprintf (buf, 256, "BD");
+                ret = dict_set_dynstr (volumes, key, buf);
+                if (ret) {
+                        GF_FREE (buf);
+                        goto out;
+                }
+
+                if (volinfo->caps & CAPS_THIN) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "thin");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret) {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+                if (volinfo->caps & CAPS_OFFLOAD_COPY) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "offload_copy");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret) {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+                if (volinfo->caps & CAPS_OFFLOAD_SNAPSHOT) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "offload_snapshot");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret)  {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+        }
+#endif
+
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 char    brick[1024] = {0,};
+                char    brick_uuid[64] = {0,};
                 snprintf (key, 256, "volume%d.brick%d", count, i);
                 snprintf (brick, 1024, "%s:%s", brickinfo->hostname,
                           brickinfo->path);
@@ -355,6 +497,25 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
                 ret = dict_set_dynstr (volumes, key, buf);
                 if (ret)
                         goto out;
+                snprintf (key, 256, "volume%d.brick%d.uuid", count, i);
+                snprintf (brick_uuid, 64, "%s", uuid_utoa (brickinfo->uuid));
+                buf = gf_strdup (brick_uuid);
+                if (!buf)
+                        goto out;
+                ret = dict_set_dynstr (volumes, key, buf);
+                if (ret)
+                        goto out;
+
+#ifdef HAVE_BD_XLATOR
+                if (volinfo->caps & CAPS_BD) {
+                        snprintf (key, 256, "volume%d.vg%d", count, i);
+                        snprintf (brick, 1024, "%s", brickinfo->vg);
+                        buf = gf_strdup (brick);
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret)
+                                goto out;
+                }
+#endif
                 i++;
         }
 
@@ -364,25 +525,14 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
                 goto out;
         }
 
-        pairs = dict->members_list;
+        pack.dict = volumes;
+        pack.vol_count = count;
+        pack.opt_count = 0;
+        dict_foreach (dict, _build_option_key, (void *) &pack);
+        dict_foreach (priv->opts, _build_option_key, &pack);
 
-        while (pairs) {
-                if (1 == glusterd_check_option_exists (pairs->key, NULL)) {
-                        value = pairs->value;
-                        if (!value)
-                                continue;
-
-                        snprintf (reconfig_key, 256, "volume%d.option.%s", count,
-                                  pairs->key);
-                        ret = dict_set_str  (volumes, reconfig_key, value->data);
-                        if (!ret)
-                            opt_count++;
-                }
-                pairs = pairs->next;
-        }
-
-        snprintf (key, 256, "volume%d.opt_count", count);
-        ret = dict_set_int32 (volumes, key, opt_count);
+        snprintf (key, 256, "volume%d.opt_count", pack.vol_count);
+        ret = dict_set_int32 (volumes, key, pack.opt_count);
 out:
         return ret;
 }
@@ -392,13 +542,18 @@ glusterd_friend_find (uuid_t uuid, char *hostname,
                       glusterd_peerinfo_t **peerinfo)
 {
         int     ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         if (uuid) {
                 ret = glusterd_friend_find_by_uuid (uuid, peerinfo);
 
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_INFO,
-                                 "Unable to find peer by uuid");
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                 "Unable to find peer by uuid: %s",
+                                 uuid_utoa (uuid));
                 } else {
                         goto out;
                 }
@@ -409,7 +564,7 @@ glusterd_friend_find (uuid_t uuid, char *hostname,
                 ret = glusterd_friend_find_by_hostname (hostname, peerinfo);
 
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_INFO,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "Unable to find hostname: %s", hostname);
                 } else {
                         goto out;
@@ -421,73 +576,200 @@ out:
 }
 
 int32_t
-glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx)
+glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                       char *err_str, size_t err_len)
 {
-        int32_t                  ret    = -1;
-        xlator_t                *this   = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        int32_t                  locked = 0;
+        int32_t                     ret             = -1;
+        dict_t                     *dict            = NULL;
+        xlator_t                   *this            = NULL;
+        glusterd_conf_t            *priv            = NULL;
+        int32_t                     locked          = 0;
+        char                       *tmp             = NULL;
+        char                       *volname         = NULL;
+        uuid_t                     *txn_id          = NULL;
+        uuid_t                     *originator_uuid = NULL;
+        glusterd_op_info_t          txn_op_info     = {{0},};
+        glusterd_op_sm_event_type_t event_type      = GD_OP_EVENT_NONE;
 
         GF_ASSERT (req);
         GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
         GF_ASSERT (NULL != ctx);
 
         this = THIS;
+        GF_ASSERT (this);
         priv = this->private;
         GF_ASSERT (priv);
 
-        ret = glusterd_lock (priv->uuid);
+        dict = ctx;
+
+        /* Generate a transaction-id for this operation and
+         * save it in the dict. This transaction id distinguishes
+         * each transaction, and helps separate opinfos in the
+         * op state machine. */
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!txn_id)
+                goto out;
+
+        uuid_generate (*txn_id);
+
+        ret = dict_set_bin (dict, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Transaction_id = %s", uuid_utoa (*txn_id));
+
+        /* Save the MY_UUID as the originator_uuid. This originator_uuid
+         * will be used by is_origin_glusterd() to determine if a node
+         * is the originator node for a command. */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
 
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (dict, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to acquire local lock, ret: %d", ret);
+                        "Failed to set originator uuid.");
                 goto out;
         }
 
+        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+        if (priv->op_version < 3) {
+                ret = glusterd_lock (MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock on localhost, ret: %d",
+                                ret);
+                        snprintf (err_str, err_len,
+                                  "Another transaction is in progress. "
+                                  "Please try again after sometime.");
+                        goto out;
+                }
+        } else {
+                /* If no volname is given as a part of the command, locks will
+                 * not be held */
+                ret = dict_get_str (dict, "volname", &tmp);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to get volume "
+                                "name");
+                        goto local_locking_done;
+                } else {
+                        /* Use a copy of volname, as cli response will be
+                         * sent before the unlock, and the volname in the
+                         * dict, might be removed */
+                        volname = gf_strdup (tmp);
+                        if (!volname)
+                                goto out;
+                }
+
+                ret = glusterd_mgmt_v3_lock (volname, MY_UUID, "vol");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock for %s", volname);
+                        snprintf (err_str, err_len,
+                                  "Another transaction is in progress for %s. "
+                                  "Please try again after sometime.", volname);
+                        goto out;
+                }
+        }
+
         locked = 1;
-        gf_log (this->name, GF_LOG_INFO, "Acquired local lock");
+        gf_log (this->name, GF_LOG_DEBUG, "Acquired lock on localhost");
+
+local_locking_done:
+
+        /* If no volname is given as a part of the command, locks will
+         * not be held, hence sending stage event. */
+        if (volname)
+                event_type = GD_OP_EVENT_START_LOCK;
+        else {
+                txn_op_info.state.state = GD_OP_STATE_LOCK_SENT;
+                event_type = GD_OP_EVENT_ALL_ACC;
+        }
+
+        /* Save opinfo for this transaction with the transaction id */
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, ctx, req);
+
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                if (ctx)
+                        dict_unref (ctx);
+                goto out;
+        }
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_START_LOCK, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, ctx);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to acquire cluster"
                         " lock.");
                 goto out;
         }
 
-        glusterd_op_set_op (op);
-        glusterd_op_set_ctx (ctx);
-        glusterd_op_set_req (req);
-
-
 out:
-        if (locked && ret)
-                glusterd_unlock (priv->uuid);
+        if (locked && ret) {
+                /* Based on the op-version, we release the
+                 * cluster or mgmt_v3 lock */
+                if (priv->op_version < 3)
+                        glusterd_unlock (MY_UUID);
+                else {
+                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                        ret = -1;
+                }
+        }
+
+        if (volname)
+                GF_FREE (volname);
 
         gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
-glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+__glusterd_handle_cluster_lock (rpcsvc_request_t *req)
 {
-        gd1_mgmt_cluster_lock_req       lock_req = {{0},};
-        int32_t                         ret = -1;
-        glusterd_op_lock_ctx_t          *ctx = NULL;
-        glusterd_peerinfo_t             *peerinfo = NULL;
+        dict_t                         *op_ctx      = NULL;
+        int32_t                         ret         = -1;
+        gd1_mgmt_cluster_lock_req       lock_req    = {{0},};
+        glusterd_op_lock_ctx_t         *ctx         = NULL;
+        glusterd_op_t                   op          = GD_OP_EVENT_LOCK;
+        glusterd_peerinfo_t            *peerinfo    = NULL;
+        glusterd_op_info_t              txn_op_info = {{0},};
+        uuid_t                         *txn_id      = &global_txn_id;
+        xlator_t                       *this        = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &lock_req, (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
+                        "request received from peer");
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received LOCK from uuid: %s", uuid_utoa (lock_req.uuid));
+        gf_log (this->name, GF_LOG_DEBUG, "Received LOCK from uuid: %s",
+                uuid_utoa (lock_req.uuid));
 
         if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
-                gf_log (THIS->name, GF_LOG_WARNING, "%s doesn't "
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
                         "belong to the cluster. Ignoring request.",
                         uuid_utoa (lock_req.uuid));
                 ret = -1;
@@ -503,11 +785,32 @@ glusterd_handle_cluster_lock (rpcsvc_request_t *req)
 
         uuid_copy (ctx->uuid, lock_req.uuid);
         ctx->req = req;
+        ctx->dict = NULL;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK, ctx);
+        op_ctx =  dict_new ();
+        if (!op_ctx) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set new dict");
+                goto out;
+        }
+
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, op_ctx, req);
+
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                dict_unref (txn_op_info.op_ctx);
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK, txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_LOCK");
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
@@ -516,6 +819,13 @@ out:
 }
 
 int
+glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cluster_lock);
+}
+
+int
 glusterd_req_ctx_create (rpcsvc_request_t *rpc_req,
                          glusterd_op_t op, uuid_t uuid,
                          char *buf_val, size_t buf_len,
@@ -526,10 +836,13 @@ glusterd_req_ctx_create (rpcsvc_request_t *rpc_req,
         char                str[50] = {0,};
         glusterd_req_ctx_t *req_ctx = NULL;
         dict_t             *dict    = NULL;
+        xlator_t           *this    = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         uuid_unparse (uuid, str);
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received op from uuid: %s", str);
+        gf_log (this->name, GF_LOG_DEBUG, "Received op from uuid %s", str);
 
         dict = dict_new ();
         if (!dict)
@@ -544,7 +857,7 @@ glusterd_req_ctx_create (rpcsvc_request_t *rpc_req,
         req_ctx->op = op;
         ret = dict_unserialize (buf_val, buf_len, &dict);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_WARNING,
                         "failed to unserialize the dictionary");
                 goto out;
         }
@@ -557,29 +870,38 @@ out:
         if (ret) {
                 if (dict)
                         dict_unref (dict);
-                if (req_ctx)
-                        GF_FREE (req_ctx);
+                GF_FREE (req_ctx);
         }
         return ret;
 }
 
 int
-glusterd_handle_stage_op (rpcsvc_request_t *req)
+__glusterd_handle_stage_op (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         glusterd_req_ctx_t              *req_ctx = NULL;
         gd1_mgmt_stage_op_req           op_req = {{0},};
         glusterd_peerinfo_t             *peerinfo = NULL;
+        xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
+        glusterd_op_info_t              txn_op_info = {{0},};
+        glusterd_op_sm_state_info_t     state;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
-        if (!xdr_to_generic (req->msg[0], &op_req, (xdrproc_t)xdr_gd1_mgmt_stage_op_req)) {
-                //failed to decode msg;
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode stage "
+                        "request received from peer");
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
         if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
-                gf_log (THIS->name, GF_LOG_WARNING, "%s doesn't "
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
                         "belong to the cluster. Ignoring request.",
                         uuid_utoa (op_req.uuid));
                 ret = -1;
@@ -592,34 +914,76 @@ glusterd_handle_stage_op (rpcsvc_request_t *req)
         if (ret)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_OP, req_ctx);
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        /* In cases where there is no volname, the receivers won't have a
+         * transaction opinfo created, as for those operations, the locking
+         * phase where the transaction opinfos are created, won't be called. */
+        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No transaction's opinfo set");
+
+                state.state = GD_OP_STATE_LOCKED;
+                glusterd_txn_opinfo_init (&txn_op_info, &state,
+                                          &op_req.op, req_ctx->dict, req);
+
+                ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set transaction's opinfo");
+                        dict_unref (req_ctx->dict);
+                        goto out;
+                }
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_OP,
+                                           txn_id, req_ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_STAGE_OP");
 
  out:
-        if (op_req.buf.buf_val)
-                free (op_req.buf.buf_val);//malloced by xdr
+        free (op_req.buf.buf_val);//malloced by xdr
         glusterd_friend_sm ();
         glusterd_op_sm ();
         return ret;
 }
 
 int
-glusterd_handle_commit_op (rpcsvc_request_t *req)
+glusterd_handle_stage_op (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_stage_op);
+}
+
+
+int
+__glusterd_handle_commit_op (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         glusterd_req_ctx_t              *req_ctx = NULL;
         gd1_mgmt_commit_op_req          op_req = {{0},};
         glusterd_peerinfo_t             *peerinfo = NULL;
+        xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &op_req, (xdrproc_t)xdr_gd1_mgmt_commit_op_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
+                        "request received from peer");
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
         if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
-                gf_log (THIS->name, GF_LOG_WARNING, "%s doesn't "
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
                         "belong to the cluster. Ignoring request.",
                         uuid_utoa (op_req.uuid));
                 ret = -1;
@@ -634,69 +998,124 @@ glusterd_handle_commit_op (rpcsvc_request_t *req)
         if (ret)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_OP, req_ctx);
-        if (ret)
-                goto out;
-        ret = glusterd_op_init_ctx (op_req.op);
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_OP,
+                                           txn_id, req_ctx);
 
 out:
-        if (op_req.buf.buf_val)
-                free (op_req.buf.buf_val);//malloced by xdr
+        free (op_req.buf.buf_val);//malloced by xdr
         glusterd_friend_sm ();
         glusterd_op_sm ();
         return ret;
 }
+
 int
-glusterd_handle_cli_probe (rpcsvc_request_t *req)
+glusterd_handle_commit_op (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_commit_op);
+}
+
+int
+__glusterd_handle_cli_probe (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
-        gf1_cli_probe_req               cli_req = {0,};
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        gf_boolean_t                    run_fsm = _gf_true;
+        gf_cli_req                  cli_req = {{0,},};
+        glusterd_peerinfo_t       *peerinfo = NULL;
+        gf_boolean_t                run_fsm = _gf_true;
+        xlator_t                      *this = NULL;
+        char                     *bind_name = NULL;
+        dict_t                        *dict = NULL;
+        char                      *hostname = NULL;
+        int                            port = 0;
+
         GF_ASSERT (req);
+        this = THIS;
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf1_cli_probe_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0)  {
                 //failed to decode msg;
                 gf_log ("", GF_LOG_ERROR, "xdr decoding error");
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
-        gf_cmd_log ("peer probe", " on host %s:%d", cli_req.hostname,
-                    cli_req.port);
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len, &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "unserialize req-buffer to dictionary");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get hostname");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "port", &port);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get port");
+                goto out;
+        }
+
+        if (glusterd_is_any_volume_in_server_quorum (this) &&
+            !does_gd_meet_server_quorum (this)) {
+                glusterd_xfer_cli_probe_resp (req, -1, GF_PROBE_QUORUM_NOT_MET,
+                                              NULL, hostname, port, dict);
+                gf_log (this->name, GF_LOG_ERROR, "Quorum does not meet, "
+                        "rejecting operation");
+                ret = 0;
+                goto out;
+        }
+
         gf_log ("glusterd", GF_LOG_INFO, "Received CLI probe req %s %d",
-                cli_req.hostname, cli_req.port);
+                hostname, port);
 
-        if (!(ret = glusterd_is_local_addr(cli_req.hostname))) {
+        if (dict_get_str(this->options,"transport.socket.bind-address",
+                         &bind_name) == 0) {
+                gf_log ("glusterd", GF_LOG_DEBUG,
+                        "only checking probe address vs. bind address");
+                ret = gf_is_same_address (bind_name, hostname);
+        }
+        else {
+                ret = gf_is_local_addr (hostname);
+        }
+        if (ret) {
                 glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_LOCALHOST,
-                                              cli_req.hostname, cli_req.port);
+                                              NULL, hostname, port, dict);
+                ret = 0;
                 goto out;
         }
 
-        if (!(ret = glusterd_friend_find_by_hostname(cli_req.hostname,
-                                         &peerinfo))) {
-                if (strcmp (peerinfo->hostname, cli_req.hostname) == 0) {
+        if (!(ret = glusterd_friend_find_by_hostname (hostname, &peerinfo))) {
+                if (strcmp (peerinfo->hostname, hostname) == 0) {
 
-                        gf_log ("glusterd", GF_LOG_DEBUG, "Probe host %s port %d"
-                               " already a peer", cli_req.hostname, cli_req.port);
+                        gf_log ("glusterd", GF_LOG_DEBUG, "Probe host %s port "
+                                "%d already a peer", hostname, port);
                         glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND,
-                                                      cli_req.hostname, cli_req.port);
+                                                      NULL, hostname, port,
+                                                      dict);
                         goto out;
                 }
         }
-        ret = glusterd_probe_begin (req, cli_req.hostname, cli_req.port);
-
-        gf_cmd_log ("peer probe","on host %s:%d %s",cli_req.hostname, cli_req.port,
-                    (ret) ? "FAILED" : "SUCCESS");
+        ret = glusterd_probe_begin (req, hostname, port, dict);
 
         if (ret == GLUSTERD_CONNECTION_AWAITED) {
                 //fsm should be run after connection establishes
                 run_fsm = _gf_false;
                 ret = 0;
         }
+
 out:
-        if (cli_req.hostname)
-                free (cli_req.hostname);//its malloced by xdr
+        free (cli_req.dict.dict_val);
 
         if (run_fsm) {
                 glusterd_friend_sm ();
@@ -707,14 +1126,24 @@ out:
 }
 
 int
-glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+glusterd_handle_cli_probe (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_cli_probe);
+}
+
+int
+__glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
-        gf1_cli_deprobe_req               cli_req = {0,};
-        uuid_t                          uuid = {0};
-        int                             op_errno = 0;
-        xlator_t                        *this = NULL;
-        glusterd_conf_t                 *priv = NULL;
+        gf_cli_req                  cli_req = {{0,},};
+        uuid_t                         uuid = {0};
+        int                        op_errno = 0;
+        xlator_t                      *this = NULL;
+        glusterd_conf_t               *priv = NULL;
+        dict_t                        *dict = NULL;
+        char                      *hostname = NULL;
+        int                            port = 0;
+        int                           flags = 0;
 
         this = THIS;
         GF_ASSERT (this);
@@ -722,62 +1151,99 @@ glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
         GF_ASSERT (priv);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-            (xdrproc_t)xdr_gf1_cli_deprobe_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len, &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "unserialize req-buffer to dictionary");
+                        goto out;
+                }
+        }
+
         gf_log ("glusterd", GF_LOG_INFO, "Received CLI deprobe req");
 
-        ret = glusterd_hostname_to_uuid (cli_req.hostname, uuid);
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get hostname");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "port", &port);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get port");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "flags", &flags);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get flags");
+                goto out;
+        }
+
+        ret = glusterd_hostname_to_uuid (hostname, uuid);
         if (ret) {
                 op_errno = GF_DEPROBE_NOT_FRIEND;
                 goto out;
         }
 
-        if (!uuid_compare (uuid, priv->uuid)) {
+        if (!uuid_compare (uuid, MY_UUID)) {
                 op_errno = GF_DEPROBE_LOCALHOST;
                 ret = -1;
                 goto out;
         }
 
-        if (!uuid_is_null (uuid) && !(cli_req.flags & GF_CLI_FLAG_OP_FORCE)) {
-                /* Check if peers are connected, except peer being detached*/
-                if (!glusterd_chk_peers_connected_befriended (uuid)) {
-                        ret = -1;
-                        op_errno = GF_DEPROBE_FRIEND_DOWN;
-                        goto out;
+        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+                if (!uuid_is_null (uuid)) {
+                        /* Check if peers are connected, except peer being detached*/
+                        if (!glusterd_chk_peers_connected_befriended (uuid)) {
+                                ret = -1;
+                                op_errno = GF_DEPROBE_FRIEND_DOWN;
+                                goto out;
+                        }
+                        ret = glusterd_all_volume_cond_check (
+                                                 glusterd_friend_brick_belongs,
+                                                 -1, &uuid);
+                        if (ret) {
+                                op_errno = GF_DEPROBE_BRICK_EXIST;
+                                goto out;
+                        }
                 }
-                ret = glusterd_all_volume_cond_check (
-                                                glusterd_friend_brick_belongs,
-                                                -1, &uuid);
-                if (ret) {
-                        op_errno = GF_DEPROBE_BRICK_EXIST;
+
+                if (glusterd_is_any_volume_in_server_quorum (this) &&
+                    !does_gd_meet_server_quorum (this)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Quorum does not "
+                                "meet, rejecting operation");
+                        ret = -1;
+                        op_errno = GF_DEPROBE_QUORUM_NOT_MET;
                         goto out;
                 }
         }
 
         if (!uuid_is_null (uuid)) {
-                ret = glusterd_deprobe_begin (req, cli_req.hostname,
-                                              cli_req.port, uuid);
+                ret = glusterd_deprobe_begin (req, hostname, port, uuid, dict);
         } else {
-                ret = glusterd_deprobe_begin (req, cli_req.hostname,
-                                              cli_req.port, NULL);
+                ret = glusterd_deprobe_begin (req, hostname, port, NULL, dict);
         }
 
-        gf_cmd_log ("peer deprobe", "on host %s:%d %s", cli_req.hostname,
-                    cli_req.port, (ret) ? "FAILED" : "SUCCESS");
 out:
+        free (cli_req.dict.dict_val);
+
         if (ret) {
-                ret = glusterd_xfer_cli_deprobe_resp (req, ret, op_errno,
-                                                      cli_req.hostname);
+                ret = glusterd_xfer_cli_deprobe_resp (req, ret, op_errno, NULL,
+                                                      hostname, dict);
         }
 
-        if (cli_req.hostname)
-                free (cli_req.hostname);//malloced by xdr
-
         glusterd_friend_sm ();
         glusterd_op_sm ();
 
@@ -785,7 +1251,13 @@ out:
 }
 
 int
-glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_cli_deprobe);
+}
+
+int
+__glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf1_cli_peer_list_req           cli_req = {0,};
@@ -793,7 +1265,9 @@ glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf1_cli_peer_list_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf1_cli_peer_list_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -831,7 +1305,14 @@ out:
 }
 
 int
-glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_list_friends);
+}
+
+int
+__glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -840,7 +1321,8 @@ glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -883,31 +1365,313 @@ out:
         return ret;
 }
 
-int32_t
-glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx)
+int
+glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
 {
-        int             ret = -1;
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_get_volume);
+}
+
+int
+__glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+{
+        int                     ret     = -1;
+        dict_t                  *dict   = NULL;
+        xlator_t                *this   = NULL;
+        glusterd_conf_t         *priv   = NULL;
+        uuid_t                  uuid    = {0};
+        gf_cli_rsp              rsp     = {0,};
+        gf_cli_req              cli_req = {{0,}};
+        char                    msg_str[2048] = {0,};
+
+        GF_ASSERT (req);
 
-        ret = glusterd_op_txn_begin (req, op, ctx);
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Received uuid reset req");
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (msg_str, sizeof (msg_str), "Unable to decode "
+                                  "the buffer");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        /* In the above section if dict_unserialize is successful, ret is set
+         * to zero.
+         */
+        ret = -1;
+        // Do not allow peer reset if there are any volumes in the cluster
+        if (!list_empty (&priv->volumes)) {
+                snprintf (msg_str, sizeof (msg_str), "volumes are already "
+                          "present in the cluster. Resetting uuid is not "
+                          "allowed");
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg_str);
+                goto out;
+        }
+
+        // Do not allow peer reset if trusted storage pool is already formed
+        if (!list_empty (&priv->peers)) {
+                snprintf (msg_str, sizeof (msg_str),"trusted storage pool "
+                          "has been already formed. Please detach this peer "
+                          "from the pool and reset its uuid.");
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg_str);
+                goto out;
+        }
+
+        uuid_copy (uuid, priv->uuid);
+        ret = glusterd_uuid_generate_save ();
+
+        if (!uuid_compare (uuid, MY_UUID)) {
+                snprintf (msg_str, sizeof (msg_str), "old uuid and the new uuid"
+                          " are same. Try gluster peer reset again");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg_str);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        if (ret) {
+                rsp.op_ret = -1;
+                if (msg_str[0] == '\0')
+                        snprintf (msg_str, sizeof (msg_str), "Operation "
+                                  "failed");
+                rsp.op_errstr = msg_str;
+                ret = 0;
+        } else {
+                rsp.op_errstr = "";
+        }
+
+        glusterd_to_cli (req, &rsp, NULL, 0, NULL,
+                         (xdrproc_t)xdr_gf_cli_rsp, dict);
 
         return ret;
 }
 
+int
+glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_uuid_reset);
+}
+
+int
+__glusterd_handle_cli_uuid_get (rpcsvc_request_t *req)
+{
+        int                     ret         = -1;
+        dict_t                  *dict       = NULL;
+        dict_t                  *rsp_dict   = NULL;
+        xlator_t                *this       = NULL;
+        glusterd_conf_t         *priv       = NULL;
+        gf_cli_rsp              rsp         = {0,};
+        gf_cli_req              cli_req     = {{0,}};
+        char                    msg_str[2048] = {0,};
+        char                    uuid_str[64] = {0,};
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
 
+        gf_log ("glusterd", GF_LOG_DEBUG, "Received uuid get req");
+
+        if (cli_req.dict.dict_len) {
+                dict  = dict_new ();
+                if (!dict) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (msg_str, sizeof (msg_str), "Unable to decode "
+                                  "the buffer");
+                        goto out;
 
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+
+                }
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        ret = dict_set_str (rsp_dict, "uuid", uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set uuid in "
+                        "dictionary.");
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize "
+                        "dictionary.");
+                goto out;
+        }
+        ret = 0;
+out:
+        if (ret) {
+                rsp.op_ret = -1;
+                if (msg_str[0] == '\0')
+                        snprintf (msg_str, sizeof (msg_str), "Operation "
+                                  "failed");
+                rsp.op_errstr = msg_str;
+
+        } else {
+                rsp.op_errstr = "";
+
+        }
+
+        glusterd_to_cli (req, &rsp, NULL, 0, NULL,
+                         (xdrproc_t)xdr_gf_cli_rsp, dict);
+
+        return 0;
+}
 int
-glusterd_handle_reset_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_uuid_get (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_uuid_get);
+}
+
+int
+__glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+{
+        int                     ret = -1;
+        dict_t                  *dict = NULL;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        int                     count = 0;
+        char                    key[1024] = {0,};
+        gf_cli_rsp              rsp = {0,};
+
+        GF_ASSERT (req);
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d", count);
+                ret = dict_set_str (dict, key, volinfo->volname);
+                if (ret)
+                        goto out;
+                count++;
+        }
+
+        ret = dict_set_int32 (dict, "count", count);
+        if (ret)
+                goto out;
+
+        ret = dict_allocate_and_serialize (dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret)
+                goto out;
+
+        ret = 0;
+
+out:
+        rsp.op_ret = ret;
+        if (ret)
+                rsp.op_errstr = "Error listing volumes";
+        else
+                rsp.op_errstr = "";
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gf_cli_rsp);
+        ret = 0;
+
+        if (dict)
+                dict_unref (dict);
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_list_volume);
+}
+
+int32_t
+glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                   char *err_str, size_t err_len)
+{
+        int             ret = -1;
+
+        ret = glusterd_op_txn_begin (req, op, ctx, err_str, err_len);
+
+        return ret;
+}
+
+int
+__glusterd_handle_reset_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
         dict_t                          *dict = NULL;
         glusterd_op_t                   cli_op = GD_OP_RESET_VOLUME;
         char                            *volname = NULL;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode request "
+                          "received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -920,8 +1684,10 @@ glusterd_handle_reset_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                     "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -930,31 +1696,37 @@ glusterd_handle_reset_volume (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
+        gf_log (this->name, GF_LOG_DEBUG, "Received volume reset request for "
+                "volume %s", volname);
 
-        gf_cmd_log ("Volume reset", "volume  : %s", volname);
-        ret = glusterd_op_begin (req, GD_OP_RESET_VOLUME, dict);
-        gf_cmd_log ("Volume reset", " on volume %s %s ", volname,
-                ((ret == 0)? " SUCCEDED":" FAILED"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_RESET_VOLUME, dict);
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
         }
 
         return ret;
 }
 
+int
+glusterd_handle_reset_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_reset_volume);
+}
 
 int
-glusterd_handle_set_volume (rpcsvc_request_t *req)
+__glusterd_handle_set_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -963,11 +1735,21 @@ glusterd_handle_set_volume (rpcsvc_request_t *req)
         char                            *key = NULL;
         char                            *value = NULL;
         char                            *volname = NULL;
+        char                            *op_errstr = NULL;
+        gf_boolean_t                    help = _gf_false;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode "
+                          "request received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -980,9 +1762,11 @@ glusterd_handle_set_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -991,65 +1775,81 @@ glusterd_handle_set_volume (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Unable to get volume name, while"
-                        "handling volume set command");
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name while handling volume set command");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        if (strcmp (volname, "help") == 0 ||
+            strcmp (volname, "help-xml") == 0) {
+                ret = glusterd_volset_help (dict, &op_errstr);
+                help = _gf_true;
                 goto out;
         }
 
         ret = dict_get_str (dict, "key1", &key);
         if (ret) {
-                if (strcmp (volname, "help-xml") && strcmp (volname, "help")) {
-                        gf_log ("", GF_LOG_WARNING, "Unable to get key, while "
-                                "handling volume set for %s",volname);
-                        goto out;
-                }
+                snprintf (err_str, sizeof (err_str), "Failed to get key while"
+                          " handling volume set for %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
         }
 
         ret = dict_get_str (dict, "value1", &value);
         if (ret) {
-                if (strcmp (volname, "help-xml") && strcmp (volname, "help")) {
-                        gf_log ("", GF_LOG_WARNING, "Unable to get value, while"
-                                "handling volume set for %s",volname);
-                        goto out;
-                }
+                snprintf (err_str, sizeof (err_str), "Failed to get value while"
+                          " handling volume set for %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
         }
+        gf_log (this->name, GF_LOG_DEBUG, "Received volume set request for "
+                "volume %s", volname);
 
+        ret = glusterd_op_begin_synctask (req, GD_OP_SET_VOLUME, dict);
 
-        gf_cmd_log ("volume set", "volume-name:%s: key:%s, value:%s",volname,
-                    key, value);
-        ret = glusterd_op_begin (req, GD_OP_SET_VOLUME, dict);
-        gf_cmd_log ("volume set", "volume-name:%s: key:%s, value:%s %s",
-                    volname, key, value, (ret == 0)? "SUCCEDED" : "FAILED" );
 out:
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
-        if (ret) {
-                if (dict)
-                        dict_unref (dict);
+        if (help)
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req, dict,
+                                                     (op_errstr)? op_errstr:"");
+        else if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
         }
+        if (op_errstr)
+                GF_FREE (op_errstr);
+
         return ret;
 }
 
 int
-glusterd_handle_sync_volume (rpcsvc_request_t *req)
+glusterd_handle_set_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_set_volume);
+}
+
+int
+__glusterd_handle_sync_volume (rpcsvc_request_t *req)
 {
         int32_t                          ret     = -1;
         gf_cli_req                       cli_req = {{0,}};
         dict_t                           *dict = NULL;
         gf_cli_rsp                       cli_rsp = {0.};
         char                             msg[2048] = {0,};
-        glusterd_volinfo_t               *volinfo = NULL;
         char                             *volname = NULL;
         gf1_cli_sync_volume              flags = 0;
         char                             *hostname = NULL;
+        xlator_t                         *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1063,9 +1863,11 @@ glusterd_handle_sync_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -1074,7 +1876,8 @@ glusterd_handle_sync_volume (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "hostname", &hostname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get hostname");
+                snprintf (msg, sizeof (msg), "Failed to get hostname");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
@@ -1082,48 +1885,25 @@ glusterd_handle_sync_volume (rpcsvc_request_t *req)
         if (ret) {
                 ret = dict_get_int32 (dict, "flags", (int32_t*)&flags);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "Unable to get volume"
-                                "name, or flags");
+                        snprintf (msg, sizeof (msg), "Failed to get volume name"
+                                  " or flags");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                         goto out;
                 }
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received volume sync req "
-                "for volume %s",
-                (flags & GF_CLI_SYNC_ALL) ? "all" : volname);
+        gf_log (this->name, GF_LOG_INFO, "Received volume sync req "
+                "for volume %s", (flags & GF_CLI_SYNC_ALL) ? "all" : volname);
 
-        if (!glusterd_is_local_addr (hostname)) {
+        if (gf_is_local_addr (hostname)) {
                 ret = -1;
                 snprintf (msg, sizeof (msg), "sync from localhost"
                           " not allowed");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
-        if (!flags) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (!ret) {
-                        snprintf (msg, sizeof (msg), "please delete the "
-                                 "volume: %s before sync", volname);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_dynmstr (dict, "volname", volname);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "volume name set failed");
-                        snprintf (msg, sizeof (msg), "volume name set failed");
-                        goto out;
-                }
-        } else {
-                if (glusterd_volume_count_get ()) {
-                        snprintf (msg, sizeof (msg), "please delete all the "
-                                 "volumes before full sync");
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        ret = glusterd_op_begin (req, GD_OP_SYNC_VOLUME, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_SYNC_VOLUME, dict);
 
 out:
         if (ret) {
@@ -1131,22 +1911,22 @@ out:
                 cli_rsp.op_errstr = msg;
                 if (msg[0] == '\0')
                         snprintf (msg, sizeof (msg), "Operation failed");
-                glusterd_submit_reply(req, &cli_rsp, NULL, 0, NULL,
-                                      (xdrproc_t)xdr_gf_cli_rsp);
-                if (dict)
-                        dict_unref (dict);
-
+                glusterd_to_cli (req, &cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
 
                 ret = 0; //sent error to cli, prevent second reply
         }
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
         return ret;
 }
 
 int
+glusterd_handle_sync_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_sync_volume);
+}
+
+int
 glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
                             char *op_errstr, dict_t *dict)
 {
@@ -1161,12 +1941,11 @@ glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
         rsp.op_errstr = op_errstr;
         if (rsp.op_ret == 0)
                 ret = dict_allocate_and_serialize (dict, &rsp.fsm_log.fsm_log_val,
-                                                (size_t *)&rsp.fsm_log.fsm_log_len);
+                                                &rsp.fsm_log.fsm_log_len);
 
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
-        if (rsp.fsm_log.fsm_log_val)
-                GF_FREE (rsp.fsm_log.fsm_log_val);
+        GF_FREE (rsp.fsm_log.fsm_log_val);
 
         gf_log ("glusterd", GF_LOG_DEBUG, "Responded, ret: %d", ret);
 
@@ -1174,7 +1953,7 @@ glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
 }
 
 int
-glusterd_handle_fsm_log (rpcsvc_request_t *req)
+__glusterd_handle_fsm_log (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf1_cli_fsm_log_req             cli_req = {0,};
@@ -1187,7 +1966,9 @@ glusterd_handle_fsm_log (rpcsvc_request_t *req)
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf1_cli_fsm_log_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf1_cli_fsm_log_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 snprintf (msg, sizeof (msg), "Garbage request");
@@ -1218,8 +1999,7 @@ glusterd_handle_fsm_log (rpcsvc_request_t *req)
         ret = glusterd_sm_tr_log_add_to_dict (dict, log);
 out:
         (void)glusterd_fsm_log_send_resp (req, ret, msg, dict);
-        if (cli_req.name)
-                free (cli_req.name);//malloced by xdr
+        free (cli_req.name);//malloced by xdr
         if (dict)
                 dict_unref (dict);
 
@@ -1230,6 +2010,12 @@ out:
 }
 
 int
+glusterd_handle_fsm_log (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_fsm_log);
+}
+
+int
 glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status)
 {
 
@@ -1243,8 +2029,7 @@ glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status)
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Responded, ret: %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to lock, ret: %d", ret);
 
         return 0;
 }
@@ -1263,35 +2048,91 @@ glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Responded to unlock, ret: %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to unlock, ret: %d", ret);
 
         return ret;
 }
 
 int
-glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+glusterd_op_mgmt_v3_lock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+                                    int32_t status)
+{
+
+        gd1_mgmt_v3_lock_rsp           rsp = {{0},};
+        int                            ret = -1;
+
+        GF_ASSERT (req);
+        GF_ASSERT (txn_id);
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+        uuid_copy (rsp.txn_id, *txn_id);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to mgmt_v3 lock, ret: %d",
+                ret);
+
+        return ret;
+}
+
+int
+glusterd_op_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+                                      int32_t status)
+{
+
+        gd1_mgmt_v3_unlock_rsp          rsp = {{0},};
+        int                             ret = -1;
+
+        GF_ASSERT (req);
+        GF_ASSERT (txn_id);
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+        glusterd_get_uuid (&rsp.uuid);
+        uuid_copy (rsp.txn_id, *txn_id);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG,
+                "Responded to mgmt_v3 unlock, ret: %d",
+                ret);
+
+        return ret;
+}
+
+int
+__glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
 {
         gd1_mgmt_cluster_unlock_req     unlock_req = {{0}, };
         int32_t                         ret = -1;
         glusterd_op_lock_ctx_t          *ctx = NULL;
         glusterd_peerinfo_t             *peerinfo = NULL;
+        xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &unlock_req,
-                             (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &unlock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
+                        "request received from peer");
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
 
-        gf_log ("glusterd", GF_LOG_INFO,
+        gf_log (this->name, GF_LOG_DEBUG,
                 "Received UNLOCK from uuid: %s", uuid_utoa (unlock_req.uuid));
 
         if (glusterd_friend_find_by_uuid (unlock_req.uuid, &peerinfo)) {
-                gf_log (THIS->name, GF_LOG_WARNING, "%s doesn't "
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
                         "belong to the cluster. Ignoring request.",
                         uuid_utoa (unlock_req.uuid));
                 ret = -1;
@@ -1306,8 +2147,9 @@ glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
         }
         uuid_copy (ctx->uuid, unlock_req.uuid);
         ctx->req = req;
+        ctx->dict = NULL;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK, ctx);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK, txn_id, ctx);
 
 out:
         glusterd_friend_sm ();
@@ -1317,14 +2159,25 @@ out:
 }
 
 int
+glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cluster_unlock);
+}
+
+int
 glusterd_op_stage_send_resp (rpcsvc_request_t   *req,
                              int32_t op, int32_t status,
                              char *op_errstr, dict_t *rsp_dict)
 {
         gd1_mgmt_stage_op_rsp           rsp      = {{0},};
         int                             ret      = -1;
+        xlator_t                       *this     = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
+
         rsp.op_ret = status;
         glusterd_get_uuid (&rsp.uuid);
         rsp.op = op;
@@ -1333,11 +2186,10 @@ glusterd_op_stage_send_resp (rpcsvc_request_t   *req,
         else
                 rsp.op_errstr = "";
 
-        ret = dict_allocate_and_serialize (rsp_dict,
-                                           &rsp.dict.dict_val,
-                                           (size_t *)&rsp.dict.dict_len);
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
         if (ret < 0) {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_ERROR,
                         "failed to get serialized length of dict");
                 return ret;
         }
@@ -1345,10 +2197,8 @@ glusterd_op_stage_send_resp (rpcsvc_request_t   *req,
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Responded to stage, ret: %d", ret);
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to stage, ret: %d", ret);
+        GF_FREE (rsp.dict.dict_val);
 
         return ret;
 }
@@ -1360,7 +2210,10 @@ glusterd_op_commit_send_resp (rpcsvc_request_t *req,
 {
         gd1_mgmt_commit_op_rsp          rsp      = {{0}, };
         int                             ret      = -1;
+        xlator_t                        *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
         rsp.op_ret = status;
         glusterd_get_uuid (&rsp.uuid);
@@ -1372,11 +2225,10 @@ glusterd_op_commit_send_resp (rpcsvc_request_t *req,
                 rsp.op_errstr = "";
 
         if (rsp_dict) {
-                ret = dict_allocate_and_serialize (rsp_dict,
-                                                   &rsp.dict.dict_val,
-                                                   (size_t *)&rsp.dict.dict_len);
+                ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                                   &rsp.dict.dict_len);
                 if (ret < 0) {
-                        gf_log ("", GF_LOG_DEBUG,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to get serialized length of dict");
                         goto out;
                 }
@@ -1386,24 +2238,24 @@ glusterd_op_commit_send_resp (rpcsvc_request_t *req,
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Responded to commit, ret: %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to commit, ret: %d", ret);
 
 out:
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
+        GF_FREE (rsp.dict.dict_val);
         return ret;
 }
 
 int
-glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+__glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
 {
         int32_t                 ret = -1;
         gd1_mgmt_friend_req     friend_req = {{0},};
         gf_boolean_t            run_fsm = _gf_true;
 
         GF_ASSERT (req);
-        if (!xdr_to_generic (req->msg[0], &friend_req, (xdrproc_t)xdr_gd1_mgmt_friend_req)) {
+        ret = xdr_to_generic (req->msg[0], &friend_req,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1422,8 +2274,7 @@ glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
         }
 
 out:
-        if (friend_req.hostname)
-                free (friend_req.hostname);//malloced by xdr
+        free (friend_req.hostname);//malloced by xdr
 
         if (run_fsm) {
                 glusterd_friend_sm ();
@@ -1434,14 +2285,23 @@ out:
 }
 
 int
-glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_incoming_friend_req);
+}
+
+int
+__glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
 {
         int32_t                 ret = -1;
         gd1_mgmt_friend_req     friend_req = {{0},};
         char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
 
         GF_ASSERT (req);
-        if (!xdr_to_generic (req->msg[0], &friend_req, (xdrproc_t)xdr_gd1_mgmt_friend_req)) {
+        ret = xdr_to_generic (req->msg[0], &friend_req,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1460,10 +2320,8 @@ glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
                                             remote_hostname, friend_req.port);
 
 out:
-        if (friend_req.hostname)
-                free (friend_req.hostname);//malloced by xdr
-        if (friend_req.vols.vols_val)
-                free (friend_req.vols.vols_val);//malloced by xdr
+        free (friend_req.hostname);//malloced by xdr
+        free (friend_req.vols.vols_val);//malloced by xdr
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
@@ -1472,6 +2330,14 @@ out:
 }
 
 int
+glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_incoming_unfriend_req);
+
+}
+
+int
 glusterd_handle_friend_update_delete (dict_t *dict)
 {
         char                    *hostname = NULL;
@@ -1517,7 +2383,7 @@ out:
 }
 
 int
-glusterd_handle_friend_update (rpcsvc_request_t *req)
+__glusterd_handle_friend_update (rpcsvc_request_t *req)
 {
         int32_t                 ret = -1;
         gd1_mgmt_friend_update     friend_req = {{0},};
@@ -1543,7 +2409,9 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
         priv = this->private;
         GF_ASSERT (priv);
 
-        if (!xdr_to_generic (req->msg[0], &friend_req, (xdrproc_t)xdr_gd1_mgmt_friend_update)) {
+        ret = xdr_to_generic (req->msg[0], &friend_req,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_update);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1603,7 +2471,13 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
                 gf_log ("", GF_LOG_INFO, "Received uuid: %s, hostname:%s",
                                 uuid_buf, hostname);
 
-                if (!uuid_compare (uuid, priv->uuid)) {
+                if (uuid_is_null (uuid)) {
+                        gf_log (this->name, GF_LOG_WARNING, "Updates mustn't "
+                                "contain peer with 'null' uuid");
+                        continue;
+                }
+
+                if (!uuid_compare (uuid, MY_UUID)) {
                         gf_log ("", GF_LOG_INFO, "Received my uuid as Friend");
                         i++;
                         continue;
@@ -1628,7 +2502,7 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
         }
 
 out:
-        uuid_copy (rsp.uuid, priv->uuid);
+        uuid_copy (rsp.uuid, MY_UUID);
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
         if (dict) {
@@ -1636,8 +2510,7 @@ out:
                         free (friend_req.friends.friends_val);//malloced by xdr
                 dict_unref (dict);
         } else {
-                if (friend_req.friends.friends_val)
-                        free (friend_req.friends.friends_val);//malloced by xdr
+                free (friend_req.friends.friends_val);//malloced by xdr
         }
 
         glusterd_friend_sm ();
@@ -1647,7 +2520,14 @@ out:
 }
 
 int
-glusterd_handle_probe_query (rpcsvc_request_t *req)
+glusterd_handle_friend_update (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_friend_update);
+}
+
+int
+__glusterd_handle_probe_query (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         xlator_t                        *this = NULL;
@@ -1661,7 +2541,9 @@ glusterd_handle_probe_query (rpcsvc_request_t *req)
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &probe_req, (xdrproc_t)xdr_gd1_mgmt_probe_req)) {
+        ret = xdr_to_generic (req->msg[0], &probe_req,
+                              (xdrproc_t)xdr_gd1_mgmt_probe_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1678,6 +2560,20 @@ glusterd_handle_probe_query (rpcsvc_request_t *req)
         gf_log ("glusterd", GF_LOG_INFO,
                 "Received probe from uuid: %s", uuid_utoa (probe_req.uuid));
 
+        /* Check for uuid collision and handle it in a user friendly way by
+         * sending the error.
+         */
+        if (!uuid_compare (probe_req.uuid, MY_UUID)) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Peer uuid %s is same as "
+                        "local uuid. Please check the uuid of both the peers "
+                        "from %s/%s", uuid_utoa (probe_req.uuid),
+                        GLUSTERD_DEFAULT_WORKDIR, GLUSTERD_INFO_FILE);
+                rsp.op_ret = -1;
+                rsp.op_errno = GF_PROBE_SAME_UUID;
+                rsp.port = port;
+                goto respond;
+        }
+
         ret = glusterd_remote_hostname_get (req, remote_hostname,
                                             sizeof (remote_hostname));
         if (ret) {
@@ -1702,20 +2598,22 @@ glusterd_handle_probe_query (rpcsvc_request_t *req)
                 }
         }
 
-        uuid_copy (rsp.uuid, conf->uuid);
+respond:
+        uuid_copy (rsp.uuid, MY_UUID);
 
         rsp.hostname = probe_req.hostname;
+        rsp.op_errstr = "";
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
+        ret = 0;
 
         gf_log ("glusterd", GF_LOG_INFO, "Responded to %s, op_ret: %d, "
-                "op_errno: %d, ret: %d", probe_req.hostname,
+                "op_errno: %d, ret: %d", remote_hostname,
                 rsp.op_ret, rsp.op_errno, ret);
 
 out:
-        if (probe_req.hostname)
-                free (probe_req.hostname);//malloced by xdr
+        free (probe_req.hostname);//malloced by xdr
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
@@ -1723,8 +2621,13 @@ out:
         return ret;
 }
 
+int glusterd_handle_probe_query (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_probe_query);
+}
+
 int
-glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret     = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -1732,10 +2635,15 @@ glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
         glusterd_op_t                   cli_op = GD_OP_PROFILE_VOLUME;
         char                            *volname = NULL;
         int32_t                         op = 0;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1751,42 +2659,50 @@ glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_log (THIS->name, GF_LOG_INFO, "Received volume profile req "
+        gf_log (this->name, GF_LOG_INFO, "Received volume profile req "
                 "for volume %s", volname);
         ret = dict_get_int32 (dict, "op", &op);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get op");
+                snprintf (err_str, sizeof (err_str), "Unable to get operation");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_cmd_log ("Volume stats", "volume  : %s, op: %d", volname, op);
-        ret = glusterd_op_begin (req, cli_op, dict);
-        gf_cmd_log ("Volume stats", " on volume %s, op: %d %s ",
-                    volname, op,
-                    ((ret == 0)? " SUCCEDED":" FAILED"));
+        ret = glusterd_op_begin (req, cli_op, dict, err_str, sizeof (err_str));
 
 out:
         glusterd_friend_sm ();
         glusterd_op_sm ();
 
-        if (ret && dict)
-                dict_unref (dict);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);
-        if (ret)
+        free (cli_req.dict.dict_val);
+
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
+        }
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
-glusterd_handle_getwd (rpcsvc_request_t *req)
+glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_profile_volume);
+}
+
+int
+__glusterd_handle_getwd (rpcsvc_request_t *req)
 {
         int32_t                 ret = -1;
         gf1_cli_getwd_rsp     rsp = {0,};
@@ -1801,8 +2717,9 @@ glusterd_handle_getwd (rpcsvc_request_t *req)
 
         rsp.wd = priv->workdir;
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+        ret = 0;
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
@@ -1810,18 +2727,27 @@ glusterd_handle_getwd (rpcsvc_request_t *req)
         return ret;
 }
 
+int
+glusterd_handle_getwd (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_getwd);
+}
 
 int
-glusterd_handle_mount (rpcsvc_request_t *req)
+__glusterd_handle_mount (rpcsvc_request_t *req)
 {
         gf1_cli_mount_req mnt_req = {0,};
         gf1_cli_mount_rsp rsp     = {0,};
         dict_t *dict              = NULL;
         int ret                   = 0;
+        glusterd_conf_t     *priv   = NULL;
 
         GF_ASSERT (req);
+	priv = THIS->private;
 
-        if (!xdr_to_generic (req->msg[0], &mnt_req, (xdrproc_t)xdr_gf1_cli_mount_req)) {
+        ret = xdr_to_generic (req->msg[0], &mnt_req,
+                              (xdrproc_t)xdr_gf1_cli_mount_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 rsp.op_ret = -1;
@@ -1850,15 +2776,18 @@ glusterd_handle_mount (rpcsvc_request_t *req)
                 }
         }
 
+	synclock_unlock (&priv->big_lock);
         rsp.op_ret = glusterd_do_mount (mnt_req.label, dict,
                                         &rsp.path, &rsp.op_errno);
+	synclock_lock (&priv->big_lock);
 
  out:
         if (!rsp.path)
                 rsp.path = "";
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_mount_rsp);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_mount_rsp);
+        ret = 0;
 
         if (dict)
                 dict_unref (dict);
@@ -1872,7 +2801,13 @@ glusterd_handle_mount (rpcsvc_request_t *req)
 }
 
 int
-glusterd_handle_umount (rpcsvc_request_t *req)
+glusterd_handle_mount (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_mount);
+}
+
+int
+__glusterd_handle_umount (rpcsvc_request_t *req)
 {
         gf1_cli_umount_req umnt_req = {0,};
         gf1_cli_umount_rsp rsp      = {0,};
@@ -1885,11 +2820,15 @@ glusterd_handle_umount (rpcsvc_request_t *req)
         gf_boolean_t dir_ok         = _gf_false;
         char *pdir                  = NULL;
         char *t                     = NULL;
+        glusterd_conf_t     *priv   = NULL;
 
         GF_ASSERT (req);
         GF_ASSERT (this);
+        priv = this->private;
 
-        if (!xdr_to_generic (req->msg[0], &umnt_req, (xdrproc_t)xdr_gf1_cli_umount_req)) {
+        ret = xdr_to_generic (req->msg[0], &umnt_req,
+                              (xdrproc_t)xdr_gf1_cli_umount_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 rsp.op_ret = -1;
@@ -1928,7 +2867,9 @@ glusterd_handle_umount (rpcsvc_request_t *req)
         runner_add_args (&runner, "umount", umnt_req.path, NULL);
         if (umnt_req.lazy)
                 runner_add_arg (&runner, "-l");
+        synclock_unlock (&priv->big_lock);
         rsp.op_ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
         if (rsp.op_ret == 0) {
                 if (realpath (umnt_req.path, mntp))
                         rmdir (mntp);
@@ -1946,8 +2887,9 @@ glusterd_handle_umount (rpcsvc_request_t *req)
         if (rsp.op_errno)
                 rsp.op_ret = -1;
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_umount_rsp);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_umount_rsp);
+        ret = 0;
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
@@ -1956,6 +2898,12 @@ glusterd_handle_umount (rpcsvc_request_t *req)
 }
 
 int
+glusterd_handle_umount (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_umount);
+}
+
+int
 glusterd_friend_remove (uuid_t uuid, char *hostname)
 {
         int                           ret = 0;
@@ -1988,8 +2936,9 @@ glusterd_rpc_create (struct rpc_clnt **rpc,
         GF_ASSERT (this);
 
         GF_ASSERT (options);
-        new_rpc = rpc_clnt_new (options, this->ctx, this->name);
 
+        /* TODO: is 32 enough? or more ? */
+        new_rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
         if (!new_rpc)
                 goto out;
 
@@ -2005,7 +2954,7 @@ out:
                 }
         }
 
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
@@ -2028,8 +2977,8 @@ glusterd_transport_keepalive_options_get (int *interval, int *time)
 }
 
 int
-glusterd_transport_inet_keepalive_options_build (dict_t **options,
-                                                 const char *hostname, int port)
+glusterd_transport_inet_options_build (dict_t **options, const char *hostname,
+                                       int port)
 {
         dict_t  *dict = NULL;
         int32_t interval = -1;
@@ -2041,10 +2990,25 @@ glusterd_transport_inet_keepalive_options_build (dict_t **options,
 
         if (!port)
                 port = GLUSTERD_DEFAULT_PORT;
+
+        /* Build default transport options */
         ret = rpc_transport_inet_options_build (&dict, hostname, port);
         if (ret)
                 goto out;
 
+        /* Set frame-timeout to 10mins. Default timeout of 30 mins is too long
+         * when compared to 2 mins for cli timeout. This ensures users don't
+         * wait too long after cli timesout before being able to resume normal
+         * operations
+         */
+        ret = dict_set_int32 (dict, "frame-timeout", 600);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "Failed to set frame-timeout");
+                goto out;
+        }
+
+        /* Set keepalive options */
         glusterd_transport_keepalive_options_get (&interval, &time);
 
         if ((interval > 0) || (time > 0))
@@ -2056,65 +3020,117 @@ out:
 }
 
 int
-glusterd_friend_add (const char *hoststr, int port,
-                     glusterd_friend_sm_state_t state,
-                     uuid_t *uuid,
-                     glusterd_peerinfo_t **friend,
-                     gf_boolean_t restore,
-                     glusterd_peerctx_args_t *args)
+glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                            glusterd_peerctx_args_t *args)
 {
-        int                     ret = 0;
-        glusterd_conf_t        *conf = NULL;
-        glusterd_peerctx_t     *peerctx = NULL;
         dict_t                 *options = NULL;
-        gf_boolean_t            handover = _gf_false;
-
-        conf = THIS->private;
-        GF_ASSERT (conf)
-        GF_ASSERT (hoststr);
+        int                    ret = -1;
+        glusterd_peerctx_t     *peerctx = NULL;
+        data_t                 *data = NULL;
 
         peerctx = GF_CALLOC (1, sizeof (*peerctx), gf_gld_mt_peerctx_t);
-        if (!peerctx) {
-                ret = -1;
+        if (!peerctx)
                 goto out;
-        }
 
         if (args)
                 peerctx->args = *args;
 
-        ret = glusterd_peerinfo_new (friend, state, uuid, hoststr);
+        peerctx->peerinfo = peerinfo;
+
+        ret = glusterd_transport_inet_options_build (&options,
+                                                     peerinfo->hostname,
+                                                     peerinfo->port);
         if (ret)
                 goto out;
 
-        peerctx->peerinfo = *friend;
+        /*
+         * For simulated multi-node testing, we need to make sure that we
+         * create our RPC endpoint with the same address that the peer would
+         * use to reach us.
+         */
+        if (this->options) {
+                data = dict_get(this->options,"transport.socket.bind-address");
+                if (data) {
+                        ret = dict_set(options,
+                                       "transport.socket.source-addr",data);
+                }
+        }
 
-        ret = glusterd_transport_inet_keepalive_options_build (&options,
-                                                         hoststr, port);
-        if (ret)
+        ret = glusterd_rpc_create (&peerinfo->rpc, options,
+                                   glusterd_peer_rpc_notify, peerctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to create rpc for"
+                        " peer %s", peerinfo->hostname);
                 goto out;
+        }
+        peerctx = NULL;
+        ret = 0;
+out:
+        GF_FREE (peerctx);
+        return ret;
+}
 
-        ret = glusterd_rpc_create (&(*friend)->rpc, options,
-                                   glusterd_peer_rpc_notify,
-                                   peerctx);
+int
+glusterd_friend_add (const char *hoststr, int port,
+                     glusterd_friend_sm_state_t state,
+                     uuid_t *uuid,
+                     glusterd_peerinfo_t **friend,
+                     gf_boolean_t restore,
+                     glusterd_peerctx_args_t *args)
+{
+        int                     ret = 0;
+        xlator_t               *this = NULL;
+        glusterd_conf_t        *conf = NULL;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (conf);
+        GF_ASSERT (hoststr);
+
+        ret = glusterd_peerinfo_new (friend, state, uuid, hoststr, port);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "failed to create rpc for"
-                        " peer %s", (char*)hoststr);
                 goto out;
         }
-        handover = _gf_true;
 
-out:
-        if (ret && !handover) {
-                        (void) glusterd_friend_cleanup (*friend);
-                        *friend = NULL;
+        /*
+         * We can't add to the list after calling glusterd_friend_rpc_create,
+         * even if it succeeds, because by then the callback to take it back
+         * off and free might have happened already (notably in the case of an
+         * invalid peer name).  That would mean we're adding something that had
+         * just been free, and we're likely to crash later.
+         */
+        list_add_tail (&(*friend)->uuid_list, &conf->peers);
+
+        //restore needs to first create the list of peers, then create rpcs
+        //to keep track of quorum in race-free manner. In restore for each peer
+        //rpc-create calls rpc_notify when the friend-list is partially
+        //constructed, leading to wrong quorum calculations.
+        if (!restore) {
+                ret = glusterd_store_peerinfo (*friend);
+                if (ret == 0) {
+                        synclock_unlock (&conf->big_lock);
+                        ret = glusterd_friend_rpc_create (this, *friend, args);
+                        synclock_lock (&conf->big_lock);
+                }
+                else {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store peerinfo");
+                }
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "connect returned %d", ret);
+        if (ret) {
+                (void) glusterd_friend_cleanup (*friend);
+                *friend = NULL;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_INFO, "connect returned %d", ret);
         return ret;
 }
 
 int
-glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
+glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
+                      dict_t *dict)
 {
         int                             ret = -1;
         glusterd_peerinfo_t             *peerinfo = NULL;
@@ -2130,6 +3146,7 @@ glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
                         " for host: %s (%d)", hoststr, port);
                 args.mode = GD_MODE_ON;
                 args.req  = req;
+                args.dict = dict;
                 ret = glusterd_friend_add ((char *)hoststr, port,
                                            GD_FRIEND_STATE_DEFAULT,
                                            NULL, &peerinfo, 0, &args);
@@ -2150,11 +3167,12 @@ glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
                         event->peerinfo = peerinfo;
                         ret = glusterd_friend_sm_inject_event (event);
                         glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_SUCCESS,
-                                                      (char*)hoststr, port);
+                                                      NULL, (char*)hoststr,
+                                                      port, dict);
                 }
         } else {
-                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND,
-                                              (char*)hoststr, port);
+                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND, NULL,
+                                              (char*)hoststr, port, dict);
         }
 
 out:
@@ -2164,7 +3182,7 @@ out:
 
 int
 glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
-                        uuid_t uuid)
+                        uuid_t uuid, dict_t *dict)
 {
         int                             ret = -1;
         glusterd_peerinfo_t             *peerinfo = NULL;
@@ -2205,6 +3223,7 @@ glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
         ctx->hostname = gf_strdup (hoststr);
         ctx->port = port;
         ctx->req = req;
+        ctx->dict = dict;
 
         event->ctx = ctx;
 
@@ -2239,7 +3258,7 @@ glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int por
 
         conf = this->private;
 
-        uuid_copy (rsp.uuid, conf->uuid);
+        uuid_copy (rsp.uuid, MY_UUID);
         rsp.hostname = hostname;
         rsp.port = port;
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
@@ -2252,76 +3271,236 @@ glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int por
 
 
 int
-glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *hostname, int port,
-                               int32_t op_ret, int32_t op_errno)
+glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *myhostname,
+                               char *remote_hostname, int port, int32_t op_ret,
+                               int32_t op_errno)
 {
         gd1_mgmt_friend_rsp  rsp = {{0}, };
         int32_t              ret = -1;
         xlator_t             *this = NULL;
         glusterd_conf_t      *conf = NULL;
 
-        GF_ASSERT (hostname);
+        GF_ASSERT (myhostname);
 
         this = THIS;
         GF_ASSERT (this);
 
         conf = this->private;
 
-        uuid_copy (rsp.uuid, conf->uuid);
+        uuid_copy (rsp.uuid, MY_UUID);
         rsp.op_ret = op_ret;
         rsp.op_errno = op_errno;
-        rsp.hostname = gf_strdup (hostname);
+        rsp.hostname = gf_strdup (myhostname);
         rsp.port = port;
 
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
 
         gf_log ("glusterd", GF_LOG_INFO,
-                "Responded to %s (%d), ret: %d", hostname, port, ret);
-        if (rsp.hostname)
-                GF_FREE (rsp.hostname)
+                "Responded to %s (%d), ret: %d", remote_hostname, port, ret);
+        GF_FREE (rsp.hostname);
         return ret;
 }
 
+static void
+set_probe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
+                     size_t len, char *hostname, int port)
+{
+        if ((op_errstr) && (strcmp (op_errstr, ""))) {
+                snprintf (errstr, len, "%s", op_errstr);
+                return;
+        }
+
+        if (!op_ret) {
+                switch (op_errno) {
+                        case GF_PROBE_LOCALHOST:
+                                snprintf (errstr, len, "Probe on localhost not "
+                                          "needed");
+                                break;
+
+                        case GF_PROBE_FRIEND:
+                                snprintf (errstr, len, "Host %s port %d already"
+                                          " in peer list", hostname, port);
+                                break;
+
+                        default:
+                                if (op_errno != 0)
+                                        snprintf (errstr, len, "Probe returned "
+                                                  "with unknown errno %d",
+                                                  op_errno);
+                                break;
+                }
+        } else {
+                switch (op_errno) {
+                        case GF_PROBE_ANOTHER_CLUSTER:
+                                snprintf (errstr, len, "%s is already part of "
+                                          "another cluster", hostname);
+                                break;
+
+                        case GF_PROBE_VOLUME_CONFLICT:
+                                snprintf (errstr, len, "Atleast one volume on "
+                                          "%s conflicts with existing volumes "
+                                          "in the cluster", hostname);
+                                break;
+
+                        case GF_PROBE_UNKNOWN_PEER:
+                                snprintf (errstr, len, "%s responded with "
+                                          "'unknown peer' error, this could "
+                                          "happen if %s doesn't have localhost "
+                                          "in its peer database", hostname,
+                                          hostname);
+                                break;
+
+                        case GF_PROBE_ADD_FAILED:
+                                snprintf (errstr, len, "Failed to add peer "
+                                          "information on %s", hostname);
+                                break;
+
+                        case GF_PROBE_SAME_UUID:
+                                snprintf (errstr, len, "Peer uuid (host %s) is "
+                                          "same as local uuid", hostname);
+                                break;
+
+                        case GF_PROBE_QUORUM_NOT_MET:
+                                snprintf (errstr, len, "Cluster quorum is not "
+                                          "met. Changing peers is not allowed "
+                                          "in this state");
+                                break;
+
+                        default:
+                                snprintf (errstr, len, "Probe returned with "
+                                          "unknown errno %d", op_errno);
+                                break;
+                }
+        }
+}
+
 int
 glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                              int32_t op_errno, char *hostname, int port)
+                              int32_t op_errno, char *op_errstr, char *hostname,
+                              int port, dict_t *dict)
 {
-        gf1_cli_probe_rsp    rsp = {0, };
+        gf_cli_rsp           rsp = {0,};
         int32_t              ret = -1;
+        char        errstr[2048] = {0,};
+        char            *cmd_str = NULL;
+        xlator_t           *this = THIS;
 
         GF_ASSERT (req);
+        GF_ASSERT (this);
+
+        (void) set_probe_error_str (op_ret, op_errno, op_errstr, errstr,
+                                    sizeof (errstr), hostname, port);
+
+        if (dict) {
+                ret = dict_get_str (dict, "cmd-str", &cmd_str);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "command string");
+        }
 
         rsp.op_ret = op_ret;
         rsp.op_errno = op_errno;
-        rsp.hostname = hostname;
-        rsp.port = port;
+        rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
+
+        gf_cmd_log ("", "%s : %s %s %s", cmd_str,
+                    (op_ret) ? "FAILED" : "SUCCESS",
+                    (errstr[0] != '\0') ? ":" : " ",
+                    (errstr[0] != '\0') ? errstr : " ");
 
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_probe_rsp);
+                                     (xdrproc_t)xdr_gf_cli_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO, "Responded to CLI, ret: %d",ret);
+        if (dict)
+                dict_unref (dict);
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to CLI, ret: %d",ret);
 
         return ret;
 }
 
+static void
+set_deprobe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
+                       size_t len, char *hostname)
+{
+        if ((op_errstr) && (strcmp (op_errstr, ""))) {
+                snprintf (errstr, len, "%s", op_errstr);
+                return;
+        }
+
+        if (op_ret) {
+                switch (op_errno) {
+                        case GF_DEPROBE_LOCALHOST:
+                                snprintf (errstr, len, "%s is localhost",
+                                hostname);
+                                break;
+
+                        case GF_DEPROBE_NOT_FRIEND:
+                                snprintf (errstr, len, "%s is not part of "
+                                          "cluster", hostname);
+                                break;
+
+                        case GF_DEPROBE_BRICK_EXIST:
+                                snprintf (errstr, len, "Brick(s) with the peer "
+                                          "%s exist in cluster", hostname);
+                                break;
+
+                        case GF_DEPROBE_FRIEND_DOWN:
+                                snprintf (errstr, len, "One of the peers is "
+                                          "probably down. Check with "
+                                          "'peer status'");
+                                break;
+
+                        case GF_DEPROBE_QUORUM_NOT_MET:
+                                snprintf (errstr, len, "Cluster quorum is not "
+                                          "met. Changing peers is not allowed "
+                                          "in this state");
+                                break;
+
+                        default:
+                                snprintf (errstr, len, "Detach returned with "
+                                          "unknown errno %d", op_errno);
+                                break;
+
+                }
+        }
+}
+
+
 int
 glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                                int32_t op_errno, char *hostname)
+                                int32_t op_errno, char *op_errstr,
+                                char *hostname, dict_t *dict)
 {
-        gf1_cli_deprobe_rsp    rsp = {0, };
+        gf_cli_rsp             rsp = {0,};
         int32_t                ret = -1;
+        char              *cmd_str = NULL;
+        char          errstr[2048] = {0,};
 
         GF_ASSERT (req);
 
+        (void) set_deprobe_error_str (op_ret, op_errno, op_errstr, errstr,
+                                      sizeof (errstr), hostname);
+
+        if (dict) {
+                ret = dict_get_str (dict, "cmd-str", &cmd_str);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_ERROR, "Failed to get "
+                                "command string");
+        }
+
         rsp.op_ret = op_ret;
         rsp.op_errno = op_errno;
-        rsp.hostname = hostname;
+        rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
+
+        gf_cmd_log ("", "%s : %s %s %s", cmd_str,
+                    (op_ret) ? "FAILED" : "SUCCESS",
+                    (errstr[0] != '\0') ? ":" : " ",
+                    (errstr[0] != '\0') ? errstr : " ");
 
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_deprobe_rsp);
+                                     (xdrproc_t)xdr_gf_cli_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO, "Responded to CLI, ret: %d",ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to CLI, ret: %d",ret);
 
         return ret;
 }
@@ -2335,39 +3514,52 @@ glusterd_list_friends (rpcsvc_request_t *req, dict_t *dict, int32_t flags)
         int32_t                 count = 0;
         dict_t                  *friends = NULL;
         gf1_cli_peer_list_rsp   rsp = {0,};
+        char                    my_uuid_str[64] = {0,};
+        char                    key[256] = {0,};
 
         priv = THIS->private;
         GF_ASSERT (priv);
 
-        if (!list_empty (&priv->peers)) {
-                friends = dict_new ();
-                if (!friends) {
-                        gf_log ("", GF_LOG_WARNING, "Out of Memory");
-                        goto out;
-                }
-        } else {
-                ret = 0;
+        friends = dict_new ();
+        if (!friends) {
+                gf_log ("", GF_LOG_WARNING, "Out of Memory");
                 goto out;
         }
-
-        if (flags == GF_CLI_LIST_ALL) {
-                        list_for_each_entry (entry, &priv->peers, uuid_list) {
-                                count++;
-                                ret = glusterd_add_peer_detail_to_dict (entry,
+        if (!list_empty (&priv->peers)) {
+                list_for_each_entry (entry, &priv->peers, uuid_list) {
+                        count++;
+                        ret = glusterd_add_peer_detail_to_dict (entry,
                                                                 friends, count);
-                                if (ret)
-                                        goto out;
+                        if (ret)
+                                goto out;
+                }
+        }
 
-                        }
+        if (flags == GF_CLI_LIST_POOL_NODES) {
+                count++;
+                snprintf (key, 256, "friend%d.uuid", count);
+                uuid_utoa_r (MY_UUID, my_uuid_str);
+                ret = dict_set_str (friends, key, my_uuid_str);
+                if (ret)
+                        goto out;
 
-                        ret = dict_set_int32 (friends, "count", count);
+                snprintf (key, 256, "friend%d.hostname", count);
+                ret = dict_set_str (friends, key, "localhost");
+                if (ret)
+                        goto out;
 
-                        if (ret)
-                                goto out;
+                snprintf (key, 256, "friend%d.connected", count);
+                ret = dict_set_int32 (friends, key, 1);
+                if (ret)
+                        goto out;
         }
 
+        ret = dict_set_int32 (friends, "count", count);
+        if (ret)
+                goto out;
+
         ret = dict_allocate_and_serialize (friends, &rsp.friends.friends_val,
-                                           (size_t *)&rsp.friends.friends_len);
+                                           &rsp.friends.friends_len);
 
         if (ret)
                 goto out;
@@ -2380,10 +3572,10 @@ out:
 
         rsp.op_ret = ret;
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
-        if (rsp.friends.friends_val)
-                GF_FREE (rsp.friends.friends_val);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
+        ret = 0;
+        GF_FREE (rsp.friends.friends_val);
 
         return ret;
 }
@@ -2474,7 +3666,7 @@ respond:
         if (ret)
                 goto out;
         ret = dict_allocate_and_serialize (volumes, &rsp.dict.dict_val,
-                                           (size_t *)&rsp.dict.dict_len);
+                                           &rsp.dict.dict_len);
 
         if (ret)
                 goto out;
@@ -2484,19 +3676,19 @@ out:
         rsp.op_ret = ret;
 
         rsp.op_errstr = "";
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_cli_rsp);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf_cli_rsp);
+        ret = 0;
 
         if (volumes)
                 dict_unref (volumes);
 
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
+        GF_FREE (rsp.dict.dict_val);
         return ret;
 }
 
 int
-glusterd_handle_status_volume (rpcsvc_request_t *req)
+__glusterd_handle_status_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret     = -1;
         uint32_t                        cmd     = 0;
@@ -2504,11 +3696,15 @@ glusterd_handle_status_volume (rpcsvc_request_t *req)
         char                           *volname = 0;
         gf_cli_req                      cli_req = {{0,}};
         glusterd_op_t                   cli_op  = GD_OP_STATUS_VOLUME;
+        char                            err_str[2048] = {0,};
+        xlator_t                       *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -2521,8 +3717,10 @@ glusterd_handle_status_volume (rpcsvc_request_t *req)
                 ret = dict_unserialize (cli_req.dict.dict_val,
                                         cli_req.dict.dict_len, &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                 "unserialize buffer");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 }
 
@@ -2535,46 +3733,172 @@ glusterd_handle_status_volume (rpcsvc_request_t *req)
         if (!(cmd & GF_CLI_STATUS_ALL)) {
                 ret = dict_get_str (dict, "volname", &volname);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "failed to get volname");
+                        snprintf (err_str, sizeof (err_str), "Unable to get "
+                                  "volume name");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         goto out;
                 }
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "Received status volume req "
-                        "for volume %s", volname);
+                gf_log (this->name, GF_LOG_INFO,
+                        "Received status volume req for volume %s", volname);
 
         }
 
-        ret = glusterd_op_begin (req, GD_OP_STATUS_VOLUME, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_STATUS_VOLUME, dict);
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        free (cli_req.dict.dict_val);
 
-        if (ret)
+        return ret;
+}
+
+int
+glusterd_handle_status_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_status_volume);
+}
+
+int
+__glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        glusterd_op_t                   cli_op = GD_OP_CLEARLOCKS_VOLUME;
+        char                            *volname = NULL;
+        dict_t                          *dict = NULL;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = -1;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to unserialize req-buffer to"
+                                " dictionary");
+                        snprintf (err_str, sizeof (err_str), "unable to decode "
+                                  "the command");
+                        goto out;
+                }
+
+        } else {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Empty cli request.");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Received clear-locks volume req "
+                "for volume %s", volname);
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_CLEARLOCKS_VOLUME, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);
+                                                     dict, err_str);
+        }
+        free (cli_req.dict.dict_val);
 
         return ret;
 }
 
 int
-glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event,
-                          void *data)
+glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_clearlocks_volume);
+}
+
+static int
+get_brickinfo_from_brickid (char *brickid, glusterd_brickinfo_t **brickinfo)
+{
+        glusterd_volinfo_t      *volinfo    = NULL;
+        char                    *volid_str  = NULL;
+        char                    *brick      = NULL;
+        char                    *brickid_dup = NULL;
+        uuid_t                  volid       = {0};
+        int                     ret         = -1;
+
+        brickid_dup = gf_strdup (brickid);
+        if (!brickid_dup)
+                goto out;
+
+        volid_str = brickid_dup;
+        brick = strchr (brickid_dup, ':');
+        *brick = '\0';
+        brick++;
+        if (!volid_str || !brick)
+                goto out;
+
+        uuid_parse (volid_str, volid);
+        ret = glusterd_volinfo_find_by_volume_id (volid, &volinfo);
+        if (ret) {
+                /* Check if it a snapshot volume */
+                ret = glusterd_snap_volinfo_find_by_volume_id (volid, &volinfo);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                      brickinfo);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        GF_FREE (brickid_dup);
+        return ret;
+}
+
+int
+__glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
 {
         xlator_t                *this = NULL;
         glusterd_conf_t         *conf = NULL;
         int                     ret = 0;
+        char                    *brickid = NULL;
         glusterd_brickinfo_t    *brickinfo = NULL;
 
-        brickinfo = mydata;
-        if (!brickinfo)
+        brickid = mydata;
+        if (!brickid)
+                return 0;
+
+        ret = get_brickinfo_from_brickid (brickid, &brickinfo);
+        if (ret)
                 return 0;
 
         this = THIS;
@@ -2584,15 +3908,21 @@ glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
 
         switch (event) {
         case RPC_CLNT_CONNECT:
-                gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
+                gf_log (this->name, GF_LOG_DEBUG, "Connected to %s:%s",
+                        brickinfo->hostname, brickinfo->path);
                 glusterd_set_brick_status (brickinfo, GF_BRICK_STARTED);
                 ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
 
                 break;
 
         case RPC_CLNT_DISCONNECT:
-                gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT");
+                if (GF_BRICK_STARTED == brickinfo->status)
+                        gf_log (this->name, GF_LOG_INFO, "Disconnected from "
+                                "%s:%s", brickinfo->hostname, brickinfo->path);
+
                 glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
+                if (rpc_clnt_is_disabled (rpc))
+                        GF_FREE (brickid);
                 break;
 
         default:
@@ -2605,12 +3935,20 @@ glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
 }
 
 int
-glusterd_shd_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                         rpc_clnt_event_t event,
-                         void *data)
+glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                           rpc_clnt_event_t event, void *data)
+{
+        return glusterd_big_locked_notify (rpc, mydata, event, data,
+                                           __glusterd_brick_rpc_notify);
+}
+
+int
+__glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                               rpc_clnt_event_t event, void *data)
 {
         xlator_t                *this = NULL;
         glusterd_conf_t         *conf = NULL;
+        char                    *server = NULL;
         int                     ret = 0;
 
         this = THIS;
@@ -2618,17 +3956,21 @@ glusterd_shd_rpc_notify (struct rpc_clnt *rpc, void *mydata,
         conf = this->private;
         GF_ASSERT (conf);
 
+        server = mydata;
+        if (!server)
+                return 0;
+
         switch (event) {
         case RPC_CLNT_CONNECT:
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
-                (void) glusterd_shd_set_running (_gf_true);
+                (void) glusterd_nodesvc_set_online_status (server, _gf_true);
                 ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
 
                 break;
 
         case RPC_CLNT_DISCONNECT:
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT");
-                (void) glusterd_shd_set_running (_gf_false);
+                (void) glusterd_nodesvc_set_online_status (server, _gf_false);
                 break;
 
         default:
@@ -2641,10 +3983,29 @@ glusterd_shd_rpc_notify (struct rpc_clnt *rpc, void *mydata,
 }
 
 int
-glusterd_friend_remove_notify (glusterd_peerinfo_t *peerinfo, rpcsvc_request_t *req)
+glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                             rpc_clnt_event_t event, void *data)
 {
-        int ret = -1;
-        glusterd_friend_sm_event_t *new_event = NULL;
+        return glusterd_big_locked_notify (rpc, mydata, event, data,
+                                           __glusterd_nodesvc_rpc_notify);
+}
+
+int
+glusterd_friend_remove_notify (glusterd_peerctx_t *peerctx)
+{
+        int                             ret = -1;
+        glusterd_friend_sm_event_t      *new_event = NULL;
+        glusterd_peerinfo_t             *peerinfo = peerctx->peerinfo;
+        rpcsvc_request_t                *req = peerctx->args.req;
+        char                            *errstr = peerctx->errstr;
+        dict_t                          *dict = NULL;
+
+        GF_ASSERT (peerctx);
+
+        peerinfo = peerctx->peerinfo;
+        req = peerctx->args.req;
+        dict = peerctx->args.dict;
+        errstr = peerctx->errstr;
 
         ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
                                             &new_event);
@@ -2656,12 +4017,12 @@ glusterd_friend_remove_notify (glusterd_peerinfo_t *peerinfo, rpcsvc_request_t *
                         goto out;
                 }
 
-                glusterd_xfer_cli_probe_resp (req, -1, ENOTCONN,
-                                              peerinfo->hostname, peerinfo->port);
+                glusterd_xfer_cli_probe_resp (req, -1, ENOTCONN, errstr,
+                                              peerinfo->hostname,
+                                              peerinfo->port, dict);
 
                 new_event->peerinfo = peerinfo;
                 ret = glusterd_friend_sm_inject_event (new_event);
-                glusterd_friend_sm ();
 
         } else {
                 gf_log ("glusterd", GF_LOG_ERROR,
@@ -2674,17 +4035,16 @@ out:
 }
 
 int
-glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event,
-                          void *data)
+__glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                            rpc_clnt_event_t event, void *data)
 {
         xlator_t             *this        = NULL;
         glusterd_conf_t      *conf        = NULL;
         int                   ret         = 0;
         glusterd_peerinfo_t  *peerinfo    = NULL;
         glusterd_peerctx_t   *peerctx     = NULL;
-        uuid_t                owner       = {0,};
-        uuid_t               *peer_uuid   = NULL;
+        gf_boolean_t         quorum_action = _gf_false;
+        glusterd_volinfo_t   *volinfo     = NULL;
 
         peerctx = mydata;
         if (!peerctx)
@@ -2699,17 +4059,9 @@ glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
         {
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
                 peerinfo->connected = 1;
-                ret = glusterd_store_peerinfo (peerinfo);
-                if (ret) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to store "
-                                "peerinfo");
-                        break;
-                }
-
-                list_add_tail (&peerinfo->uuid_list, &conf->peers);
+                peerinfo->quorum_action = _gf_true;
 
-                ret = glusterd_peer_handshake (this, rpc, peerctx);
+                ret = glusterd_peer_dump_version (this, rpc, peerctx);
                 if (ret)
                         gf_log ("", GF_LOG_ERROR, "glusterd handshake failed");
                 break;
@@ -2720,43 +4072,36 @@ glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT %d",
                         peerinfo->state.state);
 
-                peerinfo->connected = 0;
+                if (peerinfo->connected) {
+                        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                                ret = glusterd_mgmt_v3_unlock (volinfo->volname,
+                                                               peerinfo->uuid,
+                                                               "vol");
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_TRACE,
+                                                "Lock not released for %s",
+                                                volinfo->volname);
+                        }
 
-                /*
-                  local glusterd (thinks that it) is the owner of the cluster
-                  lock and 'fails' the operation on the first disconnect from
-                  a peer.
-                */
-                glusterd_get_lock_owner (&owner);
-                if (!uuid_compare (conf->uuid, owner)) {
-                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_START_UNLOCK,
-                                                           NULL);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR, "Unable"
-                                        " to enqueue cluster unlock event");
-                        break;
+                        ret = 0;
                 }
 
-                peer_uuid = GF_CALLOC (1, sizeof (*peer_uuid), gf_common_mt_char);
-                if (!peer_uuid) {
-                        ret = -1;
-                        break;
+                if ((peerinfo->quorum_contrib != QUORUM_DOWN) &&
+                    (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)) {
+                        peerinfo->quorum_contrib = QUORUM_DOWN;
+                        quorum_action = _gf_true;
+                        peerinfo->quorum_action = _gf_false;
                 }
 
-                uuid_copy (*peer_uuid, peerinfo->uuid);
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP,
-                                                   peer_uuid);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Unable"
-                                " to enque local lock flush event.");
-
-                //Inject friend disconnected here
-                if (peerinfo->state.state == GD_FRIEND_STATE_DEFAULT)  {
-                        glusterd_friend_remove_notify (peerinfo,
-                                                       peerctx->args.req);
+                /* Remove peer if it is not a friend and connection/handshake
+                *  fails, and notify cli. Happens only during probe.
+                */
+                if (peerinfo->state.state == GD_FRIEND_STATE_DEFAULT) {
+                        glusterd_friend_remove_notify (peerctx);
+                        goto out;
                 }
 
-                //default_notify (this, GF_EVENT_CHILD_DOWN, NULL);
+                peerinfo->connected = 0;
                 break;
         }
         default:
@@ -2766,12 +4111,23 @@ glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
                 break;
         }
 
+out:
         glusterd_friend_sm ();
         glusterd_op_sm ();
+        if (quorum_action)
+                glusterd_do_quorum_action ();
         return ret;
 }
 
 int
+glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
+{
+        return glusterd_big_locked_notify (rpc, mydata, event, data,
+                                           __glusterd_peer_rpc_notify);
+}
+
+int
 glusterd_null (rpcsvc_request_t *req)
 {
 
@@ -2779,11 +4135,11 @@ glusterd_null (rpcsvc_request_t *req)
 }
 
 rpcsvc_actor_t gd_svc_mgmt_actors[] = {
-        [GLUSTERD_MGMT_NULL]           = { "NULL", GLUSTERD_MGMT_NULL, glusterd_null, NULL, NULL, 0},
-        [GLUSTERD_MGMT_CLUSTER_LOCK]   = { "CLUSTER_LOCK", GLUSTERD_MGMT_CLUSTER_LOCK, glusterd_handle_cluster_lock, NULL, NULL, 0},
-        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = { "CLUSTER_UNLOCK", GLUSTERD_MGMT_CLUSTER_UNLOCK, glusterd_handle_cluster_unlock, NULL, NULL, 0},
-        [GLUSTERD_MGMT_STAGE_OP]       = { "STAGE_OP", GLUSTERD_MGMT_STAGE_OP, glusterd_handle_stage_op, NULL, NULL, 0},
-        [GLUSTERD_MGMT_COMMIT_OP]      = { "COMMIT_OP", GLUSTERD_MGMT_COMMIT_OP, glusterd_handle_commit_op, NULL, NULL, 0},
+        [GLUSTERD_MGMT_NULL]           = { "NULL",           GLUSTERD_MGMT_NULL,           glusterd_null,                  NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_CLUSTER_LOCK]   = { "CLUSTER_LOCK",   GLUSTERD_MGMT_CLUSTER_LOCK,   glusterd_handle_cluster_lock,   NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = { "CLUSTER_UNLOCK", GLUSTERD_MGMT_CLUSTER_UNLOCK, glusterd_handle_cluster_unlock, NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_STAGE_OP]       = { "STAGE_OP",       GLUSTERD_MGMT_STAGE_OP,       glusterd_handle_stage_op,       NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_COMMIT_OP]      = { "COMMIT_OP",      GLUSTERD_MGMT_COMMIT_OP,      glusterd_handle_commit_op,      NULL, 0, DRC_NA},
 };
 
 struct rpcsvc_program gd_svc_mgmt_prog = {
@@ -2792,14 +4148,15 @@ struct rpcsvc_program gd_svc_mgmt_prog = {
         .progver   = GD_MGMT_VERSION,
         .numactors = GLUSTERD_MGMT_MAXVALUE,
         .actors    = gd_svc_mgmt_actors,
+	.synctask  = _gf_true,
 };
 
 rpcsvc_actor_t gd_svc_peer_actors[] = {
-        [GLUSTERD_FRIEND_NULL]    = { "NULL", GLUSTERD_MGMT_NULL, glusterd_null, NULL, NULL, 0},
-        [GLUSTERD_PROBE_QUERY]    = { "PROBE_QUERY", GLUSTERD_PROBE_QUERY, glusterd_handle_probe_query, NULL, NULL, 0},
-        [GLUSTERD_FRIEND_ADD]     = { "FRIEND_ADD", GLUSTERD_FRIEND_ADD, glusterd_handle_incoming_friend_req, NULL, NULL, 0},
-        [GLUSTERD_FRIEND_REMOVE]  = { "FRIEND_REMOVE", GLUSTERD_FRIEND_REMOVE, glusterd_handle_incoming_unfriend_req, NULL, NULL, 0},
-        [GLUSTERD_FRIEND_UPDATE]  = { "FRIEND_UPDATE", GLUSTERD_FRIEND_UPDATE, glusterd_handle_friend_update, NULL, NULL, 0},
+        [GLUSTERD_FRIEND_NULL]    = { "NULL",          GLUSTERD_MGMT_NULL,     glusterd_null,                         NULL, 0, DRC_NA},
+        [GLUSTERD_PROBE_QUERY]    = { "PROBE_QUERY",   GLUSTERD_PROBE_QUERY,   glusterd_handle_probe_query,           NULL, 0, DRC_NA},
+        [GLUSTERD_FRIEND_ADD]     = { "FRIEND_ADD",    GLUSTERD_FRIEND_ADD,    glusterd_handle_incoming_friend_req,   NULL, 0, DRC_NA},
+        [GLUSTERD_FRIEND_REMOVE]  = { "FRIEND_REMOVE", GLUSTERD_FRIEND_REMOVE, glusterd_handle_incoming_unfriend_req, NULL, 0, DRC_NA},
+        [GLUSTERD_FRIEND_UPDATE]  = { "FRIEND_UPDATE", GLUSTERD_FRIEND_UPDATE, glusterd_handle_friend_update,         NULL, 0, DRC_NA},
 };
 
 struct rpcsvc_program gd_svc_peer_prog = {
@@ -2808,37 +4165,45 @@ struct rpcsvc_program gd_svc_peer_prog = {
         .progver   = GD_FRIEND_VERSION,
         .numactors = GLUSTERD_FRIEND_MAXVALUE,
         .actors    = gd_svc_peer_actors,
+	.synctask  = _gf_false,
 };
 
 
 
 rpcsvc_actor_t gd_svc_cli_actors[] = {
-        [GLUSTER_CLI_PROBE]         = { "CLI_PROBE", GLUSTER_CLI_PROBE, glusterd_handle_cli_probe, NULL, NULL, 0},
-        [GLUSTER_CLI_CREATE_VOLUME] = { "CLI_CREATE_VOLUME", GLUSTER_CLI_CREATE_VOLUME, glusterd_handle_create_volume, NULL,NULL, 0},
-        [GLUSTER_CLI_DEFRAG_VOLUME] = { "CLI_DEFRAG_VOLUME", GLUSTER_CLI_DEFRAG_VOLUME, glusterd_handle_defrag_volume, NULL,NULL, 0},
-        [GLUSTER_CLI_DEPROBE]       = { "FRIEND_REMOVE", GLUSTER_CLI_DEPROBE, glusterd_handle_cli_deprobe, NULL, NULL, 0},
-        [GLUSTER_CLI_LIST_FRIENDS]  = { "LIST_FRIENDS", GLUSTER_CLI_LIST_FRIENDS, glusterd_handle_cli_list_friends, NULL, NULL, 0},
-        [GLUSTER_CLI_START_VOLUME]  = { "START_VOLUME", GLUSTER_CLI_START_VOLUME, glusterd_handle_cli_start_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_STOP_VOLUME]   = { "STOP_VOLUME", GLUSTER_CLI_STOP_VOLUME, glusterd_handle_cli_stop_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_DELETE_VOLUME] = { "DELETE_VOLUME", GLUSTER_CLI_DELETE_VOLUME, glusterd_handle_cli_delete_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_GET_VOLUME]    = { "GET_VOLUME", GLUSTER_CLI_GET_VOLUME, glusterd_handle_cli_get_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_ADD_BRICK]     = { "ADD_BRICK", GLUSTER_CLI_ADD_BRICK, glusterd_handle_add_brick, NULL, NULL, 0},
-        [GLUSTER_CLI_REPLACE_BRICK] = { "REPLACE_BRICK", GLUSTER_CLI_REPLACE_BRICK, glusterd_handle_replace_brick, NULL, NULL, 0},
-        [GLUSTER_CLI_REMOVE_BRICK]  = { "REMOVE_BRICK", GLUSTER_CLI_REMOVE_BRICK, glusterd_handle_remove_brick, NULL, NULL, 0},
-        [GLUSTER_CLI_LOG_ROTATE]    = { "LOG FILENAME", GLUSTER_CLI_LOG_ROTATE, glusterd_handle_log_rotate, NULL, NULL, 0},
-        [GLUSTER_CLI_SET_VOLUME]    = { "SET_VOLUME", GLUSTER_CLI_SET_VOLUME, glusterd_handle_set_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_SYNC_VOLUME]   = { "SYNC_VOLUME", GLUSTER_CLI_SYNC_VOLUME, glusterd_handle_sync_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_RESET_VOLUME]  = { "RESET_VOLUME", GLUSTER_CLI_RESET_VOLUME, glusterd_handle_reset_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_FSM_LOG]       = { "FSM_LOG", GLUSTER_CLI_FSM_LOG, glusterd_handle_fsm_log, NULL, NULL, 0},
-        [GLUSTER_CLI_GSYNC_SET]     = { "GSYNC_SET", GLUSTER_CLI_GSYNC_SET, glusterd_handle_gsync_set, NULL, NULL, 0},
-        [GLUSTER_CLI_PROFILE_VOLUME] = { "STATS_VOLUME", GLUSTER_CLI_PROFILE_VOLUME, glusterd_handle_cli_profile_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_QUOTA]         = { "QUOTA", GLUSTER_CLI_QUOTA, glusterd_handle_quota, NULL, NULL, 0},
-        [GLUSTER_CLI_GETWD]         = { "GETWD", GLUSTER_CLI_GETWD, glusterd_handle_getwd, NULL, NULL, 1},
-        [GLUSTER_CLI_STATUS_VOLUME]  = {"STATUS_VOLUME", GLUSTER_CLI_STATUS_VOLUME, glusterd_handle_status_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_MOUNT]         = { "MOUNT", GLUSTER_CLI_MOUNT, glusterd_handle_mount, NULL, NULL, 1},
-        [GLUSTER_CLI_UMOUNT]        = { "UMOUNT", GLUSTER_CLI_UMOUNT, glusterd_handle_umount, NULL, NULL, 1},
-        [GLUSTER_CLI_HEAL_VOLUME]  = { "HEAL_VOLUME", GLUSTER_CLI_HEAL_VOLUME, glusterd_handle_cli_heal_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME", GLUSTER_CLI_STATEDUMP_VOLUME, glusterd_handle_cli_statedump_volume, NULL, NULL, 0},
+        [GLUSTER_CLI_PROBE]              = { "CLI_PROBE",         GLUSTER_CLI_PROBE,            glusterd_handle_cli_probe,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_CREATE_VOLUME]      = { "CLI_CREATE_VOLUME", GLUSTER_CLI_CREATE_VOLUME,    glusterd_handle_create_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_DEFRAG_VOLUME]      = { "CLI_DEFRAG_VOLUME", GLUSTER_CLI_DEFRAG_VOLUME,    glusterd_handle_defrag_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_DEPROBE]            = { "FRIEND_REMOVE",     GLUSTER_CLI_DEPROBE,          glusterd_handle_cli_deprobe,           NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
+        [GLUSTER_CLI_UUID_RESET]         = { "UUID_RESET",        GLUSTER_CLI_UUID_RESET,       glusterd_handle_cli_uuid_reset,        NULL, 0, DRC_NA},
+        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_START_VOLUME]       = { "START_VOLUME",      GLUSTER_CLI_START_VOLUME,     glusterd_handle_cli_start_volume,      NULL, 0, DRC_NA},
+        [GLUSTER_CLI_STOP_VOLUME]        = { "STOP_VOLUME",       GLUSTER_CLI_STOP_VOLUME,      glusterd_handle_cli_stop_volume,       NULL, 0, DRC_NA},
+        [GLUSTER_CLI_DELETE_VOLUME]      = { "DELETE_VOLUME",     GLUSTER_CLI_DELETE_VOLUME,    glusterd_handle_cli_delete_volume,     NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
+        [GLUSTER_CLI_ADD_BRICK]          = { "ADD_BRICK",         GLUSTER_CLI_ADD_BRICK,        glusterd_handle_add_brick,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_REPLACE_BRICK]      = { "REPLACE_BRICK",     GLUSTER_CLI_REPLACE_BRICK,    glusterd_handle_replace_brick,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_REMOVE_BRICK]       = { "REMOVE_BRICK",      GLUSTER_CLI_REMOVE_BRICK,     glusterd_handle_remove_brick,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LOG_ROTATE]         = { "LOG FILENAME",      GLUSTER_CLI_LOG_ROTATE,       glusterd_handle_log_rotate,            NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SET_VOLUME]         = { "SET_VOLUME",        GLUSTER_CLI_SET_VOLUME,       glusterd_handle_set_volume,            NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SYNC_VOLUME]        = { "SYNC_VOLUME",       GLUSTER_CLI_SYNC_VOLUME,      glusterd_handle_sync_volume,           NULL, 0, DRC_NA},
+        [GLUSTER_CLI_RESET_VOLUME]       = { "RESET_VOLUME",      GLUSTER_CLI_RESET_VOLUME,     glusterd_handle_reset_volume,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_FSM_LOG]            = { "FSM_LOG",           GLUSTER_CLI_FSM_LOG,          glusterd_handle_fsm_log,               NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GSYNC_SET]          = { "GSYNC_SET",         GLUSTER_CLI_GSYNC_SET,        glusterd_handle_gsync_set,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_PROFILE_VOLUME]     = { "STATS_VOLUME",      GLUSTER_CLI_PROFILE_VOLUME,   glusterd_handle_cli_profile_volume,    NULL, 0, DRC_NA},
+        [GLUSTER_CLI_QUOTA]              = { "QUOTA",             GLUSTER_CLI_QUOTA,            glusterd_handle_quota,                 NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
+        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_MOUNT]              = { "MOUNT",             GLUSTER_CLI_MOUNT,            glusterd_handle_mount,                 NULL, 1, DRC_NA},
+        [GLUSTER_CLI_UMOUNT]             = { "UMOUNT",            GLUSTER_CLI_UMOUNT,           glusterd_handle_umount,                NULL, 1, DRC_NA},
+        [GLUSTER_CLI_HEAL_VOLUME]        = { "HEAL_VOLUME",       GLUSTER_CLI_HEAL_VOLUME,      glusterd_handle_cli_heal_volume,       NULL, 0, DRC_NA},
+        [GLUSTER_CLI_STATEDUMP_VOLUME]   = {"STATEDUMP_VOLUME",   GLUSTER_CLI_STATEDUMP_VOLUME, glusterd_handle_cli_statedump_volume,  NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
+        [GLUSTER_CLI_CLRLOCKS_VOLUME]    = {"CLEARLOCKS_VOLUME",  GLUSTER_CLI_CLRLOCKS_VOLUME,  glusterd_handle_cli_clearlocks_volume, NULL, 0, DRC_NA},
+        [GLUSTER_CLI_COPY_FILE]          = {"COPY_FILE",          GLUSTER_CLI_COPY_FILE,        glusterd_handle_copy_file,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SYS_EXEC]           = {"SYS_EXEC",           GLUSTER_CLI_SYS_EXEC,         glusterd_handle_sys_exec,              NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SNAP]               = {"SNAP",               GLUSTER_CLI_SNAP,             glusterd_handle_snapshot,              NULL, 0, DRC_NA},
 };
 
 struct rpcsvc_program gd_svc_cli_prog = {
@@ -2847,4 +4212,26 @@ struct rpcsvc_program gd_svc_cli_prog = {
         .progver   = GLUSTER_CLI_VERSION,
         .numactors = GLUSTER_CLI_MAXVALUE,
         .actors    = gd_svc_cli_actors,
+	.synctask  = _gf_true,
+};
+
+/* This is a minimal RPC prog, which contains only the readonly RPC procs from
+ * the cli rpcsvc
+ */
+rpcsvc_actor_t gd_svc_cli_actors_ro[] = {
+        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
+        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
+        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_cli_prog_ro = {
+        .progname  = "GlusterD svc cli read-only",
+        .prognum   = GLUSTER_CLI_PROGRAM,
+        .progver   = GLUSTER_CLI_VERSION,
+        .numactors = GLUSTER_CLI_MAXVALUE,
+        .actors    = gd_svc_cli_actors_ro,
+	.synctask  = _gf_true,
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index 04e5fae7b..0f0357c4c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -31,6 +21,7 @@
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-op-sm.h"
+#include "glusterd-store.h"
 
 #include "glusterfs3.h"
 #include "protocol-common.h"
@@ -39,31 +30,170 @@
 
 extern struct rpc_clnt_program gd_peer_prog;
 extern struct rpc_clnt_program gd_mgmt_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+
+
+#define TRUSTED_PREFIX         "trusted-"
 
 typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *data);
 
+static int
+get_snap_volname_and_volinfo (const char *volpath, char **volname,
+                              glusterd_volinfo_t **volinfo)
+{
+        int              ret            = -1;
+        char            *save_ptr       = NULL;
+        char            *str_token      = NULL;
+        char            *snapname       = NULL;
+        char            *volname_token  = NULL;
+        char            *vol            = NULL;
+        glusterd_snap_t *snap           = NULL;
+        xlator_t        *this           = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (volpath);
+        GF_ASSERT (volinfo);
+
+        str_token = gf_strdup (volpath);
+        if (NULL == str_token) {
+                goto out;
+        }
+
+        /* Input volname will have below formats:
+         * /snaps/<snapname>/<volname>.<hostname>
+         * or
+         * /snaps/<snapname>/<parent-volname>
+         * We need to extract snapname and parent_volname */
+
+        /*split string by "/" */
+        strtok_r (str_token, "/",  &save_ptr);
+        snapname  = strtok_r(NULL, "/", &save_ptr);
+        if (!snapname) {
+                gf_log(this->name, GF_LOG_ERROR, "Invalid path: %s", volpath);
+                goto out;
+        }
+
+        volname_token = strtok_r(NULL, "/", &save_ptr);
+        if (!volname_token) {
+                gf_log(this->name, GF_LOG_ERROR, "Invalid path: %s", volpath);
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                gf_log(this->name, GF_LOG_ERROR, "Failed to "
+                                "fetch snap %s", snapname);
+                goto out;
+        }
+
+        /* Find if its a parent volume name or snap volume
+         * name. This function will succeed if volname_token
+         * is a parent volname
+         */
+        ret = glusterd_volinfo_find (volname_token, volinfo);
+        if (ret) {
+                *volname = gf_strdup (volname_token);
+                if (NULL == *volname) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_snap_volinfo_find (volname_token, snap,
+                                                  volinfo);
+                if (ret) {
+                        /* Split the volume name */
+                        vol = strtok_r (volname_token, ".", &save_ptr);
+                        if (!vol) {
+                                gf_log(this->name, GF_LOG_ERROR, "Invalid "
+                                                "volname (%s)", volname_token);
+                                goto out;
+                        }
+
+                        ret = glusterd_snap_volinfo_find (vol, snap, volinfo);
+                        if (ret) {
+                                gf_log(this->name, GF_LOG_ERROR, "Failed to "
+                                       "fetch snap volume from volname (%s)",
+                                       vol);
+                                goto out;
+                        }
+                }
+        } else {
+                /*volname_token is parent volname*/
+                ret = glusterd_snap_volinfo_find_from_parent_volname (
+                                volname_token, snap, volinfo);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_ERROR, "Failed to "
+                                        "fetch snap volume from parent "
+                                        "volname (%s)", volname_token);
+                        goto out;
+                }
+
+                /* Since volname_token is a parent volname we should
+                 * get the snap volname here*/
+                *volname = gf_strdup ((*volinfo)->volname);
+                if (NULL == *volname) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+out:
+        if (ret && NULL != *volname) {
+                GF_FREE (*volname);
+                *volname = NULL;
+        }
+        return ret;
+}
+
 static size_t
 build_volfile_path (const char *volname, char *path,
-                    size_t path_len)
+                    size_t path_len, char *trusted_str)
 {
-        struct stat         stbuf       = {0,};
-        int32_t             ret         = -1;
-        glusterd_conf_t    *priv        = NULL;
-        char               *vol         = NULL;
-        char               *dup_volname = NULL;
-        char               *free_ptr    = NULL;
-        char               *tmp         = NULL;
-        glusterd_volinfo_t *volinfo     = NULL;
-        char               *server      = NULL;
-
-        priv    = THIS->private;
+        struct stat              stbuf                  = {0,};
+        int32_t                  ret                    = -1;
+        glusterd_conf_t         *priv                   = NULL;
+        char                    *vol                    = NULL;
+        char                    *dup_volname            = NULL;
+        char                    *free_ptr               = NULL;
+        char                    *save_ptr               = NULL;
+        char                    *str_token              = NULL;
+        glusterd_volinfo_t      *volinfo                = NULL;
+        char                    *server                 = NULL;
+        const char              *volname_ptr            = NULL;
+        char                     path_prefix [PATH_MAX] = {0,};
+        xlator_t                *this                   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volname);
+        GF_ASSERT (path);
 
         if (strstr (volname, "gluster/")) {
                 server = strchr (volname, '/') + 1;
                 glusterd_get_nodesvc_volfile (server, priv->workdir,
-                                                    path, path_len);
+                                              path, path_len);
                 ret = 1;
                 goto out;
+        } else if ((str_token = strstr (volname, "/snaps/"))) {
+                ret = get_snap_volname_and_volinfo (str_token, &dup_volname,
+                                                    &volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get snap"
+                                " volinfo from path (%s)", volname);
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (path_prefix, sizeof (path_prefix), "%s/snaps/%s",
+                          priv->workdir, volinfo->snapshot->snapname);
+
+                free_ptr = dup_volname;
+                volname_ptr = dup_volname;
+                goto gotvolinfo;
         } else if (volname[0] != '/') {
                 /* Normal behavior */
                 dup_volname = gf_strdup (volname);
@@ -74,66 +204,221 @@ build_volfile_path (const char *volname, char *path,
                 dup_volname = gf_strdup (&volname[1]);
         }
 
+        if (!dup_volname) {
+                gf_log(THIS->name, GF_LOG_ERROR, "strdup failed");
+                ret = -1;
+                goto out;
+        }
         free_ptr = dup_volname;
+        volname_ptr = volname;
+
+        snprintf (path_prefix, sizeof (path_prefix), "%s/vols",
+                  priv->workdir);
 
         ret = glusterd_volinfo_find (dup_volname, &volinfo);
+
         if (ret) {
                 /* Split the volume name */
-                vol = strtok_r (dup_volname, ".", &tmp);
+                vol = strtok_r (dup_volname, ".", &save_ptr);
                 if (!vol)
                         goto out;
+
                 ret = glusterd_volinfo_find (vol, &volinfo);
                 if (ret)
                         goto out;
         }
-        ret = snprintf (path, path_len, "%s/vols/%s/%s.vol",
-                        priv->workdir, volinfo->volname, volname);
+
+gotvolinfo:
+        if (!glusterd_auth_get_username (volinfo))
+                trusted_str = NULL;
+
+        ret = snprintf (path, path_len, "%s/%s/%s.vol", path_prefix,
+                        volinfo->volname, volname_ptr);
         if (ret == -1)
                 goto out;
 
         ret = stat (path, &stbuf);
+
         if ((ret == -1) && (errno == ENOENT)) {
-                ret = snprintf (path, path_len, "%s/vols/%s/%s-fuse.vol",
-                                priv->workdir, volinfo->volname, volname);
+                snprintf (path, path_len, "%s/%s/%s%s-fuse.vol",
+                          path_prefix, volinfo->volname,
+                          (trusted_str ? trusted_str : ""),
+                          dup_volname);
+
                 ret = stat (path, &stbuf);
         }
+
         if ((ret == -1) && (errno == ENOENT)) {
-                ret = snprintf (path, path_len, "%s/vols/%s/%s-tcp.vol",
-                                priv->workdir, volinfo->volname, volname);
+                snprintf (path, path_len, "%s/%s/%s-tcp.vol",
+                          path_prefix, volinfo->volname, volname_ptr);
         }
 
         ret = 1;
 out:
-        if (free_ptr)
-                GF_FREE (free_ptr);
+        GF_FREE (free_ptr);
         return ret;
 }
 
-int
-server_getspec (rpcsvc_request_t *req)
+/* Get and store op-versions of the clients sending the getspec request
+ * Clients of versions <= 3.3, don't send op-versions, their op-versions are
+ * defaulted to 1
+ */
+static int
+_get_client_op_versions (gf_getspec_req *args, peer_info_t *peerinfo)
 {
-        int32_t               ret = -1;
-        int32_t               op_errno = 0;
-        int32_t               spec_fd = -1;
-        size_t                file_len = 0;
-        char                  filename[ZR_PATH_MAX] = {0,};
-        struct stat           stbuf = {0,};
-        char                 *volume = NULL;
-        int                   cookie = 0;
+        int    ret                   = 0;
+        int    client_max_op_version = 1;
+        int    client_min_op_version = 1;
+        dict_t *dict                 = NULL;
+
+        GF_ASSERT (args);
+        GF_ASSERT (peerinfo);
+
+        if (args->xdata.xdata_len) {
+                dict = dict_new ();
+                if (!dict) {
+                        ret = -1;
+                        goto out;
+                }
 
-        gf_getspec_req    args = {0,};
-        gf_getspec_rsp    rsp  = {0,};
+                ret = dict_unserialize (args->xdata.xdata_val,
+                                        args->xdata.xdata_len, &dict);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Failed to unserialize request dictionary");
+                        goto out;
+                }
 
+                ret = dict_get_int32 (dict, "min-op-version",
+                                      &client_min_op_version);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Failed to get client-min-op-version");
+                        goto out;
+                }
 
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gf_getspec_req)) {
+                ret = dict_get_int32 (dict, "max-op-version",
+                                      &client_max_op_version);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Failed to get client-max-op-version");
+                        goto out;
+                }
+        }
+
+        peerinfo->max_op_version = client_max_op_version;
+        peerinfo->min_op_version = client_min_op_version;
+
+out:
+        return ret;
+}
+
+/* Checks if the client supports the volume, ie. client can understand all the
+ * options in the volfile
+ */
+static gf_boolean_t
+_client_supports_volume (peer_info_t *peerinfo, int32_t *op_errno)
+{
+        gf_boolean_t       ret       = _gf_true;
+        glusterd_volinfo_t *volinfo  = NULL;
+
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (op_errno);
+
+
+        /* Only check when the volfile being requested is a volume. Not finding
+         * a volinfo implies that the volfile requested for is not of a gluster
+         * volume. A non volume volfile is requested by the local gluster
+         * services like shd and nfs-server. These need not be checked as they
+         * will be running at the same op-version as glusterd and will be able
+         * to support all the features
+         */
+        if ((glusterd_volinfo_find (peerinfo->volname, &volinfo) == 0) &&
+            ((peerinfo->min_op_version > volinfo->client_op_version) ||
+             (peerinfo->max_op_version < volinfo->client_op_version))) {
+                ret = _gf_false;
+                *op_errno = ENOTSUP;
+                gf_log ("glusterd", GF_LOG_INFO,
+                        "Client %s (%d -> %d) doesn't support required "
+                        "op-version (%d). Rejecting volfile request.",
+                        peerinfo->identifier, peerinfo->min_op_version,
+                        peerinfo->max_op_version, volinfo->client_op_version);
+        }
+
+        return ret;
+}
+
+int
+__server_getspec (rpcsvc_request_t *req)
+{
+        int32_t               ret                    = -1;
+        int32_t               op_errno               = 0;
+        int32_t               spec_fd                = -1;
+        size_t                file_len               = 0;
+        char                  filename[PATH_MAX]  = {0,};
+        struct stat           stbuf                  = {0,};
+        char                 *volume                 = NULL;
+        char                 *tmp                    = NULL;
+        int                   cookie                 = 0;
+        rpc_transport_t      *trans                  = NULL;
+        gf_getspec_req        args                   = {0,};
+        gf_getspec_rsp        rsp                    = {0,};
+        char                  addrstr[RPCSVC_PEER_STRLEN] = {0};
+        peer_info_t          *peerinfo               = NULL;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_getspec_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
 
+        peerinfo = &req->trans->peerinfo;
+
         volume = args.key;
+        /* Need to strip leading '/' from volnames. This was introduced to
+         * support nfs style mount parameters for native gluster mount
+         */
+        if (volume[0] == '/')
+                strncpy (peerinfo->volname, &volume[1], strlen(&volume[1]));
+        else
+                strncpy (peerinfo->volname, volume, strlen(volume));
+
+        ret = _get_client_op_versions (&args, peerinfo);
+        if (ret)
+                goto fail;
+
+        if (!_client_supports_volume (peerinfo, &op_errno)) {
+                ret = -1;
+                goto fail;
+        }
 
-        ret = build_volfile_path (volume, filename, sizeof (filename));
+        trans = req->trans;
+        /* addrstr will be empty for cli socket connections */
+        ret = rpcsvc_transport_peername (trans, (char *)&addrstr,
+                                         sizeof (addrstr));
+        if (ret)
+                goto fail;
+
+        tmp  = strrchr (addrstr, ':');
+        if (tmp)
+                *tmp = '\0';
+
+        /* The trusted volfiles are given to the glusterd owned process like NFS
+         * server, self-heal daemon etc., so that they are not inadvertently
+         * blocked by a auth.{allow,reject} setting. The trusted volfile is not
+         * meant for external users.
+         */
+        if (strlen (addrstr) && gf_is_local_addr (addrstr)) {
+
+                ret = build_volfile_path (volume, filename,
+                                          sizeof (filename),
+                                          TRUSTED_PREFIX);
+        } else {
+                ret = build_volfile_path (volume, filename,
+                                          sizeof (filename), NULL);
+        }
 
         if (ret > 0) {
                 /* to allocate the proper buffer to hold the file data */
@@ -179,45 +464,360 @@ fail:
                 rsp.op_errno = cookie;
 
         if (!rsp.spec)
-                rsp.spec = "";
+                rsp.spec = strdup ("");
 
         glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                (xdrproc_t)xdr_gf_getspec_rsp);
-        if (args.key)
-                free (args.key);//malloced by xdr
-        if (rsp.spec && (strcmp (rsp.spec, "")))
-                free (rsp.spec);
+        free (args.key);//malloced by xdr
+        free (rsp.spec);
 
         return 0;
 }
 
+int
+server_getspec (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __server_getspec);
+}
+
+int32_t
+__server_event_notify (rpcsvc_request_t *req)
+{
+        int32_t                 ret             = -1;
+        int32_t                 op_errno        =  0;
+        gf_event_notify_req     args            = {0,};
+        gf_event_notify_rsp     rsp             = {0,};
+        dict_t                 *dict            = NULL;
+        gf_boolean_t            need_rsp        = _gf_true;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_event_notify_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        if (args.dict.dict_len) {
+                dict = dict_new ();
+                if (!dict)
+                        return ret;
+                ret = dict_unserialize (args.dict.dict_val,
+                                        args.dict.dict_len, &dict);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to unserialize req");
+                        goto fail;
+                }
+        }
+
+        switch (args.op) {
+        case GF_EN_DEFRAG_STATUS:
+                gf_log ("", GF_LOG_INFO,
+                        "received defrag status updated");
+                if (dict) {
+                        glusterd_defrag_event_notify_handle (dict);
+                        need_rsp = _gf_false;
+                }
+                break;
+        default:
+                gf_log ("", GF_LOG_ERROR, "Unknown op received in event "
+                        "notify");
+                ret = -1;
+                break;
+        }
+
+fail:
+        rsp.op_ret   = ret;
+
+        if (op_errno)
+                rsp.op_errno = gf_errno_to_error (op_errno);
+
+        if (need_rsp)
+                glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                       (xdrproc_t)xdr_gf_event_notify_rsp);
+        if (dict)
+                dict_unref (dict);
+        free (args.dict.dict_val);//malloced by xdr
+
+        return 0;
+}
+
+int32_t
+server_event_notify (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __server_event_notify);
+}
+
+int
+gd_validate_cluster_op_version (xlator_t *this, int cluster_op_version,
+                                char *peerid)
+{
+        int              ret  = -1;
+        glusterd_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (cluster_op_version > GD_OP_VERSION_MAX) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "operating version %d is more than the maximum "
+                        "supported (%d) on the machine (as per peer request "
+                        "from %s)", cluster_op_version, GD_OP_VERSION_MAX,
+                        peerid);
+                goto out;
+        }
+
+        /* The peer can only reduce its op-version when it doesn't have any
+         * volumes. Reducing op-version when it already contains volumes can
+         * lead to inconsistencies in the cluster
+         */
+        if ((cluster_op_version < conf->op_version) &&
+            !list_empty (&conf->volumes)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot reduce operating version to %d from current "
+                        "version %d as volumes exist (as per peer request from "
+                        "%s)", cluster_op_version, conf->op_version, peerid);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+__glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+{
+        dict_t            *dict            = NULL;
+        xlator_t          *this            = NULL;
+        glusterd_conf_t   *conf            = NULL;
+        int                ret             = -1;
+        int                op_errno        = EINVAL;
+        gf_mgmt_hndsk_req  args            = {{0,},};
+        gf_mgmt_hndsk_rsp  rsp             = {0,};
+
+        this = THIS;
+        conf = this->private;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_int32 (dict, GD_OP_VERSION_KEY, conf->op_version);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set operating version");
+                rsp.op_ret = ret;
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, GD_MIN_OP_VERSION_KEY, GD_OP_VERSION_MIN);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s", GD_MIN_OP_VERSION_KEY);
+                rsp.op_ret = ret;
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, GD_MAX_OP_VERSION_KEY, GD_OP_VERSION_MAX);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s", GD_MAX_OP_VERSION_KEY);
+                rsp.op_ret = ret;
+                goto out;
+        }
+
+        ret = 0;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, (&rsp.hndsk.hndsk_val),
+                                    rsp.hndsk.hndsk_len, op_errno, out);
+out:
+
+        rsp.op_ret = ret;
+        rsp.op_errno = op_errno;
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+
+        ret = 0;
+
+        if (dict)
+                dict_unref (dict);
+
+        if (args.hndsk.hndsk_val)
+                free (args.hndsk.hndsk_val);
+
+        if (rsp.hndsk.hndsk_val)
+                GF_FREE (rsp.hndsk.hndsk_val);
+
+        return ret;
+}
+
+int
+glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_mgmt_hndsk_versions);
+}
+
+int
+__glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+{
+        dict_t            *clnt_dict       = NULL;
+        xlator_t          *this            = NULL;
+        glusterd_conf_t   *conf            = NULL;
+        int                ret             = -1;
+        int                op_errno        = EINVAL;
+        int                peer_op_version = 0;
+        gf_mgmt_hndsk_req  args            = {{0,},};
+        gf_mgmt_hndsk_rsp  rsp             = {0,};
+
+        this = THIS;
+        conf = this->private;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, clnt_dict, args.hndsk.hndsk_val,
+                                      (args.hndsk.hndsk_len), ret, op_errno,
+                                      out);
+
+        ret = dict_get_int32 (clnt_dict, GD_OP_VERSION_KEY, &peer_op_version);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to get the op-version key peer=%s",
+                        req->trans->peerinfo.identifier);
+                goto out;
+        }
+
+        ret = gd_validate_cluster_op_version (this, peer_op_version,
+                                              req->trans->peerinfo.identifier);
+        if (ret)
+                goto out;
+
+
+        /* As this is ACK from the Cluster for the versions supported,
+           can set the op-version of 'this' glusterd to the one
+           received. */
+        gf_log (this->name, GF_LOG_INFO, "using the op-version %d",
+                peer_op_version);
+        conf->op_version = peer_op_version;
+        ret = glusterd_store_global_info (this);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store op-version");
+
+out:
+        rsp.op_ret = ret;
+        rsp.op_errno = op_errno;
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+
+        ret = 0;
+
+        if (clnt_dict)
+                dict_unref (clnt_dict);
+
+        if (args.hndsk.hndsk_val)
+                free (args.hndsk.hndsk_val);
+
+        return ret;
+}
+
+int
+glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_mgmt_hndsk_versions_ack);
+}
 
 rpcsvc_actor_t gluster_handshake_actors[] = {
-        [GF_HNDSK_NULL]      = {"NULL",      GF_HNDSK_NULL,      NULL, NULL, NULL, 0},
-        [GF_HNDSK_GETSPEC]   = {"GETSPEC",   GF_HNDSK_GETSPEC,   server_getspec, NULL, NULL, 0},
+        [GF_HNDSK_NULL]         = {"NULL",        GF_HNDSK_NULL,         NULL,                NULL, 0, DRC_NA},
+        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
+        [GF_HNDSK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_HNDSK_EVENT_NOTIFY, server_event_notify, NULL, 0, DRC_NA},
 };
 
 
 struct rpcsvc_program gluster_handshake_prog = {
-        .progname  = "GlusterFS Handshake",
+        .progname  = "Gluster Handshake",
         .prognum   = GLUSTER_HNDSK_PROGRAM,
         .progver   = GLUSTER_HNDSK_VERSION,
         .actors    = gluster_handshake_actors,
         .numactors = GF_HNDSK_MAXVALUE,
 };
 
+/* A minimal RPC program just for the cli getspec command */
+rpcsvc_actor_t gluster_cli_getspec_actors[] = {
+        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gluster_cli_getspec_prog = {
+        .progname  = "Gluster Handshake (CLI Getspec)",
+        .prognum   = GLUSTER_HNDSK_PROGRAM,
+        .progver   = GLUSTER_HNDSK_VERSION,
+        .actors    = gluster_cli_getspec_actors,
+        .numactors = GF_HNDSK_MAXVALUE,
+};
+
+
 char *glusterd_dump_proc[GF_DUMP_MAXVALUE] = {
         [GF_DUMP_NULL] = "NULL",
         [GF_DUMP_DUMP] = "DUMP",
 };
 
 rpc_clnt_prog_t glusterd_dump_prog = {
-        .progname  = "GLUSTERD-DUMP",
-        .prognum   = GLUSTER_DUMP_PROGRAM,
-        .progver   = GLUSTER_DUMP_VERSION,
-        .procnames = glusterd_dump_proc,
+        .progname       = "GLUSTERD-DUMP",
+        .prognum        = GLUSTER_DUMP_PROGRAM,
+        .progver        = GLUSTER_DUMP_VERSION,
+        .procnames      = glusterd_dump_proc,
+};
+
+
+rpcsvc_actor_t glusterd_mgmt_hndsk_actors[] = {
+        [GD_MGMT_HNDSK_NULL]            = {"NULL", GD_MGMT_HNDSK_NULL, NULL,
+                                           NULL, 0},
+        [GD_MGMT_HNDSK_VERSIONS]        = {"MGMT-VERS", GD_MGMT_HNDSK_VERSIONS,
+                                           glusterd_mgmt_hndsk_versions, NULL,
+                                           0},
+        [GD_MGMT_HNDSK_VERSIONS_ACK]    = {"MGMT-VERS-ACK",
+                                           GD_MGMT_HNDSK_VERSIONS_ACK,
+                                           glusterd_mgmt_hndsk_versions_ack,
+                                           NULL, 0},
 };
 
+struct rpcsvc_program glusterd_mgmt_hndsk_prog = {
+        .progname       = "Gluster MGMT Handshake",
+        .prognum        = GD_MGMT_HNDSK_PROGRAM,
+        .progver        = GD_MGMT_HNDSK_VERSION,
+        .actors         = glusterd_mgmt_hndsk_actors,
+        .numactors      = GD_MGMT_HNDSK_MAXVALUE,
+};
+
+char *glusterd_mgmt_hndsk_proc[GD_MGMT_HNDSK_MAXVALUE] = {
+        [GD_MGMT_HNDSK_NULL]          = "NULL",
+        [GD_MGMT_HNDSK_VERSIONS] = "MGMT-VERS",
+        [GD_MGMT_HNDSK_VERSIONS_ACK] = "MGMT-VERS-ACK",
+};
+
+rpc_clnt_prog_t gd_clnt_mgmt_hndsk_prog = {
+        .progname  = "Gluster MGMT Handshake",
+        .prognum   = GD_MGMT_HNDSK_PROGRAM,
+        .progver   = GD_MGMT_HNDSK_VERSION,
+        .procnames = glusterd_mgmt_hndsk_proc,
+};
+
+
 static int
 glusterd_event_connected_inject (glusterd_peerctx_t *peerctx)
 {
@@ -249,6 +849,7 @@ glusterd_event_connected_inject (glusterd_peerctx_t *peerctx)
         ctx->hostname = gf_strdup (peerinfo->hostname);
         ctx->port = peerinfo->port;
         ctx->req = peerctx->args.req;
+        ctx->dict = peerctx->args.dict;
 
         event->peerinfo = peerinfo;
         event->ctx = ctx;
@@ -266,12 +867,286 @@ out:
         return ret;
 }
 
+
+int
+gd_validate_peer_op_version (xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                             dict_t *dict, char **errstr)
+{
+        int              ret  = -1;
+        glusterd_conf_t *conf = NULL;
+        int32_t          peer_op_version = 0;
+        int32_t          peer_min_op_version = 0;
+        int32_t          peer_max_op_version = 0;
+
+        if (!dict && !this && !peerinfo)
+                goto out;
+
+        conf = this->private;
+
+        ret = dict_get_int32 (dict, GD_OP_VERSION_KEY, &peer_op_version);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, GD_MAX_OP_VERSION_KEY,
+                              &peer_max_op_version);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, GD_MIN_OP_VERSION_KEY,
+                              &peer_min_op_version);
+        if (ret)
+                goto out;
+
+        ret = -1;
+        /* Check if peer can support our op_version */
+        if ((peer_max_op_version < conf->op_version) ||
+            (peer_min_op_version > conf->op_version)) {
+                ret = gf_asprintf (errstr, "Peer %s does not support required "
+                                   "op-version", peerinfo->hostname);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name , GF_LOG_DEBUG, "Peer %s %s", peerinfo->hostname,
+                ((ret < 0) ? "rejected" : "accepted"));
+        return ret;
+}
+
+int
+__glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
+{
+        int                  ret      = -1;
+        int                  op_errno = EINVAL;
+        gf_mgmt_hndsk_rsp    rsp      = {0,};
+        xlator_t            *this     = NULL;
+        call_frame_t        *frame    = NULL;
+        glusterd_peerinfo_t *peerinfo = NULL;
+        glusterd_peerctx_t  *peerctx  = NULL;
+        char                msg[1024] = {0,};
+
+        this = THIS;
+        frame = myframe;
+        peerctx = frame->local;
+        peerinfo = peerctx->peerinfo;
+
+        if (-1 == req->rpc_status) {
+                snprintf (msg, sizeof (msg),
+                          "Error through RPC layer, retry again later");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+        if (ret < 0) {
+                snprintf (msg, sizeof (msg), "Failed to decode XDR");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        op_errno = rsp.op_errno;
+        if (-1 == rsp.op_ret) {
+                ret = -1;
+                snprintf (msg, sizeof (msg),
+                          "Failed to get handshake ack from remote server");
+                gf_log (frame->this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        /* TODO: this is hardcoded as of now, but I don't forsee any problems
+         * with this as long as we are properly handshaking operating versions
+         */
+        peerinfo->mgmt = &gd_mgmt_prog;
+        peerinfo->peer = &gd_peer_prog;
+        peerinfo->mgmt_v3 = &gd_mgmt_v3_prog;
+
+        ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
+
+        if (GD_MODE_ON == peerctx->args.mode) {
+                ret = glusterd_event_connected_inject (peerctx);
+                peerctx->args.req = NULL;
+        } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
+                peerctx->args.mode = GD_MODE_ON;
+        } else {
+                gf_log (this->name, GF_LOG_WARNING, "unknown mode %d",
+                        peerctx->args.mode);
+        }
+
+        glusterd_friend_sm ();
+
+        ret = 0;
+out:
+
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+
+        if (ret != 0)
+                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
+
+        if (rsp.hndsk.hndsk_val)
+                free (rsp.hndsk.hndsk_val);
+
+        return 0;
+}
+
+int
+glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_mgmt_hndsk_version_ack_cbk);
+}
+
+int
+__glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        int                  ret       = -1;
+        int                  op_errno  = EINVAL;
+        gf_mgmt_hndsk_rsp    rsp       = {0,};
+        gf_mgmt_hndsk_req    arg       = {{0,}};
+        xlator_t            *this      = NULL;
+        call_frame_t        *frame     = NULL;
+        glusterd_peerinfo_t *peerinfo  = NULL;
+        glusterd_peerctx_t  *peerctx   = NULL;
+        dict_t              *dict      = NULL;
+        dict_t              *rsp_dict  = NULL;
+        glusterd_conf_t     *conf      = NULL;
+        char                 msg[1024] = {0,};
+
+        this = THIS;
+        conf = this->private;
+        frame = myframe;
+        peerctx = frame->local;
+        peerinfo = peerctx->peerinfo;
+
+        if (-1 == req->rpc_status) {
+                ret = -1;
+                snprintf (msg, sizeof (msg),
+                          "Error through RPC layer, retry again later");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+        if (ret < 0) {
+                snprintf (msg, sizeof (msg), "Failed to decode management "
+                          "handshake response");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, dict, rsp.hndsk.hndsk_val,
+                                      rsp.hndsk.hndsk_len, ret, op_errno,
+                                      out);
+
+        op_errno = rsp.op_errno;
+        if (-1 == rsp.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get the 'versions' from peer (%s)",
+                        req->conn->trans->peerinfo.identifier);
+                goto out;
+        }
+
+        /* Check if peer can be part of cluster */
+        ret = gd_validate_peer_op_version (this, peerinfo, dict,
+                                           &peerctx->errstr);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to validate the operating version of peer (%s)",
+                        peerinfo->hostname);
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict)
+                goto out;
+
+        ret = dict_set_int32 (rsp_dict, GD_OP_VERSION_KEY, conf->op_version);
+        if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to set operating version in dict");
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, rsp_dict, (&arg.hndsk.hndsk_val),
+                                    arg.hndsk.hndsk_len, op_errno, out);
+
+        ret = glusterd_submit_request (peerctx->peerinfo->rpc, &arg, frame,
+                                       &gd_clnt_mgmt_hndsk_prog,
+                                       GD_MGMT_HNDSK_VERSIONS_ACK, NULL, this,
+                                       glusterd_mgmt_hndsk_version_ack_cbk,
+                                       (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+
+out:
+        if (ret) {
+                frame->local = NULL;
+                STACK_DESTROY (frame->root);
+                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
+        }
+
+        if (rsp.hndsk.hndsk_val)
+                free (rsp.hndsk.hndsk_val);
+
+        if (arg.hndsk.hndsk_val)
+                GF_FREE (arg.hndsk.hndsk_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        return 0;
+}
+
+int
+glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_mgmt_hndsk_version_cbk);
+}
+
+int
+glusterd_mgmt_handshake (xlator_t *this, glusterd_peerctx_t *peerctx)
+{
+        call_frame_t        *frame    = NULL;
+        gf_mgmt_hndsk_req    req      = {{0,},};
+        int                  ret      = -1;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        frame->local = peerctx;
+
+        ret = glusterd_submit_request (peerctx->peerinfo->rpc, &req, frame,
+                                       &gd_clnt_mgmt_hndsk_prog,
+                                       GD_MGMT_HNDSK_VERSIONS, NULL, this,
+                                       glusterd_mgmt_hndsk_version_cbk,
+                                       (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+        ret = 0;
+out:
+        if (ret && frame)
+                STACK_DESTROY (frame->root);
+
+        return ret;
+}
+
 int
 glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo,
                                 gf_prog_detail *prog)
 {
-        gf_prog_detail *trav     = NULL;
-        int             ret      = -1;
+        gf_prog_detail  *trav   = NULL;
+        int             ret     = -1;
 
         if (!peerinfo || !prog)
                 goto out;
@@ -279,45 +1154,79 @@ glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo,
         trav = prog;
 
         while (trav) {
-                /* Select 'programs' */
+                ret = -1;
                 if ((gd_mgmt_prog.prognum == trav->prognum) &&
                     (gd_mgmt_prog.progver == trav->progver)) {
                         peerinfo->mgmt = &gd_mgmt_prog;
                         ret = 0;
                 }
+
                 if ((gd_peer_prog.prognum == trav->prognum) &&
                     (gd_peer_prog.progver == trav->progver)) {
                         peerinfo->peer = &gd_peer_prog;
                         ret = 0;
                 }
+
                 if (ret) {
                         gf_log ("", GF_LOG_DEBUG,
                                 "%s (%"PRId64":%"PRId64") not supported",
                                 trav->progname, trav->prognum,
                                 trav->progver);
                 }
+
                 trav = trav->next;
         }
 
         if (peerinfo->mgmt) {
-                gf_log ("", GF_LOG_INFO,
-                        "Using Program %s, Num (%d), Version (%d)",
-                        peerinfo->mgmt->progname, peerinfo->mgmt->prognum,
-                        peerinfo->mgmt->progver);
+                 gf_log ("", GF_LOG_INFO,
+                         "Using Program %s, Num (%d), Version (%d)",
+                         peerinfo->mgmt->progname, peerinfo->mgmt->prognum,
+                         peerinfo->mgmt->progver);
         }
+
         if (peerinfo->peer) {
-                gf_log ("", GF_LOG_INFO,
-                        "Using Program %s, Num (%d), Version (%d)",
-                        peerinfo->peer->progname, peerinfo->peer->prognum,
-                        peerinfo->peer->progver);
+                 gf_log ("", GF_LOG_INFO,
+                         "Using Program %s, Num (%d), Version (%d)",
+                         peerinfo->peer->progname, peerinfo->peer->prognum,
+                         peerinfo->peer->progver);
         }
 
+        if (peerinfo->mgmt_v3) {
+                 gf_log ("", GF_LOG_INFO,
+                         "Using Program %s, Num (%d), Version (%d)",
+                         peerinfo->mgmt_v3->progname, peerinfo->mgmt_v3->prognum,
+                         peerinfo->mgmt_v3->progver);
+        }
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+static gf_boolean_t
+_mgmt_hndsk_prog_present (gf_prog_detail *prog) {
+        gf_boolean_t    ret = _gf_false;
+        gf_prog_detail  *trav = NULL;
+
+        GF_ASSERT (prog);
+
+        trav = prog;
+
+        while (trav) {
+                if ((trav->prognum == GD_MGMT_HNDSK_PROGRAM) &&
+                    (trav->progver == GD_MGMT_HNDSK_VERSION)) {
+                        ret = _gf_true;
+                        goto out;
+                }
+                trav = trav->next;
+        }
 out:
         return ret;
 }
 
 int
-glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
                                 int count, void *myframe)
 {
         int                  ret      = -1;
@@ -328,26 +1237,51 @@ glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
         call_frame_t        *frame    = NULL;
         glusterd_peerinfo_t *peerinfo = NULL;
         glusterd_peerctx_t  *peerctx  = NULL;
+        glusterd_conf_t     *conf     = NULL;
+        char                msg[1024] = {0,};
 
         this = THIS;
+        conf = this->private;
         frame = myframe;
         peerctx = frame->local;
         peerinfo = peerctx->peerinfo;
 
         if (-1 == req->rpc_status) {
-                gf_log ("", GF_LOG_ERROR,
-                        "error through RPC layer, retry again later");
+                snprintf (msg, sizeof (msg),
+                          "Error through RPC layer, retry again later");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
                 goto out;
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "failed to decode XDR");
+                snprintf (msg, sizeof (msg), "Failed to decode XDR");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
                 goto out;
         }
         if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to get the 'versions' from remote server");
+                snprintf (msg, sizeof (msg),
+                          "Failed to get the 'versions' from remote server");
+                gf_log (frame->this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        if (_mgmt_hndsk_prog_present (rsp.prog)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Proceeding to op-version handshake with peer %s",
+                        peerinfo->hostname);
+                ret = glusterd_mgmt_handshake (this, peerctx);
+                goto out;
+        } else if (conf->op_version > 1) {
+                ret = -1;
+                snprintf (msg, sizeof (msg),
+                          "Peer %s does not support required op-version",
+                          peerinfo->hostname);
+                peerctx->errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
@@ -370,10 +1304,11 @@ glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
                         peerctx->args.mode);
         }
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        glusterd_friend_sm();
+        glusterd_op_sm();
 
         ret = 0;
+
 out:
 
         /* don't use GF_FREE, buffer was allocated by libc */
@@ -398,8 +1333,16 @@ out:
 
 
 int
-glusterd_peer_handshake (xlator_t *this, struct rpc_clnt *rpc,
-                         glusterd_peerctx_t *peerctx)
+glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_peer_dump_version_cbk);
+}
+
+int
+glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc,
+                            glusterd_peerctx_t *peerctx)
 {
         call_frame_t        *frame    = NULL;
         gf_dump_req          req      = {0,};
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.c b/xlators/mgmt/glusterd/src/glusterd-hooks.c
new file mode 100644
index 000000000..2b43a452e
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.c
@@ -0,0 +1,531 @@
+/*
+   Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "dict.h"
+#include "xlator.h"
+#include "logging.h"
+#include "run.h"
+#include "defaults.h"
+#include "compat.h"
+#include "compat-errno.h"
+#include "glusterd.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+#include "glusterd-hooks.h"
+
+#include <fnmatch.h>
+
+#define EMPTY ""
+char glusterd_hook_dirnames[GD_OP_MAX][256] =
+{
+        [GD_OP_NONE]                    = EMPTY,
+        [GD_OP_CREATE_VOLUME]           = "create",
+        [GD_OP_START_BRICK]             = EMPTY,
+        [GD_OP_STOP_BRICK]              = EMPTY,
+        [GD_OP_DELETE_VOLUME]           = "delete",
+        [GD_OP_START_VOLUME]            = "start",
+        [GD_OP_STOP_VOLUME]             = "stop",
+        [GD_OP_DEFRAG_VOLUME]           = EMPTY,
+        [GD_OP_ADD_BRICK]               = "add-brick",
+        [GD_OP_REMOVE_BRICK]            = "remove-brick",
+        [GD_OP_REPLACE_BRICK]           = EMPTY,
+        [GD_OP_SET_VOLUME]              = "set",
+        [GD_OP_RESET_VOLUME]            = EMPTY,
+        [GD_OP_SYNC_VOLUME]             = EMPTY,
+        [GD_OP_LOG_ROTATE]              = EMPTY,
+        [GD_OP_GSYNC_CREATE]            = "gsync-create",
+        [GD_OP_GSYNC_SET]               = EMPTY,
+        [GD_OP_PROFILE_VOLUME]          = EMPTY,
+        [GD_OP_QUOTA]                   = EMPTY,
+        [GD_OP_STATUS_VOLUME]           = EMPTY,
+        [GD_OP_REBALANCE]               = EMPTY,
+        [GD_OP_HEAL_VOLUME]             = EMPTY,
+        [GD_OP_STATEDUMP_VOLUME]        = EMPTY,
+        [GD_OP_LIST_VOLUME]             = EMPTY,
+        [GD_OP_CLEARLOCKS_VOLUME]       = EMPTY,
+        [GD_OP_DEFRAG_BRICK_VOLUME]     = EMPTY,
+};
+#undef EMPTY
+
+static inline gf_boolean_t
+glusterd_is_hook_enabled (char *script)
+{
+        return (script[0] == 'S');
+}
+
+int
+glusterd_hooks_create_hooks_directory (char *basedir)
+{
+        int  ret                                        = -1;
+        int  op                                         = GD_OP_NONE;
+        int  type                                       = GD_COMMIT_HOOK_NONE;
+        char version_dir[PATH_MAX]                      = {0, };
+        char path[PATH_MAX]                             = {0, };
+        char *cmd_subdir                                = NULL;
+        char type_subdir[GD_COMMIT_HOOK_MAX][256]       = {{0, },
+                                                           "pre",
+                                                           "post"};
+        glusterd_conf_t *priv                           = NULL;
+
+        priv = THIS->private;
+
+        snprintf (path, sizeof (path), "%s/hooks", basedir);
+        ret = mkdir_p (path, 0777, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Unable to create %s due"
+                         "to %s", path, strerror (errno));
+                goto out;
+        }
+
+        GLUSTERD_GET_HOOKS_DIR (version_dir, GLUSTERD_HOOK_VER, priv);
+        ret = mkdir_p (version_dir, 0777, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Unable to create %s due "
+                        "to %s", version_dir, strerror (errno));
+                goto out;
+        }
+
+        for (op = GD_OP_NONE+1; op < GD_OP_MAX; op++) {
+                cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
+                if (strlen (cmd_subdir) == 0)
+                        continue;
+
+                snprintf (path, sizeof (path), "%s/%s", version_dir,
+                          cmd_subdir);
+                ret = mkdir_p (path, 0777, _gf_true);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_CRITICAL,
+                                "Unable to create %s due to %s",
+                                path, strerror (errno));
+                        goto out;
+                }
+
+                for (type = GD_COMMIT_HOOK_PRE; type < GD_COMMIT_HOOK_MAX;
+                     type++) {
+                        snprintf (path, sizeof (path), "%s/%s/%s",
+                                  version_dir, cmd_subdir, type_subdir[type]);
+                        ret = mkdir_p (path, 0777, _gf_true);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_CRITICAL,
+                                        "Unable to create %s due to %s",
+                                        path, strerror (errno));
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+char*
+glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op)
+{
+        GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
+
+        return glusterd_hook_dirnames[op];
+}
+
+int
+glusterd_hooks_set_volume_args (dict_t *dict, runner_t *runner)
+{
+        int     i               = 0;
+        int     count           = 0;
+        int     ret             = -1;
+        char    query[1024]     = {0,};
+        char    *key            = NULL;
+        char    *value          = NULL;
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                goto out;
+
+        /* This will not happen unless op_ctx
+         * is corrupted*/
+        if (!count)
+                goto out;
+
+        runner_add_arg (runner, "-o");
+        for (i = 1; (ret == 0); i++) {
+                snprintf (query, sizeof (query), "key%d", i);
+                ret = dict_get_str (dict, query, &key);
+                if (ret)
+                       continue;
+
+                snprintf (query, sizeof (query), "value%d", i);
+                ret = dict_get_str (dict, query, &value);
+                if (ret)
+                       continue;
+
+                runner_argprintf (runner, "%s=%s", key, value);
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+glusterd_hooks_add_op_args (runner_t *runner, glusterd_op_t op,
+                            dict_t *op_ctx, glusterd_commit_hook_type_t type)
+{
+        char                   *hooks_args       = NULL;
+        int                     vol_count        = 0;
+        gf_boolean_t            truth            = _gf_false;
+        glusterd_volinfo_t      *voliter         = NULL;
+        glusterd_conf_t         *priv            = NULL;
+        int                     ret              = -1;
+
+        priv = THIS->private;
+        list_for_each_entry (voliter, &priv->volumes,
+                             vol_list) {
+                if (glusterd_is_volume_started (voliter))
+                        vol_count++;
+        }
+
+        ret = 0;
+        switch (op) {
+                case GD_OP_START_VOLUME:
+                        if (type == GD_COMMIT_HOOK_PRE &&
+                            vol_count == 0)
+                                truth = _gf_true;
+
+                        else if (type == GD_COMMIT_HOOK_POST &&
+                                 vol_count == 1)
+                                truth = _gf_true;
+
+                        else
+                                truth = _gf_false;
+
+                        runner_argprintf (runner, "--first=%s",
+                                          truth? "yes":"no");
+                        break;
+
+                case GD_OP_STOP_VOLUME:
+                        if (type == GD_COMMIT_HOOK_PRE &&
+                            vol_count == 1)
+                                truth = _gf_true;
+
+                        else if (type == GD_COMMIT_HOOK_POST &&
+                                 vol_count == 0)
+                                truth = _gf_true;
+
+                        else
+                                truth = _gf_false;
+
+                        runner_argprintf (runner, "--last=%s",
+                                          truth? "yes":"no");
+                        break;
+
+                case GD_OP_SET_VOLUME:
+                        ret = glusterd_hooks_set_volume_args (op_ctx, runner);
+                        break;
+
+                case GD_OP_GSYNC_CREATE:
+                        ret = dict_get_str (op_ctx, "hooks_args", &hooks_args);
+                        if (ret)
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "No Hooks Arguments.");
+                        else
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "Hooks Args = %s", hooks_args);
+                        if (hooks_args)
+                                runner_argprintf (runner, "%s", hooks_args);
+                        break;
+
+                default:
+                        break;
+
+        }
+
+        return ret;
+}
+
+int
+glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
+                          glusterd_commit_hook_type_t type)
+{
+        xlator_t        *this                   = NULL;
+        glusterd_conf_t *priv                   = NULL;
+        runner_t        runner                  = {0, };
+        struct dirent   *entry                  = NULL;
+        DIR             *hookdir                = NULL;
+        char            *volname                = NULL;
+        char            **lines                 = NULL;
+        int             N                       = 8; /*arbitrary*/
+        int             lineno                  = 0;
+        int             line_count              = 0;
+        int             ret                     = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        ret = dict_get_str (op_ctx, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to get volname "
+                        "from operation context");
+                goto out;
+        }
+
+        hookdir = opendir (hooks_path);
+        if (!hookdir) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s, due "
+                        "to %s", hooks_path, strerror (errno));
+                goto out;
+        }
+
+        lines = GF_CALLOC (1, N * sizeof (*lines), gf_gld_mt_charptr);
+        if (!lines) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = -1;
+        line_count = 0;
+        glusterd_for_each_entry (entry, hookdir);
+        while (entry) {
+                if (line_count == N-1) {
+                        N *= 2;
+                        lines = GF_REALLOC (lines, N * sizeof (char *));
+                        if (!lines)
+                                goto out;
+                }
+
+                if (glusterd_is_hook_enabled (entry->d_name)) {
+                        lines[line_count] = gf_strdup (entry->d_name);
+                        line_count++;
+                }
+
+                glusterd_for_each_entry (entry, hookdir);
+        }
+
+        lines[line_count] = NULL;
+        lines = GF_REALLOC (lines, (line_count + 1) * sizeof (char *));
+        if (!lines)
+                goto out;
+
+        qsort (lines, line_count, sizeof (*lines), glusterd_compare_lines);
+
+        for (lineno = 0; lineno < line_count; lineno++) {
+
+                runinit (&runner);
+                runner_argprintf (&runner, "%s/%s", hooks_path, lines[lineno]);
+                /*Add future command line arguments to hook scripts below*/
+                runner_argprintf (&runner, "--volname=%s", volname);
+                ret = glusterd_hooks_add_op_args (&runner, op, op_ctx, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to add "
+                                "command specific arguments");
+                        goto out;
+                }
+
+                ret = runner_run_reuse (&runner);
+                if (ret) {
+                        runner_log (&runner, this->name, GF_LOG_ERROR,
+                                    "Failed to execute script");
+                } else {
+                        runner_log (&runner, this->name, GF_LOG_INFO,
+                                    "Ran script");
+                }
+                runner_end (&runner);
+        }
+
+        ret = 0;
+out:
+        if (lines) {
+                for (lineno = 0; lineno < line_count+1; lineno++)
+                        GF_FREE (lines[lineno]);
+
+                GF_FREE (lines);
+        }
+
+        if (hookdir)
+                closedir (hookdir);
+
+        return ret;
+}
+
+int
+glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
+                                  dict_t *op_ctx)
+{
+        int                     ret     = -1;
+        glusterd_hooks_stub_t   *stub   = NULL;
+        glusterd_hooks_private_t *hooks_priv = NULL;
+        glusterd_conf_t         *conf   = NULL;
+
+        conf = THIS->private;
+        hooks_priv = conf->hooks_priv;
+
+        ret = glusterd_hooks_stub_init (&stub, scriptdir, op, op_ctx);
+        if (ret)
+                goto out;
+
+        pthread_mutex_lock (&hooks_priv->mutex);
+        {
+                hooks_priv->waitcount++;
+                list_add_tail (&stub->all_hooks, &hooks_priv->list);
+                pthread_cond_signal (&hooks_priv->cond);
+        }
+        pthread_mutex_unlock (&hooks_priv->mutex);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
+                          glusterd_op_t op, dict_t *op_ctx)
+{
+        int                     ret             = -1;
+        glusterd_hooks_stub_t   *hooks_stub     = NULL;
+
+        GF_ASSERT (stub);
+        if (!stub)
+                goto out;
+
+        hooks_stub = GF_CALLOC (1, sizeof (*hooks_stub),
+                                gf_gld_mt_hooks_stub_t);
+        if (!hooks_stub)
+                goto out;
+
+        INIT_LIST_HEAD (&hooks_stub->all_hooks);
+        hooks_stub->op = op;
+        hooks_stub->scriptdir = gf_strdup (scriptdir);
+        if (!hooks_stub->scriptdir)
+                goto out;
+
+        hooks_stub->op_ctx = dict_copy_with_ref (op_ctx, hooks_stub->op_ctx);
+        if (!hooks_stub->op_ctx)
+                goto out;
+
+        *stub = hooks_stub;
+        ret = 0;
+out:
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to initialize "
+                        "post hooks stub");
+                glusterd_hooks_stub_cleanup (hooks_stub);
+        }
+
+        return ret;
+}
+
+void
+glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub)
+{
+        if (!stub) {
+                gf_log_callingfn (THIS->name, GF_LOG_WARNING,
+                                  "hooks_stub is NULL");
+                return;
+        }
+
+        if (stub->op_ctx)
+                dict_unref (stub->op_ctx);
+
+        GF_FREE (stub->scriptdir);
+
+        GF_FREE (stub);
+}
+
+static void*
+hooks_worker (void *args)
+{
+        glusterd_conf_t *conf = NULL;
+        glusterd_hooks_private_t *hooks_priv = NULL;
+        glusterd_hooks_stub_t *stub = NULL;
+
+        THIS = args;
+        conf = THIS->private;
+        hooks_priv = conf->hooks_priv;
+
+        for (;;) {
+                pthread_mutex_lock (&hooks_priv->mutex);
+                {
+                        while (list_empty (&hooks_priv->list)) {
+                                pthread_cond_wait (&hooks_priv->cond,
+                                                   &hooks_priv->mutex);
+                        }
+                        stub = list_entry (hooks_priv->list.next,
+                                           glusterd_hooks_stub_t,
+                                           all_hooks);
+                        list_del_init (&stub->all_hooks);
+                        hooks_priv->waitcount--;
+
+                }
+                pthread_mutex_unlock (&hooks_priv->mutex);
+
+                glusterd_hooks_run_hooks (stub->scriptdir, stub->op,
+                                          stub->op_ctx, GD_COMMIT_HOOK_POST);
+                glusterd_hooks_stub_cleanup (stub);
+        }
+
+        return NULL;
+}
+
+int
+glusterd_hooks_priv_init (glusterd_hooks_private_t **new)
+{
+        int                      ret            = -1;
+        glusterd_hooks_private_t *hooks_priv    = NULL;
+
+        if (!new)
+                goto out;
+
+        hooks_priv = GF_CALLOC (1, sizeof (*hooks_priv),
+                                gf_gld_mt_hooks_priv_t);
+        if (!hooks_priv)
+                goto out;
+
+        pthread_mutex_init (&hooks_priv->mutex, NULL);
+        pthread_cond_init (&hooks_priv->cond, NULL);
+        INIT_LIST_HEAD (&hooks_priv->list);
+        hooks_priv->waitcount = 0;
+
+        *new = hooks_priv;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_hooks_spawn_worker (xlator_t *this)
+{
+        int                       ret          = -1;
+        glusterd_conf_t           *conf        = NULL;
+        glusterd_hooks_private_t  *hooks_priv  = NULL;
+
+
+        ret = glusterd_hooks_priv_init (&hooks_priv);
+        if (ret)
+                goto out;
+
+        conf = this->private;
+        conf->hooks_priv = hooks_priv;
+        ret = pthread_create (&hooks_priv->worker, NULL, hooks_worker,
+                              (void *)this);
+        if (ret)
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn post "
+                        "hooks worker thread");
+out:
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.h b/xlators/mgmt/glusterd/src/glusterd-hooks.h
new file mode 100644
index 000000000..c597ddd2a
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.h
@@ -0,0 +1,89 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_HOOKS_H_
+#define _GLUSTERD_HOOKS_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <fnmatch.h>
+
+#define GLUSTERD_GET_HOOKS_DIR(path, version, priv) \
+        snprintf (path, PATH_MAX, "%s/hooks/%d", priv->workdir,\
+                  version);
+
+#define GLUSTERD_HOOK_VER       1
+
+#define GD_HOOKS_SPECIFIC_KEY   "user.*"
+
+typedef enum glusterd_commit_hook_type {
+        GD_COMMIT_HOOK_NONE = 0,
+        GD_COMMIT_HOOK_PRE,
+        GD_COMMIT_HOOK_POST,
+        GD_COMMIT_HOOK_MAX
+} glusterd_commit_hook_type_t;
+
+typedef struct hooks_private {
+        struct list_head        list;
+        int                     waitcount; //debug purposes
+        pthread_mutex_t         mutex;
+        pthread_cond_t          cond;
+        pthread_t               worker;
+} glusterd_hooks_private_t;
+
+typedef struct hooks_stub {
+        struct list_head        all_hooks;
+        char                    *scriptdir;
+        glusterd_op_t           op;
+        dict_t                  *op_ctx;
+
+} glusterd_hooks_stub_t;
+
+
+static inline gf_boolean_t
+is_key_glusterd_hooks_friendly (char *key)
+{
+        gf_boolean_t is_friendly = _gf_false;
+
+        /* This is very specific to hooks friendly behavior */
+        if (fnmatch (GD_HOOKS_SPECIFIC_KEY, key, FNM_NOESCAPE) == 0) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "user namespace key %s", key);
+                is_friendly = _gf_true;
+        }
+
+        return is_friendly;
+}
+
+int
+glusterd_hooks_create_hooks_directory (char *basedir);
+
+char *
+glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op);
+
+int
+glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
+                          glusterd_commit_hook_type_t type);
+int
+glusterd_hooks_spawn_worker (xlator_t *this);
+
+int
+glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
+                          glusterd_op_t op, dict_t *op_ctx);
+void
+glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub);
+
+int
+glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
+                                      dict_t *op_ctx);
+int
+glusterd_hooks_priv_init (glusterd_hooks_private_t **new);
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.c b/xlators/mgmt/glusterd/src/glusterd-locks.c
new file mode 100644
index 000000000..0af2a186f
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.c
@@ -0,0 +1,637 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "common-utils.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "glusterd-locks.h"
+#include "run.h"
+#include "syscall.h"
+
+#include <signal.h>
+
+#define GF_MAX_LOCKING_ENTITIES 2
+
+/* Valid entities that the mgmt_v3 lock can hold locks upon    *
+ * To add newer entities to be locked, we can just add more    *
+ * entries to this table along with the type and default value */
+valid_entities   valid_types[] = {
+        { "vol",  _gf_true  },
+        { "snap", _gf_false },
+        { NULL              },
+};
+
+static dict_t *mgmt_v3_lock;
+
+/* Checks if the lock request is for a valid entity */
+gf_boolean_t
+glusterd_mgmt_v3_is_type_valid (char *type)
+{
+        int32_t         i   = 0;
+        gf_boolean_t    ret = _gf_false;
+
+        GF_ASSERT (type);
+
+        for (i = 0; valid_types[i].type; i++) {
+                if (!strcmp (type, valid_types[i].type)) {
+                        ret = _gf_true;
+                        break;
+                }
+        }
+
+        return ret;
+}
+
+/* Initialize the global mgmt_v3 lock list(dict) when
+ * glusterd is spawned */
+int32_t
+glusterd_mgmt_v3_lock_init ()
+{
+        int32_t ret = -1;
+
+        mgmt_v3_lock = dict_new ();
+        if (!mgmt_v3_lock) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/* Destroy the global mgmt_v3 lock list(dict) when
+ * glusterd cleanup is performed */
+void
+glusterd_mgmt_v3_lock_fini ()
+{
+        if (mgmt_v3_lock)
+                dict_destroy (mgmt_v3_lock);
+}
+
+int32_t
+glusterd_get_mgmt_v3_lock_owner (char *key, uuid_t *uuid)
+{
+        int32_t             ret      = -1;
+        mgmt_v3_lock_obj   *lock_obj = NULL;
+        uuid_t              no_owner = {"\0"};
+        xlator_t           *this = NULL;
+
+        GF_ASSERT(THIS);
+        this = THIS;
+
+        if (!key || !uuid) {
+                gf_log (this->name, GF_LOG_ERROR, "key or uuid is null.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(mgmt_v3_lock, key, (void **) &lock_obj);
+        if (!ret)
+                uuid_copy (*uuid, lock_obj->lock_owner);
+        else
+                uuid_copy (*uuid, no_owner);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* This function is called with the locked_count and type, to   *
+ * release all the acquired locks. */
+static int32_t
+glusterd_release_multiple_locks_per_entity (dict_t *dict, uuid_t uuid,
+                                            int32_t locked_count,
+                                            char *type)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        i                     = -1;
+        int32_t        op_ret                = 0;
+        int32_t        ret                   = -1;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        if (locked_count == 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No %s locked as part of this transaction",
+                        type);
+                goto out;
+        }
+
+        /* Release all the locks held */
+        for (i = 0; i < locked_count; i++) {
+                snprintf (name_buf, sizeof(name_buf),
+                          "%sname%d", type, i+1);
+
+                /* Looking for volname1, volname2 or snapname1, *
+                 * as key in the dict snapname2 */
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get %s locked_count = %d",
+                                name_buf, locked_count);
+                        op_ret = ret;
+                        continue;
+                }
+
+                ret = glusterd_mgmt_v3_unlock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release lock for %s.",
+                                name);
+                        op_ret = ret;
+               }
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", op_ret);
+        return op_ret;
+}
+
+/* Given the count and type of the entity this function acquires     *
+ * locks on multiple elements of the same entity. For example:       *
+ * If type is "vol" this function tries to acquire locks on multiple *
+ * volumes */
+static int32_t
+glusterd_acquire_multiple_locks_per_entity (dict_t *dict, uuid_t uuid,
+                                            int32_t count, char *type)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        i                     = -1;
+        int32_t        ret                   = -1;
+        int32_t        locked_count          = 0;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        /* Locking one element after other */
+        for (i = 0; i < count; i++) {
+                snprintf (name_buf, sizeof(name_buf),
+                          "%sname%d", type, i+1);
+
+                /* Looking for volname1, volname2 or snapname1, *
+                 * as key in the dict snapname2 */
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get %s count = %d",
+                                name_buf, count);
+                        break;
+                }
+
+                ret = glusterd_mgmt_v3_lock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire lock for %s %s "
+                                "on behalf of %s. Reversing "
+                                "this transaction", type, name,
+                                uuid_utoa(uuid));
+                        break;
+                }
+                locked_count++;
+        }
+
+        if (count == locked_count) {
+                /* If all locking ops went successfuly, return as success */
+                ret = 0;
+                goto out;
+        }
+
+        /* If we failed to lock one element, unlock others and return failure */
+        ret = glusterd_release_multiple_locks_per_entity (dict, uuid,
+                                                          locked_count,
+                                                          type);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to release multiple %s locks",
+                        type);
+        }
+        ret = -1;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Given the type of entity, this function figures out if it should unlock a   *
+ * single element of multiple elements of the said entity. For example:        *
+ * if the type is "vol", this function will accordingly unlock a single volume *
+ * or multiple volumes */
+static int32_t
+glusterd_mgmt_v3_unlock_entity (dict_t *dict, uuid_t uuid, char *type,
+                                gf_boolean_t default_value)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        count                 = -1;
+        int32_t        ret                   = -1;
+        gf_boolean_t   hold_locks            = _gf_false;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        snprintf (name_buf, sizeof(name_buf), "hold_%s_locks", type);
+        hold_locks = dict_get_str_boolean (dict, name_buf, default_value);
+
+        if (hold_locks == _gf_false) {
+                /* Locks were not held for this particular entity *
+                 * Hence nothing to release */
+                ret = 0;
+                goto out;
+        }
+
+        /* Looking for volcount or snapcount in the dict */
+        snprintf (name_buf, sizeof(name_buf), "%scount", type);
+        ret = dict_get_int32 (dict, name_buf, &count);
+        if (ret) {
+                /* count is not present. Only one *
+                 * element name needs to be unlocked */
+                snprintf (name_buf, sizeof(name_buf), "%sname",
+                          type);
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch %sname", type);
+                        goto out;
+                }
+
+                ret = glusterd_mgmt_v3_unlock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release lock for %s %s "
+                                "on behalf of %s.", type, name,
+                                uuid_utoa(uuid));
+                        goto out;
+                }
+        } else {
+                /* Unlocking one element name after another */
+                ret = glusterd_release_multiple_locks_per_entity (dict,
+                                                                  uuid,
+                                                                  count,
+                                                                  type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release all %s locks", type);
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Given the type of entity, this function figures out if it should lock a   *
+ * single element or multiple elements of the said entity. For example:      *
+ * if the type is "vol", this function will accordingly lock a single volume *
+ * or multiple volumes */
+static int32_t
+glusterd_mgmt_v3_lock_entity (dict_t *dict, uuid_t uuid, char *type,
+                              gf_boolean_t default_value)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        count                 = -1;
+        int32_t        ret                   = -1;
+        gf_boolean_t   hold_locks            = _gf_false;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        snprintf (name_buf, sizeof(name_buf), "hold_%s_locks", type);
+        hold_locks = dict_get_str_boolean (dict, name_buf, default_value);
+
+        if (hold_locks == _gf_false) {
+                /* Not holding locks for this particular entity */
+                ret = 0;
+                goto out;
+        }
+
+        /* Looking for volcount or snapcount in the dict */
+        snprintf (name_buf, sizeof(name_buf), "%scount", type);
+        ret = dict_get_int32 (dict, name_buf, &count);
+        if (ret) {
+                /* count is not present. Only one *
+                 * element name needs to be locked */
+                snprintf (name_buf, sizeof(name_buf), "%sname",
+                          type);
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch %sname", type);
+                        goto out;
+                }
+
+                ret = glusterd_mgmt_v3_lock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire lock for %s %s "
+                                "on behalf of %s.", type, name,
+                                uuid_utoa(uuid));
+                        goto out;
+                }
+        } else {
+                /* Locking one element name after another */
+                ret = glusterd_acquire_multiple_locks_per_entity (dict,
+                                                                  uuid,
+                                                                  count,
+                                                                  type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire all %s locks", type);
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Try to release locks of multiple entities like *
+ * volume, snaps etc. */
+int32_t
+glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid)
+{
+        int32_t        i                     = -1;
+        int32_t        ret                   = -1;
+        int32_t        op_ret                = 0;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "dict is null.");
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; valid_types[i].type; i++) {
+                ret = glusterd_mgmt_v3_unlock_entity
+                                            (dict, uuid,
+                                             valid_types[i].type,
+                                             valid_types[i].default_value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to unlock all %s",
+                                valid_types[i].type);
+                        op_ret = ret;
+                }
+        }
+
+        ret = op_ret;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Try to acquire locks on multiple entities like *
+ * volume, snaps etc. */
+int32_t
+glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid)
+{
+        int32_t        i                     = -1;
+        int32_t        ret                   = -1;
+        int32_t        locked_count          = 0;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "dict is null.");
+                ret = -1;
+                goto out;
+        }
+
+        /* Locking one entity after other */
+        for (i = 0; valid_types[i].type; i++) {
+                ret = glusterd_mgmt_v3_lock_entity
+                                            (dict, uuid,
+                                             valid_types[i].type,
+                                             valid_types[i].default_value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to lock all %s",
+                                valid_types[i].type);
+                        break;
+                }
+                locked_count++;
+        }
+
+        if (locked_count == GF_MAX_LOCKING_ENTITIES) {
+                /* If all locking ops went successfuly, return as success */
+                ret = 0;
+                goto out;
+        }
+
+        /* If we failed to lock one entity, unlock others and return failure */
+        for (i = 0; i < locked_count; i++) {
+                ret = glusterd_mgmt_v3_unlock_entity
+                                              (dict, uuid,
+                                               valid_types[i].type,
+                                               valid_types[i].default_value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to unlock all %s",
+                                valid_types[i].type);
+                }
+        }
+        ret = -1;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, char *type)
+{
+        char                 key[PATH_MAX] = "";
+        int32_t              ret           = -1;
+        mgmt_v3_lock_obj    *lock_obj      = NULL;
+        gf_boolean_t         is_valid      = _gf_true;
+        uuid_t               owner         = {0};
+        xlator_t            *this          = NULL;
+
+        GF_ASSERT(THIS);
+        this = THIS;
+
+        if (!name || !type) {
+                gf_log (this->name, GF_LOG_ERROR, "name or type is null.");
+                ret = -1;
+                goto out;
+        }
+
+        is_valid = glusterd_mgmt_v3_is_type_valid (type);
+        if (is_valid != _gf_true) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid entity. Cannot perform locking "
+                        "operation on %s types", type);
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (key, sizeof(key), "%s_%s", name, type);
+        if (ret != strlen(name) + 1 + strlen(type)) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create key");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Trying to acquire lock of %s %s for %s as %s",
+                type, name, uuid_utoa (uuid), key);
+
+        ret = glusterd_get_mgmt_v3_lock_owner (key, &owner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to get mgmt_v3 lock owner");
+                goto out;
+        }
+
+        /* If the lock has already been held for the given volume
+         * we fail */
+        if (!uuid_is_null (owner)) {
+                gf_log (this->name, GF_LOG_ERROR, "Lock for %s held by %s",
+                        name, uuid_utoa (owner));
+                ret = -1;
+                goto out;
+        }
+
+        lock_obj = GF_CALLOC (1, sizeof(mgmt_v3_lock_obj),
+                              gf_common_mt_mgmt_v3_lock_obj_t);
+        if (!lock_obj) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (lock_obj->lock_owner, uuid);
+
+        ret = dict_set_bin (mgmt_v3_lock, key, lock_obj,
+                            sizeof(*lock_obj));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set lock owner in mgmt_v3 lock");
+                if (lock_obj)
+                        GF_FREE (lock_obj);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Lock for %s %s successfully held by %s",
+                type, name, uuid_utoa (uuid));
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type)
+{
+        char           key[PATH_MAX] = "";
+        int32_t        ret           = -1;
+        gf_boolean_t   is_valid      = _gf_true;
+        uuid_t         owner         = {0};
+        xlator_t            *this          = NULL;
+
+        GF_ASSERT(THIS);
+        this = THIS;
+
+        if (!name || !type) {
+                gf_log (this->name, GF_LOG_ERROR, "name is null.");
+                ret = -1;
+                goto out;
+        }
+
+        is_valid = glusterd_mgmt_v3_is_type_valid (type);
+        if (is_valid != _gf_true) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid entity. Cannot perform unlocking "
+                        "operation on %s types", type);
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (key, sizeof(key), "%s_%s",
+                        name, type);
+        if (ret != strlen(name) + 1 + strlen(type)) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create key");
+                ret = -1;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Trying to release lock of %s %s for %s as %s",
+                type, name, uuid_utoa (uuid), key);
+
+        ret = glusterd_get_mgmt_v3_lock_owner (key, &owner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to get mgmt_v3 lock owner");
+                goto out;
+        }
+
+        if (uuid_is_null (owner)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Lock for %s %s not held", type, name);
+                ret = -1;
+                goto out;
+        }
+
+        ret = uuid_compare (uuid, owner);
+        if (ret) {
+
+                gf_log (this->name, GF_LOG_ERROR, "Lock owner mismatch. "
+                        "Lock for %s %s held by %s",
+                        type, name, uuid_utoa (owner));
+                goto out;
+        }
+
+        /* Removing the mgmt_v3 lock from the global list */
+        dict_del (mgmt_v3_lock, key);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Lock for %s %s successfully released",
+                type, name);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.h b/xlators/mgmt/glusterd/src/glusterd-locks.h
new file mode 100644
index 000000000..83eb8c997
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.h
@@ -0,0 +1,51 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_LOCKS_H_
+#define _GLUSTERD_LOCKS_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+typedef struct mgmt_v3_lock_object_ {
+        uuid_t              lock_owner;
+} mgmt_v3_lock_obj;
+
+typedef struct mgmt_v3_lock_valid_entities {
+        char          *type;          /* Entity type like vol, snap */
+        gf_boolean_t   default_value; /* The default value that  *
+                                       * determines if the locks *
+                                       * should be held for that *
+                                       * entity */
+} valid_entities;
+
+int32_t
+glusterd_mgmt_v3_lock_init ();
+
+void
+glusterd_mgmt_v3_lock_fini ();
+
+int32_t
+glusterd_get_mgmt_v3_lock_owner (char *volname, uuid_t *uuid);
+
+int32_t
+glusterd_mgmt_v3_lock (const char *key, uuid_t uuid, char *type);
+
+int32_t
+glusterd_mgmt_v3_unlock (const char *key, uuid_t uuid, char *type);
+
+int32_t
+glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid);
+
+int32_t
+glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid);
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-log-ops.c b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
index 93e037f87..33bd95c03 100644
--- a/xlators/mgmt/glusterd/src/glusterd-log-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -34,18 +24,22 @@
 #include <signal.h>
 
 int
-glusterd_handle_log_rotate (rpcsvc_request_t *req)
+__glusterd_handle_log_rotate (rpcsvc_request_t *req)
 {
         int32_t                 ret     = -1;
         gf_cli_req              cli_req = {{0,}};
         dict_t                 *dict    = NULL;
         glusterd_op_t           cli_op = GD_OP_LOG_ROTATE;
         char                   *volname = NULL;
+        char                   msg[2048] = {0,};
+        xlator_t               *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -59,44 +53,50 @@ glusterd_handle_log_rotate (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (msg, sizeof (msg), "Failed to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received log rotate req "
+        gf_log (this->name, GF_LOG_INFO, "Received log rotate req "
                 "for volume %s", volname);
 
         ret = dict_set_uint64 (dict, "rotate-key", (uint64_t)time (NULL));
         if (ret)
                 goto out;
 
-        ret = glusterd_op_begin (req, GD_OP_LOG_ROTATE, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_LOG_ROTATE, dict);
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
-        if (ret)
+        if (ret) {
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, msg);
+        }
 
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);
+        free (cli_req.dict.dict_val);
         return ret;
 }
 
+int
+glusterd_handle_log_rotate (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_log_rotate);
+}
+
 /* op-sm */
 int
 glusterd_op_stage_log_rotate (dict_t *dict, char **op_errstr)
@@ -142,16 +142,13 @@ glusterd_op_stage_log_rotate (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        if (strchr (brick, ':')) {
-                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo, NULL,
-                                                              GF_PATH_COMPLETE);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "Incorrect brick %s "
-                                  "for volume %s", brick, volname);
-                        gf_log ("", GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
+        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo, NULL);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Incorrect brick %s "
+                          "for volume %s", brick, volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
         }
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
@@ -170,7 +167,6 @@ glusterd_op_log_rotate (dict_t *dict)
         xlator_t             *this               = NULL;
         char                 *volname            = NULL;
         char                 *brick              = NULL;
-        char                  path[PATH_MAX]     = {0,};
         char                  logfile[PATH_MAX]  = {0,};
         char                  pidfile[PATH_MAX]  = {0,};
         FILE                 *file               = NULL;
@@ -202,15 +198,11 @@ glusterd_op_log_rotate (dict_t *dict)
         if (ret)
                 goto cont;
 
-        if (!strchr (brick, ':'))
-                brick = NULL;
-        else {
-                ret = glusterd_brickinfo_from_brick (brick, &tmpbrkinfo);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "cannot get brickinfo from brick");
-                        goto out;
-                }
+        ret = glusterd_brickinfo_new_from_brick (brick, &tmpbrkinfo);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "cannot get brickinfo from brick");
+                goto out;
         }
 
 cont:
@@ -220,7 +212,7 @@ cont:
 
         ret = -1;
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (uuid_compare (brickinfo->uuid, priv->uuid))
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
                         continue;
 
                 if (brick &&
@@ -230,10 +222,7 @@ cont:
 
                 valid_brick = 1;
 
-                GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname,
-                                            brickinfo->path);
-
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
                 file = fopen (pidfile, "r+");
                 if (!file) {
                         gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
index 196f5f50b..e6f6a0333 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __GLUSTERD_MEM_TYPES_H__
 #define __GLUSTERD_MEM_TYPES_H__
 
@@ -70,8 +60,16 @@ typedef enum gf_gld_mem_types_ {
         gf_gld_mt_mount_comp_container          = gf_common_mt_end + 44,
         gf_gld_mt_mount_component               = gf_common_mt_end + 45,
         gf_gld_mt_mount_spec                    = gf_common_mt_end + 46,
-        gf_gld_mt_nodesrv_t                     = gf_common_mt_end + 47,
-        gf_gld_mt_end                           = gf_common_mt_end + 48,
+        gf_gld_mt_georep_meet_spec              = gf_common_mt_end + 47,
+        gf_gld_mt_nodesrv_t                     = gf_common_mt_end + 48,
+        gf_gld_mt_charptr                       = gf_common_mt_end + 49,
+        gf_gld_mt_hooks_stub_t                  = gf_common_mt_end + 50,
+        gf_gld_mt_hooks_priv_t                  = gf_common_mt_end + 51,
+        gf_gld_mt_mop_commit_req_t              = gf_common_mt_end + 52,
+        gf_gld_mt_int                           = gf_common_mt_end + 53,
+        gf_gld_mt_snap_t                        = gf_common_mt_end + 54,
+        gf_gld_mt_missed_snapinfo_t             = gf_common_mt_end + 55,
+        gf_gld_mt_end                           = gf_common_mt_end + 56,
 } gf_gld_mem_types_t;
 #endif
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
new file mode 100644
index 000000000..a5f38ce9c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
@@ -0,0 +1,924 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+/* rpc related syncops */
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+#include "glusterd-syncop.h"
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-locks.h"
+#include "glusterd-mgmt.h"
+#include "glusterd-op-sm.h"
+
+static int
+glusterd_mgmt_v3_null (rpcsvc_request_t *req)
+{
+        return 0;
+}
+
+static int
+glusterd_mgmt_v3_lock_send_resp (rpcsvc_request_t *req, int32_t status)
+{
+
+        gd1_mgmt_v3_lock_rsp          rsp   = {{0},};
+        int                           ret   = -1;
+        xlator_t                     *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+
+        glusterd_get_uuid (&rsp.uuid);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to mgmt_v3 lock, ret: %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_synctasked_mgmt_v3_lock (rpcsvc_request_t *req,
+                                  gd1_mgmt_v3_lock_req *lock_req,
+                                  glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                         ret         = -1;
+        xlator_t                       *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (ctx);
+        GF_ASSERT (ctx->dict);
+
+        /* Trying to acquire multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_lock (ctx->dict, ctx->uuid);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to acquire mgmt_v3 locks for %s",
+                         uuid_utoa (ctx->uuid));
+
+        ret = glusterd_mgmt_v3_lock_send_resp (req, ret);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_op_state_machine_mgmt_v3_lock (rpcsvc_request_t *req,
+                                        gd1_mgmt_v3_lock_req *lock_req,
+                                        glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                         ret         = -1;
+        xlator_t                       *this        = NULL;
+        glusterd_op_info_t              txn_op_info = {{0},};
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &lock_req->op,
+                                  ctx->dict, req);
+
+        ret = glusterd_set_txn_opinfo (&lock_req->txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK,
+                                           &lock_req->txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_LOCK");
+
+out:
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_mgmt_v3_lock_fn (rpcsvc_request_t *req)
+{
+        gd1_mgmt_v3_lock_req            lock_req    = {{0},};
+        int32_t                         ret         = -1;
+        glusterd_peerinfo_t            *peerinfo    = NULL;
+        glusterd_op_lock_ctx_t         *ctx         = NULL;
+        xlator_t                       *this        = NULL;
+        gf_boolean_t                    is_synctasked = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received mgmt_v3 lock req "
+                "from uuid: %s", uuid_utoa (lock_req.uuid));
+
+        if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (lock_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+        if (!ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ctx->uuid, lock_req.uuid);
+        ctx->req = req;
+
+        ctx->dict = dict_new ();
+        if (!ctx->dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (lock_req.dict.dict_val,
+                                lock_req.dict.dict_len, &ctx->dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        is_synctasked = dict_get_str_boolean (ctx->dict,
+                                              "is_synctasked", _gf_false);
+        if (is_synctasked)
+                ret = glusterd_synctasked_mgmt_v3_lock (req, &lock_req, ctx);
+        else
+                ret = glusterd_op_state_machine_mgmt_v3_lock (req, &lock_req,
+                                                              ctx);
+
+out:
+
+        if (ret) {
+                if (ctx->dict)
+                        dict_unref (ctx->dict);
+                if (ctx)
+                        GF_FREE (ctx);
+        }
+
+        free (lock_req.dict.dict_val);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_pre_validate_send_resp (rpcsvc_request_t   *req,
+                                        int32_t op, int32_t status,
+                                        char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_pre_val_rsp          rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to pre validation, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_pre_validate_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_pre_val_req          op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode pre validation "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_pre_validate_fn (op_req.op, dict, &op_errstr,
+                                         rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pre Validation failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_pre_validate_send_resp (req, op_req.op,
+                                                      ret, op_errstr,
+                                                      rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send Pre Validation "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_brick_op_send_resp (rpcsvc_request_t   *req,
+                                    int32_t op, int32_t status,
+                                    char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_brick_op_rsp         rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to brick op, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_brick_op_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_brick_op_req         op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode brick op "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_brick_op_fn (op_req.op, dict, &op_errstr,
+                                     rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Brick Op failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_brick_op_send_resp (req, op_req.op,
+                                                  ret, op_errstr,
+                                                  rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send brick op "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_commit_send_resp (rpcsvc_request_t   *req,
+                                  int32_t op, int32_t status,
+                                  char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_commit_rsp           rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to commit, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_commit_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_commit_req           op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_commit_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_commit_fn (op_req.op, dict, &op_errstr,
+                                   rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "commit failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_commit_send_resp (req, op_req.op,
+                                                ret, op_errstr,
+                                                rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send commit "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_post_validate_send_resp (rpcsvc_request_t   *req,
+                                         int32_t op, int32_t status,
+                                         char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_post_val_rsp         rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to post validation, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_post_validate_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_post_val_req         op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_post_val_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode post validation "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_post_validate_fn (op_req.op, op_req.op_ret, dict,
+                                           &op_errstr, rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Post Validation failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_post_validate_send_resp (req, op_req.op,
+                                                       ret, op_errstr,
+                                                       rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send Post Validation "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
+{
+
+        gd1_mgmt_v3_unlock_rsp          rsp   = {{0},};
+        int                             ret   = -1;
+        xlator_t                       *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+
+        glusterd_get_uuid (&rsp.uuid);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to mgmt_v3 unlock, ret: %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_synctasked_mgmt_v3_unlock (rpcsvc_request_t *req,
+                                    gd1_mgmt_v3_unlock_req *unlock_req,
+                                    glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                         ret         = -1;
+        xlator_t                       *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (ctx);
+
+        /* Trying to release multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_unlock (ctx->dict, ctx->uuid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to release mgmt_v3 locks for %s",
+                        uuid_utoa(ctx->uuid));
+        }
+
+        ret = glusterd_mgmt_v3_unlock_send_resp (req, ret);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+
+static int
+glusterd_op_state_machine_mgmt_v3_unlock (rpcsvc_request_t *req,
+                                          gd1_mgmt_v3_unlock_req *lock_req,
+                                          glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                           ret      = -1;
+        xlator_t                         *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK,
+                                           &lock_req->txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_UNLOCK");
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_mgmt_v3_unlock_fn (rpcsvc_request_t *req)
+{
+        gd1_mgmt_v3_unlock_req            lock_req = {{0},};
+        int32_t                           ret      = -1;
+        glusterd_op_lock_ctx_t           *ctx      = NULL;
+        glusterd_peerinfo_t              *peerinfo = NULL;
+        xlator_t                         *this     = NULL;
+        gf_boolean_t                      is_synctasked = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received mgmt_v3 unlock req "
+                "from uuid: %s", uuid_utoa (lock_req.uuid));
+
+        if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (lock_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+        if (!ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ctx->uuid, lock_req.uuid);
+        ctx->req = req;
+
+        ctx->dict = dict_new ();
+        if (!ctx->dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (lock_req.dict.dict_val,
+                                lock_req.dict.dict_len, &ctx->dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        is_synctasked = dict_get_str_boolean (ctx->dict,
+                                              "is_synctasked", _gf_false);
+        if (is_synctasked)
+                ret = glusterd_synctasked_mgmt_v3_unlock (req, &lock_req, ctx);
+        else
+                ret = glusterd_op_state_machine_mgmt_v3_unlock (req, &lock_req,
+                                                                ctx);
+
+out:
+
+        if (ret) {
+                if (ctx->dict)
+                        dict_unref (ctx->dict);
+                if (ctx)
+                        GF_FREE (ctx);
+        }
+
+        free (lock_req.dict.dict_val);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_handle_mgmt_v3_lock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_mgmt_v3_lock_fn);
+}
+
+static int
+glusterd_handle_pre_validate (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_pre_validate_fn);
+}
+
+static int
+glusterd_handle_brick_op (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_brick_op_fn);
+}
+
+static int
+glusterd_handle_commit (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_commit_fn);
+}
+
+static int
+glusterd_handle_post_validate (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_post_validate_fn);
+}
+
+int
+glusterd_handle_mgmt_v3_unlock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_mgmt_v3_unlock_fn);
+}
+
+rpcsvc_actor_t gd_svc_mgmt_v3_actors[] = {
+        [GLUSTERD_MGMT_V3_NULL]          = { "NULL",           GLUSTERD_MGMT_V3_NULL,          glusterd_mgmt_v3_null,          NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_LOCK]          = { "MGMT_V3_LOCK",   GLUSTERD_MGMT_V3_LOCK,          glusterd_handle_mgmt_v3_lock,   NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_PRE_VALIDATE]  = { "PRE_VAL",        GLUSTERD_MGMT_V3_PRE_VALIDATE,  glusterd_handle_pre_validate,   NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_BRICK_OP]      = { "BRCK_OP",        GLUSTERD_MGMT_V3_BRICK_OP,      glusterd_handle_brick_op,       NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_COMMIT]        = { "COMMIT",         GLUSTERD_MGMT_V3_COMMIT,        glusterd_handle_commit,         NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_POST_VALIDATE] = { "POST_VAL",       GLUSTERD_MGMT_V3_POST_VALIDATE, glusterd_handle_post_validate,  NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_UNLOCK]        = { "MGMT_V3_UNLOCK", GLUSTERD_MGMT_V3_UNLOCK,        glusterd_handle_mgmt_v3_unlock, NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_mgmt_v3_prog = {
+        .progname  = "GlusterD svc mgmt v3",
+        .prognum   = GD_MGMT_V3_PROGRAM,
+        .progver   = GD_MGMT_V3_VERSION,
+        .numactors = GLUSTERD_MGMT_V3_MAXVALUE,
+        .actors    = gd_svc_mgmt_v3_actors,
+        .synctask  = _gf_true,
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
new file mode 100644
index 000000000..d52532e54
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
@@ -0,0 +1,1893 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+/* rpc related syncops */
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+#include "glusterd-syncop.h"
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-locks.h"
+#include "glusterd-mgmt.h"
+#include "glusterd-op-sm.h"
+
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+
+
+static void
+gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                           char *op_errstr, int op_code,
+                           glusterd_peerinfo_t *peerinfo, u_char *uuid)
+{
+        char      *peer_str          = NULL;
+        char       err_str[PATH_MAX] = "Please check log file for details.";
+        char       op_err[PATH_MAX]  = "";
+        int32_t    len               = -1;
+        xlator_t  *this              = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (args);
+        GF_ASSERT (uuid);
+
+        if (op_ret) {
+                args->op_ret = op_ret;
+                args->op_errno = op_errno;
+
+                if (peerinfo)
+                        peer_str = peerinfo->hostname;
+                else
+                        peer_str = uuid_utoa (uuid);
+
+                if (op_errstr && strcmp (op_errstr, "")) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "Error: %s", op_errstr);
+                        err_str[len] = '\0';
+                }
+
+                switch (op_code){
+                case GLUSTERD_MGMT_V3_LOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Locking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_PRE_VALIDATE:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Pre Validation failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_BRICK_OP:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Brick ops failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_COMMIT:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Commit failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_POST_VALIDATE:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Post Validation failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_UNLOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Unlocking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                }
+                op_err[len] = '\0';
+
+                if (args->errstr) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s\n%s", args->errstr,
+                                        op_err);
+                        GF_FREE (args->errstr);
+                        args->errstr = NULL;
+                } else
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s", op_err);
+                err_str[len] = '\0';
+
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_err);
+                args->errstr = gf_strdup (err_str);
+        }
+
+        return;
+}
+
+int32_t
+gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
+                            char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+               case GD_OP_SNAP:
+               {
+                       ret = glusterd_snapshot_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                       if (ret) {
+                               gf_log (this->name, GF_LOG_WARNING,
+                                       "Snapshot Prevalidate Failed");
+                               goto out;
+                       }
+
+                       break;
+               }
+               default:
+                       break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
+                        char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+        case GD_OP_SNAP:
+        {
+                ret = glusterd_snapshot_brickop (dict, op_errstr, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "snapshot brickop "
+                                "failed");
+                        goto out;
+                }
+                break;
+        }
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
+                      char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+               case GD_OP_SNAP:
+               {
+                       ret = glusterd_snapshot (dict, op_errstr, rsp_dict);
+                       if (ret) {
+                               gf_log (this->name, GF_LOG_WARNING,
+                                       "Snapshot Commit Failed");
+                               goto out;
+                       }
+                       break;
+               }
+               default:
+                       break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
+                             char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+               case GD_OP_SNAP:
+               {
+                       ret = glusterd_snapshot_postvalidate (dict, op_ret,
+                                                             op_errstr,
+                                                             rsp_dict);
+                       if (ret) {
+                               gf_log (this->name, GF_LOG_WARNING,
+                                       "postvalidate operation failed");
+                               goto out;
+                       }
+                       break;
+               }
+               default:
+                       break;
+        }
+
+        ret = 0;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_lock_rsp        rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        /* Even though the lock command has failed, while collating the errors
+           (gd_mgmt_v3_collate_errors), args->op_ret and args->op_errno will be
+           used. @args is obtained from frame->local. So before checking the
+           status of the request and going out if its a failure, args should be
+           set to frame->local. Otherwise, while collating args will be NULL.
+           This applies to other phases such as prevalidate, brickop, commit and
+           postvalidate also.
+        */
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_LOCK,
+                                   peerinfo, rsp.uuid);
+        free (rsp.dict.dict_val);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                     int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_lock_cbk_fn);
+}
+
+int
+gd_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
+                 glusterd_peerinfo_t *peerinfo,
+                 struct syncargs *args, uuid_t my_uuid,
+                 uuid_t recv_uuid)
+{
+        gd1_mgmt_v3_lock_req     req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+        int32_t                  ret  = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_LOCK,
+                                        gd_mgmt_v3_lock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_lock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_initiate_lockdown (glusterd_conf_t  *conf, glusterd_op_t op,
+                                    dict_t *dict, char **op_errstr, int npeers,
+                                    gf_boolean_t  *is_acquired)
+{
+        char                *volname    = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (is_acquired);
+
+        peers = &conf->xaction_peers;
+
+        /* Trying to acquire multiple mgmt_v3 locks on local node */
+        ret = glusterd_multiple_mgmt_v3_lock (dict, MY_UUID);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to acquire mgmt_v3 locks on localhost");
+                goto out;
+        }
+
+        *is_acquired = _gf_true;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending mgmt_v3 lock req to other nodes in the cluster */
+        gd_syncargs_init (&args, NULL);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_lock (op, dict, peerinfo, &args,
+                                 MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.errstr)
+                *op_errstr = gf_strdup (args.errstr);
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent lock op req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        if (ret) {
+                if (*op_errstr)
+                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                *op_errstr);
+
+                if (volname)
+                        ret = gf_asprintf (op_errstr,
+                                           "Another transaction is in progress "
+                                           "for %s. Please try again after "
+                                           "sometime.", volname);
+                else
+                        ret = gf_asprintf (op_errstr,
+                                           "Another transaction is in progress "
+                                           "Please try again after sometime.");
+
+                if (ret == -1)
+                        *op_errstr = NULL;
+
+                ret = -1;
+        }
+
+        return ret;
+}
+
+int
+glusterd_pre_validate_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp)
+{
+        int32_t              ret      = 0;
+        xlator_t            *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (aggr);
+        GF_ASSERT (rsp);
+
+        switch (op) {
+        case GD_OP_SNAP:
+                ret = glusterd_snap_pre_validate_use_rsp_dict (aggr, rsp);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to aggregate prevalidate "
+                                "response dictionaries.");
+                        goto out;
+                }
+                break;
+        default:
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Invalid op (%s)", gd_op_list[op]);
+
+                break;
+        }
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_pre_validate_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_pre_val_rsp      rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        dict_t                     *rsp_dict   = NULL;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        pthread_mutex_lock (&args->lock_dict);
+        {
+                ret = glusterd_pre_validate_aggr_rsp_dict (rsp.op, args->dict,
+                                                           rsp_dict);
+        }
+        pthread_mutex_unlock (&args->lock_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                if (!rsp.op_ret)
+                        op_ret = ret;
+                else {
+                        op_ret = rsp.op_ret;
+                        op_errno = rsp.op_errno;
+                }
+        } else {
+                op_ret = rsp.op_ret;
+                op_errno = rsp.op_errno;
+        }
+
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                  GLUSTERD_MGMT_V3_PRE_VALIDATE,
+                                  peerinfo, rsp.uuid);
+
+        if (rsp.op_errstr)
+                free (rsp.op_errstr);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_pre_validate_cbk (struct rpc_req *req, struct iovec *iov,
+                             int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_pre_validate_cbk_fn);
+}
+
+int
+gd_mgmt_v3_pre_validate_req (glusterd_op_t op, dict_t *op_ctx,
+                             glusterd_peerinfo_t *peerinfo,
+                             struct syncargs *args, uuid_t my_uuid,
+                             uuid_t recv_uuid)
+{
+        int32_t                  ret   = -1;
+        gd1_mgmt_v3_pre_val_req  req   = {{0},};
+        glusterd_conf_t         *conf  = THIS->private;
+        xlator_t                *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_PRE_VALIDATE,
+                                        gd_mgmt_v3_pre_validate_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_pre_val_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_pre_validate (glusterd_conf_t  *conf, glusterd_op_t op,
+                               dict_t *req_dict, char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Pre Validation on local node */
+        ret = gd_mgmt_v3_pre_validate_fn (op, req_dict, op_errstr,
+                                          rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pre Validation failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Pre-validation failed "
+                                           "on localhost. Please "
+                                           "check log file for details");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        ret = glusterd_pre_validate_aggr_rsp_dict (op, req_dict,
+                                                   rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending Pre Validation req to other nodes in the cluster */
+        gd_syncargs_init (&args, req_dict);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_pre_validate_req (op, req_dict, peerinfo, &args,
+                                             MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pre Validation failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent pre valaidation req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_build_payload (dict_t **req, char **op_errstr, dict_t *dict,
+                                glusterd_op_t op)
+{
+        int32_t                 ret      = -1;
+        dict_t                 *req_dict = NULL;
+        xlator_t               *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (dict);
+
+        req_dict = dict_new ();
+        if (!req_dict)
+                goto out;
+
+        switch (op) {
+                case GD_OP_SNAP:
+                        dict_copy (dict, req_dict);
+                        break;
+                default:
+                        break;
+        }
+
+        *req = req_dict;
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_brick_op_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_brick_op_rsp     rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        /* If the operation failed, then iov can be NULL. So better check the
+           status of the operation and then worry about iov (if the status of
+           the command is success)
+        */
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_BRICK_OP,
+                                   peerinfo, rsp.uuid);
+
+        if (rsp.op_errstr)
+                free (rsp.op_errstr);
+
+        free (rsp.dict.dict_val);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_brick_op_cbk_fn);
+}
+
+int
+gd_mgmt_v3_brick_op_req (glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_peerinfo_t *peerinfo,
+                         struct syncargs *args, uuid_t my_uuid,
+                         uuid_t recv_uuid)
+{
+        int32_t                   ret  = -1;
+        gd1_mgmt_v3_brick_op_req  req  = {{0},};
+        glusterd_conf_t          *conf = THIS->private;
+        xlator_t                 *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_BRICK_OP,
+                                        gd_mgmt_v3_brick_op_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_brick_op_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_brick_op (glusterd_conf_t  *conf, glusterd_op_t op,
+                           dict_t *req_dict, char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Perform brick op on local node */
+        ret = gd_mgmt_v3_brick_op_fn (op, req_dict, op_errstr,
+                                     rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Brick ops failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Brick ops failed "
+                                           "on localhost. Please "
+                                           "check log file for details");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending brick op req to other nodes in the cluster */
+        gd_syncargs_init (&args, NULL);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_brick_op_req (op, req_dict, peerinfo, &args,
+                                         MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Brick ops failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent brick op req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_commit_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_commit_rsp       rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        dict_t                     *rsp_dict   = NULL;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        pthread_mutex_lock (&args->lock_dict);
+        {
+                ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
+                                                     rsp_dict);
+        }
+        pthread_mutex_unlock (&args->lock_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                if (!rsp.op_ret)
+                        op_ret = ret;
+                else {
+                        op_ret = rsp.op_ret;
+                        op_errno = rsp.op_errno;
+                }
+        } else {
+                op_ret = rsp.op_ret;
+                op_errno = rsp.op_errno;
+        }
+
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                  GLUSTERD_MGMT_V3_COMMIT,
+                                  peerinfo, rsp.uuid);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_commit_cbk (struct rpc_req *req, struct iovec *iov,
+                       int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_commit_cbk_fn);
+}
+
+int
+gd_mgmt_v3_commit_req (glusterd_op_t op, dict_t *op_ctx,
+                       glusterd_peerinfo_t *peerinfo,
+                       struct syncargs *args, uuid_t my_uuid,
+                       uuid_t recv_uuid)
+{
+        int32_t                  ret  = -1;
+        gd1_mgmt_v3_commit_req   req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_COMMIT,
+                                        gd_mgmt_v3_commit_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_commit_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_commit (glusterd_conf_t  *conf, glusterd_op_t op,
+                         dict_t *op_ctx, dict_t *req_dict,
+                         char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Commit on local node */
+        ret = gd_mgmt_v3_commit_fn (op, req_dict, op_errstr,
+                                   rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Commit failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Commit failed "
+                                           "on localhost. Please "
+                                           "check log file for details.");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx,
+                                             rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending commit req to other nodes in the cluster */
+        gd_syncargs_init (&args, op_ctx);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_commit_req (op, req_dict, peerinfo, &args,
+                                       MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Commit failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent commit req for %s to %d "
+                "peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_post_validate_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_post_val_rsp     rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                  GLUSTERD_MGMT_V3_POST_VALIDATE,
+                                  peerinfo, rsp.uuid);
+        if (rsp.op_errstr)
+                free (rsp.op_errstr);
+
+        free (rsp.dict.dict_val);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_post_validate_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_post_validate_cbk_fn);
+}
+
+int
+gd_mgmt_v3_post_validate_req (glusterd_op_t op, int32_t op_ret, dict_t *op_ctx,
+                              glusterd_peerinfo_t *peerinfo,
+                              struct syncargs *args, uuid_t my_uuid,
+                              uuid_t recv_uuid)
+{
+        int32_t                   ret  = -1;
+        gd1_mgmt_v3_post_val_req  req  = {{0},};
+        glusterd_conf_t          *conf = THIS->private;
+        xlator_t                 *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        req.op_ret = op_ret;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_POST_VALIDATE,
+                                        gd_mgmt_v3_post_validate_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_post_val_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_post_validate (glusterd_conf_t  *conf, glusterd_op_t op,
+                                int32_t op_ret, dict_t *dict, dict_t *req_dict,
+                                char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+        GF_ASSERT (peers);
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Copy the contents of dict like missed snaps info to req_dict */
+        dict_copy (dict, req_dict);
+
+        /* Post Validation on local node */
+        ret = gd_mgmt_v3_post_validate_fn (op, op_ret, req_dict, op_errstr,
+                                           rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Post Validation failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Post-validation failed "
+                                           "on localhost. Please check "
+                                           "log file for details");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending Post Validation req to other nodes in the cluster */
+        gd_syncargs_init (&args, req_dict);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_post_validate_req (op, op_ret, req_dict, peerinfo,
+                                              &args, MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Post Validation failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent post valaidation req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_unlock_rsp      rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_UNLOCK,
+                                   peerinfo, rsp.uuid);
+        free (rsp.dict.dict_val);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                       int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_unlock_cbk_fn);
+}
+
+int
+gd_mgmt_v3_unlock (glusterd_op_t op, dict_t *op_ctx,
+                   glusterd_peerinfo_t *peerinfo,
+                   struct syncargs *args, uuid_t my_uuid,
+                   uuid_t recv_uuid)
+{
+        int32_t                  ret  = -1;
+        gd1_mgmt_v3_unlock_req   req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_UNLOCK,
+                                        gd_mgmt_v3_unlock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_unlock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_release_peer_locks (glusterd_conf_t  *conf, glusterd_op_t op,
+                                     dict_t *dict, int32_t op_ret,
+                                     char **op_errstr, int npeers,
+                                     gf_boolean_t  is_acquired)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        /* If the lock has not been held during this
+         * transaction, do not send unlock requests */
+        if (!is_acquired)
+                goto out;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending mgmt_v3 unlock req to other nodes in the cluster */
+        gd_syncargs_init (&args, NULL);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_unlock (op, dict, peerinfo, &args,
+                                   MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unlock failed on peers");
+
+                if (!op_ret && args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent unlock op req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                      dict_t *dict)
+{
+        int32_t                     ret              = -1;
+        int32_t                     op_ret           = -1;
+        int32_t                     npeers           = 0;
+        dict_t                      *req_dict        = NULL;
+        dict_t                      *tmp_dict        = NULL;
+        glusterd_conf_t             *conf            = NULL;
+        char                        *op_errstr       = NULL;
+        xlator_t                    *this            = NULL;
+        gf_boolean_t                is_acquired      = _gf_false;
+        uuid_t                      *originator_uuid = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        /* Save the MY_UUID as the originator_uuid. This originator_uuid
+         * will be used by is_origin_glusterd() to determine if a node
+         * is the originator node for a command. */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (dict, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
+        /* Marking the operation as complete synctasked */
+        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set synctasked flag.");
+                goto out;
+        }
+
+        /* Use a copy at local unlock as cli response will be sent before
+         * the unlock and the volname in the dict might be removed */
+        tmp_dict = dict_new();
+        if (!tmp_dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create dict");
+                goto out;
+        }
+        dict_copy (dict, tmp_dict);
+
+        /* BUILD PEERS LIST */
+        INIT_LIST_HEAD (&conf->xaction_peers);
+        npeers = gd_build_peers_list  (&conf->peers, &conf->xaction_peers, op);
+
+        /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+        ret = glusterd_mgmt_v3_initiate_lockdown (conf, op, dict, &op_errstr,
+                                                  npeers, &is_acquired);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "mgmt_v3 lockdown failed.");
+                goto out;
+        }
+
+        /* BUILD PAYLOAD */
+        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                goto out;
+        }
+
+        /* PRE-COMMIT VALIDATE PHASE */
+        ret = glusterd_mgmt_v3_pre_validate (conf, op, req_dict,
+                                             &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Pre Validation Failed");
+                goto out;
+        }
+
+        /* COMMIT OP PHASE */
+        ret = glusterd_mgmt_v3_commit (conf, op, dict, req_dict,
+                                       &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Commit Op Failed");
+                goto out;
+        }
+
+        /* POST-COMMIT VALIDATE PHASE */
+        /* As of now, post_validate is not handling any other
+           commands other than snapshot. So as of now, I am
+           sending 0 (op_ret as 0).
+        */
+        ret = glusterd_mgmt_v3_post_validate (conf, op, 0, dict, req_dict,
+                                              &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Post Validation Failed");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        op_ret = ret;
+        /* UNLOCK PHASE FOR PEERS*/
+        (void) glusterd_mgmt_v3_release_peer_locks (conf, op, dict,
+                                                    op_ret, &op_errstr,
+                                                    npeers, is_acquired);
+
+        /* LOCAL VOLUME(S) UNLOCK */
+        if (is_acquired) {
+                /* Trying to release multiple mgmt_v3 locks */
+                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release mgmt_v3 locks on localhost");
+                        op_ret = ret;
+                }
+        }
+
+        /* SEND CLI RESPONSE */
+        glusterd_op_send_cli_response (op, op_ret, 0, req, dict, op_errstr);
+
+        if (req_dict)
+                dict_unref (req_dict);
+
+        if (tmp_dict)
+                dict_unref (tmp_dict);
+
+        if (op_errstr) {
+                GF_FREE (op_errstr);
+                op_errstr = NULL;
+        }
+
+        return 0;
+}
+
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                       dict_t *dict)
+{
+        int32_t                     ret              = -1;
+        int32_t                     op_ret           = -1;
+        int32_t                     npeers           = 0;
+        dict_t                      *req_dict        = NULL;
+        dict_t                      *tmp_dict        = NULL;
+        glusterd_conf_t             *conf            = NULL;
+        char                        *op_errstr       = NULL;
+        xlator_t                    *this            = NULL;
+        gf_boolean_t                is_acquired      = _gf_false;
+        uuid_t                      *originator_uuid = NULL;
+        gf_boolean_t                success          = _gf_false;
+        char                        *tmp_errstr      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        /* Save the MY_UUID as the originator_uuid. This originator_uuid
+         * will be used by is_origin_glusterd() to determine if a node
+         * is the originator node for a command. */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (dict, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
+        /* Marking the operation as complete synctasked */
+        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set synctasked flag.");
+                goto out;
+        }
+
+        /* Use a copy at local unlock as cli response will be sent before
+         * the unlock and the volname in the dict might be removed */
+        tmp_dict = dict_new();
+        if (!tmp_dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create dict");
+                goto out;
+        }
+        dict_copy (dict, tmp_dict);
+
+        /* BUILD PEERS LIST */
+        INIT_LIST_HEAD (&conf->xaction_peers);
+        npeers = gd_build_peers_list  (&conf->peers, &conf->xaction_peers, op);
+
+        /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+        ret = glusterd_mgmt_v3_initiate_lockdown (conf, op, dict, &op_errstr,
+                                                  npeers, &is_acquired);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "mgmt_v3 lockdown failed.");
+                goto out;
+        }
+
+        /* BUILD PAYLOAD */
+        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                goto out;
+        }
+
+        /* PRE-COMMIT VALIDATE PHASE */
+        ret = glusterd_mgmt_v3_pre_validate (conf, op, req_dict,
+                                            &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Pre Validation Failed");
+                goto out;
+        }
+
+        /* BRICK OP PHASE for initiating barrier*/
+        ret = dict_set_int32 (req_dict, "barrier", 1);
+        if (ret)
+                goto out;
+        ret = glusterd_mgmt_v3_brick_op (conf, op, req_dict,
+                                        &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Brick Ops Failed");
+                goto unbarrier;
+        }
+
+        /* COMMIT OP PHASE */
+        /* TODO: As of now, the plan is to do quorum check before sending the
+           commit fop and if the quorum succeeds, then commit is sent to all
+           the other glusterds.
+           snap create functionality now creates the in memory and on disk
+           objects for the snapshot (marking them as incomplete), takes the lvm
+           snapshot and then updates the status of the in memory and on disk
+           snap objects as complete. Suppose one of the glusterds goes down
+           after taking the lvm snapshot, but before updating the snap object,
+           then treat it as a snapshot create failure and trigger cleanup.
+           i.e the number of commit responses received by the originator
+           glusterd shold be the same as the number of peers it has sent the
+           request to (i.e npeers variable). If not, then originator glusterd
+           will initiate cleanup in post-validate fop.
+           Question: What if one of the other glusterds goes down as explained
+           above and along with it the originator glusterd also goes down?
+           Who will initiate the cleanup?
+        */
+        ret = glusterd_mgmt_v3_commit (conf, op, dict, req_dict,
+                                       &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Commit Op Failed");
+                /* If the main op fails, we should save the error string.
+                   Because, op_errstr will be used for unbarrier and
+                   unlock ops also. We might lose the actual error that
+                   caused the failure.
+                */
+                tmp_errstr = op_errstr;
+                op_errstr = NULL;
+                goto unbarrier;
+        }
+
+        success = _gf_true;
+unbarrier:
+        /* BRICK OP PHASE for removing the barrier*/
+        ret = dict_set_int32 (req_dict, "barrier", 0);
+        if (ret)
+                goto out;
+        ret = glusterd_mgmt_v3_brick_op (conf, op, req_dict,
+                                         &op_errstr, npeers);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Brick Ops Failed");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        op_ret = ret;
+
+        if (success == _gf_false)
+                op_ret = -1;
+
+        /* POST-COMMIT VALIDATE PHASE */
+        ret = glusterd_mgmt_v3_post_validate (conf, op, op_ret, dict, req_dict,
+                                              &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Post Validation Failed");
+                op_ret = -1;
+        }
+
+        /* UNLOCK PHASE FOR PEERS*/
+        (void) glusterd_mgmt_v3_release_peer_locks (conf, op, dict,
+                                                    op_ret, &op_errstr,
+                                                    npeers, is_acquired);
+
+        /* If the commit op (snapshot taking) failed, then the error is stored
+           in tmp_errstr and unbarrier is called. Suppose, if unbarrier also
+           fails, then the error happened in unbarrier is logged and freed.
+           The error happened in commit op, which is stored in tmp_errstr
+           is sent to cli.
+        */
+        if (tmp_errstr) {
+                if (op_errstr) {
+                        gf_log (this->name, GF_LOG_ERROR, "unbarrier brick op"
+                                "failed with the error %s", op_errstr);
+                        GF_FREE (op_errstr);
+                        op_errstr = NULL;
+                }
+                op_errstr = tmp_errstr;
+        }
+
+        /* LOCAL VOLUME(S) UNLOCK */
+        if (is_acquired) {
+                /* Trying to release multiple mgmt_v3 locks */
+                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release mgmt_v3 locks on localhost");
+                        op_ret = ret;
+                }
+        }
+
+        /* SEND CLI RESPONSE */
+        glusterd_op_send_cli_response (op, op_ret, 0, req, dict, op_errstr);
+
+        if (req_dict)
+                dict_unref (req_dict);
+
+        if (tmp_dict)
+                dict_unref (tmp_dict);
+
+        if (op_errstr) {
+                GF_FREE (op_errstr);
+                op_errstr = NULL;
+        }
+
+        return 0;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
new file mode 100644
index 000000000..b185a9bec
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
@@ -0,0 +1,45 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_MGMT_H_
+#define _GLUSTERD_MGMT_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+int32_t
+gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
+                           char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
+                       char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
+                     char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
+                            char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                     dict_t *dict);
+
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                      dict_t *dict);
+
+int
+glusterd_snap_pre_validate_use_rsp_dict (dict_t *dst, dict_t *src);
+
+#endif /* _GLUSTERD_MGMT_H_ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
index 32adf4ddd..0d67d1303 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -244,12 +234,15 @@ const char *georep_mnt_desc_template =
                 "xlator-option=\\*-dht.assert-no-child-down=true "
                 "volfile-server=localhost "
                 "client-pid=%d "
-                "volfile-id=%s "
                 "user-map-root=%s "
         ")"
         "SUB+("
                 "log-file="DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"*/* "
                 "log-level=* "
+                "volfile-id=* "
+        ")"
+        "MEET("
+                "%s"
         ")";
 
 const char *hadoop_mnt_desc_template =
@@ -265,18 +258,69 @@ const char *hadoop_mnt_desc_template =
         ")";
 
 int
-make_georep_mountspec (gf_mount_spec_t *mspec, const char *volname,
+make_georep_mountspec (gf_mount_spec_t *mspec, const char *volnames,
                        char *user)
 {
         char *georep_mnt_desc = NULL;
+        char *meetspec        = NULL;
+        char *vols            = NULL;
+        char *vol             = NULL;
+        char *p               = NULL;
+        char *savetok         = NULL;
+        char *fa[3]           = {0,};
+        size_t siz            = 0;
+        int vc                = 0;
+        int i                 = 0;
         int ret               = 0;
 
+        vols = gf_strdup ((char *)volnames);
+        if (!vols)
+                goto out;
+
+        for (vc = 1, p = vols; *p; p++) {
+                if (*p == ',')
+                        vc++;
+        }
+        siz = strlen (volnames) + vc * strlen("volfile-id=");
+        meetspec = GF_CALLOC (1, siz + 1, gf_gld_mt_georep_meet_spec);
+        if (!meetspec)
+                goto out;
+
+        for (p = vols;;) {
+                vol = strtok_r (p, ",", &savetok);
+                if (!vol) {
+                        GF_ASSERT (vc == 0);
+                        break;
+                }
+                p = NULL;
+                strcat (meetspec, "volfile-id=");
+                strcat (meetspec, vol);
+                if (--vc > 0)
+                        strcat (meetspec, " ");
+        }
+
         ret = gf_asprintf (&georep_mnt_desc, georep_mnt_desc_template,
-                           GF_CLIENT_PID_GSYNCD, volname, user);
-        if (ret == -1)
-                return ret;
+                           GF_CLIENT_PID_GSYNCD, user, meetspec);
+        if (ret == -1) {
+                georep_mnt_desc = NULL;
+                goto out;
+        }
+
+        ret = parse_mount_pattern_desc (mspec, georep_mnt_desc);
+
+ out:
+        fa[0] = meetspec;
+        fa[1] = vols;
+        fa[2] = georep_mnt_desc;
 
-        return parse_mount_pattern_desc (mspec, georep_mnt_desc);
+        for (i = 0; i < 3; i++) {
+                if (fa[i] == NULL)
+                        ret = -1;
+                else
+                        GF_FREE (fa[i]);
+        }
+
+        return ret;
 }
 
 int
@@ -425,6 +469,7 @@ evaluate_mount_request (gf_mount_spec_t *mspec, dict_t *argdict)
                         break;
                 case SET_INTERSECT:
                         match = sd.common;
+                        break;
                 default:
                         GF_ASSERT(!"unreached");
                 }
@@ -642,8 +687,7 @@ glusterd_do_mount (char *label, dict_t *argdict, char **path, int *op_errno)
                 *path = cookie;
         }
 
-        if (mtptemp)
-                GF_FREE (mtptemp);
+        GF_FREE (mtptemp);
 
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.h b/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
index 729cf3fbc..426252ebe 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #define MB_HIVE "mb_hive"
 
 typedef enum {
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index d0d280a09..1666f5e4d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -45,7 +35,9 @@
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
+#include "glusterd-hooks.h"
 #include "glusterd-volgen.h"
+#include "glusterd-locks.h"
 #include "syscall.h"
 #include "cli1-xdr.h"
 #include "common-utils.h"
@@ -55,9 +47,186 @@
 #include <signal.h>
 #include <sys/wait.h>
 
+#define ALL_VOLUME_OPTION_CHECK(volname, key, ret, op_errstr, label)           \
+        do {                                                                   \
+                gf_boolean_t    _all = !strcmp ("all", volname);               \
+                gf_boolean_t    _ratio = !strcmp (key,                         \
+                                                  GLUSTERD_QUORUM_RATIO_KEY);  \
+                if (_all && !_ratio) {                                         \
+                        ret = -1;                                              \
+                        *op_errstr = gf_strdup ("Not a valid option for all "  \
+                                                "volumes");                    \
+                        goto label;                                            \
+                } else if (!_all && _ratio) {                                  \
+                        ret = -1;                                              \
+                        *op_errstr = gf_strdup ("Not a valid option for "      \
+                                                "single volume");              \
+                        goto label;                                            \
+                }                                                              \
+         } while (0)
+
 static struct list_head gd_op_sm_queue;
 pthread_mutex_t       gd_op_sm_lock;
 glusterd_op_info_t    opinfo = {{0},};
+uuid_t                global_txn_id = {"\0"}; /* To be used in
+                                               * heterogeneous
+                                               * cluster with no
+                                               * transaction ids */
+
+static dict_t *txn_opinfo;
+
+struct txn_opinfo_object_ {
+        glusterd_op_info_t    opinfo;
+};
+typedef struct txn_opinfo_object_ txn_opinfo_obj;
+
+int32_t
+glusterd_txn_opinfo_dict_init ()
+{
+        int32_t ret = -1;
+
+        txn_opinfo = dict_new ();
+        if (!txn_opinfo) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+glusterd_txn_opinfo_dict_fini ()
+{
+        if (txn_opinfo)
+                dict_destroy (txn_opinfo);
+}
+
+void
+glusterd_txn_opinfo_init (glusterd_op_info_t  *opinfo,
+                          glusterd_op_sm_state_info_t *state,
+                          glusterd_op_t *op,
+                          dict_t *op_ctx,
+                          rpcsvc_request_t *req)
+{
+        GF_ASSERT (opinfo);
+
+        if (state)
+                opinfo->state = *state;
+
+        if (op)
+                opinfo->op = *op;
+
+        opinfo->op_ctx = dict_ref(op_ctx);
+
+        if (req)
+                opinfo->req = req;
+
+        return;
+}
+
+int32_t
+glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+{
+        int32_t          ret         = -1;
+        txn_opinfo_obj  *opinfo_obj  = NULL;
+
+        if (!txn_id || !opinfo) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Empty transaction id or opinfo received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(txn_opinfo, uuid_utoa (*txn_id),
+                           (void **) &opinfo_obj);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get transaction opinfo");
+                goto out;
+        }
+
+        (*opinfo) = opinfo_obj->opinfo;
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+{
+        int32_t          ret        = -1;
+        txn_opinfo_obj  *opinfo_obj = NULL;
+
+        if (!txn_id) {
+                gf_log ("", GF_LOG_ERROR, "Empty transaction id received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(txn_opinfo, uuid_utoa (*txn_id),
+                           (void **) &opinfo_obj);
+        if (ret) {
+                opinfo_obj = GF_CALLOC (1, sizeof(txn_opinfo_obj),
+                                        gf_common_mt_txn_opinfo_obj_t);
+                if (!opinfo_obj) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_bin(txn_opinfo, uuid_utoa (*txn_id), opinfo_obj,
+                                   sizeof(txn_opinfo_obj));
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to set opinfo for transaction ID : %s",
+                                uuid_utoa (*txn_id));
+                        goto out;
+                }
+        }
+
+        opinfo_obj->opinfo = (*opinfo);
+
+        ret = 0;
+out:
+        if (ret)
+                if (opinfo_obj)
+                        GF_FREE (opinfo_obj);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_clear_txn_opinfo (uuid_t *txn_id)
+{
+        int32_t               ret         = -1;
+        glusterd_op_info_t    txn_op_info = {{0},};
+
+        if (!txn_id) {
+                gf_log ("", GF_LOG_ERROR, "Empty transaction id received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Transaction opinfo not found");
+                goto out;
+        }
+
+        dict_unref (txn_op_info.op_ctx);
+
+        dict_del(txn_opinfo, uuid_utoa (*txn_id));
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
 static int glusterfs_port = GLUSTERD_DEFAULT_PORT;
 static char *glusterd_op_sm_state_names[] = {
         "Default",
@@ -96,6 +265,8 @@ static char *glusterd_op_sm_event_names[] = {
         "GD_OP_EVENT_INVALID"
 };
 
+extern struct volopt_map_entry glusterd_volopt_map[];
+
 char*
 glusterd_op_sm_state_name_get (int state)
 {
@@ -136,10 +307,10 @@ glusterd_is_volume_started (glusterd_volinfo_t  *volinfo)
 }
 
 static int
-glusterd_op_sm_inject_all_acc ()
+glusterd_op_sm_inject_all_acc (uuid_t *txn_id)
 {
         int32_t                 ret = -1;
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, txn_id, NULL);
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
@@ -150,6 +321,13 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
 {
         int                     ret = -1;
         gd1_mgmt_brick_op_req   *brick_req = NULL;
+        char                    *volname = NULL;
+        char                    name[1024] = {0,};
+        gf_xl_afr_op_t          heal_op = GF_AFR_OP_INVALID;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (op < GD_OP_MAX);
         GF_ASSERT (op > GD_OP_NONE);
@@ -161,10 +339,8 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
         case GD_OP_STOP_VOLUME:
                 brick_req = GF_CALLOC (1, sizeof (*brick_req),
                                        gf_gld_mt_mop_brick_req_t);
-                if (!brick_req) {
-                        gf_log ("", GF_LOG_ERROR, "Out of Memory");
+                if (!brick_req)
                         goto out;
-                }
                 brick_req->op = GLUSTERD_BRICK_TERMINATE;
                 brick_req->name = "";
         break;
@@ -172,10 +348,8 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
                 brick_req = GF_CALLOC (1, sizeof (*brick_req),
                                        gf_gld_mt_mop_brick_req_t);
 
-                if (!brick_req) {
-                        gf_log ("", GF_LOG_ERROR, "Out of Memory");
+                if (!brick_req)
                         goto out;
-                }
 
                 brick_req->op = GLUSTERD_BRICK_XLATOR_INFO;
                 brick_req->name = brickinfo->path;
@@ -188,61 +362,209 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
                 if (!brick_req)
                         goto out;
 
-                brick_req->op = GLUSTERD_BRICK_XLATOR_HEAL;
+                brick_req->op = GLUSTERD_BRICK_XLATOR_OP;
                 brick_req->name = "";
+                ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
+                if (ret)
+                        goto out;
+                ret = dict_set_int32 (dict, "xl-op", heal_op);
         }
                 break;
         case GD_OP_STATUS_VOLUME:
         {
                 brick_req = GF_CALLOC (1, sizeof (*brick_req),
                                        gf_gld_mt_mop_brick_req_t);
-                if (!brick_req) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "Out of memory");
+                if (!brick_req)
                         goto out;
-                }
                 brick_req->op = GLUSTERD_BRICK_STATUS;
                 brick_req->name = "";
         }
                 break;
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_BRICK_XLATOR_DEFRAG;
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret)
+                        goto out;
+                snprintf (name, 1024, "%s-dht",volname);
+                brick_req->name = gf_strdup (name);
+
+                break;
+        case GD_OP_SNAP:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_VOLUME_BARRIER_OP;
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret)
+                        goto out;
+                snprintf (name, 1024, "%s-server",volname);
+                brick_req->name = gf_strdup (name);
+
+                break;
         default:
                 goto out;
         break;
         }
 
         ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
-                                           (size_t*)&brick_req->input.input_len);
+                                           &brick_req->input.input_len);
+        if (ret)
+                goto out;
+        *req = brick_req;
+        ret = 0;
+
+out:
+        if (ret && brick_req)
+                GF_FREE (brick_req);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_node_op_build_payload (glusterd_op_t op, gd1_mgmt_brick_op_req **req,
+                                dict_t *dict)
+{
+        int                     ret = -1;
+        gd1_mgmt_brick_op_req   *brick_req = NULL;
+
+        GF_ASSERT (op < GD_OP_MAX);
+        GF_ASSERT (op > GD_OP_NONE);
+        GF_ASSERT (req);
+
+        switch (op) {
+        case GD_OP_PROFILE_VOLUME:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_NODE_PROFILE;
+                brick_req->name = "";
+
+                break;
+
+        case GD_OP_STATUS_VOLUME:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_NODE_STATUS;
+                brick_req->name = "";
+
+                break;
+
+        default:
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
+                                           &brick_req->input.input_len);
+
         if (ret)
                 goto out;
+
         *req = brick_req;
         ret = 0;
 
 out:
         if (ret && brick_req)
                 GF_FREE (brick_req);
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_validate_quorum_options (xlator_t *this, char *fullkey, char *value,
+                                  char **op_errstr)
+{
+        int             ret = 0;
+        char            *key = NULL;
+        volume_option_t *opt = NULL;
+
+        if (!glusterd_is_quorum_option (fullkey))
+                goto out;
+        key = strchr (fullkey, '.');
+        key++;
+        opt = xlator_volume_option_get (this, key);
+        ret = xlator_option_validate (this, key, value, opt, op_errstr);
+out:
         return ret;
 }
 
+static int
+glusterd_check_client_op_version_support (char *volname, uint32_t op_version,
+                                          char **op_errstr)
+{
+        int                     ret = 0;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+        rpc_transport_t         *xprt = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        priv = this->private;
+        GF_ASSERT(priv);
+
+        pthread_mutex_lock (&priv->xprt_lock);
+        list_for_each_entry (xprt, &priv->xprt_list, list) {
+                if ((!strcmp(volname, xprt->peerinfo.volname)) &&
+                    ((op_version > xprt->peerinfo.max_op_version) ||
+                     (op_version < xprt->peerinfo.min_op_version))) {
+                        ret = -1;
+                        break;
+                }
+        }
+        pthread_mutex_unlock (&priv->xprt_lock);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "One or more clients "
+                        "don't support the required op-version");
+                ret = gf_asprintf (op_errstr, "One or more connected clients "
+                                   "cannot support the feature being set. "
+                                   "These clients need to be upgraded or "
+                                   "disconnected before running this command"
+                                   " again");
+                return -1;
+        }
+        return 0;
+}
 
 static int
 glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
 {
-        int                                      ret           = 0;
-        char                                    *volname       = NULL;
-        int                                      exists        = 0;
-        char                                    *key               = NULL;
-        char                                    *key_fixed     = NULL;
-        char                                    *value         = NULL;
-        char                                     str[100]      = {0, };
-        int                                      count         = 0;
-        int                                      dict_count    = 0;
-        char                                     errstr[2048]  = {0, };
-        glusterd_volinfo_t                      *volinfo       = NULL;
-        dict_t                                  *val_dict      = NULL;
-        gf_boolean_t                             global_opt    = _gf_false;
-        glusterd_volinfo_t                      *voliter       = NULL;
-        glusterd_conf_t                         *priv          = NULL;
-        xlator_t                                *this          = NULL;
+        int                             ret                     = -1;
+        char                            *volname                = NULL;
+        int                             exists                  = 0;
+        char                            *key                    = NULL;
+        char                            *key_fixed              = NULL;
+        char                            *value                  = NULL;
+        char                            str[100]                = {0, };
+        int                             count                   = 0;
+        int                             dict_count              = 0;
+        char                            errstr[2048]            = {0, };
+        glusterd_volinfo_t              *volinfo                = NULL;
+        dict_t                          *val_dict               = NULL;
+        gf_boolean_t                    global_opt              = _gf_false;
+        glusterd_volinfo_t              *voliter                = NULL;
+        glusterd_conf_t                 *priv                   = NULL;
+        xlator_t                        *this                   = NULL;
+        uint32_t                        new_op_version          = 0;
+        uint32_t                        local_new_op_version    = 0;
+        uint32_t                        key_op_version          = 0;
+        uint32_t                        local_key_op_version    = 0;
+        gf_boolean_t                    origin_glusterd         = _gf_true;
+        gf_boolean_t                    check_op_version        = _gf_true;
+        gf_boolean_t                    all_vol                 = _gf_false;
+        struct volopt_map_entry         *vme                    = NULL;
 
         GF_ASSERT (dict);
         this = THIS;
@@ -254,35 +576,73 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
         if (!val_dict)
                 goto out;
 
-        ret = dict_get_int32 (dict, "count", &dict_count);
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        /* Check if we can support the required op-version
+         * This check is not done on the originator glusterd. The originator
+         * glusterd sets this value.
+         */
+        origin_glusterd = is_origin_glusterd (dict);
+
+        if (!origin_glusterd) {
+                /* Check for v3.3.x origin glusterd */
+                check_op_version = dict_get_str_boolean (dict,
+                                                         "check-op-version",
+                                                         _gf_false);
+
+                if (check_op_version) {
+                        ret = dict_get_uint32 (dict, "new-op-version",
+                                               &new_op_version);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get new_op_version");
+                                goto out;
+                        }
 
+                        if ((new_op_version > GD_OP_VERSION_MAX) ||
+                            (new_op_version < GD_OP_VERSION_MIN)) {
+                                ret = -1;
+                                snprintf (errstr, sizeof (errstr),
+                                          "Required op_version (%d) is not "
+                                          "supported", new_op_version);
+                                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                                goto out;
+                        }
+                }
+        }
+
+        ret = dict_get_int32 (dict, "count", &dict_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Count(dict),not set in Volume-Set");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Count(dict),not set in Volume-Set");
                 goto out;
         }
 
-        if ( dict_count == 0 ) {
-                /*No options would be specified of volume set help */
+        if (dict_count == 0) {
+        /*No options would be specified of volume set help */
                 if (dict_get (dict, "help" ))  {
                         ret = 0;
                         goto out;
                 }
 
                 if (dict_get (dict, "help-xml" )) {
-
 #if (HAVE_LIB_XML)
                         ret = 0;
                         goto out;
 #else
                         ret  = -1;
-                        gf_log ("", GF_LOG_ERROR, "libxml not present in the"
-                                   "system");
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "libxml not present in the system");
                         *op_errstr = gf_strdup ("Error: xml libraries not "
                                                 "present to produce xml-output");
                         goto out;
 #endif
                 }
-                gf_log ("", GF_LOG_ERROR, "No options received ");
+                gf_log (this->name, GF_LOG_ERROR, "No options received ");
                 *op_errstr = gf_strdup ("Options not specified");
                 ret = -1;
                 goto out;
@@ -290,84 +650,171 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
-        exists = glusterd_check_volume_exists (volname);
-        if (!exists) {
-                snprintf (errstr, sizeof (errstr), "Volume %s does not exist",
-                          volname);
-                gf_log ("", GF_LOG_ERROR, "%s", errstr);
-                *op_errstr = gf_strdup (errstr);
-                ret = -1;
-                goto out;
-        }
+        if (strcasecmp (volname, "all") != 0) {
+                exists = glusterd_check_volume_exists (volname);
+                if (!exists) {
+                        snprintf (errstr, sizeof (errstr),
+                                  FMTSTR_CHECK_VOL_EXISTS, volname);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        ret = -1;
+                        goto out;
+                }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
-                goto out;
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                FMTSTR_CHECK_VOL_EXISTS, volname);
+                        goto out;
+                }
+
+                ret = glusterd_validate_volume_id (dict, volinfo);
+                if (ret)
+                        goto out;
+        } else {
+                all_vol = _gf_true;
         }
 
+        local_new_op_version = priv->op_version;
+
         for ( count = 1; ret != 1 ; count++ ) {
                 global_opt = _gf_false;
                 sprintf (str, "key%d", count);
                 ret = dict_get_str (dict, str, &key);
-
-
                 if (ret)
                         break;
 
+                sprintf (str, "value%d", count);
+                ret = dict_get_str (dict, str, &value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid key,value pair in 'volume set'");
+                        ret = -1;
+                        goto out;
+                }
+
+                if (strcmp (key, "config.memory-accounting") == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "enabling memory accounting for volume %s",
+                                volname);
+                        ret = 0;
+                }
+
+                if (strcmp (key, "config.transport") == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "changing transport-type for volume %s",
+                                volname);
+                        ret = 0;
+                        /* if value is none of 'tcp/rdma/tcp,rdma' error out */
+                        if (!((strcasecmp (value, "rdma") == 0) ||
+                              (strcasecmp (value, "tcp") == 0) ||
+                              (strcasecmp (value, "tcp,rdma") == 0) ||
+                              (strcasecmp (value, "rdma,tcp") == 0))) {
+                                ret = snprintf (errstr, sizeof (errstr),
+                                                "transport-type %s does "
+                                                "not exist", value);
+                                /* lets not bother about above return value,
+                                   its a failure anyways */
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                if (is_key_glusterd_hooks_friendly (key))
+                        continue;
+
+                for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
+                        if ((vme->validate_fn) &&
+                            ((!strcmp (key, vme->key)) ||
+                             (!strcmp (key, strchr (vme->key, '.') + 1)))) {
+                                ret = vme->validate_fn (dict, key, value,
+                                                        op_errstr);
+                                if (ret)
+                                        goto out;
+                                break;
+                        }
+                }
+
                 exists = glusterd_check_option_exists (key, &key_fixed);
                 if (exists == -1) {
                         ret = -1;
                         goto out;
                 }
+
                 if (!exists) {
-                        gf_log ("", GF_LOG_ERROR, "Option with name: %s "
-                                "does not exist", key);
-                        ret = snprintf (errstr, 2048,
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Option with name: %s does not exist", key);
+                        ret = snprintf (errstr, sizeof (errstr),
                                        "option : %s does not exist",
                                        key);
                         if (key_fixed)
-                                snprintf (errstr + ret, 2048 - ret,
+                                snprintf (errstr + ret, sizeof (errstr) - ret,
                                           "\nDid you mean %s?", key_fixed);
-                        *op_errstr = gf_strdup (errstr);
-                        ret = -1;
-                        goto out;
-                }
-
-                sprintf (str, "value%d", count);
-                ret = dict_get_str (dict, str, &value);
-
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "invalid key,value pair in 'volume set'");
                         ret = -1;
                         goto out;
                 }
 
                 if (key_fixed)
                         key = key_fixed;
-
-                ret = glusterd_check_globaloption (key);
+                ALL_VOLUME_OPTION_CHECK (volname, key, ret, op_errstr, out);
+                ret = glusterd_validate_quorum_options (this, key, value,
+                                                        op_errstr);
                 if (ret)
+                        goto out;
+
+                local_key_op_version = glusterd_get_op_version_for_key (key);
+                if (local_key_op_version > local_new_op_version)
+                        local_new_op_version = local_key_op_version;
+
+                sprintf (str, "op-version%d", count);
+                if (origin_glusterd) {
+                        ret = dict_set_uint32 (dict, str, local_key_op_version);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set key-op-version in dict");
+                                goto out;
+                        }
+                } else if (check_op_version) {
+                        ret = dict_get_uint32 (dict, str, &key_op_version);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get key-op-version from"
+                                        " dict");
+                                goto out;
+                        }
+                        if (local_key_op_version != key_op_version) {
+                                ret = -1;
+                                snprintf (errstr, sizeof (errstr),
+                                          "option: %s op-version mismatch",
+                                          key);
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "%s, required op-version = %"PRIu32", "
+                                        "available op-version = %"PRIu32,
+                                        errstr, key_op_version,
+                                        local_key_op_version);
+                                goto out;
+                        }
+                }
+
+                if (glusterd_check_globaloption (key))
                         global_opt = _gf_true;
 
                 ret = dict_set_str (val_dict, key, value);
 
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Unable to set the options in 'volume set'");
                         ret = -1;
                         goto out;
                 }
 
                 *op_errstr = NULL;
-                if (!global_opt)
+                if (!global_opt && !all_vol)
                         ret = glusterd_validate_reconfopts (volinfo, val_dict, op_errstr);
-                else {
+                else if (!all_vol) {
                         voliter = NULL;
                         list_for_each_entry (voliter, &priv->volumes, vol_list) {
                                 ret = glusterd_validate_globalopts (voliter, val_dict, op_errstr);
@@ -377,8 +824,9 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
                 }
 
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_DEBUG, "Could not create temp "
-                                "volfile, some option failed: %s", *op_errstr);
+                        gf_log (this->name, GF_LOG_ERROR, "Could not create "
+                                "temp volfile, some option failed: %s",
+                                *op_errstr);
                         goto out;
                 }
                 dict_del (val_dict, key);
@@ -389,6 +837,35 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
                 }
         }
 
+        // Check if all the connected clients support the new op-version
+        ret = glusterd_check_client_op_version_support (volname,
+                                                        local_new_op_version,
+                                                        op_errstr);
+        if (ret)
+                goto out;
+
+        if (origin_glusterd) {
+                ret = dict_set_uint32 (dict, "new-op-version",
+                                       local_new_op_version);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set new-op-version in dict");
+                        goto out;
+                }
+                /* Set this value in dict so other peers know to check for
+                 * op-version. This is a hack for 3.3.x compatibility
+                 *
+                 * TODO: Remove this and the other places this is referred once
+                 * 3.3.x compatibility is not required
+                 */
+                ret = dict_set_uint32 (dict, "check-op-version",
+                                       _gf_true);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set check-op-version in dict");
+                        goto out;
+                }
+        }
 
         ret = 0;
 
@@ -396,17 +873,18 @@ out:
         if (val_dict)
                 dict_unref (val_dict);
 
-        if (key_fixed)
-                GF_FREE (key_fixed);
+        GF_FREE (key_fixed);
+        if (errstr[0] != '\0')
+                *op_errstr = gf_strdup (errstr);
 
         if (ret) {
                 if (!(*op_errstr)) {
                         *op_errstr = gf_strdup ("Error, Validation Failed");
-                        gf_log ("glsuterd", GF_LOG_DEBUG,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "Error, Cannot Validate option :%s",
                                 *op_errstr);
                 } else {
-                        gf_log ("glsuterd", GF_LOG_DEBUG,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "Error, Cannot Validate option");
                 }
         }
@@ -422,28 +900,42 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
         char                                    msg[2048]      = {0};
         char                                    *key = NULL;
         char                                    *key_fixed = NULL;
+        glusterd_volinfo_t                      *volinfo       = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
-        exists = glusterd_check_volume_exists (volname);
+        if (strcasecmp (volname, "all") != 0) {
+                exists = glusterd_check_volume_exists (volname);
+                if (!exists) {
+                        snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS,
+                                  volname);
+                        ret = -1;
+                        goto out;
+                }
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS,
+                                  volname);
+                        goto out;
+                }
 
-        if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not "
-                          "exist", volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
+                ret = glusterd_validate_volume_id (dict, volinfo);
+                if (ret)
+                        goto out;
         }
 
         ret = dict_get_str (dict, "key", &key);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get option key");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get option key");
                 goto out;
         }
         if (strcmp(key, "all")) {
@@ -453,24 +945,30 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
                         goto out;
                 }
                 if (!exists) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "Option %s does not exist", key);
-                        ret = snprintf (msg, 2048,
+                        ret = snprintf (msg, sizeof (msg),
                                         "Option %s does not exist", key);
                         if (key_fixed)
-                                snprintf (msg + ret, 2048 - ret,
+                                snprintf (msg + ret, sizeof (msg) - ret,
                                           "\nDid you mean %s?", key_fixed);
-                        *op_errstr = gf_strdup (msg);
                         ret = -1;
                         goto out;
+                } else if (exists > 0) {
+                        if (key_fixed)
+                                key = key_fixed;
+                        ALL_VOLUME_OPTION_CHECK (volname, key, ret,
+                                                 op_errstr, out);
                 }
         }
 
 out:
-        if (key_fixed)
-                GF_FREE (key_fixed);
+        GF_FREE (key_fixed);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -486,6 +984,7 @@ glusterd_op_stage_sync_volume (dict_t *dict, char **op_errstr)
         gf_boolean_t                            exists = _gf_false;
         glusterd_peerinfo_t                     *peerinfo = NULL;
         char                                    msg[2048] = {0,};
+        glusterd_volinfo_t                      *volinfo  = NULL;
 
         ret = dict_get_str (dict, "hostname", &hostname);
         if (ret) {
@@ -495,8 +994,26 @@ glusterd_op_stage_sync_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        ret = glusterd_is_local_addr (hostname);
-        if (ret) {
+        if (gf_is_local_addr (hostname)) {
+                //volname is not present in case of sync all
+                ret = dict_get_str (dict, "volname", &volname);
+                if (!ret) {
+                        exists = glusterd_check_volume_exists (volname);
+                        if (!exists) {
+                                snprintf (msg, sizeof (msg), "Volume %s "
+                                         "does not exist", volname);
+                                *op_errstr = gf_strdup (msg);
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret)
+                                goto out;
+
+                } else {
+                        ret = 0;
+                }
+         } else {
                 ret = glusterd_friend_find (NULL, hostname, &peerinfo);
                 if (ret) {
                         snprintf (msg, sizeof (msg), "%s, is not a friend",
@@ -512,22 +1029,7 @@ glusterd_op_stage_sync_volume (dict_t *dict, char **op_errstr)
                         ret = -1;
                         goto out;
                 }
-        } else {
 
-                //volname is not present in case of sync all
-                ret = dict_get_str (dict, "volname", &volname);
-                if (!ret) {
-                        exists = glusterd_check_volume_exists (volname);
-                        if (!exists) {
-                                snprintf (msg, sizeof (msg), "Volume %s "
-                                         "does not exist", volname);
-                                *op_errstr = gf_strdup (msg);
-                                ret = -1;
-                                goto out;
-                        }
-                } else {
-                        ret = 0;
-                }
         }
 
 out:
@@ -547,8 +1049,10 @@ glusterd_op_stage_status_volume (dict_t *dict, char **op_errstr)
         xlator_t              *this           = NULL;
         glusterd_conf_t       *priv           = NULL;
         glusterd_brickinfo_t  *brickinfo      = NULL;
-        glusterd_brickinfo_t  *tmpbrickinfo   = NULL;
         glusterd_volinfo_t    *volinfo        = NULL;
+        dict_t                *vol_opts       = NULL;
+        gf_boolean_t           nfs_disabled   = _gf_false;
+        gf_boolean_t           shd_enabled    = _gf_true;
 
         GF_ASSERT (dict);
         this = THIS;
@@ -572,32 +1076,68 @@ glusterd_op_stage_status_volume (dict_t *dict, char **op_errstr)
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                snprintf (msg, sizeof(msg), "Volume %s does not exist", volname);
+                snprintf (msg, sizeof(msg), "Volume %s does not exist",
+                          volname);
                 gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
                 ret = -1;
                 goto out;
         }
 
-        if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
-                ret = dict_get_str (dict, "brick", &brick);
-                if (ret)
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_is_volume_started (volinfo);
+        if (!ret) {
+                snprintf (msg, sizeof (msg), "Volume %s is not started",
+                          volname);
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                ret = -1;
+                goto out;
+        }
+
+        vol_opts = volinfo->dict;
+
+        if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+                nfs_disabled = dict_get_str_boolean (vol_opts, "nfs.disable",
+                                                     _gf_false);
+                if (nfs_disabled) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg),
+                                  "NFS server is disabled for volume %s",
+                                  volname);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                        goto out;
+                }
+        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+                if (!glusterd_is_volume_replicate (volinfo)) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg),
+                                  "Volume %s is not of type replicate",
+                                  volname);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
                         goto out;
+                }
 
-                ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "%s is not a brick",
-                                  brick);
+                shd_enabled = dict_get_str_boolean (vol_opts,
+                                                    "cluster.self-heal-daemon",
+                                                    _gf_true);
+                if (!shd_enabled) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg),
+                                  "Self-heal Daemon is disabled for volume %s",
+                                  volname);
                         gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
                         goto out;
                 }
 
-                ret = glusterd_volume_brickinfo_get (NULL,
-                                                     brickinfo->hostname,
-                                                     brickinfo->path,
-                                                     volinfo,
-                                                     &tmpbrickinfo,
-                                                     GF_PATH_COMPLETE);
+        } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+                ret = dict_get_str (dict, "brick", &brick);
+                if (ret)
+                        goto out;
 
+                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                              &brickinfo);
                 if (ret) {
                         snprintf (msg, sizeof(msg), "No brick %s in"
                                   " volume %s", brick, volname);
@@ -669,6 +1209,10 @@ glusterd_op_stage_stats_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         ret = dict_get_int32 (dict, "op", &stats_op);
         if (ret) {
                 snprintf (msg, sizeof (msg), "Volume profile op get failed");
@@ -715,60 +1259,82 @@ out:
 }
 
 
-void
+static int
 _delete_reconfig_opt (dict_t *this, char *key, data_t *value, void *data)
 {
-        int             exists = 0;
-        int32_t         is_force = 0;
+        int32_t        *is_force = 0;
 
         GF_ASSERT (data);
-        is_force = *((int32_t*)data);
-        exists = glusterd_check_option_exists(key, NULL);
-
-        if (exists != 1)
-                goto out;
-
-        if ((!is_force) &&
-            (_gf_true == glusterd_check_voloption_flags (key,
-                                                         OPT_FLAG_FORCE)))
-                goto out;
+        is_force = (int32_t*)data;
+
+        if (*is_force != 1) {
+                if (_gf_true == glusterd_check_voloption_flags (key,
+                                                         OPT_FLAG_FORCE)) {
+                /* indicate to caller that we don't set the option
+                 * due to being protected
+                 */
+                        *is_force = *is_force | GD_OP_PROTECTED;
+                        goto out;
+                } else {
+                        *is_force = *is_force | GD_OP_UNPROTECTED;
+                }
+        }
 
         gf_log ("", GF_LOG_DEBUG, "deleting dict with key=%s,value=%s",
                 key, value->data);
         dict_del (this, key);
 out:
-        return;
+        return 0;
 }
 
-int
+static int
+_delete_reconfig_global_opt (dict_t *this, char *key, data_t *value, void *data)
+{
+        int32_t        *is_force = 0;
+
+        GF_ASSERT (data);
+        is_force = (int32_t*)data;
+
+        if (strcmp (GLUSTERD_GLOBAL_OPT_VERSION, key) == 0)
+                goto out;
+
+        _delete_reconfig_opt (this, key, value, data);
+out:
+        return 0;
+}
+
+static int
 glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key,
-                        int32_t is_force)
+                        int32_t *is_force)
 {
         int                      ret = 0;
         data_t                  *value = NULL;
+        char                    *key_fixed = NULL;
+        xlator_t                *this = NULL;
 
-        gf_log ("", GF_LOG_DEBUG, "Received volume set reset command");
-
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (volinfo->dict);
         GF_ASSERT (key);
 
         if (!strncmp(key, "all", 3))
-                dict_foreach (volinfo->dict, _delete_reconfig_opt, &is_force);
+                dict_foreach (volinfo->dict, _delete_reconfig_opt, is_force);
         else {
                 value = dict_get (volinfo->dict, key);
                 if (!value) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "Could not get value");
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "no value set for option %s", key);
                         goto out;
                 }
-                _delete_reconfig_opt (volinfo->dict, key, value, &is_force);
+                _delete_reconfig_opt (volinfo->dict, key, value, is_force);
         }
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        gd_update_volume_op_versions (volinfo);
 
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to create volfile for"
-                        " 'volume set'");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create volfile for"
+                        " 'volume reset'");
                 ret = -1;
                 goto out;
         }
@@ -786,23 +1352,120 @@ glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key,
         ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        GF_FREE (key_fixed);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+static int
+glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)
+{
+        char            *key            = NULL;
+        char            *key_fixed      = NULL;
+        int             ret             = -1;
+        int32_t         is_force        = 0;
+        glusterd_conf_t *conf           = NULL;
+        dict_t          *dup_opt        = NULL;
+        gf_boolean_t    all             = _gf_false;
+        char            *next_version   = NULL;
+        gf_boolean_t    quorum_action   = _gf_false;
+
+        conf = this->private;
+        ret = dict_get_str (dict, "key", &key);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get key");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "force", &is_force);
+        if (ret)
+                is_force = 0;
+
+        if (strcmp (key, "all")) {
+                ret = glusterd_check_option_exists (key, &key_fixed);
+                if (ret <= 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Option %s does not "
+                                "exist", key);
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                all = _gf_true;
+        }
+
+        if (key_fixed)
+                key = key_fixed;
+
+        ret = -1;
+        dup_opt = dict_new ();
+        if (!dup_opt)
+                goto out;
+        if (!all) {
+                dict_copy (conf->opts, dup_opt);
+                dict_del (dup_opt, key);
+        }
+        ret = glusterd_get_next_global_opt_version_str (conf->opts,
+                                                        &next_version);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_options (this, dup_opt);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_quorum_changed (conf->opts, key, NULL))
+                quorum_action = _gf_true;
+
+        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                               next_version);
+        if (ret)
+                goto out;
+        else
+                next_version = NULL;
+
+        if (!all) {
+                dict_del (conf->opts, key);
+        } else {
+                dict_foreach (conf->opts, _delete_reconfig_global_opt,
+                              &is_force);
+        }
+out:
+        GF_FREE (key_fixed);
+        if (dup_opt)
+                dict_unref (dup_opt);
+
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        GF_FREE (next_version);
+        return ret;
+}
 
 static int
-glusterd_op_reset_volume (dict_t *dict)
+glusterd_op_reset_volume (dict_t *dict, char **op_rspstr)
 {
         glusterd_volinfo_t      *volinfo    = NULL;
         int                     ret         = -1;
         char                    *volname    = NULL;
         char                    *key        = NULL;
+        char                    *key_fixed  = NULL;
         int32_t                 is_force    = 0;
+        gf_boolean_t            quorum_action = _gf_false;
+        xlator_t                *this         = NULL;
 
+        this = THIS;
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name " );
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name" );
+                goto out;
+        }
+
+        if (strcasecmp (volname, "all") == 0) {
+                ret = glusterd_op_reset_all_volume_options (this, dict);
                 goto out;
         }
 
@@ -812,32 +1475,64 @@ glusterd_op_reset_volume (dict_t *dict)
 
         ret = dict_get_str (dict, "key", &key);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get option key");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get option key");
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
                 goto out;
         }
 
-        ret = glusterd_options_reset (volinfo, key, is_force);
+        if (strcmp (key, "all") &&
+            glusterd_check_option_exists (key, &key_fixed) != 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "volinfo dict inconsistency: option %s not found",
+                        key);
+                ret = -1;
+                goto out;
+        }
+        if (key_fixed)
+                key = key_fixed;
+
+        if (glusterd_is_quorum_changed (volinfo->dict, key, NULL))
+                quorum_action = _gf_true;
+
+        ret = glusterd_options_reset (volinfo, key, &is_force);
+        if (ret == -1) {
+                gf_asprintf(op_rspstr, "Volume reset : failed");
+        } else if (is_force & GD_OP_PROTECTED) {
+                if (is_force & GD_OP_UNPROTECTED) {
+                        gf_asprintf (op_rspstr, "All unprotected fields were"
+                                     " reset. To reset the protected fields,"
+                                     " use 'force'.");
+                } else {
+                        ret = -1;
+                        gf_asprintf (op_rspstr, "'%s' is protected. To reset"
+                                     " use 'force'.", key);
+                }
+        }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "'volume reset' returning %d", ret);
-        return ret;
+        GF_FREE (key_fixed);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
 
+        gf_log (this->name, GF_LOG_DEBUG, "'volume reset' returning %d", ret);
+        return ret;
 }
 
-
 int
 glusterd_stop_bricks (glusterd_volinfo_t *volinfo)
 {
         glusterd_brickinfo_t                    *brickinfo = NULL;
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (glusterd_brick_stop (volinfo, brickinfo))
+            /*TODO: Need to change @del_brick in brick_stop to _gf_true
+             * once we enable synctask in peer rpc prog */
+                if (glusterd_brick_stop (volinfo, brickinfo, _gf_false))
                         return -1;
         }
 
@@ -847,32 +1542,109 @@ glusterd_stop_bricks (glusterd_volinfo_t *volinfo)
 int
 glusterd_start_bricks (glusterd_volinfo_t *volinfo)
 {
-        glusterd_brickinfo_t                    *brickinfo = NULL;
+        int                      ret            = -1;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+
+        GF_ASSERT (volinfo);
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (glusterd_brick_start (volinfo, brickinfo))
-                        return -1;
+                ret = glusterd_brick_start (volinfo, brickinfo, _gf_false);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "Failed to start %s:%s for %s",
+                                brickinfo->hostname, brickinfo->path,
+                                volinfo->volname);
+                        goto out;
+                }
         }
 
-        return 0;
+        ret = 0;
+out:
+        return ret;
 }
 
 static int
-glusterd_volset_help (dict_t *dict)
+glusterd_op_set_all_volume_options (xlator_t *this, dict_t *dict)
 {
-        int                     ret = -1;
-        gf_boolean_t            xml_out = _gf_false;
+        char            *key            = NULL;
+        char            *key_fixed      = NULL;
+        char            *value          = NULL;
+        char            *dup_value      = NULL;
+        int             ret             = -1;
+        glusterd_conf_t *conf           = NULL;
+        dict_t          *dup_opt        = NULL;
+        char            *next_version   = NULL;
+        gf_boolean_t    quorum_action   = _gf_false;
+
+        conf = this->private;
+        ret = dict_get_str (dict, "key1", &key);
+        if (ret)
+                goto out;
 
-        if (dict_get (dict, "help" ))
-                xml_out = _gf_false;
-        else if (dict_get (dict, "help-xml" ))
-                xml_out = _gf_true;
-        else
+        ret = dict_get_str (dict, "value1", &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "invalid key,value pair in 'volume set'");
                 goto out;
+        }
+        ret = glusterd_check_option_exists (key, &key_fixed);
+        if (ret <= 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid key %s", key);
+                ret = -1;
+                goto out;
+        }
 
-        ret = glusterd_get_volopt_content (xml_out);
- out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        if (key_fixed)
+                key = key_fixed;
+
+        ret = -1;
+        dup_opt = dict_new ();
+        if (!dup_opt)
+                goto out;
+        dict_copy (conf->opts, dup_opt);
+        ret = dict_set_str (dup_opt, key, value);
+        if (ret)
+                goto out;
+
+        ret = glusterd_get_next_global_opt_version_str (conf->opts,
+                                                        &next_version);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
+        if (ret)
+                goto out;
+
+        dup_value = gf_strdup (value);
+        if (!dup_value)
+                goto out;
+
+        ret = glusterd_store_options (this, dup_opt);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_quorum_changed (conf->opts, key, value))
+                quorum_action = _gf_true;
+
+        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                               next_version);
+        if (ret)
+                goto out;
+        else
+                next_version = NULL;
+
+        ret = dict_set_dynstr (conf->opts, key, dup_value);
+        if (ret)
+                goto out;
+out:
+        GF_FREE (key_fixed);
+        if (dup_opt)
+                dict_unref (dup_opt);
+
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        GF_FREE (next_version);
         return ret;
 }
 
@@ -889,9 +1661,14 @@ glusterd_op_set_volume (dict_t *dict)
         char                                    *key_fixed = NULL;
         char                                    *value = NULL;
         char                                     str[50] = {0, };
+        char                                    *op_errstr = NULL;
         gf_boolean_t                             global_opt    = _gf_false;
+        gf_boolean_t                             global_opts_set = _gf_false;
         glusterd_volinfo_t                      *voliter = NULL;
         int32_t                                  dict_count = 0;
+        gf_boolean_t                             check_op_version = _gf_false;
+        uint32_t                                 new_op_version = 0;
+        gf_boolean_t                            quorum_action  = _gf_false;
 
         this = THIS;
         GF_ASSERT (this);
@@ -901,68 +1678,111 @@ glusterd_op_set_volume (dict_t *dict)
 
         ret = dict_get_int32 (dict, "count", &dict_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Count(dict),not set in Volume-Set");
+                gf_log (this->name, GF_LOG_ERROR, "Count(dict),not set in Volume-Set");
                 goto out;
         }
 
-        if ( dict_count == 0 ) {
-                ret = glusterd_volset_help (dict);
-                if (ret)
-                        gf_log ("glusterd", GF_LOG_ERROR, "Volume set help"
-                                                        "internal error");
+        if (dict_count == 0) {
+                ret = glusterd_volset_help (NULL, &op_errstr);
+                if (ret) {
+                        op_errstr = (op_errstr)? op_errstr:
+                                     "Volume set help internal error";
+                        gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+                }
                 goto out;
-        }
+         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        if (strcasecmp (volname, "all") == 0) {
+                ret = glusterd_op_set_all_volume_options (this, dict);
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
                 goto out;
         }
 
-        for ( count = 1; ret != -1 ; count++ ) {
+        // TODO: Remove this once v3.3 compatability is not required
+        check_op_version = dict_get_str_boolean (dict, "check-op-version",
+                                                 _gf_false);
+
+        if (check_op_version) {
+                ret = dict_get_uint32 (dict, "new-op-version", &new_op_version);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get new op-version from dict");
+                        goto out;
+                }
+        }
+
+        for (count = 1; ret != -1 ; count++) {
 
-                global_opt = _gf_false;
                 sprintf (str, "key%d", count);
                 ret = dict_get_str (dict, str, &key);
+                if (ret)
+                        break;
 
+                sprintf (str, "value%d", count);
+                ret = dict_get_str (dict, str, &value);
                 if (ret) {
-                        break;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid key,value pair in 'volume set'");
+                        ret = -1;
+                        goto out;
                 }
 
-                if (!ret) {
+                if (strcmp (key, "config.memory-accounting") == 0) {
+                        ret = gf_string2boolean (value,
+                                                 &volinfo->memory_accounting);
+                }
+
+                if (strcmp (key, "config.transport") == 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "changing transport-type for volume %s to %s",
+                                volname, value);
+                        ret = 0;
+                        if (strcasecmp (value, "rdma") == 0) {
+                                volinfo->transport_type = GF_TRANSPORT_RDMA;
+                        } else if (strcasecmp (value, "tcp") == 0) {
+                                volinfo->transport_type = GF_TRANSPORT_TCP;
+                        } else if ((strcasecmp (value, "tcp,rdma") == 0) ||
+                                   (strcasecmp (value, "rdma,tcp") == 0)) {
+                                volinfo->transport_type =
+                                        GF_TRANSPORT_BOTH_TCP_RDMA;
+                        } else {
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                if (!is_key_glusterd_hooks_friendly (key)) {
                         ret = glusterd_check_option_exists (key, &key_fixed);
                         GF_ASSERT (ret);
-                        if (ret == -1) {
+                        if (ret <= 0) {
                                 key_fixed = NULL;
                                 goto out;
                         }
-                        ret = 0;
                 }
 
-                ret = glusterd_check_globaloption (key);
-                if (ret)
+                global_opt = _gf_false;
+                if (glusterd_check_globaloption (key)) {
                         global_opt = _gf_true;
-
-                sprintf (str, "value%d", count);
-                ret = dict_get_str (dict, str, &value);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "invalid key,value pair in 'volume set'");
-                        ret = -1;
-                        goto out;
+                        global_opts_set = _gf_true;
                 }
 
                 if (!global_opt)
                         value = gf_strdup (value);
 
                 if (!value) {
-                        gf_log ("", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Unable to set the options in 'volume set'");
                         ret = -1;
                         goto out;
@@ -971,6 +1791,9 @@ glusterd_op_set_volume (dict_t *dict)
                 if (key_fixed)
                         key = key_fixed;
 
+                if (glusterd_is_quorum_changed (volinfo->dict, key, value))
+                        quorum_action = _gf_true;
+
                 if (global_opt) {
                        list_for_each_entry (voliter, &priv->volumes, vol_list) {
                                value = gf_strdup (value);
@@ -978,8 +1801,7 @@ glusterd_op_set_volume (dict_t *dict)
                                if (ret)
                                        goto out;
                        }
-                }
-                else {
+                } else {
                         ret = dict_set_dynstr (volinfo->dict, key, value);
                         if (ret)
                                 goto out;
@@ -987,21 +1809,35 @@ glusterd_op_set_volume (dict_t *dict)
 
                 if (key_fixed) {
                         GF_FREE (key_fixed);
-
                         key_fixed = NULL;
                 }
         }
 
-        if ( count == 1 ) {
-                gf_log ("", GF_LOG_ERROR, "No options received ");
+        if (count == 1) {
+                gf_log (this->name, GF_LOG_ERROR, "No options received ");
                 ret = -1;
                 goto out;
         }
 
-        if (!global_opt) {
+        /* Update the cluster op-version before regenerating volfiles so that
+         * correct volfiles are generated
+         */
+        if (new_op_version > priv->op_version) {
+                priv->op_version = new_op_version;
+                ret = glusterd_store_global_info (this);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store op-version");
+                        goto out;
+                }
+        }
+
+        if (!global_opts_set) {
+                gd_update_volume_op_versions (volinfo);
                 ret = glusterd_create_volfiles_and_notify_services (volinfo);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to create volfile for"
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to create volfile for"
                                 " 'volume set'");
                         ret = -1;
                         goto out;
@@ -1014,19 +1850,20 @@ glusterd_op_set_volume (dict_t *dict)
                 if (GLUSTERD_STATUS_STARTED == volinfo->status) {
                         ret = glusterd_nodesvcs_handle_reconfigure (volinfo);
                         if (ret) {
-                                gf_log ("", GF_LOG_WARNING,
+                                gf_log (this->name, GF_LOG_WARNING,
                                          "Unable to restart NFS-Server");
                                 goto out;
                         }
                 }
 
-        }
-        else {
+        } else {
                 list_for_each_entry (voliter, &priv->volumes, vol_list) {
                         volinfo = voliter;
+                        gd_update_volume_op_versions (volinfo);
                         ret = glusterd_create_volfiles_and_notify_services (volinfo);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to create volfile for"
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to create volfile for"
                                         " 'volume set'");
                                 ret = -1;
                                 goto out;
@@ -1040,7 +1877,7 @@ glusterd_op_set_volume (dict_t *dict)
                         if (GLUSTERD_STATUS_STARTED == volinfo->status) {
                                 ret = glusterd_nodesvcs_handle_reconfigure (volinfo);
                                 if (ret) {
-                                        gf_log ("", GF_LOG_WARNING,
+                                        gf_log (this->name, GF_LOG_WARNING,
                                                 "Unable to restart NFS-Server");
                                         goto out;
                                 }
@@ -1048,11 +1885,11 @@ glusterd_op_set_volume (dict_t *dict)
                 }
         }
 
-        ret = 0;
  out:
-        if (key_fixed)
-                GF_FREE (key_fixed);
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        GF_FREE (key_fixed);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
         return ret;
 }
 
@@ -1084,7 +1921,7 @@ glusterd_op_sync_volume (dict_t *dict, char **op_errstr,
                 goto out;
         }
 
-        if (glusterd_is_local_addr (hostname)) {
+        if (!gf_is_local_addr (hostname)) {
                 ret = 0;
                 goto out;
         }
@@ -1255,12 +2092,249 @@ out:
 }
 
 static int
+_add_brick_name_to_dict (dict_t *dict, char *key, glusterd_brickinfo_t *brick)
+{
+        int     ret = -1;
+        char    tmp[1024] = {0,};
+        char    *brickname = NULL;
+        xlator_t *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key);
+        GF_ASSERT (brick);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        snprintf (tmp, sizeof (tmp), "%s:%s", brick->hostname, brick->path);
+        brickname = gf_strdup (tmp);
+        if (!brickname) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to dup brick name");
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dict, key, brickname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to add brick name to dict");
+                goto out;
+        }
+        brickname = NULL;
+out:
+        if (brickname)
+                GF_FREE (brickname);
+        return ret;
+}
+
+static int
+_add_remove_bricks_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo,
+                            char *prefix)
+{
+        int             ret = -1;
+        int             count = 0;
+        int             i = 0;
+        char            brick_key[1024] = {0,};
+        char            dict_key[1024] ={0,};
+        char            *brick = NULL;
+        xlator_t        *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+        GF_ASSERT (prefix);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_int32 (volinfo->rebal.dict, "count", &count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get brick count");
+                goto out;
+        }
+
+        snprintf (dict_key, sizeof (dict_key), "%s.count", prefix);
+        ret = dict_set_int32 (dict, dict_key, count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set brick count in dict");
+                goto out;
+        }
+
+        for (i = 1; i <= count; i++) {
+                memset (brick_key, 0, sizeof (brick_key));
+                snprintf (brick_key, sizeof (brick_key), "brick%d", i);
+
+                ret = dict_get_str (volinfo->rebal.dict, brick_key, &brick);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get %s", brick_key);
+                        goto out;
+                }
+
+                memset (dict_key, 0, sizeof (dict_key));
+                snprintf (dict_key, sizeof (dict_key), "%s.%s", prefix,
+                          brick_key);
+                ret = dict_set_str (dict, dict_key, brick);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add brick to dict");
+                        goto out;
+                }
+                brick = NULL;
+        }
+
+out:
+        return ret;
+}
+
+/* This adds the respective task-id and all available parameters of a task into
+ * a dictionary
+ */
+static int
+_add_task_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
+{
+
+        int             ret = -1;
+        char            key[128] = {0,};
+        char            *uuid_str = NULL;
+        int             status = 0;
+        xlator_t        *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        switch (op) {
+        case GD_OP_REMOVE_BRICK:
+                snprintf (key, sizeof (key), "task%d", index);
+                ret = _add_remove_bricks_to_dict (dict, volinfo, key);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add remove bricks to dict");
+                        goto out;
+                }
+        case GD_OP_REBALANCE:
+                uuid_str = gf_strdup (uuid_utoa (volinfo->rebal.rebalance_id));
+                status = volinfo->rebal.defrag_status;
+                break;
+
+        case GD_OP_REPLACE_BRICK:
+                snprintf (key, sizeof (key), "task%d.src-brick", index);
+                ret = _add_brick_name_to_dict (dict, key,
+                                               volinfo->rep_brick.src_brick);
+                if (ret)
+                        goto out;
+                memset (key, 0, sizeof (key));
+
+                snprintf (key, sizeof (key), "task%d.dst-brick", index);
+                ret = _add_brick_name_to_dict (dict, key,
+                                               volinfo->rep_brick.dst_brick);
+                if (ret)
+                        goto out;
+                memset (key, 0, sizeof (key));
+
+                uuid_str = gf_strdup (uuid_utoa (volinfo->rep_brick.rb_id));
+                status = volinfo->rep_brick.rb_status;
+                break;
+
+        default:
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "%s operation doesn't have a"
+                        " task_id", gd_op_list[op]);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "task%d.type", index);
+        ret = dict_set_str (dict, key, (char *)gd_op_list[op]);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting task type in dict");
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "task%d.id", index);
+
+        if (!uuid_str)
+                goto out;
+        ret = dict_set_dynstr (dict, key, uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting task id in dict");
+                goto out;
+        }
+        uuid_str = NULL;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "task%d.status", index);
+        ret = dict_set_int32 (dict, key, status);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting task status in dict");
+                goto out;
+        }
+
+out:
+        if (uuid_str)
+                GF_FREE (uuid_str);
+        return ret;
+}
+
+static int
+glusterd_aggregate_task_status (dict_t *rsp_dict, glusterd_volinfo_t *volinfo)
+{
+        int        ret   = -1;
+        int        tasks = 0;
+        xlator_t  *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+                ret = _add_task_to_dict (rsp_dict, volinfo, volinfo->rebal.op,
+                                         tasks);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add task details to dict");
+                        goto out;
+                }
+                tasks++;
+        }
+
+        if (!uuid_is_null (volinfo->rep_brick.rb_id)) {
+                ret = _add_task_to_dict (rsp_dict, volinfo, GD_OP_REPLACE_BRICK,
+                                         tasks);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add task details to dict");
+                        goto out;
+                }
+                tasks++;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "tasks", tasks);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting tasks count in dict");
+                goto out;
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static int
 glusterd_op_status_volume (dict_t *dict, char **op_errstr,
                            dict_t *rsp_dict)
 {
         int                     ret             = -1;
-        int                     brick_count     = 0;
+        int                     node_count      = 0;
         int                     brick_index     = -1;
+        int                     other_count     = 0;
+        int                     other_index     = 0;
         uint32_t                cmd             = 0;
         char                   *volname         = NULL;
         char                   *brick           = NULL;
@@ -1268,6 +2342,10 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
         glusterd_volinfo_t     *volinfo         = NULL;
         glusterd_brickinfo_t   *brickinfo       = NULL;
         glusterd_conf_t        *priv            = NULL;
+        dict_t                 *vol_opts        = NULL;
+        gf_boolean_t            nfs_disabled    = _gf_false;
+        gf_boolean_t            shd_enabled     = _gf_true;
+        gf_boolean_t            origin_glusterd = _gf_false;
 
         this = THIS;
         GF_ASSERT (this);
@@ -1277,23 +2355,21 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
 
         GF_ASSERT (dict);
 
+        origin_glusterd = is_origin_glusterd (dict);
+
         ret = dict_get_uint32 (dict, "cmd", &cmd);
         if (ret)
                 goto out;
 
-        if (!rsp_dict) {
-                //this should happen only on source
+        if (origin_glusterd) {
                 ret = 0;
-                rsp_dict = glusterd_op_get_ctx ();
-
                 if ((cmd & GF_CLI_STATUS_ALL)) {
                         ret = glusterd_get_all_volnames (rsp_dict);
                         if (ret)
-                                gf_log (THIS->name, GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "failed to get all volume "
                                         "names for status");
                 }
-
         }
 
         ret = dict_set_uint32 (rsp_dict, "cmd", cmd);
@@ -1309,37 +2385,57 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Volume with name: %s "
+                gf_log (this->name, GF_LOG_ERROR, "Volume with name: %s "
                         "does not exist", volname);
                 goto out;
         }
+        vol_opts = volinfo->dict;
 
-        if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+        if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+                ret = glusterd_add_node_to_dict ("nfs", rsp_dict, 0, vol_opts);
+                if (ret)
+                        goto out;
+                other_count++;
+                node_count++;
+
+        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+                ret = glusterd_add_node_to_dict ("glustershd", rsp_dict, 0,
+                                                 vol_opts);
+                if (ret)
+                        goto out;
+                other_count++;
+                node_count++;
+
+        } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
                 ret = dict_get_str (dict, "brick", &brick);
                 if (ret)
                         goto out;
 
                 ret = glusterd_volume_brickinfo_get_by_brick (brick,
                                                               volinfo,
-                                                              &brickinfo,
-                                                              GF_PATH_COMPLETE);
+                                                              &brickinfo);
                 if (ret)
                         goto out;
 
-                if (uuid_compare (brickinfo->uuid, priv->uuid))
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
                         goto out;
 
-                glusterd_add_brick_to_dict (volinfo, brickinfo, rsp_dict, 0);
+                glusterd_add_brick_to_dict (volinfo, brickinfo, rsp_dict,
+                                            ++brick_index);
                 if (cmd & GF_CLI_STATUS_DETAIL)
                         glusterd_add_brick_detail_to_dict (volinfo, brickinfo,
-                                                           rsp_dict, 0);
-                ret = dict_set_int32 (rsp_dict, "count", 1);
+                                                           rsp_dict,
+                                                           brick_index);
+                node_count++;
 
+        } else if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
+                ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
                 goto out;
+
         } else {
                 list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                         brick_index++;
-                        if (uuid_compare (brickinfo->uuid, priv->uuid))
+                        if (uuid_compare (brickinfo->uuid, MY_UUID))
                                 continue;
 
                         glusterd_add_brick_to_dict (volinfo, brickinfo,
@@ -1351,14 +2447,79 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
                                                                    rsp_dict,
                                                                    brick_index);
                         }
-                        brick_count++;
+                        node_count++;
                 }
+
+                if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
+                        other_index = brick_index + 1;
+
+                        nfs_disabled = dict_get_str_boolean (vol_opts,
+                                                             "nfs.disable",
+                                                             _gf_false);
+                        if (!nfs_disabled) {
+                                ret = glusterd_add_node_to_dict ("nfs",
+                                                                 rsp_dict,
+                                                                 other_index,
+                                                                 vol_opts);
+                                if (ret)
+                                        goto out;
+                                other_index++;
+                                other_count++;
+                                node_count++;
+                        }
+
+                        shd_enabled = dict_get_str_boolean
+                                        (vol_opts, "cluster.self-heal-daemon",
+                                         _gf_true);
+                        if (glusterd_is_volume_replicate (volinfo)
+                            && shd_enabled) {
+                                ret = glusterd_add_node_to_dict ("glustershd",
+                                                                 rsp_dict,
+                                                                 other_index,
+                                                                 vol_opts);
+                                if (ret)
+                                        goto out;
+                                other_count++;
+                                node_count++;
+                        }
+                }
+        }
+
+        ret = dict_set_int32 (rsp_dict, "brick-index-max", brick_index);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting brick-index-max to dict");
+                goto out;
+        }
+        ret = dict_set_int32 (rsp_dict, "other-count", other_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting other-count to dict");
+                goto out;
+        }
+        ret = dict_set_int32 (rsp_dict, "count", node_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting node count to dict");
+                goto out;
         }
 
-        ret = dict_set_int32 (rsp_dict, "count", brick_count);
+        /* Active tasks */
+        /* Tasks are added only for normal volume status request for either a
+         * single volume or all volumes, and only by the origin glusterd
+         */
+        if (((cmd & GF_CLI_STATUS_MASK) != GF_CLI_STATUS_NONE) ||
+            !(cmd & (GF_CLI_STATUS_VOL | GF_CLI_STATUS_ALL)) ||
+            !origin_glusterd)
+                goto out;
+
+        ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
+        if (ret)
+                goto out;
+        ret = 0;
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1368,7 +2529,7 @@ glusterd_op_ac_none (glusterd_op_sm_event_t *event, void *ctx)
 {
         int ret = 0;
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
@@ -1382,6 +2543,7 @@ glusterd_op_ac_send_lock (glusterd_op_sm_event_t *event, void *ctx)
         xlator_t             *this     = NULL;
         glusterd_peerinfo_t  *peerinfo = NULL;
         uint32_t             pending_count = 0;
+        dict_t               *dict     = NULL;
 
         this = THIS;
         priv = this->private;
@@ -1396,21 +2558,60 @@ glusterd_op_ac_send_lock (glusterd_op_sm_event_t *event, void *ctx)
                     (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
                         continue;
 
-                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_LOCK];
-                if (proc->fn) {
-                        ret = proc->fn (NULL, this, peerinfo);
-                        if (ret)
-                                continue;
-                        pending_count++;
+                /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+                if (priv->op_version < 3) {
+                        proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_LOCK];
+                        if (proc->fn) {
+                                ret = proc->fn (NULL, this, peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send lock request "
+                                                "for operation 'Volume %s' to "
+                                                "peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
+                } else {
+                        dict = glusterd_op_get_ctx ();
+                        dict_ref (dict);
+
+                        proc = &peerinfo->mgmt_v3->proctable
+                                                   [GLUSTERD_MGMT_V3_LOCK];
+                        if (proc->fn) {
+                                ret = dict_set_static_ptr (dict, "peerinfo",
+                                                           peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "failed to set peerinfo");
+                                        dict_unref (dict);
+                                        goto out;
+                                }
+
+                                ret = proc->fn (NULL, this, dict);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send mgmt_v3 lock "
+                                                "request for operation "
+                                                "'Volume %s' to peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        dict_unref (dict);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
                 }
         }
 
         opinfo.pending_count = pending_count;
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc ();
-
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
 
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
@@ -1423,17 +2624,12 @@ glusterd_op_ac_send_unlock (glusterd_op_sm_event_t *event, void *ctx)
         xlator_t             *this     = NULL;
         glusterd_peerinfo_t  *peerinfo = NULL;
         uint32_t             pending_count = 0;
+        dict_t               *dict     = NULL;
 
         this = THIS;
         priv = this->private;
         GF_ASSERT (priv);
 
-        /*ret = glusterd_unlock (priv->uuid);
-
-        if (ret)
-                goto out;
-        */
-
         list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
                 GF_ASSERT (peerinfo);
 
@@ -1443,23 +2639,62 @@ glusterd_op_ac_send_unlock (glusterd_op_sm_event_t *event, void *ctx)
                     (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
                         continue;
 
-                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_UNLOCK];
-                if (proc->fn) {
-                        ret = proc->fn (NULL, this, peerinfo);
-                        if (ret)
-                                continue;
-                        pending_count++;
+                /* Based on the op_version, release the *
+                 * cluster or mgmt_v3 lock */
+                if (priv->op_version < 3) {
+                        proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_UNLOCK];
+                        if (proc->fn) {
+                                ret = proc->fn (NULL, this, peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send unlock request "
+                                                "for operation 'Volume %s' to "
+                                                "peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
+                } else {
+                        dict = glusterd_op_get_ctx ();
+                        dict_ref (dict);
+
+                        proc = &peerinfo->mgmt_v3->proctable
+                                                   [GLUSTERD_MGMT_V3_UNLOCK];
+                        if (proc->fn) {
+                                ret = dict_set_static_ptr (dict, "peerinfo",
+                                                           peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "failed to set peerinfo");
+                                        dict_unref (dict);
+                                        goto out;
+                                }
+
+                                ret = proc->fn (NULL, this, dict);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send mgmt_v3 unlock "
+                                                "request for operation "
+                                                "'Volume %s' to peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        dict_unref (dict);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
                 }
         }
 
         opinfo.pending_count = pending_count;
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc ();
-
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
 
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
-
 }
 
 static int
@@ -1471,9 +2706,10 @@ glusterd_op_ac_ack_drain (glusterd_op_sm_event_t *event, void *ctx)
                 opinfo.pending_count--;
 
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, NULL);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                                   &event->txn_id, NULL);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
@@ -1487,41 +2723,97 @@ glusterd_op_ac_send_unlock_drain (glusterd_op_sm_event_t *event, void *ctx)
 static int
 glusterd_op_ac_lock (glusterd_op_sm_event_t *event, void *ctx)
 {
-        glusterd_op_lock_ctx_t   *lock_ctx = NULL;
-        int32_t                  ret = 0;
-
+        int32_t                         ret             = 0;
+        char                           *volname         = NULL;
+        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
+        glusterd_conf_t                *priv            = NULL;
+        xlator_t                       *this            = NULL;
 
         GF_ASSERT (event);
         GF_ASSERT (ctx);
 
+        this = THIS;
+        priv = this->private;
+
         lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
 
-        ret = glusterd_lock (lock_ctx->uuid);
+        /* If the req came from a node running on older op_version
+         * the dict won't be present. Based on it acquiring a cluster
+         * or mgmt_v3 lock */
+        if (lock_ctx->dict == NULL) {
+                ret = glusterd_lock (lock_ctx->uuid);
+                glusterd_op_lock_send_resp (lock_ctx->req, ret);
+        } else {
+                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+                else {
+                        ret = glusterd_mgmt_v3_lock (volname, lock_ctx->uuid,
+                                                     "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to acquire lock for %s",
+                                        volname);
+                }
 
-        gf_log ("", GF_LOG_DEBUG, "Lock Returned %d", ret);
+                glusterd_op_mgmt_v3_lock_send_resp (lock_ctx->req,
+                                                    &event->txn_id, ret);
 
-        glusterd_op_lock_send_resp (lock_ctx->req, ret);
+                dict_unref (lock_ctx->dict);
+        }
 
+        gf_log (THIS->name, GF_LOG_DEBUG, "Lock Returned %d", ret);
         return ret;
 }
 
 static int
 glusterd_op_ac_unlock (glusterd_op_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
-        glusterd_op_lock_ctx_t   *lock_ctx = NULL;
+        int32_t                         ret             = 0;
+        char                           *volname         = NULL;
+        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
+        glusterd_conf_t                *priv            = NULL;
+        xlator_t                       *this            = NULL;
+
 
         GF_ASSERT (event);
         GF_ASSERT (ctx);
 
+        this = THIS;
+        priv = this->private;
+
         lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
 
-        ret = glusterd_unlock (lock_ctx->uuid);
+        /* If the req came from a node running on older op_version
+         * the dict won't be present. Based on it releasing the cluster
+         * or mgmt_v3 lock */
+        if (lock_ctx->dict == NULL) {
+                ret = glusterd_unlock (lock_ctx->uuid);
+                glusterd_op_unlock_send_resp (lock_ctx->req, ret);
+        } else {
+                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+                else {
+                        ret = glusterd_mgmt_v3_unlock (volname, lock_ctx->uuid,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s", volname);
+                }
+
+                glusterd_op_mgmt_v3_unlock_send_resp (lock_ctx->req,
+                                                      &event->txn_id, ret);
 
-        gf_log ("", GF_LOG_DEBUG, "Unlock Returned %d", ret);
+                dict_unref (lock_ctx->dict);
+        }
 
-        glusterd_op_unlock_send_resp (lock_ctx->req, ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
 
+        if (priv->pending_quorum_action)
+                glusterd_do_quorum_action ();
         return ret;
 }
 
@@ -1538,7 +2830,7 @@ glusterd_op_ac_local_unlock (glusterd_op_sm_event_t *event, void *ctx)
 
         ret = glusterd_unlock (*originator);
 
-        gf_log ("", GF_LOG_DEBUG, "Unlock Returned %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
 
         return ret;
 }
@@ -1556,72 +2848,243 @@ glusterd_op_ac_rcvd_lock_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
+                                           &event->txn_id, NULL);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
 out:
         return ret;
 }
 
+static int
+glusterd_dict_set_volid (dict_t *dict, char *volname, char **op_errstr)
+{
+        int                     ret = -1;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *volid = NULL;
+        char                    msg[1024] = {0,};
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!dict || !volname)
+                goto out;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                goto out;
+        }
+        volid = gf_strdup (uuid_utoa (volinfo->volume_id));
+        if (!volid) {
+                ret = -1;
+                goto out;
+        }
+        ret = dict_set_dynstr (dict, "vol-id", volid);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to set volume id of volume"
+                          " %s", volname);
+                goto out;
+        }
+out:
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+        return ret;
+}
+
 int
-glusterd_op_build_payload (dict_t **req)
+glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx)
 {
         int                     ret = -1;
         void                    *ctx = NULL;
+        dict_t                  *dict = NULL;
         dict_t                  *req_dict = NULL;
         glusterd_op_t           op = GD_OP_NONE;
+        char                    *volname = NULL;
+        uint32_t                status_cmd = GF_CLI_STATUS_NONE;
+        char                    *errstr = NULL;
+        xlator_t                *this = NULL;
 
         GF_ASSERT (req);
 
+        this = THIS;
+        GF_ASSERT (this);
+
         req_dict = dict_new ();
         if (!req_dict)
                 goto out;
 
-        op  = glusterd_op_get_op ();
-        ctx = (void*)glusterd_op_get_ctx ();
-        if (!ctx) {
-                gf_log ("", GF_LOG_ERROR, "Null Context for "
-                        "op %d", op);
-                ret = -1;
-                goto out;
+        if (!op_ctx) {
+                op  = glusterd_op_get_op ();
+                ctx = (void*)glusterd_op_get_ctx ();
+                if (!ctx) {
+                        gf_log (this->name, GF_LOG_ERROR, "Null Context for "
+                                "op %d", op);
+                        ret = -1;
+                        goto out;
+                }
+
+        } else {
+#define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
+                ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, (int32_t*)&op);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get volume"
+                                " operation");
+                        goto out;
+                }
+                ctx = op_ctx;
+#undef GD_SYNC_OPCODE_KEY
         }
 
+        dict = ctx;
         switch (op) {
                 case GD_OP_CREATE_VOLUME:
                         {
-                                dict_t  *dict = ctx;
                                 ++glusterfs_port;
-                                ret = dict_set_int32 (dict, "port", glusterfs_port);
-                                if (ret)
+                                ret = dict_set_int32 (dict, "port",
+                                                      glusterfs_port);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to set port in "
+                                                "dictionary");
+                                        goto out;
+                                }
+                                dict_copy (dict, req_dict);
+                        }
+                        break;
+
+                case GD_OP_GSYNC_CREATE:
+                case GD_OP_GSYNC_SET:
+                        {
+                                ret = glusterd_op_gsync_args_get (dict,
+                                                                  &errstr,
+                                                                  &volname,
+                                                                  NULL, NULL);
+                                if (ret == 0) {
+                                        ret = glusterd_dict_set_volid
+                                                (dict, volname, op_errstr);
+                                        if (ret)
+                                                goto out;
+                                }
+                                dict_copy (dict, req_dict);
+                        }
+                        break;
+
+                case GD_OP_SET_VOLUME:
+                        {
+                                ret = dict_get_str (dict, "volname", &volname);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_CRITICAL,
+                                                "volname is not present in "
+                                                "operation ctx");
                                         goto out;
+                                }
+                                if (strcmp (volname, "help") &&
+                                    strcmp (volname, "help-xml") &&
+                                    strcasecmp (volname, "all")) {
+                                        ret = glusterd_dict_set_volid
+                                                 (dict, volname, op_errstr);
+                                        if (ret)
+                                                goto out;
+                                }
+                                dict_destroy (req_dict);
+                                req_dict = dict_ref (dict);
+                        }
+                        break;
+
+                case GD_OP_SYNC_VOLUME:
+                        {
                                 dict_copy (dict, req_dict);
+                                break;
+                        }
+
+                case GD_OP_REMOVE_BRICK:
+                        {
+                                dict_t *dict = ctx;
+                                ret = dict_get_str (dict, "volname", &volname);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_CRITICAL,
+                                                "volname is not present in "
+                                                "operation ctx");
+                                        goto out;
+                                }
+
+                                ret = glusterd_dict_set_volid (dict, volname,
+                                                               op_errstr);
+                                if (ret)
+                                        goto out;
+
+                                dict_destroy (req_dict);
+                                req_dict = dict_ref (dict);
                         }
                         break;
 
+                case GD_OP_STATUS_VOLUME:
+                        {
+                                ret = dict_get_uint32 (dict, "cmd",
+                                                       &status_cmd);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Status command not present "
+                                                "in op ctx");
+                                        goto out;
+                                }
+                                if (GF_CLI_STATUS_ALL & status_cmd) {
+                                        dict_copy (dict, req_dict);
+                                        break;
+                                }
+                        }
+                        /*fall-through*/
                 case GD_OP_DELETE_VOLUME:
                 case GD_OP_START_VOLUME:
                 case GD_OP_STOP_VOLUME:
                 case GD_OP_ADD_BRICK:
                 case GD_OP_REPLACE_BRICK:
-                case GD_OP_SET_VOLUME:
                 case GD_OP_RESET_VOLUME:
-                case GD_OP_REMOVE_BRICK:
                 case GD_OP_LOG_ROTATE:
-                case GD_OP_SYNC_VOLUME:
                 case GD_OP_QUOTA:
-                case GD_OP_GSYNC_SET:
                 case GD_OP_PROFILE_VOLUME:
-                case GD_OP_STATUS_VOLUME:
                 case GD_OP_REBALANCE:
                 case GD_OP_HEAL_VOLUME:
                 case GD_OP_STATEDUMP_VOLUME:
+                case GD_OP_CLEARLOCKS_VOLUME:
+                case GD_OP_DEFRAG_BRICK_VOLUME:
                         {
-                                dict_t  *dict = ctx;
+                                ret = dict_get_str (dict, "volname", &volname);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_CRITICAL,
+                                                "volname is not present in "
+                                                "operation ctx");
+                                        goto out;
+                                }
+
+                                if (strcasecmp (volname, "all")) {
+                                        ret = glusterd_dict_set_volid (dict,
+                                                                       volname,
+                                                                     op_errstr);
+                                        if (ret)
+                                                goto out;
+                                }
                                 dict_copy (dict, req_dict);
                         }
                         break;
 
+                case GD_OP_COPY_FILE:
+                        {
+                                dict_copy (dict, req_dict);
+                                break;
+                        }
+
+                case GD_OP_SYS_EXEC:
+                        {
+                                dict_copy (dict, req_dict);
+                                break;
+                        }
+
                 default:
                         break;
         }
@@ -1633,6 +3096,105 @@ out:
         return ret;
 }
 
+gf_boolean_t
+glusterd_is_get_op (xlator_t *this, glusterd_op_t op, dict_t *dict)
+{
+        char            *key = NULL;
+        char            *volname = NULL;
+        int             ret = 0;
+
+        if (op == GD_OP_STATUS_VOLUME)
+                return _gf_true;
+
+        if ((op == GD_OP_SET_VOLUME)) {
+                //check for set volume help
+                ret = dict_get_str (dict, "volname", &volname);
+                if (volname &&
+                    ((strcmp (volname, "help") == 0) ||
+                     (strcmp (volname, "help-xml") == 0))) {
+                        ret = dict_get_str (dict, "key1", &key);
+                        if (ret < 0)
+                                return _gf_true;
+                }
+        }
+
+        return _gf_false;
+}
+
+gf_boolean_t
+glusterd_is_op_quorum_validation_required (xlator_t *this, glusterd_op_t op,
+                                           dict_t *dict)
+{
+        gf_boolean_t    required = _gf_true;
+        char            *key = NULL;
+        char            *key_fixed = NULL;
+        int             ret = -1;
+
+        if (glusterd_is_get_op (this, op, dict)) {
+                required = _gf_false;
+                goto out;
+        }
+        if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
+                goto out;
+        if (op == GD_OP_SET_VOLUME)
+                ret = dict_get_str (dict, "key1", &key);
+        else if (op == GD_OP_RESET_VOLUME)
+                ret = dict_get_str (dict, "key", &key);
+        if (ret)
+                goto out;
+        ret = glusterd_check_option_exists (key, &key_fixed);
+        if (ret <= 0)
+                goto out;
+        if (key_fixed)
+                key = key_fixed;
+        if (glusterd_is_quorum_option (key))
+                required = _gf_false;
+out:
+        GF_FREE (key_fixed);
+        return required;
+}
+
+static int
+glusterd_op_validate_quorum (xlator_t *this, glusterd_op_t op,
+                             dict_t *dict, char **op_errstr)
+{
+        int                     ret = 0;
+        char                    *volname = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *errstr = NULL;
+
+
+        errstr = "Quorum not met. Volume operation not allowed.";
+        if (!glusterd_is_op_quorum_validation_required (this, op, dict))
+                goto out;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        if (does_gd_meet_server_quorum (this)) {
+                ret = 0;
+                goto out;
+        }
+
+        if (glusterd_is_volume_in_server_quorum (volinfo)) {
+                ret = -1;
+                *op_errstr = gf_strdup (errstr);
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
 static int
 glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
 {
@@ -1653,14 +3215,32 @@ glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
 
         op = glusterd_op_get_op ();
 
-        ret = glusterd_op_build_payload (&dict);
-        if (ret)
+        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                opinfo.op_errstr = op_errstr;
                 goto out;
+        }
+
+        ret = glusterd_op_validate_quorum (this, op, dict, &op_errstr);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+                opinfo.op_errstr = op_errstr;
+                goto out;
+        }
 
         /* rsp_dict NULL from source */
         ret = glusterd_op_stage_validate (op, dict, &op_errstr, NULL);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Staging failed");
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_STAGE_FAIL,
+                        gd_op_list[op], "localhost",
+                        (op_errstr) ? ":" : " ", (op_errstr) ? op_errstr : " ");
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_STAGE_FAIL,
+                                     "localhost");
                 opinfo.op_errstr = op_errstr;
                 goto out;
         }
@@ -1679,13 +3259,19 @@ glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
                 if (proc->fn) {
                         ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
+                                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                        "set peerinfo");
                                 goto out;
                         }
 
                         ret = proc->fn (NULL, this, dict);
-                        if (ret)
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
+                                        "send stage request for operation "
+                                        "'Volume %s' to peer %s",
+                                        gd_op_list[op], peerinfo->hostname);
                                 continue;
+                        }
                         pending_count++;
                 }
         }
@@ -1695,27 +3281,29 @@ out:
         if (dict)
                 dict_unref (dict);
         if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             &event->txn_id, NULL);
                 opinfo.op_ret = ret;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Sent op req to %d peers",
+        gf_log (this->name, GF_LOG_DEBUG, "Sent stage op request for "
+                "'Volume %s' to %d peers", gd_op_list[op],
                 opinfo.pending_count);
 
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc ();
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 
 }
 
 static int32_t
-glusterd_op_start_rb_timer (dict_t *dict)
+glusterd_op_start_rb_timer (dict_t *dict, uuid_t *txn_id)
 {
         int32_t         op = 0;
-        struct timeval  timeout = {0, };
+        struct timespec timeout = {0, };
         glusterd_conf_t *priv = NULL;
         int32_t         ret = -1;
         dict_t          *rb_ctx = NULL;
@@ -1730,13 +3318,13 @@ glusterd_op_start_rb_timer (dict_t *dict)
                 goto out;
         }
 
-        if (op == GF_REPLACE_OP_START ||
-            op == GF_REPLACE_OP_ABORT)
-                timeout.tv_sec  = 5;
-        else
-                timeout.tv_sec = 1;
+        if (op != GF_REPLACE_OP_START) {
+                ret = glusterd_op_sm_inject_all_acc (txn_id);
+                goto out;
+        }
 
-        timeout.tv_usec = 0;
+        timeout.tv_sec  = 5;
+        timeout.tv_nsec = 0;
 
 
         rb_ctx = dict_copy (dict, rb_ctx);
@@ -1746,6 +3334,17 @@ glusterd_op_start_rb_timer (dict_t *dict)
                 ret = -1;
                 goto out;
         }
+
+        ret = dict_set_bin (rb_ctx, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+                goto out;
+        } else
+               gf_log ("", GF_LOG_DEBUG,
+                        "transaction_id = %s", uuid_utoa (*txn_id));
+
         priv->timer = gf_timer_call_after (THIS->ctx, timeout,
                                            glusterd_do_replace_brick,
                                            (void *) rb_ctx);
@@ -1756,6 +3355,360 @@ out:
         return ret;
 }
 
+/* This function takes a dict and converts the uuid values of key specified
+ * into hostnames
+ */
+static int
+glusterd_op_volume_dict_uuid_to_hostname (dict_t *dict, const char *key_fmt,
+                                          int idx_min, int idx_max)
+{
+        int             ret = -1;
+        int             i = 0;
+        char            key[1024];
+        char            *uuid_str = NULL;
+        uuid_t          uuid = {0,};
+        char            *hostname = NULL;
+        xlator_t        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key_fmt);
+
+        for (i = idx_min; i < idx_max; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), key_fmt, i);
+                ret = dict_get_str (dict, key, &uuid_str);
+                if (ret)
+                        continue;
+
+                gf_log (this->name, GF_LOG_DEBUG, "Got uuid %s",
+                        uuid_str);
+
+                ret = uuid_parse (uuid_str, uuid);
+                /* if parsing fails don't error out
+                 * let the original value be retained
+                 */
+                if (ret)
+                        continue;
+
+                hostname = glusterd_uuid_to_hostname (uuid);
+                if (hostname) {
+                        gf_log (this->name, GF_LOG_DEBUG, "%s -> %s",
+                                uuid_str, hostname);
+                        ret = dict_set_dynstr (dict, key, hostname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Error setting hostname %s to dict",
+                                        hostname);
+                                GF_FREE (hostname);
+                                goto out;
+                        }
+                }
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+reassign_defrag_status (dict_t *dict, char *key, gf_defrag_status_t *status)
+{
+        int ret = 0;
+
+        if (!*status)
+                return ret;
+
+        switch (*status) {
+        case GF_DEFRAG_STATUS_STARTED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED;
+                break;
+
+        case GF_DEFRAG_STATUS_STOPPED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED;
+                break;
+
+        case GF_DEFRAG_STATUS_COMPLETE:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
+                break;
+
+        case GF_DEFRAG_STATUS_FAILED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED;
+                break;
+        default:
+                break;
+         }
+
+        ret = dict_set_int32(dict, key, *status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to reset defrag %s in dict", key);
+
+        return ret;
+}
+
+/* Check and reassign the defrag_status enum got from the rebalance process
+ * of all peers so that the rebalance-status CLI command can display if a
+ * full-rebalance or just a fix-layout was carried out.
+ */
+static int
+glusterd_op_check_peer_defrag_status (dict_t *dict, int count)
+{
+        glusterd_volinfo_t *volinfo  = NULL;
+        gf_defrag_status_t status    = GF_DEFRAG_STATUS_NOT_STARTED;
+        char               key[256]  = {0,};
+        char               *volname  = NULL;
+        int                ret       = -1;
+        int                i         = 1;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        if (volinfo->rebal.defrag_cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                /* Fix layout was not issued; we don't need to reassign
+                   the status */
+                ret = 0;
+                goto out;
+        }
+
+        do {
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", i);
+                ret = dict_get_int32 (dict, key, (int32_t *)&status);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to get defrag %s", key);
+                        goto out;
+                }
+                ret = reassign_defrag_status (dict, key, &status);
+                if (ret)
+                        goto out;
+                i++;
+        } while (i <= count);
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+/* This function is used to modify the op_ctx dict before sending it back
+ * to cli. This is useful in situations like changing the peer uuids to
+ * hostnames etc.
+ */
+void
+glusterd_op_modify_op_ctx (glusterd_op_t op, void *ctx)
+{
+        int             ret = -1;
+        dict_t          *op_ctx = NULL;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             count = 0;
+        uint32_t        cmd = GF_CLI_STATUS_NONE;
+        xlator_t        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (ctx)
+                op_ctx = ctx;
+        else
+                op_ctx = glusterd_op_get_ctx();
+
+        if (!op_ctx) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Operation context is not present.");
+                goto out;
+        }
+
+        switch (op) {
+        case GD_OP_STATUS_VOLUME:
+                ret = dict_get_uint32 (op_ctx, "cmd", &cmd);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get status cmd");
+                        goto out;
+                }
+                if (!(cmd & GF_CLI_STATUS_NFS || cmd & GF_CLI_STATUS_SHD ||
+                    (cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "op_ctx modification not required for status "
+                                "operation being performed");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (op_ctx, "brick-index-max",
+                                      &brick_index_max);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get brick-index-max");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (op_ctx, "other-count", &other_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get other-count");
+                        goto out;
+                }
+
+                count = brick_index_max + other_count + 1;
+
+                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
+                                                                "brick%d.path",
+                                                                0, count);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Failed uuid to hostname conversion");
+
+                break;
+
+        case GD_OP_PROFILE_VOLUME:
+                ret = dict_get_str_boolean (op_ctx, "nfs", _gf_false);
+                if (!ret)
+                        goto out;
+
+                ret = dict_get_int32 (op_ctx, "count", &count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get brick count");
+                        goto out;
+                }
+
+                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
+                                                                "%d-brick",
+                                                                1, (count + 1));
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Failed uuid to hostname conversion");
+
+                break;
+
+        /* For both rebalance and remove-brick status, the glusterd op is the
+         * same
+         */
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = dict_get_int32 (op_ctx, "count", &count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get count");
+                        goto out;
+                }
+
+                /* add 'node-name-%d' into op_ctx with value uuid_str.
+                   this will be used to convert to hostname later */
+                {
+                        char  key[1024];
+                        char *uuid_str = NULL;
+                        int   i;
+
+                        for (i = 1; i <= count; i++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key), "node-uuid-%d", i);
+                                ret = dict_get_str (op_ctx, key, &uuid_str);
+                                if (!ret) {
+                                        memset (key, 0, sizeof (key));
+                                        snprintf (key, sizeof (key),
+                                                  "node-name-%d", i);
+                                        ret = dict_set_str (op_ctx, key,
+                                                            uuid_str);
+                                }
+                        }
+                }
+
+                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
+                                                                "node-name-%d",
+                                                                1, (count + 1));
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Failed uuid to hostname conversion");
+
+                ret = glusterd_op_check_peer_defrag_status (op_ctx, count);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to reset defrag status for fix-layout");
+                break;
+
+        default:
+                ret = 0;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "op_ctx modification not required");
+                break;
+
+        }
+
+out:
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "op_ctx modification failed");
+        return;
+}
+
+static int
+glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_commit_hook_type_t type)
+{
+        glusterd_conf_t *priv                   = NULL;
+        char            hookdir[PATH_MAX]       = {0, };
+        char            scriptdir[PATH_MAX]     = {0, };
+        char            type_subdir[256]        = {0, };
+        char            *cmd_subdir             = NULL;
+        int             ret                     = -1;
+
+        priv = THIS->private;
+        switch (type) {
+                case GD_COMMIT_HOOK_NONE:
+                case GD_COMMIT_HOOK_MAX:
+                        /*Won't be called*/
+                        break;
+
+                case GD_COMMIT_HOOK_PRE:
+                        strcpy (type_subdir, "pre");
+                        break;
+                case GD_COMMIT_HOOK_POST:
+                        strcpy (type_subdir, "post");
+                        break;
+        }
+
+        cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
+        if (strlen (cmd_subdir) == 0)
+                return -1;
+
+        GLUSTERD_GET_HOOKS_DIR (hookdir, GLUSTERD_HOOK_VER, priv);
+        snprintf (scriptdir, sizeof (scriptdir), "%s/%s/%s",
+                  hookdir, cmd_subdir, type_subdir);
+
+        switch (type) {
+                case GD_COMMIT_HOOK_NONE:
+                case GD_COMMIT_HOOK_MAX:
+                        /*Won't be called*/
+                        break;
+
+                case GD_COMMIT_HOOK_PRE:
+                        ret = glusterd_hooks_run_hooks (scriptdir, op, op_ctx,
+                                                        type);
+                        break;
+                case GD_COMMIT_HOOK_POST:
+                        ret = glusterd_hooks_post_stub_enqueue (scriptdir, op,
+                                                                op_ctx);
+                        break;
+        }
+
+        return ret;
+}
+
 static int
 glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
 {
@@ -1775,19 +3728,32 @@ glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
         priv = this->private;
         GF_ASSERT (priv);
 
-        op = glusterd_op_get_op ();
-        ret = glusterd_op_build_payload (&dict);
+        op      = glusterd_op_get_op ();
+        op_dict = glusterd_op_get_ctx ();
 
-        if (ret)
+        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                opinfo.op_errstr = op_errstr;
                 goto out;
+        }
 
         ret = glusterd_op_commit_perform (op, dict, &op_errstr, NULL); //rsp_dict invalid for source
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Commit failed");
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_COMMIT_FAIL,
+                        gd_op_list[op], "localhost", (op_errstr) ? ":" : " ",
+                        (op_errstr) ? op_errstr : " ");
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_COMMIT_FAIL,
+                                     "localhost");
                 opinfo.op_errstr = op_errstr;
                 goto out;
         }
 
+
         list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
                 GF_ASSERT (peerinfo);
 
@@ -1802,41 +3768,48 @@ glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
                 if (proc->fn) {
                         ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to set peerinfo");
                                 goto out;
                         }
                         ret = proc->fn (NULL, this, dict);
-                        if (ret)
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
+                                        "send commit request for operation "
+                                        "'Volume %s' to peer %s",
+                                        gd_op_list[op], peerinfo->hostname);
                                 continue;
+                        }
                         pending_count++;
                 }
         }
 
         opinfo.pending_count = pending_count;
-        gf_log ("glusterd", GF_LOG_INFO, "Sent op req to %d peers",
-                opinfo.pending_count);
+        gf_log (this->name, GF_LOG_DEBUG, "Sent commit op req for 'Volume %s' "
+                "to %d peers", gd_op_list[op], opinfo.pending_count);
 out:
         if (dict)
                 dict_unref (dict);
         if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             &event->txn_id, NULL);
                 opinfo.op_ret = ret;
         }
 
         if (!opinfo.pending_count) {
                 if (op == GD_OP_REPLACE_BRICK) {
-                        op_dict = glusterd_op_get_ctx ();
-                        ret = glusterd_op_start_rb_timer (op_dict);
+                        ret = glusterd_op_start_rb_timer (op_dict,
+                                                          &event->txn_id);
 
                 } else {
-
-                        ret = glusterd_op_sm_inject_all_acc ();
+                        glusterd_op_modify_op_ctx (op, NULL);
+                        ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
                 }
                 goto err;
         }
 
 err:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 
@@ -1855,10 +3828,11 @@ glusterd_op_ac_rcvd_stage_op_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_ACC,
+                                           &event->txn_id, NULL);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1876,10 +3850,11 @@ glusterd_op_ac_stage_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, NULL);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1897,10 +3872,11 @@ glusterd_op_ac_commit_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, NULL);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1911,6 +3887,10 @@ glusterd_op_ac_brick_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         int                     ret = 0;
         glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
         gf_boolean_t                free_errstr = _gf_false;
+        xlator_t                    *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (event);
         GF_ASSERT (ctx);
@@ -1918,7 +3898,7 @@ glusterd_op_ac_brick_op_failed (glusterd_op_sm_event_t *event, void *ctx)
 
         ret = glusterd_remove_pending_entry (&opinfo.pending_bricks, ev_ctx->pending_node->node);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "unknown response received ");
+                gf_log (this->name, GF_LOG_ERROR, "unknown response received ");
                 ret = -1;
                 free_errstr = _gf_true;
                 goto out;
@@ -1936,7 +3916,8 @@ glusterd_op_ac_brick_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.brick_pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, ev_ctx->commit_ctx);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, ev_ctx->commit_ctx);
 
 out:
         if (ev_ctx->rsp_dict)
@@ -1944,7 +3925,7 @@ out:
         if (free_errstr && ev_ctx->op_errstr)
                 GF_FREE (ev_ctx->op_errstr);
         GF_FREE (ctx);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1952,11 +3933,14 @@ out:
 static int
 glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
 {
-        dict_t                 *dict              = NULL;
+        dict_t                 *op_ctx            = NULL;
         int                     ret               = 0;
         gf_boolean_t            commit_ack_inject = _gf_true;
         glusterd_op_t           op                = GD_OP_NONE;
+        xlator_t               *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         op = glusterd_op_get_op ();
         GF_ASSERT (event);
 
@@ -1967,17 +3951,17 @@ glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
                 goto out;
 
         if (op == GD_OP_REPLACE_BRICK) {
-                dict = glusterd_op_get_ctx ();
-                if (!dict) {
-                        gf_log (THIS->name, GF_LOG_CRITICAL, "Operation "
+                op_ctx = glusterd_op_get_ctx ();
+                if (!op_ctx) {
+                        gf_log (this->name, GF_LOG_CRITICAL, "Operation "
                                 "context is not present.");
                         ret = -1;
                         goto out;
                 }
 
-                ret = glusterd_op_start_rb_timer (dict);
+                ret = glusterd_op_start_rb_timer (op_ctx, &event->txn_id);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "Couldn't start "
+                        gf_log (this->name, GF_LOG_ERROR, "Couldn't start "
                                 "replace-brick operation.");
                         goto out;
                 }
@@ -1986,13 +3970,17 @@ glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
                 goto out;
         }
 
+
 out:
         if (commit_ack_inject) {
                 if (ret)
-                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
-                else if (!opinfo.pending_count)
-                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC, NULL);
-
+                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                                           &event->txn_id, NULL);
+                else if (!opinfo.pending_count) {
+                        glusterd_op_modify_op_ctx (op, NULL);
+                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC,
+                                                           &event->txn_id, NULL);
+                }
                 /*else do nothing*/
         }
 
@@ -2012,9 +4000,10 @@ glusterd_op_ac_rcvd_unlock_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
+                                           &event->txn_id, NULL);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
 out:
         return ret;
@@ -2046,7 +4035,7 @@ glusterd_op_reset_ctx ()
 }
 
 int32_t
-glusterd_op_txn_complete ()
+glusterd_op_txn_complete (uuid_t *txn_id)
 {
         int32_t                 ret = -1;
         glusterd_conf_t         *priv = NULL;
@@ -2056,9 +4045,13 @@ glusterd_op_txn_complete ()
         rpcsvc_request_t        *req = NULL;
         void                    *ctx = NULL;
         char                    *op_errstr = NULL;
+        char                    *volname = NULL;
+        xlator_t                *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
 
-        priv = THIS->private;
+        priv = this->private;
         GF_ASSERT (priv);
 
         op  = glusterd_op_get_op ();
@@ -2075,32 +4068,55 @@ glusterd_op_txn_complete ()
         glusterd_op_reset_ctx ();
         glusterd_op_clear_errstr ();
 
-        ret = glusterd_unlock (priv->uuid);
-
-        /* unlock cant/shouldnt fail here!! */
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to clear local lock, ret: %d", ret);
+        /* Based on the op-version, we release the cluster or mgmt_v3 lock */
+        if (priv->op_version < 3) {
+                ret = glusterd_unlock (MY_UUID);
+                /* unlock cant/shouldnt fail here!! */
+                if (ret)
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to clear local lock, ret: %d", ret);
+                else
+                        gf_log (this->name, GF_LOG_DEBUG, "Cleared local lock");
         } else {
-                gf_log ("glusterd", GF_LOG_INFO, "Cleared local lock");
+                ret = dict_get_str (ctx, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+
+                if (volname) {
+                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                }
         }
 
         ret = glusterd_op_send_cli_response (op, op_ret,
                                              op_errno, req, ctx, op_errstr);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Responding to cli failed, ret: %d",
-                        ret);
+                gf_log (this->name, GF_LOG_ERROR, "Responding to cli failed, "
+                        "ret: %d", ret);
                 //Ignore this error, else state machine blocks
                 ret = 0;
         }
 
-        glusterd_op_free_ctx (op, ctx);
         if (op_errstr && (strcmp (op_errstr, "")))
                 GF_FREE (op_errstr);
 
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        if (priv->pending_quorum_action)
+                glusterd_do_quorum_action ();
+
+        /* Clearing the transaction opinfo */
+        ret = glusterd_clear_txn_opinfo (txn_id);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to clear transaction's opinfo");
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -2111,9 +4127,9 @@ glusterd_op_ac_unlocked_all (glusterd_op_sm_event_t *event, void *ctx)
 
         GF_ASSERT (event);
 
-        ret = glusterd_op_txn_complete ();
+        ret = glusterd_op_txn_complete (&event->txn_id);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -2127,7 +4143,11 @@ glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
         dict_t                  *rsp_dict  = NULL;
         char                    *op_errstr = NULL;
         dict_t                  *dict = NULL;
+        xlator_t                *this = NULL;
+        uuid_t                  *txn_id = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (ctx);
 
         req_ctx = ctx;
@@ -2136,8 +4156,8 @@ glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
 
         rsp_dict = dict_new ();
         if (!rsp_dict) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "Out of memory");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
                 return -1;
         }
 
@@ -2145,16 +4165,31 @@ glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
                                              rsp_dict);
 
         if (status) {
-                gf_log ("", GF_LOG_ERROR, "Validate failed: %d", status);
+                gf_log (this->name, GF_LOG_ERROR, "Stage failed on operation"
+                        " 'Volume %s', Status : %d", gd_op_list[req_ctx->op],
+                        status);
         }
 
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+
+        if (txn_id)
+                uuid_copy (*txn_id, event->txn_id);
+        else
+                gf_log (this->name, GF_LOG_ERROR, "Out of Memory");
+
+        ret = dict_set_bin (rsp_dict, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+
         ret = glusterd_op_stage_send_resp (req_ctx->req, req_ctx->op,
                                            status, op_errstr, rsp_dict);
 
         if (op_errstr && (strcmp (op_errstr, "")))
                 GF_FREE (op_errstr);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         if (rsp_dict)
                 dict_unref (rsp_dict);
@@ -2172,6 +4207,8 @@ glusterd_need_brick_op (glusterd_op_t op)
         switch (op) {
         case GD_OP_PROFILE_VOLUME:
         case GD_OP_STATUS_VOLUME:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+        case GD_OP_HEAL_VOLUME:
                 ret = _gf_true;
                 break;
         default:
@@ -2181,7 +4218,7 @@ glusterd_need_brick_op (glusterd_op_t op)
         return ret;
 }
 
-static dict_t*
+dict_t*
 glusterd_op_init_commit_rsp_dict (glusterd_op_t op)
 {
         dict_t                  *rsp_dict = NULL;
@@ -2204,12 +4241,16 @@ static int
 glusterd_op_ac_commit_op (glusterd_op_sm_event_t *event, void *ctx)
 {
         int                       ret        = 0;
-        glusterd_req_ctx_t       *req_ctx = NULL;
+        glusterd_req_ctx_t       *req_ctx    = NULL;
         int32_t                   status     = 0;
         char                     *op_errstr  = NULL;
         dict_t                   *dict       = NULL;
-        dict_t                   *rsp_dict = NULL;
+        dict_t                   *rsp_dict   = NULL;
+        xlator_t                 *this       = NULL;
+        uuid_t                   *txn_id     = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (ctx);
 
         req_ctx = ctx;
@@ -2219,24 +4260,46 @@ glusterd_op_ac_commit_op (glusterd_op_sm_event_t *event, void *ctx)
         rsp_dict = glusterd_op_init_commit_rsp_dict (req_ctx->op);
         if (NULL == rsp_dict)
                 return -1;
-        status = glusterd_op_commit_perform (req_ctx->op, dict, &op_errstr,
-                                             rsp_dict);
 
-        if (status) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Commit failed: %d", status);
+
+        if (GD_OP_CLEARLOCKS_VOLUME == req_ctx->op) {
+                /*clear locks should be run only on
+                 * originator glusterd*/
+                status = 0;
+
+        } else {
+                status = glusterd_op_commit_perform (req_ctx->op, dict,
+                                                     &op_errstr, rsp_dict);
         }
 
+        if (status)
+                gf_log (this->name, GF_LOG_ERROR, "Commit of operation "
+                        "'Volume %s' failed: %d", gd_op_list[req_ctx->op],
+                        status);
+
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+
+        if (txn_id)
+                uuid_copy (*txn_id, event->txn_id);
+        else
+                gf_log (this->name, GF_LOG_ERROR, "Out of Memory");
+
+        ret = dict_set_bin (rsp_dict, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+
         ret = glusterd_op_commit_send_resp (req_ctx->req, req_ctx->op,
                                             status, op_errstr, rsp_dict);
 
-        glusterd_op_fini_ctx ();
         if (op_errstr && (strcmp (op_errstr, "")))
                 GF_FREE (op_errstr);
 
         if (rsp_dict)
                 dict_unref (rsp_dict);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
@@ -2258,13 +4321,12 @@ glusterd_op_ac_send_commit_failed (glusterd_op_sm_event_t *event, void *ctx)
                                             opinfo.op_ret, opinfo.op_errstr,
                                             op_ctx);
 
-        glusterd_op_fini_ctx ();
         if (opinfo.op_errstr && (strcmp (opinfo.op_errstr, ""))) {
                 GF_FREE (opinfo.op_errstr);
                 opinfo.op_errstr = NULL;
         }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
@@ -2295,6 +4357,7 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
                             dict_t *rsp_dict)
 {
         int ret = -1;
+        xlator_t *this = THIS;
 
         switch (op) {
                 case GD_OP_CREATE_VOLUME:
@@ -2342,6 +4405,10 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         ret = glusterd_op_stage_sync_volume (dict, op_errstr);
                         break;
 
+                case GD_OP_GSYNC_CREATE:
+                        ret = glusterd_op_stage_gsync_create (dict, op_errstr);
+                        break;
+
                 case GD_OP_GSYNC_SET:
                         ret = glusterd_op_stage_gsync_set (dict, op_errstr);
                         break;
@@ -2359,6 +4426,7 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         break;
 
                 case GD_OP_REBALANCE:
+                case GD_OP_DEFRAG_BRICK_VOLUME:
                         ret = glusterd_op_stage_rebalance (dict, op_errstr);
                         break;
 
@@ -2370,14 +4438,25 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         ret = glusterd_op_stage_statedump_volume (dict,
                                                                   op_errstr);
                         break;
+                case GD_OP_CLEARLOCKS_VOLUME:
+                        ret = glusterd_op_stage_clearlocks_volume (dict,
+                                                                   op_errstr);
+                        break;
+
+                case GD_OP_COPY_FILE:
+                        ret = glusterd_op_stage_copy_file (dict, op_errstr);
+                        break;
+
+                case GD_OP_SYS_EXEC:
+                        ret = glusterd_op_stage_sys_exec (dict, op_errstr);
+                        break;
 
                 default:
-                        gf_log ("", GF_LOG_ERROR, "Unknown op %d",
-                                op);
+                        gf_log (this->name, GF_LOG_ERROR, "Unknown op %s",
+                                gd_op_list[op]);
         }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-
+        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
         return ret;
 }
 
@@ -2387,7 +4466,9 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                             dict_t *rsp_dict)
 {
         int ret = -1;
+        xlator_t *this = THIS;
 
+        glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_PRE);
         switch (op) {
                 case GD_OP_CREATE_VOLUME:
                         ret = glusterd_op_create_volume (dict, op_errstr);
@@ -2418,7 +4499,7 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         break;
 
                 case GD_OP_RESET_VOLUME:
-                        ret = glusterd_op_reset_volume (dict);
+                        ret = glusterd_op_reset_volume (dict, op_errstr);
                         break;
 
                 case GD_OP_REMOVE_BRICK:
@@ -2433,6 +4514,11 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         ret = glusterd_op_sync_volume (dict, op_errstr, rsp_dict);
                         break;
 
+                case GD_OP_GSYNC_CREATE:
+                        ret = glusterd_op_gsync_create (dict, op_errstr,
+                                                        rsp_dict);
+                        break;
+
                 case GD_OP_GSYNC_SET:
                         ret = glusterd_op_gsync_set (dict, op_errstr, rsp_dict);
                         break;
@@ -2443,7 +4529,7 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         break;
 
                 case GD_OP_QUOTA:
-                        ret = glusterd_op_quota (dict, op_errstr);
+                        ret = glusterd_op_quota (dict, op_errstr, rsp_dict);
                         break;
 
                 case GD_OP_STATUS_VOLUME:
@@ -2451,6 +4537,7 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         break;
 
                 case GD_OP_REBALANCE:
+                case GD_OP_DEFRAG_BRICK_VOLUME:
                         ret = glusterd_op_rebalance (dict, op_errstr, rsp_dict);
                         break;
 
@@ -2462,150 +4549,36 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         ret = glusterd_op_statedump_volume (dict, op_errstr);
                         break;
 
-                default:
-                        gf_log ("", GF_LOG_ERROR, "Unknown op %d",
-                                op);
+                case GD_OP_CLEARLOCKS_VOLUME:
+                        ret = glusterd_op_clearlocks_volume (dict, op_errstr,
+                                                             rsp_dict);
                         break;
-        }
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
 
-void
-_profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
-                             void *data)
-{
-        char    new_key[256] = {0};
-        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
-        data_t  *new_value = NULL;
-
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        GF_ASSERT (new_value);
-        snprintf (new_key, sizeof (new_key), "%d-%s", rsp_ctx->count, key);
-        dict_set (rsp_ctx->dict, new_key, new_value);
-}
-
-int
-glusterd_profile_volume_brick_rsp (glusterd_brickinfo_t *brickinfo,
-                                   dict_t *rsp_dict, dict_t *op_ctx,
-                                   char **op_errstr)
-{
-        int     ret = 0;
-        glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
-        int32_t count = 0;
-        char    brick[PATH_MAX+1024] = {0};
-        char    key[256] = {0};
-        char    *full_brick = NULL;
-
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (brickinfo);
-
-        ret = dict_get_int32 (op_ctx, "count", &count);
-        if (ret) {
-                count = 1;
-        } else {
-                count++;
-        }
-        snprintf (key, sizeof (key), "%d-brick", count);
-        snprintf (brick, sizeof (brick), "%s:%s", brickinfo->hostname,
-                  brickinfo->path);
-        full_brick = gf_strdup (brick);
-        GF_ASSERT (full_brick);
-        ret = dict_set_dynstr (op_ctx, key, full_brick);
-
-        rsp_ctx.count = count;
-        rsp_ctx.dict = op_ctx;
-        dict_foreach (rsp_dict, _profile_volume_add_brick_rsp, &rsp_ctx);
-        dict_del (op_ctx, "count");
-        ret = dict_set_int32 (op_ctx, "count", count);
-        return ret;
-}
-
-void
-_status_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
-                              void *data)
-{
-        char                            new_key[256] = {0,};
-        data_t                          *new_value = 0;
-        glusterd_pr_brick_rsp_conv_t    *rsp_ctx = NULL;
-
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        snprintf (new_key, sizeof (new_key), "brick%d.%s", rsp_ctx->count, key);
-        dict_set (rsp_ctx->dict, new_key, new_value);
-
-        return;
-}
-
-int
-glusterd_status_volume_brick_rsp (glusterd_brickinfo_t *brickinfo,
-                                  dict_t *rsp_dict, dict_t *op_ctx,
-                                  char **op_errstr)
-{
-        int                             ret = 0;
-        glusterd_pr_brick_rsp_conv_t    rsp_ctx = {0};
-        int32_t                         count = 0;
-        int                             index = 0;
+                case GD_OP_COPY_FILE:
+                        ret = glusterd_op_copy_file (dict, op_errstr);
+                        break;
 
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (brickinfo);
+                case GD_OP_SYS_EXEC:
+                        ret = glusterd_op_sys_exec (dict, op_errstr, rsp_dict);
+                        break;
 
-        ret = dict_get_int32 (op_ctx, "count", &count);
-        if (ret) {
-                count = 0;
-        } else {
-                count++;
+                default:
+                        gf_log (this->name, GF_LOG_ERROR, "Unknown op %s",
+                                gd_op_list[op]);
+                        break;
         }
-        ret = dict_get_int32 (rsp_dict, "index", &index);
-        if (ret)
-                goto out;
-        dict_del (rsp_dict, "index");
 
-        rsp_ctx.count = index;
-        rsp_ctx.dict = op_ctx;
-        dict_foreach (rsp_dict, _status_volume_add_brick_rsp, &rsp_ctx);
-        ret = dict_set_int32 (op_ctx, "count", count);
+        if (ret == 0)
+            glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_POST);
 
-out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-int32_t
-glusterd_handle_brick_rsp (glusterd_brickinfo_t *brickinfo,
-                           glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
-                           char **op_errstr)
-{
-        int     ret = 0;
-
-        GF_ASSERT (op_errstr);
-
-        switch (op) {
-        case GD_OP_PROFILE_VOLUME:
-                ret = glusterd_profile_volume_brick_rsp (brickinfo, rsp_dict,
-                                                         op_ctx, op_errstr);
-                break;
-        case GD_OP_STATUS_VOLUME:
-                ret = glusterd_status_volume_brick_rsp (brickinfo, rsp_dict,
-                                                        op_ctx, op_errstr);
-                break;
-
-        default:
-                break;
-        }
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
 
 static int
-glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr,
+                                    struct list_head *selected)
 {
         int                                     ret = 0;
         int                                     flags = 0;
@@ -2614,15 +4587,17 @@ glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr)
         glusterd_brickinfo_t                    *brickinfo = NULL;
         glusterd_pending_node_t                 *pending_node = NULL;
 
-
         ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
         if (ret)
                 goto out;
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
                 goto out;
+        }
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 if (glusterd_is_brick_started (brickinfo)) {
@@ -2634,7 +4609,7 @@ glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr)
                         } else {
                                 pending_node->node = brickinfo;
                                 pending_node->type = GD_NODE_BRICK;
-                                list_add_tail (&pending_node->list, &opinfo.pending_bricks);
+                                list_add_tail (&pending_node->list, selected);
                                 pending_node = NULL;
                         }
                 }
@@ -2645,7 +4620,8 @@ out:
 }
 
 static int
-glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
+                                     struct list_head *selected)
 {
         int                                     ret = -1;
         char                                    *volname = NULL;
@@ -2658,6 +4634,8 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr)
         glusterd_pending_node_t                 *pending_node = NULL;
         int32_t                                 force = 0;
 
+
+
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
@@ -2694,8 +4672,7 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr)
                 }
 
                 ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo,
-                                                              GF_PATH_COMPLETE);
+                                                              &brickinfo);
                 if (ret)
                         goto out;
                 if (glusterd_is_brick_started (brickinfo)) {
@@ -2707,7 +4684,7 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr)
                         } else {
                                 pending_node->node = brickinfo;
                                 pending_node->type = GD_NODE_BRICK;
-                                list_add_tail (&pending_node->list, &opinfo.pending_bricks);
+                                list_add_tail (&pending_node->list, selected);
                                 pending_node = NULL;
                         }
                 }
@@ -2719,7 +4696,8 @@ out:
 }
 
 static int
-glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr,
+                                       struct list_head *selected)
 {
         int                                     ret = -1;
         char                                    *volname = NULL;
@@ -2732,11 +4710,14 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
         glusterd_pending_node_t                 *pending_node = NULL;
         char                                    *brick = NULL;
 
+
+
         this = THIS;
         GF_ASSERT (this);
         priv = this->private;
         GF_ASSERT (priv);
 
+
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
                 gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
@@ -2765,6 +4746,29 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
                 goto out;
                 break;
         case GF_CLI_STATS_INFO:
+                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
+                if (ret) {
+                        if (!glusterd_is_nodesvc_online ("nfs")) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "NFS server"
+                                        " is not running");
+                                goto out;
+                        }
+                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                  gf_gld_mt_pending_node_t);
+                        if (!pending_node) {
+                                ret = -1;
+                                goto out;
+                        }
+                        pending_node->node = priv->nfs;
+                        pending_node->type = GD_NODE_NFS;
+                        list_add_tail (&pending_node->list, selected);
+                        pending_node = NULL;
+
+                        ret = 0;
+                        goto out;
+
+                }
                 list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                         if (glusterd_is_brick_started (brickinfo)) {
                                 pending_node = GF_CALLOC (1, sizeof (*pending_node),
@@ -2776,7 +4780,7 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
                                         pending_node->node = brickinfo;
                                         pending_node->type = GD_NODE_BRICK;
                                         list_add_tail (&pending_node->list,
-                                                       &opinfo.pending_bricks);
+                                                       selected);
                                         pending_node = NULL;
                                 }
                         }
@@ -2784,11 +4788,33 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
                 break;
 
         case GF_CLI_STATS_TOP:
+                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
+                if (ret) {
+                        if (!glusterd_is_nodesvc_online ("nfs")) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "NFS server"
+                                        " is not running");
+                                goto out;
+                        }
+                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                  gf_gld_mt_pending_node_t);
+                        if (!pending_node) {
+                                ret = -1;
+                                goto out;
+                        }
+                        pending_node->node = priv->nfs;
+                        pending_node->type = GD_NODE_NFS;
+                        list_add_tail (&pending_node->list, selected);
+                        pending_node = NULL;
+
+                        ret = 0;
+                        goto out;
+
+                }
                 ret = dict_get_str (dict, "brick", &brick);
                 if (!ret) {
                         ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                                      &brickinfo,
-                                                                      GF_PATH_COMPLETE);
+                                                                      &brickinfo);
                         if (ret)
                                 goto out;
 
@@ -2804,7 +4830,7 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
                                 pending_node->node = brickinfo;
                                 pending_node->type = GD_NODE_BRICK;
                                 list_add_tail (&pending_node->list,
-                                               &opinfo.pending_bricks);
+                                               selected);
                                 pending_node = NULL;
                                 goto out;
                         }
@@ -2821,7 +4847,7 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
                                         pending_node->node = brickinfo;
                                         pending_node->type = GD_NODE_BRICK;
                                         list_add_tail (&pending_node->list,
-                                                       &opinfo.pending_bricks);
+                                                       selected);
                                         pending_node = NULL;
                                 }
                         }
@@ -2851,31 +4877,305 @@ _add_rxlator_to_dict (dict_t *dict, char *volname, int index, int count)
         char    key[128]        = {0,};
         char    *xname          = NULL;
 
-        snprintf (key, sizeof (key), "heal-%d", count);
+        snprintf (key, sizeof (key), "xl-%d", count);
         ret = gf_asprintf (&xname, "%s-replicate-%d", volname, index);
         if (ret == -1)
                 goto out;
 
         ret = dict_set_dynstr (dict, key, xname);
+        if (ret)
+                goto out;
+
+        ret = dict_set_int32 (dict, xname, index);
+out:
+        return ret;
+}
+
+int
+get_replica_index_for_per_replica_cmd (glusterd_volinfo_t *volinfo,
+                                       dict_t *dict) {
+        int                     ret = 0;
+        char                    *hostname = NULL;
+        char                    *path = NULL;
+        int                     index = 0;
+        glusterd_brickinfo_t   *brickinfo = NULL;
+        int                     cmd_replica_index = -1;
+        int                     replica_count = -1;
+
+
+        if (!dict)  {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "per-replica-cmd-hostname", &hostname);
+        if (ret)
+                goto out;
+        ret = dict_get_str (dict, "per-replica-cmd-path", &path);
+        if (ret)
+                goto out;
+
+        replica_count = volinfo->replica_count;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+                if (!strcmp (brickinfo->path,  path) &&
+                    !strcmp (brickinfo->hostname, hostname)) {
+                        cmd_replica_index = index/(replica_count);
+                        goto out;
+                }
+                index++;
+        }
+
+
+out:
+        if (ret)
+                cmd_replica_index = -1;
+
+        return cmd_replica_index;
+}
+
+int
+_select_rxlators_with_local_bricks (xlator_t *this, glusterd_volinfo_t *volinfo,
+                                    dict_t *dict, cli_cmd_type type)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_conf_t         *priv   = NULL;
+        int                     index = 0;
+        int                     rxlator_count = 0;
+        int                     replica_count = 0;
+        gf_boolean_t            add     = _gf_false;
+        int                     ret = 0;
+        int                     cmd_replica_index = -1;
+
+        priv = this->private;
+        replica_count = volinfo->replica_count;
+
+        if (type == PER_REPLICA) {
+
+                cmd_replica_index = get_replica_index_for_per_replica_cmd
+                                    (volinfo, dict);
+                if (cmd_replica_index == -1) {
+                        ret = -1;
+                        goto err;
+                }
+        }
+
+        index = 1;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+
+                switch (type) {
+                        case ALL_REPLICA:
+                                if (!uuid_compare (MY_UUID, brickinfo->uuid))
+                                        add = _gf_true;
+                                break;
+                        case PER_REPLICA:
+                                if (!uuid_compare (MY_UUID, brickinfo->uuid) &&
+                                 ((index-1)/replica_count == cmd_replica_index))
+
+                                                add = _gf_true;
+                                break;
+                }
+
+                if (index % replica_count == 0) {
+                        if (add) {
+                                _add_rxlator_to_dict (dict, volinfo->volname,
+                                                      (index-1)/replica_count,
+                                                      rxlator_count);
+                                rxlator_count++;
+                        }
+                        add = _gf_false;
+                }
+
+                index++;
+        }
+err:
+        if (ret)
+                rxlator_count = -1;
+
+        return rxlator_count;
+}
+
+int
+_select_rxlators_for_full_self_heal (xlator_t *this,
+                                     glusterd_volinfo_t *volinfo,
+                                     dict_t *dict)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_conf_t         *priv   = NULL;
+        int                     index = 1;
+        int                     rxlator_count = 0;
+        int                     replica_count = 0;
+        uuid_t                  candidate = {0};
+
+        priv = this->private;
+        replica_count = volinfo->replica_count;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+
+                if (uuid_compare (brickinfo->uuid, candidate) > 0)
+                        uuid_copy (candidate, brickinfo->uuid);
+
+                if (index % replica_count == 0) {
+                        if (!uuid_compare (MY_UUID, candidate)) {
+                                _add_rxlator_to_dict (dict, volinfo->volname,
+                                                      (index-1)/replica_count,
+                                                      rxlator_count);
+                                rxlator_count++;
+                        }
+                        uuid_clear (candidate);
+                }
+
+                index++;
+        }
+        return rxlator_count;
+}
+
+
+static int
+glusterd_bricks_select_snap (dict_t *dict, char **op_errstr,
+                             struct list_head *selected)
+{
+        int                        ret           = -1;
+        glusterd_conf_t            *priv         = NULL;
+        xlator_t                   *this         = NULL;
+        glusterd_pending_node_t    *pending_node = NULL;
+        glusterd_volinfo_t         *volinfo      = NULL;
+        char                       *volname      = NULL;
+        glusterd_brickinfo_t       *brickinfo    = NULL;
+        int                         brick_index  = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get"
+                        " volname");
+                goto out;
+        }
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret)
+                goto out;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                brick_index++;
+                if (uuid_compare (brickinfo->uuid, MY_UUID) ||
+                    !glusterd_is_brick_started (brickinfo)) {
+                        continue;
+                }
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
+                pending_node->node = brickinfo;
+                pending_node->type = GD_NODE_BRICK;
+                pending_node->index = brick_index;
+                list_add_tail (&pending_node->list,
+                               selected);
+                pending_node = NULL;
+        }
+
+        ret = 0;
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning ret %d", ret);
+        return ret;
+}
+
+static int
+fill_shd_status_for_local_bricks (dict_t *dict, glusterd_volinfo_t *volinfo,
+                                  cli_cmd_type type, dict_t *req_dict)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        char                    msg[1024] = {0,};
+        char                    key[1024]  = {0,};
+        char                    value[1024] = {0,};
+        int                     index = 0;
+        int                     ret = 0;
+        xlator_t               *this = NULL;
+        int                     cmd_replica_index = -1;
+
+        this = THIS;
+        snprintf (msg, sizeof (msg), "self-heal-daemon is not running on");
+
+        if (type == PER_REPLICA) {
+                cmd_replica_index = get_replica_index_for_per_replica_cmd
+                                    (volinfo, req_dict);
+                if (cmd_replica_index == -1) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Could not find the "
+                                "replica index for per replica type command");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+
+                if (uuid_compare (MY_UUID, brickinfo->uuid)) {
+                        index++;
+                        continue;
+                }
+
+                if (type == PER_REPLICA) {
+                      if (cmd_replica_index != (index/volinfo->replica_count)) {
+                              index++;
+                              continue;
+                        }
+
+                }
+                snprintf (key, sizeof (key), "%d-status",index);
+                snprintf (value, sizeof (value), "%s %s",msg,
+                          uuid_utoa(MY_UUID));
+                ret = dict_set_dynstr (dict, key, gf_strdup(value));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to"
+                                "set the dictionary for shd status msg");
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "%d-shd-status",index);
+                ret = dict_set_str (dict, key, "off");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to"
+                                " set dictionary for shd status msg");
+                        goto out;
+                }
+
+                index++;
+        }
+
 out:
         return ret;
+
 }
 
+
 static int
-glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr,
+                                    struct list_head *selected,
+                                    dict_t *rsp_dict)
 {
         int                                     ret = -1;
         char                                    *volname = NULL;
         glusterd_conf_t                         *priv = NULL;
         glusterd_volinfo_t                      *volinfo = NULL;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
         xlator_t                                *this = NULL;
         char                                    msg[2048] = {0,};
-        int                                     replica_count = 0;
-        int                                     index = 1;
-        int                                     rxlator_count = 0;
-        uuid_t                                  candidate = {0};
         glusterd_pending_node_t                 *pending_node = NULL;
+        gf_xl_afr_op_t                          heal_op = GF_AFR_OP_INVALID;
+        int                                     rxlator_count = 0;
 
         this = THIS;
         GF_ASSERT (this);
@@ -2898,48 +5198,102 @@ glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        replica_count = volinfo->replica_count;
+        ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
+        if (ret || (heal_op == GF_AFR_OP_INVALID)) {
+                gf_log ("glusterd", GF_LOG_ERROR, "heal op invalid");
+                goto out;
+        }
 
-        index = 1;
-        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
+        switch (heal_op) {
+                case GF_AFR_OP_INDEX_SUMMARY:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        if (!rsp_dict) {
+                                gf_log (this->name, GF_LOG_ERROR, "Received "
+                                        "empty ctx.");
+                                goto out;
+                        }
 
-                if (uuid_compare (brickinfo->uuid, candidate) > 0)
-                        uuid_copy (candidate, brickinfo->uuid);
+                        ret = fill_shd_status_for_local_bricks (rsp_dict,
+                                                                volinfo,
+                                                                ALL_REPLICA,
+                                                                dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "fill the shd status for the local "
+                                        "bricks");
+                        goto out;
 
-                if (index % replica_count == 0) {
-                        if (!uuid_compare (priv->uuid, candidate)) {
-                                _add_rxlator_to_dict (dict, volname,
-                                                      (index-1)/replica_count,
-                                                      rxlator_count);
-                                rxlator_count++;
+                }
+                break;
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        if (!rsp_dict) {
+                                gf_log (this->name, GF_LOG_ERROR, "Received "
+                                        "empty ctx.");
+                                goto out;
                         }
-                        uuid_clear (candidate);
+                        ret = fill_shd_status_for_local_bricks (rsp_dict,
+                                                                volinfo,
+                                                                PER_REPLICA,
+                                                                dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "fill the shd status for the local"
+                                        " bricks.");
+                        goto out;
+
                 }
+                break;
+                default:
+                        break;
+        }
 
-                index++;
+
+        switch (heal_op) {
+        case GF_AFR_OP_HEAL_FULL:
+                rxlator_count = _select_rxlators_for_full_self_heal (this,
+                                                                     volinfo,
+                                                                     dict);
+                break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                rxlator_count = _select_rxlators_with_local_bricks (this,
+                                                                   volinfo,
+                                                                   dict,
+                                                                   PER_REPLICA);
+                break;
+        default:
+                rxlator_count = _select_rxlators_with_local_bricks (this,
+                                                                    volinfo,
+                                                                    dict,
+                                                                   ALL_REPLICA);
+                break;
         }
+        if (!rxlator_count)
+                goto out;
+        if (rxlator_count == -1){
+                gf_log (this->name, GF_LOG_ERROR, "Could not determine the"
+                        "translator count");
+                ret = -1;
+                goto out;
+        }
+
         ret = dict_set_int32 (dict, "count", rxlator_count);
         if (ret)
                 goto out;
 
-        if (rxlator_count) {
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                } else {
-                        pending_node->node = priv->shd;
-                        pending_node->type = GD_NODE_SHD;
-                        list_add_tail (&pending_node->list,
-                                       &opinfo.pending_bricks);
-                        pending_node = NULL;
-                }
+        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                  gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+                ret = -1;
+                goto out;
+        } else {
+                pending_node->node = priv->shd;
+                pending_node->type = GD_NODE_SHD;
+                list_add_tail (&pending_node->list, selected);
+                pending_node = NULL;
         }
 
-
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning ret %d", ret);
         return ret;
@@ -2947,7 +5301,57 @@ out:
 }
 
 static int
-glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
+                                         struct list_head *selected)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        xlator_t                                *this = NULL;
+        char                                    msg[2048] = {0,};
+        glusterd_pending_node_t                 *pending_node = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Volume %s does not exist",
+                          volname);
+
+                *op_errstr = gf_strdup (msg);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                  gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+                ret = -1;
+                goto out;
+        } else {
+                pending_node->node = volinfo;
+                pending_node->type = GD_NODE_REBALANCE;
+                list_add_tail (&pending_node->list,
+                               &opinfo.pending_bricks);
+                pending_node = NULL;
+        }
+
+out:
+        return ret;
+}
+
+
+
+
+static int
+glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr,
+                                      struct list_head *selected)
 {
         int                     ret = -1;
         int                     cmd = 0;
@@ -2969,7 +5373,7 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
 
         ret = dict_get_int32 (dict, "cmd", &cmd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to get status type");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get status type");
                 goto out;
         }
 
@@ -2982,13 +5386,15 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
         case GF_CLI_STATUS_INODE:
         case GF_CLI_STATUS_FD:
         case GF_CLI_STATUS_CALLPOOL:
+        case GF_CLI_STATUS_NFS:
+        case GF_CLI_STATUS_SHD:
                 break;
         default:
                 goto out;
         }
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to get volname");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volname");
                 goto out;
         }
         ret = glusterd_volinfo_find (volname, &volinfo);
@@ -2999,18 +5405,17 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
         if ( (cmd & GF_CLI_STATUS_BRICK) != 0) {
                 ret = dict_get_str (dict, "brick", &brickname);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Unable to get brick");
                         goto out;
                 }
                 ret = glusterd_volume_brickinfo_get_by_brick (brickname,
                                                               volinfo,
-                                                              &brickinfo,
-                                                              GF_PATH_COMPLETE);
+                                                              &brickinfo);
                 if (ret)
                         goto out;
 
-                if (uuid_compare (brickinfo->uuid, priv->uuid)||
+                if (uuid_compare (brickinfo->uuid, MY_UUID)||
                     !glusterd_is_brick_started (brickinfo))
                         goto out;
 
@@ -3023,13 +5428,51 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
                 pending_node->node = brickinfo;
                 pending_node->type = GD_NODE_BRICK;
                 pending_node->index = 0;
-                list_add_tail (&pending_node->list, &opinfo.pending_bricks);
+                list_add_tail (&pending_node->list, selected);
+
+                ret = 0;
+        } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+                if (!glusterd_is_nodesvc_online ("nfs")) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "NFS server is not running");
+                        goto out;
+                }
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
+                pending_node->node = priv->nfs;
+                pending_node->type = GD_NODE_NFS;
+                pending_node->index = 0;
+                list_add_tail (&pending_node->list, selected);
+
+                ret = 0;
+        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Self-heal daemon is not running");
+                        goto out;
+                }
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
+                pending_node->node = priv->shd;
+                pending_node->type = GD_NODE_SHD;
+                pending_node->index = 0;
+                list_add_tail (&pending_node->list, selected);
 
                 ret = 0;
         } else {
                 list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                         brick_index++;
-                        if (uuid_compare (brickinfo->uuid, priv->uuid) ||
+                        if (uuid_compare (brickinfo->uuid, MY_UUID) ||
                             !glusterd_is_brick_started (brickinfo)) {
                                 continue;
                         }
@@ -3044,8 +5487,7 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
                         pending_node->node = brickinfo;
                         pending_node->type = GD_NODE_BRICK;
                         pending_node->index = brick_index;
-                        list_add_tail (&pending_node->list,
-                                       &opinfo.pending_bricks);
+                        list_add_tail (&pending_node->list, selected);
                         pending_node = NULL;
                 }
         }
@@ -3062,6 +5504,7 @@ glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
         xlator_t                        *this = NULL;
         glusterd_op_t                   op = GD_OP_NONE;
         glusterd_req_ctx_t              *req_ctx = NULL;
+        char                            *op_errstr = NULL;
 
         this = THIS;
         priv = this->private;
@@ -3073,10 +5516,18 @@ glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
                                      gf_gld_mt_op_allack_ctx_t);
                 op = glusterd_op_get_op ();
                 req_ctx->op = op;
-                uuid_copy (req_ctx->uuid, priv->uuid);
-                ret = glusterd_op_build_payload (&req_ctx->dict);
-                if (ret)//TODO:what to do??
+                uuid_copy (req_ctx->uuid, MY_UUID);
+                ret = glusterd_op_build_payload (&req_ctx->dict, &op_errstr,
+                                                 NULL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                                gd_op_list[op]);
+                        if (op_errstr == NULL)
+                                gf_asprintf (&op_errstr,
+                                             OPERRSTR_BUILD_PAYLOAD);
+                        opinfo.op_errstr = op_errstr;
                         goto out;
+                }
         }
 
         proc = &priv->gfs_mgmt->proctable[GLUSTERD_BRICK_OP];
@@ -3088,11 +5539,12 @@ glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
 
         if (!opinfo.pending_count && !opinfo.brick_pending_count) {
                 glusterd_clear_pending_nodes (&opinfo.pending_bricks);
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, req_ctx);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                                   &event->txn_id, req_ctx);
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
@@ -3105,10 +5557,14 @@ glusterd_op_ac_rcvd_brick_op_acc (glusterd_op_sm_event_t *event, void *ctx)
         glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
         char                        *op_errstr = NULL;
         glusterd_op_t               op = GD_OP_NONE;
+        gd_node_type                type = GD_NODE_NONE;
         dict_t                      *op_ctx = NULL;
         glusterd_req_ctx_t          *req_ctx = NULL;
         void                        *pending_entry = NULL;
+        xlator_t                    *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (event);
         GF_ASSERT (ctx);
         ev_ctx = ctx;
@@ -3119,11 +5575,12 @@ glusterd_op_ac_rcvd_brick_op_acc (glusterd_op_sm_event_t *event, void *ctx)
         op = req_ctx->op;
         op_ctx = glusterd_op_get_ctx ();
         pending_entry = ev_ctx->pending_node->node;
+        type = ev_ctx->pending_node->type;
 
         ret = glusterd_remove_pending_entry (&opinfo.pending_bricks,
                                              pending_entry);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "unknown response received ");
+                gf_log (this->name, GF_LOG_ERROR, "unknown response received ");
                 ret = -1;
                 goto out;
         }
@@ -3131,25 +5588,27 @@ glusterd_op_ac_rcvd_brick_op_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.brick_pending_count > 0)
                 opinfo.brick_pending_count--;
 
-        glusterd_handle_brick_rsp (pending_entry, op, ev_ctx->rsp_dict,
-                                   op_ctx, &op_errstr);
+        glusterd_handle_node_rsp (req_ctx->dict, pending_entry, op, ev_ctx->rsp_dict,
+                                  op_ctx, &op_errstr, type);
 
         if (opinfo.brick_pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, ev_ctx->commit_ctx);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                           ev_ctx->commit_ctx);
 
 out:
         if (ev_ctx->rsp_dict)
                 dict_unref (ev_ctx->rsp_dict);
         GF_FREE (ev_ctx);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
 
 int32_t
-glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr)
+glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           struct list_head *selected, dict_t *rsp_dict)
 {
         int     ret = 0;
 
@@ -3160,30 +5619,42 @@ glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr)
 
         switch (op) {
         case GD_OP_STOP_VOLUME:
-                ret = glusterd_bricks_select_stop_volume (dict, op_errstr);
+                ret = glusterd_bricks_select_stop_volume (dict, op_errstr,
+                                                          selected);
                 break;
 
         case GD_OP_REMOVE_BRICK:
-                ret = glusterd_bricks_select_remove_brick (dict, op_errstr);
+                ret = glusterd_bricks_select_remove_brick (dict, op_errstr,
+                                                           selected);
                 break;
 
         case GD_OP_PROFILE_VOLUME:
-                ret = glusterd_bricks_select_profile_volume (dict, op_errstr);
+                ret = glusterd_bricks_select_profile_volume (dict, op_errstr,
+                                                             selected);
                 break;
 
         case GD_OP_HEAL_VOLUME:
-                ret = glusterd_bricks_select_heal_volume (dict, op_errstr);
+                ret = glusterd_bricks_select_heal_volume (dict, op_errstr,
+                                                          selected, rsp_dict);
                 break;
 
         case GD_OP_STATUS_VOLUME:
-                ret = glusterd_bricks_select_status_volume (dict, op_errstr);
+                ret = glusterd_bricks_select_status_volume (dict, op_errstr,
+                                                            selected);
                 break;
 
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = glusterd_bricks_select_rebalance_volume (dict, op_errstr,
+                                                               selected);
+                break;
+        case GD_OP_SNAP:
+                ret = glusterd_bricks_select_snap (dict, op_errstr, selected);
+                break;
         default:
                 break;
          }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -3500,7 +5971,7 @@ glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
 
 int
 glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
-                             void *ctx)
+                             uuid_t *txn_id, void *ctx)
 {
         int32_t                 ret = -1;
         glusterd_op_sm_event_t  *event = NULL;
@@ -3515,7 +5986,10 @@ glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
 
         event->ctx = ctx;
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Enqueue event: '%s'",
+        if (txn_id)
+                uuid_copy (event->txn_id, *txn_id);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Enqueue event: '%s'",
                 glusterd_op_sm_event_name_get (event->event));
         list_add_tail (&event->list, &gd_op_sm_queue);
 
@@ -3574,9 +6048,14 @@ glusterd_op_sm ()
         glusterd_op_sm_ac_fn            handler = NULL;
         glusterd_op_sm_t                *state = NULL;
         glusterd_op_sm_event_type_t     event_type = GD_OP_EVENT_NONE;
+        xlator_t                        *this = NULL;
+        glusterd_op_info_t              txn_op_info;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         if ((lock_err = pthread_mutex_trylock (&gd_op_sm_lock))) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "lock failed due to %s",
+                gf_log (this->name, GF_LOG_ERROR, "lock failed due to %s",
                         strerror (lock_err));
                 goto lock_failed;
         }
@@ -3587,9 +6066,24 @@ glusterd_op_sm ()
 
                         list_del_init (&event->list);
                         event_type = event->event;
-                        gf_log ("", GF_LOG_DEBUG, "Dequeued event of type: '%s'",
+                        gf_log (this->name, GF_LOG_DEBUG, "Dequeued event of "
+                                "type: '%s'",
                                 glusterd_op_sm_event_name_get(event_type));
 
+                        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s",
+                                uuid_utoa (event->txn_id));
+
+                        ret = glusterd_get_txn_opinfo (&event->txn_id,
+                                                       &txn_op_info);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to get transaction's opinfo");
+                                glusterd_destroy_op_event_ctx (event);
+                                GF_FREE (event);
+                                continue;
+                        } else
+                                opinfo = txn_op_info;
+
                         state = glusterd_op_state_table[opinfo.state.state];
 
                         GF_ASSERT (state);
@@ -3600,7 +6094,7 @@ glusterd_op_sm ()
                         ret = handler (event, event->ctx);
 
                         if (ret) {
-                                gf_log ("glusterd", GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "handler returned: %d", ret);
                                 glusterd_destroy_op_event_ctx (event);
                                 GF_FREE (event);
@@ -3611,7 +6105,7 @@ glusterd_op_sm ()
                                                                 event_type);
 
                         if (ret) {
-                                gf_log ("glusterd", GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "Unable to transition"
                                         "state from '%s' to '%s'",
                          glusterd_op_sm_state_name_get(opinfo.state.state),
@@ -3620,8 +6114,27 @@ glusterd_op_sm ()
                                 return ret;
                         }
 
+                        if ((state[event_type].next_state ==
+                             GD_OP_STATE_DEFAULT) &&
+                           (event_type == GD_OP_EVENT_UNLOCK)) {
+                                /* Clearing the transaction opinfo */
+                                ret = glusterd_clear_txn_opinfo(&event->txn_id);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Unable to clear "
+                                                "transaction's opinfo");
+                        } else {
+                                ret = glusterd_set_txn_opinfo (&event->txn_id,
+                                                               &opinfo);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Unable to set "
+                                                "transaction's opinfo");
+                        }
+
                         glusterd_destroy_op_event_ctx (event);
                         GF_FREE (event);
+
                 }
         }
 
@@ -3675,48 +6188,6 @@ glusterd_op_clear_op (glusterd_op_t op)
 }
 
 int32_t
-glusterd_op_init_ctx (glusterd_op_t op)
-{
-        int     ret = 0;
-        dict_t *dict = NULL;
-
-        GF_ASSERT (GD_OP_NONE < op && op < GD_OP_MAX);
-
-        if (_gf_false == glusterd_need_brick_op (op)) {
-                gf_log ("", GF_LOG_DEBUG, "Received op: %d, returning", op);
-                goto out;
-        }
-        dict = dict_new ();
-        if (dict == NULL) {
-                ret = -1;
-                goto out;
-        }
-        ret = glusterd_op_set_ctx (dict);
-        if (ret)
-                goto out;
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-
-
-int32_t
-glusterd_op_fini_ctx ()
-{
-        dict_t *dict = NULL;
-
-        dict = glusterd_op_get_ctx ();
-        if (dict)
-                dict_unref (dict);
-
-        glusterd_op_reset_ctx ();
-        return 0;
-}
-
-
-
-int32_t
 glusterd_op_free_ctx (glusterd_op_t op, void *ctx)
 {
 
@@ -3740,6 +6211,8 @@ glusterd_op_free_ctx (glusterd_op_t op, void *ctx)
                 case GD_OP_REBALANCE:
                 case GD_OP_HEAL_VOLUME:
                 case GD_OP_STATEDUMP_VOLUME:
+                case GD_OP_CLEARLOCKS_VOLUME:
+                case GD_OP_DEFRAG_BRICK_VOLUME:
                         dict_unref (ctx);
                         break;
                 default:
@@ -3768,4 +6241,3 @@ glusterd_op_sm_init ()
         pthread_mutex_init (&gd_op_sm_lock, NULL);
         return 0;
 }
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
index 12aa139f5..cf57b78e0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_OP_SM_H_
 #define _GLUSTERD_OP_SM_H_
 
@@ -25,8 +15,8 @@
 #include "config.h"
 #endif
 
-#ifndef GSYNC_CONF
-#define GSYNC_CONF GEOREP"/gsyncd.conf"
+#ifndef GSYNC_CONF_TEMPLATE
+#define GSYNC_CONF_TEMPLATE GEOREP"/gsyncd_template.conf"
 #endif
 
 #include <pthread.h>
@@ -42,6 +32,8 @@
 #include "protocol-common.h"
 
 #define GD_VOLUME_NAME_MAX 256
+#define GD_OP_PROTECTED    (0x02)
+#define GD_OP_UNPROTECTED  (0x04)
 
 typedef enum glusterd_op_sm_state_ {
         GD_OP_STATE_DEFAULT = 0,
@@ -85,6 +77,7 @@ struct glusterd_op_sm_event_ {
         struct list_head                list;
         void                            *ctx;
         glusterd_op_sm_event_type_t     event;
+        uuid_t                          txn_id;
 };
 
 typedef struct glusterd_op_sm_event_ glusterd_op_sm_event_t;
@@ -127,6 +120,7 @@ typedef struct glusterd_op_log_filename_ctx_ glusterd_op_log_filename_ctx_t;
 
 struct glusterd_op_lock_ctx_ {
         uuid_t                  uuid;
+        dict_t                 *dict;
         rpcsvc_request_t        *req;
 };
 
@@ -154,21 +148,36 @@ typedef struct glusterd_pr_brick_rsp_conv_t {
         dict_t *dict;
 } glusterd_pr_brick_rsp_conv_t;
 
+typedef struct glusterd_heal_rsp_conv_ {
+        dict_t *dict;
+        glusterd_volinfo_t *volinfo;
+        xlator_t *this;
+} glusterd_heal_rsp_conv_t;
+
 typedef struct glusterd_status_rsp_conv_ {
         int count;
+        int brick_index_max;
+        int other_count;
         dict_t *dict;
 } glusterd_status_rsp_conv_t;
 
 typedef struct glusterd_gsync_status_temp {
         dict_t *rsp_dict;
         glusterd_volinfo_t *volinfo;
+        char *node;
 }glusterd_gsync_status_temp_t;
+
+typedef enum cli_cmd_type_ {
+        PER_REPLICA,
+        ALL_REPLICA,
+ } cli_cmd_type;
+
 int
 glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
                           glusterd_op_sm_event_t **new_event);
 int
 glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
-                             void *ctx);
+                             uuid_t *txn_id, void *ctx);
 
 int
 glusterd_op_sm_init ();
@@ -183,7 +192,7 @@ int32_t
 glusterd_op_set_op (glusterd_op_t op);
 
 int
-glusterd_op_build_payload (dict_t **req);
+glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx);
 
 int32_t
 glusterd_op_stage_validate (glusterd_op_t op, dict_t *req, char **op_errstr,
@@ -193,6 +202,13 @@ int32_t
 glusterd_op_commit_perform (glusterd_op_t op, dict_t *req, char **op_errstr,
                             dict_t* dict);
 
+int32_t
+glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                       char *err_str, size_t err_len);
+
+int32_t
+glusterd_op_txn_complete ();
+
 void *
 glusterd_op_get_ctx ();
 
@@ -220,8 +236,6 @@ set_xlator_option (dict_t *dict, char *key, char *value);
 
 void
 glusterd_do_replace_brick (void *data);
-int
-glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key, int32_t is_force);
 
 char*
 glusterd_op_sm_state_name_get (int state);
@@ -229,18 +243,25 @@ glusterd_op_sm_state_name_get (int state);
 char*
 glusterd_op_sm_event_name_get (int event);
 int32_t
-glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr);
+glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           struct list_head *selected, dict_t *rsp_dict);
 int
 glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickinfo,
                                  gd1_mgmt_brick_op_req **req, dict_t *dict);
+int
+glusterd_node_op_build_payload (glusterd_op_t op, gd1_mgmt_brick_op_req **req,
+                               dict_t *dict);
 int32_t
-glusterd_handle_brick_rsp (glusterd_brickinfo_t *brickinfo,
-                           glusterd_op_t op, dict_t *rsp_dict, dict_t *ctx_dict,
-                           char **op_errstr);
-int32_t
-glusterd_op_init_ctx (glusterd_op_t op);
-int32_t
-glusterd_op_fini_ctx ();
+glusterd_handle_brick_rsp (void *pending_entry, glusterd_op_t op,
+                           dict_t *rsp_dict, dict_t *ctx_dict, char **op_errstr,
+                           gd_node_type type);
+
+dict_t*
+glusterd_op_init_commit_rsp_dict (glusterd_op_t op);
+
+void
+glusterd_op_modify_op_ctx (glusterd_op_t op, void *op_ctx);
+
 int32_t
 glusterd_volume_stats_read_perf (char *brick_path, int32_t blk_size,
                 int32_t blk_count, double *throughput, double *time);
@@ -256,14 +277,23 @@ glusterd_are_all_volumes_stopped ();
 int
 glusterd_stop_bricks (glusterd_volinfo_t *volinfo);
 int
-glusterd_get_gsync_status_mst_slv( glusterd_volinfo_t *volinfo,
-                                   char *slave, dict_t *rsp_dict);
+gsync_status (char *master, char *slave, char *conf_path, int *status);
+
 int
-gsync_status (char *master, char *slave, int *status);
+glusterd_check_gsync_running (glusterd_volinfo_t *volinfo, gf_boolean_t *flag);
 
 int
-glusterd_gsync_get_param_file (char *prmfile, const char *ext, char *master,
-                                char *slave, char *gl_workdir);
+glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
+                                 dict_t *op_ctx);
 int
-glusterd_check_gsync_running (glusterd_volinfo_t *volinfo, gf_boolean_t *flag);
+glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg);
+
+int32_t
+glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+
+int32_t
+glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+
+int32_t
+glusterd_clear_txn_opinfo (uuid_t *txn_id);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c
index 08e7aedca..a153ca1a9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.c
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -62,8 +52,8 @@ pmap_port_isfree (int port)
 }
 
 
-struct pmap_registry *
-pmap_registry_new (void)
+static struct pmap_registry *
+pmap_registry_new (xlator_t *this)
 {
         struct pmap_registry *pmap = NULL;
         int                   i = 0;
@@ -79,8 +69,8 @@ pmap_registry_new (void)
                         pmap->ports[i].type = GF_PMAP_PORT_FOREIGN;
         }
 
-        pmap->base_port = GF_DEFAULT_BASE_PORT + 2;
-        pmap->last_alloc = GF_DEFAULT_BASE_PORT + 2;
+        pmap->base_port = pmap->last_alloc =
+                ((glusterd_conf_t *)(this->private))->base_port;
 
         return pmap;
 }
@@ -96,7 +86,7 @@ pmap_registry_get (xlator_t *this)
 
         pmap = priv->pmap;
         if (!pmap) {
-                pmap = pmap_registry_new ();
+                pmap = pmap_registry_new (this);
                 if (!pmap)
                         return NULL;
                 priv->pmap = pmap;
@@ -228,8 +218,7 @@ pmap_registry_bind (xlator_t *this, int port, const char *brickname,
 
         p = port;
         pmap->ports[p].type = type;
-        if (pmap->ports[p].brickname)
-                free (pmap->ports[p].brickname);
+        free (pmap->ports[p].brickname);
         pmap->ports[p].brickname = strdup (brickname);
         pmap->ports[p].type = type;
         pmap->ports[p].xprt = xprt;
@@ -281,8 +270,7 @@ remove:
         gf_log ("pmap", GF_LOG_INFO, "removing brick %s on port %d",
                 pmap->ports[p].brickname, p);
 
-        if (pmap->ports[p].brickname)
-                free (pmap->ports[p].brickname);
+        free (pmap->ports[p].brickname);
 
         pmap->ports[p].brickname = NULL;
         pmap->ports[p].xprt = NULL;
@@ -292,15 +280,17 @@ out:
 }
 
 int
-gluster_pmap_portbybrick (rpcsvc_request_t *req)
+__gluster_pmap_portbybrick (rpcsvc_request_t *req)
 {
         pmap_port_by_brick_req    args = {0,};
         pmap_port_by_brick_rsp    rsp  = {0,};
         char                     *brick = NULL;
         int                       port = 0;
+        int                       ret = -1;
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_pmap_port_by_brick_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_port_by_brick_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
@@ -317,21 +307,29 @@ gluster_pmap_portbybrick (rpcsvc_request_t *req)
 fail:
         glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                (xdrproc_t)xdr_pmap_port_by_brick_rsp);
-        if (args.brick)
-                free (args.brick);//malloced by xdr
+        free (args.brick);//malloced by xdr
 
         return 0;
 }
 
 
 int
-gluster_pmap_brickbyport (rpcsvc_request_t *req)
+gluster_pmap_portbybrick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_portbybrick);
+}
+
+
+int
+__gluster_pmap_brickbyport (rpcsvc_request_t *req)
 {
         pmap_brick_by_port_req    args = {0,};
         pmap_brick_by_port_rsp    rsp  = {0,};
+        int                       ret = -1;
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_pmap_brick_by_port_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_brick_by_port_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
@@ -349,6 +347,14 @@ fail:
         return 0;
 }
 
+
+int
+gluster_pmap_brickbyport (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_brickbyport);
+}
+
+
 static int
 glusterd_brick_update_signin (glusterd_brickinfo_t *brickinfo,
                               gf_boolean_t value)
@@ -359,14 +365,16 @@ glusterd_brick_update_signin (glusterd_brickinfo_t *brickinfo,
 }
 
 int
-gluster_pmap_signup (rpcsvc_request_t *req)
+__gluster_pmap_signup (rpcsvc_request_t *req)
 {
         pmap_signup_req    args = {0,};
         pmap_signup_rsp    rsp  = {0,};
+        int                ret = -1;
 
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_pmap_signup_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_signup_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
@@ -377,22 +385,28 @@ gluster_pmap_signup (rpcsvc_request_t *req)
 fail:
         glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                (xdrproc_t)xdr_pmap_signup_rsp);
-        if (args.brick)
-                free (args.brick);//malloced by xdr
+        free (args.brick);//malloced by xdr
 
         return 0;
 }
 
 int
-gluster_pmap_signin (rpcsvc_request_t *req)
+gluster_pmap_signup (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_signup);
+}
+
+int
+__gluster_pmap_signin (rpcsvc_request_t *req)
 {
         pmap_signin_req    args = {0,};
         pmap_signin_rsp    rsp  = {0,};
         glusterd_brickinfo_t *brickinfo = NULL;
         int                ret = -1;
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_pmap_signin_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_signin_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
@@ -405,8 +419,7 @@ gluster_pmap_signin (rpcsvc_request_t *req)
 fail:
         glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                (xdrproc_t)xdr_pmap_signin_rsp);
-        if (args.brick)
-                free (args.brick);//malloced by xdr
+        free (args.brick);//malloced by xdr
 
         if (!ret)
                 glusterd_brick_update_signin (brickinfo, _gf_true);
@@ -415,17 +428,24 @@ fail:
 }
 
 
+int
+gluster_pmap_signin (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_signin);
+}
+
 
 int
-gluster_pmap_signout (rpcsvc_request_t *req)
+__gluster_pmap_signout (rpcsvc_request_t *req)
 {
         pmap_signout_req    args = {0,};
         pmap_signout_rsp    rsp  = {0,};
         int                 ret = -1;
         glusterd_brickinfo_t *brickinfo = NULL;
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_pmap_signout_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_signout_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
@@ -439,8 +459,7 @@ gluster_pmap_signout (rpcsvc_request_t *req)
 fail:
         glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                (xdrproc_t)xdr_pmap_signout_rsp);
-        if (args.brick)
-                free (args.brick);//malloced by xdr
+        free (args.brick);//malloced by xdr
 
         if (!ret)
                 glusterd_brick_update_signin (brickinfo, _gf_false);
@@ -448,18 +467,19 @@ fail:
         return 0;
 }
 
+int
+gluster_pmap_signout (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_signout);
+}
+
 rpcsvc_actor_t gluster_pmap_actors[] = {
-        [GF_PMAP_NULL] = {"NULL", GF_PMAP_NULL, NULL, NULL, NULL, 0},
-        [GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", GF_PMAP_PORTBYBRICK,
-                                 gluster_pmap_portbybrick, NULL, NULL, 0},
-        [GF_PMAP_BRICKBYPORT] = {"BRICKBYPORT", GF_PMAP_BRICKBYPORT,
-                                 gluster_pmap_brickbyport, NULL, NULL, 0},
-        [GF_PMAP_SIGNIN] = {"SIGNIN", GF_PMAP_SIGNIN,
-                              gluster_pmap_signin, NULL, NULL, 0},
-        [GF_PMAP_SIGNOUT] = {"SIGNOUT", GF_PMAP_SIGNOUT,
-                              gluster_pmap_signout, NULL, NULL, 0},
-        [GF_PMAP_SIGNUP] = {"SIGNUP", GF_PMAP_SIGNUP,
-                              gluster_pmap_signup, NULL, NULL, 0},
+        [GF_PMAP_NULL]        = {"NULL",        GF_PMAP_NULL,        NULL,                     NULL, 0, DRC_NA},
+        [GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", GF_PMAP_PORTBYBRICK, gluster_pmap_portbybrick, NULL, 0, DRC_NA},
+        [GF_PMAP_BRICKBYPORT] = {"BRICKBYPORT", GF_PMAP_BRICKBYPORT, gluster_pmap_brickbyport, NULL, 0, DRC_NA},
+        [GF_PMAP_SIGNIN]      = {"SIGNIN",      GF_PMAP_SIGNIN,      gluster_pmap_signin,      NULL, 0, DRC_NA},
+        [GF_PMAP_SIGNOUT]     = {"SIGNOUT",     GF_PMAP_SIGNOUT,     gluster_pmap_signout,     NULL, 0, DRC_NA},
+        [GF_PMAP_SIGNUP]      = {"SIGNUP",      GF_PMAP_SIGNUP,      gluster_pmap_signup,      NULL, 0, DRC_NA},
 };
 
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.h b/xlators/mgmt/glusterd/src/glusterd-pmap.h
index 8f1bb4500..6336ee998 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.h
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_PMAP_H_
 #define _GLUSTERD_PMAP_H_
 
@@ -38,6 +28,8 @@
 #include "rpcsvc.h"
 
 
+#define GF_IANA_PRIV_PORTS_START 49152 /* RFC 6335 */
+
 struct pmap_port_status {
         gf_pmap_port_type_t type;
         char  *brickname;
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index 32264b7b6..318267199 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -35,7 +25,7 @@
 #include <sys/wait.h>
 
 int
-glusterd_handle_quota (rpcsvc_request_t *req)
+__glusterd_handle_quota (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -44,11 +34,15 @@ glusterd_handle_quota (rpcsvc_request_t *req)
         char                            operation[256] = {0, };
         char                           *volname = NULL;
         int32_t                         type = 0;
+        char                            msg[2048] = {0,};
+        xlator_t                       *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -62,8 +56,10 @@ glusterd_handle_quota (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                     "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -72,16 +68,18 @@ glusterd_handle_quota (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Unable to get volume name, while"
-                        "handling quota command");
+                snprintf (msg, sizeof (msg), "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name, "
+                        "while handling quota command");
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Unable to get type of cmd. , while"
-                        "handling quota command");
-                goto out;
+                snprintf (msg, sizeof (msg), "Unable to get type of command");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get type of cmd, "
+                        "while handling quota command");
+               goto out;
         }
 
         switch (type) {
@@ -101,25 +99,25 @@ glusterd_handle_quota (rpcsvc_request_t *req)
                 strncpy (operation, "remove", sizeof (operation));
                 break;
         }
-        gf_cmd_log ("volume quota", " %s command on %s", operation, volname);
-        ret = glusterd_op_begin (req, GD_OP_QUOTA, dict);
-        gf_cmd_log ("volume quota", " %s command on %s %s", operation,volname,
-                    (ret != 0)? "FAILED" : "SUCCEEDED");
+        ret = glusterd_op_begin_synctask (req, GD_OP_QUOTA, dict);
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, msg);
         }
 
         return ret;
 }
 
+int
+glusterd_handle_quota (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_quota);
+}
+
 int32_t
 glusterd_check_if_quota_trans_enabled (glusterd_volinfo_t *volinfo)
 {
@@ -237,8 +235,7 @@ _glusterd_quota_remove_limits (char **quota_limits, char *path,
         }
 
 out:
-        if (limits)
-                GF_FREE (limits);
+        GF_FREE (limits);
 
         return ret;
 }
@@ -246,11 +243,11 @@ out:
 int32_t
 glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
 {
-        int32_t   ret = 0;
-        pid_t     pid;
-        char      mountdir [] = "/tmp/mntXXXXXX";
-        runner_t  runner = {0,};
-        int       status = 0;
+        pid_t                      pid;
+        int32_t                    ret              = 0;
+        int                        status           = 0;
+        char                       mountdir[]       = "/tmp/mntXXXXXX";
+        runner_t                   runner           = {0};
 
         if (mkdtemp (mountdir) == NULL) {
                 gf_log ("glusterd", GF_LOG_DEBUG,
@@ -260,12 +257,15 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
         }
 
         runinit (&runner);
-        runner_add_args (&runner, SBIN_DIR"/glusterfs", "-s",
-                         "localhost", "--volfile-id", volname, "-l",
-                         DEFAULT_LOG_FILE_DIRECTORY"/quota-crawl.log",
+        runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                         "-s", "localhost",
+                         "--volfile-id", volname,
+                         "-l", DEFAULT_LOG_FILE_DIRECTORY"/quota-crawl.log",
                          mountdir, NULL);
 
+        synclock_unlock (&priv->big_lock);
         ret = runner_run_reuse (&runner);
+        synclock_lock (&priv->big_lock);
         if (ret == -1) {
                 runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
                 runner_end (&runner);
@@ -384,18 +384,17 @@ glusterd_quota_get_limit_usages (glusterd_conf_t *priv,
                                  glusterd_volinfo_t *volinfo,
                                  char *volname,
                                  dict_t *dict,
-                                 char **op_errstr)
+                                 char **op_errstr,
+                                 dict_t *rsp_dict)
 {
         int32_t i               = 0;
         int32_t ret             = 0;
         int32_t count           = 0;
         char    *path           = NULL;
-        dict_t  *ctx            = NULL;
         char    cmd_str [1024]  = {0, };
         char   *ret_str         = NULL;
 
-        ctx = glusterd_op_get_ctx ();
-        if (ctx == NULL)
+        if (rsp_dict == NULL)
                 return 0;
 
         ret = dict_get_int32 (dict, "count", &count);
@@ -419,7 +418,7 @@ glusterd_quota_get_limit_usages (glusterd_conf_t *priv,
         }
 
         if (ret_str) {
-                ret = dict_set_dynstr (ctx, "limit_list", ret_str);
+                ret = dict_set_dynstr (rsp_dict, "limit_list", ret_str);
         }
 out:
         return ret;
@@ -674,12 +673,11 @@ out:
 
 
 int
-glusterd_op_quota (dict_t *dict, char **op_errstr)
+glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
         glusterd_volinfo_t     *volinfo      = NULL;
         int32_t                 ret          = -1;
         char                   *volname      = NULL;
-        dict_t                 *ctx          = NULL;
         int                     type         = -1;
         gf_boolean_t            start_crawl  = _gf_false;
         glusterd_conf_t        *priv         = NULL;
@@ -744,7 +742,7 @@ glusterd_op_quota (dict_t *dict, char **op_errstr)
                 }
 
                 ret = glusterd_quota_get_limit_usages (priv, volinfo, volname,
-                                                       dict, op_errstr);
+                                                       dict, op_errstr, rsp_dict);
 
                 goto out;
         }
@@ -767,12 +765,11 @@ create_vol:
         ret = 0;
 
 out:
-        ctx = glusterd_op_get_ctx ();
-        if (ctx && start_crawl == _gf_true)
+        if (rsp_dict && start_crawl == _gf_true)
                 glusterd_quota_initiate_fs_crawl (priv, volname);
 
-        if (ctx && *op_errstr) {
-                ret = dict_set_dynstr (ctx, "errstr", *op_errstr);
+        if (rsp_dict && *op_errstr) {
+                ret = dict_set_dynstr (rsp_dict, "errstr", *op_errstr);
                 if (ret) {
                         GF_FREE (*op_errstr);
                         gf_log ("", GF_LOG_DEBUG,
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index e5a907eaa..b7b974c68 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -41,581 +31,336 @@
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
 #include "run.h"
+#include "glusterd-volgen.h"
 
 #include "syscall.h"
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 
-/* return values - 0: success, +ve: stopped, -ve: failure */
+int32_t
+glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe);
 int
-gf_glusterd_rebalance_move_data (glusterd_volinfo_t *volinfo, const char *dir)
+glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
+                                size_t len, glusterd_op_t op)
 {
-        int                     ret                    = -1;
-        DIR                    *fd                     = NULL;
-        glusterd_defrag_info_t *defrag                 = NULL;
-        struct dirent          *entry                  = NULL;
-        struct stat             stbuf                  = {0,};
-        char                    full_path[PATH_MAX]    = {0,};
-        char                    linkinfo[PATH_MAX]     = {0,};
-        char                    force_string[64]       = {0,};
-
-        if (!volinfo->defrag)
-                goto out;
-
-        defrag = volinfo->defrag;
-
-        fd = opendir (dir);
-        if (!fd)
+        int      ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        /* Check only if operation is not remove-brick */
+        if ((GD_OP_REMOVE_BRICK != op) &&
+            !gd_is_remove_brick_committed (volinfo)) {
+                gf_log (this->name, GF_LOG_DEBUG, "A remove-brick task on "
+                        "volume %s is not yet committed", volinfo->volname);
+                snprintf (op_errstr, len, "A remove-brick task on volume %s is"
+                          " not yet committed. Either commit or stop the "
+                          "remove-brick task.", volinfo->volname);
                 goto out;
-
-        if ((defrag->cmd == GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE) ||
-            (defrag->cmd == GF_DEFRAG_CMD_START_FORCE)) {
-                strcpy (force_string, "force");
-        } else {
-                strcpy (force_string, "not-force");
         }
 
-        while ((entry = readdir (fd))) {
-                if (!entry)
-                        break;
-
-                /* We have to honor 'stop' (or 'pause'|'commit') as early
-                   as possible */
-                if (volinfo->defrag_status !=
-                    GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED) {
-                        /* It can be one of 'stopped|paused|commit' etc */
-                        closedir (fd);
-                        ret = 1;
-                        goto out;
-                }
-
-                if (!strcmp (entry->d_name, ".") || !strcmp (entry->d_name, ".."))
-                        continue;
-
-                snprintf (full_path, PATH_MAX, "%s/%s", dir, entry->d_name);
-
-                ret = lstat (full_path, &stbuf);
-                if (ret == -1)
-                        continue;
-
-                if (S_ISDIR (stbuf.st_mode))
-                        continue;
-
-                defrag->num_files_lookedup += 1;
-
-                /* TODO: bring in feature to support hardlink rebalance */
-                if (stbuf.st_nlink > 1)
-                        continue;
-
-                /* if distribute is present, it will honor this key.
-                   -1 is returned if distribute is not present or file doesn't
-                   have a link-file. If file has link-file, the path of
-                   link-file will be the value, and also that guarantees
-                   that file has to be mostly migrated */
-                ret = sys_lgetxattr (full_path, GF_XATTR_LINKINFO_KEY,
-                                     &linkinfo, PATH_MAX);
-                if (ret <= 0)
-                        continue;
-
-                ret = sys_lsetxattr (full_path, "distribute.migrate-data",
-                                     force_string, strlen (force_string), 0);
-
-                /* if errno is not ENOSPC or ENOTCONN, we can still continue
-                   with rebalance process */
-                if ((ret == -1) && ((errno != ENOSPC) ||
-                                    (errno != ENOTCONN)))
-                        continue;
-
-                if ((ret == -1) && (errno == ENOTCONN)) {
-                        /* Most probably mount point went missing (mostly due
-                           to a brick down), say rebalance failure to user,
-                           let him restart it if everything is fine */
-                        volinfo->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                        break;
-                }
-
-                if ((ret == -1) && (errno == ENOSPC)) {
-                        /* rebalance process itself failed, may be
-                           remote brick went down, or write failed due to
-                           disk full etc etc.. */
-                        volinfo->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                        break;
-                }
-
-                LOCK (&defrag->lock);
-                {
-                        defrag->total_files += 1;
-                        defrag->total_data += stbuf.st_size;
-                }
-                UNLOCK (&defrag->lock);
+        if (glusterd_is_defrag_on (volinfo)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "rebalance on volume %s already started",
+                        volinfo->volname);
+                snprintf (op_errstr, len, "Rebalance on %s is already started",
+                          volinfo->volname);
+                goto out;
         }
-        closedir (fd);
 
-        fd = opendir (dir);
-        if (!fd)
+        if (glusterd_is_rb_started (volinfo) ||
+            glusterd_is_rb_paused (volinfo)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Rebalance failed as replace brick is in progress on volume %s",
+                        volinfo->volname);
+                snprintf (op_errstr, len, "Rebalance failed as replace brick is in progress on "
+                          "volume %s", volinfo->volname);
                 goto out;
-        while ((entry = readdir (fd))) {
-                if (!entry)
-                        break;
-
-                /* We have to honor 'stop' (or 'pause'|'commit') as early
-                   as possible */
-                if (volinfo->defrag_status !=
-                    GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED) {
-                        /* It can be one of 'stopped|paused|commit' etc */
-                        closedir (fd);
-                        ret = 1;
-                        goto out;
-                }
-
-                if (!strcmp (entry->d_name, ".") || !strcmp (entry->d_name, ".."))
-                        continue;
-
-                snprintf (full_path, 1024, "%s/%s", dir, entry->d_name);
-
-                ret = lstat (full_path, &stbuf);
-                if (ret == -1)
-                        continue;
-
-                if (!S_ISDIR (stbuf.st_mode))
-                        continue;
-
-                ret = gf_glusterd_rebalance_move_data (volinfo, full_path);
-                if (ret)
-                        break;
         }
-        closedir (fd);
-
-        if (!entry)
-                ret = 0;
+        ret = 0;
 out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-/* return values - 0: success, +ve: stopped, -ve: failure */
-int
-gf_glusterd_rebalance_fix_layout (glusterd_volinfo_t *volinfo, const char *dir)
-{
-        int            ret             = -1;
-        char           full_path[1024] = {0,};
-        struct stat    stbuf           = {0,};
-        DIR           *fd              = NULL;
-        struct dirent *entry           = NULL;
-
-        if (!volinfo->defrag)
-                goto out;
-
-        fd = opendir (dir);
-        if (!fd)
-                goto out;
-
-        while ((entry = readdir (fd))) {
-                if (!entry)
-                        break;
-
-                /* We have to honor 'stop' (or 'pause'|'commit') as early
-                   as possible */
-                if (volinfo->defrag_status !=
-                    GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED) {
-                        /* It can be one of 'stopped|paused|commit' etc */
-                        closedir (fd);
-                        ret = 1;
-                        goto out;
-                }
-
-                if (!strcmp (entry->d_name, ".") || !strcmp (entry->d_name, ".."))
-                        continue;
-
-                snprintf (full_path, 1024, "%s/%s", dir, entry->d_name);
-
-                ret = lstat (full_path, &stbuf);
-                if (ret == -1)
-                        continue;
 
-                if (S_ISDIR (stbuf.st_mode)) {
-                        /* Fix the layout of the directory */
-                        /* TODO: isn't error code not important ? */
-                        sys_lsetxattr (full_path, "distribute.fix.layout",
-                                       "yes", 3, 0);
-
-                        volinfo->defrag->total_files += 1;
-
-                        /* Traverse into subdirectory */
-                        ret = gf_glusterd_rebalance_fix_layout (volinfo,
-                                                                full_path);
-                        if (ret)
-                                break;
-                }
-        }
-        closedir (fd);
-
-        if (!entry)
-                ret = 0;
+int32_t
+__glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
+{
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_defrag_info_t  *defrag  = NULL;
+        int                     ret      = 0;
+        char                    pidfile[PATH_MAX];
+        glusterd_conf_t        *priv    = NULL;
 
-out:
-        return ret;
-}
+        priv = THIS->private;
+        if (!priv)
+                return 0;
 
-void *
-glusterd_defrag_start (void *data)
-{
-        glusterd_volinfo_t     *volinfo = data;
-        glusterd_defrag_info_t *defrag  = NULL;
-        int                     ret     = -1;
-        struct stat             stbuf   = {0,};
+        volinfo = mydata;
+        if (!volinfo)
+                return 0;
 
-        THIS = volinfo->xl;
-        defrag = volinfo->defrag;
+        defrag = volinfo->rebal.defrag;
         if (!defrag)
-                goto out;
+                return 0;
 
-        sleep (1);
-        ret = lstat (defrag->mount, &stbuf);
-        if ((ret == -1) && (errno == ENOTCONN)) {
-                /* Wait for some more time before starting rebalance */
-                sleep (2);
-                ret = lstat (defrag->mount, &stbuf);
-                if (ret == -1) {
-                        volinfo->defrag_status   = GF_DEFRAG_STATUS_FAILED;
-                        volinfo->rebalance_files = 0;
-                        volinfo->rebalance_data  = 0;
-                        volinfo->lookedup_files  = 0;
-                        goto out;
-                }
-        }
+        if ((event == RPC_CLNT_DISCONNECT) && defrag->connected)
+                volinfo->rebal.defrag = NULL;
 
-        /* Fix the root ('/') first */
-        sys_lsetxattr (defrag->mount, "distribute.fix.layout",
-                       "yes", 3, 0);
+        GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
 
-        if ((defrag->cmd == GF_DEFRAG_CMD_START) ||
-            (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX)) {
-                /* root's layout got fixed */
-                defrag->total_files = 1;
+        switch (event) {
+        case RPC_CLNT_CONNECT:
+        {
+                if (defrag->connected)
+                        return 0;
 
-                /* Step 1: Fix layout of all the directories */
-                ret = gf_glusterd_rebalance_fix_layout (volinfo, defrag->mount);
-                if (ret < 0)
-                        volinfo->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                /* in both 'stopped' or 'failure' cases goto out */
-                if (ret) {
-                        goto out;
+                LOCK (&defrag->lock);
+                {
+                        defrag->connected = 1;
                 }
+                UNLOCK (&defrag->lock);
 
-                /* Completed first step */
-                volinfo->defrag_status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
+               gf_log ("", GF_LOG_DEBUG, "%s got RPC_CLNT_CONNECT",
+                        rpc->conn.trans->name);
+               break;
         }
 
-        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
-                /* It was used by number of layout fixes on directories */
-                defrag->total_files = 0;
-
-                volinfo->defrag_status = GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED;
+        case RPC_CLNT_DISCONNECT:
+        {
+                if (!defrag->connected)
+                        return 0;
 
-                /* Step 2: Iterate over directories to move data */
-                ret = gf_glusterd_rebalance_move_data (volinfo, defrag->mount);
-                if (ret < 0)
-                        volinfo->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                /* in both 'stopped' or 'failure' cases goto out */
-                if (ret) {
-                        goto out;
+                LOCK (&defrag->lock);
+                {
+                        defrag->connected = 0;
                 }
+                UNLOCK (&defrag->lock);
 
-                /* Completed second step */
-                volinfo->defrag_status = GF_DEFRAG_STATUS_MIGRATE_DATA_COMPLETE;
-        }
-
-        /* Completed whole process */
-        if ((defrag->cmd == GF_DEFRAG_CMD_START) ||
-            (defrag->cmd == GF_DEFRAG_CMD_START_FORCE))
-                volinfo->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
-
-        volinfo->rebalance_files = defrag->total_files;
-        volinfo->rebalance_data  = defrag->total_data;
-        volinfo->lookedup_files  = defrag->num_files_lookedup;
-out:
-        volinfo->defrag = NULL;
-        if (defrag) {
-                gf_log ("rebalance", GF_LOG_INFO, "rebalance on %s complete",
-                        defrag->mount);
+                if (!glusterd_is_service_running (pidfile, NULL)) {
+                        if (volinfo->rebal.defrag_status ==
+                                                GF_DEFRAG_STATUS_STARTED) {
+                                volinfo->rebal.defrag_status =
+                                                   GF_DEFRAG_STATUS_FAILED;
+                        }
+                 }
 
-                ret = runcmd ("umount", "-l", defrag->mount, NULL);
-                LOCK_DESTROY (&defrag->lock);
+                glusterd_store_perform_node_state_store (volinfo);
 
-                if (defrag->cbk_fn) {
-                        defrag->cbk_fn (volinfo, volinfo->defrag_status);
+                if (defrag->rpc) {
+                        rpc_clnt_unref (defrag->rpc);
+                        defrag->rpc = NULL;
                 }
+                if (defrag->cbk_fn)
+                        defrag->cbk_fn (volinfo,
+                                        volinfo->rebal.defrag_status);
 
                 GF_FREE (defrag);
+                gf_log ("", GF_LOG_DEBUG, "%s got RPC_CLNT_DISCONNECT",
+                        rpc->conn.trans->name);
+                break;
         }
-        return NULL;
+        default:
+                gf_log ("", GF_LOG_TRACE,
+                        "got some other RPC event %d", event);
+                ret = 0;
+                break;
+        }
+
+        return ret;
 }
 
-int
-glusterd_defrag_stop_validate (glusterd_volinfo_t *volinfo,
-                               char *op_errstr, size_t len)
+int32_t
+glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
+                        rpc_clnt_event_t event, void *data)
 {
-        int     ret = -1;
-        if (glusterd_is_defrag_on (volinfo) == 0) {
-                snprintf (op_errstr, len, "Rebalance on %s is either Completed "
-                          "or not yet started", volinfo->volname);
-                goto out;
-        }
-        ret = 0;
-out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+        return glusterd_big_locked_notify (rpc, mydata, event,
+                                           data, __glusterd_defrag_notify);
 }
 
 int
-glusterd_defrag_stop (glusterd_volinfo_t *volinfo, u_quad_t *files,
-                      u_quad_t *size, char *op_errstr, size_t len)
+glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
+                              size_t len, int cmd, defrag_cbk_fn_t cbk,
+                              glusterd_op_t op)
 {
-        /* TODO: set a variable 'stop_defrag' here, it should be checked
-           in defrag loop */
-        int     ret = -1;
+        int                    ret = -1;
+        glusterd_defrag_info_t *defrag =  NULL;
+        runner_t               runner = {0,};
+        glusterd_conf_t        *priv = NULL;
+        char                   defrag_path[PATH_MAX];
+        char                   sockfile[PATH_MAX] = {0,};
+        char                   pidfile[PATH_MAX] = {0,};
+        char                   logfile[PATH_MAX] = {0,};
+        dict_t                 *options = NULL;
+        char                   valgrind_logfile[PATH_MAX] = {0,};
+
+        priv    = THIS->private;
+
         GF_ASSERT (volinfo);
-        GF_ASSERT (files);
-        GF_ASSERT (size);
         GF_ASSERT (op_errstr);
 
-        if (!volinfo) {
-                ret = -1;
+        ret = glusterd_defrag_start_validate (volinfo, op_errstr, len, op);
+        if (ret)
                 goto out;
-        }
-
-        ret = glusterd_defrag_stop_validate (volinfo, op_errstr, len);
-        if (ret) {
-                /* rebalance may be happening on other nodes */
-                ret = 0;
+        if (!volinfo->rebal.defrag)
+                volinfo->rebal.defrag =
+                        GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
+                                   gf_gld_mt_defrag_info);
+        if (!volinfo->rebal.defrag)
                 goto out;
-        }
 
-        ret = 0;
-        if (volinfo->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
-                goto out;
-        }
+        defrag = volinfo->rebal.defrag;
 
-        LOCK (&volinfo->defrag->lock);
-        {
-                volinfo->defrag_status = GF_DEFRAG_STATUS_STOPPED;
-                *files = volinfo->defrag->total_files;
-                *size = volinfo->defrag->total_data;
-        }
-        UNLOCK (&volinfo->defrag->lock);
+        defrag->cmd = cmd;
 
-        ret = 0;
-out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
+        volinfo->rebal.defrag_cmd = cmd;
+        volinfo->rebal.op = op;
 
-int
-glusterd_defrag_status_get (glusterd_volinfo_t *volinfo,
-                            dict_t *dict)
-{
-        int      ret    = 0;
-        uint64_t files  = 0;
-        uint64_t size   = 0;
-        uint64_t lookup = 0;
+        LOCK_INIT (&defrag->lock);
 
-        if (!volinfo || !dict)
-                goto out;
+        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
 
-        ret = 0;
-        if (volinfo->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
-                goto out;
+        glusterd_volinfo_reset_defrag_stats (volinfo);
+        glusterd_store_perform_node_state_store (volinfo);
 
-        if (volinfo->defrag) {
-                LOCK (&volinfo->defrag->lock);
-                {
-                        files  = volinfo->defrag->total_files;
-                        size   = volinfo->defrag->total_data;
-                        lookup = volinfo->defrag->num_files_lookedup;
-                }
-                UNLOCK (&volinfo->defrag->lock);
-        } else {
-                files  = volinfo->rebalance_files;
-                size   = volinfo->rebalance_data;
-                lookup = volinfo->lookedup_files;
+        GLUSTERD_GET_DEFRAG_DIR (defrag_path, volinfo, priv);
+        ret = mkdir_p (defrag_path, 0777, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create "
+                        "directory %s", defrag_path);
+                goto out;
         }
 
-        ret = dict_set_uint64 (dict, "files", files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set file count");
-
-        ret = dict_set_uint64 (dict, "size", size);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set size of xfer");
-
-        ret = dict_set_uint64 (dict, "lookups", lookup);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set lookedup file count");
-
-        ret = dict_set_int32 (dict, "status", volinfo->defrag_status);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set status");
+        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo, priv);
+        GLUSTERD_GET_DEFRAG_PID_FILE (pidfile, volinfo, priv);
+        snprintf (logfile, PATH_MAX, "%s/%s-rebalance.log",
+                    DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname);
+        runinit (&runner);
 
-out:
-        return 0;
-}
+        if (priv->valgrind) {
+                snprintf (valgrind_logfile, PATH_MAX,
+                          "%s/valgrind-%s-rebalance.log",
+                          DEFAULT_LOG_FILE_DIRECTORY,
+                          volinfo->volname);
 
-void
-glusterd_rebalance_cmd_attempted_log (int cmd, char *volname)
-{
-        switch (cmd) {
-                case GF_DEFRAG_CMD_START_LAYOUT_FIX:
-                        gf_cmd_log ("Volume rebalance"," on volname: %s "
-                                    "cmd: start fix layout , attempted",
-                                    volname);
-                        gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
-                                "volume start layout fix on %s", volname);
-                        break;
-                case GF_DEFRAG_CMD_START_MIGRATE_DATA:
-                case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
-                        gf_cmd_log ("Volume rebalance"," on volname: %s "
-                                    "cmd: start data migrate attempted",
-                                    volname);
-                        gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
-                                "volume start migrate data on %s", volname);
-                        break;
-                case GF_DEFRAG_CMD_START:
-                        gf_cmd_log ("Volume rebalance"," on volname: %s "
-                                    "cmd: start, attempted", volname);
-                        gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
-                                "volume start on %s", volname);
-                        break;
-                case GF_DEFRAG_CMD_STOP:
-                        gf_cmd_log ("Volume rebalance"," on volname: %s "
-                                    "cmd: stop, attempted", volname);
-                        gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
-                                "volume stop on %s", volname);
-                        break;
-                default:
-                        break;
+                runner_add_args (&runner, "valgrind", "--leak-check=full",
+                                 "--trace-children=yes", "--track-origins=yes",
+                                 NULL);
+                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
         }
-}
 
-void
-glusterd_rebalance_cmd_log (int cmd, char *volname, int status)
-{
-        if (cmd != GF_DEFRAG_CMD_STATUS) {
-                gf_cmd_log ("volume rebalance"," on volname: %s %d %s",
-                            volname, cmd, ((status)?"FAILED":"SUCCESS"));
+        runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                         "-s", "localhost", "--volfile-id", volinfo->volname,
+                         "--xlator-option", "*dht.use-readdirp=yes",
+                         "--xlator-option", "*dht.lookup-unhashed=yes",
+                         "--xlator-option", "*dht.assert-no-child-down=yes",
+                         "--xlator-option", "*replicate*.data-self-heal=off",
+                         "--xlator-option",
+                         "*replicate*.metadata-self-heal=off",
+                         "--xlator-option", "*replicate*.entry-self-heal=off",
+                         "--xlator-option", "*replicate*.readdir-failover=off",
+                         "--xlator-option", "*dht.readdir-optimize=on",
+                         NULL);
+        runner_add_arg (&runner, "--xlator-option");
+        runner_argprintf ( &runner, "*dht.rebalance-cmd=%d",cmd);
+        runner_add_arg (&runner, "--xlator-option");
+        runner_argprintf (&runner, "*dht.node-uuid=%s", uuid_utoa(MY_UUID));
+        runner_add_arg (&runner, "--socket-file");
+        runner_argprintf (&runner, "%s",sockfile);
+        runner_add_arg (&runner, "--pid-file");
+        runner_argprintf (&runner, "%s",pidfile);
+        runner_add_arg (&runner, "-l");
+        runner_argprintf (&runner, logfile);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
+
+        ret = runner_run_nowait (&runner);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_DEBUG, "rebalance command failed");
+                goto out;
         }
-}
 
-int
-glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
-                                size_t len)
-{
-        int     ret = -1;
+        sleep (5);
 
-        if (glusterd_is_defrag_on (volinfo)) {
-                gf_log ("glusterd", GF_LOG_DEBUG,
-                        "rebalance on volume %s already started",
-                        volinfo->volname);
-                snprintf (op_errstr, len, "Rebalance on %s is already started",
-                          volinfo->volname);
+        /* Setting frame-timeout to 10mins (600seconds).
+         * Unix domain sockets ensures that the connection is reliable. The
+         * default timeout of 30mins used for unreliable network connections is
+         * too long for unix domain socket connections.
+         */
+        ret = rpc_transport_unix_options_build (&options, sockfile, 600);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Unix options build failed");
                 goto out;
         }
 
-        if (glusterd_is_rb_started (volinfo) ||
-            glusterd_is_rb_paused (volinfo)) {
-                gf_log ("glusterd", GF_LOG_DEBUG,
-                        "Rebalance failed as replace brick is in progress on volume %s",
-                        volinfo->volname);
-                snprintf (op_errstr, len, "Rebalance failed as replace brick is in progress on "
-                          "volume %s", volinfo->volname);
+        synclock_unlock (&priv->big_lock);
+        ret = glusterd_rpc_create (&defrag->rpc, options,
+                                   glusterd_defrag_notify, volinfo);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "RPC create failed");
                 goto out;
         }
-        ret = 0;
+
+        if (cbk)
+                defrag->cbk_fn = cbk;
+
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+
 int
-glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
-                              size_t len, int cmd, defrag_cbk_fn_t cbk)
+glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
+                               glusterd_conf_t *priv, int cmd)
 {
-        int                    ret = -1;
-        glusterd_defrag_info_t *defrag =  NULL;
-        runner_t               runner = {0,};
-        glusterd_conf_t        *priv = NULL;
+        dict_t                  *options = NULL;
+        char                     sockfile[PATH_MAX] = {0,};
+        int                      ret = -1;
+        glusterd_defrag_info_t  *defrag =  NULL;
 
-        priv    = THIS->private;
+        if (!volinfo->rebal.defrag)
+                volinfo->rebal.defrag =
+                        GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
+                                   gf_gld_mt_defrag_info);
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (op_errstr);
-
-        ret = glusterd_defrag_start_validate (volinfo, op_errstr, len);
-        if (ret)
-                goto out;
-        if (!volinfo->defrag)
-                volinfo->defrag = GF_CALLOC (1, sizeof (glusterd_defrag_info_t),
-                                             gf_gld_mt_defrag_info);
-        if (!volinfo->defrag)
+        if (!volinfo->rebal.defrag)
                 goto out;
 
-        defrag = volinfo->defrag;
+        defrag = volinfo->rebal.defrag;
 
         defrag->cmd = cmd;
 
         LOCK_INIT (&defrag->lock);
-        snprintf (defrag->mount, 1024, "%s/mount/%s",
-                  priv->workdir, volinfo->volname);
-        /* Create a directory, mount glusterfs over it, start glusterfs-defrag */
-        runinit (&runner);
-        runner_add_args (&runner, "mkdir", "-p", defrag->mount, NULL);
-        ret = runner_run_reuse (&runner);
-        if (ret) {
-                runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
-                runner_end (&runner);
-                goto out;
-        }
-        runner_end (&runner);
 
-        runinit (&runner);
-        runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                         "-s", "localhost", "--volfile-id", volinfo->volname,
-                         "--xlator-option", "*dht.use-readdirp=yes",
-                         "--xlator-option", "*dht.lookup-unhashed=yes",
-                         "--xlator-option", "*dht.assert-no-child-down=yes",
-                         defrag->mount, NULL);
-        ret = runner_run_reuse (&runner);
+        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo, priv);
+
+        /* Setting frame-timeout to 10mins (600seconds).
+         * Unix domain sockets ensures that the connection is reliable. The
+         * default timeout of 30mins used for unreliable network connections is
+         * too long for unix domain socket connections.
+         */
+        ret = rpc_transport_unix_options_build (&options, sockfile, 600);
         if (ret) {
-                runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
-                runner_end (&runner);
+                gf_log (THIS->name, GF_LOG_ERROR, "Unix options build failed");
                 goto out;
         }
-        runner_end (&runner);
-
-        volinfo->defrag_status = GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED;
-        if ((cmd == GF_DEFRAG_CMD_START_MIGRATE_DATA) ||
-            (cmd == GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE)) {
-                volinfo->defrag_status = GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED;
-        }
 
-        if (cbk)
-                defrag->cbk_fn = cbk;
-
-        ret = pthread_create (&defrag->th, NULL, glusterd_defrag_start,
-                              volinfo);
+        synclock_unlock (&priv->big_lock);
+        ret = glusterd_rpc_create (&defrag->rpc, options,
+                                   glusterd_defrag_notify, volinfo);
+        synclock_lock (&priv->big_lock);
         if (ret) {
-                runinit (&runner);
-                runner_add_args (&runner, "umount", "-l", defrag->mount, NULL);
-                ret = runner_run_reuse (&runner);
-                if (ret)
-                        runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
-                runner_end (&runner);
+                gf_log (THIS->name, GF_LOG_ERROR, "RPC create failed");
+                goto out;
         }
+        ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -658,26 +403,32 @@ out:
 }
 
 int
-glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+__glusterd_handle_defrag_volume (rpcsvc_request_t *req)
 {
-        int32_t                 ret     = -1;
-        gf_cli_req              cli_req = {{0,}};
-        glusterd_conf_t        *priv    = NULL;
-        dict_t                 *dict    = NULL;
-        char                   *volname = NULL;
-        gf_cli_defrag_type      cmd     = 0;
+        int32_t                 ret       = -1;
+        gf_cli_req              cli_req   = {{0,}};
+        glusterd_conf_t        *priv      = NULL;
+        dict_t                 *dict      = NULL;
+        char                   *volname   = NULL;
+        gf_cli_defrag_type      cmd       = 0;
+        char                    msg[2048] = {0,};
+        xlator_t               *this      = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        priv = THIS->private;
+        priv = this->private;
+        GF_ASSERT (priv);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
-       if (cli_req.dict.dict_len) {
+
+        if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
                 dict  = dict_new ();
 
@@ -685,34 +436,39 @@ glusterd_handle_defrag_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to get volname");
+                snprintf (msg, sizeof (msg), "Failed to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "rebalance-command", (int32_t*)&cmd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to get command");
+                snprintf (msg, sizeof (msg), "Failed to get command");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
-        glusterd_rebalance_cmd_attempted_log (cmd, volname);
-
-        ret = dict_set_static_bin (dict, "node-uuid", priv->uuid, 16);
+        ret = dict_set_static_bin (dict, "node-uuid", MY_UUID, 16);
         if (ret)
                 goto out;
 
-        ret = glusterd_op_begin (req, GD_OP_REBALANCE, dict);
+        if ((cmd == GF_DEFRAG_CMD_STATUS) ||
+              (cmd == GF_DEFRAG_CMD_STOP)) {
+                ret = glusterd_op_begin (req, GD_OP_DEFRAG_BRICK_VOLUME,
+                                         dict, msg, sizeof (msg));
+        } else
+                ret = glusterd_op_begin (req, GD_OP_REBALANCE, dict,
+                                         msg, sizeof (msg));
 
 out:
 
@@ -720,57 +476,100 @@ out:
         glusterd_op_sm ();
 
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
-                ret = glusterd_op_send_cli_response (GD_OP_REBALANCE, ret, 0, req,
-                                                     NULL, "operation failed");
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
+                ret = glusterd_op_send_cli_response (GD_OP_REBALANCE, ret, 0,
+                                                     req, dict, msg);
+
         }
 
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);//malloced by xdr
+        free (cli_req.dict.dict_val);//malloced by xdr
 
         return 0;
 }
 
+int
+glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_defrag_volume);
+}
+
 
 int
 glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
 {
-        char *volname = NULL;
-        int ret = 0;
-        int32_t cmd = 0;
-        char msg[2048] = {0};
-        glusterd_volinfo_t  *volinfo = NULL;
+        char                    *volname = NULL;
+        int                     ret = 0;
+        int32_t                 cmd = 0;
+        char                    msg[2048] = {0};
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *task_id_str = NULL;
+        dict_t                  *op_ctx = NULL;
+        xlator_t                *this = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "volname not found");
+                gf_log (this->name, GF_LOG_DEBUG, "volname not found");
                 goto out;
         }
+
         ret = dict_get_int32 (dict, "rebalance-command", &cmd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "cmd not found");
+                gf_log (this->name, GF_LOG_DEBUG, "cmd not found");
                 goto out;
         }
 
         ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
                                                msg, sizeof (msg));
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "failed to validate");
+                gf_log (this->name, GF_LOG_DEBUG, "failed to validate");
                 goto out;
         }
         switch (cmd) {
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
-        case GF_DEFRAG_CMD_START_MIGRATE_DATA:
-        case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
-                ret = glusterd_defrag_start_validate (volinfo,
-                                                      msg, sizeof (msg));
+        case GF_DEFRAG_CMD_START_FORCE:
+                if (is_origin_glusterd (dict)) {
+                        op_ctx = glusterd_op_get_ctx ();
+                        if (!op_ctx) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get op_ctx");
+                                goto out;
+                        }
+
+                        ret = glusterd_generate_and_set_task_id
+                                (op_ctx, GF_REBALANCE_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to generate task-id");
+                                goto out;
+                        }
+                } else {
+                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                snprintf (msg, sizeof (msg),
+                                          "Missing rebalance-id");
+                                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
+                                ret = 0;
+                        }
+                }
+                ret = glusterd_defrag_start_validate (volinfo, msg,
+                                                      sizeof (msg),
+                                                      GD_OP_REBALANCE);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "start validate failed");
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                        "start validate failed");
                         goto out;
                 }
+                break;
+        case GF_DEFRAG_CMD_STATUS:
+        case GF_DEFRAG_CMD_STOP:
+                break;
         default:
                 break;
         }
@@ -787,101 +586,164 @@ out:
 int
 glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        char               *volname   = NULL;
-        int                 ret       = 0;
-        int32_t             cmd       = 0;
-        char                msg[2048] = {0};
-        glusterd_volinfo_t *volinfo   = NULL;
-        uint64_t            files     = 0;
-        uint64_t            size      = 0;
-        void               *node_uuid = NULL;
-        glusterd_conf_t    *priv      = NULL;
-        dict_t             *tmp_dict  = NULL;
-
-        priv = THIS->private;
+        char                    *volname   = NULL;
+        int                     ret       = 0;
+        int32_t                 cmd       = 0;
+        char                    msg[2048] = {0};
+        glusterd_volinfo_t      *volinfo   = NULL;
+        glusterd_conf_t         *priv      = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_brickinfo_t    *tmp      = NULL;
+        gf_boolean_t            volfile_update = _gf_false;
+        char                    *task_id_str = NULL;
+        dict_t                  *ctx = NULL;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "volname not given");
+                gf_log (this->name, GF_LOG_DEBUG, "volname not given");
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "rebalance-command", &cmd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "command not given");
+                gf_log (this->name, GF_LOG_DEBUG, "command not given");
                 goto out;
         }
 
+
         ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
                                                msg, sizeof (msg));
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "cmd validate failed");
+                gf_log (this->name, GF_LOG_DEBUG, "cmd validate failed");
                 goto out;
         }
 
-        if ((cmd != GF_DEFRAG_CMD_STATUS) &&
-            (cmd != GF_DEFRAG_CMD_STOP)) {
-                ret = dict_get_ptr (dict, "node-uuid", &node_uuid);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG, "node-uuid not found");
-                        goto out;
-                }
-
-                /* perform this on only the node which has
-                   issued the command */
-                if (uuid_compare (node_uuid, priv->uuid)) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "not the source node %s", uuid_utoa (priv->uuid));
-                        goto out;
+        /* Set task-id, if available, in op_ctx dict for operations other than
+         * start
+         */
+        if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STOP) {
+                if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+                        ctx = glusterd_op_get_ctx ();
+                        if (!ctx) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get op_ctx");
+                                ret = -1;
+                                goto out;
+                        }
+
+                        if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
+                                ret = glusterd_copy_uuid_to_dict
+                                        (volinfo->rebal.rebalance_id, ctx,
+                                         GF_REMOVE_BRICK_TID_KEY);
+                        else
+                                ret = glusterd_copy_uuid_to_dict
+                                        (volinfo->rebal.rebalance_id, ctx,
+                                         GF_REBALANCE_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set task-id");
+                                goto out;
+                        }
                 }
         }
 
         switch (cmd) {
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
-        case GF_DEFRAG_CMD_START_MIGRATE_DATA:
-        case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
+        case GF_DEFRAG_CMD_START_FORCE:
+                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Missing rebalance "
+                                "id");
+                        ret = 0;
+                } else {
+                        uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
+                        volinfo->rebal.op = GD_OP_REBALANCE;
+                }
                 ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
-                                                    cmd, NULL);
+                                                    cmd, NULL, GD_OP_REBALANCE);
                  break;
-         case GF_DEFRAG_CMD_STOP:
-                 ret = glusterd_defrag_stop (volinfo, &files, &size,
-                                             msg, sizeof (msg));
-                if (!ret && rsp_dict) {
-                        ret = dict_set_uint64 (rsp_dict, "files", files);
-                        if (ret)
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "failed to set file count");
-
-                        ret = dict_set_uint64 (rsp_dict, "size", size);
-                        if (ret)
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "failed to set xfer size");
-
-                        /* Don't want to propagate errors from dict_set() */
+        case GF_DEFRAG_CMD_STOP:
+                /* Clear task-id only on explicitly stopping rebalance.
+                 * Also clear the stored operation, so it doesn't cause trouble
+                 * with future rebalance/remove-brick starts
+                 */
+                uuid_clear (volinfo->rebal.rebalance_id);
+                volinfo->rebal.op = GD_OP_NONE;
+
+                /* Fall back to the old volume file in case of decommission*/
+                list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
+                                          brick_list) {
+                        if (!brickinfo->decommissioned)
+                                continue;
+                        brickinfo->decommissioned = 0;
+                        volfile_update = _gf_true;
+                }
+
+                if (volfile_update == _gf_false) {
                         ret = 0;
+                        break;
                 }
-                break;
-        case GF_DEFRAG_CMD_STATUS:
 
-                if (rsp_dict)
-                        tmp_dict = rsp_dict;
+                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to create volfiles");
+                        goto out;
+                }
+
+                ret = glusterd_store_volinfo (volinfo,
+                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to store volinfo");
+                        goto out;
+                }
 
-                /* On source node, there will be no 'rsp_dict' */
-                if (!tmp_dict)
-                        tmp_dict = glusterd_op_get_ctx (GD_OP_REBALANCE);
+                ret = 0;
+                break;
 
-                ret = glusterd_defrag_status_get (volinfo, tmp_dict);
+        case GF_DEFRAG_CMD_STATUS:
                 break;
         default:
                 break;
         }
 
-        glusterd_rebalance_cmd_log (cmd, volname, ret);
-
 out:
         if (ret && op_errstr && msg[0])
                 *op_errstr = gf_strdup (msg);
 
         return ret;
 }
+
+int32_t
+glusterd_defrag_event_notify_handle (dict_t *dict)
+{
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *volname = NULL;
+        int32_t                  ret     = -1;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Failed to get volname");
+                return ret;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Failed to get volinfo for %s"
+                        , volname);
+                return ret;
+        }
+
+        ret = glusterd_defrag_volume_status_update (volinfo, dict);
+
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "Failed to update status");
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 40202dd82..54b830870 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -36,6 +26,13 @@
 
 #include <signal.h>
 
+#define GLUSTERD_GET_RB_MNTPT(path, len, volinfo)                           \
+        snprintf (path, len,                                                \
+                  DEFAULT_VAR_RUN_DIRECTORY"/%s-"RB_CLIENT_MOUNTPOINT,      \
+                  volinfo->volname);
+
+extern uuid_t global_txn_id;
+
 int
 glusterd_get_replace_op_str (gf1_cli_replace_op op, char *op_str)
 {
@@ -74,7 +71,7 @@ out:
 }
 
 int
-glusterd_handle_replace_brick (rpcsvc_request_t *req)
+__glusterd_handle_replace_brick (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -85,17 +82,21 @@ glusterd_handle_replace_brick (rpcsvc_request_t *req)
         char                            operation[256];
         glusterd_op_t                   cli_op = GD_OP_REPLACE_BRICK;
         char                            *volname = NULL;
+        char                            msg[2048] = {0,};
+        xlator_t                        *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received replace brick req");
+        gf_log (this->name, GF_LOG_INFO, "Received replace brick req");
 
         if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
@@ -105,70 +106,79 @@ glusterd_handle_replace_brick (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "could not get volname");
+                snprintf (msg, sizeof (msg), "Could not get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "operation", &op);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "dict_get on operation failed");
+                snprintf (msg, sizeof (msg), "Could not get operation");
                 goto out;
         }
 
         ret = dict_get_str (dict, "src-brick", &src_brick);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get src brick");
+                snprintf (msg, sizeof (msg), "Failed to get src brick");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
-        gf_log ("", GF_LOG_DEBUG,
+        gf_log (this->name, GF_LOG_DEBUG,
                 "src brick=%s", src_brick);
 
         ret = dict_get_str (dict, "dst-brick", &dst_brick);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get dest brick");
+                snprintf (msg, sizeof (msg), "Failed to get dest brick");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
         (void) glusterd_get_replace_op_str (op, operation);
-        gf_log ("", GF_LOG_DEBUG, "dst brick=%s", dst_brick);
-        gf_log ("glusterd", GF_LOG_INFO, "Received replace brick %s request",
+        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
+        gf_log (this->name, GF_LOG_INFO, "Received replace brick %s request",
                 operation);
-        gf_cmd_log ("Volume replace-brick","volname: %s src_brick:%s"
-                    " dst_brick:%s op:%s", volname, src_brick, dst_brick,
-                    operation);
 
-        ret = glusterd_op_begin (req, GD_OP_REPLACE_BRICK, dict);
-        gf_cmd_log ("Volume replace-brick","on volname: %s %s", volname,
-                   (ret) ? "FAILED" : "SUCCESS");
+        ret = glusterd_op_begin (req, GD_OP_REPLACE_BRICK, dict,
+                                 msg, sizeof (msg));
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);//malloced by xdr
+        free (cli_req.dict.dict_val);//malloced by xdr
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
 
-        if (ret)
+        if (ret) {
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, msg);
+        }
 
         return ret;
 }
 
+int
+glusterd_handle_replace_brick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_replace_brick);
+}
+
 static int
 glusterd_get_rb_dst_brickinfo (glusterd_volinfo_t *volinfo,
                                glusterd_brickinfo_t **brickinfo)
@@ -178,7 +188,7 @@ glusterd_get_rb_dst_brickinfo (glusterd_volinfo_t *volinfo,
         if (!volinfo || !brickinfo)
                 goto out;
 
-        *brickinfo = volinfo->dst_brick;
+        *brickinfo = volinfo->rep_brick.dst_brick;
 
         ret = 0;
 
@@ -191,6 +201,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                                  dict_t *rsp_dict)
 {
         int                                      ret           = 0;
+        int32_t                                  port          = 0;
         char                                    *src_brick     = NULL;
         char                                    *dst_brick     = NULL;
         char                                    *volname       = NULL;
@@ -199,47 +210,53 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         glusterd_brickinfo_t                    *src_brickinfo = NULL;
         char                                    *host          = NULL;
         char                                    *path          = NULL;
-        char                                    msg[2048]      = {0};
+        char                                     msg[2048]     = {0};
         char                                    *dup_dstbrick  = NULL;
         glusterd_peerinfo_t                     *peerinfo = NULL;
         glusterd_brickinfo_t                    *dst_brickinfo = NULL;
-        gf_boolean_t                            is_run         = _gf_false;
+        gf_boolean_t                             is_run        = _gf_false;
         dict_t                                  *ctx           = NULL;
         glusterd_conf_t                         *priv          = NULL;
-        char                                    voldir[PATH_MAX] = {0};
-        char                                    pidfile[PATH_MAX] = {0};
+        char                                    *savetok       = NULL;
+        char                                     pidfile[PATH_MAX] = {0};
+        char                                    *task_id_str = NULL;
+        xlator_t                                *this = NULL;
+        gf_boolean_t                            is_force = _gf_false;
 
-        priv = THIS->private;
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
         GF_ASSERT (priv);
 
         ret = dict_get_str (dict, "src-brick", &src_brick);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get src brick");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get src brick");
                 goto out;
         }
 
-        gf_log ("", GF_LOG_DEBUG, "src brick=%s", src_brick);
+        gf_log (this->name, GF_LOG_DEBUG, "src brick=%s", src_brick);
 
         ret = dict_get_str (dict, "dst-brick", &dst_brick);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get dest brick");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get dest brick");
                 goto out;
         }
 
-        gf_log ("", GF_LOG_DEBUG, "dst brick=%s", dst_brick);
+        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
 
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "operation", (int32_t *)&replace_op);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "dict get on replace-brick operation failed");
                 goto out;
         }
@@ -264,7 +281,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 !glusterd_is_valid_volfpath (volname, dst_brick)) {
                 snprintf (msg, sizeof (msg), "brick path %s is too "
                           "long.", dst_brick);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 *op_errstr = gf_strdup (msg);
 
                 ret = -1;
@@ -273,10 +290,10 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
 
         ret = glusterd_check_gsync_running (volinfo, &is_run);
         if (ret && (is_run == _gf_false))
-                gf_log ("", GF_LOG_WARNING, "Unable to get the status"
+                gf_log (this->name, GF_LOG_WARNING, "Unable to get the status"
                                 " of active "GEOREP" session");
         if (is_run) {
-                gf_log ("", GF_LOG_WARNING, GEOREP" sessions active"
+                gf_log (this->name, GF_LOG_WARNING, GEOREP" sessions active"
                         "for the volume %s ", volname);
                 snprintf (msg, sizeof(msg), GEOREP" sessions are active "
                                 "for the volume %s.\nStop "GEOREP " sessions "
@@ -291,29 +308,61 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         if (glusterd_is_defrag_on(volinfo)) {
                 snprintf (msg, sizeof(msg), "Volume name %s rebalance is in "
                           "progress. Please retry after completion", volname);
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 *op_errstr = gf_strdup (msg);
                 ret = -1;
                 goto out;
         }
 
+        ctx = glusterd_op_get_ctx();
+
         switch (replace_op) {
         case GF_REPLACE_OP_START:
                 if (glusterd_is_rb_started (volinfo)) {
-                        gf_log ("", GF_LOG_ERROR, "Replace brick is already "
-                                "started for volume ");
+                        snprintf (msg, sizeof (msg), "Replace brick is already "
+                                  "started for volume");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                        *op_errstr = gf_strdup (msg);
                         ret = -1;
                         goto out;
                 }
+                if (is_origin_glusterd (dict)) {
+                        if (!ctx) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get op_ctx");
+                                goto out;
+                        }
+
+                        ret = glusterd_generate_and_set_task_id
+                                (ctx, GF_REPLACE_BRICK_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to generate task-id");
+                                goto out;
+                        }
+
+                } else {
+                        ret = dict_get_str (dict, GF_REPLACE_BRICK_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Missing replace-brick-id");
+                                ret = 0;
+                        }
+                }
+                is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
                 break;
+
         case GF_REPLACE_OP_PAUSE:
                 if (glusterd_is_rb_paused (volinfo)) {
-                        gf_log ("", GF_LOG_ERROR, "Replace brick is already "
-                                "paused for volume ");
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is "
+                                "already paused for volume ");
                         ret = -1;
                         goto out;
                 } else if (!glusterd_is_rb_started(volinfo)) {
-                        gf_log ("", GF_LOG_ERROR, "Replace brick is not"
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not"
                                 " started for volume ");
                         ret = -1;
                         goto out;
@@ -321,9 +370,8 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 break;
 
         case GF_REPLACE_OP_ABORT:
-                if ((!glusterd_is_rb_paused (volinfo)) &&
-                     (!glusterd_is_rb_started (volinfo))) {
-                        gf_log ("", GF_LOG_ERROR, "Replace brick is not"
+                if (!glusterd_is_rb_ongoing (volinfo)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not"
                                 " started or paused for volume ");
                         ret = -1;
                         goto out;
@@ -331,15 +379,17 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 break;
 
         case GF_REPLACE_OP_COMMIT:
-                if (!glusterd_is_rb_started (volinfo)) {
-                        gf_log ("", GF_LOG_ERROR, "Replace brick is not "
+                if (!glusterd_is_rb_ongoing (volinfo)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not "
                                 "started for volume ");
                         ret = -1;
                         goto out;
                 }
                 break;
 
-        case GF_REPLACE_OP_COMMIT_FORCE: break;
+        case GF_REPLACE_OP_COMMIT_FORCE:
+                is_force = _gf_true;
+                break;
 
         case GF_REPLACE_OP_STATUS:
 
@@ -352,7 +402,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                                 goto out;
                         }
 
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
                         ret = -1;
                         goto out;
                 }
@@ -364,8 +414,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         }
 
         ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo,
-                                                      GF_PATH_COMPLETE);
+                                                      &src_brickinfo);
         if (ret) {
                 snprintf (msg, sizeof (msg), "brick: %s does not exist in "
                           "volume: %s", src_brick, volname);
@@ -373,10 +422,9 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 goto out;
         }
 
-        ctx = glusterd_op_get_ctx();
         if (ctx) {
                 if (!glusterd_is_fuse_available ()) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "Unable to open /dev/"
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to open /dev/"
                                 "fuse (%s), replace-brick command failed",
                                 strerror (errno));
                         snprintf (msg, sizeof(msg), "Fuse unavailable\n "
@@ -387,8 +435,8 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 }
         }
 
-        if (!glusterd_is_local_addr (src_brickinfo->hostname)) {
-                gf_log ("", GF_LOG_DEBUG,
+        if (gf_is_local_addr (src_brickinfo->hostname)) {
+                gf_log (this->name, GF_LOG_DEBUG,
                         "I AM THE SOURCE HOST");
                 if (src_brickinfo->port && rsp_dict) {
                         ret = dict_set_int32 (rsp_dict, "src-brick-port",
@@ -400,11 +448,10 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                         }
                 }
 
-                GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, priv);
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, voldir,
-                                            src_brickinfo->hostname,
-                                            src_brickinfo->path);
-                if (!glusterd_is_service_running (pidfile, NULL)) {
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, src_brickinfo,
+                                            priv);
+                if ((replace_op != GF_REPLACE_OP_COMMIT_FORCE) &&
+                    !glusterd_is_service_running (pidfile, NULL)) {
                         snprintf(msg, sizeof(msg), "Source brick %s:%s "
                                  "is not online.", src_brickinfo->hostname,
                                  src_brickinfo->path);
@@ -419,53 +466,61 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         dup_dstbrick = gf_strdup (dst_brick);
         if (!dup_dstbrick) {
                 ret = -1;
-                gf_log ("", GF_LOG_ERROR, "Memory allocation failed");
+                gf_log (this->name, GF_LOG_ERROR, "Memory allocation failed");
                 goto out;
         }
-        host = strtok (dup_dstbrick, ":");
-        path = strtok (NULL, ":");
+        host = strtok_r (dup_dstbrick, ":", &savetok);
+        path = strtok_r (NULL, ":", &savetok);
 
         if (!host || !path) {
-                gf_log ("", GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "dst brick %s is not of form <HOSTNAME>:<export-dir>",
                         dst_brick);
                 ret = -1;
                 goto out;
         }
-        if (!glusterd_brickinfo_get (NULL, host, path, NULL)) {
-                snprintf(msg, sizeof(msg), "Brick: %s:%s already in use",
-                         host, path);
+
+        ret = glusterd_brickinfo_new_from_brick (dst_brick, &dst_brickinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_new_brick_validate (dst_brick, dst_brickinfo,
+                                           msg, sizeof (msg));
+        if (ret) {
                 *op_errstr = gf_strdup (msg);
                 ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
                 goto out;
         }
 
-        if ((volinfo->rb_status ==GF_RB_STATUS_NONE) &&
-            (replace_op == GF_REPLACE_OP_START)) {
-                ret = glusterd_brickinfo_from_brick (dst_brick, &dst_brickinfo);
-                volinfo->src_brick = src_brickinfo;
-                volinfo->dst_brick = dst_brickinfo;
-        } else {
-                ret = glusterd_get_rb_dst_brickinfo (volinfo, &dst_brickinfo);
+       if (!glusterd_is_rb_ongoing (volinfo) &&
+            (replace_op == GF_REPLACE_OP_START ||
+             replace_op == GF_REPLACE_OP_COMMIT_FORCE)) {
+
+                volinfo->rep_brick.src_brick = src_brickinfo;
+                volinfo->rep_brick.dst_brick = dst_brickinfo;
         }
 
         if (glusterd_rb_check_bricks (volinfo, src_brickinfo, dst_brickinfo)) {
 
                 ret = -1;
-                *op_errstr = gf_strdup ("incorrect source or "
+                *op_errstr = gf_strdup ("Incorrect source or "
                                         "destination brick");
                 if (*op_errstr)
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
                 goto out;
        }
 
-        if (!glusterd_is_local_addr (host)) {
-                ret = glusterd_brick_create_path (host, path,
-                                                  volinfo->volume_id, 0777,
-                                                  op_errstr);
+        if (!glusterd_is_rb_ongoing (volinfo) &&
+            gf_is_local_addr (host)) {
+                ret = glusterd_validate_and_create_brickpath (dst_brickinfo,
+                                                  volinfo->volume_id,
+                                                  op_errstr, is_force);
                 if (ret)
                         goto out;
-        } else {
+        }
+
+        if (!gf_is_local_addr (host)) {
                 ret = glusterd_friend_find (NULL, host, &peerinfo);
                 if (ret) {
                         snprintf (msg, sizeof (msg), "%s, is not a friend",
@@ -490,13 +545,72 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                         goto out;
                 }
         }
+
+        if (replace_op == GF_REPLACE_OP_START &&
+            gf_is_local_addr (volinfo->rep_brick.dst_brick->hostname)) {
+                port = pmap_registry_alloc (THIS);
+                if (!port) {
+                        gf_log (THIS->name, GF_LOG_CRITICAL,
+                                "No free ports available");
+                        ret = -1;
+                        goto out;
+                }
+
+                ctx = glusterd_op_get_ctx();
+                ret = dict_set_int32 ((ctx)?ctx:rsp_dict, "dst-brick-port",
+                                      port);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Failed to set dst "
+                                "brick port");
+                        goto out;
+                }
+                volinfo->rep_brick.dst_brick->port = port;
+        }
+
         ret = 0;
 
 out:
-        if (dup_dstbrick)
-                GF_FREE (dup_dstbrick);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        GF_FREE (dup_dstbrick);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+rb_set_mntfd (int mntfd)
+{
+        int     ret = -1;
+        dict_t *ctx = NULL;
+
+        ctx = glusterd_op_get_ctx ();
+        if (!ctx) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Failed to get op ctx");
+                goto out;
+        }
+        ret = dict_set_int32 (ctx, "mntfd", mntfd);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_DEBUG, "Failed to set mnt fd "
+                        "in op ctx");
+out:
+        return ret;
+}
+
+static int
+rb_get_mntfd (int *mntfd)
+{
+        int     ret = -1;
+        dict_t *ctx = NULL;
 
+        ctx = glusterd_op_get_ctx ();
+        if (!ctx) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Failed to get op ctx");
+                goto out;
+        }
+        ret = dict_get_int32 (ctx, "mntfd", mntfd);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_DEBUG, "Failed to get mnt fd "
+                        "from op ctx");
+out:
         return ret;
 }
 
@@ -522,6 +636,8 @@ rb_regenerate_volfiles (glusterd_volinfo_t *volinfo,
 
         ret = glusterd_create_rb_volfiles (volinfo, brickinfo);
 
+        dict_del (dict, "enable-pump");
+
 out:
         return ret;
 }
@@ -543,7 +659,8 @@ rb_src_brick_restart (glusterd_volinfo_t *volinfo,
                         ret);
         }
 
-        ret = glusterd_volume_stop_glusterfs (volinfo, src_brickinfo);
+        ret = glusterd_volume_stop_glusterfs (volinfo, src_brickinfo,
+                                              _gf_false);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to stop "
                         "glusterfs, ret: %d", ret);
@@ -570,7 +687,8 @@ rb_src_brick_restart (glusterd_volinfo_t *volinfo,
         }
 
         sleep (2);
-        ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo);
+        ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo,
+                                               _gf_false);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to start "
                         "glusterfs, ret: %d", ret);
@@ -592,22 +710,19 @@ rb_send_xattr_command (glusterd_volinfo_t *volinfo,
                        glusterd_brickinfo_t *dst_brickinfo,
                        const char *xattr_key, const char *value)
 {
-        glusterd_conf_t  *priv                        = NULL;
-        xlator_t         *this                        = NULL;
-        char              mount_point_path[PATH_MAX]  = {0,};
-        int               ret                         = -1;
+        int             ret   = -1;
+        int             mntfd = -1;
 
-        this = THIS;
-        priv = this->private;
-
-        snprintf (mount_point_path, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname, RB_CLIENT_MOUNTPOINT);
+        ret = rb_get_mntfd (&mntfd);
+        if (ret)
+                goto out;
 
-        ret = sys_lsetxattr (mount_point_path, xattr_key, value,
-                             strlen (value) + 1, 0);
+        ret = sys_fsetxattr (mntfd, xattr_key, value, strlen (value) + 1, 0);
         if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "setxattr on key: "
-                        "%s failed", xattr_key);
+                gf_log (THIS->name, GF_LOG_DEBUG, "setxattr on key: "
+                        "%s, reason: %s", xattr_key, strerror (errno));
+
+out:
         return ret;
 }
 
@@ -622,9 +737,7 @@ rb_spawn_dst_brick (glusterd_volinfo_t *volinfo,
 
         priv = THIS->private;
 
-        port = pmap_registry_alloc (THIS);
-        brickinfo->port = port;
-
+        port = brickinfo->port;
         GF_ASSERT (port);
 
         runinit (&runner);
@@ -635,9 +748,13 @@ rb_spawn_dst_brick (glusterd_volinfo_t *volinfo,
                           priv->workdir, volinfo->volname);
         runner_add_arg (&runner, "--xlator-option");
         runner_argprintf (&runner, "src-server.listen-port=%d", port);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
 
-        ret = runner_run (&runner);
+        ret = runner_run_nowait (&runner);
         if (ret) {
+                pmap_registry_remove (THIS, 0, brickinfo->path,
+                                      GF_PMAP_PORT_BRICKSERVER, NULL);
                 gf_log ("", GF_LOG_DEBUG,
                         "Could not start glusterfs");
                 goto out;
@@ -657,51 +774,70 @@ static int
 rb_spawn_glusterfs_client (glusterd_volinfo_t *volinfo,
                            glusterd_brickinfo_t *brickinfo)
 {
-        glusterd_conf_t    *priv           = NULL;
-        char                cmd_str[8192]  = {0,};
-        runner_t            runner         = {0,};
-        struct stat         buf;
-        int                 ret            = -1;
+        xlator_t           *this            = NULL;
+        glusterd_conf_t    *priv            = NULL;
+        runner_t            runner          = {0,};
+        struct stat         buf             = {0,};
+        char                mntpt[PATH_MAX] = {0,};
+        int                 mntfd           = -1;
+        int                 ret             = -1;
 
-        priv = THIS->private;
+        this = THIS;
+        priv = this->private;
 
+        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
         runinit (&runner);
         runner_add_arg (&runner, SBIN_DIR"/glusterfs");
         runner_argprintf (&runner, "-f" "%s/vols/%s/"RB_CLIENTVOL_FILENAME,
                           priv->workdir, volinfo->volname);
-        runner_argprintf (&runner, "%s/vols/%s/"RB_CLIENT_MOUNTPOINT,
-                          priv->workdir, volinfo->volname);
+        runner_add_arg (&runner, mntpt);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
 
-        ret = runner_run (&runner);
+        ret = runner_run_reuse (&runner);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "Could not start glusterfs");
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Could not start glusterfs");
+                runner_end (&runner);
                 goto out;
+        } else {
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Successfully started  glusterfs");
+                runner_end (&runner);
         }
 
-        gf_log ("", GF_LOG_DEBUG,
-                "Successfully started glusterfs: brick=%s:%s",
-                brickinfo->hostname, brickinfo->path);
+        ret = stat (mntpt, &buf);
+        if (ret) {
+                 gf_log (this->name, GF_LOG_DEBUG, "stat on mount point %s "
+                         "failed", mntpt);
+                 goto out;
+        }
 
-        memset (cmd_str, 0, sizeof (cmd_str));
+        mntfd = open (mntpt, O_DIRECTORY);
+        if (mntfd == -1)
+                goto out;
 
-        snprintf (cmd_str, 4096, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENT_MOUNTPOINT);
+        ret = rb_set_mntfd (mntfd);
+        if (ret)
+                goto out;
 
-        ret = stat (cmd_str, &buf);
+        runinit (&runner);
+        runner_add_args (&runner, "/bin/umount", "-l", mntpt, NULL);
+        ret = runner_run_reuse (&runner);
         if (ret) {
-                 gf_log ("", GF_LOG_DEBUG,
-                         "stat on mountpoint failed");
-                 goto out;
-         }
-
-         gf_log ("", GF_LOG_DEBUG,
-                 "stat on mountpoint succeeded");
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Lazy unmount failed on maintenance client");
+                runner_end (&runner);
+                goto out;
+        } else {
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Successfully unmounted  maintenance client");
+                runner_end (&runner);
+        }
 
-        ret = 0;
 
 out:
+
         return ret;
 }
 
@@ -711,6 +847,8 @@ static const char *client_volfile_str =  "volume mnt-client\n"
         " option remote-subvolume %s\n"
         " option remote-port %d\n"
         " option transport-type %s\n"
+        " option username %s\n"
+        " option password %s\n"
         "end-volume\n"
         "volume mnt-wb\n"
         " type performance/write-behind\n"
@@ -722,23 +860,33 @@ rb_generate_client_volfile (glusterd_volinfo_t *volinfo,
                             glusterd_brickinfo_t *src_brickinfo)
 {
         glusterd_conf_t  *priv                  = NULL;
+        xlator_t         *this                  = NULL;
         FILE             *file                  = NULL;
         char              filename[PATH_MAX]    = {0, };
         int               ret                   = -1;
+        int               fd                    = -1;
         char             *ttype                 = NULL;
 
-        priv = THIS->private;
+        this = THIS;
+        priv = this->private;
 
-        gf_log ("", GF_LOG_DEBUG,
-                "Creating volfile");
+        gf_log (this->name, GF_LOG_DEBUG, "Creating volfile");
 
         snprintf (filename, PATH_MAX, "%s/vols/%s/%s",
                   priv->workdir, volinfo->volname,
                   RB_CLIENTVOL_FILENAME);
 
+        fd = open (filename, O_CREAT | O_RDONLY, S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s", strerror (errno));
+                goto out;
+        }
+        close (fd);
+
         file = fopen (filename, "w+");
         if (!file) {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Open of volfile failed");
                 ret = -1;
                 goto out;
@@ -753,7 +901,10 @@ rb_generate_client_volfile (glusterd_volinfo_t *volinfo,
 	}
 
         fprintf (file, client_volfile_str, src_brickinfo->hostname,
-                 src_brickinfo->path, src_brickinfo->port, ttype);
+                 src_brickinfo->path,
+                 src_brickinfo->port, ttype,
+                 glusterd_auth_get_username (volinfo),
+                 glusterd_auth_get_password (volinfo));
 
         fclose (file);
         GF_FREE (ttype);
@@ -775,6 +926,8 @@ static const char *dst_brick_volfile_str = "volume src-posix\n"
         "end-volume\n"
         "volume src-server\n"
         " type protocol/server\n"
+        " option auth.login.%s.allow %s\n"
+        " option auth.login.%s.password %s\n"
         " option auth.addr.%s.allow *\n"
         " option transport-type %s\n"
         " subvolumes %s\n"
@@ -785,23 +938,34 @@ rb_generate_dst_brick_volfile (glusterd_volinfo_t *volinfo,
                                glusterd_brickinfo_t *dst_brickinfo)
 {
         glusterd_conf_t    *priv                = NULL;
+        xlator_t           *this                = NULL;
         FILE               *file                = NULL;
         char                filename[PATH_MAX]  = {0, };
         int                 ret                 = -1;
+        int                 fd                  = -1;
         char               *trans_type          = NULL;
 
-        priv = THIS->private;
+        this = THIS;
+        priv = this->private;
 
-        gf_log ("", GF_LOG_DEBUG,
+        gf_log (this->name, GF_LOG_DEBUG,
                 "Creating volfile");
 
         snprintf (filename, PATH_MAX, "%s/vols/%s/%s",
                   priv->workdir, volinfo->volname,
                   RB_DSTBRICKVOL_FILENAME);
 
+        fd = creat (filename, S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s", strerror (errno));
+                goto out;
+        }
+        close (fd);
+
         file = fopen (filename, "w+");
         if (!file) {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Open of volfile failed");
                 ret = -1;
                 goto out;
@@ -813,10 +977,17 @@ rb_generate_dst_brick_volfile (glusterd_volinfo_t *volinfo,
 		goto out;
 	}
 
-        fprintf (file, dst_brick_volfile_str, dst_brickinfo->path,
+        fprintf (file, dst_brick_volfile_str,
+                 dst_brickinfo->path,
                  uuid_utoa (volinfo->volume_id),
-                 dst_brickinfo->path, dst_brickinfo->path,
-                 trans_type, dst_brickinfo->path);
+                 dst_brickinfo->path,
+                 dst_brickinfo->path,
+                 glusterd_auth_get_username (volinfo),
+                 glusterd_auth_get_username (volinfo),
+                 glusterd_auth_get_password (volinfo),
+                 dst_brickinfo->path,
+                 trans_type,
+                 dst_brickinfo->path);
 
 	GF_FREE (trans_type);
 
@@ -833,20 +1004,14 @@ static int
 rb_mountpoint_mkdir (glusterd_volinfo_t *volinfo,
                      glusterd_brickinfo_t *src_brickinfo)
 {
-        glusterd_conf_t *priv                       = NULL;
-        char             mount_point_path[PATH_MAX] = {0,};
-        int              ret                        = -1;
+        char             mntpt[PATH_MAX]        = {0,};
+        int              ret                    = -1;
 
-        priv = THIS->private;
-
-        snprintf (mount_point_path, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENT_MOUNTPOINT);
-
-        ret = mkdir (mount_point_path, 0777);
+        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
+        ret = mkdir (mntpt, 0777);
         if (ret && (errno != EEXIST)) {
-                gf_log ("", GF_LOG_DEBUG, "mkdir failed, errno: %d",
-                        errno);
+                gf_log ("", GF_LOG_DEBUG, "mkdir failed, due to %s",
+                        strerror (errno));
                 goto out;
         }
 
@@ -860,19 +1025,14 @@ static int
 rb_mountpoint_rmdir (glusterd_volinfo_t *volinfo,
                      glusterd_brickinfo_t *src_brickinfo)
 {
-        glusterd_conf_t *priv                       = NULL;
-        char             mount_point_path[PATH_MAX] = {0,};
+        char             mntpt[PATH_MAX] = {0,};
         int              ret                        = -1;
 
-        priv = THIS->private;
-
-        snprintf (mount_point_path, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENT_MOUNTPOINT);
-
-        ret = rmdir (mount_point_path);
+        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
+        ret = rmdir (mntpt);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "rmdir failed");
+                gf_log ("", GF_LOG_DEBUG, "rmdir failed, reason: %s",
+                        strerror (errno));
                 goto out;
         }
 
@@ -884,42 +1044,42 @@ out:
 
 static int
 rb_destroy_maintenance_client (glusterd_volinfo_t *volinfo,
-                                glusterd_brickinfo_t *src_brickinfo)
+                               glusterd_brickinfo_t *src_brickinfo)
 {
+        xlator_t         *this                        = NULL;
         glusterd_conf_t  *priv                        = NULL;
-        runner_t          runner                      = {0,};
-        char              filename[PATH_MAX]          = {0,};
+        char              volfile[PATH_MAX]           = {0,};
         int               ret                         = -1;
+        int               mntfd                       = -1;
 
-        priv = THIS->private;
+        this = THIS;
+        priv = this->private;
 
-        runinit (&runner);
-        runner_add_args (&runner, "/bin/umount", "-f", NULL);
-        runner_argprintf (&runner, "%s/vols/%s/"RB_CLIENT_MOUNTPOINT,
-                          priv->workdir, volinfo->volname);
+        ret = rb_get_mntfd (&mntfd);
+        if (ret)
+                goto out;
 
-        ret = runner_run (&runner);
+        ret = close (mntfd);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "umount failed on maintenance client");
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to close mount "
+                        "point directory");
                 goto out;
         }
 
         ret = rb_mountpoint_rmdir (volinfo, src_brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "rmdir of mountpoint failed");
+                gf_log (this->name, GF_LOG_DEBUG, "rmdir of mountpoint "
+                        "failed");
                 goto out;
         }
 
-        snprintf (filename, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENTVOL_FILENAME);
+        snprintf (volfile, PATH_MAX, "%s/vols/%s/%s", priv->workdir,
+                  volinfo->volname, RB_CLIENTVOL_FILENAME);
 
-        ret = unlink (filename);
+        ret = unlink (volfile);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "unlink failed");
+                gf_log ("", GF_LOG_DEBUG, "unlink of %s failed, reason: %s",
+                        volfile, strerror (errno));
                 goto out;
         }
 
@@ -1008,23 +1168,18 @@ rb_get_xattr_command (glusterd_volinfo_t *volinfo,
                       const char *xattr_key,
                       char *value)
 {
-        glusterd_conf_t  *priv                        = NULL;
-        xlator_t         *this                        = NULL;
-        char              mount_point_path[PATH_MAX]  = {0,};
-        int               ret                         = -1;
+        int             ret    = -1;
+        int             mntfd  = -1;
 
-        this = THIS;
-        priv = THIS->private;
-
-        snprintf (mount_point_path, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENT_MOUNTPOINT);
+        ret = rb_get_mntfd (&mntfd);
+        if (ret)
+                goto out;
 
-        ret = sys_lgetxattr (mount_point_path, xattr_key, value, 8192);
+        ret = sys_fgetxattr (mntfd, xattr_key, value, 8192);
 
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "getxattr on key: %s "
-                        "failed", xattr_key);
+                gf_log (THIS->name, GF_LOG_DEBUG, "getxattr on key: %s "
+                        "failed, reason: %s", xattr_key, strerror (errno));
                 goto out;
         }
 
@@ -1039,11 +1194,17 @@ rb_send_cmd (glusterd_volinfo_t *volinfo,
              glusterd_brickinfo_t *dst,
              gf1_cli_replace_op op)
 {
-        char         start_value[8192] = {0,};
-        char         status[8192]      = {0,};
-        char        *status_reply      = NULL;
-        dict_t      *ctx               = NULL;
-        int          ret               = 0;
+        char         start_value[8192]          = {0,};
+        char         status_str[8192]           = {0,};
+        char        *status_reply               = NULL;
+        char        *tmp                        = NULL;
+        char        *save_ptr                   = NULL;
+        char         filename[PATH_MAX]         = {0,};
+        char        *current_file               = NULL;
+        uint64_t     files                      = 0;
+        int          status                     = 0;
+        dict_t      *ctx                        = NULL;
+        int          ret                        = 0;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (src);
@@ -1087,7 +1248,7 @@ rb_send_cmd (glusterd_volinfo_t *volinfo,
                 {
                         ret = rb_get_xattr_command (volinfo, src, dst,
                                                     RB_PUMP_CMD_STATUS,
-                                                    status);
+                                                    status_str);
                         if (ret)
                                 goto out;
 
@@ -1100,7 +1261,76 @@ rb_send_cmd (glusterd_volinfo_t *volinfo,
                                 goto out;
                         }
 
-                        status_reply = gf_strdup (status);
+                        /* Split status reply into different parts */
+                        tmp = strtok_r (status_str, ":", &save_ptr);
+                        if (!tmp) {
+                                ret = -1;
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Couldn't tokenize status string");
+                                goto out;
+                        }
+                        sscanf (tmp, "status=%d", &status);
+                        ret = dict_set_int32 (ctx, "status", status);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't "
+                                        "set rb status in context");
+                                goto out;
+                        }
+
+                        tmp = NULL;
+                        tmp = strtok_r (NULL, ":", &save_ptr);
+                        if (!tmp) {
+                                ret = -1;
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Couldn't tokenize status string");
+                                goto out;
+                        }
+                        sscanf (tmp, "no_of_files=%"SCNu64, &files);
+                        ret = dict_set_uint64 (ctx, "files", files);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't "
+                                        "set rb files in context");
+                                goto out;
+                        }
+
+                        if (status == 0) {
+                                tmp = NULL;
+                                tmp = strtok_r (NULL, ":", &save_ptr);
+                                if (!tmp) {
+                                        ret = -1;
+                                        gf_log (THIS->name, GF_LOG_ERROR,
+                                                "Couldn't tokenize status "
+                                                "string");
+                                        goto out;
+                                }
+                                sscanf (tmp, "current_file=%s", filename);
+                                current_file = gf_strdup (filename);
+                                ret = dict_set_dynstr (ctx, "current_file",
+                                                       current_file);
+                                if (ret) {
+                                        GF_FREE (current_file);
+                                        gf_log (THIS->name, GF_LOG_ERROR,
+                                                "Couldn't set rb current file "
+                                                "in context");
+                                        goto out;
+                                }
+                        }
+                        if (status) {
+                                ret = gf_asprintf (&status_reply,
+                                                  "Number of files migrated = %"
+                                                  PRIu64"\tMigration complete",
+                                                  files);
+                        } else {
+                                ret = gf_asprintf (&status_reply,
+                                                  "Number of files migrated = %"
+                                                  PRIu64"\tCurrent file = %s",
+                                                  files, filename);
+                        }
+                        if (ret == -1) {
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Failed to create status_reply string");
+                                goto out;
+                        }
                         ret = dict_set_dynstr (ctx, "status-reply",
                                                status_reply);
                         if (ret) {
@@ -1174,14 +1404,11 @@ rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict,
 
         this = THIS;
 
-        ctx = glusterd_op_get_ctx ();
-        if (ctx) {
-                dict_ret = dict_get_int32 (req_dict, "src-brick-port", &src_port);
-                if (src_port)
-                        src_brickinfo->port = src_port;
-        }
+        dict_ret = dict_get_int32 (req_dict, "src-brick-port", &src_port);
+        if (src_port)
+                src_brickinfo->port = src_port;
 
-        if (!glusterd_is_local_addr (src_brickinfo->hostname)) {
+        if (gf_is_local_addr (src_brickinfo->hostname)) {
                 gf_log ("", GF_LOG_INFO,
                         "adding src-brick port no");
 
@@ -1230,15 +1457,12 @@ rb_update_dstbrick_port (glusterd_brickinfo_t *dst_brickinfo, dict_t *rsp_dict,
         int     dict_ret      = 0;
         int     dst_port      = 0;
 
-        ctx = glusterd_op_get_ctx ();
-        if (ctx) {
-                dict_ret = dict_get_int32 (req_dict, "dst-brick-port", &dst_port);
-                if (dst_port)
-                        dst_brickinfo->port = dst_port;
+        dict_ret = dict_get_int32 (req_dict, "dst-brick-port", &dst_port);
+        if (!dict_ret)
+                dst_brickinfo->port = dst_port;
 
-        }
 
-        if (!glusterd_is_local_addr (dst_brickinfo->hostname)) {
+        if (gf_is_local_addr (dst_brickinfo->hostname)) {
                 gf_log ("", GF_LOG_INFO,
                         "adding dst-brick port no");
 
@@ -1277,18 +1501,18 @@ glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,
 
         GF_ASSERT (volinfo);
 
-        ret = glusterd_brickinfo_from_brick (new_brick,
-                                             &new_brickinfo);
+        ret = glusterd_brickinfo_new_from_brick (new_brick,
+                                                 &new_brickinfo);
         if (ret)
                 goto out;
 
-        ret = glusterd_volume_brickinfo_get_by_brick (old_brick, volinfo,
-                                                      &old_brickinfo,
-                                                      GF_PATH_COMPLETE);
+        ret = glusterd_resolve_brick (new_brickinfo);
+
         if (ret)
                 goto out;
 
-        ret = glusterd_resolve_brick (new_brickinfo);
+        ret = glusterd_volume_brickinfo_get_by_brick (old_brick,
+                                                      volinfo, &old_brickinfo);
         if (ret)
                 goto out;
 
@@ -1306,13 +1530,13 @@ glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,
                 goto out;
 
         if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                ret = glusterd_brick_start (volinfo, new_brickinfo);
+                ret = glusterd_brick_start (volinfo, new_brickinfo, _gf_false);
                 if (ret)
                         goto out;
         }
 
-
 out:
+
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
@@ -1331,6 +1555,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
         char                                    *dst_brick = NULL;
         glusterd_brickinfo_t                    *src_brickinfo = NULL;
         glusterd_brickinfo_t                    *dst_brickinfo = NULL;
+        char                                    *task_id_str = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -1340,26 +1565,23 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
 
         ret = dict_get_str (dict, "src-brick", &src_brick);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get src brick");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get src brick");
                 goto out;
         }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "src brick=%s", src_brick);
+        gf_log (this->name, GF_LOG_DEBUG, "src brick=%s", src_brick);
 
         ret = dict_get_str (dict, "dst-brick", &dst_brick);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get dst brick");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get dst brick");
                 goto out;
         }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "dst brick=%s", dst_brick);
+        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
 
         ret = dict_get_str (dict, "volname", &volname);
-
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
@@ -1372,29 +1594,30 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
                 goto out;
         }
 
         ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo,
-                                                      GF_PATH_COMPLETE);
+                                                      &src_brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to get src-brickinfo");
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to get src-brickinfo");
                 goto out;
         }
 
 
         ret = glusterd_get_rb_dst_brickinfo (volinfo, &dst_brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get "
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get "
                          "replace brick destination brickinfo");
                 goto out;
         }
 
         ret = glusterd_resolve_brick (dst_brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to resolve dst-brickinfo");
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to resolve dst-brickinfo");
                 goto out;
         }
 
@@ -1403,45 +1626,82 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
         if (ret)
                 goto out;
 
+
 	if ((GF_REPLACE_OP_START != replace_op)) {
-                ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
-                                               dict, replace_op);
-                if (ret)
-                        goto out;
+
+                /* Set task-id, if available, in op_ctx dict for operations
+                 * other than start
+                 */
+                if  (is_origin_glusterd (dict)) {
+                        ctx = glusterd_op_get_ctx();
+                        if (!ctx) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "get op_ctx");
+                                ret = -1;
+                                goto out;
+                        }
+                        if (!uuid_is_null (volinfo->rep_brick.rb_id)) {
+                                ret = glusterd_copy_uuid_to_dict
+                                        (volinfo->rep_brick.rb_id, ctx,
+                                         GF_REPLACE_BRICK_TID_KEY);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to set "
+                                                "replace-brick-id");
+                                        goto out;
+                                }
+                        }
+                }
 	}
+        ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
+                                       dict, replace_op);
+        if (ret)
+                goto out;
 
         switch (replace_op) {
         case GF_REPLACE_OP_START:
         {
-                if (!glusterd_is_local_addr (dst_brickinfo->hostname)) {
-                        gf_log ("", GF_LOG_INFO,
+                ret = dict_get_str (dict, GF_REPLACE_BRICK_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Missing replace-brick-id");
+                        ret = 0;
+                } else {
+                        uuid_parse (task_id_str, volinfo->rep_brick.rb_id);
+                }
+
+                if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                        gf_log (this->name, GF_LOG_INFO,
                                 "I AM THE DESTINATION HOST");
                         if (!glusterd_is_rb_paused (volinfo)) {
-                                ret = rb_spawn_destination_brick (volinfo, dst_brickinfo);
+                                ret = rb_spawn_destination_brick
+                                        (volinfo, dst_brickinfo);
                                 if (ret) {
-                                        gf_log ("", GF_LOG_DEBUG,
-                                                "Failed to spawn destination brick");
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "Failed to spawn destination "
+                                                "brick");
                                         goto out;
                                 }
                         } else {
-                                gf_log ("", GF_LOG_ERROR, "Replace brick is already "
-                                        "started=> no need to restart dst brick ");
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Replace brick is already started=> no "
+                                        "need to restart dst brick ");
                         }
 		}
 
 
-		if (!glusterd_is_local_addr (src_brickinfo->hostname)) {
+		if (gf_is_local_addr (src_brickinfo->hostname)) {
 		        ret = rb_src_brick_restart (volinfo, src_brickinfo,
 				                    1);
 		        if (ret) {
-				gf_log ("", GF_LOG_DEBUG,
-	                        "Could not restart src-brick");
+				gf_log (this->name, GF_LOG_DEBUG,
+	                                "Could not restart src-brick");
 			        goto out;
 			}
 		}
 
-		if (!glusterd_is_local_addr (dst_brickinfo->hostname)) {
-			gf_log ("", GF_LOG_INFO,
+		if (gf_is_local_addr (dst_brickinfo->hostname)) {
+			gf_log (this->name, GF_LOG_INFO,
 				"adding dst-brick port no");
 
                         ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
@@ -1455,7 +1715,6 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
 	}
 
         case GF_REPLACE_OP_COMMIT:
-        case GF_REPLACE_OP_COMMIT_FORCE:
         {
                 ctx = glusterd_op_get_ctx ();
                 if (ctx) {
@@ -1463,71 +1722,64 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                                                dst_brickinfo,
                                                GF_REPLACE_OP_COMMIT);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "Commit operation failed");
                                 goto out;
                         }
                 }
-
-                ret = dict_set_int32 (volinfo->dict, "enable-pump", 0);
-                gf_log ("", GF_LOG_DEBUG,
-                        "Received commit - will be adding dst brick and "
-                        "removing src brick");
-
-                if (!glusterd_is_local_addr (dst_brickinfo->hostname) &&
-                    replace_op != GF_REPLACE_OP_COMMIT_FORCE) {
-                        gf_log ("", GF_LOG_INFO,
+        }
+                /* fall through */
+        case GF_REPLACE_OP_COMMIT_FORCE:
+        {
+                if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "I AM THE DESTINATION HOST");
-                        ret = rb_kill_destination_brick (volinfo, dst_brickinfo);
+                        ret = rb_kill_destination_brick (volinfo,
+                                                         dst_brickinfo);
                         if (ret) {
-                                gf_log ("", GF_LOG_DEBUG,
-                                        "Failed to kill destination brick");
+                                gf_log (this->name, GF_LOG_CRITICAL,
+                                        "Unable to cleanup dst brick");
                                 goto out;
                         }
                 }
 
-                if (ret) {
-                        gf_log ("", GF_LOG_CRITICAL,
-                                "Unable to cleanup dst brick");
-                        goto out;
-                }
-
-
                 ret = glusterd_nodesvcs_stop (volinfo);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Unable to stop nfs server, ret: %d", ret);
                 }
 
 		ret = glusterd_op_perform_replace_brick (volinfo, src_brick,
 							 dst_brick);
 		if (ret) {
-			gf_log ("", GF_LOG_CRITICAL, "Unable to add "
-				"dst-brick: %s to volume: %s",
-				dst_brick, volinfo->volname);
+			gf_log (this->name, GF_LOG_CRITICAL, "Unable to add "
+				"dst-brick: %s to volume: %s", dst_brick,
+                                volinfo->volname);
 		        (void) glusterd_nodesvcs_handle_graph_change (volinfo);
 			goto out;
 		}
 
-		volinfo->defrag_status = 0;
+		volinfo->rebal.defrag_status = 0;
 
 		ret = glusterd_nodesvcs_handle_graph_change (volinfo);
 		if (ret) {
-                        gf_log ("", GF_LOG_CRITICAL,
+                        gf_log (this->name, GF_LOG_CRITICAL,
                                 "Failed to generate nfs volume file");
 		}
 
 
 		ret = glusterd_fetchspec_notify (THIS);
                 glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE);
-                glusterd_brickinfo_delete (volinfo->dst_brick);
-                volinfo->src_brick = volinfo->dst_brick = NULL;
+                glusterd_brickinfo_delete (volinfo->rep_brick.dst_brick);
+                volinfo->rep_brick.src_brick = NULL;
+                volinfo->rep_brick.dst_brick = NULL;
+                uuid_clear (volinfo->rep_brick.rb_id);
         }
         break;
 
         case GF_REPLACE_OP_PAUSE:
         {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Received pause - doing nothing");
                 ctx = glusterd_op_get_ctx ();
                 if (ctx) {
@@ -1535,7 +1787,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                                                dst_brickinfo,
                                                GF_REPLACE_OP_PAUSE);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "Pause operation failed");
                                 goto out;
                         }
@@ -1554,18 +1806,13 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                                                dst_brickinfo,
                                                GF_REPLACE_OP_ABORT);
                         if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "Abort operation failed");
                                 goto out;
                         }
                 }
 
-                ret = dict_set_int32 (volinfo->dict, "enable-pump", 0);
-		if (ret) {
-			gf_log (THIS->name, GF_LOG_CRITICAL, "Unable to disable pump");
-		}
-
-                if (!glusterd_is_local_addr (src_brickinfo->hostname)) {
+                if (gf_is_local_addr (src_brickinfo->hostname)) {
                         ret = rb_src_brick_restart (volinfo, src_brickinfo,
                                                     0);
                         if (ret) {
@@ -1576,8 +1823,8 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                         }
                 }
 
-                if (!glusterd_is_local_addr (dst_brickinfo->hostname)) {
-                        gf_log (THIS->name, GF_LOG_INFO,
+                if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                        gf_log (this->name, GF_LOG_INFO,
                                 "I AM THE DESTINATION HOST");
                         ret = rb_kill_destination_brick (volinfo, dst_brickinfo);
                         if (ret) {
@@ -1587,14 +1834,15 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                         }
                 }
                 glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE);
-                glusterd_brickinfo_delete (volinfo->dst_brick);
-                volinfo->src_brick = volinfo->dst_brick = NULL;
+                glusterd_brickinfo_delete (volinfo->rep_brick.dst_brick);
+                volinfo->rep_brick.src_brick = NULL;
+                volinfo->rep_brick.dst_brick = NULL;
         }
         break;
 
         case GF_REPLACE_OP_STATUS:
         {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "received status - doing nothing");
                 ctx = glusterd_op_get_ctx ();
                 if (ctx) {
@@ -1602,7 +1850,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                                 ret = dict_set_str (ctx, "status-reply",
                                                  "replace brick has been paused");
                                 if (ret)
-                                        gf_log (THIS->name, GF_LOG_ERROR,
+                                        gf_log (this->name, GF_LOG_ERROR,
                                                 "failed to set pump status"
                                                 " in ctx");
                                 goto out;
@@ -1626,7 +1874,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
 		ret = glusterd_store_volinfo (volinfo,
                                               GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't store"
+                gf_log (this->name, GF_LOG_ERROR, "Couldn't store"
                         " replace brick operation's state");
 
 out:
@@ -1647,6 +1895,7 @@ glusterd_do_replace_brick (void *data)
         glusterd_brickinfo_t   *src_brickinfo = NULL;
         glusterd_brickinfo_t   *dst_brickinfo = NULL;
 	glusterd_conf_t	       *priv = NULL;
+        uuid_t                 *txn_id = &global_txn_id;
 
         int ret = 0;
 
@@ -1666,6 +1915,10 @@ glusterd_do_replace_brick (void *data)
         gf_log ("", GF_LOG_DEBUG,
                 "Replace brick operation detected");
 
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         ret = dict_get_int32 (dict, "operation", &op);
         if (ret) {
                 gf_log ("", GF_LOG_DEBUG,
@@ -1704,8 +1957,7 @@ glusterd_do_replace_brick (void *data)
         }
 
         ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo,
-                                                      GF_PATH_COMPLETE);
+                                                      &src_brickinfo);
         if (ret) {
                 gf_log ("", GF_LOG_DEBUG, "Unable to get src-brickinfo");
                 goto out;
@@ -1762,12 +2014,11 @@ glusterd_do_replace_brick (void *data)
 
 out:
         if (ret)
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                                   txn_id, NULL);
         else
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC, NULL);
-
-        if (dict)
-                dict_unref (dict);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC,
+                                                   txn_id, NULL);
 
         glusterd_op_sm ();
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index 21d8ab0ac..d5200a4ae 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -43,6 +33,7 @@
 
 
 extern glusterd_op_info_t opinfo;
+extern uuid_t global_txn_id;
 
 int32_t
 glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
@@ -59,10 +50,11 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         int32_t         status = 0;
         int32_t         count = 0;
         gf_cli_rsp      rsp = {0,};
+        xlator_t        *this = NULL;
 
-        GF_ASSERT (THIS);
-
-        conf = THIS->private;
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
 
         GF_ASSERT (conf);
 
@@ -82,16 +74,18 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
                 break;
         }
         case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
         {
                 if (ctx) {
                         ret = dict_get_int32 (ctx, "status", &status);
                         if (ret) {
-                                gf_log (THIS->name, GF_LOG_TRACE,
+                                gf_log (this->name, GF_LOG_TRACE,
                                         "failed to get status");
                         }
                 }
                 break;
         }
+        case GD_OP_GSYNC_CREATE:
         case GD_OP_GSYNC_SET:
         {
                if (ctx) {
@@ -115,7 +109,7 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
                 if (ctx && dict_get_int32 (ctx, "count", &count)) {
                         ret = dict_set_int32 (ctx, "count", 0);
                         if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "failed to set count in dictionary");
                         }
                 }
@@ -124,13 +118,14 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         case GD_OP_START_BRICK:
         case GD_OP_STOP_BRICK:
         {
-                gf_log ("", GF_LOG_DEBUG, "not supported op %d", op);
+                gf_log (this->name, GF_LOG_DEBUG, "op '%s' not supported",
+                        gd_op_list[op]);
                 break;
         }
         case GD_OP_NONE:
         case GD_OP_MAX:
         {
-                gf_log ("", GF_LOG_ERROR, "invalid operation %d", op);
+                gf_log (this->name, GF_LOG_ERROR, "invalid operation");
                 break;
         }
         case GD_OP_CREATE_VOLUME:
@@ -141,15 +136,33 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         case GD_OP_ADD_BRICK:
         case GD_OP_LOG_ROTATE:
         case GD_OP_SYNC_VOLUME:
-        case GD_OP_HEAL_VOLUME:
         case GD_OP_STATEDUMP_VOLUME:
         case GD_OP_REPLACE_BRICK:
         case GD_OP_STATUS_VOLUME:
         case GD_OP_SET_VOLUME:
+        case GD_OP_LIST_VOLUME:
+        case GD_OP_CLEARLOCKS_VOLUME:
+        case GD_OP_HEAL_VOLUME:
+        case GD_OP_SNAP:
         {
                 /*nothing specific to be done*/
                 break;
         }
+        case GD_OP_COPY_FILE:
+        {
+               if (ctx)
+                        ret = dict_get_str (ctx, "errstr", &errstr);
+               break;
+        }
+        case GD_OP_SYS_EXEC:
+        {
+               if (ctx) {
+                        ret = dict_get_str (ctx, "errstr", &errstr);
+                        ret = dict_set_str (ctx, "glusterd_workdir",
+                                            conf->workdir);
+               }
+               break;
+        }
         }
 
         rsp.op_ret = op_ret;
@@ -164,9 +177,9 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
 
         if (ctx) {
                 ret = dict_allocate_and_serialize (ctx, &rsp.dict.dict_val,
-                                                   (size_t*)&rsp.dict.dict_len);
+                                                   &rsp.dict.dict_len);
                 if (ret < 0 )
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                 "serialize buffer");
                 else
                         free_ptr = rsp.dict.dict_val;
@@ -179,17 +192,31 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         cli_rsp = &rsp;
         xdrproc = (xdrproc_t) xdr_gf_cli_rsp;
 
-        ret = glusterd_submit_reply (req, cli_rsp, NULL, 0, NULL,
-                                     xdrproc);
+        glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                         xdrproc, ctx);
+        ret = 0;
 
-        if (free_ptr)
-                GF_FREE (free_ptr);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        GF_FREE (free_ptr);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
-glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
+glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe, fop_cbk_fn_t fn)
+{
+        glusterd_conf_t *priv = THIS->private;
+        int             ret   = -1;
+
+        synclock_lock (&priv->big_lock);
+        ret = fn (req, iov, count, myframe);
+        synclock_unlock (&priv->big_lock);
+
+        return ret;
+}
+
+int
+__glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
                         int count, void *myframe)
 {
         gd1_mgmt_probe_rsp    rsp   = {{0},};
@@ -222,7 +249,9 @@ glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
                 if (ctx->req) {
                         glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
                                                       rsp.op_errno,
-                                                      ctx->hostname, ctx->port);
+                                                      rsp.op_errstr,
+                                                      ctx->hostname, ctx->port,
+                                                      ctx->dict);
                 }
 
                 glusterd_destroy_probe_ctx (ctx);
@@ -249,7 +278,9 @@ glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
                 if (ctx->req) {
                         glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
                                                       rsp.op_errno,
-                                                      ctx->hostname, ctx->port);
+                                                      rsp.op_errstr,
+                                                      ctx->hostname, ctx->port,
+                                                      ctx->dict);
                 }
 
                 glusterd_destroy_probe_ctx (ctx);
@@ -283,14 +314,22 @@ glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
         gf_log ("glusterd", GF_LOG_INFO, "Received resp to probe req");
 
 out:
-        if (rsp.hostname)
-                free (rsp.hostname);//malloced by xdr
+        free (rsp.hostname);//malloced by xdr
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
 int
-glusterd3_1_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
+glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_probe_cbk);
+}
+
+
+int
+__glusterd_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
                             int count, void *myframe)
 {
         gd1_mgmt_friend_rsp           rsp   = {{0},};
@@ -369,21 +408,29 @@ out:
 
         if (ctx->req)//reverse probe doesn't have req
                 ret = glusterd_xfer_cli_probe_resp (ctx->req, op_ret, op_errno,
-                                                    ctx->hostname, ctx->port);
+                                                    NULL, ctx->hostname,
+                                                    ctx->port, ctx->dict);
         if (!ret) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
         }
         if (ctx)
                 glusterd_destroy_probe_ctx (ctx);
-        if (rsp.hostname)
-                free (rsp.hostname);//malloced by xdr
+        free (rsp.hostname);//malloced by xdr
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
 int
-glusterd3_1_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
+glusterd_friend_add_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_friend_add_cbk);
+}
+
+int
+__glusterd_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
                                int count, void *myframe)
 {
         gd1_mgmt_friend_rsp             rsp   = {{0},};
@@ -459,8 +506,8 @@ inject:
 
 
 respond:
-        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, op_ret, op_errno,
-                                              ctx->hostname);
+        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, op_ret, op_errno, NULL,
+                                              ctx->hostname, ctx->dict);
         if (!ret && move_sm_now) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
@@ -471,37 +518,62 @@ respond:
                 glusterd_destroy_probe_ctx (ctx);
         }
 
-        if (rsp.hostname)
-                free (rsp.hostname);//malloced by xdr
+        free (rsp.hostname);//malloced by xdr
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
+int
+glusterd_friend_remove_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_friend_remove_cbk);
+}
+
 int32_t
-glusterd3_1_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
                               int count, void *myframe)
 {
-        int                           ret   = -1;
-        int32_t                       op_ret = 0;
-        char                          str[50] = {0,};
+        int                           ret    = -1;
+        gd1_mgmt_friend_update_rsp    rsp    = {{0}, };
+        xlator_t                      *this  = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
 
         if (-1 == req->rpc_status) {
+                gf_log (this->name, GF_LOG_ERROR, "RPC Error");
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s",
-                (op_ret)?"RJT":"ACC", str);
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize friend"
+                        " update repsonse");
+                goto out;
+        }
 
+        ret = 0;
 out:
+        gf_log (this->name, GF_LOG_INFO, "Received %s from uuid: %s",
+                (ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
+int
+glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_friend_update_cbk);
+}
+
 int32_t
-glusterd3_1_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
                               int count, void *myframe)
 {
         gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
@@ -509,7 +581,11 @@ glusterd3_1_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
         int32_t                       op_ret = -1;
         glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
         glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = &global_txn_id;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
         if (-1 == req->rpc_status) {
@@ -520,7 +596,8 @@ glusterd3_1_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
+                        "response received from peer");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
                 goto out;
@@ -529,25 +606,28 @@ glusterd3_1_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         op_ret = rsp.op_ret;
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received lock %s from uuid: %s", (op_ret) ? "RJT" : "ACC",
+                uuid_utoa (rsp.uuid));
 
         ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Lock response received from "
-                        "unknown peer: %s", uuid_utoa (rsp.uuid));
+                gf_log (this->name, GF_LOG_CRITICAL, "Lock response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
         }
 
         if (op_ret) {
                 event_type = GD_OP_EVENT_RCVD_RJT;
                 opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
         } else {
                 event_type = GD_OP_EVENT_RCVD_ACC;
         }
 
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
@@ -559,16 +639,27 @@ out:
 }
 
 int32_t
-glusterd3_1_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
 {
-        gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_cluster_lock_cbk);
+}
+
+static int32_t
+glusterd_mgmt_v3_lock_peers_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                    int count, void *myframe)
+{
+        gd1_mgmt_v3_lock_rsp          rsp   = {{0},};
         int                           ret   = -1;
         int32_t                       op_ret = -1;
         glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
         glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
 
-
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
         if (-1 == req->rpc_status) {
@@ -577,9 +668,12 @@ glusterd3_1_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode mgmt_v3 lock "
+                        "response received from peer");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
                 goto out;
@@ -588,25 +682,30 @@ glusterd3_1_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         op_ret = rsp.op_ret;
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+        txn_id = &rsp.txn_id;
 
-        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received mgmt_v3 lock %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC", uuid_utoa (rsp.uuid));
 
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Unlock response received from "
-                        "unknown peer %s", uuid_utoa (rsp.uuid));
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "mgmt_v3 lock response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
         }
 
         if (op_ret) {
                 event_type = GD_OP_EVENT_RCVD_RJT;
                 opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
         } else {
                 event_type = GD_OP_EVENT_RCVD_ACC;
         }
 
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
@@ -617,197 +716,167 @@ out:
         return ret;
 }
 
-static int32_t
-glusterd_append_gsync_status (dict_t *dst, dict_t *src)
+int32_t
+glusterd_mgmt_v3_lock_peers_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
 {
-        int                ret = 0;
-        char               *stop_msg = NULL;
-
-        ret = dict_get_str (src, "gsync-status", &stop_msg);
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (dst, "gsync-status", gf_strdup (stop_msg));
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_WARNING, "Unable to set the stop"
-                        "message in the ctx dictionary");
-                goto out;
-        }
-
-        ret = 0;
- out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        glusterd_mgmt_v3_lock_peers_cbk_fn);
 }
 
 static int32_t
-glusterd_append_status_dicts (dict_t *dst, dict_t *src)
+glusterd_mgmt_v3_unlock_peers_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                      int count, void *myframe)
 {
-        int              dst_count = 0;
-        int              src_count = 0;
-        int              i = 0;
-        int              ret = 0;
-        char             mst[PATH_MAX] = {0,};
-        char             slv[PATH_MAX] = {0, };
-        char             sts[PATH_MAX] = {0, };
-        char             *mst_val = NULL;
-        char             *slv_val = NULL;
-        char             *sts_val = NULL;
-
-        GF_ASSERT (dst);
-
-        if (src == NULL)
-                goto out;
+        gd1_mgmt_v3_unlock_rsp        rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
 
-        ret = dict_get_int32 (dst, "gsync-count", &dst_count);
-        if (ret)
-                dst_count = 0;
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
 
-        ret = dict_get_int32 (src, "gsync-count", &src_count);
-        if (ret || !src_count) {
-                gf_log ("", GF_LOG_DEBUG, "Source brick empty");
-                ret = 0;
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
                 goto out;
         }
 
-        for (i = 1; i <= src_count; i++) {
-                snprintf (mst, sizeof(mst), "master%d", i);
-                snprintf (slv, sizeof(slv), "slave%d", i);
-                snprintf (sts, sizeof(sts), "status%d", i);
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode mgmt_v3 unlock "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
 
-                ret = dict_get_str (src, mst, &mst_val);
-                if (ret)
-                        goto out;
+out:
+        op_ret = rsp.op_ret;
 
-                ret = dict_get_str (src, slv, &slv_val);
-                if (ret)
-                        goto out;
+        txn_id = &rsp.txn_id;
 
-                ret = dict_get_str (src, sts, &sts_val);
-                if (ret)
-                        goto out;
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received mgmt_v3 unlock %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC",
+                uuid_utoa (rsp.uuid));
 
-                snprintf (mst, sizeof(mst), "master%d", i+dst_count);
-                snprintf (slv, sizeof(slv), "slave%d", i+dst_count);
-                snprintf (sts, sizeof(sts), "status%d", i+dst_count);
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
-                ret = dict_set_dynstr (dst, mst, gf_strdup (mst_val));
-                if (ret)
-                        goto out;
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "mgmt_v3 unlock response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
+        }
 
-                ret = dict_set_dynstr (dst, slv, gf_strdup (slv_val));
-                if (ret)
-                        goto out;
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+        }
 
-                ret = dict_set_dynstr (dst, sts, gf_strdup (sts_val));
-                if (ret)
-                        goto out;
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
         }
 
-        ret = dict_set_int32 (dst, "gsync-count", dst_count+src_count);
-
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
-
 }
 
-static int32_t
-glusterd_gsync_use_rsp_dict (dict_t *rsp_dict, char *op_errstr)
+int32_t
+glusterd_mgmt_v3_unlock_peers_cbk (struct rpc_req *req, struct iovec *iov,
+                                   int count, void *myframe)
 {
-        dict_t             *ctx = NULL;
-        int                ret = 0;
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        glusterd_mgmt_v3_unlock_peers_cbk_fn);
+}
 
-        ctx = glusterd_op_get_ctx ();
-        if (!ctx) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Operation Context is not present");
-                GF_ASSERT (0);
-        }
+int32_t
+__glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = &global_txn_id;
 
-        if (rsp_dict) {
-                ret = glusterd_append_status_dicts (ctx, rsp_dict);
-                if (ret)
-                        goto out;
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
 
-                ret = glusterd_append_gsync_status (ctx, rsp_dict);
-                if (ret)
-                        goto out;
-        }
-        if (strcmp ("", op_errstr)) {
-                ret = dict_set_dynstr (ctx, "errstr", gf_strdup(op_errstr));
-                if (ret)
-                        goto out;
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
         }
 
-        ret = 0;
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
-        return ret;
-}
-static int32_t
-glusterd_rb_use_rsp_dict (dict_t *rsp_dict)
-{
-        int32_t  src_port = 0;
-        int32_t  dst_port = 0;
-        int      ret      = 0;
-        dict_t  *ctx      = NULL;
-
-
-        ctx = glusterd_op_get_ctx ();
-        if (!ctx) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Operation Context is not present");
-                GF_ASSERT (0);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
         }
 
-        if (rsp_dict) {
-                ret = dict_get_int32 (rsp_dict, "src-brick-port", &src_port);
-                if (ret == 0) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "src-brick-port=%d found", src_port);
-                }
+out:
+        op_ret = rsp.op_ret;
 
-                ret = dict_get_int32 (rsp_dict, "dst-brick-port", &dst_port);
-                if (ret == 0) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "dst-brick-port=%d found", dst_port);
-                }
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received unlock %s from uuid: %s",
+                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Unlock response received "
+                        "from unknown peer %s", uuid_utoa (rsp.uuid));
         }
 
-        if (src_port) {
-                ret = dict_set_int32 (ctx, "src-brick-port",
-                                      src_port);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "Could not set src-brick");
-                        goto out;
-                }
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
         }
 
-        if (dst_port) {
-                ret = dict_set_int32 (ctx, "dst-brick-port",
-                                      dst_port);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "Could not set dst-brick");
-                        goto out;
-                }
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
         }
 
-out:
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
+}
 
+int32_t
+glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_cluster_unlock_cbk);
 }
 
 int32_t
-glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
 {
         gd1_mgmt_stage_op_rsp         rsp   = {{0},};
@@ -818,22 +887,32 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
         dict_t                        *dict   = NULL;
         char                          err_str[2048] = {0};
         char                          *peer_str = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "error";
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("error");
                 goto out;
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode stage "
+                        "response received from peer");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "error";
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("Failed to decode stage response "
+                                        "received from peer.");
                 goto out;
         }
 
@@ -845,7 +924,7 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
                                         rsp.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize rsp-buffer to dictionary");
 			event_type = GD_OP_EVENT_RCVD_RJT;
@@ -858,15 +937,19 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         op_ret = rsp.op_ret;
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received stage %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC", uuid_utoa (rsp.uuid));
+
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
 
         ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Stage response received from "
-                        "unknown peer: %s", uuid_utoa (rsp.uuid));
+                gf_log (this->name, GF_LOG_CRITICAL, "Stage response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
         }
 
         if (op_ret) {
@@ -879,12 +962,11 @@ out:
                                 peer_str = peerinfo->hostname;
                         else
                                 peer_str = uuid_utoa (rsp.uuid);
-                        snprintf (err_str, sizeof (err_str), "Operation failed "
-                                  "on %s", peer_str);
+                        snprintf (err_str, sizeof (err_str),
+                                  OPERRSTR_STAGE_FAIL, peer_str);
                         opinfo.op_errstr = gf_strdup (err_str);
                 }
                 if (!opinfo.op_errstr) {
-                        gf_log ("", GF_LOG_ERROR, "memory allocation failed");
                         ret = -1;
                         goto out;
                 }
@@ -894,218 +976,39 @@ out:
 
         switch (rsp.op) {
         case GD_OP_REPLACE_BRICK:
-                glusterd_rb_use_rsp_dict (dict);
+                glusterd_rb_use_rsp_dict (NULL, dict);
                 break;
         }
 
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
         }
 
-        if (rsp.op_errstr && strcmp (rsp.op_errstr, "error"))
-                free (rsp.op_errstr); //malloced by xdr
+        free (rsp.op_errstr); //malloced by xdr
         if (dict) {
                 if (!dict->extra_stdfree && rsp.dict.dict_val)
                         free (rsp.dict.dict_val); //malloced by xdr
                 dict_unref (dict);
         } else {
-                if (rsp.dict.dict_val)
-                        free (rsp.dict.dict_val); //malloced by xdr
+                free (rsp.dict.dict_val); //malloced by xdr
         }
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
-static int32_t
-glusterd_sync_use_rsp_dict (dict_t *rsp_dict)
-{
-        int      ret      = 0;
-
-        GF_ASSERT (rsp_dict);
-
-        if (!rsp_dict) {
-                goto out;
-        }
-
-        ret = glusterd_import_friend_volumes (rsp_dict);
-out:
-        return ret;
-
-}
-
-void
-_profile_volume_add_friend_rsp (dict_t *this, char *key, data_t *value,
-                               void *data)
-{
-        char    new_key[256] = {0};
-        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
-        data_t  *new_value = NULL;
-        int     brick_count = 0;
-        char    brick_key[256];
-
-        if (strcmp (key, "count") == 0)
-                return;
-        sscanf (key, "%d%s", &brick_count, brick_key);
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        GF_ASSERT (new_value);
-        snprintf (new_key, sizeof (new_key), "%d%s",
-                  rsp_ctx->count + brick_count, brick_key);
-        dict_set (rsp_ctx->dict, new_key, new_value);
-}
-
-int
-glusterd_profile_volume_use_rsp_dict (dict_t *rsp_dict)
-{
-        int     ret = 0;
-        glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
-        int32_t brick_count = 0;
-        int32_t count = 0;
-        dict_t  *ctx_dict = NULL;
-        glusterd_op_t   op = GD_OP_NONE;
-
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_int32 (rsp_dict, "count", &brick_count);
-        if (ret) {
-                ret = 0; //no bricks in the rsp
-                goto out;
-        }
-
-        op = glusterd_op_get_op ();
-        GF_ASSERT (GD_OP_PROFILE_VOLUME == op);
-        ctx_dict = glusterd_op_get_ctx ();
-
-        ret = dict_get_int32 (ctx_dict, "count", &count);
-        rsp_ctx.count = count;
-        rsp_ctx.dict = ctx_dict;
-        dict_foreach (rsp_dict, _profile_volume_add_friend_rsp, &rsp_ctx);
-        dict_del (ctx_dict, "count");
-        ret = dict_set_int32 (ctx_dict, "count", count + brick_count);
-out:
-        return ret;
-}
-
-void
-glusterd_volume_status_add_peer_rsp (dict_t *this, char *key, data_t *value,
-                                     void *data)
+int32_t
+glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
 {
-        glusterd_status_rsp_conv_t      *rsp_ctx = NULL;
-        data_t                          *new_value = NULL;
-        int32_t                         ret = 0;
-
-        if (strcmp (key, "count") == 0)
-                return;
-
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        GF_ASSERT (new_value);
-
-        ret = dict_set (rsp_ctx->dict, key, new_value);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "Unable to set key: %s in dict",
-                        key);
-
-        return;
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_stage_op_cbk);
 }
 
-int
-glusterd_volume_status_use_rsp_dict (dict_t *rsp_dict)
-{
-        int                             ret = 0;
-        glusterd_status_rsp_conv_t      rsp_ctx = {0};
-        int32_t                         brick_count = 0;
-        int32_t                         count = 0;
-        dict_t                          *ctx_dict = NULL;
-        glusterd_op_t                   op = GD_OP_NONE;
-
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_int32 (rsp_dict, "count", &brick_count);
-        if (ret) {
-                ret = 0; //no bricks in the rsp
-                goto out;
-        }
-
-        op = glusterd_op_get_op ();
-        GF_ASSERT (GD_OP_STATUS_VOLUME == op);
-        ctx_dict = glusterd_op_get_ctx (op);
-
-        ret = dict_get_int32 (ctx_dict, "count", &count);
-        rsp_ctx.count = count;
-        rsp_ctx.dict = ctx_dict;
-        dict_foreach (rsp_dict, glusterd_volume_status_add_peer_rsp, &rsp_ctx);
-
-        ret = dict_set_int32 (ctx_dict, "count", count + brick_count);
-out:
-        return ret;
-}
-
-int
-glusterd_volume_rebalance_use_rsp_dict (dict_t *rsp_dict)
-{
-        int            ret      = 0;
-        dict_t        *ctx_dict = NULL;
-        glusterd_op_t  op       = GD_OP_NONE;
-        uint64_t       value    = 0;
-        int32_t        value32  = 0;
-
-        GF_ASSERT (rsp_dict);
-
-        op = glusterd_op_get_op ();
-        GF_ASSERT (GD_OP_REBALANCE == op);
-
-        ctx_dict = glusterd_op_get_ctx (op);
-
-        if (!ctx_dict)
-                goto out;
-
-        ret = dict_get_uint64 (rsp_dict, "files", &value);
-        if (!ret) {
-                ret = dict_set_uint64 (ctx_dict, "files", value);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set the file count");
-                }
-        }
-
-        ret = dict_get_uint64 (rsp_dict, "size", &value);
-        if (!ret) {
-                ret = dict_set_uint64 (ctx_dict, "size", value);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set the size of migration");
-                }
-        }
-
-        ret = dict_get_uint64 (rsp_dict, "lookups", &value);
-        if (!ret) {
-                ret = dict_set_uint64 (ctx_dict, "lookups", value);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set lookuped file count");
-                }
-        }
-
-        ret = dict_get_int32 (rsp_dict, "status", &value32);
-        if (!ret) {
-                ret = dict_set_int32 (ctx_dict, "status", value32);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set status");
-                }
-        }
-
-out:
-        return ret;
-}
-
-
 int32_t
-glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
 {
         gd1_mgmt_commit_op_rsp         rsp   = {{0},};
@@ -1116,24 +1019,34 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
         dict_t                        *dict = NULL;
         char                          err_str[2048] = {0};
         char                          *peer_str = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
 
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "error";
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("error");
 		event_type = GD_OP_EVENT_RCVD_RJT;
                 goto out;
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
+                        "response received from peer");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "error";
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("Failed to decode commit response "
+                                        "received from peer.");
 		event_type = GD_OP_EVENT_RCVD_RJT;
                 goto out;
         }
@@ -1146,7 +1059,7 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                                         rsp.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize rsp-buffer to dictionary");
 			event_type = GD_OP_EVENT_RCVD_RJT;
@@ -1158,15 +1071,20 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
 
         op_ret = rsp.op_ret;
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s",
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received commit %s from uuid: %s",
                 (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
 
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Commit response received from "
-                        "unknown peer: %s", uuid_utoa (rsp.uuid));
+                gf_log (this->name, GF_LOG_CRITICAL, "Commit response for "
+                        "'Volume %s' received from unknown peer: %s",
+                        gd_op_list[opinfo.op], uuid_utoa (rsp.uuid));
         }
 
         if (op_ret) {
@@ -1179,12 +1097,11 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                                 peer_str = peerinfo->hostname;
                         else
                                 peer_str = uuid_utoa (rsp.uuid);
-                        snprintf (err_str, sizeof (err_str), "Operation failed "
-                                  "on %s", peer_str);
+                        snprintf (err_str, sizeof (err_str),
+                                  OPERRSTR_COMMIT_FAIL, peer_str);
                         opinfo.op_errstr = gf_strdup (err_str);
                 }
                 if (!opinfo.op_errstr) {
-                        gf_log ("", GF_LOG_ERROR, "memory allocation failed");
                         ret = -1;
                         goto out;
                 }
@@ -1192,37 +1109,44 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                 event_type = GD_OP_EVENT_RCVD_ACC;
                 switch (rsp.op) {
                 case GD_OP_REPLACE_BRICK:
-                        ret = glusterd_rb_use_rsp_dict (dict);
+                        ret = glusterd_rb_use_rsp_dict (NULL, dict);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_SYNC_VOLUME:
-                        ret = glusterd_sync_use_rsp_dict (dict);
+                        ret = glusterd_sync_use_rsp_dict (NULL, dict);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_PROFILE_VOLUME:
-                        ret = glusterd_profile_volume_use_rsp_dict (dict);
+                        ret = glusterd_profile_volume_use_rsp_dict (NULL, dict);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_GSYNC_SET:
-                        ret = glusterd_gsync_use_rsp_dict (dict, rsp.op_errstr);
+                        ret = glusterd_gsync_use_rsp_dict (NULL, dict, rsp.op_errstr);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_STATUS_VOLUME:
-                        ret = glusterd_volume_status_use_rsp_dict (dict);
+                        ret = glusterd_volume_status_copy_to_op_ctx_dict (NULL, dict);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_REBALANCE:
-                        ret = glusterd_volume_rebalance_use_rsp_dict (dict);
+                case GD_OP_DEFRAG_BRICK_VOLUME:
+                        ret = glusterd_volume_rebalance_use_rsp_dict (NULL, dict);
+                        if (ret)
+                                goto out;
+                break;
+
+                case GD_OP_HEAL_VOLUME:
+                        ret = glusterd_volume_heal_use_rsp_dict (NULL, dict);
                         if (ret)
                                 goto out;
 
@@ -1234,7 +1158,7 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
 out:
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
@@ -1243,17 +1167,22 @@ out:
 
         if (dict)
                 dict_unref (dict);
-        if (rsp.op_errstr && strcmp (rsp.op_errstr, "error"))
-                free (rsp.op_errstr); //malloced by xdr
+        free (rsp.op_errstr); //malloced by xdr
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
-
+int32_t
+glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_commit_op_cbk);
+}
 
 int32_t
-glusterd3_1_probe (call_frame_t *frame, xlator_t *this,
-                   void *data)
+glusterd_rpc_probe (call_frame_t *frame, xlator_t *this,
+                    void *data)
 {
         gd1_mgmt_probe_req      req = {{0},};
         int                     ret = 0;
@@ -1283,26 +1212,25 @@ glusterd3_1_probe (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
-        uuid_copy (req.uuid, priv->uuid);
+        uuid_copy (req.uuid, MY_UUID);
         req.hostname = gf_strdup (hostname);
         req.port = port;
 
         ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
                                        GLUSTERD_PROBE_QUERY,
-                                       NULL, this, glusterd3_1_probe_cbk,
+                                       NULL, this, glusterd_probe_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_probe_req);
 
 out:
-        if (req.hostname)
-                GF_FREE (req.hostname);
+        GF_FREE (req.hostname);
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 
 int32_t
-glusterd3_1_friend_add (call_frame_t *frame, xlator_t *this,
-                        void *data)
+glusterd_rpc_friend_add (call_frame_t *frame, xlator_t *this,
+                         void *data)
 {
         gd1_mgmt_friend_req         req      = {{0},};
         int                         ret      = 0;
@@ -1328,24 +1256,23 @@ glusterd3_1_friend_add (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
-        uuid_copy (req.uuid, priv->uuid);
+        uuid_copy (req.uuid, MY_UUID);
         req.hostname = peerinfo->hostname;
         req.port = peerinfo->port;
 
         ret = dict_allocate_and_serialize (vols, &req.vols.vols_val,
-                                           (size_t *)&req.vols.vols_len);
+                                           &req.vols.vols_len);
         if (ret)
                 goto out;
 
         ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
                                        GLUSTERD_FRIEND_ADD,
-                                       NULL, this, glusterd3_1_friend_add_cbk,
+                                       NULL, this, glusterd_friend_add_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_friend_req);
 
 
 out:
-        if (req.vols.vols_val)
-                GF_FREE (req.vols.vols_val);
+        GF_FREE (req.vols.vols_val);
 
         if (vols)
                 dict_unref (vols);
@@ -1355,8 +1282,8 @@ out:
 }
 
 int32_t
-glusterd3_1_friend_remove (call_frame_t *frame, xlator_t *this,
-                           void *data)
+glusterd_rpc_friend_remove (call_frame_t *frame, xlator_t *this,
+                            void *data)
 {
         gd1_mgmt_friend_req             req = {{0},};
         int                             ret = 0;
@@ -1376,12 +1303,12 @@ glusterd3_1_friend_remove (call_frame_t *frame, xlator_t *this,
 
         peerinfo = event->peerinfo;
 
-        uuid_copy (req.uuid, priv->uuid);
+        uuid_copy (req.uuid, MY_UUID);
         req.hostname = peerinfo->hostname;
         req.port = peerinfo->port;
         ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
                                        GLUSTERD_FRIEND_REMOVE, NULL,
-                                       this, glusterd3_1_friend_remove_cbk,
+                                       this, glusterd_friend_remove_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_friend_req);
 
 out:
@@ -1391,15 +1318,13 @@ out:
 
 
 int32_t
-glusterd3_1_friend_update (call_frame_t *frame, xlator_t *this,
-                           void *data)
+glusterd_rpc_friend_update (call_frame_t *frame, xlator_t *this,
+                            void *data)
 {
         gd1_mgmt_friend_update  req         = {{0},};
         int                     ret         = 0;
         glusterd_conf_t        *priv        = NULL;
         dict_t                 *friends     = NULL;
-        char                   *dict_buf    = NULL;
-        size_t                  len         = -1;
         call_frame_t           *dummy_frame = NULL;
         glusterd_peerinfo_t    *peerinfo    = NULL;
 
@@ -1414,32 +1339,29 @@ glusterd3_1_friend_update (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
-        ret = dict_allocate_and_serialize (friends, &dict_buf, (size_t *)&len);
+        ret = dict_allocate_and_serialize (friends, &req.friends.friends_val,
+                                           &req.friends.friends_len);
         if (ret)
                 goto out;
 
-        req.friends.friends_val = dict_buf;
-        req.friends.friends_len = len;
-
-        uuid_copy (req.uuid, priv->uuid);
+        uuid_copy (req.uuid, MY_UUID);
 
         dummy_frame = create_frame (this, this->ctx->pool);
         ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
                                        peerinfo->peer,
                                        GLUSTERD_FRIEND_UPDATE, NULL,
-                                       this, glusterd3_1_friend_update_cbk,
+                                       this, glusterd_friend_update_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_friend_update);
 
 out:
-        if (req.friends.friends_val)
-                GF_FREE (req.friends.friends_val);
+        GF_FREE (req.friends.friends_val);
 
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd3_1_cluster_lock (call_frame_t *frame, xlator_t *this,
+glusterd_cluster_lock (call_frame_t *frame, xlator_t *this,
                           void *data)
 {
         gd1_mgmt_cluster_lock_req       req = {{0},};
@@ -1465,15 +1387,143 @@ glusterd3_1_cluster_lock (call_frame_t *frame, xlator_t *this,
         ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
                                        peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_LOCK,
                                        NULL,
-                                       this, glusterd3_1_cluster_lock_cbk,
+                                       this, glusterd_cluster_lock_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_lock_peers (call_frame_t *frame, xlator_t *this,
+                             void *data)
+{
+        gd1_mgmt_v3_lock_req             req         = {{0},};
+        int                              ret         = -1;
+        glusterd_peerinfo_t             *peerinfo    = NULL;
+        glusterd_conf_t                 *priv        = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+        dict_t                          *dict        = NULL;
+        uuid_t                          *txn_id      = NULL;
+
+        if (!this)
+                goto out;
+
+        dict = data;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        //peerinfo should not be in payload
+        dict_del (dict, "peerinfo");
+
+        glusterd_get_uuid (&req.uuid);
+
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
+                goto out;
+        }
+
+        /* Sending valid transaction ID to peers */
+        ret = dict_get_bin (dict, "transaction_id",
+                            (void **)&txn_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to get transaction id.");
+                goto out;
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+                uuid_copy (req.txn_id, *txn_id);
+        }
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt_v3,
+                                       GLUSTERD_MGMT_V3_LOCK, NULL,
+                                       this, glusterd_mgmt_v3_lock_peers_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd3_1_cluster_unlock (call_frame_t *frame, xlator_t *this,
+glusterd_mgmt_v3_unlock_peers (call_frame_t *frame, xlator_t *this,
+                               void *data)
+{
+        gd1_mgmt_v3_unlock_req           req         = {{0},};
+        int                              ret         = -1;
+        glusterd_peerinfo_t             *peerinfo    = NULL;
+        glusterd_conf_t                 *priv        = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+        dict_t                          *dict        = NULL;
+        uuid_t                          *txn_id      = NULL;
+
+        if (!this)
+                goto out;
+
+        dict = data;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        //peerinfo should not be in payload
+        dict_del (dict, "peerinfo");
+
+        glusterd_get_uuid (&req.uuid);
+
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
+                goto out;
+        }
+
+        /* Sending valid transaction ID to peers */
+        ret = dict_get_bin (dict, "transaction_id",
+                            (void **)&txn_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to get transaction id.");
+                goto out;
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+                uuid_copy (req.txn_id, *txn_id);
+        }
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt_v3,
+                                       GLUSTERD_MGMT_V3_UNLOCK, NULL,
+                                       this, glusterd_mgmt_v3_unlock_peers_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_cluster_unlock (call_frame_t *frame, xlator_t *this,
                             void *data)
 {
         gd1_mgmt_cluster_lock_req       req = {{0},};
@@ -1499,15 +1549,15 @@ glusterd3_1_cluster_unlock (call_frame_t *frame, xlator_t *this,
         ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
                                        peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_UNLOCK,
                                        NULL,
-                                       this, glusterd3_1_cluster_unlock_cbk,
+                                       this, glusterd_cluster_unlock_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd3_1_stage_op (call_frame_t *frame, xlator_t *this,
+glusterd_stage_op (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
         gd1_mgmt_stage_op_req           req = {{0,},};
@@ -1538,9 +1588,12 @@ glusterd3_1_stage_op (call_frame_t *frame, xlator_t *this,
         req.op = glusterd_op_get_op ();
 
         ret = dict_allocate_and_serialize (dict, &req.buf.buf_val,
-                                           (size_t *)&req.buf.buf_len);
-        if (ret)
+                                           &req.buf.buf_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
                 goto out;
+        }
 
 
         dummy_frame = create_frame (this, this->ctx->pool);
@@ -1550,19 +1603,19 @@ glusterd3_1_stage_op (call_frame_t *frame, xlator_t *this,
         ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
                                        peerinfo->mgmt, GLUSTERD_MGMT_STAGE_OP,
                                        NULL,
-                                       this, glusterd3_1_stage_op_cbk,
+                                       this, glusterd_stage_op_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
 
 out:
         if ((_gf_true == is_alloc) && req.buf.buf_val)
                 GF_FREE (req.buf.buf_val);
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd3_1_commit_op (call_frame_t *frame, xlator_t *this,
+glusterd_commit_op (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
         gd1_mgmt_commit_op_req  req         = {{0,},};
@@ -1592,9 +1645,12 @@ glusterd3_1_commit_op (call_frame_t *frame, xlator_t *this,
         req.op = glusterd_op_get_op ();
 
         ret = dict_allocate_and_serialize (dict, &req.buf.buf_val,
-                                           (size_t *)&req.buf.buf_len);
-        if (ret)
+                                           &req.buf.buf_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict to "
+                        "request buffer");
                 goto out;
+        }
 
         dummy_frame = create_frame (this, this->ctx->pool);
         if (!dummy_frame)
@@ -1603,19 +1659,19 @@ glusterd3_1_commit_op (call_frame_t *frame, xlator_t *this,
         ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
                                        peerinfo->mgmt, GLUSTERD_MGMT_COMMIT_OP,
                                        NULL,
-                                       this, glusterd3_1_commit_op_cbk,
+                                       this, glusterd_commit_op_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
 
 out:
         if ((_gf_true == is_alloc) && req.buf.buf_val)
                 GF_FREE (req.buf.buf_val);
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
 {
         gd1_mgmt_brick_op_rsp         rsp   = {0};
@@ -1628,6 +1684,11 @@ glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
         int                           index = 0;
         glusterd_req_ctx_t            *req_ctx = NULL;
         glusterd_pending_node_t       *node = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = &global_txn_id;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (req);
         frame = myframe;
@@ -1636,14 +1697,17 @@ glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "error";
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("error");
 		event_type = GD_OP_EVENT_RCVD_RJT;
                 goto out;
         }
 
         ret =  xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode brick op "
+                        "response received");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
                 rsp.op_errstr = strdup ("Unable to decode brick op response");
@@ -1659,8 +1723,7 @@ glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                                         rsp.output.output_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
                                 "unserialize rsp-buffer to dictionary");
 			event_type = GD_OP_EVENT_RCVD_RJT;
                         goto out;
@@ -1677,7 +1740,7 @@ glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                 index = node->index;
                 ret = dict_set_int32 (dict, "index", index);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Error setting index on brick status rsp dict");
                         rsp.op_ret = -1;
                         event_type = GD_OP_EVENT_RCVD_RJT;
@@ -1685,6 +1748,11 @@ glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                 }
         }
 out:
+
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         ev_ctx = GF_CALLOC (1, sizeof (*ev_ctx), gf_gld_mt_brick_rsp_ctx_t);
         GF_ASSERT (ev_ctx);
         if (op_ret) {
@@ -1697,7 +1765,7 @@ out:
         ev_ctx->pending_node = frame->cookie;
         ev_ctx->rsp_dict  = dict;
         ev_ctx->commit_ctx = frame->local;
-        ret = glusterd_op_sm_inject_event (event_type, ev_ctx);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, ev_ctx);
         if (!ret) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
@@ -1705,16 +1773,24 @@ out:
 
         if (ret && dict)
                 dict_unref (dict);
-        if (rsp.op_errstr && strcmp (rsp.op_errstr, "error"))
-                free (rsp.op_errstr); //malloced by xdr
+        free (rsp.op_errstr); //malloced by xdr
         GLUSTERD_STACK_DESTROY (frame);
         return ret;
 }
 
 int32_t
-glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
+glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_brick_op_cbk);
+}
+
+int32_t
+glusterd_brick_op (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
+
         gd1_mgmt_brick_op_req           *req = NULL;
         int                             ret = 0;
         glusterd_conf_t                 *priv = NULL;
@@ -1722,8 +1798,10 @@ glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
         char                            *op_errstr = NULL;
         int                             pending_bricks = 0;
         glusterd_pending_node_t         *pending_node;
-        glusterd_req_ctx_t               *req_ctx = NULL;
+        glusterd_req_ctx_t              *req_ctx = NULL;
         struct rpc_clnt                 *rpc = NULL;
+        dict_t                          *op_ctx = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
 
         if (!this) {
                 ret = -1;
@@ -1735,31 +1813,72 @@ glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
         req_ctx = data;
         GF_ASSERT (req_ctx);
         INIT_LIST_HEAD (&opinfo.pending_bricks);
-        ret = glusterd_op_bricks_select (req_ctx->op, req_ctx->dict, &op_errstr);
+        ret = glusterd_op_bricks_select (req_ctx->op, req_ctx->dict, &op_errstr,
+                                         &opinfo.pending_bricks, NULL);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Brick Op failed");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to select bricks "
+                        "while performing brick op during 'Volume %s'",
+                        gd_op_list[opinfo.op]);
                 opinfo.op_errstr = op_errstr;
                 goto out;
         }
 
+
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         list_for_each_entry (pending_node, &opinfo.pending_bricks, list) {
                 dummy_frame = create_frame (this, this->ctx->pool);
                 if (!dummy_frame)
                         continue;
 
-                ret = glusterd_brick_op_build_payload (req_ctx->op,
-                                                       pending_node->node,
-                                                       (gd1_mgmt_brick_op_req **)&req,
-                                                       req_ctx->dict);
-                if (ret)
-                        goto out;
+                if ((pending_node->type == GD_NODE_NFS) ||
+                    ((pending_node->type == GD_NODE_SHD) &&
+                    (req_ctx->op == GD_OP_STATUS_VOLUME)))
+                        ret = glusterd_node_op_build_payload
+                                (req_ctx->op,
+                                 (gd1_mgmt_brick_op_req **)&req,
+                                 req_ctx->dict);
+                else {
+                        ret = glusterd_brick_op_build_payload
+                                (req_ctx->op, pending_node->node,
+                                 (gd1_mgmt_brick_op_req **)&req,
+                                 req_ctx->dict);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "build brick op payload during "
+                                        "'Volume %s'", gd_op_list[req_ctx->op]);
+                                goto out;
+                        }
+                }
 
                 dummy_frame->local = data;
                 dummy_frame->cookie = pending_node;
 
                 rpc = glusterd_pending_node_get_rpc (pending_node);
                 if (!rpc) {
+                        if (pending_node->type == GD_NODE_REBALANCE) {
+                                opinfo.brick_pending_count = 0;
+                                ret = 0;
+                                if (req) {
+                                        GF_FREE (req->input.input_val);
+                                        GF_FREE (req);
+                                        req = NULL;
+                                }
+                                GLUSTERD_STACK_DESTROY (dummy_frame);
+
+                                op_ctx = glusterd_op_get_ctx ();
+                                if (!op_ctx)
+                                        goto out;
+                                glusterd_defrag_volume_node_rsp (req_ctx->dict,
+                                                                 NULL, op_ctx);
+
+                                goto out;
+                        }
+
                         ret = -1;
                         gf_log (this->name, GF_LOG_ERROR, "Brick Op failed "
                                 "due to rpc failure.");
@@ -1769,11 +1888,10 @@ glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
                 ret = glusterd_submit_request (rpc, req, dummy_frame,
                                                priv->gfs_mgmt,
                                                req->op, NULL,
-                                               this, glusterd3_1_brick_op_cbk,
+                                               this, glusterd_brick_op_cbk,
                                                (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
                 if (req) {
-                        if (req->input.input_val)
-                                GF_FREE (req->input.input_val);
+                        GF_FREE (req->input.input_val);
                         GF_FREE (req);
                         req = NULL;
                 }
@@ -1781,38 +1899,46 @@ glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
                         pending_bricks++;
         }
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Sent op req to %d bricks",
+        gf_log (this->name, GF_LOG_DEBUG, "Sent brick op req for operation "
+                "'Volume %s' to %d bricks", gd_op_list[req_ctx->op],
                 pending_bricks);
         opinfo.brick_pending_count = pending_bricks;
 
 out:
         if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, data);
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             txn_id, data);
                 opinfo.op_ret = ret;
         }
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 struct rpc_clnt_procedure gd_brick_actors[GLUSTERD_BRICK_MAXVALUE] = {
         [GLUSTERD_BRICK_NULL] = {"NULL", NULL },
-        [GLUSTERD_BRICK_OP]   = {"BRICK_OP", glusterd3_1_brick_op},
+        [GLUSTERD_BRICK_OP]   = {"BRICK_OP", glusterd_brick_op},
 };
 
 struct rpc_clnt_procedure gd_peer_actors[GLUSTERD_FRIEND_MAXVALUE] = {
         [GLUSTERD_FRIEND_NULL]   = {"NULL", NULL },
-        [GLUSTERD_PROBE_QUERY]   = {"PROBE_QUERY", glusterd3_1_probe},
-        [GLUSTERD_FRIEND_ADD]    = {"FRIEND_ADD", glusterd3_1_friend_add},
-        [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE", glusterd3_1_friend_remove},
-        [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd3_1_friend_update},
+        [GLUSTERD_PROBE_QUERY]   = {"PROBE_QUERY", glusterd_rpc_probe},
+        [GLUSTERD_FRIEND_ADD]    = {"FRIEND_ADD", glusterd_rpc_friend_add},
+        [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE", glusterd_rpc_friend_remove},
+        [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd_rpc_friend_update},
 };
 
 struct rpc_clnt_procedure gd_mgmt_actors[GLUSTERD_MGMT_MAXVALUE] = {
         [GLUSTERD_MGMT_NULL]           = {"NULL", NULL },
-        [GLUSTERD_MGMT_CLUSTER_LOCK]   = {"CLUSTER_LOCK", glusterd3_1_cluster_lock},
-        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK", glusterd3_1_cluster_unlock},
-        [GLUSTERD_MGMT_STAGE_OP]       = {"STAGE_OP", glusterd3_1_stage_op},
-        [GLUSTERD_MGMT_COMMIT_OP]      = {"COMMIT_OP", glusterd3_1_commit_op},
+        [GLUSTERD_MGMT_CLUSTER_LOCK]   = {"CLUSTER_LOCK", glusterd_cluster_lock},
+        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK", glusterd_cluster_unlock},
+        [GLUSTERD_MGMT_STAGE_OP]       = {"STAGE_OP", glusterd_stage_op},
+        [GLUSTERD_MGMT_COMMIT_OP]      = {"COMMIT_OP", glusterd_commit_op},
+};
+
+struct rpc_clnt_procedure gd_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = {
+        [GLUSTERD_MGMT_V3_NULL]           = {"NULL", NULL },
+        [GLUSTERD_MGMT_V3_LOCK]           = {"MGMT_V3_LOCK", glusterd_mgmt_v3_lock_peers},
+        [GLUSTERD_MGMT_V3_UNLOCK]         = {"MGMT_V3_UNLOCK", glusterd_mgmt_v3_unlock_peers},
 };
 
 struct rpc_clnt_program gd_mgmt_prog = {
@@ -1839,3 +1965,10 @@ struct rpc_clnt_program gd_peer_prog = {
         .numproc   = GLUSTERD_FRIEND_MAXVALUE,
 };
 
+struct rpc_clnt_program gd_mgmt_v3_prog = {
+        .progname  = "glusterd mgmt v3",
+        .prognum   = GD_MGMT_V3_PROGRAM,
+        .progver   = GD_MGMT_V3_VERSION,
+        .proctable = gd_mgmt_v3_actors,
+        .numproc   = GLUSTERD_MGMT_V3_MAXVALUE,
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index 259c247f4..c671edf68 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -100,8 +90,7 @@ glusterd_destroy_probe_ctx (glusterd_probe_ctx_t *ctx)
         if (!ctx)
                 return;
 
-        if (ctx->hostname)
-                GF_FREE (ctx->hostname);
+        GF_FREE (ctx->hostname);
         GF_FREE (ctx);
 }
 
@@ -113,8 +102,7 @@ glusterd_destroy_friend_req_ctx (glusterd_friend_req_ctx_t *ctx)
 
         if (ctx->vols)
                 dict_unref (ctx->vols);
-        if (ctx->hostname)
-                GF_FREE (ctx->hostname);
+        GF_FREE (ctx->hostname);
         GF_FREE (ctx);
 }
 
@@ -123,8 +111,7 @@ glusterd_destroy_friend_update_ctx (glusterd_friend_update_ctx_t *ctx)
 {
         if (!ctx)
                 return;
-        if (ctx->hostname)
-                GF_FREE (ctx->hostname);
+        GF_FREE (ctx->hostname);
         GF_FREE (ctx);
 }
 
@@ -257,12 +244,9 @@ glusterd_ac_reverse_probe_begin (glusterd_friend_sm_event_t *event, void *ctx)
 
 out:
         if (ret) {
-                if (new_event)
-                        GF_FREE (new_event);
-                if (new_ev_ctx->hostname)
-                        GF_FREE (new_ev_ctx->hostname);
-                if (new_ev_ctx)
-                        GF_FREE (new_ev_ctx);
+                GF_FREE (new_event);
+                GF_FREE (new_ev_ctx->hostname);
+                GF_FREE (new_ev_ctx);
         }
         gf_log ("", GF_LOG_DEBUG, "returning with %d", ret);
         return ret;
@@ -415,7 +399,9 @@ glusterd_ac_send_friend_remove_req (glusterd_friend_sm_event_t *event,
 
                 if (ctx)
                         ret = glusterd_xfer_cli_deprobe_resp (ctx->req, ret, 0,
-                                                              ctx->hostname);
+                                                              NULL,
+                                                              ctx->hostname,
+                                                              ctx->dict);
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
 
@@ -444,22 +430,36 @@ out:
         return ret;
 }
 
+static gf_boolean_t
+glusterd_should_update_peer (glusterd_peerinfo_t *peerinfo,
+                             glusterd_peerinfo_t *cur_peerinfo)
+{
+        gf_boolean_t is_valid = _gf_false;
+
+        if ((peerinfo == cur_peerinfo) ||
+            (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED))
+                is_valid = _gf_true;
+
+        return is_valid;
+}
+
 static int
 glusterd_ac_send_friend_update (glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                           ret         = 0;
-        glusterd_peerinfo_t          *peerinfo    = NULL;
-        rpc_clnt_procedure_t         *proc        = NULL;
-        xlator_t                     *this        = NULL;
-        glusterd_friend_update_ctx_t  ev_ctx      = {{0}};
-        glusterd_conf_t              *priv        = NULL;
-        dict_t                       *friends     = NULL;
-        char                          key[100]    = {0,};
-        char                         *dup_buf     = NULL;
-        int32_t                       count       = 0;
+        int                           ret               = 0;
+        glusterd_peerinfo_t          *cur_peerinfo      = NULL;
+        glusterd_peerinfo_t          *peerinfo          = NULL;
+        rpc_clnt_procedure_t         *proc              = NULL;
+        xlator_t                     *this              = NULL;
+        glusterd_friend_update_ctx_t  ev_ctx            = {{0}};
+        glusterd_conf_t              *priv              = NULL;
+        dict_t                       *friends           = NULL;
+        char                          key[100]          = {0,};
+        char                         *dup_buf           = NULL;
+        int32_t                       count             = 0;
 
         GF_ASSERT (event);
-        peerinfo = event->peerinfo;
+        cur_peerinfo = event->peerinfo;
 
         this = THIS;
         priv = this->private;
@@ -478,6 +478,9 @@ glusterd_ac_send_friend_update (glusterd_friend_sm_event_t *event, void *ctx)
                 goto out;
 
         list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
+                        continue;
+
                 count++;
                 snprintf (key, sizeof (key), "friend%d.uuid", count);
                 dup_buf = gf_strdup (uuid_utoa (peerinfo->uuid));
@@ -500,6 +503,9 @@ glusterd_ac_send_friend_update (glusterd_friend_sm_event_t *event, void *ctx)
                 if (!peerinfo->connected || !peerinfo->peer)
                         continue;
 
+                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
+                        continue;
+
                 ret = dict_set_static_ptr (friends, "peerinfo", peerinfo);
                 if (ret) {
                         gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
@@ -533,7 +539,7 @@ glusterd_peer_detach_cleanup (glusterd_conf_t *priv)
         list_for_each_entry_safe (volinfo,tmp_volinfo,
                                   &priv->volumes, vol_list) {
                 if (!glusterd_friend_contains_vol_bricks (volinfo,
-                                                          priv->uuid)) {
+                                                          MY_UUID)) {
                         gf_log (THIS->name, GF_LOG_INFO,
                                 "Deleting stale volume %s", volinfo->volname);
                         ret = glusterd_delete_volume (volinfo);
@@ -645,7 +651,8 @@ glusterd_ac_handle_friend_add_req (glusterd_friend_sm_event_t *event, void *ctx)
         uuid_copy (peerinfo->uuid, ev_ctx->uuid);
 
         //Build comparison logic here.
-        ret = glusterd_compare_friend_data (ev_ctx->vols, &status);
+        ret = glusterd_compare_friend_data (ev_ctx->vols, &status,
+                                            peerinfo->hostname);
         if (ret)
                 goto out;
 
@@ -682,7 +689,8 @@ glusterd_ac_handle_friend_add_req (glusterd_friend_sm_event_t *event, void *ctx)
         glusterd_friend_sm_inject_event (new_event);
 
         ret = glusterd_xfer_friend_add_resp (ev_ctx->req, ev_ctx->hostname,
-                                             ev_ctx->port, op_ret, op_errno);
+                                             peerinfo->hostname, ev_ctx->port,
+                                             op_ret, op_errno);
 
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
@@ -954,6 +962,27 @@ glusterd_destroy_friend_event_context (glusterd_friend_sm_event_t *event)
         }
 }
 
+gf_boolean_t
+gd_does_peer_affect_quorum (glusterd_friend_sm_state_t old_state,
+                            glusterd_friend_sm_event_type_t event_type,
+                            glusterd_peerinfo_t *peerinfo)
+{
+        gf_boolean_t    affects = _gf_false;
+
+        //When glusterd comes up with friends in BEFRIENDED state in store,
+        //wait until compare-data happens.
+        if ((old_state == GD_FRIEND_STATE_BEFRIENDED) &&
+            (event_type != GD_FRIEND_EVENT_RCVD_ACC) &&
+            (event_type != GD_FRIEND_EVENT_LOCAL_ACC))
+                goto out;
+        if ((peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)
+            && peerinfo->connected) {
+                affects = _gf_true;
+        }
+out:
+        return affects;
+}
+
 int
 glusterd_friend_sm ()
 {
@@ -965,6 +994,8 @@ glusterd_friend_sm ()
         glusterd_peerinfo_t             *peerinfo   = NULL;
         glusterd_friend_sm_event_type_t  event_type = 0;
         gf_boolean_t                     is_await_conn = _gf_false;
+        gf_boolean_t                     quorum_action = _gf_false;
+        glusterd_friend_sm_state_t       old_state = GD_FRIEND_STATE_DEFAULT;
 
         while (!list_empty (&gd_friend_sm_queue)) {
                 list_for_each_entry_safe (event, tmp, &gd_friend_sm_queue, list) {
@@ -984,6 +1015,7 @@ glusterd_friend_sm ()
                                 glusterd_friend_sm_event_name_get (event_type));
 
 
+                        old_state = peerinfo->state.state;
                         state = glusterd_friend_state_table[peerinfo->state.state];
 
                         GF_ASSERT (state);
@@ -1024,6 +1056,15 @@ glusterd_friend_sm ()
                                 goto out;
                         }
 
+                        if (gd_does_peer_affect_quorum (old_state, event_type,
+                                                        peerinfo)) {
+                                peerinfo->quorum_contrib = QUORUM_UP;
+                                if (peerinfo->quorum_action) {
+                                        peerinfo->quorum_action = _gf_false;
+                                        quorum_action = _gf_true;
+                                }
+                        }
+
                         ret = glusterd_store_peerinfo (peerinfo);
 
                         glusterd_destroy_friend_event_context (event);
@@ -1037,6 +1078,25 @@ glusterd_friend_sm ()
 
         ret = 0;
 out:
+        if (quorum_action) {
+            /* When glusterd is restarted, it needs to wait until the 'friends' view
+             * of the volumes settle, before it starts any of the internal daemons.
+             *
+             * Every friend that was part of the cluster, would send its
+             * cluster-view, 'our' way. For every friend, who belongs to
+             * a partition which has a different cluster-view from our
+             * partition, we may update our cluster-view. For subsequent
+             * friends from that partition would agree with us, if the first
+             * friend wasn't rejected. For every first friend, whom we agreed with,
+             * we would need to start internal daemons/bricks belonging to the
+             * new volumes.
+             * glusterd_spawn_daemons calls functions that are idempotent. ie,
+             * the functions spawn process(es) only if they are not started yet.
+             *
+             * */
+                glusterd_spawn_daemons (NULL);
+                glusterd_do_quorum_action ();
+        }
         return ret;
 }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index 7baaf0236..b9bedbe69 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_SM_H_
 #define _GLUSTERD_SM_H_
 
@@ -37,14 +27,21 @@
 #include "byte-order.h"
 //#include "glusterd.h"
 #include "rpcsvc.h"
-
-struct glusterd_store_handle_ {
-        char    *path;
-        int     fd;
-        FILE    *read;
-};
-
-typedef struct glusterd_store_handle_  glusterd_store_handle_t;
+#include "store.h"
+
+typedef enum gd_quorum_contribution_ {
+        QUORUM_NONE,
+        QUORUM_WAITING,
+        QUORUM_DOWN,
+        QUORUM_UP
+} gd_quorum_contrib_t;
+
+typedef enum gd_quorum_status_ {
+        QUORUM_UNKNOWN,
+        QUORUM_NOT_APPLICABLE,
+        QUORUM_MEETS,
+        QUORUM_DOES_NOT_MEET
+} gd_quorum_status_t;
 
 typedef enum glusterd_friend_sm_state_ {
         GD_FRIEND_STATE_DEFAULT = 0,
@@ -89,7 +86,9 @@ typedef struct glusterd_sm_tr_log_ {
 
 struct glusterd_peerinfo_ {
         uuid_t                          uuid;
-        char                            uuid_str[50];
+        char                            uuid_str[50]; /* Retrieve this using
+                                                       * gd_peer_uuid_str ()
+                                                       */
         glusterd_peer_state_info_t      state;
         char                            *hostname;
         int                             port;
@@ -98,9 +97,13 @@ struct glusterd_peerinfo_ {
         struct rpc_clnt                 *rpc;
         rpc_clnt_prog_t                 *mgmt;
         rpc_clnt_prog_t                 *peer;
+        rpc_clnt_prog_t                 *mgmt_v3;
         int                             connected;
-        glusterd_store_handle_t         *shandle;
+        gf_store_handle_t         *shandle;
         glusterd_sm_tr_log_t            sm_log;
+        gf_boolean_t                    quorum_action;
+        gd_quorum_contrib_t             quorum_contrib;
+        gf_boolean_t                    locked;
 };
 
 typedef struct glusterd_peerinfo_ glusterd_peerinfo_t;
@@ -114,11 +117,13 @@ typedef enum glusterd_ev_gen_mode_ {
 typedef struct glusterd_peer_ctx_args_ {
         rpcsvc_request_t        *req;
         glusterd_ev_gen_mode_t  mode;
+        dict_t                  *dict;
 } glusterd_peerctx_args_t;
 
 typedef struct glusterd_peer_ctx_ {
         glusterd_peerctx_args_t        args;
         glusterd_peerinfo_t            *peerinfo;
+        char                           *errstr;
 } glusterd_peerctx_t;
 
 typedef enum glusterd_friend_sm_event_type_ {
@@ -179,6 +184,7 @@ typedef struct glusterd_probe_ctx_ {
         char                    *hostname;
         rpcsvc_request_t        *req;
         int                      port;
+        dict_t                  *dict;
 } glusterd_probe_ctx_t;
 int
 glusterd_friend_sm_new_event (glusterd_friend_sm_event_type_t event_type,
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
new file mode 100644
index 000000000..9b811cd05
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -0,0 +1,5590 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <inttypes.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/resource.h>
+#include <sys/statvfs.h>
+#include <sys/mount.h>
+
+#include "globals.h"
+#include "compat.h"
+#include "protocol-common.h"
+#include "xlator.h"
+#include "logging.h"
+#include "timer.h"
+#include "glusterd-mem-types.h"
+#include "glusterd.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+#include "run.h"
+#include "glusterd-volgen.h"
+#include "glusterd-mgmt.h"
+#include "glusterd-syncop.h"
+
+#include "syscall.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+
+#ifdef GF_LINUX_HOST_OS
+#include <mntent.h>
+#endif
+
+char snap_mount_folder[PATH_MAX];
+
+static int32_t
+glusterd_find_missed_snap (dict_t *rsp_dict, glusterd_volinfo_t *vol,
+                           char *snap_uuid, struct list_head *peers,
+                           int32_t op);
+
+/* This function will restore a snapshot volumes
+ *
+ * @param dict          dictionary containing snapshot restore request
+ * @param op_errstr     In case of any failure error message will be returned
+ *                      in this variable
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_snapshot_restore (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int                     ret             = -1;
+        char                    *volname        = NULL;
+        char                    *snapname       = NULL;
+        xlator_t                *this           = NULL;
+        glusterd_volinfo_t      *snap_volinfo   = NULL;
+        glusterd_volinfo_t      *volinfo        = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        glusterd_conf_t         *priv           = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                        "snap name");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (NULL == snap) {
+                ret = gf_asprintf (op_errstr, "Snap (%s) not found",
+                                        snapname);
+                if (ret < 0) {
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                ret = -1;
+                goto out;
+        }
+
+        /* TODO : As of now there is only volume in snapshot.
+         * Change this when multiple volume snapshot is introduced
+         */
+        snap_volinfo = list_entry (snap->volumes.next, glusterd_volinfo_t,
+                                   vol_list);
+
+        ret = glusterd_volinfo_find (snap_volinfo->parent_volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not get volinfo of "
+                        "%s", snap_volinfo->parent_volname);
+                goto out;
+        }
+
+        if (is_origin_glusterd (dict) == _gf_true) {
+                /* From origin glusterd check if      *
+                 * any peers with snap bricks is down */
+                ret = glusterd_find_missed_snap (rsp_dict, snap_volinfo,
+                                                 snap_volinfo->volname,
+                                                 &priv->peers,
+                                                 GF_SNAP_OPTION_TYPE_RESTORE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to find missed snap restores");
+                        goto out;
+                }
+        }
+
+        ret = gd_restore_snap_volume (rsp_dict, volinfo, snap_volinfo);
+        if (ret) {
+                /* No need to update op_errstr because it is assumed
+                 * that the called function will do that in case of
+                 * failure.
+                 */
+                gf_log (this->name, GF_LOG_ERROR, "Failed to restore "
+                        "snap for %s volume", volname);
+                goto out;
+        }
+
+        ret = 0;
+
+        /* TODO: Need to check if we need to delete the snap after the
+         * operation is successful or not. Also need to persist the state
+         * of restore operation in the store.
+         */
+out:
+        return ret;
+}
+
+/* This function is called before actual restore is taken place. This function
+ * will validate whether the snapshot volumes are ready to be restored or not.
+ *
+ * @param dict          dictionary containing snapshot restore request
+ * @param op_errstr     In case of any failure error message will be returned
+ *                      in this variable
+ * @param rsp_dict      response dictionary
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_snapshot_restore_prevalidate (dict_t *dict, char **op_errstr,
+                                       dict_t *rsp_dict)
+{
+        int                     ret             = -1;
+        int32_t                 i               = 0;
+        int32_t                 volcount        = 0;
+        gf_boolean_t            snap_restored   = _gf_false;
+        char                    key[PATH_MAX]   = {0, };
+        char                    *volname        = NULL;
+        char                    *snapname       = NULL;
+        glusterd_volinfo_t      *volinfo        = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        xlator_t                *this           = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                        "snap name");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (NULL == snap) {
+                ret = gf_asprintf (op_errstr, "Snap (%s) not found",
+                                snapname);
+                if (ret < 0) {
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                ret = -1;
+                goto out;
+        }
+
+        snap_restored = snap->snap_restored;
+
+        if (snap_restored) {
+                ret = gf_asprintf (op_errstr, "Snap (%s) is already "
+                                  "restored", snapname);
+                if (ret < 0) {
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (rsp_dict, "snapname", snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "snap name");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "volcount", &volcount);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume count");
+                goto out;
+        }
+
+        /* Snapshot restore will only work if all the volumes,
+           that are part of the snapshot, are stopped. */
+        for (i = 1; i <= volcount; ++i) {
+                snprintf (key, sizeof (key), "volname%d", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "get volume name");
+                        goto out;
+                }
+
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        ret = gf_asprintf (op_errstr, "Volume (%s) not found",
+                                           volname);
+                        if (ret < 0) {
+                                goto out;
+                        }
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (glusterd_is_volume_started (volinfo)) {
+                        ret = gf_asprintf (op_errstr, "Volume (%s) has been "
+                        "started. Volume needs to be stopped before restoring "
+                        "a snapshot.", volname);
+                        if (ret < 0) {
+                                goto out;
+                        }
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+snap_max_hard_limits_validate (dict_t *dict, char *volname,
+                          uint64_t value, char **op_errstr)
+{
+        char                err_str[PATH_MAX] = "";
+        glusterd_conf_t    *conf              = NULL;
+        glusterd_volinfo_t *volinfo           = NULL;
+        int                 ret               = -1;
+        uint64_t            max_limit         = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+        xlator_t           *this              = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        if (volname) {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (!ret) {
+                        if (volinfo->is_snap_volume) {
+                                ret = -1;
+                                snprintf (err_str, PATH_MAX,
+                                         "%s is a snap volume. Configuring "
+                                         "snap-max-hard-limit for a snap "
+                                         "volume is prohibited.", volname);
+                                goto out;
+                        }
+                }
+        }
+
+        if (value) {
+                /* Max limit for the system is GLUSTERD_SNAPS_MAX_HARD_LIMIT
+                 * but max limit for a volume is conf->snap_max_hard_limit.
+                 */
+                if (volname) {
+                        max_limit = conf->snap_max_hard_limit;
+                } else {
+                        max_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+                }
+        }
+
+        if ((value < 0) || (value > max_limit)) {
+                ret = -1;
+                snprintf (err_str, PATH_MAX, "Invalid snap-max-hard-limit"
+                          "%"PRIu64 ". Expected range 0 - %"PRIu64,
+                          value, max_limit);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                *op_errstr = gf_strdup (err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+        }
+        return ret;
+}
+
+int
+glusterd_snapshot_config_prevalidate (dict_t *dict, char **op_errstr)
+{
+        char               *volname             = NULL;
+        glusterd_volinfo_t *volinfo             = NULL;
+        xlator_t           *this                = NULL;
+        int                 ret                 = -1;
+        int                 config_command      = 0;
+        char                err_str[PATH_MAX]   = {0,};
+        glusterd_conf_t    *conf                = NULL;
+        uint64_t            value               = 0;
+        uint64_t            hard_limit          = 0;
+        uint64_t            soft_limit          = 0;
+        gf_loglevel_t       loglevel            = GF_LOG_ERROR;
+        uint64_t            max_limit           = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        ret = dict_get_int32 (dict, "config-command", &config_command);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str),
+                          "failed to get config-command type");
+                goto out;
+        }
+
+        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
+
+        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (volname) {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "Volume %s does not exist.", volname);
+                        goto out;
+                }
+        }
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+                if (hard_limit) {
+                        /* Validations for snap-max-hard-limits */
+                        ret = snap_max_hard_limits_validate (dict, volname,
+                                                         hard_limit, op_errstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "snap-max-hard-limit validation "
+                                        "failed.");
+                                goto out;
+                        }
+                }
+
+                if (soft_limit) {
+                        max_limit = GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT;
+                        if ((soft_limit < 0) || (soft_limit > max_limit)) {
+                                ret = -1;
+                                snprintf (err_str, PATH_MAX, "Invalid "
+                                         "snap-max-soft-limit ""%"
+                                         PRIu64 ". Expected range 0 - %"PRIu64,
+                                         value, max_limit);
+                                goto out;
+                        }
+                        break;
+                }
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+
+        if (ret && err_str[0] != '\0') {
+                gf_log (this->name, loglevel, "%s", err_str);
+                *op_errstr = gf_strdup (err_str);
+        }
+
+        return ret;
+}
+
+int
+glusterd_snap_create_pre_val_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char                  *snap_brick_dir        = NULL;
+        char                  *snap_device           = NULL;
+        char                  *tmpstr                = NULL;
+        char                   key[PATH_MAX]         = "";
+        char                   snapbrckcnt[PATH_MAX] = "";
+        char                   snapbrckord[PATH_MAX] = "";
+        int                    ret                   = -1;
+        int64_t                i                     = -1;
+        int64_t                j                     = -1;
+        int64_t                volume_count          = 0;
+        int64_t                brick_count           = 0;
+        int64_t                brick_order           = 0;
+        xlator_t              *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dst);
+        GF_ASSERT (src);
+
+        ret = dict_get_int64 (src, "volcount", &volume_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                        "get the volume count");
+                goto out;
+        }
+
+        for (i = 0; i < volume_count; i++) {
+                memset (snapbrckcnt, '\0', sizeof(snapbrckcnt));
+                ret = snprintf (snapbrckcnt, sizeof(snapbrckcnt) - 1,
+                                "vol%ld_brickcount", i+1);
+                ret = dict_get_int64 (src, snapbrckcnt, &brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "No bricks for this volume in this dict");
+                        continue;
+                }
+
+                for (j = 0; j < brick_count; j++) {
+                        /* Fetching data from source dict */
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brickdir%ld", i+1, j);
+
+                        ret = dict_get_ptr (src, key,
+                                            (void **)&snap_brick_dir);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Unable to fetch %s", key);
+                                continue;
+                        }
+
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brick_snapdevice%ld", i+1, j);
+
+                        ret = dict_get_ptr (src, key,
+                                            (void **)&snap_device);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to fetch snap_device");
+                                goto out;
+                        }
+
+                        snprintf (snapbrckord, sizeof(snapbrckord) - 1,
+                                  "vol%ld.brick%ld.order", i+1, j);
+
+                        ret = dict_get_int64 (src, snapbrckord, &brick_order);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get brick order");
+                                goto out;
+                        }
+
+                        /* Adding the data in the dst dict */
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brickdir%ld", i+1, brick_order);
+
+                        tmpstr = gf_strdup (snap_brick_dir);
+                        if (!tmpstr) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Out Of Memory");
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = dict_set_dynstr (dst, key, tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                GF_FREE (tmpstr);
+                                goto out;
+                        }
+
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brick_snapdevice%ld",
+                                  i+1, brick_order);
+
+                        tmpstr = gf_strdup (snap_device);
+                        if (!tmpstr) {
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = dict_set_dynstr (dst, key, tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                GF_FREE (tmpstr);
+                                goto out;
+                        }
+
+                }
+        }
+
+        ret = 0;
+out:
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_snap_pre_validate_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        int             ret             = -1;
+        int32_t         snap_command    = 0;
+        xlator_t       *this            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!dst || !src) {
+                gf_log (this->name, GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = glusterd_snap_create_pre_val_use_rsp_dict (dst, src);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to use "
+                                "rsp dict");
+                        goto out;
+                }
+                break;
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_snapshot_create_prevalidate (dict_t *dict, char **op_errstr,
+                                      dict_t *rsp_dict)
+{
+        char                  *volname                   = NULL;
+        char                  *snapname                  = NULL;
+        char                  *device                    = NULL;
+        char                  *tmpstr                    = NULL;
+        char                  *brick_dir                 = NULL;
+        char                   snap_brick_dir[PATH_MAX]  = "";
+        char                  *mnt_pt                    = NULL;
+        char                   key[PATH_MAX]             = "";
+        char                   snap_mount[PATH_MAX]      = "";
+        char                   snap_volname[64]          = "";
+        char                   err_str[PATH_MAX]         = "";
+        int                    ret                       = -1;
+        int64_t                i                         = 0;
+        int64_t                volcount                  = 0;
+        int64_t                brick_count               = 0;
+        int64_t                brick_order               = 0;
+        glusterd_brickinfo_t  *brickinfo                 = NULL;
+        glusterd_volinfo_t    *volinfo                   = NULL;
+        xlator_t              *this                      = NULL;
+        uuid_t                *snap_volid                = NULL;
+        gf_loglevel_t          loglevel                  = GF_LOG_ERROR;
+        glusterd_conf_t       *conf                      = NULL;
+        int64_t                effective_max_limit       = 0;
+
+        this = THIS;
+        GF_ASSERT (op_errstr);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = dict_get_int64 (dict, "volcount", &volcount);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to "
+                          "get the volume count");
+                goto out;
+        }
+        if (volcount <= 0) {
+                snprintf (err_str, sizeof (err_str), "Invalid volume count %ld "
+                          "supplied", volcount);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to get snapname");
+                goto out;
+        }
+
+        if (glusterd_find_snap_by_name (snapname)) {
+                ret = -1;
+                snprintf (err_str, sizeof (err_str), "Snap %s already exists",
+                          snapname);
+                goto out;
+        }
+
+        for (i = 1; i <= volcount; i++) {
+                snprintf (key, sizeof (key), "volname%ld", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "failed to get volume name");
+                        goto out;
+                }
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "Volume (%s) does not exist ", volname);
+                        goto out;
+                }
+
+                ret = -1;
+                if (!glusterd_is_volume_started (volinfo)) {
+                        snprintf (err_str, sizeof (err_str), "volume %s is "
+                                  "not started", volinfo->volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+                if (glusterd_is_defrag_on (volinfo)) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "rebalance process is running for the "
+                                  "volume %s", volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+                /* TODO: Also check whether geo replication is running */
+
+                if (volinfo->is_snap_volume == _gf_true) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "Volume %s is a snap volume", volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+
+                if (volinfo->snap_max_hard_limit < conf->snap_max_hard_limit)
+                        effective_max_limit = volinfo->snap_max_hard_limit;
+                else
+                        effective_max_limit = conf->snap_max_hard_limit;
+
+                if (volinfo->snap_count >= effective_max_limit) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "The number of existing snaps has reached "
+                                  "the effective maximum limit of %"PRIu64" ,"
+                                  "for the volume %s", effective_max_limit,
+                                  volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+
+                snprintf (key, sizeof(key) - 1, "vol%ld_volid", i);
+                ret = dict_get_bin (dict, key, (void **)&snap_volid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch snap_volid");
+                        goto out;
+                }
+
+                /* snap volume uuid is used as lvm snapshot name.
+                   This will avoid restrictions on snapshot names
+                   provided by user */
+                GLUSTERD_GET_UUID_NOHYPHEN (snap_volname, *snap_volid);
+
+                brick_count = 0;
+                brick_order = 0;
+                /* Adding snap bricks mount paths to the dict */
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
+                                brick_order++;
+                                continue;
+                        }
+
+                        if (!glusterd_is_brick_started (brickinfo)) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "brick %s:%s is not started",
+                                        brickinfo->hostname,
+                                        brickinfo->path);
+                                brick_order++;
+                                brick_count++;
+                                continue;
+                        }
+
+                        device = glusterd_get_brick_mount_details (brickinfo);
+                        if (!device) {
+                                snprintf (err_str, sizeof (err_str),
+                                          "getting device name for the brick "
+                                          "%s:%s failed", brickinfo->hostname,
+                                          brickinfo->path);
+                                ret = -1;
+                                goto out;
+                        }
+
+                        device = glusterd_build_snap_device_path (device,
+                                                                  snap_volname);
+                        if (!device) {
+                                snprintf (err_str, sizeof (err_str),
+                                          "cannot copy the snapshot device "
+                                          "name (volname: %s, snapname: %s)",
+                                          volinfo->volname, snapname);
+                                loglevel = GF_LOG_WARNING;
+                                ret = -1;
+                                goto out;
+                        }
+
+                        snprintf (key, sizeof(key),
+                                  "vol%ld.brick_snapdevice%ld", i,
+                                  brick_count);
+                        ret = dict_set_dynstr (rsp_dict, key, device);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                GF_FREE (device);
+                                goto out;
+                        }
+
+                        ret = glusterd_get_brick_root (brickinfo->path,
+                                                       &mnt_pt);
+                        if (ret) {
+                                snprintf (err_str, sizeof (err_str),
+                                "could not get the root of the brick path %s",
+                                brickinfo->path);
+                                loglevel = GF_LOG_WARNING;
+                                goto out;
+                        }
+                        if (strncmp (brickinfo->path, mnt_pt, strlen(mnt_pt))) {
+                                snprintf (err_str, sizeof (err_str),
+                                "brick: %s brick mount: %s",
+                                 brickinfo->path, mnt_pt);
+                                loglevel = GF_LOG_WARNING;
+                                goto out;
+                        }
+
+                        brick_dir = &brickinfo->path[strlen (mnt_pt)];
+                        brick_dir++;
+
+                        snprintf (snap_brick_dir, sizeof (snap_brick_dir),
+                                  "/%s", brick_dir);
+
+                        tmpstr = gf_strdup (snap_brick_dir);
+                        if (!tmpstr) {
+                                ret = -1;
+                                goto out;
+                        }
+                        snprintf (key, sizeof(key), "vol%ld.brickdir%ld", i,
+                                  brick_count);
+                        ret = dict_set_dynstr (rsp_dict, key, tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", snap_mount);
+                                goto out;
+                        }
+                        tmpstr = NULL;
+
+                        snprintf (key, sizeof(key) - 1, "vol%ld.brick%ld.order",
+                                  i, brick_count);
+                        ret = dict_set_int64 (rsp_dict, key, brick_order);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                goto out;
+                        }
+
+                        brick_count++;
+                        brick_order++;
+                }
+                snprintf (key, sizeof(key) - 1, "vol%ld_brickcount", i);
+                ret = dict_set_int64 (rsp_dict, key, brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
+                                key);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int64 (rsp_dict, "volcount", volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret)
+                GF_FREE (tmpstr);
+
+        if (ret && err_str[0] != '\0') {
+                gf_log (this->name, loglevel, "%s", err_str);
+                *op_errstr = gf_strdup (err_str);
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+glusterd_snap_t*
+glusterd_new_snap_object()
+{
+        glusterd_snap_t        *snap    = NULL;
+
+        snap = GF_CALLOC (1, sizeof (*snap), gf_gld_mt_snap_t);
+
+        if (snap) {
+                if (LOCK_INIT (&snap->lock)) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Failed initiating"
+                                " snap lock");
+                        GF_FREE (snap);
+                        return NULL;
+                }
+
+                INIT_LIST_HEAD (&snap->snap_list);
+                INIT_LIST_HEAD (&snap->volumes);
+                snap->snapname[0] = 0;
+                snap->snap_status = GD_SNAP_STATUS_INIT;
+        }
+
+        return snap;
+
+};
+
+/* Function glusterd_list_add_snapvol adds the volinfo object (snapshot volume)
+   to the snapshot object list and to the parent volume list */
+int32_t
+glusterd_list_add_snapvol (glusterd_volinfo_t *origin_vol,
+                           glusterd_volinfo_t *snap_vol)
+{
+        int ret               = -1;
+        glusterd_snap_t *snap = NULL;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", origin_vol, out);
+        GF_VALIDATE_OR_GOTO ("glusterd", snap_vol, out);
+
+        snap = snap_vol->snapshot;
+        GF_ASSERT (snap);
+
+        list_add_tail (&snap_vol->vol_list, &snap->volumes);
+        LOCK (&origin_vol->lock);
+        {
+                list_add_order (&snap_vol->snapvol_list,
+                               &origin_vol->snap_volumes,
+                               glusterd_compare_snap_vol_time);
+                origin_vol->snap_count++;
+        }
+        UNLOCK (&origin_vol->lock);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Snap %s added to the list",
+                snap->snapname);
+        ret = 0;
+out:
+        return ret;
+}
+
+glusterd_snap_t*
+glusterd_find_snap_by_name (char *snapname)
+{
+        glusterd_snap_t *snap  = NULL;
+        glusterd_conf_t *priv  = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snapname);
+
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                if (!strcmp (snap->snapname, snapname)) {
+                        gf_log (THIS->name, GF_LOG_DEBUG, "Found "
+                                "snap %s (%s)", snap->snapname,
+                                uuid_utoa (snap->snap_id));
+                        goto out;
+                }
+        }
+        snap = NULL;
+out:
+        return snap;
+}
+
+glusterd_snap_t*
+glusterd_find_snap_by_id (uuid_t snap_id)
+{
+        glusterd_snap_t *snap  = NULL;
+        glusterd_conf_t *priv  = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        if (uuid_is_null(snap_id))
+                goto out;
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                if (!uuid_compare (snap->snap_id, snap_id)) {
+                        gf_log (THIS->name, GF_LOG_DEBUG, "Found "
+                                "snap %s (%s)", snap->snapname,
+                                uuid_utoa (snap->snap_id));
+                        goto out;
+                }
+        }
+        snap = NULL;
+out:
+        return snap;
+}
+
+int
+glusterd_do_lvm_snapshot_remove (glusterd_volinfo_t *snap_vol,
+                                 glusterd_brickinfo_t *brickinfo,
+                                 const char *mount_pt, const char *snap_device)
+{
+        int                     ret               = -1;
+        xlator_t               *this              = NULL;
+        glusterd_conf_t        *priv              = NULL;
+        runner_t                runner            = {0,};
+        char                    msg[1024]         = {0, };
+        char                    pidfile[PATH_MAX] = {0, };
+        pid_t                   pid               = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!brickinfo) {
+                gf_log (this->name, GF_LOG_ERROR, "brickinfo NULL");
+                goto out;
+        }
+
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (mount_pt);
+        GF_ASSERT (snap_device);
+
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_vol, brickinfo, priv);
+        if (glusterd_is_service_running (pidfile, &pid)) {
+                ret = kill (pid, SIGKILL);
+                if (ret && errno != ESRCH) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to kill pid "
+                                "%d reason : %s", pid, strerror(errno));
+                        goto out;
+                }
+        }
+
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "umount the snapshot mounted path %s",
+                  mount_pt);
+        runner_add_args (&runner, "umount", mount_pt, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+
+        /* We need not do synclock_unlock => runner_run => synclock_lock here.
+           Because it is needed if we are running a glusterfs process in
+           runner_run, so that when the glusterfs process started wants to
+           communicate to glusterd, glusterd wont be able to respond if it
+           has held the big lock. So we do unlock, run glusterfs process
+           (thus communicate to glusterd), lock. But since this is not a
+           glusterfs command that is being run, unlocking and then relocking
+           is not needed.
+        */
+        ret = runner_run (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "unmounting the "
+                        "path %s (brick: %s) failed (%s)", mount_pt,
+                        brickinfo->path, strerror (errno));
+                goto out;
+        }
+
+        runinit (&runner);
+        snprintf (msg, sizeof(msg), "remove snapshot of the brick %s:%s, "
+                  "device: %s", brickinfo->hostname, brickinfo->path,
+                  snap_device);
+        runner_add_args (&runner, "/sbin/lvremove", "-f", snap_device, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+
+        ret = runner_run (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "removing snapshot of the "
+                        "brick (%s:%s) of device %s failed",
+                        brickinfo->hostname, brickinfo->path, snap_device);
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
+{
+        char                 *mnt_pt        = NULL;
+        struct mntent        *entry         = NULL;
+        int32_t               brick_count   = -1;
+        int32_t               ret           = -1;
+        glusterd_brickinfo_t *brickinfo     = NULL;
+        xlator_t             *this          = NULL;
+        FILE                 *mtab          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_vol);
+
+        if (!snap_vol) {
+                gf_log (this->name, GF_LOG_ERROR, "snap volinfo is NULL");
+                goto out;
+        }
+
+        brick_count = -1;
+        list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
+                brick_count++;
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                if (brickinfo->snap_status == -1) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "snapshot was pending. lvm not present "
+                                "for brick %s:%s of the snap %s.",
+                                brickinfo->hostname, brickinfo->path,
+                                snap_vol->snapshot->snapname);
+
+                        /* Adding missed delete to the dict */
+                        ret = glusterd_add_missed_snaps_to_dict
+                                                   (rsp_dict,
+                                                    snap_vol->volname,
+                                                    brickinfo,
+                                                    brick_count + 1,
+                                                    GF_SNAP_OPTION_TYPE_DELETE);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to add missed snapshot info "
+                                        "for %s:%s in the rsp_dict",
+                                        brickinfo->hostname,
+                                        brickinfo->path);
+                                goto out;
+                        }
+
+                        continue;
+                }
+
+                ret = glusterd_get_brick_root (brickinfo->path, &mnt_pt);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "getting the root "
+                               "of the brick for volume %s (snap %s) failed ",
+                               snap_vol->volname, snap_vol->snapshot->snapname);
+                        goto out;
+                }
+
+                entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+                if (!entry) {
+                        gf_log (this->name, GF_LOG_WARNING, "getting the mount"
+                                " entry for the brick %s:%s of the snap %s "
+                                "(volume: %s) failed", brickinfo->hostname,
+                                brickinfo->path, snap_vol->snapshot->snapname,
+                                snap_vol->volname);
+                        ret = -1;
+                        goto out;
+                }
+                ret = glusterd_do_lvm_snapshot_remove (snap_vol, brickinfo,
+                                                       mnt_pt,
+                                                       entry->mnt_fsname);
+                if (mtab)
+                        endmntent (mtab);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "remove the snapshot %s (%s)",
+                                brickinfo->path, entry->mnt_fsname);
+                        goto out;
+                }
+
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snap_volume_remove (dict_t *rsp_dict,
+                             glusterd_volinfo_t *snap_vol,
+                             gf_boolean_t remove_lvm,
+                             gf_boolean_t force)
+{
+        int                   ret         = -1;
+        int                   save_ret    =  0;
+        glusterd_brickinfo_t *brickinfo   = NULL;
+        glusterd_volinfo_t   *origin_vol  = NULL;
+        xlator_t             *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_vol);
+
+        if (!snap_vol) {
+                gf_log(this->name, GF_LOG_WARNING, "snap_vol in NULL");
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                ret = glusterd_brick_stop (snap_vol, brickinfo, _gf_false);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_WARNING, "Failed to stop "
+                               "brick for volume %s", snap_vol->volname);
+                        save_ret = ret;
+
+                        /* Continue to cleaning up the snap in case of error
+                           if force flag is enabled */
+                        if (!force)
+                                goto out;
+                }
+        }
+
+        /* Only remove the backend lvm when required */
+        if (remove_lvm) {
+                ret = glusterd_lvm_snapshot_remove (rsp_dict, snap_vol);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_WARNING, "Failed to remove "
+                               "lvm snapshot volume %s", snap_vol->volname);
+                        save_ret = ret;
+                        if (!force)
+                                goto out;
+                }
+        }
+
+        ret = glusterd_store_delete_volume (snap_vol);
+        if (ret) {
+                gf_log(this->name, GF_LOG_WARNING, "Failed to remove volume %s "
+                       "from store", snap_vol->volname);
+                save_ret = ret;
+                if (!force)
+                        goto out;
+        }
+
+        if (!list_empty(&snap_vol->snapvol_list)) {
+                ret = glusterd_volinfo_find (snap_vol->parent_volname,
+                                             &origin_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "parent volinfo %s  for volume  %s",
+                                snap_vol->parent_volname, snap_vol->volname);
+                        save_ret = ret;
+                        if (!force)
+                                goto out;
+                }
+                origin_vol->snap_count--;
+        }
+
+        ret = glusterd_volinfo_delete (snap_vol);
+        if (ret) {
+                gf_log(this->name, GF_LOG_WARNING, "Failed to remove volinfo "
+                       "%s ", snap_vol->volname);
+                save_ret = ret;
+                if (!force)
+                        goto out;
+        }
+
+        if (save_ret)
+                ret = save_ret;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_snapobject_delete (glusterd_snap_t *snap)
+{
+        if (snap == NULL) {
+                gf_log(THIS->name, GF_LOG_WARNING, "snap is NULL");
+                return -1;
+        }
+
+        list_del_init (&snap->snap_list);
+        list_del_init (&snap->volumes);
+        if (LOCK_DESTROY(&snap->lock))
+                gf_log (THIS->name, GF_LOG_WARNING, "Failed destroying lock"
+                        "of snap %s", snap->snapname);
+
+        GF_FREE (snap->description);
+        GF_FREE (snap);
+
+        return 0;
+}
+
+int32_t
+glusterd_snap_remove (dict_t *rsp_dict,
+                      glusterd_snap_t *snap,
+                      gf_boolean_t remove_lvm,
+                      gf_boolean_t force)
+{
+        int                 ret       = -1;
+        int                 save_ret  =  0;
+        glusterd_volinfo_t *snap_vol  = NULL;
+        glusterd_volinfo_t *tmp       = NULL;
+        xlator_t           *this      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap);
+
+        if (!snap) {
+                gf_log(this->name, GF_LOG_WARNING, "snap is NULL");
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
+                ret = glusterd_snap_volume_remove (rsp_dict, snap_vol,
+                                                   remove_lvm, force);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_WARNING, "Failed to remove "
+                               "volinfo %s for snap %s", snap_vol->volname,
+                               snap->snapname);
+                        save_ret = ret;
+
+                        /* Continue to cleaning up the snap in case of error
+                           if force flag is enabled */
+                        if (!force)
+                                goto out;
+                }
+        }
+
+        ret = glusterd_store_delete_snap (snap);
+        if (ret) {
+                gf_log(this->name, GF_LOG_WARNING, "Failed to remove snap %s "
+                       "from store", snap->snapname);
+                save_ret = ret;
+                if (!force)
+                        goto out;
+        }
+
+        ret = glusterd_snapobject_delete (snap);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
+                        "snap object %s", snap->snapname);
+
+        if (save_ret)
+                ret = save_ret;
+out:
+        gf_log (THIS->name, GF_LOG_TRACE, "returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_snapshot_get_snapvol_detail (dict_t *dict,
+                                      glusterd_volinfo_t *snap_vol,
+                                      char *keyprefix, int detail)
+{
+        int                 ret           = -1;
+        int                 snap_limit    = 0;
+        char                key[PATH_MAX] = {0,};
+        char               *value         = NULL;
+        glusterd_volinfo_t *origin_vol    = NULL;
+        glusterd_conf_t    *conf          = NULL;
+        xlator_t           *this          = NULL;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (keyprefix);
+
+        /* Volume Name */
+        value = gf_strdup (snap_vol->volname);
+        if (!value)
+                goto out;
+
+        snprintf (key, sizeof (key), "%s.volname", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "volume name in dictionary: %s", key);
+                goto out;
+        }
+
+        /* Volume ID */
+        value = gf_strdup (uuid_utoa (snap_vol->volume_id));
+        if (NULL == value) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.vol-id", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "volume id in dictionary: %s", key);
+                goto out;
+        }
+        value = NULL;
+
+        /* volume status */
+        snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
+        switch (snap_vol->status) {
+        case GLUSTERD_STATUS_STARTED:
+                ret = dict_set_str (dict, key, "Started");
+                break;
+        case GLUSTERD_STATUS_STOPPED:
+                ret = dict_set_str (dict, key, "Stopped");
+                break;
+        case GD_SNAP_STATUS_NONE:
+                ret = dict_set_str (dict, key, "None");
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Invalid volume status");
+                ret = -1;
+                goto out;
+        }
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volume status"
+                        " in dictionary: %s", key);
+                goto out;
+        }
+
+
+        ret = glusterd_volinfo_find (snap_vol->parent_volname, &origin_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to get the parent "
+                        "volinfo for the volume %s", snap_vol->volname);
+                goto out;
+        }
+
+        /* Snaps available */
+        if (conf->snap_max_hard_limit < origin_vol->snap_max_hard_limit) {
+                snap_limit = conf->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "system snap-max-hard-limit is"
+                       " lesser than volume snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        } else {
+                snap_limit = origin_vol->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "volume snap-max-hard-limit is"
+                       " lesser than system snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        }
+
+        snprintf (key, sizeof (key), "%s.snaps-available", keyprefix);
+        if (snap_limit > origin_vol->snap_count)
+                ret = dict_set_int32 (dict, key,
+                        snap_limit - origin_vol->snap_count);
+        else
+                ret = dict_set_int32 (dict, key, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set available snaps");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.snapcount", keyprefix);
+        ret = dict_set_int32 (dict, key, origin_vol->snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save snapcount");
+                goto out;
+        }
+
+        if (!detail)
+                goto out;
+
+        /* Parent volume name */
+        value = gf_strdup (snap_vol->parent_volname);
+        if (!value)
+                goto out;
+
+        snprintf (key, sizeof (key), "%s.origin-volname", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set parent "
+                        "volume name in dictionary: %s", key);
+                goto out;
+        }
+        value = NULL;
+
+        ret = 0;
+out:
+        if (value)
+                GF_FREE (value);
+
+        return ret;
+}
+
+static int
+glusterd_snapshot_get_snap_detail (dict_t *dict, glusterd_snap_t *snap,
+                                char *keyprefix, glusterd_volinfo_t *volinfo)
+{
+        int                 ret           = -1;
+        int                 volcount      = 0;
+        char                key[PATH_MAX] = {0,};
+        char               *value         = NULL;
+        char               *timestr       = NULL;
+        struct tm          *tmptr         = NULL;
+        glusterd_volinfo_t *snap_vol      = NULL;
+        glusterd_volinfo_t *tmp_vol       = NULL;
+        xlator_t           *this          = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (snap);
+        GF_ASSERT (keyprefix);
+
+        /* Snap Name */
+        value = gf_strdup (snap->snapname);
+        if (!value)
+                goto out;
+
+        snprintf (key, sizeof (key), "%s.snapname", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "snap name in dictionary");
+                goto out;
+        }
+
+        /* Snap ID */
+        value = gf_strdup (uuid_utoa (snap->snap_id));
+        if (NULL == value) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.snap-id", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "snap id in dictionary");
+                goto out;
+        }
+        value = NULL;
+
+        tmptr = localtime (&(snap->time_stamp));
+        if (NULL == tmptr) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to convert "
+                                "time_t to *tm");
+                ret = -1;
+                goto out;
+        }
+
+        timestr = GF_CALLOC (1, PATH_MAX, gf_gld_mt_char);
+        if (NULL == timestr) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = strftime (timestr, PATH_MAX, "%Y-%m-%d %H:%M:%S", tmptr);
+        if (0 == ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to convert time_t "
+                        "to string");
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.snap-time", keyprefix);
+        ret = dict_set_dynstr (dict, key, timestr);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "snap time stamp in dictionary");
+                goto out;
+        }
+        timestr = NULL;
+
+        /* If snap description is provided then add that into dictionary */
+        if (NULL != snap->description) {
+                value = gf_strdup (snap->description);
+                if (NULL == value) {
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "%s.snap-desc", keyprefix);
+                ret = dict_set_dynstr (dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                        "snap description in dictionary");
+                        goto out;
+                }
+                value = NULL;
+        }
+
+        snprintf (key, sizeof (key), "%s.snap-status", keyprefix);
+        switch (snap->snap_status) {
+        case GD_SNAP_STATUS_INIT:
+                ret = dict_set_str (dict, key, "Init");
+                break;
+        case GD_SNAP_STATUS_IN_USE:
+                ret = dict_set_str (dict, key, "In-use");
+                break;
+        case GD_SNAP_STATUS_DECOMMISSION:
+                ret = dict_set_str (dict, key, "Decommisioned");
+                break;
+        case GD_SNAP_STATUS_RESTORED:
+                ret = dict_set_str (dict, key, "Restored");
+                break;
+        case GD_SNAP_STATUS_NONE:
+                ret = dict_set_str (dict, key, "None");
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Invalid snap status");
+                ret = -1;
+                goto out;
+        }
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap status "
+                        "in dictionary");
+                goto out;
+        }
+
+        if (volinfo) {
+                volcount = 1;
+                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, volcount);
+                ret = glusterd_snapshot_get_snapvol_detail (dict,
+                                                        volinfo, key, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "get volume detail %s for snap %s",
+                                snap_vol->volname, snap->snapname);
+                        goto out;
+                }
+                goto done;
+        }
+
+        list_for_each_entry_safe (snap_vol, tmp_vol, &snap->volumes, vol_list) {
+                volcount++;
+                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, volcount);
+                ret = glusterd_snapshot_get_snapvol_detail (dict,
+                                                        snap_vol, key, 1);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "get volume detail %s for snap %s",
+                                snap_vol->volname, snap->snapname);
+                        goto out;
+                }
+        }
+
+done:
+        snprintf (key, sizeof (key), "%s.vol-count", keyprefix);
+        ret = dict_set_int32 (dict, key, volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
+                        key);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (value)
+                GF_FREE (value);
+
+        if (timestr)
+                GF_FREE(timestr);
+
+        return ret;
+}
+
+static int
+glusterd_snapshot_get_all_snap_info (dict_t *dict)
+{
+        int                 ret           = -1;
+        int                 snapcount     = 0;
+        char                key[PATH_MAX] = {0,};
+        glusterd_snap_t    *snap          = NULL;
+        glusterd_snap_t    *tmp_snap      = NULL;
+        glusterd_conf_t    *priv          = NULL;
+        xlator_t           *this          = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* General parameter validation */
+        GF_ASSERT (dict);
+
+        list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots, snap_list) {
+                snapcount++;
+                snprintf (key, sizeof (key), "snap%d", snapcount);
+                ret = glusterd_snapshot_get_snap_detail (dict, snap, key, NULL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "snapdetail for snap %s", snap->snapname);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_get_info_by_volume (dict_t *dict, char *volname,
+                                      char *err_str, size_t len)
+{
+        int                  ret           = -1;
+        int                  snapcount     = 0;
+        int                  snap_limit    = 0;
+        char                *value         = NULL;
+        char                 key[PATH_MAX] = "";
+        glusterd_volinfo_t  *volinfo       = NULL;
+        glusterd_volinfo_t  *snap_vol      = NULL;
+        glusterd_volinfo_t  *tmp_vol       = NULL;
+        glusterd_conf_t     *conf          = NULL;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volname);
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (err_str, len, "Volume (%s) does not exist", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        /* Snaps available */
+        if (conf->snap_max_hard_limit < volinfo->snap_max_hard_limit) {
+                snap_limit = conf->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "system snap-max-hard-limit is"
+                       " lesser than volume snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        } else {
+                snap_limit = volinfo->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "volume snap-max-hard-limit is"
+                       " lesser than system snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        }
+
+        if (snap_limit > volinfo->snap_count)
+                ret = dict_set_int32 (dict, "snaps-available",
+                        snap_limit - volinfo->snap_count);
+        else
+                ret = dict_set_int32 (dict, "snaps-available", 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set available snaps");
+                goto out;
+        }
+
+        /* Origin volume name */
+        value = gf_strdup (volinfo->volname);
+        if (!value)
+                goto out;
+
+        ret = dict_set_dynstr (dict, "origin-volname", value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set parent "
+                        "volume name in dictionary: %s", key);
+                goto out;
+        }
+        value = NULL;
+
+        list_for_each_entry_safe (snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                  snapvol_list) {
+                snapcount++;
+                snprintf (key, sizeof (key), "snap%d", snapcount);
+                ret = glusterd_snapshot_get_snap_detail (dict,
+                                                         snap_vol->snapshot,
+                                                         key, snap_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "snapdetail for snap %s",
+                                snap_vol->snapshot->snapname);
+                        goto out;
+                }
+        }
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (value)
+                GF_FREE (value);
+
+        return ret;
+}
+
+/* This function will be called from RPC handler routine.
+ * This function is responsible for getting the requested
+ * snapshot info into the dictionary.
+ *
+ * @param req   RPC request object. Required for sending a response back.
+ * @param op    glusterd operation. Required for sending a response back.
+ * @param dict  pointer to dictionary which will contain both
+ *              request and response key-pair values.
+ * @return -1 on error and 0 on success
+ */
+int
+glusterd_handle_snapshot_info (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int              ret             =       -1;
+        int8_t          snap_driven      =        1;
+        char            *volname         =      NULL;
+        char            *snapname        =      NULL;
+        glusterd_snap_t *snap            =      NULL;
+        xlator_t        *this            =      NULL;
+        int32_t         cmd              =      GF_SNAP_INFO_TYPE_ALL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_VALIDATE_OR_GOTO (this->name, req, out);
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+
+
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get type "
+                        "of snapshot info");
+                goto out;
+        }
+
+        switch (cmd) {
+                case GF_SNAP_INFO_TYPE_ALL:
+                {
+                        ret = glusterd_snapshot_get_all_snap_info (dict);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get info of all snaps");
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_INFO_TYPE_SNAP:
+                {
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get snap name");
+                                goto out;
+                        }
+
+                        ret = dict_set_int32 (dict, "snap-count", 1);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set snapcount");
+                                goto out;
+                        }
+
+                        snap = glusterd_find_snap_by_name (snapname);
+                        if (!snap) {
+                                snprintf (err_str, len,
+                                         "Snap (%s) does not exist", snapname);
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "%s", err_str);
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = glusterd_snapshot_get_snap_detail (dict, snap,
+                                                                "snap1", NULL);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get snap detail of snap "
+                                        "%s", snap->snapname);
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_INFO_TYPE_VOL:
+                {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get volname");
+                                goto out;
+                        }
+                        ret = glusterd_snapshot_get_info_by_volume (dict,
+                                                        volname, err_str, len);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get volume info of volume "
+                                        "%s", volname);
+                                goto out;
+                        }
+                        snap_driven = 0;
+                        break;
+                }
+        }
+
+        ret = dict_set_int8 (dict, "snap-driven", snap_driven);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap-driven");
+                goto out;
+        }
+
+        /* If everything is successful then send the response back to cli.
+         * In case of failure the caller of this function will take care
+           of the response */
+        ret = glusterd_op_send_cli_response (op, 0, 0, req, dict, err_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to send cli "
+                                "response");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+/* This function sets all the snapshot names in the dictionary */
+int
+glusterd_snapshot_get_all_snapnames (dict_t *dict)
+{
+        int              ret             = -1;
+        int              snapcount       = 0;
+        char            *snapname        = NULL;
+        char             key[PATH_MAX]   = {0,};
+        glusterd_snap_t *snap            = NULL;
+        glusterd_snap_t *tmp_snap        = NULL;
+        glusterd_conf_t *priv            = NULL;
+        xlator_t        *this            = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (dict);
+
+        list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots, snap_list) {
+                snapcount++;
+                snapname = gf_strdup (snap->snapname);
+                if (!snapname) {
+                        gf_log (this->name, GF_LOG_ERROR, "strdup failed");
+                        ret = -1;
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "snapname%d", snapcount);
+                ret = dict_set_dynstr (dict, key, snapname);
+                if (ret) {
+                        GF_FREE (snapname);
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
+                                key);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+
+/* This function sets all the snapshot names
+   under a given volume in the dictionary */
+int
+glusterd_snapshot_get_vol_snapnames (dict_t *dict, glusterd_volinfo_t *volinfo)
+{
+        int                 ret             = -1;
+        int                 snapcount       = 0;
+        char               *snapname        = NULL;
+        char                key[PATH_MAX]   = {0,};
+        glusterd_volinfo_t *snap_vol        = NULL;
+        glusterd_volinfo_t *tmp_vol         = NULL;
+        xlator_t           *this            = NULL;
+
+        this = THIS;
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry_safe (snap_vol, tmp_vol,
+                                  &volinfo->snap_volumes, snapvol_list) {
+                snapcount++;
+                snapname = gf_strdup (snap_vol->snapshot->snapname);
+                if (!snapname) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "strdup failed");
+                        ret = -1;
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "snapname%d", snapcount);
+                ret = dict_set_dynstr (dict, key, snapname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "set %s", key);
+                        GF_FREE (snapname);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+
+int
+glusterd_handle_snapshot_list (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int                 ret             = -1;
+        char               *volname         = NULL;
+        glusterd_volinfo_t *volinfo         = NULL;
+        xlator_t           *this            = NULL;
+
+        this = THIS;
+
+        GF_VALIDATE_OR_GOTO (this->name, req, out);
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+
+        /* Ignore error for getting volname as it is optional */
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (NULL == volname) {
+                ret = glusterd_snapshot_get_all_snapnames (dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get snapshot list");
+                        goto out;
+                }
+        } else {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, len,
+                                 "Volume (%s) does not exist", volname);
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s", err_str);
+                        goto out;
+                }
+
+                ret = glusterd_snapshot_get_vol_snapnames (dict, volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get snapshot list for volume %s",
+                                volname);
+                        goto out;
+                }
+        }
+
+        /* If everything is successful then send the response back to cli.
+        In case of failure the caller of this function will take of response.*/
+        ret = glusterd_op_send_cli_response (op, 0, 0, req, dict, err_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to send cli "
+                                "response");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+/* This is a snapshot create handler function. This function will be
+ * executed in the originator node. This function is responsible for
+ * calling mgmt_v3 framework to do the actual snap creation on all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot restore request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_handle_snapshot_create (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int           ret                     = -1;
+        char         *volname                 = NULL;
+        char         *snapname                = NULL;
+        int64_t       volcount                = 0;
+        xlator_t     *this                    = NULL;
+        char          key[PATH_MAX]           = "";
+        char         *username                = NULL;
+        char         *password                = NULL;
+        uuid_t       *uuid_ptr                = NULL;
+        uuid_t        tmp_uuid                = {0};
+        int           i                       = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_int64 (dict, "volcount", &volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                        "get the volume count");
+                goto out;
+        }
+        if (volcount <= 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid volume count %ld "
+                        "supplied", volcount);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to get the snapname");
+                goto out;
+        }
+
+        if (strlen(snapname) >= GLUSTERD_MAX_SNAP_NAME) {
+                snprintf (err_str, len, "snapname cannot exceed 255 "
+                          "characters");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+        uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!uuid_ptr) {
+                gf_log (this->name, GF_LOG_ERROR, "Out Of Memory");
+                ret = -1;
+                goto out;
+        }
+
+        uuid_generate (*uuid_ptr);
+        ret = dict_set_bin (dict, "snap-id", uuid_ptr, sizeof(uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to set snap-id");
+                GF_FREE (uuid_ptr);
+                goto out;
+        }
+        uuid_ptr = NULL;
+
+        ret = dict_set_int64 (dict, "snap-time", (int64_t)time(NULL));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to set snap-time");
+                goto out;
+        }
+
+        for (i = 1; i <= volcount; i++) {
+                snprintf (key, sizeof (key), "volname%d", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get volume name");
+                        goto out;
+                }
+
+                /* generate internal username and password  for the snap*/
+                uuid_generate (tmp_uuid);
+                username = gf_strdup (uuid_utoa (tmp_uuid));
+                snprintf (key, sizeof(key), "volume%d_username", i);
+                ret = dict_set_dynstr (dict, key, username);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
+                                "username for volume %s", volname);
+                        GF_FREE (username);
+                        goto out;
+                }
+
+                uuid_generate (tmp_uuid);
+                password = gf_strdup (uuid_utoa (tmp_uuid));
+                snprintf (key, sizeof(key), "volume%d_password", i);
+                ret = dict_set_dynstr (dict, key, password);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
+                                "password for volume %s", volname);
+                        GF_FREE (password);
+                        goto out;
+                }
+
+                uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+                if (!uuid_ptr) {
+                        gf_log (this->name, GF_LOG_ERROR, "Out Of Memory");
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (key, sizeof(key) - 1, "vol%d_volid", i);
+                uuid_generate (*uuid_ptr);
+                ret = dict_set_bin (dict, key, uuid_ptr, sizeof(uuid_t));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set snap_volid");
+                        GF_FREE (uuid_ptr);
+                        goto out;
+                }
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
+                                "phases");
+        }
+
+out:
+        return ret;
+}
+
+/* This is a snapshot status handler function. This function will be
+ * executed in a originator node. This function is responsible for
+ * calling mgmt v3 framework to get the actual snapshot status from
+ * all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot status request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * return :  0  in case of success.
+ *          -1  in case of failure.
+ *
+ */
+int
+glusterd_handle_snapshot_status (rpcsvc_request_t *req, glusterd_op_t op,
+                                 dict_t *dict, char *err_str, size_t len)
+{
+        int                     ret             =       -1;
+        char                    *volname        =       NULL;
+        char                    *snapname       =       NULL;
+        char                    *buf            =       NULL;
+        glusterd_conf_t         *conf           =       NULL;
+        xlator_t                *this           =       NULL;
+        int32_t                 cmd             =       -1;
+        int                     i               =       0;
+        dict_t                  *voldict        =       NULL;
+        char                    key[PATH_MAX]   =       "";
+        glusterd_volinfo_t     *volinfo         =       NULL;
+        glusterd_snap_t         *snap           =       NULL;
+        glusterd_volinfo_t      *snap_volinfo   =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not get status type");
+                goto out;
+        }
+        switch (cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL:
+                {
+                        /* IF we give "gluster snapshot status"
+                         * then lock is held on all snaps.
+                         * This is the place where necessary information
+                         * (snapname and snapcount)is populated in dictionary
+                         * for locking.
+                         */
+                        ++i;
+                        list_for_each_entry (snap, &conf->snapshots, snap_list)
+                        {
+                                snprintf (key, sizeof (key), "snapname%d", i);
+                                buf = gf_strdup (snap->snapname);
+                                if (!buf) {
+                                        ret = -1;
+                                        goto out;
+                                }
+                                ret = dict_set_dynstr (dict, key, buf);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Could not save snapname (%s) "
+                                                "in the dictionary",
+                                                snap->snapname);
+                                        GF_FREE (buf);
+                                        goto out;
+                                }
+
+                                buf = NULL;
+                                i++;
+                        }
+
+                        ret = dict_set_int32 (dict, "snapcount", i - 1);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Could not "
+                                        "save snapcount in the dictionary");
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_STATUS_TYPE_SNAP:
+                {
+                        /* IF we give "gluster snapshot status <snapname>"
+                         * then lock is held on single snap.
+                         * This is the place where necessary information
+                         * (snapname)is populated in dictionary
+                         * for locking.
+                         */
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to fetch snap name");
+                                goto out;
+                        }
+
+                        snap = glusterd_find_snap_by_name (snapname);
+                        if (!snap) {
+                                snprintf (err_str, len, "Snap (%s)"
+                                          "does not exist", snapname);
+                                gf_log(this->name, GF_LOG_ERROR,
+                                        "%s", err_str);
+                                ret = -1;
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_VOL:
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to fetch volname");
+                                goto out;
+                        }
+
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret) {
+                                snprintf (err_str, len, "Volume (%s) "
+                                          "does not exist", volname);
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "%s", err_str);
+                                goto out;
+                        }
+
+                        i = 1;
+                        list_for_each_entry (snap_volinfo,
+                                        &volinfo->snap_volumes, snapvol_list) {
+                                snprintf (key, sizeof (key), "snapname%d", i);
+
+                                buf = gf_strdup
+                                             (snap_volinfo->snapshot->snapname);
+                                if (!buf) {
+                                        ret = -1;
+                                        goto out;
+                                }
+
+                                ret = dict_set_dynstr (dict, key, buf);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Could not save snapname");
+                                        GF_FREE (buf);
+                                        goto out;
+                                }
+
+                                buf = NULL;
+                                i++;
+                        }
+
+                        ret = dict_set_int32 (dict, "snapcount", i-1);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save snapcount");
+                                goto out;
+                        }
+                        break;
+                default:
+                {
+                        gf_log (this->name, GF_LOG_ERROR, "Unknown type");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Volume lock is not necessary for snapshot status, hence
+         * turning it off
+         */
+        ret = dict_set_int8 (dict, "hold_vol_locks", 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Setting volume lock "
+                        "flag failed");
+                goto out;
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate "
+                        "snap phases");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (voldict) {
+                dict_unref (voldict);
+        }
+        return ret;
+}
+
+
+/* This is a snapshot restore handler function. This function will be
+ * executed in the originator node. This function is responsible for
+ * calling mgmt_v3 framework to do the actual restore on all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot restore request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_handle_snapshot_restore (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int                     ret             = -1;
+        char                    *snapname       = NULL;
+        char                    *buf            = NULL;
+        glusterd_conf_t         *conf           = NULL;
+        xlator_t                *this           = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        glusterd_volinfo_t      *snap_volinfo   = NULL;
+        int32_t                 i               =    0;
+        char                    key[PATH_MAX]   =    "";
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                        "get snapname");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                snprintf (err_str, len, "Snap (%s) does not exist", snapname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (snap_volinfo, &snap->volumes, vol_list) {
+                i++;
+                snprintf (key, sizeof (key), "volname%d", i);
+                buf = gf_strdup (snap_volinfo->parent_volname);
+                if (!buf) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_dynstr (dict, key, buf);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not set "
+                                "parent volume name %s in the dict",
+                                snap_volinfo->parent_volname);
+                        GF_FREE (buf);
+                        goto out;
+                }
+                buf = NULL;
+        }
+
+        ret = dict_set_int32 (dict, "volcount", i);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not save volume count");
+                goto out;
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
+                                "phases");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+glusterd_snap_t*
+glusterd_create_snap_object (dict_t *dict, dict_t *rsp_dict)
+{
+        char              *snapname    = NULL;
+        uuid_t            *snap_id     = NULL;
+        char              *description = NULL;
+        glusterd_snap_t   *snap        = NULL;
+        xlator_t          *this        = NULL;
+        glusterd_conf_t   *priv        = NULL;
+        int                ret         = -1;
+        int64_t            time_stamp  = 0;
+
+        this = THIS;
+        priv = this->private;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        /* Fetch snapname, description, id and time from dict */
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snapname");
+                goto out;
+        }
+
+        /* Ignore ret value for description*/
+        ret = dict_get_str (dict, "description", &description);
+
+        ret = dict_get_bin (dict, "snap-id", (void **)&snap_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snap_id");
+                goto out;
+        }
+
+        ret = dict_get_int64 (dict, "snap-time", &time_stamp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snap-time");
+                goto out;
+        }
+        if (time_stamp <= 0) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Invalid time-stamp: %ld",
+                        time_stamp);
+                goto out;
+        }
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                if (!strcmp (snap->snapname, snapname) ||
+                    !uuid_compare (snap->snap_id, *snap_id)) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "Found duplicate snap %s (%s)",
+                                snap->snapname, uuid_utoa (snap->snap_id));
+                        ret = -1;
+                        break;
+                }
+        }
+        if (ret) {
+                snap = NULL;
+                goto out;
+        }
+
+        snap = glusterd_new_snap_object ();
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not create "
+                        "the snap object for snap %s", snapname);
+                goto out;
+        }
+
+        strcpy (snap->snapname, snapname);
+        uuid_copy (snap->snap_id, *snap_id);
+        snap->time_stamp = (time_t)time_stamp;
+        /* Set the status as GD_SNAP_STATUS_INIT and once the backend snapshot
+           is taken and snap is really ready to use, set the status to
+           GD_SNAP_STATUS_IN_USE. This helps in identifying the incomplete
+           snapshots and cleaning them up.
+        */
+        snap->snap_status = GD_SNAP_STATUS_INIT;
+        if (description) {
+                snap->description = gf_strdup (description);
+                if (snap->description == NULL) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Saving the Snap Description Failed");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = glusterd_store_snap (snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Could not store snap"
+                        "object %s", snap->snapname);
+                goto out;
+        }
+
+        list_add_order (&snap->snap_list, &priv->snapshots,
+                        glusterd_compare_snap_time);
+
+        gf_log (this->name, GF_LOG_TRACE, "Snap %s added to the list",
+                snap->snapname);
+
+        ret = 0;
+
+out:
+        if (ret) {
+                if (snap)
+                        glusterd_snap_remove (rsp_dict, snap,
+                                              _gf_true, _gf_true);
+                snap = NULL;
+        }
+
+        return snap;
+}
+
+/* This function is called to get the device path of the snap lvm. Usually
+   if /dev/mapper/<group-name>-<lvm-name> is the device for the lvm,
+   then the snap device will be /dev/<group-name>/<snapname>.
+   This function takes care of building the path for the snap device.
+*/
+char *
+glusterd_build_snap_device_path (char *device, char *snapname)
+{
+        char        snap[PATH_MAX]      = "";
+        char        msg[1024]           = "";
+        char        volgroup[PATH_MAX]  = "";
+        char       *snap_device         = NULL;
+        xlator_t   *this                = NULL;
+        runner_t    runner              = {0,};
+        char       *ptr                 = NULL;
+        int         ret                 = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR, "device is NULL");
+                goto out;
+        }
+        if (!snapname) {
+                gf_log (this->name, GF_LOG_ERROR, "snapname is NULL");
+                goto out;
+        }
+
+        runinit (&runner);
+        runner_add_args (&runner, "/sbin/lvs", "--noheadings", "-o", "vg_name",
+                         device, NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        snprintf (msg, sizeof (msg), "Get volume group for device %s", device);
+        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
+        ret = runner_start (&runner);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume group "
+                        "for device %s", device);
+                runner_end (&runner);
+                goto out;
+        }
+        ptr = fgets(volgroup, sizeof(volgroup),
+                    runner_chio (&runner, STDOUT_FILENO));
+        if (!ptr || !strlen(volgroup)) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume group "
+                        "for snap %s", snapname);
+                runner_end (&runner);
+                ret = -1;
+                goto out;
+        }
+        runner_end (&runner);
+
+        snprintf (snap, sizeof(snap), "/dev/%s/%s", gf_trim(volgroup),
+                  snapname);
+        snap_device = gf_strdup (snap);
+        if (!snap_device) {
+                gf_log (this->name, GF_LOG_WARNING, "Cannot copy the "
+                        "snapshot device name for snapname: %s)", snapname);
+        }
+
+out:
+        return snap_device;
+}
+
+/* This function actually calls the command (or the API) for taking the
+   snapshot of the backend brick filesystem. If this is successful,
+   then call the glusterd_snap_create function to create the snap object
+   for glusterd
+*/
+char *
+glusterd_take_lvm_snapshot (glusterd_volinfo_t *snap_vol,
+                            glusterd_brickinfo_t *brickinfo)
+{
+        char             msg[NAME_MAX]    = "";
+        char             buf[PATH_MAX]    = "";
+        char            *snap_device      = NULL;
+        char            *ptr              = NULL;
+        char            *device           = NULL;
+        int              ret              = -1;
+        gf_boolean_t     match            = _gf_false;
+        runner_t         runner           = {0,};
+        xlator_t        *this             = NULL;
+
+        this = THIS;
+
+        if (!brickinfo) {
+                gf_log (this->name, GF_LOG_ERROR, "brickinfo NULL");
+                goto out;
+        }
+
+        device = glusterd_get_brick_mount_details (brickinfo);
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR, "getting device name for "
+                        "the brick %s:%s failed", brickinfo->hostname,
+                        brickinfo->path);
+                goto out;
+        }
+
+        /* Figuring out if setactivationskip flag is supported or not */
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "running lvcreate help");
+        runner_add_args (&runner, "/sbin/lvcreate", "--help", NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to run lvcreate help");
+                runner_end (&runner);
+                goto out;
+        }
+
+        /* Looking for setactivationskip in lvcreate --help */
+        do {
+                ptr = fgets(buf, sizeof(buf),
+                            runner_chio (&runner, STDOUT_FILENO));
+                if (ptr) {
+                        if (strstr(buf, "setactivationskip")) {
+                                match = _gf_true;
+                                break;
+                        }
+                }
+        } while (ptr != NULL);
+        runner_end (&runner);
+
+        /* Takng the actual snapshot */
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "taking snapshot of the brick %s:%s",
+                  brickinfo->hostname, brickinfo->path);
+        if (match == _gf_true)
+                runner_add_args (&runner, "/sbin/lvcreate", "-s", device,
+                                 "--setactivationskip", "n", "--name",
+                                 snap_vol->volname, NULL);
+        else
+                runner_add_args (&runner, "/sbin/lvcreate", "-s", device,
+                                 "--name", snap_vol->volname, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "taking snapshot of the "
+                        "brick (%s:%s) of device %s failed",
+                        brickinfo->hostname, brickinfo->path, device);
+                runner_end (&runner);
+                goto out;
+        }
+        runner_end (&runner);
+
+        snap_device = glusterd_build_snap_device_path (device,
+                                                       snap_vol->volname);
+        if (!snap_device) {
+                gf_log (this->name, GF_LOG_WARNING, "Cannot copy the snapshot "
+                        "device name for snap %s (volume id: %s)",
+                        snap_vol->snapshot->snapname, snap_vol->volname);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        return snap_device;
+}
+
+int32_t
+glusterd_snap_brick_create (char *device, glusterd_volinfo_t *snap_volinfo,
+                            glusterd_brickinfo_t *original_brickinfo,
+                            int32_t brick_count, char *snap_brick_dir)
+{
+        int32_t          ret                             = -1;
+        xlator_t        *this                            = NULL;
+        glusterd_conf_t *priv                            = NULL;
+        char             snap_brick_mount_path[PATH_MAX] = "";
+        char             snap_brick_path[PATH_MAX]       = "";
+        char             msg[1024]                       = "";
+        struct stat      statbuf                         = {0, };
+        runner_t         runner                          = {0, };
+
+        this = THIS;
+        priv = this->private;
+
+        GF_ASSERT (device);
+        GF_ASSERT (snap_volinfo);
+        GF_ASSERT (original_brickinfo);
+        GF_ASSERT (snap_brick_dir);
+
+        snprintf (snap_brick_mount_path, sizeof (snap_brick_mount_path),
+                  "%s/%s/brick%d",  snap_mount_folder, snap_volinfo->volname,
+                  brick_count+1);
+
+        snprintf (snap_brick_path, sizeof (snap_brick_path), "%s%s",
+                  snap_brick_mount_path, snap_brick_dir);
+
+        ret = mkdir_p (snap_brick_mount_path, 0777, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "creating the brick directory"
+                        " %s for the snapshot %s(device: %s) failed",
+                        snap_brick_mount_path, snap_volinfo->volname, device);
+                goto out;
+        }
+        /* mount the snap logical device on the directory inside
+           /run/gluster/snaps/<snapname>/@snap_brick_mount_path
+           Way to mount the snap brick via mount api is this.
+           ret = mount (device, snap_brick_mount_path, entry->mnt_type,
+                        MS_MGC_VAL, "nouuid");
+           But for now, mounting using runner apis.
+        */
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "mounting snapshot of the brick %s:%s",
+                  original_brickinfo->hostname, original_brickinfo->path);
+        runner_add_args (&runner, "mount", "-o", "nouuid", device,
+                         snap_brick_mount_path, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+
+        /* let glusterd get blocked till snapshot is over */
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "mounting the snapshot "
+                        "logical device %s failed (error: %s)", device,
+                        strerror (errno));
+                goto out;
+        } else
+                gf_log (this->name, GF_LOG_DEBUG, "mounting the snapshot "
+                        "logical device %s successful", device);
+
+        ret = stat (snap_brick_path, &statbuf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "stat of the brick %s"
+                        "(brick mount: %s) failed (%s)", snap_brick_path,
+                        snap_brick_mount_path, strerror (errno));
+                goto out;
+        }
+        ret = sys_lsetxattr (snap_brick_path,
+                             GF_XATTR_VOL_ID_KEY,
+                             snap_volinfo->volume_id, 16,
+                             XATTR_REPLACE);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "extended attribute %s on %s. Reason: "
+                        "%s, snap: %s", GF_XATTR_VOL_ID_KEY,
+                        snap_brick_path, strerror (errno),
+                        snap_volinfo->volname);
+                goto out;
+        }
+
+out:
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "unmounting the snap brick"
+                        " mount %s", snap_brick_mount_path);
+                umount (snap_brick_mount_path);
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Added missed_snap_entry to rsp_dict */
+int32_t
+glusterd_add_missed_snaps_to_dict (dict_t *rsp_dict, char *snap_uuid,
+                                   glusterd_brickinfo_t *brickinfo,
+                                   int32_t brick_number, int32_t op)
+{
+        char                   *buf                             = NULL;
+        char                    missed_snap_entry[PATH_MAX]     = "";
+        char                    name_buf[PATH_MAX]              = "";
+        int32_t                 missed_snap_count               = -1;
+        int32_t                 ret                             = -1;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_uuid);
+        GF_ASSERT (brickinfo);
+
+        snprintf (missed_snap_entry, sizeof(missed_snap_entry),
+                  "%s:%s=%d:%s:%d:%d", uuid_utoa(brickinfo->uuid),
+                  snap_uuid, brick_number, brickinfo->path, op,
+                  GD_MISSED_SNAP_PENDING);
+
+        buf = gf_strdup (missed_snap_entry);
+        if (!buf) {
+                ret = -1;
+                goto out;
+        }
+
+        /* Fetch the missed_snap_count from the dict */
+        ret = dict_get_int32 (rsp_dict, "missed_snap_count",
+                              &missed_snap_count);
+        if (ret) {
+                /* Initialize the missed_snap_count for the first time */
+                missed_snap_count = 0;
+        }
+
+        /* Setting the missed_snap_entry in the rsp_dict */
+        snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                  missed_snap_count);
+        ret = dict_set_dynstr (rsp_dict, name_buf, buf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set missed_snap_entry (%s) "
+                        "in the rsp_dict.", buf);
+                GF_FREE (buf);
+                goto out;
+        }
+        missed_snap_count++;
+
+        /* Setting the new missed_snap_count in the dict */
+        ret = dict_set_int32 (rsp_dict, "missed_snap_count",
+                              missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set missed_snap_count for %s "
+                        "in the rsp_dict.", missed_snap_entry);
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int32_t
+glusterd_add_bricks_to_snap_volume (dict_t *dict, dict_t *rsp_dict,
+                                    glusterd_volinfo_t  *snap_vol,
+                                    glusterd_brickinfo_t *original_brickinfo,
+                                    glusterd_brickinfo_t *snap_brickinfo,
+                                    char **snap_brick_dir, int64_t volcount,
+                                    int32_t brick_count)
+{
+        char                    key[PATH_MAX]                   = "";
+        char                    snap_brick_path[PATH_MAX]       = "";
+        char                   *snap_device                     = NULL;
+        gf_boolean_t            add_missed_snap                 = _gf_false;
+        int32_t                 ret                             = -1;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (original_brickinfo);
+        GF_ASSERT (snap_brickinfo);
+        GF_ASSERT (snap_brick_dir);
+
+        snprintf (key, sizeof(key) - 1, "vol%ld.brickdir%d", volcount,
+                  brick_count);
+        ret = dict_get_ptr (dict, key, (void **)snap_brick_dir);
+        if (ret) {
+                /* Using original brickinfo here because it will be a
+                 * pending snapshot and storing the original brickinfo
+                 * will help in mapping while recreating the missed snapshot
+                 */
+                gf_log (this->name, GF_LOG_WARNING, "Unable to fetch "
+                        "snap mount path (%s). Using original brickinfo", key);
+                snap_brickinfo->snap_status = -1;
+                strcpy (snap_brick_path, original_brickinfo->path);
+
+                /* In origiator node add snaps missed
+                 * from different nodes to the dict
+                 */
+                if (is_origin_glusterd (dict) == _gf_true)
+                        add_missed_snap = _gf_true;
+        } else {
+                /* Create brick-path in the format /var/run/gluster/snaps/ *
+                 * <snap-uuid>/<original-brick#>/snap-brick-dir *
+                 */
+                snprintf (snap_brick_path, sizeof(snap_brick_path),
+                          "%s/%s/brick%d%s", snap_mount_folder,
+                          snap_vol->volname, brick_count+1,
+                          *snap_brick_dir);
+        }
+
+        if ((snap_brickinfo->snap_status != -1) &&
+            (!uuid_compare (original_brickinfo->uuid, MY_UUID)) &&
+            (!glusterd_is_brick_started (original_brickinfo))) {
+                /* In case if the brick goes down after prevalidate. */
+                gf_log (this->name, GF_LOG_WARNING, "brick %s:%s is not"
+                        " started (snap: %s)",
+                        original_brickinfo->hostname,
+                        original_brickinfo->path,
+                        snap_vol->snapshot->snapname);
+
+                snap_brickinfo->snap_status = -1;
+                strcpy (snap_brick_path, original_brickinfo->path);
+                add_missed_snap = _gf_true;
+        }
+
+        if (add_missed_snap) {
+                ret = glusterd_add_missed_snaps_to_dict (rsp_dict,
+                                            snap_vol->volname,
+                                            original_brickinfo,
+                                            brick_count + 1,
+                                            GF_SNAP_OPTION_TYPE_CREATE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to add missed"
+                                " snapshot info for %s:%s in the rsp_dict",
+                                original_brickinfo->hostname,
+                                original_brickinfo->path);
+                        goto out;
+                }
+        }
+
+        snprintf (key, sizeof(key), "vol%ld.brick_snapdevice%d",
+                  volcount, brick_count);
+        ret = dict_get_ptr (dict, key, (void **)&snap_device);
+        if (ret) {
+                /* If the device name is empty, so will be the brick path
+                 * Hence the missed snap has already been added above
+                 */
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch "
+                        "snap device (%s). Leaving empty", key);
+        } else
+                strcpy (snap_brickinfo->device_path, snap_device);
+
+        ret = gf_canonicalize_path (snap_brick_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to canonicalize path");
+                goto out;
+        }
+
+        strcpy (snap_brickinfo->hostname, original_brickinfo->hostname);
+        strcpy (snap_brickinfo->path, snap_brick_path);
+        uuid_copy (snap_brickinfo->uuid, original_brickinfo->uuid);
+        list_add_tail (&snap_brickinfo->brick_list, &snap_vol->bricks);
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int32_t
+glusterd_take_brick_snapshot (glusterd_volinfo_t *origin_vol,
+                              glusterd_volinfo_t *snap_vol, dict_t *rsp_dict,
+                              glusterd_brickinfo_t *original_brickinfo,
+                              glusterd_brickinfo_t *snap_brickinfo,
+                              char *snap_brick_dir, int32_t brick_count)
+{
+        char                   *device                          = NULL;
+        int32_t                 ret                             = -1;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (origin_vol);
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (original_brickinfo);
+        GF_ASSERT (snap_brickinfo);
+        GF_ASSERT (snap_brick_dir);
+
+        device = glusterd_take_lvm_snapshot (snap_vol, original_brickinfo);
+        /* Fail the snapshot even though snapshot on one of
+           the bricks fails. At the end when we check whether
+           the snapshot volume meets quorum or not, then the
+           the snapshot can either be treated as success, or
+           in case of failure we can undo the changes and return
+           failure to cli.  */
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to take snapshot of %s:%s",
+                        original_brickinfo->hostname,
+                        original_brickinfo->path);
+                goto out;
+        }
+
+        /* create the complete brick here */
+        ret = glusterd_snap_brick_create (device, snap_vol,
+                                          original_brickinfo,
+                                          brick_count, snap_brick_dir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "not able to"
+                        " create the brickinfo for the snap %s"
+                        ", volume %s", snap_vol->snapshot->snapname,
+                        origin_vol->volname);
+                goto out;
+        }
+
+out:
+        if (device)
+                GF_FREE (device);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Look for disconnected peers, for missed snap creates or deletes */
+static int32_t
+glusterd_find_missed_snap (dict_t *rsp_dict, glusterd_volinfo_t *vol,
+                           char *snap_uuid, struct list_head *peers,
+                           int32_t op)
+{
+        int32_t                   brick_count          = -1;
+        int32_t                   ret                  = -1;
+        xlator_t                 *this                 = NULL;
+        glusterd_peerinfo_t      *peerinfo             = NULL;
+        glusterd_brickinfo_t     *brickinfo            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (peers);
+        GF_ASSERT (vol);
+        GF_ASSERT (snap_uuid);
+
+        brick_count = 0;
+        list_for_each_entry (brickinfo, &vol->bricks, brick_list) {
+                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+                        /* If the brick belongs to the same node */
+                        brick_count++;
+                        continue;
+                }
+
+                list_for_each_entry (peerinfo, peers, uuid_list) {
+                        if (uuid_compare (peerinfo->uuid, brickinfo->uuid)) {
+                                /* If the brick doesnt belong to this peer */
+                                continue;
+                        }
+
+                        /* Found peer who owns the brick,    *
+                         * if peer is not connected or not   *
+                         * friend add it to missed snap list */
+                        if (!(peerinfo->connected) ||
+                           (peerinfo->state.state !=
+                             GD_FRIEND_STATE_BEFRIENDED)) {
+                                ret = glusterd_add_missed_snaps_to_dict
+                                                   (rsp_dict,
+                                                    snap_uuid,
+                                                    brickinfo,
+                                                    brick_count + 1,
+                                                    op);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to add missed snapshot "
+                                                "info for %s:%s in the "
+                                                "rsp_dict", brickinfo->hostname,
+                                                brickinfo->path);
+                                        goto out;
+                                }
+                        }
+                }
+                brick_count++;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+glusterd_volinfo_t *
+glusterd_do_snap_vol (glusterd_volinfo_t *origin_vol, glusterd_snap_t *snap,
+                      dict_t *dict, dict_t *rsp_dict, int64_t volcount)
+{
+        char                    key[PATH_MAX]                   = "";
+        char                   *snap_brick_dir                  = NULL;
+        char                   *username                        = NULL;
+        char                   *password                        = NULL;
+        glusterd_brickinfo_t   *brickinfo                       = NULL;
+        glusterd_conf_t        *priv                            = NULL;
+        glusterd_volinfo_t     *snap_vol                        = NULL;
+        uuid_t                 *snap_volid                      = NULL;
+        int32_t                 ret                             = -1;
+        int32_t                 brick_count                     = 0;
+        glusterd_brickinfo_t   *snap_brickinfo                  = NULL;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (origin_vol);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        /* fetch username, password and vol_id from dict*/
+        snprintf (key, sizeof(key), "volume%ld_username", volcount);
+        ret = dict_get_str (dict, key, &username);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get %s for "
+                        "snap %s", key, snap->snapname);
+                goto out;
+        }
+
+        snprintf (key, sizeof(key), "volume%ld_password", volcount);
+        ret = dict_get_str (dict, key, &password);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get %s for "
+                        "snap %s", key, snap->snapname);
+                goto out;
+        }
+        snprintf (key, sizeof(key) - 1, "vol%ld_volid", volcount);
+        ret = dict_get_bin (dict, key, (void **)&snap_volid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to fetch snap_volid");
+                goto out;
+        }
+
+        /* We are not setting the username and password here as
+         * we need to set the user name and password passed in
+         * the dictionary
+         */
+        ret = glusterd_volinfo_dup (origin_vol, &snap_vol, _gf_false);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to duplicate volinfo "
+                        "for the snapshot %s", snap->snapname);
+                goto out;
+        }
+
+        /* uuid is used as lvm snapshot name.
+           This will avoid restrictions on snapshot names provided by user */
+        GLUSTERD_GET_UUID_NOHYPHEN (snap_vol->volname, *snap_volid);
+        uuid_copy (snap_vol->volume_id, *snap_volid);
+        snap_vol->is_snap_volume = _gf_true;
+        strcpy (snap_vol->parent_volname, origin_vol->volname);
+        snap_vol->snapshot = snap;
+
+        glusterd_auth_set_username (snap_vol, username);
+        glusterd_auth_set_password (snap_vol, password);
+
+        /* Adding snap brickinfos to the snap volinfo */
+        brick_count = 0;
+        list_for_each_entry (brickinfo, &origin_vol->bricks, brick_list) {
+                snap_brickinfo = NULL;
+
+                ret = glusterd_brickinfo_new (&snap_brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "initializing the brick for the snap "
+                                "volume failed (snapname: %s)", snap->snapname);
+                        goto out;
+                }
+
+                ret = glusterd_add_bricks_to_snap_volume (dict, rsp_dict,
+                                                          snap_vol,
+                                                          brickinfo,
+                                                          snap_brickinfo,
+                                                          &snap_brick_dir,
+                                                          volcount,
+                                                          brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add the snap brick for "
+                                "%s:%s to the snap volume",
+                                brickinfo->hostname, brickinfo->path);
+                        GF_FREE (snap_brickinfo);
+                        goto out;
+                }
+
+                /* Take snapshot of the brick */
+                if ((uuid_compare (brickinfo->uuid, MY_UUID)) ||
+                    (snap_brickinfo->snap_status == -1)) {
+                        brick_count++;
+                        continue;
+                }
+
+                ret = glusterd_take_brick_snapshot (origin_vol, snap_vol,
+                                                    rsp_dict, brickinfo,
+                                                    snap_brickinfo,
+                                                    snap_brick_dir,
+                                                    brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to take snapshot for %s:%s",
+                                brickinfo->hostname, brickinfo->path);
+                        goto out;
+                }
+
+                brick_count++;
+        }
+
+        /*TODO: the quorum check of the snap volume here */
+
+        ret = glusterd_store_volinfo (snap_vol,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store snapshot "
+                        "volinfo (%s) for snap %s", snap_vol->volname,
+                        snap->snapname);
+                goto out;
+        }
+
+        ret = generate_brick_volfiles (snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "generating the brick "
+                        "volfiles for the snap %s (volume: %s) failed",
+                        snap->snapname, origin_vol->volname);
+                goto out;
+        }
+
+        ret = generate_client_volfiles (snap_vol, GF_CLIENT_TRUSTED);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "generating the trusted "
+                        "client volfiles for the snap %s (volume: %s) failed",
+                        snap->snapname, origin_vol->volname);
+                goto out;
+        }
+        ret = generate_client_volfiles (snap_vol, GF_CLIENT_OTHER);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "generating the client "
+                        "volfiles for the snap %s (volume: %s) failed",
+                        snap->snapname, origin_vol->volname);
+                goto out;
+        }
+
+        ret = glusterd_list_add_snapvol (origin_vol, snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "could not add the snap "
+                        "volume %s to the list", snap_vol->volname);
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                if (brickinfo->snap_status == -1) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "not starting snap brick %s:%s for "
+                                "for the snap %s (volume: %s)",
+                                brickinfo->hostname, brickinfo->path,
+                                snap->snapname, origin_vol->volname);
+                        continue;
+                }
+
+                ret = glusterd_brick_start (snap_vol, brickinfo, _gf_true);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "starting the "
+                                "brick %s:%s for the snap %s (volume: %s) "
+                                "failed", brickinfo->hostname, brickinfo->path,
+                                snap->snapname, origin_vol->volname);
+                        goto out;
+                }
+        }
+
+        snap_vol->status = GLUSTERD_STATUS_STARTED;
+        ret = glusterd_store_volinfo (snap_vol,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to store snap volinfo");
+                goto out;
+        }
+
+out:
+        if (ret) {
+                if (snap_vol)
+                        glusterd_snap_volume_remove (rsp_dict, snap_vol,
+                                                     _gf_true, _gf_true);
+                snap_vol = NULL;
+        }
+
+        return snap_vol;
+}
+
+/* This is a snapshot remove handler function. This function will be
+ * executed in the originator node. This function is responsible for
+ * calling mgmt v3 framework to do the actual remove on all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot remove request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_handle_snapshot_remove (rpcsvc_request_t *req, glusterd_op_t op,
+                                 dict_t *dict, char *err_str, size_t len)
+{
+        int                      ret            = -1;
+        int64_t                  volcount       = 0;
+        char                    *snapname       = NULL;
+        char                    *volname        = NULL;
+        char                     key[PATH_MAX]  = "";
+        glusterd_snap_t         *snap           = NULL;
+        glusterd_volinfo_t      *snap_vol       = NULL;
+        glusterd_volinfo_t      *tmp            = NULL;
+        xlator_t                *this           = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get snapname");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                snprintf (err_str, len, "Snap (%s) does not exist", snapname);
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+        /* Set volnames in the dict to get mgmt_v3 lock */
+        list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
+                volcount++;
+                volname = gf_strdup (snap_vol->parent_volname);
+                if (!volname) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR, "strdup failed");
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "volname%ld", volcount);
+                ret = dict_set_dynstr (dict, key, volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "volume name in dictionary");
+                        GF_FREE (volname);
+                        goto out;
+                }
+                volname = NULL;
+        }
+        ret = dict_set_int64 (dict, "volcount", volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
+                goto out;
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
+                        "phases");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_remove_prevalidate (dict_t *dict, char **op_errstr,
+                                      dict_t *rsp_dict)
+{
+        int32_t             ret          = -1;
+        char               *snapname     = NULL;
+        xlator_t           *this        = NULL;
+        glusterd_snap_t    *snap        = NULL;
+
+        this = THIS;
+
+        if (!dict || !op_errstr) {
+                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
+                        "failed");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Snap %s does not exist",
+                        snapname);
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_status_prevalidate (dict_t *dict, char **op_errstr,
+                                      dict_t *rsp_dict)
+{
+        int                     ret             =       -1;
+        char                    *snapname       =       NULL;
+        glusterd_conf_t         *conf           =       NULL;
+        xlator_t                *this           =       NULL;
+        int32_t                 cmd             =       -1;
+        glusterd_volinfo_t      *volinfo        =       NULL;
+        char                    *volname        =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (op_errstr);
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Input dict is NULL");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not fetch status cmd");
+                goto out;
+        }
+
+        switch (cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL:
+                {
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_SNAP:
+                {
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not fetch snapname");
+                                goto out;
+                        }
+
+                        if (!glusterd_find_snap_by_name (snapname)) {
+                                ret = gf_asprintf (op_errstr, "Snap (%s) "
+                                                  "not found", snapname);
+                                if (ret < 0) {
+                                        goto out;
+                                }
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "Snap (%s) "
+                                        "not found", snapname);
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_VOL:
+                {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not fetch volname");
+                                goto out;
+                        }
+
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret) {
+                                ret = gf_asprintf (op_errstr, "Volume (%s)"
+                                                  "not found", volname);
+                                if (ret < 0) {
+                                        goto out;
+                                }
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "Volume "
+                                        "%s not present", volname);
+                                goto out;
+                        }
+                        break;
+
+                }
+                default:
+                {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid command");
+                        break;
+                }
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snapshot_remove_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        int32_t                   ret                  = -1;
+        char                     *snapname             = NULL;
+        char                     *dup_snapname         = NULL;
+        glusterd_snap_t          *snap                 = NULL;
+        glusterd_conf_t          *priv                 = NULL;
+        glusterd_volinfo_t       *snap_volinfo         = NULL;
+        xlator_t                 *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_errstr);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!dict || !op_errstr) {
+                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
+                        "failed");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Snap %s does not exist",
+                        snapname);
+                ret = -1;
+                goto out;
+        }
+
+        if (is_origin_glusterd (dict) == _gf_true) {
+                /* TODO : As of now there is only volume in snapshot.
+                 * Change this when multiple volume snapshot is introduced
+                 */
+                snap_volinfo = list_entry (snap->volumes.next,
+                                           glusterd_volinfo_t,
+                                           vol_list);
+                if (!snap_volinfo) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch snap_volinfo");
+                        ret = -1;
+                        goto out;
+                }
+
+                /* From origin glusterd check if      *
+                 * any peers with snap bricks is down */
+                ret = glusterd_find_missed_snap (rsp_dict, snap_volinfo,
+                                                 snap_volinfo->volname,
+                                                 &priv->peers,
+                                                 GF_SNAP_OPTION_TYPE_DELETE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to find missed snap deletes");
+                        goto out;
+                }
+        }
+
+        ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_false);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to remove snap %s",
+                        snapname);
+                goto out;
+        }
+
+        dup_snapname = gf_strdup (snapname);
+        if (!dup_snapname) {
+                gf_log (this->name, GF_LOG_ERROR, "Strdup failed");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, "snapname", dup_snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set the snapname");
+                GF_FREE (dup_snapname);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_do_snap_cleanup (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t                  ret                   = -1;
+        char                     *name                 = NULL;
+        xlator_t                 *this                 = NULL;
+        glusterd_conf_t          *conf                 = NULL;
+        glusterd_volinfo_t       *volinfo              = NULL;
+        glusterd_snap_t          *snap                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (!dict || !op_errstr) {
+                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "getting the snap "
+                        "name failed (volume: %s)", volinfo->volname);
+                goto out;
+        }
+
+        /*
+          If the snapname is not found that means the failure happened at
+          staging, or in commit, before the snap object is created, in which
+          case there is nothing to cleanup. So set ret to 0.
+        */
+        snap = glusterd_find_snap_by_name (name);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_INFO, "snap %s is not found", name);
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "removing the snap %s failed",
+                        name);
+                goto out;
+        }
+
+        name = NULL;
+
+        ret = 0;
+
+out:
+
+        return ret;
+}
+
+/* In case of a successful, delete or create operation, during post_validate *
+ * look for missed snap operations and update the missed snap lists */
+int32_t
+glusterd_snapshot_update_snaps_post_validate (dict_t *dict, char **op_errstr,
+                                              dict_t *rsp_dict)
+{
+        int32_t                  ret                    = -1;
+        int32_t                  missed_snap_count      = -1;
+        xlator_t                *this                   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_errstr);
+
+        ret = dict_get_int32 (dict, "missed_snap_count",
+                              &missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_store_update_missed_snaps (dict, missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to update missed_snaps_list");
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        int                     ret                     = -1;
+        int64_t                 i                       = 0;
+        int64_t                 volcount                = 0;
+        char                    *snapname               = NULL;
+        char                    *volname                = NULL;
+        char                    *tmp_name               = NULL;
+        char                    key[PATH_MAX]           = "";
+        xlator_t                *this                   = NULL;
+        glusterd_snap_t         *snap                   = NULL;
+        glusterd_volinfo_t      *origin_vol             = NULL;
+        glusterd_volinfo_t      *snap_vol               = NULL;
+        glusterd_conf_t *priv   = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+        GF_ASSERT(op_errstr);
+        GF_ASSERT(rsp_dict);
+        priv = this->private;
+        GF_ASSERT(priv);
+
+        ret = dict_get_int64 (dict, "volcount", &volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                        "get the volume count");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snapname");
+                goto out;
+        }
+        tmp_name = gf_strdup (snapname);
+        if (!tmp_name) {
+                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, "snapname", tmp_name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set snapname in rsp_dict");
+                GF_FREE (tmp_name);
+                goto out;
+        }
+        tmp_name = NULL;
+
+        snap = glusterd_create_snap_object (dict, rsp_dict);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "creating the"
+                        "snap object %s failed", snapname);
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 1; i <= volcount; i++) {
+                snprintf (key, sizeof (key), "volname%ld", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to get volume name");
+                        goto out;
+                }
+
+                ret = glusterd_volinfo_find (volname, &origin_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to get the volinfo for "
+                                "the volume %s", volname);
+                        goto out;
+                }
+
+                /* TODO: Create a stub where the bricks are
+                   added parallely by worker threads so that
+                   the snap creating happens parallely.  */
+                snap_vol = glusterd_do_snap_vol (origin_vol, snap, dict,
+                                                 rsp_dict, i);
+                if (!snap_vol) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_WARNING, "taking the "
+                                "snapshot of the volume %s failed", volname);
+                        goto out;
+                }
+        }
+
+        snap->snap_status = GD_SNAP_STATUS_IN_USE;
+        ret = glusterd_store_snap (snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Could not store snap"
+                        "object %s", snap->snapname);
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (ret) {
+                if (snap)
+                        glusterd_snap_remove (rsp_dict, snap,
+                                              _gf_true, _gf_true);
+                snap = NULL;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+snap_max_hard_limit_set_commit (dict_t *dict, uint64_t value,
+                                char *volname, char **op_errstr)
+{
+        char                err_str[PATH_MAX]    = "";
+        glusterd_conf_t    *conf                 = NULL;
+        glusterd_volinfo_t *volinfo              = NULL;
+        int                 ret                  = -1;
+        xlator_t           *this                 = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (volname);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        /* TODO: Initiate auto deletion when there is a limit change */
+        if (!volname) {
+                /* For system limit */
+                conf->snap_max_hard_limit = value;
+
+                ret = glusterd_store_global_info (this);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to store "
+                                  "snap-max-hard-limit for system");
+                        goto out;
+                }
+        } else {
+               /*  For one volume */
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to get the"
+                                   " volinfo for volume %s", volname);
+                        goto out;
+                }
+
+                volinfo->snap_max_hard_limit = value;
+
+                ret = glusterd_store_volinfo (volinfo,
+                                        GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to store "
+                                 "snap-max-hard-limit for volume %s", volname);
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                *op_errstr = gf_strdup (err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+        }
+        return ret;
+}
+
+int
+snap_max_limits_display_commit (dict_t *rsp_dict, char *volname,
+                                char **op_errstr)
+{
+        char                err_str[PATH_MAX]    = "";
+        char                buf[PATH_MAX]        = "";
+        glusterd_conf_t    *conf                 = NULL;
+        glusterd_volinfo_t *volinfo              = NULL;
+        int                 ret                  = -1;
+        uint64_t            active_hard_limit    = 0;
+        uint64_t            snap_max_limit       = 0;
+        uint64_t            soft_limit_value     = -1;
+        uint64_t            count                = 0;
+        xlator_t           *this                 = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (volname);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        if (!volname) {
+                /* For system limit */
+                list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                        if (volinfo->is_snap_volume == _gf_true)
+                                continue;
+                        snap_max_limit = volinfo->snap_max_hard_limit;
+                        if (snap_max_limit > conf->snap_max_hard_limit)
+                                active_hard_limit = conf->snap_max_hard_limit;
+                        else
+                                active_hard_limit = snap_max_limit;
+                        soft_limit_value = (active_hard_limit *
+                                            conf->snap_max_soft_limit) / 100;
+
+                        snprintf (buf, sizeof(buf), "volume%ld-volname", count);
+                        ret = dict_set_str (rsp_dict, buf, volinfo->volname);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-hard-limit", count);
+                        ret = dict_set_uint64 (rsp_dict, buf, snap_max_limit);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-active-hard-limit", count);
+                        ret = dict_set_uint64 (rsp_dict, buf,
+                                               active_hard_limit);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-soft-limit", count);
+                        ret = dict_set_uint64 (rsp_dict, buf, soft_limit_value);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+                        count++;
+                }
+
+                ret = dict_set_uint64 (rsp_dict, "voldisplaycount", count);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set voldisplaycount");
+                        goto out;
+                }
+        } else {
+                /*  For one volume */
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to get the"
+                                   " volinfo for volume %s", volname);
+                        goto out;
+                }
+
+                snap_max_limit = volinfo->snap_max_hard_limit;
+                if (snap_max_limit > conf->snap_max_hard_limit)
+                        active_hard_limit = conf->snap_max_hard_limit;
+                else
+                        active_hard_limit = snap_max_limit;
+
+                soft_limit_value = (active_hard_limit *
+                                    conf->snap_max_soft_limit) / 100;
+
+                snprintf (buf, sizeof(buf), "volume%ld-volname", count);
+                ret = dict_set_str (rsp_dict, buf, volinfo->volname);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                snprintf (buf, sizeof(buf),
+                          "volume%ld-snap-max-hard-limit", count);
+                ret = dict_set_uint64 (rsp_dict, buf, snap_max_limit);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                snprintf (buf, sizeof(buf),
+                          "volume%ld-active-hard-limit", count);
+                ret = dict_set_uint64 (rsp_dict, buf, active_hard_limit);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                snprintf (buf, sizeof(buf),
+                          "volume%ld-snap-max-soft-limit", count);
+                ret = dict_set_uint64 (rsp_dict, buf, soft_limit_value);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                count++;
+
+                ret = dict_set_uint64 (rsp_dict, "voldisplaycount", count);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set voldisplaycount");
+                        goto out;
+                }
+
+        }
+
+        ret = dict_set_uint64 (rsp_dict, "snap-max-hard-limit",
+                               conf->snap_max_hard_limit);
+        if (ret) {
+                snprintf (err_str, PATH_MAX,
+                          "Failed to set sys-snap-max-hard-limit ");
+                goto out;
+        }
+
+        ret = dict_set_uint64 (rsp_dict, "snap-max-soft-limit",
+                               conf->snap_max_soft_limit);
+        if (ret) {
+                snprintf (err_str, PATH_MAX,
+                          "Failed to set sys-snap-max-hard-limit ");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                *op_errstr = gf_strdup (err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+        }
+        return ret;
+}
+
+int
+glusterd_snapshot_config_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        char               *volname              = NULL;
+        xlator_t           *this                 = NULL;
+        int                 ret                  = -1;
+        char                err_str[PATH_MAX]    = {0,};
+        glusterd_conf_t    *conf                 = NULL;
+        int                 config_command       = 0;
+        uint64_t            hard_limit           = 0;
+        uint64_t            soft_limit           = 0;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        ret = dict_get_int32 (dict, "config-command", &config_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get config-command type");
+                goto out;
+        }
+
+        /* Ignore the return value of the following dict_get,
+         * as they are optional
+         */
+        ret = dict_get_str (dict, "volname", &volname);
+
+        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
+
+        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+                if (hard_limit) {
+                        /* Commit ops for snap-max-hard-limit */
+                        ret = snap_max_hard_limit_set_commit (dict, hard_limit,
+                                                              volname,
+                                                              op_errstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "snap-max-hard-limit set "
+                                        "commit failed.");
+                                goto out;
+                        }
+                }
+
+                if (soft_limit) {
+                        /* For system limit */
+                        conf->snap_max_soft_limit = soft_limit;
+
+                        ret = glusterd_store_global_info (this);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX, "Failed to store "
+                                          "snap-max-soft-limit for system");
+                                *op_errstr = gf_strdup (err_str);
+                                gf_log (this->name, GF_LOG_ERROR, "%s",
+                                        err_str);
+                                goto out;
+                        }
+                }
+                break;
+
+        case GF_SNAP_CONFIG_DISPLAY:
+                /* Reading data from local node only */
+                if (!is_origin_glusterd (dict)) {
+                        ret = 0;
+                        break;
+                }
+
+                ret = snap_max_limits_display_commit (rsp_dict, volname,
+                                                      op_errstr);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "snap-max-limit "
+                                "display commit failed.");
+                        goto out;
+                }
+                break;
+        default:
+                break;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_get_brick_lvm_details (dict_t *rsp_dict,
+                               glusterd_brickinfo_t *brickinfo, char *volname,
+                                char *device, char *key_prefix)
+{
+
+        int                     ret             =       -1;
+        glusterd_conf_t         *priv           =       NULL;
+        runner_t                runner          =       {0,};
+        xlator_t                *this           =       NULL;
+        char                    msg[PATH_MAX]   =       "";
+        char                    buf[PATH_MAX]   =       "";
+        char                    *ptr            =       NULL;
+        char                    *token          =       NULL;
+        char                    key[PATH_MAX]   =       "";
+        char                    *value          =       NULL;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (volname);
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        device = glusterd_get_brick_mount_details (brickinfo);
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR, "Getting device name for "
+                        "the brick %s:%s failed", brickinfo->hostname,
+                         brickinfo->path);
+                goto out;
+        }
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "running lvs command, "
+                  "for getting snap status");
+        /* Using lvs command fetch the Volume Group name,
+         * Percentage of data filled and Logical Volume size
+         *
+         * "-o" argument is used to get the desired information,
+         * example : "lvs /dev/VolGroup/thin_vol -o vgname,lv_size",
+         * will get us Volume Group name and Logical Volume size.
+         *
+         * Here separator used is ":",
+         * for the above given command with separator ":",
+         * The output will be "vgname:lvsize"
+         */
+        runner_add_args (&runner, "lvs", device, "--noheading", "-o",
+                         "vg_name,data_percent,lv_size",
+                         "--separator", ":", NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not perform lvs action");
+                goto end;
+        }
+        do {
+                ptr = fgets (buf, sizeof (buf),
+                             runner_chio (&runner, STDOUT_FILENO));
+
+                if (ptr == NULL)
+                        break;
+                token = strtok (buf, ":");
+                if (token != NULL) {
+                        while (token && token[0] == ' ')
+                                token++;
+                        if (!token) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Invalid vg entry");
+                                goto end;
+                        }
+                        value = gf_strdup (token);
+                        if (!value) {
+                                ret = -1;
+                                goto end;
+                        }
+                        ret = snprintf (key, sizeof (key), "%s.vgname",
+                                        key_prefix);
+                        if (ret < 0) {
+                                goto end;
+                        }
+
+                        ret = dict_set_dynstr (rsp_dict, key, value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save vgname ");
+                                goto end;
+                        }
+                }
+
+                token = strtok (NULL, ":");
+                if (token != NULL) {
+                        value = gf_strdup (token);
+                        if (!value) {
+                                ret = -1;
+                                goto end;
+                        }
+                        ret = snprintf (key, sizeof (key), "%s.data",
+                                        key_prefix);
+                        if (ret < 0) {
+                                goto end;
+                        }
+
+                        ret = dict_set_dynstr (rsp_dict, key, value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save data percent ");
+                                goto end;
+                        }
+                }
+                token = strtok (NULL, ":");
+                if (token != NULL) {
+                        value = gf_strdup (token);
+                        if (!value) {
+                                ret = -1;
+                                goto end;
+                        }
+                        ret = snprintf (key, sizeof (key), "%s.lvsize",
+                                        key_prefix);
+                        if (ret < 0) {
+                                goto end;
+                        }
+
+                        ret = dict_set_dynstr (rsp_dict, key, value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save meta data percent ");
+                                goto end;
+                        }
+                }
+
+        } while (ptr != NULL);
+
+        ret = 0;
+
+end:
+        runner_end (&runner);
+
+out:
+        if (ret && value) {
+                GF_FREE (value);
+        }
+
+        return ret;
+}
+
+int
+glusterd_get_single_brick_status (char **op_errstr, dict_t *rsp_dict,
+                                 char *keyprefix, int index,
+                                 glusterd_volinfo_t *snap_volinfo,
+                                 glusterd_brickinfo_t *brickinfo)
+{
+        int             ret                     = -1;
+        xlator_t        *this                   = NULL;
+        glusterd_conf_t *priv                   = NULL;
+        char            key[PATH_MAX]           = "";
+        char            *device                 = NULL;
+        char            *value                  = NULL;
+        char            brick_path[PATH_MAX]    = "";
+        char            pidfile[PATH_MAX]       = "";
+        pid_t           pid                     = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (keyprefix);
+        GF_ASSERT (snap_volinfo);
+        GF_ASSERT (brickinfo);
+
+        ret = snprintf (key, sizeof (key), "%s.brick%d.path", keyprefix,
+                        index);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = snprintf (brick_path, sizeof (brick_path),
+                        "%s:%s", brickinfo->hostname, brickinfo->path);
+        if (ret < 0) {
+                goto out;
+        }
+
+        value = gf_strdup (brick_path);
+        if (!value) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to store "
+                        "brick_path %s", brickinfo->path);
+                goto out;
+        }
+
+        if (brickinfo->snap_status == -1) {
+                /* Setting vgname as "Pending Snapshot" */
+                value = gf_strdup ("Pending Snapshot");
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "%s.brick%d.vgname",
+                          keyprefix, index);
+                ret = dict_set_dynstr (rsp_dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save vgname ");
+                        goto out;
+                }
+
+                ret = 0;
+                goto out;
+        }
+        value = NULL;
+
+        ret = snprintf (key, sizeof (key), "%s.brick%d.status",
+                        keyprefix, index);
+        if (ret < 0) {
+                goto out;
+        }
+
+        if (brickinfo->status == GF_BRICK_STOPPED) {
+                value = gf_strdup ("No");
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (rsp_dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save brick status");
+                        goto out;
+                }
+                value = NULL;
+        } else {
+                value = gf_strdup ("Yes");
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (rsp_dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save brick status");
+                        goto out;
+                }
+                value = NULL;
+
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_volinfo,
+                                            brickinfo, priv);
+                ret = glusterd_is_service_running (pidfile, &pid);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.pid",
+                                keyprefix, index);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_set_int32 (rsp_dict, key, pid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save pid %d", pid);
+                        goto out;
+                }
+        }
+
+        ret = snprintf (key, sizeof (key), "%s.brick%d",
+                        keyprefix, index);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = glusterd_get_brick_lvm_details (rsp_dict, brickinfo,
+                                              snap_volinfo->volname,
+                                              device, key);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                        "brick LVM details");
+                goto out;
+        }
+out:
+        if (ret && value) {
+                GF_FREE (value);
+        }
+
+        return ret;
+}
+
+int
+glusterd_get_single_snap_status (char **op_errstr, dict_t *rsp_dict,
+                                 char *keyprefix, glusterd_snap_t *snap)
+{
+        int                      ret                 =       -1;
+        xlator_t                *this                =       NULL;
+        char                     key[PATH_MAX]       =       "";
+        char                     brickkey[PATH_MAX]  =       "";
+        glusterd_volinfo_t      *snap_volinfo        =       NULL;
+        glusterd_volinfo_t      *tmp_volinfo         =       NULL;
+        glusterd_brickinfo_t    *brickinfo           =       NULL;
+        int                      volcount            =       0;
+        int                      brickcount          =       0;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (keyprefix);
+        GF_ASSERT (snap);
+
+        list_for_each_entry_safe (snap_volinfo, tmp_volinfo, &snap->volumes,
+                                  vol_list) {
+                ret = snprintf (key, sizeof (key), "%s.vol%d", keyprefix,
+                                volcount);
+                if (ret < 0) {
+                        goto out;
+                }
+                list_for_each_entry (brickinfo, &snap_volinfo->bricks,
+                                     brick_list) {
+                        if (!glusterd_is_local_brick (this, snap_volinfo,
+                            brickinfo)) {
+                                brickcount++;
+                                continue;
+                        }
+
+                        ret = glusterd_get_single_brick_status (op_errstr,
+                                                  rsp_dict, key, brickcount,
+                                                  snap_volinfo, brickinfo);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Getting "
+                                        "single snap status failed");
+                                goto out;
+                        }
+                        brickcount++;
+                }
+                ret = snprintf (brickkey, sizeof (brickkey), "%s.brickcount",
+                                key);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_set_int32 (rsp_dict, brickkey, brickcount);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save brick count");
+                        goto out;
+                }
+                volcount++;
+        }
+
+        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_set_int32 (rsp_dict, key, volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not save volcount");
+                goto out;
+        }
+
+out:
+
+        return ret;
+}
+
+int
+glusterd_get_each_snap_object_status (char **op_errstr, dict_t *rsp_dict,
+                                      glusterd_snap_t *snap, char *keyprefix)
+{
+        int                     ret             =       -1;
+        char                    key[PATH_MAX]   =       "";
+        char                    *temp           =       NULL;
+        xlator_t                *this           =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap);
+        GF_ASSERT (keyprefix);
+
+        /* TODO : Get all the snap volume info present in snap object,
+         * as of now, There will be only one snapvolinfo per snap object
+         */
+        ret = snprintf (key, sizeof (key), "%s.snapname", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        temp = gf_strdup (snap->snapname);
+        if (temp == NULL) {
+                ret = -1;
+                goto out;
+        }
+        ret = dict_set_dynstr (rsp_dict, key, temp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save "
+                        "snap name");
+                goto out;
+        }
+
+        temp = NULL;
+
+        ret = snprintf (key, sizeof (key), "%s.uuid", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        temp = gf_strdup (uuid_utoa (snap->snap_id));
+        if (temp == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, key, temp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save "
+                        "snap UUID");
+                goto out;
+        }
+
+        temp = NULL;
+
+        ret = glusterd_get_single_snap_status (op_errstr, rsp_dict, keyprefix,
+                                               snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not get single snap status");
+                goto out;
+        }
+
+        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_set_int32 (rsp_dict, key, 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save volcount");
+                goto out;
+        }
+out:
+        if (ret && temp)
+                GF_FREE (temp);
+
+        return ret;
+}
+
+int
+glusterd_get_snap_status_of_volume (char **op_errstr, dict_t *rsp_dict,
+                                    char *volname, char *keyprefix) {
+        int                     ret              =       -1;
+        glusterd_volinfo_t      *snap_volinfo    =       NULL;
+        glusterd_volinfo_t      *temp_volinfo    =       NULL;
+        glusterd_volinfo_t      *volinfo         =       NULL;
+        char                    key[PATH_MAX]    =        "";
+        xlator_t                *this            =       NULL;
+        glusterd_conf_t         *priv            =       NULL;
+        int                     i                =        0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (volname);
+        GF_ASSERT (keyprefix);
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volinfo of "
+                        "volume %s", volname);
+                goto out;
+        }
+
+        list_for_each_entry_safe (snap_volinfo, temp_volinfo,
+                             &volinfo->snap_volumes, snapvol_list) {
+                ret = snprintf (key, sizeof (key), "status.snap%d", i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = glusterd_get_each_snap_object_status (op_errstr,
+                                       rsp_dict, snap_volinfo->snapshot, key);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Function : "
+                                "glusterd_get_single_snap_status failed");
+                        goto out;
+                }
+                i++;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "status.snapcount", i);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to save snapcount");
+                ret = -1;
+                goto out;
+        }
+out:
+        return ret;
+}
+
+int
+glusterd_get_all_snapshot_status (dict_t *dict, char **op_errstr,
+                              dict_t *rsp_dict)
+{
+        int32_t                 i               =       0;
+        int                     ret             =       -1;
+        char                    key[PATH_MAX]   =       "";
+        glusterd_conf_t         *priv           =       NULL;
+        glusterd_snap_t         *snap           =       NULL;
+        glusterd_snap_t         *tmp_snap       =       NULL;
+        xlator_t                *this           =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        list_for_each_entry_safe (snap, tmp_snap,
+                                  &priv->snapshots, snap_list) {
+                ret = snprintf (key, sizeof (key), "status.snap%d", i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = glusterd_get_each_snap_object_status (op_errstr,
+                                                         rsp_dict, snap, key);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not get "
+                                "the details of a snap object: %s",
+                                snap->snapname);
+                        goto out;
+                }
+                i++;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "status.snapcount", i);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+glusterd_snapshot_status_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        xlator_t        *this             =       NULL;
+        int             ret               =       -1;
+        glusterd_conf_t *conf             =       NULL;
+        char            *get_buffer       =       NULL;
+        int32_t         cmd               =       -1;
+        char            *snapname         =       NULL;
+        glusterd_snap_t *snap             =       NULL;
+        char            *volname          =       NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get status cmd type");
+                goto out;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "cmd", cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not save status cmd in rsp dictionary");
+                goto out;
+        }
+        switch (cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL:
+                {
+                        ret = glusterd_get_all_snapshot_status (dict, op_errstr,
+                                                           rsp_dict);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get snapshot status");
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_SNAP:
+                {
+
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get snap name");
+                                goto out;
+                        }
+
+                        snap = glusterd_find_snap_by_name (snapname);
+                        if (!snap) {
+                                ret = gf_asprintf (op_errstr, "Snap (%s) "
+                                                  "not found", snapname);
+                                if (ret < 0) {
+                                        goto out;
+                                }
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get snap volinfo");
+                                goto out;
+                        }
+                        ret = glusterd_get_each_snap_object_status (op_errstr,
+                                          rsp_dict, snap, "status.snap0");
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get status of snap %s", get_buffer);
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_VOL:
+                {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to"
+                                        " get volume name");
+                                goto out;
+                        }
+
+                        ret = glusterd_get_snap_status_of_volume (op_errstr,
+                                           rsp_dict, volname, "status.vol0");
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Function :"
+                                        " glusterd_get_snap_status_of_volume "
+                                        "failed");
+                                goto out;
+                        }
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snapshot_create_postvalidate (dict_t *dict, int32_t op_ret,
+                                       char **op_errstr, dict_t *rsp_dict)
+{
+        xlator_t        *this           = NULL;
+        glusterd_conf_t *priv           = NULL;
+        int              ret            = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (op_ret) {
+                ret = glusterd_do_snap_cleanup (dict, op_errstr, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "cleanup operation "
+                                "failed");
+                        goto out;
+                }
+        } else {
+                ret = glusterd_snapshot_update_snaps_post_validate (dict,
+                                                                    op_errstr,
+                                                                    rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "create snapshot");
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snapshot (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+
+        xlator_t        *this           = NULL;
+        glusterd_conf_t *priv           = NULL;
+        int32_t          snap_command   = 0;
+        int              ret            = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case (GF_SNAP_OPTION_TYPE_CREATE):
+                ret = glusterd_snapshot_create_commit (dict, op_errstr,
+                                                       rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "create snapshot");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                ret = glusterd_snapshot_config_commit (dict, op_errstr,
+                                                       rsp_dict);
+                break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_snapshot_remove_commit (dict, op_errstr,
+                                                       rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "delete snapshot");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_snapshot_restore (dict, op_errstr,
+                                                 rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Failed to "
+                                "restore snapshot");
+                        goto out;
+                }
+
+                break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = glusterd_snapshot_status_commit (dict, op_errstr,
+                                                       rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "show snapshot status");
+                        goto out;
+                }
+                break;
+
+
+        default:
+                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
+                goto out;
+                break;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_brickop (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int            ret       = -1;
+        int64_t        vol_count = 0;
+        int64_t        count     = 1;
+        char           key[1024] = {0,};
+        char           *volname  = NULL;
+        int32_t        snap_command = 0;
+        xlator_t       *this     = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = dict_get_int64 (dict, "volcount", &vol_count);
+                if (ret)
+                        goto out;
+                while (count <= vol_count) {
+                        snprintf (key, 1024, "volname%"PRId64, count);
+                        ret = dict_get_str (dict, key, &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to get volname");
+                                goto out;
+                        }
+                        ret = dict_set_str (dict, "volname", volname);
+                        if (ret)
+                                goto out;
+
+                        ret = gd_brick_op_phase (GD_OP_SNAP, NULL, dict,
+                                        op_errstr);
+                        if (ret)
+                                goto out;
+                        volname = NULL;
+                        count++;
+                }
+
+                dict_del (dict, "volname");
+                ret = 0;
+                break;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                break;
+        default:
+                break;
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_prevalidate (dict_t *dict, char **op_errstr,
+                               dict_t *rsp_dict)
+{
+        int                snap_command          = 0;
+        xlator_t           *this                 = NULL;
+        int                ret                   = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case (GF_SNAP_OPTION_TYPE_CREATE):
+                ret = glusterd_snapshot_create_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
+                                "pre-validation failed");
+                        goto out;
+                }
+                break;
+
+        case (GF_SNAP_OPTION_TYPE_CONFIG):
+                ret = glusterd_snapshot_config_prevalidate (dict, op_errstr);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot config "
+                                "pre-validation failed");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_snapshot_restore_prevalidate (dict, op_errstr,
+                                                             rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot restore "
+                                        "validation failed");
+                        goto out;
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_snapshot_remove_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot remove "
+                                "validation failed");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = glusterd_snapshot_status_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot status "
+                                "validation failed");
+                        goto out;
+                }
+                break;
+
+        default:
+                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_postvalidate (dict_t *dict, int32_t op_ret, char **op_errstr,
+                                dict_t *rsp_dict)
+{
+        int                snap_command          = 0;
+        xlator_t           *this                 = NULL;
+        int                ret                   = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = glusterd_snapshot_create_postvalidate (dict, op_ret,
+                                                             op_errstr,
+                                                             rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
+                                "post-validation failed");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_snapshot_update_snaps_post_validate (dict,
+                                                                    op_errstr,
+                                                                    rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "update missed snaps list");
+                        goto out;
+                }
+                break;
+
+        default:
+                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_handle_snapshot_fn (rpcsvc_request_t *req)
+{
+        int32_t               ret            = 0;
+        dict_t               *dict           = NULL;
+        gf_cli_req            cli_req        = {{0},};
+        glusterd_op_t         cli_op         = GD_OP_SNAP;
+        int                   type           = 0;
+        glusterd_conf_t      *conf           = NULL;
+        char                 *host_uuid      = NULL;
+        char                  err_str[2048]  = {0,};
+        xlator_t             *this           = NULL;
+        char                 *volname        = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len > 0) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                }
+
+                dict->extra_stdfree = cli_req.dict.dict_val;
+
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
+                if (host_uuid == NULL) {
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
+                if (ret) {
+                        GF_FREE (host_uuid);
+                        goto out;
+                }
+
+
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "request dict length is %d",
+                        cli_req.dict.dict_len);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Command type not found");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = glusterd_handle_snapshot_create (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
+                                        "failed: %s", err_str);
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_handle_snapshot_restore (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot restore "
+                                        "failed: %s", err_str);
+                }
+
+                break;
+        case GF_SNAP_OPTION_TYPE_INFO:
+                ret = glusterd_handle_snapshot_info (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot info "
+                                        "failed");
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_LIST:
+                ret = glusterd_handle_snapshot_list (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot list "
+                                        "failed");
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                /* TODO : Type of lock to be taken when we are setting
+                 * limits system wide
+                 */
+                ret = dict_get_str (dict, "volname", &volname);
+                if (!volname) {
+                        ret = dict_set_int32 (dict, "hold_vol_locks",
+                                              _gf_false);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Unable to set hold_vol_locks value "
+                                        "as _gf_false");
+                                goto out;
+                        }
+
+                }
+                ret = glusterd_mgmt_v3_initiate_all_phases (req, cli_op, dict);
+                break;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_handle_snapshot_remove (req, cli_op, dict,
+                                                       err_str,
+                                                       sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot delete "
+                                "failed: %s", err_str);
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_START:
+        case GF_SNAP_OPTION_TYPE_STOP:
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = glusterd_handle_snapshot_status (req, cli_op, dict,
+                                                       err_str,
+                                                       sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot status "
+                                "failed: %s", err_str);
+                }
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Unkown snapshot request "
+                        "type (%d)", type);
+                ret = -1; /* Failure */
+        }
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_snapshot (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, glusterd_handle_snapshot_fn);
+}
+
+static inline void
+glusterd_free_snap_op (glusterd_snap_op_t *snap_op)
+{
+        if (snap_op) {
+                if (snap_op->brick_path)
+                        GF_FREE (snap_op->brick_path);
+
+                GF_FREE (snap_op);
+        }
+}
+
+/* Look for duplicates and accordingly update the list */
+int32_t
+glusterd_update_missed_snap_entry (glusterd_missed_snap_info *missed_snapinfo,
+                                   glusterd_snap_op_t *missed_snap_op)
+{
+        int32_t                        ret                         = -1;
+        glusterd_snap_op_t            *snap_opinfo                 = NULL;
+        gf_boolean_t                   match                       = _gf_false;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(missed_snapinfo);
+        GF_ASSERT(missed_snap_op);
+
+        list_for_each_entry (snap_opinfo, &missed_snapinfo->snap_ops,
+                             snap_ops_list) {
+                if ((!strcmp (snap_opinfo->brick_path,
+                              missed_snap_op->brick_path)) &&
+                    (snap_opinfo->op == missed_snap_op->op)) {
+                        /* If two entries have conflicting status
+                         * GD_MISSED_SNAP_DONE takes precedence
+                         */
+                        if ((snap_opinfo->status == GD_MISSED_SNAP_PENDING) &&
+                            (missed_snap_op->status == GD_MISSED_SNAP_DONE)) {
+                                snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "Updating missed snap status "
+                                        "for %s:%d:%s:%d as DONE",
+                                        missed_snapinfo->node_snap_info,
+                                        snap_opinfo->brick_num,
+                                        snap_opinfo->brick_path,
+                                        snap_opinfo->op);
+                                ret = 0;
+                                glusterd_free_snap_op (missed_snap_op);
+                                goto out;
+                        }
+                        match = _gf_true;
+                        break;
+                } else if ((snap_opinfo->brick_num ==
+                            missed_snap_op->brick_num) &&
+                            (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
+                            (missed_snap_op->op ==
+                             GF_SNAP_OPTION_TYPE_DELETE)) {
+                        /* Optimizing create and delete entries for the same
+                         * brick and same node
+                         */
+                        gf_log (this->name, GF_LOG_INFO,
+                                "Updating missed snap status "
+                                "for %s:%d:%s:%d as DONE",
+                                missed_snapinfo->node_snap_info,
+                                snap_opinfo->brick_num,
+                                snap_opinfo->brick_path,
+                                snap_opinfo->op);
+                        snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                        ret = 0;
+                        glusterd_free_snap_op (missed_snap_op);
+                        goto out;
+                }
+        }
+
+        if (match == _gf_true) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Duplicate entry. Not updating");
+                glusterd_free_snap_op (missed_snap_op);
+        } else {
+                list_add_tail (&missed_snap_op->snap_ops_list,
+                               &missed_snapinfo->snap_ops);
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Add new missed snap entry to the missed_snaps list. */
+int32_t
+glusterd_store_missed_snaps_list (char *missed_info, int32_t brick_num,
+                                  char *brick_path, int32_t snap_op,
+                                  int32_t snap_status)
+{
+        int32_t                        ret                         = -1;
+        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
+        glusterd_snap_op_t            *missed_snap_op              = NULL;
+        glusterd_conf_t               *priv                        = NULL;
+        gf_boolean_t                   match                       = _gf_false;
+        gf_boolean_t                   free_missed_snap_info       = _gf_false;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(missed_info);
+        GF_ASSERT(brick_path);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Create the snap_op object consisting of the *
+         * snap id and the op */
+        ret = glusterd_missed_snap_op_new (&missed_snap_op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create new missed snap object.");
+                ret = -1;
+                goto out;
+        }
+
+        missed_snap_op->brick_path = gf_strdup(brick_path);
+        if (!missed_snap_op->brick_path) {
+                ret = -1;
+                goto out;
+        }
+        missed_snap_op->brick_num = brick_num;
+        missed_snap_op->op = snap_op;
+        missed_snap_op->status = snap_status;
+
+        /* Look for other entries for the same node and same snap */
+        list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
+                             missed_snaps) {
+                if (!strcmp (missed_snapinfo->node_snap_info,
+                             missed_info)) {
+                        /* Found missed snapshot info for *
+                         * the same node and same snap */
+                        match = _gf_true;
+                        break;
+                }
+        }
+
+        if (match == _gf_false) {
+                /* First snap op missed for the brick */
+                ret = glusterd_missed_snapinfo_new (&missed_snapinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to create missed snapinfo");
+                        goto out;
+                }
+                free_missed_snap_info = _gf_true;
+                missed_snapinfo->node_snap_info = gf_strdup(missed_info);
+                if (!missed_snapinfo->node_snap_info) {
+                        ret = -1;
+                        goto out;
+                }
+
+                list_add_tail (&missed_snap_op->snap_ops_list,
+                               &missed_snapinfo->snap_ops);
+                list_add_tail (&missed_snapinfo->missed_snaps,
+                               &priv->missed_snaps_list);
+
+                ret = 0;
+                goto out;
+        } else {
+                ret = glusterd_update_missed_snap_entry (missed_snapinfo,
+                                                         missed_snap_op);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to update existing missed snap entry.");
+                        goto out;
+                }
+        }
+
+out:
+        if (ret) {
+                glusterd_free_snap_op (missed_snap_op);
+
+                if (missed_snapinfo &&
+                    (free_missed_snap_info == _gf_true)) {
+                        if (missed_snapinfo->node_snap_info)
+                                GF_FREE (missed_snapinfo->node_snap_info);
+
+                        GF_FREE (missed_snapinfo);
+                }
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Add  missing snap entries to the in-memory conf->missed_snap_list */
+int32_t
+glusterd_add_missed_snaps_to_list (dict_t *dict, int32_t missed_snap_count)
+{
+        char                          *buf                         = NULL;
+        char                          *tmp                         = NULL;
+        char                          *save_ptr                    = NULL;
+        char                          *nodeid                      = NULL;
+        char                          *snap_uuid                   = NULL;
+        char                          *brick_path                  = NULL;
+        char                           missed_info[PATH_MAX]       = "";
+        char                           name_buf[PATH_MAX]          = "";
+        int32_t                        i                           = -1;
+        int32_t                        ret                         = -1;
+        int32_t                        brick_num                   = -1;
+        int32_t                        snap_op                     = -1;
+        int32_t                        snap_status                 = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* We can update the missed_snaps_list without acquiring *
+         * any additional locks as big lock will be held.        */
+        for (i = 0; i < missed_snap_count; i++) {
+                snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                          i);
+                ret = dict_get_str (dict, name_buf, &buf);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch %s", name_buf);
+                        goto out;
+                }
+
+                gf_log (this->name, GF_LOG_DEBUG, "missed_snap_entry = %s",
+                        buf);
+
+                /* Need to make a duplicate string coz the same dictionary *
+                 * is resent to the non-originator nodes */
+                tmp = gf_strdup (buf);
+                if (!tmp) {
+                        ret = -1;
+                        goto out;
+                }
+
+                /* Fetch the node-id, snap-id, brick_num,
+                 * brick_path, snap_op and snap status
+                 */
+                nodeid = strtok_r (tmp, ":", &save_ptr);
+                snap_uuid = strtok_r (NULL, "=", &save_ptr);
+                brick_num = atoi(strtok_r (NULL, ":", &save_ptr));
+                brick_path = strtok_r (NULL, ":", &save_ptr);
+                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
+                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
+
+                if (!nodeid || !snap_uuid || !brick_path ||
+                    brick_num < 1 || snap_op < 1 ||
+                    snap_status < 1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Invalid missed_snap_entry");
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (missed_info, sizeof(missed_info), "%s:%s",
+                          nodeid, snap_uuid);
+
+                ret = glusterd_store_missed_snaps_list (missed_info,
+                                                        brick_num,
+                                                        brick_path,
+                                                        snap_op,
+                                                        snap_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store missed snaps_list");
+                        goto out;
+                }
+
+                GF_FREE (tmp);
+                tmp = NULL;
+        }
+
+out:
+        if (tmp)
+                GF_FREE (tmp);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 4fe8f71cb..1c2ec58e8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -1,21 +1,13 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   Copyright (c) 2007-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
+
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -42,6 +34,8 @@
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-hooks.h"
+#include "store.h"
 #include "glusterd-store.h"
 
 #include "rpc-clnt.h"
@@ -51,102 +45,8 @@
 #include <inttypes.h>
 #include <dirent.h>
 
-static int32_t
-glusterd_store_mkdir (char *path)
-{
-        int32_t     ret = -1;
-
-        ret = mkdir (path, 0777);
-
-        if ((-1 == ret) && (EEXIST != errno)) {
-                gf_log ("", GF_LOG_ERROR, "mkdir() failed on path %s,"
-                        "errno: %s", path, strerror (errno));
-        } else {
-                ret = 0;
-        }
-
-        return ret;
-}
-
-int32_t
-glusterd_store_handle_create_on_absence (glusterd_store_handle_t **shandle,
-                                         char *path)
-{
-        GF_ASSERT (shandle);
-        int32_t     ret = 0;
-
-        if (*shandle == NULL) {
-                ret = glusterd_store_handle_new (path, shandle);
-
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to create store"
-                                " handle for path: %s", path);
-                }
-        }
-        return ret;
-}
-
-int32_t
-glusterd_store_mkstemp (glusterd_store_handle_t *shandle)
-{
-        int     fd = -1;
-        char    tmppath[PATH_MAX] = {0,};
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (shandle->path);
-
-        snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
-        fd = open (tmppath, O_RDWR | O_CREAT | O_TRUNC, 0644);
-        if (fd <= 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to open %s, "
-                        "error: %s", tmppath, strerror (errno));
-        }
-
-        return fd;
-}
-
-int32_t
-glusterd_store_rename_tmppath (glusterd_store_handle_t *shandle)
-{
-        int32_t         ret = -1;
-        char            tmppath[PATH_MAX] = {0,};
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (shandle->path);
-
-        snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
-        ret = rename (tmppath, shandle->path);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to mv %s to %s, "
-                        "error: %s", tmppath, shandle->path, strerror (errno));
-        }
-
-        return ret;
-}
-
-int32_t
-glusterd_store_unlink_tmppath (glusterd_store_handle_t *shandle)
-{
-        int32_t         ret = -1;
-        char            tmppath[PATH_MAX] = {0,};
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (shandle->path);
-
-        snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
-        ret = unlink (tmppath);
-        if (ret && (errno != ENOENT)) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to mv %s to %s, "
-                        "error: %s", tmppath, shandle->path, strerror (errno));
-        } else {
-                ret = 0;
-        }
-
-        return ret;
-}
-
-static void
-glusterd_replace_slash_with_hipen (char *str)
+void
+glusterd_replace_slash_with_hyphen (char *str)
 {
         char                    *ptr = NULL;
 
@@ -171,7 +71,7 @@ glusterd_store_create_brick_dir (glusterd_volinfo_t *volinfo)
         GF_ASSERT (priv);
 
         GLUSTERD_GET_BRICK_DIR (brickdirpath, volinfo, priv);
-        ret = glusterd_store_mkdir (brickdirpath);
+        ret = gf_store_mkdir (brickdirpath);
 
         return ret;
 }
@@ -185,7 +85,7 @@ glusterd_store_key_vol_brick_set (glusterd_brickinfo_t *brickinfo,
         GF_ASSERT (len >= PATH_MAX);
 
         snprintf (key_vol_brick, len, "%s", brickinfo->path);
-        glusterd_replace_slash_with_hipen (key_vol_brick);
+        glusterd_replace_slash_with_hyphen (key_vol_brick);
 }
 
 static void
@@ -232,20 +132,31 @@ glusterd_store_is_valid_brickpath (char *volname, char *brick)
         glusterd_brickinfo_t    *brickinfo = NULL;
         glusterd_volinfo_t      *volinfo = NULL;
         int32_t                 ret = 0;
+        size_t                  volname_len = strlen (volname);
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
-        ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
+        ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "brick path validation failed");
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create brick "
+                        "info for brick %s", brick);
                 ret = 0;
                 goto out;
         }
         ret = glusterd_volinfo_new (&volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "brick path validation failed");
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create volinfo");
+                ret = 0;
+                goto out;
+        }
+        if (volname_len >= sizeof (volinfo->volname)) {
+                gf_log (this->name, GF_LOG_WARNING, "volume name too long");
                 ret = 0;
                 goto out;
         }
-        strncpy (volinfo->volname, volname, sizeof (volinfo->volname));
+        memcpy (volinfo->volname, volname, volname_len+1);
         glusterd_store_brickinfopath_set (volinfo, brickinfo, brickpath,
                                                 sizeof (brickpath));
 
@@ -273,7 +184,11 @@ glusterd_store_volinfo_brick_fname_write (int vol_fd,
                   brick_count);
         glusterd_store_brickinfofname_set (brickinfo, brickfname,
                                         sizeof (brickfname));
-        ret = glusterd_store_save_value (vol_fd, key, brickfname);
+        ret = gf_store_save_value (vol_fd, key, brickfname);
+        if (ret)
+                goto out;
+
+out:
         return ret;
 }
 
@@ -288,9 +203,9 @@ glusterd_store_create_brick_shandle_on_absence (glusterd_volinfo_t *volinfo,
         GF_ASSERT (brickinfo);
 
         glusterd_store_brickinfopath_set (volinfo, brickinfo, brickpath,
-                                      sizeof (brickpath));
-        ret = glusterd_store_handle_create_on_absence (&brickinfo->shandle,
-                                                       brickpath);
+                                          sizeof (brickpath));
+        ret = gf_store_handle_create_on_absence (&brickinfo->shandle,
+                                                 brickpath);
         return ret;
 }
 
@@ -303,30 +218,50 @@ glusterd_store_brickinfo_write (int fd, glusterd_brickinfo_t *brickinfo)
         GF_ASSERT (brickinfo);
         GF_ASSERT (fd > 0);
 
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
-                                         brickinfo->hostname);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
+                                   brickinfo->hostname);
         if (ret)
                 goto out;
 
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PATH,
-                                         brickinfo->path);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PATH,
+                                   brickinfo->path);
         if (ret)
                 goto out;
 
         snprintf (value, sizeof(value), "%d", brickinfo->port);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PORT,
-                                         value);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PORT, value);
 
         snprintf (value, sizeof(value), "%d", brickinfo->rdma_port);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
-                                         value);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                   value);
 
         snprintf (value, sizeof(value), "%d", brickinfo->decommissioned);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
-                                         value);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
+                                   value);
+        if (ret)
+                goto out;
+
+        if (strlen(brickinfo->device_path) > 0) {
+                snprintf (value, sizeof(value), "%s", brickinfo->device_path);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
+                                           value);
+                if (ret)
+                        goto out;
+        }
 
+        snprintf (value, sizeof(value), "%d", brickinfo->snap_status);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                                   value);
+        if (ret)
+                goto out;
+
+        if (!brickinfo->vg[0])
+                goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                                         brickinfo->vg);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -337,7 +272,7 @@ glusterd_store_perform_brick_store (glusterd_brickinfo_t *brickinfo)
         int32_t                     ret = -1;
         GF_ASSERT (brickinfo);
 
-        fd = glusterd_store_mkstemp (brickinfo->shandle);
+        fd = gf_store_mkstemp (brickinfo->shandle);
         if (fd <= 0) {
                 ret = -1;
                 goto out;
@@ -347,13 +282,12 @@ glusterd_store_perform_brick_store (glusterd_brickinfo_t *brickinfo)
         if (ret)
                 goto out;
 
-        ret = glusterd_store_rename_tmppath (brickinfo->shandle);
 out:
         if (ret && (fd > 0))
-                glusterd_store_unlink_tmppath (brickinfo->shandle);
+                gf_store_unlink_tmppath (brickinfo->shandle);
         if (fd > 0)
                 close (fd);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -382,31 +316,29 @@ glusterd_store_brickinfo (glusterd_volinfo_t *volinfo,
                 goto out;
 
         ret = glusterd_store_perform_brick_store (brickinfo);
+
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
 int32_t
-glusterd_store_delete_brick (glusterd_volinfo_t *volinfo,
-                             glusterd_brickinfo_t *brickinfo)
+glusterd_store_delete_brick (glusterd_brickinfo_t *brickinfo, char *delete_path)
 {
         int32_t                 ret = -1;
         glusterd_conf_t         *priv = NULL;
-        char                    path[PATH_MAX] = {0,};
         char                    brickpath[PATH_MAX] = {0,};
         char                    *ptr = NULL;
         char                    *tmppath = NULL;
+        xlator_t                *this = NULL;
 
-        GF_ASSERT (volinfo);
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (brickinfo);
 
-        priv = THIS->private;
-
+        priv = this->private;
         GF_ASSERT (priv);
 
-        GLUSTERD_GET_BRICK_DIR (path, volinfo, priv);
-
         tmppath = gf_strdup (brickinfo->path);
 
         ptr = strchr (tmppath, '/');
@@ -416,16 +348,17 @@ glusterd_store_delete_brick (glusterd_volinfo_t *volinfo,
                 ptr = strchr (tmppath, '/');
         }
 
-        snprintf (brickpath, sizeof (brickpath), "%s/%s:%s",
-                  path, brickinfo->hostname, tmppath);
+        snprintf (brickpath, sizeof (brickpath),
+                  "%s/"GLUSTERD_BRICK_INFO_DIR"/%s:%s", delete_path,
+                  brickinfo->hostname, tmppath);
 
         GF_FREE (tmppath);
 
         ret = unlink (brickpath);
 
         if ((ret < 0) && (errno != ENOENT)) {
-                gf_log ("", GF_LOG_ERROR, "Unlink failed on %s, reason: %s",
-                        brickpath, strerror(errno));
+                gf_log (this->name, GF_LOG_DEBUG, "Unlink failed on %s, "
+                        "reason: %s", brickpath, strerror(errno));
                 ret = -1;
                 goto out;
         } else {
@@ -434,15 +367,15 @@ glusterd_store_delete_brick (glusterd_volinfo_t *volinfo,
 
 out:
         if (brickinfo->shandle) {
-                glusterd_store_handle_destroy (brickinfo->shandle);
+                gf_store_handle_destroy (brickinfo->shandle);
                 brickinfo->shandle = NULL;
         }
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
 int32_t
-glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo)
+glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo, char *delete_path)
 {
         int32_t                 ret = 0;
         glusterd_brickinfo_t    *tmp = NULL;
@@ -451,19 +384,24 @@ glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo)
         DIR                     *dir = NULL;
         struct dirent           *entry = NULL;
         char                    path[PATH_MAX] = {0,};
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (volinfo);
 
         list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
-                ret = glusterd_store_delete_brick (volinfo, tmp);
+                ret = glusterd_store_delete_brick (tmp, delete_path);
                 if (ret)
                         goto out;
         }
 
-        priv = THIS->private;
+        priv = this->private;
         GF_ASSERT (priv);
 
-        GLUSTERD_GET_BRICK_DIR (brickdir, volinfo, priv);
+        snprintf (brickdir, sizeof (brickdir), "%s/%s", delete_path,
+                  GLUSTERD_BRICK_INFO_DIR);
 
         dir = opendir (brickdir);
 
@@ -474,7 +412,7 @@ glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo)
                           brickdir, entry->d_name);
                 ret = unlink (path);
                 if (ret && errno != ENOENT) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to unlink %s, "
+                        gf_log (this->name, GF_LOG_DEBUG, "Unable to unlink %s, "
                                 "reason: %s", path, strerror(errno));
                 }
                 glusterd_for_each_entry (entry, dir);
@@ -485,17 +423,21 @@ glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo)
         ret = rmdir (brickdir);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
-static void
+static int
 _storeslaves (dict_t *this, char *key, data_t *value, void *data)
 {
         int32_t                      ret = 0;
-        glusterd_store_handle_t  *shandle = NULL;
+        gf_store_handle_t           *shandle = NULL;
+        xlator_t                    *xl = NULL;
+
+        xl = THIS;
+        GF_ASSERT (xl);
 
-        shandle = (glusterd_store_handle_t*)data;
+        shandle = (gf_store_handle_t*)data;
 
         GF_ASSERT (shandle);
         GF_ASSERT (shandle->fd > 0);
@@ -504,32 +446,37 @@ _storeslaves (dict_t *this, char *key, data_t *value, void *data)
         GF_ASSERT (value && value->data);
 
         if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
-                return;
+                return -1;
 
         if (!key)
-                return;
+                return -1;
         if (!value || !value->data)
-                return;
+                return -1;
 
-        gf_log ("", GF_LOG_DEBUG, "Storing in volinfo:key= %s, val=%s",
+        gf_log (xl->name, GF_LOG_DEBUG, "Storing in volinfo:key= %s, val=%s",
                 key, value->data);
 
-        ret = glusterd_store_save_value (shandle->fd, key, (char*)value->data);
+        ret = gf_store_save_value (shandle->fd, key, (char*)value->data);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to write into store"
+                gf_log (xl->name, GF_LOG_ERROR, "Unable to write into store"
                                 " handle for path: %s", shandle->path);
-                return;
+                return -1;
         }
+        return 0;
 }
 
 
-void _storeopts (dict_t *this, char *key, data_t *value, void *data)
+int _storeopts (dict_t *this, char *key, data_t *value, void *data)
 {
         int32_t                      ret = 0;
         int32_t                      exists = 0;
-        glusterd_store_handle_t  *shandle = NULL;
+        gf_store_handle_t           *shandle = NULL;
+        xlator_t                    *xl = NULL;
 
-        shandle = (glusterd_store_handle_t*)data;
+        xl = THIS;
+        GF_ASSERT (xl);
+
+        shandle = (gf_store_handle_t*)data;
 
         GF_ASSERT (shandle);
         GF_ASSERT (shandle->fd > 0);
@@ -538,94 +485,158 @@ void _storeopts (dict_t *this, char *key, data_t *value, void *data)
         GF_ASSERT (value && value->data);
 
         if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
-                return;
+                return -1;
 
         if (!key)
-                return;
+                return -1;
         if (!value || !value->data)
-                return;
+                return -1;
+
+        if (is_key_glusterd_hooks_friendly (key)) {
+                exists = 1;
+
+        } else {
+                exists = glusterd_check_option_exists (key, NULL);
+        }
 
-        exists = glusterd_check_option_exists (key, NULL);
         if (1 == exists) {
-                gf_log ("", GF_LOG_DEBUG, "Storing in volinfo:key= %s, val=%s",
-                        key, value->data);
+                gf_log (xl->name, GF_LOG_DEBUG, "Storing in volinfo:key= %s, "
+                        "val=%s", key, value->data);
+
         } else {
-                gf_log ("", GF_LOG_DEBUG, "Discarding:key= %s, val=%s",
+                gf_log (xl->name, GF_LOG_DEBUG, "Discarding:key= %s, val=%s",
                         key, value->data);
-                return;
+                return 0;
         }
 
-        ret = glusterd_store_save_value (shandle->fd, key, (char*)value->data);
+        ret = gf_store_save_value (shandle->fd, key, (char*)value->data);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to write into store"
-                                " handle for path: %s", shandle->path);
-                return;
+                gf_log (xl->name, GF_LOG_ERROR, "Unable to write into store"
+                        " handle for path: %s", shandle->path);
+                return -1;
         }
+        return 0;
 }
 
 int32_t
 glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
 {
+        char    *str            = NULL;
+        char    buf[PATH_MAX]   = {0,};
+        int32_t ret             = -1;
+
         GF_ASSERT (fd > 0);
         GF_ASSERT (volinfo);
 
-        char                    buf[PATH_MAX] = {0,};
-        int32_t                 ret           = -1;
-
         snprintf (buf, sizeof (buf), "%d", volinfo->type);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TYPE, buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TYPE, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->brick_count);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_COUNT, buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_COUNT, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->status);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STATUS, buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STATUS, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->sub_count);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_SUB_COUNT,
-                                         buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_SUB_COUNT, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->stripe_count);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT,
-                                         buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->replica_count);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
-                                         buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
+                                   buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->version);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_VERSION,
-                                         buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_VERSION, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->transport_type);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
-                                         buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TRANSPORT, buf);
         if (ret)
                 goto out;
 
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_ID,
-                                         uuid_utoa (volinfo->volume_id));
+        snprintf (buf, sizeof (buf), "%s", volinfo->parent_volname);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PARENT_VOLNAME, buf);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to store "
+                        GLUSTERD_STORE_KEY_PARENT_VOLNAME);
+                goto out;
+        }
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_ID,
+                                   uuid_utoa (volinfo->volume_id));
         if (ret)
                 goto out;
 
+        str = glusterd_auth_get_username (volinfo);
+        if (str) {
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_USERNAME,
+                                           str);
+                if (ret)
+                goto out;
+        }
+
+        str = glusterd_auth_get_password (volinfo);
+        if (str) {
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PASSWORD,
+                                           str);
+                if (ret)
+                        goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->op_version);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_OP_VERSION, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->client_op_version);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                                   buf);
+        if (ret)
+                goto out;
+        if (volinfo->caps) {
+                snprintf (buf, sizeof (buf), "%d", volinfo->caps);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CAPS,
+                                           buf);
+                if (ret)
+                        goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->is_volume_restored);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_IS_RESTORED, buf);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to write is_volume_restored");
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%"PRIu64, volinfo->snap_max_hard_limit);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                                   buf);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to write snap-max-hard-limit");
+                goto out;
+        }
+
 out:
         if (ret)
-                gf_log ("", GF_LOG_ERROR, "Unable to write volume values"
-                        " for %s", volinfo->volname);
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to write volume "
+                        "values for %s", volinfo->volname);
         return ret;
 }
 
@@ -639,8 +650,7 @@ glusterd_store_voldirpath_set (glusterd_volinfo_t *volinfo, char *voldirpath,
         priv = THIS->private;
         GF_ASSERT (priv);
 
-        snprintf (voldirpath, len, "%s/%s/%s", priv->workdir,
-                  GLUSTERD_VOLUME_DIR_PREFIX, volinfo->volname);
+        GLUSTERD_GET_VOLUME_DIR (voldirpath, volinfo, priv);
 }
 
 static int32_t
@@ -653,8 +663,30 @@ glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo)
 
         glusterd_store_voldirpath_set (volinfo, voldirpath,
                                        sizeof (voldirpath));
-        ret = glusterd_store_mkdir (voldirpath);
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        ret = gf_store_mkdir (voldirpath);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+static int32_t
+glusterd_store_create_snap_dir (glusterd_snap_t *snap)
+{
+        int32_t            ret                   = -1;
+        char               snapdirpath[PATH_MAX] = {0,};
+        glusterd_conf_t   *priv                  = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+
+        GLUSTERD_GET_SNAP_DIR (snapdirpath, snap, priv);
+
+        ret = mkdir_p (snapdirpath, 0755, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snaps dir "
+                        "%s", snapdirpath);
+        }
         return ret;
 }
 
@@ -662,7 +694,7 @@ int32_t
 glusterd_store_volinfo_write (int fd, glusterd_volinfo_t *volinfo)
 {
         int32_t                         ret = -1;
-        glusterd_store_handle_t         *shandle = NULL;
+        gf_store_handle_t              *shandle = NULL;
         GF_ASSERT (fd > 0);
         GF_ASSERT (volinfo);
         GF_ASSERT (volinfo->shandle);
@@ -678,7 +710,50 @@ glusterd_store_volinfo_write (int fd, glusterd_volinfo_t *volinfo)
         dict_foreach (volinfo->gsync_slaves, _storeslaves, shandle);
         shandle->fd = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_snapinfo_write (glusterd_snap_t *snap)
+{
+        int32_t  ret            = -1;
+        int      fd             = 0;
+        char     buf[PATH_MAX]  = "";
+
+        GF_ASSERT (snap);
+
+        fd = gf_store_mkstemp (snap->shandle);
+        if (fd <= 0)
+               goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_ID,
+                                   uuid_utoa (snap->snap_id));
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", snap->snap_status);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_STATUS, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", snap->snap_restored);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_RESTORED, buf);
+        if (ret)
+                goto out;
+
+        if (snap->description) {
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_DESC,
+                                           snap->description);
+                if (ret)
+                        goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%ld", snap->time_stamp);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP, buf);
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -689,7 +764,7 @@ glusterd_store_rbstatepath_set (glusterd_volinfo_t *volinfo, char *rbstatepath,
         char    voldirpath[PATH_MAX] = {0,};
         GF_ASSERT (volinfo);
         GF_ASSERT (rbstatepath);
-        GF_ASSERT (len >= PATH_MAX);
+        GF_ASSERT (len <= PATH_MAX);
 
         glusterd_store_voldirpath_set (volinfo, voldirpath,
                                        sizeof (voldirpath));
@@ -704,13 +779,59 @@ glusterd_store_volfpath_set (glusterd_volinfo_t *volinfo, char *volfpath,
         char    voldirpath[PATH_MAX] = {0,};
         GF_ASSERT (volinfo);
         GF_ASSERT (volfpath);
-        GF_ASSERT (len >= PATH_MAX);
+        GF_ASSERT (len <= PATH_MAX);
 
         glusterd_store_voldirpath_set (volinfo, voldirpath,
                                        sizeof (voldirpath));
         snprintf (volfpath, len, "%s/%s", voldirpath, GLUSTERD_VOLUME_INFO_FILE);
 }
 
+static void
+glusterd_store_node_state_path_set (glusterd_volinfo_t *volinfo,
+                                    char *node_statepath, size_t len)
+{
+        char    voldirpath[PATH_MAX] = {0,};
+        GF_ASSERT (volinfo);
+        GF_ASSERT (node_statepath);
+        GF_ASSERT (len <= PATH_MAX);
+
+        glusterd_store_voldirpath_set (volinfo, voldirpath,
+                                       sizeof (voldirpath));
+        snprintf (node_statepath, len, "%s/%s", voldirpath,
+                  GLUSTERD_NODE_STATE_FILE);
+}
+
+static void
+glusterd_store_missed_snaps_list_path_set (char *missed_snaps_list,
+                                           size_t len)
+{
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (missed_snaps_list);
+        GF_ASSERT (len <= PATH_MAX);
+
+        snprintf (missed_snaps_list, len, "%s/snaps/"
+                  GLUSTERD_MISSED_SNAPS_LIST_FILE, priv->workdir);
+}
+
+static void
+glusterd_store_snapfpath_set (glusterd_snap_t *snap, char *snap_fpath,
+                              size_t len)
+{
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+        GF_ASSERT (snap_fpath);
+        GF_ASSERT (len <= PATH_MAX);
+
+        snprintf (snap_fpath, len, "%s/snaps/%s/%s", priv->workdir,
+                  snap->snapname, GLUSTERD_SNAP_INFO_FILE);
+}
+
 int32_t
 glusterd_store_create_rbstate_shandle_on_absence (glusterd_volinfo_t *volinfo)
 {
@@ -720,8 +841,8 @@ glusterd_store_create_rbstate_shandle_on_absence (glusterd_volinfo_t *volinfo)
         GF_ASSERT (volinfo);
 
         glusterd_store_rbstatepath_set (volinfo, rbstatepath, sizeof (rbstatepath));
-        ret = glusterd_store_handle_create_on_absence (&volinfo->rb_shandle,
-                                                       rbstatepath);
+        ret = gf_store_handle_create_on_absence (&volinfo->rb_shandle,
+                                                 rbstatepath);
         return ret;
 }
 
@@ -734,8 +855,60 @@ glusterd_store_create_vol_shandle_on_absence (glusterd_volinfo_t *volinfo)
         GF_ASSERT (volinfo);
 
         glusterd_store_volfpath_set (volinfo, volfpath, sizeof (volfpath));
-        ret = glusterd_store_handle_create_on_absence (&volinfo->shandle,
-                                                       volfpath);
+        ret = gf_store_handle_create_on_absence (&volinfo->shandle, volfpath);
+        return ret;
+}
+
+int32_t
+glusterd_store_create_nodestate_sh_on_absence (glusterd_volinfo_t *volinfo)
+{
+        char            node_state_path[PATH_MAX] = {0};
+        int32_t         ret                   = 0;
+
+        GF_ASSERT (volinfo);
+
+        glusterd_store_node_state_path_set (volinfo, node_state_path,
+                                            sizeof (node_state_path));
+        ret =
+          gf_store_handle_create_on_absence (&volinfo->node_state_shandle,
+                                             node_state_path);
+
+        return ret;
+}
+
+static int32_t
+glusterd_store_create_missed_snaps_list_shandle_on_absence ()
+{
+        char               missed_snaps_list[PATH_MAX] = "";
+        int32_t            ret                         = -1;
+        glusterd_conf_t   *priv                        = NULL;
+        xlator_t          *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        glusterd_store_missed_snaps_list_path_set (missed_snaps_list,
+                                                   sizeof(missed_snaps_list));
+
+        ret = gf_store_handle_create_on_absence
+                                            (&priv->missed_snaps_list_shandle,
+                                             missed_snaps_list);
+        return ret;
+}
+
+int32_t
+glusterd_store_create_snap_shandle_on_absence (glusterd_snap_t *snap)
+{
+        char            snapfpath[PATH_MAX] = {0};
+        int32_t         ret = 0;
+
+        GF_ASSERT (snap);
+
+        glusterd_store_snapfpath_set (snap, snapfpath, sizeof (snapfpath));
+        ret = gf_store_handle_create_on_absence (&snap->shandle, snapfpath);
         return ret;
 }
 
@@ -756,7 +929,7 @@ glusterd_store_brickinfos (glusterd_volinfo_t *volinfo, int vol_fd)
                 brick_count++;
         }
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -764,36 +937,56 @@ int32_t
 glusterd_store_rbstate_write (int fd, glusterd_volinfo_t *volinfo)
 {
         int     ret             = -1;
+        int     port            = 0;
         char    buf[PATH_MAX]   = {0, };
 
         GF_ASSERT (fd > 0);
         GF_ASSERT (volinfo);
 
-        snprintf (buf, sizeof (buf), "%d", volinfo->rb_status);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_STATUS,
-                                         buf);
+        snprintf (buf, sizeof (buf), "%d", volinfo->rep_brick.rb_status);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_STATUS, buf);
         if (ret)
                 goto out;
 
-        if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+        if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
 
                 snprintf (buf, sizeof (buf), "%s:%s",
-                          volinfo->src_brick->hostname,
-                          volinfo->src_brick->path);
-                ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
-                                                 buf);
+                          volinfo->rep_brick.src_brick->hostname,
+                          volinfo->rep_brick.src_brick->path);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                                           buf);
                 if (ret)
                         goto out;
 
                 snprintf (buf, sizeof (buf), "%s:%s",
-                          volinfo->dst_brick->hostname,
-                          volinfo->dst_brick->path);
-                ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_BRICK,
-                                                 buf);
+                          volinfo->rep_brick.dst_brick->hostname,
+                          volinfo->rep_brick.dst_brick->path);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                                           buf);
                 if (ret)
                         goto out;
+
+                switch (volinfo->transport_type) {
+                        case GF_TRANSPORT_RDMA:
+                                port = volinfo->rep_brick.dst_brick->rdma_port;
+                                break;
+
+                        case GF_TRANSPORT_TCP:
+                        case GF_TRANSPORT_BOTH_TCP_RDMA:
+                                port = volinfo->rep_brick.dst_brick->port;
+                                break;
+                }
+
+                snprintf (buf, sizeof (buf), "%d", port);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_PORT,
+                                           buf);
+                if (ret)
+                        goto out;
+                uuid_unparse (volinfo->rep_brick.rb_id, buf);
+                ret = gf_store_save_value (fd, GF_REPLACE_BRICK_TID_KEY, buf);
         }
 
+        ret = 0;
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -806,7 +999,7 @@ glusterd_store_perform_rbstate_store (glusterd_volinfo_t *volinfo)
         int32_t                     ret = -1;
         GF_ASSERT (volinfo);
 
-        fd = glusterd_store_mkstemp (volinfo->rb_shandle);
+        fd = gf_store_mkstemp (volinfo->rb_shandle);
         if (fd <= 0) {
                 ret = -1;
                 goto out;
@@ -816,14 +1009,79 @@ glusterd_store_perform_rbstate_store (glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
-        ret = glusterd_store_rename_tmppath (volinfo->rb_shandle);
+        ret = gf_store_rename_tmppath (volinfo->rb_shandle);
+        if (ret)
+                goto out;
 
 out:
         if (ret && (fd > 0))
-                glusterd_store_unlink_tmppath (volinfo->rb_shandle);
+                gf_store_unlink_tmppath (volinfo->rb_shandle);
         if (fd > 0)
                 close (fd);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo)
+{
+        int     ret             = -1;
+        char    buf[PATH_MAX]   = {0, };
+
+        GF_ASSERT (fd > 0);
+        GF_ASSERT (volinfo);
+
+        if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_STATUS) {
+                ret = 0;
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.defrag_cmd);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_DEFRAG, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.op);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_DEFRAG_OP, buf);
+        if (ret)
+                goto out;
+
+        if (volinfo->rebal.defrag_cmd) {
+                uuid_unparse (volinfo->rebal.rebalance_id, buf);
+                ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf);
+        }
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo)
+{
+        int                         fd = -1;
+        int32_t                     ret = -1;
+        GF_ASSERT (volinfo);
+
+        fd = gf_store_mkstemp (volinfo->node_state_shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_node_state_write (fd, volinfo);
+        if (ret)
+                goto out;
+
+        ret = gf_store_rename_tmppath (volinfo->node_state_shandle);
+        if (ret)
+                goto out;
+
+out:
+        if (ret && (fd > 0))
+                gf_store_unlink_tmppath (volinfo->node_state_shandle);
+        if (fd > 0)
+                close (fd);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -834,7 +1092,7 @@ glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo)
         int32_t                     ret = -1;
         GF_ASSERT (volinfo);
 
-        fd = glusterd_store_mkstemp (volinfo->shandle);
+        fd = gf_store_mkstemp (volinfo->shandle);
         if (fd <= 0) {
                 ret = -1;
                 goto out;
@@ -848,13 +1106,12 @@ glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
-        ret = glusterd_store_rename_tmppath (volinfo->shandle);
 out:
         if (ret && (fd > 0))
-                glusterd_store_unlink_tmppath (volinfo->shandle);
+                gf_store_unlink_tmppath (volinfo->shandle);
         if (fd > 0)
                 close (fd);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -870,7 +1127,148 @@ glusterd_perform_volinfo_version_action (glusterd_volinfo_t *volinfo,
         case GLUSTERD_VOLINFO_VER_AC_INCREMENT:
                 volinfo->version++;
         break;
+        case GLUSTERD_VOLINFO_VER_AC_DECREMENT:
+                volinfo->version--;
+        break;
+        }
+}
+
+void
+glusterd_store_bricks_cleanup_tmp (glusterd_volinfo_t *volinfo)
+{
+        glusterd_brickinfo_t    *brickinfo           = NULL;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                gf_store_unlink_tmppath (brickinfo->shandle);
+        }
+}
+
+void
+glusterd_store_volume_cleanup_tmp (glusterd_volinfo_t *volinfo)
+{
+        GF_ASSERT (volinfo);
+
+        glusterd_store_bricks_cleanup_tmp (volinfo);
+
+        gf_store_unlink_tmppath (volinfo->shandle);
+
+        gf_store_unlink_tmppath (volinfo->rb_shandle);
+
+        gf_store_unlink_tmppath (volinfo->node_state_shandle);
+}
+
+void
+glusterd_store_snap_cleanup_tmp (glusterd_snap_t *snap)
+{
+        GF_ASSERT (snap);
+
+        gf_store_unlink_tmppath (snap->shandle);
+}
+
+int32_t
+glusterd_store_brickinfos_atomic_update (glusterd_volinfo_t *volinfo)
+{
+        int                      ret            = -1;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = gf_store_rename_tmppath (brickinfo->shandle);
+                if (ret)
+                        goto out;
+        }
+out:
+        return ret;
+}
+
+int32_t
+glusterd_store_volinfo_atomic_update (glusterd_volinfo_t *volinfo)
+{
+        int ret = -1;
+        GF_ASSERT (volinfo);
+
+        ret = gf_store_rename_tmppath (volinfo->shandle);
+        if (ret)
+                goto out;
+
+out:
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't rename "
+                        "temporary file(s): Reason %s", strerror (errno));
+        return ret;
+}
+
+int32_t
+glusterd_store_volume_atomic_update (glusterd_volinfo_t *volinfo)
+{
+        int ret = -1;
+        GF_ASSERT (volinfo);
+
+        ret = glusterd_store_brickinfos_atomic_update (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_volinfo_atomic_update (volinfo);
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_store_snap_atomic_update (glusterd_snap_t *snap)
+{
+        int ret = -1;
+        GF_ASSERT (snap);
+
+        ret = gf_store_rename_tmppath (snap->shandle);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't rename "
+                        "temporary file(s): Reason %s", strerror (errno));
+
+        return ret;
+}
+
+int32_t
+glusterd_store_snap (glusterd_snap_t *snap)
+{
+        int32_t                 ret = -1;
+
+        GF_ASSERT (snap);
+
+        ret = glusterd_store_create_snap_dir (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snap dir");
+                goto out;
+        }
+
+        ret = glusterd_store_create_snap_shandle_on_absence (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snap info "
+                        "file");
+                goto out;
+        }
+
+        ret = glusterd_store_snapinfo_write (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to write snap info");
+                goto out;
         }
+
+        ret = glusterd_store_snap_atomic_update (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,"Failed to do automic update");
+                goto out;
+        }
+
+out:
+        if (ret)
+                glusterd_store_snap_cleanup_tmp (snap);
+
+        gf_log (THIS->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
 }
 
 int32_t
@@ -893,61 +1291,113 @@ glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t a
         if (ret)
                 goto out;
 
+        ret = glusterd_store_create_nodestate_sh_on_absence (volinfo);
+        if (ret)
+                goto out;
+
         ret = glusterd_store_perform_volume_store (volinfo);
         if (ret)
                 goto out;
 
+        ret = glusterd_store_volume_atomic_update (volinfo);
+        if (ret) {
+                glusterd_perform_volinfo_version_action (volinfo,
+                                                         GLUSTERD_VOLINFO_VER_AC_DECREMENT);
+                goto out;
+        }
+
         ret = glusterd_store_perform_rbstate_store (volinfo);
         if (ret)
                 goto out;
 
+        ret = glusterd_store_perform_node_state_store (volinfo);
+        if (ret)
+                goto out;
+
         //checksum should be computed at the end
         ret = glusterd_volume_compute_cksum (volinfo);
         if (ret)
                 goto out;
+
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (ret)
+                glusterd_store_volume_cleanup_tmp (volinfo);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
 
-
 int32_t
 glusterd_store_delete_volume (glusterd_volinfo_t *volinfo)
 {
-        char    pathname[PATH_MAX] = {0,};
-        int32_t ret = 0;
-        glusterd_conf_t *priv = NULL;
-        DIR     *dir = NULL;
-        struct dirent *entry = NULL;
-        char path[PATH_MAX] = {0,};
-        struct stat     st = {0, };
+        char             pathname[PATH_MAX]    = {0,};
+        int32_t          ret                   = 0;
+        glusterd_conf_t *priv                  = NULL;
+        DIR             *dir                   = NULL;
+        struct dirent   *entry                 = NULL;
+        char             path[PATH_MAX]        = {0,};
+        char             delete_path[PATH_MAX] = {0,};
+        char             trashdir[PATH_MAX]    = {0,};
+        struct stat      st                    = {0, };
+        xlator_t        *this                  = NULL;
+        gf_boolean_t     rename_fail           = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (volinfo);
-        priv = THIS->private;
+        priv = this->private;
 
         GF_ASSERT (priv);
-        snprintf (pathname, sizeof (pathname), "%s/vols/%s", priv->workdir,
-                  volinfo->volname);
 
-        dir = opendir (pathname);
-        if (!dir)
+        GLUSTERD_GET_VOLUME_DIR (pathname, volinfo, priv);
+
+        snprintf (delete_path, sizeof (delete_path),
+        "%s/"GLUSTERD_TRASH"/%s.deleted", priv->workdir,
+        uuid_utoa (volinfo->volume_id));
+
+        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
+                  priv->workdir);
+
+        ret = mkdir (trashdir, 0777);
+        if (ret && errno != EEXIST) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create trash "
+                        "directory, reason : %s", strerror (errno));
+                ret = -1;
                 goto out;
-        ret = glusterd_store_remove_bricks (volinfo);
+        }
 
+        ret = rename (pathname, delete_path);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Remove bricks failed for %s",
+                gf_log (this->name, GF_LOG_ERROR, "Failed to rename volume "
+                        "directory for volume %s", volinfo->volname);
+                rename_fail = _gf_true;
+                goto out;
+        }
+
+        dir = opendir (delete_path);
+        if (!dir) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to open directory %s."
+                        " Reason : %s", delete_path, strerror (errno));
+                ret = 0;
+                goto out;
+        }
+        ret = glusterd_store_remove_bricks (volinfo, delete_path);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Remove bricks failed for %s",
                         volinfo->volname);
         }
 
         glusterd_for_each_entry (entry, dir);
         while (entry) {
 
-                snprintf (path, PATH_MAX, "%s/%s", pathname, entry->d_name);
+                snprintf (path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
                 ret = stat (path, &st);
                 if (ret == -1) {
-                        gf_log ("", GF_LOG_ERROR, "Failed to stat entry: %s:%s",
-                                path, strerror (errno));
+                        gf_log (this->name, GF_LOG_DEBUG, "Failed to stat "
+                                "entry %s : %s", path, strerror (errno));
                         goto stat_failed;
                 }
 
@@ -956,12 +1406,13 @@ glusterd_store_delete_volume (glusterd_volinfo_t *volinfo)
                 else
                         ret = unlink (path);
 
-                if (ret)
-                        gf_log ("", GF_LOG_INFO, "errno:%d (%s)", errno,
-                                strerror (errno));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, " Failed to remove "
+                                "%s. Reason : %s", path, strerror (errno));
+                }
 
-                gf_log ("", GF_LOG_INFO, "%s %s",
-                                ret?"Failed to remove":"Removed",
+                gf_log (this->name, GF_LOG_DEBUG, "%s %s",
+                                ret ? "Failed to remove":"Removed",
                                 entry->d_name);
 stat_failed:
                 memset (path, 0, sizeof(path));
@@ -970,339 +1421,320 @@ stat_failed:
 
         ret = closedir (dir);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Failed to close dir, errno:%d",
-                        errno);
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to close dir %s. "
+                        "Reason : %s",delete_path, strerror (errno));
         }
 
-        ret = rmdir (pathname);
+        ret = rmdir (delete_path);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Failed to rmdir: %s, err: %s",
-                        pathname, strerror (errno));
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s,err: %s",
+                        delete_path, strerror (errno));
+        }
+        ret = rmdir (trashdir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s, Reason:"
+                        " %s", trashdir, strerror (errno));
         }
-
 
 out:
         if (volinfo->shandle) {
-                glusterd_store_handle_destroy (volinfo->shandle);
+                gf_store_handle_destroy (volinfo->shandle);
                 volinfo->shandle = NULL;
         }
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        ret = (rename_fail == _gf_true) ? -1: 0;
 
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+/*TODO: cleanup the duplicate code and implement a generic function for
+ * deleting snap/volume depending on the parameter flag */
+int32_t
+glusterd_store_delete_snap (glusterd_snap_t *snap)
+{
+        char             pathname[PATH_MAX]    = {0,};
+        int32_t          ret                   = 0;
+        glusterd_conf_t *priv                  = NULL;
+        DIR             *dir                   = NULL;
+        struct dirent   *entry                 = NULL;
+        char             path[PATH_MAX]        = {0,};
+        char             delete_path[PATH_MAX] = {0,};
+        char             trashdir[PATH_MAX]    = {0,};
+        struct stat      st                    = {0, };
+        xlator_t        *this                  = NULL;
+        gf_boolean_t     rename_fail           = _gf_false;
 
-int
-glusterd_store_read_and_tokenize (FILE *file, char *str,
-                                  char **iter_key, char **iter_val,
-                                  glusterd_store_op_errno_t *store_errno)
-{
-        int32_t  ret = -1;
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (snap);
+        GLUSTERD_GET_SNAP_DIR (pathname, snap, priv);
 
-        GF_ASSERT (file);
-        GF_ASSERT (str);
-        GF_ASSERT (iter_key);
-        GF_ASSERT (iter_val);
-        GF_ASSERT (store_errno);
+        snprintf (delete_path, sizeof (delete_path),
+        "%s/"GLUSTERD_TRASH"/snap-%s.deleted", priv->workdir,
+        uuid_utoa (snap->snap_id));
 
-        ret = fscanf (file, "%s", str);
-        if (ret <= 0 || feof (file)) {
+        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
+                  priv->workdir);
+
+        ret = mkdir (trashdir, 0777);
+        if (ret && errno != EEXIST) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create trash "
+                        "directory, reason : %s", strerror (errno));
                 ret = -1;
-                *store_errno = GD_STORE_EOF;
                 goto out;
         }
 
-        *iter_key = strtok (str, "=");
-        if (*iter_key == NULL) {
-                ret = -1;
-                *store_errno = GD_STORE_KEY_NULL;
+        ret = rename (pathname, delete_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to rename snap "
+                        "directory %s to %s", snap->snapname, delete_path);
+                rename_fail = _gf_true;
                 goto out;
         }
 
-        *iter_val = strtok (NULL, "=");
-        if (*iter_key == NULL) {
-                ret = -1;
-                *store_errno = GD_STORE_VALUE_NULL;
+        dir = opendir (delete_path);
+        if (!dir) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to open directory %s."
+                        " Reason : %s", delete_path, strerror (errno));
+                ret = 0;
                 goto out;
         }
 
-        *store_errno = GD_STORE_SUCCESS;
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-glusterd_store_retrieve_value (glusterd_store_handle_t *handle,
-                               char *key, char **value)
-{
-        int32_t         ret = -1;
-        char            *scan_str = NULL;
-        char            *iter_key = NULL;
-        char            *iter_val = NULL;
-        char            *free_str = NULL;
-        struct stat     st        = {0,};
-        glusterd_store_op_errno_t store_errno = GD_STORE_SUCCESS;
-
-        GF_ASSERT (handle);
+        glusterd_for_each_entry (entry, dir);
+        while (entry) {
+                snprintf (path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
+                ret = stat (path, &st);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Failed to stat "
+                                "entry %s : %s", path, strerror (errno));
+                        goto stat_failed;
+                }
 
-        handle->fd = open (handle->path, O_RDWR);
+                if (S_ISDIR (st.st_mode))
+                        ret = rmdir (path);
+                else
+                        ret = unlink (path);
 
-        if (!handle->read)
-                handle->read = fdopen (handle->fd, "r");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, " Failed to remove "
+                                "%s. Reason : %s", path, strerror (errno));
+                }
 
-        if (!handle->read) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open file %s errno: %d",
-                        handle->path, errno);
-                goto out;
+                gf_log (this->name, GF_LOG_DEBUG, "%s %s",
+                                ret ? "Failed to remove":"Removed",
+                                entry->d_name);
+stat_failed:
+                memset (path, 0, sizeof(path));
+                glusterd_for_each_entry (entry, dir);
         }
 
-        ret = fstat (handle->fd, &st);
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_WARNING,
-                        "stat on file failed");
-                ret = -1;
-                store_errno = GD_STORE_STAT_FAILED;
-                goto out;
+        ret = closedir (dir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to close dir %s. "
+                        "Reason : %s",delete_path, strerror (errno));
         }
 
-        scan_str = GF_CALLOC (1, st.st_size,
-                              gf_gld_mt_char);
-        if (scan_str == NULL) {
-                ret = -1;
-                store_errno = GD_STORE_ENOMEM;
-                goto out;
+        ret = rmdir (delete_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s,err: %s",
+                        delete_path, strerror (errno));
+        }
+        ret = rmdir (trashdir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s, Reason:"
+                        " %s", trashdir, strerror (errno));
         }
 
-        free_str = scan_str;
-
-        do {
-                ret = glusterd_store_read_and_tokenize (handle->read, scan_str,
-                                                        &iter_key, &iter_val,
-                                                        &store_errno);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                gf_log ("", GF_LOG_DEBUG, "key %s read", iter_key);
-
-                if (!strcmp (key, iter_key)) {
-                        gf_log ("", GF_LOG_DEBUG, "key %s found", key);
-                        ret = 0;
-                        if (iter_val)
-                                *value = gf_strdup (iter_val);
-                        goto out;
-                }
-        } while (1);
 out:
-        if (handle->fd > 0) {
-                close (handle->fd);
-                handle->read = NULL;
+        if (snap->shandle) {
+                gf_store_handle_destroy (snap->shandle);
+                snap->shandle = NULL;
         }
+        ret = (rename_fail == _gf_true) ? -1: 0;
 
-        if (free_str)
-                GF_FREE (free_str);
-
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-int32_t
-glusterd_store_save_value (int fd, char *key, char *value)
+int
+glusterd_store_global_info (xlator_t *this)
 {
-        int32_t         ret = -1;
-        FILE           *fp  = NULL;
+        int                     ret                     = -1;
+        glusterd_conf_t         *conf                   = NULL;
+        char                    op_version_str[15]      = {0,};
+        char                    path[PATH_MAX]          = {0,};
+        gf_store_handle_t       *handle                 = NULL;
+        char                    *uuid_str               = NULL;
+        char                     buf[256]               = {0, };
 
-        GF_ASSERT (fd > 0);
-        GF_ASSERT (key);
-        GF_ASSERT (value);
+        conf = this->private;
 
-        fp = fdopen (fd, "a+");
-        if (fp == NULL) {
-                gf_log ("", GF_LOG_WARNING, "fdopen failed.");
-                ret = -1;
+        uuid_str = gf_strdup (uuid_utoa (MY_UUID));
+        if (!uuid_str)
                 goto out;
-        }
 
-        ret = fprintf (fp, "%s=%s\n", key, value);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "Unable to store key: %s,"
-                        "value: %s, error: %s", key, value,
-                        strerror (errno));
-                ret = -1;
+        if (!conf->handle) {
+                snprintf (path, PATH_MAX, "%s/%s", conf->workdir,
+                          GLUSTERD_INFO_FILE);
+                ret = gf_store_handle_new (path, &handle);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get store handle");
+                        goto out;
+                }
+
+                conf->handle = handle;
+        } else
+                handle = conf->handle;
+
+        /* These options need to be available for all users */
+        ret = chmod (handle->path, 0644);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "chmod error for %s: %s",
+                        GLUSTERD_INFO_FILE, strerror (errno));
                 goto out;
         }
 
-        ret = fflush (fp);
-        if (feof (fp)) {
-                gf_log ("", GF_LOG_WARNING,
-                        "fflush failed, error: %s",
-                        strerror (errno));
+        handle->fd = gf_store_mkstemp (handle);
+        if (handle->fd <= 0) {
                 ret = -1;
                 goto out;
         }
 
-        ret = 0;
-out:
-
-        gf_log ("", GF_LOG_DEBUG, "returning: %d", ret);
-        return ret;
-}
-
-int32_t
-glusterd_store_handle_new (char *path, glusterd_store_handle_t **handle)
-{
-        int32_t                 ret = -1;
-        glusterd_store_handle_t *shandle = NULL;
-        int                     fd = -1;
-        char                    *spath = NULL;
-
-        shandle = GF_CALLOC (1, sizeof (*shandle), gf_gld_mt_store_handle_t);
-        if (!shandle)
-                goto out;
-
-        spath = gf_strdup (path);
-
-        if (!spath)
-                goto out;
-
-        fd = open (path, O_RDWR | O_CREAT | O_APPEND, 0644);
-        if (fd <= 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to open file: %s, "
-                        "error: %s", path, strerror (errno));
+        ret = gf_store_save_value (handle->fd, GLUSTERD_STORE_UUID_KEY,
+                                   uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Storing uuid failed ret = %d", ret);
                 goto out;
         }
 
-        shandle->path = spath;
-        *handle = shandle;
-
-        ret = 0;
-
-out:
-        if (fd > 0)
-                close (fd);
-
-        if (ret == -1) {
-                if (spath)
-                        GF_FREE (spath);
-                if (shandle) {
-                        GF_FREE (shandle);
-                }
+        snprintf (op_version_str, 15, "%d", conf->op_version);
+        ret = gf_store_save_value (handle->fd, GD_OP_VERSION_KEY,
+                                   op_version_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Storing op-version failed ret = %d", ret);
+                goto out;
         }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int
-glusterd_store_handle_retrieve (char *path, glusterd_store_handle_t **handle)
-{
-        int32_t                 ret = -1;
-        struct stat statbuf = {0};
-
-        ret = stat (path, &statbuf);
+        snprintf (buf, sizeof (buf), "%"PRIu64, conf->snap_max_hard_limit);
+        ret = gf_store_save_value (handle->fd,
+                                   GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT, buf);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to retrieve store "
-                        "handle for %s, error: %s", path, strerror (errno));
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Storing snap-max-hard-limit failed ret = %d", ret);
                 goto out;
         }
-        ret =  glusterd_store_handle_new (path, handle);
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
 
-int32_t
-glusterd_store_handle_destroy (glusterd_store_handle_t *handle)
-{
-        int32_t                 ret = -1;
-
-        if (!handle) {
-                ret = 0;
+        snprintf (buf, sizeof (buf), "%"PRIu64, conf->snap_max_soft_limit);
+        ret = gf_store_save_value (handle->fd,
+                                   GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT, buf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Storing snap-max-soft-limit failed ret = %d", ret);
                 goto out;
         }
 
-        GF_FREE (handle->path);
+        ret = gf_store_rename_tmppath (handle);
+out:
+        if (ret && (handle->fd > 0))
+                gf_store_unlink_tmppath (handle);
 
-        GF_FREE (handle);
+        if (handle->fd > 0) {
+                close (handle->fd);
+                handle->fd = 0;
+        }
 
-        ret = 0;
+        if (uuid_str)
+                GF_FREE (uuid_str);
 
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to store glusterd global-info");
 
         return ret;
 }
 
-int32_t
-glusterd_store_uuid ()
+int
+glusterd_retrieve_op_version (xlator_t *this, int *op_version)
 {
-        glusterd_conf_t *priv = NULL;
-        char            path[PATH_MAX] = {0,};
-        int32_t         ret = -1;
-        glusterd_store_handle_t *handle = NULL;
-
-        priv = THIS->private;
+        char                    *op_version_str = NULL;
+        glusterd_conf_t         *priv           = NULL;
+        int                     ret             = -1;
+        int                     tmp_version     = 0;
+        char                    *tmp            = NULL;
+        char                    path[PATH_MAX]  = {0,};
+        gf_store_handle_t       *handle         = NULL;
 
-        snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                  GLUSTERD_INFO_FILE);
+        priv = this->private;
 
         if (!priv->handle) {
-                ret = glusterd_store_handle_new (path, &handle);
+                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                          GLUSTERD_INFO_FILE);
+                ret = gf_store_handle_retrieve (path, &handle);
 
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to get store"
-                                " handle!");
+                        gf_log ("", GF_LOG_DEBUG, "Unable to get store "
+                                "handle!");
                         goto out;
                 }
 
                 priv->handle = handle;
-        } else {
-                handle = priv->handle;
         }
 
-        handle->fd = open (handle->path, O_RDWR | O_CREAT | O_TRUNC, 0644);
-        if (handle->fd <= 0) {
-                ret = -1;
+        ret = gf_store_retrieve_value (priv->handle, GD_OP_VERSION_KEY,
+                                       &op_version_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No previous op_version present");
                 goto out;
         }
-        ret = glusterd_store_save_value (handle->fd, GLUSTERD_STORE_UUID_KEY,
-                                         uuid_utoa (priv->uuid));
 
-        if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Storing uuid failed"
-                        "ret = %d", ret);
+        tmp_version = strtol (op_version_str, &tmp, 10);
+        if ((tmp_version <= 0) || (tmp && strlen (tmp) > 1)) {
+                gf_log (this->name, GF_LOG_WARNING, "invalid version number");
                 goto out;
         }
 
+        *op_version = tmp_version;
 
+        ret = 0;
 out:
-        if (handle->fd > 0) {
-                close (handle->fd);
-                handle->fd = 0;
-        }
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (op_version_str)
+                GF_FREE (op_version_str);
+
         return ret;
 }
 
-int32_t
-glusterd_retrieve_uuid ()
+int
+glusterd_retrieve_sys_snap_max_limit (xlator_t *this, uint64_t *limit,
+                                      char *key)
 {
-        char            *uuid_str = NULL;
-        int32_t         ret = -1;
-        glusterd_store_handle_t *handle = NULL;
-        glusterd_conf_t *priv = NULL;
-        char            path[PATH_MAX] = {0,};
+        char                    *limit_str      = NULL;
+        glusterd_conf_t         *priv           = NULL;
+        int                     ret             = -1;
+        uint64_t                tmp_limit       = 0;
+        char                    *tmp            = NULL;
+        char                    path[PATH_MAX]  = {0,};
+        gf_store_handle_t       *handle         = NULL;
 
-        priv = THIS->private;
+        GF_ASSERT (this);
+        priv = this->private;
 
+        GF_ASSERT (priv);
+        GF_ASSERT (limit);
+        GF_ASSERT (key);
 
         if (!priv->handle) {
                 snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
                           GLUSTERD_INFO_FILE);
-                ret = glusterd_store_handle_retrieve (path, &handle);
+                ret = gf_store_handle_retrieve (path, &handle);
 
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to get store "
+                        gf_log ("", GF_LOG_DEBUG, "Unable to get store "
                                 "handle!");
                         goto out;
                 }
@@ -1310,262 +1742,152 @@ glusterd_retrieve_uuid ()
                 priv->handle = handle;
         }
 
-        ret = glusterd_store_retrieve_value (priv->handle,
-                                             GLUSTERD_STORE_UUID_KEY,
-                                             &uuid_str);
-
+        ret = gf_store_retrieve_value (priv->handle,
+                                       key,
+                                       &limit_str);
         if (ret) {
-                gf_log ("", GF_LOG_INFO, "No previous uuid is present");
-                goto out;
-        }
-
-        uuid_parse (uuid_str, priv->uuid);
-
-out:
-        if (uuid_str)
-                GF_FREE (uuid_str);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int32_t
-glusterd_store_iter_new (glusterd_store_handle_t  *shandle,
-                         glusterd_store_iter_t  **iter)
-{
-        int32_t                 ret = -1;
-        glusterd_store_iter_t   *tmp_iter = NULL;
-        int                     fd = -1;
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (iter);
-
-        tmp_iter = GF_CALLOC (1, sizeof (*tmp_iter),
-                             gf_gld_mt_store_iter_t);
-
-        if (!tmp_iter) {
-                gf_log ("", GF_LOG_ERROR, "Out of Memory");
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No previous %s present", key);
                 goto out;
         }
 
-        fd = open (shandle->path, O_RDWR);
-
-        if (fd < 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open %s, errno: %d",
-                        shandle->path, errno);
+        tmp_limit = strtoul (limit_str, &tmp, 10);
+        if ((tmp_limit <= 0) || (tmp && strlen (tmp) > 1)) {
+                gf_log (this->name, GF_LOG_WARNING, "invalid version number");
                 goto out;
         }
 
-        tmp_iter->fd = fd;
+        *limit = tmp_limit;
 
-        tmp_iter->file = fdopen (tmp_iter->fd, "r");
-
-        if (!tmp_iter->file) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open file %s errno: %d",
-                        shandle->path, errno);
-                goto out;
-        }
-
-        strncpy (tmp_iter->filepath, shandle->path, sizeof (tmp_iter->filepath));
-        *iter = tmp_iter;
         ret = 0;
-
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-}
-
-int32_t
-glusterd_store_validate_key_value (char *storepath, char *key, char*val,
-                                   glusterd_store_op_errno_t *op_errno)
-{
-        int ret = 0;
-
-        GF_ASSERT (op_errno);
-        GF_ASSERT (storepath);
-
-        if ((key == NULL) && (val == NULL)) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Glusterd store may be "
-                        "corrupted, Invalid key and value (null) in %s",
-                        storepath);
-                *op_errno = GD_STORE_KEY_VALUE_NULL;
-        } else if (key == NULL) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Glusterd store may be "
-                        "corrupted, Invalid key (null) in %s", storepath);
-                *op_errno = GD_STORE_KEY_NULL;
-        } else if (val == NULL) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Glusterd store may be "
-                        "corrupted, Invalid value (null) for key %s in %s",
-                        key, storepath);
-                *op_errno = GD_STORE_VALUE_NULL;
-        } else {
-                ret = 0;
-                *op_errno = GD_STORE_SUCCESS;
-        }
+        if (limit_str)
+                GF_FREE (limit_str);
 
         return ret;
 }
-
-int32_t
-glusterd_store_iter_get_next (glusterd_store_iter_t *iter,
-                              char  **key, char **value,
-                              glusterd_store_op_errno_t *op_errno)
+static int
+glusterd_restore_op_version (xlator_t *this)
 {
-        int32_t         ret = -1;
-        char            *scan_str = NULL;
-        char            *free_str = NULL;
-        char            *iter_key = NULL;
-        char            *iter_val = NULL;
-        struct stat     st        = {0,};
-        glusterd_store_op_errno_t store_errno = GD_STORE_SUCCESS;
-
-        GF_ASSERT (iter);
-        GF_ASSERT (iter->file);
-        GF_ASSERT (key);
-        GF_ASSERT (value);
-
-        ret = fstat (iter->fd, &st);
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_WARNING,
-                        "stat on file failed");
-                ret = -1;
-                store_errno = GD_STORE_STAT_FAILED;
-                goto out;
-        }
-
-        scan_str = GF_CALLOC (1, st.st_size,
-                              gf_gld_mt_char);
-        if (scan_str == NULL) {
-                ret = -1;
-                store_errno = GD_STORE_ENOMEM;
-                goto out;
-        }
-
-        *key = NULL;
-        *value = NULL;
+        glusterd_conf_t *conf           = NULL;
+        int              ret            = 0;
+        int              op_version     = 0;
 
-        free_str = scan_str;
+        conf = this->private;
 
-        ret = glusterd_store_read_and_tokenize (iter->file, scan_str,
-                                                &iter_key, &iter_val,
-                                                &store_errno);
-        if (ret < 0) {
-                goto out;
-        }
-
-
-        ret = glusterd_store_validate_key_value (iter->filepath, iter_key,
-                                                 iter_val, &store_errno);
-        if (ret)
-                goto out;
-
-        *value = gf_strdup (iter_val);
-
-        *key   = gf_strdup (iter_key);
-        if (!iter_key || !iter_val) {
-                ret = -1;
-                store_errno = GD_STORE_ENOMEM;
-                goto out;
+        ret = glusterd_retrieve_sys_snap_max_limit (this,
+                                                    &conf->snap_max_hard_limit,
+                                                    GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unable to retrieve system snap-max-hard-limit, "
+                        "setting it to default value(%d)",
+                        GLUSTERD_SNAPS_MAX_HARD_LIMIT);
+                conf->snap_max_hard_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
         }
 
-        ret = 0;
-
-out:
+        ret = glusterd_retrieve_sys_snap_max_limit (this,
+                                                    &conf->snap_max_soft_limit,
+                                                    GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
         if (ret) {
-                if (*key) {
-                        GF_FREE (*key);
-                        *key = NULL;
-                }
-                if (*value) {
-                        GF_FREE (*value);
-                        *value = NULL;
-                }
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unable to retrieve system snap-max-soft-limit, "
+                        "setting it to default value(%d)",
+                        GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT);
+                conf->snap_max_soft_limit = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
         }
-        if (free_str)
-                GF_FREE (free_str);
-        if (op_errno)
-                *op_errno = store_errno;
-
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-}
 
-int32_t
-glusterd_store_iter_get_matching (glusterd_store_iter_t *iter,
-                              char  *key, char **value)
-{
-        int32_t ret = -1;
-        char    *tmp_key = NULL;
-        char    *tmp_value = NULL;
-
-        ret = glusterd_store_iter_get_next (iter, &tmp_key, &tmp_value,
-                                            NULL);
-        while (!ret) {
-                if (!strncmp (key, tmp_key, strlen (key))){
-                        *value = tmp_value;
-                        GF_FREE (tmp_key);
+        ret = glusterd_retrieve_op_version (this, &op_version);
+        if (!ret) {
+                if ((op_version < GD_OP_VERSION_MIN) ||
+                    (op_version > GD_OP_VERSION_MAX)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "wrong op-version (%d) retrieved", op_version);
+                        ret = -1;
                         goto out;
                 }
-                GF_FREE (tmp_key);
-                GF_FREE (tmp_value);
-                ret = glusterd_store_iter_get_next (iter, &tmp_key,
-                                                    &tmp_value, NULL);
+                conf->op_version = op_version;
+                gf_log ("glusterd", GF_LOG_INFO,
+                        "retrieved op-version: %d", conf->op_version);
+                goto out;
+        }
+
+        /* op-version can be missing from the store file in 2 cases,
+         * 1. This is a new install of glusterfs
+         * 2. This is an upgrade of glusterfs from a version without op-version
+         *    to a version with op-version (eg. 3.3 -> 3.4)
+         *
+         * Detection of a new install or an upgrade from an older install can be
+         * done by checking for the presence of the its peer-id in the store
+         * file.  If peer-id is present, the installation is an upgrade else, it
+         * is a new install.
+         *
+         * For case 1, set op-version to GD_OP_VERSION_MAX.
+         * For case 2, set op-version to GD_OP_VERSION_MIN.
+         */
+        ret = glusterd_retrieve_uuid();
+        if (ret) {
+                gf_log (this->name, GF_LOG_INFO, "Detected new install. Setting"
+                        " op-version to maximum : %d", GD_OP_VERSION_MAX);
+                conf->op_version = GD_OP_VERSION_MAX;
+        } else {
+                gf_log (this->name, GF_LOG_INFO, "Upgrade detected. Setting"
+                        " op-version to minimum : %d", GD_OP_VERSION_MIN);
+                conf->op_version = GD_OP_VERSION_MIN;
         }
+        ret = 0;
 out:
         return ret;
 }
 
 int32_t
-glusterd_store_iter_destroy (glusterd_store_iter_t *iter)
+glusterd_retrieve_uuid ()
 {
+        char            *uuid_str = NULL;
         int32_t         ret = -1;
+        gf_store_handle_t *handle = NULL;
+        glusterd_conf_t *priv = NULL;
+        char            path[PATH_MAX] = {0,};
 
-        GF_ASSERT (iter);
-        GF_ASSERT (iter->fd > 0);
+        priv = THIS->private;
 
-        ret = fclose (iter->file);
+        if (!priv->handle) {
+                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                          GLUSTERD_INFO_FILE);
+                ret = gf_store_handle_retrieve (path, &handle);
+
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Unable to get store"
+                                "handle!");
+                        goto out;
+                }
+
+                priv->handle = handle;
+        }
+
+        ret = gf_store_retrieve_value (priv->handle, GLUSTERD_STORE_UUID_KEY,
+                                       &uuid_str);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to close fd: %d, ret: %d, "
-                        "errno: %d" ,iter->fd, ret, errno);
+                gf_log ("", GF_LOG_DEBUG, "No previous uuid is present");
+                goto out;
         }
 
-        GF_FREE (iter);
+        uuid_parse (uuid_str, priv->uuid);
 
+out:
+        GF_FREE (uuid_str);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-char*
-glusterd_store_strerror (glusterd_store_op_errno_t op_errno)
-{
-        switch (op_errno) {
-        case GD_STORE_SUCCESS:
-                return "Success";
-        case GD_STORE_KEY_NULL:
-                return "Invalid Key";
-        case GD_STORE_VALUE_NULL:
-                return "Invalid Value";
-        case GD_STORE_KEY_VALUE_NULL:
-                return "Invalid Key and Value";
-        case GD_STORE_EOF:
-                return "No data";
-        case GD_STORE_ENOMEM:
-                return "No memory";
-        default:
-                return "Invalid errno";
-        }
-        return "Invalid errno";
-}
 
 int32_t
 glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
 {
-
         int32_t                 ret = 0;
         glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_store_iter_t   *iter = NULL;
+        gf_store_iter_t         *iter = NULL;
         char                    *key = NULL;
         char                    *value = NULL;
         char                    brickdir[PATH_MAX] = {0,};
@@ -1573,20 +1895,19 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
         glusterd_conf_t         *priv = NULL;
         int32_t                 brick_count = 0;
         char                    tmpkey[4096] = {0,};
-        glusterd_store_iter_t   *tmpiter = NULL;
+        gf_store_iter_t         *tmpiter = NULL;
         char                    *tmpvalue = NULL;
-        struct pmap_registry *pmap = NULL;
-        glusterd_store_op_errno_t       op_errno = GD_STORE_SUCCESS;
+        struct pmap_registry    *pmap = NULL;
+        gf_store_op_errno_t     op_errno = GD_STORE_SUCCESS;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (volinfo->volname);
 
         priv = THIS->private;
 
-        GLUSTERD_GET_BRICK_DIR (brickdir, volinfo, priv)
-
-        ret = glusterd_store_iter_new (volinfo->shandle, &tmpiter);
+        GLUSTERD_GET_BRICK_DIR (brickdir, volinfo, priv);
 
+        ret = gf_store_iter_new (volinfo->shandle, &tmpiter);
         if (ret)
                 goto out;
 
@@ -1597,30 +1918,28 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                         goto out;
                 snprintf (tmpkey, sizeof (tmpkey), "%s-%d",
                           GLUSTERD_STORE_KEY_VOL_BRICK,brick_count);
-                ret = glusterd_store_iter_get_matching (tmpiter, tmpkey,
-                                                        &tmpvalue);
+                ret = gf_store_iter_get_matching (tmpiter, tmpkey, &tmpvalue);
                 snprintf (path, sizeof (path), "%s/%s", brickdir, tmpvalue);
 
                 GF_FREE (tmpvalue);
 
                 tmpvalue = NULL;
 
-                ret = glusterd_store_handle_retrieve (path, &brickinfo->shandle);
+                ret = gf_store_handle_retrieve (path, &brickinfo->shandle);
 
                 if (ret)
                         goto out;
 
-                ret = glusterd_store_iter_new (brickinfo->shandle, &iter);
+                ret = gf_store_iter_new (brickinfo->shandle, &iter);
 
                 if (ret)
                         goto out;
 
-                ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                    &op_errno);
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
                 if (ret) {
                         gf_log ("glusterd", GF_LOG_ERROR, "Unable to iterate "
                                 "the store for brick: %s, reason: %s", path,
-                                glusterd_store_strerror (op_errno));
+                                gf_store_strerror (op_errno));
                         goto out;
                 }
                 while (!ret) {
@@ -1634,22 +1953,52 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                         } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_PORT,
                                     strlen (GLUSTERD_STORE_KEY_BRICK_PORT))) {
                                 gf_string2int (value, &brickinfo->port);
-                                /* This is required to have proper ports
-                                   assigned to bricks after restart */
-                                pmap = pmap_registry_get (THIS);
-                                if (pmap->last_alloc <= brickinfo->port)
-                                        pmap->last_alloc = brickinfo->port + 1;
+
+                                if (brickinfo->port < priv->base_port) {
+                                        /* This is required to adhere to the
+                                           IANA standards */
+                                        brickinfo->port = 0;
+                                } else {
+                                        /* This is required to have proper ports
+                                           assigned to bricks after restart */
+                                        pmap = pmap_registry_get (THIS);
+                                        if (pmap->last_alloc <= brickinfo->port)
+                                                pmap->last_alloc =
+                                                        brickinfo->port + 1;
+                                }
                         } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
                                     strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
                                 gf_string2int (value, &brickinfo->rdma_port);
-                                /* This is required to have proper ports
-                                   assigned to bricks after restart */
-                                pmap = pmap_registry_get (THIS);
-                                if (pmap->last_alloc <= brickinfo->rdma_port)
-                                        pmap->last_alloc = brickinfo->rdma_port + 1;
+
+                                if (brickinfo->rdma_port < priv->base_port) {
+                                        /* This is required to adhere to the
+                                           IANA standards */
+                                        brickinfo->rdma_port = 0;
+                                } else {
+                                        /* This is required to have proper ports
+                                           assigned to bricks after restart */
+                                        pmap = pmap_registry_get (THIS);
+                                        if (pmap->last_alloc <=
+                                            brickinfo->rdma_port)
+                                                pmap->last_alloc =
+                                                        brickinfo->rdma_port +1;
+                                }
+
                         } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
                                              strlen (GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
                                 gf_string2int (value, &brickinfo->decommissioned);
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
+                                             strlen (GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH))) {
+                                strncpy (brickinfo->device_path, value,
+                                         sizeof (brickinfo->device_path));
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                                             strlen (GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
+                                gf_string2int (value, &brickinfo->snap_status);
+                        } else if (!strncmp (key,
+                                    GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                                    strlen (GLUSTERD_STORE_KEY_BRICK_VGNAME))) {
+                                strncpy (brickinfo->vg, value,
+                                         sizeof (brickinfo->vg));
                         } else {
                                 gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
                                         key);
@@ -1660,13 +2009,13 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                         key = NULL;
                         value = NULL;
 
-                        ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                            &op_errno);
+                        ret = gf_store_iter_get_next (iter, &key, &value,
+                                                      &op_errno);
                 }
 
                 if (op_errno != GD_STORE_EOF)
                         goto out;
-                ret = glusterd_store_iter_destroy (iter);
+                ret = gf_store_iter_destroy (iter);
 
                 if (ret)
                         goto out;
@@ -1675,7 +2024,7 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                 brick_count++;
         }
 
-        ret = glusterd_store_iter_destroy (tmpiter);
+        ret = gf_store_iter_destroy (tmpiter);
         if (ret)
                 goto out;
 out:
@@ -1686,64 +2035,79 @@ out:
 
 
 int32_t
-glusterd_store_retrieve_rbstate (char   *volname)
+glusterd_store_retrieve_rbstate (glusterd_volinfo_t *volinfo)
 {
         int32_t                   ret                   = -1;
-        glusterd_volinfo_t        *volinfo              = NULL;
-        glusterd_store_iter_t     *iter                 = NULL;
+        gf_store_iter_t           *iter                 = NULL;
         char                      *key                  = NULL;
         char                      *value                = NULL;
         char                      volpath[PATH_MAX]     = {0,};
         glusterd_conf_t           *priv                 = NULL;
         char                      path[PATH_MAX]        = {0,};
-        glusterd_store_op_errno_t op_errno              = GD_STORE_SUCCESS;
+        gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
+        xlator_t                 *this                  = NULL;
 
-        priv = THIS->private;
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get"
-                        "volinfo for %s.", volname);
-                goto out;
-        }
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volinfo);
 
         GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
         snprintf (path, sizeof (path), "%s/%s", volpath,
                   GLUSTERD_VOLUME_RBSTATE_FILE);
 
-        ret = glusterd_store_handle_retrieve (path, &volinfo->rb_shandle);
+        ret = gf_store_handle_retrieve (path, &volinfo->rb_shandle);
 
         if (ret)
                 goto out;
 
-        ret = glusterd_store_iter_new (volinfo->rb_shandle, &iter);
+        ret = gf_store_iter_new (volinfo->rb_shandle, &iter);
 
         if (ret)
                 goto out;
 
-        ret = glusterd_store_iter_get_next (iter, &key, &value, &op_errno);
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
         if (ret)
                 goto out;
 
         while (!ret) {
                 if (!strncmp (key, GLUSTERD_STORE_KEY_RB_STATUS,
                               strlen (GLUSTERD_STORE_KEY_RB_STATUS))) {
-                        volinfo->rb_status = atoi (value);
+                        volinfo->rep_brick.rb_status = atoi (value);
                 }
 
-                if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+                if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
                         if (!strncmp (key, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
                                       strlen (GLUSTERD_STORE_KEY_RB_SRC_BRICK))) {
-                                ret = glusterd_brickinfo_from_brick (value,
-                                                                     &volinfo->src_brick);
+                                ret = glusterd_brickinfo_new_from_brick (value,
+                                                &volinfo->rep_brick.src_brick);
                                 if (ret)
                                         goto out;
                         } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_BRICK,
                                              strlen (GLUSTERD_STORE_KEY_RB_DST_BRICK))) {
-                                ret = glusterd_brickinfo_from_brick (value,
-                                                                     &volinfo->dst_brick);
+                                ret = glusterd_brickinfo_new_from_brick (value,
+                                                &volinfo->rep_brick.dst_brick);
                                 if (ret)
                                         goto out;
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_PORT,
+                                             strlen (GLUSTERD_STORE_KEY_RB_DST_PORT))) {
+                                switch (volinfo->transport_type) {
+                                case GF_TRANSPORT_RDMA:
+                                volinfo->rep_brick.dst_brick->rdma_port =
+                                                 atoi (value);
+                                        break;
+
+                                case GF_TRANSPORT_TCP:
+                                case GF_TRANSPORT_BOTH_TCP_RDMA:
+                                volinfo->rep_brick.dst_brick->port =
+                                                atoi (value);
+                                        break;
+                                }
+                        } else if (!strncmp (key, GF_REPLACE_BRICK_TID_KEY,
+                                             strlen (GF_REPLACE_BRICK_TID_KEY))) {
+                                        uuid_parse (value,
+                                                    volinfo->rep_brick.rb_id);
                         }
                 }
 
@@ -1752,66 +2116,144 @@ glusterd_store_retrieve_rbstate (char   *volname)
                 key = NULL;
                 value = NULL;
 
-                ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                    &op_errno);
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
         }
 
         if (op_errno != GD_STORE_EOF)
                 goto out;
 
-        ret = glusterd_store_iter_destroy (iter);
+        ret = gf_store_iter_destroy (iter);
 
         if (ret)
                 goto out;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
 
         return ret;
 }
 
 int32_t
-glusterd_store_retrieve_volume (char    *volname)
+glusterd_store_retrieve_node_state (glusterd_volinfo_t *volinfo)
 {
         int32_t                   ret                   = -1;
-        glusterd_volinfo_t        *volinfo              = NULL;
-        glusterd_store_iter_t     *iter                 = NULL;
+        gf_store_iter_t           *iter                 = NULL;
         char                      *key                  = NULL;
         char                      *value                = NULL;
         char                      volpath[PATH_MAX]     = {0,};
         glusterd_conf_t           *priv                 = NULL;
         char                      path[PATH_MAX]        = {0,};
-        int                       exists                = 0;
-        glusterd_store_op_errno_t op_errno              = GD_STORE_SUCCESS;
+        gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
+        xlator_t                 *this                  = NULL;
 
-        ret = glusterd_volinfo_new (&volinfo);
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volinfo);
 
+        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
+        snprintf (path, sizeof (path), "%s/%s", volpath,
+                  GLUSTERD_NODE_STATE_FILE);
+
+        ret = gf_store_handle_retrieve (path, &volinfo->node_state_shandle);
         if (ret)
                 goto out;
 
-        strncpy (volinfo->volname, volname, GLUSTERD_MAX_VOLUME_NAME);
+        ret = gf_store_iter_new (volinfo->node_state_shandle, &iter);
 
-        priv = THIS->private;
+        if (ret)
+                goto out;
 
-        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
-        snprintf (path, sizeof (path), "%s/%s", volpath,
-                  GLUSTERD_VOLUME_INFO_FILE);
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret)
+                goto out;
 
-        ret = glusterd_store_handle_retrieve (path, &volinfo->shandle);
+        while (ret == 0) {
+                if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
+                              strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
+                        volinfo->rebal.defrag_cmd = atoi (value);
+                }
 
-        if (ret)
+                if (volinfo->rebal.defrag_cmd) {
+                        if (!strncmp (key, GF_REBALANCE_TID_KEY,
+                                       strlen (GF_REBALANCE_TID_KEY)))
+                                uuid_parse (value, volinfo->rebal.rebalance_id);
+
+                        if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP,
+                                      strlen (GLUSTERD_STORE_KEY_DEFRAG_OP)))
+                                volinfo->rebal.op = atoi (value);
+                }
+
+                GF_FREE (key);
+                GF_FREE (value);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+
+        if (op_errno != GD_STORE_EOF)
                 goto out;
 
-        ret = glusterd_store_iter_new (volinfo->shandle, &iter);
+        ret = gf_store_iter_destroy (iter);
 
         if (ret)
                 goto out;
 
-        ret = glusterd_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret)
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+
+        return ret;
+}
+
+
+int
+glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
+{
+        int                     ret = -1;
+        int                     exists                  = 0;
+        char                    *key                    = NULL;
+        char                    *value                  = NULL;
+        char                    volpath[PATH_MAX]       = {0,};
+        char                    path[PATH_MAX]          = {0,};
+        xlator_t                *this                   = NULL;
+        glusterd_conf_t         *conf                   = NULL;
+        gf_store_iter_t         *iter                   = NULL;
+        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = THIS->private;
+        GF_ASSERT (volinfo);
+
+        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
+
+        snprintf (path, sizeof (path), "%s/%s", volpath,
+                  GLUSTERD_VOLUME_INFO_FILE);
+
+        ret = gf_store_handle_retrieve (path, &volinfo->shandle);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "volinfo handle is NULL");
+                goto out;
+        }
+
+        ret = gf_store_iter_new (volinfo->shandle, &iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get new store "
+                        "iter");
+                goto out;
+        }
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get next store "
+                        "iter");
                 goto out;
+        }
 
         while (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "key = %s value = %s", key, value);
                 if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_TYPE,
                               strlen (GLUSTERD_STORE_KEY_VOL_TYPE))) {
                         volinfo->type = atoi (value);
@@ -1850,6 +2292,16 @@ glusterd_store_retrieve_volume (char    *volname)
                                 gf_log ("", GF_LOG_WARNING,
                                         "failed to parse uuid");
 
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_USERNAME,
+                                     strlen (GLUSTERD_STORE_KEY_USERNAME))) {
+
+                        glusterd_auth_set_username (volinfo, value);
+
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_PASSWORD,
+                                     strlen (GLUSTERD_STORE_KEY_PASSWORD))) {
+
+                        glusterd_auth_set_password (volinfo, value);
+
                 } else if (strstr (key, "slave")) {
                         ret = dict_set_dynstr (volinfo->gsync_slaves, key,
                                                 gf_strdup (value));
@@ -1860,28 +2312,57 @@ glusterd_store_retrieve_volume (char    *volname)
                         }
                         gf_log ("", GF_LOG_DEBUG, "Parsed as "GEOREP" "
                                 " slave:key=%s,value:%s", key, value);
-                }
-                else {
-                        exists = glusterd_check_option_exists (key, NULL);
-                        if (exists == -1) {
+
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_OP_VERSION,
+                                strlen (GLUSTERD_STORE_KEY_VOL_OP_VERSION))) {
+                        volinfo->op_version = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                                strlen (GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION))) {
+                        volinfo->client_op_version = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CAPS,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_CAPS))) {
+                        volinfo->caps = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                                strlen (GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT))) {
+                        volinfo->snap_max_hard_limit = (uint64_t) atoll (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_IS_RESTORED,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_IS_RESTORED))) {
+                        volinfo->is_volume_restored = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_PARENT_VOLNAME,
+                                strlen (GLUSTERD_STORE_KEY_PARENT_VOLNAME))) {
+                        strncpy (volinfo->parent_volname, value, sizeof(volinfo->parent_volname) - 1);
+                } else {
+
+                        if (is_key_glusterd_hooks_friendly (key)) {
+                                exists = 1;
+
+                        } else  {
+                                exists = glusterd_check_option_exists (key,
+                                                                       NULL);
+                        }
+
+                        switch (exists) {
+                        case -1:
                                 ret = -1;
                                 goto out;
-                        }
-                        if (exists) {
+
+                        case 0:
+                                gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
+                                        key);
+                                break;
+
+                        case 1:
                                 ret = dict_set_str(volinfo->dict, key,
-                                                     gf_strdup (value));
+                                                   gf_strdup (value));
                                 if (ret) {
                                         gf_log ("",GF_LOG_ERROR, "Error in "
-                                                        "dict_set_str");
+                                                "dict_set_str");
                                         goto out;
                                 }
                                 gf_log ("", GF_LOG_DEBUG, "Parsed as Volume-"
-                                                "set:key=%s,value:%s",
-                                                                key, value);
+                                        "set:key=%s,value:%s", key, value);
+                                break;
                         }
-                        else
-                                gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
-                                        key);
                 }
 
                 GF_FREE (key);
@@ -1889,8 +2370,7 @@ glusterd_store_retrieve_volume (char    *volname)
                 key = NULL;
                 value = NULL;
 
-                ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                    &op_errno);
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
         }
 
         /* backward compatibility */
@@ -1924,18 +2404,64 @@ glusterd_store_retrieve_volume (char    *volname)
                         break;
                 }
 
-                volinfo->dist_leaf_count = (volinfo->stripe_count *
-                                            volinfo->replica_count);
+                volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+
+                volinfo->subvol_count = (volinfo->brick_count /
+                                         volinfo->dist_leaf_count);
+
+                /* Only calculate volume op-versions if they are not found */
+                if (!volinfo->op_version && !volinfo->client_op_version)
+                        gd_update_volume_op_versions (volinfo);
         }
 
         if (op_errno != GD_STORE_EOF)
                 goto out;
 
-        ret = glusterd_store_iter_destroy (iter);
+        ret = gf_store_iter_destroy (iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to destroy store "
+                        "iter");
+                goto out;
+        }
 
+        ret = 0;
+out:
+        return ret;
+}
+
+glusterd_volinfo_t*
+glusterd_store_retrieve_volume (char *volname, glusterd_snap_t *snap)
+{
+        int32_t                    ret                  = -1;
+        glusterd_volinfo_t        *volinfo              = NULL;
+        glusterd_volinfo_t        *origin_volinfo       = NULL;
+        glusterd_conf_t           *priv                 = NULL;
+        xlator_t                  *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volname);
+
+        ret = glusterd_volinfo_new (&volinfo);
         if (ret)
                 goto out;
 
+        priv = THIS->private;
+
+        strncpy (volinfo->volname, volname, GLUSTERD_MAX_VOLUME_NAME);
+        volinfo->snapshot = snap;
+        if (snap)
+                volinfo->is_snap_volume = _gf_true;
+
+        ret = glusterd_store_update_volinfo (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to update volinfo "
+                        "for %s volume", volname);
+                goto out;
+        }
+
         ret = glusterd_store_retrieve_bricks (volinfo);
         if (ret)
                 goto out;
@@ -1944,19 +2470,130 @@ glusterd_store_retrieve_volume (char    *volname)
         if (ret)
                 goto out;
 
-        list_add_tail (&volinfo->vol_list, &priv->volumes);
-
+        if (!snap) {
+                list_add_tail (&volinfo->vol_list, &priv->volumes);
+        } else {
+                ret = glusterd_volinfo_find (volinfo->parent_volname,
+                                             &origin_volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Parent volinfo "
+                                "not found for %s volume", volname);
+                        goto out;
+                }
+                glusterd_list_add_snapvol (origin_volinfo, volinfo);
+        }
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        if (ret) {
+                if (volinfo)
+                        glusterd_volinfo_delete (volinfo);
+                volinfo = NULL;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+
+        return volinfo;
+}
+
+inline void
+glusterd_store_set_options_path (glusterd_conf_t *conf, char *path, size_t len)
+{
+        snprintf (path, len, "%s/options", conf->workdir);
+}
+
+int
+_store_global_opts (dict_t *this, char *key, data_t *value, void *data)
+{
+        gf_store_handle_t *shandle = data;
+
+        gf_store_save_value (shandle->fd, key, (char*)value->data);
+        return 0;
+}
+
+int32_t
+glusterd_store_options (xlator_t *this, dict_t *opts)
+{
+        gf_store_handle_t               *shandle = NULL;
+        glusterd_conf_t                 *conf = NULL;
+        char                            path[PATH_MAX] = {0};
+        int                             fd = -1;
+        int32_t                         ret = -1;
+
+        conf = this->private;
+        glusterd_store_set_options_path (conf, path, sizeof (path));
 
+        ret = gf_store_handle_new (path, &shandle);
+        if (ret)
+                goto out;
+
+        fd = gf_store_mkstemp (shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        shandle->fd = fd;
+        dict_foreach (opts, _store_global_opts, shandle);
+        shandle->fd = 0;
+        ret = gf_store_rename_tmppath (shandle);
+        if (ret)
+                goto out;
+out:
+        gf_store_handle_destroy (shandle);
+        if (fd >=0 )
+                close (fd);
         return ret;
 }
 
+int32_t
+glusterd_store_retrieve_options (xlator_t *this)
+{
+        char                            path[PATH_MAX] = {0};
+        glusterd_conf_t                 *conf = NULL;
+        gf_store_handle_t               *shandle = NULL;
+        gf_store_iter_t                 *iter = NULL;
+        char                            *key = NULL;
+        char                            *value = NULL;
+        gf_store_op_errno_t             op_errno = 0;
+        int                             ret = -1;
+
+        conf = this->private;
+        glusterd_store_set_options_path (conf, path, sizeof (path));
+
+        ret = gf_store_handle_retrieve (path, &shandle);
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_new (shandle, &iter);
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        while (!ret) {
+                ret = dict_set_dynstr (conf->opts, key, value);
+                if (ret) {
+                        GF_FREE (key);
+                        GF_FREE (value);
+                        goto out;
+                }
+                GF_FREE (key);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+        ret = 0;
+out:
+        gf_store_iter_destroy (iter);
+        gf_store_handle_destroy (shandle);
+        return ret;
+}
 
 int32_t
-glusterd_store_retrieve_volumes (xlator_t  *this)
+glusterd_store_retrieve_volumes (xlator_t  *this, glusterd_snap_t *snap)
 {
-        int32_t                ret              = 0;
+        int32_t                ret              = -1;
         char                   path[PATH_MAX]   = {0,};
         glusterd_conf_t       *priv             = NULL;
         DIR                   *dir              = NULL;
@@ -1968,39 +2605,58 @@ glusterd_store_retrieve_volumes (xlator_t  *this)
 
         GF_ASSERT (priv);
 
-        snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                        GLUSTERD_VOLUME_DIR_PREFIX);
+        if (snap)
+                snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,
+                          snap->snapname);
+        else
+                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                          GLUSTERD_VOLUME_DIR_PREFIX);
 
         dir = opendir (path);
 
         if (!dir) {
                 gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
-                ret = -1;
                 goto out;
         }
 
         glusterd_for_each_entry (entry, dir);
 
         while (entry) {
-                ret = glusterd_store_retrieve_volume (entry->d_name);
-                if (ret) {
+                if ( entry->d_type != DT_DIR )
+                        goto next;
+
+                volinfo = glusterd_store_retrieve_volume (entry->d_name, snap);
+                if (!volinfo) {
                         gf_log ("", GF_LOG_ERROR, "Unable to restore "
                                 "volume: %s", entry->d_name);
+                        ret = -1;
                         goto out;
                 }
 
-                ret = glusterd_store_retrieve_rbstate (entry->d_name);
+                ret = glusterd_store_retrieve_rbstate (volinfo);
                 if (ret) {
                         /* Backward compatibility */
                         gf_log ("", GF_LOG_INFO, "Creating a new rbstate "
                                 "for volume: %s.", entry->d_name);
-                        ret = glusterd_volinfo_find (entry->d_name, &volinfo);
                         ret = glusterd_store_create_rbstate_shandle_on_absence (volinfo);
                         ret = glusterd_store_perform_rbstate_store (volinfo);
                 }
+
+                ret = glusterd_store_retrieve_node_state (volinfo);
+                if (ret) {
+                        /* Backward compatibility */
+                        gf_log ("", GF_LOG_INFO, "Creating a new node_state "
+                                "for volume: %s.", entry->d_name);
+                        glusterd_store_create_nodestate_sh_on_absence (volinfo);
+                        ret = glusterd_store_perform_node_state_store (volinfo);
+
+                }
+
+next:
                 glusterd_for_each_entry (entry, dir);
         }
 
+        ret = 0;
 out:
         if (dir)
                 closedir (dir);
@@ -2010,6 +2666,482 @@ out:
 }
 
 int32_t
+glusterd_resolve_snap_bricks (xlator_t  *this, glusterd_snap_t *snap)
+{
+        int32_t                 ret = -1;
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+
+        GF_ASSERT (this);
+        GF_VALIDATE_OR_GOTO (this->name, snap, out);
+
+        list_for_each_entry (volinfo, &snap->volumes, vol_list) {
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        ret = glusterd_resolve_brick (brickinfo);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "resolve brick failed in restore");
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_store_update_snap (glusterd_snap_t *snap)
+{
+        int                     ret = -1;
+        char                    *key                    = NULL;
+        char                    *value                  = NULL;
+        char                    snappath[PATH_MAX]      = {0,};
+        char                    path[PATH_MAX]          = {0,};
+        xlator_t                *this                   = NULL;
+        glusterd_conf_t         *conf                   = NULL;
+        gf_store_iter_t         *iter                   = NULL;
+        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (snap);
+
+        GLUSTERD_GET_SNAP_DIR (snappath, snap, conf);
+
+        snprintf (path, sizeof (path), "%s/%s", snappath,
+                  GLUSTERD_SNAP_INFO_FILE);
+
+        ret = gf_store_handle_retrieve (path, &snap->shandle);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "snap handle is NULL");
+                goto out;
+        }
+
+        ret = gf_store_iter_new (snap->shandle, &iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get new store "
+                        "iter");
+                goto out;
+        }
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get next store "
+                        "iter");
+                goto out;
+        }
+
+        while (!ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "key = %s value = %s",
+                        key, value);
+
+                if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_ID,
+                                     strlen (GLUSTERD_STORE_KEY_SNAP_ID))) {
+                        ret = uuid_parse (value, snap->snap_id);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Failed to parse uuid");
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_RESTORED,
+                                   strlen (GLUSTERD_STORE_KEY_SNAP_RESTORED))) {
+                        snap->snap_restored = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_STATUS,
+                                     strlen (GLUSTERD_STORE_KEY_SNAP_STATUS))) {
+                        snap->snap_status = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_DESC,
+                                     strlen (GLUSTERD_STORE_KEY_SNAP_DESC))) {
+                        snap->description = gf_strdup (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP,
+                                  strlen (GLUSTERD_STORE_KEY_SNAP_TIMESTAMP))) {
+                        snap->time_stamp = atoi (value);
+                }
+
+                GF_FREE (key);
+                GF_FREE (value);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+
+        ret = gf_store_iter_destroy (iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to destroy store "
+                        "iter");
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_snap (char *snapname)
+{
+        int32_t                ret    = -1;
+        dict_t                *dict   = NULL;
+        glusterd_snap_t       *snap   = NULL;
+        glusterd_conf_t       *priv   = NULL;
+        xlator_t              *this   = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snapname);
+
+        dict = dict_new();
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create dict");
+                ret = -1;
+                goto out;
+        }
+
+        snap = glusterd_new_snap_object ();
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create "
+                                                  " snap object");
+                goto out;
+        }
+
+        strncpy (snap->snapname, snapname, strlen(snapname));
+        ret = glusterd_store_update_snap (snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to update snapshot "
+                        "for %s snap", snapname);
+                goto out;
+        }
+
+        ret = glusterd_store_retrieve_volumes (this, snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to retrieve "
+                        "snap volumes for snap %s", snapname);
+                goto out;
+        }
+
+        /* Unlike bricks of normal volumes which are resolved at the end of
+           the glusterd restore, the bricks belonging to the snap volumes of
+           each snap should be resolved as part of snapshot restore itself.
+           Because if the snapshot has to be removed, then resolving bricks
+           helps glusterd in understanding what all bricks have its own uuid
+           and killing those bricks.
+        */
+        ret = glusterd_resolve_snap_bricks (this, snap);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "resolving the snap bricks"
+                        " failed (snap: %s)", snap?snap->snapname:"");
+
+        /* When the snapshot command from cli is received, the on disk and
+           in memory structures for the snapshot are created (with the status)
+           being marked as GD_SNAP_STATUS_INIT. Once the backend snapshot is
+           taken, the status is changed to GD_SNAP_STATUS_IN_USE. If glusterd
+           dies after taking the backend snapshot, but before updating the
+           status, then when glusterd comes up, it should treat that snapshot
+           as a failed snapshot and clean it up.
+        */
+        if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
+                ret = glusterd_snap_remove (dict, snap, _gf_true, _gf_true);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING, "failed to remove"
+                                " the snapshot %s", snap->snapname);
+                goto out;
+        }
+
+        /* TODO: list_add_order can do 'N-square' comparisions and
+           is not efficient. Find a better solution to store the snap
+           in order */
+        list_add_order (&snap->snap_list, &priv->snapshots,
+                        glusterd_compare_snap_time);
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+/* Read the missed_snap_list and update the in-memory structs */
+int32_t
+glusterd_store_retrieve_missed_snaps_list (xlator_t  *this)
+{
+        char                   buf[PATH_MAX]    = "";
+        char                   path[PATH_MAX]   = "";
+        char                  *missed_node_info = NULL;
+        char                  *brick_path       = NULL;
+        char                  *value            = NULL;
+        char                  *save_ptr         = NULL;
+        FILE                  *fp               = NULL;
+        int32_t                brick_num        = -1;
+        int32_t                snap_op          = -1;
+        int32_t                snap_status      = -1;
+        int32_t                ret              = -1;
+        glusterd_conf_t       *priv             = NULL;
+        gf_store_op_errno_t    store_errno      = GD_STORE_SUCCESS;
+
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Get the path of the missed_snap_list */
+        glusterd_store_missed_snaps_list_path_set (path, sizeof(path));
+
+        fp = fopen (path, "r");
+        if (!fp) {
+                /* If errno is ENOENT then there are no missed snaps yet */
+                if (errno != ENOENT) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to open %s. "
+                                "Error: %s", path, strerror(errno));
+                } else {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "No missed snaps list.");
+                        ret = 0;
+                }
+                goto out;
+        }
+
+        do {
+                ret = gf_store_read_and_tokenize (fp, buf,
+                                                  &missed_node_info, &value,
+                                                  &store_errno);
+                if (ret) {
+                        if (store_errno == GD_STORE_EOF) {
+                                gf_log (this->name,
+                                        GF_LOG_DEBUG,
+                                        "EOF for missed_snap_list");
+                                ret = 0;
+                                break;
+                        }
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to fetch data from "
+                                "missed_snaps_list. Error: %s",
+                                gf_store_strerror (store_errno));
+                        goto out;
+                }
+
+                /* Fetch the brick_num, brick_path, snap_op and snap status */
+                brick_num = atoi(strtok_r (value, ":", &save_ptr));
+                brick_path = strtok_r (NULL, ":", &save_ptr);
+                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
+                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
+
+                if (!missed_node_info || !brick_path ||
+                    brick_num < 1 || snap_op < 1 ||
+                    snap_status < 1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Invalid missed_snap_entry");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_store_missed_snaps_list (missed_node_info,
+                                                        brick_num,
+                                                        brick_path,
+                                                        snap_op,
+                                                        snap_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store missed snaps_list");
+                        goto out;
+                }
+
+        } while (store_errno == GD_STORE_SUCCESS);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_snaps (xlator_t  *this)
+{
+        int32_t                ret              = 0;
+        char                   path[PATH_MAX]   = {0,};
+        glusterd_conf_t       *priv             = NULL;
+        DIR                   *dir              = NULL;
+        struct dirent         *entry            = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        snprintf (path, PATH_MAX, "%s/snaps", priv->workdir);
+
+        dir = opendir (path);
+
+        if (!dir) {
+                /* If snaps dir doesn't exists ignore the error for
+                   backward compatibility */
+                if (errno != ENOENT) {
+                        ret = -1;
+                        gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
+                }
+                goto out;
+        }
+
+        glusterd_for_each_entry (entry, dir);
+
+        while (entry) {
+                if (entry->d_type == DT_DIR) {
+                        ret = glusterd_store_retrieve_snap (entry->d_name);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to restore snapshot: %s",
+                                        entry->d_name);
+                                goto out;
+                        }
+                }
+
+                glusterd_for_each_entry (entry, dir);
+        }
+
+        /* Retrieve missed_snaps_list */
+        ret = glusterd_store_retrieve_missed_snaps_list (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Failed to retrieve missed_snaps_list");
+                goto out;
+        }
+
+out:
+        if (dir)
+                closedir (dir);
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+/* Writes all the contents of conf->missed_snap_list */
+int32_t
+glusterd_store_write_missed_snapinfo (int32_t fd)
+{
+        char                           value[PATH_MAX]             = "";
+        int32_t                        ret                         = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
+        glusterd_snap_op_t            *snap_opinfo                 = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Write the missed_snap_entry */
+        list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
+                             missed_snaps) {
+                list_for_each_entry (snap_opinfo,
+                                     &missed_snapinfo->snap_ops,
+                                     snap_ops_list) {
+                        snprintf (value, sizeof(value), "%d:%s:%d:%d",
+                                  snap_opinfo->brick_num,
+                                  snap_opinfo->brick_path,
+                                  snap_opinfo->op, snap_opinfo->status);
+                        ret = gf_store_save_value
+                                           (fd,
+                                            missed_snapinfo->node_snap_info,
+                                            value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to write missed snapinfo");
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Adds the missed snap entries to the in-memory conf->missed_snap_list *
+ * and writes them to disk */
+int32_t
+glusterd_store_update_missed_snaps (dict_t *dict, int32_t missed_snap_count)
+{
+        int32_t                        fd                          = -1;
+        int32_t                        ret                         = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (missed_snap_count < 1) {
+                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_store_create_missed_snaps_list_shandle_on_absence ();
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to obtain "
+                        "missed_snaps_list store handle.");
+                goto out;
+        }
+
+        fd = gf_store_mkstemp (priv->missed_snaps_list_shandle);
+        if (fd <= 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create tmp file");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_add_missed_snaps_to_list (dict, missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to add missed snaps to list");
+                goto out;
+        }
+
+        ret = glusterd_store_write_missed_snapinfo (fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to write missed snaps to disk");
+                goto out;
+        }
+
+        ret = gf_store_rename_tmppath (priv->missed_snaps_list_shandle);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to rename the tmp file");
+                goto out;
+        }
+out:
+        if (ret && (fd > 0)) {
+                ret = gf_store_unlink_tmppath (priv->missed_snaps_list_shandle);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to unlink the tmp file");
+                }
+                ret = -1;
+        }
+
+        if (fd > 0)
+                close (fd);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
 glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo)
 {
         int32_t                         ret = -1;
@@ -2057,7 +3189,7 @@ glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo)
 
 out:
         if (peerinfo->shandle) {
-                glusterd_store_handle_destroy (peerinfo->shandle);
+                gf_store_handle_destroy (peerinfo->shandle);
                 peerinfo->shandle = NULL;
         }
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
@@ -2083,7 +3215,7 @@ glusterd_store_create_peer_dir ()
         char                            path[PATH_MAX];
 
         glusterd_store_peerinfo_dirpath_set (path, sizeof (path));
-        ret = glusterd_store_mkdir (path);
+        ret = gf_store_mkdir (path);
 
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
@@ -2127,8 +3259,8 @@ glusterd_store_peerinfo_hostname_shandle_create (glusterd_peerinfo_t *peerinfo)
 
         glusterd_store_hostname_peerpath_set (peerinfo, peerfpath,
                                               sizeof (peerfpath));
-        ret = glusterd_store_handle_create_on_absence (&peerinfo->shandle,
-                                                       peerfpath);
+        ret = gf_store_handle_create_on_absence (&peerinfo->shandle,
+                                                 peerfpath);
         return ret;
 }
 
@@ -2140,8 +3272,8 @@ glusterd_store_peerinfo_uuid_shandle_create (glusterd_peerinfo_t *peerinfo)
 
         glusterd_store_uuid_peerpath_set (peerinfo, peerfpath,
                                           sizeof (peerfpath));
-        ret = glusterd_store_handle_create_on_absence (&peerinfo->shandle,
-                                                       peerfpath);
+        ret = gf_store_handle_create_on_absence (&peerinfo->shandle,
+                                                 peerfpath);
         return ret;
 }
 
@@ -2157,7 +3289,7 @@ glusterd_peerinfo_hostname_shandle_check_destroy (glusterd_peerinfo_t *peerinfo)
         ret = stat (peerfpath, &stbuf);
         if (!ret) {
                 if (peerinfo->shandle)
-                        glusterd_store_handle_destroy (peerinfo->shandle);
+                        gf_store_handle_destroy (peerinfo->shandle);
                 peerinfo->shandle = NULL;
                 ret = unlink (peerfpath);
         }
@@ -2186,18 +3318,21 @@ glusterd_store_peer_write (int fd, glusterd_peerinfo_t *peerinfo)
         char                    buf[50] = {0};
         int32_t                 ret = 0;
 
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_UUID,
-                                         uuid_utoa (peerinfo->uuid));
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_UUID,
+                                   uuid_utoa (peerinfo->uuid));
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", peerinfo->state.state);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_STATE, buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_STATE, buf);
+        if (ret)
+                goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_HOSTNAME "1",
+                                   peerinfo->hostname);
         if (ret)
                 goto out;
 
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_HOSTNAME "1",
-                                         peerinfo->hostname);
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
@@ -2211,7 +3346,7 @@ glusterd_store_perform_peer_store (glusterd_peerinfo_t *peerinfo)
 
         GF_ASSERT (peerinfo);
 
-        fd = glusterd_store_mkstemp (peerinfo->shandle);
+        fd = gf_store_mkstemp (peerinfo->shandle);
         if (fd <= 0) {
                 ret = -1;
                 goto out;
@@ -2221,10 +3356,10 @@ glusterd_store_perform_peer_store (glusterd_peerinfo_t *peerinfo)
         if (ret)
                 goto out;
 
-        ret = glusterd_store_rename_tmppath (peerinfo->shandle);
+        ret = gf_store_rename_tmppath (peerinfo->shandle);
 out:
         if (ret && (fd > 0))
-                glusterd_store_unlink_tmppath (peerinfo->shandle);
+                gf_store_unlink_tmppath (peerinfo->shandle);
         if (fd > 0)
                 close (fd);
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2264,13 +3399,13 @@ glusterd_store_retrieve_peers (xlator_t *this)
         uuid_t                  uuid = {0,};
         char                    *hostname = NULL;
         int32_t                 state = 0;
-        glusterd_store_handle_t *shandle = NULL;
+        gf_store_handle_t       *shandle = NULL;
         char                    filepath[PATH_MAX] = {0,};
-        glusterd_store_iter_t   *iter = NULL;
+        gf_store_iter_t         *iter = NULL;
         char                    *key = NULL;
         char                    *value = NULL;
         glusterd_peerctx_args_t args = {0};
-        glusterd_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+        gf_store_op_errno_t     op_errno = GD_STORE_SUCCESS;
 
         GF_ASSERT (this);
         priv = this->private;
@@ -2292,16 +3427,15 @@ glusterd_store_retrieve_peers (xlator_t *this)
 
         while (entry) {
                 snprintf (filepath, PATH_MAX, "%s/%s", path, entry->d_name);
-                ret = glusterd_store_handle_retrieve (filepath, &shandle);
+                ret = gf_store_handle_retrieve (filepath, &shandle);
                 if (ret)
                         goto out;
 
-                ret = glusterd_store_iter_new (shandle, &iter);
+                ret = gf_store_iter_new (shandle, &iter);
                 if (ret)
                         goto out;
 
-                ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                    &op_errno);
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
                 if (ret)
                         goto out;
 
@@ -2329,17 +3463,16 @@ glusterd_store_retrieve_peers (xlator_t *this)
                         key = NULL;
                         value = NULL;
 
-                        ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                            &op_errno);
+                        ret = gf_store_iter_get_next (iter, &key, &value,
+                                                      &op_errno);
                 }
                 if (op_errno != GD_STORE_EOF)
                         goto out;
 
-                (void) glusterd_store_iter_destroy (iter);
+                (void) gf_store_iter_destroy (iter);
 
-                args.mode = GD_MODE_SWITCH_ON;
                 ret = glusterd_friend_add (hostname, 0, state, &uuid,
-                                           &peerinfo, 1, &args);
+                                           &peerinfo, 1, NULL);
 
                 GF_FREE (hostname);
                 if (ret)
@@ -2349,6 +3482,13 @@ glusterd_store_retrieve_peers (xlator_t *this)
                 glusterd_for_each_entry (entry, dir);
         }
 
+        args.mode = GD_MODE_ON;
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                ret = glusterd_friend_rpc_create (this, peerinfo, &args);
+                if (ret)
+                        goto out;
+        }
+
 out:
         if (dir)
                 closedir (dir);
@@ -2395,8 +3535,18 @@ glusterd_restore ()
 
         this = THIS;
 
-        ret = glusterd_store_retrieve_volumes (this);
+        ret = glusterd_restore_op_version (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to restore op_version");
+                goto out;
+        }
+
+        ret = glusterd_store_retrieve_volumes (this, NULL);
+        if (ret)
+                goto out;
 
+        ret = glusterd_store_retrieve_snaps (this);
         if (ret)
                 goto out;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index f1413955b..1b5cebc0c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_HA_H_
 #define _GLUSTERD_HA_H_
 
@@ -30,6 +20,7 @@
 
 #include "glusterfs.h"
 #include "xlator.h"
+#include "run.h"
 #include "logging.h"
 #include "call-stub.h"
 #include "fd.h"
@@ -40,35 +31,59 @@
 typedef enum glusterd_store_ver_ac_{
         GLUSTERD_VOLINFO_VER_AC_NONE = 0,
         GLUSTERD_VOLINFO_VER_AC_INCREMENT = 1,
+        GLUSTERD_VOLINFO_VER_AC_DECREMENT = 2,
 } glusterd_volinfo_ver_ac_t;
 
 
-#define GLUSTERD_STORE_UUID_KEY           "UUID"
-
-#define GLUSTERD_STORE_KEY_VOL_TYPE       "type"
-#define GLUSTERD_STORE_KEY_VOL_COUNT      "count"
-#define GLUSTERD_STORE_KEY_VOL_STATUS     "status"
-#define GLUSTERD_STORE_KEY_VOL_PORT       "port"
-#define GLUSTERD_STORE_KEY_VOL_SUB_COUNT  "sub_count"
-#define GLUSTERD_STORE_KEY_VOL_STRIPE_CNT  "stripe_count"
-#define GLUSTERD_STORE_KEY_VOL_REPLICA_CNT "replica_count"
-#define GLUSTERD_STORE_KEY_VOL_BRICK      "brick"
-#define GLUSTERD_STORE_KEY_VOL_VERSION    "version"
-#define GLUSTERD_STORE_KEY_VOL_TRANSPORT  "transport-type"
-#define GLUSTERD_STORE_KEY_VOL_ID         "volume-id"
-#define GLUSTERD_STORE_KEY_RB_STATUS      "rb_status"
-#define GLUSTERD_STORE_KEY_RB_SRC_BRICK   "rb_src"
-#define GLUSTERD_STORE_KEY_RB_DST_BRICK   "rb_dst"
-
-#define GLUSTERD_STORE_KEY_BRICK_HOSTNAME "hostname"
-#define GLUSTERD_STORE_KEY_BRICK_PATH     "path"
-#define GLUSTERD_STORE_KEY_BRICK_PORT     "listen-port"
-#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT "rdma.listen-port"
+#define GLUSTERD_STORE_UUID_KEY                 "UUID"
+
+#define GLUSTERD_STORE_KEY_VOL_TYPE             "type"
+#define GLUSTERD_STORE_KEY_VOL_COUNT            "count"
+#define GLUSTERD_STORE_KEY_VOL_STATUS           "status"
+#define GLUSTERD_STORE_KEY_VOL_PORT             "port"
+#define GLUSTERD_STORE_KEY_VOL_SUB_COUNT        "sub_count"
+#define GLUSTERD_STORE_KEY_VOL_STRIPE_CNT       "stripe_count"
+#define GLUSTERD_STORE_KEY_VOL_REPLICA_CNT      "replica_count"
+#define GLUSTERD_STORE_KEY_VOL_BRICK            "brick"
+#define GLUSTERD_STORE_KEY_VOL_VERSION          "version"
+#define GLUSTERD_STORE_KEY_VOL_TRANSPORT        "transport-type"
+#define GLUSTERD_STORE_KEY_VOL_ID               "volume-id"
+#define GLUSTERD_STORE_KEY_VOL_IS_RESTORED      "is-volume-restored"
+#define GLUSTERD_STORE_KEY_RB_STATUS            "rb_status"
+#define GLUSTERD_STORE_KEY_RB_SRC_BRICK         "rb_src"
+#define GLUSTERD_STORE_KEY_RB_DST_BRICK         "rb_dst"
+#define GLUSTERD_STORE_KEY_RB_DST_PORT          "rb_port"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG           "rebalance_status"
+#define GLUSTERD_STORE_KEY_DEFRAG_OP            "rebalance_op"
+#define GLUSTERD_STORE_KEY_USERNAME             "username"
+#define GLUSTERD_STORE_KEY_PASSWORD             "password"
+#define GLUSTERD_STORE_KEY_PARENT_VOLNAME       "parent_volname"
+#define GLUSTERD_STORE_KEY_VOL_OP_VERSION       "op-version"
+#define GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION "client-op-version"
+
+#define GLUSTERD_STORE_KEY_SNAP_NAME            "name"
+#define GLUSTERD_STORE_KEY_SNAP_ID              "snap-id"
+#define GLUSTERD_STORE_KEY_SNAP_DESC            "desc"
+#define GLUSTERD_STORE_KEY_SNAP_TIMESTAMP       "time-stamp"
+#define GLUSTERD_STORE_KEY_SNAP_STATUS          "status"
+#define GLUSTERD_STORE_KEY_SNAP_RESTORED        "snap-restored"
+#define GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT  "snap-max-hard-limit"
+#define GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT  "snap-max-soft-limit"
+
+#define GLUSTERD_STORE_KEY_BRICK_HOSTNAME       "hostname"
+#define GLUSTERD_STORE_KEY_BRICK_PATH           "path"
+#define GLUSTERD_STORE_KEY_BRICK_PORT           "listen-port"
+#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT      "rdma.listen-port"
 #define GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED "decommissioned"
+#define GLUSTERD_STORE_KEY_BRICK_VGNAME         "vg"
+#define GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH    "device_path"
+#define GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS    "snap-status"
 
-#define GLUSTERD_STORE_KEY_PEER_UUID      "uuid"
-#define GLUSTERD_STORE_KEY_PEER_HOSTNAME  "hostname"
-#define GLUSTERD_STORE_KEY_PEER_STATE     "state"
+#define GLUSTERD_STORE_KEY_PEER_UUID            "uuid"
+#define GLUSTERD_STORE_KEY_PEER_HOSTNAME        "hostname"
+#define GLUSTERD_STORE_KEY_PEER_STATE           "state"
+
+#define GLUSTERD_STORE_KEY_VOL_CAPS             "caps"
 
 #define glusterd_for_each_entry(entry, dir) \
         do {\
@@ -76,21 +91,13 @@ typedef enum glusterd_store_ver_ac_{
                 if (dir) {\
                         entry = readdir (dir);\
                         while (entry && (!strcmp (entry->d_name, ".") ||\
+                            !fnmatch ("*.tmp", entry->d_name, 0) ||\
                             !strcmp (entry->d_name, ".."))) {\
                                 entry = readdir (dir);\
                         }\
                 }\
         } while (0); \
 
-typedef enum {
-        GD_STORE_SUCCESS,
-        GD_STORE_KEY_NULL,
-        GD_STORE_VALUE_NULL,
-        GD_STORE_KEY_VALUE_NULL,
-        GD_STORE_EOF,
-        GD_STORE_ENOMEM,
-        GD_STORE_STAT_FAILED
-} glusterd_store_op_errno_t;
 
 int32_t
 glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac);
@@ -99,17 +106,7 @@ int32_t
 glusterd_store_delete_volume (glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_store_uuid ();
-
-int32_t
-glusterd_store_handle_new (char *path, glusterd_store_handle_t **handle);
-
-int32_t
-glusterd_store_save_value (int fd, char *key, char *value);
-
-int32_t
-glusterd_store_retrieve_value (glusterd_store_handle_t *handle,
-                               char *key, char **value);
+glusterd_store_delete_snap (glusterd_snap_t *snap);
 
 int32_t
 glusterd_retrieve_uuid ();
@@ -121,11 +118,8 @@ int32_t
 glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo);
 
 int32_t
-glusterd_store_delete_brick (glusterd_volinfo_t *volinfo,
-                             glusterd_brickinfo_t *brickinfo);
-
-int32_t
-glusterd_store_handle_destroy (glusterd_store_handle_t *handle);
+glusterd_store_delete_brick (glusterd_brickinfo_t *brickinfo,
+                             char *delete_path);
 
 int32_t
 glusterd_restore ();
@@ -135,4 +129,36 @@ glusterd_perform_volinfo_version_action (glusterd_volinfo_t *volinfo,
                                          glusterd_volinfo_ver_ac_t ac);
 gf_boolean_t
 glusterd_store_is_valid_brickpath (char *volname, char *brick);
+
+int32_t
+glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_retrieve_op_version (xlator_t *this, int *op_version);
+
+int
+glusterd_store_global_info (xlator_t *this);
+
+int32_t
+glusterd_store_retrieve_options (xlator_t *this);
+
+int32_t
+glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_store_options (xlator_t *this, dict_t *opts);
+
+void
+glusterd_replace_slash_with_hyphen (char *str);
+
+int32_t
+glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_store_snap (glusterd_snap_t *snap);
+
+int32_t
+glusterd_store_update_missed_snaps (dict_t *dict,
+                                    int32_t missed_snap_count);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
new file mode 100644
index 000000000..438df8266
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -0,0 +1,1639 @@
+/*
+   Copyright (c) 2012-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+/* rpc related syncops */
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+#include "glusterd-syncop.h"
+
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-locks.h"
+
+extern glusterd_op_info_t opinfo;
+
+void
+gd_synctask_barrier_wait (struct syncargs *args, int count)
+{
+        glusterd_conf_t *conf = THIS->private;
+
+        synclock_unlock (&conf->big_lock);
+        synctask_barrier_wait (args, count);
+        synclock_lock (&conf->big_lock);
+
+	syncbarrier_destroy (&args->barrier);
+}
+
+static void
+gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                           char *op_errstr, int op_code,
+                           glusterd_peerinfo_t *peerinfo, u_char *uuid)
+{
+        char     err_str[PATH_MAX] = "Please check log file for details.";
+        char     op_err[PATH_MAX] = "";
+        int      len               = -1;
+        char    *peer_str      = NULL;
+
+        if (op_ret) {
+                args->op_ret = op_ret;
+                args->op_errno = op_errno;
+
+                if (peerinfo)
+                        peer_str = peerinfo->hostname;
+                else
+                        peer_str = uuid_utoa (uuid);
+
+                if (op_errstr && strcmp (op_errstr, "")) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "Error: %s", op_errstr);
+                        err_str[len] = '\0';
+                }
+
+                switch (op_code){
+                        case GLUSTERD_MGMT_V3_LOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Locking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_V3_UNLOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Unlocking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                }
+                op_err[len] = '\0';
+
+                if (args->errstr) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s\n%s", args->errstr,
+                                        op_err);
+                        GF_FREE (args->errstr);
+                        args->errstr = NULL;
+                } else
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s", op_err);
+                err_str[len] = '\0';
+
+                gf_log ("", GF_LOG_ERROR, "%s", op_err);
+                args->errstr = gf_strdup (err_str);
+        }
+
+        return;
+}
+
+static void
+gd_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                   char *op_errstr, int op_code,
+                   glusterd_peerinfo_t *peerinfo, u_char *uuid)
+{
+        char     err_str[PATH_MAX] = "Please check log file for details.";
+        char     op_err[PATH_MAX] = "";
+        int      len               = -1;
+        char    *peer_str      = NULL;
+
+        if (op_ret) {
+                args->op_ret = op_ret;
+                args->op_errno = op_errno;
+
+                if (peerinfo)
+                        peer_str = peerinfo->hostname;
+                else
+                        peer_str = uuid_utoa (uuid);
+
+                if (op_errstr && strcmp (op_errstr, "")) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "Error: %s", op_errstr);
+                        err_str[len] = '\0';
+                }
+
+                switch (op_code){
+                        case GLUSTERD_MGMT_CLUSTER_UNLOCK :
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Unlocking failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_STAGE_OP :
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Staging failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_COMMIT_OP :
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Commit failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                }
+                op_err[len] = '\0';
+
+                if (args->errstr) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s\n%s", args->errstr,
+                                        op_err);
+                        GF_FREE (args->errstr);
+                        args->errstr = NULL;
+                } else
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s", op_err);
+                err_str[len] = '\0';
+
+                gf_log ("", GF_LOG_ERROR, "%s", op_err);
+                args->errstr = gf_strdup (err_str);
+        }
+
+        return;
+}
+
+void
+gd_syncargs_init (struct syncargs *args, dict_t *op_ctx)
+{
+        args->dict = op_ctx;
+        pthread_mutex_init (&args->lock_dict, NULL);
+}
+
+static void
+gd_stage_op_req_free (gd1_mgmt_stage_op_req *req)
+{
+        if (!req)
+                return;
+
+        GF_FREE (req->buf.buf_val);
+        GF_FREE (req);
+}
+
+static void
+gd_commit_op_req_free (gd1_mgmt_commit_op_req *req)
+{
+        if (!req)
+                return;
+
+        GF_FREE (req->buf.buf_val);
+        GF_FREE (req);
+}
+
+static void
+gd_brick_op_req_free (gd1_mgmt_brick_op_req *req)
+{
+        if (!req)
+                return;
+
+        if (strcmp (req->name, "") != 0)
+                GF_FREE (req->name);
+        GF_FREE (req->input.input_val);
+        GF_FREE (req);
+}
+
+int
+gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local,
+                          void *cookie, rpc_clnt_prog_t *prog, int procnum,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+        int            ret      = -1;
+        struct iobuf  *iobuf    = NULL;
+        struct iobref *iobref   = NULL;
+        int            count    = 0;
+        struct iovec   iov      = {0, };
+        ssize_t        req_size = 0;
+        call_frame_t  *frame    = NULL;
+
+        GF_ASSERT (rpc);
+        if (!req)
+                goto out;
+
+        req_size = xdr_sizeof (xdrproc, req);
+        iobuf = iobuf_get2 (rpc->ctx->iobuf_pool, req_size);
+        if (!iobuf)
+                goto out;
+
+        iobref = iobref_new ();
+        if (!iobref)
+                goto out;
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto out;
+
+        iobref_add (iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len  = iobuf_pagesize (iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic (iov, req, xdrproc);
+        if (ret == -1)
+                goto out;
+
+        iov.iov_len = ret;
+        count = 1;
+
+        frame->local = local;
+        frame->cookie = cookie;
+
+        /* Send the msg */
+        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn,
+                               &iov, count, NULL, 0, iobref,
+                               frame, NULL, 0, NULL, 0, NULL);
+
+        /* TODO: do we need to start ping also? */
+
+out:
+        iobref_unref (iobref);
+        iobuf_unref (iobuf);
+
+        return ret;
+}
+
+/* Defined in glusterd-rpc-ops.c */
+extern struct rpc_clnt_program gd_mgmt_prog;
+extern struct rpc_clnt_program gd_brick_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+
+int
+glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp)
+{
+        int ret = 0;
+
+        switch (op) {
+        case GD_OP_REPLACE_BRICK:
+                ret = glusterd_rb_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_SYNC_VOLUME:
+                ret = glusterd_sync_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_PROFILE_VOLUME:
+                ret = glusterd_profile_volume_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_GSYNC_CREATE:
+        break;
+
+        case GD_OP_GSYNC_SET:
+                ret = glusterd_gsync_use_rsp_dict (aggr, rsp, NULL);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_STATUS_VOLUME:
+                ret = glusterd_volume_status_copy_to_op_ctx_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = glusterd_volume_rebalance_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_HEAL_VOLUME:
+                ret = glusterd_volume_heal_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+
+        break;
+
+        case GD_OP_QUOTA:
+        case GD_OP_CLEARLOCKS_VOLUME:
+                ret = glusterd_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+
+        break;
+
+        case GD_OP_SYS_EXEC:
+                ret = glusterd_sys_exec_output_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_SNAP:
+                ret = glusterd_snap_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        default:
+        break;
+        }
+out:
+        return ret;
+}
+
+int32_t
+_gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_cluster_lock_rsp   rsp         = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        /* Set peer as locked, so we unlock only the locked peers */
+        if (rsp.op_ret == 0)
+                peerinfo->locked = _gf_true;
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_collate_errors (args, op_ret, op_errno, NULL,
+                           GLUSTERD_MGMT_CLUSTER_LOCK, peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_mgmt_lock_cbk);
+}
+
+int
+gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                     uuid_t my_uuid, uuid_t recv_uuid)
+{
+        int                       ret = -1;
+        gd1_mgmt_cluster_lock_req req  = {{0},};
+        glusterd_conf_t           *conf = THIS->private;
+
+        uuid_copy (req.uuid, my_uuid);
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_CLUSTER_LOCK,
+                                        gd_syncop_mgmt_lock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
+        synclock_lock (&conf->big_lock);
+        return ret;
+}
+
+int32_t
+gd_syncop_mgmt_v3_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                               int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_v3_lock_rsp         rsp        = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        GF_ASSERT(req);
+        GF_ASSERT(iov);
+        GF_ASSERT(myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_LOCK,
+                                   peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_v3_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_syncop_mgmt_v3_lock_cbk_fn);
+}
+
+int
+gd_syncop_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
+                        glusterd_peerinfo_t *peerinfo,
+                        struct syncargs *args, uuid_t my_uuid,
+                        uuid_t recv_uuid, uuid_t txn_id)
+{
+        int                      ret  = -1;
+        gd1_mgmt_v3_lock_req     req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+
+        GF_ASSERT(op_ctx);
+        GF_ASSERT(peerinfo);
+        GF_ASSERT(args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        uuid_copy (req.txn_id, txn_id);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_LOCK,
+                                        gd_syncop_mgmt_v3_lock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_lock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+gd_syncop_mgmt_v3_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_v3_unlock_rsp       rsp        = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        GF_ASSERT(req);
+        GF_ASSERT(iov);
+        GF_ASSERT(myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        /* Set peer as locked, so we unlock only the locked peers */
+        if (rsp.op_ret == 0)
+                peerinfo->locked = _gf_true;
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_UNLOCK,
+                                   peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_v3_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_syncop_mgmt_v3_unlock_cbk_fn);
+}
+
+int
+gd_syncop_mgmt_v3_unlock (dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
+                          struct syncargs *args, uuid_t my_uuid,
+                          uuid_t recv_uuid, uuid_t txn_id)
+{
+        int                          ret  = -1;
+        gd1_mgmt_v3_unlock_req       req  = {{0},};
+        glusterd_conf_t             *conf = THIS->private;
+
+        GF_ASSERT(op_ctx);
+        GF_ASSERT(peerinfo);
+        GF_ASSERT(args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        uuid_copy (req.txn_id, txn_id);
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_UNLOCK,
+                                        gd_syncop_mgmt_v3_unlock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_unlock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+_gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_cluster_unlock_rsp rsp         = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        frame = myframe;
+        args  = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        peerinfo->locked = _gf_false;
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_collate_errors (args, op_ret, op_errno, NULL,
+                           GLUSTERD_MGMT_CLUSTER_UNLOCK, peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                           int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_mgmt_unlock_cbk);
+}
+
+
+int
+gd_syncop_mgmt_unlock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                       uuid_t my_uuid, uuid_t recv_uuid)
+{
+        int                         ret     = -1;
+        gd1_mgmt_cluster_unlock_req req     = {{0},};
+        glusterd_conf_t             *conf   = THIS->private;
+
+        uuid_copy (req.uuid, my_uuid);
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_CLUSTER_UNLOCK,
+                                        gd_syncop_mgmt_unlock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
+        synclock_lock (&conf->big_lock);
+        return ret;
+}
+
+int32_t
+_gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        int                         ret         = -1;
+        gd1_mgmt_stage_op_rsp       rsp         = {{0},};
+        struct syncargs             *args       = NULL;
+        xlator_t                    *this       = NULL;
+        dict_t                      *rsp_dict   = NULL;
+        call_frame_t                *frame      = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        this  = THIS;
+        frame = myframe;
+        args  = frame->local;
+        frame->local = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Staging response "
+                        "for 'Volume %s' received from unknown "
+                        "peer: %s", gd_op_list[rsp.op],
+                        uuid_utoa (rsp.uuid));
+                goto out;
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        if (rsp.op == GD_OP_REPLACE_BRICK) {
+                pthread_mutex_lock (&args->lock_dict);
+                {
+                        ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
+                                                             rsp_dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "%s",
+                                        "Failed to aggregate response from "
+                                        " node/brick");
+                }
+                pthread_mutex_unlock (&args->lock_dict);
+        }
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
+                           GLUSTERD_MGMT_STAGE_OP, peerinfo, rsp.uuid);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_stage_op_cbk);
+}
+
+
+int
+gd_syncop_mgmt_stage_op (struct rpc_clnt *rpc, struct syncargs *args,
+                         uuid_t my_uuid, uuid_t recv_uuid, int op,
+                         dict_t *dict_out, dict_t *op_ctx)
+{
+        gd1_mgmt_stage_op_req *req  = NULL;
+        glusterd_conf_t       *conf = THIS->private;
+        int                   ret   = -1;
+
+        req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_stage_req_t);
+        if (!req)
+                goto out;
+
+        uuid_copy (req->uuid, my_uuid);
+        req->op = op;
+
+        ret = dict_allocate_and_serialize (dict_out,
+                                           &req->buf.buf_val, &req->buf.buf_len);
+        if (ret)
+                goto out;
+
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (rpc, req, args, NULL, &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_STAGE_OP,
+                                        gd_syncop_stage_op_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_stage_op_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gd_stage_op_req_free (req);
+        return ret;
+
+}
+
+int32_t
+_gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        struct syncargs        *args  = NULL;
+        gd1_mgmt_brick_op_rsp  rsp   = {0,};
+        int                    ret   = -1;
+        call_frame_t           *frame = NULL;
+
+        frame = myframe;
+        args = frame->local;
+        frame->local = NULL;
+
+        /* initialize */
+        args->op_ret   = -1;
+        args->op_errno = EINVAL;
+
+        if (-1 == req->rpc_status) {
+                args->op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.output.output_len) {
+                args->dict  = dict_new ();
+                if (!args->dict) {
+                        ret = -1;
+                        args->op_errno = ENOMEM;
+                        goto out;
+                }
+
+                ret = dict_unserialize (rsp.output.output_val,
+                                        rsp.output.output_len,
+                                        &args->dict);
+                if (ret < 0)
+                        goto out;
+        }
+
+        args->op_ret = rsp.op_ret;
+        args->op_errno = rsp.op_errno;
+        args->errstr = gf_strdup (rsp.op_errstr);
+
+out:
+        if ((rsp.op_errstr) && (strcmp (rsp.op_errstr, "") != 0))
+                free (rsp.op_errstr);
+        free (rsp.output.output_val);
+
+        STACK_DESTROY (frame->root);
+        __wake (args);
+
+        return 0;
+}
+
+int32_t
+gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_brick_op_cbk);
+}
+
+int
+gd_syncop_mgmt_brick_op (struct rpc_clnt *rpc, glusterd_pending_node_t *pnode,
+                         int op, dict_t *dict_out, dict_t *op_ctx,
+                         char **errstr)
+{
+        struct syncargs        args = {0, };
+        gd1_mgmt_brick_op_req  *req  = NULL;
+        int                    ret  = 0;
+        xlator_t               *this = NULL;
+
+        this = THIS;
+        args.op_ret = -1;
+        args.op_errno = ENOTCONN;
+
+        if ((pnode->type == GD_NODE_NFS) ||
+            ((pnode->type == GD_NODE_SHD) &&
+            (op == GD_OP_STATUS_VOLUME))) {
+                ret = glusterd_node_op_build_payload
+                        (op, &req, dict_out);
+
+        } else {
+                ret = glusterd_brick_op_build_payload
+                        (op, pnode->node, &req, dict_out);
+
+        }
+
+        if (ret)
+                goto out;
+
+        GD_SYNCOP (rpc, (&args), NULL, gd_syncop_brick_op_cbk, req,
+                   &gd_brick_prog, req->op, xdr_gd1_mgmt_brick_op_req);
+
+        if (args.errstr) {
+                if ((strlen(args.errstr) > 0) && errstr)
+                        *errstr = args.errstr;
+                else
+                        GF_FREE (args.errstr);
+        }
+
+        if (GD_OP_STATUS_VOLUME == op) {
+                ret = dict_set_int32 (args.dict, "index", pnode->index);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting index on brick status"
+                                " rsp dict");
+                        args.op_ret = -1;
+                        goto out;
+                }
+        }
+        if (args.op_ret == 0)
+                glusterd_handle_node_rsp (dict_out, pnode->node, op,
+                                          args.dict, op_ctx, errstr,
+                                          pnode->type);
+
+out:
+        errno = args.op_errno;
+        if (args.dict)
+                dict_unref (args.dict);
+        gd_brick_op_req_free (req);
+        return args.op_ret;
+
+}
+
+int32_t
+_gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int                          ret           = -1;
+        gd1_mgmt_commit_op_rsp       rsp           = {{0},};
+        struct syncargs             *args          = NULL;
+        xlator_t                    *this          = NULL;
+        dict_t                      *rsp_dict      = NULL;
+        call_frame_t                *frame         = NULL;
+        glusterd_peerinfo_t         *peerinfo      = NULL;
+        int                          op_ret        = -1;
+        int                          op_errno      = -1;
+
+        this  = THIS;
+        frame = myframe;
+        args  = frame->local;
+        frame->local = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+        if (ret < 0) {
+                goto out;
+        }
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Commit response "
+                        "for 'Volume %s' received from unknown "
+                        "peer: %s", gd_op_list[rsp.op],
+                        uuid_utoa (rsp.uuid));
+                goto out;
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        pthread_mutex_lock (&args->lock_dict);
+        {
+                ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
+                                                     rsp_dict);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                "Failed to aggregate response from "
+                                " node/brick");
+        }
+        pthread_mutex_unlock (&args->lock_dict);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
+                           GLUSTERD_MGMT_COMMIT_OP, peerinfo, rsp.uuid);
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+
+        return 0;
+}
+
+int32_t
+gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_commit_op_cbk);
+}
+
+
+int
+gd_syncop_mgmt_commit_op (struct rpc_clnt *rpc, struct syncargs *args,
+                          uuid_t my_uuid, uuid_t recv_uuid,
+                          int op, dict_t *dict_out, dict_t *op_ctx)
+{
+        glusterd_conf_t        *conf = THIS->private;
+        gd1_mgmt_commit_op_req *req  = NULL;
+        int                    ret  = -1;
+
+        req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_commit_req_t);
+        if (!req)
+                goto out;
+
+        uuid_copy (req->uuid, my_uuid);
+        req->op = op;
+
+        ret = dict_allocate_and_serialize (dict_out,
+                                           &req->buf.buf_val, &req->buf.buf_len);
+        if (ret)
+                goto out;
+
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (rpc, req, args, NULL, &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_COMMIT_OP ,
+                                        gd_syncop_commit_op_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_commit_op_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gd_commit_op_req_free (req);
+        return ret;
+}
+
+
+int
+gd_build_peers_list (struct list_head *peers, struct list_head *xact_peers,
+                     glusterd_op_t op)
+{
+        glusterd_peerinfo_t *peerinfo = NULL;
+        int                 npeers      = 0;
+
+        list_for_each_entry (peerinfo, peers, uuid_list) {
+                if (!peerinfo->connected)
+                        continue;
+                if (op != GD_OP_SYNC_VOLUME &&
+                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+                        continue;
+
+                list_add_tail (&peerinfo->op_peers_list, xact_peers);
+                npeers++;
+        }
+        return npeers;
+}
+
+int
+gd_lock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, dict_t *op_ctx,
+                  char **op_errstr, int npeers, uuid_t txn_id)
+{
+        int                 ret         = -1;
+        int                 peer_cnt    = 0;
+        uuid_t              peer_uuid   = {0};
+        xlator_t            *this       = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs     args        = {0};
+        struct list_head   *peers       = NULL;
+
+        peers = &conf->xaction_peers;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        this = THIS;
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                if (conf->op_version < 3) {
+                        /* Reset lock status */
+                        peerinfo->locked = _gf_false;
+                        gd_syncop_mgmt_lock (peerinfo, &args,
+                                             MY_UUID, peer_uuid);
+                } else
+                        gd_syncop_mgmt_v3_lock (op, op_ctx, peerinfo, &args,
+                                                MY_UUID, peer_uuid, txn_id);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+                else {
+                        ret = gf_asprintf (op_errstr, "Another transaction could be "
+                                           "in progress. Please try again after "
+                                           "sometime.");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire lock");
+
+                }
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent lock op req for 'Volume %s' "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int
+gd_stage_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
+                   dict_t *req_dict, char **op_errstr, int npeers)
+{
+        int                 ret             = -1;
+        int                 peer_cnt           = 0;
+        dict_t              *rsp_dict       = NULL;
+        char                *hostname       = NULL;
+        xlator_t            *this           = NULL;
+        glusterd_peerinfo_t *peerinfo       = NULL;
+        uuid_t              tmp_uuid        = {0};
+        char                *errstr         = NULL;
+        struct syncargs     args            = {0};
+
+        this = THIS;
+        rsp_dict = dict_new ();
+        if (!rsp_dict)
+                goto out;
+
+        ret = glusterd_op_stage_validate (op, req_dict, op_errstr, rsp_dict);
+        if (ret) {
+                hostname = "localhost";
+                goto stage_done;
+        }
+
+        if ((op == GD_OP_REPLACE_BRICK)) {
+                ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                "Failed to aggregate response from node/brick");
+                        goto out;
+                }
+        }
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+stage_done:
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_STAGE_FAIL,
+                        gd_op_list[op], hostname, (*op_errstr) ? ":" : " ",
+                        (*op_errstr) ? *op_errstr : " ");
+                if (*op_errstr == NULL)
+                        gf_asprintf (op_errstr, OPERRSTR_STAGE_FAIL, hostname);
+                goto out;
+        }
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        gd_syncargs_init (&args, op_ctx);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                ret = gd_syncop_mgmt_stage_op (peerinfo->rpc, &args,
+                                               MY_UUID, tmp_uuid,
+                                               op, req_dict, op_ctx);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.errstr)
+                 *op_errstr = gf_strdup (args.errstr);
+        else if (dict_get_str (op_ctx, "errstr", &errstr) == 0)
+                *op_errstr = gf_strdup (errstr);
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent stage op req for 'Volume %s' "
+                "to %d peers", gd_op_list[op], peer_cnt);
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+        return ret;
+}
+
+int
+gd_commit_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
+                    dict_t *req_dict, char **op_errstr, int npeers)
+{
+        dict_t              *rsp_dict       = NULL;
+        int                 peer_cnt           = -1;
+        int                 ret             = -1;
+        char                *hostname       = NULL;
+        glusterd_peerinfo_t *peerinfo       = NULL;
+        xlator_t            *this           = NULL;
+        uuid_t              tmp_uuid        = {0};
+        char                *errstr         = NULL;
+        struct syncargs     args            = {0};
+
+        this = THIS;
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_op_commit_perform (op, req_dict, op_errstr, rsp_dict);
+        if (ret) {
+                hostname = "localhost";
+                goto commit_done;
+        }
+        if (op != GD_OP_SYNC_VOLUME) {
+                ret =  glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                "Failed to aggregate response "
+                                "from node/brick");
+                        goto out;
+                }
+        }
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+commit_done:
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_COMMIT_FAIL,
+                        gd_op_list[op], hostname, (*op_errstr) ? ":" : " ",
+                        (*op_errstr) ? *op_errstr : " ");
+                if (*op_errstr == NULL)
+                        gf_asprintf (op_errstr, OPERRSTR_COMMIT_FAIL,
+                                     hostname);
+                goto out;
+        }
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        gd_syncargs_init (&args, op_ctx);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                ret = gd_syncop_mgmt_commit_op (peerinfo->rpc, &args,
+                                                MY_UUID, tmp_uuid,
+                                                op, req_dict, op_ctx);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+        ret = args.op_ret;
+        if (args.errstr)
+                 *op_errstr = gf_strdup (args.errstr);
+        else if (dict_get_str (op_ctx, "errstr", &errstr) == 0)
+                *op_errstr = gf_strdup (errstr);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent commit op req for 'Volume %s' "
+                "to %d peers", gd_op_list[op], peer_cnt);
+out:
+        if (!ret)
+                glusterd_op_modify_op_ctx (op, op_ctx);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        GF_FREE (args.errstr);
+        args.errstr = NULL;
+
+        return ret;
+}
+
+int
+gd_unlock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, int op_ret,
+                    rpcsvc_request_t *req, dict_t *op_ctx, char *op_errstr,
+                    int npeers, char *volname, gf_boolean_t is_acquired,
+                    uuid_t txn_id)
+{
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        glusterd_peerinfo_t *tmp        = NULL;
+        uuid_t              tmp_uuid    = {0};
+        int                 peer_cnt       = 0;
+        int                 ret         = -1;
+        xlator_t            *this       = NULL;
+        struct syncargs     args        = {0};
+        struct list_head   *peers       = NULL;
+
+        peers = &conf->xaction_peers;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* If the lock has not been held during this
+         * transaction, do not send unlock requests */
+        if (!is_acquired)
+                goto out;
+
+        this = THIS;
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry_safe (peerinfo, tmp, peers, op_peers_list) {
+                if (conf->op_version < 3) {
+                        /* Only unlock peers that were locked */
+                        if (peerinfo->locked)
+                                gd_syncop_mgmt_unlock (peerinfo, &args,
+                                                       MY_UUID, tmp_uuid);
+                } else
+                        gd_syncop_mgmt_v3_unlock (op_ctx, peerinfo,
+                                                  &args, MY_UUID,
+                                                  tmp_uuid, txn_id);
+                peer_cnt++;
+                list_del_init (&peerinfo->op_peers_list);
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+        ret = args.op_ret;
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to unlock "
+                        "on some peer(s)");
+        }
+
+out:
+        glusterd_op_send_cli_response (op, op_ret, 0, req, op_ctx, op_errstr);
+        glusterd_op_clear_op (op);
+        if (is_acquired) {
+                /* Based on the op-version, we release *
+                 * the cluster or mgmt_v3 lock */
+                if (conf->op_version < 3)
+                        glusterd_unlock (MY_UUID);
+                else {
+                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                }
+        }
+
+        return 0;
+}
+
+int
+gd_get_brick_count (struct list_head *bricks)
+{
+        glusterd_pending_node_t *pending_node = NULL;
+        int                     npeers        = 0;
+        list_for_each_entry (pending_node, bricks, list) {
+                npeers++;
+        }
+        return npeers;
+}
+
+int
+gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                   char **op_errstr)
+{
+        glusterd_pending_node_t *pending_node = NULL;
+        struct list_head        selected = {0,};
+        xlator_t                *this = NULL;
+        int                     brick_count = 0;
+        int                     ret = -1;
+        rpc_clnt_t              *rpc = NULL;
+        dict_t                  *rsp_dict = NULL;
+        glusterd_conf_t         *conf = NULL;
+
+        this = THIS;
+        conf = this->private;
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&selected);
+        ret = glusterd_op_bricks_select (op, req_dict, op_errstr, &selected, rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                       (*op_errstr)? *op_errstr: "Brick op failed. Check "
+                       "glusterd log file for more details.");
+                goto out;
+        }
+
+        if (op == GD_OP_HEAL_VOLUME) {
+                ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict);
+                if (ret)
+                        goto out;
+        }
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        brick_count = 0;
+        list_for_each_entry (pending_node, &selected, list) {
+                rpc = glusterd_pending_node_get_rpc (pending_node);
+                if (!rpc) {
+                        if (pending_node->type == GD_NODE_REBALANCE) {
+                                ret = 0;
+                                glusterd_defrag_volume_node_rsp (req_dict,
+                                                                 NULL, op_ctx);
+                                goto out;
+                        }
+
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR, "Brick Op failed "
+                                "due to rpc failure.");
+                        goto out;
+                }
+                ret = gd_syncop_mgmt_brick_op (rpc, pending_node, op, req_dict,
+                                               op_ctx, op_errstr);
+                if (ret)
+                        goto out;
+
+                brick_count++;
+        }
+
+        ret = 0;
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+        gf_log (this->name, GF_LOG_DEBUG, "Sent op req to %d bricks",
+                brick_count);
+        return ret;
+}
+
+void
+gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
+{
+        int                         ret              = -1;
+        int                         npeers           = 0;
+        dict_t                      *req_dict        = NULL;
+        glusterd_conf_t             *conf            = NULL;
+        glusterd_op_t               op               = 0;
+        int32_t                     tmp_op           = 0;
+        char                        *op_errstr       = NULL;
+        char                        *tmp             = NULL;
+        char                        *volname         = NULL;
+        xlator_t                    *this            = NULL;
+        gf_boolean_t                is_acquired      = _gf_false;
+        uuid_t                      *txn_id          = NULL;
+        uuid_t                      *originator_uuid = NULL;
+        glusterd_op_info_t          txn_opinfo;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        /* Generate a transaction-id for this operation and
+         * save it in the dict */
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!txn_id) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_generate (*txn_id);
+
+        ret = dict_set_bin (op_ctx, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set transaction id.");
+                goto out;
+        } else
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+
+        /* Save the MY_UUID as the originator_uuid */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (op_ctx, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, &tmp_op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume "
+                        "operation");
+                goto out;
+        }
+
+        op = tmp_op;
+
+        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+        if (conf->op_version < 3) {
+                ret = glusterd_lock (MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock");
+                        gf_asprintf (&op_errstr,
+                                     "Another transaction is in progress. "
+                                     "Please try again after sometime.");
+                        goto out;
+                }
+        } else {
+
+                /* If no volname is given as a part of the command, locks will
+                 * not be held */
+                ret = dict_get_str (op_ctx, "volname", &tmp);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Failed to get volume "
+                                "name");
+                        goto local_locking_done;
+                } else {
+                        /* Use a copy of volname, as cli response will be
+                         * sent before the unlock, and the volname in the
+                         * dict, might be removed */
+                        volname = gf_strdup (tmp);
+                        if (!volname)
+                                goto out;
+                }
+
+                ret = glusterd_mgmt_v3_lock (volname, MY_UUID, "vol");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock for %s", volname);
+                        gf_asprintf (&op_errstr,
+                                     "Another transaction is in progress "
+                                     "for %s. Please try again after sometime.",
+                                     volname);
+                        goto out;
+                }
+        }
+
+        is_acquired = _gf_true;
+
+local_locking_done:
+
+        /* Save opinfo for this transaction with the transaction id */
+        glusterd_txn_opinfo_init (&txn_opinfo, NULL, &op, NULL, NULL);
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_opinfo);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+
+        opinfo = txn_opinfo;
+
+        INIT_LIST_HEAD (&conf->xaction_peers);
+
+        /* Make 'volume status tasks' command a local operation.
+         * This is accomplished by setting npeers to 0.
+         */
+        if (!glusterd_is_status_tasks_op (op, op_ctx))
+                npeers = gd_build_peers_list  (&conf->peers,
+                                               &conf->xaction_peers, op);
+
+        /* If no volname is given as a part of the command, locks will
+         * not be held */
+        if (volname) {
+                ret = gd_lock_op_phase (conf, op, op_ctx, &op_errstr, npeers, *txn_id);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_op_build_payload (&req_dict, &op_errstr, op_ctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                goto out;
+        }
+
+        ret = gd_stage_op_phase (&conf->xaction_peers, op, op_ctx, req_dict,
+                                 &op_errstr, npeers);
+        if (ret)
+                goto out;
+
+        ret = gd_brick_op_phase (op, op_ctx, req_dict, &op_errstr);
+        if (ret)
+                goto out;
+
+        ret = gd_commit_op_phase (&conf->xaction_peers, op, op_ctx, req_dict,
+                                  &op_errstr, npeers);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        (void) gd_unlock_op_phase (conf, op, ret, req, op_ctx, op_errstr,
+                                   npeers, volname, is_acquired, *txn_id);
+
+        /* Clearing the transaction opinfo */
+        ret = glusterd_clear_txn_opinfo (txn_id);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to clear transaction's opinfo");
+
+        if (volname)
+                GF_FREE (volname);
+
+        if (req_dict)
+                dict_unref (req_dict);
+
+        if (op_errstr) {
+                GF_FREE (op_errstr);
+                op_errstr = NULL;
+        }
+
+        return;
+}
+
+int32_t
+glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
+                            void *dict)
+{
+        int              ret = 0;
+
+        ret = dict_set_int32 (dict, GD_SYNC_OPCODE_KEY, op);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "dict set failed for setting operations");
+                goto out;
+        }
+
+        gd_sync_task_begin (dict, req);
+        ret = 0;
+out:
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.h b/xlators/mgmt/glusterd/src/glusterd-syncop.h
new file mode 100644
index 000000000..e83ea2f4c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.h
@@ -0,0 +1,71 @@
+/*
+   Copyright (c) 2012-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __RPC_SYNCOP_H
+#define __RPC_SYNCOP_H
+
+#include "syncop.h"
+#include "glusterd-sm.h"
+#include "glusterd.h"
+
+#define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
+
+/* gd_syncop_* */
+#define GD_SYNCOP(rpc, stb, cookie, cbk, req, prog, procnum, xdrproc) do {     \
+                int ret = 0;                                                   \
+                struct  synctask        *task = NULL;                          \
+                glusterd_conf_t         *conf= THIS->private;                  \
+                                                                               \
+                task = synctask_get ();                                        \
+                stb->task = task;                                              \
+                                                                               \
+                /*This is to ensure that the brick_op_cbk is able to           \
+                 * take the big lock*/                                         \
+                synclock_unlock (&conf->big_lock);                             \
+                ret = gd_syncop_submit_request (rpc, req, stb, cookie,         \
+                                                prog, procnum, cbk,            \
+                                                (xdrproc_t)xdrproc);           \
+                if (!ret)                                                      \
+                        synctask_yield (stb->task);                            \
+                synclock_lock (&conf->big_lock);                               \
+        } while (0)
+
+
+int gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local,
+                              void *cookie, rpc_clnt_prog_t *prog, int procnum,
+                              fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+
+
+int gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
+                         uuid_t my_uuid, uuid_t recv_uuid);
+int gd_syncop_mgmt_unlock (glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
+                           uuid_t my_uuid, uuid_t recv_uuid);
+int gd_syncop_mgmt_stage_op (struct rpc_clnt *rpc, struct syncargs *arg,
+                             uuid_t my_uuid, uuid_t recv_uuid, int op,
+                             dict_t *dict_out, dict_t *op_ctx);
+int gd_syncop_mgmt_commit_op (struct rpc_clnt *rpc, struct syncargs *arg,
+                              uuid_t my_uuid, uuid_t recv_uuid, int op,
+                              dict_t *dict_out, dict_t *op_ctx);
+
+void
+gd_synctask_barrier_wait (struct syncargs *args, int count);
+
+int
+gd_build_peers_list (struct list_head *peers, struct list_head *xact_peers,
+                     glusterd_op_t op);
+int
+gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                   char **op_errstr);
+
+int
+glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp);
+
+void
+gd_syncargs_init (struct syncargs *args, dict_t *op_ctx);
+#endif /* __RPC_SYNCOP_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index a71258250..e8ae05851 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -32,7 +22,7 @@
 #include "timer.h"
 #include "defaults.h"
 #include "compat.h"
-#include "md5.h"
+#include "syncop.h"
 #include "run.h"
 #include "compat-errno.h"
 #include "statedump.h"
@@ -45,9 +35,10 @@
 #include "glusterd-store.h"
 #include "glusterd-volgen.h"
 #include "glusterd-pmap.h"
+#include "glusterfs-acl.h"
+#include "glusterd-locks.h"
 
 #include "xdr-generic.h"
-
 #include <sys/resource.h>
 #include <inttypes.h>
 #include <signal.h>
@@ -59,21 +50,48 @@
 #include <unistd.h>
 #include <fnmatch.h>
 #include <sys/statvfs.h>
+#include <ifaddrs.h>
+#ifdef HAVE_BD_XLATOR
+#include <lvm2app.h>
+#endif
+
 
+#ifdef GF_LINUX_HOST_OS
+#include <mntent.h>
+#endif
 
 #ifdef GF_SOLARIS_HOST_OS
 #include <sys/sockio.h>
 #endif
 
-#define MOUNT_PROGRAM 100005
-#define NFS_PROGRAM 100003
-#define NFSV3_VERSION 3
-#define MOUNTV3_VERSION 3
-#define MOUNTV1_VERSION 1
+#define NFS_PROGRAM         100003
+#define NFSV3_VERSION       3
+
+#define MOUNT_PROGRAM       100005
+#define MOUNTV3_VERSION     3
+#define MOUNTV1_VERSION     1
+
+#define NLM_PROGRAM         100021
+#define NLMV4_VERSION       4
+#define NLMV1_VERSION       1
+
+#define CEILING_POS(X) (((X)-(int)(X)) > 0 ? (int)((X)+1) : (int)(X))
 
-char    *glusterd_sock_dir = "/tmp";
 static glusterd_lock_t lock;
 
+char*
+gd_peer_uuid_str (glusterd_peerinfo_t *peerinfo)
+{
+        if ((peerinfo == NULL) || uuid_is_null (peerinfo->uuid))
+                return NULL;
+
+        if (peerinfo->uuid_str[0] == '\0')
+                uuid_utoa_r (peerinfo->uuid, peerinfo->uuid_str);
+
+        return peerinfo->uuid_str;
+}
+
+
 int32_t
 glusterd_get_lock_owner (uuid_t *uuid)
 {
@@ -111,129 +129,6 @@ glusterd_is_fuse_available ()
                 return _gf_false;
 }
 
-gf_boolean_t
-glusterd_is_loopback_localhost (const struct sockaddr *sa, char *hostname)
-{
-        GF_ASSERT (sa);
-
-        gf_boolean_t is_local = _gf_false;
-        const struct in_addr *addr4 = NULL;
-        const struct in6_addr *addr6 = NULL;
-        uint8_t      *ap   = NULL;
-        struct in6_addr loopbackaddr6 = IN6ADDR_LOOPBACK_INIT;
-
-        switch (sa->sa_family) {
-                case AF_INET:
-                        addr4 = &(((struct sockaddr_in *)sa)->sin_addr);
-                        ap = (uint8_t*)&addr4->s_addr;
-                        if (ap[0] == 127)
-                                is_local = _gf_true;
-                        break;
-
-                case AF_INET6:
-                        addr6 = &(((struct sockaddr_in6 *)sa)->sin6_addr);
-                        if (memcmp (addr6, &loopbackaddr6,
-                                    sizeof (loopbackaddr6)) == 0)
-                                is_local = _gf_true;
-                        break;
-
-                default:
-                        if (hostname)
-                                gf_log ("glusterd", GF_LOG_ERROR,
-                                        "unknown address family %d for %s",
-                                        sa->sa_family, hostname);
-                        break;
-        }
-
-        return is_local;
-}
-
-char *
-get_ip_from_addrinfo (struct addrinfo *addr, char **ip)
-{
-        char buf[64];
-        void *in_addr = NULL;
-        struct sockaddr_in *s4 = NULL;
-        struct sockaddr_in6 *s6 = NULL;
-
-        switch (addr->ai_family)
-        {
-                case AF_INET:
-                        s4 = (struct sockaddr_in *)addr->ai_addr;
-                        in_addr = &s4->sin_addr;
-                        break;
-
-                case AF_INET6:
-                        s6 = (struct sockaddr_in6 *)addr->ai_addr;
-                        in_addr = &s6->sin6_addr;
-                        break;
-
-                default:
-                        gf_log ("glusterd", GF_LOG_ERROR, "Invalid family");
-                        return NULL;
-        }
-
-        if (!inet_ntop(addr->ai_family, in_addr, buf, sizeof(buf))) {
-                gf_log ("glusterd", GF_LOG_ERROR, "String conversion failed");
-                return NULL;
-        }
-
-        *ip = strdup (buf);
-        return *ip;
-}
-
-int32_t
-glusterd_is_local_addr (char *hostname)
-{
-        int32_t         ret = -1;
-        struct          addrinfo *result = NULL;
-        struct          addrinfo *res = NULL;
-        int32_t         found = 0;
-        int             sd = -1;
-        char            *ip = NULL;
-
-        ret = getaddrinfo (hostname, NULL, NULL, &result);
-
-        if (ret != 0) {
-                gf_log ("", GF_LOG_ERROR, "error in getaddrinfo: %s\n",
-                        gai_strerror(ret));
-                goto out;
-        }
-
-        for (res = result; res != NULL; res = res->ai_next) {
-                found = glusterd_is_loopback_localhost (res->ai_addr, hostname);
-                if (found)
-                        goto out;
-        }
-
-        for (res = result; res != NULL; res = res->ai_next) {
-                gf_log ("glusterd", GF_LOG_DEBUG, "%s ", get_ip_from_addrinfo (res, &ip));
-                sd = socket (res->ai_family, SOCK_DGRAM, 0);
-                if (sd == -1)
-                        goto out;
-                /*If bind succeeds then its a local address*/
-                ret = bind (sd, res->ai_addr, res->ai_addrlen);
-                if (ret == 0) {
-                        found = _gf_true;
-                        gf_log ("glusterd", GF_LOG_INFO, "%s is local", get_ip_from_addrinfo (res, &ip));
-                        close (sd);
-                        break;
-                }
-                close (sd);
-        }
-
-out:
-        if (result)
-                freeaddrinfo (result);
-
-        if (found)
-                gf_log ("glusterd", GF_LOG_DEBUG, "%s is local", hostname);
-        else
-                gf_log ("glusterd", GF_LOG_DEBUG, "%s is not local", hostname);
-
-        return !found;
-}
-
 int32_t
 glusterd_lock (uuid_t   uuid)
 {
@@ -242,13 +137,17 @@ glusterd_lock (uuid_t   uuid)
         char    new_owner_str[50];
         char    owner_str[50];
         int     ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (uuid);
 
         glusterd_get_lock_owner (&owner);
 
         if (!uuid_is_null (owner)) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get lock"
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get lock"
                         " for uuid: %s, lock held by: %s",
                         uuid_utoa_r (uuid, new_owner_str),
                         uuid_utoa_r (owner, owner_str));
@@ -258,7 +157,7 @@ glusterd_lock (uuid_t   uuid)
         ret = glusterd_set_lock_owner (uuid);
 
         if (!ret) {
-                gf_log ("glusterd", GF_LOG_INFO, "Cluster lock held by"
+                gf_log (this->name, GF_LOG_DEBUG, "Cluster lock held by"
                          " %s", uuid_utoa (uuid));
         }
 
@@ -274,21 +173,25 @@ glusterd_unlock (uuid_t uuid)
         char    new_owner_str[50];
         char    owner_str[50];
         int32_t ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (uuid);
 
         glusterd_get_lock_owner (&owner);
 
-        if (NULL == owner) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Cluster lock not held!");
+        if (uuid_is_null (owner)) {
+                gf_log (this->name, GF_LOG_ERROR, "Cluster lock not held!");
                 goto out;
         }
 
         ret = uuid_compare (uuid, owner);
 
         if (ret) {
-               gf_log ("glusterd", GF_LOG_ERROR, "Cluster lock held by %s"
-                        " ,unlock req from %s!", uuid_utoa_r (owner ,owner_str)
+               gf_log (this->name, GF_LOG_ERROR, "Cluster lock held by %s ,"
+                       "unlock req from %s!", uuid_utoa_r (owner ,owner_str)
                         , uuid_utoa_r (uuid, new_owner_str));
                goto out;
         }
@@ -296,7 +199,7 @@ glusterd_unlock (uuid_t uuid)
         ret = glusterd_unset_lock_owner (uuid);
 
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to clear cluster "
+                gf_log (this->name, GF_LOG_ERROR, "Unable to clear cluster "
                         "lock");
                 goto out;
         }
@@ -317,7 +220,7 @@ glusterd_get_uuid (uuid_t *uuid)
 
         GF_ASSERT (priv);
 
-        uuid_copy (*uuid, priv->uuid);
+        uuid_copy (*uuid, MY_UUID);
 
         return 0;
 }
@@ -510,7 +413,7 @@ glusterd_check_volume_exists (char *volname)
         ret = stat (pathname, &stbuf);
 
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Volume %s does not exist."
+                gf_log (THIS->name, GF_LOG_DEBUG, "Volume %s does not exist."
                         "stat failed with errno : %d on path: %s",
                         volname, errno, pathname);
                 return _gf_false;
@@ -533,25 +436,30 @@ glusterd_volinfo_new (glusterd_volinfo_t **volinfo)
         if (!new_volinfo)
                 goto out;
 
+        LOCK_INIT (&new_volinfo->lock);
         INIT_LIST_HEAD (&new_volinfo->vol_list);
+        INIT_LIST_HEAD (&new_volinfo->snapvol_list);
         INIT_LIST_HEAD (&new_volinfo->bricks);
+        INIT_LIST_HEAD (&new_volinfo->snap_volumes);
 
         new_volinfo->dict = dict_new ();
         if (!new_volinfo->dict) {
-                if (new_volinfo)
-                        GF_FREE (new_volinfo);
+                GF_FREE (new_volinfo);
 
                 goto out;
         }
 
         new_volinfo->gsync_slaves = dict_new ();
         if (!new_volinfo->gsync_slaves) {
-                if (new_volinfo)
-                        GF_FREE (new_volinfo);
+                GF_FREE (new_volinfo);
 
                 goto out;
         }
 
+        snprintf (new_volinfo->parent_volname, GLUSTERD_MAX_VOLUME_NAME, "N/A");
+
+        new_volinfo->snap_max_hard_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+
         new_volinfo->xl = THIS;
 
         *volinfo = new_volinfo;
@@ -559,10 +467,269 @@ glusterd_volinfo_new (glusterd_volinfo_t **volinfo)
         ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* This function will create a new volinfo and then
+ * dup the entries from volinfo to the new_volinfo.
+ *
+ * @param volinfo       volinfo which will be duplicated
+ * @param dup_volinfo   new volinfo which will be created
+ * @param set_userauth  if this true then auth info is also set
+ *
+ * @return 0 on success else -1
+ */
+int32_t
+glusterd_volinfo_dup (glusterd_volinfo_t *volinfo,
+                      glusterd_volinfo_t **dup_volinfo,
+                      gf_boolean_t set_userauth)
+{
+        int32_t                 ret             = -1;
+        xlator_t                *this           = NULL;
+        glusterd_volinfo_t      *new_volinfo    = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, dup_volinfo, out);
+
+        ret = glusterd_volinfo_new (&new_volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "not able to create the "
+                        "duplicate volinfo for the volume %s",
+                        volinfo->volname);
+                goto out;
+        }
+
+        new_volinfo->type = volinfo->type;
+        new_volinfo->replica_count = volinfo->replica_count;
+        new_volinfo->stripe_count = volinfo->stripe_count;
+        new_volinfo->dist_leaf_count = volinfo->dist_leaf_count;
+        new_volinfo->sub_count = volinfo->sub_count;
+        new_volinfo->transport_type = volinfo->transport_type;
+        new_volinfo->nfs_transport_type = volinfo->nfs_transport_type;
+        new_volinfo->brick_count = volinfo->brick_count;
+
+        dict_copy (volinfo->dict, new_volinfo->dict);
+        gd_update_volume_op_versions (new_volinfo);
+
+        if (set_userauth) {
+                glusterd_auth_set_username (new_volinfo,
+                                            volinfo->auth.username);
+                glusterd_auth_set_password (new_volinfo,
+                                            volinfo->auth.password);
+        }
+
+        *dup_volinfo = new_volinfo;
+        ret = 0;
+out:
+        if (ret && (NULL != new_volinfo)) {
+                (void) glusterd_volinfo_delete (new_volinfo);
+        }
+        return ret;
+}
+
+/* This function will duplicate brickinfo
+ *
+ * @param brickinfo     Source brickinfo
+ * @param dup_brickinfo Destination brickinfo
+ *
+ * @return 0 on success else -1
+ */
+int32_t
+glusterd_brickinfo_dup (glusterd_brickinfo_t *brickinfo,
+                        glusterd_brickinfo_t *dup_brickinfo)
+{
+        int32_t         ret     = -1;
+        xlator_t        *this   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_VALIDATE_OR_GOTO (this->name, brickinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, dup_brickinfo, out);
+
+        strcpy (dup_brickinfo->hostname, brickinfo->hostname);
+        strcpy (dup_brickinfo->path, brickinfo->path);
+        strcpy (dup_brickinfo->device_path, brickinfo->device_path);
+        ret = gf_canonicalize_path (dup_brickinfo->path);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to canonicalize "
+                        "brick path");
+                goto out;
+        }
+        uuid_copy (dup_brickinfo->uuid, brickinfo->uuid);
+
+        dup_brickinfo->port = brickinfo->port;
+        dup_brickinfo->rdma_port = brickinfo->rdma_port;
+        if (NULL != brickinfo->logfile) {
+                dup_brickinfo->logfile = gf_strdup (brickinfo->logfile);
+                if (NULL == dup_brickinfo->logfile) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        dup_brickinfo->status = brickinfo->status;
+        dup_brickinfo->snap_status = brickinfo->snap_status;
+out:
+        return ret;
+}
+
+/* This function will copy snap volinfo to the new
+ * passed volinfo and regenerate backend store files
+ * for the restored snap.
+ *
+ * @param new_volinfo   new volinfo
+ * @param snap_volinfo  volinfo of snap volume
+ *
+ * @return 0 on success and -1 on failure
+ *
+ * TODO: Duplicate all members of volinfo, e.g. geo-rep sync slaves
+ */
+int32_t
+glusterd_snap_volinfo_restore (dict_t *rsp_dict,
+                               glusterd_volinfo_t *new_volinfo,
+                               glusterd_volinfo_t *snap_volinfo)
+{
+        int32_t                 brick_count     = -1;
+        int32_t                 ret             = -1;
+        xlator_t                *this           = NULL;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+        glusterd_brickinfo_t    *new_brickinfo  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+
+        GF_VALIDATE_OR_GOTO (this->name, new_volinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, snap_volinfo, out);
+
+        brick_count = 0;
+        list_for_each_entry (brickinfo, &snap_volinfo->bricks, brick_list) {
+                ret = glusterd_brickinfo_new (&new_brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to create "
+                                "new brickinfo");
+                        goto out;
+                }
+
+                /* Duplicate brickinfo */
+                ret = glusterd_brickinfo_dup (brickinfo, new_brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to dup "
+                                "brickinfo");
+                        goto out;
+                }
+
+                /* If the brick is not of this peer, or snapshot is missed *
+                 * for the brick do not replace the xattr for it */
+                if ((!uuid_compare (brickinfo->uuid, MY_UUID)) &&
+                    (brickinfo->snap_status != -1)) {
+                        /* We need to replace the volume id of all the bricks
+                         * to the volume id of the origin volume. new_volinfo
+                         * has the origin volume's volume id*/
+                        ret = sys_lsetxattr (new_brickinfo->path,
+                                             GF_XATTR_VOL_ID_KEY,
+                                             new_volinfo->volume_id,
+                                             sizeof (new_volinfo->volume_id),
+                                             XATTR_REPLACE);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "set extended attribute %s on %s. "
+                                        "Reason: %s, snap: %s",
+                                        GF_XATTR_VOL_ID_KEY,
+                                        new_brickinfo->path, strerror (errno),
+                                        new_volinfo->volname);
+                                goto out;
+                        }
+                }
+
+                /* If a snapshot is pending for this brick then
+                 * restore should also be pending
+                 */
+                if (brickinfo->snap_status == -1) {
+                        /* Adding missed delete to the dict */
+                        ret = glusterd_add_missed_snaps_to_dict
+                                                (rsp_dict,
+                                                 snap_volinfo->volname,
+                                                 brickinfo,
+                                                 brick_count + 1,
+                                                 GF_SNAP_OPTION_TYPE_RESTORE);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to add missed snapshot info "
+                                        "for %s:%s in the rsp_dict",
+                                        brickinfo->hostname,
+                                        brickinfo->path);
+                                goto out;
+                        }
+                }
+
+                list_add_tail (&new_brickinfo->brick_list,
+                                &new_volinfo->bricks);
+                /* ownership of new_brickinfo is passed to new_volinfo */
+                new_brickinfo = NULL;
+                brick_count++;
+        }
+
+        /* Regenerate all volfiles */
+        ret = glusterd_create_volfiles_and_notify_services (new_volinfo);
+
+out:
+        if (ret && (NULL != new_brickinfo)) {
+                (void) glusterd_brickinfo_delete (new_brickinfo);
+        }
+
         return ret;
 }
 
+void
+glusterd_auth_cleanup (glusterd_volinfo_t *volinfo) {
+
+        GF_ASSERT (volinfo);
+
+        GF_FREE (volinfo->auth.username);
+
+        GF_FREE (volinfo->auth.password);
+}
+
+char *
+glusterd_auth_get_username (glusterd_volinfo_t *volinfo) {
+
+        GF_ASSERT (volinfo);
+
+        return volinfo->auth.username;
+}
+
+char *
+glusterd_auth_get_password (glusterd_volinfo_t *volinfo) {
+
+        GF_ASSERT (volinfo);
+
+        return volinfo->auth.password;
+}
+
+int32_t
+glusterd_auth_set_username (glusterd_volinfo_t *volinfo, char *username) {
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (username);
+
+        volinfo->auth.username = gf_strdup (username);
+        return 0;
+}
+
+int32_t
+glusterd_auth_set_password (glusterd_volinfo_t *volinfo, char *password) {
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (password);
+
+        volinfo->auth.password = gf_strdup (password);
+        return 0;
+}
+
 int32_t
 glusterd_brickinfo_delete (glusterd_brickinfo_t *brickinfo)
 {
@@ -572,8 +739,7 @@ glusterd_brickinfo_delete (glusterd_brickinfo_t *brickinfo)
 
         list_del_init (&brickinfo->brick_list);
 
-        if (brickinfo->logfile)
-                GF_FREE (brickinfo->logfile);
+        GF_FREE (brickinfo->logfile);
         GF_FREE (brickinfo);
 
         ret = 0;
@@ -598,7 +764,7 @@ glusterd_volume_brickinfos_delete (glusterd_volinfo_t *volinfo)
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -610,6 +776,7 @@ glusterd_volinfo_delete (glusterd_volinfo_t *volinfo)
         GF_ASSERT (volinfo);
 
         list_del_init (&volinfo->vol_list);
+        list_del_init (&volinfo->snapvol_list);
 
         ret = glusterd_volume_brickinfos_delete (volinfo);
         if (ret)
@@ -618,18 +785,18 @@ glusterd_volinfo_delete (glusterd_volinfo_t *volinfo)
                 dict_unref (volinfo->dict);
         if (volinfo->gsync_slaves)
                 dict_unref (volinfo->gsync_slaves);
-        if (volinfo->logdir)
-                GF_FREE (volinfo->logdir);
+        GF_FREE (volinfo->logdir);
+
+        glusterd_auth_cleanup (volinfo);
 
         GF_FREE (volinfo);
         ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-
 int32_t
 glusterd_brickinfo_new (glusterd_brickinfo_t **brickinfo)
 {
@@ -651,7 +818,7 @@ glusterd_brickinfo_new (glusterd_brickinfo_t **brickinfo)
         ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -659,17 +826,21 @@ int32_t
 glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo)
 {
         int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (brickinfo);
 
         ret = glusterd_hostname_to_uuid (brickinfo->hostname, brickinfo->uuid);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd_brickinfo_from_brick (char *brick,
-                               glusterd_brickinfo_t **brickinfo)
+glusterd_brickinfo_new_from_brick (char *brick,
+                                   glusterd_brickinfo_t **brickinfo)
 {
         int32_t                 ret = -1;
         glusterd_brickinfo_t    *new_brickinfo = NULL;
@@ -677,6 +848,7 @@ glusterd_brickinfo_from_brick (char *brick,
         char                    *path = NULL;
         char                    *tmp_host = NULL;
         char                    *tmp_path = NULL;
+        char                    *vg       = NULL;
 
         GF_ASSERT (brick);
         GF_ASSERT (brickinfo);
@@ -692,7 +864,21 @@ glusterd_brickinfo_from_brick (char *brick,
         GF_ASSERT (path);
 
         ret = glusterd_brickinfo_new (&new_brickinfo);
+        if (ret)
+                goto out;
 
+#ifdef HAVE_BD_XLATOR
+        vg = strchr (path, '?');
+        /* ? is used as a delimiter for vg */
+        if (vg) {
+                strncpy (new_brickinfo->vg, vg + 1, PATH_MAX - 1);
+                *vg = '\0';
+        }
+        new_brickinfo->caps = CAPS_BD;
+#else
+        vg = NULL; /* Avoid compiler warnings when BD not enabled */
+#endif
+        ret = gf_canonicalize_path (path);
         if (ret)
                 goto out;
 
@@ -703,23 +889,272 @@ glusterd_brickinfo_from_brick (char *brick,
 
         ret = 0;
 out:
-        if (tmp_host)
-                GF_FREE (tmp_host);
+        GF_FREE (tmp_host);
         if (tmp_host)
                 GF_FREE (tmp_path);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static gf_boolean_t
+_is_prefix (char *str1, char *str2)
+{
+        GF_ASSERT (str1);
+        GF_ASSERT (str2);
+
+        int             i = 0;
+        int             len1 = 0;
+        int             len2 = 0;
+        int             small_len = 0;
+        char            *bigger = NULL;
+        gf_boolean_t    prefix = _gf_true;
+
+        len1 = strlen (str1);
+        len2 = strlen (str2);
+        small_len = min (len1, len2);
+        for (i = 0; i < small_len; i++) {
+                if (str1[i] != str2[i]) {
+                        prefix = _gf_false;
+                        break;
+                }
+        }
+
+        if (len1 < len2)
+            bigger = str2;
+
+        else if (len1 > len2)
+            bigger = str1;
+
+        else
+            return prefix;
+
+        if (bigger[small_len] != '/')
+            prefix = _gf_false;
+
+        return prefix;
+}
+
+/* Checks if @path is available in the peer identified by @uuid
+ * 'availability' is determined by querying current state of volumes
+ * in the cluster. */
+gf_boolean_t
+glusterd_is_brickpath_available (uuid_t uuid, char *path)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_volinfo_t      *volinfo   = NULL;
+        glusterd_conf_t         *priv      = NULL;
+        gf_boolean_t            available  = _gf_false;
+        char                    tmp_path[PATH_MAX+1] = {0};
+        char                    tmp_brickpath[PATH_MAX+1] = {0};
+
+        priv = THIS->private;
+
+        strncpy (tmp_path, path, PATH_MAX);
+        /* path may not yet exist */
+        if (!realpath (path, tmp_path)) {
+                if (errno != ENOENT) {
+                        goto out;
+                }
+                /* When realpath(3) fails, tmp_path is undefined. */
+                strncpy(tmp_path,path,PATH_MAX);
+        }
+
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        if (uuid_compare (uuid, brickinfo->uuid))
+                                continue;
+
+                        if (!realpath (brickinfo->path, tmp_brickpath)) {
+                            if (errno == ENOENT)
+                                strncpy (tmp_brickpath, brickinfo->path,
+                                         PATH_MAX);
+                            else
+                                goto out;
+                        }
+
+                        if (_is_prefix (tmp_brickpath, tmp_path))
+                                goto out;
+                }
+        }
+        available = _gf_true;
+out:
+        return available;
+}
+
+#ifdef HAVE_BD_XLATOR
+/*
+ * Sets the tag of the format "trusted.glusterfs.volume-id:<uuid>" in
+ * the brick VG. It is used to avoid using same VG for another brick.
+ * @volume-id - gfid, @brick - brick info, @msg - Error message returned
+ * to the caller
+ */
+int
+glusterd_bd_set_vg_tag (unsigned char *volume_id, glusterd_brickinfo_t *brick,
+                        char *msg, int msg_size)
+{
+        lvm_t        handle    = NULL;
+        vg_t         vg        = NULL;
+        char        *uuid      = NULL;
+        int          ret       = -1;
+
+        gf_asprintf (&uuid, "%s:%s", GF_XATTR_VOL_ID_KEY,
+                     uuid_utoa (volume_id));
+        if (!uuid) {
+                snprintf (msg, sizeof(*msg), "Could not allocate memory "
+                          "for tag");
+                return -1;
+        }
+
+        handle = lvm_init (NULL);
+        if (!handle) {
+                snprintf (msg, sizeof(*msg), "lvm_init failed");
+                goto out;
+        }
+
+        vg = lvm_vg_open (handle, brick->vg, "w", 0);
+        if (!vg) {
+                snprintf (msg, sizeof(*msg), "Could not open VG %s",
+                          brick->vg);
+                goto out;
+        }
+
+        if (lvm_vg_add_tag (vg, uuid) < 0) {
+                snprintf (msg, sizeof(*msg), "Could not set tag %s for "
+                          "VG %s", uuid, brick->vg);
+                goto out;
+        }
+        lvm_vg_write (vg);
+        ret = 0;
+out:
+        GF_FREE (uuid);
+
+        if (vg)
+                lvm_vg_close (vg);
+        if (handle)
+                lvm_quit (handle);
+
+        return ret;
+}
+#endif
+
+int
+glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
+                                        uuid_t volume_id, char **op_errstr,
+                                        gf_boolean_t is_force)
+{
+        int          ret                 = -1;
+        char         parentdir[PATH_MAX] = {0,};
+        struct stat  parent_st           = {0,};
+        struct stat  brick_st            = {0,};
+        struct stat  root_st             = {0,};
+        char         msg[2048]           = {0,};
+        gf_boolean_t is_created          = _gf_false;
+
+        ret = mkdir (brickinfo->path, 0777);
+        if (ret) {
+                if (errno != EEXIST) {
+                        snprintf (msg, sizeof (msg), "Failed to create brick "
+                                  "directory for brick %s:%s. Reason : %s ",
+                                  brickinfo->hostname, brickinfo->path,
+                                  strerror (errno));
+                        goto out;
+                }
+        } else {
+                is_created = _gf_true;
+        }
+
+        ret = lstat (brickinfo->path, &brick_st);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "lstat failed on %s. Reason : %s",
+                          brickinfo->path, strerror (errno));
+                goto out;
+        }
+
+        if ((!is_created) && (!S_ISDIR (brick_st.st_mode))) {
+                snprintf (msg, sizeof (msg), "The provided path %s which is "
+                          "already present, is not a directory",
+                          brickinfo->path);
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (parentdir, sizeof (parentdir), "%s/..", brickinfo->path);
+
+        ret = lstat ("/", &root_st);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "lstat failed on /. Reason : %s",
+                          strerror (errno));
+                goto out;
+        }
+
+        ret = lstat (parentdir, &parent_st);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "lstat failed on %s. Reason : %s",
+                          parentdir, strerror (errno));
+                goto out;
+        }
+
+        if (!is_force) {
+                if (brick_st.st_dev != parent_st.st_dev) {
+                        snprintf (msg, sizeof (msg), "The brick %s:%s is a "
+                                  "mount point. Please create a sub-directory "
+                                  "under the mount point and use that as the "
+                                  "brick directory. Or use 'force' at the end "
+                                  "of the command if you want to override this "
+                                  "behavior.", brickinfo->hostname,
+                                  brickinfo->path);
+                        ret = -1;
+                        goto out;
+                }
+                else if (parent_st.st_dev == root_st.st_dev) {
+                        snprintf (msg, sizeof (msg), "The brick %s:%s is "
+                                  "is being created in the root partition. It "
+                                  "is recommended that you don't use the "
+                                  "system's root partition for storage backend."
+                                  " Or use 'force' at the end of the command if"
+                                  " you want to override this behavior.",
+                                  brickinfo->hostname, brickinfo->path);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+#ifdef HAVE_BD_XLATOR
+        if (brickinfo->vg[0]) {
+                ret = glusterd_bd_set_vg_tag (volume_id, brickinfo, msg,
+                                              sizeof(msg));
+                if (ret)
+                        goto out;
+        }
+#endif
+        ret = glusterd_check_and_set_brick_xattr (brickinfo->hostname,
+                                                  brickinfo->path, volume_id,
+                                                  op_errstr, is_force);
+        if (ret)
+                goto out;
+
+        ret = 0;
+
+out:
+        if (ret && is_created)
+                rmdir (brickinfo->path);
+        if (ret && !*op_errstr && msg[0] != '\0')
+                *op_errstr = gf_strdup (msg);
+
         return ret;
 }
 
 int32_t
 glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path,
                                glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t **brickinfo,
-                               gf_path_match_t path_match)
+                               glusterd_brickinfo_t **brickinfo)
 {
         glusterd_brickinfo_t    *brickiter = NULL;
         uuid_t                  peer_uuid = {0};
         int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
 
         if (uuid) {
                 uuid_copy (peer_uuid, uuid);
@@ -731,77 +1166,48 @@ glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path,
         ret = -1;
         list_for_each_entry (brickiter, &volinfo->bricks, brick_list) {
 
-                if (uuid_is_null (brickiter->uuid) &&
-                    glusterd_resolve_brick (brickiter))
+                if ((uuid_is_null (brickiter->uuid)) &&
+                    (glusterd_resolve_brick (brickiter) != 0))
                         goto out;
                 if (uuid_compare (peer_uuid, brickiter->uuid))
                         continue;
 
-                if (!strcmp (brickiter->path, path)) {
-                        gf_log (THIS->name, GF_LOG_INFO, "Found brick");
+                if (strcmp (brickiter->path, path) == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG, LOGSTR_FOUND_BRICK,
+                                brickiter->hostname, brickiter->path,
+                                volinfo->volname);
                         ret = 0;
                         if (brickinfo)
                                 *brickinfo = brickiter;
                         break;
                 }
-                if (path_match != GF_PATH_PARTIAL)
-                        continue;
-
-                if (!fnmatch (path, brickiter->path, FNM_LEADING_DIR) ||
-                    !fnmatch (brickiter->path, path, FNM_LEADING_DIR)) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "paths %s and %s are recursive",
-                                path, brickiter->path);
-                        *brickinfo = brickiter;
-                        ret = 0;
-                        break;
-                }
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
 glusterd_volume_brickinfo_get_by_brick (char *brick,
                                         glusterd_volinfo_t *volinfo,
-                                        glusterd_brickinfo_t **brickinfo,
-                                        gf_path_match_t path_match)
+                                        glusterd_brickinfo_t **brickinfo)
 {
         int32_t                 ret = -1;
-        char                    *hostname = NULL;
-        char                    *path = NULL;
-        char                    *tmp_host = NULL;
-        char                    *tmp_path = NULL;
+        glusterd_brickinfo_t    *tmp_brickinfo = NULL;
 
         GF_ASSERT (brick);
         GF_ASSERT (volinfo);
 
-        gf_log ("", GF_LOG_INFO, "brick: %s", brick);
-
-        tmp_host = gf_strdup (brick);
-        if (tmp_host)
-                get_host_name (tmp_host, &hostname);
-        tmp_path = gf_strdup (brick);
-        if (tmp_path)
-                get_path_name (tmp_path, &path);
-
-        if (!hostname || !path) {
-                gf_log ("", GF_LOG_ERROR,
-                        "brick %s is not of form <HOSTNAME>:<export-dir>",
-                        brick);
-                ret = -1;
+        ret = glusterd_brickinfo_new_from_brick (brick, &tmp_brickinfo);
+        if (ret)
                 goto out;
-        }
 
-        ret = glusterd_volume_brickinfo_get (NULL, hostname, path, volinfo,
-                                             brickinfo, path_match);
+        ret = glusterd_volume_brickinfo_get (NULL, tmp_brickinfo->hostname,
+                                             tmp_brickinfo->path, volinfo,
+                                             brickinfo);
+        (void) glusterd_brickinfo_delete (tmp_brickinfo);
 out:
-        if (tmp_host)
-                GF_FREE (tmp_host);
-        if (tmp_path)
-                GF_FREE (tmp_path);
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
@@ -815,7 +1221,7 @@ glusterd_is_brick_decommissioned (glusterd_volinfo_t *volinfo, char *hostname,
         int                     ret = -1;
 
         ret = glusterd_volume_brickinfo_get (NULL, hostname, path, volinfo,
-                                             &brickinfo, GF_PATH_COMPLETE);
+                                             &brickinfo);
         if (ret)
                 goto out;
         decommissioned = brickinfo->decommissioned;
@@ -828,20 +1234,95 @@ glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo)
 {
         GF_ASSERT (peerinfo);
         glusterd_peerctx_t      *peerctx = NULL;
+        gf_boolean_t            quorum_action = _gf_false;
+        glusterd_conf_t         *priv = THIS->private;
 
+        if (peerinfo->quorum_contrib != QUORUM_NONE)
+                quorum_action = _gf_true;
         if (peerinfo->rpc) {
+                /* cleanup the saved-frames before last unref */
+                synclock_unlock (&priv->big_lock);
+                rpc_clnt_connection_cleanup (&peerinfo->rpc->conn);
+                synclock_lock (&priv->big_lock);
+
                 peerctx = peerinfo->rpc->mydata;
                 peerinfo->rpc->mydata = NULL;
                 peerinfo->rpc = rpc_clnt_unref (peerinfo->rpc);
                 peerinfo->rpc = NULL;
-                if (peerctx)
+                if (peerctx) {
+                        GF_FREE (peerctx->errstr);
                         GF_FREE (peerctx);
+                }
         }
         glusterd_peer_destroy (peerinfo);
 
+        if (quorum_action)
+                glusterd_do_quorum_action ();
         return 0;
 }
 
+int
+glusterd_volinfo_find_by_volume_id (uuid_t volume_id, glusterd_volinfo_t **volinfo)
+{
+        int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_volinfo_t      *voliter = NULL;
+        glusterd_conf_t         *priv = NULL;
+
+        if (!volume_id)
+                return -1;
+
+        this = THIS;
+        priv = this->private;
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (uuid_compare (volume_id, voliter->volume_id))
+                        continue;
+                *volinfo = voliter;
+                ret = 0;
+                gf_log (this->name, GF_LOG_DEBUG, "Volume %s found",
+                        voliter->volname);
+                break;
+        }
+        return ret;
+}
+
+int
+glusterd_snap_volinfo_find_by_volume_id (uuid_t volume_id,
+                                         glusterd_volinfo_t **volinfo)
+{
+        int32_t                  ret     = -1;
+        xlator_t                *this    = NULL;
+        glusterd_volinfo_t      *voliter = NULL;
+        glusterd_snap_t         *snap    = NULL;
+        glusterd_conf_t         *priv    = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volinfo);
+
+        if (uuid_is_null(volume_id)) {
+                gf_log (this->name, GF_LOG_WARNING, "Volume UUID is NULL");
+                goto out;
+        }
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                list_for_each_entry (voliter, &snap->volumes, vol_list) {
+                        if (uuid_compare (volume_id, voliter->volume_id))
+                                continue;
+                        *volinfo = voliter;
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_WARNING, "Snap volume not found");
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
 int32_t
 glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo)
 {
@@ -851,134 +1332,149 @@ glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo)
         glusterd_conf_t         *priv = NULL;
 
         GF_ASSERT (volname);
-
         this = THIS;
         GF_ASSERT (this);
 
         priv = this->private;
+        GF_ASSERT (priv);
 
         list_for_each_entry (tmp_volinfo, &priv->volumes, vol_list) {
                 if (!strcmp (tmp_volinfo->volname, volname)) {
-                        gf_log ("", GF_LOG_DEBUG, "Volume %s found", volname);
+                        gf_log (this->name, GF_LOG_DEBUG, "Volume %s found",
+                                volname);
                         ret = 0;
                         *volinfo = tmp_volinfo;
                         break;
                 }
         }
 
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-
 int32_t
-glusterd_service_stop (const char *service, char *pidfile, int sig,
-                       gf_boolean_t force_kill)
+glusterd_snap_volinfo_find (char *snap_volname, glusterd_snap_t *snap,
+                            glusterd_volinfo_t **volinfo)
 {
-        int32_t  ret = -1;
-        pid_t    pid = -1;
-        FILE    *file = NULL;
-        gf_boolean_t is_locked = _gf_false;
+        int32_t                  ret         = -1;
+        xlator_t                *this        = NULL;
+        glusterd_volinfo_t      *snap_vol    = NULL;
+        glusterd_conf_t         *priv        = NULL;
 
-        file = fopen (pidfile, "r+");
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+        GF_ASSERT (snap_volname);
 
-        if (!file) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
-                                pidfile);
-                if (errno == ENOENT) {
-                        gf_log ("",GF_LOG_TRACE, "%s may not be running",
-                                service);
+        list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
+                if (!strcmp (snap_vol->volname, snap_volname)) {
                         ret = 0;
+                        *volinfo = snap_vol;
                         goto out;
                 }
-                ret = -1;
-                goto out;
         }
-        ret = lockf (fileno (file), F_TLOCK, 0);
-        if (!ret) {
-                is_locked = _gf_true;
-                ret = unlink (pidfile);
-                if (ret && (ENOENT != errno)) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to "
-                                "unlink stale pidfile: %s", pidfile);
-                } else if (ret && (ENOENT == errno)){
+
+        gf_log (this->name, GF_LOG_WARNING, "Snap volume %s not found",
+                snap_volname);
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_snap_volinfo_find_from_parent_volname (char *origin_volname,
+                                      glusterd_snap_t *snap,
+                                      glusterd_volinfo_t **volinfo)
+{
+        int32_t                  ret         = -1;
+        xlator_t                *this        = NULL;
+        glusterd_volinfo_t      *snap_vol    = NULL;
+        glusterd_conf_t         *priv        = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+        GF_ASSERT (origin_volname);
+
+        list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
+                if (!strcmp (snap_vol->parent_volname, origin_volname)) {
                         ret = 0;
-                        gf_log ("", GF_LOG_INFO, "Brick already stopped");
+                        *volinfo = snap_vol;
+                        goto out;
                 }
-                goto out;
         }
 
+        gf_log (this->name, GF_LOG_DEBUG, "Snap volume not found(snap: %s, "
+                "origin-volume: %s", snap->snapname, origin_volname);
 
-        ret = fscanf (file, "%d", &pid);
-        if (ret <= 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to read pidfile: %s",
-                                pidfile);
-                ret = -1;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_service_stop (const char *service, char *pidfile, int sig,
+                       gf_boolean_t force_kill)
+{
+        int32_t  ret = -1;
+        pid_t    pid = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        if (!glusterd_is_service_running (pidfile, &pid)) {
+                ret = 0;
+                gf_log (this->name, GF_LOG_INFO, "%s already stopped", service);
                 goto out;
         }
-        fclose (file);
-        file = NULL;
-
-        gf_log ("", GF_LOG_INFO, "Stopping gluster %s running in pid: %d",
-                service, pid);
+        gf_log (this->name, GF_LOG_DEBUG, "Stopping gluster %s running in pid: "
+                "%d", service, pid);
 
         ret = kill (pid, sig);
+        if (!force_kill)
+                goto out;
 
-        if (force_kill) {
-                sleep (1);
-                file = fopen (pidfile, "r+");
-                if (!file) {
-                        ret = 0;
-                        goto out;
-                }
-                ret = lockf (fileno (file), F_TLOCK, 0);
-                if (ret && ((EAGAIN == errno) || (EACCES == errno))) {
-                        ret = kill (pid, SIGKILL);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to "
-                                        "kill pid %d reason: %s", pid,
-                                        strerror(errno));
-                                goto out;
-                        }
-
-                } else if (0 == ret){
-                        is_locked = _gf_true;
-                }
-                ret = unlink (pidfile);
-                if (ret && (ENOENT != errno)) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to "
-                                "unlink pidfile: %s", pidfile);
+        sleep (1);
+        if (glusterd_is_service_running (pidfile, NULL)) {
+                ret = kill (pid, SIGKILL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                "kill pid %d reason: %s", pid,
+                                strerror(errno));
                         goto out;
                 }
         }
 
         ret = 0;
 out:
-        if (is_locked && file)
-                if (lockf (fileno (file), F_ULOCK, 0) < 0)
-                        gf_log ("", GF_LOG_WARNING, "Cannot unlock pidfile: %s"
-                                " reason: %s", pidfile, strerror(errno));
-        if (file)
-                fclose (file);
         return ret;
 }
 
 void
+glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len)
+{
+        char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};
+
+        md5_wrapper ((unsigned char *) sock_filepath, strlen(sock_filepath), md5_sum);
+        snprintf (sockpath, len, "%s/%s.socket", GLUSTERD_SOCK_DIR, md5_sum);
+}
+
+void
 glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo,
                                     glusterd_brickinfo_t *brickinfo,
                                     char *sockpath, size_t len)
 {
         char                    export_path[PATH_MAX] = {0,};
         char                    sock_filepath[PATH_MAX] = {0,};
-        char                    md5_sum[MD5_DIGEST_LEN*2+1] = {0,};
         char                    volume_dir[PATH_MAX] = {0,};
         xlator_t                *this = NULL;
         glusterd_conf_t         *priv = NULL;
         int                     expected_file_len = 0;
 
-        expected_file_len = strlen (glusterd_sock_dir) + strlen ("/") +
-                            MD5_DIGEST_LEN*2 + strlen (".socket") + 1;
+        expected_file_len = strlen (GLUSTERD_SOCK_DIR) + strlen ("/") +
+                            MD5_DIGEST_LENGTH*2 + strlen (".socket") + 1;
         GF_ASSERT (len >= expected_file_len);
         this = THIS;
         GF_ASSERT (this);
@@ -989,10 +1485,8 @@ glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo,
         GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, export_path);
         snprintf (sock_filepath, PATH_MAX, "%s/run/%s-%s",
                   volume_dir, brickinfo->hostname, export_path);
-        _get_md5_str (md5_sum, sizeof (md5_sum),
-                              (uint8_t*)sock_filepath, strlen (sock_filepath));
 
-        snprintf (sockpath, len, "%s/%s.socket", glusterd_sock_dir, md5_sum);
+        glusterd_set_socket_filepath (sock_filepath, sockpath, len);
 }
 
 /* connection happens only if it is not aleady connected,
@@ -1000,26 +1494,41 @@ glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo,
  */
 int32_t
 glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
-                        glusterd_brickinfo_t  *brickinfo)
+                        glusterd_brickinfo_t  *brickinfo, char *socketpath)
 {
         int                     ret = 0;
-        char                    socketpath[PATH_MAX] = {0};
+        char                    volume_id_str[64];
+        char                    *brickid = NULL;
         dict_t                  *options = NULL;
         struct rpc_clnt         *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
+        GF_ASSERT (socketpath);
 
         if (brickinfo->rpc == NULL) {
-                glusterd_set_brick_socket_filepath (volinfo, brickinfo,
-                                                    socketpath,
-                                                    sizeof (socketpath));
-                ret = rpc_clnt_transport_unix_options_build (&options, socketpath);
+                /* Setting frame-timeout to 10mins (600seconds).
+                 * Unix domain sockets ensures that the connection is reliable.
+                 * The default timeout of 30mins used for unreliable network
+                 * connections is too long for unix domain socket connections.
+                 */
+                ret = rpc_transport_unix_options_build (&options, socketpath,
+                                                        600);
                 if (ret)
                         goto out;
+
+                uuid_utoa_r (volinfo->volume_id, volume_id_str);
+                ret = gf_asprintf (&brickid, "%s:%s:%s", volume_id_str,
+                                   brickinfo->hostname, brickinfo->path);
+                if (ret < 0)
+                        goto out;
+
+                synclock_unlock (&priv->big_lock);
                 ret = glusterd_rpc_create (&rpc, options,
                                            glusterd_brick_rpc_notify,
-                                           brickinfo);
+                                           brickid);
+                synclock_lock (&priv->big_lock);
                 if (ret)
                         goto out;
                 brickinfo->rpc = rpc;
@@ -1029,28 +1538,54 @@ out:
         return ret;
 }
 
+/* Caller should ensure that brick process is not running*/
+static void
+_reap_brick_process (char *pidfile, char *brickpath)
+{
+        unlink (pidfile);
+        /* Brick process is not running and pmap may have an entry for it.*/
+        pmap_registry_remove (THIS, 0, brickpath,
+                              GF_PMAP_PORT_BRICKSERVER, NULL);
+}
+
+static int
+_mk_rundir_p (glusterd_volinfo_t *volinfo)
+{
+        char voldir[PATH_MAX]   = {0,};
+        char rundir[PATH_MAX]   = {0,};
+        glusterd_conf_t *priv   = NULL;
+        xlator_t        *this   = NULL;
+        int             ret     = -1;
+
+        this = THIS;
+        priv = this->private;
+        GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, priv);
+        snprintf (rundir, sizeof (rundir)-1, "%s/run", voldir);
+        ret =  mkdir_p (rundir, 0777, _gf_true);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create rundir");
+        return ret;
+}
+
 int32_t
 glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
-                                 glusterd_brickinfo_t  *brickinfo)
+                                 glusterd_brickinfo_t  *brickinfo,
+                                 gf_boolean_t wait)
 {
         int32_t                 ret = -1;
         xlator_t                *this = NULL;
         glusterd_conf_t         *priv = NULL;
-        char                    pidfile[PATH_MAX] = {0,};
+        char                    pidfile[PATH_MAX+1] = {0,};
         char                    volfile[PATH_MAX] = {0,};
-        char                    path[PATH_MAX] = {0,};
         runner_t                runner = {0,};
-        char                    rundir[PATH_MAX] = {0,};
         char                    exp_path[PATH_MAX] = {0,};
         char                    logfile[PATH_MAX] = {0,};
         int                     port = 0;
         int                     rdma_port = 0;
-        FILE                    *file = NULL;
-        gf_boolean_t            is_locked = _gf_false;
         char                    socketpath[PATH_MAX] = {0};
-#ifdef DEBUG
+        char                    glusterd_uuid[1024] = {0,};
         char                    valgrind_logfile[PATH_MAX] = {0};
-#endif
+
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
 
@@ -1058,105 +1593,93 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         GF_ASSERT (this);
 
         priv = this->private;
+        GF_ASSERT (priv);
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-        snprintf (rundir, PATH_MAX, "%s/run", path);
-        ret = mkdir (rundir, 0777);
-
-        if ((ret == -1) && (EEXIST != errno)) {
-                gf_log ("", GF_LOG_ERROR, "Unable to create rundir %s",
-                        rundir);
+        if (brickinfo->snap_status == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Snapshot is pending on %s:%s. "
+                        "Hence not starting the brick",
+                        brickinfo->hostname,
+                        brickinfo->path);
+                ret = 0;
                 goto out;
         }
 
+        ret = _mk_rundir_p (volinfo);
+        if (ret)
+                goto out;
+
         glusterd_set_brick_socket_filepath (volinfo, brickinfo, socketpath,
                                             sizeof (socketpath));
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname,
-                                    brickinfo->path);
-
-        file = fopen (pidfile, "r+");
-        if (file) {
-                ret = lockf (fileno (file), F_TLOCK, 0);
-                if (ret && ((EAGAIN == errno) || (EACCES == errno))) {
-                        ret = 0;
-                        gf_log ("", GF_LOG_INFO, "brick %s:%s "
-                                "already started", brickinfo->hostname,
-                                brickinfo->path);
-                        goto connect;
-                }
-        }
 
-        ret = pmap_registry_search (this, brickinfo->path,
-                                    GF_PMAP_PORT_BRICKSERVER);
-        if (ret) {
-                ret = 0;
-                file = fopen (pidfile, "r+");
-                if (file) {
-                        ret = lockf (fileno (file), F_TLOCK, 0);
-                        if (ret && ((EAGAIN == errno) || (EACCES == errno))) {
-                                ret = 0;
-                                gf_log ("", GF_LOG_INFO, "brick %s:%s "
-                                        "already started", brickinfo->hostname,
-                                        brickinfo->path);
-                                goto connect;
-                        } else if (0 == ret) {
-                                is_locked = _gf_true;
-                        }
-                }
-                /* This means, pmap has the entry, remove it */
-                ret = pmap_registry_remove (this, 0, brickinfo->path,
-                                            GF_PMAP_PORT_BRICKSERVER, NULL);
-        }
-        unlink (pidfile);
-
-        gf_log ("", GF_LOG_INFO, "About to start glusterfs"
-                " for brick %s:%s", brickinfo->hostname,
-                brickinfo->path);
-        GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);
-        snprintf (volfile, PATH_MAX, "%s.%s.%s", volinfo->volname,
-                  brickinfo->hostname, exp_path);
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
+        if (glusterd_is_service_running (pidfile, NULL))
+                goto connect;
 
-        if (!brickinfo->logfile && volinfo->logdir) {
-                snprintf (logfile, PATH_MAX, "%s/%s.log", volinfo->logdir,
-                                                          exp_path);
-                brickinfo->logfile = gf_strdup (logfile);
-        } else if (!brickinfo->logfile) {
-                snprintf (logfile, PATH_MAX, "%s/bricks/%s.log",
-                          DEFAULT_LOG_FILE_DIRECTORY, exp_path);
-                brickinfo->logfile = gf_strdup (logfile);
-        }
+        _reap_brick_process (pidfile, brickinfo->path);
 
         port = brickinfo->port;
         if (!port)
                 port = pmap_registry_alloc (THIS);
 
+        /* Build the exp_path, before starting the glusterfsd even in
+           valgrind mode. Otherwise all the glusterfsd processes start
+           writing the valgrind log to the same file.
+        */
+        GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);
         runinit (&runner);
 
-#ifdef DEBUG
         if (priv->valgrind) {
+                /* Run bricks with valgrind */
                 if (volinfo->logdir) {
                         snprintf (valgrind_logfile, PATH_MAX,
-                                  "%s/valgrind-%s-%s.log", volinfo->logdir,
+                                  "%s/valgrind-%s-%s.log",
+                                  volinfo->logdir,
                                   volinfo->volname, exp_path);
                 } else {
-                         snprintf (valgrind_logfile, PATH_MAX,
-                                   "%s/bricks/valgrnd-%s-%s.log",
-                                   DEFAULT_LOG_FILE_DIRECTORY,
-                                   volinfo->volname, exp_path);
+                        snprintf (valgrind_logfile, PATH_MAX,
+                                  "%s/bricks/valgrind-%s-%s.log",
+                                  DEFAULT_LOG_FILE_DIRECTORY,
+                                  volinfo->volname, exp_path);
                 }
-                /* Run bricks with valgrind */
+
                 runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                "--trace-children=yes", NULL);
+                                 "--trace-children=yes", "--track-origins=yes",
+                                 NULL);
                 runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
-	}
-#endif
-	runner_add_args (&runner, SBIN_DIR"/glusterfsd",
-                         "-s", "localhost", "--volfile-id", volfile,
+        }
+
+        if (volinfo->is_snap_volume) {
+                snprintf (volfile, PATH_MAX,"/%s/%s/%s.%s.%s",
+                          GLUSTERD_VOL_SNAP_DIR_PREFIX,
+                          volinfo->snapshot->snapname, volinfo->volname,
+                          brickinfo->hostname, exp_path);
+        } else {
+                snprintf (volfile, PATH_MAX, "%s.%s.%s", volinfo->volname,
+                          brickinfo->hostname, exp_path);
+        }
+
+        if (volinfo->logdir) {
+                snprintf (logfile, PATH_MAX, "%s/%s.log",
+                          volinfo->logdir, exp_path);
+        } else {
+                snprintf (logfile, PATH_MAX, "%s/bricks/%s.log",
+                          DEFAULT_LOG_FILE_DIRECTORY, exp_path);
+        }
+        if (!brickinfo->logfile)
+                brickinfo->logfile = gf_strdup (logfile);
+
+        (void) snprintf (glusterd_uuid, 1024, "*-posix.glusterd-uuid=%s",
+                         uuid_utoa (MY_UUID));
+        runner_add_args (&runner, SBIN_DIR"/glusterfsd",
+                         "-s", brickinfo->hostname, "--volfile-id", volfile,
                          "-p", pidfile, "-S", socketpath,
                          "--brick-name", brickinfo->path,
-                         "-l", brickinfo->logfile, NULL);
+                         "-l", brickinfo->logfile,
+                         "--xlator-option", glusterd_uuid,
+                         NULL);
 
-	runner_add_arg (&runner, "--brick-port");
+        runner_add_arg (&runner, "--brick-port");
         if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
                 runner_argprintf (&runner, "%d", port);
         } else {
@@ -1173,26 +1696,34 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         runner_argprintf (&runner, "%s-server.listen-port=%d",
                           volinfo->volname, port);
 
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
+
         runner_log (&runner, "", GF_LOG_DEBUG, "Starting GlusterFS");
-        ret = runner_run (&runner);
+        if (wait) {
+                synclock_unlock (&priv->big_lock);
+                ret = runner_run (&runner);
+                synclock_lock (&priv->big_lock);
 
-        if (ret == 0) {
-                //pmap_registry_bind (THIS, port, brickinfo->path);
-                brickinfo->port = port;
-                brickinfo->rdma_port = rdma_port;
+        } else {
+                ret = runner_run_nowait (&runner);
         }
 
-connect:
-        ret = glusterd_brick_connect (volinfo, brickinfo);
         if (ret)
                 goto out;
+
+        brickinfo->port = port;
+        brickinfo->rdma_port = rdma_port;
+
+connect:
+        ret = glusterd_brick_connect (volinfo, brickinfo, socketpath);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to connect to brick %s:%s on %s",
+                        brickinfo->hostname, brickinfo->path, socketpath);
+                goto out;
+        }
 out:
-        if (is_locked && file)
-                if (lockf (fileno (file), F_ULOCK, 0) < 0)
-                        gf_log ("", GF_LOG_WARNING, "Cannot unlock pidfile: %s"
-                                " reason: %s", pidfile, strerror(errno));
-        if (file)
-                fclose (file);
         return ret;
 }
 
@@ -1220,7 +1751,7 @@ glusterd_brick_unlink_socket_file (glusterd_volinfo_t *volinfo,
         if (ret && (ENOENT == errno)) {
                 ret = 0;
         } else {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to remove %s"
+                gf_log (this->name, GF_LOG_ERROR, "Failed to remove %s"
                         " error: %s", socketpath, strerror (errno));
         }
 
@@ -1230,24 +1761,37 @@ glusterd_brick_unlink_socket_file (glusterd_volinfo_t *volinfo,
 int32_t
 glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo)
 {
+        rpc_clnt_t              *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
+
         GF_ASSERT (brickinfo);
 
-        if (brickinfo->rpc) {
-                rpc_clnt_unref (brickinfo->rpc);
-                brickinfo->rpc = NULL;
+        if (!brickinfo) {
+                gf_log_callingfn ("glusterd", GF_LOG_WARNING, "!brickinfo");
+                return -1;
+        }
+
+        rpc            = brickinfo->rpc;
+        brickinfo->rpc = NULL;
+
+        if (rpc) {
+                synclock_unlock (&priv->big_lock);
+                rpc_clnt_unref (rpc);
+                synclock_lock (&priv->big_lock);
         }
+
         return 0;
 }
 
 int32_t
 glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
-                                glusterd_brickinfo_t   *brickinfo)
+                                glusterd_brickinfo_t   *brickinfo,
+                                gf_boolean_t del_brick)
 {
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
-        char                    pidfile[PATH_MAX] = {0,};
-        char                    path[PATH_MAX] = {0,};
-        int                     ret = 0;
+        xlator_t        *this                   = NULL;
+        glusterd_conf_t *priv                   = NULL;
+        char            pidfile[PATH_MAX]       = {0,};
+        int             ret                     = 0;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
@@ -1256,17 +1800,22 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
         GF_ASSERT (this);
 
         priv = this->private;
-        (void) glusterd_brick_disconnect (brickinfo);
-
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname,
-                                    brickinfo->path);
+        if (del_brick)
+                list_del_init (&brickinfo->brick_list);
 
-        ret = glusterd_service_stop ("brick", pidfile, SIGTERM, _gf_false);
-        if (ret == 0) {
-                glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
-                (void) glusterd_brick_unlink_socket_file (volinfo, brickinfo);
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                (void) glusterd_brick_disconnect (brickinfo);
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
+                ret = glusterd_service_stop ("brick", pidfile, SIGTERM, _gf_false);
+                if (ret == 0) {
+                        glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
+                        (void) glusterd_brick_unlink_socket_file (volinfo, brickinfo);
+                }
         }
+
+        if (del_brick)
+                glusterd_delete_brick (volinfo, brickinfo);
+
         return ret;
 }
 
@@ -1296,6 +1845,114 @@ out:
         return ret;
 }
 
+/* Free LINE[0..N-1] and then the LINE buffer.  */
+static void
+free_lines (char **line, size_t n)
+{
+  size_t i;
+  for (i = 0; i < n; i++)
+    GF_FREE (line[i]);
+  GF_FREE (line);
+}
+
+char **
+glusterd_readin_file (const char *filepath, int *line_count)
+{
+        int         ret                    = -1;
+        int         n                      = 8;
+        int         counter                = 0;
+        char        buffer[PATH_MAX + 256] = {0};
+        char      **lines                  = NULL;
+        FILE       *fp                     = NULL;
+        void       *p;
+
+        fp = fopen (filepath, "r");
+        if (!fp)
+                goto out;
+
+        lines = GF_CALLOC (1, n * sizeof (*lines), gf_gld_mt_charptr);
+        if (!lines)
+                goto out;
+
+        for (counter = 0; fgets (buffer, sizeof (buffer), fp); counter++) {
+
+                if (counter == n-1) {
+                        n *= 2;
+                        p = GF_REALLOC (lines, n * sizeof (char *));
+                        if (!p) {
+                                free_lines (lines, n/2);
+                                lines = NULL;
+                                goto out;
+                        }
+                        lines = p;
+                }
+
+                lines[counter] = gf_strdup (buffer);
+        }
+
+        lines[counter] = NULL;
+        /* Reduce allocation to minimal size.  */
+        p = GF_REALLOC (lines, (counter + 1) * sizeof (char *));
+        if (!p) {
+                free_lines (lines, counter);
+                lines = NULL;
+                goto out;
+        }
+        lines = p;
+
+        *line_count = counter;
+        ret = 0;
+
+ out:
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", strerror (errno));
+        if (fp)
+                fclose (fp);
+
+        return lines;
+}
+
+int
+glusterd_compare_lines (const void *a, const void *b) {
+
+        return strcmp(* (char * const *) a, * (char * const *) b);
+}
+
+int
+glusterd_sort_and_redirect (const char *src_filepath, int dest_fd)
+{
+        int            ret          = -1;
+        int            line_count   = 0;
+        int            counter      = 0;
+        char         **lines        = NULL;
+
+
+        if (!src_filepath || dest_fd < 0)
+                goto out;
+
+        lines = glusterd_readin_file (src_filepath, &line_count);
+        if (!lines)
+                goto out;
+
+        qsort (lines, line_count, sizeof (*lines), glusterd_compare_lines);
+
+        for (counter = 0; lines[counter]; counter++) {
+
+                ret = write (dest_fd, lines[counter],
+                             strlen (lines[counter]));
+                if (ret < 0)
+                        goto out;
+
+                GF_FREE (lines[counter]);
+        }
+
+        ret = 0;
+ out:
+        GF_FREE (lines);
+
+        return ret;
+}
+
 int
 glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo)
 {
@@ -1310,10 +1967,10 @@ glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo)
         char                    sort_filepath[PATH_MAX] = {0};
         gf_boolean_t            unlink_sortfile = _gf_false;
         int                     sort_fd = 0;
-        runner_t                runner;
+        xlator_t               *this = NULL;
 
         GF_ASSERT (volinfo);
-
+        this = THIS;
         priv = THIS->private;
         GF_ASSERT (priv);
 
@@ -1322,10 +1979,10 @@ glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo)
         snprintf (cksum_path, sizeof (cksum_path), "%s/%s",
                   path, GLUSTERD_CKSUM_FILE);
 
-        fd = open (cksum_path, O_RDWR | O_APPEND | O_CREAT| O_TRUNC, 0644);
+        fd = open (cksum_path, O_RDWR | O_APPEND | O_CREAT| O_TRUNC, 0600);
 
         if (-1 == fd) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to open %s, errno: %d",
+                gf_log (this->name, GF_LOG_ERROR, "Unable to open %s, errno: %d",
                         cksum_path, errno);
                 ret = -1;
                 goto out;
@@ -1333,12 +1990,14 @@ glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo)
 
         snprintf (filepath, sizeof (filepath), "%s/%s", path,
                   GLUSTERD_VOLUME_INFO_FILE);
-        snprintf (sort_filepath, sizeof (sort_filepath), "/tmp/%s.XXXXXX",
-                  volinfo->volname);
+        snprintf (sort_filepath, sizeof (sort_filepath),
+                  "/tmp/%s.XXXXXX", volinfo->volname);
+
         sort_fd = mkstemp (sort_filepath);
         if (sort_fd < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Could not generate temp file, "
-                        "reason: %s for volume: %s", strerror (errno),
+                gf_log (this->name, GF_LOG_ERROR, "Could not generate temp "
+                        "file, reason: %s for %s: %s", strerror (errno),
+                        (volinfo->is_snap_volume)?"snap":"volume",
                         volinfo->volname);
                 goto out;
         } else {
@@ -1346,21 +2005,21 @@ glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo)
         }
 
         /* sort the info file, result in sort_filepath */
-        runinit (&runner);
-        runner_add_args (&runner, "sort", filepath, NULL);
-        runner_redir (&runner, STDOUT_FILENO, sort_fd);
 
-        ret = runner_run (&runner);
-        close (sort_fd);
+        ret = glusterd_sort_and_redirect (filepath, sort_fd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to sort file %s to %s",
-                        filepath, sort_filepath);
+                gf_log (this->name, GF_LOG_ERROR, "sorting info file failed");
                 goto out;
         }
+
+        ret = close (sort_fd);
+        if (ret)
+                goto out;
+
         ret = get_checksum_for_path (sort_filepath, &cksum);
 
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to get checksum"
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get checksum"
                         " for path: %s", sort_filepath);
                 goto out;
         }
@@ -1379,55 +2038,88 @@ glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo)
                 goto out;
 
         volinfo->cksum = cksum;
-
 out:
         if (fd > 0)
                close (fd);
         if (unlink_sortfile)
                unlink (sort_filepath);
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
 
-void
-_add_volinfo_dict_to_prdict (dict_t *this, char *key, data_t *value, void *data)
+int
+_add_dict_to_prdict (dict_t *this, char *key, data_t *value, void *data)
 {
-        glusterd_voldict_ctx_t   *ctx = NULL;
+        glusterd_dict_ctx_t     *ctx = NULL;
         char                    optkey[512] = {0,};
         int                     ret = -1;
 
         ctx = data;
-        snprintf (optkey, sizeof (optkey), "volume%d.%s%d", ctx->count,
+        snprintf (optkey, sizeof (optkey), "%s.%s%d", ctx->prefix,
                   ctx->key_name, ctx->opt_count);
         ret = dict_set_str (ctx->dict, optkey, key);
         if (ret)
                 gf_log ("", GF_LOG_ERROR, "option add for %s%d %s",
-                        ctx->key_name, ctx->count, key);
-        snprintf (optkey, sizeof (optkey), "volume%d.%s%d", ctx->count,
+                        ctx->key_name, ctx->opt_count, key);
+        snprintf (optkey, sizeof (optkey), "%s.%s%d", ctx->prefix,
                   ctx->val_name, ctx->opt_count);
         ret = dict_set_str (ctx->dict, optkey, value->data);
         if (ret)
                 gf_log ("", GF_LOG_ERROR, "option add for %s%d %s",
-                        ctx->val_name, ctx->count, value->data);
+                        ctx->val_name, ctx->opt_count, value->data);
         ctx->opt_count++;
 
-        return;
+        return ret;
+}
+
+int32_t
+glusterd_add_bricks_hname_path_to_dict (dict_t *dict,
+                                        glusterd_volinfo_t *volinfo)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        int                     ret = 0;
+        char                    key[256] = {0};
+        int                     index = 0;
+
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                snprintf (key, sizeof (key), "%d-hostname", index);
+                ret = dict_set_str (dict, key, brickinfo->hostname);
+                if (ret)
+                        goto out;
+
+                snprintf (key, sizeof (key), "%d-path", index);
+                ret = dict_set_str (dict, key, brickinfo->path);
+                if (ret)
+                        goto out;
+
+                index++;
+        }
+out:
+        return ret;
 }
 
 int32_t
 glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                              dict_t  *dict, int32_t count)
 {
-        int32_t                 ret             = -1;
-        char                    key[512]        = {0,};
-        glusterd_brickinfo_t    *brickinfo      = NULL;
-        int32_t                 i               = 1;
-        char                    *volume_id_str  = NULL;
-        char                    *src_brick      = NULL;
-        char                    *dst_brick      = NULL;
-        glusterd_voldict_ctx_t   ctx            = {0};
+        int32_t                 ret               = -1;
+        char                    prefix[512]       = {0,};
+        char                    key[512]          = {0,};
+        glusterd_brickinfo_t    *brickinfo        = NULL;
+        int32_t                 i                 = 1;
+        char                    *volume_id_str    = NULL;
+        char                    *src_brick        = NULL;
+        char                    *dst_brick        = NULL;
+        char                    *str              = NULL;
+        glusterd_dict_ctx_t     ctx               = {0};
+        char                    *rebalance_id_str = NULL;
+        char                    *rb_id_str        = NULL;
+        xlator_t                *this             = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (dict);
         GF_ASSERT (volinfo);
 
@@ -1442,6 +2134,15 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
+        snprintf (key, sizeof (key), "volume%d.is_volume_restored", count);
+        ret = dict_set_int32 (dict, key, volinfo->is_volume_restored);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set "
+                        "is_volume_restored option for %s volume",
+                        volinfo->volname);
+                goto out;
+        }
+
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.brick_count", count);
         ret = dict_set_int32 (dict, key, volinfo->brick_count);
@@ -1496,30 +2197,91 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
-        volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
-        if (!volume_id_str)
+        snprintf (key, sizeof (key), "volume%d.is_snap_volume", count);
+        ret = dict_set_uint32 (dict, key, volinfo->is_snap_volume);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to set %s", key);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "volume%d.snap-max-hard-limit", count);
+        ret = dict_set_uint64 (dict, key, volinfo->snap_max_hard_limit);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to set %s", key);
                 goto out;
+        }
 
+        volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
+        if (!volume_id_str) {
+                ret = -1;
+                goto out;
+        }
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.volume_id", count);
         ret = dict_set_dynstr (dict, key, volume_id_str);
         if (ret)
                 goto out;
+        volume_id_str = NULL;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.username", count);
+        str = glusterd_auth_get_username (volinfo);
+        if (str) {
+                ret = dict_set_dynstr (dict, key, gf_strdup (str));
+                if (ret)
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.password", count);
+        str = glusterd_auth_get_password (volinfo);
+        if (str) {
+                ret = dict_set_dynstr (dict, key, gf_strdup (str));
+                if (ret)
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d.rebalance", count);
+        ret = dict_set_int32 (dict, key, volinfo->rebal.defrag_cmd);
+        if (ret)
+                goto out;
+
+        if (volinfo->rebal.defrag_cmd) {
+                rebalance_id_str = gf_strdup (uuid_utoa
+                                                (volinfo->rebal.rebalance_id));
+                if (!rebalance_id_str) {
+                        ret = -1;
+                        goto out;
+                }
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d.rebalance-id", count);
+                ret = dict_set_dynstr (dict, key, rebalance_id_str);
+                if (ret)
+                        goto out;
+                rebalance_id_str = NULL;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebalance-op", count);
+        ret = dict_set_uint32 (dict, key, volinfo->rebal.op);
+        if (ret)
+                goto out;
 
         memset (key, 0, sizeof (key));
         snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
-        ret = dict_set_int32 (dict, key, volinfo->rb_status);
+        ret = dict_set_int32 (dict, key, volinfo->rep_brick.rb_status);
         if (ret)
                 goto out;
 
-        if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+        if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
                           count);
                 gf_asprintf (&src_brick, "%s:%s",
-                             volinfo->src_brick->hostname,
-                             volinfo->src_brick->path);
+                             volinfo->rep_brick.src_brick->hostname,
+                             volinfo->rep_brick.src_brick->path);
                 ret = dict_set_dynstr (dict, key, src_brick);
                 if (ret)
                         goto out;
@@ -1528,21 +2290,35 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                 snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_DST_BRICK,
                           count);
                 gf_asprintf (&dst_brick, "%s:%s",
-                             volinfo->dst_brick->hostname,
-                             volinfo->dst_brick->path);
+                             volinfo->rep_brick.dst_brick->hostname,
+                             volinfo->rep_brick.dst_brick->path);
                 ret = dict_set_dynstr (dict, key, dst_brick);
                 if (ret)
                         goto out;
+
+                rb_id_str = gf_strdup (uuid_utoa (volinfo->rep_brick.rb_id));
+                if (!rb_id_str) {
+                        ret = -1;
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.rb_id", count);
+                ret = dict_set_dynstr (dict, key, rb_id_str);
+                if (ret)
+                        goto out;
+                rb_id_str = NULL;
         }
 
+        snprintf (prefix, sizeof (prefix), "volume%d", count);
         ctx.dict = dict;
-        ctx.count = count;
+        ctx.prefix = prefix;
         ctx.opt_count = 1;
         ctx.key_name = "key";
         ctx.val_name = "value";
         GF_ASSERT (volinfo->dict);
 
-        dict_foreach (volinfo->dict, _add_volinfo_dict_to_prdict, &ctx);
+        dict_foreach (volinfo->dict, _add_dict_to_prdict, &ctx);
         ctx.opt_count--;
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.opt-count", count);
@@ -1551,13 +2327,13 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                 goto out;
 
         ctx.dict = dict;
-        ctx.count = count;
+        ctx.prefix = prefix;
         ctx.opt_count = 1;
         ctx.key_name = "slave-num";
         ctx.val_name = "slave-val";
         GF_ASSERT (volinfo->gsync_slaves);
 
-        dict_foreach (volinfo->gsync_slaves, _add_volinfo_dict_to_prdict, &ctx);
+        dict_foreach (volinfo->gsync_slaves, _add_dict_to_prdict, &ctx);
         ctx.opt_count--;
 
         memset (key, 0, sizeof (key));
@@ -1581,11 +2357,48 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                 if (ret)
                         goto out;
 
+                snprintf (key, sizeof (key), "volume%d.brick%d.snap_status",
+                          count, i);
+                ret = dict_set_int32 (dict, key, brickinfo->snap_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set snap_status for %s:%s",
+                                brickinfo->hostname,
+                                brickinfo->path);
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "volume%d.brick%d.device_path",
+                          count, i);
+                ret = dict_set_str (dict, key, brickinfo->device_path);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set snap_device for %s:%s",
+                                brickinfo->hostname,
+                                brickinfo->path);
+                        goto out;
+                }
+
                 i++;
         }
 
+        /* Add volume op-versions to dict. This prevents volume inconsistencies
+         * in the cluster
+         */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.op-version", count);
+        ret = dict_set_int32 (dict, key, volinfo->op_version);
+        if (ret)
+                goto out;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.client-op-version", count);
+        ret = dict_set_int32 (dict, key, volinfo->client_op_version);
 
 out:
+        GF_FREE (volume_id_str);
+        GF_FREE (rebalance_id_str);
+        GF_FREE (rb_id_str);
+
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
@@ -1599,6 +2412,7 @@ glusterd_build_volume_dict (dict_t **vols)
         glusterd_conf_t         *priv = NULL;
         glusterd_volinfo_t      *volinfo = NULL;
         int32_t                 count = 0;
+        glusterd_dict_ctx_t     ctx            = {0};
 
         priv = THIS->private;
 
@@ -1619,6 +2433,17 @@ glusterd_build_volume_dict (dict_t **vols)
         if (ret)
                 goto out;
 
+        ctx.dict = dict;
+        ctx.prefix = "global";
+        ctx.opt_count = 1;
+        ctx.key_name = "key";
+        ctx.val_name = "val";
+        dict_foreach (priv->opts, _add_dict_to_prdict, &ctx);
+        ctx.opt_count--;
+        ret = dict_set_int32 (dict, "global-opt-count", ctx.opt_count);
+        if (ret)
+                goto out;
+
         *vols = dict;
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
@@ -1629,7 +2454,8 @@ out:
 }
 
 int32_t
-glusterd_compare_friend_volume (dict_t *vols, int32_t count, int32_t *status)
+glusterd_compare_friend_volume (dict_t *vols, int32_t count, int32_t *status,
+                                char *hostname)
 {
 
         int32_t                 ret = -1;
@@ -1665,8 +2491,8 @@ glusterd_compare_friend_volume (dict_t *vols, int32_t count, int32_t *status)
                 //Mismatch detected
                 ret = 0;
                 gf_log ("", GF_LOG_ERROR, "Version of volume %s differ."
-                        "local version = %d, remote version = %d",
-                        volinfo->volname, volinfo->version, version);
+                        "local version = %d, remote version = %d on peer %s",
+                        volinfo->volname, volinfo->version, version, hostname);
                 *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
                 goto out;
         } else if (version < volinfo->version) {
@@ -1685,8 +2511,8 @@ glusterd_compare_friend_volume (dict_t *vols, int32_t count, int32_t *status)
         if (cksum != volinfo->cksum) {
                 ret = 0;
                 gf_log ("", GF_LOG_ERROR, "Cksums of volume %s differ."
-                        " local cksum = %d, remote cksum = %d",
-                        volinfo->volname, volinfo->cksum, cksum);
+                        " local cksum = %u, remote cksum = %u on peer %s",
+                        volinfo->volname, volinfo->cksum, cksum, hostname);
                 *status = GLUSTERD_VOL_COMP_RJT;
                 goto out;
         }
@@ -1700,8 +2526,8 @@ out:
 }
 
 static int32_t
-import_prdict_volinfo_dict (dict_t *vols, dict_t  *dst_dict, char *key_prefix,
-                            char *value_prefix, int opt_count, int count)
+import_prdict_dict (dict_t *vols, dict_t  *dst_dict, char *key_prefix,
+                    char *value_prefix, int opt_count, char *prefix)
 {
         char                    key[512] = {0,};
         int32_t                 ret = 0;
@@ -1713,8 +2539,8 @@ import_prdict_volinfo_dict (dict_t *vols, dict_t  *dst_dict, char *key_prefix,
 
         while (i <= opt_count) {
                 memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.%s%d",
-                          count, key_prefix, i);
+                snprintf (key, sizeof (key), "%s.%s%d",
+                          prefix, key_prefix, i);
                 ret = dict_get_str (vols, key, &opt_key);
                 if (ret) {
                         snprintf (msg, sizeof (msg), "Volume dict key not "
@@ -1723,8 +2549,8 @@ import_prdict_volinfo_dict (dict_t *vols, dict_t  *dst_dict, char *key_prefix,
                 }
 
                 memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.%s%d",
-                          count, value_prefix, i);
+                snprintf (key, sizeof (key), "%s.%s%d",
+                          prefix, value_prefix, i);
                 ret = dict_get_str (vols, key, &opt_val);
                 if (ret) {
                         snprintf (msg, sizeof (msg), "Volume dict value not "
@@ -1753,6 +2579,265 @@ out:
 
 }
 
+gf_boolean_t
+glusterd_is_quorum_option (char *option)
+{
+        gf_boolean_t    res = _gf_false;
+        int             i = 0;
+        char            *keys[] = {GLUSTERD_QUORUM_TYPE_KEY,
+                                   GLUSTERD_QUORUM_RATIO_KEY, NULL};
+
+        for (i = 0; keys[i]; i++) {
+                if (strcmp (option, keys[i]) == 0) {
+                        res = _gf_true;
+                        break;
+                }
+        }
+        return res;
+}
+
+gf_boolean_t
+glusterd_is_quorum_changed (dict_t *options, char *option, char *value)
+{
+        int             ret = 0;
+        gf_boolean_t    reconfigured = _gf_false;
+        gf_boolean_t    all = _gf_false;
+        char            *oldquorum = NULL;
+        char            *newquorum = NULL;
+        char            *oldratio = NULL;
+        char            *newratio = NULL;
+
+        if ((strcmp ("all", option) != 0) &&
+            !glusterd_is_quorum_option (option))
+                goto out;
+
+        if (strcmp ("all", option) == 0)
+                all = _gf_true;
+
+        if (all || (strcmp (GLUSTERD_QUORUM_TYPE_KEY, option) == 0)) {
+                newquorum = value;
+                ret = dict_get_str (options, GLUSTERD_QUORUM_TYPE_KEY,
+                                    &oldquorum);
+        }
+
+        if (all || (strcmp (GLUSTERD_QUORUM_RATIO_KEY, option) == 0)) {
+                newratio = value;
+                ret = dict_get_str (options, GLUSTERD_QUORUM_RATIO_KEY,
+                                    &oldratio);
+        }
+
+        reconfigured = _gf_true;
+
+        if (oldquorum && newquorum && (strcmp (oldquorum, newquorum) == 0))
+                reconfigured = _gf_false;
+        if (oldratio && newratio && (strcmp (oldratio, newratio) == 0))
+                reconfigured = _gf_false;
+
+        if ((oldratio == NULL) && (newratio == NULL) && (oldquorum == NULL) &&
+            (newquorum == NULL))
+                reconfigured = _gf_false;
+out:
+        return reconfigured;
+}
+
+static inline gf_boolean_t
+_is_contributing_to_quorum (gd_quorum_contrib_t contrib)
+{
+        if ((contrib == QUORUM_UP) || (contrib == QUORUM_DOWN))
+                return _gf_true;
+        return _gf_false;
+}
+
+static inline gf_boolean_t
+_does_quorum_meet (int active_count, int quorum_count)
+{
+        return (active_count >= quorum_count);
+}
+
+int
+glusterd_get_quorum_cluster_counts (xlator_t *this, int *active_count,
+                                    int *quorum_count)
+{
+        glusterd_peerinfo_t *peerinfo      = NULL;
+        glusterd_conf_t     *conf          = NULL;
+        int                 ret            = -1;
+        int                 inquorum_count = 0;
+        char                *val           = NULL;
+        double              quorum_percentage = 0.0;
+        gf_boolean_t        ratio          = _gf_false;
+        int                 count          = 0;
+
+        conf = this->private;
+        //Start with counting self
+        inquorum_count = 1;
+        if (active_count)
+                *active_count = 1;
+        list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
+                if (peerinfo->quorum_contrib == QUORUM_WAITING)
+                        goto out;
+
+                if (_is_contributing_to_quorum (peerinfo->quorum_contrib))
+                        inquorum_count = inquorum_count + 1;
+
+                if (active_count && (peerinfo->quorum_contrib == QUORUM_UP))
+                        *active_count = *active_count + 1;
+        }
+
+        ret = dict_get_str (conf->opts, GLUSTERD_QUORUM_RATIO_KEY, &val);
+        if (ret == 0) {
+                ratio = _gf_true;
+                ret = gf_string2percent (val, &quorum_percentage);
+                if (!ret)
+                        ratio = _gf_true;
+        }
+        if (ratio)
+                count = CEILING_POS (inquorum_count *
+                                     quorum_percentage / 100.0);
+        else
+                count = (inquorum_count * 50 / 100) + 1;
+
+        *quorum_count = count;
+        ret = 0;
+out:
+        return ret;
+}
+
+gf_boolean_t
+glusterd_is_volume_in_server_quorum (glusterd_volinfo_t *volinfo)
+{
+        gf_boolean_t    res = _gf_false;
+        char            *quorum_type = NULL;
+        int             ret = 0;
+
+        ret = dict_get_str (volinfo->dict, GLUSTERD_QUORUM_TYPE_KEY,
+                            &quorum_type);
+        if (ret)
+                goto out;
+
+        if (strcmp (quorum_type, GLUSTERD_SERVER_QUORUM) == 0)
+                res = _gf_true;
+out:
+        return res;
+}
+
+gf_boolean_t
+glusterd_is_any_volume_in_server_quorum (xlator_t *this)
+{
+        glusterd_conf_t         *conf = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+
+        conf = this->private;
+        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                if (glusterd_is_volume_in_server_quorum (volinfo)) {
+                        return _gf_true;
+                }
+        }
+        return _gf_false;
+}
+
+gf_boolean_t
+does_gd_meet_server_quorum (xlator_t *this)
+{
+        int                     quorum_count = 0;
+        int                     active_count   = 0;
+        gf_boolean_t            in = _gf_false;
+        glusterd_conf_t         *conf = NULL;
+        int                     ret = -1;
+
+        conf = this->private;
+        ret = glusterd_get_quorum_cluster_counts (this, &active_count,
+                                                  &quorum_count);
+        if (ret)
+                goto out;
+
+        if (!_does_quorum_meet (active_count, quorum_count)) {
+                goto out;
+        }
+
+        in = _gf_true;
+out:
+        return in;
+}
+
+int
+glusterd_spawn_daemons (void *opaque)
+{
+        glusterd_conf_t *conf = THIS->private;
+        gf_boolean_t    start_bricks = !conf->restart_done;
+
+        if (start_bricks) {
+                glusterd_restart_bricks (conf);
+                conf->restart_done = _gf_true;
+        }
+        glusterd_restart_gsyncds (conf);
+        glusterd_restart_rebalance (conf);
+        return 0;
+}
+
+void
+glusterd_do_volume_quorum_action (xlator_t *this, glusterd_volinfo_t *volinfo,
+                                  gf_boolean_t meets_quorum)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_conf_t         *conf      = NULL;
+
+        conf = this->private;
+        if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                goto out;
+
+        if (!glusterd_is_volume_in_server_quorum (volinfo))
+                meets_quorum = _gf_true;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (!glusterd_is_local_brick (this, volinfo, brickinfo))
+                        continue;
+                if (meets_quorum)
+                        glusterd_brick_start (volinfo, brickinfo, _gf_false);
+                else
+                        glusterd_brick_stop (volinfo, brickinfo, _gf_false);
+        }
+out:
+        return;
+}
+
+int
+glusterd_do_quorum_action ()
+{
+        xlator_t            *this          = NULL;
+        glusterd_conf_t     *conf          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                 ret            = 0;
+        int                 active_count   = 0;
+        int                 quorum_count   = 0;
+        gf_boolean_t        meets          = _gf_false;
+
+        this = THIS;
+        conf = this->private;
+
+        conf->pending_quorum_action = _gf_true;
+        ret = glusterd_lock (conf->uuid);
+        if (ret)
+                goto out;
+
+        {
+                ret = glusterd_get_quorum_cluster_counts (this, &active_count,
+                                                          &quorum_count);
+                if (ret)
+                        goto unlock;
+
+                if (_does_quorum_meet (active_count, quorum_count))
+                        meets = _gf_true;
+                list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                        glusterd_do_volume_quorum_action (this, volinfo, meets);
+                }
+        }
+unlock:
+        (void)glusterd_unlock (conf->uuid);
+        conf->pending_quorum_action = _gf_false;
+out:
+        return ret;
+}
+
 int32_t
 glusterd_import_friend_volume_opts (dict_t *vols, int count,
                                     glusterd_volinfo_t *volinfo)
@@ -1761,6 +2846,7 @@ glusterd_import_friend_volume_opts (dict_t *vols, int count,
         int32_t                 ret = -1;
         int                     opt_count = 0;
         char                    msg[2048] = {0};
+        char                    volume_prefix[1024] = {0};
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.opt-count", count);
@@ -1771,8 +2857,9 @@ glusterd_import_friend_volume_opts (dict_t *vols, int count,
                 goto out;
         }
 
-        ret = import_prdict_volinfo_dict (vols, volinfo->dict, "key",
-                                          "value", opt_count, count);
+        snprintf (volume_prefix, sizeof (volume_prefix), "volume%d", count);
+        ret = import_prdict_dict (vols, volinfo->dict, "key", "value",
+                                  opt_count, volume_prefix);
         if (ret) {
                 snprintf (msg, sizeof (msg), "Unable to import options dict "
                           "specified for %s", volinfo->volname);
@@ -1788,9 +2875,8 @@ glusterd_import_friend_volume_opts (dict_t *vols, int count,
                 goto out;
         }
 
-        ret = import_prdict_volinfo_dict (vols, volinfo->gsync_slaves,
-                                          "slave-num", "slave-val", opt_count,
-                                          count);
+        ret = import_prdict_dict (vols, volinfo->gsync_slaves, "slave-num",
+                                  "slave-val", opt_count, volume_prefix);
         if (ret) {
                 snprintf (msg, sizeof (msg), "Unable to import gsync sessions "
                           "specified for %s", volinfo->volname);
@@ -1811,6 +2897,8 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
 {
         char                    key[512] = {0,};
         int                     ret = -1;
+        int32_t                 snap_status = 0;
+        char                    *snap_device = NULL;
         char                    *hostname = NULL;
         char                    *path = NULL;
         glusterd_brickinfo_t    *new_brickinfo = NULL;
@@ -1838,12 +2926,31 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
                 goto out;
         }
 
+        snprintf (key, sizeof (key), "volume%d.brick%d.snap_status",
+                  vol_count, brick_count);
+        ret = dict_get_int32 (vols, key, &snap_status);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload", key);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "volume%d.brick%d.device_path",
+                  vol_count, brick_count);
+        ret = dict_get_str (vols, key, &snap_device);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload", key);
+                goto out;
+        }
+
         ret = glusterd_brickinfo_new (&new_brickinfo);
         if (ret)
                 goto out;
 
         strcpy (new_brickinfo->path, path);
         strcpy (new_brickinfo->hostname, hostname);
+        strcpy (new_brickinfo->device_path, snap_device);
+        new_brickinfo->snap_status = snap_status;
+
         //peerinfo might not be added yet
         (void) glusterd_resolve_brick (new_brickinfo);
         ret = 0;
@@ -1893,7 +3000,13 @@ glusterd_import_volinfo (dict_t *vols, int count,
         char               msg[2048]         = {0};
         char               *src_brick        = NULL;
         char               *dst_brick        = NULL;
+        char               *str              = NULL;
         int                rb_status         = 0;
+        char               *rebalance_id_str = NULL;
+        char               *rb_id_str        = NULL;
+        int                op_version        = 0;
+        int                client_op_version = 0;
+        uint32_t           is_snap_volume    = 0;
 
         GF_ASSERT (vols);
         GF_ASSERT (volinfo);
@@ -1905,6 +3018,22 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 goto out;
         }
 
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.is_snap_volume", count);
+        ret = dict_get_uint32 (vols, key, &is_snap_volume);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        if (is_snap_volume == _gf_true) {
+                gf_log (THIS->name, GF_LOG_DEBUG,
+                        "Not syncing snap volume %s", volname);
+                ret = 0;
+                goto out;
+        }
+
         ret = glusterd_volinfo_new (&new_volinfo);
         if (ret)
                 goto out;
@@ -2001,6 +3130,26 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 goto out;
         }
 
+        uuid_parse (volume_id_str, new_volinfo->volume_id);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.username", count);
+        ret = dict_get_str (vols, key, &str);
+        if (!ret) {
+                ret = glusterd_auth_set_username (new_volinfo, str);
+                if (ret)
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.password", count);
+        ret = dict_get_str (vols, key, &str);
+        if (!ret) {
+                ret = glusterd_auth_set_password (new_volinfo, str);
+                if (ret)
+                        goto out;
+        }
+
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.transport_type", count);
         ret = dict_get_uint32 (vols, key, &new_volinfo->transport_type);
@@ -2010,16 +3159,67 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 goto out;
         }
 
-        uuid_parse (volume_id_str, new_volinfo->volume_id);
+        new_volinfo->is_snap_volume = is_snap_volume;
+
+        snprintf (key, sizeof (key), "volume%d.is_volume_restored", count);
+        ret = dict_get_uint32 (vols, key, &new_volinfo->is_volume_restored);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get "
+                        "is_volume_restored option for %s",
+                        volname);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "volume%d.snap-max-hard-limit", count);
+        ret = dict_get_uint64 (vols, key, &new_volinfo->snap_max_hard_limit);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebalance", count);
+        ret = dict_get_uint32 (vols, key, &new_volinfo->rebal.defrag_cmd);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        if (new_volinfo->rebal.defrag_cmd) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.rebalance-id", count);
+                ret = dict_get_str (vols, key, &rebalance_id_str);
+                if (ret) {
+                        /* This is not present in older glusterfs versions,
+                         * so don't error out
+                         */
+                        ret = 0;
+                } else {
+                        uuid_parse (rebalance_id_str,
+                                    new_volinfo->rebal.rebalance_id);
+                }
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebalance-op", count);
+        ret = dict_get_uint32 (vols, key,(uint32_t *) &new_volinfo->rebal.op);
+        if (ret) {
+                /* This is not present in older glusterfs versions,
+                 * so don't error out
+                 */
+                ret = 0;
+        }
 
         memset (key, 0, sizeof (key));
         snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
         ret = dict_get_int32 (vols, key, &rb_status);
         if (ret)
                 goto out;
-        new_volinfo->rb_status = rb_status;
+        new_volinfo->rep_brick.rb_status = rb_status;
 
-        if (new_volinfo->rb_status > GF_RB_STATUS_NONE) {
+        if (new_volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
@@ -2028,8 +3228,8 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 if (ret)
                         goto out;
 
-                ret = glusterd_brickinfo_from_brick (src_brick,
-                                                     &new_volinfo->src_brick);
+                ret = glusterd_brickinfo_new_from_brick (src_brick,
+                                        &new_volinfo->rep_brick.src_brick);
                 if (ret) {
                         gf_log ("", GF_LOG_ERROR, "Unable to create"
                                 " src brickinfo");
@@ -2043,19 +3243,65 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 if (ret)
                         goto out;
 
-                ret = glusterd_brickinfo_from_brick (dst_brick,
-                                                     &new_volinfo->dst_brick);
+                ret = glusterd_brickinfo_new_from_brick (dst_brick,
+                                     &new_volinfo->rep_brick.dst_brick);
                 if (ret) {
                         gf_log ("", GF_LOG_ERROR, "Unable to create"
                                 " dst brickinfo");
                         goto out;
                 }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.rb_id", count);
+                ret = dict_get_str (vols, key, &rb_id_str);
+                if (ret) {
+                        /* This is not present in older glusterfs versions,
+                         * so don't error out
+                         */
+                        ret = 0;
+                } else {
+                        uuid_parse (rb_id_str, new_volinfo->rep_brick.rb_id);
+                }
         }
 
 
         ret = glusterd_import_friend_volume_opts (vols, count, new_volinfo);
         if (ret)
                 goto out;
+
+        /* Import the volume's op-versions if available else set it to 1.
+         * Not having op-versions implies this informtation was obtained from a
+         * op-version 1 friend (gluster-3.3), ergo the cluster is at op-version
+         * 1 and all volumes are at op-versions 1.
+         *
+         * Either both the volume op-versions should be absent or both should be
+         * present. Only one being present is a failure
+         */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.op-version", count);
+        ret = dict_get_int32 (vols, key, &op_version);
+        if (ret)
+                ret = 0;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.client-op-version", count);
+        ret = dict_get_int32 (vols, key, &client_op_version);
+        if (ret)
+                ret = 0;
+
+        if (op_version && client_op_version) {
+                new_volinfo->op_version = op_version;
+                new_volinfo->client_op_version = client_op_version;
+        } else if (((op_version == 0) && (client_op_version != 0)) ||
+                   ((op_version != 0) && (client_op_version == 0))) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "Only one volume op-version found");
+                goto out;
+        } else {
+                new_volinfo->op_version = 1;
+                new_volinfo->client_op_version = 1;
+        }
+
         ret = glusterd_import_bricks (vols, count, new_volinfo);
         if (ret)
                 goto out;
@@ -2105,8 +3351,7 @@ glusterd_volinfo_copy_brick_portinfo (glusterd_volinfo_t *new_volinfo,
                 ret = glusterd_volume_brickinfo_get (new_brickinfo->uuid,
                                                      new_brickinfo->hostname,
                                                      new_brickinfo->path,
-                                                     old_volinfo, &old_brickinfo,
-                                                     GF_PATH_COMPLETE);
+                                                     old_volinfo, &old_brickinfo);
                 if ((0 == ret) && glusterd_is_brick_started (old_brickinfo)) {
                         new_brickinfo->port = old_brickinfo->port;
                 }
@@ -2132,10 +3377,13 @@ glusterd_volinfo_stop_stale_bricks (glusterd_volinfo_t *new_volinfo,
                 ret = glusterd_volume_brickinfo_get (old_brickinfo->uuid,
                                                      old_brickinfo->hostname,
                                                      old_brickinfo->path,
-                                                     new_volinfo, &new_brickinfo,
-                                                     GF_PATH_COMPLETE);
+                                                     new_volinfo, &new_brickinfo);
                 if (ret) {
-                        ret = glusterd_brick_stop (old_volinfo, old_brickinfo);
+                        /*TODO: may need to switch to 'atomic' flavour of
+                         * brick_stop, once we make peer rpc program also
+                         * synctask enabled*/
+                        ret = glusterd_brick_stop (old_volinfo, old_brickinfo,
+                                                   _gf_false);
                         if (ret)
                                 gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
                                         "stop brick %s:%s", old_brickinfo->hostname,
@@ -2160,7 +3408,8 @@ glusterd_delete_stale_volume (glusterd_volinfo_t *stale_volinfo,
          * stop stale bricks. Stale volume information is going to be deleted.
          * Which deletes the valid brick information inside stale volinfo.
          * We dont want brick_rpc_notify to access already deleted brickinfo.
-         * Disconnect valid bricks.
+         * Disconnect all bricks from stale_volinfo (unconditionally), since
+         * they are being deleted subsequently.
          */
         if (glusterd_is_volume_started (stale_volinfo)) {
                 if (glusterd_is_volume_started (valid_volinfo)) {
@@ -2169,10 +3418,12 @@ glusterd_delete_stale_volume (glusterd_volinfo_t *stale_volinfo,
                         //Only valid bricks will be running now.
                         (void) glusterd_volinfo_copy_brick_portinfo (valid_volinfo,
                                                                      stale_volinfo);
-                        (void) glusterd_volume_disconnect_all_bricks (stale_volinfo);
+
                 } else {
                         (void) glusterd_stop_bricks (stale_volinfo);
                 }
+
+                (void) glusterd_volume_disconnect_all_bricks (stale_volinfo);
         }
         /* Delete all the bricks and stores and vol files. They will be created
          * again by the valid_volinfo. Volume store delete should not be
@@ -2182,7 +3433,7 @@ glusterd_delete_stale_volume (glusterd_volinfo_t *stale_volinfo,
         (void) glusterd_delete_all_bricks (stale_volinfo);
         if (stale_volinfo->shandle) {
                 unlink (stale_volinfo->shandle->path);
-                (void) glusterd_store_handle_destroy (stale_volinfo->shandle);
+                (void) gf_store_handle_destroy (stale_volinfo->shandle);
                 stale_volinfo->shandle = NULL;
         }
         (void) glusterd_volinfo_delete (stale_volinfo);
@@ -2209,6 +3460,12 @@ glusterd_import_friend_volume (dict_t *vols, size_t count)
         if (ret)
                 goto out;
 
+        if (!new_volinfo) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Not importing snap volume");
+                goto out;
+        }
+
         ret = glusterd_volinfo_find (new_volinfo->volname, &old_volinfo);
         if (0 == ret) {
                 (void) glusterd_delete_stale_volume (old_volinfo, new_volinfo);
@@ -2254,8 +3511,97 @@ out:
         return ret;
 }
 
+int
+glusterd_get_global_opt_version (dict_t *opts, uint32_t *version)
+{
+        int     ret = -1;
+        char    *version_str = NULL;
+
+        ret = dict_get_str (opts, GLUSTERD_GLOBAL_OPT_VERSION, &version_str);
+        if (ret)
+                goto out;
+
+        ret = gf_string2uint (version_str, version);
+        if (ret)
+                goto out;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_get_next_global_opt_version_str (dict_t *opts, char **version_str)
+{
+        int             ret = -1;
+        char            version_string[64] = {0};
+        uint32_t        version = 0;
+
+        ret = glusterd_get_global_opt_version (opts, &version);
+        if (ret)
+                goto out;
+        version++;
+        snprintf (version_string, sizeof (version_string), "%"PRIu32, version);
+        *version_str = gf_strdup (version_string);
+        if (*version_str)
+                ret = 0;
+out:
+        return ret;
+}
+
 int32_t
-glusterd_compare_friend_data (dict_t  *vols, int32_t *status)
+glusterd_import_global_opts (dict_t *friend_data)
+{
+        xlator_t        *this = NULL;
+        glusterd_conf_t *conf = NULL;
+        int             ret = -1;
+        dict_t          *import_options = NULL;
+        int             count = 0;
+        uint32_t        local_version = 0;
+        uint32_t        remote_version = 0;
+
+        this = THIS;
+        conf = this->private;
+
+        ret = dict_get_int32 (friend_data, "global-opt-count", &count);
+        if (ret) {
+                //old version peer
+                ret = 0;
+                goto out;
+        }
+
+        import_options = dict_new ();
+        if (!import_options)
+                goto out;
+        ret = import_prdict_dict (friend_data, import_options, "key", "val",
+                                  count, "global");
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to import"
+                        " global options");
+                goto out;
+        }
+
+        ret = glusterd_get_global_opt_version (conf->opts, &local_version);
+        if (ret)
+                goto out;
+        ret = glusterd_get_global_opt_version (import_options, &remote_version);
+        if (ret)
+                goto out;
+        if (remote_version > local_version) {
+                ret = glusterd_store_options (this, import_options);
+                if (ret)
+                        goto out;
+                dict_unref (conf->opts);
+                conf->opts = dict_ref (import_options);
+        }
+        ret = 0;
+out:
+        if (import_options)
+                dict_unref (import_options);
+        return ret;
+}
+
+int32_t
+glusterd_compare_friend_data (dict_t  *vols, int32_t *status, char *hostname)
 {
         int32_t                 ret = -1;
         int32_t                 count = 0;
@@ -2272,7 +3618,8 @@ glusterd_compare_friend_data (dict_t  *vols, int32_t *status)
                 goto out;
 
         while (i <= count) {
-                ret = glusterd_compare_friend_volume (vols, i, status);
+                ret = glusterd_compare_friend_volume (vols, i, status,
+                                                      hostname);
                 if (ret)
                         goto out;
 
@@ -2291,6 +3638,9 @@ glusterd_compare_friend_data (dict_t  *vols, int32_t *status)
                         stale_nfs = _gf_true;
                 if (glusterd_is_nodesvc_running ("glustershd"))
                         stale_shd = _gf_true;
+                ret = glusterd_import_global_opts (vols);
+                if (ret)
+                        goto out;
                 ret = glusterd_import_friend_volumes (vols);
                 if (ret)
                         goto out;
@@ -2387,42 +3737,52 @@ glusterd_get_nodesvc_volfile (char *server, char *workdir,
 }
 
 void
-glusterd_shd_set_running (gf_boolean_t status)
+glusterd_nodesvc_set_online_status (char *server, gf_boolean_t status)
 {
         glusterd_conf_t *priv = NULL;
 
+        GF_ASSERT (server);
         priv = THIS->private;
         GF_ASSERT (priv);
         GF_ASSERT (priv->shd);
+        GF_ASSERT (priv->nfs);
 
-        priv->shd->running = status;
+        if (!strcmp("glustershd", server))
+                priv->shd->online = status;
+        else if (!strcmp ("nfs", server))
+                priv->nfs->online = status;
 }
 
 gf_boolean_t
-glusterd_shd_is_running ()
+glusterd_is_nodesvc_online (char *server)
 {
         glusterd_conf_t *conf = NULL;
+        gf_boolean_t    online = _gf_false;
 
+        GF_ASSERT (server);
         conf = THIS->private;
         GF_ASSERT (conf);
         GF_ASSERT (conf->shd);
+        GF_ASSERT (conf->nfs);
 
-        return conf->shd->running;
+        if (!strcmp (server, "glustershd"))
+                online = conf->shd->online;
+        else if (!strcmp (server, "nfs"))
+                online = conf->nfs->online;
+
+        return online;
 }
 
 int32_t
-glusterd_shd_set_socket_filepath (char *rundir, uuid_t uuid,
-                                  char *socketpath, int len)
+glusterd_nodesvc_set_socket_filepath (char *rundir, uuid_t uuid,
+                                      char *socketpath, int len)
 {
         char                    sockfilepath[PATH_MAX] = {0,};
-        char                    md5_str[PATH_MAX] = {0,};
 
         snprintf (sockfilepath, sizeof (sockfilepath), "%s/run-%s",
                   rundir, uuid_utoa (uuid));
-        _get_md5_str (md5_str, sizeof (md5_str),
-                      (uint8_t *)sockfilepath, sizeof (sockfilepath));
-        snprintf (socketpath, len, "%s/%s.socket", glusterd_sock_dir,
-                  md5_str);
+
+        glusterd_set_socket_filepath (sockfilepath, socketpath, len);
         return 0;
 }
 
@@ -2432,6 +3792,9 @@ glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node)
         struct rpc_clnt *rpc = NULL;
         glusterd_brickinfo_t    *brickinfo = NULL;
         nodesrv_t               *shd       = NULL;
+        glusterd_volinfo_t      *volinfo   = NULL;
+        nodesrv_t               *nfs       = NULL;
+
         GF_VALIDATE_OR_GOTO (THIS->name, pending_node, out);
         GF_VALIDATE_OR_GOTO (THIS->name, pending_node->node, out);
 
@@ -2443,6 +3806,15 @@ glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node)
                 shd       = pending_node->node;
                 rpc       = shd->rpc;
 
+        } else if (pending_node->type == GD_NODE_REBALANCE) {
+                volinfo = pending_node->node;
+                if (volinfo->rebal.defrag)
+                        rpc = volinfo->rebal.defrag->rpc;
+
+        } else if (pending_node->type == GD_NODE_NFS) {
+                nfs = pending_node->node;
+                rpc = nfs->rpc;
+
         } else {
                 GF_ASSERT (0);
         }
@@ -2452,19 +3824,27 @@ out:
 }
 
 struct rpc_clnt*
-glusterd_shd_get_rpc (void)
+glusterd_nodesvc_get_rpc (char *server)
 {
         glusterd_conf_t *priv   = NULL;
+        struct rpc_clnt *rpc    = NULL;
 
+        GF_ASSERT (server);
         priv = THIS->private;
         GF_ASSERT (priv);
         GF_ASSERT (priv->shd);
+        GF_ASSERT (priv->nfs);
 
-        return priv->shd->rpc;
+        if (!strcmp (server, "glustershd"))
+                rpc = priv->shd->rpc;
+        else if (!strcmp (server, "nfs"))
+                rpc = priv->nfs->rpc;
+
+        return rpc;
 }
 
 int32_t
-glusterd_shd_set_rpc (struct rpc_clnt *rpc)
+glusterd_nodesvc_set_rpc (char *server, struct rpc_clnt *rpc)
 {
         int             ret   = 0;
         xlator_t        *this = NULL;
@@ -2475,47 +3855,77 @@ glusterd_shd_set_rpc (struct rpc_clnt *rpc)
         priv = this->private;
         GF_ASSERT (priv);
         GF_ASSERT (priv->shd);
+        GF_ASSERT (priv->nfs);
 
-        priv->shd->rpc = rpc;
+        if (!strcmp ("glustershd", server))
+                priv->shd->rpc = rpc;
+        else if (!strcmp ("nfs", server))
+                priv->nfs->rpc = rpc;
 
         return ret;
 }
 
 int32_t
-glusterd_shd_connect (char *socketpath) {
+glusterd_nodesvc_connect (char *server, char *socketpath) {
         int                     ret = 0;
         dict_t                  *options = NULL;
         struct rpc_clnt         *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
 
-        ret = rpc_clnt_transport_unix_options_build (&options, socketpath);
-        if (ret)
-                goto out;
-        ret = glusterd_rpc_create (&rpc, options,
-                                   glusterd_shd_rpc_notify,
-                                   NULL);
-        if (ret)
-                goto out;
-        (void) glusterd_shd_set_rpc (rpc);
+        rpc = glusterd_nodesvc_get_rpc (server);
+
+        if (rpc == NULL) {
+                /* Setting frame-timeout to 10mins (600seconds).
+                 * Unix domain sockets ensures that the connection is reliable.
+                 * The default timeout of 30mins used for unreliable network
+                 * connections is too long for unix domain socket connections.
+                 */
+                ret = rpc_transport_unix_options_build (&options, socketpath,
+                                                        600);
+                if (ret)
+                        goto out;
+                synclock_unlock (&priv->big_lock);
+                ret = glusterd_rpc_create (&rpc, options,
+                                           glusterd_nodesvc_rpc_notify,
+                                           server);
+                synclock_lock (&priv->big_lock);
+                if (ret)
+                        goto out;
+                (void) glusterd_nodesvc_set_rpc (server, rpc);
+        }
 out:
         return ret;
 }
 
 int32_t
-glusterd_nodesvc_start (char *server, gf_boolean_t pmap_signin)
+glusterd_nodesvc_disconnect (char *server)
 {
-        int32_t                 ret = -1;
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
-        runner_t                runner = {0,};
-        char                    pidfile[PATH_MAX] = {0,};
-        char                    logfile[PATH_MAX] = {0,};
-        char                    volfile[PATH_MAX] = {0,};
-        char                    rundir[PATH_MAX] = {0,};
-        char                    shd_sockfpath[PATH_MAX] = {0,};
-        char                    volfileid[256]   = {0};
-#ifdef DEBUG
+        struct rpc_clnt         *rpc = NULL;
+
+        rpc = glusterd_nodesvc_get_rpc (server);
+        (void)glusterd_nodesvc_set_rpc (server, NULL);
+
+        if (rpc)
+                rpc_clnt_unref (rpc);
+
+        return 0;
+}
+
+int32_t
+glusterd_nodesvc_start (char *server)
+{
+        int32_t                 ret                        = -1;
+        xlator_t               *this                       = NULL;
+        glusterd_conf_t        *priv                       = NULL;
+        runner_t                runner                     = {0,};
+        char                    pidfile[PATH_MAX]          = {0,};
+        char                    logfile[PATH_MAX]          = {0,};
+        char                    volfile[PATH_MAX]          = {0,};
+        char                    rundir[PATH_MAX]           = {0,};
+        char                    sockfpath[PATH_MAX] = {0,};
+        char                    volfileid[256]             = {0};
+        char                    glusterd_uuid_option[1024] = {0};
         char                    valgrind_logfile[PATH_MAX] = {0};
-#endif
 
         this = THIS;
         GF_ASSERT(this);
@@ -2523,7 +3933,7 @@ glusterd_nodesvc_start (char *server, gf_boolean_t pmap_signin)
         priv = this->private;
 
         glusterd_get_nodesvc_rundir (server, priv->workdir,
-                                           rundir, sizeof (rundir));
+                                     rundir, sizeof (rundir));
         ret = mkdir (rundir, 0777);
 
         if ((ret == -1) && (EEXIST != errno)) {
@@ -2533,9 +3943,9 @@ glusterd_nodesvc_start (char *server, gf_boolean_t pmap_signin)
         }
 
         glusterd_get_nodesvc_pidfile (server, priv->workdir,
-                                            pidfile, sizeof (pidfile));
+                                      pidfile, sizeof (pidfile));
         glusterd_get_nodesvc_volfile (server, priv->workdir,
-                                            volfile, sizeof (volfile));
+                                      volfile, sizeof (volfile));
         ret = access (volfile, F_OK);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "%s Volfile %s is not present",
@@ -2547,46 +3957,42 @@ glusterd_nodesvc_start (char *server, gf_boolean_t pmap_signin)
                   server);
         snprintf (volfileid, sizeof (volfileid), "gluster/%s", server);
 
-        if (!strcmp (server, "glustershd")) {
-                glusterd_shd_set_socket_filepath (rundir,
-                                                  priv->uuid,
-                                                  shd_sockfpath,
-                                                  sizeof (shd_sockfpath));
-        }
+        glusterd_nodesvc_set_socket_filepath (rundir, MY_UUID,
+                                              sockfpath, sizeof (sockfpath));
 
         runinit (&runner);
-        //TODO: kp:change the assumption that shd is the one which signs in
-        // use runner_add_args?
-#ifdef DEBUG
+
         if (priv->valgrind) {
                 snprintf (valgrind_logfile, PATH_MAX,
-                          "%s/valgrnd-%s.log",
+                          "%s/valgrind-%s.log",
                           DEFAULT_LOG_FILE_DIRECTORY,
                           server);
 
                 runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                 "--trace-children=yes", NULL);
+                                 "--trace-children=yes", "--track-origins=yes",
+                                 NULL);
                 runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
         }
-#endif
 
-        if (pmap_signin) {
-                runner_add_args (&runner, SBIN_DIR"/glusterfs", "-s",
-                                 "localhost", "--volfile-id", volfileid,
-                                 "-p", pidfile, "-l", logfile,
-                                 "-S", shd_sockfpath, NULL);
-        } else {
-                runner_add_args (&runner, SBIN_DIR"/glusterfs", "-f", volfile,
-                                 "-p", pidfile, "-l", logfile, NULL);
-        }
+        runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                         "-s", "localhost",
+                         "--volfile-id", volfileid,
+                         "-p", pidfile,
+                         "-l", logfile,
+                         "-S", sockfpath, NULL);
 
-        runner_log (&runner, "", GF_LOG_DEBUG, "Starting the nfs/glustershd "
-                    "services");
+        if (!strcmp (server, "glustershd")) {
+                snprintf (glusterd_uuid_option, sizeof (glusterd_uuid_option),
+                          "*replicate*.node-uuid=%s", uuid_utoa (MY_UUID));
+                runner_add_args (&runner, "--xlator-option",
+                                 glusterd_uuid_option, NULL);
+        }
+        runner_log (&runner, "", GF_LOG_DEBUG,
+                    "Starting the nfs/glustershd services");
 
-        ret = runner_run (&runner);
+        ret = runner_run_nowait (&runner);
         if (ret == 0) {
-                if (pmap_signin)
-                        glusterd_shd_connect (shd_sockfpath);
+                glusterd_nodesvc_connect (server, sockfpath);
         }
 out:
         return ret;
@@ -2595,13 +4001,13 @@ out:
 int
 glusterd_nfs_server_start ()
 {
-        return glusterd_nodesvc_start ("nfs", _gf_false);
+        return glusterd_nodesvc_start ("nfs");
 }
 
 int
 glusterd_shd_start ()
 {
-        return glusterd_nodesvc_start ("glustershd", _gf_true);
+        return glusterd_nodesvc_start ("glustershd");
 }
 
 gf_boolean_t
@@ -2616,6 +4022,31 @@ glusterd_is_nodesvc_running (char *server)
 }
 
 int32_t
+glusterd_nodesvc_unlink_socket_file (char *server)
+{
+        int             ret = 0;
+        char            sockfpath[PATH_MAX] = {0,};
+        char            rundir[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = THIS->private;
+
+        glusterd_get_nodesvc_rundir (server, priv->workdir,
+                                     rundir, sizeof (rundir));
+
+        glusterd_nodesvc_set_socket_filepath (rundir, MY_UUID,
+                                              sockfpath, sizeof (sockfpath));
+
+        ret = unlink (sockfpath);
+        if (ret && (ENOENT == errno)) {
+                ret = 0;
+        } else {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to remove %s"
+                        " error: %s", sockfpath, strerror (errno));
+        }
+
+        return ret;
+}
+
+int32_t
 glusterd_nodesvc_stop (char *server, int sig)
 {
         char                    pidfile[PATH_MAX] = {0,};
@@ -2624,9 +4055,17 @@ glusterd_nodesvc_stop (char *server, int sig)
 
         if (!glusterd_is_nodesvc_running (server))
                 goto out;
+
+        (void)glusterd_nodesvc_disconnect (server);
+
         glusterd_get_nodesvc_pidfile (server, priv->workdir,
                                             pidfile, sizeof (pidfile));
         ret = glusterd_service_stop (server, pidfile, sig, _gf_true);
+
+        if (ret == 0) {
+                glusterd_nodesvc_set_online_status (server, _gf_false);
+                (void)glusterd_nodesvc_unlink_socket_file (server);
+        }
 out:
         return ret;
 }
@@ -2649,6 +4088,20 @@ glusterd_nfs_pmap_deregister ()
         else
                 gf_log ("", GF_LOG_ERROR, "De-register NFSV3 is unsuccessful");
 
+        if (pmap_unset (NLM_PROGRAM, NLMV4_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered NLM v4 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-registration of NLM v4 failed");
+
+        if (pmap_unset (NLM_PROGRAM, NLMV1_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered NLM v1 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-registration of NLM v1 failed");
+
+        if (pmap_unset (ACL_PROGRAM, ACLV3_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered ACL v3 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-registration of ACL v3 failed");
 }
 
 int
@@ -2675,6 +4128,83 @@ glusterd_shd_stop ()
 }
 
 int
+glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
+                           dict_t *vol_opts)
+{
+        int                     ret = -1;
+        glusterd_conf_t         *priv = THIS->private;
+        char                    pidfile[PATH_MAX] = {0,};
+        gf_boolean_t            running = _gf_false;
+        int                     pid = -1;
+        int                     port = 0;
+        char                    key[1024] = {0,};
+
+        glusterd_get_nodesvc_pidfile (server, priv->workdir, pidfile,
+                                      sizeof (pidfile));
+        //Consider service to be running only when glusterd sees it Online
+        if (glusterd_is_nodesvc_online (server))
+                running = glusterd_is_service_running (pidfile, &pid);
+
+        /* For nfs-servers/self-heal-daemon setting
+         * brick<n>.hostname = "NFS Server" / "Self-heal Daemon"
+         * brick<n>.path = uuid
+         * brick<n>.port = 0
+         *
+         * This might be confusing, but cli displays the name of
+         * the brick as hostname+path, so this will make more sense
+         * when output.
+         */
+        snprintf (key, sizeof (key), "brick%d.hostname", count);
+        if (!strcmp (server, "nfs"))
+                ret = dict_set_str (dict, key, "NFS Server");
+        else if (!strcmp (server, "glustershd"))
+                ret = dict_set_str (dict, key, "Self-heal Daemon");
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.path", count);
+        ret = dict_set_dynstr (dict, key, gf_strdup (uuid_utoa (MY_UUID)));
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.port", count);
+        /* Port is available only for the NFS server.
+         * Self-heal daemon doesn't provide any port for access
+         * by entities other than gluster.
+         */
+        if (!strcmp (server, "nfs")) {
+                if (dict_get (vol_opts, "nfs.port")) {
+                        ret = dict_get_int32 (vol_opts, "nfs.port", &port);
+                        if (ret)
+                                goto out;
+                } else
+                        port = GF_NFS3_PORT;
+        }
+        ret = dict_set_int32 (dict, key, port);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.pid", count);
+        ret = dict_set_int32 (dict, key, pid);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.status", count);
+        ret = dict_set_int32 (dict, key, running);
+        if (ret)
+                goto out;
+
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
 glusterd_remote_hostname_get (rpcsvc_request_t *req, char *remote_host, int len)
 {
         GF_ASSERT (req);
@@ -2702,8 +4232,7 @@ glusterd_remote_hostname_get (rpcsvc_request_t *req, char *remote_host, int len)
 
 
 out:
-        if (tmp_host)
-                GF_FREE (tmp_host);
+        GF_FREE (tmp_host);
         return ret;
 }
 
@@ -2748,6 +4277,58 @@ glusterd_reconfigure_shd ()
 }
 
 int
+glusterd_reconfigure_nfs ()
+{
+        int             ret             = -1;
+        gf_boolean_t    identical       = _gf_false;
+
+        /*
+         * Check both OLD and NEW volfiles, if they are SAME by size
+         * and cksum i.e. "character-by-character". If YES, then
+         * NOTHING has been changed, just return.
+         */
+        ret = glusterd_check_nfs_volfile_identical (&identical);
+        if (ret)
+                goto out;
+
+        if (identical) {
+                ret = 0;
+                goto out;
+        }
+
+        /*
+         * They are not identical. Find out if the topology is changed
+         * OR just the volume options. If just the options which got
+         * changed, then inform the xlator to reconfigure the options.
+         */
+        identical = _gf_false; /* RESET the FLAG */
+        ret = glusterd_check_nfs_topology_identical (&identical);
+        if (ret)
+                goto out;
+
+        /* Topology is not changed, but just the options. But write the
+         * options to NFS volfile, so that NFS will be reconfigured.
+         */
+        if (identical) {
+                ret = glusterd_create_nfs_volfile();
+                if (ret == 0) {/* Only if above PASSES */
+                        ret = glusterd_fetchspec_notify (THIS);
+                }
+                goto out;
+        }
+
+        /*
+         * NFS volfile's topology has been changed. NFS server needs
+         * to be RESTARTED to ACT on the changed volfile.
+         */
+        ret = glusterd_check_generate_start_nfs ();
+
+out:
+        return ret;
+}
+
+
+int
 glusterd_check_generate_start_nfs ()
 {
         int ret = 0;
@@ -2795,8 +4376,8 @@ int
 glusterd_nodesvcs_start (glusterd_volinfo_t *volinfo)
 {
         return glusterd_nodesvcs_batch_op (volinfo,
-                                            glusterd_nfs_server_start,
-                                            glusterd_shd_start);
+                                           glusterd_nfs_server_start,
+                                           glusterd_shd_start);
 }
 
 int
@@ -2871,7 +4452,7 @@ int
 glusterd_nodesvcs_handle_reconfigure (glusterd_volinfo_t *volinfo)
 {
         return glusterd_nodesvcs_batch_op (volinfo,
-                                           glusterd_check_generate_start_nfs,
+                                           glusterd_reconfigure_nfs,
                                            glusterd_reconfigure_shd);
 }
 
@@ -2917,9 +4498,9 @@ glusterd_brickinfo_get (uuid_t uuid, char *hostname, char *path,
         list_for_each_entry (volinfo, &priv->volumes, vol_list) {
 
                 ret = glusterd_volume_brickinfo_get (uuid, hostname, path,
-                                                     volinfo, brickinfo,
-                                                     GF_PATH_COMPLETE);
-                if (!ret)
+                                                     volinfo, brickinfo);
+                if (ret == 0)
+                        /*Found*/
                         goto out;
         }
 out:
@@ -2928,7 +4509,8 @@ out:
 
 int
 glusterd_brick_start (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *brickinfo)
+                      glusterd_brickinfo_t *brickinfo,
+                      gf_boolean_t wait)
 {
         int                                     ret   = -1;
         xlator_t                                *this = NULL;
@@ -2945,61 +4527,83 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo,
         if (uuid_is_null (brickinfo->uuid)) {
                 ret = glusterd_resolve_brick (brickinfo);
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "cannot resolve brick: %s:%s",
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
                                 brickinfo->hostname, brickinfo->path);
                         goto out;
                 }
         }
 
-        if (uuid_compare (brickinfo->uuid, conf->uuid)) {
+        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
                 ret = 0;
                 goto out;
         }
-        ret = glusterd_volume_start_glusterfs (volinfo, brickinfo);
+        ret = glusterd_volume_start_glusterfs (volinfo, brickinfo, wait);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to start "
-                        "glusterfs, ret: %d", ret);
+                gf_log (this->name, GF_LOG_ERROR, "Unable to start brick %s:%s",
+                        brickinfo->hostname, brickinfo->path);
                 goto out;
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d ", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
         return ret;
 }
 
 int
 glusterd_restart_bricks (glusterd_conf_t *conf)
 {
-        glusterd_volinfo_t       *volinfo = NULL;
-        glusterd_brickinfo_t     *brickinfo = NULL;
-        int                      ret = 0;
-        gf_boolean_t             start_nodesvcs = _gf_false;
+        int                   ret            = 0;
+        glusterd_volinfo_t   *volinfo        = NULL;
+        glusterd_brickinfo_t *brickinfo      = NULL;
+        glusterd_snap_t      *snap           = NULL;
+        gf_boolean_t          start_nodesvcs = _gf_false;
+        xlator_t             *this           = NULL;
 
-        GF_ASSERT (conf);
+        this = THIS;
+        GF_ASSERT (this);
 
         list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                //If volume status is not started, do not proceed
-                if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+                if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                        continue;
+                start_nodesvcs = _gf_true;
+                gf_log (this->name, GF_LOG_DEBUG, "starting the volume %s",
+                        volinfo->volname);
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        glusterd_brick_start (volinfo, brickinfo, _gf_false);
+                }
+        }
+
+        list_for_each_entry (snap, &conf->snapshots, snap_list) {
+                list_for_each_entry (volinfo, &snap->volumes, vol_list) {
+                        if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                                continue;
+                        start_nodesvcs = _gf_true;
+                        gf_log (this->name, GF_LOG_DEBUG, "starting the snap "
+                                "volume %s", volinfo->volname);
                         list_for_each_entry (brickinfo, &volinfo->bricks,
                                              brick_list) {
-                                glusterd_brick_start (volinfo, brickinfo);
+                                glusterd_brick_start (volinfo, brickinfo,
+                                                      _gf_false);
                         }
-                        start_nodesvcs = _gf_true;
                 }
         }
+
         if (start_nodesvcs)
                 glusterd_nodesvcs_handle_graph_change (NULL);
+
         return ret;
 }
 
-void
+int
 _local_gsyncd_start (dict_t *this, char *key, data_t *value, void *data)
 {
+        char                        *path_list = NULL;
         char                        *slave = NULL;
         int                          uuid_len = 0;
+        int                          ret = 0;
         char                         uuid_str[64] = {0};
         glusterd_volinfo_t           *volinfo = NULL;
+        char                         *conf_path = NULL;
 
         volinfo = data;
         GF_ASSERT (volinfo);
@@ -3007,12 +4611,28 @@ _local_gsyncd_start (dict_t *this, char *key, data_t *value, void *data)
         if (slave)
                 slave ++;
         else
-                return;
+                return 0;
         uuid_len = (slave - value->data - 1);
 
-
         strncpy (uuid_str, (char*)value->data, uuid_len);
-        glusterd_start_gsync (volinfo, slave, uuid_str, NULL);
+
+        ret = glusterd_get_local_brickpaths (volinfo, &path_list);
+
+        ret = dict_get_str (this, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
+
+        glusterd_start_gsync (volinfo, slave, path_list, conf_path,
+                              uuid_str, NULL);
+
+        GF_FREE (path_list);
+        path_list = NULL;
+
+out:
+        return ret;
 }
 
 int
@@ -3036,6 +4656,15 @@ glusterd_restart_gsyncds (glusterd_conf_t *conf)
         return ret;
 }
 
+inline int
+glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo)
+{
+    int rcount = volinfo->replica_count;
+    int scount = volinfo->stripe_count;
+
+    return (rcount ? rcount : 1) * (scount ? scount : 1);
+}
+
 int
 glusterd_get_brickinfo (xlator_t *this, const char *brickname, int port,
                         gf_boolean_t localhost, glusterd_brickinfo_t **brickinfo)
@@ -3052,7 +4681,7 @@ glusterd_get_brickinfo (xlator_t *this, const char *brickname, int port,
         list_for_each_entry (volinfo, &priv->volumes, vol_list) {
                 list_for_each_entry (tmpbrkinfo, &volinfo->bricks,
                                      brick_list) {
-                        if (localhost && glusterd_is_local_addr (tmpbrkinfo->hostname))
+                        if (localhost && !gf_is_local_addr (tmpbrkinfo->hostname))
                                 continue;
                         if (!strcmp(tmpbrkinfo->path, brickname) &&
                             (tmpbrkinfo->port == port)) {
@@ -3064,9 +4693,23 @@ glusterd_get_brickinfo (xlator_t *this, const char *brickname, int port,
         return ret;
 }
 
+glusterd_brickinfo_t*
+glusterd_get_brickinfo_by_position (glusterd_volinfo_t *volinfo, uint32_t pos)
+{
+        glusterd_brickinfo_t    *tmpbrkinfo = NULL;
+
+        list_for_each_entry (tmpbrkinfo, &volinfo->bricks,
+                             brick_list) {
+                if (pos == 0)
+                        return tmpbrkinfo;
+                pos--;
+        }
+        return NULL;
+}
+
 void
 glusterd_set_brick_status (glusterd_brickinfo_t  *brickinfo,
-                            gf_brick_status_t status)
+                           gf_brick_status_t status)
 {
         GF_ASSERT (brickinfo);
         brickinfo->status = status;
@@ -3109,26 +4752,28 @@ out:
         return -1;
 }
 
+#ifdef GF_LINUX_HOST_OS
 int
 glusterd_get_brick_root (char *path, char **mount_point)
 {
         char           *ptr            = NULL;
+        char           *mnt_pt         = NULL;
         struct stat     brickstat      = {0};
         struct stat     buf            = {0};
 
         if (!path)
                 goto err;
-        *mount_point = gf_strdup (path);
-        if (!*mount_point)
+        mnt_pt = gf_strdup (path);
+        if (!mnt_pt)
                 goto err;
-        if (stat (*mount_point, &brickstat))
+        if (stat (mnt_pt, &brickstat))
                 goto err;
 
-        while ((ptr = strrchr (*mount_point, '/')) &&
-               ptr != *mount_point) {
+        while ((ptr = strrchr (mnt_pt, '/')) &&
+               ptr != mnt_pt) {
 
                 *ptr = '\0';
-                if (stat (*mount_point, &buf)) {
+                if (stat (mnt_pt, &buf)) {
                         gf_log (THIS->name, GF_LOG_ERROR, "error in "
                                 "stat: %s", strerror (errno));
                         goto err;
@@ -3140,35 +4785,58 @@ glusterd_get_brick_root (char *path, char **mount_point)
                 }
         }
 
-        if (ptr == *mount_point) {
+        if (ptr == mnt_pt) {
                 if (stat ("/", &buf)) {
                         gf_log (THIS->name, GF_LOG_ERROR, "error in "
                                 "stat: %s", strerror (errno));
                         goto err;
                 }
                 if (brickstat.st_dev == buf.st_dev)
-                        strcpy (*mount_point, "/");
+                        strcpy (mnt_pt, "/");
         }
 
+        *mount_point = mnt_pt;
         return 0;
 
  err:
-        if (*mount_point)
-                GF_FREE (*mount_point);
+        GF_FREE (mnt_pt);
         return -1;
 }
 
+static char*
+glusterd_parse_inode_size (char *stream, char *pattern)
+{
+        char *needle = NULL;
+        char *trail  = NULL;
+
+        needle = strstr (stream, pattern);
+        if (!needle)
+                goto out;
+
+        needle = nwstrtail (needle, pattern);
+
+        trail = needle;
+        while (trail && isdigit (*trail)) trail++;
+        if (trail)
+                *trail = '\0';
+
+out:
+        return needle;
+}
+
 static int
 glusterd_add_inode_size_to_dict (dict_t *dict, int count)
 {
         int             ret               = -1;
-        int             fd                = -1;
         char            key[1024]         = {0};
         char            buffer[4096]      = {0};
-        char            cmd_str[4096]     = {0};
         char           *inode_size        = NULL;
         char           *device            = NULL;
         char           *fs_name           = NULL;
+        char           *cur_word          = NULL;
+        char           *pattern           = NULL;
+        char           *trail             = NULL;
+        runner_t        runner            = {0, };
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.device", count);
@@ -3182,26 +4850,18 @@ glusterd_add_inode_size_to_dict (dict_t *dict, int count)
         if (ret)
                 goto out;
 
+        runinit (&runner);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
         /* get inode size for xfs or ext2/3/4 */
         if (!strcmp (fs_name, "xfs")) {
 
-                snprintf (cmd_str, sizeof (cmd_str),
-                          "xfs_info %s | "
-                          "grep isize | "
-                          "cut -d ' ' -f 2-  | "
-                          "cut -d '=' -f 2 | "
-                          "cut -d ' ' -f 1 "
-                          "> /tmp/gf_status.txt ",
-                          device);
+                runner_add_args (&runner, "xfs_info", device, NULL);
+                pattern = "isize=";
 
         } else if (IS_EXT_FS(fs_name)) {
 
-                snprintf (cmd_str, sizeof (cmd_str),
-                          "tune2fs -l %s | "
-                          "grep -i 'inode size' | "
-                          "awk '{print $3}' "
-                          "> /tmp/gf_status.txt ",
-                          device);
+                runner_add_args (&runner, "tune2fs", "-l", device, NULL);
+                pattern = "Inode size:";
 
         } else {
                 ret = 0;
@@ -3211,7 +4871,7 @@ glusterd_add_inode_size_to_dict (dict_t *dict, int count)
                 goto out;
         }
 
-        ret = runcmd ("/bin/sh", "-c", cmd_str, NULL);
+        ret = runner_start (&runner);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_ERROR, "could not get inode "
                         "size for %s : %s package missing", fs_name,
@@ -3220,53 +4880,85 @@ glusterd_add_inode_size_to_dict (dict_t *dict, int count)
                 goto out;
         }
 
-        fd = open ("/tmp/gf_status.txt", O_RDONLY);
-        unlink ("/tmp/gf_status.txt");
-        if (fd < 0) {
-                ret = -1;
+        for (;;) {
+                if (fgets (buffer, sizeof (buffer),
+                    runner_chio (&runner, STDOUT_FILENO)) == NULL)
+                        break;
+                trail = strrchr (buffer, '\n');
+                if (trail)
+                        *trail = '\0';
+
+                cur_word = glusterd_parse_inode_size (buffer, pattern);
+                if (cur_word)
+                        break;
+        }
+
+        ret = runner_end (&runner);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "%s exited with non-zero "
+                        "exit status", ((!strcmp (fs_name, "xfs")) ?
+                        "xfs_info" : "tune2fs"));
                 goto out;
         }
-        memset (buffer, 0, sizeof (buffer));
-        ret = read (fd, buffer, sizeof (buffer));
-        if (ret < 2) {
+        if (!cur_word) {
                 ret = -1;
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to retrieve inode "
+                        "size using %s",
+                        (!strcmp (fs_name, "xfs")? "xfs_info": "tune2fs"));
                 goto out;
         }
 
+        inode_size = gf_strdup (cur_word);
+
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.inode_size", count);
 
-        inode_size = get_nth_word (buffer, 1);
-        if (!inode_size) {
-                ret = -1;
-                goto out;
-        }
-
         ret = dict_set_dynstr (dict, key, inode_size);
 
  out:
-        if (fd >= 0)
-                close (fd);
         if (ret)
                 gf_log (THIS->name, GF_LOG_ERROR, "failed to get inode size");
         return ret;
 }
 
+struct mntent *
+glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab)
+{
+        struct mntent  *entry                = NULL;
+
+        mtab = setmntent (_PATH_MOUNTED, "r");
+        if (!mtab)
+                goto out;
+
+        entry = getmntent (mtab);
+
+        while (1) {
+                if (!entry)
+                        goto out;
+
+                if (!strcmp (entry->mnt_dir, mnt_pt) &&
+                    strcmp (entry->mnt_type, "rootfs"))
+                        break;
+                entry = getmntent (mtab);
+        }
+
+out:
+        return entry;
+}
+
 static int
 glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
                                   dict_t *dict, int count)
 {
         int             ret                  = -1;
-        int             fd                   = -1;
         char            key[1024]            = {0};
         char            base_key[1024]       = {0};
-        char            buffer[4096]         = {0};
-        char            cmd_str[1024]        = {0};
         char           *mnt_pt               = NULL;
         char           *fs_name              = NULL;
         char           *mnt_options          = NULL;
         char           *device               = NULL;
-        runner_t        runner               = {0};
+        struct mntent  *entry                = NULL;
+        FILE           *mtab                 = NULL;
 
         snprintf (base_key, sizeof (base_key), "brick%d", count);
 
@@ -3274,33 +4966,17 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
         if (ret)
                 goto out;
 
-        /* get mount details of brick in back-end */
-        snprintf (cmd_str, sizeof (cmd_str), " %s ", mnt_pt);
-
-        runinit (&runner);
-        runner_add_args (&runner, "grep", cmd_str, "/etc/mtab", NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-
-        ret = runner_start (&runner);
-        if (ret)
-                goto out;
-
-        if (!fgets (buffer, sizeof(buffer),
-                    runner_chio (&runner, STDOUT_FILENO))) {
+        entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+        if (!entry) {
                 ret = -1;
                 goto out;
         }
 
-        runner_end (&runner);
-
         /* get device file */
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "%s.device", base_key);
 
-        device = get_nth_word (buffer, 1);
-        if (!device)
-                goto out;
-
+        device = gf_strdup (entry->mnt_fsname);
         ret = dict_set_dynstr (dict, key, device);
         if (ret)
                 goto out;
@@ -3309,10 +4985,7 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "%s.fs_name", base_key);
 
-        fs_name = get_nth_word (buffer, 3);
-        if (!fs_name)
-                goto out;
-
+        fs_name = gf_strdup (entry->mnt_type);
         ret = dict_set_dynstr (dict, key, fs_name);
         if (ret)
                 goto out;
@@ -3321,19 +4994,57 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "%s.mnt_options", base_key);
 
-        mnt_options = get_nth_word (buffer, 4);
-        if (!mnt_options)
-                goto out;
+        mnt_options = gf_strdup (entry->mnt_opts);
         ret = dict_set_dynstr (dict, key, mnt_options);
 
  out:
-        if (mnt_pt)
-                GF_FREE (mnt_pt);
-        if (fd >= 0)
-                close (fd);
+        GF_FREE (mnt_pt);
+        if (mtab)
+                endmntent (mtab);
+
         return ret;
 }
 
+char*
+glusterd_get_brick_mount_details (glusterd_brickinfo_t *brickinfo)
+{
+        int             ret                  = -1;
+        char           *mnt_pt               = NULL;
+        char           *device               = NULL;
+        FILE           *mtab                 = NULL;
+        struct mntent  *entry                = NULL;
+        xlator_t       *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (brickinfo);
+
+        ret = glusterd_get_brick_root (brickinfo->path, &mnt_pt);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get mount point "
+                        "for %s brick", brickinfo->path);
+                goto out;
+        }
+
+        entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+        if (NULL == entry) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get mnt entry "
+                        "for %s mount path", mnt_pt);
+                goto out;
+        }
+
+        /* get the fs_name/device */
+        device = gf_strdup (entry->mnt_fsname);
+
+out:
+        if (NULL != mtab) {
+                endmntent (mtab);
+        }
+
+        return device;
+}
+#endif
+
 int
 glusterd_add_brick_detail_to_dict (glusterd_volinfo_t *volinfo,
                                    glusterd_brickinfo_t *brickinfo,
@@ -3348,7 +5059,9 @@ glusterd_add_brick_detail_to_dict (glusterd_volinfo_t *volinfo,
         char            key[1024]         = {0};
         char            base_key[1024]    = {0};
         struct statvfs  brickstat         = {0};
+        xlator_t       *this              = NULL;
 
+        this = THIS;
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
         GF_ASSERT (dict);
@@ -3357,7 +5070,7 @@ glusterd_add_brick_detail_to_dict (glusterd_volinfo_t *volinfo,
 
         ret = statvfs (brickinfo->path, &brickstat);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "statfs error: %s ",
+                gf_log (this->name, GF_LOG_ERROR, "statfs error: %s ",
                         strerror (errno));
                 goto out;
         }
@@ -3404,16 +5117,16 @@ glusterd_add_brick_detail_to_dict (glusterd_volinfo_t *volinfo,
                 if (ret)
                         goto out;
         }
-
+#ifdef GF_LINUX_HOST_OS
         ret = glusterd_add_brick_mount_details (brickinfo, dict, count);
         if (ret)
                 goto out;
 
         ret = glusterd_add_inode_size_to_dict (dict, count);
-
+#endif
  out:
         if (ret)
-                gf_log (THIS->name, GF_LOG_DEBUG, "Error adding brick"
+                gf_log (this->name, GF_LOG_DEBUG, "Error adding brick"
                         " detail to dict: %s", strerror (errno));
         return ret;
 }
@@ -3430,7 +5143,6 @@ glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
         char            key[1024]             = {0};
         char            base_key[1024]        = {0};
         char            pidfile[PATH_MAX]     = {0};
-        char            path[PATH_MAX]        = {0};
         xlator_t        *this                 = NULL;
         glusterd_conf_t *priv                 = NULL;
 
@@ -3463,9 +5175,7 @@ glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname,
-                                    brickinfo->path);
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
 
         brick_online = glusterd_is_service_running (pidfile, &pid);
 
@@ -3481,7 +5191,7 @@ glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
 
 out:
         if (ret)
-                gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+                gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -3553,11 +5263,14 @@ glusterd_friend_find_by_uuid (uuid_t uuid,
         int                     ret = -1;
         glusterd_conf_t         *priv = NULL;
         glusterd_peerinfo_t     *entry = NULL;
+        xlator_t                *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (peerinfo);
 
         *peerinfo = NULL;
-        priv    = THIS->private;
+        priv    = this->private;
 
         GF_ASSERT (priv);
 
@@ -3567,7 +5280,7 @@ glusterd_friend_find_by_uuid (uuid_t uuid,
         list_for_each_entry (entry, &priv->peers, uuid_list) {
                 if (!uuid_compare (entry->uuid, uuid)) {
 
-                        gf_log ("glusterd", GF_LOG_DEBUG,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                  "Friend found... state: %s",
                         glusterd_friend_sm_state_name_get (entry->state.state));
                         *peerinfo = entry;
@@ -3575,7 +5288,7 @@ glusterd_friend_find_by_uuid (uuid_t uuid,
                 }
         }
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Friend with uuid: %s, not found",
+        gf_log (this->name, GF_LOG_DEBUG, "Friend with uuid: %s, not found",
                 uuid_utoa (uuid));
         return ret;
 }
@@ -3595,12 +5308,15 @@ glusterd_friend_find_by_hostname (const char *hoststr,
         struct sockaddr_in      *s4 = NULL;
         struct in_addr          *in_addr = NULL;
         char                    hname[1024] = {0,};
+        xlator_t                *this  = NULL;
 
+
+        this = THIS;
         GF_ASSERT (hoststr);
         GF_ASSERT (peerinfo);
 
         *peerinfo = NULL;
-        priv    = THIS->private;
+        priv    = this->private;
 
         GF_ASSERT (priv);
 
@@ -3608,7 +5324,7 @@ glusterd_friend_find_by_hostname (const char *hoststr,
                 if (!strncasecmp (entry->hostname, hoststr,
                                   1024)) {
 
-                        gf_log ("glusterd", GF_LOG_DEBUG,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                  "Friend %s found.. state: %d", hoststr,
                                   entry->state.state);
                         *peerinfo = entry;
@@ -3616,9 +5332,10 @@ glusterd_friend_find_by_hostname (const char *hoststr,
                 }
         }
 
-        ret = getaddrinfo(hoststr, NULL, NULL, &addr);
+        ret = getaddrinfo (hoststr, NULL, NULL, &addr);
         if (ret != 0) {
-                gf_log ("", GF_LOG_ERROR, "error in getaddrinfo: %s\n",
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error in getaddrinfo: %s\n",
                         gai_strerror(ret));
                 goto out;
         }
@@ -3647,7 +5364,7 @@ glusterd_friend_find_by_hostname (const char *hoststr,
                         if (!strncasecmp (entry->hostname, host,
                             1024) || !strncasecmp (entry->hostname,hname,
                             1024)) {
-                                gf_log ("glusterd", GF_LOG_DEBUG,
+                                gf_log (this->name, GF_LOG_DEBUG,
                                         "Friend %s found.. state: %d",
                                         hoststr, entry->state.state);
                                 *peerinfo = entry;
@@ -3658,7 +5375,7 @@ glusterd_friend_find_by_hostname (const char *hoststr,
         }
 
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Unable to find friend: %s", hoststr);
+        gf_log (this->name, GF_LOG_DEBUG, "Unable to find friend: %s", hoststr);
         if (addr)
                 freeaddrinfo (addr);
         return -1;
@@ -3682,23 +5399,26 @@ glusterd_hostname_to_uuid (char *hostname, uuid_t uuid)
 
         ret = glusterd_friend_find_by_hostname (hostname, &peerinfo);
         if (ret) {
-                ret = glusterd_is_local_addr (hostname);
-                if (ret)
+                if (gf_is_local_addr (hostname)) {
+                        uuid_copy (uuid, MY_UUID);
+                        ret = 0;
+                } else {
+                        ret = 0;
                         goto out;
-                else
-                        uuid_copy (uuid, priv->uuid);
+                }
         } else {
                 uuid_copy (uuid, peerinfo->uuid);
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
 int
 glusterd_brick_stop (glusterd_volinfo_t *volinfo,
-                     glusterd_brickinfo_t *brickinfo)
+                     glusterd_brickinfo_t *brickinfo,
+                     gf_boolean_t del_brick)
 {
         int                                     ret   = -1;
         xlator_t                                *this = NULL;
@@ -3715,38 +5435,39 @@ glusterd_brick_stop (glusterd_volinfo_t *volinfo,
         if (uuid_is_null (brickinfo->uuid)) {
                 ret = glusterd_resolve_brick (brickinfo);
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "cannot resolve brick: %s:%s",
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
                                 brickinfo->hostname, brickinfo->path);
                         goto out;
                 }
         }
 
-        if (uuid_compare (brickinfo->uuid, conf->uuid)) {
+        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
                 ret = 0;
+                if (del_brick)
+                        glusterd_delete_brick (volinfo, brickinfo);
                 goto out;
         }
 
-        gf_log ("", GF_LOG_INFO, "About to stop glusterfs"
+        gf_log (this->name, GF_LOG_DEBUG, "About to stop glusterfs"
                 " for brick %s:%s", brickinfo->hostname,
                 brickinfo->path);
-        ret = glusterd_volume_stop_glusterfs (volinfo, brickinfo);
+        ret = glusterd_volume_stop_glusterfs (volinfo, brickinfo, del_brick);
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Unable to remove"
+                gf_log (this->name, GF_LOG_CRITICAL, "Unable to stop"
                         " brick: %s:%s", brickinfo->hostname,
                         brickinfo->path);
                 goto out;
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d ", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
         return ret;
 }
 
 int
 glusterd_is_defrag_on (glusterd_volinfo_t *volinfo)
 {
-        return (volinfo->defrag != NULL);
+        return (volinfo->rebal.defrag != NULL);
 }
 
 gf_boolean_t
@@ -3768,7 +5489,6 @@ glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
                              char *op_errstr, size_t len)
 {
         glusterd_brickinfo_t    *newbrickinfo = NULL;
-        glusterd_brickinfo_t    *tmpbrkinfo = NULL;
         int                     ret = -1;
         gf_boolean_t            is_allocated = _gf_false;
         glusterd_peerinfo_t     *peerinfo = NULL;
@@ -3785,7 +5505,7 @@ glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
         GF_ASSERT (op_errstr);
 
         if (!brickinfo) {
-                ret = glusterd_brickinfo_from_brick (brick, &newbrickinfo);
+                ret = glusterd_brickinfo_new_from_brick (brick, &newbrickinfo);
                 if (ret)
                         goto out;
                 is_allocated = _gf_true;
@@ -3795,42 +5515,54 @@ glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
 
         ret = glusterd_resolve_brick (newbrickinfo);
         if (ret) {
-                snprintf (op_errstr, len, "Host %s not a friend",
-                          newbrickinfo->hostname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", op_errstr);
+                snprintf(op_errstr, len, "Host %s is not in \'Peer "
+                         "in Cluster\' state", newbrickinfo->hostname);
                 goto out;
         }
 
-        if (!uuid_compare (priv->uuid, newbrickinfo->uuid))
-                goto brick_validation;
-        ret = glusterd_friend_find_by_uuid (newbrickinfo->uuid, &peerinfo);
-        if (ret)
-                goto out;
-        if ((!peerinfo->connected) ||
-            (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)) {
-                snprintf(op_errstr, len, "Host %s not connected",
-                         newbrickinfo->hostname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", op_errstr);
-                ret = -1;
-                goto out;
-        }
-brick_validation:
-        ret = glusterd_brickinfo_get (newbrickinfo->uuid,
-                                      newbrickinfo->hostname,
-                                      newbrickinfo->path, &tmpbrkinfo);
-        if (!ret) {
-                snprintf(op_errstr, len, "Brick: %s already in use",
-                         brick);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", op_errstr);
-                ret = -1;
-                goto out;
+        if (!uuid_compare (MY_UUID, newbrickinfo->uuid)) {
+                /* brick is local */
+                if (!glusterd_is_brickpath_available (newbrickinfo->uuid,
+                                                      newbrickinfo->path)) {
+                        snprintf(op_errstr, len, "Brick: %s not available."
+                                 " Brick may be containing or be contained "
+                                 "by an existing brick", brick);
+                        ret = -1;
+                        goto out;
+                }
+
         } else {
-                ret = 0;
+                ret = glusterd_friend_find_by_uuid (newbrickinfo->uuid,
+                                                    &peerinfo);
+                if (ret) {
+                        snprintf (op_errstr, len, "Failed to find host %s",
+                                  newbrickinfo->hostname);
+                        goto out;
+                }
+
+                if ((!peerinfo->connected)) {
+                        snprintf(op_errstr, len, "Host %s not connected",
+                                 newbrickinfo->hostname);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) {
+                        snprintf(op_errstr, len, "Host %s is not in \'Peer "
+                                 "in Cluster\' state",
+                                 newbrickinfo->hostname);
+                        ret = -1;
+                        goto out;
+                }
         }
+
+        ret = 0;
 out:
-        if (is_allocated && newbrickinfo)
+        if (is_allocated)
                 glusterd_brickinfo_delete (newbrickinfo);
-        gf_log (THIS->name, GF_LOG_DEBUG, "returning %d ", ret);
+        if (op_errstr[0] != '\0')
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
         return ret;
 }
 
@@ -3838,8 +5570,8 @@ int
 glusterd_is_rb_started(glusterd_volinfo_t *volinfo)
 {
         gf_log ("", GF_LOG_DEBUG,
-                "is_rb_started:status=%d", volinfo->rb_status);
-        return (volinfo->rb_status == GF_RB_STATUS_STARTED);
+                "is_rb_started:status=%d", volinfo->rep_brick.rb_status);
+        return (volinfo->rep_brick.rb_status == GF_RB_STATUS_STARTED);
 
 }
 
@@ -3847,9 +5579,9 @@ int
 glusterd_is_rb_paused ( glusterd_volinfo_t *volinfo)
 {
         gf_log ("", GF_LOG_DEBUG,
-                "is_rb_paused:status=%d", volinfo->rb_status);
+                "is_rb_paused:status=%d", volinfo->rep_brick.rb_status);
 
-        return (volinfo->rb_status == GF_RB_STATUS_PAUSED);
+        return (volinfo->rep_brick.rb_status == GF_RB_STATUS_PAUSED);
 }
 
 inline int
@@ -3857,10 +5589,10 @@ glusterd_set_rb_status (glusterd_volinfo_t *volinfo, gf_rb_status_t status)
 {
         gf_log ("", GF_LOG_DEBUG,
                 "setting status from %d to %d",
-                volinfo->rb_status,
+                volinfo->rep_brick.rb_status,
                 status);
 
-        volinfo->rb_status = status;
+        volinfo->rep_brick.rb_status = status;
         return 0;
 }
 
@@ -3868,142 +5600,192 @@ inline int
 glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
                           glusterd_brickinfo_t *src, glusterd_brickinfo_t *dst)
 {
-        if (!volinfo->src_brick || !volinfo->dst_brick)
+        glusterd_replace_brick_t        *rb = NULL;
+
+        GF_ASSERT (volinfo);
+
+        rb = &volinfo->rep_brick;
+
+        if (!rb->src_brick || !rb->dst_brick)
                 return -1;
 
-        if (strcmp (volinfo->src_brick->hostname, src->hostname) ||
-            strcmp (volinfo->src_brick->path, src->path)) {
+        if (strcmp (rb->src_brick->hostname, src->hostname) ||
+            strcmp (rb->src_brick->path, src->path)) {
                 gf_log("", GF_LOG_ERROR, "Replace brick src bricks differ");
                 return -1;
         }
-        if (strcmp (volinfo->dst_brick->hostname, dst->hostname) ||
-            strcmp (volinfo->dst_brick->path, dst->path)) {
+
+        if (strcmp (rb->dst_brick->hostname, dst->hostname) ||
+            strcmp (rb->dst_brick->path, dst->path)) {
                 gf_log ("", GF_LOG_ERROR, "Replace brick dst bricks differ");
                 return -1;
         }
+
         return 0;
 }
 
-int
-glusterd_brick_create_path (char *host, char *path, uuid_t uuid, mode_t mode,
-                            char **op_errstr)
+/*path needs to be absolute; works only on gfid, volume-id*/
+static int
+glusterd_is_uuid_present (char *path, char *xattr, gf_boolean_t *present)
 {
-        int     ret = -1;
-        char    msg[2048] = {0};
-        struct  stat st_buf = {0};
-        uuid_t  gfid = {0,};
-        uuid_t  old_uuid = {0,};
-        char    old_uuid_buf[64] = {0,};
-
-        ret = stat (path, &st_buf);
-        if ((!ret) && (!S_ISDIR (st_buf.st_mode))) {
-                snprintf (msg, sizeof (msg), "brick %s:%s, "
-                          "path %s is not a directory", host, path, path);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                ret = -1;
+        GF_ASSERT (path);
+        GF_ASSERT (xattr);
+        GF_ASSERT (present);
+
+        int     ret      = -1;
+        uuid_t  uid     = {0,};
+
+        if (!path || !xattr || !present)
                 goto out;
-        } else if (!ret) {
-                goto check_xattr;
-        } else {
-                ret = mkdir (path, mode);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "brick: %s:%s, path "
-                                  "creation failed, reason: %s",
-                                  host, path, strerror(errno));
-                        gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
+
+        ret = sys_lgetxattr (path, xattr, &uid, 16);
+
+        if (ret >= 0) {
+                *present = _gf_true;
+                ret = 0;
+                goto out;
+        }
+
+        switch (errno) {
+#if defined(ENODATA)
+                case ENODATA: /* FALLTHROUGH */
+#endif
+#if defined(ENOATTR) && (ENOATTR != ENODATA)
+                case ENOATTR: /* FALLTHROUGH */
+#endif
+                case ENOTSUP:
+                        *present = _gf_false;
+                        ret = 0;
+                        break;
+                default:
+                        break;
+        }
+out:
+        return ret;
+}
+
+/*path needs to be absolute*/
+static int
+glusterd_is_path_in_use (char *path, gf_boolean_t *in_use, char **op_errstr)
+{
+        int             i               = 0;
+        int             ret             = -1;
+        gf_boolean_t    used            = _gf_false;
+        char            dir[PATH_MAX]   = {0,};
+        char            *curdir         = NULL;
+        char            msg[2048]       = {0};
+        char            *keys[3]         = {GFID_XATTR_KEY,
+                                            GF_XATTR_VOL_ID_KEY,
+                                            NULL};
+
+        GF_ASSERT (path);
+        if (!path)
+                goto out;
+
+        strcpy (dir, path);
+        curdir = dir;
+        do {
+                for (i = 0; !used && keys[i]; i++) {
+                        ret = glusterd_is_uuid_present (curdir, keys[i], &used);
+                        if (ret)
+                                goto out;
+                }
+
+                if (used)
+                        break;
+
+                curdir = dirname (curdir);
+                if (!strcmp (curdir, "."))
                         goto out;
+
+
+        } while (strcmp (curdir, "/"));
+
+        if (!strcmp (curdir, "/")) {
+                for (i = 0; !used && keys[i]; i++) {
+                        ret = glusterd_is_uuid_present (curdir, keys[i], &used);
+                        if (ret)
+                                goto out;
+                }
+        }
+
+        ret = 0;
+        *in_use = used;
+out:
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to get extended "
+                          "attribute %s, reason: %s", keys[i],
+                          strerror (errno));
+        }
+
+        if (*in_use) {
+                if (!strcmp (path, curdir)) {
+                        snprintf (msg, sizeof (msg), "%s is already part of a "
+                          "volume", path);
                 } else {
-                        goto check_xattr;
+                        snprintf (msg, sizeof (msg), "parent directory %s is "
+                          "already part of a volume", curdir);
                 }
         }
 
-/* To check if filesystem is read-only
-   and if it supports extended attributes */
-check_xattr:
+        if (strlen (msg)) {
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+
+        return ret;
+}
+
+int
+glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
+                                    char **op_errstr, gf_boolean_t is_force)
+{
+        int             ret             = -1;
+        char            msg[2048]       = {0,};
+        gf_boolean_t    in_use          = _gf_false;
+        int             flags           = 0;
+
+        /* Check for xattr support in backend fs */
         ret = sys_lsetxattr (path, "trusted.glusterfs.test",
                              "working", 8, 0);
         if (ret) {
-                snprintf (msg, sizeof (msg), "glusterfs is not"
+                snprintf (msg, sizeof (msg), "Glusterfs is not"
                           " supported on brick: %s:%s.\nSetting"
                           " extended attributes failed, reason:"
                           " %s.", host, path, strerror(errno));
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
                 goto out;
+
         } else {
-                /* Remove xattr *cannot* fail after setting it succeeded */
                 sys_lremovexattr (path, "trusted.glusterfs.test");
         }
 
-        /* Now check if the export directory has some other 'gfid',
-           other than that of root '/' */
-        ret = sys_lgetxattr (path, "trusted.gfid", gfid, 16);
-        if (ret == 16) {
-                if (!__is_root_gfid (gfid)) {
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "%s: gfid (%s) is not that of glusterfs '/' ",
-                                path, uuid_utoa (gfid));
-                        snprintf (msg, sizeof (msg),
-                                  "'%s:%s' gfid (%s) is not that of "
-                                  "glusterfs '/' ", host, path, uuid_utoa (gfid));
-                        ret = -1;
-                        goto out;
-                }
-        } else if (ret != -1) {
-                /* Wrong 'gfid' is set, it should be error */
-                ret = -1;
-                snprintf (msg, sizeof (msg), "'%s:%s' has wrong entry"
-                          "for 'gfid'.", host, path);
-                goto out;
-        } else if ((ret == -1) && (errno != ENODATA)) {
-                /* Wrong 'gfid' is set, it should be error */
-                snprintf (msg, sizeof (msg), "'%s:%s' has failed to fetch "
-                          "'gfid' (%s)", host, path, strerror (errno));
+        ret = glusterd_is_path_in_use (path, &in_use, op_errstr);
+        if (ret)
                 goto out;
-        }
 
-        ret = 0;
-        if (!uuid)
-                goto out;
-
-        /* This 'key' is set when the volume is started for the first time */
-        ret = sys_lgetxattr (path, "trusted.glusterfs.volume-id",
-                             old_uuid, 16);
-        if (ret == 16) {
-                if (uuid_compare (old_uuid, uuid)) {
-                        uuid_utoa_r (old_uuid, old_uuid_buf);
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "%s: mismatching volume-id (%s) recieved. "
-                                "already is a part of volume %s ",
-                                path, uuid_utoa (uuid), old_uuid_buf);
-                        snprintf (msg, sizeof (msg), "'%s:%s' has been part of "
-                                  "a deleted volume with id %s. Please "
-                                  "re-create the brick directory.",
-                                  host, path, old_uuid_buf);
-                        ret = -1;
-                        goto out;
-                }
-        } else if (ret != -1) {
-                /* Wrong 'volume-id' is set, it should be error */
+        if (in_use && !is_force) {
                 ret = -1;
-                snprintf (msg, sizeof (msg), "'%s:%s' has wrong entry"
-                          "for 'volume-id'.", host, path);
-                goto out;
-        } else if ((ret == -1) && (errno != ENODATA)) {
-                /* Wrong 'volume-id' is set, it should be error */
-                snprintf (msg, sizeof (msg), "'%s:%s' : failed to fetch "
-                          "'volume-id' (%s)", host, path, strerror (errno));
                 goto out;
+        }
+
 
+        if (!is_force)
+                flags = XATTR_CREATE;
+
+        ret = sys_lsetxattr (path, GF_XATTR_VOL_ID_KEY, uuid, 16,
+                             flags);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to set extended "
+                          "attributes %s, reason: %s",
+                          GF_XATTR_VOL_ID_KEY, strerror (errno));
+                goto out;
         }
-        /* if 'ret == -1' then 'volume-id' not set, seems to be a fresh
-           directory */
 
         ret = 0;
 out:
-        if (msg[0] != '\0')
+        if (strlen (msg))
                 *op_errstr = gf_strdup (msg);
 
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
@@ -4014,9 +5796,8 @@ glusterd_sm_tr_log_transition_add_to_dict (dict_t *dict,
 {
         int     ret = -1;
         char    key[512] = {0};
-	char    timestr[256] = {0,};
+	char    timestr[64] = {0,};
         char    *str = NULL;
-	struct tm   tm = {0};
 
         GF_ASSERT (dict);
         GF_ASSERT (log);
@@ -4045,9 +5826,8 @@ glusterd_sm_tr_log_transition_add_to_dict (dict_t *dict,
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "log%d-time", count);
-	localtime_r ((const time_t*)&log->transitions[i].time, &tm);
-        memset (timestr, 0, sizeof (timestr));
-	strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", &tm);
+        gf_time_fmt (timestr, sizeof timestr, log->transitions[i].time,
+                     gf_timefmt_FT);
         str = gf_strdup (timestr);
         ret = dict_set_dynstr (dict, key, str);
         if (ret)
@@ -4135,8 +5915,7 @@ glusterd_sm_tr_log_delete (glusterd_sm_tr_log_t *log)
 {
         if (!log)
                 return;
-        if (log->transitions)
-                GF_FREE (log->transitions);
+        GF_FREE (log->transitions);
         return;
 }
 
@@ -4148,6 +5927,10 @@ glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
         glusterd_sm_transition_t *transitions = NULL;
         int                      ret = -1;
         int                      next = 0;
+        xlator_t                 *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (log);
         if (!log)
@@ -4170,18 +5953,18 @@ glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
         if (log->count < log->size)
                 log->count++;
         ret = 0;
-        gf_log ("glusterd", GF_LOG_DEBUG, "Transitioning from '%s' to '%s' "
+        gf_log (this->name, GF_LOG_DEBUG, "Transitioning from '%s' to '%s' "
                 "due to event '%s'", log->state_name_get (old_state),
                 log->state_name_get (new_state), log->event_name_get (event));
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
 int
 glusterd_peerinfo_new (glusterd_peerinfo_t **peerinfo,
-                       glusterd_friend_sm_state_t state,
-                       uuid_t *uuid, const char *hostname)
+                       glusterd_friend_sm_state_t state, uuid_t *uuid,
+                       const char *hostname, int port)
 {
         glusterd_peerinfo_t      *new_peer = NULL;
         int                      ret = -1;
@@ -4211,6 +5994,9 @@ glusterd_peerinfo_new (glusterd_peerinfo_t **peerinfo,
         if (ret)
                 goto out;
 
+        if (new_peer->state.state == GD_FRIEND_STATE_BEFRIENDED)
+                new_peer->quorum_contrib = QUORUM_WAITING;
+        new_peer->port = port;
         *peerinfo = new_peer;
 out:
         if (ret && new_peer)
@@ -4234,8 +6020,7 @@ glusterd_peer_destroy (glusterd_peerinfo_t *peerinfo)
         }
 
         list_del_init (&peerinfo->uuid_list);
-        if (peerinfo->hostname)
-                GF_FREE (peerinfo->hostname);
+        GF_FREE (peerinfo->hostname);
         glusterd_sm_tr_log_delete (&peerinfo->sm_log);
         GF_FREE (peerinfo);
         peerinfo = NULL;
@@ -4262,7 +6047,7 @@ glusterd_remove_pending_entry (struct list_head *list, void *elem)
                 }
         }
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 
 }
@@ -4304,7 +6089,7 @@ glusterd_delete_volume (glusterd_volinfo_t *volinfo)
 
         ret = glusterd_volinfo_delete (volinfo);
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
@@ -4313,18 +6098,15 @@ glusterd_delete_brick (glusterd_volinfo_t* volinfo,
                        glusterd_brickinfo_t *brickinfo)
 {
         int             ret = 0;
+        char      voldir[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = THIS->private;
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
 
-#ifdef DEBUG
-        ret = glusterd_volume_brickinfo_get (brickinfo->uuid,
-                                             brickinfo->hostname,
-                                             brickinfo->path, volinfo,
-                                             NULL, GF_PATH_COMPLETE);
-        GF_ASSERT (0 == ret);
-#endif
+        GLUSTERD_GET_VOLUME_DIR(voldir, volinfo, priv);
+
         glusterd_delete_volfile (volinfo, brickinfo);
-        glusterd_store_delete_brick (volinfo, brickinfo);
+        glusterd_store_delete_brick (brickinfo, voldir);
         glusterd_brickinfo_delete (brickinfo);
         volinfo->brick_count--;
         return ret;
@@ -4346,28 +6128,92 @@ glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo)
 }
 
 int
-mkdir_if_missing (char *path)
-{
-        struct stat st = {0,};
-        int        ret = 0;
+glusterd_get_local_brickpaths (glusterd_volinfo_t *volinfo, char **pathlist)
+{
+        char                 **path_tokens  = NULL;
+        char                  *tmp_path_list = NULL;
+        char                   path[PATH_MAX] = "";
+        int32_t                count          = 0;
+        int32_t                pathlen        = 0;
+        int32_t                total_len      = 0;
+        int32_t                ret            = 0;
+        int                    i              = 0;
+        glusterd_brickinfo_t  *brickinfo      = NULL;
+
+        if ((!volinfo) || (!pathlist))
+            goto out;
+
+        path_tokens = GF_CALLOC (sizeof(char*), volinfo->brick_count,
+                                 gf_gld_mt_charptr);
+        if (!path_tokens) {
+                gf_log ("", GF_LOG_DEBUG, "Could not allocate memory.");
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
 
-        ret = mkdir (path, 0777);
-        if (!ret || errno == EEXIST)
-                ret = stat (path, &st);
-        if (ret == -1 || !S_ISDIR (st.st_mode))
-                gf_log ("", GF_LOG_WARNING, "Failed to create the"
-                        " directory %s", path);
+                 pathlen = snprintf (path, sizeof(path),
+                                     "--path=%s ", brickinfo->path);
+                 if (pathlen < sizeof(path))
+                        path[pathlen] = '\0';
+                 else
+                        path[sizeof(path)-1] = '\0';
+                 path_tokens[count] = gf_strdup (path);
+                 if (!path_tokens[count]) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not allocate memory.");
+                        ret = -1;
+                        goto out;
+                 }
+                 count++;
+                 total_len += pathlen;
+        }
+
+        tmp_path_list = GF_CALLOC (sizeof(char), total_len + 1,
+                                   gf_gld_mt_char);
+        if (!tmp_path_list) {
+                gf_log ("", GF_LOG_DEBUG, "Could not allocate memory.");
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; i < count; i++)
+                strcat (tmp_path_list, path_tokens[i]);
 
+        if (count)
+                *pathlist = tmp_path_list;
+
+        ret = count;
+out:
+        for (i = 0; i < count; i++) {
+                GF_FREE (path_tokens[i]);
+                path_tokens[i] = NULL;
+        }
+
+        GF_FREE (path_tokens);
+        path_tokens = NULL;
+
+        if (ret == 0) {
+                gf_log ("", GF_LOG_DEBUG, "No Local Bricks Present.");
+                GF_FREE (tmp_path_list);
+                tmp_path_list = NULL;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
 glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
-                      char *glusterd_uuid_str, char **op_errstr)
+                      char *path_list, char *conf_path,
+                      char *glusterd_uuid_str,
+                      char **op_errstr)
 {
         int32_t         ret     = 0;
         int32_t         status  = 0;
-        char            buf[PATH_MAX]   = {0,};
         char            uuid_str [64] = {0};
         runner_t        runner = {0,};
         xlator_t        *this = NULL;
@@ -4379,48 +6225,46 @@ glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
         priv = this->private;
         GF_ASSERT (priv);
 
-        uuid_utoa_r (priv->uuid, uuid_str);
-        if (strcmp (uuid_str, glusterd_uuid_str))
-                goto out;
+        uuid_utoa_r (MY_UUID, uuid_str);
 
-        ret = gsync_status (master_vol->volname, slave, &status);
-        if (status == 0)
-                goto out;
-
-        snprintf (buf, PATH_MAX, "%s/"GEOREP"/%s", priv->workdir, master_vol->volname);
-        ret = mkdir_if_missing (buf);
-        if (ret) {
-                errcode = -1;
+        if (!path_list) {
+                ret = 0;
+                gf_log ("", GF_LOG_DEBUG, "No Bricks in this node."
+                        " Not starting gsyncd.");
                 goto out;
         }
 
-        snprintf (buf, PATH_MAX, DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/%s",
-                  master_vol->volname);
-        ret = mkdir_if_missing (buf);
-        if (ret) {
-                errcode = -1;
+        ret = gsync_status (master_vol->volname, slave, conf_path, &status);
+        if (status == 0)
                 goto out;
-        }
 
         uuid_utoa_r (master_vol->volume_id, uuid_str);
         runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, priv->workdir);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
+                          path_list, "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
         runner_argprintf (&runner, ":%s", master_vol->volname);
         runner_add_args  (&runner, slave, "--config-set", "session-owner",
                           uuid_str, NULL);
+        synclock_unlock (&priv->big_lock);
         ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
         if (ret == -1) {
                 errcode = -1;
                 goto out;
         }
 
         runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "--monitor", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, priv->workdir);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
+                          path_list, "--monitor", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
         runner_argprintf (&runner, ":%s", master_vol->volname);
+        runner_argprintf (&runner, "--glusterd-uuid=%s",
+                          uuid_utoa (priv->uuid));
         runner_add_arg   (&runner, slave);
+        synclock_unlock (&priv->big_lock);
         ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
         if (ret == -1) {
                 gf_asprintf (op_errstr, GEOREP" start failed for %s %s",
                              master_vol->volname, slave);
@@ -4432,7 +6276,7 @@ glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
 out:
         if ((ret != 0) && errcode == -1) {
                 if (op_errstr)
-                        *op_errstr = gf_strdup ("internal error, cannot start"
+                        *op_errstr = gf_strdup ("internal error, cannot start "
                                                 "the " GEOREP " session");
         }
 
@@ -4441,17 +6285,38 @@ out:
 }
 
 int32_t
-glusterd_recreate_bricks (glusterd_conf_t *conf)
+glusterd_recreate_volfiles (glusterd_conf_t *conf)
 {
 
         glusterd_volinfo_t      *volinfo = NULL;
         int                      ret = 0;
+        int                      op_ret = 0;
 
         GF_ASSERT (conf);
         list_for_each_entry (volinfo, &conf->volumes, vol_list) {
                 ret = generate_brick_volfiles (volinfo);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                "regenerate brick volfiles for %s",
+                                volinfo->volname);
+                        op_ret = ret;
+                }
+                ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                "regenerate trusted client volfiles for %s",
+                                volinfo->volname);
+                        op_ret = ret;
+                }
+                ret = generate_client_volfiles (volinfo, GF_CLIENT_OTHER);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                "regenerate client volfiles for %s",
+                                volinfo->volname);
+                        op_ret = ret;
+                }
         }
-        return ret;
+        return op_ret;
 }
 
 int32_t
@@ -4461,7 +6326,8 @@ glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf)
         char            *type                           = NULL;
         gf_boolean_t     upgrade                        = _gf_false;
         gf_boolean_t     downgrade                      = _gf_false;
-        gf_boolean_t     regenerate_brick_volfiles      = _gf_false;
+        gf_boolean_t     regenerate_volfiles            = _gf_false;
+        gf_boolean_t     terminate                      = _gf_false;
 
         ret = dict_get_str (options, "upgrade", &type);
         if (!ret) {
@@ -4473,7 +6339,7 @@ glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf)
                         goto out;
                 }
                 if (_gf_true == upgrade)
-                        regenerate_brick_volfiles = _gf_true;
+                        regenerate_volfiles = _gf_true;
         }
 
         ret = dict_get_str (options, "downgrade", &type);
@@ -4496,10 +6362,14 @@ glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf)
 
         if (!upgrade && !downgrade)
                 ret = 0;
-        if (regenerate_brick_volfiles) {
-                ret = glusterd_recreate_bricks (conf);
+        else
+                terminate = _gf_true;
+        if (regenerate_volfiles) {
+                ret = glusterd_recreate_volfiles (conf);
         }
 out:
+        if (terminate && (ret == 0))
+                kill (getpid(), SIGTERM);
         return ret;
 }
 
@@ -4536,7 +6406,7 @@ glusterd_set_dump_options (char *dumpoptions_path, char *options,
                 goto out;
         }
         dup_options = gf_strdup (options);
-        gf_log ("", GF_LOG_INFO, "Recieved following statedump options: %s",
+        gf_log ("", GF_LOG_INFO, "Received following statedump options: %s",
                 dup_options);
         option = strtok_r (dup_options, " ", &tmpptr);
         while (option) {
@@ -4557,8 +6427,7 @@ glusterd_set_dump_options (char *dumpoptions_path, char *options,
 out:
         if (fp)
                 fclose (fp);
-        if (dup_options)
-                GF_FREE (dup_options);
+        GF_FREE (dup_options);
         return ret;
 }
 
@@ -4571,7 +6440,6 @@ glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
         xlator_t                *this = NULL;
         glusterd_conf_t         *conf = NULL;
         char                    pidfile_path[PATH_MAX] = {0,};
-        char                    path[PATH_MAX] = {0,};
         char                    dumpoptions_path[PATH_MAX] = {0,};
         FILE                    *pidfile = NULL;
         pid_t                   pid = -1;
@@ -4591,14 +6459,12 @@ glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
                 }
         }
 
-        if (uuid_compare (brickinfo->uuid, conf->uuid)) {
+        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
                 ret = 0;
                 goto out;
         }
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, conf);
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile_path, path, brickinfo->hostname,
-                                    brickinfo->path);
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile_path, volinfo, brickinfo, conf);
 
         pidfile = fopen (pidfile_path, "r");
         if (!pidfile) {
@@ -4616,7 +6482,7 @@ glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
         }
 
         snprintf (dumpoptions_path, sizeof (dumpoptions_path),
-                  "/tmp/glusterdump.%d.options", pid);
+                  DEFAULT_VAR_RUN_DIRECTORY"/glusterdump.%d.options", pid);
         ret = glusterd_set_dump_options (dumpoptions_path, options, option_cnt);
         if (ret < 0) {
                 gf_log ("", GF_LOG_ERROR, "error while parsing the statedump "
@@ -4689,7 +6555,7 @@ glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr)
         }
 
         snprintf (dumpoptions_path, sizeof (dumpoptions_path),
-                  "/tmp/glusterdump.%d.options", pid);
+                  DEFAULT_VAR_RUN_DIRECTORY"/glusterdump.%d.options", pid);
         ret = glusterd_set_dump_options (dumpoptions_path, options, option_cnt);
         if (ret < 0) {
                 gf_log ("", GF_LOG_ERROR, "error while parsing the statedump "
@@ -4710,8 +6576,7 @@ out:
         if (pidfile)
                 fclose (pidfile);
         unlink (dumpoptions_path);
-        if (dup_options)
-                GF_FREE (dup_options);
+        GF_FREE (dup_options);
         return ret;
 }
 
@@ -4798,3 +6663,2232 @@ glusterd_chk_peers_connected_befriended (uuid_t skip_uuid)
                 (ret?"TRUE":"FALSE"));
         return ret;
 }
+
+void
+glusterd_get_client_filepath (char *filepath, glusterd_volinfo_t *volinfo,
+                              gf_transport_type type)
+{
+        char  path[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+
+        if ((volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) &&
+            (type == GF_TRANSPORT_RDMA))
+                snprintf (filepath, PATH_MAX, "%s/%s.rdma-fuse.vol",
+                          path, volinfo->volname);
+        else
+                snprintf (filepath, PATH_MAX, "%s/%s-fuse.vol",
+                          path, volinfo->volname);
+}
+
+void
+glusterd_get_trusted_client_filepath (char *filepath,
+                                      glusterd_volinfo_t *volinfo,
+                                      gf_transport_type type)
+{
+        char  path[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+
+        if ((volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) &&
+            (type == GF_TRANSPORT_RDMA))
+                snprintf (filepath, PATH_MAX,
+                          "%s/trusted-%s.rdma-fuse.vol",
+                          path, volinfo->volname);
+        else
+                snprintf (filepath, PATH_MAX,
+                          "%s/trusted-%s-fuse.vol",
+                          path, volinfo->volname);
+}
+
+int
+glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
+                              size_t len, int cmd, defrag_cbk_fn_t cbk)
+{
+        glusterd_conf_t         *priv                   = NULL;
+        char                     pidfile[PATH_MAX];
+        int                      ret                    = -1;
+        pid_t                    pid;
+
+        priv = THIS->private;
+        if (!priv)
+                return ret;
+
+        GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
+
+        if (!glusterd_is_service_running (pidfile, &pid)) {
+                glusterd_handle_defrag_start (volinfo, op_errstr, len, cmd,
+                                              cbk, volinfo->rebal.op);
+        } else {
+                glusterd_rebalance_rpc_create (volinfo, priv, cmd);
+        }
+
+        return ret;
+}
+
+int
+glusterd_restart_rebalance (glusterd_conf_t *conf)
+{
+        glusterd_volinfo_t       *volinfo = NULL;
+        int                      ret = 0;
+        char                     op_errstr[256];
+
+        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                if (!volinfo->rebal.defrag_cmd)
+                        continue;
+                glusterd_volume_defrag_restart (volinfo, op_errstr, 256,
+                                        volinfo->rebal.defrag_cmd, NULL);
+        }
+        return ret;
+}
+
+void
+glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo)
+{
+        glusterd_rebalance_t *rebal = NULL;
+        GF_ASSERT (volinfo);
+
+        rebal = &volinfo->rebal;
+        rebal->rebalance_files = 0;
+        rebal->rebalance_data = 0;
+        rebal->lookedup_files = 0;
+        rebal->rebalance_failures = 0;
+        rebal->rebalance_time = 0;
+        rebal->skipped_files = 0;
+
+}
+
+/* Return hostname for given uuid if it exists
+ * else return NULL
+ */
+char *
+glusterd_uuid_to_hostname (uuid_t uuid)
+{
+        char                    *hostname = NULL;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_peerinfo_t     *entry = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        if (!uuid_compare (MY_UUID, uuid)) {
+                hostname = gf_strdup ("localhost");
+        }
+        if (!list_empty (&priv->peers)) {
+                list_for_each_entry (entry, &priv->peers, uuid_list) {
+                        if (!uuid_compare (entry->uuid, uuid)) {
+                                hostname = gf_strdup (entry->hostname);
+                                break;
+                        }
+                }
+        }
+
+        return hostname;
+}
+
+gf_boolean_t
+glusterd_is_local_brick (xlator_t *this, glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo)
+{
+        gf_boolean_t    local = _gf_false;
+        int             ret = 0;
+        glusterd_conf_t *conf = NULL;
+
+        if (uuid_is_null (brickinfo->uuid)) {
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret)
+                        goto out;
+        }
+        conf = this->private;
+        local = !uuid_compare (brickinfo->uuid, MY_UUID);
+out:
+        return local;
+}
+int
+glusterd_validate_volume_id (dict_t *op_dict, glusterd_volinfo_t *volinfo)
+{
+        int     ret             = -1;
+        char    *volid_str      = NULL;
+        uuid_t  vol_uid         = {0, };
+        xlator_t *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_str (op_dict, "vol-id", &volid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume id for "
+                        "volume %s", volinfo->volname);
+                goto out;
+        }
+        ret = uuid_parse (volid_str, vol_uid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to parse volume id "
+                        "for volume %s", volinfo->volname);
+                goto out;
+        }
+
+        if (uuid_compare (vol_uid, volinfo->volume_id)) {
+                gf_log (this->name, GF_LOG_ERROR, "Volume ids of volume %s - %s"
+                        " and %s - are different. Possibly a split brain among "
+                        "peers.", volinfo->volname, volid_str,
+                        uuid_utoa (volinfo->volume_id));
+                ret = -1;
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
+                                      dict_t *rsp_dict)
+{
+        int                             ret = 0;
+        uint64_t                        files = 0;
+        uint64_t                        size = 0;
+        uint64_t                        lookup = 0;
+        gf_defrag_status_t              status = GF_DEFRAG_STATUS_NOT_STARTED;
+        uint64_t                        failures = 0;
+        uint64_t                        skipped = 0;
+        xlator_t                       *this = NULL;
+        double                          run_time = 0;
+
+        this = THIS;
+
+        ret = dict_get_uint64 (rsp_dict, "files", &files);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get file count");
+
+        ret = dict_get_uint64 (rsp_dict, "size", &size);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get size of xfer");
+
+        ret = dict_get_uint64 (rsp_dict, "lookups", &lookup);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get lookedup file count");
+
+        ret = dict_get_int32 (rsp_dict, "status", (int32_t *)&status);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get status");
+
+        ret = dict_get_uint64 (rsp_dict, "failures", &failures);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get failure count");
+
+        ret = dict_get_uint64 (rsp_dict, "skipped", &skipped);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get skipped count");
+
+        ret = dict_get_double (rsp_dict, "run-time", &run_time);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get run-time");
+
+        if (files)
+                volinfo->rebal.rebalance_files = files;
+        if (size)
+                volinfo->rebal.rebalance_data = size;
+        if (lookup)
+                volinfo->rebal.lookedup_files = lookup;
+        if (status)
+                volinfo->rebal.defrag_status = status;
+        if (failures)
+                volinfo->rebal.rebalance_failures = failures;
+        if (skipped)
+                volinfo->rebal.skipped_files = skipped;
+        if (run_time)
+                volinfo->rebal.rebalance_time = run_time;
+
+        return ret;
+}
+
+int
+glusterd_check_topology_identical (const char *filename1,
+                                   const char *filename2,
+                                   gf_boolean_t *identical)
+{
+        int                     ret = -1; /* FAILURE */
+        xlator_t                *this = NULL;
+        FILE                    *fp1 = NULL;
+        FILE                    *fp2 = NULL;
+        glusterfs_graph_t       *grph1 = NULL;
+        glusterfs_graph_t       *grph2 = NULL;
+
+        if ((!filename1) || (!filename2) || (!identical))
+                goto out;
+
+        this = THIS;
+
+        errno = 0; /* RESET the errno */
+
+        /* fopen() the volfile1 to create the graph */
+        fp1 = fopen (filename1, "r");
+        if (fp1 == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed "
+                        "(%s)", filename1, strerror (errno));
+                goto out;
+        }
+
+        /* fopen() the volfile2 to create the graph */
+        fp2 = fopen (filename2, "r");
+        if (fp2 == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed "
+                        "(%s)", filename2, strerror (errno));
+                goto out;
+        }
+
+        /* create the graph for filename1 */
+        grph1 = glusterfs_graph_construct(fp1);
+        if (grph1 == NULL)
+                goto out;
+
+        /* create the graph for filename2 */
+        grph2 = glusterfs_graph_construct(fp2);
+        if (grph2 == NULL)
+                goto out;
+
+        /* compare the graph topology */
+        *identical = is_graph_topology_equal(grph1, grph2);
+        ret = 0; /* SUCCESS */
+out:
+        if (fp1)
+                fclose(fp1);
+        if (fp2)
+                fclose(fp2);
+        if (grph1)
+                glusterfs_graph_destroy(grph1);
+        if (grph2)
+                glusterfs_graph_destroy(grph2);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int
+glusterd_check_files_identical (char *filename1, char *filename2,
+                                gf_boolean_t *identical)
+{
+        int                     ret = -1;
+        struct stat             buf1 = {0,};
+        struct stat             buf2 = {0,};
+        uint32_t                cksum1 = 0;
+        uint32_t                cksum2 = 0;
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (filename1);
+        GF_ASSERT (filename2);
+        GF_ASSERT (identical);
+
+        this = THIS;
+
+        ret = stat (filename1, &buf1);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "stat on file: %s failed "
+                        "(%s)", filename1, strerror (errno));
+                goto out;
+        }
+
+        ret = stat (filename2, &buf2);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "stat on file: %s failed "
+                        "(%s)", filename2, strerror (errno));
+                goto out;
+        }
+
+        if (buf1.st_size != buf2.st_size) {
+                *identical = _gf_false;
+                goto out;
+        }
+
+        ret = get_checksum_for_path (filename1, &cksum1);
+        if (ret)
+                goto out;
+
+
+        ret = get_checksum_for_path (filename2, &cksum2);
+        if (ret)
+                goto out;
+
+        if (cksum1 != cksum2)
+                *identical = _gf_false;
+        else
+                *identical = _gf_true;
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int
+glusterd_volset_help (dict_t *dict, char **op_errstr)
+{
+        int                     ret = -1;
+        gf_boolean_t            xml_out = _gf_false;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+
+        if (!dict) {
+                if (!(dict = glusterd_op_get_ctx ())) {
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        if (dict_get (dict, "help" )) {
+                xml_out = _gf_false;
+
+        } else if (dict_get (dict, "help-xml" )) {
+                xml_out = _gf_true;
+#if (HAVE_LIB_XML)
+                ret = 0;
+#else
+                gf_log (this->name, GF_LOG_ERROR,
+                        "libxml not present in the system");
+                if (op_errstr)
+                        *op_errstr = gf_strdup ("Error: xml libraries not "
+                                                "present to produce "
+                                                "xml-output");
+                goto out;
+#endif
+
+        } else {
+                goto out;
+        }
+
+        ret = glusterd_get_volopt_content (dict, xml_out);
+        if (ret && op_errstr)
+                *op_errstr = gf_strdup ("Failed to get volume options help");
+ out:
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_to_cli (rpcsvc_request_t *req, gf_cli_rsp *arg, struct iovec *payload,
+                 int payloadcount, struct iobref *iobref, xdrproc_t xdrproc,
+                 dict_t *dict)
+{
+        int                ret = -1;
+        char               *cmd = NULL;
+        int                op_ret = 0;
+        char               *op_errstr = NULL;
+        int                op_errno = 0;
+        xlator_t           *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        op_ret = arg->op_ret;
+        op_errstr = arg->op_errstr;
+        op_errno = arg->op_errno;
+
+        ret = dict_get_str (dict, "cmd-str", &cmd);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get command "
+                        "string");
+
+        if (cmd) {
+                if (op_ret)
+                        gf_cmd_log ("", "%s : FAILED %s %s", cmd,
+                                       (op_errstr)? ":" : " ",
+                                       (op_errstr)? op_errstr : " ");
+                else
+                        gf_cmd_log ("", "%s : SUCCESS", cmd);
+        }
+
+        glusterd_submit_reply (req, arg, payload, payloadcount, iobref,
+                               (xdrproc_t) xdrproc);
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+static int32_t
+glusterd_append_gsync_status (dict_t *dst, dict_t *src)
+{
+        int                ret = 0;
+        char               *stop_msg = NULL;
+
+        ret = dict_get_str (src, "gsync-status", &stop_msg);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dst, "gsync-status", gf_strdup (stop_msg));
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_WARNING, "Unable to set the stop"
+                        "message in the ctx dictionary");
+                goto out;
+        }
+
+        ret = 0;
+ out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int32_t
+glusterd_append_status_dicts (dict_t *dst, dict_t *src)
+{
+        int              dst_count = 0;
+        int              src_count = 0;
+        int              i = 0;
+        int              ret = 0;
+        char             mst[PATH_MAX] = {0,};
+        char             slv[PATH_MAX] = {0, };
+        char             sts[PATH_MAX] = {0, };
+        char             nds[PATH_MAX] = {0, };
+        char             *mst_val = NULL;
+        char             *slv_val = NULL;
+        char             *sts_val = NULL;
+        char             *nds_val = NULL;
+
+        GF_ASSERT (dst);
+
+        if (src == NULL)
+                goto out;
+
+        ret = dict_get_int32 (dst, "gsync-count", &dst_count);
+        if (ret)
+                dst_count = 0;
+
+        ret = dict_get_int32 (src, "gsync-count", &src_count);
+        if (ret || !src_count) {
+                gf_log ("", GF_LOG_DEBUG, "Source brick empty");
+                ret = 0;
+                goto out;
+        }
+
+        for (i = 1; i <= src_count; i++) {
+                snprintf (nds, sizeof(nds), "node%d", i);
+                snprintf (mst, sizeof(mst), "master%d", i);
+                snprintf (slv, sizeof(slv), "slave%d", i);
+                snprintf (sts, sizeof(sts), "status%d", i);
+
+                ret = dict_get_str (src, nds, &nds_val);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (src, mst, &mst_val);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (src, slv, &slv_val);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (src, sts, &sts_val);
+                if (ret)
+                        goto out;
+
+                snprintf (nds, sizeof(nds), "node%d", i+dst_count);
+                snprintf (mst, sizeof(mst), "master%d", i+dst_count);
+                snprintf (slv, sizeof(slv), "slave%d", i+dst_count);
+                snprintf (sts, sizeof(sts), "status%d", i+dst_count);
+
+                ret = dict_set_dynstr (dst, nds, gf_strdup (nds_val));
+                if (ret)
+                        goto out;
+
+                ret = dict_set_dynstr (dst, mst, gf_strdup (mst_val));
+                if (ret)
+                        goto out;
+
+                ret = dict_set_dynstr (dst, slv, gf_strdup (slv_val));
+                if (ret)
+                        goto out;
+
+                ret = dict_set_dynstr (dst, sts, gf_strdup (sts_val));
+                if (ret)
+                        goto out;
+
+        }
+
+        ret = dict_set_int32 (dst, "gsync-count", dst_count+src_count);
+
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+int32_t
+glusterd_gsync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict, char *op_errstr)
+{
+        dict_t             *ctx = NULL;
+        int                ret = 0;
+        char               *conf_path = NULL;
+
+        if (aggr) {
+                ctx = aggr;
+
+        } else {
+                ctx = glusterd_op_get_ctx ();
+                if (!ctx) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Operation Context is not present");
+                        GF_ASSERT (0);
+                }
+        }
+
+        if (rsp_dict) {
+                ret = glusterd_append_status_dicts (ctx, rsp_dict);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_append_gsync_status (ctx, rsp_dict);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (rsp_dict, "conf_path", &conf_path);
+                if (!ret && conf_path) {
+                        ret = dict_set_dynstr (ctx, "conf_path",
+                                            gf_strdup(conf_path));
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to store conf path.");
+                                goto out;
+                        }
+                }
+        }
+        if ((op_errstr) && (strcmp ("", op_errstr))) {
+                ret = dict_set_dynstr (ctx, "errstr", gf_strdup(op_errstr));
+                if (ret)
+                        goto out;
+        }
+
+        ret = 0;
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
+        return ret;
+}
+
+int32_t
+glusterd_rb_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int32_t  src_port = 0;
+        int32_t  dst_port = 0;
+        int      ret      = 0;
+        dict_t  *ctx      = NULL;
+
+
+        if (aggr) {
+                ctx = aggr;
+
+        } else {
+                ctx = glusterd_op_get_ctx ();
+                if (!ctx) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Operation Context is not present");
+                        GF_ASSERT (0);
+                }
+        }
+
+        if (rsp_dict) {
+                ret = dict_get_int32 (rsp_dict, "src-brick-port", &src_port);
+                if (ret == 0) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "src-brick-port=%d found", src_port);
+                }
+
+                ret = dict_get_int32 (rsp_dict, "dst-brick-port", &dst_port);
+                if (ret == 0) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "dst-brick-port=%d found", dst_port);
+                }
+
+        }
+
+        if (src_port) {
+                ret = dict_set_int32 (ctx, "src-brick-port",
+                                      src_port);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not set src-brick");
+                        goto out;
+                }
+        }
+
+        if (dst_port) {
+                ret = dict_set_int32 (ctx, "dst-brick-port",
+                                      dst_port);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not set dst-brick");
+                        goto out;
+                }
+
+        }
+
+out:
+        return ret;
+
+}
+
+int32_t
+glusterd_sync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int      ret      = 0;
+
+        GF_ASSERT (rsp_dict);
+
+        if (!rsp_dict) {
+                goto out;
+        }
+
+        ret = glusterd_import_friend_volumes (rsp_dict);
+out:
+        return ret;
+
+}
+
+static int
+_profile_volume_add_friend_rsp (dict_t *this, char *key, data_t *value,
+                               void *data)
+{
+        char    new_key[256] = {0};
+        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
+        data_t  *new_value = NULL;
+        int     brick_count = 0;
+        char    brick_key[256];
+
+        if (strcmp (key, "count") == 0)
+                return 0;
+        sscanf (key, "%d%s", &brick_count, brick_key);
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        GF_ASSERT (new_value);
+        snprintf (new_key, sizeof (new_key), "%d%s",
+                  rsp_ctx->count + brick_count, brick_key);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+        return 0;
+}
+
+int
+glusterd_profile_volume_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int     ret = 0;
+        glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
+        int32_t brick_count = 0;
+        int32_t count = 0;
+        dict_t  *ctx_dict = NULL;
+        glusterd_op_t   op = GD_OP_NONE;
+
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (rsp_dict, "count", &brick_count);
+        if (ret) {
+                ret = 0; //no bricks in the rsp
+                goto out;
+        }
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT (GD_OP_PROFILE_VOLUME == op);
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx ();
+        }
+
+        ret = dict_get_int32 (ctx_dict, "count", &count);
+        rsp_ctx.count = count;
+        rsp_ctx.dict = ctx_dict;
+        dict_foreach (rsp_dict, _profile_volume_add_friend_rsp, &rsp_ctx);
+        dict_del (ctx_dict, "count");
+        ret = dict_set_int32 (ctx_dict, "count", count + brick_count);
+out:
+        return ret;
+}
+
+static int
+glusterd_volume_status_add_peer_rsp (dict_t *this, char *key, data_t *value,
+                                     void *data)
+{
+        glusterd_status_rsp_conv_t      *rsp_ctx = NULL;
+        data_t                          *new_value = NULL;
+        char                            brick_key[1024] = {0,};
+        char                            new_key[1024] = {0,};
+        int32_t                         index = 0;
+        int32_t                         ret = 0;
+
+        /* Skip the following keys, they are already present in the ctx_dict */
+        if (!strcmp (key, "count") || !strcmp (key, "cmd") ||
+            !strcmp (key, "brick-index-max") || !strcmp (key, "other-count"))
+                return 0;
+
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        GF_ASSERT (new_value);
+
+        sscanf (key, "brick%d.%s", &index, brick_key);
+
+        if (index > rsp_ctx->brick_index_max) {
+                snprintf (new_key, sizeof (new_key), "brick%d.%s",
+                          index + rsp_ctx->other_count, brick_key);
+        } else {
+                strncpy (new_key, key, sizeof (new_key));
+                new_key[sizeof (new_key) - 1] = 0;
+        }
+
+        ret = dict_set (rsp_ctx->dict, new_key, new_value);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "Unable to set key: %s in dict",
+                        key);
+
+        return 0;
+}
+
+int
+glusterd_volume_status_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int                             ret = 0;
+        glusterd_status_rsp_conv_t      rsp_ctx = {0};
+        int32_t                         cmd = GF_CLI_STATUS_NONE;
+        int32_t                         node_count = 0;
+        int32_t                         other_count = 0;
+        int32_t                         brick_index_max = -1;
+        int32_t                         rsp_node_count = 0;
+        int32_t                         rsp_other_count = 0;
+        int                             vol_count = -1;
+        int                             i = 0;
+        dict_t                          *ctx_dict = NULL;
+        char                            key[PATH_MAX] = {0,};
+        char                            *volname = NULL;
+
+        GF_ASSERT (rsp_dict);
+
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx (GD_OP_STATUS_VOLUME);
+
+        }
+
+        ret = dict_get_int32 (ctx_dict, "cmd", &cmd);
+        if (ret)
+                goto out;
+
+        if (cmd & GF_CLI_STATUS_ALL && is_origin_glusterd (ctx_dict)) {
+                ret = dict_get_int32 (rsp_dict, "vol_count", &vol_count);
+                if (ret == 0) {
+                        ret = dict_set_int32 (ctx_dict, "vol_count",
+                                              vol_count);
+                        if (ret)
+                                goto out;
+
+                        for (i = 0; i < vol_count; i++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key), "vol%d", i);
+                                ret = dict_get_str (rsp_dict, key, &volname);
+                                if (ret)
+                                        goto out;
+
+                                ret = dict_set_str (ctx_dict, key, volname);
+                                if (ret)
+                                        goto out;
+                        }
+                }
+        }
+
+        if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
+                dict_copy (rsp_dict, aggr);
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_int32 (rsp_dict, "count", &rsp_node_count);
+        if (ret) {
+                ret = 0; //no bricks in the rsp
+                goto out;
+        }
+
+        ret = dict_get_int32 (rsp_dict, "other-count", &rsp_other_count);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to get other count from rsp_dict");
+                goto out;
+        }
+
+        ret = dict_get_int32 (ctx_dict, "count", &node_count);
+        ret = dict_get_int32 (ctx_dict, "other-count", &other_count);
+        if (!dict_get (ctx_dict, "brick-index-max")) {
+                ret = dict_get_int32 (rsp_dict, "brick-index-max", &brick_index_max);
+                if (ret)
+                        goto out;
+                ret = dict_set_int32 (ctx_dict, "brick-index-max", brick_index_max);
+                if (ret)
+                        goto out;
+
+        } else {
+                ret = dict_get_int32 (ctx_dict, "brick-index-max", &brick_index_max);
+        }
+
+        rsp_ctx.count = node_count;
+        rsp_ctx.brick_index_max = brick_index_max;
+        rsp_ctx.other_count = other_count;
+        rsp_ctx.dict = ctx_dict;
+
+        dict_foreach (rsp_dict, glusterd_volume_status_add_peer_rsp, &rsp_ctx);
+
+        ret = dict_set_int32 (ctx_dict, "count", node_count + rsp_node_count);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to update node count");
+                goto out;
+        }
+
+        ret = dict_set_int32 (ctx_dict, "other-count",
+                              (other_count + rsp_other_count));
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to update other-count");
+out:
+        return ret;
+}
+
+int
+glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        char                 key[256]      = {0,};
+        char                *node_uuid     = NULL;
+        char                *node_uuid_str = NULL;
+        char                *volname       = NULL;
+        dict_t              *ctx_dict      = NULL;
+        double               elapsed_time  = 0;
+        glusterd_conf_t     *conf          = NULL;
+        glusterd_op_t        op            = GD_OP_NONE;
+        glusterd_peerinfo_t *peerinfo      = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        int32_t              index         = 0;
+        int32_t              count         = 0;
+        int32_t              current_index = 2;
+        int32_t              value32       = 0;
+        uint64_t             value         = 0;
+        char                *peer_uuid_str = NULL;
+
+        GF_ASSERT (rsp_dict);
+        conf = THIS->private;
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT ((GD_OP_REBALANCE == op) ||
+                   (GD_OP_DEFRAG_BRICK_VOLUME == op));
+
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx (op);
+
+        }
+
+        if (!ctx_dict)
+                goto out;
+
+        ret = dict_get_str (ctx_dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (rsp_dict, "count", &index);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "failed to get index");
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "node-uuid-%d", index);
+        ret = dict_get_str (rsp_dict, key, &node_uuid);
+        if (!ret) {
+                node_uuid_str = gf_strdup (node_uuid);
+
+                /* Finding the index of the node-uuid in the peer-list */
+                list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
+                        peer_uuid_str = gd_peer_uuid_str (peerinfo);
+                        if (strcmp (peer_uuid_str, node_uuid_str) == 0)
+                                break;
+
+                        current_index++;
+                }
+
+                /* Setting the largest index value as the total count. */
+                ret = dict_get_int32 (ctx_dict, "count", &count);
+                if (count < current_index) {
+                    ret = dict_set_int32 (ctx_dict, "count", current_index);
+                    if (ret)
+                            gf_log ("", GF_LOG_ERROR, "Failed to set count");
+                }
+
+                /* Setting the same index for the node, as is in the peerlist.*/
+                memset (key, 0, 256);
+                snprintf (key, 256, "node-uuid-%d", current_index);
+                ret = dict_set_dynstr (ctx_dict, key, node_uuid_str);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set node-uuid");
+                }
+        }
+
+        snprintf (key, 256, "files-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "files-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set the file count");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "size-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "size-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set the size of migration");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "lookups-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "lookups-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set lookuped file count");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "status-%d", index);
+        ret = dict_get_int32 (rsp_dict, key, &value32);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", current_index);
+                ret = dict_set_int32 (ctx_dict, key, value32);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set status");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "failures-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "failures-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set failure count");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "skipped-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "skipped-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set skipped count");
+                }
+        }
+        memset (key, 0, 256);
+        snprintf (key, 256, "run-time-%d", index);
+        ret = dict_get_double (rsp_dict, key, &elapsed_time);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "run-time-%d", current_index);
+                ret = dict_set_double (ctx_dict, key, elapsed_time);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set run-time");
+                }
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+glusterd_snap_config_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char           buf[PATH_MAX]        = "";
+        char          *volname              = NULL;
+        int            ret                  = -1;
+        int            config_command       = 0;
+        uint64_t       i                    = 0;
+        uint64_t       value                = 0;
+        uint64_t       voldisplaycount      = 0;
+
+        if (!dst || !src) {
+                gf_log ("", GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "config-command", &config_command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "failed to get config-command type");
+                goto out;
+        }
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_DISPLAY:
+                ret = dict_get_uint64 (src, "snap-max-hard-limit", &value);
+                if (!ret) {
+                        ret = dict_set_uint64 (dst, "snap-max-hard-limit", value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set snap_max_hard_limit");
+                                goto out;
+                        }
+                } else {
+                        /* Received dummy response from other nodes */
+                        ret = 0;
+                        goto out;
+                }
+
+                ret = dict_get_uint64 (src, "snap-max-soft-limit", &value);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to get snap_max_soft_limit");
+                        goto out;
+                }
+
+                ret = dict_set_uint64 (dst, "snap-max-soft-limit", value);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to set snap_max_soft_limit");
+                        goto out;
+                }
+
+                ret = dict_get_uint64 (src, "voldisplaycount",
+                                       &voldisplaycount);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to get voldisplaycount");
+                        goto out;
+                }
+
+                ret = dict_set_uint64 (dst, "voldisplaycount",
+                                       voldisplaycount);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to set voldisplaycount");
+                        goto out;
+                }
+
+                for (i = 0; i < voldisplaycount; i++) {
+                        snprintf (buf, sizeof(buf), "volume%ld-volname", i);
+                        ret = dict_get_str (src, buf, &volname);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_str (dst, buf, volname);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-hard-limit", i);
+                        ret = dict_get_uint64 (src, buf, &value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_uint64 (dst, buf, value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-active-hard-limit", i);
+                        ret = dict_get_uint64 (src, buf, &value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_uint64 (dst, buf, value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-soft-limit", i);
+                        ret = dict_get_uint64 (src, buf, &value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_uint64 (dst, buf, value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+                }
+
+                break;
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Aggregate missed_snap_counts from different nodes and save it *
+ * in the req_dict of the originator node */
+int
+glusterd_snap_create_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char          *buf                      = NULL;
+        char          *tmp_str                  = NULL;
+        char           name_buf[PATH_MAX]       = "";
+        int32_t        i                        = -1;
+        int32_t        ret                      = -1;
+        int32_t        src_missed_snap_count    = -1;
+        int32_t        dst_missed_snap_count    = -1;
+        xlator_t      *this                     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!dst || !src) {
+                gf_log (this->name, GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (src, "missed_snap_count",
+                              &src_missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "missed_snap_count",
+                              &dst_missed_snap_count);
+        if (ret) {
+                /* Initialize dst_missed_count for the first time */
+                dst_missed_snap_count = 0;
+        }
+
+        for (i = 0; i < src_missed_snap_count; i++) {
+                 snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                           i);
+                 ret = dict_get_str (src, name_buf, &buf);
+                 if (ret) {
+                         gf_log (this->name, GF_LOG_ERROR,
+                                 "Unable to fetch %s", name_buf);
+                         goto out;
+                 }
+
+                 snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                           dst_missed_snap_count);
+
+                 tmp_str = gf_strdup (buf);
+                 if (!tmp_str) {
+                         ret = -1;
+                         goto out;
+                 }
+
+                 ret = dict_set_dynstr (dst, name_buf, tmp_str);
+                 if (ret) {
+                         gf_log (this->name, GF_LOG_ERROR,
+                                 "Unable to set %s", name_buf);
+                         goto out;
+                 }
+
+                 tmp_str = NULL;
+                 dst_missed_snap_count++;
+        }
+
+        ret = dict_set_int32 (dst, "missed_snap_count", dst_missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set dst_missed_snap_count");
+                goto out;
+        }
+
+out:
+        if (ret && tmp_str)
+                GF_FREE(tmp_str);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_snap_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        int            ret            = -1;
+        int32_t        snap_command   = 0;
+
+        if (!dst || !src) {
+                gf_log ("", GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "type", &snap_command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_snap_create_use_rsp_dict (dst, src);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to use rsp dict");
+                        goto out;
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                ret = glusterd_snap_config_use_rsp_dict (dst, src);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to use rsp dict");
+                        goto out;
+                }
+                break;
+        default:
+                // copy the response dictinary's contents to the dict to be
+                // sent back to the cli
+                dict_copy (src, dst);
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_sys_exec_output_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char           output_name[PATH_MAX] = "";
+        char          *output = NULL;
+        int            ret      = 0;
+        int            i      = 0;
+        int            len    = 0;
+        int            src_output_count      = 0;
+        int            dst_output_count      = 0;
+
+        if (!dst || !src) {
+                gf_log ("", GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "output_count", &dst_output_count);
+
+        ret = dict_get_int32 (src, "output_count", &src_output_count);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "No output from source");
+                ret = 0;
+                goto out;
+        }
+
+        for (i = 1; i <= src_output_count; i++) {
+                len = snprintf (output_name, sizeof(output_name) - 1,
+                                "output_%d", i);
+                output_name[len] = '\0';
+                ret = dict_get_str (src, output_name, &output);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch %s",
+                                output_name);
+                        goto out;
+                }
+
+                len = snprintf (output_name, sizeof(output_name) - 1,
+                                "output_%d", i+dst_output_count);
+                output_name[len] = '\0';
+                ret = dict_set_dynstr (dst, output_name, gf_strdup (output));
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to set %s",
+                                output_name);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dst, "output_count",
+                              dst_output_count+src_output_count);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int            ret      = 0;
+        glusterd_op_t  op       = GD_OP_NONE;
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT (aggr);
+        GF_ASSERT (rsp_dict);
+
+        if (!aggr)
+                goto out;
+        dict_copy (rsp_dict, aggr);
+out:
+        return ret;
+}
+
+int
+glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int            ret      = 0;
+        dict_t        *ctx_dict = NULL;
+        glusterd_op_t  op       = GD_OP_NONE;
+
+        GF_ASSERT (rsp_dict);
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT (GD_OP_HEAL_VOLUME == op);
+
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx (op);
+        }
+
+        if (!ctx_dict)
+                goto out;
+        dict_copy (rsp_dict, ctx_dict);
+out:
+        return ret;
+}
+
+int
+_profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
+                             void *data)
+{
+        char    new_key[256] = {0};
+        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
+        data_t  *new_value = NULL;
+
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        GF_ASSERT (new_value);
+        snprintf (new_key, sizeof (new_key), "%d-%s", rsp_ctx->count, key);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+        return 0;
+}
+
+int
+glusterd_profile_volume_brick_rsp (void *pending_entry,
+                                   dict_t *rsp_dict, dict_t *op_ctx,
+                                   char **op_errstr, gd_node_type type)
+{
+        int                             ret = 0;
+        glusterd_pr_brick_rsp_conv_t    rsp_ctx = {0};
+        int32_t                         count = 0;
+        char                            brick[PATH_MAX+1024] = {0};
+        char                            key[256] = {0};
+        char                            *full_brick = NULL;
+        glusterd_brickinfo_t            *brickinfo = NULL;
+        xlator_t                        *this = NULL;
+        glusterd_conf_t                 *priv = NULL;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (pending_entry);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_int32 (op_ctx, "count", &count);
+        if (ret) {
+                count = 1;
+        } else {
+                count++;
+        }
+        snprintf (key, sizeof (key), "%d-brick", count);
+        if (type == GD_NODE_BRICK) {
+                brickinfo = pending_entry;
+                snprintf (brick, sizeof (brick), "%s:%s", brickinfo->hostname,
+                          brickinfo->path);
+        } else if (type == GD_NODE_NFS) {
+                snprintf (brick, sizeof (brick), "%s", uuid_utoa (MY_UUID));
+        }
+        full_brick = gf_strdup (brick);
+        GF_ASSERT (full_brick);
+        ret = dict_set_dynstr (op_ctx, key, full_brick);
+
+        rsp_ctx.count = count;
+        rsp_ctx.dict = op_ctx;
+        dict_foreach (rsp_dict, _profile_volume_add_brick_rsp, &rsp_ctx);
+        dict_del (op_ctx, "count");
+        ret = dict_set_int32 (op_ctx, "count", count);
+        return ret;
+}
+
+//input-key: <replica-id>:<child-id>-*
+//output-key: <brick-id>-*
+int
+_heal_volume_add_shd_rsp (dict_t *this, char *key, data_t *value, void *data)
+{
+        char                            new_key[256] = {0,};
+        char                            int_str[16] = {0};
+        data_t                          *new_value = NULL;
+        char                            *rxl_end = NULL;
+        char                            *rxl_child_end = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        int                             rxl_id = 0;
+        int                             rxl_child_id = 0;
+        int                             brick_id = 0;
+        int                             int_len = 0;
+        int                             ret = 0;
+        glusterd_heal_rsp_conv_t        *rsp_ctx = NULL;
+        glusterd_brickinfo_t            *brickinfo = NULL;
+
+        rsp_ctx = data;
+        rxl_end = strchr (key, '-');
+        if (!rxl_end)
+                goto out;
+
+        int_len = strlen (key) - strlen (rxl_end);
+        strncpy (int_str, key, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_id);
+        if (ret)
+                goto out;
+
+        rxl_child_end = strchr (rxl_end + 1, '-');
+        if (!rxl_child_end)
+                goto out;
+
+        int_len = strlen (rxl_end) - strlen (rxl_child_end) - 1;
+        strncpy (int_str, rxl_end + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_child_id);
+        if (ret)
+                goto out;
+
+        volinfo = rsp_ctx->volinfo;
+        brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
+
+        if (!strcmp (rxl_child_end, "-status")) {
+                brickinfo = glusterd_get_brickinfo_by_position (volinfo,
+                                                                brick_id);
+                if (!brickinfo)
+                        goto out;
+                if (!glusterd_is_local_brick (rsp_ctx->this, volinfo,
+                                              brickinfo))
+                        goto out;
+        }
+        new_value = data_copy (value);
+        snprintf (new_key, sizeof (new_key), "%d%s", brick_id, rxl_child_end);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+
+out:
+        return 0;
+}
+
+int
+_heal_volume_add_shd_rsp_of_statistics (dict_t *this, char *key, data_t
+                                             *value, void *data)
+{
+        char                            new_key[256] = {0,};
+        char                            int_str[16] = {0,};
+        char                            key_begin_string[128] = {0,};
+        data_t                          *new_value = NULL;
+        char                            *rxl_end = NULL;
+        char                            *rxl_child_end = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        char                            *key_begin_str = NULL;
+        int                             rxl_id = 0;
+        int                             rxl_child_id = 0;
+        int                             brick_id = 0;
+        int                             int_len = 0;
+        int                             ret = 0;
+        glusterd_heal_rsp_conv_t        *rsp_ctx = NULL;
+        glusterd_brickinfo_t            *brickinfo = NULL;
+
+        rsp_ctx = data;
+        key_begin_str = strchr (key, '-');
+        if (!key_begin_str)
+                goto out;
+
+        int_len = strlen (key) - strlen (key_begin_str);
+        strncpy (key_begin_string, key, int_len);
+        key_begin_string[int_len] = '\0';
+
+        rxl_end = strchr (key_begin_str + 1, '-');
+        if (!rxl_end)
+                goto out;
+
+        int_len = strlen (key_begin_str) - strlen (rxl_end) - 1;
+        strncpy (int_str, key_begin_str + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_id);
+        if (ret)
+                goto out;
+
+
+        rxl_child_end = strchr (rxl_end + 1, '-');
+        if (!rxl_child_end)
+                goto out;
+
+        int_len = strlen (rxl_end) - strlen (rxl_child_end) - 1;
+        strncpy (int_str, rxl_end + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_child_id);
+        if (ret)
+                goto out;
+
+        volinfo = rsp_ctx->volinfo;
+        brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
+
+        brickinfo = glusterd_get_brickinfo_by_position (volinfo, brick_id);
+        if (!brickinfo)
+                goto out;
+        if (!glusterd_is_local_brick (rsp_ctx->this, volinfo, brickinfo))
+                goto out;
+
+        new_value = data_copy (value);
+        snprintf (new_key, sizeof (new_key), "%s-%d%s", key_begin_string,
+                  brick_id, rxl_child_end);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+
+out:
+        return 0;
+
+}
+
+int
+glusterd_heal_volume_brick_rsp (dict_t *req_dict, dict_t *rsp_dict,
+                                dict_t *op_ctx, char **op_errstr)
+{
+        int                             ret = 0;
+        glusterd_heal_rsp_conv_t        rsp_ctx = {0};
+        char                            *volname = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        int                             heal_op = -1;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (op_errstr);
+
+        ret = dict_get_str (req_dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = dict_get_int32 (req_dict, "heal-op", &heal_op);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get heal_op");
+                goto out;
+        }
+
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret)
+                goto out;
+
+        rsp_ctx.dict = op_ctx;
+        rsp_ctx.volinfo = volinfo;
+        rsp_ctx.this = THIS;
+        if (heal_op == GF_AFR_OP_STATISTICS)
+                dict_foreach (rsp_dict, _heal_volume_add_shd_rsp_of_statistics,
+                              &rsp_ctx);
+        else
+                dict_foreach (rsp_dict, _heal_volume_add_shd_rsp, &rsp_ctx);
+
+
+out:
+        return ret;
+}
+
+int
+_status_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
+                              void *data)
+{
+        char                            new_key[256] = {0,};
+        data_t                          *new_value = 0;
+        glusterd_pr_brick_rsp_conv_t    *rsp_ctx = NULL;
+
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        snprintf (new_key, sizeof (new_key), "brick%d.%s", rsp_ctx->count, key);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+
+        return 0;
+}
+
+int
+glusterd_status_volume_brick_rsp (dict_t *rsp_dict, dict_t *op_ctx,
+                                  char **op_errstr)
+{
+        int                             ret = 0;
+        glusterd_pr_brick_rsp_conv_t    rsp_ctx = {0};
+        int32_t                         count = 0;
+        int                             index = 0;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (op_errstr);
+
+        ret = dict_get_int32 (op_ctx, "count", &count);
+        if (ret) {
+                count = 0;
+        } else {
+                count++;
+        }
+        ret = dict_get_int32 (rsp_dict, "index", &index);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get node index");
+                goto out;
+        }
+        dict_del (rsp_dict, "index");
+
+        rsp_ctx.count = index;
+        rsp_ctx.dict = op_ctx;
+        dict_foreach (rsp_dict, _status_volume_add_brick_rsp, &rsp_ctx);
+        ret = dict_set_int32 (op_ctx, "count", count);
+
+out:
+        return ret;
+}
+
+int
+glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
+                                 dict_t *op_ctx)
+{
+        int                             ret = 0;
+        char                            *volname = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        char                            key[256] = {0,};
+        int32_t                         i = 0;
+        char                            buf[1024] = {0,};
+        char                            *node_str = NULL;
+        glusterd_conf_t                 *priv = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (req_dict);
+
+        ret = dict_get_str (req_dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret)
+                goto out;
+
+        if (rsp_dict) {
+                ret = glusterd_defrag_volume_status_update (volinfo,
+                                                            rsp_dict);
+        }
+
+        if (!op_ctx) {
+                dict_copy (rsp_dict, op_ctx);
+                goto out;
+        }
+
+        ret = dict_get_int32 (op_ctx, "count", &i);
+        i++;
+
+        ret = dict_set_int32 (op_ctx, "count", i);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set count");
+
+        snprintf (buf, 1024, "%s", uuid_utoa (MY_UUID));
+        node_str = gf_strdup (buf);
+
+        snprintf (key, 256, "node-uuid-%d",i);
+        ret = dict_set_dynstr (op_ctx, key, node_str);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set node-uuid");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "files-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set file count");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "size-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_data);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set size of xfer");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "lookups-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.lookedup_files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set lookedup file count");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "status-%d", i);
+        ret = dict_set_int32 (op_ctx, key, volinfo->rebal.defrag_status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set status");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "failures-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set failure count");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "skipped-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.skipped_files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set skipped count");
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "run-time-%d", i);
+        ret = dict_set_double (op_ctx, key, volinfo->rebal.rebalance_time);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set run-time");
+
+out:
+        return ret;
+}
+int32_t
+glusterd_handle_node_rsp (dict_t *req_dict, void *pending_entry,
+                          glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
+                          char **op_errstr, gd_node_type type)
+{
+        int                     ret = 0;
+
+        GF_ASSERT (op_errstr);
+
+        switch (op) {
+        case GD_OP_PROFILE_VOLUME:
+                ret = glusterd_profile_volume_brick_rsp (pending_entry,
+                                                         rsp_dict, op_ctx,
+                                                         op_errstr, type);
+                break;
+        case GD_OP_STATUS_VOLUME:
+                ret = glusterd_status_volume_brick_rsp (rsp_dict, op_ctx,
+                                                        op_errstr);
+                break;
+
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                glusterd_defrag_volume_node_rsp (req_dict,
+                                                 rsp_dict, op_ctx);
+                break;
+
+        case GD_OP_HEAL_VOLUME:
+                ret = glusterd_heal_volume_brick_rsp (req_dict, rsp_dict,
+                                                      op_ctx, op_errstr);
+                break;
+        default:
+                break;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Should be used only when an operation is in progress, as that is the only
+ * time a lock_owner is set
+ */
+gf_boolean_t
+is_origin_glusterd (dict_t *dict)
+{
+        gf_boolean_t  ret              = _gf_false;
+        uuid_t        lock_owner       = {0,};
+        uuid_t        *originator_uuid = NULL;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_bin (dict, "originator_uuid",
+                            (void **) &originator_uuid);
+        if (ret) {
+                /* If not originator_uuid has been set, then the command
+                 * has been originated from a glusterd running on older version
+                 * Hence fetching the lock owner */
+                ret = glusterd_get_lock_owner (&lock_owner);
+                if (ret) {
+                        ret = _gf_false;
+                        goto out;
+                }
+                ret = !uuid_compare (MY_UUID, lock_owner);
+        } else
+                ret = !uuid_compare (MY_UUID, *originator_uuid);
+
+out:
+        return ret;
+}
+
+int
+glusterd_generate_and_set_task_id (dict_t *dict, char *key)
+{
+        int             ret = -1;
+        uuid_t          task_id = {0,};
+        char            *uuid_str = NULL;
+        xlator_t        *this = NULL;
+
+        GF_ASSERT (dict);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        uuid_generate (task_id);
+        uuid_str = gf_strdup (uuid_utoa (task_id));
+        if (!uuid_str) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dict, key, uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set %s in dict",
+                        key);
+                goto out;
+        }
+        gf_log (this->name, GF_LOG_INFO, "Generated task-id %s for key %s",
+                uuid_str, key);
+
+out:
+        if (ret)
+                GF_FREE (uuid_str);
+        return ret;
+}
+
+int
+glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key)
+{
+        int             ret = -1;
+        char            tmp_str[40] = {0,};
+        char            *task_id_str = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key);
+
+        uuid_unparse (uuid, tmp_str);
+        task_id_str = gf_strdup (tmp_str);
+        if (!task_id_str)
+                return -1;
+
+        ret = dict_set_dynstr (dict, key, task_id_str);
+        if (ret) {
+                GF_FREE (task_id_str);
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Error setting uuid in dict with key %s", key);
+        }
+
+        return 0;
+}
+
+int
+_update_volume_op_versions (dict_t *this, char *key, data_t *value, void *data)
+{
+        int                op_version = 0;
+        glusterd_volinfo_t *ctx       = NULL;
+        gf_boolean_t       enabled    = _gf_true;
+        int                ret        = -1;
+
+        GF_ASSERT (data);
+        ctx = data;
+
+        op_version = glusterd_get_op_version_for_key (key);
+
+        if (gd_is_xlator_option (key) || gd_is_boolean_option (key)) {
+                ret = gf_string2boolean (value->data, &enabled);
+                if (ret)
+                        return 0;
+
+                if (!enabled)
+                        return 0;
+        }
+
+        if (op_version > ctx->op_version)
+                ctx->op_version = op_version;
+
+        if (gd_is_client_option (key) &&
+            (op_version > ctx->client_op_version))
+                ctx->client_op_version = op_version;
+
+        return 0;
+}
+
+void
+gd_update_volume_op_versions (glusterd_volinfo_t *volinfo)
+{
+        glusterd_conf_t *conf = NULL;
+        gf_boolean_t    ob_enabled = _gf_false;
+
+        GF_ASSERT (volinfo);
+
+        conf = THIS->private;
+        GF_ASSERT (conf);
+
+        /* Reset op-versions to minimum */
+        volinfo->op_version = 1;
+        volinfo->client_op_version = 1;
+
+        dict_foreach (volinfo->dict, _update_volume_op_versions, volinfo);
+
+        /* Special case for open-behind
+         * If cluster op-version >= 2 and open-behind hasn't been explicitly
+         * disabled, volume op-versions must be updated to account for it
+         */
+
+        /* TODO: Remove once we have a general way to update automatically
+         * enabled features
+         */
+        if (conf->op_version >= 2) {
+                ob_enabled = dict_get_str_boolean (volinfo->dict,
+                                                   "performance.open-behind",
+                                                   _gf_true);
+                if (ob_enabled) {
+
+                        if (volinfo->op_version < 2)
+                                volinfo->op_version = 2;
+                        if (volinfo->client_op_version < 2)
+                                volinfo->client_op_version = 2;
+                }
+        }
+
+        return;
+}
+
+/* A task is committed/completed once the task-id for it is cleared */
+gf_boolean_t
+gd_is_remove_brick_committed (glusterd_volinfo_t *volinfo)
+{
+        GF_ASSERT (volinfo);
+
+        if ((GD_OP_REMOVE_BRICK == volinfo->rebal.op) &&
+            !uuid_is_null (volinfo->rebal.rebalance_id))
+                        return _gf_false;
+
+        return _gf_true;
+}
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
+                               struct list_head *peers,
+                               char **down_peerstr)
+{
+        glusterd_peerinfo_t   *peerinfo  = NULL;
+        glusterd_brickinfo_t  *brickinfo = NULL;
+        gf_boolean_t           ret       = _gf_false;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (!uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                list_for_each_entry (peerinfo, peers, uuid_list) {
+                        if (uuid_compare (peerinfo->uuid, brickinfo->uuid))
+                                continue;
+
+                        /*Found peer who owns the brick, return false
+                         * if peer is not connected or not friend */
+                        if (!(peerinfo->connected) ||
+                           (peerinfo->state.state !=
+                             GD_FRIEND_STATE_BEFRIENDED)) {
+                                *down_peerstr = gf_strdup (peerinfo->hostname);
+                                gf_log ("", GF_LOG_DEBUG, "Peer %s is down. ",
+                                        peerinfo->hostname);
+                                goto out;
+                        }
+                }
+       }
+
+        ret = _gf_true;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+gf_boolean_t
+glusterd_is_status_tasks_op (glusterd_op_t op, dict_t *dict)
+{
+        int           ret             = -1;
+        uint32_t      cmd             = GF_CLI_STATUS_NONE;
+        gf_boolean_t  is_status_tasks = _gf_false;
+
+        if (op != GD_OP_STATUS_VOLUME)
+                goto out;
+
+        ret = dict_get_uint32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get opcode");
+                goto out;
+        }
+
+        if (cmd & GF_CLI_STATUS_TASKS)
+                is_status_tasks = _gf_true;
+
+out:
+        return is_status_tasks;
+}
+
+int
+glusterd_compare_snap_time(struct list_head *list1, struct list_head *list2)
+{
+        glusterd_snap_t *snap1 = NULL;
+        glusterd_snap_t *snap2 = NULL;
+        double diff_time       = 0;
+
+        GF_ASSERT (list1);
+        GF_ASSERT (list2);
+
+        snap1 = list_entry(list1, glusterd_snap_t, snap_list);
+        snap2 = list_entry(list2, glusterd_snap_t, snap_list);
+        diff_time = difftime(snap1->time_stamp, snap2->time_stamp);
+
+        return ((int)diff_time);
+}
+
+int
+glusterd_compare_snap_vol_time(struct list_head *list1, struct list_head *list2)
+{
+        glusterd_volinfo_t *snapvol1 = NULL;
+        glusterd_volinfo_t *snapvol2 = NULL;
+        double diff_time             = 0;
+
+        GF_ASSERT (list1);
+        GF_ASSERT (list2);
+
+        snapvol1 = list_entry(list1, glusterd_volinfo_t, snapvol_list);
+        snapvol2 = list_entry(list2, glusterd_volinfo_t, snapvol_list);
+        diff_time = difftime(snapvol1->snapshot->time_stamp,
+                             snapvol2->snapshot->time_stamp);
+
+        return ((int)diff_time);
+}
+
+int32_t
+glusterd_missed_snapinfo_new (glusterd_missed_snap_info **missed_snapinfo)
+{
+        glusterd_missed_snap_info      *new_missed_snapinfo = NULL;
+        int32_t                         ret                 = -1;
+        xlator_t                       *this                = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (missed_snapinfo);
+
+        new_missed_snapinfo = GF_CALLOC (1, sizeof(*new_missed_snapinfo),
+                                         gf_gld_mt_missed_snapinfo_t);
+
+        if (!new_missed_snapinfo)
+                goto out;
+
+        new_missed_snapinfo->node_snap_info = NULL;
+        INIT_LIST_HEAD (&new_missed_snapinfo->missed_snaps);
+        INIT_LIST_HEAD (&new_missed_snapinfo->snap_ops);
+
+        *missed_snapinfo = new_missed_snapinfo;
+
+        ret = 0;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_missed_snap_op_new (glusterd_snap_op_t **snap_op)
+{
+        glusterd_snap_op_t      *new_snap_op = NULL;
+        int32_t                  ret         = -1;
+        xlator_t                *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (snap_op);
+
+        new_snap_op = GF_CALLOC (1, sizeof(*new_snap_op),
+                                 gf_gld_mt_missed_snapinfo_t);
+
+        if (!new_snap_op)
+                goto out;
+
+        new_snap_op->brick_path = NULL;
+        new_snap_op->brick_num = -1;
+        new_snap_op->op = -1;
+        new_snap_op->status = -1;
+        INIT_LIST_HEAD (&new_snap_op->snap_ops_list);
+
+        *snap_op = new_snap_op;
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index e6a655ca7..56bb799bf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_UTILS_H
 #define _GLUSTERD_UTILS_H_
 
@@ -38,24 +28,23 @@
 #include "rpc-clnt.h"
 #include "protocol-common.h"
 
-/* For brick search using path: partial or complete */
-typedef enum {
-        GF_PATH_COMPLETE = 0,
-        GF_PATH_PARTIAL
-} gf_path_match_t;
+#define GLUSTERD_SOCK_DIR "/var/run"
 
 struct glusterd_lock_ {
         uuid_t  owner;
         time_t  timestamp;
 };
 
-typedef struct glusterd_voldict_ctx_ {
+typedef struct glusterd_dict_ctx_ {
         dict_t  *dict;
-        int     count;
         int     opt_count;
         char    *key_name;
         char    *val_name;
-} glusterd_voldict_ctx_t;
+        char    *prefix;
+} glusterd_dict_ctx_t;
+
+int
+glusterd_compare_lines (const void *a, const void *b);
 
 typedef int (*glusterd_condition_func) (glusterd_volinfo_t *volinfo,
                                         glusterd_brickinfo_t *brickinfo,
@@ -80,6 +69,11 @@ glusterd_submit_reply (rpcsvc_request_t *req, void *arg,
                        struct iobref *iobref, xdrproc_t xdrproc);
 
 int
+glusterd_to_cli (rpcsvc_request_t *req, gf_cli_rsp *arg, struct iovec *payload,
+                 int payloadcount, struct iobref *iobref, xdrproc_t xdrproc,
+                 dict_t *dict);
+
+int
 glusterd_submit_request (struct rpc_clnt *rpc, void *req,
                          call_frame_t *frame, rpc_clnt_prog_t *prog,
                          int procnum, struct iobref *iobref,
@@ -87,6 +81,26 @@ glusterd_submit_request (struct rpc_clnt *rpc, void *req,
 int32_t
 glusterd_volinfo_new (glusterd_volinfo_t **volinfo);
 
+int32_t
+glusterd_volinfo_dup (glusterd_volinfo_t *volinfo,
+                      glusterd_volinfo_t **dup_volinfo,
+                      gf_boolean_t set_userauth);
+
+char *
+glusterd_auth_get_username (glusterd_volinfo_t *volinfo);
+
+char *
+glusterd_auth_get_password (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_auth_set_username (glusterd_volinfo_t *volinfo, char *username);
+
+int32_t
+glusterd_auth_set_password (glusterd_volinfo_t *volinfo, char *password);
+
+void
+glusterd_auth_cleanup (glusterd_volinfo_t *volinfo);
+
 gf_boolean_t
 glusterd_check_volume_exists (char *volname);
 
@@ -94,7 +108,7 @@ int32_t
 glusterd_brickinfo_new (glusterd_brickinfo_t **brickinfo);
 
 int32_t
-glusterd_brickinfo_from_brick (char *brick, glusterd_brickinfo_t **brickinfo);
+glusterd_brickinfo_new_from_brick (char *brick, glusterd_brickinfo_t **brickinfo);
 
 int32_t
 glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo);
@@ -106,8 +120,23 @@ int32_t
 glusterd_peer_hostname_new (char *hostname, glusterd_peer_hostname_t **name);
 
 int32_t
+glusterd_snap_volinfo_find (char *volname, glusterd_snap_t *snap,
+                            glusterd_volinfo_t **volinfo);
+int32_t
+glusterd_snap_volinfo_find_from_parent_volname (char *origin_volname,
+                                      glusterd_snap_t *snap,
+                                      glusterd_volinfo_t **volinfo);
+
+int32_t
 glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo);
 
+int
+glusterd_volinfo_find_by_volume_id (uuid_t volume_id, glusterd_volinfo_t **volinfo);
+
+int
+glusterd_snap_volinfo_find_by_volume_id (uuid_t volume_id,
+                                         glusterd_volinfo_t **volinfo);
+
 int32_t
 glusterd_service_stop(const char *service, char *pidfile, int sig,
                       gf_boolean_t force_kill);
@@ -117,11 +146,13 @@ glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo);
 
 int32_t
 glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
-                                 glusterd_brickinfo_t   *brickinfo);
+                                 glusterd_brickinfo_t   *brickinfo,
+                                 gf_boolean_t wait);
 
 int32_t
 glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
-                                glusterd_brickinfo_t   *brickinfo);
+                                glusterd_brickinfo_t   *brickinfo,
+                                gf_boolean_t del_brick);
 
 int32_t
 glusterd_volinfo_delete (glusterd_volinfo_t *volinfo);
@@ -135,16 +166,13 @@ glusterd_is_cli_op_req (int32_t op);
 int32_t
 glusterd_volume_brickinfo_get_by_brick (char *brick,
                                         glusterd_volinfo_t *volinfo,
-                                        glusterd_brickinfo_t **brickinfo,
-                                        gf_path_match_t path_match);
-int32_t
-glusterd_is_local_addr (char *hostname);
+                                        glusterd_brickinfo_t **brickinfo);
 
 int32_t
 glusterd_build_volume_dict (dict_t **vols);
 
 int32_t
-glusterd_compare_friend_data (dict_t  *vols, int32_t *status);
+glusterd_compare_friend_data (dict_t  *vols, int32_t *status, char *hostname);
 
 int
 glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo);
@@ -177,27 +205,30 @@ glusterd_shd_start ();
 int32_t
 glusterd_shd_stop ();
 
+void
+glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len);
+
 int32_t
-glusterd_shd_set_socket_filepath (char *rundir, uuid_t uuid,
-                                  char *socketpath, int len);
+glusterd_nodesvc_set_socket_filepath (char *rundir, uuid_t uuid,
+                                      char *socketpath, int len);
 
 struct rpc_clnt*
 glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node);
 
 struct rpc_clnt*
-glusterd_shd_get_rpc (void);
+glusterd_nodesvc_get_rpc (char *server);
 
 int32_t
-glusterd_shd_set_rpc (struct rpc_clnt *rpc);
+glusterd_nodesvc_set_rpc (char *server, struct rpc_clnt *rpc);
 
 int32_t
-glusterd_shd_connect (char *socketpath);
+glusterd_nodesvc_connect (char *server, char *socketpath);
 
 void
-glusterd_shd_set_running (gf_boolean_t status);
+glusterd_nodesvc_set_online_status (char *server, gf_boolean_t status);
 
 gf_boolean_t
-glusterd_shd_is_running ();
+glusterd_is_nodesvc_online (char *server);
 
 int
 glusterd_remote_hostname_get (rpcsvc_request_t *req,
@@ -231,8 +262,8 @@ int32_t
 glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                              dict_t  *dict, int32_t count);
 int
-glusterd_get_brickinfo (xlator_t *this, const char *brickname, 
-                        int port, gf_boolean_t localhost, 
+glusterd_get_brickinfo (xlator_t *this, const char *brickname,
+                        int port, gf_boolean_t localhost,
                         glusterd_brickinfo_t **brickinfo);
 
 void
@@ -256,16 +287,19 @@ glusterd_all_volume_cond_check (glusterd_condition_func func, int status,
                                 void *ctx);
 int
 glusterd_brick_start (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *brickinfo);
+                      glusterd_brickinfo_t *brickinfo,
+                      gf_boolean_t wait);
 int
 glusterd_brick_stop (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *brickinfo);
+                     glusterd_brickinfo_t *brickinfo,
+                     gf_boolean_t del_brick);
 
 int
 glusterd_is_defrag_on (glusterd_volinfo_t *volinfo);
 
 int32_t
 glusterd_volinfo_bricks_delete (glusterd_volinfo_t *volinfo);
+
 int
 glusterd_friend_find_by_uuid (uuid_t uuid,
                               glusterd_peerinfo_t  **peerinfo);
@@ -274,11 +308,12 @@ glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
                              char *op_errstr, size_t len);
 int32_t
 glusterd_volume_brickinfos_delete (glusterd_volinfo_t *volinfo);
+
 int32_t
 glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path,
                                glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t **brickinfo,
-                               gf_path_match_t path_match);
+                               glusterd_brickinfo_t **brickinfo);
+
 int
 glusterd_brickinfo_get (uuid_t uuid, char *hostname, char *path,
                        glusterd_brickinfo_t **brickinfo);
@@ -296,19 +331,25 @@ glusterd_is_rb_ongoing (glusterd_volinfo_t *volinfo);
 
 int
 glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
-                          glusterd_brickinfo_t *src_brick, glusterd_brickinfo_t *dst_brick);
+                          glusterd_brickinfo_t *src_brick,
+                          glusterd_brickinfo_t *dst_brick);
+
+int
+glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
+                                    char **op_errstr, gf_boolean_t is_force);
 
 int
-glusterd_brick_create_path (char *host, char *path, uuid_t uuid, mode_t mode,
-                            char **op_errstr);
+glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
+                                        uuid_t volume_id, char **op_errstr,
+                                        gf_boolean_t is_force);
 int
 glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
                                            int old_state, int new_state,
                                            int event);
 int
 glusterd_peerinfo_new (glusterd_peerinfo_t **peerinfo,
-                       glusterd_friend_sm_state_t state,
-                       uuid_t *uuid, const char *hostname);
+                       glusterd_friend_sm_state_t state, uuid_t *uuid,
+                       const char *hostname, int port);
 int
 glusterd_sm_tr_log_init (glusterd_sm_tr_log_t *log,
                          char * (*state_name_get) (int),
@@ -328,7 +369,7 @@ gf_boolean_t
 glusterd_peerinfo_is_uuid_unknown (glusterd_peerinfo_t *peerinfo);
 int32_t
 glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
-                        glusterd_brickinfo_t  *brickinfo);
+                        glusterd_brickinfo_t  *brickinfo, char *socketpath);
 int32_t
 glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo);
 int32_t
@@ -336,22 +377,31 @@ glusterd_delete_volume (glusterd_volinfo_t *volinfo);
 int32_t
 glusterd_delete_brick (glusterd_volinfo_t* volinfo,
                        glusterd_brickinfo_t *brickinfo);
+
 int32_t
 glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo);
+
+int
+glusterd_spawn_daemons (void *opaque);
+
 int
 glusterd_restart_gsyncds (glusterd_conf_t *conf);
+
 int
 glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
-                      char *glusterd_uuid_str, char **op_errstr);
+                      char *path_list, char *conf_path,
+                      char *glusterd_uuid_str,
+                      char **op_errstr);
+int
+glusterd_get_local_brickpaths (glusterd_volinfo_t *volinfo,
+                               char **pathlist);
+
 int32_t
 glusterd_recreate_bricks (glusterd_conf_t *conf);
 int32_t
 glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf);
 
 int
-glusterd_get_brick_root (char *path, char **mount_point);
-
-int
 glusterd_add_brick_detail_to_dict (glusterd_volinfo_t *volinfo,
                                    glusterd_brickinfo_t *brickinfo,
                                    dict_t  *dict, int32_t count);
@@ -386,4 +436,199 @@ glusterd_friend_remove_cleanup_vols (uuid_t uuid);
 
 gf_boolean_t
 glusterd_chk_peers_connected_befriended (uuid_t skip_uuid);
+
+void
+glusterd_get_client_filepath (char *filepath,
+                              glusterd_volinfo_t *volinfo,
+                              gf_transport_type type);
+void
+glusterd_get_trusted_client_filepath (char *filepath,
+                                      glusterd_volinfo_t *volinfo,
+                                      gf_transport_type type);
+int
+glusterd_restart_rebalance (glusterd_conf_t *conf);
+
+int32_t
+glusterd_add_bricks_hname_path_to_dict (dict_t *dict,
+                                        glusterd_volinfo_t *volinfo);
+
+int
+glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
+                           dict_t *vol_opts);
+
+char *
+glusterd_uuid_to_hostname (uuid_t uuid);
+
+int
+glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo);
+
+glusterd_brickinfo_t*
+glusterd_get_brickinfo_by_position (glusterd_volinfo_t *volinfo, uint32_t pos);
+
+gf_boolean_t
+glusterd_is_local_brick (xlator_t *this, glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo);
+int
+glusterd_validate_volume_id (dict_t *op_dict, glusterd_volinfo_t *volinfo);
+
+int
+glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
+                                      dict_t *rsp_dict);
+
+int
+glusterd_check_files_identical (char *filename1, char *filename2,
+                                gf_boolean_t *identical);
+
+int
+glusterd_check_topology_identical (const char *filename1,
+                                   const char *filename2,
+                                   gf_boolean_t *identical);
+
+void
+glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo);
+int
+glusterd_volset_help (dict_t *dict, char **op_errstr);
+
+int32_t
+glusterd_sync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int32_t
+glusterd_gsync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict, char *op_errstr);
+int32_t
+glusterd_rb_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_profile_volume_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_status_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_sys_exec_output_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_snap_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int32_t
+glusterd_handle_node_rsp (dict_t *req_ctx, void *pending_entry,
+                          glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
+                          char **op_errstr, gd_node_type type);
+int
+glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+_profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
+                             void *data);
+int
+glusterd_profile_volume_brick_rsp (void *pending_entry,
+                                   dict_t *rsp_dict, dict_t *op_ctx,
+                                   char **op_errstr, gd_node_type type);
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
+                               struct list_head *peers,
+                               char **down_peerstr);
+
+/* Should be used only when an operation is in progress, as that is the only
+ * time a lock_owner is set
+ */
+gf_boolean_t
+is_origin_glusterd (dict_t *dict);
+
+gf_boolean_t
+glusterd_is_quorum_changed (dict_t *options, char *option, char *value);
+
+int
+glusterd_do_quorum_action ();
+
+int
+glusterd_get_quorum_cluster_counts (xlator_t *this, int *active_count,
+                                    int *quorum_count);
+
+int
+glusterd_get_next_global_opt_version_str (dict_t *opts, char **version_str);
+gf_boolean_t
+glusterd_is_quorum_option (char *option);
+gf_boolean_t
+glusterd_is_volume_in_server_quorum (glusterd_volinfo_t *volinfo);
+gf_boolean_t
+glusterd_is_any_volume_in_server_quorum (xlator_t *this);
+gf_boolean_t
+does_gd_meet_server_quorum (xlator_t *this);
+
+int
+glusterd_generate_and_set_task_id (dict_t *dict, char *key);
+
+int
+glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key);
+
+gf_boolean_t
+glusterd_is_same_address (char *name1, char *name2);
+
+void
+gd_update_volume_op_versions (glusterd_volinfo_t *volinfo);
+
+char*
+gd_peer_uuid_str (glusterd_peerinfo_t *peerinfo);
+
+gf_boolean_t
+gd_is_remove_brick_committed (glusterd_volinfo_t *volinfo);
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
+                               struct list_head *peers,
+                               char **down_peerstr);
+
+int
+glusterd_get_slave_details_confpath (glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char **slave_ip, char **slave_vol,
+                                     char **conf_path, char **op_errstr);
+
+int
+glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
+                                      dict_t *resp_dict, char *path_list,
+                                      char *conf_path, gf_boolean_t is_force);
+
+int
+glusterd_check_gsync_running_local (char *master, char *slave,
+                                    char *conf_path,
+                                    gf_boolean_t *is_run);
+
+gf_boolean_t
+glusterd_is_status_tasks_op (glusterd_op_t op, dict_t *dict);
+
+#ifdef GF_LINUX_HOST_OS
+char*
+glusterd_get_brick_mount_details (glusterd_brickinfo_t *brickinfo);
+struct mntent *
+glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab);
+int
+glusterd_get_brick_root (char *path, char **mount_point);
+#endif //LINUX_HOST
+
+int
+glusterd_compare_snap_time(struct list_head *, struct list_head *);
+
+int
+glusterd_compare_snap_vol_time(struct list_head *, struct list_head *);
+
+int32_t
+glusterd_snap_volinfo_restore (dict_t *rsp_dict,
+                               glusterd_volinfo_t *new_volinfo,
+                               glusterd_volinfo_t *snap_volinfo);
+int32_t
+glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol);
+
+int32_t
+glusterd_missed_snapinfo_new (glusterd_missed_snap_info **missed_snapinfo);
+
+int32_t
+glusterd_missed_snap_op_new (glusterd_snap_op_t **snap_op);
+
+int32_t
+glusterd_add_missed_snaps_to_dict (dict_t *rsp_dict, char *snap_uuid,
+                                   glusterd_brickinfo_t *brickinfo,
+                                   int32_t brick_number, int32_t op);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 798ff0565..6f3c69e7d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -25,6 +15,7 @@
 
 #include <fnmatch.h>
 #include <sys/wait.h>
+#include <dlfcn.h>
 
 #if (HAVE_LIB_XML)
 #include <libxml/encoding.h>
@@ -37,6 +28,8 @@
 #include "logging.h"
 #include "dict.h"
 #include "graph-utils.h"
+#include "glusterd-store.h"
+#include "glusterd-hooks.h"
 #include "trie.h"
 #include "glusterd-mem-types.h"
 #include "cli1-xdr.h"
@@ -44,167 +37,9 @@
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "run.h"
+#include "options.h"
 
-
-/* dispatch table for VOLUME SET
- * -----------------------------
- *
- * Format of entries:
- *
- * First field is the <key>, for the purpose of looking it up
- * in volume dictionary. Each <key> is of the format "<domain>.<specifier>".
- *
- * Second field is <voltype>.
- *
- * Third field is <option>, if its unset, it's assumed to be
- * the same as <specifier>.
- *
- * Fourth field is <value>. In this context they are used to specify
- * a default. That is, even the volume dict doesn't have a value,
- * we procced as if the default value were set for it.
- *
- * There are two type of entries: basic and special.
- *
- * - Basic entries are the ones where the <option> does _not_ start with
- *   the bang! character ('!').
- *
- *   In their case, <option> is understood as an option for an xlator of
- *   type <voltype>. Their effect is to copy over the volinfo->dict[<key>]
- *   value to all graph nodes of type <voltype> (if such a value is set).
- *
- *   You are free to add entries of this type, they will become functional
- *   just by being present in the table.
- *
- * - Special entries where the <option> starts with the bang!.
- *
- *   They are not applied to all graphs during generation, and you cannot
- *   extend them in a trivial way which could be just picked up. Better
- *   not touch them unless you know what you do.
- *
- * "NODOC" entries are not part of the public interface and are subject
- * to change at any time.
- *
- * Another kind of grouping for options, according to visibility:
- *
- * - Exported: one which is used in the code. These are characterized by
- *   being used a macro as <key> (of the format VKEY_..., defined in
- *   glusterd-volgen.h
- *
- * - Non-exported: the rest; these have string literal <keys>.
- *
- * Adhering to this policy, option name changes shall be one-liners.
- *
- */
-typedef enum  { DOC, NO_DOC, GLOBAL_DOC, GLOBAL_NO_DOC } option_type_t;
-
-
-struct volopt_map_entry {
-        char *key;
-        char *voltype;
-        char *option;
-        char *value;
-        option_type_t type;
-        uint32_t flags;
-};
-
-static struct volopt_map_entry glusterd_volopt_map[] = {
-
-        {"cluster.lookup-unhashed",              "cluster/distribute", NULL, NULL, NO_DOC, 0    },
-        {"cluster.min-free-disk",                "cluster/distribute", NULL, NULL, NO_DOC, 0    },
-	{"cluster.min-free-inodes",              "cluster/distribute", NULL, NULL, NO_DOC, 0    },
-
-        {"cluster.entry-change-log",             "cluster/replicate",  NULL, NULL, NO_DOC, 0     },
-        {"cluster.read-subvolume",               "cluster/replicate",  NULL, NULL, NO_DOC, 0    },
-        {"cluster.background-self-heal-count",   "cluster/replicate",  NULL, NULL, NO_DOC, 0    },
-        {"cluster.metadata-self-heal",           "cluster/replicate",  NULL, NULL, NO_DOC, 0     },
-        {"cluster.data-self-heal",               "cluster/replicate",  NULL, NULL, NO_DOC, 0     },
-        {"cluster.entry-self-heal",              "cluster/replicate",  NULL, NULL, NO_DOC, 0     },
-        {"cluster.self-heal-daemon",             "cluster/replicate",  "!self-heal-daemon" , NULL, NO_DOC, 0     },
-        {"cluster.strict-readdir",               "cluster/replicate",  NULL, NULL, NO_DOC, 0     },
-        {"cluster.self-heal-window-size",        "cluster/replicate",         "data-self-heal-window-size", NULL, DOC, 0},
-        {"cluster.data-change-log",              "cluster/replicate",  NULL, NULL, NO_DOC, 0     },
-        {"cluster.metadata-change-log",          "cluster/replicate",  NULL, NULL, NO_DOC, 0     },
-        {"cluster.data-self-heal-algorithm",     "cluster/replicate",         "data-self-heal-algorithm", NULL,DOC, 0},
-        {"cluster.quorum-type",                  "cluster/replicate",  "quorum-type", NULL, NO_DOC, 0},
-        {"cluster.quorum-count",                 "cluster/replicate",  "quorum-count", NULL, NO_DOC, 0},
-
-        {"cluster.stripe-block-size",            "cluster/stripe",            "block-size", NULL, DOC, 0},
-
-        {VKEY_DIAG_LAT_MEASUREMENT,              "debug/io-stats",     "latency-measurement", "off", NO_DOC, 0      },
-        {"diagnostics.dump-fd-stats",            "debug/io-stats",     NULL, NULL, NO_DOC, 0     },
-        {VKEY_DIAG_CNT_FOP_HITS,                 "debug/io-stats",     "count-fop-hits", "off", NO_DOC, 0     },
-
-        {"diagnostics.brick-log-level",          "debug/io-stats",     "!brick-log-level", NULL, DOC, 0},
-        {"diagnostics.client-log-level",         "debug/io-stats",     "!client-log-level", NULL, DOC, 0},
-        {"diagnostics.brick-sys-log-level",      "debug/io-stats",     "!sys-log-level", NULL, DOC, 0},
-        {"diagnostics.client-sys-log-level",     "debug/io-stats",     "!sys-log-level", NULL, DOC, 0},
-
-        {"performance.cache-max-file-size",      "performance/io-cache",      "max-file-size", NULL, DOC, 0},
-        {"performance.cache-min-file-size",      "performance/io-cache",      "min-file-size", NULL, DOC, 0},
-        {"performance.cache-refresh-timeout",    "performance/io-cache",      "cache-timeout", NULL, DOC, 0},
-        {"performance.cache-priority",           "performance/io-cache",      "priority", NULL, DOC, 0},
-        {"performance.cache-size",               "performance/io-cache",   NULL, NULL, NO_DOC, 0 },
-        {"performance.cache-size",               "performance/quick-read", NULL, NULL, NO_DOC, 0 },
-        {"performance.flush-behind",             "performance/write-behind",      "flush-behind", NULL, DOC, 0},
-
-        {"performance.io-thread-count",          "performance/io-threads",    "thread-count", DOC, 0},
-
-        {"performance.disk-usage-limit",         "performance/quota",   NULL, NULL, NO_DOC, 0    },
-        {"performance.min-free-disk-limit",      "performance/quota",   NULL, NULL, NO_DOC, 0    },
-
-        {"performance.write-behind-window-size", "performance/write-behind",  "cache-size", NULL, DOC},
-
-        {"performance.read-ahead-page-count",    "performance/read-ahead",  "page-count", NULL, DOC},
-
-        {"network.frame-timeout",                "protocol/client",    NULL, NULL, NO_DOC, 0     },
-        {"network.ping-timeout",                 "protocol/client",    NULL, NULL, NO_DOC, 0     },
-        {"network.inode-lru-limit",              "protocol/server",    NULL, NULL, NO_DOC, 0     },
-
-        {"auth.allow",                           "protocol/server",           "!server-auth", "*", DOC, 0},
-        {"auth.reject",                          "protocol/server",           "!server-auth", NULL, DOC, 0},
-
-        {"transport.keepalive",                   "protocol/server",           "transport.socket.keepalive", NULL, NO_DOC, 0},
-        {"server.allow-insecure",                 "protocol/server",          "rpc-auth-allow-insecure", NULL, NO_DOC, 0},
-
-        {"performance.write-behind",             "performance/write-behind",  "!perf", "on", NO_DOC, 0},
-        {"performance.read-ahead",               "performance/read-ahead",    "!perf", "on", NO_DOC, 0},
-        {"performance.io-cache",                 "performance/io-cache",      "!perf", "on", NO_DOC, 0},
-        {"performance.quick-read",               "performance/quick-read",    "!perf", "on", NO_DOC, 0},
-        {VKEY_PERF_STAT_PREFETCH,                "performance/stat-prefetch", "!perf", "on", NO_DOC, 0},
-        {"performance.client-io-threads",        "performance/io-threads",    "!perf", "off", NO_DOC, 0},
-        {VKEY_MARKER_XTIME,                      "features/marker",           "xtime", "off", NO_DOC, OPT_FLAG_FORCE},
-        {VKEY_MARKER_XTIME,                      "features/marker",           "!xtime", "off", NO_DOC, OPT_FLAG_FORCE},
-
-        {"nfs.enable-ino32",                     "nfs/server",                "nfs.enable-ino32", NULL, GLOBAL_DOC, 0},
-        {"nfs.mem-factor",                       "nfs/server",                "nfs.mem-factor", NULL, GLOBAL_DOC, 0},
-        {"nfs.export-dirs",                      "nfs/server",                "nfs3.export-dirs", NULL, GLOBAL_DOC, 0},
-        {"nfs.export-volumes",                   "nfs/server",                "nfs3.export-volumes", NULL, GLOBAL_DOC, 0},
-        {"nfs.addr-namelookup",                  "nfs/server",                "rpc-auth.addr.namelookup", NULL, GLOBAL_DOC, 0},
-        {"nfs.dynamic-volumes",                  "nfs/server",                "nfs.dynamic-volumes", NULL, GLOBAL_NO_DOC, 0},
-        {"nfs.register-with-portmap",            "nfs/server",                "rpc.register-with-portmap", NULL, GLOBAL_DOC, 0},
-        {"nfs.port",                             "nfs/server",                "nfs.port", NULL, GLOBAL_DOC, 0},
-
-        {"nfs.rpc-auth-unix",                    "nfs/server",                "!rpc-auth.auth-unix.*", NULL, DOC, 0},
-        {"nfs.rpc-auth-null",                    "nfs/server",                "!rpc-auth.auth.null.*", NULL, DOC, 0},
-        {"nfs.rpc-auth-allow",                   "nfs/server",                "!rpc-auth.addr.*.allow", NULL, DOC, 0},
-        {"nfs.rpc-auth-reject",                  "nfs/server",                "!rpc-auth.addr.*.reject", NULL, DOC, 0},
-        {"nfs.ports-insecure",                   "nfs/server",                "!rpc-auth.ports.*.insecure", NULL, DOC, 0},
-        {"nfs.transport-type",                   "nfs/server",                "!nfs.transport-type", NULL, DOC, 0},
-
-        {"nfs.trusted-sync",                     "nfs/server",                "!nfs3.*.trusted-sync", NULL, DOC, 0},
-        {"nfs.trusted-write",                    "nfs/server",                "!nfs3.*.trusted-write", NULL, DOC, 0},
-        {"nfs.volume-access",                    "nfs/server",                "!nfs3.*.volume-access", NULL, DOC, 0},
-        {"nfs.export-dir",                       "nfs/server",                "!nfs3.*.export-dir", NULL, DOC, 0},
-        {"nfs.disable",                          "nfs/server",                "!nfs-disable", NULL, DOC, 0},
-
-        {VKEY_FEATURES_QUOTA,                    "features/marker",           "quota", "off", NO_DOC, OPT_FLAG_FORCE},
-        {VKEY_FEATURES_LIMIT_USAGE,              "features/quota",            "limit-set", NULL, NO_DOC, 0},
-        {"features.quota-timeout",               "features/quota",            "timeout", "0", DOC, 0},
-        {"server.statedump-path",                "protocol/server",           "statedump-path", NULL, NO_DOC, 0},
-        {NULL,                                                                }
-};
-
-
+extern struct volopt_map_entry glusterd_volopt_map[];
 
 /*********************************************
  *
@@ -254,13 +89,14 @@ xlator_instantiate_va (const char *type, const char *format, va_list arg)
         xl->name = volname;
         INIT_LIST_HEAD (&xl->volume_options);
 
+        xl->ctx = THIS->ctx;
+
         return xl;
 
  error:
         gf_log ("", GF_LOG_ERROR, "creating xlator of type %s failed",
                 type);
-        if (volname)
-                GF_FREE (volname);
+        GF_FREE (volname);
         if (xl)
                 xlator_destroy (xl);
 
@@ -583,8 +419,7 @@ volopt_trie (char *key, char **hint)
         }
 
  out:
-        if (patt[0])
-                GF_FREE (patt[0]);
+        GF_FREE (patt[0]);
         if (ret)
                 *hint = NULL;
 
@@ -616,26 +451,21 @@ struct opthandler_data {
         void *param;
 };
 
-#define pattern_match_options 0
-
-
-static void
-process_option (dict_t *dict, char *key, data_t *value, void *param)
+static int
+process_option (char *key, data_t *value, void *param)
 {
         struct opthandler_data *odt = param;
         struct volopt_map_entry vme = {0,};
 
         if (odt->rv)
-                return;
-#if pattern_match_options
-        if (fnmatch (odt->vme->key, key, 0) != 0)
-                return;
-#endif
+                return 0;
         odt->found = _gf_true;
 
         vme.key = key;
         vme.voltype = odt->vme->voltype;
         vme.option = odt->vme->option;
+        vme.op_version = odt->vme->op_version;
+
         if (!vme.option) {
                 vme.option = strrchr (key, '.');
                 if (vme.option)
@@ -649,6 +479,7 @@ process_option (dict_t *dict, char *key, data_t *value, void *param)
                 vme.value = value->data;
 
         odt->rv = odt->handler (odt->graph, &vme, odt->param);
+        return 0;
 }
 
 static int
@@ -669,14 +500,10 @@ volgen_graph_set_options_generic (volgen_graph_t *graph, dict_t *dict,
                 odt.found = _gf_false;
                 odt.data_t_fake = _gf_false;
 
-#if pattern_match_options
-                dict_foreach (dict, process_option, &odt);
-#else
                 data = dict_get (dict, vme->key);
 
                 if (data)
-                        process_option (dict, vme->key, data, &odt);
-#endif
+                        process_option (vme->key, data, &odt);
                 if (odt.rv)
                         return odt.rv;
 
@@ -690,8 +517,7 @@ volgen_graph_set_options_generic (volgen_graph_t *graph, dict_t *dict,
                          * in this context
                          */
                         odt.data_t_fake = _gf_true;
-                        process_option (NULL, vme->key, (data_t *)vme->value,
-                                        &odt);
+                        process_option (vme->key, (data_t *)vme->value, &odt);
                         if (odt.rv)
                                 return odt.rv;
                 }
@@ -751,6 +577,108 @@ optget_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
         return 0;
 }
 
+static glusterd_server_xlator_t
+get_server_xlator (char *xlator)
+{
+        glusterd_server_xlator_t subvol = GF_XLATOR_NONE;
+
+        if (strcmp (xlator, "posix") == 0)
+                subvol = GF_XLATOR_POSIX;
+        if (strcmp (xlator, "acl") == 0)
+                subvol = GF_XLATOR_ACL;
+        if (strcmp (xlator, "locks") == 0)
+                subvol = GF_XLATOR_LOCKS;
+        if (strcmp (xlator, "io-threads") == 0)
+                subvol = GF_XLATOR_IOT;
+        if (strcmp (xlator, "index") == 0)
+                subvol = GF_XLATOR_INDEX;
+        if (strcmp (xlator, "marker") == 0)
+                subvol = GF_XLATOR_MARKER;
+        if (strcmp (xlator, "io-stats") == 0)
+                subvol = GF_XLATOR_IO_STATS;
+        if (strcmp (xlator, "bd") == 0)
+                subvol = GF_XLATOR_BD;
+
+        return subvol;
+}
+
+static glusterd_client_xlator_t
+get_client_xlator (char *xlator)
+{
+        glusterd_client_xlator_t subvol = GF_CLNT_XLATOR_NONE;
+
+        if (strcmp (xlator, "client") == 0)
+                subvol = GF_CLNT_XLATOR_FUSE;
+
+        return subvol;
+}
+
+static int
+debugxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                       void *param)
+{
+        char *volname = NULL;
+        gf_boolean_t enabled = _gf_false;
+
+        volname = param;
+
+        if (strcmp (vme->option, "!debug") != 0)
+                return 0;
+
+        if (!strcmp (vme->key , "debug.trace") ||
+            !strcmp (vme->key, "debug.error-gen")) {
+                if (get_server_xlator (vme->value) == GF_XLATOR_NONE &&
+                    get_client_xlator (vme->value) == GF_CLNT_XLATOR_NONE)
+                        return 0;
+                else
+                        goto add_graph;
+        }
+
+        if (gf_string2boolean (vme->value, &enabled) == -1)
+                return -1;
+        if (!enabled)
+                return 0;
+
+add_graph:
+        if (volgen_graph_add (graph, vme->voltype, volname))
+                return 0;
+        else
+                return -1;
+}
+
+int
+check_and_add_debug_xl (volgen_graph_t *graph, dict_t *set_dict, char *volname,
+                        char *xlname)
+{
+        int       ret       = 0;
+        char     *value_str = NULL;
+
+        ret = dict_get_str (set_dict, "debug.trace", &value_str);
+        if (!ret) {
+                if (strcmp (xlname, value_str) == 0) {
+                        ret = volgen_graph_set_options_generic (graph, set_dict, volname,
+                                                                &debugxl_option_handler);
+                        if (ret)
+                                goto out;
+                }
+        }
+
+        ret = dict_get_str (set_dict, "debug.error-gen", &value_str);
+        if (!ret) {
+                if (strcmp (xlname, value_str) == 0) {
+                        ret = volgen_graph_set_options_generic (graph, set_dict, volname,
+                                                                &debugxl_option_handler);
+                        if (ret)
+                                goto out;
+                }
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
 /* This getter considers defaults also. */
 static int
 volgen_dict_get (dict_t *dict, char *key, char **value)
@@ -919,20 +847,20 @@ glusterd_check_voloption (char *key)
 int
 glusterd_check_option_exists (char *key, char **completion)
 {
-        dict_t *dict = NULL;
         struct volopt_map_entry vme = {0,};
         struct volopt_map_entry *vmep = NULL;
         int ret = 0;
+        xlator_t *this = THIS;
 
         (void)vme;
         (void)vmep;
-        (void)dict;
 
         if (!strchr (key, '.')) {
                 if (completion) {
                         ret = option_complete (key, completion);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Out of memory");
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Out of memory");
                                 return -1;
                         }
 
@@ -945,42 +873,12 @@ glusterd_check_option_exists (char *key, char **completion)
                         return 0;
         }
 
-#if !pattern_match_options
         for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
                 if (strcmp (vmep->key, key) == 0) {
                         ret = 1;
                         break;
                 }
         }
-#else
-        vme.key = key;
-
-        /* We are getting a bit anal here to avoid typing
-         * fnmatch one more time. Orthogonality foremost!
-         * The internal logic of looking up in the volopt_map table
-         * should be coded exactly once.
-         *
-         * [[Ha-ha-ha, so now if I ever change the internals then I'll
-         * have to update the fnmatch in this comment also :P ]]
-         */
-        dict = get_new_dict ();
-        if (!dict || dict_set_str (dict, key, "")) {
-                gf_log ("", GF_LOG_ERROR, "Out of memory");
-
-                return -1;
-        }
-
-        ret = volgen_graph_set_options_generic (NULL, dict, &vme,
-                                                &optget_option_handler);
-        dict_destroy (dict);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Out of memory");
-
-                return -1;
-        }
-
-        ret = !!vme.value;
-#endif
 
         if (ret || !completion)
                 return ret;
@@ -988,7 +886,7 @@ glusterd_check_option_exists (char *key, char **completion)
  trie:
         ret = volopt_trie (key, completion);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "Some error occurred during keyword hinting");
         }
 
@@ -1117,8 +1015,12 @@ free_fp:
 static int
 volgen_write_volfile (volgen_graph_t *graph, char *filename)
 {
-        char *ftmp = NULL;
-        FILE *f = NULL;
+        char        *ftmp = NULL;
+        FILE        *f = NULL;
+        int          fd   = 0;
+        xlator_t    *this = NULL;
+
+        this = THIS;
 
         if (gf_asprintf (&ftmp, "%s.tmp", filename) == -1) {
                 ftmp = NULL;
@@ -1126,15 +1028,35 @@ volgen_write_volfile (volgen_graph_t *graph, char *filename)
                 goto error;
         }
 
+        fd = creat (ftmp, S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        strerror (errno));
+                goto error;
+        }
+
+        close (fd);
+
         f = fopen (ftmp, "w");
         if (!f)
                 goto error;
 
         if (glusterfs_graph_print_file (f, &graph->graph) == -1)
-	goto error;
+                goto error;
 
-        if (fclose (f) == -1)
+        if (fclose (f) != 0) {
+                gf_log (THIS->name, GF_LOG_ERROR, "fclose on the file %s "
+                        "failed (%s)", ftmp, strerror (errno));
+                /*
+                 * Even though fclose has failed here, we have to set f to NULL.
+                 * Otherwise when the code path goes to error, there again we
+                 * try to close it which might cause undefined behavior such as
+                 * process crash.
+                 */
+                f = NULL;
                 goto error;
+        }
+
         f = NULL;
 
         if (rename (ftmp, filename) == -1)
@@ -1148,12 +1070,12 @@ volgen_write_volfile (volgen_graph_t *graph, char *filename)
 
  error:
 
-        if (ftmp)
-                GF_FREE (ftmp);
+        GF_FREE (ftmp);
         if (f)
                 fclose (f);
 
-        gf_log ("", GF_LOG_ERROR, "failed to create volfile %s", filename);
+        gf_log (this->name, GF_LOG_ERROR,
+                "failed to create volfile %s", filename);
 
         return -1;
 }
@@ -1484,21 +1406,28 @@ static int
 server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                       dict_t *set_dict, void *param)
 {
-        char     *volname               = NULL;
-        char     *path                  = NULL;
-        int       pump                  = 0;
-        xlator_t *xl                    = NULL;
-        xlator_t *txl                   = NULL;
-        xlator_t *rbxl                  = NULL;
-        char      transt[16]            = {0,};
-        char     *ptranst               = NULL;
-        char      volume_id[64]         = {0,};
-        char      tstamp_file[PATH_MAX] = {0,};
-        int       ret                   = 0;
-        char     *xlator                = NULL;
-        char     *loglevel              = NULL;
-
-        path = param;
+        char     *volname                 = NULL;
+        char     *path                    = NULL;
+        int       pump                    = 0;
+        xlator_t *xl                      = NULL;
+        xlator_t *txl                     = NULL;
+        xlator_t *rbxl                    = NULL;
+        char      transt[16]              = {0,};
+        char     *ptranst                 = NULL;
+        char      volume_id[64]           = {0,};
+        char      tstamp_file[PATH_MAX]   = {0,};
+        int       ret                     = 0;
+        char     *xlator                  = NULL;
+        char     *loglevel                = NULL;
+        char     *username                = NULL;
+        char     *password                = NULL;
+        char     index_basepath[PATH_MAX] = {0};
+        char     key[1024]                = {0};
+        glusterd_brickinfo_t *brickinfo   = NULL;
+        char changelog_basepath[PATH_MAX] = {0,};
+
+        brickinfo = param;
+        path      = brickinfo->path;
         volname = volinfo->volname;
         get_vol_transport_type (volinfo, transt);
 
@@ -1527,23 +1456,82 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
                 return -1;
 
+        ret = check_and_add_debug_xl (graph, set_dict, volname,
+                                      "posix");
+        if (ret)
+                return -1;
+#ifdef HAVE_BD_XLATOR
+        if (*brickinfo->vg != '\0') {
+                /* Now add BD v2 xlator if volume is BD type */
+                xl = volgen_graph_add (graph, "storage/bd", volname);
+                if (!xl)
+                        return -1;
+
+                ret = xlator_set_option (xl, "device", "vg");
+                if (ret)
+                        return -1;
+                ret = xlator_set_option (xl, "export", brickinfo->vg);
+                if (ret)
+                        return -1;
+
+                ret = check_and_add_debug_xl (graph, set_dict, volname, "bd");
+                if (ret)
+                        return -1;
+
+        }
+#endif
+
+        xl = volgen_graph_add (graph, "features/changelog", volname);
+        if (!xl)
+                return -1;
+
+        ret = xlator_set_option (xl, "changelog-brick", path);
+        if (ret)
+                return -1;
+
+        snprintf (changelog_basepath, sizeof (changelog_basepath),
+                  "%s/%s", path, ".glusterfs/changelogs");
+        ret = xlator_set_option (xl, "changelog-dir", changelog_basepath);
+        if (ret)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "changelog");
+        if (ret)
+                return -1;
+
         xl = volgen_graph_add (graph, "features/access-control", volname);
         if (!xl)
                 return -1;
 
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "acl");
+        if (ret)
+                return -1;
+
         xl = volgen_graph_add (graph, "features/locks", volname);
         if (!xl)
                 return -1;
 
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "locks");
+        if (ret)
+                return -1;
+
         xl = volgen_graph_add (graph, "performance/io-threads", volname);
         if (!xl)
                 return -1;
 
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "io-threads");
+        if (ret)
+                return -1;
+
         ret = dict_get_int32 (volinfo->dict, "enable-pump", &pump);
         if (ret == -ENOENT)
                 ret = pump = 0;
         if (ret)
                 return -1;
+
+        username = glusterd_auth_get_username (volinfo);
+        password = glusterd_auth_get_password (volinfo);
+
         if (pump) {
                 txl = first_of (graph);
 
@@ -1556,6 +1544,18 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
 		if (NULL == ptranst)
 			return -1;
 
+                if (username) {
+                        ret = xlator_set_option (rbxl, "username", username);
+                        if (ret)
+                                return -1;
+                }
+
+                if (password) {
+                        ret = xlator_set_option (rbxl, "password", password);
+                        if (ret)
+                                return -1;
+                }
+
                 ret = xlator_set_option (rbxl, "transport-type", ptranst);
                 GF_FREE (ptranst);
                 if (ret)
@@ -1573,9 +1573,25 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                         return -1;
         }
 
+        xl = volgen_graph_add (graph, "features/index", volname);
+        if (!xl)
+                return -1;
+
+        snprintf (index_basepath, sizeof (index_basepath), "%s/%s",
+                  path, ".glusterfs/indices");
+        ret = xlator_set_option (xl, "index-base", index_basepath);
+        if (ret)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname,
+                                      "index");
+        if (ret)
+                return -1;
+
         xl = volgen_graph_add (graph, "features/marker", volname);
         if (!xl)
                 return -1;
+
         uuid_unparse (volinfo->volume_id, volume_id);
         ret = xlator_set_option (xl, "volume-uuid", volume_id);
         if (ret)
@@ -1585,6 +1601,49 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
                 return -1;
 
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "marker");
+        if (ret)
+                return -1;
+
+        if (dict_get_str_boolean (set_dict, "features.read-only", 0) &&
+            dict_get_str_boolean (set_dict, "features.worm",0)) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "read-only and worm cannot be set together");
+                ret = -1;
+                goto out;
+        }
+
+        /* Check for read-only volume option, and add it to the graph */
+        if (dict_get_str_boolean (set_dict, "features.read-only", 0)
+                || volinfo -> is_snap_volume) {
+                xl = volgen_graph_add (graph, "features/read-only", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Check for worm volume option, and add it to the graph */
+        if (dict_get_str_boolean (set_dict, "features.worm", 0)) {
+                xl = volgen_graph_add (graph, "features/worm", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Check for compress volume option, and add it to the graph on server side */
+        if (dict_get_str_boolean (set_dict, "features.compress", 0)) {
+                xl = volgen_graph_add (graph, "features/cdc", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (set_dict, "compress.mode", "server");
+                if (ret)
+                        goto out;
+        }
+
         xl = volgen_graph_add_as (graph, "debug/io-stats", path);
         if (!xl)
                 return -1;
@@ -1596,6 +1655,35 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
                 return -1;
 
+        /*In the case of running multiple glusterds on a single machine,
+         * we should ensure that bricks don't listen on all IPs on that
+         * machine and break the IP based separation being brought about.*/
+        if (dict_get (THIS->options, "transport.socket.bind-address")) {
+                ret = xlator_set_option (xl, "transport.socket.bind-address",
+                                         brickinfo->hostname);
+                if (ret)
+                        return -1;
+        }
+
+        if (username) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "auth.login.%s.allow", path);
+
+                ret = xlator_set_option (xl, key, username);
+                if (ret)
+                        return -1;
+        }
+
+        if (password) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "auth.login.%s.password",
+                          username);
+
+                ret = xlator_set_option (xl, key, password);
+                if (ret)
+                        return -1;
+        }
+
         ret = volgen_graph_set_options_generic (graph, set_dict,
                                                 (xlator && loglevel) ? (void *)set_dict : volinfo,
                                                 (xlator && loglevel) ?  &server_spec_extended_option_handler :
@@ -1609,9 +1697,9 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
 /* builds a graph for server role , with option overrides in mod_dict */
 static int
 build_server_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                    dict_t *mod_dict, char *path)
+                    dict_t *mod_dict, glusterd_brickinfo_t *brickinfo)
 {
-        return build_graph_generic (graph, volinfo, mod_dict, path,
+        return build_graph_generic (graph, volinfo, mod_dict, brickinfo,
                                     &server_graph_builder);
 }
 
@@ -1619,12 +1707,42 @@ static int
 perfxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
                        void *param)
 {
+        gf_boolean_t enabled = _gf_false;
+        glusterd_volinfo_t *volinfo = NULL;
+
+        GF_ASSERT (param);
+        volinfo = param;
+
+        if (strcmp (vme->option, "!perf") != 0)
+                return 0;
+
+        if (gf_string2boolean (vme->value, &enabled) == -1)
+                return -1;
+        if (!enabled)
+                return 0;
+
+        /* Check op-version before adding the 'open-behind' xlator in the graph
+         */
+        if (!strcmp (vme->key, "performance.open-behind") &&
+            (vme->op_version > volinfo->client_op_version))
+                return 0;
+
+        if (volgen_graph_add (graph, vme->voltype, volinfo->volname))
+                return 0;
+        else
+                return -1;
+}
+
+static int
+nfsperfxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                       void *param)
+{
         char *volname = NULL;
         gf_boolean_t enabled = _gf_false;
 
         volname = param;
 
-        if (strcmp (vme->option, "!perf") != 0)
+        if (strcmp (vme->option, "!nfsperf") != 0)
                 return 0;
 
         if (gf_string2boolean (vme->value, &enabled) == -1)
@@ -1695,8 +1813,7 @@ init_sethelp_xml_doc (xmlTextWriterPtr *writer, xmlBufferPtr  *buf)
                 goto out;
         }
 
-        ret = xmlTextWriterStartElement(*writer,
-                                        (xmlChar *)"volumeOptionsDefaults");
+        ret = xmlTextWriterStartElement(*writer, (xmlChar *)"options");
         if (ret < 0) {
                 gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
                         "xmlElemetnt");
@@ -1722,7 +1839,7 @@ xml_add_volset_element (xmlTextWriterPtr writer, const char *name,
 
         GF_ASSERT (name);
 
-        ret = xmlTextWriterStartElement(writer, (xmlChar *) "volumeOption");
+        ret = xmlTextWriterStartElement(writer, (xmlChar *) "option");
         if (ret < 0) {
                 gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
                         "xmlElemetnt");
@@ -1775,7 +1892,7 @@ xml_add_volset_element (xmlTextWriterPtr writer, const char *name,
 #endif
 
 static int
-get_key_from_volopt ( struct volopt_map_entry *vme, char **key)
+_get_xlator_opt_key_from_vme ( struct volopt_map_entry *vme, char **key)
 {
         int ret = 0;
 
@@ -1783,25 +1900,32 @@ get_key_from_volopt ( struct volopt_map_entry *vme, char **key)
         GF_ASSERT (key);
 
 
-        if (vme->option) {
-                if  (vme->option[0] == '!') {
-                        *key = vme->option + 1;
-                        if (!*key[0])
-                                ret = -1;
+        if (!strcmp (vme->key, AUTH_ALLOW_MAP_KEY))
+                *key = gf_strdup (AUTH_ALLOW_OPT_KEY);
+        else if (!strcmp (vme->key, AUTH_REJECT_MAP_KEY))
+                *key = gf_strdup (AUTH_REJECT_OPT_KEY);
+        else if (!strcmp (vme->key, NFS_DISABLE_MAP_KEY))
+                *key = gf_strdup (NFS_DISABLE_OPT_KEY);
+        else {
+                if (vme->option) {
+                        if  (vme->option[0] == '!') {
+                                *key = vme->option + 1;
+                                if (!*key[0])
+                                        ret = -1;
+                        } else {
+                                *key = vme->option;
+                        }
                 } else {
-                        *key = vme->option;
-                }
-        } else  {
-                *key = strchr (vme->key, '.');
-                if (*key) {
-                        (*key) ++;
-                        if (!*key[0])
+                        *key = strchr (vme->key, '.');
+                        if (*key) {
+                                (*key) ++;
+                                if (!*key[0])
+                                        ret = -1;
+                        } else {
                                 ret = -1;
-                } else {
-                        ret = -1;
+                        }
                 }
         }
-
         if (ret)
                 gf_log ("glusterd", GF_LOG_ERROR, "Wrong entry found in  "
                         "glusterd_volopt_map entry %s", vme->key);
@@ -1811,23 +1935,32 @@ get_key_from_volopt ( struct volopt_map_entry *vme, char **key)
         return ret;
 }
 
+static void
+_free_xlator_opt_key (char *key)
+{
+        GF_ASSERT (key);
+
+        if (!strcmp (key, AUTH_ALLOW_OPT_KEY) ||
+            !strcmp (key, AUTH_REJECT_OPT_KEY) ||
+            !strcmp (key, NFS_DISABLE_OPT_KEY))
+                GF_FREE (key);
+
+        return;
+}
 
 int
-glusterd_get_volopt_content (gf_boolean_t xml_out)
+glusterd_get_volopt_content (dict_t * ctx, gf_boolean_t xml_out)
 {
-
-        char                    *xlator_type = NULL;
         void                    *dl_handle = NULL;
-        volume_opt_list_t          vol_opt_handle = {{0},};
+        volume_opt_list_t        vol_opt_handle = {{0},};
         char                    *key = NULL;
         struct volopt_map_entry *vme = NULL;
         int                      ret = -1;
         char                    *def_val = NULL;
         char                    *descr = NULL;
-        char                     output_string[8192] = {0, };
+        char                     output_string[25600] = {0, };
         char                    *output = NULL;
-        char                     tmp_str[1024] = {0, };
-        dict_t                  *ctx = NULL;
+        char                     tmp_str[2048] = {0, };
 #if (HAVE_LIB_XML)
         xmlTextWriterPtr         writer = NULL;
         xmlBufferPtr             buf = NULL;
@@ -1839,51 +1972,73 @@ glusterd_get_volopt_content (gf_boolean_t xml_out)
         }
 #endif
 
-        ctx = glusterd_op_get_ctx ();
-
-        if (!ctx) {
-                /*extract the vol-set-help output only in host glusterd*/
-                ret = 0;
-                goto out;
-        }
-
         INIT_LIST_HEAD (&vol_opt_handle.list);
 
         for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
 
-                if ( ( vme->type == NO_DOC) || (vme->type == GLOBAL_NO_DOC) )
+                if ((vme->type == NO_DOC) || (vme->type == GLOBAL_NO_DOC))
                         continue;
 
-                if (get_key_from_volopt (vme, &key))
-                                goto out; /*Some error while getin key*/
+                if (vme->description) {
+                        descr = vme->description;
+                        def_val = vme->value;
+                } else {
+                        if (_get_xlator_opt_key_from_vme (vme, &key)) {
+                                gf_log ("glusterd", GF_LOG_DEBUG, "Failed to "
+                                        "get %s key from volume option entry",
+                                        vme->key);
+                                goto out; /*Some error while geting key*/
+                        }
 
-                if (!xlator_type || strcmp (vme->voltype, xlator_type)){
                         ret = xlator_volopt_dynload (vme->voltype,
-                                                        &dl_handle,
-                                                        &vol_opt_handle);
-                        if (ret)
-                                continue;
-                }
+                                                     &dl_handle,
+                                                     &vol_opt_handle);
 
-                ret = xlator_option_info_list (&vol_opt_handle, key,
-                                               &def_val, &descr);
-                if (ret) /*Swallow Error i.e if option not found*/
-                        continue;
+                        if (ret) {
+                                gf_log ("glusterd", GF_LOG_DEBUG,
+                                        "xlator_volopt_dynload error(%d)", ret);
+                                ret = 0;
+                                goto cont;
+                        }
+
+                        ret = xlator_option_info_list (&vol_opt_handle, key,
+                                                       &def_val, &descr);
+                        if (ret) { /*Swallow Error i.e if option not found*/
+                                gf_log ("glusterd", GF_LOG_DEBUG,
+                                        "Failed to get option for %s key", key);
+                                ret = 0;
+                                goto cont;
+                        }
+                }
 
                 if (xml_out) {
 #if (HAVE_LIB_XML)
                         if (xml_add_volset_element (writer,vme->key,
-                                                        def_val, descr))
-                                goto out;
+                                                    def_val, descr)) {
+                                ret = -1;
+                                goto cont;
+                        }
 #else
                         gf_log ("glusterd", GF_LOG_ERROR, "Libxml not present");
 #endif
                 } else {
-                        snprintf (tmp_str, 1024, "Option: %s\nDefault "
+                        snprintf (tmp_str, sizeof (tmp_str), "Option: %s\nDefault "
                                         "Value: %s\nDescription: %s\n\n",
                                         vme->key, def_val, descr);
                         strcat (output_string, tmp_str);
                 }
+cont:
+                if (dl_handle) {
+                        dlclose (dl_handle);
+                        dl_handle = NULL;
+                        vol_opt_handle.given_opt = NULL;
+                }
+                if (key) {
+                        _free_xlator_opt_key (key);
+                        key = NULL;
+                }
+                if (ret)
+                        goto out;
         }
 
 #if (HAVE_LIB_XML)
@@ -1910,7 +2065,7 @@ glusterd_get_volopt_content (gf_boolean_t xml_out)
         }
 
         ret = dict_set_dynstr (ctx, "help-str", output);
- out:
+out:
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 
@@ -1922,10 +2077,14 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
 {
         int                      i                  = 0;
         int                      ret                = -1;
+        uint32_t                 client_type        = GF_CLIENT_OTHER;
         char                     transt[16]         = {0,};
         char                    *volname            = NULL;
-        glusterd_brickinfo_t    *brick = NULL;
-        xlator_t                *xl                = NULL;
+        char                    *str                = NULL;
+        glusterd_brickinfo_t    *brick              = NULL;
+        xlator_t                *xl                 = NULL;
+        char                    *ssl_str            = NULL;
+        gf_boolean_t             ssl_bool;
 
         volname = volinfo->volname;
 
@@ -1968,8 +2127,45 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 ret = xlator_set_option (xl, "transport-type", transt);
                 if (ret)
                         goto out;
+
+                ret = dict_get_uint32 (set_dict, "trusted-client",
+                                       &client_type);
+
+                if (!ret && client_type == GF_CLIENT_TRUSTED) {
+                        str = NULL;
+                        str = glusterd_auth_get_username (volinfo);
+                        if (str) {
+                                ret = xlator_set_option (xl, "username",
+                                                         str);
+                                if (ret)
+                                        goto out;
+                        }
+
+                        str = glusterd_auth_get_password (volinfo);
+                        if (str) {
+                                ret = xlator_set_option (xl, "password",
+                                                         str);
+                                if (ret)
+                                        goto out;
+                        }
+                }
+
+                if (dict_get_str(set_dict,"client.ssl",&ssl_str) == 0) {
+                        if (gf_string2boolean(ssl_str,&ssl_bool) == 0) {
+                                if (ssl_bool) {
+                                        ret = xlator_set_option(xl,
+                                                "transport.socket.ssl-enabled",
+                                                "true");
+                                        if (ret) {
+                                                goto out;
+                                        }
+                                }
+                        }
+                }
+
                 i++;
         }
+
         if (i != volinfo->brick_count) {
                 gf_log ("", GF_LOG_ERROR,
                         "volume inconsistency: actual number of bricks (%d) "
@@ -2020,6 +2216,7 @@ volgen_graph_build_clusters (volgen_graph_t *graph,
 
                 if (trav == txl)
                         break;
+
                 i++;
         }
 
@@ -2130,13 +2327,33 @@ volgen_graph_build_dht_cluster (volgen_graph_t *graph,
         int                     ret                      = -1;
         char                    *decommissioned_children = NULL;
         xlator_t                *dht                     = NULL;
+        char                    *voltype                 = "cluster/distribute";
+
+        /* NUFA and Switch section */
+        if (dict_get_str_boolean (volinfo->dict, "cluster.nufa", 0) &&
+            dict_get_str_boolean (volinfo->dict, "cluster.switch", 0)) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "nufa and switch cannot be set together");
+                ret = -1;
+                goto out;
+        }
+
+        /* Check for NUFA volume option, and change the voltype */
+        if (dict_get_str_boolean (volinfo->dict, "cluster.nufa", 0))
+                voltype = "cluster/nufa";
+
+        /* Check for switch volume option, and change the voltype */
+        if (dict_get_str_boolean (volinfo->dict, "cluster.switch", 0))
+                voltype = "cluster/switch";
 
-        GF_ASSERT (child_count > 1);
         clusters = volgen_graph_build_clusters (graph,  volinfo,
-                                                "cluster/distribute", "%s-dht",
-                                                child_count, child_count);
+                                                voltype,
+                                                "%s-dht",
+                                                child_count,
+                                                child_count);
         if (clusters < 0)
                 goto out;
+
         dht = first_of (graph);
         ret = _graph_get_decommissioned_children (dht, volinfo,
                                                   &decommissioned_children);
@@ -2150,8 +2367,7 @@ volgen_graph_build_dht_cluster (volgen_graph_t *graph,
         }
         ret = 0;
 out:
-        if (decommissioned_children)
-                GF_FREE (decommissioned_children);
+        GF_FREE (decommissioned_children);
         return ret;
 }
 
@@ -2224,24 +2440,51 @@ volume_volgen_graph_build_clusters (volgen_graph_t *graph,
 
 build_distribute:
         dist_count = volinfo->brick_count / volinfo->dist_leaf_count;
-        if (dist_count > 1) {
-                ret = volgen_graph_build_dht_cluster (graph, volinfo,
-                                                      dist_count);
-                if (ret)
-                        goto out;
+        if (!dist_count) {
+                ret = -1;
+                goto out;
         }
+
+        ret = volgen_graph_build_dht_cluster (graph, volinfo,
+                                              dist_count);
+        if (ret == -1)
+                goto out;
+
         ret = 0;
 out:
         return ret;
 }
 
+static int client_graph_set_perf_options(volgen_graph_t *graph,
+					 glusterd_volinfo_t *volinfo,
+					 dict_t *set_dict)
+{
+	data_t *tmp_data = NULL;
+	char *volname = NULL;
+
+	/*
+	 * Logic to make sure NFS doesn't have performance translators by
+	 * default for a volume
+	 */
+	volname = volinfo->volname;
+	tmp_data = dict_get (set_dict, "nfs-volume-file");
+	if (!tmp_data)
+		return volgen_graph_set_options_generic(graph, set_dict,
+							volname,
+							&perfxl_option_handler);
+	else
+		return volgen_graph_set_options_generic(graph, set_dict,
+							volname,
+							&nfsperfxl_option_handler);
+}
+
 static int
 client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                       dict_t *set_dict, void *param)
 {
-        int                      ret                = 0;
-        xlator_t                *xl                 = NULL;
-        char                    *volname            = NULL;
+        int       ret      = 0;
+        xlator_t *xl       = NULL;
+        char     *volname  = NULL;
 
         volname = volinfo->volname;
         ret = volgen_graph_build_clients (graph, volinfo, set_dict, param);
@@ -2249,9 +2492,34 @@ client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 goto out;
 
         ret = volume_volgen_graph_build_clusters (graph, volinfo);
-        if (ret)
+        if (ret == -1)
                 goto out;
 
+        /* Check for compress volume option, and add it to the graph on client side */
+        if (dict_get_str_boolean (set_dict, "features.compress", 0)) {
+                xl = volgen_graph_add (graph, "features/cdc", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (set_dict, "compress.mode", "client");
+                if (ret)
+                        goto out;
+
+        }
+
+        ret = glusterd_volinfo_get_boolean (volinfo, "features.encryption");
+        if (ret == -1)
+                goto out;
+        if (ret) {
+                xl = volgen_graph_add (graph, "encryption/crypt", volname);
+
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
         ret = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
         if (ret == -1)
                 goto out;
@@ -2264,11 +2532,29 @@ client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 }
         }
 
-        ret = volgen_graph_set_options_generic (graph, set_dict, volname,
-                                                &perfxl_option_handler);
+
+        ret = glusterd_volinfo_get_boolean (volinfo, "features.file-snapshot");
+        if (ret == -1)
+                goto out;
+        if (ret) {
+                xl = volgen_graph_add (graph, "features/qemu-block", volname);
+
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+	ret = client_graph_set_perf_options(graph, volinfo, set_dict);
         if (ret)
                 goto out;
 
+        /* add debug translators depending on the options */
+        ret = check_and_add_debug_xl (graph, set_dict, volname,
+                                      "client");
+        if (ret)
+                return -1;
+
         ret = -1;
         xl = volgen_graph_add_as (graph, "debug/io-stats", volname);
         if (!xl)
@@ -2277,9 +2563,15 @@ client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         ret = volgen_graph_set_options_generic (graph, set_dict, "client",
                                                 &loglevel_option_handler);
 
-        if (!ret)
-                ret = volgen_graph_set_options_generic (graph, set_dict, "client",
-                                                        &sys_loglevel_option_handler);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING, "changing client log level"
+                        " failed");
+
+        ret = volgen_graph_set_options_generic (graph, set_dict, "client",
+                                                &sys_loglevel_option_handler);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING, "changing client syslog "
+                        "level failed");
 out:
         return ret;
 }
@@ -2294,20 +2586,35 @@ build_client_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                                     &client_graph_builder);
 }
 
+char *gd_shd_options[] = {
+        "!self-heal-daemon",
+        "!heal-timeout",
+        NULL
+};
+
+char*
+gd_get_matching_option (char **options, char *option)
+{
+        while (*options && strcmp (*options, option))
+                options++;
+        return *options;
+}
+
 static int
 shd_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
-                           void *param)
+                    void *param)
 {
         int                     ret = 0;
         struct volopt_map_entry new_vme = {0};
-        int                     shd = 0;
+        char                    *shd_option = NULL;
 
-        shd = !strcmp (vme->option, "!self-heal-daemon");
-        if ((vme->option[0] == '!') && !shd)
+        shd_option = gd_get_matching_option (gd_shd_options, vme->option);
+        if ((vme->option[0] == '!') && !shd_option)
                 goto out;
         new_vme = *vme;
-        if (shd)
-                new_vme.option = "self-heal-daemon";
+        if (shd_option) {
+                new_vme.option = shd_option + 1;//option with out '!'
+        }
 
         ret = no_filter_option_handler (graph, &new_vme, param);
 out:
@@ -2331,7 +2638,7 @@ nfs_option_handler (volgen_graph_t *graph,
 
                 ret = xlator_set_option (xl, vme->key, vme->value);
         }*/
-        if ( !volinfo || !volinfo->volname)
+        if (!volinfo || (volinfo->volname[0] == '\0'))
                 return 0;
 
         if (! strcmp (vme->option, "!rpc-auth.addr.*.allow")) {
@@ -2361,7 +2668,7 @@ nfs_option_handler (volgen_graph_t *graph,
         }
 
         if (! strcmp (vme->option, "!rpc-auth.auth-unix.*")) {
-                ret = gf_asprintf (&aa, "rpc-auth.auth.unix.%s",
+                ret = gf_asprintf (&aa, "rpc-auth.auth-unix.%s",
                                         volinfo->volname);
 
                 if (ret != -1) {
@@ -2372,8 +2679,8 @@ nfs_option_handler (volgen_graph_t *graph,
                 if (ret)
                         return -1;
         }
-        if (! strcmp (vme->option, "!rpc-auth.auth.null.*")) {
-                ret = gf_asprintf (&aa, "rpc-auth.auth.null.%s",
+        if (! strcmp (vme->option, "!rpc-auth.auth-null.*")) {
+                ret = gf_asprintf (&aa, "rpc-auth.auth-null.%s",
                                         volinfo->volname);
 
                 if (ret != -1) {
@@ -2429,6 +2736,10 @@ nfs_option_handler (volgen_graph_t *graph,
                                         volinfo->volname);
 
                 if (ret != -1) {
+                        ret = gf_canonicalize_path (vme->value);
+                        if (ret)
+                                return -1;
+
                         ret = xlator_set_option (xl, aa, vme->value);
                         GF_FREE (aa);
                 }
@@ -2491,6 +2802,23 @@ nfs_option_handler (volgen_graph_t *graph,
 }
 
 static int
+volgen_graph_set_iam_shd (volgen_graph_t *graph)
+{
+        xlator_t        *trav;
+        int             ret = 0;
+
+        for (trav = first_of (graph); trav; trav = trav->next) {
+                if (strcmp (trav->type, "cluster/replicate") != 0)
+                        continue;
+
+                ret = xlator_set_option (trav, "iam-self-heal-daemon", "yes");
+                if (ret)
+                        break;
+        }
+        return ret;
+}
+
+static int
 build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
 {
         volgen_graph_t     cgraph         = {0};
@@ -2501,8 +2829,9 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
         int                ret            = 0;
         gf_boolean_t       valid_config   = _gf_false;
         xlator_t           *iostxl        = NULL;
-        int                rclusters       = 0;
+        int                rclusters      = 0;
         int                replica_count  = 0;
+        gf_boolean_t       graph_check    = _gf_false;
 
         this = THIS;
         priv = this->private;
@@ -2513,6 +2842,7 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 goto out;
         }
 
+        graph_check = dict_get_str_boolean (mod_dict, "graph-check", 0);
         iostxl = volgen_graph_add_as (graph, "debug/io-stats", "glustershd");
         if (!iostxl) {
                 ret = -1;
@@ -2520,7 +2850,8 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
         }
 
         list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (voliter->status != GLUSTERD_STATUS_STARTED)
+                if (!graph_check &&
+                   (voliter->status != GLUSTERD_STATUS_STARTED))
                         continue;
 
                 if (!glusterd_is_volume_replicate (voliter))
@@ -2534,6 +2865,11 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 if (ret)
                         goto out;
 
+                ret = dict_set_uint32 (set_dict, "trusted-client",
+                                       GF_CLIENT_TRUSTED);
+                if (ret)
+                        goto out;
+
                 dict_copy (voliter->dict, set_dict);
                 if (mod_dict)
                         dict_copy (mod_dict, set_dict);
@@ -2559,10 +2895,29 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 if (ret)
                         goto out;
 
+                ret = volgen_graph_set_iam_shd (&cgraph);
+                if (ret)
+                        goto out;
+
                 ret = volgen_graph_merge_sub (graph, &cgraph, rclusters);
                 if (ret)
                         goto out;
 
+                ret = volgen_graph_set_options_generic (graph, set_dict,
+                                                        "client",
+                                                 &loglevel_option_handler);
+
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING, "changing loglevel "
+                                "of self-heal daemon failed");
+
+                ret = volgen_graph_set_options_generic (graph, set_dict,
+                                                        "client",
+                                                 &sys_loglevel_option_handler);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING, "changing syslog "
+                                "level of self-heal daemon failed");
+
                 ret = dict_reset (set_dict);
                 if (ret)
                         goto out;
@@ -2588,6 +2943,8 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
         char               *skey          = NULL;
         int                 ret           = 0;
         char               nfs_xprt[16]   = {0,};
+        char               *volname       = NULL;
+        data_t             *data          = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -2609,6 +2966,14 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
         if (ret)
                 goto out;
 
+        ret = xlator_set_option (nfsxl, "nfs.nlm", "on");
+        if (ret)
+                goto out;
+
+        ret = xlator_set_option (nfsxl, "nfs.drc", "on");
+        if (ret)
+                goto out;
+
         list_for_each_entry (voliter, &priv->volumes, vol_list) {
                 if (voliter->status != GLUSTERD_STATUS_STARTED)
                         continue;
@@ -2646,16 +3011,35 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 else
                         get_transport_type (voliter, voliter->dict, nfs_xprt, _gf_true);
 
-                ret = dict_set_str (set_dict, VKEY_PERF_STAT_PREFETCH, "off");
+                ret = dict_set_str (set_dict, "performance.stat-prefetch", "off");
                 if (ret)
                         goto out;
 
-                ret = dict_set_str (set_dict, "performance.client-io-threads", "off");
+                ret = dict_set_str (set_dict, "performance.client-io-threads",
+                                    "off");
                 if (ret)
                         goto out;
 
                 ret = dict_set_str (set_dict, "client-transport-type",
                                     nfs_xprt);
+                if (ret)
+                        goto out;
+
+                ret = dict_set_uint32 (set_dict, "trusted-client",
+                                       GF_CLIENT_TRUSTED);
+                if (ret)
+                        goto out;
+
+                ret = dict_set_str (set_dict, "nfs-volume-file", "yes");
+                if (ret)
+                        goto out;
+
+                if (mod_dict && (data = dict_get (mod_dict, "volume-name"))) {
+                        volname = data->data;
+                        if (strcmp (volname, voliter->volname) == 0)
+                                dict_copy (mod_dict, set_dict);
+                }
+
                 ret = build_client_graph (&cgraph, voliter, set_dict);
                 if (ret)
                         goto out;
@@ -2738,16 +3122,21 @@ glusterd_is_valid_volfpath (char *volname, char *brick)
         glusterd_brickinfo_t    *brickinfo = NULL;
         glusterd_volinfo_t      *volinfo = NULL;
         int32_t                 ret = 0;
+        xlator_t                *this = NULL;
 
-        ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "brick path validation failed");
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create brickinfo"
+                        " for brick %s", brick );
                 ret = 0;
                 goto out;
         }
         ret = glusterd_volinfo_new (&volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "brick path validation failed");
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create volinfo");
                 ret = 0;
                 goto out;
         }
@@ -2777,7 +3166,7 @@ glusterd_generate_brick_volfile (glusterd_volinfo_t *volinfo,
 
         get_brick_filepath (filename, volinfo, brickinfo);
 
-        ret = build_server_graph (&graph, volinfo, NULL, brickinfo->path);
+        ret = build_server_graph (&graph, volinfo, NULL, brickinfo);
         if (!ret)
                 ret = volgen_write_volfile (&graph, filename);
 
@@ -2786,8 +3175,6 @@ glusterd_generate_brick_volfile (glusterd_volinfo_t *volinfo,
         return ret;
 }
 
-
-
 static void
 get_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo)
 {
@@ -2814,7 +3201,7 @@ generate_brick_volfiles (glusterd_volinfo_t *volinfo)
         get_vol_tstamp_file (tstamp_file, volinfo);
 
         if (ret) {
-                ret = open (tstamp_file, O_WRONLY|O_CREAT|O_EXCL, 0644);
+                ret = open (tstamp_file, O_WRONLY|O_CREAT|O_EXCL, 0600);
                 if (ret == -1 && errno == EEXIST) {
                         gf_log ("", GF_LOG_DEBUG, "timestamp file exist");
                         ret = -2;
@@ -2871,25 +3258,6 @@ generate_single_transport_client_volfile (glusterd_volinfo_t *volinfo,
         return ret;
 }
 
-void
-get_client_filepath (char *filepath, glusterd_volinfo_t *volinfo, gf_transport_type type)
-{
-        char  path[PATH_MAX] = {0,};
-        glusterd_conf_t *priv = NULL;
-
-        priv = THIS->private;
-
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-
-        if ((volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) &&
-            (type == GF_TRANSPORT_RDMA))
-                snprintf (filepath, PATH_MAX, "%s/%s.rdma-fuse.vol",
-                          path, volinfo->volname);
-        else
-                snprintf (filepath, PATH_MAX, "%s/%s-fuse.vol",
-                          path, volinfo->volname);
-}
-
 static void
 enumerate_transport_reqs (gf_transport_type type, char **types)
 {
@@ -2907,8 +3275,9 @@ enumerate_transport_reqs (gf_transport_type type, char **types)
         }
 }
 
-static int
-generate_client_volfiles (glusterd_volinfo_t *volinfo)
+int
+generate_client_volfiles (glusterd_volinfo_t *volinfo,
+                          glusterd_client_type_t client_type)
 {
         char               filepath[PATH_MAX] = {0,};
         int                ret = -1;
@@ -2927,7 +3296,21 @@ generate_client_volfiles (glusterd_volinfo_t *volinfo)
                 if (ret)
                         goto out;
                 type = transport_str_to_type (types[i]);
-                get_client_filepath (filepath, volinfo, type);
+
+                ret = dict_set_uint32 (dict, "trusted-client", client_type);
+                if (ret)
+                        goto out;
+
+                if (client_type == GF_CLIENT_TRUSTED) {
+                        glusterd_get_trusted_client_filepath (filepath,
+                                                              volinfo,
+                                                              type);
+                } else {
+                        glusterd_get_client_filepath (filepath,
+                                                      volinfo,
+                                                      type);
+                }
+
                 ret = generate_single_transport_client_volfile (volinfo,
                                                                 filepath,
                                                                 dict);
@@ -2948,7 +3331,7 @@ glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
 
         ret = glusterd_generate_brick_volfile (volinfo, brickinfo);
         if (!ret)
-                ret = generate_client_volfiles (volinfo);
+                ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
         if (!ret)
                 ret = glusterd_fetchspec_notify (THIS);
 
@@ -2958,23 +3341,33 @@ glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
 int
 glusterd_create_volfiles_and_notify_services (glusterd_volinfo_t *volinfo)
 {
-        int ret = -1;
+        int        ret  = -1;
+        xlator_t  *this = NULL;
+
+        this = THIS;
 
         ret = generate_brick_volfiles (volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "Could not generate volfiles for bricks");
                 goto out;
         }
 
-        ret = generate_client_volfiles (volinfo);
+        ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Could not generate volfile for client");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not generate trusted client volfiles");
                 goto out;
         }
 
-        ret = glusterd_fetchspec_notify (THIS);
+        ret = generate_client_volfiles (volinfo, GF_CLIENT_OTHER);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not generate client volfiles");
+                goto out;
+        }
+
+        ret = glusterd_fetchspec_notify (this);
 
 out:
         return ret;
@@ -3029,6 +3422,14 @@ glusterd_create_shd_volfile ()
         if (ret)
                 goto out;
 
+        ret = dict_set_str (mod_dict, "cluster.metadata-self-heal", "on");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (mod_dict, "cluster.entry-self-heal", "on");
+        if (ret)
+                goto out;
+
         glusterd_get_nodesvc_volfile ("glustershd", conf->workdir,
                                       filepath, sizeof (filepath));
         ret = glusterd_create_global_volfile (build_shd_graph, filepath,
@@ -3040,6 +3441,106 @@ out:
 }
 
 int
+glusterd_check_nfs_topology_identical (gf_boolean_t *identical)
+{
+        char            nfsvol[PATH_MAX]        = {0,};
+        char            tmpnfsvol[PATH_MAX]     = {0,};
+        glusterd_conf_t *conf                   = NULL;
+        xlator_t        *this                   = THIS;
+        int             ret                     = -1;
+        int             tmpclean                = 0;
+        int             tmpfd                   = -1;
+
+        if ((!identical) || (!this) || (!this->private))
+                goto out;
+
+        conf = (glusterd_conf_t *) this->private;
+
+        /* Fetch the original NFS volfile */
+        glusterd_get_nodesvc_volfile ("nfs", conf->workdir,
+                                      nfsvol, sizeof (nfsvol));
+
+        /* Create the temporary NFS volfile */
+        snprintf (tmpnfsvol, sizeof (tmpnfsvol), "/tmp/gnfs-XXXXXX");
+        tmpfd = mkstemp (tmpnfsvol);
+        if (tmpfd < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                                    "Unable to create temp file %s: (%s)",
+                                    tmpnfsvol, strerror (errno));
+                goto out;
+        }
+
+        tmpclean = 1; /* SET the flag to unlink() tmpfile */
+
+        ret = glusterd_create_global_volfile (build_nfs_graph,
+                                              tmpnfsvol, NULL);
+        if (ret)
+                goto out;
+
+        /* Compare the topology of volfiles */
+        ret = glusterd_check_topology_identical (nfsvol, tmpnfsvol,
+                                                 identical);
+out:
+        if (tmpfd >= 0)
+                close (tmpfd);
+        if (tmpclean)
+                unlink (tmpnfsvol);
+        return ret;
+}
+
+int
+glusterd_check_nfs_volfile_identical (gf_boolean_t *identical)
+{
+        char            nfsvol[PATH_MAX]        = {0,};
+        char            tmpnfsvol[PATH_MAX]     = {0,};
+        glusterd_conf_t *conf                   = NULL;
+        xlator_t        *this                   = NULL;
+        int             ret                     = -1;
+        int             need_unlink             = 0;
+        int             tmp_fd                  = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (identical);
+
+        conf = this->private;
+
+        glusterd_get_nodesvc_volfile ("nfs", conf->workdir,
+                                      nfsvol, sizeof (nfsvol));
+
+        snprintf (tmpnfsvol, sizeof (tmpnfsvol), "/tmp/gnfs-XXXXXX");
+
+        tmp_fd = mkstemp (tmpnfsvol);
+        if (tmp_fd < 0) {
+                gf_log ("", GF_LOG_WARNING, "Unable to create temp file %s: "
+                                "(%s)", tmpnfsvol, strerror (errno));
+                goto out;
+        }
+
+        need_unlink = 1;
+
+        ret = glusterd_create_global_volfile (build_nfs_graph,
+                                              tmpnfsvol, NULL);
+        if (ret)
+                goto out;
+
+        ret = glusterd_check_files_identical (nfsvol, tmpnfsvol,
+                                              identical);
+        if (ret)
+                goto out;
+
+out:
+        if (need_unlink)
+                unlink (tmpnfsvol);
+
+        if (tmp_fd >= 0)
+                close (tmp_fd);
+
+        return ret;
+}
+
+int
 glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
                          glusterd_brickinfo_t *brickinfo)
 {
@@ -3058,6 +3559,35 @@ glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
 }
 
 int
+validate_shdopts (glusterd_volinfo_t *volinfo,
+                  dict_t *val_dict,
+                  char **op_errstr)
+{
+        volgen_graph_t graph = {0,};
+        int     ret = -1;
+
+        graph.errstr = op_errstr;
+
+        if (!glusterd_is_volume_replicate (volinfo)) {
+                ret = 0;
+                goto out;
+        }
+        ret = dict_set_str (val_dict, "graph-check", "on");
+        if (ret)
+                goto out;
+        ret = build_shd_graph (&graph, val_dict);
+        if (!ret)
+                ret = graph_reconf_validateopt (&graph.graph, op_errstr);
+
+        volgen_graph_free (&graph);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+out:
+        dict_del (val_dict, "graph-check");
+        return ret;
+}
+
+int
 validate_nfsopts (glusterd_volinfo_t *volinfo,
                     dict_t *val_dict,
                     char **op_errstr)
@@ -3067,6 +3597,9 @@ validate_nfsopts (glusterd_volinfo_t *volinfo,
         char    transport_type[16] = {0,};
         char    *tt                = NULL;
         char    err_str[4096]      = {0,};
+        xlator_t *this             = THIS;
+
+        GF_ASSERT (this);
 
         graph.errstr = op_errstr;
 
@@ -3077,7 +3610,7 @@ validate_nfsopts (glusterd_volinfo_t *volinfo,
                         snprintf (err_str, sizeof (err_str), "Changing nfs "
                                   "transport type is allowed only for volumes "
                                   "of transport type tcp,rdma");
-                        gf_log ("", GF_LOG_ERROR, "%s", err_str);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         *op_errstr = gf_strdup (err_str);
                         ret = -1;
                         goto out;
@@ -3091,6 +3624,12 @@ validate_nfsopts (glusterd_volinfo_t *volinfo,
                 }
         }
 
+        ret = dict_set_str (val_dict, "volume-name", volinfo->volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volume name");
+                goto out;
+        }
+
         ret = build_nfs_graph (&graph, val_dict);
         if (!ret)
                 ret = graph_reconf_validateopt (&graph.graph, op_errstr);
@@ -3098,17 +3637,20 @@ validate_nfsopts (glusterd_volinfo_t *volinfo,
         volgen_graph_free (&graph);
 
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        if (dict_get (val_dict, "volume-name"))
+                dict_del (val_dict, "volume-name");
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+
 int
 validate_clientopts (glusterd_volinfo_t *volinfo,
-                    dict_t *val_dict,
-                    char **op_errstr)
+                     dict_t *val_dict,
+                     char **op_errstr)
 {
         volgen_graph_t graph = {0,};
-        int     ret = -1;
+        int            ret   = -1;
 
         GF_ASSERT (volinfo);
 
@@ -3126,18 +3668,18 @@ validate_clientopts (glusterd_volinfo_t *volinfo,
 
 int
 validate_brickopts (glusterd_volinfo_t *volinfo,
-                    char *brickinfo_path,
+                    glusterd_brickinfo_t *brickinfo,
                     dict_t *val_dict,
                     char **op_errstr)
 {
         volgen_graph_t graph = {0,};
-        int     ret = -1;
+        int            ret   = -1;
 
         GF_ASSERT (volinfo);
 
         graph.errstr = op_errstr;
 
-        ret = build_server_graph (&graph, volinfo, val_dict, brickinfo_path);
+        ret = build_server_graph (&graph, volinfo, val_dict, brickinfo);
         if (!ret)
                 ret = graph_reconf_validateopt (&graph.graph, op_errstr);
 
@@ -3159,28 +3701,30 @@ glusterd_validate_brickreconf (glusterd_volinfo_t *volinfo,
                 gf_log ("", GF_LOG_DEBUG,
                         "Validating %s", brickinfo->hostname);
 
-                ret = validate_brickopts (volinfo, brickinfo->path, val_dict,
+                ret = validate_brickopts (volinfo, brickinfo, val_dict,
                                           op_errstr);
                 if (ret)
                         goto out;
         }
 
         ret = 0;
-out:
 
+out:
         return ret;
 }
 
-static void
+static int
 _check_globalopt (dict_t *this, char *key, data_t *value, void *ret_val)
 {
         int *ret = NULL;
 
         ret = ret_val;
         if (*ret)
-                return;
+                return 0;
         if (!glusterd_check_globaloption (key))
                 *ret = 1;
+
+        return 0;
 }
 
 int
@@ -3210,23 +3754,34 @@ glusterd_validate_globalopts (glusterd_volinfo_t *volinfo,
         }
 
         ret = validate_nfsopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate nfs");
+                goto out;
+        }
 
+        ret = validate_shdopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate self-heald");
+                goto out;
+        }
 
 out:
-                gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-                return ret;
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
 }
 
-static void
+static int
 _check_localopt (dict_t *this, char *key, data_t *value, void *ret_val)
 {
         int *ret = NULL;
 
         ret = ret_val;
         if (*ret)
-                return;
+                return 0;
         if (!glusterd_check_localoption (key))
                 *ret = 1;
+
+        return 0;
 }
 
 int
@@ -3256,9 +3811,255 @@ glusterd_validate_reconfopts (glusterd_volinfo_t *volinfo, dict_t *val_dict,
         }
 
         ret = validate_nfsopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate nfs");
+                goto out;
+        }
+
+
+        ret = validate_shdopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate self-heald");
+                goto out;
+        }
 
 
 out:
-                gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-                return ret;
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static struct volopt_map_entry *
+_gd_get_vmep (char *key) {
+        char *completion = NULL;
+        struct volopt_map_entry *vmep = NULL;
+        int ret = 0;
+
+        COMPLETE_OPTION ((char *)key, completion, ret);
+        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+                if (strcmp (vmep->key, key) == 0)
+                        return vmep;
+        }
+
+        return NULL;
+}
+
+uint32_t
+glusterd_get_op_version_for_key (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep)
+                return vmep->op_version;
+
+        return 0;
+}
+
+gf_boolean_t
+gd_is_client_option (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep && (vmep->flags & OPT_FLAG_CLIENT_OPT))
+                return _gf_true;
+
+        return _gf_false;
+}
+
+gf_boolean_t
+gd_is_xlator_option (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep && (vmep->flags & OPT_FLAG_XLATOR_OPT))
+                return _gf_true;
+
+        return _gf_false;
+}
+
+volume_option_type_t
+_gd_get_option_type (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+        void                    *dl_handle = NULL;
+        volume_opt_list_t       vol_opt_list = {{0},};
+        int                     ret = -1;
+        volume_option_t         *opt = NULL;
+        char                    *xlopt_key = NULL;
+        volume_option_type_t    opt_type = GF_OPTION_TYPE_MAX;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+
+        if (vmep) {
+                INIT_LIST_HEAD (&vol_opt_list.list);
+                ret = xlator_volopt_dynload (vmep->voltype, &dl_handle,
+                                             &vol_opt_list);
+                if (ret)
+                        goto out;
+
+                if (_get_xlator_opt_key_from_vme (vmep, &xlopt_key))
+                        goto out;
+
+                opt = xlator_volume_option_get_list (&vol_opt_list, xlopt_key);
+                _free_xlator_opt_key (xlopt_key);
+
+                if (opt)
+                        opt_type = opt->type;
+        }
+
+out:
+        if (dl_handle) {
+                dlclose (dl_handle);
+                dl_handle = NULL;
+        }
+
+        return opt_type;
+}
+
+gf_boolean_t
+gd_is_boolean_option (char *key)
+{
+        GF_ASSERT (key);
+
+        if (GF_OPTION_TYPE_BOOL == _gd_get_option_type (key))
+                return _gf_true;
+
+        return _gf_false;
+}
+
+/* This function will restore origin volume to it's snap.
+ * The restore operation will simply replace the Gluster origin
+ * volume with the snap volume.
+ * TODO: Multi-volume delete to be done.
+ *       Cleanup in case of restore failure is pending.
+ *
+ * @param orig_vol      volinfo of origin volume
+ * @param snap_vol      volinfo of snapshot volume
+ *
+ * @return 0 on success and negative value on error
+ */
+int
+gd_restore_snap_volume (dict_t *rsp_dict,
+                        glusterd_volinfo_t *orig_vol,
+                        glusterd_volinfo_t *snap_vol)
+{
+        int                     ret             = -1;
+        glusterd_volinfo_t      *new_volinfo    = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        xlator_t                *this           = NULL;
+        glusterd_conf_t         *conf           = NULL;
+        glusterd_volinfo_t      *temp_volinfo   = NULL;
+        glusterd_volinfo_t      *voliter        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_VALIDATE_OR_GOTO (this->name, orig_vol, out);
+        GF_VALIDATE_OR_GOTO (this->name, snap_vol, out);
+        snap = snap_vol->snapshot;
+        GF_VALIDATE_OR_GOTO (this->name, snap, out);
+
+        /* Snap volume must be stoped before performing the
+         * restore operation.
+         */
+        ret = glusterd_stop_volume (snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to stop "
+                        "snap volume");
+                goto out;
+        }
+
+        /* Create a new volinfo for the restored volume */
+        ret = glusterd_volinfo_dup (snap_vol, &new_volinfo, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create volinfo");
+                goto out;
+        }
+
+        /* Following entries need to be derived from origin volume. */
+        strcpy (new_volinfo->volname, orig_vol->volname);
+        uuid_copy (new_volinfo->volume_id, orig_vol->volume_id);
+        new_volinfo->snap_count = orig_vol->snap_count;
+        new_volinfo->snap_max_hard_limit = orig_vol->snap_max_hard_limit;
+        new_volinfo->is_volume_restored = _gf_true;
+
+        /* Bump the version of the restored volume, so that nodes *
+         * which are done can sync during handshake */
+        new_volinfo->version = orig_vol->version;
+
+        list_for_each_entry_safe (voliter, temp_volinfo,
+                         &orig_vol->snap_volumes, snapvol_list) {
+                list_add_tail (&voliter->snapvol_list,
+                               &new_volinfo->snap_volumes);
+        }
+        /* Copy the snap vol info to the new_volinfo.*/
+        ret = glusterd_snap_volinfo_restore (rsp_dict, new_volinfo, snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to restore snap");
+                (void)glusterd_volinfo_delete (new_volinfo);
+                goto out;
+        }
+
+        /* If the orig_vol is already restored then we should delete
+         * the backend LVMs */
+        if (orig_vol->is_volume_restored) {
+                ret = glusterd_lvm_snapshot_remove (rsp_dict, orig_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to remove "
+                                "LVM backend");
+                        (void)glusterd_volinfo_delete (new_volinfo);
+                        goto out;
+                }
+        }
+
+        /* Once the new_volinfo is completely constructed then delete
+         * the orinal volinfo
+         */
+        ret = glusterd_volinfo_delete (orig_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to delete volinfo");
+                (void)glusterd_volinfo_delete (new_volinfo);
+                goto out;
+        }
+
+        /* New volinfo always shows the status as created. Therefore
+         * set the status to stop. */
+        glusterd_set_volume_status (new_volinfo, GLUSTERD_STATUS_STOPPED);
+
+        list_add_tail (&new_volinfo->vol_list, &conf->volumes);
+
+        /* Now delete the snap entry. As a first step delete the snap
+         * volume information stored in store. */
+        ret = glusterd_snap_remove (rsp_dict, snap, _gf_false, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
+                        "snap %s", snap->snapname);
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo (new_volinfo,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 974aed934..fcbaaf93e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_VOLGEN_H_
 #define _GLUSTERD_VOLGEN_H_
 
@@ -33,8 +23,22 @@
 #define VKEY_DIAG_LAT_MEASUREMENT "diagnostics.latency-measurement"
 #define VKEY_FEATURES_LIMIT_USAGE "features.limit-usage"
 #define VKEY_MARKER_XTIME         GEOREP".indexing"
+#define VKEY_MARKER_XTIME_FORCE   GEOREP".ignore-pid-check"
+#define VKEY_CHANGELOG            "changelog.changelog"
 #define VKEY_FEATURES_QUOTA       "features.quota"
-#define VKEY_PERF_STAT_PREFETCH   "performance.stat-prefetch"
+
+#define AUTH_ALLOW_MAP_KEY "auth.allow"
+#define AUTH_REJECT_MAP_KEY "auth.reject"
+#define NFS_DISABLE_MAP_KEY "nfs.disable"
+#define AUTH_ALLOW_OPT_KEY "auth.addr.*.allow"
+#define AUTH_REJECT_OPT_KEY "auth.addr.*.reject"
+#define NFS_DISABLE_OPT_KEY "nfs.*.disable"
+
+
+typedef enum {
+        GF_CLIENT_TRUSTED,
+        GF_CLIENT_OTHER
+} glusterd_client_type_t;
 
 #define COMPLETE_OPTION(key, completion, ret)                           \
         do {                                                            \
@@ -58,9 +62,54 @@
 
 typedef enum gd_volopt_flags_ {
         OPT_FLAG_NONE,
-        OPT_FLAG_FORCE = 1,
+        OPT_FLAG_FORCE = 0x01,      // option needs force to be reset
+        OPT_FLAG_XLATOR_OPT = 0x02, // option enables/disables xlators
+        OPT_FLAG_CLIENT_OPT = 0x04, // option affects clients
 } gd_volopt_flags_t;
 
+typedef enum {
+        GF_XLATOR_POSIX = 0,
+        GF_XLATOR_ACL,
+        GF_XLATOR_LOCKS,
+        GF_XLATOR_IOT,
+        GF_XLATOR_INDEX,
+        GF_XLATOR_MARKER,
+        GF_XLATOR_IO_STATS,
+        GF_XLATOR_BD,
+        GF_XLATOR_NONE,
+} glusterd_server_xlator_t;
+
+/* As of now debug xlators can be loaded only below fuse in the client
+ * graph via cli. More xlators can be added below when the cli option
+ * for adding debug xlators anywhere in the client graph has to be made
+ * available.
+ */
+typedef enum {
+        GF_CLNT_XLATOR_FUSE = 0,
+        GF_CLNT_XLATOR_NONE,
+} glusterd_client_xlator_t;
+
+typedef enum  { DOC, NO_DOC, GLOBAL_DOC, GLOBAL_NO_DOC } option_type_t;
+
+typedef int (*vme_option_validation) (dict_t *dict, char *key, char *value,
+                                      char **op_errstr);
+
+struct volopt_map_entry {
+        char *key;
+        char *voltype;
+        char *option;
+        char *value;
+        option_type_t type;
+        uint32_t flags;
+        uint32_t op_version;
+        char *description;
+        vme_option_validation validate_fn;
+        /* If client_option is true, the option affects clients.
+         * this is used to calculate client-op-version of volumes
+         */
+        //gf_boolean_t client_option;
+};
+
 int glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
                                  glusterd_brickinfo_t *brickinfo);
 
@@ -75,6 +124,10 @@ int glusterd_create_shd_volfile ();
 
 int glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
                              glusterd_brickinfo_t *brickinfo);
+int
+glusterd_delete_snap_volfile (glusterd_volinfo_t *volinfo,
+                              glusterd_volinfo_t *snap_volinfo,
+                              glusterd_brickinfo_t *brickinfo);
 
 int glusterd_volinfo_get (glusterd_volinfo_t *volinfo, char *key, char **value);
 int glusterd_volinfo_get_boolean (glusterd_volinfo_t *volinfo, char *key);
@@ -88,7 +141,36 @@ glusterd_check_voloption_flags (char *key, int32_t flags);
 gf_boolean_t
 glusterd_is_valid_volfpath (char *volname, char *brick);
 int generate_brick_volfiles (glusterd_volinfo_t *volinfo);
-int glusterd_get_volopt_content (gf_boolean_t xml_out);
+int generate_snap_brick_volfiles (glusterd_volinfo_t *volinfo,
+                                  glusterd_volinfo_t *snap_volinfo);
+int generate_client_volfiles (glusterd_volinfo_t *volinfo,
+                              glusterd_client_type_t client_type);
+int
+generate_snap_client_volfiles (glusterd_volinfo_t *actual_volinfo,
+                               glusterd_volinfo_t *snap_volinfo,
+                               glusterd_client_type_t client_type,
+                               gf_boolean_t vol_restore);
+int glusterd_get_volopt_content (dict_t *dict, gf_boolean_t xml_out);
 char*
 glusterd_get_trans_type_rb (gf_transport_type ttype);
+int
+glusterd_check_nfs_volfile_identical (gf_boolean_t *identical);
+int
+glusterd_check_nfs_topology_identical (gf_boolean_t *identical);
+
+uint32_t
+glusterd_get_op_version_for_key (char *key);
+
+gf_boolean_t
+gd_is_client_option (char *key);
+
+gf_boolean_t
+gd_is_xlator_option (char *key);
+
+gf_boolean_t
+gd_is_boolean_option (char *key);
+
+int gd_restore_snap_volume (dict_t *rsp_dict,
+                            glusterd_volinfo_t *orig_vol,
+                            glusterd_volinfo_t *snap_vol);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 3208ad355..0d322b9ad 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -1,28 +1,23 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
+#ifdef HAVE_BD_XLATOR
+#include <lvm2app.h>
+#endif
+
 #include "common-utils.h"
+#include "syscall.h"
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -35,20 +30,16 @@
 #define glusterd_op_start_volume_args_get(dict, volname, flags) \
         glusterd_op_stop_volume_args_get (dict, volname, flags)
 
+
 int
-glusterd_handle_create_volume (rpcsvc_request_t *req)
+__glusterd_handle_create_volume (rpcsvc_request_t *req)
 {
         int32_t                 ret         = -1;
         gf_cli_req              cli_req     = {{0,}};
         dict_t                 *dict        = NULL;
-        glusterd_brickinfo_t   *brickinfo   = NULL;
-        char                   *brick       = NULL;
         char                   *bricks      = NULL;
         char                   *volname     = NULL;
-        int                    brick_count = 0;
-        char                   *tmpptr      = NULL;
-        int                    i           = 0;
-        char                   *brick_list  = NULL;
+        int                    brick_count  = 0;
         void                   *cli_rsp     = NULL;
         char                    err_str[2048] = {0,};
         gf_cli_rsp              rsp         = {0,};
@@ -56,27 +47,27 @@ glusterd_handle_create_volume (rpcsvc_request_t *req)
         char                   *free_ptr    = NULL;
         char                   *trans_type  = NULL;
         uuid_t                  volume_id   = {0,};
-        glusterd_brickinfo_t    *tmpbrkinfo = NULL;
-        glusterd_volinfo_t      tmpvolinfo = {{0},};
-        int32_t                 type       = 0;
+        uuid_t                  tmp_uuid    = {0};
+        int32_t                 type        = 0;
+        char                   *username    = NULL;
+        char                   *password    = NULL;
 
         GF_ASSERT (req);
 
-        INIT_LIST_HEAD (&tmpvolinfo.bricks);
-
         this = THIS;
         GF_ASSERT(this);
 
         ret = -1;
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
-                snprintf (err_str, sizeof (err_str), "Garbage args received");
+                snprintf (err_str, sizeof (err_str), "Failed to decode request "
+                          "received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received create volume req");
+        gf_log (this->name, GF_LOG_DEBUG, "Received create volume req");
 
         if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
@@ -86,11 +77,11 @@ glusterd_handle_create_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
                         snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the buffer");
+                                  "the command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -100,47 +91,52 @@ glusterd_handle_create_volume (rpcsvc_request_t *req)
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
                 snprintf (err_str, sizeof (err_str), "Unable to get volume "
                           "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
-        gf_cmd_log ("Volume create", "on volname: %s attempted", volname);
 
         if ((ret = glusterd_check_volume_exists (volname))) {
-                snprintf(err_str, 2048, "Volume %s already exists", volname);
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                snprintf (err_str, sizeof (err_str), "Volume %s already exists",
+                          volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "brick count");
+                snprintf (err_str, sizeof (err_str), "Unable to get brick count"
+                          " for volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "type", &type);
-        /*if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get type");
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "type");
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get type of "
+                          "volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
-        }*/
+        }
 
         ret = dict_get_str (dict, "transport", &trans_type);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get transport-type");
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "transport-type");
+                snprintf (err_str, sizeof (err_str), "Unable to get "
+                          "transport-type of volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
         ret = dict_get_str (dict, "bricks", &bricks);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get bricks");
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "bricks");
+                snprintf (err_str, sizeof (err_str), "Unable to get bricks for "
+                          "volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        if (!dict_get (dict, "force")) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get 'force' flag");
                 goto out;
         }
 
@@ -148,102 +144,82 @@ glusterd_handle_create_volume (rpcsvc_request_t *req)
         free_ptr = gf_strdup (uuid_utoa (volume_id));
         ret = dict_set_dynstr (dict, "volume-id", free_ptr);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "unable to set volume-id");
                 snprintf (err_str, sizeof (err_str), "Unable to set volume "
-                          "id");
+                          "id of volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
         free_ptr = NULL;
 
-        if (bricks) {
-                brick_list = gf_strdup (bricks);
-                free_ptr = brick_list;
-        }
-
-        gf_cmd_log ("Volume create", "on volname: %s type:%s count:%d bricks:%s",
-                    volname, ((type == 0)? "DEFAULT":
-                    ((type == 1)? "STRIPE":"REPLICATE")), brick_count, bricks);
-
+        /* generate internal username and password */
 
-        while ( i < brick_count) {
-                i++;
-                brick= strtok_r (brick_list, " \n", &tmpptr);
-                brick_list = tmpptr;
-                ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str), "Unable to get "
-                                  "brick info from brick %s", brick);
-                        goto out;
-                }
+        uuid_generate (tmp_uuid);
+        username = gf_strdup (uuid_utoa (tmp_uuid));
+        ret = dict_set_dynstr (dict, "internal-username", username);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set username for "
+                        "volume %s", volname);
+                goto out;
+        }
 
-                ret = glusterd_new_brick_validate (brick, brickinfo, err_str,
-                                                   sizeof (err_str));
-                if (ret)
-                        goto out;
-                ret = glusterd_volume_brickinfo_get (brickinfo->uuid,
-                                                     brickinfo->hostname,
-                                                     brickinfo->path,
-                                                     &tmpvolinfo, &tmpbrkinfo,
-                                                     GF_PATH_PARTIAL);
-                if (!ret) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str), "Brick: %s:%s, %s"
-                                  " one of the bricks contain the other",
-                                  tmpbrkinfo->hostname, tmpbrkinfo->path, brick);
-                        goto out;
-                }
-                list_add_tail (&brickinfo->brick_list, &tmpvolinfo.bricks);
-                brickinfo = NULL;
+        uuid_generate (tmp_uuid);
+        password = gf_strdup (uuid_utoa (tmp_uuid));
+        ret = dict_set_dynstr (dict, "internal-password", password);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set password for "
+                        "volume %s", volname);
+                goto out;
         }
 
-        ret = glusterd_op_begin (req, GD_OP_CREATE_VOLUME, dict);
-        gf_cmd_log ("Volume create", "on volname: %s %s", volname,
-                    (ret != 0) ? "FAILED": "SUCCESS");
+        ret = glusterd_op_begin_synctask (req, GD_OP_CREATE_VOLUME, dict);
 
 out:
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
                 rsp.op_ret = -1;
                 rsp.op_errno = 0;
                 if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str), "Operation failed");
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 rsp.op_errstr = err_str;
                 cli_rsp = &rsp;
-                glusterd_submit_reply(req, cli_rsp, NULL, 0, NULL,
-                                      (xdrproc_t)xdr_gf_cli_rsp);
-
+                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
                 ret = 0; //Client response sent, prevent second response
         }
 
-        if (free_ptr)
-                GF_FREE(free_ptr);
-
-        glusterd_volume_brickinfos_delete (&tmpvolinfo);
-        if (brickinfo)
-                glusterd_brickinfo_delete (brickinfo);
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        GF_FREE(free_ptr);
 
         return ret;
 }
 
 int
-glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
+glusterd_handle_create_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_create_volume);
+}
+
+int
+__glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
-        char                            *dup_volname = NULL;
+        char                            *volname = NULL;
         dict_t                          *dict = NULL;
         glusterd_op_t                   cli_op = GD_OP_START_VOLUME;
+        char                            errstr[2048] = {0,};
+        xlator_t                        *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (errstr, sizeof (errstr), "Failed to decode message "
+                        "received from cli");
                 req->rpc_err = GARBAGE_ARGS;
+                gf_log (this->name, sizeof (errstr), "%s", errstr);
                 goto out;
         }
 
@@ -255,58 +231,68 @@ glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (errstr, sizeof (errstr), "Unable to decode "
+                                  "the command");
                         goto out;
                 }
         }
 
-        ret = dict_get_str (dict, "volname", &dup_volname);
+        ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict get failed");
+                snprintf (errstr, sizeof (errstr), "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received start vol req"
-                "for volume %s", dup_volname);
-        ret = glusterd_op_begin (req, GD_OP_START_VOLUME, dict);
+        gf_log (this->name, GF_LOG_DEBUG, "Received start vol req"
+                " for volume %s", volname);
 
-        gf_cmd_log ("volume start","on volname: %s %s", dup_volname,
-                    ((ret == 0) ? "SUCCESS": "FAILED"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_START_VOLUME, dict);
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        free (cli_req.dict.dict_val); //its malloced by xdr
 
-        if (ret)
+        if (ret) {
+                if(errstr[0] == '\0')
+                        snprintf (errstr, sizeof (errstr), "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, errstr);
+        }
 
         return ret;
 }
 
+int
+glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_start_volume);
+}
 
 int
-glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
         char                            *dup_volname = NULL;
         dict_t                          *dict = NULL;
         glusterd_op_t                   cli_op = GD_OP_STOP_VOLUME;
+        xlator_t                        *this = NULL;
+        char                            err_str[2048] = {0,};
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode message "
+                          "received from cli");
                 req->rpc_err = GARBAGE_ARGS;
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
         if (cli_req.dict.dict_len) {
@@ -317,9 +303,11 @@ glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 }
         }
@@ -327,48 +315,59 @@ glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
         ret = dict_get_str (dict, "volname", &dup_volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received stop vol req"
+        gf_log (this->name, GF_LOG_DEBUG, "Received stop vol req "
                 "for volume %s", dup_volname);
 
-        ret = glusterd_op_begin (req, GD_OP_STOP_VOLUME, dict);
-        gf_cmd_log ("Volume stop","on volname: %s %s", dup_volname,
-                    ((ret)?"FAILED":"SUCCESS"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_STOP_VOLUME, dict);
 
 out:
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        free (cli_req.dict.dict_val); //its malloced by xdr
 
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
         }
 
         return ret;
 }
 
 int
-glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_stop_volume);
+}
+
+int
+__glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
 {
         int32_t        ret         = -1;
         gf_cli_req     cli_req     = {{0,},};
         glusterd_op_t  cli_op      = GD_OP_DELETE_VOLUME;
         dict_t        *dict        = NULL;
         char          *volname     = NULL;
+        char          err_str[2048]= {0,};
+        xlator_t      *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode request "
+                          "received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -381,65 +380,74 @@ glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to get volname");
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
-        gf_cmd_log ("Volume delete","on volname: %s attempted", volname);
-
-        gf_log ("glusterd", GF_LOG_INFO, "Received delete vol req"
+        gf_log (this->name, GF_LOG_DEBUG, "Received delete vol req"
                 "for volume %s", volname);
 
-        ret = glusterd_op_begin (req, GD_OP_DELETE_VOLUME, dict);
-        gf_cmd_log ("Volume delete", "on volname: %s %s", volname,
-                   ((ret) ? "FAILED" : "SUCCESS"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_DELETE_VOLUME, dict);
 
 out:
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
-        if (ret && dict)
-                dict_unref (dict);
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        free (cli_req.dict.dict_val); //its malloced by xdr
 
         if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
         }
 
         return ret;
 }
 
 int
-glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_delete_volume);
+}
+
+int
+__glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
         dict_t                          *dict = NULL;
         glusterd_op_t                   cli_op = GD_OP_HEAL_VOLUME;
         char                            *volname = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        xlator_t                        *this = NULL;
+        char                            op_errstr[2048] = {0,};
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
+        this = THIS;
+        GF_ASSERT (this);
+
         if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
                 dict  = dict_new ();
@@ -448,44 +456,67 @@ glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (op_errstr, sizeof (op_errstr),
+                        "Unable to decode the command");
                         goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (op_errstr, sizeof (op_errstr), "Unable to find "
+                          "volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
                 goto out;
         }
-        gf_log ("glusterd", GF_LOG_INFO, "Received heal vol req"
+
+        gf_log (this->name, GF_LOG_INFO, "Received heal vol req "
                 "for volume %s", volname);
 
-        ret = glusterd_op_begin (req, GD_OP_HEAL_VOLUME, dict);
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (op_errstr, sizeof (op_errstr),
+                          "Volume %s does not exist", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+                goto out;
+        }
 
-        gf_cmd_log ("volume heal","on volname: %s %s", volname,
-                    ((ret == 0) ? "SUCCESS": "FAILED"));
+        ret = glusterd_add_bricks_hname_path_to_dict (dict, volinfo);
+        if (ret)
+                goto out;
 
-out:
-        if (ret && dict)
-                dict_unref (dict);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
+        ret = dict_set_int32 (dict, "count", volinfo->brick_count);
+        if (ret)
+                goto out;
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        ret = glusterd_op_begin_synctask (req, GD_OP_HEAL_VOLUME, dict);
 
-        if (ret)
+out:
+        if (ret) {
+                if (op_errstr[0] == '\0')
+                        snprintf (op_errstr, sizeof (op_errstr),
+                                  "operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, op_errstr);
+        }
 
         return ret;
 }
 
 int
-glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_heal_volume);
+}
+
+int
+__glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -493,12 +524,18 @@ glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
         char                            *options = NULL;
         dict_t                          *dict = NULL;
         int32_t                         option_cnt = 0;
+        glusterd_op_t                   cli_op = GD_OP_STATEDUMP_VOLUME;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (req);
 
         ret = -1;
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -510,50 +547,158 @@ glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to "
+                                  "decode the command");
                         goto out;
                 }
         }
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (err_str, sizeof (err_str), "Unable to get the volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_str (dict, "options", &options);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get options");
+                snprintf (err_str, sizeof (err_str), "Unable to get options");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "option_cnt", &option_cnt);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get option cnt");
+                snprintf (err_str , sizeof (err_str), "Unable to get option "
+                          "count");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
 
-        gf_log ("glusterd", GF_LOG_INFO, "Recieved statedump request for "
+        gf_log (this->name, GF_LOG_INFO, "Received statedump request for "
                 "volume %s with options %s", volname, options);
 
-        ret = glusterd_op_begin (req, GD_OP_STATEDUMP_VOLUME, dict);
-
-        gf_cmd_log ("statedump", "on volume %s %s", volname,
-                    ((0 == ret) ? "SUCCEEDED" : "FAILED"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_STATEDUMP_VOLUME, dict);
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);
-        glusterd_friend_sm ();
-        glusterd_op_sm();
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        free (cli_req.dict.dict_val);
 
         return ret;
 }
 
+int
+glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_statedump_volume);
+}
+
+#ifdef HAVE_BD_XLATOR
+/*
+ * Validates if given VG in the brick exists or not. Also checks if VG has
+ * GF_XATTR_VOL_ID_KEY tag set to avoid using same VG for multiple bricks.
+ * Tag is checked only during glusterd_op_stage_create_volume. Tag is set during
+ * glusterd_validate_and_create_brickpath().
+ * @brick - brick info, @check_tag - check for VG tag or not
+ * @msg - Error message to return to caller
+ */
+int
+glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg)
+{
+        lvm_t                     handle     = NULL;
+        vg_t                      vg         = NULL;
+        char                     *vg_name    = NULL;
+        int                       retval     = 0;
+        char                     *p          = NULL;
+        char                     *ptr        = NULL;
+        struct dm_list           *dm_lvlist = NULL;
+        struct dm_list           *dm_seglist = NULL;
+        struct lvm_lv_list       *lv_list    = NULL;
+        struct lvm_property_value prop       = {0, };
+        struct lvm_lvseg_list    *seglist    = NULL;
+        struct dm_list           *taglist    = NULL;
+        struct lvm_str_list      *strl       = NULL;
+
+        handle = lvm_init (NULL);
+        if (!handle) {
+                sprintf (msg, "lvm_init failed, could not validate vg");
+                return -1;
+        }
+        if (*brick->vg == '\0') { /* BD xlator has vg in brick->path */
+                p = gf_strdup (brick->path);
+                vg_name = strtok_r (p, "/", &ptr);
+        } else
+                vg_name = brick->vg;
+
+        vg = lvm_vg_open (handle, vg_name, "r", 0);
+        if (!vg) {
+                sprintf (msg, "no such vg: %s", vg_name);
+                retval = -1;
+                goto out;
+        }
+        if (!check_tag)
+                goto next;
+
+        taglist = lvm_vg_get_tags (vg);
+        if (!taglist)
+                goto next;
+
+        dm_list_iterate_items (strl, taglist) {
+                if (!strncmp(strl->str, GF_XATTR_VOL_ID_KEY,
+                             strlen (GF_XATTR_VOL_ID_KEY))) {
+                            sprintf (msg, "VG %s is already part of"
+                                    " a brick", vg_name);
+                            retval = -1;
+                            goto out;
+                    }
+        }
+next:
+
+        brick->caps = CAPS_BD | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+
+        dm_lvlist = lvm_vg_list_lvs (vg);
+        if (!dm_lvlist)
+                goto out;
+
+        dm_list_iterate_items (lv_list, dm_lvlist) {
+                dm_seglist = lvm_lv_list_lvsegs (lv_list->lv);
+                dm_list_iterate_items (seglist, dm_seglist) {
+                        prop = lvm_lvseg_get_property (seglist->lvseg,
+                                                       "segtype");
+                        if (!prop.is_valid || !prop.value.string)
+                                continue;
+                        if (!strcmp (prop.value.string, "thin-pool")) {
+                                brick->caps |= CAPS_THIN;
+                                gf_log (THIS->name, GF_LOG_INFO, "Thin Pool "
+                                        "\"%s\" will be used for thin LVs",
+                                        lvm_lv_get_name (lv_list->lv));
+                                break;
+                        }
+                }
+        }
+
+        retval = 0;
+out:
+        if (vg)
+                lvm_vg_close (vg);
+        lvm_quit (handle);
+        if (p)
+                GF_FREE (p);
+        return retval;
+}
+#endif
+
 /* op-sm */
 int
 glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
@@ -569,73 +714,68 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
         int32_t                                 i = 0;
         char                                    *brick = NULL;
         char                                    *tmpptr = NULL;
-        char                                    cmd_str[1024];
         xlator_t                                *this = NULL;
         glusterd_conf_t                         *priv = NULL;
         char                                    msg[2048] = {0};
         uuid_t                                  volume_uuid;
         char                                    *volume_uuid_str;
+        gf_boolean_t                             is_force = _gf_false;
 
         this = THIS;
-        if (!this) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "this is NULL");
-                goto out;
-        }
-
+        GF_ASSERT (this);
         priv = this->private;
-        if (!priv) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "priv is NULL");
-                goto out;
-        }
+        GF_ASSERT (priv);
 
         ret = dict_get_str (dict, "volname", &volname);
-
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         exists = glusterd_check_volume_exists (volname);
-
         if (exists) {
                 snprintf (msg, sizeof (msg), "Volume %s already exists",
                           volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
                 ret = -1;
                 goto out;
         } else {
                 ret = 0;
         }
+
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count "
+                        "for volume %s", volname);
                 goto out;
         }
+
         ret = dict_get_str (dict, "volume-id", &volume_uuid_str);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume id");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume id of "
+                        "volume %s", volname);
                 goto out;
         }
+
         ret = uuid_parse (volume_uuid_str, volume_uuid);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to parse volume id");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to parse volume id of"
+                        " volume %s", volname);
                 goto out;
         }
 
         ret = dict_get_str (dict, "bricks", &bricks);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get bricks");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get bricks for "
+                        "volume %s", volname);
                 goto out;
         }
 
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
         if (bricks) {
                 brick_list = gf_strdup (bricks);
                 if (!brick_list) {
                         ret = -1;
-                        gf_log ("", GF_LOG_ERROR, "Out of memory");
                         goto out;
                 } else {
                         free_ptr = brick_list;
@@ -651,30 +791,38 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
                         !glusterd_is_valid_volfpath (volname, brick)) {
                         snprintf (msg, sizeof (msg), "brick path %s is too "
                                   "long.", brick);
-                        gf_log ("", GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-
                         ret = -1;
                         goto out;
                 }
 
-                ret = glusterd_brickinfo_from_brick (brick, &brick_info);
+                ret = glusterd_brickinfo_new_from_brick (brick, &brick_info);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_new_brick_validate (brick, brick_info, msg,
+                                                   sizeof (msg));
                 if (ret)
                         goto out;
-                snprintf (cmd_str, 1024, "%s", brick_info->path);
+
                 ret = glusterd_resolve_brick (brick_info);
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "cannot resolve "
-                                "brick: %s:%s", brick_info->hostname,
-                                brick_info->path);
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                brick_info->hostname, brick_info->path);
                         goto out;
                 }
 
-                if (!uuid_compare (brick_info->uuid, priv->uuid)) {
-                        ret = glusterd_brick_create_path (brick_info->hostname,
-                                                          brick_info->path,
-                                                          volume_uuid,
-                                                          0777, op_errstr);
+                if (!uuid_compare (brick_info->uuid, MY_UUID)) {
+
+#ifdef HAVE_BD_XLATOR
+                        if (brick_info->vg[0]) {
+                                ret = glusterd_is_valid_vg (brick_info, 1, msg);
+                                if (ret)
+                                        goto out;
+                        }
+#endif
+                        ret = glusterd_validate_and_create_brickpath (brick_info,
+                                                          volume_uuid, op_errstr,
+                                                          is_force);
                         if (ret)
                                 goto out;
                         brick_list = tmpptr;
@@ -683,11 +831,15 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
                 brick_info = NULL;
         }
 out:
-        if (free_ptr)
-                GF_FREE (free_ptr);
+        GF_FREE (free_ptr);
         if (brick_info)
                 glusterd_brickinfo_delete (brick_info);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -696,19 +848,23 @@ int
 glusterd_op_stop_volume_args_get (dict_t *dict, char** volname, int *flags)
 {
         int ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         if (!dict || !volname || !flags)
                 goto out;
 
         ret = dict_get_str (dict, "volname", volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "flags", flags);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get flags");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get flags");
                 goto out;
         }
 out:
@@ -757,14 +913,16 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
         glusterd_brickinfo_t                    *brickinfo = NULL;
         char                                    msg[2048];
         glusterd_conf_t                         *priv = NULL;
+        xlator_t                                *this = NULL;
+        uuid_t                                  volume_id = {0,};
+        char                                    volid[50] = {0,};
+        char                                    xattr_volid[50] = {0,};
+        int                                     caps = 0;
 
-        priv = THIS->private;
-        if (!priv) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "priv is NULL");
-                ret = -1;
-                goto out;
-        }
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
 
         ret = glusterd_op_start_volume_args_get (dict, &volname, &flags);
         if (ret)
@@ -773,53 +931,110 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
         exists = glusterd_check_volume_exists (volname);
 
         if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist", volname);
-                gf_log ("", GF_LOG_ERROR, "%s",
-                        msg);
-                *op_errstr = gf_strdup (msg);
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
                 ret = -1;
-        } else {
-                ret = 0;
+                goto out;
         }
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
         if (ret)
                 goto out;
 
+        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+                if (glusterd_is_volume_started (volinfo)) {
+                        snprintf (msg, sizeof (msg), "Volume %s already "
+                                  "started", volname);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 ret = glusterd_resolve_brick (brickinfo);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to resolve brick %s:%s",
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
                                 brickinfo->hostname, brickinfo->path);
                         goto out;
                 }
 
-                if (!uuid_compare (brickinfo->uuid, priv->uuid)) {
-                        ret = glusterd_brick_create_path (brickinfo->hostname,
-                                                          brickinfo->path,
-                                                          volinfo->volume_id,
-                                                          0777, op_errstr);
-                        if (ret)
-                                goto out;
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                ret = gf_lstat_dir (brickinfo->path, NULL);
+                if (ret && (flags & GF_CLI_FLAG_OP_FORCE)) {
+                        continue;
+                } else if (ret) {
+                        snprintf (msg, sizeof (msg), "Failed to find "
+                                          "brick directory %s for volume %s. "
+                                          "Reason : %s", brickinfo->path,
+                                          volname, strerror (errno));
+                        goto out;
                 }
-
-                if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
-                        if (glusterd_is_volume_started (volinfo)) {
-                                snprintf (msg, sizeof (msg), "Volume %s already"
-                                          " started", volname);
-                                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
-                                *op_errstr = gf_strdup (msg);
-                                ret = -1;
+                ret = sys_lgetxattr (brickinfo->path, GF_XATTR_VOL_ID_KEY,
+                                     volume_id, 16);
+                if (ret < 0 && (!(flags & GF_CLI_FLAG_OP_FORCE))) {
+                        snprintf (msg, sizeof (msg), "Failed to get "
+                                  "extended attribute %s for brick dir %s. "
+                                  "Reason : %s", GF_XATTR_VOL_ID_KEY,
+                                  brickinfo->path, strerror (errno));
+                        ret = -1;
+                        goto out;
+                } else if (ret < 0) {
+                        ret = sys_lsetxattr (brickinfo->path,
+                                             GF_XATTR_VOL_ID_KEY,
+                                             volinfo->volume_id, 16,
+                                             XATTR_CREATE);
+                        if (ret) {
+                                snprintf (msg, sizeof (msg), "Failed to set "
+                                        "extended attribute %s on %s. Reason: "
+                                        "%s", GF_XATTR_VOL_ID_KEY,
+                                        brickinfo->path, strerror (errno));
                                 goto out;
+                        } else {
+                                continue;
                         }
                 }
+                if (uuid_compare (volinfo->volume_id, volume_id)) {
+                        snprintf (msg, sizeof (msg), "Volume id mismatch for "
+                                  "brick %s:%s. Expected volume id %s, "
+                                  "volume id %s found", brickinfo->hostname,
+                                  brickinfo->path,
+                                  uuid_utoa_r (volinfo->volume_id, volid),
+                                  uuid_utoa_r (volume_id, xattr_volid));
+                        ret = -1;
+                        goto out;
+                }
+#ifdef HAVE_BD_XLATOR
+                if (brickinfo->vg[0])
+                        caps = CAPS_BD | CAPS_THIN |
+                                CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+                /* Check for VG/thin pool if its BD volume */
+                if (brickinfo->vg[0]) {
+                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                        if (ret)
+                                goto out;
+                        /* if anyone of the brick does not have thin support,
+                           disable it for entire volume */
+                        caps &= brickinfo->caps;
+                } else
+                        caps = 0;
+#endif
         }
 
+        volinfo->caps = caps;
         ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-
+        if (ret && (msg[0] != '\0')) {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
         return ret;
 }
 
@@ -833,7 +1048,10 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
         gf_boolean_t                            is_run = _gf_false;
         glusterd_volinfo_t                      *volinfo = NULL;
         char                                    msg[2048] = {0};
+        xlator_t                                *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
         if (ret)
@@ -842,16 +1060,20 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
         exists = glusterd_check_volume_exists (volname);
 
         if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist", volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 ret = -1;
                 goto out;
-        } else {
-                ret = 0;
         }
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
         if (ret)
                 goto out;
 
@@ -862,25 +1084,22 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
         if (_gf_false == glusterd_is_volume_started (volinfo)) {
                 snprintf (msg, sizeof(msg), "Volume %s "
                           "is not in the started state", volname);
-                gf_log ("", GF_LOG_ERROR, "Volume %s "
-                        "has not been started", volname);
-                *op_errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 ret = -1;
                 goto out;
         }
         ret = glusterd_check_gsync_running (volinfo, &is_run);
         if (ret && (is_run == _gf_false))
-                gf_log ("", GF_LOG_WARNING, "Unable to get the status"
+                gf_log (this->name, GF_LOG_WARNING, "Unable to get the status"
                         " of active "GEOREP" session");
         if (is_run) {
-                gf_log ("", GF_LOG_WARNING, GEOREP" sessions active"
+                gf_log (this->name, GF_LOG_WARNING, GEOREP" sessions active"
                         "for the volume %s ", volname);
                 snprintf (msg, sizeof(msg), GEOREP" sessions are active "
                           "for the volume '%s'.\nUse 'volume "GEOREP" "
-                          "status' command for more info. Use 'force'"
-                          "option to ignore and stop stop the volume",
+                          "status' command for more info. Use 'force' "
+                          "option to ignore and stop the volume.",
                           volname);
-                *op_errstr = gf_strdup (msg);
                 ret = -1;
                 goto out;
         }
@@ -889,8 +1108,8 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
                 snprintf (msg, sizeof (msg), "Replace brick is in progress on "
                           "volume %s. Please retry after replace-brick "
                           "operation is committed or aborted", volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_WARNING, "replace-brick in progress "
+                        "on volume %s", volname);
                 ret = -1;
                 goto out;
         }
@@ -898,23 +1117,22 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
         if (glusterd_is_defrag_on (volinfo)) {
                 snprintf (msg, sizeof(msg), "rebalance session is "
                           "in progress for the volume '%s'", volname);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
                 ret = -1;
                 goto out;
         }
-        if (volinfo->rb_status != GF_RB_STATUS_NONE) {
+        if (volinfo->rep_brick.rb_status != GF_RB_STATUS_NONE) {
                 snprintf (msg, sizeof(msg), "replace-brick session is "
                           "in progress for the volume '%s'", volname);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
                 ret = -1;
                 goto out;
         }
 
 out:
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (msg[0] != 0)
+                *op_errstr = gf_strdup (msg);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -927,21 +1145,20 @@ glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
         gf_boolean_t                            exists = _gf_false;
         glusterd_volinfo_t                      *volinfo = NULL;
         char                                    msg[2048] = {0};
+        xlator_t                                *this = NULL;
 
-        ret = dict_get_str (dict, "volname", &volname);
+        this = THIS;
+        GF_ASSERT (this);
 
+        ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         exists = glusterd_check_volume_exists (volname);
-
         if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist",
-                          volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
                 ret = -1;
                 goto out;
         } else {
@@ -949,7 +1166,12 @@ glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
         }
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                goto out;
+        }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
         if (ret)
                 goto out;
 
@@ -957,8 +1179,16 @@ glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
                 snprintf (msg, sizeof (msg), "Volume %s has been started."
                           "Volume needs to be stopped before deletion.",
                           volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        if (volinfo->snap_count > 0 || !list_empty(&volinfo->snap_volumes)) {
+                snprintf (msg, sizeof (msg), "Cannot delete Volume %s ,"
+                        "as it has %ld snapshots. "
+                        "To delete the volume, "
+                        "first delete all the snapshots under it.",
+                          volname, volinfo->snap_count);
                 ret = -1;
                 goto out;
         }
@@ -966,7 +1196,11 @@ glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
         ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -981,11 +1215,14 @@ glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
         char                                    msg[2048];
         glusterd_conf_t                         *priv = NULL;
         dict_t                                  *opt_dict = NULL;
+        gf_xl_afr_op_t                          heal_op = GF_AFR_OP_INVALID;
+        xlator_t                                *this = NULL;
 
-        priv = THIS->private;
+        this = THIS;
+        priv = this->private;
         if (!priv) {
                 ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "priv is NULL");
                 goto out;
         }
@@ -1000,17 +1237,21 @@ glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
         if (ret) {
                 ret = -1;
                 snprintf (msg, sizeof (msg), "Volume %s does not exist", volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 *op_errstr = gf_strdup (msg);
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         if (!glusterd_is_volume_replicate (volinfo)) {
                 ret = -1;
-                snprintf (msg, sizeof (msg), "Volume %s is not a replicate "
-                          "type volume", volname);
+                snprintf (msg, sizeof (msg), "Volume %s is not of type "
+                          "replicate", volname);
                 *op_errstr = gf_strdup (msg);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
                 goto out;
         }
 
@@ -1036,20 +1277,37 @@ glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
                 snprintf (msg, sizeof (msg), "Self-heal-daemon is "
                           "disabled. Heal will not be triggered on volume %s",
                           volname);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
                 *op_errstr = gf_strdup (msg);
                 goto out;
         }
 
-        if (!glusterd_shd_is_running ()) {
+        ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
+        if (ret || (heal_op == GF_AFR_OP_INVALID)) {
                 ret = -1;
-                snprintf (msg, sizeof (msg), "Self-heal daemon is not "
-                          "running. Check self-heal daemon log file.");
-                *op_errstr = gf_strdup (msg);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
+                *op_errstr = gf_strdup("Invalid heal-op");
+                gf_log (this->name, GF_LOG_WARNING, "%s", "Invalid heal-op");
                 goto out;
         }
 
+        switch (heal_op) {
+                case GF_AFR_OP_INDEX_SUMMARY:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                        break;
+                default:
+                        if (!glusterd_is_nodesvc_online("glustershd")){
+                                ret = -1;
+                                *op_errstr = gf_strdup ("Self-heal daemon is "
+                                                "not running. Check self-heal "
+                                                "daemon log file.");
+                                gf_log (this->name, GF_LOG_WARNING, "%s",
+                                        "Self-heal daemon is not running."
+                                        "Check self-heal daemon log file.");
+                                goto out;
+                        }
+        }
+
         ret = 0;
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
@@ -1082,6 +1340,10 @@ glusterd_op_stage_statedump_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         is_running = glusterd_is_volume_started (volinfo);
         if (!is_running) {
                 snprintf (msg, sizeof(msg), "Volume %s is not in a started"
@@ -1098,6 +1360,76 @@ out:
 }
 
 int
+glusterd_op_stage_clearlocks_volume (dict_t *dict, char **op_errstr)
+{
+        int                     ret = -1;
+        char                    *volname = NULL;
+        char                    *path    = NULL;
+        char                    *type    = NULL;
+        char                    *kind    = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    msg[2048] = {0,};
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Failed to get volume name");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "path", &path);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Failed to get path");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "kind", &kind);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Failed to get kind");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "type", &type);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Failed to get type");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Volume %s does not exist",
+                          volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        if (!glusterd_is_volume_started (volinfo)) {
+                snprintf (msg, sizeof(msg), "Volume %s is not started",
+                          volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
 glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 {
         int                   ret        = 0;
@@ -1116,6 +1448,10 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         char                 *saveptr    = NULL;
         char                 *trans_type = NULL;
         char                 *str        = NULL;
+        char                 *username   = NULL;
+        char                 *password   = NULL;
+        int                   caps       = 0;
+        char                  msg[1024] __attribute__((unused)) = {0, };
 
         this = THIS;
         GF_ASSERT (this);
@@ -1126,14 +1462,15 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         ret = glusterd_volinfo_new (&volinfo);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to allocate memory for volinfo");
                 goto out;
         }
 
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
@@ -1142,19 +1479,21 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 
         ret = dict_get_int32 (dict, "type", &volinfo->type);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get type");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get type of volume"
+                        " %s", volname);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "count", &volinfo->brick_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count of"
+                        " volume %s", volname);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "port", &volinfo->port);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get port");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get port");
                 goto out;
         }
 
@@ -1162,7 +1501,8 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 
         ret = dict_get_str (dict, "bricks", &bricks);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get bricks");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get bricks for "
+                        "volume %s", volname);
                 goto out;
         }
 
@@ -1174,28 +1514,44 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         if (GF_CLUSTER_TYPE_REPLICATE == volinfo->type) {
                 ret = dict_get_int32 (dict, "replica-count",
                                       &volinfo->replica_count);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                 "replica count for volume %s", volname);
                         goto out;
+                }
         } else if (GF_CLUSTER_TYPE_STRIPE == volinfo->type) {
                 ret = dict_get_int32 (dict, "stripe-count",
                                       &volinfo->stripe_count);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get stripe"
+                                " count for volume %s", volname);
                         goto out;
+                }
         } else if (GF_CLUSTER_TYPE_STRIPE_REPLICATE == volinfo->type) {
                 ret = dict_get_int32 (dict, "stripe-count",
                                       &volinfo->stripe_count);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get stripe"
+                                " count for volume %s", volname);
                         goto out;
+                }
                 ret = dict_get_int32 (dict, "replica-count",
                                       &volinfo->replica_count);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "replica count for volume %s", volname);
                         goto out;
+                }
         }
 
         /* dist-leaf-count is the count of brick nodes for a given
            subvolume of distribute */
-        volinfo->dist_leaf_count = (volinfo->stripe_count *
-                                    volinfo->replica_count);
+        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+
+        /* subvol_count is the count of number of subvolumes present
+           for a given distribute volume */
+        volinfo->subvol_count = (volinfo->brick_count /
+                                 volinfo->dist_leaf_count);
 
         /* Keep sub-count same as earlier, for the sake of backward
            compatibility */
@@ -1204,20 +1560,41 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 
         ret = dict_get_str (dict, "transport", &trans_type);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get transport");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to get transport type of volume %s", volname);
                 goto out;
         }
 
         ret = dict_get_str (dict, "volume-id", &str);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume-id");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to get volume-id of volume %s", volname);
                 goto out;
         }
         ret = uuid_parse (str, volinfo->volume_id);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "unable to parse uuid %s", str);
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to parse uuid %s of volume %s", str, volname);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "internal-username", &username);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to get internal username of volume %s",
+                        volname);
+                goto out;
+        }
+        glusterd_auth_set_username (volinfo, username);
+
+        ret = dict_get_str (dict, "internal-password", &password);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to get internal password of volume %s",
+                        volname);
                 goto out;
         }
+        glusterd_auth_set_password (volinfo, password);
 
         if (strcasecmp (trans_type, "rdma") == 0) {
                 volinfo->transport_type = GF_TRANSPORT_RDMA;
@@ -1237,20 +1614,49 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 
         if (count)
                 brick = strtok_r (brick_list+1, " \n", &saveptr);
+        caps = CAPS_BD | CAPS_THIN | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
 
         while ( i <= count) {
-                ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
+                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
                 if (ret)
                         goto out;
 
                 ret = glusterd_resolve_brick (brickinfo);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                brickinfo->hostname, brickinfo->path);
                         goto out;
+                }
+
+#ifdef HAVE_BD_XLATOR
+                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+                        if (brickinfo->vg[0]) {
+                                ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                                msg);
+                                        goto out;
+                                }
+
+                                /* if anyone of the brick does not have thin
+                                   support, disable it for entire volume */
+                                caps &= brickinfo->caps;
+
+
+                        } else
+                                caps = 0;
+                }
+#endif
+
                 list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
                 brick = strtok_r (NULL, " \n", &saveptr);
                 i++;
         }
 
+        gd_update_volume_op_versions (volinfo);
+
+        volinfo->caps = caps;
+
         ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret) {
                 glusterd_store_delete_volume (volinfo);
@@ -1264,18 +1670,12 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        ret = glusterd_volume_compute_cksum (volinfo);
-        if (ret) {
-                *op_errstr = gf_strdup ("Failed to compute checksum of volume");
-                goto out;
-        }
-
-        volinfo->defrag_status = 0;
+        volinfo->rebal.defrag_status = 0;
         list_add_tail (&volinfo->vol_list, &priv->volumes);
         vol_added = _gf_true;
+
 out:
-        if (free_ptr)
-                GF_FREE(free_ptr);
+        GF_FREE(free_ptr);
         if (!vol_added && volinfo)
                 glusterd_volinfo_delete (volinfo);
         return ret;
@@ -1289,18 +1689,28 @@ glusterd_op_start_volume (dict_t *dict, char **op_errstr)
         int                                     flags = 0;
         glusterd_volinfo_t                      *volinfo = NULL;
         glusterd_brickinfo_t                    *brickinfo = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = glusterd_op_start_volume_args_get (dict, &volname, &flags);
         if (ret)
                 goto out;
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
                 goto out;
+        }
+
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_brick_start (volinfo, brickinfo);
-                if (ret)
+                ret = glusterd_brick_start (volinfo, brickinfo, _gf_true);
+                /* If 'force' try to start all bricks regardless of success or
+                 * failure
+                 */
+                if (!(flags & GF_CLI_FLAG_OP_FORCE) && ret)
                         goto out;
         }
 
@@ -1313,7 +1723,48 @@ glusterd_op_start_volume (dict_t *dict, char **op_errstr)
         ret = glusterd_nodesvcs_handle_graph_change (volinfo);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d ", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
+        return ret;
+}
+
+int
+glusterd_stop_volume (glusterd_volinfo_t *volinfo)
+{
+        int                     ret             = -1;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+        xlator_t                *this           = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = glusterd_brick_stop (volinfo, brickinfo, _gf_false);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to stop "
+                                "brick (%s)", brickinfo->path);
+                        goto out;
+                }
+        }
+
+        glusterd_set_volume_status (volinfo, GLUSTERD_STATUS_STOPPED);
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo of "
+                        "%s volume", volinfo->volname);
+                goto out;
+        }
+
+        ret = glusterd_nodesvcs_handle_graph_change (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to notify graph "
+                        "change for %s volume", volinfo->volname);
+                goto out;
+        }
+
+out:
         return ret;
 }
 
@@ -1325,30 +1776,28 @@ glusterd_op_stop_volume (dict_t *dict)
         int                                     flags = 0;
         char                                    *volname = NULL;
         glusterd_volinfo_t                      *volinfo = NULL;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
         if (ret)
                 goto out;
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
                 goto out;
-
-        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_brick_stop (volinfo, brickinfo);
-                if (ret)
-                        goto out;
         }
 
-        glusterd_set_volume_status (volinfo, GLUSTERD_STATUS_STOPPED);
-
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
+        ret = glusterd_stop_volume (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to stop %s volume",
+                        volname);
                 goto out;
-
-        ret = glusterd_nodesvcs_handle_graph_change (volinfo);
+        }
 out:
         return ret;
 }
@@ -1369,18 +1818,20 @@ glusterd_op_delete_volume (dict_t *dict)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
                 goto out;
+        }
 
         ret = glusterd_delete_volume (volinfo);
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
@@ -1422,8 +1873,14 @@ glusterd_op_statedump_volume (dict_t *dict, char **op_errstr)
                         ret = glusterd_brick_statedump (volinfo, brickinfo,
                                                         options, option_cnt,
                                                         op_errstr);
+                        /* Let us take the statedump of other bricks instead of
+                         * exiting, if statedump of this brick fails.
+                         */
                         if (ret)
-                                goto out;
+                                gf_log (THIS->name, GF_LOG_WARNING, "could not "
+                                        "take the statedump of the brick %s:%s."
+                                        " Proceeding to other bricks",
+                                        brickinfo->hostname, brickinfo->path);
                 }
         }
 
@@ -1431,3 +1888,338 @@ out:
         return ret;
 }
 
+int
+glusterd_clearlocks_send_cmd (glusterd_volinfo_t *volinfo, char *cmd,
+                              char *path, char *result, char *errstr,
+                              int err_len, char *mntpt)
+{
+        int               ret                   = -1;
+        glusterd_conf_t  *priv                  = NULL;
+        char             abspath[PATH_MAX]      = {0, };
+
+        priv = THIS->private;
+
+        snprintf (abspath, sizeof (abspath), "%s/%s", mntpt, path);
+        ret = sys_lgetxattr (abspath, cmd, result, PATH_MAX);
+        if (ret < 0) {
+                snprintf (errstr, err_len, "clear-locks getxattr command "
+                          "failed. Reason: %s", strerror (errno));
+                gf_log (THIS->name, GF_LOG_DEBUG, "%s", errstr);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_clearlocks_rmdir_mount (glusterd_volinfo_t *volinfo, char *mntpt)
+{
+        int              ret               = -1;
+        glusterd_conf_t *priv              = NULL;
+
+        priv = THIS->private;
+
+        ret = rmdir (mntpt);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "rmdir failed");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+glusterd_clearlocks_unmount (glusterd_volinfo_t *volinfo, char *mntpt)
+{
+        glusterd_conf_t  *priv               = NULL;
+        runner_t          runner             = {0,};
+        int               ret                = 0;
+
+        priv = THIS->private;
+
+        /*umount failures are ignored. Using stat we could have avoided
+         * attempting to unmount a non-existent filesystem. But a failure of
+         * stat() on mount can be due to network failures.*/
+
+        runinit (&runner);
+        runner_add_args (&runner, "/bin/umount", "-f", NULL);
+        runner_argprintf (&runner, "%s", mntpt);
+
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                ret = 0;
+                gf_log ("", GF_LOG_DEBUG,
+                        "umount failed on maintenance client");
+        }
+
+        return;
+}
+
+int
+glusterd_clearlocks_create_mount (glusterd_volinfo_t *volinfo, char **mntpt)
+{
+        int              ret                    = -1;
+        glusterd_conf_t *priv                   = NULL;
+        char             template[PATH_MAX]     = {0,};
+        char            *tmpl                   = NULL;
+
+        priv = THIS->private;
+
+        snprintf (template, sizeof (template), "/tmp/%s.XXXXXX",
+                  volinfo->volname);
+        tmpl = mkdtemp (template);
+        if (!tmpl) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "Couldn't create temporary "
+                        "mount directory. Reason %s", strerror (errno));
+                goto out;
+        }
+
+        *mntpt = gf_strdup (tmpl);
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_clearlocks_mount (glusterd_volinfo_t *volinfo, char **xl_opts,
+                           char *mntpt)
+{
+        int             ret                             = -1;
+        int             i                               = 0;
+        glusterd_conf_t *priv                           = NULL;
+        runner_t        runner                          = {0,};
+        char            client_volfpath[PATH_MAX]       = {0,};
+        char            self_heal_opts[3][1024]      = {"*replicate*.data-self-heal=off",
+                                                        "*replicate*.metadata-self-heal=off",
+                                                        "*replicate*.entry-self-heal=off"};
+
+        priv = THIS->private;
+
+        runinit (&runner);
+        glusterd_get_trusted_client_filepath (client_volfpath, volinfo,
+                                      volinfo->transport_type);
+        runner_add_args (&runner, SBIN_DIR"/glusterfs", "-f", NULL);
+        runner_argprintf (&runner, "%s", client_volfpath);
+        runner_add_arg (&runner, "-l");
+        runner_argprintf (&runner, DEFAULT_LOG_FILE_DIRECTORY
+                          "/%s-clearlocks-mnt.log", volinfo->volname);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
+
+        for (i = 0; i < volinfo->brick_count && xl_opts[i]; i++) {
+                runner_add_arg (&runner, "--xlator-option");
+                runner_argprintf (&runner, "%s", xl_opts[i]);
+        }
+
+        for (i = 0; i < 3; i++) {
+                runner_add_args (&runner, "--xlator-option",
+                                 self_heal_opts[i], NULL);
+        }
+
+        runner_argprintf (&runner, "%s", mntpt);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG,
+                        "Could not start glusterfs");
+                goto out;
+        }
+        gf_log (THIS->name, GF_LOG_DEBUG,
+                "Started glusterfs successfully");
+
+out:
+        return ret;
+}
+
+int
+glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo,
+                                            char **xl_opts)
+{
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+        glusterd_conf_t         *priv           = NULL;
+        int                     index           = 0;
+        int                     ret             = -1;
+        int                     i               = 0;
+        int                     port            = 0;
+
+        GF_ASSERT (xl_opts);
+        if (!xl_opts) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "Should pass non-NULL "
+                        "xl_opts");
+                goto out;
+        }
+
+        priv = THIS->private;
+
+        index = -1;
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                index++;
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                port = pmap_registry_search (THIS, brickinfo->path,
+                                             GF_PMAP_PORT_BRICKSERVER);
+                if (!port) {
+                        ret = -1;
+                        gf_log (THIS->name, GF_LOG_DEBUG, "Couldn't get port "
+                                " for brick %s:%s", brickinfo->hostname,
+                                brickinfo->path);
+                        goto out;
+                }
+
+                ret = gf_asprintf (&xl_opts[i], "%s-client-%d.remote-port=%d",
+                                   volinfo->volname, index, port);
+                if (ret == -1) {
+                        xl_opts[i] = NULL;
+                        goto out;
+                }
+                i++;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t                         ret                 = -1;
+        int                             i                   = 0;
+        char                            *volname            = NULL;
+        char                            *path               = NULL;
+        char                            *kind               = NULL;
+        char                            *type               = NULL;
+        char                            *opts               = NULL;
+        char                            *cmd_str            = NULL;
+        char                            *free_ptr           = NULL;
+        char                            msg[PATH_MAX]       = {0,};
+        char                            result[PATH_MAX]    = {0,};
+        char                            *mntpt              = NULL;
+        char                            **xl_opts           = NULL;
+        glusterd_volinfo_t              *volinfo            = NULL;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get volume name");
+                goto out;
+        }
+        gf_log ("", GF_LOG_DEBUG, "Performing clearlocks on volume %s", volname);
+
+        ret = dict_get_str (dict, "path", &path);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get path");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "kind", &kind);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get kind");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "type", &type);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get type");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "opts", &opts);
+        if (ret)
+                ret = 0;
+
+        gf_log (THIS->name, GF_LOG_INFO, "Received clear-locks request for "
+                "volume %s with kind %s type %s and options %s", volname,
+                kind, type, opts);
+
+        if (opts)
+                ret = gf_asprintf (&cmd_str, GF_XATTR_CLRLK_CMD".t%s.k%s.%s",
+                                   type, kind, opts);
+        else
+                ret = gf_asprintf (&cmd_str, GF_XATTR_CLRLK_CMD".t%s.k%s",
+                                   type, kind);
+        if (ret == -1)
+                goto out;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Volume %s doesn't exist.",
+                          volname);
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        xl_opts = GF_CALLOC (volinfo->brick_count+1, sizeof (char*),
+                             gf_gld_mt_charptr);
+        if (!xl_opts)
+                goto out;
+
+        ret = glusterd_clearlocks_get_local_client_ports (volinfo, xl_opts);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Couldn't get port numbers of "
+                          "local bricks");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = glusterd_clearlocks_create_mount (volinfo, &mntpt);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Creating mount directory "
+                          "for clear-locks failed.");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = glusterd_clearlocks_mount (volinfo, xl_opts, mntpt);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to mount clear-locks "
+                          "maintenance client.");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = glusterd_clearlocks_send_cmd (volinfo, cmd_str, path, result,
+                                            msg, sizeof (msg), mntpt);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                goto umount;
+        }
+
+        free_ptr = gf_strdup(result);
+        if (dict_set_dynstr (rsp_dict, "lk-summary", free_ptr)) {
+                GF_FREE (free_ptr);
+                snprintf (msg, sizeof (msg), "Failed to set clear-locks "
+                          "result");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+        }
+
+umount:
+        glusterd_clearlocks_unmount (volinfo, mntpt);
+
+        if (glusterd_clearlocks_rmdir_mount (volinfo, mntpt))
+                gf_log (THIS->name, GF_LOG_WARNING, "Couldn't unmount "
+                        "clear-locks mount point");
+
+out:
+        if (ret)
+                *op_errstr = gf_strdup (msg);
+
+        if (xl_opts) {
+                for (i = 0; i < volinfo->brick_count && xl_opts[i]; i++)
+                        GF_FREE (xl_opts[i]);
+                GF_FREE (xl_opts);
+        }
+
+        GF_FREE (cmd_str);
+
+        GF_FREE (mntpt);
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
new file mode 100644
index 000000000..665a8b298
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -0,0 +1,1452 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterd-volgen.h"
+#include "glusterd-utils.h"
+
+static int
+check_dict_key_value (dict_t *dict, char *key, char *value)
+{
+        glusterd_conf_t     *priv          = NULL;
+        int                  ret           = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Received Empty Dict.");
+                ret = -1;
+                goto out;
+        }
+
+        if (!key) {
+                gf_log (this->name, GF_LOG_ERROR, "Received Empty Key.");
+                ret = -1;
+                goto out;
+        }
+
+        if (!value) {
+                gf_log (this->name, GF_LOG_ERROR, "Received Empty Value.");
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+get_volname_volinfo (dict_t *dict, char **volname, glusterd_volinfo_t **volinfo)
+{
+        glusterd_conf_t     *priv          = NULL;
+        int                  ret           = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (*volname, volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_cache_max_min_size (dict_t *dict, char *key, char *value,
+                             char **op_errstr)
+{
+        char                *current_max_value = NULL;
+        char                *current_min_value = NULL;
+        char                 errstr[2048]  = "";
+        char                *volname       = NULL;
+        glusterd_conf_t     *priv          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        uint64_t             max_value     = 0;
+        uint64_t             min_value     = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        if ((!strcmp (key, "performance.cache-min-file-size")) ||
+            (!strcmp (key, "cache-min-file-size"))) {
+                glusterd_volinfo_get (volinfo,
+                                      "performance.cache-max-file-size",
+                                      &current_max_value);
+                if (current_max_value) {
+                        gf_string2bytesize (current_max_value, &max_value);
+                        gf_string2bytesize (value, &min_value);
+                        current_min_value = value;
+                }
+        } else  if ((!strcmp (key, "performance.cache-max-file-size")) ||
+                    (!strcmp (key, "cache-max-file-size"))) {
+                glusterd_volinfo_get (volinfo,
+                                      "performance.cache-min-file-size",
+                                      &current_min_value);
+                if (current_min_value) {
+                        gf_string2bytesize (current_min_value, &min_value);
+                        gf_string2bytesize (value, &max_value);
+                        current_max_value = value;
+                }
+        }
+
+        if (min_value > max_value) {
+                snprintf (errstr, sizeof (errstr),
+                          "cache-min-file-size (%s) is greater than "
+                          "cache-max-file-size (%s)",
+                          current_min_value, current_max_value);
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_quota (dict_t *dict, char *key, char *value,
+                char **op_errstr)
+{
+        char                 errstr[2048] = "";
+        char                *volname      = NULL;
+        glusterd_conf_t     *priv         = NULL;
+        glusterd_volinfo_t  *volinfo      = NULL;
+        int                  ret          = 0;
+        xlator_t            *this         = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get the quota status");
+                goto out;
+        }
+
+        if (ret == _gf_false) {
+                snprintf (errstr, sizeof (errstr),
+                          "Cannot set %s. Enable quota first.", key);
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_stripe (dict_t *dict, char *key, char *value, char **op_errstr)
+{
+        char                 errstr[2048]  = "";
+        char                *volname       = NULL;
+        glusterd_conf_t     *priv          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        if (volinfo->stripe_count == 1) {
+                snprintf (errstr, sizeof (errstr),
+                          "Cannot set %s for a non-stripe volume.", key);
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+               goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_subvols_per_directory (dict_t *dict, char *key, char *value,
+                                char **op_errstr)
+{
+        char                 errstr[2048]  = "";
+        char                *volname       = NULL;
+        glusterd_conf_t     *priv          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        int                  subvols       = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        subvols = atoi(value);
+
+        /* Checking if the subvols-per-directory exceed the total
+           number of subvolumes. */
+        if (subvols > volinfo->subvol_count) {
+                snprintf (errstr, sizeof(errstr),
+                          "subvols-per-directory(%d) is greater "
+                          "than the number of subvolumes(%d).",
+                          subvols, volinfo->subvol_count);
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s.", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+
+/* dispatch table for VOLUME SET
+ * -----------------------------
+ *
+ * Format of entries:
+ *
+ * First field is the <key>, for the purpose of looking it up
+ * in volume dictionary. Each <key> is of the format "<domain>.<specifier>".
+ *
+ * Second field is <voltype>.
+ *
+ * Third field is <option>, if its unset, it's assumed to be
+ * the same as <specifier>.
+ *
+ * Fourth field is <value>. In this context they are used to specify
+ * a default. That is, even the volume dict doesn't have a value,
+ * we procced as if the default value were set for it.
+ *
+ * Fifth field is <doctype>, which decides if the option is public and available
+ * in "set help" or not. "NO_DOC" entries are not part of the public interface
+ * and are subject to change at any time. This also decides if an option is
+ * global (apllies to all volumes) or normal (applies to only specified volume).
+ *
+ * Sixth field is <flags>.
+ *
+ * Seventh field is <op-version>.
+ *
+ * Eight field is description of option: If NULL, tried to fetch from
+ * translator code's xlator_options table.
+ *
+ * Nineth field is validation function: If NULL, xlator's option specific
+ * validation will be tried, otherwise tried at glusterd code itself.
+ *
+ * There are two type of entries: basic and special.
+ *
+ * - Basic entries are the ones where the <option> does _not_ start with
+ *   the bang! character ('!').
+ *
+ *   In their case, <option> is understood as an option for an xlator of
+ *   type <voltype>. Their effect is to copy over the volinfo->dict[<key>]
+ *   value to all graph nodes of type <voltype> (if such a value is set).
+ *
+ *   You are free to add entries of this type, they will become functional
+ *   just by being present in the table.
+ *
+ * - Special entries where the <option> starts with the bang!.
+ *
+ *   They are not applied to all graphs during generation, and you cannot
+ *   extend them in a trivial way which could be just picked up. Better
+ *   not touch them unless you know what you do.
+ *
+ *
+ * Another kind of grouping for options, according to visibility:
+ *
+ * - Exported: one which is used in the code. These are characterized by
+ *   being used a macro as <key> (of the format VKEY_..., defined in
+ *   glusterd-volgen.h
+ *
+ * - Non-exported: the rest; these have string literal <keys>.
+ *
+ * Adhering to this policy, option name changes shall be one-liners.
+ *
+ */
+
+struct volopt_map_entry glusterd_volopt_map[] = {
+        /* DHT xlator options */
+        { .key        = "cluster.lookup-unhashed",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.min-free-disk",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.min-free-inodes",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.rebalance-stats",
+          .voltype    = "cluster/distribute",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "cluster.subvols-per-directory",
+          .voltype     = "cluster/distribute",
+          .option      = "directory-layout-spread",
+          .op_version  = 2,
+          .validate_fn = validate_subvols_per_directory,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.readdir-optimize",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.rsync-hash-regex",
+          .voltype    = "cluster/distribute",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.extra-hash-regex",
+          .voltype    = "cluster/distribute",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.dht-xattr-name",
+          .voltype    = "cluster/distribute",
+          .option     = "xattr-name",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* NUFA xlator options (Distribute special case) */
+        { .key        = "cluster.nufa",
+          .voltype    = "cluster/distribute",
+          .option     = "!nufa",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.local-volume-name",
+          .voltype    = "cluster/nufa",
+          .option     = "local-volume-name",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Switch xlator options (Distribute special case) */
+        { .key        = "cluster.switch",
+          .voltype    = "cluster/distribute",
+          .option     = "!switch",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.switch-pattern",
+          .voltype    = "cluster/switch",
+          .option     = "pattern.switch.case",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* AFR xlator options */
+        { .key        = "cluster.entry-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-subvolume",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags     = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-subvolume-index",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-hash-mode",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.background-self-heal-count",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.metadata-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.entry-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key           = "cluster.self-heal-daemon",
+          .voltype       = "cluster/replicate",
+          .option        = "!self-heal-daemon",
+          .op_version    = 1
+        },
+        { .key        = "cluster.heal-timeout",
+          .voltype    = "cluster/replicate",
+          .option     = "!heal-timeout",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.strict-readdir",
+          .voltype    = "cluster/replicate",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.self-heal-window-size",
+          .voltype    = "cluster/replicate",
+          .option     = "data-self-heal-window-size",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.metadata-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-self-heal-algorithm",
+          .voltype    = "cluster/replicate",
+          .option     = "data-self-heal-algorithm",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.eager-lock",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.quorum-type",
+          .voltype    = "cluster/replicate",
+          .option     = "quorum-type",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.quorum-count",
+          .voltype    = "cluster/replicate",
+          .option     = "quorum-count",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.choose-local",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.self-heal-readdir-size",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.post-op-delay-secs",
+          .voltype    = "cluster/replicate",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.readdir-failover",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.ensure-durability",
+          .voltype    = "cluster/replicate",
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Stripe xlator options */
+        { .key         = "cluster.stripe-block-size",
+          .voltype     = "cluster/stripe",
+          .option      = "block-size",
+          .op_version  = 1,
+          .validate_fn = validate_stripe,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.stripe-coalesce",
+          .voltype    = "cluster/stripe",
+          .option     = "coalesce",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* IO-stats xlator options */
+        { .key         = VKEY_DIAG_LAT_MEASUREMENT,
+          .voltype     = "debug/io-stats",
+          .option      = "latency-measurement",
+          .value       = "off",
+          .op_version  = 1
+        },
+        { .key         = "diagnostics.dump-fd-stats",
+          .voltype     = "debug/io-stats",
+          .op_version  = 1
+        },
+        { .key         = VKEY_DIAG_CNT_FOP_HITS,
+          .voltype     = "debug/io-stats",
+          .option      = "count-fop-hits",
+          .value       = "off",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "diagnostics.brick-log-level",
+          .voltype     = "debug/io-stats",
+          .option      = "!brick-log-level",
+          .op_version  = 1
+        },
+        { .key        = "diagnostics.client-log-level",
+          .voltype    = "debug/io-stats",
+          .option     = "!client-log-level",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "diagnostics.brick-sys-log-level",
+          .voltype     = "debug/io-stats",
+          .option      = "!sys-log-level",
+          .op_version  = 1
+        },
+        { .key        = "diagnostics.client-sys-log-level",
+          .voltype    = "debug/io-stats",
+          .option     = "!sys-log-level",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* IO-cache xlator options */
+        { .key         = "performance.cache-max-file-size",
+          .voltype     = "performance/io-cache",
+          .option      = "max-file-size",
+          .op_version  = 1,
+          .validate_fn = validate_cache_max_min_size,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "performance.cache-min-file-size",
+          .voltype     = "performance/io-cache",
+          .option      = "min-file-size",
+          .op_version  = 1,
+          .validate_fn = validate_cache_max_min_size,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-refresh-timeout",
+          .voltype    = "performance/io-cache",
+          .option     = "cache-timeout",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-priority",
+          .voltype    = "performance/io-cache",
+          .option     = "priority",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-size",
+          .voltype    = "performance/io-cache",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* IO-threads xlator options */
+        { .key         = "performance.io-thread-count",
+          .voltype     = "performance/io-threads",
+          .option      = "thread-count",
+          .op_version  = 1
+        },
+        { .key         = "performance.high-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.normal-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.low-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.least-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.enable-least-priority",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.least-rate-limit",
+          .voltype     = "performance/io-threads",
+          .op_version  = 2
+        },
+
+        /* Other perf xlators' options */
+        { .key        = "performance.cache-size",
+          .voltype    = "performance/quick-read",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.flush-behind",
+          .voltype    = "performance/write-behind",
+          .option     = "flush-behind",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.write-behind-window-size",
+          .voltype    = "performance/write-behind",
+          .option     = "cache-size",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.strict-o-direct",
+          .voltype    = "performance/write-behind",
+          .option     = "strict-O_DIRECT",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.strict-write-ordering",
+          .voltype    = "performance/write-behind",
+          .option     = "strict-write-ordering",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.lazy-open",
+          .voltype    = "performance/open-behind",
+          .option     = "lazy-open",
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.read-ahead-page-count",
+          .voltype    = "performance/read-ahead",
+          .option     = "page-count",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.md-cache-timeout",
+          .voltype    = "performance/md-cache",
+          .option     = "md-cache-timeout",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+ 	/* Crypt xlator options */
+
+	{ .key         = "features.encryption",
+	  .voltype     = "encryption/crypt",
+	  .option      = "!feat",
+	  .value       = "off",
+	  .op_version  = 3,
+	  .description = "enable/disable client-side encryption for "
+                         "the volume.",
+	  .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+	},
+
+        { .key         = "encryption.master-key",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "encryption.data-key-size",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "encryption.block-size",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Client xlator options */
+        { .key        = "network.frame-timeout",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.ping-timeout",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.tcp-window-size",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "features.lock-heal",
+          .voltype    = "protocol/client",
+          .option     = "lk-heal",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "features.grace-timeout",
+          .voltype    = "protocol/client",
+          .option     = "grace-timeout",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "client.ssl",
+          .voltype    = "protocol/client",
+          .option     = "transport.socket.ssl-enabled",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.remote-dio",
+          .voltype    = "protocol/client",
+          .option     = "filter-O_DIRECT",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Server xlator options */
+        { .key         = "network.tcp-window-size",
+          .voltype     = "protocol/server",
+          .op_version  = 1
+        },
+        { .key         = "network.inode-lru-limit",
+          .voltype     = "protocol/server",
+          .op_version  = 1
+        },
+        { .key         = AUTH_ALLOW_MAP_KEY,
+          .voltype     = "protocol/server",
+          .option      = "!server-auth",
+          .value       = "*",
+          .op_version  = 1
+        },
+        { .key         = AUTH_REJECT_MAP_KEY,
+          .voltype     = "protocol/server",
+          .option      = "!server-auth",
+          .op_version  = 1
+        },
+        { .key         = "transport.keepalive",
+          .voltype     = "protocol/server",
+          .option      = "transport.socket.keepalive",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "server.allow-insecure",
+          .voltype     = "protocol/server",
+          .option      = "rpc-auth-allow-insecure",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "server.root-squash",
+          .voltype     = "protocol/server",
+          .option      = "root-squash",
+          .op_version  = 2
+        },
+        { .key         = "server.statedump-path",
+          .voltype     = "protocol/server",
+          .option      = "statedump-path",
+          .op_version  = 1
+        },
+        { .key         = "server.outstanding-rpc-limit",
+          .voltype     = "protocol/server",
+          .option      = "rpc.outstanding-rpc-limit",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "features.lock-heal",
+          .voltype     = "protocol/server",
+          .option      = "lk-heal",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "features.grace-timeout",
+          .voltype     = "protocol/server",
+          .option      = "grace-timeout",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "server.ssl",
+          .voltype     = "protocol/server",
+          .option      = "transport.socket.ssl-enabled",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+
+        /* Performance xlators enable/disbable options */
+        { .key         = "performance.write-behind",
+          .voltype     = "performance/write-behind",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable write-behind translator in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "performance.read-ahead",
+          .voltype     = "performance/read-ahead",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable read-ahead translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "performance.readdir-ahead",
+          .voltype     = "performance/readdir-ahead",
+          .option      = "!perf",
+          .value       = "off",
+          .op_version  = 3,
+          .description = "enable/disable readdir-ahead translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+
+        { .key         = "performance.io-cache",
+          .voltype     = "performance/io-cache",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable io-cache translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "performance.quick-read",
+          .voltype     = "performance/quick-read",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable quick-read translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+
+        },
+        { .key         = "performance.open-behind",
+          .voltype     = "performance/open-behind",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 2,
+          .description = "enable/disable open-behind translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+
+        },
+        { .key         = "performance.stat-prefetch",
+          .voltype     = "performance/md-cache",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable meta-data caching translator in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "performance.client-io-threads",
+          .voltype     = "performance/io-threads",
+          .option      = "!perf",
+          .value       = "off",
+          .op_version  = 1,
+          .description = "enable/disable io-threads translator in the client "
+                         "graph of volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.write-behind",
+          .voltype    = "performance/write-behind",
+          .option     = "!nfsperf",
+          .value      = "on",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.read-ahead",
+          .voltype    = "performance/read-ahead",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.io-cache",
+          .voltype    = "performance/io-cache",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.quick-read",
+          .voltype    = "performance/quick-read",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.stat-prefetch",
+          .voltype    = "performance/md-cache",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.io-threads",
+          .voltype    = "performance/io-threads",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.force-readdirp",
+          .voltype    = "performance/md-cache",
+          .option     = "force-readdirp",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+	/* Feature translators */
+        { .key         = "features.file-snapshot",
+          .voltype     = "features/qemu-block",
+          .option      = "!feat",
+          .value       = "off",
+          .op_version  = 3,
+          .description = "enable/disable file-snapshot feature in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+
+#ifdef HAVE_LIB_Z
+        /* Compressor-decompressor xlator options
+         * defaults used from xlator/feature/compress/src/cdc.h
+         */
+        { .key         = "features.compress",
+          .voltype     = "features/cdc",
+          .option      = "!compress",
+          .value       = "off",
+          .type        = NO_DOC,
+          .op_version  = 2,
+          .description = "enable/disable compression translator"
+        },
+        { .key         = "compress.mode",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.window-size",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.mem-level",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.min-size",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.compression-level",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.debug",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+ #endif
+
+        /* Quota xlator options */
+        { .key        = VKEY_FEATURES_LIMIT_USAGE,
+          .voltype    = "features/quota",
+          .option     = "limit-set",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "features.quota-timeout",
+          .voltype     = "features/quota",
+          .option      = "timeout",
+          .value       = "0",
+          .op_version  = 1,
+          .validate_fn = validate_quota,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "features.quota-deem-statfs",
+          .voltype     = "features/quota",
+          .option      = "deem-statfs",
+          .value       = "off",
+          .type        = DOC,
+          .op_version  = 3,
+          .validate_fn = validate_quota,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Marker xlator options */
+        { .key         = VKEY_MARKER_XTIME,
+          .voltype     = "features/marker",
+          .option      = "xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 1
+        },
+        { .key         = VKEY_MARKER_XTIME,
+          .voltype     = "features/marker",
+          .option      = "!xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 1
+        },
+        { .key         = VKEY_MARKER_XTIME_FORCE,
+          .voltype     = "features/marker",
+          .option      = "gsync-force-xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 2
+        },
+        { .key         = VKEY_MARKER_XTIME_FORCE,
+          .voltype     = "features/marker",
+          .option      = "!gsync-force-xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 2
+        },
+        { .key         = VKEY_FEATURES_QUOTA,
+          .voltype     = "features/marker",
+          .option      = "quota",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 1
+        },
+
+        /* Debug xlators options */
+        { .key        = "debug.trace",
+          .voltype    = "debug/trace",
+          .option     = "!debug",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "debug.log-history",
+          .voltype     = "debug/trace",
+          .option      = "log-history",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "debug.log-file",
+          .voltype     = "debug/trace",
+          .option      = "log-file",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "debug.exclude-ops",
+          .voltype     = "debug/trace",
+          .option      = "exclude-ops",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "debug.include-ops",
+          .voltype     = "debug/trace",
+          .option      = "include-ops",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key        = "debug.error-gen",
+          .voltype    = "debug/error-gen",
+          .option     = "!debug",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "debug.error-failure",
+          .voltype     = "debug/error-gen",
+          .option      = "failure",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "debug.error-number",
+          .voltype     = "debug/error-gen",
+          .option      = "error-no",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "debug.random-failure",
+          .voltype     = "debug/error-gen",
+          .option      = "random-failure",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "debug.error-fops",
+          .voltype     = "debug/error-gen",
+          .option      = "enable",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+
+
+        /* NFS xlator options */
+        { .key         = "nfs.enable-ino32",
+          .voltype     = "nfs/server",
+          .option      = "nfs.enable-ino32",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.mem-factor",
+          .voltype     = "nfs/server",
+          .option      = "nfs.mem-factor",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.export-dirs",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.export-dirs",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.export-volumes",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.export-volumes",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.addr-namelookup",
+          .voltype     = "nfs/server",
+          .option      = "rpc-auth.addr.namelookup",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.dynamic-volumes",
+          .voltype     = "nfs/server",
+          .option      = "nfs.dynamic-volumes",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.register-with-portmap",
+          .voltype     = "nfs/server",
+          .option      = "rpc.register-with-portmap",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.outstanding-rpc-limit",
+          .voltype     = "nfs/server",
+          .option      = "rpc.outstanding-rpc-limit",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.port",
+          .voltype     = "nfs/server",
+          .option      = "nfs.port",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-unix",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.auth-unix.*",
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-null",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.auth-null.*",
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-allow",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.addr.*.allow",
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-reject",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.addr.*.reject",
+          .op_version  = 1
+        },
+        { .key         = "nfs.ports-insecure",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.ports.*.insecure",
+          .op_version  = 1
+        },
+        { .key         = "nfs.transport-type",
+          .voltype     = "nfs/server",
+          .option      = "!nfs.transport-type",
+          .value       = "tcp",
+          .op_version  = 1,
+          .description = "Specifies the nfs transport type. Valid "
+                         "transport types are 'tcp' and 'rdma'."
+        },
+        { .key         = "nfs.trusted-sync",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.trusted-sync",
+          .op_version  = 1
+        },
+        { .key         = "nfs.trusted-write",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.trusted-write",
+          .op_version  = 1
+        },
+        { .key         = "nfs.volume-access",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.volume-access",
+          .op_version  = 1
+        },
+        { .key         = "nfs.export-dir",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.export-dir",
+          .op_version  = 1
+        },
+        { .key         = NFS_DISABLE_MAP_KEY,
+          .voltype     = "nfs/server",
+          .option      = "!nfs-disable",
+          .op_version  = 1
+        },
+        { .key         = "nfs.nlm",
+          .voltype     = "nfs/server",
+          .option      = "nfs.nlm",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.acl",
+          .voltype     = "nfs/server",
+          .option      = "nfs.acl",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.mount-udp",
+          .voltype     = "nfs/server",
+          .option      = "nfs.mount-udp",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.mount-rmtab",
+          .voltype     = "nfs/server",
+          .option      = "nfs.mount-rmtab",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.server-aux-gids",
+          .voltype     = "nfs/server",
+          .option      = "nfs.server-aux-gids",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "nfs.drc",
+          .voltype     = "nfs/server",
+          .option      = "nfs.drc",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.drc-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs.drc-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.read-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.read-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.write-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.write-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.readdir-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.readdir-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+
+        /* Other options which don't fit any place above */
+        { .key        = "features.read-only",
+          .voltype    = "features/read-only",
+          .option     = "!read-only",
+          .value      = "off",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "features.worm",
+          .voltype    = "features/worm",
+          .option     = "!worm",
+          .value      = "off",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "storage.linux-aio",
+          .voltype     = "storage/posix",
+          .op_version  = 1
+        },
+        { .key         = "storage.batch-fsync-mode",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.batch-fsync-delay-usec",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.owner-uid",
+          .voltype     = "storage/posix",
+          .option      = "brick-uid",
+          .op_version  = 1
+        },
+        { .key         = "storage.owner-gid",
+          .voltype     = "storage/posix",
+          .option      = "brick-gid",
+          .op_version  = 1
+        },
+        { .key         = "storage.node-uuid-pathinfo",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.health-check-interval",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.bd-aio",
+          .voltype     = "storage/bd",
+          .op_version  = 3
+        },
+        { .key        = "config.memory-accounting",
+          .voltype    = "configuration",
+          .option     = "!config",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "config.transport",
+          .voltype     = "configuration",
+          .option      = "!config",
+          .op_version  = 2
+        },
+        { .key         = GLUSTERD_QUORUM_TYPE_KEY,
+          .voltype     = "mgmt/glusterd",
+          .value       = "off",
+          .op_version  = 2
+        },
+        { .key         = GLUSTERD_QUORUM_RATIO_KEY,
+          .voltype     = "mgmt/glusterd",
+          .value       = "0",
+          .op_version  = 2
+        },
+        /* changelog translator - global tunables */
+        { .key         = "changelog.changelog",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "changelog.changelog-dir",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "changelog.encoding",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "changelog.rollover-time",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "changelog.fsync-interval",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = NULL
+        }
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index e9f3bd055..59288ada0 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -44,20 +34,29 @@
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-store.h"
+#include "glusterd-hooks.h"
 #include "glusterd-utils.h"
+#include "glusterd-locks.h"
 #include "common-utils.h"
 #include "run.h"
 
+#include "syncop.h"
+
 #include "glusterd-mountbroker.h"
 
-static uuid_t glusterd_uuid;
 extern struct rpcsvc_program gluster_handshake_prog;
+extern struct rpcsvc_program gluster_cli_getspec_prog;
 extern struct rpcsvc_program gluster_pmap_prog;
 extern glusterd_op_info_t opinfo;
 extern struct rpcsvc_program gd_svc_mgmt_prog;
+extern struct rpcsvc_program gd_svc_mgmt_v3_prog;
 extern struct rpcsvc_program gd_svc_peer_prog;
 extern struct rpcsvc_program gd_svc_cli_prog;
+extern struct rpcsvc_program gd_svc_cli_prog_ro;
 extern struct rpc_clnt_program gd_brick_prog;
+extern struct rpcsvc_program glusterd_mgmt_hndsk_prog;
+
+extern char snap_mount_folder[PATH_MAX];
 
 rpcsvc_cbk_program_t glusterd_cbk_prog = {
         .progname  = "Gluster Callback",
@@ -65,6 +64,57 @@ rpcsvc_cbk_program_t glusterd_cbk_prog = {
         .progver   = GLUSTER_CBK_VERSION,
 };
 
+struct rpcsvc_program *gd_inet_programs[] = {
+        &gd_svc_peer_prog,
+        &gd_svc_cli_prog_ro,
+        &gd_svc_mgmt_prog,
+        &gd_svc_mgmt_v3_prog,
+        &gluster_pmap_prog,
+        &gluster_handshake_prog,
+        &glusterd_mgmt_hndsk_prog,
+};
+int gd_inet_programs_count = (sizeof (gd_inet_programs) /
+                              sizeof (gd_inet_programs[0]));
+
+struct rpcsvc_program *gd_uds_programs[] = {
+        &gd_svc_cli_prog,
+        &gluster_cli_getspec_prog,
+};
+int gd_uds_programs_count = (sizeof (gd_uds_programs) /
+                             sizeof (gd_uds_programs[0]));
+
+const char *gd_op_list[GD_OP_MAX + 1] = {
+        [GD_OP_NONE]                    = "Invalid op",
+        [GD_OP_CREATE_VOLUME]           = "Create",
+        [GD_OP_START_BRICK]             = "Start Brick",
+        [GD_OP_STOP_BRICK]              = "Stop Brick",
+        [GD_OP_DELETE_VOLUME]           = "Delete",
+        [GD_OP_START_VOLUME]            = "Start",
+        [GD_OP_STOP_VOLUME]             = "Stop",
+        [GD_OP_DEFRAG_VOLUME]           = "Rebalance",
+        [GD_OP_ADD_BRICK]               = "Add brick",
+        [GD_OP_REMOVE_BRICK]            = "Remove brick",
+        [GD_OP_REPLACE_BRICK]           = "Replace brick",
+        [GD_OP_SET_VOLUME]              = "Set",
+        [GD_OP_RESET_VOLUME]            = "Reset",
+        [GD_OP_SYNC_VOLUME]             = "Sync",
+        [GD_OP_LOG_ROTATE]              = "Log rotate",
+        [GD_OP_GSYNC_SET]               = "Geo-replication",
+        [GD_OP_PROFILE_VOLUME]          = "Profile",
+        [GD_OP_QUOTA]                   = "Quota",
+        [GD_OP_STATUS_VOLUME]           = "Status",
+        [GD_OP_REBALANCE]               = "Rebalance",
+        [GD_OP_HEAL_VOLUME]             = "Heal",
+        [GD_OP_STATEDUMP_VOLUME]        = "Statedump",
+        [GD_OP_LIST_VOLUME]             = "Lists",
+        [GD_OP_CLEARLOCKS_VOLUME]       = "Clear locks",
+        [GD_OP_DEFRAG_BRICK_VOLUME]     = "Rebalance",
+        [GD_OP_COPY_FILE]               = "Copy File",
+        [GD_OP_SYS_EXEC]                = "Execute system commands",
+        [GD_OP_GSYNC_CREATE]            = "Geo-replication Create",
+        [GD_OP_SNAP]                    = "Snapshot",
+        [GD_OP_MAX]                     = "Invalid op"
+};
 
 static int
 glusterd_opinfo_init ()
@@ -76,35 +126,30 @@ glusterd_opinfo_init ()
         return ret;
 }
 
-static int
-glusterd_uuid_init (int flag)
+
+int
+glusterd_uuid_init ()
 {
         int             ret = -1;
+        xlator_t        *this = NULL;
         glusterd_conf_t *priv = NULL;
 
-        priv = THIS->private;
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
 
-        if (!flag) {
-                ret = glusterd_retrieve_uuid ();
-                if (!ret) {
-                        uuid_copy (glusterd_uuid, priv->uuid);
-                        gf_log ("glusterd", GF_LOG_INFO,
-                                "retrieved UUID: %s", uuid_utoa (priv->uuid));
-                        return 0;
-                }
+        ret = glusterd_retrieve_uuid ();
+        if (ret == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "retrieved UUID: %s", uuid_utoa (priv->uuid));
+                return 0;
         }
 
-        uuid_generate (glusterd_uuid);
-
-        gf_log ("glusterd", GF_LOG_INFO,
-                        "generated UUID: %s", uuid_utoa (glusterd_uuid));
-        uuid_copy (priv->uuid, glusterd_uuid);
-
-        ret = glusterd_store_uuid ();
+        ret = glusterd_uuid_generate_save ();
 
         if (ret) {
                 gf_log ("glusterd", GF_LOG_ERROR,
-                          "Unable to store generated UUID");
+                          "Unable to generate and save new UUID");
                 return ret;
         }
 
@@ -112,6 +157,63 @@ glusterd_uuid_init (int flag)
 }
 
 int
+glusterd_uuid_generate_save ()
+{
+        int               ret = -1;
+        glusterd_conf_t   *priv = NULL;
+        xlator_t          *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        uuid_generate (priv->uuid);
+
+        gf_log (this->name, GF_LOG_INFO, "generated UUID: %s",
+                uuid_utoa (priv->uuid));
+
+        ret = glusterd_store_global_info (this);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to store the generated uuid %s",
+                        uuid_utoa (priv->uuid));
+
+        return ret;
+}
+
+int
+glusterd_options_init (xlator_t *this)
+{
+        int             ret = -1;
+        glusterd_conf_t *priv = NULL;
+        char            *initial_version = "0";
+
+        priv = this->private;
+
+        priv->opts = dict_new ();
+        if (!priv->opts)
+                goto out;
+
+        ret = glusterd_store_retrieve_options (this);
+        if (ret == 0)
+                goto out;
+
+        ret = dict_set_str (priv->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                            initial_version);
+        if (ret)
+                goto out;
+        ret = glusterd_store_options (this, priv->opts);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to store version");
+                return ret;
+        }
+out:
+
+        return 0;
+}
+int
 glusterd_fetchspec_notify (xlator_t *this)
 {
         int              ret   = -1;
@@ -120,10 +222,15 @@ glusterd_fetchspec_notify (xlator_t *this)
 
         priv = this->private;
 
-        list_for_each_entry (trans, &priv->xprt_list, list) {
-                rpcsvc_callback_submit (priv->rpc, trans, &glusterd_cbk_prog,
-                                        GF_CBK_FETCHSPEC, NULL, 0);
+        pthread_mutex_lock (&priv->xprt_lock);
+        {
+                list_for_each_entry (trans, &priv->xprt_list, list) {
+                        rpcsvc_callback_submit (priv->rpc, trans,
+                                                &glusterd_cbk_prog,
+                                                GF_CBK_FETCHSPEC, NULL, 0);
+                }
         }
+        pthread_mutex_unlock (&priv->xprt_lock);
 
         ret = 0;
 
@@ -181,12 +288,16 @@ glusterd_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
         {
                 INIT_LIST_HEAD (&xprt->list);
 
+                pthread_mutex_lock (&priv->xprt_lock);
                 list_add_tail (&xprt->list, &priv->xprt_list);
+                pthread_mutex_unlock (&priv->xprt_lock);
                 break;
         }
         case RPCSVC_EVENT_DISCONNECT:
         {
+                pthread_mutex_lock (&priv->xprt_lock);
                 list_del (&xprt->list);
+                pthread_mutex_unlock (&priv->xprt_lock);
                 pmap_registry_remove (this, 0, NULL, GF_PMAP_PORT_NONE, xprt);
                 break;
         }
@@ -243,18 +354,6 @@ out:
         return ret;
 }
 
-/* defined in usterd-utils.c -- no
- * glusterd header where it would be
- * appropriate to put to, and too
- * accidental routine to place in
- * libglusterfs.
- *
- * (Indeed, XXX: we'd rather need a general
- * "mkdir -p" like routine in
- * libglusterfs)
- */
-extern int mkdir_if_missing (char *path);
-
 #if SYNCDAEMON_COMPILE
 static int
 glusterd_check_gsync_present (int *valid_state)
@@ -347,13 +446,13 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
         if (strlen (conf->workdir)+2 > PATH_MAX-strlen(GEOREP)) {
                 ret = -1;
                 gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" directory %s",
-                        georepdir);
+                        "directory path %s/"GEOREP" is longer than PATH_MAX",
+                        conf->workdir);
                 goto out;
         }
 
         snprintf (georepdir, PATH_MAX, "%s/"GEOREP, conf->workdir);
-        ret = mkdir_if_missing (georepdir);
+        ret = mkdir_p (georepdir, 0777, _gf_true);
         if (-1 == ret) {
                 gf_log ("glusterd", GF_LOG_CRITICAL,
                         "Unable to create "GEOREP" directory %s",
@@ -364,31 +463,49 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
         if (strlen (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP) >= PATH_MAX) {
                 ret = -1;
                 gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" directory %s",
-                        georepdir);
+                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"
+                        GEOREP" is longer than PATH_MAX");
                 goto out;
         }
-        ret = mkdir_if_missing (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP);
+        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP, 0777, _gf_true);
         if (-1 == ret) {
                 gf_log ("glusterd", GF_LOG_CRITICAL,
                         "Unable to create "GEOREP" log directory");
                 goto out;
         }
 
+        /* Slave log file directory */
         if (strlen(DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves") >= PATH_MAX) {
                 ret = -1;
                 gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" directory %s",
-                        georepdir);
+                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"
+                        GEOREP"-slaves"" is longer than PATH_MAX");
                 goto out;
         }
-        ret = mkdir_if_missing (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves");
+        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves", 0777,
+                      _gf_true);
         if (-1 == ret) {
                 gf_log ("glusterd", GF_LOG_CRITICAL,
                         "Unable to create "GEOREP" slave log directory");
                 goto out;
         }
 
+        /* MountBroker log file directory */
+        if (strlen(DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr") >= PATH_MAX) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP
+                        "-slaves/mbr"" is longer than PATH_MAX");
+                goto out;
+        }
+        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr", 0777,
+                      _gf_true);
+        if (-1 == ret) {
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Unable to create "GEOREP" mountbroker slave log directory");
+                goto out;
+        }
+
         ret = dict_get_str (THIS->options, GEOREP"-log-group", &greplg_s);
         if (ret)
                 ret = 0;
@@ -406,20 +523,22 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
                 if (ret == 0)
                         ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
                                                  GEOREP"-slaves", gr->gr_gid);
+                if (ret == 0)
+                        ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
+                                                 GEOREP"-slaves/mbr", gr->gr_gid);
         }
 
  out:
         gf_log("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
-#endif
 
 static void
 runinit_gsyncd_setrx (runner_t *runner, glusterd_conf_t *conf)
 {
         runinit (runner);
         runner_add_args (runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (runner, "%s/"GSYNC_CONF,conf->workdir);
+        runner_argprintf (runner, "%s/"GSYNC_CONF_TEMPLATE, conf->workdir);
         runner_add_arg (runner, "--config-set-rx");
 }
 
@@ -436,7 +555,6 @@ configure_syncdaemon (glusterd_conf_t *conf)
 } while (0)
 {
         int ret = 0;
-#if SYNCDAEMON_COMPILE
         runner_t runner = {0,};
         char georepdir[PATH_MAX] = {0,};
         int valid_state = 0;
@@ -469,8 +587,8 @@ configure_syncdaemon (glusterd_conf_t *conf)
         RUN_GSYNCD_CMD;
 
         runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "remote-gsyncd",
-                         "/usr/local/libexec/glusterfs/gsyncd", ".", "^ssh:", NULL);
+        runner_add_args (&runner, "remote-gsyncd", "/nonexistent/gsyncd",
+                         ".", "^ssh:", NULL);
         RUN_GSYNCD_CMD;
 
         /* gluster-command-dir */
@@ -482,7 +600,7 @@ configure_syncdaemon (glusterd_conf_t *conf)
         /* gluster-params */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_args (&runner, "gluster-params",
-                         "xlator-option=*-dht.assert-no-child-down=true",
+                         "aux-gfid-mount xlator-option=*-dht.assert-no-child-down=true",
                          ".", ".", NULL);
         RUN_GSYNCD_CMD;
 
@@ -499,17 +617,45 @@ configure_syncdaemon (glusterd_conf_t *conf)
         /* pid-file */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_arg (&runner, "pid-file");
-        runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.pid", georepdir);
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.pid", georepdir);
         runner_add_args (&runner, ".", ".", NULL);
         RUN_GSYNCD_CMD;
 
         /* state-file */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_arg (&runner, "state-file");
-        runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.status", georepdir);
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.status", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-detail-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "state-detail-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+                          georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-detail-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "state-detail-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+                          georepdir);
         runner_add_args (&runner, ".", ".", NULL);
         RUN_GSYNCD_CMD;
 
+        /* state-socket */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "state-socket-unencoded");
+        runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.socket", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* socketdir */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
         /* log-file */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_args (&runner,
@@ -522,10 +668,32 @@ configure_syncdaemon (glusterd_conf_t *conf)
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_args (&runner,
                          "gluster-log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.gluster.log",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}.gluster.log",
                          ".", ".", NULL);
         RUN_GSYNCD_CMD;
 
+        /* ignore-deletes */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "ignore-deletes", "true", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* special-sync-mode */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "special-sync-mode", "partial", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* change-detector == changelog */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg(&runner, "working-dir");
+        runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
+                         DEFAULT_VAR_RUN_DIRECTORY);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
         /************
          * slave pre-configuration
          ************/
@@ -539,7 +707,7 @@ configure_syncdaemon (glusterd_conf_t *conf)
         /* gluster-params */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_args (&runner, "gluster-params",
-                         "xlator-option=*-dht.assert-no-child-down=true",
+                         "aux-gfid-mount xlator-option=*-dht.assert-no-child-down=true",
                          ".", NULL);
         RUN_GSYNCD_CMD;
 
@@ -551,6 +719,14 @@ configure_syncdaemon (glusterd_conf_t *conf)
                          ".", NULL);
         RUN_GSYNCD_CMD;
 
+        /* MountBroker log-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner,
+                         "log-file-mbr",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr/${session_owner}:${eSlave}.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
         /* gluster-log-file */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_args (&runner,
@@ -560,12 +736,17 @@ configure_syncdaemon (glusterd_conf_t *conf)
         RUN_GSYNCD_CMD;
 
  out:
-#else
-        (void)conf;
-#endif
         return ret ? -1 : 0;
 }
 #undef RUN_GSYNCD_CMD
+#else /* SYNCDAEMON_COMPILE */
+static int
+configure_syncdaemon (glusterd_conf_t *conf)
+{
+        return 0;
+}
+#endif /* !SYNCDAEMON_COMPILE */
+
 
 static int
 check_prepare_mountbroker_root (char *mountbroker_root)
@@ -597,6 +778,11 @@ check_prepare_mountbroker_root (char *mountbroker_root)
                 ret = -1;
                 goto out;
         }
+        if (!(st.st_mode & (S_IXGRP|S_IXOTH))) {
+                gf_log ("", GF_LOG_WARNING,
+                        "permissions on mountbroker-root directory %s are "
+                        "probably too strict", mountbroker_root);
+        }
 
         dfd0 = dup (dfd);
 
@@ -625,6 +811,11 @@ check_prepare_mountbroker_root (char *mountbroker_root)
                         ret = -1;
                         goto out;
                 }
+                if (!(st.st_mode & (S_IXGRP|S_IXOTH))) {
+                        gf_log ("", GF_LOG_WARNING,
+                                "permissions on ancestors of mountbroker-root "
+                                "directory are probably too strict");
+                }
 
                 close (dfd);
                 dfd = dfd2;
@@ -647,14 +838,17 @@ check_prepare_mountbroker_root (char *mountbroker_root)
         ret = 0;
 
  out:
-        close (dfd0);
-        close (dfd);
-        close (dfd2);
+        if (dfd0 != -1)
+                close (dfd0);
+        if (dfd != -1)
+                close (dfd);
+        if (dfd2 != -1)
+                close (dfd2);
 
         return ret;
 }
 
-static void
+static int
 _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
 {
         glusterd_conf_t *priv           = THIS->private;
@@ -663,15 +857,11 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
         gf_boolean_t     ghadoop        = _gf_false;
         char            *pdesc          = value->data;
         char            *volname        = NULL;
-        int             *ret            = data;
         int              rv             = 0;
         gf_mount_spec_t *mspec          = NULL;
         char            *user           = NULL;
         char            *volfile_server = NULL;
 
-        if (*ret == -1)
-                return;
-
         label = strtail (key, "mountbroker.");
 
         /* check for presence of geo-rep/hadoop label */
@@ -679,14 +869,15 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
                 label = strtail (key, "mountbroker-"GEOREP".");
                 if (label)
                         georep = _gf_true;
-
-                label = strtail (key, "mountbroker-"GHADOOP".");
-                if (label)
-                        ghadoop = _gf_true;
+                else {
+                        label = strtail (key, "mountbroker-"GHADOOP".");
+                        if (label)
+                                ghadoop = _gf_true;
+                }
         }
 
         if (!label)
-                return;
+                return 0;
 
         mspec = GF_CALLOC (1, sizeof (*mspec), gf_gld_mt_mount_spec);
         if (!mspec)
@@ -725,14 +916,222 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
 
         list_add_tail (&mspec->speclist, &priv->mount_specs);
 
-        return;
+        return 0;
  err:
 
         gf_log ("", GF_LOG_ERROR,
                 "adding %smount spec failed: label: %s desc: %s",
                 georep ? GEOREP" " : "", label, pdesc);
 
-        *ret = -1;
+        return -1;
+}
+
+
+static int
+gd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+        glusterd_conf_t     *priv = THIS->private;
+        synclock_unlock (&priv->big_lock);
+        return ret;
+}
+
+static void
+glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
+{
+        xlator_t        *this = NULL;
+        glusterd_conf_t *priv = NULL;
+        int             ret   = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        synclock_lock (&priv->big_lock);
+        ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
+                            opaque);
+        if (ret)
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn bricks"
+                        " and other volume related services");
+}
+
+int
+glusterd_uds_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                            void *data)
+{
+        /* glusterd_rpcsvc_notify() does stuff that calls coming in from the
+         * unix domain socket don't need. This is just an empty function to be
+         * used for the uds listener. This will be used later if required.
+         */
+        return 0;
+}
+
+/* The glusterd unix domain socket listener only listens for cli */
+rpcsvc_t *
+glusterd_init_uds_listener (xlator_t *this)
+{
+        int             ret = -1;
+        dict_t          *options = NULL;
+        rpcsvc_t        *rpc = NULL;
+        data_t          *sock_data = NULL;
+        char            sockfile[PATH_MAX+1] = {0,};
+        int             i = 0;
+
+
+        GF_ASSERT (this);
+
+        sock_data = dict_get (this->options, "glusterd-sockfile");
+        if (!sock_data) {
+                strncpy (sockfile, DEFAULT_GLUSTERD_SOCKFILE, PATH_MAX);
+        } else {
+                strncpy (sockfile, sock_data->data, PATH_MAX);
+        }
+
+        options = dict_new ();
+        if (!options)
+                goto out;
+
+        ret = rpcsvc_transport_unix_options_build (&options, sockfile);
+        if (ret)
+                goto out;
+
+        rpc = rpcsvc_init (this, this->ctx, options, 8);
+        if (rpc == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = rpcsvc_register_notify (rpc, glusterd_uds_rpcsvc_notify,
+                                      this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Failed to register notify function");
+                goto out;
+        }
+
+        ret = rpcsvc_create_listeners (rpc, options, this->name);
+        if (ret != 1) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to create listener");
+                goto out;
+        }
+        ret = 0;
+
+        for (i = 0; i < gd_uds_programs_count; i++) {
+                ret = glusterd_program_register (this, rpc, gd_uds_programs[i]);
+                if (ret) {
+                        i--;
+                        for (; i >= 0; i--)
+                                rpcsvc_program_unregister (rpc,
+                                                           gd_uds_programs[i]);
+
+                        goto out;
+                }
+        }
+
+out:
+        if (options)
+                dict_unref (options);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to start glusterd "
+                        "unix domain socket listener.");
+                if (rpc) {
+                        GF_FREE (rpc);
+                        rpc = NULL;
+                }
+        }
+        return rpc;
+}
+
+void
+glusterd_stop_uds_listener (xlator_t *this)
+{
+        glusterd_conf_t         *conf = NULL;
+        rpcsvc_listener_t       *listener = NULL;
+        rpcsvc_listener_t       *next = NULL;
+
+        GF_ASSERT (this);
+        conf = this->private;
+
+        (void) rpcsvc_program_unregister (conf->uds_rpc, &gd_svc_cli_prog);
+        (void) rpcsvc_program_unregister (conf->uds_rpc, &gluster_handshake_prog);
+
+        list_for_each_entry_safe (listener, next, &conf->uds_rpc->listeners,
+                                  list) {
+                rpcsvc_listener_destroy (listener);
+        }
+
+        (void) rpcsvc_unregister_notify (conf->uds_rpc, glusterd_rpcsvc_notify,
+                                         this);
+
+        unlink (DEFAULT_GLUSTERD_SOCKFILE);
+
+        GF_FREE (conf->uds_rpc);
+        conf->uds_rpc = NULL;
+
+        return;
+}
+
+static int
+glusterd_init_snap_folder (xlator_t *this)
+{
+        int             ret = -1;
+        struct stat     buf = {0,};
+
+        GF_ASSERT (this);
+
+        /* Snapshot volumes are mounted under /var/run/gluster/snaps folder.
+         * But /var/run is normally a symbolic link to /run folder, which
+         * creates problems as the entry point in the mtab for the mount point
+         * and glusterd maintained entry point will be different. Therefore
+         * identify the correct run folder and use it for snap volume mounting.
+         */
+        ret = lstat (GLUSTERD_VAR_RUN_DIR, &buf);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat fails on %s, exiting. (errno = %d)",
+                        GLUSTERD_VAR_RUN_DIR, errno);
+                goto out;
+        }
+
+        /* If /var/run is symlink then use /run folder */
+        if (S_ISLNK (buf.st_mode)) {
+                strcpy (snap_mount_folder, GLUSTERD_RUN_DIR);
+        } else {
+                strcpy (snap_mount_folder, GLUSTERD_VAR_RUN_DIR);
+        }
+
+        strcat (snap_mount_folder, GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);
+
+        ret = stat (snap_mount_folder, &buf);
+        if ((ret != 0) && (ENOENT != errno)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat fails on %s, exiting. (errno = %d)",
+                        snap_mount_folder, errno);
+                ret = -1;
+                goto out;
+        }
+
+        if ((!ret) && (!S_ISDIR(buf.st_mode))) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Provided snap path %s is not a directory,"
+                        "exiting", snap_mount_folder);
+                ret = -1;
+                goto out;
+        }
+
+        if ((-1 == ret) && (ENOENT == errno)) {
+                /* Create missing folders */
+                ret = mkdir_p (snap_mount_folder, 0777, _gf_false);
+
+                if (-1 == ret) {
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to create directory %s"
+                                " ,errno = %d", snap_mount_folder, errno);
+                        goto out;
+                }
+        }
+
+out:
+        return ret;
 }
 
 /*
@@ -746,57 +1145,67 @@ init (xlator_t *this)
 {
         int32_t            ret               = -1;
         rpcsvc_t          *rpc               = NULL;
+        rpcsvc_t          *uds_rpc           = NULL;
         glusterd_conf_t   *conf              = NULL;
         data_t            *dir_data          = NULL;
         struct stat        buf               = {0,};
-        char               voldir [PATH_MAX] = {0,};
-        char               dirname [PATH_MAX];
+        char               storedir [PATH_MAX] = {0,};
+        char               workdir [PATH_MAX] = {0,};
+        char               hooks_dir [PATH_MAX] = {0,};
         char               cmd_log_filename [PATH_MAX] = {0,};
         int                first_time        = 0;
         char              *mountbroker_root  = NULL;
-
-#ifdef DEBUG
+        int                i                 = 0;
         char              *valgrind_str      = NULL;
-#endif
+
         dir_data = dict_get (this->options, "working-directory");
 
         if (!dir_data) {
                 //Use default working dir
-                strncpy (dirname, GLUSTERD_DEFAULT_WORKDIR, PATH_MAX);
+                strncpy (workdir, GLUSTERD_DEFAULT_WORKDIR, PATH_MAX);
         } else {
-                strncpy (dirname, dir_data->data, PATH_MAX);
+                strncpy (workdir, dir_data->data, PATH_MAX);
         }
 
-        ret = stat (dirname, &buf);
+        ret = stat (workdir, &buf);
         if ((ret != 0) && (ENOENT != errno)) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "stat fails on %s, exiting. (errno = %d)",
-			dirname, errno);
+                        workdir, errno);
                 exit (1);
         }
 
         if ((!ret) && (!S_ISDIR(buf.st_mode))) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Provided working area %s is not a directory,"
-                        "exiting", dirname);
+                        "exiting", workdir);
                 exit (1);
         }
 
 
         if ((-1 == ret) && (ENOENT == errno)) {
-                ret = mkdir (dirname, 0777);
+                ret = mkdir (workdir, 0777);
 
                 if (-1 == ret) {
                         gf_log (this->name, GF_LOG_CRITICAL,
                                 "Unable to create directory %s"
-                                " ,errno = %d", dirname, errno);
+                                " ,errno = %d", workdir, errno);
                         exit (1);
                 }
+
                 first_time = 1;
         }
 
+        setenv ("GLUSTERD_WORKING_DIR", workdir, 1);
         gf_log (this->name, GF_LOG_INFO, "Using %s as working directory",
-                dirname);
+                workdir);
+
+        ret = glusterd_init_snap_folder (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Unable to create "
+                        "snap backend folder");
+                 exit (1);
+         }
 
         snprintf (cmd_log_filename, PATH_MAX,"%s/.cmd_log_history",
                   DEFAULT_LOG_FILE_DIRECTORY);
@@ -808,59 +1217,68 @@ init (xlator_t *this)
                 exit (1);
         }
 
-        snprintf (voldir, PATH_MAX, "%s/vols", dirname);
+        snprintf (storedir, PATH_MAX, "%s/vols", workdir);
 
-        ret = mkdir (voldir, 0777);
+        ret = mkdir (storedir, 0777);
 
         if ((-1 == ret) && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Unable to create volume directory %s"
-                        " ,errno = %d", voldir, errno);
+                        " ,errno = %d", storedir, errno);
                 exit (1);
         }
 
-        snprintf (voldir, PATH_MAX, "%s/peers", dirname);
+        snprintf (storedir, PATH_MAX, "%s/peers", workdir);
 
-        ret = mkdir (voldir, 0777);
+        ret = mkdir (storedir, 0777);
 
         if ((-1 == ret) && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Unable to create peers directory %s"
-                        " ,errno = %d", voldir, errno);
+                        " ,errno = %d", storedir, errno);
                 exit (1);
         }
 
-        snprintf (voldir, PATH_MAX, "%s/bricks", DEFAULT_LOG_FILE_DIRECTORY);
-        ret = mkdir (voldir, 0777);
+        snprintf (storedir, PATH_MAX, "%s/bricks", DEFAULT_LOG_FILE_DIRECTORY);
+        ret = mkdir (storedir, 0777);
         if ((-1 == ret) && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Unable to create logs directory %s"
-                        " ,errno = %d", voldir, errno);
+                        " ,errno = %d", storedir, errno);
                 exit (1);
         }
 
-        snprintf (voldir, PATH_MAX, "%s/nfs", dirname);
-        ret = mkdir (voldir, 0777);
+        snprintf (storedir, PATH_MAX, "%s/nfs", workdir);
+        ret = mkdir (storedir, 0777);
         if ((-1 == ret) && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Unable to create nfs directory %s"
-                        " ,errno = %d", voldir, errno);
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
+        snprintf (storedir, PATH_MAX, "%s/glustershd", workdir);
+        ret = mkdir (storedir, 0777);
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create glustershd directory %s"
+                        " ,errno = %d", storedir, errno);
                 exit (1);
         }
 
-        snprintf (voldir, PATH_MAX, "%s/glustershd", dirname);
-        ret = mkdir (voldir, 0777);
+        snprintf (storedir, PATH_MAX, "%s/groups", workdir);
+        ret = mkdir (storedir, 0777);
         if ((-1 == ret) && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Unable to create glustershd directory %s"
-                        " ,errno = %d", voldir, errno);
+                        " ,errno = %d", storedir, errno);
                 exit (1);
         }
 
         ret = glusterd_rpcsvc_options_build (this->options);
         if (ret)
                 goto out;
-        rpc = rpcsvc_init (this, this->ctx, this->options);
+        rpc = rpcsvc_init (this, this->ctx, this->options, 64);
         if (rpc == NULL) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to init rpc");
@@ -886,38 +1304,26 @@ init (xlator_t *this)
                 goto out;
         }
 
-        ret = glusterd_program_register (this, rpc, &gd_svc_peer_prog);
-        if (ret) {
-                goto out;
-        }
-
-        ret = glusterd_program_register (this, rpc, &gd_svc_cli_prog);
-        if (ret) {
-                rpcsvc_program_unregister (rpc, &gd_svc_peer_prog);
-                goto out;
-        }
-
-        ret = glusterd_program_register (this, rpc, &gd_svc_mgmt_prog);
-        if (ret) {
-                rpcsvc_program_unregister (rpc, &gd_svc_peer_prog);
-                rpcsvc_program_unregister (rpc, &gd_svc_cli_prog);
-                goto out;
-        }
+        for (i = 0; i < gd_inet_programs_count; i++) {
+                ret = glusterd_program_register (this, rpc,
+                                                 gd_inet_programs[i]);
+                if (ret) {
+                        i--;
+                        for (; i >= 0; i--)
+                                rpcsvc_program_unregister (rpc,
+                                                           gd_inet_programs[i]);
 
-        ret = glusterd_program_register (this, rpc, &gluster_pmap_prog);
-        if (ret) {
-                rpcsvc_program_unregister (rpc, &gd_svc_peer_prog);
-                rpcsvc_program_unregister (rpc, &gd_svc_cli_prog);
-                rpcsvc_program_unregister (rpc, &gd_svc_mgmt_prog);
-                goto out;
+                        goto out;
+                }
         }
 
-        ret = glusterd_program_register (this, rpc, &gluster_handshake_prog);
-        if (ret) {
-                rpcsvc_program_unregister (rpc, &gd_svc_peer_prog);
-                rpcsvc_program_unregister (rpc, &gluster_pmap_prog);
-                rpcsvc_program_unregister (rpc, &gd_svc_cli_prog);
-                rpcsvc_program_unregister (rpc, &gd_svc_mgmt_prog);
+        /* Start a unix domain socket listener just for cli commands
+         * This should prevent ports from being wasted by being in TIMED_WAIT
+         * when cli commands are done continuously
+         */
+        uds_rpc = glusterd_init_uds_listener (this);
+        if (uds_rpc == NULL) {
+                ret = -1;
                 goto out;
         }
 
@@ -927,19 +1333,30 @@ init (xlator_t *this)
         conf->shd = GF_CALLOC (1, sizeof (nodesrv_t),
                                gf_gld_mt_nodesrv_t);
         GF_VALIDATE_OR_GOTO(this->name, conf->shd, out);
+        conf->nfs = GF_CALLOC (1, sizeof (nodesrv_t),
+                               gf_gld_mt_nodesrv_t);
+        GF_VALIDATE_OR_GOTO(this->name, conf->nfs, out);
 
         INIT_LIST_HEAD (&conf->peers);
         INIT_LIST_HEAD (&conf->volumes);
+        INIT_LIST_HEAD (&conf->snapshots);
+        INIT_LIST_HEAD (&conf->missed_snaps_list);
+
         pthread_mutex_init (&conf->mutex, NULL);
         conf->rpc = rpc;
+        conf->uds_rpc = uds_rpc;
         conf->gfs_mgmt = &gd_brick_prog;
-        strncpy (conf->workdir, dirname, PATH_MAX);
+        strncpy (conf->workdir, workdir, PATH_MAX);
 
+        synclock_init (&conf->big_lock);
+        pthread_mutex_init (&conf->xprt_lock, NULL);
         INIT_LIST_HEAD (&conf->xprt_list);
 
         glusterd_friend_sm_init ();
         glusterd_op_sm_init ();
         glusterd_opinfo_init ();
+        glusterd_mgmt_v3_lock_init ();
+        glusterd_txn_opinfo_dict_init ();
         ret = glusterd_sm_tr_log_init (&conf->op_sm_log,
                                        glusterd_op_sm_state_name_get,
                                        glusterd_op_sm_event_name_get,
@@ -947,31 +1364,42 @@ init (xlator_t *this)
         if (ret)
                 goto out;
 
+         conf->base_port = GF_IANA_PRIV_PORTS_START;
+         if (dict_get_uint32(this->options, "base-port", &conf->base_port) == 0) {
+                 gf_log (this->name, GF_LOG_INFO,
+                         "base-port override: %d", conf->base_port);
+         }
+
         /* Set option to run bricks on valgrind if enabled in glusterd.vol */
-#ifdef DEBUG
         conf->valgrind = _gf_false;
-        ret = dict_get_str (this->options, "brick-with-valgrind", &valgrind_str);
+        ret = dict_get_str (this->options, "run-with-valgrind", &valgrind_str);
         if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "cannot get brick-with-valgrind value");
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "cannot get run-with-valgrind value");
         }
         if (valgrind_str) {
                 if (gf_string2boolean (valgrind_str, &(conf->valgrind))) {
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "brick-with-valgrind value not a boolean string");
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "run-with-valgrind value not a boolean string");
                 }
         }
-#endif
+
         this->private = conf;
-        (void) glusterd_shd_set_running (_gf_false);
-        /* this->ctx->top = this;*/
+        (void) glusterd_nodesvc_set_online_status ("glustershd", _gf_false);
 
-        ret = glusterd_uuid_init (first_time);
-        if (ret < 0)
-                goto out;
+        GLUSTERD_GET_HOOKS_DIR (hooks_dir, GLUSTERD_HOOK_VER, conf);
+        if (stat (hooks_dir, &buf)) {
+                ret = glusterd_hooks_create_hooks_directory (conf->workdir);
+                if (-1 == ret) {
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to create hooks directory ");
+                        exit (1);
+                }
+        }
 
         INIT_LIST_HEAD (&conf->mount_specs);
-        dict_foreach (this->options, _install_mount_spec, &ret);
+
+        ret = dict_foreach (this->options, _install_mount_spec, NULL);
         if (ret)
                 goto out;
         ret = dict_get_str (this->options, "mountbroker-root",
@@ -991,14 +1419,25 @@ init (xlator_t *this)
         if (ret < 0)
                 goto out;
 
+        /* If there are no 'friends', this would be the best time to
+         * spawn process/bricks that may need (re)starting since last
+         * time (this) glusterd was up.*/
+
+        if (list_empty (&conf->peers)) {
+                glusterd_launch_synctask (glusterd_spawn_daemons, NULL);
+        }
+        ret = glusterd_options_init (this);
+        if (ret < 0)
+                goto out;
+
         ret = glusterd_handle_upgrade_downgrade (this->options, conf);
         if (ret)
                 goto out;
 
-        glusterd_restart_bricks (conf);
-        ret = glusterd_restart_gsyncds (conf);
+        ret = glusterd_hooks_spawn_worker (this);
         if (ret)
                 goto out;
+
         ret = 0;
 out:
         if (ret < 0) {
@@ -1030,12 +1469,17 @@ fini (xlator_t *this)
                 goto out;
 
         conf = this->private;
-        if (conf->pmap)
-                FREE (conf->pmap);
+
+        glusterd_stop_uds_listener (this);
+
+        FREE (conf->pmap);
         if (conf->handle)
-                glusterd_store_handle_destroy (conf->handle);
+                gf_store_handle_destroy (conf->handle);
         glusterd_sm_tr_log_delete (&conf->op_sm_log);
+        glusterd_mgmt_v3_lock_fini ();
+        glusterd_txn_opinfo_dict_fini ();
         GF_FREE (conf);
+
         this->private = NULL;
 out:
         return;
@@ -1073,11 +1517,9 @@ notify (xlator_t *this, int32_t event, void *data, ...)
 }
 
 
-struct xlator_fops fops = {
-};
+struct xlator_fops fops;
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct xlator_dumpops dumpops = {
         .priv  = glusterd_priv,
@@ -1124,8 +1566,32 @@ struct volume_options options[] = {
         { .key = {GEOREP"-log-group"},
           .type = GF_OPTION_TYPE_ANY,
         },
-        { .key = {"brick-with-valgrind"},
+        { .key = {"run-with-valgrind"},
           .type = GF_OPTION_TYPE_BOOL,
         },
+        { .key = {"server-quorum-type"},
+          .type = GF_OPTION_TYPE_STR,
+          .value = { "none", "server"},
+          .description = "If set to server, enables the specified "
+          "volume to participate in quorum."
+        },
+        { .key = {"server-quorum-ratio"},
+          .type = GF_OPTION_TYPE_PERCENT,
+          .description = "Sets the quorum percentage for the trusted "
+          "storage pool."
+        },
+        { .key = {"glusterd-sockfile"},
+          .type = GF_OPTION_TYPE_PATH,
+          .description = "The socket file on which glusterd should listen for "
+                        "cli requests. Default is "DEFAULT_GLUSTERD_SOCKFILE "."
+        },
+        { .key = {"base-port"},
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Sets the base port for portmap query"
+        },
+        { .key = {"snap-brick-path"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "directory where the bricks for the snapshots will be created"
+        },
         { .key   = {NULL} },
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 3c671da64..3aa395ebc 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_H_
 #define _GLUSTERD_H_
 
@@ -46,13 +36,47 @@
 #include "protocol-common.h"
 #include "glusterd-pmap.h"
 #include "cli1-xdr.h"
+#include "syncop.h"
+#include "store.h"
 
 #define GLUSTERD_MAX_VOLUME_NAME        1000
-#define DEFAULT_LOG_FILE_DIRECTORY      DATADIR "/log/glusterfs"
 #define GLUSTERD_TR_LOG_SIZE            50
 #define GLUSTERD_NAME                   "glusterd"
 #define GLUSTERD_SOCKET_LISTEN_BACKLOG  128
+#define GLUSTERD_QUORUM_TYPE_KEY        "cluster.server-quorum-type"
+#define GLUSTERD_QUORUM_RATIO_KEY       "cluster.server-quorum-ratio"
+#define GLUSTERD_GLOBAL_OPT_VERSION     "global-option-version"
+#define GLUSTERD_COMMON_PEM_PUB_FILE    "/geo-replication/common_secret.pem.pub"
+#define GEO_CONF_MAX_OPT_VALS           5
+#define GLUSTERD_CREATE_HOOK_SCRIPT     "/hooks/1/gsync-create/post/" \
+                                        "S56glusterd-geo-rep-create-post.sh"
+
+
+#define GLUSTERD_SNAPS_MAX_HARD_LIMIT 256
+#define GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT 90
+#define GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT 100
+#define GLUSTERD_SERVER_QUORUM "server"
+
+#define FMTSTR_CHECK_VOL_EXISTS "Volume %s does not exist"
+#define FMTSTR_RESOLVE_BRICK "Could not find peer on which brick %s:%s resides"
+
+#define LOGSTR_FOUND_BRICK  "Found brick %s:%s in volume %s"
+#define LOGSTR_BUILD_PAYLOAD "Failed to build payload for operation 'Volume %s'"
+#define LOGSTR_STAGE_FAIL "Staging of operation 'Volume %s' failed on %s %s %s"
+#define LOGSTR_COMMIT_FAIL "Commit of operation 'Volume %s' failed on %s %s %s"
+
+#define OPERRSTR_BUILD_PAYLOAD "Failed to build payload. Please check the log "\
+                               "file for more details."
+#define OPERRSTR_STAGE_FAIL "Staging failed on %s. Please check the log file " \
+                            "for more details."
+#define OPERRSTR_COMMIT_FAIL "Commit failed on %s. Please check the log file "\
+                             "for more details."
 
+struct glusterd_volinfo_;
+typedef struct glusterd_volinfo_ glusterd_volinfo_t;
+
+struct glusterd_snap_;
+typedef struct glusterd_snap_ glusterd_snap_t;
 
 typedef enum glusterd_op_ {
         GD_OP_NONE = 0,
@@ -77,68 +101,98 @@ typedef enum glusterd_op_ {
         GD_OP_REBALANCE,
         GD_OP_HEAL_VOLUME,
         GD_OP_STATEDUMP_VOLUME,
+        GD_OP_LIST_VOLUME,
+        GD_OP_CLEARLOCKS_VOLUME,
+        GD_OP_DEFRAG_BRICK_VOLUME,
+        GD_OP_COPY_FILE,
+        GD_OP_SYS_EXEC,
+        GD_OP_GSYNC_CREATE,
+        GD_OP_SNAP,
         GD_OP_MAX,
 } glusterd_op_t;
 
-
-struct glusterd_store_iter_ {
-        int     fd;
-        FILE    *file;
-        char    filepath[PATH_MAX];
-};
-
-typedef struct glusterd_store_iter_     glusterd_store_iter_t;
+extern const char * gd_op_list[];
 
 struct glusterd_volgen {
         dict_t *dict;
 };
+
 typedef struct {
-        struct rpc_clnt         *rpc;
-        gf_boolean_t            running;
+        struct rpc_clnt  *rpc;
+        gf_boolean_t      online;
 } nodesrv_t;
 
 typedef struct {
-        struct _volfile_ctx *volfile;
-	pthread_mutex_t   mutex;
-	struct list_head  peers;
-//	struct list_head  pending_peers;
-        gf_boolean_t      verify_volfile_checksum;
-        gf_boolean_t      trace;
-        uuid_t            uuid;
-        char              workdir[PATH_MAX];
-        rpcsvc_t          *rpc;
-        nodesrv_t         *shd;
-        struct pmap_registry *pmap;
-        struct list_head  volumes;
-        struct list_head  xprt_list;
-        glusterd_store_handle_t *handle;
-        gf_timer_t *timer;
-        glusterd_sm_tr_log_t op_sm_log;
-        struct rpc_clnt_program *gfs_mgmt;
-        struct list_head mount_specs;
-#ifdef DEBUG
-        gf_boolean_t      valgrind;
-#endif
+        gf_boolean_t    quorum;
+        double          quorum_ratio;
+        uint64_t        gl_opt_version;
+} gd_global_opts_t;
+
+typedef struct {
+        struct _volfile_ctx      *volfile;
+        pthread_mutex_t           mutex;
+        struct list_head          peers;
+        struct list_head          xaction_peers;
+        gf_boolean_t              verify_volfile_checksum;
+        gf_boolean_t              trace;
+        uuid_t                    uuid;
+        char                      workdir[PATH_MAX];
+        rpcsvc_t                 *rpc;
+        nodesrv_t                *shd;
+        nodesrv_t                *nfs;
+        struct pmap_registry     *pmap;
+        struct list_head          volumes;
+        struct list_head          snapshots; /*List of snap volumes */
+        pthread_mutex_t           xprt_lock;
+        struct list_head          xprt_list;
+        gf_store_handle_t        *handle;
+        gf_timer_t               *timer;
+        glusterd_sm_tr_log_t      op_sm_log;
+        struct rpc_clnt_program  *gfs_mgmt;
+
+        struct list_head           mount_specs;
+        gf_boolean_t               valgrind;
+        pthread_t                  brick_thread;
+        void                      *hooks_priv;
+        /* need for proper handshake_t */
+        int                        op_version; /* Starts with 1 for 3.3.0 */
+        xlator_t                  *xl;  /* Should be set to 'THIS' before creating thread */
+        gf_boolean_t               pending_quorum_action;
+        dict_t                    *opts;
+        synclock_t                 big_lock;
+        gf_boolean_t               restart_done;
+        rpcsvc_t                  *uds_rpc; /* RPCSVC for the unix domain socket */
+        uint32_t                   base_port;
+        uint64_t                   snap_max_hard_limit;
+        uint64_t                   snap_max_soft_limit;
+        char                      *snap_bricks_directory;
+        gf_store_handle_t         *missed_snaps_list_shandle;
+        struct list_head           missed_snaps_list;
 } glusterd_conf_t;
 
+
 typedef enum gf_brick_status {
         GF_BRICK_STOPPED,
         GF_BRICK_STARTED,
 } gf_brick_status_t;
 
 struct glusterd_brickinfo {
-        char    hostname[1024];
-        char    path[PATH_MAX];
-        struct list_head  brick_list;
-        uuid_t  uuid;
-        int     port;
-        int     rdma_port;
-        char   *logfile;
-        gf_boolean_t signed_in;
-        glusterd_store_handle_t *shandle;
-        gf_brick_status_t status;
-        struct rpc_clnt *rpc;
-        int decommissioned;
+        char               hostname[1024];
+        char               path[PATH_MAX];
+        char               device_path[PATH_MAX];
+        struct list_head   brick_list;
+        uuid_t             uuid;
+        int                port;
+        int                rdma_port;
+        char              *logfile;
+        gf_boolean_t       signed_in;
+        gf_store_handle_t *shandle;
+        gf_brick_status_t  status;
+        struct rpc_clnt   *rpc;
+        int                decommissioned;
+        char vg[PATH_MAX]; /* FIXME: Use max size for length of vg */
+        int     caps; /* Capability */
+        int32_t            snap_status;
 };
 
 typedef struct glusterd_brickinfo glusterd_brickinfo_t;
@@ -149,9 +203,6 @@ struct gf_defrag_brickinfo_ {
         int   size;
 };
 
-struct glusterd_volinfo_;
-typedef struct glusterd_volinfo_ glusterd_volinfo_t;
-
 typedef int (*defrag_cbk_fn_t) (glusterd_volinfo_t *volinfo,
                                 gf_defrag_status_t status);
 
@@ -159,9 +210,13 @@ struct glusterd_defrag_info_ {
         uint64_t                     total_files;
         uint64_t                     total_data;
         uint64_t                     num_files_lookedup;
+        uint64_t                     total_failures;
         gf_lock_t                    lock;
         int                          cmd;
         pthread_t                    th;
+        gf_defrag_status_t           defrag_status;
+        struct rpc_clnt             *rpc;
+        uint32_t                     connected;
         char                         mount[1024];
         char                         databuf[131072];
         struct gf_defrag_brickinfo_ *bricks; /* volinfo->brick_count */
@@ -186,56 +241,168 @@ typedef enum gf_rb_status_ {
         GF_RB_STATUS_PAUSED,
 } gf_rb_status_t;
 
+struct _auth {
+        char       *username;
+        char       *password;
+};
+
+typedef struct _auth auth_t;
+
+/* Capabilities of xlator */
+#define CAPS_BD               0x00000001
+#define CAPS_THIN             0x00000002
+#define CAPS_OFFLOAD_COPY     0x00000004
+#define CAPS_OFFLOAD_SNAPSHOT 0x00000008
+
+struct glusterd_rebalance_ {
+        gf_defrag_status_t       defrag_status;
+        uint64_t                 rebalance_files;
+        uint64_t                 rebalance_data;
+        uint64_t                 lookedup_files;
+        uint64_t                 skipped_files;
+        glusterd_defrag_info_t  *defrag;
+        gf_cli_defrag_type       defrag_cmd;
+        uint64_t                 rebalance_failures;
+        uuid_t                   rebalance_id;
+        double                   rebalance_time;
+        glusterd_op_t            op;
+        dict_t                  *dict; /* Dict to store misc information
+                                        * like list of bricks being removed */
+};
+
+typedef struct glusterd_rebalance_ glusterd_rebalance_t;
+
+struct glusterd_replace_brick_ {
+        gf_rb_status_t          rb_status;
+        glusterd_brickinfo_t   *src_brick;
+        glusterd_brickinfo_t   *dst_brick;
+        uuid_t                  rb_id;
+};
+
+typedef struct glusterd_replace_brick_ glusterd_replace_brick_t;
+
 struct glusterd_volinfo_ {
-        char                    volname[GLUSTERD_MAX_VOLUME_NAME];
-        int                     type;
-        int                     brick_count;
-        struct list_head        vol_list;
-        struct list_head        bricks;
-        glusterd_volume_status  status;
-        int                     sub_count;  /* backward compatibility */
-        int                     stripe_count;
-        int                     replica_count;
-        int                     dist_leaf_count; /* Number of bricks in one
+        gf_lock_t                 lock;
+        char                      volname[GLUSTERD_MAX_VOLUME_NAME];
+        gf_boolean_t              is_snap_volume;
+        glusterd_snap_t          *snapshot;
+        gf_boolean_t              is_volume_restored;
+        char                      parent_volname[GLUSTERD_MAX_VOLUME_NAME];
+                                         /* In case of a snap volume
+                                            i.e (is_snap_volume == TRUE) this
+                                            field will contain the name of
+                                            the volume which is snapped. In
+                                            case of a non-snap volume, this
+                                            field will be initialized as N/A */
+        int                       type;
+        int                       brick_count;
+        uint64_t                  snap_count;
+        uint64_t                  snap_max_hard_limit;
+        struct list_head          vol_list;
+                                      /* In case of a snap volume
+                                         i.e (is_snap_volume == TRUE) this
+                                         is linked to glusterd_snap_t->volumes.
+                                         In case of a non-snap volume, this is
+                                         linked to glusterd_conf_t->volumes */
+        struct list_head          snapvol_list;
+                                      /* This is a current pointer for
+                                         glusterd_volinfo_t->snap_volumes */
+        struct list_head          bricks;
+        struct list_head          snap_volumes;
+                                      /* TODO : Need to remove this, as this
+                                       * is already part of snapshot object.
+                                       */
+        glusterd_volume_status    status;
+        int                       sub_count;  /* backward compatibility */
+        int                       stripe_count;
+        int                       replica_count;
+        int                       subvol_count; /* Number of subvolumes in a
+                                                 distribute volume */
+        int                       dist_leaf_count; /* Number of bricks in one
                                                     distribute subvolume */
-        int                     port;
-        glusterd_store_handle_t *shandle;
-        glusterd_store_handle_t *rb_shandle;
+        int                       port;
+        gf_store_handle_t        *shandle;
+        gf_store_handle_t        *rb_shandle;
+        gf_store_handle_t        *node_state_shandle;
 
         /* Defrag/rebalance related */
-        gf_defrag_status_t      defrag_status;
-        uint64_t                rebalance_files;
-        uint64_t                rebalance_data;
-        uint64_t                lookedup_files;
-        glusterd_defrag_info_t  *defrag;
+        glusterd_rebalance_t      rebal;
 
         /* Replace brick status */
-        gf_rb_status_t          rb_status;
-        glusterd_brickinfo_t    *src_brick;
-        glusterd_brickinfo_t    *dst_brick;
+        glusterd_replace_brick_t  rep_brick;
 
-        int                     version;
-        uint32_t                cksum;
-        gf_transport_type       transport_type;
-        gf_transport_type   nfs_transport_type;
+        int                       version;
+        uint32_t                  cksum;
+        gf_transport_type         transport_type;
+        gf_transport_type         nfs_transport_type;
 
-        dict_t                  *dict;
+        dict_t                   *dict;
 
-        uuid_t                  volume_id;
-        char                    *logdir;
+        uuid_t                    volume_id;
+        auth_t                    auth;
+        char                     *logdir;
 
-        dict_t                  *gsync_slaves;
+        dict_t                   *gsync_slaves;
 
-        int                      decommission_in_progress;
-        xlator_t                *xl;
+        int                       decommission_in_progress;
+        xlator_t                 *xl;
+
+        gf_boolean_t              memory_accounting;
+        int                      caps; /* Capability */
+
+        int                       op_version;
+        int                       client_op_version;
 };
 
+typedef enum gd_snap_status_ {
+        GD_SNAP_STATUS_NONE,
+        GD_SNAP_STATUS_INIT,
+        GD_SNAP_STATUS_IN_USE,
+        GD_SNAP_STATUS_DECOMMISSION,
+        GD_SNAP_STATUS_RESTORED,
+} gd_snap_status_t;
+
+struct glusterd_snap_ {
+        gf_lock_t                lock;
+        struct  list_head        volumes;
+        struct  list_head        snap_list;
+        char                     snapname[GLUSTERD_MAX_SNAP_NAME];
+        uuid_t                   snap_id;
+        char                    *description;
+        time_t                   time_stamp;
+        gf_boolean_t             snap_restored;
+        gd_snap_status_t         snap_status;
+        gf_store_handle_t       *shandle;
+};
+
+typedef struct glusterd_snap_op_ {
+        int32_t                brick_num;
+        char                  *brick_path;
+        int32_t                op;
+        int32_t                status;
+        struct list_head       snap_ops_list;
+} glusterd_snap_op_t;
+
+typedef struct glusterd_missed_snap_ {
+        char                   *node_snap_info;
+        struct list_head        missed_snaps;
+        struct list_head        snap_ops;
+} glusterd_missed_snap_info;
+
 typedef enum gd_node_type_ {
         GD_NODE_NONE,
         GD_NODE_BRICK,
-        GD_NODE_SHD
+        GD_NODE_SHD,
+        GD_NODE_REBALANCE,
+        GD_NODE_NFS,
 } gd_node_type;
 
+typedef enum missed_snap_stat {
+        GD_MISSED_SNAP_NONE,
+        GD_MISSED_SNAP_PENDING,
+        GD_MISSED_SNAP_DONE,
+} missed_snap_stat;
+
 typedef struct glusterd_pending_node_ {
         struct list_head list;
         void   *node;
@@ -243,6 +410,13 @@ typedef struct glusterd_pending_node_ {
         int32_t index;
 } glusterd_pending_node_t;
 
+struct gsync_config_opt_vals_ {
+        char           *op_name;
+        int             no_of_pos_vals;
+        gf_boolean_t    case_sensitive;
+        char           *values[GEO_CONF_MAX_OPT_VALS];
+};
+
 enum glusterd_op_ret {
         GLUSTERD_CONNECTION_AWAITED = 100,
 };
@@ -254,15 +428,24 @@ enum glusterd_vol_comp_status_ {
         GLUSTERD_VOL_COMP_RJT,
 };
 
-#define GLUSTERD_DEFAULT_WORKDIR "/etc/glusterd"
+#define GLUSTERD_DEFAULT_WORKDIR "/var/lib/glusterd"
 #define GLUSTERD_DEFAULT_PORT    GF_DEFAULT_BASE_PORT
 #define GLUSTERD_INFO_FILE      "glusterd.info"
 #define GLUSTERD_VOLUME_DIR_PREFIX "vols"
 #define GLUSTERD_PEER_DIR_PREFIX "peers"
 #define GLUSTERD_VOLUME_INFO_FILE "info"
+#define GLUSTERD_SNAP_INFO_FILE "info"
 #define GLUSTERD_VOLUME_RBSTATE_FILE "rbstate"
 #define GLUSTERD_BRICK_INFO_DIR "bricks"
 #define GLUSTERD_CKSUM_FILE "cksum"
+#define GLUSTERD_TRASH "trash"
+#define GLUSTERD_NODE_STATE_FILE "node_state.info"
+#define GLUSTERD_MISSED_SNAPS_LIST_FILE "missed_snaps_list"
+#define GLUSTERD_VOL_SNAP_DIR_PREFIX "snaps"
+
+#define GLUSTERD_DEFAULT_SNAPS_BRICK_DIR        "/gluster/snaps"
+#define GLUSTERD_VAR_RUN_DIR                    "/var/run"
+#define GLUSTERD_RUN_DIR                        "/run"
 
 /* definitions related to replace brick */
 #define RB_CLIENT_MOUNTPOINT    "rb_mount"
@@ -275,14 +458,29 @@ enum glusterd_vol_comp_status_ {
 
 typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
 
-#define GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv) \
-        snprintf (path, PATH_MAX, "%s/vols/%s", priv->workdir,\
-                  volinfo->volname);
+#define GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv)                       \
+        if (volinfo->is_snap_volume) {                                     \
+                snprintf (path, PATH_MAX, "%s/snaps/%s/%s", priv->workdir, \
+                          volinfo->snapshot->snapname, volinfo->volname);  \
+        } else {                                                           \
+                snprintf (path, PATH_MAX, "%s/vols/%s", priv->workdir,     \
+                          volinfo->volname);                               \
+        }
 
-#define GLUSTERD_GET_BRICK_DIR(path, volinfo, priv) \
-        snprintf (path, PATH_MAX, "%s/%s/%s/%s", priv->workdir,\
-                  GLUSTERD_VOLUME_DIR_PREFIX, volinfo->volname, \
-                  GLUSTERD_BRICK_INFO_DIR);
+#define GLUSTERD_GET_SNAP_DIR(path, snap, priv)                           \
+                snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,   \
+                          snap->snapname);
+
+#define GLUSTERD_GET_BRICK_DIR(path, volinfo, priv)                           \
+        if (volinfo->is_snap_volume) {                                        \
+                snprintf (path, PATH_MAX, "%s/snaps/%s/%s/%s", priv->workdir, \
+                          volinfo->snapshot->snapname, volinfo->volname,      \
+                          GLUSTERD_BRICK_INFO_DIR);                           \
+        } else {                                                              \
+                snprintf (path, PATH_MAX, "%s/%s/%s/%s", priv->workdir,       \
+                          GLUSTERD_VOLUME_DIR_PREFIX, volinfo->volname,       \
+                          GLUSTERD_BRICK_INFO_DIR);                           \
+        }
 
 #define GLUSTERD_GET_NFS_DIR(path, priv) \
         snprintf (path, PATH_MAX, "%s/nfs", priv->workdir);
@@ -296,12 +494,14 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                 }                                                       \
         } while (0)
 
-#define GLUSTERD_GET_BRICK_PIDFILE(pidfile,volpath,hostname,brickpath) { \
-                char exp_path[PATH_MAX] = {0,};                         \
-                GLUSTERD_REMOVE_SLASH_FROM_PATH (brickpath, exp_path);  \
-                snprintf (pidfile, PATH_MAX, "%s/run/%s-%s.pid",        \
-                          volpath, hostname, exp_path);                 \
-        }
+#define GLUSTERD_GET_BRICK_PIDFILE(pidfile,volinfo,brickinfo, priv) do {      \
+                char exp_path[PATH_MAX] = {0,};                               \
+                char volpath[PATH_MAX]  = {0,};                               \
+                GLUSTERD_GET_VOLUME_DIR (volpath, volinfo, priv);             \
+                GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);  \
+                snprintf (pidfile, PATH_MAX, "%s/run/%s-%s.pid",              \
+                          volpath, brickinfo->hostname, exp_path);      \
+        } while (0)
 
 #define GLUSTERD_GET_NFS_PIDFILE(pidfile,nfspath) {                     \
                 snprintf (pidfile, PATH_MAX, "%s/run/nfs.pid",          \
@@ -309,19 +509,81 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
         }
 
 #define GLUSTERD_STACK_DESTROY(frame) do {\
-		frame->local = NULL;	  \
-		STACK_DESTROY (frame->root);\
-	} while (0)
+                frame->local = NULL;                                    \
+                STACK_DESTROY (frame->root);                            \
+        } while (0)
+
+#define GLUSTERD_GET_DEFRAG_DIR(path, volinfo, priv) do {               \
+                char vol_path[PATH_MAX];                                \
+                GLUSTERD_GET_VOLUME_DIR(vol_path, volinfo, priv);       \
+                snprintf (path, PATH_MAX, "%s/rebalance",vol_path);     \
+        } while (0)
+
+#define GLUSTERD_GET_DEFRAG_SOCK_FILE(path, volinfo, priv) do {         \
+                char defrag_path[PATH_MAX];                             \
+                GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);    \
+                snprintf (path, PATH_MAX, "%s/%s.sock", defrag_path,    \
+                           uuid_utoa(MY_UUID));                         \
+        } while (0)
+
+#define GLUSTERD_GET_DEFRAG_PID_FILE(path, volinfo, priv) do {          \
+                char defrag_path[PATH_MAX];                             \
+                GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);    \
+                snprintf (path, PATH_MAX, "%s/%s.pid", defrag_path,     \
+                           uuid_utoa(MY_UUID));                         \
+        } while (0)
+
+#define GLUSTERD_GET_UUID_NOHYPHEN(ret_string, uuid) do {               \
+                char *snap_volname_ptr = ret_string;                    \
+                char *snap_volid_ptr = uuid_utoa(uuid);                 \
+                while (*snap_volid_ptr) {                               \
+                        if (*snap_volid_ptr == '-') {                   \
+                                snap_volid_ptr++;                       \
+                        } else {                                        \
+                                (*snap_volname_ptr++) =                 \
+                                (*snap_volid_ptr++);                    \
+                        }                                               \
+                }                                                       \
+                *snap_volname_ptr = '\0';                               \
+        } while (0)
+
+int glusterd_uuid_init();
+
+int glusterd_uuid_generate_save ();
+
+#define MY_UUID (__glusterd_uuid())
+
+static inline unsigned char *
+__glusterd_uuid()
+{
+        glusterd_conf_t *priv = THIS->private;
+
+        if (uuid_is_null (priv->uuid))
+                glusterd_uuid_init();
+        return &priv->uuid[0];
+}
+
+int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
+                                rpc_clnt_event_t event,
+                                void *data, rpc_clnt_notify_t notify_fn);
+
+int
+glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe, fop_cbk_fn_t fn);
+
+int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn);
 
 int32_t
 glusterd_brick_from_brickinfo (glusterd_brickinfo_t *brickinfo,
                                char **new_brick);
 int
-glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port);
+glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
+                      dict_t *dict);
 
 int
-glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *hostname,
-                               int port, int32_t op_ret, int32_t op_errno);
+glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *myhostname,
+                               char *remote_hostname, int port, int32_t op_ret,
+                               int32_t op_errno);
 
 int
 glusterd_friend_find (uuid_t uuid, char *hostname,
@@ -334,6 +596,9 @@ glusterd_friend_add (const char *hoststr, int port,
                      gf_boolean_t restore, glusterd_peerctx_args_t *args);
 
 int
+glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                            glusterd_peerctx_args_t *args);
+int
 glusterd_friend_remove (uuid_t uuid, char *hostname);
 
 int
@@ -343,6 +608,14 @@ int
 glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status);
 
 int
+glusterd_op_mgmt_v3_lock_send_resp (rpcsvc_request_t *req,
+                                    uuid_t *txn_id, int32_t status);
+
+int
+glusterd_op_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req,
+                                      uuid_t *txn_id, int32_t status);
+
+int
 glusterd_op_stage_send_resp (rpcsvc_request_t *req,
                              int32_t op, int32_t status,
                              char *op_errstr, dict_t *rsp_dict);
@@ -386,7 +659,8 @@ glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req);
 
 int
 glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                              int32_t op_errno, char *hostname, int port);
+                              int32_t op_errno, char *op_errstr, char *hostname,
+                              int port, dict_t *dict);
 
 int
 glusterd_op_commit_send_resp (rpcsvc_request_t *req,
@@ -398,7 +672,7 @@ glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int por
 
 int
 glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
-                        uuid_t uuid);
+                        uuid_t uuid, dict_t *dict);
 
 int
 glusterd_handle_cli_deprobe (rpcsvc_request_t *req);
@@ -473,6 +747,12 @@ int
 glusterd_handle_reset_volume (rpcsvc_request_t *req);
 
 int
+glusterd_handle_copy_file (rpcsvc_request_t *req);
+
+int
+glusterd_handle_sys_exec (rpcsvc_request_t *req);
+
+int
 glusterd_handle_gsync_set (rpcsvc_request_t *req);
 
 int
@@ -483,7 +763,8 @@ glusterd_handle_fsm_log (rpcsvc_request_t *req);
 
 int
 glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                                int32_t op_errno, char *hostname);
+                                int32_t op_errno, char *op_errstr,
+                                char *hostname, dict_t *dict);
 
 int
 glusterd_fetchspec_notify (xlator_t *this);
@@ -500,8 +781,8 @@ glusterd_volume_txn (rpcsvc_request_t *req, char *volname, int flags,
                      glusterd_op_t op);
 
 int
-glusterd_peer_handshake (xlator_t *this, struct rpc_clnt *rpc,
-                         glusterd_peerctx_t *peerctx);
+glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc,
+                            glusterd_peerctx_t *peerctx);
 
 int
 glusterd_validate_reconfopts (glusterd_volinfo_t *volinfo, dict_t *val_dict, char **op_errstr);
@@ -522,8 +803,8 @@ glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
                           rpc_clnt_event_t event, void *data);
 
 int
-glusterd_shd_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event, void *data);
+glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                             rpc_clnt_event_t event, void *data);
 
 int
 glusterd_rpc_create (struct rpc_clnt **rpc, dict_t *options,
@@ -531,22 +812,40 @@ glusterd_rpc_create (struct rpc_clnt **rpc, dict_t *options,
 
 
 /* handler functions */
-int32_t glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx);
+int32_t glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                           char *err_str, size_t size);
 
 /* removed other definitions as they have been defined elsewhere in this file*/
 
 int glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req);
+int glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req);
 
 int glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
-                                  size_t len, int cmd, defrag_cbk_fn_t cbk);
+                                  size_t len, int cmd, defrag_cbk_fn_t cbk,
+                                  glusterd_op_t op);
+int
+glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
+                               glusterd_conf_t *priv, int cmd);
+
 int glusterd_handle_cli_heal_volume (rpcsvc_request_t *req);
 
+int glusterd_handle_cli_list_volume (rpcsvc_request_t *req);
+
+int
+glusterd_handle_snapshot (rpcsvc_request_t *req);
+
 /* op-sm functions */
 int glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr);
 int glusterd_op_heal_volume (dict_t *dict, char **op_errstr);
 int glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr);
 int glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
-int glusterd_op_quota (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_copy_file (dict_t *dict, char **op_errstr);
+int glusterd_op_copy_file (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_sys_exec (dict_t *dict, char **op_errstr);
+int glusterd_op_sys_exec (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int glusterd_op_stage_gsync_create (dict_t *dict, char **op_errstr);
+int glusterd_op_gsync_create (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
 int glusterd_op_stage_quota (dict_t *dict, char **op_errstr);
 int glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                                      dict_t *rsp_dict);
@@ -573,6 +872,10 @@ int glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
 int glusterd_op_stage_statedump_volume (dict_t *dict, char **op_errstr);
 int glusterd_op_statedump_volume (dict_t *dict, char **op_errstr);
 
+int glusterd_op_stage_clearlocks_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr,
+                                   dict_t *rsp_dict);
+
 /* misc */
 void glusterd_do_replace_brick (void *data);
 int glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
@@ -581,4 +884,69 @@ int glusterd_op_stop_volume_args_get (dict_t *dict, char** volname, int *flags);
 int glusterd_op_statedump_volume_args_get (dict_t *dict, char **volname,
                                            char **options, int *option_cnt);
 
+int glusterd_op_gsync_args_get (dict_t *dict, char **op_errstr,
+                                char **master, char **slave, char **host_uuid);
+int glusterd_stop_volume (glusterd_volinfo_t *volinfo);
+
+/* Synctask part */
+int32_t glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
+                                    void *dict);
+int32_t
+glusterd_defrag_event_notify_handle (dict_t *dict);
+
+int32_t
+glusterd_txn_opinfo_dict_init ();
+
+void
+glusterd_txn_opinfo_dict_fini ();
+
+void
+glusterd_txn_opinfo_init ();
+
+/* snapshot */
+glusterd_snap_t*
+glusterd_new_snap_object();
+
+int32_t
+glusterd_list_add_snapvol (glusterd_volinfo_t *origin_vol,
+                           glusterd_volinfo_t *snap_vol);
+
+glusterd_snap_t*
+glusterd_remove_snap_by_id (uuid_t snap_id);
+
+glusterd_snap_t*
+glusterd_remove_snap_by_name (char *snap_name);
+
+glusterd_snap_t*
+glusterd_find_snap_by_name (char *snap_name);
+
+glusterd_snap_t*
+glusterd_find_snap_by_id (uuid_t snap_id);
+
+int
+glusterd_snapshot_prevalidate (dict_t *dict, char **op_errstr,
+                               dict_t *rsp_dict);
+int
+glusterd_snapshot_brickop (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int
+glusterd_snapshot (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int
+glusterd_snapshot_postvalidate (dict_t *dict, int32_t op_ret, char **op_errstr,
+                                dict_t *rsp_dict);
+char *
+glusterd_build_snap_device_path (char *device, char *snapname);
+int32_t
+glusterd_snap_remove (dict_t *rsp_dict, glusterd_snap_t *snap,
+                      gf_boolean_t remove_lvm, gf_boolean_t force);
+int32_t
+glusterd_snapshot_cleanup (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+glusterd_add_missed_snaps_to_list (dict_t *dict, int32_t missed_snap_count);
+
+int32_t
+glusterd_store_missed_snaps_list (char *missed_info, int32_t brick_num,
+                                  char *brick_path, int32_t snap_op,
+                                  int32_t snap_status);
+
 #endif
diff --git a/xlators/mount/fuse/src/Makefile.am b/xlators/mount/fuse/src/Makefile.am
index 7bb3931ec..653121d18 100644
--- a/xlators/mount/fuse/src/Makefile.am
+++ b/xlators/mount/fuse/src/Makefile.am
@@ -1,26 +1,36 @@
-noinst_HEADERS = $(CONTRIBDIR)/fuse-include/fuse_kernel.h\
-	$(CONTRIBDIR)/fuse-include/fuse-mount.h\
+noinst_HEADERS_linux = $(CONTRIBDIR)/fuse-include/fuse_kernel.h\
+	$(CONTRIBDIR)/fuse-include/mount_util.h\
+	$(CONTRIBDIR)/fuse-lib/mount-gluster-compat.h
+noinst_HEADERS_darwin =	$(CONTRIBDIR)/fuse-include/fuse_kernel_macfuse.h
+noinst_HEADERS_common = $(CONTRIBDIR)/fuse-include/fuse-mount.h\
 	$(CONTRIBDIR)/fuse-include/fuse-misc.h fuse-mem-types.h \
 	fuse-bridge.h
 
+if GF_DARWIN_HOST_OS
+    noinst_HEADERS = $(noinst_HEADERS_common) $(noinst_HEADERS_darwin)
+else
+    noinst_HEADERS = $(noinst_HEADERS_common) $(noinst_HEADERS_linux)
+endif
+
 xlator_LTLIBRARIES = fuse.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
 
 if GF_DARWIN_HOST_OS
     mount_source=$(CONTRIBDIR)/macfuse/mount_darwin.c
 else
-    mount_source=$(CONTRIBDIR)/fuse-lib/mount.c
+    mount_source=$(CONTRIBDIR)/fuse-lib/mount.c $(CONTRIBDIR)/fuse-lib/mount-common.c
 endif
 
 fuse_la_SOURCES = fuse-helpers.c fuse-resolve.c fuse-bridge.c \
 	$(CONTRIBDIR)/fuse-lib/misc.c $(mount_source)
 
-fuse_la_LDFLAGS = -module -avoidversion -shared -nostartfiles
+fuse_la_LDFLAGS = -module -avoid-version
 fuse_la_LIBADD = @GF_FUSE_LDADD@
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) -Wall \
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/fuse-include \
-	 $(GF_CFLAGS) $(GF_FUSE_CFLAGS)
+	-I$(CONTRIBDIR)/fuse-lib $(GF_FUSE_CFLAGS)
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 1da2dcdb2..6a5587c2d 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -1,31 +1,100 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
+#include <sys/wait.h>
 #include "fuse-bridge.h"
+#include "mount-gluster-compat.h"
+#include "glusterfs.h"
+#include "glusterfs-acl.h"
+
+#ifdef __NetBSD__
+#undef open /* in perfuse.h, pulled from mount-gluster-compat.h */
+#endif
 
 static int gf_fuse_conn_err_log;
 static int gf_fuse_xattr_enotsup_log;
 
 void fini (xlator_t *this_xl);
 
+static void fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino);
+
+/*
+ * Send an invalidate notification up to fuse to purge the file from local
+ * page cache.
+ */
+static int32_t
+fuse_invalidate(xlator_t *this, inode_t *inode)
+{
+        fuse_private_t *priv = this->private;
+        uint64_t nodeid;
+
+        /*
+         * NOTE: We only invalidate at the moment if fopen_keep_cache is
+         * enabled because otherwise this is a departure from default
+         * behavior. Specifically, the performance/write-behind xlator
+         * causes unconditional invalidations on write requests.
+         */
+        if (!priv->fopen_keep_cache)
+                return 0;
+
+        nodeid = inode_to_fuse_nodeid(inode);
+        gf_log(this->name, GF_LOG_DEBUG, "Invalidate inode id %lu.", nodeid);
+        fuse_log_eh (this, "Sending invalidate inode id: %lu gfid: %s", nodeid,
+                     uuid_utoa (inode->gfid));
+        fuse_invalidate_inode(this, nodeid);
+
+        return 0;
+}
+
+static int32_t
+fuse_forget_cbk (xlator_t *this, inode_t *inode)
+{
+        //Nothing to free in inode ctx, hence return.
+        return 0;
+}
+
+void
+fuse_inode_set_need_lookup (inode_t *inode, xlator_t *this)
+{
+	uint64_t          need_lookup = 1;
+
+	if (!inode || !this)
+		return;
+
+	inode_ctx_set (inode, this, &need_lookup);
+
+	return;
+}
+
+
+gf_boolean_t
+fuse_inode_needs_lookup (inode_t *inode, xlator_t *this)
+{
+	uint64_t          need_lookup = 0;
+	gf_boolean_t      ret = _gf_false;
+
+	if (!inode || !this)
+		return ret;
+
+	inode_ctx_get (inode, this, &need_lookup);
+	if (need_lookup)
+		ret = _gf_true;
+	need_lookup = 0;
+	inode_ctx_set (inode, this, &need_lookup);
+
+	return ret;
+}
+
+
 fuse_fd_ctx_t *
-__fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
+__fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd)
 {
         uint64_t       val    = 0;
         int32_t        ret    = 0;
@@ -38,7 +107,9 @@ __fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
         if (fd_ctx == NULL) {
                 fd_ctx = GF_CALLOC (1, sizeof (*fd_ctx),
                                     gf_fuse_mt_fd_ctx_t);
-
+                if (!fd_ctx) {
+                    goto out;
+                }
                 ret = __fd_ctx_set (fd, this,
                                     (uint64_t)(unsigned long)fd_ctx);
                 if (ret < 0) {
@@ -48,12 +119,12 @@ __fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
                         fd_ctx = NULL;
                 }
         }
-
+out:
         return fd_ctx;
 }
 
 fuse_fd_ctx_t *
-fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
+fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd)
 {
         fuse_fd_ctx_t *fd_ctx = NULL;
 
@@ -63,7 +134,7 @@ fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
 
         LOCK (&fd->lock);
         {
-                fd_ctx = __fuse_fd_ctx_check_n_create (fd, this);
+                fd_ctx = __fuse_fd_ctx_check_n_create (this, fd);
         }
         UNLOCK (&fd->lock);
 
@@ -71,6 +142,23 @@ out:
         return fd_ctx;
 }
 
+fuse_fd_ctx_t *
+fuse_fd_ctx_get (xlator_t *this, fd_t *fd)
+{
+        fuse_fd_ctx_t *fdctx = NULL;
+        uint64_t       value = 0;
+        int            ret   = 0;
+
+        ret = fd_ctx_get (fd, this, &value);
+        if (ret < 0) {
+                goto out;
+        }
+
+        fdctx = (fuse_fd_ctx_t *) (unsigned long)value;
+
+out:
+        return fdctx;
+}
 
 /*
  * iov_out should contain a fuse_out_header at zeroth position.
@@ -86,7 +174,7 @@ send_fuse_iov (xlator_t *this, fuse_in_header_t *finh, struct iovec *iov_out,
 
         if (!this || !finh || !iov_out) {
                 gf_log ("send_fuse_iov", GF_LOG_ERROR,"Invalid arguments");
-                return -1;
+                return EINVAL;
         }
         priv = this->private;
 
@@ -141,7 +229,7 @@ send_fuse_data (xlator_t *this, fuse_in_header_t *finh, void *data, size_t size)
 
 
 static void
-fuse_invalidate (xlator_t *this, uint64_t fuse_ino)
+fuse_invalidate_entry (xlator_t *this, uint64_t fuse_ino)
 {
         struct fuse_out_header             *fouh   = NULL;
         struct fuse_notify_inval_entry_out *fnieo  = NULL;
@@ -150,8 +238,7 @@ fuse_invalidate (xlator_t *this, uint64_t fuse_ino)
         inode_t                            *inode  = NULL;
         size_t                              nlen   = 0;
         int                                 rv     = 0;
-
-        char inval_buf[INVAL_BUF_SIZE] = {0,};
+        char inval_buf[INVAL_BUF_SIZE]             = {0,};
 
         fouh  = (struct fuse_out_header *)inval_buf;
         fnieo = (struct fuse_notify_inval_entry_out *)(fouh + 1);
@@ -184,7 +271,73 @@ fuse_invalidate (xlator_t *this, uint64_t fuse_ino)
 
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE, "INVALIDATE entry: "
                         "%"PRIu64"/%s", fnieo->parent, dentry->name);
+
+                if (dentry->parent) {
+                        fuse_log_eh (this, "Invalidated entry %s (parent: %s)",
+                                     dentry->name,
+                                     uuid_utoa (dentry->parent->gfid));
+                } else {
+                        fuse_log_eh (this, "Invalidated entry %s(nodeid: %ld)",
+                                     dentry->name, fnieo->parent);
+                }
         }
+
+        if (inode)
+                inode_unref (inode);
+}
+
+/*
+ * Send an inval inode notification to fuse. This causes an invalidation of the
+ * entire page cache mapping on the inode.
+ */
+static void
+fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino)
+{
+        struct fuse_out_header *fouh = NULL;
+        struct fuse_notify_inval_inode_out *fniio = NULL;
+        fuse_private_t *priv = NULL;
+        int rv = 0;
+        char inval_buf[INVAL_BUF_SIZE] = {0};
+        inode_t    *inode = NULL;
+
+        fouh = (struct fuse_out_header *) inval_buf;
+        fniio = (struct fuse_notify_inval_inode_out *) (fouh + 1);
+
+        priv = this->private;
+
+        if (priv->revchan_out < 0)
+                return;
+
+        fouh->unique = 0;
+        fouh->error = FUSE_NOTIFY_INVAL_INODE;
+        fouh->len = sizeof(struct fuse_out_header) +
+                sizeof(struct fuse_notify_inval_inode_out);
+
+        /* inval the entire mapping until we learn how to be more granular */
+        fniio->ino = fuse_ino;
+        fniio->off = 0;
+        fniio->len = -1;
+
+        inode = fuse_ino_to_inode (fuse_ino, this);
+
+        rv = write(priv->revchan_out, inval_buf, fouh->len);
+        if (rv != fouh->len) {
+                gf_log("glusterfs-fuse", GF_LOG_ERROR, "kernel notification "
+                        "daemon defunct");
+                close(priv->fd);
+        }
+
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "INVALIDATE inode: %lu", fuse_ino);
+
+        if (inode) {
+                fuse_log_eh (this, "Invalidated inode %lu (gfid: %s)",
+                             fuse_ino, uuid_utoa (inode->gfid));
+        } else {
+                fuse_log_eh (this, "Invalidated inode %lu ", fuse_ino);
+        }
+
+        if (inode)
+                inode_unref (inode);
 }
 
 int
@@ -192,41 +345,80 @@ send_fuse_err (xlator_t *this, fuse_in_header_t *finh, int error)
 {
         struct fuse_out_header fouh = {0, };
         struct iovec iov_out;
+        inode_t  *inode = NULL;
 
         fouh.error = -error;
         iov_out.iov_base = &fouh;
 
+        inode = fuse_ino_to_inode (finh->nodeid, this);
+
+        // filter out ENOENT
+        if (error != ENOENT) {
+                if (inode) {
+                        fuse_log_eh (this,"Sending %s for operation %d on "
+                                     "inode %s", strerror (error), finh->opcode,
+                                     uuid_utoa (inode->gfid));
+                } else {
+                        fuse_log_eh (this, "Sending %s for operation %d on "
+                                     "inode %ld", strerror (error),
+                                     finh->opcode, finh->nodeid);
+                }
+        }
+
+        if (inode)
+                inode_unref (inode);
+
         return send_fuse_iov (this, finh, &iov_out, 1);
 }
 
-
 static int
 fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
-                inode_t *inode, struct iatt *buf)
+                inode_t *inode, struct iatt *buf, dict_t *xdata)
 {
-        fuse_state_t            *state = NULL;
-        fuse_in_header_t        *finh = NULL;
-        struct fuse_entry_out    feo = {0, };
-        fuse_private_t          *priv = NULL;
-        inode_t                 *linked_inode = NULL;
+        fuse_state_t          *state        = NULL;
+        fuse_in_header_t      *finh         = NULL;
+        struct fuse_entry_out  feo          = {0, };
+        fuse_private_t        *priv         = NULL;
+        inode_t               *linked_inode = NULL;
 
         priv = this->private;
         state = frame->root->state;
         finh = state->finh;
 
-        if (!op_ret && __is_root_gfid (state->loc.inode->gfid)) {
-                buf->ia_ino = 1;
+        if (op_ret == 0) {
+                if (__is_root_gfid (state->loc.inode->gfid))
+                        buf->ia_ino = 1;
+                if (uuid_is_null (buf->ia_gfid)) {
+                        /* With a NULL gfid inode linking is
+                           not possible. Let's not pretend this
+                           call was a "success".
+                        */
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "Received NULL gfid for %s. Forcing EIO",
+                                state->loc.path);
+                        op_ret = -1;
+                        op_errno = EIO;
+                }
         }
 
+        /* log into the event-history after the null uuid check is done, since
+         * the op_ret and op_errno are being changed if the gfid is NULL.
+         */
+        fuse_log_eh (this, "op_ret: %d op_errno: %d "
+                     "%"PRIu64": %s() %s => %s", op_ret, op_errno,
+                     frame->root->unique, gf_fop_list[frame->root->op],
+                     state->loc.path, (op_ret == 0)?
+                     uuid_utoa(buf->ia_gfid):uuid_utoa(state->loc.gfid));
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRId64,
+                        "%"PRIu64": %s() %s => %"PRIu64,
                         frame->root->unique, gf_fop_list[frame->root->op],
                         state->loc.path, buf->ia_ino);
 
                 buf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (buf, &feo.attr);
+                gf_fuse_stat2attr (buf, &feo.attr, priv->enable_ino32);
 
                 if (!buf->ia_ino) {
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
@@ -270,7 +462,16 @@ fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
                         gf_fop_list[frame->root->op], state->loc.path,
                         strerror (op_errno));
-                send_fuse_err (this, state->finh, op_errno);
+
+		if ((op_errno == ENOENT) && (priv->negative_timeout != 0)) {
+			feo.entry_valid =
+				calc_timeout_sec (priv->negative_timeout);
+			feo.entry_valid_nsec =
+				calc_timeout_nsec (priv->negative_timeout);
+			send_fuse_obj (this, finh, &feo);
+		} else {
+			send_fuse_err (this, state->finh, op_errno);
+                }
         }
 
         free_fuse_state (state);
@@ -278,18 +479,17 @@ fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 static int
 fuse_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
                    inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
-        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, buf);
+        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, buf,
+                        xdata);
         return 0;
 }
 
-
 static int
 fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
@@ -305,6 +505,13 @@ fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (op_ret == -1 && state->is_revalidate == 1) {
                 itable = state->itable;
+		/*
+		 * A stale mapping might exist for a dentry/inode that has been
+		 * removed from another client.
+		 */
+		if (op_errno == ENOENT)
+			inode_unlink(state->loc.inode, state->loc.parent,
+				     state->loc.name);
                 inode_unref (state->loc.inode);
                 state->loc.inode = inode_new (itable);
                 state->is_revalidate = 2;
@@ -314,11 +521,12 @@ fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STACK_WIND (frame, fuse_lookup_cbk,
                             prev->this, prev->this->fops->lookup,
-                            &state->loc, state->dict);
+                            &state->loc, state->xdata);
                 return 0;
         }
 
-        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, stat);
+        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, stat,
+                        dict);
         return 0;
 }
 
@@ -333,6 +541,14 @@ fuse_lookup_resume (fuse_state_t *state)
                 return;
         }
 
+        /* parent was resolved, entry could not, may be a missing gfid?
+         * Hence try to do a regular lookup
+         */
+        if ((state->resolve.op_ret == -1)
+            && (state->resolve.op_errno == ENODATA)) {
+                state->resolve.op_ret = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": LOOKUP %s(%s)", state->finh->unique,
@@ -343,35 +559,48 @@ fuse_lookup_resume (fuse_state_t *state)
                         "%"PRIu64": LOOKUP %s", state->finh->unique,
                         state->loc.path);
                 state->loc.inode = inode_new (state->loc.parent->table);
+                if (uuid_is_null (state->gfid))
+                        uuid_generate (state->gfid);
+                fuse_gfid_set (state);
         }
 
         FUSE_FOP (state, fuse_lookup_cbk, GF_FOP_LOOKUP,
-                  lookup, &state->loc, state->dict);
+                  lookup, &state->loc, state->xdata);
 }
 
 static void
 fuse_lookup (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        char         *name      = msg;
-        fuse_state_t *state     = NULL;
-        int32_t       ret       = -1;
+        char           *name     = msg;
+        fuse_state_t   *state    = NULL;
 
         GET_STATE (this, finh, state);
 
-        ret = fuse_resolve_entry_init (state, &state->resolve,
-				       finh->nodeid, name);
+        (void) fuse_resolve_entry_init (state, &state->resolve,
+                                        finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_lookup_resume);
+
+        return;
 }
 
+static inline void
+do_forget(xlator_t *this, uint64_t unique, uint64_t nodeid, uint64_t nlookup)
+{
+	inode_t *fuse_inode = fuse_ino_to_inode(nodeid, this);
+
+	fuse_log_eh(this, "%"PRIu64": FORGET %"PRIu64"/%"PRIu64" gfid: (%s)",
+		    unique, nodeid, nlookup, uuid_utoa(fuse_inode->gfid));
+
+	inode_forget(fuse_inode, nlookup);
+	inode_unref(fuse_inode);
+}
 
 static void
 fuse_forget (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
 {
-        struct fuse_forget_in *ffi = msg;
-
-        inode_t      *fuse_inode;
+        struct fuse_forget_in *ffi        = msg;
 
         if (finh->nodeid == 1) {
                 GF_FREE (finh);
@@ -382,19 +611,35 @@ fuse_forget (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 "%"PRIu64": FORGET %"PRIu64"/%"PRIu64,
                 finh->unique, finh->nodeid, ffi->nlookup);
 
-        fuse_inode = fuse_ino_to_inode (finh->nodeid, this);
-
-        inode_forget (fuse_inode, ffi->nlookup);
-        inode_unref (fuse_inode);
+	do_forget(this, finh->unique, finh->nodeid, ffi->nlookup);
 
         GF_FREE (finh);
 }
 
+static void
+fuse_batch_forget(xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+	struct fuse_batch_forget_in *fbfi = msg;
+	struct fuse_forget_one *ffo = (struct fuse_forget_one *) (fbfi + 1);
+	int i;
+
+	gf_log("glusterfs-fuse", GF_LOG_TRACE,
+		"%"PRIu64": BATCH_FORGET %"PRIu64"/%"PRIu32,
+		finh->unique, finh->nodeid, fbfi->count);
+
+	for (i = 0; i < fbfi->count; i++) {
+                if (ffo[i].nodeid == 1)
+                        continue;
+		do_forget(this, finh->unique, ffo[i].nodeid, ffo[i].nlookup);
+        }
+
+	GF_FREE(finh);
+}
 
 static int
 fuse_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         fuse_state_t     *state;
         fuse_in_header_t *finh;
@@ -405,15 +650,17 @@ fuse_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh  = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRId64, frame->root->unique,
+                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
                         gf_fop_list[frame->root->op],
                         state->loc.path ? state->loc.path : "ERR",
                         prebuf->ia_ino);
 
                 postbuf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (postbuf, &fao.attr);
+                gf_fuse_stat2attr (postbuf, &fao.attr, priv->enable_ino32);
 
                 fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
                 fao.attr_valid_nsec =
@@ -443,10 +690,9 @@ fuse_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 static int
 fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf)
+               int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         fuse_state_t     *state;
         fuse_in_header_t *finh;
@@ -457,15 +703,19 @@ fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh  = state->finh;
 
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() %s => "
+                   "gfid: %s", op_ret, op_errno, frame->root->unique,
+                   gf_fop_list[frame->root->op], state->loc.path,
+                   state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRId64, frame->root->unique,
+                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
                         gf_fop_list[frame->root->op],
                         state->loc.path ? state->loc.path : "ERR",
                         buf->ia_ino);
 
                 buf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (buf, &fao.attr);
+                gf_fuse_stat2attr (buf, &fao.attr, priv->enable_ino32);
 
                 fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
                 fao.attr_valid_nsec =
@@ -480,9 +730,9 @@ fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 send_fuse_obj (this, finh, &fao);
 #endif
         } else {
-                GF_LOG_OCCASIONALLY ( gf_fuse_conn_err_log, "glusterfs-fuse", 
-                                      GF_LOG_WARNING, 
-                                      "%"PRIu64": %s() %s => -1 (%s)", 
+                GF_LOG_OCCASIONALLY ( gf_fuse_conn_err_log, "glusterfs-fuse",
+                                      GF_LOG_WARNING,
+                                      "%"PRIu64": %s() %s => -1 (%s)",
                                       frame->root->unique,
                                       gf_fop_list[frame->root->op],
                                       state->loc.path ? state->loc.path : "ERR",
@@ -497,14 +747,13 @@ fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 static int
 fuse_root_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
                       inode_t *inode, struct iatt *stat, dict_t *dict,
                       struct iatt *postparent)
 {
-        fuse_attr_cbk (frame, cookie, this, op_ret, op_errno, stat);
+        fuse_attr_cbk (frame, cookie, this, op_ret, op_errno, stat, dict);
 
         return 0;
 }
@@ -516,15 +765,15 @@ fuse_getattr_resume (fuse_state_t *state)
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "%"PRIu64": GETATTR %"PRIu64" (%s) resolution failed",
                         state->finh->unique, state->finh->nodeid,
-			uuid_utoa (state->resolve.gfid));
+                        uuid_utoa (state->resolve.gfid));
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
         }
 
-	if (!IA_ISDIR (state->loc.inode->ia_type)) {
-		state->fd = fd_lookup (state->loc.inode, state->finh->pid);
-	}
+        if (!IA_ISDIR (state->loc.inode->ia_type)) {
+                state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+        }
 
         if (!state->fd) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -533,7 +782,7 @@ fuse_getattr_resume (fuse_state_t *state)
                         state->loc.path);
 
                 FUSE_FOP (state, fuse_attr_cbk, GF_FOP_STAT,
-                          stat, &state->loc);
+                          stat, &state->loc, state->xdata);
         } else {
 
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -542,7 +791,7 @@ fuse_getattr_resume (fuse_state_t *state)
                         state->loc.path, state->fd);
 
                 FUSE_FOP (state, fuse_attr_cbk, GF_FOP_FSTAT,
-                          fstat, state->fd);
+                          fstat, state->fd, state->xdata);
         }
 }
 
@@ -570,7 +819,7 @@ fuse_getattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 fuse_gfid_set (state);
 
                 FUSE_FOP (state, fuse_root_lookup_cbk, GF_FOP_LOOKUP,
-                          lookup, &state->loc, state->dict);
+                          lookup, &state->loc, state->xdata);
                 return;
         }
 
@@ -579,14 +828,12 @@ fuse_getattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_resolve_and_resume (state, fuse_getattr_resume);
 }
 
-
 static int32_t
 fuse_fd_inherit_directio (xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
 {
         int32_t        ret    = 0;
         fuse_fd_ctx_t *fdctx  = NULL, *tmp_fdctx = NULL;
         fd_t          *tmp_fd = NULL;
-        uint64_t       val    = 0;
 
         GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", this, out, ret,
                                         -EINVAL);
@@ -595,7 +842,7 @@ fuse_fd_inherit_directio (xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
         GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", foo, out, ret,
                                         -EINVAL);
 
-        fdctx = fuse_fd_ctx_check_n_create (fd, this);
+        fdctx = fuse_fd_ctx_get (this, fd);
         if (!fdctx) {
                 ret = -ENOMEM;
                 goto out;
@@ -603,14 +850,11 @@ fuse_fd_inherit_directio (xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
 
         tmp_fd = fd_lookup (fd->inode, 0);
         if (tmp_fd) {
-                ret = fd_ctx_get (tmp_fd, this, &val);
-                if (!ret) {
-                        tmp_fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
-                        if (tmp_fdctx) {
-                                foo->open_flags &= ~FOPEN_DIRECT_IO;
-                                foo->open_flags |= (tmp_fdctx->open_flags
-                                                    & FOPEN_DIRECT_IO);
-                        }
+                tmp_fdctx = fuse_fd_ctx_get (this, tmp_fd);
+                if (tmp_fdctx) {
+                        foo->open_flags &= ~FOPEN_DIRECT_IO;
+                        foo->open_flags |= (tmp_fdctx->open_flags
+                                            & FOPEN_DIRECT_IO);
                 }
         }
 
@@ -625,21 +869,22 @@ out:
         return ret;
 }
 
-
 static int
 fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd)
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        fuse_state_t         *state = NULL;
-        fuse_in_header_t     *finh  = NULL;
-        fuse_private_t       *priv  = NULL;
-        int32_t               ret   = 0;
-        struct fuse_open_out  foo   = {0, };
+        fuse_state_t         *state  = NULL;
+        fuse_in_header_t     *finh   = NULL;
+        fuse_private_t       *priv   = NULL;
+        int32_t               ret    = 0;
+        struct fuse_open_out  foo    = {0, };
 
         priv = this->private;
         state = frame->root->state;
         finh = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret >= 0) {
                 foo.fh = (uintptr_t) fd;
                 foo.open_flags = 0;
@@ -650,17 +895,27 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                             || (priv->direct_io_mode == 1))
                                 foo.open_flags |= FOPEN_DIRECT_IO;
 #ifdef GF_DARWIN_HOST_OS
-                                /* In Linux: by default, buffer cache
-                                 * is purged upon open, setting
-                                 * FOPEN_KEEP_CACHE implies no-purge
-                                 *
-                                 * In MacFUSE: by default, buffer cache
-                                 * is left intact upon open, setting
-                                 * FOPEN_PURGE_UBC implies purge
-                                 *
-                                 * [[Interesting...]]
-                                 */
+                        /* In Linux: by default, buffer cache
+                         * is purged upon open, setting
+                         * FOPEN_KEEP_CACHE implies no-purge
+                         *
+                         * In MacFUSE: by default, buffer cache
+                         * is left intact upon open, setting
+                         * FOPEN_PURGE_UBC implies purge
+                         *
+                         * [[Interesting...]]
+                         */
+                        if (!priv->fopen_keep_cache)
                                 foo.open_flags |= FOPEN_PURGE_UBC;
+#else
+                        /*
+                         * If fopen-keep-cache is enabled, we set the associated
+                         * flag here such that files are not invalidated on open.
+                         * File invalidations occur either in fuse or explicitly
+                         * when the cache is set invalid on the inode.
+                         */
+                        if (priv->fopen_keep_cache)
+                                foo.open_flags |= FOPEN_KEEP_CACHE;
 #endif
                 }
 
@@ -672,17 +927,16 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (ret < 0) {
                         op_errno = -ret;
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "cannot inherit direct-io values from fds "
-                                "already opened");
+                                "cannot inherit direct-io values for fd "
+                                "(ptr:%p inode-gfid:%s) from fds already "
+                                "opened", fd, uuid_utoa (fd->inode->gfid));
                         goto err;
                 }
 
-                fd_ref (fd);
-
                 if (send_fuse_obj (this, finh, &foo) == ENOENT) {
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                                 "open(%s) got EINTR", state->loc.path);
-                        fd_unref (fd);
+                        gf_fd_put (priv->fdtable, state->fd_no);
                         goto out;
                 }
 
@@ -695,6 +949,7 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         strerror (op_errno));
 
                 send_fuse_err (this, finh, op_errno);
+                gf_fd_put (priv->fdtable, state->fd_no);
         }
 out:
         free_fuse_state (state);
@@ -702,26 +957,24 @@ out:
         return 0;
 }
 
-
 static void
 fuse_do_truncate (fuse_state_t *state, size_t size)
 {
         if (state->fd) {
                 FUSE_FOP (state, fuse_truncate_cbk, GF_FOP_FTRUNCATE,
-                          ftruncate, state->fd, size);
+                          ftruncate, state->fd, size, state->xdata);
         } else {
                 FUSE_FOP (state, fuse_truncate_cbk, GF_FOP_TRUNCATE,
-                          truncate, &state->loc, size);
+                          truncate, &state->loc, size, state->xdata);
         }
 
         return;
 }
 
-
 static int
 fuse_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *statpre, struct iatt *statpost)
+                  struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
         fuse_state_t     *state;
         fuse_in_header_t *finh;
@@ -734,15 +987,20 @@ fuse_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh  = state->finh;
 
+        fuse_log_eh(this, "op_ret: %d, op_errno: %d, %"PRIu64", %s() %s => "
+                    "gfid: %s", op_ret, op_errno, frame->root->unique,
+                    gf_fop_list[frame->root->op], state->loc.path,
+                    state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRId64, frame->root->unique,
+                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
                         gf_fop_list[frame->root->op],
                         state->loc.path ? state->loc.path : "ERR",
                         statpost->ia_ino);
 
                 statpost->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (statpost, &fao.attr);
+                gf_fuse_stat2attr (statpost, &fao.attr, priv->enable_ino32);
 
                 fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
                 fao.attr_valid_nsec =
@@ -781,7 +1039,6 @@ fuse_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 static int32_t
 fattr_to_gf_set_attr (int32_t valid)
 {
@@ -808,7 +1065,6 @@ fattr_to_gf_set_attr (int32_t valid)
         return gf_valid;
 }
 
-
 #define FATTR_MASK   (FATTR_SIZE                        \
                       | FATTR_UID | FATTR_GID           \
                       | FATTR_ATIME | FATTR_MTIME       \
@@ -821,7 +1077,7 @@ fuse_setattr_resume (fuse_state_t *state)
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "%"PRIu64": SETATTR %"PRIu64" (%s) resolution failed",
                         state->finh->unique, state->finh->nodeid,
-			uuid_utoa (state->resolve.gfid));
+                        uuid_utoa (state->resolve.gfid));
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -848,11 +1104,13 @@ fuse_setattr_resume (fuse_state_t *state)
 
                         FUSE_FOP (state, fuse_setattr_cbk, GF_FOP_FSETATTR,
                                   fsetattr, state->fd, &state->attr,
-                                  fattr_to_gf_set_attr (state->valid));
+                                  fattr_to_gf_set_attr (state->valid),
+                                  state->xdata);
                 } else {
                         FUSE_FOP (state, fuse_setattr_cbk, GF_FOP_SETATTR,
                                   setattr, &state->loc, &state->attr,
-                                  fattr_to_gf_set_attr (state->valid));
+                                  fattr_to_gf_set_attr (state->valid),
+                                  state->xdata);
                 }
         } else {
                 fuse_do_truncate (state, state->size);
@@ -875,10 +1133,10 @@ fuse_setattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 /* We need no loc if kernel sent us an fd and
                  * we are not fiddling with times */
                 state->fd = FH_TO_FD (fsi->fh);
-		fuse_resolve_fd_init (state, &state->resolve, state->fd);
-	} else {
-		fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
-	}
+                fuse_resolve_fd_init (state, &state->resolve, state->fd);
+        } else {
+                fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        }
 
         /*
          * This is just stub code demonstrating how to retrieve
@@ -923,14 +1181,15 @@ fuse_setattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_resolve_and_resume (state, fuse_setattr_resume);
 }
 
-
 static int
 fuse_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         fuse_state_t *state = frame->root->state;
         fuse_in_header_t *finh = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": %s() %s => 0", frame->root->unique,
@@ -955,19 +1214,17 @@ fuse_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 static int
 fuse_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                struct iatt *postbuf)
+                struct iatt *postbuf, dict_t *xdata)
 {
-        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno);
+        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 }
 
-
 static int
 fuse_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         if (op_ret == -1 && op_errno == ENOTSUP)
                 GF_LOG_OCCASIONALLY (gf_fuse_xattr_enotsup_log,
@@ -975,14 +1232,13 @@ fuse_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                      "extended attribute not supported "
                                      "by the backend storage");
 
-        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno);
+        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 }
 
-
 static int
 fuse_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         fuse_state_t     *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -990,11 +1246,14 @@ fuse_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh = state->finh;
 
-        if (op_ret == 0)
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() %s => "
+                     "gfid: %s", op_ret, op_errno, frame->root->unique,
+                     gf_fop_list[frame->root->op], state->loc.path,
+                     state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
 
         if (op_ret == 0) {
+                inode_unlink (state->loc.inode, state->loc.parent,
+                              state->loc.name);
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": %s() %s => 0", frame->root->unique,
                         gf_fop_list[frame->root->op], state->loc.path);
@@ -1016,7 +1275,6 @@ fuse_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 void
 fuse_access_resume (fuse_state_t *state)
 {
@@ -1024,7 +1282,7 @@ fuse_access_resume (fuse_state_t *state)
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "%"PRIu64": ACCESS %"PRIu64" (%s) resolution failed",
                         state->finh->unique, state->finh->nodeid,
-			uuid_utoa (state->resolve.gfid));
+                        uuid_utoa (state->resolve.gfid));
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -1036,10 +1294,9 @@ fuse_access_resume (fuse_state_t *state)
                 state->finh->nodeid, state->mask);
 
         FUSE_FOP (state, fuse_err_cbk, GF_FOP_ACCESS, access,
-                  &state->loc, state->mask);
+                  &state->loc, state->mask, state->xdata);
 }
 
-
 static void
 fuse_access (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1057,11 +1314,10 @@ fuse_access (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 static int
 fuse_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, const char *linkname,
-                   struct iatt *buf)
+                   struct iatt *buf, dict_t *xdata)
 {
         fuse_state_t     *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -1069,6 +1325,12 @@ fuse_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh = state->finh;
 
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d %"PRIu64": %s() => %s"
+                     " linkname: %s, gfid: %s", op_ret, op_errno,
+                     frame->root->unique, gf_fop_list[frame->root->op],
+                     state->loc.gfid, linkname,
+                     uuid_utoa (state->loc.gfid));
+
         if (op_ret > 0) {
                 ((char *)linkname)[op_ret] = '\0';
 
@@ -1091,7 +1353,6 @@ fuse_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 void
 fuse_readlink_resume (fuse_state_t *state)
 {
@@ -1109,10 +1370,9 @@ fuse_readlink_resume (fuse_state_t *state)
                 state->loc.path, uuid_utoa (state->loc.inode->gfid));
 
         FUSE_FOP (state, fuse_readlink_cbk, GF_FOP_READLINK,
-                  readlink, &state->loc, 4096);
+                  readlink, &state->loc, 4096, state->xdata);
 }
 
-
 static void
 fuse_readlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1120,31 +1380,35 @@ fuse_readlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         fuse_resolve_and_resume (state, fuse_readlink_resume);
 
         return;
 }
 
-
 void
 fuse_mknod_resume (fuse_state_t *state)
 {
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"MKNOD %"PRId64"/%s (%s/%s) resolution failed",
+                        "MKNOD %"PRIu64"/%s (%s/%s) resolution failed",
                         state->finh->nodeid, state->resolve.bname,
-			uuid_utoa (state->resolve.gfid), state->resolve.bname);
+                        uuid_utoa (state->resolve.gfid), state->resolve.bname);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
-		state->loc.inode = NULL;
+                state->loc.inode = NULL;
         }
 
         state->loc.inode = inode_new (state->loc.parent->table);
@@ -1154,10 +1418,10 @@ fuse_mknod_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_MKNOD,
-                  mknod, &state->loc, state->mode, state->rdev, state->dict);
+                  mknod, &state->loc, state->mode, state->rdev, state->umask,
+                  state->xdata);
 }
 
-
 static void
 fuse_mknod (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1178,43 +1442,14 @@ fuse_mknod (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         uuid_generate (state->gfid);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         state->mode = fmi->mode;
         state->rdev = fmi->rdev;
 
         priv = this->private;
 #if FUSE_KERNEL_MINOR_VERSION >=12
-        if (priv->proto_minor >= 12)
-                state->mode &= ~fmi->umask;
-        if (priv->proto_minor >= 12 && priv->acl) {
-                state->dict = dict_new ();
-                if (!state->dict) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKNOD Failed to allocate a param dictionary");
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "umask", fmi->umask);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKNOD Failed adding umask to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "mode", fmi->mode);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKNOD Failed adding mode to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-        }
+        FUSE_ENTRY_CREATE(this, priv, finh, state, fmi, "MKNOD");
 #endif
 
         fuse_resolve_and_resume (state, fuse_mknod_resume);
@@ -1222,24 +1457,28 @@ fuse_mknod (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 void
 fuse_mkdir_resume (fuse_state_t *state)
 {
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"MKDIR %"PRId64" (%s/%s) resolution failed",
+                        "MKDIR %"PRIu64" (%s/%s) resolution failed",
                         state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-			state->resolve.bname);
+                        state->resolve.bname);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
-		state->loc.inode = NULL;
+                state->loc.inode = NULL;
         }
 
         state->loc.inode = inode_new (state->loc.parent->table);
@@ -1249,10 +1488,9 @@ fuse_mkdir_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_MKDIR,
-                  mkdir, &state->loc, state->mode, state->dict);
+                  mkdir, &state->loc, state->mode, state->umask, state->xdata);
 }
 
-
 static void
 fuse_mkdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1267,42 +1505,13 @@ fuse_mkdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         uuid_generate (state->gfid);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         state->mode = fmi->mode;
 
         priv = this->private;
 #if FUSE_KERNEL_MINOR_VERSION >=12
-        if (priv->proto_minor >= 12)
-                state->mode &= ~fmi->umask;
-        if (priv->proto_minor >= 12 && priv->acl) {
-                state->dict = dict_new ();
-                if (!state->dict) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKDIR Failed to allocate a param dictionary");
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "umask", fmi->umask);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKDIR Failed adding umask to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "mode", fmi->mode);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKDIR Failed adding mode to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-        }
+        FUSE_ENTRY_CREATE(this, priv, finh, state, fmi, "MKDIR");
 #endif
 
         fuse_resolve_and_resume (state, fuse_mkdir_resume);
@@ -1310,15 +1519,14 @@ fuse_mkdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 void
 fuse_unlink_resume (fuse_state_t *state)
 {
         if (!state->loc.parent || !state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"UNLINK %"PRId64" (%s/%s) resolution failed",
-			state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-			state->resolve.bname);
+                        "UNLINK %"PRIu64" (%s/%s) resolution failed",
+                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
+                        state->resolve.bname);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -1329,10 +1537,9 @@ fuse_unlink_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_unlink_cbk, GF_FOP_UNLINK,
-                  unlink, &state->loc);
+                  unlink, &state->loc, 0, state->xdata);
 }
 
-
 static void
 fuse_unlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1341,7 +1548,7 @@ fuse_unlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_unlink_resume);
 
@@ -1353,9 +1560,9 @@ fuse_rmdir_resume (fuse_state_t *state)
 {
         if (!state->loc.parent || !state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"RMDIR %"PRId64" (%s/%s) resolution failed",
-			state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-			state->resolve.bname);
+                        "RMDIR %"PRIu64" (%s/%s) resolution failed",
+                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
+                        state->resolve.bname);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -1366,10 +1573,9 @@ fuse_rmdir_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_unlink_cbk, GF_FOP_RMDIR,
-                  rmdir, &state->loc, 0);
+                  rmdir, &state->loc, 0, state->xdata);
 }
 
-
 static void
 fuse_rmdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1378,31 +1584,35 @@ fuse_rmdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_rmdir_resume);
 
         return;
 }
 
-
 void
 fuse_symlink_resume (fuse_state_t *state)
 {
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"SYMLINK %"PRId64" (%s/%s) -> %s resolution failed",
+                        "SYMLINK %"PRIu64" (%s/%s) -> %s resolution failed",
                         state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-			state->resolve.bname, state->name);
+                        state->resolve.bname, state->name);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
-		state->loc.inode = NULL;
+                state->loc.inode = NULL;
         }
 
         state->loc.inode = inode_new (state->loc.parent->table);
@@ -1412,10 +1622,9 @@ fuse_symlink_resume (fuse_state_t *state)
                 state->loc.path, state->name);
 
         FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_SYMLINK,
-                  symlink, state->name, &state->loc, state->dict);
+                  symlink, state->name, &state->loc, state->umask, state->xdata);
 }
 
-
 static void
 fuse_symlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1427,7 +1636,7 @@ fuse_symlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         uuid_generate (state->gfid);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         state->name = gf_strdup (linkname);
 
@@ -1436,12 +1645,12 @@ fuse_symlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 int
 fuse_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
                  struct iatt *preoldparent, struct iatt *postoldparent,
-                 struct iatt *prenewparent, struct iatt *postnewparent)
+                 struct iatt *prenewparent, struct iatt *postnewparent,
+                 dict_t *xdata)
 {
         fuse_state_t     *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -1449,9 +1658,18 @@ fuse_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh  = state->finh;
 
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() "
+                     "path: %s parent: %s ==> path: %s parent: %s"
+                     "gfid: %s", op_ret, op_errno, frame->root->unique,
+                     gf_fop_list[frame->root->op], state->loc.path,
+                     state->loc.parent?uuid_utoa (state->loc.parent->gfid):"",
+                     state->loc2.path,
+                     state->loc2.parent?uuid_utoa (state->loc2.parent->gfid):"",
+                     state->loc.inode?uuid_utoa (state->loc.inode->gfid):"");
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s -> %s => 0 (buf->ia_ino=%"PRId64")",
+                        "%"PRIu64": %s -> %s => 0 (buf->ia_ino=%"PRIu64")",
                         frame->root->unique, state->loc.path, state->loc2.path,
                         buf->ia_ino);
 
@@ -1491,11 +1709,11 @@ fuse_rename_resume (fuse_state_t *state)
         if (!state->loc.parent || !state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "RENAME %"PRIu64" %s/%s -> %s/%s src resolution failed",
-			state->finh->unique,
-			uuid_utoa_r (state->resolve.gfid, loc_uuid),
-			state->resolve.bname,
-			uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
-			state->resolve2.bname);
+                        state->finh->unique,
+                        uuid_utoa_r (state->resolve.gfid, loc_uuid),
+                        state->resolve.bname,
+                        uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
+                        state->resolve2.bname);
 
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
@@ -1505,27 +1723,29 @@ fuse_rename_resume (fuse_state_t *state)
         if (!state->loc2.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "RENAME %"PRIu64" %s/%s -> %s/%s dst resolution failed",
-			state->finh->unique,
-			uuid_utoa_r (state->resolve.gfid, loc_uuid),
-			state->resolve.bname,
-			uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
-			state->resolve2.bname);
+                        state->finh->unique,
+                        uuid_utoa_r (state->resolve.gfid, loc_uuid),
+                        state->resolve.bname,
+                        uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
+                        state->resolve2.bname);
 
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
         }
 
+        state->resolve.op_ret = 0;
+        state->resolve2.op_ret = 0;
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RENAME `%s (%s)' -> `%s (%s)'",
                 state->finh->unique, state->loc.path, loc_uuid,
                 state->loc2.path, loc2_uuid);
 
         FUSE_FOP (state, fuse_rename_cbk, GF_FOP_RENAME,
-                  rename, &state->loc, &state->loc2);
+                  rename, &state->loc, &state->loc2, state->xdata);
 }
 
-
 static void
 fuse_rename (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1536,16 +1756,15 @@ fuse_rename (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, oldname);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, oldname);
 
-	fuse_resolve_entry_init (state, &state->resolve2, fri->newdir, newname);
+        fuse_resolve_entry_init (state, &state->resolve2, fri->newdir, newname);
 
         fuse_resolve_and_resume (state, fuse_rename_resume);
 
         return;
 }
 
-
 void
 fuse_link_resume (fuse_state_t *state)
 {
@@ -1558,11 +1777,14 @@ fuse_link_resume (fuse_state_t *state)
                 return;
         }
 
-	if (state->loc.inode) {
-		inode_unref (state->loc.inode);
-		state->loc.inode = NULL;
-	}
-	state->loc.inode = inode_ref (state->loc2.inode);
+        state->resolve.op_ret = 0;
+        state->resolve2.op_ret = 0;
+
+        if (state->loc.inode) {
+                inode_unref (state->loc.inode);
+                state->loc.inode = NULL;
+        }
+        state->loc.inode = inode_ref (state->loc2.inode);
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": LINK() %s -> %s",
@@ -1570,10 +1792,9 @@ fuse_link_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_LINK,
-                  link, &state->loc2, &state->loc);
+                  link, &state->loc2, &state->loc, state->xdata);
 }
 
-
 static void
 fuse_link (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1583,36 +1804,37 @@ fuse_link (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve2, fli->oldnodeid);
+        fuse_resolve_inode_init (state, &state->resolve2, fli->oldnodeid);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_link_resume);
 
         return;
 }
 
-
 static int
 fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  fd_t *fd, inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        fuse_state_t            *state = NULL;
-        fuse_in_header_t        *finh = NULL;
-        fuse_private_t          *priv = NULL;
-        struct fuse_out_header   fouh = {0, };
-        struct fuse_entry_out    feo = {0, };
-        struct fuse_open_out     foo = {0, };
-        struct iovec             iov_out[3];
-        inode_t                 *linked_inode = NULL;
+        fuse_state_t           *state        = NULL;
+        fuse_in_header_t       *finh         = NULL;
+        fuse_private_t         *priv         = NULL;
+        struct fuse_out_header  fouh         = {0, };
+        struct fuse_entry_out   feo          = {0, };
+        struct fuse_open_out    foo          = {0, };
+        struct iovec            iov_out[3];
+        inode_t                *linked_inode = NULL;
 
         state    = frame->root->state;
         priv     = this->private;
         finh     = state->finh;
         foo.open_flags = 0;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret >= 0) {
                 foo.fh = (uintptr_t) fd;
 
@@ -1622,12 +1844,12 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         foo.open_flags |= FOPEN_DIRECT_IO;
 
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %p (ino=%"PRId64")",
+                        "%"PRIu64": %s() %s => %p (ino=%"PRIu64")",
                         frame->root->unique, gf_fop_list[frame->root->op],
                         state->loc.path, fd, buf->ia_ino);
 
                 buf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (buf, &feo.attr);
+                gf_fuse_stat2attr (buf, &feo.attr, priv->enable_ino32);
 
                 linked_inode = inode_link (inode, state->loc.parent,
                                            state->loc.name, buf);
@@ -1645,8 +1867,6 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 inode_unref (linked_inode);
 
-                fd_ref (fd);
-
                 feo.nodeid = inode_to_fuse_nodeid (linked_inode);
 
                 feo.entry_valid = calc_timeout_sec (priv->entry_timeout);
@@ -1667,11 +1887,12 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 #endif
                 iov_out[2].iov_base = &foo;
                 iov_out[2].iov_len = sizeof (foo);
+
                 if (send_fuse_iov (this, finh, iov_out, 3) == ENOENT) {
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                                 "create(%s) got EINTR", state->loc.path);
                         inode_forget (inode, 1);
-                        fd_unref (fd);
+                        gf_fd_put (priv->fdtable, state->fd_no);
                         goto out;
                 }
 
@@ -1681,6 +1902,7 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%"PRIu64": %s => -1 (%s)", finh->unique,
                         state->loc.path, strerror (op_errno));
                 send_fuse_err (this, finh, op_errno);
+                gf_fd_put (priv->fdtable, state->fd_no);
         }
 out:
         free_fuse_state (state);
@@ -1689,31 +1911,62 @@ out:
         return 0;
 }
 
-
 void
 fuse_create_resume (fuse_state_t *state)
 {
-        fd_t *fd = NULL;
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
 
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                         "%"PRIu64" CREATE %s/%s resolution failed",
                         state->finh->unique, uuid_utoa (state->resolve.gfid),
-			state->resolve.bname);
+                        state->resolve.bname);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
-                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
+                gf_log (state->this->name, GF_LOG_DEBUG,
+                        "inode already present");
                 inode_unref (state->loc.inode);
         }
 
         state->loc.inode = inode_new (state->loc.parent->table);
 
         fd = fd_create (state->loc.inode, state->finh->pid);
-        state->fd = fd;
+        if (fd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64" CREATE cannot create a new fd",
+                        state->finh->unique);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64" CREATE creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
+        priv = state->this->private;
+
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+
+        state->fd = fd_ref (fd);
         fd->flags = state->flags;
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -1722,11 +1975,10 @@ fuse_create_resume (fuse_state_t *state)
 
         FUSE_FOP (state, fuse_create_cbk, GF_FOP_CREATE,
                   create, &state->loc, state->flags, state->mode,
-                  fd, state->dict);
+                  state->umask, fd, state->xdata);
 
 }
 
-
 static void
 fuse_create (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1751,55 +2003,26 @@ fuse_create (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         uuid_generate (state->gfid);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         state->mode = fci->mode;
         state->flags = fci->flags;
 
         priv = this->private;
 #if FUSE_KERNEL_MINOR_VERSION >=12
-        if (priv->proto_minor >= 12)
-                state->mode &= ~fci->umask;
-        if (priv->proto_minor >= 12 && priv->acl) {
-                state->dict = dict_new ();
-                if (!state->dict) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "CREATE Failed to allocate a param dictionary");
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "umask", fci->umask);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "CREATE Failed adding umask to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "mode", fci->mode);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "CREATE Failed adding mode to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-        }
+        FUSE_ENTRY_CREATE(this, priv, finh, state, fci, "CREATE");
 #endif
-
         fuse_resolve_and_resume (state, fuse_create_resume);
 
         return;
 }
 
-
 void
 fuse_open_resume (fuse_state_t *state)
 {
-        fd_t *fd = NULL;
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
 
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
@@ -1820,7 +2043,21 @@ fuse_open_resume (fuse_state_t *state)
                 return;
         }
 
-        state->fd = fd;
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPEN creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
+        priv = state->this->private;
+
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+        state->fd = fd_ref (fd);
         fd->flags = state->flags;
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -1828,10 +2065,9 @@ fuse_open_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_fd_cbk, GF_FOP_OPEN,
-                  open, &state->loc, state->flags, fd, 0);
+                  open, &state->loc, state->flags, fd, state->xdata);
 }
 
-
 static void
 fuse_open (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1840,7 +2076,7 @@ fuse_open (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         state->flags = foi->flags;
 
@@ -1849,12 +2085,11 @@ fuse_open (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 static int
 fuse_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
                 struct iovec *vector, int32_t count,
-                struct iatt *stbuf, struct iobref *iobref)
+                struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
         fuse_state_t *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -1864,9 +2099,11 @@ fuse_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret >= 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": READ => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRId64,
+                        "%"PRIu64": READ => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRIu64,
                         frame->root->unique,
                         op_ret, state->size, state->off, stbuf->ia_size);
 
@@ -1901,8 +2138,8 @@ fuse_readv_resume (fuse_state_t *state)
                 "%"PRIu64": READ (%p, size=%zu, offset=%"PRIu64")",
                 state->finh->unique, state->fd, state->size, state->off);
 
-        FUSE_FOP (state, fuse_readv_cbk, GF_FOP_READ,
-                  readv, state->fd, state->size, state->off, state->io_flags);
+        FUSE_FOP (state, fuse_readv_cbk, GF_FOP_READ, readv, state->fd,
+                  state->size, state->off, state->io_flags, state->xdata);
 }
 
 static void
@@ -1919,7 +2156,7 @@ fuse_readv (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fd = FH_TO_FD (fri->fh);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         /* See comment by similar code in fuse_settatr */
         priv = this->private;
@@ -1936,11 +2173,10 @@ fuse_readv (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_resolve_and_resume (state, fuse_readv_resume);
 }
 
-
 static int
 fuse_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *stbuf, struct iatt *postbuf)
+                 struct iatt *stbuf, struct iatt *postbuf, dict_t *xdata)
 {
         fuse_state_t *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -1949,9 +2185,11 @@ fuse_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret >= 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": WRITE => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRId64,
+                        "%"PRIu64": WRITE => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRIu64,
                         frame->root->unique,
                         op_ret, state->size, state->off, stbuf->ia_size);
 
@@ -1993,11 +2231,12 @@ fuse_write_resume (fuse_state_t *state)
         iobref_add (iobref, iobuf);
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": WRITE (%p, size=%"PRId64", offset=%"PRId64")",
+                "%"PRIu64": WRITE (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
                 state->finh->unique, state->fd, state->size, state->off);
 
         FUSE_FOP (state, fuse_writev_cbk, GF_FOP_WRITE, writev, state->fd,
-                  &state->vector, 1, state->off, state->io_flags, iobref);
+                  &state->vector, 1, state->off, state->io_flags, iobref,
+                  state->xdata);
 
         iobref_unref (iobref);
 }
@@ -2030,7 +2269,7 @@ fuse_write (xlator_t *this, fuse_in_header_t *finh, void *msg)
         */
 
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         /* See comment by similar code in fuse_settatr */
         priv = this->private;
@@ -2047,15 +2286,13 @@ fuse_write (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 void
 fuse_flush_resume (fuse_state_t *state)
 {
         FUSE_FOP (state, fuse_err_cbk, GF_FOP_FLUSH,
-                  flush, state->fd);
+                  flush, state->fd, state->xdata);
 }
 
-
 static void
 fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -2068,7 +2305,7 @@ fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fd = FH_TO_FD (ffi->fh);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         state->lk_owner = ffi->lock_owner;
 
@@ -2080,22 +2317,27 @@ fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 static void
 fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_release_in *fri        = msg;
-        fd_t                   *new_fd     = NULL;
-        fd_t                   *fd         = NULL;
-        uint64_t                val        = 0;
-        int                     ret        = 0;
-        fuse_state_t           *state      = NULL;
-        fuse_fd_ctx_t          *fdctx      = NULL;
+        struct fuse_release_in *fri       = msg;
+        fd_t                   *activefd = NULL;
+        fd_t                   *fd        = NULL;
+        uint64_t                val       = 0;
+        int                     ret       = 0;
+        fuse_state_t           *state     = NULL;
+        fuse_fd_ctx_t          *fdctx     = NULL;
+        fuse_private_t         *priv      = NULL;
 
         GET_STATE (this, finh, state);
         fd = FH_TO_FD (fri->fh);
         state->fd = fd;
 
+        priv = this->private;
+
+        fuse_log_eh (this, "RELEASE(): %"PRIu64":, fd: %p, gfid: %s",
+                     finh->unique, fd, uuid_utoa (fd->inode->gfid));
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RELEASE %p", finh->unique, state->fd);
 
@@ -2103,9 +2345,9 @@ fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
         if (!ret) {
                 fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
                 if (fdctx) {
-                        new_fd = fdctx->fd;
-                        if (new_fd) {
-                                fd_unref (new_fd);
+                        activefd = fdctx->activefd;
+                        if (activefd) {
+                                fd_unref (activefd);
                         }
 
                         GF_FREE (fdctx);
@@ -2113,26 +2355,28 @@ fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
         }
         fd_unref (fd);
 
+        state->fd = NULL;
+
+        gf_fdptr_put (priv->fdtable, fd);
+
         send_fuse_err (this, finh, 0);
 
         free_fuse_state (state);
         return;
 }
 
-
 void
 fuse_fsync_resume (fuse_state_t *state)
 {
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": FSYNC %p", state->finh->unique,
-		state->fd);
+                state->fd);
 
         /* fsync_flags: 1 means "datasync" (no defines for this) */
         FUSE_FOP (state, fuse_fsync_cbk, GF_FOP_FSYNC,
-                  fsync, state->fd, state->flags & 1);
+                  fsync, state->fd, (state->flags & 1), state->xdata);
 }
 
-
 static void
 fuse_fsync (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -2145,18 +2389,21 @@ fuse_fsync (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fd = FH_TO_FD (fsi->fh);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         state->flags = fsi->fsync_flags;
         fuse_resolve_and_resume (state, fuse_fsync_resume);
         return;
 }
 
-
 void
 fuse_opendir_resume (fuse_state_t *state)
 {
-        fd_t *fd = NULL;
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
+
+        priv = state->this->private;
 
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
@@ -2168,17 +2415,36 @@ fuse_opendir_resume (fuse_state_t *state)
         }
 
         fd = fd_create (state->loc.inode, state->finh->pid);
-        state->fd = fd;
+        if (fd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPENDIR fd creation failed",
+                        state->finh->unique);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+        }
+
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPENDIR creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
+        state->fd = fd_ref (fd);
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": OPENDIR %s", state->finh->unique,
                 state->loc.path);
 
         FUSE_FOP (state, fuse_fd_cbk, GF_FOP_OPENDIR,
-                  opendir, &state->loc, fd);
+                  opendir, &state->loc, fd, state->xdata);
 }
 
-
 static void
 fuse_opendir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -2195,7 +2461,6 @@ fuse_opendir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_resolve_and_resume (state, fuse_opendir_resume);
 }
 
-
 unsigned char
 d_type_from_stat (struct iatt *buf)
 {
@@ -2229,10 +2494,10 @@ d_type_from_stat (struct iatt *buf)
         return d_type;
 }
 
-
 static int
 fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                  dict_t *xdata)
 {
         fuse_state_t *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -2240,9 +2505,13 @@ fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         char         *buf = NULL;
         gf_dirent_t  *entry = NULL;
         struct fuse_dirent *fde = NULL;
+        fuse_private_t *priv = NULL;
 
         state = frame->root->state;
         finh  = state->finh;
+        priv = state->this->private;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
         if (op_ret < 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
@@ -2262,6 +2531,11 @@ fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                            strlen (entry->d_name));
         }
 
+	if (size <= 0) {
+		send_fuse_data (this, finh, 0, 0);
+		goto out;
+	}
+
         buf = GF_CALLOC (1, size, gf_fuse_mt_char);
         if (!buf) {
                 gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
@@ -2274,10 +2548,7 @@ fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         size = 0;
         list_for_each_entry (entry, &entries->list, list) {
                 fde = (struct fuse_dirent *)(buf + size);
-                fde->ino = entry->d_ino;
-                fde->off = entry->d_off;
-                fde->namelen = strlen (entry->d_name);
-                strncpy (fde->name, entry->d_name, fde->namelen);
+                gf_fuse_fill_dirent (entry, fde, priv->enable_ino32);
                 size += FUSE_DIRENT_SIZE (fde);
         }
 
@@ -2289,25 +2560,22 @@ fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         free_fuse_state (state);
         STACK_DESTROY (frame->root);
-        if (buf)
-                GF_FREE (buf);
+        GF_FREE (buf);
         return 0;
 
 }
 
-
 void
 fuse_readdir_resume (fuse_state_t *state)
 {
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": READDIR (%p, size=%zu, offset=%"PRId64")",
+                "%"PRIu64": READDIR (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
                 state->finh->unique, state->fd, state->size, state->off);
 
         FUSE_FOP (state, fuse_readdir_cbk, GF_FOP_READDIR,
-                  readdir, state->fd, state->size, state->off);
+                  readdir, state->fd, state->size, state->off, state->xdata);
 }
 
-
 static void
 fuse_readdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -2322,25 +2590,207 @@ fuse_readdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fd = FH_TO_FD (fri->fh);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         fuse_resolve_and_resume (state, fuse_readdir_resume);
 }
 
 
+static int
+fuse_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+		   dict_t *xdata)
+{
+	fuse_state_t *state = NULL;
+	fuse_in_header_t *finh = NULL;
+	int           size = 0;
+	char         *buf = NULL;
+	gf_dirent_t  *entry = NULL;
+	struct fuse_direntplus *fde = NULL;
+	struct fuse_entry_out *feo = NULL;
+	fuse_private_t         *priv   = NULL;
+
+	state = frame->root->state;
+	finh  = state->finh;
+	priv = this->private;
+
+	if (op_ret < 0) {
+		gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+			"%"PRIu64": READDIRP => -1 (%s)", frame->root->unique,
+			strerror (op_errno));
+
+		send_fuse_err (this, finh, op_errno);
+		goto out;
+	}
+
+	gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+		"%"PRIu64": READDIRP => %d/%"GF_PRI_SIZET",%"PRId64,
+		frame->root->unique, op_ret, state->size, state->off);
+
+	list_for_each_entry (entry, &entries->list, list) {
+		size += FUSE_DIRENT_ALIGN (FUSE_NAME_OFFSET_DIRENTPLUS +
+					   strlen (entry->d_name));
+	}
+
+	if (size <= 0) {
+		send_fuse_data (this, finh, 0, 0);
+		goto out;
+	}
+
+	buf = GF_CALLOC (1, size, gf_fuse_mt_char);
+	if (!buf) {
+		gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+			"%"PRIu64": READDIRP => -1 (%s)", frame->root->unique,
+			strerror (ENOMEM));
+		send_fuse_err (this, finh, ENOMEM);
+		goto out;
+	}
+
+	size = 0;
+	list_for_each_entry (entry, &entries->list, list) {
+		inode_t *linked_inode;
+
+		fde = (struct fuse_direntplus *)(buf + size);
+		feo = &fde->entry_out;
+		fde->dirent.ino = entry->d_ino;
+		fde->dirent.off = entry->d_off;
+		fde->dirent.type = entry->d_type;
+		fde->dirent.namelen = strlen (entry->d_name);
+		strncpy (fde->dirent.name, entry->d_name, fde->dirent.namelen);
+		size += FUSE_DIRENTPLUS_SIZE (fde);
+
+		if (!entry->inode)
+			continue;
+
+		entry->d_stat.ia_blksize = this->ctx->page_size;
+		gf_fuse_stat2attr (&entry->d_stat, &feo->attr, priv->enable_ino32);
+
+		linked_inode = inode_link (entry->inode, state->fd->inode,
+					   entry->d_name, &entry->d_stat);
+		if (!linked_inode)
+			continue;
+
+		inode_lookup (linked_inode);
+
+		feo->nodeid = inode_to_fuse_nodeid (linked_inode);
+
+		fuse_inode_set_need_lookup (linked_inode, this);
+
+		inode_unref (linked_inode);
+
+		feo->entry_valid =
+			calc_timeout_sec (priv->entry_timeout);
+		feo->entry_valid_nsec =
+			calc_timeout_nsec (priv->entry_timeout);
+		feo->attr_valid =
+			calc_timeout_sec (priv->attribute_timeout);
+		feo->attr_valid_nsec =
+			calc_timeout_nsec (priv->attribute_timeout);
+	}
+
+	send_fuse_data (this, finh, buf, size);
+out:
+	free_fuse_state (state);
+	STACK_DESTROY (frame->root);
+	GF_FREE (buf);
+	return 0;
+
+}
+
+
+void
+fuse_readdirp_resume (fuse_state_t *state)
+{
+	gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+		"%"PRIu64": READDIRP (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
+		state->finh->unique, state->fd, state->size, state->off);
+
+	FUSE_FOP (state, fuse_readdirp_cbk, GF_FOP_READDIRP,
+		  readdirp, state->fd, state->size, state->off, state->xdata);
+}
+
+
+static void
+fuse_readdirp (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+	struct fuse_read_in *fri = msg;
+
+	fuse_state_t *state = NULL;
+	fd_t         *fd = NULL;
+
+	GET_STATE (this, finh, state);
+	state->size = fri->size;
+	state->off = fri->offset;
+	fd = FH_TO_FD (fri->fh);
+	state->fd = fd;
+
+	fuse_resolve_fd_init (state, &state->resolve, fd);
+
+	fuse_resolve_and_resume (state, fuse_readdirp_resume);
+}
+
+static int
+fuse_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		   struct iatt *postbuf, dict_t *xdata)
+{
+	return fuse_err_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+}
+
+static void
+fuse_fallocate_resume(fuse_state_t *state)
+{
+	gf_log("glusterfs-fuse", GF_LOG_TRACE,
+	       "%"PRIu64": FALLOCATE (%p, flags=%d, size=%zu, offset=%"PRId64")",
+	       state->finh->unique, state->fd, state->flags, state->size,
+	       state->off);
+
+	if (state->flags & FALLOC_FL_PUNCH_HOLE)
+		FUSE_FOP(state, fuse_fallocate_cbk, GF_FOP_DISCARD, discard,
+			 state->fd, state->off, state->size, state->xdata);
+	else
+		FUSE_FOP(state, fuse_fallocate_cbk, GF_FOP_FALLOCATE, fallocate,
+			 state->fd, (state->flags & FALLOC_FL_KEEP_SIZE),
+			 state->off, state->size, state->xdata);
+}
+
+static void
+fuse_fallocate(xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+	struct fuse_fallocate_in *ffi = msg;
+	fuse_state_t *state = NULL;
+
+	GET_STATE(this, finh, state);
+	state->off = ffi->offset;
+	state->size = ffi->length;
+	state->flags = ffi->mode;
+	state->fd = FH_TO_FD(ffi->fh);
+
+	fuse_resolve_fd_init(state, &state->resolve, state->fd);
+	fuse_resolve_and_resume(state, fuse_fallocate_resume);
+}
+
+
 static void
 fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_release_in *fri        = msg;
-        fd_t                   *new_fd     = NULL;
-        uint64_t                val        = 0;
-        int                     ret        = 0;
-        fuse_state_t           *state      = NULL;
-        fuse_fd_ctx_t          *fdctx      = NULL;
+        struct fuse_release_in *fri       = msg;
+        fd_t                   *activefd = NULL;
+        uint64_t                val       = 0;
+        int                     ret       = 0;
+        fuse_state_t           *state     = NULL;
+        fuse_fd_ctx_t          *fdctx     = NULL;
+        fuse_private_t         *priv      = NULL;
 
         GET_STATE (this, finh, state);
         state->fd = FH_TO_FD (fri->fh);
 
+        priv = this->private;
+
+        fuse_log_eh (this, "RELEASEDIR (): %"PRIu64": fd: %p, gfid: %s",
+                     finh->unique, state->fd,
+                     uuid_utoa (state->fd->inode->gfid));
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RELEASEDIR %p", finh->unique, state->fd);
 
@@ -2349,9 +2799,9 @@ fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         if (!ret) {
                 fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
                 if (fdctx) {
-                        new_fd = fdctx->fd;
-                        if (new_fd) {
-                                fd_unref (new_fd);
+                        activefd = fdctx->activefd;
+                        if (activefd) {
+                                fd_unref (activefd);
                         }
 
                         GF_FREE (fdctx);
@@ -2360,6 +2810,10 @@ fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         fd_unref (state->fd);
 
+        gf_fdptr_put (priv->fdtable, state->fd);
+
+        state->fd = NULL;
+
         send_fuse_err (this, finh, 0);
 
         free_fuse_state (state);
@@ -2371,7 +2825,7 @@ void
 fuse_fsyncdir_resume (fuse_state_t *state)
 {
         FUSE_FOP (state, fuse_err_cbk, GF_FOP_FSYNCDIR,
-                  fsyncdir, state->fd, state->flags & 1);
+                  fsyncdir, state->fd, (state->flags & 1), state->xdata);
 
 }
 
@@ -2388,7 +2842,7 @@ fuse_fsyncdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         GET_STATE (this, finh, state);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         state->flags = fsi->fsync_flags;
         fuse_resolve_and_resume (state, fuse_fsyncdir_resume);
@@ -2396,10 +2850,10 @@ fuse_fsyncdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 static int
 fuse_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                 int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                 dict_t *xdata)
 {
         fuse_state_t *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -2410,6 +2864,10 @@ fuse_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         priv  = this->private;
         finh  = state->finh;
 
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s()",
+                     op_ret, op_errno, frame->root->unique,
+                     gf_fop_list[frame->root->op]);
+
         if (op_ret == 0) {
 #ifndef GF_DARWIN_HOST_OS
                 /* MacFUSE doesn't respect anyof these tweaks */
@@ -2449,7 +2907,6 @@ fuse_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 void
 fuse_statfs_resume (fuse_state_t *state)
 {
@@ -2467,7 +2924,7 @@ fuse_statfs_resume (fuse_state_t *state)
                 "%"PRIu64": STATFS", state->finh->unique);
 
         FUSE_FOP (state, fuse_statfs_cbk, GF_FOP_STATFS,
-                  statfs, &state->loc);
+                  statfs, &state->loc, state->xdata);
 }
 
 
@@ -2478,9 +2935,9 @@ fuse_statfs (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
-	fuse_resolve_and_resume (state, fuse_statfs_resume);
+        fuse_resolve_and_resume (state, fuse_statfs_resume);
 }
 
 
@@ -2492,7 +2949,7 @@ fuse_setxattr_resume (fuse_state_t *state)
                         "%"PRIu64": SETXATTR %s/%"PRIu64" (%s) "
                         "resolution failed",
                         state->finh->unique, uuid_utoa (state->resolve.gfid),
-			state->finh->nodeid, state->name);
+                        state->finh->nodeid, state->name);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -2508,14 +2965,16 @@ fuse_setxattr_resume (fuse_state_t *state)
                         state->fd, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_setxattr_cbk, GF_FOP_FSETXATTR,
-                          fsetxattr, state->fd, state->dict, state->flags);
+                          fsetxattr, state->fd, state->xattr, state->flags,
+                          state->xdata);
         } else {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": SETXATTR %s/%"PRIu64" (%s)", state->finh->unique,
                         state->loc.path, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_setxattr_cbk, GF_FOP_SETXATTR,
-                          setxattr, &state->loc, state->dict, state->flags);
+                          setxattr, &state->loc, state->xattr, state->flags,
+                          state->xdata);
         }
 }
 
@@ -2547,22 +3006,27 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         }
 #endif
 
+        if (fuse_ignore_xattr_set (priv, name)) {
+                (void) send_fuse_err (this, finh, 0);
+                return;
+        }
+
         if (!priv->acl) {
-                if ((strcmp (name, "system.posix_acl_access") == 0) ||
-                    (strcmp (name, "system.posix_acl_default") == 0)) {
+                if ((strcmp (name, POSIX_ACL_ACCESS_XATTR) == 0) ||
+                    (strcmp (name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
                         send_fuse_err (this, finh, EOPNOTSUPP);
                         GF_FREE (finh);
                         return;
                 }
         }
 
-#ifdef DISABLE_SELINUX
-        if (!strncmp (name, "security.", 9)) {
-                send_fuse_err (this, finh, EOPNOTSUPP);
-                GF_FREE (finh);
-                return;
+        if (!priv->selinux) {
+                if (strncmp (name, "security.", 9) == 0) {
+                        send_fuse_err (this, finh, EOPNOTSUPP);
+                        GF_FREE (finh);
+                        return;
+                }
         }
-#endif
 
         /* Check if the command is for changing the log
            level of process or specific xlator */
@@ -2577,7 +3041,7 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 gf_log ("fuse", GF_LOG_TRACE,
                         "got request to invalidate %"PRIu64, finh->nodeid);
                 send_fuse_err (this, finh, 0);
-                fuse_invalidate (this, finh->nodeid);
+                fuse_invalidate_entry (this, finh->nodeid);
                 GF_FREE (finh);
                 return;
         }
@@ -2591,10 +3055,10 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         GET_STATE (this, finh, state);
         state->size = fsi->size;
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
-        state->dict = get_new_dict ();
-        if (!state->dict) {
+        state->xattr = get_new_dict ();
+        if (!state->xattr) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "%"PRIu64": SETXATTR dict allocation failed",
                         finh->unique);
@@ -2617,9 +3081,9 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 gf_log (THIS->name, GF_LOG_ERROR, "value size zero");
                 dict_value = NULL;
         }
-        dict_set (state->dict, newkey,
+        dict_set (state->xattr, newkey,
                   data_from_dynptr ((void *)dict_value, fsi->size));
-        dict_ref (state->dict);
+        dict_ref (state->xattr);
 
         state->flags = fsi->flags;
         state->name = newkey;
@@ -2653,10 +3117,28 @@ send_fuse_xattr (xlator_t *this, fuse_in_header_t *finh, const char *value,
         }
 }
 
+/* filter out xattrs that need not be visible on the
+ * mount point. this is _specifically_ for geo-rep
+ * as of now, to prevent Rsync from crying out loud
+ * when it tries to setxattr() for selinux xattrs
+ */
+static int
+fuse_filter_xattr(char *key)
+{
+        int need_filter = 0;
+        struct fuse_private *priv = THIS->private;
+
+        if ((priv->client_pid == GF_CLIENT_PID_GSYNCD)
+            && fnmatch ("*.selinux*", key, FNM_PERIOD) == 0)
+                need_filter = 1;
+
+        return need_filter;
+}
+
 
 static int
 fuse_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, dict_t *dict)
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         int             need_to_free_dict = 0;
         char           *value = "";
@@ -2665,11 +3147,13 @@ fuse_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         data_t         *value_data = NULL;
         int             ret = -1;
         int32_t         len = 0;
-        data_pair_t    *trav = NULL;
+        int32_t         len_next = 0;
 
         state = frame->root->state;
         finh  = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret >= 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": %s() %s => %d", frame->root->unique,
@@ -2691,22 +3175,24 @@ fuse_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         } /* if(value_data)...else */
                 } else {
                         /* if callback for listxattr */
-                        trav = dict->members_list;
-                        while (trav) {
-                                len += strlen (trav->key) + 1;
-                                trav = trav->next;
-                        } /* while(trav) */
+                        /* we need to invoke fuse_filter_xattr() twice. Once
+                         * while counting size and then while filling buffer
+                         */
+                        len = dict_keys_join (NULL, 0, dict, fuse_filter_xattr);
+                        if (len < 0)
+                                goto out;
+
                         value = alloca (len + 1);
                         if (!value)
                                 goto out;
-                        len = 0;
-                        trav = dict->members_list;
-                        while (trav) {
-                                strcpy (value + len, trav->key);
-                                value[len + strlen (trav->key)] = '\0';
-                                len += strlen (trav->key) + 1;
-                                trav = trav->next;
-                        } /* while(trav) */
+
+                        len_next = dict_keys_join (value, len, dict,
+                                                   fuse_filter_xattr);
+                        if (len_next != len)
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "sizes not equal %d != %d",
+                                        len, len_next);
+
                         send_fuse_xattr (this, finh, value, len, state->size);
                 } /* if(state->name)...else */
         } else {
@@ -2751,13 +3237,15 @@ out:
 void
 fuse_getxattr_resume (fuse_state_t *state)
 {
+        char *value  = NULL;
+
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                         "%"PRIu64": GETXATTR %s/%"PRIu64" (%s) "
-			"resolution failed",
+                        "resolution failed",
                         state->finh->unique,
-			uuid_utoa (state->resolve.gfid),
-			state->finh->nodeid, state->name);
+                        uuid_utoa (state->resolve.gfid),
+                        state->finh->nodeid, state->name);
 
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
@@ -2768,20 +3256,60 @@ fuse_getxattr_resume (fuse_state_t *state)
         state->fd = fd_lookup (state->loc.inode, state->finh->pid);
 #endif /* GF_TEST_FFOP */
 
+        if (state->name &&
+            (strcmp (state->name, VIRTUAL_GFID_XATTR_KEY) == 0)) {
+                /* send glusterfs gfid in binary form */
+
+                value = GF_CALLOC (16 + 1, sizeof(char),
+                                   gf_common_mt_char);
+                if (!value) {
+                        send_fuse_err (state->this, state->finh, ENOMEM);
+                        goto internal_out;
+                }
+                memcpy (value, state->loc.inode->gfid, 16);
+
+                send_fuse_xattr (THIS, state->finh, value, 16, state->size);
+                GF_FREE (value);
+        internal_out:
+                free_fuse_state (state);
+                return;
+        }
+
+        if (state->name &&
+            (strcmp (state->name, VIRTUAL_GFID_XATTR_KEY_STR) == 0)) {
+                /* transform binary gfid to canonical form */
+
+                value = GF_CALLOC (UUID_CANONICAL_FORM_LEN + 1, sizeof(char),
+                                   gf_common_mt_char);
+                if (!value) {
+                        send_fuse_err (state->this, state->finh, ENOMEM);
+                        goto internal_out1;
+                }
+                uuid_utoa_r (state->loc.inode->gfid, value);
+
+                send_fuse_xattr (THIS, state->finh, value,
+                                 UUID_CANONICAL_FORM_LEN, state->size);
+                GF_FREE (value);
+        internal_out1:
+                free_fuse_state (state);
+                return;
+        }
+
+
         if (state->fd) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": GETXATTR %p/%"PRIu64" (%s)", state->finh->unique,
                         state->fd, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_FGETXATTR,
-                          fgetxattr, state->fd, state->name);
+                          fgetxattr, state->fd, state->name, state->xdata);
         } else {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": GETXATTR %s/%"PRIu64" (%s)", state->finh->unique,
                         state->loc.path, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_GETXATTR,
-                          getxattr, &state->loc, state->name);
+                          getxattr, &state->loc, state->name, state->xdata);
         }
 }
 
@@ -2789,15 +3317,16 @@ fuse_getxattr_resume (fuse_state_t *state)
 static void
 fuse_getxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_getxattr_in *fgxi = msg;
-        char         *name = (char *)(fgxi + 1);
-
-        fuse_state_t *state = NULL;
-        struct fuse_private *priv = NULL;
-        int rv = 0;
-        char *newkey = NULL;
+        struct fuse_getxattr_in *fgxi     = msg;
+        char                    *name     = (char *)(fgxi + 1);
+        fuse_state_t            *state    = NULL;
+        struct fuse_private     *priv     = NULL;
+        int                      rv       = 0;
+        int                     op_errno  = EINVAL;
+        char                    *newkey   = NULL;
 
         priv = this->private;
+        GET_STATE (this, finh, state);
 
 #ifdef GF_DARWIN_HOST_OS
         if (fgxi->position) {
@@ -2813,45 +3342,45 @@ fuse_getxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                         "%"PRIu64": GETXATTR %s/%"PRIu64" (%s):"
                         "refusing positioned getxattr",
                         finh->unique, state->loc.path, finh->nodeid, name);
-                send_fuse_err (this, finh, EINVAL);
-                FREE (finh);
-                return;
+                op_errno = EINVAL;
+                goto err;
         }
 #endif
 
         if (!priv->acl) {
-                if ((strcmp (name, "system.posix_acl_access") == 0) ||
-                    (strcmp (name, "system.posix_acl_default") == 0)) {
-                        send_fuse_err (this, finh, ENOTSUP);
-                        GF_FREE (finh);
-                        return;
+                if ((strcmp (name, POSIX_ACL_ACCESS_XATTR) == 0) ||
+                    (strcmp (name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
+                        op_errno = ENOTSUP;
+                        goto err;
                 }
         }
 
-#ifdef DISABLE_SELINUX
-        if (!strncmp (name, "security.", 9)) {
-                send_fuse_err (this, finh, ENODATA);
-                GF_FREE (finh);
-                return;
+        if (!priv->selinux) {
+                if (strncmp (name, "security.", 9) == 0) {
+                        op_errno = ENODATA;
+                        goto err;
+                }
         }
-#endif
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         rv = fuse_flip_xattr_ns (priv, name, &newkey);
         if (rv) {
-                send_fuse_err (this, finh, ENOMEM);
-                free_fuse_state (state);
-                goto out;
+                op_errno = ENOMEM;
+                goto err;
         }
 
         state->size = fgxi->size;
         state->name = newkey;
 
         fuse_resolve_and_resume (state, fuse_getxattr_resume);
- out:
+
+        return;
+ err:
+        send_fuse_err (this, finh, op_errno);
+        free_fuse_state (state);
         return;
 }
 
@@ -2863,7 +3392,7 @@ fuse_listxattr_resume (fuse_state_t *state)
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                         "%"PRIu64": LISTXATTR %s/%"PRIu64
                         "resolution failed", state->finh->unique,
-			uuid_utoa (state->resolve.gfid), state->finh->nodeid);
+                        uuid_utoa (state->resolve.gfid), state->finh->nodeid);
 
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
@@ -2880,14 +3409,14 @@ fuse_listxattr_resume (fuse_state_t *state)
                         state->fd, state->finh->nodeid);
 
                 FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_FGETXATTR,
-                          fgetxattr, state->fd, NULL);
+                          fgetxattr, state->fd, NULL, state->xdata);
         } else {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": LISTXATTR %s/%"PRIu64, state->finh->unique,
                         state->loc.path, state->finh->nodeid);
 
                 FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_GETXATTR,
-                          getxattr, &state->loc, NULL);
+                          getxattr, &state->loc, NULL, state->xdata);
         }
 }
 
@@ -2900,7 +3429,7 @@ fuse_listxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         state->size = fgxi->size;
 
@@ -2916,9 +3445,9 @@ fuse_removexattr_resume (fuse_state_t *state)
          if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                         "%"PRIu64": REMOVEXATTR %s/%"PRIu64" (%s) "
-			"resolution failed",
+                        "resolution failed",
                         state->finh->unique, uuid_utoa (state->resolve.gfid),
-			state->finh->nodeid, state->name);
+                        state->finh->nodeid, state->name);
 
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
@@ -2935,14 +3464,14 @@ fuse_removexattr_resume (fuse_state_t *state)
                         state->fd, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_err_cbk, GF_FOP_FREMOVEXATTR,
-                          fremovexattr, state->fd, state->name);
+                          fremovexattr, state->fd, state->name, state->xdata);
         } else {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": REMOVEXATTR %s/%"PRIu64" (%s)", state->finh->unique,
                         state->loc.path, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_err_cbk, GF_FOP_REMOVEXATTR,
-                          removexattr, &state->loc, state->name);
+                          removexattr, &state->loc, state->name, state->xdata);
         }
 }
 
@@ -2957,11 +3486,17 @@ fuse_removexattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         int32_t       ret = -1;
         char *newkey = NULL;
 
+        if (!strcmp (GFID_XATTR_KEY, name)) {
+                send_fuse_err (this, finh, EPERM);
+                GF_FREE (finh);
+                return;
+        }
+
         priv = this->private;
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         ret = fuse_flip_xattr_ns (priv, name, &newkey);
         if (ret) {
@@ -2981,13 +3516,16 @@ static int gf_fuse_lk_enosys_log;
 
 static int
 fuse_getlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
 {
         fuse_state_t *state = NULL;
 
         state = frame->root->state;
         struct fuse_lk_out flo = {{0, }, };
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": ERR => 0", frame->root->unique);
@@ -3033,7 +3571,7 @@ fuse_getlk_resume (fuse_state_t *state)
                 "%"PRIu64": GETLK %p", state->finh->unique, state->fd);
 
         FUSE_FOP (state, fuse_getlk_cbk, GF_FOP_LK,
-                  lk, state->fd, F_GETLK, &state->lk_lock);
+                  lk, state->fd, F_GETLK, &state->lk_lock, state->xdata);
 }
 
 
@@ -3049,7 +3587,7 @@ fuse_getlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
         GET_STATE (this, finh, state);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         convert_fuse_file_lock (&fli->lk, &state->lk_lock,
                                 fli->owner);
@@ -3064,15 +3602,24 @@ fuse_getlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
 static int
 fuse_setlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
 {
+        uint32_t      op    = 0;
         fuse_state_t *state = NULL;
 
         state = frame->root->state;
+        op    = state->finh->opcode;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": ERR => 0", frame->root->unique);
+                fd_lk_insert_and_merge (state->fd,
+                                        (op == FUSE_SETLK) ? F_SETLK : F_SETLKW,
+                                        &state->lk_lock);
+
                 send_fuse_err (this, state->finh, 0);
         } else {
                 if (op_errno == ENOSYS) {
@@ -3087,11 +3634,10 @@ fuse_setlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                                 "Returning EAGAIN Flock: "
                                 "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
-                                (unsigned long long) lock->l_start,
-                                (unsigned long long) lock->l_len,
-                                (unsigned long long) lock->l_pid,
+                                (unsigned long long) state->lk_lock.l_start,
+                                (unsigned long long) state->lk_lock.l_len,
+                                (unsigned long long) state->lk_lock.l_pid,
                                 lkowner_utoa (&frame->root->lk_owner));
-
                 } else  {
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                                 "%"PRIu64": ERR => -1 (%s)",
@@ -3117,7 +3663,7 @@ fuse_setlk_resume (fuse_state_t *state)
 
         FUSE_FOP (state, fuse_setlk_cbk, GF_FOP_LK, lk, state->fd,
                   state->finh->opcode == FUSE_SETLK ? F_SETLK : F_SETLKW,
-                  &state->lk_lock);
+                  &state->lk_lock, state->xdata);
 }
 
 
@@ -3134,7 +3680,7 @@ fuse_setlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
         state->finh = finh;
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         convert_fuse_file_lock (&fli->lk, &state->lk_lock,
                                 fli->owner);
@@ -3221,6 +3767,10 @@ fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fino.max_readahead = 1 << 17;
         fino.max_write = 1 << 17;
         fino.flags = FUSE_ASYNC_READ | FUSE_POSIX_LOCKS;
+#if FUSE_KERNEL_MINOR_VERSION >= 17
+	if (fini->minor >= 17)
+		fino.flags |= FUSE_FLOCK_LOCKS;
+#endif
 #if FUSE_KERNEL_MINOR_VERSION >= 12
         if (fini->minor >= 12) {
             /* let fuse leave the umask processing to us, so that it does not
@@ -3249,8 +3799,8 @@ fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 }
                 priv->revchan_in  = pfd[0];
                 priv->revchan_out = pfd[1];
-                ret = pthread_create (&messenger, NULL, notify_kernel_loop,
-                                      this);
+                ret = gf_thread_create (&messenger, NULL, notify_kernel_loop,
+					this);
                 if (ret != 0) {
                         gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                                 "failed to start messenger daemon (%s)",
@@ -3260,15 +3810,70 @@ fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
                         goto out;
                 }
                 priv->reverse_fuse_thread_started = _gf_true;
+        } else {
+                /*
+                 * FUSE minor < 12 does not implement invalidate notifications.
+                 * This mechanism is required for fopen-keep-cache to operate
+                 * correctly. Disable and warn the user.
+                 */
+                if (priv->fopen_keep_cache) {
+                        gf_log("glusterfs-fuse", GF_LOG_WARNING, "FUSE version "
+                                "%d.%d does not support inval notifications. "
+                                "fopen-keep-cache disabled.", fini->major,
+                                fini->minor);
+                        priv->fopen_keep_cache = 0;
+                }
         }
+
         if (fini->minor >= 13) {
-                /* these values seemed to work fine during testing */
-                fino.max_background = 64;
-                fino.congestion_threshold = 48;
+                fino.max_background = priv->background_qlen;
+                fino.congestion_threshold = priv->congestion_threshold;
         }
         if (fini->minor < 9)
                 *priv->msg0_len_p = sizeof(*finh) + FUSE_COMPAT_WRITE_IN_SIZE;
 #endif
+        if (priv->use_readdirp) {
+                if (fini->flags & FUSE_DO_READDIRPLUS)
+                        fino.flags |= FUSE_DO_READDIRPLUS;
+        }
+
+	if (priv->fopen_keep_cache == 2) {
+		/* If user did not explicitly set --fopen-keep-cache[=off],
+		   then check if kernel support FUSE_AUTO_INVAL_DATA and ...
+		*/
+		if (fini->flags & FUSE_AUTO_INVAL_DATA) {
+			/* ... enable fopen_keep_cache mode if supported.
+			*/
+			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "Detected "
+				"support for FUSE_AUTO_INVAL_DATA. Enabling "
+				"fopen_keep_cache automatically.");
+			fino.flags |= FUSE_AUTO_INVAL_DATA;
+			priv->fopen_keep_cache = 1;
+		} else {
+			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "No support "
+				"for FUSE_AUTO_INVAL_DATA. Disabling "
+				"fopen_keep_cache.");
+			/* ... else disable. */
+			priv->fopen_keep_cache = 0;
+		}
+	} else if (priv->fopen_keep_cache == 1) {
+		/* If user explicitly set --fopen-keep-cache[=on],
+		   then enable FUSE_AUTO_INVAL_DATA if possible.
+		*/
+		if (fini->flags & FUSE_AUTO_INVAL_DATA) {
+			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "fopen_keep_cache "
+				"is explicitly set. Enabling FUSE_AUTO_INVAL_DATA");
+			fino.flags |= FUSE_AUTO_INVAL_DATA;
+		} else {
+			gf_log ("glusterfs-fuse", GF_LOG_WARNING, "fopen_keep_cache "
+				"is explicitly set. Support for "
+				"FUSE_AUTO_INVAL_DATA is missing");
+		}
+	}
+
+	if (fini->flags & FUSE_ASYNC_DIO)
+		fino.flags |= FUSE_ASYNC_DIO;
+
         ret = send_fuse_obj (this, finh, &fino);
         if (ret == 0)
                 gf_log ("glusterfs-fuse", GF_LOG_INFO,
@@ -3403,11 +4008,507 @@ fuse_first_lookup (xlator_t *this)
 
 
 int
+fuse_nameless_lookup (xlator_t *xl, uuid_t gfid, loc_t *loc)
+{
+        int          ret          = -1;
+        dict_t      *xattr_req    = NULL;
+        struct iatt  iatt         = {0, };
+        inode_t     *linked_inode = NULL;
+
+        if ((loc == NULL) || (xl == NULL)) {
+                goto out;
+        }
+
+        if (loc->inode == NULL) {
+                loc->inode = inode_new (xl->itable);
+                if (loc->inode == NULL) {
+                        goto out;
+                }
+        }
+
+        uuid_copy (loc->gfid, gfid);
+
+        xattr_req = dict_new ();
+        if (xattr_req == NULL) {
+                goto out;
+        }
+
+        ret = syncop_lookup (xl, loc, xattr_req, &iatt, NULL, NULL);
+        if (ret < 0) {
+                goto out;
+        }
+
+        linked_inode = inode_link (loc->inode, NULL, NULL, &iatt);
+        inode_unref (loc->inode);
+        loc->inode = linked_inode;
+
+        ret = 0;
+out:
+        if (xattr_req != NULL) {
+                dict_unref (xattr_req);
+        }
+
+        return ret;
+}
+
+
+int
+fuse_migrate_fd_open (xlator_t *this, fd_t *basefd, fd_t *oldfd,
+                      xlator_t *old_subvol, xlator_t *new_subvol)
+{
+        loc_t          loc       = {0, };
+        fd_t          *newfd     = NULL, *old_activefd = NULL;
+        fuse_fd_ctx_t *basefd_ctx = NULL;
+        fuse_fd_ctx_t *newfd_ctx = NULL;
+        int            ret       = 0, flags = 0;
+
+        ret = inode_path (basefd->inode, NULL, (char **)&loc.path);
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "cannot construct path of gfid (%s) failed"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                        uuid_utoa (basefd->inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+        uuid_copy (loc.gfid, basefd->inode->gfid);
+
+        loc.inode = inode_find (new_subvol->itable, basefd->inode->gfid);
+
+        if (loc.inode == NULL) {
+                ret = fuse_nameless_lookup (new_subvol, basefd->inode->gfid,
+                                            &loc);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "name-less lookup of gfid (%s) failed (%s)"
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                uuid_utoa (basefd->inode->gfid),
+                                strerror (errno),
+                                old_subvol->name, old_subvol->graph->id,
+                                new_subvol->name, new_subvol->graph->id);
+                        goto out;
+                }
+
+        }
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        newfd = fd_create (loc.inode, basefd->pid);
+        if (newfd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "cannot create new fd, hence not migrating basefd "
+                        "(ptr:%p inode-gfid:%s) "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (loc.inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+        newfd->flags = basefd->flags;
+	if (newfd->lk_ctx)
+		fd_lk_ctx_unref (newfd->lk_ctx);
+
+        newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
+
+        newfd_ctx = fuse_fd_ctx_check_n_create (this, newfd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", newfd_ctx, out);
+
+        if (IA_ISDIR (basefd->inode->ia_type)) {
+                ret = syncop_opendir (new_subvol, &loc, newfd);
+        } else {
+                flags = basefd->flags & ~(O_CREAT | O_EXCL | O_TRUNC);
+                ret = syncop_open (new_subvol, &loc, flags, newfd);
+        }
+
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "open on basefd (ptr:%p inode-gfid:%s) failed (%s)"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid), strerror (errno),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+        fd_bind (newfd);
+
+        LOCK (&basefd->lock);
+        {
+                if (basefd_ctx->activefd != NULL) {
+                        old_activefd = basefd_ctx->activefd;
+                }
+
+                basefd_ctx->activefd = newfd;
+        }
+        UNLOCK (&basefd->lock);
+
+        if (old_activefd != NULL) {
+                fd_unref (old_activefd);
+        }
+
+        gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                "migrated basefd (%p) to newfd (%p) (inode-gfid:%s)"
+                "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd, newfd,
+                uuid_utoa (basefd->inode->gfid),
+		old_subvol->name, old_subvol->graph->id,
+                new_subvol->name, new_subvol->graph->id);
+
+        ret = 0;
+
+out:
+        loc_wipe (&loc);
+
+        return ret;
+}
+
+int
+fuse_migrate_locks (xlator_t *this, fd_t *basefd, fd_t *oldfd,
+                    xlator_t *old_subvol, xlator_t *new_subvol)
+{
+        int	 ret      = -1;
+        dict_t	*lockinfo = NULL;
+        void    *ptr      = NULL;
+	fd_t    *newfd    = NULL;
+        fuse_fd_ctx_t *basefd_ctx         = NULL;
+
+
+	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
+		return 0;
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        LOCK (&basefd->lock);
+        {
+                newfd = fd_ref (basefd_ctx->activefd);
+        }
+        UNLOCK (&basefd->lock);
+
+        ret = syncop_fgetxattr (old_subvol, oldfd, &lockinfo,
+                                GF_XATTR_LOCKINFO_KEY);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+			"getting lockinfo failed while migrating locks"
+			"(oldfd:%p newfd:%p inode-gfid:%s)"
+			"(old-subvol:%s-%d new-subvol:%s-%d)",
+			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
+			old_subvol->name, old_subvol->graph->id,
+			new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+        ret = dict_get_ptr (lockinfo, GF_XATTR_LOCKINFO_KEY, &ptr);
+        if (ptr == NULL) {
+                ret = 0;
+                gf_log (this->name, GF_LOG_INFO,
+                        "No lockinfo present on any of the bricks "
+			"(oldfd: %p newfd:%p inode-gfid:%s) "
+                        "(old-subvol:%s-%d new-subvol:%s-%d)",
+			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
+			old_subvol->name, old_subvol->graph->id,
+			new_subvol->name, new_subvol->graph->id);
+
+                goto out;
+        }
+
+        ret = syncop_fsetxattr (new_subvol, newfd, lockinfo, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+			"migrating locks failed (oldfd:%p newfd:%p "
+			"inode-gfid:%s) (old-subvol:%s-%d new-subvol:%s-%d)",
+			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
+			old_subvol->name, old_subvol->graph->id,
+			new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+out:
+	if (newfd)
+		fd_unref (newfd);
+
+        if (lockinfo != NULL) {
+                dict_unref (lockinfo);
+        }
+
+        return ret;
+}
+
+
+int
+fuse_migrate_fd (xlator_t *this, fd_t *basefd, xlator_t *old_subvol,
+                 xlator_t *new_subvol)
+{
+        int            ret                = -1;
+        char           create_in_progress = 0;
+        fuse_fd_ctx_t *basefd_ctx         = NULL;
+        fd_t          *oldfd              = NULL;
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        LOCK (&basefd->lock);
+        {
+                oldfd = basefd_ctx->activefd ? basefd_ctx->activefd
+                        : basefd;
+                fd_ref (oldfd);
+        }
+        UNLOCK (&basefd->lock);
+
+        LOCK (&oldfd->inode->lock);
+        {
+                if (uuid_is_null (oldfd->inode->gfid)) {
+                        create_in_progress = 1;
+                } else {
+                        create_in_progress = 0;
+                }
+        }
+        UNLOCK (&oldfd->inode->lock);
+
+        if (create_in_progress) {
+                gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                        "create call on fd (%p) is in progress "
+                        "(basefd-ptr:%p basefd-inode.gfid:%s), "
+                        "hence deferring migration till application does an "
+                        "fd based operation on this fd"
+                        "(old-subvolume:%s-%d, new-subvolume:%s-%d)",
+                        oldfd, basefd, uuid_utoa (basefd->inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+
+                ret = 0;
+                goto out;
+        }
+
+        if (oldfd->inode->table->xl == old_subvol) {
+                ret = syncop_fsync (old_subvol, oldfd, 0);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "syncop_fsync failed (%s) on fd (%p)"
+                                "(basefd:%p basefd-inode.gfid:%s) "
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                strerror (errno), oldfd, basefd,
+                                uuid_utoa (basefd->inode->gfid),
+                                old_subvol->name, old_subvol->graph->id,
+                                new_subvol->name, new_subvol->graph->id);
+                }
+        } else {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "basefd (ptr:%p inode-gfid:%s) was not "
+                        "migrated during previous graph switch"
+                        "(old-subvolume:%s-%d new-subvolume: %s-%d)", basefd,
+                        basefd->inode->gfid,
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+        }
+
+        ret = fuse_migrate_fd_open (this, basefd, oldfd, old_subvol,
+                                    new_subvol);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "open corresponding to "
+                        "basefd (ptr:%p inode-gfid:%s) in new graph failed "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid), old_subvol->name,
+                        old_subvol->graph->id, new_subvol->name,
+                        new_subvol->graph->id);
+                goto out;
+        }
+
+        ret = fuse_migrate_locks (this, basefd, oldfd, old_subvol,
+                                  new_subvol);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "migrating locks from old-subvolume (%s-%d) to "
+                        "new-subvolume (%s-%d) failed (inode-gfid:%s oldfd:%p "
+                        "basefd:%p)", old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id,
+                        uuid_utoa (basefd->inode->gfid), oldfd, basefd);
+
+        }
+out:
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "migration of basefd "
+                        "(ptr:%p inode-gfid:%s) failed"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        oldfd ? uuid_utoa (oldfd->inode->gfid) : NULL,
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+        }
+
+        fd_unref (oldfd);
+
+        return ret;
+}
+
+
+int
+fuse_handle_opened_fds (xlator_t *this, xlator_t *old_subvol,
+                        xlator_t *new_subvol)
+{
+        fuse_private_t *priv      = NULL;
+        fdentry_t      *fdentries = NULL;
+        uint32_t        count     = 0;
+        fdtable_t      *fdtable   = NULL;
+        int             i         = 0;
+        fd_t           *fd        = NULL;
+        int32_t         ret       = 0;
+        fuse_fd_ctx_t  *fdctx     = NULL;
+
+        priv = this->private;
+
+        fdtable = priv->fdtable;
+
+        fdentries = gf_fd_fdtable_copy_all_fds (fdtable, &count);
+        if (fdentries != NULL) {
+                for (i = 0; i < count; i++) {
+                        fd = fdentries[i].fd;
+                        if (fd == NULL)
+                                continue;
+
+                        ret = fuse_migrate_fd (this, fd, old_subvol,
+                                               new_subvol);
+
+                        fdctx = fuse_fd_ctx_get (this, fd);
+                        if (fdctx) {
+                                LOCK (&fd->lock);
+                                {
+                                        if (ret < 0) {
+                                                fdctx->migration_failed = 1;
+                                        } else {
+                                                fdctx->migration_failed = 0;
+                                        }
+                                }
+                                UNLOCK (&fd->lock);
+                        }
+                }
+
+                for (i = 0; i < count ; i++) {
+                        fd = fdentries[i].fd;
+                        if (fd)
+                                fd_unref (fd);
+                }
+
+                GF_FREE (fdentries);
+        }
+
+        return 0;
+}
+
+
+static int
+fuse_handle_blocked_locks (xlator_t *this, xlator_t *old_subvol,
+                           xlator_t *new_subvol)
+{
+        return 0;
+}
+
+
+static int
+fuse_graph_switch_task (void *data)
+{
+        fuse_graph_switch_args_t *args = NULL;
+
+        args = data;
+        if (args == NULL) {
+                goto out;
+        }
+
+        /* don't change the order of handling open fds and blocked locks, since
+         * the act of opening files also reacquires granted locks in new graph.
+         */
+        fuse_handle_opened_fds (args->this, args->old_subvol, args->new_subvol);
+
+        fuse_handle_blocked_locks (args->this, args->old_subvol,
+                                   args->new_subvol);
+
+out:
+        return 0;
+}
+
+
+fuse_graph_switch_args_t *
+fuse_graph_switch_args_alloc (void)
+{
+        fuse_graph_switch_args_t *args = NULL;
+
+        args = GF_CALLOC (1, sizeof (*args), gf_fuse_mt_graph_switch_args_t);
+        if (args == NULL) {
+                goto out;
+        }
+
+out:
+        return args;
+}
+
+
+void
+fuse_graph_switch_args_destroy (fuse_graph_switch_args_t *args)
+{
+        if (args == NULL) {
+                goto out;
+        }
+
+        GF_FREE (args);
+out:
+        return;
+}
+
+
+int
+fuse_handle_graph_switch (xlator_t *this, xlator_t *old_subvol,
+                          xlator_t *new_subvol)
+{
+        call_frame_t             *frame = NULL;
+        int32_t                   ret   = -1;
+        fuse_graph_switch_args_t *args  = NULL;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (frame == NULL) {
+                goto out;
+        }
+
+        args = fuse_graph_switch_args_alloc ();
+        if (args == NULL) {
+                goto out;
+        }
+
+        args->this = this;
+        args->old_subvol = old_subvol;
+        args->new_subvol = new_subvol;
+
+        ret = synctask_new (this->ctx->env, fuse_graph_switch_task, NULL, frame,
+                            args);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING, "starting sync-task to "
+                        "handle graph switch failed");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (args != NULL) {
+                fuse_graph_switch_args_destroy (args);
+        }
+
+        if (frame != NULL) {
+                STACK_DESTROY (frame->root);
+        }
+
+        return ret;
+}
+
+
+int
 fuse_graph_sync (xlator_t *this)
 {
-        fuse_private_t   *priv = NULL;
-        int               need_first_lookup = 0;
-        int               ret = 0;
+        fuse_private_t *priv                = NULL;
+        int             need_first_lookup   = 0;
+        int             ret                 = 0;
+        xlator_t       *old_subvol          = NULL, *new_subvol = NULL;
+        uint64_t        winds_on_old_subvol = 0;
 
         priv = this->private;
 
@@ -3416,7 +4517,8 @@ fuse_graph_sync (xlator_t *this)
                 if (!priv->next_graph)
                         goto unlock;
 
-                priv->active_subvol = priv->next_graph->top;
+                old_subvol = priv->active_subvol;
+                new_subvol = priv->active_subvol = priv->next_graph->top;
                 priv->next_graph = NULL;
                 need_first_lookup = 1;
 
@@ -3424,11 +4526,11 @@ fuse_graph_sync (xlator_t *this)
                         ret = pthread_cond_wait (&priv->sync_cond,
                                                  &priv->sync_mutex);
                         if (ret != 0) {
-			          gf_log (this->name, GF_LOG_DEBUG,
-					  "timedwait returned non zero value "
-					  "ret: %d errno: %d", ret, errno);
-				  break;
-			}
+                                  gf_log (this->name, GF_LOG_DEBUG,
+                                          "timedwait returned non zero value "
+                                          "ret: %d errno: %d", ret, errno);
+                                  break;
+                        }
                 }
         }
 unlock:
@@ -3438,23 +4540,57 @@ unlock:
                 fuse_first_lookup (this);
         }
 
+        if ((old_subvol != NULL) && (new_subvol != NULL)) {
+                fuse_handle_graph_switch (this, old_subvol, new_subvol);
+
+                pthread_mutex_lock (&priv->sync_mutex);
+                {
+                        old_subvol->switched = 1;
+                        winds_on_old_subvol = old_subvol->winds;
+                }
+                pthread_mutex_unlock (&priv->sync_mutex);
+
+                if (winds_on_old_subvol == 0) {
+                        xlator_notify (old_subvol, GF_EVENT_PARENT_DOWN,
+                                       old_subvol, NULL);
+                }
+        }
+
         return 0;
 }
 
+int
+fuse_get_mount_status (xlator_t *this)
+{
+        int             kid_status = -1;
+        fuse_private_t *priv = this->private;
+
+        if (read(priv->status_pipe[0],&kid_status, sizeof(kid_status)) < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "could not get mount status");
+                kid_status = -1;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "mount status is %d", kid_status);
+
+        close(priv->status_pipe[0]);
+        close(priv->status_pipe[1]);
+        return kid_status;
+}
 
 static void *
 fuse_thread_proc (void *data)
 {
-        char           *mount_point = NULL;
-        xlator_t       *this = NULL;
-        fuse_private_t *priv = NULL;
-        ssize_t         res = 0;
-        struct iobuf   *iobuf = NULL;
-        fuse_in_header_t *finh;
-        struct iovec iov_in[2];
-        void *msg = NULL;
-        const size_t msg0_size = sizeof (*finh) + 128;
-        fuse_handler_t **fuse_ops = NULL;
+        char                     *mount_point = NULL;
+        xlator_t                 *this = NULL;
+        fuse_private_t           *priv = NULL;
+        ssize_t                   res = 0;
+        struct iobuf             *iobuf = NULL;
+        fuse_in_header_t         *finh;
+        struct iovec              iov_in[2];
+        void                     *msg = NULL;
+        const size_t              msg0_size = sizeof (*finh) + 128;
+        fuse_handler_t          **fuse_ops = NULL;
+        struct pollfd             pfd[2] = {{0,}};
+        gf_boolean_t              mount_finished = _gf_false;
 
         this = data;
         priv = this->private;
@@ -3471,10 +4607,50 @@ fuse_thread_proc (void *data)
                 /* THIS has to be reset here */
                 THIS = this;
 
+                if (!mount_finished) {
+                        memset(pfd,0,sizeof(pfd));
+                        pfd[0].fd = priv->status_pipe[0];
+                        pfd[0].events = POLLIN | POLLHUP | POLLERR;
+                        pfd[1].fd = priv->fd;
+                        pfd[1].events = POLLIN | POLLHUP | POLLERR;
+                        if (poll(pfd,2,-1) < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "poll error %s", strerror(errno));
+                                break;
+                        }
+                        if (pfd[0].revents & POLLIN) {
+                                if (fuse_get_mount_status(this) != 0) {
+                                        break;
+                                }
+                                mount_finished = _gf_true;
+                        }
+                        else if (pfd[0].revents) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "mount pipe closed without status");
+                                break;
+                        }
+                        if (!pfd[1].revents) {
+                                continue;
+                        }
+                }
+
+                /*
+                 * We don't want to block on readv while we're still waiting
+                 * for mount status.  That means we only want to get here if
+                 * mount_status is true (meaning that our wait completed
+                 * already) or if we already called poll(2) on priv->fd to
+                 * make sure it's ready.
+                 */
+
                 if (priv->init_recvd)
                         fuse_graph_sync (this);
 
+                /* TODO: This place should always get maximum supported buffer
+                   size from 'fuse', which is as of today 128KB. If we bring in
+                   support for higher block sizes support, then we should be
+                   changing this one too */
                 iobuf = iobuf_get (this->ctx->iobuf_pool);
+
                 /* Add extra 128 byte to the first iov so that it can
                  * accommodate "ordinary" non-write requests. It's not
                  * guaranteed to be big enough, as SETXATTR and namespace
@@ -3505,13 +4681,20 @@ fuse_thread_proc (void *data)
                                         "terminating upon getting %s when "
                                         "reading /dev/fuse",
                                         errno == ENODEV ? "ENODEV" : "EBADF");
-
+                                fuse_log_eh (this, "glusterfs-fuse: terminating"
+                                             " upon getting %s when "
+                                             "reading /dev/fuse",
+                                             errno == ENODEV ? "ENODEV":
+                                             "EBADF");
                                 break;
                         }
                         if (errno != EINTR) {
                                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                                         "read from /dev/fuse returned -1 (%s)",
                                         strerror (errno));
+                                fuse_log_eh (this, "glusterfs-fuse: read from "
+                                             "/dev/fuse returned -1 (%s)",
+                                             strerror (errno));
                         }
 
                         goto cont_err;
@@ -3519,6 +4702,8 @@ fuse_thread_proc (void *data)
                 if (res < sizeof (finh)) {
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                                 "short read on /dev/fuse");
+                        fuse_log_eh (this, "glusterfs-fuse: short read on "
+                                     "/dev/fuse");
                         break;
                 }
 
@@ -3537,6 +4722,8 @@ fuse_thread_proc (void *data)
                    ) {
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                                 "inconsistent read on /dev/fuse");
+                        fuse_log_eh (this, "glusterfs-fuse: inconsistent read "
+                                     "on /dev/fuse");
                         break;
                 }
 
@@ -3546,11 +4733,12 @@ fuse_thread_proc (void *data)
                         msg = iov_in[1].iov_base;
                 else {
                         if (res > msg0_size) {
-                                iov_in[0].iov_base =
-                                  GF_REALLOC (iov_in[0].iov_base, res);
-                                if (iov_in[0].iov_base)
+                                void *b = GF_REALLOC (iov_in[0].iov_base, res);
+                                if (b) {
+                                        iov_in[0].iov_base = b;
                                         finh = (fuse_in_header_t *)
                                                  iov_in[0].iov_base;
+                                }
                                 else {
                                         gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                                                 "Out of memory");
@@ -3571,13 +4759,11 @@ fuse_thread_proc (void *data)
                     finh->uid == priv->uid_map_root)
                         finh->uid = 0;
 
-#ifdef GF_DARWIN_HOST_OS
                 if (finh->opcode >= FUSE_OP_HIGH)
                         /* turn down MacFUSE specific messages */
                         fuse_enosys (this, finh, msg);
                 else
-#endif
-                fuse_ops[finh->opcode] (this, finh, msg);
+                        fuse_ops[finh->opcode] (this, finh, msg);
 
                 iobuf_unref (iobuf);
                 continue;
@@ -3587,8 +4773,11 @@ fuse_thread_proc (void *data)
                 GF_FREE (iov_in[0].iov_base);
         }
 
-        iobuf_unref (iobuf);
-        GF_FREE (iov_in[0].iov_base);
+        /*
+         * We could be in all sorts of states with respect to iobuf and iov_in
+         * by the time we get here, and it's just not worth untangling them if
+         * we're about to kill ourselves anyway.
+         */
 
         if (dict_get (this->options, ZR_MOUNTPOINT_OPT))
                 mount_point = data_to_str (dict_get (this->options,
@@ -3596,11 +4785,10 @@ fuse_thread_proc (void *data)
         if (mount_point) {
                 gf_log (this->name, GF_LOG_INFO,
                         "unmounting %s", mount_point);
-                dict_del (this->options, ZR_MOUNTPOINT_OPT);
         }
 
+        /* Kill the whole process, not just this thread. */
         kill (getpid(), SIGTERM);
-
         return NULL;
 }
 
@@ -3614,7 +4802,7 @@ fuse_itable_dump (xlator_t  *this)
         gf_proc_dump_add_section("xlator.mount.fuse.itable");
         inode_table_dump(this->itable, "xlator.mount.fuse.itable");
 
-	return 0;
+        return 0;
 }
 
 int32_t
@@ -3657,10 +4845,51 @@ fuse_priv_dump (xlator_t  *this)
                             (int)private->strict_volfile_check);
         gf_proc_dump_write("reverse_thread_started", "%d",
                            (int)private->reverse_fuse_thread_started);
+        gf_proc_dump_write("use_readdirp", "%d", private->use_readdirp);
 
         return 0;
 }
 
+int
+fuse_history_dump (xlator_t *this)
+{
+        int      ret    = -1;
+        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
+
+        GF_VALIDATE_OR_GOTO ("fuse", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
+
+        gf_proc_dump_build_key (key_prefix, "xlator.mount.fuse",
+                                "history");
+        gf_proc_dump_add_section (key_prefix);
+        eh_dump (this->history, NULL, dump_history_fuse);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+dump_history_fuse (circular_buffer_t *cb, void *data)
+{
+        char       *string   =  NULL;
+        struct tm  *tm       =  NULL;
+        char       timestr[256] = {0,};
+
+        string = (char *)cb->data;
+        tm = localtime (&cb->tv.tv_sec);
+
+        if (tm) {
+                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
+                gf_proc_dump_write ("TIME", "%s", timestr);
+        }
+
+        gf_proc_dump_write ("message", "%s\n", string);
+
+        return 0;
+}
 
 int
 fuse_graph_setup (xlator_t *this, glusterfs_graph_t *graph)
@@ -3745,8 +4974,8 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                 if (!private->fuse_thread_started) {
                         private->fuse_thread_started = 1;
 
-                        ret = pthread_create (&private->fuse_thread, NULL,
-                                              fuse_thread_proc, this);
+                        ret = gf_thread_create (&private->fuse_thread, NULL,
+						fuse_thread_proc, this);
                         if (ret != 0) {
                                 gf_log (this->name, GF_LOG_DEBUG,
                                         "pthread_create() failed (%s)",
@@ -3794,45 +5023,52 @@ mem_acct_init (xlator_t *this)
 
 
 static fuse_handler_t *fuse_std_ops[FUSE_OP_HIGH] = {
-        [FUSE_INIT]        = fuse_init,
-        [FUSE_DESTROY]     = fuse_destroy,
         [FUSE_LOOKUP]      = fuse_lookup,
         [FUSE_FORGET]      = fuse_forget,
         [FUSE_GETATTR]     = fuse_getattr,
         [FUSE_SETATTR]     = fuse_setattr,
-        [FUSE_OPENDIR]     = fuse_opendir,
-        [FUSE_READDIR]     = fuse_readdir,
-        [FUSE_RELEASEDIR]  = fuse_releasedir,
-        [FUSE_ACCESS]      = fuse_access,
         [FUSE_READLINK]    = fuse_readlink,
+        [FUSE_SYMLINK]     = fuse_symlink,
         [FUSE_MKNOD]       = fuse_mknod,
         [FUSE_MKDIR]       = fuse_mkdir,
         [FUSE_UNLINK]      = fuse_unlink,
         [FUSE_RMDIR]       = fuse_rmdir,
-        [FUSE_SYMLINK]     = fuse_symlink,
         [FUSE_RENAME]      = fuse_rename,
         [FUSE_LINK]        = fuse_link,
-        [FUSE_CREATE]      = fuse_create,
         [FUSE_OPEN]        = fuse_open,
         [FUSE_READ]        = fuse_readv,
         [FUSE_WRITE]       = fuse_write,
-        [FUSE_FLUSH]       = fuse_flush,
+        [FUSE_STATFS]      = fuse_statfs,
         [FUSE_RELEASE]     = fuse_release,
         [FUSE_FSYNC]       = fuse_fsync,
-        [FUSE_FSYNCDIR]    = fuse_fsyncdir,
-        [FUSE_STATFS]      = fuse_statfs,
         [FUSE_SETXATTR]    = fuse_setxattr,
         [FUSE_GETXATTR]    = fuse_getxattr,
         [FUSE_LISTXATTR]   = fuse_listxattr,
         [FUSE_REMOVEXATTR] = fuse_removexattr,
+        [FUSE_FLUSH]       = fuse_flush,
+        [FUSE_INIT]        = fuse_init,
+        [FUSE_OPENDIR]     = fuse_opendir,
+        [FUSE_READDIR]     = fuse_readdir,
+        [FUSE_RELEASEDIR]  = fuse_releasedir,
+        [FUSE_FSYNCDIR]    = fuse_fsyncdir,
         [FUSE_GETLK]       = fuse_getlk,
         [FUSE_SETLK]       = fuse_setlk,
         [FUSE_SETLKW]      = fuse_setlk,
+        [FUSE_ACCESS]      = fuse_access,
+        [FUSE_CREATE]      = fuse_create,
+     /* [FUSE_INTERRUPT] */
+     /* [FUSE_BMAP] */
+        [FUSE_DESTROY]     = fuse_destroy,
+     /* [FUSE_IOCTL] */
+     /* [FUSE_POLL] */
+     /* [FUSE_NOTIFY_REPLY] */
+	[FUSE_BATCH_FORGET]= fuse_batch_forget,
+	[FUSE_FALLOCATE]   = fuse_fallocate,
+	[FUSE_READDIRPLUS] = fuse_readdirp,
 };
 
 
-static fuse_handler_t *fuse_dump_ops[FUSE_OP_HIGH] = {
-};
+static fuse_handler_t *fuse_dump_ops[FUSE_OP_HIGH];
 
 
 static void
@@ -3860,7 +5096,7 @@ fuse_dumper (xlator_t *this, fuse_in_header_t *finh, void *msg)
                         "failed to dump fuse message (R): %s",
                         strerror (errno));
 
-        return priv->fuse_ops0[finh->opcode] (this, finh, msg);
+        priv->fuse_ops0[finh->opcode] (this, finh, msg);
 }
 
 
@@ -3878,8 +5114,11 @@ init (xlator_t *this_xl)
         int                xl_name_allocated = 0;
         int                fsname_allocated = 0;
         glusterfs_ctx_t   *ctx = NULL;
-        gf_boolean_t       sync_mtab = _gf_false;
+        gf_boolean_t       sync_to_mount = _gf_false;
+        gf_boolean_t       fopen_keep_cache = _gf_false;
+        unsigned long      mntflags = 0;
         char              *mnt_args = NULL;
+        eh_t              *event = NULL;
 
         if (this_xl == NULL)
                 return -1;
@@ -3887,7 +5126,7 @@ init (xlator_t *this_xl)
         if (this_xl->options == NULL)
                 return -1;
 
-        ctx = glusterfs_ctx_get ();
+        ctx = this_xl->ctx;
         if (!ctx)
                 return -1;
 
@@ -3958,25 +5197,21 @@ init (xlator_t *this_xl)
                 goto cleanup_exit;
         }
 
-        ret = dict_get_double (options, "attribute-timeout",
-                               &priv->attribute_timeout);
-        if (ret != 0)
-                priv->attribute_timeout = 1.0; /* default */
+        GF_OPTION_INIT ("attribute-timeout", priv->attribute_timeout, double,
+                        cleanup_exit);
 
-        ret = dict_get_double (options, "entry-timeout",
-                               &priv->entry_timeout);
-        if (ret != 0)
-                priv->entry_timeout = 1.0; /* default */
+        GF_OPTION_INIT ("entry-timeout", priv->entry_timeout, double,
+                        cleanup_exit);
 
-        ret = dict_get_int32 (options, "client-pid",
-                              &priv->client_pid);
-        if (ret == 0)
-                priv->client_pid_set = _gf_true;
+        GF_OPTION_INIT ("negative-timeout", priv->negative_timeout, double,
+                        cleanup_exit);
 
-        ret = dict_get_uint32 (options, "uid-map-root",
-                               &priv->uid_map_root);
-        if (ret != 0)
-                priv->uid_map_root = 0;
+        GF_OPTION_INIT ("client-pid", priv->client_pid, int32, cleanup_exit);
+        /* have to check & register the presence of client-pid manually */
+        priv->client_pid_set = !!dict_get (this_xl->options, "client-pid");
+
+        GF_OPTION_INIT ("uid-map-root", priv->uid_map_root, uint32,
+                        cleanup_exit);
 
         priv->direct_io_mode = 2;
         ret = dict_get_str (options, ZR_DIRECT_IO_OPT, &value_string);
@@ -3985,29 +5220,21 @@ init (xlator_t *this_xl)
                 GF_ASSERT (ret == 0);
         }
 
-        priv->strict_volfile_check = 0;
-        ret = dict_get_str (options, ZR_STRICT_VOLFILE_CHECK, &value_string);
-        if (ret == 0) {
-                ret = gf_string2boolean (value_string,
-                                         &priv->strict_volfile_check);
-                GF_ASSERT (ret == 0);
-        }
+        GF_OPTION_INIT (ZR_STRICT_VOLFILE_CHECK, priv->strict_volfile_check,
+                        bool, cleanup_exit);
+
+        GF_OPTION_INIT ("acl", priv->acl, bool, cleanup_exit);
 
-        priv->acl = 0;
-        ret = dict_get_str (options, "acl", &value_string);
-        if (ret == 0) {
-                ret = gf_string2boolean (value_string, &priv->acl);
-                GF_ASSERT (ret == 0);
-        }
         if (priv->uid_map_root)
                 priv->acl = 1;
 
-        priv->read_only = 0;
-        ret = dict_get_str (options, "read-only", &value_string);
-        if (ret == 0) {
-                ret = gf_string2boolean (value_string, &priv->read_only);
-                GF_ASSERT (ret == 0);
-        }
+        GF_OPTION_INIT ("selinux", priv->selinux, bool, cleanup_exit);
+
+        GF_OPTION_INIT ("read-only", priv->read_only, bool, cleanup_exit);
+
+        GF_OPTION_INIT ("enable-ino32", priv->enable_ino32, bool, cleanup_exit);
+
+        GF_OPTION_INIT ("use-readdirp", priv->use_readdirp, bool, cleanup_exit);
 
         priv->fuse_dump_fd = -1;
         ret = dict_get_str (options, "dump-fuse", &value_string);
@@ -4026,14 +5253,58 @@ init (xlator_t *this_xl)
                 priv->fuse_dump_fd = ret;
         }
 
-        sync_mtab = _gf_false;
-        ret = dict_get_str (options, "sync-mtab", &value_string);
+        sync_to_mount = _gf_false;
+        ret = dict_get_str (options, "sync-to-mount", &value_string);
         if (ret == 0) {
                 ret = gf_string2boolean (value_string,
-                                         &sync_mtab);
+                                         &sync_to_mount);
                 GF_ASSERT (ret == 0);
         }
 
+	priv->fopen_keep_cache = 2;
+	if (dict_get (options, "fopen-keep-cache")) {
+		GF_OPTION_INIT("fopen-keep-cache", fopen_keep_cache, bool,
+			       cleanup_exit);
+		priv->fopen_keep_cache = fopen_keep_cache;
+	}
+
+        GF_OPTION_INIT("gid-timeout", priv->gid_cache_timeout, int32,
+                cleanup_exit);
+
+        GF_OPTION_INIT ("fuse-mountopts", priv->fuse_mountopts, str, cleanup_exit);
+
+        if (gid_cache_init(&priv->gid_cache, priv->gid_cache_timeout) < 0) {
+                gf_log("glusterfs-fuse", GF_LOG_ERROR, "Failed to initialize "
+                        "group cache.");
+                goto cleanup_exit;
+        }
+
+        /* default values seemed to work fine during testing */
+        GF_OPTION_INIT ("background-qlen", priv->background_qlen, int32,
+                        cleanup_exit);
+        GF_OPTION_INIT ("congestion-threshold", priv->congestion_threshold,
+                        int32, cleanup_exit);
+
+        /* user has set only background-qlen, not congestion-threshold,
+           use the fuse kernel driver formula to set congestion. ie, 75% */
+        if (dict_get (this_xl->options, "background-qlen") &&
+            !dict_get (this_xl->options, "congestion-threshold")) {
+                priv->congestion_threshold = (priv->background_qlen * 3) / 4;
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "setting congestion control as 75%% of "
+                        "background-queue length (ie, (.75 * %d) = %d",
+                        priv->background_qlen, priv->congestion_threshold);
+        }
+
+        /* congestion should not be higher than background queue length */
+        if (priv->congestion_threshold > priv->background_qlen) {
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "setting congestion control same as "
+                        "background-queue length (%d)",
+                        priv->background_qlen);
+                priv->congestion_threshold = priv->background_qlen;
+        }
+
         cmd_args = &this_xl->ctx->cmd_args;
         fsname = cmd_args->volfile;
         if (!fsname && cmd_args->volfile_server) {
@@ -4057,17 +5328,42 @@ init (xlator_t *this_xl)
         if (!fsname)
                 fsname = "glusterfs";
 
-        gf_asprintf (&mnt_args, "%s%sallow_other,max_read=131072",
-                     priv->read_only ? "ro," : "",
-                     priv->acl ? "" : "default_permissions,");
+        priv->fdtable = gf_fd_fdtable_alloc ();
+        if (priv->fdtable == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
+                goto cleanup_exit;
+        }
+
+        if (priv->read_only)
+                mntflags |= MS_RDONLY;
+        gf_asprintf (&mnt_args, "%s%s%sallow_other,max_read=131072",
+                     priv->acl ? "" : "default_permissions,",
+                     priv->fuse_mountopts ? priv->fuse_mountopts : "",
+                     priv->fuse_mountopts ? "," : "");
         if (!mnt_args)
                 goto cleanup_exit;
 
-        priv->fd = gf_fuse_mount (priv->mount_point, fsname, mnt_args,
-                                  sync_mtab ? &ctx->mtab_pid : NULL);
+        if (pipe(priv->status_pipe) < 0) {
+                gf_log (this_xl->name, GF_LOG_ERROR,
+                        "could not create pipe to separate mount process");
+                goto cleanup_exit;
+        }
+
+        priv->fd = gf_fuse_mount (priv->mount_point, fsname, mntflags, mnt_args,
+                                  sync_to_mount ? &ctx->mnt_pid : NULL,
+                                  priv->status_pipe[1]);
         if (priv->fd == -1)
                 goto cleanup_exit;
 
+        event = eh_new (FUSE_EVENT_HISTORY_SIZE, _gf_false, NULL);
+        if (!event) {
+                gf_log (this_xl->name, GF_LOG_ERROR,
+                        "could not create a new event history");
+                goto cleanup_exit;
+        }
+
+        this_xl->history = event;
+
         pthread_mutex_init (&priv->fuse_dump_mutex, NULL);
         pthread_cond_init (&priv->sync_cond, NULL);
         pthread_mutex_init (&priv->sync_mutex, NULL);
@@ -4097,12 +5393,13 @@ cleanup_exit:
                 GF_FREE (fsname);
         if (priv) {
                 GF_FREE (priv->mount_point);
-                close (priv->fd);
-                close (priv->fuse_dump_fd);
+                if (priv->fd != -1)
+                        close (priv->fd);
+                if (priv->fuse_dump_fd != -1)
+                        close (priv->fuse_dump_fd);
                 GF_FREE (priv);
         }
-        if (mnt_args)
-                GF_FREE (mnt_args);
+        GF_FREE (mnt_args);
         return -1;
 }
 
@@ -4126,26 +5423,28 @@ fini (xlator_t *this_xl)
                 gf_log (this_xl->name, GF_LOG_INFO,
                         "Unmounting '%s'.", mount_point);
 
-                dict_del (this_xl->options, ZR_MOUNTPOINT_OPT);
                 gf_fuse_unmount (mount_point, priv->fd);
                 close (priv->fuse_dump_fd);
+                dict_del (this_xl->options, ZR_MOUNTPOINT_OPT);
         }
         /* Process should terminate once fuse xlator is finished.
          * Required for AUTH_FAILED event.
          */
-        raise (SIGTERM);
+        kill (getpid (), SIGTERM);
 }
 
-struct xlator_fops fops = {
-};
+struct xlator_fops fops;
 
 struct xlator_cbks cbks = {
+        .invalidate = fuse_invalidate,
+        .forget     = fuse_forget_cbk,
 };
 
 
 struct xlator_dumpops dumpops = {
         .priv  = fuse_priv_dump,
         .inode = fuse_itable_dump,
+        .history = fuse_history_dump,
 };
 
 struct volume_options options[] = {
@@ -4159,13 +5458,20 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_PATH
         },
         { .key  = {ZR_ATTR_TIMEOUT_OPT},
-          .type = GF_OPTION_TYPE_DOUBLE
+          .type = GF_OPTION_TYPE_DOUBLE,
+          .default_value = "1.0"
         },
         { .key  = {ZR_ENTRY_TIMEOUT_OPT},
-          .type = GF_OPTION_TYPE_DOUBLE
+          .type = GF_OPTION_TYPE_DOUBLE,
+          .default_value = "1.0"
+        },
+        { .key  = {ZR_NEGATIVE_TIMEOUT_OPT},
+          .type = GF_OPTION_TYPE_DOUBLE,
+          .default_value = "0.0"
         },
         { .key  = {ZR_STRICT_VOLFILE_CHECK},
-          .type = GF_OPTION_TYPE_BOOL
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
         },
         { .key  = {"client-pid"},
           .type = GF_OPTION_TYPE_INT
@@ -4173,11 +5479,50 @@ struct volume_options options[] = {
         { .key  = {"uid-map-root"},
           .type = GF_OPTION_TYPE_INT
         },
-        { .key  = {"sync-mtab"},
+        { .key  = {"sync-to-mount"},
           .type = GF_OPTION_TYPE_BOOL
         },
         { .key = {"read-only"},
           .type = GF_OPTION_TYPE_BOOL
         },
+        { .key = {"fopen-keep-cache"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key = {"gid-timeout"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "2"
+        },
+        { .key = {"acl"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key = {"selinux"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key = {"enable-ino32"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key  = {"background-qlen"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "64",
+          .min = 16,
+          .max = (64 * GF_UNIT_KB),
+        },
+        { .key  = {"congestion-threshold"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "48",
+          .min = 12,
+          .max = (64 * GF_UNIT_KB),
+        },
+        { .key = {"fuse-mountopts"},
+          .type = GF_OPTION_TYPE_STR
+        },
+        { .key = {"use-readdirp"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "yes"
+        },
         { .key = {NULL} },
 };
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index d9783f2e0..34794b6ea 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _GF_FUSE_BRIDGE_H_
 #define _GF_FUSE_BRIDGE_H_
 
@@ -54,9 +44,11 @@
 
 #include "list.h"
 #include "dict.h"
+#include "syncop.h"
+#include "gidcache.h"
 
 #if defined(GF_LINUX_HOST_OS) || defined(__NetBSD__)
-#define FUSE_OP_HIGH (FUSE_POLL + 1)
+#define FUSE_OP_HIGH (FUSE_READDIRPLUS + 1)
 #endif
 #ifdef GF_DARWIN_HOST_OS
 #define FUSE_OP_HIGH (FUSE_DESTROY + 1)
@@ -65,8 +57,6 @@
 
 #define MAX_FUSE_PROC_DELAY 1
 
-#define DISABLE_SELINUX 1
-
 typedef struct fuse_in_header fuse_in_header_t;
 typedef void (fuse_handler_t) (xlator_t *this, fuse_in_header_t *finh,
                                void *msg);
@@ -86,6 +76,7 @@ struct fuse_private {
         size_t              *msg0_len_p;
 
         double               entry_timeout;
+        double               negative_timeout;
         double               attribute_timeout;
 
         pthread_cond_t       sync_cond;
@@ -108,20 +99,45 @@ struct fuse_private {
         gf_boolean_t         client_pid_set;
         unsigned             uid_map_root;
         gf_boolean_t         acl;
+        gf_boolean_t         selinux;
         gf_boolean_t         read_only;
+	int32_t	             fopen_keep_cache;
+	int32_t		     gid_cache_timeout;
+        gf_boolean_t         enable_ino32;
+        fdtable_t           *fdtable;
+	gid_cache_t	     gid_cache;
+        char                *fuse_mountopts;
 
         /* For fuse-reverse-validation */
         int                  revchan_in;
         int                  revchan_out;
         gf_boolean_t         reverse_fuse_thread_started;
+
+        /* For communicating with separate mount thread. */
+        int                  status_pipe[2];
+
+        /* for fuse queue length and congestion threshold */
+        int background_qlen;
+        int congestion_threshold;
+
+        /* for using fuse-kernel readdirp*/
+        gf_boolean_t use_readdirp;
 };
 typedef struct fuse_private fuse_private_t;
 
+struct fuse_graph_switch_args {
+        xlator_t        *this;
+        xlator_t        *old_subvol;
+        xlator_t        *new_subvol;
+};
+typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
+
 #define INVAL_BUF_SIZE (sizeof (struct fuse_out_header) +               \
                         max (sizeof (struct fuse_notify_inval_inode_out), \
                              sizeof (struct fuse_notify_inval_entry_out) + \
                              NAME_MAX + 1))
 
+#define FUSE_EVENT_HISTORY_SIZE 1024
 
 #define _FH_TO_FD(fh) ((fd_t *)(uintptr_t)(fh))
 
@@ -129,8 +145,10 @@ typedef struct fuse_private fuse_private_t;
 
 #define FUSE_FOP(state, ret, op_num, fop, args ...)                     \
         do {                                                            \
-                call_frame_t *frame = NULL;                             \
-                xlator_t *xl = NULL;                                    \
+                call_frame_t   *frame   = NULL;                         \
+                xlator_t       *xl      = NULL;                         \
+                int32_t         op_ret  = 0, op_errno = 0;              \
+                fuse_resolve_t *resolve = NULL;                         \
                                                                         \
                 frame = get_call_frame_for_req (state);                 \
                 if (!frame) {                                           \
@@ -140,7 +158,7 @@ typedef struct fuse_private fuse_private_t;
                           * better than trying to go on with a NULL     \
                           * frame ...                                   \
                           */                                            \
-                        gf_log ("glusterfs-fuse",                       \
+                        gf_log_callingfn ("glusterfs-fuse",             \
                                 GF_LOG_ERROR,                           \
                                 "FUSE message"                          \
                                 " unique %"PRIu64" opcode %d:"          \
@@ -157,14 +175,66 @@ typedef struct fuse_private fuse_private_t;
                 frame->root->op    = op_num;                            \
                 frame->op          = op_num;                            \
                                                                         \
+                if ( state->resolve_now ) {                             \
+                        resolve = state->resolve_now;                   \
+                } else {                                                \
+                        resolve = &(state->resolve);                    \
+                }                                                       \
+                                                                        \
                 xl = state->active_subvol;				\
                 if (!xl) {                                              \
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,         \
-                                "xl is NULL");                          \
-                        send_fuse_err (state->this, state->finh, ENOENT); \
+                        gf_log_callingfn ("glusterfs-fuse", GF_LOG_ERROR, \
+                                          "xl is NULL");                \
+                        op_errno = ENOENT;                              \
+                        op_ret = -1;                                    \
+                } else if (resolve->op_ret < 0) {                       \
+                        op_errno = resolve->op_errno;                   \
+                        op_ret = -1;                                    \
+                        if (op_num == GF_FOP_LOOKUP) {                  \
+                                gf_log ("glusterfs-fuse",               \
+                                        (op_errno == ENOENT ? GF_LOG_TRACE \
+                                         : GF_LOG_WARNING),             \
+                                        "%"PRIu64": %s() %s => -1 (%s)", \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        resolve->resolve_loc.path,      \
+                                        strerror (op_errno));           \
+                        } else {                                        \
+                                gf_log ("glusterfs-fuse",               \
+                                        GF_LOG_WARNING,                 \
+                                        "%"PRIu64": %s() inode "        \
+                                        "migration of %s failed (%s)",  \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        resolve->resolve_loc.path,      \
+                                        strerror (op_errno));           \
+                        }                                               \
+                } else if (state->resolve2.op_ret < 0) {                \
+                        op_errno = state->resolve2.op_errno;            \
+                        op_ret = -1;                                    \
+                        gf_log ("glusterfs-fuse",                       \
+                                GF_LOG_WARNING,                         \
+                                "%"PRIu64": %s() inode "                \
+                                "migration of %s failed (%s)",          \
+                                frame->root->unique,                    \
+                                gf_fop_list[frame->root->op],           \
+                                state->resolve2.resolve_loc.path,       \
+                                strerror (op_errno));                   \
+                }                                                       \
+                                                                        \
+                if (op_ret < 0) {                                       \
+                        send_fuse_err (state->this, state->finh, op_errno); \
                         free_fuse_state (state);                        \
                         STACK_DESTROY (frame->root);                    \
                 } else {                                                \
+                        if (state->this->history)                       \
+                                gf_log_eh ("%"PRIu64", %s, path: (%s), gfid: " \
+                                           "(%s)", frame->root->unique, \
+                                           gf_fop_list[frame->root->op], \
+                                           state->loc.path,             \
+                                           (state->fd == NULL)?         \
+                                           uuid_utoa (state->loc.gfid): \
+                                           uuid_utoa (state->fd->inode->gfid));\
                         STACK_WIND (frame, ret, xl, xl->fops->fop, args); \
                 }                                                       \
                                                                         \
@@ -173,7 +243,9 @@ typedef struct fuse_private fuse_private_t;
 
 #define FUSE_FOP_COOKIE(state, xl, ret, cky, op_num, fop, args ...)     \
         do {                                                            \
-                call_frame_t *frame = NULL;                             \
+                call_frame_t *frame  = NULL;                            \
+                xlator_t     *xl     = NULL;                            \
+                int32_t       op_ret = 0, op_errno = 0;                 \
                                                                         \
                 frame = get_call_frame_for_req (state);                 \
                 if (!frame) {                                           \
@@ -191,7 +263,62 @@ typedef struct fuse_private fuse_private_t;
                 frame->root->state = state;                             \
                 frame->root->op    = op_num;                            \
 		frame->op          = op_num;				\
-                STACK_WIND_COOKIE (frame, ret, cky, xl, xl->fops->fop, args); \
+                                                                        \
+                xl = state->active_subvol;				\
+                if (!xl) {                                              \
+                        gf_log_callingfn ("glusterfs-fuse", GF_LOG_ERROR, \
+                                          "xl is NULL");                \
+                        op_errno = ENOENT;                              \
+                        op_ret = -1;                                    \
+                } else if (state->resolve.op_ret < 0) {                 \
+                        op_errno = state->resolve.op_errno;             \
+                        op_ret = -1;                                    \
+                        if (op_num == GF_FOP_LOOKUP) {                  \
+                                gf_log ("glusterfs-fuse",               \
+                                        (op_errno == ENOENT ? GF_LOG_TRACE \
+                                         : GF_LOG_WARNING),             \
+                                        "%"PRIu64": %s() %s => -1 (%s)", \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        state->resolve.resolve_loc.path, \
+                                        strerror (op_errno));           \
+                        } else {                                        \
+                                gf_log ("glusterfs-fuse",               \
+                                        GF_LOG_WARNING,                 \
+                                        "%"PRIu64": %s() inode "        \
+                                        "migration of %s failed (%s)",  \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        state->resolve.resolve_loc.path, \
+                                        strerror (op_errno));           \
+                        }                                               \
+                } else if (state->resolve2.op_ret < 0) {                \
+                        op_errno = state->resolve2.op_errno;            \
+                        op_ret = -1;                                    \
+                        gf_log ("glusterfs-fuse",                       \
+                                GF_LOG_WARNING,                         \
+                                "%"PRIu64": %s() inode "                \
+                                "migration of %s failed (%s)",          \
+                                frame->root->unique,                    \
+                                gf_fop_list[frame->root->op],           \
+                                state->resolve2.resolve_loc.path,       \
+                                strerror (op_errno));                   \
+                }                                                       \
+                                                                        \
+                if (op_ret < 0) {                                       \
+                        send_fuse_err (state->this, state->finh, op_errno); \
+                        free_fuse_state (state);                        \
+                        STACK_DESTROY (frame->root);                    \
+                } else {                                                \
+                        if (xl->history)                                \
+                                gf_log_eh ("%"PRIu64", %s, path: (%s), gfid: " \
+                                           "(%s)", frame->root->unique, \
+                                           gf_fop_list[frame->root->op], \
+                                           state->loc.path,             \
+                                           uuid_utoa (state->loc.gfid)); \
+                        STACK_WIND_COOKIE (frame, ret, cky, xl, xl->fops->fop, \
+                                           args);                       \
+                }                                                       \
         } while (0)
 
 #define GF_SELECT_LOG_LEVEL(_errno)                     \
@@ -215,7 +342,80 @@ typedef struct fuse_private fuse_private_t;
                 }                                                          \
         } while (0)
 
+#define FUSE_ENTRY_CREATE(this, priv, finh, state, fci, op)             \
+                do {                                                    \
+                        if (priv->proto_minor >= 12)                    \
+                                state->mode &= ~fci->umask;             \
+                        if (priv->proto_minor >= 12 && priv->acl) {     \
+                                state->xdata = dict_new ();             \
+                                if (!state->xdata) {                    \
+                                        gf_log ("glusterfs-fuse",       \
+                                                GF_LOG_WARNING,         \
+                                                "%s failed to allocate " \
+                                                "a param dictionary", op); \
+                                        send_fuse_err (this, finh, ENOMEM); \
+                                        free_fuse_state (state);        \
+                                        return;                         \
+                                }                                       \
+                                state->umask = fci->umask;              \
+                                                                        \
+/* TODO: remove this after 3.4.0 release. keeping it for the            \
+   sake of backward compatibility with old (3.3.[01])                   \
+   releases till then. */                                               \
+                                ret = dict_set_int16 (state->xdata, "umask", \
+                                                      fci->umask);      \
+                                if (ret < 0) {                          \
+                                        gf_log ("glusterfs-fuse",       \
+                                                GF_LOG_WARNING,         \
+                                                "%s Failed adding umask"\
+                                                " to request", op);     \
+                                        dict_destroy (state->xdata);    \
+                                        send_fuse_err (this, finh, ENOMEM); \
+                                        free_fuse_state (state);        \
+                                        return;                         \
+                                }                                       \
+                                ret = dict_set_int16 (state->xdata, "mode", \
+                                                      fci->mode);       \
+                                if (ret < 0) {                          \
+                                        gf_log ("glusterfs-fuse",       \
+                                                GF_LOG_WARNING,         \
+                                                "%s Failed adding mode " \
+                                                "to request", op);         \
+                                        dict_destroy (state->xdata);    \
+                                        send_fuse_err (this, finh, ENOMEM); \
+                                        free_fuse_state (state);        \
+                                        return;                         \
+                                }                                       \
+                        }                                               \
+                } while (0)
+
+#define fuse_log_eh_fop(this, state, frame, op_ret, op_errno)               \
+        do {                                                            \
+                if (this->history) {                                    \
+                        if (state->fd)                                  \
+                                gf_log_eh ("op_ret: %d, op_errno: %d, " \
+                                           "%"PRIu64", %s () => %p, gfid: %s", \
+                                           op_ret, op_errno,            \
+                                           frame->root->unique,         \
+                                           gf_fop_list[frame->root->op], \
+                                           state->fd,                   \
+                                           uuid_utoa (state->fd->inode->gfid)); \
+                        else                                            \
+                                gf_log_eh ("op_ret: %d, op_errno: %d, " \
+                                           "%"PRIu64", %s () => %s, gfid: %s", \
+                                           op_ret, op_errno,            \
+                                           frame->root->unique,         \
+                                           gf_fop_list[frame->root->op], \
+                                           state->loc.path,             \
+                                           uuid_utoa (state->loc.gfid)); \
+                }                                                       \
+        } while(0)
 
+#define fuse_log_eh(this, args...)              \
+        do {                                    \
+                if (this->history)              \
+                        gf_log_eh(args);        \
+        } while (0)
 
 static inline xlator_t *
 fuse_active_subvol (xlator_t *fuse)
@@ -266,7 +466,8 @@ typedef struct {
         size_t            size;
         unsigned long     nlookup;
         fd_t             *fd;
-        dict_t           *dict;
+        dict_t           *xattr;
+        dict_t           *xdata;
         char             *name;
         char              is_revalidate;
         gf_boolean_t      truncate_needed;
@@ -287,17 +488,20 @@ typedef struct {
         int            mask;
         dev_t          rdev;
         mode_t         mode;
+        mode_t         umask;
         struct iatt    attr;
         struct gf_flock   lk_lock;
         struct iovec   vector;
 
         uuid_t         gfid;
         uint32_t       io_flags;
+        int32_t        fd_no;
 } fuse_state_t;
 
-typedef struct fuse_fd_ctx {
+typedef struct {
         uint32_t  open_flags;
-        fd_t     *fd;
+        char      migration_failed;
+        fd_t     *activefd;
 } fuse_fd_ctx_t;
 
 typedef void (*fuse_resume_fn_t) (fuse_state_t *state);
@@ -308,17 +512,18 @@ fuse_loc_fill (loc_t *loc, fuse_state_t *state, ino_t ino,
 call_frame_t *get_call_frame_for_req (fuse_state_t *state);
 fuse_state_t *get_fuse_state (xlator_t *this, fuse_in_header_t *finh);
 void free_fuse_state (fuse_state_t *state);
-void gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa);
+void gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa,
+                        gf_boolean_t enable_ino32);
+void gf_fuse_fill_dirent (gf_dirent_t *entry, struct fuse_dirent *fde,
+                          gf_boolean_t enable_ino32);
 uint64_t inode_to_fuse_nodeid (inode_t *inode);
 xlator_t *fuse_active_subvol (xlator_t *fuse);
 inode_t *fuse_ino_to_inode (uint64_t ino, xlator_t *fuse);
 int send_fuse_err (xlator_t *this, fuse_in_header_t *finh, int error);
 int fuse_gfid_set (fuse_state_t *state);
 int fuse_flip_xattr_ns (struct fuse_private *priv, char *okey, char **nkey);
-int fuse_flip_user_to_trusted (char *okey, char **nkey);
-int fuse_xattr_alloc_default (char *okey, char **nkey);
-fuse_fd_ctx_t * __fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this);
-fuse_fd_ctx_t * fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this);
+fuse_fd_ctx_t * __fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd);
+fuse_fd_ctx_t * fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd);
 
 int fuse_resolve_and_resume (fuse_state_t *state, fuse_resume_fn_t fn);
 int fuse_resolve_inode_init (fuse_state_t *state, fuse_resolve_t *resolve,
@@ -327,4 +532,6 @@ int fuse_resolve_entry_init (fuse_state_t *state, fuse_resolve_t *resolve,
 			     ino_t par, char *name);
 int fuse_resolve_fd_init (fuse_state_t *state, fuse_resolve_t *resolve,
 			  fd_t *fd);
+int fuse_ignore_xattr_set (fuse_private_t *priv, char *key);
+int dump_history_fuse (circular_buffer_t *cb, void *data);
 #endif /* _GF_FUSE_BRIDGE_H_ */
diff --git a/xlators/mount/fuse/src/fuse-helpers.c b/xlators/mount/fuse/src/fuse-helpers.c
index 81994ad96..4d478b919 100644
--- a/xlators/mount/fuse/src/fuse-helpers.c
+++ b/xlators/mount/fuse/src/fuse-helpers.c
@@ -1,21 +1,15 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
+#ifdef __NetBSD__
+#define _KMEMUSER
+#endif
 
 #include "fuse-bridge.h"
 #if defined(GF_SOLARIS_HOST_OS)
@@ -28,14 +22,14 @@
 static void
 fuse_resolve_wipe (fuse_resolve_t *resolve)
 {
-        if (resolve->path)
-                GF_FREE ((void *)resolve->path);
+        GF_FREE ((void *)resolve->path);
+
+        GF_FREE ((void *)resolve->bname);
 
-        if (resolve->bname)
-                GF_FREE ((void *)resolve->bname);
+        GF_FREE ((void *)resolve->resolved);
 
-        if (resolve->resolved)
-                GF_FREE ((void *)resolve->resolved);
+        if (resolve->fd)
+                fd_unref (resolve->fd);
 
         loc_wipe (&resolve->resolve_loc);
 
@@ -54,14 +48,26 @@ fuse_resolve_wipe (fuse_resolve_t *resolve)
 void
 free_fuse_state (fuse_state_t *state)
 {
+        xlator_t       *this     = NULL;
+        fuse_private_t *priv     = NULL;
+        uint64_t        winds    = 0;
+        char            switched = 0;
+
+        this = state->this;
+
+        priv = this->private;
+
         loc_wipe (&state->loc);
 
         loc_wipe (&state->loc2);
 
-        if (state->dict) {
-                dict_unref (state->dict);
-                state->dict = (void *)0xaaaaeeee;
+        if (state->xdata) {
+                dict_unref (state->xdata);
+                state->xdata = (void *)0xaaaaeeee;
         }
+        if (state->xattr)
+                dict_unref (state->xattr);
+
         if (state->name) {
                 GF_FREE (state->name);
                 state->name = NULL;
@@ -78,6 +84,18 @@ free_fuse_state (fuse_state_t *state)
         fuse_resolve_wipe (&state->resolve);
         fuse_resolve_wipe (&state->resolve2);
 
+        pthread_mutex_lock (&priv->sync_mutex);
+        {
+                winds = --state->active_subvol->winds;
+                switched = state->active_subvol->switched;
+        }
+        pthread_mutex_unlock (&priv->sync_mutex);
+
+        if ((winds == 0) && (switched)) {
+                xlator_notify (state->active_subvol, GF_EVENT_PARENT_DOWN,
+                               state->active_subvol, NULL);
+        }
+
 #ifdef DEBUG
         memset (state, 0x90, sizeof (*state));
 #endif
@@ -89,8 +107,9 @@ free_fuse_state (fuse_state_t *state)
 fuse_state_t *
 get_fuse_state (xlator_t *this, fuse_in_header_t *finh)
 {
-        fuse_state_t *state = NULL;
-	xlator_t     *active_subvol = NULL;
+        fuse_state_t   *state         = NULL;
+	xlator_t       *active_subvol = NULL;
+        fuse_private_t *priv          = NULL;
 
         state = (void *)GF_CALLOC (1, sizeof (*state),
                                    gf_fuse_mt_fuse_state_t);
@@ -98,7 +117,15 @@ get_fuse_state (xlator_t *this, fuse_in_header_t *finh)
                 return NULL;
 
 	state->this = THIS;
-        active_subvol = fuse_active_subvol (state->this);
+        priv = this->private;
+
+        pthread_mutex_lock (&priv->sync_mutex);
+        {
+                active_subvol = fuse_active_subvol (state->this);
+                active_subvol->winds++;
+        }
+        pthread_mutex_unlock (&priv->sync_mutex);
+
 	state->active_subvol = active_subvol;
 	state->itable = active_subvol->itable;
 
@@ -112,6 +139,7 @@ get_fuse_state (xlator_t *this, fuse_in_header_t *finh)
 }
 
 
+#define FUSE_MAX_AUX_GROUPS 32 /* We can get only up to 32 aux groups from /proc */
 void
 frame_fill_groups (call_frame_t *frame)
 {
@@ -134,6 +162,9 @@ frame_fill_groups (call_frame_t *frame)
         if (!fp)
                 goto out;
 
+	if (call_stack_alloc_groups (frame->root, FUSE_MAX_AUX_GROUPS) != 0)
+		goto out;
+
         while ((ptr = fgets (line, sizeof line, fp))) {
                 if (strncmp (ptr, "Groups:", 7) != 0)
                         continue;
@@ -150,7 +181,7 @@ frame_fill_groups (call_frame_t *frame)
                         if (!endptr || *endptr)
                                 break;
                         frame->root->groups[idx++] = id;
-                        if (idx == GF_MAX_AUX_GROUPS)
+                        if (idx == FUSE_MAX_AUX_GROUPS)
                                 break;
                 }
 
@@ -166,6 +197,7 @@ out:
         prcred_t    *prcred = (prcred_t *) scratch;
         FILE        *fp = NULL;
         int          ret = 0;
+	int          ngrps;
 
         ret = snprintf (filename, sizeof filename,
                         "/proc/%d/cred", frame->root->pid);
@@ -174,8 +206,11 @@ out:
                 fp = fopen (filename, "r");
                 if (fp != NULL) {
                         if (fgets (scratch, sizeof scratch, fp) != NULL) {
-                                frame->root->ngrps = MIN(prcred->pr_ngroups,
-                                                         GF_REQUEST_MAXGROUPS);
+                                ngrps = MIN(prcred->pr_ngroups,
+					    GF_MAX_AUX_GROUPS);
+				if (call_stack_alloc_groups (frame->root,
+							     ngrps) != 0)
+					return;
                         }
                         fclose (fp);
                  }
@@ -200,7 +235,9 @@ out:
 
         if (sysctl(name, namelen, &kp, &kplen, NULL, 0) != 0)
                 return;
-        ngroups = MIN(kp.kp_eproc.e_ucred.cr_ngroups, GF_REQUEST_MAXGROUPS);
+        ngroups = MIN(kp.kp_eproc.e_ucred.cr_ngroups, GF_MAX_AUX_GROUPS);
+	if (call_stack_alloc_groups (frame->root, ngroups) != 0)
+		return;
         for (i = 0; i < ngroups; i++)
                 frame->root->groups[i] = kp.kp_eproc.e_ucred.cr_groups[i];
         frame->root->ngrps = ngroups;
@@ -209,6 +246,53 @@ out:
 #endif /* GF_LINUX_HOST_OS */
 }
 
+/*
+ * Get the groups for the PID associated with this frame. If enabled,
+ * use the gid cache to reduce group list collection.
+ */
+static void get_groups(fuse_private_t *priv, call_frame_t *frame)
+{
+	int i;
+	const gid_list_t *gl;
+	gid_list_t agl;
+
+        if (-1 == priv->gid_cache_timeout) {
+                frame->root->ngrps = 0;
+                return;
+        }
+
+	if (!priv->gid_cache_timeout) {
+		frame_fill_groups(frame);
+		return;
+	}
+
+	gl = gid_cache_lookup(&priv->gid_cache, frame->root->pid);
+	if (gl) {
+		if (call_stack_alloc_groups (frame->root, gl->gl_count) != 0)
+			return;
+		frame->root->ngrps = gl->gl_count;
+		for (i = 0; i < gl->gl_count; i++)
+			frame->root->groups[i] = gl->gl_list[i];
+		gid_cache_release(&priv->gid_cache, gl);
+		return;
+	}
+
+	frame_fill_groups (frame);
+
+	agl.gl_id = frame->root->pid;
+	agl.gl_count = frame->root->ngrps;
+	agl.gl_list = GF_CALLOC(frame->root->ngrps, sizeof(gid_t),
+			gf_fuse_mt_gids_t);
+	if (!agl.gl_list)
+		return;
+
+	for (i = 0; i < frame->root->ngrps; i++)
+		agl.gl_list[i] = frame->root->groups[i];
+
+	if (gid_cache_add(&priv->gid_cache, &agl) != 1)
+		GF_FREE(agl.gl_list);
+}
+
 call_frame_t *
 get_call_frame_for_req (fuse_state_t *state)
 {
@@ -236,7 +320,7 @@ get_call_frame_for_req (fuse_state_t *state)
                                           state->lk_owner);
         }
 
-        frame_fill_groups (frame);
+	get_groups(priv, frame);
 
         if (priv && priv->client_pid_set)
                 frame->root->pid = priv->client_pid;
@@ -268,7 +352,9 @@ fuse_ino_to_inode (uint64_t ino, xlator_t *fuse)
 uint64_t
 inode_to_fuse_nodeid (inode_t *inode)
 {
-        if (!inode || __is_root_gfid (inode->gfid))
+        if (!inode)
+		return 0;
+	if (__is_root_gfid (inode->gfid))
                 return 1;
 
         return (unsigned long) inode;
@@ -356,54 +442,81 @@ fuse_loc_fill (loc_t *loc, fuse_state_t *state, ino_t ino,
         }
         ret = 0;
 fail:
+        /* this should not happen as inode_path returns -1 when buf is NULL
+           for sure */
+        if (path && !loc->path)
+                GF_FREE (path);
         return ret;
 }
 
+/* Use the same logic as the Linux NFS-client */
+#define GF_FUSE_SQUASH_INO(ino) ((uint32_t) ino) ^ (ino >> 32)
 
 /* courtesy of folly */
 void
-gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa)
+gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa, gf_boolean_t enable_ino32)
 {
-        fa->ino        = st->ia_ino;
-        fa->size       = st->ia_size;
-        fa->blocks     = st->ia_blocks;
-        fa->atime      = st->ia_atime;
-        fa->mtime      = st->ia_mtime;
-        fa->ctime      = st->ia_ctime;
-        fa->atimensec  = st->ia_atime_nsec;
-        fa->mtimensec  = st->ia_mtime_nsec;
-        fa->ctimensec  = st->ia_ctime_nsec;
-        fa->mode       = st_mode_from_ia (st->ia_prot, st->ia_type);
-        fa->nlink      = st->ia_nlink;
-        fa->uid        = st->ia_uid;
-        fa->gid        = st->ia_gid;
-        fa->rdev       = makedev (ia_major (st->ia_rdev),
-                                  ia_minor (st->ia_rdev));
+        if (enable_ino32)
+                fa->ino = GF_FUSE_SQUASH_INO(st->ia_ino);
+        else
+                fa->ino = st->ia_ino;
+
+        fa->size        = st->ia_size;
+        fa->blocks      = st->ia_blocks;
+        fa->atime       = st->ia_atime;
+        fa->mtime       = st->ia_mtime;
+        fa->ctime       = st->ia_ctime;
+        fa->atimensec   = st->ia_atime_nsec;
+        fa->mtimensec   = st->ia_mtime_nsec;
+        fa->ctimensec   = st->ia_ctime_nsec;
+        fa->mode        = st_mode_from_ia (st->ia_prot, st->ia_type);
+        fa->nlink       = st->ia_nlink;
+        fa->uid         = st->ia_uid;
+        fa->gid         = st->ia_gid;
+        fa->rdev        = makedev (ia_major (st->ia_rdev),
+                                   ia_minor (st->ia_rdev));
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-        fa->blksize    = st->ia_blksize;
+        fa->blksize     = st->ia_blksize;
 #endif
 #ifdef GF_DARWIN_HOST_OS
-        fa->crtime     = (uint64_t)-1;
-        fa->crtimensec = (uint32_t)-1;
-        fa->flags      = 0;
+        fa->crtime      = (uint64_t)-1;
+        fa->crtimensec  = (uint32_t)-1;
+        fa->flags       = 0;
 #endif
 }
 
-int
-fuse_flip_user_to_trusted (char *okey, char **nkey)
+void
+gf_fuse_fill_dirent (gf_dirent_t *entry, struct fuse_dirent *fde, gf_boolean_t enable_ino32)
+{
+        if (enable_ino32)
+                fde->ino = GF_FUSE_SQUASH_INO(entry->d_ino);
+        else
+                fde->ino = entry->d_ino;
+
+        fde->off         = entry->d_off;
+        fde->type        = entry->d_type;
+        fde->namelen     = strlen (entry->d_name);
+        strncpy (fde->name, entry->d_name, fde->namelen);
+}
+
+static int
+fuse_do_flip_xattr_ns (char *okey, const char *nns, char **nkey)
 {
         int   ret = 0;
         char *key = NULL;
 
-        key = GF_CALLOC (1, strlen(okey) + 10, gf_common_mt_char);
+        okey = strchr (okey, '.');
+        GF_ASSERT (okey);
+
+        key = GF_CALLOC (1, strlen (nns) + strlen(okey) + 1,
+                         gf_common_mt_char);
         if (!key) {
                 ret = -1;
                 goto out;
         }
 
-        okey += 5;
-        strncpy(key, "trusted.", 8);
-        strncat(key+8, okey, strlen(okey));
+        strcpy (key, nns);
+        strcat (key, okey);
 
         *nkey = key;
 
@@ -411,7 +524,7 @@ fuse_flip_user_to_trusted (char *okey, char **nkey)
         return ret;
 }
 
-int
+static int
 fuse_xattr_alloc_default (char *okey, char **nkey)
 {
         int ret = 0;
@@ -422,56 +535,71 @@ fuse_xattr_alloc_default (char *okey, char **nkey)
         return ret;
 }
 
+#define PRIV_XA_NS   "trusted"
+#define UNPRIV_XA_NS "system"
+
 int
 fuse_flip_xattr_ns (fuse_private_t *priv, char *okey, char **nkey)
 {
         int             ret       = 0;
         gf_boolean_t    need_flip = _gf_false;
-        gf_client_pid_t npid      = 0;
-
-        npid = priv->client_pid;
-        if (gf_client_pid_check (npid)) {
-                ret = fuse_xattr_alloc_default (okey, nkey);
-                goto out;
-        }
-
-        switch (npid) {
-                /*
-                 * These two cases will never execute as we check the
-                 * pid range above, but are kept to keep the compiler
-                 * happy.
-                 */
-        case GF_CLIENT_PID_MAX:
-        case GF_CLIENT_PID_MIN:
-                goto out;
 
+        switch (priv->client_pid) {
         case GF_CLIENT_PID_GSYNCD:
                 /* valid xattr(s): *xtime, volume-mark* */
                 gf_log("glusterfs-fuse", GF_LOG_DEBUG, "PID: %d, checking xattr(s): "
-                       "volume-mark*, *xtime", npid);
-                if ( (strcmp (okey, "user.glusterfs.volume-mark") == 0)
-                     || (fnmatch (okey, "user.glusterfs.volume-mark.*", FNM_PERIOD) == 0)
-                     || (fnmatch ("user.glusterfs.*.xtime", okey, FNM_PERIOD) == 0) )
+                       "volume-mark*, *xtime", priv->client_pid);
+                if ( (strcmp (okey, UNPRIV_XA_NS".glusterfs.volume-mark") == 0)
+                     || (fnmatch (UNPRIV_XA_NS".glusterfs.volume-mark.*", okey, FNM_PERIOD) == 0)
+                     || (fnmatch (UNPRIV_XA_NS".glusterfs.*.xtime", okey, FNM_PERIOD) == 0) )
                         need_flip = _gf_true;
                 break;
 
         case GF_CLIENT_PID_HADOOP:
                 /* valid xattr(s): pathinfo */
                 gf_log("glusterfs-fuse", GF_LOG_DEBUG, "PID: %d, checking xattr(s): "
-                       "pathinfo", npid);
-                if (strcmp (okey, "user.glusterfs.pathinfo") == 0)
+                       "pathinfo", priv->client_pid);
+                if (strcmp (okey, UNPRIV_XA_NS".glusterfs.pathinfo") == 0)
                         need_flip = _gf_true;
                 break;
         }
 
         if (need_flip) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "flipping %s to trusted equivalent",
+                gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "flipping %s to "PRIV_XA_NS" equivalent",
                         okey);
-                ret = fuse_flip_user_to_trusted (okey, nkey);
+                ret = fuse_do_flip_xattr_ns (okey, PRIV_XA_NS, nkey);
         } else {
                 /* if we cannot match, continue with what we got */
                 ret = fuse_xattr_alloc_default (okey, nkey);
         }
+
+        return ret;
+}
+
+int
+fuse_ignore_xattr_set (fuse_private_t *priv, char *key)
+{
+        int ret = 0;
+
+        /* don't mess with user namespace */
+        if (fnmatch ("user.*", key, FNM_PERIOD) == 0)
+                goto out;
+
+        if (priv->client_pid != GF_CLIENT_PID_GSYNCD)
+                goto out;
+
+        /* trusted NS check */
+        if (!((fnmatch ("*.glusterfs.*.xtime", key, FNM_PERIOD) == 0)
+              || (fnmatch ("*.glusterfs.volume-mark",
+                           key, FNM_PERIOD) == 0)
+              || (fnmatch ("*.glusterfs.volume-mark.*",
+                           key, FNM_PERIOD) == 0)))
+                ret = -1;
+
  out:
+        gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "%s setxattr: key [%s], "
+                " client pid [%d]", (ret ? "disallowing" : "allowing"), key,
+                priv->client_pid);
+
         return ret;
 }
diff --git a/xlators/mount/fuse/src/fuse-mem-types.h b/xlators/mount/fuse/src/fuse-mem-types.h
index 1fb959c3a..28b4dfbdd 100644
--- a/xlators/mount/fuse/src/fuse-mem-types.h
+++ b/xlators/mount/fuse/src/fuse-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __FUSE_MEM_TYPES_H__
 #define __FUSE_MEM_TYPES_H__
 
@@ -30,6 +20,8 @@ enum gf_fuse_mem_types_ {
         gf_fuse_mt_iov_base,
         gf_fuse_mt_fuse_state_t,
         gf_fuse_mt_fd_ctx_t,
+        gf_fuse_mt_graph_switch_args_t,
+	gf_fuse_mt_gids_t,
         gf_fuse_mt_end
 };
 #endif
diff --git a/xlators/mount/fuse/src/fuse-resolve.c b/xlators/mount/fuse/src/fuse-resolve.c
index 1af80b93c..8565ce0e4 100644
--- a/xlators/mount/fuse/src/fuse-resolve.c
+++ b/xlators/mount/fuse/src/fuse-resolve.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -30,7 +20,13 @@ fuse_resolve_all (fuse_state_t *state);
 int fuse_resolve_continue (fuse_state_t *state);
 int fuse_resolve_entry_simple (fuse_state_t *state);
 int fuse_resolve_inode_simple (fuse_state_t *state);
+int fuse_migrate_fd (xlator_t *this, fd_t *fd, xlator_t *old_subvol,
+                     xlator_t *new_subvol);
 
+fuse_fd_ctx_t *
+fuse_fd_ctx_get (xlator_t *this, fd_t *fd);
+
+gf_boolean_t fuse_inode_needs_lookup (inode_t *inode, xlator_t *this);
 
 static int
 fuse_resolve_loc_touchup (fuse_state_t *state)
@@ -80,11 +76,13 @@ fuse_resolve_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         STACK_DESTROY (frame->root);
 
         if (op_ret == -1) {
-                gf_log (this->name, ((op_errno == ENOENT) ? GF_LOG_DEBUG :
-                                     GF_LOG_WARNING),
+                gf_log (this->name, (op_errno == ENOENT)
+                        ? GF_LOG_DEBUG : GF_LOG_WARNING,
                         "%s/%s: failed to resolve (%s)",
                         uuid_utoa (resolve_loc->pargfid), resolve_loc->name,
                         strerror (op_errno));
+                resolve->op_ret = -1;
+                resolve->op_errno = op_errno;
                 goto out;
         }
 
@@ -92,6 +90,7 @@ fuse_resolve_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                  resolve_loc->name, buf);
 
 	state->loc_now->inode = link_inode;
+
 out:
         loc_wipe (resolve_loc);
 
@@ -126,8 +125,8 @@ fuse_resolve_entry (fuse_state_t *state)
 
 int
 fuse_resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-                       dict_t *xattr, struct iatt *postparent)
+                       int op_ret, int op_errno, inode_t *inode,
+                       struct iatt *buf, dict_t *xattr, struct iatt *postparent)
 {
         fuse_state_t   *state      = NULL;
         fuse_resolve_t *resolve    = NULL;
@@ -141,12 +140,26 @@ fuse_resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         STACK_DESTROY (frame->root);
 
         if (op_ret == -1) {
-                gf_log (this->name, ((op_errno == ENOENT) ? GF_LOG_DEBUG :
-                                     GF_LOG_WARNING),
+                gf_log (this->name, (op_errno == ENOENT)
+                        ? GF_LOG_DEBUG : GF_LOG_WARNING,
                         "%s: failed to resolve (%s)",
                         uuid_utoa (resolve->resolve_loc.gfid),
 			strerror (op_errno));
                 loc_wipe (&resolve->resolve_loc);
+
+                /* resolve->op_ret can have 3 values: 0, -1, -2.
+                 * 0 : resolution was successful.
+                 * -1: parent inode could not be resolved.
+                 * -2: entry (inode corresponding to path) could not be resolved
+                 */
+
+                if (uuid_is_null (resolve->gfid)) {
+                        resolve->op_ret = -1;
+                } else {
+                        resolve->op_ret = -2;
+                }
+
+                resolve->op_errno = op_errno;
                 goto out;
         }
 
@@ -163,6 +176,7 @@ fuse_resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 	}
 
 	loc_now->parent = link_inode;
+        uuid_copy (loc_now->pargfid, link_inode->gfid);
 
 	fuse_resolve_entry (state);
 
@@ -189,13 +203,16 @@ fuse_resolve_gfid (fuse_state_t *state)
                 uuid_copy (resolve_loc->gfid, resolve->gfid);
         }
 
-	resolve_loc->inode = inode_new (state->itable);
-	ret = inode_path (resolve_loc->inode, NULL,
-			  (char **)&resolve_loc->path);
+	/* inode may already exist in case we are looking up an inode which was
+	   linked through readdirplus */
+	resolve_loc->inode = inode_find (state->itable, resolve_loc->gfid);
+	if (!resolve_loc->inode)
+		resolve_loc->inode = inode_new (state->itable);
+	ret = loc_path (resolve_loc, NULL);
 
         if (ret <= 0) {
                 gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to get the path from inode %s",
+                        "failed to get the path for inode %s",
                         uuid_utoa (resolve->gfid));
         }
 
@@ -228,22 +245,42 @@ fuse_resolve_parent_simple (fuse_state_t *state)
 
 	parent = resolve->parhint;
 	if (parent->table == state->itable) {
+		if (fuse_inode_needs_lookup (parent, THIS))
+			return 1;
+
 		/* no graph switches since */
 		loc->parent = inode_ref (parent);
+		uuid_copy (loc->pargfid, parent->gfid);
 		loc->inode = inode_grep (state->itable, parent, loc->name);
+
+                /* nodeid for root is 1 and we blindly take the latest graph's
+                 * table->root as the parhint and because of this there is
+                 * ambiguity whether the entry should have existed or not, and
+                 * we took the conservative approach of assuming entry should
+                 * have been there even though it need not have (bug #804592).
+                 */
+                if ((loc->inode == NULL)
+                    && __is_root_gfid (parent->gfid)) {
+                        /* non decisive result - entry missing */
+                        return -1;
+                }
+
 		/* decisive result - resolution success */
 		return 0;
 	}
 
         parent = inode_find (state->itable, resolve->pargfid);
 	if (!parent) {
-		resolve->op_ret   = -1;
-		resolve->op_errno = ENOENT;
 		/* non decisive result - parent missing */
 		return 1;
 	}
+	if (fuse_inode_needs_lookup (parent, THIS)) {
+		inode_unref (parent);
+		return 1;
+	}
 
 	loc->parent = parent;
+        uuid_copy (loc->pargfid, resolve->pargfid);
 
 	inode = inode_grep (state->itable, parent, loc->name);
 	if (inode) {
@@ -261,9 +298,6 @@ int
 fuse_resolve_parent (fuse_state_t *state)
 {
         int    ret = 0;
-        loc_t *loc = NULL;
-
-        loc  = state->loc_now;
 
         ret = fuse_resolve_parent_simple (state);
         if (ret > 0) {
@@ -293,18 +327,18 @@ fuse_resolve_inode_simple (fuse_state_t *state)
 	loc = state->loc_now;
 
 	inode = resolve->hint;
-	if (inode->table == state->itable) {
+	if (inode->table == state->itable)
 		inode_ref (inode);
-		goto found;
+	else
+		inode = inode_find (state->itable, resolve->gfid);
+
+        if (inode) {
+		if (!fuse_inode_needs_lookup (inode, THIS))
+			goto found;
+		/* inode was linked through readdirplus */
+		inode_unref (inode);
 	}
 
-        inode = inode_find (state->itable, resolve->gfid);
-        if (inode)
-		goto found;
-
-	resolve->op_ret   = -1;
-	resolve->op_errno = ENOENT;
-
         return 1;
 found:
 	loc->inode = inode;
@@ -316,9 +350,6 @@ int
 fuse_resolve_inode (fuse_state_t *state)
 {
         int                 ret = 0;
-        loc_t              *loc = NULL;
-
-        loc  = state->loc_now;
 
         ret = fuse_resolve_inode_simple (state);
 
@@ -332,19 +363,200 @@ fuse_resolve_inode (fuse_state_t *state)
         return 0;
 }
 
+
+int
+fuse_migrate_fd_task (void *data)
+{
+        int            ret        = -1;
+        fuse_state_t  *state      = NULL;
+        fd_t          *basefd     = NULL, *oldfd = NULL;
+        fuse_fd_ctx_t *basefd_ctx = NULL;
+        xlator_t      *old_subvol = NULL;
+
+        state = data;
+        if (state == NULL) {
+                goto out;
+        }
+
+        basefd = state->fd;
+
+        basefd_ctx = fuse_fd_ctx_get (state->this, basefd);
+
+        LOCK (&basefd->lock);
+        {
+                oldfd = basefd_ctx->activefd ? basefd_ctx->activefd : basefd;
+                fd_ref (oldfd);
+        }
+        UNLOCK (&basefd->lock);
+
+        old_subvol = oldfd->inode->table->xl;
+
+        ret = fuse_migrate_fd (state->this, basefd, old_subvol,
+                               state->active_subvol);
+
+        LOCK (&basefd->lock);
+        {
+                if (ret < 0) {
+                        basefd_ctx->migration_failed = 1;
+                } else {
+                        basefd_ctx->migration_failed = 0;
+                }
+        }
+        UNLOCK (&basefd->lock);
+
+        ret = 0;
+
+out:
+        if (oldfd)
+                fd_unref (oldfd);
+
+        return ret;
+}
+
+
+static inline int
+fuse_migrate_fd_error (xlator_t *this, fd_t *fd)
+{
+        fuse_fd_ctx_t *fdctx = NULL;
+        char           error = 0;
+
+        fdctx = fuse_fd_ctx_get (this, fd);
+        if (fdctx != NULL) {
+                if (fdctx->migration_failed) {
+                        error = 1;
+                }
+        }
+
+        return error;
+}
+
+#define FUSE_FD_GET_ACTIVE_FD(activefd, basefd)                 \
+        do {                                                    \
+                LOCK (&basefd->lock);                           \
+                {                                               \
+                        activefd = basefd_ctx->activefd ?       \
+                                basefd_ctx->activefd : basefd;  \
+                        if (activefd != basefd) {               \
+                                fd_ref (activefd);              \
+                        }                                       \
+                }                                               \
+                UNLOCK (&basefd->lock);                         \
+                                                                \
+                if (activefd == basefd) {                       \
+                        fd_ref (activefd);                      \
+                }                                               \
+        } while (0);
+
+
 static int
 fuse_resolve_fd (fuse_state_t *state)
 {
-        fuse_resolve_t  *resolve    = NULL;
-	fd_t            *fd         = NULL;
-	xlator_t        *active_subvol = NULL;
+        fuse_resolve_t *resolve            = NULL;
+	fd_t           *basefd             = NULL, *activefd = NULL;
+	xlator_t       *active_subvol      = NULL, *this = NULL;
+        int             ret                = 0;
+        char            fd_migration_error = 0;
+        fuse_fd_ctx_t  *basefd_ctx         = NULL;
 
         resolve = state->resolve_now;
 
-        fd = resolve->fd;
-	active_subvol = fd->inode->table->xl;
+        this = state->this;
+
+        basefd = resolve->fd;
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        if (basefd_ctx == NULL) {
+                gf_log (state->this->name, GF_LOG_WARNING,
+                        "fdctx is NULL for basefd (ptr:%p inode-gfid:%s), "
+                        "resolver erroring out with errno EINVAL",
+                        basefd, uuid_utoa (basefd->inode->gfid));
+                resolve->op_ret = -1;
+                resolve->op_errno = EINVAL;
+                goto resolve_continue;
+        }
+
+        FUSE_FD_GET_ACTIVE_FD (activefd, basefd);
+
+        active_subvol = activefd->inode->table->xl;
+
+        fd_migration_error = fuse_migrate_fd_error (state->this, basefd);
+        if (fd_migration_error) {
+                resolve->op_ret = -1;
+                resolve->op_errno = EBADF;
+        } else if (state->active_subvol != active_subvol) {
+                ret = synctask_new (state->this->ctx->env, fuse_migrate_fd_task,
+                                    NULL, NULL, state);
+
+                fd_migration_error = fuse_migrate_fd_error (state->this,
+                                                            basefd);
+                fd_unref (activefd);
+
+                FUSE_FD_GET_ACTIVE_FD (activefd, basefd);
+                active_subvol = activefd->inode->table->xl;
+
+                if ((ret == -1) || fd_migration_error
+                    || (state->active_subvol != active_subvol)) {
+                        if (ret == -1) {
+                                gf_log (state->this->name, GF_LOG_WARNING,
+                                        "starting sync-task to migrate "
+                                        "basefd (ptr:%p inode-gfid:%s) failed "
+                                        "(old-subvolume:%s-%d "
+                                        "new-subvolume:%s-%d)",
+                                        basefd,
+                                        uuid_utoa (basefd->inode->gfid),
+                                        active_subvol->name,
+                                        active_subvol->graph->id,
+                                        state->active_subvol->name,
+                                        state->active_subvol->graph->id);
+                        } else {
+                                gf_log (state->this->name, GF_LOG_WARNING,
+                                        "fd migration of basefd "
+                                        "(ptr:%p inode-gfid:%s) failed "
+                                        "(old-subvolume:%s-%d "
+                                        "new-subvolume:%s-%d)",
+                                        basefd,
+                                        uuid_utoa (basefd->inode->gfid),
+                                        active_subvol->name,
+                                        active_subvol->graph->id,
+                                        state->active_subvol->name,
+                                        state->active_subvol->graph->id);
+                        }
+
+                        resolve->op_ret = -1;
+                        resolve->op_errno = EBADF;
+                } else {
+                        gf_log (state->this->name, GF_LOG_DEBUG,
+                                "basefd (ptr:%p inode-gfid:%s) migrated "
+                                "successfully in resolver "
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                basefd, uuid_utoa (basefd->inode->gfid),
+                                active_subvol->name, active_subvol->graph->id,
+                                state->active_subvol->name,
+                                state->active_subvol->graph->id);
+                }
+        }
+
+        if ((resolve->op_ret == -1) && (resolve->op_errno == EBADF)) {
+                gf_log ("fuse-resolve", GF_LOG_WARNING,
+                        "migration of basefd (ptr:%p inode-gfid:%s) "
+                        "did not complete, failing fop with EBADF "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid),
+                        active_subvol->name, active_subvol->graph->id,
+                        state->active_subvol->name,
+                        state->active_subvol->graph->id);
+        }
+
+        if (activefd != basefd) {
+                state->fd = fd_ref (activefd);
+                fd_unref (basefd);
+        }
+
+	/* state->active_subvol = active_subvol; */
 
-	state->active_subvol = active_subvol;
+resolve_continue:
+        if (activefd != NULL) {
+                fd_unref (activefd);
+        }
 
         fuse_resolve_continue (state);
 
@@ -360,15 +572,15 @@ fuse_gfid_set (fuse_state_t *state)
         if (uuid_is_null (state->gfid))
                 goto out;
 
-        if (!state->dict)
-                state->dict = dict_new ();
+        if (!state->xdata)
+                state->xdata = dict_new ();
 
-        if (!state->dict) {
+        if (!state->xdata) {
                 ret = -1;
                 goto out;
         }
 
-        ret = dict_set_static_bin (state->dict, "gfid-req",
+        ret = dict_set_static_bin (state->xdata, "gfid-req",
                                    state->gfid, sizeof (state->gfid));
 out:
         return ret;
@@ -434,10 +646,6 @@ fuse_resolve (fuse_state_t *state)
                 fuse_resolve_inode (state);
 
         } else {
-
-                resolve->op_ret = 0;
-                resolve->op_errno = EINVAL;
-
                 fuse_resolve_all (state);
         }
 
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index 40fd5781e..a192d6059 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 # (C) 2006, 2007, 2008 Gluster Inc. <http://www.gluster.com>
 #
 # This program is free software; you can redistribute it and/or
@@ -27,137 +27,233 @@ _init ()
     LOG_DEBUG=DEBUG;
     LOG_TRACE=TRACE;
 
+    HOST_NAME_MAX=64;
+
     prefix="@prefix@";
     exec_prefix=@exec_prefix@;
     cmd_line=$(echo "@sbindir@/glusterfs");
 
+    case `uname -s` in
+    NetBSD)
+        getinode="stat -f %i"
+        getdev="stat -f %d"
+        lgetinode="${getinode} -L"
+        lgetdev="${getdev} -L"
+
+        mounttab=/proc/mounts
+        ;;
+    Linux)
+        getinode="stat -c %i $i"
+        getdev="stat -c %d $d"
+        lgetinode="${getinode} -L"
+        lgetdev="${getdev} -L"
+
+        mounttab=/etc/mtab
+        ;;
+    esac
+
     UPDATEDBCONF=/etc/updatedb.conf
 }
 
+parse_backup_volfile_servers ()
+{
+    local server_list=$1
+    local servers=""
+    local new_servers=""
+
+    servers=$(echo ${server_list} | sed 's/\:/ /g')
+    for server in ${servers}; do
+        length=$(echo $server | wc -c)
+        if [ ${length} -gt ${HOST_NAME_MAX} ]; then
+            echo "Hostname:${server} provided is too long.. skipping"
+            continue
+        fi
+        new_servers=$(echo "$new_servers $server")
+    done
+    echo ${new_servers}
+}
+
 start_glusterfs ()
 {
+    # lets the comparsion be case insensitive for all strings
+
     if [ -n "$log_level_str" ]; then
-	case "$log_level_str" in
-	    "ERROR")
-		log_level=$LOG_ERROR;
-		;;
+        case "$( echo $log_level_str | tr '[a-z]' '[A-Z]')" in
+            "ERROR")
+                log_level=$LOG_ERROR;
+                ;;
             "INFO")
                 log_level=$LOG_INFO
                 ;;
-	    "DEBUG")
-		log_level=$LOG_DEBUG;
-		;;
-	    "CRITICAL")
-		log_level=$LOG_CRITICAL;
-		;;
-	    "WARNING")
-		log_level=$LOG_WARNING;
-		;;
-	    "TRACE")
-		log_level=$LOG_TRACE;
-		;;
-	    "NONE")
-		log_level=$LOG_NONE;
-		;;
-	    *)
-		echo "invalid log level $log_level_str, using INFO";
-		log_level=$LOG_INFO;
-		;;
-	esac
-    fi
-    if [ -n "$log_level" ]; then
-        cmd_line=$(echo "$cmd_line --log-level=$log_level");
+            "DEBUG")
+                log_level=$LOG_DEBUG;
+                ;;
+            "CRITICAL")
+                log_level=$LOG_CRITICAL;
+                ;;
+            "WARNING")
+                log_level=$LOG_WARNING;
+                ;;
+            "TRACE")
+                log_level=$LOG_TRACE;
+                ;;
+            "NONE")
+                log_level=$LOG_NONE;
+                ;;
+            *)
+                echo "invalid log level $log_level_str, using INFO";
+                log_level=$LOG_INFO;
+                ;;
+        esac
     fi
 
+#options without values start here
     if [ -n "$read_only" ]; then
-	cmd_line=$(echo "$cmd_line --read-only");
+        cmd_line=$(echo "$cmd_line --read-only");
     fi
 
     if [ -n "$acl" ]; then
-	cmd_line=$(echo "$cmd_line --acl");
+        cmd_line=$(echo "$cmd_line --acl");
+    fi
+
+    if [ -n "$selinux" ]; then
+        cmd_line=$(echo "$cmd_line --selinux");
+    fi
+
+    if [ -n "$enable_ino32" ]; then
+        cmd_line=$(echo "$cmd_line --enable-ino32");
     fi
 
     if [ -n "$worm" ]; then
         cmd_line=$(echo "$cmd_line --worm");
     fi
 
-    if [ -n "$log_file" ]; then
-	cmd_line=$(echo "$cmd_line --log-file=$log_file");
+    if [ -n "$fopen_keep_cache" ]; then
+        cmd_line=$(echo "$cmd_line --fopen-keep-cache");
     fi
 
     if [ -n "$volfile_check" ]; then
-	cmd_line=$(echo "$cmd_line --volfile-check");
+        cmd_line=$(echo "$cmd_line --volfile-check");
+    fi
+
+    if [ -n "$mem_accounting" ]; then
+        cmd_line=$(echo "$cmd_line --mem-accounting");
+    fi
+
+    if [ -n "$aux_gfid_mount" ]; then
+        cmd_line=$(echo "$cmd_line --aux-gfid-mount");
+    fi
+
+#options with values start here
+    if [ -n "$log_level" ]; then
+        cmd_line=$(echo "$cmd_line --log-level=$log_level");
+    fi
+
+    if [ -n "$log_file" ]; then
+        cmd_line=$(echo "$cmd_line --log-file=$log_file");
     fi
 
     if [ -n "$direct_io_mode" ]; then
-	cmd_line=$(echo "$cmd_line --direct-io-mode=$direct_io_mode");
+        cmd_line=$(echo "$cmd_line --direct-io-mode=$direct_io_mode");
+    fi
+
+    if [ -n "$use_readdirp" ]; then
+        cmd_line=$(echo "$cmd_line --use-readdirp=$use_readdirp");
     fi
 
     if [ -n "$volume_name" ]; then
         cmd_line=$(echo "$cmd_line --volume-name=$volume_name");
     fi
 
-    if [ -n "$log_server" ]; then
-        if [ -n "$log_server_port" ]; then
-            cmd_line=$(echo "$cmd_line \
---log-server=$log_server \
---log-server-port=$log_server_port");
-        fi
+    if [ -n "$attribute_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --attribute-timeout=$attribute_timeout");
+    fi
+
+    if [ -n "$entry_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --entry-timeout=$entry_timeout");
+    fi
+
+    if [ -n "$negative_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --negative-timeout=$negative_timeout");
+    fi
+
+    if [ -n "$gid_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --gid-timeout=$gid_timeout");
+    fi
+
+    if [ -n "$bg_qlen" ]; then
+        cmd_line=$(echo "$cmd_line --background-qlen=$bg_qlen");
+    fi
+
+    if [ -n "$cong_threshold" ]; then
+        cmd_line=$(echo "$cmd_line --congestion-threshold=$cong_threshold");
+    fi
+
+    if [ -n "$fuse_mountopts" ]; then
+        cmd_line=$(echo "$cmd_line --fuse-mountopts=$fuse_mountopts");
     fi
 
+    if [ -n "$xlator_option" ]; then
+        xlator_option=$(echo $xlator_option | sed s/"xlator-option="/"--xlator-option "/g)
+        cmd_line=$(echo "$cmd_line $xlator_option");
+    fi
+
+    # for rdma volume, we have to fetch volfile with '.rdma' added
+    # to volume name, so that it fetches the right client vol file
+    volume_id_rdma="";
+
     if [ -z "$volfile_loc" ]; then
         if  [ -n "$server_ip" ]; then
+
+            cmd_line=$(echo "$cmd_line --volfile-server=$server_ip");
+
+            if [ -n "$backup_volfile_servers" ]; then
+                servers=$(parse_backup_volfile_servers ${backup_volfile_servers})
+                for i in $(echo ${servers}); do
+                    cmd_line=$(echo "$cmd_line --volfile-server=$i");
+                done
+            fi
+
             if [ -n "$server_port" ]; then
                 cmd_line=$(echo "$cmd_line --volfile-server-port=$server_port");
             fi
-	    if [ -n "$transport" ]; then
+
+            if [ -n "$transport" ]; then
                 cmd_line=$(echo "$cmd_line --volfile-server-transport=$transport");
+                if [ "$transport" = "rdma" ]; then
+                    volume_id_rdma=".rdma";
+                fi
             fi
+
             if [ -n "$volume_id" ]; then
+                if [ -n "$volume_id_rdma" ]; then
+                    volume_id="$volume_id$volume_id_rdma";
+                fi
                 cmd_line=$(echo "$cmd_line --volfile-id=$volume_id");
             fi
-
-            if [ -n "$backupvolfile_server" ]; then
-                cmd_line1=$(echo "$cmd_line --volfile-server=$backupvolfile_server");
-            fi
-            if [ -n "$volfile_max_fetch_attempts" ]; then
-                cmd_line=$(echo "$cmd_line --volfile-max-fetch-attempts=$volfile_max_fetch_attempts");
-            fi
-	    cmd_line=$(echo "$cmd_line --volfile-server=$server_ip");
         fi
     else
         cmd_line=$(echo "$cmd_line --volfile=$volfile_loc");
     fi
 
+    if [ -n "$fuse_mountopts" ]; then
+        cmd_line=$(echo "$cmd_line --fuse-mountopts=$fuse_mountopts");
+    fi
+
     cmd_line=$(echo "$cmd_line $mount_point");
     err=0;
     $cmd_line;
 
 
-    inode=$(stat -c %i $mount_point 2>/dev/null);
+    inode=$( ${getinode} $mount_point 2>/dev/null);
 
     # this is required if the stat returns error
     if [ -z "$inode" ]; then
         inode="0";
     fi
 
-    # retry the failover
-    # if [ $? != "0" ]; then # <--- TODO: Once glusterfs returns proper error code, change it.
     if [ $inode -ne 1 ]; then
         err=1;
-        if [ -n "$cmd_line1" ]; then
-            cmd_line1=$(echo "$cmd_line1 $mount_point");
-            $cmd_line1
-
-            inode=$(stat -c %i $mount_point 2>/dev/null);
-            # this is required if the stat returns error
-            if [ -z "$inode" ]; then
-                inode="0";
-            fi
-            if [ $inode -ne 1]; then
-                err=1;
-            fi
-        fi
     fi
 
     if [ $err -eq "1" ]; then
@@ -182,13 +278,13 @@ mount.glusterfs --version"
 # check for recursive mounts. i.e, mounting over an existing brick
 check_recursive_mount ()
 {
-    if [ $2 = "/" ]; then
+    if [ $1 = "/" ]; then
         echo Cannot mount over root;
         exit 2;
     fi
     # GFID check first
     # remove trailing / from mount point
-    mnt_dir=${2%/};
+    mnt_dir=${1%/};
 
     export PATH;
     # check whether getfattr exists
@@ -204,18 +300,18 @@ check_recursive_mount ()
     fi
 
     # check if the mount point is a brick's parent directory
-    GLUSTERD_WORKDIR="/etc/glusterd";
+    GLUSTERD_WORKDIR="/var/lib/glusterd";
 
     ls -L "$GLUSTERD_WORKDIR"/vols/*/bricks/* > /dev/null 2>&1;
     if [ $? -ne 0 ]; then
         return;
     fi
 
-    brick_path=(`grep ^path "$GLUSTERD_WORKDIR"/vols/*/bricks/* | cut -d "=" -f 2`);
-    root_inode=`stat -Lc %i /`;
-    root_dev=`stat -Lc %d /`;
-    mnt_inode=`stat -Lc %i $mnt_dir`;
-    mnt_dev=`stat -Lc %d $mnt_dir`;
+    brick_path=`grep ^path "$GLUSTERD_WORKDIR"/vols/*/bricks/* | cut -d "=" -f 2`;
+    root_inode=`${lgetinode} /`;
+    root_dev=`${lgetdev} /`;
+    mnt_inode=`${lgetinode} $mnt_dir`;
+    mnt_dev=`${lgetdev} $mnt_dir`;
     for brick in "$brick_path";
     do
         # evaluate brick path to see if this is local, if non-local, skip iteration
@@ -232,8 +328,8 @@ check_recursive_mount ()
             do
                 tmp_brick="$brick";
                 brick="$brick"/..;
-                brick_dev=`stat -Lc %d $brick`;
-                brick_inode=`stat -Lc %i $brick`;
+                brick_dev=`${lgetdev} $brick`;
+                brick_inode=`${lgetinode} $brick`;
                 if [ "$mnt_inode" -eq "$brick_inode" -a "$mnt_dev" -eq "$brick_dev" ]; then
                     echo ERROR: $mnt_dir is a parent of the brick $tmp_brick;
                     exit 2;
@@ -247,44 +343,79 @@ check_recursive_mount ()
 main ()
 {
     helper=$(echo "$@" | sed -n 's/.*\--[ ]*\([^ ]*\).*/\1/p');
-
-    options=$(echo "$@" | sed -n 's/.*\-o[ ]*\([^ ]*\).*/\1/p');
-
-    new_log_level=$(echo "$options" | sed -n 's/.*log-level=\([^,]*\).*/\1/p');
-
-    [ -n "$new_log_level" ] && {
-	log_level_str="$new_log_level";
-    }
-
-    log_file=$(echo "$options" | sed -n 's/.*log-file=\([^,]*\).*/\1/p');
-
-    read_only=$(echo "$options" | sed -n 's/.*\(ro\)[^,]*.*/\1/p');
-
-    acl=$(echo "$options" | sed -n 's/.*\(acl\)[^,]*.*/\1/p');
-
-    worm=$(echo "$options" | sed -n 's/.*\(worm\)[^,]*.*/\1/p');
-
-    transport=$(echo "$options" | sed -n 's/.*transport=\([^,]*\).*/\1/p');
-
-    direct_io_mode=$(echo "$options" | sed -n 's/.*direct-io-mode=\([^,]*\).*/\1/p');
-
-    volume_name=$(echo "$options" | sed -n 's/.*volume-name=\([^,]*\).*/\1/p');
-
-    volume_id=$(echo "$options" | sed -n 's/.*volume_id=\([^,]*\).*/\1/p');
-
-    volfile_check=$(echo "$options" | sed -n 's/.*volfile-check=\([^,]*\).*/\1/p');
-
-    volfile_max_fetch_attempts=$(echo "$options" | sed -n 's/.*fetch-attempts=\([^,]*\).*/\1/p');
-
-    server_port=$(echo "$options" | sed -n 's/.*server-port=\([^,]*\).*/\1/p');
-
-    backupvolfile_server=$(echo "$options" | sed -n 's/.*backupvolfile-server=\([^,]*\).*/\1/p');
-
-    log_server=$(echo "$options" | sed -n 's/.*log-server=\([^,]*\).*/\1/p');
-
-    log_server_port=$(echo "$options" | sed -n 's/.*log-server-port=\([^,]*\).*/\1/p');
-
-    volfile_loc="$1";
+    in_opt="no"
+    pos_args=0
+    for opt in "$@"; do
+        if [ "$in_opt" = "yes" ]; then
+            for pair in $(echo "$opt" | tr "," " "); do
+                # Handle options without values.
+                case "$pair" in
+                    "ro")               read_only=1 ;;
+                    "acl")              acl=1 ;;
+                    "selinux")          selinux=1 ;;
+                    "worm")             worm=1 ;;
+                    "fopen-keep-cache") fopen_keep_cache=1 ;;
+                    "enable-ino32")     enable_ino32=1 ;;
+                    "mem-accounting")   mem_accounting=1;;
+                    "aux-gfid-mount")
+                        if [ `uname -s` = "Linux" ]; then
+                            aux_gfid_mount=1
+                        fi
+                        ;;
+                    # "mount -t glusterfs" sends this, but it's useless.
+                    "rw")       ;;
+                    # these ones are interpreted during system initialization
+                    "noauto")   ;;
+                    "_netdev")  ;;
+                    *)
+                        key=$(echo "$pair" | cut -f1 -d'=');
+                        value=$(echo "$pair" | cut -f2- -d'=');
+
+                        # Handle options with values.
+                        case "$key" in
+                            "log-level")        log_level_str=$value ;;
+                            "log-file")         log_file=$value ;;
+                            "transport")        transport=$value ;;
+                            "direct-io-mode")   direct_io_mode=$value ;;
+                            "volume-name")      volume_name=$value ;;
+                            "volume-id")        volume_id=$value ;;
+                            "volfile-check")    volfile_check=$value ;;
+                            "server-port")      server_port=$value ;;
+                            "attribute-timeout")
+                                attribute_timeout=$value ;;
+                            "entry-timeout")    entry_timeout=$value ;;
+                            "negative-timeout")    negative_timeout=$value ;;
+                            "gid-timeout")   gid_timeout=$value ;;
+                            "background-qlen") bg_qlen=$value ;;
+                            "backup-volfile-servers") backup_volfile_servers=$value ;;
+                            "congestion-threshold")  cong_threshold=$value ;;
+                            "xlator-option")  xlator_option=$xlator_option" "$pair ;;
+                            "fuse-mountopts")  fuse_mountopts=$value ;;
+                            "use-readdirp") use_readdirp=$value ;;
+                            *)
+                                # Passthru
+                                [ -z "$fuse_mountopts" ] || fuse_mountopts="$fuse_mountopts,"
+                                fuse_mountopts="$fuse_mountopts$pair"
+                                ;;
+                        esac
+                esac
+            done
+            in_opt="no"
+        elif [ "$opt" = "-o" ]; then
+            in_opt="yes"
+        else
+            case $pos_args in
+                0)      volfile_loc=$opt ;;
+                1)      mount_point=$opt ;;
+                *)      echo "extra arguments at end (ignored)" ;;
+            esac
+            pos_args=$((pos_args+1))
+        fi
+    done
+    if [ $in_opt = "yes" -o $pos_args -lt 2 ]; then
+        usage
+        exit 1
+    fi
 
     [ -r "$volfile_loc" ] || {
         server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
@@ -292,25 +423,9 @@ main ()
         [ -n "$test_str" ] && {
             volume_id="$test_str";
         }
-	volfile_loc="";
+        volfile_loc="";
     }
 
-    new_fs_options=$(echo "$options" | sed -e 's/[,]*log-file=[^,]*//' \
-        -e 's/[,]*log-level=[^,]*//' \
-        -e 's/[,]*volume-name=[^,]*//' \
-        -e 's/[,]*direct-io-mode=[^,]*//' \
-        -e 's/[,]*volfile-check=[^,]*//' \
-        -e 's/[,]*transport=[^,]*//' \
-        -e 's/[,]*backupvolfile-server=[^,]*//' \
-        -e 's/[,]*server-port=[^,]*//' \
-        -e 's/[,]*volume-id=[^,]*//' \
-        -e 's/[,]*fetch-attempts=[^,]*//' \
-        -e 's/[,]*log-server=[^,]*//' \
-        -e 's/[,]*ro[^,]*//' \
-        -e 's/[,]*acl[^,]*//' \
-        -e 's/[,]*worm[^,]*//' \
-        -e 's/[,]*log-server-port=[^,]*//');
-
     #
     [ -n "$helper" ] && {
         cmd_line=$(echo "$cmd_line --$helper");
@@ -318,31 +433,22 @@ main ()
         exit 0;
     }
 
-    mount_point=""
-    for arg in "$@"; do
-        [ -d "$arg" ] && {
-            mount_point=$arg
-        }
-    done
-
     # No need to do a ! -d test, it is taken care while initializing the
     # variable mount_point
-    [ -z "$mount_point" ] && {
+    [ -z "$mount_point" -o ! -d "$mount_point" ] && {
         echo "ERROR: Mount point does not exist."
         usage;
         exit 0;
     }
 
     # Simple check to avoid multiple identical mounts
-    if grep -q " $mount_point fuse" /etc/mtab; then
+    if grep -q "[[:space:]+]${mount_point}[[:space:]+]fuse" $mounttab; then
         echo -n "$0: according to mtab, GlusterFS is already mounted on "
         echo "$mount_point"
         exit 0;
     fi
 
-    check_recursive_mount "$@";
-
-    fs_options=$(echo "$fs_options,$new_fs_options");
+    check_recursive_mount "$mount_point";
 
     # Append fuse.glusterfs to PRUNEFS variable in updatedb.conf(5). updatedb(8)
     # should not index files under GlusterFS, indexing will slow down GlusteFS
@@ -359,4 +465,3 @@ main ()
 }
 
 _init "$@" && main "$@";
-
diff --git a/xlators/mount/fuse/utils/mount_glusterfs.in b/xlators/mount/fuse/utils/mount_glusterfs.in
index 3d4d28620..b12b4e04e 100755
--- a/xlators/mount/fuse/utils/mount_glusterfs.in
+++ b/xlators/mount/fuse/utils/mount_glusterfs.in
@@ -85,7 +85,7 @@ start_glusterfs ()
 --volfile-server-transport=$transport");
 	else
 	    cmd_line=$(echo "$cmd_line \
---volfile-server=$server_ip \
+--volfile-server=$server_ip");
 	fi
     else
 	cmd_line=$(echo "$cmd_line --volfile=$volfile_loc");
@@ -173,6 +173,7 @@ main ()
     
     [ -r "$volfile_loc" ] || {
         server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
+        volume_id=$(echo "$volfile_loc" | sed -n 's/[a-zA-Z0-9:.\-]*:\(.*\)/\1/p');
 	volfile_loc="";
     }
     # following line is product of love towards sed
diff --git a/xlators/nfs/server/src/Makefile.am b/xlators/nfs/server/src/Makefile.am
index 8fd286385..62fbf6535 100644
--- a/xlators/nfs/server/src/Makefile.am
+++ b/xlators/nfs/server/src/Makefile.am
@@ -1,17 +1,24 @@
 xlator_LTLIBRARIES = server.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/nfs
 nfsrpclibdir = $(top_srcdir)/rpc/rpc-lib/src
-server_la_LDFLAGS = -module -avoidversion
-server_la_SOURCES = nfs.c nfs-common.c nfs-fops.c nfs-inodes.c nfs-generics.c mount3.c nfs3-fh.c nfs3.c nfs3-helpers.c
+server_la_LDFLAGS = -module -avoid-version
+server_la_SOURCES = nfs.c nfs-common.c nfs-fops.c nfs-inodes.c \
+	nfs-generics.c mount3.c nfs3-fh.c nfs3.c nfs3-helpers.c nlm4.c \
+	nlmcbk_svc.c mount3udp_svc.c acl3.c
 server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = nfs.h nfs-common.h nfs-fops.h nfs-inodes.h nfs-generics.h mount3.h nfs3-fh.h nfs3.h nfs3-helpers.h nfs-mem-types.h
+noinst_HEADERS = nfs.h nfs-common.h nfs-fops.h nfs-inodes.h nfs-generics.h \
+	mount3.h nfs3-fh.h nfs3.h nfs3-helpers.h nfs-mem-types.h nlm4.h \
+	acl3.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)\
-	-I$(nfsrpclibdir) -L$(xlatordir)/ -I$(CONTRIBDIR)/rbtree\
-	-I$(top_srcdir)/rpc/xdr/src/ \
-	-I$(top_srcdir)/contrib/md5/
+	-I$(top_srcdir)/libglusterfs/src \
+	-I$(nfsrpclibdir) -I$(CONTRIBDIR)/rbtree \
+	-I$(top_srcdir)/rpc/xdr/src/
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+AM_LDFLAGS = -L$(xlatordir)
 
 CLEANFILES =
diff --git a/xlators/nfs/server/src/acl3.c b/xlators/nfs/server/src/acl3.c
new file mode 100644
index 000000000..08b099b4e
--- /dev/null
+++ b/xlators/nfs/server/src/acl3.c
@@ -0,0 +1,708 @@
+/*
+ * Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "rpcsvc.h"
+#include "dict.h"
+#include "xlator.h"
+#include "nfs.h"
+#include "mem-pool.h"
+#include "logging.h"
+#include "nfs-fops.h"
+#include "inode.h"
+#include "nfs3.h"
+#include "nfs-mem-types.h"
+#include "nfs3-helpers.h"
+#include "nfs3-fh.h"
+#include "nfs-generics.h"
+#include "acl3.h"
+
+
+typedef ssize_t (*acl3_serializer) (struct iovec outmsg, void *args);
+
+extern void nfs3_call_state_wipe (nfs3_call_state_t *cs);
+
+extern nfs3_call_state_t *
+nfs3_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v);
+
+extern int
+nfs3_fh_validate (struct nfs3_fh *fh);
+
+extern fattr3
+nfs3_stat_to_fattr3 (struct iatt *buf);
+
+#define acl3_validate_nfs3_state(request, state, status, label, retval) \
+        do      {                                                       \
+                state = rpcsvc_request_program_private (request);       \
+                if (!state) {                                           \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "NFSv3 state "    \
+                                "missing from RPC request");            \
+                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
+                        status = NFS3ERR_SERVERFAULT;                   \
+                        goto label;                                     \
+                }                                                       \
+        } while (0);                                                    \
+
+#define acl3_validate_gluster_fh(handle, status, errlabel)              \
+        do {                                                            \
+                if (!nfs3_fh_validate (handle)) {                       \
+                        status = NFS3ERR_SERVERFAULT;                   \
+                        goto errlabel;                                  \
+                }                                                       \
+        } while (0)                                                     \
+
+
+extern xlator_t *
+nfs3_fh_to_xlator (struct nfs3_state *nfs3, struct nfs3_fh *fh);
+
+#define acl3_map_fh_to_volume(nfs3state, handle, req, volume, status, label) \
+        do {                                                            \
+                char exportid[256], gfid[256];                          \
+                rpc_transport_t *trans = NULL;                          \
+                volume = nfs3_fh_to_xlator ((nfs3state), handle);      \
+                if (!volume) {                                          \
+                        uuid_unparse (handle->exportid, exportid);       \
+                        uuid_unparse (handle->gfid, gfid);               \
+                        trans = rpcsvc_request_transport (req);         \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "Failed to map "  \
+                                "FH to vol: client=%s, exportid=%s, gfid=%s",\
+                                trans->peerinfo.identifier, exportid,   \
+                                gfid);                                  \
+                        gf_log (GF_ACL, GF_LOG_ERROR,                   \
+                                "Stale nfs client %s must be trying to "\
+                                "connect to a deleted volume, please "  \
+                                "unmount it.", trans->peerinfo.identifier);\
+                        status = NFS3ERR_STALE;                         \
+                        goto label;                                     \
+                } else {                                                \
+                        gf_log (GF_ACL, GF_LOG_TRACE, "FH to Volume: %s"\
+                                ,volume->name);                         \
+                        rpcsvc_request_set_private (req, volume);       \
+                }                                                       \
+        } while (0);                                                    \
+
+#define acl3_volume_started_check(nfs3state, vlm, rtval, erlbl)         \
+        do {                                                            \
+              if ((!nfs_subvolume_started (nfs_state (nfs3state->nfsx), vlm))){\
+                      gf_log (GF_ACL, GF_LOG_ERROR, "Volume is disabled: %s",\
+                              vlm->name);                               \
+                      rtval = RPCSVC_ACTOR_IGNORE;                      \
+                      goto erlbl;                                       \
+              }                                                         \
+        } while (0)                                                     \
+
+#define acl3_check_fh_resolve_status(cst, nfstat, erlabl)               \
+        do {                                                            \
+                xlator_t *xlatorp = NULL;                               \
+                char buf[256], gfid[256];                               \
+                rpc_transport_t *trans = NULL;                          \
+                if ((cst)->resolve_ret < 0) {                           \
+                        trans = rpcsvc_request_transport (cst->req);    \
+                        xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
+                                                     &cst->resolvefh);  \
+                        uuid_unparse (cst->resolvefh.gfid, gfid);       \
+                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
+                                  trans->peerinfo.identifier,           \
+                                  xlatorp ? xlatorp->name : "ERR",      \
+                                  gfid);                                \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "Unable to resolve FH"\
+                                ": %s", buf);                           \
+                        nfstat = nfs3_errno_to_nfsstat3 (cst->resolve_errno);\
+                        goto erlabl;                                    \
+                }                                                       \
+        } while (0)                                                     \
+
+#define acl3_handle_call_state_init(nfs3state, calls, rq, v, opstat, errlabel)\
+        do {                                                            \
+                calls = nfs3_call_state_init ((nfs3state), (rq), v); \
+                if (!calls) {                                           \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "Failed to "      \
+                                "init call state");                     \
+                        opstat = NFS3ERR_SERVERFAULT;                   \
+                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
+                        goto errlabel;                                  \
+                }                                                       \
+        } while (0)                                                     \
+
+
+int
+acl3svc_submit_reply (rpcsvc_request_t *req, void *arg, acl3_serializer sfunc)
+{
+        struct iovec            outmsg  = {0, };
+        struct iobuf            *iob    = NULL;
+        struct nfs3_state       *nfs3   = NULL;
+        int                     ret     = -1;
+        ssize_t                 msglen  = 0;
+        struct iobref           *iobref = NULL;
+
+        if (!req)
+                return -1;
+
+        nfs3 = (struct nfs3_state *)rpcsvc_request_program_private (req);
+        if (!nfs3) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "mount state not found");
+                goto ret;
+        }
+
+        /* First, get the io buffer into which the reply in arg will
+         * be serialized.
+         */
+        iob = iobuf_get (nfs3->iobpool);
+        if (!iob) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to get iobuf");
+                goto ret;
+        }
+
+        iobuf_to_iovec (iob, &outmsg);
+        /* Use the given serializer to translate the give C structure in arg
+         * to XDR format which will be written into the buffer in outmsg.
+         */
+        msglen = sfunc (outmsg, arg);
+        if (msglen < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to encode message");
+                goto ret;
+        }
+        outmsg.iov_len = msglen;
+
+        iobref = iobref_new ();
+        if (iobref == NULL) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to get iobref");
+                goto ret;
+        }
+
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
+
+        /* Then, submit the message for transmission. */
+        ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Reply submission failed");
+                goto ret;
+        }
+
+        ret = 0;
+ret:
+        if (iob)
+                iobuf_unref (iob);
+        if (iobref)
+                iobref_unref (iobref);
+
+        return ret;
+}
+
+
+int
+acl3svc_null (rpcsvc_request_t *req)
+{
+        struct iovec    dummyvec = {0, };
+
+        if (!req) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Got NULL request!");
+                return 0;
+        }
+        rpcsvc_submit_generic (req, &dummyvec, 1,  NULL, 0, NULL);
+        return 0;
+}
+
+int
+acl3_getacl_reply (nfs3_call_state_t *cs, getaclreply *reply)
+{
+        acl3svc_submit_reply (cs->req, (void *)reply,
+                              (acl3_serializer)xdr_serialize_getaclreply);
+        return 0;
+}
+
+
+int
+acl3_getacl_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict,
+                 dict_t *xdata)
+{
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs3_call_state_t              *cs = NULL;
+        data_t                         *data = NULL;
+        int                            *p = NULL;
+        int                             i = 0;
+        getaclreply                     *getaclreply = NULL;
+
+        if (!frame->local) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Invalid argument,"
+                       " frame->local NULL");
+                return  EINVAL;
+        }
+        cs = frame->local;
+        getaclreply = &cs->args.getaclreply;
+        if (op_ret == -1) {
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
+                goto err;
+        }
+
+        getaclreply->aclentry.aclentry_val = cs->aclentry;
+        getaclreply->daclentry.daclentry_val = cs->daclentry;
+
+        /* FIXME: use posix_acl_from_xattr() */
+        data = dict_get (dict, POSIX_ACL_ACCESS_XATTR);
+        if (data && (p = data_to_bin (data))) {
+                /* POSIX_ACL_VERSION */
+                p++;
+                while ((char *)p < (data->data + data->len)) {
+                        getaclreply->aclentry.aclentry_val[i].type = *(*(short **)&p)++;
+                        getaclreply->aclentry.aclentry_val[i].perm = *(*(short **)&p)++;
+                        getaclreply->aclentry.aclentry_val[i].uid = *(*(int **)&p)++;
+                        i++;
+                }
+                getaclreply->aclcount = getaclreply->aclentry.aclentry_len = i;
+        }
+        i = 0;
+
+        data = dict_get (dict, POSIX_ACL_DEFAULT_XATTR);
+        if (data && (p = data_to_bin (data))) {
+                /* POSIX_ACL_VERSION */
+                p++;
+                while ((char *)p < (data->data + data->len)) {
+                        getaclreply->daclentry.daclentry_val[i].type = *(*(short **)&p)++;
+                        getaclreply->daclentry.daclentry_val[i].perm = *(*(short **)&p)++;
+                        getaclreply->daclentry.daclentry_val[i].uid = *(*(int **)&p)++;
+                        i++;
+                }
+                getaclreply->daclcount = getaclreply->daclentry.daclentry_len = i;
+        }
+
+        acl3_getacl_reply (cs, getaclreply);
+        nfs3_call_state_wipe (cs);
+        return 0;
+
+err:
+        if (getaclreply)
+                getaclreply->status = stat;
+        acl3_getacl_reply (cs, getaclreply);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+int
+acl3_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
+{
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs3_call_state_t              *cs = NULL;
+        getaclreply                     *getaclreply = NULL;
+        int                             ret = -1;
+        nfs_user_t                      nfu = {0, };
+
+        if (!frame->local) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Invalid argument,"
+                       " frame->local NULL");
+                return EINVAL;
+        }
+
+        cs = frame->local;
+        getaclreply = &cs->args.getaclreply;
+
+        if (op_ret == -1) {
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
+                goto err;
+        }
+
+        getaclreply->attr_follows = 1;
+        getaclreply->attr = nfs3_stat_to_fattr3 (buf);
+        getaclreply->mask = 0xf;
+        nfs_request_user_init (&nfu, cs->req);
+        ret = nfs_getxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, NULL, NULL,
+                            acl3_getacl_cbk, cs);
+        if (ret == -1) {
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
+                goto err;
+        }
+        return 0;
+err:
+        getaclreply->status = stat;
+        acl3_getacl_reply (cs, getaclreply);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+
+int
+acl3_getacl_resume (void *carg)
+{
+        int                             ret = -1;
+        nfs3_call_state_t               *cs = NULL;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs_user_t                      nfu = {0, };
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        acl3_check_fh_resolve_status (cs, stat, acl3err);
+        nfs_request_user_init (&nfu, cs->req);
+
+        ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                        acl3_stat_cbk, cs);
+        stat = -ret;
+acl3err:
+        if (ret < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to open_and_resume");
+                cs->args.getaclreply.status = nfs3_errno_to_nfsstat3 (stat);
+                acl3_getacl_reply (cs, &cs->args.getaclreply);
+                nfs3_call_state_wipe (cs);
+        }
+
+        return ret;
+}
+
+
+int
+acl3svc_getacl (rpcsvc_request_t *req)
+{
+        xlator_t                        *vol = NULL;
+        struct nfs_state               *nfs = NULL;
+        nfs3_state_t                   *nfs3 = NULL;
+        nfs3_call_state_t               *cs = NULL;
+        int                             ret = RPCSVC_ACTOR_ERROR;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        struct nfs3_fh                  fh, *fhp = NULL;
+        getaclargs                      getaclargs;
+
+        if (!req)
+                return ret;
+
+        acl3_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        memset (&getaclargs, 0, sizeof (getaclargs));
+        getaclargs.fh.n_bytes = (char *)&fh;
+        if (xdr_to_getaclargs(req->msg[0], &getaclargs) <= 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+        fhp = &fh;
+        acl3_validate_gluster_fh (&fh, stat, acl3err);
+        acl3_map_fh_to_volume (nfs->nfs3state, fhp, req,
+                               vol, stat, acl3err);
+        acl3_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     vol, stat, rpcerr);
+
+        cs->vol = vol;
+        acl3_volume_started_check (nfs3, vol, ret, acl3err);
+
+        ret = nfs3_fh_resolve_and_resume (cs, fhp,
+                                          NULL, acl3_getacl_resume);
+
+acl3err:
+        if (ret < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to resolve and resume");
+                if (cs) {
+                        cs->args.getaclreply.status = stat;
+                        acl3_getacl_reply (cs, &cs->args.getaclreply);
+                        nfs3_call_state_wipe (cs);
+                }
+                return 0;
+        }
+
+rpcerr:
+        return ret;
+}
+
+int
+acl3_setacl_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno,
+                 dict_t *xdata)
+{
+        nfs3_call_state_t               *cs = NULL;
+        cs = frame->local;
+        if (op_ret < 0) {
+                nfsstat3 status = nfs3_cbk_errno_status (op_ret, op_errno);
+                cs->args.setaclreply.status = status;
+        }
+
+        acl3svc_submit_reply (cs->req, (void *)&cs->args.setaclreply,
+                              (acl3_serializer)xdr_serialize_setaclreply);
+        return 0;
+}
+
+int
+acl3_setacl_resume (void *carg)
+{
+        int                             ret = -1;
+        nfs3_call_state_t               *cs = NULL;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs_user_t                      nfu = {0, };
+        dict_t                          *xattr = NULL;
+
+        if (!carg)
+                return ret;
+        cs = (nfs3_call_state_t *)carg;
+        acl3_check_fh_resolve_status (cs, stat, acl3err);
+        nfs_request_user_init (&nfu, cs->req);
+        xattr = dict_new();
+        if (cs->aclcount)
+        ret = dict_set_static_bin (xattr, POSIX_ACL_ACCESS_XATTR, cs->aclxattr,
+                                   cs->aclcount * 8 + 4);
+        if (cs->daclcount)
+        ret = dict_set_static_bin (xattr, POSIX_ACL_DEFAULT_XATTR,
+                                   cs->daclxattr, cs->daclcount * 8 + 4);
+
+        ret = nfs_setxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, xattr,
+                            0, NULL, acl3_setacl_cbk, cs);
+        dict_unref (xattr);
+
+acl3err:
+        if (ret < 0) {
+                stat = -ret;
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to open_and_resume");
+                cs->args.setaclreply.status = nfs3_errno_to_nfsstat3 (stat);
+                acl3svc_submit_reply (cs->req, (void *)&cs->args.setaclreply,
+                                      (acl3_serializer)xdr_serialize_setaclreply);
+                nfs3_call_state_wipe (cs);
+        }
+
+        return ret;
+}
+
+
+int
+acl3svc_setacl (rpcsvc_request_t *req)
+{
+        xlator_t                        *vol = NULL;
+        struct nfs_state               *nfs = NULL;
+        nfs3_state_t                   *nfs3 = NULL;
+        nfs3_call_state_t               *cs = NULL;
+        int                             ret = RPCSVC_ACTOR_ERROR;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        struct nfs3_fh                  fh;
+        struct nfs3_fh                 *fhp = NULL;
+        setaclargs                      setaclargs;
+        aclentry                        *aclentry = NULL;
+        struct aclentry                 *daclentry = NULL;
+        int                             i = 0;
+        struct posix_acl_xattr_header   *bufheader = NULL;
+        struct posix_acl_xattr_entry    *bufentry  = NULL;
+
+        if (!req)
+                return ret;
+        aclentry =  GF_CALLOC (NFS_ACL_MAX_ENTRIES, sizeof(*aclentry),
+                               gf_nfs_mt_arr);
+        if (!aclentry) {
+                goto rpcerr;
+        }
+        daclentry = GF_CALLOC (NFS_ACL_MAX_ENTRIES, sizeof(*daclentry),
+                               gf_nfs_mt_arr);
+        if (!daclentry) {
+                goto rpcerr;
+        }
+
+        acl3_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        memset (&setaclargs, 0, sizeof (setaclargs));
+        memset (&fh, 0, sizeof (fh));
+        setaclargs.fh.n_bytes = (char *)&fh;
+        setaclargs.aclentry.aclentry_val = aclentry;
+        setaclargs.daclentry.daclentry_val = daclentry;
+        if (xdr_to_setaclargs(req->msg[0], &setaclargs) <= 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+        fhp = &fh;
+        acl3_validate_gluster_fh (fhp, stat, acl3err);
+        acl3_map_fh_to_volume (nfs->nfs3state, fhp, req,
+                               vol, stat, acl3err);
+        acl3_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     vol, stat, rpcerr);
+
+        cs->vol = vol;
+        acl3_volume_started_check (nfs3, vol, ret, rpcerr);
+
+        cs->aclcount = setaclargs.aclcount;
+        cs->daclcount = setaclargs.daclcount;
+
+        if ((cs->aclcount > NFS_ACL_MAX_ENTRIES) ||
+                        (cs->daclcount > NFS_ACL_MAX_ENTRIES))
+                goto acl3err;
+        /* FIXME: use posix_acl_to_xattr() */
+        /* Populate xattr buffer for user ACL */
+        bufheader = (struct posix_acl_xattr_header *)(cs->aclxattr);
+        bufheader->version = htole32(POSIX_ACL_VERSION);
+        bufentry  = bufheader->entries;
+        for (i = 0; i < cs->aclcount; i++) {
+                int uaceuid;
+                const struct aclentry *uace = &aclentry[i];
+                switch (uace->type) {
+                    case POSIX_ACL_USER:
+                    case POSIX_ACL_GROUP:
+                        uaceuid = uace->uid;
+                        break;
+                    default:
+                        uaceuid = POSIX_ACL_UNDEFINED_ID;
+                        break;
+                }
+                bufentry->tag  = htole16(uace->type);
+                bufentry->perm = htole16(uace->perm);
+                bufentry->id   = htole32(uaceuid);
+
+                bufentry++;
+        }
+
+        /* Populate xattr buffer for Default ACL */
+        bufheader = (struct posix_acl_xattr_header *)(cs->aclxattr);
+        bufheader->version = htole32(POSIX_ACL_VERSION);
+        bufentry  = bufheader->entries;
+        for (i = 0; i < cs->daclcount; i++) {
+                int daceuid;
+                int dacetype;
+                const struct aclentry *dace = &daclentry[i];
+                /*
+                 * For "default ACL", NFSv3 handles the 'type' differently
+                 * i.e. by logical OR'ing 'type' with NFS_ACL_DEFAULT.
+                 * Which the backend File system does not understand and
+                 * that needs to be masked OFF.
+                 */
+                dacetype = (dace->type & ~(NFS_ACL_DEFAULT));
+                switch (dacetype) {
+                    case POSIX_ACL_USER:
+                    case POSIX_ACL_GROUP:
+                        daceuid = dace->uid;
+                        break;
+                    default:
+                        daceuid = POSIX_ACL_UNDEFINED_ID;
+                        break;
+                }
+                bufentry->tag  = htole16(dacetype);
+                bufentry->perm = htole16(dace->perm);
+                bufentry->id   = htole32(daceuid);
+
+                bufentry++;
+        }
+
+
+        ret = nfs3_fh_resolve_and_resume (cs, fhp,
+                                          NULL, acl3_setacl_resume);
+
+acl3err:
+        if (ret < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to resolve and resume");
+                cs->args.setaclreply.status = stat;
+                acl3svc_submit_reply (cs->req, (void *)&cs->args.setaclreply,
+                                      (acl3_serializer)xdr_serialize_setaclreply);
+                nfs3_call_state_wipe (cs);
+                GF_FREE(aclentry);
+                GF_FREE(daclentry);
+                return 0;
+        }
+
+rpcerr:
+        if (ret < 0)
+                nfs3_call_state_wipe (cs);
+        if (aclentry)
+                GF_FREE (aclentry);
+        if (daclentry)
+                GF_FREE (daclentry);
+        return ret;
+}
+
+
+
+rpcsvc_actor_t  acl3svc_actors[ACL3_PROC_COUNT] = {
+        {"NULL",       ACL3_NULL,      acl3svc_null,   NULL,   0},
+        {"GETACL",     ACL3_GETACL,    acl3svc_getacl, NULL,   0},
+        {"SETACL",     ACL3_SETACL,    acl3svc_setacl, NULL,   0},
+};
+
+rpcsvc_program_t        acl3prog = {
+        .progname       = "ACL3",
+        .prognum        = ACL_PROGRAM,
+        .progver        = ACLV3_VERSION,
+        .progport       = GF_NFS3_PORT,
+        .actors         = acl3svc_actors,
+        .numactors      = ACL3_PROC_COUNT,
+        .min_auth       = AUTH_NULL,
+};
+
+rpcsvc_program_t *
+acl3svc_init(xlator_t *nfsx)
+{
+        struct nfs3_state *ns = NULL;
+        struct nfs_state *nfs = NULL;
+        dict_t *options = NULL;
+        int ret = -1;
+        char *portstr = NULL;
+        static gf_boolean_t acl3_inited = _gf_false;
+
+        /* Already inited */
+        if (acl3_inited)
+                return &acl3prog;
+
+        nfs = (struct nfs_state*)nfsx->private;
+
+        ns = nfs->nfs3state;
+        if (!ns) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "ACL3 init failed");
+                goto err;
+        }
+        acl3prog.private = ns;
+
+        options = dict_new ();
+
+        ret = gf_asprintf (&portstr, "%d", GF_ACL3_PORT);
+        if (ret == -1)
+                goto err;
+
+        ret = dict_set_dynstr (options, "transport.socket.listen-port",
+                               portstr);
+        if (ret == -1)
+                goto err;
+        ret = dict_set_str (options, "transport-type", "socket");
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                goto err;
+        }
+
+        if (nfs->allow_insecure) {
+                ret = dict_set_str (options, "rpc-auth-allow-insecure", "on");
+                if (ret == -1) {
+                        gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                        goto err;
+                }
+                ret = dict_set_str (options, "rpc-auth.ports.insecure", "on");
+                if (ret == -1) {
+                        gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                        goto err;
+                }
+        }
+
+        ret = dict_set_str (options, "transport.address-family", "inet");
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                goto err;
+        }
+
+        ret = rpcsvc_create_listeners (nfs->rpcsvc, options, "ACL");
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Unable to create listeners");
+                dict_unref (options);
+                goto err;
+        }
+
+        acl3_inited = _gf_true;
+        return &acl3prog;
+err:
+        return NULL;
+}
diff --git a/xlators/nfs/server/src/acl3.h b/xlators/nfs/server/src/acl3.h
new file mode 100644
index 000000000..e0e61281a
--- /dev/null
+++ b/xlators/nfs/server/src/acl3.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+#ifndef _ACL3_H
+#define _ACL3_H
+
+#include "glusterfs-acl.h"
+
+#define GF_ACL3_PORT            38469
+#define GF_ACL                  GF_NFS"-ACL"
+
+/*
+ * NFSv3, identifies the default ACL by NFS_ACL_DEFAULT. Gluster
+ * NFS needs to mask it OFF before sending it upto POSIX layer
+ * or File system layer.
+ */
+#define NFS_ACL_DEFAULT             0x1000
+
+#define NFS_ACL_MAX_ENTRIES         1024
+
+rpcsvc_program_t *
+acl3svc_init(xlator_t *nfsx);
+
+#endif
diff --git a/xlators/nfs/server/src/mount3.c b/xlators/nfs/server/src/mount3.c
index 0f14c8d5a..b0824bf10 100644
--- a/xlators/nfs/server/src/mount3.c
+++ b/xlators/nfs/server/src/mount3.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -39,23 +30,71 @@
 #include "nfs-mem-types.h"
 #include "nfs.h"
 #include "common-utils.h"
-
+#include "store.h"
 
 #include <errno.h>
 #include <sys/socket.h>
 #include <sys/uio.h>
 
+
+#define IPv4_ADDR_SIZE  32
+
+/* Macro to typecast the parameter to struct sockaddr_in
+ */
+#define SA(addr) ((struct sockaddr_in*)(addr))
+
+/* Macro will mask the ip address with netmask.
+ */
+#define MASKED_IP(ipv4addr, netmask)                    \
+                (ntohl(SA(ipv4addr)->sin_addr.s_addr) & (netmask))
+
+/* Macro will compare two IP address after applying the mask
+ */
+#define COMPARE_IPv4_ADDRS(ip1, ip2, netmask)           \
+                ((MASKED_IP(ip1, netmask)) == (MASKED_IP(ip2, netmask)))
+
+/* This macro will assist in freeing up entire link list
+ * of host_auth_spec structure.
+ */
+#define FREE_HOSTSPEC(exp) do {                                 \
+                struct host_auth_spec *host= exp->hostspec;     \
+                while (NULL != host){                           \
+                        struct host_auth_spec* temp = host;     \
+                        host = host->next;                      \
+                        if (NULL != temp->host_addr) {          \
+                                GF_FREE (temp->host_addr);      \
+                        }                                       \
+                        GF_FREE (temp);                         \
+                }                                               \
+                exp->hostspec = NULL;                           \
+        } while (0)
+
 typedef ssize_t (*mnt3_serializer) (struct iovec outmsg, void *args);
 
+extern void *
+mount3udp_thread (void *argv);
+
+static inline void
+mnt3_export_free (struct mnt3_export *exp)
+{
+        if (!exp)
+                return;
+
+        if (exp->exptype == MNT3_EXPTYPE_DIR)
+                FREE_HOSTSPEC (exp);
+        GF_FREE (exp->expname);
+        GF_FREE (exp);
+}
 
 /* Generic reply function for MOUNTv3 specific replies. */
 int
 mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
 {
-        struct iovec            outmsg = {0, };
-        struct iobuf            *iob = NULL;
-        struct mount3_state     *ms = NULL;
-        int                     ret = -1;
+        struct iovec            outmsg  = {0, };
+        struct iobuf            *iob    = NULL;
+        struct mount3_state     *ms     = NULL;
+        int                     ret     = -1;
+        ssize_t                 msglen  = 0;
         struct iobref           *iobref = NULL;
 
         if (!req)
@@ -70,6 +109,7 @@ mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
         /* First, get the io buffer into which the reply in arg will
          * be serialized.
          */
+        /* TODO: use 'xdrproc_t' instead of 'sfunc' to get the xdr-size */
         iob = iobuf_get (ms->iobpool);
         if (!iob) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get iobuf");
@@ -80,7 +120,12 @@ mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
         /* Use the given serializer to translate the give C structure in arg
          * to XDR format which will be written into the buffer in outmsg.
          */
-        outmsg.iov_len = sfunc (outmsg, arg);
+        msglen = sfunc (outmsg, arg);
+        if (msglen < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to encode message");
+                goto ret;
+        }
+        outmsg.iov_len = msglen;
 
         iobref = iobref_new ();
         if (iobref == NULL) {
@@ -88,12 +133,14 @@ mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
                 goto ret;
         }
 
-        iobref_add (iobref, iob);
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
 
         /* Then, submit the message for transmission. */
         ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
-        iobuf_unref (iob);
-        iobref_unref (iobref);
         if (ret == -1) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Reply submission failed");
                 goto ret;
@@ -101,6 +148,11 @@ mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
 
         ret = 0;
 ret:
+        if (NULL != iob)
+                iobuf_unref (iob);
+        if (NULL != iobref)
+                iobref_unref (iobref);
+
         return ret;
 }
 
@@ -184,13 +236,278 @@ mnt3svc_set_mountres3 (mountstat3 stat, struct nfs3_fh *fh, int *authflavor,
         return res;
 }
 
+/* Read the rmtab from the store_handle and append (or not) the entries to the
+ * mountlist.
+ *
+ * Requires the store_handle to be locked.
+ */
+static int
+__mount_read_rmtab (gf_store_handle_t *sh, struct list_head *mountlist,
+                    gf_boolean_t append)
+{
+        int                     ret = 0;
+        unsigned int            idx = 0;
+        struct mountentry       *me = NULL, *tmp = NULL;
+                                /* me->hostname is a char[MNTPATHLEN] */
+        char                    key[MNTPATHLEN + 11];
+
+        GF_ASSERT (sh && mountlist);
+
+        if (!gf_store_locked_local (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Not reading unlocked %s",
+                        sh->path);
+                return -1;
+        }
+
+        if (!append) {
+                list_for_each_entry_safe (me, tmp, mountlist, mlist) {
+                        list_del (&me->mlist);
+                        GF_FREE (me);
+                }
+                me = NULL;
+        }
+
+        for (;;) {
+                char *value = NULL;
+
+                if (me && append) {
+                        /* do not add duplicates */
+                        list_for_each_entry (tmp, mountlist, mlist) {
+                                if (!strcmp(tmp->hostname, me->hostname) &&
+                                    !strcmp(tmp->exname, me->exname)) {
+                                        GF_FREE (me);
+                                        goto dont_add;
+                                }
+                        }
+                        list_add_tail (&me->mlist, mountlist);
+                } else if (me) {
+                        list_add_tail (&me->mlist, mountlist);
+                }
+
+dont_add:
+                me = GF_CALLOC (1, sizeof (*me), gf_nfs_mt_mountentry);
+                if (!me) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory");
+                        ret = -1;
+                        goto out;
+                }
+
+                INIT_LIST_HEAD (&me->mlist);
+
+                snprintf (key, 9 + MNTPATHLEN, "hostname-%d", idx);
+                ret = gf_store_retrieve_value (sh, key, &value);
+                if (ret)
+                        break;
+                strncpy (me->hostname, value, MNTPATHLEN);
+                GF_FREE (value);
+
+                snprintf (key, 11 + MNTPATHLEN, "mountpoint-%d", idx);
+                ret = gf_store_retrieve_value (sh, key, &value);
+                if (ret)
+                        break;
+                strncpy (me->exname, value, MNTPATHLEN);
+                GF_FREE (value);
+
+                idx++;
+                gf_log (GF_MNT, GF_LOG_TRACE, "Read entries %s:%s", me->hostname, me->exname);
+        }
+        gf_log (GF_MNT, GF_LOG_DEBUG, "Read %d entries from '%s'", idx, sh->path);
+        GF_FREE (me);
+out:
+        return ret;
+}
+
+/* Overwrite the contents of the rwtab with te in-memory client list.
+ * Fail gracefully if the stora_handle is not locked.
+ */
+static void
+__mount_rewrite_rmtab(struct mount3_state *ms, gf_store_handle_t *sh)
+{
+        struct mountentry       *me = NULL;
+        char                    key[16];
+        int                     fd, ret;
+        unsigned int            idx = 0;
+
+        if (!gf_store_locked_local (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Not modifying unlocked %s",
+                        sh->path);
+                return;
+        }
+
+        fd = gf_store_mkstemp (sh);
+        if (fd == -1) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to open %s", sh->path);
+                return;
+        }
+
+        list_for_each_entry (me, &ms->mountlist, mlist) {
+                snprintf (key, 16, "hostname-%d", idx);
+                ret = gf_store_save_value (fd, key, me->hostname);
+                if (ret)
+                        goto fail;
+
+                snprintf (key, 16, "mountpoint-%d", idx);
+                ret = gf_store_save_value (fd, key, me->exname);
+                if (ret)
+                        goto fail;
+
+                idx++;
+        }
+
+        gf_log (GF_MNT, GF_LOG_DEBUG, "Updated rmtab with %d entries", idx);
+
+        close (fd);
+        if (gf_store_rename_tmppath (sh))
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to overwrite rwtab %s",
+                        sh->path);
+
+        return;
+
+fail:
+        gf_log (GF_MNT, GF_LOG_ERROR, "Failed to update %s", sh->path);
+        close (fd);
+        gf_store_unlink_tmppath (sh);
+}
+
+/* Read the rmtab into a clean ms->mountlist.
+ */
+static void
+mount_read_rmtab (struct mount3_state *ms)
+{
+        gf_store_handle_t       *sh = NULL;
+        struct nfs_state        *nfs = NULL;
+        int                     ret;
+
+        nfs = (struct nfs_state *)ms->nfsx->private;
 
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                return;
+        }
+
+        if (gf_store_lock (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to lock '%s'",
+                        nfs->rmtab);
+                goto out;
+        }
+
+        __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
+        gf_store_unlock (sh);
+
+out:
+        gf_store_handle_destroy (sh);
+}
+
+/* Write the ms->mountlist to the rmtab.
+ *
+ * The rmtab could be empty, or it can exists and have been updated by a
+ * different storage server without our knowing.
+ *
+ * 1. takes the store_handle lock on the current rmtab
+ *    - blocks if an other storage server rewrites the rmtab at the same time
+ * 2. [if new_rmtab] takes the store_handle lock on the new rmtab
+ * 3. reads/merges the entries from the current rmtab
+ * 4. [if new_rmtab] reads/merges the entries from the new rmtab
+ * 5. [if new_rmtab] writes the new rmtab
+ * 6. [if not new_rmtab] writes the current rmtab
+ * 7  [if new_rmtab] replaces nfs->rmtab to point to the new location
+ * 8. [if new_rmtab] releases the store_handle lock of the new rmtab
+ * 9. releases the store_handle lock of the old rmtab
+ */
+void
+mount_rewrite_rmtab (struct mount3_state *ms, char *new_rmtab)
+{
+        gf_store_handle_t       *sh = NULL, *nsh = NULL;
+        struct nfs_state        *nfs = NULL;
+        int                     ret;
+        char                    *rmtab = NULL;
+
+        nfs = (struct nfs_state *)ms->nfsx->private;
+
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                return;
+        }
+
+        if (gf_store_lock (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Not rewriting '%s'",
+                        nfs->rmtab);
+                goto free_sh;
+        }
+
+        if (new_rmtab) {
+                ret = gf_store_handle_new (new_rmtab, &nsh);
+                if (ret) {
+                        gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                                new_rmtab);
+                        goto unlock_sh;
+                }
+
+                if (gf_store_lock (nsh)) {
+                        gf_log (GF_MNT, GF_LOG_WARNING, "Not rewriting '%s'",
+                                new_rmtab);
+                        goto free_nsh;
+                }
+        }
+
+        /* always read the currently used rmtab */
+        __mount_read_rmtab (sh, &ms->mountlist, _gf_true);
+
+        if (new_rmtab) {
+                /* read the new rmtab and write changes to the new location */
+                __mount_read_rmtab (nsh, &ms->mountlist, _gf_true);
+                __mount_rewrite_rmtab (ms, nsh);
+
+                /* replace the nfs->rmtab reference to the new rmtab */
+                rmtab = gf_strdup(new_rmtab);
+                if (rmtab == NULL) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory, keeping "
+                                "%s as rmtab", nfs->rmtab);
+                } else {
+                        GF_FREE (nfs->rmtab);
+                        nfs->rmtab = new_rmtab;
+                }
+
+                gf_store_unlock (nsh);
+        } else {
+                /* rewrite the current (unchanged location) rmtab */
+                __mount_rewrite_rmtab (ms, sh);
+        }
+
+free_nsh:
+        if (new_rmtab)
+                gf_store_handle_destroy (nsh);
+unlock_sh:
+        gf_store_unlock (sh);
+free_sh:
+        gf_store_handle_destroy (sh);
+}
+
+/* Add a new NFS-client to the ms->mountlist and update the rmtab if we can.
+ *
+ * A NFS-client will only be removed from the ms->mountlist in case the
+ * NFS-client sends a unmount request. It is possible that a NFS-client
+ * crashed/rebooted had network loss or something else prevented the NFS-client
+ * to unmount cleanly. In this case, a duplicate entry would be added to the
+ * ms->mountlist, which is wrong and we should prevent.
+ *
+ * It is fully acceptible that the ms->mountlist is not 100% correct, this is a
+ * common issue for all(?) NFS-servers.
+ */
 int
 mnt3svc_update_mountlist (struct mount3_state *ms, rpcsvc_request_t *req,
                           char  *expname)
 {
         struct mountentry       *me = NULL;
+        struct mountentry       *cur = NULL;
         int                     ret = -1;
+        char                    *colon = NULL;
+        struct nfs_state        *nfs = NULL;
+        gf_store_handle_t       *sh = NULL;
 
         if ((!ms) || (!req) || (!expname))
                 return -1;
@@ -200,21 +517,62 @@ mnt3svc_update_mountlist (struct mount3_state *ms, rpcsvc_request_t *req,
         if (!me)
                 return -1;
 
-        strcpy (me->exname, expname);
+        nfs = (struct nfs_state *)ms->nfsx->private;
+
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                goto free_err;
+        }
+
+        strncpy (me->exname, expname, MNTPATHLEN);
+
         INIT_LIST_HEAD (&me->mlist);
         /* Must get the IP or hostname of the client so we
          * can map it into the mount entry.
          */
         ret = rpcsvc_transport_peername (req->trans, me->hostname, MNTPATHLEN);
         if (ret == -1)
-                goto free_err;
+                goto free_err2;
 
+        colon = strrchr (me->hostname, ':');
+        if (colon) {
+                *colon = '\0';
+        }
         LOCK (&ms->mountlock);
         {
+                /* in case locking fails, we just don't write the rmtab */
+                if (gf_store_lock (sh)) {
+                        gf_log (GF_MNT, GF_LOG_WARNING, "Failed to lock '%s'"
+                                ", changes will not be written", nfs->rmtab);
+                } else {
+                        __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
+                }
+
+                /* do not add duplicates */
+                list_for_each_entry (cur, &ms->mountlist, mlist) {
+                        if (!strcmp(cur->hostname, me->hostname) &&
+                            !strcmp(cur->exname, me->exname)) {
+                                GF_FREE (me);
+                                goto dont_add;
+                        }
+                }
                 list_add_tail (&me->mlist, &ms->mountlist);
+
+                /* only write the rmtab in case it was locked */
+                if (gf_store_locked_local (sh))
+                        __mount_rewrite_rmtab (ms, sh);
         }
+dont_add:
+        if (gf_store_locked_local (sh))
+                gf_store_unlock (sh);
+
         UNLOCK (&ms->mountlock);
 
+free_err2:
+        gf_store_handle_destroy (sh);
+
 free_err:
         if (ret == -1)
                 GF_FREE (me);
@@ -233,6 +591,7 @@ __mnt3_get_volume_id (struct mount3_state *ms, xlator_t *mntxl,
         if ((!ms) || (!mntxl))
                 return ret;
 
+        LOCK (&ms->mountlock);
         list_for_each_entry (exp, &ms->exportlist, explist) {
                 if (exp->vol == mntxl) {
                         uuid_copy (volumeid, exp->volumeid);
@@ -242,6 +601,7 @@ __mnt3_get_volume_id (struct mount3_state *ms, xlator_t *mntxl,
         }
 
 out:
+        UNLOCK (&ms->mountlock);
         return ret;
 }
 
@@ -262,7 +622,7 @@ mnt3svc_lookup_mount_cbk (call_frame_t *frame, void  *cookie,
         rpcsvc_t                *svc = NULL;
         xlator_t                *mntxl = NULL;
         uuid_t                  volumeid = {0, };
-        char                    fhstr[1024];
+        char                    fhstr[1024], *path = NULL;
 
         req = (rpcsvc_request_t *)frame->local;
 
@@ -284,7 +644,15 @@ mnt3svc_lookup_mount_cbk (call_frame_t *frame, void  *cookie,
         if (status != MNT3_OK)
                 goto xmit_res;
 
-        mnt3svc_update_mountlist (ms, req, mntxl->name);
+        path = GF_CALLOC (PATH_MAX, sizeof (char), gf_nfs_mt_char);
+        if (!path) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory");
+                goto xmit_res;
+        }
+
+        snprintf (path, PATH_MAX, "/%s", mntxl->name);
+        mnt3svc_update_mountlist (ms, req, path);
+        GF_FREE (path);
         if (gf_nfs_dvm_off (nfs_state (ms->nfsx))) {
                 fh = nfs3_fh_build_indexed_root_fh (ms->nfsx->children, mntxl);
                 goto xmit_res;
@@ -294,7 +662,7 @@ mnt3svc_lookup_mount_cbk (call_frame_t *frame, void  *cookie,
         fh = nfs3_fh_build_uuid_root_fh (volumeid);
 
 xmit_res:
-        nfs3_fh_to_str (&fh, fhstr);
+        nfs3_fh_to_str (&fh, fhstr, sizeof (fhstr));
         gf_log (GF_MNT, GF_LOG_DEBUG, "MNT reply: fh %s, status: %d", fhstr,
                 status);
         if (op_ret == 0) {
@@ -315,7 +683,7 @@ int
 mnt3_match_dirpath_export (char *expname, char *dirpath)
 {
         int     ret = 0;
-        int     dlen = 0;
+        size_t  dlen;
 
         if ((!expname) || (!dirpath))
                 return 0;
@@ -326,7 +694,7 @@ mnt3_match_dirpath_export (char *expname, char *dirpath)
          * compare.
          */
         dlen = strlen (dirpath);
-        if (dirpath [dlen - 1] == '/')
+        if (dlen && dirpath [dlen - 1] == '/')
                 dirpath [dlen - 1] = '\0';
 
         if (dirpath[0] != '/')
@@ -458,8 +826,8 @@ mnt3_resolve_state_wipe (mnt3_resolve_t *mres)
 /* Sets up the component argument to contain the next component in the path and
  * sets up path as an absolute path starting from the next component.
  */
-char *
-__setup_next_component (char *path, char *component)
+static char *
+setup_next_component (char *path, size_t plen, char *component, size_t clen)
 {
         char    *comp = NULL;
         char    *nextcomp = NULL;
@@ -467,7 +835,7 @@ __setup_next_component (char *path, char *component)
         if ((!path) || (!component))
                 return NULL;
 
-        strcpy (component, path);
+        strncpy (component, path, clen);
         comp = index (component, (int)'/');
         if (!comp)
                 goto err;
@@ -475,7 +843,7 @@ __setup_next_component (char *path, char *component)
         comp++;
         nextcomp = index (comp, (int)'/');
         if (nextcomp) {
-                strcpy (path, nextcomp);
+                strncpy (path, nextcomp, plen);
                 *nextcomp = '\0';
         } else
                 path[0] = '\0';
@@ -505,7 +873,9 @@ __mnt3_resolve_export_subdir_comp (mnt3_resolve_t *mres)
         if (!mres)
                 return ret;
 
-        nextcomp = __setup_next_component (mres->remainingdir, dupsubdir);
+        nextcomp = setup_next_component (mres->remainingdir,
+                                         sizeof (mres->remainingdir),
+                                         dupsubdir, sizeof (dupsubdir));
         if (!nextcomp)
                 goto err;
 
@@ -517,7 +887,7 @@ __mnt3_resolve_export_subdir_comp (mnt3_resolve_t *mres)
         if ((ret < 0) && (ret != -2)) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Failed to resolve and create "
                         "inode: parent gfid %s, entry %s",
-                        uuid_utoa (mres->resolveloc.inode->gfid), nextcomp);
+                        uuid_utoa (gfid), nextcomp);
                 ret = -EFAULT;
                 goto err;
         }
@@ -545,6 +915,7 @@ mnt3_resolve_subdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         rpcsvc_t                *svc = NULL;
         mountres3               res = {0, };
         xlator_t                *mntxl = NULL;
+        char                    *path = NULL;
 
         mres = frame->local;
         mntxl = (xlator_t *)cookie;
@@ -562,15 +933,23 @@ mnt3_resolve_subdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (strlen (mres->remainingdir) <= 0) {
                 op_ret = -1;
                 mntstat = MNT3_OK;
+                path = GF_CALLOC (PATH_MAX, sizeof (char), gf_nfs_mt_char);
+                if (!path) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation "
+                                "failed");
+                        goto err;
+                }
+                snprintf (path, PATH_MAX, "/%s%s", mres->exp->vol->name,
+                         mres->resolveloc.path);
                 mnt3svc_update_mountlist (mres->mstate, mres->req,
-                                          mres->exp->expname);
-                goto err;
+                                          path);
+                GF_FREE (path);
+        } else {
+                mres->parentfh = fh;
+                op_ret = __mnt3_resolve_export_subdir_comp (mres);
+                if (op_ret < 0)
+                        mntstat = mnt3svc_errno_to_mnterr (-op_ret);
         }
-
-        mres->parentfh = fh;
-        op_ret = __mnt3_resolve_export_subdir_comp (mres);
-        if (op_ret < 0)
-                mntstat = mnt3svc_errno_to_mnterr (-op_ret);
 err:
         if (op_ret == -1) {
                 gf_log (GF_MNT, GF_LOG_DEBUG, "Mount reply status: %d",
@@ -613,7 +992,9 @@ __mnt3_resolve_subdir (mnt3_resolve_t *mres)
         if (!mres)
                 return ret;
 
-        firstcomp = __setup_next_component (mres->remainingdir, dupsubdir);
+        firstcomp = setup_next_component (mres->remainingdir,
+                                          sizeof (mres->remainingdir),
+                                          dupsubdir, sizeof (dupsubdir));
         if (!firstcomp)
                 goto err;
 
@@ -636,6 +1017,132 @@ err:
 }
 
 
+/**
+ * This function will verify if the client is allowed to mount
+ * the directory or not. Client's IP address will be compared with
+ * allowed IP list or range present in mnt3_export structure.
+ *
+ * @param req - RPC request. This structure contains client's IP address.
+ * @param export - mnt3_export structure. Contains allowed IP list/range.
+ *
+ * @return 0 - on Success and -EACCES on failure.
+ */
+int
+mnt3_verify_auth (rpcsvc_request_t *req, struct mnt3_export *export)
+{
+        int                     retvalue = -EACCES;
+        int                     ret = 0;
+        int                     shiftbits = 0;
+        uint32_t                ipv4netmask = 0;
+        uint32_t                routingprefix = 0;
+        struct host_auth_spec   *host = NULL;
+        struct sockaddr_in      *client_addr = NULL;
+        struct sockaddr_in      *allowed_addr = NULL;
+        struct addrinfo         *allowed_addrinfo = NULL;
+
+        /* Sanity check */
+        if ((NULL == req) ||
+            (NULL == req->trans) ||
+            (NULL == export) ||
+            (NULL == export->hostspec)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Invalid argument");
+                return retvalue;
+        }
+
+        host = export->hostspec;
+
+
+        /* Client's IP address. */
+        client_addr = (struct sockaddr_in *)(&(req->trans->peerinfo.sockaddr));
+
+        /* Try to see if the client IP matches the allowed IP list.*/
+        while (NULL != host){
+                GF_ASSERT (host->host_addr);
+
+                if (NULL != allowed_addrinfo) {
+                        freeaddrinfo (allowed_addrinfo);
+                        allowed_addrinfo = NULL;
+                }
+
+                /* Get the addrinfo for the allowed host (host_addr). */
+                ret = getaddrinfo (host->host_addr,
+                                NULL,
+                                NULL,
+                                &allowed_addrinfo);
+                if (0 != ret){
+                        gf_log (GF_MNT, GF_LOG_ERROR, "getaddrinfo: %s\n",
+                                gai_strerror (ret));
+                        host = host->next;
+
+                        /* Failed to get IP addrinfo. Continue to check other
+                         * allowed IPs in the list.
+                         */
+                        continue;
+                }
+
+                allowed_addr = (struct sockaddr_in *)(allowed_addrinfo->ai_addr);
+
+                if (NULL == allowed_addr) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Invalid structure");
+                        break;
+                }
+
+                if (AF_INET == allowed_addr->sin_family){
+                        if (IPv4_ADDR_SIZE < host->routeprefix) {
+                                gf_log (GF_MNT, GF_LOG_ERROR, "invalid IP "
+                                        "configured for export-dir AUTH");
+                                host = host->next;
+                                continue;
+                        }
+
+                        /* -1 means no route prefix is provided. In this case
+                         * the IP should be an exact match. Which is same as
+                         * providing a route prefix of IPv4_ADDR_SIZE.
+                         */
+                        if (-1 == host->routeprefix) {
+                                routingprefix = IPv4_ADDR_SIZE;
+                        } else {
+                                routingprefix = host->routeprefix;
+                        }
+
+                        /* Create a mask from the routing prefix. User provided
+                         * CIDR address is split into IP address (host_addr) and
+                         * routing prefix (routeprefix). This CIDR address may
+                         * denote a single, distinct interface address or the
+                         * beginning address of an entire network.
+                         *
+                         * e.g. the IPv4 block 192.168.100.0/24 represents the
+                         * 256 IPv4 addresses from 192.168.100.0 to
+                         * 192.168.100.255.
+                         * Therefore to check if an IP matches 192.168.100.0/24
+                         * we should mask the IP with FFFFFF00 and compare it
+                         * with host address part of CIDR.
+                         */
+                        shiftbits = IPv4_ADDR_SIZE - routingprefix;
+                        ipv4netmask = 0xFFFFFFFFUL << shiftbits;
+
+                        /* Mask both the IPs and then check if they match
+                         * or not. */
+                        if (COMPARE_IPv4_ADDRS (allowed_addr,
+                                                client_addr,
+                                                ipv4netmask)){
+                                retvalue = 0;
+                                break;
+                        }
+                }
+
+                /* Client IP didn't match the allowed IP.
+                 * Check with the next allowed IP.*/
+               host = host->next;
+        }
+
+        if (NULL != allowed_addrinfo) {
+               freeaddrinfo (allowed_addrinfo);
+        }
+
+        return retvalue;
+}
+
 int
 mnt3_resolve_subdir (rpcsvc_request_t *req, struct mount3_state *ms,
                      struct mnt3_export *exp, char *subdir)
@@ -647,6 +1154,16 @@ mnt3_resolve_subdir (rpcsvc_request_t *req, struct mount3_state *ms,
         if ((!req) || (!ms) || (!exp) || (!subdir))
                 return ret;
 
+        /* Need to check AUTH */
+        if (NULL != exp->hostspec) {
+                ret = mnt3_verify_auth (req, exp);
+                if (0 != ret) {
+                        gf_log (GF_MNT,GF_LOG_ERROR,
+                                        "AUTH verification failed");
+                        return ret;
+                }
+        }
+
         mres = GF_CALLOC (1, sizeof (mnt3_resolve_t), gf_nfs_mt_mnt3_resolve);
         if (!mres) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
@@ -656,7 +1173,7 @@ mnt3_resolve_subdir (rpcsvc_request_t *req, struct mount3_state *ms,
         mres->exp = exp;
         mres->mstate = ms;
         mres->req = req;
-        strcpy (mres->remainingdir, subdir);
+        strncpy (mres->remainingdir, subdir, MNTPATHLEN);
         if (gf_nfs_dvm_off (nfs_state (ms->nfsx)))
                 pfh = nfs3_fh_build_indexed_root_fh (mres->mstate->nfsx->children, mres->exp->vol);
         else
@@ -731,6 +1248,7 @@ mnt3_mntpath_to_export (struct mount3_state *ms, char *dirpath)
         if ((!ms) || (!dirpath))
                 return NULL;
 
+        LOCK (&ms->mountlock);
         list_for_each_entry (exp, &ms->exportlist, explist) {
 
                 /* Search for the an exact match with the volume */
@@ -744,6 +1262,7 @@ mnt3_mntpath_to_export (struct mount3_state *ms, char *dirpath)
 
         gf_log (GF_MNT, GF_LOG_DEBUG, "Export not found");
 foundexp:
+        UNLOCK (&ms->mountlock);
         return found;
 }
 
@@ -753,24 +1272,36 @@ mnt3_check_client_net (struct mount3_state *ms, rpcsvc_request_t *req,
                        xlator_t *targetxl)
 {
 
-        rpcsvc_t        *svc = NULL;
-        int             ret = -1;
+        rpcsvc_t                *svc = NULL;
+        rpc_transport_t         *trans = NULL;
+        struct sockaddr_storage sastorage = {0,};
+        char                    peer[RPCSVC_PEER_STRLEN] = {0,};
+        int                     ret = -1;
 
         if ((!ms) || (!req) || (!targetxl))
                 return -1;
 
         svc = rpcsvc_request_service (req);
-        ret = rpcsvc_transport_peer_check (svc->options, targetxl->name,
-                                           rpcsvc_request_transport (req));
+
+        trans = rpcsvc_request_transport (req);
+        ret = rpcsvc_transport_peeraddr (trans, peer, RPCSVC_PEER_STRLEN,
+                                         &sastorage, sizeof (sastorage));
+        if (ret != 0) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to get peer addr: %s",
+                        gai_strerror (ret));
+        }
+
+        ret = rpcsvc_auth_check (svc, targetxl->name, trans);
         if (ret == RPCSVC_AUTH_REJECT) {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Peer not allowed");
+                gf_log (GF_MNT, GF_LOG_INFO, "Peer %s  not allowed", peer);
                 goto err;
         }
 
         ret = rpcsvc_transport_privport_check (svc, targetxl->name,
                                                rpcsvc_request_transport (req));
         if (ret == RPCSVC_AUTH_REJECT) {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Unprivileged port not allowed");
+                gf_log (GF_MNT, GF_LOG_INFO, "Peer %s rejected. Unprivileged "
+                        "port not allowed", peer);
                 goto err;
         }
 
@@ -955,6 +1486,9 @@ __build_mountlist (struct mount3_state *ms, int *count)
         if ((!ms) || (!count))
                 return NULL;
 
+        /* read rmtab, other peers might have updated it */
+        mount_read_rmtab(ms);
+
         *count = 0;
         gf_log (GF_MNT, GF_LOG_DEBUG, "Building mount list:");
         list_for_each_entry (me, &ms->mountlist, mlist) {
@@ -965,6 +1499,8 @@ __build_mountlist (struct mount3_state *ms, int *count)
                                 " failed");
                         goto free_list;
                 }
+                if (!first)
+                        first = mlist;
 
                 mlist->ml_directory = GF_CALLOC (namelen + 2, sizeof (char),
                                                  gf_nfs_mt_char);
@@ -974,8 +1510,7 @@ __build_mountlist (struct mount3_state *ms, int *count)
                         goto free_list;
                 }
 
-                strcpy (mlist->ml_directory, "/");
-                strcat (mlist->ml_directory, me->exname);
+                strcpy (mlist->ml_directory, me->exname);
 
                 namelen = strlen (me->hostname);
                 mlist->ml_hostname = GF_CALLOC (namelen + 2, sizeof (char),
@@ -996,9 +1531,6 @@ __build_mountlist (struct mount3_state *ms, int *count)
                 } else
                         prev = mlist;
 
-                if (!first)
-                        first = mlist;
-
                 (*count)++;
         }
 
@@ -1075,67 +1607,71 @@ rpcerr:
 
 
 int
-__mnt3svc_umount (struct mount3_state *ms, char *dirpath, char *hostname)
+mnt3svc_umount (struct mount3_state *ms, char *dirpath, char *hostname)
 {
         struct mountentry       *me = NULL;
-        char                    *exname = NULL;
         int                     ret = -1;
+        gf_store_handle_t       *sh = NULL;
+        struct nfs_state        *nfs = NULL;
 
         if ((!ms) || (!dirpath) || (!hostname))
                 return -1;
 
-        if (list_empty (&ms->mountlist))
-                return 0;
-
-        if (dirpath[0] == '/')
-                exname = dirpath+1;
-        else
-                exname = dirpath;
+        nfs = (struct nfs_state *)ms->nfsx->private;
 
-        list_for_each_entry (me, &ms->mountlist, mlist) {
-               if ((strcmp (me->exname, exname) == 0) &&
-                    (strcmp (me->hostname, hostname) == 0)) {
-                       ret = 0;
-                       break;
-               }
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                return 0;
         }
 
-        /* Need this check here because at the end of the search me might still
-         * be pointing to the last entry, which may not be the one we're
-         * looking for.
-         */
-        if (ret == -1)  {/* Not found in list. */
-                gf_log (GF_MNT, GF_LOG_DEBUG, "Export not found");
-                goto ret;
+        ret = gf_store_lock (sh);
+        if (ret) {
+                goto out_free;
         }
 
-        if (!me)
-                goto ret;
+        LOCK (&ms->mountlock);
+        {
+                __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
+                if (list_empty (&ms->mountlist)) {
+                        ret = 0;
+                        goto out_unlock;
+                }
 
-        gf_log (GF_MNT, GF_LOG_DEBUG, "Unmounting: dir %s, host: %s",
-                me->exname, me->hostname);
-        list_del (&me->mlist);
-        GF_FREE (me);
-        ret = 0;
-ret:
-        return ret;
-}
+                ret = -1;
+                list_for_each_entry (me, &ms->mountlist, mlist) {
+                       if ((strcmp (me->exname, dirpath) == 0) &&
+                            (strcmp (me->hostname, hostname) == 0)) {
+                               ret = 0;
+                               break;
+                       }
+                }
 
+                /* Need this check here because at the end of the search me
+                 * might still be pointing to the last entry, which may not be
+                 * the one we're looking for.
+                 */
+                if (ret == -1)  {/* Not found in list. */
+                        gf_log (GF_MNT, GF_LOG_TRACE, "Export not found");
+                        goto out_unlock;
+                }
 
+                if (!me)
+                        goto out_unlock;
 
-int
-mnt3svc_umount (struct mount3_state *ms, char *dirpath, char *hostname)
-{
-        int ret = -1;
-        if ((!ms) || (!dirpath) || (!hostname))
-                return -1;
+                gf_log (GF_MNT, GF_LOG_DEBUG, "Unmounting: dir %s, host: %s",
+                        me->exname, me->hostname);
 
-        LOCK (&ms->mountlock);
-        {
-               ret = __mnt3svc_umount (ms, dirpath, hostname);
+                list_del (&me->mlist);
+                GF_FREE (me);
+                __mount_rewrite_rmtab (ms, sh);
         }
+out_unlock:
         UNLOCK (&ms->mountlock);
-
+        gf_store_unlock (sh);
+out_free:
+        gf_store_handle_destroy (sh);
         return ret;
 }
 
@@ -1149,6 +1685,7 @@ mnt3svc_umnt (rpcsvc_request_t *req)
         int                     ret = -1;
         struct mount3_state     *ms = NULL;
         mountstat3              mstat = MNT3_OK;
+        char                    *colon = NULL;
 
         if (!req)
                 return -1;
@@ -1175,37 +1712,24 @@ mnt3svc_umnt (rpcsvc_request_t *req)
         if (ret != 0) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get remote name: %s",
                         gai_strerror (ret));
-                goto try_umount_with_addr;
-        }
-
-        gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s, hostname: %s", dirpath,
-                hostname);
-        ret = mnt3svc_umount (ms, dirpath, hostname);
-
-        /* Unmount succeeded with the given hostname. */
-        if (ret == 0)
-                goto snd_reply;
-
-try_umount_with_addr:
-        if (ret != 0)
-                ret = rpcsvc_transport_peeraddr (req->trans, hostname,
-                                                 MNTPATHLEN, NULL, 0);
-
-        if (ret != 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get remote addr: %s",
-                        gai_strerror (ret));
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
                 goto rpcerr;
         }
 
+        colon = strrchr (hostname, ':');
+        if (colon) {
+                *colon= '\0';
+        }
         gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s, hostname: %s", dirpath,
                 hostname);
         ret = mnt3svc_umount (ms, dirpath, hostname);
-        if (ret == -1)
-                mstat = MNT3ERR_INVAL;
 
-        ret = 0;
-snd_reply:
+        if (ret == -1) {
+                ret = 0;
+                mstat = MNT3ERR_NOENT;
+        }
+        /* FIXME: also take care of the corner case where the
+         * client was resolvable at mount but not at the umount - vice-versa.
+         */
         mnt3svc_submit_reply (req, &mstat,
                               (mnt3_serializer)xdr_serialize_mountstat3);
 
@@ -1295,6 +1819,10 @@ mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms)
                 return NULL;
 
         nfs = (struct nfs_state *)ms->nfsx->private;
+        if (!nfs)
+                return NULL;
+
+        LOCK (&ms->mountlock);
         list_for_each_entry(ent, &ms->exportlist, explist) {
 
                 /* If volume is not started yet, do not list it for tools like
@@ -1310,7 +1838,8 @@ mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms)
                                 " failed");
                         goto free_list;
                 }
-
+                 if (!first)
+                         first = elist;
                 elist->ex_dir = GF_CALLOC (namelen + 2, sizeof (char),
                                            gf_nfs_mt_char);
                 if (!elist->ex_dir) {
@@ -1318,16 +1847,10 @@ mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms)
                                 " failed");
                         goto free_list;
                 }
-
                 strcpy (elist->ex_dir, ent->expname);
 
                 addrstr = rpcsvc_volume_allowed (svc->options,
                                                  ent->vol->name);
-                if (addrstr)
-                        addrstr = gf_strdup (addrstr);
-                else
-                        addrstr = gf_strdup ("No Access");
-
                 elist->ex_groups = GF_CALLOC (1, sizeof (struct groupnode),
                                               gf_nfs_mt_groupnode);
                 if (!elist->ex_groups) {
@@ -1335,21 +1858,29 @@ mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms)
                                 " failed");
                         goto free_list;
                 }
+                /*This check has to be done after checking
+                 * elist->ex_groups allocation check to avoid resource leak;
+                */
+                if (addrstr)
+                        addrstr = gf_strdup (addrstr);
+                else
+                        addrstr = gf_strdup ("No Access");
 
+                if (!addrstr) {
+                        goto free_list;
+                }
                 elist->ex_groups->gr_name = addrstr;
                 if (prev) {
                         prev->ex_next = elist;
                         prev = elist;
                 } else
                         prev = elist;
-
-                if (!first)
-                        first = elist;
         }
 
         ret = 0;
 
 free_list:
+        UNLOCK (&ms->mountlock);
         if (ret == -1) {
                 xdr_free_exports_list (first);
                 first = NULL;
@@ -1399,7 +1930,233 @@ err:
         return ret;
 }
 
+/* just declaring, definition is way down below */
+rpcsvc_program_t        mnt3prog;
+
+/* nfs3_rootfh used by mount3udp thread needs to access mount3prog.private
+ * directly as we don't have nfs xlator pointer to dereference it. But thats OK
+ */
+
+struct nfs3_fh *
+nfs3_rootfh (char* path)
+{
+        struct mount3_state     *ms = NULL;
+        struct nfs3_fh          *fh = NULL;
+        struct mnt3_export      *exp = NULL;
+        inode_t                 *inode = NULL;
+        char                    *tmp = NULL;
+
+        ms = mnt3prog.private;
+        exp = mnt3_mntpath_to_export (ms, path);
+        if (exp == NULL)
+                goto err;
+
+        tmp = (char *)path;
+        tmp = strchr (tmp, '/');
+        if (tmp == NULL)
+                tmp = "/";
+
+        inode = inode_from_path (exp->vol->itable, tmp);
+        if (inode == NULL)
+                goto err;
+
+        fh = GF_CALLOC (1, sizeof(*fh), gf_nfs_mt_nfs3_fh);
+        if (fh == NULL)
+                goto err;
+        nfs3_build_fh (inode, exp->volumeid, fh);
+
+err:
+        if (inode)
+                inode_unref (inode);
+        return fh;
+}
+
+int
+mount3udp_add_mountlist (char *host, dirpath *expname)
+{
+        struct mountentry       *me = NULL;
+        struct mount3_state     *ms = NULL;
+        char                    *export = NULL;
 
+        ms = mnt3prog.private;
+        me = GF_CALLOC (1, sizeof (*me), gf_nfs_mt_mountentry);
+        if (!me)
+                return -1;
+        export = (char *)expname;
+        while (*export == '/')
+                export++;
+
+        strncpy (me->exname, export, MNTPATHLEN);
+        strncpy (me->hostname, host, MNTPATHLEN);
+        INIT_LIST_HEAD (&me->mlist);
+        LOCK (&ms->mountlock);
+        {
+                list_add_tail (&me->mlist, &ms->mountlist);
+                mount_rewrite_rmtab(ms, NULL);
+        }
+        UNLOCK (&ms->mountlock);
+        return 0;
+}
+
+int
+mount3udp_delete_mountlist (char *hostname, dirpath *expname)
+{
+        struct mount3_state     *ms = NULL;
+        char                    *export = NULL;
+
+        ms = mnt3prog.private;
+        export = (char *)expname;
+        while (*export == '/')
+                export++;
+        mnt3svc_umount (ms, export, hostname);
+        return 0;
+}
+
+/**
+ * This function will parse the hostip (IP addres, IP range, or hostname)
+ * and fill the host_auth_spec structure.
+ *
+ * @param hostspec - struct host_auth_spec
+ * @param hostip   - IP address, IP range (CIDR format) or hostname
+ *
+ * @return 0 - on success and -1 on failure
+ */
+int
+mnt3_export_fill_hostspec (struct host_auth_spec* hostspec, const char* hostip)
+{
+        char *ipdupstr = NULL;
+        char *savptr = NULL;
+        char *ip = NULL;
+        char *token = NULL;
+        int  ret = -1;
+
+        /* Create copy of the string so that the source won't change
+         */
+        ipdupstr = gf_strdup (hostip);
+        if (NULL == ipdupstr) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
+                goto err;
+        }
+
+        ip = strtok_r (ipdupstr, "/", &savptr);
+        hostspec->host_addr = gf_strdup (ip);
+        if (NULL == hostspec->host_addr) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
+                goto err;
+        }
+
+        /* Check if the IP is in <IP address> / <Range> format.
+         * If yes, then strip the range and store it separately.
+         */
+        token = strtok_r (NULL, "/", &savptr);
+
+        if (NULL == token) {
+              hostspec->routeprefix = -1;
+        } else {
+              hostspec->routeprefix = atoi (token);
+        }
+
+        // success
+        ret = 0;
+err:
+        if (NULL != ipdupstr) {
+                GF_FREE (ipdupstr);
+        }
+        return ret;
+}
+
+
+/**
+ * This function will parse the AUTH parameter passed along with
+ * "export-dir" option. If AUTH parameter is present then it will be
+ * stripped from exportpath and stored in mnt3_export (exp) structure.
+ *
+ * @param exp - mnt3_export structure. Holds information needed for mount.
+ * @param exportpath - Value of "export-dir" key. Holds both export path
+ *                     and AUTH parameter for the path.
+ *                     exportpath format: <abspath>[(hostdesc[|hostspec|...])]
+ *
+ * @return This function will return 0 on success and -1 on failure.
+ */
+int
+mnt3_export_parse_auth_param (struct mnt3_export* exp, char* exportpath)
+{
+        char *token = NULL;
+        char *savPtr = NULL;
+        char *hostip = NULL;
+        struct host_auth_spec *host = NULL;
+        int ret = 0;
+
+        /* Using exportpath directly in strtok_r because we want
+         * to strip off AUTH parameter from exportpath. */
+        token = strtok_r (exportpath, "(", &savPtr);
+
+        /* Get the next token, which will be the AUTH parameter. */
+        token = strtok_r (NULL, ")", &savPtr);
+
+        if (NULL == token) {
+                /* If AUTH is not present then we should return success. */
+                return 0;
+        }
+
+        /* Free any previously allocated hostspec structure. */
+        if (NULL != exp->hostspec) {
+                GF_FREE (exp->hostspec);
+                exp->hostspec = NULL;
+        }
+
+        exp->hostspec = GF_CALLOC (1,
+                        sizeof (*(exp->hostspec)),
+                        gf_nfs_mt_auth_spec);
+        if (NULL == exp->hostspec){
+                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
+                return -1;
+        }
+
+        /* AUTH parameter can have multiple entries. For each entry
+         * a host_auth_spec structure is created. */
+        host = exp->hostspec;
+
+        hostip = strtok_r (token, "|", &savPtr);
+
+        /* Parse all AUTH parameters separated by '|' */
+        while (NULL != hostip){
+                ret = mnt3_export_fill_hostspec (host, hostip);
+                if (0 != ret) {
+                        gf_log(GF_MNT, GF_LOG_WARNING,
+                                        "Failed to parse hostspec: %s", hostip);
+                        goto err;
+                }
+
+                hostip = strtok_r (NULL, "|", &savPtr);
+                if (NULL == hostip) {
+                        break;
+                }
+
+                host->next = GF_CALLOC (1, sizeof (*(host)),
+                                gf_nfs_mt_auth_spec);
+                if (NULL == host->next){
+                        gf_log (GF_MNT,GF_LOG_ERROR,
+                                        "Memory allocation failed");
+                        goto err;
+                }
+                host = host->next;
+        }
+
+        /* In case of success return from here */
+        return 0;
+err:
+        /* In case of failure free up hostspec structure.  */
+        FREE_HOSTSPEC (exp);
+
+        return -1;
+}
+
+/**
+ * exportpath will also have AUTH options (ip address, subnet address or
+ * hostname) mentioned.
+ * exportpath format: <abspath>[(hostdesc[|hostspec|...])]
+ */
 struct mnt3_export *
 mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
                       uuid_t volumeid)
@@ -1417,6 +2174,20 @@ mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
                 return NULL;
         }
 
+        if (NULL != exportpath) {
+                /* If exportpath is not NULL then we should check if AUTH
+                 * parameter is present or not. If AUTH parameter is present
+                 * then it will be stripped and stored in mnt3_export (exp)
+                 * structure.
+                 */
+                if (0 != mnt3_export_parse_auth_param (exp, exportpath)){
+                        gf_log (GF_MNT, GF_LOG_ERROR,
+                                                "Failed to parse auth param");
+                        goto err;
+                }
+        }
+
+
         INIT_LIST_HEAD (&exp->explist);
         if (exportpath)
                 alloclen = strlen (xl->name) + 2 + strlen (exportpath);
@@ -1426,8 +2197,6 @@ mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
         exp->expname = GF_CALLOC (alloclen, sizeof (char), gf_nfs_mt_char);
         if (!exp->expname) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
-                GF_FREE (exp);
-                exp = NULL;
                 goto err;
         }
 
@@ -1444,8 +2213,9 @@ mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
                 ret = snprintf (exp->expname, alloclen, "/%s", xl->name);
         }
         if (ret < 0) {
-                gf_log (xl->name, GF_LOG_WARNING,
-                        "failed to get the export name");
+                gf_log (xl->name, GF_LOG_ERROR,
+                        "Failed to set the export name");
+                goto err;
         }
         /* Just copy without discrimination, we'll determine whether to
          * actually use it when a mount request comes in and a file handle
@@ -1453,7 +2223,16 @@ mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
          */
         uuid_copy (exp->volumeid, volumeid);
         exp->vol = xl;
+
+        /* On success we should return from here*/
+        return exp;
 err:
+        /* On failure free exp and it's members.*/
+        if (NULL != exp) {
+                mnt3_export_free (exp);
+                exp = NULL;
+        }
+
         return exp;
 }
 
@@ -1493,8 +2272,7 @@ __mnt3_init_volume_direxports (struct mount3_state *ms, xlator_t *xlator,
 
         ret = 0;
 err:
-        if (dupopt)
-                GF_FREE (dupopt);
+        GF_FREE (dupopt);
 
         return ret;
 }
@@ -1615,8 +2393,11 @@ __mnt3_init_volume_export (struct mount3_state *ms, dict_t *opts)
                 goto err;
         }
 
-        gf_string2boolean (optstr, &boolt);
-        ret = 0;
+        ret = gf_string2boolean (optstr, &boolt);
+        if (ret < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to convert"
+                        " string to boolean");
+        }
 
 err:
         if (boolt == _gf_false) {
@@ -1655,8 +2436,11 @@ __mnt3_init_dir_export (struct mount3_state *ms, dict_t *opts)
                 goto err;
         }
 
-        gf_string2boolean (optstr, &boolt);
-        ret = 0;
+        ret = gf_string2boolean (optstr, &boolt);
+        if (ret < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to convert"
+                        " string to boolean");
+         }
 
 err:
         if (boolt == _gf_false) {
@@ -1754,18 +2538,20 @@ out:
 }
 
 rpcsvc_actor_t  mnt3svc_actors[MOUNT3_PROC_COUNT] = {
-        {"NULL", MOUNT3_NULL, mnt3svc_null, NULL, NULL, 0},
-        {"MNT", MOUNT3_MNT, mnt3svc_mnt, NULL, NULL, 0},
-        {"DUMP", MOUNT3_DUMP, mnt3svc_dump, NULL, NULL, 0},
-        {"UMNT", MOUNT3_UMNT, mnt3svc_umnt, NULL, NULL, 0},
-        {"UMNTALL", MOUNT3_UMNTALL, mnt3svc_umntall, NULL, NULL, 0},
-        {"EXPORT", MOUNT3_EXPORT, mnt3svc_export, NULL, NULL, 0}
+        {"NULL",    MOUNT3_NULL,    mnt3svc_null,    NULL, 0, DRC_NA},
+        {"MNT",     MOUNT3_MNT,     mnt3svc_mnt,     NULL, 0, DRC_NA},
+        {"DUMP",    MOUNT3_DUMP,    mnt3svc_dump,    NULL, 0, DRC_NA},
+        {"UMNT",    MOUNT3_UMNT,    mnt3svc_umnt,    NULL, 0, DRC_NA},
+        {"UMNTALL", MOUNT3_UMNTALL, mnt3svc_umntall, NULL, 0, DRC_NA},
+        {"EXPORT",  MOUNT3_EXPORT,  mnt3svc_export,  NULL, 0, DRC_NA}
 };
 
 
 
 /* Static init parts are assigned here, dynamic ones are done in
  * mnt3svc_init and mnt3_init_state.
+ * Making MOUNT3 a synctask so that the blocking DNS calls during rpc auth
+ * gets offloaded to syncenv, keeping the main/poll thread unblocked
  */
 rpcsvc_program_t        mnt3prog = {
                         .progname       = "MOUNT3",
@@ -1775,9 +2561,11 @@ rpcsvc_program_t        mnt3prog = {
                         .actors         = mnt3svc_actors,
                         .numactors      = MOUNT3_PROC_COUNT,
                         .min_auth       = AUTH_NULL,
+                        .synctask       = _gf_true,
 };
 
 
+
 rpcsvc_program_t *
 mnt3svc_init (xlator_t *nfsx)
 {
@@ -1786,6 +2574,7 @@ mnt3svc_init (xlator_t *nfsx)
         dict_t                  *options = NULL;
         char                    *portstr = NULL;
         int                      ret = -1;
+        pthread_t                udp_thread;
 
         if (!nfsx || !nfsx->private)
                 return NULL;
@@ -1828,13 +2617,16 @@ mnt3svc_init (xlator_t *nfsx)
                 }
         }
 
-        rpcsvc_create_listeners (nfs->rpcsvc, options, nfsx->name);
+        ret= rpcsvc_create_listeners (nfs->rpcsvc, options, nfsx->name);
         if (ret == -1) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "Unable to create listeners");
                 dict_unref (options);
                 goto err;
         }
 
+        if (nfs->mount_udp) {
+                pthread_create (&udp_thread, NULL, mount3udp_thread, NULL);
+        }
         return &mnt3prog;
 err:
         return NULL;
@@ -1842,12 +2634,12 @@ err:
 
 
 rpcsvc_actor_t  mnt1svc_actors[MOUNT1_PROC_COUNT] = {
-        {"NULL", MOUNT1_NULL, mnt3svc_null, NULL, NULL, 0},
-        {{0, 0}, },
-        {"DUMP", MOUNT1_DUMP, mnt3svc_dump, NULL, NULL, 0},
-        {"UMNT", MOUNT1_UMNT, mnt3svc_umnt, NULL, NULL, 0},
-        {{0, 0}, },
-        {"EXPORT", MOUNT1_EXPORT, mnt3svc_export, NULL, NULL, 0}
+        {"NULL",    MOUNT1_NULL,    mnt3svc_null,   NULL, 0, DRC_NA},
+        {"MNT",     MOUNT1_MNT,     NULL,           NULL, 0, DRC_NA },
+        {"DUMP",    MOUNT1_DUMP,    mnt3svc_dump,   NULL, 0, DRC_NA},
+        {"UMNT",    MOUNT1_UMNT,    mnt3svc_umnt,   NULL, 0, DRC_NA},
+        {"UMNTALL", MOUNT1_UMNTALL, NULL,           NULL, 0, DRC_NA},
+        {"EXPORT",  MOUNT1_EXPORT,  mnt3svc_export, NULL, 0, DRC_NA}
 };
 
 rpcsvc_program_t        mnt1prog = {
@@ -1858,6 +2650,7 @@ rpcsvc_program_t        mnt1prog = {
                         .actors         = mnt1svc_actors,
                         .numactors      = MOUNT1_PROC_COUNT,
                         .min_auth       = AUTH_NULL,
+                        .synctask       = _gf_true,
 };
 
 
@@ -1912,7 +2705,7 @@ mnt1svc_init (xlator_t *nfsx)
                 }
         }
 
-        rpcsvc_create_listeners (nfs->rpcsvc, options, nfsx->name);
+        ret = rpcsvc_create_listeners (nfs->rpcsvc, options, nfsx->name);
         if (ret == -1) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "Unable to create listeners");
                 dict_unref (options);
@@ -1924,4 +2717,43 @@ err:
         return NULL;
 }
 
+int
+mount_reconfigure_state (xlator_t *nfsx, dict_t *options)
+{
+        int                    ret      = -1;
+        struct nfs_state       *nfs     = NULL;
+        struct mount3_state    *ms      = NULL;
+        struct mnt3_export     *exp     = NULL;
+        struct mnt3_export     *texp  = NULL;
+
+        if ((!nfsx) || (!options))
+                return (-1);
+
+        nfs = (struct nfs_state *)nfs_state (nfsx);
+        if (!nfs)
+                return (-1);
+
+        ms = nfs->mstate;
+        if (!ms)
+                return (-1);
+
+        /*
+         * Free() up the old export list. mnt3_init_options() will
+         * rebuild the export list from scratch. Do it with locking
+         * to avoid unnecessary race conditions.
+         */
+        LOCK (&ms->mountlock);
+        list_for_each_entry_safe (exp, texp, &ms->exportlist, explist) {
+                list_del (&exp->explist);
+                mnt3_export_free (exp);
+        }
+        ret = mnt3_init_options (ms, options);
+        UNLOCK (&ms->mountlock);
+
+        if (ret < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Options reconfigure failed");
+                return (-1);
+        }
 
+        return (0);
+}
diff --git a/xlators/nfs/server/src/mount3.h b/xlators/nfs/server/src/mount3.h
index c0eae3644..7fc16ed57 100644
--- a/xlators/nfs/server/src/mount3.h
+++ b/xlators/nfs/server/src/mount3.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _MOUNT3_H_
@@ -53,6 +44,12 @@ mnt1svc_init (xlator_t *nfsx);
 extern int
 mount_init_state (xlator_t *nfsx);
 
+extern int
+mount_reconfigure_state (xlator_t *nfsx, dict_t *options);
+
+void
+mount_rewrite_rmtab (struct mount3_state *ms, char *new_rmtab);
+
 /* Data structure used to store the list of mounts points currently
  * in use by NFS clients.
  */
@@ -68,6 +65,13 @@ struct mountentry {
 #define MNT3_EXPTYPE_VOLUME     1
 #define MNT3_EXPTYPE_DIR        2
 
+/* Structure to hold export-dir AUTH parameter */
+struct host_auth_spec {
+        char                    *host_addr;    /* Allowed IP or host name */
+        int                     routeprefix;   /* Routing prefix */
+        struct host_auth_spec   *next;         /* Pointer to next AUTH struct */
+};
+
 struct mnt3_export {
         struct list_head        explist;
 
@@ -75,6 +79,11 @@ struct mnt3_export {
          * is exported or the subdirectory in the volume.
          */
         char                    *expname;
+        /*
+         * IP address, hostname or subnets who are allowed to connect to expname
+         * subvolume or subdirectory
+         */
+        struct host_auth_spec*  hostspec;
         xlator_t                *vol;
         int                     exptype;
 
@@ -101,8 +110,8 @@ struct mount3_state {
         gf_lock_t               mountlock;
 
         /* Set to 0 if exporting full volumes is disabled. On by default. */
-        int                     export_volumes;
-        int                     export_dirs;
+        gf_boolean_t            export_volumes;
+        gf_boolean_t            export_dirs;
 };
 
 #define gf_mnt3_export_dirs(mst)        ((mst)->export_dirs)
diff --git a/xlators/nfs/server/src/mount3udp_svc.c b/xlators/nfs/server/src/mount3udp_svc.c
new file mode 100644
index 000000000..fb59e282c
--- /dev/null
+++ b/xlators/nfs/server/src/mount3udp_svc.c
@@ -0,0 +1,189 @@
+/*
+  Copyright (c) 2012 Gluster, Inc. <http://www.gluster.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#include "xdr-nfs3.h"
+#include "logging.h"
+#include "mem-pool.h"
+#include "nfs-mem-types.h"
+#include "mount3.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <rpc/pmap_clnt.h>
+#include <string.h>
+#include <memory.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+
+
+extern struct nfs3_fh* nfs3_rootfh (char *dp);
+extern mountres3 mnt3svc_set_mountres3 (mountstat3 stat, struct nfs3_fh *fh,
+                                        int *authflavor, u_int aflen);
+extern int
+mount3udp_add_mountlist (char *host, dirpath *expname);
+
+extern int
+mount3udp_delete_mountlist (char *host, dirpath *expname);
+
+
+/* only this thread will use this, no locking needed */
+char mnthost[INET_ADDRSTRLEN+1];
+
+mountres3 *
+mountudpproc3_mnt_3_svc(dirpath **dpp, struct svc_req *req)
+{
+        struct mountres3        *res = NULL;
+        int                     *autharr = NULL;
+        struct nfs3_fh          *fh = NULL;
+        char                    *tmp = NULL;
+
+        tmp = (char *)*dpp;
+        while (*tmp == '/')
+                tmp++;
+        fh = nfs3_rootfh (tmp);
+        if (fh == NULL) {
+                gf_log (GF_MNT, GF_LOG_DEBUG, "unable to get fh for %s", tmp);
+                goto err;
+        }
+
+        res = GF_CALLOC (1, sizeof(*res), gf_nfs_mt_mountres3);
+        if (res == NULL) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "unable to allocate memory");
+                goto err;
+        }
+        autharr = GF_CALLOC (1, sizeof(*autharr), gf_nfs_mt_int);
+        if (autharr == NULL) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "unable to allocate memory");
+                goto err;
+        }
+        autharr[0] = AUTH_UNIX;
+        *res = mnt3svc_set_mountres3 (MNT3_OK, fh, autharr, 1);
+        mount3udp_add_mountlist (mnthost, *dpp);
+        return res;
+
+ err:
+        GF_FREE (fh);
+        GF_FREE (res);
+        GF_FREE (autharr);
+        return NULL;
+}
+
+mountstat3 *
+mountudpproc3_umnt_3_svc(dirpath **dp, struct svc_req *req)
+{
+        mountstat3 *stat = NULL;
+
+        stat = GF_CALLOC (1, sizeof(mountstat3), gf_nfs_mt_mountstat3);
+        if (stat == NULL) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "unable to allocate memory");
+                return NULL;
+        }
+        *stat = MNT3_OK;
+        mount3udp_delete_mountlist (mnthost, *dp);
+        return stat;
+}
+
+static void
+mountudp_program_3(struct svc_req *rqstp, register SVCXPRT *transp)
+{
+        union {
+                dirpath mountudpproc3_mnt_3_arg;
+        } argument;
+        char                    *result = NULL;
+        xdrproc_t               _xdr_argument = NULL, _xdr_result = NULL;
+        char *(*local)(char *, struct svc_req *) = NULL;
+        mountres3               *res = NULL;
+        struct sockaddr_in      *sin = NULL;
+
+        sin = svc_getcaller (transp);
+        inet_ntop (AF_INET, &sin->sin_addr, mnthost, INET_ADDRSTRLEN+1);
+
+        switch (rqstp->rq_proc) {
+        case NULLPROC:
+                (void) svc_sendreply (transp, (xdrproc_t) xdr_void,
+                                      (char *)NULL);
+                return;
+
+        case MOUNT3_MNT:
+                _xdr_argument = (xdrproc_t) xdr_dirpath;
+                _xdr_result = (xdrproc_t) xdr_mountres3;
+                local = (char *(*)(char *,
+                                   struct svc_req *)) mountudpproc3_mnt_3_svc;
+                break;
+
+        case MOUNT3_UMNT:
+                _xdr_argument = (xdrproc_t) xdr_dirpath;
+                _xdr_result = (xdrproc_t) xdr_mountstat3;
+                local = (char *(*)(char *,
+                                   struct svc_req *)) mountudpproc3_umnt_3_svc;
+                break;
+
+        default:
+                svcerr_noproc (transp);
+                return;
+        }
+        memset ((char *)&argument, 0, sizeof (argument));
+        if (!svc_getargs (transp, (xdrproc_t) _xdr_argument,
+                          (caddr_t) &argument)) {
+                svcerr_decode (transp);
+                return;
+        }
+        result = (*local)((char *)&argument, rqstp);
+        if (result == NULL) {
+                gf_log (GF_MNT, GF_LOG_DEBUG, "PROC returned error");
+                svcerr_systemerr (transp);
+        }
+        if (result != NULL && !svc_sendreply(transp, (xdrproc_t) _xdr_result,
+                                             result)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "svc_sendreply returned error");
+                svcerr_systemerr (transp);
+        }
+        if (!svc_freeargs (transp, (xdrproc_t) _xdr_argument,
+                           (caddr_t) &argument)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "unable to free arguments");
+        }
+        if (result == NULL)
+                return;
+        /* free the result */
+        switch (rqstp->rq_proc) {
+        case MOUNT3_MNT:
+                res = (mountres3 *) result;
+                GF_FREE (res->mountres3_u.mountinfo.fhandle.fhandle3_val);
+                GF_FREE (res->mountres3_u.mountinfo.auth_flavors.auth_flavors_val);
+                GF_FREE (res);
+                break;
+
+        case MOUNT3_UMNT:
+                GF_FREE (result);
+                break;
+        }
+        return;
+}
+
+void *
+mount3udp_thread (void *argv)
+{
+        register SVCXPRT *transp = NULL;
+
+        transp = svcudp_create(RPC_ANYSOCK);
+        if (transp == NULL) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "svcudp_create error");
+                return NULL;
+        }
+        if (!svc_register(transp, MOUNT_PROGRAM, MOUNT_V3,
+                          mountudp_program_3, IPPROTO_UDP)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "svc_register error");
+                return NULL;
+        }
+
+        svc_run ();
+        gf_log (GF_MNT, GF_LOG_ERROR, "svc_run returned");
+        return NULL;
+}
diff --git a/xlators/nfs/server/src/nfs-common.c b/xlators/nfs/server/src/nfs-common.c
index b5c56d03d..f74396ee8 100644
--- a/xlators/nfs/server/src/nfs-common.c
+++ b/xlators/nfs/server/src/nfs-common.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -88,13 +79,13 @@ nfs_mntpath_to_xlator (xlator_list_t *cl, char *path)
 {
         char            volname[MNTPATHLEN];
         char            *volptr = NULL;
-        int             pathlen = 0;
+        size_t          pathlen;
         xlator_t        *targetxl = NULL;
 
         if ((!cl) || (!path))
                 return NULL;
 
-        strcpy (volname, path);
+        strncpy (volname, path, MNTPATHLEN);
         pathlen = strlen (volname);
         gf_log (GF_NFS, GF_LOG_TRACE, "Subvolume search: %s", path);
         if (volname[0] == '/')
@@ -102,7 +93,7 @@ nfs_mntpath_to_xlator (xlator_list_t *cl, char *path)
         else
                 volptr = &volname[0];
 
-        if (volname[pathlen - 1] == '/')
+        if (pathlen && volname[pathlen - 1] == '/')
                 volname[pathlen - 1] = '\0';
 
         while (cl) {
@@ -203,11 +194,26 @@ nfs_inode_loc_fill (inode_t *inode, loc_t *loc, int how)
         if ((!inode) || (!loc))
                 return ret;
 
-        ret = inode_path (inode, NULL, &resolvedpath);
-        if (ret < 0) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "path resolution failed %s",
-                                resolvedpath);
-                goto err;
+        /* If gfid is not null, then the inode is already linked to
+         * the inode table, and not a newly created one. For newly
+         * created inode, inode_path returns null gfid as the path.
+         */
+        if (!uuid_is_null (inode->gfid)) {
+                ret = inode_path (inode, NULL, &resolvedpath);
+                if (ret < 0) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "path resolution failed "
+                                "%s", resolvedpath);
+                        goto err;
+                }
+        }
+
+        if (resolvedpath == NULL) {
+                char tmp_path[GFID_STR_PFX_LEN + 1] = {0,};
+                snprintf (tmp_path, sizeof (tmp_path), "<gfid:%s>",
+                          uuid_utoa (loc->gfid));
+                resolvedpath = gf_strdup (tmp_path);
+        } else {
+                parent = inode_parent (inode, loc->pargfid, NULL);
         }
 
         ret = nfs_loc_fill (loc, inode, parent, resolvedpath);
@@ -222,8 +228,7 @@ err:
         if (parent)
                 inode_unref (parent);
 
-        if (resolvedpath)
-                GF_FREE (resolvedpath);
+        GF_FREE (resolvedpath);
 
         return ret;
 }
@@ -384,8 +389,7 @@ err:
         if (entryinode)
                 inode_unref (entryinode);
 
-        if (resolvedpath)
-                GF_FREE (resolvedpath);
+        GF_FREE (resolvedpath);
 
         return ret;
 }
@@ -404,6 +408,9 @@ nfs_hash_gfid (uuid_t gfid)
         uint32_t                b1 = 0;
         uint32_t                b2 = 0;
 
+        if (__is_root_gfid (gfid))
+                return 0x1;
+
         memcpy (&msb64, &gfid[8], 8);
         memcpy (&lsb64, &gfid[0], 8);
 
@@ -421,3 +428,38 @@ nfs_hash_gfid (uuid_t gfid)
 }
 
 
+void
+nfs_fix_generation (xlator_t *this, inode_t *inode)
+{
+        uint64_t                 raw_ctx        = 0;
+        struct nfs_inode_ctx    *ictx           = NULL;
+        struct nfs_state        *priv           = NULL;
+        int                      ret            = -1;
+
+        if (!inode) {
+                return;
+        }
+        priv = this->private;
+
+        if (inode_ctx_get(inode,this,&raw_ctx) == 0) {
+                ictx = (struct nfs_inode_ctx *)raw_ctx;
+                ictx->generation = priv->generation;
+        }
+        else {
+                ictx = GF_CALLOC (1, sizeof (struct nfs_inode_ctx),
+                                  gf_nfs_mt_inode_ctx);
+                if (!ictx) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not allocate nfs inode ctx");
+                        return;
+                }
+                INIT_LIST_HEAD(&ictx->shares);
+                ictx->generation = priv->generation;
+                ret = inode_ctx_put (inode, this, (uint64_t)ictx);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not store nfs inode ctx");
+                        return;
+                }
+        }
+}
diff --git a/xlators/nfs/server/src/nfs-common.h b/xlators/nfs/server/src/nfs-common.h
index 58dea70d0..2e97f1563 100644
--- a/xlators/nfs/server/src/nfs-common.h
+++ b/xlators/nfs/server/src/nfs-common.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS_COMMON_H_
@@ -37,7 +28,7 @@
 #define NFS_PATH_MAX    4096
 #define NFS_NAME_MAX    NAME_MAX
 
-#define NFS_DEFAULT_CREATE_MODE 0644
+#define NFS_DEFAULT_CREATE_MODE 0600
 
 extern xlator_t *
 nfs_xlid_to_xlator (xlator_list_t *cl, uint8_t xlid);
@@ -84,4 +75,7 @@ nfs_hash_gfid (uuid_t gfid);
 
 extern int
 nfs_gfid_loc_fill (inode_table_t *itable, uuid_t gfid, loc_t *loc, int how);
+
+void
+nfs_fix_generation (xlator_t *this, inode_t *inode);
 #endif
diff --git a/xlators/nfs/server/src/nfs-fops.c b/xlators/nfs/server/src/nfs-fops.c
index 8774da075..236b80c76 100644
--- a/xlators/nfs/server/src/nfs-fops.c
+++ b/xlators/nfs/server/src/nfs-fops.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -22,6 +13,9 @@
 #include "config.h"
 #endif
 
+#include <grp.h>
+#include <pwd.h>
+
 #include "dict.h"
 #include "xlator.h"
 #include "iobuf.h"
@@ -32,9 +26,79 @@
 #include "inode.h"
 #include "nfs-common.h"
 #include "nfs3-helpers.h"
+#include "nfs-mem-types.h"
 #include <libgen.h>
 #include <semaphore.h>
 
+void
+nfs_fix_groups (xlator_t *this, call_stack_t *root)
+{
+        struct passwd    mypw;
+        char             mystrs[1024];
+        struct passwd    *result;
+        gid_t            mygroups[GF_MAX_AUX_GROUPS];
+        int              ngroups;
+        int              i;
+        struct nfs_state *priv = this->private;
+        const gid_list_t *agl;
+	gid_list_t gl;
+
+        if (!priv->server_aux_gids) {
+                return;
+        }
+
+	agl = gid_cache_lookup(&priv->gid_cache, root->uid);
+	if (agl) {
+		for (ngroups = 0; ngroups < agl->gl_count; ngroups++) 
+			root->groups[ngroups] = agl->gl_list[ngroups];
+		root->ngrps = ngroups;
+		gid_cache_release(&priv->gid_cache, agl);
+		return;
+	}
+
+	/* No cached list found. */
+        if (getpwuid_r(root->uid,&mypw,mystrs,sizeof(mystrs),&result) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "getpwuid_r(%u) failed", root->uid);
+                return;
+        }
+
+        if (!result) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "getpwuid_r(%u) found nothing", root->uid);
+                return;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "mapped %u => %s",
+                root->uid, result->pw_name);
+
+        ngroups = GF_MAX_AUX_GROUPS;
+        if (getgrouplist(result->pw_name,root->gid,mygroups,&ngroups) == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not map %s to group list", result->pw_name);
+                return;
+        }
+
+	/* Add the group data to the cache. */
+	gl.gl_list = GF_CALLOC(ngroups, sizeof(gid_t), gf_nfs_mt_aux_gids);
+	if (gl.gl_list) {
+		/* It's not fatal if the alloc failed. */
+		gl.gl_id = root->uid;
+		gl.gl_count = ngroups;
+		memcpy(gl.gl_list, mygroups, sizeof(gid_t) * ngroups);
+		if (gid_cache_add(&priv->gid_cache, &gl) != 1)
+			GF_FREE(gl.gl_list);
+	}
+
+	/* Copy data to the frame. */
+        for (i = 0; i < ngroups; ++i) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "%s is in group %u", result->pw_name, mygroups[i]);
+                root->groups[i] = mygroups[i];
+        }
+        root->ngrps = ngroups;
+}
+
 struct nfs_fop_local *
 nfs_fop_local_init (xlator_t *nfsx)
 {
@@ -118,22 +182,33 @@ nfs_create_frame (xlator_t *xl, nfs_user_t *nfu)
         frame = create_frame (xl, (call_pool_t *)xl->ctx->pool);
         if (!frame)
                 goto err;
+	if (call_stack_alloc_groups (frame->root, nfu->ngrps) != 0) {
+		STACK_DESTROY (frame->root);
+		frame = NULL;
+		goto err;
+	}
+
         frame->root->pid = NFS_PID;
         frame->root->uid = nfu->uid;
         frame->root->gid = nfu->gids[NFS_PRIMGID_IDX];
-        if (nfu->ngrps == 1)
-                goto err;       /* Done, we only got primary gid */
+        frame->root->lk_owner = nfu->lk_owner;
 
-        frame->root->ngrps = nfu->ngrps - 1;
+        if (nfu->ngrps != 1) {
+                frame->root->ngrps = nfu->ngrps - 1;
 
-        gf_log (GF_NFS, GF_LOG_TRACE,"uid: %d, gid %d, gids: %d",
-                frame->root->uid, frame->root->gid, frame->root->ngrps);
-        for(y = 0, x = 1;  y < frame->root->ngrps; x++,y++) {
-                gf_log (GF_NFS, GF_LOG_TRACE, "gid: %d", nfu->gids[x]);
-                frame->root->groups[y] = nfu->gids[x];
+                gf_log (GF_NFS, GF_LOG_TRACE,"uid: %d, gid %d, gids: %d",
+                        frame->root->uid, frame->root->gid, frame->root->ngrps);
+                for(y = 0, x = 1;  y < frame->root->ngrps; x++,y++) {
+                        gf_log (GF_NFS, GF_LOG_TRACE, "gid: %d", nfu->gids[x]);
+                        frame->root->groups[y] = nfu->gids[x];
+                }
         }
 
-        set_lk_owner_from_uint64 (&frame->root->lk_owner, nfs_frame_getctr ());
+        /*
+         * It's tempting to do this *instead* of using nfu above, but we need
+         * to have those values in case nfs_fix_groups doesn't do anything.
+         */
+        nfs_fix_groups(xl,frame->root);
 
 err:
         return frame;
@@ -141,7 +216,7 @@ err:
 
 #define nfs_fop_handle_frame_create(fram, xla, nfuser, retval, errlabel)      \
         do {                                                                  \
-                fram = nfs_create_frame (xla, (nfuser));                      \
+                fram = nfs_create_frame (xla, (nfuser));                \
                 if (!fram) {                                                  \
                         retval = (-ENOMEM);                                   \
                         gf_log (GF_NFS, GF_LOG_ERROR,"Frame creation failed");\
@@ -275,9 +350,6 @@ out:
                 }                                                       \
         } while (0)                                                     \
 
-
-
-
 /* Fops Layer Explained
  * The fops layer has three types of functions. They can all be identified by
  * their names. Here are the three patterns:
@@ -311,6 +383,10 @@ nfs_fop_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         struct nfs_fop_local    *local = NULL;
         fop_lookup_cbk_t        progcbk;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (local, progcbk, frame);
         nfs_fop_restore_root_ino (local, op_ret, buf, NULL, NULL, postparent);
         if (progcbk)
@@ -354,14 +430,14 @@ err:
 
 int32_t
 nfs_fop_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_access_cbk_t        progcbk = NULL;
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno);
+                progcbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -386,7 +462,7 @@ nfs_fop_access (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
 
         accessbits = nfs3_request_to_accessbits (accesstest);
         STACK_WIND_COOKIE (frame, nfs_fop_access_cbk, xl, xl, xl->fops->access,
-                           loc, accessbits);
+                           loc, accessbits, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -399,7 +475,8 @@ err:
 
 int32_t
 nfs_fop_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_stat_cbk_t          progcbk = NULL;
@@ -407,7 +484,7 @@ nfs_fop_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf);
+                progcbk (frame, cookie, this, op_ret, op_errno, buf, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -431,7 +508,7 @@ nfs_fop_stat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
         nfs_fop_save_root_ino (nfl, loc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_stat_cbk, xl, xl, xl->fops->stat,
-                           loc);
+                           loc, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -445,7 +522,8 @@ err:
 
 int32_t
 nfs_fop_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_fstat_cbk_t         progcbk = NULL;
@@ -453,7 +531,7 @@ nfs_fop_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf);
+                progcbk (frame, cookie, this, op_ret, op_errno, buf, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -477,7 +555,7 @@ nfs_fop_fstat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         nfs_fop_save_root_fd_ino (nfl, fd);
 
         STACK_WIND_COOKIE (frame, nfs_fop_fstat_cbk, xl, xl, xl->fops->fstat,
-                           fd);
+                           fd, NULL);
 
         ret = 0;
 err:
@@ -492,14 +570,14 @@ err:
 
 int32_t
 nfs_fop_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd)
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_opendir_cbk_t       progcbk = NULL;
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd);
+                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -521,7 +599,7 @@ nfs_fop_opendir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
 
         STACK_WIND_COOKIE (frame, nfs_fop_opendir_cbk, xl, xl,
-                           xl->fops->opendir, pathloc, dirfd);
+                           xl->fops->opendir, pathloc, dirfd, NULL);
         ret = 0;
 
 err:
@@ -535,14 +613,14 @@ err:
 
 int
 nfs_fop_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_flush_cbk_t         progcbk = NULL;
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno);
+                progcbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -564,7 +642,7 @@ nfs_fop_flush (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
 
         STACK_WIND_COOKIE (frame, nfs_fop_flush_cbk, xl, xl, xl->fops->flush,
-                           fd);
+                           fd, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -578,14 +656,15 @@ err:
 
 int32_t
 nfs_fop_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                      dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_readdirp_cbk_t      progcbk = NULL;
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, entries);
+                progcbk (frame, cookie, this, op_ret, op_errno, entries, xdata);
 
         nfs_stack_destroy (nfl, frame);
 
@@ -625,7 +704,8 @@ err:
 
 int32_t
 nfs_fop_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                    int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                    dict_t *xdata)
 {
 
         struct nfs_fop_local    *nfl = NULL;
@@ -633,7 +713,7 @@ nfs_fop_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf);
+                progcbk (frame, cookie, this, op_ret, op_errno, buf, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -656,7 +736,7 @@ nfs_fop_statfs (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
 
         STACK_WIND_COOKIE (frame, nfs_fop_statfs_cbk, xl, xl,
-                           xl->fops->statfs, pathloc);
+                           xl->fops->statfs, pathloc, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -672,17 +752,21 @@ int32_t
 nfs_fop_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_create_cbk_t        progcbk = NULL;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, preparent,
                                   postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, fd, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, NULL);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -708,7 +792,7 @@ nfs_fop_create (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
 
         STACK_WIND_COOKIE (frame, nfs_fop_create_cbk, xl, xl, xl->fops->create,
-                           pathloc, flags, mode, fd, nfl->dictgfid);
+                           pathloc, flags, mode, 0, fd, nfl->dictgfid);
 
         ret = 0;
 err:
@@ -724,7 +808,7 @@ err:
 int32_t
 nfs_fop_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                     struct iatt *post)
+                     struct iatt *post, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_setattr_cbk_t       progcbk = NULL;
@@ -732,7 +816,8 @@ nfs_fop_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, pre, post, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, pre, post);
+                progcbk (frame, cookie, this, op_ret, op_errno, pre, post,
+                         xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -757,7 +842,7 @@ nfs_fop_setattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_save_root_ino (nfl, pathloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_setattr_cbk, xl, xl,
-                           xl->fops->setattr, pathloc, buf, valid);
+                           xl->fops->setattr, pathloc, buf, valid, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -773,16 +858,20 @@ int32_t
 nfs_fop_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_mkdir_cbk_t         progcbk = NULL;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL,preparent, postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -806,7 +895,7 @@ nfs_fop_mkdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
 
         STACK_WIND_COOKIE  (frame, nfs_fop_mkdir_cbk, xl, xl, xl->fops->mkdir,
-                            pathloc, mode, nfl->dictgfid);
+                            pathloc, mode, 0, nfl->dictgfid);
         ret = 0;
 err:
         if (ret < 0) {
@@ -822,16 +911,20 @@ int32_t
 nfs_fop_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_symlink_cbk_t       progcbk = NULL;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret,buf, NULL, preparent, postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -854,7 +947,8 @@ nfs_fop_symlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
         nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
 
         STACK_WIND_COOKIE  (frame, nfs_fop_symlink_cbk, xl, xl,
-                            xl->fops->symlink, target, pathloc, nfl->dictgfid);
+                            xl->fops->symlink, target, pathloc,
+                            0, nfl->dictgfid);
         ret = 0;
 err:
         if (ret < 0) {
@@ -869,7 +963,7 @@ err:
 int32_t
 nfs_fop_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf)
+                      struct iatt *buf, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_readlink_cbk_t      progcbk = NULL;
@@ -877,7 +971,8 @@ nfs_fop_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, path, buf);
+                progcbk (frame, cookie, this, op_ret, op_errno, path, buf,
+                         xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -900,7 +995,7 @@ nfs_fop_readlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_save_root_ino (nfl, pathloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_readlink_cbk, xl, xl,
-                           xl->fops->readlink, pathloc, size);
+                           xl->fops->readlink, pathloc, size, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -916,16 +1011,20 @@ int32_t
 nfs_fop_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_mknod_cbk_t         progcbk = NULL;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret,buf, NULL, preparent, postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -949,7 +1048,7 @@ nfs_fop_mknod (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
 
         STACK_WIND_COOKIE  (frame, nfs_fop_mknod_cbk, xl, xl, xl->fops->mknod,
-                            pathloc, mode, dev, nfl->dictgfid);
+                            pathloc, mode, dev, 0, nfl->dictgfid);
         ret = 0;
 err:
         if (ret < 0) {
@@ -963,7 +1062,7 @@ err:
 int32_t
 nfs_fop_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = frame->local;
         fop_rmdir_cbk_t         progcbk = NULL;
@@ -973,7 +1072,7 @@ nfs_fop_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                   postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                         postparent);
+                         postparent, NULL);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -997,7 +1096,7 @@ nfs_fop_rmdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_save_root_ino (nfl, pathloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_rmdir_cbk, xl, xl, xl->fops->rmdir,
-                           pathloc, 0);
+                           pathloc, 0, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1013,7 +1112,7 @@ err:
 int32_t
 nfs_fop_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = frame->local;
         fop_unlink_cbk_t         progcbk = NULL;
@@ -1023,7 +1122,7 @@ nfs_fop_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                   postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                         postparent);
+                         postparent, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -1046,7 +1145,7 @@ nfs_fop_unlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_save_root_ino (nfl, pathloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_unlink_cbk, xl, xl,
-                           xl->fops->unlink, pathloc);
+                           xl->fops->unlink, pathloc, 0, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1063,17 +1162,21 @@ int32_t
 nfs_fop_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_link_cbk_t          progcbk = NULL;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, preparent,
                                   postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -1098,7 +1201,7 @@ nfs_fop_link (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
         nfs_fop_save_root_ino (nfl, newloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_link_cbk, xl, xl, xl->fops->link,
-                           oldloc, newloc);
+                           oldloc, newloc, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1114,7 +1217,8 @@ int32_t
 nfs_fop_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
                     struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent)
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
 
         struct nfs_fop_local    *nfl = NULL;
@@ -1132,7 +1236,7 @@ nfs_fop_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, buf,
                          preoldparent, postoldparent, prenewparent,
-                         postnewparent);
+                         postnewparent, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -1157,7 +1261,7 @@ nfs_fop_rename (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
         nfs_fop_newloc_save_root_ino (nfl, newloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_rename_cbk, xl, xl,
-                           xl->fops->rename, oldloc, newloc);
+                           xl->fops->rename, oldloc, newloc, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1171,14 +1275,14 @@ err:
 
 int32_t
 nfs_fop_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd)
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_open_cbk_t          progcbk = NULL;
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd);
+                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
         nfs_stack_destroy (nfl, frame);
 
         return 0;
@@ -1186,7 +1290,7 @@ nfs_fop_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 nfs_fop_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-              int32_t flags, fd_t *fd, int32_t wbflags, fop_open_cbk_t cbk,
+              int32_t flags, fd_t *fd, fop_open_cbk_t cbk,
               void *local)
 {
         call_frame_t            *frame = NULL;
@@ -1201,7 +1305,7 @@ nfs_fop_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
         nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
 
         STACK_WIND_COOKIE (frame, nfs_fop_open_cbk, xl, xl, xl->fops->open,
-                           loc, flags, fd, wbflags);
+                           loc, flags, fd, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1216,7 +1320,7 @@ err:
 int32_t
 nfs_fop_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf)
+                    struct iatt *postbuf, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_writev_cbk_t        progcbk = NULL;
@@ -1224,7 +1328,8 @@ nfs_fop_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, prebuf, postbuf, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,postbuf);
+                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
 
         nfs_stack_destroy (nfl, frame);
 
@@ -1258,7 +1363,7 @@ nfs_fop_write (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         iobref_add (nfl->iobref, srciob);
 */
         STACK_WIND_COOKIE (frame, nfs_fop_writev_cbk, xl, xl,xl->fops->writev,
-                           fd, vector, count, offset, 0, srciobref);
+                           fd, vector, count, offset, fd->flags, srciobref, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1273,7 +1378,7 @@ err:
 int32_t
 nfs_fop_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_fsync_cbk_t         progcbk = NULL;
@@ -1281,7 +1386,9 @@ nfs_fop_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, prebuf, postbuf, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,postbuf);
+                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
+
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -1304,7 +1411,7 @@ nfs_fop_fsync (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         nfs_fop_save_root_fd_ino (nfl, fd);
 
         STACK_WIND_COOKIE (frame, nfs_fop_fsync_cbk, xl, xl,
-                           xl->fops->fsync, fd, datasync);
+                           xl->fops->fsync, fd, datasync, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1319,7 +1426,8 @@ err:
 int32_t
 nfs_fop_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                   dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_readv_cbk_t         progcbk = NULL;
@@ -1328,7 +1436,7 @@ nfs_fop_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfs_fop_restore_root_ino (nfl, op_ret, stbuf, NULL, NULL, NULL);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, vector, count,
-                         stbuf, iobref);
+                         stbuf, iobref, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -1351,7 +1459,149 @@ nfs_fop_read (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         nfs_fop_save_root_fd_ino (nfl, fd);
 
         STACK_WIND_COOKIE (frame, nfs_fop_readv_cbk, xl, xl, xl->fops->readv,
-                           fd, size, offset, 0);
+                           fd, size, offset, 0, NULL);
+        ret = 0;
+err:
+        if (ret < 0) {
+                if (frame)
+                        nfs_stack_destroy (nfl, frame);
+        }
+
+        return ret;
+}
+
+int32_t
+nfs_fop_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                dict_t *xdata)
+{
+        struct nfs_fop_local    *nfl = NULL;
+        fop_lk_cbk_t         progcbk = NULL;
+
+        nfl_to_prog_data (nfl, progcbk, frame);
+
+        if (!op_ret)
+                fd_lk_insert_and_merge (nfl->fd, nfl->cmd, &nfl->flock);
+
+        fd_unref (nfl->fd);
+
+        if (progcbk)
+                progcbk (frame, cookie, this, op_ret, op_errno, flock, xdata);
+
+        nfs_stack_destroy (nfl, frame);
+        return 0;
+}
+
+
+int
+nfs_fop_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+            int cmd, struct gf_flock *flock, fop_lk_cbk_t cbk, void *local)
+{
+        call_frame_t            *frame = NULL;
+        int                     ret = -EFAULT;
+        struct nfs_fop_local    *nfl = NULL;
+
+        if ((!xl) || (!fd) || (!nfu))
+                return ret;
+
+        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
+        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+
+        nfl->cmd   = cmd;
+        nfl->fd    = fd_ref (fd);
+        nfl->flock = *flock;
+
+        STACK_WIND_COOKIE (frame, nfs_fop_lk_cbk, xl, xl, xl->fops->lk,
+                           fd, cmd, flock, NULL);
+        ret = 0;
+err:
+        if (ret < 0) {
+                if (frame)
+                        nfs_stack_destroy (nfl, frame);
+        }
+
+        return ret;
+}
+
+int32_t
+nfs_fop_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict,
+                dict_t *xdata)
+{
+        struct nfs_fop_local    *nfl = NULL;
+        fop_getxattr_cbk_t         progcbk = NULL;
+
+        nfl_to_prog_data (nfl, progcbk, frame);
+
+        if (progcbk)
+                progcbk (frame, cookie, this, op_ret, op_errno, dict, xdata);
+
+        nfs_stack_destroy (nfl, frame);
+        return 0;
+}
+
+
+int
+nfs_fop_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                  char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local)
+{
+        call_frame_t            *frame = NULL;
+        int                     ret = -EFAULT;
+        struct nfs_fop_local    *nfl = NULL;
+
+        if ((!xl) || (!loc) || (!nfu))
+                return ret;
+
+        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
+        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+
+        STACK_WIND_COOKIE (frame, nfs_fop_getxattr_cbk, xl, xl, xl->fops->getxattr,
+                           loc, name, NULL);
+        ret = 0;
+err:
+        if (ret < 0) {
+                if (frame)
+                        nfs_stack_destroy (nfl, frame);
+        }
+
+        return ret;
+}
+
+
+int32_t
+nfs_fop_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        struct nfs_fop_local    *nfl = NULL;
+        fop_setxattr_cbk_t         progcbk = NULL;
+
+        nfl_to_prog_data (nfl, progcbk, frame);
+
+        if (progcbk)
+                progcbk (frame, cookie, this, op_ret, op_errno, xdata);
+
+        nfs_stack_destroy (nfl, frame);
+        return 0;
+}
+
+
+int
+nfs_fop_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
+                  loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
+                  fop_setxattr_cbk_t cbk, void *local)
+{
+        call_frame_t            *frame = NULL;
+        int                     ret = -EFAULT;
+        struct nfs_fop_local    *nfl = NULL;
+
+        if ((!xl) || (!loc) || (!nfu))
+                return ret;
+
+        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
+        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+
+        STACK_WIND_COOKIE (frame, nfs_fop_setxattr_cbk, xl, xl, xl->fops->setxattr,
+                           loc, dict, flags, xdata);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1366,7 +1616,7 @@ err:
 int32_t
 nfs_fop_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_truncate_cbk_t      progcbk = NULL;
@@ -1374,7 +1624,8 @@ nfs_fop_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, prebuf, postbuf, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,postbuf);
+                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -1397,7 +1648,7 @@ nfs_fop_truncate (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
         nfs_fop_save_root_ino (nfl, loc);
 
         STACK_WIND_COOKIE  (frame, nfs_fop_truncate_cbk, xl, xl,
-                            xl->fops->truncate, loc, offset);
+                            xl->fops->truncate, loc, offset, NULL);
 
         ret = 0;
 err:
diff --git a/xlators/nfs/server/src/nfs-fops.h b/xlators/nfs/server/src/nfs-fops.h
index 48565215e..44e99c66b 100644
--- a/xlators/nfs/server/src/nfs-fops.h
+++ b/xlators/nfs/server/src/nfs-fops.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS_FOPS_H_
@@ -100,6 +91,10 @@ struct nfs_fop_local {
         char            newpath[NFS_NAME_MAX + 1];
         xlator_t        *nfsx;
         dict_t          *dictgfid;
+
+        fd_t            *fd;
+        int             cmd;
+        struct gf_flock flock;
 };
 
 extern struct nfs_fop_local *
@@ -116,7 +111,7 @@ nfs_fop_local_wipe (xlator_t *xl, struct nfs_fop_local *l);
                 nflocal = nfs_fop_local_init (nf);                      \
                 if (nflocal) {                                          \
                         nflocal->proglocal = plocal;                    \
-                        nflocal->progcbk = pcbk;                        \
+                        nflocal->progcbk = *VOID(&pcbk);                        \
                         nflocal->nfsx = nf;                             \
                         if (fram)                                       \
                                 ((call_frame_t *)fram)->local = nflocal;\
@@ -134,7 +129,7 @@ nfs_fop_local_wipe (xlator_t *xl, struct nfs_fop_local *l);
 
 #define nfs_fop_handle_local_init(fram,nfx, nfloc, cbck,prgloc,retval,lab)  \
         do {                                                                \
-                prog_data_to_nfl (nfx, nfloc, fram, cbck, prgloc);          \
+                prog_data_to_nfl (nfx, nfloc, fram, cbck, prgloc);      \
                 if (!nfloc) {                                               \
                         gf_log (GF_NFS,GF_LOG_ERROR,"Failed to init local");\
                         retval = -ENOMEM;                                   \
@@ -185,7 +180,7 @@ nfs_fop_write (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
 
 extern int
 nfs_fop_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-              int32_t flags, fd_t *fd, int32_t wbflags, fop_open_cbk_t cbk,
+              int32_t flags, fd_t *fd, fop_open_cbk_t cbk,
               void *local);
 
 extern int
@@ -236,4 +231,18 @@ nfs_fop_stat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
 extern int
 nfs_fop_access (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
                 int32_t accesstest, fop_access_cbk_t cbk, void *local);
+
+extern int
+nfs_fop_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+            int cmd, struct gf_flock *flock, fop_lk_cbk_t cbk, void *local);
+
+extern int
+nfs_fop_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                  char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local);
+
+extern int
+nfs_fop_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
+                  loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
+                  fop_setxattr_cbk_t cbk, void *local);
+
 #endif
diff --git a/xlators/nfs/server/src/nfs-generics.c b/xlators/nfs/server/src/nfs-generics.c
index 642879b8a..cb32b7f1b 100644
--- a/xlators/nfs/server/src/nfs-generics.c
+++ b/xlators/nfs/server/src/nfs-generics.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -151,7 +142,6 @@ nfs_truncate (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         return ret;
 }
 
-
 int
 nfs_read (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd, size_t size,
           off_t offset, fop_readv_cbk_t cbk, void *local)
@@ -159,6 +149,28 @@ nfs_read (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd, size_t size,
         return nfs_fop_read (nfsx, xl, nfu, fd, size, offset, cbk, local);
 }
 
+int
+nfs_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+        int cmd, struct gf_flock *flock, fop_lk_cbk_t cbk, void *local)
+{
+        return nfs_fop_lk ( nfsx, xl, nfu, fd, cmd, flock, cbk, local);
+}
+
+int
+nfs_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+              char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local)
+{
+        return nfs_fop_getxattr (nfsx, xl, nfu, loc, name, xdata, cbk, local);
+}
+
+int
+nfs_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
+              fop_setxattr_cbk_t cbk, void *local)
+{
+        return nfs_fop_setxattr (nfsx, xl, nfu, loc, dict, flags, xdata, cbk,
+                                local);
+}
 
 int
 nfs_fsync (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
@@ -187,7 +199,7 @@ nfs_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
                 return ret;
 
-        ret = nfs_inode_open (nfsx, xl, nfu, pathloc, flags, GF_OPEN_NOWB, cbk,
+        ret = nfs_inode_open (nfsx, xl, nfu, pathloc, flags, cbk,
                               local);
         return ret;
 }
@@ -330,3 +342,4 @@ nfs_opendir (xlator_t *nfsx, xlator_t *fopxl, nfs_user_t *nfu, loc_t *pathloc,
 
         return nfs_inode_opendir (nfsx, fopxl, nfu, pathloc, cbk, local);
 }
+
diff --git a/xlators/nfs/server/src/nfs-generics.h b/xlators/nfs/server/src/nfs-generics.h
index 237a0ad09..01876d68e 100644
--- a/xlators/nfs/server/src/nfs-generics.h
+++ b/xlators/nfs/server/src/nfs-generics.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS_GENERICS_H_
@@ -161,4 +152,17 @@ nfs_opendir (xlator_t *nfsx, xlator_t *fopxl, nfs_user_t *nfu, loc_t *pathloc,
 extern int
 nfs_access (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
             int32_t accesstest, fop_access_cbk_t cbk, void *local);
+extern int
+nfs_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+        int cmd, struct gf_flock *flock, fop_lk_cbk_t cbk, void *local);
+
+extern int
+nfs_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+              char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local);
+
+extern int
+nfs_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
+              fop_setxattr_cbk_t cbk, void *local);
+
 #endif
diff --git a/xlators/nfs/server/src/nfs-inodes.c b/xlators/nfs/server/src/nfs-inodes.c
index 4867dd71d..63d5e8a19 100644
--- a/xlators/nfs/server/src/nfs-inodes.c
+++ b/xlators/nfs/server/src/nfs-inodes.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -36,7 +27,7 @@
         do {                                                            \
                 nflocal = fram->local;                                  \
                 fram->local = nflocal->proglocal;                       \
-                pcbk = nflocal->progcbk;                                \
+                *VOID(&pcbk) = nflocal->progcbk;                                \
                 nfs_fop_local_wipe (nflocal->nfsx, nflocal);            \
         } while (0)                                                     \
 
@@ -57,10 +48,10 @@ nfl_inodes_init (struct nfs_fop_local *nfl, inode_t *inode, inode_t *parent,
                 nfl->newparent = inode_ref (newparent);
 
         if (name)
-                strcpy (nfl->path, name);
+                strncpy (nfl->path, name, NFS_NAME_MAX);
 
         if (newname)
-                strcpy (nfl->newpath, newname);
+                strncpy (nfl->newpath, newname, NFS_NAME_MAX);
 
         return;
 }
@@ -70,7 +61,7 @@ int32_t
 nfs_inode_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode
                       , struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent)
+                      struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = frame->local;
         fop_create_cbk_t        progcbk = NULL;
@@ -90,7 +81,7 @@ do_not_link:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, fd, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         if (linked_inode) {
                 inode_lookup (linked_inode);
@@ -142,7 +133,7 @@ int32_t
 nfs_inode_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = frame->local;
         fop_mkdir_cbk_t         progcbk = NULL;
@@ -157,7 +148,7 @@ do_not_link:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         if (linked_inode) {
                 inode_lookup (linked_inode);
@@ -193,7 +184,7 @@ err:
 
 int32_t
 nfs_inode_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd)
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
 
         struct nfs_fop_local    *nfl = NULL;
@@ -209,14 +200,14 @@ nfs_inode_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 */
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd);
+                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
 
 int
 nfs_inode_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                int32_t flags, int32_t wbflags, fop_open_cbk_t cbk, void *local)
+                int32_t flags, fop_open_cbk_t cbk, void *local)
 {
         struct nfs_fop_local    *nfl = NULL;
         fd_t                    *newfd = NULL;
@@ -233,7 +224,7 @@ nfs_inode_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
         }
 
         nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, fd_err);
-        ret = nfs_fop_open (nfsx, xl, nfu, loc, flags, newfd, wbflags,
+        ret = nfs_fop_open (nfsx, xl, nfu, loc, flags, newfd,
                             nfs_inode_open_cbk, nfl);
 
         if (ret < 0)
@@ -255,7 +246,8 @@ int32_t
 nfs_inode_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *buf,
                       struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent)
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_rename_cbk_t        progcbk = NULL;
@@ -272,7 +264,7 @@ do_not_link:
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, buf,
                          preoldparent, postoldparent, prenewparent,
-                         postnewparent);
+                         postnewparent, xdata);
         return 0;
 }
 
@@ -305,7 +297,7 @@ int32_t
 nfs_inode_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_link_cbk_t          progcbk = NULL;
@@ -321,7 +313,7 @@ do_not_link:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         if (linked_inode) {
                 inode_lookup (linked_inode);
@@ -358,7 +350,7 @@ err:
 int32_t
 nfs_inode_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                      struct iatt *postparent)
+                      struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_unlink_cbk_t        progcbk = NULL;
@@ -375,7 +367,7 @@ do_not_unlink:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                         postparent);
+                         postparent, xdata);
         return 0;
 }
 
@@ -406,7 +398,7 @@ err:
 int32_t
 nfs_inode_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_rmdir_cbk_t         progcbk = NULL;
@@ -423,7 +415,7 @@ do_not_unlink:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                        postparent);
+                         postparent, xdata);
 
         return 0;
 }
@@ -456,7 +448,7 @@ int32_t
 nfs_inode_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_mknod_cbk_t         progcbk = NULL;
@@ -473,7 +465,7 @@ do_not_link:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         if (linked_inode) {
                 inode_lookup (linked_inode);
@@ -513,7 +505,7 @@ int32_t
 nfs_inode_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, inode_t *inode,
                        struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent)
+                       struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_symlink_cbk_t       progcbk = NULL;
@@ -529,7 +521,7 @@ do_not_link:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         if (linked_inode) {
                 inode_lookup (linked_inode);
@@ -565,21 +557,20 @@ err:
 
 int32_t
 nfs_inode_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, fd_t *fd)
+                       int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
 
         struct nfs_fop_local    *nfl = NULL;
         fop_open_cbk_t          progcbk = NULL;
 
-        if ((op_ret == -1) && (fd))
-                fd_unref (fd);
-        else
+        if (op_ret != -1)
                 fd_bind (fd);
 
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
 
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd);
+                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
+
         return 0;
 }
 
diff --git a/xlators/nfs/server/src/nfs-inodes.h b/xlators/nfs/server/src/nfs-inodes.h
index 12c8eec0d..ba7a57124 100644
--- a/xlators/nfs/server/src/nfs-inodes.h
+++ b/xlators/nfs/server/src/nfs-inodes.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS_INODES_H_
@@ -48,7 +39,7 @@ nfs_inode_mkdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
 
 extern int
 nfs_inode_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                int32_t flags, int32_t wbflags, fop_open_cbk_t cbk,
+                int32_t flags, fop_open_cbk_t cbk,
                 void *local);
 
 extern int
diff --git a/xlators/nfs/server/src/nfs-mem-types.h b/xlators/nfs/server/src/nfs-mem-types.h
index 57f5f90b2..450b6f2fe 100644
--- a/xlators/nfs/server/src/nfs-mem-types.h
+++ b/xlators/nfs/server/src/nfs-mem-types.h
@@ -2,19 +2,10 @@
    Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -37,12 +28,25 @@ enum gf_nfs_mem_types_ {
         gf_nfs_mt_entry3,
         gf_nfs_mt_entryp3,
         gf_nfs_mt_nfs3_fd_entry,
+        gf_nfs_mt_nfs3_fh,
         gf_nfs_mt_nfs_initer_list,
         gf_nfs_mt_xlator_t,
         gf_nfs_mt_list_head,
         gf_nfs_mt_mnt3_resolve,
         gf_nfs_mt_mnt3_export,
+        gf_nfs_mt_int,
+        gf_nfs_mt_mountres3,
+        gf_nfs_mt_mountstat3,
         gf_nfs_mt_inode_q,
+        gf_nfs_mt_nlm4_state,
+        gf_nfs_mt_nlm4_cm,
+        gf_nfs_mt_nlm4_fde,
+        gf_nfs_mt_nlm4_nlmclnt,
+        gf_nfs_mt_nlm4_share,
+        gf_nfs_mt_aux_gids,
+        gf_nfs_mt_inode_ctx,
+        gf_nfs_mt_auth_spec,
+        gf_nfs_mt_arr,
         gf_nfs_mt_end
 };
 #endif
diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index d92a0e126..75c8fe44e 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /* This is the primary translator source for NFS.
@@ -40,6 +31,208 @@
 #include "nfs3.h"
 #include "nfs-mem-types.h"
 #include "nfs3-helpers.h"
+#include "nlm4.h"
+#include "options.h"
+#include "acl3.h"
+#include "rpc-drc.h"
+
+#define STRINGIFY(val) #val
+#define TOSTRING(val) STRINGIFY(val)
+
+#define OPT_SERVER_AUX_GIDS             "nfs.server-aux-gids"
+#define OPT_SERVER_GID_CACHE_TIMEOUT    "nfs.server.aux-gid-timeout"
+
+/* TODO: DATADIR should be based on configure's $(localstatedir) */
+#define DATADIR                         "/var/lib/glusterd"
+#define NFS_DATADIR                     DATADIR "/nfs"
+
+/* Forward declaration */
+int nfs_add_initer (struct list_head *list, nfs_version_initer_t init);
+
+static int
+nfs_init_version (xlator_t *this, nfs_version_initer_t init)
+{
+        int                       ret       = -1;
+        struct nfs_initer_list    *version  = NULL;
+        struct nfs_initer_list    *tmp      = NULL;
+        rpcsvc_program_t          *prog     = NULL;
+        struct list_head          *versions = NULL;
+        struct nfs_state          *nfs      = NULL;
+        gf_boolean_t              found     = _gf_false;
+
+        if ((!this) || (!this->private) || (!init))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        ret = nfs_add_initer (&nfs->versions, init);
+        if (ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Failed to add protocol initializer");
+                goto err;
+        }
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (version->init == init) {
+                        prog = init(this);
+                        if (!prog) {
+                                ret = -1;
+                                goto err;
+                        }
+                        version->program = prog;
+                        found = _gf_true;
+                        break;
+                }
+        }
+
+        /* program not added */
+        if (!found) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Program: %s NOT found", prog->progname);
+                goto err;
+        }
+
+        /* Check if nfs.port is configured */
+        if (nfs->override_portnum)
+                prog->progport = nfs->override_portnum;
+
+        gf_log (GF_NFS, GF_LOG_DEBUG, "Starting program: %s", prog->progname);
+
+        ret = rpcsvc_program_register (nfs->rpcsvc, prog);
+        if (ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR, "Program: %s init failed",
+                                                      prog->progname);
+                goto err;
+        }
+
+        /* Registration with portmapper is disabled, Nothing to do */
+        if (!nfs->register_portmap)
+                goto err;
+
+        ret = rpcsvc_program_register_portmap (prog, prog->progport);
+        if (ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Program  %s registration failed",
+                                prog->progname);
+                goto err;
+        }
+        ret = 0; /* All well */
+err:
+        return ret;
+}
+
+static int
+nfs_deinit_version (struct nfs_state *nfs, nfs_version_initer_t init)
+{
+        int                       ret       = -1;
+        struct nfs_initer_list    *version  = NULL;
+        struct nfs_initer_list    *tmp      = NULL;
+        rpcsvc_program_t          *prog     = NULL;
+        struct list_head          *versions = NULL;
+
+        if ((!nfs) || (!init))
+                return (-1);
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (version->init == init) {
+                        prog = version->program;
+                        ret = rpcsvc_program_unregister (nfs->rpcsvc, prog);
+                        if (ret != 0)
+                                return (-1);
+                        list_del (&version->list);
+                        GF_FREE (version);
+                        return (0);
+                }
+        }
+
+        return (-1);
+}
+
+static int
+nfs_reconfigure_acl3 (xlator_t *this)
+{
+        struct nfs_state          *nfs     = NULL;
+
+        if ((!this) || (!this->private))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* ACL is enabled */
+        if (nfs->enable_acl)
+                return nfs_init_version (this, acl3svc_init);
+
+        /* ACL is disabled */
+        return nfs_deinit_version (nfs, acl3svc_init);
+}
+
+static int
+nfs_reconfigure_nlm4 (xlator_t *this)
+{
+        struct nfs_state          *nfs     = NULL;
+
+        if ((!this) || (!this->private))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* NLM is enabled */
+        if (nfs->enable_nlm)
+                return nfs_init_version (this, nlm4svc_init);
+
+        /* NLM is disabled */
+        return nfs_deinit_version (nfs, nlm4svc_init);
+}
+
+static int
+nfs_program_register_portmap_all (struct nfs_state *nfs)
+{
+        struct list_head                *versions = NULL;
+        struct nfs_initer_list          *version = NULL;
+        struct nfs_initer_list          *tmp = NULL;
+        rpcsvc_program_t                *prog = NULL;
+
+        if (nfs == NULL)
+                return (-1);
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (prog == NULL)
+                        continue;
+                if (nfs->override_portnum)
+                        prog->progport = nfs->override_portnum;
+                (void) rpcsvc_program_register_portmap (prog, prog->progport);
+        }
+
+        return (0);
+}
+
+static int
+nfs_program_unregister_portmap_all (struct nfs_state *nfs)
+{
+        struct list_head                *versions = NULL;
+        struct nfs_initer_list          *version = NULL;
+        struct nfs_initer_list          *tmp = NULL;
+        rpcsvc_program_t                *prog = NULL;
+
+        if (nfs == NULL)
+                return (-1);
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (prog == NULL)
+                        continue;
+                (void) rpcsvc_program_unregister_portmap (prog);
+        }
+
+        return (0);
+}
 
 /* Every NFS version must call this function with the init function
  * for its particular version.
@@ -117,7 +310,7 @@ nfs_init_versions (struct nfs_state *nfs, xlator_t *this)
                         ret = -1;
                         goto err;
                 }
-//                prog->actorxl = this;
+
                 version->program = prog;
                 if (nfs->override_portnum)
                         prog->progport = nfs->override_portnum;
@@ -126,17 +319,21 @@ nfs_init_versions (struct nfs_state *nfs, xlator_t *this)
 
                 ret = rpcsvc_program_register (nfs->rpcsvc, prog);
                 if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Program init failed");
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Program: %s init failed",
+                                                      prog->progname);
                         goto err;
                 }
-                if (rpcsvc_register_portmap_enabled(nfs->rpcsvc)) {
+                if (nfs->register_portmap) {
                         ret = rpcsvc_program_register_portmap (prog,
                                                                prog->progport);
                         if (ret == -1) {
-                                gf_log (GF_NFS, GF_LOG_ERROR, "Program registration failed");
+                                gf_log (GF_NFS, GF_LOG_ERROR,
+                                        "Program  %s registration failed",
+                                        prog->progname);
                                 goto err;
                         }
                 }
+
         }
 
         ret = 0;
@@ -153,25 +350,43 @@ nfs_add_all_initiators (struct nfs_state *nfs)
         /* Add the initializers for all versions. */
         ret = nfs_add_initer (&nfs->versions, mnt3svc_init);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                        " initializer");
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "MOUNT3 protocol initializer");
                 goto ret;
         }
 
         ret = nfs_add_initer (&nfs->versions, mnt1svc_init);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                        " initializer");
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "MOUNT1 protocol initializer");
                 goto ret;
         }
 
         ret = nfs_add_initer (&nfs->versions, nfs3svc_init);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                        " initializer");
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "NFS3 protocol initializer");
                 goto ret;
         }
 
+        if (nfs->enable_nlm == _gf_true) {
+                ret = nfs_add_initer (&nfs->versions, nlm4svc_init);
+                if (ret == -1) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
+                                " initializer");
+                        goto ret;
+                }
+        }
+
+        if (nfs->enable_acl == _gf_true) {
+                ret = nfs_add_initer (&nfs->versions, acl3svc_init);
+                if (ret == -1) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "ACL protocol initializer");
+                        goto ret;
+                }
+        }
+
         ret = 0;
 ret:
         return ret;
@@ -500,14 +715,16 @@ nfs_init_state (xlator_t *this)
         unsigned int            fopspoolsize = 0;
         char                    *optstr = NULL;
         gf_boolean_t            boolt = _gf_false;
+        struct stat             stbuf = {0,};
 
         if (!this)
                 return NULL;
 
-        if ((!this->children) || (!this->children->xlator)) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "nfs must have at least one"
-                        " child subvolume");
-                return NULL;
+        if (!this->children) {
+                gf_log (GF_NFS, GF_LOG_INFO,
+                        "NFS is manually disabled: Exiting");
+                /* Nothing for nfs process to do, exit cleanly */
+                kill (getpid (), SIGTERM);
         }
 
         nfs = GF_CALLOC (1, sizeof (*nfs), gf_nfs_mt_nfs_state);
@@ -562,6 +779,20 @@ nfs_init_state (xlator_t *this)
                         nfs->dynamicvolumes = GF_NFS_DVM_ON;
         }
 
+        nfs->enable_nlm = _gf_true;
+        ret = dict_get_str_boolean (this->options, "nfs.nlm", _gf_true);
+        if (ret == _gf_false) {
+                gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually disabled");
+                nfs->enable_nlm = _gf_false;
+        }
+
+        nfs->enable_acl = _gf_true;
+        ret = dict_get_str_boolean (this->options, "nfs.acl", _gf_true);
+        if (ret == _gf_false) {
+                gf_log (GF_NFS, GF_LOG_INFO, "ACL is manually disabled");
+                nfs->enable_acl = _gf_false;
+        }
+
         nfs->enable_ino32 = 0;
         if (dict_get (this->options, "nfs.enable-ino32")) {
                 ret = dict_get_str (this->options, "nfs.enable-ino32",
@@ -624,6 +855,34 @@ nfs_init_state (xlator_t *this)
                 }
         }
 
+        nfs->mount_udp = 0;
+        if (dict_get(this->options, "nfs.mount-udp")) {
+                ret = dict_get_str (this->options, "nfs.mount-udp", &optstr);
+                if (ret == -1) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
+                        goto free_foppool;
+                }
+
+                ret = gf_string2boolean (optstr, &boolt);
+                if (ret < 0) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse bool "
+                                "string");
+                        goto free_foppool;
+                }
+
+                if (boolt == _gf_true)
+                        nfs->mount_udp = 1;
+        }
+
+        nfs->rmtab = gf_strdup (NFS_DATADIR "/rmtab");
+        if (dict_get(this->options, "nfs.mount-rmtab")) {
+                ret = dict_get_str (this->options, "nfs.mount-rmtab", &nfs->rmtab);
+                if (ret == -1) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
+                        goto free_foppool;
+                }
+        }
+
         /* support both options rpc-auth.ports.insecure and
          * rpc-auth-allow-insecure for backward compatibility
          */
@@ -684,15 +943,34 @@ nfs_init_state (xlator_t *this)
                 }
         }
 
-        nfs->rpcsvc =  rpcsvc_init (this, this->ctx, this->options);
+        GF_OPTION_INIT (OPT_SERVER_AUX_GIDS, nfs->server_aux_gids,
+                        bool, free_foppool);
+        GF_OPTION_INIT (OPT_SERVER_GID_CACHE_TIMEOUT, nfs->server_aux_gids_max_age,
+                        uint32, free_foppool);
+
+        if (gid_cache_init(&nfs->gid_cache, nfs->server_aux_gids_max_age) < 0) {
+                gf_log(GF_NFS, GF_LOG_ERROR, "Failed to initialize group cache.");
+                goto free_foppool;
+        }
+
+        if (stat("/sbin/rpc.statd", &stbuf) == -1) {
+                gf_log (GF_NFS, GF_LOG_WARNING, "/sbin/rpc.statd not found. "
+                        "Disabling NLM");
+                nfs->enable_nlm = _gf_false;
+        }
+
+        nfs->rpcsvc =  rpcsvc_init (this, this->ctx, this->options, 0);
         if (!nfs->rpcsvc) {
                 ret = -1;
                 gf_log (GF_NFS, GF_LOG_ERROR, "RPC service init failed");
                 goto free_foppool;
         }
 
+        nfs->register_portmap = rpcsvc_register_portmap_enabled (nfs->rpcsvc);
+
         this->private = (void *)nfs;
         INIT_LIST_HEAD (&nfs->versions);
+        nfs->generation = 1965;
 
         ret = 0;
 
@@ -711,6 +989,243 @@ free_rpcsvc:
         return nfs;
 }
 
+int
+nfs_drc_init (xlator_t *this)
+{
+        int       ret     = -1;
+        rpcsvc_t *svc     = NULL;
+
+        svc = ((struct nfs_state *)(this->private))->rpcsvc;
+        if (!svc)
+                goto out;
+
+        ret = rpcsvc_drc_init (svc, this->options);
+
+ out:
+        return ret;
+}
+
+int
+nfs_reconfigure_state (xlator_t *this, dict_t *options)
+{
+        int                 ret = 0;
+        int                 keyindx = 0;
+        char                *optstr = NULL;
+        gf_boolean_t        optbool;
+        uint32_t            optuint32;
+        struct nfs_state    *nfs = NULL;
+        char                *blacklist_keys[] = {
+                                        "nfs.port",
+                                        "nfs.transport-type",
+                                        "nfs.mem-factor",
+                                        NULL};
+
+        GF_VALIDATE_OR_GOTO (GF_NFS, this, out);
+        GF_VALIDATE_OR_GOTO (GF_NFS, this->private, out);
+        GF_VALIDATE_OR_GOTO (GF_NFS, options, out);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* Black listed options can't be reconfigured, they need
+         * NFS to be restarted. There are two cases 1. SET 2. UNSET.
+         * 1. SET */
+        while (blacklist_keys[keyindx]) {
+                if (dict_get (options, blacklist_keys[keyindx])) {
+                        gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Reconfiguring %s needs NFS restart",
+                                blacklist_keys[keyindx]);
+                        goto out;
+                }
+                keyindx ++;
+        }
+
+        /* UNSET for nfs.mem-factor */
+        if ((!dict_get (options, "nfs.mem-factor")) &&
+            (nfs->memfactor != GF_NFS_DEFAULT_MEMFACTOR)) {
+                gf_log (GF_NFS, GF_LOG_INFO,
+                        "Reconfiguring nfs.mem-factor needs NFS restart");
+                goto out;
+        }
+
+        /* UNSET for nfs.port */
+        if ((!dict_get (options, "nfs.port")) &&
+            (nfs->override_portnum)) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "Reconfiguring nfs.port needs NFS restart");
+                goto out;
+        }
+
+        /* reconfig nfs.mount-rmtab */
+        optstr = NFS_DATADIR "/rmtab";
+        if (dict_get (options, "nfs.mount-rmtab")) {
+                ret = dict_get_str (options, "nfs.mount-rmtab", &optstr);
+                if (ret < 0) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to read "
+                                "reconfigured option: nfs.mount-rmtab");
+                        goto out;
+                }
+                gf_path_strip_trailing_slashes (optstr);
+        }
+        if (strcmp (nfs->rmtab, optstr) != 0) {
+                mount_rewrite_rmtab (nfs->mstate, optstr);
+                gf_log (GF_NFS, GF_LOG_INFO,
+                                "Reconfigured nfs.mount-rmtab path: %s",
+                                nfs->rmtab);
+        }
+
+        GF_OPTION_RECONF (OPT_SERVER_AUX_GIDS, optbool,
+                                               options, bool, out);
+        if (nfs->server_aux_gids != optbool) {
+                nfs->server_aux_gids = optbool;
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured %s with value %d",
+                               OPT_SERVER_AUX_GIDS, optbool);
+        }
+
+        GF_OPTION_RECONF (OPT_SERVER_GID_CACHE_TIMEOUT, optuint32,
+                                               options, uint32, out);
+        if (nfs->server_aux_gids_max_age != optuint32) {
+                nfs->server_aux_gids_max_age = optuint32;
+                gid_cache_reconf (&nfs->gid_cache, optuint32);
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured %s with value %d",
+                               OPT_SERVER_GID_CACHE_TIMEOUT, optuint32);
+        }
+
+        /* reconfig nfs.dynamic-volumes */
+        ret = dict_get_str_boolean (options, "nfs.dynamic-volumes",
+                                             GF_NFS_DVM_OFF);
+        switch (ret) {
+        case GF_NFS_DVM_ON:
+        case GF_NFS_DVM_OFF:
+                optbool = ret;
+                break;
+        default:
+                optbool = GF_NFS_DVM_OFF;
+                break;
+        }
+        if (nfs->dynamicvolumes != optbool) {
+                nfs->dynamicvolumes = optbool;
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured nfs.dynamic-volumes"
+                                            " with value %d", optbool);
+        }
+
+        optbool = _gf_false;
+        if (dict_get (options, "nfs.enable-ino32")) {
+                ret = dict_get_str_boolean (options, "nfs.enable-ino32",
+                                                      _gf_false);
+                if (ret < 0) {
+                        gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Failed to read reconfigured option: "
+                                "nfs.enable-ino32");
+                        goto out;
+                }
+                optbool = ret;
+        }
+        if (nfs->enable_ino32 != optbool) {
+                nfs->enable_ino32 = optbool;
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured nfs.enable-ino32"
+                                            " with value %d", optbool);
+        }
+
+        /* nfs.nlm is enabled by default */
+        ret = dict_get_str_boolean (options, "nfs.nlm", _gf_true);
+        if (ret < 0) {
+                optbool = _gf_true;
+        } else {
+                optbool = ret;
+        }
+        if (nfs->enable_nlm != optbool) {
+                gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually %s",
+                                (optbool ? "enabled":"disabled"));
+                nfs->enable_nlm = optbool;
+                nfs_reconfigure_nlm4 (this);
+        }
+
+        /* nfs.acl is enabled by default */
+        ret = dict_get_str_boolean (options, "nfs.acl", _gf_true);
+        if (ret < 0) {
+                optbool = _gf_true;
+        } else {
+                optbool = ret;
+        }
+        if (nfs->enable_acl != optbool) {
+                gf_log (GF_NFS, GF_LOG_INFO, "ACL is manually %s",
+                                (optbool ? "enabled":"disabled"));
+                nfs->enable_acl = optbool;
+                nfs_reconfigure_acl3 (this);
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+/*
+ * reconfigure() for NFS server xlator.
+ */
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int                      ret = 0;
+        struct nfs_state         *nfs = NULL;
+        gf_boolean_t             regpmap = _gf_true;
+
+        if ((!this) || (!this->private) || (!options))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* Reconfigure nfs options */
+        ret = nfs_reconfigure_state(this, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "nfs reconfigure state failed");
+                return (-1);
+        }
+
+        /* Reconfigure nfs3 options */
+        ret = nfs3_reconfigure_state(this, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "nfs3 reconfigure state failed");
+                return (-1);
+        }
+
+        /* Reconfigure mount options */
+        ret = mount_reconfigure_state(this, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "mount reconfigure state failed");
+                return (-1);
+        }
+
+        /* Reconfigure rpc layer */
+        ret = rpcsvc_reconfigure_options (nfs->rpcsvc, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "rpcsvc reconfigure options failed");
+                return (-1);
+        }
+        regpmap = rpcsvc_register_portmap_enabled(nfs->rpcsvc);
+        if (nfs->register_portmap != regpmap) {
+                nfs->register_portmap = regpmap;
+                if (regpmap) {
+                        (void) nfs_program_register_portmap_all (nfs);
+                } else {
+                        (void) nfs_program_unregister_portmap_all (nfs);
+                }
+        }
+
+        /* Reconfigure drc */
+        ret = rpcsvc_drc_reconfigure (nfs->rpcsvc, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "rpcsvc DRC reconfigure failed");
+                return (-1);
+        }
+
+        return (0);
+}
 
 int
 init (xlator_t *this) {
@@ -747,6 +1262,13 @@ init (xlator_t *this) {
                 goto err;
         }
 
+        ret = nlm4_init_state (this);
+        if (ret == -1) {
+                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init NLM"
+                        "state");
+                goto err;
+        }
+
         ret = nfs_init_versions (nfs, this);
         if (ret == -1) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "Failed to initialize "
@@ -759,7 +1281,9 @@ init (xlator_t *this) {
                 goto err;
         }
 
-        gf_log (GF_NFS, GF_LOG_INFO, "NFS service started");
+        ret = nfs_drc_init (this);
+        if (ret == 0)
+                gf_log (GF_NFS, GF_LOG_INFO, "NFS service started");
 err:
 
         return ret;
@@ -770,24 +1294,26 @@ int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
         xlator_t                *subvol = NULL;
+        struct nfs_state        *priv   = NULL;
 
         subvol = (xlator_t *)data;
 
         gf_log (GF_NFS, GF_LOG_TRACE, "Notification received: %d",
                 event);
-        switch (event)
-        {
-                case GF_EVENT_CHILD_UP:
-                {
-                        nfs_startup_subvolume (this, subvol);
-                        break;
-                }
 
-                case GF_EVENT_PARENT_UP:
-                {
-                        default_notify (this, GF_EVENT_PARENT_UP, data);
-                        break;
-                }
+        switch (event) {
+        case GF_EVENT_CHILD_UP:
+                nfs_startup_subvolume (this, subvol);
+                break;
+
+        case GF_EVENT_CHILD_MODIFIED:
+                priv = this->private;
+                ++(priv->generation);
+                break;
+
+        case GF_EVENT_PARENT_UP:
+                default_notify (this, GF_EVENT_PARENT_UP, data);
+                break;
         }
 
         return 0;
@@ -809,14 +1335,138 @@ fini (xlator_t *this)
 int32_t
 nfs_forget (xlator_t *this, inode_t *inode)
 {
+        uint64_t                 ctx    = 0;
+        struct nfs_inode_ctx    *ictx   = NULL;
+
+        if (inode_ctx_del (inode, this, &ctx))
+                return -1;
+
+        ictx = (struct nfs_inode_ctx *)ctx;
+        GF_FREE (ictx);
+
         return 0;
 }
 
+gf_boolean_t
+_nfs_export_is_for_vol (char *exname, char *volname)
+{
+        gf_boolean_t    ret = _gf_false;
+        char            *tmp = NULL;
+
+        tmp = exname;
+        if (tmp[0] == '/')
+                tmp++;
+
+        if (!strcmp (tmp, volname))
+                ret = _gf_true;
+
+        return ret;
+}
+
+int
+nfs_priv_to_dict (xlator_t *this, dict_t *dict)
+{
+        int                     ret = -1;
+        struct nfs_state        *priv = NULL;
+        struct mountentry       *mentry = NULL;
+        char                    *volname = NULL;
+        char                    key[1024] = {0,};
+        int                     count = 0;
+
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, dict, out);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not get volname");
+                goto out;
+        }
+
+        list_for_each_entry (mentry, &priv->mstate->mountlist, mlist) {
+                if (!_nfs_export_is_for_vol (mentry->exname, volname))
+                        continue;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "client%d.hostname", count);
+                ret = dict_set_str (dict, key, mentry->hostname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error writing hostname to dict");
+                        goto out;
+                }
+
+                /* No connection data available yet in nfs server.
+                 * Hence, setting to 0 to prevent cli failing
+                 */
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "client%d.bytesread", count);
+                ret = dict_set_uint64 (dict, key, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error writing bytes read to dict");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "client%d.byteswrite", count);
+                ret = dict_set_uint64 (dict, key, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error writing bytes write to dict");
+                        goto out;
+                }
+
+                count++;
+        }
+
+        ret = dict_set_int32 (dict, "clientcount", count);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error writing client count to dict");
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+extern int32_t
+nlm_priv (xlator_t *this);
+
+int32_t
+nfs_priv (xlator_t *this)
+{
+        int32_t ret = -1;
+
+        /* DRC needs the global drc structure, xl is of no use to it. */
+        ret = rpcsvc_drc_priv (((struct nfs_state *)(this->private))->rpcsvc->drc);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Statedump of DRC failed");
+                goto out;
+        }
+
+        ret = nlm_priv (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Statedump of NLM failed");
+                goto out;
+        }
+ out:
+        return ret;
+}
+
+
 struct xlator_cbks cbks = {
         .forget      = nfs_forget,
 };
 
-struct xlator_fops fops = { };
+struct xlator_fops fops;
+
+struct xlator_dumpops dumpops = {
+        .priv           = nfs_priv,
+        .priv_to_dict   = nfs_priv_to_dict,
+};
 
 /* TODO: If needed, per-volume options below can be extended to be export
 + * specific also because after export-dir is introduced, a volume is not
@@ -826,29 +1476,53 @@ struct xlator_fops fops = { };
 struct volume_options options[] = {
         { .key  = {"nfs3.read-size"},
           .type = GF_OPTION_TYPE_SIZET,
-          .description = "Size in which the client should issue read requests"
-                         " to the Gluster NFSv3 server. Must be a multiple of"
-                         " 4KiB."
+          .min  = GF_NFS3_RTMIN,
+          .max  = GF_NFS3_RTMAX,
+          .default_value = TOSTRING(GF_NFS3_RTPREF),
+          .description = "Size in which the client should issue read requests "
+                         "to the Gluster NFSv3 server. Must be a multiple of "
+                         "4KB (4096). Min and Max supported values are 4KB "
+                         "(4096) and 1MB (1048576) respectively. If the "
+                         "specified value is within the supported range but "
+                         "not a multiple of 4096, it is rounded up to the "
+                         "nearest multiple of 4096."
         },
         { .key  = {"nfs3.write-size"},
           .type = GF_OPTION_TYPE_SIZET,
-          .description = "Size in which the client should issue write requests"
-                         " to the Gluster NFSv3 server. Must be a multiple of"
-                         " 4KiB."
+          .min  = GF_NFS3_WTMIN,
+          .max  = GF_NFS3_WTMAX,
+          .default_value = TOSTRING(GF_NFS3_WTPREF),
+          .description = "Size in which the client should issue write requests "
+                         "to the Gluster NFSv3 server. Must be a multiple of "
+                         "1KB (1024). Min and Max supported values are "
+                         "4KB (4096) and 1MB(1048576) respectively. If the "
+                         "specified value is within the supported range but "
+                         "not a multiple of 4096, it is rounded up to the "
+                         "nearest multiple of 4096."
         },
         { .key  = {"nfs3.readdir-size"},
           .type = GF_OPTION_TYPE_SIZET,
+          .min  = GF_NFS3_DTMIN,
+          .max  = GF_NFS3_DTMAX,
+          .default_value = TOSTRING(GF_NFS3_DTPREF),
           .description = "Size in which the client should issue directory "
-                         " reading requests."
+                         "reading requests to the Gluster NFSv3 server. Must "
+                         "be a multiple of 1KB (1024). Min and Max supported "
+                         "values are 4KB (4096) and 1MB (1048576) respectively."
+                         "If the specified value is within the supported range "
+                         "but not a multiple of 4096, it is rounded up to the "
+                         "nearest multiple of 4096."
         },
         { .key  = {"nfs3.*.volume-access"},
           .type = GF_OPTION_TYPE_STR,
           .value = {"read-only", "read-write"},
+          .default_value = "read-write",
           .description = "Type of access desired for this subvolume: "
                          " read-only, read-write(default)"
         },
         { .key  = {"nfs3.*.trusted-write"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "On an UNSTABLE write from client, return STABLE flag"
                          " to force client to not send a COMMIT request. In "
                          "some environments, combined with a replicated "
@@ -863,6 +1537,7 @@ struct volume_options options[] = {
         },
         { .key  = {"nfs3.*.trusted-sync"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "All writes and COMMIT requests are treated as async."
                          " This implies that no write requests are guaranteed"
                          " to be on server disks when the write reply is "
@@ -872,25 +1547,40 @@ struct volume_options options[] = {
         },
         { .key  = {"nfs3.*.export-dir"},
           .type = GF_OPTION_TYPE_PATH,
+          .default_value = "",
           .description = "By default, all subvolumes of nfs are exported as "
                          "individual exports. There are cases where a "
                          "subdirectory or subdirectories in the volume need to "
                          "be exported separately. This option can also be used "
                          "in conjunction with nfs3.export-volumes option to "
                          "restrict exports only to the subdirectories specified"
-                         " through this option. Must be an absolute path."
+                         " through this option. Must be an absolute path. Along"
+                         " with path allowed list of IPs/hostname can be "
+                         "associated with each subdirectory. If provided "
+                         "connection will allowed only from these IPs. By "
+                         "default connections from all IPs are allowed. "
+                         "Format: <dir>[(hostspec[|hostspec|...])][,...]. Where"
+                         " hostspec can be an IP address, hostname or an IP "
+                         "range in CIDR notation. "
+                         "e.g. /foo(192.168.1.0/24|host1|10.1.1.8),/host2."
+                         " NOTE: Care must be taken while configuring this "
+                         "option as invalid entries and/or unreachable DNS "
+                         "servers can introduce unwanted delay in all the mount"
+                         " calls."
         },
         { .key  = {"nfs3.export-dirs"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "By default, all subvolumes of nfs are exported as "
                          "individual exports. There are cases where a "
                          "subdirectory or subdirectories in the volume need to "
                          "be exported separately. Enabling this option allows "
                          "any directory on a volumes to be exported separately."
-                         " Directory exports are enabled by default."
+                         "Directory exports are enabled by default."
         },
         { .key  = {"nfs3.export-volumes"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "Enable or disable exporting whole volumes, instead "
                          "if used in conjunction with nfs3.export-dir, can "
                          "allow setting up only subdirectories as exports. On "
@@ -898,6 +1588,7 @@ struct volume_options options[] = {
         },
         { .key  = {"rpc-auth.auth-unix"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "Disable or enable the AUTH_UNIX authentication type."
                          "Must always be enabled for better interoperability."
                          "However, can be disabled if needed. Enabled by"
@@ -905,14 +1596,16 @@ struct volume_options options[] = {
         },
         { .key  = {"rpc-auth.auth-null"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "Disable or enable the AUTH_NULL authentication type."
                          "Must always be enabled. This option is here only to"
                          " avoid unrecognized option warnings"
         },
         { .key  = {"rpc-auth.auth-unix.*"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "Disable or enable the AUTH_UNIX authentication type "
-                         "for a particular exported volume over-riding defaults"
+                         "for a particular exported volume overriding defaults"
                          " and general setting for AUTH_UNIX scheme. Must "
                          "always be enabled for better interoperability."
                          "However, can be disabled if needed. Enabled by"
@@ -920,8 +1613,9 @@ struct volume_options options[] = {
         },
         { .key  = {"rpc-auth.auth-unix.*.allow"},
           .type = GF_OPTION_TYPE_STR,
+          .default_value = "on",
           .description = "Disable or enable the AUTH_UNIX authentication type "
-                         "for a particular exported volume over-riding defaults"
+                         "for a particular exported volume overriding defaults"
                          " and general setting for AUTH_UNIX scheme. Must "
                          "always be enabled for better interoperability."
                          "However, can be disabled if needed. Enabled by"
@@ -929,42 +1623,48 @@ struct volume_options options[] = {
         },
         { .key  = {"rpc-auth.auth-null.*"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "Disable or enable the AUTH_NULL authentication type "
-                         "for a particular exported volume over-riding defaults"
+                         "for a particular exported volume overriding defaults"
                          " and general setting for AUTH_NULL. Must always be "
                          "enabled. This option is here only to avoid "
                          "unrecognized option warnings."
         },
         { .key  = {"rpc-auth.addr.allow"},
-          .type = GF_OPTION_TYPE_STR,
+          .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .default_value = "all",
           .description = "Allow a comma separated list of addresses and/or"
                          " hostnames to connect to the server. By default, all"
-                         " connections are disallowed. This allows users to "
+                         " connections are allowed. This allows users to "
                          "define a general rule for all exported volumes."
         },
         { .key  = {"rpc-auth.addr.reject"},
-          .type = GF_OPTION_TYPE_STR,
+          .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .default_value = "none",
           .description = "Reject a comma separated list of addresses and/or"
                          " hostnames from connecting to the server. By default,"
-                         " all connections are disallowed. This allows users to"
+                         " all connections are allowed. This allows users to"
                          "define a general rule for all exported volumes."
         },
         { .key  = {"rpc-auth.addr.*.allow"},
-          .type = GF_OPTION_TYPE_STR,
+          .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .default_value = "all",
           .description = "Allow a comma separated list of addresses and/or"
                          " hostnames to connect to the server. By default, all"
-                         " connections are disallowed. This allows users to "
+                         " connections are allowed. This allows users to "
                          "define a rule for a specific exported volume."
         },
         { .key  = {"rpc-auth.addr.*.reject"},
-          .type = GF_OPTION_TYPE_STR,
+          .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .default_value = "none",
           .description = "Reject a comma separated list of addresses and/or"
                          " hostnames from connecting to the server. By default,"
-                         " all connections are disallowed. This allows users to"
+                         " all connections are allowed. This allows users to"
                          "define a rule for a specific exported volume."
         },
         { .key  = {"rpc-auth.ports.insecure"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "Allow client connections from unprivileged ports. By "
                          "default only privileged ports are allowed. This is a"
                          "global setting in case insecure ports are to be "
@@ -972,31 +1672,35 @@ struct volume_options options[] = {
         },
         { .key  = {"rpc-auth.ports.*.insecure"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "Allow client connections from unprivileged ports. By "
                          "default only privileged ports are allowed. Use this"
-                         " option to set enable or disable insecure ports for "
-                         "a specific subvolume and to over-ride global setting "
-                         " set by the previous option."
+                         " option to enable or disable insecure ports for "
+                         "a specific subvolume and to override the global "
+                         "setting set by the previous option."
         },
         { .key  = {"rpc-auth.addr.namelookup"},
           .type = GF_OPTION_TYPE_BOOL,
-          .description = "Users have the option of turning off name lookup for"
-                  " incoming client connections using this option. In some "
+          .default_value = "off",
+          .description = "Users have the option of turning on name lookup for"
+                  " incoming client connections using this option. Use this "
+                  "option to turn on name lookups during address-based "
+                  "authentication. Turning this on will enable you to"
+                  " use hostnames in nfs.rpc-auth-* filters. In some "
                   "setups, the name server can take too long to reply to DNS "
-                  "queries resulting in timeouts of mount requests. Use this "
-                  "option to turn off name lookups during address "
-                  "authentication. Note, turning this off will prevent you from"
-                  " using hostnames in rpc-auth.addr.* filters. By default, "
-                  " name lookup is on."
+                  "queries resulting in timeouts of mount requests. By "
+                  "default, name lookup is off"
         },
         { .key  = {"nfs.dynamic-volumes"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "Internal option set to tell gnfs to use a different"
                          " scheme for encoding file handles when DVM is being"
                          " used."
         },
         { .key  = {"nfs3.*.volume-id"},
           .type = GF_OPTION_TYPE_STR,
+          .default_value = "",
           .description = "When nfs.dynamic-volumes is set, gnfs expects every "
                          "subvolume to have this option set for it, so that "
                          "gnfs can use this option to identify the volume. "
@@ -1005,25 +1709,42 @@ struct volume_options options[] = {
         },
         { .key  = {"nfs.enable-ino32"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
           .description = "For nfs clients or apps that do not support 64-bit "
                          "inode numbers, use this option to make NFS return "
-                         "32-bit inode numbers instead. Disabled by default so "
-                         "NFS returns 64-bit inode numbers by default."
+                         "32-bit inode numbers instead. Disabled by default, so"
+                         " NFS returns 64-bit inode numbers."
         },
         { .key  = {"rpc.register-with-portmap"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "For systems that need to run multiple nfs servers, "
                          "only one registration is possible with "
                          "portmap service. Use this option to turn off portmap "
                          "registration for Gluster NFS. On by default"
         },
+        { .key  = {"rpc.outstanding-rpc-limit"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = RPCSVC_MIN_OUTSTANDING_RPC_LIMIT,
+          .max  = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT,
+          .default_value = TOSTRING(RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT),
+          .description = "Parameter to throttle the number of incoming RPC "
+                         "requests from a client. 0 means no limit (can "
+                         "potentially run out of memory)"
+        },
         { .key  = {"nfs.port"},
           .type = GF_OPTION_TYPE_INT,
+          .min  = 1,
+          .max  = 0xffff,
+          .default_value = TOSTRING(GF_NFS3_PORT),
           .description = "Use this option on systems that need Gluster NFS to "
                          "be associated with a non-default port number."
         },
         { .key  = {"nfs.mem-factor"},
           .type = GF_OPTION_TYPE_INT,
+          .min  = 1,
+          .max  = 1024,
+          .default_value = TOSTRING(GF_NFS_DEFAULT_MEMFACTOR),
           .description = "Use this option to make NFS be faster on systems by "
                          "using more memory. This option specifies a multiple "
                          "that determines the total amount of memory used. "
@@ -1034,9 +1755,71 @@ struct volume_options options[] = {
         },
         { .key  = {"nfs.*.disable"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false",
           .description = "This option is used to start or stop NFS server"
                          "for individual volume."
         },
+
+        { .key  = {"nfs.nlm"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "This option, if set to 'off', disables NLM server "
+                         "by not registering the service with the portmapper."
+                         " Set it to 'on' to re-enable it. Default value: 'on'"
+        },
+
+        { .key = {"nfs.mount-udp"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "set the option to 'on' to enable mountd on UDP. "
+                         "Required for some Solaris and AIX NFS clients. "
+                         "The need for enabling this option often depends "
+                         "on the usage of NLM."
+        },
+        { .key = {"nfs.mount-rmtab"},
+          .type = GF_OPTION_TYPE_PATH,
+          .default_value = DATADIR "/rmtab",
+          .description = "Set the location of the cache file that is used to "
+                         "list all the NFS-clients that have connected "
+                         "through the MOUNT protocol. If this is on shared "
+                         "storage, all GlusterFS servers will update and "
+                         "output (with 'showmount') the same list."
+        },
+        { .key = {OPT_SERVER_AUX_GIDS},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Let the server look up which groups a user belongs "
+                         "to, overwriting the list passed from the client. "
+                         "This enables support for group lists longer than "
+                         "can be passed through the NFS protocol, but is not "
+                         "secure unless users and groups are well synchronized "
+                         "between clients and servers."
+        },
+        { .key = {OPT_SERVER_GID_CACHE_TIMEOUT},
+          .type = GF_OPTION_TYPE_INT,
+          .min = 0,
+          .max = 3600,
+          .default_value = "5",
+          .description = "Number of seconds to cache auxiliary-GID data, when "
+                         OPT_SERVER_AUX_GIDS " is set."
+        },
+        { .key = {"nfs.acl"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "This option is used to control ACL support for NFS."
+        },
+        { .key  = {"nfs.drc"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "on",
+          .description = "Enable Duplicate Request Cache in gNFS server to "
+                         "improve correctness of non-idempotent operations like "
+                         "write, delete, link, et al"
+        },
+        { .key  = {"nfs.drc-size"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "0x20000",
+          .description = "Sets the number of non-idempotent "
+                         "requests to cache in drc"
+        },
         { .key  = {NULL} },
 };
-
diff --git a/xlators/nfs/server/src/nfs.h b/xlators/nfs/server/src/nfs.h
index 3113e32cc..00c7f8046 100644
--- a/xlators/nfs/server/src/nfs.h
+++ b/xlators/nfs/server/src/nfs.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __NFS_H__
@@ -28,6 +19,8 @@
 #include "rpcsvc.h"
 #include "dict.h"
 #include "xlator.h"
+#include "lkowner.h"
+#include "gidcache.h"
 
 #define GF_NFS                  "nfs"
 
@@ -43,7 +36,7 @@
 #define GF_NFS_MAX_MEMFACTOR            30
 
 #define GF_NFS_DVM_ON                   1
-#define GF_NFS_DVM_OFF                  2
+#define GF_NFS_DVM_OFF                  0
 
 /* This corresponds to the max 16 number of group IDs that are sent through an
  * RPC request. Since NFS is the only one going to set this, we can be safe
@@ -64,11 +57,12 @@ struct nfs_initer_list {
         rpcsvc_program_t        *program;
 };
 
-
 struct nfs_state {
         rpcsvc_t                *rpcsvc;
         struct list_head        versions;
         struct mount3_state     *mstate;
+        struct nfs3_state       *nfs3state;
+        struct nlm4_state       *nlm4state;
         struct mem_pool         *foppool;
         unsigned int            memfactor;
         xlator_list_t           *subvols;
@@ -82,6 +76,21 @@ struct nfs_state {
         int                     enable_ino32;
         unsigned int            override_portnum;
         int                     allow_insecure;
+        int                     enable_nlm;
+        int                     enable_acl;
+        int                     mount_udp;
+        char                    *rmtab;
+        struct rpc_clnt         *rpc_clnt;
+        gf_boolean_t            server_aux_gids;
+	uint32_t		server_aux_gids_max_age;
+	gid_cache_t		gid_cache;
+        uint32_t                generation;
+        gf_boolean_t            register_portmap;
+};
+
+struct nfs_inode_ctx {
+        struct list_head        shares;
+        uint32_t                generation;
 };
 
 #define gf_nfs_dvm_on(nfsstt)   (((struct nfs_state *)nfsstt)->dynamicvolumes == GF_NFS_DVM_ON)
@@ -102,6 +111,7 @@ typedef struct nfs_user_info {
         uid_t   uid;
         gid_t   gids[NFS_NGROUPS];
         int     ngrps;
+        gf_lkowner_t lk_owner;
 } nfs_user_t;
 
 extern int
@@ -119,4 +129,7 @@ nfs_request_primary_user_init (nfs_user_t *nfu, rpcsvc_request_t *req,
                                uid_t uid, gid_t gid);
 extern int
 nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl);
+
+extern void
+nfs_fix_groups (xlator_t *this, call_stack_t *root);
 #endif
diff --git a/xlators/nfs/server/src/nfs3-fh.c b/xlators/nfs/server/src/nfs3-fh.c
index 9aea88137..e199c56dc 100644
--- a/xlators/nfs/server/src/nfs3-fh.c
+++ b/xlators/nfs/server/src/nfs3-fh.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -46,6 +37,12 @@ nfs3_fh_validate (struct nfs3_fh *fh)
 	if (fh->ident[1] != GF_NFSFH_IDENT1)
 		return 0;
 
+       if (fh->ident[2] != GF_NFSFH_IDENT2)
+               return 0;
+
+       if (fh->ident[3] != GF_NFSFH_IDENT3)
+               return 0;
+
         return 1;
 }
 
@@ -58,8 +55,9 @@ nfs3_fh_init (struct nfs3_fh *fh, struct iatt *buf)
 
         fh->ident[0] = GF_NFSFH_IDENT0;
         fh->ident[1] = GF_NFSFH_IDENT1;
+        fh->ident[2] = GF_NFSFH_IDENT2;
+        fh->ident[3] = GF_NFSFH_IDENT3;
 
-        fh->hashcount = 0;
         uuid_copy (fh->gfid, buf->ia_gfid);
 }
 
@@ -112,99 +110,33 @@ nfs3_fh_is_root_fh (struct nfs3_fh *fh)
 }
 
 
-nfs3_hash_entry_t
-nfs3_fh_hash_entry (uuid_t gfid)
-{
-        nfs3_hash_entry_t       hash = 0;
-        int                     shiftsize = 48;
-        uint64_t                ino = 0;
-        uint64_t                gen = 0;
-        nfs3_hash_entry_t       inomsb = 0;
-        nfs3_hash_entry_t       inolsb = 0;
-        nfs3_hash_entry_t       inols23b = 0;
-
-        nfs3_hash_entry_t       genmsb = 0;
-        nfs3_hash_entry_t       genlsb = 0;
-        nfs3_hash_entry_t       genls23b = 0;
-
-        memcpy (&ino, &gfid[8], 8);
-        hash = ino;
-        while (shiftsize != 0) {
-                hash ^= (ino >> shiftsize);
-                shiftsize -= 16;
-        }
-/*
-        gf_log ("FILEHANDLE", GF_LOG_TRACE, "INO %"PRIu64, ino);
-        gf_log ("FILEHANDLE",GF_LOG_TRACE, "PRI HASH %d", hash);
-*/
-        inomsb = (ino >> 56);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inomsb %d", inomsb);
-
-        inolsb = ((ino << 56) >> 56);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inolsb %d", inolsb);
-
-        inolsb = (inolsb << 8);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inolsb to inomsb %d", inolsb);
-        inols23b = ((ino << 40) >> 48);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inols23b %d", inols23b);
-
-        inols23b = (inols23b << 8);
-//        gf_log ("FILEHDNALE", GF_LOG_TRACE, "inols23b  %d", inols23b);
-
-        memcpy (&gen, &gfid[0], 8);
-        genmsb = (gen >> 56);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inomsb %d", inomsb);
-
-        genlsb = ((gen << 56) >> 56);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inolsb %d", inolsb);
-
-        genlsb = (genlsb << 8);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inolsb to inomsb %d", inolsb);
-
-        genls23b = ((gen << 40) >> 48);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inols23b %d", inols23b);
-
-        genls23b = (genls23b << 8);
-//        gf_log ("FILEHDNALE", GF_LOG_TRACE, "inols23b  %d", inols23b);
-
-        hash ^= inolsb ^ inomsb ^ inols23b ^ genmsb ^ genlsb ^ genls23b;
-        return hash;
-
-}
-
 void
-nfs3_fh_to_str (struct nfs3_fh *fh, char *str)
+nfs3_fh_to_str (struct nfs3_fh *fh, char *str, size_t len)
 {
-        char            gfid[512];
-        char            exportid[512];
+        char            gfid[GF_UUID_BUF_SIZE];
+        char            exportid[GF_UUID_BUF_SIZE];
 
         if ((!fh) || (!str))
                 return;
 
-        sprintf (str, "FH: hashcount %d, exportid %s, gfid %s",
-                 fh->hashcount, uuid_utoa_r (fh->exportid, exportid),
-                 uuid_utoa_r (fh->gfid, gfid));
+        snprintf (str, len, "FH: exportid %s, gfid %s",
+                  uuid_utoa_r (fh->exportid, exportid),
+                  uuid_utoa_r (fh->gfid, gfid));
 }
 
 void
 nfs3_log_fh (struct nfs3_fh *fh)
 {
-//        int     x = 0;
         char    gfidstr[512];
         char    exportidstr[512];
 
         if (!fh)
                 return;
 
-        gf_log ("nfs3-fh", GF_LOG_TRACE, "filehandle: hashcount %d, exportid "
-                "0x%s, gfid 0x%s", fh->hashcount,
+        gf_log ("nfs3-fh", GF_LOG_TRACE, "filehandle: exportid "
+                "0x%s, gfid 0x%s",
                  uuid_utoa_r (fh->exportid, exportidstr),
                  uuid_utoa_r (fh->gfid, gfidstr));
-/*
-        for (; x < fh->hashcount; ++x)
-                gf_log ("FILEHANDLE", GF_LOG_TRACE, "Hash %d: %d", x,
-                        fh->entryhash[x]);
-*/
 }
 
 int
@@ -216,53 +148,35 @@ nfs3_fh_build_parent_fh (struct nfs3_fh *child, struct iatt *newstat,
 
         nfs3_fh_init (newfh, newstat);
         uuid_copy (newfh->exportid, child->exportid);
-        if (newstat->ia_ino == 1)
-                goto done;
 
-        newfh->hashcount = child->hashcount - 1;
-        memcpy (newfh->entryhash, child->entryhash,
-                newfh->hashcount * GF_NFSFH_ENTRYHASH_SIZE);
+        return 0;
+}
 
-done:
-//        nfs3_log_fh (newfh);
+int
+nfs3_build_fh (inode_t *inode, uuid_t exportid, struct nfs3_fh *newfh)
+{
+        if (!newfh || !inode)
+                return -1;
 
+        newfh->ident[0] = GF_NFSFH_IDENT0;
+        newfh->ident[1] = GF_NFSFH_IDENT1;
+        newfh->ident[2] = GF_NFSFH_IDENT2;
+        newfh->ident[3] = GF_NFSFH_IDENT3;
+        uuid_copy (newfh->gfid, inode->gfid);
+        uuid_copy (newfh->exportid, exportid);
         return 0;
 }
 
-
 int
 nfs3_fh_build_child_fh (struct nfs3_fh *parent, struct iatt *newstat,
                         struct nfs3_fh *newfh)
 {
-        int             hashcount = 0;
-        int             entry = 0;
-
         if ((!parent) || (!newstat) || (!newfh))
                 return -1;
 
         nfs3_fh_init (newfh, newstat);
         uuid_copy (newfh->exportid, parent->exportid);
 
-        newfh->hashcount = parent->hashcount + 1;
-        /* Only copy the hashes that are available in the parent file
-         * handle. */
-        if (parent->hashcount > GF_NFSFH_MAXHASHES)
-                hashcount = GF_NFSFH_MAXHASHES;
-        else
-                hashcount = parent->hashcount;
-
-        memcpy (newfh->entryhash, parent->entryhash,
-                hashcount * GF_NFSFH_ENTRYHASH_SIZE);
-
-        /* Do not insert parent dir hash if there is no space left in the hash
-         * array of the child entry. */
-        if (newfh->hashcount <= GF_NFSFH_MAXHASHES) {
-                entry = newfh->hashcount - 1;
-                newfh->entryhash[entry] = nfs3_fh_hash_entry (parent->gfid);
-        }
-
-//        nfs3_log_fh (newfh);
-
         return 0;
 }
 
@@ -270,34 +184,5 @@ nfs3_fh_build_child_fh (struct nfs3_fh *parent, struct iatt *newstat,
 uint32_t
 nfs3_fh_compute_size (struct nfs3_fh *fh)
 {
-        uint32_t        fhlen = 0;
-
-        if (!fh)
-                return 0;
-
-        if (fh->hashcount <= GF_NFSFH_MAXHASHES)
-                fhlen = nfs3_fh_hashcounted_size (fh->hashcount);
-        else
-                fhlen = nfs3_fh_hashcounted_size (GF_NFSFH_MAXHASHES);
-
-        return fhlen;
-}
-
-
-/* There is no point searching at a directory level which is beyond that of
- * the hashcount given in the file handle.
- */
-int
-nfs3_fh_hash_index_is_beyond (struct nfs3_fh *fh, int hashidx)
-{
-        if (!fh)
-                return 1;
-
-        if (fh->hashcount >= hashidx)
-                return 0;
-        else
-                return 1;
-
-        return 1;
+        return GF_NFSFH_STATIC_SIZE;
 }
-
diff --git a/xlators/nfs/server/src/nfs3-fh.h b/xlators/nfs/server/src/nfs3-fh.h
index e7c0625ee..1049cdc96 100644
--- a/xlators/nfs/server/src/nfs3-fh.h
+++ b/xlators/nfs/server/src/nfs3-fh.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS_FH_H_
@@ -35,17 +26,10 @@
  * handles for now. This will change if and when we need v4. */
 #define GF_NFSFH_IDENT0         ':'
 #define GF_NFSFH_IDENT1         'O'
-#define GF_NFSFH_IDENT_SIZE     (sizeof(char) * 2)
-#define GF_NFSFH_STATIC_SIZE    (GF_NFSFH_IDENT_SIZE + (2*sizeof (uuid_t)) + sizeof (uint16_t))
-#define GF_NFSFH_MAX_HASH_BYTES (NFS3_FHSIZE - GF_NFSFH_STATIC_SIZE)
-
-/* Each hash element in the file handle is of 2 bytes thus giving
- * us theoretically 65536 unique entries in a directory.
- */
-typedef uint16_t                nfs3_hash_entry_t;
-#define GF_NFSFH_ENTRYHASH_SIZE (sizeof (nfs3_hash_entry_t))
-#define GF_NFSFH_MAXHASHES      ((int)(GF_NFSFH_MAX_HASH_BYTES / GF_NFSFH_ENTRYHASH_SIZE))
-#define nfs3_fh_hashcounted_size(hcount) (GF_NFSFH_STATIC_SIZE + (hcount * GF_NFSFH_ENTRYHASH_SIZE))
+#define GF_NFSFH_IDENT2         'G'
+#define GF_NFSFH_IDENT3         'L'
+#define GF_NFSFH_IDENT_SIZE     (sizeof(char) * 4)
+#define GF_NFSFH_STATIC_SIZE    (GF_NFSFH_IDENT_SIZE + (2*sizeof (uuid_t)))
 
 #define nfs3_fh_exportid_to_index(exprtid)      ((uint16_t)exprtid[15])
 /* ATTENTION: Change in size of the structure below should be reflected in the
@@ -54,9 +38,9 @@ typedef uint16_t                nfs3_hash_entry_t;
 struct nfs3_fh {
 
         /* Used to ensure that a bunch of bytes are actually a GlusterFS NFS
-         * file handle. Should contain ":O"
+         * file handle. Should contain ":OGL"
          */
-        char                    ident[2];
+        char                    ident[4];
 
         /* UUID that identifies an export. The value stored in exportid
          * depends on the usage of gluster nfs. If the DVM is enabled using
@@ -72,11 +56,11 @@ struct nfs3_fh {
 
         /* File/dir gfid. */
         uuid_t                  gfid;
-
-        /* Number of file/ino hash elements that follow the ino. */
-        uint16_t                hashcount;
-
-        nfs3_hash_entry_t       entryhash[GF_NFSFH_MAXHASHES];
+        /* This structure must be exactly NFS3_FHSIZE (64) bytes long.
+           Having the structure shorter results in buffer overflows
+           during XDR decoding.
+        */
+        unsigned char padding[NFS3_FHSIZE - GF_NFSFH_STATIC_SIZE];
 } __attribute__((__packed__));
 
 #define GF_NFS3FH_STATIC_INITIALIZER    {{0},}
@@ -84,9 +68,6 @@ struct nfs3_fh {
 extern uint32_t
 nfs3_fh_compute_size (struct nfs3_fh *fh);
 
-extern int
-nfs3_fh_hash_index_is_beyond (struct nfs3_fh *fh, int hashidx);
-
 extern uint16_t
 nfs3_fh_hash_entry (uuid_t gfid);
 
@@ -107,7 +88,7 @@ extern void
 nfs3_log_fh (struct nfs3_fh *fh);
 
 extern void
-nfs3_fh_to_str (struct nfs3_fh *fh, char *str);
+nfs3_fh_to_str (struct nfs3_fh *fh, char *str, size_t len);
 
 extern int
 nfs3_fh_build_parent_fh (struct nfs3_fh *child, struct iatt *newstat,
@@ -115,4 +96,8 @@ nfs3_fh_build_parent_fh (struct nfs3_fh *child, struct iatt *newstat,
 
 extern struct nfs3_fh
 nfs3_fh_build_uuid_root_fh (uuid_t volumeid);
+
+extern int
+nfs3_build_fh (inode_t *inode, uuid_t exportid, struct nfs3_fh *newfh);
+
 #endif
diff --git a/xlators/nfs/server/src/nfs3-helpers.c b/xlators/nfs/server/src/nfs3-helpers.c
index 7075b3e6c..9059fc341 100644
--- a/xlators/nfs/server/src/nfs3-helpers.c
+++ b/xlators/nfs/server/src/nfs3-helpers.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -102,21 +93,18 @@ struct nfs3stat_strerror nfs3stat_strerror_table[] = {
 uint64_t
 nfs3_iatt_gfid_to_ino (struct iatt *buf)
 {
-        uuid_t          gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
-        uint64_t        ino = 0;
+        uint64_t ino  = 0;
 
         if (!buf)
                 return 0;
 
-        if ((buf->ia_ino != 1) && (uuid_compare (buf->ia_gfid, gfid) != 0)) {
-                if (gf_nfs_enable_ino32()) {
-                        ino = (uint32_t )nfs_hash_gfid (buf->ia_gfid);
-                        goto hashout;
-                }
+        if (gf_nfs_enable_ino32()) {
+                ino = (uint32_t )nfs_hash_gfid (buf->ia_gfid);
+                goto hashout;
+        }
 
-                memcpy (&ino, &buf->ia_gfid[8], sizeof (uint64_t));
-        } else
-                ino = 1;
+        /* from posix its guaranteed to send unique ino */
+        ino = buf->ia_ino;
 
 hashout:
         return ino;
@@ -268,6 +256,20 @@ nfs3_errno_to_nfsstat3 (int errnum)
         return stat;
 }
 
+/*
+ * Special case: If op_ret is -1, it's very unusual op_errno being
+ * 0 which means something came wrong from upper layer(s). If it
+ * happens by any means, then set NFS3 status to NFS3ERR_SERVERFAULT.
+ */
+inline nfsstat3
+nfs3_cbk_errno_status (int32_t op_ret, int32_t op_errno)
+{
+        if ((op_ret == -1) && (op_errno == 0)) {
+                return NFS3ERR_SERVERFAULT;
+        }
+
+        return nfs3_errno_to_nfsstat3 (op_errno);
+}
 
 void
 nfs3_fill_lookup3res_error (lookup3res *res, nfsstat3 stat,
@@ -287,6 +289,9 @@ nfs3_stat_to_fattr3 (struct iatt *buf)
 {
         fattr3          fa = {0, };
 
+        if (buf == NULL)
+                goto out;
+
         if (IA_ISDIR (buf->ia_type))
                 fa.type = NF3DIR;
         else if (IA_ISREG (buf->ia_type))
@@ -356,6 +361,7 @@ nfs3_stat_to_fattr3 (struct iatt *buf)
         fa.mtime.seconds = buf->ia_mtime;
         fa.mtime.nseconds = buf->ia_mtime_nsec;
 
+out:
         return fa;
 }
 
@@ -499,7 +505,7 @@ nfs3_fill_fsinfo3res (struct nfs3_state *nfs3, fsinfo3res *res,
         resok.wtpref = nfs3->writesize;
         resok.wtmult = GF_NFS3_WTMULT;
         resok.dtpref = nfs3->readdirsize;
-        resok.maxfilesize = GF_NFS3_MAXFILE;
+        resok.maxfilesize = GF_NFS3_MAXFILESIZE;
         resok.time_delta = tdelta;
         resok.properties = GF_NFS3_FS_PROP;
 
@@ -567,7 +573,7 @@ nfs3_accessbits (int32_t accbits)
                 accresult |= ACCESS3_READ;
 
         if (accbits & POSIX_WRITE)
-                accresult |= (ACCESS3_MODIFY | ACCESS3_EXTEND);
+                accresult |= (ACCESS3_MODIFY | ACCESS3_EXTEND | ACCESS3_DELETE);
 
         /* lookup on directory allowed only in case of execute permission */
         if (accbits & POSIX_EXEC)
@@ -584,7 +590,7 @@ nfs3_request_to_accessbits (int32_t accbits)
         if (accbits & ACCESS3_READ)
                 acc_request |= POSIX_READ;
 
-        if (accbits & (ACCESS3_MODIFY | ACCESS3_EXTEND))
+        if (accbits & (ACCESS3_MODIFY | ACCESS3_EXTEND | ACCESS3_DELETE))
                 acc_request |= POSIX_WRITE;
 
         /* For lookup on directory check for execute permission */
@@ -594,7 +600,8 @@ nfs3_request_to_accessbits (int32_t accbits)
         return acc_request;
 }
 void
-nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits)
+nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits,
+		      int32_t reqaccbits)
 {
         uint32_t        accres = 0;
 
@@ -605,7 +612,8 @@ nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits)
 
         accres = nfs3_accessbits (accbits);
 
-        res->access3res_u.resok.access = accres;
+	/* do not answer what was not asked */
+        res->access3res_u.resok.access = accres & reqaccbits;
 }
 
 void
@@ -1608,13 +1616,14 @@ err:
 
 void
 nfs3_stat_to_errstr (uint32_t xid, char *op, nfsstat3 stat, int pstat,
-                     char *errstr)
+                     char *errstr, size_t len)
 {
         if ((!op) || (!errstr))
                 return;
 
-        sprintf (errstr, "XID: %x, %s: NFS: %d(%s), POSIX: %d(%s)", xid, op,
-                 stat, nfsstat3_strerror (stat), pstat, strerror (pstat));
+        snprintf (errstr, len, "XID: %x, %s: NFS: %d(%s), POSIX: %d(%s)",
+                  xid, op,stat, nfsstat3_strerror (stat), pstat,
+                  strerror (pstat));
 }
 
 void
@@ -1622,10 +1631,10 @@ nfs3_log_common_call (uint32_t xid, char *op, struct nfs3_fh *fh)
 {
         char    fhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
 
-        nfs3_fh_to_str (fh, fhstr);
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, %s: args: %s", xid, op,
                 fhstr);
 }
@@ -1637,9 +1646,9 @@ nfs3_log_fh_entry_call (uint32_t xid, char *op, struct nfs3_fh *fh,
 {
         char    fhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, fhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, %s: args: %s, name: %s", xid,
                 op, fhstr, name);
 }
@@ -1652,10 +1661,10 @@ nfs3_log_rename_call (uint32_t xid, struct nfs3_fh *src, char *sname,
         char    sfhstr[1024];
         char    dfhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (src, sfhstr);
-        nfs3_fh_to_str (dst, dfhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (src, sfhstr, sizeof (sfhstr));
+        nfs3_fh_to_str (dst, dfhstr, sizeof (dfhstr));
         gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, RENAME: args: Src: %s, "
                 "name: %s, Dst: %s, name: %s", xid, sfhstr, sname, dfhstr,
                 dname);
@@ -1673,9 +1682,9 @@ nfs3_log_create_call (uint32_t xid, struct nfs3_fh *fh, char *name,
         char    unchkd[] = "UNCHECKED";
         char    guarded[] = "GUARDED";
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, fhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         if (mode == EXCLUSIVE)
                 modestr = exclmode;
         else if (mode == GUARDED)
@@ -1698,9 +1707,9 @@ nfs3_log_mknod_call (uint32_t xid, struct nfs3_fh *fh, char *name, int type)
         char    sock[] = "SOCK";
         char    fifo[] = "FIFO";
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, fhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         if (type == NF3CHR)
                 modestr = chr;
         else if (type == NF3BLK)
@@ -1721,9 +1730,9 @@ nfs3_log_symlink_call (uint32_t xid, struct nfs3_fh *fh, char *name, char *tgt)
 {
         char    fhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, fhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, SYMLINK: args: %s, name: %s,"
                 " target: %s", xid, fhstr, name, tgt);
 }
@@ -1736,10 +1745,10 @@ nfs3_log_link_call (uint32_t xid, struct nfs3_fh *fh, char *name,
         char    dfhstr[1024];
         char    tfhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, dfhstr);
-        nfs3_fh_to_str (tgt, tfhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, dfhstr, sizeof (dfhstr));
+        nfs3_fh_to_str (tgt, tfhstr, sizeof (tfhstr));
         gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, LINK: args: %s, name: %s,"
                 " target: %s", xid, dfhstr, name, tfhstr);
 }
@@ -1751,9 +1760,9 @@ nfs3_log_rw_call (uint32_t xid, char *op, struct nfs3_fh *fh, offset3 offt,
 {
         char    fhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, fhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         if (stablewrite == -1)
                 gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, %s: args: %s, offset:"
                         " %"PRIu64",  count: %"PRIu32, xid, op, fhstr, offt,
@@ -3384,11 +3393,11 @@ void
 nfs3_log_common_res (uint32_t xid, int op, nfsstat3 stat, int pstat)
 {
         char    errstr[1024];
-	int	ll = nfs3_loglevel (op, stat);
+        int     ll = nfs3_loglevel (op, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, nfs3op_strings[op].str, stat, pstat, errstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, nfs3op_strings[op].str, stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s", errstr);
 }
 
@@ -3396,14 +3405,14 @@ void
 nfs3_log_readlink_res (uint32_t xid, nfsstat3 stat, int pstat, char *linkpath)
 {
         char    errstr[1024];
-	int	ll = nfs3_loglevel (NFS3_READLINK, stat);
+        int     ll = nfs3_loglevel (NFS3_READLINK, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
+        if (THIS->ctx->log.loglevel < ll)
+                return;
 
-        nfs3_stat_to_errstr (xid, "READLINK", stat, pstat, errstr);
+        nfs3_stat_to_errstr (xid, "READLINK", stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s, target: %s",
-		errstr, linkpath);
+                errstr, linkpath);
 
 }
 
@@ -3412,12 +3421,12 @@ nfs3_log_read_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
                    int is_eof, struct iovec *vec, int32_t veccount)
 {
         char    errstr[1024];
-	int	ll = GF_LOG_DEBUG;
+        int     ll = GF_LOG_DEBUG;
 
 	ll = nfs3_loglevel (NFS3_READ, stat);
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, "READ", stat, pstat, errstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, "READ", stat, pstat, errstr, sizeof (errstr));
         if (vec)
                 gf_log (GF_NFS3, ll, "%s, count: %"PRIu32", is_eof:"
                         " %d, vector: count: %d, len: %zd", errstr, count,
@@ -3433,12 +3442,12 @@ nfs3_log_write_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
                     int stable, uint64_t wverf)
 {
         char    errstr[1024];
-	int	ll = nfs3_loglevel (NFS3_WRITE, stat);
+        int     ll = nfs3_loglevel (NFS3_WRITE, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
+        if (THIS->ctx->log.loglevel < ll)
+                 return;
 
-        nfs3_stat_to_errstr (xid, "WRITE", stat, pstat, errstr);
+        nfs3_stat_to_errstr (xid, "WRITE", stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s, count: %"PRIu32", %s,wverf: %"PRIu64
                 , errstr, count, (stable == UNSTABLE)?"UNSTABLE":"STABLE",
                 wverf);
@@ -3451,12 +3460,12 @@ nfs3_log_newfh_res (uint32_t xid, int op, nfsstat3 stat, int pstat,
 {
         char    errstr[1024];
         char    fhstr[1024];
-	int	ll = nfs3_loglevel (op, stat);
+        int     ll = nfs3_loglevel (op, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, nfs3op_strings[op].str, stat, pstat, errstr);
-        nfs3_fh_to_str (newfh, fhstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, nfs3op_strings[op].str, stat, pstat, errstr, sizeof (errstr));
+        nfs3_fh_to_str (newfh, fhstr, sizeof (fhstr));
 
         gf_log (GF_NFS3, nfs3_loglevel (op, stat), "%s, %s", errstr, fhstr);
 }
@@ -3467,11 +3476,11 @@ nfs3_log_readdir_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
                       count3 count, int is_eof)
 {
         char    errstr[1024];
-	int	ll = nfs3_loglevel (NFS3_READDIR, stat);
+        int     ll = nfs3_loglevel (NFS3_READDIR, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, "READDIR", stat, pstat, errstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, "READDIR", stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s, count: %"PRIu32", cverf: %"PRIu64
                 ", is_eof: %d", errstr, count, cverf, is_eof);
 }
@@ -3484,9 +3493,9 @@ nfs3_log_readdirp_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
         char    errstr[1024];
 	int	ll = nfs3_loglevel (NFS3_READDIRP, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, "READDIRPLUS", stat, pstat, errstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, "READDIRPLUS", stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s, dircount: %"PRIu32", maxcount: %"
                 PRIu32", cverf: %"PRIu64", is_eof: %d", errstr, dircount,
                 maxcount, cverf, is_eof);
@@ -3499,9 +3508,9 @@ nfs3_log_commit_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t wverf)
         char    errstr[1024];
 	int	ll = nfs3_loglevel (NFS3_COMMIT, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, "COMMIT", stat, pstat, errstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, "COMMIT", stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s, wverf: %"PRIu64, errstr, wverf);
 }
 
@@ -3512,10 +3521,10 @@ nfs3_log_readdir_call (uint32_t xid, struct nfs3_fh *fh, count3 dircount,
 {
         char    fhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
 
-        nfs3_fh_to_str (fh, fhstr);
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
 
         if (maxcount == 0)
                 gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, READDIR: args: %s,"
@@ -3548,55 +3557,6 @@ err:
         return ret;
 }
 
-#define GF_NFS3_FHRESOLVE_FOUND         1
-#define GF_NFS3_FHRESOLVE_NOTFOUND      2
-#define GF_NFS3_FHRESOLVE_DIRFOUND      3
-
-int
-nfs3_fh_resolve_check_entry (struct nfs3_fh *fh, gf_dirent_t *candidate,
-                             int hashidx)
-{
-        struct iatt             *ia = NULL;
-        int                     ret = GF_NFS3_FHRESOLVE_NOTFOUND;
-        nfs3_hash_entry_t       entryhash = 0;
-
-        if ((!fh) || (!candidate))
-                return ret;
-
-        if ((strcmp (candidate->d_name, ".") == 0) ||
-            (strcmp (candidate->d_name, "..") == 0))
-                goto found_entry;
-
-        ia = &candidate->d_stat;
-        if ((uuid_compare (candidate->d_stat.ia_gfid, fh->gfid)) == 0) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Found entry: gfid: %s, "
-                        "name: %s, hashcount %d",
-                         uuid_utoa (candidate->d_stat.ia_gfid),
-                         candidate->d_name, hashidx);
-                ret = GF_NFS3_FHRESOLVE_FOUND;
-                goto found_entry;
-        }
-
-        /* This condition ensures that we never have to be afraid of having
-         * a directory hash conflict with a file hash. The consequence of
-         * this condition is that we can now have unlimited files in a directory
-         * and upto 65536 sub-directories in a directory.
-         */
-        if (!IA_ISDIR (candidate->d_stat.ia_type))
-                goto found_entry;
-        entryhash = fh->entryhash[hashidx];
-        if (entryhash == nfs3_fh_hash_entry (ia->ia_gfid)) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Found hash match: %s: %d, "
-                        "hashidx: %d", candidate->d_name, entryhash, hashidx);
-                ret = GF_NFS3_FHRESOLVE_DIRFOUND;
-                goto found_entry;
-        }
-
-found_entry:
-
-        return ret;
-}
-
 
 int32_t
 nfs3_fh_resolve_entry_lookup_cbk (call_frame_t *frame, void *cookie,
@@ -3627,7 +3587,8 @@ nfs3_fh_resolve_entry_lookup_cbk (call_frame_t *frame, void *cookie,
                                    cs->resolvedloc.name, buf);
         if (linked_inode) {
                 inode_lookup (linked_inode);
-                inode_unref (linked_inode);
+                inode_unref (cs->resolvedloc.inode);
+                cs->resolvedloc.inode = linked_inode;
         }
 err:
         nfs3_call_resume (cs);
@@ -3635,119 +3596,6 @@ err:
 }
 
 
-int
-nfs3_fh_resolve_found_entry (nfs3_call_state_t *cs, gf_dirent_t *candidate)
-{
-        int             ret = 0;
-        nfs_user_t      nfu = {0, };
-        uuid_t          gfid = {0, };
-
-        if ((!cs) || (!candidate))
-                return -EFAULT;
-
-        uuid_copy (gfid, cs->resolvedloc.inode->gfid);
-        nfs_loc_wipe (&cs->resolvedloc);
-        ret = nfs_entry_loc_fill (cs->vol->itable, gfid, candidate->d_name,
-                                  &cs->resolvedloc, NFS_RESOLVE_CREATE);
-        if (ret == -ENOENT) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry not in itable, needs"
-                        " lookup");
-                nfs_user_root_create (&nfu);
-                ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                  nfs3_fh_resolve_entry_lookup_cbk,
-                                  cs);
-        } else {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry got from itable");
-                nfs3_call_resume (cs);
-        }
-
-        return ret;
-}
-
-
-int32_t
-nfs3_fh_resolve_parent_lookup_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int32_t op_ret,
-                                  int32_t op_errno, inode_t *inode,
-                                  struct iatt *buf, dict_t *xattr,
-                                  struct iatt *postparent)
-{
-        nfs3_call_state_t       *cs = NULL;
-        inode_t                 *linked_inode = NULL;
-
-        cs = frame->local;
-        cs->resolve_ret = op_ret;
-        cs->resolve_errno = op_errno;
-
-        if (op_ret == -1) {
-                gf_log (GF_NFS3, (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_ERROR),
-                        "Lookup failed: %s: %s",
-                        cs->resolvedloc.path, strerror (op_errno));
-                nfs3_call_resume (cs);
-                goto err;
-        } else
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry looked up: %s",
-                        cs->resolvedloc.path);
-
-        linked_inode = inode_link (inode, cs->resolvedloc.parent,
-                                   cs->resolvedloc.name, buf);
-        if (linked_inode) {
-                inode_lookup (linked_inode);
-                inode_unref (linked_inode);
-        }
-        nfs3_fh_resolve_entry_hard (cs);
-
-err:
-        return 0;
-}
-
-
-int
-nfs3_fh_resolve_found_parent (nfs3_call_state_t *cs, gf_dirent_t *candidate)
-{
-        int             ret = 0;
-        nfs_user_t      nfu = {0, };
-        uuid_t          gfid = {0, };
-
-        if ((!cs) || (!candidate))
-                return -EFAULT;
-
-        uuid_copy (gfid, cs->resolvedloc.inode->gfid);
-        nfs_loc_wipe (&cs->resolvedloc);
-        ret = nfs_entry_loc_fill (cs->vol->itable, gfid, candidate->d_name,
-                                  &cs->resolvedloc, NFS_RESOLVE_CREATE);
-        if (ret == -ENOENT) {
-                nfs_user_root_create (&nfu);
-                ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                  nfs3_fh_resolve_parent_lookup_cbk,
-                                  cs);
-        } else
-                nfs3_fh_resolve_entry_hard (cs);
-
-        return ret;
-}
-
-
-int
-nfs3_fh_resolve_found (nfs3_call_state_t *cs, gf_dirent_t *candidate)
-{
-        int             ret = 0;
-
-        if ((!cs) || (!candidate))
-                return -EFAULT;
-
-        if (!cs->resolventry) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Candidate entry was found");
-                ret = nfs3_fh_resolve_found_entry (cs, candidate);
-        } else {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry's parent was found");
-                ret = nfs3_fh_resolve_found_parent (cs, candidate);
-        }
-
-        return ret;
-}
-
-
 int32_t
 nfs3_fh_resolve_inode_lookup_cbk (call_frame_t *frame, void *cookie,
                                   xlator_t *this, int32_t op_ret,
@@ -3794,80 +3642,6 @@ err:
 }
 
 
-/* Validate the depth of the dir such that we do not end up opening and
- * reading directories beyond those that are needed for resolving the file
- * handle.
- * Returns 1 if fh resolution can continue, 0 otherwise.
- */
-int
-nfs3_fh_resolve_validate_dirdepth (nfs3_call_state_t *cs)
-{
-        int     ret = 1;
-
-        if (!cs)
-                return 0;
-
-        /* This condition will generally never be hit because the
-         * hash-matching scheme will prevent us from going into a
-         * directory that is not part of the hash-array.
-         */
-        if (nfs3_fh_hash_index_is_beyond (&cs->resolvefh, cs->hashidx)) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Hash index is beyond: idx %d,"
-                        " fh idx: %d", cs->hashidx, cs->resolvefh.hashcount);
-                ret = 0;
-                goto out;
-        }
-
-        if (cs->hashidx >= GF_NFSFH_MAXHASHES) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Hash index beyond max hashes:"
-                        " hashidx %d, max: %d", cs->hashidx,
-                        GF_NFSFH_MAXHASHES);
-                ret = 0;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-/*
- * Called in a recursive code path, so if another
- * directory was opened in an earlier call during fh resolution, we must unref
- * through this reference before opening another fd_t.
- */
-#define nfs3_fh_resolve_close_cwd(cst)                                  \
-        do {                                                            \
-                if ((cst)->resolve_dir_fd) {                            \
-                        gf_log (GF_NFS3, GF_LOG_TRACE, "resolve fd "    \
-                                "unrefing: 0x%lx, ref: %d",             \
-                                (long)(cst)->resolve_dir_fd,            \
-                                (cst)->resolve_dir_fd->refcount);       \
-                                fd_unref ((cst)->resolve_dir_fd);       \
-                }                                                       \
-        } while (0)                                                     \
-
-
-int
-nfs3_fh_resolve_determine_response (nfs3_call_state_t *cs) {
-
-        int     response = GF_NFS3_FHRESOLVE_NOTFOUND;
-
-        if (!cs)
-                return response;
-
-        if ((cs->hashmatch) && (cs->entrymatch))
-                response = GF_NFS3_FHRESOLVE_FOUND;
-        else if ((cs->hashmatch) && (!cs->entrymatch))
-                response = GF_NFS3_FHRESOLVE_DIRFOUND;
-        else if ((!cs->hashmatch) && (cs->entrymatch))
-                response = GF_NFS3_FHRESOLVE_FOUND;
-        else if ((!cs->hashmatch) && (!cs->entrymatch))
-                response = GF_NFS3_FHRESOLVE_NOTFOUND;
-
-        return response;
-}
-
 
 /* Needs no extra argument since it knows that the fh to be resolved is in
  * resolvefh and that it needs to start looking from the root.
diff --git a/xlators/nfs/server/src/nfs3-helpers.h b/xlators/nfs/server/src/nfs3-helpers.h
index 67935d143..4de1d5623 100644
--- a/xlators/nfs/server/src/nfs3-helpers.h
+++ b/xlators/nfs/server/src/nfs3-helpers.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS3_HELPER_H_
@@ -44,6 +35,9 @@ nfs3_extract_lookup_name (lookup3args *args);
 extern nfsstat3
 nfs3_errno_to_nfsstat3 (int errnum);
 
+extern nfsstat3
+nfs3_cbk_errno_status (int32_t, int32_t);
+
 extern void
 nfs3_fill_lookup3res (lookup3res *res, nfsstat3 stat, struct nfs3_fh *newfh,
                       struct iatt *stbuf, struct iatt *postparent,
@@ -99,7 +93,8 @@ extern void
 nfs3_prep_access3args (access3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits);
+nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits,
+		      int32_t reqaccbits);
 
 extern char *
 nfs3_fhcache_getpath (struct nfs3_state *nfs3, struct nfs3_fh *fh);
diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c
index 422646bc0..f914c3193 100644
--- a/xlators/nfs/server/src/nfs3.c
+++ b/xlators/nfs/server/src/nfs3.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -32,6 +23,7 @@
 #include "nfs3.h"
 #include "mem-pool.h"
 #include "logging.h"
+#include "nfs-common.h"
 #include "nfs-fops.h"
 #include "nfs-inodes.h"
 #include "nfs-generics.h"
@@ -216,47 +208,60 @@ out:
 }
 
 
-#define nfs3_map_fh_to_volume(nfs3state, handle, rqst, volume, status, label) \
+#define nfs3_map_fh_to_volume(nfs3state, handle, req, volume, status, label) \
         do {                                                            \
-                char buf[256];                                          \
+                char exportid[256], gfid[256];                          \
+                rpc_transport_t *trans = NULL;                          \
                 volume = nfs3_fh_to_xlator ((nfs3state), handle);       \
                 if (!volume) {                                          \
-                        uuid_unparse (handle->exportid, buf);           \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to map " \
-                                "FH to vol, exportid=%s", buf);         \
+                        uuid_unparse (handle->exportid, exportid);      \
+                        uuid_unparse (handle->gfid, gfid);              \
+                        trans = rpcsvc_request_transport (req);         \
+                        GF_LOG_OCCASIONALLY (nfs3state->occ_logger,     \
+                                GF_NFS3, GF_LOG_ERROR, "Failed to map " \
+                                "FH to vol: client=%s, exportid=%s, "   \
+                                "gfid=%s", trans->peerinfo.identifier,  \
+                                exportid, gfid);                        \
+                        GF_LOG_OCCASIONALLY (nfs3state->occ_logger,     \
+                                GF_NFS3, GF_LOG_ERROR, "Stale nfs "     \
+                                "client %s must be trying to connect to"\
+                                " a deleted volume, please unmount it.",\
+                                trans->peerinfo.identifier);            \
                         status = NFS3ERR_STALE;                         \
                         goto label;                                     \
                 } else {                                                \
-                        gf_log (GF_NFS3, GF_LOG_TRACE, "FH to Volume: %s"\
-                                ,volume->name);                         \
-                        rpcsvc_request_set_private (req, volume);   \
+                        gf_log (GF_NFS3, GF_LOG_TRACE, "FH to Volume:"  \
+                                "%s", volume->name);                    \
+                        rpcsvc_request_set_private (req, volume);       \
                 }                                                       \
         } while (0);                                                    \
 
 
 #define nfs3_validate_gluster_fh(handle, status, errlabel)              \
         do {                                                            \
-                if ((handle)) {                                         \
-                        if (!nfs3_fh_validate (handle)) {               \
-                                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad Handle");\
-                                status = NFS3ERR_BADHANDLE;             \
-                                goto errlabel;                          \
-                        }                                               \
+                if (!nfs3_fh_validate (handle)) {                       \
+                        gf_log (GF_NFS3, GF_LOG_ERROR, "Bad Handle");   \
+                        status = NFS3ERR_BADHANDLE;                     \
+                        goto errlabel;                                  \
                 }                                                       \
         } while (0)                                                     \
 
 #define nfs3_check_fh_resolve_status(cst, nfstat, erlabl)               \
         do {                                                            \
                 xlator_t *xlatorp = NULL;                               \
-                char buf[256], gfid[256];                                \
+                char buf[256], gfid[256];                               \
+                rpc_transport_t *trans = NULL;                          \
                 if ((cst)->resolve_ret < 0) {                           \
+                        trans = rpcsvc_request_transport (cst->req);    \
                         xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
                                                      &cst->resolvefh);  \
-                        uuid_unparse (cst->fh.gfid, gfid);              \
-                        sprintf (buf, "%s : %s", xlatorp ?              \
-                                 xlatorp->name : "ERR", gfid);          \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Unable to resolve FH"\
-                                ": %s", buf);                           \
+                        uuid_unparse (cst->resolvefh.gfid, gfid);       \
+                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
+                                  trans->peerinfo.identifier,           \
+                                  xlatorp ? xlatorp->name : "ERR",      \
+                                  gfid );                                \
+                        gf_log (GF_NFS3, GF_LOG_ERROR, "%s: %s",        \
+                                strerror(cst->resolve_errno), buf);     \
                         nfstat = nfs3_errno_to_nfsstat3 (cst->resolve_errno);\
                         goto erlabl;                                    \
                 }                                                       \
@@ -265,16 +270,20 @@ out:
 #define nfs3_check_new_fh_resolve_status(cst, nfstat, erlabl)           \
         do {                                                            \
                 xlator_t *xlatorp = NULL;                               \
-                char buf[256], gfid[256];                                \
+                char buf[256], gfid[256];                               \
+                rpc_transport_t *trans = NULL;                          \
                 if (((cst)->resolve_ret < 0) &&                         \
                     ((cst)->resolve_errno != ENOENT)) {                 \
+                        trans = rpcsvc_request_transport (cst->req);    \
                         xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
-                                                     &cst->resolvefh);   \
-                        uuid_unparse (cst->fh.gfid, gfid);             \
-                        sprintf (buf, "%s : %s", xlatorp ?              \
-                                 xlatorp->name : "ERR", gfid);          \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Unable to resolve FH"\
-                                ": %s", buf);                           \
+                                                     &cst->resolvefh);  \
+                        uuid_unparse (cst->resolvefh.gfid, gfid);       \
+                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
+                                  trans->peerinfo.identifier,     \
+                                  xlatorp ? xlatorp->name : "ERR",      \
+                                  gfid);         \
+                        gf_log (GF_NFS3, GF_LOG_ERROR, "%s: %s",        \
+                                strerror(cst->resolve_errno), buf);     \
                         nfstat = nfs3_errno_to_nfsstat3 (cs->resolve_errno);\
                         goto erlabl;                                    \
                 }                                                       \
@@ -435,11 +444,9 @@ nfs3_call_state_wipe (nfs3_call_state_t *cs)
                 fd_unref (cs->fd);
         }
 
-        if (cs->resolventry)
-                GF_FREE (cs->resolventry);
+        GF_FREE (cs->resolventry);
 
-        if (cs->pathname)
-                GF_FREE (cs->pathname);
+        GF_FREE (cs->pathname);
 
         if (!list_empty (&cs->entries.list))
                 gf_dirent_free (&cs->entries);
@@ -450,6 +457,8 @@ nfs3_call_state_wipe (nfs3_call_state_t *cs)
                 iobuf_unref (cs->iob);
         if (cs->iobref)
                 iobref_unref (cs->iobref);
+        if (cs->trans)
+                rpc_transport_unref (cs->trans);
         memset (cs, 0, sizeof (*cs));
         mem_put (cs);
         /* Already refd by fd_lookup, so no need to ref again. */
@@ -487,6 +496,8 @@ nfs3_serialize_reply (rpcsvc_request_t *req, void *arg, nfs3_serializer sfunc,
         /* First, get the io buffer into which the reply in arg will
          * be serialized.
          */
+        /* TODO: get rid of 'sfunc' and use 'xdrproc_t' so we
+           can have 'xdr_sizeof' */
         iob = iobuf_get (nfs3->iobpool);
         if (!iob) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to get iobuf");
@@ -538,23 +549,18 @@ nfs3svc_submit_reply (rpcsvc_request_t *req, void *arg, nfs3_serializer sfunc)
 
         iobref = iobref_new ();
         if (!iobref) {
-                iobuf_unref (iob);
                 gf_log (GF_NFS3, GF_LOG_ERROR, "failed on iobref_new()");
                 goto ret;
         }
 
-        iobref_add (iobref, iob);
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
 
         /* Then, submit the message for transmission. */
         ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
-
-        /* Now that we've done our job of handing the message to the RPC layer
-         * we can safely unref the iob in the hope that RPC layer must have
-         * ref'ed the iob on receiving into the txlist.
-         */
-        iobuf_unref (iob);
-        iobref_unref (iobref);
-
         if (ret == -1) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Reply submission failed");
                 goto ret;
@@ -562,6 +568,14 @@ nfs3svc_submit_reply (rpcsvc_request_t *req, void *arg, nfs3_serializer sfunc)
 
         ret = 0;
 ret:
+        /* Now that we've done our job of handing the message to the RPC layer
+         * we can safely unref the iob in the hope that RPC layer must have
+         * ref'ed the iob on receiving into the txlist.
+         */
+        if (NULL != iob)
+                iobuf_unref (iob);
+        if (NULL != iobref)
+                iobref_unref (iobref);
         return ret;
 }
 
@@ -593,19 +607,14 @@ nfs3svc_submit_vector_reply (rpcsvc_request_t *req, void *arg,
                 new_iobref = 1;
         }
 
-        iobref_add (iobref, iob);
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
 
         /* Then, submit the message for transmission. */
         ret = rpcsvc_submit_message (req, &outmsg, 1, payload, vcount, iobref);
-
-        /* Now that we've done our job of handing the message to the RPC layer
-         * we can safely unref the iob in the hope that RPC layer must have
-         * ref'ed the iob on receiving into the txlist.
-         */
-        iobuf_unref (iob);
-        if (new_iobref)
-                iobref_unref (iobref);
-
         if (ret == -1) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Reply submission failed");
                 goto ret;
@@ -613,6 +622,14 @@ nfs3svc_submit_vector_reply (rpcsvc_request_t *req, void *arg,
 
         ret = 0;
 ret:
+        /* Now that we've done our job of handing the message to the RPC layer
+         * we can safely unref the iob in the hope that RPC layer must have
+         * ref'ed the iob on receiving into the txlist.
+         */
+        if (NULL != iob)
+                iobuf_unref (iob);
+        if (new_iobref)
+                iobref_unref (iobref);
         return ret;
 }
 
@@ -677,11 +694,20 @@ nfs3svc_getattr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         cs = frame->local;
 
+        /*
+         * Somewhat counter-intuitively, we don't need to look for sh-failed
+         * here. Failing this getattr will generate a new lookup from the
+         * client, and nfs_fop_lookup_cbk will detect any self-heal failures.
+         */
+
         if (op_ret == -1) {
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
+        }
+        else {
+                nfs_fix_generation(this,inode);
         }
 
         nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_GETATTR,
@@ -696,7 +722,8 @@ nfs3svc_getattr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 nfs3svc_getattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          dict_t *xdata)
 {
         nfsstat3                status = NFS3_OK;
         nfs3_call_state_t       *cs = NULL;
@@ -707,7 +734,7 @@ nfs3svc_getattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
         }
 
         nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_GETATTR,
@@ -727,6 +754,9 @@ nfs3_getattr_resume (void *carg)
         int                             ret = -EFAULT;
         nfs_user_t                      nfu = {0, };
         nfs3_call_state_t               *cs = NULL;
+        uint64_t                         raw_ctx = 0;
+        struct nfs_inode_ctx            *ictx = NULL;
+        struct nfs_state                *priv = NULL;
 
         if (!carg)
                 return ret;
@@ -753,9 +783,28 @@ nfs3_getattr_resume (void *carg)
 		goto nfs3err;
 	}
 
+        /*
+         * If brick state changed, we need to force a proper lookup cycle (as
+         * would happen in native protocol) to do self-heal checks. We detect
+         * this by comparing the generation number for the last successful
+         * creation/lookup on the inode to the current number, so inodes that
+         * haven't been validated since the state change are affected.
+         */
+        if (inode_ctx_get(cs->resolvedloc.inode,cs->nfsx,&raw_ctx) == 0) {
+                ictx = (struct nfs_inode_ctx *)raw_ctx;
+                priv = cs->nfsx->private;
+                if (ictx->generation != priv->generation) {
+                        ret = nfs_lookup (cs->nfsx, cs->vol, &nfu,
+                                          &cs->resolvedloc,
+                                          nfs3svc_getattr_lookup_cbk, cs);
+                        goto check_err;
+                }
+        }
+
         ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
                         nfs3svc_getattr_stat_cbk, cs);
 
+check_err:
         if (ret < 0) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Stat fop failed: %s: %s",
                         cs->oploc.path, strerror (-ret));
@@ -858,7 +907,7 @@ nfs3_setattr_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preop,
 int32_t
 nfs3svc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         struct iatt             *prestat = NULL;
@@ -869,7 +918,7 @@ nfs3svc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -896,7 +945,7 @@ nfs3err:
 int32_t
 nfs3svc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *preop,
-                     struct iatt *postop)
+                     struct iatt *postop, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     ret = -1;
@@ -909,7 +958,7 @@ nfs3svc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -923,7 +972,8 @@ nfs3svc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
          * truncation and also only if this is not a directory.
          */
         if ((gf_attr_size_set (cs->setattr_valid)) &&
-            (!IA_ISDIR (postop->ia_type))) {
+            (!IA_ISDIR (postop->ia_type)) &&
+            (preop->ia_size != cs->stbuf.ia_size)) {
                 nfs_request_user_init (&nfu, cs->req);
                 ret = nfs_truncate (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
                                     cs->stbuf.ia_size, nfs3svc_truncate_cbk,cs);
@@ -950,7 +1000,8 @@ nfs3err:
 
 int32_t
 nfs3svc_setattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          dict_t *xdata)
 {
 
         int                     ret = -EFAULT;
@@ -963,7 +1014,7 @@ nfs3svc_setattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -1177,7 +1228,7 @@ nfs3svc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_WARNING),
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto xmit_res;
         }
 
@@ -1221,7 +1272,7 @@ nfs3svc_lookup_parentdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto xmit_res;
         }
 
@@ -1263,7 +1314,11 @@ nfs3_lookup_parentdir_resume (void *carg)
         nfs3_call_state_t               *cs = NULL;
         inode_t                         *parent = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, carg, nfs3err);
+        if (!carg) {
+                gf_log (GF_NFS3, GF_LOG_ERROR, "Invalid argument,"
+                        " carg value NULL");
+                return EINVAL;
+        }
 
         cs = (nfs3_call_state_t *)carg;
         nfs3_check_fh_resolve_status (cs, stat, nfs3err);
@@ -1334,10 +1389,16 @@ nfs3_lookup_resume (void *carg)
         nfs3_call_state_t               *cs = NULL;
         struct nfs3_fh                  newfh = {{0},};
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, carg, nfs3err);
+        if (!carg) {
+                gf_log (GF_NFS3, GF_LOG_ERROR, "Invalid argument,"
+                        " carg value NULL");
+                return EINVAL;
+        }
 
         cs = (nfs3_call_state_t *)carg;
         nfs3_check_fh_resolve_status (cs, stat, nfs3err);
+        cs->parent = cs->resolvefh;
+
 	if (cs->hardresolved) {
 		stat = NFS3_OK;
 		nfs3_fh_build_child_fh (&cs->parent, &cs->stbuf, &newfh);
@@ -1345,7 +1406,6 @@ nfs3_lookup_resume (void *carg)
 	}
 
         nfs_request_user_init (&nfu, cs->req);
-        cs->parent = cs->resolvefh;
         ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
                           nfs3svc_lookup_cbk, cs);
         if (ret < 0)
@@ -1390,12 +1450,8 @@ nfs3_lookup (rpcsvc_request_t *req, struct nfs3_fh *fh, int fhlen, char *name)
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
         cs->lookuptype = GF_NFS3_REVALIDATE;
-        if (!nfs3_is_parentdir_entry (name))
-                ret = nfs3_fh_resolve_and_resume (cs, fh, name,
-                                                  nfs3_lookup_resume);
-        else
-                ret = nfs3_fh_resolve_and_resume (cs, fh, NULL,
-                                                  nfs3_lookup_parentdir_resume);
+        ret = nfs3_fh_resolve_and_resume (cs, fh, name,
+                                          nfs3_lookup_resume);
 
         if (ret < 0) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "failed to start hard reslove");
@@ -1449,11 +1505,12 @@ rpcerr:
 
 
 int
-nfs3_access_reply (rpcsvc_request_t *req, nfsstat3 status, int32_t accbits)
+nfs3_access_reply (rpcsvc_request_t *req, nfsstat3 status, int32_t accbits,
+		   int32_t reqaccbits)
 {
         access3res      res;
 
-        nfs3_fill_access3res (&res, status, accbits);
+        nfs3_fill_access3res (&res, status, accbits, reqaccbits);
         nfs3svc_submit_reply (req, &res,
                               (nfs3_serializer)xdr_serialize_access3res);
         return 0;
@@ -1462,7 +1519,7 @@ nfs3_access_reply (rpcsvc_request_t *req, nfsstat3 status, int32_t accbits)
 
 int32_t
 nfs3svc_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         nfsstat3                status = NFS3_OK;
         nfs3_call_state_t       *cs = NULL;
@@ -1473,11 +1530,11 @@ nfs3svc_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
         }
         nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_ACCESS, status,
                              op_errno);
-        nfs3_access_reply (cs->req, status, op_errno);
+        nfs3_access_reply (cs->req, status, op_errno, cs->accessbits);
         nfs3_call_state_wipe (cs);
 
         return 0;
@@ -1491,7 +1548,11 @@ nfs3_access_resume (void *carg)
         nfs_user_t              nfu = {0, };
         nfs3_call_state_t       *cs = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, carg, nfs3err);
+        if (!carg) {
+                gf_log (GF_NFS3, GF_LOG_ERROR, "Invalid argument,"
+                        " carg value NULL");
+                return EINVAL;
+        }
 
         cs = (nfs3_call_state_t *)carg;
         nfs3_check_fh_resolve_status (cs, stat, nfs3err);
@@ -1506,7 +1567,7 @@ nfs3err:
         if (ret < 0) {
                 nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_ACCESS,
                                      stat, -ret);
-                nfs3_access_reply (cs->req, stat, 0);
+                nfs3_access_reply (cs->req, stat, 0, 0);
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
@@ -1542,7 +1603,7 @@ nfs3err:
         if (ret < 0) {
                 nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_ACCESS,
                                      stat, -ret);
-                nfs3_access_reply (req, stat, 0);
+                nfs3_access_reply (req, stat, 0, 0);
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
@@ -1599,7 +1660,7 @@ nfs3_readlink_reply (rpcsvc_request_t *req, nfsstat3 stat, char *path,
 int32_t
 nfs3svc_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf)
+                      struct iatt *buf, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -1609,7 +1670,7 @@ nfs3svc_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -1764,7 +1825,8 @@ nfs3_read_reply (rpcsvc_request_t *req, nfsstat3 stat, count3 count,
 int32_t
 nfs3svc_read_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                  int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     is_eof = 0;
@@ -1775,7 +1837,7 @@ nfs3svc_read_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto err;
         } else
                 stat = NFS3_OK;
@@ -1948,7 +2010,7 @@ nfs3_write_reply (rpcsvc_request_t *req, nfsstat3 stat, count3 count,
 int32_t
 nfs3svc_write_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf)
+                         struct iatt *postbuf, dict_t *xdata)
 {
         struct nfs3_state       *nfs3 = NULL;
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
@@ -1961,7 +2023,7 @@ nfs3svc_write_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else
                 stat = NFS3_OK;
 
@@ -1974,41 +2036,6 @@ nfs3svc_write_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 
-/*
- * If this logic determines that the write should return a reply to the client
- * after this function, the return value is -1 and the writetype is reset to
- * the type of write we want to signify to the client.
- *
- * In case the write should continue to serve the request according to the type
- * of stable write, a 0 is returned and writetype is left as it is.
- */
-int
-nfs3_write_how (int *writetype, int write_trusted, int sync_trusted)
-{
-        int     ret = -1;
-
-        if (*writetype == UNSTABLE) {
-                /* On an UNSTABLE write, only return STABLE when trusted-write
-                 * is set. TW is also set when trusted-sync is set.
-                 */
-                if (write_trusted)
-                        *writetype = FILE_SYNC;
-
-                goto err;
-        } else if ((*writetype == DATA_SYNC) || (*writetype == FILE_SYNC)) {
-
-                /* On a STABLE write, if sync-trusted is on, only then, return
-                 * without syncing.
-                 */
-                if (sync_trusted)
-                        goto err;
-        }
-
-        ret = 0;
-err:
-        return ret;
-}
-
 
 /*
  * Before going into the write reply logic, here is a matrix that shows the
@@ -2044,15 +2071,11 @@ err:
 int32_t
 nfs3svc_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs_user_t              nfu = {0, };
         nfs3_call_state_t       *cs = NULL;
         struct nfs3_state       *nfs3 = NULL;
-        int                     write_trusted = 0;
-        int                     sync_trusted = 0;
 
         cs = frame->local;
         nfs3 = rpcsvc_request_program_private (cs->req);
@@ -2060,42 +2083,21 @@ nfs3svc_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto err;
         }
 
         stat = NFS3_OK;
         cs->maxcount = op_ret;
 
-        write_trusted = nfs3_export_write_trusted (cs->nfs3state,
-                                                   cs->resolvefh.exportid);
-        sync_trusted = nfs3_export_sync_trusted (cs->nfs3state,
-                                                 cs->resolvefh.exportid);
-        ret = nfs3_write_how (&cs->writetype, write_trusted, sync_trusted);
-        if (ret == -1)
-                goto err;
-
-        nfs_request_user_init (&nfu, cs->req);
-        /* Store the current preattr so that this can be used as the pre attr
-         * when fsync returns. We dont want to use the preattr in fsync because
-         * the write fop happened before the fsync.
-         */
-        cs->stbuf = *prebuf;
-        ret = nfs_fsync (cs->nfsx, cs->vol, &nfu, cs->fd, 0,
-                         nfs3svc_write_fsync_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
-
 err:
-        if (ret < 0) {
-                nfs3_log_write_res (rpcsvc_request_xid (cs->req), stat,
-                                    op_errno, cs->maxcount, cs->writetype,
-                                    nfs3->serverstart);
-                nfs3_write_reply (cs->req, stat, cs->maxcount,
-                                  cs->writetype, nfs3->serverstart, prebuf,
-                                  postbuf);
-                nfs3_call_state_wipe (cs);
-        }
+	nfs3_log_write_res (rpcsvc_request_xid (cs->req), stat,
+			    op_errno, cs->maxcount, cs->writetype,
+			    nfs3->serverstart);
+	nfs3_write_reply (cs->req, stat, cs->maxcount,
+			  cs->writetype, nfs3->serverstart, prebuf,
+			  postbuf);
+	nfs3_call_state_wipe (cs);
 
         return 0;
 }
@@ -2151,6 +2153,25 @@ nfs3_write_resume (void *carg)
         }
 
         cs->fd = fd;    /* Gets unrefd when the call state is wiped. */
+
+/*
+  enum stable_how {
+  UNSTABLE = 0,
+  DATA_SYNC = 1,
+  FILE_SYNC = 2,
+  };
+*/
+	switch (cs->writetype) {
+	case UNSTABLE:
+		break;
+	case DATA_SYNC:
+		fd->flags |= O_DSYNC;
+		break;
+	case FILE_SYNC:
+		fd->flags |= O_SYNC;
+		break;
+	}
+
         ret = __nfs3_write_resume (cs);
         if (ret < 0)
                 stat = nfs3_errno_to_nfsstat3 (-ret);
@@ -2221,17 +2242,18 @@ out:
 
 
 int
-nfs3svc_write_vecsizer (int state, ssize_t *readsize, char *addr)
+nfs3svc_write_vecsizer (int state, ssize_t *readsize, char *base_addr,
+                        char *curr_addr)
 {
-        int         ret = 0;
-        uint32_t        fhlen = 0;
-        uint32_t        fhlen_n = 0;
+        int      ret     = 0;
+        uint32_t fhlen   = 0;
+        uint32_t fhlen_n = 0;
 
         if (state == 0) {
                 ret = NFS3_VECWRITE_READFHLEN;
                 *readsize = 4;
         } else if (state == NFS3_VECWRITE_READFHLEN) {
-                fhlen_n = *(uint32_t *)(addr - 4);
+                fhlen_n = *(uint32_t *)(curr_addr - 4);
                 fhlen = ntohl (fhlen_n);
                 *readsize = xdr_length_round_up (fhlen, NFS3_FHSIZE);
                 ret = NFS3_VECWRITE_READFH;
@@ -2282,12 +2304,6 @@ rpcerr:
         return ret;
 }
 
-int
-nfs3svc_write_vec (rpcsvc_request_t *req, struct iovec *payload,
-                   int payload_count, struct iobref *iobref)
-{
-        return nfs3svc_write (req);
-}
 
 int
 nfs3_create_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *newfh,
@@ -2309,7 +2325,7 @@ nfs3_create_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *newfh,
 int32_t
 nfs3svc_create_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                             int32_t op_ret, int32_t op_errno,
-                            struct iatt *preop, struct iatt *postop)
+                            struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -2319,7 +2335,7 @@ nfs3svc_create_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -2339,7 +2355,7 @@ int32_t
 nfs3svc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     ret = -EFAULT;
@@ -2352,7 +2368,7 @@ nfs3svc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -2442,7 +2458,8 @@ nfs3_create_common (nfs3_call_state_t *cs)
 
 int32_t
 nfs3svc_create_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                         dict_t *xdata)
 {
         int                     ret = -EFAULT;
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
@@ -2456,7 +2473,7 @@ nfs3svc_create_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
                 ret = -op_errno;
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -2627,13 +2644,7 @@ nfs3svc_create (rpcsvc_request_t *req)
         }
 
         cval = (uint64_t *)args.how.createhow3_u.verf;
-        if (cval)
-                cverf = *cval;
-        else {
-                gf_log(GF_NFS3, GF_LOG_ERROR,
-                       "Error getting createverf3 from args");
-                goto rpcerr;
-        }
+        cverf = *cval;
 
         ret = nfs3_create (req, &dirfh, name, args.how.mode,
                            &args.how.createhow3_u.obj_attributes, cverf);
@@ -2667,7 +2678,7 @@ nfs3_mkdir_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *fh,
 int32_t
 nfs3svc_mkdir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            int32_t op_ret, int32_t op_errno,
-                           struct iatt *preop, struct iatt *postop)
+                           struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -2677,7 +2688,7 @@ nfs3svc_mkdir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -2697,7 +2708,7 @@ int32_t
 nfs3svc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                        stat = NFS3ERR_SERVERFAULT;
         int                             ret = -EFAULT;
@@ -2709,7 +2720,7 @@ nfs3svc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -2877,7 +2888,7 @@ int32_t
 nfs3svc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                        stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t               *cs = NULL;
@@ -2887,7 +2898,7 @@ nfs3svc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -3039,7 +3050,7 @@ nfs3_mknod_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *fh,
 int32_t
 nfs3svc_mknod_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            int32_t op_ret, int32_t op_errno,
-                           struct iatt *preop, struct iatt *postop)
+                           struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -3049,7 +3060,7 @@ nfs3svc_mknod_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -3069,7 +3080,7 @@ int32_t
 nfs3svc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                        stat = NFS3ERR_SERVERFAULT;
         int                             ret = -1;
@@ -3081,7 +3092,7 @@ nfs3svc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -3329,7 +3340,7 @@ nfs3_remove_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preparent
 int32_t
 nfs3svc_remove_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -3339,7 +3350,7 @@ nfs3svc_remove_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         }
 
         if (op_ret == 0)
@@ -3495,7 +3506,7 @@ nfs3_rmdir_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preparent,
 int32_t
 nfs3svc_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -3505,7 +3516,7 @@ nfs3svc_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else {
                 stat = NFS3_OK;
         }
@@ -3647,7 +3658,8 @@ int32_t
 nfs3svc_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
                     struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent)
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
         int                     ret = -EFAULT;
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
@@ -3659,7 +3671,7 @@ nfs3svc_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%x: rename %s -> %s => -1 (%s)",
                         rpcsvc_request_xid (cs->req), cs->oploc.path,
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -3853,7 +3865,7 @@ int32_t
 nfs3svc_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -3864,7 +3876,7 @@ nfs3svc_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%x: link %s <- %s => -1 (%s)",
                         rpcsvc_request_xid (cs->req), cs->oploc.path,
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else
                 stat = NFS3_OK;
 
@@ -4062,7 +4074,8 @@ nfs3_readdir_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *dirfh,
 
 int32_t
 nfs3svc_readdir_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                           int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                           dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     is_eof = 0;
@@ -4073,7 +4086,7 @@ nfs3svc_readdir_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -4115,7 +4128,8 @@ nfs3err:
 
 int32_t
 nfs3svc_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     ret = -EFAULT;
@@ -4127,7 +4141,7 @@ nfs3svc_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto err;
         }
 
@@ -4224,12 +4238,30 @@ nfs3err:
 }
 
 
+int32_t
+nfs3svc_readdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, fd_t *fd,
+                             dict_t *xdata)
+{
+        /*
+         * We don't really need this, it's just an artifact of forcing the
+         * opendir to happen.
+         */
+        if (fd) {
+                fd_unref(fd);
+        }
+
+        return 0;
+}
+
+
 int
 nfs3_readdir_open_resume (void *carg)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     ret = -EFAULT;
         nfs3_call_state_t       *cs = NULL;
+        nfs_user_t               nfu = {0, };
 
         if (!carg)
                 return ret;
@@ -4242,6 +4274,22 @@ nfs3_readdir_open_resume (void *carg)
                 goto nfs3err;
         }
 
+        /*
+         * NFS client will usually send us a readdirp without an opendir,
+         * which would cause us to skip our usual self-heal checks which occur
+         * in opendir for native protocol. To make sure those checks do happen,
+         * our most reliable option is to do our own opendir for any readdirp
+         * at the beginning of the directory.
+         */
+        if (cs->cookie == 0) {
+                nfs_request_user_init (&nfu, cs->req);
+                ret = nfs_opendir (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                                   nfs3svc_readdir_opendir_cbk, cs);
+                if (ret < 0) {
+                        gf_log (GF_NFS3, GF_LOG_ERROR, "auto-opendir failed");
+                }
+        }
+
         ret = nfs3_readdir_read_resume (cs);
         if (ret < 0)
                 stat = nfs3_errno_to_nfsstat3 (-ret);
@@ -4341,16 +4389,8 @@ nfs3svc_readdir (rpcsvc_request_t *req)
                 rpcsvc_request_seterr (req, GARBAGE_ARGS);
                 goto rpcerr;
         }
-
         cval = (uint64_t *) ra.cookieverf;
-
-        if (cval)
-                verf =  *cval;
-        else {
-                gf_log(GF_NFS3, GF_LOG_ERROR,
-                       "Error getting cookieverf from readdir args");
-                goto rpcerr;
-        }
+        verf =  *cval;
 
         ret = nfs3_readdir (req, &fh, ra.cookie, verf, ra.count, 0);
         if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
@@ -4381,16 +4421,8 @@ nfs3svc_readdirp (rpcsvc_request_t *req)
                 rpcsvc_request_seterr (req, GARBAGE_ARGS);
                 goto rpcerr;
         }
-
         cval = (uint64_t *) ra.cookieverf;
-
-        if (cval)
-                cverf = *cval;
-        else {
-                gf_log (GF_NFS3, GF_LOG_ERROR,
-                        "Error getting cookieverf from readdirp args");
-                goto rpcerr;
-	}
+        cverf = *cval;
 
         ret = nfs3_readdir (req, &fh, ra.cookie, cverf, ra.dircount,
                             ra.maxcount);
@@ -4422,7 +4454,8 @@ nfs3_fsstat_reply (rpcsvc_request_t *req, nfsstat3 stat, struct statvfs *fsbuf,
 
 int32_t
 nfs3_fsstat_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                      dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -4432,7 +4465,7 @@ nfs3_fsstat_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else
                 stat = NFS3_OK;
 
@@ -4446,7 +4479,8 @@ nfs3_fsstat_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 nfs3_fsstat_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                        int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                        dict_t *xdata)
 {
         nfs_user_t              nfu = {0, };
         int                     ret = -EFAULT;
@@ -4459,7 +4493,7 @@ nfs3_fsstat_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
                 ret = -op_errno;
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto err;
         }
 
@@ -4606,7 +4640,8 @@ nfs3_fsinfo_reply (rpcsvc_request_t *req, nfsstat3 status, struct iatt *fsroot)
 
 int32_t
 nfs3svc_fsinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    dict_t *xdata)
 {
         nfsstat3                status = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -4617,7 +4652,7 @@ nfs3svc_fsinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
         }else
                 status = NFS3_OK;
 
@@ -4747,7 +4782,8 @@ nfs3_pathconf_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *buf)
 
 int32_t
 nfs3svc_pathconf_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                      dict_t *xdata)
 {
         struct iatt             *sbuf = NULL;
         nfs3_call_state_t       *cs = NULL;
@@ -4758,7 +4794,7 @@ nfs3svc_pathconf_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else {
                 /* If stat fop failed, we can still send the other components
                  * in a pathconf reply.
@@ -4891,7 +4927,7 @@ nfs3_commit_reply (rpcsvc_request_t *req, nfsstat3 stat, uint64_t wverf,
 
 int32_t
 nfs3svc_commit_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -4902,7 +4938,7 @@ nfs3svc_commit_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else
                 stat = NFS3_OK;
 
@@ -5063,28 +5099,28 @@ rpcerr:
 
 
 rpcsvc_actor_t          nfs3svc_actors[NFS3_PROC_COUNT] = {
-        {"NULL",        NFS3_NULL,      nfs3svc_null,   NULL,   NULL, 0},
-        {"GETATTR",     NFS3_GETATTR,   nfs3svc_getattr,NULL,   NULL, 0},
-        {"SETATTR",     NFS3_SETATTR,   nfs3svc_setattr,NULL,   NULL, 0},
-        {"LOOKUP",      NFS3_LOOKUP,    nfs3svc_lookup, NULL,   NULL, 0},
-        {"ACCESS",      NFS3_ACCESS,    nfs3svc_access, NULL,   NULL, 0},
-        {"READLINK",    NFS3_READLINK,  nfs3svc_readlink,NULL,  NULL, 0},
-        {"READ",        NFS3_READ,      nfs3svc_read,   NULL,   NULL, 0},
-        {"WRITE",       NFS3_WRITE,     nfs3svc_write, nfs3svc_write_vec, nfs3svc_write_vecsizer, 0},
-        {"CREATE",      NFS3_CREATE,    nfs3svc_create, NULL,   NULL, 0},
-        {"MKDIR",       NFS3_MKDIR,     nfs3svc_mkdir,  NULL,   NULL, 0},
-        {"SYMLINK",     NFS3_SYMLINK,   nfs3svc_symlink,NULL,   NULL, 0},
-        {"MKNOD",       NFS3_MKNOD,     nfs3svc_mknod,  NULL,   NULL, 0},
-        {"REMOVE",      NFS3_REMOVE,    nfs3svc_remove, NULL,   NULL, 0},
-        {"RMDIR",       NFS3_RMDIR,     nfs3svc_rmdir,  NULL,   NULL, 0},
-        {"RENAME",      NFS3_RENAME,    nfs3svc_rename, NULL,   NULL, 0},
-        {"LINK",        NFS3_LINK,      nfs3svc_link,   NULL,   NULL, 0},
-        {"READDIR",     NFS3_READDIR,   nfs3svc_readdir,NULL,   NULL, 0},
-        {"READDIRPLUS", NFS3_READDIRP,  nfs3svc_readdirp,NULL,  NULL, 0},
-        {"FSSTAT",      NFS3_FSSTAT,    nfs3svc_fsstat, NULL,   NULL, 0},
-        {"FSINFO",      NFS3_FSINFO,    nfs3svc_fsinfo, NULL,   NULL, 0},
-        {"PATHCONF",    NFS3_PATHCONF,  nfs3svc_pathconf,NULL,  NULL, 0},
-        {"COMMIT",      NFS3_COMMIT,    nfs3svc_commit, NULL,   NULL, 0}
+        {"NULL",        NFS3_NULL,      nfs3svc_null,      NULL,   0, DRC_IDEMPOTENT},
+        {"GETATTR",     NFS3_GETATTR,   nfs3svc_getattr,   NULL,   0, DRC_IDEMPOTENT},
+        {"SETATTR",     NFS3_SETATTR,   nfs3svc_setattr,   NULL,   0, DRC_NON_IDEMPOTENT},
+        {"LOOKUP",      NFS3_LOOKUP,    nfs3svc_lookup,    NULL,   0, DRC_IDEMPOTENT},
+        {"ACCESS",      NFS3_ACCESS,    nfs3svc_access,    NULL,   0, DRC_IDEMPOTENT},
+        {"READLINK",    NFS3_READLINK,  nfs3svc_readlink,  NULL,   0, DRC_IDEMPOTENT},
+        {"READ",        NFS3_READ,      nfs3svc_read,      NULL,   0, DRC_IDEMPOTENT},
+        {"WRITE",       NFS3_WRITE,     nfs3svc_write,     nfs3svc_write_vecsizer, 0, DRC_NON_IDEMPOTENT},
+        {"CREATE",      NFS3_CREATE,    nfs3svc_create,    NULL,   0, DRC_NON_IDEMPOTENT},
+        {"MKDIR",       NFS3_MKDIR,     nfs3svc_mkdir,     NULL,   0, DRC_NON_IDEMPOTENT},
+        {"SYMLINK",     NFS3_SYMLINK,   nfs3svc_symlink,   NULL,   0, DRC_NON_IDEMPOTENT},
+        {"MKNOD",       NFS3_MKNOD,     nfs3svc_mknod,     NULL,   0, DRC_NON_IDEMPOTENT},
+        {"REMOVE",      NFS3_REMOVE,    nfs3svc_remove,    NULL,   0, DRC_NON_IDEMPOTENT},
+        {"RMDIR",       NFS3_RMDIR,     nfs3svc_rmdir,     NULL,   0, DRC_NON_IDEMPOTENT},
+        {"RENAME",      NFS3_RENAME,    nfs3svc_rename,    NULL,   0, DRC_NON_IDEMPOTENT},
+        {"LINK",        NFS3_LINK,      nfs3svc_link,      NULL,   0, DRC_NON_IDEMPOTENT},
+        {"READDIR",     NFS3_READDIR,   nfs3svc_readdir,   NULL,   0, DRC_IDEMPOTENT},
+        {"READDIRPLUS", NFS3_READDIRP,  nfs3svc_readdirp,  NULL,   0, DRC_IDEMPOTENT},
+        {"FSSTAT",      NFS3_FSSTAT,    nfs3svc_fsstat,    NULL,   0, DRC_IDEMPOTENT},
+        {"FSINFO",      NFS3_FSINFO,    nfs3svc_fsinfo,    NULL,   0, DRC_IDEMPOTENT},
+        {"PATHCONF",    NFS3_PATHCONF,  nfs3svc_pathconf,  NULL,   0, DRC_IDEMPOTENT},
+        {"COMMIT",      NFS3_COMMIT,    nfs3svc_commit,    NULL,   0, DRC_IDEMPOTENT}
 };
 
 
@@ -5101,21 +5137,48 @@ rpcsvc_program_t        nfs3prog = {
                         .min_auth       = AUTH_NULL,
 };
 
+/*
+ * This function rounds up the input value to multiple of 4096. Min and Max
+ * supported I/O size limits are 4KB (GF_NFS3_FILE_IO_SIZE_MIN) and
+ * 1MB (GF_NFS3_FILE_IO_SIZE_MAX).
+ */
+void
+nfs3_iosize_roundup_4KB (uint64_t *ioszptr)
+{
+        uint64_t iosize;
+        uint64_t iopages;
+
+        if (!ioszptr)
+                return;
+
+        iosize  = *ioszptr;
+        iopages = (iosize + GF_NFS3_IO_SIZE -1) >> GF_NFS3_IO_SHIFT;
+        iosize  = (iopages * GF_NFS3_IO_SIZE);
+
+        /* Double check - boundary conditions */
+        if (iosize < GF_NFS3_FILE_IO_SIZE_MIN) {
+                iosize = GF_NFS3_FILE_IO_SIZE_MIN;
+        } else if (iosize > GF_NFS3_FILE_IO_SIZE_MAX) {
+                iosize = GF_NFS3_FILE_IO_SIZE_MAX;
+        }
+
+        *ioszptr = iosize;
+}
 
 int
-nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx)
+nfs3_init_options (struct nfs3_state *nfs3, dict_t *options)
 {
         int      ret = -1;
         char     *optstr = NULL;
         uint64_t size64 = 0;
 
-        if ((!nfs3) || (!nfsx))
+        if ((!nfs3) || (!options))
                 return -1;
 
         /* nfs3.read-size */
         nfs3->readsize = GF_NFS3_RTPREF;
-        if (dict_get (nfsx->options, "nfs3.read-size")) {
-                ret = dict_get_str (nfsx->options, "nfs3.read-size", &optstr);
+        if (dict_get (options, "nfs3.read-size")) {
+                ret = dict_get_str (options, "nfs3.read-size", &optstr);
                 if (ret < 0) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read "
                                 " option: nfs3.read-size");
@@ -5124,19 +5187,21 @@ nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx)
                 }
 
                 ret = gf_string2bytesize (optstr, &size64);
-                nfs3->readsize = size64;
                 if (ret == -1) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format"
                                 " option: nfs3.read-size");
                         ret = -1;
                         goto err;
                 }
+
+                nfs3_iosize_roundup_4KB (&size64);
+                nfs3->readsize = size64;
         }
 
         /* nfs3.write-size */
         nfs3->writesize = GF_NFS3_WTPREF;
-        if (dict_get (nfsx->options, "nfs3.write-size")) {
-                ret = dict_get_str (nfsx->options, "nfs3.write-size", &optstr);
+        if (dict_get (options, "nfs3.write-size")) {
+                ret = dict_get_str (options, "nfs3.write-size", &optstr);
                 if (ret < 0) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read "
                                 " option: nfs3.write-size");
@@ -5145,19 +5210,21 @@ nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx)
                 }
 
                 ret = gf_string2bytesize (optstr, &size64);
-                nfs3->writesize = size64;
                 if (ret == -1) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format"
                                 " option: nfs3.write-size");
                         ret = -1;
                         goto err;
                 }
+
+                nfs3_iosize_roundup_4KB (&size64);
+                nfs3->writesize = size64;
         }
 
         /* nfs3.readdir.size */
         nfs3->readdirsize = GF_NFS3_DTPREF;
-        if (dict_get (nfsx->options, "nfs3.readdir-size")) {
-                ret = dict_get_str (nfsx->options,"nfs3.readdir-size", &optstr);
+        if (dict_get (options, "nfs3.readdir-size")) {
+                ret = dict_get_str (options,"nfs3.readdir-size", &optstr);
                 if (ret < 0) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read"
                                 " option: nfs3.readdir-size");
@@ -5166,15 +5233,16 @@ nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx)
                 }
 
                 ret = gf_string2bytesize (optstr, &size64);
-                nfs3->readdirsize = size64;
                 if (ret == -1) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format"
                                 " option: nfs3.readdir-size");
                         ret = -1;
                         goto err;
                 }
-        }
 
+                nfs3_iosize_roundup_4KB (&size64);
+                nfs3->readdirsize = size64;
+        }
 
         /* We want to use the size of the biggest param for the io buffer size.
          */
@@ -5195,9 +5263,10 @@ err:
         return ret;
 }
 
-
 int
-nfs3_init_subvolume_options (struct nfs3_state *nfs3, struct nfs3_export *exp)
+nfs3_init_subvolume_options (xlator_t *nfsx,
+                             struct nfs3_export *exp,
+                             dict_t *options)
 {
         int             ret = -1;
         char            *optstr = NULL;
@@ -5205,14 +5274,20 @@ nfs3_init_subvolume_options (struct nfs3_state *nfs3, struct nfs3_export *exp)
         char            *name = NULL;
         gf_boolean_t    boolt = _gf_false;
         uuid_t          volumeid = {0, };
-        dict_t          *options = NULL;
 
-        if ((!exp) || (!nfs3))
+        if ((!nfsx) || (!exp))
                 return -1;
 
-        options = nfs3->nfsx->options;
+        /* For init, fetch options from xlator but for
+         * reconfigure, take the parameter */
+        if (!options)
+                options = nfsx->options;
+
+        if (!options)
+                return (-1);
+
         uuid_clear (volumeid);
-        if (gf_nfs_dvm_off (nfs_state (nfs3->nfsx)))
+        if (gf_nfs_dvm_off (nfs_state (nfsx)))
                 goto no_dvm;
 
         ret = snprintf (searchkey, 1024, "nfs3.%s.volume-id",exp->subvol->name);
@@ -5378,7 +5453,7 @@ nfs3_init_subvolume (struct nfs3_state *nfs3, xlator_t *subvol)
         INIT_LIST_HEAD (&exp->explist);
         gf_log (GF_NFS3, GF_LOG_TRACE, "Initing state: %s", exp->subvol->name);
 
-        ret = nfs3_init_subvolume_options (nfs3, exp);
+        ret = nfs3_init_subvolume_options (nfs3->nfsx, exp, NULL);
         if (ret == -1) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to init subvol");
                 goto exp_free;
@@ -5443,7 +5518,7 @@ nfs3_init_state (xlator_t *nfsx)
         }
 
         nfs = nfsx->private;
-        ret = nfs3_init_options (nfs3, nfsx);
+        ret = nfs3_init_options (nfs3, nfsx->options);
         if (ret == -1) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to init options");
                 goto ret;
@@ -5475,12 +5550,13 @@ nfs3_init_state (xlator_t *nfsx)
         LOCK_INIT (&nfs3->fdlrulock);
         nfs3->fdcount = 0;
 
-        rpcsvc_create_listeners (nfs->rpcsvc, nfsx->options, nfsx->name);
+        ret = rpcsvc_create_listeners (nfs->rpcsvc, nfsx->options, nfsx->name);
         if (ret == -1) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "Unable to create listeners");
                 goto free_localpool;
         }
 
+        nfs->nfs3state = nfs3;
         ret = 0;
 
 free_localpool:
@@ -5516,4 +5592,39 @@ nfs3svc_init (xlator_t *nfsx)
         return &nfs3prog;
 }
 
+int
+nfs3_reconfigure_state (xlator_t *nfsx, dict_t *options)
+{
+        int                   ret   = -1;
+        struct nfs3_export    *exp  = NULL;
+        struct nfs_state      *nfs  = NULL;
+        struct nfs3_state     *nfs3 = NULL;
+
+        if ((!nfsx) || (!nfsx->private) || (!options))
+                goto out;
 
+        nfs = (struct nfs_state *)nfsx->private;
+        nfs3 = nfs->nfs3state;
+        if (!nfs3)
+                goto out;
+
+        ret = nfs3_init_options (nfs3, options);
+        if (ret) {
+                gf_log (GF_NFS3, GF_LOG_ERROR,
+                        "Failed to reconfigure options");
+                goto out;
+        }
+
+        list_for_each_entry (exp, &nfs3->exports, explist) {
+                ret = nfs3_init_subvolume_options (nfsx, exp, options);
+                if (ret) {
+                        gf_log (GF_NFS3, GF_LOG_ERROR,
+                                "Failed to reconfigure subvol options");
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/xlators/nfs/server/src/nfs3.h b/xlators/nfs/server/src/nfs3.h
index 3e99184c7..0c35445a4 100644
--- a/xlators/nfs/server/src/nfs3.h
+++ b/xlators/nfs/server/src/nfs3.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS3_H_
@@ -34,11 +25,12 @@
 #include "nfs-common.h"
 #include "xdr-nfs3.h"
 #include "mem-pool.h"
-
+#include "nlm4.h"
+#include "acl3-xdr.h"
+#include "acl3.h"
 #include <sys/statvfs.h>
 
 #define GF_NFS3                 GF_NFS"-nfsv3"
-#define GF_NFS3_PORT            38467
 
 #define GF_NFS3_DEFAULT_MEMFACTOR       15
 #define GF_NFS3_IOBPOOL_MULT            GF_NFS_CONCURRENT_OPS_MULT
@@ -47,16 +39,39 @@
 
 
 /* Static values used for FSINFO
-FIXME: This should be configurable */
-#define GF_NFS3_RTMAX      (64 * GF_UNIT_KB)
-#define GF_NFS3_RTPREF     (64 * GF_UNIT_KB)
-#define GF_NFS3_RTMULT     (4 * GF_UNIT_KB)
-#define GF_NFS3_WTMAX      (64 * GF_UNIT_KB)
-#define GF_NFS3_WTPREF     (64 * GF_UNIT_KB)
-#define GF_NFS3_WTMULT     (4 * GF_UNIT_KB)
-#define GF_NFS3_DTMIN      (4 * GF_UNIT_KB)
-#define GF_NFS3_DTPREF     (64 * GF_UNIT_KB)
-#define GF_NFS3_MAXFILE    (1 * GF_UNIT_PB)
+ * To change the maximum rsize and wsize supported by the NFS client, adjust
+ * GF_NFS3_FILE_IO_SIZE_MAX. The Gluster NFS server defaults to 1MB(1048576)
+ * (same as kernel NFS server). For slower network, rsize/wsize can be trimmed
+ * to 16/32/64-KB. rsize and wsize can be tuned through nfs.read-size and
+ * nfs.write-size respectively.
+ *
+ * NB: For Kernel-NFS, NFS_MAX_FILE_IO_SIZE is 1048576U (1MB).
+ */
+#define GF_NFS3_FILE_IO_SIZE_MAX     (1  * GF_UNIT_MB) /* 1048576 */
+#define GF_NFS3_FILE_IO_SIZE_MIN     (4  * GF_UNIT_KB) /* 4096 */
+
+#define GF_NFS3_FILE_IO_SIZE_DEF     GF_NFS3_FILE_IO_SIZE_MAX
+
+#define GF_NFS3_RTMAX          GF_NFS3_FILE_IO_SIZE_MAX
+#define GF_NFS3_RTMIN          GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_RTPREF         GF_NFS3_FILE_IO_SIZE_DEF
+#define GF_NFS3_RTMULT         GF_NFS3_FILE_IO_SIZE_MIN
+
+#define GF_NFS3_WTMAX          GF_NFS3_FILE_IO_SIZE_MAX
+#define GF_NFS3_WTMIN          GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_WTPREF         GF_NFS3_FILE_IO_SIZE_DEF
+#define GF_NFS3_WTMULT         GF_NFS3_FILE_IO_SIZE_MIN
+
+/* This can be tuned through nfs.readdir-size */
+#define GF_NFS3_DTMAX          GF_NFS3_FILE_IO_SIZE_MAX
+#define GF_NFS3_DTMIN          GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_DTPREF         GF_NFS3_FILE_IO_SIZE_DEF
+
+#define GF_NFS3_MAXFILESIZE    (1 * GF_UNIT_PB)
+
+#define GF_NFS3_IO_SIZE        4096 /* 4-KB */
+#define GF_NFS3_IO_SHIFT       12   /* 2^12 = 4KB */
+
 /* FIXME: Handle time resolutions */
 #define GF_NFS3_TIMEDELTA_SECS     {1,0}
 #define GF_NFS3_TIMEDELTA_NSECS    {0,1}
@@ -94,7 +109,7 @@ struct nfs3_export {
 #define GF_NFS3_DEFAULT_VOLACCESS       (GF_NFS3_VOLACCESS_RW)
 
 /* The NFSv3 protocol state */
-struct nfs3_state {
+typedef struct nfs3_state {
 
         /* The NFS xlator pointer. The NFS xlator can be running
          * multiple versions of the NFS protocol.
@@ -119,27 +134,50 @@ struct nfs3_state {
         uint64_t                serverstart;
 
         /* NFSv3 Protocol configurables */
-        size_t                  readsize;
-        size_t                  writesize;
-        size_t                  readdirsize;
+        uint64_t                readsize;
+        uint64_t                writesize;
+        uint64_t                readdirsize;
 
         /* Size of the iobufs used, depends on the sizes of the three params
          * above.
          */
-        size_t                  iobsize;
+        uint64_t                iobsize;
 
         unsigned int            memfactor;
 
         struct list_head        fdlru;
         gf_lock_t               fdlrulock;
         int                     fdcount;
-};
+        uint32_t                occ_logger;
+} nfs3_state_t;
 
 typedef enum nfs3_lookup_type {
         GF_NFS3_REVALIDATE = 1,
         GF_NFS3_FRESH,
 } nfs3_lookup_type_t;
 
+typedef union args_ {
+        nlm4_stat nlm4_stat;
+        nlm4_holder nlm4_holder;
+        nlm4_lock nlm4_lock;
+        nlm4_share nlm4_share;
+        nlm4_testrply nlm4_testrply;
+        nlm4_testres nlm4_testres;
+        nlm4_testargs nlm4_testargs;
+        nlm4_res nlm4_res;
+        nlm4_lockargs nlm4_lockargs;
+        nlm4_cancargs nlm4_cancargs;
+        nlm4_unlockargs nlm4_unlockargs;
+        nlm4_shareargs nlm4_shareargs;
+        nlm4_shareres nlm4_shareres;
+        nlm4_freeallargs nlm4_freeallargs;
+        getaclargs getaclargs;
+        setaclargs setaclargs;
+        getaclreply getaclreply;
+        setaclreply setaclreply;
+} args;
+
+
 typedef int (*nfs3_resume_fn_t) (void *cs);
 /* Structure used to communicate state between a fop and its callback.
  * Not all members are used at all times. Usage is fop and NFS request
@@ -208,6 +246,22 @@ struct nfs3_local {
         gf_dirent_t             *hashmatch;
         gf_dirent_t             *entrymatch;
         off_t                   lastentryoffset;
+        struct flock            flock;
+        args                    args;
+        nlm4_lkowner_t          lkowner;
+        char                    cookiebytes[1024];
+        struct nfs3_fh          lockfh;
+        int                     monitor;
+        rpc_transport_t         *trans;
+        call_frame_t            *frame;
+
+        /* ACL */
+        aclentry                aclentry[NFS_ACL_MAX_ENTRIES];
+        aclentry                daclentry[NFS_ACL_MAX_ENTRIES];
+        int                     aclcount;
+        char                    aclxattr[NFS_ACL_MAX_ENTRIES*8 + 4];
+        int                     daclcount;
+        char                    daclxattr[NFS_ACL_MAX_ENTRIES*8 + 4];
 };
 
 #define nfs3_is_revalidate_lookup(cst) ((cst)->lookuptype == GF_NFS3_REVALIDATE)
@@ -223,9 +277,9 @@ struct inode_op_queue {
         pthread_mutex_t         qlock;
 };
 
-
-
-
 extern rpcsvc_program_t *
 nfs3svc_init (xlator_t *nfsx);
+
+extern int
+nfs3_reconfigure_state (xlator_t *nfsx, dict_t *options);
 #endif
diff --git a/xlators/nfs/server/src/nlm4.c b/xlators/nfs/server/src/nlm4.c
new file mode 100644
index 000000000..5c5d87412
--- /dev/null
+++ b/xlators/nfs/server/src/nlm4.c
@@ -0,0 +1,2525 @@
+/*
+  Copyright (c) 2012 Gluster, Inc. <http://www.gluster.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "rpcsvc.h"
+#include "dict.h"
+#include "xlator.h"
+#include "nfs.h"
+#include "mem-pool.h"
+#include "logging.h"
+#include "nfs-fops.h"
+#include "inode.h"
+#include "mount3.h"
+#include "nfs3.h"
+#include "nfs-mem-types.h"
+#include "nfs3-helpers.h"
+#include "nfs3-fh.h"
+#include "nlm4.h"
+#include "nlm4-xdr.h"
+#include "msg-nfs3.h"
+#include "nfs-generics.h"
+#include "rpc-clnt.h"
+#include "nsm-xdr.h"
+#include "nlmcbk-xdr.h"
+#include "run.h"
+#include <unistd.h>
+#include <rpc/pmap_clnt.h>
+#include <rpc/rpc.h>
+#include <rpc/xdr.h>
+#include <statedump.h>
+
+/* TODO:
+ * 1) 2 opens racing .. creating an fd leak.
+ * 2) use mempool for nlmclnt - destroy if no fd exists, create during 1st call
+ */
+
+typedef ssize_t (*nlm4_serializer) (struct iovec outmsg, void *args);
+
+extern void nfs3_call_state_wipe (nfs3_call_state_t *cs);
+
+struct list_head nlm_client_list;
+gf_lock_t nlm_client_list_lk;
+
+/* race on this is harmless */
+int nlm_grace_period = 50;
+
+#define nlm4_validate_nfs3_state(request, state, status, label, retval) \
+        do      {                                                       \
+                state = rpcsvc_request_program_private (request);       \
+                if (!state) {                                           \
+                        gf_log (GF_NLM, GF_LOG_ERROR, "NFSv3 state "    \
+                                "missing from RPC request");            \
+                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
+                        status = nlm4_failed;                           \
+                        goto label;                                     \
+                }                                                       \
+        } while (0);                                                    \
+
+nfs3_call_state_t *
+nfs3_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v);
+
+#define nlm4_handle_call_state_init(nfs3state, calls, rq, opstat, errlabel)\
+        do {                                                            \
+                calls = nlm4_call_state_init ((nfs3state), (rq));       \
+                if (!calls) {                                           \
+                        gf_log (GF_NLM, GF_LOG_ERROR, "Failed to "      \
+                                "init call state");                     \
+                        opstat = nlm4_failed;                           \
+                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
+                        goto errlabel;                                  \
+                }                                                       \
+        } while (0)                                                     \
+
+#define nlm4_validate_gluster_fh(handle, status, errlabel)              \
+        do {                                                            \
+                if (!nfs3_fh_validate (handle)) {                       \
+                        status = nlm4_stale_fh;                         \
+                        goto errlabel;                                  \
+                }                                                       \
+        } while (0)                                                     \
+
+xlator_t *
+nfs3_fh_to_xlator (struct nfs3_state *nfs3, struct nfs3_fh *fh);
+
+#define nlm4_map_fh_to_volume(nfs3state, handle, req, volume, status, label) \
+        do {                                                            \
+                char exportid[256], gfid[256];                          \
+                rpc_transport_t *trans = NULL;                          \
+                volume = nfs3_fh_to_xlator ((nfs3state), &handle);      \
+                if (!volume) {                                          \
+                        uuid_unparse (handle.exportid, exportid);       \
+                        uuid_unparse (handle.gfid, gfid);               \
+                        trans = rpcsvc_request_transport (req);         \
+                        gf_log (GF_NLM, GF_LOG_ERROR, "Failed to map "  \
+                                "FH to vol: client=%s, exportid=%s, gfid=%s",\
+                                trans->peerinfo.identifier, exportid,   \
+                                gfid);                                  \
+                        gf_log (GF_NLM, GF_LOG_ERROR,                   \
+                                "Stale nfs client %s must be trying to "\
+                                "connect to a deleted volume, please "  \
+                                "unmount it.", trans->peerinfo.identifier);\
+                        status = nlm4_stale_fh;                         \
+                        goto label;                                     \
+                } else {                                                \
+                        gf_log (GF_NLM, GF_LOG_TRACE, "FH to Volume: %s"\
+                                ,volume->name);                         \
+                        rpcsvc_request_set_private (req, volume);       \
+                }                                                       \
+        } while (0);                                                    \
+
+#define nlm4_volume_started_check(nfs3state, vlm, rtval, erlbl)         \
+        do {                                                            \
+              if ((!nfs_subvolume_started (nfs_state (nfs3state->nfsx), vlm))){\
+                      gf_log (GF_NLM, GF_LOG_ERROR, "Volume is disabled: %s",\
+                              vlm->name);                               \
+                      rtval = RPCSVC_ACTOR_IGNORE;                      \
+                      goto erlbl;                                       \
+              }                                                         \
+        } while (0)                                                     \
+
+#define nlm4_check_fh_resolve_status(cst, nfstat, erlabl)               \
+        do {                                                            \
+                xlator_t *xlatorp = NULL;                               \
+                char buf[256], gfid[256];                               \
+                rpc_transport_t *trans = NULL;                          \
+                if ((cst)->resolve_ret < 0) {                           \
+                        trans = rpcsvc_request_transport (cst->req);    \
+                        xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
+                                                     &cst->resolvefh);  \
+                        uuid_unparse (cst->resolvefh.gfid, gfid);       \
+                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
+                                  trans->peerinfo.identifier,           \
+                                  xlatorp ? xlatorp->name : "ERR",      \
+                                  gfid);                                \
+                        gf_log (GF_NLM, GF_LOG_ERROR, "Unable to resolve FH"\
+                                ": %s", buf);                           \
+                        nfstat = nlm4_errno_to_nlm4stat (cst->resolve_errno);\
+                        goto erlabl;                                    \
+                }                                                       \
+        } while (0)                                                     \
+
+
+void
+nlm4_prep_nlm4_testargs (nlm4_testargs *args, struct nfs3_fh *fh,
+                         nlm4_lkowner_t *oh, char *cookiebytes)
+{
+        memset (args, 0, sizeof (*args));
+        args->alock.fh.n_bytes = (void *)fh;
+        args->alock.oh.n_bytes = (void *)oh;
+        args->cookie.n_bytes = (void *)cookiebytes;
+}
+
+void
+nlm4_prep_nlm4_lockargs (nlm4_lockargs *args, struct nfs3_fh *fh,
+                         nlm4_lkowner_t *oh, char *cookiebytes)
+{
+        memset (args, 0, sizeof (*args));
+        args->alock.fh.n_bytes = (void *)fh;
+        args->alock.oh.n_bytes = (void *)oh;
+        args->cookie.n_bytes = (void *)cookiebytes;
+}
+
+void
+nlm4_prep_nlm4_cancargs (nlm4_cancargs *args, struct nfs3_fh *fh,
+                           nlm4_lkowner_t *oh, char *cookiebytes)
+{
+        memset (args, 0, sizeof (*args));
+        args->alock.fh.n_bytes = (void *)fh;
+        args->alock.oh.n_bytes = (void *)oh;
+        args->cookie.n_bytes = (void *)cookiebytes;
+}
+
+void
+nlm4_prep_nlm4_unlockargs (nlm4_unlockargs *args, struct nfs3_fh *fh,
+                           nlm4_lkowner_t *oh, char *cookiebytes)
+{
+        memset (args, 0, sizeof (*args));
+        args->alock.fh.n_bytes = (void *)fh;
+        args->alock.oh.n_bytes = (void *)oh;
+        args->cookie.n_bytes = (void *)cookiebytes;
+}
+
+void
+nlm4_prep_shareargs (nlm4_shareargs *args, struct nfs3_fh *fh,
+                     nlm4_lkowner_t *oh, char *cookiebytes)
+{
+        memset (args, 0, sizeof (*args));
+        args->share.fh.n_bytes = (void *)fh;
+        args->share.oh.n_bytes = (void *)oh;
+        args->cookie.n_bytes = (void *)cookiebytes;
+}
+
+void
+nlm4_prep_freeallargs (nlm4_freeallargs *args, nlm4_lkowner_t *oh)
+{
+        memset (args, 0, sizeof (*args));
+        args->name = (void *)oh;
+}
+
+void
+nlm_copy_lkowner (gf_lkowner_t *dst, netobj *src)
+{
+        dst->len = src->n_len;
+        memcpy (dst->data, src->n_bytes, dst->len);
+}
+
+int
+nlm_is_oh_same_lkowner (gf_lkowner_t *a, netobj *b)
+{
+        if (!a || !b) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "invalid args");
+                return -1;
+        }
+
+        return (a->len == b->n_len &&
+                !memcmp (a->data, b->n_bytes, a->len));
+}
+
+nlm4_stats
+nlm4_errno_to_nlm4stat (int errnum)
+{
+        nlm4_stats        stat = nlm4_denied;
+
+        switch (errnum) {
+        case 0:
+                stat = nlm4_granted;
+                break;
+        case EROFS:
+                stat = nlm4_rofs;
+                break;
+        case ESTALE:
+                stat = nlm4_stale_fh;
+                break;
+        case ENOLCK:
+                stat = nlm4_failed;
+                break;
+        default:
+                stat = nlm4_denied;
+                break;
+        }
+
+        return stat;
+}
+
+nfs3_call_state_t *
+nlm4_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req)
+{
+        nfs3_call_state_t       *cs = NULL;
+
+        if ((!s) || (!req))
+                return NULL;
+
+        cs = (nfs3_call_state_t *) mem_get (s->localpool);
+        if (!cs)
+                return NULL;
+
+        memset (cs, 0, sizeof (*cs));
+        INIT_LIST_HEAD (&cs->entries.list);
+        INIT_LIST_HEAD (&cs->openwait_q);
+        cs->operrno = EINVAL;
+        cs->req = req;
+        cs->nfsx = s->nfsx;
+        cs->nfs3state = s;
+        cs->monitor = 1;
+
+        return cs;
+}
+
+int
+nlm_monitor (char *caller_name)
+{
+        nlm_client_t *nlmclnt = NULL;
+        int monitor = -1;
+
+        LOCK (&nlm_client_list_lk);
+        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
+                if (!strcmp(caller_name, nlmclnt->caller_name)) {
+                        monitor = nlmclnt->nsm_monitor;
+                        nlmclnt->nsm_monitor = 1;
+                        break;
+                }
+        }
+        UNLOCK (&nlm_client_list_lk);
+
+        if (monitor == -1)
+                gf_log (GF_NLM, GF_LOG_ERROR, "%s was not found in "
+                        "the nlmclnt list", caller_name);
+
+        return monitor;
+}
+
+rpc_clnt_t *
+nlm_get_rpc_clnt (char *caller_name)
+{
+        nlm_client_t *nlmclnt = NULL;
+        int nlmclnt_found = 0;
+        rpc_clnt_t *rpc_clnt = NULL;
+
+        LOCK (&nlm_client_list_lk);
+        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
+                if (!strcmp(caller_name, nlmclnt->caller_name)) {
+                        nlmclnt_found = 1;
+                        break;
+                }
+        }
+        if (!nlmclnt_found)
+                goto ret;
+        if (nlmclnt->rpc_clnt)
+                rpc_clnt = rpc_clnt_ref (nlmclnt->rpc_clnt);
+ret:
+        UNLOCK (&nlm_client_list_lk);
+        return rpc_clnt;
+}
+
+int
+nlm_set_rpc_clnt (rpc_clnt_t *rpc_clnt, char *caller_name)
+{
+        nlm_client_t *nlmclnt = NULL;
+        int nlmclnt_found = 0;
+        int ret = -1;
+
+        LOCK (&nlm_client_list_lk);
+        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
+                if (!strcmp(caller_name, nlmclnt->caller_name)) {
+                        nlmclnt_found = 1;
+                        break;
+                }
+        }
+
+        if (!nlmclnt_found) {
+                nlmclnt = GF_CALLOC (1, sizeof(*nlmclnt),
+                                     gf_nfs_mt_nlm4_nlmclnt);
+                if (nlmclnt == NULL)
+                        goto ret;
+
+                INIT_LIST_HEAD(&nlmclnt->fdes);
+                INIT_LIST_HEAD(&nlmclnt->nlm_clients);
+                INIT_LIST_HEAD(&nlmclnt->shares);
+
+                list_add (&nlmclnt->nlm_clients, &nlm_client_list);
+                nlmclnt->caller_name = gf_strdup (caller_name);
+        }
+
+        if (nlmclnt->rpc_clnt == NULL) {
+                nlmclnt->rpc_clnt = rpc_clnt_ref (rpc_clnt);
+        }
+        ret = 0;
+ret:
+        UNLOCK (&nlm_client_list_lk);
+        return ret;
+}
+
+int
+nlm_unset_rpc_clnt (rpc_clnt_t *rpc)
+{
+        nlm_client_t *nlmclnt = NULL;
+        rpc_clnt_t *rpc_clnt = NULL;
+
+        LOCK (&nlm_client_list_lk);
+        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
+                if (rpc == nlmclnt->rpc_clnt) {
+                        rpc_clnt = nlmclnt->rpc_clnt;
+                        nlmclnt->rpc_clnt = NULL;
+                        break;
+                }
+        }
+        UNLOCK (&nlm_client_list_lk);
+        if (rpc_clnt == NULL) {
+                return -1;
+        }
+        if (rpc_clnt) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&rpc_clnt->conn);
+
+                rpc_clnt_unref (rpc_clnt);
+        }
+        return 0;
+}
+
+int
+nlm_add_nlmclnt (char *caller_name)
+{
+        nlm_client_t *nlmclnt = NULL;
+        int nlmclnt_found = 0;
+        int ret = -1;
+
+        LOCK (&nlm_client_list_lk);
+        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
+                if (!strcmp(caller_name, nlmclnt->caller_name)) {
+                        nlmclnt_found = 1;
+                        break;
+                }
+        }
+        if (!nlmclnt_found) {
+                nlmclnt = GF_CALLOC (1, sizeof(*nlmclnt),
+                                     gf_nfs_mt_nlm4_nlmclnt);
+                if (nlmclnt == NULL) {
+                        gf_log (GF_NLM, GF_LOG_DEBUG, "malloc error");
+                        goto ret;
+                }
+
+                INIT_LIST_HEAD(&nlmclnt->fdes);
+                INIT_LIST_HEAD(&nlmclnt->nlm_clients);
+                INIT_LIST_HEAD(&nlmclnt->shares);
+
+                list_add (&nlmclnt->nlm_clients, &nlm_client_list);
+                nlmclnt->caller_name = gf_strdup (caller_name);
+        }
+        ret = 0;
+ret:
+        UNLOCK (&nlm_client_list_lk);
+        return ret;
+}
+
+int
+nlm4svc_submit_reply (rpcsvc_request_t *req, void *arg, nlm4_serializer sfunc)
+{
+        struct iovec            outmsg  = {0, };
+        struct iobuf            *iob    = NULL;
+        struct nfs3_state       *nfs3   = NULL;
+        int                     ret     = -1;
+        ssize_t                 msglen  = 0;
+        struct iobref           *iobref = NULL;
+
+        if (!req)
+                return -1;
+
+        nfs3 = (struct nfs3_state *)rpcsvc_request_program_private (req);
+        if (!nfs3) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "mount state not found");
+                goto ret;
+        }
+
+        /* First, get the io buffer into which the reply in arg will
+         * be serialized.
+         */
+        iob = iobuf_get (nfs3->iobpool);
+        if (!iob) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to get iobuf");
+                goto ret;
+        }
+
+        iobuf_to_iovec (iob, &outmsg);
+        /* Use the given serializer to translate the give C structure in arg
+         * to XDR format which will be written into the buffer in outmsg.
+         */
+        msglen = sfunc (outmsg, arg);
+        if (msglen < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to encode message");
+                goto ret;
+        }
+        outmsg.iov_len = msglen;
+
+        iobref = iobref_new ();
+        if (iobref == NULL) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to get iobref");
+                goto ret;
+        }
+
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
+
+        /* Then, submit the message for transmission. */
+        ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Reply submission failed");
+                goto ret;
+        }
+
+        ret = 0;
+ret:
+        if (iob)
+                iobuf_unref (iob);
+        if (iobref)
+                iobref_unref (iobref);
+
+        return ret;
+}
+
+typedef int (*nlm4_resume_fn_t) (void *cs);
+
+int32_t
+nlm4_file_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        nfs3_call_state_t *cs = frame->local;
+
+        if (op_ret == 0)
+                fd_bind (cs->fd);
+        cs->resolve_ret = op_ret;
+        cs->resume_fn (cs);
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+        return 0;
+}
+
+void *
+nsm_monitor(void *arg)
+{
+        CLIENT *clnt = NULL;
+        enum clnt_stat ret;
+        struct mon nsm_mon;
+        struct sm_stat_res res;
+        struct timeval tout = { 5, 0 };
+        char *host = NULL;
+
+        host = arg;
+        nsm_mon.mon_id.mon_name = gf_strdup(host);
+        nsm_mon.mon_id.my_id.my_name = gf_strdup("localhost");
+        nsm_mon.mon_id.my_id.my_prog = NLMCBK_PROGRAM;
+        nsm_mon.mon_id.my_id.my_vers = NLMCBK_V1;
+        nsm_mon.mon_id.my_id.my_proc = NLMCBK_SM_NOTIFY;
+        /* nothing to put in the private data */
+#define SM_PROG 100024
+#define SM_VERS 1
+#define SM_MON 2
+
+        /* create a connection to nsm on the localhost */
+        clnt = clnt_create("localhost", SM_PROG, SM_VERS, "tcp");
+        if(!clnt)
+        {
+                gf_log (GF_NLM, GF_LOG_ERROR, "%s",
+                        clnt_spcreateerror ("Clnt_create()"));
+                goto out;
+        }
+
+        ret = clnt_call(clnt, SM_MON,
+                        (xdrproc_t) xdr_mon, (caddr_t) & nsm_mon,
+                        (xdrproc_t) xdr_sm_stat_res, (caddr_t) & res, tout);
+        if(ret != RPC_SUCCESS)
+        {
+                gf_log (GF_NLM, GF_LOG_ERROR, "clnt_call(): %s",
+                        clnt_sperrno(ret));
+                goto out;
+        }
+        if(res.res_stat != STAT_SUCC)
+        {
+                gf_log (GF_NLM, GF_LOG_ERROR, "clnt_call(): %s",
+                        clnt_sperrno(ret));
+                goto out;
+        }
+
+out:
+        GF_FREE(nsm_mon.mon_id.mon_name);
+        GF_FREE(nsm_mon.mon_id.my_id.my_name);
+        if (clnt != NULL)
+                clnt_destroy(clnt);
+        return NULL;
+}
+
+nlm_client_t *
+__nlm_get_uniq (char *caller_name)
+{
+        nlm_client_t *nlmclnt = NULL;
+
+        if (!caller_name)
+                return NULL;
+
+        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
+                if (!strcmp(caller_name, nlmclnt->caller_name))
+                        return nlmclnt;
+        }
+
+        return NULL;
+}
+
+nlm_client_t *
+nlm_get_uniq (char *caller_name)
+{
+        nlm_client_t *nlmclnt = NULL;
+
+        LOCK (&nlm_client_list_lk);
+        nlmclnt = __nlm_get_uniq (caller_name);
+        UNLOCK (&nlm_client_list_lk);
+
+        return nlmclnt;
+}
+
+
+int
+nlm4_file_open_and_resume(nfs3_call_state_t *cs, nlm4_resume_fn_t resume)
+{
+        fd_t *fd = NULL;
+        int ret = -1;
+        int flags = 0;
+        nlm_client_t *nlmclnt = NULL;
+        call_frame_t *frame = NULL;
+
+        if (cs->args.nlm4_lockargs.exclusive == _gf_false)
+                flags = O_RDONLY;
+        else
+                flags = O_WRONLY;
+
+        nlmclnt = nlm_get_uniq (cs->args.nlm4_lockargs.alock.caller_name);
+        if (nlmclnt == NULL) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "nlm_get_uniq() returned NULL");
+                ret = -ENOLCK;
+                goto err;
+        }
+        cs->resume_fn = resume;
+        fd = fd_lookup_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
+        if (fd) {
+                cs->fd = fd;
+                cs->resolve_ret = 0;
+                cs->resume_fn(cs);
+                ret = 0;
+                goto err;
+        }
+
+        fd = fd_create_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
+        if (fd == NULL) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "fd_create_uint64() returned NULL");
+                ret = -ENOLCK;
+                goto err;
+        }
+
+        cs->fd = fd;
+
+        frame = create_frame (cs->nfsx, cs->nfsx->ctx->pool);
+        if (!frame) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to create frame");
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        frame->root->pid = NFS_PID;
+        frame->root->uid = rpcsvc_request_uid (cs->req);
+        frame->root->gid = rpcsvc_request_gid (cs->req);
+        frame->local = cs;
+        nfs_fix_groups (cs->nfsx, frame->root);
+
+        STACK_WIND_COOKIE (frame, nlm4_file_open_cbk, cs->vol, cs->vol,
+                          cs->vol->fops->open, &cs->resolvedloc, flags,
+                          cs->fd, NULL);
+        ret = 0;
+err:
+        return ret;
+}
+
+int
+nlm4_generic_reply (rpcsvc_request_t *req, netobj cookie, nlm4_stats stat)
+{
+        nlm4_res res;
+
+        memset (&res, 0, sizeof (res));
+        res.cookie = cookie;
+        res.stat.stat = stat;
+
+        nlm4svc_submit_reply (req, (void *)&res,
+                              (nlm4_serializer)xdr_serialize_nlm4_res);
+        return 0;
+}
+
+int
+nlm4svc_null (rpcsvc_request_t *req)
+{
+        struct iovec    dummyvec = {0, };
+
+        if (!req) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Got NULL request!");
+                return 0;
+        }
+        rpcsvc_submit_generic (req, &dummyvec, 1,  NULL, 0, NULL);
+        return 0;
+}
+
+int
+nlm4_gf_flock_to_holder (nlm4_holder *holder, struct gf_flock *flock)
+{
+        switch (flock->l_type) {
+        case GF_LK_F_WRLCK:
+                holder->exclusive = 1;
+                break;
+        }
+
+        holder->svid = flock->l_pid;
+        holder->l_offset = flock->l_start;
+        holder->l_len = flock->l_len;
+        return 0;
+}
+
+int
+nlm4_lock_to_gf_flock (struct gf_flock *flock, nlm4_lock *lock, int excl)
+{
+        flock->l_pid = lock->svid;
+        flock->l_start = lock->l_offset;
+        flock->l_len = lock->l_len;
+        if (excl)
+                flock->l_type = F_WRLCK;
+        else
+                flock->l_type = F_RDLCK;
+        flock->l_whence = SEEK_SET;
+        nlm_copy_lkowner (&flock->l_owner, &lock->oh);
+        return 0;
+}
+
+rpc_clnt_procedure_t nlm4_clnt_actors[NLM4_PROC_COUNT] = {
+        [NLM4_NULL] = {"NULL", NULL},
+        [NLM4_GRANTED] = {"GRANTED", NULL},
+};
+
+char *nlm4_clnt_names[NLM4_PROC_COUNT] = {
+        [NLM4_NULL] = "NULL",
+        [NLM4_GRANTED] = "GRANTED",
+};
+
+rpc_clnt_prog_t nlm4clntprog = {
+        .progname = "NLMv4",
+        .prognum = NLM_PROGRAM,
+        .progver = NLM_V4,
+        .numproc = NLM4_PROC_COUNT,
+        .proctable = nlm4_clnt_actors,
+        .procnames = nlm4_clnt_names,
+};
+
+int
+nlm4_test_reply (nfs3_call_state_t *cs, nlm4_stats stat, struct gf_flock *flock)
+{
+        nlm4_testres res;
+
+        memset (&res, 0, sizeof (res));
+        res.cookie = cs->args.nlm4_testargs.cookie;
+        res.stat.stat = stat;
+        if (stat == nlm4_denied)
+                nlm4_gf_flock_to_holder (&res.stat.nlm4_testrply_u.holder,
+                                         flock);
+
+        nlm4svc_submit_reply (cs->req, (void *)&res,
+                              (nlm4_serializer)xdr_serialize_nlm4_testres);
+        return 0;
+}
+
+int
+nlm4svc_test_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                  dict_t *xdata)
+{
+        nlm4_stats                      stat = nlm4_denied;
+        nfs3_call_state_t              *cs = NULL;
+
+        cs = frame->local;
+        if (op_ret == -1) {
+                stat = nlm4_errno_to_nlm4stat (op_errno);
+                goto err;
+        } else if (flock->l_type == F_UNLCK)
+                stat = nlm4_granted;
+
+err:
+        nlm4_test_reply (cs, stat, flock);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+int
+nlm4_test_fd_resume (void *carg)
+{
+        int                             ret = -EFAULT;
+        nfs_user_t                      nfu = {0, };
+        nfs3_call_state_t               *cs = NULL;
+        struct gf_flock                 flock = {0, };
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        nfs_request_user_init (&nfu, cs->req);
+        nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_testargs.alock,
+                               cs->args.nlm4_testargs.exclusive);
+        nlm_copy_lkowner (&nfu.lk_owner, &cs->args.nlm4_testargs.alock.oh);
+        ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_GETLK, &flock,
+                      nlm4svc_test_cbk, cs);
+
+        return ret;
+}
+
+
+int
+nlm4_test_resume (void *carg)
+{
+        nlm4_stats                      stat = nlm4_failed;
+        int                             ret = -1;
+        nfs3_call_state_t               *cs = NULL;
+        fd_t                            *fd = NULL;
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
+        fd = fd_anonymous (cs->resolvedloc.inode);
+        if (!fd)
+                goto nlm4err;
+        cs->fd = fd;
+        ret = nlm4_test_fd_resume (cs);
+
+nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to open_and_resume");
+                stat = nlm4_errno_to_nlm4stat (-ret);
+                nlm4_test_reply (cs, stat, NULL);
+                nfs3_call_state_wipe (cs);
+        }
+
+        return ret;
+}
+
+int
+nlm4svc_test (rpcsvc_request_t *req)
+{
+        xlator_t                        *vol = NULL;
+        nlm4_stats                      stat = nlm4_failed;
+        struct nfs_state               *nfs = NULL;
+        nfs3_state_t                   *nfs3 = NULL;
+        nfs3_call_state_t               *cs = NULL;
+        int                             ret = RPCSVC_ACTOR_ERROR;
+        struct nfs3_fh                  fh = {{0}, };
+
+        if (!req)
+                return ret;
+
+        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     stat, rpcerr);
+
+        nlm4_prep_nlm4_testargs (&cs->args.nlm4_testargs, &fh, &cs->lkowner,
+                                 cs->cookiebytes);
+        if (xdr_to_nlm4_testargs(req->msg[0], &cs->args.nlm4_testargs) <= 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+        if (nlm_grace_period) {
+                gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
+                stat = nlm4_denied_grace_period;
+                nlm4_test_reply (cs, stat, NULL);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+        cs->vol = vol;
+        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+
+        ret = nfs3_fh_resolve_and_resume (cs, &fh,
+                                          NULL, nlm4_test_resume);
+
+nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
+                nlm4_test_reply (cs, stat, NULL);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+rpcerr:
+        if (ret < 0)
+                nfs3_call_state_wipe (cs);
+
+        return ret;
+}
+
+int
+nlm4svc_send_granted_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+        STACK_DESTROY (((call_frame_t*)myframe)->root);
+        return 0;
+}
+
+void
+nlm4svc_send_granted (nfs3_call_state_t *cs);
+
+int
+nlm_rpcclnt_notify (struct rpc_clnt *rpc_clnt, void *mydata,
+                    rpc_clnt_event_t fn, void *data)
+{
+        int                ret         = 0;
+        char              *caller_name = NULL;
+        nfs3_call_state_t *cs          = NULL;
+
+        cs = mydata;
+        caller_name = cs->args.nlm4_lockargs.alock.caller_name;
+
+        switch (fn) {
+        case RPC_CLNT_CONNECT:
+                ret = nlm_set_rpc_clnt (rpc_clnt, caller_name);
+                if (ret == -1) {
+                        gf_log (GF_NLM, GF_LOG_ERROR, "Failed to set rpc clnt");
+                        goto err;
+                }
+                rpc_clnt_unref (rpc_clnt);
+                nlm4svc_send_granted (cs);
+
+                break;
+
+        case RPC_CLNT_MSG:
+                break;
+
+        case RPC_CLNT_DISCONNECT:
+                nlm_unset_rpc_clnt (rpc_clnt);
+                break;
+        }
+
+ err:
+        return 0;
+}
+
+void *
+nlm4_establish_callback (void *csarg)
+{
+        int                           ret                        = -1;
+        nfs3_call_state_t            *cs                         = NULL;
+        union gf_sock_union           sock_union;
+        dict_t                       *options                    = NULL;
+        char                          peerip[INET6_ADDRSTRLEN+1] = {0};
+        char                         *portstr                    = NULL;
+        char                          myip[INET6_ADDRSTRLEN+1]   = {0};
+        rpc_clnt_t                   *rpc_clnt                   = NULL;
+        int                           port                       = -1;
+
+
+        cs = (nfs3_call_state_t *) csarg;
+        glusterfs_this_set (cs->nfsx);
+
+        rpc_transport_get_peeraddr (cs->trans, NULL, 0, &sock_union.storage,
+                                    sizeof (sock_union.storage));
+
+        switch (sock_union.sa.sa_family) {
+        case AF_INET6:
+        /* can not come here as NLM listens on IPv4 */
+                gf_log (GF_NLM, GF_LOG_ERROR, "NLM is not supported on IPv6 in"
+                        " this release");
+                goto err;
+/*
+                inet_ntop (AF_INET6,
+                           &((struct sockaddr_in6 *)sockaddr)->sin6_addr,
+                           peerip, INET6_ADDRSTRLEN+1);
+                break;
+*/
+        case AF_INET:
+                inet_ntop (AF_INET, &sock_union.sin.sin_addr, peerip,
+                           INET6_ADDRSTRLEN+1);
+                inet_ntop (AF_INET, &(((struct sockaddr_in *)&cs->trans->myinfo.sockaddr)->sin_addr),
+                           myip, INET6_ADDRSTRLEN + 1);
+
+                break;
+        default:
+                break;
+                /* FIXME: handle the error */
+        }
+
+        /* looks like libc rpc supports only ipv4 */
+        port = pmap_getport (&sock_union.sin, NLM_PROGRAM,
+                             NLM_V4, IPPROTO_TCP);
+
+        if (port == 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Unable to get NLM port of the "
+                        "client. Is the firewall running on client?");
+                goto err;
+        }
+
+        options = dict_new();
+        ret = dict_set_str (options, "transport-type", "socket");
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
+                goto err;
+        }
+
+        ret = dict_set_dynstr (options, "remote-host", gf_strdup (peerip));
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
+                goto err;
+        }
+
+        ret = gf_asprintf (&portstr, "%d", port);
+        if (ret == -1)
+                goto err;
+
+        ret = dict_set_dynstr (options, "remote-port",
+                               portstr);
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_dynstr error");
+                goto err;
+        }
+
+        /* needed in case virtual IP is used */
+        ret = dict_set_dynstr (options, "transport.socket.source-addr",
+                               gf_strdup (myip));
+        if (ret == -1)
+                goto err;
+
+        ret = dict_set_str (options, "auth-null", "on");
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_dynstr error");
+                goto err;
+        }
+
+        /* TODO: is 32 frames in transit enough ? */
+        rpc_clnt = rpc_clnt_new (options, cs->nfsx->ctx, "NLM-client", 32);
+        if (rpc_clnt == NULL) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "rpc_clnt NULL");
+                goto err;
+        }
+
+        ret = rpc_clnt_register_notify (rpc_clnt, nlm_rpcclnt_notify, cs);
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR,"rpc_clnt_register_connect error");
+                goto err;
+        }
+
+        /* After this connect succeeds, granted msg is sent in notify */
+        ret = rpc_transport_connect (rpc_clnt->conn.trans, port);
+
+        if (ret == -1 && EINPROGRESS == errno)
+                ret = 0;
+
+err:
+        if (ret == -1 && rpc_clnt) {
+                rpc_clnt_unref (rpc_clnt);
+        }
+
+        return rpc_clnt;
+}
+
+void
+nlm4svc_send_granted (nfs3_call_state_t *cs)
+{
+        int ret = -1;
+        rpc_clnt_t *rpc_clnt = NULL;
+        struct iovec            outmsg = {0, };
+        nlm4_testargs testargs;
+        struct iobuf *iobuf = NULL;
+        struct iobref *iobref = NULL;
+        char peerip[INET6_ADDRSTRLEN+1];
+        union gf_sock_union sock_union;
+
+        rpc_clnt = nlm_get_rpc_clnt (cs->args.nlm4_lockargs.alock.caller_name);
+        if (rpc_clnt == NULL) {
+                nlm4_establish_callback ((void*)cs);
+                return;
+        }
+
+        rpc_transport_get_peeraddr (cs->trans, NULL, 0, &sock_union.storage,
+                                    sizeof (sock_union.storage));
+
+        switch (sock_union.sa.sa_family) {
+        case AF_INET6:
+                inet_ntop (AF_INET6, &sock_union.sin6.sin6_addr, peerip,
+                           INET6_ADDRSTRLEN+1);
+                break;
+        case AF_INET:
+                inet_ntop (AF_INET, &sock_union.sin.sin_addr, peerip,
+                           INET6_ADDRSTRLEN+1);
+                break;
+        default:
+                break;
+        }
+
+        testargs.cookie = cs->args.nlm4_lockargs.cookie;
+        testargs.exclusive = cs->args.nlm4_lockargs.exclusive;
+        testargs.alock = cs->args.nlm4_lockargs.alock;
+
+        iobuf = iobuf_get (cs->nfs3state->iobpool);
+        if (!iobuf) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to get iobuf");
+                goto ret;
+        }
+
+        iobuf_to_iovec (iobuf, &outmsg);
+        /* Use the given serializer to translate the give C structure in arg
+         * to XDR format which will be written into the buffer in outmsg.
+         */
+        outmsg.iov_len = xdr_serialize_nlm4_testargs (outmsg, &testargs);
+
+        iobref = iobref_new ();
+        if (iobref == NULL) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to get iobref");
+                goto ret;
+        }
+
+        ret = iobref_add (iobref, iobuf);
+        if (ret) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
+
+        ret = rpc_clnt_submit (rpc_clnt, &nlm4clntprog, NLM4_GRANTED,
+                               nlm4svc_send_granted_cbk, &outmsg, 1,
+                               NULL, 0, iobref, cs->frame, NULL, 0,
+                               NULL, 0, NULL);
+
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "rpc_clnt_submit error");
+                goto ret;
+        }
+ret:
+        if (iobref)
+                iobref_unref (iobref);
+        if (iobuf)
+                iobuf_unref (iobuf);
+
+        rpc_clnt_unref (rpc_clnt);
+        nfs3_call_state_wipe (cs);
+        return;
+}
+
+int
+nlm_cleanup_fds (char *caller_name)
+{
+        int nlmclnt_found = 0;
+        nlm_fde_t *fde = NULL, *tmp = NULL;
+        nlm_client_t *nlmclnt = NULL;
+
+        LOCK (&nlm_client_list_lk);
+        list_for_each_entry (nlmclnt,
+                             &nlm_client_list, nlm_clients) {
+                if (!strcmp(caller_name, nlmclnt->caller_name)) {
+                        nlmclnt_found = 1;
+                        break;
+                }
+        }
+
+        if (!nlmclnt_found)
+                goto ret;
+
+        if (list_empty (&nlmclnt->fdes))
+                goto ret;
+
+        list_for_each_entry_safe (fde, tmp, &nlmclnt->fdes, fde_list) {
+                fd_unref (fde->fd);
+                list_del (&fde->fde_list);
+                GF_FREE (fde);
+        }
+
+ret:
+        UNLOCK (&nlm_client_list_lk);
+        return 0;
+}
+
+void
+nlm_search_and_delete (fd_t *fd, char *caller_name)
+{
+        nlm_fde_t *fde = NULL;
+        nlm_client_t *nlmclnt = NULL;
+        int nlmclnt_found = 0;
+        int fde_found = 0;
+        int transit_cnt = 0;
+
+        LOCK (&nlm_client_list_lk);
+        list_for_each_entry (nlmclnt,
+                             &nlm_client_list, nlm_clients) {
+                if (!strcmp(caller_name, nlmclnt->caller_name)) {
+                        nlmclnt_found = 1;
+                        break;
+                }
+        }
+
+        if (!nlmclnt_found)
+                goto ret;
+
+        list_for_each_entry (fde, &nlmclnt->fdes, fde_list) {
+                if (fde->fd == fd) {
+                        fde_found = 1;
+                        break;
+                }
+        }
+
+        if (!fde_found)
+                goto ret;
+        transit_cnt = fde->transit_cnt;
+        if (transit_cnt)
+                goto ret;
+        list_del (&fde->fde_list);
+
+ret:
+        UNLOCK (&nlm_client_list_lk);
+
+        if (fde_found && !transit_cnt) {
+                fd_unref (fde->fd);
+                GF_FREE (fde);
+        }
+        return;
+}
+
+int
+nlm_dec_transit_count (fd_t *fd, char *caller_name)
+{
+        nlm_fde_t *fde = NULL;
+        nlm_client_t *nlmclnt = NULL;
+        int nlmclnt_found = 0;
+        int fde_found = 0;
+        int transit_cnt = -1;
+
+        LOCK (&nlm_client_list_lk);
+        list_for_each_entry (nlmclnt,
+                             &nlm_client_list, nlm_clients) {
+                if (!strcmp(caller_name, nlmclnt->caller_name)) {
+                        nlmclnt_found = 1;
+                        break;
+                }
+        }
+
+        if (!nlmclnt_found) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "nlmclnt not found");
+                nlmclnt = NULL;
+                goto ret;
+        }
+
+        list_for_each_entry (fde, &nlmclnt->fdes, fde_list) {
+                if (fde->fd == fd) {
+                        fde_found = 1;
+                        break;
+                }
+        }
+
+        if (fde_found) {
+                transit_cnt = --fde->transit_cnt;
+                goto ret;
+        }
+ret:
+
+        UNLOCK (&nlm_client_list_lk);
+        return transit_cnt;
+}
+
+
+nlm_client_t *
+nlm_search_and_add (fd_t *fd, char *caller_name)
+{
+        nlm_fde_t *fde = NULL;
+        nlm_client_t *nlmclnt = NULL;
+        int nlmclnt_found = 0;
+        int fde_found = 0;
+
+        LOCK (&nlm_client_list_lk);
+        list_for_each_entry (nlmclnt,
+                             &nlm_client_list, nlm_clients) {
+                if (!strcmp(caller_name, nlmclnt->caller_name)) {
+                        nlmclnt_found = 1;
+                        break;
+                }
+        }
+
+        if (!nlmclnt_found) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "nlmclnt not found");
+                nlmclnt = NULL;
+                goto ret;
+        }
+
+        list_for_each_entry (fde, &nlmclnt->fdes, fde_list) {
+                if (fde->fd == fd) {
+                        fde_found = 1;
+                        break;
+                }
+        }
+
+        if (fde_found)
+                goto ret;
+
+        fde = GF_CALLOC (1, sizeof (*fde), gf_nfs_mt_nlm4_fde);
+
+        fde->fd = fd_ref (fd);
+        list_add (&fde->fde_list, &nlmclnt->fdes);
+ret:
+        if (nlmclnt_found && fde)
+                fde->transit_cnt++;
+        UNLOCK (&nlm_client_list_lk);
+        return nlmclnt;
+}
+
+int
+nlm4svc_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                  dict_t *xdata)
+{
+        nlm4_stats                       stat        = nlm4_denied;
+        int                              transit_cnt = -1;
+        char                            *caller_name = NULL;
+        nfs3_call_state_t               *cs          = NULL;
+        pthread_t                        thr;
+
+        cs = frame->local;
+        caller_name = cs->args.nlm4_lockargs.alock.caller_name;
+        transit_cnt = nlm_dec_transit_count (cs->fd, caller_name);
+
+        if (op_ret == -1) {
+                if (transit_cnt == 0)
+                        nlm_search_and_delete (cs->fd, caller_name);
+                stat = nlm4_errno_to_nlm4stat (op_errno);
+                goto err;
+        } else {
+                stat = nlm4_granted;
+                if (cs->monitor && !nlm_monitor (caller_name)) {
+                        /* FIXME: handle nsm_monitor failure */
+                        pthread_create (&thr, NULL, nsm_monitor, (void*)caller_name);
+                }
+        }
+
+err:
+        if (cs->args.nlm4_lockargs.block) {
+                cs->frame = copy_frame (frame);
+                frame->local = NULL;
+                nlm4svc_send_granted (cs);
+        } else {
+                nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
+                                    stat);
+                nfs3_call_state_wipe (cs);
+        }
+        return 0;
+}
+
+int
+nlm4_lock_fd_resume (void *carg)
+{
+        nlm4_stats                      stat = nlm4_denied;
+        int                             ret = -EFAULT;
+        nfs_user_t                      nfu = {0, };
+        nfs3_call_state_t               *cs = NULL;
+        struct gf_flock                 flock = {0, };
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
+        (void) nlm_search_and_add (cs->fd,
+                                   cs->args.nlm4_lockargs.alock.caller_name);
+        nfs_request_user_init (&nfu, cs->req);
+        nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_lockargs.alock,
+                               cs->args.nlm4_lockargs.exclusive);
+        nlm_copy_lkowner (&nfu.lk_owner, &cs->args.nlm4_lockargs.alock.oh);
+        if (cs->args.nlm4_lockargs.block) {
+                nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
+                                    nlm4_blocked);
+                ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLKW,
+                              &flock, nlm4svc_lock_cbk, cs);
+                /* FIXME: handle error from nfs_lk() specially  by just
+                 * cleaning up cs and unblock the client lock request.
+                 */
+                ret = 0;
+        } else
+                ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK,
+                              &flock, nlm4svc_lock_cbk, cs);
+
+nlm4err:
+        if (ret < 0) {
+                stat = nlm4_errno_to_nlm4stat (-ret);
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to call lk()");
+                nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
+                                    stat);
+                nfs3_call_state_wipe (cs);
+        }
+
+        return ret;
+}
+
+
+int
+nlm4_lock_resume (void *carg)
+{
+        nlm4_stats                      stat = nlm4_failed;
+        int                             ret  = -1;
+        nfs3_call_state_t              *cs   = NULL;
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
+        ret = nlm4_file_open_and_resume (cs, nlm4_lock_fd_resume);
+
+nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to open and resume");
+                stat = nlm4_errno_to_nlm4stat (-ret);
+                nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
+                                    stat);
+                nfs3_call_state_wipe (cs);
+        }
+
+        return ret;
+}
+
+int
+nlm4svc_lock_common (rpcsvc_request_t *req, int mon)
+{
+        int                              ret      = RPCSVC_ACTOR_ERROR;
+        nlm4_stats                       stat     = nlm4_failed;
+        struct nfs3_fh                   fh       = {{0}, };
+        xlator_t                        *vol      = NULL;
+        nfs3_state_t                    *nfs3     = NULL;
+        nfs3_call_state_t               *cs       = NULL;
+        struct nfs_state                *nfs      = NULL;
+
+        if (!req)
+                return ret;
+
+        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     stat, rpcerr);
+
+        nlm4_prep_nlm4_lockargs (&cs->args.nlm4_lockargs, &cs->lockfh,
+                                 &cs->lkowner, cs->cookiebytes);
+        if (xdr_to_nlm4_lockargs(req->msg[0], &cs->args.nlm4_lockargs) <= 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        fh = cs->lockfh;
+        cs->monitor = mon;
+        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+        if (nlm_grace_period && !cs->args.nlm4_lockargs.reclaim) {
+                gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
+                stat = nlm4_denied_grace_period;
+                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+        cs->vol = vol;
+        cs->trans = rpcsvc_request_transport_ref(req);
+        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+
+        ret = nlm_add_nlmclnt (cs->args.nlm4_lockargs.alock.caller_name);
+
+        ret = nfs3_fh_resolve_and_resume (cs, &fh,
+                                          NULL, nlm4_lock_resume);
+
+nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
+                nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
+                                    stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+rpcerr:
+        if (ret < 0) {
+                nfs3_call_state_wipe (cs);
+        }
+
+        return ret;
+}
+
+int
+nlm4svc_lock (rpcsvc_request_t *req)
+{
+        return nlm4svc_lock_common (req, 1);
+}
+
+int
+nlm4svc_nm_lock (rpcsvc_request_t *req)
+{
+        return nlm4svc_lock_common (req, 0);
+}
+
+int
+nlm4svc_cancel_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                    dict_t *xdata)
+{
+        nlm4_stats                      stat = nlm4_denied;
+        nfs3_call_state_t              *cs = NULL;
+
+        cs = frame->local;
+        if (op_ret == -1) {
+                stat = nlm4_errno_to_nlm4stat (op_errno);
+                goto err;
+        } else
+                stat = nlm4_granted;
+
+err:
+        nlm4_generic_reply (cs->req, cs->args.nlm4_cancargs.cookie,
+                            stat);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+int
+nlm4_cancel_fd_resume (void *carg)
+{
+        int                             ret = -EFAULT;
+        nfs_user_t                      nfu = {0, };
+        nfs3_call_state_t               *cs = NULL;
+        struct gf_flock                 flock = {0, };
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        nfs_request_user_init (&nfu, cs->req);
+        nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_cancargs.alock,
+                                cs->args.nlm4_cancargs.exclusive);
+        nlm_copy_lkowner (&nfu.lk_owner, &cs->args.nlm4_cancargs.alock.oh);
+        flock.l_type = F_UNLCK;
+        ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK,
+                      &flock, nlm4svc_cancel_cbk, cs);
+
+        return ret;
+}
+
+int
+nlm4_cancel_resume (void *carg)
+{
+        nlm4_stats                      stat = nlm4_failed;
+        int                             ret = -EFAULT;
+        nfs3_call_state_t               *cs = NULL;
+        nlm_client_t                    *nlmclnt = NULL;
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
+
+        nlmclnt = nlm_get_uniq (cs->args.nlm4_cancargs.alock.caller_name);
+        if (nlmclnt == NULL) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "nlm_get_uniq() returned NULL");
+                goto nlm4err;
+        }
+        cs->fd = fd_lookup_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
+        if (cs->fd == NULL) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "fd_lookup_uint64 retrned NULL");
+                goto nlm4err;
+        }
+        ret = nlm4_cancel_fd_resume (cs);
+
+nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_WARNING, "unable to unlock_fd_resume()");
+                stat = nlm4_errno_to_nlm4stat (-ret);
+                nlm4_generic_reply (cs->req, cs->args.nlm4_cancargs.cookie,
+                                    stat);
+
+                nfs3_call_state_wipe (cs);
+        }
+        /* clean up is taken care of */
+        return 0;
+}
+
+int
+nlm4svc_cancel (rpcsvc_request_t *req)
+{
+        xlator_t                        *vol = NULL;
+        nlm4_stats                      stat = nlm4_failed;
+        struct nfs_state               *nfs = NULL;
+        nfs3_state_t                   *nfs3 = NULL;
+        nfs3_call_state_t               *cs = NULL;
+        int                             ret = RPCSVC_ACTOR_ERROR;
+        struct nfs3_fh                  fh = {{0}, };
+
+        if (!req)
+                return ret;
+
+        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     stat, rpcerr);
+
+        nlm4_prep_nlm4_cancargs (&cs->args.nlm4_cancargs, &fh, &cs->lkowner,
+                                 cs->cookiebytes);
+        if (xdr_to_nlm4_cancelargs(req->msg[0], &cs->args.nlm4_cancargs) <= 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+        if (nlm_grace_period) {
+                gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
+                stat = nlm4_denied_grace_period;
+                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+        cs->vol = vol;
+        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+
+        ret = nfs3_fh_resolve_and_resume (cs, &fh,
+                                          NULL, nlm4_cancel_resume);
+
+nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
+                nlm4_generic_reply (cs->req, cs->args.nlm4_cancargs.cookie,
+                                    stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+rpcerr:
+        if (ret < 0) {
+                nfs3_call_state_wipe (cs);
+        }
+        return ret;
+}
+
+int
+nlm4svc_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                    dict_t *xdata)
+{
+        nlm4_stats                      stat = nlm4_denied;
+        nfs3_call_state_t              *cs = NULL;
+
+        cs = frame->local;
+        if (op_ret == -1) {
+                stat = nlm4_errno_to_nlm4stat (op_errno);
+                goto err;
+        } else {
+                stat = nlm4_granted;
+                if (flock->l_type == F_UNLCK)
+                        nlm_search_and_delete (cs->fd,
+                                               cs->args.nlm4_unlockargs.alock.caller_name);
+        }
+
+err:
+        nlm4_generic_reply (cs->req, cs->args.nlm4_unlockargs.cookie, stat);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+int
+nlm4_unlock_fd_resume (void *carg)
+{
+        int                             ret = -EFAULT;
+        nfs_user_t                      nfu = {0, };
+        nfs3_call_state_t               *cs = NULL;
+        struct gf_flock                 flock = {0, };
+
+        if (!carg)
+                return ret;
+        cs = (nfs3_call_state_t *)carg;
+        nfs_request_user_init (&nfu, cs->req);
+        nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_unlockargs.alock, 0);
+        nlm_copy_lkowner (&nfu.lk_owner, &cs->args.nlm4_unlockargs.alock.oh);
+        flock.l_type = F_UNLCK;
+        ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK,
+                      &flock, nlm4svc_unlock_cbk, cs);
+
+        return ret;
+}
+
+int
+nlm4_unlock_resume (void *carg)
+{
+        nlm4_stats                      stat = nlm4_failed;
+        int                             ret = -1;
+        nfs3_call_state_t               *cs = NULL;
+        nlm_client_t                    *nlmclnt = NULL;
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
+
+        nlmclnt = nlm_get_uniq (cs->args.nlm4_unlockargs.alock.caller_name);
+        if (nlmclnt == NULL) {
+                stat = nlm4_granted;
+                gf_log (GF_NLM, GF_LOG_WARNING, "nlm_get_uniq() returned NULL");
+                goto nlm4err;
+        }
+        cs->fd = fd_lookup_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
+        if (cs->fd == NULL) {
+                stat = nlm4_granted;
+                gf_log (GF_NLM, GF_LOG_WARNING, "fd_lookup_uint64() returned "
+                        "NULL");
+                goto nlm4err;
+        }
+        ret = nlm4_unlock_fd_resume (cs);
+
+nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_WARNING, "unable to unlock_fd_resume");
+                stat = nlm4_errno_to_nlm4stat (-ret);
+                nlm4_generic_reply (cs->req, cs->args.nlm4_unlockargs.cookie,
+                                    stat);
+
+                nfs3_call_state_wipe (cs);
+        }
+        /* we have already taken care of cleanup */
+        return 0;
+}
+
+int
+nlm4svc_unlock (rpcsvc_request_t *req)
+{
+        xlator_t                        *vol = NULL;
+        nlm4_stats                      stat = nlm4_failed;
+        struct nfs_state               *nfs = NULL;
+        nfs3_state_t                   *nfs3 = NULL;
+        nfs3_call_state_t               *cs = NULL;
+        int                             ret = RPCSVC_ACTOR_ERROR;
+        struct nfs3_fh                  fh = {{0}, };
+
+        if (!req)
+                return ret;
+
+        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     stat, rpcerr);
+
+        nlm4_prep_nlm4_unlockargs (&cs->args.nlm4_unlockargs, &fh, &cs->lkowner,
+                                   cs->cookiebytes);
+        if (xdr_to_nlm4_unlockargs(req->msg[0], &cs->args.nlm4_unlockargs) <= 0)
+        {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+        if (nlm_grace_period) {
+                gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
+                stat = nlm4_denied_grace_period;
+                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+        cs->vol = vol;
+        /* FIXME: check if trans is being used at all for unlock */
+        cs->trans = rpcsvc_request_transport_ref(req);
+        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+
+        ret = nfs3_fh_resolve_and_resume (cs, &fh,
+                                          NULL, nlm4_unlock_resume);
+
+nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
+                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+rpcerr:
+        if (ret < 0) {
+                nfs3_call_state_wipe (cs);
+        }
+        return ret;
+}
+
+int
+nlm4_share_reply (nfs3_call_state_t *cs, nlm4_stats stat)
+{
+        nlm4_shareres  res = {{0}, 0, 0};
+
+        if (!cs)
+                return -1;
+
+        res.cookie = cs->args.nlm4_shareargs.cookie;
+        res.stat = stat;
+        res.sequence = 0;
+
+        nlm4svc_submit_reply (cs->req, (void *)&res,
+                              (nlm4_serializer)xdr_serialize_nlm4_shareres);
+        return 0;
+}
+
+nlm_share_t *
+nlm4_share_new ()
+{
+        nlm_share_t      *share     = NULL;
+
+        share = GF_CALLOC (1, sizeof (nlm_share_t),
+                           gf_nfs_mt_nlm4_share);
+        if (!share)
+                goto out;
+
+        INIT_LIST_HEAD (&share->client_list);
+        INIT_LIST_HEAD (&share->inode_list);
+ out:
+        return share;
+}
+
+int
+nlm4_add_share_to_inode (nlm_share_t *share)
+{
+        int                    ret     = -1;
+        uint64_t               ctx     = 0;
+        struct list_head      *head    = NULL;
+        xlator_t              *this    = NULL;
+        inode_t               *inode   = NULL;
+        struct nfs_inode_ctx  *ictx    = NULL;
+        struct nfs_state      *priv    = NULL;
+
+        this = THIS;
+        priv = this->private;
+        inode = share->inode;
+        ret = inode_ctx_get (inode, this, &ctx);
+
+        if (ret == -1) {
+                ictx = GF_CALLOC (1, sizeof (struct nfs_inode_ctx),
+                                  gf_nfs_mt_inode_ctx);
+                if (!ictx ) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not allocate nfs inode ctx");
+                        ret = -1;
+                        goto out;
+                }
+                ictx->generation = priv->generation;
+
+                head = &ictx->shares;
+                INIT_LIST_HEAD (head);
+
+                ret = inode_ctx_put (inode, this, (uint64_t)ictx);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not store share list");
+                        goto out;
+                }
+        }
+        else {
+                ictx = (struct nfs_inode_ctx *)ctx;
+                head = &ictx->shares;
+        }
+
+        list_add (&share->inode_list, head);
+
+ out:
+        if (ret && head)
+                GF_FREE (head);
+
+        return ret;
+}
+
+int
+nlm4_approve_share_reservation (nfs3_call_state_t *cs)
+{
+        int                       ret               = -1;
+        uint64_t                  ctx               = 0;
+        fsh_mode                  req_mode          = 0;
+        fsh_access                req_access        = 0;
+        inode_t                  *inode             = NULL;
+        nlm_share_t              *share             = NULL;
+        struct list_head         *head              = NULL;
+        struct nfs_inode_ctx     *ictx              = NULL;
+
+        if (!cs)
+                goto out;
+
+        inode = cs->resolvedloc.inode;
+
+        ret = inode_ctx_get (inode, THIS, &ctx);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+        ictx = (struct nfs_inode_ctx *)ctx;
+
+        head = &ictx->shares;
+        if (!head || list_empty (head))
+                goto out;
+
+        req_mode = cs->args.nlm4_shareargs.share.mode;
+        req_access = cs->args.nlm4_shareargs.share.access;
+
+        list_for_each_entry (share, head, inode_list) {
+                ret = (((req_mode & share->access) == 0) &&
+                       ((req_access & share->mode) == 0));
+                if (!ret) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        ret = 0;
+
+ out:
+         return ret;
+}
+
+int
+nlm4_create_share_reservation (nfs3_call_state_t *cs)
+{
+        int                       ret        = -1;
+        nlm_share_t              *share      = NULL;
+        nlm_client_t             *client     = NULL;
+        inode_t                  *inode      = NULL;
+
+        LOCK (&nlm_client_list_lk);
+
+        inode = inode_ref (cs->resolvedloc.inode);
+        if (!inode) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "inode not found");
+                goto out;
+        }
+
+        client = __nlm_get_uniq (cs->args.nlm4_shareargs.share.caller_name);
+        if (!client) {
+                /* DO NOT add client. the client is supposed
+                   to be here, since nlm4svc_share adds it */
+                gf_log (GF_NLM, GF_LOG_ERROR, "client not found");
+                goto out;
+        }
+
+        ret = nlm4_approve_share_reservation (cs);
+        if (ret)
+                goto out;
+
+        share = nlm4_share_new ();
+        if (!share) {
+                ret = -1;
+                goto out;
+        }
+
+        share->inode  = inode;
+        share->mode   = cs->args.nlm4_shareargs.share.mode;
+        share->access = cs->args.nlm4_shareargs.share.access;
+        nlm_copy_lkowner (&share->lkowner,
+                          &cs->args.nlm4_shareargs.share.oh);
+
+        ret = nlm4_add_share_to_inode (share);
+        if (ret)
+                goto out;
+
+        list_add (&share->client_list, &client->shares);
+
+ out:
+        if (ret && inode) {
+                inode_unref (inode);
+                GF_FREE (share);
+        }
+
+        UNLOCK (&nlm_client_list_lk);
+        return ret;
+}
+
+/*
+  SHARE and UNSHARE calls DO NOT perform STACK_WIND,
+  the (non-monitored) share reservations are maintained
+  at *nfs xlator level only*, in memory
+*/
+int
+nlm4_share_resume (void *call_state)
+{
+        int                             ret             = -1;
+        nlm4_stats                      stat            = nlm4_failed;
+        nfs3_call_state_t              *cs              = NULL;
+
+        if (!call_state)
+                return ret;
+
+        cs = (nfs3_call_state_t *)call_state;
+        nlm4_check_fh_resolve_status (cs, stat, out);
+
+        ret = nlm4_create_share_reservation (cs);
+        if (!ret)
+                stat = nlm4_granted;
+
+ out:
+        nlm4_share_reply (cs, stat);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+int
+nlm4svc_share (rpcsvc_request_t *req)
+{
+        nlm4_stats                    stat      = nlm4_failed;
+        xlator_t                     *vol       = NULL;
+        nfs3_state_t                 *nfs3      = NULL;
+        nfs3_call_state_t            *cs        = NULL;
+        struct nfs_state             *nfs       = NULL;
+        struct nfs3_fh                fh        = {{0}, };
+        int                           ret       = RPCSVC_ACTOR_ERROR;
+
+        if (!req)
+                return ret;
+
+        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     stat, rpcerr);
+
+        nlm4_prep_shareargs (&cs->args.nlm4_shareargs, &cs->lockfh,
+                             &cs->lkowner, cs->cookiebytes);
+
+        if (xdr_to_nlm4_shareargs (req->msg[0],
+                                   &cs->args.nlm4_shareargs) <= 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding SHARE args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        fh = cs->lockfh;
+        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req,
+                               vol, stat, nlm4err);
+
+        if (nlm_grace_period && !cs->args.nlm4_shareargs.reclaim) {
+                gf_log (GF_NLM, GF_LOG_DEBUG, "NLM in grace period");
+                stat = nlm4_denied_grace_period;
+                nlm4_share_reply (cs, stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+        cs->vol = vol;
+        cs->trans = rpcsvc_request_transport_ref(req);
+        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+
+        ret = nlm_add_nlmclnt (cs->args.nlm4_shareargs.share.caller_name);
+
+        ret = nfs3_fh_resolve_and_resume (cs, &fh, NULL, nlm4_share_resume);
+
+ nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "SHARE call failed");
+                nlm4_share_reply (cs, stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+ rpcerr:
+        if (ret < 0)
+                nfs3_call_state_wipe (cs);
+
+        return ret;
+}
+
+int
+nlm4_remove_share_reservation (nfs3_call_state_t *cs)
+{
+        int                      ret        = -1;
+        uint64_t                 ctx        = 0;
+        fsh_mode                 req_mode   = 0;
+        fsh_access               req_access = 0;
+        nlm_share_t             *share      = NULL;
+        nlm_share_t             *tmp        = NULL;
+        nlm_client_t            *client     = NULL;
+        char                    *caller     = NULL;
+        inode_t                 *inode      = NULL;
+        xlator_t                *this       = NULL;
+        struct list_head        *head       = NULL;
+        nlm4_shareargs          *args       = NULL;
+        struct nfs_inode_ctx    *ictx       = NULL;
+
+        LOCK (&nlm_client_list_lk);
+
+        args = &cs->args.nlm4_shareargs;
+        caller = args->share.caller_name;
+
+        client = __nlm_get_uniq (caller);
+        if (!client) {
+                gf_log (GF_NLM, GF_LOG_ERROR,
+                        "client not found: %s", caller);
+                goto out;
+        }
+
+        inode = cs->resolvedloc.inode;
+        if (!inode) {
+                gf_log (GF_NLM, GF_LOG_ERROR,
+                        "inode not found: client: %s", caller);
+                goto out;
+        }
+
+        this = THIS;
+        ret = inode_ctx_get (inode, this, &ctx);
+        if (ret) {
+                gf_log (GF_NLM, GF_LOG_ERROR,
+                        "no shares found for inode:"
+                        "gfid: %s; client: %s",
+                        inode->gfid, caller);
+                goto out;
+        }
+        ictx = (struct nfs_inode_ctx *)ctx;
+
+        head = &ictx->shares;
+        if (list_empty (head)) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+        req_mode = args->share.mode;
+        req_access = args->share.access;
+
+        list_for_each_entry_safe (share, tmp, head, inode_list) {
+                ret = ((req_mode == share->mode) &&
+                       (req_access == share->access) &&
+                       nlm_is_oh_same_lkowner (&share->lkowner, &args->share.oh));
+                if (ret) {
+                        list_del (&share->client_list);
+                        list_del (&share->inode_list);
+                        inode_unref (share->inode);
+                        GF_FREE (share);
+                        break;
+                }
+        }
+
+        ret = 0;
+ out:
+        UNLOCK (&nlm_client_list_lk);
+        return ret;
+
+}
+
+int
+nlm4_unshare_resume (void *call_state)
+{
+        int                ret        = -1;
+        nlm4_stats         stat       = nlm4_failed;
+        nfs3_call_state_t *cs         = NULL;
+
+        if (!call_state)
+                return ret;
+
+        cs = (nfs3_call_state_t *)call_state;
+
+        nlm4_check_fh_resolve_status (cs, stat, out);
+        ret = nlm4_remove_share_reservation (cs);
+        if (!ret)
+                stat = nlm4_granted;
+
+ out:
+        nlm4_share_reply (cs, stat);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+int
+nlm4svc_unshare (rpcsvc_request_t *req)
+{
+        nlm4_stats                    stat      = nlm4_failed;
+        xlator_t                     *vol       = NULL;
+        nfs3_state_t                 *nfs3      = NULL;
+        nfs3_call_state_t            *cs        = NULL;
+        struct nfs_state             *nfs       = NULL;
+        struct nfs3_fh                fh        = {{0}, };
+        int                           ret       = RPCSVC_ACTOR_ERROR;
+
+        if (!req)
+                return ret;
+
+        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     stat, rpcerr);
+
+        nlm4_prep_shareargs (&cs->args.nlm4_shareargs, &cs->lockfh,
+                             &cs->lkowner, cs->cookiebytes);
+
+        if (xdr_to_nlm4_shareargs (req->msg[0],
+                                   &cs->args.nlm4_shareargs) <= 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding UNSHARE args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        fh = cs->lockfh;
+        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req,
+                               vol, stat, nlm4err);
+
+        if (nlm_grace_period && !cs->args.nlm4_shareargs.reclaim) {
+                gf_log (GF_NLM, GF_LOG_DEBUG, "NLM in grace period");
+                stat = nlm4_denied_grace_period;
+                nlm4_share_reply (cs, stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+        cs->vol = vol;
+        cs->trans = rpcsvc_request_transport_ref(req);
+        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+
+        ret = nfs3_fh_resolve_and_resume (cs, &fh, NULL,
+                                          nlm4_unshare_resume);
+
+ nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "UNSHARE call failed");
+                nlm4_share_reply (cs, stat);
+                ret = 0;
+                return 0;
+        }
+
+ rpcerr:
+        if (ret < 0)
+                nfs3_call_state_wipe (cs);
+
+        return ret;
+}
+
+int
+nlm4_free_all_shares (char *caller_name)
+{
+        nlm_share_t             *share      = NULL;
+        nlm_share_t             *tmp        = NULL;
+        nlm_client_t            *client     = NULL;
+
+        LOCK (&nlm_client_list_lk);
+
+        client = __nlm_get_uniq (caller_name);
+        if (!client) {
+                gf_log (GF_NLM, GF_LOG_DEBUG,
+                        "client not found: %s", caller_name);
+                goto out;
+        }
+
+        list_for_each_entry_safe (share, tmp, &client->shares, client_list) {
+                list_del (&share->inode_list);
+                list_del (&share->client_list);
+                inode_unref (share->inode);
+                GF_FREE (share);
+        }
+ out:
+        UNLOCK (&nlm_client_list_lk);
+        return 0;
+}
+
+int
+nlm4svc_free_all (rpcsvc_request_t *req)
+{
+        int                           ret       = RPCSVC_ACTOR_ERROR;
+        nlm4_stats                    stat      = nlm4_failed;
+        nfs3_state_t                 *nfs3      = NULL;
+        nfs3_call_state_t            *cs        = NULL;
+        struct nfs_state             *nfs       = NULL;
+
+        nlm4_validate_nfs3_state (req, nfs3, stat, err, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        nlm4_handle_call_state_init (nfs->nfs3state, cs,
+                                     req, stat, err);
+
+        nlm4_prep_freeallargs (&cs->args.nlm4_freeallargs,
+                               &cs->lkowner);
+
+        if (xdr_to_nlm4_freeallargs (req->msg[0],
+                                     &cs->args.nlm4_freeallargs) <= 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding FREE_ALL args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto err;
+        }
+
+        ret = nlm4_free_all_shares (cs->args.nlm4_freeallargs.name);
+        if (ret)
+                goto err;
+
+        ret = nlm_cleanup_fds (cs->args.nlm4_freeallargs.name);
+        if (ret)
+                goto err;
+
+ err:
+        nfs3_call_state_wipe (cs);
+        if (ret)
+                gf_log (GF_NLM, GF_LOG_DEBUG,
+                        "error in free all; stat: %d", stat);
+        return ret;
+
+}
+
+void
+nlm4svc_sm_notify (struct nlm_sm_status *status)
+{
+        gf_log (GF_NLM, GF_LOG_INFO, "sm_notify: %s, state: %d",
+                status->mon_name,
+                status->state);
+        nlm_cleanup_fds (status->mon_name);
+}
+
+rpcsvc_actor_t  nlm4svc_actors[NLM4_PROC_COUNT] = {
+        /* 0 */
+        {"NULL",       NLM4_NULL,         nlm4svc_null,      NULL, 0, DRC_IDEMPOTENT},
+        {"TEST",       NLM4_TEST,         nlm4svc_test,      NULL, 0, DRC_IDEMPOTENT},
+        {"LOCK",       NLM4_LOCK,         nlm4svc_lock,      NULL, 0, DRC_NON_IDEMPOTENT},
+        {"CANCEL",     NLM4_CANCEL,       nlm4svc_cancel,    NULL, 0, DRC_NON_IDEMPOTENT},
+        {"UNLOCK",     NLM4_UNLOCK,       nlm4svc_unlock,    NULL, 0, DRC_NON_IDEMPOTENT},
+        /* 5 */
+        {"GRANTED",    NLM4_GRANTED,      NULL,              NULL, 0, DRC_NA},
+        {"TEST",       NLM4_TEST_MSG,     NULL,              NULL, 0, DRC_NA},
+        {"LOCK",       NLM4_LOCK_MSG,     NULL,              NULL, 0, DRC_NA},
+        {"CANCEL",     NLM4_CANCEL_MSG,   NULL,              NULL, 0, DRC_NA},
+        {"UNLOCK",     NLM4_UNLOCK_MSG,   NULL,              NULL, 0, DRC_NA},
+        /* 10 */
+        {"GRANTED",    NLM4_GRANTED_MSG,  NULL,              NULL, 0, DRC_NA},
+        {"TEST",       NLM4_TEST_RES,     NULL,              NULL, 0, DRC_NA},
+        {"LOCK",       NLM4_LOCK_RES,     NULL,              NULL, 0, DRC_NA},
+        {"CANCEL",     NLM4_CANCEL_RES,   NULL,              NULL, 0, DRC_NA},
+        {"UNLOCK",     NLM4_UNLOCK_RES,   NULL,              NULL, 0, DRC_NA},
+        /* 15 ; procedures 17,18,19 are not defined by nlm */
+        {"GRANTED",    NLM4_GRANTED_RES,  NULL,              NULL, 0, DRC_NA},
+        {"SM_NOTIFY",  NLM4_SM_NOTIFY,    NULL,              NULL, 0, DRC_NA},
+        {"SEVENTEEN",  NLM4_SEVENTEEN,    NULL,              NULL, 0, DRC_NA},
+        {"EIGHTEEN",   NLM4_EIGHTEEN,     NULL,              NULL, 0, DRC_NA},
+        {"NINETEEN",   NLM4_NINETEEN,     NULL,              NULL, 0, DRC_NA},
+        /* 20 */
+        {"SHARE",      NLM4_SHARE,        nlm4svc_share,     NULL, 0, DRC_NON_IDEMPOTENT},
+        {"UNSHARE",    NLM4_UNSHARE,      nlm4svc_unshare,   NULL, 0, DRC_NON_IDEMPOTENT},
+        {"NM_LOCK",    NLM4_NM_LOCK,      nlm4svc_nm_lock,   NULL, 0, DRC_NON_IDEMPOTENT},
+        {"FREE_ALL",   NLM4_FREE_ALL,     nlm4svc_free_all,  NULL, 0, DRC_IDEMPOTENT},
+};
+
+rpcsvc_program_t        nlm4prog = {
+        .progname       = "NLM4",
+        .prognum        = NLM_PROGRAM,
+        .progver        = NLM_V4,
+        .progport       = GF_NLM4_PORT,
+        .actors         = nlm4svc_actors,
+        .numactors      = NLM4_PROC_COUNT,
+        .min_auth       = AUTH_NULL,
+};
+
+
+int
+nlm4_init_state (xlator_t *nfsx)
+{
+        return 0;
+}
+
+extern void *nsm_thread (void *argv);
+
+void nlm_grace_period_over(void *arg)
+{
+        nlm_grace_period = 0;
+}
+
+rpcsvc_program_t *
+nlm4svc_init(xlator_t *nfsx)
+{
+        struct nfs3_state *ns = NULL;
+        struct nfs_state *nfs = NULL;
+        dict_t *options = NULL;
+        int ret = -1;
+        char *portstr = NULL;
+        pthread_t thr;
+        struct timespec timeout = {0,};
+        FILE   *pidfile = NULL;
+        pid_t   pid     = -1;
+        static gf_boolean_t nlm4_inited = _gf_false;
+
+        /* Already inited */
+        if (nlm4_inited)
+                return &nlm4prog;
+
+        nfs = (struct nfs_state*)nfsx->private;
+
+        ns = nfs->nfs3state;
+        if (!ns) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "NLM4 init failed");
+                goto err;
+        }
+        nlm4prog.private = ns;
+
+        options = dict_new ();
+
+        ret = gf_asprintf (&portstr, "%d", GF_NLM4_PORT);
+        if (ret == -1)
+                goto err;
+
+        ret = dict_set_dynstr (options, "transport.socket.listen-port",
+                               portstr);
+        if (ret == -1)
+                goto err;
+        ret = dict_set_str (options, "transport-type", "socket");
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
+                goto err;
+        }
+
+        if (nfs->allow_insecure) {
+                ret = dict_set_str (options, "rpc-auth-allow-insecure", "on");
+                if (ret == -1) {
+                        gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
+                        goto err;
+                }
+                ret = dict_set_str (options, "rpc-auth.ports.insecure", "on");
+                if (ret == -1) {
+                        gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
+                        goto err;
+                }
+        }
+
+        ret = dict_set_str (options, "transport.address-family", "inet");
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
+                goto err;
+        }
+
+        ret = rpcsvc_create_listeners (nfs->rpcsvc, options, "NLM");
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Unable to create listeners");
+                dict_unref (options);
+                goto err;
+        }
+        INIT_LIST_HEAD(&nlm_client_list);
+        LOCK_INIT (&nlm_client_list_lk);
+
+        /* unlink sm-notify.pid so that when we restart rpc.statd/sm-notify
+         * it thinks that the machine has restarted and sends NOTIFY to clients.
+         */
+        ret = unlink ("/var/run/sm-notify.pid");
+        if (ret == -1 && errno != ENOENT) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to unlink sm-notify");
+                goto err;
+        }
+        /* temporary work around to restart statd, not distro/OS independant.
+         * Need to figure out a more graceful way
+         * killall will cause problems on solaris.
+         */
+
+        pidfile = fopen ("/var/run/rpc.statd.pid", "r");
+        if (pidfile) {
+                ret = fscanf (pidfile, "%d", &pid);
+                if (ret <= 0) {
+                        gf_log (GF_NLM, GF_LOG_WARNING, "unable to get pid of "
+                                "rpc.statd");
+                        ret = runcmd ("killall", "-9", "rpc.statd", NULL);
+                } else
+                        kill (pid, SIGKILL);
+
+                fclose (pidfile);
+        } else {
+                gf_log (GF_NLM, GF_LOG_WARNING, "opening the pid file of "
+                        "rpc.statd failed (%s)", strerror (errno));
+                /* if ret == -1, do nothing - case either statd was not
+                 * running or was running in valgrind mode
+                 */
+                ret = runcmd ("killall", "-9", "rpc.statd", NULL);
+        }
+
+        ret = unlink ("/var/run/rpc.statd.pid");
+        if (ret == -1 && errno != ENOENT) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to unlink rpc.statd");
+                goto err;
+        }
+
+        ret = runcmd ("/sbin/rpc.statd", NULL);
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to start rpc.statd");
+                goto err;
+        }
+        pthread_create (&thr, NULL, nsm_thread, (void*)NULL);
+
+        timeout.tv_sec = nlm_grace_period;
+        timeout.tv_nsec = 0;
+
+        gf_timer_call_after (nfsx->ctx, timeout, nlm_grace_period_over, NULL);
+        nlm4_inited = _gf_true;
+        return &nlm4prog;
+err:
+        return NULL;
+}
+
+int32_t
+nlm_priv (xlator_t *this)
+{
+        int32_t                  ret                       = -1;
+        uint32_t                 client_count              = 0;
+        uint64_t                 file_count                = 0;
+        nlm_client_t            *client                    = NULL;
+        nlm_fde_t               *fde                       = NULL;
+        char                     key[GF_DUMP_MAX_BUF_LEN]  = {0};
+        char                     gfid_str[64]              = {0};
+
+        gf_proc_dump_add_section("nfs.nlm");
+
+        if (TRY_LOCK (&nlm_client_list_lk))
+                goto out;
+
+        list_for_each_entry (client, &nlm_client_list, nlm_clients) {
+
+                gf_proc_dump_build_key (key, "client", "%d.hostname", client_count);
+                gf_proc_dump_write (key, "%s\n", client->caller_name);
+
+                file_count = 0;
+                list_for_each_entry (fde, &client->fdes, fde_list) {
+                        gf_proc_dump_build_key (key, "file", "%ld.gfid", file_count);
+                        memset (gfid_str, 0, 64);
+                        uuid_utoa_r (fde->fd->inode->gfid, gfid_str);
+                        gf_proc_dump_write (key, "%s", gfid_str);
+                        file_count++;
+                }
+
+                gf_proc_dump_build_key (key, "client", "files-locked");
+                gf_proc_dump_write (key, "%ld\n", file_count);
+                client_count++;
+        }
+
+        gf_proc_dump_build_key (key, "nlm", "client-count");
+        gf_proc_dump_write (key, "%d", client_count);
+        ret = 0;
+        UNLOCK (&nlm_client_list_lk);
+
+ out:
+        if (ret) {
+                gf_proc_dump_build_key (key, "nlm", "statedump_error");
+                gf_proc_dump_write (key, "Unable to dump nlm state because "
+                                    "nlm_client_list_lk lock couldn't be acquired");
+        }
+
+        return ret;
+}
diff --git a/xlators/nfs/server/src/nlm4.h b/xlators/nfs/server/src/nlm4.h
new file mode 100644
index 000000000..9b5d54081
--- /dev/null
+++ b/xlators/nfs/server/src/nlm4.h
@@ -0,0 +1,77 @@
+/*
+  Copyright (c) 2012 Gluster, Inc. <http://www.gluster.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _NLM4_H_
+#define _NLM4_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/types.h>
+#include <signal.h>
+#include "rpcsvc.h"
+#include "dict.h"
+#include "xlator.h"
+#include "iobuf.h"
+#include "nfs.h"
+#include "list.h"
+#include "xdr-nfs3.h"
+#include "locking.h"
+#include "nfs3-fh.h"
+#include "uuid.h"
+#include "nlm4-xdr.h"
+#include "lkowner.h"
+
+/* Registered with portmap */
+#define GF_NLM4_PORT            38468
+#define GF_NLM                  GF_NFS"-NLM"
+
+extern rpcsvc_program_t *
+nlm4svc_init (xlator_t *nfsx);
+
+extern int
+nlm4_init_state (xlator_t *nfsx);
+
+#define NLM_PROGRAM 100021
+#define NLM_V4 4
+
+typedef struct nlm4_lwowner {
+        char temp[1024];
+} nlm4_lkowner_t;
+
+typedef struct nlm_client {
+        struct sockaddr_storage sa;
+        pid_t uniq;
+        struct list_head nlm_clients;
+        struct list_head fdes;
+        struct list_head shares;
+        struct rpc_clnt *rpc_clnt;
+        char *caller_name;
+        int nsm_monitor;
+} nlm_client_t;
+
+typedef struct nlm_share {
+        struct list_head     client_list;
+        struct list_head     inode_list;
+        gf_lkowner_t         lkowner;
+        inode_t             *inode;
+        fsh_mode             mode;
+        fsh_access           access;
+} nlm_share_t;
+
+typedef struct nlm_fde {
+        struct list_head fde_list;
+        fd_t *fd;
+        int transit_cnt;
+} nlm_fde_t;
+
+#endif
diff --git a/xlators/nfs/server/src/nlmcbk_svc.c b/xlators/nfs/server/src/nlmcbk_svc.c
new file mode 100644
index 000000000..e1b588765
--- /dev/null
+++ b/xlators/nfs/server/src/nlmcbk_svc.c
@@ -0,0 +1,117 @@
+/*
+  Copyright (c) 2012 Gluster, Inc. <http://www.gluster.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#include "nlmcbk-xdr.h"
+#include "nlm4.h"
+#include "logging.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <rpc/pmap_clnt.h>
+#include <string.h>
+#include <memory.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+
+#ifndef SIG_PF
+#define SIG_PF void(*)(int)
+#endif
+
+void
+nlm4svc_sm_notify (struct nlm_sm_status *status);
+
+void *nlmcbk_sm_notify_0_svc(struct nlm_sm_status *status, struct svc_req *req)
+{
+        nlm4svc_sm_notify (status);
+        return NULL;
+}
+
+static void
+nlmcbk_program_0(struct svc_req *rqstp, register SVCXPRT *transp)
+{
+	union {
+		struct nlm_sm_status nlmcbk_sm_notify_0_arg;
+	} argument;
+	char *result;
+	xdrproc_t _xdr_argument, _xdr_result;
+	char *(*local)(char *, struct svc_req *);
+
+	switch (rqstp->rq_proc) {
+	case NULLPROC:
+		(void) svc_sendreply (transp, (xdrproc_t) xdr_void, (char *)NULL);
+		return;
+
+	case NLMCBK_SM_NOTIFY:
+		_xdr_argument = (xdrproc_t) xdr_nlm_sm_status;
+		_xdr_result = (xdrproc_t) xdr_void;
+		local = (char *(*)(char *, struct svc_req *)) nlmcbk_sm_notify_0_svc;
+		break;
+
+	default:
+		svcerr_noproc (transp);
+		return;
+	}
+	memset ((char *)&argument, 0, sizeof (argument));
+	if (!svc_getargs (transp, (xdrproc_t) _xdr_argument, (caddr_t) &argument)) {
+		svcerr_decode (transp);
+		return;
+	}
+	result = (*local)((char *)&argument, rqstp);
+	if (!svc_sendreply(transp, (xdrproc_t) _xdr_result, result)) {
+		svcerr_systemerr (transp);
+	}
+
+	if (!svc_freeargs (transp, (xdrproc_t) _xdr_argument, (caddr_t) &argument)) {
+		gf_log (GF_NLM, GF_LOG_ERROR, "unable to free arguments");
+                return;
+	}
+	return;
+}
+
+void *
+nsm_thread (void *argv)
+{
+	register SVCXPRT *transp;
+        int ret = 0;
+
+        ret = pmap_unset (NLMCBK_PROGRAM, NLMCBK_V1);
+        if (ret == 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "pmap_unset failed");
+                return NULL;
+        }
+        transp = svcudp_create(RPC_ANYSOCK);
+	if (transp == NULL) {
+		gf_log (GF_NLM, GF_LOG_ERROR, "cannot create udp service.");
+                return NULL;
+	}
+	if (!svc_register(transp, NLMCBK_PROGRAM, NLMCBK_V1, nlmcbk_program_0, IPPROTO_UDP)) {
+		gf_log (GF_NLM, GF_LOG_ERROR, "unable to register (NLMCBK_PROGRAM, NLMCBK_V0, udp).");
+                return NULL;
+	}
+
+	transp = svctcp_create(RPC_ANYSOCK, 0, 0);
+	if (transp == NULL) {
+		gf_log (GF_NLM, GF_LOG_ERROR, "cannot create tcp service.");
+                return NULL;
+	}
+	if (!svc_register(transp, NLMCBK_PROGRAM, NLMCBK_V1, nlmcbk_program_0, IPPROTO_TCP)) {
+		gf_log (GF_NLM, GF_LOG_ERROR, "unable to register (NLMCBK_PROGRAM, NLMCBK_V0, tcp).");
+                return NULL;
+	}
+
+	svc_run ();
+	gf_log (GF_NLM, GF_LOG_ERROR, "svc_run returned");
+        return NULL;
+	/* NOTREACHED */
+}
diff --git a/xlators/performance/Makefile.am b/xlators/performance/Makefile.am
index e91d5f6ef..a494190ba 100644
--- a/xlators/performance/Makefile.am
+++ b/xlators/performance/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = write-behind read-ahead io-threads io-cache symlink-cache quick-read stat-prefetch
+SUBDIRS = write-behind read-ahead readdir-ahead io-threads io-cache symlink-cache quick-read md-cache open-behind
 
 CLEANFILES = 
diff --git a/xlators/performance/io-cache/src/Makefile.am b/xlators/performance/io-cache/src/Makefile.am
index 6dd270e8f..155be9988 100644
--- a/xlators/performance/io-cache/src/Makefile.am
+++ b/xlators/performance/io-cache/src/Makefile.am
@@ -1,14 +1,16 @@
 xlator_LTLIBRARIES = io-cache.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-io_cache_la_LDFLAGS = -module -avoidversion 
+io_cache_la_LDFLAGS = -module -avoid-version 
 
 io_cache_la_SOURCES = io-cache.c page.c ioc-inode.c
 io_cache_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = io-cache.h ioc-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/rbtree -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(CONTRIBDIR)/rbtree
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/io-cache/src/io-cache.c b/xlators/performance/io-cache/src/io-cache.c
index 0ab36454c..201777b38 100644
--- a/xlators/performance/io-cache/src/io-cache.c
+++ b/xlators/performance/io-cache/src/io-cache.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -37,13 +28,10 @@ int ioc_log2_page_size;
 uint32_t
 ioc_get_priority (ioc_table_t *table, const char *path);
 
-uint32_t
-ioc_get_priority (ioc_table_t *table, const char *path);
-
 struct volume_options options[];
 
 
-inline uint32_t
+static inline uint32_t
 ioc_hashfn (void *data, int len)
 {
         off_t offset;
@@ -53,7 +41,7 @@ ioc_hashfn (void *data, int len)
         return (offset >> ioc_log2_page_size);
 }
 
-inline ioc_inode_t *
+static inline ioc_inode_t *
 ioc_inode_reupdate (ioc_inode_t *ioc_inode)
 {
         ioc_table_t *table = NULL;
@@ -66,7 +54,7 @@ ioc_inode_reupdate (ioc_inode_t *ioc_inode)
         return ioc_inode;
 }
 
-inline ioc_inode_t *
+static inline ioc_inode_t *
 ioc_get_inode (dict_t *dict, char *name)
 {
         ioc_inode_t *ioc_inode      = NULL;
@@ -157,15 +145,16 @@ ioc_inode_flush (ioc_inode_t *ioc_inode)
 int32_t
 ioc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *preop, struct iatt *postop)
+                 struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop);
+        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop,
+                             xdata);
         return 0;
 }
 
 int32_t
 ioc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid)
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         uint64_t ioc_inode = 0;
 
@@ -177,7 +166,7 @@ ioc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
 
         STACK_WIND (frame, ioc_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid);
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
 
         return 0;
 }
@@ -185,7 +174,7 @@ ioc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 int32_t
 ioc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret,	int32_t op_errno, inode_t *inode,
-                struct iatt *stbuf, dict_t *dict, struct iatt *postparent)
+                struct iatt *stbuf, dict_t *xdata, struct iatt *postparent)
 {
         ioc_inode_t *ioc_inode         = NULL;
         ioc_table_t *table             = NULL;
@@ -264,19 +253,18 @@ out:
         }
 
         STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, stbuf,
-                             dict, postparent);
+                             xdata, postparent);
         return 0;
 }
 
 int32_t
 ioc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            dict_t *xattr_req)
+            dict_t *xdata)
 {
         ioc_local_t *local    = NULL;
         int32_t      op_errno = -1, ret = -1;
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_ioc_mt_ioc_local_t);
+        local = mem_get0 (this->local_pool);
         if (local == NULL) {
                 op_errno = ENOMEM;
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
@@ -293,7 +281,7 @@ ioc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         frame->local = local;
 
         STACK_WIND (frame, ioc_lookup_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lookup, loc, xattr_req);
+                    FIRST_CHILD (this)->fops->lookup, loc, xdata);
 
         return 0;
 
@@ -325,6 +313,20 @@ ioc_forget (xlator_t *this, inode_t *inode)
         return 0;
 }
 
+static int32_t
+ioc_invalidate(xlator_t *this, inode_t *inode)
+{
+	uint64_t     ioc_addr = 0;
+	ioc_inode_t *ioc_inode = NULL;
+
+	inode_ctx_get(inode, this, (uint64_t *) &ioc_addr);
+	ioc_inode = (void *) ioc_addr;
+
+	if (ioc_inode)
+		ioc_inode_flush(ioc_inode);
+
+	return 0;
+}
 
 /*
  * ioc_cache_validate_cbk -
@@ -339,7 +341,8 @@ ioc_forget (xlator_t *this, inode_t *inode)
  */
 int32_t
 ioc_cache_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
+                        int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                        dict_t *xdata)
 {
         ioc_local_t *local        = NULL;
         ioc_inode_t *ioc_inode    = NULL;
@@ -455,8 +458,7 @@ ioc_cache_validate (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
         int32_t       ret            = 0;
 
         local = frame->local;
-        validate_local = GF_CALLOC (1, sizeof (ioc_local_t),
-                                    gf_ioc_mt_ioc_local_t);
+        validate_local = mem_get0 (THIS->local_pool);
         if (validate_local == NULL) {
                 ret = -1;
                 local->op_ret = -1;
@@ -471,7 +473,7 @@ ioc_cache_validate (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
                 ret = -1;
                 local->op_ret = -1;
                 local->op_errno = ENOMEM;
-                GF_FREE (validate_local);
+                mem_put (validate_local);
                 gf_log (ioc_inode->table->xl->name, GF_LOG_ERROR,
                         "out of memory");
                 goto out;
@@ -483,13 +485,13 @@ ioc_cache_validate (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
 
         STACK_WIND (validate_frame, ioc_cache_validate_cbk,
                     FIRST_CHILD (frame->this),
-                    FIRST_CHILD (frame->this)->fops->fstat, fd);
+                    FIRST_CHILD (frame->this)->fops->fstat, fd, NULL);
 
 out:
         return ret;
 }
 
-inline uint32_t
+static inline uint32_t
 is_match (const char *path, const char *pattern)
 {
         int32_t ret = 0;
@@ -530,7 +532,7 @@ ioc_get_priority (ioc_table_t *table, const char *path)
  */
 int32_t
 ioc_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, fd_t *fd)
+              int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         uint64_t     tmp_ioc_inode = 0;
         ioc_local_t *local         = NULL;
@@ -551,6 +553,13 @@ ioc_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
                 inode_ctx_get (fd->inode, this, &tmp_ioc_inode);
                 ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
 
+                //TODO: see why inode context is NULL and handle it.
+                if (!ioc_inode) {
+                        gf_log (this->name, GF_LOG_ERROR, "inode context is "
+                                "NULL (%s)", uuid_utoa (fd->inode->gfid));
+                        goto out;
+                }
+
                 ioc_table_lock (ioc_inode->table);
                 {
                         list_move_tail (&ioc_inode->inode_lru,
@@ -575,10 +584,6 @@ ioc_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
                          */
                         fd_ctx_set (fd, this, 1);
                 }
-                if ((local->wbflags & GF_OPEN_NOWB) != 0) {
-                        /* disable caching as asked by NFS */
-                        fd_ctx_set (fd, this, 1);
-                }
 
                 /* weight = 0, we disable caching on it */
                 if (weight == 0) {
@@ -589,10 +594,10 @@ ioc_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
         }
 
 out:
-        GF_FREE (local);
+        mem_put (local);
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 
         return 0;
 }
@@ -614,7 +619,7 @@ int32_t
 ioc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret,	int32_t op_errno, fd_t *fd,
                 inode_t *inode,	struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
         ioc_local_t *local     = NULL;
         ioc_table_t *table     = NULL;
@@ -686,10 +691,10 @@ ioc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         frame->local = NULL;
-        GF_FREE (local);
+        mem_put (local);
 
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         return 0;
 }
@@ -699,7 +704,7 @@ int32_t
 ioc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, inode_t *inode,
                struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
+               struct iatt *postparent, dict_t *xdata)
 {
         ioc_local_t *local     = NULL;
         ioc_table_t *table     = NULL;
@@ -739,23 +744,22 @@ out:
         frame->local = NULL;
 
         loc_wipe (&local->file_loc);
-        GF_FREE (local);
+        mem_put (local);
 
         STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 ioc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t rdev, dict_t *params)
+           dev_t rdev, mode_t umask, dict_t *xdata)
 {
         ioc_local_t *local    = NULL;
         int32_t      op_errno = -1, ret = -1;
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_ioc_mt_ioc_local_t);
+        local = mem_get0 (this->local_pool);
         if (local == NULL) {
                 op_errno = ENOMEM;
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
@@ -774,17 +778,17 @@ ioc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         STACK_WIND (frame, ioc_mknod_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, params);
+                    loc, mode, rdev, umask, xdata);
         return 0;
 
 unwind:
         if (local != NULL) {
                 loc_wipe (&local->file_loc);
-                GF_FREE (local);
+                mem_put (local);
         }
 
         STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
-                             NULL, NULL);
+                             NULL, NULL, NULL);
 
         return 0;
 }
@@ -800,27 +804,27 @@ unwind:
  */
 int32_t
 ioc_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags)
+          fd_t *fd, dict_t *xdata)
 {
 
         ioc_local_t *local = NULL;
 
-        local = GF_CALLOC (1, sizeof (ioc_local_t), gf_ioc_mt_ioc_local_t);
+        local = mem_get0 (this->local_pool);
         if (local == NULL) {
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
         local->flags = flags;
         local->file_loc.path = loc->path;
         local->file_loc.inode = loc->inode;
-        local->wbflags = wbflags;
 
         frame->local = local;
 
         STACK_WIND (frame, ioc_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd,
+                    xdata);
 
         return 0;
 }
@@ -837,15 +841,15 @@ ioc_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
  */
 int32_t
 ioc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-            mode_t mode, fd_t *fd, dict_t *params)
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         ioc_local_t *local = NULL;
 
-        local = GF_CALLOC (1, sizeof (ioc_local_t), gf_ioc_mt_ioc_local_t);
+        local = mem_get0 (this->local_pool);
         if (local == NULL) {
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
                 STACK_UNWIND_STRICT (create, frame, -1, ENOMEM, NULL, NULL,
-                                     NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -855,7 +859,7 @@ ioc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
 
         STACK_WIND (frame, ioc_create_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create, loc, flags, mode,
-                    fd, params);
+                    umask, fd, xdata);
 
         return 0;
 }
@@ -892,10 +896,10 @@ int32_t
 ioc_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret,	int32_t op_errno, struct iovec *vector,
                         int32_t count, struct iatt *stbuf,
-                        struct iobref *iobref)
+                        struct iobref *iobref, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
+                             stbuf, iobref, xdata);
         return 0;
 }
 
@@ -1002,7 +1006,8 @@ ioc_dispatch_requests (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
                                                 "cache hit for trav_offset=%"
                                                 PRId64"/local_offset=%"PRId64"",
                                                 trav_offset, local_offset);
-                                        waitq = __ioc_page_wakeup (trav);
+                                        waitq = __ioc_page_wakeup (trav,
+                                                                   trav->op_errno);
                                 } else {
                                         /* if waitq already exists, fstat
                                          * revalidate is
@@ -1019,7 +1024,8 @@ ioc_dispatch_requests (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
                                                 local->op_errno = -ret;
                                                 need_validate = 0;
 
-                                                waitq = __ioc_page_wakeup (trav);
+                                                waitq = __ioc_page_wakeup (trav,
+                                                                           trav->op_errno);
                                                 ioc_inode_unlock (ioc_inode);
 
                                                 ioc_waitq_return (waitq);
@@ -1051,7 +1057,8 @@ ioc_dispatch_requests (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
                         if (ret == -1) {
                                 ioc_inode_lock (ioc_inode);
                                 {
-                                        waitq = __ioc_page_wakeup (trav);
+                                        waitq = __ioc_page_wakeup (trav,
+                                                                   trav->op_errno);
                                 }
                                 ioc_inode_unlock (ioc_inode);
 
@@ -1087,14 +1094,13 @@ out:
  */
 int32_t
 ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           size_t size, off_t offset, uint32_t flags)
+           size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         uint64_t     tmp_ioc_inode = 0;
         ioc_inode_t *ioc_inode     = NULL;
         ioc_local_t *local         = NULL;
         uint32_t     weight        = 0;
         ioc_table_t *table         = NULL;
-        uint32_t     num_pages     = 0;
         int32_t      op_errno      = -1;
 
         if (!this) {
@@ -1108,7 +1114,7 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 STACK_WIND (frame, ioc_readv_disabled_cbk,
                             FIRST_CHILD (frame->this),
                             FIRST_CHILD (frame->this)->fops->readv, fd, size,
-                            offset, flags);
+                            offset, flags, xdata);
                 return 0;
         }
 
@@ -1121,29 +1127,6 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto out;
         }
 
-
-        ioc_table_lock (table);
-        {
-                if (!table->mem_pool) {
-
-                        num_pages = (table->cache_size / table->page_size)
-                                + ((table->cache_size % table->page_size)
-                                   ? 1 : 0);
-
-                        table->mem_pool
-                                =  mem_pool_new (rbthash_entry_t, num_pages);
-
-                        if (!table->mem_pool) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to allocate mem_pool");
-                                op_errno = ENOMEM;
-                                ioc_table_unlock (table);
-                                goto out;
-                        }
-                }
-        }
-        ioc_table_unlock (table);
-
         ioc_inode_lock (ioc_inode);
         {
                 if (!ioc_inode->cache.page_table) {
@@ -1167,12 +1150,11 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 STACK_WIND (frame, ioc_readv_disabled_cbk,
                             FIRST_CHILD (frame->this),
                             FIRST_CHILD (frame->this)->fops->readv, fd, size,
-                            offset, flags);
+                            offset, flags, xdata);
                 return 0;
         }
 
-        local = (ioc_local_t *) GF_CALLOC (1, sizeof (ioc_local_t),
-                                           gf_ioc_mt_ioc_local_t);
+        local = mem_get0 (this->local_pool);
         if (local == NULL) {
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
                 op_errno = ENOMEM;
@@ -1205,7 +1187,8 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         return 0;
 
 out:
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                             NULL);
         return 0;
 }
 
@@ -1222,7 +1205,7 @@ out:
 int32_t
 ioc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret,	int32_t op_errno, struct iatt *prebuf,
-                struct iatt *postbuf)
+                struct iatt *postbuf, dict_t *xdata)
 {
         ioc_local_t *local     = NULL;
         uint64_t     ioc_inode = 0;
@@ -1233,7 +1216,8 @@ ioc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (ioc_inode)
                 ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
         return 0;
 }
 
@@ -1251,16 +1235,16 @@ ioc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 ioc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref)
+            uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         ioc_local_t *local     = NULL;
         uint64_t     ioc_inode = 0;
 
-        local = GF_CALLOC (1, sizeof (ioc_local_t), gf_ioc_mt_ioc_local_t);
+        local = mem_get0 (this->local_pool);
         if (local == NULL) {
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
 
-                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1274,7 +1258,7 @@ ioc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         STACK_WIND (frame, ioc_writev_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
-                    flags, iobref);
+                    flags, iobref, xdata);
 
         return 0;
 }
@@ -1293,11 +1277,11 @@ ioc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int32_t
 ioc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
 
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
         return 0;
 }
 
@@ -1316,11 +1300,11 @@ ioc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 ioc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
 
         STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
         return 0;
 }
 
@@ -1335,7 +1319,8 @@ ioc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  *
  */
 int32_t
-ioc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+ioc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
 {
         uint64_t ioc_inode = 0;
 
@@ -1345,7 +1330,7 @@ ioc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
                 ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
 
         STACK_WIND (frame, ioc_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
         return 0;
 }
 
@@ -1359,7 +1344,8 @@ ioc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
  *
  */
 int32_t
-ioc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+ioc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
 {
         uint64_t ioc_inode = 0;
 
@@ -1369,21 +1355,21 @@ ioc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
                 ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
 
         STACK_WIND (frame, ioc_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
         return 0;
 }
 
 int32_t
 ioc_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-            int32_t op_errno, struct gf_flock *lock)
+            int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
         return 0;
 }
 
 int32_t
 ioc_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-        struct gf_flock *lock)
+        struct gf_flock *lock, dict_t *xdata)
 {
         ioc_inode_t *ioc_inode = NULL;
         uint64_t     tmp_inode = 0;
@@ -1393,7 +1379,7 @@ ioc_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
         if (!ioc_inode) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "inode context is NULL: returning EBADFD");
-                STACK_UNWIND_STRICT (lk, frame, -1, EBADFD, NULL);
+                STACK_UNWIND_STRICT (lk, frame, -1, EBADFD, NULL, NULL);
                 return 0;
         }
 
@@ -1404,14 +1390,14 @@ ioc_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
         ioc_inode_unlock (ioc_inode);
 
         STACK_WIND (frame, ioc_lk_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lk, fd, cmd, lock);
+                    FIRST_CHILD (this)->fops->lk, fd, cmd, lock, xdata);
 
         return 0;
 }
 
 int
 ioc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, gf_dirent_t *entries)
+                  int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
         gf_dirent_t *entry = NULL;
 
@@ -1423,7 +1409,7 @@ ioc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 
         return 0;
 }
@@ -1438,6 +1424,58 @@ ioc_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         return 0;
 }
 
+static int32_t
+ioc_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		int32_t op_ret, int32_t op_errno, struct iatt *pre,
+		struct iatt *post, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, pre, post, xdata);
+	return 0;
+}
+
+static int32_t
+ioc_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	    size_t len, dict_t *xdata)
+{
+	uint64_t ioc_inode = 0;
+
+	inode_ctx_get (fd->inode, this, &ioc_inode);
+
+	if (ioc_inode)
+		ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
+
+	STACK_WIND(frame, ioc_discard_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+       return 0;
+}
+
+static int32_t
+ioc_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                struct iatt *post, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT(zerofill, frame, op_ret,
+                            op_errno, pre, post, xdata);
+        return 0;
+}
+
+static int32_t
+ioc_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+        uint64_t ioc_inode = 0;
+
+        inode_ctx_get (fd->inode, this, &ioc_inode);
+
+        if (ioc_inode)
+                ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
+
+        STACK_WIND(frame, ioc_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+       return 0;
+}
+
+
 int32_t
 ioc_get_priority_list (const char *opt_str, struct list_head *first)
 {
@@ -1518,13 +1556,9 @@ ioc_get_priority_list (const char *opt_str, struct list_head *first)
                 stripe_str = strtok_r (NULL, ",", &tmp_str);
         }
 out:
-        if (string != NULL) {
-                GF_FREE (string);
-        }
+        GF_FREE (string);
 
-        if (dup_str != NULL) {
-                GF_FREE (dup_str);
-        }
+        GF_FREE (dup_str);
 
         if (max_pri == -1) {
                 list_for_each_entry_safe (curr, tmp, first, list) {
@@ -1557,7 +1591,7 @@ mem_acct_init (xlator_t *this)
 }
 
 
-gf_boolean_t
+static gf_boolean_t
 check_cache_size_ok (xlator_t *this, uint64_t cache_size)
 {
         gf_boolean_t            ret = _gf_true;
@@ -1580,7 +1614,7 @@ check_cache_size_ok (xlator_t *this, uint64_t cache_size)
         else
                 max_cache_size = total_mem;
 
-        gf_log (this->name, GF_LOG_INFO, "Max cache size is %"PRIu64,
+        gf_log (this->name, GF_LOG_DEBUG, "Max cache size is %"PRIu64,
                 max_cache_size);
 
         if (cache_size > max_cache_size) {
@@ -1676,6 +1710,7 @@ init (xlator_t *this)
         int32_t          ret               = -1;
         glusterfs_ctx_t *ctx               = NULL;
         data_t          *data              = 0;
+        uint32_t         num_pages         = 0;
 
         xl_options = this->options;
 
@@ -1751,8 +1786,28 @@ init (xlator_t *this)
         for (index = 0; index < (table->max_pri); index++)
                 INIT_LIST_HEAD (&table->inode_lru[index]);
 
+        this->local_pool = mem_pool_new (ioc_local_t, 64);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
         pthread_mutex_init (&table->table_lock, NULL);
         this->private = table;
+
+        num_pages = (table->cache_size / table->page_size)
+                + ((table->cache_size % table->page_size)
+                   ? 1 : 0);
+
+        table->mem_pool = mem_pool_new (rbthash_entry_t, num_pages);
+        if (!table->mem_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to allocate mem_pool");
+                goto out;
+        }
+
         ret = 0;
 
         ctx = this->ctx;
@@ -1812,7 +1867,16 @@ void
 __ioc_page_dump (ioc_page_t *page, char *prefix)
 {
 
-        ioc_page_lock (page);
+        int    ret = -1;
+
+        if (!page)
+                return;
+        /* ioc_page_lock can be used to hold the mutex. But in statedump
+         * its better to use trylock to avoid deadlocks.
+         */
+        ret = pthread_mutex_trylock (&page->page_lock);
+        if (ret)
+                goto out;
         {
                 gf_proc_dump_write ("offset", "%"PRId64, page->offset);
                 gf_proc_dump_write ("size", "%"PRId64, page->size);
@@ -1820,7 +1884,14 @@ __ioc_page_dump (ioc_page_t *page, char *prefix)
                 gf_proc_dump_write ("ready", "%s", page->ready ? "yes" : "no");
                 ioc_page_waitq_dump (page, prefix);
         }
-        ioc_page_unlock (page);
+        pthread_mutex_unlock (&page->page_lock);
+
+out:
+        if (ret && page)
+                gf_proc_dump_write ("Unable to dump the page information",
+                                    "(Lock acquisition failed) %p", page);
+
+        return;
 }
 
 void
@@ -1830,7 +1901,6 @@ __ioc_cache_dump (ioc_inode_t *ioc_inode, char *prefix)
         ioc_table_t *table                    = NULL;
         ioc_page_t  *page                     = NULL;
         int          i                        = 0;
-        struct tm   *tm                       = NULL;
         char         key[GF_DUMP_MAX_BUF_LEN] = {0, };
         char         timestr[256]             = {0, };
 
@@ -1840,12 +1910,15 @@ __ioc_cache_dump (ioc_inode_t *ioc_inode, char *prefix)
 
         table = ioc_inode->table;
 
-        tm = localtime (&ioc_inode->cache.tv.tv_sec);
-        strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-        snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                  ".%"GF_PRI_SUSECONDS, ioc_inode->cache.tv.tv_usec);
+        if (ioc_inode->cache.tv.tv_sec) {
+                gf_time_fmt (timestr, sizeof timestr,
+                             ioc_inode->cache.tv.tv_sec, gf_timefmt_FT);
+                snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, ioc_inode->cache.tv.tv_usec);
 
-        gf_proc_dump_write ("last-cache-validation-time", "%s", timestr);
+                gf_proc_dump_write ("last-cache-validation-time", "%s",
+                                    timestr);
+        }
 
         for (offset = 0; offset < ioc_inode->ia_size;
              offset += table->page_size) {
@@ -1862,41 +1935,91 @@ out:
 }
 
 
-void
-ioc_inode_dump (ioc_inode_t *ioc_inode, char *prefix)
+int
+ioc_inode_dump (xlator_t *this, inode_t *inode)
 {
 
-       if ((ioc_inode == NULL) || (prefix == NULL)) {
+        char         *path                            = NULL;
+        int           ret                             = -1;
+        char          key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
+        uint64_t      tmp_ioc_inode                   = 0;
+        ioc_inode_t  *ioc_inode                       = NULL;
+        gf_boolean_t  section_added                   = _gf_false;
+        char          uuid_str[64]                    = {0,};
+
+        if (this == NULL || inode == NULL)
+                goto out;
+
+        gf_proc_dump_build_key (key_prefix, "io-cache", "inode");
+
+        inode_ctx_get (inode, this, &tmp_ioc_inode);
+        ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
+        if (ioc_inode == NULL)
+                goto out;
+
+        /* Similar to ioc_page_dump function its better to use
+         * pthread_mutex_trylock and not to use gf_log in statedump
+         * to avoid deadlocks.
+         */
+        ret = pthread_mutex_trylock (&ioc_inode->inode_lock);
+        if (ret)
                 goto out;
-        }
 
-        ioc_inode_lock (ioc_inode);
         {
+                if (uuid_is_null (ioc_inode->inode->gfid))
+                        goto unlock;
+
+                gf_proc_dump_add_section (key_prefix);
+                section_added = _gf_true;
+
+                __inode_path (ioc_inode->inode, NULL, &path);
+
                 gf_proc_dump_write ("inode.weight", "%d", ioc_inode->weight);
-                __ioc_cache_dump (ioc_inode, prefix);
-                __ioc_inode_waitq_dump (ioc_inode, prefix);
+
+                if (path) {
+                        gf_proc_dump_write ("path", "%s", path);
+                        GF_FREE (path);
+                }
+
+                gf_proc_dump_write ("uuid", "%s", uuid_utoa_r
+                                    (ioc_inode->inode->gfid, uuid_str));
+                __ioc_cache_dump (ioc_inode, key_prefix);
+                __ioc_inode_waitq_dump (ioc_inode, key_prefix);
         }
-        ioc_inode_unlock (ioc_inode);
+unlock:
+        pthread_mutex_unlock (&ioc_inode->inode_lock);
+
 out:
-        return;
+        if (ret && ioc_inode) {
+                if (section_added == _gf_false)
+                        gf_proc_dump_add_section (key_prefix);
+                gf_proc_dump_write ("Unable to print the status of ioc_inode",
+                                    "(Lock acquisition failed) %s",
+                                    uuid_utoa (inode->gfid));
+        }
+        return ret;
 }
 
 int
 ioc_priv_dump (xlator_t *this)
 {
         ioc_table_t *priv                            = NULL;
-        ioc_inode_t *ioc_inode                       = NULL;
         char         key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
+        int          ret                             = -1;
+        gf_boolean_t add_section                     = _gf_false;
 
         if (!this || !this->private)
                 goto out;
 
         priv = this->private;
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.io-cache",
-                                "priv");
+
+        gf_proc_dump_build_key (key_prefix, "io-cache", "priv");
         gf_proc_dump_add_section (key_prefix);
+        add_section = _gf_true;
 
-        ioc_table_lock (priv);
+        ret = pthread_mutex_trylock (&priv->table_lock);
+        if (ret)
+                goto out;
         {
                 gf_proc_dump_write ("page_size", "%ld", priv->page_size);
                 gf_proc_dump_write ("cache_size", "%ld", priv->cache_size);
@@ -1905,13 +2028,20 @@ ioc_priv_dump (xlator_t *this)
                 gf_proc_dump_write ("cache_timeout", "%u", priv->cache_timeout);
                 gf_proc_dump_write ("min-file-size", "%u", priv->min_file_size);
                 gf_proc_dump_write ("max-file-size", "%u", priv->max_file_size);
-
-                list_for_each_entry (ioc_inode, &priv->inodes, inode_list) {
-                        ioc_inode_dump (ioc_inode, key_prefix);
-                }
         }
-        ioc_table_unlock (priv);
+        pthread_mutex_unlock (&priv->table_lock);
 out:
+        if (ret && priv) {
+                if (!add_section) {
+                        gf_proc_dump_build_key (key_prefix, "xlator."
+                                                "performance.io-cache", "priv");
+                        gf_proc_dump_add_section (key_prefix);
+                }
+                gf_proc_dump_write ("Unable to dump the state of private "
+                                    "structure of io-cache xlator", "(Lock "
+                                    "acquisition failed) %s", this->name);
+        }
+
         return 0;
 }
 
@@ -1924,18 +2054,33 @@ out:
 void
 fini (xlator_t *this)
 {
-        ioc_table_t *table = NULL;
+        ioc_table_t         *table = NULL;
+        struct ioc_priority *curr  = NULL, *tmp = NULL;
+        int                  i     = 0;
 
         table = this->private;
 
         if (table == NULL)
                 return;
 
+        this->private = NULL;
+
         if (table->mem_pool != NULL) {
                 mem_pool_destroy (table->mem_pool);
                 table->mem_pool = NULL;
         }
 
+        list_for_each_entry_safe (curr, tmp, &table->priority_list, list) {
+                list_del_init (&curr->list);
+                GF_FREE (curr->pattern);
+                GF_FREE (curr);
+        }
+
+        for (i = 0; i < table->max_pri; i++) {
+                GF_ASSERT (list_empty (&table->inode_lru[i]));
+        }
+
+        GF_ASSERT (list_empty (&table->inodes));
         pthread_mutex_destroy (&table->table_lock);
         GF_FREE (table);
 
@@ -1956,21 +2101,25 @@ struct xlator_fops fops = {
         .mknod       = ioc_mknod,
 
         .readdirp    = ioc_readdirp,
+	.discard     = ioc_discard,
+        .zerofill    = ioc_zerofill,
 };
 
 
 struct xlator_dumpops dumpops = {
         .priv        = ioc_priv_dump,
+        .inodectx    = ioc_inode_dump,
 };
 
 struct xlator_cbks cbks = {
         .forget      = ioc_forget,
-        .release     = ioc_release
+        .release     = ioc_release,
+	.invalidate  = ioc_invalidate,
 };
 
 struct volume_options options[] = {
         { .key  = {"priority"},
-          .type = GF_OPTION_TYPE_ANY,
+          .type = GF_OPTION_TYPE_PRIORITY_LIST,
           .default_value = "",
           .description = "Assigns priority to filenames with specific "
           "patterns so that when a page needs to be ejected "
diff --git a/xlators/performance/io-cache/src/io-cache.h b/xlators/performance/io-cache/src/io-cache.h
index eec24f143..46d758a66 100644
--- a/xlators/performance/io-cache/src/io-cache.h
+++ b/xlators/performance/io-cache/src/io-cache.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __IO_CACHE_H
@@ -84,7 +75,6 @@ struct ioc_fill {
 struct ioc_local {
         mode_t           mode;
         int32_t          flags;
-        int32_t          wbflags;
         loc_t            file_loc;
         off_t            offset;
         size_t           size;
@@ -126,6 +116,8 @@ struct ioc_page {
         struct ioc_waitq    *waitq;
         struct iobref       *iobref;
         pthread_mutex_t     page_lock;
+        int32_t             op_errno;
+        char                stale;
 };
 
 struct ioc_cache {
@@ -196,7 +188,7 @@ int32_t
 ioc_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iovec *vector,
                         int32_t count, struct iatt *stbuf,
-                        struct iobref *iobref);
+                        struct iobref *iobref, dict_t *xdata);
 
 ioc_page_t *
 __ioc_page_get (ioc_inode_t *ioc_inode, off_t offset);
@@ -212,7 +204,7 @@ __ioc_wait_on_page (ioc_page_t *page, call_frame_t *frame, off_t offset,
                   size_t size);
 
 ioc_waitq_t *
-__ioc_page_wakeup (ioc_page_t *page);
+__ioc_page_wakeup (ioc_page_t *page, int32_t op_errno);
 
 void
 ioc_page_flush (ioc_page_t *page);
@@ -228,7 +220,7 @@ ioc_waitq_return (ioc_waitq_t *waitq);
 
 int32_t
 ioc_frame_fill (ioc_page_t *page, call_frame_t *frame, off_t offset,
-                size_t size);
+                size_t size, int32_t op_errno);
 
 #define ioc_inode_lock(ioc_inode)                                       \
         do {                                                            \
@@ -337,6 +329,4 @@ ioc_prune (ioc_table_t *table);
 int32_t
 ioc_need_prune (ioc_table_t *table);
 
-inline uint32_t
-ioc_hashfn (void *data, int len);
 #endif /* __IO_CACHE_H */
diff --git a/xlators/performance/io-cache/src/ioc-inode.c b/xlators/performance/io-cache/src/ioc-inode.c
index 4c48c41d1..86a54bb14 100644
--- a/xlators/performance/io-cache/src/ioc-inode.c
+++ b/xlators/performance/io-cache/src/ioc-inode.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -123,7 +114,8 @@ ioc_inode_wakeup (call_frame_t *frame, ioc_inode_t *ioc_inode,
                                 ioc_inode_lock (ioc_inode);
                                 {
                                         page_waitq =
-                                                __ioc_page_wakeup (waiter_page);
+                                                __ioc_page_wakeup (waiter_page,
+                                                                   waiter_page->op_errno);
                                 }
                                 ioc_inode_unlock (ioc_inode);
                                 if (page_waitq)
diff --git a/xlators/performance/io-cache/src/ioc-mem-types.h b/xlators/performance/io-cache/src/ioc-mem-types.h
index 421485e26..9b68f9fce 100644
--- a/xlators/performance/io-cache/src/ioc-mem-types.h
+++ b/xlators/performance/io-cache/src/ioc-mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __IOC_MT_H__
@@ -26,7 +17,6 @@ enum gf_ioc_mem_types_ {
         gf_ioc_mt_iovec  = gf_common_mt_end + 1,
         gf_ioc_mt_ioc_table_t,
         gf_ioc_mt_char,
-        gf_ioc_mt_ioc_local_t,
         gf_ioc_mt_ioc_waitq_t,
         gf_ioc_mt_ioc_priority,
         gf_ioc_mt_list_head,
diff --git a/xlators/performance/io-cache/src/page.c b/xlators/performance/io-cache/src/page.c
index 93c4a51de..b2e20ba65 100644
--- a/xlators/performance/io-cache/src/page.c
+++ b/xlators/performance/io-cache/src/page.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -111,6 +102,7 @@ __ioc_page_destroy (ioc_page_t *page)
         if (page->waitq) {
                 /* frames waiting on this page, do not destroy this page */
                 page_size = -1;
+                page->stale = 1;
         } else {
                 rbthash_remove (page->inode->cache.page_table, &page->offset,
                                 sizeof (page->offset));
@@ -416,7 +408,8 @@ ioc_waitq_return (ioc_waitq_t *waitq)
 int
 ioc_fault_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iovec *vector,
-               int32_t count, struct iatt *stbuf, struct iobref *iobref)
+               int32_t count, struct iatt *stbuf, struct iobref *iobref,
+               dict_t *xdata)
 {
         ioc_local_t *local            = NULL;
         off_t        offset           = 0;
@@ -517,6 +510,7 @@ ioc_fault_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                  * byte replies */
                                 page_size = iov_length(vector, count);
                                 page->size = page_size;
+                                page->op_errno = op_errno;
 
                                 iobref_page_size = iobref_size (page->iobref);
 
@@ -524,7 +518,8 @@ ioc_fault_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         /* wake up all the frames waiting on
                                          * this page, including
                                          * the frame which triggered fault */
-                                        waitq = __ioc_page_wakeup (page);
+                                        waitq = __ioc_page_wakeup (page,
+                                                                   op_errno);
                                 } /* if(page->waitq) */
                         } /* if(!page)...else */
                 } /* if(op_ret < 0)...else */
@@ -602,8 +597,7 @@ ioc_page_fault (ioc_inode_t *ioc_inode, call_frame_t *frame, fd_t *fd,
                 goto err;
         }
 
-        fault_local = GF_CALLOC (1, sizeof (ioc_local_t),
-                                 gf_ioc_mt_ioc_local_t);
+        fault_local = mem_get0 (THIS->local_pool);
         if (fault_local == NULL) {
                 op_ret = -1;
                 op_errno = ENOMEM;
@@ -630,7 +624,7 @@ ioc_page_fault (ioc_inode_t *ioc_inode, call_frame_t *frame, fd_t *fd,
 
         STACK_WIND (fault_frame, ioc_fault_cbk, FIRST_CHILD(fault_frame->this),
                     FIRST_CHILD(fault_frame->this)->fops->readv, fd,
-                    table->page_size, offset, 0);
+                    table->page_size, offset, 0, NULL);
         return;
 
 err:
@@ -651,7 +645,7 @@ err:
 
 int32_t
 __ioc_frame_fill (ioc_page_t *page, call_frame_t *frame, off_t offset,
-                  size_t size)
+                  size_t size, int32_t op_errno)
 {
         ioc_local_t *local      = NULL;
         ioc_fill_t  *fill       = NULL;
@@ -686,7 +680,13 @@ __ioc_frame_fill (ioc_page_t *page, call_frame_t *frame, off_t offset,
         /* immediately move this page to the end of the page_lru list */
         list_move_tail (&page->page_lru, &ioc_inode->cache.page_lru);
         /* fill local->pending_size bytes from local->pending_offset */
-        if (local->op_ret != -1 && page->size) {
+        if (local->op_ret != -1) {
+                local->op_errno = op_errno;
+
+                if (page->size == 0) {
+                        goto done;
+                }
+
                 if (offset > page->offset)
                         /* offset is offset in file, convert it to offset in
                          * page */
@@ -779,6 +779,7 @@ __ioc_frame_fill (ioc_page_t *page, call_frame_t *frame, off_t offset,
                 local->op_ret += copy_size;
         }
 
+done:
         ret = 0;
 out:
         return ret;
@@ -803,7 +804,7 @@ ioc_frame_unwind (call_frame_t *frame)
         int32_t        copied = 0;
         struct iobref *iobref = NULL;
         struct iatt    stbuf  = {0,};
-        int32_t        op_ret = 0;
+        int32_t        op_ret = 0, op_errno = 0;
 
         GF_ASSERT (frame);
 
@@ -812,16 +813,21 @@ ioc_frame_unwind (call_frame_t *frame)
                 gf_log (frame->this->name, GF_LOG_WARNING,
                         "local is NULL");
                 op_ret = -1;
-                local->op_errno = ENOMEM;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        if (local->op_ret < 0) {
+                op_ret = local->op_ret;
+                op_errno = local->op_errno;
                 goto unwind;
         }
 
         //  ioc_local_lock (local);
-        frame->local = NULL;
         iobref = iobref_new ();
         if (iobref == NULL) {
                 op_ret = -1;
-                local->op_errno = ENOMEM;
+                op_errno = ENOMEM;
         }
 
         if (list_empty (&local->fill_list)) {
@@ -838,7 +844,7 @@ ioc_frame_unwind (call_frame_t *frame)
         vector = GF_CALLOC (count, sizeof (*vector), gf_ioc_mt_iovec);
         if (vector == NULL) {
                 op_ret = -1;
-                local->op_errno = ENOMEM;
+                op_errno = ENOMEM;
         }
 
         list_for_each_entry_safe (fill, next, &local->fill_list, list) {
@@ -868,8 +874,9 @@ unwind:
 
         //  ioc_local_unlock (local);
 
-        STACK_UNWIND_STRICT (readv, frame, op_ret, local->op_errno, vector,
-                             count, &stbuf, iobref);
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
+                             count, &stbuf, iobref, NULL);
 
         if (iobref != NULL) {
                 iobref_unref (iobref);
@@ -881,7 +888,8 @@ unwind:
         }
 
         pthread_mutex_destroy (&local->local_lock);
-        GF_FREE (local);
+        if (local)
+                mem_put (local);
 
         return;
 }
@@ -923,7 +931,7 @@ ioc_frame_return (call_frame_t *frame)
  * to be called only when a frame is waiting on an in-transit page
  */
 ioc_waitq_t *
-__ioc_page_wakeup (ioc_page_t *page)
+__ioc_page_wakeup (ioc_page_t *page, int32_t op_errno)
 {
         ioc_waitq_t  *waitq = NULL, *trav = NULL;
         call_frame_t *frame = NULL;
@@ -942,12 +950,16 @@ __ioc_page_wakeup (ioc_page_t *page)
         for (trav = waitq; trav; trav = trav->next) {
                 frame = trav->data;
                 ret = __ioc_frame_fill (page, frame, trav->pending_offset,
-                                        trav->pending_size);
+                                        trav->pending_size, op_errno);
                 if (ret == -1) {
                         break;
                 }
         }
 
+        if (page->stale) {
+                __ioc_page_destroy (page);
+        }
+
 out:
         return waitq;
 }
diff --git a/xlators/performance/io-threads/src/Makefile.am b/xlators/performance/io-threads/src/Makefile.am
index 72f9a8012..d63042e7c 100644
--- a/xlators/performance/io-threads/src/Makefile.am
+++ b/xlators/performance/io-threads/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = io-threads.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-io_threads_la_LDFLAGS = -module -avoidversion 
+io_threads_la_LDFLAGS = -module -avoid-version 
 
 io_threads_la_SOURCES = io-threads.c
 io_threads_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = io-threads.h iot-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index d3196d50b..bbcf4ed26 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -39,16 +30,69 @@ int __iot_workers_scale (iot_conf_t *conf);
 struct volume_options options[];
 
 call_stub_t *
-__iot_dequeue (iot_conf_t *conf, int *pri)
+__iot_dequeue (iot_conf_t *conf, int *pri, struct timespec *sleep)
 {
         call_stub_t  *stub = NULL;
         int           i = 0;
+	struct timeval curtv = {0,}, difftv = {0,};
 
         *pri = -1;
+	sleep->tv_sec = 0;
+	sleep->tv_nsec = 0;
         for (i = 0; i < IOT_PRI_MAX; i++) {
                 if (list_empty (&conf->reqs[i]) ||
                    (conf->ac_iot_count[i] >= conf->ac_iot_limit[i]))
                         continue;
+
+		if (i == IOT_PRI_LEAST) {
+			pthread_mutex_lock(&conf->throttle.lock);
+			if (!conf->throttle.sample_time.tv_sec) {
+				/* initialize */
+				gettimeofday(&conf->throttle.sample_time, NULL);
+			} else {
+				/*
+				 * Maintain a running count of least priority
+				 * operations that are handled over a particular
+				 * time interval. The count is provided via
+				 * state dump and is used as a measure against
+				 * least priority op throttling.
+				 */
+				gettimeofday(&curtv, NULL);
+				timersub(&curtv, &conf->throttle.sample_time,
+					 &difftv);
+				if (difftv.tv_sec >= IOT_LEAST_THROTTLE_DELAY) {
+					conf->throttle.cached_rate =
+						conf->throttle.sample_cnt;
+					conf->throttle.sample_cnt = 0;
+					conf->throttle.sample_time = curtv;
+				}
+
+				/*
+				 * If we're over the configured rate limit,
+				 * provide an absolute time to the caller that
+				 * represents the soonest we're allowed to
+				 * return another least priority request.
+				 */
+				if (conf->throttle.rate_limit &&
+				    conf->throttle.sample_cnt >=
+						conf->throttle.rate_limit) {
+					struct timeval delay;
+					delay.tv_sec = IOT_LEAST_THROTTLE_DELAY;
+					delay.tv_usec = 0;
+
+					timeradd(&conf->throttle.sample_time,
+						 &delay, &curtv);
+					TIMEVAL_TO_TIMESPEC(&curtv, sleep);
+
+					pthread_mutex_unlock(
+						&conf->throttle.lock);
+					break;
+				}
+			}
+			conf->throttle.sample_cnt++;
+			pthread_mutex_unlock(&conf->throttle.lock);
+		}
+
                 stub = list_entry (conf->reqs[i].next, call_stub_t, list);
                 conf->ac_iot_count[i]++;
                 *pri = i;
@@ -59,6 +103,7 @@ __iot_dequeue (iot_conf_t *conf, int *pri)
                 return NULL;
 
         conf->queue_size--;
+        conf->queue_sizes[*pri]--;
         list_del_init (&stub->list);
 
         return stub;
@@ -74,6 +119,7 @@ __iot_enqueue (iot_conf_t *conf, call_stub_t *stub, int pri)
         list_add_tail (&stub->list, &conf->reqs[pri]);
 
         conf->queue_size++;
+        conf->queue_sizes[pri]++;
 
         return;
 }
@@ -90,6 +136,7 @@ iot_worker (void *data)
         int               pri = -1;
         char              timeout = 0;
         char              bye = 0;
+	struct timespec	  sleep = {0,};
 
         conf = data;
         this = conf->this;
@@ -130,7 +177,13 @@ iot_worker (void *data)
                                 }
                         }
 
-                        stub = __iot_dequeue (conf, &pri);
+                        stub = __iot_dequeue (conf, &pri, &sleep);
+			if (!stub && (sleep.tv_sec || sleep.tv_nsec)) {
+				pthread_cond_timedwait(&conf->cond,
+						       &conf->mutex, &sleep);
+				pthread_mutex_unlock(&conf->mutex);
+				continue;
+			}
                 }
                 pthread_mutex_unlock (&conf->mutex);
 
@@ -199,8 +252,9 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
 {
         int             ret = -1;
         iot_pri_t       pri = IOT_PRI_MAX - 1;
+        iot_conf_t      *conf = this->private;
 
-        if (frame->root->pid < 0) {
+        if ((frame->root->pid < GF_CLIENT_PID_MAX) && conf->least_priority) {
                 pri = IOT_PRI_LEAST;
                 goto out;
         }
@@ -252,11 +306,11 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_FSYNCDIR:
         case GF_FOP_XATTROP:
         case GF_FOP_FXATTROP:
-                pri = IOT_PRI_LO;
-                break;
-
         case GF_FOP_RCHECKSUM:
-                pri = IOT_PRI_LEAST;
+	case GF_FOP_FALLOCATE:
+	case GF_FOP_DISCARD:
+        case GF_FOP_ZEROFILL:
+                pri = IOT_PRI_LO;
                 break;
 
         case GF_FOP_NULL:
@@ -270,19 +324,19 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
                 break;
         }
 out:
-        ret = do_iot_schedule (this->private, stub, pri);
         gf_log (this->name, GF_LOG_DEBUG, "%s scheduled as %s fop",
                 gf_fop_list[stub->fop], iot_get_pri_meaning (pri));
+        ret = do_iot_schedule (this->private, stub, pri);
         return ret;
 }
 
 int
 iot_lookup_cbk (call_frame_t *frame, void * cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
-                inode_t *inode, struct iatt *buf, dict_t *xattr,
+                inode_t *inode, struct iatt *buf, dict_t *xdata,
                 struct iatt *postparent)
 {
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xattr,
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xdata,
                              postparent);
         return 0;
 }
@@ -290,23 +344,23 @@ iot_lookup_cbk (call_frame_t *frame, void * cookie, xlator_t *this,
 
 int
 iot_lookup_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    dict_t *xattr_req)
+                    dict_t *xdata)
 {
         STACK_WIND (frame, iot_lookup_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->lookup,
-                    loc, xattr_req);
+                    loc, xdata);
         return 0;
 }
 
 
 int
-iot_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+iot_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int              ret = -1;
 
-        stub = fop_lookup_stub (frame, iot_lookup_wrapper, loc, xattr_req);
+        stub = fop_lookup_stub (frame, iot_lookup_wrapper, loc, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "cannot create lookup stub (out of memory)");
@@ -332,33 +386,35 @@ out:
 int
 iot_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *preop, struct iatt *postop)
+                 struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop);
+        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop,
+                             xdata);
         return 0;
 }
 
 
 int
 iot_setattr_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     struct iatt *stbuf, int32_t valid)
+                     struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         STACK_WIND (frame, iot_setattr_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->setattr,
-                    loc, stbuf, valid);
+                    loc, stbuf, valid, xdata);
         return 0;
 }
 
 
 int
 iot_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid)
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int              ret = -1;
 
-        stub = fop_setattr_stub (frame, iot_setattr_wrapper, loc, stbuf, valid);
+        stub = fop_setattr_stub (frame, iot_setattr_wrapper, loc, stbuf, valid,
+                                 xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "Cannot create setattr stub"
                         "(Out of memory)");
@@ -374,7 +430,7 @@ out:
                         call_stub_destroy (stub);
                 }
 
-                STACK_UNWIND_STRICT (setattr, frame, -1, -ret, NULL, NULL);
+                STACK_UNWIND_STRICT (setattr, frame, -1, -ret, NULL, NULL, NULL);
         }
 
         return 0;
@@ -384,32 +440,34 @@ out:
 int
 iot_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *preop, struct iatt *postop)
+                  struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, preop, postop);
+        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, preop, postop,
+                             xdata);
         return 0;
 }
 
 
 int
 iot_fsetattr_wrapper (call_frame_t *frame, xlator_t *this,
-                      fd_t *fd, struct iatt *stbuf, int32_t valid)
+                      fd_t *fd, struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         STACK_WIND (frame, iot_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid);
+                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid,
+                    xdata);
         return 0;
 }
 
 
 int
 iot_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iatt *stbuf, int32_t valid)
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int              ret = -1;
 
         stub = fop_fsetattr_stub (frame, iot_fsetattr_wrapper, fd, stbuf,
-                                  valid);
+                                  valid, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create fsetattr stub"
                         "(out of memory)");
@@ -421,7 +479,8 @@ iot_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, -ret, NULL, NULL);
+                STACK_UNWIND_STRICT (fsetattr, frame, -1, -ret, NULL, NULL,
+                                     NULL);
                 if (stub != NULL) {
                         call_stub_destroy (stub);
                 }
@@ -432,30 +491,31 @@ out:
 
 int
 iot_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno)
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 iot_access_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    int32_t mask)
+                    int32_t mask, dict_t *xdata)
 {
         STACK_WIND (frame, iot_access_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->access, loc, mask);
+                    FIRST_CHILD (this)->fops->access, loc, mask, xdata);
         return 0;
 }
 
 
 int
-iot_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+iot_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+            dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
-        stub = fop_access_stub (frame, iot_access_wrapper, loc, mask);
+        stub = fop_access_stub (frame, iot_access_wrapper, loc, mask, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create access stub"
                         "(out of memory)");
@@ -466,7 +526,7 @@ iot_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (access, frame, -1, -ret);
+                STACK_UNWIND_STRICT (access, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -479,32 +539,33 @@ out:
 int
 iot_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, const char *path,
-                  struct iatt *stbuf)
+                  struct iatt *stbuf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, stbuf);
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, stbuf,
+                             xdata);
         return 0;
 }
 
 
 int
 iot_readlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      size_t size)
+                      size_t size, dict_t *xdata)
 {
         STACK_WIND (frame, iot_readlink_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->readlink,
-                    loc, size);
+                    loc, size, xdata);
         return 0;
 }
 
 
 int
-iot_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+iot_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
-        stub = fop_readlink_stub (frame, iot_readlink_wrapper, loc, size);
+        stub = fop_readlink_stub (frame, iot_readlink_wrapper, loc, size, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create readlink stub"
                         "(out of memory)");
@@ -516,7 +577,7 @@ iot_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
 
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (readlink, frame, -1, -ret, NULL, NULL);
+                STACK_UNWIND_STRICT (readlink, frame, -1, -ret, NULL, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -531,33 +592,34 @@ int
 iot_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, inode_t *inode,
                struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
+               struct iatt *postparent, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 iot_mknod_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-                   dev_t rdev, dict_t *params)
+                   dev_t rdev, mode_t umask, dict_t *xdata)
 {
         STACK_WIND (frame, iot_mknod_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->mknod, loc, mode, rdev, params);
+                    FIRST_CHILD (this)->fops->mknod, loc, mode, rdev, umask,
+                    xdata);
         return 0;
 }
 
 
 int
 iot_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t rdev, dict_t *params)
+           dev_t rdev, mode_t umask, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_mknod_stub (frame, iot_mknod_wrapper, loc, mode, rdev,
-                               params);
+                               umask, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create mknod stub"
                         "(out of memory)");
@@ -570,7 +632,7 @@ iot_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (mknod, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
+                                     NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -584,32 +646,33 @@ int
 iot_mkdir_cbk (call_frame_t *frame, void * cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, inode_t *inode,
                struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
+               struct iatt *postparent, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 iot_mkdir_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-                   dict_t *params)
+                   mode_t umask, dict_t *xdata)
 {
         STACK_WIND (frame, iot_mkdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->mkdir, loc, mode, params);
+                    FIRST_CHILD (this)->fops->mkdir, loc, mode, umask, xdata);
         return 0;
 }
 
 
 int
 iot_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dict_t *params)
+           mode_t umask, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
-        stub = fop_mkdir_stub (frame, iot_mkdir_wrapper, loc, mode, params);
+        stub = fop_mkdir_stub (frame, iot_mkdir_wrapper, loc, mode, umask,
+                               xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create mkdir stub"
                         "(out of memory)");
@@ -622,7 +685,7 @@ iot_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (mkdir, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
+                                     NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -635,30 +698,30 @@ out:
 int
 iot_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-               struct iatt *postparent)
+               struct iatt *postparent, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent);
+                             postparent, xdata);
         return 0;
 }
 
 
 int
-iot_rmdir_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+iot_rmdir_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
 {
         STACK_WIND (frame, iot_rmdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rmdir, loc, flags);
+                    FIRST_CHILD (this)->fops->rmdir, loc, flags, xdata);
         return 0;
 }
 
 
 int
-iot_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+iot_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
-        stub = fop_rmdir_stub (frame, iot_rmdir_wrapper, loc, flags);
+        stub = fop_rmdir_stub (frame, iot_rmdir_wrapper, loc, flags, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create rmdir stub"
                         "(out of memory)");
@@ -669,7 +732,7 @@ iot_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (rmdir, frame, -1, -ret, NULL, NULL);
+                STACK_UNWIND_STRICT (rmdir, frame, -1, -ret, NULL, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -683,33 +746,34 @@ int
 iot_symlink_cbk (call_frame_t *frame, void * cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
                  struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 iot_symlink_wrapper (call_frame_t *frame, xlator_t *this, const char *linkname,
-                     loc_t *loc, dict_t *params)
+                     loc_t *loc, mode_t umask, dict_t *xdata)
 {
         STACK_WIND (frame, iot_symlink_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->symlink, linkname, loc, params);
+                    FIRST_CHILD (this)->fops->symlink, linkname, loc, umask,
+                    xdata);
         return 0;
 }
 
 
 int
 iot_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-             loc_t *loc, dict_t *params)
+             loc_t *loc, mode_t umask, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_symlink_stub (frame, iot_symlink_wrapper, linkname, loc,
-                                 params);
+                                 umask, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create symlink stub"
                         "(out of memory)");
@@ -722,7 +786,7 @@ iot_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (symlink, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
+                                     NULL, NULL);
                 if (stub != NULL) {
                         call_stub_destroy (stub);
                 }
@@ -736,31 +800,33 @@ int
 iot_rename_cbk (call_frame_t *frame, void * cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
                 struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent)
+                struct iatt *prenewparent, struct iatt *postnewparent,
+                dict_t *xdata)
 {
         STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent);
+                             postoldparent, prenewparent, postnewparent, xdata);
         return 0;
 }
 
 
 int
 iot_rename_wrapper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                    loc_t *newloc)
+                    loc_t *newloc, dict_t *xdata)
 {
         STACK_WIND (frame, iot_rename_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rename, oldloc, newloc);
+                    FIRST_CHILD (this)->fops->rename, oldloc, newloc, xdata);
         return 0;
 }
 
 
 int
-iot_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+iot_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
-        stub = fop_rename_stub (frame, iot_rename_wrapper, oldloc, newloc);
+        stub = fop_rename_stub (frame, iot_rename_wrapper, oldloc, newloc, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_DEBUG, "cannot create rename stub"
                         "(out of memory)");
@@ -773,7 +839,7 @@ iot_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (rename, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL, NULL);
+                                     NULL, NULL, NULL);
                 if (stub != NULL) {
                         call_stub_destroy (stub);
                 }
@@ -785,31 +851,33 @@ out:
 
 int
 iot_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, fd_t *fd)
+              int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 	return 0;
 }
 
 
 int
 iot_open_wrapper (call_frame_t * frame, xlator_t * this, loc_t *loc,
-                  int32_t flags, fd_t * fd, int32_t wbflags)
+                  int32_t flags, fd_t * fd, dict_t *xdata)
 {
 	STACK_WIND (frame, iot_open_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->open, loc, flags, fd, wbflags);
+                    FIRST_CHILD (this)->fops->open, loc, flags, fd,
+                    xdata);
 	return 0;
 }
 
 
 int
 iot_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags)
+          fd_t *fd, dict_t *xdata)
 {
         call_stub_t	*stub = NULL;
         int             ret = -1;
 
-        stub = fop_open_stub (frame, iot_open_wrapper, loc, flags, fd, wbflags);
+        stub = fop_open_stub (frame, iot_open_wrapper, loc, flags, fd,
+                              xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "cannot create open call stub"
@@ -822,7 +890,7 @@ iot_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
 
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (open, frame, -1, -ret, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -837,35 +905,36 @@ int
 iot_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                 struct iatt *stbuf, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, stbuf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 	return 0;
 }
 
 
 int
 iot_create_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+                    int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                    dict_t *xdata)
 {
 	STACK_WIND (frame, iot_create_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, fd, params);
+		    loc, flags, mode, umask, fd, xdata);
 	return 0;
 }
 
 
 int
 iot_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-            mode_t mode, fd_t *fd, dict_t *params)
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_create_stub (frame, iot_create_wrapper, loc, flags, mode,
-                                fd, params);
+                                umask, fd, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "cannot create \"create\" call stub"
@@ -879,7 +948,7 @@ iot_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (create, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL, NULL);
+                                     NULL, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -893,10 +962,11 @@ out:
 int
 iot_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iovec *vector,
-               int32_t count, struct iatt *stbuf, struct iobref *iobref)
+               int32_t count, struct iatt *stbuf, struct iobref *iobref,
+               dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
+                             stbuf, iobref, xdata);
 
 	return 0;
 }
@@ -904,24 +974,25 @@ iot_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 iot_readv_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                   off_t offset, uint32_t flags)
+                   off_t offset, uint32_t flags, dict_t *xdata)
 {
 	STACK_WIND (frame, iot_readv_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readv,
-		    fd, size, offset, flags);
+		    fd, size, offset, flags, xdata);
 	return 0;
 }
 
 
 int
 iot_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-           off_t offset, uint32_t flags)
+           off_t offset, uint32_t flags, dict_t *xdata)
 {
 	call_stub_t *stub = NULL;
         int         ret = -1;
 
-	stub = fop_readv_stub (frame, iot_readv_wrapper, fd, size, offset, flags);
+	stub = fop_readv_stub (frame, iot_readv_wrapper, fd, size, offset,
+                               flags, xdata);
 	if (!stub) {
 		gf_log (this->name, GF_LOG_ERROR,
 			"cannot create readv call stub"
@@ -935,7 +1006,7 @@ iot_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 out:
         if (ret < 0) {
 		STACK_UNWIND_STRICT (readv, frame, -1, -ret, NULL, -1, NULL,
-                                     NULL);
+                                     NULL, NULL);
                 if (stub != NULL) {
                         call_stub_destroy (stub);
                 }
@@ -946,31 +1017,31 @@ out:
 
 int
 iot_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno)
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
 	return 0;
 }
 
 
 int
-iot_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd)
+iot_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
 	STACK_WIND (frame, iot_flush_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->flush,
-		    fd);
+		    fd, xdata);
 	return 0;
 }
 
 
 int
-iot_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+iot_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
 	call_stub_t *stub = NULL;
         int         ret = -1;
 
-	stub = fop_flush_stub (frame, iot_flush_wrapper, fd);
+	stub = fop_flush_stub (frame, iot_flush_wrapper, fd, xdata);
 	if (!stub) {
 		gf_log (this->name, GF_LOG_ERROR,
                         "cannot create flush_cbk call stub"
@@ -982,7 +1053,7 @@ iot_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-		STACK_UNWIND_STRICT (flush, frame, -1, -ret);
+		STACK_UNWIND_STRICT (flush, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -995,32 +1066,34 @@ out:
 int
 iot_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+               struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 	return 0;
 }
 
 
 int
 iot_fsync_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   int32_t datasync)
+                   int32_t datasync, dict_t *xdata)
 {
 	STACK_WIND (frame, iot_fsync_cbk,
 		    FIRST_CHILD (this),
 		    FIRST_CHILD (this)->fops->fsync,
-		    fd, datasync);
+		    fd, datasync, xdata);
 	return 0;
 }
 
 
 int
-iot_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync)
+iot_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+           dict_t *xdata)
 {
 	call_stub_t *stub = NULL;
         int         ret = -1;
 
-	stub = fop_fsync_stub (frame, iot_fsync_wrapper, fd, datasync);
+	stub = fop_fsync_stub (frame, iot_fsync_wrapper, fd, datasync, xdata);
 	if (!stub) {
 		gf_log (this->name, GF_LOG_ERROR,
                         "cannot create fsync_cbk call stub"
@@ -1033,7 +1106,7 @@ iot_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync)
 
 out:
         if (ret < 0) {
-		STACK_UNWIND_STRICT (fsync, frame, -1, -ret, NULL, NULL);
+		STACK_UNWIND_STRICT (fsync, frame, -1, -ret, NULL, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1046,9 +1119,10 @@ out:
 int
 iot_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                struct iatt *postbuf)
+                struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 	return 0;
 }
 
@@ -1056,12 +1130,13 @@ iot_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 iot_writev_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
                     struct iovec *vector, int32_t count,
-                    off_t offset, uint32_t flags, struct iobref *iobref)
+                    off_t offset, uint32_t flags, struct iobref *iobref,
+                    dict_t *xdata)
 {
 	STACK_WIND (frame, iot_writev_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->writev,
-		    fd, vector, count, offset, flags, iobref);
+		    fd, vector, count, offset, flags, iobref, xdata);
 	return 0;
 }
 
@@ -1069,13 +1144,13 @@ iot_writev_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int
 iot_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref)
+            uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
 	call_stub_t *stub = NULL;
         int         ret = -1;
 
-	stub = fop_writev_stub (frame, iot_writev_wrapper,
-				fd, vector, count, offset, flags, iobref);
+	stub = fop_writev_stub (frame, iot_writev_wrapper, fd, vector,
+                                count, offset, flags, iobref, xdata);
 
 	if (!stub) {
 		gf_log (this->name, GF_LOG_ERROR,
@@ -1088,7 +1163,7 @@ iot_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-		STACK_UNWIND_STRICT (writev, frame, -1, -ret, NULL, NULL);
+		STACK_UNWIND_STRICT (writev, frame, -1, -ret, NULL, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1101,33 +1176,34 @@ out:
 
 int32_t
 iot_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *flock)
+            int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+            dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock);
+	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock, xdata);
 	return 0;
 }
 
 
 int
 iot_lk_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                int32_t cmd, struct gf_flock *flock)
+                int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
 	STACK_WIND (frame, iot_lk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->lk,
-		    fd, cmd, flock);
+		    fd, cmd, flock, xdata);
 	return 0;
 }
 
 
 int
 iot_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-	struct gf_flock *flock)
+	struct gf_flock *flock, dict_t *xdata)
 {
 	call_stub_t *stub = NULL;
         int         ret = -1;
 
-	stub = fop_lk_stub (frame, iot_lk_wrapper, fd, cmd, flock);
+	stub = fop_lk_stub (frame, iot_lk_wrapper, fd, cmd, flock, xdata);
 
 	if (!stub) {
 		gf_log (this->name, GF_LOG_ERROR,
@@ -1140,7 +1216,7 @@ iot_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-		STACK_UNWIND_STRICT (lk, frame, -1, -ret, NULL);
+		STACK_UNWIND_STRICT (lk, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1152,31 +1228,31 @@ out:
 
 int
 iot_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct iatt *buf)
+              int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
 	return 0;
 }
 
 
 int
-iot_stat_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc)
+iot_stat_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
 	STACK_WIND (frame, iot_stat_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->stat,
-		    loc);
+		    loc, xdata);
 	return 0;
 }
 
 
 int
-iot_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+iot_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
 	call_stub_t *stub = NULL;
         int         ret = -1;
 
-        stub = fop_stat_stub (frame, iot_stat_wrapper, loc);
+        stub = fop_stat_stub (frame, iot_stat_wrapper, loc, xdata);
 	if (!stub) {
 		gf_log (this->name, GF_LOG_ERROR,
                         "cannot create fop_stat call stub"
@@ -1189,7 +1265,7 @@ iot_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 out:
         if (ret < 0) {
-		STACK_UNWIND_STRICT (stat, frame, -1, -ret, NULL);
+		STACK_UNWIND_STRICT (stat, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1201,31 +1277,31 @@ out:
 
 int
 iot_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf)
+               int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
+	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
 	return 0;
 }
 
 
 int
-iot_fstat_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd)
+iot_fstat_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
 	STACK_WIND (frame, iot_fstat_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fstat,
-		    fd);
+		    fd, xdata);
 	return 0;
 }
 
 
 int
-iot_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+iot_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
 	call_stub_t *stub = NULL;
         int         ret = -1;
 
-	stub = fop_fstat_stub (frame, iot_fstat_wrapper, fd);
+	stub = fop_fstat_stub (frame, iot_fstat_wrapper, fd, xdata);
 	if (!stub) {
 		gf_log (this->name, GF_LOG_ERROR,
                         "cannot create fop_fstat call stub"
@@ -1237,7 +1313,7 @@ iot_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-		STACK_UNWIND_STRICT (fstat, frame, -1, -ret, NULL);
+		STACK_UNWIND_STRICT (fstat, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1250,34 +1326,35 @@ out:
 int
 iot_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
 	return 0;
 }
 
 
 int
 iot_truncate_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      off_t offset)
+                      off_t offset, dict_t *xdata)
 {
 	STACK_WIND (frame, iot_truncate_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->truncate,
-		    loc, offset);
+		    loc, offset, xdata);
 	return 0;
 }
 
 
 int
-iot_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+iot_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
 {
 	call_stub_t *stub;
         int         ret = -1;
 
-        stub = fop_truncate_stub (frame, iot_truncate_wrapper, loc, offset);
-
+        stub = fop_truncate_stub (frame, iot_truncate_wrapper, loc, offset,
+                                  xdata);
 	if (!stub) {
 		gf_log (this->name, GF_LOG_ERROR,
                         "cannot create fop_stat call stub"
@@ -1290,7 +1367,8 @@ iot_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
 
 out:
         if (ret < 0) {
-		STACK_UNWIND_STRICT (truncate, frame, -1, -ret, NULL, NULL);
+		STACK_UNWIND_STRICT (truncate, frame, -1, -ret, NULL, NULL,
+                                     NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1304,33 +1382,35 @@ out:
 int
 iot_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
 	return 0;
 }
 
 
 int
 iot_ftruncate_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       off_t offset)
+                       off_t offset, dict_t *xdata)
 {
 	STACK_WIND (frame, iot_ftruncate_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd, offset);
+		    fd, offset, xdata);
 	return 0;
 }
 
 
 int
-iot_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+iot_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
 {
 	call_stub_t *stub = NULL;
         int         ret = -1;
 
-	stub = fop_ftruncate_stub (frame, iot_ftruncate_wrapper, fd, offset);
+	stub = fop_ftruncate_stub (frame, iot_ftruncate_wrapper, fd, offset,
+                                   xdata);
 	if (!stub) {
 		gf_log (this->name, GF_LOG_ERROR,
                         "cannot create fop_ftruncate call stub"
@@ -1342,7 +1422,7 @@ iot_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-		STACK_UNWIND_STRICT (ftruncate, frame, -1, -ret, NULL, NULL);
+		STACK_UNWIND_STRICT (ftruncate, frame, -1, -ret, NULL, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1356,32 +1436,34 @@ out:
 int
 iot_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent);
+                             postparent, xdata);
 	return 0;
 }
 
 
 int
-iot_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc)
+iot_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    int32_t xflag, dict_t *xdata)
 {
 	STACK_WIND (frame, iot_unlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->unlink,
-		    loc);
+		    loc, xflag, xdata);
 	return 0;
 }
 
 
 int
-iot_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+iot_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+            dict_t *xdata)
 {
 	call_stub_t *stub = NULL;
         int         ret = -1;
 
-	stub = fop_unlink_stub (frame, iot_unlink_wrapper, loc);
+	stub = fop_unlink_stub (frame, iot_unlink_wrapper, loc, xflag, xdata);
 	if (!stub) {
 		gf_log (this->name, GF_LOG_ERROR,
                         "cannot create fop_unlink call stub"
@@ -1394,7 +1476,7 @@ iot_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 out:
         if (ret < 0) {
-		STACK_UNWIND_STRICT (unlink, frame, -1, -ret, NULL, NULL);
+		STACK_UNWIND_STRICT (unlink, frame, -1, -ret, NULL, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1408,31 +1490,34 @@ out:
 int
 iot_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int32_t op_ret, int32_t op_errno, inode_t *inode,
-              struct iatt *buf, struct iatt *preparent, struct iatt *postparent)
+              struct iatt *buf, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
 {
         STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
-iot_link_wrapper (call_frame_t *frame, xlator_t *this, loc_t *old, loc_t *new)
+iot_link_wrapper (call_frame_t *frame, xlator_t *this, loc_t *old, loc_t *new,
+                  dict_t *xdata)
 {
         STACK_WIND (frame, iot_link_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->link, old, new);
+                    FIRST_CHILD (this)->fops->link, old, new, xdata);
 
         return 0;
 }
 
 
 int
-iot_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+iot_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
-        stub = fop_link_stub (frame, iot_link_wrapper, oldloc, newloc);
+        stub = fop_link_stub (frame, iot_link_wrapper, oldloc, newloc, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create link stub"
                         "(out of memory)");
@@ -1444,7 +1529,7 @@ iot_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
 out:
         if (ret < 0) {
                 STACK_UNWIND_STRICT (link, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
+                                     NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1456,29 +1541,31 @@ out:
 
 int
 iot_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd)
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
 
 int
-iot_opendir_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+iot_opendir_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                     dict_t *xdata)
 {
         STACK_WIND (frame, iot_opendir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->opendir, loc, fd);
+                    FIRST_CHILD (this)->fops->opendir, loc, fd, xdata);
         return 0;
 }
 
 
 int
-iot_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+iot_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+             dict_t *xdata)
 {
         call_stub_t     *stub  = NULL;
         int             ret = -1;
 
-        stub = fop_opendir_stub (frame, iot_opendir_wrapper, loc, fd);
+        stub = fop_opendir_stub (frame, iot_opendir_wrapper, loc, fd, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create opendir stub"
                         "(out of memory)");
@@ -1489,7 +1576,7 @@ iot_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (opendir, frame, -1, -ret, NULL);
+                STACK_UNWIND_STRICT (opendir, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1501,30 +1588,32 @@ out:
 
 int
 iot_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 iot_fsyncdir_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      int datasync)
+                      int datasync, dict_t *xdata)
 {
         STACK_WIND (frame, iot_fsyncdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsyncdir, fd, datasync);
+                    FIRST_CHILD (this)->fops->fsyncdir, fd, datasync, xdata);
         return 0;
 }
 
 
 int
-iot_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+iot_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+              dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
-        stub = fop_fsyncdir_stub (frame, iot_fsyncdir_wrapper, fd, datasync);
+        stub = fop_fsyncdir_stub (frame, iot_fsyncdir_wrapper, fd, datasync,
+                                  xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create fsyncdir stub"
                         "(out of memory)");
@@ -1535,7 +1624,7 @@ iot_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, -ret);
+                STACK_UNWIND_STRICT (fsyncdir, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1547,29 +1636,31 @@ out:
 
 int
 iot_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
-iot_statfs_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc)
+iot_statfs_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
         STACK_WIND (frame, iot_statfs_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->statfs, loc);
+                    FIRST_CHILD (this)->fops->statfs, loc, xdata);
         return 0;
 }
 
 
 int
-iot_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+iot_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int              ret = -1;
 
-        stub = fop_statfs_stub (frame, iot_statfs_wrapper, loc);
+        stub = fop_statfs_stub (frame, iot_statfs_wrapper, loc, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create statfs stub"
                         "(out of memory)");
@@ -1580,7 +1671,7 @@ iot_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (statfs, frame, -1, -ret, NULL);
+                STACK_UNWIND_STRICT (statfs, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1592,32 +1683,32 @@ out:
 
 int
 iot_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 iot_setxattr_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      dict_t *dict, int32_t flags)
+                      dict_t *dict, int32_t flags, dict_t *xdata)
 {
         STACK_WIND (frame, iot_setxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setxattr, loc, dict, flags);
+                    FIRST_CHILD (this)->fops->setxattr, loc, dict, flags, xdata);
         return 0;
 }
 
 
 int
 iot_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-              int32_t flags)
+              int32_t flags, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int              ret = -1;
 
         stub = fop_setxattr_stub (frame, iot_setxattr_wrapper, loc, dict,
-                                  flags);
+                                  flags, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create setxattr stub"
                         "(out of memory)");
@@ -1629,7 +1720,7 @@ iot_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
 
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (setxattr, frame, -1, -ret);
+                STACK_UNWIND_STRICT (setxattr, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1641,31 +1732,31 @@ out:
 
 int
 iot_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict)
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 iot_getxattr_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      const char *name)
+                      const char *name, dict_t *xdata)
 {
         STACK_WIND (frame, iot_getxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->getxattr, loc, name);
+                    FIRST_CHILD (this)->fops->getxattr, loc, name, xdata);
         return 0;
 }
 
 
 int
 iot_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              const char *name)
+              const char *name, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int              ret = -1;
 
-        stub = fop_getxattr_stub (frame, iot_getxattr_wrapper, loc, name);
+        stub = fop_getxattr_stub (frame, iot_getxattr_wrapper, loc, name, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create getxattr stub"
                         "(out of memory)");
@@ -1677,7 +1768,7 @@ iot_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, NULL);
+                STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1689,31 +1780,32 @@ out:
 
 int
 iot_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *dict)
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 iot_fgetxattr_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       const char *name)
+                       const char *name, dict_t *xdata)
 {
         STACK_WIND (frame, iot_fgetxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fgetxattr, fd, name);
+                    FIRST_CHILD (this)->fops->fgetxattr, fd, name, xdata);
         return 0;
 }
 
 
 int
 iot_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               const char *name)
+               const char *name, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
-        stub = fop_fgetxattr_stub (frame, iot_fgetxattr_wrapper, fd, name);
+        stub = fop_fgetxattr_stub (frame, iot_fgetxattr_wrapper, fd, name, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create fgetxattr stub"
                         "(out of memory)");
@@ -1724,7 +1816,7 @@ iot_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (fgetxattr, frame, -1, -ret, NULL);
+                STACK_UNWIND_STRICT (fgetxattr, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1736,32 +1828,33 @@ out:
 
 int
 iot_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 iot_fsetxattr_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       dict_t *dict, int32_t flags)
+                       dict_t *dict, int32_t flags, dict_t *xdata)
 {
         STACK_WIND (frame, iot_fsetxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetxattr, fd, dict, flags);
+                    FIRST_CHILD (this)->fops->fsetxattr, fd, dict, flags,
+                    xdata);
         return 0;
 }
 
 
 int
 iot_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-               int32_t flags)
+               int32_t flags, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_fsetxattr_stub (frame, iot_fsetxattr_wrapper, fd, dict,
-                                        flags);
+                                   flags, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create fsetxattr stub"
                         "(out of memory)");
@@ -1772,7 +1865,7 @@ iot_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, -ret);
+                STACK_UNWIND_STRICT (fsetxattr, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1784,32 +1877,32 @@ out:
 
 int
 iot_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 iot_removexattr_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         const char *name)
+                         const char *name, dict_t *xdata)
 {
         STACK_WIND (frame, iot_removexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->removexattr, loc, name);
+                    FIRST_CHILD (this)->fops->removexattr, loc, name, xdata);
         return 0;
 }
 
 
 int
 iot_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_removexattr_stub (frame, iot_removexattr_wrapper, loc,
-                                     name);
+                                     name, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR,"cannot get removexattr fop"
                         "(out of memory)");
@@ -1820,7 +1913,7 @@ iot_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (removexattr, frame, -1, -ret);
+                STACK_UNWIND_STRICT (removexattr, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1831,32 +1924,32 @@ out:
 
 int
 iot_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 iot_fremovexattr_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          const char *name)
+                          const char *name, dict_t *xdata)
 {
         STACK_WIND (frame, iot_fremovexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fremovexattr, fd, name);
+                    FIRST_CHILD (this)->fops->fremovexattr, fd, name, xdata);
         return 0;
 }
 
 
 int
 iot_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_fremovexattr_stub (frame, iot_fremovexattr_wrapper, fd,
-                                      name);
+                                      name, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR,"cannot get fremovexattr fop"
                         "(out of memory)");
@@ -1867,7 +1960,7 @@ iot_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (fremovexattr, frame, -1, -ret);
+                STACK_UNWIND_STRICT (fremovexattr, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1879,32 +1972,33 @@ out:
 
 int
 iot_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                  dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
         return 0;
 }
 
 
 int
 iot_readdirp_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      size_t size, off_t offset, dict_t *dict)
+                      size_t size, off_t offset, dict_t *xdata)
 {
         STACK_WIND (frame, iot_readdirp_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readdirp, fd, size, offset, dict);
+                    FIRST_CHILD (this)->fops->readdirp, fd, size, offset, xdata);
         return 0;
 }
 
 
 int
 iot_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, dict_t *dict)
+              off_t offset, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_readdirp_stub (frame, iot_readdirp_wrapper, fd, size,
-                                  offset, dict);
+                                  offset, xdata);
         if (!stub) {
                 gf_log (this->private, GF_LOG_ERROR,"cannot get readdir stub"
                         "(out of memory)");
@@ -1915,7 +2009,7 @@ iot_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (readdirp, frame, -1, -ret, NULL);
+                STACK_UNWIND_STRICT (readdirp, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1927,31 +2021,33 @@ out:
 
 int
 iot_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                 dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
         return 0;
 }
 
 
 int
 iot_readdir_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     size_t size, off_t offset)
+                     size_t size, off_t offset, dict_t *xdata)
 {
         STACK_WIND (frame, iot_readdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readdir, fd, size, offset);
+                    FIRST_CHILD (this)->fops->readdir, fd, size, offset, xdata);
         return 0;
 }
 
 
 int
 iot_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset)
+             off_t offset, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
-        stub = fop_readdir_stub (frame, iot_readdir_wrapper, fd, size, offset);
+        stub = fop_readdir_stub (frame, iot_readdir_wrapper, fd, size, offset,
+                                 xdata);
         if (!stub) {
                 gf_log (this->private, GF_LOG_ERROR,"cannot get readdir stub"
                         "(out of memory)");
@@ -1962,7 +2058,7 @@ iot_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, -ret, NULL);
+                STACK_UNWIND_STRICT (readdir, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -1972,33 +2068,36 @@ out:
 }
 
 int
-iot_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+iot_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 iot_inodelk_wrapper (call_frame_t *frame, xlator_t *this, const char *volume,
-                     loc_t *loc, int32_t cmd, struct gf_flock *lock)
+                     loc_t *loc, int32_t cmd, struct gf_flock *lock,
+                     dict_t *xdata)
 {
         STACK_WIND (frame, iot_inodelk_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->inodelk, volume, loc, cmd, lock);
+                    FIRST_CHILD (this)->fops->inodelk, volume, loc, cmd, lock,
+                    xdata);
         return 0;
 }
 
 
 int
 iot_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
+             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock,
+             dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_inodelk_stub (frame, iot_inodelk_wrapper,
-                                 volume, loc, cmd, lock);
+                                 volume, loc, cmd, lock, xdata);
         if (!stub) {
                 ret = -ENOMEM;
                 goto out;
@@ -2007,7 +2106,7 @@ iot_inodelk (call_frame_t *frame, xlator_t *this,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (inodelk, frame, -1, -ret);
+                STACK_UNWIND_STRICT (inodelk, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -2018,9 +2117,9 @@ out:
 
 int
 iot_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -2028,23 +2127,25 @@ iot_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 iot_finodelk_wrapper (call_frame_t *frame, xlator_t *this,
                       const char *volume, fd_t *fd, int32_t cmd,
-                      struct gf_flock *lock)
+                      struct gf_flock *lock, dict_t *xdata)
 {
         STACK_WIND (frame, iot_finodelk_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->finodelk, volume, fd, cmd, lock);
+                    FIRST_CHILD (this)->fops->finodelk, volume, fd, cmd, lock,
+                    xdata);
         return 0;
 }
 
 
 int
 iot_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
+              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock,
+              dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_finodelk_stub (frame, iot_finodelk_wrapper,
-                                  volume, fd, cmd, lock);
+                                  volume, fd, cmd, lock, xdata);
         if (!stub) {
                 gf_log (this->private, GF_LOG_ERROR,"cannot get finodelk stub"
                         "(out of memory)");
@@ -2055,7 +2156,7 @@ iot_finodelk (call_frame_t *frame, xlator_t *this,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (finodelk, frame, -1, -ret);
+                STACK_UNWIND_STRICT (finodelk, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -2066,9 +2167,9 @@ out:
 
 int
 iot_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno)
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -2076,11 +2177,11 @@ iot_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 iot_entrylk_wrapper (call_frame_t *frame, xlator_t *this,
                      const char *volume, loc_t *loc, const char *basename,
-                     entrylk_cmd cmd, entrylk_type type)
+                     entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         STACK_WIND (frame, iot_entrylk_cbk, FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
         return 0;
 }
 
@@ -2088,13 +2189,13 @@ iot_entrylk_wrapper (call_frame_t *frame, xlator_t *this,
 int
 iot_entrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, loc_t *loc, const char *basename,
-             entrylk_cmd cmd, entrylk_type type)
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_entrylk_stub (frame, iot_entrylk_wrapper,
-                                 volume, loc, basename, cmd, type);
+                                 volume, loc, basename, cmd, type, xdata);
         if (!stub) {
                 gf_log (this->private, GF_LOG_ERROR,"cannot get entrylk stub"
                         "(out of memory)");
@@ -2105,7 +2206,7 @@ iot_entrylk (call_frame_t *frame, xlator_t *this,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (entrylk, frame, -1, -ret);
+                STACK_UNWIND_STRICT (entrylk, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -2116,9 +2217,9 @@ out:
 
 int
 iot_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -2126,11 +2227,11 @@ iot_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 iot_fentrylk_wrapper (call_frame_t *frame, xlator_t *this,
                       const char *volume, fd_t *fd, const char *basename,
-                      entrylk_cmd cmd, entrylk_type type)
+                      entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         STACK_WIND (frame, iot_fentrylk_cbk, FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
+                    volume, fd, basename, cmd, type, xdata);
         return 0;
 }
 
@@ -2138,13 +2239,13 @@ iot_fentrylk_wrapper (call_frame_t *frame, xlator_t *this,
 int
 iot_fentrylk (call_frame_t *frame, xlator_t *this,
               const char *volume, fd_t *fd, const char *basename,
-              entrylk_cmd cmd, entrylk_type type)
+              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_fentrylk_stub (frame, iot_fentrylk_wrapper,
-                                  volume, fd, basename, cmd, type);
+                                  volume, fd, basename, cmd, type, xdata);
         if (!stub) {
                 gf_log (this->private, GF_LOG_ERROR,"cannot get fentrylk stub"
                         "(out of memory)");
@@ -2155,7 +2256,7 @@ iot_fentrylk (call_frame_t *frame, xlator_t *this,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (fentrylk, frame, -1, -ret);
+                STACK_UNWIND_STRICT (fentrylk, frame, -1, -ret, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -2167,32 +2268,32 @@ out:
 
 int
 iot_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xattr)
+                 int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, xattr);
+        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, xattr, xdata);
         return 0;
 }
 
 
 int
 iot_xattrop_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     gf_xattrop_flags_t optype, dict_t *xattr)
+                     gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         STACK_WIND (frame, iot_xattrop_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->xattrop, loc, optype, xattr);
+                    FIRST_CHILD (this)->fops->xattrop, loc, optype, xattr, xdata);
         return 0;
 }
 
 
 int
 iot_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr)
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_xattrop_stub (frame, iot_xattrop_wrapper, loc, optype,
-                                        xattr);
+                                        xattr, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create xattrop stub"
                         "(out of memory)");
@@ -2203,7 +2304,7 @@ iot_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (xattrop, frame, -1, -ret, NULL);
+                STACK_UNWIND_STRICT (xattrop, frame, -1, -ret, NULL, NULL);
 
                 if (stub != NULL) {
                         call_stub_destroy (stub);
@@ -2215,31 +2316,31 @@ out:
 
 int
 iot_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xattr)
+                  int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, xattr);
+        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, xattr, xdata);
         return 0;
 }
 
 int
 iot_fxattrop_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      gf_xattrop_flags_t optype, dict_t *xattr)
+                      gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         STACK_WIND (frame, iot_fxattrop_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fxattrop, fd, optype, xattr);
+                    FIRST_CHILD (this)->fops->fxattrop, fd, optype, xattr, xdata);
         return 0;
 }
 
 
 int
 iot_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr)
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_fxattrop_stub (frame, iot_fxattrop_wrapper, fd, optype,
-                                        xattr);
+                                  xattr, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create fxattrop stub"
                         "(out of memory)");
@@ -2250,7 +2351,7 @@ iot_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, -ret, NULL);
+                STACK_UNWIND_STRICT (fxattrop, frame, -1, -ret, NULL, NULL);
                 if (stub != NULL) {
                         call_stub_destroy (stub);
                 }
@@ -2262,33 +2363,33 @@ out:
 int32_t
 iot_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                   uint8_t *strong_checksum)
+                   uint8_t *strong_checksum, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum);
+                             strong_checksum, xdata);
         return 0;
 }
 
 
 int32_t
 iot_rchecksum_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       off_t offset, int32_t len)
+                       off_t offset, int32_t len, dict_t *xdata)
 {
         STACK_WIND (frame, iot_rchecksum_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rchecksum, fd, offset, len);
+                    FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
         return 0;
 }
 
 
 int32_t
 iot_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               int32_t len)
+               int32_t len, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         int             ret = -1;
 
         stub = fop_rchecksum_stub (frame, iot_rchecksum_wrapper, fd, offset,
-                                   len);
+                                   len, xdata);
         if (!stub) {
                 gf_log (this->name, GF_LOG_ERROR, "cannot create rchecksum stub"
                         "(out of memory)");
@@ -2299,7 +2400,7 @@ iot_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         ret = iot_schedule (frame, this, stub);
 out:
         if (ret < 0) {
-                STACK_UNWIND_STRICT (rchecksum, frame, -1, -ret, -1, NULL);
+                STACK_UNWIND_STRICT (rchecksum, frame, -1, -ret, -1, NULL, NULL);
                 if (stub != NULL) {
                         call_stub_destroy (stub);
                 }
@@ -2308,24 +2409,172 @@ out:
         return 0;
 }
 
+int
+iot_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *preop, struct iatt *postop, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (fallocate, frame, op_ret, op_errno, preop, postop,
+                             xdata);
+        return 0;
+}
+
+
+int
+iot_fallocate_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		      off_t offset, size_t len, dict_t *xdata)
+{
+        STACK_WIND (frame, iot_fallocate_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->fallocate, fd, mode, offset, len,
+		    xdata);
+        return 0;
+}
+
+
+int
+iot_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+	      off_t offset, size_t len, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+        int              ret = -1;
+
+        stub = fop_fallocate_stub(frame, iot_fallocate_wrapper, fd, mode, offset,
+				  len, xdata);
+        if (!stub) {
+                gf_log (this->name, GF_LOG_ERROR, "cannot create fallocate stub"
+                        "(out of memory)");
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        ret = iot_schedule (frame, this, stub);
+
+out:
+        if (ret < 0) {
+                STACK_UNWIND_STRICT (fallocate, frame, -1, -ret, NULL, NULL,
+                                     NULL);
+                if (stub != NULL) {
+                        call_stub_destroy (stub);
+                }
+        }
+        return 0;
+}
+
+int
+iot_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *preop, struct iatt *postop, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, preop, postop,
+                             xdata);
+        return 0;
+}
+
+
+int
+iot_discard_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+		    size_t len, dict_t *xdata)
+{
+        STACK_WIND (frame, iot_discard_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->discard, fd, offset, len, xdata);
+        return 0;
+}
+
+
+int
+iot_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	    size_t len, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+        int              ret = -1;
+
+        stub = fop_discard_stub(frame, iot_discard_wrapper, fd, offset, len,
+				xdata);
+        if (!stub) {
+                gf_log (this->name, GF_LOG_ERROR, "cannot create discard stub"
+                        "(out of memory)");
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        ret = iot_schedule (frame, this, stub);
+
+out:
+        if (ret < 0) {
+                STACK_UNWIND_STRICT (discard, frame, -1, -ret, NULL, NULL,
+                                     NULL);
+                if (stub != NULL) {
+                        call_stub_destroy (stub);
+                }
+        }
+        return 0;
+}
+
+int
+iot_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *preop, struct iatt *postop, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, preop, postop,
+                             xdata);
+        return 0;
+}
+
+int
+iot_zerofill_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     off_t offset, size_t len, dict_t *xdata)
+{
+        STACK_WIND (frame, iot_zerofill_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->zerofill, fd, offset, len, xdata);
+        return 0;
+}
+
+int
+iot_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+        call_stub_t     *stub     = NULL;
+        int              ret      = -1;
+
+        stub = fop_zerofill_stub(frame, iot_zerofill_wrapper, fd,
+                                 offset, len, xdata);
+        if (!stub) {
+                gf_log (this->name, GF_LOG_ERROR, "cannot create zerofill stub"
+                        "(out of memory)");
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        ret = iot_schedule (frame, this, stub);
+
+out:
+        if (ret < 0) {
+                STACK_UNWIND_STRICT (zerofill, frame, -1, -ret, NULL, NULL,
+                                     NULL);
+                if (stub != NULL) {
+                        call_stub_destroy (stub);
+                }
+        }
+        return 0;
+}
+
 
 int
 __iot_workers_scale (iot_conf_t *conf)
 {
-        int       log2 = 0;
         int       scale = 0;
         int       diff = 0;
         pthread_t thread;
         int       ret = 0;
+        int       i = 0;
 
-        log2 = log_base2 (conf->queue_size);
-
-        scale = log2;
+        for (i = 0; i < IOT_PRI_MAX; i++)
+                scale += min (conf->queue_sizes[i], conf->ac_iot_limit[i]);
 
-        if (log2 < IOT_MIN_THREADS)
+        if (scale < IOT_MIN_THREADS)
                 scale = IOT_MIN_THREADS;
 
-        if (log2 > conf->max_count)
+        if (scale > conf->max_count)
                 scale = conf->max_count;
 
         if (conf->curr_count < scale) {
@@ -2335,7 +2584,7 @@ __iot_workers_scale (iot_conf_t *conf)
         while (diff) {
                 diff --;
 
-                ret = pthread_create (&thread, &conf->w_attr, iot_worker, conf);
+                ret = gf_thread_create (&thread, &conf->w_attr, iot_worker, conf);
                 if (ret == 0) {
                         conf->curr_count++;
                         gf_log (conf->this->name, GF_LOG_DEBUG,
@@ -2376,13 +2625,24 @@ set_stack_size (iot_conf_t *conf)
 {
         int     err = 0;
         size_t  stacksize = IOT_THREAD_STACK_SIZE;
+        xlator_t *this = NULL;
+
+        this = THIS;
 
         pthread_attr_init (&conf->w_attr);
         err = pthread_attr_setstacksize (&conf->w_attr, stacksize);
         if (err == EINVAL) {
-                gf_log (conf->this->name, GF_LOG_WARNING,
-                        "Using default thread stack size");
+                err = pthread_attr_getstacksize (&conf->w_attr, &stacksize);
+                if (!err)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Using default thread stack size %zd",
+                                stacksize);
+                else
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Using default thread stack size");
         }
+
+        conf->stack_size = stacksize;
 }
 
 
@@ -2405,6 +2665,44 @@ mem_acct_init (xlator_t *this)
         return ret;
 }
 
+int
+iot_priv_dump (xlator_t *this)
+{
+        iot_conf_t     *conf   =   NULL;
+        char           key_prefix[GF_DUMP_MAX_BUF_LEN];
+
+        if (!this)
+                return 0;
+
+        conf = this->private;
+        if (!conf)
+                return 0;
+
+        snprintf (key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type,
+                  this->name);
+
+        gf_proc_dump_add_section(key_prefix);
+
+        gf_proc_dump_write("maximum_threads_count", "%d", conf->max_count);
+        gf_proc_dump_write("current_threads_count", "%d", conf->curr_count);
+        gf_proc_dump_write("sleep_count", "%d", conf->sleep_count);
+        gf_proc_dump_write("idle_time", "%d", conf->idle_time);
+        gf_proc_dump_write("stack_size", "%zd", conf->stack_size);
+        gf_proc_dump_write("high_priority_threads", "%d",
+                           conf->ac_iot_limit[IOT_PRI_HI]);
+        gf_proc_dump_write("normal_priority_threads", "%d",
+                           conf->ac_iot_limit[IOT_PRI_NORMAL]);
+        gf_proc_dump_write("low_priority_threads", "%d",
+                           conf->ac_iot_limit[IOT_PRI_LO]);
+        gf_proc_dump_write("least_priority_threads", "%d",
+                           conf->ac_iot_limit[IOT_PRI_LEAST]);
+
+	gf_proc_dump_write("cached least rate", "%u",
+			   conf->throttle.cached_rate);
+	gf_proc_dump_write("least rate limit", "%u", conf->throttle.rate_limit);
+
+        return 0;
+}
 
 int
 reconfigure (xlator_t *this, dict_t *options)
@@ -2431,6 +2729,11 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("least-prio-threads",
                           conf->ac_iot_limit[IOT_PRI_LEAST], options, int32,
                           out);
+        GF_OPTION_RECONF ("enable-least-priority", conf->least_priority,
+                          options, bool, out);
+
+	GF_OPTION_RECONF("least-rate-limit", conf->throttle.rate_limit, options,
+			 int32, out);
 
 	ret = 0;
 out:
@@ -2441,9 +2744,9 @@ out:
 int
 init (xlator_t *this)
 {
-        iot_conf_t      *conf = NULL;
-        int              ret = -1;
-        int              i = 0;
+        iot_conf_t *conf = NULL;
+        int         ret  = -1;
+        int         i    = 0;
 
 	if (!this->children || this->children->next) {
 		gf_log ("io-threads", GF_LOG_ERROR,
@@ -2493,6 +2796,16 @@ init (xlator_t *this)
                         conf->ac_iot_limit[IOT_PRI_LEAST], int32, out);
 
         GF_OPTION_INIT ("idle-time", conf->idle_time, int32, out);
+        GF_OPTION_INIT ("enable-least-priority", conf->least_priority,
+                        bool, out);
+
+	GF_OPTION_INIT("least-rate-limit", conf->throttle.rate_limit, int32,
+		       out);
+        if ((ret = pthread_mutex_init(&conf->throttle.lock, NULL)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_mutex_init failed (%d)", ret);
+                goto out;
+        }
 
         conf->this = this;
 
@@ -2505,13 +2818,15 @@ init (xlator_t *this)
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "cannot initialize worker threads, exiting init");
-                GF_FREE (conf);
                 goto out;
         }
 
 	this->private = conf;
         ret = 0;
 out:
+        if (ret)
+                GF_FREE (conf);
+
 	return ret;
 }
 
@@ -2527,6 +2842,9 @@ fini (xlator_t *this)
 	return;
 }
 
+struct xlator_dumpops dumpops = {
+        .priv    = iot_priv_dump,
+};
 
 struct xlator_fops fops = {
 	.open        = iot_open,
@@ -2570,10 +2888,12 @@ struct xlator_fops fops = {
         .xattrop     = iot_xattrop,
 	.fxattrop    = iot_fxattrop,
         .rchecksum   = iot_rchecksum,
+	.fallocate   = iot_fallocate,
+	.discard     = iot_discard,
+        .zerofill    = iot_zerofill,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
 	{ .key  = {"thread-count"},
@@ -2620,12 +2940,25 @@ struct volume_options options[] = {
           .description = "Max number of threads in IO threads translator which "
                          "perform least priority IO operations at a given time"
 	},
+        { .key  = {"enable-least-priority"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Enable/Disable least priority"
+        },
         {.key   = {"idle-time"},
          .type  = GF_OPTION_TYPE_INT,
          .min   = 1,
          .max   = 0x7fffffff,
          .default_value = "120",
         },
+	{.key	= {"least-rate-limit"},
+	 .type	= GF_OPTION_TYPE_INT,
+	 .min	= 0,
+         .max	= INT_MAX,
+	 .default_value = "0",
+	 .description = "Max number of least priority operations to handle "
+			"per-second"
+	},
 	{ .key  = {NULL},
         },
 };
diff --git a/xlators/performance/io-threads/src/io-threads.h b/xlators/performance/io-threads/src/io-threads.h
index d09fec94d..1a9dee9ae 100644
--- a/xlators/performance/io-threads/src/io-threads.h
+++ b/xlators/performance/io-threads/src/io-threads.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __IOT_H
@@ -37,6 +28,7 @@
 #include "locking.h"
 #include "iot-mem-types.h"
 #include <semaphore.h>
+#include "statedump.h"
 
 
 struct iot_conf;
@@ -61,6 +53,14 @@ typedef enum {
         IOT_PRI_MAX,
 } iot_pri_t;
 
+#define IOT_LEAST_THROTTLE_DELAY 1	/* sample interval in seconds */
+struct iot_least_throttle {
+	struct timeval	sample_time;	/* timestamp of current sample */
+	uint32_t	sample_cnt;	/* sample count for active interval */
+	uint32_t	cached_rate;	/* the most recently measured rate */
+	int32_t		rate_limit;	/* user-specified rate limit */
+	pthread_mutex_t	lock;
+};
 
 struct iot_conf {
         pthread_mutex_t      mutex;
@@ -76,10 +76,15 @@ struct iot_conf {
 
         int32_t              ac_iot_limit[IOT_PRI_MAX];
         int32_t              ac_iot_count[IOT_PRI_MAX];
+        int                  queue_sizes[IOT_PRI_MAX];
         int                  queue_size;
         pthread_attr_t       w_attr;
+        gf_boolean_t         least_priority; /*Enable/Disable least-priority */
 
         xlator_t            *this;
+        size_t              stack_size;
+
+	struct iot_least_throttle throttle;
 };
 
 typedef struct iot_conf iot_conf_t;
diff --git a/xlators/performance/io-threads/src/iot-mem-types.h b/xlators/performance/io-threads/src/iot-mem-types.h
index 14400cd08..4fa8302d1 100644
--- a/xlators/performance/io-threads/src/iot-mem-types.h
+++ b/xlators/performance/io-threads/src/iot-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
diff --git a/xlators/performance/md-cache/Makefile.am b/xlators/performance/md-cache/Makefile.am
new file mode 100644
index 000000000..af437a64d
--- /dev/null
+++ b/xlators/performance/md-cache/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = src
diff --git a/xlators/performance/md-cache/src/Makefile.am b/xlators/performance/md-cache/src/Makefile.am
new file mode 100644
index 000000000..8c9f5a858
--- /dev/null
+++ b/xlators/performance/md-cache/src/Makefile.am
@@ -0,0 +1,25 @@
+xlator_LTLIBRARIES = md-cache.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
+
+md_cache_la_LDFLAGS = -module -avoid-version
+
+md_cache_la_SOURCES = md-cache.c
+md_cache_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = md-cache-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(CONTRIBDIR)/rbtree
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
+
+stat-prefetch-compat:
+	mkdir -p $(DESTDIR)$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
+	rm -rf $(DESTDIR)$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance/stat-prefetch.so
+	ln -s ./md-cache.so $(DESTDIR)$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance/stat-prefetch.so
+
+
+install-exec-local: stat-prefetch-compat
diff --git a/xlators/performance/md-cache/src/md-cache-mem-types.h b/xlators/performance/md-cache/src/md-cache-mem-types.h
new file mode 100644
index 000000000..6634cf962
--- /dev/null
+++ b/xlators/performance/md-cache/src/md-cache-mem-types.h
@@ -0,0 +1,24 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __MDC_MEM_TYPES_H__
+#define __MDC_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_mdc_mem_types_ {
+        gf_mdc_mt_mdc_local_t   = gf_common_mt_end + 1,
+	gf_mdc_mt_md_cache_t,
+	gf_mdc_mt_mdc_conf_t,
+        gf_mdc_mt_end
+};
+#endif
+
diff --git a/xlators/performance/md-cache/src/md-cache.c b/xlators/performance/md-cache/src/md-cache.c
new file mode 100644
index 000000000..84c363ad9
--- /dev/null
+++ b/xlators/performance/md-cache/src/md-cache.c
@@ -0,0 +1,2303 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "md-cache-mem-types.h"
+#include "glusterfs-acl.h"
+#include <assert.h>
+#include <sys/time.h>
+
+
+/* TODO:
+   - cache symlink() link names and nuke symlink-cache
+   - send proper postbuf in setattr_cbk even when op_ret = -1
+*/
+
+
+struct mdc_conf {
+	int  timeout;
+	gf_boolean_t cache_posix_acl;
+	gf_boolean_t cache_selinux;
+	gf_boolean_t force_readdirp;
+};
+
+
+static struct mdc_key {
+	const char *name;
+	int         load;
+	int         check;
+} mdc_keys[] = {
+	{
+		.name = POSIX_ACL_ACCESS_XATTR,
+		.load = 0,
+		.check = 1,
+	},
+	{
+		.name = POSIX_ACL_DEFAULT_XATTR,
+		.load = 0,
+		.check = 1,
+	},
+	{
+		.name = GF_SELINUX_XATTR_KEY,
+		.load = 0,
+		.check = 1,
+	},
+	{
+		.name = "security.capability",
+		.load = 0,
+		.check = 1,
+	},
+	{
+		.name = "gfid-req",
+		.load = 0,
+		.check = 1,
+	},
+        {
+                .name = NULL,
+                .load = 0,
+                .check = 0,
+        }
+};
+
+
+static uint64_t
+gfid_to_ino (uuid_t gfid)
+{
+	uint64_t  ino = 0;
+	int       i = 0, j = 0;
+
+        for (i = 15; i > (15 - 8); i--) {
+                ino += (uint64_t)(gfid[i]) << j;
+                j += 8;
+        }
+
+	return ino;
+}
+
+
+struct mdc_local;
+typedef struct mdc_local mdc_local_t;
+
+#define MDC_STACK_UNWIND(fop, frame, params ...) do {           \
+                mdc_local_t *__local = NULL;                    \
+                xlator_t    *__xl    = NULL;                    \
+                if (frame) {                                    \
+                        __xl         = frame->this;             \
+                        __local      = frame->local;            \
+                        frame->local = NULL;                    \
+                }                                               \
+                STACK_UNWIND_STRICT (fop, frame, params);       \
+                mdc_local_wipe (__xl, __local);                 \
+        } while (0)
+
+
+struct md_cache {
+        ia_prot_t     md_prot;
+        uint32_t      md_nlink;
+        uint32_t      md_uid;
+        uint32_t      md_gid;
+        uint32_t      md_atime;
+        uint32_t      md_atime_nsec;
+        uint32_t      md_mtime;
+        uint32_t      md_mtime_nsec;
+        uint32_t      md_ctime;
+        uint32_t      md_ctime_nsec;
+        uint64_t      md_rdev;
+        uint64_t      md_size;
+        uint64_t      md_blocks;
+        dict_t       *xattr;
+        char         *linkname;
+	time_t        ia_time;
+	time_t        xa_time;
+        gf_lock_t     lock;
+};
+
+
+struct mdc_local {
+        loc_t   loc;
+        loc_t   loc2;
+        fd_t   *fd;
+        char   *linkname;
+	char   *key;
+        dict_t *xattr;
+};
+
+
+int
+__mdc_inode_ctx_get (xlator_t *this, inode_t *inode, struct md_cache **mdc_p)
+{
+        int              ret = 0;
+        struct md_cache *mdc = NULL;
+        uint64_t         mdc_int = 0;
+
+	ret = __inode_ctx_get (inode, this, &mdc_int);
+	mdc = (void *) (long) (mdc_int);
+	if (ret == 0 && mdc_p)
+		*mdc_p = mdc;
+
+	return ret;
+}
+
+
+int
+mdc_inode_ctx_get (xlator_t *this, inode_t *inode, struct md_cache **mdc_p)
+{
+	int   ret;
+
+	LOCK(&inode->lock);
+	{
+		ret = __mdc_inode_ctx_get (this, inode, mdc_p);
+	}
+	UNLOCK(&inode->lock);
+
+	return ret;
+}
+
+
+int
+__mdc_inode_ctx_set (xlator_t *this, inode_t *inode, struct md_cache *mdc)
+{
+        int              ret = 0;
+        uint64_t         mdc_int = 0;
+
+	mdc_int = (long) mdc;
+	ret = __inode_ctx_set (inode, this, &mdc_int);
+
+	return ret;
+}
+
+
+int
+mdc_inode_ctx_set (xlator_t *this, inode_t *inode, struct md_cache *mdc)
+{
+	int   ret;
+
+	LOCK(&inode->lock);
+	{
+		ret = __mdc_inode_ctx_set (this, inode, mdc);
+	}
+	UNLOCK(&inode->lock);
+
+	return ret;
+}
+
+
+mdc_local_t *
+mdc_local_get (call_frame_t *frame)
+{
+        mdc_local_t *local = NULL;
+
+        local = frame->local;
+        if (local)
+                goto out;
+
+        local = GF_CALLOC (sizeof (*local), 1, gf_mdc_mt_mdc_local_t);
+        if (!local)
+                goto out;
+
+        frame->local = local;
+out:
+        return local;
+}
+
+
+void
+mdc_local_wipe (xlator_t *this, mdc_local_t *local)
+{
+        if (!local)
+                return;
+
+        loc_wipe (&local->loc);
+
+        loc_wipe (&local->loc2);
+
+        if (local->fd)
+                fd_unref (local->fd);
+
+        GF_FREE (local->linkname);
+
+        GF_FREE (local->key);
+
+        if (local->xattr)
+                dict_unref (local->xattr);
+
+        GF_FREE (local);
+        return;
+}
+
+
+int
+mdc_inode_wipe (xlator_t *this, inode_t *inode)
+{
+        int              ret = 0;
+        uint64_t         mdc_int = 0;
+        struct md_cache *mdc = NULL;
+
+        ret = inode_ctx_del (inode, this, &mdc_int);
+        if (ret != 0)
+                goto out;
+
+        mdc = (void *) (long) mdc_int;
+
+        if (mdc->xattr)
+                dict_unref (mdc->xattr);
+
+        GF_FREE (mdc->linkname);
+
+        GF_FREE (mdc);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+struct md_cache *
+mdc_inode_prep (xlator_t *this, inode_t *inode)
+{
+        int              ret = 0;
+        struct md_cache *mdc = NULL;
+
+        LOCK (&inode->lock);
+        {
+		ret = __mdc_inode_ctx_get (this, inode, &mdc);
+                if (ret == 0)
+                        goto unlock;
+
+                mdc = GF_CALLOC (sizeof (*mdc), 1, gf_mdc_mt_md_cache_t);
+                if (!mdc) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "out of memory :(");
+                        goto unlock;
+                }
+
+                LOCK_INIT (&mdc->lock);
+
+                ret = __mdc_inode_ctx_set (this, inode, mdc);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "out of memory :(");
+                        GF_FREE (mdc);
+                        mdc = NULL;
+                }
+        }
+unlock:
+        UNLOCK (&inode->lock);
+
+        return mdc;
+}
+
+
+static gf_boolean_t
+is_md_cache_iatt_valid (xlator_t *this, struct md_cache *mdc)
+{
+	struct mdc_conf *conf = NULL;
+	time_t           now = 0;
+        gf_boolean_t     ret = _gf_true;
+	conf = this->private;
+
+	time (&now);
+
+        LOCK (&mdc->lock);
+        {
+                if (now >= (mdc->ia_time + conf->timeout))
+                        ret = _gf_false;
+        }
+        UNLOCK (&mdc->lock);
+
+	return ret;
+}
+
+
+static gf_boolean_t
+is_md_cache_xatt_valid (xlator_t *this, struct md_cache *mdc)
+{
+	struct mdc_conf *conf = NULL;
+	time_t           now = 0;
+        gf_boolean_t     ret = _gf_true;
+
+	conf = this->private;
+
+	time (&now);
+
+        LOCK (&mdc->lock);
+        {
+                if (now >= (mdc->xa_time + conf->timeout))
+                        ret = _gf_false;
+        }
+        UNLOCK (&mdc->lock);
+
+	return ret;
+}
+
+
+void
+mdc_from_iatt (struct md_cache *mdc, struct iatt *iatt)
+{
+        mdc->md_prot       = iatt->ia_prot;
+        mdc->md_nlink      = iatt->ia_nlink;
+        mdc->md_uid        = iatt->ia_uid;
+        mdc->md_gid        = iatt->ia_gid;
+        mdc->md_atime      = iatt->ia_atime;
+        mdc->md_atime_nsec = iatt->ia_atime_nsec;
+        mdc->md_mtime      = iatt->ia_mtime;
+        mdc->md_mtime_nsec = iatt->ia_mtime_nsec;
+        mdc->md_ctime      = iatt->ia_ctime;
+        mdc->md_ctime_nsec = iatt->ia_ctime_nsec;
+        mdc->md_rdev       = iatt->ia_rdev;
+        mdc->md_size       = iatt->ia_size;
+        mdc->md_blocks     = iatt->ia_blocks;
+}
+
+
+void
+mdc_to_iatt (struct md_cache *mdc, struct iatt *iatt)
+{
+        iatt->ia_prot       = mdc->md_prot;
+        iatt->ia_nlink      = mdc->md_nlink;
+        iatt->ia_uid        = mdc->md_uid;
+        iatt->ia_gid        = mdc->md_gid;
+        iatt->ia_atime      = mdc->md_atime;
+        iatt->ia_atime_nsec = mdc->md_atime_nsec;
+        iatt->ia_mtime      = mdc->md_mtime;
+        iatt->ia_mtime_nsec = mdc->md_mtime_nsec;
+        iatt->ia_ctime      = mdc->md_ctime;
+        iatt->ia_ctime_nsec = mdc->md_ctime_nsec;
+        iatt->ia_rdev       = mdc->md_rdev;
+        iatt->ia_size       = mdc->md_size;
+        iatt->ia_blocks     = mdc->md_blocks;
+}
+
+
+int
+mdc_inode_iatt_set_validate(xlator_t *this, inode_t *inode, struct iatt *prebuf,
+			    struct iatt *iatt)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+
+        mdc = mdc_inode_prep (this, inode);
+        if (!mdc)
+                goto out;
+
+        LOCK (&mdc->lock);
+        {
+                if (!iatt || !iatt->ia_ctime) {
+                        mdc->ia_time = 0;
+                        goto unlock;
+                }
+
+		/*
+		 * Invalidate the inode if the mtime or ctime has changed
+		 * and the prebuf doesn't match the value we have cached.
+		 * TODO: writev returns with a NULL iatt due to
+		 * performance/write-behind, causing invalidation on writes.
+		 */
+		if (IA_ISREG(inode->ia_type) &&
+		    ((iatt->ia_mtime != mdc->md_mtime) ||
+		    (iatt->ia_ctime != mdc->md_ctime)))
+			if (!prebuf || (prebuf->ia_ctime != mdc->md_ctime) ||
+			    (prebuf->ia_mtime != mdc->md_mtime))
+				inode_invalidate(inode);
+
+                mdc_from_iatt (mdc, iatt);
+
+                time (&mdc->ia_time);
+        }
+unlock:
+        UNLOCK (&mdc->lock);
+        ret = 0;
+out:
+        return ret;
+}
+
+int mdc_inode_iatt_set(xlator_t *this, inode_t *inode, struct iatt *iatt)
+{
+	return mdc_inode_iatt_set_validate(this, inode, NULL, iatt);
+}
+
+int
+mdc_inode_iatt_get (xlator_t *this, inode_t *inode, struct iatt *iatt)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+
+        if (mdc_inode_ctx_get (this, inode, &mdc) != 0)
+                goto out;
+
+	if (!is_md_cache_iatt_valid (this, mdc))
+		goto out;
+
+        LOCK (&mdc->lock);
+        {
+                mdc_to_iatt (mdc, iatt);
+        }
+        UNLOCK (&mdc->lock);
+
+        uuid_copy (iatt->ia_gfid, inode->gfid);
+        iatt->ia_ino    = gfid_to_ino (inode->gfid);
+        iatt->ia_dev    = 42;
+        iatt->ia_type   = inode->ia_type;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+struct updatedict {
+	dict_t *dict;
+	int ret;
+};
+
+static int
+updatefn(dict_t *dict, char *key, data_t *value, void *data)
+{
+	struct updatedict *u = data;
+	const char *mdc_key;
+	int i = 0;
+
+	for (mdc_key = mdc_keys[i].name; (mdc_key = mdc_keys[i].name); i++) {
+		if (!mdc_keys[i].check)
+			continue;
+		if (strcmp(mdc_key, key))
+			continue;
+
+		if (!u->dict) {
+			u->dict = dict_new();
+			if (!u->dict) {
+				u->ret = -1;
+				return -1;
+			}
+		}
+
+		if (dict_set(u->dict, key, value) < 0) {
+			u->ret = -1;
+			return -1;
+		}
+
+		break;
+	}
+        return 0;
+}
+
+static int
+mdc_dict_update(dict_t **tgt, dict_t *src)
+{
+	struct updatedict u = {
+		.dict = *tgt,
+		.ret = 0,
+	};
+
+	dict_foreach(src, updatefn, &u);
+
+	if (*tgt)
+		return u.ret;
+
+	if ((u.ret < 0) && u.dict) {
+		dict_unref(u.dict);
+		return u.ret;
+	}
+
+	*tgt = u.dict;
+
+	return u.ret;
+}
+
+int
+mdc_inode_xatt_set (xlator_t *this, inode_t *inode, dict_t *dict)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+	dict_t		*newdict = NULL;
+
+        mdc = mdc_inode_prep (this, inode);
+        if (!mdc)
+                goto out;
+
+        if (!dict)
+                goto out;
+
+        LOCK (&mdc->lock);
+        {
+                if (mdc->xattr) {
+                        dict_unref (mdc->xattr);
+			mdc->xattr = NULL;
+		}
+
+		ret = mdc_dict_update(&newdict, dict);
+		if (ret < 0) {
+			UNLOCK(&mdc->lock);
+			goto out;
+		}
+
+		if (newdict)
+			mdc->xattr = newdict;
+
+                time (&mdc->xa_time);
+        }
+        UNLOCK (&mdc->lock);
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+mdc_inode_xatt_update (xlator_t *this, inode_t *inode, dict_t *dict)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+
+        mdc = mdc_inode_prep (this, inode);
+        if (!mdc)
+                goto out;
+
+        if (!dict)
+                goto out;
+
+        LOCK (&mdc->lock);
+        {
+		ret = mdc_dict_update(&mdc->xattr, dict);
+		if (ret < 0) {
+			UNLOCK(&mdc->lock);
+			goto out;
+		}
+
+                time (&mdc->xa_time);
+        }
+        UNLOCK (&mdc->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+mdc_inode_xatt_unset (xlator_t *this, inode_t *inode, char *name)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+
+        mdc = mdc_inode_prep (this, inode);
+        if (!mdc)
+                goto out;
+
+        if (!name)
+                goto out;
+
+        LOCK (&mdc->lock);
+        {
+		dict_del (mdc->xattr, name);
+        }
+        UNLOCK (&mdc->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+mdc_inode_xatt_get (xlator_t *this, inode_t *inode, dict_t **dict)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+
+        if (mdc_inode_ctx_get (this, inode, &mdc) != 0)
+                goto out;
+
+	if (!is_md_cache_xatt_valid (this, mdc))
+		goto out;
+
+        LOCK (&mdc->lock);
+        {
+                ret = 0;
+		/* Missing xattr only means no keys were there, i.e
+		   a negative cache for the "loaded" keys
+		*/
+                if (!mdc->xattr)
+                        goto unlock;
+
+                if (dict)
+                        *dict = dict_ref (mdc->xattr);
+        }
+unlock:
+        UNLOCK (&mdc->lock);
+
+out:
+        return ret;
+}
+
+
+int
+mdc_inode_iatt_invalidate (xlator_t *this, inode_t *inode)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+
+        if (mdc_inode_ctx_get (this, inode, &mdc) != 0)
+                goto out;
+
+        LOCK (&mdc->lock);
+        {
+		mdc->ia_time = 0;
+        }
+        UNLOCK (&mdc->lock);
+
+out:
+        return ret;
+}
+
+
+int
+mdc_inode_xatt_invalidate (xlator_t *this, inode_t *inode)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+
+        if (mdc_inode_ctx_get (this, inode, &mdc) != 0)
+                goto out;
+
+        LOCK (&mdc->lock);
+        {
+		mdc->xa_time = 0;
+        }
+        UNLOCK (&mdc->lock);
+
+out:
+        return ret;
+}
+
+
+void
+mdc_load_reqs (xlator_t *this, dict_t *dict)
+{
+	const char *mdc_key = NULL;
+	int  i = 0;
+	int  ret = 0;
+
+	for (mdc_key = mdc_keys[i].name; (mdc_key = mdc_keys[i].name); i++) {
+		if (!mdc_keys[i].load)
+			continue;
+		ret = dict_set_int8 (dict, (char *)mdc_key, 0);
+		if (ret)
+			return;
+	}
+}
+
+
+struct checkpair {
+	int  ret;
+	dict_t *rsp;
+};
+
+
+static int
+is_mdc_key_satisfied (const char *key)
+{
+	const char *mdc_key = NULL;
+	int  i = 0;
+
+	if (!key)
+		return 0;
+
+	for (mdc_key = mdc_keys[i].name; (mdc_key = mdc_keys[i].name); i++) {
+		if (!mdc_keys[i].load)
+			continue;
+		if (strcmp (mdc_key, key) == 0)
+			return 1;
+	}
+
+	return 0;
+}
+
+
+static int
+checkfn (dict_t *this, char *key, data_t *value, void *data)
+{
+        struct checkpair *pair = data;
+
+	if (!is_mdc_key_satisfied (key))
+		pair->ret = 0;
+
+        return 0;
+}
+
+
+int
+mdc_xattr_satisfied (xlator_t *this, dict_t *req, dict_t *rsp)
+{
+        struct checkpair pair = {
+                .ret = 1,
+                .rsp = rsp,
+        };
+
+        dict_foreach (req, checkfn, &pair);
+
+        return pair.ret;
+}
+
+
+int
+mdc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret,	int32_t op_errno, inode_t *inode,
+                struct iatt *stbuf, dict_t *dict, struct iatt *postparent)
+{
+        mdc_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        if (local->loc.parent) {
+                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+        }
+
+        if (local->loc.inode) {
+                mdc_inode_iatt_set (this, local->loc.inode, stbuf);
+                mdc_inode_xatt_set (this, local->loc.inode, dict);
+        }
+out:
+        MDC_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf,
+                          dict, postparent);
+        return 0;
+}
+
+
+int
+mdc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            dict_t *xdata)
+{
+        int          ret = 0;
+        struct iatt  stbuf = {0, };
+        struct iatt  postparent = {0, };
+        dict_t      *xattr_rsp = NULL;
+        dict_t      *xattr_alloc = NULL;
+        mdc_local_t *local = NULL;
+
+
+        local = mdc_local_get (frame);
+        if (!local)
+                goto uncached;
+
+	if (!loc->name)
+		/* A nameless discovery is dangerous to cache. We
+		   perform nameless lookup with the intention of
+		   re-establishing an inode "properly"
+		*/
+		goto uncached;
+
+        loc_copy (&local->loc, loc);
+
+        ret = mdc_inode_iatt_get (this, loc->inode, &stbuf);
+        if (ret != 0)
+                goto uncached;
+
+        if (xdata) {
+                ret = mdc_inode_xatt_get (this, loc->inode, &xattr_rsp);
+                if (ret != 0)
+                        goto uncached;
+
+                if (!mdc_xattr_satisfied (this, xdata, xattr_rsp))
+                        goto uncached;
+        }
+
+        MDC_STACK_UNWIND (lookup, frame, 0, 0, loc->inode, &stbuf,
+                          xattr_rsp, &postparent);
+
+        if (xattr_rsp)
+                dict_unref (xattr_rsp);
+
+        return 0;
+
+uncached:
+	if (!xdata)
+		xdata = xattr_alloc = dict_new ();
+	if (xdata)
+		mdc_load_reqs (this, xdata);
+
+        STACK_WIND (frame, mdc_lookup_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->lookup, loc, xdata);
+
+        if (xattr_rsp)
+                dict_unref (xattr_rsp);
+	if (xattr_alloc)
+		dict_unref (xattr_alloc);
+        return 0;
+}
+
+
+int
+mdc_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        if (op_ret != 0)
+                goto out;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set (this, local->loc.inode, buf);
+
+out:
+        MDC_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        int           ret;
+        struct iatt   stbuf;
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+        if (!local)
+                goto uncached;
+
+        loc_copy (&local->loc, loc);
+
+        ret = mdc_inode_iatt_get (this, loc->inode, &stbuf);
+        if (ret != 0)
+                goto uncached;
+
+        MDC_STACK_UNWIND (stat, frame, 0, 0, &stbuf, xdata);
+
+        return 0;
+
+uncached:
+        STACK_WIND (frame, mdc_stat_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+        return 0;
+}
+
+
+int
+mdc_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        if (op_ret != 0)
+                goto out;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set (this, local->fd->inode, buf);
+
+out:
+        MDC_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        int           ret;
+        struct iatt   stbuf;
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+        if (!local)
+                goto uncached;
+
+        local->fd = fd_ref (fd);
+
+        ret = mdc_inode_iatt_get (this, fd->inode, &stbuf);
+        if (ret != 0)
+                goto uncached;
+
+        MDC_STACK_UNWIND (fstat, frame, 0, 0, &stbuf, xdata);
+
+        return 0;
+
+uncached:
+        STACK_WIND (frame, mdc_fstat_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+        return 0;
+}
+
+
+int
+mdc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->loc.inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+
+        return 0;
+}
+
+
+int
+mdc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              off_t offset, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        local->loc.inode = inode_ref (loc->inode);
+
+        STACK_WIND (frame, mdc_truncate_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+                    loc, offset, xdata);
+        return 0;
+}
+
+
+int
+mdc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+
+        return 0;
+}
+
+
+int
+mdc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               off_t offset, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        local->fd = fd_ref (fd);
+
+        STACK_WIND (frame, mdc_ftruncate_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->ftruncate,
+                    fd, offset, xdata);
+        return 0;
+}
+
+
+int
+mdc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+        mdc_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        if (local->loc.parent) {
+                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+        }
+
+        if (local->loc.inode) {
+                mdc_inode_iatt_set (this, local->loc.inode, buf);
+                mdc_inode_xatt_set (this, local->loc.inode, local->xattr);
+        }
+out:
+        MDC_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
+                          preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+mdc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+           mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        loc_copy (&local->loc, loc);
+        local->xattr = dict_ref (xdata);
+
+        STACK_WIND (frame, mdc_mknod_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                    loc, mode, rdev, umask, xdata);
+        return 0;
+}
+
+
+int
+mdc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+        mdc_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        if (local->loc.parent) {
+                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+        }
+
+        if (local->loc.inode) {
+                mdc_inode_iatt_set (this, local->loc.inode, buf);
+                mdc_inode_xatt_set (this, local->loc.inode, local->xattr);
+        }
+out:
+        MDC_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
+                          preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+mdc_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+           mode_t mode, mode_t umask, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        loc_copy (&local->loc, loc);
+        local->xattr = dict_ref (xdata);
+
+        STACK_WIND (frame, mdc_mkdir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                    loc, mode, umask, xdata);
+        return 0;
+}
+
+
+int
+mdc_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        mdc_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        if (local->loc.parent) {
+                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+        }
+
+        if (local->loc.inode) {
+                mdc_inode_iatt_set (this, local->loc.inode, NULL);
+        }
+
+out:
+        MDC_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                          preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+mdc_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+            dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        loc_copy (&local->loc, loc);
+
+        STACK_WIND (frame, mdc_unlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+        return 0;
+}
+
+
+int
+mdc_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        mdc_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        if (local->loc.parent) {
+                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+        }
+
+out:
+        MDC_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                          preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+mdc_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+           dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        loc_copy (&local->loc, loc);
+
+        STACK_WIND (frame, mdc_rmdir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flag, xdata);
+        return 0;
+}
+
+
+int
+mdc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+        mdc_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        if (local->loc.parent) {
+                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+        }
+
+        if (local->loc.inode) {
+                mdc_inode_iatt_set (this, local->loc.inode, buf);
+        }
+out:
+        MDC_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
+                          preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+mdc_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
+             loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        loc_copy (&local->loc, loc);
+
+        local->linkname = gf_strdup (linkname);
+
+        STACK_WIND (frame, mdc_symlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkname, loc, umask, xdata);
+        return 0;
+}
+
+
+int
+mdc_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                struct iatt *preoldparent, struct iatt *postoldparent,
+                struct iatt *prenewparent, struct iatt *postnewparent,
+                dict_t *xdata)
+{
+        mdc_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        if (local->loc.parent) {
+                mdc_inode_iatt_set (this, local->loc.parent, postoldparent);
+        }
+
+        if (local->loc.inode) {
+		/* TODO: fix dht_rename() not to return linkfile
+		   attributes before setting attributes here
+		*/
+
+		mdc_inode_iatt_set (this, local->loc.inode, NULL);
+        }
+
+        if (local->loc2.parent) {
+                mdc_inode_iatt_set (this, local->loc2.parent, postnewparent);
+        }
+out:
+        MDC_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
+                          preoldparent, postoldparent, prenewparent,
+                          postnewparent, xdata);
+        return 0;
+}
+
+
+int
+mdc_rename (call_frame_t *frame, xlator_t *this,
+            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        loc_copy (&local->loc, oldloc);
+        loc_copy (&local->loc2, newloc);
+
+        STACK_WIND (frame, mdc_rename_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+
+int
+mdc_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        mdc_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        if (local->loc.inode) {
+                mdc_inode_iatt_set (this, local->loc.inode, buf);
+        }
+
+        if (local->loc2.parent) {
+                mdc_inode_iatt_set (this, local->loc2.parent, postparent);
+        }
+out:
+        MDC_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
+                          preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+mdc_link (call_frame_t *frame, xlator_t *this,
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        loc_copy (&local->loc, oldloc);
+        loc_copy (&local->loc2, newloc);
+
+        STACK_WIND (frame, mdc_link_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+
+int
+mdc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        mdc_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        if (local->loc.parent) {
+                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+        }
+
+        if (local->loc.inode) {
+                mdc_inode_iatt_set (this, inode, buf);
+                mdc_inode_xatt_set (this, local->loc.inode, local->xattr);
+        }
+out:
+        MDC_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
+                          preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+mdc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        loc_copy (&local->loc, loc);
+        local->xattr = dict_ref (xdata);
+
+        STACK_WIND (frame, mdc_create_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+}
+
+
+int
+mdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iovec *vector, int32_t count,
+               struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set (this, local->fd->inode, stbuf);
+
+out:
+        MDC_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
+                          stbuf, iobref, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t offset, uint32_t flags, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        local->fd = fd_ref (fd);
+
+        STACK_WIND (frame, mdc_readv_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+        return 0;
+}
+
+
+int
+mdc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+
+        return 0;
+}
+
+
+int
+mdc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+            int count, off_t offset, uint32_t flags, struct iobref *iobref,
+            dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        local->fd = fd_ref (fd);
+
+        STACK_WIND (frame, mdc_writev_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+                    fd, vector, count, offset, flags, iobref, xdata);
+        return 0;
+}
+
+
+int
+mdc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0) {
+		mdc_inode_iatt_set (this, local->loc.inode, NULL);
+                goto out;
+	}
+
+        if (!local)
+                goto out;
+
+	mdc_inode_iatt_set_validate(this, local->loc.inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND (setattr, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+
+        return 0;
+}
+
+
+int
+mdc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             struct iatt *stbuf, int valid, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        loc_copy (&local->loc, loc);
+
+        STACK_WIND (frame, mdc_setattr_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+        return 0;
+}
+
+
+int
+mdc_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+
+        return 0;
+}
+
+
+int
+mdc_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iatt *stbuf, int valid, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        local->fd = fd_ref (fd);
+
+        STACK_WIND (frame, mdc_fsetattr_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+        return 0;
+}
+
+
+int
+mdc_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+
+        return 0;
+}
+
+
+int
+mdc_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+           dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        local->fd = fd_ref (fd);
+
+        STACK_WIND (frame, mdc_fsync_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                    fd, datasync, xdata);
+        return 0;
+}
+
+
+int
+mdc_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_xatt_update (this, local->loc.inode, local->xattr);
+
+	mdc_inode_iatt_invalidate (this, local->loc.inode);
+
+out:
+        MDC_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xattr, int flags, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+        loc_copy (&local->loc, loc);
+        local->xattr = dict_ref (xattr);
+
+        STACK_WIND (frame, mdc_setxattr_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                    loc, xattr, flags, xdata);
+        return 0;
+}
+
+
+int
+mdc_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_xatt_update (this, local->fd->inode, local->xattr);
+
+	mdc_inode_iatt_invalidate (this, local->fd->inode);
+out:
+        MDC_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	       dict_t *xattr, int flags, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+	local->fd = fd_ref (fd);
+        local->xattr = dict_ref (xattr);
+
+        STACK_WIND (frame, mdc_fsetxattr_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetxattr,
+                    fd, xattr, flags, xdata);
+        return 0;
+}
+
+int
+mdc_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                  dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        if (op_ret != 0)
+                goto out;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        mdc_inode_xatt_update (this, local->loc.inode, xattr);
+
+out:
+        MDC_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key,
+              dict_t *xdata)
+{
+        int           ret;
+	int           op_errno = ENODATA;
+        mdc_local_t  *local = NULL;
+	dict_t       *xattr = NULL;
+
+        local = mdc_local_get (frame);
+        if (!local)
+                goto uncached;
+
+        loc_copy (&local->loc, loc);
+
+	if (!is_mdc_key_satisfied (key))
+		goto uncached;
+
+	ret = mdc_inode_xatt_get (this, loc->inode, &xattr);
+	if (ret != 0)
+		goto uncached;
+
+	if (!xattr || !dict_get (xattr, (char *)key)) {
+		ret = -1;
+		op_errno = ENODATA;
+	}
+
+        MDC_STACK_UNWIND (getxattr, frame, ret, op_errno, xattr, xdata);
+
+        return 0;
+
+uncached:
+        STACK_WIND (frame, mdc_getxattr_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr,
+                    loc, key, xdata);
+        return 0;
+}
+
+
+int
+mdc_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		   int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                   dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        if (op_ret != 0)
+                goto out;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        mdc_inode_xatt_update (this, local->fd->inode, xattr);
+
+out:
+        MDC_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+               dict_t *xdata)
+{
+        int           ret;
+        mdc_local_t  *local = NULL;
+	dict_t       *xattr = NULL;
+	int           op_errno = ENODATA;
+
+        local = mdc_local_get (frame);
+        if (!local)
+                goto uncached;
+
+        local->fd = fd_ref (fd);
+
+	if (!is_mdc_key_satisfied (key))
+		goto uncached;
+
+	ret = mdc_inode_xatt_get (this, fd->inode, &xattr);
+	if (ret != 0)
+		goto uncached;
+
+	if (!xattr || !dict_get (xattr, (char *)key)) {
+		ret = -1;
+		op_errno = ENODATA;
+	}
+
+        MDC_STACK_UNWIND (fgetxattr, frame, ret, op_errno, xattr, xdata);
+
+        return 0;
+
+uncached:
+        STACK_WIND (frame, mdc_fgetxattr_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fgetxattr,
+                    fd, key, xdata);
+        return 0;
+}
+
+int
+mdc_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+	if (local->key)
+		mdc_inode_xatt_unset (this, local->loc.inode, local->key);
+	else
+		mdc_inode_xatt_invalidate (this, local->loc.inode);
+
+	mdc_inode_iatt_invalidate (this, local->loc.inode);
+out:
+        MDC_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+		 const char *name, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+	loc_copy (&local->loc, loc);
+
+	local->key = gf_strdup (name);
+
+        STACK_WIND (frame, mdc_removexattr_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+
+int
+mdc_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+	if (local->key)
+		mdc_inode_xatt_unset (this, local->fd->inode, local->key);
+	else
+		mdc_inode_xatt_invalidate (this, local->fd->inode);
+
+	mdc_inode_iatt_invalidate (this, local->fd->inode);
+out:
+        MDC_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		  const char *name, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+	local->fd = fd_ref (fd);
+
+	local->key = gf_strdup (name);
+
+        STACK_WIND (frame, mdc_fremovexattr_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+
+int
+mdc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+        gf_dirent_t *entry      = NULL;
+
+	if (op_ret <= 0)
+		goto unwind;
+
+        list_for_each_entry (entry, &entries->list, list) {
+                if (!entry->inode)
+			continue;
+                mdc_inode_iatt_set (this, entry->inode, &entry->d_stat);
+                mdc_inode_xatt_set (this, entry->inode, entry->dict);
+        }
+
+unwind:
+	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+
+int
+mdc_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	      size_t size, off_t offset, dict_t *xdata)
+{
+	dict_t *xattr_alloc = NULL;
+
+	if (!xdata)
+		xdata = xattr_alloc = dict_new ();
+	if (xdata)
+		mdc_load_reqs (this, xdata);
+
+	STACK_WIND (frame, mdc_readdirp_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readdirp,
+		    fd, size, offset, xdata);
+	if (xattr_alloc)
+		dict_unref (xattr_alloc);
+	return 0;
+}
+
+int
+mdc_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+		int op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+int
+mdc_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	     size_t size, off_t offset, dict_t *xdata)
+{
+        int need_unref = 0;
+	struct mdc_conf *conf = this->private;
+
+	if (!conf->force_readdirp) {
+		STACK_WIND(frame, mdc_readdir_cbk, FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->readdir, fd, size, offset,
+			   xdata);
+		return 0;
+	}
+
+	if (!xdata) {
+                xdata = dict_new ();
+                need_unref = 1;
+        }
+
+        if (xdata)
+		mdc_load_reqs (this, xdata);
+
+	STACK_WIND(frame, mdc_readdirp_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp, fd, size, offset,
+		   xdata);
+
+        if (need_unref && xdata)
+                dict_unref (xdata);
+
+	return 0;
+}
+
+int
+mdc_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND (fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+
+        return 0;
+}
+
+int mdc_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		  off_t offset, size_t len, dict_t *xdata)
+{
+	mdc_local_t *local;
+
+	local = mdc_local_get(frame);
+	local->fd = fd_ref(fd);
+
+	STACK_WIND(frame, mdc_fallocate_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		   xdata);
+
+	return 0;
+}
+
+int
+mdc_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno,
+                struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND(discard, frame, op_ret, op_errno, prebuf, postbuf,
+                         xdata);
+
+        return 0;
+}
+
+int mdc_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+		size_t len, dict_t *xdata)
+{
+	mdc_local_t *local;
+
+	local = mdc_local_get(frame);
+	local->fd = fd_ref(fd);
+
+	STACK_WIND(frame, mdc_discard_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->discard, fd, offset, len,
+		   xdata);
+
+	return 0;
+}
+
+int
+mdc_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno,
+                struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                         xdata);
+
+        return 0;
+}
+
+int mdc_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                size_t len, dict_t *xdata)
+{
+        mdc_local_t *local;
+
+        local = mdc_local_get(frame);
+        local->fd = fd_ref(fd);
+
+        STACK_WIND(frame, mdc_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len,
+                   xdata);
+
+        return 0;
+}
+
+
+int
+mdc_forget (xlator_t *this, inode_t *inode)
+{
+        mdc_inode_wipe (this, inode);
+
+        return 0;
+}
+
+
+int
+is_strpfx (const char *str1, const char *str2)
+{
+	/* is one of the string a prefix of the other? */
+	int i;
+
+	for (i = 0; str1[i] == str2[i]; i++) {
+		if (!str1[i] || !str2[i])
+			break;
+	}
+
+	return !(str1[i] && str2[i]);
+}
+
+
+int
+mdc_key_load_set (struct mdc_key *keys, char *pattern, gf_boolean_t val)
+{
+	struct mdc_key *key = NULL;
+
+	for (key = keys; key->name; key++) {
+		if (is_strpfx (key->name, pattern))
+			key->load = val;
+	}
+
+	return 0;
+}
+
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+	struct mdc_conf *conf = NULL;
+
+	conf = this->private;
+
+	GF_OPTION_RECONF ("md-cache-timeout", conf->timeout, options, int32, out);
+
+	GF_OPTION_RECONF ("cache-selinux", conf->cache_selinux, options, bool, out);
+	mdc_key_load_set (mdc_keys, "security.", conf->cache_selinux);
+
+	GF_OPTION_RECONF ("cache-posix-acl", conf->cache_posix_acl, options, bool, out);
+	mdc_key_load_set (mdc_keys, "system.posix_acl_", conf->cache_posix_acl);
+
+	GF_OPTION_RECONF("force-readdirp", conf->force_readdirp, options, bool, out);
+
+out:
+	return 0;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_mdc_mt_end + 1);
+        return ret;
+}
+
+int
+init (xlator_t *this)
+{
+	struct mdc_conf *conf = NULL;
+
+	conf = GF_CALLOC (sizeof (*conf), 1, gf_mdc_mt_mdc_conf_t);
+	if (!conf) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"out of memory");
+		return -1;
+	}
+
+        GF_OPTION_INIT ("md-cache-timeout", conf->timeout, int32, out);
+
+	GF_OPTION_INIT ("cache-selinux", conf->cache_selinux, bool, out);
+	mdc_key_load_set (mdc_keys, "security.", conf->cache_selinux);
+
+	GF_OPTION_INIT ("cache-posix-acl", conf->cache_posix_acl, bool, out);
+	mdc_key_load_set (mdc_keys, "system.posix_acl_", conf->cache_posix_acl);
+
+	GF_OPTION_INIT("force-readdirp", conf->force_readdirp, bool, out);
+out:
+	this->private = conf;
+
+        return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+        return;
+}
+
+
+struct xlator_fops fops = {
+        .lookup      = mdc_lookup,
+        .stat        = mdc_stat,
+        .fstat       = mdc_fstat,
+        .truncate    = mdc_truncate,
+        .ftruncate   = mdc_ftruncate,
+        .mknod       = mdc_mknod,
+        .mkdir       = mdc_mkdir,
+        .unlink      = mdc_unlink,
+        .rmdir       = mdc_rmdir,
+        .symlink     = mdc_symlink,
+        .rename      = mdc_rename,
+        .link        = mdc_link,
+        .create      = mdc_create,
+        .readv       = mdc_readv,
+        .writev      = mdc_writev,
+        .setattr     = mdc_setattr,
+        .fsetattr    = mdc_fsetattr,
+        .fsync       = mdc_fsync,
+        .setxattr    = mdc_setxattr,
+        .fsetxattr   = mdc_fsetxattr,
+        .getxattr    = mdc_getxattr,
+        .fgetxattr   = mdc_fgetxattr,
+	.removexattr = mdc_removexattr,
+	.fremovexattr= mdc_fremovexattr,
+	.readdirp    = mdc_readdirp,
+	.readdir     = mdc_readdir,
+	.fallocate   = mdc_fallocate,
+	.discard     = mdc_discard,
+        .zerofill    = mdc_zerofill,
+};
+
+
+struct xlator_cbks cbks = {
+        .forget      = mdc_forget,
+};
+
+struct volume_options options[] = {
+	{ .key = {"cache-selinux"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "false",
+	},
+	{ .key = {"cache-posix-acl"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "false",
+	},
+        { .key = {"md-cache-timeout"},
+          .type = GF_OPTION_TYPE_INT,
+          .min = 0,
+          .max = 60,
+          .default_value = "1",
+          .description = "Time period after which cache has to be refreshed",
+        },
+	{ .key = {"force-readdirp"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "true",
+	  .description = "Convert all readdir requests to readdirplus to "
+			 "collect stat info on each entry.",
+	},
+    { .key = {NULL} },
+};
diff --git a/xlators/performance/open-behind/Makefile.am b/xlators/performance/open-behind/Makefile.am
new file mode 100644
index 000000000..af437a64d
--- /dev/null
+++ b/xlators/performance/open-behind/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = src
diff --git a/xlators/performance/open-behind/src/Makefile.am b/xlators/performance/open-behind/src/Makefile.am
new file mode 100644
index 000000000..125285707
--- /dev/null
+++ b/xlators/performance/open-behind/src/Makefile.am
@@ -0,0 +1,15 @@
+xlator_LTLIBRARIES = open-behind.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
+
+open_behind_la_LDFLAGS = -module -avoid-version
+
+open_behind_la_SOURCES = open-behind.c
+open_behind_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = open-behind-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/performance/open-behind/src/open-behind-mem-types.h b/xlators/performance/open-behind/src/open-behind-mem-types.h
new file mode 100644
index 000000000..1e94296f4
--- /dev/null
+++ b/xlators/performance/open-behind/src/open-behind-mem-types.h
@@ -0,0 +1,21 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __OB_MEM_TYPES_H__
+#define __OB_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_ob_mem_types_ {
+        gf_ob_mt_fd_t   = gf_common_mt_end + 1,
+	gf_ob_mt_conf_t,
+        gf_ob_mt_end
+};
+#endif
diff --git a/xlators/performance/open-behind/src/open-behind.c b/xlators/performance/open-behind/src/open-behind.c
new file mode 100644
index 000000000..7e5b57278
--- /dev/null
+++ b/xlators/performance/open-behind/src/open-behind.c
@@ -0,0 +1,1001 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "open-behind-mem-types.h"
+#include "xlator.h"
+#include "statedump.h"
+#include "call-stub.h"
+#include "defaults.h"
+
+typedef struct ob_conf {
+	gf_boolean_t  use_anonymous_fd; /* use anonymous FDs wherever safe
+					   e.g - fstat() readv()
+
+					   whereas for fops like writev(), lk(),
+					   the fd is important for side effects
+					   like mandatory locks
+					*/
+	gf_boolean_t  lazy_open; /* delay backend open as much as possible */
+} ob_conf_t;
+
+
+typedef struct ob_fd {
+	call_frame_t     *open_frame;
+	loc_t             loc;
+	dict_t           *xdata;
+	int               flags;
+	int               op_errno;
+	struct list_head  list;
+} ob_fd_t;
+
+
+ob_fd_t *
+__ob_fd_ctx_get (xlator_t *this, fd_t *fd)
+{
+	uint64_t    value = 0;
+	int         ret = -1;
+	ob_fd_t    *ob_fd = NULL;
+
+	ret = __fd_ctx_get (fd, this, &value);
+	if (ret)
+		return NULL;
+
+	ob_fd = (void *) ((long) value);
+
+	return ob_fd;
+}
+
+
+ob_fd_t *
+ob_fd_ctx_get (xlator_t *this, fd_t *fd)
+{
+	ob_fd_t  *ob_fd = NULL;
+
+	LOCK (&fd->lock);
+	{
+		ob_fd = __ob_fd_ctx_get (this, fd);
+	}
+	UNLOCK (&fd->lock);
+
+	return ob_fd;
+}
+
+
+int
+__ob_fd_ctx_set (xlator_t *this, fd_t *fd, ob_fd_t *ob_fd)
+{
+	uint64_t    value = 0;
+	int         ret = -1;
+
+	value = (long) ((void *) ob_fd);
+
+	ret = __fd_ctx_set (fd, this, value);
+
+	return ret;
+}
+
+
+int
+ob_fd_ctx_set (xlator_t *this, fd_t *fd, ob_fd_t *ob_fd)
+{
+	int ret = -1;
+
+	LOCK (&fd->lock);
+	{
+		ret = __ob_fd_ctx_set (this, fd, ob_fd);
+	}
+	UNLOCK (&fd->lock);
+
+	return ret;
+}
+
+
+ob_fd_t *
+ob_fd_new (void)
+{
+	ob_fd_t  *ob_fd = NULL;
+
+	ob_fd = GF_CALLOC (1, sizeof (*ob_fd), gf_ob_mt_fd_t);
+
+	INIT_LIST_HEAD (&ob_fd->list);
+
+	return ob_fd;
+}
+
+
+void
+ob_fd_free (ob_fd_t *ob_fd)
+{
+	loc_wipe (&ob_fd->loc);
+
+	if (ob_fd->xdata)
+		dict_unref (ob_fd->xdata);
+
+	if (ob_fd->open_frame)
+		STACK_DESTROY (ob_fd->open_frame->root);
+
+	GF_FREE (ob_fd);
+}
+
+
+int
+ob_wake_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	     int op_ret, int op_errno, fd_t *fd_ret, dict_t *xdata)
+{
+	fd_t              *fd = NULL;
+	struct list_head   list;
+	ob_fd_t           *ob_fd = NULL;
+	call_stub_t       *stub = NULL, *tmp = NULL;
+
+	fd = frame->local;
+	frame->local = NULL;
+
+	INIT_LIST_HEAD (&list);
+
+	LOCK (&fd->lock);
+	{
+		ob_fd = __ob_fd_ctx_get (this, fd);
+
+		list_splice_init (&ob_fd->list, &list);
+
+		if (op_ret < 0) {
+			/* mark fd BAD for ever */
+			ob_fd->op_errno = op_errno;
+		} else {
+			__fd_ctx_del (fd, this, NULL);
+			ob_fd_free (ob_fd);
+		}
+	}
+	UNLOCK (&fd->lock);
+
+	list_for_each_entry_safe (stub, tmp, &list, list) {
+		list_del_init (&stub->list);
+
+		if (op_ret < 0)
+			call_unwind_error (stub, -1, op_errno);
+		else
+			call_resume (stub);
+	}
+
+	fd_unref (fd);
+
+	STACK_DESTROY (frame->root);
+
+	return 0;
+}
+
+
+int
+ob_fd_wake (xlator_t *this, fd_t *fd)
+{
+	call_frame_t *frame = NULL;
+	ob_fd_t      *ob_fd = NULL;
+
+	LOCK (&fd->lock);
+	{
+		ob_fd = __ob_fd_ctx_get (this, fd);
+		if (!ob_fd)
+			goto unlock;
+
+		frame = ob_fd->open_frame;
+		ob_fd->open_frame = NULL;
+	}
+unlock:
+	UNLOCK (&fd->lock);
+
+	if (frame) {
+		frame->local = fd_ref (fd);
+
+		STACK_WIND (frame, ob_wake_cbk, FIRST_CHILD (this),
+			    FIRST_CHILD (this)->fops->open,
+			    &ob_fd->loc, ob_fd->flags, fd, ob_fd->xdata);
+	}
+
+	return 0;
+}
+
+
+int
+open_and_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
+{
+	ob_fd_t  *ob_fd = NULL;
+	int       op_errno = 0;
+
+	if (!fd)
+		goto nofd;
+
+	LOCK (&fd->lock);
+	{
+		ob_fd = __ob_fd_ctx_get (this, fd);
+		if (!ob_fd)
+			goto unlock;
+
+		if (ob_fd->op_errno) {
+			op_errno = ob_fd->op_errno;
+			goto unlock;
+		}
+
+		list_add_tail (&stub->list, &ob_fd->list);
+	}
+unlock:
+	UNLOCK (&fd->lock);
+
+nofd:
+	if (op_errno)
+		call_unwind_error (stub, -1, op_errno);
+	else if (ob_fd)
+		ob_fd_wake (this, fd);
+	else
+		call_resume (stub);
+
+	return 0;
+}
+
+
+int
+ob_open_behind (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+		fd_t *fd, dict_t *xdata)
+{
+	ob_fd_t    *ob_fd = NULL;
+	int         ret = -1;
+	ob_conf_t  *conf = NULL;
+
+
+	conf = this->private;
+
+	if (flags & O_TRUNC) {
+		STACK_WIND (frame, default_open_cbk,
+			    FIRST_CHILD (this), FIRST_CHILD (this)->fops->open,
+			    loc, flags, fd, xdata);
+		return 0;
+	}
+
+	ob_fd = ob_fd_new ();
+	if (!ob_fd)
+		goto enomem;
+
+	ob_fd->open_frame = copy_frame (frame);
+	if (!ob_fd->open_frame)
+		goto enomem;
+	ret = loc_copy (&ob_fd->loc, loc);
+	if (ret)
+		goto enomem;
+
+	ob_fd->flags = flags;
+	if (xdata)
+		ob_fd->xdata = dict_ref (xdata);
+
+	ret = ob_fd_ctx_set (this, fd, ob_fd);
+	if (ret)
+		goto enomem;
+
+	fd_ref (fd);
+
+	STACK_UNWIND_STRICT (open, frame, 0, 0, fd, xdata);
+
+	if (!conf->lazy_open)
+		ob_fd_wake (this, fd);
+
+	fd_unref (fd);
+
+	return 0;
+enomem:
+	if (ob_fd) {
+		if (ob_fd->open_frame)
+			STACK_DESTROY (ob_fd->open_frame->root);
+		loc_wipe (&ob_fd->loc);
+		if (ob_fd->xdata)
+			dict_unref (ob_fd->xdata);
+		GF_FREE (ob_fd);
+	}
+
+	return -1;
+}
+
+
+int
+ob_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+	 fd_t *fd, dict_t *xdata)
+{
+	fd_t         *old_fd = NULL;
+	int           ret = -1;
+	int           op_errno = 0;
+	call_stub_t  *stub = NULL;
+
+	old_fd = fd_lookup (fd->inode, 0);
+	if (old_fd) {
+		/* open-behind only when this is the first FD */
+		stub = fop_open_stub (frame, default_open_resume,
+				      loc, flags, fd, xdata);
+		if (!stub) {
+			op_errno = ENOMEM;
+			fd_unref (old_fd);
+			goto err;
+		}
+
+		open_and_resume (this, old_fd, stub);
+
+		fd_unref (old_fd);
+
+		return 0;
+	}
+
+	ret = ob_open_behind (frame, this, loc, flags, fd, xdata);
+	if (ret) {
+		op_errno = ENOMEM;
+		goto err;
+	}
+
+	return 0;
+err:
+	gf_log (this->name, GF_LOG_ERROR, "%s: %s", loc->path,
+		strerror (op_errno));
+
+	STACK_UNWIND_STRICT (open, frame, -1, op_errno, 0, 0);
+
+	return 0;
+}
+
+
+fd_t *
+ob_get_wind_fd (xlator_t *this, fd_t *fd)
+{
+	ob_conf_t  *conf = NULL;
+	ob_fd_t    *ob_fd = NULL;
+
+	conf = this->private;
+
+	ob_fd = ob_fd_ctx_get (this, fd);
+
+	if (ob_fd && conf->use_anonymous_fd)
+		return fd_anonymous (fd->inode);
+
+	return fd_ref (fd);
+}
+
+
+int
+ob_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	  off_t offset, uint32_t flags, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+	fd_t         *wind_fd = NULL;
+
+	wind_fd = ob_get_wind_fd (this, fd);
+
+	stub = fop_readv_stub (frame, default_readv_resume, wind_fd,
+			       size, offset, flags, xdata);
+	fd_unref (wind_fd);
+
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, wind_fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM, 0, 0, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
+	   int count, off_t offset, uint32_t flags, struct iobref *iobref,
+	   dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_writev_stub (frame, default_writev_resume, fd, iov, count,
+				offset, flags, iobref, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+	fd_t         *wind_fd = NULL;
+
+	wind_fd = ob_get_wind_fd (this, fd);
+
+	stub = fop_fstat_stub (frame, default_fstat_resume, wind_fd, xdata);
+
+	fd_unref (wind_fd);
+
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, wind_fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+	call_stub_t   *stub = NULL;
+	ob_fd_t       *ob_fd = NULL;
+	gf_boolean_t   unwind = _gf_false;
+
+	LOCK (&fd->lock);
+	{
+		ob_fd = __ob_fd_ctx_get (this, fd);
+		if (ob_fd && ob_fd->open_frame)
+			/* if open() was never wound to backend,
+			   no need to wind flush() either.
+			*/
+			unwind = _gf_true;
+	}
+	UNLOCK (&fd->lock);
+
+	if (unwind)
+		goto unwind;
+
+	stub = fop_flush_stub (frame, default_flush_resume, fd, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, 0);
+
+	return 0;
+
+unwind:
+	STACK_UNWIND_STRICT (flush, frame, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int flag,
+	  dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fsync_stub (frame, default_fsync_resume, fd, flag, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+       struct gf_flock *flock, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_lk_stub (frame, default_lk_resume, fd, cmd, flock, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM, 0, 0);
+
+	return 0;
+}
+
+int
+ob_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	      dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_ftruncate_stub (frame, default_ftruncate_resume, fd, offset,
+				   xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+	      int flags, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume, fd, xattr,
+				   flags, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, 0);
+
+	return 0;
+}
+
+
+int
+ob_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+	      dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume, fd, name,
+				   xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		 const char *name, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fremovexattr_stub (frame, default_fremovexattr_resume, fd,
+				      name, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fremovexattr, frame, -1, ENOMEM, 0);
+
+	return 0;
+}
+
+
+int
+ob_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+	     int cmd, struct gf_flock *flock, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_finodelk_stub (frame, default_finodelk_resume, volume, fd,
+				  cmd, flock, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, 0);
+
+	return 0;
+}
+
+
+int
+ob_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+	     const char *basename, entrylk_cmd cmd, entrylk_type type,
+	     dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fentrylk_stub (frame, default_fentrylk_resume, volume, fd,
+				  basename, cmd, type, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, 0);
+
+	return 0;
+}
+
+
+int
+ob_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	     gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fxattrop_stub (frame, default_fxattrop_resume, fd, optype,
+				  xattr, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	     struct iatt *iatt, int valid, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fsetattr_stub (frame, default_fsetattr_resume, fd,
+				  iatt, valid, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, 0, 0, 0);
+
+	return 0;
+}
+
+int
+ob_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+	     off_t offset, size_t len, dict_t *xdata)
+{
+	call_stub_t *stub;
+
+	stub = fop_fallocate_stub(frame, default_fallocate_resume, fd, mode,
+				  offset, len, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume(this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT(fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
+	return 0;
+}
+
+int
+ob_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	   size_t len, dict_t *xdata)
+{
+	call_stub_t *stub;
+
+	stub = fop_discard_stub(frame, default_discard_resume, fd, offset, len,
+				xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume(this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT(discard, frame, -1, ENOMEM, NULL, NULL, NULL);
+	return 0;
+}
+
+int
+ob_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
+{
+        call_stub_t *stub;
+
+        stub = fop_zerofill_stub(frame, default_zerofill_resume, fd,
+                                 offset, len, xdata);
+        if (!stub)
+                goto err;
+
+        open_and_resume(this, fd, stub);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT(zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+ob_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+	   dict_t *xdata)
+{
+	fd_t         *fd = NULL;
+	call_stub_t  *stub = NULL;
+
+	stub = fop_unlink_stub (frame, default_unlink_resume, loc,
+				xflags, xdata);
+	if (!stub)
+		goto err;
+
+	fd = fd_lookup (loc->inode, 0);
+
+	open_and_resume (this, fd, stub);
+        if (fd)
+                fd_unref (fd);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_rename (call_frame_t *frame, xlator_t *this, loc_t *src, loc_t *dst,
+	   dict_t *xdata)
+{
+	fd_t         *fd = NULL;
+	call_stub_t  *stub = NULL;
+
+	stub = fop_rename_stub (frame, default_rename_resume, src, dst, xdata);
+	if (!stub)
+		goto err;
+
+	if (dst->inode)
+		fd = fd_lookup (dst->inode, 0);
+
+	open_and_resume (this, fd, stub);
+        if (fd)
+                fd_unref (fd);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM, 0, 0, 0, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_release (xlator_t *this, fd_t *fd)
+{
+	ob_fd_t *ob_fd = NULL;
+
+	ob_fd = ob_fd_ctx_get (this, fd);
+
+	ob_fd_free (ob_fd);
+
+	return 0;
+}
+
+
+int
+ob_priv_dump (xlator_t *this)
+{
+        ob_conf_t        *conf       = NULL;
+        char              key_prefix[GF_DUMP_MAX_BUF_LEN];
+
+        conf = this->private;
+
+        if (!conf)
+                return -1;
+
+        gf_proc_dump_build_key (key_prefix, "xlator.performance.open-behind",
+                                "priv");
+
+        gf_proc_dump_add_section (key_prefix);
+
+        gf_proc_dump_write ("use_anonymous_fd", "%d", conf->use_anonymous_fd);
+
+        gf_proc_dump_write ("lazy_open", "%d", conf->lazy_open);
+
+        return 0;
+}
+
+
+int
+ob_fdctx_dump (xlator_t *this, fd_t *fd)
+{
+	ob_fd_t   *ob_fd = NULL;
+        char       key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
+	int        ret = 0;
+
+	ret = TRY_LOCK (&fd->lock);
+	if (ret)
+		return 0;
+
+	ob_fd = __ob_fd_ctx_get (this, fd);
+	if (!ob_fd) {
+		UNLOCK (&fd->lock);
+		return 0;
+	}
+
+        gf_proc_dump_build_key (key_prefix, "xlator.performance.open-behind",
+                                "file");
+        gf_proc_dump_add_section (key_prefix);
+
+        gf_proc_dump_write ("fd", "%p", fd);
+
+        gf_proc_dump_write ("open_frame", "%p", ob_fd->open_frame);
+
+        gf_proc_dump_write ("open_frame.root.unique", "%p",
+			    ob_fd->open_frame->root->unique);
+
+	gf_proc_dump_write ("loc.path", "%s", ob_fd->loc.path);
+
+	gf_proc_dump_write ("loc.ino", "%s", uuid_utoa (ob_fd->loc.gfid));
+
+        gf_proc_dump_write ("flags", "%p", ob_fd->open_frame);
+
+	UNLOCK (&fd->lock);
+
+	return 0;
+}
+
+
+int
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_ob_mt_end + 1);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting failed");
+
+        return ret;
+}
+
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        ob_conf_t  *conf = NULL;
+	int         ret = -1;
+
+        conf = this->private;
+
+        GF_OPTION_RECONF ("use-anonymous-fd", conf->use_anonymous_fd, options,
+			  bool, out);
+
+        GF_OPTION_RECONF ("lazy-open", conf->lazy_open, options, bool, out);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+init (xlator_t *this)
+{
+        ob_conf_t    *conf = NULL;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "FATAL: volume (%s) not configured with exactly one "
+                        "child", this->name);
+                return -1;
+        }
+
+        if (!this->parents)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+
+        conf = GF_CALLOC (1, sizeof (*conf), gf_ob_mt_conf_t);
+        if (!conf)
+                goto err;
+
+        GF_OPTION_INIT ("use-anonymous-fd", conf->use_anonymous_fd, bool, err);
+
+        GF_OPTION_INIT ("lazy-open", conf->lazy_open, bool, err);
+
+        this->private = conf;
+
+	return 0;
+err:
+	if (conf)
+                GF_FREE (conf);
+
+        return -1;
+}
+
+
+void
+fini (xlator_t *this)
+{
+        ob_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        GF_FREE (conf);
+
+	return;
+}
+
+
+struct xlator_fops fops = {
+        .open        = ob_open,
+        .readv       = ob_readv,
+        .writev      = ob_writev,
+	.flush       = ob_flush,
+	.fsync       = ob_fsync,
+	.fstat       = ob_fstat,
+	.ftruncate   = ob_ftruncate,
+	.fsetxattr   = ob_fsetxattr,
+	.fgetxattr   = ob_fgetxattr,
+	.fremovexattr = ob_fremovexattr,
+	.finodelk    = ob_finodelk,
+	.fentrylk    = ob_fentrylk,
+	.fxattrop    = ob_fxattrop,
+	.fsetattr    = ob_fsetattr,
+	.fallocate   = ob_fallocate,
+	.discard     = ob_discard,
+        .zerofill    = ob_zerofill,
+	.unlink      = ob_unlink,
+	.rename      = ob_rename,
+	.lk          = ob_lk,
+};
+
+struct xlator_cbks cbks = {
+        .release  = ob_release,
+};
+
+struct xlator_dumpops dumpops = {
+	.priv    = ob_priv_dump,
+	.fdctx   = ob_fdctx_dump,
+};
+
+
+struct volume_options options[] = {
+        { .key  = {"use-anonymous-fd"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "yes",
+          .description = "For read operations, use anonymous FD when "
+          "original FD is open-behind and not yet opened in the backend.",
+        },
+        { .key  = {"lazy-open"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "yes",
+          .description = "Perform open in the backend only when a necessary "
+          "FOP arrives (e.g writev on the FD, unlink of the file). When option "
+          "is disabled, perform backend open right after unwinding open().",
+        },
+        { .key  = {NULL} }
+
+};
diff --git a/xlators/performance/quick-read/src/Makefile.am b/xlators/performance/quick-read/src/Makefile.am
index db917f897..4906f408a 100644
--- a/xlators/performance/quick-read/src/Makefile.am
+++ b/xlators/performance/quick-read/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = quick-read.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-quick_read_la_LDFLAGS = -module -avoidversion 
+quick_read_la_LDFLAGS = -module -avoid-version 
 
 quick_read_la_SOURCES = quick-read.c
 quick_read_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = quick-read.h quick-read-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/quick-read/src/quick-read-mem-types.h b/xlators/performance/quick-read/src/quick-read-mem-types.h
index b6a65e57c..78547f641 100644
--- a/xlators/performance/quick-read/src/quick-read-mem-types.h
+++ b/xlators/performance/quick-read/src/quick-read-mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __QR_MEM_TYPES_H__
@@ -24,12 +15,13 @@
 
 enum gf_qr_mem_types_ {
         gf_qr_mt_qr_inode_t   = gf_common_mt_end + 1,
+	gf_qr_mt_content_t,
         gf_qr_mt_qr_fd_ctx_t,
-        gf_qr_mt_qr_local_t,
         gf_qr_mt_iovec,
         gf_qr_mt_qr_conf_t,
         gf_qr_mt_qr_priority_t,
         gf_qr_mt_qr_private_t,
+        gf_qr_mt_qr_unlink_ctx_t,
         gf_qr_mt_end
 };
 #endif
diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c
index 14ff58b51..445ea8658 100644
--- a/xlators/performance/quick-read/src/quick-read.c
+++ b/xlators/performance/quick-read/src/quick-read.c
@@ -1,177 +1,112 @@
 /*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "quick-read.h"
 #include "statedump.h"
 
-#define QR_DEFAULT_CACHE_SIZE 134217728
+qr_inode_t *qr_inode_ctx_get (xlator_t *this, inode_t *inode);
+void __qr_inode_prune (qr_inode_table_t *table, qr_inode_t *qr_inode);
 
-struct volume_options options[];
 
-void
-qr_local_free (qr_local_t *local)
+int
+__qr_inode_ctx_set (xlator_t *this, inode_t *inode, qr_inode_t *qr_inode)
 {
-        if (local == NULL) {
-                goto out;
-        }
-
-        if (local->stub != NULL) {
-                call_stub_destroy (local->stub);
-        }
+	uint64_t    value = 0;
+	int         ret = -1;
 
-        if (local->path != NULL) {
-                GF_FREE (local->path);
-        }
+	value = (long) qr_inode;
 
-        GF_FREE (local);
+	ret = __inode_ctx_set (inode, this, &value);
 
-out:
-        return;
+	return ret;
 }
 
 
-int32_t
-qr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags);
-
-
-static void
-qr_loc_wipe (loc_t *loc)
+qr_inode_t *
+__qr_inode_ctx_get (xlator_t *this, inode_t *inode)
 {
-        if (loc == NULL) {
-                goto out;
-        }
-
-        if (loc->path) {
-                GF_FREE ((char *)loc->path);
-                loc->path = NULL;
-        }
+	qr_inode_t *qr_inode = NULL;
+	uint64_t    value = 0;
+	int         ret = -1;
 
-        if (loc->inode) {
-                inode_unref (loc->inode);
-                loc->inode = NULL;
-        }
+	ret = __inode_ctx_get (inode, this, &value);
+	if (ret)
+		return NULL;
 
-        if (loc->parent) {
-                inode_unref (loc->parent);
-                loc->parent = NULL;
-        }
+	qr_inode = (void *) ((long) value);
 
-out:
-        return;
+	return qr_inode;
 }
 
 
-static int32_t
-qr_loc_fill (loc_t *loc, inode_t *inode, char *path)
+qr_inode_t *
+qr_inode_ctx_get (xlator_t *this, inode_t *inode)
 {
-        int32_t  ret    = -1;
+	qr_inode_t *qr_inode = NULL;
 
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quick-read", loc, out, errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quick-read", inode, out, errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quick-read", path, out, errno, EINVAL);
+	LOCK (&inode->lock);
+	{
+		qr_inode = __qr_inode_ctx_get (this, inode);
+	}
+	UNLOCK (&inode->lock);
 
-        loc->inode = inode_ref (inode);
-        uuid_copy (loc->gfid, inode->gfid);
-
-        loc->path = gf_strdup (path);
-        if (!loc->path)
-                goto out;
-
-        ret = 0;
-out:
-        if (ret == -1) {
-                qr_loc_wipe (loc);
-        }
-
-        return ret;
+	return qr_inode;
 }
 
 
-void
-qr_resume_pending_ops (qr_fd_ctx_t *qr_fd_ctx, int32_t op_ret, int32_t op_errno)
+qr_inode_t *
+qr_inode_new (xlator_t *this, inode_t *inode)
 {
-        call_stub_t      *stub        = NULL, *tmp = NULL;
-        struct list_head  waiting_ops = {0, };
-
-        GF_VALIDATE_OR_GOTO ("quick-read", qr_fd_ctx, out);
-
-        INIT_LIST_HEAD (&waiting_ops);
-
-        LOCK (&qr_fd_ctx->lock);
-        {
-                qr_fd_ctx->open_in_transit = 0;
-                list_splice_init (&qr_fd_ctx->waiting_ops,
-                                  &waiting_ops);
-        }
-        UNLOCK (&qr_fd_ctx->lock);
-
-        if (!list_empty (&waiting_ops)) {
-                list_for_each_entry_safe (stub, tmp, &waiting_ops, list) {
-                        list_del_init (&stub->list);
-                        if (op_ret < 0) {
-                                qr_local_t *local = NULL;
-
-                                local = stub->frame->local;
-                                local->op_ret = op_ret;
-                                local->op_errno = op_errno;
-                        }
-
-                        call_resume (stub);
-                }
-        }
-
-out:
-        return;
-}
-
+        qr_inode_t    *qr_inode = NULL;
 
-static void
-qr_fd_ctx_free (qr_fd_ctx_t *qr_fd_ctx)
-{
-        GF_VALIDATE_OR_GOTO ("quick-read", qr_fd_ctx, out);
+        qr_inode = GF_CALLOC (1, sizeof (*qr_inode), gf_qr_mt_qr_inode_t);
+        if (!qr_inode)
+                return NULL;
 
-        GF_ASSERT (list_empty (&qr_fd_ctx->waiting_ops));
+        INIT_LIST_HEAD (&qr_inode->lru);
 
-        GF_FREE (qr_fd_ctx->path);
-        GF_FREE (qr_fd_ctx);
+        qr_inode->priority = 0; /* initial priority */
 
-out:
-        return;
+        return qr_inode;
 }
 
 
-static inline uint32_t
-is_match (const char *path, const char *pattern)
-{
-        int32_t  ret = 0;
-        uint32_t match = 0;
-
-        GF_VALIDATE_OR_GOTO ("quick-read", path, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", pattern, out);
-
-        ret = fnmatch (pattern, path, FNM_NOESCAPE);
-        match = (ret == 0);
+qr_inode_t *
+qr_inode_ctx_get_or_new (xlator_t *this, inode_t *inode)
+{
+	qr_inode_t   *qr_inode = NULL;
+	int           ret = -1;
+	qr_private_t *priv = NULL;
+
+	priv = this->private;
+
+	LOCK (&inode->lock);
+	{
+		qr_inode = __qr_inode_ctx_get (this, inode);
+		if (qr_inode)
+			goto unlock;
+
+		qr_inode = qr_inode_new (this, inode);
+		if (!qr_inode)
+			goto unlock;
+
+		ret = __qr_inode_ctx_set (this, inode, qr_inode);
+		if (ret) {
+			__qr_inode_prune (&priv->table, qr_inode);
+			GF_FREE (qr_inode);
+		}
+	}
+unlock:
+	UNLOCK (&inode->lock);
 
-out:
-        return match;
+	return qr_inode;
 }
 
 
@@ -181,2980 +116,578 @@ qr_get_priority (qr_conf_t *conf, const char *path)
         uint32_t            priority = 0;
         struct qr_priority *curr     = NULL;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", conf, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", path, out);
-
         list_for_each_entry (curr, &conf->priority_list, list) {
-                if (is_match (path, curr->pattern))
+                if (fnmatch (curr->pattern, path, FNM_NOESCAPE) == 0)
                         priority = curr->priority;
         }
 
-out:
         return priority;
 }
 
 
-/* To be called with this-priv->table.lock held */
-qr_inode_t *
-__qr_inode_alloc (xlator_t *this, char *path, inode_t *inode)
-{
-        qr_inode_t    *qr_inode = NULL;
-        qr_private_t  *priv     = NULL;
-        int            priority = 0;
-
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, path, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        qr_inode = GF_CALLOC (1, sizeof (*qr_inode), gf_qr_mt_qr_inode_t);
-        if (qr_inode == NULL) {
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&qr_inode->lru);
-
-        priority = qr_get_priority (&priv->conf, path);
-
-        list_add_tail (&qr_inode->lru, &priv->table.lru[priority]);
-
-        qr_inode->inode = inode;
-        qr_inode->priority = priority;
-out:
-        return qr_inode;
-}
-
-
-/* To be called with qr_inode->table->lock held */
 void
-__qr_inode_free (qr_inode_t *qr_inode)
+__qr_inode_register (qr_inode_table_t *table, qr_inode_t *qr_inode)
 {
-        GF_VALIDATE_OR_GOTO ("quick-read", qr_inode, out);
-
-        if (qr_inode->xattr) {
-                dict_unref (qr_inode->xattr);
-        }
+	if (!qr_inode->data)
+		return;
 
-        list_del (&qr_inode->lru);
+	if (list_empty (&qr_inode->lru))
+		/* first time addition of this qr_inode into table */
+		table->cache_used += qr_inode->size;
+	else
+		list_del_init (&qr_inode->lru);
 
-        GF_FREE (qr_inode);
-out:
-        return;
+	list_add_tail (&qr_inode->lru, &table->lru[qr_inode->priority]);
 }
 
-/* To be called with priv->table.lock held */
+
 void
-__qr_cache_prune (xlator_t *this)
+qr_inode_set_priority (xlator_t *this, inode_t *inode, const char *path)
 {
-        qr_private_t     *priv          = NULL;
-        qr_conf_t        *conf          = NULL;
-        qr_inode_table_t *table         = NULL;
-        qr_inode_t        *curr         = NULL, *next = NULL;
-        int32_t           index         = 0;
-        uint64_t          size_to_prune = 0;
-        uint64_t          size_pruned   = 0;
-
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+	uint32_t          priority = 0;
+	qr_inode_table_t *table = NULL;
+	qr_inode_t       *qr_inode = NULL;
+        qr_private_t     *priv = NULL;
+	qr_conf_t        *conf = NULL;
 
-        table = &priv->table;
-        conf = &priv->conf;
-
-        size_to_prune = table->cache_used - conf->cache_size;
-
-        for (index=0; index < conf->max_pri; index++) {
-                list_for_each_entry_safe (curr, next, &table->lru[index], lru) {
-                        size_pruned += curr->stbuf.ia_size;
-                        inode_ctx_del (curr->inode, this, NULL);
-                        __qr_inode_free (curr);
-                        if (size_pruned >= size_to_prune)
-                                goto done;
-                }
-        }
-
-done:
-        table->cache_used -= size_pruned;
+	qr_inode = qr_inode_ctx_get (this, inode);
+	if (!qr_inode)
+		return;
 
-out:
-        return;
-}
-
-/* To be called with table->lock held */
-inline char
-__qr_need_cache_prune (qr_conf_t *conf, qr_inode_table_t *table)
-{
-        char need_prune = 0;
+	priv = this->private;
+	table = &priv->table;
+	conf = &priv->conf;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", conf, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", table, out);
+	if (path)
+		priority = qr_get_priority (conf, path);
+	else
+		/* retain existing priority, just bump LRU */
+		priority = qr_inode->priority;
 
-        need_prune = (table->cache_used > conf->cache_size);
+	LOCK (&table->lock);
+	{
+		qr_inode->priority = priority;
 
-out:
-        return need_prune;
+		__qr_inode_register (table, qr_inode);
+	}
+	UNLOCK (&table->lock);
 }
 
 
-int32_t
-qr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *dict, struct iatt *postparent)
+/* To be called with priv->table.lock held */
+void
+__qr_inode_prune (qr_inode_table_t *table, qr_inode_t *qr_inode)
 {
-        data_t           *content  = NULL;
-        qr_inode_t       *qr_inode = NULL;
-        uint64_t          value    = 0;
-        int               ret      = -1;
-        qr_conf_t        *conf     = NULL;
-        qr_inode_table_t *table    = NULL;
-        qr_private_t     *priv     = NULL;
-        qr_local_t       *local    = NULL;
-
-        GF_ASSERT (frame);
-
-        if ((op_ret == -1) || (dict == NULL)) {
-                goto out;
-        }
-
-        if ((this == NULL) || (this->private == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "quick-read configuration is not found");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        priv = this->private;
-        conf = &priv->conf;
-        table = &priv->table;
-
-        local = frame->local;
-
-        if (buf->ia_size > conf->max_file_size) {
-                goto out;
-        }
+	GF_FREE (qr_inode->data);
+	qr_inode->data = NULL;
 
-        if (IA_ISDIR (buf->ia_type)) {
-                goto out;
-        }
+	if (!list_empty (&qr_inode->lru)) {
+		table->cache_used -= qr_inode->size;
+		qr_inode->size = 0;
 
-        if (inode == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "lookup returned a NULL inode");
-                goto out;
-        }
-
-        content = dict_get (dict, GF_CONTENT_KEY);
-        if (content == NULL) {
-                goto out;
-        }
-
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (inode, this, &value);
-                if (ret == -1) {
-                        qr_inode = __qr_inode_alloc (this, local->path, inode);
-                        if (qr_inode == NULL) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto unlock;
-                        }
-
-                        ret = inode_ctx_put (inode, this,
-                                             (uint64_t)(long)qr_inode);
-                        if (ret == -1) {
-                                __qr_inode_free (qr_inode);
-                                qr_inode = NULL;
-                                op_ret = -1;
-                                op_errno = EINVAL;
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot set quick-read context in "
-                                        "inode (gfid:%s)",
-                                        uuid_utoa (inode->gfid));
-                                goto unlock;
-                        }
-                } else {
-                        qr_inode = (qr_inode_t *)(long)value;
-                        if (qr_inode == NULL) {
-                                op_ret = -1;
-                                op_errno = EINVAL;
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot find quick-read context in "
-                                        "inode (gfid:%s)",
-                                        uuid_utoa (inode->gfid));
-                                goto unlock;
-                        }
-                }
+		list_del_init (&qr_inode->lru);
+	}
 
-                if (qr_inode->xattr) {
-                        dict_unref (qr_inode->xattr);
-                        qr_inode->xattr = NULL;
-                        table->cache_used -= qr_inode->stbuf.ia_size;
-                }
-
-                qr_inode->xattr = dict_ref (dict);
-                qr_inode->stbuf = *buf;
-                table->cache_used += buf->ia_size;
-
-                gettimeofday (&qr_inode->tv, NULL);
-                if (__qr_need_cache_prune (conf, table)) {
-                        __qr_cache_prune (this);
-                }
-        }
-unlock:
-        UNLOCK (&table->lock);
-
-out:
-        /*
-         * FIXME: content size in dict can be greater than the size application
-         * requested for. Applications need to be careful till this is fixed.
-         */
-        QR_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf, dict,
-                         postparent);
-
-        return 0;
+	memset (&qr_inode->buf, 0, sizeof (qr_inode->buf));
 }
 
 
-int32_t
-qr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+void
+qr_inode_prune (xlator_t *this, inode_t *inode)
 {
-        qr_conf_t        *conf           = NULL;
-        dict_t           *new_req_dict   = NULL;
-        int32_t           op_ret         = -1, op_errno = EINVAL;
-        data_t           *content        = NULL;
-        uint64_t          requested_size = 0, size = 0, value = 0;
-        char              cached         = 0;
-        qr_inode_t       *qr_inode       = NULL;
-        qr_private_t     *priv           = NULL;
-        qr_inode_table_t *table          = NULL;
-        qr_local_t       *local          = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (frame->this->name, priv, unwind);
-
-        conf = &priv->conf;
-        if (conf == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        table = &priv->table;
-        local = GF_CALLOC (1, sizeof (*local), gf_qr_mt_qr_local_t);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, unwind, op_errno,
-                                        ENOMEM);
-
-        frame->local = local;
-
-        local->path = gf_strdup (loc->path);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, unwind, op_errno,
-                                        ENOMEM);
-        LOCK (&table->lock);
-        {
-                op_ret = inode_ctx_get (loc->inode, this, &value);
-                if (op_ret == 0) {
-                        qr_inode = (qr_inode_t *)(long)value;
-                        if (qr_inode != NULL) {
-                                if (qr_inode->xattr) {
-                                        cached = 1;
-                                }
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-        if ((xattr_req == NULL) && (conf->max_file_size > 0)) {
-                new_req_dict = xattr_req = dict_new ();
-                if (xattr_req == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-        }
-
-        if (!cached) {
-                if (xattr_req) {
-                        content = dict_get (xattr_req, GF_CONTENT_KEY);
-                        if (content) {
-                                requested_size = data_to_uint64 (content);
-                        }
-                }
-
-                if ((conf->max_file_size > 0)
-                    && (conf->max_file_size != requested_size)) {
-                        size = (conf->max_file_size > requested_size) ?
-                                conf->max_file_size : requested_size;
-
-                        op_ret = dict_set (xattr_req, GF_CONTENT_KEY,
-                                           data_from_uint64 (size));
-                        if (op_ret < 0) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot set key in request dict to "
-                                        "request file "
-                                        "content during lookup cbk");
-                                goto unwind;
-                        }
-                }
-        }
-
-        STACK_WIND (frame, qr_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
-
-        if (new_req_dict) {
-                dict_unref (new_req_dict);
-        }
-
-        return 0;
+        qr_private_t     *priv          = NULL;
+        qr_inode_table_t *table         = NULL;
+	qr_inode_t       *qr_inode      = NULL;
 
-unwind:
-        QR_STACK_UNWIND (lookup, frame, op_ret, op_errno, NULL, NULL, NULL,
-                         NULL);
+	qr_inode = qr_inode_ctx_get (this, inode);
+	if (!qr_inode)
+		return;
 
-        if (new_req_dict) {
-                dict_unref (new_req_dict);
-        }
+	priv = this->private;
+	table = &priv->table;
 
-        return 0;
+	LOCK (&table->lock);
+	{
+		__qr_inode_prune (table, qr_inode);
+	}
+	UNLOCK (&table->lock);
 }
 
 
-int32_t
-qr_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, fd_t *fd)
+/* To be called with priv->table.lock held */
+void
+__qr_cache_prune (qr_inode_table_t *table, qr_conf_t *conf)
 {
-        uint64_t          value     = 0;
-        int32_t           ret       = -1;
-        qr_local_t       *local     = NULL;
-        qr_inode_t       *qr_inode  = NULL;
-        qr_fd_ctx_t      *qr_fd_ctx = NULL;
-        call_stub_t      *stub      = NULL, *tmp = NULL;
-        char              is_open   = 0;
-        qr_private_t     *priv      = NULL;
-        qr_inode_table_t *table     = NULL;
-        struct list_head  waiting_ops;
-
-        GF_ASSERT (frame);
-
-        priv = this->private;
-        table = &priv->table;
-
-        local = frame->local;
-        if (local != NULL) {
-                is_open = local->is_open;
-        }
-
-        INIT_LIST_HEAD (&waiting_ops);
+        qr_inode_t        *curr = NULL;
+	qr_inode_t        *next = NULL;
+        int                index = 0;
+	size_t             size_pruned = 0;
 
-        ret = fd_ctx_get (fd, this, &value);
-        if ((ret == -1) && (op_ret != -1)) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot find quick-read context in fd (%p) opened on "
-                        "inode (gfid: %s)", fd, uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        if (value) {
-                qr_fd_ctx = (qr_fd_ctx_t *) (long)value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        qr_fd_ctx->open_in_transit = 0;
-
-                        if (op_ret == 0) {
-                                qr_fd_ctx->opened = 1;
-                        }
-                        list_splice_init (&qr_fd_ctx->waiting_ops,
-                                          &waiting_ops);
-                }
-                UNLOCK (&qr_fd_ctx->lock);
-
-                if (local && local->is_open
-                    && ((local->open_flags & O_TRUNC) == O_TRUNC)) {
-                        LOCK (&table->lock);
-                        {
-                                ret = inode_ctx_del (fd->inode, this, &value);
-                                if (ret == 0) {
-                                        qr_inode = (qr_inode_t *)(long) value;
-
-                                        if (qr_inode != NULL) {
-                                                __qr_inode_free (qr_inode);
-                                        }
-                                }
-                        }
-                        UNLOCK (&table->lock);
-                }
+        for (index = 0; index < conf->max_pri; index++) {
+                list_for_each_entry_safe (curr, next, &table->lru[index], lru) {
 
-                if (!list_empty (&waiting_ops)) {
-                        list_for_each_entry_safe (stub, tmp, &waiting_ops,
-                                                  list) {
-                                list_del_init (&stub->list);
-                                if (op_ret < 0) {
-                                        qr_local_t *local = NULL;
+                        size_pruned += curr->size;
 
-                                        local = stub->frame->local;
-                                        local->op_ret = op_ret;
-                                        local->op_errno = op_errno;
-                                }
+                        __qr_inode_prune (table, curr);
 
-                                call_resume (stub);
-                        }
+                        if (table->cache_used < conf->cache_size)
+				return;
                 }
         }
-out:
-        if (is_open) {
-                QR_STACK_UNWIND (open, frame, op_ret, op_errno, fd);
-        } else {
-                STACK_DESTROY (frame->root);
-        }
 
-        return 0;
+        return;
 }
 
 
-int32_t
-qr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+void
+qr_cache_prune (xlator_t *this)
 {
-        qr_inode_t       *qr_inode       = NULL;
-        int32_t           ret            = -1;
-        uint64_t          filep          = 0;
-        char              content_cached = 0;
-        qr_fd_ctx_t      *qr_fd_ctx      = NULL, *tmp_fd_ctx = NULL;
-        int32_t           op_ret         = -1, op_errno = EINVAL;
-        qr_local_t       *local          = NULL;
-        qr_private_t     *priv           = NULL;
-        qr_inode_table_t *table          = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this->private, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+        qr_private_t      *priv = NULL;
+        qr_conf_t         *conf = NULL;
+        qr_inode_table_t  *table = NULL;
 
         priv = this->private;
         table = &priv->table;
+        conf = &priv->conf;
 
-        tmp_fd_ctx = qr_fd_ctx = GF_CALLOC (1, sizeof (*qr_fd_ctx),
-                                            gf_qr_mt_qr_fd_ctx_t);
-        if (qr_fd_ctx == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        LOCK_INIT (&qr_fd_ctx->lock);
-        INIT_LIST_HEAD (&qr_fd_ctx->waiting_ops);
-
-        qr_fd_ctx->path = gf_strdup (loc->path);
-        if (qr_fd_ctx->path == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        qr_fd_ctx->flags = flags;
-        qr_fd_ctx->wbflags = wbflags;
-
-        ret = fd_ctx_set (fd, this, (uint64_t)(long)qr_fd_ctx);
-        if (ret == -1) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot set quick-read context in "
-                        "fd (%p) opened on inode (gfid:%s)", fd,
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-        tmp_fd_ctx = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_qr_mt_qr_local_t);
-        if (local == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        local->is_open = 1;
-        local->open_flags = flags;
-        frame->local = local;
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (fd->inode, this, &filep);
-                if (ret == 0) {
-                        qr_inode = (qr_inode_t *)(long) filep;
-                        if (qr_inode) {
-                                if (qr_inode->xattr) {
-                                        content_cached = 1;
-                                }
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-        if (content_cached && (flags & O_DIRECTORY)) {
-                op_ret = -1;
-                op_errno = ENOTDIR;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open with O_DIRECTORY flag received on non-directory");
-                goto unwind;
-        }
-
-        if (!content_cached || ((flags & O_ACCMODE) == O_WRONLY)
-            || ((flags & O_TRUNC) == O_TRUNC)
-            || ((flags & O_DIRECT) == O_DIRECT)
-            || ((wbflags & GF_OPEN_NOWB) != 0)) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        /*
-                         * we really need not set this flag, since open is
-                         * not yet unwound.
-                         */
-
-                        qr_fd_ctx->open_in_transit = 1;
-                        if (((flags & O_DIRECT) == O_DIRECT)
-                            || ((wbflags & GF_OPEN_NOWB)) != 0) {
-                                qr_fd_ctx->disabled = 1;
-                        }
-                }
-                UNLOCK (&qr_fd_ctx->lock);
-                goto wind;
-        } else {
-                op_ret = 0;
-                op_errno = 0;
-                goto unwind;
-        }
-
-unwind:
-        if (tmp_fd_ctx != NULL) {
-                qr_fd_ctx_free (tmp_fd_ctx);
-        }
-
-        QR_STACK_UNWIND (open, frame, op_ret, op_errno, fd);
-        return 0;
-
-wind:
-        STACK_WIND (frame, qr_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
-        return 0;
-}
-
-
-static inline time_t
-qr_time_elapsed (struct timeval *now, struct timeval *then)
-{
-        time_t time_elapsed = 0;
-
-        GF_VALIDATE_OR_GOTO ("quick-read", now, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", then, out);
-
-        time_elapsed = now->tv_sec - then->tv_sec;
-
-out:
-        return time_elapsed;
+	LOCK (&table->lock);
+	{
+		if (table->cache_used > conf->cache_size)
+			__qr_cache_prune (table, conf);
+	}
+	UNLOCK (&table->lock);
 }
 
 
-static inline char
-qr_need_validation (qr_conf_t *conf, qr_inode_t *qr_inode)
+void *
+qr_content_extract (dict_t *xdata)
 {
-        struct timeval now             = {0, };
-        char           need_validation = 0;
+	data_t  *data = NULL;
+	void    *content = NULL;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", conf, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", qr_inode, out);
+	data = dict_get (xdata, GF_CONTENT_KEY);
+	if (!data)
+		return NULL;
 
-        gettimeofday (&now, NULL);
+	content = GF_CALLOC (1, data->len, gf_qr_mt_content_t);
+	if (!content)
+		return NULL;
 
-        if (qr_time_elapsed (&now, &qr_inode->tv) >= conf->cache_timeout)
-                need_validation = 1;
+	memcpy (content, data->data, data->len);
 
-out:
-        return need_validation;
+	return content;
 }
 
 
-static int32_t
-qr_validate_cache_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *buf)
+void
+qr_content_update (xlator_t *this, qr_inode_t *qr_inode, void *data,
+		   struct iatt *buf)
 {
-        qr_inode_t       *qr_inode  = NULL;
-        qr_local_t       *local     = NULL;
-        uint64_t          value     = 0;
-        int32_t           ret       = 0;
-        qr_private_t     *priv      = NULL;
-        qr_inode_table_t *table     = NULL;
-        call_stub_t      *stub      = NULL;
-
-        GF_ASSERT (frame);
-        if (this == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "xlator object (this) is NULL");
-                goto unwind;
-        }
-
-        local = frame->local;
-        if ((local == NULL) || ((local->fd) == NULL)) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (local == NULL) ? "local is NULL"
-                        : "fd is not stored in local");
-                goto unwind;
-        }
-
-        local->just_validated = 1;
-
-        if (op_ret == -1) {
-                goto unwind;
-        }
+        qr_private_t      *priv = NULL;
+        qr_inode_table_t  *table = NULL;
 
         priv = this->private;
         table = &priv->table;
 
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (local->fd->inode, this, &value);
-                if (ret == 0) {
-                        qr_inode = (qr_inode_t *)(long) value;
-                }
+	LOCK (&table->lock);
+	{
+		__qr_inode_prune (table, qr_inode);
 
-                if (qr_inode != NULL) {
-                        gettimeofday (&qr_inode->tv, NULL);
+		qr_inode->data = data;
+		qr_inode->size = buf->ia_size;
 
-                        if ((qr_inode->stbuf.ia_mtime != buf->ia_mtime)
-                            || (qr_inode->stbuf.ia_mtime_nsec
-                                != buf->ia_mtime_nsec)) {
-                                inode_ctx_del (local->fd->inode, this, NULL);
-                                __qr_inode_free (qr_inode);
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
+		qr_inode->ia_mtime = buf->ia_mtime;
+		qr_inode->ia_mtime_nsec = buf->ia_mtime_nsec;
 
-        stub = local->stub;
-        local->stub = NULL;
+		qr_inode->buf = *buf;
 
-        call_resume (stub);
+		gettimeofday (&qr_inode->last_refresh, NULL);
 
-        return 0;
+		__qr_inode_register (table, qr_inode);
+	}
+	UNLOCK (&table->lock);
 
-unwind:
-        /* this is actually unwind of readv */
-        QR_STACK_UNWIND (readv, frame, op_ret, op_errno, NULL, -1, NULL, NULL);
-        return 0;
+	qr_cache_prune (this);
 }
 
 
-int32_t
-qr_validate_cache_helper (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        qr_local_t *local  = NULL;
-        int32_t     op_ret = -1, op_errno = -1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, out);
-
-        local = frame->local;
-        if (local == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        } else {
-                op_ret = local->op_ret;
-                op_errno = local->op_errno;
-        }
-
-out:
-        if (op_ret == -1) {
-                qr_validate_cache_cbk (frame, NULL, this, op_ret, op_errno,
-                                       NULL);
-        } else {
-                STACK_WIND (frame, qr_validate_cache_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fstat, fd);
-        }
-
-        return 0;
-}
-
-
-int
-qr_validate_cache (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   call_stub_t *stub)
-{
-        int           ret           = -1;
-        int           flags         = 0;
-        uint64_t      value         = 0;
-        loc_t         loc           = {0, };
-        char         *path          = NULL;
-        qr_local_t   *local         = NULL;
-        qr_fd_ctx_t  *qr_fd_ctx     = NULL;
-        call_stub_t  *validate_stub = NULL;
-        char          need_open     = 0, can_wind = 0, validate_cbk_called = 0;
-        call_frame_t *open_frame    = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, stub, out);
-
-        if (frame->local == NULL) {
-                local = GF_CALLOC (1, sizeof (*local), gf_qr_mt_qr_local_t);
-                if (local == NULL) {
-                        goto out;
-                }
-        } else {
-                local = frame->local;
-        }
-
-        local->fd = fd;
-        local->stub = stub;
-        frame->local = local;
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                validate_stub = fop_fstat_stub (frame,
-                                                                qr_validate_cache_helper,
-                                                                fd);
-                                if (validate_stub == NULL) {
-                                        ret = -1;
-                                        if (need_open) {
-                                                qr_fd_ctx->open_in_transit = 0;
-                                        }
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&validate_stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-
-                if (ret == -1) {
-                        goto out;
-                }
-        } else {
-                can_wind = 1;
-        }
-
-        if (need_open) {
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        validate_cbk_called = 1;
-                        goto out;
-                }
-
-                ret = qr_loc_fill (&loc, fd->inode, path);
-                if (ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        validate_cbk_called = 1;
-                        STACK_DESTROY (open_frame->root);
-                        goto out;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open,
-                            &loc, flags, fd, qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_validate_cache_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fstat, fd);
-        }
-
-        ret = 0;
-out:
-        if ((ret < 0) && !validate_cbk_called) {
-                if (frame->local == NULL) {
-                        call_stub_destroy (stub);
-                }
-
-                qr_validate_cache_cbk (frame, NULL, this, -1, errno, NULL);
-        }
-        return ret;
-}
-
-
-int32_t
-qr_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iovec *vector, int32_t count,
-              struct iatt *stbuf, struct iobref *iobref)
+gf_boolean_t
+qr_size_fits (qr_conf_t *conf, struct iatt *buf)
 {
-        GF_ASSERT (frame);
-
-        QR_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                         stbuf, iobref);
-        return 0;
+	return (buf->ia_size <= conf->max_file_size);
 }
 
 
-int32_t
-qr_readv_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t offset, uint32_t flags)
+gf_boolean_t
+qr_mtime_equal (qr_inode_t *qr_inode, struct iatt *buf)
 {
-        qr_local_t  *local    = NULL;
-        int32_t      op_errno = EINVAL, ret = 0;
-        uint64_t     value    = 0;
-        qr_fd_ctx_t *fdctx   = NULL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding read call",
-                        fdctx ? fdctx->path : NULL, strerror (errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_readv_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readv, fd, size, offset, flags);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
-        return 0;
+	return (qr_inode->ia_mtime == buf->ia_mtime &&
+		qr_inode->ia_mtime_nsec == buf->ia_mtime_nsec);
 }
 
 
-int32_t
-qr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t read_flags)
+void
+__qr_content_refresh (xlator_t *this, qr_inode_t *qr_inode, struct iatt *buf)
 {
-        qr_inode_t        *qr_inode       = NULL;
-        int32_t            ret            = -1, op_ret = -1, op_errno = -1;
-        uint64_t           value          = 0;
-        int                count          = -1, flags = 0, i = 0;
-        char               content_cached = 0, need_validation = 0;
-        char               need_open      = 0, can_wind = 0, need_unwind = 0;
-        struct iobuf      *iobuf          = NULL;
-        struct iobref     *iobref         = NULL;
-        struct iatt        stbuf          = {0, };
-        data_t            *content        = NULL;
-        qr_fd_ctx_t       *qr_fd_ctx      = NULL;
-        call_stub_t       *stub           = NULL;
-        loc_t              loc            = {0, };
-        qr_conf_t         *conf           = NULL;
-        struct iovec      *vector         = NULL;
-        char              *path           = NULL;
-        off_t              start          = 0, end = 0;
-        size_t             len            = 0;
-        struct iobuf_pool *iobuf_pool     = NULL;
-        qr_local_t        *local          = NULL;
-        char               just_validated = 0;
-        qr_private_t      *priv           = NULL;
-        qr_inode_table_t  *table          = NULL;
-        call_frame_t      *open_frame     = NULL;
-
-        op_ret = 0;
+        qr_private_t      *priv = NULL;
+        qr_inode_table_t  *table = NULL;
+	qr_conf_t         *conf = NULL;
 
         priv = this->private;
-        conf = &priv->conf;
         table = &priv->table;
+	conf = &priv->conf;
 
-        local = frame->local;
-
-        if (local != NULL) {
-                just_validated = local->just_validated;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-                if (qr_fd_ctx != NULL) {
-                        if (qr_fd_ctx->disabled) {
-                                goto out;
-                        }
-                }
-        }
-
-        iobuf_pool = this->ctx->iobuf_pool;
-
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (fd->inode, this, &value);
-                if (ret)
-                        goto unlock;
-
-                qr_inode = (qr_inode_t *)(long)value;
-                if (!qr_inode || !qr_inode->xattr)
-                        goto unlock;
-
-                if (!just_validated
-                    && qr_need_validation (conf, qr_inode)) {
-                        need_validation = 1;
-                        goto unlock;
-                }
-
-                content = dict_get (qr_inode->xattr, GF_CONTENT_KEY);
-
-                stbuf = qr_inode->stbuf;
-                content_cached = 1;
-                list_move_tail (&qr_inode->lru,
-                                &table->lru[qr_inode->priority]);
-
-                if (offset > content->len) {
-                        op_ret = 0;
-                        end = content->len;
-                } else {
-                        if ((offset + size) > content->len) {
-                                op_ret = content->len - offset;
-                                end = content->len;
-                        } else {
-                                op_ret = size;
-                                end =  offset + size;
-                        }
-                }
-
-                count = (op_ret  / iobuf_pool->default_page_size);
-                if ((op_ret % iobuf_pool->default_page_size) != 0) {
-                        count++;
-                }
-
-                if (count == 0) {
-                        op_ret = 0;
-                        goto unlock;
-                }
-
-                vector = GF_CALLOC (count, sizeof (*vector), gf_qr_mt_iovec);
-                if (vector == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        need_unwind = 1;
-                        goto unlock;
-                }
-
-                iobref = iobref_new ();
-                if (iobref == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        need_unwind = 1;
-                        goto unlock;
-                }
-
-                for (i = 0; i < count; i++) {
-                        iobuf = iobuf_get (iobuf_pool);
-                        if (iobuf == NULL) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                need_unwind = 1;
-                                goto unlock;
-                        }
+	if (qr_size_fits (conf, buf) && qr_mtime_equal (qr_inode, buf)) {
+		qr_inode->buf = *buf;
 
-                        start = offset + (iobuf_pool->default_page_size * i);
+		gettimeofday (&qr_inode->last_refresh, NULL);
 
-                        if (start > end) {
-                                len = 0;
-                        } else {
-                                len = (iobuf_pool->default_page_size >
-                                       ((end - start)) ? (end - start) :
-                                       iobuf_pool->default_page_size);
+		__qr_inode_register (table, qr_inode);
+	} else {
+		__qr_inode_prune (table, qr_inode);
+	}
 
-                                memcpy (iobuf->ptr, content->data + start, len);
-                        }
-
-                        iobref_add (iobref, iobuf);
-                        iobuf_unref (iobuf);
-
-                        vector[i].iov_base = iobuf->ptr;
-                        vector[i].iov_len = len;
-                }
-        }
-unlock:
-        UNLOCK (&table->lock);
-
-out:
-        if (content_cached || need_unwind) {
-                QR_STACK_UNWIND (readv, frame, op_ret, op_errno, vector,
-                                 count, &stbuf, iobref);
-
-        } else if (need_validation) {
-                stub = fop_readv_stub (frame, qr_readv, fd, size, offset,
-                                       read_flags);
-                if (stub == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-                qr_validate_cache (frame, this, fd, stub);
-        } else {
-                if (qr_fd_ctx) {
-                        LOCK (&qr_fd_ctx->lock);
-                        {
-                                path = qr_fd_ctx->path;
-                                flags = qr_fd_ctx->flags;
-
-                                if (!(qr_fd_ctx->opened
-                                      || qr_fd_ctx->open_in_transit)) {
-                                        need_open = 1;
-                                        qr_fd_ctx->open_in_transit = 1;
-                                }
-
-                                if (qr_fd_ctx->opened) {
-                                        can_wind = 1;
-                                } else {
-                                        if (frame->local == NULL) {
-                                                frame->local = GF_CALLOC (1,
-                                                                          sizeof (qr_local_t),
-                                                                          gf_qr_mt_qr_local_t);
-                                                if (frame->local == NULL) {
-                                                        op_ret = -1;
-                                                        op_errno = ENOMEM;
-                                                        need_unwind = 1;
-                                                        qr_fd_ctx->open_in_transit = 0;
-                                                        goto fdctx_unlock;
-                                                }
-                                        }
-
-                                        stub = fop_readv_stub (frame,
-                                                               qr_readv_helper,
-                                                               fd, size,
-                                                               offset,
-                                                               read_flags);
-                                        if (stub == NULL) {
-                                                op_ret = -1;
-                                                op_errno = ENOMEM;
-                                                need_unwind = 1;
-                                                qr_fd_ctx->open_in_transit = 0;
-                                                goto fdctx_unlock;
-                                        }
-
-                                        list_add_tail (&stub->list,
-                                                       &qr_fd_ctx->waiting_ops);
-                                }
-                        }
-                fdctx_unlock:
-                        UNLOCK (&qr_fd_ctx->lock);
-
-                        if (op_ret == -1) {
-                                need_unwind = 1;
-                                goto out;
-                        }
-                } else {
-                        can_wind = 1;
-                }
-
-                if (need_open) {
-                        op_ret = qr_loc_fill (&loc, fd->inode, path);
-                        if (op_ret == -1) {
-                                qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                                goto ret;
-                        }
-
-                        open_frame = create_frame (this, this->ctx->pool);
-                        if (open_frame == NULL) {
-                                qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                                qr_loc_wipe (&loc);
-                                goto ret;
-                        }
-
-                        STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->open,
-                                    &loc, flags, fd, qr_fd_ctx->wbflags);
-
-                        qr_loc_wipe (&loc);
-                } else if (can_wind) {
-                        STACK_WIND (frame, qr_readv_cbk, FIRST_CHILD (this),
-                                    FIRST_CHILD (this)->fops->readv, fd, size,
-                                    offset, read_flags);
-                }
-        }
-
-ret:
-        if (vector) {
-                GF_FREE (vector);
-        }
-
-        if (iobref) {
-                iobref_unref (iobref);
-        }
-
-        return 0;
-}
-
-
-int32_t
-qr_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf);
-        return 0;
-}
-
-
-int32_t
-qr_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  struct iovec *vector, int32_t count, off_t off,
-                  uint32_t flags, struct iobref *iobref)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding write call",
-                        fdctx ? fdctx->path : NULL, strerror (errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_writev_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev, fd, vector, count, off,
-                    flags, iobref);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
-        return 0;
+	return;
 }
 
 
-int32_t
-qr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t wr_flags, struct iobref *iobref)
+void
+qr_content_refresh (xlator_t *this, qr_inode_t *qr_inode, struct iatt *buf)
 {
-        uint64_t          value      = 0;
-        int               flags      = 0;
-        call_stub_t      *stub       = NULL;
-        char             *path       = NULL;
-        loc_t             loc        = {0, };
-        qr_inode_t       *qr_inode   = NULL;
-        qr_fd_ctx_t      *qr_fd_ctx  = NULL;
-        int32_t           op_ret     = -1, op_errno = -1, ret = -1;
-        char              can_wind   = 0, need_unwind = 0, need_open = 0;
-        qr_private_t     *priv       = NULL;
-        qr_inode_table_t *table      = NULL;
-        call_frame_t     *open_frame = NULL;
+        qr_private_t      *priv = NULL;
+        qr_inode_table_t  *table = NULL;
 
         priv = this->private;
         table = &priv->table;
 
-        ret = fd_ctx_get (fd, this, &value);
-
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (fd->inode, this, &value);
-                if (ret == 0) {
-                        qr_inode = (qr_inode_t *)(long)value;
-                        if (qr_inode != NULL) {
-                                inode_ctx_del (fd->inode, this, NULL);
-                                __qr_inode_free (qr_inode);
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = GF_CALLOC (1,
-                                                          sizeof (qr_local_t),
-                                                          gf_qr_mt_qr_local_t);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_writev_stub (frame, qr_writev_helper,
-                                                        fd, vector, count, off,
-                                                        wr_flags, iobref);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-        if (need_unwind) {
-                QR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_writev_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->writev, fd, vector, count,
-                            off, wr_flags, iobref);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *buf)
-{
-        QR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf);
-        return 0;
-}
-
-
-int32_t
-qr_fstat_helper (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fstat call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_fstat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fstat, fd);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-qr_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        uint64_t      value      = 0;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        call_stub_t  *stub       = NULL;
-        loc_t         loc        = {0, };
-        char         *path       = NULL;
-        int           flags      = 0;
-        call_frame_t *open_frame = NULL;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto unwind;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = GF_CALLOC (1,
-                                                          sizeof (qr_local_t),
-                                                          gf_qr_mt_qr_local_t);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fstat_stub (frame, qr_fstat_helper,
-                                                       fd);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-unwind:
-        if (need_unwind) {
-                QR_STACK_UNWIND (fstat, frame, op_ret, op_errno, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fstat_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fstat, fd);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
+	LOCK (&table->lock);
+	{
+		__qr_content_refresh (this, qr_inode, buf);
+	}
+	UNLOCK (&table->lock);
 }
 
 
-int32_t
-qr_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *preop, struct iatt *postop)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, preop, postop);
-        return 0;
-}
-
-
-int32_t
-qr_fsetattr_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fsetattr "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_fsetattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr, fd, stbuf,
-                    valid);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-qr_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid)
-{
-        uint64_t      value      = 0;
-        int           flags      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL" :
-                        "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = GF_CALLOC (1,
-                                                          sizeof (qr_local_t),
-                                                          gf_qr_mt_qr_local_t);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fsetattr_stub (frame,
-                                                          qr_fsetattr_helper,
-                                                          fd, stbuf, valid);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, NULL, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fsetattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsetattr, fd, stbuf,
-                            valid);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int32_t
-qr_fsetxattr_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     dict_t *dict, int32_t flags)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fsetxattr "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_fsetxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetxattr, fd, dict, flags);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (fsetxattr, frame, -1, op_errno);
-        return 0;
-}
-
-
-int32_t
-qr_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags)
-{
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        int           open_flags = 0;
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) "
-                        "is NULL" : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        open_flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = GF_CALLOC (1,
-                                                          sizeof (qr_local_t),
-                                                          gf_qr_mt_qr_local_t);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fsetxattr_stub (frame,
-                                                           qr_fsetxattr_helper,
-                                                           fd, dict, flags);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fsetxattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsetxattr, fd, dict,
-                            flags);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, open_flags,
-                            fd, qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict);
-        return 0;
-}
-
-
-int32_t
-qr_fgetxattr_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     const char *name)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fgetxattr "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_fgetxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fgetxattr, fd, name);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-qr_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name)
+gf_boolean_t
+__qr_cache_is_fresh (xlator_t *this, qr_inode_t *qr_inode)
 {
-        int           flags      = 0;
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        /*
-         * FIXME: Can quick-read use the extended attributes stored in the
-         * cache? this needs to be discussed.
-         */
+	qr_conf_t        *conf = NULL;
+	qr_private_t     *priv = NULL;
+	struct timeval    now;
+	struct timeval    diff;
 
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL" :
-                        "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
+	priv = this->private;
+	conf = &priv->conf;
 
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
+	gettimeofday (&now, NULL);
 
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
+	timersub (&now, &qr_inode->last_refresh, &diff);
 
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
+	if (diff.tv_sec >= conf->cache_timeout)
+		return _gf_false;
 
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = GF_CALLOC (1,
-                                                          sizeof (qr_local_t),
-                                                          gf_qr_mt_qr_local_t);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fgetxattr_stub (frame,
-                                                           qr_fgetxattr_helper,
-                                                           fd, name);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (open, frame, op_ret, op_errno, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fgetxattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fgetxattr, fd, name);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (flush, frame, op_ret, op_errno);
-        return 0;
+	return _gf_true;
 }
 
 
-int32_t
-qr_flush_helper (call_frame_t *frame, xlator_t *this, fd_t *fd)
+int
+qr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode_ret,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding flush call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_flush_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->flush, fd);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (flush, frame, -1, op_errno);
-        return 0;
-}
+        void             *content  = NULL;
+        qr_inode_t       *qr_inode = NULL;
+	inode_t          *inode    = NULL;
 
+	inode = frame->local;
+	frame->local = NULL;
 
-int32_t
-qr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        uint64_t     value     = 0;
-        call_stub_t *stub      = NULL;
-        qr_fd_ctx_t *qr_fd_ctx = NULL;
-        int32_t      ret       = -1, op_ret = -1, op_errno = EINVAL;
-        char         can_wind  = 0, need_unwind = 0;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
+        if (op_ret == -1) {
+		qr_inode_prune (this, inode);
                 goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else if (qr_fd_ctx->open_in_transit) {
-                                frame->local = GF_CALLOC (1,
-                                                          sizeof (qr_local_t),
-                                                          gf_qr_mt_qr_local_t);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_flush_stub (frame, qr_flush_helper,
-                                                       fd);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        } else {
-                                op_ret = 0;
-                                need_unwind = 1;
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
+	}
+
+	if (dict_get (xdata, "sh-failed")) {
+		qr_inode_prune (this, inode);
+		goto out;
+	}
+
+	content = qr_content_extract (xdata);
+
+	if (content) {
+		/* new content came along, always replace old content */
+		qr_inode = qr_inode_ctx_get_or_new (this, inode);
+		if (!qr_inode)
+			/* no harm done */
+			goto out;
+
+		qr_content_update (this, qr_inode, content, buf);
+	} else {
+		/* purge old content if necessary */
+		qr_inode = qr_inode_ctx_get (this, inode);
+		if (!qr_inode)
+			/* usual path for large files */
+			goto out;
+
+		qr_content_refresh (this, qr_inode, buf);
+	}
 out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (flush, frame, op_ret, op_errno);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_flush_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->flush, fd);
-        }
-
-        return 0;
-}
+	if (inode)
+		inode_unref (inode);
 
-
-int32_t
-qr_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (fentrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode_ret,
+			     buf, xdata, postparent);
         return 0;
 }
 
 
-int32_t
-qr_fentrylk_helper (call_frame_t *frame, xlator_t *this, const char *volume,
-                    fd_t *fd, const char *basename, entrylk_cmd cmd,
-                    entrylk_type type)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fentrylk "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND(frame, qr_fentrylk_cbk, FIRST_CHILD(this),
-                   FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename,
-                   cmd, type);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (fentrylk, frame, -1, op_errno);
-        return 0;
-}
-
-
-int32_t
-qr_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-             const char *basename, entrylk_cmd cmd, entrylk_type type)
+int
+qr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int           flags      = 0;
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = GF_CALLOC (1,
-                                                          sizeof (qr_local_t),
-                                                          gf_qr_mt_qr_local_t);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fentrylk_stub (frame,
-                                                          qr_fentrylk_helper,
-                                                          volume, fd, basename,
-                                                          cmd, type);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (fentrylk, frame, op_ret, op_errno);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fentrylk_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fentrylk, volume, fd,
-                            basename, cmd, type);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
+        qr_private_t     *priv           = NULL;
+        qr_conf_t        *conf           = NULL;
+        qr_inode_t       *qr_inode       = NULL;
+	int               ret            = -1;
+	dict_t           *new_xdata      = NULL;
 
-                qr_loc_wipe (&loc);
-        }
+        priv = this->private;
+        conf = &priv->conf;
 
-ret:
-        return 0;
-}
+	qr_inode = qr_inode_ctx_get (this, loc->inode);
+	if (qr_inode && qr_inode->data)
+		/* cached. only validate in qr_lookup_cbk */
+		goto wind;
+
+	if (!xdata)
+		xdata = new_xdata = dict_new ();
+
+	if (!xdata)
+		goto wind;
+
+	ret = 0;
+	if (conf->max_file_size)
+		ret = dict_set (xdata, GF_CONTENT_KEY,
+				data_from_uint64 (conf->max_file_size));
+	if (ret)
+		gf_log (this->name, GF_LOG_WARNING,
+			"cannot set key in request dict (%s)",
+			loc->path);
+wind:
+	frame->local = inode_ref (loc->inode);
 
+        STACK_WIND (frame, qr_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
-int32_t
-qr_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno)
+	if (new_xdata)
+		dict_unref (new_xdata);
 
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (finodelk, frame, op_ret, op_errno);
         return 0;
 }
 
 
-int32_t
-qr_finodelk_helper (call_frame_t *frame, xlator_t *this, const char *volume,
-                    fd_t *fd, int32_t cmd, struct gf_flock *lock)
+int
+qr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
+        gf_dirent_t *entry      = NULL;
+	qr_inode_t  *qr_inode   = NULL;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+	if (op_ret <= 0)
+		goto unwind;
 
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
+        list_for_each_entry (entry, &entries->list, list) {
+                if (!entry->inode)
+			continue;
 
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
+		qr_inode = qr_inode_ctx_get (this, entry->inode);
+		if (!qr_inode)
+			/* no harm */
+			continue;
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding finodelk "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
+		qr_content_refresh (this, qr_inode, &entry->d_stat);
         }
 
-        STACK_WIND (frame, qr_finodelk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, lock);
-        return 0;
-
 unwind:
-        QR_STACK_UNWIND (finodelk, frame, -1, op_errno);
-        return 0;
+	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-             int32_t cmd, struct gf_flock *lock)
-{
-        int           flags      = 0;
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = GF_CALLOC (1,
-                                                          sizeof (qr_local_t),
-                                                          gf_qr_mt_qr_local_t);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_finodelk_stub (frame,
-                                                          qr_finodelk_helper,
-                                                          volume, fd, cmd,
-                                                          lock);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (finodelk, frame, op_ret, op_errno);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_finodelk_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->finodelk, volume, fd,
-                            cmd, lock);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf)
+int
+qr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	     size_t size, off_t offset, dict_t *xdata)
 {
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf);
-        return 0;
+	STACK_WIND (frame, qr_readdirp_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readdirp,
+		    fd, size, offset, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_fsync_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+int
+qr_readv_cached (call_frame_t *frame, qr_inode_t *qr_inode, size_t size,
+		 off_t offset, uint32_t flags, dict_t *xdata)
 {
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
+	xlator_t         *this = NULL;
+	qr_private_t     *priv = NULL;
+	qr_inode_table_t *table = NULL;
+	int               op_ret = -1;
+	struct iobuf     *iobuf = NULL;
+	struct iobref    *iobref = NULL;
+	struct iovec      iov = {0, };
+	struct iatt       buf = {0, };
 
-        GF_ASSERT (frame);
+	this = frame->this;
+	priv = this->private;
+	table = &priv->table;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+	LOCK (&table->lock);
+	{
+		op_ret = -1;
 
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
+		if (!qr_inode->data)
+			goto unlock;
 
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
+		if (offset >= qr_inode->size)
+			goto unlock;
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fsync call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
+		if (!__qr_cache_is_fresh (this, qr_inode))
+			goto unlock;
 
-        STACK_WIND (frame, qr_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->fsync, fd, flags);
-        return 0;
+		op_ret = min (size, (qr_inode->size - offset));
 
-unwind:
-        QR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
+		iobuf = iobuf_get2 (this->ctx->iobuf_pool, op_ret);
+		if (!iobuf) {
+			op_ret = -1;
+			goto unlock;
+		}
 
+		iobref = iobref_new ();
+		if (!iobref) {
+			op_ret = -1;
+			iobuf_unref (iobuf);
+			goto unlock;
+		}
 
-int32_t
-qr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
-{
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        int           open_flags = 0;
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
+		iobref_add (iobref, iobuf);
 
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        open_flags = qr_fd_ctx->flags;
+		memcpy (iobuf->ptr, qr_inode->data + offset, op_ret);
 
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
+		buf = qr_inode->buf;
 
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = GF_CALLOC (1,
-                                                          sizeof (qr_local_t),
-                                                          gf_qr_mt_qr_local_t);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fsync_stub (frame, qr_fsync_helper,
-                                                       fd, flags);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
+		/* bump LRU */
+		__qr_inode_register (table, qr_inode);
+	}
+unlock:
+	UNLOCK (&table->lock);
 
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (fsync, frame, op_ret, op_errno, NULL, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fsync_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsync, fd, flags);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
+	if (op_ret > 0) {
+		iov.iov_base = iobuf->ptr;
+		iov.iov_len = op_ret;
 
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
+		STACK_UNWIND_STRICT (readv, frame, op_ret, 0, &iov, 1,
+				     &buf, iobref, xdata);
+	}
 
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, open_flags,
-                            fd, qr_fd_ctx->wbflags);
+	if (iobuf)
+		iobuf_unref (iobuf);
 
-                qr_loc_wipe (&loc);
-        }
+	if (iobref)
+		iobref_unref (iobref);
 
-ret:
-        return 0;
+	return op_ret;
 }
 
 
-int32_t
-qr_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+int
+qr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        int32_t           ret      = 0;
-        uint64_t          value    = 0;
-        qr_inode_t       *qr_inode = NULL;
-        qr_local_t       *local    = NULL;
-        qr_private_t     *priv     = NULL;
-        qr_inode_table_t *table    = NULL;
-
-        GF_ASSERT (frame);
-
-        if (op_ret == -1) {
-                goto out;
-        }
-
-        local = frame->local;
-        if ((local == NULL) || (local->fd == NULL)
-            || (local->fd->inode == NULL)) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (frame->this->name, GF_LOG_WARNING, "cannot get inode");
-                goto out;
-        }
+	qr_inode_t *qr_inode = NULL;
 
-        if ((this == NULL) || (this->private == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "cannot get quick read configuration from xlator "
-                        "object");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+	qr_inode = qr_inode_ctx_get (this, fd->inode);
+	if (!qr_inode)
+		goto wind;
 
-        priv = this->private;
-        table = &priv->table;
+	if (qr_readv_cached (frame, qr_inode, size, offset, flags, xdata) <= 0)
+		goto wind;
 
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (local->fd->inode, this, &value);
-                if (ret == 0) {
-                        qr_inode = (qr_inode_t *)(long) value;
-
-                        if (qr_inode) {
-                                if (qr_inode->stbuf.ia_size != postbuf->ia_size)
-                                {
-                                        inode_ctx_del (local->fd->inode, this,
-                                                       NULL);
-                                        __qr_inode_free (qr_inode);
-                                }
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-out:
-        QR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf,
-                         postbuf);
-        return 0;
+	return 0;
+wind:
+	STACK_WIND (frame, default_readv_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
+		    fd, size, offset, flags, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_ftruncate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     off_t offset)
+int
+qr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
+	   int count, off_t offset, uint32_t flags, struct iobref *iobref,
+	   dict_t *xdata)
 {
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding ftruncate "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
-        return 0;
+	qr_inode_prune (this, fd->inode);
 
-unwind:
-        QR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
-        return 0;
+	STACK_WIND (frame, default_writev_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
+		    fd, iov, count, offset, flags, iobref, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+int
+qr_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+	     dict_t *xdata)
 {
-        int           flags      = 0;
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_local_t   *local      = NULL;
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_qr_mt_qr_local_t);
-        if (local == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                need_unwind = 1;
-                goto out;
-        }
-
-        local->fd = fd;
-        frame->local = local;
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
+	qr_inode_prune (this, loc->inode);
 
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                stub = fop_ftruncate_stub (frame,
-                                                           qr_ftruncate_helper,
-                                                           fd, offset);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL,
-                                 NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_ftruncate_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate, fd, offset);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-           int32_t op_errno, struct gf_flock *lock)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (lk, frame, op_ret, op_errno, lock);
-        return 0;
+	STACK_WIND (frame, default_truncate_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
+		    loc, offset, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_lk_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-              struct gf_flock *lock)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding lk call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_lk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk, fd, cmd, lock);
-
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-qr_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-       struct gf_flock *lock)
+int
+qr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	      dict_t *xdata)
 {
-        int           flags      = 0;
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
+	qr_inode_prune (this, fd->inode);
 
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = GF_CALLOC (1,
-                                                          sizeof (qr_local_t),
-                                                          gf_qr_mt_qr_local_t);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_lk_stub (frame, qr_lk_helper, fd,
-                                                    cmd, lock);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (lk, frame, op_ret, op_errno, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_lk_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lk, fd, cmd, lock);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
+	STACK_WIND (frame, default_ftruncate_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
+		    fd, offset, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_release (xlator_t *this, fd_t *fd)
+int
+qr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+	 fd_t *fd, dict_t *xdata)
 {
-        qr_fd_ctx_t *qr_fd_ctx = NULL;
-        int32_t      ret       = 0;
-        uint64_t     value     = 0;
+	qr_inode_set_priority (this, fd->inode, loc->path);
 
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        ret = fd_ctx_del (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-                if (qr_fd_ctx) {
-                        qr_fd_ctx_free (qr_fd_ctx);
-                }
-        }
-
-out:
-        return 0;
+	STACK_WIND (frame, default_open_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->open,
+		    loc, flags, fd, xdata);
+	return 0;
 }
 
-
-int32_t
+int
 qr_forget (xlator_t *this, inode_t *inode)
 {
         qr_inode_t   *qr_inode = NULL;
-        uint64_t      value     = 0;
-        int32_t       ret       = -1;
-        qr_private_t *priv      = NULL;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+	qr_inode = qr_inode_ctx_get (this, inode);
 
-        priv = this->private;
+	if (!qr_inode)
+		return 0;
 
-        LOCK (&priv->table.lock);
-        {
-                ret = inode_ctx_del (inode, this, &value);
-                if (ret == 0) {
-                        qr_inode = (qr_inode_t *)(long) value;
-                        __qr_inode_free (qr_inode);
-                }
-        }
-        UNLOCK (&priv->table.lock);
+	qr_inode_prune (this, inode);
 
-out:
-        return 0;
+	GF_FREE (qr_inode);
+
+	return 0;
 }
 
 
@@ -3162,101 +695,35 @@ int32_t
 qr_inodectx_dump (xlator_t *this, inode_t *inode)
 {
         qr_inode_t *qr_inode = NULL;
-        uint64_t    value    = 0;
         int32_t     ret      = -1;
         char        key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
         char        buf[256]                        = {0, };
-        struct tm  *tm                              = NULL;
-        ret = inode_ctx_get (inode, this, &value);
-        if (ret != 0) {
-                goto out;
-        }
 
-        qr_inode = (qr_inode_t *)(long)value;
-        if (qr_inode == NULL) {
+        qr_inode = qr_inode_ctx_get (this, inode);
+        if (!qr_inode)
                 goto out;
-        }
 
         gf_proc_dump_build_key (key_prefix, "xlator.performance.quick-read",
                                 "inodectx");
         gf_proc_dump_add_section (key_prefix);
 
-        gf_proc_dump_write ("entire-file-cached", "%s", qr_inode->xattr ? "yes" : "no");
-
-        tm = localtime (&qr_inode->tv.tv_sec);
-        strftime (buf, 256, "%Y-%m-%d %H:%M:%S", tm);
-        snprintf (buf + strlen (buf), 256 - strlen (buf),
-                  ".%"GF_PRI_SUSECONDS, qr_inode->tv.tv_usec);
-
-        gf_proc_dump_write ("last-cache-validation-time", "%s", buf);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-qr_fdctx_dump (xlator_t *this, fd_t *fd)
-{
-        qr_fd_ctx_t *fdctx = NULL;
-        uint64_t     value = 0;
-        int32_t      ret   = 0, i = 0;
-        char         key[GF_DUMP_MAX_BUF_LEN]        = {0, };
-        char         key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-        call_stub_t *stub                            = NULL;
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret != 0) {
-                goto out;
-        }
-
-        fdctx = (qr_fd_ctx_t *)(long)value;
-        if (fdctx == NULL) {
-                goto out;
-        }
-
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.quick-read",
-                                "fdctx");
-        gf_proc_dump_add_section (key_prefix);
-
-        gf_proc_dump_write ("fd", "%p", fd);
-
-        gf_proc_dump_write ("path", "%s", fdctx->path);
-
-        LOCK (&fdctx->lock);
-        {
-                gf_proc_dump_write ("opened", "%s", fdctx->opened ? "yes" : "no");
-
-                gf_proc_dump_write ("open-in-progress", "%s", fdctx->open_in_transit ?
-                                    "yes" : "no");
-
-                gf_proc_dump_write ("caching disabled (for this fd)", "%s",
-                                    fdctx->disabled ? "yes" : "no");
-
-                gf_proc_dump_write ("flags", "%d", fdctx->flags);
-
-                gf_proc_dump_write ("wbflags", "%d", fdctx->wbflags);
+        gf_proc_dump_write ("entire-file-cached", "%s", qr_inode->data ? "yes" : "no");
 
-                list_for_each_entry (stub, &fdctx->waiting_ops, list) {
-                        gf_proc_dump_build_key (key, "",
-                                                "waiting-ops[%d].frame", i);
-                        gf_proc_dump_write (key, "%"PRId64,
-                                            stub->frame->root->unique);
+        if (qr_inode->last_refresh.tv_sec) {
+                gf_time_fmt (buf, sizeof buf, qr_inode->last_refresh.tv_sec,
+                             gf_timefmt_FT);
+                snprintf (buf + strlen (buf), sizeof buf - strlen (buf),
+                          ".%"GF_PRI_SUSECONDS, qr_inode->last_refresh.tv_usec);
 
-                        gf_proc_dump_build_key (key, "",
-                                                "waiting-ops[%d].fop", i);
-                        gf_proc_dump_write (key, "%s", gf_fop_list[stub->fop]);
-
-                        i++;
-                }
+                gf_proc_dump_write ("last-cache-validation-time", "%s", buf);
         }
-        UNLOCK (&fdctx->lock);
 
         ret = 0;
 out:
         return ret;
 }
 
+
 int
 qr_priv_dump (xlator_t *this)
 {
@@ -3276,14 +743,11 @@ qr_priv_dump (xlator_t *this)
         priv = this->private;
         conf = &priv->conf;
 
-        if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING, "conf null in xlator");
+        if (!conf)
                 return -1;
-        }
 
         table = &priv->table;
 
-
         gf_proc_dump_build_key (key_prefix, "xlator.performance.quick-read",
                                 "priv");
 
@@ -3293,13 +757,12 @@ qr_priv_dump (xlator_t *this)
         gf_proc_dump_write ("cache_timeout", "%d", conf->cache_timeout);
 
         if (!table) {
-                gf_log (this->name, GF_LOG_WARNING, "table is NULL");
                 goto out;
         } else {
                 for (i = 0; i < conf->max_pri; i++) {
                         list_for_each_entry (curr, &table->lru[i], lru) {
                                 file_count++;
-                                total_size += curr->stbuf.ia_size;
+                                total_size += curr->size;
                         }
                 }
         }
@@ -3331,7 +794,8 @@ mem_acct_init (xlator_t *this)
         return ret;
 }
 
-gf_boolean_t
+
+static gf_boolean_t
 check_cache_size_ok (xlator_t *this, int64_t cache_size)
 {
         int                     ret = _gf_true;
@@ -3354,7 +818,7 @@ check_cache_size_ok (xlator_t *this, int64_t cache_size)
         else
                 max_cache_size = total_mem;
 
-        gf_log (this->name, GF_LOG_INFO, "Max cache size is %"PRIu64,
+        gf_log (this->name, GF_LOG_DEBUG, "Max cache size is %"PRIu64,
                 max_cache_size);
         if (cache_size > max_cache_size) {
                 ret = _gf_false;
@@ -3374,6 +838,7 @@ reconfigure (xlator_t *this, dict_t *options)
         qr_private_t *priv           = NULL;
         qr_conf_t    *conf           = NULL;
         uint64_t       cache_size_new = 0;
+
         GF_VALIDATE_OR_GOTO ("quick-read", this, out);
         GF_VALIDATE_OR_GOTO (this->name, this->private, out);
         GF_VALIDATE_OR_GOTO (this->name, options, out);
@@ -3485,13 +950,9 @@ qr_get_priority_list (const char *opt_str, struct list_head *first)
                 priority_str = strtok_r (NULL, ",", &tmp_str);
         }
 out:
-        if (string != NULL) {
-                GF_FREE (string);
-        }
+        GF_FREE (string);
 
-        if (dup_str != NULL) {
-                GF_FREE (dup_str);
-        }
+        GF_FREE (dup_str);
 
         if (max_pri == -1) {
                 list_for_each_entry_safe (curr, tmp, first, list) {
@@ -3584,37 +1045,79 @@ out:
 
 
 void
+qr_inode_table_destroy (qr_private_t *priv)
+{
+        int        i    = 0;
+        qr_conf_t *conf = NULL;
+
+        conf = &priv->conf;
+
+        for (i = 0; i < conf->max_pri; i++) {
+                GF_ASSERT (list_empty (&priv->table.lru[i]));
+        }
+
+        LOCK_DESTROY (&priv->table.lock);
+
+        return;
+}
+
+
+void
+qr_conf_destroy (qr_conf_t *conf)
+{
+        struct qr_priority *curr = NULL, *tmp = NULL;
+
+        list_for_each_entry_safe (curr, tmp, &conf->priority_list, list) {
+                list_del (&curr->list);
+                GF_FREE (curr->pattern);
+                GF_FREE (curr);
+        }
+
+        return;
+}
+
+
+void
 fini (xlator_t *this)
 {
+        qr_private_t *priv = NULL;
+
+        if (this == NULL) {
+                goto out;
+        }
+
+        priv = this->private;
+        if (priv == NULL) {
+                goto out;
+        }
+
+        qr_inode_table_destroy (priv);
+        qr_conf_destroy (&priv->conf);
+
+        this->private = NULL;
+
+        GF_FREE (priv);
+out:
         return;
 }
 
 struct xlator_fops fops = {
         .lookup      = qr_lookup,
+	.readdirp    = qr_readdirp,
         .open        = qr_open,
         .readv       = qr_readv,
-        .writev      = qr_writev,
-        .fstat       = qr_fstat,
-        .fsetxattr   = qr_fsetxattr,
-        .fgetxattr   = qr_fgetxattr,
-        .flush       = qr_flush,
-        .fentrylk    = qr_fentrylk,
-        .finodelk    = qr_finodelk,
-        .fsync       = qr_fsync,
-        .ftruncate   = qr_ftruncate,
-        .lk          = qr_lk,
-        .fsetattr    = qr_fsetattr,
+	.writev      = qr_writev,
+	.truncate    = qr_truncate,
+	.ftruncate   = qr_ftruncate
 };
 
 struct xlator_cbks cbks = {
         .forget  = qr_forget,
-        .release = qr_release,
 };
 
 struct xlator_dumpops dumpops = {
         .priv      =  qr_priv_dump,
         .inodectx  =  qr_inodectx_dump,
-        .fdctx     =  qr_fdctx_dump
 };
 
 struct volume_options options[] = {
@@ -3640,4 +1143,5 @@ struct volume_options options[] = {
           .max  = 1 * GF_UNIT_KB * 1000,
           .default_value = "64KB",
         },
+        { .key  = {NULL} }
 };
diff --git a/xlators/performance/quick-read/src/quick-read.h b/xlators/performance/quick-read/src/quick-read.h
index 064151b63..6f0a05417 100644
--- a/xlators/performance/quick-read/src/quick-read.h
+++ b/xlators/performance/quick-read/src/quick-read.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __QUICK_READ_H
@@ -43,40 +34,20 @@
 #include <fnmatch.h>
 #include "quick-read-mem-types.h"
 
-struct qr_fd_ctx {
-        char              opened;
-        char              disabled;
-        char              open_in_transit;
-        char             *path;
-        int               flags;
-        int               wbflags;
-        struct list_head  waiting_ops;
-        gf_lock_t         lock;
-};
-typedef struct qr_fd_ctx qr_fd_ctx_t;
-
-struct qr_local {
-        char         is_open;
-        char        *path;
-        char         just_validated;
-        fd_t        *fd;
-        int          open_flags;
-        int32_t      op_ret;
-        int32_t      op_errno;
-        call_stub_t *stub;
-};
-typedef struct qr_local qr_local_t;
 
 struct qr_inode {
-        dict_t           *xattr;
-        inode_t          *inode;
+	void             *data;
+	size_t            size;
         int               priority;
-        struct iatt       stbuf;
-        struct timeval    tv;
+	uint32_t          ia_mtime;
+	uint32_t          ia_mtime_nsec;
+	struct iatt       buf;
+        struct timeval    last_refresh;
         struct list_head  lru;
 };
 typedef struct qr_inode qr_inode_t;
 
+
 struct qr_priority {
         char             *pattern;
         int32_t           priority;
@@ -106,13 +77,5 @@ struct qr_private {
 };
 typedef struct qr_private qr_private_t;
 
-void qr_local_free (qr_local_t *local);
-
-#define QR_STACK_UNWIND(op, frame, params ...) do {             \
-                qr_local_t *__local = frame->local;             \
-                frame->local = NULL;                            \
-                STACK_UNWIND_STRICT (op, frame, params);        \
-                qr_local_free (__local);                        \
-        } while (0)
 
 #endif /* #ifndef __QUICK_READ_H */
diff --git a/xlators/performance/read-ahead/src/Makefile.am b/xlators/performance/read-ahead/src/Makefile.am
index b46020aac..be80ae7ac 100644
--- a/xlators/performance/read-ahead/src/Makefile.am
+++ b/xlators/performance/read-ahead/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = read-ahead.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-read_ahead_la_LDFLAGS = -module -avoidversion
+read_ahead_la_LDFLAGS = -module -avoid-version
 
 read_ahead_la_SOURCES = read-ahead.c page.c
 read_ahead_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = read-ahead.h read-ahead-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/read-ahead/src/page.c b/xlators/performance/read-ahead/src/page.c
index 8aa55c065..e79e7ae78 100644
--- a/xlators/performance/read-ahead/src/page.c
+++ b/xlators/performance/read-ahead/src/page.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -142,7 +133,8 @@ ra_waitq_return (ra_waitq_t *waitq)
 int
 ra_fault_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int32_t op_ret, int32_t op_errno, struct iovec *vector,
-              int32_t count, struct iatt *stbuf, struct iobref *iobref)
+              int32_t count, struct iatt *stbuf, struct iobref *iobref,
+              dict_t *xdata)
 {
         ra_local_t   *local          = NULL;
         off_t         pending_offset = 0;
@@ -233,7 +225,7 @@ unlock:
 
         fd_unref (local->fd);
 
-        GF_FREE (frame->local);
+        mem_put (frame->local);
         frame->local = NULL;
 
 out:
@@ -261,7 +253,7 @@ ra_page_fault (ra_file_t *file, call_frame_t *frame, off_t offset)
                 goto err;
         }
 
-        fault_local = GF_CALLOC (1, sizeof (ra_local_t), gf_ra_mt_ra_local_t);
+        fault_local = mem_get0 (THIS->local_pool);
         if (fault_local == NULL) {
                 STACK_DESTROY (fault_frame->root);
                 op_ret = -1;
@@ -278,7 +270,7 @@ ra_page_fault (ra_file_t *file, call_frame_t *frame, off_t offset)
         STACK_WIND (fault_frame, ra_fault_cbk,
                     FIRST_CHILD (fault_frame->this),
                     FIRST_CHILD (fault_frame->this)->fops->readv,
-                    file->fd, file->page_size, offset, 0);
+                    file->fd, file->page_size, offset, 0, NULL);
 
         return;
 
@@ -447,11 +439,11 @@ ra_frame_unwind (call_frame_t *frame)
         file = (ra_file_t *)(long)tmp_file;
 
         STACK_UNWIND_STRICT (readv, frame, local->op_ret, local->op_errno,
-                             vector, count, &file->stbuf, iobref);
+                             vector, count, &file->stbuf, iobref, NULL);
 
         iobref_unref (iobref);
         pthread_mutex_destroy (&local->local_lock);
-        GF_FREE (local);
+        mem_put (local);
         GF_FREE (vector);
 
 out:
@@ -508,6 +500,9 @@ ra_page_wakeup (ra_page_t *page)
                 ra_frame_fill (page, frame);
         }
 
+        if (page->stale) {
+                ra_page_purge (page);
+        }
 out:
         return waitq;
 }
diff --git a/xlators/performance/read-ahead/src/read-ahead-mem-types.h b/xlators/performance/read-ahead/src/read-ahead-mem-types.h
index 7ca093696..219e29289 100644
--- a/xlators/performance/read-ahead/src/read-ahead-mem-types.h
+++ b/xlators/performance/read-ahead/src/read-ahead-mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -25,7 +16,6 @@
 
 enum gf_ra_mem_types_ {
         gf_ra_mt_ra_file_t   = gf_common_mt_end + 1,
-        gf_ra_mt_ra_local_t,
         gf_ra_mt_ra_conf_t,
         gf_ra_mt_ra_page_t,
         gf_ra_mt_ra_waitq_t,
diff --git a/xlators/performance/read-ahead/src/read-ahead.c b/xlators/performance/read-ahead/src/read-ahead.c
index f58c4078d..069ab1f1a 100644
--- a/xlators/performance/read-ahead/src/read-ahead.c
+++ b/xlators/performance/read-ahead/src/read-ahead.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /*
@@ -44,12 +35,11 @@ read_ahead (call_frame_t *frame, ra_file_t *file);
 
 int
 ra_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd)
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         ra_conf_t  *conf    = NULL;
         ra_file_t  *file    = NULL;
         int         ret     = 0;
-        long        wbflags = 0;
 
         GF_ASSERT (frame);
         GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
@@ -60,8 +50,6 @@ ra_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto unwind;
         }
 
-        wbflags = (long)frame->local;
-
         file = GF_CALLOC (1, sizeof (*file), gf_ra_mt_ra_file_t);
         if (!file) {
                 op_ret = -1;
@@ -74,10 +62,6 @@ ra_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((fd->flags & O_DIRECT) || ((fd->flags & O_ACCMODE) == O_WRONLY))
                 file->disabled = 1;
 
-        if (wbflags & GF_OPEN_NOWB) {
-                file->disabled = 1;
-        }
-
         file->offset = (unsigned long long) 0;
         file->conf = conf;
         file->pages.next = &file->pages;
@@ -116,7 +100,7 @@ ra_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unwind:
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 
         return 0;
 }
@@ -126,7 +110,7 @@ int
 ra_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
+               struct iatt *postparent, dict_t *xdata)
 {
         ra_conf_t  *conf = NULL;
         ra_file_t  *file = NULL;
@@ -187,7 +171,7 @@ ra_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 unwind:
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         return 0;
 }
@@ -195,17 +179,15 @@ unwind:
 
 int
 ra_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+         fd_t *fd, dict_t *xdata)
 {
         GF_ASSERT (frame);
         GF_ASSERT (this);
 
-        frame->local = (void *)(long)wbflags;
-
         STACK_WIND (frame, ra_open_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
 
         return 0;
 }
@@ -213,7 +195,7 @@ ra_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
 
 int
 ra_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, fd_t *fd, dict_t *params)
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         GF_ASSERT (frame);
         GF_ASSERT (this);
@@ -221,7 +203,7 @@ ra_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         STACK_WIND (frame, ra_create_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
 
         return 0;
 }
@@ -248,8 +230,12 @@ flush_region (call_frame_t *frame, ra_file_t *file, off_t offset, off_t size,
                                 if (!trav->waitq) {
                                         ra_page_purge (trav);
                                 }
-                                else if (for_write) {
-                                        trav->poisoned = 1;
+                                else {
+                                        trav->stale = 1;
+
+                                        if (for_write) {
+                                                trav->poisoned = 1;
+                                        }
                                 }
                         }
                         trav = next;
@@ -358,7 +344,8 @@ out:
 int
 ra_need_atime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                   dict_t *xdata)
 {
         GF_ASSERT (frame);
         STACK_DESTROY (frame->root);
@@ -449,7 +436,7 @@ dispatch_requests (call_frame_t *frame, ra_file_t *file)
                 STACK_WIND (ra_frame, ra_need_atime_cbk,
                             FIRST_CHILD (frame->this),
                             FIRST_CHILD (frame->this)->fops->readv,
-                            file->fd, 1, 1, 0);
+                            file->fd, 1, 1, 0, NULL);
         }
 
 out:
@@ -460,12 +447,13 @@ out:
 int
 ra_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                       int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                       int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                       dict_t *xdata)
 {
         GF_ASSERT (frame);
 
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
+                             stbuf, iobref, xdata);
 
         return 0;
 }
@@ -473,7 +461,7 @@ ra_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 ra_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags)
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
         ra_file_t   *file            = NULL;
         ra_local_t  *local           = NULL;
@@ -510,7 +498,7 @@ ra_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                         "expected offset (%"PRId64") when page_count=%d",
                         offset, file->page_count);
 
-                if (file->expected < (conf->page_size * conf->page_count)) {
+                if (file->expected < (file->page_size * conf->page_count)) {
                         file->expected += size;
                         file->page_count = min ((file->expected
                                                  / file->page_size),
@@ -522,7 +510,7 @@ ra_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                 flush_region (frame, file, 0, file->pages.prev->offset + 1, 0);
         }
 
-        local = (void *) GF_CALLOC (1, sizeof (*local), gf_ra_mt_ra_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -553,7 +541,8 @@ ra_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                             NULL);
 
         return 0;
 
@@ -561,17 +550,17 @@ disabled:
         STACK_WIND (frame, ra_readv_disabled_cbk,
                     FIRST_CHILD (frame->this),
                     FIRST_CHILD (frame->this)->fops->readv,
-                    fd, size, offset, flags);
+                    fd, size, offset, flags, xdata);
         return 0;
 }
 
 
 int
 ra_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno)
+              int32_t op_errno, dict_t *xdata)
 {
         GF_ASSERT (frame);
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -579,16 +568,18 @@ ra_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
 
 int
 ra_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf)
+              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
         GF_ASSERT (frame);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
         return 0;
 }
 
 
 int
-ra_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+ra_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         ra_file_t *file     = NULL;
         uint64_t   tmp_file = 0;
@@ -606,17 +597,18 @@ ra_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
         }
 
         STACK_WIND (frame, ra_flush_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->flush, fd);
+                    FIRST_CHILD (this)->fops->flush, fd, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int
-ra_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync)
+ra_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata)
 {
         ra_file_t *file     = NULL;
         uint64_t   tmp_file = 0;
@@ -634,11 +626,11 @@ ra_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync)
         }
 
         STACK_WIND (frame, ra_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsync, fd, datasync);
+                    FIRST_CHILD (this)->fops->fsync, fd, datasync, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL);
+        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -646,7 +638,7 @@ unwind:
 int
 ra_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+               struct iatt *postbuf, dict_t *xdata)
 {
         ra_file_t *file     = NULL;
 
@@ -659,14 +651,16 @@ ra_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
         return 0;
 }
 
 
 int
 ra_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref)
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
         ra_file_t *file    = NULL;
         uint64_t  tmp_file = 0;
@@ -688,12 +682,12 @@ ra_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
         STACK_WIND (frame, ra_writev_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
 
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -701,29 +695,30 @@ unwind:
 int
 ra_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf)
+                 struct iatt *postbuf, dict_t *xdata)
 {
         GF_ASSERT (frame);
 
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
         return 0;
 }
 
 
 int
 ra_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, struct iatt *buf)
+             int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         GF_ASSERT (frame);
 
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
-ra_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+ra_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
         ra_file_t *file     = NULL;
         fd_t      *iter_fd  = NULL;
@@ -762,11 +757,11 @@ ra_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         STACK_WIND (frame, ra_truncate_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->truncate,
-                    loc, offset);
+                    loc, offset, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -863,7 +858,7 @@ out:
 }
 
 int
-ra_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+ra_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         ra_file_t *file     = NULL;
         fd_t      *iter_fd  = NULL;
@@ -892,17 +887,18 @@ ra_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
         UNLOCK (&inode->lock);
 
         STACK_WIND (frame, ra_attr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fstat, fd);
+                    FIRST_CHILD (this)->fops->fstat, fd, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL);
+        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int
-ra_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+ra_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
         ra_file_t *file    = NULL;
         fd_t      *iter_fd = NULL;
@@ -938,14 +934,114 @@ ra_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         UNLOCK (&inode->lock);
 
         STACK_WIND (frame, ra_truncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ftruncate, fd, offset);
+                    FIRST_CHILD (this)->fops->ftruncate, fd, offset, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+ra_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+        GF_ASSERT (frame);
+
+        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
         return 0;
 }
 
+static int
+ra_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	     size_t len, dict_t *xdata)
+{
+        ra_file_t *file    = NULL;
+        fd_t      *iter_fd = NULL;
+        inode_t   *inode   = NULL;
+        uint64_t  tmp_file = 0;
+        int32_t   op_errno = EINVAL;
+
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
+        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+
+        inode = fd->inode;
+
+        LOCK (&inode->lock);
+        {
+                list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                        fd_ctx_get (iter_fd, this, &tmp_file);
+                        file = (ra_file_t *)(long)tmp_file;
+                        if (!file)
+                                continue;
+
+                        flush_region(frame, file, offset, len, 1);
+                }
+        }
+        UNLOCK (&inode->lock);
+
+        STACK_WIND (frame, ra_discard_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->discard, fd, offset, len, xdata);
+        return 0;
+
+unwind:
+        STACK_UNWIND_STRICT (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+ra_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+        GF_ASSERT (frame);
+
+        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+        return 0;
+}
+
+static int
+ra_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+        ra_file_t *file    = NULL;
+        fd_t      *iter_fd = NULL;
+        inode_t   *inode   = NULL;
+        uint64_t  tmp_file = 0;
+        int32_t   op_errno = EINVAL;
+
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
+        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+
+        inode = fd->inode;
+
+        LOCK (&inode->lock);
+        {
+                list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                        fd_ctx_get (iter_fd, this, &tmp_file);
+                        file = (ra_file_t *)(long)tmp_file;
+                        if (!file)
+                                continue;
+
+                        flush_region(frame, file, offset, len, 1);
+                }
+        }
+        UNLOCK (&inode->lock);
+
+        STACK_WIND (frame, ra_zerofill_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->zerofill, fd,
+                    offset, len, xdata);
+        return 0;
+
+unwind:
+        STACK_UNWIND_STRICT (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
 int
 ra_priv_dump (xlator_t *this)
@@ -953,6 +1049,7 @@ ra_priv_dump (xlator_t *this)
         ra_conf_t       *conf                           = NULL;
         int             ret                             = -1;
         char            key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
+        gf_boolean_t    add_section                     = _gf_false;
 
         if (!this) {
                 goto out;
@@ -964,26 +1061,32 @@ ra_priv_dump (xlator_t *this)
                 goto out;
         }
 
-        ret = pthread_mutex_trylock (&conf->conf_lock);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Unable to lock client %s "
-                        "(%s)", this->name, strerror (ret));
-                ret = -1;
-                goto out;
-        }
-
         gf_proc_dump_build_key (key_prefix, "xlator.performance.read-ahead",
                                 "priv");
 
         gf_proc_dump_add_section (key_prefix);
-        gf_proc_dump_write ("page_size", "%d", conf->page_size);
-        gf_proc_dump_write ("page_count", "%d", conf->page_count);
-        gf_proc_dump_write ("force_atime_update", "%d", conf->force_atime_update);
+        add_section = _gf_true;
 
+        ret = pthread_mutex_trylock (&conf->conf_lock);
+        if (ret)
+                goto out;
+        {
+                gf_proc_dump_write ("page_size", "%d", conf->page_size);
+                gf_proc_dump_write ("page_count", "%d", conf->page_count);
+                gf_proc_dump_write ("force_atime_update", "%d",
+                                    conf->force_atime_update);
+        }
         pthread_mutex_unlock (&conf->conf_lock);
 
         ret = 0;
 out:
+        if (ret && conf) {
+                if (add_section == _gf_false)
+                        gf_proc_dump_add_section (key_prefix);
+
+                gf_proc_dump_write ("Unable to dump priv",
+                                    "(Lock acquisition failed) %s", this->name);
+        }
         return ret;
 }
 
@@ -1021,6 +1124,8 @@ reconfigure (xlator_t *this, dict_t *options)
 
         GF_OPTION_RECONF ("page-count", conf->page_count, options, uint32, out);
 
+	GF_OPTION_RECONF ("page-size", conf->page_size, options, size, out);
+
         ret = 0;
  out:
         return ret;
@@ -1053,6 +1158,8 @@ init (xlator_t *this)
 
         conf->page_size = this->ctx->page_size;
 
+	GF_OPTION_INIT ("page-size", conf->page_size, size, out);
+
         GF_OPTION_INIT ("page-count", conf->page_count, uint32, out);
 
         GF_OPTION_INIT ("force-atime-update", conf->force_atime_update, bool, out);
@@ -1061,14 +1168,21 @@ init (xlator_t *this)
         conf->files.prev = &conf->files;
 
         pthread_mutex_init (&conf->conf_lock, NULL);
+
+        this->local_pool = mem_pool_new (ra_local_t, 64);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
         this->private = conf;
         ret = 0;
 
 out:
         if (ret == -1) {
-                if (conf != NULL) {
-                        GF_FREE (conf);
-                }
+                GF_FREE (conf);
         }
 
         return ret;
@@ -1087,11 +1201,14 @@ fini (xlator_t *this)
                 goto out;
         }
 
+        this->private = NULL;
+
+        GF_ASSERT ((conf->files.next == &conf->files)
+                   && (conf->files.prev == &conf->files));
+
         pthread_mutex_destroy (&conf->conf_lock);
         GF_FREE (conf);
 
-        this->private = NULL;
-
 out:
         return;
 }
@@ -1106,6 +1223,8 @@ struct xlator_fops fops = {
         .truncate    = ra_truncate,
         .ftruncate   = ra_ftruncate,
         .fstat       = ra_fstat,
+	.discard     = ra_discard,
+        .zerofill    = ra_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -1129,5 +1248,12 @@ struct volume_options options[] = {
           .default_value = "4",
           .description = "Number of pages that will be pre-fetched"
         },
+	{ .key = {"page-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .min = 4096,
+	  .max = 1048576 * 64,
+	  .default_value = "131072",
+	  .description = "Page size with which read-ahead performs server I/O"
+	},
         { .key = {NULL} },
 };
diff --git a/xlators/performance/read-ahead/src/read-ahead.h b/xlators/performance/read-ahead/src/read-ahead.h
index f5e73cb66..d1d768c34 100644
--- a/xlators/performance/read-ahead/src/read-ahead.h
+++ b/xlators/performance/read-ahead/src/read-ahead.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __READ_AHEAD_H
@@ -85,6 +76,7 @@ struct ra_page {
         size_t            size;
         struct ra_waitq  *waitq;
         struct iobref    *iobref;
+        char              stale;
 };
 
 
diff --git a/xlators/performance/readdir-ahead/Makefile.am b/xlators/performance/readdir-ahead/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/performance/readdir-ahead/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/performance/readdir-ahead/src/Makefile.am b/xlators/performance/readdir-ahead/src/Makefile.am
new file mode 100644
index 000000000..cdabd1428
--- /dev/null
+++ b/xlators/performance/readdir-ahead/src/Makefile.am
@@ -0,0 +1,15 @@
+xlator_LTLIBRARIES = readdir-ahead.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
+
+readdir_ahead_la_LDFLAGS = -module -avoidversion
+
+readdir_ahead_la_SOURCES = readdir-ahead.c
+readdir_ahead_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = readdir-ahead.h readdir-ahead-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead-mem-types.h b/xlators/performance/readdir-ahead/src/readdir-ahead-mem-types.h
new file mode 100644
index 000000000..39e2c5369
--- /dev/null
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead-mem-types.h
@@ -0,0 +1,24 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __RDA_MEM_TYPES_H__
+#define __RDA_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_rda_mem_types_ {
+        gf_rda_mt_rda_local   = gf_common_mt_end + 1,
+	gf_rda_mt_rda_fd_ctx,
+	gf_rda_mt_rda_priv,
+        gf_rda_mt_end
+};
+
+#endif
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead.c b/xlators/performance/readdir-ahead/src/readdir-ahead.c
new file mode 100644
index 000000000..53e6756f0
--- /dev/null
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead.c
@@ -0,0 +1,560 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+ * performance/readdir-ahead preloads a local buffer with directory entries
+ * on opendir. The optimization involves using maximum sized gluster rpc
+ * requests (128k) to minimize overhead of smaller client requests.
+ *
+ * For example, fuse currently supports a maximum readdir buffer of 4k
+ * (regardless of the filesystem client's buffer size). readdir-ahead should
+ * effectively convert these smaller requests into fewer, larger sized requests
+ * for simple, sequential workloads (i.e., ls).
+ *
+ * The translator is currently designed to handle the simple, sequential case
+ * only. If a non-sequential directory read occurs, readdir-ahead disables
+ * preloads on the directory.
+ */
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "call-stub.h"
+#include "readdir-ahead.h"
+#include "readdir-ahead-mem-types.h"
+#include "defaults.h"
+
+static int rda_fill_fd(call_frame_t *, xlator_t *, fd_t *);
+
+/*
+ * Get (or create) the fd context for storing prepopulated directory
+ * entries.
+ */
+static struct
+rda_fd_ctx *get_rda_fd_ctx(fd_t *fd, xlator_t *this)
+{
+	uint64_t val;
+	struct rda_fd_ctx *ctx;
+
+	LOCK(&fd->lock);
+
+	if (__fd_ctx_get(fd, this, &val) < 0) {
+		ctx = GF_CALLOC(1, sizeof(struct rda_fd_ctx),
+				gf_rda_mt_rda_fd_ctx);
+		if (!ctx)
+			goto out;
+
+		LOCK_INIT(&ctx->lock);
+		INIT_LIST_HEAD(&ctx->entries.list);
+		ctx->state = RDA_FD_NEW;
+		/* ctx offset values initialized to 0 */
+
+		if (__fd_ctx_set(fd, this, (uint64_t) ctx) < 0) {
+			GF_FREE(ctx);
+			ctx = NULL;
+			goto out;
+		}
+	} else {
+		ctx = (struct rda_fd_ctx *) val;
+	}
+out:
+	UNLOCK(&fd->lock);
+	return ctx;
+}
+
+/*
+ * Reset the tracking state of the context.
+ */
+static void
+rda_reset_ctx(struct rda_fd_ctx *ctx)
+{
+	ctx->state = RDA_FD_NEW;
+	ctx->cur_offset = 0;
+	ctx->cur_size = 0;
+	ctx->next_offset = 0;
+	gf_dirent_free(&ctx->entries);
+}
+
+/*
+ * Check whether we can handle a request. Offset verification is done by the
+ * caller, so we only check whether the preload buffer has completion status
+ * (including an error) or has some data to return.
+ */
+static gf_boolean_t
+rda_can_serve_readdirp(struct rda_fd_ctx *ctx, size_t request_size)
+{
+	if ((ctx->state & RDA_FD_EOD) ||
+	    (ctx->state & RDA_FD_ERROR) ||
+	    (!(ctx->state & RDA_FD_PLUGGED) && (ctx->cur_size > 0)))
+		return _gf_true;
+
+	return _gf_false;
+}
+
+/*
+ * Serve a request from the fd dentry list based on the size of the request
+ * buffer. ctx must be locked.
+ */
+static int32_t
+__rda_serve_readdirp(xlator_t *this, gf_dirent_t *entries, size_t request_size,
+		   struct rda_fd_ctx *ctx)
+{
+	gf_dirent_t *dirent, *tmp;
+	size_t dirent_size, size = 0;
+	int32_t count = 0;
+	struct rda_priv *priv = this->private;
+
+	list_for_each_entry_safe(dirent, tmp, &ctx->entries.list, list) {
+		dirent_size = gf_dirent_size(dirent->d_name);
+		if (size + dirent_size > request_size)
+			break;
+
+		size += dirent_size;
+		list_del_init(&dirent->list);
+		ctx->cur_size -= dirent_size;
+
+		list_add_tail(&dirent->list, &entries->list);
+		ctx->cur_offset = dirent->d_off;
+		count++;
+	}
+
+	if (ctx->cur_size <= priv->rda_low_wmark)
+		ctx->state |= RDA_FD_PLUGGED;
+
+	return count;
+}
+
+static int32_t
+rda_readdirp_stub(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+		  off_t offset, dict_t *xdata)
+{
+	gf_dirent_t entries;
+	int32_t ret;
+	struct rda_fd_ctx *ctx;
+	int op_errno = 0;
+
+	ctx = get_rda_fd_ctx(fd, this);
+	INIT_LIST_HEAD(&entries.list);
+	ret = __rda_serve_readdirp(this, &entries, size, ctx);
+
+	if (!ret && (ctx->state & RDA_FD_ERROR)) {
+		ret = -1;
+		op_errno = ctx->op_errno;
+		ctx->state &= ~RDA_FD_ERROR;
+
+		/*
+		 * the preload has stopped running in the event of an error, so
+		 * pass all future requests along
+		 */
+		ctx->state |= RDA_FD_BYPASS;
+	}
+
+	STACK_UNWIND_STRICT(readdirp, frame, ret, op_errno, &entries, xdata);
+	gf_dirent_free(&entries);
+
+	return 0;
+}
+
+static int32_t
+rda_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	     off_t off, dict_t *xdata)
+{
+	struct rda_fd_ctx *ctx;
+	call_stub_t *stub;
+	int fill = 0;
+
+	ctx = get_rda_fd_ctx(fd, this);
+	if (!ctx)
+		goto err;
+
+	if (ctx->state & RDA_FD_BYPASS)
+		goto bypass;
+
+	LOCK(&ctx->lock);
+
+	/* recheck now that we have the lock */
+	if (ctx->state & RDA_FD_BYPASS) {
+		UNLOCK(&ctx->lock);
+		goto bypass;
+	}
+
+	/*
+	 * If a new read comes in at offset 0 and the buffer has been
+	 * completed, reset the context and kickstart the filler again.
+	 */
+	if (!off && (ctx->state & RDA_FD_EOD) && (ctx->cur_size == 0)) {
+		rda_reset_ctx(ctx);
+		fill = 1;
+	}
+
+	/*
+	 * If a readdir occurs at an unexpected offset or we already have a
+	 * request pending, admit defeat and just get out of the way.
+	 */
+	if (off != ctx->cur_offset || ctx->stub) {
+		ctx->state |= RDA_FD_BYPASS;
+		UNLOCK(&ctx->lock);
+		goto bypass;
+	}
+
+	stub = fop_readdirp_stub(frame, rda_readdirp_stub, fd, size, off, xdata);
+	if (!stub) {
+		UNLOCK(&ctx->lock);
+		goto err;
+	}
+
+	/*
+	 * If we haven't bypassed the preload, this means we can either serve
+	 * the request out of the preload or the request that enables us to do
+	 * so is in flight...
+	 */
+	if (rda_can_serve_readdirp(ctx, size))
+		call_resume(stub);
+	else
+		ctx->stub = stub;
+
+	UNLOCK(&ctx->lock);
+
+	if (fill)
+		rda_fill_fd(frame, this, fd);
+
+	return 0;
+
+bypass:
+	STACK_WIND(frame, default_readdirp_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+	return 0;
+
+err:
+	STACK_UNWIND_STRICT(readdirp, frame, -1, ENOMEM, NULL, NULL);
+	return 0;
+}
+
+static int32_t
+rda_fill_fd_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+		 dict_t *xdata)
+{
+	gf_dirent_t *dirent, *tmp;
+	struct rda_local *local = frame->local;
+	struct rda_fd_ctx *ctx = local->ctx;
+	struct rda_priv *priv = this->private;
+	int fill = 1;
+
+	LOCK(&ctx->lock);
+
+	/* Verify that the preload buffer is still pending on this data. */
+	if (ctx->next_offset != local->offset) {
+		gf_log(this->name, GF_LOG_ERROR,
+			"Out of sequence directory preload.");
+		ctx->state |= (RDA_FD_BYPASS|RDA_FD_ERROR);
+		ctx->op_errno = EUCLEAN;
+
+		goto out;
+	}
+
+	if (entries) {
+		list_for_each_entry_safe(dirent, tmp, &entries->list, list) {
+			list_del_init(&dirent->list);
+			/* must preserve entry order */
+			list_add_tail(&dirent->list, &ctx->entries.list);
+
+			ctx->cur_size += gf_dirent_size(dirent->d_name);
+			ctx->next_offset = dirent->d_off;
+		}
+	}
+
+	if (ctx->cur_size >= priv->rda_high_wmark)
+		ctx->state &= ~RDA_FD_PLUGGED;
+
+	if (!op_ret) {
+		/* we've hit eod */
+		ctx->state &= ~RDA_FD_RUNNING;
+		ctx->state |= RDA_FD_EOD;
+	} else if (op_ret == -1) {
+		/* kill the preload and pend the error */
+		ctx->state &= ~RDA_FD_RUNNING;
+		ctx->state |= RDA_FD_ERROR;
+		ctx->op_errno = op_errno;
+	}
+
+	/*
+	 * NOTE: The strict bypass logic in readdirp() means a pending request
+	 * is always based on ctx->cur_offset.
+	 */
+	if (ctx->stub &&
+	    rda_can_serve_readdirp(ctx, ctx->stub->args.size)) {
+		call_resume(ctx->stub);
+		ctx->stub = NULL;
+	}
+
+out:
+	/*
+	 * If we have been marked for bypass and have no pending stub, clear the
+	 * run state so we stop preloading the context with entries.
+	 */
+	if ((ctx->state & RDA_FD_BYPASS) && !ctx->stub)
+		ctx->state &= ~RDA_FD_RUNNING;
+
+	if (!(ctx->state & RDA_FD_RUNNING)) {
+		fill = 0;
+		STACK_DESTROY(ctx->fill_frame->root);
+		ctx->fill_frame = NULL;
+	}
+
+	UNLOCK(&ctx->lock);
+
+	if (fill)
+		rda_fill_fd(frame, this, local->fd);
+
+	return 0;
+}
+
+/*
+ * Start prepopulating the fd context with directory entries.
+ */
+static int
+rda_fill_fd(call_frame_t *frame, xlator_t *this, fd_t *fd)
+{
+	call_frame_t *nframe = NULL;
+	struct rda_local *local = NULL;
+	struct rda_fd_ctx *ctx;
+	off_t offset;
+	struct rda_priv *priv = this->private;
+
+	ctx = get_rda_fd_ctx(fd, this);
+	if (!ctx)
+		goto err;
+
+	LOCK(&ctx->lock);
+
+	if (ctx->state & RDA_FD_NEW) {
+		ctx->state &= ~RDA_FD_NEW;
+		ctx->state |= RDA_FD_RUNNING;
+		if (priv->rda_low_wmark)
+			ctx->state |= RDA_FD_PLUGGED;
+	}
+
+	offset = ctx->next_offset;
+
+	if (!ctx->fill_frame) {
+		nframe = copy_frame(frame);
+		if (!nframe) {
+			UNLOCK(&ctx->lock);
+			goto err;
+		}
+
+		local = mem_get0(this->local_pool);
+		if (!local) {
+			UNLOCK(&ctx->lock);
+			goto err;
+		}
+
+		local->ctx = ctx;
+		local->fd = fd;
+		nframe->local = local;
+
+		ctx->fill_frame = nframe;
+	} else {
+		nframe = ctx->fill_frame;
+		local = nframe->local;
+	}
+
+	local->offset = offset;
+
+	UNLOCK(&ctx->lock);
+
+	STACK_WIND(nframe, rda_fill_fd_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp, fd, priv->rda_req_size,
+		   offset, NULL);
+
+	return 0;
+
+err:
+	if (nframe)
+		FRAME_DESTROY(nframe);
+
+	return -1;
+}
+
+static int32_t
+rda_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+	if (!op_ret)
+		rda_fill_fd(frame, this, fd);
+
+	STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, xdata);
+	return 0;
+}
+
+static int32_t
+rda_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+		dict_t *xdata)
+{
+	STACK_WIND(frame, rda_opendir_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+	return 0;
+}
+
+static int32_t
+rda_releasedir(xlator_t *this, fd_t *fd)
+{
+	uint64_t val;
+	struct rda_fd_ctx *ctx;
+
+	if (fd_ctx_del(fd, this, &val) < 0)
+		return -1;
+
+	ctx = (struct rda_fd_ctx *) val;
+	if (!ctx)
+		return 0;
+
+	rda_reset_ctx(ctx);
+
+	if (ctx->fill_frame)
+		STACK_DESTROY(ctx->fill_frame->root);
+
+	if (ctx->stub)
+		gf_log(this->name, GF_LOG_ERROR,
+			"released a directory with a pending stub");
+
+	GF_FREE(ctx);
+	return 0;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+	int ret = -1;
+
+	if (!this)
+		goto out;
+
+	ret = xlator_mem_acct_init(this, gf_rda_mt_end + 1);
+
+	if (ret != 0)
+		gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+		       "failed");
+
+out:
+	return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+	struct rda_priv *priv = this->private;
+
+	GF_OPTION_RECONF("rda-request-size", priv->rda_req_size, options,
+			 uint32, err);
+	GF_OPTION_RECONF("rda-low-wmark", priv->rda_low_wmark, options, size,
+			 err);
+	GF_OPTION_RECONF("rda-high-wmark", priv->rda_high_wmark, options, size,
+			 err);
+
+	return 0;
+err:
+	return -1;
+}
+
+int
+init(xlator_t *this)
+{
+	struct rda_priv *priv = NULL;
+
+        GF_VALIDATE_OR_GOTO("readdir-ahead", this, err);
+
+        if (!this->children || this->children->next) {
+                gf_log(this->name,  GF_LOG_ERROR,
+                        "FATAL: readdir-ahead not configured with exactly one"
+                        " child");
+                goto err;
+        }
+
+        if (!this->parents) {
+                gf_log(this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+	priv = GF_CALLOC(1, sizeof(struct rda_priv), gf_rda_mt_rda_priv);
+	if (!priv)
+		goto err;
+	this->private = priv;
+
+	this->local_pool = mem_pool_new(struct rda_local, 32);
+	if (!this->local_pool)
+		goto err;
+
+	GF_OPTION_INIT("rda-request-size", priv->rda_req_size, uint32, err);
+	GF_OPTION_INIT("rda-low-wmark", priv->rda_low_wmark, size, err);
+	GF_OPTION_INIT("rda-high-wmark", priv->rda_high_wmark, size, err);
+
+	return 0;
+
+err:
+	if (this->local_pool)
+		mem_pool_destroy(this->local_pool);
+	if (priv)
+		GF_FREE(priv);
+
+        return -1;
+}
+
+
+void
+fini(xlator_t *this)
+{
+        GF_VALIDATE_OR_GOTO ("readdir-ahead", this, out);
+
+	GF_FREE(this->private);
+
+out:
+        return;
+}
+
+struct xlator_fops fops = {
+	.opendir	= rda_opendir,
+	.readdirp	= rda_readdirp,
+};
+
+struct xlator_cbks cbks = {
+	.releasedir	= rda_releasedir,
+};
+
+struct volume_options options[] = {
+	{ .key = {"rda-request-size"},
+	  .type = GF_OPTION_TYPE_INT,
+	  .min = 4096,
+	  .max = 131072,
+	  .default_value = "131072",
+	  .description = "readdir-ahead request size",
+	},
+	{ .key = {"rda-low-wmark"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .min = 0,
+	  .max = 10 * GF_UNIT_MB,
+	  .default_value = "4096",
+	  .description = "the value under which we plug",
+	},
+	{ .key = {"rda-high-wmark"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .min = 0,
+	  .max = 100 * GF_UNIT_MB,
+	  .default_value = "131072",
+	  .description = "the value over which we unplug",
+	},
+        { .key = {NULL} },
+};
+
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead.h b/xlators/performance/readdir-ahead/src/readdir-ahead.h
new file mode 100644
index 000000000..e48786dae
--- /dev/null
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead.h
@@ -0,0 +1,46 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __READDIR_AHEAD_H
+#define __READDIR_AHEAD_H
+
+/* state flags */
+#define RDA_FD_NEW	(1 << 0)
+#define RDA_FD_RUNNING	(1 << 1)
+#define RDA_FD_EOD	(1 << 2)
+#define RDA_FD_ERROR	(1 << 3)
+#define RDA_FD_BYPASS	(1 << 4)
+#define RDA_FD_PLUGGED	(1 << 5)
+
+struct rda_fd_ctx {
+	off_t cur_offset;	/* current head of the ctx */
+	size_t cur_size;	/* current size of the preload */
+	off_t next_offset;	/* tail of the ctx */
+	uint32_t state;
+	gf_lock_t lock;
+	gf_dirent_t entries;
+	call_frame_t *fill_frame;
+	call_stub_t *stub;
+	int op_errno;
+};
+
+struct rda_local {
+	struct rda_fd_ctx *ctx;
+	fd_t *fd;
+	off_t offset;
+};
+
+struct rda_priv {
+	uint32_t rda_req_size;
+	uint64_t rda_low_wmark;
+	uint64_t rda_high_wmark;
+};
+
+#endif /* __READDIR_AHEAD_H */
diff --git a/xlators/performance/stat-prefetch/src/Makefile.am b/xlators/performance/stat-prefetch/src/Makefile.am
deleted file mode 100644
index cfb130714..000000000
--- a/xlators/performance/stat-prefetch/src/Makefile.am
+++ /dev/null
@@ -1,14 +0,0 @@
-xlator_LTLIBRARIES = stat-prefetch.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
-
-stat_prefetch_la_LDFLAGS = -module -avoidversion
-stat_prefetch_la_SOURCES = stat-prefetch.c
-noinst_HEADERS = stat-prefetch.h stat-prefetch-mem-types.h
-
-stat_prefetch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/rbtree -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/performance/stat-prefetch/src/stat-prefetch-mem-types.h b/xlators/performance/stat-prefetch/src/stat-prefetch-mem-types.h
deleted file mode 100644
index 156b3472b..000000000
--- a/xlators/performance/stat-prefetch/src/stat-prefetch-mem-types.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __SP_MEM_TYPES_H__
-#define __SP_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_sp_mem_types_ {
-        gf_sp_mt_sp_cache_t   = gf_common_mt_end + 1,
-        gf_sp_mt_sp_fd_ctx_t,
-        gf_sp_mt_stat,
-        gf_sp_mt_sp_local_t,
-        gf_sp_mt_sp_inode_ctx_t,
-        gf_sp_mt_sp_private_t,
-        gf_sp_mt_fd_wrapper_t,
-        gf_sp_mt_end
-};
-#endif
diff --git a/xlators/performance/stat-prefetch/src/stat-prefetch.c b/xlators/performance/stat-prefetch/src/stat-prefetch.c
deleted file mode 100644
index 25ed6fa57..000000000
--- a/xlators/performance/stat-prefetch/src/stat-prefetch.c
+++ /dev/null
@@ -1,4268 +0,0 @@
-/*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include "stat-prefetch.h"
-#include "statedump.h"
-#include "fd.h"
-
-#define GF_SP_CACHE_BUCKETS 1
-#define GF_SP_CACHE_ENTRIES_EXPECTED (128 * 1024) //1048576
-
-typedef enum {
-        SP_EXPECT,
-        SP_DONT_EXPECT,
-        SP_DONT_CARE
-} sp_expect_t;
-
-
-void
-sp_inode_ctx_free (xlator_t *this, sp_inode_ctx_t *ctx)
-{
-        call_stub_t *stub = NULL, *tmp = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
-
-        LOCK (&ctx->lock);
-        {
-                if (!list_empty (&ctx->waiting_ops)) {
-                        gf_log (this->name, GF_LOG_WARNING, "inode ctx is "
-                                "being freed even when there are file "
-                                "operations waiting for lookup-behind to "
-                                "complete. The operations in the waiting list "
-                                "are:");
-                        list_for_each_entry_safe (stub, tmp, &ctx->waiting_ops,
-                                                  list) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "OP (%s)", gf_fop_list[stub->fop]);
-
-                                list_del_init (&stub->list);
-                                call_stub_destroy (stub);
-                        }
-                }
-        }
-        UNLOCK (&ctx->lock);
-
-        LOCK_DESTROY (&ctx->lock);
-        GF_FREE (ctx);
-
-out:
-        return;
-}
-
-
-sp_inode_ctx_t *
-sp_inode_ctx_init ()
-{
-        sp_inode_ctx_t *inode_ctx = NULL;
-
-        inode_ctx = GF_CALLOC (1, sizeof (*inode_ctx), gf_sp_mt_sp_inode_ctx_t);
-        if (inode_ctx == NULL) {
-                goto out;
-        }
-
-        LOCK_INIT (&inode_ctx->lock);
-        INIT_LIST_HEAD (&inode_ctx->waiting_ops);
-
-out:
-        return inode_ctx;
-}
-
-
-int
-sp_update_inode_ctx (xlator_t *this, inode_t *inode, int32_t *op_ret,
-                     int32_t *op_errno, char *lookup_in_progress,
-                     char *looked_up, struct iatt *stbuf,
-                     struct list_head *waiting_ops, int32_t *error)
-{
-        int32_t         ret       = -1;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        uint64_t        value     = 0;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = inode_ctx_get (inode, this, &value);
-        if (ret == 0) {
-                inode_ctx = (sp_inode_ctx_t *)(long)value;
-        }
-
-        if (inode_ctx == NULL) {
-                ret = -1;
-                if (error != NULL) {
-                        *error = EINVAL;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode "
-                        "(gfid:%s)", uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        LOCK (&inode_ctx->lock);
-        {
-                if (op_ret != NULL) {
-                        inode_ctx->op_ret = *op_ret;
-                }
-
-                if (op_errno != NULL) {
-                        inode_ctx->op_errno = *op_errno;
-                }
-
-                if (looked_up != NULL) {
-                        inode_ctx->looked_up = *looked_up;
-                }
-
-                if (lookup_in_progress != NULL) {
-                        inode_ctx->lookup_in_progress = *lookup_in_progress;
-                }
-
-                if ((op_ret != NULL ) && (*op_ret == 0) && (stbuf != NULL)
-                    && IA_ISDIR (stbuf->ia_type)) {
-                        memcpy (&inode_ctx->stbuf, stbuf, sizeof (*stbuf));
-                }
-
-                if (waiting_ops != NULL) {
-                        list_splice_init (&inode_ctx->waiting_ops, waiting_ops);
-                }
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-
-sp_inode_ctx_t *
-sp_check_and_create_inode_ctx (xlator_t *this, inode_t *inode,
-                               sp_expect_t expect)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &value);
-                if (ret == 0) {
-                        inode_ctx = (sp_inode_ctx_t *)(long)value;
-
-                        if ((expect == SP_DONT_EXPECT) && (inode_ctx != NULL)) {
-                                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                                  "stat-prefetch context is "
-                                                  "present in inode "
-                                                  "(gfid:%s) "
-                                                  "when it is supposed to be "
-                                                  "not present",
-                                                  uuid_utoa (inode->gfid));
-                        }
-                } else {
-                        if (expect == SP_EXPECT) {
-                                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                                  "stat-prefetch context is "
-                                                  "not present in inode "
-                                                  "(gfid:%s)"
-                                                  " when it is supposed to be "
-                                                  "present",
-                                                  uuid_utoa (inode->gfid));
-                        }
-
-                        inode_ctx = sp_inode_ctx_init ();
-                        if (inode_ctx != NULL) {
-                                ret = __inode_ctx_put (inode, this,
-                                                       (long)inode_ctx);
-                                if (ret == -1) {
-                                        sp_inode_ctx_free (this, inode_ctx);
-                                        inode_ctx = NULL;
-                                }
-                        }
-                }
-        }
-        UNLOCK (&inode->lock);
-
-out:
-        return inode_ctx;
-}
-
-
-sp_cache_t *
-sp_cache_ref (sp_cache_t *cache)
-{
-        if (cache == NULL) {
-                goto out;
-        }
-
-        LOCK (&cache->lock);
-        {
-                cache->ref++;
-        }
-        UNLOCK (&cache->lock);
-
-out:
-        return cache;;
-}
-
-
-void
-sp_cache_unref (sp_cache_t *cache)
-{
-        int refcount = 0;
-
-        if (cache == NULL) {
-                goto out;
-        }
-
-        LOCK (&cache->lock);
-        {
-                refcount = --cache->ref;
-        }
-        UNLOCK (&cache->lock);
-
-        if (refcount == 0) {
-                rbthash_table_destroy (cache->table);
-                GF_FREE (cache);
-        }
-
-out:
-        return;
-}
-
-
-int32_t
-sp_process_inode_ctx (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      call_stub_t *stub, char *need_unwind, char *need_lookup,
-                      char *can_wind, int32_t *error)
-{
-        int32_t         ret          = -1, op_errno = EINVAL;
-        sp_local_t     *local        = NULL;
-        sp_inode_ctx_t *inode_ctx    = NULL;
-        uint64_t        value        = 0;
-
-        if (need_unwind != NULL) {
-                *need_unwind = 1;
-        }
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, need_unwind, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, need_lookup, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, can_wind, out);
-
-        inode_ctx_get (loc->inode, this, &value);
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        if (inode_ctx == NULL) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                  "stat-prefetch context not set in inode "
-                                  "(gfid:%s)", uuid_utoa (loc->inode->gfid));
-                *can_wind = 1;
-                *need_unwind = 0;
-                op_errno = 0;
-                ret = 0;
-                goto out;
-        }
-
-        LOCK (&inode_ctx->lock);
-        {
-                if (!(inode_ctx->looked_up || inode_ctx->lookup_in_progress)) {
-                        if (frame->local == NULL) {
-                                local = GF_CALLOC (1, sizeof (*local),
-                                                   gf_sp_mt_sp_local_t);
-                                if (local == NULL) {
-                                        op_errno = ENOMEM;
-                                        goto unlock;
-                                }
-
-                                frame->local = local;
-
-                                ret = loc_copy (&local->loc, loc);
-                                if (ret == -1) {
-                                        op_errno = errno;
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "loc_copy failed (%s)",
-                                                strerror (op_errno));
-                                        goto unlock;
-                                }
-                        }
-
-                        *need_lookup = 1;
-                        inode_ctx->lookup_in_progress = 1;
-                }
-
-                if (inode_ctx->looked_up) {
-                        *can_wind = 1;
-                } else {
-                        list_add_tail (&stub->list, &inode_ctx->waiting_ops);
-                        stub = NULL;
-                }
-
-                *need_unwind = 0;
-                ret = 0;
-        }
-unlock:
-        UNLOCK (&inode_ctx->lock);
-
-out:
-        if (stub != NULL) {
-                call_stub_destroy (stub);
-        }
-
-        if (error != NULL) {
-                *error = op_errno;
-        }
-
-        return ret;
-}
-
-
-inline uint32_t
-sp_hashfn (void *data, int len)
-{
-        return gf_dm_hashfn ((const char *)data, len);
-}
-
-
-sp_cache_t *
-sp_cache_init (xlator_t *this)
-{
-        sp_cache_t      *cache = NULL;
-        sp_private_t    *priv  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, priv->mem_pool, out);
-
-        cache = GF_CALLOC (1, sizeof (*cache), gf_sp_mt_sp_cache_t);
-        if (cache) {
-                cache->table =
-                        rbthash_table_init (GF_SP_CACHE_BUCKETS,
-                                            sp_hashfn, __gf_free,
-                                            0, priv->mem_pool);
-                if (cache->table == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot init a new rbthash table to hold "
-                                "cache");
-                        GF_FREE (cache);
-                        cache = NULL;
-                        goto out;
-                }
-
-                LOCK_INIT (&cache->lock);
-                cache->this = this;
-        }
-
-out:
-        return cache;
-}
-
-
-void
-sp_local_free (sp_local_t *local)
-{
-        if (local) {
-                loc_wipe (&local->loc);
-                GF_FREE (local);
-        }
-}
-
-
-int32_t
-sp_cache_remove_entry (sp_cache_t *cache, char *name, char remove_all)
-{
-        int32_t          ret   = -1;
-        rbthash_table_t *table = NULL;
-        xlator_t        *this  = NULL;
-        sp_private_t    *priv  = NULL;
-        void            *data  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", cache, out);
-        if ((name == NULL) && !remove_all) {
-                gf_log ((cache->this ? cache->this->name : "stat-prefetch"),
-                        GF_LOG_WARNING,
-                        "request to remove a single entry from cache and is no "
-                        "name passed to identify it");
-                goto out;
-        }
-
-        this = cache->this;
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        priv = this->private;
-
-        LOCK (&cache->lock);
-        {
-                if (remove_all) {
-                        table = cache->table;
-                        cache->table = rbthash_table_init (GF_SP_CACHE_BUCKETS,
-                                                           sp_hashfn, __gf_free,
-                                                           0, priv->mem_pool);
-                        if (cache->table == NULL) {
-                                cache->table = table;
-                        } else {
-                                rbthash_table_destroy (table);
-                                ret = 0;
-                                if (priv) {
-                                        LOCK (&priv->lock);
-                                        {
-                                                priv->entries = 0;
-                                        }
-                                        UNLOCK (&priv->lock);
-                                }
-                        }
-                } else {
-                        data = rbthash_remove (cache->table, name,
-                                               strlen (name));
-                        GF_FREE (data);
-                        ret = 0;
-                        if (priv) {
-                                LOCK (&priv->lock);
-                                {
-                                        priv->entries--;
-                                }
-                                UNLOCK (&priv->lock);
-                        }
-                }
-        }
-        UNLOCK (&cache->lock);
-
-out:
-        return ret;
-}
-
-
-int32_t
-sp_cache_get_entry (sp_cache_t *cache, char *name, gf_dirent_t **entry)
-{
-        int32_t      ret = -1;
-        gf_dirent_t *tmp = NULL, *new = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", cache, out);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", cache->this, out);
-        GF_VALIDATE_OR_GOTO (cache->this->name, name, out);
-        GF_VALIDATE_OR_GOTO (cache->this->name, entry, out);
-
-        LOCK (&cache->lock);
-        {
-                tmp = rbthash_get (cache->table, name, strlen (name));
-                if (tmp != NULL) {
-                        new = gf_dirent_for_name (tmp->d_name);
-                        if (new == NULL) {
-                                gf_log (cache->this->name, GF_LOG_WARNING,
-                                        "cannot create a new dentry to copy "
-                                        "from cache");
-                                goto unlock;
-                        }
-
-                        new->d_ino  = tmp->d_ino;
-                        new->d_off  = tmp->d_off;
-                        new->d_len  = tmp->d_len;
-                        new->d_type = tmp->d_type;
-                        new->d_stat = tmp->d_stat;
-
-                        *entry = new;
-                        ret = 0;
-                }
-        }
-unlock:
-        UNLOCK (&cache->lock);
-
-out:
-        return ret;
-}
-
-
-void
-sp_cache_free (sp_cache_t *cache)
-{
-        sp_cache_remove_entry (cache, NULL, 1);
-        sp_cache_unref (cache);
-}
-
-
-sp_cache_t *
-__sp_get_cache_fd (xlator_t *this, fd_t *fd)
-{
-        int32_t      ret   = -1;
-        sp_cache_t  *cache = NULL;
-        uint64_t     value = 0;
-        sp_fd_ctx_t *fd_ctx = NULL;
-
-        ret = __fd_ctx_get (fd, this, &value);
-        if (ret == -1) {
-                goto out;
-        }
-
-        fd_ctx = (void *)(long) value;
-
-        cache = fd_ctx->cache;
-
-out:
-        return cache;
-}
-
-
-sp_cache_t *
-sp_get_cache_fd (xlator_t *this, fd_t *fd)
-{
-        sp_cache_t  *cache     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        LOCK (&fd->lock);
-        {
-                cache = __sp_get_cache_fd (this, fd);
-                if (cache != NULL) {
-                        sp_cache_ref (cache);
-                }
-        }
-        UNLOCK (&fd->lock);
-
-out:
-        return cache;
-}
-
-
-void
-sp_fd_ctx_free (sp_fd_ctx_t *fd_ctx)
-{
-        if (fd_ctx == NULL) {
-                goto out;
-        }
-
-        if (fd_ctx->parent_inode) {
-                inode_unref (fd_ctx->parent_inode);
-                fd_ctx->parent_inode = NULL;
-        }
-
-        if (fd_ctx->name) {
-                GF_FREE (fd_ctx->name);
-                fd_ctx->name = NULL;
-        }
-
-        if (fd_ctx->cache) {
-                sp_cache_free (fd_ctx->cache);
-        }
-
-        GF_FREE (fd_ctx);
-out:
-        return;
-}
-
-
-inline sp_fd_ctx_t *
-sp_fd_ctx_init (void)
-{
-        sp_fd_ctx_t *fd_ctx = NULL;
-
-        fd_ctx = GF_CALLOC (1, sizeof (*fd_ctx), gf_sp_mt_sp_fd_ctx_t);
-
-        return fd_ctx;
-}
-
-
-sp_fd_ctx_t *
-sp_fd_ctx_new (xlator_t *this, inode_t *parent, char *name, sp_cache_t *cache)
-{
-        sp_fd_ctx_t *fd_ctx = NULL;
-
-        fd_ctx = sp_fd_ctx_init ();
-        if (fd_ctx == NULL) {
-                goto out;
-        }
-
-        if (parent) {
-                fd_ctx->parent_inode = inode_ref (parent);
-        }
-
-        if (name) {
-                fd_ctx->name = gf_strdup (name);
-                if (fd_ctx->name == NULL) {
-                        sp_fd_ctx_free (fd_ctx);
-                        fd_ctx = NULL;
-                        goto out;
-                }
-        }
-
-        fd_ctx->cache = cache;
-
-out:
-        return fd_ctx;
-}
-
-
-sp_cache_t *
-sp_del_cache_fd (xlator_t *this, fd_t *fd)
-{
-        sp_cache_t  *cache  = NULL;
-        uint64_t     value  = 0;
-        int32_t      ret    = -1;
-        sp_fd_ctx_t *fd_ctx = NULL;
-
-        if (fd == NULL) {
-                goto out;
-        }
-
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fd_ctx = (void *)(long) value;
-                        cache = fd_ctx->cache;
-                        fd_ctx->cache = NULL;
-                }
-        }
-        UNLOCK (&fd->lock);
-
-out:
-        return cache;
-}
-
-
-sp_cache_t *
-sp_get_cache_inode (xlator_t *this, inode_t *inode, int32_t pid)
-{
-        fd_t       *fd    = NULL;
-        sp_cache_t *cache = NULL;
-
-        if (inode == NULL) {
-                goto out;
-        }
-
-        fd = fd_lookup (inode, pid);
-        if (fd == NULL) {
-                goto out;
-        }
-
-        cache = sp_get_cache_fd (this, fd);
-
-        fd_unref (fd);
-out:
-        return cache;
-}
-
-
-void
-sp_remove_caches_from_all_fds_opened (xlator_t *this, inode_t *inode,
-                                      char *name)
-{
-        fd_t       *fd    = NULL;
-        sp_cache_t *cache = NULL;
-        struct fd_wrapper {
-                fd_t             *fd;
-                struct list_head  list;
-        };
-
-        struct fd_wrapper *wrapper    = NULL, *tmp = NULL;
-        struct list_head   head       = {0, };
-        char               remove_all = 0;
-
-        wrapper = NULL;
-
-        INIT_LIST_HEAD (&head);
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        remove_all = (name == NULL);
-
-        LOCK (&inode->lock);
-        {
-                list_for_each_entry (fd, &inode->fd_list, inode_list) {
-                        wrapper = GF_CALLOC (1, sizeof (*wrapper),
-                                             gf_sp_mt_fd_wrapper_t);
-                        if (wrapper == NULL) {
-                                goto unlock;
-                        }
-
-                        INIT_LIST_HEAD (&wrapper->list);
-
-                        wrapper->fd = __fd_ref (fd);
-                        list_add_tail (&wrapper->list, &head);
-                }
-        }
-unlock:
-        UNLOCK (&inode->lock);
-
-        list_for_each_entry_safe (wrapper, tmp, &head, list) {
-                cache = sp_get_cache_fd (this, wrapper->fd);
-                if (cache) {
-                        sp_cache_remove_entry (cache, name, remove_all);
-                        sp_cache_unref (cache);
-                }
-
-                list_del (&wrapper->list);
-                fd_unref (wrapper->fd);
-                GF_FREE (wrapper);
-        }
-
-out:
-        return;
-}
-
-
-inline int32_t
-__sp_put_cache (xlator_t *this, fd_t *fd, sp_cache_t *cache)
-{
-        sp_fd_ctx_t *fd_ctx = NULL;
-        int32_t      ret    = -1;
-        uint64_t     value  = 0;
-
-        ret = __fd_ctx_get (fd, this, &value);
-        if (!ret) {
-                fd_ctx = (void *)(long)value;
-        } else {
-                fd_ctx = sp_fd_ctx_init ();
-                if (fd_ctx == NULL) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = __fd_ctx_set (fd, this, (long)(void *)fd_ctx);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot set stat-prefetch context in fd (%p) "
-                                "opened on inode (gfid:%s)",
-                                fd, uuid_utoa (fd->inode->gfid));
-                        sp_fd_ctx_free (fd_ctx);
-                        goto out;
-                }
-        }
-
-        if (fd_ctx->cache) {
-                sp_cache_free (fd_ctx->cache);
-        }
-
-        fd_ctx->cache = cache;
-
-out:
-        return ret;
-}
-
-
-inline int32_t
-sp_put_cache (xlator_t *this, fd_t *fd, sp_cache_t *cache)
-{
-        int32_t ret = -1;
-
-        if (fd != NULL) {
-                LOCK (&fd->lock);
-                {
-                        ret = __sp_put_cache (this, fd, cache);
-                }
-                UNLOCK (&fd->lock);
-        }
-
-        return ret;
-}
-
-
-int32_t
-sp_cache_add_entries (sp_cache_t *cache, gf_dirent_t *entries)
-{
-        gf_dirent_t *entry           = NULL, *new = NULL;
-        int32_t      ret             = -1;
-        uint64_t     expected_offset = 0;
-        xlator_t     *this           = NULL;
-        sp_private_t *priv           = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", cache, out);
-
-        this = cache->this;
-        if (this && this->private) {
-                priv = this->private;
-        }
-
-        LOCK (&cache->lock);
-        {
-                list_for_each_entry (entry, &entries->list, list) {
-                        if (IA_ISDIR (entry->d_stat.ia_type)) {
-                                continue;
-                        }
-
-                        if (uuid_is_null (entry->d_stat.ia_gfid))
-                                continue;
-
-                        new = gf_dirent_for_name (entry->d_name);
-                        if (new == NULL) {
-                                gf_log (cache->this->name, GF_LOG_WARNING,
-                                        "cannot create a new dentry to store "
-                                        "in cache");
-                                goto unlock;
-                        }
-
-                        new->d_ino  = entry->d_ino;
-                        new->d_off  = entry->d_off;
-                        new->d_len  = entry->d_len;
-                        new->d_type = entry->d_type;
-                        new->d_stat = entry->d_stat;
-
-                        ret = rbthash_insert (cache->table, new, new->d_name,
-                                              strlen (new->d_name));
-                        if (ret == -1) {
-                                gf_log (this->name, GF_LOG_WARNING, "cannot "
-                                        "insert dentry (name:%s) into cache",
-                                        new->d_name);
-
-                                GF_FREE (new);
-                                continue;
-                        }
-
-                        expected_offset = new->d_off;
-                        if (priv) {
-                                LOCK (&priv->lock);
-                                {
-                                        priv->entries++;
-                                }
-                                UNLOCK (&priv->lock);
-                        }
-                }
-
-                cache->expected_offset = expected_offset;
-
-                ret = 0;
-        }
-unlock:
-        UNLOCK (&cache->lock);
-
-out:
-        return ret;
-}
-
-
-int32_t
-sp_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *dict, struct iatt *postparent)
-{
-        struct list_head     waiting_ops = {0, };
-        call_stub_t         *stub        = NULL, *tmp = NULL;
-        sp_local_t          *local       = NULL;
-        int                  need_unwind = 0;
-        char                 looked_up   = 0, lookup_in_progress = 0;
-
-        GF_ASSERT (frame);
-
-        INIT_LIST_HEAD (&waiting_ops);
-
-        local = frame->local;
-        if (local == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL, but it is "
-                        "needed to find and resume operations waiting on "
-                        "lookup");
-                goto out;
-        }
-
-        if (this == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (frame->this ? frame->this->name : "stat-prefetch",
-                        GF_LOG_WARNING, "xlator object (this) is NULL");
-                goto out;
-        }
-
-        /* For '/' Entry is never cached, don't try to remove it */
-        if ((op_ret == -1) && local->loc.parent) {
-                sp_remove_caches_from_all_fds_opened (this, local->loc.parent,
-                                                      (char *)local->loc.name);
-        }
-
-        if (local->is_lookup)
-                need_unwind = 1;
-
-        lookup_in_progress = 0;
-        looked_up = 1;
-        sp_update_inode_ctx (this, local->loc.inode, &op_ret, &op_errno,
-                             &lookup_in_progress, &looked_up, buf,
-                             &waiting_ops, &op_errno);
-
-        list_for_each_entry_safe (stub, tmp, &waiting_ops, list) {
-                list_del_init (&stub->list);
-                call_resume (stub);
-        }
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-                                 dict, postparent);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_get_ancestors (char *path, char **parent, char **grand_parent)
-{
-        int32_t  ret = -1, i = 0;
-        char    *cpy = NULL, *tmp = NULL;
-
-        if (!path || !parent || !grand_parent) {
-                ret = 0;
-                goto out;
-        }
-
-        for (i = 0; i < 2; i++) {
-                if (!strcmp (path, "/")) {
-                        break;
-                }
-
-                tmp = cpy;
-
-                cpy = gf_strdup (path);
-
-                if (tmp != NULL) {
-                        GF_FREE (tmp);
-                }
-
-                if (cpy == NULL) {
-                        ret = -errno;
-                        goto out;
-                }
-
-                path = dirname (cpy);
-                switch (i)
-                {
-                case 0:
-                        *parent = gf_strdup (path);
-			if (*parent == NULL)
-				goto out;
-                        break;
-                case 1:
-                        *grand_parent = gf_strdup (path);
-			if (*grand_parent == NULL)
-				goto out;
-                        break;
-                }
-        }
-
-        ret = 0;
-out:
-	if (cpy != NULL)
-		GF_FREE(cpy);
-        return ret;
-}
-
-
-int32_t
-sp_cache_remove_parent_entry (call_frame_t *frame, xlator_t *this,
-                              inode_table_t *itable, char *path)
-{
-        char       *parent    = NULL, *grand_parent = NULL, *cpy = NULL;
-        inode_t    *inode_gp  = NULL;
-        int32_t     ret       = -1;
-
-        ret = sp_get_ancestors (path, &parent, &grand_parent);
-        if (ret < 0) {
-                goto out;
-        }
-
-        if (grand_parent && strcmp (grand_parent, "/")) {
-                inode_gp = inode_from_path (itable, grand_parent);
-                if (inode_gp) {
-                        cpy = gf_strdup (parent);
-                        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name,
-                                                        cpy, out, ret,
-                                                        -ENOMEM);
-                        path = basename (cpy);
-                        sp_remove_caches_from_all_fds_opened (this, inode_gp,
-                                                              path);
-                        GF_FREE (cpy);
-
-                        inode_unref (inode_gp);
-                }
-        }
-
-        ret = 0;
-out:
-        if (parent) {
-                GF_FREE (parent);
-        }
-
-        if (grand_parent) {
-                GF_FREE (grand_parent);
-        }
-
-        return ret;
-}
-
-
-void
-sp_is_empty (dict_t *this, char *key, data_t *value, void *data)
-{
-        char *ptr = data;
-
-        if (strcmp (key, "gfid-req") == 0)
-                return;
-
-        if (ptr && *ptr) {
-                *ptr = 0;
-        }
-}
-
-
-int32_t
-sp_lookup_helper (call_frame_t *frame,xlator_t *this, loc_t *loc,
-                  dict_t *xattr_req)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-        call_stub_t    *stub      = NULL;
-        char            can_wind  = 0;
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode "
-                        "(gfid:%s)", uuid_utoa (loc->inode->gfid));
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, inode_ctx, unwind, op_errno,
-                                        EINVAL);
-
-        stub = fop_lookup_stub (frame, sp_lookup_helper, loc, xattr_req);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, stub, unwind,
-                                        op_errno, ENOMEM);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-                if (op_ret == 0) {
-                        if (!inode_ctx->lookup_in_progress) {
-                                inode_ctx->lookup_in_progress = 1;
-                                can_wind = 1;
-                        } else {
-                                list_add_tail (&stub->list,
-                                               &inode_ctx->waiting_ops);
-                                stub = NULL;
-                        }
-                }
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                goto unwind;
-        }
-
-        if (can_wind) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc,
-                            xattr_req);
-        }
-
-        if (stub != NULL) {
-                call_stub_destroy (stub);
-        }
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        if (stub != NULL) {
-                call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-/*
- * TODO: implement sending lookups for every fop done on this path. As of now
- * lookup on the path is sent only for the first fop on this path.
- */
-int32_t
-sp_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
-{
-        gf_dirent_t    *dirent          = NULL;
-        char            entry_cached    = 0;
-        uint64_t        value           = 0;
-        char            xattr_req_empty = 1, can_wind = 0;
-        sp_cache_t     *cache           = NULL;
-        struct iatt     postparent      = {0, }, buf = {0, };
-        int32_t         ret             = -1, op_ret = -1, op_errno = EINVAL;
-        sp_inode_ctx_t *inode_ctx       = NULL, *parent_inode_ctx = NULL;
-        sp_local_t     *local           = NULL;
-        call_stub_t    *stub            = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc->inode, unwind);
-
-        inode_ctx = sp_check_and_create_inode_ctx (this, loc->inode,
-                                                   SP_DONT_CARE);
-        if (inode_ctx == NULL) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot create stat-prefetch context in inode "
-                        "(gfid:%s)(%s)",
-                        uuid_utoa (loc->inode->gfid), strerror (op_errno));
-                goto unwind;
-        }
-
-        if ((loc->parent == NULL) || (loc->name == NULL)) {
-                goto wind;
-        }
-
-        if (xattr_req != NULL) {
-                dict_foreach (xattr_req, sp_is_empty, &xattr_req_empty);
-        }
-
-        if (!xattr_req_empty) {
-                goto wind;
-        }
-
-        cache = sp_get_cache_inode (this, loc->parent, frame->root->pid);
-        if (cache) {
-                ret = sp_cache_get_entry (cache, (char *)loc->name, &dirent);
-                if (ret == 0) {
-                        if (!uuid_is_null (loc->inode->gfid)
-                            && (uuid_compare (loc->inode->gfid,
-                                              dirent->d_stat.ia_gfid))
-                                != 0) {
-                                op_ret = -1;
-                                op_errno = ESTALE;
-                                goto unwind;
-                        }
-
-                        ret = inode_ctx_get (loc->parent, this, &value);
-                        if ((ret == 0) && (value != 0)) {
-                                parent_inode_ctx = (void *)(long)value;
-                                postparent = parent_inode_ctx->stbuf;
-                                buf = dirent->d_stat;
-                                op_ret = 0;
-                                op_errno = 0;
-                                entry_cached = 1;
-                        }
-
-                        GF_FREE (dirent);
-                }
-        } else if (IA_ISDIR (loc->inode->ia_type)) {
-                cache = sp_get_cache_inode (this, loc->inode, frame->root->pid);
-                if (cache) {
-                        ret = sp_cache_get_entry (cache, ".", &dirent);
-                        if (ret == 0) {
-                                ret = inode_ctx_get (loc->parent, this, &value);
-                                if ((ret == 0) && (value != 0)) {
-                                        parent_inode_ctx = (void *)(long)value;
-                                        postparent = parent_inode_ctx->stbuf;
-                                        buf = dirent->d_stat;
-                                        op_ret = 0;
-                                        op_errno = 0;
-                                        entry_cached = 1;
-                                }
-
-                                GF_FREE (dirent);
-                        }
-                }
-        }
-
-wind:
-        if (entry_cached) {
-                if (cache) {
-                        cache->hits++;
-                        sp_cache_unref (cache);
-                }
-        } else {
-                if (cache) {
-                        cache->miss++;
-                        sp_cache_unref (cache);
-                }
-
-                stub = fop_lookup_stub (frame, sp_lookup_helper, loc,
-                                        xattr_req);
-                GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, stub, unwind,
-                                                op_errno, ENOMEM);
-
-                local = GF_CALLOC (1, sizeof (*local), gf_sp_mt_sp_local_t);
-                GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, unwind,
-                                                op_errno, ENOMEM);
-
-                frame->local = local;
-
-                ret = loc_copy (&local->loc, loc);
-                if (ret == -1) {
-                        op_errno = errno;
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "loc_copy failed (%s)", strerror (op_errno));
-                        goto unwind;
-                }
-
-                local->is_lookup = 1;
-
-                LOCK (&inode_ctx->lock);
-                {
-                        if (inode_ctx->lookup_in_progress) {
-                                list_add_tail (&stub->list,
-                                               &inode_ctx->waiting_ops);
-                                stub = NULL;
-                        } else {
-                                can_wind = 1;
-                                inode_ctx->lookup_in_progress = 1;
-                        }
-                }
-                UNLOCK (&inode_ctx->lock);
-
-                if (can_wind) {
-                        STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->lookup, loc,
-                                    xattr_req);
-                }
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-
-                return 0;
-        }
-
-unwind:
-        SP_STACK_UNWIND (lookup, frame, op_ret, op_errno, (loc)?loc->inode:NULL,
-                         &buf, NULL, &postparent);
-
-        return 0;
-}
-
-
-int32_t
-sp_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        sp_local_t      *local       = NULL;
-        sp_cache_t      *cache       = NULL;
-        fd_t            *fd          = NULL;
-        int32_t          ret         = 0;
-        char             was_present = 1;
-        sp_private_t    *priv        = NULL;
-
-        GF_ASSERT (frame);
-        if (op_ret == -1) {
-                goto out;
-        }
-
-        if ((this == NULL) || (this->private == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "stat-prefetch configuration (this->private) is "
-                        "NULL");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (frame->this->name, GF_LOG_WARNING, "local is NULL");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        fd = local->fd;
-
-        priv = this->private;
-
-        LOCK (&priv->lock);
-        {
-                if (!priv->mem_pool)
-                        priv->mem_pool =  mem_pool_new (rbthash_entry_t,
-                                                        GF_SP_CACHE_ENTRIES_EXPECTED);
-        }
-        UNLOCK (&priv->lock);
-
-        if (!priv->mem_pool)
-                goto out;
-
-        LOCK (&fd->lock);
-        {
-                cache = __sp_get_cache_fd (this, fd);
-                if (cache == NULL) {
-                        was_present = 0;
-                        cache = sp_cache_init (this);
-                        if (cache == NULL) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "creation of stat-prefetch cache "
-                                        "for fd (%p) opened on inode "
-                                        "(gfid:%s) failed", fd,
-                                        uuid_utoa (fd->inode->gfid));
-                                goto unlock;
-                        }
-
-                        ret = __sp_put_cache (this, fd, cache);
-                        if (ret == -1) {
-                                sp_cache_free (cache);
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot store cache in fd (%p) opened "
-                                        "on inode (gfid:%s)", fd,
-                                        uuid_utoa (fd->inode->gfid));
-                                goto unlock;
-                        }
-                }
-
-                sp_cache_ref (cache);
-        }
-unlock:
-        UNLOCK (&fd->lock);
-
-        if (cache != NULL) {
-                sp_cache_add_entries (cache, entries);
-                if (was_present) {
-                        sp_cache_unref (cache);
-                }
-        }
-
-out:
-        SP_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries);
-        return 0;
-}
-
-
-int32_t
-sp_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t off, dict_t *dict)
-{
-        sp_cache_t *cache    = NULL;
-        sp_local_t *local    = NULL;
-        char       *path     = NULL;
-        int32_t     ret      = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        cache = sp_get_cache_fd (this, fd);
-        if (cache) {
-                if (off != cache->expected_offset) {
-                        sp_cache_remove_entry (cache, NULL, 1);
-                }
-
-                sp_cache_unref (cache);
-        }
-
-        ret = inode_path (fd->inode, NULL, &path);
-        if (ret < 0) {
-                op_errno = -ret;
-                gf_log (this->name, GF_LOG_WARNING, "cannot construct path on "
-                        "which fd (%p) is opened (gfid = %s) (%s)", fd,
-                        uuid_utoa (fd->inode->gfid), strerror (op_errno));
-                goto unwind;
-        }
-
-        ret = sp_cache_remove_parent_entry (frame, this, fd->inode->table,
-                                            path);
-
-        if (ret < 0) {
-                op_errno = -ret;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot remove parent entry from grand-parent's cache"
-                        " for path %s", path);
-                goto unwind;
-        }
-
-        GF_FREE (path);
-
-        local = GF_CALLOC (1, sizeof (*local), gf_sp_mt_sp_local_t);
-        if (local) {
-                local->fd = fd;
-                frame->local = local;
-        }
-
-        STACK_WIND (frame, sp_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp, fd, size, off, dict);
-
-        return 0;
-
-unwind:
-        if (path != NULL) {
-                GF_FREE (path);
-        }
-
-        SP_STACK_UNWIND (readdir, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-int32_t
-sp_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-            off_t off)
-{
-        return sp_readdirp (frame, this, fd, size, off, NULL);
-}
-
-int32_t
-sp_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf)
-{
-        GF_ASSERT (frame);
-
-        SP_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf, postbuf);
-        return 0;
-}
-
-
-
-int32_t
-sp_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf,
-               struct iatt *preoldparent, struct iatt *postoldparent,
-               struct iatt *prenewparent, struct iatt *postnewparent)
-{
-        GF_ASSERT (frame);
-        SP_STACK_UNWIND (rename, frame, op_ret, op_errno, buf, preoldparent,
-                         postoldparent, prenewparent, postnewparent);
-        return 0;
-}
-
-
-int32_t
-sp_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-           int32_t op_errno, fd_t *fd)
-{
-        sp_local_t  *local = NULL;
-        sp_fd_ctx_t *fd_ctx = NULL;
-
-        GF_ASSERT (frame);
-
-        if (op_ret == -1) {
-                goto out;
-        }
-
-        if (this == NULL) {
-                gf_log (frame->this ? frame->this->name : "stat-prefetch",
-                        GF_LOG_WARNING, "xlator object (this) is NULL");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (fd == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "fd is NULL");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        fd_ctx = sp_fd_ctx_new (this, local->loc.parent,
-                                (char *)local->loc.name, NULL);
-        if (fd_ctx == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        op_ret = fd_ctx_set (fd, this, (long)(void *)fd_ctx);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot set stat-prefetch context in fd (%p) opened on "
-                        "inode (gfid:%s)", fd, uuid_utoa (fd->inode->gfid));
-                sp_fd_ctx_free (fd_ctx);
-                op_errno = ENOMEM;
-        }
-
-out:
-        SP_STACK_UNWIND (open, frame, op_ret, op_errno, fd);
-        return 0;
-}
-
-
-int32_t
-sp_open_helper (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-                fd_t *fd, int32_t wbflags)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode "
-                        "(gfid:%s)", uuid_utoa (loc->inode->gfid));
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, inode_ctx, unwind, op_errno,
-                                        EINVAL);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if ((op_ret == -1) && ((op_errno != ENOENT)
-                               || !((op_errno == ENOENT)
-                                    && (flags & O_CREAT)))) {
-                gf_log (this->name, GF_LOG_WARNING, "lookup-behind has failed "
-                        "for path (%s)(%s), unwinding open call waiting on "
-                        "it", loc->path, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_fd_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (open, frame, -1, op_errno, fd);
-        return 0;
-}
-
-
-int32_t
-sp_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int wbflags)
-{
-        call_stub_t    *stub         = NULL;
-        sp_local_t     *local        = NULL;
-        int32_t         op_errno     = EINVAL, ret = -1;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        local = GF_CALLOC (1, sizeof (*local), gf_sp_mt_sp_local_t);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, out, op_errno,
-                                        ENOMEM);
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret == -1) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed (%s)",
-                        strerror (op_errno));
-                goto out;
-        }
-
-        stub = fop_open_stub (frame, sp_open_helper, loc, flags, fd, wbflags);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (open, frame, -1, op_errno, fd);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_fd_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, loc, flags, fd,
-                            wbflags);
-        }
-
-        return 0;
-
-}
-
-static int32_t
-sp_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
-{
-        sp_local_t     *local              = NULL;
-        sp_fd_ctx_t    *fd_ctx             = NULL;
-        char            lookup_in_progress = 0, looked_up = 0;
-
-        GF_ASSERT (frame);
-
-        if (op_ret == -1) {
-                goto out;
-        }
-
-        if (this == NULL) {
-                gf_log (frame->this ? frame->this->name : "stat-prefetch",
-                        GF_LOG_WARNING, "xlator object (this) is NULL");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        looked_up = 1;
-        op_ret = sp_update_inode_ctx (this, local->loc.inode, &op_ret,
-                                      &op_errno, &lookup_in_progress,
-                                      &looked_up, buf, NULL, &op_errno);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "updating stat-prefetch context in inode "
-                        "(gfid:%s) (path: %s) failed (%s)",
-                        uuid_utoa (local->loc.inode->gfid), local->loc.path,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        op_ret = sp_update_inode_ctx (this, local->loc.parent, NULL, NULL, NULL,
-                                      NULL, postparent, NULL, &op_errno);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "updating stat-prefetch context in parent inode failed "
-                        "for path (%s)(%s)", local->loc.path,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        fd_ctx = sp_fd_ctx_new (this, local->loc.parent,
-                                (char *)local->loc.name, NULL);
-        if (fd_ctx == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        op_ret = fd_ctx_set (fd, this, (long)(void *)fd_ctx);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot set stat-prefetch context in fd (%p) opened on "
-                        "inode (gfid:%s)", fd, uuid_utoa (fd->inode->gfid));
-                sp_fd_ctx_free (fd_ctx);
-                op_errno = ENOMEM;
-        }
-
-out:
-        SP_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
-                         preparent, postparent);
-        return 0;
-}
-
-
-int32_t
-sp_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, fd_t *fd, dict_t *params)
-{
-        sp_local_t        *local        = NULL;
-        int32_t            op_errno     = -1, ret = -1;
-        char               need_unwind  = 1;
-        sp_inode_ctx_t    *inode_ctx    = NULL;
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, loc, out, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, loc->path, out, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, loc->name, out, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, loc->inode, out,
-                                        op_errno, EINVAL);
-
-        ret = sp_cache_remove_parent_entry (frame, this, loc->inode->table,
-                                            (char *)loc->path);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_sp_mt_sp_local_t);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, out, op_errno,
-                                        ENOMEM);
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret == -1) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "loc_copy failed (%s)", strerror (op_errno));
-                goto out;
-        }
-
-        inode_ctx = sp_check_and_create_inode_ctx (this, loc->inode,
-                                                   SP_DONT_EXPECT);
-        if (inode_ctx == NULL) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot create stat-prefetch context in inode "
-                        "(gfid:%s)(%s)", uuid_utoa (loc->inode->gfid),
-                        strerror (op_errno));
-                goto out;
-        }
-
-        need_unwind = 0;
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                                 NULL, NULL);
-        } else {
-                STACK_WIND (frame, sp_create_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->create, loc, flags,
-                            mode, fd, params);
-        }
-        return 0;
-}
-
-
-int32_t
-sp_opendir_helper (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode "
-                        "(gfid:%s)", uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "lookup-behind has failed "
-                        "for path (%s)(%s), unwinding opendir call waiting "
-                        "on it", loc->path, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_fd_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (opendir, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
-{
-        sp_local_t     *local        = NULL;
-        call_stub_t    *stub         = NULL;
-        int32_t         op_errno     = EINVAL, ret = -1;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        local = GF_CALLOC (1, sizeof (*local), gf_sp_mt_sp_local_t);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, out, op_errno,
-                                        ENOMEM);
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret == -1) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed (%s)",
-                        strerror (op_errno));
-                goto out;
-        }
-
-        stub = fop_opendir_stub (frame, sp_opendir_helper, loc, fd);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (opendir, frame, -1, op_errno, NULL);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_fd_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->opendir, loc, fd);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_new_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        sp_local_t *local              = NULL;
-        char        lookup_in_progress = 0, looked_up = 0;
-
-        GF_ASSERT (frame);
-
-        if (op_ret == -1) {
-                goto out;
-        }
-
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (frame->this->name, GF_LOG_WARNING, "local is NULL");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (this == NULL) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "xlator object (this) is NULL");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        looked_up = 1;
-        op_ret = sp_update_inode_ctx (this, local->loc.inode, &op_ret,
-                                      &op_errno, &lookup_in_progress,
-                                      &looked_up, buf, NULL, &op_errno);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "updating stat-prefetch context in inode "
-                        "(gfid:%s) (path: %s) failed (%s)",
-                        uuid_utoa (local->loc.inode->gfid), local->loc.path,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        op_ret = sp_update_inode_ctx (this, local->loc.parent, NULL, NULL, NULL,
-                                      NULL, postparent, NULL, &op_errno);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "updating stat-prefetch context in parent inode failed "
-                        "for path (%s)(%s)", local->loc.path,
-                        strerror (op_errno));
-        }
-
-out:
-        SP_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf, preparent,
-                         postparent);
-        return 0;
-}
-
-
-int
-sp_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dict_t *params)
-{
-        int32_t         ret          = -1, op_errno = EINVAL;
-        char            need_unwind  = 1;
-        sp_inode_ctx_t *inode_ctx    = NULL;
-        sp_local_t     *local        = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->path, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = sp_cache_remove_parent_entry (frame, this, loc->inode->table,
-                                            (char *)loc->path);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot remove parent entry from grand-parent's cache "
-                        "for path (%s)", loc->path);
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_sp_mt_sp_local_t);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, out, op_errno,
-                                        ENOMEM);
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret == -1) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed (%s)",
-                        strerror (op_errno));
-                goto out;
-        }
-
-        inode_ctx = sp_check_and_create_inode_ctx (this, loc->inode,
-                                                   SP_DONT_EXPECT);
-        if (inode_ctx == NULL) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot create stat-prefetch context in inode "
-                        "(gfid:%s)(%s)", uuid_utoa (loc->inode->gfid),
-                        strerror (op_errno));
-                goto out;
-        }
-
-        need_unwind = 0;
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                                 NULL);
-        } else {
-                STACK_WIND (frame, sp_new_entry_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mkdir, loc, mode, params);
-        }
-
-        return 0;
-}
-
-
-int
-sp_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, dict_t *params)
-{
-        int32_t         op_errno     = EINVAL, ret = -1;
-        char            need_unwind  = 1;
-        sp_inode_ctx_t *inode_ctx    = NULL;
-        sp_local_t     *local        = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->path, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = sp_cache_remove_parent_entry (frame, this, loc->inode->table,
-                                            (char *)loc->path);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot remove parent entry from grand-parent's cache "
-                        "for path (%s)", loc->path);
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_sp_mt_sp_local_t);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, out, op_errno,
-                                        ENOMEM);
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret == -1) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed (%s)",
-                        strerror (op_errno));
-                goto out;
-        }
-
-        inode_ctx = sp_check_and_create_inode_ctx (this, loc->inode,
-                                                   SP_DONT_EXPECT);
-        if (inode_ctx == NULL) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot create stat-prefetch context in inode "
-                        "(gfid:%s)(%s)", uuid_utoa (loc->inode->gfid),
-                        strerror (op_errno));
-                goto out;
-        }
-
-        need_unwind = 0;
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL,
-                                 NULL);
-        } else {
-                STACK_WIND (frame, sp_new_entry_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mknod, loc, mode,
-                            rdev, params);
-        }
-
-        return 0;
-}
-
-
-int
-sp_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-            loc_t *loc, dict_t *params)
-{
-        int32_t         ret          = -1, op_errno = EINVAL;
-        char            need_unwind  = 1;
-        sp_inode_ctx_t *inode_ctx    = NULL;
-        sp_local_t     *local        = NULL;
-
-        GF_ASSERT (frame);
-
-        GF_VALIDATE_OR_GOTO ((frame->this ? frame->this->name
-                              : "stat-prefetch"),
-                             this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->path, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = sp_cache_remove_parent_entry (frame, this, loc->inode->table,
-                                            (char *)loc->path);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot remove parent entry from grand-parent's cache "
-                        "for path (%s)", loc->path);
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_sp_mt_sp_local_t);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, out, op_errno,
-                                        ENOMEM);
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret == -1) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed (%s)",
-                        strerror (op_errno));
-                goto out;
-        }
-
-        inode_ctx = sp_check_and_create_inode_ctx (this, loc->inode,
-                                                   SP_DONT_EXPECT);
-        if (inode_ctx == NULL) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: cannot create stat-prefetch context (gfid:%s)(%s)",
-                        loc->path, loc->inode->gfid, strerror (op_errno));
-                goto out;
-        }
-
-        need_unwind = 0;
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
-                                 NULL);
-        } else {
-                STACK_WIND (frame, sp_new_entry_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->symlink, linkpath, loc,
-                            params);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, inode_t *inode,
-             struct iatt *buf, struct iatt *preparent,
-             struct iatt *postparent)
-{
-        GF_ASSERT (frame);
-        SP_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf, preparent,
-                         postparent);
-        return 0;
-}
-
-
-int32_t
-sp_link_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, newloc, unwind);
-
-        ret = inode_ctx_get (oldloc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (oldloc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, inode_ctx, unwind, op_errno,
-                                        EINVAL);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "lookup-behind has failed "
-                        "for path (%s)(%s), unwinding link call waiting on "
-                        "it", oldloc->path, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
-{
-        call_stub_t *stub         = NULL;
-        int32_t      ret          = 0, op_errno = EINVAL;
-        char         can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, out);
-        GF_VALIDATE_OR_GOTO (this->name, newloc, out);
-        GF_VALIDATE_OR_GOTO (this->name, newloc->path, out);
-        if (uuid_is_null (newloc->gfid))
-            GF_VALIDATE_OR_GOTO (this->name, newloc->name, out);
-        GF_VALIDATE_OR_GOTO (this->name, newloc->inode, out);
-        if (uuid_is_null (oldloc->gfid))
-                GF_VALIDATE_OR_GOTO (this->name, oldloc->name, out);
-
-        ret = sp_cache_remove_parent_entry (frame, this, newloc->parent->table,
-                                            (char *)newloc->path);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot remove parent entry from grand-parent's cache "
-                        "for path (%s)", newloc->path);
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_remove_caches_from_all_fds_opened (this, oldloc->parent,
-                                              (char *)oldloc->name);
-
-        stub = fop_link_stub (frame, sp_link_helper, oldloc, newloc);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, oldloc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL,
-                                 NULL);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, oldloc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_link_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->link, oldloc, newloc);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_truncate_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, inode_ctx, unwind, op_errno,
-                                        EINVAL);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "lookup-behind has failed "
-                        "for path (%s)(%s), unwinding truncate call "
-                        "waiting on it", loc->path, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
-{
-        int32_t         op_errno     = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        if (uuid_is_null (loc->gfid))
-                GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-
-        sp_remove_caches_from_all_fds_opened (this, loc->parent,
-                                              (char *)loc->name);
-
-        stub = fop_truncate_stub (frame, sp_truncate_helper, loc, offset);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_truncate_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, loc, offset);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
-{
-        sp_fd_ctx_t *fd_ctx = NULL;
-        uint64_t     value  = 0;
-        int32_t      ret    = 0, op_errno = EINVAL;
-        inode_t     *parent = NULL;
-        char        *name   = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "stat-prefetch context not "
-                        "set in fd (%p) opened on inode (gfid:%s)", fd,
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-        fd_ctx = (void *)(long)value;
-        name   = fd_ctx->name;
-        parent = fd_ctx->parent_inode;
-
-        sp_remove_caches_from_all_fds_opened (this, parent, (char *)name);
-
-        STACK_WIND (frame, sp_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                struct iatt *prestat, struct iatt *poststat)
-{
-        GF_ASSERT (frame);
-        SP_STACK_UNWIND (setattr, frame, op_ret, op_errno, prestat, poststat);
-        return 0;
-}
-
-
-int
-sp_setattr_helper (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, struct iatt *buf, int32_t valid)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "lookup-behind has failed "
-                        "for path (%s)(%s), unwinding setattr call "
-                        "waiting on it", loc->path, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, buf, valid);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int
-sp_setattr (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, struct iatt *buf, int32_t valid)
-{
-        int32_t         op_errno     = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        if (uuid_is_null (loc->gfid))
-            GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-
-        sp_remove_caches_from_all_fds_opened (this, loc->parent,
-                                              (char *)loc->name);
-
-        stub = fop_setattr_stub (frame, sp_setattr_helper, loc, buf, valid);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_setattr_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->setattr, loc, buf, valid);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, const char *path,
-                 struct iatt *buf)
-{
-        GF_ASSERT (frame);
-        SP_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, buf);
-        return 0;
-}
-
-
-int32_t
-sp_readlink_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    size_t size)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "lookup-behind has failed "
-                        "for path (%s)(%s), unwinding readlink call "
-                        "waiting on it", loc->path, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_readlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink, loc, size);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
-{
-        int32_t         op_errno     = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        if (uuid_is_null (loc->gfid))
-                GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-
-        sp_remove_caches_from_all_fds_opened (this, loc->parent,
-                                              (char *)loc->name);
-
-        stub = fop_readlink_stub (frame, sp_readlink_helper, loc, size);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_readlink_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readlink, loc, size);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-               struct iatt *postparent)
-{
-        GF_ASSERT (frame);
-        SP_STACK_UNWIND (unlink, frame, op_ret, op_errno, preparent,
-                         postparent);
-        return 0;
-}
-
-
-
-int32_t
-sp_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno)
-{
-        GF_ASSERT (frame);
-        SP_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int32_t
-sp_unlink_helper (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "lookup-behind has failed "
-                        "for path (%s)(%s), unwinding unlink call "
-                        "waiting on it", loc->path, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        int32_t         ret          = -1, op_errno = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-
-        sp_remove_caches_from_all_fds_opened (this, loc->parent,
-                                              (char *)loc->name);
-
-        ret = sp_cache_remove_parent_entry (frame, this, loc->parent->table,
-                                            (char *)loc->path);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot remove parent entry from grand-parent's cache "
-                        "for path (%s)", loc->path);
-                goto out;
-        }
-
-        stub = fop_unlink_stub (frame, sp_unlink_helper, loc);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_unlink_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc);
-        }
-
-        return 0;
-}
-
-
-int
-sp_rmdir_helper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "lookup-behind has failed "
-                        "for path (%s)(%s), unwinding rmdir call "
-                        "waiting on it", loc->path, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int
-sp_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
-{
-        int32_t         ret          = -1, op_errno = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->path, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        sp_remove_caches_from_all_fds_opened (this, loc->inode, NULL);
-
-        ret = sp_cache_remove_parent_entry (frame, this, loc->inode->table,
-                                            (char *)loc->path);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot remove parent entry from grand-parent's cache "
-                        "for path (%s)", loc->path);
-                goto out;
-        }
-
-        stub = fop_rmdir_stub (frame, sp_rmdir_helper, loc, flags);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_unlink_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rmdir, loc, flags);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iovec *vector, int32_t count,
-              struct iatt *stbuf, struct iobref *iobref)
-{
-        GF_ASSERT (frame);
-        SP_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
-                         iobref);
-        return 0;
-}
-
-
-int32_t
-sp_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags)
-{
-        sp_fd_ctx_t *fd_ctx = NULL;
-        uint64_t     value  = 0;
-        int32_t      ret    = 0, op_errno = EINVAL;
-        inode_t     *parent = NULL;
-        char        *name   = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "stat-prefetch context not "
-                        "set in fd (%p) opened on inode (gfid:%s)", fd,
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-        fd_ctx = (void *)(long)value;
-        name   = fd_ctx->name;
-        parent = fd_ctx->parent_inode;
-
-        sp_remove_caches_from_all_fds_opened (this, parent, (char *)name);
-
-        STACK_WIND (frame, sp_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags);
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (readv, frame, -1, op_errno, NULL, -1, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t flags, struct iobref *iobref)
-{
-        sp_fd_ctx_t *fd_ctx = NULL;
-        uint64_t     value  = 0;
-        int32_t      ret    = 0, op_errno = EINVAL;
-        inode_t     *parent = NULL;
-        char        *name   = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "stat-prefetch context not "
-                        "set in fd (%p) opened on inode (gfid:%s)", fd,
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-        fd_ctx = (void *)(long)value;
-        name   = fd_ctx->name;
-        parent = fd_ctx->parent_inode;
-
-        sp_remove_caches_from_all_fds_opened (this, parent, (char *)name);
-
-        STACK_WIND (frame, sp_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
-                    flags, iobref);
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
-{
-        sp_fd_ctx_t *fd_ctx = NULL;
-        uint64_t     value  = 0;
-        int32_t      ret    = 0, op_errno = EINVAL;
-        inode_t     *parent = NULL;
-        char        *name   = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "stat-prefetch context not "
-                        "set in fd (%p) opened on inode (gfid:%s)", fd,
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-        fd_ctx = (void *)(long)value;
-        name   = fd_ctx->name;
-        parent = fd_ctx->parent_inode;
-
-        sp_remove_caches_from_all_fds_opened (this, parent, (char *)name);
-
-        STACK_WIND (frame, sp_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, flags);
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_rename_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                  loc_t *newloc)
-{
-        uint64_t        value               = 0;
-        char            need_unwind         = 0;
-        char            can_wind            = 0;
-        int32_t         ret                 = 0, op_errno = EINVAL;
-        int32_t         old_op_ret          = -1, old_op_errno = -1;
-        int32_t         new_op_ret          = -1, new_op_errno = -1;
-        char            old_inode_looked_up = 0, new_inode_looked_up = 0;
-        sp_inode_ctx_t *old_inode_ctx       = NULL, *new_inode_ctx = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, newloc, unwind);
-
-        ret = inode_ctx_get (oldloc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (oldloc->inode->gfid));
-                goto unwind;
-        }
-
-        old_inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, old_inode_ctx, unwind);
-
-        LOCK (&old_inode_ctx->lock);
-        {
-                old_inode_looked_up = old_inode_ctx->looked_up;
-                old_op_ret = old_inode_ctx->op_ret;
-                old_op_errno = old_inode_ctx->op_errno;
-                need_unwind = old_inode_ctx->need_unwind;
-        }
-        UNLOCK (&old_inode_ctx->lock);
-
-        if (need_unwind) {
-                /* there was an error while queuing up lookup stub for newloc */
-                gf_log (this->name, GF_LOG_WARNING,
-                        "could not queue lookup stub for path (%s)",
-                        newloc->path);
-                goto unwind;
-        }
-
-        if (newloc->inode != NULL) {
-                ret = inode_ctx_get (newloc->inode, this, &value);
-                if (ret == 0) {
-                        new_inode_ctx = (sp_inode_ctx_t *)(long)value;
-                        if (new_inode_ctx != NULL) {
-                                LOCK (&new_inode_ctx->lock);
-                                {
-                                        new_inode_looked_up
-                                                = new_inode_ctx->looked_up;
-                                        new_op_ret = new_inode_ctx->op_ret;
-                                        new_op_errno = new_inode_ctx->op_errno;
-                                }
-                                UNLOCK (&new_inode_ctx->lock);
-                        }
-                }
-        }
-
-        if (new_inode_ctx == NULL) {
-                if (old_op_ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "lookup-behind has failed "
-                                "for path (%s)(%s), unwinding rename call "
-                                "waiting on it", oldloc->path,
-                                strerror (old_op_errno));
-
-                        op_errno = old_op_errno;
-                        goto unwind;
-                } else {
-                        can_wind = 1;
-                }
-        } else {
-                if (new_inode_looked_up && old_inode_looked_up) {
-                        if ((old_op_ret == -1)
-                            || ((new_op_ret == -1)
-                                && (new_op_errno != ENOENT))) {
-                                if (old_op_ret == -1) {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "lookup-behind has failed "
-                                                "for path (%s)(%s), unwinding "
-                                                "rename call waiting on it",
-                                                oldloc->path,
-                                                strerror (old_op_errno));
-                                        op_errno = old_op_errno;
-                                } else {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "lookup-behind has failed "
-                                                "for path (%s)(%s), unwinding "
-                                                "rename call waiting on it",
-                                                newloc->path,
-                                                strerror (new_op_errno));
-                                        op_errno = new_op_errno;
-                                }
-
-                                goto unwind;
-                        } else {
-                                can_wind = 1;
-                        }
-                }
-        }
-
-        if (can_wind) {
-                STACK_WIND (frame, sp_rename_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rename, oldloc, newloc);
-        }
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                         NULL);
-        return 0;
-}
-
-
-int32_t
-sp_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,loc_t *newloc)
-{
-        char            need_unwind           = 1;
-        uint64_t        value                 = 0;
-        call_stub_t    *stub                  = NULL;
-        sp_inode_ctx_t *inode_ctx             = NULL;
-        int32_t         ret                   = -1, op_errno = EINVAL;
-        char            old_inode_can_wind    = 0, new_inode_can_wind = 0;
-        char            old_inode_need_lookup = 0, new_inode_need_lookup = 0;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this ? frame->this->name : "stat-prefetch",
-                             this, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc->path, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc->name, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc->inode, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, newloc, out);
-        GF_VALIDATE_OR_GOTO (this->name, newloc->path, out);
-
-        sp_remove_caches_from_all_fds_opened (this, oldloc->parent,
-                                              (char *)oldloc->name);
-
-        sp_remove_caches_from_all_fds_opened (this, newloc->parent,
-                                              (char *)newloc->name);
-
-        ret = sp_cache_remove_parent_entry (frame, this, oldloc->parent->table,
-                                            (char *)oldloc->path);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot remove parent entry from grand-parent's cache "
-                        "for path (%s)", oldloc->path);
-                goto out;
-        }
-
-        ret = sp_cache_remove_parent_entry (frame, this, newloc->parent->table,
-                                            (char *)newloc->path);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot remove parent entry from grand-parent's cache "
-                        "for path (%s)", newloc->path);
-                goto out;
-        }
-
-        if (IA_ISDIR (oldloc->inode->ia_type)) {
-                sp_remove_caches_from_all_fds_opened (this, oldloc->inode,
-                                                      NULL);
-        }
-
-        stub = fop_rename_stub (frame, sp_rename_helper, oldloc, newloc);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        ret = sp_process_inode_ctx (frame, this, oldloc, stub, &need_unwind,
-                                    &old_inode_need_lookup, &old_inode_can_wind,
-                                    &op_errno);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "processing stat-prefetch "
-                        "context in inode (gfid:%s) (path:%s) failed (%s)",
-                        uuid_utoa (oldloc->inode->gfid), oldloc->path,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        if (newloc->inode != NULL) {
-                stub = fop_rename_stub (frame, sp_rename_helper, oldloc,
-                                        newloc);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-                ret = sp_process_inode_ctx (frame, this, newloc, stub,
-                                            &need_unwind,
-                                            &new_inode_need_lookup,
-                                            &new_inode_can_wind, &op_errno);
-                if (ret == -1) {
-                        ret = inode_ctx_get (oldloc->inode, this, &value);
-
-                        inode_ctx = (sp_inode_ctx_t *)(long)value;
-                        if (inode_ctx == NULL) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "stat-prefetch context not set in inode"
-                                        " (gfid:%s) (path:%s)",
-                                        uuid_utoa (oldloc->inode->gfid),
-                                        oldloc->path);
-                                goto out;
-                        }
-
-                        LOCK (&inode_ctx->lock);
-                        {
-                                if (!inode_ctx->looked_up) {
-                                        /* unwind in sp_rename_helper */
-                                        need_unwind = 0;
-                                        inode_ctx->need_unwind = 1;
-                                }
-                        }
-                        UNLOCK (&inode_ctx->lock);
-                }
-
-        } else {
-                new_inode_can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
-                                 NULL, NULL);
-        } else if (old_inode_need_lookup || new_inode_need_lookup) {
-                if (old_inode_need_lookup) {
-                        STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->lookup, oldloc,
-                                    NULL);
-                }
-
-                if (new_inode_need_lookup) {
-                        STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->lookup, newloc,
-                                    NULL);
-                }
-        } else if (old_inode_can_wind && new_inode_can_wind) {
-                STACK_WIND (frame, sp_rename_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rename, oldloc, newloc);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_setxattr_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    dict_t *dict, int32_t flags)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "lookup-behind has failed for path (%s)(%s), "
-                        "unwinding setxattr call waiting on it", loc->path,
-                        strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_err_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict,
-                    flags);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (setxattr, frame, -1, op_errno);
-        return 0;
-}
-
-
-int32_t
-sp_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags)
-{
-        int32_t         op_errno     = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-
-        sp_remove_caches_from_all_fds_opened (this, loc->parent,
-                                              (char *)loc->name);
-
-        stub = fop_setxattr_stub (frame, sp_setxattr_helper, loc, dict, flags);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (setxattr, frame, -1, op_errno);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_err_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->setxattr, loc, dict,
-                            flags);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_removexattr_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       const char *name)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "lookup-behind has failed for path (%s)(%s), "
-                        "unwinding setxattr call waiting on it", loc->path,
-                        strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_err_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (removexattr, frame, -1, op_errno);
-        return 0;
-}
-
-
-int32_t
-sp_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name)
-{
-        int32_t         op_errno     = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-
-        sp_remove_caches_from_all_fds_opened (this, loc->parent,
-                                              (char *)loc->name);
-
-        stub = fop_removexattr_stub (frame, sp_removexattr_helper, loc, name);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (removexattr, frame, -1, op_errno);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_err_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->removexattr, loc, name);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        GF_ASSERT (frame);
-        SP_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
-        return 0;
-}
-
-
-int32_t
-sp_getxattr_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "lookup-behind has failed for path (%s)(%s), "
-                        "unwinding getxattr call waiting on it", loc->path,
-                        strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name)
-{
-        int32_t         op_errno     = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        stub = fop_getxattr_stub (frame, sp_getxattr_helper, loc, name);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_getxattr_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->getxattr, loc, name);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        GF_ASSERT (frame);
-        SP_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict);
-        return 0;
-}
-
-
-int32_t
-sp_xattrop_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   gf_xattrop_flags_t flags, dict_t *dict)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "lookup-behind has failed for path (%s)(%s), "
-                        "unwinding xattrop call waiting on it", loc->path,
-                        strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_xattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            gf_xattrop_flags_t flags, dict_t *dict)
-{
-        int32_t         op_errno     = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
-
-        sp_remove_caches_from_all_fds_opened (this, loc->parent,
-                                              (char *)loc->name);
-
-        stub = fop_xattrop_stub (frame, sp_xattrop_helper, loc, flags, dict);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_xattrop_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->xattrop, loc, flags, dict);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             gf_xattrop_flags_t flags, dict_t *dict)
-{
-        sp_fd_ctx_t *fd_ctx = NULL;
-        uint64_t     value  = 0;
-        int32_t      ret    = 0, op_errno = EINVAL;
-        inode_t     *parent = NULL;
-        char        *name   = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "stat-prefetch context not "
-                        "set in fd (%p) opened on inode (gfid:%s", fd,
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-        fd_ctx = (void *)(long)value;
-        name   = fd_ctx->name;
-        parent = fd_ctx->parent_inode;
-
-        sp_remove_caches_from_all_fds_opened (this, parent, name);
-
-        STACK_WIND (frame, sp_xattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict);
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-int32_t
-sp_stbuf_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *buf)
-{
-        GF_ASSERT (frame);
-        SP_STACK_UNWIND (stat, frame, op_ret, op_errno, buf);
-        return 0;
-}
-
-
-int32_t
-sp_stat_helper (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "lookup-behind has failed for path (%s)(%s), "
-                        "unwinding stat call waiting on it", loc->path,
-                        strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_stbuf_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-sp_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        int32_t         op_errno     = -1;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        stub = fop_stat_stub (frame, sp_stat_helper, loc);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_stbuf_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->stat, loc);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_access_helper (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "lookup-behind has failed for path (%s)(%s), "
-                        "unwinding access call waiting on it", loc->path,
-                        strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_err_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access, loc, mask);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (access, frame, -1, op_errno);
-        return 0;
-}
-
-
-int32_t
-sp_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
-{
-        int32_t         op_errno     = -1;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        stub = fop_access_stub (frame, sp_access_helper, loc, mask);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (access, frame, -1, op_errno);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_err_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->access, loc, mask);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_inodelk_helper (call_frame_t *frame, xlator_t *this, const char *volume,
-                   loc_t *loc, int32_t cmd, struct gf_flock *lock)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "lookup-behind has failed for path (%s)(%s), "
-                        "unwinding inodelk call waiting on it", loc->path,
-                        strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_err_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, lock);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (inodelk, frame, -1, op_errno);
-        return 0;
-}
-
-
-int32_t
-sp_inodelk (call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
-            int32_t cmd, struct gf_flock *lock)
-{
-        int32_t         op_errno     = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        stub = fop_inodelk_stub (frame, sp_inodelk_helper, volume, loc, cmd,
-                                 lock);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (inodelk, frame, -1, op_errno);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_err_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd,
-                            lock);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_entrylk_helper (call_frame_t *frame, xlator_t *this, const char *volume,
-                   loc_t *loc, const char *basename, entrylk_cmd cmd,
-                   entrylk_type type)
-{
-        uint64_t        value     = 0;
-        sp_inode_ctx_t *inode_ctx = NULL;
-        int32_t         ret       = 0, op_ret = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "stat-prefetch context not set in inode (gfid:%s)",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        inode_ctx = (sp_inode_ctx_t *)(long) value;
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, unwind);
-
-        LOCK (&inode_ctx->lock);
-        {
-                op_ret = inode_ctx->op_ret;
-                op_errno = inode_ctx->op_errno;
-        }
-        UNLOCK (&inode_ctx->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "lookup-behind has failed for path (%s)(%s), "
-                        "unwinding entrylk call waiting on it", loc->path,
-                        strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, sp_err_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->entrylk, volume, loc, basename,
-                    cmd, type);
-
-        return 0;
-
-unwind:
-        SP_STACK_UNWIND (entrylk, frame, -1, op_errno);
-        return 0;
-}
-
-
-int32_t
-sp_entrylk (call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
-            const char *basename, entrylk_cmd cmd, entrylk_type type)
-{
-        int32_t         op_errno     = EINVAL;
-        call_stub_t    *stub         = NULL;
-        char            can_wind     = 0, need_lookup = 0, need_unwind = 1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        stub = fop_entrylk_stub (frame, sp_entrylk_helper, volume, loc,
-                                 basename, cmd, type);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        sp_process_inode_ctx (frame, this, loc, stub, &need_unwind,
-                              &need_lookup, &can_wind, &op_errno);
-
-out:
-        if (need_unwind) {
-                SP_STACK_UNWIND (entrylk, frame, -1, op_errno);
-        } else if (need_lookup) {
-                STACK_WIND (frame, sp_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, sp_err_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->entrylk, volume, loc,
-                            basename, cmd, type);
-        }
-
-        return 0;
-}
-
-
-int32_t
-sp_forget (xlator_t *this, inode_t *inode)
-{
-        struct iatt *buf   = NULL;
-        uint64_t     value = 0;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        inode_ctx_del (inode, this, &value);
-
-        if (value) {
-                buf = (void *)(long)value;
-                GF_FREE (buf);
-        }
-
-out:
-        return 0;
-}
-
-
-int32_t
-sp_release (xlator_t *this, fd_t *fd)
-{
-        sp_fd_ctx_t *fd_ctx = NULL;
-        uint64_t     value  = 0;
-        int32_t      ret    = 0;
-        sp_cache_t  *cache  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        ret = fd_ctx_del (fd, this, &value);
-        if (!ret) {
-                fd_ctx = (void *)(long) value;
-                cache = fd_ctx->cache;
-                if (cache) {
-                        gf_log (this->name, GF_LOG_TRACE, "cache hits: %lu, "
-                                "cache miss: %lu", cache->hits, cache->miss);
-                }
-
-                sp_fd_ctx_free (fd_ctx);
-        }
-
-out:
-        return 0;
-}
-
-
-struct sp_cache_dump {
-        int i;
-        char *key_prefix;
-};
-typedef struct sp_cache_dump sp_cache_dump_t;
-
-void
-sp_cache_traverse (void *data, void *mydata)
-{
-        gf_dirent_t     *dirent                   = NULL;
-        char             key[GF_DUMP_MAX_BUF_LEN] = {0, };
-        char             uuidbuf[256]             = {0, };
-        sp_cache_dump_t *dump                     = NULL;
-
-        if ((data == NULL) || (mydata == NULL)) {
-                goto out;
-        }
-
-        dirent = data;
-        dump = mydata;
-
-        gf_proc_dump_build_key (key, dump->key_prefix, "entry[%d].name",
-                                dump->i);
-        gf_proc_dump_write (key, "%s", dirent->d_name);
-
-        uuid_unparse (dirent->d_stat.ia_gfid, uuidbuf);
-        gf_proc_dump_build_key (key, dump->key_prefix, "entry[%d].inode.gfid",
-                                dump->i);
-        gf_proc_dump_write (key, "%s", uuidbuf);
-
-        dump->i++;
-out:
-        return;
-}
-
-
-int32_t
-sp_fdctx_dump (xlator_t *this, fd_t *fd)
-{
-        char            key[GF_DUMP_MAX_BUF_LEN]        = {0, };
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-        char            uuidbuf[256]                    = {0, };
-        sp_cache_t     *cache                           = NULL;
-        int32_t         ret                             = -1;
-        sp_cache_dump_t *dump                           = NULL;
-        char            *parent                         = NULL;
-
-        cache = sp_get_cache_fd (this, fd);
-        if (cache == NULL) {
-                ret = 0;
-                goto out;
-        }
-
-        dump = GF_CALLOC(1, sizeof (*dump), gf_common_mt_char);
-        if (dump == NULL) {
-                goto out;
-        }
-
-        gf_proc_dump_build_key (key_prefix,
-                                "xlator.performance.stat-prefetch",
-                                "fdctx");
-        gf_proc_dump_add_section (key_prefix);
-
-        gf_proc_dump_build_key (key, key_prefix, "fd");
-        gf_proc_dump_write (key, "%p", fd);
-
-        ret = __inode_path (fd->inode, NULL, &parent);
-        if (parent != NULL) {
-                gf_proc_dump_build_key (key, key_prefix, "name");
-                gf_proc_dump_write (key, "%s", parent);
-                GF_FREE (parent);
-        }
-
-        uuid_unparse (fd->inode->gfid, uuidbuf);
-        gf_proc_dump_build_key (key, key_prefix, "fd.inode.gfid");
-        gf_proc_dump_write (key, "%s", uuidbuf);
-
-        gf_proc_dump_build_key (key, key_prefix, "miss");
-        gf_proc_dump_write (key, "%lu", cache->miss);
-
-        gf_proc_dump_build_key (key, key_prefix, "hits");
-        gf_proc_dump_write (key, "%lu", cache->hits);
-
-        gf_proc_dump_build_key (key, key_prefix, "cache");
-        dump->key_prefix = key;
-
-        rbthash_table_traverse (cache->table, sp_cache_traverse, dump);
-
-        GF_FREE (dump);
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-sp_inodectx_dump (xlator_t *this, inode_t *inode)
-{
-        char            key[GF_DUMP_MAX_BUF_LEN]        = {0, };
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-        sp_inode_ctx_t *inode_ctx                       = NULL;
-        call_stub_t    *stub                            = NULL;
-        uint64_t        value                           = 0;
-        int32_t         ret                             = -1, i = 0;
-
-        if ((this == NULL) || (inode == NULL)) {
-                goto out;
-        }
-
-        ret = inode_ctx_get (inode, this, &value);
-        if (ret == 0) {
-                inode_ctx = (sp_inode_ctx_t *)(long)value;
-        }
-
-        if (inode_ctx == NULL) {
-                goto out;
-        }
-
-        gf_proc_dump_build_key (key_prefix,
-                                "stat-prefetch",
-                                "inodectx");
-        gf_proc_dump_add_section (key_prefix);
-
-        LOCK (&inode_ctx->lock);
-        {
-                gf_proc_dump_write ("looked_up", "%s",
-                                    inode_ctx->looked_up ? "yes" : "no");
-
-                gf_proc_dump_write ("lookup_in_progress", "%s",
-                                    inode_ctx->lookup_in_progress ?
-                                    "yes" : "no");
-
-                gf_proc_dump_write ("need_unwind", "%s", inode_ctx->need_unwind ?
-                                    "yes" : "no");
-
-                gf_proc_dump_write ("op_ret", "%d", inode_ctx->op_ret);
-
-                gf_proc_dump_write ("op_errno", "%d", inode_ctx->op_errno);
-
-                list_for_each_entry (stub, &inode_ctx->waiting_ops, list) {
-                        gf_proc_dump_build_key (key, "",
-                                                "waiting-ops[%d].frame", i);
-                        gf_proc_dump_write (key, "%"PRId64,
-                                            stub->frame->root->unique);
-
-                        gf_proc_dump_build_key (key, "",
-                                                "waiting-ops[%d].fop", i);
-                        gf_proc_dump_write (key, "%s", gf_fop_list[stub->fop]);
-
-                        i++;
-                }
-        }
-        UNLOCK (&inode_ctx->lock);
-out:
-        return ret;
-}
-
-int
-sp_priv_dump (xlator_t *this)
-{
-        sp_private_t            *priv         = NULL;
-        uint32_t                total_entries = 0;
-        uint32_t                ret           = -1;
-        char                    key[GF_DUMP_MAX_BUF_LEN];
-        char                    key_prefix[GF_DUMP_MAX_BUF_LEN];
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        priv = this->private;
-
-        total_entries = priv->entries;
-
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.stat-prefetch",
-                                "priv");
-        gf_proc_dump_add_section (key_prefix);
-
-        gf_proc_dump_build_key (key, key_prefix, "max_allowed_entries");
-        gf_proc_dump_write (key, "%lu", GF_SP_CACHE_ENTRIES_EXPECTED);
-        gf_proc_dump_build_key (key, key_prefix, "num_entries_cached");
-        gf_proc_dump_write (key, "%lu",(unsigned long)total_entries);
-        ret = 0;
-
-out:
-        return ret;
-}
-
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_sp_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int32_t
-init (xlator_t *this)
-{
-        int32_t         ret  = -1;
-        sp_private_t   *priv = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stat-prefetch", this, out);
-
-        if (!this->children || this->children->next) {
-                gf_log ("stat-prefetch",
-                        GF_LOG_ERROR,
-                        "FATAL: translator %s does not have exactly one child "
-                        "node", this->name);
-                goto out;
-        }
-
-        priv = GF_CALLOC (1, sizeof(sp_private_t), gf_sp_mt_sp_private_t);
-        LOCK_INIT (&priv->lock);
-
-        this->private = priv;
-
-        ret = 0;
-out:
-        return ret;
-}
-
-void
-fini (xlator_t *this)
-{
-        sp_private_t  *priv = NULL;
-
-        if (!this)
-                goto out;
-        else {
-                priv = this->private;
-                if (priv) {
-                        if (priv->mem_pool)
-                                mem_pool_destroy (priv->mem_pool);
-                        LOCK_DESTROY (&priv->lock);
-                        GF_FREE (priv);
-                        this->private = NULL;
-                }
-        }
-out:
-        return;
-}
-
-
-struct xlator_fops fops = {
-        .lookup      = sp_lookup,
-        .readdir     = sp_readdir,
-        .readdirp    = sp_readdirp,
-        .open        = sp_open,
-        .create      = sp_create,
-        .opendir     = sp_opendir,
-        .mkdir       = sp_mkdir,
-        .mknod       = sp_mknod,
-        .symlink     = sp_symlink,
-        .link        = sp_link,
-        .truncate    = sp_truncate,
-        .ftruncate   = sp_ftruncate,
-        .readlink    = sp_readlink,
-        .unlink      = sp_unlink,
-        .rmdir       = sp_rmdir,
-        .readv       = sp_readv,
-        .writev      = sp_writev,
-        .fsync       = sp_fsync,
-        .rename      = sp_rename,
-        .setxattr    = sp_setxattr,
-        .removexattr = sp_removexattr,
-        .xattrop     = sp_xattrop,
-        .fxattrop    = sp_fxattrop,
-        .setattr     = sp_setattr,
-        .stat        = sp_stat,
-        .access      = sp_access,
-        .getxattr    = sp_getxattr,
-        .inodelk     = sp_inodelk,
-        .entrylk     = sp_entrylk,
-};
-
-struct xlator_cbks cbks = {
-        .forget     = sp_forget,
-        .release    = sp_release,
-        .releasedir = sp_release
-};
-
-struct xlator_dumpops dumpops = {
-        .priv = sp_priv_dump,
-        .inodectx = sp_inodectx_dump,
-        .fdctx = sp_fdctx_dump
-};
diff --git a/xlators/performance/stat-prefetch/src/stat-prefetch.h b/xlators/performance/stat-prefetch/src/stat-prefetch.h
deleted file mode 100644
index ed84719e4..000000000
--- a/xlators/performance/stat-prefetch/src/stat-prefetch.h
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _STAT_PREFETCH_H
-#define _STAT_PREFETCH_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "locking.h"
-#include "inode.h"
-#include "glusterfs.h"
-#include "dict.h"
-#include "xlator.h"
-#include "rbthash.h"
-#include "hashfn.h"
-#include "call-stub.h"
-#include "stat-prefetch-mem-types.h"
-#include <libgen.h>
-
-struct sp_cache {
-        rbthash_table_t *table;
-        xlator_t        *this;
-        uint64_t         expected_offset;    /* Offset where the next read will
-                                              * happen.
-                                              */
-        gf_lock_t        lock;
-        unsigned long    miss;
-        unsigned long    hits;
-        uint32_t         ref;
-};
-typedef struct sp_cache sp_cache_t;
-
-struct sp_fd_ctx {
-        sp_cache_t *cache;
-        inode_t    *parent_inode;       /*
-                                         * inode corresponding to dirname (path)
-                                         */
-        char       *name;               /*
-                                         * basename of path on which this fd is
-                                         * opened
-                                         */
-};
-typedef struct sp_fd_ctx sp_fd_ctx_t;
-
-struct sp_local {
-        loc_t  loc;
-        fd_t  *fd;
-        char   is_lookup;
-};
-typedef struct sp_local sp_local_t;
-
-struct sp_inode_ctx {
-        char             looked_up;
-        char             lookup_in_progress;
-        char             need_unwind;
-        int32_t          op_ret;
-        int32_t          op_errno;
-        struct iatt      stbuf;
-        gf_lock_t        lock;
-        struct list_head waiting_ops;
-};
-typedef struct sp_inode_ctx sp_inode_ctx_t;
-
-struct sp_private {
-        struct mem_pool  *mem_pool;
-        uint32_t         entries;
-        gf_lock_t        lock;
-};
-typedef struct sp_private sp_private_t;
-
-void sp_local_free (sp_local_t *local);
-
-#define SP_STACK_UNWIND(op, frame, params ...) do {             \
-                sp_local_t *__local = frame->local;             \
-                frame->local = NULL;                            \
-                STACK_UNWIND_STRICT (op, frame, params);        \
-                sp_local_free (__local);                        \
-        } while (0)
-
-#define SP_STACK_DESTROY(frame) do {                    \
-                sp_local_t *__local = frame->local;     \
-                frame->local = NULL;                    \
-                STACK_DESTROY (frame->root);            \
-                sp_local_free (__local);                \
-        } while (0)
-
-#endif  /* #ifndef _STAT_PREFETCH_H */
diff --git a/xlators/performance/symlink-cache/src/Makefile.am b/xlators/performance/symlink-cache/src/Makefile.am
index 06e85fc92..4091c3293 100644
--- a/xlators/performance/symlink-cache/src/Makefile.am
+++ b/xlators/performance/symlink-cache/src/Makefile.am
@@ -1,12 +1,13 @@
 xlator_LTLIBRARIES = symlink-cache.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/performance
 
-symlink_cache_la_LDFLAGS = -module -avoidversion 
+symlink_cache_la_LDFLAGS = -module -avoid-version 
 
 symlink_cache_la_SOURCES = symlink-cache.c
 symlink_cache_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/symlink-cache/src/symlink-cache.c b/xlators/performance/symlink-cache/src/symlink-cache.c
index a82786cce..3b5fbc252 100644
--- a/xlators/performance/symlink-cache/src/symlink-cache.c
+++ b/xlators/performance/symlink-cache/src/symlink-cache.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -146,8 +137,7 @@ sc_cache_set (xlator_t *this, inode_t *inode, struct iatt *buf,
 err:
 
 	if (sc) {
-		if (sc->readlink)
-			FREE (sc->readlink);
+		FREE (sc->readlink);
 		sc->readlink = NULL;
 		FREE (sc);
 	}
@@ -242,7 +232,7 @@ sc_cache_get (xlator_t *this, inode_t *inode, char **link)
 int
 sc_readlink_cbk (call_frame_t *frame, void *cookie,
 		 xlator_t *this, int op_ret, int op_errno,
-		 const char *link, struct iatt *sbuf)
+		 const char *link, struct iatt *sbuf, dict_t *xdata)
 {
 	if (op_ret > 0)
 		sc_cache_update (this, frame->local, link);
@@ -250,14 +240,15 @@ sc_readlink_cbk (call_frame_t *frame, void *cookie,
 	inode_unref (frame->local);
 	frame->local = NULL;
 
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, link, sbuf);
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, link, sbuf,
+                             xdata);
         return 0;
 }
 
 
 int
 sc_readlink (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, size_t size)
+	     loc_t *loc, size_t size, dict_t *xdata)
 {
 	char *link = NULL;
         struct iatt buf = {0, };
@@ -275,7 +266,8 @@ sc_readlink (call_frame_t *frame, xlator_t *this,
                   using buf in readlink_cbk should be aware that @buf
                   is 0 filled
                 */
-		STACK_UNWIND_STRICT (readlink, frame, strlen (link), 0, link, &buf);
+		STACK_UNWIND_STRICT (readlink, frame, strlen (link), 0, link,
+                                     &buf, NULL);
 		FREE (link);
 		return 0;
 	}
@@ -285,7 +277,7 @@ sc_readlink (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, sc_readlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readlink,
-                    loc, size);
+                    loc, size, xdata);
 
 	return 0;
 }
@@ -295,7 +287,7 @@ int
 sc_symlink_cbk (call_frame_t *frame, void *cookie,
 		xlator_t *this, int op_ret, int op_errno,
                 inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
 	if (op_ret == 0) {
 		if (frame->local) {
@@ -303,22 +295,22 @@ sc_symlink_cbk (call_frame_t *frame, void *cookie,
 		}
 	}
 
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf, preparent,
-                      postparent);
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 sc_symlink (call_frame_t *frame, xlator_t *this,
-	    const char *dst, loc_t *src, dict_t *params)
+	    const char *dst, loc_t *src, mode_t umask, dict_t *xdata)
 {
 	frame->local = strdup (dst);
 
         STACK_WIND (frame, sc_symlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->symlink,
-                    dst, src, params);
+                    dst, src, umask, xdata);
 
 	return 0;
 }
@@ -327,7 +319,7 @@ sc_symlink (call_frame_t *frame, xlator_t *this,
 int
 sc_lookup_cbk (call_frame_t *frame, void *cookie,
 	       xlator_t *this, int op_ret, int op_errno,
-	       inode_t *inode, struct iatt *buf, dict_t *xattr,
+	       inode_t *inode, struct iatt *buf, dict_t *xdata,
                struct iatt *postparent)
 {
 	if (op_ret == 0)
@@ -335,19 +327,20 @@ sc_lookup_cbk (call_frame_t *frame, void *cookie,
 	else
 		sc_cache_flush (this, inode);
 
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xattr, postparent);
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
         return 0;
 }
 
 
 int
 sc_lookup (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, dict_t *xattr_req)
+	   loc_t *loc, dict_t *xdata)
 {
         STACK_WIND (frame, sc_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
-                    loc, xattr_req);
+                    loc, xdata);
 
         return 0;
 }
@@ -363,10 +356,9 @@ sc_forget (xlator_t *this,
 }
 
 
-int32_t 
+int32_t
 init (xlator_t *this)
 {
-	
         if (!this->children || this->children->next)
         {
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/performance/write-behind/src/Makefile.am b/xlators/performance/write-behind/src/Makefile.am
index a5ebc90bd..6c829d8ee 100644
--- a/xlators/performance/write-behind/src/Makefile.am
+++ b/xlators/performance/write-behind/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = write-behind.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-write_behind_la_LDFLAGS = -module -avoidversion 
+write_behind_la_LDFLAGS = -module -avoid-version 
 
 write_behind_la_SOURCES = write-behind.c
 write_behind_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = write-behind-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/write-behind/src/write-behind-mem-types.h b/xlators/performance/write-behind/src/write-behind-mem-types.h
index 5a3ee4aed..f64f429ce 100644
--- a/xlators/performance/write-behind/src/write-behind-mem-types.h
+++ b/xlators/performance/write-behind/src/write-behind-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -25,10 +16,10 @@
 
 enum gf_wb_mem_types_ {
         gf_wb_mt_wb_file_t   = gf_common_mt_end + 1,
-        gf_wb_mt_wb_local_t,
         gf_wb_mt_wb_request_t,
         gf_wb_mt_iovec,
         gf_wb_mt_wb_conf_t,
+        gf_wb_mt_wb_inode_t,
         gf_wb_mt_end
 };
 #endif
diff --git a/xlators/performance/write-behind/src/write-behind.c b/xlators/performance/write-behind/src/write-behind.c
index 7c666b403..95c5921c6 100644
--- a/xlators/performance/write-behind/src/write-behind.c
+++ b/xlators/performance/write-behind/src/write-behind.c
@@ -1,24 +1,13 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-/*TODO: check for non null wb_file_data before getting wb_file */
-
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -35,2646 +24,1774 @@
 #include "common-utils.h"
 #include "call-stub.h"
 #include "statedump.h"
+#include "defaults.h"
 #include "write-behind-mem-types.h"
 
-#define MAX_VECTOR_COUNT  8
-#define WB_AGGREGATE_SIZE 131072 /* 128 KB */
-#define WB_WINDOW_SIZE    1048576 /* 1MB */
+#define MAX_VECTOR_COUNT          8
+#define WB_AGGREGATE_SIZE         131072 /* 128 KB */
+#define WB_WINDOW_SIZE            1048576 /* 1MB */
 
 typedef struct list_head list_head_t;
 struct wb_conf;
-struct wb_page;
-struct wb_file;
-
-typedef struct wb_file {
-        int          disabled;
-        uint64_t     disable_till;
-        size_t       window_conf;
-        size_t       window_current;
-        int32_t      flags;
-        size_t       aggregate_current;
-        int32_t      refcount;
-        int32_t      op_ret;
-        int32_t      op_errno;
-        list_head_t  request;
-        list_head_t  passive_requests;
-        fd_t        *fd;
+struct wb_inode;
+
+typedef struct wb_inode {
+        ssize_t      window_conf;
+        ssize_t      window_current;
+	ssize_t      transit; /* size of data stack_wound, and yet
+				 to be fulfilled (wb_fulfill_cbk).
+				 used for trickling_writes
+			      */
+
+        list_head_t  all;    /* All requests, from enqueue() till destroy().
+				Used only for resetting generation
+				number when empty.
+			     */
+        list_head_t  todo;   /* Work to do (i.e, STACK_WIND to server).
+				Once we STACK_WIND, the entry is taken
+				off the list. If it is non-sync write,
+				then we continue to track it via @liability
+				or @temptation depending on the status
+				of its writeback.
+			     */
+        list_head_t  liability;   /* Non-sync writes which are lied
+				     (STACK_UNWIND'ed to caller) but ack
+				     from server not yet complete. This
+				     is the "liability" which we hold, and
+				     must guarantee that dependent operations
+				     which arrive later (which overlap, etc.)
+				     are issued only after their dependencies
+				     in this list are "fulfilled".
+
+				     Server acks for entries in this list
+				     shrinks the window.
+
+				     The sum total of all req->write_size
+				     of entries in this list must be kept less
+				     than the permitted window size.
+				  */
+        list_head_t  temptation;  /* Operations for which we are tempted
+				     to 'lie' (write-behind), but temporarily
+				     holding off (because of insufficient
+				     window capacity, etc.)
+
+				     This is the list to look at to grow
+				     the window (in __wb_pick_unwinds()).
+
+				     Entries typically get chosen from
+				     write-behind from this list, and therefore
+				     get "upgraded" to the "liability" list.
+			     */
+	list_head_t  wip; /* List of write calls in progress, SYNC or non-SYNC
+			     which are currently STACK_WIND'ed towards the server.
+			     This is for guaranteeing that no two overlapping
+			     writes are in progress at the same time. Modules
+			     like eager-lock in AFR depend on this behavior.
+			  */
+	uint64_t     gen;    /* Liability generation number. Represents
+				the current 'state' of liability. Every
+				new addition to the liability list bumps
+				the generation number.
+
+				a newly arrived request is only required
+				to perform causal checks against the entries
+				in the liability list which were present
+				at the time of its addition. the generation
+				number at the time of its addition is stored
+				in the request and used during checks.
+
+				the liability list can grow while the request
+				waits in the todo list waiting for its
+				dependent operations to complete. however
+				it is not of the request's concern to depend
+				itself on those new entries which arrived
+				after it arrived (i.e, those that have a
+				liability generation higher than itself)
+			     */
         gf_lock_t    lock;
         xlator_t    *this;
-}wb_file_t;
+} wb_inode_t;
+
 
 typedef struct wb_request {
-        list_head_t     list;
-        list_head_t     winds;
-        list_head_t     unwinds;
-        list_head_t     other_requests;
-        call_stub_t    *stub;
-        size_t          write_size;
-        int32_t         refcount;
-        wb_file_t      *file;
-        glusterfs_fop_t fop;
-        union {
-                struct  {
-                        char write_behind;
-                        char stack_wound;
-                        char got_reply;
-                        char virgin;
-                        char flush_all;     /* while trying to sync to back-end,
-                                             * don't wait till a data of size
-                                             * equal to configured aggregate-size
-                                             * is accumulated, instead sync
-                                             * whatever data currently present in
-                                             * request queue.
-                                             */
-
-                }write_request;
-
-                struct {
-                        char marked_for_resume;
-                }other_requests;
-        }flags;
+        list_head_t           all;
+        list_head_t           todo;
+	list_head_t           lie;  /* either in @liability or @temptation */
+        list_head_t           winds;
+        list_head_t           unwinds;
+        list_head_t           wip;
+
+        call_stub_t          *stub;
+
+        ssize_t               write_size;  /* currently held size
+					      (after collapsing) */
+	size_t                orig_size;   /* size which arrived with the request.
+					      This is the size by which we grow
+					      the window when unwinding the frame.
+					   */
+        size_t                total_size;  /* valid only in @head in wb_fulfill().
+					      This is the size with which we perform
+					      STACK_WIND to server and therefore the
+					      amount by which we shrink the window.
+					   */
+
+	int                   op_ret;
+	int                   op_errno;
+
+        int32_t               refcount;
+        wb_inode_t           *wb_inode;
+        glusterfs_fop_t       fop;
+        gf_lkowner_t          lk_owner;
+	struct iobref        *iobref;
+	uint64_t              gen;  /* inode liability state at the time of
+				       request arrival */
+
+	fd_t                 *fd;
+	struct {
+		size_t        size;          /* 0 size == till infinity */
+		off_t         off;
+		int           append:1;      /* offset is invalid. only one
+						outstanding append at a time */
+		int           tempted:1;     /* true only for non-sync writes */
+		int           lied:1;        /* sin committed */
+		int           fulfilled:1;   /* got server acknowledgement */
+		int           go:1;          /* enough aggregating, good to go */
+	} ordering;
 } wb_request_t;
 
-struct wb_conf {
-        uint64_t     aggregate_size;
-        uint64_t     window_size;
-        uint64_t     disable_till;
-        gf_boolean_t enable_O_SYNC;
-        gf_boolean_t flush_behind;
-        gf_boolean_t enable_trickling_writes;
-};
-
-typedef struct wb_local {
-        list_head_t     winds;
-        int32_t         flags;
-        int32_t         wbflags;
-        struct wb_file *file;
-        wb_request_t   *request;
-        int             op_ret;
-        int             op_errno;
-        call_frame_t   *frame;
-        int32_t         reply_count;
-} wb_local_t;
-
-typedef struct wb_conf wb_conf_t;
-typedef struct wb_page wb_page_t;
 
-int32_t
-wb_process_queue (call_frame_t *frame, wb_file_t *file);
+typedef struct wb_conf {
+        uint64_t         aggregate_size;
+        uint64_t         window_size;
+        gf_boolean_t     flush_behind;
+        gf_boolean_t     trickling_writes;
+	gf_boolean_t     strict_write_ordering;
+	gf_boolean_t     strict_O_DIRECT;
+} wb_conf_t;
 
-ssize_t
-wb_sync (call_frame_t *frame, wb_file_t *file, list_head_t *winds);
 
-ssize_t
-__wb_mark_winds (list_head_t *list, list_head_t *winds, size_t aggregate_size,
-                 char enable_trickling_writes);
+void
+wb_process_queue (wb_inode_t *wb_inode);
 
 
-static int
-__wb_request_unref (wb_request_t *this)
+wb_inode_t *
+__wb_inode_ctx_get (xlator_t *this, inode_t *inode)
 {
-        int ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
-
-        if (this->refcount <= 0) {
-                gf_log ("wb-request", GF_LOG_WARNING,
-                        "refcount(%d) is <= 0", this->refcount);
-                goto out;
-        }
+        uint64_t    value    = 0;
+        wb_inode_t *wb_inode = NULL;
 
-        ret = --this->refcount;
-        if (this->refcount == 0) {
-                list_del_init (&this->list);
-                if (this->stub && this->stub->fop == GF_FOP_WRITE) {
-                        call_stub_destroy (this->stub);
-                }
-
-                GF_FREE (this);
-        }
+        __inode_ctx_get (inode, this, &value);
+        wb_inode = (wb_inode_t *)(unsigned long) value;
 
-out:
-        return ret;
+        return wb_inode;
 }
 
 
-static int
-wb_request_unref (wb_request_t *this)
+wb_inode_t *
+wb_inode_ctx_get (xlator_t *this, inode_t *inode)
 {
-        wb_file_t *file = NULL;
-        int        ret  = -1;
+        wb_inode_t *wb_inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("write-behind", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
 
-        file = this->file;
-
-        LOCK (&file->lock);
+        LOCK (&inode->lock);
         {
-                ret = __wb_request_unref (this);
+                wb_inode = __wb_inode_ctx_get (this, inode);
         }
-        UNLOCK (&file->lock);
-
+        UNLOCK (&inode->lock);
 out:
-        return ret;
+        return wb_inode;
 }
 
 
-static wb_request_t *
-__wb_request_ref (wb_request_t *this)
+gf_boolean_t
+wb_fd_err (fd_t *fd, xlator_t *this, int32_t *op_errno)
 {
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
+        gf_boolean_t err   = _gf_false;
+        uint64_t     value = 0;
+        int32_t      tmp   = 0;
 
-        if (this->refcount < 0) {
-                gf_log ("wb-request", GF_LOG_WARNING,
-                        "refcount(%d) is < 0", this->refcount);
-                this = NULL;
-                goto out;
-        }
+	if (fd_ctx_get (fd, this, &value) == 0) {
+                if (op_errno) {
+                        tmp = value;
+                        *op_errno = tmp;
+                }
 
-        this->refcount++;
+                err = _gf_true;
+        }
 
-out:
-        return this;
+        return err;
 }
 
 
-wb_request_t *
-wb_request_ref (wb_request_t *this)
-{
-        wb_file_t *file = NULL;
+/*
+  Below is a succinct explanation of the code deciding whether two regions
+  overlap, from Pavan <tcp@gluster.com>.
 
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
+  For any two ranges to be non-overlapping, either the end of the first
+  range is lesser than the start of the second, or vice versa. Example -
 
-        file = this->file;
-        LOCK (&file->lock);
-        {
-                this = __wb_request_ref (this);
-        }
-        UNLOCK (&file->lock);
+  <--------->       <-------------->
+  p         q       x              y
 
-out:
-        return this;
-}
+  ( q < x ) or (y < p) = > No overlap.
 
+  To check for *overlap*, we can negate this (using de morgan's laws), and
+  it becomes -
 
-wb_request_t *
-wb_enqueue (wb_file_t *file, call_stub_t *stub)
-{
-        wb_request_t *request = NULL, *tmp = NULL;
-        call_frame_t *frame   = NULL;
-        wb_local_t   *local   = NULL;
-        struct iovec *vector  = NULL;
-        int32_t       count   = 0;
+  (q >= x ) and (y >= p)
 
-        GF_VALIDATE_OR_GOTO ("write-behind", file, out);
-        GF_VALIDATE_OR_GOTO (file->this->name, stub, out);
+  Either that, or you write the negation using -
 
-        request = GF_CALLOC (1, sizeof (*request), gf_wb_mt_wb_request_t);
-        if (request == NULL) {
-                goto out;
-        }
+  if (! ((q < x) or (y < p)) ) {
+  "Overlap"
+  }
+*/
 
-        INIT_LIST_HEAD (&request->list);
-        INIT_LIST_HEAD (&request->winds);
-        INIT_LIST_HEAD (&request->unwinds);
-        INIT_LIST_HEAD (&request->other_requests);
+gf_boolean_t
+wb_requests_overlap (wb_request_t *req1, wb_request_t *req2)
+{
+        uint64_t         r1_start = 0;
+	uint64_t         r1_end = 0;
+	uint64_t         r2_start = 0;
+	uint64_t         r2_end = 0;
+        enum _gf_boolean do_overlap = 0;
+
+        r1_start = req1->ordering.off;
+	if (req1->ordering.size)
+		r1_end = r1_start + req1->ordering.size - 1;
+	else
+		r1_end = ULLONG_MAX;
+
+        r2_start = req2->ordering.off;
+	if (req2->ordering.size)
+		r2_end = r2_start + req2->ordering.size - 1;
+	else
+		r2_end = ULLONG_MAX;
+
+        do_overlap = ((r1_end >= r2_start) && (r2_end >= r1_start));
+
+        return do_overlap;
+}
 
-        request->stub = stub;
-        request->file = file;
-        request->fop  = stub->fop;
 
-        frame = stub->frame;
-        local = frame->local;
-        if (local) {
-                local->request = request;
-        }
+gf_boolean_t
+wb_requests_conflict (wb_request_t *lie, wb_request_t *req)
+{
+	wb_conf_t  *conf = NULL;
 
-        if (stub->fop == GF_FOP_WRITE) {
-                vector = stub->args.writev.vector;
-                count = stub->args.writev.count;
+	conf = req->wb_inode->this->private;
 
-                request->write_size = iov_length (vector, count);
-                if (local) {
-                        local->op_ret = request->write_size;
-                        local->op_errno = 0;
-                }
+	if (lie == req)
+		/* request cannot conflict with itself */
+		return _gf_false;
 
-                request->flags.write_request.virgin = 1;
-        }
+	if (lie->gen >= req->gen)
+		/* this liability entry was behind
+		   us in the todo list */
+		return _gf_false;
 
-        LOCK (&file->lock);
-        {
-                list_add_tail (&request->list, &file->request);
-                if (stub->fop == GF_FOP_WRITE) {
-                        /* reference for stack winding */
-                        __wb_request_ref (request);
-
-                        /* reference for stack unwinding */
-                        __wb_request_ref (request);
-
-                        file->aggregate_current += request->write_size;
-                } else {
-                        list_for_each_entry (tmp, &file->request, list) {
-                                if (tmp->stub && tmp->stub->fop
-                                    == GF_FOP_WRITE) {
-                                        tmp->flags.write_request.flush_all = 1;
-                                }
-                        }
-
-                        /*reference for resuming */
-                        __wb_request_ref (request);
-                }
-        }
-        UNLOCK (&file->lock);
+	if (lie->ordering.append)
+		/* all modifications wait for the completion
+		   of outstanding append */
+		return _gf_true;
 
-out:
-        return request;
+	if (conf->strict_write_ordering)
+		/* We are sure (lie->gen < req->gen) by now. So
+		   skip overlap check if strict write ordering is
+		   requested and always return "conflict" against a
+		   lower generation lie. */
+		return _gf_true;
+
+	return wb_requests_overlap (lie, req);
 }
 
 
-wb_file_t *
-wb_file_create (xlator_t *this, fd_t *fd, int32_t flags)
+gf_boolean_t
+wb_liability_has_conflict (wb_inode_t *wb_inode, wb_request_t *req)
 {
-        wb_file_t *file = NULL;
-        wb_conf_t *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        conf = this->private;
+        wb_request_t *each     = NULL;
 
-        file = GF_CALLOC (1, sizeof (*file), gf_wb_mt_wb_file_t);
-        if (file == NULL) {
-                goto out;
+        list_for_each_entry (each, &wb_inode->liability, lie) {
+		if (wb_requests_conflict (each, req))
+			return _gf_true;
         }
 
-        INIT_LIST_HEAD (&file->request);
-        INIT_LIST_HEAD (&file->passive_requests);
-
-        /*
-          fd_ref() not required, file should never decide the existence of
-          an fd
-        */
-        file->fd= fd;
-        file->disable_till = conf->disable_till;
-        file->this = this;
-        file->refcount = 1;
-        file->window_conf = conf->window_size;
-        file->flags = flags;
-
-        LOCK_INIT (&file->lock);
-
-        fd_ctx_set (fd, this, (uint64_t)(long)file);
-
-out:
-        return file;
+        return _gf_false;
 }
 
 
-void
-wb_file_destroy (wb_file_t *file)
+gf_boolean_t
+wb_wip_has_conflict (wb_inode_t *wb_inode, wb_request_t *req)
 {
-        int32_t refcount = 0;
+        wb_request_t *each     = NULL;
 
-        GF_VALIDATE_OR_GOTO ("write-behind", file, out);
+	if (req->stub->fop != GF_FOP_WRITE)
+		/* non-writes fundamentally never conflict with WIP requests */
+		return _gf_false;
 
-        LOCK (&file->lock);
-        {
-                refcount = --file->refcount;
-        }
-        UNLOCK (&file->lock);
+        list_for_each_entry (each, &wb_inode->wip, wip) {
+		if (each == req)
+			/* request never conflicts with itself,
+			   though this condition should never occur.
+			*/
+			continue;
 
-        if (!refcount){
-                LOCK_DESTROY (&file->lock);
-                GF_FREE (file);
+		if (wb_requests_overlap (each, req))
+			return _gf_true;
         }
 
-out:
-        return;
+        return _gf_false;
 }
 
 
-int32_t
-wb_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf)
+static int
+__wb_request_unref (wb_request_t *req)
 {
-        wb_local_t   *local             = NULL;
-        list_head_t  *winds             = NULL;
-        wb_file_t    *file              = NULL;
-        wb_request_t *request           = NULL, *dummy = NULL;
-        wb_local_t   *per_request_local = NULL;
-        int32_t       ret               = -1;
-        fd_t         *fd                = NULL;
+        int         ret = -1;
+	wb_inode_t *wb_inode = NULL;
 
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+	wb_inode = req->wb_inode;
 
-        local = frame->local;
-        winds = &local->winds;
+        if (req->refcount <= 0) {
+                gf_log ("wb-request", GF_LOG_WARNING,
+                        "refcount(%d) is <= 0", req->refcount);
+                goto out;
+        }
 
-        file = local->file;
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
+        ret = --req->refcount;
+        if (req->refcount == 0) {
+                list_del_init (&req->todo);
+                list_del_init (&req->lie);
+		list_del_init (&req->wip);
 
-        LOCK (&file->lock);
-        {
-                list_for_each_entry_safe (request, dummy, winds, winds) {
-                        request->flags.write_request.got_reply = 1;
+		list_del_init (&req->all);
+		if (list_empty (&wb_inode->all)) {
+			wb_inode->gen = 0;
+			/* in case of accounting errors? */
+			wb_inode->window_current = 0;
+		}
 
-                        if (!request->flags.write_request.write_behind
-                            && (op_ret == -1)) {
-                                per_request_local = request->stub->frame->local;
-                                per_request_local->op_ret = op_ret;
-                                per_request_local->op_errno = op_errno;
-                        }
+		list_del_init (&req->winds);
+		list_del_init (&req->unwinds);
 
-                        if (request->flags.write_request.write_behind) {
-                                file->window_current -= request->write_size;
-                        }
+                if (req->stub && req->ordering.tempted) {
+                        call_stub_destroy (req->stub);
+			req->stub = NULL;
+                } /* else we would have call_resume()'ed */
 
-                        __wb_request_unref (request);
-                }
+		if (req->iobref)
+			iobref_unref (req->iobref);
 
-                if (op_ret == -1) {
-                        file->op_ret = op_ret;
-                        file->op_errno = op_errno;
-                }
-                fd = file->fd;
-        }
-        UNLOCK (&file->lock);
-
-        ret = wb_process_queue (frame, file);
-        if (ret == -1) {
-                if (errno == ENOMEM) {
-                        LOCK (&file->lock);
-                        {
-                                file->op_ret = -1;
-                                file->op_errno = ENOMEM;
-                        }
-                        UNLOCK (&file->lock);
-                }
+		if (req->fd)
+			fd_unref (req->fd);
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "request queue processing failed");
+                GF_FREE (req);
         }
-
-        /* safe place to do fd_unref */
-        fd_unref (fd);
-
-        STACK_DESTROY (frame->root);
-
 out:
-        return 0;
+        return ret;
 }
 
 
-ssize_t
-wb_sync (call_frame_t *frame, wb_file_t *file, list_head_t *winds)
+static int
+wb_request_unref (wb_request_t *req)
 {
-        wb_request_t   *dummy         = NULL, *request = NULL;
-        wb_request_t   *first_request = NULL, *next = NULL;
-        size_t          total_count   = 0, count = 0;
-        size_t          copied        = 0;
-        call_frame_t   *sync_frame    = NULL;
-        struct iobref  *iobref        = NULL;
-        wb_local_t     *local         = NULL;
-        struct iovec   *vector        = NULL;
-        ssize_t         current_size  = 0, bytes = 0;
-        size_t          bytecount     = 0;
-        wb_conf_t      *conf          = NULL;
-        fd_t           *fd            = NULL;
-        int32_t         op_errno      = -1;
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ((file ? file->this->name
-                                         : "write-behind"), frame,
-                                        out, bytes, -1);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, file, out, bytes,
-                                        -1);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, winds, out, bytes,
-                                        -1);
-
-        conf = file->this->private;
-        list_for_each_entry (request, winds, winds) {
-                total_count += request->stub->args.writev.count;
-                if (total_count > 0) {
-                        break;
-                }
-        }
-
-        if (total_count == 0) {
-                gf_log (file->this->name, GF_LOG_TRACE, "no vectors are to be"
-                        "synced");
-                goto out;
-        }
-
-        list_for_each_entry_safe (request, dummy, winds, winds) {
-                if (!vector) {
-                        vector = GF_MALLOC (VECTORSIZE (MAX_VECTOR_COUNT),
-                                            gf_wb_mt_iovec);
-                        if (vector == NULL) {
-                                bytes = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        iobref = iobref_new ();
-                        if (iobref == NULL) {
-                                bytes = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        local = GF_CALLOC (1, sizeof (*local),
-                                           gf_wb_mt_wb_local_t);
-                        if (local == NULL) {
-                                bytes = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        INIT_LIST_HEAD (&local->winds);
-
-                        first_request = request;
-                        current_size = 0;
-                }
+        wb_inode_t *wb_inode = NULL;
+        int         ret      = -1;
 
-                count += request->stub->args.writev.count;
-                bytecount = VECTORSIZE (request->stub->args.writev.count);
-                memcpy (((char *)vector)+copied,
-                        request->stub->args.writev.vector,
-                        bytecount);
-                copied += bytecount;
+        GF_VALIDATE_OR_GOTO ("write-behind", req, out);
 
-                current_size += request->write_size;
+        wb_inode = req->wb_inode;
 
-                if (request->stub->args.writev.iobref) {
-                        iobref_merge (iobref,
-                                      request->stub->args.writev.iobref);
-                }
+        LOCK (&wb_inode->lock);
+        {
+                ret = __wb_request_unref (req);
+        }
+        UNLOCK (&wb_inode->lock);
 
-                next = NULL;
-                if (request->winds.next != winds) {
-                        next = list_entry (request->winds.next,
-                                           wb_request_t, winds);
-                }
+out:
+        return ret;
+}
 
-                list_del_init (&request->winds);
-                list_add_tail (&request->winds, &local->winds);
-
-                if ((!next)
-                    || ((count + next->stub->args.writev.count)
-                        > MAX_VECTOR_COUNT)
-                    || ((current_size + next->write_size)
-                        > conf->aggregate_size)) {
-
-                        sync_frame = copy_frame (frame);
-                        if (sync_frame == NULL) {
-                                bytes = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        sync_frame->local = local;
-                        local->file = file;
-
-                        LOCK (&file->lock);
-                        {
-                                fd = file->fd;
-                        }
-                        UNLOCK (&file->lock);
-
-                        fd_ref (fd);
-
-                        bytes += current_size;
-                        STACK_WIND (sync_frame, wb_sync_cbk,
-                                    FIRST_CHILD(sync_frame->this),
-                                    FIRST_CHILD(sync_frame->this)->fops->writev,
-                                    fd, vector, count,
-                                    first_request->stub->args.writev.off,
-                                    first_request->stub->args.writev.flags,
-                                    iobref);
 
-                        iobref_unref (iobref);
-                        GF_FREE (vector);
-                        first_request = NULL;
-                        iobref = NULL;
-                        vector = NULL;
-                        sync_frame = NULL;
-                        local = NULL;
-                        copied = count = 0;
-                }
-        }
+static wb_request_t *
+__wb_request_ref (wb_request_t *req)
+{
+        GF_VALIDATE_OR_GOTO ("write-behind", req, out);
 
-out:
-        if (sync_frame != NULL) {
-                sync_frame->local = NULL;
-                STACK_DESTROY (sync_frame->root);
+        if (req->refcount < 0) {
+                gf_log ("wb-request", GF_LOG_WARNING,
+                        "refcount(%d) is < 0", req->refcount);
+                req = NULL;
+                goto out;
         }
 
-        if (local != NULL) {
-                /* had we winded these requests, we would have unrefed
-                 * in wb_sync_cbk.
-                 */
-                list_for_each_entry_safe (request, dummy, &local->winds,
-                                          winds) {
-                        wb_request_unref (request);
-                }
+        req->refcount++;
 
-                GF_FREE (local);
-                local = NULL;
-        }
+out:
+        return req;
+}
 
-        if (iobref != NULL) {
-                iobref_unref (iobref);
-        }
 
-        if (vector != NULL) {
-                GF_FREE (vector);
-        }
+wb_request_t *
+wb_request_ref (wb_request_t *req)
+{
+        wb_inode_t *wb_inode = NULL;
 
-        if (bytes == -1) {
-                /*
-                 * had we winded these requests, we would have unrefed
-                 * in wb_sync_cbk.
-                 */
-                if (local) {
-                        list_for_each_entry_safe (request, dummy, &local->winds,
-                                                  winds) {
-                                wb_request_unref (request);
-                        }
-                }
+        GF_VALIDATE_OR_GOTO ("write-behind", req, out);
 
-                if (file != NULL) {
-                        LOCK (&file->lock);
-                        {
-                                file->op_ret = -1;
-                                file->op_errno = op_errno;
-                        }
-                        UNLOCK (&file->lock);
-                }
+        wb_inode = req->wb_inode;
+        LOCK (&wb_inode->lock);
+        {
+                req = __wb_request_ref (req);
         }
+        UNLOCK (&wb_inode->lock);
 
-        return bytes;
+out:
+        return req;
 }
 
 
-int32_t
-wb_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, struct iatt *buf)
+gf_boolean_t
+wb_enqueue_common (wb_inode_t *wb_inode, call_stub_t *stub, int tempted)
 {
-        wb_local_t   *local         = NULL;
-        wb_request_t *request       = NULL;
-        call_frame_t *process_frame = NULL;
-        wb_file_t    *file          = NULL;
-        int32_t       ret           = -1;
-        fd_t         *fd            = NULL;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-
-        local = frame->local;
-        file = local->file;
-
-        request = local->request;
-        if (request) {
-                process_frame = copy_frame (frame);
-                if (process_frame == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
-        }
+        wb_request_t *req     = NULL;
 
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+        GF_VALIDATE_OR_GOTO ("write-behind", wb_inode, out);
+        GF_VALIDATE_OR_GOTO (wb_inode->this->name, stub, out);
 
-        if (request != NULL) {
-                wb_request_unref (request);
-        }
+        req = GF_CALLOC (1, sizeof (*req), gf_wb_mt_wb_request_t);
+        if (!req)
+                goto out;
 
-        if (process_frame != NULL) {
-                ret = wb_process_queue (process_frame, file);
-                if (ret == -1) {
-                        if ((errno == ENOMEM) && (file != NULL)) {
-                                LOCK (&file->lock);
-                                {
-                                        file->op_ret = -1;
-                                        file->op_errno = ENOMEM;
-                                }
-                                UNLOCK (&file->lock);
-                        }
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
+        INIT_LIST_HEAD (&req->all);
+        INIT_LIST_HEAD (&req->todo);
+        INIT_LIST_HEAD (&req->lie);
+        INIT_LIST_HEAD (&req->winds);
+        INIT_LIST_HEAD (&req->unwinds);
+        INIT_LIST_HEAD (&req->wip);
 
-                STACK_DESTROY (process_frame->root);
-        }
+        req->stub = stub;
+        req->wb_inode = wb_inode;
+        req->fop  = stub->fop;
+	req->ordering.tempted = tempted;
 
-        if (file) {
-                LOCK (&file->lock);
-                {
-                        fd = file->fd;
-                }
-                UNLOCK (&file->lock);
+        if (stub->fop == GF_FOP_WRITE) {
+                req->write_size = iov_length (stub->args.vector,
+					      stub->args.count);
 
-                fd_unref (fd);
-        }
+		/* req->write_size can change as we collapse
+		   small writes. But the window needs to grow
+		   only by how much we acknowledge the app. so
+		   copy the original size in orig_size for the
+		   purpose of accounting.
+		*/
+		req->orig_size = req->write_size;
 
-        return 0;
-}
+		/* Let's be optimistic that we can
+		   lie about it
+		*/
+		req->op_ret = req->write_size;
+		req->op_errno = 0;
 
+		if (stub->args.fd->flags & O_APPEND)
+			req->ordering.append = 1;
+        }
 
-static int32_t
-wb_stat_helper (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+        req->lk_owner = stub->frame->root->lk_owner;
 
-        STACK_WIND (frame, wb_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
-        return 0;
-}
+	switch (stub->fop) {
+	case GF_FOP_WRITE:
+		req->ordering.off = stub->args.offset;
+		req->ordering.size = req->write_size;
 
+		req->fd = fd_ref (stub->args.fd);
 
-int32_t
-wb_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        wb_file_t    *file     = NULL;
-        fd_t         *iter_fd  = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
-
-        if (loc->inode) {
-                /* FIXME: fd_lookup extends life of fd till stat returns */
-                iter_fd = fd_lookup (loc->inode, frame->root->pid);
-                if (iter_fd) {
-                        if (!fd_ctx_get (iter_fd, this, &tmp_file)) {
-                                file = (wb_file_t *)(long)tmp_file;
-                        } else {
-                                fd_unref (iter_fd);
-                                iter_fd = NULL;
-                        }
-                }
-        }
+		break;
+	case GF_FOP_READ:
+		req->ordering.off = stub->args.offset;
+		req->ordering.size = stub->args.size;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_wb_mt_wb_local_t);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+		req->fd = fd_ref (stub->args.fd);
 
-        local->file = file;
+		break;
+	case GF_FOP_TRUNCATE:
+		req->ordering.off = stub->args.offset;
+		req->ordering.size = 0; /* till infinity */
+		break;
+	case GF_FOP_FTRUNCATE:
+		req->ordering.off = stub->args.offset;
+		req->ordering.size = 0; /* till infinity */
 
-        frame->local = local;
+		req->fd = fd_ref (stub->args.fd);
 
-        if (file) {
-                stub = fop_stat_stub (frame, wb_stat_helper, loc);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+		break;
+	default:
+		break;
+	}
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        LOCK (&wb_inode->lock);
+        {
+                list_add_tail (&req->all, &wb_inode->all);
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_stat_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->stat, loc);
-        }
+		req->gen = wb_inode->gen;
 
-        return 0;
-unwind:
-        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL);
+                list_add_tail (&req->todo, &wb_inode->todo);
+		__wb_request_ref (req); /* for wind */
 
-        if (stub) {
-                call_stub_destroy (stub);
+		if (req->ordering.tempted) {
+			list_add_tail (&req->lie, &wb_inode->temptation);
+			__wb_request_ref (req); /* for unwind */
+		}
         }
+        UNLOCK (&wb_inode->lock);
 
-        if (iter_fd != NULL) {
-                fd_unref (iter_fd);
-        }
+out:
+	if (!req)
+		return _gf_false;
 
-        return 0;
+	return _gf_true;
 }
 
 
-int32_t
-wb_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *buf)
+gf_boolean_t
+wb_enqueue (wb_inode_t *wb_inode, call_stub_t *stub)
 {
-        wb_local_t   *local   = NULL;
-        wb_request_t *request = NULL;
-        wb_file_t    *file    = NULL;
-        int32_t       ret     = -1;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        file = local->file;
-
-        request = local->request;
-        if ((file != NULL) && (request != NULL)) {
-                wb_request_unref (request);
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        if (errno == ENOMEM) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                        }
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        }
-
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
-
-        return 0;
+	return wb_enqueue_common (wb_inode, stub, 0);
 }
 
 
-int32_t
-wb_fstat_helper (call_frame_t *frame, xlator_t *this, fd_t *fd)
+gf_boolean_t
+wb_enqueue_tempted (wb_inode_t *wb_inode, call_stub_t *stub)
 {
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-
-        STACK_WIND (frame, wb_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
-        return 0;
+	return wb_enqueue_common (wb_inode, stub, 1);
 }
 
 
-int32_t
-wb_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+wb_inode_t *
+__wb_inode_create (xlator_t *this, inode_t *inode)
 {
-        wb_file_t    *file     = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1;
-        int           op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
-
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_wb_mt_wb_local_t);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+        wb_inode_t *wb_inode = NULL;
+        wb_conf_t  *conf     = NULL;
 
-        local->file = file;
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
 
-        frame->local = local;
+        conf = this->private;
 
-        if (file) {
-                stub = fop_fstat_stub (frame, wb_fstat_helper, fd);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        wb_inode = GF_CALLOC (1, sizeof (*wb_inode), gf_wb_mt_wb_inode_t);
+        if (!wb_inode)
+                goto out;
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        INIT_LIST_HEAD (&wb_inode->all);
+        INIT_LIST_HEAD (&wb_inode->todo);
+        INIT_LIST_HEAD (&wb_inode->liability);
+        INIT_LIST_HEAD (&wb_inode->temptation);
+        INIT_LIST_HEAD (&wb_inode->wip);
 
-                /*
-                  FIXME:should the request queue be emptied in case of error?
-                */
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_fstat_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fstat, fd);
-        }
+        wb_inode->this = this;
 
-        return 0;
+        wb_inode->window_conf = conf->window_size;
 
-unwind:
-        STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL);
+        LOCK_INIT (&wb_inode->lock);
 
-        if (stub) {
-                call_stub_destroy (stub);
-        }
+        __inode_ctx_put (inode, this, (uint64_t)(unsigned long)wb_inode);
 
-        return 0;
+out:
+        return wb_inode;
 }
 
 
-int32_t
-wb_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf)
+wb_inode_t *
+wb_inode_create (xlator_t *this, inode_t *inode)
 {
-        wb_local_t   *local         = NULL;
-        wb_request_t *request       = NULL;
-        wb_file_t    *file          = NULL;
-        call_frame_t *process_frame = NULL;
-        int32_t       ret           = -1;
-        fd_t         *fd            = NULL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        file = local->file;
-        request = local->request;
-
-        if ((request != NULL) && (file != NULL)) {
-                process_frame = copy_frame (frame);
-                if (process_frame == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
-        }
-
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
-
-        if (request) {
-                wb_request_unref (request);
-        }
-
-        if (process_frame != NULL) {
-                ret = wb_process_queue (process_frame, file);
-                if (ret == -1) {
-                        if ((errno == ENOMEM) && (file != NULL)) {
-                                LOCK (&file->lock);
-                                {
-                                        file->op_ret = -1;
-                                        file->op_errno = ENOMEM;
-                                }
-                                UNLOCK (&file->lock);
-                        }
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
+        wb_inode_t *wb_inode = NULL;
 
-                STACK_DESTROY (process_frame->root);
-        }
-
-        if (file) {
-                LOCK (&file->lock);
-                {
-                        fd = file->fd;
-                }
-                UNLOCK (&file->lock);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
 
-                fd_unref (fd);
+        LOCK (&inode->lock);
+        {
+                wb_inode = __wb_inode_ctx_get (this, inode);
+                if (!wb_inode)
+                        wb_inode = __wb_inode_create (this, inode);
         }
+        UNLOCK (&inode->lock);
 
-        return 0;
+out:
+        return wb_inode;
 }
 
 
-static int32_t
-wb_truncate_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset)
+void
+wb_inode_destroy (wb_inode_t *wb_inode)
 {
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+        GF_VALIDATE_OR_GOTO ("write-behind", wb_inode, out);
 
-        STACK_WIND (frame, wb_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
-
-        return 0;
+        LOCK_DESTROY (&wb_inode->lock);
+        GF_FREE (wb_inode);
+out:
+        return;
 }
 
 
-int32_t
-wb_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+void
+__wb_fulfill_request (wb_request_t *req)
 {
-        wb_file_t    *file     = NULL;
-        fd_t         *iter_fd  = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
-
-        if (loc->inode) {
-                /*
-                  FIXME: fd_lookup extends life of fd till the execution of
-                  truncate_cbk
-                */
-                iter_fd = fd_lookup (loc->inode, frame->root->pid);
-                if (iter_fd) {
-                        if (!fd_ctx_get (iter_fd, this, &tmp_file)){
-                                file = (wb_file_t *)(long)tmp_file;
-                        } else {
-                                fd_unref (iter_fd);
-                        }
-                }
-        }
-
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_wb_mt_wb_local_t);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        local->file = file;
-
-        frame->local = local;
-        if (file) {
-                stub = fop_truncate_stub (frame, wb_truncate_helper, loc,
-                                          offset);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_truncate_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, loc, offset);
-        }
+	wb_inode_t *wb_inode = NULL;
 
-        return 0;
+	wb_inode = req->wb_inode;
 
-unwind:
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL);
+	req->ordering.fulfilled = 1;
+	wb_inode->window_current -= req->total_size;
+	wb_inode->transit -= req->total_size;
 
-        if (stub) {
-                call_stub_destroy (stub);
-        }
+	if (!req->ordering.lied) {
+		/* TODO: fail the req->frame with error if
+		   necessary
+		*/
+	}
 
-        return 0;
+	__wb_request_unref (req);
 }
 
 
-int32_t
-wb_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+void
+wb_head_done (wb_request_t *head)
 {
-        wb_local_t   *local   = NULL;
-        wb_request_t *request = NULL;
-        wb_file_t    *file    = NULL;
-        int32_t       ret     = -1;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        file = local->file;
-        request = local->request;
-
-        if ((request != NULL) && (file != NULL)) {
-                wb_request_unref (request);
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        if (errno == ENOMEM) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                        }
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        }
-
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
-
-        return 0;
+	wb_request_t *req = NULL;
+	wb_request_t *tmp     = NULL;
+	wb_inode_t   *wb_inode = NULL;
+
+	wb_inode = head->wb_inode;
+
+	LOCK (&wb_inode->lock);
+	{
+		list_for_each_entry_safe (req, tmp, &head->winds, winds) {
+			__wb_fulfill_request (req);
+		}
+		__wb_fulfill_request (head);
+	}
+	UNLOCK (&wb_inode->lock);
 }
 
 
-static int32_t
-wb_ftruncate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     off_t offset)
+void
+wb_fulfill_err (wb_request_t *head, int op_errno)
 {
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-
-        STACK_WIND (frame, wb_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
-        return 0;
+	wb_inode_t *wb_inode;
+	wb_request_t *req;
+
+	wb_inode = head->wb_inode;
+
+	/* for all future requests yet to arrive */
+	fd_ctx_set (head->fd, THIS, op_errno);
+
+	LOCK (&wb_inode->lock);
+	{
+		/* for all requests already arrived */
+		list_for_each_entry (req, &wb_inode->all, all) {
+			if (req->fd != head->fd)
+				continue;
+			req->op_ret = -1;
+			req->op_errno = op_errno;
+		}
+	}
+	UNLOCK (&wb_inode->lock);
 }
 
 
-int32_t
-wb_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+int
+wb_fulfill_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		struct iatt *postbuf, dict_t *xdata)
 {
-        wb_file_t    *file     = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1;
-        int           op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
+        wb_inode_t   *wb_inode   = NULL;
+        wb_request_t *head       = NULL;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_wb_mt_wb_local_t);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+	head = frame->local;
+	frame->local = NULL;
 
-        local->file = file;
+        wb_inode = head->wb_inode;
 
-        frame->local = local;
+	if (op_ret == -1) {
+		wb_fulfill_err (head, op_errno);
+	} else if (op_ret < head->total_size) {
+		/*
+		 * We've encountered a short write, for whatever reason.
+		 * Set an EIO error for the next fop. This should be
+		 * valid for writev or flush (close).
+		 *
+		 * TODO: Retry the write so we can potentially capture
+		 * a real error condition (i.e., ENOSPC).
+		 */
+		wb_fulfill_err (head, EIO);
+	}
 
-        if (file) {
-                stub = fop_ftruncate_stub (frame, wb_ftruncate_helper, fd,
-                                           offset);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	wb_head_done (head);
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        wb_process_queue (wb_inode);
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_ftruncate_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate, fd, offset);
-        }
+        STACK_DESTROY (frame->root);
 
         return 0;
+}
 
-unwind:
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno, NULL, NULL);
 
-        if (stub) {
-                call_stub_destroy (stub);
-        }
+#define WB_IOV_LOAD(vec, cnt, req, head) do {				\
+		memcpy (&vec[cnt], req->stub->args.vector,		\
+			(req->stub->args.count * sizeof(vec[0])));	\
+		cnt += req->stub->args.count;				\
+		head->total_size += req->write_size;			\
+	} while (0)
 
-        return 0;
-}
 
-
-int32_t
-wb_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                struct iatt *statpost)
+int
+wb_fulfill_head (wb_inode_t *wb_inode, wb_request_t *head)
 {
-        wb_local_t   *local         = NULL;
-        wb_request_t *request       = NULL;
-        call_frame_t *process_frame = NULL;
-        wb_file_t    *file          = NULL;
-        int32_t       ret           = -1;
-        fd_t         *fd            = NULL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        file = local->file;
-        request = local->request;
-
-        if (request) {
-                process_frame = copy_frame (frame);
-                if (process_frame == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
-        }
+	struct iovec  vector[MAX_VECTOR_COUNT];
+	int           count    = 0;
+	wb_request_t *req      = NULL;
+	call_frame_t *frame    = NULL;
+        gf_boolean_t  fderr    = _gf_false;
+        xlator_t     *this     = NULL;
 
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+        this = THIS;
 
-        if (request) {
-                wb_request_unref (request);
-        }
+        /* make sure head->total_size is updated before we run into any
+         * errors
+         */
 
-        if (request && (process_frame != NULL)) {
-                ret = wb_process_queue (process_frame, file);
-                if (ret == -1) {
-                        if ((errno == ENOMEM) && (file != NULL)) {
-                                LOCK (&file->lock);
-                                {
-                                        file->op_ret = -1;
-                                        file->op_errno = ENOMEM;
-                                }
-                                UNLOCK (&file->lock);
-                        }
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
+	WB_IOV_LOAD (vector, count, head, head);
 
-                STACK_DESTROY (process_frame->root);
-        }
+	list_for_each_entry (req, &head->winds, winds) {
+		WB_IOV_LOAD (vector, count, req, head);
 
-        if (file) {
-                LOCK (&file->lock);
-                {
-                        fd = file->fd;
-                }
-                UNLOCK (&file->lock);
+		iobref_merge (head->stub->args.iobref,
+			      req->stub->args.iobref);
+	}
 
-                fd_unref (fd);
+        if (wb_fd_err (head->fd, this, NULL)) {
+                fderr = _gf_true;
+                goto err;
         }
 
-        return 0;
-}
-
+	frame = create_frame (wb_inode->this, wb_inode->this->ctx->pool);
+	if (!frame)
+		goto err;
 
-static int32_t
-wb_setattr_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid)
-{
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+	frame->root->lk_owner = head->lk_owner;
+	frame->local = head;
 
-        STACK_WIND (frame, wb_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid);
-        return 0;
-}
+	LOCK (&wb_inode->lock);
+	{
+		wb_inode->transit += head->total_size;
+	}
+	UNLOCK (&wb_inode->lock);
 
+	STACK_WIND (frame, wb_fulfill_cbk, FIRST_CHILD (frame->this),
+		    FIRST_CHILD (frame->this)->fops->writev,
+		    head->fd, vector, count,
+		    head->stub->args.offset,
+		    head->stub->args.flags,
+		    head->stub->args.iobref, NULL);
 
-int32_t
-wb_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid)
-{
-        wb_file_t    *file     = NULL;
-        fd_t         *iter_fd  = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
-
-        local = GF_CALLOC (1, sizeof (*local), gf_wb_mt_wb_local_t);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
+	return 0;
+err:
+        if (!fderr) {
+                /* frame creation failure */
+                fderr = ENOMEM;
+                wb_fulfill_err (head, fderr);
         }
 
-        frame->local = local;
+	wb_head_done (head);
 
-        if (!(valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME))) {
-                STACK_WIND (frame, wb_setattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr, loc, stbuf,
-                            valid);
-                goto out;
-        }
-
-        if (loc->inode) {
-                /*
-                  FIXME: fd_lookup extends life of fd till the execution
-                  of wb_utimens_cbk
-                */
-                iter_fd = fd_lookup (loc->inode, frame->root->pid);
-                if (iter_fd) {
-                        if (!fd_ctx_get (iter_fd, this, &tmp_file)) {
-                                file = (wb_file_t *)(long)tmp_file;
-                        } else {
-                                fd_unref (iter_fd);
-                        }
-                }
-
-        }
-
-        local->file = file;
-
-        if (file) {
-                stub = fop_setattr_stub (frame, wb_setattr_helper, loc, stbuf,
-                                         valid);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	return fderr;
+}
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_setattr_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->setattr, loc, stbuf,
-                            valid);
-        }
+#define NEXT_HEAD(head, req) do {					\
+		if (head)						\
+			ret |= wb_fulfill_head (wb_inode, head);	\
+		head = req;						\
+		expected_offset = req->stub->args.offset +		\
+			req->write_size;				\
+		curr_aggregate = 0;					\
+		vector_count = 0;					\
+	} while (0)
 
-        return 0;
-unwind:
-        STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL);
 
-        if (stub) {
-                call_stub_destroy (stub);
-        }
-out:
-        return 0;
+int
+wb_fulfill (wb_inode_t *wb_inode, list_head_t *liabilities)
+{
+	wb_request_t  *req     = NULL;
+	wb_request_t  *head    = NULL;
+	wb_request_t  *tmp     = NULL;
+	wb_conf_t     *conf    = NULL;
+	off_t          expected_offset = 0;
+	size_t         curr_aggregate = 0;
+	size_t         vector_count = 0;
+        int            ret          = 0;
+
+	conf = wb_inode->this->private;
+
+	list_for_each_entry_safe (req, tmp, liabilities, winds) {
+		list_del_init (&req->winds);
+
+		if (!head) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		if (req->fd != head->fd) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		if (!is_same_lkowner (&req->lk_owner, &head->lk_owner)) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		if (expected_offset != req->stub->args.offset) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		if ((curr_aggregate + req->write_size) > conf->aggregate_size) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		if (vector_count + req->stub->args.count >
+		    MAX_VECTOR_COUNT) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		list_add_tail (&req->winds, &head->winds);
+		curr_aggregate += req->write_size;
+		vector_count += req->stub->args.count;
+	}
+
+	if (head)
+		ret |= wb_fulfill_head (wb_inode, head);
+
+	return ret;
 }
 
 
-int32_t
-wb_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, fd_t *fd)
+void
+wb_do_unwinds (wb_inode_t *wb_inode, list_head_t *lies)
 {
-        int32_t     wbflags = 0, flags = 0;
-        wb_file_t  *file    = NULL;
-        wb_conf_t  *conf    = NULL;
-        wb_local_t *local   = NULL;
+	wb_request_t *req = NULL;
+	wb_request_t *tmp = NULL;
+	call_frame_t *frame = NULL;
+	struct iatt   buf = {0, };
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, out, op_errno,
-                                        EINVAL);
+        list_for_each_entry_safe (req, tmp, lies, unwinds) {
+                frame = req->stub->frame;
 
-        conf = this->private;
+                STACK_UNWIND_STRICT (writev, frame, req->op_ret, req->op_errno,
+				     &buf, &buf, NULL); /* :O */
+		req->stub->frame = NULL;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, out, op_errno,
-                                        EINVAL);
-
-        flags = local->flags;
-        wbflags = local->wbflags;
-
-        if (op_ret != -1) {
-                file = wb_file_create (this, fd, flags);
-                if (file == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-                LOCK (&file->lock);
-                {
-                        /* If O_DIRECT then, we disable caching */
-                        if (((flags & O_DIRECT) == O_DIRECT)
-                            || ((flags & O_ACCMODE) == O_RDONLY)
-                            || (((flags & O_SYNC) == O_SYNC)
-                                && conf->enable_O_SYNC == _gf_true)) {
-                                file->window_conf = 0;
-                        }
-
-                        if (wbflags & GF_OPEN_NOWB) {
-                                file->disabled = 1;
-                        }
-                }
-                UNLOCK (&file->lock);
+		list_del_init (&req->unwinds);
+                wb_request_unref (req);
         }
 
-out:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
-        return 0;
+        return;
 }
 
 
-int32_t
-wb_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+void
+__wb_pick_unwinds (wb_inode_t *wb_inode, list_head_t *lies)
 {
-        wb_local_t *local    = NULL;
-        int32_t     op_errno = EINVAL;
+        wb_request_t *req = NULL;
+        wb_request_t *tmp = NULL;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_wb_mt_wb_local_t);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+	list_for_each_entry_safe (req, tmp, &wb_inode->temptation, lie) {
+		if (!req->ordering.fulfilled &&
+		    wb_inode->window_current > wb_inode->window_conf)
+			continue;
 
-        local->flags = flags;
-        local->wbflags = wbflags;
+		list_del_init (&req->lie);
+		list_move_tail (&req->unwinds, lies);
 
-        frame->local = local;
+		wb_inode->window_current += req->orig_size;
 
-        STACK_WIND (frame, wb_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
-        return 0;
+		if (!req->ordering.fulfilled) {
+			/* burden increased */
+			list_add_tail (&req->lie, &wb_inode->liability);
 
-unwind:
-        STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL);
-        return 0;
-}
+			req->ordering.lied = 1;
 
+			wb_inode->gen++;
+		}
+	}
 
-int32_t
-wb_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
-{
-        long       flags = 0;
-        wb_file_t *file  = NULL;
-        wb_conf_t *conf  = NULL;
+        return;
+}
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, out,
-                                        op_errno, EINVAL);
 
-        conf = this->private;
-        if (op_ret != -1) {
-                if (frame->local) {
-                        flags = (long) frame->local;
+int
+__wb_collapse_small_writes (wb_request_t *holder, wb_request_t *req)
+{
+        char          *ptr    = NULL;
+        struct iobuf  *iobuf  = NULL;
+        struct iobref *iobref = NULL;
+        int            ret    = -1;
+        ssize_t        required_size = 0;
+        size_t         holder_len = 0;
+        size_t         req_len = 0;
+
+        if (!holder->iobref) {
+                holder_len = iov_length (holder->stub->args.vector,
+                                         holder->stub->args.count);
+                req_len = iov_length (req->stub->args.vector,
+                                      req->stub->args.count);
+
+                required_size = max ((THIS->ctx->page_size),
+                                     (holder_len + req_len));
+                iobuf = iobuf_get2 (req->wb_inode->this->ctx->iobuf_pool,
+                                    required_size);
+                if (iobuf == NULL) {
+                        goto out;
                 }
 
-                file = wb_file_create (this, fd, flags);
-                if (file == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
+                iobref = iobref_new ();
+                if (iobref == NULL) {
+                        iobuf_unref (iobuf);
                         goto out;
                 }
 
-                LOCK (&file->lock);
-                {
-                        /* If O_DIRECT then, we disable caching */
-                        if (frame->local) {
-                                if (((flags & O_DIRECT) == O_DIRECT)
-                                    || ((flags & O_ACCMODE) == O_RDONLY)
-                                    || (((flags & O_SYNC) == O_SYNC)
-                                        && (conf->enable_O_SYNC == _gf_true))) {
-                                        file->window_conf = 0;
-                                }
-                        }
+                ret = iobref_add (iobref, iobuf);
+                if (ret != 0) {
+                        iobuf_unref (iobuf);
+                        iobref_unref (iobref);
+                        gf_log (req->wb_inode->this->name, GF_LOG_WARNING,
+                                "cannot add iobuf (%p) into iobref (%p)",
+                                iobuf, iobref);
+                        goto out;
                 }
-                UNLOCK (&file->lock);
-        }
 
-        frame->local = NULL;
+                iov_unload (iobuf->ptr, holder->stub->args.vector,
+                            holder->stub->args.count);
+                holder->stub->args.vector[0].iov_base = iobuf->ptr;
+		holder->stub->args.count = 1;
 
-out:
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
-        return 0;
-}
+                iobref_unref (holder->stub->args.iobref);
+                holder->stub->args.iobref = iobref;
 
+                iobuf_unref (iobuf);
 
-int32_t
-wb_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, fd_t *fd, dict_t *params)
-{
-        int32_t op_errno = EINVAL;
+                holder->iobref = iobref_ref (iobref);
+        }
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
+        ptr = holder->stub->args.vector[0].iov_base + holder->write_size;
 
-        frame->local = (void *)(long)flags;
+        iov_unload (ptr, req->stub->args.vector,
+                    req->stub->args.count);
 
-        STACK_WIND (frame, wb_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
-        return 0;
+        holder->stub->args.vector[0].iov_len += req->write_size;
+        holder->write_size += req->write_size;
+        holder->ordering.size += req->write_size;
 
-unwind:
-        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
-        return 0;
+        ret = 0;
+out:
+        return ret;
 }
 
 
-/* Mark all the contiguous write requests for winding starting from head of
- * request list. Stops marking at the first non-write request found. If
- * file is opened with O_APPEND, make sure all the writes marked for winding
- * will fit into a single write call to server.
- */
-size_t
-__wb_mark_wind_all (wb_file_t *file, list_head_t *list, list_head_t *winds)
+void
+__wb_preprocess_winds (wb_inode_t *wb_inode)
 {
-        wb_request_t *request         = NULL;
-        size_t        size            = 0;
-        char          first_request   = 1;
         off_t         offset_expected = 0;
-        wb_conf_t    *conf            = NULL;
-        int           count           = 0;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", file, out);
-        GF_VALIDATE_OR_GOTO (file->this->name, list, out);
-        GF_VALIDATE_OR_GOTO (file->this->name, winds, out);
-
-        conf = file->this->private;
-
-        list_for_each_entry (request, list, list)
-        {
-                if ((request->stub == NULL)
-                    || (request->stub->fop != GF_FOP_WRITE)) {
-                        break;
+        ssize_t       space_left      = 0;
+	wb_request_t *req             = NULL;
+	wb_request_t *tmp             = NULL;
+	wb_request_t *holder          = NULL;
+	wb_conf_t    *conf            = NULL;
+        int           ret             = 0;
+	ssize_t       page_size       = 0;
+
+	/* With asynchronous IO from a VM guest (as a file), there
+	   can be two sequential writes happening in two regions
+	   of the file. But individual (broken down) IO requests
+	   can arrive interleaved.
+
+	   TODO: cycle for each such sequence sifting
+	   through the interleaved ops
+	*/
+
+	page_size = wb_inode->this->ctx->page_size;
+	conf = wb_inode->this->private;
+
+        list_for_each_entry_safe (req, tmp, &wb_inode->todo, todo) {
+		if (!req->ordering.tempted) {
+			if (holder) {
+				if (wb_requests_conflict (holder, req))
+					/* do not hold on write if a
+					   dependent write is in queue */
+					holder->ordering.go = 1;
+			}
+			/* collapse only non-sync writes */
+			continue;
+		} else if (!holder) {
+			/* holder is always a non-sync write */
+			holder = req;
+			continue;
+		}
+
+		offset_expected = holder->stub->args.offset
+			+ holder->write_size;
+
+		if (req->stub->args.offset != offset_expected) {
+			holder->ordering.go = 1;
+			holder = req;
+			continue;
+		}
+
+		if (!is_same_lkowner (&req->lk_owner, &holder->lk_owner)) {
+			holder->ordering.go = 1;
+			holder = req;
+			continue;
+		}
+
+                if (req->fd != holder->fd) {
+                        holder->ordering.go = 1;
+                        holder = req;
+                        continue;
                 }
 
-                if (!request->flags.write_request.stack_wound) {
-                        if (first_request) {
-                                first_request = 0;
-                                offset_expected
-                                        = request->stub->args.writev.off;
-                        }
-
-                        if (request->stub->args.writev.off != offset_expected) {
-                                break;
-                        }
-
-                        if ((file->flags & O_APPEND)
-                            && (((size + request->write_size)
-                                 > conf->aggregate_size)
-                                || ((count + request->stub->args.writev.count)
-                                    > MAX_VECTOR_COUNT))) {
-                                break;
-                        }
-
-                        size += request->write_size;
-                        offset_expected += request->write_size;
-                        file->aggregate_current -= request->write_size;
-                        count += request->stub->args.writev.count;
-
-                        request->flags.write_request.stack_wound = 1;
-                        list_add_tail (&request->winds, winds);
-                }
-        }
+		space_left = page_size - holder->write_size;
 
-out:
-        return size;
-}
+		if (space_left < req->write_size) {
+			holder->ordering.go = 1;
+			holder = req;
+			continue;
+		}
 
+		ret = __wb_collapse_small_writes (holder, req);
+		if (ret)
+			continue;
 
-int32_t
-__wb_can_wind (list_head_t *list, char *other_fop_in_queue,
-               char *non_contiguous_writes, char *incomplete_writes,
-               char *wind_all)
-{
-        wb_request_t *request         = NULL;
-        char          first_request   = 1;
-        off_t         offset_expected = 0;
-        int32_t       ret             = -1;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", list, out);
+		/* collapsed request is as good as wound
+		   (from its p.o.v)
+		*/
+		list_del_init (&req->todo);
+		__wb_fulfill_request (req);
 
-        list_for_each_entry (request, list, list)
-        {
-                if ((request->stub == NULL)
-                    || (request->stub->fop != GF_FOP_WRITE)) {
-                        if (request->stub && other_fop_in_queue) {
-                                *other_fop_in_queue = 1;
-                        }
-                        break;
-                }
+               /* Only the last @holder in queue which
 
-                if (request->flags.write_request.stack_wound
-                    && !request->flags.write_request.got_reply
-                    && (incomplete_writes != NULL)) {
-                        *incomplete_writes = 1;
-                        break;
-                }
+                  - does not have any non-buffered-writes following it
+                  - has not yet filled its capacity
 
-                if (!request->flags.write_request.stack_wound) {
-                        if (first_request) {
-                                char flush = 0;
-                                first_request = 0;
-                                offset_expected
-                                        = request->stub->args.writev.off;
-
-                                flush = request->flags.write_request.flush_all;
-                                if (wind_all != NULL) {
-                                        *wind_all = flush;
-                                }
-                        }
-
-                        if (offset_expected != request->stub->args.writev.off) {
-                                if (non_contiguous_writes) {
-                                        *non_contiguous_writes = 1;
-                                }
-                                break;
-                        }
-
-                        offset_expected += request->write_size;
-                }
+                  does not get its 'go' set, in anticipation of the arrival
+                  of consecutive smaller writes.
+               */
         }
 
-        ret = 0;
-out:
-        return ret;
+	/* but if trickling writes are enabled, then do not hold back
+	   writes if there are no outstanding requests
+	*/
+
+	if (conf->trickling_writes && !wb_inode->transit && holder)
+		holder->ordering.go = 1;
+
+        return;
 }
 
 
-ssize_t
-__wb_mark_winds (list_head_t *list, list_head_t *winds, size_t aggregate_conf,
-                 char enable_trickling_writes)
+void
+__wb_pick_winds (wb_inode_t *wb_inode, list_head_t *tasks,
+		 list_head_t *liabilities)
 {
-        size_t        size                   = 0;
-        char          other_fop_in_queue     = 0;
-        char          incomplete_writes      = 0;
-        char          non_contiguous_writes  = 0;
-        wb_request_t *request                = NULL;
-        wb_file_t    *file                   = NULL;
-        char          wind_all               = 0;
-        int32_t       ret                    = 0;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", list, out);
-        GF_VALIDATE_OR_GOTO ("write-behind", winds, out);
-
-        if (list_empty (list)) {
-                goto out;
-        }
+	wb_request_t *req = NULL;
+	wb_request_t *tmp = NULL;
 
-        request = list_entry (list->next, typeof (*request), list);
-        file = request->file;
+	list_for_each_entry_safe (req, tmp, &wb_inode->todo, todo) {
+		if (wb_liability_has_conflict (wb_inode, req))
+			continue;
 
-        ret = __wb_can_wind (list, &other_fop_in_queue,
-                             &non_contiguous_writes, &incomplete_writes,
-                             &wind_all);
-        if (ret == -1) {
-                gf_log (file->this->name, GF_LOG_WARNING,
-                        "cannot decide whether to wind or not");
-                goto out;
-        }
+		if (req->ordering.tempted && !req->ordering.go)
+			/* wait some more */
+			continue;
 
-        if (!incomplete_writes && ((enable_trickling_writes)
-                                   || (wind_all) || (non_contiguous_writes)
-                                   || (other_fop_in_queue)
-                                   || (file->aggregate_current
-                                       >= aggregate_conf))) {
-                size = __wb_mark_wind_all (file, list, winds);
-        }
+		if (req->stub->fop == GF_FOP_WRITE) {
+			if (wb_wip_has_conflict (wb_inode, req))
+				continue;
 
-out:
-        return size;
-}
+			list_add_tail (&req->wip, &wb_inode->wip);
 
+			if (!req->ordering.tempted)
+				/* unrefed in wb_writev_cbk */
+				req->stub->frame->local =
+					__wb_request_ref (req);
+		}
 
-size_t
-__wb_mark_unwind_till (list_head_t *list, list_head_t *unwinds, size_t size)
-{
-        size_t        written_behind = 0;
-        wb_request_t *request        = NULL;
-        wb_file_t    *file           = NULL;
+		list_del_init (&req->todo);
 
-        if (list_empty (list)) {
-                goto out;
-        }
+		if (req->ordering.tempted)
+			list_add_tail (&req->winds, liabilities);
+		else
+			list_add_tail (&req->winds, tasks);
+	}
+}
 
-        request = list_entry (list->next, typeof (*request), list);
-        file = request->file;
 
-        list_for_each_entry (request, list, list)
-        {
-                if ((request->stub == NULL)
-                    || (request->stub->fop != GF_FOP_WRITE)) {
-                        continue;
-                }
+void
+wb_do_winds (wb_inode_t *wb_inode, list_head_t *tasks)
+{
+	wb_request_t *req = NULL;
+	wb_request_t *tmp = NULL;
 
-                if (written_behind <= size) {
-                        if (!request->flags.write_request.write_behind) {
-                                written_behind += request->write_size;
-                                request->flags.write_request.write_behind = 1;
-                                list_add_tail (&request->unwinds, unwinds);
-
-                                if (!request->flags.write_request.got_reply) {
-                                        file->window_current
-                                                += request->write_size;
-                                }
-                        }
-                } else {
-                        break;
-                }
-        }
+	list_for_each_entry_safe (req, tmp, tasks, winds) {
+		list_del_init (&req->winds);
 
-out:
-        return written_behind;
+		call_resume (req->stub);
+
+		wb_request_unref (req);
+	}
 }
 
 
 void
-__wb_mark_unwinds (list_head_t *list, list_head_t *unwinds)
+wb_process_queue (wb_inode_t *wb_inode)
 {
-        wb_request_t *request = NULL;
-        wb_file_t    *file    = NULL;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", list, out);
-        GF_VALIDATE_OR_GOTO ("write-behind", unwinds, out);
-
-        if (list_empty (list)) {
-                goto out;
-        }
+        list_head_t tasks  = {0, };
+	list_head_t lies = {0, };
+	list_head_t liabilities = {0, };
+        int         retry       = 0;
 
-        request = list_entry (list->next, typeof (*request), list);
-        file = request->file;
+        INIT_LIST_HEAD (&tasks);
+        INIT_LIST_HEAD (&lies);
+        INIT_LIST_HEAD (&liabilities);
 
-        if (file->window_current <= file->window_conf) {
-                __wb_mark_unwind_till (list, unwinds,
-                                       file->window_conf
-                                       - file->window_current);
-        }
+        do {
+                LOCK (&wb_inode->lock);
+                {
+                        __wb_preprocess_winds (wb_inode);
 
-out:
-        return;
-}
+                        __wb_pick_winds (wb_inode, &tasks, &liabilities);
 
+                        __wb_pick_unwinds (wb_inode, &lies);
 
-uint32_t
-__wb_get_other_requests (list_head_t *list, list_head_t *other_requests)
-{
-        wb_request_t *request = NULL;
-        uint32_t      count   = 0;
+                }
+                UNLOCK (&wb_inode->lock);
 
-        GF_VALIDATE_OR_GOTO ("write-behind", list, out);
-        GF_VALIDATE_OR_GOTO ("write-behind", other_requests, out);
+                wb_do_unwinds (wb_inode, &lies);
 
-        list_for_each_entry (request, list, list) {
-                if ((request->stub == NULL)
-                    || (request->stub->fop == GF_FOP_WRITE)) {
-                        break;
-                }
+                wb_do_winds (wb_inode, &tasks);
 
-                if (!request->flags.other_requests.marked_for_resume) {
-                        request->flags.other_requests.marked_for_resume = 1;
-                        list_add_tail (&request->other_requests,
-                                       other_requests);
-                        count++;
-                }
-        }
+                /* fd might've been marked bad due to previous errors.
+                 * Since, caller of wb_process_queue might be the last fop on
+                 * inode, make sure we keep processing request queue, till there
+                 * are no requests left.
+                 */
+                retry = wb_fulfill (wb_inode, &liabilities);
+        } while (retry);
 
-out:
-        return count;
+        return;
 }
 
 
-int32_t
-wb_stack_unwind (list_head_t *unwinds)
+int
+wb_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	       int32_t op_ret, int32_t op_errno,
+	       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        struct iatt   buf     = {0,};
-        wb_request_t *request = NULL, *dummy = NULL;
-        call_frame_t *frame   = NULL;
-        wb_local_t   *local   = NULL;
-        int           ret     = 0, write_requests_removed = 0;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", unwinds, out);
+	wb_request_t *req = NULL;
+	wb_inode_t *wb_inode;
 
-        list_for_each_entry_safe (request, dummy, unwinds, unwinds) {
-                frame = request->stub->frame;
-                local = frame->local;
+	req = frame->local;
+	frame->local = NULL;
+	wb_inode = req->wb_inode;
 
-                STACK_UNWIND (frame, local->op_ret, local->op_errno, &buf,
-                              &buf);
+	wb_request_unref (req);
 
-                ret = wb_request_unref (request);
-                if (ret == 0) {
-                        write_requests_removed++;
-                }
-        }
+	/* requests could be pending while this was in progress */
+	wb_process_queue(wb_inode);
 
-out:
-        return write_requests_removed;
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+			     xdata);
+	return 0;
 }
 
 
-int32_t
-wb_resume_other_requests (call_frame_t *frame, wb_file_t *file,
-                          list_head_t *other_requests)
+int
+wb_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		  struct iovec *vector, int32_t count, off_t offset,
+		  uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        int32_t       ret          = -1;
-        wb_request_t *request      = NULL, *dummy = NULL;
-        int32_t       fops_removed = 0;
-        char          wind         = 0;
-        call_stub_t  *stub         = NULL;
-
-        GF_VALIDATE_OR_GOTO ((file ? file->this->name : "write-behind"), frame,
-                             out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, file, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, other_requests, out);
+	STACK_WIND (frame, wb_writev_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
+		    fd, vector, count, offset, flags, iobref, xdata);
+	return 0;
+}
 
-        if (list_empty (other_requests)) {
-                ret = 0;
-                goto out;
-        }
 
-        list_for_each_entry_safe (request, dummy, other_requests,
-                                  other_requests) {
-                wind = request->stub->wind;
-                stub = request->stub;
+int
+wb_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
+{
+        wb_inode_t   *wb_inode      = NULL;
+	wb_conf_t    *conf          = NULL;
+        gf_boolean_t  wb_disabled   = 0;
+        call_stub_t  *stub          = NULL;
+        int           ret           = -1;
+        int32_t       op_errno      = EINVAL;
+	int           o_direct      = O_DIRECT;
 
-                LOCK (&file->lock);
-                {
-                        request->stub = NULL;
-                }
-                UNLOCK (&file->lock);
+	conf = this->private;
 
-                if (!wind) {
-                        wb_request_unref (request);
-                        fops_removed++;
-                }
-
-                call_resume (stub);
+        if (wb_fd_err (fd, this, &op_errno)) {
+                goto unwind;
         }
 
-        ret = 0;
+        wb_inode = wb_inode_create (this, fd->inode);
+	if (!wb_inode) {
+		op_errno = ENOMEM;
+		goto unwind;
+	}
 
-        if (fops_removed > 0) {
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (frame->this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        }
+	if (!conf->strict_O_DIRECT)
+		o_direct = 0;
 
-out:
-        return ret;
-}
+	if (fd->flags & (O_SYNC|O_DSYNC|o_direct))
+		wb_disabled = 1;
 
+	if (flags & (O_SYNC|O_DSYNC|o_direct))
+		wb_disabled = 1;
 
-int32_t
-wb_do_ops (call_frame_t *frame, wb_file_t *file, list_head_t *winds,
-           list_head_t *unwinds, list_head_t *other_requests)
-{
-        int32_t ret = -1, write_requests_removed = 0;
+	if (wb_disabled)
+		stub = fop_writev_stub (frame, wb_writev_helper, fd, vector,
+					count, offset, flags, iobref, xdata);
+	else
+		stub = fop_writev_stub (frame, NULL, fd, vector, count, offset,
+					flags, iobref, xdata);
+        if (!stub) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
 
-        GF_VALIDATE_OR_GOTO ((file ? file->this->name : "write-behind"),
-                             frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, file, out);
+	if (wb_disabled)
+		ret = wb_enqueue (wb_inode, stub);
+	else
+		ret = wb_enqueue_tempted (wb_inode, stub);
 
-        ret = wb_stack_unwind (unwinds);
+	if (!ret) {
+		op_errno = ENOMEM;
+		goto unwind;
+	}
 
-        write_requests_removed = ret;
+        wb_process_queue (wb_inode);
 
-        ret = wb_sync (frame, file, winds);
-        if (ret == -1) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "syncing of write requests failed");
-        }
+        return 0;
 
-        ret = wb_resume_other_requests (frame, file, other_requests);
-        if (ret == -1) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "cannot resume non-write requests in request queue");
-        }
+unwind:
+        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        /* wb_stack_unwind does wb_request_unref after unwinding a write
-         * request. Hence if a write-request was just freed in wb_stack_unwind,
-         * we have to process request queue once again to unblock requests
-         * blocked on the writes just unwound.
-         */
-        if (write_requests_removed > 0) {
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (frame->this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        }
+        if (stub)
+                call_stub_destroy (stub);
 
-out:
-        return ret;
+        return 0;
 }
 
 
-inline int
-__wb_copy_into_holder (wb_request_t *holder, wb_request_t *request)
+int
+wb_readv_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, uint32_t flags, dict_t *xdata)
 {
-        char          *ptr    = NULL;
-        struct iobuf  *iobuf  = NULL;
-        struct iobref *iobref = NULL;
-        int            ret    = -1;
-
-        if (holder->flags.write_request.virgin) {
-                iobuf = iobuf_get (request->file->this->ctx->iobuf_pool);
-                if (iobuf == NULL) {
-                        goto out;
-                }
+        STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                    xdata);
+        return 0;
+}
 
-                iobref = iobref_new ();
-                if (iobref == NULL) {
-                        iobuf_unref (iobuf);
-                        goto out;
-                }
 
-                ret = iobref_add (iobref, iobuf);
-                if (ret != 0) {
-                        iobuf_unref (iobuf);
-                        iobref_unref (iobref);
-                        gf_log (request->file->this->name, GF_LOG_WARNING,
-                                "cannot add iobuf (%p) into iobref (%p)",
-                                iobuf, iobref);
-                        goto out;
-                }
+int
+wb_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
+{
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
 
-                iov_unload (iobuf->ptr, holder->stub->args.writev.vector,
-                            holder->stub->args.writev.count);
-                holder->stub->args.writev.vector[0].iov_base = iobuf->ptr;
+        wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-                iobref_unref (holder->stub->args.writev.iobref);
-                holder->stub->args.writev.iobref = iobref;
+	stub = fop_readv_stub (frame, wb_readv_helper, fd, size,
+			       offset, flags, xdata);
+	if (!stub)
+		goto unwind;
 
-                iobuf_unref (iobuf);
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-                holder->flags.write_request.virgin = 0;
-        }
+	wb_process_queue (wb_inode);
 
-        ptr = holder->stub->args.writev.vector[0].iov_base + holder->write_size;
+        return 0;
 
-        iov_unload (ptr, request->stub->args.writev.vector,
-                    request->stub->args.writev.count);
+unwind:
+        STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL,
+                             NULL);
+        return 0;
 
-        holder->stub->args.writev.vector[0].iov_len += request->write_size;
-        holder->write_size += request->write_size;
+noqueue:
+        STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                    xdata);
+        return 0;
+}
 
-        request->flags.write_request.stack_wound = 1;
-        list_move_tail (&request->list, &request->file->passive_requests);
 
-        ret = 0;
-out:
-        return ret;
+int
+wb_flush_bg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        STACK_DESTROY (frame->root);
+        return 0;
 }
 
 
-/* this procedure assumes that write requests have only one vector to write */
-void
-__wb_collapse_write_bufs (list_head_t *requests, size_t page_size)
+int
+wb_flush_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        off_t         offset_expected = 0;
-        size_t        space_left      = 0;
-        wb_request_t *request         = NULL, *tmp = NULL, *holder = NULL;
-        int           ret             = 0;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", requests, out);
+        wb_conf_t    *conf        = NULL;
+        wb_inode_t   *wb_inode    = NULL;
+        call_frame_t *bg_frame    = NULL;
+	int32_t       op_errno    = 0;
+	int           op_ret      = 0;
 
-        list_for_each_entry_safe (request, tmp, requests, list) {
-                if ((request->stub == NULL)
-                    || (request->stub->fop != GF_FOP_WRITE)
-                    || (request->flags.write_request.stack_wound)) {
-                        holder = NULL;
-                        continue;
-                }
+        conf = this->private;
 
-                if (request->flags.write_request.write_behind) {
-                        if (holder == NULL) {
-                                holder = request;
-                                continue;
-                        }
-
-                        offset_expected = holder->stub->args.writev.off
-                                + holder->write_size;
-
-                        if (request->stub->args.writev.off != offset_expected) {
-                                holder = request;
-                                continue;
-                        }
-
-                        space_left = page_size - holder->write_size;
-
-                        if (space_left >= request->write_size) {
-                                ret = __wb_copy_into_holder (holder, request);
-                                if (ret != 0) {
-                                        break;
-                                }
-
-                                __wb_request_unref (request);
-                        } else {
-                                holder = request;
-                        }
-                } else {
-                        break;
-                }
-        }
+	wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode) {
+		op_ret = -1;
+		op_errno = EINVAL;
+		goto unwind;
+	}
+
+	if (wb_fd_err (fd, this, &op_errno)) {
+		op_ret = -1;
+		goto unwind;
+	}
+
+        if (conf->flush_behind)
+		goto flushbehind;
+
+	STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->flush, fd, xdata);
+	return 0;
+
+flushbehind:
+	bg_frame = copy_frame (frame);
+	if (!bg_frame) {
+		op_ret = -1;
+		op_errno = ENOMEM;
+		goto unwind;
+	}
+
+	STACK_WIND (bg_frame, wb_flush_bg_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->flush, fd, xdata);
+	/* fall through */
+unwind:
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, NULL);
 
-out:
-        return;
+        return 0;
 }
 
 
-int32_t
-wb_process_queue (call_frame_t *frame, wb_file_t *file)
+int
+wb_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        list_head_t winds  = {0, }, unwinds = {0, }, other_requests = {0, };
-        size_t      size   = 0;
-        wb_conf_t  *conf   = NULL;
-        uint32_t    count  = 0;
-        int32_t     ret    = -1;
-
-        INIT_LIST_HEAD (&winds);
-        INIT_LIST_HEAD (&unwinds);
-        INIT_LIST_HEAD (&other_requests);
-
-        GF_VALIDATE_OR_GOTO ((file ? file->this->name : "write-behind"), frame,
-                             out);
-        GF_VALIDATE_OR_GOTO (file->this->name, frame, out);
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
 
-        conf = file->this->private;
-        GF_VALIDATE_OR_GOTO (file->this->name, conf, out);
+        wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-        size = conf->aggregate_size;
-        LOCK (&file->lock);
-        {
-                /*
-                 * make sure requests are marked for unwinding and adjacent
-                 * contiguous write buffers (each of size less than that of
-                 * an iobuf) are packed properly so that iobufs are filled to
-                 * their maximum capacity, before calling __wb_mark_winds.
-                 */
-                __wb_mark_unwinds (&file->request, &unwinds);
+	stub = fop_flush_stub (frame, wb_flush_helper, fd, xdata);
+	if (!stub)
+		goto unwind;
 
-                __wb_collapse_write_bufs (&file->request,
-                                          file->this->ctx->page_size);
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-                count = __wb_get_other_requests (&file->request,
-                                                 &other_requests);
+        wb_process_queue (wb_inode);
 
-                if (count == 0) {
-                        __wb_mark_winds (&file->request, &winds, size,
-                                         conf->enable_trickling_writes);
-                }
+        return 0;
 
-        }
-        UNLOCK (&file->lock);
+unwind:
+        STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
 
-        ret = wb_do_ops (frame, file, &winds, &unwinds, &other_requests);
+        return 0;
 
-out:
-        return ret;
+noqueue:
+        STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->flush, fd, xdata);
+        return 0;
 }
 
 
-int32_t
-wb_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
-{
-        GF_ASSERT (frame);
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+int
+wb_fsync_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 int32_t datasync, dict_t *xdata)
+{
+        STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
         return 0;
 }
 
 
-int32_t
-wb_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref)
+int
+wb_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata)
 {
-        wb_file_t    *file          = NULL;
-        char          wb_disabled   = 0;
-        call_frame_t *process_frame = NULL;
-        size_t        size          = 0;
-        uint64_t      tmp_file      = 0;
-        call_stub_t  *stub          = NULL;
-        wb_local_t   *local         = NULL;
-        wb_request_t *request       = NULL;
-        int32_t       ret           = -1;
-        int32_t       op_ret        = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("write-behind", this, unwind, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, fd, unwind, op_errno,
-                                        EINVAL);
-
-        if (vector != NULL)
-                size = iov_length (vector, count);
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
-
-        if (file != NULL) {
-                LOCK (&file->lock);
-                {
-                        op_ret = file->op_ret;
-                        op_errno = file->op_errno;
-
-                        file->op_ret = 0;
-
-                        if ((op_ret == 0)
-                            && (file->disabled || file->disable_till)) {
-                                if (size > file->disable_till) {
-                                        file->disable_till = 0;
-                                } else {
-                                        file->disable_till -= size;
-                                }
-                                wb_disabled = 1;
-                        }
-                }
-                UNLOCK (&file->lock);
-        } else {
-                wb_disabled = 1;
-        }
-
-        if (op_ret == -1) {
-                STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, NULL,
-                                     NULL);
-                return 0;
-        }
-
-        if (wb_disabled) {
-                STACK_WIND (frame, wb_writev_cbk, FIRST_CHILD (frame->this),
-                            FIRST_CHILD (frame->this)->fops->writev,
-                            fd, vector, count, offset, flags, iobref);
-                return 0;
-        }
-
-        process_frame = copy_frame (frame);
-        if (process_frame == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_wb_mt_wb_local_t);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
+	int32_t       op_errno     = EINVAL;
 
-        frame->local = local;
-        local->file = file;
+	if (wb_fd_err (fd, this, &op_errno))
+		goto unwind;
 
-        stub = fop_writev_stub (frame, NULL, fd, vector, count, offset, flags,
-                                iobref);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+        wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-        request = wb_enqueue (file, stub);
-        if (request == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+	stub = fop_fsync_stub (frame, wb_fsync_helper, fd, datasync, xdata);
+	if (!stub)
+		goto unwind;
 
-        ret = wb_process_queue (process_frame, file);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "request queue processing failed");
-        }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-        STACK_DESTROY (process_frame->root);
+	wb_process_queue (wb_inode);
 
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
-        if (process_frame) {
-                STACK_DESTROY (process_frame->root);
-        }
+        return 0;
+
+noqueue:
+        STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+        return 0;
+}
 
-        if (stub) {
-                call_stub_destroy (stub);
-        }
 
+int
+wb_stat_helper (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
         return 0;
 }
 
 
-int32_t
-wb_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iovec *vector, int32_t count,
-              struct iatt *stbuf, struct iobref *iobref)
+int
+wb_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        wb_local_t   *local   = NULL;
-        wb_file_t    *file    = NULL;
-        wb_request_t *request = NULL;
-        int32_t       ret     = 0;
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
 
-        GF_ASSERT (frame);
 
-        local = frame->local;
-        file = local->file;
-        request = local->request;
+	wb_inode = wb_inode_ctx_get (this, loc->inode);
+        if (!wb_inode)
+		goto noqueue;
 
-        if ((request != NULL) && (file != NULL)) {
-                wb_request_unref (request);
+	stub = fop_stat_stub (frame, wb_stat_helper, loc, xdata);
+	if (!stub)
+		goto unwind;
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        if (errno == ENOMEM) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                        }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-                        gf_log (frame->this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        }
+	wb_process_queue (wb_inode);
+
+        return 0;
 
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
+unwind:
+        STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM, NULL, NULL);
 
+        if (stub)
+                call_stub_destroy (stub);
         return 0;
+
+noqueue:
+        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+	return 0;
 }
 
 
-static int32_t
-wb_readv_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t offset, uint32_t flags)
+int
+wb_fstat_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, wb_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags);
-
+        STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
         return 0;
 }
 
 
-int32_t
-wb_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags)
+int
+wb_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        wb_file_t    *file     = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        int32_t       ret      = -1, op_errno = 0;
-        wb_request_t *request  = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, unwind,
-                                        op_errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, fd, unwind, op_errno,
-                                        EINVAL);
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
+        wb_inode_t   *wb_inode     = NULL;
+	call_stub_t  *stub         = NULL;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_wb_mt_wb_local_t);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
 
-        local->file = file;
+        wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-        frame->local = local;
-        if (file) {
-                stub = fop_readv_stub (frame, wb_readv_helper, fd, size,
-                                       offset, flags);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	stub = fop_fstat_stub (frame, wb_fstat_helper, fd, xdata);
+	if (!stub)
+		goto unwind;
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        call_stub_destroy (stub);
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_readv_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readv,
-                            fd, size, offset, flags);
-        }
+	wb_process_queue (wb_inode);
 
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, NULL, NULL);
+
+        if (stub)
+                call_stub_destroy (stub);
+        return 0;
+
+noqueue:
+        STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
         return 0;
 }
 
 
-int32_t
-wb_ffr_bg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno)
+int
+wb_truncate_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    off_t offset, dict_t *xdata)
 {
-        STACK_DESTROY (frame->root);
+        STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
         return 0;
 }
 
 
-int32_t
-wb_ffr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-            int32_t op_errno)
+int
+wb_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        wb_local_t *local = NULL;
-        wb_file_t  *file  = NULL;
-
-        GF_ASSERT (frame);
+        wb_inode_t   *wb_inode     = NULL;
+	call_stub_t  *stub         = NULL;
 
-        local = frame->local;
-        file = local->file;
+	wb_inode = wb_inode_create (this, loc->inode);
+	if (!wb_inode)
+		goto unwind;
 
-        if (file != NULL) {
-                LOCK (&file->lock);
-                {
-                        if (file->op_ret == -1) {
-                                op_ret = file->op_ret;
-                                op_errno = file->op_errno;
+	stub = fop_truncate_stub (frame, wb_truncate_helper, loc,
+				  offset, xdata);
+	if (!stub)
+		goto unwind;
 
-                                file->op_ret = 0;
-                        }
-                }
-                UNLOCK (&file->lock);
-        }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+	wb_process_queue (wb_inode);
 
         return 0;
-}
 
+unwind:
+        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-int32_t
-wb_flush_helper (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        wb_conf_t    *conf        = NULL;
-        wb_local_t   *local       = NULL;
-        wb_file_t    *file        = NULL;
-        call_frame_t *flush_frame = NULL, *process_frame = NULL;
-        int32_t       op_ret      = -1, op_errno = -1, ret = -1;
+        if (stub)
+                call_stub_destroy (stub);
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, unwind,
-                                        op_errno, EINVAL);
+        return 0;
+}
 
-        conf = this->private;
 
-        local = frame->local;
-        file = local->file;
+int
+wb_ftruncate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     off_t offset, dict_t *xdata)
+{
+        STACK_WIND (frame, default_ftruncate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        return 0;
+}
 
-        LOCK (&file->lock);
-        {
-                op_ret = file->op_ret;
-                op_errno = file->op_errno;
-        }
-        UNLOCK (&file->lock);
 
-        if (local && local->request) {
-                process_frame = copy_frame (frame);
-                if (process_frame == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+int
+wb_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
+{
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
+        int32_t       op_errno     = 0;
 
-                wb_request_unref (local->request);
+        wb_inode = wb_inode_create (this, fd->inode);
+	if (!wb_inode) {
+                op_errno = ENOMEM;
+		goto unwind;
         }
 
-        if (conf->flush_behind) {
-                flush_frame = copy_frame (frame);
-                if (flush_frame == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	if (wb_fd_err (fd, this, &op_errno))
+		goto unwind;
 
-                STACK_WIND (flush_frame, wb_ffr_bg_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->flush, fd);
-        } else {
-                STACK_WIND (frame, wb_ffr_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->flush, fd);
+	stub = fop_ftruncate_stub (frame, wb_ftruncate_helper, fd,
+				   offset, xdata);
+	if (!stub) {
+                op_errno = ENOMEM;
+		goto unwind;
         }
 
-        if (process_frame != NULL) {
-                ret = wb_process_queue (process_frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-
-                STACK_DESTROY (process_frame->root);
+	if (!wb_enqueue (wb_inode, stub)) {
+                op_errno = ENOMEM;
+		goto unwind;
         }
 
-        if (conf->flush_behind) {
-                STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
-        }
+	wb_process_queue (wb_inode);
 
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        if (stub)
+                call_stub_destroy (stub);
         return 0;
 }
 
 
-int32_t
-wb_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+int
+wb_setattr_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        wb_conf_t    *conf        = NULL;
-        wb_file_t    *file        = NULL;
-        wb_local_t   *local       = NULL;
-        uint64_t      tmp_file    = 0;
-        call_stub_t  *stub        = NULL;
-        call_frame_t *flush_frame = NULL;
-        wb_request_t *request     = NULL;
-        int32_t       ret         = 0, op_errno = 0;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, unwind,
-                                        op_errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, fd, unwind, op_errno,
-                                        EINVAL);
-
-        conf = this->private;
-
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
+        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+        return 0;
+}
 
-        if (file != NULL) {
-                local = GF_CALLOC (1, sizeof (*local), gf_wb_mt_wb_local_t);
-                if (local == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
 
-                local->file = file;
+int
+wb_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
 
-                frame->local = local;
+	wb_inode = wb_inode_ctx_get (this, loc->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-                stub = fop_flush_stub (frame, wb_flush_helper, fd);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	stub = fop_setattr_stub (frame, wb_setattr_helper, loc, stbuf,
+                                         valid, xdata);
+	if (!stub)
+		goto unwind;
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        call_stub_destroy (stub);
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                if (conf->flush_behind) {
-                        flush_frame = copy_frame (frame);
-                        if (flush_frame == NULL) {
-                                op_errno = ENOMEM;
-                                goto unwind;
-                        }
-
-                        STACK_UNWIND_STRICT (flush, frame, 0, 0);
-
-                        STACK_WIND (flush_frame, wb_ffr_bg_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->flush, fd);
-                } else {
-                        STACK_WIND (frame, wb_ffr_cbk, FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->flush, fd);
-                }
-        }
+	wb_process_queue (wb_inode);
 
         return 0;
-
 unwind:
-        STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
+        STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        if (stub)
+                call_stub_destroy (stub);
+	return 0;
+
+noqueue:
+        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
         return 0;
 }
 
 
-static int32_t
-wb_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf)
+int
+wb_fsetattr_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		    struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        wb_local_t   *local   = NULL;
-        wb_file_t    *file    = NULL;
-        wb_request_t *request = NULL;
-        int32_t       ret     = -1;
+        STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+        return 0;
+}
 
-        GF_ASSERT (frame);
 
-        local = frame->local;
-        file = local->file;
-        request = local->request;
+int
+wb_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	     struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
 
-        if (file != NULL) {
-                LOCK (&file->lock);
-                {
-                        if (file->op_ret == -1) {
-                                op_ret = file->op_ret;
-                                op_errno = file->op_errno;
+	wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-                                file->op_ret = 0;
-                        }
-                }
-                UNLOCK (&file->lock);
-
-                if (request) {
-                        wb_request_unref (request);
-                        ret = wb_process_queue (frame, file);
-                        if (ret == -1) {
-                                if (errno == ENOMEM) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                }
-
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "request queue processing failed");
-                        }
-                }
+	stub = fop_fsetattr_stub (frame, wb_fsetattr_helper, fd, stbuf,
+				  valid, xdata);
+	if (!stub)
+		goto unwind;
 
-        }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+	wb_process_queue (wb_inode);
 
         return 0;
-}
+unwind:
+        STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
+        if (stub)
+                call_stub_destroy (stub);
+	return 0;
 
-static int32_t
-wb_fsync_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 int32_t datasync)
-{
-        STACK_WIND (frame, wb_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, datasync);
+noqueue:
+        STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
         return 0;
 }
 
 
-int32_t
-wb_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync)
+int
+wb_forget (xlator_t *this, inode_t *inode)
 {
-        wb_file_t    *file     = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1, op_errno = 0;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, unwind,
-                                        op_errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, fd, unwind,
-                                        op_errno, EINVAL);
-
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_wb_mt_wb_local_t);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        local->file = file;
+        uint64_t    tmp      = 0;
+        wb_inode_t *wb_inode = NULL;
 
-        frame->local = local;
+        inode_ctx_del (inode, this, &tmp);
 
-        if (file) {
-                stub = fop_fsync_stub (frame, wb_fsync_helper, fd, datasync);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        wb_inode = (wb_inode_t *)(long)tmp;
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        call_stub_destroy (stub);
-                        goto unwind;
-                }
+	if (!wb_inode)
+		return 0;
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_fsync_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsync, fd, datasync);
-        }
+        GF_ASSERT (list_empty (&wb_inode->todo));
+        GF_ASSERT (list_empty (&wb_inode->liability));
+        GF_ASSERT (list_empty (&wb_inode->temptation));
 
-        return 0;
+        GF_FREE (wb_inode);
 
-unwind:
-        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
-int32_t
+int
 wb_release (xlator_t *this, fd_t *fd)
 {
-        uint64_t   file_ptr = 0;
-        wb_file_t *file     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+        uint64_t    tmp      = 0;
 
-        fd_ctx_get (fd, this, &file_ptr);
-        file = (wb_file_t *) (long) file_ptr;
+        fd_ctx_del (fd, this, &tmp);
 
-        if (file != NULL) {
-                LOCK (&file->lock);
-                {
-                        GF_ASSERT (list_empty (&file->request));
-                }
-                UNLOCK (&file->lock);
-
-                wb_file_destroy (file);
-        }
-
-out:
         return 0;
 }
 
@@ -2698,11 +1815,8 @@ wb_priv_dump (xlator_t *this)
 
         gf_proc_dump_write ("aggregate_size", "%d", conf->aggregate_size);
         gf_proc_dump_write ("window_size", "%d", conf->window_size);
-        gf_proc_dump_write ("disable_till", "%d", conf->disable_till);
-        gf_proc_dump_write ("enable_O_SYNC", "%d", conf->enable_O_SYNC);
         gf_proc_dump_write ("flush_behind", "%d", conf->flush_behind);
-        gf_proc_dump_write ("enable_trickling_writes", "%d",
-                            conf->enable_trickling_writes);
+        gf_proc_dump_write ("trickling_writes", "%d", conf->trickling_writes);
 
         ret = 0;
 out:
@@ -2711,131 +1825,111 @@ out:
 
 
 void
-__wb_dump_requests (struct list_head *head, char *prefix, char passive)
+__wb_dump_requests (struct list_head *head, char *prefix)
 {
         char          key[GF_DUMP_MAX_BUF_LEN]        = {0, };
         char          key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, }, flag = 0;
-        wb_request_t *request                         = NULL;
+        wb_request_t *req                             = NULL;
 
-        list_for_each_entry (request, head, list) {
-                gf_proc_dump_build_key (key, prefix, passive ? "passive-request"
-                                        : "active-request");
+        list_for_each_entry (req, head, all) {
                 gf_proc_dump_build_key (key_prefix, key,
-                                        gf_fop_list[request->fop]);
+                                        (char *)gf_fop_list[req->fop]);
 
                 gf_proc_dump_add_section(key_prefix);
 
-                gf_proc_dump_write ("request-ptr", "%p", request);
+                gf_proc_dump_write ("request-ptr", "%p", req);
 
-                gf_proc_dump_write ("refcount", "%d", request->refcount);
+                gf_proc_dump_write ("refcount", "%d", req->refcount);
 
-                if (request->fop == GF_FOP_WRITE) {
-                        flag = request->flags.write_request.stack_wound;
-                        gf_proc_dump_write ("stack_wound", "%d", flag);
+		if (list_empty (&req->todo))
+			gf_proc_dump_write ("wound", "yes");
+		else
+			gf_proc_dump_write ("wound", "no");
 
+                if (req->fop == GF_FOP_WRITE) {
                         gf_proc_dump_write ("size", "%"GF_PRI_SIZET,
-                                            request->write_size);
+                                            req->write_size);
 
                         gf_proc_dump_write ("offset", "%"PRId64,
-                                            request->stub->args.writev.off);
+                                            req->stub->args.offset);
 
-                        flag = request->flags.write_request.write_behind;
-                        gf_proc_dump_write ("write_behind", "%d", flag);
+                        flag = req->ordering.lied;
+                        gf_proc_dump_write ("lied", "%d", flag);
 
-                        flag = request->flags.write_request.got_reply;
-                        gf_proc_dump_write ("got_reply", "%d", flag);
+                        flag = req->ordering.append;
+                        gf_proc_dump_write ("append", "%d", flag);
 
-                        flag = request->flags.write_request.virgin;
-                        gf_proc_dump_write ("virgin", "%d", flag);
+                        flag = req->ordering.fulfilled;
+                        gf_proc_dump_write ("fulfilled", "%d", flag);
 
-                        flag = request->flags.write_request.flush_all;
-                        gf_proc_dump_write ("flush_all", "%d", flag);
-                } else {
-                        flag = request->flags.other_requests.marked_for_resume;
-                        gf_proc_dump_write ("marked_for_resume", "%d", flag);
+                        flag = req->ordering.go;
+                        gf_proc_dump_write ("go", "%d", flag);
                 }
         }
 }
 
 
 int
-wb_file_dump (xlator_t *this, fd_t *fd)
+wb_inode_dump (xlator_t *this, inode_t *inode)
 {
-        wb_file_t *file                            = NULL;
-        uint64_t   tmp_file                        = 0;
-        int32_t    ret                             = -1;
-        char      *path                            = NULL;
+        wb_inode_t *wb_inode                       = NULL;
+        int32_t     ret                            = -1;
+        char       *path                           = NULL;
         char       key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
+        char       uuid_str[64] = {0,};
 
-        if ((fd == NULL) || (this == NULL)) {
+        if ((inode == NULL) || (this == NULL)) {
                 ret = 0;
                 goto out;
         }
 
-        ret = fd_ctx_get (fd, this, &tmp_file);
-        if (ret == -1) {
-                ret = 0;
-                goto out;
-        }
-
-        file = (wb_file_t *)(long)tmp_file;
-        if (file == NULL) {
+        wb_inode = wb_inode_ctx_get (this, inode);
+        if (wb_inode == NULL) {
                 ret = 0;
                 goto out;
         }
 
         gf_proc_dump_build_key (key_prefix, "xlator.performance.write-behind",
-                                "file");
+                                "wb_inode");
 
         gf_proc_dump_add_section (key_prefix);
 
-        __inode_path (fd->inode, NULL, &path);
+        __inode_path (inode, NULL, &path);
         if (path != NULL) {
                 gf_proc_dump_write ("path", "%s", path);
                 GF_FREE (path);
         }
 
-        gf_proc_dump_write ("fd", "%p", fd);
-
-        gf_proc_dump_write ("disabled", "%d", file->disabled);
-
-        gf_proc_dump_write ("disable_till", "%lu", file->disable_till);
+        gf_proc_dump_write ("inode", "%p", inode);
 
-        gf_proc_dump_write ("window_conf", "%"GF_PRI_SIZET, file->window_conf);
+        gf_proc_dump_write ("window_conf", "%"GF_PRI_SIZET,
+                            wb_inode->window_conf);
 
-        gf_proc_dump_write ("window_current", "%"GF_PRI_SIZET, file->window_current);
+        gf_proc_dump_write ("window_current", "%"GF_PRI_SIZET,
+                            wb_inode->window_current);
 
-        gf_proc_dump_write ("flags", "%s", (file->flags & O_APPEND) ? "O_APPEND"
-                            : "!O_APPEND");
 
-        gf_proc_dump_write ("aggregate_current", "%"GF_PRI_SIZET, file->aggregate_current);
-
-        gf_proc_dump_write ("refcount", "%d", file->refcount);
-
-        gf_proc_dump_write ("op_ret", "%d", file->op_ret);
-
-        gf_proc_dump_write ("op_errno", "%d", file->op_errno);
-
-        LOCK (&file->lock);
+        ret = TRY_LOCK (&wb_inode->lock);
+        if (!ret)
         {
-                if (!list_empty (&file->request)) {
-                        __wb_dump_requests (&file->request, key_prefix, 0);
-                }
-
-                if (!list_empty (&file->passive_requests)) {
-                        __wb_dump_requests (&file->passive_requests, key_prefix,
-                                            1);
+                if (!list_empty (&wb_inode->all)) {
+                        __wb_dump_requests (&wb_inode->all, key_prefix);
                 }
+                UNLOCK (&wb_inode->lock);
         }
-        UNLOCK (&file->lock);
 
+        if (ret && wb_inode)
+                gf_proc_dump_write ("Unable to dump the inode information",
+                                    "(Lock acquisition failed) %p (gfid: %s)",
+                                    wb_inode,
+                                    uuid_utoa_r (inode->gfid, uuid_str));
         ret = 0;
 out:
         return ret;
 }
 
 
-int32_t
+int
 mem_acct_init (xlator_t *this)
 {
         int     ret = -1;
@@ -2869,6 +1963,14 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("flush-behind", conf->flush_behind, options, bool,
                           out);
 
+        GF_OPTION_RECONF ("trickling-writes", conf->trickling_writes, options,
+			  bool, out);
+
+        GF_OPTION_RECONF ("strict-O_DIRECT", conf->strict_O_DIRECT, options,
+			  bool, out);
+
+        GF_OPTION_RECONF ("strict-write-ordering", conf->strict_write_ordering,
+			  options, bool, out);
         ret = 0;
 out:
         return ret;
@@ -2891,7 +1993,7 @@ init (xlator_t *this)
 
         if (this->parents == NULL) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
+                        "dangling volume. check volfilex");
         }
 
         conf = GF_CALLOC (1, sizeof (*conf), gf_wb_mt_wb_conf_t);
@@ -2899,14 +2001,9 @@ init (xlator_t *this)
                 goto out;
         }
 
-        GF_OPTION_INIT("enable-O_SYNC", conf->enable_O_SYNC, bool, out);
-
         /* configure 'options aggregate-size <size>' */
         conf->aggregate_size = WB_AGGREGATE_SIZE;
 
-        GF_OPTION_INIT("disable-for-first-nbytes", conf->disable_till, size,
-                       out);
-
         /* configure 'option window-size <size>' */
         GF_OPTION_INIT ("cache-size", conf->window_size, size, out);
 
@@ -2929,16 +2026,19 @@ init (xlator_t *this)
         /* configure 'option flush-behind <on/off>' */
         GF_OPTION_INIT ("flush-behind", conf->flush_behind, bool, out);
 
-        GF_OPTION_INIT ("enable-trickling-writes", conf->enable_trickling_writes,
-                        bool, out);
+        GF_OPTION_INIT ("trickling-writes", conf->trickling_writes, bool, out);
+
+        GF_OPTION_INIT ("strict-O_DIRECT", conf->strict_O_DIRECT, bool, out);
+
+        GF_OPTION_INIT ("strict-write-ordering", conf->strict_write_ordering,
+			bool, out);
 
         this->private = conf;
         ret = 0;
 
 out:
         if (ret) {
-                if (conf)
-                        GF_FREE (conf);
+                GF_FREE (conf);
         }
         return ret;
 }
@@ -2966,8 +2066,6 @@ out:
 
 struct xlator_fops fops = {
         .writev      = wb_writev,
-        .open        = wb_open,
-        .create      = wb_create,
         .readv       = wb_readv,
         .flush       = wb_flush,
         .fsync       = wb_fsync,
@@ -2976,17 +2074,22 @@ struct xlator_fops fops = {
         .truncate    = wb_truncate,
         .ftruncate   = wb_ftruncate,
         .setattr     = wb_setattr,
+        .fsetattr    = wb_fsetattr,
 };
 
+
 struct xlator_cbks cbks = {
+        .forget   = wb_forget,
         .release  = wb_release
 };
 
+
 struct xlator_dumpops dumpops = {
         .priv      =  wb_priv_dump,
-        .fdctx     =  wb_file_dump,
+        .inodectx  =  wb_inode_dump,
 };
 
+
 struct volume_options options[] = {
         { .key  = {"flush-behind"},
           .type = GF_OPTION_TYPE_BOOL,
@@ -2995,29 +2098,31 @@ struct volume_options options[] = {
                           "translator to perform flush in background, by "
                           "returning success (or any errors, if any of "
                           "previous  writes were failed) to application even "
-                          "before flush is sent to backend filesystem. "
+                          "before flush FOP is sent to backend filesystem. "
         },
         { .key  = {"cache-size", "window-size"},
           .type = GF_OPTION_TYPE_SIZET,
           .min  = 512 * GF_UNIT_KB,
           .max  = 1 * GF_UNIT_GB,
           .default_value = "1MB",
-          .description = "Size of the per-file write-behind buffer. "
-
+          .description = "Size of the write-behind buffer for a single file "
+                         "(inode)."
         },
-        { .key = {"disable-for-first-nbytes"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .min = 0,
-          .max = 1 * GF_UNIT_MB,
-          .default_value = "0",
-        },
-        { .key = {"enable-O_SYNC"},
+        { .key = {"trickling-writes"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
         },
-        { .key = {"enable-trickling-writes"},
+        { .key = {"strict-O_DIRECT"},
           .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
+          .default_value = "off",
+          .description = "This option when set to off, ignores the "
+          "O_DIRECT flag."
+        },
+        { .key = {"strict-write-ordering"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+	  .description = "Do not let later writes overtake earlier writes even "
+	                  "if they do not overlap",
         },
         { .key = {NULL} },
 };
diff --git a/xlators/playground/Makefile.am b/xlators/playground/Makefile.am
new file mode 100644
index 000000000..e7de6b31a
--- /dev/null
+++ b/xlators/playground/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = template
+CLEANFILES =
diff --git a/xlators/playground/template/Makefile.am b/xlators/playground/template/Makefile.am
new file mode 100644
index 000000000..f26892443
--- /dev/null
+++ b/xlators/playground/template/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = src
+
diff --git a/xlators/playground/template/src/Makefile.am b/xlators/playground/template/src/Makefile.am
new file mode 100644
index 000000000..21f1c5f6b
--- /dev/null
+++ b/xlators/playground/template/src/Makefile.am
@@ -0,0 +1,16 @@
+xlator_LTLIBRARIES = template.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
+
+template_la_LDFLAGS = -module -avoid-version
+
+template_la_SOURCES = template.c
+template_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = template.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
diff --git a/xlators/playground/template/src/template.c b/xlators/playground/template/src/template.c
new file mode 100644
index 000000000..37a7794a0
--- /dev/null
+++ b/xlators/playground/template/src/template.c
@@ -0,0 +1,49 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "template.h"
+
+int32_t
+init (xlator_t *this)
+{
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "not configured with exactly one child. exiting");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+        return 0;
+}
+
+void
+fini (xlator_t *this)
+{
+        return;
+}
+
+struct xlator_fops fops = {
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {NULL} },
+};
diff --git a/xlators/playground/template/src/template.h b/xlators/playground/template/src/template.h
new file mode 100644
index 000000000..d6aced501
--- /dev/null
+++ b/xlators/playground/template/src/template.h
@@ -0,0 +1,24 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __TEMPLATE_H__
+#define __TEMPLATE_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "defaults.h"
+
+#endif /* __TEMPLATE_H__ */
diff --git a/xlators/protocol/auth/addr/src/Makefile.am b/xlators/protocol/auth/addr/src/Makefile.am
index 7f1dd7445..426e7c2fb 100644
--- a/xlators/protocol/auth/addr/src/Makefile.am
+++ b/xlators/protocol/auth/addr/src/Makefile.am
@@ -1,11 +1,14 @@
 auth_LTLIBRARIES = addr.la
 authdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/auth
 
-addr_la_LDFLAGS = -module -avoidversion
+addr_la_LDFLAGS = -module -avoid-version
 
 addr_la_SOURCES = addr.c
 addr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
-	-I$(top_srcdir)/xlators/protocol/server/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/protocol/server/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src/
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/protocol/auth/addr/src/addr.c b/xlators/protocol/auth/addr/src/addr.c
index 0035d55ba..64e8d0fc6 100644
--- a/xlators/protocol/auth/addr/src/addr.c
+++ b/xlators/protocol/auth/addr/src/addr.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -28,6 +19,7 @@
 #include <netdb.h>
 #include "authenticate.h"
 #include "dict.h"
+#include "rpc-transport.h"
 
 #define ADDR_DELIMITER " ,"
 #define PRIVILEGED_PORT_CEILING 1024
@@ -36,13 +28,6 @@
 #define AF_INET_SDP 27
 #endif
 
-/* TODO: duplicate declaration */
-typedef struct peer_info {
-        struct sockaddr_storage sockaddr;
-        socklen_t sockaddr_len;
-        char identifier[UNIX_PATH_MAX];
-}peer_info_t;
-
 auth_result_t
 gf_auth (dict_t *input_params, dict_t *config_params)
 {
@@ -224,22 +209,21 @@ gf_auth (dict_t *input_params, dict_t *config_params)
         }
 
 out:
-        if (addr_cpy)
-                GF_FREE (addr_cpy);
+        GF_FREE (addr_cpy);
 
         return result;
 }
 
 struct volume_options options[] = {
         { .key   = {"auth.addr.*.allow"},
-          .type  = GF_OPTION_TYPE_ANY
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST
         },
         { .key   = {"auth.addr.*.reject"},
-          .type  = GF_OPTION_TYPE_ANY
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST
         },
         /* Backword compatibility */
         { .key   = {"auth.ip.*.allow"},
-          .type  = GF_OPTION_TYPE_ANY
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST
         },
         { .key = {NULL} }
 };
diff --git a/xlators/protocol/auth/login/src/Makefile.am b/xlators/protocol/auth/login/src/Makefile.am
index b4719d1a7..d84db91c4 100644
--- a/xlators/protocol/auth/login/src/Makefile.am
+++ b/xlators/protocol/auth/login/src/Makefile.am
@@ -1,11 +1,12 @@
 auth_LTLIBRARIES = login.la
 authdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/auth
 
-login_la_LDFLAGS = -module -avoidversion
+login_la_LDFLAGS = -module -avoid-version
 
 login_la_SOURCES = login.c
 login_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/protocol/server/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/protocol/auth/login/src/login.c b/xlators/protocol/auth/login/src/login.c
index 87ee77fcf..c2f0bf0d0 100644
--- a/xlators/protocol/auth/login/src/login.c
+++ b/xlators/protocol/auth/login/src/login.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -121,8 +112,7 @@ auth_result_t gf_auth (dict_t *input_params, dict_t *config_params)
         }
 
 out:
-        if (username_cpy)
-                GF_FREE (username_cpy);
+        GF_FREE (username_cpy);
 
         return result;
 }
diff --git a/xlators/protocol/client/src/Makefile.am b/xlators/protocol/client/src/Makefile.am
index 29c6e9dde..cf89d42da 100644
--- a/xlators/protocol/client/src/Makefile.am
+++ b/xlators/protocol/client/src/Makefile.am
@@ -2,16 +2,19 @@
 xlator_LTLIBRARIES = client.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/protocol
 
-client_la_LDFLAGS = -module -avoidversion
+client_la_LDFLAGS = -module -avoid-version
 
 client_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
-client_la_SOURCES = client.c client-helpers.c client3_1-fops.c  \
+client_la_SOURCES = client.c client-helpers.c client-rpc-fops.c  \
 	client-handshake.c client-callback.c client-lk.c
+
 noinst_HEADERS = client.h client-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)  \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src/
-\ No newline at end of file
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src/
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/protocol/client/src/client-callback.c b/xlators/protocol/client/src/client-callback.c
index 2d960ab1a..d886862f7 100644
--- a/xlators/protocol/client/src/client-callback.c
+++ b/xlators/protocol/client/src/client-callback.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -26,7 +17,7 @@
 #include "rpc-clnt.h"
 
 int
-client_cbk_null (void *data)
+client_cbk_null (struct rpc_clnt *rpc, void *mydata, void *data)
 {
         gf_log (THIS->name, GF_LOG_WARNING,
                 "this function should not be called");
@@ -34,7 +25,7 @@ client_cbk_null (void *data)
 }
 
 int
-client_cbk_fetchspec (void *data)
+client_cbk_fetchspec (struct rpc_clnt *rpc, void *mydata, void *data)
 {
         gf_log (THIS->name, GF_LOG_WARNING,
                 "this function should not be called");
@@ -42,7 +33,7 @@ client_cbk_fetchspec (void *data)
 }
 
 int
-client_cbk_ino_flush (void *data)
+client_cbk_ino_flush (struct rpc_clnt *rpc, void *mydata, void *data)
 {
         gf_log (THIS->name, GF_LOG_WARNING,
                 "this function should not be called");
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 91cda6d0c..5668fea53 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -22,6 +13,7 @@
 #include "config.h"
 #endif
 
+#include "fd-lk.h"
 #include "client.h"
 #include "xlator.h"
 #include "defaults.h"
@@ -33,12 +25,25 @@
 #include "portmap-xdr.h"
 #include "rpc-common-xdr.h"
 
-extern rpc_clnt_prog_t clnt3_1_fop_prog;
+#define CLIENT_REOPEN_MAX_ATTEMPTS 1024
+extern rpc_clnt_prog_t clnt3_3_fop_prog;
 extern rpc_clnt_prog_t clnt_pmap_prog;
 
 int client_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe);
 
+int client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
+                               int count, void *myframe);
+
+int client_set_lk_version (xlator_t *this);
+
+typedef struct client_fd_lk_local {
+        int             ref;
+        gf_boolean_t    error;
+        gf_lock_t       lock;
+        clnt_fd_ctx_t *fdctx;
+}clnt_fd_lk_local_t;
+
 /* Handshake */
 
 void
@@ -48,7 +53,7 @@ rpc_client_ping_timer_expired (void *data)
         rpc_clnt_connection_t   *conn               = NULL;
         int                      disconnect         = 0;
         int                      transport_activity = 0;
-        struct timeval           timeout            = {0, };
+        struct timespec          timeout            = {0, };
         struct timeval           current            = {0, };
         struct rpc_clnt         *clnt               = NULL;
         xlator_t                *this               = NULL;
@@ -96,7 +101,7 @@ rpc_client_ping_timer_expired (void *data)
                                 "ping timer expired but transport activity "
                                 "detected - not bailing transport");
                         timeout.tv_sec = conf->opt.ping_timeout;
-                        timeout.tv_usec = 0;
+                        timeout.tv_nsec = 0;
 
                         conn->ping_timer =
                                 gf_timer_call_after (this->ctx, timeout,
@@ -135,7 +140,7 @@ client_start_ping (void *data)
         clnt_conf_t             *conf        = NULL;
         rpc_clnt_connection_t   *conn        = NULL;
         int32_t                  ret         = -1;
-        struct timeval           timeout     = {0, };
+        struct timespec          timeout     = {0, };
         call_frame_t            *frame       = NULL;
         int                      frame_count = 0;
 
@@ -191,7 +196,7 @@ client_start_ping (void *data)
                 }
 
                 timeout.tv_sec = conf->opt.ping_timeout;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
 
                 conn->ping_timer =
                         gf_timer_call_after (this->ctx, timeout,
@@ -214,14 +219,14 @@ client_start_ping (void *data)
         ret = client_submit_request (this, NULL, frame, conf->handshake,
                                      GF_HNDSK_PING, client_ping_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL, (xdrproc_t)NULL);
-        if (ret)
-                goto fail;
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to start ping timer");
+        }
 
         return;
-fail:
-        gf_log (THIS->name, GF_LOG_ERROR,
-                "failed to start ping timer");
 
+fail:
         if (frame) {
                 STACK_DESTROY (frame->root);
         }
@@ -236,42 +241,47 @@ client_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
 {
         xlator_t              *this    = NULL;
         rpc_clnt_connection_t *conn    = NULL;
-        struct timeval         timeout = {0, };
+        struct timespec        timeout = {0, };
         call_frame_t          *frame   = NULL;
         clnt_conf_t           *conf    = NULL;
 
         if (!myframe) {
-                gf_log (THIS->name, GF_LOG_WARNING, "frame with the request is NULL");
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "frame with the request is NULL");
                 goto out;
         }
         frame = myframe;
         this = frame->this;
         if (!this || !this->private) {
-                gf_log (THIS->name, GF_LOG_WARNING, "xlator private is not set");
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "xlator private is not set");
                 goto out;
         }
 
         conf = this->private;
         conn = &conf->rpc->conn;
 
-        if (req->rpc_status == -1) {
-                if (conn->ping_timer != NULL) {
-                        gf_log (this->name, GF_LOG_WARNING, "socket or ib"
-                                " related error");
-                        gf_timer_call_cancel (this->ctx, conn->ping_timer);
-                        conn->ping_timer = NULL;
-                } else {
-                        /* timer expired and transport bailed out */
-                        gf_log (this->name, GF_LOG_WARNING, "timer must have "
-                                "expired");
-                }
-                goto out;
-        }
-
         pthread_mutex_lock (&conn->lock);
         {
+                if (req->rpc_status == -1) {
+                        if (conn->ping_timer != NULL) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "socket or ib related error");
+                                gf_timer_call_cancel (this->ctx,
+                                                      conn->ping_timer);
+                                conn->ping_timer = NULL;
+                        } else {
+                                /* timer expired and transport bailed out */
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "timer must have expired");
+                        }
+
+                        goto unlock;
+                }
+
+
                 timeout.tv_sec  = conf->opt.ping_timeout;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
 
                 gf_timer_call_cancel (this->ctx,
                                       conn->ping_timer);
@@ -284,6 +294,7 @@ client_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         gf_log (this->name, GF_LOG_WARNING,
                                 "failed to set the ping timer");
         }
+unlock:
         pthread_mutex_unlock (&conn->lock);
 out:
         if (frame)
@@ -333,11 +344,11 @@ client3_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
 out:
-        STACK_UNWIND_STRICT (getspec, frame, rsp.op_ret, rsp.op_errno, rsp.spec);
+        CLIENT_STACK_UNWIND (getspec, frame, rsp.op_ret, rsp.op_errno,
+                             rsp.spec);
 
         /* Don't use 'GF_FREE', this is allocated by libc */
-        if (rsp.spec)
-                free (rsp.spec);
+        free (rsp.spec);
 
         return 0;
 }
@@ -363,13 +374,14 @@ int32_t client3_getspec (call_frame_t *frame, xlator_t *this, void *data)
                                      NULL, NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gf_getspec_req);
 
-        if (ret)
-                goto unwind;
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to send the request");
+        }
 
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the request");
-        STACK_UNWIND_STRICT (getspec, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (getspec, frame, -1, op_errno, NULL);
         return 0;
 
 }
@@ -391,24 +403,554 @@ client_notify_parents_child_up (xlator_t *this)
 }
 
 int
-client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
+clnt_fd_lk_reacquire_failed (xlator_t *this, clnt_fd_ctx_t *fdctx,
+                             clnt_conf_t *conf)
+{
+        int      ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx->remote_fd     = -1;
+                fdctx->lk_heal_state = GF_LK_HEAL_DONE;
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
+                           int count, void *myframe)
+{
+        int32_t           ret    = -1;
+        call_frame_t     *fr     = NULL;
+        gf_set_lk_ver_rsp rsp    = {0,};
+
+        fr = (call_frame_t *) myframe;
+        GF_VALIDATE_OR_GOTO ("client", fr, out);
+
+        if (req->rpc_status == -1) {
+                gf_log (fr->this->name, GF_LOG_WARNING,
+                        "received RPC status error");
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_set_lk_ver_rsp);
+        if (ret < 0)
+                gf_log (fr->this->name, GF_LOG_WARNING,
+                        "xdr decoding failed");
+        else
+                gf_log (fr->this->name, GF_LOG_INFO,
+                        "Server lk version = %d", rsp.lk_ver);
+
+        ret = 0;
+out:
+        if (fr)
+                STACK_DESTROY (fr->root);
+
+        return ret;
+}
+
+//TODO: Check for all released fdctx and destroy them
+int
+client_set_lk_version (xlator_t *this)
+{
+        int                 ret      = -1;
+        clnt_conf_t        *conf     = NULL;
+        call_frame_t       *frame    = NULL;
+        gf_set_lk_ver_req   req      = {0, };
+
+        GF_VALIDATE_OR_GOTO ("client", this, err);
+
+        conf = (clnt_conf_t *) this->private;
+
+        req.lk_ver = client_get_lk_ver (conf);
+        ret = gf_asprintf (&req.uid, "%s-%s-%d",
+                           this->ctx->process_uuid, this->name,
+                           this->graph->id);
+        if (ret == -1)
+                goto err;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sending SET_LK_VERSION");
+
+        ret = client_submit_request (this, &req, frame,
+                                     conf->handshake,
+                                     GF_HNDSK_SET_LK_VER,
+                                     client_set_lk_version_cbk,
+                                     NULL, NULL, 0, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gf_set_lk_ver_req);
+out:
+        GF_FREE (req.uid);
+        return ret;
+err:
+        gf_log (this->name, GF_LOG_WARNING,
+                "Failed to send SET_LK_VERSION to server");
+
+        return ret;
+}
+
+int
+client_fd_lk_count (fd_lk_ctx_t *lk_ctx)
+{
+        int               count     = 0;
+        fd_lk_ctx_node_t *fd_lk   = NULL;
+
+        GF_VALIDATE_OR_GOTO ("client", lk_ctx, err);
+
+        LOCK (&lk_ctx->lock);
+        {
+                list_for_each_entry (fd_lk, &lk_ctx->lk_list, next)
+                        count++;
+        }
+        UNLOCK (&lk_ctx->lock);
+
+        return count;
+err:
+        return -1;
+}
+
+clnt_fd_lk_local_t *
+clnt_fd_lk_local_ref (xlator_t *this, clnt_fd_lk_local_t *local)
+{
+        GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+        LOCK (&local->lock);
+        {
+                local->ref++;
+        }
+        UNLOCK (&local->lock);
+out:
+        return local;
+}
+
+int
+clnt_fd_lk_local_unref (xlator_t *this, clnt_fd_lk_local_t *local)
+{
+        int   ref = -1;
+
+        GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+        LOCK (&local->lock);
+        {
+                ref = --local->ref;
+        }
+        UNLOCK (&local->lock);
+
+        if (ref == 0) {
+                LOCK_DESTROY (&local->lock);
+                GF_FREE (local);
+        }
+out:
+        return ref;
+}
+
+clnt_fd_lk_local_t *
+clnt_fd_lk_local_create (clnt_fd_ctx_t *fdctx)
+{
+        clnt_fd_lk_local_t *local = NULL;
+
+        local = GF_CALLOC (1, sizeof (clnt_fd_lk_local_t),
+                           gf_client_mt_clnt_fd_lk_local_t);
+        if (!local)
+                goto out;
+
+        local->ref    = 1;
+        local->error  = _gf_false;
+        local->fdctx = fdctx;
+
+        LOCK_INIT (&local->lock);
+out:
+        return local;
+}
+
+void
+clnt_mark_fd_bad (clnt_conf_t *conf, clnt_fd_ctx_t *fdctx)
+{
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx->remote_fd = -1;
+        }
+        pthread_mutex_unlock (&conf->lock);
+}
+
+int
+clnt_release_reopen_fd_cbk (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        xlator_t       *this   = NULL;
+        call_frame_t   *frame  = NULL;
+        clnt_conf_t    *conf   = NULL;
+        clnt_fd_ctx_t  *fdctx  = NULL;
+
+        frame  = myframe;
+        this   = frame->this;
+        fdctx  = (clnt_fd_ctx_t *) frame->local;
+        conf   = (clnt_conf_t *) this->private;
+
+        clnt_fd_lk_reacquire_failed (this, fdctx, conf);
+
+        fdctx->reopen_done (fdctx, this);
+
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+
+        return 0;
+}
+
+int
+clnt_release_reopen_fd (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+        int               ret     = -1;
+        clnt_conf_t      *conf    = NULL;
+        call_frame_t     *frame   = NULL;
+        gfs3_release_req  req     = {{0,},};
+
+        conf = (clnt_conf_t *) this->private;
+
+        frame  = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        frame->local = (void *) fdctx;
+        req.fd       = fdctx->remote_fd;
+
+        ret    = client_submit_request (this, &req, frame, conf->fops,
+                                        GFS3_OP_RELEASE,
+                                        clnt_release_reopen_fd_cbk, NULL,
+                                        NULL, 0, NULL, 0, NULL,
+                                        (xdrproc_t)xdr_gfs3_releasedir_req);
+        return 0;
+ out:
+        if (ret) {
+                clnt_fd_lk_reacquire_failed (this, fdctx, conf);
+                fdctx->reopen_done (fdctx, this);
+                if (frame) {
+                        frame->local = NULL;
+                        STACK_DESTROY (frame->root);
+                }
+        }
+        return 0;
+}
+
+int
+clnt_reacquire_lock_error (xlator_t *this, clnt_fd_ctx_t *fdctx,
+                           clnt_conf_t *conf)
+{
+        int32_t   ret       = -1;
+
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        clnt_release_reopen_fd (this, fdctx);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+gf_boolean_t
+clnt_fd_lk_local_error_status (xlator_t *this,
+                               clnt_fd_lk_local_t *local)
+{
+        gf_boolean_t error = _gf_false;
+
+        LOCK (&local->lock);
+        {
+                error = local->error;
+        }
+        UNLOCK (&local->lock);
+
+        return error;
+}
+
+int
+clnt_fd_lk_local_mark_error (xlator_t *this,
+                             clnt_fd_lk_local_t *local)
+{
+        int32_t       ret   = -1;
+        clnt_conf_t  *conf  = NULL;
+        gf_boolean_t  error = _gf_false;
+
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+        conf = (clnt_conf_t *) this->private;
+
+        LOCK (&local->lock);
+        {
+                error        = local->error;
+                local->error = _gf_true;
+        }
+        UNLOCK (&local->lock);
+
+        if (!error)
+                clnt_reacquire_lock_error (this, local->fdctx, conf);
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                           int count, void *myframe)
+{
+        int32_t             ret        = -1;
+        xlator_t           *this       = NULL;
+        gfs3_lk_rsp         rsp        = {0,};
+        call_frame_t       *frame      = NULL;
+        clnt_conf_t        *conf       = NULL;
+        clnt_fd_ctx_t      *fdctx      = NULL;
+        clnt_fd_lk_local_t *local      = NULL;
+        struct gf_flock     lock       = {0,};
+
+        frame = (call_frame_t *) myframe;
+        this  = frame->this;
+        local = (clnt_fd_lk_local_t *) frame->local;
+        conf  = (clnt_conf_t *) this->private;
+
+        if (req->rpc_status == -1) {
+                gf_log ("client", GF_LOG_WARNING,
+                        "request failed at rpc");
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lk_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+                goto out;
+        }
+
+        if (rsp.op_ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "lock request failed");
+                ret = -1;
+                goto out;
+        }
+
+        fdctx = local->fdctx;
+
+        gf_proto_flock_to_flock (&rsp.flock, &lock);
+
+        gf_log (this->name, GF_LOG_DEBUG, "%s type lock reacquired on file "
+                "with gfid %s from %"PRIu64 " to %"PRIu64,
+                get_lk_type (lock.l_type), uuid_utoa (fdctx->gfid),
+                lock.l_start, lock.l_start + lock.l_len);
+
+        if (!clnt_fd_lk_local_error_status (this, local) &&
+            clnt_fd_lk_local_unref (this, local) == 0) {
+                pthread_mutex_lock (&conf->lock);
+                {
+                        fdctx->lk_heal_state = GF_LK_HEAL_DONE;
+                }
+                pthread_mutex_unlock (&conf->lock);
+
+                fdctx->reopen_done (fdctx, this);
+        }
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                clnt_fd_lk_local_mark_error (this, local);
+
+                clnt_fd_lk_local_unref (this, local);
+        }
+
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+
+        return ret;
+}
+
+int
+_client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+        int32_t             ret       = -1;
+        int32_t             gf_cmd    = 0;
+        int32_t             gf_type   = 0;
+        gfs3_lk_req         req       = {{0,},};
+        struct gf_flock     flock     = {0,};
+        fd_lk_ctx_t        *lk_ctx    = NULL;
+        clnt_fd_lk_local_t *local     = NULL;
+        fd_lk_ctx_node_t   *fd_lk     = NULL;
+        call_frame_t       *frame     = NULL;
+        clnt_conf_t        *conf      = NULL;
+
+        conf   = (clnt_conf_t *) this->private;
+        lk_ctx = fdctx->lk_ctx;
+
+        local = clnt_fd_lk_local_create (fdctx);
+        if (!local) {
+                gf_log (this->name, GF_LOG_WARNING, "clnt_fd_lk_local_create "
+                        "failed, aborting reacquring of locks on %s.",
+                        uuid_utoa (fdctx->gfid));
+                clnt_reacquire_lock_error (this, fdctx, conf);
+                goto out;
+        }
+
+        list_for_each_entry (fd_lk, &lk_ctx->lk_list, next) {
+                memcpy (&flock, &fd_lk->user_flock,
+                        sizeof (struct gf_flock));
+
+                /* Always send F_SETLK even if the cmd was F_SETLKW */
+                /* to avoid frame being blocked if lock cannot be granted. */
+                ret = client_cmd_to_gf_cmd (F_SETLK, &gf_cmd);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "client_cmd_to_gf_cmd failed, "
+                                "aborting reacquiring of locks");
+                        break;
+                }
+
+                gf_type   = client_type_to_gf_type (flock.l_type);
+                req.fd    = fdctx->remote_fd;
+                req.cmd   = gf_cmd;
+                req.type  = gf_type;
+                (void) gf_proto_flock_from_flock (&req.flock,
+                                                  &flock);
+
+                memcpy (req.gfid, fdctx->gfid, 16);
+
+                frame = create_frame (this, this->ctx->pool);
+                if (!frame) {
+                        ret = -1;
+                        break;
+                }
+
+                frame->local          = clnt_fd_lk_local_ref (this, local);
+                frame->root->lk_owner = fd_lk->user_flock.l_owner;
+
+                ret = client_submit_request (this, &req, frame,
+                                             conf->fops, GFS3_OP_LK,
+                                             client_reacquire_lock_cbk,
+                                             NULL, NULL, 0, NULL, 0, NULL,
+                                             (xdrproc_t)xdr_gfs3_lk_req);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "reacquiring locks failed on file with gfid %s",
+                                uuid_utoa (fdctx->gfid));
+                        break;
+                }
+
+                ret   = 0;
+                frame = NULL;
+        }
+
+        if (local)
+                (void) clnt_fd_lk_local_unref (this, local);
+out:
+        return ret;
+}
+
+int
+client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+        int32_t          ret       = -1;
+        fd_lk_ctx_t     *lk_ctx    = NULL;
+
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
+
+        if (client_fd_lk_list_empty (fdctx->lk_ctx, _gf_false)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "fd lock list is empty");
+                fdctx->reopen_done (fdctx, this);
+        } else {
+                lk_ctx = fdctx->lk_ctx;
+
+                LOCK (&lk_ctx->lock);
+                {
+                        (void) _client_reacquire_lock (this, fdctx);
+                }
+                UNLOCK (&lk_ctx->lock);
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+client_default_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
+{
+        gf_log_callingfn (this->name, GF_LOG_WARNING,
+                          "This function should never be called");
+}
+
+void
+client_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
+{
+        clnt_conf_t  *conf    = NULL;
+        gf_boolean_t destroy  = _gf_false;
+
+        conf = this->private;
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx->reopen_attempts = 0;
+                if (!fdctx->released)
+                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
+                else
+                        destroy = _gf_true;
+                fdctx->reopen_done = client_default_reopen_done;
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        if (destroy)
+                client_fdctx_destroy (this, fdctx);
+}
+
+void
+client_child_up_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
+{
+        clnt_conf_t  *conf    = NULL;
+        uint64_t     fd_count = 0;
+
+        conf = this->private;
+
+        LOCK (&conf->rec_lock);
+        {
+                fd_count = --(conf->reopen_fd_count);
+        }
+        UNLOCK (&conf->rec_lock);
+
+        client_reopen_done (fdctx, this);
+        if (fd_count == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "last fd open'd/lock-self-heal'd - notifying CHILD-UP");
+                client_set_lk_version (this);
+                client_notify_parents_child_up (this);
+        }
+}
+
+int
+client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void           *myframe)
 {
         int32_t        ret                   = -1;
         gfs3_open_rsp  rsp                   = {0,};
-        int            attempt_lock_recovery = _gf_false;
-        uint64_t       fd_count              = 0;
+        gf_boolean_t   attempt_lock_recovery = _gf_false;
         clnt_local_t  *local                 = NULL;
         clnt_conf_t   *conf                  = NULL;
         clnt_fd_ctx_t *fdctx                 = NULL;
         call_frame_t  *frame                 = NULL;
+        xlator_t      *this                  = NULL;
 
         frame = myframe;
-        if (!frame || !frame->this)
-                goto out;
-
+        this  = frame->this;
+        conf  = this->private;
         local = frame->local;
-        conf  = frame->this->private;
+        fdctx = local->fdctx;
 
         if (-1 == req->rpc_status) {
                 gf_log (frame->this->name, GF_LOG_WARNING,
@@ -441,50 +983,38 @@ client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
-        fdctx = local->fdctx;
-
-        if (!fdctx) {
-                gf_log (frame->this->name, GF_LOG_WARNING, "fdctx not found");
-                ret = -1;
-                goto out;
-        }
-
         pthread_mutex_lock (&conf->lock);
         {
                 fdctx->remote_fd = rsp.fd;
                 if (!fdctx->released) {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                        if (!list_empty (&fdctx->lock_list))
+                        if (conf->lk_heal &&
+                            !client_fd_lk_list_empty (fdctx->lk_ctx,
+                                                      _gf_false)) {
                                 attempt_lock_recovery = _gf_true;
-                        fdctx = NULL;
+                                fdctx->lk_heal_state  = GF_LK_HEAL_IN_PROGRESS;
+                        }
                 }
         }
         pthread_mutex_unlock (&conf->lock);
 
         ret = 0;
 
-        attempt_lock_recovery = _gf_false; /* temporarily */
-
         if (attempt_lock_recovery) {
-                ret = client_attempt_lock_recovery (frame->this, local->fdctx);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "lock recovery not attempted on fd");
-                } else {
-                        gf_log (frame->this->name, GF_LOG_INFO,
-                                "need to attempt lock recovery on %"PRIu64
-                                " open fds", fd_count);
+                /* Delay decrementing the reopen fd count untill all the
+                   locks corresponding to this fd are acquired.*/
+                gf_log (this->name, GF_LOG_DEBUG, "acquiring locks "
+                        "on %s", local->loc.path);
+                ret = client_reacquire_lock (frame->this, local->fdctx);
+                if (ret) {
+                        clnt_reacquire_lock_error (this, local->fdctx, conf);
+                        gf_log (this->name, GF_LOG_WARNING, "acquiring locks "
+                                "failed on %s", local->loc.path);
                 }
-        } else {
-                fd_count = decrement_reopen_fd_count (frame->this, conf);
         }
-out:
-
-        if (fdctx)
-                client_fdctx_destroy (frame->this, fdctx);
 
-        if ((ret < 0) && frame && frame->this && conf)
-                decrement_reopen_fd_count (frame->this, conf);
+out:
+        if (!attempt_lock_recovery)
+                fdctx->reopen_done (fdctx, this);
 
         frame->local = NULL;
         STACK_DESTROY (frame->root);
@@ -495,7 +1025,7 @@ out:
 }
 
 int
-client3_1_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                          void           *myframe)
 {
         int32_t        ret   = -1;
@@ -506,11 +1036,10 @@ client3_1_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
         call_frame_t  *frame = NULL;
 
         frame = myframe;
-        if (!frame || !frame->this)
-                goto out;
+        local = frame->local;
+        fdctx = local->fdctx;
+        conf  = frame->this->private;
 
-        local        = frame->local;
-        conf         = frame->this->private;
 
         if (-1 == req->rpc_status) {
                 gf_log (frame->this->name, GF_LOG_WARNING,
@@ -543,78 +1072,44 @@ client3_1_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
-        fdctx = local->fdctx;
-        if (!fdctx) {
-                gf_log (frame->this->name, GF_LOG_WARNING, "fdctx not found");
-                ret = -1;
-                goto out;
-        }
-
         pthread_mutex_lock (&conf->lock);
         {
                 fdctx->remote_fd = rsp.fd;
-
-                if (!fdctx->released) {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                        fdctx = NULL;
-                }
         }
         pthread_mutex_unlock (&conf->lock);
 
-        decrement_reopen_fd_count (frame->this, conf);
-        ret = 0;
-
 out:
-        if (fdctx)
-                client_fdctx_destroy (frame->this, fdctx);
-
-        if ((ret < 0) && frame && frame->this && conf)
-                decrement_reopen_fd_count (frame->this, conf);
-
-        if (frame) {
-                frame->local = NULL;
-                STACK_DESTROY (frame->root);
-        }
+        fdctx->reopen_done (fdctx, frame->this);
 
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
         client_local_wipe (local);
 
         return 0;
 }
 
-int
-protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx)
+static int
+protocol_client_reopendir (clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
         int               ret   = -1;
         gfs3_opendir_req  req   = {{0,},};
         clnt_local_t     *local = NULL;
-        inode_t          *inode = NULL;
-        char             *path  = NULL;
         call_frame_t     *frame = NULL;
         clnt_conf_t      *conf  = NULL;
 
-        if (!this || !fdctx)
-                goto out;
-
-        inode = fdctx->inode;
         conf = this->private;
 
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "couldn't build path from inode %s",
-                        uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 ret = -1;
                 goto out;
         }
-
         local->fdctx    = fdctx;
-        local->loc.path = path;
-        path            = NULL;
+
+        uuid_copy (local->loc.gfid, fdctx->gfid);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
 
         frame = create_frame (this, this->ctx->pool);
         if (!frame) {
@@ -622,27 +1117,26 @@ protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx)
                 goto out;
         }
 
-        memcpy (req.gfid, inode->gfid, 16);
+        memcpy (req.gfid, fdctx->gfid, 16);
 
         gf_log (frame->this->name, GF_LOG_DEBUG,
                 "attempting reopen on %s", local->loc.path);
 
-        frame->local = local; local = NULL;
+        frame->local = local;
 
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_OPENDIR,
-                                     client3_1_reopendir_cbk, NULL,
+                                     client3_3_reopendir_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_opendir_req);
-        if (ret)
-                goto out;
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to send the re-opendir request");
+        }
 
-        return ret;
+        return 0;
 
 out:
-        gf_log (THIS->name, GF_LOG_ERROR,
-                "failed to send the re-opendir request");
-
         if (frame) {
                 frame->local = NULL;
                 STACK_DESTROY (frame->root);
@@ -651,79 +1145,62 @@ out:
         if (local)
                 client_local_wipe (local);
 
-        if (path)
-                GF_FREE (path);
-        if ((ret < 0) && this && conf) {
-                decrement_reopen_fd_count (this, conf);
-        }
+        fdctx->reopen_done (fdctx, this);
 
         return 0;
 
 }
 
-int
-protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx)
+static int
+protocol_client_reopenfile (clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
         int            ret   = -1;
         gfs3_open_req  req   = {{0,},};
         clnt_local_t  *local = NULL;
-        inode_t       *inode = NULL;
-        char          *path  = NULL;
         call_frame_t  *frame = NULL;
         clnt_conf_t   *conf  = NULL;
 
-        if (!this || !fdctx)
-                goto out;
-
-        inode = fdctx->inode;
         conf  = this->private;
 
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "couldn't build path from inode %s",
-                        uuid_utoa (inode->gfid));
-                goto out;
-        }
-
         frame = create_frame (this, this->ctx->pool);
         if (!frame) {
                 ret = -1;
                 goto out;
         }
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 ret = -1;
                 goto out;
         }
 
         local->fdctx    = fdctx;
-        local->loc.path = path;
-        path            = NULL;
+        uuid_copy (local->loc.gfid, fdctx->gfid);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
+
         frame->local    = local;
 
-        memcpy (req.gfid, inode->gfid, 16);
+        memcpy (req.gfid, fdctx->gfid, 16);
         req.flags    = gf_flags_from_flags (fdctx->flags);
-        req.wbflags  = fdctx->wbflags;
+        req.flags    = req.flags & (~(O_TRUNC|O_CREAT|O_EXCL));
 
         gf_log (frame->this->name, GF_LOG_DEBUG,
                 "attempting reopen on %s", local->loc.path);
 
-        local = NULL;
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPEN, client3_1_reopen_cbk, NULL,
+                                     GFS3_OP_OPEN, client3_3_reopen_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_open_req);
-        if (ret)
-                goto out;
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to send the re-open request");
+        }
 
-        return ret;
+        return 0;
 
 out:
-        gf_log (THIS->name, GF_LOG_ERROR,
-                "failed to send the re-open request");
-
         if (frame) {
                 frame->local = NULL;
                 STACK_DESTROY (frame->root);
@@ -732,17 +1209,65 @@ out:
         if (local)
                 client_local_wipe (local);
 
-        if (path)
-                GF_FREE (path);
-
-        if ((ret < 0) && this && conf) {
-                decrement_reopen_fd_count (this, conf);
-        }
+        fdctx->reopen_done (fdctx, this);
 
         return 0;
 
 }
 
+static void
+protocol_client_reopen (clnt_fd_ctx_t *fdctx, xlator_t *this)
+{
+        if (fdctx->is_dir)
+                protocol_client_reopendir (fdctx, this);
+        else
+                protocol_client_reopenfile (fdctx, this);
+}
+
+gf_boolean_t
+__is_fd_reopen_in_progress (clnt_fd_ctx_t *fdctx)
+{
+        if (fdctx->reopen_done == client_default_reopen_done)
+                return _gf_false;
+        return _gf_true;
+}
+
+void
+client_attempt_reopen (fd_t *fd, xlator_t *this)
+{
+        clnt_conf_t   *conf  = NULL;
+        clnt_fd_ctx_t *fdctx = NULL;
+        gf_boolean_t  reopen = _gf_false;
+
+        if (!fd || !this)
+                goto out;
+
+        conf = this->private;
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx = this_fd_get_ctx (fd, this);
+                if (!fdctx)
+                        goto unlock;
+                if (__is_fd_reopen_in_progress (fdctx))
+                        goto unlock;
+                if (fdctx->remote_fd != -1)
+                        goto unlock;
+
+                if (fdctx->reopen_attempts == CLIENT_REOPEN_MAX_ATTEMPTS) {
+                        reopen = _gf_true;
+                        fdctx->reopen_done = client_reopen_done;
+                        list_del_init (&fdctx->sfd_pos);
+                } else {
+                        fdctx->reopen_attempts++;
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&conf->lock);
+        if (reopen)
+                protocol_client_reopen (fdctx, this);
+out:
+        return;
+}
 
 int
 client_post_handshake (call_frame_t *frame, xlator_t *this)
@@ -767,6 +1292,7 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
                         if (fdctx->remote_fd != -1)
                                 continue;
 
+                        fdctx->reopen_done = client_child_up_reopen_done;
                         list_del_init (&fdctx->sfd_pos);
                         list_add_tail (&fdctx->sfd_pos, &reopen_head);
                         count++;
@@ -785,14 +1311,12 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
                 list_for_each_entry_safe (fdctx, tmp, &reopen_head, sfd_pos) {
                         list_del_init (&fdctx->sfd_pos);
 
-                        if (fdctx->is_dir)
-                                protocol_client_reopendir (this, fdctx);
-                        else
-                                protocol_client_reopen (this, fdctx);
+                        protocol_client_reopen (fdctx, this);
                 }
         } else {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "no open fds - notifying all parents child up");
+                        "No fds to open - notifying all parents child up");
+                client_set_lk_version (this);
                 client_notify_parents_child_up (this);
         }
 out:
@@ -814,6 +1338,7 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
         int32_t               op_ret        = 0;
         int32_t               op_errno      = 0;
         gf_boolean_t          auth_fail     = _gf_false;
+        uint32_t              lk_ver        = 0;
 
         frame = myframe;
         this  = frame->this;
@@ -872,8 +1397,7 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
                         remote_error ? remote_error : strerror (op_errno));
                 errno = op_errno;
                 if (remote_error &&
-                   (strncmp ("Authentication failed",remote_error,
-                             sizeof (remote_error)) == 0)) {
+                    (strcmp ("Authentication failed", remote_error) == 0)) {
                         auth_fail = _gf_true;
                         op_ret = 0;
                 }
@@ -895,6 +1419,15 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
                 goto out;
         }
 
+        ret = dict_get_uint32 (reply, "clnt-lk-version", &lk_ver);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to find key 'clnt-lk-version' in the options");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "clnt-lk-version = %d, "
+                "server-lk-version = %d", client_get_lk_ver (conf), lk_ver);
         /* TODO: currently setpeer path is broken */
         /*
         if (process_uuid && req->conn &&
@@ -930,8 +1463,19 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
 
         conf->need_different_port = 0;
 
-        /* TODO: more to test */
-        client_post_handshake (frame, frame->this);
+        if (lk_ver != client_get_lk_ver (conf)) {
+                gf_log (this->name, GF_LOG_INFO, "Server and Client "
+                        "lk-version numbers are not same, reopening the fds");
+                client_mark_fd_bad (this);
+                client_post_handshake (frame, frame->this);
+        } else {
+                /*TODO: Traverse the saved fd list, and send
+                  release to the server on fd's that were closed
+                  during grace period */
+                gf_log (this->name, GF_LOG_INFO, "Server and Client "
+                        "lk-version numbers are same, no need to "
+                        "reopen the fds");
+        }
 
 out:
         if (auth_fail) {
@@ -960,8 +1504,7 @@ out:
                 ret = 0;
         }
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
 
         STACK_DESTROY (frame->root);
 
@@ -1005,8 +1548,13 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
                 }
         }
 
-        ret = gf_asprintf (&process_uuid_xl, "%s-%s", this->ctx->process_uuid,
-                           this->name);
+        /* With multiple graphs possible in the same process, we need a
+           field to bring the uniqueness. Graph-ID should be enough to get the
+           job done
+        */
+        ret = gf_asprintf (&process_uuid_xl, "%s-%s-%d",
+                           this->ctx->process_uuid, this->name,
+                           this->graph->id);
         if (-1 == ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "asprintf failed while setting process_uuid");
@@ -1043,13 +1591,22 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
                                 "failed to set 'volfile-checksum'");
         }
 
-        req.dict.dict_len = dict_serialized_length (options);
-        if (req.dict.dict_len < 0) {
+        ret = dict_set_int16 (options, "clnt-lk-version",
+                              client_get_lk_ver (conf));
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set clnt-lk-version(%"PRIu32") in handshake msg",
+                        client_get_lk_ver (conf));
+        }
+
+        ret = dict_serialized_length (options);
+        if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to get serialized length of dict");
                 ret = -1;
                 goto fail;
         }
+        req.dict.dict_len = ret;
         req.dict.dict_val = GF_CALLOC (1, req.dict.dict_len,
                                        gf_client_mt_clnt_req_buf_t);
         ret = dict_serialize (options, req.dict.dict_val);
@@ -1069,8 +1626,7 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
                                      (xdrproc_t)xdr_gf_setvolume_req);
 
 fail:
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
@@ -1093,9 +1649,9 @@ select_server_supported_programs (xlator_t *this, gf_prog_detail *prog)
 
         while (trav) {
                 /* Select 'programs' */
-                if ((clnt3_1_fop_prog.prognum == trav->prognum) &&
-                    (clnt3_1_fop_prog.progver == trav->progver)) {
-                        conf->fops = &clnt3_1_fop_prog;
+                if ((clnt3_3_fop_prog.prognum == trav->prognum) &&
+                    (clnt3_3_fop_prog.progver == trav->progver)) {
+                        conf->fops = &clnt3_3_fop_prog;
                         gf_log (this->name, GF_LOG_INFO,
                                 "Using Program %s, Num (%"PRId64"), "
                                 "Version (%"PRId64")",
@@ -1180,16 +1736,21 @@ client_query_portmap_cbk (struct rpc_req *req, struct iovec *iov, int count, voi
                 ret = -1;
                 gf_log (this->name, ((!conf->portmap_err_logged) ?
                                      GF_LOG_ERROR : GF_LOG_DEBUG),
-                        "failed to get the port number for remote subvolume");
+                        "failed to get the port number for remote subvolume. "
+                        "Please run 'gluster volume status' on server to see "
+                        "if brick process is running.");
                 conf->portmap_err_logged = 1;
                 goto out;
         }
 
         conf->portmap_err_logged = 0;
+        conf->disconnect_err_logged = 0;
 
         config.remote_port = rsp.port;
         rpc_clnt_reconfig (conf->rpc, &config);
+
         conf->skip_notify = 1;
+	conf->quick_reconnect = 1;
 
 out:
         if (frame)
@@ -1199,7 +1760,6 @@ out:
                 /* Need this to connect the same transport on different port */
                 /* ie, glusterd to glusterfsd */
                 rpc_transport_disconnect (conf->rpc->conn.trans);
-                rpc_clnt_reconnect (conf->rpc->conn.trans);
         }
 
         return ret;
@@ -1366,6 +1926,7 @@ char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
         [GF_HNDSK_SETVOLUME]    = "SETVOLUME",
         [GF_HNDSK_GETSPEC]      = "GETSPEC",
         [GF_HNDSK_PING]         = "PING",
+        [GF_HNDSK_SET_LK_VER]   = "SET_LK_VER"
 };
 
 rpc_clnt_prog_t clnt_handshake_prog = {
diff --git a/xlators/protocol/client/src/client-helpers.c b/xlators/protocol/client/src/client-helpers.c
index 5759f3062..5d9f00fdc 100644
--- a/xlators/protocol/client/src/client-helpers.c
+++ b/xlators/protocol/client/src/client-helpers.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -25,6 +16,31 @@
 #include "client.h"
 #include "fd.h"
 
+int
+client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t try_lock)
+{
+        int  ret = 1;
+
+        if (!lk_ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        if (try_lock) {
+                ret = TRY_LOCK (&lk_ctx->lock);
+                if (ret != 0) {
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                LOCK (&lk_ctx->lock);
+        }
+
+        ret = list_empty (&lk_ctx->lk_list);
+        UNLOCK (&lk_ctx->lock);
+out:
+        return ret;
+}
 
 clnt_fd_ctx_t *
 this_fd_del_ctx (fd_t *file, xlator_t *this)
@@ -106,6 +122,7 @@ client_local_wipe (clnt_local_t *local)
 {
         if (local) {
                 loc_wipe (&local->loc);
+                loc_wipe (&local->loc2);
 
                 if (local->fd) {
                         fd_unref (local->fd);
@@ -115,7 +132,9 @@ client_local_wipe (clnt_local_t *local)
                         iobref_unref (local->iobref);
                 }
 
-                 GF_FREE (local);
+                GF_FREE (local->name);
+
+                mem_put (local);
         }
 
         return 0;
@@ -190,6 +209,8 @@ unserialize_rsp_direntp (xlator_t *this, fd_t *fd,
                         if (!buf)
                                 goto out;
 
+                        entry->dict = dict_new ();
+
                         ret = dict_unserialize (buf, trav->dict.dict_len,
                                                 &entry->dict);
                         if (ret < 0) {
@@ -227,6 +248,7 @@ clnt_readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp)
         while (trav) {
                 trav = trav->nextentry;
                 /* on client, the rpc lib allocates this */
+                free (prev->dict.dict_val);
                 free (prev->name);
                 free (prev);
                 prev = trav;
@@ -253,3 +275,75 @@ clnt_readdir_rsp_cleanup (gfs3_readdir_rsp *rsp)
 
         return 0;
 }
+
+int
+client_get_remote_fd (xlator_t *this, fd_t *fd, int flags, int64_t *remote_fd)
+{
+        clnt_fd_ctx_t *fdctx    = NULL;
+        clnt_conf_t   *conf     = NULL;
+
+        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+        GF_VALIDATE_OR_GOTO (this->name, remote_fd, out);
+
+        conf = this->private;
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx = this_fd_get_ctx (fd, this);
+                if (!fdctx)
+                        *remote_fd = GF_ANON_FD_NO;
+                else if (__is_fd_reopen_in_progress (fdctx))
+                        *remote_fd = -1;
+                else
+                        *remote_fd = fdctx->remote_fd;
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        if ((flags & FALLBACK_TO_ANON_FD) && (*remote_fd == -1))
+                *remote_fd = GF_ANON_FD_NO;
+
+        return 0;
+out:
+        return -1;
+}
+
+gf_boolean_t
+client_is_reopen_needed (fd_t *fd, xlator_t *this, int64_t remote_fd)
+{
+        clnt_fd_ctx_t   *fdctx = NULL;
+
+        fdctx = this_fd_get_ctx (fd, this);
+        if (fdctx && (fdctx->remote_fd == -1) &&
+            (remote_fd == GF_ANON_FD_NO))
+                return _gf_true;
+        return _gf_false;
+}
+
+int
+client_fd_fop_prepare_local (call_frame_t *frame, fd_t *fd, int64_t remote_fd)
+{
+        xlator_t     *this  = NULL;
+        clnt_conf_t  *conf  = NULL;
+        clnt_local_t *local = NULL;
+        int          ret    = 0;
+
+        this = frame->this;
+        conf = this->private;
+
+        if (!frame || !fd) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        frame->local = mem_get0 (this->local_pool);
+        if (frame->local == NULL) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        local = frame->local;
+        local->fd = fd_ref (fd);
+        local->attempt_reopen = client_is_reopen_needed (fd, this, remote_fd);
+        return 0;
+out:
+        return ret;
+}
diff --git a/xlators/protocol/client/src/client-lk.c b/xlators/protocol/client/src/client-lk.c
index 1bdde87df..1fd8f0d50 100644
--- a/xlators/protocol/client/src/client-lk.c
+++ b/xlators/protocol/client/src/client-lk.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "common-utils.h"
@@ -25,9 +16,6 @@
 static void
 __insert_and_merge (clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock);
 
-static int
-client_send_recovery_lock (call_frame_t *frame, xlator_t *this,
-                           client_posix_lock_t *lock);
 static void
 __dump_client_lock (client_posix_lock_t *lock)
 {
@@ -375,9 +363,8 @@ delete_granted_locks_owner (fd_t *fd, gf_lkowner_t *owner)
         this = THIS;
         fdctx = this_fd_get_ctx (fd, this);
         if (!fdctx) {
-                if (!fd_is_anonymous (fd))
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "fdctx not valid");
+                gf_log (this->name, GF_LOG_WARNING,
+                        "fdctx not valid");
                 ret = -1;
                 goto out;
         }
@@ -440,21 +427,6 @@ delete_granted_locks_fd (clnt_fd_ctx_t *fdctx)
         return  ret;
 }
 
-static void
-client_mark_bad_fd (fd_t *fd, clnt_fd_ctx_t *fdctx)
-{
-        xlator_t *this = NULL;
-
-        this = THIS;
-        if (fdctx)
-                fdctx->remote_fd = -1;
-
-        gf_log (this->name, GF_LOG_WARNING,
-                "marking the file descriptor (%p) bad", fd);
-
-        this_fd_set_ctx (fd, this, NULL, fdctx);
-}
-
 int32_t
 client_cmd_to_gf_cmd (int32_t cmd, int32_t *gf_cmd)
 {
@@ -563,347 +535,6 @@ out:
 
 }
 
-static int
-construct_reserve_unlock (struct gf_flock *lock, call_frame_t *frame,
-                          client_posix_lock_t *client_lock)
-{
-        GF_ASSERT (lock);
-        GF_ASSERT (frame);
-
-        lock->l_type = F_UNLCK;
-        lock->l_start = 0;
-        lock->l_whence = SEEK_SET;
-        lock->l_len = 0; /* Whole file */
-        lock->l_pid = (uint64_t)(unsigned long)frame->root;
-        lock->l_owner = client_lock->owner;
-
-        frame->root->lk_owner = client_lock->owner;
-
-        return 0;
-}
-
-static int
-construct_reserve_lock (client_posix_lock_t *client_lock, call_frame_t *frame,
-                        struct gf_flock *lock)
-{
-        GF_ASSERT (client_lock);
-
-        memcpy (lock, &(client_lock->user_flock), sizeof (struct gf_flock));
-
-        frame->root->lk_owner = client_lock->owner;
-
-        return 0;
-}
-
-uint64_t
-decrement_reopen_fd_count (xlator_t *this, clnt_conf_t *conf)
-{
-        uint64_t fd_count = 0;
-
-        LOCK (&conf->rec_lock);
-        {
-                fd_count = --(conf->reopen_fd_count);
-        }
-        UNLOCK (&conf->rec_lock);
-
-        if (fd_count == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "last fd open'd/lock-self-heal'd - notifying CHILD-UP");
-                client_notify_parents_child_up (this);
-        }
-
-        return fd_count;
-}
-
-int32_t
-client_remove_reserve_lock_cbk (call_frame_t *frame,
-                                void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret,
-                                int32_t op_errno,
-                                struct gf_flock *lock)
-{
-        clnt_local_t *local = NULL;
-        clnt_conf_t  *conf  = NULL;
-
-        uint64_t fd_count = 0;
-
-        local = frame->local;
-        conf  = this->private;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "removing reserver lock on fd failed: %s",
-                        strerror(op_errno));
-                goto cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Removing reserve lock was successful.");
-
-cleanup:
-        frame->local = NULL;
-
-        client_mark_bad_fd (local->client_lock->fd, local->fdctx);
-
-        destroy_client_lock (local->client_lock);
-        client_local_wipe (local);
-        STACK_DESTROY (frame->root);
-
-        fd_count = decrement_reopen_fd_count (this, conf);
-        gf_log (this->name, GF_LOG_TRACE,
-                "Need to attempt lock recovery on %lld open fds",
-                (unsigned long long) fd_count);
-	return 0;
-}
-
-static void
-client_remove_reserve_lock (xlator_t *this, call_frame_t *frame,
-                            client_posix_lock_t *lock)
-{
-        struct gf_flock unlock;
-
-        construct_reserve_unlock (&unlock, frame, lock);
-
-        STACK_WIND (frame, client_remove_reserve_lock_cbk,
-                    this, this->fops->lk,
-                    lock->fd, F_RESLK_UNLCK, &unlock);
-}
-
-static client_posix_lock_t *
-get_next_recovery_lock (xlator_t *this, clnt_local_t *local)
-{
-        client_posix_lock_t *lock = NULL;
-
-        pthread_mutex_lock (&local->mutex);
-        {
-                if (list_empty (&local->lock_list)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "lock-list empty");
-                        goto unlock;
-                }
-
-                lock = list_entry ((local->lock_list).next, typeof (*lock), list);
-                list_del_init (&lock->list);
-        }
-unlock:
-        pthread_mutex_unlock (&local->mutex);
-
-        return lock;
-
-}
-
-int32_t
-client_reserve_lock_cbk (call_frame_t *frame,
-                         void *cookie,
-                         xlator_t *this,
-                         int32_t op_ret,
-                         int32_t op_errno,
-                         struct gf_flock *lock)
-{
-
-        clnt_local_t *local = NULL;
-        clnt_conf_t  *conf  = NULL;
-
-        uint64_t fd_count = 0;
-
-        local = frame->local;
-        conf  = this->private;
-
-        /* Got the reserve lock. Check if lock is grantable and proceed
-           with the real lk call */
-
-        if (op_ret >= 0) {
-                /* Lock is grantable if flock reflects a successful getlk() call*/
-                if (lock->l_type == F_UNLCK && lock->l_pid) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Got the reservelk, but the lock is not grantable. ");
-                        client_remove_reserve_lock (this, frame, local->client_lock);
-                        goto out;
-                }
-
-                gf_log (this->name, GF_LOG_DEBUG, "reserve lock succeeded");
-                client_send_recovery_lock (frame, this, local->client_lock);
-                goto out;
-        }
-
-        /* Somebody else has a reserve lk. Lock conflict detected.
-           Mark fd as bad */
-
-        gf_log (this->name, GF_LOG_WARNING,
-                "reservelk OP failed. aborting lock recovery");
-
-        client_mark_bad_fd (local->client_lock->fd,
-                            local->fdctx);
-        destroy_client_lock (local->client_lock);
-        frame->local = NULL;
-        client_local_wipe (local);
-        STACK_DESTROY (frame->root);
-
-        fd_count = decrement_reopen_fd_count (this, conf);
-        gf_log (this->name, GF_LOG_DEBUG,
-                "need to attempt lock recovery on %"PRIu64" open fds",
-                fd_count);
-
-out:
-	return 0;
-}
-
-int32_t
-client_recovery_lock_cbk (call_frame_t *frame,
-                          void *cookie,
-                          xlator_t *this,
-                          int32_t op_ret,
-                          int32_t op_errno,
-                          struct gf_flock *lock)
-{
-        clnt_local_t *local = NULL;
-        clnt_fd_ctx_t *fdctx = NULL;
-        clnt_conf_t   *conf  = NULL;
-        client_posix_lock_t *next_lock = NULL;
-
-        struct gf_flock reserve_flock;
-        uint64_t fd_count = 0;
-
-        local = frame->local;
-        conf  = this->private;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "lock recovery failed: %s",
-                        strerror(op_errno));
-
-                client_mark_bad_fd (local->client_lock->fd,
-                                    local->fdctx);
-                goto cleanup;
-
-                /* Lock recovered. Continue with reserve lock for next lock */
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "lock recovered successfully - continuing with next lock.");
-
-                next_lock = get_next_recovery_lock (this, local);
-                if (!next_lock) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "all locks recovered on fd");
-                        goto cleanup;
-                }
-
-                construct_reserve_lock (next_lock, frame, &reserve_flock);
-                local->fdctx       = fdctx;
-                local->client_lock = next_lock;
-
-                STACK_WIND (frame, client_reserve_lock_cbk,
-                            this, this->fops->lk,
-                            next_lock->fd, F_RESLK_LCK, &reserve_flock);
-                goto out;
-
-        }
-
-cleanup:
-
-        frame->local = NULL;
-        client_local_wipe (local);
-
-        if (local->client_lock)
-                destroy_client_lock (local->client_lock);
-
-        STACK_DESTROY (frame->root);
-
-        fd_count = decrement_reopen_fd_count (this, conf);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "need to attempt lock recovery on %"PRIu64" open fds",
-                fd_count);
-
-out:
-	return 0;
-}
-
-static int
-client_send_recovery_lock (call_frame_t *frame, xlator_t *this,
-                           client_posix_lock_t *lock)
-{
-        frame->root->lk_owner = lock->owner;
-
-        /* Send all locks as F_SETLK to prevent the frame
-           from blocking if there is a conflict */
-
-        STACK_WIND (frame, client_recovery_lock_cbk,
-                    this, this->fops->lk,
-                    lock->fd, F_SETLK,
-                    &(lock->user_flock));
-
-        return 0;
-}
-
-static int
-client_lockrec_init (clnt_fd_ctx_t *fdctx, clnt_local_t *local)
-{
-
-        INIT_LIST_HEAD (&local->lock_list);
-        pthread_mutex_init (&local->mutex, NULL);
-
-        pthread_mutex_lock (&fdctx->mutex);
-        {
-                list_splice_init (&fdctx->lock_list, &local->lock_list);
-        }
-        pthread_mutex_unlock (&fdctx->mutex);
-
-        return 0;
-}
-
-
-int
-client_attempt_lock_recovery (xlator_t *this, clnt_fd_ctx_t *fdctx)
-{
-        call_frame_t        *frame = NULL;
-        clnt_local_t        *local = NULL;
-        client_posix_lock_t *lock  = NULL;
-
-        struct gf_flock reserve_flock;
-        int ret = 0;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
-        if (!local) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        client_lockrec_init (fdctx, local);
-
-        lock = get_next_recovery_lock (this, local);
-        if (!lock) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no locks found on fd");
-                ret = -1;
-                goto out;
-        }
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "creating of frame failed, lock recovery failed");
-                ret = -1;
-                goto out;
-        }
-
-        construct_reserve_lock (lock, frame, &reserve_flock);
-
-        frame->local       = local;
-        local->fdctx       = fdctx;
-        local->client_lock = lock;
-
-        STACK_WIND (frame, client_reserve_lock_cbk,
-                    this, this->fops->lk,
-                    lock->fd, F_RESLK_LCK, &reserve_flock);
-
-out:
-        return ret;
-
-
-}
-
 int32_t
 client_dump_locks (char *name, inode_t *inode,
                    dict_t *dict)
diff --git a/xlators/protocol/client/src/client-mem-types.h b/xlators/protocol/client/src/client-mem-types.h
index c2aa690b1..f6573da2d 100644
--- a/xlators/protocol/client/src/client-mem-types.h
+++ b/xlators/protocol/client/src/client-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -25,10 +16,10 @@
 
 enum gf_client_mem_types_ {
         gf_client_mt_clnt_conf_t = gf_common_mt_end + 1,
-        gf_client_mt_clnt_local_t,
         gf_client_mt_clnt_req_buf_t,
         gf_client_mt_clnt_fdctx_t,
         gf_client_mt_clnt_lock_t,
+        gf_client_mt_clnt_fd_lk_local_t,
         gf_client_mt_end,
 };
 #endif /* __CLIENT_MEM_TYPES_H__ */
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client-rpc-fops.c
index e57373419..6355450c3 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client-rpc-fops.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -29,22 +20,25 @@
 
 int32_t client3_getspec (call_frame_t *frame, xlator_t *this, void *data);
 void client_start_ping (void *data);
-rpc_clnt_prog_t clnt3_1_fop_prog;
+rpc_clnt_prog_t clnt3_3_fop_prog;
+
 
 int
 client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
-                           rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbk,
+                           rpc_clnt_prog_t *prog, int procnum,
+                           fop_cbk_fn_t cbkfn,
                            struct iovec  *payload, int payloadcnt,
                            struct iobref *iobref, xdrproc_t xdrproc)
 {
-        int            ret        = 0;
-        clnt_conf_t   *conf       = NULL;
-        struct iovec   iov        = {0, };
-        struct iobuf  *iobuf      = NULL;
-        int            count      = 0;
-        int            start_ping = 0;
-        struct iobref *new_iobref = NULL;
-        ssize_t        xdr_size   = 0;
+        int             ret        = 0;
+        clnt_conf_t    *conf       = NULL;
+        struct iovec    iov        = {0, };
+        struct iobuf   *iobuf      = NULL;
+        int             count      = 0;
+        int             start_ping = 0;
+        struct iobref  *new_iobref = NULL;
+        ssize_t         xdr_size   = 0;
+        struct rpc_req  rpcreq     = {0, };
 
         start_ping = 0;
 
@@ -54,12 +48,12 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
                 xdr_size = xdr_sizeof (xdrproc, req);
                 iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
                 if (!iobuf) {
-                        goto out;
+                        goto unwind;
                 };
 
                 new_iobref = iobref_new ();
                 if (!new_iobref) {
-                        goto out;
+                        goto unwind;
                 }
 
                 if (iobref != NULL) {
@@ -75,7 +69,7 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
                 if (ret != 0) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "cannot add iobuf into iobref");
-                        goto out;
+                        goto unwind;
                 }
 
                 iov.iov_base = iobuf->ptr;
@@ -86,7 +80,7 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
                 if (ret == -1) {
                         gf_log_callingfn ("", GF_LOG_WARNING,
                                           "XDR function failed");
-                        goto out;
+                        goto unwind;
                 }
 
                 iov.iov_len = ret;
@@ -94,7 +88,7 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
         }
 
         /* Send the msg */
-        ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbk, &iov, count,
+        ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbkfn, &iov, count,
                                payload, payloadcnt, new_iobref, frame, NULL, 0,
                                NULL, 0, NULL);
         if (ret < 0) {
@@ -114,12 +108,23 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
         if (start_ping)
                 client_start_ping ((void *) this);
 
-out:
-        if (new_iobref != NULL) {
+        if (new_iobref)
                 iobref_unref (new_iobref);
-        }
 
-        iobuf_unref (iobuf);
+        if (iobuf)
+                iobuf_unref (iobuf);
+
+        return ret;
+
+unwind:
+        rpcreq.rpc_status = -1;
+        cbkfn (&rpcreq, NULL, 0, frame);
+
+        if (new_iobref)
+                iobref_unref (new_iobref);
+
+        if (iobuf)
+                iobuf_unref (iobuf);
 
         return ret;
 }
@@ -127,7 +132,7 @@ out:
 /* CBK */
 
 int
-client3_1_symlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_symlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t     *frame      = NULL;
@@ -139,13 +144,13 @@ client3_1_symlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
         clnt_local_t     *local      = NULL;
         inode_t          *inode      = NULL;
         xlator_t         *this       = NULL;
+        dict_t           *xdata      = NULL;
 
         this = THIS;
 
         frame = myframe;
 
         local = frame->local;
-        frame->local = NULL;
         inode = local->loc.inode;
 
         if (-1 == req->rpc_status) {
@@ -168,26 +173,36 @@ client3_1_symlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
+                /* no need to print the gfid, because it will be null, since
+                 * symlink operation failed.
+                 */
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                        "remote operation failed: %s. Path: (%s to %s)",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, local->loc2.path);
         }
-        STACK_UNWIND_STRICT (symlink, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (symlink, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), inode, &stbuf,
-                             &preparent, &postparent);
+                             &preparent, &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
 
-         client_local_wipe (local);
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_mknod_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_mknod_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         call_frame_t     *frame      = NULL;
@@ -199,13 +214,14 @@ client3_1_mknod_cbk (struct rpc_req *req, struct iovec *iov, int count,
         clnt_local_t     *local      = NULL;
         inode_t          *inode      = NULL;
         xlator_t         *this       = NULL;
+        dict_t           *xdata      = NULL;
 
         this = THIS;
 
         frame = myframe;
 
         local = frame->local;
-        frame->local = NULL;
+
         inode = local->loc.inode;
 
         if (-1 == req->rpc_status) {
@@ -228,25 +244,32 @@ client3_1_mknod_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s. Path: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path);
         }
-        STACK_UNWIND_STRICT (mknod, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (mknod, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), inode,
-                             &stbuf, &preparent, &postparent);
+                             &stbuf, &preparent, &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
 
-        client_local_wipe (local);
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_mkdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_mkdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         call_frame_t     *frame      = NULL;
@@ -258,6 +281,7 @@ client3_1_mkdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
         clnt_local_t     *local      = NULL;
         inode_t          *inode      = NULL;
         xlator_t         *this       = NULL;
+        dict_t           *xdata      = NULL;
 
         this = THIS;
 
@@ -286,42 +310,122 @@ client3_1_mkdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s. Path: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path);
         }
-        STACK_UNWIND_STRICT (mkdir, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (mkdir, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), inode,
-                             &stbuf, &preparent, &postparent);
+                             &stbuf, &preparent, &postparent, xdata);
 
-        client_local_wipe (local);
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_open_cbk (struct rpc_req *req, struct iovec *iov, int count,
+_copy_gfid_from_inode_holders (uuid_t gfid, loc_t *loc, fd_t *fd)
+{
+        int     ret = 0;
+
+        if (fd && fd->inode && !uuid_is_null (fd->inode->gfid)) {
+                uuid_copy (gfid, fd->inode->gfid);
+                goto out;
+        }
+
+        if (!loc) {
+                GF_ASSERT (0);
+                ret = -1;
+                goto out;
+        }
+
+        if (loc->inode && !uuid_is_null (loc->inode->gfid)) {
+                uuid_copy (gfid, loc->inode->gfid);
+        } else if (!uuid_is_null (loc->gfid)) {
+                uuid_copy (gfid, loc->gfid);
+        } else {
+                GF_ASSERT (0);
+                ret = -1;
+        }
+out:
+        return ret;
+}
+
+int
+client_add_fd_to_saved_fds (xlator_t *this, fd_t *fd, loc_t *loc, int32_t flags,
+                            int64_t remote_fd, int is_dir)
+{
+        int             ret = 0;
+        uuid_t          gfid = {0};
+        clnt_conf_t     *conf       = NULL;
+        clnt_fd_ctx_t   *fdctx      = NULL;
+
+        conf  = this->private;
+        ret = _copy_gfid_from_inode_holders (gfid, loc, fd);
+        if (ret) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        fdctx = GF_CALLOC (1, sizeof (*fdctx),
+                           gf_client_mt_clnt_fdctx_t);
+        if (!fdctx) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        uuid_copy (fdctx->gfid, gfid);
+        fdctx->is_dir        = is_dir;
+        fdctx->remote_fd     = remote_fd;
+        fdctx->flags         = flags;
+        fdctx->lk_ctx        = fd_lk_ctx_ref (fd->lk_ctx);
+        fdctx->lk_heal_state = GF_LK_HEAL_DONE;
+        fdctx->reopen_done   = client_default_reopen_done;
+
+        INIT_LIST_HEAD (&fdctx->sfd_pos);
+        INIT_LIST_HEAD (&fdctx->lock_list);
+
+        this_fd_set_ctx (fd, this, loc, fdctx);
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
+        }
+        pthread_mutex_unlock (&conf->lock);
+out:
+        return ret;
+}
+
+int
+client3_3_open_cbk (struct rpc_req *req, struct iovec *iov, int count,
                     void *myframe)
 {
         clnt_local_t  *local = NULL;
         clnt_conf_t   *conf  = NULL;
-        clnt_fd_ctx_t *fdctx = NULL;
         call_frame_t  *frame = NULL;
         fd_t          *fd    = NULL;
         int            ret   = 0;
         gfs3_open_rsp  rsp   = {0,};
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
 
-        frame->local = NULL;
         conf  = frame->this->private;
         fd    = local->fd;
 
@@ -339,57 +443,50 @@ client3_1_open_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
         if (-1 != rsp.op_ret) {
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_clnt_fdctx_t);
-                if (!fdctx) {
+                ret = client_add_fd_to_saved_fds (frame->this, fd, &local->loc,
+                                                  local->flags, rsp.fd, 0);
+                if (ret) {
                         rsp.op_ret = -1;
-                        rsp.op_errno = ENOMEM;
+                        rsp.op_errno = -ret;
                         goto out;
                 }
-
-                fdctx->remote_fd = rsp.fd;
-                fdctx->inode     = inode_ref (fd->inode);
-                fdctx->flags     = local->flags;
-                fdctx->wbflags   = local->wbflags;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-                INIT_LIST_HEAD (&fdctx->lock_list);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->lock);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->lock);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                        "remote operation failed: %s. Path: %s (%s)",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, loc_gfid_utoa (&local->loc));
         }
-        STACK_UNWIND_STRICT (open, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), fd);
 
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (open, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), fd, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_stat_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_stat_cbk (struct rpc_req *req, struct iovec *iov, int count,
                     void *myframe)
 {
         gfs3_stat_rsp rsp = {0,};
         call_frame_t   *frame = NULL;
         struct iatt  iatt = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -412,26 +509,38 @@ client3_1_stat_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.stat, &iatt);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (stat, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &iatt);
+
+        CLIENT_STACK_UNWIND (stat, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &iatt, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_readlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_readlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         gfs3_readlink_rsp rsp = {0,};
         call_frame_t   *frame = NULL;
         struct iatt  iatt = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -454,24 +563,35 @@ client3_1_readlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.buf, &iatt);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+                gf_log (this->name, (gf_error_to_errno(rsp.op_errno) == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_WARNING, "remote operation failed:"
+                        " %s", strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (readlink, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), rsp.path, &iatt);
+
+        CLIENT_STACK_UNWIND (readlink, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), rsp.path,
+                             &iatt, xdata);
 
         /* This is allocated by the libc while decoding RPC msg */
         /* Hence no 'GF_FREE', but just 'free' */
-        if (rsp.path)
-                free (rsp.path);
+        free (rsp.path);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_unlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_unlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         call_frame_t    *frame      = NULL;
@@ -479,7 +599,9 @@ client3_1_unlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt      preparent  = {0,};
         struct iatt      postparent = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -503,20 +625,33 @@ client3_1_unlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
+                gf_log (this->name,
+                        ((gf_error_to_errno (rsp.op_errno) == ENOENT)
+                        ? GF_LOG_DEBUG : GF_LOG_WARNING),
+                        "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (unlink, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (unlink, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &preparent,
-                             &postparent);
+                             &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_rmdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_rmdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         gfs3_rmdir_rsp rsp = {0,};
@@ -524,7 +659,9 @@ client3_1_rmdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt  preparent  = {0,};
         struct iatt  postparent = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -548,21 +685,30 @@ client3_1_rmdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (rmdir, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (rmdir, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &preparent,
-                             &postparent);
+                             &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_truncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_truncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         gfs3_truncate_rsp rsp = {0,};
@@ -570,7 +716,9 @@ client3_1_truncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt  prestat  = {0,};
         struct iatt  poststat = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -594,28 +742,39 @@ client3_1_truncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.poststat, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (truncate, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (truncate, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_statfs_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_statfs_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         gfs3_statfs_rsp rsp = {0,};
         call_frame_t   *frame = NULL;
         struct statvfs  statfs = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -638,20 +797,29 @@ client3_1_statfs_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_statfs_to_statfs (&rsp.statfs, &statfs);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (statfs, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &statfs);
+        CLIENT_STACK_UNWIND (statfs, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &statfs, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_writev_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_writev_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         gfs3_write_rsp rsp = {0,};
@@ -659,11 +827,15 @@ client3_1_writev_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt  prestat  = {0,};
         struct iatt  poststat = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+        clnt_local_t    *local = NULL;
+
 
         this = THIS;
 
         frame = myframe;
+        local = frame->local;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -684,25 +856,39 @@ client3_1_writev_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.poststat, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
+        } else if (rsp.op_ret >= 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
-        STACK_UNWIND_STRICT (writev, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (writev, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_flush_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_flush_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         call_frame_t    *frame      = NULL;
         clnt_local_t  *local      = NULL;
-        xlator_t        *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
 
@@ -723,7 +909,7 @@ client3_1_flush_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
-        if (rsp.op_ret >= 0) {
+        if (rsp.op_ret >= 0 && !fd_is_anonymous (local->fd)) {
                 /* Delete all saved locks of the owner issuing flush */
                 ret = delete_granted_locks_owner (local->fd, &local->owner);
                 gf_log (this->name, GF_LOG_TRACE,
@@ -731,23 +917,29 @@ client3_1_flush_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         lkowner_utoa (&local->owner), ret);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (flush, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (flush, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
 
-        client_local_wipe (local);
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fsync_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fsync_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         gfs3_fsync_rsp rsp = {0,};
@@ -755,7 +947,9 @@ client3_1_fsync_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt  prestat  = {0,};
         struct iatt  poststat = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -780,26 +974,37 @@ client3_1_fsync_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.poststat, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fsync, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (fsync, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_setxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_setxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t         *this       = NULL;
+        call_frame_t  *frame    = NULL;
+        gf_common_rsp  rsp      = {0,};
+        int            ret      = 0;
+        xlator_t      *this     = NULL;
+        dict_t        *xdata    = NULL;
+        int            op_errno = EINVAL;
 
         this = THIS;
 
@@ -819,19 +1024,30 @@ client3_1_setxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
+        op_errno = gf_error_to_errno (rsp.op_errno);
         if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+                gf_log (this->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_WARNING),
+                        "remote operation failed: %s",
+                        strerror (op_errno));
         }
-        STACK_UNWIND_STRICT (setxattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (setxattr, frame, rsp.op_ret, op_errno, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_getxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_getxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t      *frame    = NULL;
@@ -840,13 +1056,14 @@ client3_1_getxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
         gfs3_getxattr_rsp  rsp      = {0,};
         int                ret      = 0;
         clnt_local_t    *local    = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret = -1;
@@ -870,32 +1087,40 @@ client3_1_getxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                                               op_errno, out);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                gf_log (this->name, (((op_errno == ENOTSUP) ||
+                                      (op_errno == ENODATA) ||
+                                      (op_errno == ENOENT)) ?
+                                     GF_LOG_DEBUG : GF_LOG_WARNING),
+                        "remote operation failed: %s. Path: %s (%s). Key: %s",
                         strerror (op_errno),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, loc_gfid_utoa (&local->loc),
+                        (local->name) ? local->name : "(null)");
         }
-        STACK_UNWIND_STRICT (getxattr, frame, rsp.op_ret, op_errno, dict);
 
-        if (rsp.dict.dict_val) {
-                /* don't use GF_FREE, this memory was allocated by libc
-                 */
-                free (rsp.dict.dict_val);
-                rsp.dict.dict_val = NULL;
-        }
+        CLIENT_STACK_UNWIND (getxattr, frame, rsp.op_ret, op_errno, dict, xdata);
+
+        /* don't use GF_FREE, this memory was allocated by libc */
+        free (rsp.dict.dict_val);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         if (dict)
                 dict_unref (dict);
 
-        client_local_wipe (local);
-
         return 0;
 }
 
 int
-client3_1_fgetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fgetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                          void *myframe)
 {
         call_frame_t       *frame    = NULL;
@@ -904,13 +1129,14 @@ client3_1_fgetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
         int                 ret      = 0;
         int                 op_errno = EINVAL;
         clnt_local_t     *local    = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret = -1;
@@ -932,36 +1158,42 @@ client3_1_fgetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                                               (rsp.dict.dict_len), rsp.op_ret,
                                               op_errno, out);
         }
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_WARNING),
                         "remote operation failed: %s",
                         strerror (op_errno));
         }
-        STACK_UNWIND_STRICT (fgetxattr, frame, rsp.op_ret, op_errno, dict);
-        if (rsp.dict.dict_val) {
-                /* don't use GF_FREE, this memory was allocated by libc
-                 */
-                free (rsp.dict.dict_val);
-                rsp.dict.dict_val = NULL;
-        }
+
+        CLIENT_STACK_UNWIND (fgetxattr, frame, rsp.op_ret, op_errno, dict, xdata);
+
+        free (rsp.dict.dict_val);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         if (dict)
                 dict_unref (dict);
 
-        client_local_wipe (local);
-
         return 0;
 }
 
 int
-client3_1_removexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_removexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                            void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
@@ -981,25 +1213,37 @@ client3_1_removexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (removexattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+
+        CLIENT_STACK_UNWIND (removexattr, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fremovexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fremovexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                             void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1019,25 +1263,36 @@ client3_1_fremovexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fremovexattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (fremovexattr, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fsyncdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fsyncdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1056,25 +1311,36 @@ client3_1_fsyncdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fsyncdir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (fsyncdir, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_access_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_access_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1093,20 +1359,29 @@ client3_1_access_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (access, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (access, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_ftruncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_ftruncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
                          void *myframe)
 {
         gfs3_ftruncate_rsp rsp = {0,};
@@ -1114,7 +1389,9 @@ client3_1_ftruncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt  prestat  = {0,};
         struct iatt  poststat = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1138,27 +1415,38 @@ client3_1_ftruncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.poststat, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (ftruncate, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (ftruncate, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fstat_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fstat_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         gfs3_fstat_rsp rsp = {0,};
         call_frame_t   *frame = NULL;
         struct iatt  stat  = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1181,26 +1469,37 @@ client3_1_fstat_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.stat, &stat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fstat, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &stat);
+        CLIENT_STACK_UNWIND (fstat, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &stat,  xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_inodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_inodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1219,30 +1518,42 @@ client3_1_inodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if ((rsp.op_ret == -1) &&
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (inodelk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (inodelk, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_finodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_finodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t         *this       = NULL;
+        call_frame_t  *frame = NULL;
+        gf_common_rsp rsp    = {0,};
+        int           ret    = 0;
+        xlator_t      *this  = NULL;
+        dict_t        *xdata = NULL;
+        clnt_local_t  *local = NULL;
 
-        this = THIS;
 
         frame = myframe;
+        this = frame->this;
+        local = frame->local;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -1257,26 +1568,40 @@ client3_1_finodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if ((rsp.op_ret == -1) &&
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
+        } else if (rsp.op_ret == 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
-        STACK_UNWIND_STRICT (finodelk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (finodelk, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_entrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_entrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1295,27 +1620,38 @@ client3_1_entrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
-out:
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
 
+out:
         if ((rsp.op_ret == -1) &&
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (entrylk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+
+        CLIENT_STACK_UNWIND (entrylk, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fentrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fentrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1334,20 +1670,30 @@ client3_1_fentrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if ((rsp.op_ret == -1) &&
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fentrylk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+
+        CLIENT_STACK_UNWIND (fentrylk, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_xattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_xattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t     *frame    = NULL;
@@ -1356,13 +1702,14 @@ client3_1_xattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
         int               ret      = 0;
         int               op_errno = EINVAL;
         clnt_local_t   *local    = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret = -1;
@@ -1385,49 +1732,51 @@ client3_1_xattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
                                               op_errno, out);
         }
 
-out:
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      op_errno, out);
 
+out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                        "remote operation failed: %s. Path: %s (%s)",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, loc_gfid_utoa (&local->loc));
         }
-        STACK_UNWIND_STRICT (xattrop, frame, rsp.op_ret,
-                             gf_error_to_errno (op_errno), dict);
 
-        if (rsp.dict.dict_val) {
-                /* don't use GF_FREE, this memory was allocated by libc
-                 */
-                free (rsp.dict.dict_val);
-                rsp.dict.dict_val = NULL;
-        }
+        CLIENT_STACK_UNWIND (xattrop, frame, rsp.op_ret,
+                             gf_error_to_errno (op_errno), dict, xdata);
+
+        free (rsp.dict.dict_val);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         if (dict)
                 dict_unref (dict);
 
-        client_local_wipe (local);
-
         return 0;
 }
 
 int
-client3_1_fxattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fxattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t      *frame    = NULL;
         dict_t            *dict     = NULL;
+        dict_t            *xdata    = NULL;
         gfs3_fxattrop_rsp  rsp      = {0,};
         int                ret      = 0;
         int                op_errno = 0;
         clnt_local_t    *local    = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -1450,38 +1799,99 @@ client3_1_fxattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
                                               op_errno, out);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata,
+                                      (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), rsp.op_ret,
+                                      op_errno, out);
 out:
 
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+                        strerror (gf_error_to_errno (op_errno)));
+        } else if (rsp.op_ret == 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
-        STACK_UNWIND_STRICT (fxattrop, frame, rsp.op_ret,
-                             gf_error_to_errno (op_errno), dict);
+        CLIENT_STACK_UNWIND (fxattrop, frame, rsp.op_ret,
+                             gf_error_to_errno (op_errno), dict, xdata);
 
-        if (rsp.dict.dict_val) {
-                /* don't use GF_FREE, this memory was allocated by libc
-                 */
-                free (rsp.dict.dict_val);
-                rsp.dict.dict_val = NULL;
-        }
+        free (rsp.dict.dict_val);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         if (dict)
                 dict_unref (dict);
 
-        client_local_wipe (local);
         return 0;
 }
 
 int
-client3_1_fsetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fsetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                          void *myframe)
 {
+        call_frame_t  *frame    = NULL;
+        gf_common_rsp  rsp      = {0,};
+        int            ret      = 0;
+        xlator_t      *this     = NULL;
+        dict_t        *xdata    = NULL;
+        int            op_errno = EINVAL;
+
+        this = THIS;
+
+        frame = myframe;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = ENOTCONN;
+                goto out;
+        }
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
+out:
+        op_errno = gf_error_to_errno (rsp.op_errno);
+        if (rsp.op_ret == -1) {
+                gf_log (this->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_WARNING),
+                        "remote operation failed: %s",
+                        strerror (op_errno));
+        }
+
+        CLIENT_STACK_UNWIND (fsetxattr, frame, rsp.op_ret, op_errno, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
+}
+
+int
+client3_3_fsetattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
         call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
+        gfs3_fsetattr_rsp rsp        = {0,};
+        struct iatt      prestat    = {0,};
+        struct iatt      poststat   = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1492,7 +1902,7 @@ client3_1_fsetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 rsp.op_errno = ENOTCONN;
                 goto out;
         }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fsetattr_rsp);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
                 rsp.op_ret   = -1;
@@ -1500,27 +1910,44 @@ client3_1_fsetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        if (-1 != rsp.op_ret) {
+                gf_stat_to_iatt (&rsp.statpre, &prestat);
+                gf_stat_to_iatt (&rsp.statpost, &poststat);
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fsetxattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (fsetattr, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &prestat,
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fsetattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fallocate_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t    *frame      = NULL;
-        gfs3_fsetattr_rsp rsp        = {0,};
+        gfs3_fallocate_rsp rsp      = {0,};
         struct iatt      prestat    = {0,};
         struct iatt      poststat   = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1531,7 +1958,7 @@ client3_1_fsetattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 rsp.op_errno = ENOTCONN;
                 goto out;
         }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fsetattr_rsp);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fallocate_rsp);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
                 rsp.op_ret   = -1;
@@ -1544,21 +1971,140 @@ client3_1_fsetattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.statpost, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fsetattr, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (fallocate, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
+int
+client3_3_discard_cbk(struct rpc_req *req, struct iovec *iov, int count,
+		      void *myframe)
+{
+        call_frame_t    *frame      = NULL;
+        gfs3_discard_rsp rsp      = {0,};
+        struct iatt      prestat    = {0,};
+        struct iatt      poststat   = {0,};
+        int              ret        = 0;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
+        this = THIS;
+
+        frame = myframe;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = ENOTCONN;
+                goto out;
+        }
+        ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gfs3_discard_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        if (-1 != rsp.op_ret) {
+                gf_stat_to_iatt (&rsp.statpre, &prestat);
+                gf_stat_to_iatt (&rsp.statpost, &poststat);
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
+out:
+        if (rsp.op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
+                        strerror (gf_error_to_errno (rsp.op_errno)));
+        }
+        CLIENT_STACK_UNWIND (discard, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &prestat,
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
+}
 
 int
-client3_1_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_zerofill_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+        call_frame_t    *frame         = NULL;
+        gfs3_zerofill_rsp rsp          = {0,};
+        struct iatt      prestat       = {0,};
+        struct iatt      poststat      = {0,};
+        int              ret           = 0;
+        xlator_t *this                 = NULL;
+        dict_t  *xdata                 = NULL;
+
+        this = THIS;
+
+        frame = myframe;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = ENOTCONN;
+                goto out;
+        }
+        ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gfs3_zerofill_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        if (-1 != rsp.op_ret) {
+                gf_stat_to_iatt (&rsp.statpre, &prestat);
+                gf_stat_to_iatt (&rsp.statpost, &poststat);
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
+out:
+        if (rsp.op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "remote operation failed: %s",
+                        strerror (gf_error_to_errno (rsp.op_errno)));
+        }
+        CLIENT_STACK_UNWIND (zerofill, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &prestat,
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
+}
+
+int
+client3_3_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t    *frame      = NULL;
@@ -1566,7 +2112,9 @@ client3_1_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt      prestat    = {0,};
         struct iatt      poststat   = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1591,20 +2139,29 @@ client3_1_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.statpost, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (setattr, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (setattr, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_create_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_create_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         call_frame_t    *frame      = NULL;
@@ -1615,16 +2172,14 @@ client3_1_create_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt      postparent = {0, };
         int32_t          ret        = -1;
         clnt_local_t    *local      = NULL;
-        clnt_conf_t     *conf       = NULL;
-        clnt_fd_ctx_t   *fdctx      = NULL;
         gfs3_create_rsp  rsp        = {0,};
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
         frame = myframe;
-        local = frame->local; frame->local = NULL;
-        conf  = frame->this->private;
+        local = frame->local;
         fd    = local->fd;
         inode = local->loc.inode;
 
@@ -1647,56 +2202,51 @@ client3_1_create_cbk (struct rpc_req *req, struct iovec *iov, int count,
 
                 gf_stat_to_iatt (&rsp.preparent, &preparent);
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
-
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_clnt_fdctx_t);
-                if (!fdctx) {
+                uuid_copy (local->loc.gfid, stbuf.ia_gfid);
+                ret = client_add_fd_to_saved_fds (frame->this, fd, &local->loc,
+                                                  local->flags, rsp.fd, 0);
+                if (ret) {
                         rsp.op_ret = -1;
-                        rsp.op_errno = ENOMEM;
+                        rsp.op_errno = -ret;
                         goto out;
                 }
-
-                fdctx->remote_fd = rsp.fd;
-                fdctx->inode     = inode_ref (inode);
-                fdctx->flags     = local->flags;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-                INIT_LIST_HEAD (&fdctx->lock_list);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->lock);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->lock);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s. Path: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path);
         }
-        STACK_UNWIND_STRICT (create, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (create, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), fd, inode,
-                             &stbuf, &preparent, &postparent);
+                             &stbuf, &preparent, &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
-        client_local_wipe (local);
         return 0;
 }
 
 
 int
-client3_1_rchecksum_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_rchecksum_cbk (struct rpc_req *req, struct iovec *iov, int count,
                          void *myframe)
 {
         call_frame_t *frame = NULL;
         gfs3_rchecksum_rsp rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1716,15 +2266,20 @@ client3_1_rchecksum_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (rchecksum, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (rchecksum, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno),
                              rsp.weak_checksum,
-                             (uint8_t *)rsp.strong_checksum.strong_checksum_val);
+                             (uint8_t *)rsp.strong_checksum.strong_checksum_val,
+                             xdata);
 
         if (rsp.strong_checksum.strong_checksum_val) {
                 /* This is allocated by the libc while decoding RPC msg */
@@ -1732,11 +2287,16 @@ out:
                 free (rsp.strong_checksum.strong_checksum_val);
         }
 
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
         return 0;
 }
 
 int
-client3_1_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                   void *myframe)
 {
         call_frame_t    *frame      = NULL;
@@ -1744,7 +2304,8 @@ client3_1_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct gf_flock     lock       = {0,};
         gfs3_lk_rsp      rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
@@ -1783,37 +2344,47 @@ client3_1_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
         */
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if ((rsp.op_ret == -1) &&
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (lk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &lock);
 
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (lk, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &lock, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        free (rsp.flock.lk_owner.lk_owner_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
         return 0;
 }
 
 int
-client3_1_readdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_readdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
-        call_frame_t           *frame = NULL;
-        gfs3_readdir_rsp        rsp   = {0,};
-        int32_t                 ret   = 0;
-        clnt_local_t         *local = NULL;
-        gf_dirent_t             entries;
-        xlator_t         *this       = NULL;
+        call_frame_t     *frame    = NULL;
+        gfs3_readdir_rsp  rsp      = {0,};
+        int32_t           ret      = 0;
+        clnt_local_t     *local    = NULL;
+        gf_dirent_t       entries;
+        xlator_t         *this     = NULL;
+        dict_t           *xdata    = NULL;
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -1834,21 +2405,30 @@ client3_1_readdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 unserialize_rsp_dirent (&rsp, &entries);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata,
+                                      (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), rsp.op_ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+                        "remote operation failed: %s remote_fd = %d",
+                        strerror (gf_error_to_errno (rsp.op_errno)),
+                                  local->cmd);
         }
-        STACK_UNWIND_STRICT (readdir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &entries);
-
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (readdir, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &entries, xdata);
 
         if (rsp.op_ret != -1) {
                 gf_dirent_free (&entries);
         }
 
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
         clnt_readdir_rsp_cleanup (&rsp);
 
         return 0;
@@ -1856,7 +2436,7 @@ out:
 
 
 int
-client3_1_readdirp_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_readdirp_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t      *frame = NULL;
@@ -1865,12 +2445,12 @@ client3_1_readdirp_cbk (struct rpc_req *req, struct iovec *iov, int count,
         clnt_local_t      *local = NULL;
         gf_dirent_t        entries;
         xlator_t          *this  = NULL;
+        dict_t            *xdata = NULL;
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -1891,20 +2471,26 @@ client3_1_readdirp_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 unserialize_rsp_direntp (this, local->fd, &rsp, &entries);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (readdirp, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &entries);
-
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (readdirp, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &entries, xdata);
 
         if (rsp.op_ret != -1) {
                 gf_dirent_free (&entries);
         }
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         clnt_readdirp_rsp_cleanup (&rsp);
 
@@ -1913,7 +2499,7 @@ out:
 
 
 int
-client3_1_rename_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_rename_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         call_frame_t     *frame      = NULL;
@@ -1924,7 +2510,8 @@ client3_1_rename_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt       prenewparent  = {0,};
         struct iatt       postnewparent = {0,};
         int               ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
@@ -1954,21 +2541,30 @@ client3_1_rename_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postnewparent, &postnewparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (rename, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (rename, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno),
                              &stbuf, &preoldparent, &postoldparent,
-                             &prenewparent, &postnewparent);
+                             &prenewparent, &postnewparent, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_link_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_link_cbk (struct rpc_req *req, struct iovec *iov, int count,
                     void *myframe)
 {
         call_frame_t     *frame      = NULL;
@@ -1979,14 +2575,15 @@ client3_1_link_cbk (struct rpc_req *req, struct iovec *iov, int count,
         int               ret        = 0;
         clnt_local_t     *local      = NULL;
         inode_t          *inode      = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
         frame = myframe;
 
         local = frame->local;
-        frame->local = NULL;
         inode = local->loc.inode;
 
         if (-1 == req->rpc_status) {
@@ -2010,42 +2607,49 @@ client3_1_link_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                        "remote operation failed: %s (%s -> %s)",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, local->loc2.path);
         }
-        STACK_UNWIND_STRICT (link, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (link, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), inode,
-                             &stbuf, &preparent, &postparent);
+                             &stbuf, &preparent, &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
-        client_local_wipe (local);
         return 0;
 }
 
 
 int
-client3_1_opendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_opendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         clnt_local_t      *local = NULL;
         clnt_conf_t       *conf = NULL;
-        clnt_fd_ctx_t     *fdctx = NULL;
-        call_frame_t   *frame = NULL;
-        fd_t             *fd = NULL;
+        call_frame_t      *frame = NULL;
+        fd_t              *fd = NULL;
         int ret = 0;
         gfs3_opendir_rsp  rsp = {0,};
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
 
-        frame->local = NULL;
         conf  = frame->this->private;
         fd    = local->fd;
 
@@ -2064,49 +2668,40 @@ client3_1_opendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
         if (-1 != rsp.op_ret) {
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_clnt_fdctx_t);
-                if (!fdctx) {
+                ret = client_add_fd_to_saved_fds (frame->this, fd, &local->loc,
+                                                  0, rsp.fd, 1);
+                if (ret) {
                         rsp.op_ret = -1;
-                        rsp.op_errno = ENOMEM;
+                        rsp.op_errno = -ret;
                         goto out;
                 }
-
-                fdctx->remote_fd = rsp.fd;
-                fdctx->inode     = inode_ref (fd->inode);
-                fdctx->is_dir    = 1;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-                INIT_LIST_HEAD (&fdctx->lock_list);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->lock);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->lock);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                        "remote operation failed: %s. Path: %s (%s)",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, loc_gfid_utoa (&local->loc));
         }
-        STACK_UNWIND_STRICT (opendir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), fd);
+        CLIENT_STACK_UNWIND (opendir, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), fd, xdata);
 
-        client_local_wipe (local);
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         clnt_local_t    *local      = NULL;
@@ -2116,7 +2711,7 @@ client3_1_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt      stbuf      = {0,};
         struct iatt      postparent = {0,};
         int              op_errno   = EINVAL;
-        dict_t          *xattr      = NULL;
+        dict_t          *xdata      = NULL;
         inode_t         *inode      = NULL;
         xlator_t        *this       = NULL;
 
@@ -2125,7 +2720,6 @@ client3_1_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
         frame = myframe;
         local = frame->local;
         inode = local->loc.inode;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -2150,8 +2744,8 @@ client3_1_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
         rsp.op_ret = -1;
         gf_stat_to_iatt (&rsp.stat, &stbuf);
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xattr, (rsp.dict.dict_val),
-                                      (rsp.dict.dict_len), rsp.op_ret,
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), rsp.op_ret,
                                       op_errno, out);
 
         if ((!uuid_is_null (inode->gfid))
@@ -2167,37 +2761,31 @@ client3_1_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
 
 out:
         rsp.op_errno = op_errno;
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 /* any error other than ENOENT */
                 if (rsp.op_errno != ENOENT)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "remote operation failed: %s. Path: %s",
-                                strerror (rsp.op_errno),
-                                (local) ? local->loc.path : "--");
+                                "remote operation failed: %s. Path: %s (%s)",
+                                strerror (rsp.op_errno), local->loc.path,
+                                loc_gfid_utoa (&local->loc));
                 else
                         gf_log (this->name, GF_LOG_TRACE, "not found on remote node");
 
         }
-        STACK_UNWIND_STRICT (lookup, frame, rsp.op_ret, rsp.op_errno, inode,
-                             &stbuf, xattr, &postparent);
 
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (lookup, frame, rsp.op_ret, rsp.op_errno, inode,
+                             &stbuf, xdata, &postparent);
 
-        if (xattr)
-                dict_unref (xattr);
+        if (xdata)
+                dict_unref (xdata);
 
-        if (rsp.dict.dict_val) {
-                /* don't use GF_FREE, this memory was allocated by libc
-                 */
-                free (rsp.dict.dict_val);
-        }
+        free (rsp.xdata.xdata_val);
 
         return 0;
 }
 
 int
-client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         call_frame_t   *frame  = NULL;
@@ -2207,7 +2795,8 @@ client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
         gfs3_read_rsp   rsp    = {0,};
         int             ret    = 0, rspcount = 0;
         clnt_local_t   *local  = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
@@ -2215,7 +2804,6 @@ client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -2240,23 +2828,37 @@ client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         vector[0].iov_base = req->rsp[1].iov_base;
                 rspcount = 1;
         }
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
+#ifdef GF_TESTING_IO_XDATA
+        dict_dump (xdata);
+#endif
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
+        } else if (rsp.op_ret >= 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
-        STACK_UNWIND_STRICT (readv, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (readv, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), vector, rspcount,
-                             &stat, iobref);
+                             &stat, iobref, xdata);
+
+        free (rsp.xdata.xdata_val);
 
-        client_local_wipe (local);
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_release_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_release_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t   *frame = NULL;
@@ -2266,7 +2868,7 @@ client3_1_release_cbk (struct rpc_req *req, struct iovec *iov, int count,
         return 0;
 }
 int
-client3_1_releasedir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_releasedir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                           void *myframe)
 {
         call_frame_t   *frame = NULL;
@@ -2279,43 +2881,69 @@ client3_1_releasedir_cbk (struct rpc_req *req, struct iovec *iov, int count,
 int
 client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx)
 {
-        call_frame_t *fr = NULL;
-        int32_t       ret = -1;
+        clnt_conf_t  *conf        = NULL;
+        call_frame_t *fr          = NULL;
+        int32_t       ret         = -1;
+        char          parent_down = 0;
+        fd_lk_ctx_t  *lk_ctx      = NULL;
 
-        if (!fdctx)
-                goto out;
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
+
+        conf = (clnt_conf_t *) this->private;
 
         if (fdctx->remote_fd == -1) {
                 gf_log (this->name, GF_LOG_DEBUG, "not a valid fd");
                 goto out;
         }
 
+        pthread_mutex_lock (&conf->lock);
+        {
+                parent_down   = conf->parent_down;
+                lk_ctx        = fdctx->lk_ctx;
+                fdctx->lk_ctx = NULL;
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        if (lk_ctx)
+                fd_lk_ctx_unref (lk_ctx);
+
+        if (!parent_down)
+                rpc_clnt_ref (conf->rpc);
+        else
+                goto out;
+
         fr = create_frame (this, this->ctx->pool);
+        if (fr == NULL) {
+                goto out;
+        }
+
+        ret = 0;
 
         if (fdctx->is_dir) {
                 gfs3_releasedir_req  req = {{0,},};
                 req.fd = fdctx->remote_fd;
-                gf_log (this->name, GF_LOG_INFO, "sending releasedir on fd");
-                ret = client_submit_request (this, &req, fr, &clnt3_1_fop_prog,
-                                             GFS3_OP_RELEASEDIR,
-                                             client3_1_releasedir_cbk,
-                                             NULL, NULL, 0, NULL, 0, NULL,
-                                             (xdrproc_t)xdr_gfs3_releasedir_req);
+                gf_log (this->name, GF_LOG_TRACE, "sending releasedir on fd");
+                client_submit_request (this, &req, fr, &clnt3_3_fop_prog,
+                                       GFS3_OP_RELEASEDIR,
+                                       client3_3_releasedir_cbk,
+                                       NULL, NULL, 0, NULL, 0, NULL,
+                                       (xdrproc_t)xdr_gfs3_releasedir_req);
         } else {
                 gfs3_release_req  req = {{0,},};
                 req.fd = fdctx->remote_fd;
-                gf_log (this->name, GF_LOG_INFO, "sending release on fd");
-                ret = client_submit_request (this, &req, fr, &clnt3_1_fop_prog,
-                                             GFS3_OP_RELEASE,
-                                             client3_1_release_cbk, NULL,
-                                             NULL, 0, NULL, 0, NULL,
-                                             (xdrproc_t)xdr_gfs3_release_req);
+                gf_log (this->name, GF_LOG_TRACE, "sending release on fd");
+                client_submit_request (this, &req, fr, &clnt3_3_fop_prog,
+                                       GFS3_OP_RELEASE,
+                                       client3_3_release_cbk, NULL,
+                                       NULL, 0, NULL, 0, NULL,
+                                       (xdrproc_t)xdr_gfs3_release_req);
         }
 
+        rpc_clnt_unref (conf->rpc);
 out:
-        if (!ret && fdctx) {
+        if (fdctx) {
                 fdctx->remote_fd = -1;
-                inode_unref (fdctx->inode);
                 GF_FREE (fdctx);
         }
 
@@ -2326,18 +2954,16 @@ out:
 }
 
 int32_t
-client3_1_releasedir (call_frame_t *frame, xlator_t *this,
+client3_3_releasedir (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
-        clnt_conf_t         *conf = NULL;
-        clnt_fd_ctx_t       *fdctx = NULL;
-        clnt_args_t         *args = NULL;
-        gfs3_releasedir_req  req = {{0,},};
-        int64_t              remote_fd = -1;
-        int                  ret = 0;
+        clnt_conf_t         *conf        = NULL;
+        clnt_fd_ctx_t       *fdctx       = NULL;
+        clnt_args_t         *args        = NULL;
+        int64_t              remote_fd   = -1;
 
-        if (!frame || !this || !data)
-                goto unwind;
+        if (!this || !data)
+                goto out;
 
         args = data;
         conf = this->private;
@@ -2361,37 +2987,25 @@ client3_1_releasedir (call_frame_t *frame, xlator_t *this,
         }
         pthread_mutex_unlock (&conf->lock);
 
-        if (remote_fd != -1) {
-                req.fd = remote_fd;
-                ret = client_submit_request (this, &req, frame, conf->fops,
-                                             GFS3_OP_RELEASEDIR,
-                                             client3_1_releasedir_cbk,
-                                             NULL, NULL, 0, NULL, 0, NULL,
-                                             (xdrproc_t)xdr_gfs3_releasedir_req);
-                inode_unref (fdctx->inode);
-                GF_FREE (fdctx);
-        }
-
-unwind:
-        if (ret)
-                STACK_DESTROY (frame->root);
+        if (remote_fd != -1)
+                client_fdctx_destroy (this, fdctx);
+out:
 
         return 0;
 }
 
 int32_t
-client3_1_release (call_frame_t *frame, xlator_t *this,
+client3_3_release (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
-        int64_t           remote_fd = -1;
-        clnt_conf_t      *conf      = NULL;
-        clnt_fd_ctx_t    *fdctx     = NULL;
-        clnt_args_t      *args      = NULL;
-        gfs3_release_req  req       = {{0,},};
-        int               ret       = 0;
+        int64_t           remote_fd     = -1;
+        clnt_conf_t      *conf          = NULL;
+        clnt_fd_ctx_t    *fdctx         = NULL;
+        clnt_args_t      *args          = NULL;
+        lk_heal_state_t   lk_heal_state = GF_LK_HEAL_DONE;
 
-        if (!frame || !this || !data)
-                goto unwind;
+        if (!this || !data)
+                goto out;
 
         args = data;
         conf = this->private;
@@ -2400,14 +3014,16 @@ client3_1_release (call_frame_t *frame, xlator_t *this,
         {
                 fdctx = this_fd_del_ctx (args->fd, this);
                 if (fdctx != NULL) {
-                        remote_fd = fdctx->remote_fd;
+                        remote_fd     = fdctx->remote_fd;
+                        lk_heal_state = fdctx->lk_heal_state;
 
                         /* fdctx->remote_fd == -1 indicates a reopen attempt
                            in progress. Just mark ->released = 1 and let
                            reopen_cbk handle releasing
                         */
 
-                        if (remote_fd != -1)
+                        if (remote_fd != -1 &&
+                            lk_heal_state == GF_LK_HEAL_DONE)
                                 list_del_init (&fdctx->sfd_pos);
 
                         fdctx->released = 1;
@@ -2415,29 +3031,15 @@ client3_1_release (call_frame_t *frame, xlator_t *this,
         }
         pthread_mutex_unlock (&conf->lock);
 
-        if (remote_fd != -1) {
-                req.fd = remote_fd;
-
-                delete_granted_locks_fd (fdctx);
-
-                ret = client_submit_request (this, &req, frame, conf->fops,
-                                             GFS3_OP_RELEASE,
-                                             client3_1_release_cbk, NULL, NULL,
-                                             0, NULL, 0, NULL,
-                                             (xdrproc_t)xdr_gfs3_release_req);
-                inode_unref (fdctx->inode);
-                GF_FREE (fdctx);
-        }
-unwind:
-        if (ret)
-                STACK_DESTROY (frame->root);
-
+        if (remote_fd != -1 && lk_heal_state == GF_LK_HEAL_DONE)
+                client_fdctx_destroy (this, fdctx);
+out:
         return 0;
 }
 
 
 int32_t
-client3_1_lookup (call_frame_t *frame, xlator_t *this,
+client3_3_lookup (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_conf_t     *conf              = NULL;
@@ -2460,7 +3062,7 @@ client3_1_lookup (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
         args = data;
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -2470,6 +3072,7 @@ client3_1_lookup (call_frame_t *frame, xlator_t *this,
                 goto unwind;
 
         loc_copy (&local->loc, args->loc);
+        loc_path (&local->loc, NULL);
         frame->local = local;
 
         if (args->loc->parent) {
@@ -2484,8 +3087,8 @@ client3_1_lookup (call_frame_t *frame, xlator_t *this,
                         memcpy (req.gfid, args->loc->gfid, 16);
         }
 
-        if (args->dict) {
-                content = dict_get (args->dict, GF_CONTENT_KEY);
+        if (args->xdata) {
+                content = dict_get (args->xdata, GF_CONTENT_KEY);
                 if (content != NULL) {
                         rsp_iobref = iobref_new ();
                         if (rsp_iobref == NULL) {
@@ -2493,6 +3096,8 @@ client3_1_lookup (call_frame_t *frame, xlator_t *this,
                         }
 
                         /* TODO: what is the size we should send ? */
+                        /* This change very much depends on quick-read
+                           changes */
                         rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
                         if (rsp_iobuf == NULL) {
                                 goto unwind;
@@ -2502,75 +3107,61 @@ client3_1_lookup (call_frame_t *frame, xlator_t *this,
                         iobuf_unref (rsp_iobuf);
                         rsphdr = &vector[0];
                         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-                        rsphdr->iov_len
-                                = iobuf_pagesize (rsp_iobuf);
+                        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
                         count = 1;
-                        rsp_iobuf = NULL;
                         local->iobref = rsp_iobref;
+                        rsp_iobuf = NULL;
                         rsp_iobref = NULL;
                 }
 
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
+                GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata,
+                                            (&req.xdata.xdata_val),
+                                            req.xdata.xdata_len,
                                             op_errno, unwind);
         }
 
         if (args->loc->name)
-                req.bname         = (char *)args->loc->name;
+                req.bname = (char *)args->loc->name;
         else
                 req.bname = "";
 
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_LOOKUP, client3_1_lookup_cbk,
+                                     GFS3_OP_LOOKUP, client3_3_lookup_cbk,
                                      NULL, rsphdr, count,
                                      NULL, 0, local->iobref,
                                      (xdrproc_t)xdr_gfs3_lookup_req);
 
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.xdata.xdata_val);
 
-        if (rsp_iobref != NULL) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
+
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
 
         return 0;
 
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-
-        if (frame)
-                frame->local = NULL;
-
-        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+        CLIENT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
                              NULL);
 
-        client_local_wipe (local);
-
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.xdata.xdata_val);
 
-        if (rsp_iobref != NULL) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        if (rsp_iobuf != NULL) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
         return 0;
 }
 
-
-
 int32_t
-client3_1_stat (call_frame_t *frame, xlator_t *this,
+client3_3_stat (call_frame_t *frame, xlator_t *this,
                 void *data)
 {
         clnt_conf_t   *conf     = NULL;
@@ -2596,26 +3187,31 @@ client3_1_stat (call_frame_t *frame, xlator_t *this,
                                        unwind, op_errno, EINVAL);
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_STAT, client3_1_stat_cbk, NULL,
+                                     GFS3_OP_STAT, client3_3_stat_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_stat_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop %s",
-                strerror (op_errno));
-        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_truncate (call_frame_t *frame, xlator_t *this,
+client3_3_truncate (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_conf_t       *conf     = NULL;
@@ -2644,25 +3240,31 @@ client3_1_truncate (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_TRUNCATE,
-                                     client3_1_truncate_cbk, NULL,
+                                     client3_3_truncate_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_truncate_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_ftruncate (call_frame_t *frame, xlator_t *this,
+client3_3_ftruncate (call_frame_t *frame, xlator_t *this,
                      void *data)
 {
         clnt_args_t        *args     = NULL;
@@ -2679,32 +3281,39 @@ client3_1_ftruncate (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.offset = args->offset;
         req.fd     = remote_fd;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FTRUNCATE,
-                                     client3_1_ftruncate_cbk, NULL,
+                                     client3_3_ftruncate_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_ftruncate_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_access (call_frame_t *frame, xlator_t *this,
+client3_3_access (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_conf_t     *conf     = NULL;
@@ -2733,32 +3342,43 @@ client3_1_access (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_ACCESS,
-                                     client3_1_access_cbk, NULL,
+                                     client3_3_access_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_access_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (access, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 int32_t
-client3_1_readlink (call_frame_t *frame, xlator_t *this,
+client3_3_readlink (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
-        clnt_conf_t       *conf     = NULL;
-        clnt_args_t       *args     = NULL;
-        gfs3_readlink_req  req      = {{0,},};
-        int                ret      = 0;
-        int                op_errno = ESTALE;
+        clnt_conf_t       *conf              = NULL;
+        clnt_args_t       *args              = NULL;
+        gfs3_readlink_req  req               = {{0,},};
+        int                ret               = 0;
+        int                op_errno          = ESTALE;
+        clnt_local_t      *local             = NULL;
+        struct iobuf      *rsp_iobuf         = NULL;
+        struct iobref     *rsp_iobref        = NULL;
+        struct iovec      *rsphdr            = NULL;
+        int                count             = 0;
+        struct iovec       vector[MAX_IOVEC] = {{0}, };
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -2779,20 +3399,58 @@ client3_1_readlink (call_frame_t *frame, xlator_t *this,
         req.size = args->size;
         conf = this->private;
 
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        frame->local = local;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        rsp_iobref = iobref_new ();
+        if (rsp_iobref == NULL) {
+                goto unwind;
+        }
+
+        rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
+        if (rsp_iobuf == NULL) {
+                goto unwind;
+        }
+
+        iobref_add (rsp_iobref, rsp_iobuf);
+        iobuf_unref (rsp_iobuf);
+        rsphdr = &vector[0];
+        rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
+        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
+        count = 1;
+        local->iobref = rsp_iobref;
+        rsp_iobuf = NULL;
+        rsp_iobref = NULL;
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_READLINK,
-                                     client3_1_readlink_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_readlink_req);
+                                     client3_3_readlink_cbk, NULL,
+                                     rsphdr, count, NULL, 0,
+                                     local->iobref,
+                                     (xdrproc_t)xdr_gfs3_readlink_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL);
+        if (rsp_iobref != NULL) {
+                iobref_unref (rsp_iobref);
+        }
+
+        CLIENT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
@@ -2800,7 +3458,7 @@ unwind:
 
 
 int32_t
-client3_1_unlink (call_frame_t *frame, xlator_t *this,
+client3_3_unlink (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_conf_t     *conf     = NULL;
@@ -2828,26 +3486,32 @@ client3_1_unlink (call_frame_t *frame, xlator_t *this,
         req.bname = (char *)args->loc->name;
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_UNLINK,
-                                     client3_1_unlink_cbk, NULL,
+                                     client3_3_unlink_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_unlink_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_rmdir (call_frame_t *frame, xlator_t *this,
+client3_3_rmdir (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_conf_t    *conf     = NULL;
@@ -2873,28 +3537,33 @@ client3_1_rmdir (call_frame_t *frame, xlator_t *this,
                                        !uuid_is_null (*((uuid_t*)req.pargfid)),
                                        unwind, op_errno, EINVAL);
         req.bname = (char *)args->loc->name;
-        req.flags = args->flags;
+        req.xflags = args->flags;
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_RMDIR, client3_1_rmdir_cbk, NULL,
+                                     GFS3_OP_RMDIR, client3_3_rmdir_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_rmdir_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_symlink (call_frame_t *frame, xlator_t *this,
+client3_3_symlink (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_local_t     *local    = NULL;
@@ -2908,17 +3577,19 @@ client3_1_symlink (call_frame_t *frame, xlator_t *this,
                 goto unwind;
 
         args = data;
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
         }
 
+        frame->local = local;
+
         if (!(args->loc && args->loc->parent))
                 goto unwind;
 
         loc_copy (&local->loc, args->loc);
-        frame->local = local;
+        loc_path (&local->loc, NULL);
 
         if (!uuid_is_null (args->loc->parent->gfid))
                 memcpy (req.pargfid,  args->loc->parent->gfid, 16);
@@ -2930,42 +3601,39 @@ client3_1_symlink (call_frame_t *frame, xlator_t *this,
                                        unwind, op_errno, EINVAL);
         req.linkname = (char *)args->linkname;
         req.bname    = (char *)args->loc->name;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->dict, (&req.dict.dict_val),
-                                    req.dict.dict_len, op_errno, unwind);
+        req.umask = args->umask;
+        local->loc2.path = gf_strdup (req.linkname);
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_SYMLINK, client3_1_symlink_cbk,
+                                     GFS3_OP_SYMLINK, client3_3_symlink_cbk,
                                      NULL,  NULL, 0, NULL,
                                      0, NULL, (xdrproc_t)xdr_gfs3_symlink_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
-        }
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
 
-        STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        CLIENT_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
 
-        client_local_wipe (local);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
         return 0;
 }
 
 
 
 int32_t
-client3_1_rename (call_frame_t *frame, xlator_t *this,
+client3_3_rename (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_conf_t     *conf     = NULL;
@@ -2999,31 +3667,37 @@ client3_1_rename (call_frame_t *frame, xlator_t *this,
         GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
                                        !uuid_is_null (*((uuid_t*)req.newgfid)),
                                        unwind, op_errno, EINVAL);
-        req.oldpath = (char *)args->oldloc->path;
         req.oldbname =  (char *)args->oldloc->name;
-        req.newpath = (char *)args->newloc->path;
         req.newbname = (char *)args->newloc->name;
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_RENAME, client3_1_rename_cbk, NULL,
+                                     GFS3_OP_RENAME, client3_3_rename_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_rename_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        CLIENT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_link (call_frame_t *frame, xlator_t *this,
+client3_3_link (call_frame_t *frame, xlator_t *this,
                 void *data)
 {
         clnt_local_t  *local    = NULL;
@@ -3058,39 +3732,46 @@ client3_1_link (call_frame_t *frame, xlator_t *this,
         GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
                                        !uuid_is_null (*((uuid_t*)req.newgfid)),
                                        unwind, op_errno, EINVAL);
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
         }
 
         loc_copy (&local->loc, args->oldloc);
+        loc_path (&local->loc, NULL);
+        loc_copy (&local->loc2, args->newloc);
+        loc_path (&local->loc2, NULL);
         frame->local = local;
 
-        req.oldpath = (char *)args->oldloc->path;
-        req.newpath = (char *)args->newloc->path;
         req.newbname = (char *)args->newloc->name;
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_LINK, client3_1_link_cbk, NULL,
+                                     GFS3_OP_LINK, client3_3_link_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_link_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        CLIENT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_mknod (call_frame_t *frame, xlator_t *this,
+client3_3_mknod (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_local_t   *local    = NULL;
@@ -3105,7 +3786,7 @@ client3_1_mknod (call_frame_t *frame, xlator_t *this,
 
         args = data;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3115,6 +3796,7 @@ client3_1_mknod (call_frame_t *frame, xlator_t *this,
                 goto unwind;
 
         loc_copy (&local->loc, args->loc);
+        loc_path (&local->loc, NULL);
         frame->local = local;
 
         if (!uuid_is_null (args->loc->parent->gfid))
@@ -3128,45 +3810,36 @@ client3_1_mknod (call_frame_t *frame, xlator_t *this,
         req.bname  = (char *)args->loc->name;
         req.mode   = args->mode;
         req.dev    = args->rdev;
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
+        req.umask = args->umask;
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_MKNOD, client3_1_mknod_cbk, NULL,
+                                     GFS3_OP_MKNOD, client3_3_mknod_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_mknod_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
-        }
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
+        CLIENT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL);
 
-        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
 
-        client_local_wipe (local);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
         return 0;
 }
 
 
 
 int32_t
-client3_1_mkdir (call_frame_t *frame, xlator_t *this,
+client3_3_mkdir (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_local_t   *local    = NULL;
@@ -3181,7 +3854,7 @@ client3_1_mkdir (call_frame_t *frame, xlator_t *this,
 
         args = data;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3191,6 +3864,7 @@ client3_1_mkdir (call_frame_t *frame, xlator_t *this,
                 goto unwind;
 
         loc_copy (&local->loc, args->loc);
+        loc_path (&local->loc, NULL);
         frame->local = local;
 
         if (!uuid_is_null (args->loc->parent->gfid))
@@ -3204,44 +3878,35 @@ client3_1_mkdir (call_frame_t *frame, xlator_t *this,
 
         req.bname = (char *)args->loc->name;
         req.mode  = args->mode;
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
+        req.umask = args->umask;
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_MKDIR, client3_1_mkdir_cbk, NULL,
+                                     GFS3_OP_MKDIR, client3_3_mkdir_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_mkdir_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
-        }
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
+        CLIENT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL);
 
-        STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
 
-        client_local_wipe (local);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
         return 0;
 }
 
 
 int32_t
-client3_1_create (call_frame_t *frame, xlator_t *this,
+client3_3_create (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_local_t    *local    = NULL;
@@ -3256,7 +3921,7 @@ client3_1_create (call_frame_t *frame, xlator_t *this,
 
         args = data;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3268,6 +3933,7 @@ client3_1_create (call_frame_t *frame, xlator_t *this,
         local->flags = args->flags;
 
         loc_copy (&local->loc, args->loc);
+        loc_path (&local->loc, NULL);
         frame->local = local;
 
         if (!uuid_is_null (args->loc->parent->gfid))
@@ -3281,45 +3947,37 @@ client3_1_create (call_frame_t *frame, xlator_t *this,
         req.bname = (char *)args->loc->name;
         req.mode  = args->mode;
         req.flags = gf_flags_from_flags (args->flags);
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
+        req.umask = args->umask;
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_CREATE, client3_1_create_cbk, NULL,
+                                     GFS3_OP_CREATE, client3_3_create_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_create_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
-        }
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
+        CLIENT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
 
-        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
-        client_local_wipe (local);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
         return 0;
 }
 
 
 
 int32_t
-client3_1_open (call_frame_t *frame, xlator_t *this,
+client3_3_open (call_frame_t *frame, xlator_t *this,
                 void *data)
 {
         clnt_local_t  *local    = NULL;
@@ -3334,7 +3992,7 @@ client3_1_open (call_frame_t *frame, xlator_t *this,
 
         args = data;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3344,8 +4002,8 @@ client3_1_open (call_frame_t *frame, xlator_t *this,
 
         local->fd = fd_ref (args->fd);
         local->flags = args->flags;
-        local->wbflags = args->wbflags;
         loc_copy (&local->loc, args->loc);
+        loc_path (&local->loc, NULL);
         frame->local = local;
 
         if (!uuid_is_null (args->loc->inode->gfid))
@@ -3357,46 +4015,47 @@ client3_1_open (call_frame_t *frame, xlator_t *this,
                                        !uuid_is_null (*((uuid_t*)req.gfid)),
                                        unwind, op_errno, EINVAL);
         req.flags = gf_flags_from_flags (args->flags);
-        req.wbflags = args->wbflags;
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPEN, client3_1_open_cbk, NULL,
+                                     GFS3_OP_OPEN, client3_3_open_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_open_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
+        CLIENT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
 
-        STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
 
-        client_local_wipe (local);
         return 0;
 }
 
 
 
 int32_t
-client3_1_readv (call_frame_t *frame, xlator_t *this,
+client3_3_readv (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_args_t    *args       = NULL;
         int64_t         remote_fd  = -1;
         clnt_conf_t    *conf       = NULL;
+        clnt_local_t   *local      = NULL;
         int             op_errno   = ESTALE;
         gfs3_read_req   req        = {{0,},};
         int             ret        = 0;
         struct iovec    rsp_vec    = {0, };
         struct iobuf   *rsp_iobuf  = NULL;
         struct iobref  *rsp_iobref = NULL;
-        clnt_local_t   *local      = NULL;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -3404,7 +4063,14 @@ client3_1_readv (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
+        local = frame->local;
 
         req.size   = args->size;
         req.offset = args->offset;
@@ -3413,8 +4079,7 @@ client3_1_readv (call_frame_t *frame, xlator_t *this,
 
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-                        /* TODO: what is the size we should send ? */
-        rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
+        rsp_iobuf = iobuf_get2 (this->ctx->iobuf_pool, args->size);
         if (rsp_iobuf == NULL) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3428,6 +4093,7 @@ client3_1_readv (call_frame_t *frame, xlator_t *this,
 
         iobref_add (rsp_iobref, rsp_iobuf);
         iobuf_unref (rsp_iobuf);
+
         rsp_vec.iov_base = iobuf_ptr (rsp_iobuf);
         rsp_vec.iov_len = iobuf_pagesize (rsp_iobuf);
 
@@ -3442,51 +4108,48 @@ client3_1_readv (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
         local->iobref = rsp_iobref;
         rsp_iobref = NULL;
-        frame->local = local;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
 
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_READ, client3_1_readv_cbk, NULL,
+                                     GFS3_OP_READ, client3_3_readv_cbk, NULL,
                                      NULL, 0, &rsp_vec, 1,
                                      local->iobref,
                                      (xdrproc_t)xdr_gfs3_read_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                //unwind is done in the cbk
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        CLIENT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_writev (call_frame_t *frame, xlator_t *this, void *data)
+client3_3_writev (call_frame_t *frame, xlator_t *this, void *data)
 {
         clnt_args_t    *args     = NULL;
         int64_t         remote_fd = -1;
         clnt_conf_t    *conf     = NULL;
         gfs3_write_req  req      = {{0,},};
         int             op_errno = ESTALE;
-        int             ret        = 0;
+        int             ret      = 0;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -3494,7 +4157,13 @@ client3_1_writev (call_frame_t *frame, xlator_t *this, void *data)
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
 
         req.size   = args->size;
         req.offset = args->offset;
@@ -3503,9 +4172,21 @@ client3_1_writev (call_frame_t *frame, xlator_t *this, void *data)
 
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-        ret = client_submit_vec_request (this, &req, frame, conf->fops, GFS3_OP_WRITE,
-                                         client3_1_writev_cbk, args->vector,
-                                         args->count, args->iobref,
+#ifdef GF_TESTING_IO_XDATA
+        if (!args->xdata)
+                args->xdata = dict_new ();
+
+        ret = dict_set_str (args->xdata, "testing-the-xdata-key",
+                            "testing-the-xdata-value");
+#endif
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        ret = client_submit_vec_request (this, &req, frame, conf->fops,
+                                         GFS3_OP_WRITE, client3_3_writev_cbk,
+                                         args->vector, args->count,
+                                         args->iobref,
                                          (xdrproc_t)xdr_gfs3_write_req);
         if (ret) {
                 /*
@@ -3513,20 +4194,23 @@ client3_1_writev (call_frame_t *frame, xlator_t *this, void *data)
                  * do the unwind for us (see rpc_clnt_submit), so don't unwind
                  * here in such cases.
                  */
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to send the fop: %s", strerror (op_errno));
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_flush (call_frame_t *frame, xlator_t *this,
+client3_3_flush (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_args_t    *args     = NULL;
@@ -3543,15 +4227,15 @@ client3_1_flush (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         conf = this->private;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
-                STACK_UNWIND (frame, -1, ENOMEM);
-                return 0;
-
+                op_errno = ENOMEM;
+                goto unwind;
         }
 
         local->fd = fd_ref (args->fd);
@@ -3561,33 +4245,41 @@ client3_1_flush (call_frame_t *frame, xlator_t *this,
         req.fd = remote_fd;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FLUSH, client3_1_flush_cbk, NULL,
+                                     GFS3_OP_FLUSH, client3_3_flush_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_flush_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_fsync (call_frame_t *frame, xlator_t *this,
+client3_3_fsync (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
-        clnt_args_t    *args     = NULL;
-        gfs3_fsync_req  req      = {{0,},};
+        clnt_args_t    *args      = NULL;
+        gfs3_fsync_req  req       = {{0,},};
         int64_t         remote_fd = -1;
-        clnt_conf_t    *conf     = NULL;
-        int             op_errno = 0;
-        int           ret        = 0;
+        clnt_conf_t    *conf      = NULL;
+        int             op_errno  = 0;
+        int             ret       = 0;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -3595,31 +4287,40 @@ client3_1_fsync (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd   = remote_fd;
         req.data = args->flags;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSYNC, client3_1_fsync_cbk, NULL,
+                                     GFS3_OP_FSYNC, client3_3_fsync_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_fsync_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_fstat (call_frame_t *frame, xlator_t *this,
+client3_3_fstat (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_args_t    *args     = NULL;
@@ -3635,30 +4336,38 @@ client3_1_fstat (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd = remote_fd;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSTAT, client3_1_fstat_cbk, NULL,
+                                     GFS3_OP_FSTAT, client3_3_fstat_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_fstat_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_opendir (call_frame_t *frame, xlator_t *this,
+client3_3_opendir (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_local_t     *local    = NULL;
@@ -3673,7 +4382,7 @@ client3_1_opendir (call_frame_t *frame, xlator_t *this,
 
         args = data;
 
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3683,6 +4392,7 @@ client3_1_opendir (call_frame_t *frame, xlator_t *this,
 
         local->fd = fd_ref (args->fd);
         loc_copy (&local->loc, args->loc);
+        loc_path (&local->loc, NULL);
         frame->local = local;
 
         if (!uuid_is_null (args->loc->inode->gfid))
@@ -3696,35 +4406,40 @@ client3_1_opendir (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPENDIR, client3_1_opendir_cbk,
+                                     GFS3_OP_OPENDIR, client3_3_opendir_cbk,
                                      NULL, NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_opendir_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
-        STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL);
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_fsyncdir (call_frame_t *frame, xlator_t *this, void *data)
+client3_3_fsyncdir (call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t       *args     = NULL;
+        clnt_args_t       *args      = NULL;
         int64_t            remote_fd = -1;
-        clnt_conf_t       *conf     = NULL;
-        int                op_errno = ESTALE;
-        gfs3_fsyncdir_req  req      = {{0,},};
-        int                ret        = 0;
+        clnt_conf_t       *conf      = NULL;
+        gfs3_fsyncdir_req  req       = {{0,},};
+        int                ret       = 0;
+        int32_t            op_errno  = ESTALE;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -3732,7 +4447,8 @@ client3_1_fsyncdir (call_frame_t *frame, xlator_t *this, void *data)
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd   = remote_fd;
         req.data = args->flags;
@@ -3740,26 +4456,33 @@ client3_1_fsyncdir (call_frame_t *frame, xlator_t *this, void *data)
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSYNCDIR, client3_1_fsyncdir_cbk,
+                                     GFS3_OP_FSYNCDIR, client3_3_fsyncdir_cbk,
                                      NULL, NULL, 0,
                                      NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_fsyncdir_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_statfs (call_frame_t *frame, xlator_t *this,
+client3_3_statfs (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_conf_t     *conf     = NULL;
@@ -3790,25 +4513,32 @@ client3_1_statfs (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_STATFS, client3_1_statfs_cbk, NULL,
+                                     GFS3_OP_STATFS, client3_3_statfs_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_statfs_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_setxattr (call_frame_t *frame, xlator_t *this,
+client3_3_setxattr (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_conf_t       *conf     = NULL;
@@ -3833,8 +4563,8 @@ client3_1_setxattr (call_frame_t *frame, xlator_t *this,
         GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
                                        !uuid_is_null (*((uuid_t*)req.gfid)),
                                        unwind, op_errno, EINVAL);
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
+        if (args->xattr) {
+                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
                                             (&req.dict.dict_val),
                                             req.dict.dict_len,
                                             op_errno, unwind);
@@ -3844,32 +4574,34 @@ client3_1_setxattr (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_SETXATTR, client3_1_setxattr_cbk,
+                                     GFS3_OP_SETXATTR, client3_3_setxattr_cbk,
                                      NULL, NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_setxattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
-        }
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+        GF_FREE (req.dict.dict_val);
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        CLIENT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+        GF_FREE (req.dict.dict_val);
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_fsetxattr (call_frame_t *frame, xlator_t *this,
+client3_3_fsetxattr (call_frame_t *frame, xlator_t *this,
                      void *data)
 {
         clnt_args_t        *args     = NULL;
@@ -3885,39 +4617,42 @@ client3_1_fsetxattr (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd    = remote_fd;
         req.flags = args->flags;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
+        if (args->xattr) {
+                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
                                             (&req.dict.dict_val),
                                             req.dict.dict_len,
                                             op_errno, unwind);
         }
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSETXATTR, client3_1_fsetxattr_cbk,
+                                     GFS3_OP_FSETXATTR, client3_3_fsetxattr_cbk,
                                      NULL, NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_fsetxattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.dict.dict_val);
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        CLIENT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+        GF_FREE (req.dict.dict_val);
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
@@ -3925,7 +4660,7 @@ unwind:
 
 
 int32_t
-client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
+client3_3_fgetxattr (call_frame_t *frame, xlator_t *this,
                      void *data)
 {
         clnt_args_t        *args       = NULL;
@@ -3947,10 +4682,10 @@ client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3963,8 +4698,8 @@ client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-                        /* TODO: what is the size we should send ? */
-        rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
+        /* TODO: what is the size we should send ? */
+        rsp_iobuf = iobuf_get2 (this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
         if (rsp_iobuf == NULL) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -3972,12 +4707,13 @@ client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
 
         iobref_add (rsp_iobref, rsp_iobuf);
         iobuf_unref (rsp_iobuf);
+
         rsphdr = &vector[0];
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
         rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);;
         count = 1;
-        rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
+        rsp_iobuf = NULL;
         rsp_iobref = NULL;
 
         req.namelen = 1; /* Use it as a flag */
@@ -3989,34 +4725,38 @@ client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
         }
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FGETXATTR,
-                                     client3_1_fgetxattr_cbk, NULL,
+                                     client3_3_fgetxattr_cbk, NULL,
                                      rsphdr, count,
                                      NULL, 0, local->iobref,
                                      (xdrproc_t)xdr_gfs3_fgetxattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        return 0;
-unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
+        GF_FREE (req.xdata.xdata_val);
 
-        STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL);
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
+
+        if (rsp_iobref)
+                iobref_unref (rsp_iobref);
 
-        client_local_wipe (local);
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 }
@@ -4024,7 +4764,7 @@ unwind:
 
 
 int32_t
-client3_1_getxattr (call_frame_t *frame, xlator_t *this,
+client3_3_getxattr (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_conf_t       *conf       = NULL;
@@ -4047,17 +4787,22 @@ client3_1_getxattr (call_frame_t *frame, xlator_t *this,
         }
         args = data;
 
-        if (!(args->loc && args->loc->inode)) {
+        if (!args->loc) {
                 op_errno = EINVAL;
                 goto unwind;
         }
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
         }
+
+        loc_copy (&local->loc, args->loc);
+        loc_path (&local->loc, NULL);
+        if (args->name)
+                local->name = gf_strdup (args->name);
+
         frame->local = local;
 
         rsp_iobref = iobref_new ();
@@ -4066,8 +4811,8 @@ client3_1_getxattr (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-                        /* TODO: what is the size we should send ? */
-        rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
+        /* TODO: what is the size we should send ? */
+        rsp_iobuf = iobuf_get2 (this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
         if (rsp_iobuf == NULL) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -4075,15 +4820,16 @@ client3_1_getxattr (call_frame_t *frame, xlator_t *this,
 
         iobref_add (rsp_iobref, rsp_iobuf);
         iobuf_unref (rsp_iobuf);
+
         rsphdr = &vector[0];
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
         rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
         count = 1;
-        rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
+        rsp_iobuf = NULL;
         rsp_iobref = NULL;
 
-        if (!uuid_is_null (args->loc->inode->gfid))
+        if (args->loc->inode && !uuid_is_null (args->loc->inode->gfid))
                 memcpy (req.gfid,  args->loc->inode->gfid, 16);
         else
                 memcpy (req.gfid, args->loc->gfid, 16);
@@ -4120,34 +4866,38 @@ client3_1_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_GETXATTR,
-                                     client3_1_getxattr_cbk, NULL,
+                                     client3_3_getxattr_cbk, NULL,
                                      rsphdr, count,
                                      NULL, 0, local->iobref,
                                      (xdrproc_t)xdr_gfs3_getxattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
+
+        if (rsp_iobref)
+                iobref_unref (rsp_iobref);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s",
-                strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
-        client_local_wipe (local);
-
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        CLIENT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 }
@@ -4155,7 +4905,7 @@ unwind:
 
 
 int32_t
-client3_1_xattrop (call_frame_t *frame, xlator_t *this,
+client3_3_xattrop (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_conf_t      *conf       = NULL;
@@ -4178,8 +4928,7 @@ client3_1_xattrop (call_frame_t *frame, xlator_t *this,
         if (!(args->loc && args->loc->inode))
                 goto unwind;
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -4192,8 +4941,8 @@ client3_1_xattrop (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-                        /* TODO: what is the size we should send ? */
-        rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
+        /* TODO: what is the size we should send ? */
+        rsp_iobuf = iobuf_get2 (this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
         if (rsp_iobuf == NULL) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -4201,12 +4950,13 @@ client3_1_xattrop (call_frame_t *frame, xlator_t *this,
 
         iobref_add (rsp_iobref, rsp_iobuf);
         iobuf_unref (rsp_iobuf);
+
         rsphdr = &vector[0];
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
         rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
         count = 1;
-        rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
+        rsp_iobuf = NULL;
         rsp_iobref = NULL;
 
         if (!uuid_is_null (args->loc->inode->gfid))
@@ -4214,11 +4964,13 @@ client3_1_xattrop (call_frame_t *frame, xlator_t *this,
         else
                 memcpy (req.gfid, args->loc->gfid, 16);
 
+        loc_copy (&local->loc, args->loc);
+        loc_path (&local->loc, NULL);
         GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
                                        !uuid_is_null (*((uuid_t*)req.gfid)),
                                        unwind, op_errno, EINVAL);
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
+        if (args->xattr) {
+                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
                                             (&req.dict.dict_val),
                                             req.dict.dict_len,
                                             op_errno, unwind);
@@ -4228,41 +4980,42 @@ client3_1_xattrop (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_XATTROP,
-                                     client3_1_xattrop_cbk, NULL,
+                                     client3_3_xattrop_cbk, NULL,
                                      rsphdr, count,
                                      NULL, 0, local->iobref,
                                      (xdrproc_t)xdr_gfs3_xattrop_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
-        return 0;
-unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
+        GF_FREE (req.dict.dict_val);
 
-        STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
+
+        if (rsp_iobref)
+                iobref_unref (rsp_iobref);
+
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
 
-        client_local_wipe (local);
+        GF_FREE (req.dict.dict_val);
 
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 }
@@ -4270,17 +5023,17 @@ unwind:
 
 
 int32_t
-client3_1_fxattrop (call_frame_t *frame, xlator_t *this,
+client3_3_fxattrop (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_args_t       *args       = NULL;
         int64_t            remote_fd  = -1;
         clnt_conf_t       *conf       = NULL;
+        clnt_local_t      *local      = NULL;
         gfs3_fxattrop_req  req        = {{0,},};
         int                op_errno   = ESTALE;
         int                ret        = 0;
         int                count      = 0;
-        clnt_local_t    *local      = NULL;
         struct iobref     *rsp_iobref = NULL;
         struct iobuf      *rsp_iobuf  = NULL;
         struct iovec      *rsphdr     = NULL;
@@ -4292,28 +5045,28 @@ client3_1_fxattrop (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
+
+        local = frame->local;
 
         req.fd     = remote_fd;
         req.flags  = args->flags;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_client_mt_clnt_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
         rsp_iobref = iobref_new ();
         if (rsp_iobref == NULL) {
                 op_errno = ENOMEM;
                 goto unwind;
         }
 
-                        /* TODO: what is the size we should send ? */
-        rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
+        /* TODO: what is the size we should send ? */
+        rsp_iobuf = iobuf_get2 (this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
         if (rsp_iobuf == NULL) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -4325,61 +5078,54 @@ client3_1_fxattrop (call_frame_t *frame, xlator_t *this,
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
         rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
         count = 1;
-        rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
+        rsp_iobuf = NULL;
         rsp_iobref = NULL;
 
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
+        if (args->xattr) {
+                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
                                             (&req.dict.dict_val),
                                             req.dict.dict_len,
                                             op_errno, unwind);
         }
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FXATTROP,
-                                     client3_1_fxattrop_cbk, NULL,
+                                     client3_3_fxattrop_cbk, NULL,
                                      rsphdr, count,
                                      NULL, 0, local->iobref,
                                      (xdrproc_t)xdr_gfs3_fxattrop_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.dict.dict_val);
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
-
-        STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL);
-
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        CLIENT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
 
-        client_local_wipe (local);
+        GF_FREE (req.dict.dict_val);
 
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 }
 
 
-
 int32_t
-client3_1_removexattr (call_frame_t *frame, xlator_t *this,
+client3_3_removexattr (call_frame_t *frame, xlator_t *this,
                        void *data)
 {
         clnt_conf_t          *conf     = NULL;
@@ -4408,25 +5154,30 @@ client3_1_removexattr (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_REMOVEXATTR,
-                                     client3_1_removexattr_cbk, NULL,
+                                     client3_3_removexattr_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_removexattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 int32_t
-client3_1_fremovexattr (call_frame_t *frame, xlator_t *this,
+client3_3_fremovexattr (call_frame_t *frame, xlator_t *this,
                         void *data)
 {
         clnt_conf_t           *conf     = NULL;
@@ -4446,32 +5197,37 @@ client3_1_fremovexattr (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         memcpy (req.gfid,  args->fd->inode->gfid, 16);
         req.name = (char *)args->name;
         req.fd = remote_fd;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FREMOVEXATTR,
-                                     client3_1_fremovexattr_cbk, NULL,
+                                     client3_3_fremovexattr_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_fremovexattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
-
 int32_t
-client3_1_lk (call_frame_t *frame, xlator_t *this,
+client3_3_lk (call_frame_t *frame, xlator_t *this,
               void *data)
 {
         clnt_args_t     *args       = NULL;
@@ -4489,13 +5245,14 @@ client3_1_lk (call_frame_t *frame, xlator_t *this,
 
         args = data;
         conf = this->private;
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
         }
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         ret = client_cmd_to_gf_cmd (args->cmd, &gf_cmd);
         if (ret) {
@@ -4526,27 +5283,33 @@ client3_1_lk (call_frame_t *frame, xlator_t *this,
         req.cmd   = gf_cmd;
         req.type  = gf_type;
         gf_proto_flock_from_flock (&req.flock, args->flock);
+
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops, GFS3_OP_LK,
-                                     client3_1_lk_cbk, NULL,
+                                     client3_3_lk_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_lk_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_inodelk (call_frame_t *frame, xlator_t *this,
+client3_3_inodelk (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_conf_t      *conf     = NULL;
@@ -4604,27 +5367,32 @@ client3_1_inodelk (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_INODELK,
-                                     client3_1_inodelk_cbk, NULL,
+                                     client3_3_inodelk_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_inodelk_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_finodelk (call_frame_t *frame, xlator_t *this,
+client3_3_finodelk (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_args_t       *args     = NULL;
@@ -4634,15 +5402,20 @@ client3_1_finodelk (call_frame_t *frame, xlator_t *this,
         int64_t            remote_fd = -1;
         clnt_conf_t       *conf     = NULL;
         int                op_errno = ESTALE;
-        int           ret        = 0;
+        int                ret      = 0;
 
         if (!frame || !this || !data)
                 goto unwind;
 
         args = data;
         conf = this->private;
-
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
 
         if (args->cmd == F_GETLK || args->cmd == F_GETLK64)
                 gf_cmd = GF_LK_GETLK;
@@ -4675,26 +5448,30 @@ client3_1_finodelk (call_frame_t *frame, xlator_t *this,
         gf_proto_flock_from_flock (&req.flock, args->flock);
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FINODELK,
-                                     client3_1_finodelk_cbk, NULL,
+                                     client3_3_finodelk_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_finodelk_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_entrylk (call_frame_t *frame, xlator_t *this,
+client3_3_entrylk (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_conf_t      *conf     = NULL;
@@ -4730,27 +5507,32 @@ client3_1_entrylk (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_ENTRYLK,
-                                     client3_1_entrylk_cbk, NULL,
+                                     client3_3_entrylk_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_entrylk_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_fentrylk (call_frame_t *frame, xlator_t *this,
+client3_3_fentrylk (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_args_t       *args     = NULL;
@@ -4766,7 +5548,8 @@ client3_1_fentrylk (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd  = remote_fd;
         req.cmd = args->cmd_entrylk;
@@ -4779,26 +5562,31 @@ client3_1_fentrylk (call_frame_t *frame, xlator_t *this,
         }
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FENTRYLK,
-                                     client3_1_fentrylk_cbk, NULL,
+                                     client3_3_fentrylk_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_fentrylk_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_rchecksum (call_frame_t *frame, xlator_t *this,
+client3_3_rchecksum (call_frame_t *frame, xlator_t *this,
                      void *data)
 {
         clnt_args_t        *args     = NULL;
@@ -4814,33 +5602,40 @@ client3_1_rchecksum (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.len    = args->len;
         req.offset = args->offset;
         req.fd     = remote_fd;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_RCHECKSUM,
-                                     client3_1_rchecksum_cbk, NULL,
+                                     client3_3_rchecksum_cbk, NULL,
                                      NULL, 0, NULL,
-                                     0, NULL, (xdrproc_t)xdr_gfs3_rchecksum_req);
+                                     0, NULL,
+                                     (xdrproc_t)xdr_gfs3_rchecksum_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (rchecksum, frame, -1, op_errno, 0, NULL);
+        CLIENT_STACK_UNWIND (rchecksum, frame, -1, op_errno, 0, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_readdir (call_frame_t *frame, xlator_t *this,
+client3_3_readdir (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_args_t      *args       = NULL;
@@ -4864,15 +5659,15 @@ client3_1_readdir (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         readdir_rsp_size = xdr_sizeof ((xdrproc_t) xdr_gfs3_readdir_rsp, &rsp)
                 + args->size;
 
         if ((readdir_rsp_size + GLUSTERFS_RPC_REPLY_SIZE + GLUSTERFS_RDMA_MAX_HEADER_SIZE)
             > (GLUSTERFS_RDMA_INLINE_THRESHOLD)) {
-                local = GF_CALLOC (1, sizeof (*local),
-                                   gf_client_mt_clnt_local_t);
+                local = mem_get0 (this->local_pool);
                 if (!local) {
                         op_errno = ENOMEM;
                         goto unwind;
@@ -4884,7 +5679,9 @@ client3_1_readdir (call_frame_t *frame, xlator_t *this,
                         goto unwind;
                 }
 
-                        /* TODO: what is the size we should send ? */
+                /* TODO: what is the size we should send ? */
+                /* This iobuf will live for only receiving the response,
+                   so not harmful */
                 rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
                 if (rsp_iobuf == NULL) {
                         goto unwind;
@@ -4892,56 +5689,63 @@ client3_1_readdir (call_frame_t *frame, xlator_t *this,
 
                 iobref_add (rsp_iobref, rsp_iobuf);
                 iobuf_unref (rsp_iobuf);
+
                 rsphdr = &vector[0];
                 rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-                rsphdr->iov_len
-                        = iobuf_pagesize (rsp_iobuf);
+                rsphdr->iov_len  = iobuf_pagesize (rsp_iobuf);
                 count = 1;
-                rsp_iobuf = NULL;
                 local->iobref = rsp_iobref;
+                rsp_iobuf = NULL;
+                rsp_iobref = NULL;
         }
 
         req.size = args->size;
         req.offset = args->offset;
         req.fd = remote_fd;
+
+        local->cmd = remote_fd;
+
         memcpy (req.gfid, args->fd->inode->gfid, 16);
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
 
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_READDIR,
-                                     client3_1_readdir_cbk, NULL,
+                                     client3_3_readdir_cbk, NULL,
                                      rsphdr, count,
                                      NULL, 0, rsp_iobref,
                                      (xdrproc_t)xdr_gfs3_readdir_req);
-        rsp_iobref = NULL;
 
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
+
+        if (rsp_iobref)
+                iobref_unref (rsp_iobref);
+
         return 0;
 
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
-        client_local_wipe (local);
-
-        if (rsp_iobref != NULL) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        if (rsp_iobuf != NULL) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_readdirp (call_frame_t *frame, xlator_t *this,
+client3_3_readdirp (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_args_t      *args              = NULL;
@@ -4965,13 +5769,13 @@ client3_1_readdirp (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         readdirp_rsp_size = xdr_sizeof ((xdrproc_t) xdr_gfs3_readdirp_rsp, &rsp)
                 + args->size;
 
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_client_mt_clnt_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -4987,6 +5791,8 @@ client3_1_readdirp (call_frame_t *frame, xlator_t *this,
                 }
 
                 /* TODO: what is the size we should send ? */
+                /* This iobuf will live for only receiving the response,
+                   so not harmful */
                 rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
                 if (rsp_iobuf == NULL) {
                         goto unwind;
@@ -4994,12 +5800,13 @@ client3_1_readdirp (call_frame_t *frame, xlator_t *this,
 
                 iobref_add (rsp_iobref, rsp_iobuf);
                 iobuf_unref (rsp_iobuf);
+
                 rsphdr = &vector[0];
                 rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
                 rsphdr->iov_len  = iobuf_pagesize (rsp_iobuf);
                 count = 1;
-                rsp_iobuf = NULL;
                 local->iobref = rsp_iobref;
+                rsp_iobuf = NULL;
                 rsp_iobref = NULL;
         }
 
@@ -5010,46 +5817,45 @@ client3_1_readdirp (call_frame_t *frame, xlator_t *this,
         req.fd = remote_fd;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
+        /* dict itself is 'xdata' here */
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.dict.dict_val),
+                                    req.dict.dict_len, op_errno, unwind);
 
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_READDIRP,
-                                     client3_1_readdirp_cbk, NULL,
+                                     client3_3_readdirp_cbk, NULL,
                                      rsphdr, count, NULL,
                                      0, rsp_iobref,
                                      (xdrproc_t)xdr_gfs3_readdirp_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.dict.dict_val);
+
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
+
+        if (rsp_iobref)
+                iobref_unref (rsp_iobref);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
-        client_local_wipe (local);
-
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL);
+        GF_FREE (req.dict.dict_val);
+
+        CLIENT_STACK_UNWIND (readdirp, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
-client3_1_setattr (call_frame_t *frame, xlator_t *this,
+client3_3_setattr (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_conf_t      *conf     = NULL;
@@ -5079,25 +5885,30 @@ client3_1_setattr (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_SETATTR,
-                                     client3_1_setattr_cbk, NULL,
+                                     client3_3_setattr_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_setattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 int32_t
-client3_1_fsetattr (call_frame_t *frame, xlator_t *this, void *data)
+client3_3_fsetattr (call_frame_t *frame, xlator_t *this, void *data)
 {
         clnt_args_t       *args     = NULL;
         int64_t            remote_fd = -1;
@@ -5112,84 +5923,226 @@ client3_1_fsetattr (call_frame_t *frame, xlator_t *this, void *data)
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd = remote_fd;
         req.valid = args->valid;
         gf_stat_from_iatt (&req.stbuf, args->stbuf);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FSETATTR,
-                                     client3_1_fsetattr_cbk, NULL,
+                                     client3_3_fsetattr_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_fsetattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+        }
+
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+}
+
+int32_t
+client3_3_fallocate(call_frame_t *frame, xlator_t *this, void *data)
+{
+        clnt_args_t       *args     = NULL;
+        int64_t            remote_fd = -1;
+        clnt_conf_t       *conf     = NULL;
+	gfs3_fallocate_req req	    = {{0},};
+        int                op_errno = ESTALE;
+        int                ret        = 0;
+
+        if (!frame || !this || !data)
                 goto unwind;
+
+        args = data;
+        conf = this->private;
+
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
+
+        req.fd = remote_fd;
+	req.flags = args->flags;
+	req.offset = args->offset;
+	req.size = args->size;
+	memcpy(req.gfid, args->fd->inode->gfid, 16);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        ret = client_submit_request (this, &req, frame, conf->fops,
+                                     GFS3_OP_FALLOCATE,
+                                     client3_3_fallocate_cbk, NULL,
+                                     NULL, 0, NULL, 0,
+                                     NULL, (xdrproc_t)xdr_gfs3_fallocate_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
+int32_t
+client3_3_discard(call_frame_t *frame, xlator_t *this, void *data)
+{
+        clnt_args_t       *args     = NULL;
+        int64_t            remote_fd = -1;
+        clnt_conf_t       *conf     = NULL;
+	gfs3_discard_req   req	    = {{0},};
+        int                op_errno = ESTALE;
+        int                ret        = 0;
+
+        if (!frame || !this || !data)
+                goto unwind;
 
+        args = data;
+        conf = this->private;
+
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
+
+        req.fd = remote_fd;
+	req.offset = args->offset;
+	req.size = args->size;
+	memcpy(req.gfid, args->fd->inode->gfid, 16);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        ret = client_submit_request(this, &req, frame, conf->fops,
+                                    GFS3_OP_DISCARD, client3_3_discard_cbk,
+				    NULL, NULL, 0, NULL, 0, NULL,
+                                    (xdrproc_t) xdr_gfs3_discard_req);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+}
+
+int32_t
+client3_3_zerofill(call_frame_t *frame, xlator_t *this, void *data)
+{
+        clnt_args_t       *args        = NULL;
+        int64_t            remote_fd   = -1;
+        clnt_conf_t       *conf        = NULL;
+        gfs3_zerofill_req   req        = {{0},};
+        int                op_errno    = ESTALE;
+        int                ret         = 0;
+
+        if (!frame || !this || !data)
+                goto unwind;
+
+        args = data;
+        conf = this->private;
+
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
+
+        req.fd = remote_fd;
+        req.offset = args->offset;
+        req.size = args->size;
+        memcpy(req.gfid, args->fd->inode->gfid, 16);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        ret = client_submit_request(this, &req, frame, conf->fops,
+                                    GFS3_OP_ZEROFILL, client3_3_zerofill_cbk,
+                                    NULL, NULL, 0, NULL, 0, NULL,
+                                    (xdrproc_t) xdr_gfs3_zerofill_req);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+}
 
 /* Table Specific to FOPS */
 
 
-rpc_clnt_procedure_t clnt3_1_fop_actors[GF_FOP_MAXVALUE] = {
+rpc_clnt_procedure_t clnt3_3_fop_actors[GF_FOP_MAXVALUE] = {
         [GF_FOP_NULL]        = { "NULL",        NULL},
-        [GF_FOP_STAT]        = { "STAT",        client3_1_stat },
-        [GF_FOP_READLINK]    = { "READLINK",    client3_1_readlink },
-        [GF_FOP_MKNOD]       = { "MKNOD",       client3_1_mknod },
-        [GF_FOP_MKDIR]       = { "MKDIR",       client3_1_mkdir },
-        [GF_FOP_UNLINK]      = { "UNLINK",      client3_1_unlink },
-        [GF_FOP_RMDIR]       = { "RMDIR",       client3_1_rmdir },
-        [GF_FOP_SYMLINK]     = { "SYMLINK",     client3_1_symlink },
-        [GF_FOP_RENAME]      = { "RENAME",      client3_1_rename },
-        [GF_FOP_LINK]        = { "LINK",        client3_1_link },
-        [GF_FOP_TRUNCATE]    = { "TRUNCATE",    client3_1_truncate },
-        [GF_FOP_OPEN]        = { "OPEN",        client3_1_open },
-        [GF_FOP_READ]        = { "READ",        client3_1_readv },
-        [GF_FOP_WRITE]       = { "WRITE",       client3_1_writev },
-        [GF_FOP_STATFS]      = { "STATFS",      client3_1_statfs },
-        [GF_FOP_FLUSH]       = { "FLUSH",       client3_1_flush },
-        [GF_FOP_FSYNC]       = { "FSYNC",       client3_1_fsync },
-        [GF_FOP_SETXATTR]    = { "SETXATTR",    client3_1_setxattr },
-        [GF_FOP_GETXATTR]    = { "GETXATTR",    client3_1_getxattr },
-        [GF_FOP_REMOVEXATTR] = { "REMOVEXATTR", client3_1_removexattr },
-        [GF_FOP_OPENDIR]     = { "OPENDIR",     client3_1_opendir },
-        [GF_FOP_FSYNCDIR]    = { "FSYNCDIR",    client3_1_fsyncdir },
-        [GF_FOP_ACCESS]      = { "ACCESS",      client3_1_access },
-        [GF_FOP_CREATE]      = { "CREATE",      client3_1_create },
-        [GF_FOP_FTRUNCATE]   = { "FTRUNCATE",   client3_1_ftruncate },
-        [GF_FOP_FSTAT]       = { "FSTAT",       client3_1_fstat },
-        [GF_FOP_LK]          = { "LK",          client3_1_lk },
-        [GF_FOP_LOOKUP]      = { "LOOKUP",      client3_1_lookup },
-        [GF_FOP_READDIR]     = { "READDIR",     client3_1_readdir },
-        [GF_FOP_INODELK]     = { "INODELK",     client3_1_inodelk },
-        [GF_FOP_FINODELK]    = { "FINODELK",    client3_1_finodelk },
-        [GF_FOP_ENTRYLK]     = { "ENTRYLK",     client3_1_entrylk },
-        [GF_FOP_FENTRYLK]    = { "FENTRYLK",    client3_1_fentrylk },
-        [GF_FOP_XATTROP]     = { "XATTROP",     client3_1_xattrop },
-        [GF_FOP_FXATTROP]    = { "FXATTROP",    client3_1_fxattrop },
-        [GF_FOP_FGETXATTR]   = { "FGETXATTR",   client3_1_fgetxattr },
-        [GF_FOP_FSETXATTR]   = { "FSETXATTR",   client3_1_fsetxattr },
-        [GF_FOP_RCHECKSUM]   = { "RCHECKSUM",   client3_1_rchecksum },
-        [GF_FOP_SETATTR]     = { "SETATTR",     client3_1_setattr },
-        [GF_FOP_FSETATTR]    = { "FSETATTR",    client3_1_fsetattr },
-        [GF_FOP_READDIRP]    = { "READDIRP",    client3_1_readdirp },
-        [GF_FOP_RELEASE]     = { "RELEASE",     client3_1_release },
-        [GF_FOP_RELEASEDIR]  = { "RELEASEDIR",  client3_1_releasedir },
+        [GF_FOP_STAT]        = { "STAT",        client3_3_stat },
+        [GF_FOP_READLINK]    = { "READLINK",    client3_3_readlink },
+        [GF_FOP_MKNOD]       = { "MKNOD",       client3_3_mknod },
+        [GF_FOP_MKDIR]       = { "MKDIR",       client3_3_mkdir },
+        [GF_FOP_UNLINK]      = { "UNLINK",      client3_3_unlink },
+        [GF_FOP_RMDIR]       = { "RMDIR",       client3_3_rmdir },
+        [GF_FOP_SYMLINK]     = { "SYMLINK",     client3_3_symlink },
+        [GF_FOP_RENAME]      = { "RENAME",      client3_3_rename },
+        [GF_FOP_LINK]        = { "LINK",        client3_3_link },
+        [GF_FOP_TRUNCATE]    = { "TRUNCATE",    client3_3_truncate },
+        [GF_FOP_OPEN]        = { "OPEN",        client3_3_open },
+        [GF_FOP_READ]        = { "READ",        client3_3_readv },
+        [GF_FOP_WRITE]       = { "WRITE",       client3_3_writev },
+        [GF_FOP_STATFS]      = { "STATFS",      client3_3_statfs },
+        [GF_FOP_FLUSH]       = { "FLUSH",       client3_3_flush },
+        [GF_FOP_FSYNC]       = { "FSYNC",       client3_3_fsync },
+        [GF_FOP_SETXATTR]    = { "SETXATTR",    client3_3_setxattr },
+        [GF_FOP_GETXATTR]    = { "GETXATTR",    client3_3_getxattr },
+        [GF_FOP_REMOVEXATTR] = { "REMOVEXATTR", client3_3_removexattr },
+        [GF_FOP_OPENDIR]     = { "OPENDIR",     client3_3_opendir },
+        [GF_FOP_FSYNCDIR]    = { "FSYNCDIR",    client3_3_fsyncdir },
+        [GF_FOP_ACCESS]      = { "ACCESS",      client3_3_access },
+        [GF_FOP_CREATE]      = { "CREATE",      client3_3_create },
+        [GF_FOP_FTRUNCATE]   = { "FTRUNCATE",   client3_3_ftruncate },
+        [GF_FOP_FSTAT]       = { "FSTAT",       client3_3_fstat },
+        [GF_FOP_LK]          = { "LK",          client3_3_lk },
+        [GF_FOP_LOOKUP]      = { "LOOKUP",      client3_3_lookup },
+        [GF_FOP_READDIR]     = { "READDIR",     client3_3_readdir },
+        [GF_FOP_INODELK]     = { "INODELK",     client3_3_inodelk },
+        [GF_FOP_FINODELK]    = { "FINODELK",    client3_3_finodelk },
+        [GF_FOP_ENTRYLK]     = { "ENTRYLK",     client3_3_entrylk },
+        [GF_FOP_FENTRYLK]    = { "FENTRYLK",    client3_3_fentrylk },
+        [GF_FOP_XATTROP]     = { "XATTROP",     client3_3_xattrop },
+        [GF_FOP_FXATTROP]    = { "FXATTROP",    client3_3_fxattrop },
+        [GF_FOP_FGETXATTR]   = { "FGETXATTR",   client3_3_fgetxattr },
+        [GF_FOP_FSETXATTR]   = { "FSETXATTR",   client3_3_fsetxattr },
+        [GF_FOP_RCHECKSUM]   = { "RCHECKSUM",   client3_3_rchecksum },
+        [GF_FOP_SETATTR]     = { "SETATTR",     client3_3_setattr },
+        [GF_FOP_FSETATTR]    = { "FSETATTR",    client3_3_fsetattr },
+        [GF_FOP_READDIRP]    = { "READDIRP",    client3_3_readdirp },
+	[GF_FOP_FALLOCATE]   = { "FALLOCATE",	client3_3_fallocate },
+	[GF_FOP_DISCARD]     = { "DISCARD",	client3_3_discard },
+        [GF_FOP_ZEROFILL]    = { "ZEROFILL",    client3_3_zerofill},
+        [GF_FOP_RELEASE]     = { "RELEASE",     client3_3_release },
+        [GF_FOP_RELEASEDIR]  = { "RELEASEDIR",  client3_3_releasedir },
         [GF_FOP_GETSPEC]     = { "GETSPEC",     client3_getspec },
-        [GF_FOP_FREMOVEXATTR] = { "FREMOVEXATTR", client3_1_fremovexattr },
+        [GF_FOP_FREMOVEXATTR] = { "FREMOVEXATTR", client3_3_fremovexattr },
 };
 
 /* Used From RPC-CLNT library to log proper name of procedure based on number */
-char *clnt3_1_fop_names[GFS3_OP_MAXVALUE] = {
+char *clnt3_3_fop_names[GFS3_OP_MAXVALUE] = {
         [GFS3_OP_NULL]        = "NULL",
         [GFS3_OP_STAT]        = "STAT",
         [GFS3_OP_READLINK]    = "READLINK",
@@ -5234,13 +6187,17 @@ char *clnt3_1_fop_names[GFS3_OP_MAXVALUE] = {
         [GFS3_OP_RELEASE]     = "RELEASE",
         [GFS3_OP_RELEASEDIR]  = "RELEASEDIR",
         [GFS3_OP_FREMOVEXATTR] = "FREMOVEXATTR",
+	[GFS3_OP_FALLOCATE]   = "FALLOCATE",
+	[GFS3_OP_DISCARD]     = "DISCARD",
+        [GFS3_OP_ZEROFILL]    = "ZEROFILL",
+
 };
 
-rpc_clnt_prog_t clnt3_1_fop_prog = {
-        .progname  = "GlusterFS 3.1",
-        .prognum   = GLUSTER3_1_FOP_PROGRAM,
-        .progver   = GLUSTER3_1_FOP_VERSION,
-        .numproc   = GLUSTER3_1_FOP_PROCCNT,
-        .proctable = clnt3_1_fop_actors,
-        .procnames = clnt3_1_fop_names,
+rpc_clnt_prog_t clnt3_3_fop_prog = {
+        .progname  = "GlusterFS 3.3",
+        .prognum   = GLUSTER_FOP_PROGRAM,
+        .progver   = GLUSTER_FOP_VERSION,
+        .numproc   = GLUSTER_FOP_PROCCNT,
+        .proctable = clnt3_3_fop_actors,
+        .procnames = clnt3_3_fop_names,
 };
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 229e01917..1f7d13ea4 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -40,23 +31,134 @@ int client_handshake (xlator_t *this, struct rpc_clnt *rpc);
 void client_start_ping (void *data);
 int client_init_rpc (xlator_t *this);
 int client_destroy_rpc (xlator_t *this);
+int client_mark_fd_bad (xlator_t *this);
+
+int32_t
+client_type_to_gf_type (short l_type)
+{
+        int32_t  gf_type = GF_LK_EOL;
+
+        switch (l_type) {
+        case F_RDLCK:
+                gf_type = GF_LK_F_RDLCK;
+                break;
+        case F_WRLCK:
+                gf_type = GF_LK_F_WRLCK;
+                break;
+        case F_UNLCK:
+                gf_type = GF_LK_F_UNLCK;
+                break;
+        }
+
+        return gf_type;
+}
+
+uint32_t
+client_get_lk_ver (clnt_conf_t *conf)
+{
+        uint32_t  lk_ver = 0;
+
+        GF_VALIDATE_OR_GOTO ("client", conf, out);
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                lk_ver = conf->lk_version;
+        }
+        pthread_mutex_unlock (&conf->lock);
+out:
+        return lk_ver;
+}
+
+void
+client_grace_timeout (void *data)
+{
+        int               ver  = 0;
+        xlator_t         *this = NULL;
+        struct clnt_conf *conf = NULL;
+        struct rpc_clnt  *rpc  = NULL;
+
+        GF_VALIDATE_OR_GOTO ("client", data, out);
+
+        this = THIS;
+
+        rpc = (struct rpc_clnt *) data;
+
+        conf = (struct clnt_conf *) this->private;
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                ver = ++conf->lk_version;
+                /* ver == 0 is a special value used by server
+                   to notify client that this is a fresh connect.*/
+                if (ver == 0)
+                        ver = ++conf->lk_version;
+
+                gf_timer_call_cancel (this->ctx, conf->grace_timer);
+                conf->grace_timer = NULL;
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        gf_log (this->name, GF_LOG_WARNING,
+                "client grace timer expired, updating "
+                "the lk-version to %d", ver);
+
+        client_mark_fd_bad (this);
+out:
+        return;
+}
+
+int32_t
+client_register_grace_timer (xlator_t *this, clnt_conf_t *conf)
+{
+        int32_t  ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                if (conf->grace_timer || !conf->grace_timer_needed) {
+                        gf_log (this->name, GF_LOG_TRACE,
+				"Client grace timer is already set "
+				"or a grace-timer has already time out, "
+				"not registering a new timer");
+                } else {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "Registering a grace timer");
+
+                        conf->grace_timer_needed = _gf_false;
+
+                        conf->grace_timer =
+                                gf_timer_call_after (this->ctx,
+                                                     conf->grace_ts,
+                                                     client_grace_timeout,
+                                                     conf->rpc);
+                }
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
 
 int
 client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
-                       rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbk,
+                       rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
                        struct iobref *iobref,  struct iovec *rsphdr,
                        int rsphdr_count, struct iovec *rsp_payload,
                        int rsp_payload_count, struct iobref *rsp_iobref,
                        xdrproc_t xdrproc)
 {
-        int            ret         = -1;
-        clnt_conf_t   *conf        = NULL;
-        struct iovec   iov         = {0, };
-        struct iobuf  *iobuf       = NULL;
-        int            count       = 0;
-        char           start_ping  = 0;
-        struct iobref *new_iobref  = NULL;
-        ssize_t        xdr_size    = 0;
+        int             ret        = -1;
+        clnt_conf_t    *conf       = NULL;
+        struct iovec    iov        = {0, };
+        struct iobuf   *iobuf      = NULL;
+        int             count      = 0;
+        char            start_ping = 0;
+        struct iobref  *new_iobref = NULL;
+        ssize_t         xdr_size   = 0;
+        struct rpc_req  rpcreq     = {0, };
 
         GF_VALIDATE_OR_GOTO ("client", this, out);
         GF_VALIDATE_OR_GOTO (this->name, prog, out);
@@ -124,8 +226,8 @@ client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
         }
 
         /* Send the msg */
-        ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbk, &iov, count, NULL,
-                               0, new_iobref, frame, rsphdr, rsphdr_count,
+        ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbkfn, &iov, count,
+                               NULL, 0, new_iobref, frame, rsphdr, rsphdr_count,
                                rsp_payload, rsp_payload_count, rsp_iobref);
 
         if (ret < 0) {
@@ -146,14 +248,27 @@ client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
                 client_start_ping ((void *) this);
 
         ret = 0;
-out:
-        if (new_iobref != NULL)
+
+        if (new_iobref)
                 iobref_unref (new_iobref);
 
         if (iobuf)
                 iobuf_unref (iobuf);
 
         return ret;
+
+out:
+        rpcreq.rpc_status = -1;
+
+        cbkfn (&rpcreq, NULL, 0, frame);
+
+        if (new_iobref)
+                iobref_unref (new_iobref);
+
+        if (iobuf)
+                iobuf_unref (iobuf);
+
+        return 0;
 }
 
 
@@ -171,7 +286,6 @@ client_releasedir (xlator_t *this, fd_t *fd)
         clnt_conf_t *conf = NULL;
         rpc_clnt_procedure_t *proc = NULL;
         clnt_args_t  args = {0,};
-        call_frame_t *frame = NULL;
 
         conf = this->private;
         if (!conf || !conf->fops)
@@ -187,11 +301,7 @@ client_releasedir (xlator_t *this, fd_t *fd)
                 goto out;
         }
         if (proc->fn) {
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        goto out;
-                }
-                ret = proc->fn (frame, this, &args);
+                ret = proc->fn (NULL, this, &args);
         }
 out:
         if (ret)
@@ -207,7 +317,6 @@ client_release (xlator_t *this, fd_t *fd)
         clnt_conf_t *conf = NULL;
         rpc_clnt_procedure_t *proc = NULL;
         clnt_args_t  args = {0,};
-        call_frame_t *frame = NULL;
 
         conf = this->private;
         if (!conf || !conf->fops)
@@ -222,11 +331,7 @@ client_release (xlator_t *this, fd_t *fd)
                 goto out;
         }
         if (proc->fn) {
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        goto out;
-                }
-                ret = proc->fn (frame, this, &args);
+                ret = proc->fn (NULL, this, &args);
         }
 out:
         if (ret)
@@ -238,7 +343,7 @@ out:
 
 int32_t
 client_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xattr_req)
+               dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -250,7 +355,7 @@ client_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
 
         args.loc = loc;
-        args.dict = xattr_req;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_LOOKUP];
         if (!proc) {
@@ -272,7 +377,7 @@ out:
 
 
 int32_t
-client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -284,6 +389,7 @@ client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto out;
 
         args.loc = loc;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_STAT];
         if (!proc) {
@@ -296,14 +402,15 @@ client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (stat, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (stat, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
 
 
 int32_t
-client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 off_t offset, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -316,6 +423,7 @@ client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
 
         args.loc    = loc;
         args.offset = offset;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_TRUNCATE];
         if (!proc) {
@@ -328,7 +436,7 @@ client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (truncate, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (truncate, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 
 	return 0;
@@ -336,7 +444,8 @@ out:
 
 
 int32_t
-client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  off_t offset, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -349,6 +458,7 @@ client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
 
         args.fd     = fd;
         args.offset = offset;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FTRUNCATE];
         if (!proc) {
@@ -361,7 +471,7 @@ client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -369,7 +479,8 @@ out:
 
 
 int32_t
-client_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+client_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int32_t mask, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -382,6 +493,7 @@ client_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
 
         args.loc  = loc;
         args.mask = mask;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_ACCESS];
         if (!proc) {
@@ -394,7 +506,7 @@ client_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (access, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (access, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -403,7 +515,8 @@ out:
 
 
 int32_t
-client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 size_t size, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -416,6 +529,7 @@ client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
 
         args.loc  = loc;
         args.size = size;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_READLINK];
         if (!proc) {
@@ -428,7 +542,7 @@ client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (readlink, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (readlink, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -436,7 +550,7 @@ out:
 
 int
 client_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, dict_t *params)
+              dev_t rdev, mode_t umask, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -450,7 +564,8 @@ client_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         args.loc  = loc;
         args.mode = mode;
         args.rdev = rdev;
-        args.dict = params;
+        args.umask = umask;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_MKNOD];
         if (!proc) {
@@ -464,7 +579,7 @@ client_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (mknod, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -472,7 +587,7 @@ out:
 
 int
 client_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              mode_t mode, dict_t *params)
+              mode_t mode, mode_t umask, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -485,7 +600,8 @@ client_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         args.loc  = loc;
         args.mode = mode;
-        args.dict = params;
+        args.umask = umask;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_MKDIR];
         if (!proc) {
@@ -499,7 +615,7 @@ client_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (mkdir, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -507,7 +623,8 @@ out:
 
 
 int32_t
-client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int xflag, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -519,6 +636,8 @@ client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto out;
 
         args.loc = loc;
+        args.xdata = xdata;
+        args.flags = xflag;
 
         proc = &conf->fops->proctable[GF_FOP_UNLINK];
         if (!proc) {
@@ -532,13 +651,14 @@ client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
 out:
         if (ret)
                 STACK_UNWIND_STRICT (unlink, frame, -1, ENOTCONN,
-                                     NULL, NULL);
+                                     NULL, NULL, NULL);
 
 	return 0;
 }
 
 int32_t
-client_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+client_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -551,6 +671,7 @@ client_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 
         args.loc = loc;
         args.flags = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_RMDIR];
         if (!proc) {
@@ -565,7 +686,7 @@ out:
         /* think of avoiding a missing frame */
         if (ret)
                 STACK_UNWIND_STRICT (rmdir, frame, -1, ENOTCONN,
-                                     NULL, NULL);
+                                     NULL, NULL, NULL);
 
 	return 0;
 }
@@ -573,7 +694,7 @@ out:
 
 int
 client_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                loc_t *loc, dict_t *params)
+                loc_t *loc, mode_t umask, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -586,7 +707,8 @@ client_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
 
         args.linkname = linkpath;
         args.loc      = loc;
-        args.dict     = params;
+        args.umask    = umask;
+        args.xdata    = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_SYMLINK];
         if (!proc) {
@@ -600,7 +722,7 @@ client_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (symlink, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -609,7 +731,7 @@ out:
 
 int32_t
 client_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc)
+               loc_t *newloc, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -622,6 +744,8 @@ client_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 
         args.oldloc = oldloc;
         args.newloc = newloc;
+        args.xdata  = xdata;
+
         proc = &conf->fops->proctable[GF_FOP_RENAME];
         if (!proc) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -634,7 +758,7 @@ client_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (rename, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -643,7 +767,7 @@ out:
 
 int32_t
 client_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-             loc_t *newloc)
+             loc_t *newloc, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -656,6 +780,7 @@ client_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 
         args.oldloc = oldloc;
         args.newloc = newloc;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_LINK];
         if (!proc) {
@@ -669,7 +794,7 @@ client_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (link, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -677,8 +802,8 @@ out:
 
 
 int32_t
-client_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+client_create (call_frame_t *frame, xlator_t *this, loc_t *loc,  int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -690,10 +815,15 @@ client_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
 
         args.loc = loc;
-        args.flags = flags;
         args.mode = mode;
         args.fd = fd;
-        args.dict = params;
+        args.umask = umask;
+        args.xdata = xdata;
+
+        if (!conf->filter_o_direct)
+                args.flags = flags;
+        else
+                args.flags = (flags & ~O_DIRECT);
 
         proc = &conf->fops->proctable[GF_FOP_CREATE];
         if (!proc) {
@@ -707,7 +837,7 @@ client_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (create, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -716,7 +846,7 @@ out:
 
 int32_t
 client_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int32_t flags, fd_t *fd, int32_t wbflags)
+             int32_t flags, fd_t *fd, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -728,9 +858,13 @@ client_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
 
         args.loc = loc;
-        args.flags = flags;
         args.fd = fd;
-        args.wbflags = wbflags;
+        args.xdata = xdata;
+
+        if (!conf->filter_o_direct)
+                args.flags = flags;
+        else
+                args.flags = (flags & ~O_DIRECT);
 
         proc = &conf->fops->proctable[GF_FOP_OPEN];
         if (!proc) {
@@ -744,7 +878,7 @@ client_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 out:
         if (ret)
-                STACK_UNWIND_STRICT (open, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -753,7 +887,7 @@ out:
 
 int32_t
 client_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, uint32_t flags)
+              off_t offset, uint32_t flags, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -768,6 +902,7 @@ client_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         args.size   = size;
         args.offset = offset;
         args.flags  = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_READ];
         if (!proc) {
@@ -782,7 +917,7 @@ client_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (readv, frame, -1, ENOTCONN,
-                                     NULL, 0, NULL, NULL);
+                                     NULL, 0, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -793,7 +928,7 @@ out:
 int32_t
 client_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                struct iovec *vector, int32_t count, off_t off,
-               uint32_t flags, struct iobref *iobref)
+               uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -810,6 +945,7 @@ client_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         args.offset = off;
         args.flags  = flags;
         args.iobref = iobref;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_WRITE];
         if (!proc) {
@@ -822,15 +958,14 @@ client_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (writev, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (writev, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
 
 
-
 int32_t
-client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -842,6 +977,7 @@ client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 goto out;
 
         args.fd = fd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FLUSH];
         if (!proc) {
@@ -854,7 +990,7 @@ client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (flush, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (flush, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -863,7 +999,7 @@ out:
 
 int32_t
 client_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t flags)
+              int32_t flags, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -876,6 +1012,7 @@ client_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         args.fd    = fd;
         args.flags = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FSYNC];
         if (!proc) {
@@ -888,7 +1025,7 @@ client_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fsync, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (fsync, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -896,7 +1033,7 @@ out:
 
 
 int32_t
-client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -908,6 +1045,7 @@ client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 goto out;
 
         args.fd = fd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FSTAT];
         if (!proc) {
@@ -920,7 +1058,7 @@ client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fstat, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (fstat, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -928,7 +1066,8 @@ out:
 
 
 int32_t
-client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -941,6 +1080,7 @@ client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 
         args.loc = loc;
         args.fd  = fd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_OPENDIR];
         if (!proc) {
@@ -953,7 +1093,7 @@ client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (opendir, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (opendir, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -961,7 +1101,7 @@ out:
 
 
 int32_t
-client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -974,6 +1114,7 @@ client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
 
         args.fd    = fd;
         args.flags = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FSYNCDIR];
         if (!proc) {
@@ -986,7 +1127,7 @@ client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -994,7 +1135,7 @@ out:
 
 
 int32_t
-client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1006,6 +1147,7 @@ client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto out;
 
         args.loc = loc;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_STATFS];
         if (!proc) {
@@ -1018,7 +1160,7 @@ client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (statfs, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (statfs, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1135,8 +1277,7 @@ client_set_remote_options (char *value, xlator_t *this)
 
         ret = _gf_true;
 out:
-        if (dup_value)
-                GF_FREE (dup_value);
+        GF_FREE (dup_value);
 
         return ret;
 }
@@ -1144,7 +1285,7 @@ out:
 
 int32_t
 client_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                 int32_t flags)
+                 int32_t flags, dict_t *xdata)
 {
         int                   ret         = -1;
         int                   op_ret      = -1;
@@ -1192,8 +1333,9 @@ client_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
         }
 
         args.loc   = loc;
-        args.dict  = dict;
+        args.xattr = dict;
         args.flags = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_SETXATTR];
         if (!proc) {
@@ -1210,7 +1352,7 @@ client_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
         }
 out:
         if (need_unwind)
-                STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+                STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
 
 	return 0;
 }
@@ -1219,7 +1361,7 @@ out:
 
 int32_t
 client_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  dict_t *dict, int32_t flags)
+                  dict_t *dict, int32_t flags, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1230,9 +1372,10 @@ client_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (!conf || !conf->fops)
                 goto out;
 
-        args.fd = fd;
-        args.dict = dict;
+        args.fd    = fd;
+        args.xattr = dict;
         args.flags = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FSETXATTR];
         if (!proc) {
@@ -1245,7 +1388,7 @@ client_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1255,7 +1398,7 @@ out:
 
 int32_t
 client_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  const char *name)
+                  const char *name, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1268,6 +1411,7 @@ client_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         args.fd = fd;
         args.name = name;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FGETXATTR];
         if (!proc) {
@@ -1280,7 +1424,7 @@ client_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1289,7 +1433,7 @@ out:
 
 int32_t
 client_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1302,6 +1446,7 @@ client_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         args.name = name;
         args.loc  = loc;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_GETXATTR];
         if (!proc) {
@@ -1314,7 +1459,7 @@ client_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1323,7 +1468,7 @@ out:
 
 int32_t
 client_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                gf_xattrop_flags_t flags, dict_t *dict)
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1336,7 +1481,8 @@ client_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         args.loc = loc;
         args.flags = flags;
-        args.dict = dict;
+        args.xattr = dict;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_XATTROP];
         if (!proc) {
@@ -1349,7 +1495,7 @@ client_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1358,7 +1504,7 @@ out:
 
 int32_t
 client_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 gf_xattrop_flags_t flags, dict_t *dict)
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1371,7 +1517,8 @@ client_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         args.fd = fd;
         args.flags = flags;
-        args.dict = dict;
+        args.xattr = dict;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FXATTROP];
         if (!proc) {
@@ -1384,7 +1531,7 @@ client_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1393,7 +1540,7 @@ out:
 
 int32_t
 client_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name)
+                    const char *name, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1406,6 +1553,7 @@ client_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         args.name = name;
         args.loc  = loc;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_REMOVEXATTR];
         if (!proc) {
@@ -1418,14 +1566,14 @@ client_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
 
 int32_t
 client_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     const char *name)
+                     const char *name, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1438,6 +1586,7 @@ client_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         args.name = name;
         args.fd  = fd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FREMOVEXATTR];
         if (!proc) {
@@ -1450,15 +1599,14 @@ client_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fremovexattr, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (fremovexattr, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
 
-
 int32_t
 client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *lock)
+           struct gf_flock *lock, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1472,6 +1620,7 @@ client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
         args.fd    = fd;
         args.cmd   = cmd;
         args.flock = lock;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_LK];
         if (!proc) {
@@ -1484,7 +1633,7 @@ client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (lk, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (lk, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1492,7 +1641,7 @@ out:
 
 int32_t
 client_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                loc_t *loc, int32_t cmd, struct gf_flock *lock)
+                loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1507,6 +1656,7 @@ client_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
         args.cmd    = cmd;
         args.flock  = lock;
         args.volume = volume;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_INODELK];
         if (!proc) {
@@ -1519,7 +1669,7 @@ client_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1528,7 +1678,7 @@ out:
 
 int32_t
 client_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                 fd_t *fd, int32_t cmd, struct gf_flock *lock)
+                 fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1543,6 +1693,7 @@ client_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
         args.cmd    = cmd;
         args.flock  = lock;
         args.volume = volume;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FINODELK];
         if (!proc) {
@@ -1555,7 +1706,7 @@ client_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1564,7 +1715,7 @@ out:
 int32_t
 client_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                 loc_t *loc, const char *basename, entrylk_cmd cmd,
-                entrylk_type type)
+                entrylk_type type, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1580,6 +1731,7 @@ client_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
         args.type         = type;
         args.volume       = volume;
         args.cmd_entrylk  = cmd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_ENTRYLK];
         if (!proc) {
@@ -1592,7 +1744,7 @@ client_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1602,7 +1754,7 @@ out:
 int32_t
 client_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                  fd_t *fd, const char *basename, entrylk_cmd cmd,
-                 entrylk_type type)
+                 entrylk_type type, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1618,6 +1770,7 @@ client_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
         args.type         = type;
         args.volume       = volume;
         args.cmd_entrylk  = cmd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FENTRYLK];
         if (!proc) {
@@ -1630,7 +1783,7 @@ client_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1638,7 +1791,7 @@ out:
 
 int32_t
 client_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                  int32_t len)
+                  int32_t len, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1652,6 +1805,7 @@ client_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         args.fd = fd;
         args.offset = offset;
         args.len = len;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_RCHECKSUM];
         if (!proc) {
@@ -1664,14 +1818,14 @@ client_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOTCONN, 0, NULL);
+                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOTCONN, 0, NULL, NULL);
 
 	return 0;
 }
 
 int32_t
 client_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                size_t size, off_t off)
+                size_t size, off_t off, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1685,6 +1839,7 @@ client_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
         args.fd = fd;
         args.size = size;
         args.offset = off;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_READDIR];
         if (!proc) {
@@ -1697,7 +1852,7 @@ client_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (readdir, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1719,7 +1874,7 @@ client_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
         args.fd = fd;
         args.size = size;
         args.offset = off;
-        args.xattr_req = dict;
+        args.xdata = dict;
 
         proc = &conf->fops->proctable[GF_FOP_READDIRP];
         if (!proc) {
@@ -1732,7 +1887,7 @@ client_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1740,7 +1895,7 @@ out:
 
 int32_t
 client_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1754,6 +1909,7 @@ client_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         args.loc = loc;
         args.stbuf = stbuf;
         args.valid = valid;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_SETATTR];
         if (!proc) {
@@ -1766,14 +1922,14 @@ client_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (setattr, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (setattr, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
 
 int32_t
 client_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid)
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1787,6 +1943,7 @@ client_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         args.fd = fd;
         args.stbuf = stbuf;
         args.valid = valid;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FSETATTR];
         if (!proc) {
@@ -1799,11 +1956,115 @@ client_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOTCONN, NULL, NULL, NULL);
+
+	return 0;
+}
+
+int32_t
+client_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		 off_t offset, size_t len, dict_t *xdata)
+{
+        int          ret  = -1;
+        clnt_conf_t *conf = NULL;
+        rpc_clnt_procedure_t *proc = NULL;
+        clnt_args_t  args = {0,};
+
+        conf = this->private;
+        if (!conf || !conf->fops)
+                goto out;
+
+        args.fd = fd;
+	args.flags = mode;
+	args.offset = offset;
+	args.size = len;
+        args.xdata = xdata;
+
+        proc = &conf->fops->proctable[GF_FOP_FALLOCATE];
+        if (!proc) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "rpc procedure not found for %s",
+                        gf_fop_list[GF_FOP_FALLOCATE]);
+                goto out;
+        }
+        if (proc->fn)
+                ret = proc->fn (frame, this, &args);
+out:
+        if (ret)
+                STACK_UNWIND_STRICT (fallocate, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
 
+int32_t
+client_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       size_t len, dict_t *xdata)
+{
+        int          ret  = -1;
+        clnt_conf_t *conf = NULL;
+        rpc_clnt_procedure_t *proc = NULL;
+        clnt_args_t  args = {0,};
+
+        conf = this->private;
+        if (!conf || !conf->fops)
+                goto out;
+
+        args.fd = fd;
+	args.offset = offset;
+	args.size = len;
+        args.xdata = xdata;
+
+        proc = &conf->fops->proctable[GF_FOP_DISCARD];
+        if (!proc) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "rpc procedure not found for %s",
+                        gf_fop_list[GF_FOP_DISCARD]);
+                goto out;
+        }
+        if (proc->fn)
+                ret = proc->fn (frame, this, &args);
+out:
+        if (ret)
+                STACK_UNWIND_STRICT(discard, frame, -1, ENOTCONN, NULL, NULL, NULL);
+
+	return 0;
+}
+
+int32_t
+client_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               size_t len, dict_t *xdata)
+{
+        int          ret              = -1;
+        clnt_conf_t *conf             = NULL;
+        rpc_clnt_procedure_t *proc    = NULL;
+        clnt_args_t  args             = {0,};
+
+        conf = this->private;
+        if (!conf || !conf->fops)
+                goto out;
+
+        args.fd = fd;
+        args.offset = offset;
+        args.size = len;
+        args.xdata = xdata;
+
+        proc = &conf->fops->proctable[GF_FOP_ZEROFILL];
+        if (!proc) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "rpc procedure not found for %s",
+                        gf_fop_list[GF_FOP_ZEROFILL]);
+                goto out;
+        }
+        if (proc->fn)
+                ret = proc->fn (frame, this, &args);
+out:
+        if (ret)
+                STACK_UNWIND_STRICT(zerofill, frame, -1, ENOTCONN,
+                                    NULL, NULL, NULL);
+
+        return 0;
+}
+
 
 int32_t
 client_getspec (call_frame_t *frame, xlator_t *this, const char *key,
@@ -1841,7 +2102,7 @@ out:
 }
 
 
- int
+int
 client_mark_fd_bad (xlator_t *this)
 {
         clnt_conf_t            *conf = NULL;
@@ -1893,7 +2154,7 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
 
                 if ((ret < 0) || (strcasecmp (handshake, "on"))) {
-                        ret = client_handshake (this, conf->rpc);
+                        ret = client_handshake (this, rpc);
                         if (ret)
                                 gf_log (this->name, GF_LOG_WARNING,
                                         "handshake msg returned %d", ret);
@@ -1908,16 +2169,46 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                                 conf->last_sent_event = GF_EVENT_CHILD_UP;
                         }
                 }
+
+                /* Cancel grace timer if set */
+                pthread_mutex_lock (&conf->lock);
+                {
+                        conf->grace_timer_needed = _gf_true;
+
+                        if (conf->grace_timer) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Cancelling the grace timer");
+
+                                gf_timer_call_cancel (this->ctx,
+                                                      conf->grace_timer);
+
+                                conf->grace_timer      = NULL;
+                        }
+                }
+                pthread_mutex_unlock (&conf->lock);
+
                 break;
         }
         case RPC_CLNT_DISCONNECT:
-
-                client_mark_fd_bad (this);
+                if (!conf->lk_heal)
+                        client_mark_fd_bad (this);
+                else
+                        client_register_grace_timer (this, conf);
 
                 if (!conf->skip_notify) {
-                        if (conf->connected)
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "disconnected");
+                        if (conf->connected) {
+                               gf_log (this->name,
+                                        ((!conf->disconnect_err_logged)
+                                        ? GF_LOG_INFO : GF_LOG_DEBUG),
+                                        "disconnected from %s. Client process "
+                                        "will keep trying to connect to "
+                                        "glusterd until brick's port is "
+                                        "available",
+                                  conf->rpc->conn.trans->peerinfo.identifier);
+
+                                if (conf->portmap_err_logged)
+                                        conf->disconnect_err_logged = 1;
+                        }
 
                         /* If the CHILD_DOWN event goes to parent xlator
                            multiple times, the logic of parent xlator notify
@@ -1941,6 +2232,15 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                 conf->connected = 0;
                 conf->skip_notify = 0;
 
+                if (conf->quick_reconnect) {
+                        conf->quick_reconnect = 0;
+                        rpc_clnt_start (rpc);
+
+                } else {
+                        rpc->conn.config.remote_port = 0;
+
+                }
+
                 break;
 
         default:
@@ -1958,7 +2258,7 @@ out:
 int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
-        clnt_conf_t *conf  = NULL;
+        clnt_conf_t     *conf  = NULL;
 
         conf = this->private;
         if (!conf)
@@ -1972,8 +2272,22 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                         "on transport");
 
                 rpc_clnt_start (conf->rpc);
+                break;
         }
-        break;
+
+        case GF_EVENT_PARENT_DOWN:
+                gf_log (this->name, GF_LOG_INFO,
+                        "current graph is no longer active, destroying "
+                        "rpc_client ");
+
+                pthread_mutex_lock (&conf->lock);
+                {
+                        conf->parent_down = 1;
+                }
+                pthread_mutex_unlock (&conf->lock);
+
+                rpc_clnt_disable (conf->rpc);
+                break;
 
         default:
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -2010,6 +2324,9 @@ build_client_config (xlator_t *this, clnt_conf_t *conf)
                 gf_log (this->name, GF_LOG_WARNING,
                         "option 'remote-subvolume' not given");
 
+        GF_OPTION_INIT ("filter-O_DIRECT", conf->filter_o_direct,
+                        bool, out);
+
         ret = 0;
 out:
         return ret;
@@ -2046,6 +2363,9 @@ client_destroy_rpc (xlator_t *this)
                 goto out;
 
         if (conf->rpc) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&conf->rpc->conn);
+
                 conf->rpc = rpc_clnt_unref (conf->rpc);
                 ret = 0;
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -2076,7 +2396,7 @@ client_init_rpc (xlator_t *this)
                 goto out;
         }
 
-        conf->rpc = rpc_clnt_new (this->options, this->ctx, this->name);
+        conf->rpc = rpc_clnt_new (this->options, this->ctx, this->name, 0);
         if (!conf->rpc) {
                 gf_log (this->name, GF_LOG_ERROR, "failed to initialize RPC");
                 goto out;
@@ -2091,7 +2411,8 @@ client_init_rpc (xlator_t *this)
         conf->handshake = &clnt_handshake_prog;
         conf->dump      = &clnt_dump_prog;
 
-        ret = rpcclnt_cbk_program_register (conf->rpc, &gluster_cbk_prog);
+        ret = rpcclnt_cbk_program_register (conf->rpc, &gluster_cbk_prog,
+                                            this);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to register callback program");
@@ -2107,6 +2428,43 @@ out:
 
 
 int
+client_init_grace_timer (xlator_t *this, dict_t *options,
+                         clnt_conf_t *conf)
+{
+        char     *lk_heal        = NULL;
+        int32_t   ret            = -1;
+        int32_t   grace_timeout  = -1;
+
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, options, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        conf->lk_heal = _gf_false;
+
+        ret = dict_get_str (options, "lk-heal", &lk_heal);
+        if (!ret)
+                gf_string2boolean (lk_heal, &conf->lk_heal);
+
+        gf_log (this->name, GF_LOG_DEBUG, "lk-heal = %s",
+                (conf->lk_heal) ? "on" : "off");
+
+        ret = dict_get_int32 (options, "grace-timeout", &grace_timeout);
+        if (!ret)
+                conf->grace_ts.tv_sec = grace_timeout;
+        else
+                conf->grace_ts.tv_sec = 10;
+
+        conf->grace_ts.tv_nsec  = 0;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Client grace timeout "
+                "value = %"PRIu64, conf->grace_ts.tv_sec);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
 reconfigure (xlator_t *this, dict_t *options)
 {
 	clnt_conf_t *conf              = NULL;
@@ -2153,6 +2511,13 @@ reconfigure (xlator_t *this, dict_t *options)
                 }
         }
 
+        GF_OPTION_RECONF ("filter-O_DIRECT", conf->filter_o_direct,
+                          options, bool, out);
+
+        ret = client_init_grace_timer (this, options, conf);
+        if (ret)
+                goto out;
+
         ret = 0;
 out:
 	return ret;
@@ -2166,7 +2531,6 @@ init (xlator_t *this)
         int          ret = -1;
         clnt_conf_t *conf = NULL;
 
-        /* */
         if (this->children) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "FATAL: client protocol translator cannot have any "
@@ -2186,6 +2550,15 @@ init (xlator_t *this)
         pthread_mutex_init (&conf->lock, NULL);
         INIT_LIST_HEAD (&conf->saved_fds);
 
+        /* Initialize parameters for lock self healing*/
+        conf->lk_version         = 1;
+        conf->grace_timer        = NULL;
+        conf->grace_timer_needed = _gf_true;
+
+        ret = client_init_grace_timer (this, this->options, conf);
+        if (ret)
+                goto out;
+
         LOCK_INIT (&conf->rec_lock);
 
         conf->last_sent_event = -1; /* To start with we don't have any events */
@@ -2207,6 +2580,13 @@ init (xlator_t *this)
                 goto out;
         }
 
+        this->local_pool = mem_pool_new (clnt_local_t, 64);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
 
         ret = client_init_rpc (this);
 out:
@@ -2225,8 +2605,12 @@ fini (xlator_t *this)
         this->private = NULL;
 
         if (conf) {
-                if (conf->rpc)
-                       rpc_clnt_unref (conf->rpc);
+                if (conf->rpc) {
+                        /* cleanup the saved-frames before last unref */
+                        rpc_clnt_connection_cleanup (&conf->rpc->conn);
+
+                        rpc_clnt_unref (conf->rpc);
+                }
 
                 /* Saved Fds */
                 /* TODO: */
@@ -2238,6 +2622,54 @@ fini (xlator_t *this)
         return;
 }
 
+static void
+client_fd_lk_ctx_dump (xlator_t *this, fd_lk_ctx_t *lk_ctx, int nth_fd)
+{
+        gf_boolean_t            use_try_lock             = _gf_true;
+        int                     ret                      = -1;
+        int                     lock_no                  = 0;
+        fd_lk_ctx_t             *lk_ctx_ref              = NULL;
+        fd_lk_ctx_node_t        *plock                   = NULL;
+        char                    key[GF_DUMP_MAX_BUF_LEN] = {0,};
+
+        lk_ctx_ref = fd_lk_ctx_try_ref (lk_ctx);
+        if (!lk_ctx_ref)
+                return;
+
+        ret = client_fd_lk_list_empty (lk_ctx_ref, (use_try_lock = _gf_true));
+        if (ret != 0)
+                return;
+
+        ret = TRY_LOCK (&lk_ctx_ref->lock);
+        if (ret)
+                return;
+
+        gf_proc_dump_write ("------","------");
+
+        lock_no = 0;
+        list_for_each_entry (plock, &lk_ctx_ref->lk_list, next) {
+                snprintf (key, sizeof (key), "granted-posix-lock[%d]",
+                          lock_no++);
+                gf_proc_dump_write (key, "owner = %s, cmd = %s "
+                                    "fl_type = %s, fl_start = %"
+                                    PRId64", fl_end = %"PRId64
+                                    ", user_flock: l_type = %s, "
+                                    "l_start = %"PRId64", l_len = %"PRId64,
+                                    lkowner_utoa (&plock->user_flock.l_owner),
+                                    get_lk_cmd (plock->cmd),
+                                    get_lk_type (plock->fl_type),
+                                    plock->fl_start, plock->fl_end,
+                                    get_lk_type (plock->user_flock.l_type),
+                                    plock->user_flock.l_start,
+                                    plock->user_flock.l_len);
+        }
+        gf_proc_dump_write ("------","------");
+
+        UNLOCK (&lk_ctx_ref->lock);
+        fd_lk_ctx_unref (lk_ctx_ref);
+
+}
+
 int
 client_priv_dump (xlator_t *this)
 {
@@ -2252,18 +2684,12 @@ client_priv_dump (xlator_t *this)
                 return -1;
 
         conf = this->private;
-        if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
+        if (!conf)
                 return -1;
-        }
 
         ret = pthread_mutex_trylock(&conf->lock);
-        if (ret) {
-                gf_log(this->name, GF_LOG_WARNING, "Unable to lock client %s"
-                       " errno: %d", this->name, errno);
+        if (ret)
                 return -1;
-        }
 
         gf_proc_dump_build_key(key_prefix, "xlator.protocol.client",
                                "%s.priv", this->name);
@@ -2271,13 +2697,13 @@ client_priv_dump (xlator_t *this)
         gf_proc_dump_add_section(key_prefix);
 
         list_for_each_entry(tmp, &conf->saved_fds, sfd_pos) {
-                sprintf (key, "fd.%d.remote_fd", ++i);
+                sprintf (key, "fd.%d.remote_fd", i);
                 gf_proc_dump_write(key, "%d", tmp->remote_fd);
+                client_fd_lk_ctx_dump (this, tmp->lk_ctx, i);
+                i++;
         }
 
         gf_proc_dump_write("connecting", "%d", conf->connecting);
-        gf_proc_dump_write("last_sent", "%s", ctime(&conf->last_sent.tv_sec));
-        gf_proc_dump_write("last_received", "%s", ctime(&conf->last_received.tv_sec));
 
         if (conf->rpc) {
                 gf_proc_dump_write("total_bytes_read", "%"PRIu64,
@@ -2357,6 +2783,9 @@ struct xlator_fops fops = {
         .fxattrop    = client_fxattrop,
         .setattr     = client_setattr,
         .fsetattr    = client_fsetattr,
+	.fallocate   = client_fallocate,
+	.discard     = client_discard,
+        .zerofill    = client_zerofill,
         .getspec     = client_getspec,
 };
 
@@ -2409,5 +2838,30 @@ struct volume_options options[] = {
         { .key   = {"client-bind-insecure"},
           .type  = GF_OPTION_TYPE_BOOL
         },
+        { .key   = {"lk-heal"},
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Enables or disables the lock heal."
+        },
+        { .key   = {"grace-timeout"},
+          .type  = GF_OPTION_TYPE_INT,
+          .min   = 10,
+          .max   = 1800,
+          .description = "Sets the grace-timeout value. Valid range 10-1800."
+        },
+        {.key  = {"tcp-window-size"},
+         .type = GF_OPTION_TYPE_SIZET,
+         .min  = GF_MIN_SOCKET_WINDOW_SIZE,
+         .max  = GF_MAX_SOCKET_WINDOW_SIZE,
+         .description = "Specifies the window size for tcp socket."
+        },
+        { .key   = {"filter-O_DIRECT"},
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "disable",
+          .description = "If enabled, in open() and creat() calls, O_DIRECT "
+          "flag will be filtered at the client protocol level so server will "
+          "still continue to cache the file. This works similar to NFS's "
+          "behavior of O_DIRECT",
+        },
         { .key   = {NULL} },
 };
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 2dda451c9..afab2d74f 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CLIENT_H
@@ -29,33 +20,49 @@
 #include "client-mem-types.h"
 #include "protocol-common.h"
 #include "glusterfs3.h"
+#include "fd-lk.h"
 
 /* FIXME: Needs to be defined in a common file */
 #define CLIENT_CMD_CONNECT    "trusted.glusterfs.client-connect"
 #define CLIENT_CMD_DISCONNECT "trusted.glusterfs.client-disconnect"
 #define CLIENT_DUMP_LOCKS     "trusted.glusterfs.clientlk-dump"
+#define GF_MAX_SOCKET_WINDOW_SIZE  (1 * GF_UNIT_MB)
+#define GF_MIN_SOCKET_WINDOW_SIZE  (0)
+
+typedef enum {
+        GF_LK_HEAL_IN_PROGRESS,
+        GF_LK_HEAL_DONE,
+} lk_heal_state_t;
 
-#define CLIENT_GET_REMOTE_FD(conf, fd, remote_fd, op_errno, label)      \
+typedef enum {
+        DEFAULT_REMOTE_FD = 0,
+        FALLBACK_TO_ANON_FD = 1
+} clnt_remote_fd_flags_t;
+
+#define CLIENT_GET_REMOTE_FD(xl, fd, flags, remote_fd, op_errno, label) \
         do {                                                            \
-                clnt_fd_ctx_t      *fdctx    = NULL;                    \
-                pthread_mutex_lock (&conf->lock);                       \
-                {                                                       \
-                        fdctx = this_fd_get_ctx (fd, THIS);             \
-                }                                                       \
-                pthread_mutex_unlock (&conf->lock);                     \
-                if (!fdctx) {                                           \
-                        remote_fd = -2;                                 \
-                } else {                                                \
-                        remote_fd = fdctx->remote_fd;                   \
+                int     _ret    = 0;                                    \
+                _ret = client_get_remote_fd (xl, fd, flags, &remote_fd);\
+                if (_ret < 0) {                                         \
+                        op_errno = errno;                               \
+                        goto label;                                     \
                 }                                                       \
                 if (remote_fd == -1) {                                  \
-                        gf_log (THIS->name, GF_LOG_WARNING, " (%s) "    \
+                        gf_log (xl->name, GF_LOG_WARNING, " (%s) "      \
                                 "remote_fd is -1. EBADFD",              \
                                 uuid_utoa (fd->inode->gfid));           \
                         op_errno = EBADFD;                              \
                         goto label;                                     \
                 }                                                       \
-        } while (0);
+        } while (0)
+
+#define CLIENT_STACK_UNWIND(op, frame, params ...) do {             \
+                clnt_local_t *__local = frame->local;               \
+                frame->local = NULL;                                \
+                STACK_UNWIND_STRICT (op, frame, params);            \
+                client_local_wipe (__local);                        \
+        } while (0)
+
 
 struct clnt_options {
         char *remote_subvolume;
@@ -70,8 +77,6 @@ typedef struct clnt_conf {
         pthread_mutex_t        lock;
         int                    connecting;
         int                    connected;
-	struct timeval         last_sent;
-	struct timeval         last_received;
 
         rpc_clnt_prog_t       *fops;
         rpc_clnt_prog_t       *mgmt;
@@ -88,9 +93,33 @@ typedef struct clnt_conf {
                                                    which was sent earlier */
         char                   portmap_err_logged; /* flag used to prevent
                                                       excessive logging */
+        char                   disconnect_err_logged; /* flag used to prevent
+                                                         excessive disconnect
+                                                         logging */
+
         char                   need_different_port; /* flag used to change the
                                                        portmap path in case of
                                                        'tcp,rdma' on server */
+        gf_boolean_t           lk_heal;
+        uint16_t               lk_version; /* this variable is used to distinguish
+                                              client-server transaction while
+                                              performing lock healing */
+        struct timespec        grace_ts;
+        gf_timer_t            *grace_timer;
+        gf_boolean_t           grace_timer_needed; /* The state of this flag will
+                                                      be used to decide whether
+                                                      a new grace-timer must be
+                                                      registered or not. False
+                                                      means dont register, true
+                                                      means register */
+        char                   parent_down;
+	gf_boolean_t           quick_reconnect; /* When reconnecting after
+						   portmap query, do not let
+						   the reconnection happen after
+						   the usual 3-second wait
+						*/
+        gf_boolean_t           filter_o_direct; /* if set, filter O_DIRECT from
+                                                   the flags list of open() */
 } clnt_conf_t;
 
 typedef struct _client_fd_ctx {
@@ -98,16 +127,16 @@ typedef struct _client_fd_ctx {
                                             fd's position in the saved_fds list.
                                         */
         int64_t           remote_fd;
-        inode_t          *inode;
-        uint64_t          ino;
-        uint64_t          gen;
         char              is_dir;
         char              released;
         int32_t           flags;
-        int32_t           wbflags;
-
+        fd_lk_ctx_t      *lk_ctx;
         pthread_mutex_t   mutex;
+        lk_heal_state_t   lk_heal_state;
+        uuid_t            gfid;
+        void (*reopen_done) (struct _client_fd_ctx*, xlator_t *);
         struct list_head  lock_list;     /* List of all granted locks on this fd */
+        int32_t           reopen_attempts;
 } clnt_fd_ctx_t;
 
 typedef struct _client_posix_lock {
@@ -128,7 +157,6 @@ typedef struct client_local {
         fd_t                *fd;
         clnt_fd_ctx_t       *fdctx;
         uint32_t             flags;
-        uint32_t             wbflags;
         struct iobref       *iobref;
 
         client_posix_lock_t *client_lock;
@@ -136,18 +164,18 @@ typedef struct client_local {
         int32_t              cmd;
         struct list_head     lock_list;
         pthread_mutex_t      mutex;
+        char                *name;
+        gf_boolean_t         attempt_reopen;
 } clnt_local_t;
 
 typedef struct client_args {
         loc_t              *loc;
         fd_t               *fd;
-        dict_t             *xattr_req;
         const char         *linkname;
         struct iobref      *iobref;
         struct iovec       *vector;
         dict_t             *xattr;
         struct iatt        *stbuf;
-        dict_t             *dict;
         loc_t              *oldloc;
         loc_t              *newloc;
         const char         *name;
@@ -161,7 +189,6 @@ typedef struct client_args {
         mode_t              mode;
         dev_t               rdev;
         int32_t             flags;
-        int32_t             wbflags;
         int32_t             count;
         int32_t             datasync;
         entrylk_cmd         cmd_entrylk;
@@ -169,6 +196,9 @@ typedef struct client_args {
         gf_xattrop_flags_t  optype;
         int32_t             valid;
         int32_t             len;
+
+        mode_t              umask;
+        dict_t             *xdata;
 } clnt_args_t;
 
 typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *args);
@@ -187,9 +217,6 @@ int client_submit_request (xlator_t *this, void *req,
                            struct iovec *rsp_payload, int rsp_count,
                            struct iobref *rsp_iobref, xdrproc_t xdrproc);
 
-int protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx);
-int protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx);
-
 int unserialize_rsp_dirent (struct gfs3_readdir_rsp *rsp, gf_dirent_t *entries);
 int unserialize_rsp_direntp (xlator_t *this, fd_t *fd,
                              struct gfs3_readdirp_rsp *rsp, gf_dirent_t *entries);
@@ -200,7 +227,6 @@ int client_attempt_lock_recovery (xlator_t *this, clnt_fd_ctx_t *fdctx);
 int32_t delete_granted_locks_owner (fd_t *fd, gf_lkowner_t *owner);
 int client_add_lock_for_recovery (fd_t *fd, struct gf_flock *flock,
                                   gf_lkowner_t *owner, int32_t cmd);
-uint64_t decrement_reopen_fd_count (xlator_t *this, clnt_conf_t *conf);
 int32_t delete_granted_locks_fd (clnt_fd_ctx_t *fdctx);
 int32_t client_cmd_to_gf_cmd (int32_t cmd, int32_t *gf_cmd);
 void client_save_number_fds (clnt_conf_t *conf, int count);
@@ -211,4 +237,21 @@ int32_t client_dump_locks (char *name, inode_t *inode,
                            dict_t *dict);
 int client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx);
 
+uint32_t client_get_lk_ver (clnt_conf_t *conf);
+
+int32_t client_type_to_gf_type (short l_type);
+
+int client_mark_fd_bad (xlator_t *this);
+
+int client_set_lk_version (xlator_t *this);
+
+int client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t use_try_lock);
+void client_default_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this);
+void client_attempt_reopen (fd_t *fd, xlator_t *this);
+int client_get_remote_fd (xlator_t *this, fd_t *fd, int flags,
+                          int64_t *remote_fd);
+int client_fd_fop_prepare_local (call_frame_t *frame, fd_t *fd,
+                                 int64_t remote_fd);
+gf_boolean_t
+__is_fd_reopen_in_progress (clnt_fd_ctx_t *fdctx);
 #endif /* !_CLIENT_H */
diff --git a/xlators/protocol/legacy/Makefile.am b/xlators/protocol/legacy/Makefile.am
deleted file mode 100644
index 991486302..000000000
--- a/xlators/protocol/legacy/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = lib transport client server
-
-CLEANFILES =
diff --git a/xlators/protocol/legacy/client/Makefile.am b/xlators/protocol/legacy/client/Makefile.am
deleted file mode 100644
index d471a3f92..000000000
--- a/xlators/protocol/legacy/client/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/protocol/legacy/client/src/Makefile.am b/xlators/protocol/legacy/client/src/Makefile.am
deleted file mode 100644
index 2ae64ebd0..000000000
--- a/xlators/protocol/legacy/client/src/Makefile.am
+++ /dev/null
@@ -1,21 +0,0 @@
-
-xlator_LTLIBRARIES = client-old.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/legacy/protocol
-
-client_old_la_LDFLAGS = -module -avoidversion
-
-client_old_la_SOURCES = client-protocol.c saved-frames.c
-
-client_old_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	$(top_builddir)/xlators/protocol/legacy/lib/src/libgfproto.la
-
-noinst_HEADERS = client-protocol.h saved-frames.h client-mem-types.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)  \
-	-I$(top_srcdir)/xlators/protocol/legacy/lib/src
-
-CLEANFILES = 
-
-install-data-hook:
-	ln -sf client-old.so $(DESTDIR)$(xlatordir)/client.so
diff --git a/xlators/protocol/legacy/client/src/client-mem-types.h b/xlators/protocol/legacy/client/src/client-mem-types.h
deleted file mode 100644
index d2e2878c3..000000000
--- a/xlators/protocol/legacy/client/src/client-mem-types.h
+++ /dev/null
@@ -1,43 +0,0 @@
-
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __CLIENT_MEM_TYPES_H__
-#define __CLIENT_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_client_mem_types_ {
-        gf_client_mt_dir_entry_t = gf_common_mt_end + 1,
-        gf_client_mt_volfile_ctx,
-        gf_client_mt_client_state_t,
-        gf_client_mt_client_conf_t,
-        gf_client_mt_locker,
-        gf_client_mt_lock_table,
-        gf_client_mt_char,
-        gf_client_mt_client_connection_t,
-        gf_client_mt_client_fd_ctx_t,
-        gf_client_mt_client_local_t,
-        gf_client_mt_saved_frames,
-        gf_client_mt_saved_frame,
-        gf_client_mt_end
-};
-#endif
-
diff --git a/xlators/protocol/legacy/client/src/client-protocol.c b/xlators/protocol/legacy/client/src/client-protocol.c
deleted file mode 100644
index 5c199e4dc..000000000
--- a/xlators/protocol/legacy/client/src/client-protocol.c
+++ /dev/null
@@ -1,6683 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-#include <inttypes.h>
-
-
-#include "glusterfs.h"
-#include "client-protocol.h"
-#include "compat.h"
-#include "dict.h"
-#include "protocol.h"
-#include "transport.h"
-#include "xlator.h"
-#include "logging.h"
-#include "timer.h"
-#include "defaults.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
-#include "client-mem-types.h"
-
-#include <sys/resource.h>
-#include <inttypes.h>
-
-/* for default_*_cbk functions */
-#include "defaults.c"
-#include "saved-frames.h"
-#include "common-utils.h"
-
-int protocol_client_cleanup (transport_t *trans);
-int protocol_client_interpret (xlator_t *this, transport_t *trans,
-                               char *hdr_p, size_t hdrlen,
-                               struct iobuf *iobuf);
-int
-protocol_client_xfer (call_frame_t *frame, xlator_t *this, transport_t *trans,
-                      int type, int op,
-                      gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iovec *vector, int count,
-                      struct iobref *iobref);
-
-int
-protocol_client_post_handshake (call_frame_t *frame, xlator_t *this);
-
-static gf_op_t gf_fops[GF_PROTO_FOP_MAXVALUE];
-static gf_op_t gf_mops[GF_MOP_MAXVALUE];
-static gf_op_t gf_cbks[GF_CBK_MAXVALUE];
-
-
-transport_t *
-client_channel (xlator_t *this, int id)
-{
-        transport_t              *trans = NULL;
-        client_conf_t            *conf = NULL;
-        int                       i = 0;
-        struct client_connection *conn = NULL;
-
-        conf = this->private;
-
-        trans = conf->transport[id];
-        conn = trans->xl_private;
-
-        if (conn->connected == 1)
-                goto ret;
-
-        for (i = 0; i < CHANNEL_MAX; i++) {
-                trans = conf->transport[i];
-                conn = trans->xl_private;
-                if (conn->connected == 1)
-                        break;
-        }
-
-ret:
-        return trans;
-}
-
-
-client_fd_ctx_t *
-this_fd_del_ctx (fd_t *file, xlator_t *this)
-{
-        int         dict_ret = -1;
-        uint64_t    ctxaddr  = 0;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
-
-        dict_ret = fd_ctx_del (file, this, &ctxaddr);
-
-        if (dict_ret < 0) {
-                ctxaddr = 0;
-        }
-
-out:
-        return (client_fd_ctx_t *)(unsigned long)ctxaddr;
-}
-
-
-client_fd_ctx_t *
-this_fd_get_ctx (fd_t *file, xlator_t *this)
-{
-        int         dict_ret = -1;
-        uint64_t    ctxaddr = 0;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
-
-        dict_ret = fd_ctx_get (file, this, &ctxaddr);
-
-        if (dict_ret < 0) {
-                ctxaddr = 0;
-        }
-
-out:
-        return (client_fd_ctx_t *)(unsigned long)ctxaddr;
-}
-
-
-static void
-this_fd_set_ctx (fd_t *file, xlator_t *this, loc_t *loc, client_fd_ctx_t *ctx)
-{
-        uint64_t oldaddr = 0;
-        int32_t  ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
-
-        ret = fd_ctx_get (file, this, &oldaddr);
-        if (ret >= 0) {
-                if (loc)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s (%"PRId64"): trying duplicate remote fd set. ",
-                                loc->path, loc->inode->ino);
-                else
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%p: trying duplicate remote fd set. ",
-                                file);
-        }
-
-        ret = fd_ctx_set (file, this, (uint64_t)(unsigned long)ctx);
-        if (ret < 0) {
-                if (loc)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s (%"PRId64"): failed to set remote fd",
-                                loc->path, loc->inode->ino);
-                else
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%p: failed to set remote fd",
-                                file);
-        }
-out:
-        return;
-}
-
-
-static int
-client_local_wipe (client_local_t *local)
-{
-        if (local) {
-                loc_wipe (&local->loc);
-
-                if (local->fd)
-                        fd_unref (local->fd);
-
-                GF_FREE (local);
-        }
-
-        return 0;
-}
-
-/*
- * lookup_frame - lookup call frame corresponding to a given callid
- * @trans: transport object
- * @callid: call id of the frame
- *
- * not for external reference
- */
-
-static call_frame_t *
-lookup_frame (transport_t *trans, int32_t op, int8_t type, int64_t callid)
-{
-        client_connection_t *conn = NULL;
-        call_frame_t        *frame = NULL;
-
-        conn = trans->xl_private;
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                frame = saved_frames_get (conn->saved_frames,
-                                          op, type, callid);
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        return frame;
-}
-
-
-static void
-call_bail (void *data)
-{
-        client_connection_t  *conn = NULL;
-        struct timeval        current;
-        transport_t          *trans = NULL;
-        struct list_head      list;
-        struct saved_frame   *saved_frame = NULL;
-        struct saved_frame   *trav = NULL;
-        struct saved_frame   *tmp = NULL;
-        call_frame_t         *frame = NULL;
-        gf_hdr_common_t       hdr = {0, };
-        char                **gf_op_list = NULL;
-        gf_op_t              *gf_ops = NULL;
-        struct tm             frame_sent_tm;
-        char                  frame_sent[32] = {0,};
-        struct timeval        timeout = {0,};
-        gf_timer_cbk_t        timer_cbk = NULL;
-
-        GF_VALIDATE_OR_GOTO ("client", data, out);
-        trans = data;
-
-        conn = trans->xl_private;
-
-        gettimeofday (&current, NULL);
-        INIT_LIST_HEAD (&list);
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                /* Chaining to get call-always functionality from
-                   call-once timer */
-                if (conn->timer) {
-                        timer_cbk = conn->timer->callbk;
-
-                        timeout.tv_sec = 10;
-                        timeout.tv_usec = 0;
-
-                        gf_timer_call_cancel (trans->xl->ctx, conn->timer);
-                        conn->timer = gf_timer_call_after (trans->xl->ctx,
-                                                           timeout,
-                                                           timer_cbk,
-                                                           trans);
-                        if (conn->timer == NULL) {
-                                gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                        "Cannot create bailout timer");
-                        }
-                }
-
-                do {
-                        saved_frame =
-                                saved_frames_get_timedout (conn->saved_frames,
-                                                           GF_OP_TYPE_MOP_REQUEST,
-                                                           conn->frame_timeout,
-                                                           &current);
-                        if (saved_frame)
-                                list_add (&saved_frame->list, &list);
-
-                } while (saved_frame);
-
-                do {
-                        saved_frame =
-                                saved_frames_get_timedout (conn->saved_frames,
-                                                           GF_OP_TYPE_FOP_REQUEST,
-                                                           conn->frame_timeout,
-                                                           &current);
-                        if (saved_frame)
-                                list_add (&saved_frame->list, &list);
-                } while (saved_frame);
-
-                do {
-                        saved_frame =
-                                saved_frames_get_timedout (conn->saved_frames,
-                                                           GF_OP_TYPE_CBK_REQUEST,
-                                                           conn->frame_timeout,
-                                                           &current);
-                        if (saved_frame)
-                                list_add (&saved_frame->list, &list);
-                } while (saved_frame);
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        hdr.rsp.op_ret   = hton32 (-1);
-        hdr.rsp.op_errno = hton32 (ENOTCONN);
-
-        list_for_each_entry_safe (trav, tmp, &list, list) {
-                switch (trav->type)
-                {
-                case GF_OP_TYPE_FOP_REQUEST:
-                        gf_ops = gf_fops;
-                        gf_op_list = gf_fop_list;
-                        break;
-                case GF_OP_TYPE_MOP_REQUEST:
-                        gf_ops = gf_mops;
-                        gf_op_list = gf_mop_list;
-                        break;
-                case GF_OP_TYPE_CBK_REQUEST:
-                        gf_ops = gf_cbks;
-                        gf_op_list = gf_cbk_list;
-                        break;
-                default:
-                        goto out;
-                }
-
-                localtime_r (&trav->saved_at.tv_sec, &frame_sent_tm);
-                strftime (frame_sent, 32, "%Y-%m-%d %H:%M:%S", &frame_sent_tm);
-
-                gf_log (trans->xl->name, GF_LOG_ERROR,
-                        "bailing out frame %s(%d) "
-                        "frame sent = %s. frame-timeout = %d",
-                        gf_op_list[trav->op], trav->op,
-                        frame_sent, conn->frame_timeout);
-
-                hdr.type = hton32 (trav->type);
-                hdr.op   = hton32 (trav->op);
-
-                frame = trav->frame;
-
-                gf_ops[trav->op] (frame, &hdr, sizeof (hdr), NULL);
-
-                list_del_init (&trav->list);
-                GF_FREE (trav);
-        }
-out:
-        return;
-}
-
-
-void
-save_frame (transport_t *trans, call_frame_t *frame,
-            int32_t op, int8_t type, uint64_t callid)
-{
-        client_connection_t *conn = NULL;
-        struct timeval       timeout = {0, };
-
-
-        conn = trans->xl_private;
-
-        saved_frames_put (conn->saved_frames, frame, op, type, callid);
-
-        if (conn->timer == NULL && conn->frame_timeout) {
-                timeout.tv_sec  = 10;
-                timeout.tv_usec = 0;
-                conn->timer = gf_timer_call_after (trans->xl->ctx, timeout,
-                                                   call_bail, (void *) trans);
-        }
-}
-
-
-
-void
-client_ping_timer_expired (void *data)
-{
-        xlator_t            *this = NULL;
-        transport_t         *trans = NULL;
-        client_conf_t       *conf = NULL;
-        client_connection_t *conn = NULL;
-        int                  disconnect = 0;
-        int                  transport_activity = 0;
-        struct timeval       timeout = {0, };
-        struct timeval       current = {0, };
-
-        trans = data;
-        this  = trans->xl;
-        conf  = this->private;
-        conn  = trans->xl_private;
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (conn->ping_timer)
-                        gf_timer_call_cancel (trans->xl->ctx,
-                                              conn->ping_timer);
-                gettimeofday (&current, NULL);
-
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        if (((current.tv_sec - conf->last_received.tv_sec) <
-                             conn->ping_timeout)
-                            || ((current.tv_sec - conf->last_sent.tv_sec) <
-                                conn->ping_timeout)) {
-                                transport_activity = 1;
-                        }
-                }
-                pthread_mutex_unlock (&conf->mutex);
-
-                if (transport_activity) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "ping timer expired but transport activity "
-                                "detected - not bailing transport");
-                        conn->transport_activity = 0;
-                        timeout.tv_sec = conn->ping_timeout;
-                        timeout.tv_usec = 0;
-
-                        conn->ping_timer =
-                                gf_timer_call_after (trans->xl->ctx, timeout,
-                                                     client_ping_timer_expired,
-                                                     (void *) trans);
-                        if (conn->ping_timer == NULL)
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "unable to setup timer");
-
-                } else {
-                        conn->ping_started = 0;
-                        conn->ping_timer = NULL;
-                        disconnect = 1;
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-        if (disconnect) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Server %s has not responded in the last %d "
-                        "seconds, disconnecting.",
-                        conf->transport[0]->peerinfo.identifier,
-                        conn->ping_timeout);
-
-                transport_disconnect (conf->transport[0]);
-                transport_disconnect (conf->transport[1]);
-        }
-}
-
-
-void
-client_start_ping (void *data)
-{
-        xlator_t            *this = NULL;
-        transport_t         *trans = NULL;
-        client_conf_t       *conf = NULL;
-        client_connection_t *conn = NULL;
-        int32_t              ret = -1;
-        gf_hdr_common_t     *hdr = NULL;
-        struct timeval       timeout = {0, };
-        call_frame_t        *dummy_frame = NULL;
-        size_t               hdrlen = -1;
-        gf_mop_ping_req_t   *req = NULL;
-        int                  frame_count = 0;
-
-
-        trans = data;
-        this  = trans->xl;
-        conf  = this->private;
-        conn  = trans->xl_private;
-
-        if (!conn->ping_timeout)
-                return;
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (conn->ping_timer)
-                        gf_timer_call_cancel (trans->xl->ctx, conn->ping_timer);
-
-                conn->ping_timer = NULL;
-                conn->ping_started = 0;
-
-                if (conn->saved_frames)
-                        /* treat the case where conn->saved_frames is NULL
-                           as no pending frames */
-                        frame_count = conn->saved_frames->count;
-
-                if ((frame_count == 0) || !conn->connected) {
-                        /* using goto looked ugly here,
-                         * hence getting out this way */
-                        /* unlock */
-                        pthread_mutex_unlock (&conn->lock);
-                        return;
-                }
-
-                if (frame_count < 0) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "saved_frames->count is %"PRId64,
-                                conn->saved_frames->count);
-                        conn->saved_frames->count = 0;
-                }
-
-                timeout.tv_sec = conn->ping_timeout;
-                timeout.tv_usec = 0;
-
-                conn->ping_timer =
-                        gf_timer_call_after (trans->xl->ctx, timeout,
-                                             client_ping_timer_expired,
-                                             (void *) trans);
-
-                if (conn->ping_timer == NULL) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "unable to setup timer");
-                } else {
-                        conn->ping_started = 1;
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        if (!hdr)
-                goto err;
-
-        dummy_frame = create_frame (this, this->ctx->pool);
-
-        if (!dummy_frame)
-                goto err;
-
-        dummy_frame->local = trans;
-
-        ret = protocol_client_xfer (dummy_frame, this, trans,
-                                    GF_OP_TYPE_MOP_REQUEST, GF_MOP_PING,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return;
-err:
-        if (hdr)
-                GF_FREE (hdr);
-
-        if (dummy_frame)
-                STACK_DESTROY (dummy_frame->root);
-
-        return;
-}
-
-
-int
-client_ping_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        xlator_t            *this = NULL;
-        transport_t         *trans = NULL;
-        client_connection_t *conn = NULL;
-        struct timeval       timeout = {0, };
-        int                  op_ret = 0;
-
-        trans  = frame->local; frame->local = NULL;
-        this   = trans->xl;
-        conn   = trans->xl_private;
-
-        op_ret = ntoh32 (hdr->rsp.op_ret);
-
-        if (op_ret == -1) {
-                /* timer expired and transport bailed out */
-                gf_log (this->name, GF_LOG_DEBUG, "timer must have expired");
-                goto out;
-        }
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                timeout.tv_sec  = conn->ping_timeout;
-                timeout.tv_usec = 0;
-
-                gf_timer_call_cancel (trans->xl->ctx,
-                                      conn->ping_timer);
-
-                conn->ping_timer =
-                        gf_timer_call_after (trans->xl->ctx, timeout,
-                                             client_start_ping, (void *)trans);
-                if (conn->ping_timer == NULL)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "gf_timer_call_after() returned NULL");
-        }
-        pthread_mutex_unlock (&conn->lock);
-out:
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-int
-client_encode_groups (call_frame_t *frame, gf_hdr_common_t *hdr)
-{
-        int     i = 0;
-        if ((!frame) || (!hdr))
-                return -1;
-
-        hdr->req.ngrps = hton32 (frame->root->ngrps);
-        if (frame->root->ngrps == 0)
-                return 0;
-
-        for (; i < frame->root->ngrps; ++i)
-                hdr->req.groups[i] = hton32 (frame->root->groups[i]);
-
-        return 0;
-}
-
-
-int
-protocol_client_xfer (call_frame_t *frame, xlator_t *this, transport_t *trans,
-                      int type, int op,
-                      gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iovec *vector, int count,
-                      struct iobref *iobref)
-{
-        client_conf_t        *conf = NULL;
-        client_connection_t  *conn = NULL;
-        uint64_t              callid = 0;
-        int32_t               ret = -1;
-        int                   start_ping = 0;
-        gf_hdr_common_t       rsphdr = {0, };
-
-        conf  = this->private;
-
-        if (!trans) {
-                /* default to bulk op since it is 'safer' */
-                trans = conf->transport[CHANNEL_BULK];
-        }
-        conn  = trans->xl_private;
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                callid = ++conn->callid;
-
-                hdr->callid = hton64 (callid);
-                hdr->op     = hton32 (op);
-                hdr->type   = hton32 (type);
-
-                if (frame) {
-                        hdr->req.uid = hton32 (frame->root->uid);
-                        hdr->req.gid = hton32 (frame->root->gid);
-                        hdr->req.pid = hton32 (frame->root->pid);
-                        hdr->req.lk_owner = hton64 (frame->root->lk_owner);
-                        client_encode_groups (frame, hdr);
-                }
-
-                if (conn->connected == 0)
-                        transport_connect (trans);
-
-                ret = -1;
-
-                if (conn->connected ||
-                    ((type == GF_OP_TYPE_MOP_REQUEST) &&
-                     (op == GF_MOP_SETVOLUME))) {
-                        ret = transport_submit (trans, (char *)hdr, hdrlen,
-                                                vector, count, iobref);
-                }
-
-                if ((ret >= 0) && frame) {
-                        pthread_mutex_lock (&conf->mutex);
-                        {
-                                gettimeofday (&conf->last_sent, NULL);
-                        }
-                        pthread_mutex_unlock (&conf->mutex);
-                        save_frame (trans, frame, op, type, callid);
-                }
-
-                if (!conn->ping_started && (ret >= 0)) {
-                        start_ping = 1;
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        if (start_ping)
-                client_start_ping ((void *) trans);
-
-        if (frame && (ret < 0)) {
-                rsphdr.op = op;
-                rsphdr.rsp.op_ret   = hton32 (-1);
-                rsphdr.rsp.op_errno = hton32 (ENOTCONN);
-
-                if (type == GF_OP_TYPE_FOP_REQUEST) {
-                        rsphdr.type = GF_OP_TYPE_FOP_REPLY;
-                        gf_fops[op] (frame, &rsphdr, sizeof (rsphdr), NULL);
-                } else if (type == GF_OP_TYPE_MOP_REQUEST) {
-                        rsphdr.type = GF_OP_TYPE_MOP_REPLY;
-                        gf_mops[op] (frame, &rsphdr, sizeof (rsphdr), NULL);
-                } else {
-                        rsphdr.type = GF_OP_TYPE_CBK_REPLY;
-                        gf_cbks[op] (frame, &rsphdr, sizeof (rsphdr), NULL);
-                }
-
-                GF_FREE (hdr);
-        }
-
-        return ret;
-}
-
-
-
-/**
- * client_create - create function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @path: complete path to file
- * @flags: create flags
- * @mode: create mode
- *
- * external reference through client_protocol_xlator->fops->create
- */
-
-int
-client_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-               mode_t mode, fd_t *fd, dict_t *params)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_create_req_t *req = NULL;
-        size_t               hdrlen = 0;
-        size_t               pathlen = 0;
-        size_t               baselen = 0;
-        int32_t              ret = -1;
-        ino_t                par = 0;
-        uint64_t             gen = 0;
-        client_local_t      *local = NULL;
-
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        local->fd = fd_ref (fd);
-        loc_copy (&local->loc, loc);
-        local->flags = flags;
-
-        frame->local = local;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "CREATE %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for parent inode",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen);
-        hdr    = gf_hdr_new (req, pathlen + baselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->flags   = hton32 (gf_flags_from_flags (flags));
-        req->mode    = hton32 (mode);
-        req->par     = hton64 (par);
-        req->gen     = hton64 (gen);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_CREATE,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, fd, NULL, NULL);
-        return 0;
-
-}
-
-/**
- * client_open - open function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location of file
- * @flags: open flags
- * @mode: open modes
- *
- * external reference through client_protocol_xlator->fops->open
- */
-
-int
-client_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-             fd_t *fd, int32_t wbflags)
-{
-        int                 ret = -1;
-        gf_hdr_common_t    *hdr = NULL;
-        size_t              hdrlen = 0;
-        gf_fop_open_req_t  *req = NULL;
-        size_t              pathlen = 0;
-        ino_t               ino = 0;
-        uint64_t            gen = 0;
-        client_local_t     *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        local->fd = fd_ref (fd);
-        loc_copy (&local->loc, loc);
-        local->flags = flags;
-        local->wbflags = wbflags;
-
-        frame->local = local;
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "OPEN %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->gen   = hton64 (gen);
-        req->flags = hton32 (gf_flags_from_flags (flags));
-        req->wbflags = hton32 (wbflags);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_OPEN,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, fd);
-        return 0;
-
-}
-
-
-/**
- * client_stat - stat function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- *
- * external reference through client_protocol_xlator->fops->stat
- */
-
-int
-client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_hdr_common_t   *hdr = NULL;
-        gf_fop_stat_req_t *req = NULL;
-        size_t             hdrlen = -1;
-        int32_t            ret = -1;
-        size_t             pathlen = 0;
-        ino_t              ino = 0;
-        ino_t              gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "STAT %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_STAT,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_readlink - readlink function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- * @size:
- *
- * external reference through client_protocol_xlator->fops->readlink
- */
-int
-client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_readlink_req_t *req = NULL;
-        size_t                 hdrlen = -1;
-        int                    ret = -1;
-        size_t                 pathlen = 0;
-        ino_t                  ino = 0;
-        uint64_t               gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "READLINK %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-        req->size = hton32 (size);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_READLINK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND_STRICT (readlink, frame, -1, EINVAL,
-                             NULL, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_mknod - mknod function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @path: pathname of node
- * @mode:
- * @dev:
- *
- * external reference through client_protocol_xlator->fops->mknod
- */
-int
-client_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t dev, dict_t *params)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_mknod_req_t *req = NULL;
-        size_t              hdrlen = -1;
-        int                 ret = -1;
-        size_t              pathlen = 0;
-        size_t              baselen = 0;
-        ino_t               par = 0;
-        uint64_t            gen = 0;
-        client_local_t     *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, loc);
-
-        frame->local = local;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "MKNOD %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for parent",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen);
-        hdr    = gf_hdr_new (req, pathlen + baselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->par  = hton64 (par);
-        req->gen  = hton64 (gen);
-        req->mode = hton32 (mode);
-        req->dev  = hton64 (dev);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_MKNOD,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, loc->inode, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_mkdir - mkdir function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @path: pathname of directory
- * @mode:
- *
- * external reference through client_protocol_xlator->fops->mkdir
- */
-int
-client_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dict_t *params)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_mkdir_req_t *req = NULL;
-        size_t              hdrlen = -1;
-        int                 ret = -1;
-        size_t              pathlen = 0;
-        size_t              baselen = 0;
-        ino_t               par = 0;
-        uint64_t            gen = 0;
-        client_local_t     *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, loc);
-
-        frame->local = local;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "MKDIR %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for parent",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen);
-        hdr    = gf_hdr_new (req, pathlen + baselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->par  = hton64 (par);
-        req->gen  = hton64 (gen);
-        req->mode = hton32 (mode);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_MKDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, loc->inode, NULL);
-        return 0;
-
-}
-
-/**
- * client_unlink - unlink function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location of file
- *
- * external reference through client_protocol_xlator->fops->unlink
- */
-
-int
-client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_unlink_req_t *req = NULL;
-        size_t               hdrlen = -1;
-        int                  ret = -1;
-        size_t               pathlen = 0;
-        size_t               baselen = 0;
-        ino_t                par = 0;
-        uint64_t             gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "UNLINK %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for parent",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen);
-        hdr    = gf_hdr_new (req, pathlen + baselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->par  = hton64 (par);
-        req->gen  = hton64 (gen);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_UNLINK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-/**
- * client_rmdir - rmdir function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- *
- * external reference through client_protocol_xlator->fops->rmdir
- */
-
-int
-client_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_rmdir_req_t *req = NULL;
-        size_t              hdrlen = -1;
-        int                 ret = -1;
-        size_t              pathlen = 0;
-        size_t              baselen = 0;
-        ino_t               par = 0;
-        uint64_t            gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "RMDIR %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for parent",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen);
-        hdr    = gf_hdr_new (req, pathlen + baselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->par  = hton64 (par);
-        req->gen  = hton64 (gen);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_RMDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-
-/**
- * client_symlink - symlink function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @oldpath: pathname of target
- * @newpath: pathname of symlink
- *
- * external reference through client_protocol_xlator->fops->symlink
- */
-
-int
-client_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-                loc_t *loc, dict_t *params)
-{
-        int                   ret = -1;
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_symlink_req_t *req = NULL;
-        size_t                hdrlen  = 0;
-        size_t                pathlen = 0;
-        size_t                newlen  = 0;
-        size_t                baselen = 0;
-        ino_t                 par = 0;
-        uint64_t              gen = 0;
-        client_local_t       *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, loc);
-
-        frame->local = local;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-        newlen = STRLEN_0 (linkname);
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "SYMLINK %"PRId64"/%s (%s): failed to get remote inode"
-                        " number parent",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen + newlen);
-        hdr    = gf_hdr_new (req, pathlen + baselen + newlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->par =  hton64 (par);
-        req->gen =  hton64 (gen);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-        strcpy (req->linkname + pathlen + baselen, linkname);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_SYMLINK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, loc->inode, NULL);
-        return 0;
-
-}
-
-/**
- * client_rename - rename function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @oldloc: location of old pathname
- * @newloc: location of new pathname
- *
- * external reference through client_protocol_xlator->fops->rename
- */
-
-int
-client_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc)
-{
-        int                  ret = -1;
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_rename_req_t *req = NULL;
-        size_t               hdrlen = 0;
-        size_t               oldpathlen = 0;
-        size_t               oldbaselen = 0;
-        size_t               newpathlen = 0;
-        size_t               newbaselen = 0;
-        ino_t                oldpar = 0;
-        uint64_t             oldgen = 0;
-        ino_t                newpar = 0;
-        uint64_t             newgen = 0;
-
-        oldpathlen = STRLEN_0 (oldloc->path);
-        oldbaselen = STRLEN_0 (oldloc->name);
-        newpathlen = STRLEN_0 (newloc->path);
-        newbaselen = STRLEN_0 (newloc->name);
-        ret = inode_ctx_get2 (oldloc->parent, this, &oldpar, &oldgen);
-        if (oldloc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "RENAME %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for source parent",
-                        oldloc->parent->ino, oldloc->name, oldloc->path);
-                goto unwind;
-        }
-
-        ret = inode_ctx_get2 (newloc->parent, this, &newpar, &newgen);
-        if (newloc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "CREATE %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for destination parent",
-                        newloc->parent->ino, newloc->name, newloc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, (oldpathlen + oldbaselen +
-                                   newpathlen + newbaselen));
-        hdr    = gf_hdr_new (req, (oldpathlen + oldbaselen +
-                                   newpathlen + newbaselen));
-
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->oldpar = hton64 (oldpar);
-        req->oldgen = hton64 (oldgen);
-        req->newpar = hton64 (newpar);
-        req->newgen = hton64 (newgen);
-
-        strcpy (req->oldpath, oldloc->path);
-        strcpy (req->oldbname + oldpathlen, oldloc->name);
-        strcpy (req->newpath  + oldpathlen + oldbaselen, newloc->path);
-        strcpy (req->newbname + oldpathlen + oldbaselen + newpathlen,
-                newloc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_RENAME,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-/**
- * client_link - link function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @oldloc: location of old pathname
- * @newpath: new pathname
- *
- * external reference through client_protocol_xlator->fops->link
- */
-
-int
-client_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
-{
-        int                ret = -1;
-        gf_hdr_common_t   *hdr = NULL;
-        gf_fop_link_req_t *req = NULL;
-        size_t             hdrlen = 0;
-        size_t             oldpathlen = 0;
-        size_t             newpathlen = 0;
-        size_t             newbaselen = 0;
-        ino_t              oldino = 0;
-        uint64_t           oldgen = 0;
-        ino_t              newpar = 0;
-        uint64_t           newgen = 0;
-        client_local_t    *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, oldloc);
-
-        frame->local = local;
-
-        oldpathlen = STRLEN_0 (oldloc->path);
-        newpathlen = STRLEN_0 (newloc->path);
-        newbaselen = STRLEN_0 (newloc->name);
-
-        ret = inode_ctx_get2 (oldloc->inode, this, &oldino, &oldgen);
-        if (oldloc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "LINK %"PRId64"/%s (%s) ==> %"PRId64" (%s): "
-                        "failed to get remote inode number for source inode",
-                        newloc->parent->ino, newloc->name, newloc->path,
-                        oldloc->ino, oldloc->path);
-                goto unwind;
-        }
-
-        ret = inode_ctx_get2 (newloc->parent, this, &newpar, &newgen);
-        if (newloc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "LINK %"PRId64"/%s (%s) ==> %"PRId64" (%s): "
-                        "failed to get remote inode number destination parent",
-                        newloc->parent->ino, newloc->name, newloc->path,
-                        oldloc->ino, oldloc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, oldpathlen + newpathlen + newbaselen);
-        hdr    = gf_hdr_new (req, oldpathlen + newpathlen + newbaselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        strcpy (req->oldpath, oldloc->path);
-        strcpy (req->newpath  + oldpathlen, newloc->path);
-        strcpy (req->newbname + oldpathlen + newpathlen, newloc->name);
-
-        req->oldino = hton64 (oldino);
-        req->oldgen = hton64 (oldgen);
-        req->newpar = hton64 (newpar);
-        req->newgen = hton64 (newgen);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_LINK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, oldloc->inode, NULL);
-        return 0;
-}
-
-
-/**
- * client_truncate - truncate function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- * @offset:
- *
- * external reference through client_protocol_xlator->fops->truncate
- */
-
-int
-client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_truncate_req_t *req = NULL;
-        size_t                 hdrlen = -1;
-        int                    ret = -1;
-        size_t                 pathlen = 0;
-        ino_t                  ino = 0;
-        uint64_t               gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "TRUNCATE %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino    = hton64 (ino);
-        req->gen    = hton64 (gen);
-        req->offset = hton64 (offset);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_TRUNCATE,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_readv - readv function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @size:
- * @offset:
- *
- * external reference through client_protocol_xlator->fops->readv
- */
-
-int
-client_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_read_req_t  *req = NULL;
-        size_t              hdrlen = 0;
-        int64_t             remote_fd = -1;
-        int                 ret = -1;
-        client_fd_ctx_t    *fdctx = NULL;
-        client_conf_t      *conf = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx, EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL, 0, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx, EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL, 0, NULL);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd     = hton64 (remote_fd);
-        req->size   = hton32 (size);
-        req->offset = hton64 (offset);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_READ,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL, 0, NULL);
-        return 0;
-
-}
-
-/**
- * client_writev - writev function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @vector:
- * @count:
- * @offset:
- *
- * external reference through client_protocol_xlator->fops->writev
- */
-
-int
-client_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               struct iovec *vector, int32_t count, off_t offset,
-               struct iobref *iobref)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_write_req_t *req = NULL;
-        size_t              hdrlen = 0;
-        int64_t             remote_fd = -1;
-        int                 ret = -1;
-        client_fd_ctx_t    *fdctx = NULL;
-        client_conf_t      *conf = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd     = hton64 (remote_fd);
-        req->size   = hton32 (iov_length (vector, count));
-        req->offset = hton64 (offset);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_WRITE,
-                                    hdr, hdrlen, vector, count, iobref);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_statfs - statfs function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- *
- * external reference through client_protocol_xlator->fops->statfs
- */
-
-int
-client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_statfs_req_t *req = NULL;
-        size_t               hdrlen = -1;
-        int                  ret = -1;
-        size_t               pathlen = 0;
-        ino_t                ino = 0;
-        ino_t                gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-
-        if (loc->inode) {
-                ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-                if (loc->inode->ino && ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "STATFS %"PRId64" (%s): "
-                                "failed to get remote inode number",
-                                loc->inode->ino, loc->path);
-                        goto unwind;
-                }
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino = hton64 (ino);
-        req->gen = hton64 (gen);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_STATFS,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_flush - flush function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- *
- * external reference through client_protocol_xlator->fops->flush
- */
-
-int
-client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_flush_req_t  *req = NULL;
-        size_t               hdrlen = 0;
-        int64_t              remote_fd = -1;
-        int                  ret = -1;
-        client_fd_ctx_t     *fdctx = NULL;
-        client_conf_t       *conf = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd = hton64 (remote_fd);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FLUSH,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-/**
- * client_fsync - fsync function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @flags:
- *
- * external reference through client_protocol_xlator->fops->fsync
- */
-
-int
-client_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_fsync_req_t *req = NULL;
-        size_t              hdrlen = 0;
-        int64_t             remote_fd = -1;
-        int32_t             ret = -1;
-        client_fd_ctx_t    *fdctx = NULL;
-        client_conf_t      *conf = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd   = hton64 (remote_fd);
-        req->data = hton32 (flags);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FSYNC,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-int
-client_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                gf_xattrop_flags_t flags, dict_t *dict)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_xattrop_req_t *req = NULL;
-        size_t                hdrlen = 0;
-        size_t                dict_len = 0;
-        int32_t               ret = -1;
-        size_t                pathlen = 0;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-        char                 *buf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("client", this, unwind);
-
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        if (dict) {
-                ret = dict_allocate_and_serialize (dict, &buf, &dict_len);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to get serialized length of dict(%p)",
-                                dict);
-                        goto unwind;
-                }
-        }
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "XATTROP %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, dict_len + pathlen);
-        hdr    = gf_hdr_new (req, dict_len + pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->flags = hton32 (flags);
-        req->dict_len = hton32 (dict_len);
-        if (dict) {
-                memcpy (req->dict, buf, dict_len);
-                GF_FREE (buf);
-        }
-
-        req->ino = hton64 (ino);
-        req->gen = hton64 (gen);
-        strcpy (req->path + dict_len, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_XATTROP,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-int
-client_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 gf_xattrop_flags_t flags, dict_t *dict)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fxattrop_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        size_t                 dict_len = 0;
-        int64_t                remote_fd = -1;
-        int32_t                ret = -1;
-        ino_t                  ino = 0;
-        client_fd_ctx_t       *fdctx = NULL;
-        client_conf_t         *conf  = NULL;
-
-        conf = this->private;
-
-        if (dict) {
-                dict_len = dict_serialized_length (dict);
-                if (dict_len < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to get serialized length of dict(%p)",
-                                dict);
-                        goto unwind;
-                }
-        }
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. "
-                        "returning EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        ino = fd->inode->ino;
-        remote_fd = fdctx->remote_fd;
-
-        hdrlen = gf_hdr_len (req, dict_len);
-        hdr    = gf_hdr_new (req, dict_len);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->flags = hton32 (flags);
-        req->dict_len = hton32 (dict_len);
-        if (dict) {
-                ret = dict_serialize (dict, req->dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to serialize dictionary(%p)",
-                                dict);
-                        goto unwind;
-                }
-        }
-        req->fd = hton64 (remote_fd);
-        req->ino = hton64 (ino);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FXATTROP,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EBADFD, NULL);
-        return 0;
-
-}
-
-/**
- * client_setxattr - setxattr function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- * @dict: dictionary which contains key:value to be set.
- * @flags:
- *
- * external reference through client_protocol_xlator->fops->setxattr
- */
-
-int
-client_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 dict_t *dict, int32_t flags)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_setxattr_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        size_t                 dict_len = 0;
-        int                    ret = -1;
-        size_t                 pathlen = 0;
-        ino_t                  ino = 0;
-        uint64_t               gen = 0;
-
-        dict_len = dict_serialized_length (dict);
-        if (dict_len < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict(%p)",
-                        dict);
-                goto unwind;
-        }
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "SETXATTR %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, dict_len + pathlen);
-        hdr    = gf_hdr_new (req, dict_len + pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->gen   = hton64 (gen);
-        req->flags = hton32 (flags);
-        req->dict_len = hton32 (dict_len);
-
-        ret = dict_serialize (dict, req->dict);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to serialize dictionary(%p)",
-                        dict);
-                goto unwind;
-        }
-
-        strcpy (req->path + dict_len, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_SETXATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-}
-
-/**
- * client_fsetxattr - fsetxattr function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: fd
- * @dict: dictionary which contains key:value to be set.
- * @flags:
- *
- * external reference through client_protocol_xlator->fops->fsetxattr
- */
-
-int
-client_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  dict_t *dict, int32_t flags)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_fsetxattr_req_t *req = NULL;
-        size_t                  hdrlen = 0;
-        size_t                  dict_len = 0;
-        ino_t                   ino;
-        int                     ret = -1;
-        int64_t                 remote_fd = -1;
-        client_fd_ctx_t        *fdctx = NULL;
-        client_conf_t          *conf  = NULL;
-
-        conf = this->private;
-
-        dict_len = dict_serialized_length (dict);
-        if (dict_len < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict(%p)",
-                        dict);
-                goto unwind;
-        }
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        ino = fd->inode->ino;
-        remote_fd = fdctx->remote_fd;
-
-        hdrlen = gf_hdr_len (req, dict_len);
-        hdr    = gf_hdr_new (req, dict_len);
-
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->fd    = hton64 (remote_fd);
-        req->flags = hton32 (flags);
-        req->dict_len = hton32 (dict_len);
-
-        ret = dict_serialize (dict, req->dict);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to serialize dictionary(%p)",
-                        dict);
-                goto unwind;
-        }
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FSETXATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-}
-
-/**
- * client_getxattr - getxattr function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location structure
- *
- * external reference through client_protocol_xlator->fops->getxattr
- */
-
-int
-client_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name)
-{
-        int                    ret = -1;
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_getxattr_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        size_t                 pathlen = 0;
-        size_t                 namelen = 0;
-        ino_t                  ino = 0;
-        uint64_t               gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        if (name)
-                namelen = STRLEN_0 (name);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "GETXATTR %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + namelen);
-        hdr    = gf_hdr_new (req, pathlen + namelen);
-        GF_VALIDATE_OR_GOTO (frame->this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->gen   = hton64 (gen);
-        req->namelen = hton32 (namelen);
-        strcpy (req->path, loc->path);
-        if (name)
-                strcpy (req->name + pathlen, name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_GETXATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-/**
- * client_fgetxattr - fgetxattr function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: fd
- *
- * external reference through client_protocol_xlator->fops->fgetxattr
- */
-
-int
-client_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  const char *name)
-{
-        int                     ret = -1;
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_fgetxattr_req_t *req = NULL;
-        size_t                  hdrlen = 0;
-        int64_t                 remote_fd = -1;
-        size_t                  namelen = 0;
-        ino_t                   ino = 0;
-        client_fd_ctx_t        *fdctx = NULL;
-        client_conf_t          *conf  = NULL;
-
-        if (name)
-                namelen = STRLEN_0 (name);
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get remote fd. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        ino = fd->inode->ino;
-        remote_fd = fdctx->remote_fd;
-
-        hdrlen = gf_hdr_len (req, namelen);
-        hdr    = gf_hdr_new (req, namelen);
-
-        GF_VALIDATE_OR_GOTO (frame->this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->fd    = hton64 (remote_fd);
-        req->namelen = hton32 (namelen);
-
-        if (name)
-                strcpy (req->name, name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FGETXATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-/**
- * client_removexattr - removexattr function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location structure
- * @name:
- *
- * external reference through client_protocol_xlator->fops->removexattr
- */
-
-int
-client_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name)
-{
-        int                       ret = -1;
-        gf_hdr_common_t          *hdr = NULL;
-        gf_fop_removexattr_req_t *req = NULL;
-        size_t                    hdrlen = 0;
-        size_t                    namelen = 0;
-        size_t                    pathlen = 0;
-        ino_t                     ino = 0;
-        uint64_t                  gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        namelen = STRLEN_0 (name);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "REMOVEXATTR %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + namelen);
-        hdr    = gf_hdr_new (req, pathlen + namelen);
-        GF_VALIDATE_OR_GOTO (frame->this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->gen   = hton64 (gen);
-        strcpy (req->path, loc->path);
-        strcpy (req->name + pathlen, name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_REMOVEXATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-}
-
-/**
- * client_opendir - opendir function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location structure
- *
- * external reference through client_protocol_xlator->fops->opendir
- */
-
-int
-client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                fd_t *fd)
-{
-        gf_fop_opendir_req_t *req = NULL;
-        gf_hdr_common_t      *hdr = NULL;
-        size_t                hdrlen = 0;
-        int                   ret = -1;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-        size_t                pathlen = 0;
-        client_local_t       *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, loc);
-        local->fd = fd_ref (fd);
-
-        frame->local = local;
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "OPENDIR %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        pathlen = STRLEN_0 (loc->path);
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (frame->this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino = hton64 (ino);
-        req->gen = hton64 (gen);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_OPENDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, fd);
-        return 0;
-
-}
-
-/**
- * client_readdirp - readdirp function for client protocol
- * @frame: call frame
- * @this: this translator structure
- *
- * external reference through client_protocol_xlator->fops->readdirp
- */
-
-int
-client_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t offset)
-{
-        gf_hdr_common_t         *hdr = NULL;
-        gf_fop_readdirp_req_t   *req = NULL;
-        size_t                  hdrlen = 0;
-        int64_t                 remote_fd = -1;
-        int                     ret = -1;
-        client_fd_ctx_t         *fdctx = NULL;
-        client_conf_t           *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req->fd     = hton64 (remote_fd);
-        req->size   = hton32 (size);
-        req->offset = hton64 (offset);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_READDIRP,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EBADFD, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_readdir - readdir function for client protocol
- * @frame: call frame
- * @this: this translator structure
- *
- * external reference through client_protocol_xlator->fops->readdir
- */
-
-int
-client_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_readdir_req_t  *req = NULL;
-        size_t                 hdrlen = 0;
-        int64_t                remote_fd = -1;
-        int                    ret = -1;
-        client_fd_ctx_t       *fdctx = NULL;
-        client_conf_t         *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req->fd     = hton64 (remote_fd);
-        req->size   = hton32 (size);
-        req->offset = hton64 (offset);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_READDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EBADFD, NULL);
-        return 0;
-
-}
-
-/**
- * client_fsyncdir - fsyncdir function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @flags:
- *
- * external reference through client_protocol_xlator->fops->fsyncdir
- */
-
-int
-client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fsyncdir_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        int64_t                remote_fd = -1;
-        int32_t                ret = -1;
-        client_fd_ctx_t       *fdctx = NULL;
-        client_conf_t         *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->data = hton32 (flags);
-        req->fd   = hton64 (remote_fd);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FSYNCDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        STACK_UNWIND (frame, -1, EBADFD);
-        return 0;
-}
-
-/**
- * client_access - access function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location structure
- * @mode:
- *
- * external reference through client_protocol_xlator->fops->access
- */
-
-int
-client_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_access_req_t *req = NULL;
-        size_t               hdrlen = -1;
-        int                  ret = -1;
-        ino_t                ino = 0;
-        uint64_t             gen = 0;
-        size_t               pathlen = 0;
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "ACCESS %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        pathlen = STRLEN_0 (loc->path);
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-        req->mask = hton32 (mask);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_ACCESS,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-/**
- * client_ftrucate - ftruncate function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @offset: offset to truncate to
- *
- * external reference through client_protocol_xlator->fops->ftruncate
- */
-
-int
-client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  off_t offset)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_ftruncate_req_t *req = NULL;
-        int64_t                 remote_fd = -1;
-        size_t                  hdrlen = -1;
-        int                     ret = -1;
-        client_fd_ctx_t        *fdctx = NULL;
-        client_conf_t          *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd     = hton64 (remote_fd);
-        req->offset = hton64 (offset);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FTRUNCATE,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-/**
- * client_fstat - fstat function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- *
- * external reference through client_protocol_xlator->fops->fstat
- */
-
-int
-client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_fstat_req_t *req = NULL;
-        int64_t             remote_fd = -1;
-        size_t              hdrlen = -1;
-        int                 ret = -1;
-        client_fd_ctx_t    *fdctx = NULL;
-        client_conf_t      *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd = hton64 (remote_fd);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FSTAT,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-/**
- * client_lk - lk function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @cmd: lock command
- * @lock:
- *
- * external reference through client_protocol_xlator->fops->lk
- */
-
-int
-client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *flock)
-{
-        int              ret = -1;
-        gf_hdr_common_t *hdr = NULL;
-        gf_fop_lk_req_t *req = NULL;
-        size_t           hdrlen = 0;
-        int64_t          remote_fd = -1;
-        int32_t          gf_cmd = 0;
-        int32_t          gf_type = 0;
-        client_fd_ctx_t *fdctx = NULL;
-        client_conf_t   *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        if (cmd == F_GETLK || cmd == F_GETLK64)
-                gf_cmd = GF_LK_GETLK;
-        else if (cmd == F_SETLK || cmd == F_SETLK64)
-                gf_cmd = GF_LK_SETLK;
-        else if (cmd == F_SETLKW || cmd == F_SETLKW64)
-                gf_cmd = GF_LK_SETLKW;
-        else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unknown cmd (%d)!", gf_cmd);
-                goto unwind;
-        }
-
-        switch (flock->l_type) {
-        case F_RDLCK:
-                gf_type = GF_LK_F_RDLCK;
-                break;
-        case F_WRLCK:
-                gf_type = GF_LK_F_WRLCK;
-                break;
-        case F_UNLCK:
-                gf_type = GF_LK_F_UNLCK;
-                break;
-        }
-
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd   = hton64 (remote_fd);
-        req->cmd  = hton32 (gf_cmd);
-        req->type = hton32 (gf_type);
-        gf_flock_from_flock (&req->flock, flock);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_LK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-/**
- * client_inodelk - inodelk function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @inode: inode structure
- * @cmd: lock command
- * @lock: flock struct
- *
- * external reference through client_protocol_xlator->fops->inodelk
- */
-
-int
-client_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                loc_t *loc, int32_t cmd, struct gf_flock *flock)
-{
-        int                   ret = -1;
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_inodelk_req_t *req = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_cmd = 0;
-        int32_t               gf_type = 0;
-        ino_t                 ino  = 0;
-        uint64_t              gen  = 0;
-        size_t                pathlen = 0;
-        size_t                vollen  = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        vollen  = STRLEN_0 (volume);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "INODELK %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        if (cmd == F_GETLK || cmd == F_GETLK64)
-                gf_cmd = GF_LK_GETLK;
-        else if (cmd == F_SETLK || cmd == F_SETLK64)
-                gf_cmd = GF_LK_SETLK;
-        else if (cmd == F_SETLKW || cmd == F_SETLKW64)
-                gf_cmd = GF_LK_SETLKW;
-        else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unknown cmd (%d)!", gf_cmd);
-                goto unwind;
-        }
-
-        switch (flock->l_type) {
-        case F_RDLCK:
-                gf_type = GF_LK_F_RDLCK;
-                break;
-        case F_WRLCK:
-                gf_type = GF_LK_F_WRLCK;
-                break;
-        case F_UNLCK:
-                gf_type = GF_LK_F_UNLCK;
-                break;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + vollen);
-        hdr    = gf_hdr_new (req, pathlen + vollen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        strcpy (req->path, loc->path);
-        strcpy (req->path + pathlen, volume);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-
-        req->cmd  = hton32 (gf_cmd);
-        req->type = hton32 (gf_type);
-        gf_flock_from_flock (&req->flock, flock);
-
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST,
-                                    GF_PROTO_FOP_INODELK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-
-/**
- * client_finodelk - finodelk function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @inode: inode structure
- * @cmd: lock command
- * @lock: flock struct
- *
- * external reference through client_protocol_xlator->fops->finodelk
- */
-
-int
-client_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                 fd_t *fd, int32_t cmd, struct gf_flock *flock)
-{
-        int                    ret = -1;
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_finodelk_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        size_t                 vollen = 0;
-        int32_t                gf_cmd = 0;
-        int32_t                gf_type = 0;
-        int64_t                remote_fd = -1;
-        client_fd_ctx_t       *fdctx = NULL;
-        client_conf_t         *conf  = NULL;
-
-        vollen = STRLEN_0 (volume);
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        if (cmd == F_GETLK || cmd == F_GETLK64)
-                gf_cmd = GF_LK_GETLK;
-        else if (cmd == F_SETLK || cmd == F_SETLK64)
-                gf_cmd = GF_LK_SETLK;
-        else if (cmd == F_SETLKW || cmd == F_SETLKW64)
-                gf_cmd = GF_LK_SETLKW;
-        else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unknown cmd (%d)!", gf_cmd);
-                goto unwind;
-        }
-
-        switch (flock->l_type) {
-        case F_RDLCK:
-                gf_type = GF_LK_F_RDLCK;
-                break;
-        case F_WRLCK:
-                gf_type = GF_LK_F_WRLCK;
-                break;
-        case F_UNLCK:
-                gf_type = GF_LK_F_UNLCK;
-                break;
-        }
-
-        hdrlen = gf_hdr_len (req, vollen);
-        hdr    = gf_hdr_new (req, vollen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        strcpy (req->volume, volume);
-
-        req->fd = hton64 (remote_fd);
-
-        req->cmd  = hton32 (gf_cmd);
-        req->type = hton32 (gf_type);
-        gf_flock_from_flock (&req->flock, flock);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST,
-                                    GF_PROTO_FOP_FINODELK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-}
-
-
-int
-client_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-                loc_t *loc, const char *name, entrylk_cmd cmd,
-                entrylk_type type)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_entrylk_req_t *req = NULL;
-        size_t                pathlen = 0;
-        size_t                vollen  = 0;
-        size_t                hdrlen = -1;
-        int                   ret = -1;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-        size_t                namelen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        vollen  = STRLEN_0 (volume);
-
-        if (name)
-                namelen = STRLEN_0 (name);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "ENTRYLK %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + vollen + namelen);
-        hdr    = gf_hdr_new (req, pathlen + vollen + namelen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-        req->namelen = hton64 (namelen);
-
-        strcpy (req->path, loc->path);
-        if (name)
-                strcpy (req->name + pathlen, name);
-        strcpy (req->volume + pathlen + namelen, volume);
-
-        req->cmd  = hton32 (cmd);
-        req->type = hton32 (type);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_ENTRYLK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-
-int
-client_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-                 fd_t *fd, const char *name, entrylk_cmd cmd,
-                 entrylk_type type)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_fentrylk_req_t  *req = NULL;
-        int64_t                 remote_fd = -1;
-        size_t                  vollen  = 0;
-        size_t                  namelen = 0;
-        size_t                  hdrlen = -1;
-        int                     ret = -1;
-        client_fd_ctx_t        *fdctx = NULL;
-        client_conf_t          *conf  = NULL;
-
-        if (name)
-                namelen = STRLEN_0 (name);
-
-        conf = this->private;
-
-        vollen = STRLEN_0 (volume);
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, namelen + vollen);
-        hdr    = gf_hdr_new (req, namelen + vollen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd = hton64 (remote_fd);
-        req->namelen = hton64 (namelen);
-
-        if (name)
-                strcpy (req->name, name);
-
-        strcpy (req->volume + namelen, volume);
-
-        req->cmd  = hton32 (cmd);
-        req->type = hton32 (type);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FENTRYLK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-}
-
-/*
- * client_lookup - lookup function for client protocol
- * @frame: call frame
- * @this:
- * @loc: location
- *
- * not for external reference
- */
-
-int
-client_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xattr_req)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_lookup_req_t *req = NULL;
-        size_t               hdrlen = -1;
-        int                  ret = -1;
-        ino_t                ino = 0;
-        ino_t                par = 0;
-        uint64_t             gen = 0;
-        size_t               dictlen = 0;
-        size_t               pathlen = 0;
-        size_t               baselen = 0;
-        int32_t              op_ret = -1;
-        int32_t              op_errno = EINVAL;
-        client_local_t      *local = NULL;
-        char                *buf = NULL;
-
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->path, unwind);
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, loc);
-
-        frame->local = local;
-
-        if (loc->ino != 1 && loc->parent) {
-                ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-                if (loc->parent->ino && ret < 0) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "LOOKUP %"PRId64"/%s (%s): failed to get "
-                                "remote inode number for parent",
-                                loc->parent->ino, loc->name, loc->path);
-                        goto unwind;
-                }
-                GF_VALIDATE_OR_GOTO (this->name, loc->name, unwind);
-                baselen = STRLEN_0 (loc->name);
-        } else {
-                ino = 1;
-        }
-
-        pathlen = STRLEN_0 (loc->path);
-
-        if (xattr_req) {
-                ret = dict_allocate_and_serialize (xattr_req, &buf, &dictlen);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to get serialized length of dict(%p)",
-                                xattr_req);
-                        goto unwind;
-                }
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen + dictlen);
-        hdr    = gf_hdr_new (req, pathlen + baselen + dictlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->gen   = hton64 (gen);
-        req->par   = hton64 (par);
-        strcpy (req->path, loc->path);
-        if (baselen)
-                strcpy (req->path + pathlen, loc->name);
-
-        if (dictlen > 0) {
-                memcpy (req->dict + pathlen + baselen, buf, dictlen);
-                GF_FREE (buf);
-        }
-
-        req->dictlen = hton32 (dictlen);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_LOOKUP,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-
-unwind:
-        STACK_UNWIND (frame, op_ret, op_errno, (loc)?loc->inode:NULL, NULL, NULL);
-        return ret;
-}
-
-
-int
-client_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_setattr_req_t *req = NULL;
-        size_t                hdrlen = 0;
-        size_t                pathlen = 0;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-        int                   ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("client", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "SETATTR %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-        strcpy (req->path, loc->path);
-
-        gf_stat_from_iatt (&req->stbuf, stbuf);
-        req->valid = hton32 (valid);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_SETATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-int
-client_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fsetattr_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        int                    ret = -1;
-        client_fd_ctx_t       *fdctx = NULL;
-        int64_t                remote_fd = -1;
-        client_conf_t         *conf  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("client", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd = hton64 (remote_fd);
-
-        gf_stat_from_iatt (&req->stbuf, stbuf);
-        req->valid = hton32 (valid);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FSETATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        STACK_UNWIND (frame, -1, EINVAL, NULL, NULL);
-        return 0;
-}
-
-
-int
-client_fdctx_destroy (xlator_t *this, client_fd_ctx_t *fdctx)
-{
-        call_frame_t             *fr = NULL;
-        int32_t                   ret = -1;
-        gf_hdr_common_t          *hdr = NULL;
-        size_t                    hdrlen = 0;
-        gf_cbk_release_req_t     *req = NULL;
-        gf_cbk_releasedir_req_t  *reqdir = NULL;
-        int64_t                   remote_fd = -1;
-        int                       op = 0;
-
-        remote_fd = fdctx->remote_fd;
-
-        if (remote_fd == -1)
-                goto out;
-
-        if (fdctx->is_dir) {
-                hdrlen     = gf_hdr_len (reqdir, 0);
-                hdr        = gf_hdr_new (reqdir, 0);
-                op         = GF_CBK_RELEASEDIR;
-                reqdir     = gf_param (hdr);
-                reqdir->fd = hton64 (remote_fd);
-        } else {
-                hdrlen  = gf_hdr_len (req, 0);
-                hdr     = gf_hdr_new (req, 0);
-                op      = GF_CBK_RELEASE;
-                req     = gf_param (hdr);
-                req->fd = hton64 (remote_fd);
-        }
-
-        fr = create_frame (this, this->ctx->pool);
-
-        ret = protocol_client_xfer (fr, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_CBK_REQUEST, op,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-out:
-        inode_unref (fdctx->inode);
-        GF_FREE (fdctx);
-
-        return ret;
-}
-
-
-/**
- * client_releasedir - releasedir function for client protocol
- * @this: this translator structure
- * @fd: file descriptor structure
- *
- * external reference through client_protocol_xlator->cbks->releasedir
- */
-
-int
-client_releasedir (xlator_t *this, fd_t *fd)
-{
-        int64_t                remote_fd = -1;
-        client_conf_t         *conf = NULL;
-        client_fd_ctx_t       *fdctx = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_del_ctx (fd, this);
-                if (fdctx != NULL) {
-                        remote_fd = fdctx->remote_fd;
-
-                        /* fdctx->remote_fd == -1 indicates a reopen attempt
-                           in progress. Just mark ->released = 1 and let
-                           reopen_cbk handle releasing
-                        */
-
-                        if (remote_fd != -1)
-                                list_del_init (&fdctx->sfd_pos);
-
-                        fdctx->released = 1;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (remote_fd != -1)
-                client_fdctx_destroy (this, fdctx);
-
-        return 0;
-}
-
-
-/**
- * client_release - release function for client protocol
- * @this: this translator structure
- * @fd: file descriptor structure
- *
- * external reference through client_protocol_xlator->cbks->release
- *
- */
-int
-client_release (xlator_t *this, fd_t *fd)
-{
-        int64_t                remote_fd = -1;
-        client_conf_t         *conf = NULL;
-        client_fd_ctx_t       *fdctx = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_del_ctx (fd, this);
-                if (fdctx != NULL) {
-                        remote_fd = fdctx->remote_fd;
-
-                        /* fdctx->remote_fd == -1 indicates a reopen attempt
-                           in progress. Just mark ->released = 1 and let
-                           reopen_cbk handle releasing
-                        */
-
-                        if (remote_fd != -1)
-                                list_del_init (&fdctx->sfd_pos);
-
-                        fdctx->released = 1;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (remote_fd != -1)
-                client_fdctx_destroy (this, fdctx);
-
-        return 0;
-}
-
-/*
- * MGMT_OPS
- */
-
-/* Callbacks */
-
-int
-client_fxattrop_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_xattrop_rsp_t *rsp = NULL;
-        int32_t               op_ret   = 0;
-        int32_t               gf_errno = 0;
-        int32_t               op_errno = 0;
-        int32_t               dict_len = 0;
-        dict_t               *dict = NULL;
-        int32_t               ret = -1;
-        char                 *dictbuf = NULL;
-
-        rsp = gf_param (hdr);
-        GF_VALIDATE_OR_GOTO (frame->this->name, rsp, fail);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-
-        if (op_ret >= 0) {
-                op_ret = -1;
-                dict_len = ntoh32 (rsp->dict_len);
-
-                if (dict_len > 0) {
-                        dictbuf = memdup (rsp->dict, dict_len);
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dictbuf, fail);
-
-                        dict = dict_new();
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dict, fail);
-
-                        ret = dict_unserialize (dictbuf, dict_len, &dict);
-                        if (ret < 0) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "failed to serialize dictionary(%p)",
-                                        dict);
-                                op_errno = -ret;
-                                goto fail;
-                        } else {
-                                dict->extra_free = dictbuf;
-                                dictbuf = NULL;
-                        }
-                }
-                op_ret = 0;
-        }
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-
-fail:
-        STACK_UNWIND (frame, op_ret, op_errno, dict);
-
-        if (dictbuf)
-                GF_FREE (dictbuf);
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
-}
-
-
-int
-client_xattrop_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_xattrop_rsp_t *rsp = NULL;
-        int32_t               op_ret   = -1;
-        int32_t               gf_errno = EINVAL;
-        int32_t               op_errno = 0;
-        int32_t               dict_len = 0;
-        dict_t               *dict = NULL;
-        int32_t               ret = -1;
-        char                 *dictbuf = NULL;
-
-        rsp = gf_param (hdr);
-        GF_VALIDATE_OR_GOTO (frame->this->name, rsp, fail);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        if (op_ret >= 0) {
-                op_ret = -1;
-                dict_len = ntoh32 (rsp->dict_len);
-
-                if (dict_len > 0) {
-                        dictbuf = memdup (rsp->dict, dict_len);
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dictbuf, fail);
-
-                        dict = get_new_dict();
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dict, fail);
-                        dict_ref (dict);
-
-                        ret = dict_unserialize (dictbuf, dict_len, &dict);
-                        if (ret < 0) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "failed to serialize dictionary(%p)",
-                                        dict);
-                                goto fail;
-                        } else {
-                                dict->extra_free = dictbuf;
-                                dictbuf = NULL;
-                        }
-                }
-                op_ret = 0;
-        }
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-
-
-fail:
-        STACK_UNWIND (frame, op_ret, op_errno, dict);
-
-        if (dictbuf)
-                GF_FREE (dictbuf);
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
-}
-
-/*
- * client_create_cbk - create callback function for client protocol
- * @frame: call frame
- * @args: arguments in dictionary
- *
- * not for external reference
- */
-
-int
-client_create_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        gf_fop_create_rsp_t  *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-        fd_t                 *fd = NULL;
-        inode_t              *inode = NULL;
-        struct iatt           stbuf = {0, };
-        struct iatt           preparent = {0, };
-        struct iatt           postparent = {0, };
-        int64_t               remote_fd = 0;
-        int32_t               ret = -1;
-        client_local_t       *local = NULL;
-        client_conf_t        *conf = NULL;
-        client_fd_ctx_t      *fdctx = NULL;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-
-        local = frame->local; frame->local = NULL;
-        conf  = frame->this->private;
-        fd    = local->fd;
-        inode = local->loc.inode;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        if (op_ret >= 0) {
-                remote_fd = ntoh64 (rsp->fd);
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-
-                ino = stbuf.ia_ino;
-                gen = stbuf.ia_gen;
-        }
-
-        if (op_ret >= 0) {
-                ret = inode_ctx_put2 (local->loc.inode, frame->this, ino, gen);
-
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "CREATE %"PRId64"/%s (%s): failed to set "
-                                "remote inode number to inode ctx",
-                                local->loc.parent->ino, local->loc.name,
-                                local->loc.path);
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                        goto unwind_out;
-                }
-
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_client_fd_ctx_t);
-                if (!fdctx) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind_out;
-                }
-
-                fdctx->remote_fd = remote_fd;
-                fdctx->inode     = inode_ref (fd->inode);
-                fdctx->ino       = ino;
-                fdctx->gen       = gen;
-                fdctx->flags     = local->flags;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->mutex);
-        }
-unwind_out:
-        STACK_UNWIND (frame, op_ret, op_errno, fd, inode, &stbuf,
-                      &preparent, &postparent);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-
-/*
- * client_open_cbk - open callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-int
-client_open_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        int32_t              op_ret = -1;
-        int32_t              op_errno = ENOTCONN;
-        fd_t                *fd = NULL;
-        int64_t              remote_fd = 0;
-        gf_fop_open_rsp_t   *rsp = NULL;
-        client_local_t      *local = NULL;
-        client_conf_t       *conf = NULL;
-        client_fd_ctx_t     *fdctx = NULL;
-        ino_t                ino = 0;
-        uint64_t             gen = 0;
-
-
-        local = frame->local;
-
-        if (local->op) {
-                local->op (frame, hdr, hdrlen, iobuf);
-                return 0;
-        }
-
-        frame->local = NULL;
-        conf  = frame->this->private;
-        fd    = local->fd;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        if (op_ret >= 0) {
-                remote_fd = ntoh64 (rsp->fd);
-        }
-
-        if (op_ret >= 0) {
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_client_fd_ctx_t);
-                if (!fdctx) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind_out;
-                }
-
-                inode_ctx_get2 (fd->inode, frame->this, &ino, &gen);
-
-                fdctx->remote_fd = remote_fd;
-                fdctx->inode     = inode_ref (fd->inode);
-                fdctx->ino       = ino;
-                fdctx->gen       = gen;
-                fdctx->flags     = local->flags;
-                fdctx->wbflags   = local->wbflags;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->mutex);
-        }
-unwind_out:
-        STACK_UNWIND (frame, op_ret, op_errno, fd);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_stat_cbk - stat callback for client protocol
- * @frame: call frame
- * @args: arguments dictionary
- *
- * not for external reference
- */
-
-int
-client_stat_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        struct iatt        stbuf = {0, };
-        gf_fop_stat_rsp_t *rsp = NULL;
-        int32_t            op_ret = 0;
-        int32_t            op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &stbuf);
-
-        return 0;
-}
-
-
-/*
- * client_mknod_cbk - mknod callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_mknod_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_mknod_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-        struct iatt         stbuf = {0, };
-        inode_t            *inode = NULL;
-        client_local_t     *local = NULL;
-        int                 ret = 0;
-        struct iatt         preparent = {0,};
-        struct iatt         postparent = {0,};
-
-        local = frame->local;
-        frame->local = NULL;
-        inode = local->loc.inode;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                ret = inode_ctx_put2 (local->loc.inode, frame->this,
-                                      stbuf.ia_ino, stbuf.ia_gen);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "MKNOD %"PRId64"/%s (%s): failed to set remote"
-                                " inode number to inode ctx",
-                                local->loc.parent->ino, local->loc.name,
-                                local->loc.path);
-
-                        STACK_UNWIND (frame, -1, EINVAL, inode, NULL,
-                                      NULL, NULL);
-                        return 0;
-                }
-
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, inode, &stbuf,
-                      &preparent, &postparent);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_symlink_cbk - symlink callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_symlink_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_symlink_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-        struct iatt           stbuf = {0, };
-        struct iatt           preparent = {0,};
-        struct iatt           postparent = {0,};
-        inode_t              *inode = NULL;
-        client_local_t       *local = NULL;
-        int                   ret = 0;
-
-        local = frame->local;
-        frame->local = NULL;
-        inode = local->loc.inode;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                ret = inode_ctx_put2 (inode, frame->this,
-                                      stbuf.ia_ino, stbuf.ia_gen);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "SYMLINK %"PRId64"/%s (%s): failed to set "
-                                "remote inode number to inode ctx",
-                                local->loc.parent->ino, local->loc.name,
-                                local->loc.path);
-                        STACK_UNWIND (frame, -1, EINVAL, inode, NULL,
-                                      NULL, NULL);
-                        return 0;
-                }
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, inode, &stbuf,
-                      &preparent, &postparent);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_link_cbk - link callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_link_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_link_rsp_t *rsp = NULL;
-        int32_t            op_ret = 0;
-        int32_t            op_errno = 0;
-        struct iatt        stbuf = {0, };
-        inode_t           *inode = NULL;
-        client_local_t    *local = NULL;
-        struct iatt        preparent = {0,};
-        struct iatt        postparent = {0,};
-
-        local = frame->local;
-        frame->local = NULL;
-        inode = local->loc.inode;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, inode, &stbuf,
-                      &preparent, &postparent);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_truncate_cbk - truncate callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_truncate_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_truncate_rsp_t *rsp = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-        struct iatt         prestat = {0, };
-        struct iatt         poststat = {0, };
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->prestat, &prestat);
-                gf_stat_to_iatt (&rsp->poststat, &poststat);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &prestat, &poststat);
-
-        return 0;
-}
-
-/* client_fstat_cbk - fstat callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_fstat_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        struct iatt         stbuf = {0, };
-        gf_fop_fstat_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &stbuf);
-
-        return 0;
-}
-
-/*
- * client_ftruncate_cbk - ftruncate callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-int
-client_ftruncate_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iobuf *iobuf)
-{
-        gf_fop_ftruncate_rsp_t *rsp = NULL;
-        int32_t                 op_ret = 0;
-        int32_t                 op_errno = 0;
-        struct iatt             prestat = {0, };
-        struct iatt             poststat = {0, };
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->prestat, &prestat);
-                gf_stat_to_iatt (&rsp->poststat, &poststat);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &prestat, &poststat);
-
-        return 0;
-}
-
-
-/* client_readv_cbk - readv callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external referece
- */
-
-int
-client_readv_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_read_rsp_t  *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-        struct iovec        vector = {0, };
-        struct iatt         stbuf = {0, };
-        struct iobref      *iobref = NULL;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret != -1) {
-                iobref = iobref_new ();
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-                vector.iov_len  = op_ret;
-
-                if (op_ret > 0) {
-                        vector.iov_base = iobuf->ptr;
-                        iobref_add (iobref, iobuf);
-                }
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &vector, 1, &stbuf, iobref);
-
-        if (iobref)
-                iobref_unref (iobref);
-
-        if (iobuf)
-                iobuf_unref (iobuf);
-
-        return 0;
-}
-
-/*
- * client_write_cbk - write callback for client protocol
- * @frame: cal frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_write_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_write_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-        struct iatt         prestat = {0, };
-        struct iatt         poststat = {0, };
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_to_iatt (&rsp->prestat, &prestat);
-                gf_stat_to_iatt (&rsp->poststat, &poststat);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &prestat, &poststat);
-
-        return 0;
-}
-
-
-int
-client_readdirp_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_readdirp_rsp_t   *rsp = NULL;
-        int32_t                 op_ret = 0;
-        int32_t                 op_errno = 0;
-        uint32_t                buf_size = 0;
-        gf_dirent_t             entries;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        INIT_LIST_HEAD (&entries.list);
-        if (op_ret > 0) {
-                buf_size = ntoh32 (rsp->size);
-                gf_dirent_unserialize (&entries, rsp->buf, buf_size);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &entries);
-
-        gf_dirent_free (&entries);
-
-        return 0;
-}
-
-
-int
-client_readdir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_readdir_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-        uint32_t              buf_size = 0;
-        gf_dirent_t           entries;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        INIT_LIST_HEAD (&entries.list);
-        if (op_ret > 0) {
-                buf_size = ntoh32 (rsp->size);
-                gf_dirent_unserialize (&entries, rsp->buf, buf_size);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &entries);
-
-        gf_dirent_free (&entries);
-
-        return 0;
-}
-
-/*
- * client_fsync_cbk - fsync callback for client protocol
- *
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_fsync_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        struct iatt         prestat = {0, };
-        struct iatt         poststat = {0,};
-        gf_fop_fsync_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->prestat, &prestat);
-                gf_stat_to_iatt (&rsp->poststat, &poststat);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &prestat, &poststat);
-
-        return 0;
-}
-
-/*
- * client_unlink_cbk - unlink callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_unlink_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        gf_fop_unlink_rsp_t *rsp = NULL;
-        int32_t              op_ret = 0;
-        int32_t              op_errno = 0;
-        struct iatt          preparent = {0,};
-        struct iatt          postparent = {0,};
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &preparent, &postparent);
-
-        return 0;
-}
-
-/*
- * client_rename_cbk - rename callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_rename_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        struct iatt          stbuf = {0, };
-        gf_fop_rename_rsp_t *rsp = NULL;
-        int32_t              op_ret = 0;
-        int32_t              op_errno = 0;
-        struct iatt          preoldparent = {0, };
-        struct iatt          postoldparent = {0, };
-        struct iatt          prenewparent = {0, };
-        struct iatt          postnewparent = {0, };
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-                gf_stat_to_iatt (&rsp->preoldparent, &preoldparent);
-                gf_stat_to_iatt (&rsp->postoldparent, &postoldparent);
-                gf_stat_to_iatt (&rsp->prenewparent, &prenewparent);
-                gf_stat_to_iatt (&rsp->postnewparent, &postnewparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &stbuf, &preoldparent,
-                      &postoldparent, &prenewparent, &postnewparent);
-
-        return 0;
-}
-
-
-/*
- * client_readlink_cbk - readlink callback for client protocol
- *
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-int
-client_readlink_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_readlink_rsp_t *rsp = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-        char                  *link = NULL;
-        struct iatt            stbuf = {0,};
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret > 0) {
-                link = rsp->path;
-                gf_stat_to_iatt (&rsp->buf, &stbuf);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, link, &stbuf);
-        return 0;
-}
-
-/*
- * client_mkdir_cbk - mkdir callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_mkdir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_mkdir_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-        struct iatt         stbuf = {0, };
-        inode_t            *inode = NULL;
-        client_local_t     *local = NULL;
-        int                 ret = 0;
-        struct iatt         preparent = {0,};
-        struct iatt         postparent = {0,};
-
-        local = frame->local;
-        inode = local->loc.inode;
-        frame->local = NULL;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                ret = inode_ctx_put2 (inode, frame->this, stbuf.ia_ino,
-                                      stbuf.ia_gen);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "MKDIR %"PRId64"/%s (%s): failed to set "
-                                "remote inode number to inode ctx",
-                                local->loc.parent->ino, local->loc.name,
-                                local->loc.path);
-                        STACK_UNWIND (frame, -1, EINVAL, inode, NULL,
-                                      NULL, NULL);
-                        return 0;
-                }
-
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, inode, &stbuf,
-                      &preparent, &postparent);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_flush_cbk - flush callback for client protocol
- *
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_flush_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-/*
- * client_opendir_cbk - opendir callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_opendir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        int32_t               op_ret   = -1;
-        int32_t               op_errno = ENOTCONN;
-        fd_t                 *fd       = NULL;
-        int64_t               remote_fd = 0;
-        gf_fop_opendir_rsp_t *rsp       = NULL;
-        client_local_t       *local = NULL;
-        client_conf_t        *conf = NULL;
-        client_fd_ctx_t      *fdctx = NULL;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-
-
-        local = frame->local;
-
-        if (local->op) {
-                local->op (frame, hdr, hdrlen, iobuf);
-                return 0;
-        }
-
-        frame->local = NULL;
-        conf  = frame->this->private;
-        fd    = local->fd;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        if (op_ret >= 0) {
-                remote_fd = ntoh64 (rsp->fd);
-        }
-
-        if (op_ret >= 0) {
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_client_fd_ctx_t);
-                if (!fdctx) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind_out;
-                }
-
-                inode_ctx_get2 (fd->inode, frame->this, &ino, &gen);
-
-                fdctx->remote_fd = remote_fd;
-                fdctx->inode     = inode_ref (fd->inode);
-                fdctx->ino       = ino;
-                fdctx->gen       = gen;
-
-                fdctx->is_dir    = 1;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->mutex);
-        }
-unwind_out:
-        STACK_UNWIND (frame, op_ret, op_errno, fd);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_rmdir_cbk - rmdir callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_rmdir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_rmdir_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-        struct iatt         preparent = {0,};
-        struct iatt         postparent = {0,};
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &preparent, &postparent);
-
-        return 0;
-}
-
-/*
- * client_access_cbk - access callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_access_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        gf_fop_access_rsp_t *rsp = NULL;
-        int32_t              op_ret = 0;
-        int32_t              op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-/*
- * client_lookup_cbk - lookup callback for client protocol
- *
- * @frame: call frame
- * @args: arguments dictionary
- *
- * not for external reference
- */
-
-int
-client_lookup_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        struct iatt          stbuf = {0, };
-        struct iatt          postparent = {0, };
-        inode_t             *inode = NULL;
-        dict_t              *xattr = NULL;
-        gf_fop_lookup_rsp_t *rsp = NULL;
-        int32_t              op_ret = 0;
-        int32_t              op_errno = 0;
-        size_t               dict_len = 0;
-        char                *dictbuf = NULL;
-        int32_t              ret = -1;
-        int32_t              gf_errno = 0;
-        client_local_t      *local = NULL;
-        ino_t                oldino = 0;
-        uint64_t             oldgen = 0;
-
-        local = frame->local;
-        inode = local->loc.inode;
-        frame->local = NULL;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-
-        gf_stat_to_iatt (&rsp->postparent, &postparent);
-
-        if (op_ret == 0) {
-                op_ret = -1;
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                ret = inode_ctx_get2 (inode, frame->this, &oldino, &oldgen);
-                if (oldino != stbuf.ia_ino || oldgen != stbuf.ia_gen) {
-                        if (oldino) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "LOOKUP %"PRId64"/%s (%s): "
-                                        "inode number changed from "
-                                        "{%"PRId64",%"PRId64"} to {%"PRId64",%"PRId64"}",
-                                        local->loc.parent ?
-                                        local->loc.parent->ino : (uint64_t) 0,
-                                        local->loc.name,
-                                        local->loc.path,
-                                        oldgen, oldino, stbuf.ia_gen, stbuf.ia_ino);
-                                op_errno = ESTALE;
-                                goto fail;
-                        }
-
-                        ret = inode_ctx_put2 (inode, frame->this,
-                                              stbuf.ia_ino, stbuf.ia_gen);
-                        if (ret < 0) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "LOOKUP %"PRId64"/%s (%s) : "
-                                        "failed to set remote inode "
-                                        "number to inode ctx",
-                                        local->loc.parent ?
-                                        local->loc.parent->ino : (uint64_t) 0,
-                                        local->loc.name,
-                                        local->loc.path);
-                                op_errno = EINVAL;
-                                goto fail;
-                        }
-                }
-
-                dict_len = ntoh32 (rsp->dict_len);
-
-                if (dict_len > 0) {
-                        dictbuf = memdup (rsp->dict, dict_len);
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dictbuf, fail);
-
-                        xattr = dict_new();
-                        GF_VALIDATE_OR_GOTO (frame->this->name, xattr, fail);
-
-                        ret = dict_unserialize (dictbuf, dict_len, &xattr);
-                        if (ret < 0) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "%s (%"PRId64"): failed to "
-                                        "unserialize dictionary",
-                                        local->loc.path, inode->ino);
-                                goto fail;
-                        } else {
-                                xattr->extra_free = dictbuf;
-                                dictbuf = NULL;
-                        }
-                }
-                op_ret = 0;
-        }
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-
-fail:
-        STACK_UNWIND (frame, op_ret, op_errno, inode, &stbuf, xattr,
-                      &postparent);
-
-        client_local_wipe (local);
-
-        if (dictbuf)
-                GF_FREE (dictbuf);
-
-        if (xattr)
-                dict_unref (xattr);
-
-        return 0;
-}
-
-static int32_t
-client_setattr_cbk (call_frame_t *frame,gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        struct iatt           statpre = {0, };
-        struct iatt           statpost = {0, };
-        gf_fop_setattr_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->statpre, &statpre);
-                gf_stat_to_iatt (&rsp->statpost, &statpost);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &statpre, &statpost);
-
-        return 0;
-}
-
-static int32_t
-client_fsetattr_cbk (call_frame_t *frame,gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        struct iatt           statpre = {0, };
-        struct iatt           statpost = {0, };
-        gf_fop_setattr_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->statpre, &statpre);
-                gf_stat_to_iatt (&rsp->statpost, &statpost);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &statpre, &statpost);
-
-        return 0;
-}
-
-
-int
-gf_free_direntry (dir_entry_t *head)
-{
-        dir_entry_t *prev = NULL;
-        dir_entry_t *trav = NULL;
-
-        prev = head;
-        GF_VALIDATE_OR_GOTO ("client-protocol", prev, fail);
-
-        trav = head->next;
-        while (trav) {
-                prev->next = trav->next;
-                GF_FREE (trav->name);
-                if (IA_ISLNK (trav->buf.ia_type))
-                        GF_FREE (trav->link);
-                GF_FREE (trav);
-                trav = prev->next;
-        }
-        GF_FREE (head);
-fail:
-        return 0;
-}
-
-/*
- * client_statfs_cbk - statfs callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_statfs_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        struct statvfs       stbuf = {0, };
-        gf_fop_statfs_rsp_t *rsp = NULL;
-        int32_t              op_ret = 0;
-        int32_t              op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_statfs_to_statfs (&rsp->statfs, &stbuf);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &stbuf);
-
-        return 0;
-}
-
-/*
- * client_fsyncdir_cbk - fsyncdir callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_fsyncdir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-/*
- * client_setxattr_cbk - setxattr callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_setxattr_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_setxattr_rsp_t *rsp = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-/*
- * client_getxattr_cbk - getxattr callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_getxattr_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_getxattr_rsp_t *rsp = NULL;
-        int32_t                op_ret   = 0;
-        int32_t                gf_errno = 0;
-        int32_t                op_errno = 0;
-        int32_t                dict_len = 0;
-        dict_t                *dict = NULL;
-        int32_t                ret = -1;
-        char                  *dictbuf = NULL;
-        client_local_t        *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-
-        rsp = gf_param (hdr);
-        GF_VALIDATE_OR_GOTO (frame->this->name, rsp, fail);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-
-        if (op_ret >= 0) {
-                op_ret = -1;
-                dict_len = ntoh32 (rsp->dict_len);
-
-                if (dict_len > 0) {
-                        dictbuf = memdup (rsp->dict, dict_len);
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dictbuf, fail);
-
-                        dict = dict_new();
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dict, fail);
-
-                        ret = dict_unserialize (dictbuf, dict_len, &dict);
-                        if (ret < 0) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "%s (%"PRId64"): failed to "
-                                        "unserialize xattr dictionary",
-                                        local->loc.path,
-                                        local->loc.inode->ino);
-                                goto fail;
-                        } else {
-                                dict->extra_free = dictbuf;
-                                dictbuf = NULL;
-                        }
-                }
-                op_ret = 0;
-        }
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-fail:
-        STACK_UNWIND (frame, op_ret, op_errno, dict);
-
-        client_local_wipe (local);
-
-        if (dictbuf)
-                GF_FREE (dictbuf);
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
-}
-
-/*
- * client_removexattr_cbk - removexattr callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_removexattr_cbk (call_frame_t *frame, gf_hdr_common_t *hdr,
-                        size_t hdrlen, struct iobuf *iobuf)
-{
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-/*
- * client_lk_cbk - lk callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_lk_common_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iobuf *iobuf)
-{
-        struct gf_flock     lock = {0,};
-        gf_fop_lk_rsp_t *rsp = NULL;
-        int32_t          op_ret = 0;
-        int32_t          op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_flock_to_flock (&rsp->flock, &lock);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &lock);
-        return 0;
-}
-
-/*
- * client_gf_file_lk_cbk - gf_file_lk callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_inodelk_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_inodelk_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int
-client_finodelk_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_finodelk_rsp_t *rsp = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-        return 0;
-}
-
-/*
- * client_entrylk_cbk - entrylk callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_entrylk_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_entrylk_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-        return 0;
-}
-
-int
-client_fentrylk_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_fentrylk_rsp_t *rsp = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-        return 0;
-}
-
-
-
-/*
- * client_getspec - getspec function for client protocol
- * @frame: call frame
- * @this: client protocol xlator structure
- * @flag:
- *
- * external reference through client_protocol_xlator->fops->getspec
- */
-
-int
-client_getspec (call_frame_t *frame, xlator_t *this, const char *key,
-                int32_t flag)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_mop_getspec_req_t *req = NULL;
-        size_t                hdrlen = -1;
-        int                   keylen = 0;
-        int                   ret = -1;
-
-        if (key)
-                keylen = STRLEN_0 (key);
-
-        hdrlen = gf_hdr_len (req, keylen);
-        hdr    = gf_hdr_new (req, keylen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req        = gf_param (hdr);
-        req->flags = hton32 (flag);
-        req->keylen = hton32 (keylen);
-        if (keylen)
-                strcpy (req->key, key);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_MOP_REQUEST, GF_MOP_GETSPEC,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-/*
- * client_getspec_cbk - getspec callback for client protocol
- *
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_getspec_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_mop_getspec_rsp_t  *rsp = NULL;
-        char                  *spec_data = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-        int32_t                gf_errno = 0;
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-        rsp = gf_param (hdr);
-
-        if (op_ret >= 0) {
-                spec_data = rsp->spec;
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, spec_data);
-        return 0;
-}
-
-
-int
-client_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                  int32_t len)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_rchecksum_req_t *req = NULL;
-        size_t                  hdrlen = -1;
-        int                     ret = -1;
-
-        int64_t                remote_fd = -1;
-        client_fd_ctx_t       *fdctx     = NULL;
-        client_conf_t         *conf = NULL;
-
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        req    = gf_param (hdr);
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, 0, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, 0, NULL);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-
-        req->fd     = hton64 (remote_fd);
-        req->offset = hton64 (offset);
-        req->len    = hton32 (len);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_RCHECKSUM,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-}
-
-
-int
-client_rchecksum_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iobuf *iobuf)
-{
-        gf_fop_rchecksum_rsp_t *rsp = NULL;
-
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-        int32_t                gf_errno = 0;
-        uint32_t               weak_checksum = 0;
-        unsigned char         *strong_checksum = NULL;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-
-        if (op_ret >= 0) {
-                weak_checksum   = rsp->weak_checksum;
-                strong_checksum = rsp->strong_checksum;
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, weak_checksum, strong_checksum);
-
-        return 0;
-}
-
-
-/*
- * client_setspec_cbk - setspec callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_setspec_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-
-
-int
-protocol_client_reopendir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr,
-                               size_t hdrlen, struct iobuf *iobuf)
-{
-        int32_t              op_ret = -1;
-        int32_t              op_errno = ENOTCONN;
-        int64_t              remote_fd = -1;
-        gf_fop_open_rsp_t   *rsp = NULL;
-        client_local_t      *local = NULL;
-        client_conf_t       *conf = NULL;
-        client_fd_ctx_t     *fdctx = NULL;
-
-
-        local = frame->local; frame->local = NULL;
-        conf  = frame->this->private;
-        fdctx = local->fdctx;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        if (op_ret >= 0)
-                remote_fd = ntoh64 (rsp->fd);
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "reopendir on %s returned %d (%"PRId64")",
-                local->loc.path, op_ret, remote_fd);
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx->remote_fd = remote_fd;
-
-                if (!fdctx->released) {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                        fdctx = NULL;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx)
-                client_fdctx_destroy (frame->this, fdctx);
-
-        STACK_DESTROY (frame->root);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-
-
-int
-protocol_client_reopendir (xlator_t *this, client_fd_ctx_t *fdctx)
-{
-        int                    ret = -1;
-        gf_hdr_common_t       *hdr = NULL;
-        size_t                 hdrlen = 0;
-        gf_fop_opendir_req_t  *req = NULL;
-        size_t                 pathlen = 0;
-        client_local_t        *local = NULL;
-        inode_t               *inode = NULL;
-        char                  *path = NULL;
-        call_frame_t          *frame = NULL;
-
-        inode = fdctx->inode;
-
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        if (!local) {
-                goto out;
-        }
-
-        local->fdctx = fdctx;
-        local->op = protocol_client_reopendir_cbk;
-        local->loc.path = path; path = NULL;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                goto out;
-        }
-
-        pathlen = STRLEN_0 (local->loc.path);
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (fdctx->ino);
-        req->gen   = hton64 (fdctx->gen);
-
-        strcpy (req->path, local->loc.path);
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "attempting reopendir on %s", local->loc.path);
-
-        frame->local = local; local = NULL;
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_OPENDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-
-out:
-        if (frame)
-                STACK_DESTROY (frame->root);
-
-        if (local)
-                client_local_wipe (local);
-
-        if (path)
-                GF_FREE (path);
-
-        return 0;
-}
-
-
-int
-protocol_client_reopen_cbk (call_frame_t *frame, gf_hdr_common_t *hdr,
-                            size_t hdrlen, struct iobuf *iobuf)
-{
-        int32_t              op_ret = -1;
-        int32_t              op_errno = ENOTCONN;
-        int64_t              remote_fd = -1;
-        gf_fop_open_rsp_t   *rsp = NULL;
-        client_local_t      *local = NULL;
-        client_conf_t       *conf = NULL;
-        client_fd_ctx_t     *fdctx = NULL;
-
-
-        local = frame->local; frame->local = NULL;
-        conf  = frame->this->private;
-        fdctx = local->fdctx;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        if (op_ret >= 0)
-                remote_fd = ntoh64 (rsp->fd);
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "reopen on %s returned %d (%"PRId64")",
-                local->loc.path, op_ret, remote_fd);
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx->remote_fd = remote_fd;
-
-                if (!fdctx->released) {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                        fdctx = NULL;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx)
-                client_fdctx_destroy (frame->this, fdctx);
-
-        STACK_DESTROY (frame->root);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-
-int
-protocol_client_reopen (xlator_t *this, client_fd_ctx_t *fdctx)
-{
-        int                 ret = -1;
-        gf_hdr_common_t    *hdr = NULL;
-        size_t              hdrlen = 0;
-        gf_fop_open_req_t  *req = NULL;
-        size_t              pathlen = 0;
-        client_local_t     *local = NULL;
-        inode_t            *inode = NULL;
-        char               *path = NULL;
-        call_frame_t       *frame = NULL;
-
-        inode = fdctx->inode;
-
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        if (!local) {
-                goto out;
-        }
-
-        local->fdctx = fdctx;
-        local->op = protocol_client_reopen_cbk;
-        local->loc.path = path; path = NULL;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                goto out;
-        }
-
-        pathlen = STRLEN_0 (local->loc.path);
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (fdctx->ino);
-        req->gen   = hton64 (fdctx->gen);
-        req->flags = hton32 (gf_flags_from_flags (fdctx->flags));
-        req->wbflags = hton32 (fdctx->wbflags);
-        strcpy (req->path, local->loc.path);
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "attempting reopen on %s", local->loc.path);
-
-        frame->local = local; local = NULL;
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_OPEN,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-
-out:
-        if (frame)
-                STACK_DESTROY (frame->root);
-
-        if (local)
-                client_local_wipe (local);
-
-        if (path)
-                GF_FREE (path);
-
-        return 0;
-
-}
-
-
-int
-protocol_client_post_handshake (call_frame_t *frame, xlator_t *this)
-{
-        client_conf_t            *conf = NULL;
-        client_fd_ctx_t          *tmp = NULL;
-        client_fd_ctx_t          *fdctx = NULL;
-        xlator_list_t            *parent = NULL;
-        struct list_head          reopen_head;
-
-        conf = this->private;
-        INIT_LIST_HEAD (&reopen_head);
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                list_for_each_entry_safe (fdctx, tmp, &conf->saved_fds,
-                                          sfd_pos) {
-                        if (fdctx->remote_fd != -1)
-                                continue;
-
-                        list_del (&fdctx->sfd_pos);
-                        list_add_tail (&fdctx->sfd_pos, &reopen_head);
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        list_for_each_entry_safe (fdctx, tmp, &reopen_head, sfd_pos) {
-                list_del_init (&fdctx->sfd_pos);
-
-                if (fdctx->is_dir)
-                        protocol_client_reopendir (this, fdctx);
-                else
-                        protocol_client_reopen (this, fdctx);
-        }
-
-        parent = this->parents;
-
-        while (parent) {
-                xlator_notify (parent->xlator, GF_EVENT_CHILD_UP,
-                               this);
-                parent = parent->next;
-        }
-
-        return 0;
-}
-
-/*
- * client_setvolume_cbk - setvolume callback for client protocol
- * @frame:  call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_setvolume_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iobuf *iobuf)
-{
-        client_conf_t          *conf = NULL;
-        gf_mop_setvolume_rsp_t *rsp = NULL;
-        client_connection_t    *conn = NULL;
-        glusterfs_ctx_t        *ctx = NULL;
-        xlator_t               *this = NULL;
-        xlator_list_t          *parent = NULL;
-        transport_t            *trans = NULL;
-        dict_t                 *reply = NULL;
-        char                   *remote_subvol = NULL;
-        char                   *remote_error = NULL;
-        char                   *process_uuid = NULL;
-        int32_t                 ret = -1;
-        int32_t                 op_ret   = -1;
-        int32_t                 op_errno = EINVAL;
-        int32_t                 dict_len = 0;
-        transport_t            *peer_trans = NULL;
-        uint64_t                peer_trans_int = 0;
-
-        trans = frame->local; frame->local = NULL;
-        this  = frame->this;
-        conn  = trans->xl_private;
-        conf  = this->private;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setvolume failed (%s)",
-                        strerror (op_errno));
-                goto out;
-        }
-
-        reply = dict_new ();
-        GF_VALIDATE_OR_GOTO (this->name, reply, out);
-
-        dict_len = ntoh32 (rsp->dict_len);
-        ret = dict_unserialize (rsp->buf, dict_len, &reply);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "failed to unserialize buffer(%p) to dictionary",
-                        rsp->buf);
-                goto out;
-        }
-
-        ret = dict_get_str (reply, "ERROR", &remote_error);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get ERROR string from reply dictionary");
-        }
-
-        ret = dict_get_str (reply, "process-uuid", &process_uuid);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get 'process-uuid' from reply dictionary");
-        }
-
-        if (op_ret < 0) {
-                gf_log (trans->xl->name, GF_LOG_ERROR,
-                        "SETVOLUME on remote-host failed: %s",
-                        remote_error ? remote_error : strerror (op_errno));
-                errno = op_errno;
-                if (op_errno == ESTALE) {
-                        parent = trans->xl->parents;
-                        while (parent) {
-                                xlator_notify (parent->xlator,
-                                               GF_EVENT_VOLFILE_MODIFIED,
-                                               trans->xl);
-                                parent = parent->next;
-                        }
-                }
-
-        } else {
-                ret = dict_get_str (this->options, "remote-subvolume",
-                                    &remote_subvol);
-                if (!remote_subvol)
-                        goto out;
-
-                ctx = this->ctx;
-
-                if (process_uuid && !strcmp (ctx->process_uuid,process_uuid)) {
-                        ret = dict_get_uint64 (reply, "transport-ptr",
-                                               &peer_trans_int);
-
-                        peer_trans = (void *) (long) (peer_trans_int);
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "attaching to the local volume '%s'",
-                                remote_subvol);
-
-                        transport_setpeer (trans, peer_trans);
-
-                }
-
-                gf_log (trans->xl->name, GF_LOG_INFO,
-                        "Connected to %s, attached "
-                        "to remote volume '%s'.",
-                        trans->peerinfo.identifier, remote_subvol);
-
-                pthread_mutex_lock (&(conn->lock));
-                {
-                        conn->connected = 1;
-                }
-                pthread_mutex_unlock (&(conn->lock));
-
-                protocol_client_post_handshake (frame, frame->this);
-        }
-
-        conf->connecting = 0;
-out:
-
-        if (-1 == op_ret) {
-                /* Let the connection/re-connection happen in
-                 * background, for now, don't hang here,
-                 * tell the parents that i am all ok..
-                 */
-                parent = trans->xl->parents;
-                while (parent) {
-                        xlator_notify (parent->xlator,
-                                       GF_EVENT_CHILD_CONNECTING, trans->xl);
-                        parent = parent->next;
-                }
-                conf->connecting= 1;
-        }
-
-        STACK_DESTROY (frame->root);
-
-        if (reply)
-                dict_unref (reply);
-
-        return op_ret;
-}
-
-/*
- * client_enosys_cbk -
- * @frame: call frame
- *
- * not for external reference
- */
-
-int
-client_enosys_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
-void
-client_protocol_reconnect (void *trans_ptr)
-{
-        transport_t         *trans = NULL;
-        client_connection_t *conn = NULL;
-        struct timeval       tv = {0, 0};
-        int32_t              ret = 0;
-
-        trans = trans_ptr;
-        conn  = trans->xl_private;
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (conn->reconnect)
-                        gf_timer_call_cancel (trans->xl->ctx,
-                                              conn->reconnect);
-                conn->reconnect = 0;
-
-                if (conn->connected == 0) {
-                        tv.tv_sec = 10;
-
-                        gf_log (trans->xl->name, GF_LOG_TRACE,
-                                "attempting reconnect");
-                        ret = transport_connect (trans);
-
-                        conn->reconnect =
-                                gf_timer_call_after (trans->xl->ctx, tv,
-                                                     client_protocol_reconnect,
-                                                     trans);
-                } else {
-                        gf_log (trans->xl->name, GF_LOG_TRACE,
-                                "breaking reconnect chain");
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        if (ret == -1 && errno != EINPROGRESS) {
-                default_notify (trans->xl, GF_EVENT_CHILD_DOWN, NULL);
-        }
-}
-
-int
-protocol_client_mark_fd_bad (xlator_t *this)
-{
-        client_conf_t            *conf = NULL;
-        client_fd_ctx_t          *tmp = NULL;
-        client_fd_ctx_t          *fdctx = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                list_for_each_entry_safe (fdctx, tmp, &conf->saved_fds,
-                                          sfd_pos) {
-                        fdctx->remote_fd = -1;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        return 0;
-}
-
-/*
- * client_protocol_cleanup - cleanup function
- * @trans: transport object
- *
- */
-
-int
-protocol_client_cleanup (transport_t *trans)
-{
-        client_connection_t    *conn = NULL;
-        struct saved_frames    *saved_frames = NULL;
-
-        conn = trans->xl_private;
-
-        gf_log (trans->xl->name, GF_LOG_TRACE,
-                "cleaning up state in transport object %p", trans);
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                saved_frames = conn->saved_frames;
-                conn->saved_frames = gf_client_saved_frames_new ();
-
-                /* bailout logic cleanup */
-                if (conn->timer) {
-                        gf_timer_call_cancel (trans->xl->ctx, conn->timer);
-                        conn->timer = NULL;
-                }
-
-                if (conn->reconnect == NULL) {
-                        /* :O This part is empty.. any thing missing? */
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        gf_client_saved_frames_destroy (trans->xl, saved_frames,
-                                        gf_fops, gf_mops, gf_cbks);
-
-        return 0;
-}
-
-
-/* cbk callbacks */
-int
-client_releasedir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr,
-                       size_t hdrlen, struct iobuf *iobuf)
-{
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
-int
-client_release_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
-int
-client_forget_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        gf_log ("", GF_LOG_CRITICAL, "fop not implemented");
-        return 0;
-}
-
-
-int
-client_log_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_log ("", GF_LOG_CRITICAL, "fop not implemented");
-        return 0;
-}
-
-
-static gf_op_t gf_fops[] = {
-        [GF_PROTO_FOP_STAT]           =  client_stat_cbk,
-        [GF_PROTO_FOP_READLINK]       =  client_readlink_cbk,
-        [GF_PROTO_FOP_MKNOD]          =  client_mknod_cbk,
-        [GF_PROTO_FOP_MKDIR]          =  client_mkdir_cbk,
-        [GF_PROTO_FOP_UNLINK]         =  client_unlink_cbk,
-        [GF_PROTO_FOP_RMDIR]          =  client_rmdir_cbk,
-        [GF_PROTO_FOP_SYMLINK]        =  client_symlink_cbk,
-        [GF_PROTO_FOP_RENAME]         =  client_rename_cbk,
-        [GF_PROTO_FOP_LINK]           =  client_link_cbk,
-        [GF_PROTO_FOP_TRUNCATE]       =  client_truncate_cbk,
-        [GF_PROTO_FOP_OPEN]           =  client_open_cbk,
-        [GF_PROTO_FOP_READ]           =  client_readv_cbk,
-        [GF_PROTO_FOP_WRITE]          =  client_write_cbk,
-        [GF_PROTO_FOP_STATFS]         =  client_statfs_cbk,
-        [GF_PROTO_FOP_FLUSH]          =  client_flush_cbk,
-        [GF_PROTO_FOP_FSYNC]          =  client_fsync_cbk,
-        [GF_PROTO_FOP_SETXATTR]       =  client_setxattr_cbk,
-        [GF_PROTO_FOP_GETXATTR]       =  client_getxattr_cbk,
-        [GF_PROTO_FOP_REMOVEXATTR]    =  client_removexattr_cbk,
-        [GF_PROTO_FOP_OPENDIR]        =  client_opendir_cbk,
-        [GF_PROTO_FOP_FSYNCDIR]       =  client_fsyncdir_cbk,
-        [GF_PROTO_FOP_ACCESS]         =  client_access_cbk,
-        [GF_PROTO_FOP_CREATE]         =  client_create_cbk,
-        [GF_PROTO_FOP_FTRUNCATE]      =  client_ftruncate_cbk,
-        [GF_PROTO_FOP_FSTAT]          =  client_fstat_cbk,
-        [GF_PROTO_FOP_LK]             =  client_lk_common_cbk,
-        [GF_PROTO_FOP_LOOKUP]         =  client_lookup_cbk,
-        [GF_PROTO_FOP_READDIR]        =  client_readdir_cbk,
-        [GF_PROTO_FOP_READDIRP]       =  client_readdirp_cbk,
-        [GF_PROTO_FOP_INODELK]        =  client_inodelk_cbk,
-        [GF_PROTO_FOP_FINODELK]       =  client_finodelk_cbk,
-        [GF_PROTO_FOP_ENTRYLK]        =  client_entrylk_cbk,
-        [GF_PROTO_FOP_FENTRYLK]       =  client_fentrylk_cbk,
-        [GF_PROTO_FOP_RCHECKSUM]      =  client_rchecksum_cbk,
-        [GF_PROTO_FOP_XATTROP]        =  client_xattrop_cbk,
-        [GF_PROTO_FOP_FXATTROP]       =  client_fxattrop_cbk,
-        [GF_PROTO_FOP_SETATTR]        =  client_setattr_cbk,
-        [GF_PROTO_FOP_FSETATTR]       =  client_fsetattr_cbk
-};
-
-static gf_op_t gf_mops[] = {
-        [GF_MOP_SETVOLUME]        =  client_setvolume_cbk,
-        [GF_MOP_GETVOLUME]        =  client_enosys_cbk,
-        [GF_MOP_SETSPEC]          =  client_setspec_cbk,
-        [GF_MOP_GETSPEC]          =  client_getspec_cbk,
-        [GF_MOP_PING]             =  client_ping_cbk,
-        [GF_MOP_LOG]              =  client_log_cbk
-};
-
-static gf_op_t gf_cbks[] = {
-        [GF_CBK_FORGET]           = client_forget_cbk,
-        [GF_CBK_RELEASE]          = client_release_cbk,
-        [GF_CBK_RELEASEDIR]       = client_releasedir_cbk
-};
-
-/*
- * client_protocol_interpret - protocol interpreter
- * @trans: transport object
- * @blk: data block
- *
- */
-int
-protocol_client_interpret (xlator_t *this, transport_t *trans,
-                           char *hdr_p, size_t hdrlen, struct iobuf *iobuf)
-{
-        int                  ret = -1;
-        call_frame_t        *frame = NULL;
-        gf_hdr_common_t     *hdr = NULL;
-        uint64_t             callid = 0;
-        int                  type = -1;
-        int                  op = -1;
-        client_connection_t *conn = NULL;
-
-        conn  = trans->xl_private;
-
-        hdr  = (gf_hdr_common_t *)hdr_p;
-
-        type   = ntoh32 (hdr->type);
-        op     = ntoh32 (hdr->op);
-        callid = ntoh64 (hdr->callid);
-
-        frame  = lookup_frame (trans, op, type, callid);
-        if (frame == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "no frame for callid=%"PRId64" type=%d op=%d",
-                        callid, type, op);
-                return 0;
-        }
-
-        switch (type) {
-        case GF_OP_TYPE_FOP_REPLY:
-                if ((op > GF_PROTO_FOP_MAXVALUE) ||
-                    (op < 0)) {
-                        gf_log (trans->xl->name, GF_LOG_WARNING,
-                                "invalid fop '%d'", op);
-                } else {
-                        ret = gf_fops[op] (frame, hdr, hdrlen, iobuf);
-                }
-                break;
-        case GF_OP_TYPE_MOP_REPLY:
-                if ((op > GF_MOP_MAXVALUE) ||
-                    (op < 0)) {
-                        gf_log (trans->xl->name, GF_LOG_WARNING,
-                                "invalid fop '%d'", op);
-                } else {
-                        ret = gf_mops[op] (frame, hdr, hdrlen, iobuf);
-                }
-                break;
-        case GF_OP_TYPE_CBK_REPLY:
-                if ((op > GF_CBK_MAXVALUE) ||
-                    (op < 0)) {
-                        gf_log (trans->xl->name, GF_LOG_WARNING,
-                                "invalid cbk '%d'", op);
-                } else {
-                        ret = gf_cbks[op] (frame, hdr, hdrlen, iobuf);
-                }
-                break;
-        default:
-                gf_log (trans->xl->name, GF_LOG_DEBUG,
-                        "invalid packet type: %d", type);
-                break;
-        }
-
-        return ret;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret       = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_client_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-
-/*
- * init - initiliazation function. called during loading of client protocol
- * @this:
- *
- */
-
-int
-init (xlator_t *this)
-{
-        transport_t               *trans = NULL;
-        client_conf_t             *conf = NULL;
-        client_connection_t       *conn = NULL;
-        int32_t                    frame_timeout = 0;
-        int32_t                    ping_timeout = 0;
-        data_t                    *remote_subvolume = NULL;
-        int32_t                    ret = -1;
-        int                        i = 0;
-
-        if (this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: client protocol translator cannot have any "
-                        "subvolumes");
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Volume is dangling. ");
-        }
-
-        remote_subvolume = dict_get (this->options, "remote-subvolume");
-        if (remote_subvolume == NULL) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Option 'remote-subvolume' is not specified.");
-                goto out;
-        }
-
-        ret = dict_get_int32 (this->options, "frame-timeout",
-                              &frame_timeout);
-        if (ret >= 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setting frame-timeout to %d", frame_timeout);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "defaulting frame-timeout to 30mins");
-                frame_timeout = 1800;
-        }
-
-        ret = dict_get_int32 (this->options, "ping-timeout",
-                              &ping_timeout);
-        if (ret >= 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setting ping-timeout to %d", ping_timeout);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "defaulting ping-timeout to 42");
-                ping_timeout = GF_UNIVERSAL_ANSWER;
-        }
-
-        conf = GF_CALLOC (1, sizeof (client_conf_t),
-                          gf_client_mt_client_conf_t);
-
-        protocol_common_init ();
-
-        pthread_mutex_init (&conf->mutex, NULL);
-        INIT_LIST_HEAD (&conf->saved_fds);
-
-        this->private = conf;
-
-        for (i = 0; i < CHANNEL_MAX; i++) {
-                if (CHANNEL_LOWLAT == i) {
-                        dict_set (this->options, "transport.socket.lowlat",
-                                  data_from_dynstr (gf_strdup ("true")));
-                }
-                trans = transport_load (this->options, this);
-                if (trans == NULL) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Failed to load transport");
-                        ret = -1;
-                        goto out;
-                }
-
-                conn = GF_CALLOC (1, sizeof (*conn),
-                                  gf_client_mt_client_connection_t);
-
-                conn->saved_frames = gf_client_saved_frames_new ();
-
-                conn->callid = 1;
-
-                conn->frame_timeout = frame_timeout;
-                conn->ping_timeout = ping_timeout;
-
-                pthread_mutex_init (&conn->lock, NULL);
-
-                trans->xl_private = conn;
-                conf->transport[i] = transport_ref (trans);
-        }
-
-#ifndef GF_DARWIN_HOST_OS
-        {
-                struct rlimit lim;
-
-                lim.rlim_cur = 1048576;
-                lim.rlim_max = 1048576;
-
-                ret = setrlimit (RLIMIT_NOFILE, &lim);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "WARNING: Failed to set 'ulimit -n 1M': %s",
-                                strerror(errno));
-                        lim.rlim_cur = 65536;
-                        lim.rlim_max = 65536;
-
-                        ret = setrlimit (RLIMIT_NOFILE, &lim);
-                        if (ret == -1) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Failed to set max open fd to 64k: %s",
-                                        strerror(errno));
-                        } else {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "max open fd set to 64k");
-                        }
-
-                }
-        }
-#endif
-        ret = 0;
-out:
-        return ret;
-}
-
-/*
- * fini - finish function called during unloading of client protocol
- * @this:
- *
- */
-void
-fini (xlator_t *this)
-{
-        /* TODO: Check if its enough.. how to call transport's fini () */
-        client_conf_t *conf = NULL;
-
-        conf = this->private;
-        this->private = NULL;
-
-        if (conf) {
-                GF_FREE (conf);
-        }
-        return;
-}
-
-
-int
-protocol_client_handshake (xlator_t *this, transport_t *trans)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_mop_setvolume_req_t *req = NULL;
-        dict_t                 *options = NULL;
-        int32_t                 ret = -1;
-        int                     hdrlen = 0;
-        int                     dict_len = 0;
-        call_frame_t           *fr = NULL;
-        char                   *process_uuid_xl;
-
-        options = this->options;
-        ret = dict_set_str (options, "protocol-version", GF_PROTOCOL_VERSION);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set protocol version(%s) in handshake msg",
-                        GF_PROTOCOL_VERSION);
-        }
-
-        ret = gf_asprintf (&process_uuid_xl, "%s-%s", this->ctx->process_uuid,
-                           this->name);
-        if (-1 == ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "asprintf failed while setting process_uuid");
-                goto fail;
-        }
-        ret = dict_set_dynstr (options, "process-uuid",
-                               process_uuid_xl);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set process-uuid(%s) in handshake msg",
-                        process_uuid_xl);
-        }
-
-        if (this->ctx->cmd_args.volfile_server) {
-                if (this->ctx->cmd_args.volfile_id)
-                        ret = dict_set_str (options, "volfile-key",
-                                            this->ctx->cmd_args.volfile_id);
-                ret = dict_set_uint32 (options, "volfile-checksum",
-                                       this->graph->volfile_checksum);
-        }
-
-        dict_len = dict_serialized_length (options);
-        if (dict_len < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict(%p)",
-                        options);
-                ret = dict_len;
-                goto fail;
-        }
-
-        hdrlen = gf_hdr_len (req, dict_len);
-        hdr    = gf_hdr_new (req, dict_len);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, fail);
-
-        req    = gf_param (hdr);
-
-        ret = dict_serialize (options, req->buf);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to serialize dictionary(%p)",
-                        options);
-                goto fail;
-        }
-
-        req->dict_len = hton32 (dict_len);
-        fr  = create_frame (this, this->ctx->pool);
-        GF_VALIDATE_OR_GOTO (this->name, fr, fail);
-
-        fr->local = trans;
-        ret = protocol_client_xfer (fr, this, trans,
-                                    GF_OP_TYPE_MOP_REQUEST, GF_MOP_SETVOLUME,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-fail:
-        if (hdr)
-                GF_FREE (hdr);
-        return ret;
-}
-
-
-int
-protocol_client_pollout (xlator_t *this, transport_t *trans)
-{
-        client_conf_t *conf = NULL;
-
-        conf = trans->xl->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                gettimeofday (&conf->last_sent, NULL);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        return 0;
-}
-
-
-int
-protocol_client_pollin (xlator_t *this, transport_t *trans)
-{
-        client_conf_t *conf = NULL;
-        int            ret = -1;
-        struct iobuf  *iobuf = NULL;
-        char          *hdr = NULL;
-        size_t         hdrlen = 0;
-
-        conf = trans->xl->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                gettimeofday (&conf->last_received, NULL);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        ret = transport_receive (trans, &hdr, &hdrlen, &iobuf);
-
-        if (ret == 0)
-        {
-                ret = protocol_client_interpret (this, trans, hdr, hdrlen,
-                                                 iobuf);
-        }
-
-        /* TODO: use mem-pool */
-        GF_FREE (hdr);
-
-        return ret;
-}
-
-int
-client_priv_dump (xlator_t *this)
-{
-        client_conf_t   *conf = NULL;
-        int             ret   = -1;
-        client_fd_ctx_t *tmp = NULL;
-        int             i = 0;
-        char            key[GF_DUMP_MAX_BUF_LEN];
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-
-        if (!this)
-                return -1;
-
-        conf = this->private;
-        if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
-                return -1;
-        }
-
-        ret = pthread_mutex_trylock(&conf->mutex);
-        if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to lock client %s"
-                       " errno: %d", this->name, errno);
-                return -1;
-        }
-
-        gf_proc_dump_build_key(key_prefix, "xlator.protocol.client",
-                               "%s.priv", this->name);
-
-        gf_proc_dump_add_section(key_prefix);
-
-        list_for_each_entry(tmp, &conf->saved_fds, sfd_pos) {
-                gf_proc_dump_build_key(key, key_prefix,
-                                       "fd.%d.remote_fd", ++i);
-                gf_proc_dump_write(key, "%d", tmp->remote_fd);
-        }
-
-        gf_proc_dump_build_key(key, key_prefix, "connecting");
-        gf_proc_dump_write(key, "%d", conf->connecting);
-        gf_proc_dump_build_key(key, key_prefix, "last_sent");
-        gf_proc_dump_write(key, "%s", ctime(&conf->last_sent.tv_sec));
-        gf_proc_dump_build_key(key, key_prefix, "last_received");
-        gf_proc_dump_write(key, "%s", ctime(&conf->last_received.tv_sec));
-
-        pthread_mutex_unlock(&conf->mutex);
-
-        return 0;
-
-}
-
-int32_t
-client_inodectx_dump (xlator_t *this, inode_t *inode)
-{
-        ino_t   par = 0;
-        int     ret = -1;
-        char    key[GF_DUMP_MAX_BUF_LEN];
-
-        if (!inode)
-                return -1;
-
-        if (!this)
-                return -1;
-
-        ret = inode_ctx_get (inode, this, &par);
-
-        if (ret != 0)
-                return ret;
-
-        gf_proc_dump_build_key(key, "xlator.protocol.client",
-                               "%s.inode.%ld.par",
-                               this->name,inode->ino);
-        gf_proc_dump_write(key, "%ld", par);
-
-        return 0;
-}
-
-/*
- * client_protocol_notify - notify function for client protocol
- * @this:
- * @trans: transport object
- * @event
- *
- */
-
-int
-notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        int                  i          = 0;
-        int                  ret        = 0;
-        int                  child_down = 1;
-        int                  was_not_down = 0;
-        transport_t         *trans      = NULL;
-        client_connection_t *conn       = NULL;
-        client_conf_t       *conf       = NULL;
-        xlator_list_t       *parent = NULL;
-
-        conf = this->private;
-        trans = data;
-
-        switch (event) {
-        case GF_EVENT_POLLOUT:
-        {
-                ret = protocol_client_pollout (this, trans);
-
-                break;
-        }
-        case GF_EVENT_POLLIN:
-        {
-                ret = protocol_client_pollin (this, trans);
-
-                break;
-        }
-        /* no break for ret check to happen below */
-        case GF_EVENT_POLLERR:
-        {
-                ret = -1;
-                protocol_client_cleanup (trans);
-
-                if (conf->connecting == 0) {
-                        /* Let the connection/re-connection happen in
-                         * background, for now, don't hang here,
-                         * tell the parents that i am all ok..
-                         */
-                        parent = trans->xl->parents;
-                        while (parent) {
-                                parent->xlator->notify (parent->xlator,
-                                                        GF_EVENT_CHILD_CONNECTING,
-                                                        trans->xl);
-                                parent = parent->next;
-                        }
-                        conf->connecting = 1;
-                }
-
-                was_not_down = 0;
-                for (i = 0; i < CHANNEL_MAX; i++) {
-                        conn = conf->transport[i]->xl_private;
-                        if (conn->connected == 1)
-                                was_not_down = 1;
-                }
-
-                conn = trans->xl_private;
-                if (conn->connected) {
-                        conn->connected = 0;
-                        if (conn->reconnect == 0)
-                                client_protocol_reconnect (trans);
-                }
-
-                child_down = 1;
-                for (i = 0; i < CHANNEL_MAX; i++) {
-                        trans = conf->transport[i];
-                        conn = trans->xl_private;
-                        if (conn->connected == 1)
-                                child_down = 0;
-                }
-
-                if (child_down && was_not_down) {
-                        gf_log (this->name, GF_LOG_INFO, "disconnected");
-
-                        protocol_client_mark_fd_bad (this);
-
-                        parent = this->parents;
-                        while (parent) {
-                                xlator_notify (parent->xlator,
-                                               GF_EVENT_CHILD_DOWN, this);
-                                parent = parent->next;
-                        }
-                }
-        }
-        break;
-
-        case GF_EVENT_PARENT_UP:
-        {
-                client_conf_t *conf = NULL;
-                int            i = 0;
-                transport_t   *trans = NULL;
-
-                conf = this->private;
-                for (i = 0; i < CHANNEL_MAX; i++) {
-                        trans = conf->transport[i];
-                        if (!trans) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "transport init failed");
-                                return -1;
-                        }
-
-                        conn = trans->xl_private;
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "got GF_EVENT_PARENT_UP, attempting connect "
-                                "on transport");
-
-                        client_protocol_reconnect (trans);
-                }
-        }
-        break;
-
-        case GF_EVENT_CHILD_UP:
-        {
-                char *handshake = NULL;
-
-                ret = dict_get_str (this->options, "disable-handshake",
-                                    &handshake);
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "got GF_EVENT_CHILD_UP");
-                if ((ret < 0) ||
-                    (strcasecmp (handshake, "on"))) {
-                        ret = protocol_client_handshake (this, trans);
-                } else {
-                        conn = trans->xl_private;
-                        conn->connected = 1;
-                        ret = default_notify (this, event, trans);
-                }
-
-                if (ret)
-                        transport_disconnect (trans);
-
-        }
-        break;
-
-        default:
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "got %d, calling default_notify ()", event);
-
-                default_notify (this, event, data);
-                break;
-        }
-
-        return ret;
-}
-
-
-struct xlator_fops fops = {
-        .stat        = client_stat,
-        .readlink    = client_readlink,
-        .mknod       = client_mknod,
-        .mkdir       = client_mkdir,
-        .unlink      = client_unlink,
-        .rmdir       = client_rmdir,
-        .symlink     = client_symlink,
-        .rename      = client_rename,
-        .link        = client_link,
-        .truncate    = client_truncate,
-        .open        = client_open,
-        .readv       = client_readv,
-        .writev      = client_writev,
-        .statfs      = client_statfs,
-        .flush       = client_flush,
-        .fsync       = client_fsync,
-        .setxattr    = client_setxattr,
-        .getxattr    = client_getxattr,
-        .fsetxattr   = client_fsetxattr,
-        .fgetxattr   = client_fgetxattr,
-        .removexattr = client_removexattr,
-        .opendir     = client_opendir,
-        .readdir     = client_readdir,
-        .readdirp    = client_readdirp,
-        .fsyncdir    = client_fsyncdir,
-        .access      = client_access,
-        .ftruncate   = client_ftruncate,
-        .fstat       = client_fstat,
-        .create      = client_create,
-        .lk          = client_lk,
-        .inodelk     = client_inodelk,
-        .finodelk    = client_finodelk,
-        .entrylk     = client_entrylk,
-        .fentrylk    = client_fentrylk,
-        .lookup      = client_lookup,
-        .rchecksum   = client_rchecksum,
-        .xattrop     = client_xattrop,
-        .fxattrop    = client_fxattrop,
-        .setattr     = client_setattr,
-        .fsetattr    = client_fsetattr,
-        .getspec   = client_getspec,
-};
-
-struct xlator_cbks cbks = {
-        .release    = client_release,
-        .releasedir = client_releasedir
-};
-
-
-struct xlator_dumpops dumpops = {
-        .priv      =  client_priv_dump,
-        .inodectx  =  client_inodectx_dump,
-};
-
-struct volume_options options[] = {
-        { .key   = {"username"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key   = {"password"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key   = {"transport-type"},
-          .value = {"tcp", "socket", "ib-verbs", "unix", "ib-sdp",
-                    "tcp/client", "ib-verbs/client"},
-          .type  = GF_OPTION_TYPE_STR
-        },
-        { .key   = {"remote-host"},
-          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS
-        },
-        { .key   = {"remote-subvolume"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key   = {"frame-timeout"},
-          .type  = GF_OPTION_TYPE_TIME,
-          .min   = 0,
-          .max   = 86400,
-        },
-        { .key   = {"ping-timeout"},
-          .type  = GF_OPTION_TYPE_TIME,
-          .min   = 1,
-          .max   = 1013,
-        },
-        { .key   = {NULL} },
-};
diff --git a/xlators/protocol/legacy/client/src/client-protocol.h b/xlators/protocol/legacy/client/src/client-protocol.h
deleted file mode 100644
index cd856d50e..000000000
--- a/xlators/protocol/legacy/client/src/client-protocol.h
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CLIENT_PROTOCOL_H
-#define _CLIENT_PROTOCOL_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdio.h>
-#include <arpa/inet.h>
-#include "inode.h"
-#include "timer.h"
-#include "byte-order.h"
-#include "saved-frames.h"
-
-#define CLIENT_PORT_CEILING        1023
-
-#define GF_CLIENT_INODE_SELF   0
-#define GF_CLIENT_INODE_PARENT 1
-
-#define CLIENT_CONF(this) ((client_conf_t *)(this->private))
-
-#define RECEIVE_TIMEOUT(_cprivate,_current)     \
-        ((_cprivate->last_received.tv_sec +     \
-          _cprivate->frame_timeout) <           \
-         _current.tv_sec)
-
-#define SEND_TIMEOUT(_cprivate,_current)        \
-        ((_cprivate->last_sent.tv_sec +         \
-          _cprivate->frame_timeout) <           \
-         _current.tv_sec)
-
-enum {
-	CHANNEL_BULK = 0,
-	CHANNEL_LOWLAT = 1,
-	CHANNEL_MAX
-};
-
-#define CLIENT_CHANNEL client_channel
-
-struct client_connection;
-typedef struct client_connection client_connection_t;
-
-#include "stack.h"
-#include "xlator.h"
-#include "transport.h"
-#include "protocol.h"
-
-typedef struct _client_fd_ctx {
-        struct list_head  sfd_pos;      /*  Stores the reference to this
-                                            fd's position in the saved_fds list.
-                                        */
-        int64_t           remote_fd;
-        inode_t          *inode;
-        uint64_t          ino;
-        uint64_t          gen;
-        char              is_dir;
-        char              released;
-        int32_t           flags;
-        int32_t           wbflags;
-} client_fd_ctx_t;
-
-struct _client_conf {
-	transport_t         *transport[CHANNEL_MAX];
-	struct list_head     saved_fds;
-	struct timeval       last_sent;
-	struct timeval       last_received;
-	pthread_mutex_t      mutex;
-        int                  connecting;
-};
-typedef struct _client_conf client_conf_t;
-
-/* This will be stored in transport_t->xl_private */
-struct client_connection {
-	pthread_mutex_t      lock;
-	uint64_t             callid;
-	struct saved_frames *saved_frames;
-	int32_t              frame_timeout;
-	int32_t              ping_started;
-	int32_t              ping_timeout;
-	int32_t              transport_activity;
-	gf_timer_t          *reconnect;
-	char                 connected;
-	uint64_t             max_block_size;
-	gf_timer_t          *timer;
-	gf_timer_t          *ping_timer;
-};
-
-typedef struct {
-	loc_t              loc;
-	loc_t              loc2;
-	fd_t              *fd;
-        gf_op_t            op;
-        client_fd_ctx_t   *fdctx;
-        uint32_t           flags;
-        uint32_t           wbflags;
-} client_local_t;
-
-
-static inline void
-gf_string_to_stat(char *string, struct iatt *stbuf)
-{
-	uint64_t dev        = 0;
-	uint64_t ino        = 0;
-	uint32_t mode       = 0;
-	uint32_t nlink      = 0;
-	uint32_t uid        = 0;
-	uint32_t gid        = 0;
-	uint64_t rdev       = 0;
-	uint64_t size       = 0;
-	uint32_t blksize    = 0;
-	uint64_t blocks     = 0;
-	uint32_t atime      = 0;
-	uint32_t atime_nsec = 0;
-	uint32_t mtime      = 0;
-	uint32_t mtime_nsec = 0;
-	uint32_t ctime      = 0;
-	uint32_t ctime_nsec = 0;
-
-	sscanf (string, GF_STAT_PRINT_FMT_STR,
-		&dev,
-		&ino,
-		&mode,
-		&nlink,
-		&uid,
-		&gid,
-		&rdev,
-		&size,
-		&blksize,
-		&blocks,
-		&atime,
-		&atime_nsec,
-		&mtime,
-		&mtime_nsec,
-		&ctime,
-		&ctime_nsec);
-
-	stbuf->ia_gen   = dev;
-	stbuf->ia_ino   = ino;
-	stbuf->ia_prot  = ia_prot_from_st_mode (mode);
-        stbuf->ia_type  = ia_type_from_st_mode (mode);
-	stbuf->ia_nlink = nlink;
-	stbuf->ia_uid   = uid;
-	stbuf->ia_gid   = gid;
-	stbuf->ia_rdev  = rdev;
-	stbuf->ia_size  = size;
-	stbuf->ia_blksize = blksize;
-	stbuf->ia_blocks  = blocks;
-
-	stbuf->ia_atime = atime;
-	stbuf->ia_mtime = mtime;
-	stbuf->ia_ctime = ctime;
-
-	stbuf->ia_atime_nsec = atime_nsec;
-	stbuf->ia_mtime_nsec = mtime_nsec;
-	stbuf->ia_ctime_nsec = ctime_nsec;
-}
-
-#endif
diff --git a/xlators/protocol/legacy/client/src/saved-frames.c b/xlators/protocol/legacy/client/src/saved-frames.c
deleted file mode 100644
index b884a9796..000000000
--- a/xlators/protocol/legacy/client/src/saved-frames.c
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#include "saved-frames.h"
-#include "common-utils.h"
-#include "protocol.h"
-#include "xlator.h"
-#include "client-mem-types.h"
-
-
-
-struct saved_frames *
-gf_client_saved_frames_new (void)
-{
-	struct saved_frames *saved_frames = NULL;
-
-	saved_frames = GF_CALLOC (sizeof (*saved_frames), 1,
-                                  gf_client_mt_saved_frames);
-	if (!saved_frames) {
-		return NULL;
-	}
-
-        gf_log ("", 1, "here");
-	INIT_LIST_HEAD (&saved_frames->fops.list);
-	INIT_LIST_HEAD (&saved_frames->mops.list);
-	INIT_LIST_HEAD (&saved_frames->cbks.list);
-
-	return saved_frames;
-}
-
-
-struct saved_frame *
-get_head_frame_for_type (struct saved_frames *frames, int8_t type)
-{
-	struct saved_frame *head_frame = NULL;
-
-	switch (type) {
-	case GF_OP_TYPE_FOP_REQUEST:
-	case GF_OP_TYPE_FOP_REPLY:
-		head_frame = &frames->fops;
-		break;
-	case GF_OP_TYPE_MOP_REQUEST:
-	case GF_OP_TYPE_MOP_REPLY:
-		head_frame = &frames->mops;
-		break;
-	case GF_OP_TYPE_CBK_REQUEST:
-	case GF_OP_TYPE_CBK_REPLY:
-		head_frame = &frames->cbks;
-		break;
-	}
-
-	return head_frame;
-}
-
-
-int
-saved_frames_put (struct saved_frames *frames, call_frame_t *frame,
-		  int32_t op, int8_t type, int64_t callid)
-{
-	struct saved_frame *saved_frame = NULL;
-	struct saved_frame *head_frame = NULL;
-
-	head_frame = get_head_frame_for_type (frames, type);
-
-	saved_frame = GF_CALLOC (sizeof (*saved_frame), 1,
-                                gf_client_mt_saved_frame);
-	if (!saved_frame) {
-		return -ENOMEM;
-	}
-
-	INIT_LIST_HEAD (&saved_frame->list);
-	saved_frame->frame  = frame;
-	saved_frame->op     = op;
-	saved_frame->type   = type;
-	saved_frame->callid = callid;
-
-	gettimeofday (&saved_frame->saved_at, NULL);
-
-	list_add_tail (&saved_frame->list, &head_frame->list);
-	frames->count++;
-
-	return 0;
-}
-
-
-call_frame_t *
-saved_frames_get (struct saved_frames *frames, int32_t op,
-		  int8_t type, int64_t callid)
-{
-	struct saved_frame *saved_frame = NULL;
-	struct saved_frame *tmp = NULL;
-	struct saved_frame *head_frame = NULL;
-	call_frame_t       *frame = NULL;
-
-	head_frame = get_head_frame_for_type (frames, type);
-
-	list_for_each_entry (tmp, &head_frame->list, list) {
-		if (tmp->callid == callid) {
-			list_del_init (&tmp->list);
-			frames->count--;
-			saved_frame = tmp;
-			break;
-		}
-	}
-
-	if (saved_frame)
-		frame = saved_frame->frame;
-
-	GF_FREE (saved_frame);
-
-	return frame;
-}
-
-struct saved_frame *
-saved_frames_get_timedout (struct saved_frames *frames, int8_t type, 
-			   uint32_t timeout, struct timeval *current)
-{
-	struct saved_frame *bailout_frame = NULL, *tmp = NULL;
-	struct saved_frame *head_frame = NULL;
-
-	head_frame = get_head_frame_for_type (frames, type);
-
-	if (!list_empty(&head_frame->list)) {
-		tmp = list_entry (head_frame->list.next, typeof (*tmp), list);
-		if ((tmp->saved_at.tv_sec + timeout) < current->tv_sec) {
-			bailout_frame = tmp;
-			list_del_init (&bailout_frame->list);
-			frames->count--;
-		}
-	}
-
-	return bailout_frame;
-}
-
-void
-saved_frames_unwind (xlator_t *this, struct saved_frames *saved_frames,
-		     struct saved_frame *head,
-		     gf_op_t gf_ops[], char *gf_op_list[])
-{
-	struct saved_frame   *trav = NULL;
-	struct saved_frame   *tmp = NULL;
-
-	gf_hdr_common_t       hdr = {0, };
-	call_frame_t         *frame = NULL;
-
-	hdr.rsp.op_ret   = hton32 (-1);
-	hdr.rsp.op_errno = hton32 (ENOTCONN);
-
-	list_for_each_entry_safe (trav, tmp, &head->list, list) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"forced unwinding frame type(%d) op(%s)",
-			trav->type, gf_op_list[trav->op]);
-
-		hdr.type = hton32 (trav->type);
-		hdr.op   = hton32 (trav->op);
-
-		frame = trav->frame;
-
-		saved_frames->count--;
-
-		gf_ops[trav->op] (frame, &hdr, sizeof (hdr), NULL);
-
-		list_del_init (&trav->list);
-		GF_FREE (trav);
-	}
-}
-
-
-void
-gf_client_saved_frames_destroy (xlator_t *this, struct saved_frames *frames,
-                                gf_op_t gf_fops[], gf_op_t gf_mops[],
-                                gf_op_t gf_cbks[])
-{
-	saved_frames_unwind (this, frames, &frames->fops, gf_fops, gf_fop_list);
-	saved_frames_unwind (this, frames, &frames->mops, gf_mops, gf_mop_list);
-	saved_frames_unwind (this, frames, &frames->cbks, gf_cbks, gf_cbk_list);
-
-	GF_FREE (frames);
-}
diff --git a/xlators/protocol/legacy/client/src/saved-frames.h b/xlators/protocol/legacy/client/src/saved-frames.h
deleted file mode 100644
index 9f7fe0a7d..000000000
--- a/xlators/protocol/legacy/client/src/saved-frames.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _SAVED_FRAMES_H
-#define _SAVED_FRAMES_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdint.h>
-#include <sys/time.h>
-#include "stack.h"
-#include "list.h"
-#include "protocol.h"
-
-/* UGLY: have common typedef b/w saved-frames.c and protocol-client.c */
-typedef int32_t (*gf_op_t) (call_frame_t *frame,
-                            gf_hdr_common_t *hdr, size_t hdrlen,
-                            struct iobuf *iobuf);
-
-
-struct saved_frame {
-	union {
-		struct list_head list;
-		struct {
-			struct saved_frame *frame_next;
-			struct saved_frame *frame_prev;
-		};
-	};
-
-	struct timeval  saved_at;
-	call_frame_t   *frame;
-	int32_t         op;
-	int8_t          type;
-	uint64_t        callid;
-};
-
-
-struct saved_frames {
-	int64_t            count;
-	struct saved_frame fops;
-	struct saved_frame mops;
-	struct saved_frame cbks;
-};
-
-
-struct saved_frames *gf_client_saved_frames_new ();
-int saved_frames_put (struct saved_frames *frames, call_frame_t *frame,
-		      int32_t op, int8_t type, int64_t callid);
-call_frame_t *saved_frames_get (struct saved_frames *frames, int32_t op,
-				int8_t type, int64_t callid);
-
-struct saved_frame *
-saved_frames_get_timedout (struct saved_frames *frames, int8_t type, 
-			   uint32_t timeout, struct timeval *current);
-
-void gf_client_saved_frames_destroy (xlator_t *this, struct saved_frames *frames,
-                                     gf_op_t gf_fops[], gf_op_t gf_mops[],
-                                     gf_op_t gf_cbks[]);
-
-#endif /* _SAVED_FRAMES_H */
diff --git a/xlators/protocol/legacy/lib/Makefile.am b/xlators/protocol/legacy/lib/Makefile.am
deleted file mode 100644
index d471a3f92..000000000
--- a/xlators/protocol/legacy/lib/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/protocol/legacy/lib/src/Makefile.am b/xlators/protocol/legacy/lib/src/Makefile.am
deleted file mode 100644
index 1f0e93e30..000000000
--- a/xlators/protocol/legacy/lib/src/Makefile.am
+++ /dev/null
@@ -1,14 +0,0 @@
-lib_LTLIBRARIES = libgfproto.la
-
-libgfproto_la_CFLAGS = -fPIC  -Wall -g -shared -nostartfiles $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS)
-
-libgfproto_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D_GNU_SOURCE  \
-	-D$(GF_HOST_OS) -DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\"     \
-	-DTRANSPORTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/transport\"          \
-	-I$(CONTRIBDIR)/rbtree -I$(top_srcdir)/libglusterfs/src/
-
-libgfproto_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-libgfproto_la_SOURCES = transport.c protocol.c
-
-noinst_HEADERS = transport.h protocol.h
diff --git a/xlators/protocol/legacy/lib/src/protocol.c b/xlators/protocol/legacy/lib/src/protocol.c
deleted file mode 100644
index 63950f43d..000000000
--- a/xlators/protocol/legacy/lib/src/protocol.c
+++ /dev/null
@@ -1,108 +0,0 @@
-
-#include "globals.h"
-#include "compat.h"
-#include "protocol.h"
-
-char *gf_mop_list[GF_MOP_MAXVALUE];
-char *gf_cbk_list[GF_CBK_MAXVALUE];
-
-static int
-gf_dirent_nb_size (gf_dirent_t *entries)
-{
-	return (sizeof (struct gf_dirent_nb) + strlen (entries->d_name) + 1);
-}
-
-int
-gf_dirent_serialize (gf_dirent_t *entries, char *buf, size_t buf_size)
-{
-	struct gf_dirent_nb *entry_nb = NULL;
-	gf_dirent_t         *entry = NULL;
-	int                  size = 0;
-	int                  entry_size = 0;
-
-
-	list_for_each_entry (entry, &entries->list, list) {
-		entry_size = gf_dirent_nb_size (entry);
-
-		if (buf && (size + entry_size <= buf_size)) {
-			entry_nb = (void *) (buf + size);
-
-			entry_nb->d_ino  = hton64 (entry->d_ino);
-			entry_nb->d_off  = hton64 (entry->d_off);
-			entry_nb->d_len  = hton32 (entry->d_len);
-			entry_nb->d_type = hton32 (entry->d_type);
-
-                        gf_stat_from_iatt (&entry_nb->d_stat, &entry->d_stat);
-
-			strcpy (entry_nb->d_name, entry->d_name);
-		}
-		size += entry_size;
-	}
-
-	return size;
-}
-
-
-int
-gf_dirent_unserialize (gf_dirent_t *entries, const char *buf, size_t buf_size)
-{
-	struct gf_dirent_nb *entry_nb = NULL;
-	int                  remaining_size = 0;
-	int                  least_dirent_size = 0;
-	int                  count = 0;
-	gf_dirent_t         *entry = NULL;
-	int                  entry_strlen = 0;
-	int                  entry_len = 0;
-
-
-	remaining_size = buf_size;
-	least_dirent_size = (sizeof (struct gf_dirent_nb) + 2);
-
-	while (remaining_size >= least_dirent_size) {
-		entry_nb = (void *)(buf + (buf_size - remaining_size));
-
-		entry_strlen = strnlen (entry_nb->d_name, remaining_size);
-		if (entry_strlen == remaining_size) {
-			break;
-		}
-
-		entry_len = sizeof (gf_dirent_t) + entry_strlen + 1;
-		entry = GF_CALLOC (1, entry_len, gf_common_mt_gf_dirent_t);
-		if (!entry) {
-			break;
-		}
-
-		entry->d_ino  = ntoh64 (entry_nb->d_ino);
-		entry->d_off  = ntoh64 (entry_nb->d_off);
-		entry->d_len  = ntoh32 (entry_nb->d_len);
-		entry->d_type = ntoh32 (entry_nb->d_type);
-
-                gf_stat_to_iatt (&entry_nb->d_stat, &entry->d_stat);
-
-		strcpy (entry->d_name, entry_nb->d_name);
-
-		list_add_tail (&entry->list, &entries->list);
-
-		remaining_size -= (sizeof (*entry_nb) + entry_strlen + 1);
-		count++;
-	}
-
-	return count;
-}
-
-int
-protocol_common_init (void)
-{
-        gf_mop_list[GF_MOP_SETVOLUME]   = "SETVOLUME";
-        gf_mop_list[GF_MOP_GETVOLUME]   = "GETVOLUME";
-        gf_mop_list[GF_MOP_SETSPEC]     = "SETSPEC";
-        gf_mop_list[GF_MOP_GETSPEC]     = "GETSPEC";
-        gf_mop_list[GF_MOP_LOG]         = "LOG";
-        gf_mop_list[GF_MOP_PING]        = "PING";
-
-        gf_cbk_list[GF_CBK_FORGET]      = "FORGET";
-        gf_cbk_list[GF_CBK_RELEASE]     = "RELEASE";
-        gf_cbk_list[GF_CBK_RELEASEDIR]  = "RELEASEDIR";
-
-        return 0;
-}
diff --git a/xlators/protocol/legacy/lib/src/protocol.h b/xlators/protocol/legacy/lib/src/protocol.h
deleted file mode 100644
index 6c568ed0d..000000000
--- a/xlators/protocol/legacy/lib/src/protocol.h
+++ /dev/null
@@ -1,1118 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _PROTOCOL_H
-#define _PROTOCOL_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <inttypes.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/statvfs.h>
-#include <unistd.h>
-#include <fcntl.h>
-
-#include "byte-order.h"
-#include "iatt.h"
-
-/* Any changes in the protocol structure or adding new '[f,m]ops' needs to 
- * bump the protocol version by "0.1" 
- */
-
-#define GF_PROTOCOL_VERSION "3.0"
-
-extern char *gf_mop_list[];
-extern char *gf_cbk_list[];
-
-/* NOTE: add members ONLY at the end (just before _MAXVALUE) */
-typedef enum {
-        GF_PROTO_FOP_STAT,       /* 0 */
-        GF_PROTO_FOP_READLINK,   /* 1 */
-        GF_PROTO_FOP_MKNOD,      /* 2 */
-        GF_PROTO_FOP_MKDIR,
-        GF_PROTO_FOP_UNLINK,
-        GF_PROTO_FOP_RMDIR,      /* 5 */
-        GF_PROTO_FOP_SYMLINK,
-        GF_PROTO_FOP_RENAME,
-        GF_PROTO_FOP_LINK,
-        GF_PROTO_FOP_TRUNCATE,
-        GF_PROTO_FOP_OPEN,       /* 10 */
-        GF_PROTO_FOP_READ,
-        GF_PROTO_FOP_WRITE,
-        GF_PROTO_FOP_STATFS,     /* 15 */
-        GF_PROTO_FOP_FLUSH,
-        GF_PROTO_FOP_FSYNC,
-        GF_PROTO_FOP_SETXATTR,
-        GF_PROTO_FOP_GETXATTR,
-        GF_PROTO_FOP_REMOVEXATTR,/* 20 */
-        GF_PROTO_FOP_OPENDIR,
-        GF_PROTO_FOP_GETDENTS,
-        GF_PROTO_FOP_FSYNCDIR,
-        GF_PROTO_FOP_ACCESS,
-        GF_PROTO_FOP_CREATE,     /* 25 */
-        GF_PROTO_FOP_FTRUNCATE,
-        GF_PROTO_FOP_FSTAT,
-        GF_PROTO_FOP_LK,
-        GF_PROTO_FOP_LOOKUP,
-        GF_PROTO_FOP_SETDENTS,
-        GF_PROTO_FOP_READDIR,
-        GF_PROTO_FOP_INODELK,   /* 35 */
-        GF_PROTO_FOP_FINODELK,
-	GF_PROTO_FOP_ENTRYLK,
-	GF_PROTO_FOP_FENTRYLK,
-        GF_PROTO_FOP_CHECKSUM,
-        GF_PROTO_FOP_XATTROP,  /* 40 */
-        GF_PROTO_FOP_FXATTROP,
-        GF_PROTO_FOP_LOCK_NOTIFY,
-        GF_PROTO_FOP_LOCK_FNOTIFY,
-        GF_PROTO_FOP_FGETXATTR,
-        GF_PROTO_FOP_FSETXATTR, /* 45 */
-        GF_PROTO_FOP_RCHECKSUM,
-        GF_PROTO_FOP_SETATTR,
-        GF_PROTO_FOP_FSETATTR,
-        GF_PROTO_FOP_READDIRP,
-        GF_PROTO_FOP_MAXVALUE,
-} glusterfs_proto_fop_t;
-
-/* NOTE: add members ONLY at the end (just before _MAXVALUE) */
-typedef enum {
-        GF_MOP_SETVOLUME, /* 0 */
-        GF_MOP_GETVOLUME, /* 1 */
-        GF_MOP_STATS,
-        GF_MOP_SETSPEC,
-        GF_MOP_GETSPEC,
-	GF_MOP_PING,      /* 5 */
-        GF_MOP_LOG,
-        GF_MOP_NOTIFY,
-        GF_MOP_MAXVALUE,   /* 8 */
-} glusterfs_mop_t;
-
-typedef enum {
-	GF_CBK_FORGET,      /* 0 */
-	GF_CBK_RELEASE,     /* 1 */
-	GF_CBK_RELEASEDIR,  /* 2 */
-	GF_CBK_MAXVALUE     /* 3 */
-} glusterfs_cbk_t;
-
-typedef enum {
-        GF_OP_TYPE_FOP_REQUEST = 1,
-        GF_OP_TYPE_MOP_REQUEST,
-	GF_OP_TYPE_CBK_REQUEST,
-        GF_OP_TYPE_FOP_REPLY,
-        GF_OP_TYPE_MOP_REPLY,
-	GF_OP_TYPE_CBK_REPLY
-} glusterfs_op_type_t;
-
-
-struct gf_stat {
-	uint64_t ino;
-	uint64_t size;
-	uint64_t blocks;
-	uint64_t dev;
-	uint32_t rdev;
-	uint32_t mode;
-	uint32_t nlink;
-	uint32_t uid;
-	uint32_t gid;
-	uint32_t blksize;
-	uint32_t atime;
-	uint32_t atime_nsec;
-	uint32_t mtime ;
-	uint32_t mtime_nsec;
-	uint32_t ctime;
-	uint32_t ctime_nsec;
-} __attribute__((packed));
-
-
-static inline void
-gf_stat_to_stat (struct gf_stat *gf_stat, struct stat *stat)
-{
-	stat->st_dev          = ntoh64 (gf_stat->dev);
-	stat->st_ino          = ntoh64 (gf_stat->ino);
-	stat->st_mode         = ntoh32 (gf_stat->mode);
-	stat->st_nlink        = ntoh32 (gf_stat->nlink);
-	stat->st_uid          = ntoh32 (gf_stat->uid);
-	stat->st_gid          = ntoh32 (gf_stat->gid);
-	stat->st_rdev         = ntoh32 (gf_stat->rdev);
-	stat->st_size         = ntoh64 (gf_stat->size);
-	stat->st_blksize      = ntoh32 (gf_stat->blksize);
-	stat->st_blocks       = ntoh64 (gf_stat->blocks);
-	stat->st_atime        = ntoh32 (gf_stat->atime);
-	stat->st_mtime        = ntoh32 (gf_stat->mtime);
-	stat->st_ctime        = ntoh32 (gf_stat->ctime);
-        ST_ATIM_NSEC_SET(stat, ntoh32 (gf_stat->atime_nsec));
-        ST_MTIM_NSEC_SET(stat, ntoh32 (gf_stat->mtime_nsec));
-        ST_CTIM_NSEC_SET(stat, ntoh32 (gf_stat->ctime_nsec));
-}
-
-
-static inline void
-gf_stat_from_stat (struct gf_stat *gf_stat, struct stat *stat)
-{
-	gf_stat->dev         = hton64 (stat->st_dev);
-	gf_stat->ino         = hton64 (stat->st_ino);
-	gf_stat->mode        = hton32 (stat->st_mode);
-	gf_stat->nlink       = hton32 (stat->st_nlink);
-	gf_stat->uid         = hton32 (stat->st_uid);
-	gf_stat->gid         = hton32 (stat->st_gid);
-	gf_stat->rdev        = hton32 (stat->st_rdev);
-	gf_stat->size        = hton64 (stat->st_size);
-	gf_stat->blksize     = hton32 (stat->st_blksize);
-	gf_stat->blocks      = hton64 (stat->st_blocks);
-	gf_stat->atime       = hton32 (stat->st_atime);
-	gf_stat->mtime       = hton32 (stat->st_mtime);
-	gf_stat->ctime       = hton32 (stat->st_ctime);
-        gf_stat->atime_nsec  = hton32 (ST_ATIM_NSEC(stat));
-        gf_stat->mtime_nsec  = hton32 (ST_MTIM_NSEC(stat));
-        gf_stat->ctime_nsec  = hton32 (ST_CTIM_NSEC(stat));
-}
-
-
-static inline void
-gf_stat_to_iatt (struct gf_stat *gf_stat, struct iatt *iatt)
-{
-        iatt->ia_ino        = ntoh64 (gf_stat->ino);
-        iatt->ia_dev        = ntoh64 (gf_stat->dev);
-        iatt->ia_type       = ia_type_from_st_mode (ntoh32 (gf_stat->mode));
-        iatt->ia_prot       = ia_prot_from_st_mode (ntoh32 (gf_stat->mode));
-        iatt->ia_nlink      = ntoh32 (gf_stat->nlink);
-        iatt->ia_uid        = ntoh32 (gf_stat->uid);
-        iatt->ia_gid        = ntoh32 (gf_stat->gid);
-        iatt->ia_rdev       = ntoh64 (gf_stat->rdev);
-        iatt->ia_size       = ntoh64 (gf_stat->size);
-        iatt->ia_blksize    = ntoh32 (gf_stat->blksize);
-        iatt->ia_blocks     = ntoh64 (gf_stat->blocks);
-        iatt->ia_atime      = ntoh32 (gf_stat->atime);
-        iatt->ia_atime_nsec = ntoh32 (gf_stat->atime_nsec);
-        iatt->ia_mtime      = ntoh32 (gf_stat->mtime);
-        iatt->ia_mtime_nsec = ntoh32 (gf_stat->mtime_nsec);
-        iatt->ia_ctime      = ntoh32 (gf_stat->ctime);
-        iatt->ia_ctime_nsec = ntoh32 (gf_stat->ctime_nsec);
-
-        iatt->ia_gen        = ntoh64 (gf_stat->dev);
-}
-
-
-static inline void
-gf_stat_from_iatt (struct gf_stat *gf_stat, struct iatt *iatt)
-{
-        gf_stat->ino        = hton64 (iatt->ia_ino);
-        gf_stat->dev        = hton64 (iatt->ia_dev);
-        gf_stat->mode       = hton32 (st_mode_from_ia (iatt->ia_prot,
-                                                       iatt->ia_type));
-        gf_stat->nlink      = hton32 (iatt->ia_nlink);
-        gf_stat->uid        = hton32 (iatt->ia_uid);
-        gf_stat->gid        = hton32 (iatt->ia_gid);
-        gf_stat->rdev       = hton32 (iatt->ia_rdev);
-        gf_stat->size       = hton64 (iatt->ia_size);
-        gf_stat->blksize    = hton32 (iatt->ia_blksize);
-        gf_stat->blocks     = hton64 (iatt->ia_blocks);
-        gf_stat->atime      = hton32 (iatt->ia_atime);
-        gf_stat->atime_nsec = hton32 (iatt->ia_atime_nsec);
-        gf_stat->mtime      = hton32 (iatt->ia_mtime);
-        gf_stat->mtime_nsec = hton32 (iatt->ia_mtime_nsec);
-        gf_stat->ctime      = hton32 (iatt->ia_ctime);
-        gf_stat->ctime_nsec = hton32 (iatt->ia_ctime_nsec);
-
-        gf_stat->dev        = hton64 (iatt->ia_gen);
-
-}
-
-
-struct gf_statfs {
-	uint64_t bsize;
-	uint64_t frsize;
-	uint64_t blocks;
-	uint64_t bfree;
-	uint64_t bavail;
-	uint64_t files;
-	uint64_t ffree;
-	uint64_t favail;
-	uint64_t fsid;
-	uint64_t flag;
-	uint64_t namemax;
-} __attribute__((packed));
-
-
-static inline void
-gf_statfs_to_statfs (struct gf_statfs *gf_stat, struct statvfs *stat)
-{
-	stat->f_bsize   = ntoh64 (gf_stat->bsize);
-	stat->f_frsize  = ntoh64 (gf_stat->frsize);
-	stat->f_blocks  = ntoh64 (gf_stat->blocks);
-	stat->f_bfree   = ntoh64 (gf_stat->bfree);
-	stat->f_bavail  = ntoh64 (gf_stat->bavail);
-	stat->f_files   = ntoh64 (gf_stat->files);
-	stat->f_ffree   = ntoh64 (gf_stat->ffree);
-	stat->f_favail  = ntoh64 (gf_stat->favail);
-	stat->f_fsid    = ntoh64 (gf_stat->fsid);
-	stat->f_flag    = ntoh64 (gf_stat->flag);
-	stat->f_namemax = ntoh64 (gf_stat->namemax);
-}
-
-
-static inline void
-gf_statfs_from_statfs (struct gf_statfs *gf_stat, struct statvfs *stat)
-{
-	gf_stat->bsize   = hton64 (stat->f_bsize);
-	gf_stat->frsize  = hton64 (stat->f_frsize);
-	gf_stat->blocks  = hton64 (stat->f_blocks);
-	gf_stat->bfree   = hton64 (stat->f_bfree);
-	gf_stat->bavail  = hton64 (stat->f_bavail);
-	gf_stat->files   = hton64 (stat->f_files);
-	gf_stat->ffree   = hton64 (stat->f_ffree);
-	gf_stat->favail  = hton64 (stat->f_favail);
-	gf_stat->fsid    = hton64 (stat->f_fsid);
-	gf_stat->flag    = hton64 (stat->f_flag);
-	gf_stat->namemax = hton64 (stat->f_namemax);
-}
-
-
-struct gf_flock {
-	uint16_t type;
-	uint16_t whence;
-	uint64_t start;
-	uint64_t len;
-	uint32_t pid;
-} __attribute__((packed));
-
-
-static inline void
-gf_flock_to_flock (struct gf_flock *gf_flock, struct gf_flock *flock)
-{
-	flock->l_type   = ntoh16 (gf_flock->type);
-	flock->l_whence = ntoh16 (gf_flock->whence);
-	flock->l_start  = ntoh64 (gf_flock->start);
-	flock->l_len    = ntoh64 (gf_flock->len);
-	flock->l_pid    = ntoh32 (gf_flock->pid);
-}
-
-
-static inline void
-gf_flock_from_flock (struct gf_flock *gf_flock, struct gf_flock *flock)
-{
-	gf_flock->type   = hton16 (flock->l_type);
-	gf_flock->whence = hton16 (flock->l_whence);
-	gf_flock->start  = hton64 (flock->l_start);
-	gf_flock->len    = hton64 (flock->l_len);
-	gf_flock->pid    = hton32 (flock->l_pid);
-}
-
-
-struct gf_timespec {
-	uint32_t tv_sec;
-	uint32_t tv_nsec;
-} __attribute__((packed));
-
-
-static inline void
-gf_timespec_to_timespec (struct gf_timespec *gf_ts, struct timespec *ts)
-{
-
-	ts[0].tv_sec  = ntoh32 (gf_ts[0].tv_sec);
-	ts[0].tv_nsec = ntoh32 (gf_ts[0].tv_nsec);
-	ts[1].tv_sec  = ntoh32 (gf_ts[1].tv_sec);
-	ts[1].tv_nsec = ntoh32 (gf_ts[1].tv_nsec);
-}
-
-
-static inline void
-gf_timespec_from_timespec (struct gf_timespec *gf_ts, struct timespec *ts)
-{
-	gf_ts[0].tv_sec  = hton32 (ts[0].tv_sec);
-	gf_ts[0].tv_nsec = hton32 (ts[0].tv_nsec);
-	gf_ts[1].tv_sec  = hton32 (ts[1].tv_sec);
-	gf_ts[1].tv_nsec = hton32 (ts[1].tv_nsec);
-}
-
-
-#define GF_O_ACCMODE           003
-#define GF_O_RDONLY             00
-#define GF_O_WRONLY             01
-#define GF_O_RDWR               02
-#define GF_O_CREAT            0100
-#define GF_O_EXCL             0200
-#define GF_O_NOCTTY           0400
-#define GF_O_TRUNC           01000
-#define GF_O_APPEND          02000
-#define GF_O_NONBLOCK        04000
-#define GF_O_SYNC           010000
-#define GF_O_ASYNC          020000
-
-#define GF_O_DIRECT         040000
-#define GF_O_DIRECTORY     0200000
-#define GF_O_NOFOLLOW      0400000
-#define GF_O_NOATIME      01000000
-#define GF_O_CLOEXEC      02000000
-
-#define GF_O_LARGEFILE     0100000
-
-#define XLATE_BIT(from, to, bit)    do {                \
-                if (from & bit)                         \
-                        to = to | GF_##bit;             \
-        } while (0)
-
-#define UNXLATE_BIT(from, to, bit)  do {                \
-                if (from & GF_##bit)                    \
-                        to = to | bit;                  \
-        } while (0)
-
-#define XLATE_ACCESSMODE(from, to) do {                 \
-                switch (from & O_ACCMODE) {             \
-                case O_RDONLY: to |= GF_O_RDONLY;       \
-                        break;                          \
-                case O_WRONLY: to |= GF_O_WRONLY;       \
-                        break;                          \
-                case O_RDWR: to |= GF_O_RDWR;           \
-                        break;                          \
-                }                                       \
-        } while (0)
-
-#define UNXLATE_ACCESSMODE(from, to) do {               \
-                switch (from & GF_O_ACCMODE) {          \
-                case GF_O_RDONLY: to |= O_RDONLY;       \
-                        break;                          \
-                case GF_O_WRONLY: to |= O_WRONLY;       \
-                        break;                          \
-                case GF_O_RDWR: to |= O_RDWR;           \
-                        break;                          \
-                }                                       \
-        } while (0)
-
-static inline uint32_t
-gf_flags_from_flags (uint32_t flags)
-{
-        uint32_t gf_flags = 0;
-
-        XLATE_ACCESSMODE (flags, gf_flags);
-
-        XLATE_BIT (flags, gf_flags, O_CREAT);
-        XLATE_BIT (flags, gf_flags, O_EXCL);
-        XLATE_BIT (flags, gf_flags, O_NOCTTY);
-        XLATE_BIT (flags, gf_flags, O_TRUNC);
-        XLATE_BIT (flags, gf_flags, O_APPEND);
-        XLATE_BIT (flags, gf_flags, O_NONBLOCK);
-        XLATE_BIT (flags, gf_flags, O_SYNC);
-        XLATE_BIT (flags, gf_flags, O_ASYNC);
-
-        XLATE_BIT (flags, gf_flags, O_DIRECT);
-        XLATE_BIT (flags, gf_flags, O_DIRECTORY);
-        XLATE_BIT (flags, gf_flags, O_NOFOLLOW);
-#ifdef O_NOATIME
-        XLATE_BIT (flags, gf_flags, O_NOATIME);
-#endif
-#ifdef O_CLOEXEC
-        XLATE_BIT (flags, gf_flags, O_CLOEXEC);
-#endif
-        XLATE_BIT (flags, gf_flags, O_LARGEFILE);
-
-        return gf_flags;
-}
-
-static inline uint32_t
-gf_flags_to_flags (uint32_t gf_flags)
-{
-        uint32_t flags = 0;
-
-        UNXLATE_ACCESSMODE (gf_flags, flags);
-
-        UNXLATE_BIT (gf_flags, flags, O_CREAT);
-        UNXLATE_BIT (gf_flags, flags, O_EXCL);
-        UNXLATE_BIT (gf_flags, flags, O_NOCTTY);
-        UNXLATE_BIT (gf_flags, flags, O_TRUNC);
-        UNXLATE_BIT (gf_flags, flags, O_APPEND);
-        UNXLATE_BIT (gf_flags, flags, O_NONBLOCK);
-        UNXLATE_BIT (gf_flags, flags, O_SYNC);
-        UNXLATE_BIT (gf_flags, flags, O_ASYNC);
-
-        UNXLATE_BIT (gf_flags, flags, O_DIRECT);
-        UNXLATE_BIT (gf_flags, flags, O_DIRECTORY);
-        UNXLATE_BIT (gf_flags, flags, O_NOFOLLOW);
-#ifdef O_NOATIME
-        UNXLATE_BIT (gf_flags, flags, O_NOATIME);
-#endif
-#ifdef O_CLOEXEC
-        UNXLATE_BIT (gf_flags, flags, O_CLOEXEC);
-#endif
-        UNXLATE_BIT (gf_flags, flags, O_LARGEFILE);
-
-        return flags;
-}
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	char     path[0];     /* NULL terminated */
-} __attribute__((packed)) gf_fop_stat_req_t;;
-typedef struct {
-	struct gf_stat stat;
-} __attribute__((packed)) gf_fop_stat_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t size;
-	char     path[0];     /* NULL terminated */
-} __attribute__((packed)) gf_fop_readlink_req_t;
-typedef struct {
-        struct gf_stat buf;
-	char     path[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_readlink_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	uint64_t dev;
-	uint32_t mode;
-	char     path[0];     /* NULL terminated */
-	char     bname[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_mknod_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_mknod_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	uint32_t mode;
-	char     path[0];     /* NULL terminated */
-	char     bname[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_mkdir_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_mkdir_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	char     path[0];     /* NULL terminated */
-	char     bname[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_unlink_req_t;
-typedef struct {
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_unlink_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	char     path[0];
-	char     bname[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_rmdir_req_t;
-typedef struct {
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_rmdir_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	char     path[0];
-	char     bname[0];
-	char     linkname[0];
-} __attribute__((packed)) gf_fop_symlink_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-}__attribute__((packed)) gf_fop_symlink_rsp_t;
-
-
-typedef struct {
-	uint64_t   oldpar;
-        uint64_t   oldgen;
-	uint64_t   newpar;
-        uint64_t   newgen;
-	char       oldpath[0];
-	char       oldbname[0]; /* NULL terminated */
-	char       newpath[0];
-	char       newbname[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_rename_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat preoldparent;
-        struct gf_stat postoldparent;
-        struct gf_stat prenewparent;
-        struct gf_stat postnewparent;
-} __attribute__((packed)) gf_fop_rename_rsp_t;
-
-
-typedef struct {
-	uint64_t   oldino;
-        uint64_t   oldgen;
-	uint64_t   newpar;
-        uint64_t   newgen;
-	char       oldpath[0];
-	char       newpath[0];
-	char       newbname[0];
-}__attribute__((packed)) gf_fop_link_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_link_rsp_t;
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint64_t offset;
-	char     path[0];
-} __attribute__((packed)) gf_fop_truncate_req_t;
-typedef struct {
-	struct gf_stat prestat;
-        struct gf_stat poststat;
-} __attribute__((packed)) gf_fop_truncate_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t flags;
-        uint32_t wbflags;
-	char     path[0];
-} __attribute__((packed)) gf_fop_open_req_t;
-typedef struct {
-	int64_t fd;
-} __attribute__((packed)) gf_fop_open_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint64_t offset;
-	uint32_t size;
-} __attribute__((packed)) gf_fop_read_req_t;
-typedef struct {
-	struct gf_stat stat;
-	char buf[0];
-} __attribute__((packed)) gf_fop_read_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint64_t offset;
-	uint32_t size;
-} __attribute__((packed)) gf_fop_write_req_t;
-typedef struct {
-	struct gf_stat prestat;
-        struct gf_stat poststat;
-} __attribute__((packed)) gf_fop_write_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	char     path[0];
-} __attribute__((packed)) gf_fop_statfs_req_t;
-typedef struct {
-	struct gf_statfs statfs;
-} __attribute__((packed)) gf_fop_statfs_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-} __attribute__((packed)) gf_fop_flush_req_t;
-typedef struct { } __attribute__((packed)) gf_fop_flush_rsp_t;
-
-
-typedef struct fsync_req {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint32_t data;
-} __attribute__((packed)) gf_fop_fsync_req_t;
-typedef struct {
-        struct gf_stat prestat;
-        struct gf_stat poststat;
-} __attribute__((packed)) gf_fop_fsync_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t flags;
-	uint32_t dict_len;
-	char     dict[0];
-	char     path[0];
-} __attribute__((packed)) gf_fop_setxattr_req_t;
-typedef struct { } __attribute__((packed)) gf_fop_setxattr_rsp_t;
-
-
-typedef struct {
-        uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint32_t flags;
-	uint32_t dict_len;
-	char     dict[0];
-} __attribute__((packed)) gf_fop_fsetxattr_req_t;
-typedef struct { } __attribute__((packed)) gf_fop_fsetxattr_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t flags;
-	uint32_t dict_len;
-	char     dict[0];
-	char     path[0];
-} __attribute__((packed)) gf_fop_xattrop_req_t;
-
-typedef struct {
-	uint32_t dict_len;
-	char  dict[0];
-} __attribute__((packed)) gf_fop_xattrop_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint32_t flags;
-	uint32_t dict_len;
-	char     dict[0];
-} __attribute__((packed)) gf_fop_fxattrop_req_t;
-
-typedef struct {
-	uint32_t dict_len;
-	char  dict[0];
-} __attribute__((packed)) gf_fop_fxattrop_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t namelen;
-	char     path[0];
-	char     name[0];
-} __attribute__((packed)) gf_fop_getxattr_req_t;
-typedef struct {
-	uint32_t dict_len;
-	char     dict[0];
-} __attribute__((packed)) gf_fop_getxattr_rsp_t;
-
-
-typedef struct {
-        uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-        uint32_t namelen;
-	char     name[0];
-} __attribute__((packed)) gf_fop_fgetxattr_req_t;
-typedef struct {
-	uint32_t dict_len;
-	char     dict[0];
-} __attribute__((packed)) gf_fop_fgetxattr_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	char     path[0];
-	char     name[0];
-} __attribute__((packed)) gf_fop_removexattr_req_t;
-typedef struct { } __attribute__((packed)) gf_fop_removexattr_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	char     path[0];
-} __attribute__((packed)) gf_fop_opendir_req_t;
-typedef struct {
-	int64_t fd;
-} __attribute__((packed)) gf_fop_opendir_rsp_t;
-
-
-typedef struct fsyncdir_req {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	int32_t  data;
-} __attribute__((packed)) gf_fop_fsyncdir_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_fsyncdir_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint64_t offset;
-	uint32_t size;
-} __attribute__((packed)) gf_fop_readdir_req_t;
-typedef struct {
-	uint32_t size;
-	char     buf[0];
-} __attribute__((packed)) gf_fop_readdir_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint64_t offset;
-	uint32_t size;
-} __attribute__((packed)) gf_fop_readdirp_req_t;
-typedef struct {
-	uint32_t size;
-	char     buf[0];
-} __attribute__((packed)) gf_fop_readdirp_rsp_t;
-
-
-typedef struct  {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t mask;
-	char     path[0];
-} __attribute__((packed)) gf_fop_access_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_access_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	uint32_t flags;
-	uint32_t mode;
-	char     path[0];
-	char     bname[0];
-} __attribute__((packed)) gf_fop_create_req_t;
-typedef struct {
-	struct gf_stat stat;
-	uint64_t       fd;
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_create_rsp_t;
-
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint64_t offset;
-} __attribute__((packed)) gf_fop_ftruncate_req_t;
-typedef struct {
-	struct gf_stat prestat;
-        struct gf_stat poststat;
-} __attribute__((packed)) gf_fop_ftruncate_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-} __attribute__((packed)) gf_fop_fstat_req_t;
-typedef struct {
-	struct gf_stat stat;
-} __attribute__((packed)) gf_fop_fstat_rsp_t;
-
-
-typedef struct {
-	uint64_t        ino;
-        uint64_t        gen;
-	int64_t         fd;
-	uint32_t        cmd;
-	uint32_t        type;
-	struct gf_flock flock;
-} __attribute__((packed)) gf_fop_lk_req_t;
-typedef struct {
-	struct gf_flock flock;
-} __attribute__((packed)) gf_fop_lk_rsp_t;
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t cmd;
-	uint32_t type;
-	struct gf_flock flock;
-	char     path[0];
-        char     volume[0];
-} __attribute__((packed)) gf_fop_inodelk_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_inodelk_rsp_t;
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint32_t cmd;
-	uint32_t type;
-	struct gf_flock flock;
-        char volume[0];
-} __attribute__((packed)) gf_fop_finodelk_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_finodelk_rsp_t;
-
-typedef struct {
-	uint64_t  ino;
-        uint64_t  gen;
-	uint32_t  cmd;
-	uint32_t  type;
-	uint64_t  namelen;
-	char      path[0];
-	char      name[0];
-        char      volume[0];
-} __attribute__((packed)) gf_fop_entrylk_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_entrylk_rsp_t;
-
-typedef struct {
-	uint64_t  ino;
-        uint64_t  gen;
-	int64_t   fd;
-	uint32_t  cmd;
-	uint32_t  type;
-	uint64_t  namelen;
-	char      name[0];
-        char      volume[0];
-} __attribute__((packed)) gf_fop_fentrylk_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_fentrylk_rsp_t;
-
-typedef struct {
-	uint64_t ino; /* NOTE: used only in case of 'root' lookup */
-	uint64_t par;
-        uint64_t gen;
-	uint32_t flags;
-	uint32_t dictlen;
-	char     path[0];
-	char     bname[0];
-	char     dict[0];
-} __attribute__((packed)) gf_fop_lookup_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat postparent;
-	uint32_t       dict_len;
-	char           dict[0];
-} __attribute__((packed)) gf_fop_lookup_rsp_t;
-
-typedef struct {
-	uint64_t  ino;
-        uint64_t  gen;
-	uint32_t  flag;
-	char      path[0];
-} __attribute__((packed)) gf_fop_checksum_req_t;
-typedef struct {
-	unsigned char fchecksum[0];
-	unsigned char dchecksum[0];
-} __attribute__((packed)) gf_fop_checksum_rsp_t;
-
-typedef struct {
-        uint64_t       ino;
-        uint64_t       gen;
-        struct gf_stat stbuf;
-        int32_t        valid;
-        char           path[0];
-} __attribute__((packed)) gf_fop_setattr_req_t;
-typedef struct {
-        struct gf_stat statpre;
-        struct gf_stat statpost;
-} __attribute__((packed)) gf_fop_setattr_rsp_t;
-
-typedef struct {
-        int64_t        fd;
-        struct gf_stat stbuf;
-        int32_t        valid;
-} __attribute__((packed)) gf_fop_fsetattr_req_t;
-typedef struct {
-        struct gf_stat statpre;
-        struct gf_stat statpost;
-} __attribute__((packed)) gf_fop_fsetattr_rsp_t;
-
-typedef struct {
-        int64_t   fd;
-        uint64_t  offset;
-        uint32_t  len;
-} __attribute__((packed)) gf_fop_rchecksum_req_t;
-typedef struct {
-        uint32_t weak_checksum;
-        unsigned char strong_checksum[0];
-} __attribute__((packed)) gf_fop_rchecksum_rsp_t;
-
-typedef struct {
-	uint32_t flags;
-	uint32_t keylen;
-	char     key[0];
-} __attribute__((packed)) gf_mop_getspec_req_t;
-typedef struct {
-	char spec[0];
-} __attribute__((packed)) gf_mop_getspec_rsp_t;
-
-
-typedef struct {
-        uint32_t msglen;
-	char     msg[0];
-} __attribute__((packed)) gf_mop_log_req_t;
-typedef struct {
-} __attribute__((packed)) gf_mop_log_rsp_t;
-
-
-typedef struct {
-	uint32_t dict_len;
-	char buf[0];
-} __attribute__((packed)) gf_mop_setvolume_req_t;
-typedef struct {
-	uint32_t dict_len;
-	char buf[0];
-} __attribute__((packed)) gf_mop_setvolume_rsp_t;
-
-
-typedef struct {
-} __attribute__((packed)) gf_mop_ping_req_t;
-typedef struct {
-} __attribute__((packed)) gf_mop_ping_rsp_t;
-
-typedef struct {
-	uint32_t  flags;
-        char buf[0];
-} __attribute__((packed)) gf_mop_notify_req_t;
-typedef struct {
-	uint32_t  flags;
-        char buf[0];
-} __attribute__((packed)) gf_mop_notify_rsp_t;
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-} __attribute__((packed)) gf_cbk_releasedir_req_t;
-typedef struct {
-} __attribute__((packed)) gf_cbk_releasedir_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-} __attribute__((packed)) gf_cbk_release_req_t;
-typedef struct {
-} __attribute__((packed)) gf_cbk_release_rsp_t;
-
-
-typedef struct {
-	uint32_t count;
-	uint64_t ino_array[0];
-} __attribute__((packed)) gf_cbk_forget_req_t;
-typedef struct { } __attribute__((packed)) gf_cbk_forget_rsp_t;
-
-typedef struct {
-	uint32_t pid;
-	uint32_t uid;
-	uint32_t gid;
-
-        /* Number of groups being sent through the array above. */
-        uint32_t ngrps;
-
-        /* Array of groups to which the uid belongs apart from the primary group
-         * in gid.
-         */
-        uint32_t groups[GF_REQUEST_MAXGROUPS];
-
-        uint64_t lk_owner;
-} __attribute__ ((packed)) gf_hdr_req_t;
-
-
-typedef struct {
-	uint32_t op_ret;
-	uint32_t op_errno;
-} __attribute__ ((packed)) gf_hdr_rsp_t;
-
-
-typedef struct {
-	uint64_t callid;
-	uint32_t type;
-	uint32_t op;
-	uint32_t size;
-	union {
-		gf_hdr_req_t req;
-		gf_hdr_rsp_t rsp;
-	} __attribute__ ((packed));
-} __attribute__ ((packed)) gf_hdr_common_t;
-
-
-static inline gf_hdr_common_t *
-__gf_hdr_new (int size)
-{
-	gf_hdr_common_t *hdr = NULL;
-
-	/* TODO: use mem-pool */
-	hdr = GF_CALLOC (sizeof (gf_hdr_common_t) + size, 1,
-                         gf_common_mt_gf_hdr_common_t);
-
-	if (!hdr) {
-		return NULL;
-	}
-
-	hdr->size = hton32 (size);
-
-	return hdr;
-}
-
-
-#define gf_hdr_len(type, x) (sizeof (gf_hdr_common_t) + sizeof (*type) + x)
-#define gf_hdr_new(type, x) __gf_hdr_new (sizeof (*type) + x)
-
-
-static inline void *
-gf_param (gf_hdr_common_t *hdr)
-{
-	return ((void *)hdr) + sizeof (*hdr);
-}
-
-
-struct gf_dirent_nb {
-	uint64_t       d_ino;
-	uint64_t       d_off;
-	uint32_t       d_len;
-	uint32_t       d_type;
-        struct gf_stat d_stat;
-	char           d_name[0];
-} __attribute__((packed));
-
-int
-gf_dirent_unserialize (gf_dirent_t *entries, const char *buf, size_t buf_size);
-int
-gf_dirent_serialize (gf_dirent_t *entries, char *buf, size_t buf_size);
-
-int protocol_common_init (void);
-
-#endif
diff --git a/xlators/protocol/legacy/lib/src/transport.c b/xlators/protocol/legacy/lib/src/transport.c
deleted file mode 100644
index 4396613ec..000000000
--- a/xlators/protocol/legacy/lib/src/transport.c
+++ /dev/null
@@ -1,422 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <dlfcn.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <sys/poll.h>
-#include <fnmatch.h>
-#include <stdint.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "logging.h"
-#include "transport.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "list.h"
-
-
-transport_t *
-transport_load (dict_t *options,
-		xlator_t *xl)
-{
-	struct transport *trans = NULL, *return_trans = NULL;
-	char *name = NULL;
-	void *handle = NULL;
-	char *type = NULL;
-	char str[] = "ERROR";
-	int32_t ret = -1;
-	int8_t is_tcp = 0, is_unix = 0, is_ibsdp = 0;
-	volume_opt_list_t *vol_opt = NULL;
-
-	GF_VALIDATE_OR_GOTO("transport", options, fail);
-	GF_VALIDATE_OR_GOTO("transport", xl, fail);
-  
-	trans = GF_CALLOC (1, sizeof (struct transport),
-                           gf_common_mt_transport);
-	GF_VALIDATE_OR_GOTO("transport", trans, fail);
-
-	trans->xl = xl;
-	type = str;
-
-	/* Backward compatibility */
-	ret = dict_get_str (options, "transport-type", &type);
-	if (ret < 0) {
-		ret = dict_set_str (options, "transport-type", "socket");
-		if (ret < 0)
-			gf_log ("dict", GF_LOG_DEBUG,
-				"setting transport-type failed");
-		gf_log ("transport", GF_LOG_WARNING,
-			"missing 'option transport-type'. defaulting to "
-			"\"socket\"");
-	} else {
-		{
-			/* Backword compatibility to handle * /client,
-			 * * /server. 
-			 */
-			char *tmp = strchr (type, '/');
-			if (tmp)
-				*tmp = '\0';
-		}
-		
-		is_tcp = strcmp (type, "tcp");
-		is_unix = strcmp (type, "unix");
-		is_ibsdp = strcmp (type, "ib-sdp");
-		if ((is_tcp == 0) ||
-		    (is_unix == 0) ||
-		    (is_ibsdp == 0)) {
-			if (is_unix == 0)
-				ret = dict_set_str (options, 
-						    "transport.address-family",
-						    "unix");
-			if (is_ibsdp == 0)
-				ret = dict_set_str (options, 
-						    "transport.address-family",
-						    "inet-sdp");
-
-			if (ret < 0)
-				gf_log ("dict", GF_LOG_DEBUG,
-					"setting address-family failed");
-
-			ret = dict_set_str (options, 
-					    "transport-type", "socket");
-			if (ret < 0)
-				gf_log ("dict", GF_LOG_DEBUG,
-					"setting transport-type failed");
-		}
-	}
-
-	ret = dict_get_str (options, "transport-type", &type);
-	if (ret < 0) {
-		GF_FREE (trans);
-		gf_log ("transport", GF_LOG_ERROR,
-			"'option transport-type <xx>' missing in volume '%s'",
-			xl->name);
-		goto fail;
-	}
-
-	ret = gf_asprintf (&name, "%s/%s.so", TRANSPORTDIR, type);
-        if (-1 == ret) {
-                gf_log ("transport", GF_LOG_ERROR, "asprintf failed");
-                goto fail;
-        }
-	gf_log ("transport", GF_LOG_DEBUG,
-		"attempt to load file %s", name);
-
-	handle = dlopen (name, RTLD_NOW|RTLD_GLOBAL);
-	if (handle == NULL) {
-		gf_log ("transport", GF_LOG_ERROR, "%s", dlerror ());
-		gf_log ("transport", GF_LOG_ERROR,
-			"volume '%s': transport-type '%s' is not valid or "
-			"not found on this machine", 
-			xl->name, type);
-		GF_FREE (name);
-		GF_FREE (trans);
-		goto fail;
-	}
-	GF_FREE (name);
-	
-	trans->ops = dlsym (handle, "tops");
-	if (trans->ops == NULL) {
-		gf_log ("transport", GF_LOG_ERROR,
-			"dlsym (transport_ops) on %s", dlerror ());
-		GF_FREE (trans);
-		goto fail;
-	}
-
-	trans->init = dlsym (handle, "init");
-	if (trans->init == NULL) {
-		gf_log ("transport", GF_LOG_ERROR,
-			"dlsym (gf_transport_init) on %s", dlerror ());
-		GF_FREE (trans);
-		goto fail;
-	}
-
-	trans->fini = dlsym (handle, "fini");
-	if (trans->fini == NULL) {
-		gf_log ("transport", GF_LOG_ERROR,
-			"dlsym (gf_transport_fini) on %s", dlerror ());
-		GF_FREE (trans);
-		goto fail;
-	}
-	
-	vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
-                             gf_common_mt_volume_opt_list_t);
-	vol_opt->given_opt = dlsym (handle, "options");
-	if (vol_opt->given_opt == NULL) {
-		gf_log ("transport", GF_LOG_DEBUG,
-			"volume option validation not specified");
-	} else {
-		list_add_tail (&vol_opt->list, &xl->volume_options);
-		if (-1 == 
-		    validate_xlator_volume_options (xl, 
-						    vol_opt->given_opt)) {
-			gf_log ("transport", GF_LOG_ERROR,
-				"volume option validation failed");
-			GF_FREE (trans);
-			goto fail;
-		}
-	}
-	
-	ret = trans->init (trans);
-	if (ret != 0) {
-		gf_log ("transport", GF_LOG_ERROR,
-			"'%s' initialization failed", type);
-		GF_FREE (trans);
-		goto fail;
-	}
-
-	pthread_mutex_init (&trans->lock, NULL);
-	return_trans = trans;
-fail:
-	return return_trans;
-}
-
-
-int32_t 
-transport_submit (transport_t *this, char *buf, int32_t len,
-		  struct iovec *vector, int count,
-                  struct iobref *iobref)
-{
-	int32_t               ret = -1;
-        transport_t          *peer_trans = NULL;
-        struct iobuf         *iobuf = NULL;
-        struct transport_msg *msg = NULL;
-
-        if (this->peer_trans) {
-                peer_trans = this->peer_trans;
-
-                msg = GF_CALLOC (1, sizeof (*msg),
-                                gf_common_mt_transport_msg);
-                if (!msg) {
-                        return -ENOMEM;
-                }
-
-                msg->hdr = buf;
-                msg->hdrlen = len;
-
-                if (vector) {
-                        iobuf = iobuf_get (this->xl->ctx->iobuf_pool);
-                        if (!iobuf) {
-                                GF_FREE (msg->hdr);
-                                GF_FREE (msg);
-                                return -ENOMEM;
-                        }
-
-                        iov_unload (iobuf->ptr, vector, count);
-                        msg->iobuf = iobuf;
-                }
-
-                pthread_mutex_lock (&peer_trans->handover.mutex);
-                {
-                        list_add_tail (&msg->list, &peer_trans->handover.msgs);
-                        pthread_cond_broadcast (&peer_trans->handover.cond);
-                }
-                pthread_mutex_unlock (&peer_trans->handover.mutex);
-
-                return 0;
-        }
-
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-	GF_VALIDATE_OR_GOTO("transport", this->ops, fail);
-	
-	ret = this->ops->submit (this, buf, len, vector, count, iobref);
-fail:
-	return ret;
-}
-
-
-int32_t 
-transport_connect (transport_t *this)
-{
-	int ret = -1;
-	
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-  
-	ret = this->ops->connect (this);
-fail:
-	return ret;
-}
-
-
-int32_t
-transport_listen (transport_t *this)
-{
-	int ret = -1;
-	
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-  
-	ret = this->ops->listen (this);
-fail:
-	return ret;
-}
-
-
-int32_t 
-transport_disconnect (transport_t *this)
-{
-	int32_t ret = -1;
-	
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-  
-	ret = this->ops->disconnect (this);
-fail:
-	return ret;
-}
-
-
-int32_t 
-transport_destroy (transport_t *this)
-{
-	int32_t ret = -1;
-
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-  
-	if (this->fini)
-		this->fini (this);
-
-	pthread_mutex_destroy (&this->lock);
-	GF_FREE (this);
-fail:
-	return ret;
-}
-
-
-transport_t *
-transport_ref (transport_t *this)
-{
-	transport_t *return_this = NULL;
-
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-	
-	pthread_mutex_lock (&this->lock);
-	{
-		this->refcount ++;
-	}
-	pthread_mutex_unlock (&this->lock);
-	
-	return_this = this;
-fail:
-	return return_this;
-}
-
-
-int32_t
-transport_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
-		   struct iobuf **iobuf_p)
-{
-	int32_t ret = -1;
-
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-
-        if (this->peer_trans) {
-                *hdr_p = this->handover.msg->hdr;
-                *hdrlen_p = this->handover.msg->hdrlen;
-                *iobuf_p = this->handover.msg->iobuf;
-
-                return 0;
-        }
-
-	ret = this->ops->receive (this, hdr_p, hdrlen_p, iobuf_p);
-fail:
-	return ret;
-}
-
-
-int32_t
-transport_unref (transport_t *this)
-{
-	int32_t refcount = 0;
-	int32_t ret = -1;
-
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-  
-	pthread_mutex_lock (&this->lock);
-	{
-		refcount = --this->refcount;
-	}
-	pthread_mutex_unlock (&this->lock);
-
-	if (refcount == 0) {
-		xlator_notify (this->xl, GF_EVENT_TRANSPORT_CLEANUP, this);
-		transport_destroy (this);
-	}
-	
-	ret = 0;
-fail:
-	return ret;
-}
-
-
-void *
-transport_peerproc (void *trans_data)
-{
-        transport_t          *trans = NULL;
-        struct transport_msg *msg = NULL;
-
-        trans = trans_data;
-
-        while (1) {
-                pthread_mutex_lock (&trans->handover.mutex);
-                {
-                        while (list_empty (&trans->handover.msgs))
-                                pthread_cond_wait (&trans->handover.cond,
-                                                   &trans->handover.mutex);
-
-                        msg = list_entry (trans->handover.msgs.next,
-                                          struct transport_msg, list);
-
-                        list_del_init (&msg->list);
-                }
-                pthread_mutex_unlock (&trans->handover.mutex);
-
-                trans->handover.msg = msg;
-
-                xlator_notify (trans->xl, GF_EVENT_POLLIN, trans);
-
-                GF_FREE (msg);
-        }
-}
-
-
-int
-transport_setpeer (transport_t *trans, transport_t *peer_trans)
-{
-        trans->peer_trans = transport_ref (peer_trans);
-
-        INIT_LIST_HEAD (&trans->handover.msgs);
-        pthread_cond_init (&trans->handover.cond, NULL);
-        pthread_mutex_init (&trans->handover.mutex, NULL);
-        pthread_create (&trans->handover.thread, NULL,
-                        transport_peerproc, trans);
-
-        peer_trans->peer_trans = transport_ref (trans);
-
-        INIT_LIST_HEAD (&peer_trans->handover.msgs);
-        pthread_cond_init (&peer_trans->handover.cond, NULL);
-        pthread_mutex_init (&peer_trans->handover.mutex, NULL);
-        pthread_create (&peer_trans->handover.thread, NULL,
-                        transport_peerproc, peer_trans);
-
-        return 0;
-}
diff --git a/xlators/protocol/legacy/lib/src/transport.h b/xlators/protocol/legacy/lib/src/transport.h
deleted file mode 100644
index 5b8b62caf..000000000
--- a/xlators/protocol/legacy/lib/src/transport.h
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __TRANSPORT_H__
-#define __TRANSPORT_H__
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <inttypes.h>
-
-struct transport_ops;
-typedef struct transport transport_t;
-
-#include "xlator.h"
-#include "dict.h"
-#include "compat.h"
-
-typedef struct peer_info {
-	struct sockaddr_storage sockaddr;
-	socklen_t sockaddr_len;
-	char identifier[UNIX_PATH_MAX];
-}peer_info_t;
-
-struct transport_msg {
-        struct list_head  list;
-        char             *hdr;
-        int               hdrlen;
-        struct iobuf     *iobuf;
-};
-
-struct transport {
-	struct transport_ops  *ops;
-	void                  *private;
-	void                  *xl_private;
-	pthread_mutex_t        lock;
-	int32_t                refcount;
-
-	xlator_t              *xl;
-	void                  *dnscache;
-	data_t                *buf;
-	int32_t              (*init)   (transport_t *this);
-	void                 (*fini)   (transport_t *this);
-	/*  int                  (*notify) (transport_t *this, int event, void *data); */
-	peer_info_t     peerinfo;
-	peer_info_t     myinfo;
-
-        transport_t    *peer_trans;
-        struct {
-                pthread_mutex_t       mutex;
-                pthread_cond_t        cond;
-                pthread_t             thread;
-                struct list_head      msgs;
-                struct transport_msg *msg;
-        } handover;
-                
-};
-
-struct transport_ops {
-	int32_t (*receive)    (transport_t *this, char **hdr_p, size_t *hdrlen_p,
-                               struct iobuf **iobuf_p);
-	int32_t (*submit)     (transport_t *this, char *buf, int len,
-                               struct iovec *vector, int count,
-                               struct iobref *iobref);
-	int32_t (*connect)    (transport_t *this);
-	int32_t (*listen)     (transport_t *this);
-	int32_t (*disconnect) (transport_t *this);
-};
-
-
-int32_t transport_listen     (transport_t *this);
-int32_t transport_connect    (transport_t *this);
-int32_t transport_disconnect (transport_t *this);
-int32_t transport_notify     (transport_t *this, int event);
-int32_t transport_submit     (transport_t *this, char *buf, int len,
-                              struct iovec *vector, int count,
-                              struct iobref *iobref);
-int32_t transport_receive    (transport_t *this, char **hdr_p, size_t *hdrlen_p,
-                              struct iobuf **iobuf_p);
-int32_t transport_destroy    (transport_t *this);
-
-transport_t *transport_load  (dict_t *options, xlator_t *xl);
-transport_t *transport_ref   (transport_t *trans);
-int32_t      transport_unref (transport_t *trans);
-
-int transport_setpeer (transport_t *trans, transport_t *trans_peer);
-
-#endif /* __TRANSPORT_H__ */
diff --git a/xlators/protocol/legacy/server/Makefile.am b/xlators/protocol/legacy/server/Makefile.am
deleted file mode 100644
index d471a3f92..000000000
--- a/xlators/protocol/legacy/server/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/protocol/legacy/server/src/Makefile.am b/xlators/protocol/legacy/server/src/Makefile.am
deleted file mode 100644
index 262fec9ea..000000000
--- a/xlators/protocol/legacy/server/src/Makefile.am
+++ /dev/null
@@ -1,27 +0,0 @@
-
-xlator_LTLIBRARIES = server-old.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/legacy/protocol
-
-server_old_la_LDFLAGS = -module -avoidversion
-
-server_old_la_SOURCES = server-protocol.c server-resolve.c server-helpers.c \
-			authenticate.c
-
-server_old_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la   \
-	$(top_builddir)/xlators/protocol/legacy/lib/src/libgfproto.la
-
-noinst_HEADERS = server-protocol.h server-helpers.h server-mem-types.h \
-	authenticate.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
-	-I$(top_srcdir)/contrib/md5/ \
-	-DDATADIR=\"$(localstatedir)\" -DCONFDIR=\"$(sysconfdir)/glusterfs\" \
-	-DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" \
-	$(GF_CFLAGS) -I$(top_srcdir)/xlators/protocol/legacy/lib/src \
-	-I$(top_srcdir)/xlators/protocol/lib/src
-
-CLEANFILES =
-
-install-data-hook:
-	ln -sf server-old.so $(DESTDIR)$(xlatordir)/server.so
diff --git a/xlators/protocol/legacy/server/src/authenticate.c b/xlators/protocol/legacy/server/src/authenticate.c
deleted file mode 100644
index a01d7003c..000000000
--- a/xlators/protocol/legacy/server/src/authenticate.c
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-
-#include <stdio.h>
-#include <dlfcn.h>
-#include <errno.h>
-#include "authenticate.h"
-
-static void
-init (dict_t *this,
-      char *key,
-      data_t *value,
-      void *data)
-{
-	void *handle = NULL;
-	char *auth_file = NULL;
-	auth_handle_t *auth_handle = NULL;
-	auth_fn_t authenticate = NULL;
-	int *error = NULL;
-        int  ret = 0;
-
-	/* It gets over written */
-	error = data;
-
-	if (!strncasecmp (key, "ip", strlen ("ip"))) {
-		gf_log ("authenticate", GF_LOG_ERROR,
-			"AUTHENTICATION MODULE \"IP\" HAS BEEN REPLACED "
-			"BY \"ADDR\"");
-		dict_set (this, key, data_from_dynptr (NULL, 0));
-		/* TODO: 1.3.x backword compatibility */
-		// *error = -1;
-		// return;
-		key = "addr";
-	}
-
-	ret = gf_asprintf (&auth_file, "%s/%s.so", LIBDIR, key);
-        if (-1 == ret) {
-                gf_log ("authenticate", GF_LOG_ERROR, "asprintf failed");
-                dict_set (this, key, data_from_dynptr (NULL, 0));
-                *error = -1;
-                return;
-        }
-
-	handle = dlopen (auth_file, RTLD_LAZY);
-	if (!handle) {
-		gf_log ("authenticate", GF_LOG_ERROR, "dlopen(%s): %s\n",
-			auth_file, dlerror ());
-		dict_set (this, key, data_from_dynptr (NULL, 0));
-		GF_FREE (auth_file);
-		*error = -1;
-		return;
-	}
-	GF_FREE (auth_file);
-
-	authenticate = dlsym (handle, "gf_auth");
-	if (!authenticate) {
-		gf_log ("authenticate", GF_LOG_ERROR,
-			"dlsym(gf_auth) on %s\n", dlerror ());
-		dict_set (this, key, data_from_dynptr (NULL, 0));
-		*error = -1;
-		return;
-	}
-
-	auth_handle = GF_CALLOC (1, sizeof (*auth_handle),
-                                 gf_common_mt_auth_handle_t);
-	if (!auth_handle) {
-		gf_log ("authenticate", GF_LOG_ERROR, "Out of memory");
-		dict_set (this, key, data_from_dynptr (NULL, 0));
-		*error = -1;
-		return;
-	}
-	auth_handle->vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
-                                       gf_common_mt_volume_opt_list_t);
-	auth_handle->vol_opt->given_opt = dlsym (handle, "options");
-	if (auth_handle->vol_opt->given_opt == NULL) {
-		gf_log ("authenticate", GF_LOG_DEBUG,
-			"volume option validation not specified");
-	}
-
-	auth_handle->authenticate = authenticate;
-	auth_handle->handle = handle;
-
-	dict_set (this, key,
-		  data_from_dynptr (auth_handle, sizeof (*auth_handle)));
-}
-
-static void
-fini (dict_t *this,
-      char *key,
-      data_t *value,
-      void *data)
-{
-	auth_handle_t *handle = data_to_ptr (value);
-	if (handle) {
-		dlclose (handle->handle);
-	}
-}
-
-int32_t
-gf_auth_init (xlator_t *xl, dict_t *auth_modules)
-{
-	int ret = 0;
-	auth_handle_t *handle = NULL;
-	data_pair_t *pair = NULL;
-	dict_foreach (auth_modules, init, &ret);
-	if (!ret) {
-		pair = auth_modules->members_list;
-		while (pair) {
-			handle = data_to_ptr (pair->value);
-			if (handle) {
-				list_add_tail (&(handle->vol_opt->list),
-					       &(xl->volume_options));
-				if (-1 ==
-				    validate_xlator_volume_options (xl,
-								    handle->vol_opt->given_opt)) {
-					gf_log ("authenticate", GF_LOG_ERROR,
-						"volume option validation "
-						"failed");
-					ret = -1;
-				}
-			}
-			pair = pair->next;
-		}
-	}
-	if (ret) {
-		gf_log (xl->name, GF_LOG_ERROR, "authentication init failed");
-		dict_foreach (auth_modules, fini, &ret);
-		ret = -1;
-	}
-	return ret;
-}
-
-static dict_t *__input_params;
-static dict_t *__config_params;
-
-void
-map (dict_t *this,
-     char *key,
-     data_t *value,
-     void *data)
-{
-	dict_t *res = data;
-	auth_fn_t authenticate;
-	auth_handle_t *handle = NULL;
-
-	if (value && (handle = data_to_ptr (value)) &&
-	    (authenticate = handle->authenticate)) {
-		dict_set (res, key,
-			  int_to_data (authenticate (__input_params,
-						     __config_params)));
-	} else {
-		dict_set (res, key, int_to_data (AUTH_DONT_CARE));
-	}
-}
-
-void
-reduce (dict_t *this,
-	char *key,
-	data_t *value,
-	void *data)
-{
-	int64_t val = 0;
-	int64_t *res = data;
-	if (!data)
-		return;
-
-	val = data_to_int64 (value);
-	switch (val)
-	{
-	case AUTH_ACCEPT:
-		if (AUTH_DONT_CARE == *res)
-			*res = AUTH_ACCEPT;
-		break;
-
-	case AUTH_REJECT:
-		*res = AUTH_REJECT;
-		break;
-
-	case AUTH_DONT_CARE:
-		break;
-	}
-}
-
-
-auth_result_t
-gf_authenticate (dict_t *input_params,
-		 dict_t *config_params,
-		 dict_t *auth_modules)
-{
-	dict_t *results = NULL;
-	int64_t result = AUTH_DONT_CARE;
-
-	results = get_new_dict ();
-	__input_params = input_params;
-	__config_params = config_params;
-
-	dict_foreach (auth_modules, map, results);
-
-	dict_foreach (results, reduce, &result);
-	if (AUTH_DONT_CARE == result) {
-		data_t *peerinfo_data = dict_get (input_params, "peer-info-name");
-		char *name = NULL;
-
-		if (peerinfo_data) {
-			name = peerinfo_data->data;
-		}
-
-		gf_log ("auth", GF_LOG_ERROR,
-			"no authentication module is interested in "
-			"accepting remote-client %s", name);
-		result = AUTH_REJECT;
-	}
-
-	dict_destroy (results);
-	return result;
-}
-
-void
-gf_auth_fini (dict_t *auth_modules)
-{
-	int32_t dummy;
-
-	dict_foreach (auth_modules, fini, &dummy);
-}
diff --git a/xlators/protocol/legacy/server/src/authenticate.h b/xlators/protocol/legacy/server/src/authenticate.h
deleted file mode 100644
index d35304db2..000000000
--- a/xlators/protocol/legacy/server/src/authenticate.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _AUTHENTICATE_H
-#define _AUTHENTICATE_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-
-#include <stdio.h>
-#include <fnmatch.h>
-#include "dict.h"
-#include "compat.h"
-#include "list.h"
-#include "xlator.h"
-
-typedef enum {
-	AUTH_ACCEPT,
-	AUTH_REJECT,
-	AUTH_DONT_CARE
-} auth_result_t;
-
-typedef auth_result_t (*auth_fn_t) (dict_t *input_params,
-				    dict_t *config_params);
-
-typedef struct {
-	void              *handle;
-	auth_fn_t          authenticate;
-	volume_opt_list_t *vol_opt;
-} auth_handle_t;
-
-auth_result_t gf_authenticate (dict_t *input_params,
-			       dict_t *config_params,
-			       dict_t *auth_modules);
-int32_t gf_auth_init (xlator_t *xl, dict_t *auth_modules);
-void gf_auth_fini (dict_t *auth_modules);
-
-#endif /* _AUTHENTICATE_H */
diff --git a/xlators/protocol/legacy/server/src/server-helpers.c b/xlators/protocol/legacy/server/src/server-helpers.c
deleted file mode 100644
index 15e46b3d6..000000000
--- a/xlators/protocol/legacy/server/src/server-helpers.c
+++ /dev/null
@@ -1,622 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "server-protocol.h"
-#include "server-helpers.h"
-
-
-
-void
-old_server_loc_wipe (loc_t *loc)
-{
-        if (loc->parent) {
-                inode_unref (loc->parent);
-                loc->parent = NULL;
-        }
-
-        if (loc->inode) {
-                inode_unref (loc->inode);
-                loc->inode = NULL;
-        }
-
-	if (loc->path)
-		GF_FREE ((char *)loc->path);
-}
-
-
-static void
-old_server_resolve_wipe (server_resolve_t *resolve)
-{
-        struct resolve_comp *comp = NULL;
-        int                  i = 0;
-
-        if (resolve->path)
-                GF_FREE (resolve->path);
-
-        if (resolve->bname)
-                GF_FREE (resolve->bname);
-
-        if (resolve->resolved)
-                GF_FREE (resolve->resolved);
-
-        loc_wipe (&resolve->deep_loc);
-
-        comp = resolve->components;
-        if (comp) {
-                for (i = 0; comp[i].basename; i++) {
-                        if (comp[i].inode)
-                                inode_unref (comp[i].inode);
-                }
-                GF_FREE (resolve->components);
-        }
-}
-
-
-void
-free_old_server_state (server_state_t *state)
-{
-        if (state->trans) {
-                transport_unref (state->trans);
-                state->trans = NULL;
-        }
-
-        if (state->fd) {
-                fd_unref (state->fd);
-                state->fd = NULL;
-        }
-
-        if (state->params) {
-                dict_unref (state->params);
-                state->params = NULL;
-        }
-
-        if (state->iobref) {
-                iobref_unref (state->iobref);
-                state->iobref = NULL;
-        }
-
-        if (state->iobuf) {
-                iobuf_unref (state->iobuf);
-                state->iobuf = NULL;
-        }
-
-        if (state->dict) {
-                dict_unref (state->dict);
-                state->dict = NULL;
-        }
-
-        if (state->volume)
-                GF_FREE ((char *)state->volume);
-
-        if (state->name)
-                GF_FREE (state->name);
-
-        old_server_loc_wipe (&state->loc);
-        old_server_loc_wipe (&state->loc2);
-
-        old_server_resolve_wipe (&state->resolve);
-        old_server_resolve_wipe (&state->resolve2);
-
-	GF_FREE (state);
-}
-
-static struct _lock_table *
-gf_lock_table_new (void)
-{
-        struct _lock_table *new = NULL;
-
-	new = GF_CALLOC (1, sizeof (struct _lock_table),
-                         gf_server_mt_lock_table);
-        if (new == NULL) {
-                gf_log ("server-protocol", GF_LOG_CRITICAL,
-                        "failed to allocate memory for new lock table");
-                goto out;
-        }
-        INIT_LIST_HEAD (&new->dir_lockers);
-        INIT_LIST_HEAD (&new->file_lockers);
-        LOCK_INIT (&new->lock);
-out:
-        return new;
-}
-
-
-static int
-gf_server_nop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE(frame);
-
-        if (state)
-                free_old_server_state (state);
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
-static int
-do_lock_table_cleanup (xlator_t *this, server_connection_t *conn,
-                       call_frame_t *frame, struct _lock_table *ltable)
-{
-        struct list_head  file_lockers, dir_lockers;
-        call_frame_t     *tmp_frame = NULL;
-        struct gf_flock      flock = {0, };
-        xlator_t         *bound_xl = NULL;
-        struct _locker   *locker = NULL, *tmp = NULL;
-        int               ret = -1;
-
-        bound_xl = conn->bound_xl;
-        INIT_LIST_HEAD (&file_lockers);
-        INIT_LIST_HEAD (&dir_lockers);
-
-        LOCK (&ltable->lock);
-        {
-                list_splice_init (&ltable->file_lockers,
-                                  &file_lockers);
-
-                list_splice_init (&ltable->dir_lockers, &dir_lockers);
-        }
-        UNLOCK (&ltable->lock);
-
-        GF_FREE (ltable);
-
-        flock.l_type  = F_UNLCK;
-        flock.l_start = 0;
-        flock.l_len   = 0;
-        list_for_each_entry_safe (locker,
-                                  tmp, &file_lockers, lockers) {
-                tmp_frame = copy_frame (frame);
-                if (tmp_frame == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "out of memory");
-                        goto out;
-                }
-                /*
-                   pid = 0 is a special case that tells posix-locks
-                   to release all locks from this transport
-                */
-                tmp_frame->root->pid = 0;
-                tmp_frame->root->trans = conn;
-
-                if (locker->fd) {
-                        STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                    bound_xl,
-                                    bound_xl->fops->finodelk,
-                                    locker->volume,
-                                    locker->fd, F_SETLK, &flock);
-                        fd_unref (locker->fd);
-                } else {
-                        STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                    bound_xl,
-                                    bound_xl->fops->inodelk,
-                                    locker->volume,
-                                    &(locker->loc), F_SETLK, &flock);
-                        loc_wipe (&locker->loc);
-                }
-
-                GF_FREE (locker->volume);
-                
-                list_del_init (&locker->lockers);
-                GF_FREE (locker);
-        }
-
-        tmp = NULL;
-        locker = NULL;
-        list_for_each_entry_safe (locker, tmp, &dir_lockers, lockers) {
-                tmp_frame = copy_frame (frame);
-
-                tmp_frame->root->pid = 0;
-                tmp_frame->root->trans = conn;
-
-                if (locker->fd) {
-                        STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                    bound_xl,
-                                    bound_xl->fops->fentrylk,
-                                    locker->volume,
-                                    locker->fd, NULL,
-                                    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                        fd_unref (locker->fd);
-                } else {
-                        STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                    bound_xl,
-                                    bound_xl->fops->entrylk,
-                                    locker->volume,
-                                    &(locker->loc), NULL,
-                                    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                        loc_wipe (&locker->loc);
-                }
-
-                GF_FREE (locker->volume);
-                
-                list_del_init (&locker->lockers);
-                GF_FREE (locker);
-        }
-        ret = 0;
-
-out:
-        return ret;
-}
-
-
-static int
-server_connection_cleanup_flush_cbk (call_frame_t *frame, void *cookie,
-                                     xlator_t *this, int32_t op_ret,
-                                     int32_t op_errno)
-{
-        fd_t *fd = NULL;
-
-        fd = frame->local;
-
-        fd_unref (fd);
-        frame->local = NULL;
-
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
-static int
-do_fd_cleanup (xlator_t *this, server_connection_t *conn, call_frame_t *frame,
-               fdentry_t *fdentries, int fd_count)
-{
-        fd_t               *fd = NULL;
-        int                 i = 0, ret = -1;
-        call_frame_t       *tmp_frame = NULL;
-        xlator_t           *bound_xl = NULL;
-
-        bound_xl = conn->bound_xl;
-        for (i = 0;i < fd_count; i++) {
-                fd = fdentries[i].fd;
-
-                if (fd != NULL) {
-                        tmp_frame = copy_frame (frame);
-                        if (tmp_frame == NULL) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "out of memory");
-                                goto out;
-                        }
-                        tmp_frame->local = fd;
-
-                        tmp_frame->root->pid = 0;
-                        tmp_frame->root->trans = conn;
-                        tmp_frame->root->lk_owner = 0;
-                        STACK_WIND (tmp_frame,
-                                    server_connection_cleanup_flush_cbk,
-                                    bound_xl, bound_xl->fops->flush, fd);
-                }
-        }
-
-        GF_FREE (fdentries);
-        ret = 0;
-
-out:
-        return ret;
-}
-
-static int
-do_connection_cleanup (xlator_t *this, server_connection_t *conn,
-                       struct _lock_table *ltable, fdentry_t *fdentries, int fd_count)
-{
-        int             ret = 0;
-        int             saved_ret = 0;
-        call_frame_t   *frame = NULL;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (frame == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto out;
-        }
-
-        saved_ret = do_lock_table_cleanup (this, conn, frame, ltable);
-
-        if (fdentries != NULL) {
-                ret = do_fd_cleanup (this, conn, frame, fdentries, fd_count);
-        }
-
-        STACK_DESTROY (frame->root);
-
-        if (saved_ret || ret) {
-                ret = -1;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-gf_server_connection_cleanup (xlator_t *this, server_connection_t *conn)
-{
-        char                do_cleanup = 0;
-        struct _lock_table *ltable = NULL;
-        fdentry_t          *fdentries = NULL;
-        uint32_t            fd_count = 0;
-        int                 ret = 0; 
-
-        if (conn == NULL) {
-                goto out;
-        }
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                conn->active_transports--;
-                if (conn->active_transports == 0) {
-                        if (conn->ltable) {
-                                ltable = conn->ltable;
-                                conn->ltable = gf_lock_table_new ();
-                        }
-
-                        if (conn->fdtable) {
-                                fdentries = gf_fd_fdtable_get_all_fds (conn->fdtable,
-                                                                       &fd_count);
-                        }
-                        do_cleanup = 1;
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        if (do_cleanup && conn->bound_xl)
-                ret = do_connection_cleanup (this, conn, ltable, fdentries, fd_count);
-
-out:
-        return ret;
-}
-
-
-static int
-server_connection_destroy (xlator_t *this, server_connection_t *conn)
-{
-        call_frame_t       *frame = NULL, *tmp_frame = NULL;
-        xlator_t           *bound_xl = NULL;
-        int32_t             ret = -1;
-        struct list_head    file_lockers;
-        struct list_head    dir_lockers;
-        struct _lock_table *ltable = NULL;
-        struct _locker     *locker = NULL, *tmp = NULL;
-        struct gf_flock        flock = {0,};
-        fd_t               *fd = NULL;
-        int32_t             i = 0;
-        fdentry_t          *fdentries = NULL;
-        uint32_t             fd_count = 0;
-
-        if (conn == NULL) {
-                ret = 0;
-                goto out;
-        }
-
-        bound_xl = (xlator_t *) (conn->bound_xl);
-
-        if (bound_xl) {
-                /* trans will have ref_count = 1 after this call, but its
-                   ok since this function is called in
-                   GF_EVENT_TRANSPORT_CLEANUP */
-                frame = create_frame (this, this->ctx->pool);
-
-                pthread_mutex_lock (&(conn->lock));
-                {
-                        if (conn->ltable) {
-                                ltable = conn->ltable;
-                                conn->ltable = NULL;
-                        }
-                }
-                pthread_mutex_unlock (&conn->lock);
-
-                INIT_LIST_HEAD (&file_lockers);
-                INIT_LIST_HEAD (&dir_lockers);
-
-                if (ltable) {
-                        LOCK (&ltable->lock);
-                        {
-                                list_splice_init (&ltable->file_lockers,
-                                                  &file_lockers);
-
-                                list_splice_init (&ltable->dir_lockers, &dir_lockers);
-                        }
-                        UNLOCK (&ltable->lock);
-                        GF_FREE (ltable);
-                }
-
-                flock.l_type  = F_UNLCK;
-                flock.l_start = 0;
-                flock.l_len   = 0;
-                list_for_each_entry_safe (locker,
-                                          tmp, &file_lockers, lockers) {
-                        tmp_frame = copy_frame (frame);
-                        /*
-                           pid = 0 is a special case that tells posix-locks
-                           to release all locks from this transport
-                        */
-                        tmp_frame->root->pid = 0;
-                        tmp_frame->root->trans = conn;
-
-                        if (locker->fd) {
-                                STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                            bound_xl,
-                                            bound_xl->fops->finodelk,
-                                            locker->volume,
-                                            locker->fd, F_SETLK, &flock);
-                                fd_unref (locker->fd);
-                        } else {
-                                STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                            bound_xl,
-                                            bound_xl->fops->inodelk,
-                                            locker->volume,
-                                            &(locker->loc), F_SETLK, &flock);
-                                loc_wipe (&locker->loc);
-                        }
-
-                        GF_FREE (locker->volume);
-
-			list_del_init (&locker->lockers);
-			GF_FREE (locker);
-		}
-
-                tmp = NULL;
-                locker = NULL;
-                list_for_each_entry_safe (locker, tmp, &dir_lockers, lockers) {
-                        tmp_frame = copy_frame (frame);
-
-                        tmp_frame->root->pid = 0;
-                        tmp_frame->root->trans = conn;
-
-                        if (locker->fd) {
-                                STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                            bound_xl,
-                                            bound_xl->fops->fentrylk,
-                                            locker->volume,
-                                            locker->fd, NULL,
-                                            ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                                fd_unref (locker->fd);
-                        } else {
-                                STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                            bound_xl,
-                                            bound_xl->fops->entrylk,
-                                            locker->volume,
-                                            &(locker->loc), NULL,
-                                            ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                                loc_wipe (&locker->loc);
-                        }
-
-                        GF_FREE (locker->volume);
-
-
-			list_del_init (&locker->lockers);
-			GF_FREE (locker);
-		}
-
-                pthread_mutex_lock (&(conn->lock));
-                {
-                        if (conn->fdtable) {
-                                fdentries = gf_fd_fdtable_get_all_fds (conn->fdtable,
-                                                                       &fd_count);
-                                gf_fd_fdtable_destroy (conn->fdtable);
-                                conn->fdtable = NULL;
-                        }
-                }
-                pthread_mutex_unlock (&conn->lock);
-
-                if (fdentries != NULL) {
-                        for (i = 0; i < fd_count; i++) {
-                                fd = fdentries[i].fd;
-                                if (fd != NULL) {
-                                        tmp_frame = copy_frame (frame);
-                                        tmp_frame->local = fd;
-
-                                        STACK_WIND (tmp_frame,
-                                                    server_connection_cleanup_flush_cbk,
-                                                    bound_xl,
-                                                    bound_xl->fops->flush,
-                                                    fd);
-                                }
-                        }
-                        GF_FREE (fdentries);
-                }
-        }
-
-        if (frame) {
-                STACK_DESTROY (frame->root);
-        }
-
-        gf_log (this->name, GF_LOG_INFO, "destroyed connection of %s",
-                conn->id);
-
-	GF_FREE (conn->id);
-	GF_FREE (conn);
-
-out:
-        return ret;
-}
-
-
-server_connection_t *
-gf_server_connection_get (xlator_t *this, const char *id)
-{
-	server_connection_t *conn = NULL;
-	server_connection_t *trav = NULL;
-	server_conf_t       *conf = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                list_for_each_entry (trav, &conf->conns, list) {
-                        if (!strcmp (id, trav->id)) {
-                                conn = trav;
-                                break;
-                        }
-                }
-
-		if (!conn) {
-			conn = (void *) GF_CALLOC (1, sizeof (*conn),
-                                        gf_server_mt_server_connection_t);
-
-			conn->id = gf_strdup (id);
-			conn->fdtable = gf_fd_fdtable_alloc ();
-			conn->ltable  = gf_lock_table_new ();
-
-                        pthread_mutex_init (&conn->lock, NULL);
-
-			list_add (&conn->list, &conf->conns);
-		}
-
-                conn->ref++;
-                conn->active_transports++;
-	}
-	pthread_mutex_unlock (&conf->mutex);
-
-        return conn;
-}
-
-
-void
-gf_server_connection_put (xlator_t *this, server_connection_t *conn)
-{
-        server_conf_t       *conf = NULL;
-        server_connection_t *todel = NULL;
-
-        if (conn == NULL) {
-                goto out;
-        }
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                conn->ref--;
-
-                if (!conn->ref) {
-                        list_del_init (&conn->list);
-                        todel = conn;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (todel) {
-                server_connection_destroy (this, todel);
-        }
-
-out:
-        return;
-}
diff --git a/xlators/protocol/legacy/server/src/server-helpers.h b/xlators/protocol/legacy/server/src/server-helpers.h
deleted file mode 100644
index cda0c36ef..000000000
--- a/xlators/protocol/legacy/server/src/server-helpers.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __SERVER_HELPERS_H__
-#define __SERVER_HELPERS_H__
-
-#define CALL_STATE(frame)   ((server_state_t *)frame->root->state)
-
-#define BOUND_XL(frame)     ((xlator_t *) CALL_STATE(frame)->bound_xl)
-
-#define TRANSPORT_FROM_FRAME(frame) ((transport_t *) CALL_STATE(frame)->trans)
-
-#define SERVER_CONNECTION(frame)  \
-	((server_connection_t *) TRANSPORT_FROM_FRAME(frame)->xl_private)
-
-#define SERVER_CONF(frame) \
-	((server_conf_t *)TRANSPORT_FROM_FRAME(frame)->xl->private)
-
-#define TRANSPORT_FROM_XLATOR(this) ((((server_conf_t *)this->private))->trans)
-
-#define INODE_LRU_LIMIT(this)						\
-	(((server_conf_t *)(this->private))->inode_lru_limit)
-
-#define IS_ROOT_INODE(inode) (inode == inode->table->root)
-
-#define IS_NOT_ROOT(pathlen) ((pathlen > 2)? 1 : 0)
-
-void free_old_server_state (server_state_t *state);
-
-void old_server_loc_wipe (loc_t *loc);
-
-#endif /* __SERVER_HELPERS_H__ */
diff --git a/xlators/protocol/legacy/server/src/server-mem-types.h b/xlators/protocol/legacy/server/src/server-mem-types.h
deleted file mode 100644
index 5e15fd90f..000000000
--- a/xlators/protocol/legacy/server/src/server-mem-types.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __AFR_MEM_TYPES_H__
-#define __AFR_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_server_mem_types_ {
-        gf_server_mt_dir_entry_t = gf_common_mt_end + 1,
-        gf_server_mt_volfile_ctx,
-        gf_server_mt_server_state_t,
-        gf_server_mt_server_conf_t,
-        gf_server_mt_locker,
-        gf_server_mt_lock_table,
-        gf_server_mt_char,
-        gf_server_mt_server_connection_t,
-        gf_server_mt_resolve_comp,
-        gf_server_mt_end
-};
-#endif
-
diff --git a/xlators/protocol/legacy/server/src/server-protocol.c b/xlators/protocol/legacy/server/src/server-protocol.c
deleted file mode 100644
index abd339542..000000000
--- a/xlators/protocol/legacy/server/src/server-protocol.c
+++ /dev/null
@@ -1,6587 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-#include <time.h>
-#include <sys/uio.h>
-#include <sys/resource.h>
-
-#include <libgen.h>
-
-#include "transport.h"
-#include "fnmatch.h"
-#include "xlator.h"
-#include "protocol.h"
-#include "server-protocol.h"
-#include "server-helpers.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "list.h"
-#include "dict.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
-#include "md5.h"
-
-
-static void
-print_caller (char *str, int size, call_frame_t *frame)
-{
-        int              filled = 0;
-        server_state_t  *state = NULL;
-        transport_t     *trans = NULL;
-
-        state = CALL_STATE (frame);
-        trans = state->trans;
-
-        filled += snprintf (str + filled, size - filled,
-                            " Callid=%"PRId64", Client=%s",
-                            frame->root->unique,
-                            trans->peerinfo.identifier);
-
-        return;
-}
-
-
-static void
-server_print_resolve (char *str, int size, server_resolve_t *resolve)
-{
-        int filled = 0;
-
-        if (!resolve) {
-                snprintf (str, size, "<nul>");
-                return;
-        }
-
-        filled += snprintf (str + filled, size - filled,
-                            " Resolve={");
-        if (resolve->fd_no != -1)
-                filled += snprintf (str + filled, size - filled,
-                                    "fd=%"PRId64",", (uint64_t) resolve->fd_no);
-        if (resolve->ino)
-                filled += snprintf (str + filled, size - filled,
-                                    "ino=%"PRIu64",", (uint64_t) resolve->ino);
-        if (resolve->par)
-                filled += snprintf (str + filled, size - filled,
-                                    "par=%"PRIu64",", (uint64_t) resolve->par);
-        if (resolve->gen)
-                filled += snprintf (str + filled, size - filled,
-                                    "gen=%"PRIu64",", (uint64_t) resolve->gen);
-        if (resolve->bname)
-                filled += snprintf (str + filled, size - filled,
-                                    "bname=%s,", resolve->bname);
-        if (resolve->path)
-                filled += snprintf (str + filled, size - filled,
-                                    "path=%s", resolve->path);
-
-        filled += snprintf (str + filled, size - filled, "}");
-}
-
-
-static void
-server_print_loc (char *str, int size, loc_t *loc)
-{
-        int filled = 0;
-
-        if (!loc) {
-                snprintf (str, size, "<nul>");
-                return;
-        }
-
-        filled += snprintf (str + filled, size - filled,
-                            " Loc={");
-
-        if (loc->path)
-                filled += snprintf (str + filled, size - filled,
-                                    "path=%s,", loc->path);
-        if (loc->inode)
-                filled += snprintf (str + filled, size - filled,
-                                    "inode=%p,", loc->inode);
-        if (loc->parent)
-                filled += snprintf (str + filled, size - filled,
-                                    "parent=%p", loc->parent);
-
-        filled += snprintf (str + filled, size - filled, "}");
-}
-
-
-static void
-server_print_params (char *str, int size, server_state_t *state)
-{
-        int filled = 0;
-
-        filled += snprintf (str + filled, size - filled,
-                            " Params={");
-
-        if (state->fd)
-                filled += snprintf (str + filled, size - filled,
-                                    "fd=%p,", state->fd);
-        if (state->valid)
-                filled += snprintf (str + filled, size - filled,
-                                    "valid=%d,", state->valid);
-        if (state->flags)
-                filled += snprintf (str + filled, size - filled,
-                                    "flags=%d,", state->flags);
-        if (state->wbflags)
-                filled += snprintf (str + filled, size - filled,
-                                    "wbflags=%d,", state->wbflags);
-        if (state->size)
-                filled += snprintf (str + filled, size - filled,
-                                    "size=%zu,", state->size);
-        if (state->offset)
-                filled += snprintf (str + filled, size - filled,
-                                    "offset=%"PRId64",", state->offset);
-        if (state->cmd)
-                filled += snprintf (str + filled, size - filled,
-                                    "cmd=%d,", state->cmd);
-        if (state->type)
-                filled += snprintf (str + filled, size - filled,
-                                    "type=%d,", state->type);
-        if (state->name)
-                filled += snprintf (str + filled, size - filled,
-                                    "name=%s,", state->name);
-        if (state->mask)
-                filled += snprintf (str + filled, size - filled,
-                                    "mask=%d,", state->mask);
-        if (state->volume)
-                filled += snprintf (str + filled, size - filled,
-                                    "volume=%s,", state->volume);
-
-        filled += snprintf (str + filled, size - filled,
-                            "bound_xl=%s}", state->bound_xl->name);
-}
-
-
-static int
-server_resolve_is_empty (server_resolve_t *resolve)
-{
-        if (resolve->fd_no != -1)
-                return 0;
-
-        if (resolve->ino != 0)
-                return 0;
-
-        if (resolve->gen != 0)
-                return 0;
-
-        if (resolve->par != 0)
-                return 0;
-
-        if (resolve->path != 0)
-                return 0;
-
-        if (resolve->bname != 0)
-                return 0;
-
-        return 1;
-}
-
-void
-gf_server_print_request (call_frame_t *frame)
-{
-        server_conf_t   *conf = NULL;
-        xlator_t        *this = NULL;
-        server_state_t  *state = NULL;
-        char             resolve_vars[256];
-        char             resolve2_vars[256];
-        char             loc_vars[256];
-        char             loc2_vars[256];
-        char             other_vars[512];
-        char             caller[512];
-        char            *op = "UNKNOWN";
-
-        this = frame->this;
-        conf = this->private;
-
-        state = CALL_STATE (frame);
-
-        if (!conf->trace)
-                return;
-
-        memset (resolve_vars, '\0', 256);
-        memset (resolve2_vars, '\0', 256);
-        memset (loc_vars, '\0', 256);
-        memset (loc2_vars, '\0', 256);
-        memset (other_vars, '\0', 256);
-
-        print_caller (caller, 256, frame);
-
-        if (!server_resolve_is_empty (&state->resolve)) {
-                server_print_resolve (resolve_vars, 256, &state->resolve);
-                server_print_loc (loc_vars, 256, &state->loc);
-        }
-
-        if (!server_resolve_is_empty (&state->resolve2)) {
-                server_print_resolve (resolve2_vars, 256, &state->resolve2);
-                server_print_loc (loc2_vars, 256, &state->loc2);
-        }
-
-        server_print_params (other_vars, 512, state);
-
-        switch (frame->root->type) {
-        case GF_OP_TYPE_FOP_REQUEST:
-        case GF_OP_TYPE_FOP_REPLY:
-                op = gf_fop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_MOP_REQUEST:
-        case GF_OP_TYPE_MOP_REPLY:
-                op = gf_mop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_CBK_REQUEST:
-        case GF_OP_TYPE_CBK_REPLY:
-                op = gf_cbk_list[frame->root->op];
-                break;
-        }
-
-        gf_log (this->name, GF_LOG_INFO,
-                "%s%s%s%s%s%s%s",
-                gf_fop_list[frame->root->op], caller,
-                resolve_vars, loc_vars, resolve2_vars, loc2_vars, other_vars);
-}
-
-
-static void
-server_print_reply (call_frame_t *frame, int op_ret, int op_errno)
-{
-        server_conf_t   *conf = NULL;
-        server_state_t  *state = NULL;
-        xlator_t        *this = NULL;
-        char             caller[512];
-        char             fdstr[32];
-        char            *op = "UNKNOWN";
-
-        this = frame->this;
-        conf = this->private;
-
-        if (!conf->trace)
-                return;
-
-        state = CALL_STATE (frame);
-
-        print_caller (caller, 256, frame);
-
-        switch (frame->root->type) {
-        case GF_OP_TYPE_FOP_REQUEST:
-        case GF_OP_TYPE_FOP_REPLY:
-                op = gf_fop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_MOP_REQUEST:
-        case GF_OP_TYPE_MOP_REPLY:
-                op = gf_mop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_CBK_REQUEST:
-        case GF_OP_TYPE_CBK_REPLY:
-                op = gf_cbk_list[frame->root->op];
-                break;
-        }
-
-        fdstr[0] = '\0';
-        if (state->fd)
-                snprintf (fdstr, 32, " fd=%p", state->fd);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "%s%s => (%d, %d)%s",
-                op, caller, op_ret, op_errno, fdstr);
-}
-
-
-
-static void
-protocol_server_reply (call_frame_t *frame, int type, int op,
-                       gf_hdr_common_t *hdr, size_t hdrlen,
-                       struct iovec *vector, int count,
-                       struct iobref *iobref)
-{
-        server_state_t *state = NULL;
-        xlator_t       *bound_xl = NULL;
-        transport_t    *trans = NULL;
-        int             ret = 0;
-
-        xlator_t       *this = NULL;
-
-        bound_xl = BOUND_XL (frame);
-        state    = CALL_STATE (frame);
-        trans    = state->trans;
-        this     = frame->this;
-
-        hdr->callid = hton64 (frame->root->unique);
-        hdr->type   = hton32 (type);
-        hdr->op     = hton32 (op);
-
-        server_print_reply (frame, ntoh32 (hdr->rsp.op_ret),
-                            gf_error_to_errno (ntoh32 (hdr->rsp.op_errno)));
-
-        ret = transport_submit (trans, (char *)hdr, hdrlen, vector,
-                                count, iobref);
-        if (ret < 0) {
-                gf_log ("protocol/server", GF_LOG_ERROR,
-                        "frame %"PRId64": failed to submit. op= %d, type= %d",
-                        frame->root->unique, op, type);
-        }
-
-        STACK_DESTROY (frame->root);
-
-        if (state)
-                free_old_server_state (state);
-
-}
-
-
-static int
-gf_add_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc, fd_t *fd, pid_t pid)
-{
-        int32_t         ret = -1;
-        struct _locker *new = NULL;
-        uint8_t         dir = 0;
-
-	new = GF_CALLOC (1, sizeof (struct _locker),
-                         gf_server_mt_locker);
-        if (new == NULL) {
-                gf_log ("server", GF_LOG_ERROR,
-                        "failed to allocate memory for \'struct _locker\'");
-                goto out;
-        }
-        INIT_LIST_HEAD (&new->lockers);
-
-        new->volume = gf_strdup (volume);
-
-        if (fd == NULL) {
-                loc_copy (&new->loc, loc);
-                dir = IA_ISDIR (new->loc.inode->ia_type);
-        } else {
-                new->fd = fd_ref (fd);
-                dir = IA_ISDIR (fd->inode->ia_type);
-        }
-
-        new->pid = pid;
-
-        LOCK (&table->lock);
-        {
-                if (dir)
-                        list_add_tail (&new->lockers, &table->dir_lockers);
-                else
-                        list_add_tail (&new->lockers, &table->file_lockers);
-        }
-        UNLOCK (&table->lock);
-out:
-        return ret;
-}
-
-
-static int
-gf_del_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc, fd_t *fd, pid_t pid)
-{
-        struct _locker    *locker = NULL;
-        struct _locker    *tmp = NULL;
-        int32_t            ret = 0;
-        uint8_t            dir = 0;
-        struct list_head  *head = NULL;
-        struct list_head   del;
-
-        INIT_LIST_HEAD (&del);
-
-        if (fd) {
-                dir = IA_ISDIR (fd->inode->ia_type);
-        } else {
-                dir = IA_ISDIR (loc->inode->ia_type);
-        }
-
-        LOCK (&table->lock);
-        {
-                if (dir) {
-                        head = &table->dir_lockers;
-                } else {
-                        head = &table->file_lockers;
-                }
-
-                list_for_each_entry_safe (locker, tmp, head, lockers) {
-                        if (locker->fd && fd &&
-                            (locker->fd == fd) && (locker->pid == pid)
-                            && !strcmp (locker->volume, volume)) {
-                                list_move_tail (&locker->lockers, &del);
-                        } else if (locker->loc.inode &&
-                                   loc &&
-                                   (locker->loc.inode == loc->inode) &&
-                                   (locker->pid == pid)
-                                   && !strcmp (locker->volume, volume)) {
-                                list_move_tail (&locker->lockers, &del);
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-        tmp = NULL;
-        locker = NULL;
-
-        list_for_each_entry_safe (locker, tmp, &del, lockers) {
-                list_del_init (&locker->lockers);
-                if (locker->fd)
-                        fd_unref (locker->fd);
-                else
-                        loc_wipe (&locker->loc);
-
-                GF_FREE (locker->volume);
-		GF_FREE (locker);
-	}
-
-        return ret;
-}
-
-
-/*
- * server_lk_cbk - lk callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @lock:
- *
- * not for external reference
- */
-static int
-server_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_lk_rsp_t     *rsp = NULL;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-        server_state_t      *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_flock_from_flock (&rsp->flock, lock);
-        } else if (op_errno != ENOSYS) {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": LK %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_LK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        server_connection_t  *conn = NULL;
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_inodelk_rsp_t *rsp = NULL;
-        server_state_t       *state = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-
-        conn = SERVER_CONNECTION(frame);
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                if (state->flock.l_type == F_UNLCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid);
-        } else if (op_errno != ENOSYS) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": INODELK %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_INODELK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        server_connection_t   *conn = NULL;
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_finodelk_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        conn = SERVER_CONNECTION(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                if (state->flock.l_type == F_UNLCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       NULL, state->fd,
-                                       frame->root->pid);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       NULL, state->fd,
-                                       frame->root->pid);
-        } else if (op_errno != ENOSYS) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": FINODELK %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FINODELK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_entrylk_cbk -
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @lock:
- *
- * not for external reference
- */
-static int
-server_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        server_connection_t  *conn = NULL;
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_entrylk_rsp_t *rsp = NULL;
-        server_state_t       *state = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-
-        conn = SERVER_CONNECTION(frame);
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                if (state->cmd == ENTRYLK_UNLOCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid);
-        } else if (op_errno != ENOSYS) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": INODELK %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_ENTRYLK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        server_connection_t   *conn = NULL;
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fentrylk_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        conn = SERVER_CONNECTION(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                if (state->cmd == ENTRYLK_UNLOCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       NULL, state->fd, frame->root->pid);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       NULL, state->fd, frame->root->pid);
-        } else if (op_errno != ENOSYS) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": FENTRYLK %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FENTRYLK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_access_cbk - access callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_access_rsp_t *rsp = NULL;
-        server_state_t      *state = NULL;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_ACCESS,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_rmdir_cbk - rmdir callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_rmdir_rsp_t *rsp = NULL;
-        server_state_t     *state = NULL;
-        int32_t             gf_errno = 0;
-        size_t              hdrlen = 0;
-        inode_t            *parent = NULL;
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
-                parent = inode_parent (state->loc.inode, 0, NULL);
-                if (parent)
-                        inode_unref (parent);
-                else
-                        inode_forget (state->loc.inode, 0);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": RMDIR %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_RMDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_mkdir_cbk - mkdir callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *stbuf, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_mkdir_rsp_t *rsp = NULL;
-        server_state_t     *state = NULL;
-        size_t              hdrlen = 0;
-        int32_t             gf_errno = 0;
-        inode_t            *link_inode = NULL;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": MKDIR %s  ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_MKDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_mknod_cbk - mknod callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_mknod_rsp_t *rsp = NULL;
-        server_state_t     *state = NULL;
-        int32_t             gf_errno = 0;
-        size_t              hdrlen = 0;
-        inode_t            *link_inode = NULL;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": MKNOD %s ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_MKNOD,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_fsyncdir_cbk - fsyncdir callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fsyncdir_rsp_t *rsp = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-        server_state_t        *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        if (op_ret < 0) {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": FSYNCDIR %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FSYNCDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-
-/*
- * server_readdir_cbk - getdents callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_readdir_rsp_t *rsp = NULL;
-        size_t                hdrlen = 0;
-        size_t                buf_size = 0;
-        int32_t               gf_errno = 0;
-        server_state_t       *state = NULL;
-
-        if (op_ret > 0)
-                buf_size = gf_dirent_serialize (entries, NULL, 0);
-
-        hdrlen = gf_hdr_len (rsp, buf_size);
-        hdr    = gf_hdr_new (rsp, buf_size);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret > 0) {
-                rsp->size = hton32 (buf_size);
-                gf_dirent_serialize (entries, rsp->buf, buf_size);
-        } else {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": READDIR %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_READDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_releasedir_cbk - releasedir callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_releasedir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t         *hdr = NULL;
-        gf_cbk_releasedir_rsp_t *rsp = NULL;
-        size_t                   hdrlen = 0;
-        int32_t                  gf_errno = 0;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_CBK_REPLY, GF_CBK_RELEASEDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_opendir_cbk - opendir callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- * @fd: file descriptor structure of opened directory
- *
- * not for external reference
- */
-static int
-server_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        server_connection_t   *conn = NULL;
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_opendir_rsp_t  *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-        uint64_t               fd_no = 0;
-
-        conn = SERVER_CONNECTION (frame);
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                fd_bind (fd);
-
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd); // on behalf of the client
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": OPENDIR %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        rsp->fd           = hton64 (fd_no);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_OPENDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_statfs_cbk - statfs callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- * @buf:
- *
- * not for external reference
- */
-static int
-server_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct statvfs *buf)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_statfs_rsp_t *rsp = NULL;
-        server_state_t      *state = NULL;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                gf_statfs_from_statfs (&rsp->statfs, buf);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_STATFS,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_removexattr_cbk - removexattr callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t          *hdr = NULL;
-        gf_fop_removexattr_rsp_t *rsp = NULL;
-        server_state_t           *state = NULL;
-        size_t                    hdrlen = 0;
-        int32_t                   gf_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_REMOVEXATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_getxattr_cbk - getxattr callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- * @value:
- *
- * not for external reference
- */
-static int
-server_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_getxattr_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                len = 0;
-        int32_t                gf_errno = 0;
-        int32_t                ret = -1;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                len = dict_serialized_length (dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to get serialized length of "
-                                "reply dict",
-                                state->loc.path, state->resolve.ino);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        len = 0;
-                }
-        }
-
-        hdrlen = gf_hdr_len (rsp, len + 1);
-        hdr    = gf_hdr_new (rsp, len + 1);
-        rsp    = gf_param (hdr);
-
-        if (op_ret >= 0) {
-                ret = dict_serialize (dict, rsp->dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to serialize reply dict",
-                                state->loc.path, state->resolve.ino);
-                        op_ret = -1;
-                        op_errno = -ret;
-                }
-        }
-        rsp->dict_len = hton32 (len);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_GETXATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gf_hdr_common_t        *hdr  = NULL;
-        gf_fop_fgetxattr_rsp_t *rsp = NULL;
-        server_state_t         *state = NULL;
-        size_t                  hdrlen = 0;
-        int32_t                 len = 0;
-        int32_t                 gf_errno = 0;
-        int32_t                 ret = -1;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                len = dict_serialized_length (dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to get serialized length of "
-                                "reply dict",
-                                state->loc.path, state->resolve.ino);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        len = 0;
-                }
-        }
-
-        hdrlen = gf_hdr_len (rsp, len + 1);
-        hdr    = gf_hdr_new (rsp, len + 1);
-        rsp    = gf_param (hdr);
-
-        if (op_ret >= 0) {
-                ret = dict_serialize (dict, rsp->dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to serialize reply dict",
-                                state->loc.path, state->resolve.ino);
-                        op_ret = -1;
-                        op_errno = -ret;
-                }
-        }
-        rsp->dict_len = hton32 (len);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FGETXATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_setxattr_cbk - setxattr callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_setxattr_rsp_t  *rsp = NULL;
-        server_state_t         *state = NULL;
-        size_t                  hdrlen = 0;
-        int32_t                 gf_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_SETXATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_fsetxattr_rsp_t *rsp = NULL;
-        server_state_t         *state = NULL;
-        size_t                  hdrlen = 0;
-        int32_t                 gf_errno = 0;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FSETXATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_rename_cbk - rename callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                   struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_rename_rsp_t *rsp = NULL;
-        server_state_t      *state = NULL;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                stbuf->ia_ino  = state->loc.inode->ino;
-                stbuf->ia_type = state->loc.inode->ia_type;
-
-                gf_log (state->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": RENAME_CBK (%"PRId64") %"PRId64"/%s "
-                        "==> %"PRId64"/%s",
-                        frame->root->unique, state->loc.inode->ino,
-                        state->loc.parent->ino, state->loc.name,
-                        state->loc2.parent->ino, state->loc2.name);
-
-                inode_rename (state->itable,
-                              state->loc.parent, state->loc.name,
-                              state->loc2.parent, state->loc2.name,
-                              state->loc.inode, stbuf);
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-
-                gf_stat_from_iatt (&rsp->preoldparent, preoldparent);
-                gf_stat_from_iatt (&rsp->postoldparent, postoldparent);
-
-                gf_stat_from_iatt (&rsp->prenewparent, prenewparent);
-                gf_stat_from_iatt (&rsp->postnewparent, postnewparent);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_RENAME,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_unlink_cbk - unlink callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_unlink_rsp_t  *rsp = NULL;
-        server_state_t       *state = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-        inode_t              *parent = NULL;
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                gf_log (state->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": UNLINK_CBK %"PRId64"/%s (%"PRId64")",
-                        frame->root->unique, state->loc.parent->ino,
-                        state->loc.name, state->loc.inode->ino);
-
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
-
-                parent = inode_parent (state->loc.inode, 0, NULL);
-                if (parent)
-                        inode_unref (parent);
-                else
-                        inode_forget (state->loc.inode, 0);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": UNLINK %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_UNLINK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_symlink_cbk - symlink callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *stbuf, struct iatt *preparent,
-                    struct iatt *postparent)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_symlink_rsp_t *rsp = NULL;
-        server_state_t       *state = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-        inode_t              *link_inode = NULL;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": SYMLINK %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_SYMLINK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_link_cbk - link callback for server protocol
- * @frame: call frame
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent)
-{
-        gf_hdr_common_t   *hdr = NULL;
-        gf_fop_link_rsp_t *rsp = NULL;
-        server_state_t    *state = NULL;
-        int32_t            gf_errno = 0;
-        size_t             hdrlen = 0;
-        inode_t           *link_inode = NULL;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                stbuf->ia_ino = state->loc.inode->ino;
-
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-
-                gf_log (state->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": LINK (%"PRId64") %"PRId64"/%s ==> %"PRId64"/%s",
-                        frame->root->unique, inode->ino,
-                        state->loc2.parent->ino,
-                        state->loc2.name, state->loc.parent->ino,
-                        state->loc.name);
-
-                link_inode = inode_link (inode, state->loc2.parent,
-                                         state->loc2.name, stbuf);
-                inode_unref (link_inode);
-        } else {
-                gf_log (state->bound_xl->name, GF_LOG_DEBUG,
-                        "%"PRId64": LINK (%"PRId64") %"PRId64"/%s ==> %"PRId64"/%s "
-                        " ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve2.ino,
-                        state->resolve2.par,
-                        state->resolve2.bname, state->resolve.par,
-                        state->resolve.bname,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_LINK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_truncate_cbk - truncate callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_truncate_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->prestat, prebuf);
-                gf_stat_from_iatt (&rsp->poststat, postbuf);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": TRUNCATE %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_TRUNCATE,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_fstat_cbk - fstat callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_fstat_rsp_t *rsp = NULL;
-        size_t              hdrlen = 0;
-        int32_t             gf_errno = 0;
-        server_state_t     *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-        } else {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FSTAT %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FSTAT,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_ftruncate_cbk - ftruncate callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_ftruncate_rsp_t *rsp = NULL;
-        size_t                  hdrlen = 0;
-        int32_t                 gf_errno = 0;
-        server_state_t         *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->prestat, prebuf);
-                gf_stat_from_iatt (&rsp->poststat, postbuf);
-        } else {
-                state = CALL_STATE (frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FTRUNCATE %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FTRUNCATE,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_flush_cbk - flush callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_flush_rsp_t *rsp = NULL;
-        size_t              hdrlen = 0;
-        int32_t             gf_errno = 0;
-        server_state_t     *state = NULL;
-
-        if (op_ret < 0) {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FLUSH %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FLUSH,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_fsync_cbk - fsync callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_fsync_rsp_t *rsp = NULL;
-        size_t              hdrlen = 0;
-        int32_t             gf_errno = 0;
-        server_state_t     *state = NULL;
-
-        if (op_ret < 0) {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FSYNC %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&(rsp->prestat), prebuf);
-                gf_stat_from_iatt (&(rsp->poststat), postbuf);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FSYNC,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_release_cbk - rleease callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_release_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_cbk_release_rsp_t *rsp = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_CBK_REPLY, GF_CBK_RELEASE,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_writev_cbk - writev callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-
-static int
-server_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_write_rsp_t *rsp = NULL;
-        size_t              hdrlen = 0;
-        int32_t             gf_errno = 0;
-        server_state_t     *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->prestat, prebuf);
-                gf_stat_from_iatt (&rsp->poststat, postbuf);
-        } else {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": WRITEV %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_WRITE,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_readv_cbk - readv callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @vector:
- * @count:
- *
- * not for external reference
- */
-static int
-server_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count,
-                  struct iatt *stbuf, struct iobref *iobref)
-{
-        gf_hdr_common_t   *hdr = NULL;
-        gf_fop_read_rsp_t *rsp = NULL;
-        size_t             hdrlen = 0;
-        int32_t            gf_errno = 0;
-        server_state_t    *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-        } else {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": READV %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_READ,
-                               hdr, hdrlen, vector, count, iobref);
-
-        return 0;
-}
-
-
-/*
- * server_open_cbk - open callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @fd:
- *
- * not for external reference
- */
-static int
-server_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        server_connection_t  *conn = NULL;
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_open_rsp_t    *rsp = NULL;
-        server_state_t       *state = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-        uint64_t              fd_no = 0;
-
-        conn = SERVER_CONNECTION (frame);
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                fd_bind (fd);
-
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": OPEN %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        rsp->fd           = hton64 (fd_no);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_OPEN,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_create_cbk - create callback for server
- * @frame: call frame
- * @cookie:
- * @this:  translator structure
- * @op_ret:
- * @op_errno:
- * @fd: file descriptor
- * @inode: inode structure
- * @stbuf: struct iatt of created file
- *
- * not for external reference
- */
-static int
-server_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                   struct iatt *preparent, struct iatt *postparent)
-{
-        server_connection_t *conn = NULL;
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_create_rsp_t *rsp = NULL;
-        server_state_t      *state = NULL;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-        uint64_t             fd_no = 0;
-        inode_t             *link_inode = NULL;
-
-        conn = SERVER_CONNECTION (frame);
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                gf_log (state->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": CREATE %"PRId64"/%s (%"PRId64")",
-                        frame->root->unique, state->loc.parent->ino,
-                        state->loc.name, stbuf->ia_ino);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-
-                if (link_inode != inode) {
-
-                        /*
-                           VERY racy code (if used anywhere else)
-                           -- don't do this without understanding
-                        */
-
-                        inode_unref (fd->inode);
-                        fd->inode = inode_ref (link_inode);
-                }
-
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-
-                fd_bind (fd);
-
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd);
-
-                if ((fd_no < 0) || (fd == 0)) {
-                        op_ret = fd_no;
-                        op_errno = errno;
-                }
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": CREATE %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        rsp->fd           = hton64 (fd_no);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_CREATE,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_readlink_cbk - readlink callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @buf:
- *
- * not for external reference
- */
-static int
-server_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, const char *buf,
-                     struct iatt *sbuf)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_readlink_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        size_t                 linklen = 0;
-        int32_t                gf_errno = 0;
-
-        state  = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                linklen = strlen (buf) + 1;
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": READLINK %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, linklen);
-        hdr    = gf_hdr_new (rsp, linklen);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&(rsp->buf), sbuf);
-                strcpy (rsp->path, buf);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_READLINK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_stat_cbk - stat callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
-{
-        gf_hdr_common_t   *hdr = NULL;
-        gf_fop_stat_rsp_t *rsp = NULL;
-        server_state_t    *state = NULL;
-        size_t             hdrlen = 0;
-        int32_t            gf_errno = 0;
-
-        state  = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": STAT %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_STAT,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_setattr_cbk - setattr callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-
-static int
-server_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *statpre, struct iatt *statpost)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_setattr_rsp_t  *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        state  = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->statpre, statpre);
-                gf_stat_from_iatt (&rsp->statpost, statpost);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": SETATTR %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_SETATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_setattr_cbk - setattr callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fsetattr_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        state  = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->statpre, statpre);
-                gf_stat_from_iatt (&rsp->statpost, statpost);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FSETATTR %"PRId64" (%"PRId64") ==> "
-                        "%"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FSETATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_lookup_cbk - lookup callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @inode:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *stbuf, dict_t *dict,
-                   struct iatt *postparent)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_lookup_rsp_t *rsp = NULL;
-        server_state_t      *state = NULL;
-        inode_t             *root_inode = NULL;
-        int32_t              dict_len = 0;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-        int32_t              ret = -1;
-        inode_t             *link_inode = NULL;
-        loc_t                fresh_loc = {0,};
-
-        state = CALL_STATE(frame);
-
-        if (state->is_revalidate == 1 && op_ret == -1) {
-                state->is_revalidate = 2;
-                loc_copy (&fresh_loc, &state->loc);
-                inode_unref (fresh_loc.inode);
-                fresh_loc.inode = inode_new (state->itable);
-
-                STACK_WIND (frame, server_lookup_cbk,
-                            BOUND_XL (frame), BOUND_XL (frame)->fops->lookup,
-                            &fresh_loc, state->dict);
-
-                loc_wipe (&fresh_loc);
-                return 0;
-        }
-
-        if (dict) {
-                dict_len = dict_serialized_length (dict);
-                if (dict_len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to get serialized "
-                                "length of reply dict",
-                                state->loc.path, state->loc.inode->ino);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        dict_len = 0;
-                }
-        }
-
-        hdrlen = gf_hdr_len (rsp, dict_len);
-        hdr    = gf_hdr_new (rsp, dict_len);
-        rsp    = gf_param (hdr);
-
-        if ((op_ret >= 0) && dict) {
-                ret = dict_serialize (dict, rsp->dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to serialize reply dict",
-                                state->loc.path, state->loc.inode->ino);
-                        op_ret = -1;
-                        op_errno = -ret;
-                        dict_len = 0;
-                }
-        }
-        rsp->dict_len = hton32 (dict_len);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (postparent)
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-
-        if (op_ret == 0) {
-                root_inode = BOUND_XL(frame)->itable->root;
-                if (inode == root_inode) {
-                        /* we just looked up root ("/") */
-                        stbuf->ia_ino = 1;
-                        if (inode->ia_type == 0)
-                                inode->ia_type = stbuf->ia_type;
-                }
-
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-
-                if (inode->ino != 1) {
-                        link_inode = inode_link (inode, state->loc.parent,
-                                                 state->loc.name, stbuf);
-                        inode_lookup (link_inode);
-                        inode_unref (link_inode);
-                }
-        } else {
-                if (state->is_revalidate && op_errno == ENOENT) {
-                        if (state->loc.inode->ino != 1) {
-                                inode_unlink (state->loc.inode,
-                                              state->loc.parent,
-                                              state->loc.name);
-                        }
-                }
-
-                gf_log (this->name,
-                        (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_DEBUG),
-                        "%"PRId64": LOOKUP %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_LOOKUP,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_xattrop_rsp_t  *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                len = 0;
-        int32_t                gf_errno = 0;
-        int32_t                ret = -1;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": XATTROP %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        if ((op_ret >= 0) && dict) {
-                len = dict_serialized_length (dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to get serialized length"
-                                " for reply dict",
-                                state->loc.path, state->loc.inode->ino);
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                        len = 0;
-                }
-        }
-
-        hdrlen = gf_hdr_len (rsp, len + 1);
-        hdr    = gf_hdr_new (rsp, len + 1);
-        rsp    = gf_param (hdr);
-
-        if ((op_ret >= 0) && dict) {
-                ret = dict_serialize (dict, rsp->dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to serialize reply dict",
-                                state->loc.path, state->loc.inode->ino);
-                        op_ret = -1;
-                        op_errno = -ret;
-                        len = 0;
-                }
-        }
-        rsp->dict_len = hton32 (len);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_XATTROP,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_xattrop_rsp_t *rsp = NULL;
-        size_t                hdrlen = 0;
-        int32_t               len = 0;
-        int32_t               gf_errno = 0;
-        int32_t               ret = -1;
-        server_state_t       *state = NULL;
-
-        state = CALL_STATE(frame);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FXATTROP %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        if ((op_ret >= 0) && dict) {
-                len = dict_serialized_length (dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "fd - %"PRId64" (%"PRId64"): failed to get "
-                                "serialized length for reply dict",
-                                state->resolve.fd_no, state->fd->inode->ino);
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                        len = 0;
-                }
-        }
-
-        hdrlen = gf_hdr_len (rsp, len + 1);
-        hdr    = gf_hdr_new (rsp, len + 1);
-        rsp    = gf_param (hdr);
-
-        if ((op_ret >= 0) && dict) {
-                ret = dict_serialize (dict, rsp->dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "fd - %"PRId64" (%"PRId64"): failed to "
-                                "serialize reply dict",
-                                state->resolve.fd_no, state->fd->inode->ino);
-                        op_ret = -1;
-                        op_errno = -ret;
-                        len = 0;
-                }
-        }
-        rsp->dict_len = hton32 (len);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FXATTROP,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_lookup_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        if (!state->loc.inode)
-                state->loc.inode = inode_new (state->itable);
-        else
-                state->is_revalidate = 1;
-
-        STACK_WIND (frame, server_lookup_cbk,
-                    bound_xl, bound_xl->fops->lookup,
-                    &state->loc, state->dict);
-
-        return 0;
-err:
-        server_lookup_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL, NULL, NULL);
-
-        return 0;
-}
-
-
-static int
-server_lookup (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_lookup_req_t *req = NULL;
-        server_state_t      *state = NULL;
-        int32_t              ret = -1;
-        size_t               pathlen = 0;
-        size_t               baselen = 0;
-        size_t               dictlen = 0;
-        dict_t              *xattr_req = NULL;
-        char                *req_dictbuf = NULL;
-
-        req = gf_param (hdr);
-
-        state = CALL_STATE (frame);
-
-        pathlen = STRLEN_0 (req->path);
-        dictlen = ntoh32 (req->dictlen);
-
-        /* NOTE: lookup() uses req->ino only to identify if a lookup()
-         *       is requested for 'root' or not
-         */
-        state->resolve.ino    = ntoh64 (req->ino);
-        if (state->resolve.ino != 1)
-                state->resolve.ino = 0;
-
-        state->resolve.type   = RESOLVE_DONTCARE;
-        state->resolve.par    = ntoh64 (req->par);
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->resolve.path   = gf_strdup (req->path);
-
-        if (IS_NOT_ROOT (pathlen)) {
-                state->resolve.bname = gf_strdup (req->bname + pathlen);
-                baselen = STRLEN_0 (state->resolve.bname);
-        }
-
-        if (dictlen) {
-                /* Unserialize the dictionary */
-                req_dictbuf = memdup (req->dict + pathlen + baselen, dictlen);
-
-                xattr_req = dict_new ();
-
-                ret = dict_unserialize (req_dictbuf, dictlen, &xattr_req);
-                if (ret < 0) {
-                        gf_log (bound_xl->name, GF_LOG_ERROR,
-                                "%"PRId64": %s (%"PRId64"): failed to "
-                                "unserialize req-buffer to dictionary",
-                                frame->root->unique, state->resolve.path,
-                                state->resolve.ino);
-                        GF_FREE (req_dictbuf);
-                        goto err;
-                }
-
-                xattr_req->extra_free = req_dictbuf;
-                state->dict = xattr_req;
-        }
-
-        gf_resolve_and_resume (frame, server_lookup_resume);
-
-        return 0;
-err:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        server_lookup_cbk (frame, NULL, frame->this, -1, EINVAL, NULL, NULL,
-                           NULL, NULL);
-        return 0;
-}
-
-
-/*
- * server_forget - forget function for server protocol
- *
- * not for external reference
- */
-static int
-server_forget (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_log ("forget", GF_LOG_CRITICAL, "function not implemented");
-        return 0;
-}
-
-
-static int
-server_stat_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_stat_cbk,
-                    bound_xl, bound_xl->fops->stat, &state->loc);
-        return 0;
-err:
-        server_stat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                         state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_stat (call_frame_t *frame, xlator_t *bound_xl,
-             gf_hdr_common_t *hdr, size_t hdrlen,
-             struct iobuf *iobuf)
-{
-        gf_fop_stat_req_t *req = NULL;
-        server_state_t    *state = NULL;
-
-        req = gf_param (hdr);
-        state = CALL_STATE (frame);
-        {
-                state->resolve.type  = RESOLVE_MUST;
-                state->resolve.ino   = ntoh64 (req->ino);
-                state->resolve.gen   = ntoh64 (req->gen);
-                state->resolve.path  = gf_strdup (req->path);
-        }
-
-        gf_resolve_and_resume (frame, server_stat_resume);
-
-        return 0;
-}
-
-
-static int
-server_setattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setattr_cbk,
-                    bound_xl, bound_xl->fops->setattr,
-                    &state->loc, &state->stbuf, state->valid);
-        return 0;
-err:
-        server_setattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-static int
-server_setattr (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_setattr_req_t *req = NULL;
-        server_state_t       *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.ino   = ntoh64 (req->ino);
-        state->resolve.gen   = ntoh64 (req->gen);
-        state->resolve.path  = gf_strdup (req->path);
-
-        gf_stat_to_iatt (&req->stbuf, &state->stbuf);
-        state->valid = ntoh32 (req->valid);
-
-        gf_resolve_and_resume (frame, server_setattr_resume);
-
-        return 0;
-}
-
-
-static int
-server_fsetattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsetattr_cbk,
-                    bound_xl, bound_xl->fops->fsetattr,
-                    state->fd, &state->stbuf, state->valid);
-        return 0;
-err:
-        server_fsetattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fsetattr (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_fsetattr_req_t  *req = NULL;
-        server_state_t         *state = NULL;
-
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = ntoh64 (req->fd);
-
-        gf_stat_to_iatt (&req->stbuf, &state->stbuf);
-        state->valid = ntoh32 (req->valid);
-
-        gf_resolve_and_resume (frame, server_fsetattr_resume);
-
-        return 0;
-}
-
-
-static int
-server_readlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readlink_cbk,
-                    bound_xl, bound_xl->fops->readlink,
-                    &state->loc, state->size);
-        return 0;
-err:
-        server_readlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_readlink (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_readlink_req_t *req = NULL;
-        server_state_t        *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.ino  = ntoh64 (req->ino);
-        state->resolve.gen  = ntoh64 (req->gen);
-        state->resolve.path = gf_strdup (req->path);
-
-        state->size  = ntoh32 (req->size);
-
-        gf_resolve_and_resume (frame, server_readlink_resume);
-
-        return 0;
-}
-
-
-static int
-server_create_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-        state->fd->flags = state->flags;
-
-        STACK_WIND (frame, server_create_cbk,
-                    bound_xl, bound_xl->fops->create,
-                    &(state->loc), state->flags, state->mode,
-                    state->fd, state->params);
-
-        return 0;
-err:
-        server_create_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL, NULL,
-                           NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_create (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_create_req_t *req = NULL;
-        server_state_t      *state = NULL;
-        int                  pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type   = RESOLVE_NOT;
-        state->resolve.par    = ntoh64 (req->par);
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.bname  = gf_strdup (req->bname + pathlen);
-        state->mode           = ntoh32 (req->mode);
-        state->flags          = gf_flags_to_flags (ntoh32 (req->flags));
-
-        gf_resolve_and_resume (frame, server_create_resume);
-
-        return 0;
-}
-
-
-static int
-server_open_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-        state->fd->flags = state->flags;
-
-        STACK_WIND (frame, server_open_cbk,
-                    bound_xl, bound_xl->fops->open,
-                    &state->loc, state->flags, state->fd, 0);
-
-        return 0;
-err:
-        server_open_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                         state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_open (call_frame_t *frame, xlator_t *bound_xl,
-             gf_hdr_common_t *hdr, size_t hdrlen,
-             struct iobuf *iobuf)
-{
-        gf_fop_open_req_t  *req = NULL;
-        server_state_t     *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.ino   = ntoh64 (req->ino);
-        state->resolve.gen   = ntoh64 (req->gen);
-        state->resolve.path  = gf_strdup (req->path);
-
-        state->flags = gf_flags_to_flags (ntoh32 (req->flags));
-
-        gf_resolve_and_resume (frame, server_open_resume);
-
-        return 0;
-}
-
-
-static int
-server_readv_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readv_cbk,
-                    bound_xl, bound_xl->fops->readv,
-                    state->fd, state->size, state->offset);
-
-        return 0;
-err:
-        server_readv_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, 0, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_readv (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_read_req_t   *req = NULL;
-        server_state_t      *state = NULL;
-
-        req = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = ntoh64 (req->fd);
-        state->size           = ntoh32 (req->size);
-        state->offset         = ntoh64 (req->offset);
-
-        gf_resolve_and_resume (frame, server_readv_resume);
-
-        return 0;
-}
-
-
-static int
-server_writev_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t   *state = NULL;
-        struct iovec      iov = {0, };
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        iov.iov_len  = state->size;
-
-        if (state->iobuf) {
-                iov.iov_base = state->iobuf->ptr;
-        }
-
-        STACK_WIND (frame, server_writev_cbk,
-                    bound_xl, bound_xl->fops->writev,
-                    state->fd, &iov, 1, state->offset, state->iobref);
-
-        return 0;
-err:
-        server_writev_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_writev (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_write_req_t  *req = NULL;
-        server_state_t      *state = NULL;
-        struct iobref       *iobref = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->offset        = ntoh64 (req->offset);
-        state->size          = ntoh32 (req->size);
-
-        if (iobuf) {
-                iobref = iobref_new ();
-                iobref_add (iobref, iobuf);
-
-                state->iobuf = iobuf;
-                state->iobref = iobref;
-        }
-
-        gf_resolve_and_resume (frame, server_writev_resume);
-
-        return 0;
-}
-
-
-static int
-server_release (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_cbk_release_req_t  *req = NULL;
-        server_state_t        *state = NULL;
-        server_connection_t   *conn = NULL;
-
-        conn  = SERVER_CONNECTION (frame);
-        state = CALL_STATE (frame);
-        req   = gf_param (hdr);
-
-        state->resolve.fd_no = ntoh64 (req->fd);
-
-        gf_fd_put (conn->fdtable, state->resolve.fd_no);
-
-        server_release_cbk (frame, NULL, frame->this, 0, 0);
-
-        return 0;
-}
-
-
-static int
-server_fsync_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsync_cbk,
-                    bound_xl, bound_xl->fops->fsync,
-                    state->fd, state->flags);
-        return 0;
-err:
-        server_fsync_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fsync (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_fsync_req_t  *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->flags         = ntoh32 (req->data);
-
-        gf_resolve_and_resume (frame, server_fsync_resume);
-
-        return 0;
-}
-
-
-
-static int
-server_flush_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_flush_cbk,
-                    bound_xl, bound_xl->fops->flush, state->fd);
-        return 0;
-err:
-        server_flush_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno);
-
-        return 0;
-}
-
-
-static int
-server_flush (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_fsync_req_t  *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-
-        gf_resolve_and_resume (frame, server_flush_resume);
-
-        return 0;
-}
-
-
-
-static int
-server_ftruncate_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_ftruncate_cbk,
-                    bound_xl, bound_xl->fops->ftruncate,
-                    state->fd, state->offset);
-        return 0;
-err:
-        server_ftruncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-static int
-server_ftruncate (call_frame_t *frame, xlator_t *bound_xl,
-                  gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_ftruncate_req_t  *req = NULL;
-        server_state_t          *state = NULL;
-
-        req = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = ntoh64 (req->fd);
-        state->offset         = ntoh64 (req->offset);
-
-        gf_resolve_and_resume (frame, server_ftruncate_resume);
-
-        return 0;
-}
-
-
-static int
-server_fstat_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t     *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fstat_cbk,
-                    bound_xl, bound_xl->fops->fstat,
-                    state->fd);
-        return 0;
-err:
-        server_fstat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_fstat (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_fstat_req_t  *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.fd_no   = ntoh64 (req->fd);
-
-        gf_resolve_and_resume (frame, server_fstat_resume);
-
-        return 0;
-}
-
-
-static int
-server_truncate_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_truncate_cbk,
-                    bound_xl, bound_xl->fops->truncate,
-                    &state->loc, state->offset);
-        return 0;
-err:
-        server_truncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-
-static int
-server_truncate (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_truncate_req_t *req = NULL;
-        server_state_t        *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.path  = gf_strdup (req->path);
-        state->resolve.ino   = ntoh64 (req->ino);
-        state->resolve.gen   = ntoh64 (req->gen);
-        state->offset        = ntoh64 (req->offset);
-
-        gf_resolve_and_resume (frame, server_truncate_resume);
-
-        return 0;
-}
-
-
-static int
-server_unlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_unlink_cbk,
-                    bound_xl, bound_xl->fops->unlink,
-                    &state->loc);
-        return 0;
-err:
-        server_unlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_unlink (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_unlink_req_t *req = NULL;
-        server_state_t      *state = NULL;
-        int                  pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.par    = ntoh64 (req->par);
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.bname  = gf_strdup (req->bname + pathlen);
-
-        gf_resolve_and_resume (frame, server_unlink_resume);
-
-        return 0;
-}
-
-
-static int
-server_setxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setxattr_cbk,
-                    bound_xl, bound_xl->fops->setxattr,
-                    &state->loc, state->dict, state->flags);
-        return 0;
-err:
-        server_setxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-
-        return 0;
-}
-
-
-static int
-server_setxattr (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_setxattr_req_t *req = NULL;
-        server_state_t        *state = NULL;
-        dict_t                *dict = NULL;
-        int32_t                ret = -1;
-        size_t                 dict_len = 0;
-        char                  *req_dictbuf = NULL;
-
-        req = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        dict_len = ntoh32 (req->dict_len);
-
-        state->resolve.type     = RESOLVE_MUST;
-        state->resolve.path     = gf_strdup (req->path + dict_len);
-        state->resolve.ino      = ntoh64 (req->ino);
-        state->resolve.gen      = ntoh64 (req->gen);
-        state->flags            = ntoh32 (req->flags);
-
-        if (dict_len) {
-                req_dictbuf = memdup (req->dict, dict_len);
-
-                dict = dict_new ();
-
-                ret = dict_unserialize (req_dictbuf, dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (bound_xl->name, GF_LOG_ERROR,
-                                "%"PRId64": %s (%"PRId64"): failed to "
-                                "unserialize request buffer to dictionary",
-                                frame->root->unique, state->loc.path,
-                                state->resolve.ino);
-                        GF_FREE (req_dictbuf);
-                        goto err;
-                }
-
-                dict->extra_free = req_dictbuf;
-                state->dict = dict;
-        }
-
-        gf_resolve_and_resume (frame, server_setxattr_resume);
-
-        return 0;
-err:
-        if (dict)
-                dict_unref (dict);
-
-        server_setxattr_cbk (frame, NULL, frame->this, -1, EINVAL);
-
-        return 0;
-
-}
-
-
-static int
-server_fsetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setxattr_cbk,
-                    bound_xl, bound_xl->fops->fsetxattr,
-                    state->fd, state->dict, state->flags);
-        return 0;
-err:
-        server_fsetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno);
-
-        return 0;
-}
-
-
-static int
-server_fsetxattr (call_frame_t *frame, xlator_t *bound_xl,
-                  gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_fsetxattr_req_t *req = NULL;
-        server_state_t         *state = NULL;
-        dict_t                 *dict = NULL;
-        int32_t                 ret = -1;
-        size_t                  dict_len = 0;
-        char                   *req_dictbuf = NULL;
-
-        req = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        dict_len = ntoh32 (req->dict_len);
-
-        state->resolve.type      = RESOLVE_MUST;
-        state->resolve.fd_no     = ntoh64 (req->fd);
-        state->flags             = ntoh32 (req->flags);
-
-        if (dict_len) {
-                req_dictbuf = memdup (req->dict, dict_len);
-
-                dict = dict_new ();
-
-                ret = dict_unserialize (req_dictbuf, dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (bound_xl->name, GF_LOG_ERROR,
-                                "%"PRId64": %s (%"PRId64"): failed to "
-                                "unserialize request buffer to dictionary",
-                                frame->root->unique, state->loc.path,
-                                state->resolve.ino);
-                        GF_FREE (req_dictbuf);
-                        goto err;
-                }
-
-                dict->extra_free = req_dictbuf;
-                state->dict = dict;
-        }
-
-        gf_resolve_and_resume (frame, server_fsetxattr_resume);
-
-        return 0;
-err:
-        if (dict)
-                dict_unref (dict);
-
-        server_setxattr_cbk (frame, NULL, frame->this, -1, EINVAL);
-
-        return 0;
-}
-
-
-static int
-server_fxattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fxattrop_cbk,
-                    bound_xl, bound_xl->fops->fxattrop,
-                    state->fd, state->flags, state->dict);
-        return 0;
-err:
-        server_fxattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_fxattrop (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_fxattrop_req_t *req = NULL;
-        dict_t                *dict = NULL;
-        server_state_t        *state = NULL;
-        size_t                 dict_len = 0;
-        char                  *req_dictbuf = NULL;
-        int32_t                ret = -1;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        dict_len = ntoh32 (req->dict_len);
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.fd_no   = ntoh64 (req->fd);
-
-        state->resolve.ino     = ntoh64 (req->ino);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->flags           = ntoh32 (req->flags);
-
-        if (dict_len) {
-                /* Unserialize the dictionary */
-                req_dictbuf = memdup (req->dict, dict_len);
-
-                dict = dict_new ();
-
-                ret = dict_unserialize (req_dictbuf, dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (bound_xl->name, GF_LOG_ERROR,
-                                "fd - %"PRId64" (%"PRId64"): failed to unserialize "
-                                "request buffer to dictionary",
-                                state->resolve.fd_no, state->fd->inode->ino);
-                        GF_FREE (req_dictbuf);
-                        goto fail;
-                }
-                dict->extra_free = req_dictbuf;
-                state->dict = dict;
-                dict = NULL;
-        }
-
-        gf_resolve_and_resume (frame, server_fxattrop_resume);
-
-        return 0;
-
-fail:
-        if (dict)
-                dict_unref (dict);
-
-        server_fxattrop_cbk (frame, NULL, frame->this, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-static int
-server_xattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_xattrop_cbk,
-                    bound_xl, bound_xl->fops->xattrop,
-                    &state->loc, state->flags, state->dict);
-        return 0;
-err:
-        server_xattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_xattrop (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_xattrop_req_t  *req = NULL;
-        dict_t                *dict = NULL;
-        server_state_t        *state = NULL;
-        size_t                 dict_len = 0;
-        char                  *req_dictbuf = NULL;
-        int32_t                ret = -1;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        dict_len = ntoh32 (req->dict_len);
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.path    = gf_strdup (req->path + dict_len);
-        state->resolve.ino     = ntoh64 (req->ino);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->flags           = ntoh32 (req->flags);
-
-        if (dict_len) {
-                /* Unserialize the dictionary */
-                req_dictbuf = memdup (req->dict, dict_len);
-
-                dict = dict_new ();
-
-                ret = dict_unserialize (req_dictbuf, dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (bound_xl->name, GF_LOG_ERROR,
-                                "fd - %"PRId64" (%"PRId64"): failed to unserialize "
-                                "request buffer to dictionary",
-                                state->resolve.fd_no, state->fd->inode->ino);
-                        GF_FREE (req_dictbuf);
-                        goto fail;
-                }
-                dict->extra_free = req_dictbuf;
-                state->dict = dict;
-                dict = NULL;
-        }
-
-        gf_resolve_and_resume (frame, server_xattrop_resume);
-
-        return 0;
-
-fail:
-        if (dict)
-                dict_unref (dict);
-
-        server_xattrop_cbk (frame, NULL, frame->this, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-static int
-server_getxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_getxattr_cbk,
-                    bound_xl, bound_xl->fops->getxattr,
-                    &state->loc, state->name);
-        return 0;
-err:
-        server_getxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_getxattr (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_getxattr_req_t  *req = NULL;
-        server_state_t         *state = NULL;
-        size_t                  namelen = 0;
-        size_t                  pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.path  = gf_strdup (req->path);
-        state->resolve.ino   = ntoh64 (req->ino);
-        state->resolve.gen   = ntoh64 (req->gen);
-
-        namelen = ntoh32 (req->namelen);
-        if (namelen)
-                state->name = gf_strdup (req->name + pathlen);
-
-        gf_resolve_and_resume (frame, server_getxattr_resume);
-
-        return 0;
-}
-
-
-static int
-server_fgetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fgetxattr_cbk,
-                    bound_xl, bound_xl->fops->fgetxattr,
-                    state->fd, state->name);
-        return 0;
-err:
-        server_fgetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_fgetxattr (call_frame_t *frame, xlator_t *bound_xl,
-                  gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_fgetxattr_req_t *req = NULL;
-        server_state_t         *state = NULL;
-        size_t                  namelen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-
-        namelen = ntoh32 (req->namelen);
-        if (namelen)
-                state->name = gf_strdup (req->name);
-
-        gf_resolve_and_resume (frame, server_fgetxattr_resume);
-
-        return 0;
-}
-
-
-static int
-server_removexattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_removexattr_cbk,
-                    bound_xl, bound_xl->fops->removexattr,
-                    &state->loc, state->name);
-        return 0;
-err:
-        server_removexattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                                state->resolve.op_errno);
-        return 0;
-}
-
-
-static int
-server_removexattr (call_frame_t *frame, xlator_t *bound_xl,
-                    gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_removexattr_req_t  *req = NULL;
-        server_state_t            *state = NULL;
-        size_t                     pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.ino    = ntoh64 (req->ino);
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->name           = gf_strdup (req->name + pathlen);
-
-        gf_resolve_and_resume (frame, server_removexattr_resume);
-
-        return 0;
-}
-
-
-static int
-server_statfs_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t      *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret !=0)
-                goto err;
-
-        STACK_WIND (frame, server_statfs_cbk,
-                    bound_xl, bound_xl->fops->statfs,
-                    &state->loc);
-        return 0;
-
-err:
-        server_statfs_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_statfs (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_statfs_req_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req = gf_param (hdr);
-
-        state = CALL_STATE (frame);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.ino    = ntoh64 (req->ino);
-        if (!state->resolve.ino)
-                state->resolve.ino = 1;
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->resolve.path   = gf_strdup (req->path);
-
-        gf_resolve_and_resume (frame, server_statfs_resume);
-
-        return 0;
-}
-
-
-static int
-server_opendir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-
-        STACK_WIND (frame, server_opendir_cbk,
-                    bound_xl, bound_xl->fops->opendir,
-                    &state->loc, state->fd);
-        return 0;
-err:
-        server_opendir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_opendir (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_opendir_req_t  *req = NULL;
-        server_state_t        *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.ino    = ntoh64 (req->ino);
-        state->resolve.gen    = ntoh64 (req->gen);
-
-        gf_resolve_and_resume (frame, server_opendir_resume);
-
-        return 0;
-}
-
-
-static int
-server_releasedir (call_frame_t *frame, xlator_t *bound_xl,
-                   gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        gf_cbk_releasedir_req_t *req = NULL;
-        server_connection_t     *conn = NULL;
-        uint64_t                 fd_no = 0;
-
-        conn = SERVER_CONNECTION (frame);
-
-        req = gf_param (hdr);
-
-        fd_no = ntoh64 (req->fd);
-
-        gf_fd_put (conn->fdtable, fd_no);
-
-        server_releasedir_cbk (frame, NULL, frame->this, 0, 0);
-
-        return 0;
-}
-
-/*
- * server_readdirp_cbk - getdents callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        gf_hdr_common_t         *hdr = NULL;
-        gf_fop_readdirp_rsp_t   *rsp = NULL;
-        size_t                  hdrlen = 0;
-        size_t                  buf_size = 0;
-        int32_t                 gf_errno = 0;
-        server_state_t          *state = NULL;
-
-        if (op_ret > 0)
-                buf_size = gf_dirent_serialize (entries, NULL, 0);
-
-        hdrlen = gf_hdr_len (rsp, buf_size);
-        hdr    = gf_hdr_new (rsp, buf_size);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret > 0) {
-                rsp->size = hton32 (buf_size);
-                gf_dirent_serialize (entries, rsp->buf, buf_size);
-        } else {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": READDIRP %"PRId64" (%"PRId64") ==>"
-                        "%"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_READDIRP,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-static int
-server_readdirp_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readdirp_cbk, bound_xl,
-                    bound_xl->fops->readdirp, state->fd, state->size,
-                    state->offset);
-
-        return 0;
-err:
-        server_readdirp_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-/*
- * server_readdirp - readdirp function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- * not for external reference
- */
-static int
-server_readdirp (call_frame_t *frame, xlator_t *bound_xl, gf_hdr_common_t *hdr,
-                 size_t hdrlen, struct iobuf *iobuf)
-{
-        gf_fop_readdirp_req_t *req = NULL;
-        server_state_t *state = NULL;
-        server_connection_t *conn = NULL;
-
-        conn = SERVER_CONNECTION(frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->size   = ntoh32 (req->size);
-        state->offset = ntoh64 (req->offset);
-
-        gf_resolve_and_resume (frame, server_readdirp_resume);
-
-        return 0;
-}
-
-
-static int
- server_readdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readdir_cbk,
-                    bound_xl,
-                    bound_xl->fops->readdir,
-                    state->fd, state->size, state->offset);
-
-        return 0;
-err:
-        server_readdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-/*
- * server_readdir - readdir function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- * not for external reference
- */
-static int
-server_readdir (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_readdir_req_t *req = NULL;
-        server_state_t *state = NULL;
-        server_connection_t *conn = NULL;
-
-        conn = SERVER_CONNECTION(frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->size   = ntoh32 (req->size);
-        state->offset = ntoh64 (req->offset);
-
-        gf_resolve_and_resume (frame, server_readdir_resume);
-
-        return 0;
-}
-
-static int
-server_fsyncdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsyncdir_cbk,
-                    bound_xl,
-                    bound_xl->fops->fsyncdir,
-                    state->fd, state->flags);
-        return 0;
-
-err:
-        server_fsyncdir_cbk (frame, NULL, frame->this,
-                             state->resolve.op_ret,
-                             state->resolve.op_errno);
-        return 0;
-}
-
-/*
- * server_fsyncdir - fsyncdir function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- * not for external reference
- */
-static int
-server_fsyncdir (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_fsyncdir_req_t *req = NULL;
-        server_state_t        *state = NULL;
-        server_connection_t   *conn = NULL;
-
-        conn = SERVER_CONNECTION (frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->flags = ntoh32 (req->data);
-
-        gf_resolve_and_resume (frame, server_fsyncdir_resume);
-
-        return 0;
-}
-
-
-static int
-server_mknod_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_mknod_cbk,
-                    bound_xl, bound_xl->fops->mknod,
-                    &(state->loc), state->mode, state->dev,
-                    state->params);
-
-        return 0;
-err:
-        server_mknod_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-
-static int
-server_mknod (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_mknod_req_t *req = NULL;
-        server_state_t     *state = NULL;
-        size_t              pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type    = RESOLVE_NOT;
-        state->resolve.par     = ntoh64 (req->par);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->resolve.path    = gf_strdup (req->path);
-        state->resolve.bname   = gf_strdup (req->bname + pathlen);
-
-        state->mode = ntoh32 (req->mode);
-        state->dev  = ntoh64 (req->dev);
-
-        gf_resolve_and_resume (frame, server_mknod_resume);
-
-        return 0;
-}
-
-
-static int
-server_mkdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_mkdir_cbk,
-                    bound_xl, bound_xl->fops->mkdir,
-                    &(state->loc), state->mode,
-                    state->params);
-
-        return 0;
-err:
-        server_mkdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_mkdir (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_mkdir_req_t *req = NULL;
-        server_state_t     *state = NULL;
-        size_t              pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type    = RESOLVE_NOT;
-        state->resolve.par     = ntoh64 (req->par);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->resolve.path    = gf_strdup (req->path);
-        state->resolve.bname   = gf_strdup (req->bname + pathlen);
-
-        state->mode = ntoh32 (req->mode);
-
-        gf_resolve_and_resume (frame, server_mkdir_resume);
-
-        return 0;
-}
-
-
-static int
-server_rmdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_rmdir_cbk,
-                    bound_xl, bound_xl->fops->rmdir, &state->loc);
-        return 0;
-err:
-        server_rmdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-static int
-server_rmdir (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_rmdir_req_t *req = NULL;
-        server_state_t     *state = NULL;
-        int                 pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.par     = ntoh64 (req->par);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->resolve.path    = gf_strdup (req->path);
-        state->resolve.bname   = gf_strdup (req->bname + pathlen);
-
-        gf_resolve_and_resume (frame, server_rmdir_resume);
-
-        return 0;
-}
-
-
-static int
-server_inodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_inodelk_cbk,
-                    bound_xl, bound_xl->fops->inodelk,
-                    state->volume, &state->loc, state->cmd, &state->flock);
-        return 0;
-err:
-        server_inodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno);
-        return 0;
-}
-
-
-static int
-server_inodelk (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_inodelk_req_t *req = NULL;
-        server_state_t       *state = NULL;
-        size_t                pathlen = 0;
-        size_t                vollen  = 0;
-        int                   cmd = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-        vollen  = STRLEN_0 (req->volume + pathlen);
-
-        state->resolve.type    = RESOLVE_EXACT;
-        state->resolve.ino     = ntoh64 (req->ino);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->resolve.path    = gf_strdup (req->path);
-
-        cmd = ntoh32 (req->cmd);
-        switch (cmd) {
-        case GF_LK_GETLK:
-                state->cmd = F_GETLK;
-                break;
-        case GF_LK_SETLK:
-                state->cmd = F_SETLK;
-                break;
-        case GF_LK_SETLKW:
-                state->cmd = F_SETLKW;
-                break;
-        }
-
-        state->type = ntoh32 (req->type);
-        state->volume = gf_strdup (req->volume + pathlen);
-
-        gf_flock_to_flock (&req->flock, &state->flock);
-
-        switch (state->type) {
-        case GF_LK_F_RDLCK:
-                state->flock.l_type = F_RDLCK;
-                break;
-        case GF_LK_F_WRLCK:
-                state->flock.l_type = F_WRLCK;
-                break;
-        case GF_LK_F_UNLCK:
-                state->flock.l_type = F_UNLCK;
-                break;
-        }
-
-        gf_resolve_and_resume (frame, server_inodelk_resume);
-
-        return 0;
-}
-
-static int
-server_finodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_finodelk_cbk,
-                    BOUND_XL(frame),
-                    BOUND_XL(frame)->fops->finodelk,
-                    state->volume, state->fd, state->cmd, &state->flock);
-
-        return 0;
-err:
-        server_finodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-
-        return 0;
-}
-
-static int
-server_finodelk (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_finodelk_req_t  *req = NULL;
-        server_state_t         *state = NULL;
-        server_connection_t    *conn = NULL;
-
-        conn = SERVER_CONNECTION(frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        state->resolve.type = RESOLVE_EXACT;
-        state->volume = gf_strdup (req->volume);
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->cmd = ntoh32 (req->cmd);
-
-        switch (state->cmd) {
-        case GF_LK_GETLK:
-                state->cmd = F_GETLK;
-                break;
-        case GF_LK_SETLK:
-                state->cmd = F_SETLK;
-                break;
-        case GF_LK_SETLKW:
-                state->cmd = F_SETLKW;
-                break;
-        }
-
-        state->type = ntoh32 (req->type);
-
-        gf_flock_to_flock (&req->flock, &state->flock);
-
-        switch (state->type) {
-        case GF_LK_F_RDLCK:
-                state->flock.l_type = F_RDLCK;
-                break;
-        case GF_LK_F_WRLCK:
-                state->flock.l_type = F_WRLCK;
-                break;
-        case GF_LK_F_UNLCK:
-                state->flock.l_type = F_UNLCK;
-                break;
-        }
-
-        gf_resolve_and_resume (frame, server_finodelk_resume);
-
-        return 0;
-}
-
-
-static int
-server_entrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_entrylk_cbk,
-                    bound_xl, bound_xl->fops->entrylk,
-                    state->volume, &state->loc, state->name,
-                    state->cmd, state->type);
-        return 0;
-err:
-        server_entrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno);
-        return 0;
-}
-
-
-static int
-server_entrylk (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_entrylk_req_t *req = NULL;
-        server_state_t       *state = NULL;
-        size_t                pathlen = 0;
-        size_t                namelen = 0;
-        size_t                vollen  = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-        namelen = ntoh64 (req->namelen);
-        vollen = STRLEN_0(req->volume + pathlen + namelen);
-
-        state->resolve.type   = RESOLVE_EXACT;
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.ino    = ntoh64 (req->ino);
-        state->resolve.gen    = ntoh64 (req->gen);
-
-        if (namelen)
-                state->name   = gf_strdup (req->name + pathlen);
-        state->volume         = gf_strdup (req->volume + pathlen + namelen);
-
-        state->cmd            = ntoh32 (req->cmd);
-        state->type           = ntoh32 (req->type);
-
-        gf_resolve_and_resume (frame, server_entrylk_resume);
-
-        return 0;
-}
-
-static int
-server_fentrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fentrylk_cbk,
-                    BOUND_XL(frame),
-                    BOUND_XL(frame)->fops->fentrylk,
-                    state->volume, state->fd, state->name,
-                    state->cmd, state->type);
-
-        return 0;
-err:
-        server_fentrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-        return 0;
-}
-
-static int
-server_fentrylk (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_fentrylk_req_t *req = NULL;
-        server_state_t        *state = NULL;
-        size_t                 namelen = 0;
-        size_t                 vollen  = 0;
-        server_connection_t   *conn = NULL;
-
-        conn = SERVER_CONNECTION (frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-        vollen = STRLEN_0(req->volume + namelen);
-
-        state->resolve.type = RESOLVE_EXACT;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->cmd  = ntoh32 (req->cmd);
-        state->type = ntoh32 (req->type);
-        namelen = ntoh64 (req->namelen);
-        if (namelen)
-                state->name = req->name;
-        state->volume = gf_strdup (req->volume + namelen);
-
-
-        gf_resolve_and_resume (frame, server_fentrylk_resume);
-
-        return 0;
-}
-
-
-static int
-server_access_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_access_cbk,
-                    bound_xl, bound_xl->fops->access,
-                    &state->loc, state->mask);
-        return 0;
-err:
-        server_access_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno);
-        return 0;
-}
-
-
-static int
-server_access (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_access_req_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.ino   = hton64 (req->ino);
-        state->resolve.gen   = hton64 (req->gen);
-        state->resolve.path  = gf_strdup (req->path);
-
-        state->mask  = ntoh32 (req->mask);
-
-        gf_resolve_and_resume (frame, server_access_resume);
-
-        return 0;
-}
-
-
-static int
-server_symlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_symlink_cbk,
-                    bound_xl, bound_xl->fops->symlink,
-                    state->name, &state->loc, state->params);
-
-        return 0;
-err:
-        server_symlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-
-static int
-server_symlink (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        server_state_t       *state = NULL;
-        gf_fop_symlink_req_t *req = NULL;
-        size_t                pathlen = 0;
-        size_t                baselen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-        baselen = STRLEN_0 (req->bname + pathlen);
-
-        state->resolve.type   = RESOLVE_NOT;
-        state->resolve.par    = ntoh64 (req->par);
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.bname  = gf_strdup (req->bname + pathlen);
-        state->name           = gf_strdup (req->linkname + pathlen + baselen);
-
-        gf_resolve_and_resume (frame, server_symlink_resume);
-
-        return 0;
-}
-
-
-static int
-server_link_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-        int             op_ret = 0;
-        int             op_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        if (state->resolve2.op_ret != 0) {
-                op_ret   = state->resolve2.op_ret;
-                op_errno = state->resolve2.op_errno;
-                goto err;
-        }
-
-        state->loc2.inode = inode_ref (state->loc.inode);
-
-        STACK_WIND (frame, server_link_cbk,
-                    bound_xl, bound_xl->fops->link,
-                    &state->loc, &state->loc2);
-        return 0;
-err:
-        server_link_cbk (frame, NULL, frame->this, op_ret, op_errno,
-                         NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_link (call_frame_t *frame, xlator_t *this,
-             gf_hdr_common_t *hdr, size_t hdrlen,
-             struct iobuf *iobuf)
-{
-        gf_fop_link_req_t *req = NULL;
-        server_state_t    *state = NULL;
-        size_t             oldpathlen = 0;
-        size_t             newpathlen = 0;
-        size_t             newbaselen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        oldpathlen = STRLEN_0 (req->oldpath);
-        newpathlen = STRLEN_0 (req->newpath  + oldpathlen);
-        newbaselen = STRLEN_0 (req->newbname + oldpathlen + newpathlen);
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.path    = gf_strdup (req->oldpath);
-        state->resolve.ino     = ntoh64 (req->oldino);
-        state->resolve.gen     = ntoh64 (req->oldgen);
-
-        state->resolve2.type   = RESOLVE_NOT;
-        state->resolve2.path   = gf_strdup (req->newpath + oldpathlen);
-        state->resolve2.bname  = gf_strdup (req->newbname + oldpathlen + newpathlen);
-        state->resolve2.par    = ntoh64 (req->newpar);
-        state->resolve2.gen    = ntoh64 (req->newgen);
-
-        gf_resolve_and_resume (frame, server_link_resume);
-
-        return 0;
-}
-
-
-static int
-server_rename_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-        int             op_ret = 0;
-        int             op_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        if (state->resolve2.op_ret != 0) {
-                op_ret   = state->resolve2.op_ret;
-                op_errno = state->resolve2.op_errno;
-                goto err;
-        }
-
-        STACK_WIND (frame, server_rename_cbk,
-                    bound_xl, bound_xl->fops->rename,
-                    &state->loc, &state->loc2);
-        return 0;
-err:
-        server_rename_cbk (frame, NULL, frame->this, op_ret, op_errno,
-                           NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_rename (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_rename_req_t *req = NULL;
-        server_state_t      *state = NULL;
-        size_t               oldpathlen = 0;
-        size_t               oldbaselen = 0;
-        size_t               newpathlen = 0;
-        size_t               newbaselen = 0;
-
-        req = gf_param (hdr);
-
-        state = CALL_STATE (frame);
-        oldpathlen = STRLEN_0 (req->oldpath);
-        oldbaselen = STRLEN_0 (req->oldbname + oldpathlen);
-        newpathlen = STRLEN_0 (req->newpath  + oldpathlen + oldbaselen);
-        newbaselen = STRLEN_0 (req->newbname + oldpathlen +
-                               oldbaselen + newpathlen);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.path   = gf_strdup (req->oldpath);
-        state->resolve.bname  = gf_strdup (req->oldbname + oldpathlen);
-        state->resolve.par    = ntoh64 (req->oldpar);
-        state->resolve.gen    = ntoh64 (req->oldgen);
-
-        state->resolve2.type  = RESOLVE_MAY;
-        state->resolve2.path  = gf_strdup (req->newpath  + oldpathlen + oldbaselen);
-        state->resolve2.bname = gf_strdup (req->newbname + oldpathlen + oldbaselen +
-                                        newpathlen);
-        state->resolve2.par   = ntoh64 (req->newpar);
-        state->resolve2.gen   = ntoh64 (req->newgen);
-
-        gf_resolve_and_resume (frame, server_rename_resume);
-
-        return 0;
-}
-
-static int
-server_lk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_lk_cbk,
-                    BOUND_XL(frame),
-                    BOUND_XL(frame)->fops->lk,
-                    state->fd, state->cmd, &state->flock);
-
-        return 0;
-
-err:
-        server_lk_cbk (frame, NULL, frame->this,
-                       state->resolve.op_ret,
-                       state->resolve.op_errno,
-                       NULL);
-        return 0;
-}
-
-/*
- * server_lk - lk function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- * not for external reference
- */
-
-static int
-server_lk (call_frame_t *frame, xlator_t *bound_xl,
-           gf_hdr_common_t *hdr, size_t hdrlen,
-           struct iobuf *iobuf)
-{
-        gf_fop_lk_req_t     *req = NULL;
-        server_state_t      *state = NULL;
-        server_connection_t *conn = NULL;
-
-        conn = SERVER_CONNECTION (frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->cmd =  ntoh32 (req->cmd);
-        state->type = ntoh32 (req->type);
-
-        switch (state->cmd) {
-        case GF_LK_GETLK:
-                state->cmd = F_GETLK;
-                break;
-        case GF_LK_SETLK:
-                state->cmd = F_SETLK;
-                break;
-        case GF_LK_SETLKW:
-                state->cmd = F_SETLKW;
-                break;
-        }
-
-        gf_flock_to_flock (&req->flock, &state->flock);
-
-        switch (state->type) {
-        case GF_LK_F_RDLCK:
-                state->flock.l_type = F_RDLCK;
-                break;
-        case GF_LK_F_WRLCK:
-                state->flock.l_type = F_WRLCK;
-                break;
-        case GF_LK_F_UNLCK:
-                state->flock.l_type = F_UNLCK;
-                break;
-        default:
-                gf_log (bound_xl->name, GF_LOG_ERROR,
-                        "fd - %"PRId64" (%"PRId64"): Unknown lock type: %"PRId32"!",
-                        state->resolve.fd_no, state->fd->inode->ino, state->type);
-                break;
-        }
-
-
-        gf_resolve_and_resume (frame, server_lk_resume);
-
-        return 0;
-}
-
-/* xxx_MOPS */
-static int
-_volfile_update_checksum (xlator_t *this, char *key, uint32_t checksum)
-{
-        server_conf_t       *conf         = NULL;
-        struct _volfile_ctx *temp_volfile = NULL;
-
-        conf         = this->private;
-        temp_volfile = conf->volfile;
-
-        while (temp_volfile) {
-                if ((NULL == key) && (NULL == temp_volfile->key))
-                        break;
-                if ((NULL == key) || (NULL == temp_volfile->key)) {
-                        temp_volfile = temp_volfile->next;
-                        continue;
-                }
-                if (strcmp (temp_volfile->key, key) == 0)
-                        break;
-                temp_volfile = temp_volfile->next;
-        }
-
-        if (!temp_volfile) {
-                temp_volfile = GF_CALLOC (1, sizeof (struct _volfile_ctx),
-                                          gf_server_mt_volfile_ctx);
-                if (!temp_volfile)
-                        goto out;
-                temp_volfile->next  = conf->volfile;
-                temp_volfile->key   = (key)? gf_strdup (key): NULL;
-                temp_volfile->checksum = checksum;
-
-                conf->volfile = temp_volfile;
-                goto out;
-        }
-
-        if (temp_volfile->checksum != checksum) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "the volume file got modified between earlier access "
-                        "and now, this may lead to inconsistency between "
-                        "clients, advised to remount client");
-                temp_volfile->checksum  = checksum;
-        }
-
- out:
-        return 0;
-}
-
-
-size_t
-build_volfile_path (xlator_t *this, const char *key, char *path,
-                    size_t path_len)
-{
-        int             ret = -1;
-        int             free_filename = 0;
-        int             free_conf_dir = 0;
-        char            *filename = NULL;
-        char            *conf_dir = CONFDIR;
-        struct stat      buf      = {0,};
-        data_t *         conf_dir_data = NULL;
-        char             data_key[256] = {0,};
-
-        /* Inform users that this option is changed now */
-        ret = dict_get_str (this->options, "client-volume-filename",
-                            &filename);
-        if (ret == 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "option 'client-volume-filename' is changed to "
-                        "'volume-filename.<key>' which now takes 'key' as an "
-                        "option to choose/fetch different files from server. "
-                        "Refer documentation or contact developers for more "
-                        "info. Currently defaulting to given file '%s'",
-                        filename);
-        }
-
-        if (key && !filename) {
-                sprintf (data_key, "volume-filename.%s", key);
-                ret = dict_get_str (this->options, data_key, &filename);
-
-                if (ret < 0) {
-
-                        conf_dir_data = dict_get (this->options, "conf-dir");
-                        if (conf_dir_data) {
-                                /* Check whether the specified directory exists,
-                                   or directory specified is non standard */
-                                ret = stat (conf_dir_data->data, &buf);
-                                if ((ret != 0) || !S_ISDIR (buf.st_mode)) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Directory '%s' doesn't"
-                                                "exist, exiting.",
-                                                conf_dir_data->data);
-                                        ret = -1;
-                                        goto out;
-                                }
-                                /* Make sure that conf-dir doesn't
-                                 * contain ".." in path
-                                 */
-                                if ((gf_strstr (conf_dir_data->data,
-                                                "/", "..")) == -1) {
-                                        ret = -1;
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s: invalid conf_dir",
-                                                conf_dir_data->data);
-                                        goto out;
-                                }
-
-                                /* Make sure that key doesn't
-                                 * contain "../" in path
-                                 */
-
-                                if ((gf_strstr (key, "/", "..")) == -1) {
-                                        ret = -1;
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s: invalid key", key);
-                                        goto out;
-                                }
-
-                                conf_dir = gf_strdup (conf_dir_data->data);
-                                free_conf_dir = 1;
-                        }
-
-                        ret = gf_asprintf (&filename, "%s/%s.vol",
-                                           conf_dir, key);
-                        if (-1 == ret)
-                                goto out;
-
-                        free_filename = 1;
-                }
-        }
-
-        if (!filename) {
-                ret = dict_get_str (this->options,
-                                    "volume-filename.default", &filename);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no default volume filename given, "
-                                "defaulting to %s", DEFAULT_VOLUME_FILE_PATH);
-                        filename = DEFAULT_VOLUME_FILE_PATH;
-                }
-        }
-
-        ret = -1;
-
-        if ((filename) && (path_len > strlen (filename))) {
-                strcpy (path, filename);
-                ret = strlen (filename);
-        }
-
-out:
-        if (free_conf_dir)
-                GF_FREE (conf_dir);
-
-        if (free_filename)
-                GF_FREE (filename);
-
-        return ret;
-}
-
-static int
-_validate_volfile_checksum (xlator_t *this, char *key,
-                            uint32_t checksum)
-{
-        char                 filename[ZR_PATH_MAX] = {0,};
-        server_conf_t       *conf         = NULL;
-        struct _volfile_ctx *temp_volfile = NULL;
-        int                  ret          = 0;
-        uint32_t             local_checksum = 0;
-
-        conf         = this->private;
-        temp_volfile = conf->volfile;
-
-        if (!checksum)
-                goto out;
-
-        if (!temp_volfile) {
-                ret = build_volfile_path (this, key, filename,
-                                          sizeof (filename));
-                if (ret <= 0)
-                        goto out;
-                ret = open (filename, O_RDONLY);
-                if (-1 == ret) {
-                        ret = 0;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to open volume file (%s) : %s",
-                                filename, strerror (errno));
-                        goto out;
-                }
-                get_checksum_for_file (ret, &local_checksum);
-                _volfile_update_checksum (this, key, local_checksum);
-                close (ret);
-        }
-
-        temp_volfile = conf->volfile;
-        while (temp_volfile) {
-                if ((NULL == key) && (NULL == temp_volfile->key))
-                        break;
-                if ((NULL == key) || (NULL == temp_volfile->key)) {
-                        temp_volfile = temp_volfile->next;
-                        continue;
-                }
-                if (strcmp (temp_volfile->key, key) == 0)
-                        break;
-                temp_volfile = temp_volfile->next;
-        }
-
-        if (!temp_volfile)
-                goto out;
-
-        if ((temp_volfile->checksum) &&
-            (checksum != temp_volfile->checksum))
-                ret = -1;
-
-out:
-        return ret;
-}
-
-/* Management Calls */
-/*
- * mop_getspec - getspec function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params:
- *
- */
-static int
-mop_getspec (call_frame_t *frame, xlator_t *bound_xl,
-             gf_hdr_common_t *hdr, size_t hdrlen,
-             struct iobuf *iobuf)
-{
-        gf_hdr_common_t      *_hdr = NULL;
-        gf_mop_getspec_rsp_t *rsp = NULL;
-        int32_t               ret = -1;
-        int32_t               op_errno = ENOENT;
-        int32_t               gf_errno = 0;
-        int32_t               spec_fd = -1;
-        size_t                file_len = 0;
-        size_t                _hdrlen = 0;
-        char                  filename[ZR_PATH_MAX] = {0,};
-        struct stat           stbuf = {0,};
-        gf_mop_getspec_req_t *req = NULL;
-        uint32_t              checksum = 0;
-        uint32_t              flags  = 0;
-        uint32_t              keylen = 0;
-        char                 *key = NULL;
-        server_conf_t        *conf = NULL;
-
-        req   = gf_param (hdr);
-        flags = ntoh32 (req->flags);
-        keylen = ntoh32 (req->keylen);
-        if (keylen) {
-                key = req->key;
-        }
-
-        conf = frame->this->private;
-
-        ret = build_volfile_path (frame->this, key, filename,
-                                  sizeof (filename));
-        if (ret > 0) {
-                /* to allocate the proper buffer to hold the file data */
-                ret = stat (filename, &stbuf);
-                if (ret < 0){
-                        gf_log (frame->this->name, GF_LOG_ERROR,
-                                "Unable to stat %s (%s)",
-                                filename, strerror (errno));
-                        goto fail;
-                }
-
-                spec_fd = open (filename, O_RDONLY);
-                if (spec_fd < 0) {
-                        gf_log (frame->this->name, GF_LOG_ERROR,
-                                "Unable to open %s (%s)",
-                                filename, strerror (errno));
-                        goto fail;
-                }
-                ret = 0;
-                file_len = stbuf.st_size;
-                if (conf->verify_volfile_checksum) {
-                        get_checksum_for_file (spec_fd, &checksum);
-                        _volfile_update_checksum (frame->this, key, checksum);
-                }
-        } else {
-                errno = ENOENT;
-        }
-
-fail:
-        op_errno = errno;
-
-        _hdrlen = gf_hdr_len (rsp, file_len + 1);
-        _hdr    = gf_hdr_new (rsp, file_len + 1);
-        rsp     = gf_param (_hdr);
-
-        _hdr->rsp.op_ret = hton32 (ret);
-        gf_errno         = gf_errno_to_error (op_errno);
-        _hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (file_len) {
-                ret = read (spec_fd, rsp->spec, file_len);
-                close (spec_fd);
-        }
-        protocol_server_reply (frame, GF_OP_TYPE_MOP_REPLY, GF_MOP_GETSPEC,
-                               _hdr, _hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-
-static int
-server_checksum (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_CHECKSUM,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_rchecksum_rsp_t *rsp = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        hdrlen = gf_hdr_len (rsp, MD5_DIGEST_LEN + 1);
-        hdr    = gf_hdr_new (rsp, MD5_DIGEST_LEN + 1);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                rsp->weak_checksum = weak_checksum;
-
-                memcpy (rsp->strong_checksum,
-                        strong_checksum, MD5_DIGEST_LEN);
-
-                rsp->strong_checksum[MD5_DIGEST_LEN] = '\0';
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_RCHECKSUM,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-static int
-server_rchecksum_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-        int             op_ret = 0;
-        int             op_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        STACK_WIND (frame, server_rchecksum_cbk,
-                    bound_xl,
-                    bound_xl->fops->rchecksum,
-                    state->fd, state->offset, state->size);
-
-        return 0;
-err:
-        server_rchecksum_cbk (frame, NULL, frame->this, -1, EINVAL, 0, NULL);
-
-        return 0;
-
-}
-
-static int
-server_rchecksum (call_frame_t *frame, xlator_t *bound_xl,
-                  gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_rchecksum_req_t  *req = NULL;
-        server_state_t          *state = NULL;
-        server_connection_t     *conn = NULL;
-
-        conn = SERVER_CONNECTION(frame);
-
-        req = gf_param (hdr);
-
-        state = CALL_STATE(frame);
-
-        state->resolve.type = RESOLVE_MAY;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->offset = ntoh64 (req->offset);
-        state->size   = ntoh32 (req->len);
-
-        gf_resolve_and_resume (frame, server_rchecksum_resume);
-
-        return 0;
-}
-
-
-/*
- * mop_unlock - unlock management function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- */
-static int
-mop_getvolume (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        return 0;
-}
-
-struct __get_xl_struct {
-        const char *name;
-        xlator_t *reply;
-};
-
-static void __check_and_set (xlator_t *each, void *data)
-{
-        if (!strcmp (each->name,
-                     ((struct __get_xl_struct *) data)->name))
-                ((struct __get_xl_struct *) data)->reply = each;
-}
-
-static xlator_t *
-get_xlator_by_name (xlator_t *some_xl, const char *name)
-{
-        struct __get_xl_struct get = {
-                .name = name,
-                .reply = NULL
-        };
-
-        xlator_foreach (some_xl, __check_and_set, &get);
-
-        return get.reply;
-}
-
-
-/*
- * mop_setvolume - setvolume management function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- */
-static int
-mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *req_hdr, size_t req_hdrlen,
-               struct iobuf *iobuf)
-{
-        server_connection_t         *conn = NULL;
-        server_conf_t               *conf = NULL;
-        gf_hdr_common_t             *rsp_hdr = NULL;
-        gf_mop_setvolume_req_t      *req = NULL;
-        gf_mop_setvolume_rsp_t      *rsp = NULL;
-        peer_info_t                 *peerinfo = NULL;
-        int32_t                      ret = -1;
-        int32_t                      op_ret = -1;
-        int32_t                      op_errno = EINVAL;
-        int32_t                      gf_errno = 0;
-        dict_t                      *reply = NULL;
-        dict_t                      *config_params = NULL;
-        dict_t                      *params = NULL;
-        char                        *name = NULL;
-        char                        *version = NULL;
-        char                        *process_uuid = NULL;
-        xlator_t                    *xl = NULL;
-        transport_t                 *trans = NULL;
-        size_t                       rsp_hdrlen = -1;
-        size_t                       dict_len = -1;
-        size_t                       req_dictlen = -1;
-        char                        *msg = NULL;
-        char                        *volfile_key = NULL;
-        uint32_t                     checksum = 0;
-        int32_t                      lru_limit = 1024;
-
-        params = dict_new ();
-        reply  = dict_new ();
-
-        req    = gf_param (req_hdr);
-        req_dictlen = ntoh32 (req->dict_len);
-        ret = dict_unserialize (req->buf, req_dictlen, &params);
-
-        config_params = dict_copy_with_ref (frame->this->options, NULL);
-        trans         = TRANSPORT_FROM_FRAME(frame);
-        conf          = SERVER_CONF(frame);
-
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "Internal error: failed to unserialize "
-                                    "request dictionary");
-                if (ret < 0)
-                        gf_log (bound_xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg \"%s\"",
-                                "Internal error: failed to unserialize "
-                                "request dictionary");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        ret = dict_get_str (params, "process-uuid", &process_uuid);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "UUID not specified");
-                if (ret < 0)
-                        gf_log (bound_xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-
-        conn = gf_server_connection_get (frame->this, process_uuid);
-        if (trans->xl_private != conn)
-                trans->xl_private = conn;
-
-        ret = dict_get_str (params, "protocol-version", &version);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "No version number specified");
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        ret = strcmp (version, GF_PROTOCOL_VERSION);
-        if (ret != 0) {
-                ret = gf_asprintf (&msg, "protocol version mismatch: client(%s) "
-                                "- server(%s)", version, GF_PROTOCOL_VERSION);
-                if (-1 == ret) {
-                        gf_log (trans->xl->name, GF_LOG_ERROR,
-                                "gf_asprintf failed while setting up error msg");
-                        goto fail;
-                }
-                ret = dict_set_dynstr (reply, "ERROR", msg);
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        ret = dict_get_str (params,
-                            "remote-subvolume", &name);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "No remote-subvolume option specified");
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        xl = get_xlator_by_name (frame->this, name);
-        if (xl == NULL) {
-                ret = gf_asprintf (&msg, "remote-subvolume \"%s\" is not found",
-                                name);
-                if (-1 == ret) {
-                        gf_log (trans->xl->name, GF_LOG_ERROR,
-                                "gf_asprintf failed while setting error msg");
-                        goto fail;
-                }
-                ret = dict_set_dynstr (reply, "ERROR", msg);
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = ENOENT;
-                goto fail;
-        }
-
-        if (conf->verify_volfile_checksum) {
-                ret = dict_get_uint32 (params, "volfile-checksum", &checksum);
-                if (ret == 0) {
-                        ret = dict_get_str (params, "volfile-key",
-                                            &volfile_key);
-
-                        ret = _validate_volfile_checksum (trans->xl,
-                                                          volfile_key,
-                                                          checksum);
-                        if (-1 == ret) {
-                                ret = dict_set_str (reply, "ERROR",
-                                                    "volume-file checksum "
-                                                    "varies from earlier "
-                                                    "access");
-                                if (ret < 0)
-                                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                                "failed to set error msg");
-
-                                op_ret   = -1;
-                                op_errno = ESTALE;
-                                goto fail;
-                        }
-                }
-        }
-
-
-        peerinfo = &trans->peerinfo;
-        ret = dict_set_static_ptr (params, "peer-info", peerinfo);
-        if (ret < 0)
-                gf_log (trans->xl->name, GF_LOG_DEBUG,
-                        "failed to set peer-info");
-        ret = dict_set_str (params, "peer-info-name", peerinfo->identifier);
-        if (ret < 0)
-                gf_log (trans->xl->name, GF_LOG_DEBUG,
-                        "failed to set peer-info-name");
-
-        if (conf->auth_modules == NULL) {
-                gf_log (trans->xl->name, GF_LOG_ERROR,
-                        "Authentication module not initialized");
-        }
-
-        ret = gf_authenticate (params, config_params,
-                               conf->auth_modules);
-        if (ret == AUTH_ACCEPT) {
-                gf_log (trans->xl->name, GF_LOG_INFO,
-                        "accepted client from %s",
-                        peerinfo->identifier);
-                op_ret = 0;
-                conn->bound_xl = xl;
-                ret = dict_set_str (reply, "ERROR", "Success");
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-        } else {
-                gf_log (trans->xl->name, GF_LOG_ERROR,
-                        "Cannot authenticate client from %s",
-                        peerinfo->identifier);
-                op_ret = -1;
-                op_errno = EACCES;
-                ret = dict_set_str (reply, "ERROR", "Authentication failed");
-                if (ret < 0)
-                        gf_log (bound_xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                goto fail;
-        }
-
-        if (conn->bound_xl == NULL) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "Check volfile and handshake "
-                                    "options in protocol/client");
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EACCES;
-                goto fail;
-        }
-
-        if ((conn->bound_xl != NULL) &&
-            (ret >= 0)                   &&
-            (conn->bound_xl->itable == NULL)) {
-                /* create inode table for this bound_xl, if one doesn't
-                   already exist */
-                lru_limit = INODE_LRU_LIMIT (frame->this);
-
-                gf_log (trans->xl->name, GF_LOG_TRACE,
-                        "creating inode table with lru_limit=%"PRId32", "
-                        "xlator=%s", lru_limit, conn->bound_xl->name);
-
-                conn->bound_xl->itable =
-                        inode_table_new (lru_limit,
-                                         conn->bound_xl);
-        }
-
-        ret = dict_set_str (reply, "process-uuid",
-                            xl->ctx->process_uuid);
-
-        ret = dict_set_uint64 (reply, "transport-ptr",
-                               ((uint64_t) (long) trans));
-
-fail:
-        dict_len = dict_serialized_length (reply);
-        if (dict_len < 0) {
-                gf_log ("server", GF_LOG_DEBUG,
-                        "failed to get serialized length of reply dict");
-                op_ret   = -1;
-                op_errno = EINVAL;
-                dict_len = 0;
-        }
-
-        rsp_hdr    = gf_hdr_new (rsp, dict_len);
-        rsp_hdrlen = gf_hdr_len (rsp, dict_len);
-        rsp = gf_param (rsp_hdr);
-
-        if (dict_len) {
-                ret = dict_serialize (reply, rsp->buf);
-                if (ret < 0) {
-                        gf_log ("server", GF_LOG_DEBUG,
-                                "failed to serialize reply dict");
-                        op_ret = -1;
-                        op_errno = -ret;
-                }
-        }
-        rsp->dict_len = hton32 (dict_len);
-
-        rsp_hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno = gf_errno_to_error (op_errno);
-        rsp_hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_MOP_REPLY, GF_MOP_SETVOLUME,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        dict_unref (params);
-        dict_unref (reply);
-        dict_unref (config_params);
-
-        return 0;
-}
-
-
-static int
-mop_ping (call_frame_t *frame, xlator_t *bound_xl,
-          gf_hdr_common_t *hdr, size_t hdrlen,
-          struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;
-        size_t               rsp_hdrlen = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        hdr->rsp.op_ret = 0;
-
-        protocol_server_reply (frame, GF_OP_TYPE_MOP_REPLY, GF_MOP_PING,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-mop_log (call_frame_t *frame, xlator_t *bound_xl,
-         gf_hdr_common_t *hdr, size_t hdrlen,
-         struct iobuf *iobuf)
-{
-        gf_mop_log_req_t *    req = NULL;
-        char *                msg = NULL;
-        uint32_t           msglen = 0;
-
-        transport_t *       trans = NULL;
-
-        trans = TRANSPORT_FROM_FRAME (frame);
-
-        req    = gf_param (hdr);
-        msglen = ntoh32 (req->msglen);
-
-        if (msglen)
-                msg = req->msg;
-
-        gf_log_from_client (msg, trans->peerinfo.identifier);
-
-        return 0;
-}
-
-
-/* ENOSYS operations (for backword compatibility) */
-static int
-server_setdents (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_SETDENTS,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/* */
-static int
-server_getdents (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_GETDENTS,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/* */
-static int
-server_lock_notify (call_frame_t *frame, xlator_t *bound_xl,
-                    gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_LOCK_NOTIFY,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/* */
-static int
-server_lock_fnotify (call_frame_t *frame, xlator_t *bound_xl,
-                     gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_LOCK_FNOTIFY,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-mop_stats (call_frame_t *frame, xlator_t *bound_xl,
-           gf_hdr_common_t *hdr, size_t hdrlen,
-           struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_MOP_REPLY, GF_MOP_STATS,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * get_frame_for_transport - get call frame for specified transport object
- *
- * @trans: transport object
- *
- */
-static call_frame_t *
-get_frame_for_transport (transport_t *trans)
-{
-        call_frame_t         *frame = NULL;
-        call_pool_t          *pool = NULL;
-        server_connection_t  *conn = NULL;
-        server_state_t       *state = NULL;;
-
-        GF_VALIDATE_OR_GOTO("server", trans, out);
-
-        if (trans->xl && trans->xl->ctx)
-                pool = trans->xl->ctx->pool;
-        GF_VALIDATE_OR_GOTO("server", pool, out);
-
-        frame = create_frame (trans->xl, pool);
-        GF_VALIDATE_OR_GOTO("server", frame, out);
-
-        state = GF_CALLOC (1, sizeof (*state),
-                           gf_server_mt_server_state_t);
-        GF_VALIDATE_OR_GOTO("server", state, out);
-
-        conn = trans->xl_private;
-        if (conn) {
-                if (conn->bound_xl)
-                        state->itable = conn->bound_xl->itable;
-                state->bound_xl = conn->bound_xl;
-        }
-
-        state->trans = transport_ref (trans);
-        state->resolve.fd_no = -1;
-        state->resolve2.fd_no = -1;
-
-        frame->root->trans = conn;
-        frame->root->state = state;        /* which socket */
-        frame->root->unique = 0;           /* which call */
-
-out:
-        return frame;
-}
-
-
-static int
-server_decode_groups (call_frame_t *frame, gf_hdr_common_t *hdr)
-{
-        int     i = 0;
-
-        if ((!frame) || (!hdr))
-                return 0;
-
-        frame->root->ngrps = ntoh32 (hdr->req.ngrps);
-        if (frame->root->ngrps == 0)
-                return 0;
-
-        if (frame->root->ngrps > GF_REQUEST_MAXGROUPS)
-                return -1;
-
-        for (; i < frame->root->ngrps; ++i)
-                frame->root->groups[i] = ntoh32 (hdr->req.groups[i]);
-
-        return 0;
-}
-
-
-/*
- * get_frame_for_call - create a frame into the capable of
- *                      generating and replying the reply packet by itself.
- *                      By making a call with this frame, the last UNWIND
- *                      function will have all needed state from its
- *                      frame_t->root to send reply.
- * @trans:
- * @blk:
- * @params:
- *
- * not for external reference
- */
-static call_frame_t *
-get_frame_for_call (transport_t *trans, gf_hdr_common_t *hdr)
-{
-        call_frame_t *frame = NULL;
-        int32_t      ret = -1;
-
-        frame = get_frame_for_transport (trans);
-
-        frame->root->op   = ntoh32 (hdr->op);
-        frame->root->type = ntoh32 (hdr->type);
-
-        frame->root->uid         = ntoh32 (hdr->req.uid);
-        frame->root->unique      = ntoh64 (hdr->callid);      /* which call */
-        frame->root->gid         = ntoh32 (hdr->req.gid);
-        frame->root->pid         = ntoh32 (hdr->req.pid);
-        frame->root->lk_owner    = ntoh64 (hdr->req.lk_owner);
-        ret = server_decode_groups (frame, hdr);
-
-        if (ret) {
-                //FRAME_DESTROY (frame);
-                return NULL;
-        }
-
-        return frame;
-}
-
-/*
- * prototype of operations function for each of mop and
- * fop at server protocol level
- *
- * @frame: call frame pointer
- * @bound_xl: the xlator that this frame is bound to
- * @params: parameters dictionary
- *
- * to be used by protocol interpret, _not_ for exterenal reference
- */
-typedef int32_t (*gf_op_t) (call_frame_t *frame, xlator_t *bould_xl,
-                            gf_hdr_common_t *hdr, size_t hdrlen,
-                            struct iobuf *iobuf);
-
-
-static gf_op_t gf_fops[] = {
-        [GF_PROTO_FOP_STAT]         =  server_stat,
-        [GF_PROTO_FOP_READLINK]     =  server_readlink,
-        [GF_PROTO_FOP_MKNOD]        =  server_mknod,
-        [GF_PROTO_FOP_MKDIR]        =  server_mkdir,
-        [GF_PROTO_FOP_UNLINK]       =  server_unlink,
-        [GF_PROTO_FOP_RMDIR]        =  server_rmdir,
-        [GF_PROTO_FOP_SYMLINK]      =  server_symlink,
-        [GF_PROTO_FOP_RENAME]       =  server_rename,
-        [GF_PROTO_FOP_LINK]         =  server_link,
-        [GF_PROTO_FOP_TRUNCATE]     =  server_truncate,
-        [GF_PROTO_FOP_OPEN]         =  server_open,
-        [GF_PROTO_FOP_READ]         =  server_readv,
-        [GF_PROTO_FOP_WRITE]        =  server_writev,
-        [GF_PROTO_FOP_STATFS]       =  server_statfs,
-        [GF_PROTO_FOP_FLUSH]        =  server_flush,
-        [GF_PROTO_FOP_FSYNC]        =  server_fsync,
-        [GF_PROTO_FOP_SETXATTR]     =  server_setxattr,
-        [GF_PROTO_FOP_GETXATTR]     =  server_getxattr,
-        [GF_PROTO_FOP_FGETXATTR]    =  server_fgetxattr,
-        [GF_PROTO_FOP_FSETXATTR]    =  server_fsetxattr,
-        [GF_PROTO_FOP_REMOVEXATTR]  =  server_removexattr,
-        [GF_PROTO_FOP_OPENDIR]      =  server_opendir,
-        [GF_PROTO_FOP_FSYNCDIR]     =  server_fsyncdir,
-        [GF_PROTO_FOP_ACCESS]       =  server_access,
-        [GF_PROTO_FOP_CREATE]       =  server_create,
-        [GF_PROTO_FOP_FTRUNCATE]    =  server_ftruncate,
-        [GF_PROTO_FOP_FSTAT]        =  server_fstat,
-        [GF_PROTO_FOP_LK]           =  server_lk,
-        [GF_PROTO_FOP_LOOKUP]       =  server_lookup,
-        [GF_PROTO_FOP_READDIR]      =  server_readdir,
-        [GF_PROTO_FOP_READDIRP]     =  server_readdirp,
-        [GF_PROTO_FOP_INODELK]      =  server_inodelk,
-        [GF_PROTO_FOP_FINODELK]     =  server_finodelk,
-        [GF_PROTO_FOP_ENTRYLK]      =  server_entrylk,
-        [GF_PROTO_FOP_FENTRYLK]     =  server_fentrylk,
-        [GF_PROTO_FOP_CHECKSUM]     =  server_checksum,
-        [GF_PROTO_FOP_RCHECKSUM]    =  server_rchecksum,
-        [GF_PROTO_FOP_XATTROP]      =  server_xattrop,
-        [GF_PROTO_FOP_FXATTROP]     =  server_fxattrop,
-        [GF_PROTO_FOP_SETATTR]      =  server_setattr,
-        [GF_PROTO_FOP_FSETATTR]     =  server_fsetattr,
-        [GF_PROTO_FOP_SETDENTS]     =  server_setdents,
-        [GF_PROTO_FOP_GETDENTS]     =  server_getdents,
-        [GF_PROTO_FOP_LOCK_NOTIFY]  =  server_lock_notify,
-        [GF_PROTO_FOP_LOCK_FNOTIFY] =  server_lock_fnotify,
-};
-
-
-
-static gf_op_t gf_mops[] = {
-        [GF_MOP_SETVOLUME] = mop_setvolume,
-        [GF_MOP_GETVOLUME] = mop_getvolume,
-        [GF_MOP_GETSPEC]   = mop_getspec,
-        [GF_MOP_PING]      = mop_ping,
-        [GF_MOP_LOG]       = mop_log,
-        [GF_MOP_STATS]     = mop_stats,
-};
-
-static gf_op_t gf_cbks[] = {
-        [GF_CBK_FORGET]     = server_forget,
-        [GF_CBK_RELEASE]    = server_release,
-        [GF_CBK_RELEASEDIR] = server_releasedir
-};
-
-static int
-protocol_server_interpret (xlator_t *this, transport_t *trans,
-                           char *hdr_p, size_t hdrlen, struct iobuf *iobuf)
-{
-        server_connection_t         *conn = NULL;
-        gf_hdr_common_t             *hdr = NULL;
-        xlator_t                    *bound_xl = NULL;
-        call_frame_t                *frame = NULL;
-        peer_info_t                 *peerinfo = NULL;
-        int32_t                      type = -1;
-        int32_t                      op = -1;
-        int32_t                      ret = -1;
-
-        hdr  = (gf_hdr_common_t *)hdr_p;
-        type = ntoh32 (hdr->type);
-        op   = ntoh32 (hdr->op);
-
-        conn = trans->xl_private;
-        if (conn)
-                bound_xl = conn->bound_xl;
-
-        peerinfo = &trans->peerinfo;
-        switch (type) {
-        case GF_OP_TYPE_FOP_REQUEST:
-                if ((op < 0) || (op >= GF_PROTO_FOP_MAXVALUE)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid fop %"PRId32" from client %s",
-                                op, peerinfo->identifier);
-                        break;
-                }
-                if (bound_xl == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Received fop %"PRId32" before "
-                                "authentication.", op);
-                        break;
-                }
-                frame = get_frame_for_call (trans, hdr);
-                if (!frame) {
-                        ret = -1;
-                        goto out;
-                }
-		frame->op = op;
-                ret = gf_fops[op] (frame, bound_xl, hdr, hdrlen, iobuf);
-                break;
-
-        case GF_OP_TYPE_MOP_REQUEST:
-                if ((op < 0) || (op >= GF_MOP_MAXVALUE)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid mop %"PRId32" from client %s",
-                                op, peerinfo->identifier);
-                        break;
-                }
-                frame = get_frame_for_call (trans, hdr);
-                if (!frame) {
-                        ret = -1;
-                        goto out;
-                }
-		frame->op = op;
-                ret = gf_mops[op] (frame, bound_xl, hdr, hdrlen, iobuf);
-                break;
-
-        case GF_OP_TYPE_CBK_REQUEST:
-                if ((op < 0) || (op >= GF_CBK_MAXVALUE)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid cbk %"PRId32" from client %s",
-                                op, peerinfo->identifier);
-                        break;
-                }
-                if (bound_xl == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Received cbk %d before authentication.", op);
-                        break;
-                }
-
-                frame = get_frame_for_call (trans, hdr);
-                if (!frame) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = gf_cbks[op] (frame, bound_xl, hdr, hdrlen, iobuf);
-                break;
-
-        default:
-                break;
-        }
-out:
-        return ret;
-}
-
-
-/*
- * server_fd - fdtable dump function for server protocol
- * @this:
- *
- */
-static int
-server_fd (xlator_t *this)
-{
-         server_conf_t        *conf = NULL;
-         server_connection_t  *trav = NULL;
-         char                 key[GF_DUMP_MAX_BUF_LEN];
-         int                  i = 1;
-         int                  ret = -1;
-
-         if (!this)
-                 return -1;
-
-         conf = this->private;
-         if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
-                return -1;
-         }
-
-         gf_proc_dump_add_section("xlator.protocol.server.conn");
-
-         ret = pthread_mutex_trylock (&conf->mutex);
-         if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump fdtable"
-                " errno: %d", errno);
-                return -1;
-        }
-
-         list_for_each_entry (trav, &conf->conns, list) {
-                 if (trav->id) {
-                         gf_proc_dump_build_key(key,
-                                          "xlator.protocol.server.conn",
-                                          "%d.id", i);
-                         gf_proc_dump_write(key, "%s", trav->id);
-                 }
-
-                 gf_proc_dump_build_key(key,"xlator.protocol.server.conn",
-                                        "%d.ref",i)
-                 gf_proc_dump_write(key, "%d", trav->ref);
-                 if (trav->bound_xl) {
-                         gf_proc_dump_build_key(key,
-                                          "xlator.protocol.server.conn",
-                                          "%d.bound_xl", i);
-                         gf_proc_dump_write(key, "%s", trav->bound_xl->name);
-                 }
-
-                 gf_proc_dump_build_key(key,
-                                        "xlator.protocol.server.conn",
-                                         "%d.id", i);
-                 fdtable_dump(trav->fdtable,key);
-                 i++;
-         }
-        pthread_mutex_unlock (&conf->mutex);
-
-
-        return 0;
- }
-
-static int
-server_priv (xlator_t *this)
-{
-        return 0;
-}
-
-static int
-server_inode (xlator_t *this)
-{
-         server_conf_t        *conf = NULL;
-         server_connection_t  *trav = NULL;
-         char                 key[GF_DUMP_MAX_BUF_LEN];
-         int                  i = 1;
-         int                  ret = -1;
-
-         if (!this)
-                 return -1;
-
-         conf = this->private;
-         if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
-                return -1;
-         }
-
-         ret = pthread_mutex_trylock (&conf->mutex);
-         if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump itable"
-                " errno: %d", errno);
-                return -1;
-        }
-
-        list_for_each_entry (trav, &conf->conns, list) {
-                 if (trav->bound_xl && trav->bound_xl->itable) {
-                         gf_proc_dump_build_key(key,
-                                          "xlator.protocol.server.conn",
-                                          "%d.bound_xl.%s",
-                                          i, trav->bound_xl->name);
-                         inode_table_dump(trav->bound_xl->itable,key);
-                         i++;
-                 }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-
-        return 0;
-}
-
-
-static void
-get_auth_types (dict_t *this, char *key, data_t *value, void *data)
-{
-        dict_t   *auth_dict = NULL;
-        char     *saveptr = NULL;
-        char     *tmp = NULL;
-        char     *key_cpy = NULL;
-        int32_t   ret = -1;
-
-        auth_dict = data;
-        key_cpy = gf_strdup (key);
-        GF_VALIDATE_OR_GOTO("server", key_cpy, out);
-
-        tmp = strtok_r (key_cpy, ".", &saveptr);
-        ret = strcmp (tmp, "auth");
-        if (ret == 0) {
-                tmp = strtok_r (NULL, ".", &saveptr);
-                if (strcmp (tmp, "ip") == 0) {
-                        /* TODO: backward compatibility, remove when
-                           newer versions are available */
-                        tmp = "addr";
-                        gf_log ("server", GF_LOG_WARNING,
-                                "assuming 'auth.ip' to be 'auth.addr'");
-                }
-                ret = dict_set_dynptr (auth_dict, tmp, NULL, 0);
-                if (ret < 0) {
-                        gf_log ("server", GF_LOG_DEBUG,
-                                "failed to dict_set_dynptr");
-                }
-        }
-
-        GF_FREE (key_cpy);
-out:
-        return;
-}
-
-
-static int
-validate_auth_options (xlator_t *this, dict_t *dict)
-{
-        int            ret = -1;
-        int            error = 0;
-        xlator_list_t *trav = NULL;
-        data_pair_t   *pair = NULL;
-        char          *saveptr = NULL;
-        char          *tmp = NULL;
-        char          *key_cpy = NULL;
-
-        trav = this->children;
-        while (trav) {
-                error = -1;
-                for (pair = dict->members_list; pair; pair = pair->next) {
-                        key_cpy = gf_strdup (pair->key);
-                        tmp = strtok_r (key_cpy, ".", &saveptr);
-                        ret = strcmp (tmp, "auth");
-                        if (ret == 0) {
-                                /* for module type */
-                                tmp = strtok_r (NULL, ".", &saveptr);
-                                /* for volume name */
-                                tmp = strtok_r (NULL, ".", &saveptr);
-                        }
-
-                        if (strcmp (tmp, trav->xlator->name) == 0) {
-                                error = 0;
-                                GF_FREE (key_cpy);
-                                break;
-                        }
-                        GF_FREE (key_cpy);
-                }
-                if (-1 == error) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "volume '%s' defined as subvolume, but no "
-                                "authentication defined for the same",
-                                trav->xlator->name);
-                        break;
-                }
-                trav = trav->next;
-        }
-
-        return error;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_server_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-
-/*
- * init - called during server protocol initialization
- *
- * @this:
- *
- */
-int
-init (xlator_t *this)
-{
-        int32_t        ret = -1;
-        transport_t   *trans = NULL;
-        server_conf_t *conf = NULL;
-        data_t        *data = NULL;
-        data_t        *trace = NULL;
-
-        if (this->children == NULL) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "protocol/server should have subvolume");
-                goto out;
-        }
-
-        trans = transport_load (this->options, this);
-        if (trans == NULL) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to load transport");
-                goto out;
-        }
-
-        ret = transport_listen (trans);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to bind/listen on socket");
-                goto out;
-        }
-
-        conf = GF_CALLOC (1, sizeof (server_conf_t),
-                          gf_server_mt_server_conf_t);
-        GF_VALIDATE_OR_GOTO(this->name, conf, out);
-
-        INIT_LIST_HEAD (&conf->conns);
-        pthread_mutex_init (&conf->mutex, NULL);
-
-        conf->trans = trans;
-
-        conf->auth_modules = dict_new ();
-        GF_VALIDATE_OR_GOTO(this->name, conf->auth_modules, out);
-
-        dict_foreach (this->options, get_auth_types,
-                      conf->auth_modules);
-        ret = validate_auth_options (this, this->options);
-        if (ret == -1) {
-                /* logging already done in validate_auth_options function. */
-                goto out;
-        }
-
-        ret = gf_auth_init (this, conf->auth_modules);
-        if (ret) {
-                dict_unref (conf->auth_modules);
-                goto out;
-        }
-
-        this->private = conf;
-
-        ret = dict_get_int32 (this->options, "inode-lru-limit",
-                              &conf->inode_lru_limit);
-        if (ret < 0) {
-                conf->inode_lru_limit = 1024;
-        }
-
-        ret = dict_get_int32 (this->options, "limits.transaction-size",
-                              &conf->max_block_size);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "defaulting limits.transaction-size to %d",
-                        DEFAULT_BLOCK_SIZE);
-                conf->max_block_size = DEFAULT_BLOCK_SIZE;
-        }
-
-        conf->verify_volfile_checksum = 1;
-        data = dict_get (this->options, "verify-volfile-checksum");
-        if (data) {
-                ret = gf_string2boolean(data->data,
-                                        &conf->verify_volfile_checksum);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "wrong value for verify-volfile-checksum");
-                        conf->verify_volfile_checksum = 1;
-                }
-        }
-
-        trace = dict_get (this->options, "trace");
-	if (trace) {
-		if (gf_string2boolean (trace->data,
-				       &conf->trace) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"'trace' takes on only boolean values.");
-			return -1;
-		}
-	}
-
-#ifndef GF_DARWIN_HOST_OS
-        {
-                struct rlimit lim;
-
-                lim.rlim_cur = 1048576;
-                lim.rlim_max = 1048576;
-
-                if (setrlimit (RLIMIT_NOFILE, &lim) == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "WARNING: Failed to set 'ulimit -n 1M': %s",
-                                strerror(errno));
-                        lim.rlim_cur = 65536;
-                        lim.rlim_max = 65536;
-
-                        if (setrlimit (RLIMIT_NOFILE, &lim) == -1) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Failed to set max open fd to 64k: %s",
-                                        strerror(errno));
-                        } else {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "max open fd set to 64k");
-                        }
-                }
-        }
-#endif
-        this->graph->top = this;
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-
-static int
-protocol_server_pollin (xlator_t *this, transport_t *trans)
-{
-        char                *hdr = NULL;
-        size_t               hdrlen = 0;
-        int                  ret = -1;
-        struct iobuf        *iobuf = NULL;
-
-
-        ret = transport_receive (trans, &hdr, &hdrlen, &iobuf);
-
-        if (ret == 0)
-                ret = protocol_server_interpret (this, trans, hdr,
-                                                 hdrlen, iobuf);
-
-        /* TODO: use mem-pool */
-        GF_FREE (hdr);
-
-        return ret;
-}
-
-
-/*
- * fini - finish function for server protocol, called before
- *        unloading server protocol.
- *
- * @this:
- *
- */
-void
-fini (xlator_t *this)
-{
-        server_conf_t *conf = this->private;
-
-        GF_VALIDATE_OR_GOTO(this->name, conf, out);
-
-        if (conf->auth_modules) {
-                dict_unref (conf->auth_modules);
-        }
-
-        GF_FREE (conf);
-        this->private = NULL;
-out:
-        return;
-}
-
-/*
- * server_protocol_notify - notify function for server protocol
- * @this:
- * @trans:
- * @event:
- *
- */
-int
-notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        int          ret = 0;
-        transport_t *trans = NULL;
-        peer_info_t *peerinfo = NULL;
-        peer_info_t *myinfo = NULL;
-
-        THIS = this;
-        trans = data;
-        if (!trans) {
-                gf_log (this->name, GF_LOG_ERROR, "!trans");
-                goto out;
-        }
-
-        peerinfo = &(trans->peerinfo);
-        myinfo = &(trans->myinfo);
-
-        switch (event) {
-        case GF_EVENT_POLLIN:
-                ret = protocol_server_pollin (this, trans);
-                break;
-        case GF_EVENT_POLLERR:
-        {
-                gf_log (trans->xl->name, GF_LOG_INFO, "%s disconnected",
-                        peerinfo->identifier);
-
-                ret = -1;
-                transport_disconnect (trans);
-                if (trans->xl_private == NULL) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "POLLERR received on (%s) even before "
-                                "handshake with (%s) is successful",
-                                myinfo->identifier, peerinfo->identifier);
-                } else {
-                        /*
-                         * FIXME: shouldn't we check for return value?
-                         * what should be done if cleanup fails?
-                         */
-                        gf_server_connection_cleanup (this, trans->xl_private);
-                }
-        }
-        break;
-
-        case GF_EVENT_TRANSPORT_CLEANUP:
-        {
-                if (trans->xl_private) {
-                        gf_server_connection_put (this, trans->xl_private);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "transport (%s) cleaned up even before "
-                                "handshake with (%s) is successful",
-                                myinfo->identifier, peerinfo->identifier);
-                }
-        }
-        break;
-
-        default:
-                default_notify (this, event, data);
-                break;
-        }
-out:
-        return ret;
-}
-
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct xlator_dumpops dumpops = {
-        .inode = server_inode,
-        .priv  = server_priv,
-        .fd    = server_fd,
-};
-
-
-struct volume_options options[] = {
-        { .key   = {"transport-type"},
-          .value = {"tcp", "socket", "ib-verbs", "unix", "ib-sdp",
-                    "tcp/server", "ib-verbs/server"},
-          .type  = GF_OPTION_TYPE_STR
-        },
-        { .key   = {"volume-filename.*"},
-          .type  = GF_OPTION_TYPE_PATH,
-        },
-        { .key   = {"inode-lru-limit"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 0,
-          .max   = (1 * GF_UNIT_MB)
-        },
-        { .key   = {"client-volume-filename"},
-          .type  = GF_OPTION_TYPE_PATH
-        },
-        { .key   = {"verify-volfile-checksum"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"trace"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"conf-dir"},
-          .type  = GF_OPTION_TYPE_PATH,
-        },
-
-        { .key   = {NULL} },
-};
diff --git a/xlators/protocol/legacy/server/src/server-protocol.h b/xlators/protocol/legacy/server/src/server-protocol.h
deleted file mode 100644
index be6776033..000000000
--- a/xlators/protocol/legacy/server/src/server-protocol.h
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _SERVER_PROTOCOL_H_
-#define _SERVER_PROTOCOL_H_
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <pthread.h>
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
-#include "server-mem-types.h"
-#include "authenticate.h"
-#include "transport.h"
-
-#define DEFAULT_BLOCK_SIZE         4194304   /* 4MB */
-#define DEFAULT_VOLUME_FILE_PATH   CONFDIR "/glusterfs.vol"
-
-typedef struct _server_state server_state_t;
-
-struct _locker {
-	struct list_head  lockers;
-        char             *volume;
-	loc_t             loc;
-	fd_t             *fd;
-	pid_t             pid;
-};
-
-struct _lock_table {
-	struct list_head  file_lockers;
-	struct list_head  dir_lockers;
-	gf_lock_t         lock;
-	size_t            count;
-};
-
-
-/* private structure per connection (transport object)
- * used as transport_t->xl_private
- */
-struct _server_connection {
-	struct list_head    list;
-	char               *id;
-	int                 ref;
-        int                 active_transports;
-	pthread_mutex_t     lock;
-	char                disconnected;
-	fdtable_t          *fdtable; 
-	struct _lock_table *ltable;
-	xlator_t           *bound_xl;
-};
-
-typedef struct _server_connection server_connection_t;
-
-
-server_connection_t *
-gf_server_connection_get (xlator_t *this, const char *id);
-
-void
-gf_server_connection_put (xlator_t *this, server_connection_t *conn);
-
-int
-gf_server_connection_cleanup (xlator_t *this, server_connection_t *conn);
-
-struct _volfile_ctx {
-        struct _volfile_ctx *next;
-        char                *key;
-        uint32_t             checksum;
-};
-
-typedef struct {
-        struct _volfile_ctx *volfile;
-
-	dict_t           *auth_modules;
-	transport_t      *trans;
-	int32_t           max_block_size;
-	int32_t           inode_lru_limit;
-	pthread_mutex_t   mutex;
-	struct list_head  conns;
-        gf_boolean_t      verify_volfile_checksum;
-        gf_boolean_t      trace;
-} server_conf_t;
-
-
-typedef enum {
-        RESOLVE_MUST = 1,
-        RESOLVE_NOT,
-        RESOLVE_MAY,
-        RESOLVE_DONTCARE,
-        RESOLVE_EXACT
-} server_resolve_type_t;
-
-
-struct resolve_comp {
-        char      *basename;
-        ino_t      ino;
-        uint64_t   gen;
-        inode_t   *inode;
-};
-
-typedef struct {
-        server_resolve_type_t  type;
-        uint64_t               fd_no;
-        ino_t                  ino;
-        uint64_t               gen;
-        ino_t                  par;
-        char                  *path;
-        char                  *bname;
-	char                  *resolved;
-        int                    op_ret;
-        int                    op_errno;
-        loc_t                  deep_loc;
-        struct resolve_comp   *components;
-        int                    comp_count;
-} server_resolve_t;
-
-
-typedef int (*server_resume_fn_t) (call_frame_t *frame, xlator_t *bound_xl);
-
-int
-gf_resolve_and_resume (call_frame_t *frame, server_resume_fn_t fn);
-
-struct _server_state {
-	transport_t      *trans;
-	xlator_t         *bound_xl;
-        inode_table_t    *itable;
-
-        server_resume_fn_t resume_fn;
-
-	loc_t             loc;
-	loc_t             loc2;
-        server_resolve_t  resolve;
-        server_resolve_t  resolve2;
-
-        /* used within resolve_and_resume */
-        loc_t            *loc_now;
-        server_resolve_t *resolve_now;
-
-        struct iatt       stbuf;
-        int               valid;
-
-	fd_t             *fd;
-        dict_t           *params;
-	int               flags;
-        int               wbflags;
-        struct iobuf     *iobuf;
-        struct iobref    *iobref;
-
-	size_t            size;
-	off_t             offset;
-	mode_t            mode;
-	dev_t             dev;
-	size_t            nr_count;
-	int               cmd;
-	int               type;
-	char             *name;
-	int               name_len;
-
-	int               mask;
-	char              is_revalidate;
-	dict_t           *dict;
-	struct gf_flock      flock;
-        const char       *volume;
-        dir_entry_t      *entry;
-};
-
-
-#endif
diff --git a/xlators/protocol/legacy/server/src/server-resolve.c b/xlators/protocol/legacy/server/src/server-resolve.c
deleted file mode 100644
index 5a02be7ef..000000000
--- a/xlators/protocol/legacy/server/src/server-resolve.c
+++ /dev/null
@@ -1,658 +0,0 @@
-/*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "server-protocol.h"
-#include "server-helpers.h"
-
-#include "compat-errno.h"
-
-void
-gf_server_print_request (call_frame_t *frame);
-
-static int
-server_resolve_all (call_frame_t *frame);
-static int
-resolve_entry_simple (call_frame_t *frame);
-static int
-resolve_inode_simple (call_frame_t *frame);
-static int
-resolve_path_simple (call_frame_t *frame);
-
-
-static int
-component_count (const char *path)
-{
-        int         count = 0;
-        const char *trav = NULL;
-
-        trav = path;
-
-        for (trav = path; *trav; trav++) {
-                if (*trav == '/')
-                        count++;
-        }
-
-        return count + 2;
-}
-
-
-static int
-prepare_components (call_frame_t *frame)
-{
-        server_state_t       *state = NULL;
-        xlator_t             *this = NULL;
-        server_resolve_t     *resolve = NULL;
-        char                 *resolved = NULL;
-        int                   count = 0;
-        struct resolve_comp  *components = NULL;
-        int                   i = 0;
-        char                 *trav = NULL;
-
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        resolved = gf_strdup (resolve->path);
-        resolve->resolved = resolved;
-
-        count = component_count (resolve->path);
-        components = GF_CALLOC (sizeof (*components), count,
-                                gf_server_mt_resolve_comp);
-        resolve->components = components;
-
-        components[0].basename = "";
-        components[0].ino      = 1;
-        components[0].gen      = 0;
-        components[0].inode    = state->itable->root;
-
-        i = 1;
-        for (trav = resolved; *trav; trav++) {
-                if (*trav == '/') {
-                        components[i].basename = trav + 1;
-                        *trav = 0;
-                        i++;
-                }
-        }
-
-        return 0;
-}
-
-
-static int
-resolve_loc_touchup (call_frame_t *frame)
-{
-        server_state_t       *state = NULL;
-        server_resolve_t     *resolve = NULL;
-        loc_t                *loc = NULL;
-        char                 *path = NULL;
-        int                   ret = 0;
-
-        state = CALL_STATE (frame);
-
-        resolve = state->resolve_now;
-        loc     = state->loc_now;
-
-        if (!loc->path) {
-                if (loc->parent && resolve->bname) {
-                        ret = inode_path (loc->parent, resolve->bname, &path);
-                } else if (loc->inode) {
-                        ret = inode_path (loc->inode, NULL, &path);
-                }
-
-                if (!path)
-                        path = gf_strdup (resolve->path);
-
-                loc->path = path;
-        }
-
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name)
-                loc->name++;
-
-        if (!loc->parent && loc->inode) {
-                loc->parent = inode_parent (loc->inode, 0, NULL);
-        }
-
-        return 0;
-}
-
-
-static int
-resolve_deep_continue (call_frame_t *frame)
-{
-        server_state_t       *state = NULL;
-        xlator_t             *this = NULL;
-        server_resolve_t     *resolve = NULL;
-        int                   ret = 0;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        resolve->op_ret   = 0;
-        resolve->op_errno = 0;
-
-        if (resolve->par)
-                ret = resolve_entry_simple (frame);
-        else if (resolve->ino)
-                ret = resolve_inode_simple (frame);
-        else if (resolve->path)
-                ret = resolve_path_simple (frame);
-
-        resolve_loc_touchup (frame);
-
-        server_resolve_all (frame);
-
-        return 0;
-}
-
-
-static int
-resolve_deep_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-                  dict_t *xattr, struct iatt *postparent)
-{
-        server_state_t       *state = NULL;
-        server_resolve_t     *resolve = NULL;
-        struct resolve_comp  *components = NULL;
-        int                   i = 0;
-        inode_t              *link_inode = NULL;
-
-        state = CALL_STATE (frame);
-        resolve = state->resolve_now;
-        components = resolve->components;
-
-        i = (long) cookie;
-
-        if (op_ret == -1) {
-                goto get_out_of_here;
-        }
-
-        if (i != 0) {
-                /* no linking for root inode */
-                link_inode = inode_link (inode, resolve->deep_loc.parent,
-                                         resolve->deep_loc.name, buf);
-                inode_lookup (link_inode);
-                components[i].inode  = link_inode;
-                link_inode = NULL;
-        }
-
-        loc_wipe (&resolve->deep_loc);
-
-        i++; /* next component */
-
-        if (!components[i].basename) {
-                /* all components of the path are resolved */
-                goto get_out_of_here;
-        }
-
-        /* join the current component with the path resolved until now */
-        *(components[i].basename - 1) = '/';
-
-        resolve->deep_loc.path   = gf_strdup (resolve->resolved);
-        resolve->deep_loc.parent = inode_ref (components[i-1].inode);
-        resolve->deep_loc.inode  = inode_new (state->itable);
-        resolve->deep_loc.name   = components[i].basename;
-
-        STACK_WIND_COOKIE (frame, resolve_deep_cbk, (void *) (long) i,
-                           BOUND_XL (frame), BOUND_XL (frame)->fops->lookup,
-                           &resolve->deep_loc, NULL);
-        return 0;
-
-get_out_of_here:
-        resolve_deep_continue (frame);
-        return 0;
-}
-
-
-static int
-resolve_path_deep (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        xlator_t           *this = NULL;
-        server_resolve_t   *resolve = NULL;
-        int                 i = 0;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        gf_log (BOUND_XL (frame)->name, GF_LOG_DEBUG,
-                "RESOLVE %s() seeking deep resolution of %s",
-                gf_fop_list[frame->root->op], resolve->path);
-
-        prepare_components (frame);
-
-        /* start from the root */
-        resolve->deep_loc.inode = state->itable->root;
-        resolve->deep_loc.path  = gf_strdup ("/");
-        resolve->deep_loc.name  = "";
-
-        STACK_WIND_COOKIE (frame, resolve_deep_cbk, (void *) (long) i,
-                           BOUND_XL (frame), BOUND_XL (frame)->fops->lookup,
-                           &resolve->deep_loc, NULL);
-        return 0;
-}
-
-
-static int
-resolve_path_simple (call_frame_t *frame)
-{
-        server_state_t       *state = NULL;
-        xlator_t             *this = NULL;
-        server_resolve_t     *resolve = NULL;
-        struct resolve_comp  *components = NULL;
-        int                   ret = -1;
-        int                   par_idx = 0;
-        int                   ino_idx = 0;
-        int                   i = 0;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-        components = resolve->components;
-
-        if (!components) {
-                resolve->op_ret   = -1;
-                resolve->op_errno = ENOENT;
-                goto out;
-        }
-
-        for (i = 0; components[i].basename; i++) {
-                par_idx = ino_idx;
-                ino_idx = i;
-        }
-
-        if (!components[par_idx].inode) {
-                resolve->op_ret    = -1;
-                resolve->op_errno  = ENOENT;
-                goto out;
-        }
-
-        if (!components[ino_idx].inode &&
-            (resolve->type == RESOLVE_MUST || resolve->type == RESOLVE_EXACT)) {
-                resolve->op_ret    = -1;
-                resolve->op_errno  = ENOENT;
-                goto out;
-        }
-
-        if (components[ino_idx].inode && resolve->type == RESOLVE_NOT) {
-                resolve->op_ret    = -1;
-                resolve->op_errno  = EEXIST;
-                goto out;
-        }
-
-        if (components[ino_idx].inode)
-                state->loc_now->inode  = inode_ref (components[ino_idx].inode);
-        state->loc_now->parent = inode_ref (components[par_idx].inode);
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-/*
-  Check if the requirements are fulfilled by entries in the inode cache itself
-  Return value:
-  <= 0 - simple resolution was decisive and complete (either success or failure)
-  > 0  - indecisive, need to perform deep resolution
-*/
-
-static int
-resolve_entry_simple (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        xlator_t           *this = NULL;
-        server_resolve_t   *resolve = NULL;
-        inode_t            *parent = NULL;
-        inode_t            *inode = NULL;
-        int                 ret = 0;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        parent = inode_get (state->itable, resolve->par, 0);
-        if (!parent) {
-                /* simple resolution is indecisive. need to perform
-                   deep resolution */
-                resolve->op_ret   = -1;
-                resolve->op_errno = ENOENT;
-                ret = 1;
-
-                inode = inode_grep (state->itable, parent, resolve->bname);
-                if (inode != NULL) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%"PRId64": inode "
-                                "(pointer:%p ino: %"PRIu64") present but parent"
-                                " is NULL for path (%s)", frame->root->unique,
-                                inode, inode->ino, resolve->path);
-                        inode_unref (inode);
-                }
-                goto out;
-        }
-
-//        if (parent->ino != 1 && parent->generation != resolve->gen) {
-        if (0) {
-                /* simple resolution is decisive - request was for a
-                   stale handle */
-                resolve->op_ret   = -1;
-                resolve->op_errno = ENOENT;
-                ret = -1;
-                goto out;
-        }
-
-        /* expected @parent was found from the inode cache */
-        state->loc_now->parent = inode_ref (parent);
-
-        inode = inode_grep (state->itable, parent, resolve->bname);
-        if (!inode) {
-                switch (resolve->type) {
-                case RESOLVE_DONTCARE:
-                case RESOLVE_NOT:
-                        ret = 0;
-                        break;
-                case RESOLVE_MAY:
-                        ret = 1;
-                        break;
-                default:
-                        resolve->op_ret   = -1;
-                        resolve->op_errno = ENOENT;
-                        ret = 1;
-                        break;
-                }
-
-                goto out;
-        }
-
-        if (resolve->type == RESOLVE_NOT) {
-                gf_log (this->name, GF_LOG_DEBUG, "inode (pointer: %p ino:%"
-                        PRIu64") found for path (%s) while type is RESOLVE_NOT",
-                        inode, inode->ino, resolve->path);
-                resolve->op_ret   = -1;
-                resolve->op_errno = EEXIST;
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-
-        state->loc_now->inode  = inode_ref (inode);
-
-out:
-        if (parent)
-                inode_unref (parent);
-
-        if (inode)
-                inode_unref (inode);
-
-        return ret;
-}
-
-
-static int
-server_resolve_entry (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        xlator_t           *this = NULL;
-        server_resolve_t   *resolve = NULL;
-        int                 ret = 0;
-        loc_t              *loc = NULL;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-        loc  = state->loc_now;
-
-        ret = resolve_entry_simple (frame);
-
-        if (ret > 0) {
-                loc_wipe (loc);
-                resolve_path_deep (frame);
-                return 0;
-        }
-
-        if (ret == 0)
-                resolve_loc_touchup (frame);
-
-        server_resolve_all (frame);
-
-        return 0;
-}
-
-
-static int
-resolve_inode_simple (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        xlator_t           *this = NULL;
-        server_resolve_t   *resolve = NULL;
-        inode_t            *inode = NULL;
-        int                 ret = 0;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        if (resolve->type == RESOLVE_EXACT) {
-                inode = inode_get (state->itable, resolve->ino, resolve->gen);
-        } else {
-                inode = inode_get (state->itable, resolve->ino, 0);
-        }
-
-        if (!inode) {
-                resolve->op_ret   = -1;
-                resolve->op_errno = ENOENT;
-                ret = 1;
-                goto out;
-        }
-
-//        if (inode->ino != 1 && inode->generation != resolve->gen) {
-        if (0) {
-                resolve->op_ret      = -1;
-                resolve->op_errno    = ENOENT;
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-
-        state->loc_now->inode = inode_ref (inode);
-
-out:
-        if (inode)
-                inode_unref (inode);
-
-        return ret;
-}
-
-
-static int
-server_resolve_inode (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        int                 ret = 0;
-        loc_t              *loc = NULL;
-
-        state = CALL_STATE (frame);
-        loc  = state->loc_now;
-
-        ret = resolve_inode_simple (frame);
-
-        if (ret > 0) {
-                loc_wipe (loc);
-                resolve_path_deep (frame);
-                return 0;
-        }
-
-        if (ret == 0)
-                resolve_loc_touchup (frame);
-
-        server_resolve_all (frame);
-
-        return 0;
-}
-
-
-static int
-server_resolve_fd (call_frame_t *frame)
-{
-        server_state_t       *state = NULL;
-        xlator_t             *this = NULL;
-        server_resolve_t     *resolve = NULL;
-        server_connection_t  *conn = NULL;
-        uint64_t              fd_no = -1;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-        conn  = SERVER_CONNECTION (frame);
-
-        fd_no = resolve->fd_no;
-
-        state->fd = gf_fd_fdptr_get (conn->fdtable, fd_no);
-
-        if (!state->fd) {
-                resolve->op_ret   = -1;
-                resolve->op_errno = EBADFD;
-        }
-
-        server_resolve_all (frame);
-
-        return 0;
-}
-
-
-static int
-server_resolve (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        xlator_t           *this = NULL;
-        server_resolve_t   *resolve = NULL;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        if (resolve->fd_no != -1) {
-
-                server_resolve_fd (frame);
-
-        } else if (resolve->par) {
-
-                server_resolve_entry (frame);
-
-        } else if (resolve->ino) {
-
-                server_resolve_inode (frame);
-
-        } else if (resolve->path) {
-
-                resolve_path_deep (frame);
-
-        } else  {
-
-                resolve->op_ret = -1;
-                resolve->op_errno = EINVAL;
-
-                server_resolve_all (frame);
-        }
-
-        return 0;
-}
-
-
-static int
-server_resolve_done (call_frame_t *frame)
-{
-        server_state_t    *state = NULL;
-        xlator_t          *bound_xl = NULL;
-
-        state = CALL_STATE (frame);
-        bound_xl = BOUND_XL (frame);
-
-        gf_server_print_request (frame);
-
-        state->resume_fn (frame, bound_xl);
-
-        return 0;
-}
-
-
-/*
- * This function is called multiple times, once per resolving one location/fd.
- * state->resolve_now is used to decide which location/fd is to be resolved now
- */
-static int
-server_resolve_all (call_frame_t *frame)
-{
-        server_state_t    *state = NULL;
-        xlator_t          *this = NULL;
-
-        this  = frame->this;
-        state = CALL_STATE (frame);
-
-        if (state->resolve_now == NULL) {
-
-                state->resolve_now = &state->resolve;
-                state->loc_now     = &state->loc;
-
-                server_resolve (frame);
-
-        } else if (state->resolve_now == &state->resolve) {
-
-                state->resolve_now = &state->resolve2;
-                state->loc_now     = &state->loc2;
-
-                server_resolve (frame);
-
-        } else if (state->resolve_now == &state->resolve2) {
-
-                server_resolve_done (frame);
-
-        } else {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid pointer for state->resolve_now");
-        }
-
-        return 0;
-}
-
-
-int
-gf_resolve_and_resume (call_frame_t *frame, server_resume_fn_t fn)
-{
-        server_state_t    *state = NULL;
-        xlator_t          *this  = NULL;
-
-        state = CALL_STATE (frame);
-        state->resume_fn = fn;
-
-        this = frame->this;
-
-        server_resolve_all (frame);
-
-        return 0;
-}
diff --git a/xlators/protocol/legacy/transport/Makefile.am b/xlators/protocol/legacy/transport/Makefile.am
deleted file mode 100644
index e2f97437c..000000000
--- a/xlators/protocol/legacy/transport/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = socket $(IBVERBS_SUBDIR)
-
-CLEANFILES = 
diff --git a/xlators/protocol/legacy/transport/ib-verbs/Makefile.am b/xlators/protocol/legacy/transport/ib-verbs/Makefile.am
deleted file mode 100644
index f963effea..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS = src
-\ No newline at end of file
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/Makefile.am b/xlators/protocol/legacy/transport/ib-verbs/src/Makefile.am
deleted file mode 100644
index 3db7aff98..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-# TODO : need to change transportdir
-
-transport_LTLIBRARIES = ib-verbs.la
-transportdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/transport
-
-ib_verbs_la_LDFLAGS = -module -avoidversion
-
-ib_verbs_la_SOURCES = ib-verbs.c name.c
-ib_verbs_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la     \
-	-libverbs $(top_builddir)/xlators/protocol/legacy/lib/src/libgfproto.la
-
-noinst_HEADERS = ib-verbs.h name.h ib-verbs-mem-types.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)  \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)   \
-	-I$(top_srcdir)/xlators/protocol/legacy/transport/ib-verbs                    \
-	-I$(top_srcdir)/xlators/protocol/legacy/lib/src
-
-CLEANFILES = *~
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs-mem-types.h b/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs-mem-types.h
deleted file mode 100644
index 80e81e56b..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs-mem-types.h
+++ /dev/null
@@ -1,39 +0,0 @@
-
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __IB_VERBS_MEM_TYPES_H__
-#define __IB_VERBS_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_ib_verbs_mem_types_ {
-        gf_ibv_mt_ib_verbs_private_t = gf_common_mt_end + 1,
-        gf_ibv_mt_ib_verbs_ioq_t,
-        gf_ibv_mt_transport_t,
-        gf_ibv_mt_ib_verbs_local_t,
-        gf_ibv_mt_ib_verbs_post_t,
-        gf_ibv_mt_char,
-        gf_ibv_mt_qpent,
-        gf_ibv_mt_ib_verbs_device_t,
-        gf_ibv_mt_end
-};
-#endif
-
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.c b/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.c
deleted file mode 100644
index 8b52fa884..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.c
+++ /dev/null
@@ -1,2625 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "dict.h"
-#include "glusterfs.h"
-#include "transport.h"
-#include "protocol.h"
-#include "logging.h"
-#include "xlator.h"
-#include "name.h"
-#include "ib-verbs.h"
-#include <signal.h>
-
-int32_t
-gf_resolve_ip6 (const char *hostname, 
-                uint16_t port, 
-                int family, 
-                void **dnscache, 
-                struct addrinfo **addr_info);
-
-static uint16_t 
-ib_verbs_get_local_lid (struct ibv_context *context,
-                        int32_t port)
-{
-        struct ibv_port_attr attr;
-
-        if (ibv_query_port (context, port, &attr))
-                return 0;
-
-        return attr.lid;
-}
-
-static const char *
-get_port_state_str(enum ibv_port_state pstate)
-{
-	switch (pstate) {
-	case IBV_PORT_DOWN:          return "PORT_DOWN";
-	case IBV_PORT_INIT:          return "PORT_INIT";
-	case IBV_PORT_ARMED:         return "PORT_ARMED";
-	case IBV_PORT_ACTIVE:        return "PORT_ACTIVE";
-	case IBV_PORT_ACTIVE_DEFER:  return "PORT_ACTIVE_DEFER";
-	default:                     return "invalid state";
-	}
-}
-
-static int32_t
-ib_check_active_port (struct ibv_context *ctx, uint8_t port)
-{
-        struct ibv_port_attr port_attr;
-
-        int32_t ret           = 0;
-        const char *state_str = NULL;
-
-        if (!ctx) {
-		gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                        "Error in supplied context");
-                return -1;
-	}
-
-        ret = ibv_query_port (ctx, port, &port_attr);
-
-        if (ret) {
-                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                        "Failed to query port %u properties", port);
-                return -1;
-        }
-
-        state_str = get_port_state_str (port_attr.state);
-        gf_log ("transport/ib-verbs", GF_LOG_TRACE,
-                "Infiniband PORT: (%u) STATE: (%s)",
-                port, state_str);
-
-        if (port_attr.state == IBV_PORT_ACTIVE)
-                return 0;
-
-	return -1;
-}
-
-static int32_t
-ib_get_active_port (struct ibv_context *ib_ctx)
-{
-	struct ibv_device_attr ib_device_attr;
-
-	int32_t ret     = -1;
-	uint8_t ib_port = 0;
-
-	if (!ib_ctx) {
-		gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                        "Error in supplied context");
-		return -1;
-	}
-	if (ibv_query_device (ib_ctx, &ib_device_attr)) {
-		gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                        "Failed to query device properties");
-		return -1;
-	}
-
-	for (ib_port = 1; ib_port <= ib_device_attr.phys_port_cnt; ++ib_port) {
-                ret = ib_check_active_port (ib_ctx, ib_port);
-                if (ret == 0)
-                        return ib_port;
-
-                gf_log ("transport/ib-verbs", GF_LOG_TRACE,
-                        "Port:(%u) not active", ib_port);
-                continue;
-	}
-	return ret;
-}
-
-
-
-static void
-ib_verbs_put_post (ib_verbs_queue_t *queue,
-                   ib_verbs_post_t *post)
-{
-        pthread_mutex_lock (&queue->lock);
-        if (post->prev) {
-                queue->active_count--;
-                post->prev->next = post->next;
-        }
-        if (post->next)
-                post->next->prev = post->prev;
-        post->prev = &queue->passive_posts;
-        post->next = post->prev->next;
-        post->prev->next = post;
-        post->next->prev = post;
-        queue->passive_count++;
-        pthread_mutex_unlock (&queue->lock);
-}
-
-
-static ib_verbs_post_t *
-ib_verbs_new_post (ib_verbs_device_t *device, int32_t len)
-{
-        ib_verbs_post_t *post;
-
-        post = (ib_verbs_post_t *) GF_CALLOC (1, sizeof (*post),
-                                   gf_ibv_mt_ib_verbs_post_t);
-        if (!post)
-                return NULL;
-
-        post->buf_size = len;
-
-        post->buf = valloc (len);
-        if (!post->buf) {
-                GF_FREE (post);
-                return NULL;
-        }
-
-        post->mr = ibv_reg_mr (device->pd,
-                               post->buf,
-                               post->buf_size,
-                               IBV_ACCESS_LOCAL_WRITE);
-        if (!post->mr) {
-                free (post->buf);
-                GF_FREE (post);
-                return NULL;
-        }
-
-        return post;
-}
-
-
-static ib_verbs_post_t *
-ib_verbs_get_post (ib_verbs_queue_t *queue)
-{
-        ib_verbs_post_t *post;
-
-        pthread_mutex_lock (&queue->lock);
-        {
-                post = queue->passive_posts.next;
-                if (post == &queue->passive_posts)
-                        post = NULL;
-    
-                if (post) {
-                        if (post->prev)
-                                post->prev->next = post->next;
-                        if (post->next)
-                                post->next->prev = post->prev;
-                        post->prev = &queue->active_posts;
-                        post->next = post->prev->next;
-                        post->prev->next = post;
-                        post->next->prev = post;
-                        post->reused++;
-                        queue->active_count++;
-                }
-        }
-        pthread_mutex_unlock (&queue->lock);
-
-        return post;
-}
-
-void
-ib_verbs_destroy_post (ib_verbs_post_t *post)
-{
-        ibv_dereg_mr (post->mr);
-        free (post->buf);
-        GF_FREE (post);
-}
-
-
-static int32_t
-__ib_verbs_quota_get (ib_verbs_peer_t *peer)
-{
-        int32_t ret = -1;
-        ib_verbs_private_t *priv = peer->trans->private;
-
-        if (priv->connected && peer->quota > 0) {
-                ret = peer->quota--;
-        }
-
-        return ret;
-}
-
-/*
-  static int32_t
-  ib_verbs_quota_get (ib_verbs_peer_t *peer)
-  {
-  int32_t ret = -1;
-  ib_verbs_private_t *priv = peer->trans->private;
-
-  pthread_mutex_lock (&priv->write_mutex);
-  {
-  ret = __ib_verbs_quota_get (peer);
-  }
-  pthread_mutex_unlock (&priv->write_mutex);
-
-  return ret;
-  }
-*/
-
-static void 
-__ib_verbs_ioq_entry_free (ib_verbs_ioq_t *entry)
-{
-        list_del_init (&entry->list);
-        if (entry->iobref)
-                iobref_unref (entry->iobref);
-
-        /* TODO: use mem-pool */
-        GF_FREE (entry->buf);
-
-        /* TODO: use mem-pool */
-        GF_FREE (entry);
-}
-
-
-static void
-__ib_verbs_ioq_flush (ib_verbs_peer_t *peer)
-{
-        ib_verbs_ioq_t *entry = NULL, *dummy = NULL;
-
-        list_for_each_entry_safe (entry, dummy, &peer->ioq, list) {
-                __ib_verbs_ioq_entry_free (entry);
-        }
-}
-
-
-static int32_t
-__ib_verbs_disconnect (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        int32_t ret = 0;
-
-        if (priv->connected || priv->tcp_connected) {
-                fcntl (priv->sock, F_SETFL, O_NONBLOCK);
-                if (shutdown (priv->sock, SHUT_RDWR) != 0) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_DEBUG,
-                                "shutdown () - error: %s",
-                                strerror (errno));
-                        ret = -errno;
-                        priv->tcp_connected = 0;
-                }
-        }
-  
-        return ret;
-}
-
-
-static int32_t
-ib_verbs_post_send (struct ibv_qp *qp,
-                    ib_verbs_post_t *post,
-                    int32_t len)
-{
-        struct ibv_sge list = {
-                .addr = (unsigned long) post->buf,
-                .length = len,
-                .lkey = post->mr->lkey
-        };
-
-        struct ibv_send_wr wr = {
-                .wr_id      = (unsigned long) post,
-                .sg_list    = &list,
-                .num_sge    = 1,
-                .opcode     = IBV_WR_SEND,
-                .send_flags = IBV_SEND_SIGNALED,
-        }, *bad_wr;
-
-        if (!qp)
-                return -1;
-
-        return ibv_post_send (qp, &wr, &bad_wr);
-}
-
-
-static int32_t
-__ib_verbs_ioq_churn_entry (ib_verbs_peer_t *peer, ib_verbs_ioq_t *entry)
-{
-        int32_t ret = 0, quota = 0;
-        ib_verbs_private_t *priv = peer->trans->private;
-        ib_verbs_device_t *device = priv->device;
-        ib_verbs_options_t *options = &priv->options;
-        ib_verbs_post_t *post = NULL;
-        int32_t len = 0;
-
-        quota = __ib_verbs_quota_get (peer);
-        if (quota > 0) {
-                post = ib_verbs_get_post (&device->sendq);
-                if (!post) 
-                        post = ib_verbs_new_post (device, 
-                                                  (options->send_size + 2048));
-
-                len = iov_length ((const struct iovec *)&entry->vector, 
-                                  entry->count);
-                if  (len >= (options->send_size + 2048)) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "increase value of option 'transport.ib-verbs."
-                                "work-request-send-size' (given=> %"PRId64") "
-                                "to send bigger (%d) messages", 
-                                (options->send_size + 2048), len);
-                        return -1;
-                }
-
-                iov_unload (post->buf, 
-                            (const struct iovec *)&entry->vector, 
-                            entry->count);
-
-                ret = ib_verbs_post_send (peer->qp, post, len);
-                if (!ret) {
-                        __ib_verbs_ioq_entry_free (entry);
-                        ret = len;
-                } else {
-                        gf_log ("transport/ib-verbs", GF_LOG_DEBUG,
-                                "ibv_post_send failed with ret = %d", ret);
-                        ib_verbs_put_post (&device->sendq, post);
-                        __ib_verbs_disconnect (peer->trans);
-                        ret = -1;
-                } 
-        }
-
-        return ret;
-}
-
-
-static int32_t
-__ib_verbs_ioq_churn (ib_verbs_peer_t *peer)
-{
-        ib_verbs_ioq_t *entry = NULL;
-        int32_t ret = 0;
-
-        while (!list_empty (&peer->ioq))
-        {
-                /* pick next entry */
-                entry = peer->ioq_next;
-
-                ret = __ib_verbs_ioq_churn_entry (peer, entry);
-
-                if (ret <= 0)
-                        break;
-        }
-
-        /*
-          list_for_each_entry_safe (entry, dummy, &peer->ioq, list) {
-          ret = __ib_verbs_ioq_churn_entry (peer, entry);
-          if (ret <= 0) {
-          break;
-          }
-          }
-        */
-
-        return ret;
-}
-
-static int32_t
-__ib_verbs_quota_put (ib_verbs_peer_t *peer)
-{
-        int32_t ret;
-
-        peer->quota++;
-        ret = peer->quota;
-
-        if (!list_empty (&peer->ioq)) {
-                ret = __ib_verbs_ioq_churn (peer);
-        }
-
-        return ret;
-}
-
-
-static int32_t
-ib_verbs_quota_put (ib_verbs_peer_t *peer)
-{
-        int32_t ret;
-        ib_verbs_private_t *priv = peer->trans->private;
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                ret = __ib_verbs_quota_put (peer);
-        }
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        return ret;
-}
-
-
-static int32_t
-ib_verbs_post_recv (struct ibv_srq *srq,
-                    ib_verbs_post_t *post)
-{
-        struct ibv_sge list = {
-                .addr   = (unsigned long) post->buf,
-                .length = post->buf_size,
-                .lkey   = post->mr->lkey
-        };
-
-        struct ibv_recv_wr wr = {
-                .wr_id  = (unsigned long) post,
-                .sg_list = &list,
-                .num_sge = 1,
-        }, *bad_wr;
-
-        return ibv_post_srq_recv (srq, &wr, &bad_wr);
-}
-
-
-static int32_t
-ib_verbs_writev (transport_t *this,
-                 ib_verbs_ioq_t *entry)
-{
-        int32_t ret = 0, need_append = 1;
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_peer_t  *peer = NULL;
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                if (!priv->connected) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "ib-verbs is not connected to post a "
-                                "send request");
-                        ret = -1;
-                        goto unlock;
-                }
-
-                peer = &priv->peer;
-                if (list_empty (&peer->ioq)) {
-                        ret = __ib_verbs_ioq_churn_entry (peer, entry);
-                        if (ret != 0) {
-                                need_append = 0;
-                        }
-                }
-
-                if (need_append) {
-                        list_add_tail (&entry->list, &peer->ioq);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-        return ret;
-}
-
-
-static ib_verbs_ioq_t *
-ib_verbs_ioq_new (char *buf, int len, struct iovec *vector, 
-                  int count, struct iobref *iobref)
-{
-        ib_verbs_ioq_t *entry = NULL;
-
-        /* TODO: use mem-pool */
-        entry = GF_CALLOC (1, sizeof (*entry), gf_ibv_mt_ib_verbs_ioq_t);
-
-        GF_ASSERT (count <= (MAX_IOVEC-2));
-
-        entry->header.colonO[0] = ':';
-        entry->header.colonO[1] = 'O';
-        entry->header.colonO[2] = '\0';
-        entry->header.version   = 42;
-        entry->header.size1     = hton32 (len);
-        entry->header.size2     = hton32 (iov_length (vector, count));
-
-        entry->vector[0].iov_base = &entry->header;
-        entry->vector[0].iov_len  = sizeof (entry->header);
-        entry->count++;
-
-        entry->vector[1].iov_base = buf;
-        entry->vector[1].iov_len  = len;
-        entry->count++;
-
-        if (vector && count)
-        {
-                memcpy (&entry->vector[2], vector, sizeof (*vector) * count);
-                entry->count += count;
-        }
-
-        if (iobref)
-                entry->iobref = iobref_ref (iobref);
-
-        entry->buf = buf;
-
-        INIT_LIST_HEAD (&entry->list);
-
-        return entry;
-}
-
-
-static int32_t
-ib_verbs_submit (transport_t *this, char *buf, int32_t len,
-                 struct iovec *vector, int count, struct iobref *iobref)
-{
-        int32_t ret = 0;
-        ib_verbs_ioq_t *entry = NULL;
-  
-        entry = ib_verbs_ioq_new (buf, len, vector, count, iobref);
-        ret = ib_verbs_writev (this, entry);
-
-        if (ret > 0) {
-                ret = 0;
-        }
-
-        return ret;
-}
-
-static int
-ib_verbs_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
-                  struct iobuf **iobuf_p)
-{
-        ib_verbs_private_t *priv = this->private;
-        /* TODO: return error if !priv->connected, check with locks */
-        /* TODO: boundry checks for data_ptr/offset */
-        char *copy_from = NULL;
-        ib_verbs_header_t *header = NULL;
-        uint32_t size1, size2, data_len = 0;
-        char *hdr = NULL;
-        struct iobuf *iobuf = NULL;
-        int32_t ret = 0;
-
-        pthread_mutex_lock (&priv->recv_mutex);
-        {
-/*
-  while (!priv->data_ptr)
-  pthread_cond_wait (&priv->recv_cond, &priv->recv_mutex);
-*/
-
-                copy_from = priv->data_ptr + priv->data_offset;
-
-                priv->data_ptr = NULL;
-                data_len = priv->data_len;
-                pthread_cond_broadcast (&priv->recv_cond);
-        }
-        pthread_mutex_unlock (&priv->recv_mutex);
-
-        header = (ib_verbs_header_t *)copy_from;
-        if (strcmp (header->colonO, ":O")) {
-                gf_log ("transport/ib-verbs", GF_LOG_DEBUG,
-                        "%s: corrupt header received", this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        size1 = ntoh32 (header->size1);
-        size2 = ntoh32 (header->size2);
-
-        if (data_len != (size1 + size2 + sizeof (*header))) {
-                gf_log ("transport/ib-verbs", GF_LOG_DEBUG,
-                        "%s: sizeof data read from transport is not equal "
-                        "to the size specified in the header",
-                        this->xl->name);
-                ret = -1;
-                goto err;
-        }
-                  
-        copy_from += sizeof (*header);
-
-        if (size1) {
-                hdr = GF_CALLOC (1, size1, gf_ibv_mt_char);
-                if (!hdr) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unable to allocate header for peer %s",
-                                this->peerinfo.identifier);
-                        ret = -ENOMEM;
-                        goto err;
-                }
-                memcpy (hdr, copy_from, size1);
-                copy_from += size1;
-                *hdr_p = hdr;
-        }
-        *hdrlen_p = size1;
-
-        if (size2) {
-                iobuf = iobuf_get (this->xl->ctx->iobuf_pool);
-                if (!iobuf) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unable to allocate IO buffer for peer %s",
-                                this->peerinfo.identifier);
-                        ret = -ENOMEM;
-                        goto err;
-                }
-                memcpy (iobuf->ptr, copy_from, size2);
-                *iobuf_p = iobuf;
-        }
-
-err:
-        return ret;
-}
-
-
-static void
-ib_verbs_destroy_cq (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_device_t *device = priv->device;
-  
-        if (device->recv_cq)
-                ibv_destroy_cq (device->recv_cq);
-        device->recv_cq = NULL;
-  
-        if (device->send_cq)
-                ibv_destroy_cq (device->send_cq);
-        device->send_cq = NULL;
-
-        return;
-}
-
-
-static int32_t
-ib_verbs_create_cq (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        ib_verbs_device_t *device = priv->device;
-        int32_t ret = 0;
-
-        device->recv_cq = ibv_create_cq (priv->device->context,
-                                         options->recv_count * 2,
-                                         device,
-                                         device->recv_chan,
-                                         0);
-        if (!device->recv_cq) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_ERROR,
-                        "%s: creation of CQ failed",
-                        this->xl->name);
-                ret = -1;
-        } else if (ibv_req_notify_cq (device->recv_cq, 0)) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_ERROR,
-                        "%s: ibv_req_notify_cq on CQ failed",
-                        this->xl->name);
-                ret = -1;
-        }
-    
-        do {
-                /* TODO: make send_cq size dynamically adaptive */
-                device->send_cq = ibv_create_cq (priv->device->context,
-                                                 options->send_count * 1024,
-                                                 device,
-                                                 device->send_chan,
-                                                 0);
-                if (!device->send_cq) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_ERROR,
-                                "%s: creation of send_cq failed",
-                                this->xl->name);
-                        ret = -1;
-                        break;
-                }
-
-                if (ibv_req_notify_cq (device->send_cq, 0)) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_ERROR,
-                                "%s: ibv_req_notify_cq on send_cq failed",
-                                this->xl->name);
-                        ret = -1;
-                        break;
-                }
-        } while (0);
-
-        if (ret != 0)
-                ib_verbs_destroy_cq (this);
-
-        return ret;
-}
-
-
-static void
-ib_verbs_register_peer (ib_verbs_device_t *device,
-                        int32_t qp_num,
-                        ib_verbs_peer_t *peer)
-{
-        struct _qpent *ent;
-        ib_verbs_qpreg_t *qpreg = &device->qpreg;
-        int32_t hash = qp_num % 42;
-
-        pthread_mutex_lock (&qpreg->lock);
-        ent = qpreg->ents[hash].next;
-        while ((ent != &qpreg->ents[hash]) && (ent->qp_num != qp_num))
-                ent = ent->next;
-        if (ent->qp_num == qp_num) {
-                pthread_mutex_unlock (&qpreg->lock);
-                return;
-        }
-        ent = (struct _qpent *) GF_CALLOC (1, sizeof (*ent), gf_ibv_mt_qpent);
-        if (!ent)
-                return;
-        /* TODO: ref reg->peer */
-        ent->peer = peer;
-        ent->next = &qpreg->ents[hash];
-        ent->prev = ent->next->prev;
-        ent->next->prev = ent;
-        ent->prev->next = ent;
-        ent->qp_num = qp_num;
-        qpreg->count++;
-        pthread_mutex_unlock (&qpreg->lock);
-}
-
-
-static void
-ib_verbs_unregister_peer (ib_verbs_device_t *device,
-                          int32_t qp_num)
-{
-        struct _qpent *ent;
-        ib_verbs_qpreg_t *qpreg = &device->qpreg;
-        int32_t hash = qp_num % 42;
-
-        pthread_mutex_lock (&qpreg->lock);
-        ent = qpreg->ents[hash].next;
-        while ((ent != &qpreg->ents[hash]) && (ent->qp_num != qp_num))
-                ent = ent->next;
-        if (ent->qp_num != qp_num) {
-                pthread_mutex_unlock (&qpreg->lock);
-                return;
-        }
-        ent->prev->next = ent->next;
-        ent->next->prev = ent->prev;
-        /* TODO: unref reg->peer */
-        GF_FREE (ent);
-        qpreg->count--;
-        pthread_mutex_unlock (&qpreg->lock);
-}
-
-
-static ib_verbs_peer_t *
-__ib_verbs_lookup_peer (ib_verbs_device_t *device, int32_t qp_num)
-{
-        struct _qpent    *ent   = NULL;
-        ib_verbs_peer_t  *peer  = NULL;
-        ib_verbs_qpreg_t *qpreg = NULL;
-        int32_t hash            = 0;
-
-        qpreg = &device->qpreg;
-        hash = qp_num % 42;
-        ent = qpreg->ents[hash].next;
-        while ((ent != &qpreg->ents[hash]) && (ent->qp_num != qp_num))
-                ent = ent->next;
-
-        if (ent != &qpreg->ents[hash]) {
-                peer = ent->peer;
-        }
-
-        return peer;
-}
-
-/*
-static ib_verbs_peer_t *
-ib_verbs_lookup_peer (ib_verbs_device_t *device,
-                      int32_t qp_num)
-{
-        ib_verbs_qpreg_t *qpreg = NULL;
-        ib_verbs_peer_t  *peer  = NULL;
- 
-        qpreg = &device->qpreg;
-        pthread_mutex_lock (&qpreg->lock);
-        {
-                peer = __ib_verbs_lookup_peer (device, qp_num);
-        }
-        pthread_mutex_unlock (&qpreg->lock);
-
-        return peer;
-}
-*/
-
-
-static void
-__ib_verbs_destroy_qp (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-
-        if (priv->peer.qp) {
-                ib_verbs_unregister_peer (priv->device, priv->peer.qp->qp_num);
-                ibv_destroy_qp (priv->peer.qp);
-        }
-        priv->peer.qp = NULL;
-
-        return;
-}
-
-
-static int32_t
-ib_verbs_create_qp (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        ib_verbs_device_t *device = priv->device;
-        int32_t ret = 0;
-        ib_verbs_peer_t *peer;
-
-        peer = &priv->peer;
-        struct ibv_qp_init_attr init_attr = {
-                .send_cq        = device->send_cq,
-                .recv_cq        = device->recv_cq,
-                .srq            = device->srq,
-                .cap            = {
-                        .max_send_wr  = peer->send_count,
-                        .max_recv_wr  = peer->recv_count,
-                        .max_send_sge = 1,
-                        .max_recv_sge = 1
-                },
-                .qp_type = IBV_QPT_RC
-        };
-  
-        struct ibv_qp_attr attr = {
-                .qp_state        = IBV_QPS_INIT,
-                .pkey_index      = 0,
-                .port_num        = options->port,
-                .qp_access_flags = 0
-        };
-  
-        peer->qp = ibv_create_qp (device->pd, &init_attr);
-        if (!peer->qp) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_CRITICAL,
-                        "%s: could not create QP",
-                        this->xl->name);
-                ret = -1;
-                goto out;
-        } else if (ibv_modify_qp (peer->qp, &attr,
-                                  IBV_QP_STATE              |
-                                  IBV_QP_PKEY_INDEX         |
-                                  IBV_QP_PORT               |
-                                  IBV_QP_ACCESS_FLAGS)) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_ERROR,
-                        "%s: failed to modify QP to INIT state",
-                        this->xl->name);
-                ret = -1;
-                goto out;
-        }
-
-        peer->local_lid = ib_verbs_get_local_lid (device->context,
-                                                  options->port);
-        peer->local_qpn = peer->qp->qp_num;
-        peer->local_psn = lrand48 () & 0xffffff;
-
-        ib_verbs_register_peer (device, peer->qp->qp_num, peer);
-
-out:
-        if (ret == -1)
-                __ib_verbs_destroy_qp (this);
-
-        return ret;
-}
-
-
-static void
-ib_verbs_destroy_posts (transport_t *this)
-{
-
-}
-
-
-static int32_t
-__ib_verbs_create_posts (transport_t *this,
-                         int32_t count,
-                         int32_t size,
-                         ib_verbs_queue_t *q)
-{
-        int32_t i;
-        int32_t ret = 0;
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_device_t *device = priv->device;
-
-        for (i=0 ; i<count ; i++) {
-                ib_verbs_post_t *post;
-
-                post = ib_verbs_new_post (device, size + 2048);
-                if (!post) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_ERROR,
-                                "%s: post creation failed",
-                                this->xl->name);
-                        ret = -1;
-                        break;
-                }
-
-                ib_verbs_put_post (q, post);
-        }
-        return ret;
-}
-
-
-static int32_t
-ib_verbs_create_posts (transport_t *this)
-{
-        int32_t i, ret;
-        ib_verbs_post_t *post = NULL;
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        ib_verbs_device_t *device = priv->device;
-
-        ret =  __ib_verbs_create_posts (this, options->send_count,
-                                        options->send_size,
-                                        &device->sendq);
-        if (!ret)
-                ret =  __ib_verbs_create_posts (this, options->recv_count,
-                                                options->recv_size,
-                                                &device->recvq);
-
-        if (!ret) {
-                for (i=0 ; i<options->recv_count ; i++) {
-                        post = ib_verbs_get_post (&device->recvq);
-                        if (ib_verbs_post_recv (device->srq, post) != 0) {
-                                ret = -1;
-                                break;
-                        }
-                }
-        }
-
-        if (ret)
-                ib_verbs_destroy_posts (this);
-
-        return ret;
-}
-
-
-static int32_t
-ib_verbs_connect_qp (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        struct ibv_qp_attr attr = {
-                .qp_state               = IBV_QPS_RTR,
-                .path_mtu               = options->mtu,
-                .dest_qp_num            = priv->peer.remote_qpn,
-                .rq_psn                 = priv->peer.remote_psn,
-                .max_dest_rd_atomic     = 1,
-                .min_rnr_timer          = 12,
-                .ah_attr                = {
-                        .is_global      = 0,
-                        .dlid           = priv->peer.remote_lid,
-                        .sl             = 0,
-                        .src_path_bits  = 0,
-                        .port_num       = options->port
-                }
-        };
-        if (ibv_modify_qp (priv->peer.qp, &attr,
-                           IBV_QP_STATE              |
-                           IBV_QP_AV                 |
-                           IBV_QP_PATH_MTU           |
-                           IBV_QP_DEST_QPN           |
-                           IBV_QP_RQ_PSN             |
-                           IBV_QP_MAX_DEST_RD_ATOMIC |
-                           IBV_QP_MIN_RNR_TIMER)) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_CRITICAL,
-                        "Failed to modify QP to RTR\n");
-                return -1;
-        }
-
-        /* TODO: make timeout and retry_cnt configurable from options */
-        attr.qp_state       = IBV_QPS_RTS;
-        attr.timeout        = 14;
-        attr.retry_cnt      = 7;
-        attr.rnr_retry      = 7;
-        attr.sq_psn         = priv->peer.local_psn;
-        attr.max_rd_atomic  = 1;
-        if (ibv_modify_qp (priv->peer.qp, &attr,
-                           IBV_QP_STATE              |
-                           IBV_QP_TIMEOUT            |
-                           IBV_QP_RETRY_CNT          |
-                           IBV_QP_RNR_RETRY          |
-                           IBV_QP_SQ_PSN             |
-                           IBV_QP_MAX_QP_RD_ATOMIC)) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_CRITICAL,
-                        "Failed to modify QP to RTS\n");
-                return -1;
-        }
-
-        return 0;
-}
-
-static int32_t
-__ib_verbs_teardown (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-
-        __ib_verbs_destroy_qp (this);
-
-        if (!list_empty (&priv->peer.ioq)) {
-                __ib_verbs_ioq_flush (&priv->peer);
-        }
-
-        /* TODO: decrement cq size */
-        return 0;
-}
-
-/*
- * return value:
- *   0 = success (completed)
- *  -1 = error
- * > 0 = incomplete
- */
-
-static int
-__tcp_rwv (transport_t *this, struct iovec *vector, int count,
-           struct iovec **pending_vector, int *pending_count,
-           int write)
-{
-        ib_verbs_private_t *priv = NULL;
-        int sock = -1;
-        int ret = -1;
-        struct iovec *opvector = vector;
-        int opcount = count;
-        int moved = 0;
-
-        priv = this->private;
-        sock = priv->sock;
-
-        while (opcount)
-        {
-                if (write)
-                {
-                        ret = writev (sock, opvector, opcount);
-
-                        if (ret == 0 || (ret == -1 && errno == EAGAIN))
-                        {
-                                /* done for now */
-                                break;
-                        }
-                }
-                else
-                {
-                        ret = readv (sock, opvector, opcount);
-
-                        if (ret == -1 && errno == EAGAIN)
-                        {
-                                /* done for now */
-                                break;
-                        }
-                }
-
-                if (ret == 0)
-                {
-                        gf_log (this->xl->name, GF_LOG_DEBUG, 
-                                "EOF from peer %s", this->peerinfo.identifier);
-                        opcount = -1;
-                        errno = ENOTCONN;
-                        break;
-                }
-
-                if (ret == -1)
-                {
-                        if (errno == EINTR)
-                                continue;
-
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "%s failed (%s)", write ? "writev" : "readv",
-                                strerror (errno));
-                        if (write && !priv->connected && 
-                            (errno == ECONNREFUSED))
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "possible mismatch of 'transport-type'"
-                                        " in protocol server and client. "
-                                        "check volume file");
-                        opcount = -1;
-                        break;
-                }
-
-                moved = 0;
-
-                while (moved < ret)
-                {
-                        if ((ret - moved) >= opvector[0].iov_len)
-                        {
-                                moved += opvector[0].iov_len;
-                                opvector++;
-                                opcount--;
-                        }
-                        else
-                        {
-                                opvector[0].iov_len -= (ret - moved);
-                                opvector[0].iov_base += (ret - moved);
-                                moved += (ret - moved);
-                        }
-                        while (opcount && !opvector[0].iov_len)
-                        {
-                                opvector++;
-                                opcount--;
-                        }
-                }
-        }
-
-        if (pending_vector)
-                *pending_vector = opvector;
-
-        if (pending_count)
-                *pending_count = opcount;
-
-        return opcount;
-}
-
-
-static int
-__tcp_readv (transport_t *this, struct iovec *vector, int count,
-             struct iovec **pending_vector, int *pending_count)
-{
-        int ret = -1;
-
-        ret = __tcp_rwv (this, vector, count, 
-                         pending_vector, pending_count, 0);
-
-        return ret;
-}
-
-
-static int
-__tcp_writev (transport_t *this, struct iovec *vector, int count,
-              struct iovec **pending_vector, int *pending_count)
-{
-        int ret = -1;
-        ib_verbs_private_t *priv = this->private;
-
-        ret = __tcp_rwv (this, vector, count, pending_vector, 
-                         pending_count, 1);
-
-        if (ret > 0) {
-                /* TODO: Avoid multiple calls when socket is already 
-                   registered for POLLOUT */
-                priv->idx = event_select_on (this->xl->ctx->event_pool, 
-                                             priv->sock, priv->idx, -1, 1);
-        } else if (ret == 0) {
-                priv->idx = event_select_on (this->xl->ctx->event_pool, 
-                                             priv->sock,
-                                             priv->idx, -1, 0);
-        }
-
-        return ret;
-}
-
-
-static void *
-ib_verbs_recv_completion_proc (void *data)
-{
-        struct ibv_comp_channel *chan = data;
-        ib_verbs_private_t      *priv = NULL;
-        ib_verbs_device_t       *device;
-        ib_verbs_post_t         *post;
-        ib_verbs_peer_t         *peer;
-        struct ibv_cq           *event_cq;
-        struct ibv_wc            wc;
-        void                    *event_ctx;
-        int32_t                  ret  = 0;
-
-
-        while (1) {
-                ret = ibv_get_cq_event (chan, &event_cq, &event_ctx);
-                if (ret) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "ibv_get_cq_event failed, terminating recv "
-                                "thread %d (%d)", ret, errno);
-                        continue;
-                }
-
-                device = event_ctx;
-    
-                ret = ibv_req_notify_cq (event_cq, 0);
-                if (ret) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "ibv_req_notify_cq on %s failed, terminating "
-                                "recv thread: %d (%d)",
-                                device->device_name, ret, errno);
-                        continue;
-                }
-
-                device = (ib_verbs_device_t *) event_ctx;
-
-                while ((ret = ibv_poll_cq (event_cq, 1, &wc)) > 0) {
-                        post = (ib_verbs_post_t *) (long) wc.wr_id;
-
-                        pthread_mutex_lock (&device->qpreg.lock);
-                        {
-                                peer = __ib_verbs_lookup_peer (device,
-                                                               wc.qp_num);
-
-                                /*
-                                 * keep a refcount on transport so that it
-                                 * doesnot get freed because of some error
-                                 * indicated by wc.status till we are done
-                                 * with usage of peer and thereby that of trans.
-                                 */
-                                if (peer != NULL) {
-                                        transport_ref (peer->trans);
-                                }
-                        }
-                        pthread_mutex_unlock (&device->qpreg.lock);
-
-                        if (wc.status != IBV_WC_SUCCESS) {
-                                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                        "recv work request on `%s' returned "
-                                        "error (%d)",
-                                        device->device_name,
-                                        wc.status);
-                                if (peer) {
-                                        transport_unref (peer->trans);
-                                        transport_disconnect (peer->trans);
-                                }
-
-                                if (post) {
-                                        ib_verbs_post_recv (device->srq, post);
-                                }
-                                continue;
-                        }
-
-                        if (peer) {
-                                priv = peer->trans->private;
-        
-                                pthread_mutex_lock (&priv->recv_mutex);
-                                {
-                                        while (priv->data_ptr)
-                                                pthread_cond_wait (&priv->recv_cond,
-                                                                   &priv->recv_mutex);
-          
-                                        priv->data_ptr = post->buf;
-                                        priv->data_offset = 0;
-                                        priv->data_len = wc.byte_len;
-          
-                                        /*pthread_cond_broadcast (&priv->recv_cond);*/
-                                }
-                                pthread_mutex_unlock (&priv->recv_mutex);
-        
-                                if ((ret = xlator_notify (peer->trans->xl, GF_EVENT_POLLIN, 
-                                                          peer->trans, NULL)) == -1) {
-                                        gf_log ("transport/ib-verbs",
-                                                GF_LOG_DEBUG, 
-                                                "pollin notification to %s "
-                                                "failed, disconnecting "
-                                                "transport", 
-                                                peer->trans->xl->name);
-                                        transport_disconnect (peer->trans);
-                                }
-
-                                transport_unref (peer->trans);
-                        } else {
-                                gf_log ("transport/ib-verbs",
-                                        GF_LOG_DEBUG,
-                                        "could not lookup peer for qp_num: %d",
-                                        wc.qp_num);
-                        }
-                        ib_verbs_post_recv (device->srq, post);
-                }
-    
-                if (ret < 0) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_ERROR,
-                                "ibv_poll_cq on `%s' returned error "
-                                "(ret = %d, errno = %d)",
-                                device->device_name, ret, errno);
-                        continue;
-                }
-                ibv_ack_cq_events (event_cq, 1);
-        }
-        return NULL;
-}
-
-
-static void *
-ib_verbs_send_completion_proc (void *data)
-{
-        struct ibv_comp_channel *chan = data;
-        ib_verbs_post_t *post;
-        ib_verbs_peer_t *peer;
-        struct ibv_cq *event_cq;
-        void *event_ctx;
-        ib_verbs_device_t *device;
-        struct ibv_wc wc;
-        int32_t ret;
-        
-        while (1) {
-                ret = ibv_get_cq_event (chan, &event_cq, &event_ctx);
-                if (ret) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "ibv_get_cq_event on failed, terminating "
-                                "send thread: %d (%d)", ret, errno);
-                        continue;
-                }
-       
-                device = event_ctx;
-
-                ret = ibv_req_notify_cq (event_cq, 0);
-                if (ret) {
-                        gf_log ("transport/ib-verbs",  GF_LOG_ERROR,
-                                "ibv_req_notify_cq on %s failed, terminating "
-                                "send thread: %d (%d)",
-                                device->device_name, ret, errno);
-                        continue;
-                }
-
-                while ((ret = ibv_poll_cq (event_cq, 1, &wc)) > 0) {
-                        post = (ib_verbs_post_t *) (long) wc.wr_id;
-
-                        pthread_mutex_lock (&device->qpreg.lock);
-                        {
-                                peer = __ib_verbs_lookup_peer (device,
-                                                               wc.qp_num);
-
-                                /*
-                                 * keep a refcount on transport so that it
-                                 * doesnot get freed because of some error
-                                 * indicated by wc.status till we are done
-                                 * with usage of peer and thereby that of trans.
-                                 */
-                                if (peer != NULL) {
-                                        transport_ref (peer->trans);
-                                }
-                        }
-                        pthread_mutex_unlock (&device->qpreg.lock);
-
-                        if (wc.status != IBV_WC_SUCCESS) {
-                                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                        "send work request on `%s' returned "
-                                        "error wc.status = %d, wc.vendor_err "
-                                        "= %d, post->buf = %p, wc.byte_len = "
-                                        "%d, post->reused = %d",
-                                        device->device_name, wc.status, 
-                                        wc.vendor_err,
-                                        post->buf, wc.byte_len, post->reused);
-                                if (wc.status == IBV_WC_RETRY_EXC_ERR)
-                                        gf_log ("ib-verbs", GF_LOG_ERROR,
-                                                "connection between client and"
-                                                " server not working. check by"
-                                                " running 'ibv_srq_pingpong'. "
-                                                "also make sure subnet manager"
-                                                " is running (eg: 'opensm'), "
-                                                "or check if ib-verbs port is "
-                                                "valid (or active) by running "
-                                                " 'ibv_devinfo'. contact "
-                                                "Gluster Support Team if "
-                                                "the problem persists.");
-                                if (peer)
-                                        transport_disconnect (peer->trans);
-                        }
-
-                        if (post) {
-                                ib_verbs_put_post (&device->sendq, post);
-                        }
-      
-                        if (peer) {
-                                int quota_ret = ib_verbs_quota_put (peer);
-                                if (quota_ret < 0) {
-                                        gf_log ("ib-verbs", GF_LOG_DEBUG,
-                                                "failed to send message");
-                                        
-                                }
-
-                                transport_unref (peer->trans);
-                        } else {
-                                gf_log ("transport/ib-verbs", GF_LOG_DEBUG,
-                                        "could not lookup peer for qp_num: %d",
-                                        wc.qp_num);
-                        }
-                }
-
-                if (ret < 0) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "ibv_poll_cq on `%s' returned error (ret = %d,"
-                                " errno = %d)",
-                                device->device_name, ret, errno);
-                        continue;
-                }
-                ibv_ack_cq_events (event_cq, 1); 
-        }
-
-        return NULL;
-}
-
-static void
-ib_verbs_options_init (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        int32_t mtu;
-        data_t *temp;
-
-        /* TODO: validate arguments from options below */
-
-        options->send_size = this->xl->ctx->page_size * 4; /* 512 KB */
-        options->recv_size = this->xl->ctx->page_size * 4; /* 512 KB */
-        options->send_count = 32;
-        options->recv_count = 32;
-
-        temp = dict_get (this->xl->options,
-                         "transport.ib-verbs.work-request-send-count");
-        if (temp)
-                options->send_count = data_to_int32 (temp);
-
-        temp = dict_get (this->xl->options,
-                         "transport.ib-verbs.work-request-recv-count");
-        if (temp)
-                options->recv_count = data_to_int32 (temp);
-
-        options->port = 0;
-        temp = dict_get (this->xl->options,
-                         "transport.ib-verbs.port");
-        if (temp)
-                options->port = data_to_uint64 (temp);
-
-        options->mtu = mtu = IBV_MTU_2048;
-        temp = dict_get (this->xl->options,
-                         "transport.ib-verbs.mtu");
-        if (temp)
-                mtu = data_to_int32 (temp);
-        switch (mtu) {
-        case 256: options->mtu = IBV_MTU_256;
-                break;
-        case 512: options->mtu = IBV_MTU_512;
-                break;
-        case 1024: options->mtu = IBV_MTU_1024;
-                break;
-        case 2048: options->mtu = IBV_MTU_2048;
-                break;
-        case 4096: options->mtu = IBV_MTU_4096;
-                break;
-        default:
-                if (temp)
-                        gf_log ("transport/ib-verbs", GF_LOG_WARNING,
-                                "%s: unrecognized MTU value '%s', defaulting "
-                                "to '2048'", this->xl->name,
-                                data_to_str (temp));
-                else
-                        gf_log ("transport/ib-verbs", GF_LOG_TRACE,
-                                "%s: defaulting MTU to '2048'",
-                                this->xl->name);
-                options->mtu = IBV_MTU_2048;
-                break;
-        }
-
-        temp = dict_get (this->xl->options,
-                         "transport.ib-verbs.device-name");
-        if (temp)
-                options->device_name = gf_strdup (temp->data);
-
-        return;
-}
-
-static void
-ib_verbs_queue_init (ib_verbs_queue_t *queue)
-{
-        pthread_mutex_init (&queue->lock, NULL);
-
-        queue->active_posts.next = &queue->active_posts;
-        queue->active_posts.prev = &queue->active_posts;
-        queue->passive_posts.next = &queue->passive_posts;
-        queue->passive_posts.prev = &queue->passive_posts;
-}
-
-
-static ib_verbs_device_t *
-ib_verbs_get_device (transport_t *this,
-		     struct ibv_context *ibctx)
-{
-        glusterfs_ctx_t *ctx        = this->xl->ctx;
-        ib_verbs_private_t *priv    = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        char *device_name           = priv->options.device_name;
-        uint32_t port               = priv->options.port;
-
-        uint8_t active_port = 0;
-        int32_t ret         = 0;
-        int32_t i           = 0;
-
-        ib_verbs_device_t *trav;
-
-        trav = ctx->ib;
-        while (trav) {
-                if ((!strcmp (trav->device_name, device_name)) && 
-                    (trav->port == port))
-                        break;
-                trav = trav->next;
-        }
-
-        if (!trav) {
-
-                trav = GF_CALLOC (1, sizeof (*trav), 
-                                  gf_ibv_mt_ib_verbs_device_t);
-                if (!trav)
-                        return NULL;
-                priv->device = trav;
-
-                trav->context = ibctx;
-
-		ret = ib_get_active_port (trav->context);
-
-		if (ret < 0) {
-			if (!port) {
-				gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-					"Failed to find any active ports and "
-					"none specified in volume file,"
-                                        " exiting");
-				return NULL;
-			}
-		}
-
-		active_port = ret;
-
-                if (port) {
-                        ret = ib_check_active_port (trav->context, port);
-                        if (ret < 0) {
-                                gf_log ("transport/ib-verbs", GF_LOG_WARNING,
-                                        "On device %s: provided port:%u is "
-                                        "found to be offline, continuing to "
-                                        "use the same port", device_name, port);
-                        }
-		} else {
-			priv->options.port = active_port;
-			port = active_port;
-			gf_log ("transport/ib-verbs", GF_LOG_TRACE,
-				"Port unspecified in volume file using active "
-                                "port: %u", port);
-                }
-
-                trav->device_name = gf_strdup (device_name);
-                trav->port = port;
-
-                trav->next = ctx->ib;
-                ctx->ib = trav;
-
-                trav->send_chan = ibv_create_comp_channel (trav->context);
-                if (!trav->send_chan) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: could not create send completion channel",
-                                device_name);
-                        /* TODO: cleanup current mess */
-                        return NULL;
-                }
-    
-                trav->recv_chan = ibv_create_comp_channel (trav->context);
-                if (!trav->recv_chan) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "could not create recv completion channel");
-                        /* TODO: cleanup current mess */
-                        return NULL;
-                }
-      
-                if (ib_verbs_create_cq (this) < 0) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: could not create CQ",
-                                this->xl->name);
-                        return NULL;
-                }
-
-                /* protection domain */
-                trav->pd = ibv_alloc_pd (trav->context);
-
-                if (!trav->pd) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: could not allocate protection domain",
-                                this->xl->name);
-                        return NULL;
-                }
-
-                struct ibv_srq_init_attr attr = {
-                        .attr = {
-                                .max_wr = options->recv_count,
-                                .max_sge = 1
-                        }
-                };
-                trav->srq = ibv_create_srq (trav->pd, &attr);
-
-                if (!trav->srq) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: could not create SRQ",
-                                this->xl->name);
-                        return NULL;
-                }
-
-                /* queue init */
-                ib_verbs_queue_init (&trav->sendq);
-                ib_verbs_queue_init (&trav->recvq);
-
-                if (ib_verbs_create_posts (this) < 0) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: could not allocate posts",
-                                this->xl->name);
-                        return NULL;
-                }
-
-                /* completion threads */
-                ret = pthread_create (&trav->send_thread,
-                                      NULL,
-                                      ib_verbs_send_completion_proc,
-                                      trav->send_chan);
-                if (ret) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "could not create send completion thread");
-                        return NULL;
-                }
-                ret = pthread_create (&trav->recv_thread,
-                                      NULL,
-                                      ib_verbs_recv_completion_proc,
-                                      trav->recv_chan);
-                if (ret) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "could not create recv completion thread");
-                        return NULL;
-                }
-  
-                /* qpreg */
-                pthread_mutex_init (&trav->qpreg.lock, NULL);
-                for (i=0; i<42; i++) {
-                        trav->qpreg.ents[i].next = &trav->qpreg.ents[i];
-                        trav->qpreg.ents[i].prev = &trav->qpreg.ents[i];
-                }
-        }
-        return trav;
-}
-
-static int32_t 
-ib_verbs_init (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        struct ibv_device **dev_list;
-	struct ibv_context *ib_ctx = NULL;
-	int32_t ret = 0;
-
-        ib_verbs_options_init (this);
-
-        {
-                dev_list = ibv_get_device_list (NULL);
-
-		if (!dev_list) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_CRITICAL,
-                                "Failed to get IB devices");
-			ret = -1;
-			goto cleanup;
-                }
-
-                if (!*dev_list) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_CRITICAL,
-                                "No IB devices found");
-			ret = -1;
-                        goto cleanup;
-                }
-
-                if (!options->device_name) {
-                        if (*dev_list) {
-                                options->device_name = 
-                                        gf_strdup (ibv_get_device_name (*dev_list));
-                        } else {
-                                gf_log ("transport/ib-verbs", GF_LOG_CRITICAL,
-                                        "IB device list is empty. Check for "
-                                        "'ib_uverbs' module");
-                                return -1;
-                                goto cleanup;
-                        }
-                }
-
-		while (*dev_list) {
-                        if (!strcmp (ibv_get_device_name (*dev_list),
-                                     options->device_name)) {
-                                ib_ctx = ibv_open_device (*dev_list);
-
-                                if (!ib_ctx) {
-                                        gf_log ("transport/ib-verbs",
-                                                GF_LOG_ERROR,
-                                                "Failed to get infiniband"
-                                                "device context");
-                                        ret = -1;
-                                        goto cleanup;
-                                }
-                                break;
-                        }
-			++dev_list;
-		}
-
-		priv->device = ib_verbs_get_device (this, ib_ctx);
-
-                if (!priv->device) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "could not create ib_verbs device for %s",
-                                options->device_name);
-                        ret = -1;
-			goto cleanup;
-                }
-        }
-
-        priv->peer.trans = this;
-        INIT_LIST_HEAD (&priv->peer.ioq);
-  
-        pthread_mutex_init (&priv->read_mutex, NULL);
-        pthread_mutex_init (&priv->write_mutex, NULL);
-        pthread_mutex_init (&priv->recv_mutex, NULL);
-        pthread_cond_init (&priv->recv_cond, NULL);
-
-cleanup:
-	if (-1 == ret) {
-		if (ib_ctx)
-			ibv_close_device (ib_ctx);
-	}
-
-	if (dev_list)
-		ibv_free_device_list (dev_list);
-
-	return ret;
-}
-
-
-static int32_t
-ib_verbs_disconnect (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        int32_t ret = 0;
- 
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                ret = __ib_verbs_disconnect (this);
-        }
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        return ret;
-}
-
-
-static int32_t
-__tcp_connect_finish (int fd)
-{
-        int ret = -1;
-        int optval = 0;
-        socklen_t optlen = sizeof (int);
-
-        ret = getsockopt (fd, SOL_SOCKET, SO_ERROR,
-                          (void *)&optval, &optlen);
-
-        if (ret == 0 && optval)
-        {
-                errno = optval;
-                ret = -1;
-        }
-
-        return ret;
-}
-
-static inline void
-ib_verbs_fill_handshake_data (char *buf, struct ib_verbs_nbio *nbio, 
-                              ib_verbs_private_t *priv)
-{
-        sprintf (buf,
-                 "QP1:RECV_BLKSIZE=%08x:SEND_BLKSIZE=%08x\n"
-                 "QP1:LID=%04x:QPN=%06x:PSN=%06x\n",
-                 priv->peer.recv_size,
-                 priv->peer.send_size,
-                 priv->peer.local_lid,
-                 priv->peer.local_qpn,
-                 priv->peer.local_psn);
-
-        nbio->vector.iov_base = buf;
-        nbio->vector.iov_len = strlen (buf) + 1;
-        nbio->count = 1;
-        return;
-}
-
-static inline void
-ib_verbs_fill_handshake_ack (char *buf, struct ib_verbs_nbio *nbio)
-{
-        sprintf (buf, "DONE\n");
-        nbio->vector.iov_base = buf;
-        nbio->vector.iov_len = strlen (buf) + 1;
-        nbio->count = 1;
-        return;
-}
-
-static int
-ib_verbs_handshake_pollin (transport_t *this)
-{
-        int ret = 0;
-        ib_verbs_private_t *priv = this->private;
-        char *buf = priv->handshake.incoming.buf;
-        int32_t recv_buf_size, send_buf_size;
-        socklen_t sock_len;
-
-        if (priv->handshake.incoming.state == IB_VERBS_HANDSHAKE_COMPLETE) {
-                return -1;
-        }
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                while (priv->handshake.incoming.state != IB_VERBS_HANDSHAKE_COMPLETE)
-                {
-                        switch (priv->handshake.incoming.state) 
-                        {
-                        case IB_VERBS_HANDSHAKE_START:
-                                buf = priv->handshake.incoming.buf = GF_CALLOC (1, 256, gf_ibv_mt_char);
-                                ib_verbs_fill_handshake_data (buf, &priv->handshake.incoming, priv);
-                                buf[0] = 0;
-                                priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_RECEIVING_DATA;
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_RECEIVING_DATA:
-                                ret = __tcp_readv (this, 
-                                                   &priv->handshake.incoming.vector, 
-                                                   priv->handshake.incoming.count,
-                                                   &priv->handshake.incoming.pending_vector, 
-                                                   &priv->handshake.incoming.pending_count);
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                                "partial header read on NB socket. continue later");
-                                        goto unlock;
-                                }
-            
-                                if (!ret) {
-                                        priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_RECEIVED_DATA;
-                                }
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_RECEIVED_DATA:
-                                ret = sscanf (buf,
-                                              "QP1:RECV_BLKSIZE=%08x:SEND_BLKSIZE=%08x\n"
-                                              "QP1:LID=%04x:QPN=%06x:PSN=%06x\n",
-                                              &recv_buf_size,
-                                              &send_buf_size,
-                                              &priv->peer.remote_lid,
-                                              &priv->peer.remote_qpn,
-                                              &priv->peer.remote_psn);
-
-                                if ((ret != 5) && (strncmp (buf, "QP1:", 4))) {
-                                        gf_log ("transport/ib-verbs", 
-                                                GF_LOG_CRITICAL,
-                                                "%s: remote-host(%s)'s "
-                                                "transport type is different",
-                                                this->xl->name, 
-                                                this->peerinfo.identifier);
-                                        ret = -1;
-                                        goto unlock;
-                                }
-
-                                if (recv_buf_size < priv->peer.recv_size)
-                                        priv->peer.recv_size = recv_buf_size;
-                                if (send_buf_size < priv->peer.send_size)
-                                        priv->peer.send_size = send_buf_size;
-          
-                                gf_log ("transport/ib-verbs", GF_LOG_TRACE,
-                                        "%s: transacted recv_size=%d "
-                                        "send_size=%d",
-                                        this->xl->name, priv->peer.recv_size,
-                                        priv->peer.send_size);
-
-                                priv->peer.quota = priv->peer.send_count;
-
-                                if (ib_verbs_connect_qp (this)) {
-                                        gf_log ("transport/ib-verbs", 
-                                                GF_LOG_ERROR,
-                                                "%s: failed to connect with "
-                                                "remote QP", this->xl->name);
-                                        ret = -1;
-                                        goto unlock;
-                                }
-                                ib_verbs_fill_handshake_ack (buf, &priv->handshake.incoming);
-                                buf[0] = 0;
-                                priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_RECEIVING_ACK;
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_RECEIVING_ACK:
-                                ret = __tcp_readv (this, 
-                                                   &priv->handshake.incoming.vector, 
-                                                   priv->handshake.incoming.count,
-                                                   &priv->handshake.incoming.pending_vector, 
-                                                   &priv->handshake.incoming.pending_count);
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                                "partial header read on NB "
-                                                "socket. continue later");
-                                        goto unlock;
-                                }
-            
-                                if (!ret) {
-                                        priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_RECEIVED_ACK;
-                                }
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_RECEIVED_ACK:
-                                if (strncmp (buf, "DONE", 4)) {
-                                        gf_log ("transport/ib-verbs", 
-                                                GF_LOG_DEBUG,
-                                                "%s: handshake-3 did not "
-                                                "return 'DONE' (%s)",
-                                                this->xl->name, buf);
-                                        ret = -1;
-                                        goto unlock;
-                                }
-                                ret = 0;
-                                priv->connected = 1;
-                                sock_len = sizeof (struct sockaddr_storage);
-                                getpeername (priv->sock,
-                                             (struct sockaddr *) &this->peerinfo.sockaddr,
-                                             &sock_len);
-
-                                GF_FREE (priv->handshake.incoming.buf);
-                                priv->handshake.incoming.buf = NULL;
-                                priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_COMPLETE;
-                        }
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        if (ret == -1) {
-                transport_disconnect (this);
-        } else {
-                ret = 0;
-        }
-
-        if (!ret && priv->connected) {
-                ret = xlator_notify (this->xl, GF_EVENT_CHILD_UP, this);
-        }
-
-        return ret;
-}
-
-static int 
-ib_verbs_handshake_pollout (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        char *buf = priv->handshake.outgoing.buf;
-        int32_t ret = 0;
-
-        if (priv->handshake.outgoing.state == IB_VERBS_HANDSHAKE_COMPLETE) {
-                return 0;
-        }
-
-        pthread_mutex_unlock (&priv->write_mutex);
-        {
-                while (priv->handshake.outgoing.state != IB_VERBS_HANDSHAKE_COMPLETE)
-                {
-                        switch (priv->handshake.outgoing.state) 
-                        {
-                        case IB_VERBS_HANDSHAKE_START:
-                                buf = priv->handshake.outgoing.buf = GF_CALLOC (1, 256, gf_ibv_mt_char);
-                                ib_verbs_fill_handshake_data (buf, &priv->handshake.outgoing, priv);
-                                priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_SENDING_DATA;
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_SENDING_DATA:
-                                ret = __tcp_writev (this, 
-                                                    &priv->handshake.outgoing.vector, 
-                                                    priv->handshake.outgoing.count,
-                                                    &priv->handshake.outgoing.pending_vector, 
-                                                    &priv->handshake.outgoing.pending_count);
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                                "partial header read on NB socket. continue later");
-                                        goto unlock;
-                                }
-            
-                                if (!ret) {
-                                        priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_SENT_DATA;
-                                }
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_SENT_DATA:
-                                ib_verbs_fill_handshake_ack (buf, &priv->handshake.outgoing);
-                                priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_SENDING_ACK;
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_SENDING_ACK:
-                                ret = __tcp_writev (this,
-                                                    &priv->handshake.outgoing.vector,
-                                                    priv->handshake.outgoing.count,
-                                                    &priv->handshake.outgoing.pending_vector,
-                                                    &priv->handshake.outgoing.pending_count);
-
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                                "partial header read on NB "
-                                                "socket. continue later");
-                                        goto unlock;
-                                }
-            
-                                if (!ret) {
-                                        GF_FREE (priv->handshake.outgoing.buf);
-                                        priv->handshake.outgoing.buf = NULL;
-                                        priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_COMPLETE;
-                                }
-                                break;
-                        }
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        if (ret == -1) {
-                transport_disconnect (this);
-        } else {
-                ret = 0;
-        }
-
-        return ret;
-}
-
-static int
-ib_verbs_handshake_pollerr (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        int32_t ret = 0;
-        char need_unref = 0;
-
-        gf_log ("transport/ib-verbs", GF_LOG_DEBUG,
-                "%s: peer disconnected, cleaning up",
-                this->xl->name);
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                __ib_verbs_teardown (this);
-
-                if (priv->sock != -1) {
-                        event_unregister (this->xl->ctx->event_pool, 
-                                          priv->sock, priv->idx);
-                        need_unref = 1;
-
-                        if (close (priv->sock) != 0) {
-                                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                        "close () - error: %s",
-                                        strerror (errno));
-                                ret = -errno;
-                        }
-                        priv->tcp_connected = priv->connected = 0;
-                        priv->sock = -1;
-                }
-
-                if (priv->handshake.incoming.buf) {
-                        GF_FREE (priv->handshake.incoming.buf);
-                        priv->handshake.incoming.buf = NULL;
-                }
-
-                priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_START;
-
-                if (priv->handshake.outgoing.buf) {
-                        GF_FREE (priv->handshake.outgoing.buf);
-                        priv->handshake.outgoing.buf = NULL;
-                }
-
-                priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_START;
-        }
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        xlator_notify (this->xl, GF_EVENT_POLLERR, this, NULL);
-
-        if (need_unref)
-                transport_unref (this);
-
-        return 0;
-}
-
-
-static int
-tcp_connect_finish (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        int error = 0, ret = 0;
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                ret = __tcp_connect_finish (priv->sock);
-
-                if (!ret) {
-                        this->myinfo.sockaddr_len = 
-                                sizeof (this->myinfo.sockaddr);
-                        ret = getsockname (priv->sock,
-                                           (struct sockaddr *)&this->myinfo.sockaddr, 
-                                           &this->myinfo.sockaddr_len);
-                        if (ret == -1) 
-                        {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "getsockname on new client-socket %d "
-                                        "failed (%s)", 
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                error = 1;
-                                goto unlock;
-                        }
-
-                        gf_ibverbs_get_transport_identifiers (this);
-                        priv->tcp_connected = 1;
-                }
-
-                if (ret == -1 && errno != EINPROGRESS) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "tcp connect to %s failed (%s)", 
-                                this->peerinfo.identifier, strerror (errno));
-                        error = 1;
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        if (error) {
-                transport_disconnect (this);
-        }
-
-        return ret;
-}
-
-static int
-ib_verbs_event_handler (int fd, int idx, void *data,
-                        int poll_in, int poll_out, int poll_err)
-{
-        transport_t *this = data;
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = NULL;
-        int ret = 0;
-
-        if (!priv->tcp_connected) {
-                ret = tcp_connect_finish (this);
-                if (priv->tcp_connected) {
-                        options = &priv->options;
-
-                        priv->peer.send_count = options->send_count;
-                        priv->peer.recv_count = options->recv_count;
-                        priv->peer.send_size = options->send_size;
-                        priv->peer.recv_size = options->recv_size;
-
-                        if ((ret = ib_verbs_create_qp (this)) < 0) {
-                                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                        "%s: could not create QP",
-                                        this->xl->name);
-                                transport_disconnect (this);
-                        }
-                }
-        }
-
-        if (!ret && poll_out && priv->tcp_connected) {
-                ret = ib_verbs_handshake_pollout (this);
-        }
-
-        if (!ret && poll_in && priv->tcp_connected) {
-                if (priv->handshake.incoming.state == IB_VERBS_HANDSHAKE_COMPLETE) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: pollin received on tcp socket (peer: %s) "
-                                "after handshake is complete",
-                                this->xl->name, this->peerinfo.identifier);
-                        ib_verbs_handshake_pollerr (this);
-                        return 0;
-                }
-                ret = ib_verbs_handshake_pollin (this);
-        }
-
-        if (ret < 0 || poll_err) {
-                ret = ib_verbs_handshake_pollerr (this);
-        }
-
-        return 0;
-}
-
-static int
-__tcp_nonblock (int fd)
-{
-        int flags = 0;
-        int ret = -1;
-
-        flags = fcntl (fd, F_GETFL);
-
-        if (flags != -1)
-                ret = fcntl (fd, F_SETFL, flags | O_NONBLOCK);
-
-        return ret;
-}
-
-static int32_t
-ib_verbs_connect (struct transport *this)
-{
-        dict_t *options = this->xl->options;
-  
-        ib_verbs_private_t *priv = this->private;
-  
-        int32_t ret = 0;
-        gf_boolean_t non_blocking = 1;
-        struct sockaddr_storage sockaddr;
-        socklen_t sockaddr_len = 0;
-
-        if (priv->connected) {
-                return 0;
-        }
-
-        if (dict_get (options, "non-blocking-io")) {
-                char *nb_connect = data_to_str (dict_get (this->xl->options,
-                                                          "non-blocking-io"));
-          
-                if (gf_string2boolean (nb_connect, &non_blocking) == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "'non-blocking-io' takes only boolean "
-                                "options, not taking any action");
-                        non_blocking = 1;
-                }
-        }
-
-        ret = gf_ibverbs_client_get_remote_sockaddr (this,
-                                                     (struct sockaddr *)&sockaddr,
-                                                     &sockaddr_len);
-        if (ret != 0) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "cannot get remote address to connect");
-                return ret;
-        }
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                if (priv->sock != -1) {
-                        ret = 0;
-                        goto unlock;
-                }
-  
-                priv->sock = socket (((struct sockaddr *)&sockaddr)->sa_family,
-                                     SOCK_STREAM, 0);
-        
-                if (priv->sock == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "socket () - error: %s", strerror (errno));
-                        ret = -errno;
-                        goto unlock;
-                }
-
-                gf_log (this->xl->name, GF_LOG_TRACE,
-                        "socket fd = %d", priv->sock);
-
-                memcpy (&this->peerinfo.sockaddr, &sockaddr, sockaddr_len);
-                this->peerinfo.sockaddr_len = sockaddr_len;
-
-                ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = 
-                        ((struct sockaddr *)&this->peerinfo.sockaddr)->sa_family;
-
-                if (non_blocking) 
-                {
-                        ret = __tcp_nonblock (priv->sock);
-        
-                        if (ret == -1)
-                        {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "could not set socket %d to non "
-                                        "blocking mode (%s)",
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                priv->sock = -1;
-                                goto unlock;
-                        }
-                }
-
-                ret = gf_ibverbs_client_bind (this,
-                                              (struct sockaddr *)&this->myinfo.sockaddr,
-                                              &this->myinfo.sockaddr_len, priv->sock);
-                if (ret == -1)
-                {
-                        gf_log (this->xl->name, GF_LOG_WARNING,
-                                "client bind failed: %s", strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                ret = connect (priv->sock, 
-                               (struct sockaddr *)&this->peerinfo.sockaddr, 
-                               this->peerinfo.sockaddr_len);
-                if (ret == -1 && errno != EINPROGRESS)
-                {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "connection attempt failed (%s)", 
-                                strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                priv->tcp_connected = priv->connected = 0;
-
-                transport_ref (this);
-
-                priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_START;
-                priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_START;
-                        
-                priv->idx = event_register (this->xl->ctx->event_pool, 
-                                            priv->sock, ib_verbs_event_handler,
-                                            this, 1, 1); 
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        return ret;
-}
-
-static int
-ib_verbs_server_event_handler (int fd, int idx, void *data,
-                               int poll_in, int poll_out, int poll_err)
-{
-        int32_t main_sock = -1;
-        transport_t *this, *trans = data;
-        ib_verbs_private_t *priv = NULL;
-        ib_verbs_private_t *trans_priv = (ib_verbs_private_t *) trans->private;
-        ib_verbs_options_t *options = NULL;
-
-        if (!poll_in)
-                return 0;
-
-        this = GF_CALLOC (1, sizeof (transport_t),
-                          gf_ibv_mt_transport_t);
-        if (!this)
-                return 0;
-
-        priv = GF_CALLOC (1, sizeof (ib_verbs_private_t),
-                          gf_ibv_mt_ib_verbs_private_t);
-        if (!priv) {
-                GF_FREE (this);
-                return 0;
-        }
-
-        this->private = priv;
-        /* Copy all the ib_verbs related values in priv, from trans_priv 
-           as other than QP, all the values remain same */
-        priv->device = trans_priv->device;
-        priv->options = trans_priv->options;
-        options = &priv->options;
-
-        this->ops = trans->ops;
-        this->xl = trans->xl;
-        this->init = trans->init;
-        this->fini = trans->fini;
-
-        memcpy (&this->myinfo.sockaddr, &trans->myinfo.sockaddr, 
-                trans->myinfo.sockaddr_len);
-        this->myinfo.sockaddr_len = trans->myinfo.sockaddr_len;
-
-        main_sock = (trans_priv)->sock;
-        this->peerinfo.sockaddr_len = sizeof (this->peerinfo.sockaddr);
-        priv->sock = accept (main_sock, 
-                             (struct sockaddr *)&this->peerinfo.sockaddr, 
-                             &this->peerinfo.sockaddr_len);
-        if (priv->sock == -1) {
-                gf_log ("ib-verbs/server", GF_LOG_ERROR,
-                        "accept() failed: %s",
-                        strerror (errno));
-                GF_FREE (this->private);
-                GF_FREE (this);
-                return -1;
-        }
-
-        priv->peer.trans = this;
-        transport_ref (this);
-
-        gf_ibverbs_get_transport_identifiers (this);
-
-        priv->tcp_connected = 1;
-        priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_START;
-        priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_START;
-
-        priv->peer.send_count = options->send_count;
-        priv->peer.recv_count = options->recv_count;
-        priv->peer.send_size = options->send_size;
-        priv->peer.recv_size = options->recv_size;
-        INIT_LIST_HEAD (&priv->peer.ioq);
-
-        if (ib_verbs_create_qp (this) < 0) {
-                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                        "%s: could not create QP",
-                        this->xl->name);
-                transport_disconnect (this);
-                return -1;
-        }
-
-        priv->idx = event_register (this->xl->ctx->event_pool, priv->sock,
-                                    ib_verbs_event_handler, this, 1, 1);
-
-        pthread_mutex_init (&priv->read_mutex, NULL);
-        pthread_mutex_init (&priv->write_mutex, NULL);
-        pthread_mutex_init (&priv->recv_mutex, NULL);
-        /*  pthread_cond_init (&priv->recv_cond, NULL); */
-
-        return 0;
-}
-
-static int32_t
-ib_verbs_listen (transport_t *this)
-{
-        struct sockaddr_storage sockaddr;
-        socklen_t sockaddr_len;
-        ib_verbs_private_t *priv = this->private;
-        int opt = 1, ret = 0;
-        char service[NI_MAXSERV], host[NI_MAXHOST];
-
-        memset (&sockaddr, 0, sizeof (sockaddr));
-        ret = gf_ibverbs_server_get_local_sockaddr (this,
-                                                    (struct sockaddr *)&sockaddr,
-                                                    &sockaddr_len);
-        if (ret != 0) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "cannot find network address of server to bind to");
-                goto err;
-        }
-
-        priv->sock = socket (((struct sockaddr *)&sockaddr)->sa_family, 
-                             SOCK_STREAM, 0);
-        if (priv->sock == -1) {
-                gf_log ("ib-verbs/server", GF_LOG_CRITICAL,
-                        "init: failed to create socket, error: %s",
-                        strerror (errno));
-                GF_FREE (this->private);
-                ret = -1;
-                goto err;
-        }
-
-        memcpy (&this->myinfo.sockaddr, &sockaddr, sockaddr_len);
-        this->myinfo.sockaddr_len = sockaddr_len;
-
-        ret = getnameinfo ((struct sockaddr *)&this->myinfo.sockaddr, 
-                           this->myinfo.sockaddr_len,
-                           host, sizeof (host),
-                           service, sizeof (service),
-                           NI_NUMERICHOST);
-        if (ret != 0) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "getnameinfo failed (%s)", gai_strerror (ret));
-                goto err;
-        }
-        sprintf (this->myinfo.identifier, "%s:%s", host, service);
- 
-        setsockopt (priv->sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof (opt));
-        if (bind (priv->sock,
-                  (struct sockaddr *)&sockaddr,
-                  sockaddr_len) != 0) {
-                ret = -1;
-                gf_log ("ib-verbs/server", GF_LOG_ERROR,
-                        "init: failed to bind to socket for %s (%s)",
-                        this->myinfo.identifier, strerror (errno));
-                goto err;
-        }
-
-        if (listen (priv->sock, 10) != 0) {
-                gf_log ("ib-verbs/server", GF_LOG_ERROR,
-                        "init: listen () failed on socket for %s (%s)",
-                        this->myinfo.identifier, strerror (errno));
-                ret = -1;
-                goto err;
-        }
-
-        /* Register the main socket */
-        priv->idx = event_register (this->xl->ctx->event_pool, priv->sock,
-                                    ib_verbs_server_event_handler, 
-                                    transport_ref (this), 1, 0);
-
-err:
-        return ret;
-}
-
-struct transport_ops tops = {
-        .receive = ib_verbs_receive,
-        .submit = ib_verbs_submit,
-        .connect = ib_verbs_connect,
-        .disconnect = ib_verbs_disconnect,
-        .listen = ib_verbs_listen,
-};
-
-int32_t
-init (transport_t *this)
-{
-        ib_verbs_private_t *priv = GF_CALLOC (1, sizeof (*priv),
-                                              gf_ibv_mt_ib_verbs_private_t);
-        this->private = priv;
-        priv->sock = -1;
-
-        if (ib_verbs_init (this)) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "Failed to initialize IB Device");
-                return -1;
-        }
-
-        return 0;
-}
-
-void  
-fini (struct transport *this)
-{
-        /* TODO: verify this function does graceful finish */
-        ib_verbs_private_t *priv = this->private;
-        this->private = NULL;
-
-        pthread_mutex_destroy (&priv->recv_mutex);
-        pthread_mutex_destroy (&priv->write_mutex);
-        pthread_mutex_destroy (&priv->read_mutex);
-        /*  pthread_cond_destroy (&priv->recv_cond); */
-
-        gf_log (this->xl->name, GF_LOG_TRACE,
-                "called fini on transport: %p",
-                this);
-        GF_FREE (priv);
-        return;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_common_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-/* TODO: expand each option */
-struct volume_options options[] = {
-        { .key   = {"transport.ib-verbs.port",
-                    "ib-verbs-port"}, 
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 4,
-          .description = "check the option by 'ibv_devinfo'"
-        },
-        { .key   = {"transport.ib-verbs.mtu",
-                    "ib-verbs-mtu"}, 
-          .type  = GF_OPTION_TYPE_INT,
-        },
-        { .key   = {"transport.ib-verbs.device-name",
-                    "ib-verbs-device-name"}, 
-          .type  = GF_OPTION_TYPE_ANY,
-          .description = "check by 'ibv_devinfo'"
-        },
-        { .key   = {"transport.ib-verbs.work-request-send-count",
-                    "ib-verbs-work-request-send-count"}, 
-          .type  = GF_OPTION_TYPE_INT,
-        },
-        { .key   = {"transport.ib-verbs.work-request-recv-count",
-                    "ib-verbs-work-request-recv-count"}, 
-          .type  = GF_OPTION_TYPE_INT,
-        },
-        { .key   = {"remote-port", 
-                    "transport.remote-port",
-                    "transport.ib-verbs.remote-port"}, 
-          .type  = GF_OPTION_TYPE_INT 
-        },
-        { .key   = {"transport.ib-verbs.listen-port", "listen-port"}, 
-          .type  = GF_OPTION_TYPE_INT 
-        },
-        { .key   = {"transport.ib-verbs.connect-path", "connect-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = {"transport.ib-verbs.bind-path", "bind-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = {"transport.ib-verbs.listen-path", "listen-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = {"transport.address-family",
-                    "address-family"}, 
-          .value = {"inet", "inet6", "inet/inet6", "inet6/inet",
-                    "unix", "inet-sdp" },
-          .type  = GF_OPTION_TYPE_STR 
-        },
-        { .key   = {"transport.socket.lowlat"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key = {NULL} }
-};
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.h b/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.h
deleted file mode 100644
index 3818609b0..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.h
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _XPORT_IB_VERBS_H
-#define _XPORT_IB_VERBS_H
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef MAX_IOVEC
-#define MAX_IOVEC 16
-#endif /* MAX_IOVEC */
-
-#include "xlator.h"
-#include "event.h"
-#include "ib-verbs-mem-types.h"
-
-#include <stdio.h>
-#include <list.h>
-#include <arpa/inet.h>
-#include <infiniband/verbs.h>
-
-#define GF_DEFAULT_IBVERBS_LISTEN_PORT 6997
-
-/* options per transport end point */
-struct _ib_verbs_options {
-        int32_t port;
-        char *device_name;
-        enum ibv_mtu mtu;
-        int32_t  send_count;
-        int32_t  recv_count;
-        uint64_t recv_size;
-        uint64_t send_size;
-};
-typedef struct _ib_verbs_options ib_verbs_options_t;
-
-
-struct _ib_verbs_header {
-        char     colonO[3];
-        uint32_t size1;
-        uint32_t size2;
-        char     version;
-} __attribute__((packed));
-typedef struct _ib_verbs_header ib_verbs_header_t;
-
-struct _ib_verbs_ioq {
-        union {
-                struct list_head list;
-                struct {
-                        struct _ib_verbs_ioq    *next;
-                        struct _ib_verbs_ioq    *prev;
-                };
-        };
-        ib_verbs_header_t  header;
-        struct iovec       vector[MAX_IOVEC];
-        int                count;
-        char              *buf;
-        struct iobref     *iobref;
-};
-typedef struct _ib_verbs_ioq ib_verbs_ioq_t;
-
-/* represents one communication peer, two per transport_t */
-struct _ib_verbs_peer {
-        transport_t *trans;
-        struct ibv_qp *qp;
-
-        int32_t recv_count;
-        int32_t send_count;
-        int32_t recv_size;
-        int32_t send_size;
-
-        int32_t quota;
-        union {
-                struct list_head     ioq;
-                struct {
-                        ib_verbs_ioq_t        *ioq_next;
-                        ib_verbs_ioq_t        *ioq_prev;
-                };
-        };
-
-        /* QP attributes, needed to connect with remote QP */
-        int32_t local_lid;
-        int32_t local_psn;
-        int32_t local_qpn;
-        int32_t remote_lid;
-        int32_t remote_psn;
-        int32_t remote_qpn;
-};
-typedef struct _ib_verbs_peer ib_verbs_peer_t;
-
-
-struct _ib_verbs_post {
-        struct _ib_verbs_post *next, *prev;
-        struct ibv_mr *mr;
-        char *buf;
-        int32_t buf_size;
-        char aux;
-        int32_t reused;
-        pthread_barrier_t wait;
-};
-typedef struct _ib_verbs_post ib_verbs_post_t;
-
-
-struct _ib_verbs_queue {
-        ib_verbs_post_t active_posts, passive_posts;
-        int32_t active_count, passive_count;
-        pthread_mutex_t lock;
-};
-typedef struct _ib_verbs_queue ib_verbs_queue_t;
-
-
-struct _ib_verbs_qpreg {
-        pthread_mutex_t lock;
-        int32_t count;
-        struct _qpent {
-                struct _qpent *next, *prev;
-                int32_t qp_num;
-                ib_verbs_peer_t *peer;
-        } ents[42];
-};
-typedef struct _ib_verbs_qpreg ib_verbs_qpreg_t;
-
-/* context per device, stored in global glusterfs_ctx_t->ib */
-struct _ib_verbs_device {
-        struct _ib_verbs_device *next;
-        const char *device_name;
-        struct ibv_context *context;
-        int32_t port;
-        struct ibv_pd *pd;
-        struct ibv_srq *srq;
-        ib_verbs_qpreg_t qpreg;
-        struct ibv_comp_channel *send_chan, *recv_chan;
-        struct ibv_cq *send_cq, *recv_cq;
-        ib_verbs_queue_t sendq, recvq;
-        pthread_t send_thread, recv_thread;
-};
-typedef struct _ib_verbs_device ib_verbs_device_t;
-
-typedef enum {
-        IB_VERBS_HANDSHAKE_START = 0,
-        IB_VERBS_HANDSHAKE_SENDING_DATA,
-        IB_VERBS_HANDSHAKE_RECEIVING_DATA,
-        IB_VERBS_HANDSHAKE_SENT_DATA,
-        IB_VERBS_HANDSHAKE_RECEIVED_DATA,
-        IB_VERBS_HANDSHAKE_SENDING_ACK,
-        IB_VERBS_HANDSHAKE_RECEIVING_ACK,
-        IB_VERBS_HANDSHAKE_RECEIVED_ACK,
-        IB_VERBS_HANDSHAKE_COMPLETE,
-} ib_verbs_handshake_state_t;
-
-struct ib_verbs_nbio {
-        int state;
-        char *buf;
-        int count;
-        struct iovec vector;
-        struct iovec *pending_vector;
-        int pending_count;
-};
-
-
-struct _ib_verbs_private {
-        int32_t sock;
-        int32_t idx;
-        unsigned char connected;
-        unsigned char tcp_connected;
-        unsigned char ib_connected;
-        in_addr_t addr;
-        unsigned short port;
-
-        /* IB Verbs Driver specific variables, pointers */
-        ib_verbs_peer_t peer;
-        ib_verbs_device_t *device;
-        ib_verbs_options_t options;
-
-        /* Used by trans->op->receive */
-        char *data_ptr;
-        int32_t data_offset;
-        int32_t data_len;
-
-        /* Mutex */
-        pthread_mutex_t read_mutex;
-        pthread_mutex_t write_mutex;
-        pthread_barrier_t handshake_barrier;
-        char handshake_ret;
-
-        pthread_mutex_t recv_mutex;
-        pthread_cond_t  recv_cond;
-
-        /* used during ib_verbs_handshake */
-        struct {
-                struct ib_verbs_nbio incoming;
-                struct ib_verbs_nbio outgoing;
-                int               state;
-                ib_verbs_header_t header;
-                char *buf;
-                size_t size;
-        } handshake;
-};
-typedef struct _ib_verbs_private ib_verbs_private_t;
-
-#endif /* _XPORT_IB_VERBS_H */
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/name.c b/xlators/protocol/legacy/transport/ib-verbs/src/name.c
deleted file mode 100644
index 511a33a22..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/name.c
+++ /dev/null
@@ -1,712 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <errno.h>
-#include <netdb.h>
-#include <string.h>
-
-#ifdef CLIENT_PORT_CEILING
-#undef CLIENT_PORT_CEILING
-#endif
-
-#define CLIENT_PORT_CEILING 1024
-
-#ifndef AF_INET_SDP
-#define AF_INET_SDP 27
-#endif
-
-#include "transport.h"
-#include "ib-verbs.h"
-
-int32_t
-gf_resolve_ip6 (const char *hostname, 
-                uint16_t port, 
-                int family, 
-                void **dnscache, 
-                struct addrinfo **addr_info);
-
-static int32_t
-af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr, 
-                                 socklen_t sockaddr_len, int ceiling)
-{
-        int32_t ret = -1;
-        /*  struct sockaddr_in sin = {0, }; */
-        uint16_t port = ceiling - 1;
-
-        while (port)
-        {
-                switch (sockaddr->sa_family)
-                {
-                case AF_INET6:
-                        ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
-                        break;
-
-                case AF_INET_SDP:
-                case AF_INET:
-                        ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
-                        break;
-                }
-
-                ret = bind (fd, sockaddr, sockaddr_len);
-
-                if (ret == 0)
-                        break;
-
-                if (ret == -1 && errno == EACCES)
-                        break;
-
-                port--;
-        }
-
-        return ret;
-}
-
-static int32_t
-af_unix_client_bind (transport_t *this, 
-                     struct sockaddr *sockaddr, 
-                     socklen_t sockaddr_len, 
-                     int sock)
-{
-        data_t *path_data = NULL;
-        struct sockaddr_un *addr = NULL;
-        int32_t ret = -1;
-
-        path_data = dict_get (this->xl->options, 
-                              "transport.ib-verbs.bind-path");
-        if (path_data) {
-                char *path = data_to_str (path_data);
-                if (!path || strlen (path) > UNIX_PATH_MAX) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "transport.ib-verbs.bind-path not specfied "
-                                "for unix socket, letting connect to assign "
-                                "default value");
-                        goto err;
-                }
-
-                addr = (struct sockaddr_un *) sockaddr;
-                strcpy (addr->sun_path, path);
-                ret = bind (sock, (struct sockaddr *)addr, sockaddr_len);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "cannot bind to unix-domain socket %d (%s)", 
-                                sock, strerror (errno));
-                        goto err;
-                }
-        }
-
-err:
-        return ret;
-}
-
-static int32_t
-client_fill_address_family (transport_t *this, struct sockaddr *sockaddr)
-{
-        data_t *address_family_data = NULL;
-
-        address_family_data = dict_get (this->xl->options, 
-                                        "transport.address-family");
-        if (!address_family_data) {
-                data_t *remote_host_data = NULL, *connect_path_data = NULL;
-                remote_host_data = dict_get (this->xl->options, "remote-host");
-                connect_path_data = dict_get (this->xl->options, 
-                                              "transport.ib-verbs.connect-path");
-
-                if (!(remote_host_data || connect_path_data) || 
-                    (remote_host_data && connect_path_data)) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "address-family not specified and not able to "
-                                "determine the same from other options "
-                                "(remote-host:%s and connect-path:%s)", 
-                                data_to_str (remote_host_data), 
-                                data_to_str (connect_path_data));
-                        return -1;
-                } 
-
-                if (remote_host_data) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "address-family not specified, guessing it "
-                                "to be inet/inet6");
-                        sockaddr->sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "address-family not specified, guessing it "
-                                "to be unix");
-                        sockaddr->sa_family = AF_UNIX;
-                }
-
-        } else {
-                char *address_family = data_to_str (address_family_data);
-                if (!strcasecmp (address_family, "unix")) {
-                        sockaddr->sa_family = AF_UNIX;
-                } else if (!strcasecmp (address_family, "inet")) {
-                        sockaddr->sa_family = AF_INET;
-                } else if (!strcasecmp (address_family, "inet6")) {
-                        sockaddr->sa_family = AF_INET6;
-                } else if (!strcasecmp (address_family, "inet-sdp")) {
-                        sockaddr->sa_family = AF_INET_SDP;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        sockaddr->sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unknown address-family (%s) specified", 
-                                address_family);
-                        return -1;
-                }
-        }
-
-        return 0;
-}
-
-static int32_t
-af_inet_client_get_remote_sockaddr (transport_t *this, 
-                                    struct sockaddr *sockaddr, 
-                                    socklen_t *sockaddr_len)
-{
-        dict_t *options = this->xl->options;
-        data_t *remote_host_data = NULL;
-        data_t *remote_port_data = NULL;
-        char *remote_host = NULL;
-        uint16_t remote_port = 0;
-        struct addrinfo *addr_info = NULL;
-        int32_t ret = 0;
-
-        remote_host_data = dict_get (options, "remote-host");
-        if (remote_host_data == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-host missing in volume %s", 
-                        this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        remote_host = data_to_str (remote_host_data);
-        if (remote_host == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-host has data NULL in volume %s", 
-                        this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        remote_port_data = dict_get (options, "remote-port");
-        if (remote_port_data == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "option remote-port missing in volume %s. "
-                        "Defaulting to %d",
-                        this->xl->name, GF_DEFAULT_IBVERBS_LISTEN_PORT);
-
-                remote_port = GF_DEFAULT_IBVERBS_LISTEN_PORT;
-        }
-        else
-        {
-                remote_port = data_to_uint16 (remote_port_data);
-        }
-
-        if (remote_port == (uint16_t)-1)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-port has invalid port in volume %s",
-                        this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        /* TODO: gf_resolve is a blocking call. kick in some
-           non blocking dns techniques */
-        ret = gf_resolve_ip6 (remote_host, remote_port,
-                              sockaddr->sa_family, 
-                              &this->dnscache, &addr_info);
-        if (ret == -1) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "DNS resolution failed on host %s", remote_host);
-                goto err;
-        }
-
-        memcpy (sockaddr, addr_info->ai_addr, addr_info->ai_addrlen);
-        *sockaddr_len = addr_info->ai_addrlen;
-
-err:
-        return ret;
-}
-
-static int32_t
-af_unix_client_get_remote_sockaddr (transport_t *this, 
-                                    struct sockaddr *sockaddr, 
-                                    socklen_t *sockaddr_len)
-{
-        struct sockaddr_un *sockaddr_un = NULL;
-        char *connect_path = NULL;
-        data_t *connect_path_data = NULL;
-        int32_t ret = 0;
-
-        connect_path_data = dict_get (this->xl->options, 
-                                      "transport.ib-verbs.connect-path");
-        if (!connect_path_data) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option transport.ib-verbs.connect-path not "
-                        "specified for address-family unix");
-                ret = -1;
-                goto err;
-        }
-
-        connect_path = data_to_str (connect_path_data);
-        if (!connect_path) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "connect-path is null-string");
-                ret = -1;
-                goto err;
-        }
-
-        if (strlen (connect_path) > UNIX_PATH_MAX) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "connect-path value length %"GF_PRI_SIZET" > "
-                        "%d octets", strlen (connect_path), UNIX_PATH_MAX);
-                ret = -1;
-                goto err;
-        }
-
-        gf_log (this->xl->name,
-                GF_LOG_DEBUG,
-                "using connect-path %s", connect_path);
-        sockaddr_un = (struct sockaddr_un *)sockaddr;
-        strcpy (sockaddr_un->sun_path, connect_path);
-        *sockaddr_len = sizeof (struct sockaddr_un);
-
-err:
-        return ret;
-}
-
-static int32_t
-af_unix_server_get_local_sockaddr (transport_t *this,
-                                   struct sockaddr *addr,
-                                   socklen_t *addr_len)
-{
-        data_t *listen_path_data = NULL;
-        char *listen_path = NULL;
-        int32_t ret = 0;
-        struct sockaddr_un *sunaddr = (struct sockaddr_un *)addr;
-
-
-        listen_path_data = dict_get (this->xl->options, 
-                                     "transport.ib-verbs.listen-path");
-        if (!listen_path_data) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "missing option listen-path");
-                ret = -1;
-                goto err;
-        }
-
-        listen_path = data_to_str (listen_path_data);
-
-#ifndef UNIX_PATH_MAX
-#define UNIX_PATH_MAX 108
-#endif
-
-        if (strlen (listen_path) > UNIX_PATH_MAX) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option listen-path has value length %"GF_PRI_SIZET" > %d",
-                        strlen (listen_path), UNIX_PATH_MAX);
-                ret = -1;
-                goto err;
-        }
-
-        sunaddr->sun_family = AF_UNIX;
-        strcpy (sunaddr->sun_path, listen_path);
-        *addr_len = sizeof (struct sockaddr_un);
-
-err:
-        return ret;
-}
-
-static int32_t 
-af_inet_server_get_local_sockaddr (transport_t *this, 
-                                   struct sockaddr *addr, 
-                                   socklen_t *addr_len)
-{
-        struct addrinfo hints, *res = 0;
-        data_t *listen_port_data = NULL, *listen_host_data = NULL;
-        uint16_t listen_port = -1;
-        char service[NI_MAXSERV], *listen_host = NULL;
-        dict_t *options = NULL;
-        int32_t ret = 0;
-
-        options = this->xl->options;
-
-        listen_port_data = dict_get (options, "transport.ib-verbs.listen-port");
-        listen_host_data = dict_get (options, "transport.ib-verbs.bind-address");
-
-        if (listen_port_data)
-        {
-                listen_port = data_to_uint16 (listen_port_data);
-        } else {
-		if (addr->sa_family == AF_INET6) {
-			struct sockaddr_in6 *in = (struct sockaddr_in6 *) addr;
-			in->sin6_addr = in6addr_any;
-			in->sin6_port = htons(listen_port);
-			*addr_len = sizeof(struct sockaddr_in6);
-                        goto out;
-		} else if (addr->sa_family == AF_INET) {
-			struct sockaddr_in *in = (struct sockaddr_in *) addr;
-			in->sin_addr.s_addr = htonl(INADDR_ANY);
-			in->sin_port = htons(listen_port);
-			*addr_len = sizeof(struct sockaddr_in);
-                        goto out;
-		}
-	}
-
-        if (listen_port == (uint16_t) -1)
-                listen_port = GF_DEFAULT_IBVERBS_LISTEN_PORT;
-
-
-        if (listen_host_data)
-        {
-                listen_host = data_to_str (listen_host_data);
-        }
-
-        memset (service, 0, sizeof (service));
-        sprintf (service, "%d", listen_port);
-
-        memset (&hints, 0, sizeof (hints));
-        hints.ai_family = addr->sa_family;
-        hints.ai_socktype = SOCK_STREAM;
-        hints.ai_flags    = AI_ADDRCONFIG | AI_PASSIVE;
-
-        ret = getaddrinfo(listen_host, service, &hints, &res);
-        if (ret != 0) {
-                gf_log (this->xl->name,
-                        GF_LOG_ERROR,
-                        "getaddrinfo failed for host %s, service %s (%s)", 
-                        listen_host, service, gai_strerror (ret));
-                ret = -1;
-                goto out;
-        }
-
-        memcpy (addr, res->ai_addr, res->ai_addrlen);
-        *addr_len = res->ai_addrlen;
-
-        freeaddrinfo (res);
-
-out:
-        return ret;
-}
-
-int32_t
-gf_ibverbs_client_bind (transport_t *this,
-                        struct sockaddr *sockaddr,
-                        socklen_t *sockaddr_len,
-                        int sock)
-{
-        int ret = 0;
-
-        *sockaddr_len = sizeof (struct sockaddr_in6);
-        switch (sockaddr->sa_family)
-        {
-        case AF_INET_SDP:
-        case AF_INET:
-                *sockaddr_len = sizeof (struct sockaddr_in);
-
-        case AF_INET6:
-                ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr, 
-                                                       *sockaddr_len, 
-                                                       CLIENT_PORT_CEILING);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_WARNING,
-                                "cannot bind inet socket (%d) to port "
-                                "less than %d (%s)", 
-                                sock, CLIENT_PORT_CEILING, strerror (errno));
-                        ret = 0;
-                }
-                break;
-
-        case AF_UNIX:
-                *sockaddr_len = sizeof (struct sockaddr_un);
-                ret = af_unix_client_bind (this, (struct sockaddr *)sockaddr, 
-                                           *sockaddr_len, sock);
-                break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "unknown address family %d", sockaddr->sa_family);
-                ret = -1;
-                break;
-        }
-
-        return ret;
-}
-
-int32_t
-gf_ibverbs_client_get_remote_sockaddr (transport_t *this,
-                                       struct sockaddr *sockaddr,
-                                       socklen_t *sockaddr_len)
-{
-        int32_t ret = 0;
-        char is_inet_sdp = 0;
-
-        ret = client_fill_address_family (this, sockaddr);
-        if (ret) {
-                ret = -1;
-                goto err;
-        }
- 
-        switch (sockaddr->sa_family)
-        {
-        case AF_INET_SDP:
-                sockaddr->sa_family = AF_INET;
-                is_inet_sdp = 1;
-
-        case AF_INET:
-        case AF_INET6:
-        case AF_UNSPEC:
-                ret = af_inet_client_get_remote_sockaddr (this, 
-                                                          sockaddr, 
-                                                          sockaddr_len);
-
-                if (is_inet_sdp) {
-                        sockaddr->sa_family = AF_INET_SDP;
-                }
-
-                break;
-
-        case AF_UNIX:
-                ret = af_unix_client_get_remote_sockaddr (this, 
-                                                          sockaddr, 
-                                                          sockaddr_len);
-                break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "unknown address-family %d", sockaddr->sa_family);
-                ret = -1;
-        }
-  
-err:
-        return ret;
-}
-
-int32_t
-gf_ibverbs_server_get_local_sockaddr (transport_t *this,
-                                      struct sockaddr *addr,
-                                      socklen_t *addr_len)
-{
-        data_t *address_family_data = NULL;
-        int32_t ret = 0;
-        char is_inet_sdp = 0;
-
-        address_family_data = dict_get (this->xl->options, 
-                                        "transport.address-family");
-        if (address_family_data) {
-                char *address_family = NULL;
-                address_family = data_to_str (address_family_data);
-
-                if (!strcasecmp (address_family, "inet")) {
-                        addr->sa_family = AF_INET;
-                } else if (!strcasecmp (address_family, "inet6")) {
-                        addr->sa_family = AF_INET6;
-                } else if (!strcasecmp (address_family, "inet-sdp")) {
-                        addr->sa_family = AF_INET_SDP;
-                } else if (!strcasecmp (address_family, "unix")) {
-                        addr->sa_family = AF_UNIX;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        addr->sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unknown address family (%s) specified", 
-                                address_family);
-                        ret = -1;
-                        goto err;
-                }
-        } else {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "option address-family not specified, defaulting "
-                        "to inet/inet6");
-                addr->sa_family = AF_UNSPEC;
-        }
-
-        switch (addr->sa_family)
-        {
-        case AF_INET_SDP:
-                is_inet_sdp = 1;
-                addr->sa_family = AF_INET;
-
-        case AF_INET:
-        case AF_INET6:
-        case AF_UNSPEC:
-                ret = af_inet_server_get_local_sockaddr (this, addr, addr_len);
-                if (is_inet_sdp && !ret) {
-                        addr->sa_family = AF_INET_SDP;
-                }
-                break;
-
-        case AF_UNIX:
-                ret = af_unix_server_get_local_sockaddr (this, addr, addr_len);
-                break;
-        }
-
-err:
-        return ret;
-}
-
-int32_t 
-fill_inet6_inet_identifiers (transport_t *this, struct sockaddr_storage *addr, 
-                             int32_t addr_len, char *identifier)
-{
-        int32_t ret = 0, tmpaddr_len = 0;
-        char service[NI_MAXSERV], host[NI_MAXHOST];
-        struct sockaddr_storage tmpaddr;
-
-        memset (&tmpaddr, 0, sizeof (tmpaddr));
-        tmpaddr = *addr;
-        tmpaddr_len = addr_len;
-
-        if (((struct sockaddr *) &tmpaddr)->sa_family == AF_INET6) {
-                int32_t one_to_four, four_to_eight, twelve_to_sixteen;
-                int16_t eight_to_ten, ten_to_twelve;
-    
-                one_to_four = four_to_eight = twelve_to_sixteen = 0;
-                eight_to_ten = ten_to_twelve = 0;
-    
-                one_to_four = ((struct sockaddr_in6 *) 
-                               &tmpaddr)->sin6_addr.s6_addr32[0];
-                four_to_eight = ((struct sockaddr_in6 *) 
-                                 &tmpaddr)->sin6_addr.s6_addr32[1];
-#ifdef GF_SOLARIS_HOST_OS
-                eight_to_ten = S6_ADDR16(((struct sockaddr_in6 *) 
-                                          &tmpaddr)->sin6_addr)[4];
-#else
-                eight_to_ten = ((struct sockaddr_in6 *) 
-                                &tmpaddr)->sin6_addr.s6_addr16[4];
-#endif
-
-#ifdef GF_SOLARIS_HOST_OS
-                ten_to_twelve = S6_ADDR16(((struct sockaddr_in6 *) 
-                                           &tmpaddr)->sin6_addr)[5];
-#else
-                ten_to_twelve = ((struct sockaddr_in6 *) 
-                                 &tmpaddr)->sin6_addr.s6_addr16[5];
-#endif
-                twelve_to_sixteen = ((struct sockaddr_in6 *) 
-                                     &tmpaddr)->sin6_addr.s6_addr32[3];
-
-                /* ipv4 mapped ipv6 address has
-                   bits 0-80: 0
-                   bits 80-96: 0xffff
-                   bits 96-128: ipv4 address 
-                */
- 
-                if (one_to_four == 0 &&
-                    four_to_eight == 0 &&
-                    eight_to_ten == 0 &&
-                    ten_to_twelve == -1) {
-                        struct sockaddr_in *in_ptr = (struct sockaddr_in *)&tmpaddr;
-                        memset (&tmpaddr, 0, sizeof (tmpaddr));
-      
-                        in_ptr->sin_family = AF_INET;
-                        in_ptr->sin_port = ((struct sockaddr_in6 *)addr)->sin6_port;
-                        in_ptr->sin_addr.s_addr = twelve_to_sixteen;
-                        tmpaddr_len = sizeof (*in_ptr);
-                }
-        }
-
-        ret = getnameinfo ((struct sockaddr *) &tmpaddr,
-                           tmpaddr_len,
-                           host, sizeof (host),
-                           service, sizeof (service),
-                           NI_NUMERICHOST | NI_NUMERICSERV);
-        if (ret != 0) {
-                gf_log (this->xl->name,
-                        GF_LOG_ERROR,
-                        "getnameinfo failed (%s)", gai_strerror (ret));
-        }
-
-        sprintf (identifier, "%s:%s", host, service);
-
-        return ret;
-}
-
-int32_t
-gf_ibverbs_get_transport_identifiers (transport_t *this)
-{
-        int32_t ret = 0;
-        char is_inet_sdp = 0;
-
-        switch (((struct sockaddr *) &this->myinfo.sockaddr)->sa_family)
-        {
-        case AF_INET_SDP:
-                is_inet_sdp = 1;
-                ((struct sockaddr *) &this->peerinfo.sockaddr)->sa_family = ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = AF_INET;
-
-        case AF_INET:
-        case AF_INET6:
-        {
-                ret = fill_inet6_inet_identifiers (this, 
-                                                   &this->myinfo.sockaddr, 
-                                                   this->myinfo.sockaddr_len,
-                                                   this->myinfo.identifier);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "can't fill inet/inet6 identifier for server");
-                        goto err;
-                }
-
-                ret = fill_inet6_inet_identifiers (this,
-                                                   &this->peerinfo.sockaddr,
-                                                   this->peerinfo.sockaddr_len,
-                                                   this->peerinfo.identifier);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "can't fill inet/inet6 identifier for client");
-                        goto err;
-                }
-
-                if (is_inet_sdp) {
-                        ((struct sockaddr *) &this->peerinfo.sockaddr)->sa_family = ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = AF_INET_SDP;
-                }
-        }
-        break;
-
-        case AF_UNIX:
-        {
-                struct sockaddr_un *sunaddr = NULL;
-
-                sunaddr = (struct sockaddr_un *) &this->myinfo.sockaddr;
-                strcpy (this->myinfo.identifier, sunaddr->sun_path);
-
-                sunaddr = (struct sockaddr_un *) &this->peerinfo.sockaddr;
-                strcpy (this->peerinfo.identifier, sunaddr->sun_path);
-        }
-        break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR, 
-                        "unknown address family (%d)",
-                        ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family);
-                ret = -1;
-                break;
-        }
-
-err:
-        return ret;
-}
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/name.h b/xlators/protocol/legacy/transport/ib-verbs/src/name.h
deleted file mode 100644
index 366077fdd..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/name.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _IB_VERBS_NAME_H
-#define _IB_VERBS_NAME_H
-
-#include <sys/socket.h>
-#include <sys/un.h>
-
-#include "compat.h"
-
-int32_t
-gf_ibverbs_client_bind (transport_t *this,
-                        struct sockaddr *sockaddr,
-                        socklen_t *sockaddr_len,
-                        int sock);
-
-int32_t
-gf_ibverbs_client_get_remote_sockaddr (transport_t *this,
-                                    struct sockaddr *sockaddr,
-                                    socklen_t *sockaddr_len);
-
-int32_t
-gf_ibverbs_server_get_local_sockaddr (transport_t *this,
-                                   struct sockaddr *addr,
-                                   socklen_t *addr_len);
-
-int32_t
-gf_ibverbs_get_transport_identifiers (transport_t *this);
-
-#endif /* _IB_VERBS_NAME_H */
diff --git a/xlators/protocol/legacy/transport/socket/Makefile.am b/xlators/protocol/legacy/transport/socket/Makefile.am
deleted file mode 100644
index f963effea..000000000
--- a/xlators/protocol/legacy/transport/socket/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS = src
-\ No newline at end of file
diff --git a/xlators/protocol/legacy/transport/socket/src/Makefile.am b/xlators/protocol/legacy/transport/socket/src/Makefile.am
deleted file mode 100644
index 5952e18e9..000000000
--- a/xlators/protocol/legacy/transport/socket/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-# TODO : change to proper transport dir
-
-transport_LTLIBRARIES = socket.la
-transportdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/transport
-
-socket_la_LDFLAGS = -module -avoidversion
-
-socket_la_SOURCES = socket.c name.c
-socket_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	$(top_builddir)/xlators/protocol/legacy/lib/src/libgfproto.la
-
-noinst_HEADERS = socket.h name.h socket-mem-types.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
-	-I$(top_srcdir)/xlators/protocol/legacy/transport/socket/src   \
-	-I$(top_srcdir)/xlators/protocol/legacy/lib/src
-
-CLEANFILES = *~
diff --git a/xlators/protocol/legacy/transport/socket/src/name.c b/xlators/protocol/legacy/transport/socket/src/name.c
deleted file mode 100644
index 644a0bc5e..000000000
--- a/xlators/protocol/legacy/transport/socket/src/name.c
+++ /dev/null
@@ -1,740 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <errno.h>
-#include <netdb.h>
-#include <string.h>
-
-#ifdef CLIENT_PORT_CEILING
-#undef CLIENT_PORT_CEILING
-#endif
-
-#define CLIENT_PORT_CEILING 1024
-
-#ifndef AF_INET_SDP
-#define AF_INET_SDP 27
-#endif
-
-static int gf_name_addr_enotspec_log;
-
-#include "transport.h"
-#include "socket.h"
-
-int32_t
-gf_resolve_ip6 (const char *hostname, 
-                uint16_t port, 
-                int family, 
-                void **dnscache, 
-                struct addrinfo **addr_info);
-
-static int32_t
-af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr, 
-                                 socklen_t sockaddr_len, int ceiling)
-{
-        int32_t ret = -1;
-        /*  struct sockaddr_in sin = {0, }; */
-        uint16_t port = ceiling - 1;
-
-        while (port)
-        {
-                switch (sockaddr->sa_family)
-                {
-                case AF_INET6:
-                        ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
-                        break;
-
-                case AF_INET_SDP:
-                case AF_INET:
-                        ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
-                        break;
-                }
-
-                ret = bind (fd, sockaddr, sockaddr_len);
-
-                if (ret == 0)
-                        break;
-
-                if (ret == -1 && errno == EACCES)
-                        break;
-
-                port--;
-        }
-
-        return ret;
-}
-
-static int32_t
-af_unix_client_bind (transport_t *this, 
-                     struct sockaddr *sockaddr, 
-                     socklen_t sockaddr_len, 
-                     int sock)
-{
-        data_t *path_data = NULL;
-        struct sockaddr_un *addr = NULL;
-        int32_t ret = 0;
-
-        path_data = dict_get (this->xl->options, "transport.socket.bind-path");
-        if (path_data) {
-                char *path = data_to_str (path_data);
-                if (!path || strlen (path) > UNIX_PATH_MAX) {
-                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                "bind-path not specfied for unix socket, "
-                                "letting connect to assign default value");
-                        goto err;
-                }
-
-                addr = (struct sockaddr_un *) sockaddr;
-                strcpy (addr->sun_path, path);
-                ret = bind (sock, (struct sockaddr *)addr, sockaddr_len);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "cannot bind to unix-domain socket %d (%s)", 
-                                sock, strerror (errno));
-                        goto err;
-                }
-        } else {
-                gf_log (this->xl->name, GF_LOG_TRACE,
-                        "bind-path not specfied for unix socket, "
-                        "letting connect to assign default value");
-        }
-
-err:
-        return ret;
-}
-
-static int32_t
-client_fill_address_family (transport_t *this, sa_family_t *sa_family)
-{
-        data_t  *address_family_data = NULL;
-        int32_t  ret                 = -1;
-
-        if (sa_family == NULL) {
-                goto out;
-        }
-
-        address_family_data = dict_get (this->xl->options, 
-                                        "transport.address-family");
-        if (!address_family_data) {
-                data_t *remote_host_data = NULL, *connect_path_data = NULL;
-                remote_host_data = dict_get (this->xl->options, "remote-host");
-                connect_path_data = dict_get (this->xl->options, 
-                                              "transport.socket.connect-path");
-
-                if (!(remote_host_data || connect_path_data) || 
-                    (remote_host_data && connect_path_data)) {
-                        GF_LOG_OCCASIONALLY (gf_name_addr_enotspec_log, 
-                                             this->xl->name, GF_LOG_ERROR,
-                                             "transport.address-family not specified and "
-                                             "not able to determine the "
-                                             "same from other options (remote-host:%s and "
-                                             "transport.unix.connect-path:%s)", 
-                                             data_to_str (remote_host_data), 
-                                             data_to_str (connect_path_data));
-                        goto out;
-                } 
-
-                if (remote_host_data) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "address-family not specified, guessing it "
-                                "to be inet/inet6");
-                        *sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "address-family not specified, guessing it "
-                                "to be unix");
-                        *sa_family = AF_UNIX;
-                }
-
-        } else {
-                char *address_family = data_to_str (address_family_data);
-                if (!strcasecmp (address_family, "unix")) {
-                        *sa_family = AF_UNIX;
-                } else if (!strcasecmp (address_family, "inet")) {
-                        *sa_family = AF_INET;
-                } else if (!strcasecmp (address_family, "inet6")) {
-                        *sa_family = AF_INET6;
-                } else if (!strcasecmp (address_family, "inet-sdp")) {
-                        *sa_family = AF_INET_SDP;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        *sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unknown address-family (%s) specified", 
-                                address_family);
-                        goto out;
-                }
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-static int32_t
-af_inet_client_get_remote_sockaddr (transport_t *this, 
-                                    struct sockaddr *sockaddr, 
-                                    socklen_t *sockaddr_len)
-{
-        dict_t *options = this->xl->options;
-        data_t *remote_host_data = NULL;
-        data_t *remote_port_data = NULL;
-        char *remote_host = NULL;
-        uint16_t remote_port = 0;
-        struct addrinfo *addr_info = NULL;
-        int32_t ret = 0;
-
-        remote_host_data = dict_get (options, "remote-host");
-        if (remote_host_data == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-host missing in volume %s", this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        remote_host = data_to_str (remote_host_data);
-        if (remote_host == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-host has data NULL in volume %s", this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        remote_port_data = dict_get (options, "remote-port");
-        if (remote_port_data == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_TRACE,
-                        "option remote-port missing in volume %s. Defaulting to %d",
-                        this->xl->name, GF_DEFAULT_SOCKET_LISTEN_PORT);
-
-                remote_port = GF_DEFAULT_SOCKET_LISTEN_PORT;
-        }
-        else
-        {
-                remote_port = data_to_uint16 (remote_port_data);
-        }
-
-        if (remote_port == (uint16_t)-1)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-port has invalid port in volume %s",
-                        this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        /* TODO: gf_resolve is a blocking call. kick in some
-           non blocking dns techniques */
-        ret = gf_resolve_ip6 (remote_host, remote_port,
-                              sockaddr->sa_family, &this->dnscache, &addr_info);
-        if (ret == -1) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "DNS resolution failed on host %s", remote_host);
-                goto err;
-        }
-
-        memcpy (sockaddr, addr_info->ai_addr, addr_info->ai_addrlen);
-        *sockaddr_len = addr_info->ai_addrlen;
-
-err:
-        return ret;
-}
-
-static int32_t
-af_unix_client_get_remote_sockaddr (transport_t *this, 
-                                    struct sockaddr *sockaddr, 
-                                    socklen_t *sockaddr_len)
-{
-        struct sockaddr_un *sockaddr_un = NULL;
-        char *connect_path = NULL;
-        data_t *connect_path_data = NULL;
-        int32_t ret = 0;
-
-        connect_path_data = dict_get (this->xl->options, 
-                                      "transport.socket.connect-path");
-        if (!connect_path_data) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option transport.unix.connect-path not specified for "
-                        "address-family unix");
-                ret = -1;
-                goto err;
-        }
-
-        connect_path = data_to_str (connect_path_data);
-        if (!connect_path) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "transport.unix.connect-path is null-string");
-                ret = -1;
-                goto err;
-        }
-
-        if (strlen (connect_path) > UNIX_PATH_MAX) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "connect-path value length %"GF_PRI_SIZET" > %d octets", 
-                        strlen (connect_path), UNIX_PATH_MAX);
-                ret = -1;
-                goto err;
-        }
-
-        gf_log (this->xl->name, GF_LOG_TRACE,
-                "using connect-path %s", connect_path);
-        sockaddr_un = (struct sockaddr_un *)sockaddr;
-        strcpy (sockaddr_un->sun_path, connect_path);
-        *sockaddr_len = sizeof (struct sockaddr_un);
-
-err:
-        return ret;
-}
-
-static int32_t
-af_unix_server_get_local_sockaddr (transport_t *this,
-                                   struct sockaddr *addr,
-                                   socklen_t *addr_len)
-{
-        data_t *listen_path_data = NULL;
-        char *listen_path = NULL;
-        int32_t ret = 0;
-        struct sockaddr_un *sunaddr = (struct sockaddr_un *)addr;
-
-
-        listen_path_data = dict_get (this->xl->options, 
-                                     "transport.socket.listen-path");
-        if (!listen_path_data) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "missing option transport.socket.listen-path");
-                ret = -1;
-                goto err;
-        }
-
-        listen_path = data_to_str (listen_path_data);
-
-#ifndef UNIX_PATH_MAX
-#define UNIX_PATH_MAX 108
-#endif
-
-        if (strlen (listen_path) > UNIX_PATH_MAX) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option transport.unix.listen-path has value length "
-                        "%"GF_PRI_SIZET" > %d",
-                        strlen (listen_path), UNIX_PATH_MAX);
-                ret = -1;
-                goto err;
-        }
-
-        sunaddr->sun_family = AF_UNIX;
-        strcpy (sunaddr->sun_path, listen_path);
-        *addr_len = sizeof (struct sockaddr_un);
-
-err:
-        return ret;
-}
-
-static int32_t
-af_inet_server_get_local_sockaddr (transport_t *this,
-                                   struct sockaddr *addr,
-                                   socklen_t *addr_len)
-{
-        struct addrinfo hints, *res = 0;
-        data_t *listen_port_data = NULL, *listen_host_data = NULL;
-        uint16_t listen_port = -1;
-        char service[NI_MAXSERV], *listen_host = NULL;
-        dict_t *options = NULL;
-        int32_t ret = 0;
-
-        options = this->xl->options;
-
-        listen_port_data = dict_get (options, "transport.socket.listen-port");
-        listen_host_data = dict_get (options, "transport.socket.bind-address");
-
-        if (listen_port_data)
-        {
-                listen_port = data_to_uint16 (listen_port_data);
-        }
-
-        if (listen_port == (uint16_t) -1)
-                listen_port = GF_DEFAULT_SOCKET_LISTEN_PORT;
-
-
-        if (listen_host_data)
-        {
-                listen_host = data_to_str (listen_host_data);
-        } else {
-		if (addr->sa_family == AF_INET6) {
-			struct sockaddr_in6 *in = (struct sockaddr_in6 *) addr;
-			in->sin6_addr = in6addr_any;
-			in->sin6_port = htons(listen_port);
-			*addr_len = sizeof(struct sockaddr_in6);
-                        goto out;
-		} else if (addr->sa_family == AF_INET) {
-			struct sockaddr_in *in = (struct sockaddr_in *) addr;
-			in->sin_addr.s_addr = htonl(INADDR_ANY);
-			in->sin_port = htons(listen_port);
-			*addr_len = sizeof(struct sockaddr_in);
-			goto out;
-		}
-	}
-
-        memset (service, 0, sizeof (service));
-        sprintf (service, "%d", listen_port);
-
-        memset (&hints, 0, sizeof (hints));
-        hints.ai_family = addr->sa_family;
-        hints.ai_socktype = SOCK_STREAM;
-        hints.ai_flags    = AI_ADDRCONFIG | AI_PASSIVE;
-
-        ret = getaddrinfo(listen_host, service, &hints, &res);
-        if (ret != 0) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "getaddrinfo failed for host %s, service %s (%s)", 
-                        listen_host, service, gai_strerror (ret));
-                ret = -1;
-                goto out;
-        }
-
-        memcpy (addr, res->ai_addr, res->ai_addrlen);
-        *addr_len = res->ai_addrlen;
-
-        freeaddrinfo (res);
-
-out:
-        return ret;
-}
-
-int32_t
-gf_client_bind (transport_t *this,
-             struct sockaddr *sockaddr,
-             socklen_t *sockaddr_len,
-             int sock)
-{
-        int ret = 0;
-
-        *sockaddr_len = sizeof (struct sockaddr_in6);
-        switch (sockaddr->sa_family)
-        {
-        case AF_INET_SDP:
-        case AF_INET:
-                *sockaddr_len = sizeof (struct sockaddr_in);
-
-        case AF_INET6:
-                ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr, 
-                                                       *sockaddr_len, CLIENT_PORT_CEILING);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_WARNING,
-                                "cannot bind inet socket (%d) to port less than %d (%s)", 
-                                sock, CLIENT_PORT_CEILING, strerror (errno));
-                        ret = 0;
-                }
-                break;
-
-        case AF_UNIX:
-                *sockaddr_len = sizeof (struct sockaddr_un);
-                ret = af_unix_client_bind (this, (struct sockaddr *)sockaddr, 
-                                           *sockaddr_len, sock);
-                break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "unknown address family %d", sockaddr->sa_family);
-                ret = -1;
-                break;
-        }
-
-        return ret;
-}
-
-int32_t
-gf_socket_client_get_remote_sockaddr (transport_t *this,
-                                   struct sockaddr *sockaddr,
-                                   socklen_t *sockaddr_len,
-                                   sa_family_t *sa_family)
-{
-        int32_t ret = 0;
-
-        if ((sockaddr == NULL) || (sockaddr_len == NULL)
-            || (sa_family == NULL)) {
-                ret = -1;
-                goto err;
-        }
-
-
-        ret = client_fill_address_family (this, &sockaddr->sa_family);
-        if (ret) {
-                ret = -1;
-                goto err;
-        }
- 
-        *sa_family = sockaddr->sa_family;
-
-        switch (sockaddr->sa_family)
-        {
-        case AF_INET_SDP:
-                sockaddr->sa_family = AF_INET;
-
-        case AF_INET:
-        case AF_INET6:
-        case AF_UNSPEC:
-                ret = af_inet_client_get_remote_sockaddr (this, sockaddr,
-                                                          sockaddr_len);
-                break;
-
-        case AF_UNIX:
-                ret = af_unix_client_get_remote_sockaddr (this, sockaddr,
-                                                          sockaddr_len);
-                break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "unknown address-family %d", sockaddr->sa_family);
-                ret = -1;
-        }
-  
-        if (*sa_family == AF_UNSPEC) {
-                *sa_family = sockaddr->sa_family;
-        }
-
-err:
-        return ret;
-}
-
-
-static int32_t
-server_fill_address_family (transport_t *this, sa_family_t *sa_family)
-{
-        data_t  *address_family_data = NULL;
-        int32_t  ret                 = -1;
-        
-        if (sa_family == NULL) {
-                goto out;
-        }
-
-        address_family_data = dict_get (this->xl->options, 
-                                        "transport.address-family");
-        if (address_family_data) {
-                char *address_family = NULL;
-                address_family = data_to_str (address_family_data);
-
-                if (!strcasecmp (address_family, "inet")) {
-                        *sa_family = AF_INET;
-                } else if (!strcasecmp (address_family, "inet6")) {
-                        *sa_family = AF_INET6;
-                } else if (!strcasecmp (address_family, "inet-sdp")) {
-                        *sa_family = AF_INET_SDP;
-                } else if (!strcasecmp (address_family, "unix")) {
-                        *sa_family = AF_UNIX;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        *sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unknown address family (%s) specified", address_family);
-                        goto out;
-                }
-        } else {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "option address-family not specified, defaulting to inet/inet6");
-                *sa_family = AF_UNSPEC;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-int32_t
-gf_socket_server_get_local_sockaddr (transport_t *this, struct sockaddr *addr,
-                                  socklen_t *addr_len, sa_family_t *sa_family)
-{
-        int32_t ret = -1;
-
-        if ((addr == NULL) || (addr_len == NULL) || (sa_family == NULL)) {
-                goto err;
-        }
-
-        ret = server_fill_address_family (this, &addr->sa_family);
-        if (ret == -1) {
-                goto err;
-        }
-
-        *sa_family = addr->sa_family;
-
-        switch (addr->sa_family)
-        {
-        case AF_INET_SDP:
-                addr->sa_family = AF_INET;
-
-        case AF_INET:
-        case AF_INET6:
-        case AF_UNSPEC:
-                ret = af_inet_server_get_local_sockaddr (this, addr, addr_len);
-                break;
-
-        case AF_UNIX:
-                ret = af_unix_server_get_local_sockaddr (this, addr, addr_len);
-                break;
-        }
-
-        if (*sa_family == AF_UNSPEC) {
-                *sa_family = addr->sa_family;
-        }
-
-err:
-        return ret;
-}
-
-static int32_t
-fill_inet6_inet_identifiers (transport_t *this, struct sockaddr_storage *addr,
-                             int32_t addr_len, char *identifier)
-{
-        int32_t ret = 0, tmpaddr_len = 0;
-        char service[NI_MAXSERV], host[NI_MAXHOST];
-        struct sockaddr_storage tmpaddr;
-
-        memset (&tmpaddr, 0, sizeof (tmpaddr));
-        tmpaddr = *addr;
-        tmpaddr_len = addr_len;
-
-        if (((struct sockaddr *) &tmpaddr)->sa_family == AF_INET6) {
-                int32_t one_to_four, four_to_eight, twelve_to_sixteen;
-                int16_t eight_to_ten, ten_to_twelve;
-    
-                one_to_four = four_to_eight = twelve_to_sixteen = 0;
-                eight_to_ten = ten_to_twelve = 0;
-    
-                one_to_four = ((struct sockaddr_in6 *) &tmpaddr)->sin6_addr.s6_addr32[0];
-                four_to_eight = ((struct sockaddr_in6 *) &tmpaddr)->sin6_addr.s6_addr32[1];
-#ifdef GF_SOLARIS_HOST_OS
-                eight_to_ten = S6_ADDR16(((struct sockaddr_in6 *) &tmpaddr)->sin6_addr)[4];
-#else
-                eight_to_ten = ((struct sockaddr_in6 *) &tmpaddr)->sin6_addr.s6_addr16[4];
-#endif
-
-#ifdef GF_SOLARIS_HOST_OS
-                ten_to_twelve = S6_ADDR16(((struct sockaddr_in6 *) &tmpaddr)->sin6_addr)[5];
-#else
-                ten_to_twelve = ((struct sockaddr_in6 *) &tmpaddr)->sin6_addr.s6_addr16[5];
-#endif
-
-                twelve_to_sixteen = ((struct sockaddr_in6 *) &tmpaddr)->sin6_addr.s6_addr32[3];
-
-                /* ipv4 mapped ipv6 address has
-                   bits 0-80: 0
-                   bits 80-96: 0xffff
-                   bits 96-128: ipv4 address 
-                */
- 
-                if (one_to_four == 0 &&
-                    four_to_eight == 0 &&
-                    eight_to_ten == 0 &&
-                    ten_to_twelve == -1) {
-                        struct sockaddr_in *in_ptr = (struct sockaddr_in *)&tmpaddr;
-                        memset (&tmpaddr, 0, sizeof (tmpaddr));
-      
-                        in_ptr->sin_family = AF_INET;
-                        in_ptr->sin_port = ((struct sockaddr_in6 *)addr)->sin6_port;
-                        in_ptr->sin_addr.s_addr = twelve_to_sixteen;
-                        tmpaddr_len = sizeof (*in_ptr);
-                }
-        }
-
-        ret = getnameinfo ((struct sockaddr *) &tmpaddr,
-                           tmpaddr_len,
-                           host, sizeof (host),
-                           service, sizeof (service),
-                           NI_NUMERICHOST | NI_NUMERICSERV);
-        if (ret != 0) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "getnameinfo failed (%s)", gai_strerror (ret));
-        }
-
-        sprintf (identifier, "%s:%s", host, service);
-
-        return ret;
-}
-
-int32_t
-gf_get_transport_identifiers (transport_t *this)
-{
-        int32_t ret = 0;
-        char is_inet_sdp = 0;
-
-        switch (((struct sockaddr *) &this->myinfo.sockaddr)->sa_family)
-        {
-        case AF_INET_SDP:
-                is_inet_sdp = 1;
-                ((struct sockaddr *) &this->peerinfo.sockaddr)->sa_family = ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = AF_INET;
-
-        case AF_INET:
-        case AF_INET6:
-        {
-                ret = fill_inet6_inet_identifiers (this, 
-                                                   &this->myinfo.sockaddr, 
-                                                   this->myinfo.sockaddr_len,
-                                                   this->myinfo.identifier);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "cannot fill inet/inet6 identifier for server");
-                        goto err;
-                }
-
-                ret = fill_inet6_inet_identifiers (this,
-                                                   &this->peerinfo.sockaddr,
-                                                   this->peerinfo.sockaddr_len,
-                                                   this->peerinfo.identifier);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "cannot fill inet/inet6 identifier for client");
-                        goto err;
-                }
-
-                if (is_inet_sdp) {
-                        ((struct sockaddr *) &this->peerinfo.sockaddr)->sa_family = ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = AF_INET_SDP;
-                }
-        }
-        break;
-
-        case AF_UNIX:
-        {
-                struct sockaddr_un *sunaddr = NULL;
-
-                sunaddr = (struct sockaddr_un *) &this->myinfo.sockaddr;
-                strcpy (this->myinfo.identifier, sunaddr->sun_path);
-
-                sunaddr = (struct sockaddr_un *) &this->peerinfo.sockaddr;
-                strcpy (this->peerinfo.identifier, sunaddr->sun_path);
-        }
-        break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "unknown address family (%d)", 
-                        ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family);
-                ret = -1;
-                break;
-        }
-
-err:
-        return ret;
-}
diff --git a/xlators/protocol/legacy/transport/socket/src/name.h b/xlators/protocol/legacy/transport/socket/src/name.h
deleted file mode 100644
index 789cf13ef..000000000
--- a/xlators/protocol/legacy/transport/socket/src/name.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _SOCKET_NAME_H
-#define _SOCKET_NAME_H
-
-#include "compat.h"
-
-int32_t
-gf_client_bind (transport_t *this,
-                struct sockaddr *sockaddr,
-                socklen_t *sockaddr_len,
-                int sock);
-
-int32_t
-gf_socket_client_get_remote_sockaddr (transport_t *this,
-                                      struct sockaddr *sockaddr,
-                                      socklen_t *sockaddr_len,
-                                      sa_family_t *sa_family);
-
-int32_t
-gf_socket_server_get_local_sockaddr (transport_t *this, struct sockaddr *addr,
-                                     socklen_t *addr_len, sa_family_t *sa_family);
-
-int32_t
-gf_get_transport_identifiers (transport_t *this);
-
-#endif /* _SOCKET_NAME_H */
diff --git a/xlators/protocol/legacy/transport/socket/src/socket-mem-types.h b/xlators/protocol/legacy/transport/socket/src/socket-mem-types.h
deleted file mode 100644
index 937b9d7a8..000000000
--- a/xlators/protocol/legacy/transport/socket/src/socket-mem-types.h
+++ /dev/null
@@ -1,36 +0,0 @@
-
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __SOCKET_MEM_TYPES_H__
-#define __SOCKET_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_socket_mem_types_ {
-        gf_socket_mt_socket_private_t = gf_common_mt_end + 1,
-        gf_socket_mt_ioq,
-        gf_socket_mt_transport_t,
-        gf_socket_mt_socket_local_t,
-        gf_socket_mt_char,
-        gf_socket_mt_end
-};
-#endif
-
diff --git a/xlators/protocol/legacy/transport/socket/src/socket.c b/xlators/protocol/legacy/transport/socket/src/socket.c
deleted file mode 100644
index 9272b0584..000000000
--- a/xlators/protocol/legacy/transport/socket/src/socket.c
+++ /dev/null
@@ -1,1625 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "socket.h"
-#include "name.h"
-#include "dict.h"
-#include "transport.h"
-#include "logging.h"
-#include "xlator.h"
-#include "byte-order.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-
-#include <fcntl.h>
-#include <errno.h>
-#include <netinet/tcp.h>
-
-
-#define GF_LOG_ERRNO(errno) ((errno == ENOTCONN) ? GF_LOG_DEBUG : GF_LOG_ERROR)
-#define SA(ptr) ((struct sockaddr *)ptr)
-
-static int socket_init (transport_t *this);
-
-/*
- * return value:
- *   0 = success (completed)
- *  -1 = error
- * > 0 = incomplete
- */
-
-static int
-__socket_rwv (transport_t *this, struct iovec *vector, int count,
-              struct iovec **pending_vector, int *pending_count,
-              int write)
-{
-        socket_private_t *priv = NULL;
-        int               sock = -1;
-        int               ret = -1;
-        struct iovec     *opvector = NULL;
-        int               opcount = 0;
-        int               moved = 0;
-
-        priv = this->private;
-        sock = priv->sock;
-
-	opvector = vector;
-	opcount  = count;
-
-        while (opcount) {
-                if (write) {
-                        ret = writev (sock, opvector, opcount);
-
-                        if (ret == 0 || (ret == -1 && errno == EAGAIN)) {
-                                /* done for now */
-                                break;
-                        }
-                } else {
-                        ret = readv (sock, opvector, opcount);
-
-                        if (ret == -1 && errno == EAGAIN) {
-                                /* done for now */
-                                break;
-                        }
-                }
-
-                if (ret == 0) {
-                        /* Mostly due to 'umount' in client */
-                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                "EOF from peer %s", this->peerinfo.identifier);
-                        opcount = -1;
-                        errno = ENOTCONN;
-                        break;
-                }
-
-                if (ret == -1) {
-                        if (errno == EINTR)
-                                continue;
-
-                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                "%s failed (%s)", write ? "writev" : "readv",
-                                strerror (errno));
-                        opcount = -1;
-                        break;
-                }
-
-                moved = 0;
-
-                while (moved < ret) {
-                        if ((ret - moved) >= opvector[0].iov_len) {
-                                moved += opvector[0].iov_len;
-                                opvector++;
-                                opcount--;
-                        } else {
-                                opvector[0].iov_len -= (ret - moved);
-                                opvector[0].iov_base += (ret - moved);
-                                moved += (ret - moved);
-                        }
-                        while (opcount && !opvector[0].iov_len) {
-                                opvector++;
-                                opcount--;
-                        }
-                }
-        }
-
-        if (pending_vector)
-                *pending_vector = opvector;
-
-        if (pending_count)
-                *pending_count = opcount;
-
-        return opcount;
-}
-
-
-static int
-__socket_readv (transport_t *this, struct iovec *vector, int count,
-                struct iovec **pending_vector, int *pending_count)
-{
-        int ret = -1;
-
-        ret = __socket_rwv (this, vector, count,
-			    pending_vector, pending_count, 0);
-
-        return ret;
-}
-
-
-static int
-__socket_writev (transport_t *this, struct iovec *vector, int count,
-                 struct iovec **pending_vector, int *pending_count)
-{
-        int ret = -1;
-
-        ret = __socket_rwv (this, vector, count,
-			    pending_vector, pending_count, 1);
-
-        return ret;
-}
-
-
-static int
-__socket_disconnect (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-
-        priv = this->private;
-
-        if (priv->sock != -1) {
-                ret = shutdown (priv->sock, SHUT_RDWR);
-                priv->connected = -1;
-                gf_log (this->xl->name, GF_LOG_TRACE,
-                        "shutdown() returned %d. set connection state to -1",
-                        ret);
-        }
-
-        return ret;
-}
-
-
-static int
-__socket_server_bind (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-	int               opt = 1;
-
-	priv = this->private;
-
-        ret = setsockopt (priv->sock, SOL_SOCKET, SO_REUSEADDR,
-			  &opt, sizeof (opt));
-
-        if (ret == -1) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "setsockopt() for SO_REUSEADDR failed (%s)",
-                        strerror (errno));
-        }
-
-        ret = bind (priv->sock, (struct sockaddr *)&this->myinfo.sockaddr,
-		    this->myinfo.sockaddr_len);
-
-        if (ret == -1) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "binding to %s failed: %s",
-                        this->myinfo.identifier, strerror (errno));
-                if (errno == EADDRINUSE) {
-                        gf_log (this->xl->name, GF_LOG_ERROR, 
-                                "Port is already in use");
-                }
-        }
-
-        return ret;
-}
-
-
-static int
-__socket_nonblock (int fd)
-{
-        int flags = 0;
-        int ret = -1;
-
-        flags = fcntl (fd, F_GETFL);
-
-        if (flags != -1)
-                ret = fcntl (fd, F_SETFL, flags | O_NONBLOCK);
-
-        return ret;
-}
-
-
-static int
-__socket_nodelay (int fd)
-{
-        int     on = 1;
-        int     ret = -1;
-
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_NODELAY,
-			  &on, sizeof (on));
-        if (!ret)
-                gf_log ("", GF_LOG_TRACE,
-                        "NODELAY enabled for socket %d", fd);
-
-        return ret;
-}
-
-
-static int
-__socket_keepalive (int fd, int keepalive_intvl)
-{
-        int     on = 1;
-        int     ret = -1;
-
-        ret = setsockopt (fd, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof (on));
-        if (ret == -1)
-                goto err;
-
-        if (keepalive_intvl == GF_USE_DEFAULT_KEEPALIVE)
-                goto done;
-
-#ifndef GF_LINUX_HOST_OS
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPALIVE, &keepalive_intvl,
-                          sizeof (keepalive_intvl));
-        if (ret == -1)
-                goto err;
-#else
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepalive_intvl,
-                          sizeof (keepalive_intvl));
-        if (ret == -1)
-                goto err;
-
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPINTVL, &keepalive_intvl,
-                          sizeof (keepalive_intvl));
-        if (ret == -1)
-                goto err;
-#endif
-
-done:
-        gf_log ("", GF_LOG_TRACE, "Keep-alive enabled for socket %d, interval "
-                "%d", fd, keepalive_intvl);
-
-err:
-        return ret;
-}
-
-
-static int
-__socket_connect_finish (int fd)
-{
-        int       ret = -1;
-        int       optval = 0;
-        socklen_t optlen = sizeof (int);
-
-        ret = getsockopt (fd, SOL_SOCKET, SO_ERROR, (void *)&optval, &optlen);
-
-        if (ret == 0 && optval) {
-                errno = optval;
-                ret = -1;
-        }
-
-        return ret;
-}
-
-
-static void
-__socket_reset (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-
-        priv = this->private;
-
-        /* TODO: use mem-pool on incoming data */
-
-        if (priv->incoming.hdr_p)
-                GF_FREE (priv->incoming.hdr_p);
-
-        if (priv->incoming.iobuf)
-                iobuf_unref (priv->incoming.iobuf);
-
-        memset (&priv->incoming, 0, sizeof (priv->incoming));
-
-        event_unregister (this->xl->ctx->event_pool, priv->sock, priv->idx);
-        close (priv->sock);
-        priv->sock = -1;
-        priv->idx = -1;
-        priv->connected = -1;
-}
-
-
-static struct ioq *
-__socket_ioq_new (transport_t *this, char *buf, int len,
-                  struct iovec *vector, int count, struct iobref *iobref)
-{
-        socket_private_t *priv = NULL;
-        struct ioq       *entry = NULL;
-
-        priv = this->private;
-
-        /* TODO: use mem-pool */
-        entry = GF_CALLOC (1, sizeof (*entry),
-                           gf_common_mt_ioq);
-        if (!entry)
-                return NULL;
-
-        GF_ASSERT (count <= (MAX_IOVEC-2));
-
-        entry->header.colonO[0] = ':';
-        entry->header.colonO[1] = 'O';
-        entry->header.colonO[2] = '\0';
-        entry->header.version   = 42;
-        entry->header.size1     = hton32 (len);
-        entry->header.size2     = hton32 (iov_length (vector, count));
-
-        entry->vector[0].iov_base = &entry->header;
-        entry->vector[0].iov_len  = sizeof (entry->header);
-        entry->count++;
-
-        entry->vector[1].iov_base = buf;
-        entry->vector[1].iov_len  = len;
-        entry->count++;
-
-        if (vector && count) {
-                memcpy (&entry->vector[2], vector, sizeof (*vector) * count);
-                entry->count += count;
-        }
-
-        entry->pending_vector = entry->vector;
-        entry->pending_count  = entry->count;
-
-        if (iobref)
-                entry->iobref = iobref_ref (iobref);
-
-        entry->buf = buf;
-
-        INIT_LIST_HEAD (&entry->list);
-
-        return entry;
-}
-
-
-static void
-__socket_ioq_entry_free (struct ioq *entry)
-{
-        list_del_init (&entry->list);
-        if (entry->iobref)
-                iobref_unref (entry->iobref);
-
-        /* TODO: use mem-pool */
-        GF_FREE (entry->buf);
-
-        /* TODO: use mem-pool */
-        GF_FREE (entry);
-}
-
-
-static void
-__socket_ioq_flush (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        struct ioq       *entry = NULL;
-
-        priv = this->private;
-
-        while (!list_empty (&priv->ioq)) {
-                entry = priv->ioq_next;
-                __socket_ioq_entry_free (entry);
-        }
-
-        return;
-}
-
-
-static int
-__socket_ioq_churn_entry (transport_t *this, struct ioq *entry)
-{
-        int ret = -1;
-
-        ret = __socket_writev (this, entry->pending_vector,
-			       entry->pending_count,
-                               &entry->pending_vector,
-			       &entry->pending_count);
-
-        if (ret == 0) {
-                /* current entry was completely written */
-                GF_ASSERT (entry->pending_count == 0);
-                __socket_ioq_entry_free (entry);
-        }
-
-        return ret;
-}
-
-
-static int
-__socket_ioq_churn (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = 0;
-        struct ioq       *entry = NULL;
-
-        priv = this->private;
-
-        while (!list_empty (&priv->ioq)) {
-                /* pick next entry */
-                entry = priv->ioq_next;
-
-                ret = __socket_ioq_churn_entry (this, entry);
-
-                if (ret != 0)
-                        break;
-        }
-
-        if (list_empty (&priv->ioq)) {
-                /* all pending writes done, not interested in POLLOUT */
-                priv->idx = event_select_on (this->xl->ctx->event_pool,
-					     priv->sock, priv->idx, -1, 0);
-        }
-
-        return ret;
-}
-
-
-static int
-socket_event_poll_err (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                __socket_ioq_flush (this);
-                __socket_reset (this);
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        xlator_notify (this->xl, GF_EVENT_POLLERR, this);
-
-        return ret;
-}
-
-
-static int
-socket_event_poll_out (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                if (priv->connected == 1) {
-                        ret = __socket_ioq_churn (this);
-
-                        if (ret == -1) {
-                                __socket_disconnect (this);
-                        }
-                }
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        xlator_notify (this->xl, GF_EVENT_POLLOUT, this);
-
-        return ret;
-}
-
-
-static int
-__socket_proto_validate_header (transport_t *this,
-				struct socket_header *header,
-				size_t *size1_p, size_t *size2_p)
-{
-        size_t size1 = 0;
-	size_t size2 = 0;
-
-        if (strcmp (header->colonO, ":O")) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "socket header signature does not match :O (%x.%x.%x)",
-                        header->colonO[0], header->colonO[1],
-			header->colonO[2]);
-                return -1;
-        }
-
-        if (header->version != 42) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "socket header version does not match 42 != %d",
-			header->version);
-                return -1;
-        }
-
-        size1 = ntoh32 (header->size1);
-        size2 = ntoh32 (header->size2);
-
-        if (size1 <= 0 || size1 > 1048576) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "socket header has incorrect size1=%"GF_PRI_SIZET,
-			size1);
-                return -1;
-        }
-
-        if (size2 > (131072)) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "socket header has incorrect size2=%"GF_PRI_SIZET,
-			size2);
-                return -1;
-        }
-
-        if (size1_p)
-                *size1_p = size1;
-
-        if (size2_p)
-                *size2_p = size2;
-
-        return 0;
-}
-
-
-
-/* socket protocol state machine */
-
-static int
-__socket_proto_state_machine (transport_t *this)
-{
-        int                   ret = -1;
-        socket_private_t     *priv = NULL;
-        size_t                size1 = 0;
-	size_t                size2 = 0;
-        int                   previous_state = -1;
-	struct socket_header *hdr = NULL;
-        struct iobuf         *iobuf = NULL;
-
-
-        priv = this->private;
-
-	while (priv->incoming.state != SOCKET_PROTO_STATE_COMPLETE) {
-		/* debug check against infinite loops */
-		if (previous_state == priv->incoming.state) {
-			gf_log (this->xl->name, GF_LOG_DEBUG,
-				"state did not change! (%d) breaking",
-				previous_state);
-			ret = -1;
-			goto unlock;
-		}
-		previous_state = priv->incoming.state;
-
-		switch (priv->incoming.state) {
-
-		case SOCKET_PROTO_STATE_NADA:
-			priv->incoming.pending_vector =
-				priv->incoming.vector;
-
-			priv->incoming.pending_vector->iov_base =
-				&priv->incoming.header;
-
-			priv->incoming.pending_vector->iov_len  =
-				sizeof (struct socket_header);
-
-			priv->incoming.state =
-				SOCKET_PROTO_STATE_HEADER_COMING;
-			break;
-
-		case SOCKET_PROTO_STATE_HEADER_COMING:
-
-			ret = __socket_readv (this,
-					      priv->incoming.pending_vector, 1,
-					      &priv->incoming.pending_vector,
-					      NULL);
-			if (ret == 0) {
-				priv->incoming.state =
-					SOCKET_PROTO_STATE_HEADER_CAME;
-				break;
-			}
-
-			if (ret == -1) {
-				gf_log (this->xl->name, GF_LOG_TRACE,
-					"read (%s) in state %d (%s)",
-					strerror (errno),
-					SOCKET_PROTO_STATE_HEADER_COMING, 
-					this->peerinfo.identifier);
-				goto unlock;
-			}
-
-			if (ret > 0) {
-				gf_log (this->xl->name, GF_LOG_TRACE,
-					"partial header read on NB socket.");
-				goto unlock;
-			}
-			break;
-
-		case SOCKET_PROTO_STATE_HEADER_CAME:
-			hdr = &priv->incoming.header;
-			ret = __socket_proto_validate_header (this, hdr,
-							      &size1, &size2);
-
-			if (ret == -1) {
-				gf_log (this->xl->name, GF_LOG_ERROR,
-					"socket header validate failed (%s). "
-					"possible mismatch of transport-type "
-					"between server and client volumes, "
-					"or version mismatch",
-					this->peerinfo.identifier);
-                                        goto unlock;
-                        }
-
-                        priv->incoming.hdrlen = size1;
-                        priv->incoming.buflen = size2;
-
-                        /* TODO: use mem-pool */
-                        priv->incoming.hdr_p  = GF_MALLOC (size1, 
-                                                           gf_common_mt_char);
-                        if (size2) {
-                                /* TODO: sanity check size2 < page size
-                                 */
-                                iobuf = iobuf_get (this->xl->ctx->iobuf_pool);
-                                if (!iobuf) {
-                                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                                "unable to allocate IO buffer "
-                                                "for peer %s",
-                                                this->peerinfo.identifier);
-                                        ret = -ENOMEM;
-                                        goto unlock;
-                                }
-                                priv->incoming.iobuf = iobuf;
-                                priv->incoming.buf_p = iobuf->ptr;
-                        }
-
-                        priv->incoming.vector[0].iov_base =
-                                priv->incoming.hdr_p;
-
-                        priv->incoming.vector[0].iov_len  = size1;
-
-                        priv->incoming.vector[1].iov_base =
-                                priv->incoming.buf_p;
-
-                        priv->incoming.vector[1].iov_len  = size2;
-                        priv->incoming.count = size2 ? 2 : 1;
-
-                        priv->incoming.pending_vector =
-                                priv->incoming.vector;
-
-                        priv->incoming.pending_count  =
-                                priv->incoming.count;
-
-                        priv->incoming.state =
-                                SOCKET_PROTO_STATE_DATA_COMING;
-                        break;
-
-		case SOCKET_PROTO_STATE_DATA_COMING:
-
-			ret = __socket_readv (this,
-					      priv->incoming.pending_vector,
-					      priv->incoming.pending_count,
-					      &priv->incoming.pending_vector,
-					      &priv->incoming.pending_count);
-			if (ret == 0) {
-				priv->incoming.state =
-					SOCKET_PROTO_STATE_DATA_CAME;
-				break;
-			}
-
-			if (ret == -1) {
-				gf_log (this->xl->name, GF_LOG_DEBUG,
-					"read (%s) in state %d (%s)",
-					strerror (errno),
-					SOCKET_PROTO_STATE_DATA_COMING,
-					this->peerinfo.identifier);
-				goto unlock;
-			}
-
-			if (ret > 0) {
-				gf_log (this->xl->name, GF_LOG_TRACE,
-					"partial data read on NB socket");
-                                        goto unlock;
-			}
-			break;
-
-		case SOCKET_PROTO_STATE_DATA_CAME:
-			memset (&priv->incoming.vector, 0,
-				sizeof (priv->incoming.vector));
-			priv->incoming.pending_vector = NULL;
-			priv->incoming.pending_count  = 0;
-			priv->incoming.state = SOCKET_PROTO_STATE_COMPLETE;
-			break;
-
-		case SOCKET_PROTO_STATE_COMPLETE:
-			/* not reached */
-			break;
-
-		default:
-			gf_log (this->xl->name, GF_LOG_DEBUG,
-				"undefined state reached: %d",
-				priv->incoming.state);
-                                goto unlock;
-		}
-	}
-unlock:
-
-        return ret;
-}
-
-
-static int
-socket_proto_state_machine (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-	int               ret = 0;
-
-	priv = this->private;
-
-	pthread_mutex_lock (&priv->lock);
-	{
-		ret = __socket_proto_state_machine (this);
-	}
-        pthread_mutex_unlock (&priv->lock);
-
-	return ret;
-}
-
-
-static int
-socket_event_poll_in (transport_t *this)
-{
-        int ret = -1;
-
-        ret = socket_proto_state_machine (this);
-
-        /* call POLLIN on xlator even if complete block is not received,
-           just to keep the last_received timestamp ticking */
-
-        if (ret == 0)
-                ret = xlator_notify (this->xl, GF_EVENT_POLLIN, this);
-
-        return ret;
-}
-
-
-static int
-socket_connect_finish (transport_t *this)
-{
-        int               ret = -1;
-        socket_private_t *priv = NULL;
-        int               event = -1;
-        char              notify_xlator = 0;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-		if (priv->connected)
-			goto unlock;
-
-		ret = __socket_connect_finish (priv->sock);
-
-		if (ret == -1 && errno == EINPROGRESS)
-			ret = 1;
-
-		if (ret == -1 && errno != EINPROGRESS) {
-			if (!priv->connect_finish_log) {
-				gf_log (this->xl->name, GF_LOG_ERROR,
-					"connection to %s failed (%s)",
-                                        this->peerinfo.identifier,
-					strerror (errno));
-				priv->connect_finish_log = 1;
-			}
-			__socket_disconnect (this);
-			notify_xlator = 1;
-			event = GF_EVENT_POLLERR;
-			goto unlock;
-		}
-
-		if (ret == 0) {
-			notify_xlator = 1;
-
-			this->myinfo.sockaddr_len =
-				sizeof (this->myinfo.sockaddr);
-
-			ret = getsockname (priv->sock,
-					   SA (&this->myinfo.sockaddr),
-					   &this->myinfo.sockaddr_len);
-			if (ret == -1) {
-				gf_log (this->xl->name, GF_LOG_DEBUG,
-					"getsockname on (%d) failed (%s)", 
-					priv->sock, strerror (errno));
-				__socket_disconnect (this);
-				event = GF_EVENT_POLLERR;
-				goto unlock;
-			}
-
-			priv->connected = 1;
-			priv->connect_finish_log = 0;
-			event = GF_EVENT_CHILD_UP;
-			gf_get_transport_identifiers (this);
-		}
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-        if (notify_xlator)
-                xlator_notify (this->xl, event, this);
-
-        return 0;
-}
-
-
-static int
-socket_event_handler (int fd, int idx, void *data,
-                      int poll_in, int poll_out, int poll_err)
-{
-        transport_t      *this = NULL;
-        socket_private_t *priv = NULL;
-        int               ret = 0;
-
-        this = data;
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                priv->idx = idx;
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        if (!priv->connected) {
-                ret = socket_connect_finish (this);
-        }
-
-        if (!ret && poll_out) {
-                ret = socket_event_poll_out (this);
-        }
-
-        if (!ret && poll_in) {
-                ret = socket_event_poll_in (this);
-        }
-
-        if (ret < 0 || poll_err) {
-                socket_event_poll_err (this);
-                transport_unref (this);
-        }
-
-        return 0;
-}
-
-
-static int
-socket_server_event_handler (int fd, int idx, void *data,
-                             int poll_in, int poll_out, int poll_err)
-{
-        transport_t             *this = NULL;
-        socket_private_t        *priv = NULL;
-        int                      ret = 0;
-        int                      new_sock = -1;
-        transport_t             *new_trans = NULL;
-        struct sockaddr_storage  new_sockaddr = {0, };
-        socklen_t                addrlen = sizeof (new_sockaddr);
-        socket_private_t        *new_priv = NULL;
-	glusterfs_ctx_t         *ctx = NULL;
-
-        this = data;
-        priv = this->private;
-	ctx  = this->xl->ctx;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                priv->idx = idx;
-
-                if (poll_in) {
-                        new_sock = accept (priv->sock, SA (&new_sockaddr),
-					   &addrlen);
-
-                        if (new_sock == -1)
-                                goto unlock;
-
-                        if (!priv->bio) {
-                                ret = __socket_nonblock (new_sock);
-
-                                if (ret == -1) {
-                                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                                "NBIO on %d failed (%s)",
-                                                new_sock, strerror (errno));
-                                        close (new_sock);
-                                        goto unlock;
-                                }
-                        }
-
-                        if (priv->nodelay) {
-                                ret = __socket_nodelay (new_sock);
-                                if (ret == -1) {
-                                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                                "setsockopt() failed for "
-                                                "NODELAY (%s)",
-                                                strerror (errno));
-                                }
-                        }
-
-                        if (priv->keepalive) {
-                                ret = __socket_keepalive (new_sock,
-                                                          priv->keepaliveintvl);
-                                if (ret == -1)
-                                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                                "Failed to set keep-alive: %s",
-                                                strerror (errno));
-                        }
-
-                        new_trans = GF_CALLOC (1, sizeof (*new_trans), 
-                                               gf_common_mt_transport_t);
-                        new_trans->xl = this->xl;
-                        new_trans->fini = this->fini;
-
-                        memcpy (&new_trans->peerinfo.sockaddr, &new_sockaddr,
-				addrlen);
-                        new_trans->peerinfo.sockaddr_len = addrlen;
-
-                        new_trans->myinfo.sockaddr_len =
-				sizeof (new_trans->myinfo.sockaddr);
-
-                        ret = getsockname (new_sock, 
-                                           SA (&new_trans->myinfo.sockaddr),
-                                           &new_trans->myinfo.sockaddr_len);
-                        if (ret == -1) {
-                                gf_log (this->xl->name, GF_LOG_DEBUG,
-                                        "getsockname on %d failed (%s)", 
-                                        new_sock, strerror (errno));
-                                close (new_sock);
-                                goto unlock;
-                        }
-
-                        gf_get_transport_identifiers (new_trans);
-                        socket_init (new_trans);
-                        new_trans->ops = this->ops;
-                        new_trans->init = this->init;
-                        new_trans->fini = this->fini;
-
-                        new_priv = new_trans->private;
-
-                        pthread_mutex_lock (&new_priv->lock);
-                        {
-                                new_priv->sock = new_sock;
-                                new_priv->connected = 1;
-        
-                                transport_ref (new_trans);
-                                new_priv->idx =
-					event_register (ctx->event_pool,
-							new_sock,
-							socket_event_handler,
-							new_trans, 1, 0);
-
-                                if (new_priv->idx == -1)
-                                        ret = -1;
-                        }
-                        pthread_mutex_unlock (&new_priv->lock);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-        return ret;
-}
-
-
-static int
-socket_disconnect (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                ret = __socket_disconnect (this);
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        return ret;
-}
-
-
-static int
-socket_connect (transport_t *this)
-{
-        int                      ret = -1;
-	int                      sock = -1;
-        socket_private_t        *priv = NULL;
-        struct sockaddr_storage  sockaddr = {0, };
-        socklen_t                sockaddr_len = 0;
-	glusterfs_ctx_t         *ctx = NULL;
-        sa_family_t              sa_family = {0, };
-
-        priv = this->private;
-	ctx = this->xl->ctx;
-
-        if (!priv) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "connect() called on uninitialized transport");
-                goto err;
-        }
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                sock = priv->sock;
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        if (sock != -1) {
-                gf_log (this->xl->name, GF_LOG_TRACE,
-                        "connect () called on transport already connected");
-                ret = 0;
-                goto err;
-        }
-
-        ret = gf_socket_client_get_remote_sockaddr (this, SA (&sockaddr),
-                                                 &sockaddr_len, &sa_family);
-        if (ret == -1) {
-                /* logged inside client_get_remote_sockaddr */
-                goto err;
-        }
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                if (priv->sock != -1) {
-                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                "connect() -- already connected");
-                        goto unlock;
-                }
-
-                memcpy (&this->peerinfo.sockaddr, &sockaddr, sockaddr_len);
-                this->peerinfo.sockaddr_len = sockaddr_len;
-
-                priv->sock = socket (sa_family, SOCK_STREAM, 0);
-                if (priv->sock == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "socket creation failed (%s)",
-				strerror (errno));
-                        goto unlock;
-                }
-
-                /* Cant help if setting socket options fails. We can continue
-                 * working nonetheless.
-                 */
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "setting receive window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
-
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "setting send window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
-
-
-                if (priv->nodelay && priv->lowlat) {
-                        ret = __socket_nodelay (priv->sock);
-                        if (ret == -1) {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "setsockopt() failed for NODELAY (%s)",
-                                        strerror (errno));
-                        }
-                }
-
-                if (!priv->bio) {
-                        ret = __socket_nonblock (priv->sock);
-
-                        if (ret == -1) {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "NBIO on %d failed (%s)",
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                priv->sock = -1;
-                                goto unlock;
-                        }
-                }
-
-                if (priv->keepalive) {
-                        ret = __socket_keepalive (priv->sock,
-                                                  priv->keepaliveintvl);
-                        if (ret == -1)
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "Failed to set keep-alive: %s",
-                                        strerror (errno));
-                }
-
-                SA (&this->myinfo.sockaddr)->sa_family =
-			SA (&this->peerinfo.sockaddr)->sa_family;
-
-                ret = gf_client_bind (this, SA (&this->myinfo.sockaddr),
-				   &this->myinfo.sockaddr_len, priv->sock);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_WARNING,
-                                "client bind failed: %s", strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                ret = connect (priv->sock, SA (&this->peerinfo.sockaddr),
-			       this->peerinfo.sockaddr_len);
-
-                if (ret == -1 && errno != EINPROGRESS) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "connection attempt failed (%s)",
-				strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                priv->connected = 0;
-
-                transport_ref (this);
-
-                priv->idx = event_register (ctx->event_pool, priv->sock,
-                                            socket_event_handler, this, 1, 1);
-                if (priv->idx == -1)
-                        ret = -1;
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-err:
-        return ret;
-}
-
-
-static int
-socket_listen (transport_t *this)
-{
-        socket_private_t *       priv = NULL;
-        int                      ret = -1;
-	int                      sock = -1;
-        struct sockaddr_storage  sockaddr;
-        socklen_t                sockaddr_len;
-        peer_info_t             *myinfo = NULL;
-	glusterfs_ctx_t         *ctx = NULL;
-        sa_family_t              sa_family = {0, };
-
-	priv   = this->private;
-	myinfo = &this->myinfo;
-	ctx    = this->xl->ctx;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                sock = priv->sock;
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        if (sock != -1)  {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "already listening");
-                return ret;
-        }
-
-        ret = gf_socket_server_get_local_sockaddr (this, SA (&sockaddr),
-                                                &sockaddr_len, &sa_family);
-        if (ret == -1) {
-                return ret;
-        }
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                if (priv->sock != -1) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "already listening");
-                        goto unlock;
-                }
-
-                memcpy (&myinfo->sockaddr, &sockaddr, sockaddr_len);
-                myinfo->sockaddr_len = sockaddr_len;
-
-                priv->sock = socket (sa_family, SOCK_STREAM, 0);
-
-                if (priv->sock == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "socket creation failed (%s)",
-				strerror (errno));
-                        goto unlock;
-                }
-
-                /* Cant help if setting socket options fails. We can continue
-                 * working nonetheless.
-                 */
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "setting receive window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
-
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                               "setting send window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
-
-                if (priv->nodelay) {
-                        ret = __socket_nodelay (priv->sock);
-                        if (ret == -1) {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "setsockopt() failed for NODELAY (%s)",
-                                        strerror (errno));
-                        }
-                }
-
-                if (!priv->bio) {
-                        ret = __socket_nonblock (priv->sock);
-
-                        if (ret == -1) {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "NBIO on %d failed (%s)",
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                priv->sock = -1;
-                                goto unlock;
-                        }
-                }
-
-                ret = __socket_server_bind (this);
-
-                if (ret == -1) {
-                        /* logged inside __socket_server_bind() */
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                ret = listen (priv->sock, 10);
-
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "could not set socket %d to listen mode (%s)",
-				priv->sock, strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                transport_ref (this);
-
-                priv->idx = event_register (ctx->event_pool, priv->sock,
-                                            socket_server_event_handler,
-					    this, 1, 0);
-
-                if (priv->idx == -1) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "could not register socket %d with events",
-				priv->sock);
-                        ret = -1;
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-        return ret;
-}
-
-
-static int
-socket_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
-                struct iobuf **iobuf_p)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                if (priv->connected != 1) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "socket not connected to receive");
-                        goto unlock;
-                }
-
-                if (!hdr_p || !hdrlen_p || !iobuf_p) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "bad parameters %p %p %p",
-                                hdr_p, hdrlen_p, iobuf_p);
-                        goto unlock;
-                }
-
-                if (priv->incoming.state == SOCKET_PROTO_STATE_COMPLETE) {
-                        *hdr_p    = priv->incoming.hdr_p;
-                        *hdrlen_p = priv->incoming.hdrlen;
-                        *iobuf_p  = priv->incoming.iobuf;
-
-                        memset (&priv->incoming, 0, sizeof (priv->incoming));
-                        priv->incoming.state = SOCKET_PROTO_STATE_NADA;
-
-                        ret = 0;
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-        return ret;
-}
-
-
-/* TODO: implement per transfer limit */
-static int
-socket_submit (transport_t *this, char *buf, int len,
-               struct iovec *vector, int count,
-               struct iobref *iobref)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-        char              need_poll_out = 0;
-        char              need_append = 1;
-        struct ioq       *entry = NULL;
-	glusterfs_ctx_t  *ctx = NULL;
-
-        priv = this->private;
-	ctx  = this->xl->ctx;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                if (priv->connected != 1) {
-                        if (!priv->submit_log && !priv->connect_finish_log) {
-                                gf_log (this->xl->name, GF_LOG_DEBUG,
-                                        "not connected (priv->connected = %d)",
-                                        priv->connected);
-                                priv->submit_log = 1;
-                        }
-                        goto unlock;
-                }
-
-                priv->submit_log = 0;
-                entry = __socket_ioq_new (this, buf, len, vector, count, iobref);
-                if (!entry)
-                        goto unlock;
-
-                if (list_empty (&priv->ioq)) {
-                        ret = __socket_ioq_churn_entry (this, entry);
-
-                        if (ret == 0)
-                                need_append = 0;
-
-                        if (ret > 0)
-                                need_poll_out = 1;
-                }
-
-                if (need_append) {
-                        list_add_tail (&entry->list, &priv->ioq);
-                        ret = 0;
-                }
-
-                if (need_poll_out) {
-                        /* first entry to wait. continue writing on POLLOUT */
-                        priv->idx = event_select_on (ctx->event_pool,
-						     priv->sock,
-                                                     priv->idx, -1, 1);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-        return ret;
-}
-
-
-struct transport_ops tops = {
-        .listen     = socket_listen,
-        .connect    = socket_connect,
-        .disconnect = socket_disconnect,
-        .submit     = socket_submit,
-        .receive    = socket_receive
-};
-
-
-static int
-socket_init (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        gf_boolean_t      tmp_bool = 0;
-        uint64_t          windowsize = GF_DEFAULT_SOCKET_WINDOW_SIZE;
-        char             *optstr = NULL;
-        uint32_t          keepalive = 0;
-
-        if (this->private) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "double init attempted");
-                return -1;
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), 
-                         gf_common_mt_socket_private_t);
-        if (!priv) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "calloc (1, %"GF_PRI_SIZET") returned NULL",
-			sizeof (*priv));
-                return -1;
-        }
-
-        pthread_mutex_init (&priv->lock, NULL);
-
-        priv->sock = -1;
-        priv->idx = -1;
-        priv->connected = -1;
-
-        INIT_LIST_HEAD (&priv->ioq);
-
-        if (dict_get (this->xl->options, "non-blocking-io")) {
-                optstr = data_to_str (dict_get (this->xl->options,
-                                                          "non-blocking-io"));
-      
-                if (gf_string2boolean (optstr, &tmp_bool) == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "'non-blocking-io' takes only boolean options,"
-				" not taking any action");
-                        tmp_bool = 1;
-                }
-                priv->bio = 0;
-                if (!tmp_bool) {
-                        priv->bio = 1;
-                        gf_log (this->xl->name, GF_LOG_WARNING,
-                                "disabling non-blocking IO");
-                }
-        }
-
-        optstr = NULL;
-        
-        // By default, we enable NODELAY
-        priv->nodelay = 1;
-        if (dict_get (this->xl->options, "transport.socket.nodelay")) {
-                optstr = data_to_str (dict_get (this->xl->options,
-                                                "transport.socket.nodelay"));
-
-                if (gf_string2boolean (optstr, &tmp_bool) == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "'transport.socket.nodelay' takes only "
-                                 "boolean options, not taking any action");
-                        tmp_bool = 1;
-                }
-                if (!tmp_bool) {
-                        priv->nodelay = 0;
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "disabling nodelay");
-                }
-        }
-
-
-        optstr = NULL;
-        if (dict_get_str (this->xl->options, "transport.window-size",
-                          &optstr) == 0) {
-                if (gf_string2bytesize (optstr, &windowsize) != 0) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "invalid number format: %s", optstr);
-                        return -1;
-                }
-        }
-
-        optstr = NULL;
-
-        if (dict_get_str (this->xl->options, "transport.socket.lowlat",
-                          &optstr) == 0) {
-                priv->lowlat = 1;
-        }
-
-        /* Enable Keep-alive by default. */
-        priv->keepalive = 1;
-        priv->keepaliveintvl = GF_USE_DEFAULT_KEEPALIVE;
-        if (dict_get_str (this->xl->options, "transport.socket.keepalive",
-                          &optstr) == 0) {
-                if (gf_string2boolean (optstr, &tmp_bool) == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "'transport.socket.keepalive' takes only "
-                                 "boolean options, not taking any action");
-                        tmp_bool = 1;
-                }
-
-                if (!tmp_bool)
-                        priv->keepalive = 0;
-
-        }
-
-        if (dict_get_uint32 (this->xl->options,
-                             "transport.socket.keepalive-interval",
-                             &keepalive) == 0) {
-                priv->keepaliveintvl = keepalive;
-        }
-
-        priv->windowsize = (int)windowsize;
-        this->private = priv;
-
-        return 0;
-}
-
-
-void
-fini (transport_t *this)
-{
-        socket_private_t *priv = this->private;
-
-        if (!priv)
-                return;
-        this->private = NULL;
-        gf_log (this->xl->name, GF_LOG_TRACE,
-                "transport %p destroyed", this);
-
-        pthread_mutex_destroy (&priv->lock);
-        GF_FREE (priv);
-}
-
-
-int32_t
-init (transport_t *this)
-{
-        int ret = -1;
-
-        ret = socket_init (this);
-
-        if (ret == -1) {
-                gf_log (this->xl->name, GF_LOG_DEBUG, "socket_init() failed");
-        }
-  
-        return ret;
-}
-
-struct volume_options options[] = {
-        { .key   = {"remote-port", 
-                    "transport.remote-port",
-                    "transport.socket.remote-port"}, 
-          .type  = GF_OPTION_TYPE_INT 
-        },
-        { .key   = {"transport.socket.listen-port", "listen-port"}, 
-          .type  = GF_OPTION_TYPE_INT 
-        },
-        { .key   = {"transport.socket.bind-address", "bind-address" }, 
-          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS 
-        },
-        { .key   = {"transport.socket.connect-path", "connect-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = {"transport.socket.bind-path", "bind-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = {"transport.socket.listen-path", "listen-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = { "transport.address-family",
-                     "address-family" }, 
-          .value = {"inet", "inet6", "inet/inet6", "inet6/inet",
-                    "unix", "inet-sdp" },
-          .type  = GF_OPTION_TYPE_STR 
-        },
-
-        { .key   = {"non-blocking-io"}, 
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"transport.window-size"},
-          .type  = GF_OPTION_TYPE_SIZET,
-          .min   = GF_MIN_SOCKET_WINDOW_SIZE,
-          .max   = GF_MAX_SOCKET_WINDOW_SIZE,
-        },
-        { .key   = {"transport.socket.nodelay"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"transport.socket.lowlat"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"transport.socket.keepalive"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"transport.socket.keepalive-interval"},
-          .type  = GF_OPTION_TYPE_INT
-        },
-        { .key = {NULL} }
-};
-
diff --git a/xlators/protocol/legacy/transport/socket/src/socket.h b/xlators/protocol/legacy/transport/socket/src/socket.h
deleted file mode 100644
index e1dc9a91e..000000000
--- a/xlators/protocol/legacy/transport/socket/src/socket.h
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _SOCKET_H
-#define _SOCKET_H
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "event.h"
-#include "transport.h"
-#include "logging.h"
-#include "dict.h"
-#include "mem-pool.h"
-#include "socket-mem-types.h"
-
-#ifndef MAX_IOVEC
-#define MAX_IOVEC 16
-#endif /* MAX_IOVEC */
-
-#define GF_DEFAULT_SOCKET_LISTEN_PORT 6996
-
-/* This is the size set through setsockopt for
- * both the TCP receive window size and the
- * send buffer size.
- * Till the time iobuf size becomes configurable, this size is set to include
- * two iobufs + the GlusterFS protocol headers.
- * Linux allows us to over-ride the max values for the system.
- * Should we over-ride them? Because if we set a value larger than the default
- * setsockopt will fail. Having larger values might be beneficial for
- * IB links.
- */
-#define GF_DEFAULT_SOCKET_WINDOW_SIZE   (512 * GF_UNIT_KB)
-#define GF_MAX_SOCKET_WINDOW_SIZE       (1 * GF_UNIT_MB)
-#define GF_MIN_SOCKET_WINDOW_SIZE       (128 * GF_UNIT_KB)
-
-#define GF_USE_DEFAULT_KEEPALIVE        (-1)
-
-typedef enum {
-        SOCKET_PROTO_STATE_NADA = 0,
-        SOCKET_PROTO_STATE_HEADER_COMING,
-        SOCKET_PROTO_STATE_HEADER_CAME,
-        SOCKET_PROTO_STATE_DATA_COMING,
-        SOCKET_PROTO_STATE_DATA_CAME,
-        SOCKET_PROTO_STATE_COMPLETE,
-} socket_proto_state_t;
-
-struct socket_header {
-        char     colonO[3];
-        uint32_t size1;
-        uint32_t size2;
-        char     version;
-} __attribute__((packed));
-
-
-struct ioq {
-        union {
-                struct list_head list;
-                struct {
-                        struct ioq    *next;
-                        struct ioq    *prev;
-                };
-        };
-        struct socket_header  header;
-        struct iovec       vector[MAX_IOVEC];
-        int                count;
-        struct iovec      *pending_vector;
-        int                pending_count;
-        char              *buf;
-        struct iobref     *iobref;
-};
-
-
-typedef struct {
-        int32_t                sock;
-        int32_t                idx;
-        unsigned char          connected; // -1 = not connected. 0 = in progress. 1 = connected
-        char                   bio;
-        char                   connect_finish_log;
-        char                   submit_log;
-        union {
-                struct list_head     ioq;
-                struct {
-                        struct ioq        *ioq_next;
-                        struct ioq        *ioq_prev;
-                };
-        };
-        struct {
-                int                  state;
-                struct socket_header header;
-                char                *hdr_p;
-                size_t               hdrlen;
-                struct iobuf        *iobuf;
-                char                *buf_p;
-                size_t               buflen;
-                struct iovec         vector[2];
-                int                  count;
-                struct iovec        *pending_vector;
-                int                  pending_count;
-        } incoming;
-        pthread_mutex_t        lock;
-        int                    windowsize;
-        char                   lowlat;
-        char                   nodelay;
-        int                    keepalive;
-        int                    keepaliveintvl;
-} socket_private_t;
-
-
-#endif
diff --git a/xlators/protocol/server/src/Makefile.am b/xlators/protocol/server/src/Makefile.am
index 72a581ddf..6a18bf025 100644
--- a/xlators/protocol/server/src/Makefile.am
+++ b/xlators/protocol/server/src/Makefile.am
@@ -1,24 +1,26 @@
 xlator_LTLIBRARIES = server.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/protocol
 
-server_la_LDFLAGS = -module -avoidversion
+server_la_LDFLAGS = -module -avoid-version
 
 server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
 server_la_SOURCES = server.c server-resolve.c server-helpers.c  \
-	server3_1-fops.c server-handshake.c authenticate.c
+	server-rpc-fops.c server-handshake.c authenticate.c
 
 noinst_HEADERS = server.h server-helpers.h server-mem-types.h authenticate.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall    \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles  \
-	-DCONFDIR=\"$(sysconfdir)/glusterfs\" -D$(GF_HOST_OS)   \
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src \
+	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
 	-DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" \
-	$(GF_CFLAGS) -I$(top_srcdir)/xlators/protocol/lib/src   \
-	-I$(top_srcdir)/rpc/rpc-lib/src/       \
-	-I$(top_srcdir)/rpc/xdr/src/       \
-	-I$(top_srcdir)/contrib/md5/
+	-I$(top_srcdir)/xlators/protocol/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS) \
+            -DDATADIR=\"$(localstatedir)\"
 
 CLEANFILES = *~
diff --git a/xlators/protocol/server/src/authenticate.c b/xlators/protocol/server/src/authenticate.c
index de5bf9f43..d8d138a84 100644
--- a/xlators/protocol/server/src/authenticate.c
+++ b/xlators/protocol/server/src/authenticate.c
@@ -1,21 +1,14 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
 
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -31,15 +24,15 @@
 #include <errno.h>
 #include "authenticate.h"
 
-static void
+static int
 init (dict_t *this, char *key, data_t *value, void *data)
 {
-        void *handle = NULL;
-        char *auth_file = NULL;
-        auth_handle_t *auth_handle = NULL;
-        auth_fn_t authenticate = NULL;
-        int *error = NULL;
-        int  ret = 0;
+        void          *handle       = NULL;
+        char          *auth_file    = NULL;
+        auth_handle_t *auth_handle  = NULL;
+        auth_fn_t      authenticate = NULL;
+        int           *error        = NULL;
+        int            ret          = 0;
 
         /* It gets over written */
         error = data;
@@ -59,7 +52,7 @@ init (dict_t *this, char *key, data_t *value, void *data)
         if (-1 == ret) {
                 dict_set (this, key, data_from_dynptr (NULL, 0));
                 *error = -1;
-                return;
+                return -1;
         }
 
         handle = dlopen (auth_file, RTLD_LAZY);
@@ -69,7 +62,7 @@ init (dict_t *this, char *key, data_t *value, void *data)
                 dict_set (this, key, data_from_dynptr (NULL, 0));
                 GF_FREE (auth_file);
                 *error = -1;
-                return;
+                return -1;
         }
         GF_FREE (auth_file);
 
@@ -78,8 +71,9 @@ init (dict_t *this, char *key, data_t *value, void *data)
                 gf_log ("authenticate", GF_LOG_ERROR,
                         "dlsym(gf_auth) on %s\n", dlerror ());
                 dict_set (this, key, data_from_dynptr (NULL, 0));
+                dlclose (handle);
                 *error = -1;
-                return;
+                return -1;
         }
 
         auth_handle = GF_CALLOC (1, sizeof (*auth_handle),
@@ -87,10 +81,18 @@ init (dict_t *this, char *key, data_t *value, void *data)
         if (!auth_handle) {
                 dict_set (this, key, data_from_dynptr (NULL, 0));
                 *error = -1;
-                return;
+                dlclose (handle);
+                return -1;
         }
         auth_handle->vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
                                           gf_common_mt_volume_opt_list_t);
+        if (!auth_handle->vol_opt) {
+                dict_set (this, key, data_from_dynptr (NULL, 0));
+                *error = -1;
+                GF_FREE (auth_handle);
+                dlclose (handle);
+                return -1;
+        }
         auth_handle->vol_opt->given_opt = dlsym (handle, "options");
         if (auth_handle->vol_opt->given_opt == NULL) {
                 gf_log ("authenticate", GF_LOG_DEBUG,
@@ -102,47 +104,54 @@ init (dict_t *this, char *key, data_t *value, void *data)
 
         dict_set (this, key,
                   data_from_dynptr (auth_handle, sizeof (*auth_handle)));
+        return 0;
 }
 
-static void
+static int
 fini (dict_t *this, char *key, data_t *value, void *data)
 {
         auth_handle_t *handle = data_to_ptr (value);
         if (handle) {
                 dlclose (handle->handle);
         }
+        return 0;
+}
+
+static int
+_gf_auth_option_validate (dict_t *d, char *k, data_t *v, void *tmp)
+{
+        auth_handle_t *handle = NULL;
+        xlator_t      *xl = NULL;
+        int ret = 0;
+
+        xl = tmp;
+
+        handle = data_to_ptr (v);
+        if (!handle)
+                return 0;
+
+        list_add_tail (&(handle->vol_opt->list), &(xl->volume_options));
+
+        ret = xlator_options_validate_list (xl, xl->options,
+                                            handle->vol_opt, NULL);
+        if (ret) {
+                gf_log ("authenticate", GF_LOG_ERROR,
+                        "volume option validation failed");
+                return -1;
+        }
+        return 0;
 }
 
 int32_t
 gf_auth_init (xlator_t *xl, dict_t *auth_modules)
 {
         int ret = 0;
-        auth_handle_t *handle = NULL;
-        data_pair_t *pair = NULL;
 
         dict_foreach (auth_modules, init, &ret);
         if (ret)
                 goto out;
 
-        pair = auth_modules->members_list;
-        while (pair) {
-                handle = data_to_ptr (pair->value);
-                if (!handle) {
-                        pair = pair->next;
-                        continue;
-                }
-
-                list_add_tail (&(handle->vol_opt->list),
-                               &(xl->volume_options));
-                ret = xlator_options_validate_list (xl, xl->options,
-                                                    handle->vol_opt, NULL);
-
-                if (ret)
-                        gf_log ("authenticate", GF_LOG_ERROR,
-                                "volume option validation failed");
-
-                pair = pair->next;
-        }
+        ret = dict_foreach (auth_modules, _gf_auth_option_validate, xl);
 
 out:
         if (ret) {
@@ -156,7 +165,7 @@ out:
 static dict_t *__input_params;
 static dict_t *__config_params;
 
-void
+int
 map (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t *res = data;
@@ -171,15 +180,16 @@ map (dict_t *this, char *key, data_t *value, void *data)
         } else {
                 dict_set (res, key, int_to_data (AUTH_DONT_CARE));
         }
+        return 0;
 }
 
-void
+int
 reduce (dict_t *this, char *key, data_t *value, void *data)
 {
         int64_t val = 0;
         int64_t *res = data;
         if (!data)
-                return;
+                return 0;
 
         val = data_to_int64 (value);
         switch (val)
@@ -196,6 +206,7 @@ reduce (dict_t *this, char *key, data_t *value, void *data)
         case AUTH_DONT_CARE:
                 break;
         }
+        return 0;
 }
 
 
diff --git a/xlators/protocol/server/src/authenticate.h b/xlators/protocol/server/src/authenticate.h
index bb9f77e45..d4d43e498 100644
--- a/xlators/protocol/server/src/authenticate.h
+++ b/xlators/protocol/server/src/authenticate.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _AUTHENTICATE_H
diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c
index 2c8cf059b..d4941011d 100644
--- a/xlators/protocol/server/src/server-handshake.c
+++ b/xlators/protocol/server/src/server-handshake.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -40,7 +31,7 @@ gf_compare_client_version (rpcsvc_request_t *req, int fop_prognum,
 {
         int ret = -1;
         /* TODO: think.. */
-        if (glusterfs3_1_fop_prog.prognum == fop_prognum)
+        if (glusterfs3_3_fop_prog.prognum == fop_prognum)
                 ret = 0;
 
         return ret;
@@ -103,9 +94,9 @@ _volfile_update_checksum (xlator_t *this, char *key, uint32_t checksum)
 
         if (temp_volfile->checksum != checksum) {
                 gf_log (this->name, GF_LOG_INFO,
-                        "the volume file got modified between earlier access "
-                        "and now, this may lead to inconsistency between "
-                        "clients, advised to remount client");
+                        "the volume file was modified between a prior access "
+                        "and now. This may lead to inconsistency between "
+                        "clients, you are advised to remount client");
                 temp_volfile->checksum  = checksum;
         }
 
@@ -118,10 +109,10 @@ static size_t
 getspec_build_volfile_path (xlator_t *this, const char *key, char *path,
                             size_t path_len)
 {
-        int              ret = -1;
+        char            *filename      = NULL;
+        server_conf_t   *conf          = NULL;
+        int              ret           = -1;
         int              free_filename = 0;
-        char            *filename = NULL;
-        server_conf_t   *conf = NULL;
         char             data_key[256] = {0,};
 
         conf = this->private;
@@ -190,7 +181,7 @@ int
 _validate_volfile_checksum (xlator_t *this, char *key,
                             uint32_t checksum)
 {
-        char                 filename[ZR_PATH_MAX] = {0,};
+        char                 filename[PATH_MAX] = {0,};
         server_conf_t       *conf         = NULL;
         struct _volfile_ctx *temp_volfile = NULL;
         int                  ret          = 0;
@@ -253,7 +244,7 @@ server_getspec (rpcsvc_request_t *req)
         int32_t              op_errno               = ENOENT;
         int32_t              spec_fd                = -1;
         size_t               file_len               = 0;
-        char                 filename[ZR_PATH_MAX]  = {0,};
+        char                 filename[PATH_MAX]  = {0,};
         struct stat          stbuf                  = {0,};
         uint32_t             checksum               = 0;
         char                *key                    = NULL;
@@ -264,7 +255,9 @@ server_getspec (rpcsvc_request_t *req)
 
         this = req->svc->mydata;
         conf = this->private;
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gf_getspec_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_getspec_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 op_errno = EINVAL;
@@ -311,8 +304,6 @@ server_getspec (rpcsvc_request_t *req)
                         goto fail;
                 }
                 ret = read (spec_fd, rsp.spec, file_len);
-
-                close (spec_fd);
         }
 
         /* convert to XDR */
@@ -323,6 +314,9 @@ fail:
         rsp.op_errno = gf_errno_to_error (op_errno);
         rsp.op_ret   = ret;
 
+        if (spec_fd != -1)
+                close (spec_fd);
+
         server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
                              (xdrproc_t)xdr_gf_getspec_rsp);
 
@@ -335,14 +329,15 @@ server_setvolume (rpcsvc_request_t *req)
 {
         gf_setvolume_req     args          = {{0,},};
         gf_setvolume_rsp     rsp           = {0,};
-        server_connection_t *conn          = NULL;
+        client_t            *client        = NULL;
+        server_ctx_t        *serv_ctx      = NULL;
         server_conf_t       *conf          = NULL;
         peer_info_t         *peerinfo      = NULL;
         dict_t              *reply         = NULL;
         dict_t              *config_params = NULL;
         dict_t              *params        = NULL;
         char                *name          = NULL;
-        char                *process_uuid  = NULL;
+        char                *client_uid    = NULL;
         char                *clnt_version  = NULL;
         xlator_t            *xl            = NULL;
         char                *msg           = NULL;
@@ -354,11 +349,15 @@ server_setvolume (rpcsvc_request_t *req)
         int32_t              op_errno      = EINVAL;
         int32_t              fop_version   = 0;
         int32_t              mgmt_version  = 0;
+        uint32_t             lk_version    = 0;
         char                *buf           = NULL;
+        gf_boolean_t        cancelled      = _gf_false;
 
         params = dict_new ();
         reply  = dict_new ();
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gf_setvolume_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_setvolume_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
@@ -395,7 +394,7 @@ server_setvolume (rpcsvc_request_t *req)
         params->extra_free = buf;
         buf = NULL;
 
-        ret = dict_get_str (params, "process-uuid", &process_uuid);
+        ret = dict_get_str (params, "process-uuid", &client_uid);
         if (ret < 0) {
                 ret = dict_set_str (reply, "ERROR",
                                     "UUID not specified");
@@ -408,10 +407,46 @@ server_setvolume (rpcsvc_request_t *req)
                 goto fail;
         }
 
+        /*lk_verion :: [1..2^31-1]*/
+        ret = dict_get_uint32 (params, "clnt-lk-version", &lk_version);
+        if (ret < 0) {
+                ret = dict_set_str (reply, "ERROR",
+                                    "lock state version not supplied");
+                if (ret < 0)
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "failed to set error msg");
+
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto fail;
+        }
+
+        client = gf_client_get (this, &req->cred, client_uid);
+        if (client == NULL) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto fail;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Connected to %s", client->client_uid);
+        cancelled = server_cancel_grace_timer (this, client);
+        if (cancelled)//Do gf_client_put on behalf of grace-timer-handler.
+                gf_client_put (client, NULL);
+
+        serv_ctx = server_ctx_get (client, client->this);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto fail;
+        }
+
+        if (serv_ctx->lk_version != 0 &&
+            serv_ctx->lk_version != lk_version) {
+                (void) server_connection_cleanup (this, client,
+                                                  INTERNAL_LOCKS | POSIX_LOCKS);
+        }
 
-        conn = server_connection_get (this, process_uuid);
-        if (req->trans->xl_private != conn)
-                req->trans->xl_private = conn;
+        if (req->trans->xl_private != client)
+                req->trans->xl_private = client;
 
         ret = dict_get_int32 (params, "fops-version", &fop_version);
         if (ret < 0) {
@@ -536,10 +571,10 @@ server_setvolume (rpcsvc_request_t *req)
 
                 gf_log (this->name, GF_LOG_INFO,
                         "accepted client from %s (version: %s)",
-                        (peerinfo) ? peerinfo->identifier : "<>",
+                        client->client_uid,
                         (clnt_version) ? clnt_version : "old");
                 op_ret = 0;
-                conn->bound_xl = xl;
+                client->bound_xl = xl;
                 ret = dict_set_str (reply, "ERROR", "Success");
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_DEBUG,
@@ -547,7 +582,7 @@ server_setvolume (rpcsvc_request_t *req)
         } else {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Cannot authenticate client from %s %s",
-                        (peerinfo) ? peerinfo->identifier : "<>",
+                        client->client_uid,
                         (clnt_version) ? clnt_version : "old");
 
                 op_ret = -1;
@@ -556,11 +591,10 @@ server_setvolume (rpcsvc_request_t *req)
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "failed to set error msg");
-
                 goto fail;
         }
 
-        if (conn->bound_xl == NULL) {
+        if (client->bound_xl == NULL) {
                 ret = dict_set_str (reply, "ERROR",
                                     "Check volfile and handshake "
                                     "options in protocol/client");
@@ -573,20 +607,21 @@ server_setvolume (rpcsvc_request_t *req)
                 goto fail;
         }
 
-        if ((conn->bound_xl != NULL) &&
+        if ((client->bound_xl != NULL) &&
             (ret >= 0)                   &&
-            (conn->bound_xl->itable == NULL)) {
+            (client->bound_xl->itable == NULL)) {
                 /* create inode table for this bound_xl, if one doesn't
                    already exist */
 
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating inode table with lru_limit=%"PRId32", "
                         "xlator=%s", conf->inode_lru_limit,
-                        conn->bound_xl->name);
+                        client->bound_xl->name);
 
                 /* TODO: what is this ? */
-                conn->bound_xl->itable = inode_table_new (conf->inode_lru_limit,
-                                                          conn->bound_xl);
+                client->bound_xl->itable =
+                        inode_table_new (conf->inode_lru_limit,
+                                         client->bound_xl);
         }
 
         ret = dict_set_str (reply, "process-uuid",
@@ -595,6 +630,11 @@ server_setvolume (rpcsvc_request_t *req)
                 gf_log (this->name, GF_LOG_DEBUG,
                         "failed to set 'process-uuid'");
 
+        ret = dict_set_uint32 (reply, "clnt-lk-version", serv_ctx->lk_version);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set 'clnt-lk-version'");
+
         ret = dict_set_uint64 (reply, "transport-ptr",
                                ((uint64_t) (long) req->trans));
         if (ret)
@@ -627,23 +667,34 @@ fail:
         rsp.op_ret   = op_ret;
         rsp.op_errno = gf_errno_to_error (op_errno);
 
+        /* if bound_xl is NULL or something fails, then put the connection
+         * back. Otherwise the connection would have been added to the
+         * list of connections the server is maintaining and might segfault
+         * during statedump when bound_xl of the connection is accessed.
+         */
+        if (op_ret && !xl) {
+                /* We would have set the xl_private of the transport to the
+                 * @conn. But if we have put the connection i.e shutting down
+                 * the connection, then we should set xl_private to NULL as it
+                 * would be pointing to a freed memory and would segfault when
+                 * accessed upon getting DISCONNECT.
+                 */
+                gf_client_put (client, NULL);
+                req->trans->xl_private = NULL;
+        }
         server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
                              (xdrproc_t)xdr_gf_setvolume_rsp);
 
 
-        if (args.dict.dict_val)
-                free (args.dict.dict_val);
+        free (args.dict.dict_val);
 
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
+        GF_FREE (rsp.dict.dict_val);
 
         dict_unref (params);
         dict_unref (reply);
         dict_unref (config_params);
 
-        if (buf) {
-                GF_FREE (buf);
-        }
+        GF_FREE (buf);
 
         return 0;
 }
@@ -663,12 +714,61 @@ server_ping (rpcsvc_request_t *req)
         return 0;
 }
 
+int
+server_set_lk_version (rpcsvc_request_t *req)
+{
+        int                 op_ret   = -1;
+        int                 op_errno = EINVAL;
+        gf_set_lk_ver_req   args     = {0,};
+        gf_set_lk_ver_rsp   rsp      = {0,};
+        client_t           *client   = NULL;
+        server_ctx_t       *serv_ctx = NULL;
+        xlator_t           *this     = NULL;
+
+        this = req->svc->mydata;
+        //TODO: Decide on an appropriate errno for the error-path
+        //below
+        if (!this)
+                goto fail;
+
+        op_ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_set_lk_ver_req);
+        if (op_ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        client = gf_client_get (this, &req->cred, args.uid);
+        serv_ctx = server_ctx_get (client, client->this);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto fail;
+        }
+
+        serv_ctx->lk_version = args.lk_ver;
+        gf_client_put (client, NULL);
+
+        rsp.lk_ver   = args.lk_ver;
+
+        op_ret = 0;
+fail:
+        rsp.op_ret   = op_ret;
+        rsp.op_errno = op_errno;
+        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_set_lk_ver_rsp);
+
+        free (args.uid);
+
+        return 0;
+}
 
 rpcsvc_actor_t gluster_handshake_actors[] = {
-        [GF_HNDSK_NULL]      = {"NULL",      GF_HNDSK_NULL,      server_null, NULL, NULL, 0},
-        [GF_HNDSK_SETVOLUME] = {"SETVOLUME", GF_HNDSK_SETVOLUME, server_setvolume, NULL, NULL, 0},
-        [GF_HNDSK_GETSPEC]   = {"GETSPEC",   GF_HNDSK_GETSPEC,   server_getspec, NULL, NULL, 0},
-        [GF_HNDSK_PING]      = {"PING",      GF_HNDSK_PING,      server_ping, NULL, NULL, 0},
+        [GF_HNDSK_NULL]       = {"NULL",       GF_HNDSK_NULL,       server_null,           NULL, 0, DRC_NA},
+        [GF_HNDSK_SETVOLUME]  = {"SETVOLUME",  GF_HNDSK_SETVOLUME,  server_setvolume,      NULL, 0, DRC_NA},
+        [GF_HNDSK_GETSPEC]    = {"GETSPEC",    GF_HNDSK_GETSPEC,    server_getspec,        NULL, 0, DRC_NA},
+        [GF_HNDSK_PING]       = {"PING",       GF_HNDSK_PING,       server_ping,           NULL, 0, DRC_NA},
+        [GF_HNDSK_SET_LK_VER] = {"SET_LK_VER", GF_HNDSK_SET_LK_VER, server_set_lk_version, NULL, 0, DRC_NA},
 };
 
 
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 4980424d3..f0b040c74 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -35,6 +26,9 @@ server_decode_groups (call_frame_t *frame, rpcsvc_request_t *req)
         GF_VALIDATE_OR_GOTO ("server", frame, out);
         GF_VALIDATE_OR_GOTO ("server", req, out);
 
+        if (call_stack_alloc_groups (frame->root, req->auxgidcount) != 0)
+                return -1;
+
         frame->root->ngrps = req->auxgidcount;
         if (frame->root->ngrps == 0)
                 return 0;
@@ -48,6 +42,7 @@ out:
         return 0;
 }
 
+
 void
 server_loc_wipe (loc_t *loc)
 {
@@ -61,19 +56,16 @@ server_loc_wipe (loc_t *loc)
                 loc->inode = NULL;
         }
 
-        if (loc->path)
-                GF_FREE ((void *)loc->path);
+        GF_FREE ((void *)loc->path);
 }
 
 
 void
 server_resolve_wipe (server_resolve_t *resolve)
 {
-        if (resolve->path)
-                GF_FREE ((void *)resolve->path);
+        GF_FREE ((void *)resolve->path);
 
-        if (resolve->bname)
-                GF_FREE ((void *)resolve->bname);
+        GF_FREE ((void *)resolve->bname);
 
         loc_wipe (&resolve->resolve_loc);
 }
@@ -82,11 +74,6 @@ server_resolve_wipe (server_resolve_t *resolve)
 void
 free_state (server_state_t *state)
 {
-        if (state->conn) {
-                //xprt_svc_unref (state->conn);
-                state->conn = NULL;
-        }
-
         if (state->xprt) {
                 rpc_transport_unref (state->xprt);
                 state->xprt = NULL;
@@ -116,11 +103,14 @@ free_state (server_state_t *state)
                 state->dict = NULL;
         }
 
-        if (state->volume)
-                GF_FREE ((void *)state->volume);
+        if (state->xdata) {
+                dict_unref (state->xdata);
+                state->xdata = NULL;
+        }
 
-        if (state->name)
-                GF_FREE ((void *)state->name);
+        GF_FREE ((void *)state->volume);
+
+        GF_FREE ((void *)state->name);
 
         server_loc_wipe (&state->loc);
         server_loc_wipe (&state->loc2);
@@ -132,301 +122,26 @@ free_state (server_state_t *state)
 }
 
 
-int
-gf_add_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc, fd_t *fd, pid_t pid, gf_lkowner_t *owner,
-               glusterfs_fop_t type)
-{
-        int32_t         ret = -1;
-        struct _locker *new = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", table, out);
-        GF_VALIDATE_OR_GOTO ("server", volume, out);
-
-        new = GF_CALLOC (1, sizeof (struct _locker), gf_server_mt_locker_t);
-        if (new == NULL) {
-                goto out;
-        }
-        INIT_LIST_HEAD (&new->lockers);
-
-        new->volume = gf_strdup (volume);
-
-        if (fd == NULL) {
-                loc_copy (&new->loc, loc);
-        } else {
-                new->fd = fd_ref (fd);
-        }
-
-        new->pid   = pid;
-        new->owner = *owner;
-
-        LOCK (&table->lock);
-        {
-                if (type == GF_FOP_ENTRYLK)
-                        list_add_tail (&new->lockers, &table->entrylk_lockers);
-                else
-                        list_add_tail (&new->lockers, &table->inodelk_lockers);
-        }
-        UNLOCK (&table->lock);
-out:
-        return ret;
-}
-
-
-int
-gf_del_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc, fd_t *fd, gf_lkowner_t *owner,
-               glusterfs_fop_t type)
-{
-        struct _locker    *locker = NULL;
-        struct _locker    *tmp = NULL;
-        int32_t            ret = -1;
-        struct list_head  *head = NULL;
-        struct list_head   del;
-
-        GF_VALIDATE_OR_GOTO ("server", table, out);
-        GF_VALIDATE_OR_GOTO ("server", volume, out);
-
-        INIT_LIST_HEAD (&del);
-
-        LOCK (&table->lock);
-        {
-                if (type == GF_FOP_ENTRYLK) {
-                        head = &table->entrylk_lockers;
-                } else {
-                        head = &table->inodelk_lockers;
-                }
-
-                list_for_each_entry_safe (locker, tmp, head, lockers) {
-                        if (!is_same_lkowner (&locker->owner, owner) ||
-                            strcmp (locker->volume, volume))
-                                continue;
-
-                        if (locker->fd && fd && (locker->fd == fd))
-                                list_move_tail (&locker->lockers, &del);
-                        else if (locker->loc.inode && loc &&
-                                 (locker->loc.inode == loc->inode))
-                                list_move_tail (&locker->lockers, &del);
-                }
-        }
-        UNLOCK (&table->lock);
-
-        tmp = NULL;
-        locker = NULL;
-
-        list_for_each_entry_safe (locker, tmp, &del, lockers) {
-                list_del_init (&locker->lockers);
-                if (locker->fd)
-                        fd_unref (locker->fd);
-                else
-                        loc_wipe (&locker->loc);
-
-                GF_FREE (locker->volume);
-                GF_FREE (locker);
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-static struct _lock_table *
-gf_lock_table_new (void)
-{
-        struct _lock_table *new = NULL;
-
-        new = GF_CALLOC (1, sizeof (struct _lock_table), gf_server_mt_lock_table_t);
-        if (new == NULL) {
-                goto out;
-        }
-        INIT_LIST_HEAD (&new->entrylk_lockers);
-        INIT_LIST_HEAD (&new->inodelk_lockers);
-        LOCK_INIT (&new->lock);
-out:
-        return new;
-}
-
-static int
-server_nop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno)
-{
-        int             ret   = -1;
-        server_state_t *state = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
-        GF_VALIDATE_OR_GOTO ("server", cookie, out);
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-
-        state = CALL_STATE(frame);
-
-        if (state)
-                free_state (state);
-        STACK_DESTROY (frame->root);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int
-do_lock_table_cleanup (xlator_t *this, server_connection_t *conn,
-                       call_frame_t *frame, struct _lock_table *ltable)
-{
-        struct list_head  inodelk_lockers, entrylk_lockers;
-        call_frame_t     *tmp_frame = NULL;
-        struct gf_flock      flock = {0, };
-        xlator_t         *bound_xl = NULL;
-        struct _locker   *locker = NULL, *tmp = NULL;
-        int               ret = -1;
-        char             *path = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
-        GF_VALIDATE_OR_GOTO ("server", ltable, out);
-
-        bound_xl = conn->bound_xl;
-        INIT_LIST_HEAD (&inodelk_lockers);
-        INIT_LIST_HEAD (&entrylk_lockers);
-
-        LOCK (&ltable->lock);
-        {
-                list_splice_init (&ltable->inodelk_lockers,
-                                  &inodelk_lockers);
-
-                list_splice_init (&ltable->entrylk_lockers, &entrylk_lockers);
-        }
-        UNLOCK (&ltable->lock);
-
-        GF_FREE (ltable);
-
-        flock.l_type  = F_UNLCK;
-        flock.l_start = 0;
-        flock.l_len   = 0;
-        list_for_each_entry_safe (locker,
-                                  tmp, &inodelk_lockers, lockers) {
-                tmp_frame = copy_frame (frame);
-                if (tmp_frame == NULL) {
-                        goto out;
-                }
-                /*
-                  lock owner = 0 is a special case that tells posix-locks
-                  to release all locks from this transport
-                */
-                tmp_frame->root->pid         = 0;
-                tmp_frame->root->trans       = conn;
-                memset (&tmp_frame->root->lk_owner, 0, sizeof (gf_lkowner_t));
-
-                if (locker->fd) {
-                        GF_ASSERT (locker->fd->inode);
-
-                        ret = inode_path (locker->fd->inode, NULL, &path);
-
-                        if (ret > 0) {
-                                gf_log (this->name, GF_LOG_INFO, "finodelk "
-                                        "released on %s", path);
-                                GF_FREE (path);
-                        } else {
-
-                                gf_log (this->name, GF_LOG_INFO, "finodelk "
-                                        "released on inode with gfid %s",
-                                        uuid_utoa (locker->fd->inode->gfid));
-                        }
-
-                        STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                    bound_xl->fops->finodelk,
-                                    locker->volume,
-                                    locker->fd, F_SETLK, &flock);
-                        fd_unref (locker->fd);
-                } else {
-                        gf_log (this->name, GF_LOG_INFO, "inodelk released "
-                                "on %s", locker->loc.path);
-
-                        STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                    bound_xl->fops->inodelk,
-                                    locker->volume,
-                                    &(locker->loc), F_SETLK, &flock);
-                        loc_wipe (&locker->loc);
-                }
-
-                GF_FREE (locker->volume);
-
-                list_del_init (&locker->lockers);
-                GF_FREE (locker);
-        }
-
-        tmp = NULL;
-        locker = NULL;
-        list_for_each_entry_safe (locker, tmp, &entrylk_lockers, lockers) {
-                tmp_frame = copy_frame (frame);
-
-                tmp_frame->root->pid         = 0;
-                tmp_frame->root->trans       = conn;
-                memset (&tmp_frame->root->lk_owner, 0, sizeof (gf_lkowner_t));
-
-                if (locker->fd) {
-                        GF_ASSERT (locker->fd->inode);
-
-                        ret = inode_path (locker->fd->inode, NULL, &path);
-
-                        if (ret > 0) {
-                                gf_log (this->name, GF_LOG_INFO, "fentrylk "
-                                        "released on %s", path);
-                                GF_FREE (path);
-                        }  else {
-
-                                gf_log (this->name, GF_LOG_INFO, "fentrylk "
-                                        "released on inode with gfid %s",
-                                        uuid_utoa (locker->fd->inode->gfid));
-                        }
-
-                        STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                    bound_xl->fops->fentrylk,
-                                    locker->volume,
-                                    locker->fd, NULL,
-                                    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                        fd_unref (locker->fd);
-                } else {
-                        gf_log (this->name, GF_LOG_INFO, "entrylk released "
-                                "on %s", locker->loc.path);
-
-                        STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                    bound_xl->fops->entrylk,
-                                    locker->volume,
-                                    &(locker->loc), NULL,
-                                    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                        loc_wipe (&locker->loc);
-                }
-
-                GF_FREE (locker->volume);
-
-                list_del_init (&locker->lockers);
-                GF_FREE (locker);
-        }
-        ret = 0;
-
-out:
-        return ret;
-}
-
-
 static int
 server_connection_cleanup_flush_cbk (call_frame_t *frame, void *cookie,
                                      xlator_t *this, int32_t op_ret,
-                                     int32_t op_errno)
+                                     int32_t op_errno, dict_t *xdata)
 {
-        int32_t ret = -1;
-        fd_t *fd = NULL;
+        int32_t    ret    = -1;
+        fd_t      *fd     = NULL;
+        client_t  *client = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", this, out);
         GF_VALIDATE_OR_GOTO ("server", cookie, out);
         GF_VALIDATE_OR_GOTO ("server", frame, out);
 
         fd = frame->local;
+        client = frame->root->client;
 
         fd_unref (fd);
         frame->local = NULL;
 
+        gf_client_unref (client);
         STACK_DESTROY (frame->root);
 
         ret = 0;
@@ -435,9 +150,8 @@ out:
 }
 
 
-int
-do_fd_cleanup (xlator_t *this, server_connection_t *conn, call_frame_t *frame,
-               fdentry_t *fdentries, int fd_count)
+static int
+do_fd_cleanup (xlator_t *this, client_t* client, fdentry_t *fdentries, int fd_count)
 {
         fd_t               *fd = NULL;
         int                 i = 0, ret = -1;
@@ -446,16 +160,14 @@ do_fd_cleanup (xlator_t *this, server_connection_t *conn, call_frame_t *frame,
         char               *path     = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
         GF_VALIDATE_OR_GOTO ("server", fdentries, out);
 
-        bound_xl = conn->bound_xl;
+        bound_xl = client->bound_xl;
         for (i = 0;i < fd_count; i++) {
                 fd = fdentries[i].fd;
 
                 if (fd != NULL) {
-                        tmp_frame = copy_frame (frame);
+                        tmp_frame = create_frame (this, this->ctx->pool);
                         if (tmp_frame == NULL) {
                                 goto out;
                         }
@@ -465,26 +177,26 @@ do_fd_cleanup (xlator_t *this, server_connection_t *conn, call_frame_t *frame,
                         ret = inode_path (fd->inode, NULL, &path);
 
                         if (ret > 0) {
-                                gf_log (this->name, GF_LOG_INFO, "fd cleanup on "
-                                        "%s", path);
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "fd cleanup on %s", path);
                                 GF_FREE (path);
                         }  else {
 
-                                gf_log (this->name, GF_LOG_INFO, "fd cleanup on"
-                                        " inode with gfid %s",
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "fd cleanup on inode with gfid %s",
                                         uuid_utoa (fd->inode->gfid));
                         }
 
                         tmp_frame->local = fd;
 
                         tmp_frame->root->pid = 0;
-                        tmp_frame->root->trans = conn;
+                        gf_client_ref (client);
                         memset (&tmp_frame->root->lk_owner, 0,
                                 sizeof (gf_lkowner_t));
 
                         STACK_WIND (tmp_frame,
                                     server_connection_cleanup_flush_cbk,
-                                    bound_xl, bound_xl->fops->flush, fd);
+                                    bound_xl, bound_xl->fops->flush, fd, NULL);
                 }
         }
 
@@ -495,363 +207,72 @@ out:
         return ret;
 }
 
+
 int
-do_connection_cleanup (xlator_t *this, server_connection_t *conn,
-                       struct _lock_table *ltable, fdentry_t *fdentries, int fd_count)
+server_connection_cleanup (xlator_t *this, client_t *client,
+                           int32_t flags)
 {
-        int             ret = 0;
-        int             saved_ret = 0;
-        call_frame_t   *frame = NULL;
-        server_state_t *state = NULL;
+        server_ctx_t        *serv_ctx  = NULL;
+        fdentry_t           *fdentries = NULL;
+        uint32_t             fd_count  = 0;
+        int                  cd_ret    = 0;
+        int                  ret       = 0;
 
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-        GF_VALIDATE_OR_GOTO ("server", fdentries, out);
-        GF_VALIDATE_OR_GOTO ("server", ltable, out);
-
-        frame = create_frame (this, this->ctx->pool);
-        if (frame == NULL) {
-                goto out;
-        }
-
-        saved_ret = do_lock_table_cleanup (this, conn, frame, ltable);
+        GF_VALIDATE_OR_GOTO (this->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, client, out);
+        GF_VALIDATE_OR_GOTO (this->name, flags, out);
 
-        if (fdentries != NULL) {
-                ret = do_fd_cleanup (this, conn, frame, fdentries, fd_count);
-        }
+        serv_ctx = server_ctx_get (client, client->this);
 
-        state = CALL_STATE (frame);
-        if (state)
-                GF_FREE (state);
-
-        STACK_DESTROY (frame->root);
-
-        if (saved_ret || ret) {
-                ret = -1;
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
         }
 
-out:
-        return ret;
-}
-
-
-int
-server_connection_cleanup (xlator_t *this, server_connection_t *conn)
-{
-        struct _lock_table *ltable = NULL;
-        fdentry_t          *fdentries = NULL;
-        uint32_t            fd_count = 0;
-        int                 ret = 0;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-
-        pthread_mutex_lock (&conn->lock);
+        LOCK (&serv_ctx->fdtable_lock);
         {
-                if (conn->ltable) {
-                        ltable = conn->ltable;
-                        conn->ltable = gf_lock_table_new ();
-                }
-
-                if (conn->fdtable)
-                        fdentries = gf_fd_fdtable_get_all_fds (conn->fdtable,
+                if (serv_ctx->fdtable && (flags & POSIX_LOCKS))
+                        fdentries = gf_fd_fdtable_get_all_fds (serv_ctx->fdtable,
                                                                &fd_count);
         }
-        pthread_mutex_unlock (&conn->lock);
-
-        if (conn->bound_xl)
-                ret = do_connection_cleanup (this, conn, ltable,
-                                             fdentries, fd_count);
-
-out:
-        return ret;
-}
-
-
-int
-server_connection_destroy (xlator_t *this, server_connection_t *conn)
-{
-        call_frame_t       *frame = NULL, *tmp_frame = NULL;
-        xlator_t           *bound_xl = NULL;
-        int32_t             ret = -1;
-        server_state_t     *state = NULL;
-        struct list_head    inodelk_lockers;
-        struct list_head    entrylk_lockers;
-        struct _lock_table *ltable = NULL;
-        struct _locker     *locker = NULL, *tmp = NULL;
-        struct gf_flock        flock = {0,};
-        fd_t               *fd = NULL;
-        int32_t             i = 0;
-        fdentry_t          *fdentries = NULL;
-        uint32_t             fd_count = 0;
-        char               *path      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-
-        bound_xl = (xlator_t *) (conn->bound_xl);
-
-        if (bound_xl) {
-                /* trans will have ref_count = 1 after this call, but its
-                   ok since this function is called in
-                   GF_EVENT_TRANSPORT_CLEANUP */
-                frame = create_frame (this, this->ctx->pool);
-
-                pthread_mutex_lock (&(conn->lock));
-                {
-                        if (conn->ltable) {
-                                ltable = conn->ltable;
-                                conn->ltable = NULL;
-                        }
-                }
-                pthread_mutex_unlock (&conn->lock);
+        UNLOCK (&serv_ctx->fdtable_lock);
 
-                INIT_LIST_HEAD (&inodelk_lockers);
-                INIT_LIST_HEAD (&entrylk_lockers);
-
-                if (ltable) {
-                        LOCK (&ltable->lock);
-                        {
-                                list_splice_init (&ltable->inodelk_lockers,
-                                                  &inodelk_lockers);
-
-                                list_splice_init (&ltable->entrylk_lockers, &entrylk_lockers);
-                        }
-                        UNLOCK (&ltable->lock);
-                        GF_FREE (ltable);
-                }
-
-                flock.l_type  = F_UNLCK;
-                flock.l_start = 0;
-                flock.l_len   = 0;
-                list_for_each_entry_safe (locker,
-                                          tmp, &inodelk_lockers, lockers) {
-                        tmp_frame = copy_frame (frame);
-                        /*
-                          lock_owner = 0 is a special case that tells posix-locks
-                          to release all locks from this transport
-                        */
-                        tmp_frame->root->trans = conn;
-                        memset (&tmp_frame->root->lk_owner, 0,
-                                sizeof (gf_lkowner_t));
-
-                        if (locker->fd) {
-                                GF_ASSERT (locker->fd->inode);
-
-                                ret = inode_path (locker->fd->inode, NULL, &path);
-
-                                if (ret > 0) {
-                                        gf_log (this->name, GF_LOG_INFO, "finodelk "
-                                                "released on %s", path);
-                                        GF_FREE (path);
-                                } else {
-
-                                        gf_log (this->name, GF_LOG_INFO, "finodelk "
-                                                "released on inode with gfid %s",
-                                                uuid_utoa (locker->fd->inode->gfid));
-                                }
-
-                                STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                            bound_xl->fops->finodelk,
-                                            locker->volume,
-                                            locker->fd, F_SETLK, &flock);
-                                fd_unref (locker->fd);
-                        } else {
-                                gf_log (this->name, GF_LOG_INFO, "inodelk "
-                                        "released on %s", locker->loc.path);
-
-                                STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                            bound_xl->fops->inodelk,
-                                            locker->volume,
-                                            &(locker->loc), F_SETLK, &flock);
-                                loc_wipe (&locker->loc);
-                        }
-
-                        GF_FREE (locker->volume);
-
-                        list_del_init (&locker->lockers);
-                        GF_FREE (locker);
-                }
-
-                tmp = NULL;
-                locker = NULL;
-                list_for_each_entry_safe (locker, tmp, &entrylk_lockers, lockers) {
-                        tmp_frame = copy_frame (frame);
-
-                        tmp_frame->root->trans = conn;
-                        memset (&tmp_frame->root->lk_owner, 0,
-                                sizeof (gf_lkowner_t));
-
-                        if (locker->fd) {
-                                GF_ASSERT (locker->fd->inode);
-
-                                ret = inode_path (locker->fd->inode, NULL, &path);
-
-                                if (ret > 0) {
-                                        gf_log (this->name, GF_LOG_INFO, "fentrylk "
-                                                "released on %s", path);
-
-                                        GF_FREE (path);
-                                } else {
-
-                                        gf_log (this->name, GF_LOG_INFO, "fentrylk "
-                                                "released on inode with gfid %s",
-                                                uuid_utoa (locker->fd->inode->gfid));
-                                }
-
-                                STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                            bound_xl->fops->fentrylk,
-                                            locker->volume,
-                                            locker->fd, NULL,
-                                            ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                                fd_unref (locker->fd);
-                        } else {
-                                gf_log (this->name, GF_LOG_INFO, "entrylk "
-                                        "released on %s", locker->loc.path);
-
-                                STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                            bound_xl->fops->entrylk,
-                                            locker->volume,
-                                            &(locker->loc), NULL,
-                                            ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                                loc_wipe (&locker->loc);
-                        }
-
-                        GF_FREE (locker->volume);
-
-                        list_del_init (&locker->lockers);
-                        GF_FREE (locker);
-                }
-
-                pthread_mutex_lock (&(conn->lock));
-                {
-                        if (conn->fdtable) {
-                                fdentries = gf_fd_fdtable_get_all_fds (conn->fdtable,
-                                                                       &fd_count);
-                                gf_fd_fdtable_destroy (conn->fdtable);
-                                conn->fdtable = NULL;
-                        }
-                }
-                pthread_mutex_unlock (&conn->lock);
-
-                if (fdentries != NULL) {
-                        for (i = 0; i < fd_count; i++) {
-                                fd = fdentries[i].fd;
-                                if (fd != NULL) {
-                                        tmp_frame = copy_frame (frame);
-                                        tmp_frame->local = fd;
-
-                                        STACK_WIND (tmp_frame,
-                                                    server_connection_cleanup_flush_cbk,
-                                                    bound_xl,
-                                                    bound_xl->fops->flush,
-                                                    fd);
-                                }
-                        }
-                        GF_FREE (fdentries);
-                }
-        }
+        if (client->bound_xl == NULL)
+                goto out;
 
-        if (frame) {
-                state = CALL_STATE (frame);
-                if (state)
-                        GF_FREE (state);
-                STACK_DESTROY (frame->root);
+        if (flags & INTERNAL_LOCKS) {
+                cd_ret = gf_client_disconnect (client);
         }
 
-        gf_log (this->name, GF_LOG_INFO, "destroyed connection of %s",
-                conn->id);
+        if (fdentries != NULL)
+                ret = do_fd_cleanup (this, client, fdentries, fd_count);
+        else
+                gf_log (this->name, GF_LOG_INFO, "no fdentries to clean");
 
-        GF_FREE (conn->id);
-        GF_FREE (conn);
+        if (cd_ret || ret)
+                ret = -1;
 
 out:
         return ret;
 }
 
 
-server_connection_t *
-server_connection_get (xlator_t *this, const char *id)
-{
-        server_connection_t *conn = NULL;
-        server_conf_t       *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", id, out);
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                conn = (void *) GF_CALLOC (1, sizeof (*conn),
-                                           gf_server_mt_conn_t);
-                if (!conn)
-                        goto unlock;
-
-                conn->id = gf_strdup (id);
-                conn->fdtable = gf_fd_fdtable_alloc ();
-                conn->ltable  = gf_lock_table_new ();
-                conn->this    = this;
-                pthread_mutex_init (&conn->lock, NULL);
-
-                list_add (&conn->list, &conf->conns);
-
-                conn->ref++;
-        }
-unlock:
-        pthread_mutex_unlock (&conf->mutex);
-out:
-        return conn;
-}
-
-
-void
-server_connection_put (xlator_t *this, server_connection_t *conn)
-{
-        server_conf_t       *conf = NULL;
-        server_connection_t *todel = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO ("server", conf, out);
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                conn->ref--;
-
-                if (!conn->ref) {
-                        list_del_init (&conn->list);
-                        todel = conn;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (todel) {
-                server_connection_destroy (this, todel);
-        }
-
-out:
-        return;
-}
-
 static call_frame_t *
 server_alloc_frame (rpcsvc_request_t *req)
 {
-        call_frame_t         *frame = NULL;
-        server_state_t       *state = NULL;
-        server_connection_t  *conn  = NULL;
+        call_frame_t    *frame  = NULL;
+        server_state_t  *state  = NULL;
+        client_t        *client = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", req, out);
         GF_VALIDATE_OR_GOTO ("server", req->trans, out);
         GF_VALIDATE_OR_GOTO ("server", req->svc, out);
         GF_VALIDATE_OR_GOTO ("server", req->svc->ctx, out);
 
-        conn = (server_connection_t *)req->trans->xl_private;
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
+        client = req->trans->xl_private;
+        GF_VALIDATE_OR_GOTO ("server", client, out);
 
-        frame = create_frame (conn->this, req->svc->ctx->pool);
+        frame = create_frame (client->this, req->svc->ctx->pool);
         if (!frame)
                 goto out;
 
@@ -859,45 +280,46 @@ server_alloc_frame (rpcsvc_request_t *req)
         if (!state)
                 goto out;
 
-        if (conn->bound_xl)
-                state->itable = conn->bound_xl->itable;
+        if (client->bound_xl)
+                state->itable = client->bound_xl->itable;
 
         state->xprt  = rpc_transport_ref (req->trans);
-        state->conn  = conn;
-
         state->resolve.fd_no = -1;
         state->resolve2.fd_no = -1;
 
+        frame->root->client  = client;
         frame->root->state = state;        /* which socket */
         frame->root->unique = 0;           /* which call */
 
-        frame->this = conn->this;
+        frame->this = client->this;
 out:
         return frame;
 }
 
 
-
 call_frame_t *
 get_frame_from_request (rpcsvc_request_t *req)
 {
-        call_frame_t *frame = NULL;
+        call_frame_t  *frame = NULL;
+        client_t      *client = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", req, out);
 
+        client = req->trans->xl_private;
+
         frame = server_alloc_frame (req);
         if (!frame)
                 goto out;
 
         frame->root->op       = req->procnum;
-        frame->root->type     = req->type;
 
         frame->root->unique   = req->xid;
 
         frame->root->uid      = req->uid;
         frame->root->gid      = req->gid;
         frame->root->pid      = req->pid;
-        frame->root->trans    = req->trans->xl_private;
+        gf_client_ref (client);
+        frame->root->client   = client;
         frame->root->lk_owner = req->lk_owner;
 
         server_decode_groups (frame, req);
@@ -921,7 +343,7 @@ server_build_config (xlator_t *this, server_conf_t *conf)
         ret = dict_get_int32 (this->options, "inode-lru-limit",
                               &conf->inode_lru_limit);
         if (ret < 0) {
-                conf->inode_lru_limit = 1024;
+                conf->inode_lru_limit = 16384;
         }
 
         conf->verify_volfile = 1;
@@ -982,73 +404,6 @@ out:
         return ret;
 }
 
-server_connection_t *
-get_server_conn_state (xlator_t *this, rpc_transport_t *xprt)
-{
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", xprt, out);
-
-        return (server_connection_t *)xprt->xl_private;
-out:
-        return NULL;
-}
-
-server_connection_t *
-create_server_conn_state (xlator_t *this, rpc_transport_t *xprt)
-{
-        server_connection_t *conn = NULL;
-        int                  ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", xprt, out);
-
-        conn = GF_CALLOC (1, sizeof (*conn), gf_server_mt_conn_t);
-        if (!conn)
-                goto out;
-
-        pthread_mutex_init (&conn->lock, NULL);
-
-        conn->fdtable = gf_fd_fdtable_alloc ();
-        if (!conn->fdtable)
-                goto out;
-
-        conn->ltable  = gf_lock_table_new ();
-        if (!conn->ltable)
-                goto out;
-
-        conn->this = this;
-
-        xprt->xl_private = conn;
-
-        ret = 0;
-out:
-        if (ret)
-                destroy_server_conn_state (conn);
-
-        return conn;
-}
-
-void
-destroy_server_conn_state (server_connection_t *conn)
-{
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-
-        if (conn->ltable) {
-                /* TODO */
-                //FREE (conn->ltable);
-                ;
-        }
-
-        if (conn->fdtable)
-                gf_fd_fdtable_destroy (conn->fdtable);
-
-        pthread_mutex_destroy (&conn->lock);
-
-        GF_FREE (conn);
-out:
-        return;
-}
-
 
 void
 print_caller (char *str, int size, call_frame_t *frame)
@@ -1175,12 +530,15 @@ server_print_params (char *str, int size, server_state_t *state)
                 filled += snprintf (str + filled, size - filled,
                                     "volume=%s,", state->volume);
 
+/* FIXME
         snprintf (str + filled, size - filled,
-                  "bound_xl=%s}", state->conn->bound_xl->name);
+                  "bound_xl=%s}", state->client->bound_xl->name);
+*/
 out:
         return;
 }
 
+
 int
 server_resolve_is_empty (server_resolve_t *resolve)
 {
@@ -1196,6 +554,7 @@ server_resolve_is_empty (server_resolve_t *resolve)
         return 1;
 }
 
+
 void
 server_print_reply (call_frame_t *frame, int op_ret, int op_errno)
 {
@@ -1220,10 +579,7 @@ server_print_reply (call_frame_t *frame, int op_ret, int op_errno)
 
         switch (frame->root->type) {
         case GF_OP_TYPE_FOP:
-                op = gf_fop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_MGMT:
-                op = gf_mgmt_list[frame->root->op];
+                op = (char *)gf_fop_list[frame->root->op];
                 break;
         default:
                 op = "";
@@ -1244,16 +600,16 @@ out:
 void
 server_print_request (call_frame_t *frame)
 {
-        server_conf_t   *conf = NULL;
-        xlator_t        *this = NULL;
+        server_conf_t   *conf  = NULL;
+        xlator_t        *this  = NULL;
         server_state_t  *state = NULL;
+        char            *op    = "UNKNOWN";
         char             resolve_vars[256];
         char             resolve2_vars[256];
         char             loc_vars[256];
         char             loc2_vars[256];
         char             other_vars[512];
         char             caller[512];
-        char            *op = "UNKNOWN";
 
         GF_VALIDATE_OR_GOTO ("server", frame, out);
 
@@ -1289,10 +645,7 @@ server_print_request (call_frame_t *frame)
 
         switch (frame->root->type) {
         case GF_OP_TYPE_FOP:
-                op = gf_fop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_MGMT:
-                op = gf_mgmt_list[frame->root->op];
+                op = (char *)gf_fop_list[frame->root->op];
                 break;
         default:
                 op = "";
@@ -1307,13 +660,14 @@ out:
         return;
 }
 
+
 int
 serialize_rsp_direntp (gf_dirent_t *entries, gfs3_readdirp_rsp *rsp)
 {
         gf_dirent_t         *entry = NULL;
-        gfs3_dirplist       *trav = NULL;
-        gfs3_dirplist       *prev = NULL;
-        int                  ret = -1;
+        gfs3_dirplist       *trav  = NULL;
+        gfs3_dirplist       *prev  = NULL;
+        int                  ret   = -1;
 
         GF_VALIDATE_OR_GOTO ("server", entries, out);
         GF_VALIDATE_OR_GOTO ("server", rsp, out);
@@ -1367,10 +721,13 @@ serialize_rsp_direntp (gf_dirent_t *entries, gfs3_readdirp_rsp *rsp)
                         rsp->reply = trav;
 
                 prev = trav;
+                trav = NULL;
         }
 
         ret = 0;
 out:
+        GF_FREE (trav);
+
         return ret;
 }
 
@@ -1378,10 +735,10 @@ out:
 int
 serialize_rsp_dirent (gf_dirent_t *entries, gfs3_readdir_rsp *rsp)
 {
-        gf_dirent_t         *entry = NULL;
-        gfs3_dirlist        *trav = NULL;
-        gfs3_dirlist        *prev = NULL;
-        int                  ret = -1;
+        gf_dirent_t   *entry = NULL;
+        gfs3_dirlist  *trav  = NULL;
+        gfs3_dirlist  *prev  = NULL;
+        int           ret    = -1;
 
         GF_VALIDATE_OR_GOTO ("server", entries, out);
         GF_VALIDATE_OR_GOTO ("server", rsp, out);
@@ -1408,11 +765,12 @@ out:
         return ret;
 }
 
+
 int
 readdir_rsp_cleanup (gfs3_readdir_rsp *rsp)
 {
-        gfs3_dirlist *prev = NULL;
-        gfs3_dirlist *trav = NULL;
+        gfs3_dirlist  *prev = NULL;
+        gfs3_dirlist  *trav = NULL;
 
         trav = rsp->reply;
         prev = trav;
@@ -1425,6 +783,7 @@ readdir_rsp_cleanup (gfs3_readdir_rsp *rsp)
         return 0;
 }
 
+
 int
 readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp)
 {
@@ -1435,6 +794,7 @@ readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp)
         prev = trav;
         while (trav) {
                 trav = trav->nextentry;
+                GF_FREE (prev->dict.dict_val);
                 GF_FREE (prev);
                 prev = trav;
         }
@@ -1442,6 +802,7 @@ readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp)
         return 0;
 }
 
+
 int
 gf_server_check_getxattr_cmd (call_frame_t *frame, const char *key)
 {
@@ -1455,10 +816,14 @@ gf_server_check_getxattr_cmd (call_frame_t *frame, const char *key)
 
         if (fnmatch ("*list*mount*point*", key, 0) == 0) {
                 /* list all the client protocol connecting to this process */
-                list_for_each_entry (xprt, &conf->xprt_list, list) {
-                        gf_log ("mount-point-list", GF_LOG_INFO,
-                                "%s", xprt->peerinfo.identifier);
+                pthread_mutex_lock (&conf->mutex);
+                {
+                        list_for_each_entry (xprt, &conf->xprt_list, list) {
+                                gf_log ("mount-point-list", GF_LOG_INFO,
+                                        "%s", xprt->peerinfo.identifier);
+                        }
                 }
+                pthread_mutex_unlock (&conf->mutex);
         }
 
         /* Add more options/keys here */
@@ -1466,34 +831,431 @@ gf_server_check_getxattr_cmd (call_frame_t *frame, const char *key)
         return 0;
 }
 
+
 int
 gf_server_check_setxattr_cmd (call_frame_t *frame, dict_t *dict)
 {
 
-        data_pair_t      *pair = NULL;
-        server_conf_t    *conf = NULL;
-        rpc_transport_t  *xprt = NULL;
-        uint64_t          total_read = 0;
+        server_conf_t    *conf        = NULL;
+        rpc_transport_t  *xprt        = NULL;
+        uint64_t          total_read  = 0;
         uint64_t          total_write = 0;
 
         conf = frame->this->private;
-        if (!conf)
+        if (!conf || !dict)
                 return 0;
 
-        for (pair = dict->members_list; pair; pair = pair->next) {
-                /* this exact key is used in 'io-stats' too.
-                 * But this is better place for this information dump.
-                 */
-                if (fnmatch ("*io*stat*dump", pair->key, 0) == 0) {
-                        list_for_each_entry (xprt, &conf->xprt_list, list) {
-                                total_read  += xprt->total_bytes_read;
-                                total_write += xprt->total_bytes_write;
-                        }
-                        gf_log ("stats", GF_LOG_INFO,
-                                "total-read %"PRIu64", total-write %"PRIu64,
-                                total_read, total_write);
+        if (dict_foreach_fnmatch (dict, "*io*stat*dump",
+                                  dict_null_foreach_fn, NULL ) > 0) {
+                list_for_each_entry (xprt, &conf->xprt_list, list) {
+                        total_read  += xprt->total_bytes_read;
+                        total_write += xprt->total_bytes_write;
                 }
+                gf_log ("stats", GF_LOG_INFO,
+                        "total-read %"PRIu64", total-write %"PRIu64,
+                        total_read, total_write);
         }
 
         return 0;
 }
+
+
+gf_boolean_t
+server_cancel_grace_timer (xlator_t *this, client_t *client)
+{
+        server_ctx_t  *serv_ctx  = NULL;
+        gf_timer_t    *timer     = NULL;
+        gf_boolean_t   cancelled = _gf_false;
+
+        if (!this || !client) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Invalid arguments to cancel connection timer");
+                return cancelled;
+        }
+
+        serv_ctx = server_ctx_get (client, client->this);
+
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
+        }
+
+        LOCK (&serv_ctx->fdtable_lock);
+        {
+                if (serv_ctx->grace_timer) {
+                        timer = serv_ctx->grace_timer;
+                        serv_ctx->grace_timer = NULL;
+                }
+        }
+        UNLOCK (&serv_ctx->fdtable_lock);
+
+        if (timer) {
+                gf_timer_call_cancel (this->ctx, timer);
+                cancelled = _gf_true;
+        }
+out:
+        return cancelled;
+}
+
+server_ctx_t*
+server_ctx_get (client_t *client, xlator_t *xlator)
+{
+        void *tmp = NULL;
+        server_ctx_t *ctx = NULL;
+
+        client_ctx_get (client, xlator, &tmp);
+
+        ctx = tmp;
+
+        if (ctx != NULL)
+                goto out;
+
+        ctx = GF_CALLOC (1, sizeof (server_ctx_t), gf_server_mt_server_conf_t);
+
+        if (ctx == NULL)
+                goto out;
+
+     /* ctx->lk_version = 0; redundant */
+        ctx->fdtable = gf_fd_fdtable_alloc ();
+
+        if (ctx->fdtable == NULL) {
+                GF_FREE (ctx);
+                ctx = NULL;
+                goto out;
+        }
+
+        LOCK_INIT (&ctx->fdtable_lock);
+
+        if (client_ctx_set (client, xlator, ctx) != 0) {
+              LOCK_DESTROY (&ctx->fdtable_lock);
+              GF_FREE (ctx);
+              ctx = NULL;
+        }
+
+out:
+        return ctx;
+}
+
+int32_t
+gf_barrier_transmit (server_conf_t *conf, gf_barrier_payload_t *payload)
+{
+        gf_barrier_t            *barrier = NULL;
+        int32_t                  ret     = -1;
+        client_t                *client  = NULL;
+        gf_boolean_t             lk_heal = _gf_false;
+        call_frame_t            *frame   = NULL;
+        server_state_t          *state   = NULL;
+
+        GF_VALIDATE_OR_GOTO ("barrier", conf, out);
+        GF_VALIDATE_OR_GOTO ("barrier", conf->barrier, out);
+        GF_VALIDATE_OR_GOTO ("barrier", payload, out);
+
+        barrier = conf->barrier;
+
+        frame = payload->frame;
+        if (frame) {
+                state = CALL_STATE (frame);
+                frame->local = NULL;
+                client = frame->root->client;
+        }
+        /* currently lk fops are not barrier'ed. This is reflecting code in
+         * server_submit_reply */
+        if (client)
+                lk_heal = ((server_conf_t *) client->this->private)->lk_heal;
+
+        ret = rpcsvc_submit_generic (payload->req, &payload->rsp, 1,
+                                     payload->payload, payload->payload_count,
+                                     payload->iobref);
+        iobuf_unref (payload->iob);
+        if (ret == -1) {
+                gf_log_callingfn ("", GF_LOG_ERROR, "Reply submission failed");
+                if (frame && client && !lk_heal) {
+                        server_connection_cleanup (frame->this, client,
+                                                  INTERNAL_LOCKS | POSIX_LOCKS);
+                } else {
+                        /* TODO: Failure of open(dir), create, inodelk, entrylk
+                           or lk fops send failure must be handled specially. */
+                }
+                goto ret;
+        }
+
+        ret = 0;
+ret:
+        if (state) {
+                free_state (state);
+        }
+
+        if (frame) {
+                gf_client_unref (client);
+                STACK_DESTROY (frame->root);
+        }
+
+        if (payload->free_iobref) {
+                iobref_unref (payload->iobref);
+        }
+out:
+        return ret;
+}
+
+gf_barrier_payload_t *
+gf_barrier_dequeue (gf_barrier_t *barrier)
+{
+        gf_barrier_payload_t  *payload = NULL;
+
+        if (!barrier || list_empty (&barrier->queue))
+                return NULL;
+
+        payload = list_entry (barrier->queue.next,
+                              gf_barrier_payload_t, list);
+        if (payload) {
+                list_del_init (&payload->list);
+                barrier->cur_size--;
+        }
+
+        return payload;
+}
+
+
+void*
+gf_barrier_dequeue_start (void *data)
+{
+        server_conf_t           *conf = NULL;
+        gf_barrier_t            *barrier = NULL;
+        gf_barrier_payload_t    *payload = NULL;
+
+        conf = (server_conf_t *)data;
+        if (!conf || !conf->barrier)
+                return NULL;
+        barrier = conf->barrier;
+
+        LOCK (&barrier->lock);
+        {
+                while (barrier->cur_size) {
+                        payload = gf_barrier_dequeue (barrier);
+                        if (payload) {
+                                if (gf_barrier_transmit (conf, payload)) {
+                                        gf_log ("server", GF_LOG_WARNING,
+                                                "Failed to transmit");
+                                }
+                                GF_FREE (payload);
+                        }
+                }
+        }
+        UNLOCK (&barrier->lock);
+        return NULL;
+}
+
+void
+gf_barrier_timeout (void *data)
+{
+        server_conf_t *conf = NULL;
+        gf_barrier_t  *barrier = NULL;
+        gf_boolean_t   need_dequeue = _gf_false;
+
+        conf = (server_conf_t *)data;
+        if (!conf || !conf->barrier)
+                goto out;
+        barrier = conf->barrier;
+
+        gf_log ("", GF_LOG_INFO, "barrier timed-out");
+        LOCK (&barrier->lock);
+        {
+                need_dequeue = barrier->on;
+                barrier->on = _gf_false;
+        }
+        UNLOCK (&barrier->lock);
+
+        if (need_dequeue == _gf_true)
+                gf_barrier_dequeue_start (data);
+out:
+        return;
+}
+
+
+int32_t
+gf_barrier_start (xlator_t *this)
+{
+        server_conf_t *conf = NULL;
+        gf_barrier_t  *barrier = NULL;
+        int32_t        ret  = -1;
+        struct timespec time = {0,};
+
+        conf = this->private;
+
+        GF_VALIDATE_OR_GOTO ("server", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf->barrier, out);
+
+        barrier = conf->barrier;
+
+        gf_log (this->name, GF_LOG_INFO, "barrier start called");
+        LOCK (&barrier->lock);
+        {
+                /* if barrier is on, reset timer */
+                if (barrier->on == _gf_true) {
+                        ret = gf_timer_call_cancel (this->ctx, barrier->timer);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "unset timer, failing barrier start");
+                                goto unlock;
+                        }
+                }
+
+                barrier->on = _gf_true;
+                time.tv_sec = barrier->time_out;
+                time.tv_nsec = 0;
+
+                barrier->timer = gf_timer_call_after (this->ctx, time,
+                                                      gf_barrier_timeout,
+                                                      (void *)conf);
+                if (!barrier->timer) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "timer, failing barrier start");
+                        barrier->on = _gf_false;
+                }
+        }
+unlock:
+        UNLOCK (&barrier->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+gf_barrier_stop (xlator_t *this)
+{
+        server_conf_t *conf = NULL;
+        gf_barrier_t  *barrier = NULL;
+        int32_t        ret  = -1;
+        gf_boolean_t   need_dequeue = _gf_false;
+
+        conf = this->private;
+
+        GF_VALIDATE_OR_GOTO ("server", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf->barrier, out);
+
+        barrier = conf->barrier;
+
+        gf_log (this->name, GF_LOG_INFO, "barrier stop called");
+        LOCK (&barrier->lock);
+        {
+                need_dequeue = barrier->on;
+                barrier->on = _gf_false;
+        }
+        UNLOCK (&barrier->lock);
+
+        if (need_dequeue == _gf_true) {
+                gf_timer_call_cancel (this->ctx, barrier->timer);
+                ret = gf_thread_create (&conf->barrier_th, NULL,
+                                        gf_barrier_dequeue_start,
+                                        conf);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Failed to start un-barriering");
+                        goto out;
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+gf_barrier_fops_configure (xlator_t *this, gf_barrier_t *barrier, char *str)
+{
+        int32_t         ret = -1;
+        char           *dup_str = NULL;
+        char           *str_tok = NULL;
+        char           *save_ptr = NULL;
+        uint64_t        fops = 0;
+
+        /* by defaul fsync & flush needs to be barriered */
+
+        fops |= 1 << GFS3_OP_FSYNC;
+        fops |= 1 << GFS3_OP_FLUSH;
+
+        if (!str)
+                goto done;
+
+        dup_str = gf_strdup (str);
+        if (!dup_str)
+                goto done;
+
+        str_tok = strtok_r (dup_str, ",", &save_ptr);
+        if (!str_tok)
+                goto done;
+
+        fops = 0;
+        while (str_tok) {
+                if (!strcmp(str_tok, "writev")) {
+                        fops |= ((uint64_t)1 << GFS3_OP_WRITE);
+                } else if (!strcmp(str_tok, "fsync")) {
+                        fops |= ((uint64_t)1 << GFS3_OP_FSYNC);
+                } else if (!strcmp(str_tok, "read")) {
+                        fops |= ((uint64_t)1 << GFS3_OP_READ);
+                } else if (!strcmp(str_tok, "rename")) {
+                        fops |= ((uint64_t)1 << GFS3_OP_RENAME);
+                } else if (!strcmp(str_tok, "flush")) {
+                        fops |= ((uint64_t)1 << GFS3_OP_FLUSH);
+                } else if (!strcmp(str_tok, "ftruncate")) {
+                        fops |= ((uint64_t)1 << GFS3_OP_FTRUNCATE);
+                } else if (!strcmp(str_tok, "fallocate")) {
+                        fops |= ((uint64_t)1 << GFS3_OP_FALLOCATE);
+                } else if (!strcmp(str_tok, "rmdir")) {
+                        fops |= ((uint64_t)1 << GFS3_OP_RMDIR);
+                }  else {
+                        gf_log ("barrier", GF_LOG_ERROR,
+                                "Invalid barrier fop %s", str_tok);
+                }
+
+                str_tok = strtok_r (NULL, ",", &save_ptr);
+        }
+done:
+        LOCK (&barrier->lock);
+        {
+                barrier->fops = fops;
+        }
+        UNLOCK (&barrier->lock);
+        ret = 0;
+
+        GF_FREE (dup_str);
+        return ret;
+}
+
+void
+gf_barrier_enqueue (gf_barrier_t *barrier, gf_barrier_payload_t *payload)
+{
+        list_add_tail (&payload->list, &barrier->queue);
+        barrier->cur_size++;
+}
+
+gf_barrier_payload_t *
+gf_barrier_payload (rpcsvc_request_t *req, struct iovec *rsp,
+                    call_frame_t *frame, struct iovec *payload_orig,
+                    int payloadcount, struct iobref *iobref,
+                    struct iobuf *iob, gf_boolean_t free_iobref)
+{
+        gf_barrier_payload_t *payload = NULL;
+
+        if (!rsp)
+                return NULL;
+
+        payload = GF_CALLOC (1, sizeof (*payload),1);
+        if (!payload)
+                return NULL;
+
+        INIT_LIST_HEAD (&payload->list);
+
+        payload->req = req;
+        memcpy (&payload->rsp, rsp, sizeof (struct iovec));
+        payload->frame = frame;
+        payload->payload = payload_orig;
+        payload->payload_count = payloadcount;
+        payload->iobref = iobref;
+        payload->iob = iob;
+        payload->free_iobref = free_iobref;
+
+        return payload;
+}
diff --git a/xlators/protocol/server/src/server-helpers.h b/xlators/protocol/server/src/server-helpers.h
index 844c98c27..b455aa6df 100644
--- a/xlators/protocol/server/src/server-helpers.h
+++ b/xlators/protocol/server/src/server-helpers.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _SERVER_HELPERS_H
@@ -24,13 +15,8 @@
 
 #define CALL_STATE(frame)   ((server_state_t *)frame->root->state)
 
-#define BOUND_XL(frame)     ((xlator_t *) CALL_STATE(frame)->conn->bound_xl)
-
 #define XPRT_FROM_FRAME(frame) ((rpc_transport_t *) CALL_STATE(frame)->xprt)
 
-#define SERVER_CONNECTION(frame)                                \
-        ((server_connection_t *) CALL_STATE(frame)->conn)
-
 #define SERVER_CONF(frame)                                              \
         ((server_conf_t *)XPRT_FROM_FRAME(frame)->this->private)
 
@@ -43,39 +29,26 @@
 
 #define IS_NOT_ROOT(pathlen) ((pathlen > 2)? 1 : 0)
 
+#define is_fop_barriered(fops, procnum) (fops & ((uint64_t)1 << procnum))
+
+#define barrier_add_to_queue(barrier) (barrier->on || barrier->cur_size)
+
 void free_state (server_state_t *state);
 
 void server_loc_wipe (loc_t *loc);
 
-int32_t
-gf_add_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc,
-               fd_t *fd,
-               pid_t pid,
-               gf_lkowner_t *owner,
-               glusterfs_fop_t type);
-
-int32_t
-gf_del_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc,
-               fd_t *fd,
-               gf_lkowner_t *owner,
-               glusterfs_fop_t type);
-
 void
 server_print_request (call_frame_t *frame);
 
 call_frame_t *
 get_frame_from_request (rpcsvc_request_t *req);
 
-server_connection_t *
-get_server_conn_state (xlator_t *this, rpc_transport_t *xptr);
+int
+server_connection_cleanup (xlator_t *this, struct _client_t *client,
+                           int32_t flags);
 
-server_connection_t *
-create_server_conn_state (xlator_t *this, rpc_transport_t *xptr);
-
-void
-destroy_server_conn_state (server_connection_t *conn);
+gf_boolean_t
+server_cancel_grace_timer (xlator_t *this, struct _client_t *client);
 
 int
 server_build_config (xlator_t *this, server_conf_t *conf);
@@ -85,4 +58,17 @@ int serialize_rsp_direntp (gf_dirent_t *entries, gfs3_readdirp_rsp *rsp);
 int readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp);
 int readdir_rsp_cleanup (gfs3_readdir_rsp *rsp);
 
+server_ctx_t *server_ctx_get (client_t *client, xlator_t *xlator);
+
+int32_t gf_barrier_start (xlator_t *this);
+int32_t gf_barrier_stop (xlator_t *this);
+int32_t gf_barrier_fops_configure (xlator_t *this, gf_barrier_t *barrier,
+                                   char *str);
+void gf_barrier_enqueue (gf_barrier_t *barrier, gf_barrier_payload_t *stub);
+gf_barrier_payload_t *
+gf_barrier_payload (rpcsvc_request_t *req, struct iovec *rsp,
+                    call_frame_t *frame, struct iovec *payload,
+                    int payloadcount, struct iobref *iobref,
+                    struct iobuf *iob, gf_boolean_t free_iobref);
+
 #endif /* !_SERVER_HELPERS_H */
diff --git a/xlators/protocol/server/src/server-mem-types.h b/xlators/protocol/server/src/server-mem-types.h
index 88bae8cb4..19c3466d3 100644
--- a/xlators/protocol/server/src/server-mem-types.h
+++ b/xlators/protocol/server/src/server-mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -33,6 +24,7 @@ enum gf_server_mem_types_ {
         gf_server_mt_dirent_rsp_t,
         gf_server_mt_rsp_buf_t,
         gf_server_mt_volfile_ctx_t,
+        gf_server_mt_timer_data_t,
         gf_server_mt_end,
 };
 #endif /* __SERVER_MEM_TYPES_H__ */
diff --git a/xlators/protocol/server/src/server-resolve.c b/xlators/protocol/server/src/server-resolve.c
index 38ab368c3..cc4686a03 100644
--- a/xlators/protocol/server/src/server-resolve.c
+++ b/xlators/protocol/server/src/server-resolve.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -70,7 +61,8 @@ resolve_loc_touchup (call_frame_t *frame)
 int
 resolve_gfid_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *buf, dict_t *xattr, struct iatt *postparent)
+                        struct iatt *buf, dict_t *xdata,
+                        struct iatt *postparent)
 {
         server_state_t       *state = NULL;
         server_resolve_t     *resolve = NULL;
@@ -111,7 +103,7 @@ out:
 int
 resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-                  dict_t *xattr, struct iatt *postparent)
+                  dict_t *xdata, struct iatt *postparent)
 {
         server_state_t       *state = NULL;
         server_resolve_t     *resolve = NULL;
@@ -155,7 +147,8 @@ resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     (char **) &resolve_loc->path);
 
         STACK_WIND (frame, resolve_gfid_entry_cbk,
-                    BOUND_XL (frame), BOUND_XL (frame)->fops->lookup,
+                    frame->root->client->bound_xl,
+                    frame->root->client->bound_xl->fops->lookup,
                     &resolve->resolve_loc, NULL);
         return 0;
 out:
@@ -178,25 +171,21 @@ resolve_gfid (call_frame_t *frame)
         resolve = state->resolve_now;
         resolve_loc = &resolve->resolve_loc;
 
-        if (!uuid_is_null (resolve->pargfid)) {
+        if (!uuid_is_null (resolve->pargfid))
                 uuid_copy (resolve_loc->gfid, resolve->pargfid);
-                resolve_loc->inode = inode_new (state->itable);
-                ret = inode_path (resolve_loc->inode, NULL,
-                                  (char **)&resolve_loc->path);
-        } else if (!uuid_is_null (resolve->gfid)) {
+        else if (!uuid_is_null (resolve->gfid))
                 uuid_copy (resolve_loc->gfid, resolve->gfid);
-                resolve_loc->inode = inode_new (state->itable);
-                ret = inode_path (resolve_loc->inode, NULL,
-                                  (char **)&resolve_loc->path);
-        }
+
+        resolve_loc->inode = inode_new (state->itable);
+        ret = loc_path (resolve_loc, NULL);
 
         STACK_WIND (frame, resolve_gfid_cbk,
-                    BOUND_XL (frame), BOUND_XL (frame)->fops->lookup,
+                    frame->root->client->bound_xl,
+                    frame->root->client->bound_xl->fops->lookup,
                     &resolve->resolve_loc, NULL);
         return 0;
 }
 
-
 int
 resolve_continue (call_frame_t *frame)
 {
@@ -426,6 +415,10 @@ out:
         if (inode)
                 inode_unref (inode);
 
+        if (ret != 0)
+                gf_log ("server", GF_LOG_WARNING, "inode for the gfid (%s) is "
+                        "not found. anonymous fd creation failed",
+                        uuid_utoa (resolve->gfid));
         return ret;
 }
 
@@ -458,23 +451,34 @@ server_resolve_anonfd (call_frame_t *frame)
 int
 server_resolve_fd (call_frame_t *frame)
 {
-        server_state_t       *state = NULL;
-        server_resolve_t     *resolve = NULL;
-        server_connection_t  *conn = NULL;
-        uint64_t              fd_no = -1;
+        server_ctx_t         *serv_ctx = NULL;
+        server_state_t       *state    = NULL;
+        client_t             *client   = NULL;
+        server_resolve_t     *resolve  = NULL;
+        uint64_t              fd_no    = -1;
 
         state = CALL_STATE (frame);
         resolve = state->resolve_now;
-        conn  = SERVER_CONNECTION (frame);
 
         fd_no = resolve->fd_no;
 
-        if (fd_no == -2) {
+        if (fd_no == GF_ANON_FD_NO) {
                 server_resolve_anonfd (frame);
                 return 0;
         }
 
-        state->fd = gf_fd_fdptr_get (conn->fdtable, fd_no);
+        client = frame->root->client;
+
+        serv_ctx = server_ctx_get (client, client->this);
+
+        if (serv_ctx == NULL) {
+                gf_log ("", GF_LOG_INFO, "server_ctx_get() failed");
+                resolve->op_ret   = -1;
+                resolve->op_errno = ENOMEM;
+                return 0;
+        }
+
+        state->fd = gf_fd_fdptr_get (serv_ctx->fdtable, fd_no);
 
         if (!state->fd) {
                 gf_log ("", GF_LOG_INFO, "fd not found in context");
@@ -529,14 +533,12 @@ int
 server_resolve_done (call_frame_t *frame)
 {
         server_state_t    *state = NULL;
-        xlator_t          *bound_xl = NULL;
 
         state = CALL_STATE (frame);
-        bound_xl = BOUND_XL (frame);
 
         server_print_request (frame);
 
-        state->resume_fn (frame, bound_xl);
+        state->resume_fn (frame, frame->root->client->bound_xl);
 
         return 0;
 }
diff --git a/xlators/protocol/server/src/server-rpc-fops.c b/xlators/protocol/server/src/server-rpc-fops.c
new file mode 100644
index 000000000..138e601ce
--- /dev/null
+++ b/xlators/protocol/server/src/server-rpc-fops.c
@@ -0,0 +1,6179 @@
+/*
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <openssl/md5.h>
+
+#include "server.h"
+#include "server-helpers.h"
+#include "glusterfs3-xdr.h"
+#include "glusterfs3.h"
+#include "compat-errno.h"
+
+#include "xdr-nfs3.h"
+
+#define SERVER_REQ_SET_ERROR(req, ret)                          \
+        do {                                                    \
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);      \
+                ret = RPCSVC_ACTOR_ERROR;                       \
+        } while (0)
+
+/* Callback function section */
+int
+server_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                   dict_t *xdata)
+{
+        gfs3_statfs_rsp      rsp    = {0,};
+        rpcsvc_request_t    *req    = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "%"PRId64": STATFS (%s)",
+                        frame->root->unique, strerror (op_errno));
+                goto out;
+        }
+
+        gf_statfs_from_statfs (&rsp.statfs, buf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_statfs_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   inode_t *inode, struct iatt *stbuf, dict_t *xdata,
+                   struct iatt *postparent)
+{
+        rpcsvc_request_t    *req        = NULL;
+        server_state_t      *state      = NULL;
+        inode_t             *root_inode = NULL;
+        inode_t             *link_inode = NULL;
+        loc_t                fresh_loc  = {0,};
+        gfs3_lookup_rsp      rsp        = {0,};
+        uuid_t               rootgfid   = {0,};
+
+        state = CALL_STATE (frame);
+
+        if (state->is_revalidate == 1 && op_ret == -1) {
+                state->is_revalidate = 2;
+                loc_copy (&fresh_loc, &state->loc);
+                inode_unref (fresh_loc.inode);
+                fresh_loc.inode = inode_new (state->itable);
+
+                STACK_WIND (frame, server_lookup_cbk,
+                            frame->root->client->bound_xl,
+                            frame->root->client->bound_xl->fops->lookup,
+                            &fresh_loc, state->xdata);
+
+                loc_wipe (&fresh_loc);
+                return 0;
+        }
+
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                if (state->is_revalidate && op_errno == ENOENT) {
+                        if (!__is_root_gfid (state->resolve.gfid)) {
+                                inode_unlink (state->loc.inode,
+                                              state->loc.parent,
+                                              state->loc.name);
+                        }
+                }
+                goto out;
+        }
+
+        root_inode = frame->root->client->bound_xl->itable->root;
+        if (inode == root_inode) {
+                /* we just looked up root ("/") */
+                stbuf->ia_ino = 1;
+                rootgfid[15]  = 1;
+                uuid_copy (stbuf->ia_gfid, rootgfid);
+                if (inode->ia_type == 0)
+                        inode->ia_type = stbuf->ia_type;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+
+        if (!__is_root_gfid (inode->gfid)) {
+                link_inode = inode_link (inode, state->loc.parent,
+                                         state->loc.name, stbuf);
+                if (link_inode) {
+                        inode_lookup (link_inode);
+                        inode_unref (link_inode);
+                }
+        }
+
+out:
+        rsp.op_ret   = op_ret;
+        rsp.op_errno = gf_errno_to_error (op_errno);
+
+        if (op_ret) {
+                if (state->resolve.bname) {
+                        gf_log (this->name, ((op_errno == ENOENT) ?
+                                             GF_LOG_TRACE : GF_LOG_INFO),
+                                "%"PRId64": LOOKUP %s (%s/%s) ==> "
+                                "(%s)", frame->root->unique,
+                                state->loc.path,
+                                uuid_utoa (state->resolve.pargfid),
+                                state->resolve.bname,
+                                strerror (op_errno));
+                } else {
+                        gf_log (this->name, ((op_errno == ENOENT) ?
+                                             GF_LOG_TRACE : GF_LOG_INFO),
+                                "%"PRId64": LOOKUP %s (%s) ==> (%s)",
+                                frame->root->unique, state->loc.path,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+        }
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_lookup_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+               dict_t *xdata)
+{
+        gfs3_lk_rsp          rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
+                        state = CALL_STATE (frame);
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%"PRId64": LK %"PRId64" (%s) ==> "
+                                "(%s)", frame->root->unique,
+                                state->resolve.fd_no,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+                goto out;
+        }
+
+        switch (lock->l_type) {
+        case F_RDLCK:
+                lock->l_type = GF_LK_F_RDLCK;
+                break;
+        case F_WRLCK:
+                lock->l_type = GF_LK_F_WRLCK;
+                break;
+        case F_UNLCK:
+                lock->l_type = GF_LK_F_UNLCK;
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unknown lock type: %"PRId32"!", lock->l_type);
+                break;
+        }
+
+        gf_proto_flock_from_flock (&rsp.flock, lock);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_lk_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp    rsp        = {0,};
+        server_state_t   *state     = NULL;
+        rpcsvc_request_t *req       = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
+                        gf_log (this->name, (op_errno == ENOENT)?
+                                GF_LOG_DEBUG:GF_LOG_ERROR,
+                                "%"PRId64": INODELK %s (%s) ==> (%s)",
+                                frame->root->unique, state->loc.path,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp     rsp       = {0,};
+        server_state_t   *state     = NULL;
+        rpcsvc_request_t *req       = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%"PRId64": FINODELK %"PRId64" (%s) "
+                                "==> (%s)", frame->root->unique,
+                                state->resolve.fd_no,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp     rsp       = {0,};
+        server_state_t   *state     = NULL;
+        rpcsvc_request_t *req       = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%"PRId64": ENTRYLK %s (%s) ==> (%s)",
+                                frame->root->unique, state->loc.path,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp     rsp       = {0,};
+        server_state_t   *state     = NULL;
+        rpcsvc_request_t *req       = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%"PRId64": FENTRYLK %"PRId64" (%s) ==>(%s)",
+                                frame->root->unique, state->resolve.fd_no,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req   = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp        rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": ACCESS %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_rmdir_rsp       rsp    = {0,};
+        server_state_t      *state  = NULL;
+        inode_t             *parent = NULL;
+        rpcsvc_request_t    *req    = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": RMDIR %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        inode_unlink (state->loc.inode, state->loc.parent,
+                      state->loc.name);
+        parent = inode_parent (state->loc.inode, 0, NULL);
+        if (parent)
+                /* parent should not be found for directories after
+                 * inode_unlink, since directories cannot have
+                 * hardlinks.
+                 */
+                inode_unref (parent);
+        else
+                inode_forget (state->loc.inode, 0);
+
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_rmdir_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_mkdir_rsp       rsp        = {0,};
+        server_state_t      *state      = NULL;
+        inode_t             *link_inode = NULL;
+        rpcsvc_request_t    *req        = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": MKDIR %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        link_inode = inode_link (inode, state->loc.parent,
+                                 state->loc.name, stbuf);
+        inode_lookup (link_inode);
+        inode_unref (link_inode);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_mkdir_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_mknod_rsp       rsp        = {0,};
+        server_state_t      *state      = NULL;
+        inode_t             *link_inode = NULL;
+        rpcsvc_request_t    *req        = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": MKNOD %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        link_inode = inode_link (inode, state->loc.parent,
+                                 state->loc.name, stbuf);
+        inode_lookup (link_inode);
+        inode_unref (link_inode);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_mknod_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp        rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FSYNCDIR %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
+{
+        gfs3_readdir_rsp     rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+        int                  ret   = 0;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": READDIR %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        /* (op_ret == 0) is valid, and means EOF */
+        if (op_ret) {
+                ret = serialize_rsp_dirent (entries, &rsp);
+                if (ret == -1) {
+                        op_ret   = -1;
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_readdir_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        readdir_rsp_cleanup (&rsp);
+
+        return 0;
+}
+
+int
+server_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        server_state_t      *state    = NULL;
+        server_ctx_t        *serv_ctx = NULL;
+        rpcsvc_request_t    *req      = NULL;
+        gfs3_opendir_rsp     rsp      = {0,};
+        uint64_t             fd_no    = 0;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, (op_errno == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_ERROR,
+                        "%"PRId64": OPENDIR %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        serv_ctx = server_ctx_get (frame->root->client, this);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
+        }
+
+        fd_bind (fd);
+        fd_no = gf_fd_unused_get (serv_ctx->fdtable, fd);
+        fd_ref (fd); // on behalf of the client
+
+out:
+        rsp.fd = fd_no;
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_opendir_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp        rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": REMOVEXATTR %s (%s) of key %s ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        state->name, strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req   = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp        rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FREMOVEXATTR %"PRId64" (%s) (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), state->name,
+                        strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req   = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        gfs3_getxattr_rsp    rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, (((op_errno == ENOTSUP) ||
+                                      (op_errno == ENODATA) ||
+                                      (op_errno == ENOENT)) ?
+                                     GF_LOG_DEBUG : GF_LOG_INFO),
+                        "%"PRId64": GETXATTR %s (%s) (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        state->name, strerror (op_errno));
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, &rsp.dict.dict_val,
+                                    rsp.dict.dict_len, op_errno, out);
+
+out:
+        rsp.op_ret        = op_ret;
+        rsp.op_errno      = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_getxattr_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
+{
+        gfs3_fgetxattr_rsp   rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_INFO),
+                        "%"PRId64": FGETXATTR %"PRId64" (%s) (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        state->name, strerror (op_errno));
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, &rsp.dict.dict_val,
+                                    rsp.dict.dict_len, op_errno, out);
+
+out:
+
+        rsp.op_ret        = op_ret;
+        rsp.op_errno      = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_fgetxattr_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+/* print every key */
+static int
+_gf_server_log_setxattr_failure (dict_t *d, char *k, data_t *v,
+                                 void *tmp)
+{
+        server_state_t      *state = NULL;
+        call_frame_t        *frame = NULL;
+
+        frame = tmp;
+        state = CALL_STATE (frame);
+
+        gf_log (THIS->name, GF_LOG_INFO,
+                "%"PRId64": SETXATTR %s (%s) ==> %s",
+                frame->root->unique, state->loc.path,
+                uuid_utoa (state->resolve.gfid), k);
+        return 0;
+}
+
+int
+server_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp     rsp = {0,};
+        rpcsvc_request_t *req = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                if (op_errno != ENOTSUP)
+                        dict_foreach (state->dict,
+                                      _gf_server_log_setxattr_failure,
+                                      frame);
+
+                gf_log (THIS->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_INFO),
+                        "%s", strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+/* print every key here */
+static int
+_gf_server_log_fsetxattr_failure (dict_t *d, char *k, data_t *v,
+                                 void *tmp)
+{
+        call_frame_t        *frame = NULL;
+        server_state_t      *state = NULL;
+
+        frame = tmp;
+        state = CALL_STATE (frame);
+
+        gf_log (THIS->name, GF_LOG_INFO,
+                "%"PRId64": FSETXATTR %"PRId64" (%s) ==> %s",
+                frame->root->unique, state->resolve.fd_no,
+                uuid_utoa (state->resolve.gfid), k);
+
+        return 0;
+}
+
+int
+server_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp     rsp = {0,};
+        rpcsvc_request_t *req = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                if (op_errno != ENOTSUP) {
+                        dict_foreach (state->dict,
+                                      _gf_server_log_fsetxattr_failure,
+                                      frame);
+                }
+                gf_log (THIS->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_INFO),
+                        "%s", strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+        gfs3_rename_rsp      rsp        = {0,};
+        server_state_t      *state      = NULL;
+        rpcsvc_request_t    *req        = NULL;
+        inode_t             *tmp_inode  = NULL;
+        inode_t             *tmp_parent = NULL;
+        char         oldpar_str[50]     = {0,};
+        char         newpar_str[50]     = {0,};
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret == -1) {
+                uuid_utoa_r (state->resolve.gfid, oldpar_str);
+                uuid_utoa_r (state->resolve2.gfid, newpar_str);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": RENAME %s (%s/%s) -> %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        oldpar_str, state->resolve.bname, state->loc2.path,
+                        newpar_str, state->resolve2.bname, strerror (op_errno));
+                goto out;
+        }
+
+        stbuf->ia_type = state->loc.inode->ia_type;
+
+        /* TODO: log gfid of the inodes */
+        gf_log (frame->root->client->bound_xl->name, GF_LOG_TRACE,
+                "%"PRId64": RENAME_CBK  %s ==> %s",
+                frame->root->unique, state->loc.name, state->loc2.name);
+
+        /* Before renaming the inode, we have to get the inode for the
+         * destination entry (i.e. inode with state->loc2.parent as
+         * parent and state->loc2.name as name). If it exists, then
+         * unlink that inode, and send forget on that inode if the
+         * unlinked entry is the last entry. In case of fuse client
+         * the fuse kernel module itself sends the forget on the
+         * unlinked inode.
+         */
+        tmp_inode = inode_grep (state->loc.inode->table,
+                                state->loc2.parent, state->loc2.name);
+        if (tmp_inode) {
+                inode_unlink (tmp_inode, state->loc2.parent,
+                              state->loc2.name);
+                tmp_parent = inode_parent (tmp_inode, 0, NULL);
+                if (tmp_parent)
+                        inode_unref (tmp_parent);
+                else
+                        inode_forget (tmp_inode, 0);
+
+                inode_unref (tmp_inode);
+        }
+
+        inode_rename (state->itable,
+                      state->loc.parent, state->loc.name,
+                      state->loc2.parent, state->loc2.name,
+                      state->loc.inode, stbuf);
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+
+        gf_stat_from_iatt (&rsp.preoldparent, preoldparent);
+        gf_stat_from_iatt (&rsp.postoldparent, postoldparent);
+
+        gf_stat_from_iatt (&rsp.prenewparent, prenewparent);
+        gf_stat_from_iatt (&rsp.postnewparent, postnewparent);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req   = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_rename_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_unlink_rsp      rsp    = {0,};
+        server_state_t      *state  = NULL;
+        inode_t             *parent = NULL;
+        rpcsvc_request_t    *req    = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret) {
+                gf_log (this->name, (op_errno == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_ERROR,
+                        "%"PRId64": UNLINK %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        /* TODO: log gfid of the inodes */
+        gf_log (frame->root->client->bound_xl->name, GF_LOG_TRACE,
+                "%"PRId64": UNLINK_CBK %s",
+                frame->root->unique, state->loc.name);
+
+        inode_unlink (state->loc.inode, state->loc.parent,
+                      state->loc.name);
+
+        parent = inode_parent (state->loc.inode, 0, NULL);
+        if (parent)
+                inode_unref (parent);
+        else
+                inode_forget (state->loc.inode, 0);
+
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_unlink_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_symlink_rsp     rsp        = {0,};
+        server_state_t      *state      = NULL;
+        inode_t             *link_inode = NULL;
+        rpcsvc_request_t    *req        = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": SYMLINK %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        link_inode = inode_link (inode, state->loc.parent,
+                                 state->loc.name, stbuf);
+        inode_lookup (link_inode);
+        inode_unref (link_inode);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_symlink_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *stbuf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_link_rsp        rsp         = {0,};
+        server_state_t      *state       = NULL;
+        inode_t             *link_inode  = NULL;
+        rpcsvc_request_t    *req         = NULL;
+        char              gfid_str[50]   = {0,};
+        char              newpar_str[50] = {0,};
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret) {
+                uuid_utoa_r (state->resolve.gfid, gfid_str);
+                uuid_utoa_r (state->resolve2.pargfid, newpar_str);
+
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": LINK %s (%s) -> %s/%s ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        gfid_str, newpar_str, state->resolve2.bname,
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        link_inode = inode_link (inode, state->loc2.parent,
+                                 state->loc2.name, stbuf);
+        inode_unref (link_inode);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_link_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        gfs3_truncate_rsp    rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": TRUNCATE %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.prestat, prebuf);
+        gf_stat_from_iatt (&rsp.poststat, postbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_truncate_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                  dict_t *xdata)
+{
+        gfs3_fstat_rsp       rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FSTAT %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_fstat_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        gfs3_ftruncate_rsp   rsp   = {0};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FTRUNCATE %"PRId64" (%s)==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.prestat, prebuf);
+        gf_stat_from_iatt (&rsp.poststat, postbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_ftruncate_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp        rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, (op_errno == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_ERROR,
+                        "%"PRId64": FLUSH %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        gfs3_fsync_rsp       rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FSYNC %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&(rsp.prestat), prebuf);
+        gf_stat_from_iatt (&(rsp.poststat), postbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_fsync_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        gfs3_write_rsp       rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": WRITEV %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                         uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.prestat, prebuf);
+        gf_stat_from_iatt (&rsp.poststat, postbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_write_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iovec *vector, int32_t count,
+                  struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+        gfs3_read_rsp        rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+#ifdef GF_TESTING_IO_XDATA
+        {
+                int ret = 0;
+                if (!xdata)
+                        xdata = dict_new ();
+
+                ret = dict_set_str (xdata, "testing-the-xdata-key",
+                                       "testing-xdata-value");
+        }
+#endif
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": READV %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        rsp.size = op_ret;
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, vector, count, iobref,
+                             (xdrproc_t)xdr_gfs3_read_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      uint32_t weak_checksum, uint8_t *strong_checksum,
+                      dict_t *xdata)
+{
+        gfs3_rchecksum_rsp   rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": RCHECKSUM %"PRId64" (%s)==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        rsp.weak_checksum = weak_checksum;
+
+        rsp.strong_checksum.strong_checksum_val = (char *)strong_checksum;
+        rsp.strong_checksum.strong_checksum_len = MD5_DIGEST_LENGTH;
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_rchecksum_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        server_state_t      *state    = NULL;
+        server_ctx_t        *serv_ctx = NULL;
+        rpcsvc_request_t    *req      = NULL;
+        uint64_t             fd_no    = 0;
+        gfs3_open_rsp        rsp      = {0,};
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, (op_errno == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_ERROR,
+                        "%"PRId64": OPEN %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        serv_ctx = server_ctx_get (frame->root->client, this);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
+        }
+
+        fd_bind (fd);
+        fd_no = gf_fd_unused_get (serv_ctx->fdtable, fd);
+        fd_ref (fd);
+        rsp.fd = fd_no;
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_open_rsp);
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *stbuf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        server_state_t      *state      = NULL;
+        server_ctx_t        *serv_ctx   = NULL;
+        inode_t             *link_inode = NULL;
+        rpcsvc_request_t    *req        = NULL;
+        uint64_t             fd_no      = 0;
+        gfs3_create_rsp      rsp        = {0,};
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": CREATE %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        /* TODO: log gfid too */
+        gf_log (frame->root->client->bound_xl->name, GF_LOG_TRACE,
+                "%"PRId64": CREATE %s (%s)",
+                frame->root->unique, state->loc.name,
+                uuid_utoa (stbuf->ia_gfid));
+
+        link_inode = inode_link (inode, state->loc.parent,
+                                 state->loc.name, stbuf);
+
+        if (!link_inode) {
+                op_ret = -1;
+                op_errno = ENOENT;
+                goto out;
+        }
+
+        if (link_inode != inode) {
+                /*
+                  VERY racy code (if used anywhere else)
+                  -- don't do this without understanding
+                */
+
+                inode_unref (fd->inode);
+                fd->inode = inode_ref (link_inode);
+        }
+
+        inode_lookup (link_inode);
+        inode_unref (link_inode);
+
+        serv_ctx = server_ctx_get (frame->root->client, this);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
+        }
+
+        fd_bind (fd);
+        fd_no = gf_fd_unused_get (serv_ctx->fdtable, fd);
+        fd_ref (fd);
+
+        if ((fd_no < 0) || (fd == 0)) {
+                op_ret = fd_no;
+                op_errno = errno;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+out:
+        rsp.fd        = fd_no;
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_create_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, const char *buf,
+                     struct iatt *stbuf, dict_t *xdata)
+{
+        gfs3_readlink_rsp    rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": READLINK %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.buf, stbuf);
+        rsp.path = (char *)buf;
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        if (!rsp.path)
+                rsp.path = "";
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_readlink_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                 dict_t *xdata)
+{
+        gfs3_stat_rsp        rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state  = CALL_STATE (frame);
+                gf_log (this->name, (op_errno == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_ERROR,
+                        "%"PRId64": STAT %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_stat_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        gfs3_setattr_rsp     rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": SETATTR %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.statpre, statpre);
+        gf_stat_from_iatt (&rsp.statpost, statpost);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_setattr_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        gfs3_fsetattr_rsp    rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state  = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FSETATTR %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.statpre, statpre);
+        gf_stat_from_iatt (&rsp.statpost, statpost);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_fsetattr_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+        gfs3_xattrop_rsp     rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": XATTROP %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, &rsp.dict.dict_val,
+                                    rsp.dict.dict_len, op_errno, out);
+
+out:
+        rsp.op_ret        = op_ret;
+        rsp.op_errno      = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_xattrop_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        gfs3_xattrop_rsp     rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FXATTROP %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, &rsp.dict.dict_val,
+                                    rsp.dict.dict_len, op_errno, out);
+
+out:
+        rsp.op_ret        = op_ret;
+        rsp.op_errno      = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_fxattrop_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
+{
+        gfs3_readdirp_rsp    rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+        int                  ret   = 0;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": READDIRP %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        /* (op_ret == 0) is valid, and means EOF */
+        if (op_ret) {
+                ret = serialize_rsp_direntp (entries, &rsp);
+                if (ret == -1) {
+                        op_ret   = -1;
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+        }
+
+        /* TODO: need more clear thoughts before calling this function. */
+        /* gf_link_inodes_from_dirent (this, state->fd->inode, entries); */
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_readdirp_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        readdirp_rsp_cleanup (&rsp);
+
+        return 0;
+}
+
+int
+server_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        gfs3_fallocate_rsp rsp   = {0,};
+        server_state_t    *state = NULL;
+        rpcsvc_request_t  *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state  = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FALLOCATE %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.statpre, statpre);
+        gf_stat_from_iatt (&rsp.statpost, statpost);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply(frame, req, &rsp, NULL, 0, NULL,
+                            (xdrproc_t) xdr_gfs3_fallocate_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        gfs3_discard_rsp rsp     = {0,};
+        server_state_t    *state = NULL;
+        rpcsvc_request_t  *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state  = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": DISCARD %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.statpre, statpre);
+        gf_stat_from_iatt (&rsp.statpost, statpost);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply(frame, req, &rsp, NULL, 0, NULL,
+                            (xdrproc_t) xdr_gfs3_discard_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        gfs3_zerofill_rsp  rsp    = {0,};
+        server_state_t    *state  = NULL;
+        rpcsvc_request_t  *req    = NULL;
+
+        req = frame->local;
+        state  = CALL_STATE (frame);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, (&rsp.xdata.xdata_val),
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": ZEROFILL%"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.statpre, statpre);
+        gf_stat_from_iatt (&rsp.statpost, statpost);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        server_submit_reply(frame, req, &rsp, NULL, 0, NULL,
+                            (xdrproc_t) xdr_gfs3_zerofill_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+/* Resume function section */
+
+int
+server_rchecksum_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state    = NULL;
+        int             op_ret   = 0;
+        int             op_errno = EINVAL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0) {
+                op_ret   = state->resolve.op_ret;
+                op_errno = state->resolve.op_errno;
+                goto err;
+        }
+
+        STACK_WIND (frame, server_rchecksum_cbk, bound_xl,
+                    bound_xl->fops->rchecksum, state->fd,
+                    state->offset, state->size, state->xdata);
+
+        return 0;
+err:
+        server_rchecksum_cbk (frame, NULL, frame->this, op_ret, op_errno, 0,
+                              NULL, NULL);
+
+        return 0;
+
+}
+
+int
+server_lk_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_lk_cbk, bound_xl, bound_xl->fops->lk,
+                    state->fd, state->cmd, &state->flock, state->xdata);
+
+        return 0;
+
+err:
+        server_lk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                       state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+server_rename_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+        int             op_ret = 0;
+        int             op_errno = 0;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0) {
+                op_ret   = state->resolve.op_ret;
+                op_errno = state->resolve.op_errno;
+                goto err;
+        }
+
+        if (state->resolve2.op_ret != 0) {
+                op_ret   = state->resolve2.op_ret;
+                op_errno = state->resolve2.op_errno;
+                goto err;
+        }
+
+        STACK_WIND (frame, server_rename_cbk,
+                    bound_xl, bound_xl->fops->rename,
+                    &state->loc, &state->loc2, state->xdata);
+        return 0;
+err:
+        server_rename_cbk (frame, NULL, frame->this, op_ret, op_errno,
+                           NULL, NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_link_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+        int             op_ret = 0;
+        int             op_errno = 0;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0) {
+                op_ret   = state->resolve.op_ret;
+                op_errno = state->resolve.op_errno;
+                goto err;
+        }
+
+        if (state->resolve2.op_ret != 0) {
+                op_ret   = state->resolve2.op_ret;
+                op_errno = state->resolve2.op_errno;
+                goto err;
+        }
+
+        state->loc2.inode = inode_ref (state->loc.inode);
+
+        STACK_WIND (frame, server_link_cbk, bound_xl, bound_xl->fops->link,
+                    &state->loc, &state->loc2, state->xdata);
+
+        return 0;
+err:
+        server_link_cbk (frame, NULL, frame->this, op_ret, op_errno,
+                         NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+server_symlink_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->loc.inode = inode_new (state->itable);
+
+        STACK_WIND (frame, server_symlink_cbk,
+                    bound_xl, bound_xl->fops->symlink,
+                    state->name, &state->loc, state->umask, state->xdata);
+
+        return 0;
+err:
+        server_symlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_access_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_access_cbk,
+                    bound_xl, bound_xl->fops->access,
+                    &state->loc, state->mask, state->xdata);
+        return 0;
+err:
+        server_access_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL);
+        return 0;
+}
+
+int
+server_fentrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        GF_UNUSED  int  ret   = -1;
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        if (!state->xdata)
+                state->xdata = dict_new ();
+
+        if (state->xdata)
+                ret = dict_set_str (state->xdata, "connection-id",
+                                    frame->root->client->client_uid);
+
+        STACK_WIND (frame, server_fentrylk_cbk, bound_xl,
+                    bound_xl->fops->fentrylk,
+                    state->volume, state->fd, state->name,
+                    state->cmd, state->type, state->xdata);
+
+        return 0;
+err:
+        server_fentrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL);
+        return 0;
+}
+
+
+int
+server_entrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        GF_UNUSED int   ret   = -1;
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        if (!state->xdata)
+                state->xdata = dict_new ();
+
+        if (state->xdata)
+                ret = dict_set_str (state->xdata, "connection-id",
+                                    frame->root->client->client_uid);
+
+        STACK_WIND (frame, server_entrylk_cbk,
+                    bound_xl, bound_xl->fops->entrylk,
+                    state->volume, &state->loc, state->name,
+                    state->cmd, state->type, state->xdata);
+        return 0;
+err:
+        server_entrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL);
+        return 0;
+}
+
+
+int
+server_finodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        GF_UNUSED int   ret   = -1;
+        server_state_t *state = NULL;
+
+        gf_log (bound_xl->name, GF_LOG_WARNING, "frame %p, xlator %p",
+                frame, bound_xl);
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        if (!state->xdata)
+                state->xdata = dict_new ();
+
+        if (state->xdata)
+                ret = dict_set_str (state->xdata, "connection-id",
+                                    frame->root->client->client_uid);
+
+        STACK_WIND (frame, server_finodelk_cbk, bound_xl,
+                    bound_xl->fops->finodelk, state->volume, state->fd,
+                    state->cmd, &state->flock, state->xdata);
+
+        return 0;
+err:
+        server_finodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL);
+
+        return 0;
+}
+
+int
+server_inodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        GF_UNUSED int   ret   = -1;
+        server_state_t *state = NULL;
+
+        gf_log (bound_xl->name, GF_LOG_WARNING, "frame %p, xlator %p",
+                frame, bound_xl);
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        if (!state->xdata)
+                state->xdata = dict_new ();
+
+        if (state->xdata)
+                ret = dict_set_str (state->xdata, "connection-id",
+                                    frame->root->client->client_uid);
+
+        STACK_WIND (frame, server_inodelk_cbk, bound_xl,
+                    bound_xl->fops->inodelk, state->volume, &state->loc,
+                    state->cmd, &state->flock, state->xdata);
+        return 0;
+err:
+        server_inodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL);
+        return 0;
+}
+
+int
+server_rmdir_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_rmdir_cbk, bound_xl, bound_xl->fops->rmdir,
+                    &state->loc, state->flags, state->xdata);
+        return 0;
+err:
+        server_rmdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+server_mkdir_resume (call_frame_t *frame, xlator_t *bound_xl)
+
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->loc.inode = inode_new (state->itable);
+
+        STACK_WIND (frame, server_mkdir_cbk,
+                    bound_xl, bound_xl->fops->mkdir,
+                    &(state->loc), state->mode, state->umask, state->xdata);
+
+        return 0;
+err:
+        server_mkdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_mknod_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->loc.inode = inode_new (state->itable);
+
+        STACK_WIND (frame, server_mknod_cbk,
+                    bound_xl, bound_xl->fops->mknod,
+                    &(state->loc), state->mode, state->dev,
+                    state->umask, state->xdata);
+
+        return 0;
+err:
+        server_mknod_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_fsyncdir_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fsyncdir_cbk,
+                    bound_xl,
+                    bound_xl->fops->fsyncdir,
+                    state->fd, state->flags, state->xdata);
+        return 0;
+
+err:
+        server_fsyncdir_cbk (frame, NULL, frame->this,
+                             state->resolve.op_ret,
+                             state->resolve.op_errno, NULL);
+        return 0;
+}
+
+
+int
+server_readdir_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t  *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        GF_ASSERT (state->fd);
+
+        STACK_WIND (frame, server_readdir_cbk,
+                    bound_xl,
+                    bound_xl->fops->readdir,
+                    state->fd, state->size, state->offset, state->xdata);
+
+        return 0;
+err:
+        server_readdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+server_readdirp_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t  *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_readdirp_cbk, bound_xl,
+                    bound_xl->fops->readdirp, state->fd, state->size,
+                    state->offset, state->dict);
+
+        return 0;
+err:
+        server_readdirp_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_opendir_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t  *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->fd = fd_create (state->loc.inode, frame->root->pid);
+        if (!state->fd) {
+                gf_log ("server", GF_LOG_ERROR, "could not create the fd");
+                goto err;
+        }
+
+        STACK_WIND (frame, server_opendir_cbk,
+                    bound_xl, bound_xl->fops->opendir,
+                    &state->loc, state->fd, state->xdata);
+        return 0;
+err:
+        server_opendir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_statfs_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t      *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret !=0)
+                goto err;
+
+        STACK_WIND (frame, server_statfs_cbk,
+                    bound_xl, bound_xl->fops->statfs,
+                    &state->loc, state->xdata);
+        return 0;
+
+err:
+        server_statfs_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_removexattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_removexattr_cbk,
+                    bound_xl, bound_xl->fops->removexattr,
+                    &state->loc, state->name, state->xdata);
+        return 0;
+err:
+        server_removexattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                                state->resolve.op_errno, NULL);
+        return 0;
+}
+
+int
+server_fremovexattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fremovexattr_cbk,
+                    bound_xl, bound_xl->fops->fremovexattr,
+                    state->fd, state->name, state->xdata);
+        return 0;
+err:
+        server_fremovexattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                                 state->resolve.op_errno, NULL);
+        return 0;
+}
+
+int
+server_fgetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fgetxattr_cbk,
+                    bound_xl, bound_xl->fops->fgetxattr,
+                    state->fd, state->name, state->xdata);
+        return 0;
+err:
+        server_fgetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                              state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_xattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_xattrop_cbk,
+                    bound_xl, bound_xl->fops->xattrop,
+                    &state->loc, state->flags, state->dict, state->xdata);
+        return 0;
+err:
+        server_xattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+server_fxattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fxattrop_cbk,
+                    bound_xl, bound_xl->fops->fxattrop,
+                    state->fd, state->flags, state->dict, state->xdata);
+        return 0;
+err:
+        server_fxattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+server_fsetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_setxattr_cbk,
+                    bound_xl, bound_xl->fops->fsetxattr,
+                    state->fd, state->dict, state->flags, state->xdata);
+        return 0;
+err:
+        server_fsetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                              state->resolve.op_errno, NULL);
+
+        return 0;
+}
+
+int
+server_unlink_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_unlink_cbk,
+                    bound_xl, bound_xl->fops->unlink,
+                    &state->loc, state->flags, state->xdata);
+        return 0;
+err:
+        server_unlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+server_truncate_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_truncate_cbk,
+                    bound_xl, bound_xl->fops->truncate,
+                    &state->loc, state->offset, state->xdata);
+        return 0;
+err:
+        server_truncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+
+int
+server_fstat_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t     *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fstat_cbk,
+                    bound_xl, bound_xl->fops->fstat,
+                    state->fd, state->xdata);
+        return 0;
+err:
+        server_fstat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_setxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_setxattr_cbk,
+                    bound_xl, bound_xl->fops->setxattr,
+                    &state->loc, state->dict, state->flags, state->xdata);
+        return 0;
+err:
+        server_setxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL);
+
+        return 0;
+}
+
+
+int
+server_getxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_getxattr_cbk,
+                    bound_xl, bound_xl->fops->getxattr,
+                    &state->loc, state->name, state->xdata);
+        return 0;
+err:
+        server_getxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_ftruncate_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t    *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_ftruncate_cbk,
+                    bound_xl, bound_xl->fops->ftruncate,
+                    state->fd, state->offset, state->xdata);
+        return 0;
+err:
+        server_ftruncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                              state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+server_flush_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t    *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_flush_cbk,
+                    bound_xl, bound_xl->fops->flush, state->fd, state->xdata);
+        return 0;
+err:
+        server_flush_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL);
+
+        return 0;
+}
+
+
+int
+server_fsync_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t    *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fsync_cbk,
+                    bound_xl, bound_xl->fops->fsync,
+                    state->fd, state->flags, state->xdata);
+        return 0;
+err:
+        server_fsync_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+server_writev_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t   *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_writev_cbk,
+                    bound_xl, bound_xl->fops->writev,
+                    state->fd, state->payload_vector, state->payload_count,
+                    state->offset, state->flags, state->iobref, state->xdata);
+
+        return 0;
+err:
+        server_writev_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_readv_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t    *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_readv_cbk,
+                    bound_xl, bound_xl->fops->readv,
+                    state->fd, state->size, state->offset, state->flags, state->xdata);
+
+        return 0;
+err:
+        server_readv_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, 0, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_create_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->loc.inode = inode_new (state->itable);
+
+        state->fd = fd_create (state->loc.inode, frame->root->pid);
+        if (!state->fd) {
+                gf_log ("server", GF_LOG_ERROR, "fd creation for the inode %s "
+                        "failed", state->loc.inode?
+                        uuid_utoa (state->loc.inode->gfid):NULL);
+                state->resolve.op_ret = -1;
+                state->resolve.op_errno = ENOMEM;
+                goto err;
+        }
+        state->fd->flags = state->flags;
+
+        STACK_WIND (frame, server_create_cbk,
+                    bound_xl, bound_xl->fops->create,
+                    &(state->loc), state->flags, state->mode,
+                    state->umask, state->fd, state->xdata);
+
+        return 0;
+err:
+        server_create_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL,
+                           NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_open_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t  *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->fd = fd_create (state->loc.inode, frame->root->pid);
+        state->fd->flags = state->flags;
+
+        STACK_WIND (frame, server_open_cbk,
+                    bound_xl, bound_xl->fops->open,
+                    &state->loc, state->flags, state->fd, state->xdata);
+
+        return 0;
+err:
+        server_open_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                         state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_readlink_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_readlink_cbk,
+                    bound_xl, bound_xl->fops->readlink,
+                    &state->loc, state->size, state->xdata);
+        return 0;
+err:
+        server_readlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_fsetattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fsetattr_cbk,
+                    bound_xl, bound_xl->fops->fsetattr,
+                    state->fd, &state->stbuf, state->valid, state->xdata);
+        return 0;
+err:
+        server_fsetattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+server_setattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_setattr_cbk,
+                    bound_xl, bound_xl->fops->setattr,
+                    &state->loc, &state->stbuf, state->valid, state->xdata);
+        return 0;
+err:
+        server_setattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+server_stat_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_stat_cbk,
+                    bound_xl, bound_xl->fops->stat, &state->loc, state->xdata);
+        return 0;
+err:
+        server_stat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                         state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+server_lookup_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t    *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        if (!state->loc.inode)
+                state->loc.inode = inode_new (state->itable);
+        else
+                state->is_revalidate = 1;
+
+        STACK_WIND (frame, server_lookup_cbk,
+                    bound_xl, bound_xl->fops->lookup,
+                    &state->loc, state->xdata);
+
+        return 0;
+err:
+        server_lookup_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+server_fallocate_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fallocate_cbk,
+                    bound_xl, bound_xl->fops->fallocate,
+                    state->fd, state->flags, state->offset, state->size,
+                    state->xdata);
+        return 0;
+err:
+        server_fallocate_cbk(frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+server_discard_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_discard_cbk,
+                    bound_xl, bound_xl->fops->discard,
+                    state->fd, state->offset, state->size, state->xdata);
+        return 0;
+err:
+        server_discard_cbk(frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+server_zerofill_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_zerofill_cbk,
+                    bound_xl, bound_xl->fops->zerofill,
+                    state->fd, state->offset, state->size, state->xdata);
+        return 0;
+err:
+        server_zerofill_cbk(frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+
+/* Fop section */
+
+int
+server3_3_stat (rpcsvc_request_t *req)
+{
+        server_state_t *state    = NULL;
+        call_frame_t   *frame    = NULL;
+        gfs3_stat_req   args     = {{0,},};
+        int             ret      = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                return 0;
+
+        /* Initialize args first, then decode */
+
+        ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_stat_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_STAT;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+
+        ret = 0;
+        resolve_and_resume (frame, server_stat_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_setattr (rpcsvc_request_t *req)
+{
+        server_state_t   *state                 = NULL;
+        call_frame_t     *frame                 = NULL;
+        gfs3_setattr_req  args                  = {{0,},};
+        int               ret                   = -1;
+        int               op_errno = 0;
+
+        if (!req)
+                return 0;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_setattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_SETATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        gf_stat_to_iatt (&args.stbuf, &state->stbuf);
+        state->valid = args.valid;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_setattr_resume);
+
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        free (args.xdata.xdata_val);
+
+        return ret;
+}
+
+
+int
+server3_3_fsetattr (rpcsvc_request_t *req)
+{
+        server_state_t    *state = NULL;
+        call_frame_t      *frame = NULL;
+        gfs3_fsetattr_req  args  = {0,};
+        int                ret   = -1;
+        int                op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fsetattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FSETATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+
+        gf_stat_to_iatt (&args.stbuf, &state->stbuf);
+        state->valid = args.valid;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fsetattr_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_fallocate(rpcsvc_request_t *req)
+{
+        server_state_t    *state = NULL;
+        call_frame_t      *frame = NULL;
+        gfs3_fallocate_req args  = {{0},};
+        int                ret   = -1;
+        int                op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fallocate_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FALLOCATE;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+
+        state->flags = args.flags;
+        state->offset = args.offset;
+        state->size = args.size;
+        memcpy(state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fallocate_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_discard(rpcsvc_request_t *req)
+{
+        server_state_t    *state = NULL;
+        call_frame_t      *frame = NULL;
+        gfs3_discard_req args    = {{0},};
+        int                ret   = -1;
+        int                op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_discard_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_DISCARD;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+
+        state->offset = args.offset;
+        state->size = args.size;
+        memcpy(state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_discard_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_zerofill(rpcsvc_request_t *req)
+{
+        server_state_t       *state      = NULL;
+        call_frame_t         *frame      = NULL;
+        gfs3_zerofill_req     args       = {{0},};
+        int                   ret        = -1;
+        int                   op_errno   = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_zerofill_req);
+        if (ret < 0) {
+                /*failed to decode msg*/;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                /* something wrong, mostly insufficient memory*/
+                req->rpc_err = GARBAGE_ARGS; /* TODO */
+                goto out;
+        }
+        frame->root->op = GF_FOP_ZEROFILL;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+
+        state->offset = args.offset;
+        state->size = args.size;
+        memcpy(state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl, state->xdata,
+                                      (args.xdata.xdata_val),
+                                      (args.xdata.xdata_len), ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_zerofill_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                req->rpc_err = GARBAGE_ARGS;
+
+        return ret;
+}
+
+int
+server3_3_readlink (rpcsvc_request_t *req)
+{
+        server_state_t    *state                 = NULL;
+        call_frame_t      *frame                 = NULL;
+        gfs3_readlink_req  args                  = {{0,},};
+        int                ret                   = -1;
+        int                op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_readlink_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_READLINK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        state->size  = args.size;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_readlink_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_create (rpcsvc_request_t *req)
+{
+        server_state_t  *state    = NULL;
+        call_frame_t    *frame    = NULL;
+        gfs3_create_req  args     = {{0,},};
+        int              ret      = -1;
+        int              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_create_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_CREATE;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+
+        state->resolve.bname  = gf_strdup (args.bname);
+        state->mode           = args.mode;
+        state->umask          = args.umask;
+        state->flags          = gf_flags_to_flags (args.flags);
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+
+        if (state->flags & O_EXCL) {
+                state->resolve.type = RESOLVE_NOT;
+        } else {
+                state->resolve.type = RESOLVE_DONTCARE;
+        }
+
+        /* TODO: can do alloca for xdata field instead of stdalloc */
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_create_resume);
+
+out:
+        /* memory allocated by libc, don't use GF_FREE */
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_open (rpcsvc_request_t *req)
+{
+        server_state_t *state                 = NULL;
+        call_frame_t   *frame                 = NULL;
+        gfs3_open_req   args                  = {{0,},};
+        int             ret                   = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_open_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_OPEN;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        state->flags = gf_flags_to_flags (args.flags);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_open_resume);
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        free (args.xdata.xdata_val);
+
+        return ret;
+}
+
+
+int
+server3_3_readv (rpcsvc_request_t *req)
+{
+        server_state_t *state = NULL;
+        call_frame_t   *frame = NULL;
+        gfs3_read_req   args  = {{0,},};
+        int             ret   = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                goto out;
+
+        ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_read_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_READ;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+        state->size           = args.size;
+        state->offset         = args.offset;
+        state->flags          = args.flag;
+
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_readv_resume);
+out:
+        /* memory allocated by libc, don't use GF_FREE */
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_writev (rpcsvc_request_t *req)
+{
+        server_state_t      *state  = NULL;
+        call_frame_t        *frame  = NULL;
+        gfs3_write_req       args   = {{0,},};
+        ssize_t              len    = 0;
+        int                  i      = 0;
+        int                  ret    = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        len = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_write_req);
+        if (len < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_WRITE;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        state->offset        = args.offset;
+        state->flags         = args.flag;
+        state->iobref        = iobref_ref (req->iobref);
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        if (len < req->msg[0].iov_len) {
+                state->payload_vector[0].iov_base
+                        = (req->msg[0].iov_base + len);
+                state->payload_vector[0].iov_len
+                        = req->msg[0].iov_len - len;
+                state->payload_count = 1;
+        }
+
+        for (i = 1; i < req->count; i++) {
+                state->payload_vector[state->payload_count++]
+                        = req->msg[i];
+        }
+
+        for (i = 0; i < state->payload_count; i++) {
+                state->size += state->payload_vector[i].iov_len;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+#ifdef GF_TESTING_IO_XDATA
+        dict_dump (state->xdata);
+#endif
+
+        ret = 0;
+        resolve_and_resume (frame, server_writev_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+#define SERVER3_3_VECWRITE_START 0
+#define SERVER3_3_VECWRITE_READING_HDR 1
+#define SERVER3_3_VECWRITE_READING_OPAQUE 2
+
+int
+server3_3_writev_vecsizer (int state, ssize_t *readsize, char *base_addr,
+                        char *curr_addr)
+{
+        ssize_t         size = 0;
+        int             nextstate = 0;
+        gfs3_write_req  write_req = {{0,},};
+        XDR             xdr;
+
+        switch (state) {
+        case SERVER3_3_VECWRITE_START:
+                size = xdr_sizeof ((xdrproc_t) xdr_gfs3_write_req,
+                                   &write_req);
+                *readsize = size;
+                nextstate = SERVER3_3_VECWRITE_READING_HDR;
+                break;
+        case SERVER3_3_VECWRITE_READING_HDR:
+                size = xdr_sizeof ((xdrproc_t) xdr_gfs3_write_req,
+                                           &write_req);
+
+                xdrmem_create (&xdr, base_addr, size, XDR_DECODE);
+
+                /* This will fail if there is xdata sent from client, if not,
+                   well and good */
+                xdr_gfs3_write_req (&xdr, &write_req);
+
+                /* need to round off to proper roof (%4), as XDR packing pads
+                   the end of opaque object with '0' */
+                size = roof (write_req.xdata.xdata_len, 4);
+
+                *readsize = size;
+
+                if (!size)
+                        nextstate = SERVER3_3_VECWRITE_START;
+                else
+                        nextstate = SERVER3_3_VECWRITE_READING_OPAQUE;
+
+                free (write_req.xdata.xdata_val);
+
+                break;
+
+        case SERVER3_3_VECWRITE_READING_OPAQUE:
+                *readsize = 0;
+                nextstate = SERVER3_3_VECWRITE_START;
+                break;
+        default:
+                gf_log ("server", GF_LOG_ERROR, "wrong state: %d", state);
+        }
+
+        return nextstate;
+}
+
+
+int
+server3_3_release (rpcsvc_request_t *req)
+{
+        client_t         *client   = NULL;
+        server_ctx_t     *serv_ctx = NULL;
+        gfs3_release_req  args     = {{0,},};
+        gf_common_rsp     rsp      = {0,};
+        int               ret      = -1;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_release_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        client = req->trans->xl_private;
+        if (!client) {
+                /* Handshake is not complete yet. */
+                req->rpc_err = SYSTEM_ERR;
+                goto out;
+        }
+
+        serv_ctx = server_ctx_get (client, client->this);
+        if (serv_ctx == NULL) {
+                gf_log (req->trans->name, GF_LOG_INFO,
+                        "server_ctx_get() failed");
+                req->rpc_err = SYSTEM_ERR;
+                goto out;
+        }
+
+        gf_fd_put (serv_ctx->fdtable, args.fd);
+
+        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+server3_3_releasedir (rpcsvc_request_t *req)
+{
+        client_t            *client   = NULL;
+        server_ctx_t        *serv_ctx = NULL;
+        gfs3_releasedir_req  args     = {{0,},};
+        gf_common_rsp        rsp      = {0,};
+        int                  ret      = -1;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_release_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        client = req->trans->xl_private;
+        if (!client) {
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        serv_ctx = server_ctx_get (client, client->this);
+        if (serv_ctx == NULL) {
+                gf_log (req->trans->name, GF_LOG_INFO,
+                        "server_ctx_get() failed");
+                req->rpc_err = SYSTEM_ERR;
+                goto out;
+        }
+
+        gf_fd_put (serv_ctx->fdtable, args.fd);
+
+        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+server3_3_fsync (rpcsvc_request_t *req)
+{
+        server_state_t *state = NULL;
+        call_frame_t   *frame = NULL;
+        gfs3_fsync_req  args  = {{0,},};
+        int             ret   = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fsync_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FSYNC;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        state->flags         = args.data;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fsync_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_flush (rpcsvc_request_t *req)
+{
+        server_state_t *state = NULL;
+        call_frame_t   *frame = NULL;
+        gfs3_flush_req  args  = {{0,},};
+        int             ret   = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_flush_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FLUSH;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_flush_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_ftruncate (rpcsvc_request_t *req)
+{
+        server_state_t     *state = NULL;
+        call_frame_t       *frame = NULL;
+        gfs3_ftruncate_req  args  = {{0,},};
+        int                 ret   = -1;
+        int                 op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_ftruncate_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FTRUNCATE;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+        state->offset         = args.offset;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_ftruncate_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_fstat (rpcsvc_request_t *req)
+{
+        server_state_t *state = NULL;
+        call_frame_t   *frame = NULL;
+        gfs3_fstat_req  args  = {{0,},};
+        int             ret   = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fstat_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FSTAT;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_MUST;
+        state->resolve.fd_no   = args.fd;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fstat_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_truncate (rpcsvc_request_t *req)
+{
+        server_state_t    *state                 = NULL;
+        call_frame_t      *frame                 = NULL;
+        gfs3_truncate_req  args                  = {{0,},};
+        int                ret                   = -1;
+        int                op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_truncate_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_TRUNCATE;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+        state->offset        = args.offset;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_truncate_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_unlink (rpcsvc_request_t *req)
+{
+        server_state_t  *state                  = NULL;
+        call_frame_t    *frame                  = NULL;
+        gfs3_unlink_req  args                   = {{0,},};
+        int              ret                    = -1;
+        int              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_unlink_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_UNLINK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.bname  = gf_strdup (args.bname);
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+
+        state->flags = args.xflags;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_unlink_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_setxattr (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        dict_t              *dict                  = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_setxattr_req    args                  = {{0,},};
+        int32_t              ret                   = -1;
+        int32_t              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.dict.dict_val = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_setxattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_SETXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type     = RESOLVE_MUST;
+        state->flags            = args.flags;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      dict,
+                                      (args.dict.dict_val),
+                                      (args.dict.dict_len), ret,
+                                      op_errno, out);
+
+        state->dict = dict;
+
+        /* There can be some commands hidden in key, check and proceed */
+        gf_server_check_setxattr_cmd (frame, dict);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_setxattr_resume);
+
+        return ret;
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+
+
+int
+server3_3_fsetxattr (rpcsvc_request_t *req)
+{
+        server_state_t      *state                = NULL;
+        dict_t              *dict                 = NULL;
+        call_frame_t        *frame                = NULL;
+        gfs3_fsetxattr_req   args                 = {{0,},};
+        int32_t              ret                  = -1;
+        int32_t              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.dict.dict_val = alloca (req->msg[0].iov_len);
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fsetxattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FSETXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type      = RESOLVE_MUST;
+        state->resolve.fd_no     = args.fd;
+        state->flags             = args.flags;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      dict,
+                                      (args.dict.dict_val),
+                                      (args.dict.dict_len), ret,
+                                      op_errno, out);
+
+        state->dict = dict;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fsetxattr_resume);
+
+        return ret;
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+
+
+int
+server3_3_fxattrop (rpcsvc_request_t *req)
+{
+        dict_t              *dict                 = NULL;
+        server_state_t      *state                = NULL;
+        call_frame_t        *frame                = NULL;
+        gfs3_fxattrop_req    args                 = {{0,},};
+        int32_t              ret                  = -1;
+        int32_t              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.dict.dict_val = alloca (req->msg[0].iov_len);
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fxattrop_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FXATTROP;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_MUST;
+        state->resolve.fd_no   = args.fd;
+        state->flags           = args.flags;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      dict,
+                                      (args.dict.dict_val),
+                                      (args.dict.dict_len), ret,
+                                      op_errno, out);
+
+        state->dict = dict;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fxattrop_resume);
+
+        return ret;
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+
+
+int
+server3_3_xattrop (rpcsvc_request_t *req)
+{
+        dict_t              *dict                  = NULL;
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_xattrop_req     args                  = {{0,},};
+        int32_t              ret                   = -1;
+        int32_t              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.dict.dict_val = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_xattrop_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_XATTROP;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_MUST;
+        state->flags           = args.flags;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      dict,
+                                      (args.dict.dict_val),
+                                      (args.dict.dict_len), ret,
+                                      op_errno, out);
+
+        state->dict = dict;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_xattrop_resume);
+
+        return ret;
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+
+int
+server3_3_getxattr (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_getxattr_req    args                  = {{0,},};
+        int                  ret                   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.name = alloca (256);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_getxattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_GETXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        if (args.namelen) {
+                state->name = gf_strdup (args.name);
+                /* There can be some commands hidden in key, check and proceed */
+                gf_server_check_getxattr_cmd (frame, state->name);
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_getxattr_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_fgetxattr (rpcsvc_request_t *req)
+{
+        server_state_t      *state      = NULL;
+        call_frame_t        *frame      = NULL;
+        gfs3_fgetxattr_req   args       = {{0,},};
+        int                  ret        = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.name = alloca (256);
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fgetxattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FGETXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        if (args.namelen)
+                state->name = gf_strdup (args.name);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fgetxattr_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_removexattr (rpcsvc_request_t *req)
+{
+        server_state_t       *state                 = NULL;
+        call_frame_t         *frame                 = NULL;
+        gfs3_removexattr_req  args                  = {{0,},};
+        int                   ret                   = -1;
+        int                   op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.name = alloca (256);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_removexattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_REMOVEXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+        state->name           = gf_strdup (args.name);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_removexattr_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_fremovexattr (rpcsvc_request_t *req)
+{
+        server_state_t       *state                 = NULL;
+        call_frame_t         *frame                 = NULL;
+        gfs3_fremovexattr_req  args                  = {{0,},};
+        int                   ret                   = -1;
+        int                   op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.name = alloca (4096);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fremovexattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FREMOVEXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+        state->name           = gf_strdup (args.name);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fremovexattr_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+
+int
+server3_3_opendir (rpcsvc_request_t *req)
+{
+        server_state_t   *state                 = NULL;
+        call_frame_t     *frame                 = NULL;
+        gfs3_opendir_req  args                  = {{0,},};
+        int               ret                   = -1;
+        int               op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_opendir_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_OPENDIR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_opendir_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_readdirp (rpcsvc_request_t *req)
+{
+        server_state_t      *state        = NULL;
+        call_frame_t        *frame        = NULL;
+        gfs3_readdirp_req    args         = {{0,},};
+        size_t               headers_size = 0;
+        int                  ret          = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_readdirp_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_READDIRP;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        /* FIXME: this should go away when variable sized iobufs are introduced
+         * and transport layer can send msgs bigger than current page-size.
+         */
+        headers_size = sizeof (struct rpc_msg) + sizeof (gfs3_readdir_rsp);
+        if ((frame->this->ctx->page_size < args.size)
+            || ((frame->this->ctx->page_size - args.size) < headers_size)) {
+                state->size = frame->this->ctx->page_size - headers_size;
+        } else {
+                state->size   = args.size;
+        }
+
+        state->resolve.type = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        state->offset = args.offset;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        /* here, dict itself works as xdata */
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->dict,
+                                      (args.dict.dict_val),
+                                      (args.dict.dict_len), ret,
+                                      op_errno, out);
+
+
+        ret = 0;
+        resolve_and_resume (frame, server_readdirp_resume);
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        free (args.dict.dict_val);
+
+        return ret;
+}
+
+int
+server3_3_readdir (rpcsvc_request_t *req)
+{
+        server_state_t      *state        = NULL;
+        call_frame_t        *frame        = NULL;
+        gfs3_readdir_req     args         = {{0,},};
+        size_t               headers_size = 0;
+        int                  ret          = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_readdir_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_READDIR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        /* FIXME: this should go away when variable sized iobufs are introduced
+         * and transport layer can send msgs bigger than current page-size.
+         */
+        headers_size = sizeof (struct rpc_msg) + sizeof (gfs3_readdir_rsp);
+        if ((frame->this->ctx->page_size < args.size)
+            || ((frame->this->ctx->page_size - args.size) < headers_size)) {
+                state->size = frame->this->ctx->page_size - headers_size;
+        } else {
+                state->size   = args.size;
+        }
+
+        state->resolve.type = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        state->offset = args.offset;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_readdir_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_fsyncdir (rpcsvc_request_t *req)
+{
+        server_state_t      *state = NULL;
+        call_frame_t        *frame = NULL;
+        gfs3_fsyncdir_req    args  = {{0,},};
+        int                  ret   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fsyncdir_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FSYNCDIR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        state->flags = args.data;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fsyncdir_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_mknod (rpcsvc_request_t *req)
+{
+        server_state_t      *state                  = NULL;
+        call_frame_t        *frame                  = NULL;
+        gfs3_mknod_req       args                   = {{0,},};
+        int                  ret                    = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_mknod_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_MKNOD;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_NOT;
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+        state->resolve.bname   = gf_strdup (args.bname);
+
+        state->mode  = args.mode;
+        state->dev   = args.dev;
+        state->umask = args.umask;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_mknod_resume);
+
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        /* memory allocated by libc, don't use GF_FREE */
+        free (args.xdata.xdata_val);
+
+        return ret;
+
+}
+
+
+int
+server3_3_mkdir (rpcsvc_request_t *req)
+{
+        server_state_t      *state                  = NULL;
+        call_frame_t        *frame                  = NULL;
+        gfs3_mkdir_req       args                   = {{0,},};
+        int                  ret                    = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_mkdir_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_MKDIR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_NOT;
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+        state->resolve.bname   = gf_strdup (args.bname);
+
+        state->mode  = args.mode;
+        state->umask = args.umask;
+
+        /* TODO: can do alloca for xdata field instead of stdalloc */
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_mkdir_resume);
+
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        free (args.xdata.xdata_val);
+
+        return ret;
+}
+
+
+int
+server3_3_rmdir (rpcsvc_request_t *req)
+{
+        server_state_t      *state                  = NULL;
+        call_frame_t        *frame                  = NULL;
+        gfs3_rmdir_req       args                   = {{0,},};
+        int                  ret                    = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_rmdir_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_RMDIR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_MUST;
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+        state->resolve.bname   = gf_strdup (args.bname);
+
+        state->flags = args.xflags;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_rmdir_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_inodelk (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_inodelk_req     args                  = {{0,},};
+        int                  cmd                   = 0;
+        int                  ret                   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.volume = alloca (256);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_inodelk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_INODELK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_EXACT;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        cmd = args.cmd;
+        switch (cmd) {
+        case GF_LK_GETLK:
+                state->cmd = F_GETLK;
+                break;
+        case GF_LK_SETLK:
+                state->cmd = F_SETLK;
+                break;
+        case GF_LK_SETLKW:
+                state->cmd = F_SETLKW;
+                break;
+        }
+
+        state->type = args.type;
+        state->volume = gf_strdup (args.volume);
+
+        gf_proto_flock_to_flock (&args.flock, &state->flock);
+
+        switch (state->type) {
+        case GF_LK_F_RDLCK:
+                state->flock.l_type = F_RDLCK;
+                break;
+        case GF_LK_F_WRLCK:
+                state->flock.l_type = F_WRLCK;
+                break;
+        case GF_LK_F_UNLCK:
+                state->flock.l_type = F_UNLCK;
+                break;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_inodelk_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        free (args.flock.lk_owner.lk_owner_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_finodelk (rpcsvc_request_t *req)
+{
+        server_state_t      *state        = NULL;
+        call_frame_t        *frame        = NULL;
+        gfs3_finodelk_req    args         = {{0,},};
+        int                  ret          = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.volume = alloca (256);
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_finodelk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FINODELK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type = RESOLVE_EXACT;
+        state->volume = gf_strdup (args.volume);
+        state->resolve.fd_no = args.fd;
+        state->cmd = args.cmd;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        switch (state->cmd) {
+        case GF_LK_GETLK:
+                state->cmd = F_GETLK;
+                break;
+        case GF_LK_SETLK:
+                state->cmd = F_SETLK;
+                break;
+        case GF_LK_SETLKW:
+                state->cmd = F_SETLKW;
+                break;
+        }
+
+        state->type = args.type;
+
+        gf_proto_flock_to_flock (&args.flock, &state->flock);
+
+        switch (state->type) {
+        case GF_LK_F_RDLCK:
+                state->flock.l_type = F_RDLCK;
+                break;
+        case GF_LK_F_WRLCK:
+                state->flock.l_type = F_WRLCK;
+                break;
+        case GF_LK_F_UNLCK:
+                state->flock.l_type = F_UNLCK;
+                break;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_finodelk_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        free (args.flock.lk_owner.lk_owner_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_entrylk (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_entrylk_req     args                  = {{0,},};
+        int                  ret                   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.volume = alloca (256);
+        args.name   = alloca (256);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_entrylk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_ENTRYLK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_EXACT;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        if (args.namelen)
+                state->name   = gf_strdup (args.name);
+        state->volume         = gf_strdup (args.volume);
+
+        state->cmd            = args.cmd;
+        state->type           = args.type;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_entrylk_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_fentrylk (rpcsvc_request_t *req)
+{
+        server_state_t      *state        = NULL;
+        call_frame_t        *frame        = NULL;
+        gfs3_fentrylk_req    args         = {{0,},};
+        int                  ret          = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.name   = alloca (256);
+        args.volume = alloca (256);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fentrylk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FENTRYLK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type = RESOLVE_EXACT;
+        state->resolve.fd_no = args.fd;
+        state->cmd  = args.cmd;
+        state->type = args.type;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        if (args.namelen)
+                state->name = gf_strdup (args.name);
+        state->volume = gf_strdup (args.volume);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fentrylk_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_access (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_access_req      args                  = {{0,},};
+        int                  ret                   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_access_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_ACCESS;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+        state->mask          = args.mask;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_access_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_symlink (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_symlink_req     args                  = {{0,},};
+        int                  ret                   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname    = alloca (req->msg[0].iov_len);
+        args.linkname = alloca (4096);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_symlink_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_SYMLINK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_NOT;
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+        state->resolve.bname  = gf_strdup (args.bname);
+        state->name           = gf_strdup (args.linkname);
+        state->umask          = args.umask;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_symlink_resume);
+
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        /* memory allocated by libc, don't use GF_FREE */
+        free (args.xdata.xdata_val);
+
+        return ret;
+}
+
+
+
+int
+server3_3_link (rpcsvc_request_t *req)
+{
+        server_state_t      *state                     = NULL;
+        call_frame_t        *frame                     = NULL;
+        gfs3_link_req        args                      = {{0,},};
+        int                  ret                       = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.newbname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_link_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_LINK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.oldgfid, 16);
+
+        state->resolve2.type   = RESOLVE_NOT;
+        state->resolve2.bname  = gf_strdup (args.newbname);
+        memcpy (state->resolve2.pargfid, args.newgfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_link_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_rename (rpcsvc_request_t *req)
+{
+        server_state_t      *state                     = NULL;
+        call_frame_t        *frame                     = NULL;
+        gfs3_rename_req      args                      = {{0,},};
+        int                  ret                       = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.oldbname = alloca (req->msg[0].iov_len);
+        args.newbname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_rename_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_RENAME;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.bname  = gf_strdup (args.oldbname);
+        memcpy (state->resolve.pargfid, args.oldgfid, 16);
+
+        state->resolve2.type  = RESOLVE_MAY;
+        state->resolve2.bname = gf_strdup (args.newbname);
+        memcpy (state->resolve2.pargfid, args.newgfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_rename_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_lk (rpcsvc_request_t *req)
+{
+        server_state_t      *state = NULL;
+        call_frame_t        *frame = NULL;
+        gfs3_lk_req          args  = {{0,},};
+        int                  ret   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_lk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_LK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.fd_no = args.fd;
+        state->cmd =  args.cmd;
+        state->type = args.type;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        switch (state->cmd) {
+        case GF_LK_GETLK:
+                state->cmd = F_GETLK;
+                break;
+        case GF_LK_SETLK:
+                state->cmd = F_SETLK;
+                break;
+        case GF_LK_SETLKW:
+                state->cmd = F_SETLKW;
+                break;
+        case GF_LK_RESLK_LCK:
+                state->cmd = F_RESLK_LCK;
+                break;
+        case GF_LK_RESLK_LCKW:
+                state->cmd = F_RESLK_LCKW;
+                break;
+        case GF_LK_RESLK_UNLCK:
+                state->cmd = F_RESLK_UNLCK;
+                break;
+        case GF_LK_GETLK_FD:
+                state->cmd = F_GETLK_FD;
+                break;
+
+        }
+
+
+        gf_proto_flock_to_flock (&args.flock, &state->flock);
+
+        switch (state->type) {
+        case GF_LK_F_RDLCK:
+                state->flock.l_type = F_RDLCK;
+                break;
+        case GF_LK_F_WRLCK:
+                state->flock.l_type = F_WRLCK;
+                break;
+        case GF_LK_F_UNLCK:
+                state->flock.l_type = F_UNLCK;
+                break;
+        default:
+                gf_log (frame->root->client->bound_xl->name, GF_LOG_ERROR,
+                        "fd - %"PRId64" (%s): Unknown lock type: %"PRId32"!",
+                        state->resolve.fd_no,
+                        uuid_utoa (state->fd->inode->gfid), state->type);
+                break;
+        }
+
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_lk_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        free (args.flock.lk_owner.lk_owner_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_rchecksum (rpcsvc_request_t *req)
+{
+        server_state_t      *state = NULL;
+        call_frame_t        *frame = NULL;
+        gfs3_rchecksum_req   args  = {0,};
+        int                  ret   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_rchecksum_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_RCHECKSUM;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MAY;
+        state->resolve.fd_no = args.fd;
+        state->offset        = args.offset;
+        state->size          = args.len;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_rchecksum_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server_null (rpcsvc_request_t *req)
+{
+        gf_common_rsp rsp = {0,};
+
+        /* Accepted */
+        rsp.op_ret = 0;
+
+        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        return 0;
+}
+
+int
+server3_3_lookup (rpcsvc_request_t *req)
+{
+        call_frame_t        *frame    = NULL;
+        server_state_t      *state    = NULL;
+        gfs3_lookup_req      args     = {{0,},};
+        int                  ret      = -1;
+        int                  op_errno = 0;
+
+        GF_VALIDATE_OR_GOTO ("server", req, err);
+
+        args.bname           = alloca (req->msg[0].iov_len);
+        args.xdata.xdata_val = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_lookup_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto err;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto err;
+        }
+        frame->root->op = GF_FOP_LOOKUP;
+
+        /* NOTE: lookup() uses req->ino only to identify if a lookup()
+         *       is requested for 'root' or not
+         */
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_DONTCARE;
+
+        if (args.bname && strcmp (args.bname, "")) {
+                memcpy (state->resolve.pargfid, args.pargfid, 16);
+                state->resolve.bname = gf_strdup (args.bname);
+        } else {
+                memcpy (state->resolve.gfid, args.gfid, 16);
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_lookup_resume);
+
+        return ret;
+out:
+
+        server_lookup_cbk (frame, NULL, frame->this, -1, EINVAL, NULL, NULL,
+                           NULL, NULL);
+        ret = 0;
+err:
+        return ret;
+}
+
+int
+server3_3_statfs (rpcsvc_request_t *req)
+{
+        server_state_t      *state = NULL;
+        call_frame_t        *frame = NULL;
+        gfs3_statfs_req      args  = {{0,},};
+        int                  ret   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_statfs_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_STATFS;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_statfs_resume);
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+rpcsvc_actor_t glusterfs3_3_fop_actors[] = {
+        [GFS3_OP_NULL]         = {"NULL",         GFS3_OP_NULL,         server_null,            NULL, 0, DRC_NA},
+        [GFS3_OP_STAT]         = {"STAT",         GFS3_OP_STAT,         server3_3_stat,         NULL, 0, DRC_NA},
+        [GFS3_OP_READLINK]     = {"READLINK",     GFS3_OP_READLINK,     server3_3_readlink,     NULL, 0, DRC_NA},
+        [GFS3_OP_MKNOD]        = {"MKNOD",        GFS3_OP_MKNOD,        server3_3_mknod,        NULL, 0, DRC_NA},
+        [GFS3_OP_MKDIR]        = {"MKDIR",        GFS3_OP_MKDIR,        server3_3_mkdir,        NULL, 0, DRC_NA},
+        [GFS3_OP_UNLINK]       = {"UNLINK",       GFS3_OP_UNLINK,       server3_3_unlink,       NULL, 0, DRC_NA},
+        [GFS3_OP_RMDIR]        = {"RMDIR",        GFS3_OP_RMDIR,        server3_3_rmdir,        NULL, 0, DRC_NA},
+        [GFS3_OP_SYMLINK]      = {"SYMLINK",      GFS3_OP_SYMLINK,      server3_3_symlink,      NULL, 0, DRC_NA},
+        [GFS3_OP_RENAME]       = {"RENAME",       GFS3_OP_RENAME,       server3_3_rename,       NULL, 0, DRC_NA},
+        [GFS3_OP_LINK]         = {"LINK",         GFS3_OP_LINK,         server3_3_link,         NULL, 0, DRC_NA},
+        [GFS3_OP_TRUNCATE]     = {"TRUNCATE",     GFS3_OP_TRUNCATE,     server3_3_truncate,     NULL, 0, DRC_NA},
+        [GFS3_OP_OPEN]         = {"OPEN",         GFS3_OP_OPEN,         server3_3_open,         NULL, 0, DRC_NA},
+        [GFS3_OP_READ]         = {"READ",         GFS3_OP_READ,         server3_3_readv,        NULL, 0, DRC_NA},
+        [GFS3_OP_WRITE]        = {"WRITE",        GFS3_OP_WRITE,        server3_3_writev,       server3_3_writev_vecsizer, 0, DRC_NA},
+        [GFS3_OP_STATFS]       = {"STATFS",       GFS3_OP_STATFS,       server3_3_statfs,       NULL, 0, DRC_NA},
+        [GFS3_OP_FLUSH]        = {"FLUSH",        GFS3_OP_FLUSH,        server3_3_flush,        NULL, 0, DRC_NA},
+        [GFS3_OP_FSYNC]        = {"FSYNC",        GFS3_OP_FSYNC,        server3_3_fsync,        NULL, 0, DRC_NA},
+        [GFS3_OP_SETXATTR]     = {"SETXATTR",     GFS3_OP_SETXATTR,     server3_3_setxattr,     NULL, 0, DRC_NA},
+        [GFS3_OP_GETXATTR]     = {"GETXATTR",     GFS3_OP_GETXATTR,     server3_3_getxattr,     NULL, 0, DRC_NA},
+        [GFS3_OP_REMOVEXATTR]  = {"REMOVEXATTR",  GFS3_OP_REMOVEXATTR,  server3_3_removexattr,  NULL, 0, DRC_NA},
+        [GFS3_OP_OPENDIR]      = {"OPENDIR",      GFS3_OP_OPENDIR,      server3_3_opendir,      NULL, 0, DRC_NA},
+        [GFS3_OP_FSYNCDIR]     = {"FSYNCDIR",     GFS3_OP_FSYNCDIR,     server3_3_fsyncdir,     NULL, 0, DRC_NA},
+        [GFS3_OP_ACCESS]       = {"ACCESS",       GFS3_OP_ACCESS,       server3_3_access,       NULL, 0, DRC_NA},
+        [GFS3_OP_CREATE]       = {"CREATE",       GFS3_OP_CREATE,       server3_3_create,       NULL, 0, DRC_NA},
+        [GFS3_OP_FTRUNCATE]    = {"FTRUNCATE",    GFS3_OP_FTRUNCATE,    server3_3_ftruncate,    NULL, 0, DRC_NA},
+        [GFS3_OP_FSTAT]        = {"FSTAT",        GFS3_OP_FSTAT,        server3_3_fstat,        NULL, 0, DRC_NA},
+        [GFS3_OP_LK]           = {"LK",           GFS3_OP_LK,           server3_3_lk,           NULL, 0, DRC_NA},
+        [GFS3_OP_LOOKUP]       = {"LOOKUP",       GFS3_OP_LOOKUP,       server3_3_lookup,       NULL, 0, DRC_NA},
+        [GFS3_OP_READDIR]      = {"READDIR",      GFS3_OP_READDIR,      server3_3_readdir,      NULL, 0, DRC_NA},
+        [GFS3_OP_INODELK]      = {"INODELK",      GFS3_OP_INODELK,      server3_3_inodelk,      NULL, 0, DRC_NA},
+        [GFS3_OP_FINODELK]     = {"FINODELK",     GFS3_OP_FINODELK,     server3_3_finodelk,     NULL, 0, DRC_NA},
+        [GFS3_OP_ENTRYLK]      = {"ENTRYLK",      GFS3_OP_ENTRYLK,      server3_3_entrylk,      NULL, 0, DRC_NA},
+        [GFS3_OP_FENTRYLK]     = {"FENTRYLK",     GFS3_OP_FENTRYLK,     server3_3_fentrylk,     NULL, 0, DRC_NA},
+        [GFS3_OP_XATTROP]      = {"XATTROP",      GFS3_OP_XATTROP,      server3_3_xattrop,      NULL, 0, DRC_NA},
+        [GFS3_OP_FXATTROP]     = {"FXATTROP",     GFS3_OP_FXATTROP,     server3_3_fxattrop,     NULL, 0, DRC_NA},
+        [GFS3_OP_FGETXATTR]    = {"FGETXATTR",    GFS3_OP_FGETXATTR,    server3_3_fgetxattr,    NULL, 0, DRC_NA},
+        [GFS3_OP_FSETXATTR]    = {"FSETXATTR",    GFS3_OP_FSETXATTR,    server3_3_fsetxattr,    NULL, 0, DRC_NA},
+        [GFS3_OP_RCHECKSUM]    = {"RCHECKSUM",    GFS3_OP_RCHECKSUM,    server3_3_rchecksum,    NULL, 0, DRC_NA},
+        [GFS3_OP_SETATTR]      = {"SETATTR",      GFS3_OP_SETATTR,      server3_3_setattr,      NULL, 0, DRC_NA},
+        [GFS3_OP_FSETATTR]     = {"FSETATTR",     GFS3_OP_FSETATTR,     server3_3_fsetattr,     NULL, 0, DRC_NA},
+        [GFS3_OP_READDIRP]     = {"READDIRP",     GFS3_OP_READDIRP,     server3_3_readdirp,     NULL, 0, DRC_NA},
+        [GFS3_OP_RELEASE]      = {"RELEASE",      GFS3_OP_RELEASE,      server3_3_release,      NULL, 0, DRC_NA},
+        [GFS3_OP_RELEASEDIR]   = {"RELEASEDIR",   GFS3_OP_RELEASEDIR,   server3_3_releasedir,   NULL, 0, DRC_NA},
+        [GFS3_OP_FREMOVEXATTR] = {"FREMOVEXATTR", GFS3_OP_FREMOVEXATTR, server3_3_fremovexattr, NULL, 0, DRC_NA},
+        [GFS3_OP_FALLOCATE]    = {"FALLOCATE",    GFS3_OP_FALLOCATE,    server3_3_fallocate,    NULL, 0, DRC_NA},
+        [GFS3_OP_DISCARD]      = {"DISCARD",      GFS3_OP_DISCARD,      server3_3_discard,      NULL, 0, DRC_NA},
+        [GFS3_OP_ZEROFILL]    =  {"ZEROFILL",     GFS3_OP_ZEROFILL,     server3_3_zerofill,     NULL, 0, DRC_NA},
+};
+
+
+struct rpcsvc_program glusterfs3_3_fop_prog = {
+        .progname  = "GlusterFS 3.3",
+        .prognum   = GLUSTER_FOP_PROGRAM,
+        .progver   = GLUSTER_FOP_VERSION,
+        .numactors = GLUSTER_FOP_PROCCNT,
+        .actors    = glusterfs3_3_fop_actors,
+};
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index b0697bb7b..589bd7b36 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -34,7 +25,60 @@
 #include "statedump.h"
 #include "defaults.h"
 #include "authenticate.h"
-#include "rpcsvc.h"
+
+void
+grace_time_handler (void *data)
+{
+        client_t      *client    = NULL;
+        xlator_t      *this      = NULL;
+        gf_timer_t    *timer     = NULL;
+        server_ctx_t  *serv_ctx  = NULL;
+        gf_boolean_t   cancelled = _gf_false;
+        gf_boolean_t   detached  = _gf_false;
+
+        client = data;
+        this = client->this;
+
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+
+        gf_log (this->name, GF_LOG_INFO, "grace timer expired for %s",
+                client->client_uid);
+
+        serv_ctx = server_ctx_get (client, this);
+
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
+        }
+
+        LOCK (&serv_ctx->fdtable_lock);
+        {
+                if (serv_ctx->grace_timer) {
+                        timer = serv_ctx->grace_timer;
+                        serv_ctx->grace_timer = NULL;
+                }
+        }
+        UNLOCK (&serv_ctx->fdtable_lock);
+        if (timer) {
+                gf_timer_call_cancel (this->ctx, timer);
+                cancelled = _gf_true;
+        }
+        if (cancelled) {
+
+                /*
+                 * client must not be destroyed in gf_client_put(),
+                 * so take a ref.
+                 */
+                gf_client_ref (client);
+                gf_client_put (client, &detached);
+                if (detached)//reconnection did not happen :-(
+                        server_connection_cleanup (this, client,
+                                                   INTERNAL_LOCKS | POSIX_LOCKS);
+                gf_client_unref (client);
+        }
+out:
+        return;
+}
 
 struct iobuf *
 gfs_serialize_reply (rpcsvc_request_t *req, void *arg, struct iovec *outmsg,
@@ -86,8 +130,6 @@ ret:
         return iob;
 }
 
-
-
 int
 server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
                      struct iovec *payload, int payloadcount,
@@ -98,16 +140,25 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
         struct iovec            rsp        = {0,};
         server_state_t         *state      = NULL;
         char                    new_iobref = 0;
-        server_connection_t    *conn  = NULL;
+        client_t               *client     = NULL;
+        gf_boolean_t            lk_heal    = _gf_false;
+        server_conf_t          *conf       = NULL;
+        gf_barrier_t           *barrier    = NULL;
+        gf_barrier_payload_t   *stub       = NULL;
+        gf_boolean_t            barriered  = _gf_false;
 
         GF_VALIDATE_OR_GOTO ("server", req, ret);
 
         if (frame) {
                 state = CALL_STATE (frame);
                 frame->local = NULL;
-                conn  = SERVER_CONNECTION(frame);
+                client = frame->root->client;
+                conf = (server_conf_t *) client->this->private;
         }
 
+        if (client)
+                lk_heal = ((server_conf_t *) client->this->private)->lk_heal;
+
         if (!iobref) {
                 iobref = iobref_new ();
                 if (!iobref) {
@@ -125,6 +176,32 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
 
         iobref_add (iobref, iob);
 
+        if (conf)
+                barrier = conf->barrier;
+        if (barrier) {
+                /* todo: write's with fd flags set to O_SYNC and O_DIRECT */
+                LOCK (&barrier->lock);
+                {
+                        if (is_fop_barriered (barrier->fops, req->procnum) &&
+                            (barrier_add_to_queue (barrier))) {
+                                stub = gf_barrier_payload (req, &rsp, frame,
+                                                           payload,
+                                                           payloadcount, iobref,
+                                                           iob, new_iobref);
+                                if (stub) {
+                                        gf_barrier_enqueue (barrier, stub);
+                                        barriered = _gf_true;
+                                } else {
+                                        gf_log ("", GF_LOG_ERROR, "Failed to "
+                                                " barrier fop %"PRIu64,
+                                                ((uint64_t)1 << req->procnum));
+                                }
+                        }
+                }
+                UNLOCK (&barrier->lock);
+                if (barriered == _gf_true)
+                        goto out;
+        }
         /* Then, submit the message for transmission. */
         ret = rpcsvc_submit_generic (req, &rsp, 1, payload, payloadcount,
                                      iobref);
@@ -140,8 +217,15 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
         iobuf_unref (iob);
         if (ret == -1) {
                 gf_log_callingfn ("", GF_LOG_ERROR, "Reply submission failed");
-                if (frame && conn)
-                        server_connection_cleanup (frame->this, conn);
+                if (frame && client && !lk_heal) {
+                        server_connection_cleanup (frame->this, client,
+                                                  INTERNAL_LOCKS | POSIX_LOCKS);
+                } else {
+                        gf_log_callingfn ("", GF_LOG_ERROR,
+                                          "Reply submission failed");
+                        /* TODO: Failure of open(dir), create, inodelk, entrylk
+                           or lk fops send failure must be handled specially. */
+                }
                 goto ret;
         }
 
@@ -152,102 +236,17 @@ ret:
         }
 
         if (frame) {
+                gf_client_unref (client);
                 STACK_DESTROY (frame->root);
         }
 
         if (new_iobref) {
                 iobref_unref (iobref);
         }
-
-        return ret;
-}
-
-/* */
-int
-server_fd_to_dict (xlator_t *this, dict_t *dict)
-{
-        server_conf_t           *conf = NULL;
-        server_connection_t     *trav = NULL;
-        char                    key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        int                     count = 0;
-        int                     ret = -1;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        conf = this->private;
-        if (!conf)
-                return -1;
-
-        ret = pthread_mutex_trylock (&conf->mutex);
-        if (ret)
-                return -1;
-
-        list_for_each_entry (trav, &conf->conns, list) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "conn%d", count++);
-                fdtable_dump_to_dict (trav->fdtable, key, dict);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        ret = dict_set_int32 (dict, "conncount", count);
 out:
         return ret;
 }
 
-int
-server_fd (xlator_t *this)
-{
-        server_conf_t        *conf = NULL;
-        server_connection_t  *trav = NULL;
-        char                 key[GF_DUMP_MAX_BUF_LEN];
-        int                  i = 1;
-        int                  ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-
-        conf = this->private;
-        if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
-                return -1;
-        }
-
-        gf_proc_dump_add_section("xlator.protocol.server.conn");
-
-        ret = pthread_mutex_trylock (&conf->mutex);
-        if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump fdtable"
-                       " errno: %d", errno);
-                return -1;
-        }
-
-        list_for_each_entry (trav, &conf->conns, list) {
-                if (trav->id) {
-                        gf_proc_dump_build_key(key,
-                                               "conn","%d.id", i);
-                        gf_proc_dump_write(key, "%s", trav->id);
-                }
-
-                gf_proc_dump_build_key(key,"conn","%d.ref",i)
-                        gf_proc_dump_write(key, "%d", trav->ref);
-                if (trav->bound_xl) {
-                        gf_proc_dump_build_key(key,
-                                               "conn","%d.bound_xl", i);
-                        gf_proc_dump_write(key, "%s", trav->bound_xl->name);
-                }
-
-                gf_proc_dump_build_key(key,
-                                       "conn","%d.id", i);
-                fdtable_dump(trav->fdtable,key);
-                i++;
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        ret = 0;
-out:
-        return ret;
-}
 
 int
 server_priv_to_dict (xlator_t *this, dict_t *dict)
@@ -267,28 +266,40 @@ server_priv_to_dict (xlator_t *this, dict_t *dict)
                 return 0;
         //TODO: Dump only specific info to dict
 
-        list_for_each_entry (xprt, &conf->xprt_list, list) {
-                peerinfo = &xprt->peerinfo;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "client%d.hostname", count);
-                ret = dict_set_str (dict, key, peerinfo->identifier);
-                if (ret)
-                        goto out;
+        pthread_mutex_lock (&conf->mutex);
+        {
+                list_for_each_entry (xprt, &conf->xprt_list, list) {
+                        peerinfo = &xprt->peerinfo;
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "client%d.hostname",
+                                  count);
+                        ret = dict_set_str (dict, key, peerinfo->identifier);
+                        if (ret)
+                                goto unlock;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "client%d.bytesread", count);
-                ret = dict_set_uint64 (dict, key, xprt->total_bytes_read);
-                if (ret)
-                        goto out;
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "client%d.bytesread",
+                                  count);
+                        ret = dict_set_uint64 (dict, key,
+                                               xprt->total_bytes_read);
+                        if (ret)
+                                goto unlock;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "client%d.byteswrite", count);
-                ret = dict_set_uint64 (dict, key, xprt->total_bytes_write);
-                if (ret)
-                        goto out;
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "client%d.byteswrite",
+                                  count);
+                        ret = dict_set_uint64 (dict, key,
+                                               xprt->total_bytes_write);
+                        if (ret)
+                                goto unlock;
 
-                count++;
+                        count++;
+                }
         }
+unlock:
+        pthread_mutex_unlock (&conf->mutex);
+        if (ret)
+                goto out;
 
         ret = dict_set_int32 (dict, "clientcount", count);
 
@@ -312,10 +323,19 @@ server_priv (xlator_t *this)
         if (!conf)
                 return 0;
 
-        list_for_each_entry (xprt, &conf->xprt_list, list) {
-                total_read  += xprt->total_bytes_read;
-                total_write += xprt->total_bytes_write;
+        gf_proc_dump_build_key (key, "xlator.protocol.server", "priv");
+        gf_proc_dump_add_section (key);
+
+        ret = pthread_mutex_trylock (&conf->mutex);
+        if (ret != 0)
+                goto out;
+        {
+                list_for_each_entry (xprt, &conf->xprt_list, list) {
+                        total_read  += xprt->total_bytes_read;
+                        total_write += xprt->total_bytes_write;
+                }
         }
+        pthread_mutex_unlock (&conf->mutex);
 
         gf_proc_dump_build_key(key, "server", "total-bytes-read");
         gf_proc_dump_write(key, "%"PRIu64, total_read);
@@ -325,103 +345,16 @@ server_priv (xlator_t *this)
 
         ret = 0;
 out:
-        return ret;
-}
-
-int
-server_inode_to_dict (xlator_t *this, dict_t *dict)
-{
-        server_conf_t           *conf = NULL;
-        server_connection_t     *trav = NULL;
-        char                    key[32] = {0,};
-        int                     count = 0;
-        int                     ret = -1;
-        xlator_t                *prev_bound_xl = NULL;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        conf = this->private;
-        if (!conf)
-                return -1;
-
-        ret = pthread_mutex_trylock (&conf->mutex);
         if (ret)
-                return -1;
-
-        list_for_each_entry (trav, &conf->conns, list) {
-                if (trav->bound_xl && trav->bound_xl->itable) {
-                        /* Presently every brick contains only one
-                         * bound_xl for all connections. This will lead
-                         * to duplicating of the inode lists, if listing
-                         * is done for every connection. This simple check
-                         * prevents duplication in the present case. If
-                         * need arises the check can be improved.
-                         */
-                        if (trav->bound_xl == prev_bound_xl)
-                                continue;
-                        prev_bound_xl = trav->bound_xl;
+                gf_proc_dump_write ("Unable to print priv",
+                                    "(Lock acquisition failed) %s",
+                                    this?this->name:"server");
 
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "conn%d", count);
-                        inode_table_dump_to_dict (trav->bound_xl->itable,
-                                                  key, dict);
-                        count++;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        ret = dict_set_int32 (dict, "conncount", count);
-
-out:
-        if (prev_bound_xl)
-                prev_bound_xl = NULL;
         return ret;
 }
 
-int
-server_inode (xlator_t *this)
-{
-        server_conf_t        *conf = NULL;
-        server_connection_t  *trav = NULL;
-        char                 key[GF_DUMP_MAX_BUF_LEN];
-        int                  i = 1;
-        int                  ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-
-        conf = this->private;
-        if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
-                return -1;
-        }
-
-        ret = pthread_mutex_trylock (&conf->mutex);
-        if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump itable"
-                       " errno: %d", errno);
-                return -1;
-        }
-
-        list_for_each_entry (trav, &conf->conns, list) {
-                if (trav->bound_xl && trav->bound_xl->itable) {
-                        gf_proc_dump_build_key(key,
-                                               "conn","%d.bound_xl.%s",
-                                               i, trav->bound_xl->name);
-                        inode_table_dump(trav->bound_xl->itable,key);
-                        i++;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
 
-        ret = 0;
-out:
-        return ret;
-}
-
-
-static void
+static int
 get_auth_types (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t   *auth_dict = NULL;
@@ -458,82 +391,88 @@ get_auth_types (dict_t *this, char *key, data_t *value, void *data)
 
         GF_FREE (key_cpy);
 out:
-        return;
+        return 0;
 }
 
+int
+_check_for_auth_option (dict_t *d, char *k, data_t *v,
+                        void *tmp)
+{
+        int       ret           = 0;
+        xlator_t *xl            = NULL;
+        char     *tail          = NULL;
+        char     *tmp_addr_list = NULL;
+        char     *addr          = NULL;
+        char     *tmp_str       = NULL;
+
+        xl = tmp;
+
+        tail = strtail (k, "auth.");
+        if (!tail)
+                goto out;
+
+        /* fast fwd thru module type */
+        tail = strchr (tail, '.');
+        if (!tail)
+                goto out;
+        tail++;
+
+        tail = strtail (tail, xl->name);
+        if (!tail)
+                goto out;
+
+        if (*tail == '.') {
+                /* when we are here, the key is checked for
+                 * valid auth.allow.<xlator>
+                 * Now we verify the ip address
+                 */
+                if (!strcmp (v->data, "*")) {
+                        ret = 0;
+                        goto out;
+                }
+
+                tmp_addr_list = gf_strdup (v->data);
+                addr = strtok_r (tmp_addr_list, ",", &tmp_str);
+                if (!addr)
+                        addr = v->data;
+
+                while (addr) {
+                        if (valid_internet_address (addr, _gf_true)) {
+                                ret = 0;
+                        } else {
+                                ret = -1;
+                                gf_log (xl->name, GF_LOG_ERROR,
+                                        "internet address '%s'"
+                                        " does not conform to"
+                                        " standards.", addr);
+                                goto out;
+                        }
+                        if (tmp_str)
+                                addr = strtok_r (NULL, ",", &tmp_str);
+                        else
+                                addr = NULL;
+                }
+
+                GF_FREE (tmp_addr_list);
+                tmp_addr_list = NULL;
+        }
+out:
+        return ret;
+}
 
 int
 validate_auth_options (xlator_t *this, dict_t *dict)
 {
         int            error = -1;
         xlator_list_t *trav = NULL;
-        data_pair_t   *pair = NULL;
-        char          *tail = NULL;
-        char          *addr = NULL;
-        char          *tmp_str = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", this, out);
         GF_VALIDATE_OR_GOTO ("server", dict, out);
 
         trav = this->children;
         while (trav) {
-                error = -1;
-                for (pair = dict->members_list; pair; pair = pair->next) {
-                        tail = strtail (pair->key, "auth.");
-                        if (!tail)
-                                continue;
-                        /* fast fwd thru module type */
-                        tail = strchr (tail, '.');
-                        if (!tail)
-                                continue;
-                        tail++;
-
-                        tail = strtail (tail, trav->xlator->name);
-                        if (!tail)
-                                continue;
-
-                        if (*tail == '.') {
-                                error = 0;
-
-                                /* when we are here, the key is checked for
-                                 * valid auth.allow.<xlator>
-                                 * Now we verify the ip address
-                                 */
-                                if (!strcmp (pair->value->data, "*")) {
-                                     error = 0;
-                                     goto out;
-                                }
-
-                                addr = strtok_r (pair->value->data, ",",
-                                                &tmp_str);
-                                if (!addr)
-                                        addr = pair->value->data;
-
-                                while (addr) {
-
-                                        if (valid_internet_address (addr) ||
-                                        valid_wildcard_internet_address (addr))
-                                        {
-                                                error = 0;
-                                        } else {
-                                                error = -1;
-                                                gf_log (this->name, GF_LOG_ERROR,
-                                                        "internet address '%s'"
-                                                        " does not conform to"
-                                                        " standards.", addr);
-                                                goto out;
-
-                                        }
-                                        if (tmp_str)
-                                                addr = strtok_r (NULL, ",",
-                                                                 &tmp_str);
-                                        else
-                                                addr = NULL;
-                                }
-
-                        }
-
-                }
+                error = dict_foreach (dict, _check_for_auth_option,
+                                      trav->xlator);
 
                 if (-1 == error) {
                         gf_log (this->name, GF_LOG_ERROR,
@@ -554,11 +493,12 @@ int
 server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
                    void *data)
 {
-        xlator_t            *this = NULL;
-        rpc_transport_t     *xprt = NULL;
-        server_connection_t *conn = NULL;
-        server_conf_t       *conf = NULL;
-
+        gf_boolean_t         detached   = _gf_false;
+        xlator_t            *this       = NULL;
+        rpc_transport_t     *trans      = NULL;
+        server_conf_t       *conf       = NULL;
+        client_t            *client     = NULL;
+        server_ctx_t        *serv_ctx   = NULL;
 
         if (!xl || !data) {
                 gf_log_callingfn ("server", GF_LOG_WARNING,
@@ -567,7 +507,7 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
         }
 
         this = xl;
-        xprt = data;
+        trans = data;
         conf = this->private;
 
         switch (event) {
@@ -575,34 +515,85 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
         {
                 /* Have a structure per new connection */
                 /* TODO: Should we create anything here at all ? * /
-                   conn = create_server_conn_state (this, xprt);
-                   if (!conn)
+                   client->conn = create_server_conn_state (this, trans);
+                   if (!client->conn)
                    goto out;
 
-                   xprt->protocol_private = conn;
+                   trans->protocol_private = client->conn;
                 */
-                INIT_LIST_HEAD (&xprt->list);
+                INIT_LIST_HEAD (&trans->list);
 
-                list_add_tail (&xprt->list, &conf->xprt_list);
+                pthread_mutex_lock (&conf->mutex);
+                {
+                        list_add_tail (&trans->list, &conf->xprt_list);
+                }
+                pthread_mutex_unlock (&conf->mutex);
 
                 break;
         }
         case RPCSVC_EVENT_DISCONNECT:
-                conn = get_server_conn_state (this, xprt);
-                if (conn)
-                        server_connection_cleanup (this, conn);
+                /* transport has to be removed from the list upon disconnect
+                 * irrespective of whether lock self heal is off or on, since
+                 * new transport will be created upon reconnect.
+                 */
+                pthread_mutex_lock (&conf->mutex);
+                {
+                        list_del_init (&trans->list);
+                }
+                pthread_mutex_unlock (&conf->mutex);
+
+                client = trans->xl_private;
+                if (!client)
+                        break;
+
+                gf_log (this->name, GF_LOG_INFO, "disconnecting connection"
+                        "from %s", client->client_uid);
+
+                /* If lock self heal is off, then destroy the
+                   conn object, else register a grace timer event */
+                if (!conf->lk_heal) {
+                        gf_client_ref (client);
+                        gf_client_put (client, &detached);
+                        if (detached)
+                                server_connection_cleanup (this, client,
+                                                           INTERNAL_LOCKS | POSIX_LOCKS);
+                        gf_client_unref (client);
+                        break;
+                }
+                trans->xl_private = NULL;
+                server_connection_cleanup (this, client, INTERNAL_LOCKS);
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "disconnected connection from %s",
-                        xprt->peerinfo.identifier);
+                serv_ctx = server_ctx_get (client, this);
 
-                list_del (&xprt->list);
+                if (serv_ctx == NULL) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "server_ctx_get() failed");
+                        goto out;
+                }
 
+                LOCK (&serv_ctx->fdtable_lock);
+                {
+                        if (!serv_ctx->grace_timer) {
+
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "starting a grace timer for %s",
+                                        client->client_uid);
+
+                                serv_ctx->grace_timer =
+                                        gf_timer_call_after (this->ctx,
+                                                             conf->grace_ts,
+                                                             grace_time_handler,
+                                                             client);
+                        }
+                }
+                UNLOCK (&serv_ctx->fdtable_lock);
                 break;
         case RPCSVC_EVENT_TRANSPORT_DESTROY:
-                conn = get_server_conn_state (this, xprt);
-                if (conn)
-                        server_connection_put (this, conn);
+                /*- conn obj has been disassociated from trans on first
+                 *  disconnect.
+                 *  conn cleanup and destruction is handed over to
+                 *  grace_time_handler or the subsequent handler that 'owns'
+                 *  the conn. Nothing left to be done here. */
                 break;
         default:
                 break;
@@ -631,43 +622,82 @@ out:
 }
 
 
-static void
-_delete_auth_opt (dict_t *this,
-                char *key,
-                data_t *value,
-                void *data)
+static int
+_delete_auth_opt (dict_t *this, char *key, data_t *value, void *data)
 {
         char *auth_option_pattern[] = { "auth.addr.*.allow",
-                                        "auth.addr.*.reject"};
-        if (fnmatch ( auth_option_pattern[0], key, 0) != 0) {
-                dict_del (this, key);
-                return;
-        }
+                                        "auth.addr.*.reject",
+                                        NULL};
+        int i = 0;
 
-        if (fnmatch ( auth_option_pattern[1], key, 0) != 0) {
-                dict_del (this, key);
-                return;
+        for (i = 0; auth_option_pattern[i]; i++) {
+                if (fnmatch (auth_option_pattern[i], key, 0) == 0) {
+                        dict_del (this, key);
+                        break;
+                }
         }
+
+        return 0;
 }
 
 
-static void
-_copy_auth_opt (dict_t *unused,
-                char *key,
-                data_t *value,
-                void *xl_dict)
+static int
+_copy_auth_opt (dict_t *unused, char *key, data_t *value, void *xl_dict)
 {
         char *auth_option_pattern[] = { "auth.addr.*.allow",
-                                        "auth.addr.*.reject"};
-        if (fnmatch ( auth_option_pattern[0], key, 0) != 0)
-                dict_set ((dict_t *)xl_dict, key, (value));
+                                        "auth.addr.*.reject",
+                                        NULL};
+        int i = 0;
+
+        for (i = 0; auth_option_pattern [i]; i++) {
+                if (fnmatch (auth_option_pattern[i], key, 0) == 0) {
+                        dict_set ((dict_t *)xl_dict, key, value);
+                        break;
+                }
+        }
 
-        if (fnmatch ( auth_option_pattern[1], key, 0) != 0)
-                dict_set ((dict_t *)xl_dict, key, (value));
+        return 0;
 }
 
 
 int
+server_init_grace_timer (xlator_t *this, dict_t *options,
+                         server_conf_t *conf)
+{
+        int32_t   ret            = -1;
+        int32_t   grace_timeout  = -1;
+        char     *lk_heal        = NULL;
+
+        GF_VALIDATE_OR_GOTO ("server", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, options, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        conf->lk_heal = _gf_false;
+
+        ret = dict_get_str (options, "lk-heal", &lk_heal);
+        if (!ret)
+                gf_string2boolean (lk_heal, &conf->lk_heal);
+
+        gf_log (this->name, GF_LOG_DEBUG, "lk-heal = %s",
+                (conf->lk_heal) ? "on" : "off");
+
+        ret = dict_get_int32 (options, "grace-timeout", &grace_timeout);
+        if (!ret)
+                conf->grace_ts.tv_sec = grace_timeout;
+        else
+                conf->grace_ts.tv_sec = 10;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Server grace timeout "
+                "value = %"PRIu64, conf->grace_ts.tv_sec);
+
+        conf->grace_ts.tv_nsec  = 0;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
 reconfigure (xlator_t *this, dict_t *options)
 {
 
@@ -698,22 +728,15 @@ reconfigure (xlator_t *this, dict_t *options)
                         gf_log (this->name, GF_LOG_WARNING,
                                 "'trace' takes on only boolean values. "
                                 "Neglecting option");
-			ret = -1;
+                        ret = -1;
                         goto out;
-		}
-		conf->trace = trace;
-		gf_log (this->name, GF_LOG_TRACE, "Reconfigured trace"
-			" to %d", conf->trace);
+                }
+                conf->trace = trace;
+                gf_log (this->name, GF_LOG_TRACE, "Reconfigured trace"
+                        " to %d", conf->trace);
 
-	}
+        }
 
-        /*ret = dict_get_str (options, "statedump-path", &statedump_path);
-        if (!ret) {
-                gf_path_strip_trailing_slashes (statedump_path);
-                if (this->ctx->statedump_path)
-                        GF_FREE (this->ctx->statedump_path);
-                this->ctx->statedump_path = gf_strdup (statedump_path);
-        }*/
         GF_OPTION_RECONF ("statedump-path", statedump_path,
                           options, path, out);
         if (!statedump_path) {
@@ -723,8 +746,7 @@ reconfigure (xlator_t *this, dict_t *options)
                 goto out;
         }
         gf_path_strip_trailing_slashes (statedump_path);
-        if (this->ctx->statedump_path)
-                GF_FREE (this->ctx->statedump_path);
+        GF_FREE (this->ctx->statedump_path);
         this->ctx->statedump_path = gf_strdup (statedump_path);
 
         if (!conf->auth_modules)
@@ -752,6 +774,8 @@ reconfigure (xlator_t *this, dict_t *options)
         }
 
         (void) rpcsvc_set_allow_insecure (rpc_conf, options);
+        (void) rpcsvc_set_root_squash (rpc_conf, options);
+        (void) rpcsvc_set_outstanding_rpc_limit (rpc_conf, options);
         list_for_each_entry (listeners, &(rpc_conf->listeners), list) {
                 if (listeners->trans != NULL) {
                         if (listeners->trans->reconfigure )
@@ -761,12 +785,33 @@ reconfigure (xlator_t *this, dict_t *options)
                                         "Reconfigure not found for transport" );
                 }
         }
+        ret = server_init_grace_timer (this, options, conf);
 
 out:
         gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
+static int32_t
+client_destroy_cbk (xlator_t *this, client_t *client)
+{
+        void         *tmp = NULL;
+        server_ctx_t *ctx = NULL;
+
+        client_ctx_del (client, this, &tmp);
+ 
+        ctx = tmp;
+
+        if (ctx == NULL)
+                return 0;
+
+        gf_fd_fdtable_destroy (ctx->fdtable);
+        LOCK_DESTROY (&ctx->fdtable_lock);
+        GF_FREE (ctx);
+
+        return 0;
+}
+
 int
 init (xlator_t *this)
 {
@@ -774,6 +819,8 @@ init (xlator_t *this)
         server_conf_t     *conf     = NULL;
         rpcsvc_listener_t *listener = NULL;
         char              *statedump_path = NULL;
+        gf_barrier_t      *barrier  = NULL;
+        char              *str      = NULL;
         GF_VALIDATE_OR_GOTO ("init", this, out);
 
         if (this->children == NULL) {
@@ -793,10 +840,13 @@ init (xlator_t *this)
 
         GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        INIT_LIST_HEAD (&conf->conns);
         INIT_LIST_HEAD (&conf->xprt_list);
         pthread_mutex_init (&conf->mutex, NULL);
 
+        ret = server_init_grace_timer (this, this->options, conf);
+        if (ret)
+                goto out;
+
         ret = server_build_config (this, conf);
         if (ret)
                 goto out;
@@ -839,8 +889,7 @@ init (xlator_t *this)
         }
 
         /* RPC related */
-        //conf->rpc = rpc_svc_init (&conf->rpc_conf);
-        conf->rpc = rpcsvc_init (this, this->ctx, this->options);
+        conf->rpc = rpcsvc_init (this, this->ctx, this->options, 0);
         if (conf->rpc == NULL) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "creation of rpcsvc failed");
@@ -864,14 +913,14 @@ init (xlator_t *this)
                 goto out;
         }
 
-        glusterfs3_1_fop_prog.options = this->options;
-        ret = rpcsvc_program_register (conf->rpc, &glusterfs3_1_fop_prog);
+        glusterfs3_3_fop_prog.options = this->options;
+        ret = rpcsvc_program_register (conf->rpc, &glusterfs3_3_fop_prog);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "registration of program (name:%s, prognum:%d, "
-                        "progver:%d) failed", glusterfs3_1_fop_prog.progname,
-                        glusterfs3_1_fop_prog.prognum,
-                        glusterfs3_1_fop_prog.progver);
+                        "progver:%d) failed", glusterfs3_3_fop_prog.progname,
+                        glusterfs3_3_fop_prog.prognum,
+                        glusterfs3_3_fop_prog.progver);
                 goto out;
         }
 
@@ -883,7 +932,7 @@ init (xlator_t *this)
                         "progver:%d) failed", gluster_handshake_prog.progname,
                         gluster_handshake_prog.prognum,
                         gluster_handshake_prog.progver);
-                rpcsvc_program_unregister (conf->rpc, &glusterfs3_1_fop_prog);
+                rpcsvc_program_unregister (conf->rpc, &glusterfs3_3_fop_prog);
                 goto out;
         }
 
@@ -912,6 +961,37 @@ init (xlator_t *this)
                 }
         }
 #endif
+        /* barrier related */
+        barrier = GF_CALLOC (1, sizeof (*barrier),1);
+        if (!barrier) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "WARNING: Failed to allocate barrier");
+                ret = -1;
+                goto out;
+        }
+
+        LOCK_INIT (&barrier->lock);
+        INIT_LIST_HEAD (&barrier->queue);
+        barrier->on = _gf_false;
+
+        GF_OPTION_INIT ("barrier-queue-length", barrier->max_size,
+                        int64, out);
+        GF_OPTION_INIT ("barrier-timeout", barrier->time_out,
+                        uint64, out);
+
+        ret = dict_get_str (this->options, "barrier-fops", &str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "setting barrier fops to default value");
+        }
+        ret = gf_barrier_fops_configure (this, barrier, str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "invalid barrier fops specified");
+                goto out;
+        }
+
+        conf->barrier = barrier;
         this->private = conf;
 
         ret = 0;
@@ -965,29 +1045,65 @@ int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
         int          ret = 0;
+        int32_t      val = 0;
+        dict_t      *dict = NULL;
+        dict_t      *output = NULL;
+        va_list      ap;
+
+        dict = data;
+        va_start (ap, data);
+        output = va_arg (ap, dict_t*);
+        va_end (ap);
+
         switch (event) {
+        case GF_EVENT_VOLUME_BARRIER_OP:
+               ret = dict_get_int32 (dict, "barrier", &val);
+               if (ret) {
+                       gf_log (this->name, GF_LOG_ERROR,
+                               "Wrong BARRIER event");
+                       goto out;
+               }
+               /* !val un-barrier, if val, barrier */
+               if (val) {
+                       ret = gf_barrier_start (this);
+                       if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Barrier start failed");
+               } else {
+                       ret = gf_barrier_stop (this);
+                       if (ret)
+                               gf_log (this->name, GF_LOG_ERROR,
+                                       "Barrier stop failed");
+               }
+               ret = dict_set_int32 (output, "barrier-status", ret);
+               if (ret)
+                       gf_log (this->name, GF_LOG_ERROR,
+                               "Failed to set barrier-status in dict");
+               break;
+
+               /* todo: call default_notify to make other xlators handle it.*/
         default:
                 default_notify (this, event, data);
                 break;
         }
-
+out:
         return ret;
 }
 
 
-struct xlator_fops fops = {
-};
+struct xlator_fops fops;
 
 struct xlator_cbks cbks = {
+        .client_destroy = client_destroy_cbk,
 };
 
 struct xlator_dumpops dumpops = {
         .priv           = server_priv,
-        .fd             = server_fd,
-        .inode          = server_inode,
+        .fd             = gf_client_dump_fdtables,
+        .inode          = gf_client_dump_inodes,
         .priv_to_dict   = server_priv_to_dict,
-        .fd_to_dict     = server_fd_to_dict,
-        .inode_to_dict  = server_inode_to_dict,
+        .fd_to_dict     = gf_client_dump_fdtables_to_dict,
+        .inode_to_dict  = gf_client_dump_inodes_to_dict,
 };
 
 
@@ -1013,7 +1129,10 @@ struct volume_options options[] = {
         { .key   = {"inode-lru-limit"},
           .type  = GF_OPTION_TYPE_INT,
           .min   = 0,
-          .max   = (1 * GF_UNIT_MB)
+          .max   = (1 * GF_UNIT_MB),
+          .default_value = "16384",
+          .description = "Specifies the maximum megabytes of memory to be "
+          "used in the inode cache."
         },
         { .key   = {"verify-volfile-checksum"},
           .type  = GF_OPTION_TYPE_BOOL
@@ -1028,9 +1147,68 @@ struct volume_options options[] = {
         { .key   = {"rpc-auth-allow-insecure"},
           .type  = GF_OPTION_TYPE_BOOL,
         },
+        { .key   = {"root-squash"},
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Map  requests  from  uid/gid 0 to the anonymous "
+                         "uid/gid. Note that this does not apply to any other"
+                         "uids or gids that might be equally sensitive, such as"
+                         "user bin or group staff."
+        },
         { .key           = {"statedump-path"},
           .type          = GF_OPTION_TYPE_PATH,
-          .default_value = "/tmp"
+          .default_value = DEFAULT_VAR_RUN_DIRECTORY,
+          .description = "Specifies directory in which gluster should save its"
+                         " statedumps. By default it is the /tmp directory"
+        },
+        { .key   = {"lk-heal"},
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+        {.key  = {"grace-timeout"},
+         .type = GF_OPTION_TYPE_INT,
+         .min  = 10,
+         .max  = 1800,
+        },
+        {.key  = {"tcp-window-size"},
+         .type = GF_OPTION_TYPE_SIZET,
+         .min  = GF_MIN_SOCKET_WINDOW_SIZE,
+         .max  = GF_MAX_SOCKET_WINDOW_SIZE,
+         .description = "Specifies the window size for tcp socket."
+        },
+
+        /*  The following two options are defined in addr.c, redifined here *
+         * for the sake of validation during volume set from cli            */
+
+        { .key   = {"auth.addr.*.allow"},
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .description = "Allow a comma separated list of addresses and/or "
+                         "hostnames to connect to the server. By default, all"
+                         " connections are allowed."
+        },
+        { .key   = {"auth.addr.*.reject"},
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .description = "Reject a comma separated list of addresses and/or "
+                         "hostnames to connect to the server. By default, all"
+                         " connections are allowed."
+        },
+        {.key  = {"barrier-timeout"},
+         .type = GF_OPTION_TYPE_INT,
+         .default_value = "60",
+         .min  = 0,
+         .max  = 360,
+         .description = "Barrier timeout in seconds",
+        },
+        {.key  = {"barrier-queue-length"},
+         .type = GF_OPTION_TYPE_INT,
+         .default_value = "4096",
+         .min  = 0,
+         .max  = 16384,
+         .description = "Barrier queue length",
+        },
+        {.key = {"barrier-fops"},
+         .type = GF_OPTION_TYPE_STR,
+         .description = "Allow a comma seperated fop lists",
         },
         { .key   = {NULL} },
 };
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index 92785c5a9..782327d77 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _SERVER_H
@@ -22,59 +13,55 @@
 
 #include <pthread.h>
 
+#include "fd.h"
 #include "rpcsvc.h"
 
 #include "fd.h"
 #include "protocol-common.h"
 #include "server-mem-types.h"
 #include "glusterfs3.h"
+#include "timer.h"
+#include "client_t.h"
 
 #define DEFAULT_BLOCK_SIZE         4194304   /* 4MB */
 #define DEFAULT_VOLUME_FILE_PATH   CONFDIR "/glusterfs.vol"
-
-typedef struct _server_state server_state_t;
-
-struct _locker {
-        struct list_head  lockers;
-        char             *volume;
-        loc_t             loc;
-        fd_t             *fd;
-        gf_lkowner_t      owner;
-        pid_t             pid;
+#define GF_MAX_SOCKET_WINDOW_SIZE  (1 * GF_UNIT_MB)
+#define GF_MIN_SOCKET_WINDOW_SIZE  (0)
+
+struct _gf_barrier_payload {
+        rpcsvc_request_t        *req;
+        struct iovec            rsp;
+        call_frame_t            *frame;
+        struct iovec            *payload;
+        struct iobref           *iobref;
+        struct iobuf            *iob;
+        int                      payload_count;
+        gf_boolean_t             free_iobref;
+        struct list_head         list;
 };
 
-struct _lock_table {
-        struct list_head  inodelk_lockers;
-        struct list_head  entrylk_lockers;
-        gf_lock_t         lock;
-        size_t            count;
+typedef struct _gf_barrier_payload gf_barrier_payload_t;
+
+struct _gf_barrier {
+        gf_lock_t               lock;
+        gf_boolean_t            on;
+        gf_boolean_t            force;
+        size_t                  cur_size;
+        int64_t                 max_size;
+        uint64_t                fops;
+        gf_timer_t             *timer;
+        uint64_t                time_out;
+        struct list_head        queue;
 };
 
-/* private structure per connection (transport object)
- * used as transport_t->xl_private
- */
-struct _server_connection {
-        struct list_head    list;
-        char               *id;
-        int                 ref;
-        pthread_mutex_t     lock;
-        fdtable_t          *fdtable;
-        struct _lock_table *ltable;
-        xlator_t           *bound_xl;
-        xlator_t           *this;
-};
-
-typedef struct _server_connection server_connection_t;
-
+typedef struct _gf_barrier gf_barrier_t;
 
-server_connection_t *
-server_connection_get (xlator_t *this, const char *id);
-
-void
-server_connection_put (xlator_t *this, server_connection_t *conn);
+typedef enum {
+        INTERNAL_LOCKS = 1,
+        POSIX_LOCKS = 2,
+} server_lock_flags_t;
 
-int
-server_connection_cleanup (xlator_t *this, server_connection_t *conn);
+typedef struct _server_state server_state_t;
 
 int server_null (rpcsvc_request_t *req);
 
@@ -90,13 +77,16 @@ struct server_conf {
         int                     inode_lru_limit;
         gf_boolean_t            verify_volfile;
         gf_boolean_t            trace;
+        gf_boolean_t            lk_heal; /* If true means lock self
+                                            heal is on else off. */
         char                   *conf_dir;
         struct _volfile_ctx    *volfile;
-
+        struct timespec         grace_ts;
         dict_t                 *auth_modules;
         pthread_mutex_t         mutex;
-        struct list_head        conns;
+        gf_barrier_t           *barrier;
         struct list_head        xprt_list;
+        pthread_t               barrier_th;
 };
 typedef struct server_conf server_conf_t;
 
@@ -117,7 +107,7 @@ struct resolve_comp {
 
 typedef struct {
         server_resolve_type_t  type;
-        uint64_t               fd_no;
+        int64_t               fd_no;
         u_char                 gfid[16];
         u_char                 pargfid[16];
         char                  *path;
@@ -134,11 +124,10 @@ int
 resolve_and_resume (call_frame_t *frame, server_resume_fn_t fn);
 
 struct _server_state {
-        server_connection_t  *conn;
-        rpc_transport_t      *xprt;
-        inode_table_t        *itable;
+        rpc_transport_t  *xprt;
+        inode_table_t    *itable;
 
-        server_resume_fn_t    resume_fn;
+        server_resume_fn_t resume_fn;
 
         loc_t             loc;
         loc_t             loc2;
@@ -174,15 +163,28 @@ struct _server_state {
         int               mask;
         char              is_revalidate;
         dict_t           *dict;
-        struct gf_flock      flock;
+        struct gf_flock   flock;
         const char       *volume;
         dir_entry_t      *entry;
+
+        dict_t           *xdata;
+        mode_t            umask;
 };
 
+
 extern struct rpcsvc_program gluster_handshake_prog;
-extern struct rpcsvc_program glusterfs3_1_fop_prog;
+extern struct rpcsvc_program glusterfs3_3_fop_prog;
 extern struct rpcsvc_program gluster_ping_prog;
 
+
+typedef struct _server_ctx {
+        gf_lock_t            fdtable_lock;
+        fdtable_t           *fdtable;
+        struct _gf_timer    *grace_timer;
+        uint32_t             lk_version;
+} server_ctx_t;
+
+
 int
 server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
                      struct iovec *payload, int payloadcount,
diff --git a/xlators/protocol/server/src/server3_1-fops.c b/xlators/protocol/server/src/server3_1-fops.c
deleted file mode 100644
index 90f702569..000000000
--- a/xlators/protocol/server/src/server3_1-fops.c
+++ /dev/null
@@ -1,5015 +0,0 @@
-/*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "server.h"
-#include "server-helpers.h"
-#include "glusterfs3-xdr.h"
-#include "glusterfs3.h"
-#include "compat-errno.h"
-
-#include "md5.h"
-#include "xdr-nfs3.h"
-
-
-/* Callback function section */
-int
-server_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct statvfs *buf)
-{
-        gfs3_statfs_rsp   rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        if (op_ret >= 0) {
-                gf_statfs_from_statfs (&rsp.statfs, buf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": STATFS %"PRId32" (%s)",
-                        frame->root->unique, op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_statfs_rsp);
-
-        return 0;
-}
-
-int
-server_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *stbuf, dict_t *dict,
-                   struct iatt *postparent)
-{
-        rpcsvc_request_t *req        = NULL;
-        server_state_t   *state      = NULL;
-        inode_t          *root_inode = NULL;
-        inode_t          *link_inode = NULL;
-        loc_t             fresh_loc  = {0,};
-        gfs3_lookup_rsp   rsp        = {0,};
-        uuid_t            rootgfid   = {0,};
-
-        state = CALL_STATE(frame);
-
-        req = frame->local;
-
-        if (state->is_revalidate == 1 && op_ret == -1) {
-                state->is_revalidate = 2;
-                loc_copy (&fresh_loc, &state->loc);
-                inode_unref (fresh_loc.inode);
-                fresh_loc.inode = inode_new (state->itable);
-
-                STACK_WIND (frame, server_lookup_cbk, BOUND_XL (frame),
-                            BOUND_XL (frame)->fops->lookup,
-                            &fresh_loc, state->dict);
-
-                loc_wipe (&fresh_loc);
-                return 0;
-        }
-
-        if ((op_ret >= 0) && dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, dict,
-                                            (&rsp.dict.dict_val),
-                                            rsp.dict.dict_len,
-                                            op_errno, out);
-        }
-
-        gf_stat_from_iatt (&rsp.postparent, postparent);
-
-        if (op_ret == 0) {
-                root_inode = BOUND_XL(frame)->itable->root;
-                if (inode == root_inode) {
-                        /* we just looked up root ("/") */
-                        stbuf->ia_ino = 1;
-                        rootgfid[15]  = 1;
-                        uuid_copy (stbuf->ia_gfid, rootgfid);
-                        if (inode->ia_type == 0)
-                                inode->ia_type = stbuf->ia_type;
-                }
-
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-
-                if (!__is_root_gfid (inode->gfid)) {
-                        link_inode = inode_link (inode, state->loc.parent,
-                                                 state->loc.name, stbuf);
-                        inode_lookup (link_inode);
-                        inode_unref (link_inode);
-                }
-        } else {
-                if (state->is_revalidate && op_errno == ENOENT) {
-                        if (!__is_root_gfid (state->loc.inode->gfid)) {
-                                inode_unlink (state->loc.inode,
-                                              state->loc.parent,
-                                              state->loc.name);
-                        }
-                }
-        }
-out:
-        rsp.op_ret   = op_ret;
-        rsp.op_errno = gf_errno_to_error (op_errno);
-
-        if (op_ret) {
-                gf_log (this->name,
-                        ((op_errno == ENOENT) ? GF_LOG_TRACE : GF_LOG_INFO),
-                        "%"PRId64": LOOKUP %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_lookup_rsp);
-
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
-
-        return 0;
-}
-
-
-int
-server_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        gfs3_lk_rsp       rsp   = {0,};
-        rpcsvc_request_t *req   = NULL;
-        server_state_t   *state = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                switch (lock->l_type) {
-                case F_RDLCK:
-                        lock->l_type = GF_LK_F_RDLCK;
-                        break;
-                case F_WRLCK:
-                        lock->l_type = GF_LK_F_WRLCK;
-                        break;
-                case F_UNLCK:
-                        lock->l_type = GF_LK_F_UNLCK;
-                        break;
-                default:
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unknown lock type: %"PRId32"!", lock->l_type);
-                        break;
-                }
-
-                gf_proto_flock_from_flock (&rsp.flock, lock);
-        } else if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": LK %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_lk_rsp);
-
-        return 0;
-}
-
-
-int
-server_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp        rsp   = {0,};
-        server_connection_t *conn  = NULL;
-        server_state_t      *state = NULL;
-        rpcsvc_request_t    *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        conn  = SERVER_CONNECTION(frame);
-        state = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                if (state->flock.l_type == F_UNLCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, &frame->root->lk_owner,
-                                       GF_FOP_INODELK);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid,
-                                       &frame->root->lk_owner,
-                                       GF_FOP_INODELK);
-        } else if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": INODELK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-
-int
-server_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp        rsp   = {0,};
-        server_state_t      *state = NULL;
-        server_connection_t *conn  = NULL;
-        rpcsvc_request_t    *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        conn = SERVER_CONNECTION(frame);
-        state = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                if (state->flock.l_type == F_UNLCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       NULL, state->fd,
-                                       &frame->root->lk_owner,  GF_FOP_INODELK);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       NULL, state->fd,
-                                       frame->root->pid,
-                                       &frame->root->lk_owner, GF_FOP_INODELK);
-        } else if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FINODELK %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        server_connection_t *conn  = NULL;
-        server_state_t      *state = NULL;
-        rpcsvc_request_t    *req   = NULL;
-        gf_common_rsp        rsp   = {0,};
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        conn  = SERVER_CONNECTION(frame);
-        state = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                if (state->cmd == ENTRYLK_UNLOCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, &frame->root->lk_owner,
-                                       GF_FOP_ENTRYLK);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid,
-                                       &frame->root->lk_owner,
-                                       GF_FOP_ENTRYLK);
-        } else if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": ENTRYLK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-        return 0;
-}
-
-
-int
-server_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp        rsp   = {0,};
-        server_connection_t *conn  = NULL;
-        server_state_t      *state = NULL;
-        rpcsvc_request_t    *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        conn  = SERVER_CONNECTION(frame);
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                if (state->cmd == ENTRYLK_UNLOCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       NULL, state->fd, &frame->root->lk_owner,
-                                       GF_FOP_ENTRYLK);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       NULL, state->fd, frame->root->pid,
-                                       &frame->root->lk_owner, GF_FOP_ENTRYLK);
-        } else if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FENTRYLK %"PRId64" (%s) "
-                        " ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-
-int
-server_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-        server_state_t   *state = NULL;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-        if (op_ret)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": ACCESS %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gfs3_rmdir_rsp    rsp    = {0,};
-        server_state_t   *state  = NULL;
-        inode_t          *parent = NULL;
-        rpcsvc_request_t *req    = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
-                parent = inode_parent (state->loc.inode, 0, NULL);
-                if (parent)
-                        inode_unref (parent);
-                else
-                        inode_forget (state->loc.inode, 0);
-
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": RMDIR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_rmdir_rsp);
-
-        return 0;
-}
-
-int
-server_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *stbuf, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gfs3_mkdir_rsp    rsp        = {0,};
-        server_state_t   *state      = NULL;
-        inode_t          *link_inode = NULL;
-        rpcsvc_request_t *req        = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": MKDIR %s  ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_mkdir_rsp);
-
-        return 0;
-}
-
-int
-server_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gfs3_mknod_rsp    rsp        = {0,};
-        server_state_t   *state      = NULL;
-        inode_t          *link_inode = NULL;
-        rpcsvc_request_t *req        = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": MKNOD %s ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_mknod_rsp);
-
-
-        return 0;
-}
-
-int
-server_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FSYNCDIR %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        gfs3_readdir_rsp  rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-        int               ret   = 0;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-        if (op_ret > 0) {
-                ret = serialize_rsp_dirent (entries, &rsp);
-                if (ret == -1) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-        } else {
-                /* (op_ret == 0) is valid, and means EOF, don't log for that */
-                gf_log (this->name, (op_ret) ? GF_LOG_INFO : GF_LOG_TRACE,
-                        "%"PRId64": READDIR %"PRId64" (%s) ==> %"PRId32
-                        " (%s)", frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-unwind:
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_readdir_rsp);
-
-        readdir_rsp_cleanup (&rsp);
-
-        return 0;
-}
-
-
-int
-server_releasedir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        server_connection_t *conn  = NULL;
-        server_state_t      *state = NULL;
-        rpcsvc_request_t    *req   = NULL;
-        gfs3_opendir_rsp     rsp   = {0,};
-        uint64_t             fd_no = 0;
-
-        conn = SERVER_CONNECTION (frame);
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                fd_bind (fd);
-
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd); // on behalf of the client
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": OPENDIR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        req           = frame->local;
-
-        rsp.fd        = fd_no;
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_opendir_rsp);
-
-        return 0;
-}
-
-int
-server_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = frame->local;
-        state = CALL_STATE(frame);
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": REMOVEXATTR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = frame->local;
-        state = CALL_STATE(frame);
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FREMOVEXATTR (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, ((state->fd->inode) ?
-                                              uuid_utoa (state->fd->inode->gfid) :
-                                              "--"),
-                        op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gfs3_getxattr_rsp  rsp   = {0,};
-        rpcsvc_request_t  *req   = NULL;
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, dict,
-                                            (&rsp.dict.dict_val),
-                                            rsp.dict.dict_len,
-                                            op_errno, out);
-        }
-out:
-        req               = frame->local;
-
-        rsp.op_ret        = op_ret;
-        rsp.op_errno      = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": GETXATTR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->name, op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_getxattr_rsp);
-
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
-
-        return 0;
-}
-
-
-int
-server_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gfs3_fgetxattr_rsp  rsp   = {0,};
-        server_state_t     *state = NULL;
-        rpcsvc_request_t   *req   = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, dict,
-                                            (&rsp.dict.dict_val),
-                                            rsp.dict.dict_len,
-                                            op_errno, out);
-        }
-
-out:
-        req               = frame->local;
-
-        rsp.op_ret        = op_ret;
-        rsp.op_errno      = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FGETXATTR %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->name, op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_fgetxattr_rsp);
-
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
-
-        return 0;
-}
-
-int
-server_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": SETXATTR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-
-int
-server_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FSETXATTR %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                   struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent)
-{
-        gfs3_rename_rsp   rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                stbuf->ia_type = state->loc.inode->ia_type;
-
-                /* TODO: log gfid of the inodes */
-                gf_log (state->conn->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": RENAME_CBK  %s ==> %s",
-                        frame->root->unique, state->loc.name, state->loc2.name);
-
-                inode_rename (state->itable,
-                              state->loc.parent, state->loc.name,
-                              state->loc2.parent, state->loc2.name,
-                              state->loc.inode, stbuf);
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-
-                gf_stat_from_iatt (&rsp.preoldparent, preoldparent);
-                gf_stat_from_iatt (&rsp.postoldparent, postoldparent);
-
-                gf_stat_from_iatt (&rsp.prenewparent, prenewparent);
-                gf_stat_from_iatt (&rsp.postnewparent, postnewparent);
-        }
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": RENAME %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_rename_rsp);
-
-        return 0;
-}
-
-int
-server_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
-{
-        gfs3_unlink_rsp   rsp    = {0,};
-        server_state_t   *state  = NULL;
-        inode_t          *parent = NULL;
-        rpcsvc_request_t *req    = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                /* TODO: log gfid of the inodes */
-                gf_log (state->conn->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": UNLINK_CBK %s",
-                        frame->root->unique, state->loc.name);
-
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
-
-                parent = inode_parent (state->loc.inode, 0, NULL);
-                if (parent)
-                        inode_unref (parent);
-                else
-                        inode_forget (state->loc.inode, 0);
-
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": UNLINK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_unlink_rsp);
-
-        return 0;
-}
-
-int
-server_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *stbuf, struct iatt *preparent,
-                    struct iatt *postparent)
-{
-        gfs3_symlink_rsp  rsp        = {0,};
-        server_state_t   *state      = NULL;
-        inode_t          *link_inode = NULL;
-        rpcsvc_request_t *req        = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": SYMLINK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_symlink_rsp);
-
-        return 0;
-}
-
-
-int
-server_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent)
-{
-        gfs3_link_rsp     rsp        = {0,};
-        server_state_t   *state      = NULL;
-        inode_t          *link_inode = NULL;
-        rpcsvc_request_t *req        = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc2.parent,
-                                         state->loc2.name, stbuf);
-                inode_unref (link_inode);
-        }
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": LINK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_link_rsp);
-
-        return 0;
-}
-
-int
-server_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
-{
-        gfs3_truncate_rsp  rsp   = {0,};
-        server_state_t    *state = NULL;
-        rpcsvc_request_t  *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE (frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.prestat, prebuf);
-                gf_stat_from_iatt (&rsp.poststat, postbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": TRUNCATE %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_truncate_rsp);
-
-        return 0;
-}
-
-int
-server_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
-{
-        gfs3_fstat_rsp    rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FSTAT %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_fstat_rsp);
-
-        return 0;
-}
-
-int
-server_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
-{
-        gfs3_ftruncate_rsp  rsp   = {0};
-        server_state_t     *state = NULL;
-        rpcsvc_request_t   *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE (frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.prestat, prebuf);
-                gf_stat_from_iatt (&rsp.poststat, postbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FTRUNCATE %"PRId64" (%s)==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_ftruncate_rsp);
-
-        return 0;
-}
-
-int
-server_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FLUSH %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-
-        return 0;
-}
-
-int
-server_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
-{
-        gfs3_fsync_rsp    rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&(rsp.prestat), prebuf);
-                gf_stat_from_iatt (&(rsp.poststat), postbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FSYNC %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_fsync_rsp);
-
-        return 0;
-}
-
-int
-server_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-        gfs3_write_rsp    rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.prestat, prebuf);
-                gf_stat_from_iatt (&rsp.poststat, postbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": WRITEV %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_write_rsp);
-
-        return 0;
-}
-
-
-int
-server_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count,
-                  struct iatt *stbuf, struct iobref *iobref)
-{
-        gfs3_read_rsp     rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                rsp.size = op_ret;
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": READV %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, vector, count, iobref,
-                             (xdrproc_t)xdr_gfs3_read_rsp);
-
-        return 0;
-}
-
-int
-server_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum)
-{
-        gfs3_rchecksum_rsp  rsp = {0,};
-        rpcsvc_request_t   *req = NULL;
-        server_state_t      *state = NULL;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        if (op_ret >= 0) {
-                rsp.weak_checksum = weak_checksum;
-
-                rsp.strong_checksum.strong_checksum_val = (char *)strong_checksum;
-                rsp.strong_checksum.strong_checksum_len = MD5_DIGEST_LEN;
-        }
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": RCHECKSUM %"PRId64" (%s)==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_rchecksum_rsp);
-
-        return 0;
-}
-
-
-int
-server_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        server_connection_t *conn  = NULL;
-        server_state_t      *state = NULL;
-        rpcsvc_request_t    *req   = NULL;
-        uint64_t             fd_no = 0;
-        gfs3_open_rsp        rsp   = {0,};
-
-        conn = SERVER_CONNECTION (frame);
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                fd_bind (fd);
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": OPEN %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        req           = frame->local;
-
-        rsp.fd        = fd_no;
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_open_rsp);
-        return 0;
-}
-
-
-int
-server_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                   struct iatt *preparent, struct iatt *postparent)
-{
-        server_connection_t *conn       = NULL;
-        server_state_t      *state      = NULL;
-        inode_t             *link_inode = NULL;
-        rpcsvc_request_t    *req        = NULL;
-        uint64_t             fd_no      = 0;
-        gfs3_create_rsp      rsp        = {0,};
-
-        conn = SERVER_CONNECTION (frame);
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                /* TODO: log gfid too */
-                gf_log (state->conn->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": CREATE %s (%s)",
-                        frame->root->unique, state->loc.name,
-                        uuid_utoa (inode->gfid));
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-
-                if (!link_inode) {
-                        op_ret = -1;
-                        op_errno = ENOENT;
-                        goto out;
-                }
-
-                if (link_inode != inode) {
-                        /*
-                          VERY racy code (if used anywhere else)
-                          -- don't do this without understanding
-                        */
-
-                        inode_unref (fd->inode);
-                        fd->inode = inode_ref (link_inode);
-                }
-
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-
-                fd_bind (fd);
-
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd);
-
-                if ((fd_no < 0) || (fd == 0)) {
-                        op_ret = fd_no;
-                        op_errno = errno;
-                }
-
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": CREATE %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-out:
-        req           = frame->local;
-
-        rsp.fd        = fd_no;
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_create_rsp);
-
-        return 0;
-}
-
-int
-server_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, const char *buf,
-                     struct iatt *stbuf)
-{
-        gfs3_readlink_rsp  rsp   = {0,};
-        server_state_t    *state = NULL;
-        rpcsvc_request_t  *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-
-        state  = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.buf, stbuf);
-                rsp.path = (char *)buf;
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": READLINK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        if (!rsp.path)
-                rsp.path = "";
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_readlink_rsp);
-
-        return 0;
-}
-
-int
-server_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
-{
-        gfs3_stat_rsp     rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state  = CALL_STATE (frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": STAT %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_stat_rsp);
-
-        return 0;
-}
-
-
-int
-server_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *statpre, struct iatt *statpost)
-{
-        gfs3_setattr_rsp  rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state  = CALL_STATE (frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.statpre, statpre);
-                gf_stat_from_iatt (&rsp.statpost, statpost);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": SETATTR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_setattr_rsp);
-
-        return 0;
-}
-
-int
-server_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost)
-{
-        gfs3_fsetattr_rsp  rsp   = {0,};
-        server_state_t    *state = NULL;
-        rpcsvc_request_t  *req   = NULL;
-
-        state  = CALL_STATE (frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.statpre, statpre);
-                gf_stat_from_iatt (&rsp.statpost, statpost);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FSETATTR %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_fsetattr_rsp);
-
-        return 0;
-}
-
-
-int
-server_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gfs3_xattrop_rsp  rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": XATTROP %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-                goto out;
-        }
-
-        if ((op_ret >= 0) && dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, dict,
-                                            (&rsp.dict.dict_val),
-                                            rsp.dict.dict_len,
-                                            op_errno, out);
-        }
-out:
-        req               = frame->local;
-
-        rsp.op_ret        = op_ret;
-        rsp.op_errno      = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": XATTROP %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_xattrop_rsp);
-
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
-
-        return 0;
-}
-
-
-int
-server_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gfs3_xattrop_rsp  rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        state = CALL_STATE(frame);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FXATTROP %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-                goto out;
-        }
-
-        if ((op_ret >= 0) && dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, dict,
-                                            (&rsp.dict.dict_val),
-                                            rsp.dict.dict_len,
-                                            op_errno, out);
-        }
-out:
-        req               = frame->local;
-
-        rsp.op_ret        = op_ret;
-        rsp.op_errno      = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FXATTROP %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_fxattrop_rsp);
-
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
-
-        return 0;
-}
-
-
-int
-server_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        gfs3_readdirp_rsp  rsp   = {0,};
-        server_state_t    *state = NULL;
-        rpcsvc_request_t  *req   = NULL;
-        int                ret   = 0;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-        if (op_ret > 0) {
-                ret = serialize_rsp_direntp (entries, &rsp);
-                if (ret == -1) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-                /* TODO: need more clear thoughts before calling this function. */
-                /* gf_link_inodes_from_dirent (this, state->fd->inode, entries); */
-
-        } else {
-                /* (op_ret == 0) is valid, and means EOF, don't log for that */
-                gf_log (this->name, (op_ret) ? GF_LOG_INFO : GF_LOG_TRACE,
-                        "%"PRId64": READDIRP %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-out:
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_readdirp_rsp);
-
-        readdirp_rsp_cleanup (&rsp);
-
-        return 0;
-}
-
-/* Resume function section */
-
-int
-server_rchecksum_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state    = NULL;
-        int             op_ret   = 0;
-        int             op_errno = EINVAL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        STACK_WIND (frame, server_rchecksum_cbk, bound_xl,
-                    bound_xl->fops->rchecksum, state->fd,
-                    state->offset, state->size);
-
-        return 0;
-err:
-        server_rchecksum_cbk (frame, NULL, frame->this, op_ret, op_errno, 0, NULL);
-
-        return 0;
-
-}
-
-int
-server_lk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_lk_cbk, bound_xl, bound_xl->fops->lk,
-                    state->fd, state->cmd, &state->flock);
-
-        return 0;
-
-err:
-        server_lk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                       state->resolve.op_errno, NULL);
-        return 0;
-}
-
-int
-server_rename_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-        int             op_ret = 0;
-        int             op_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        if (state->resolve2.op_ret != 0) {
-                op_ret   = state->resolve2.op_ret;
-                op_errno = state->resolve2.op_errno;
-                goto err;
-        }
-
-        STACK_WIND (frame, server_rename_cbk,
-                    bound_xl, bound_xl->fops->rename,
-                    &state->loc, &state->loc2);
-        return 0;
-err:
-        server_rename_cbk (frame, NULL, frame->this, op_ret, op_errno,
-                           NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_link_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-        int             op_ret = 0;
-        int             op_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        if (state->resolve2.op_ret != 0) {
-                op_ret   = state->resolve2.op_ret;
-                op_errno = state->resolve2.op_errno;
-                goto err;
-        }
-
-        state->loc2.inode = inode_ref (state->loc.inode);
-
-        STACK_WIND (frame, server_link_cbk, bound_xl, bound_xl->fops->link,
-                    &state->loc, &state->loc2);
-
-        return 0;
-err:
-        server_link_cbk (frame, NULL, frame->this, op_ret, op_errno,
-                         NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-int
-server_symlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_symlink_cbk,
-                    bound_xl, bound_xl->fops->symlink,
-                    state->name, &state->loc, state->params);
-
-        return 0;
-err:
-        server_symlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_access_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_access_cbk,
-                    bound_xl, bound_xl->fops->access,
-                    &state->loc, state->mask);
-        return 0;
-err:
-        server_access_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno);
-        return 0;
-}
-
-int
-server_fentrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fentrylk_cbk, bound_xl,
-                    bound_xl->fops->fentrylk,
-                    state->volume, state->fd, state->name,
-                    state->cmd, state->type);
-
-        return 0;
-err:
-        server_fentrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-        return 0;
-}
-
-
-int
-server_entrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_entrylk_cbk,
-                    bound_xl, bound_xl->fops->entrylk,
-                    state->volume, &state->loc, state->name,
-                    state->cmd, state->type);
-        return 0;
-err:
-        server_entrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno);
-        return 0;
-}
-
-
-int
-server_finodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_finodelk_cbk, bound_xl,
-                    bound_xl->fops->finodelk,
-                    state->volume, state->fd, state->cmd, &state->flock);
-
-        return 0;
-err:
-        server_finodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-
-        return 0;
-}
-
-int
-server_inodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_inodelk_cbk,
-                    bound_xl, bound_xl->fops->inodelk,
-                    state->volume, &state->loc, state->cmd, &state->flock);
-        return 0;
-err:
-        server_inodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno);
-        return 0;
-}
-
-int
-server_rmdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_rmdir_cbk,
-                    bound_xl, bound_xl->fops->rmdir, &state->loc, state->flags);
-        return 0;
-err:
-        server_rmdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-int
-server_mkdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_mkdir_cbk,
-                    bound_xl, bound_xl->fops->mkdir,
-                    &(state->loc), state->mode, state->params);
-
-        return 0;
-err:
-        server_mkdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_mknod_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_mknod_cbk,
-                    bound_xl, bound_xl->fops->mknod,
-                    &(state->loc), state->mode, state->dev,
-                    state->params);
-
-        return 0;
-err:
-        server_mknod_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_fsyncdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsyncdir_cbk,
-                    bound_xl,
-                    bound_xl->fops->fsyncdir,
-                    state->fd, state->flags);
-        return 0;
-
-err:
-        server_fsyncdir_cbk (frame, NULL, frame->this,
-                             state->resolve.op_ret,
-                             state->resolve.op_errno);
-        return 0;
-}
-
-
-int
-server_readdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readdir_cbk,
-                    bound_xl,
-                    bound_xl->fops->readdir,
-                    state->fd, state->size, state->offset);
-
-        return 0;
-err:
-        server_readdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-
-int
-server_readdirp_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readdirp_cbk, bound_xl,
-                    bound_xl->fops->readdirp, state->fd, state->size,
-                    state->offset, state->dict);
-
-        return 0;
-err:
-        server_readdirp_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_opendir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-
-        STACK_WIND (frame, server_opendir_cbk,
-                    bound_xl, bound_xl->fops->opendir,
-                    &state->loc, state->fd);
-        return 0;
-err:
-        server_opendir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_statfs_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t      *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret !=0)
-                goto err;
-
-        STACK_WIND (frame, server_statfs_cbk,
-                    bound_xl, bound_xl->fops->statfs,
-                    &state->loc);
-        return 0;
-
-err:
-        server_statfs_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_removexattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_removexattr_cbk,
-                    bound_xl, bound_xl->fops->removexattr,
-                    &state->loc, state->name);
-        return 0;
-err:
-        server_removexattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                                state->resolve.op_errno);
-        return 0;
-}
-
-int
-server_fremovexattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fremovexattr_cbk,
-                    bound_xl, bound_xl->fops->fremovexattr,
-                    state->fd, state->name);
-        return 0;
-err:
-        server_fremovexattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                                 state->resolve.op_errno);
-        return 0;
-}
-
-int
-server_fgetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fgetxattr_cbk,
-                    bound_xl, bound_xl->fops->fgetxattr,
-                    state->fd, state->name);
-        return 0;
-err:
-        server_fgetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_xattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_xattrop_cbk,
-                    bound_xl, bound_xl->fops->xattrop,
-                    &state->loc, state->flags, state->dict);
-        return 0;
-err:
-        server_xattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-
-int
-server_fxattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fxattrop_cbk,
-                    bound_xl, bound_xl->fops->fxattrop,
-                    state->fd, state->flags, state->dict);
-        return 0;
-err:
-        server_fxattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-int
-server_fsetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setxattr_cbk,
-                    bound_xl, bound_xl->fops->fsetxattr,
-                    state->fd, state->dict, state->flags);
-        return 0;
-err:
-        server_fsetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno);
-
-        return 0;
-}
-
-int
-server_unlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_unlink_cbk,
-                    bound_xl, bound_xl->fops->unlink,
-                    &state->loc);
-        return 0;
-err:
-        server_unlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-int
-server_truncate_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_truncate_cbk,
-                    bound_xl, bound_xl->fops->truncate,
-                    &state->loc, state->offset);
-        return 0;
-err:
-        server_truncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-
-int
-server_fstat_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t     *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fstat_cbk,
-                    bound_xl, bound_xl->fops->fstat,
-                    state->fd);
-        return 0;
-err:
-        server_fstat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_setxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setxattr_cbk,
-                    bound_xl, bound_xl->fops->setxattr,
-                    &state->loc, state->dict, state->flags);
-        return 0;
-err:
-        server_setxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-
-        return 0;
-}
-
-
-int
-server_getxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_getxattr_cbk,
-                    bound_xl, bound_xl->fops->getxattr,
-                    &state->loc, state->name);
-        return 0;
-err:
-        server_getxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_ftruncate_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_ftruncate_cbk,
-                    bound_xl, bound_xl->fops->ftruncate,
-                    state->fd, state->offset);
-        return 0;
-err:
-        server_ftruncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-int
-server_flush_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_flush_cbk,
-                    bound_xl, bound_xl->fops->flush, state->fd);
-        return 0;
-err:
-        server_flush_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno);
-
-        return 0;
-}
-
-
-int
-server_fsync_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsync_cbk,
-                    bound_xl, bound_xl->fops->fsync,
-                    state->fd, state->flags);
-        return 0;
-err:
-        server_fsync_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-int
-server_writev_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t   *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_writev_cbk,
-                    bound_xl, bound_xl->fops->writev,
-                    state->fd, state->payload_vector, state->payload_count,
-                    state->offset, state->flags, state->iobref);
-
-        return 0;
-err:
-        server_writev_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_readv_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readv_cbk,
-                    bound_xl, bound_xl->fops->readv,
-                    state->fd, state->size, state->offset, state->flags);
-
-        return 0;
-err:
-        server_readv_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, 0, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_create_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-        if (!state->fd) {
-                gf_log ("server", GF_LOG_ERROR, "fd creation for the inode %s "
-                        "failed", state->loc.inode?
-                        uuid_utoa (state->loc.inode->gfid):NULL);
-                state->resolve.op_ret = -1;
-                state->resolve.op_errno = ENOMEM;
-                goto err;
-        }
-        state->fd->flags = state->flags;
-
-        STACK_WIND (frame, server_create_cbk,
-                    bound_xl, bound_xl->fops->create,
-                    &(state->loc), state->flags, state->mode,
-                    state->fd, state->params);
-
-        return 0;
-err:
-        server_create_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL, NULL,
-                           NULL, NULL);
-        return 0;
-}
-
-
-int
-server_open_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-        state->fd->flags = state->flags;
-
-        STACK_WIND (frame, server_open_cbk,
-                    bound_xl, bound_xl->fops->open,
-                    &state->loc, state->flags, state->fd, 0);
-
-        return 0;
-err:
-        server_open_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                         state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_readlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readlink_cbk,
-                    bound_xl, bound_xl->fops->readlink,
-                    &state->loc, state->size);
-        return 0;
-err:
-        server_readlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_fsetattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsetattr_cbk,
-                    bound_xl, bound_xl->fops->fsetattr,
-                    state->fd, &state->stbuf, state->valid);
-        return 0;
-err:
-        server_fsetattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-int
-server_setattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setattr_cbk,
-                    bound_xl, bound_xl->fops->setattr,
-                    &state->loc, &state->stbuf, state->valid);
-        return 0;
-err:
-        server_setattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-int
-server_stat_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_stat_cbk,
-                    bound_xl, bound_xl->fops->stat, &state->loc);
-        return 0;
-err:
-        server_stat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                         state->resolve.op_errno, NULL);
-        return 0;
-}
-
-int
-server_lookup_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        if (!state->loc.inode)
-                state->loc.inode = inode_new (state->itable);
-        else
-                state->is_revalidate = 1;
-
-        STACK_WIND (frame, server_lookup_cbk,
-                    bound_xl, bound_xl->fops->lookup,
-                    &state->loc, state->dict);
-
-        return 0;
-err:
-        server_lookup_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL, NULL, NULL);
-
-        return 0;
-}
-
-
-
-
-/* Fop section */
-
-int
-server_stat (rpcsvc_request_t *req)
-{
-        server_state_t *state                 = NULL;
-        call_frame_t   *frame                 = NULL;
-        gfs3_stat_req   args                  = {{0,},};
-        int             ret                   = -1;
-
-        if (!req)
-                return 0;
-
-        /* Initialize args first, then decode */
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_stat_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_STAT;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_stat_resume);
-out:
-        return ret;
-}
-
-
-int
-server_setattr (rpcsvc_request_t *req)
-{
-        server_state_t   *state                 = NULL;
-        call_frame_t     *frame                 = NULL;
-        gfs3_setattr_req  args                  = {{0,},};
-        int               ret                   = -1;
-
-        if (!req)
-                return 0;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_setattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_SETATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        gf_stat_to_iatt (&args.stbuf, &state->stbuf);
-        state->valid = args.valid;
-
-        ret = 0;
-        resolve_and_resume (frame, server_setattr_resume);
-out:
-        return ret;
-}
-
-
-int
-server_fsetattr (rpcsvc_request_t *req)
-{
-        server_state_t    *state = NULL;
-        call_frame_t      *frame = NULL;
-        gfs3_fsetattr_req  args  = {0,};
-        int                ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fsetattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FSETATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = args.fd;
-
-        gf_stat_to_iatt (&args.stbuf, &state->stbuf);
-        state->valid = args.valid;
-
-        ret = 0;
-        resolve_and_resume (frame, server_fsetattr_resume);
-out:
-        return ret;
-}
-
-
-int
-server_readlink (rpcsvc_request_t *req)
-{
-        server_state_t    *state                 = NULL;
-        call_frame_t      *frame                 = NULL;
-        gfs3_readlink_req  args                  = {{0,},};
-        int                ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_readlink_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_READLINK;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        state->size  = args.size;
-
-        ret = 0;
-        resolve_and_resume (frame, server_readlink_resume);
-out:
-        return ret;
-}
-
-
-int
-server_create (rpcsvc_request_t *req)
-{
-        server_state_t  *state    = NULL;
-        call_frame_t    *frame    = NULL;
-        dict_t          *params   = NULL;
-        gfs3_create_req  args     = {{0,},};
-        int              ret      = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        args.bname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_create_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_CREATE;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        /* Unserialize the dictionary */
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, params,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->params = params;
-
-        state->resolve.bname  = gf_strdup (args.bname);
-        state->mode           = args.mode;
-        state->flags          = gf_flags_to_flags (args.flags);
-        memcpy (state->resolve.pargfid, args.pargfid, 16);
-
-        if (state->flags & O_EXCL) {
-                state->resolve.type = RESOLVE_NOT;
-        } else {
-                state->resolve.type = RESOLVE_DONTCARE;
-        }
-
-        ret = 0;
-        resolve_and_resume (frame, server_create_resume);
-
-        /* memory allocated by libc, don't use GF_FREE */
-        if (args.dict.dict_val != NULL) {
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-out:
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        if (params)
-                dict_unref (params);
-
-        /* memory allocated by libc, don't use GF_FREE */
-        if (args.dict.dict_val != NULL) {
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-}
-
-
-int
-server_open (rpcsvc_request_t *req)
-{
-        server_state_t *state                 = NULL;
-        call_frame_t   *frame                 = NULL;
-        gfs3_open_req   args                  = {{0,},};
-        int             ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_open_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_OPEN;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        state->flags = gf_flags_to_flags (args.flags);
-
-        ret = 0;
-        resolve_and_resume (frame, server_open_resume);
-out:
-        return ret;
-}
-
-
-int
-server_readv (rpcsvc_request_t *req)
-{
-        server_state_t *state = NULL;
-        call_frame_t   *frame = NULL;
-        gfs3_read_req   args  = {{0,},};
-        int             ret   = -1;
-
-        if (!req)
-                goto out;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_read_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_READ;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = args.fd;
-        state->size           = args.size;
-        state->offset         = args.offset;
-        state->flags          = args.flag;
-
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_readv_resume);
-out:
-        return ret;
-}
-
-
-int
-server_writev (rpcsvc_request_t *req)
-{
-        server_state_t      *state  = NULL;
-        call_frame_t        *frame  = NULL;
-        gfs3_write_req       args   = {{0,},};
-        ssize_t              len    = 0;
-        int                  i      = 0;
-        int                  ret    = -1;
-
-        if (!req)
-                return ret;
-
-        len = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_write_req);
-        if (len == 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_WRITE;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        state->offset        = args.offset;
-        state->flags         = args.flag;
-        state->iobref        = iobref_ref (req->iobref);
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        if (len < req->msg[0].iov_len) {
-                state->payload_vector[0].iov_base
-                        = (req->msg[0].iov_base + len);
-                state->payload_vector[0].iov_len
-                        = req->msg[0].iov_len - len;
-                state->payload_count = 1;
-        }
-
-        for (i = 1; i < req->count; i++) {
-                state->payload_vector[state->payload_count++]
-                        = req->msg[i];
-        }
-
-        for (i = 0; i < state->payload_count; i++) {
-                state->size += state->payload_vector[i].iov_len;
-        }
-
-        ret = 0;
-        resolve_and_resume (frame, server_writev_resume);
-out:
-        return ret;
-}
-
-
-int
-server_writev_vec (rpcsvc_request_t *req, struct iovec *payload,
-                   int payload_count, struct iobref *iobref)
-{
-        return server_writev (req);
-}
-
-#define SERVER3_1_VECWRITE_START 0
-#define SERVER3_1_VECWRITE_READINGHDR 1
-
-int
-server_writev_vecsizer (int state, ssize_t *readsize, char *addr)
-{
-        int nextstate = 0;
-        gfs3_write_req    write_req              = {{0,},};
-
-        switch (state) {
-        case SERVER3_1_VECWRITE_START:
-                *readsize = xdr_sizeof ((xdrproc_t) xdr_gfs3_write_req, &write_req);
-                nextstate = SERVER3_1_VECWRITE_READINGHDR;
-                break;
-        case SERVER3_1_VECWRITE_READINGHDR:
-                *readsize = 0;
-                nextstate = SERVER3_1_VECWRITE_START;
-                break;
-        default:
-                gf_log ("server3_1", GF_LOG_ERROR, "wrong state: %d", state);
-        }
-        return nextstate;
-}
-
-
-int
-server_release (rpcsvc_request_t *req)
-{
-        server_connection_t *conn = NULL;
-        gfs3_release_req     args = {{0,},};
-        gf_common_rsp        rsp  = {0,};
-        int                  ret  = -1;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_release_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        conn = req->trans->xl_private;
-        gf_fd_put (conn->fdtable, args.fd);
-
-        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-        ret = 0;
-out:
-        return ret;
-}
-
-int
-server_releasedir (rpcsvc_request_t *req)
-{
-        server_connection_t *conn = NULL;
-        gfs3_releasedir_req  args = {{0,},};
-        gf_common_rsp        rsp  = {0,};
-        int                  ret  = -1;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_release_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        conn = req->trans->xl_private;
-        gf_fd_put (conn->fdtable, args.fd);
-
-        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-        ret = 0;
-out:
-        return ret;
-}
-
-
-int
-server_fsync (rpcsvc_request_t *req)
-{
-        server_state_t *state = NULL;
-        call_frame_t   *frame = NULL;
-        gfs3_fsync_req  args  = {{0,},};
-        int             ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fsync_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FSYNC;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        state->flags         = args.data;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fsync_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_flush (rpcsvc_request_t *req)
-{
-        server_state_t *state = NULL;
-        call_frame_t   *frame = NULL;
-        gfs3_flush_req  args  = {{0,},};
-        int             ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_flush_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FLUSH;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_flush_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_ftruncate (rpcsvc_request_t *req)
-{
-        server_state_t     *state = NULL;
-        call_frame_t       *frame = NULL;
-        gfs3_ftruncate_req  args  = {{0,},};
-        int                 ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_ftruncate_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FTRUNCATE;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = args.fd;
-        state->offset         = args.offset;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_ftruncate_resume);
-out:
-        return ret;
-}
-
-
-int
-server_fstat (rpcsvc_request_t *req)
-{
-        server_state_t *state = NULL;
-        call_frame_t   *frame = NULL;
-        gfs3_write_req  args  = {{0,},};
-        int             ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fstat_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FSTAT;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.fd_no   = args.fd;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fstat_resume);
-out:
-        return ret;
-}
-
-
-int
-server_truncate (rpcsvc_request_t *req)
-{
-        server_state_t    *state                 = NULL;
-        call_frame_t      *frame                 = NULL;
-        gfs3_truncate_req  args                  = {{0,},};
-        int                ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_truncate_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_TRUNCATE;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-        state->offset        = args.offset;
-
-        ret = 0;
-        resolve_and_resume (frame, server_truncate_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_unlink (rpcsvc_request_t *req)
-{
-        server_state_t  *state                  = NULL;
-        call_frame_t    *frame                  = NULL;
-        gfs3_unlink_req  args                   = {{0,},};
-        int              ret                    = -1;
-
-        if (!req)
-                return ret;
-
-        args.bname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_unlink_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_UNLINK;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.bname  = gf_strdup (args.bname);
-        memcpy (state->resolve.pargfid, args.pargfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_unlink_resume);
-out:
-        return ret;
-}
-
-
-int
-server_setxattr (rpcsvc_request_t *req)
-{
-        server_state_t      *state                 = NULL;
-        dict_t              *dict                  = NULL;
-        call_frame_t        *frame                 = NULL;
-        server_connection_t *conn                  = NULL;
-        gfs3_setxattr_req    args                  = {{0,},};
-        int32_t              ret                   = -1;
-        int                  op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        conn = req->trans->xl_private;
-
-        args.dict.dict_val = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_setxattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_SETXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type     = RESOLVE_MUST;
-        state->flags            = args.flags;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, dict,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->dict = dict;
-
-        /* There can be some commands hidden in key, check and proceed */
-        gf_server_check_setxattr_cmd (frame, dict);
-
-        ret = 0;
-        resolve_and_resume (frame, server_setxattr_resume);
-
-        return ret;
-out:
-        if (dict)
-                dict_unref (dict);
-
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        return ret;
-}
-
-
-
-int
-server_fsetxattr (rpcsvc_request_t *req)
-{
-        server_state_t      *state                = NULL;
-        dict_t              *dict                 = NULL;
-        server_connection_t *conn                 = NULL;
-        call_frame_t        *frame                = NULL;
-        gfs3_fsetxattr_req   args                 = {{0,},};
-        int32_t              ret                  = -1;
-        int                  op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        conn = req->trans->xl_private;
-
-        args.dict.dict_val = alloca (req->msg[0].iov_len);
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fsetxattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FSETXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type      = RESOLVE_MUST;
-        state->resolve.fd_no     = args.fd;
-        state->flags             = args.flags;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, dict,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->dict = dict;
-
-        ret = 0;
-        resolve_and_resume (frame, server_fsetxattr_resume);
-
-        return ret;
-out:
-        if (dict)
-                dict_unref (dict);
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-        return ret;
-}
-
-
-
-int
-server_fxattrop (rpcsvc_request_t *req)
-{
-        dict_t              *dict                 = NULL;
-        server_state_t      *state                = NULL;
-        server_connection_t *conn                 = NULL;
-        call_frame_t        *frame                = NULL;
-        gfs3_fxattrop_req    args                 = {{0,},};
-        int32_t              ret                  = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        conn = req->trans->xl_private;
-
-        args.dict.dict_val = alloca (req->msg[0].iov_len);
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fxattrop_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FXATTROP;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.fd_no   = args.fd;
-        state->flags           = args.flags;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, dict,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->dict = dict;
-
-        ret = 0;
-        resolve_and_resume (frame, server_fxattrop_resume);
-
-        return ret;
-
-out:
-        if (dict)
-                dict_unref (dict);
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        return ret;
-}
-
-
-
-int
-server_xattrop (rpcsvc_request_t *req)
-{
-        dict_t              *dict                  = NULL;
-        server_state_t      *state                 = NULL;
-        server_connection_t *conn                  = NULL;
-        call_frame_t        *frame                 = NULL;
-        gfs3_xattrop_req     args                  = {{0,},};
-        int32_t              ret                   = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        conn = req->trans->xl_private;
-
-        args.dict.dict_val = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_xattrop_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_XATTROP;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->flags           = args.flags;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, dict,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->dict = dict;
-
-        ret = 0;
-        resolve_and_resume (frame, server_xattrop_resume);
-
-        return ret;
-out:
-        if (dict)
-                dict_unref (dict);
-
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        return ret;
-}
-
-
-int
-server_getxattr (rpcsvc_request_t *req)
-{
-        server_state_t      *state                 = NULL;
-        call_frame_t        *frame                 = NULL;
-        gfs3_getxattr_req    args                  = {{0,},};
-        int                  ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        args.name = alloca (256);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_getxattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_GETXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        if (args.namelen) {
-                state->name = gf_strdup (args.name);
-                /* There can be some commands hidden in key, check and proceed */
-                gf_server_check_getxattr_cmd (frame, state->name);
-        }
-
-        ret = 0;
-        resolve_and_resume (frame, server_getxattr_resume);
-out:
-        return ret;
-}
-
-
-int
-server_fgetxattr (rpcsvc_request_t *req)
-{
-        server_state_t      *state      = NULL;
-        call_frame_t        *frame      = NULL;
-        gfs3_fgetxattr_req   args       = {{0,},};
-        int                  ret        = -1;
-
-        if (!req)
-                return ret;
-
-        args.name = alloca (256);
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fgetxattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FGETXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        if (args.namelen)
-                state->name = gf_strdup (args.name);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fgetxattr_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_removexattr (rpcsvc_request_t *req)
-{
-        server_state_t       *state                 = NULL;
-        call_frame_t         *frame                 = NULL;
-        gfs3_removexattr_req  args                  = {{0,},};
-        int                   ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        args.name = alloca (256);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_removexattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_REMOVEXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-        state->name           = gf_strdup (args.name);
-
-        ret = 0;
-        resolve_and_resume (frame, server_removexattr_resume);
-out:
-        return ret;
-}
-
-int
-server_fremovexattr (rpcsvc_request_t *req)
-{
-        server_state_t       *state                 = NULL;
-        call_frame_t         *frame                 = NULL;
-        gfs3_fremovexattr_req  args                  = {{0,},};
-        int                   ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        args.name = alloca (4096);
-
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_gfs3_fremovexattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FREMOVEXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = args.fd;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-        state->name           = gf_strdup (args.name);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fremovexattr_resume);
-out:
-        return ret;
-}
-
-
-
-
-int
-server_opendir (rpcsvc_request_t *req)
-{
-        server_state_t   *state                 = NULL;
-        call_frame_t     *frame                 = NULL;
-        gfs3_opendir_req  args                  = {{0,},};
-        int               ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_opendir_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_OPENDIR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_opendir_resume);
-out:
-        return ret;
-}
-
-
-int
-server_readdirp (rpcsvc_request_t *req)
-{
-        server_state_t      *state        = NULL;
-        call_frame_t        *frame        = NULL;
-        gfs3_readdirp_req    args         = {{0,},};
-        size_t               headers_size = 0;
-        int                  ret          = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_readdirp_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_READDIRP;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        /* FIXME: this should go away when variable sized iobufs are introduced
-         * and transport layer can send msgs bigger than current page-size.
-         */
-        headers_size = sizeof (struct rpc_msg) + sizeof (gfs3_readdir_rsp);
-        if ((frame->this->ctx->page_size < args.size)
-            || ((frame->this->ctx->page_size - args.size) < headers_size)) {
-                state->size = frame->this->ctx->page_size - headers_size;
-        } else {
-                state->size   = args.size;
-        }
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        state->offset = args.offset;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, state->dict,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-
-        ret = 0;
-        resolve_and_resume (frame, server_readdirp_resume);
-out:
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        return ret;
-}
-
-int
-server_readdir (rpcsvc_request_t *req)
-{
-        server_state_t      *state        = NULL;
-        call_frame_t        *frame        = NULL;
-        gfs3_readdir_req     args         = {{0,},};
-        size_t               headers_size = 0;
-        int                  ret          = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_readdir_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_READDIR;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        /* FIXME: this should go away when variable sized iobufs are introduced
-         * and transport layer can send msgs bigger than current page-size.
-         */
-        headers_size = sizeof (struct rpc_msg) + sizeof (gfs3_readdir_rsp);
-        if ((frame->this->ctx->page_size < args.size)
-            || ((frame->this->ctx->page_size - args.size) < headers_size)) {
-                state->size = frame->this->ctx->page_size - headers_size;
-        } else {
-                state->size   = args.size;
-        }
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        state->offset = args.offset;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_readdir_resume);
-out:
-        return ret;
-}
-
-int
-server_fsyncdir (rpcsvc_request_t *req)
-{
-        server_state_t      *state = NULL;
-        call_frame_t        *frame = NULL;
-        gfs3_fsyncdir_req    args  = {{0,},};
-        int                  ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fsyncdir_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FSYNCDIR;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        state->flags = args.data;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fsyncdir_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_mknod (rpcsvc_request_t *req)
-{
-        server_state_t      *state                  = NULL;
-        call_frame_t        *frame                  = NULL;
-        dict_t              *params                 = NULL;
-        gfs3_mknod_req       args                   = {{0,},};
-        int                  ret                    = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        args.bname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_mknod_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_MKNOD;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, params,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->params = params;
-
-        state->resolve.type    = RESOLVE_NOT;
-        memcpy (state->resolve.pargfid, args.pargfid, 16);
-        state->resolve.bname   = gf_strdup (args.bname);
-
-        state->mode = args.mode;
-        state->dev  = args.dev;
-
-        ret = 0;
-        resolve_and_resume (frame, server_mknod_resume);
-
-        /* memory allocated by libc, don't use GF_FREE */
-        if (args.dict.dict_val != NULL) {
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-out:
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        if (params)
-                dict_unref (params);
-
-        /* memory allocated by libc, don't use GF_FREE */
-        if (args.dict.dict_val != NULL) {
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-
-}
-
-
-int
-server_mkdir (rpcsvc_request_t *req)
-{
-        server_state_t      *state                  = NULL;
-        call_frame_t        *frame                  = NULL;
-        dict_t              *params                 = NULL;
-        gfs3_mkdir_req       args                   = {{0,},};
-        int                  ret                    = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        args.bname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_mkdir_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_MKDIR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, params,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->params = params;
-
-        state->resolve.type    = RESOLVE_NOT;
-        memcpy (state->resolve.pargfid, args.pargfid, 16);
-        state->resolve.bname   = gf_strdup (args.bname);
-
-        state->mode = args.mode;
-
-        ret = 0;
-        resolve_and_resume (frame, server_mkdir_resume);
-
-        if (args.dict.dict_val != NULL) {
-                /* memory allocated by libc, don't use GF_FREE */
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-out:
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        if (params)
-                dict_unref (params);
-
-        if (args.dict.dict_val != NULL) {
-                /* memory allocated by libc, don't use GF_FREE */
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-}
-
-
-int
-server_rmdir (rpcsvc_request_t *req)
-{
-        server_state_t      *state                  = NULL;
-        call_frame_t        *frame                  = NULL;
-        gfs3_rmdir_req       args                   = {{0,},};
-        int                  ret                    = -1;
-
-        if (!req)
-                return ret;
-
-        args.bname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_rmdir_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_RMDIR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type    = RESOLVE_MUST;
-        memcpy (state->resolve.pargfid, args.pargfid, 16);
-        state->resolve.bname   = gf_strdup (args.bname);
-
-        state->flags = args.flags;
-
-        ret = 0;
-        resolve_and_resume (frame, server_rmdir_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_inodelk (rpcsvc_request_t *req)
-{
-        server_state_t      *state                 = NULL;
-        call_frame_t        *frame                 = NULL;
-        gfs3_inodelk_req     args                  = {{0,},};
-        int                  cmd                   = 0;
-        int                  ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        args.volume = alloca (256);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_inodelk_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_INODELK;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type    = RESOLVE_EXACT;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        cmd = args.cmd;
-        switch (cmd) {
-        case GF_LK_GETLK:
-                state->cmd = F_GETLK;
-                break;
-        case GF_LK_SETLK:
-                state->cmd = F_SETLK;
-                break;
-        case GF_LK_SETLKW:
-                state->cmd = F_SETLKW;
-                break;
-        }
-
-        state->type = args.type;
-        state->volume = gf_strdup (args.volume);
-
-        gf_proto_flock_to_flock (&args.flock, &state->flock);
-
-        switch (state->type) {
-        case GF_LK_F_RDLCK:
-                state->flock.l_type = F_RDLCK;
-                break;
-        case GF_LK_F_WRLCK:
-                state->flock.l_type = F_WRLCK;
-                break;
-        case GF_LK_F_UNLCK:
-                state->flock.l_type = F_UNLCK;
-                break;
-        }
-
-        ret = 0;
-        resolve_and_resume (frame, server_inodelk_resume);
-out:
-        return ret;
-}
-
-int
-server_finodelk (rpcsvc_request_t *req)
-{
-        server_state_t      *state        = NULL;
-        call_frame_t        *frame        = NULL;
-        gfs3_finodelk_req    args         = {{0,},};
-        int                  ret          = -1;
-
-        if (!req)
-                return ret;
-
-        args.volume = alloca (256);
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_finodelk_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FINODELK;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type = RESOLVE_EXACT;
-        state->volume = gf_strdup (args.volume);
-        state->resolve.fd_no = args.fd;
-        state->cmd = args.cmd;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        switch (state->cmd) {
-        case GF_LK_GETLK:
-                state->cmd = F_GETLK;
-                break;
-        case GF_LK_SETLK:
-                state->cmd = F_SETLK;
-                break;
-        case GF_LK_SETLKW:
-                state->cmd = F_SETLKW;
-                break;
-        }
-
-        state->type = args.type;
-
-        gf_proto_flock_to_flock (&args.flock, &state->flock);
-
-        switch (state->type) {
-        case GF_LK_F_RDLCK:
-                state->flock.l_type = F_RDLCK;
-                break;
-        case GF_LK_F_WRLCK:
-                state->flock.l_type = F_WRLCK;
-                break;
-        case GF_LK_F_UNLCK:
-                state->flock.l_type = F_UNLCK;
-                break;
-        }
-
-        ret = 0;
-        resolve_and_resume (frame, server_finodelk_resume);
-out:
-        return ret;
-}
-
-
-int
-server_entrylk (rpcsvc_request_t *req)
-{
-        server_state_t      *state                 = NULL;
-        call_frame_t        *frame                 = NULL;
-        gfs3_entrylk_req     args                  = {{0,},};
-        int                  ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        args.volume = alloca (256);
-        args.name   = alloca (256);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_entrylk_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_ENTRYLK;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_EXACT;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        if (args.namelen)
-                state->name   = gf_strdup (args.name);
-        state->volume         = gf_strdup (args.volume);
-
-        state->cmd            = args.cmd;
-        state->type           = args.type;
-
-        ret = 0;
-        resolve_and_resume (frame, server_entrylk_resume);
-out:
-        return ret;
-}
-
-int
-server_fentrylk (rpcsvc_request_t *req)
-{
-        server_state_t      *state        = NULL;
-        call_frame_t        *frame        = NULL;
-        gfs3_fentrylk_req    args         = {{0,},};
-        int                  ret          = -1;
-
-        if (!req)
-                return ret;
-
-        args.name   = alloca (256);
-        args.volume = alloca (256);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fentrylk_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FENTRYLK;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type = RESOLVE_EXACT;
-        state->resolve.fd_no = args.fd;
-        state->cmd  = args.cmd;
-        state->type = args.type;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        if (args.namelen)
-                state->name = gf_strdup (args.name);
-        state->volume = gf_strdup (args.volume);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fentrylk_resume);
-out:
-        return ret;
-}
-
-int
-server_access (rpcsvc_request_t *req)
-{
-        server_state_t      *state                 = NULL;
-        call_frame_t        *frame                 = NULL;
-        gfs3_access_req      args                  = {{0,},};
-        int                  ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_access_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_ACCESS;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-        state->mask          = args.mask;
-
-        ret = 0;
-        resolve_and_resume (frame, server_access_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_symlink (rpcsvc_request_t *req)
-{
-        server_state_t      *state                 = NULL;
-        call_frame_t        *frame                 = NULL;
-        dict_t              *params                = NULL;
-        gfs3_symlink_req     args                  = {{0,},};
-        int                  ret                   = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        args.bname    = alloca (req->msg[0].iov_len);
-        args.linkname = alloca (4096);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_symlink_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_SYMLINK;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, params,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->params = params;
-
-        state->resolve.type   = RESOLVE_NOT;
-        memcpy (state->resolve.pargfid, args.pargfid, 16);
-        state->resolve.bname  = gf_strdup (args.bname);
-        state->name           = gf_strdup (args.linkname);
-
-        ret = 0;
-        resolve_and_resume (frame, server_symlink_resume);
-
-        /* memory allocated by libc, don't use GF_FREE */
-        if (args.dict.dict_val != NULL) {
-                free (args.dict.dict_val);
-        }
-        return ret;
-out:
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        if (params)
-                dict_unref (params);
-
-        /* memory allocated by libc, don't use GF_FREE */
-        if (args.dict.dict_val != NULL) {
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-}
-
-
-
-int
-server_link (rpcsvc_request_t *req)
-{
-        server_state_t      *state                     = NULL;
-        call_frame_t        *frame                     = NULL;
-        gfs3_link_req        args                      = {{0,},};
-        int                  ret                       = -1;
-
-        if (!req)
-                return ret;
-
-        args.oldpath  = alloca (req->msg[0].iov_len);
-        args.newpath  = alloca (req->msg[0].iov_len);
-        args.newbname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_link_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_LINK;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.path    = gf_strdup (args.oldpath);
-        memcpy (state->resolve.gfid, args.oldgfid, 16);
-
-        state->resolve2.type   = RESOLVE_NOT;
-        state->resolve2.path   = gf_strdup (args.newpath);
-        state->resolve2.bname  = gf_strdup (args.newbname);
-        memcpy (state->resolve2.pargfid, args.newgfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_link_resume);
-out:
-        return ret;
-}
-
-
-int
-server_rename (rpcsvc_request_t *req)
-{
-        server_state_t      *state                     = NULL;
-        call_frame_t        *frame                     = NULL;
-        gfs3_rename_req      args                      = {{0,},};
-        int                  ret                       = -1;
-
-        if (!req)
-                return ret;
-
-        args.oldpath  = alloca (req->msg[0].iov_len);
-        args.oldbname = alloca (req->msg[0].iov_len);
-        args.newpath  = alloca (req->msg[0].iov_len);
-        args.newbname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_rename_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_RENAME;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.path   = gf_strdup (args.oldpath);
-        state->resolve.bname  = gf_strdup (args.oldbname);
-        memcpy (state->resolve.pargfid, args.oldgfid, 16);
-
-        state->resolve2.type  = RESOLVE_MAY;
-        state->resolve2.path  = gf_strdup (args.newpath);
-        state->resolve2.bname = gf_strdup (args.newbname);
-        memcpy (state->resolve2.pargfid, args.newgfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_rename_resume);
-out:
-        return ret;
-}
-
-int
-server_lk (rpcsvc_request_t *req)
-{
-        server_state_t      *state = NULL;
-        server_connection_t *conn  = NULL;
-        call_frame_t        *frame = NULL;
-        gfs3_lk_req          args  = {{0,},};
-        int                  ret   = -1;
-
-        if (!req)
-                return ret;
-
-        conn = req->trans->xl_private;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_lk_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_LK;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.fd_no = args.fd;
-        state->cmd =  args.cmd;
-        state->type = args.type;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        switch (state->cmd) {
-        case GF_LK_GETLK:
-                state->cmd = F_GETLK;
-                break;
-        case GF_LK_SETLK:
-                state->cmd = F_SETLK;
-                break;
-        case GF_LK_SETLKW:
-                state->cmd = F_SETLKW;
-                break;
-        case GF_LK_RESLK_LCK:
-                state->cmd = F_RESLK_LCK;
-                break;
-        case GF_LK_RESLK_LCKW:
-                state->cmd = F_RESLK_LCKW;
-                break;
-        case GF_LK_RESLK_UNLCK:
-                state->cmd = F_RESLK_UNLCK;
-                break;
-        case GF_LK_GETLK_FD:
-                state->cmd = F_GETLK_FD;
-                break;
-
-        }
-
-
-        gf_proto_flock_to_flock (&args.flock, &state->flock);
-
-        switch (state->type) {
-        case GF_LK_F_RDLCK:
-                state->flock.l_type = F_RDLCK;
-                break;
-        case GF_LK_F_WRLCK:
-                state->flock.l_type = F_WRLCK;
-                break;
-        case GF_LK_F_UNLCK:
-                state->flock.l_type = F_UNLCK;
-                break;
-        default:
-                gf_log (conn->bound_xl->name, GF_LOG_ERROR,
-                        "fd - %"PRId64" (%s): Unknown lock type: %"PRId32"!",
-                        state->resolve.fd_no,
-                        uuid_utoa (state->fd->inode->gfid), state->type);
-                break;
-        }
-
-
-        ret = 0;
-        resolve_and_resume (frame, server_lk_resume);
-out:
-        return ret;
-}
-
-
-int
-server_rchecksum (rpcsvc_request_t *req)
-{
-        server_state_t      *state = NULL;
-        call_frame_t        *frame = NULL;
-        gfs3_rchecksum_req   args  = {0,};
-        int                  ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_rchecksum_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_RCHECKSUM;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MAY;
-        state->resolve.fd_no = args.fd;
-        state->offset        = args.offset;
-        state->size          = args.len;
-
-        ret = 0;
-        resolve_and_resume (frame, server_rchecksum_resume);
-out:
-        return ret;
-}
-
-int
-server_null (rpcsvc_request_t *req)
-{
-        gf_common_rsp rsp = {0,};
-
-        /* Accepted */
-        rsp.op_ret = 0;
-
-        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_lookup (rpcsvc_request_t *req)
-{
-        call_frame_t        *frame                  = NULL;
-        server_connection_t *conn                   = NULL;
-        server_state_t      *state                  = NULL;
-        dict_t              *xattr_req              = NULL;
-        gfs3_lookup_req      args                   = {{0,},};
-        int                  ret                    = -1;
-        int              op_errno = 0;
-
-        GF_VALIDATE_OR_GOTO ("server", req, err);
-
-        conn = req->trans->xl_private;
-
-        args.bname         = alloca (req->msg[0].iov_len);
-        args.dict.dict_val = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_lookup_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto err;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS;
-                goto err;
-        }
-        frame->root->op = GF_FOP_LOOKUP;
-
-        /* NOTE: lookup() uses req->ino only to identify if a lookup()
-         *       is requested for 'root' or not
-         */
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_DONTCARE;
-
-        if (args.bname && strcmp (args.bname, "")) {
-                memcpy (state->resolve.pargfid, args.pargfid, 16);
-                state->resolve.bname = gf_strdup (args.bname);
-        } else {
-                memcpy (state->resolve.gfid, args.gfid, 16);
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, xattr_req,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-        state->dict = xattr_req;
-
-        ret = 0;
-        resolve_and_resume (frame, server_lookup_resume);
-
-        return ret;
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        server_lookup_cbk (frame, NULL, frame->this, -1, EINVAL, NULL, NULL,
-                           NULL, NULL);
-        ret = 0;
-err:
-        return ret;
-}
-
-int
-server_statfs (rpcsvc_request_t *req)
-{
-        server_state_t      *state = NULL;
-        call_frame_t        *frame = NULL;
-        gfs3_statfs_req      args  = {{0,},};
-        int                  ret                    = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_statfs_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_STATFS;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_statfs_resume);
-out:
-        return ret;
-}
-
-
-rpcsvc_actor_t glusterfs3_1_fop_actors[] = {
-        [GFS3_OP_NULL]        = { "NULL",       GFS3_OP_NULL, server_null, NULL, NULL, 0},
-        [GFS3_OP_STAT]        = { "STAT",       GFS3_OP_STAT, server_stat, NULL, NULL, 0},
-        [GFS3_OP_READLINK]    = { "READLINK",   GFS3_OP_READLINK, server_readlink, NULL, NULL, 0},
-        [GFS3_OP_MKNOD]       = { "MKNOD",      GFS3_OP_MKNOD, server_mknod, NULL, NULL, 0},
-        [GFS3_OP_MKDIR]       = { "MKDIR",      GFS3_OP_MKDIR, server_mkdir, NULL, NULL, 0},
-        [GFS3_OP_UNLINK]      = { "UNLINK",     GFS3_OP_UNLINK, server_unlink, NULL, NULL, 0},
-        [GFS3_OP_RMDIR]       = { "RMDIR",      GFS3_OP_RMDIR, server_rmdir, NULL, NULL, 0},
-        [GFS3_OP_SYMLINK]     = { "SYMLINK",    GFS3_OP_SYMLINK, server_symlink, NULL, NULL, 0},
-        [GFS3_OP_RENAME]      = { "RENAME",     GFS3_OP_RENAME, server_rename, NULL, NULL, 0},
-        [GFS3_OP_LINK]        = { "LINK",       GFS3_OP_LINK, server_link, NULL, NULL, 0},
-        [GFS3_OP_TRUNCATE]    = { "TRUNCATE",   GFS3_OP_TRUNCATE, server_truncate, NULL, NULL, 0},
-        [GFS3_OP_OPEN]        = { "OPEN",       GFS3_OP_OPEN, server_open, NULL, NULL, 0},
-        [GFS3_OP_READ]        = { "READ",       GFS3_OP_READ, server_readv, NULL, NULL, 0},
-        [GFS3_OP_WRITE]       = { "WRITE",      GFS3_OP_WRITE, server_writev, server_writev_vec, server_writev_vecsizer, 0},
-        [GFS3_OP_STATFS]      = { "STATFS",     GFS3_OP_STATFS, server_statfs, NULL, NULL, 0},
-        [GFS3_OP_FLUSH]       = { "FLUSH",      GFS3_OP_FLUSH, server_flush, NULL, NULL, 0},
-        [GFS3_OP_FSYNC]       = { "FSYNC",      GFS3_OP_FSYNC, server_fsync, NULL, NULL, 0},
-        [GFS3_OP_SETXATTR]    = { "SETXATTR",   GFS3_OP_SETXATTR, server_setxattr, NULL, NULL, 0},
-        [GFS3_OP_GETXATTR]    = { "GETXATTR",   GFS3_OP_GETXATTR, server_getxattr, NULL, NULL, 0},
-        [GFS3_OP_REMOVEXATTR] = { "REMOVEXATTR", GFS3_OP_REMOVEXATTR, server_removexattr, NULL, NULL, 0},
-        [GFS3_OP_OPENDIR]     = { "OPENDIR",    GFS3_OP_OPENDIR, server_opendir, NULL, NULL, 0},
-        [GFS3_OP_FSYNCDIR]    = { "FSYNCDIR",   GFS3_OP_FSYNCDIR, server_fsyncdir, NULL, NULL, 0},
-        [GFS3_OP_ACCESS]      = { "ACCESS",     GFS3_OP_ACCESS, server_access, NULL, NULL, 0},
-        [GFS3_OP_CREATE]      = { "CREATE",     GFS3_OP_CREATE, server_create, NULL, NULL, 0},
-        [GFS3_OP_FTRUNCATE]   = { "FTRUNCATE",  GFS3_OP_FTRUNCATE, server_ftruncate, NULL, NULL, 0},
-        [GFS3_OP_FSTAT]       = { "FSTAT",      GFS3_OP_FSTAT, server_fstat, NULL, NULL, 0},
-        [GFS3_OP_LK]          = { "LK",         GFS3_OP_LK, server_lk, NULL, NULL, 0},
-        [GFS3_OP_LOOKUP]      = { "LOOKUP",     GFS3_OP_LOOKUP, server_lookup, NULL, NULL, 0},
-        [GFS3_OP_READDIR]     = { "READDIR",    GFS3_OP_READDIR, server_readdir, NULL, NULL, 0},
-        [GFS3_OP_INODELK]     = { "INODELK",    GFS3_OP_INODELK, server_inodelk, NULL, NULL, 0},
-        [GFS3_OP_FINODELK]    = { "FINODELK",   GFS3_OP_FINODELK, server_finodelk, NULL, NULL, 0},
-        [GFS3_OP_ENTRYLK]     = { "ENTRYLK",    GFS3_OP_ENTRYLK, server_entrylk, NULL, NULL, 0},
-        [GFS3_OP_FENTRYLK]    = { "FENTRYLK",   GFS3_OP_FENTRYLK, server_fentrylk, NULL, NULL, 0},
-        [GFS3_OP_XATTROP]     = { "XATTROP",    GFS3_OP_XATTROP, server_xattrop, NULL, NULL, 0},
-        [GFS3_OP_FXATTROP]    = { "FXATTROP",   GFS3_OP_FXATTROP, server_fxattrop, NULL, NULL, 0},
-        [GFS3_OP_FGETXATTR]   = { "FGETXATTR",  GFS3_OP_FGETXATTR, server_fgetxattr, NULL, NULL, 0},
-        [GFS3_OP_FSETXATTR]   = { "FSETXATTR",  GFS3_OP_FSETXATTR, server_fsetxattr, NULL, NULL, 0},
-        [GFS3_OP_RCHECKSUM]   = { "RCHECKSUM",  GFS3_OP_RCHECKSUM, server_rchecksum, NULL, NULL, 0},
-        [GFS3_OP_SETATTR]     = { "SETATTR",    GFS3_OP_SETATTR, server_setattr, NULL, NULL, 0},
-        [GFS3_OP_FSETATTR]    = { "FSETATTR",   GFS3_OP_FSETATTR, server_fsetattr, NULL, NULL, 0},
-        [GFS3_OP_READDIRP]    = { "READDIRP",   GFS3_OP_READDIRP, server_readdirp, NULL, NULL, 0},
-        [GFS3_OP_RELEASE]     = { "RELEASE",    GFS3_OP_RELEASE, server_release, NULL, NULL, 0},
-        [GFS3_OP_RELEASEDIR]  = { "RELEASEDIR", GFS3_OP_RELEASEDIR, server_releasedir, NULL, NULL, 0},
-        [GFS3_OP_FREMOVEXATTR] = { "FREMOVEXATTR", GFS3_OP_FREMOVEXATTR, server_fremovexattr, NULL, NULL, 0},
-};
-
-
-struct rpcsvc_program glusterfs3_1_fop_prog = {
-        .progname  = "GlusterFS " PACKAGE_VERSION,
-        .prognum   = GLUSTER3_1_FOP_PROGRAM,
-        .progver   = GLUSTER3_1_FOP_VERSION,
-        .numactors = GLUSTER3_1_FOP_PROCCNT,
-        .actors    = glusterfs3_1_fop_actors,
-};
diff --git a/xlators/storage/Makefile.am b/xlators/storage/Makefile.am
index 9cb9ded30..c08e8e41b 100644
--- a/xlators/storage/Makefile.am
+++ b/xlators/storage/Makefile.am
@@ -1,3 +1,7 @@
 SUBDIRS = posix
 
-CLEANFILES = 
+if ENABLE_BD_XLATOR
+SUBDIRS += bd
+endif
+
+CLEANFILES =
diff --git a/xlators/storage/bd/Makefile.am b/xlators/storage/bd/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/storage/bd/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/storage/bd/src/Makefile.am b/xlators/storage/bd/src/Makefile.am
new file mode 100644
index 000000000..3d93f7442
--- /dev/null
+++ b/xlators/storage/bd/src/Makefile.am
@@ -0,0 +1,20 @@
+if ENABLE_BD_XLATOR
+xlator_LTLIBRARIES = bd.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/storage
+
+bd_la_LDFLAGS = -module -avoid-version
+LIBBD = -llvm2app -lrt
+bd_la_SOURCES = bd.c bd-helper.c bd-aio.c
+bd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(LIBBD) $(LIBAIO)
+
+noinst_HEADERS = bd.h bd-aio.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+            -I$(top_srcdir)/rpc/xdr/src \
+            -I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -fno-strict-aliasing -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
+endif
diff --git a/xlators/storage/bd/src/bd-aio.c b/xlators/storage/bd/src/bd-aio.c
new file mode 100644
index 000000000..62d4590f7
--- /dev/null
+++ b/xlators/storage/bd/src/bd-aio.c
@@ -0,0 +1,527 @@
+/*
+  Copyright IBM, Corp. 2013
+
+  This file is part of GlusterFS.
+
+  Author: M. Mohan Kumar <mohan@in.ibm.com>
+
+  Based on posix-aio.c
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <lvm2app.h>
+#include <sys/uio.h>
+
+#include "xlator.h"
+#include "glusterfs.h"
+#include "defaults.h"
+#include "bd.h"
+#include "bd-aio.h"
+
+#ifdef HAVE_LIBAIO
+#include <libaio.h>
+
+struct bd_aio_cb {
+        struct iocb     iocb;
+        call_frame_t   *frame;
+        struct iobuf   *iobuf;
+        struct iobref  *iobref;
+        struct iatt     prebuf;
+        int             op;
+        off_t           offset;
+        fd_t           *fd;
+};
+
+void
+__bd_fd_set_odirect (fd_t *fd, bd_fd_t *bd_fd, int opflags,
+                     off_t offset, size_t size)
+{
+        int odirect = 0;
+        int flags = 0;
+        int ret = 0;
+
+        odirect = bd_fd->odirect;
+
+        if ((fd->flags|opflags) & O_DIRECT) {
+                /* if instructed, use O_DIRECT always */
+                odirect = 1;
+        } else {
+                /* else use O_DIRECT when feasible */
+                if ((offset|size) & 0xfff)
+                        odirect = 0;
+                else
+                        odirect = 1;
+        }
+
+        if (!odirect && bd_fd->odirect) {
+                flags = fcntl (bd_fd->fd, F_GETFL);
+                ret = fcntl (bd_fd->fd, F_SETFL, (flags & (~O_DIRECT)));
+                bd_fd->odirect = 0;
+        }
+
+        if (odirect && !bd_fd->odirect) {
+                flags = fcntl (bd_fd->fd, F_GETFL);
+                ret = fcntl (bd_fd->fd, F_SETFL, (flags | O_DIRECT));
+                bd_fd->odirect = 1;
+        }
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "fcntl() failed (%s). fd=%d flags=%d pfd->odirect=%d",
+                        strerror (errno), bd_fd->fd, flags, bd_fd->odirect);
+        }
+}
+
+int
+bd_aio_readv_complete (struct bd_aio_cb *paiocb, int res, int res2)
+{
+        call_frame_t   *frame = NULL;
+        xlator_t       *this = NULL;
+        struct iobuf   *iobuf = NULL;
+        struct iatt     postbuf = {0,};
+        int             op_ret = -1;
+        int             op_errno = 0;
+        struct iovec    iov;
+        struct iobref  *iobref = NULL;
+        off_t           offset = 0;
+        bd_attr_t      *bdatt = NULL;
+
+        frame = paiocb->frame;
+        this = frame->this;
+        iobuf = paiocb->iobuf;
+        offset = paiocb->offset;
+
+        if (res < 0) {
+                op_ret = -1;
+                op_errno = -res;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "readv(async) failed fd=%p,size=%lu,offset=%llu (%d/%s)",
+                        paiocb->fd, paiocb->iocb.u.c.nbytes,
+                        (unsigned long long) paiocb->offset,
+                        res, strerror (op_errno));
+                goto out;
+        }
+
+        bd_inode_ctx_get (paiocb->fd->inode, this, &bdatt);
+        memcpy (&postbuf, &bdatt->iatt, sizeof (struct iatt));
+
+        op_ret = res;
+        op_errno = 0;
+
+        iobref = iobref_new ();
+        if (!iobref) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        iobref_add (iobref, iobuf);
+
+        iov.iov_base = iobuf_ptr (iobuf);
+        iov.iov_len = op_ret;
+
+        /* Hack to notify higher layers of EOF. */
+        if (!postbuf.ia_size || (offset + iov.iov_len) >= postbuf.ia_size)
+                op_errno = ENOENT;
+
+out:
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, &iov, 1,
+                             &postbuf, iobref, NULL);
+        if (iobuf)
+                iobuf_unref (iobuf);
+        if (iobref)
+                iobref_unref (iobref);
+
+        GF_FREE (paiocb);
+
+        return 0;
+}
+
+int
+bd_aio_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+        int32_t             op_errno   = EINVAL;
+        int                 _fd        = -1;
+        struct iobuf       *iobuf      = NULL;
+        bd_fd_t            *bd_fd      = NULL;
+        int                 ret        = -1;
+        struct bd_aio_cb   *paiocb     = NULL;
+        bd_priv_t          *priv       = NULL;
+        struct iocb        *iocb       = NULL;
+        bd_attr_t          *bdatt      = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        priv = this->private;
+
+        ret = bd_fd_ctx_get (this, fd, &bd_fd);
+        if (ret < 0 || !bd_fd) {
+                STACK_WIND (frame, default_readv_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->readv, fd, size, offset,
+                            flags, xdata);
+                return 0;
+        }
+        _fd = bd_fd->fd;
+        bd_inode_ctx_get (fd->inode, this, &bdatt);
+        if (!size) {
+                op_errno = EINVAL;
+                gf_log (this->name, GF_LOG_WARNING, "size=%"GF_PRI_SIZET, size);
+                goto err;
+        }
+
+        iobuf = iobuf_get2 (this->ctx->iobuf_pool, size);
+        if (!iobuf) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        paiocb = CALLOC (1, sizeof (*paiocb));
+        if (!paiocb) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        paiocb->frame = frame;
+        paiocb->iobuf = iobuf;
+        paiocb->offset = offset;
+        paiocb->op = GF_FOP_READ;
+        paiocb->fd = fd;
+
+        paiocb->iocb.data = paiocb;
+        paiocb->iocb.aio_fildes = _fd;
+        paiocb->iocb.aio_lio_opcode = IO_CMD_PREAD;
+        paiocb->iocb.aio_reqprio = 0;
+        paiocb->iocb.u.c.buf = iobuf_ptr (iobuf);
+        paiocb->iocb.u.c.nbytes = size;
+        paiocb->iocb.u.c.offset = offset;
+
+        iocb = &paiocb->iocb;
+
+        LOCK (&fd->lock);
+        {
+                __bd_fd_set_odirect (fd, bd_fd, flags, offset, size);
+
+                ret = io_submit (priv->ctxp, 1, &iocb);
+        }
+        UNLOCK (&fd->lock);
+
+        if (ret != 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "io_submit() returned %d", ret);
+                op_errno = -ret;
+                goto err;
+        }
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
+        if (iobuf)
+                iobuf_unref (iobuf);
+
+        if (paiocb)
+                GF_FREE (paiocb);
+
+        return 0;
+}
+
+int
+bd_aio_writev_complete (struct bd_aio_cb *paiocb, int res, int res2)
+{
+        call_frame_t *frame    = NULL;
+        xlator_t     *this     = NULL;
+        struct iatt   prebuf   = {0,};
+        struct iatt   postbuf  = {0,};
+        int           op_ret   = -1;
+        int           op_errno = 0;
+        bd_attr_t    *bdatt    = NULL;
+
+        frame = paiocb->frame;
+        prebuf = paiocb->prebuf;
+        this = frame->this;
+
+        if (res < 0) {
+                op_ret = -1;
+                op_errno = -res;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "writev(async) failed fd=%p,offset=%llu (%d/%s)",
+                        paiocb->fd, (unsigned long long) paiocb->offset, res,
+                        strerror (op_errno));
+
+                goto out;
+        }
+
+        bd_inode_ctx_get (paiocb->fd->inode, this, &bdatt);
+        bd_update_amtime (&bdatt->iatt, GF_SET_ATTR_MTIME);
+        memcpy (&postbuf, &bdatt->iatt, sizeof (struct iatt));
+
+        op_ret = res;
+        op_errno = 0;
+
+out:
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, &prebuf, &postbuf,
+                             NULL);
+
+        if (paiocb) {
+                if (paiocb->iobref)
+                        iobref_unref (paiocb->iobref);
+                GF_FREE (paiocb);
+        }
+
+        return 0;
+}
+
+int
+bd_aio_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *iov, int count, off_t offset, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
+{
+        int32_t             op_errno   = EINVAL;
+        int                 _fd        = -1;
+        bd_fd_t            *bd_fd      = NULL;
+        int                 ret        = -1;
+        struct bd_aio_cb   *paiocb     = NULL;
+        bd_priv_t          *priv       = NULL;
+        struct iocb        *iocb       = NULL;
+        bd_attr_t          *bdatt      = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        priv = this->private;
+
+        ret = bd_fd_ctx_get (this, fd, &bd_fd);
+        if (ret < 0 || !bd_fd) {
+                STACK_WIND (frame, default_writev_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+                            fd, iov, count, offset, flags, iobref, xdata);
+                return 0;
+        }
+
+        bd_inode_ctx_get (fd->inode, this, &bdatt);
+
+        _fd = bd_fd->fd;
+
+        paiocb = CALLOC (1, sizeof (*paiocb));
+        if (!paiocb) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+
+        paiocb->frame = frame;
+        paiocb->offset = offset;
+        paiocb->op = GF_FOP_WRITE;
+        paiocb->fd = fd;
+
+        paiocb->iocb.data = paiocb;
+        paiocb->iocb.aio_fildes = _fd;
+        paiocb->iobref = iobref_ref (iobref);
+        paiocb->iocb.aio_lio_opcode = IO_CMD_PWRITEV;
+        paiocb->iocb.aio_reqprio = 0;
+        paiocb->iocb.u.v.vec = iov;
+        paiocb->iocb.u.v.nr = count;
+        paiocb->iocb.u.v.offset = offset;
+
+        iocb = &paiocb->iocb;
+
+        memcpy (&paiocb->prebuf, &bdatt->iatt, sizeof (struct iatt));
+        LOCK (&fd->lock);
+        {
+                __bd_fd_set_odirect (fd, bd_fd, flags, offset,
+                                     iov_length (iov, count));
+
+                ret = io_submit (priv->ctxp, 1, &iocb);
+        }
+        UNLOCK (&fd->lock);
+
+        if (ret != 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "io_submit() returned %d", ret);
+                op_errno = -ret;
+                goto err;
+        }
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, 0, 0, 0);
+
+        if (paiocb) {
+                if (paiocb->iobref)
+                        iobref_unref (paiocb->iobref);
+                GF_FREE (paiocb);
+        }
+
+        return 0;
+}
+
+void *
+bd_aio_thread (void *data)
+{
+        xlator_t           *this = NULL;
+        bd_priv_t          *priv = NULL;
+        int                 ret = 0;
+        int                 i = 0;
+        struct io_event    *event = NULL;
+        struct bd_aio_cb   *paiocb = NULL;
+        struct io_event     events[BD_AIO_MAX_NR_GETEVENTS];
+        struct timespec     ts = {0, };
+
+        this = data;
+        THIS = this;
+        priv = this->private;
+
+        ts.tv_sec = 5;
+        for (;;) {
+                memset (&events[0], 0, sizeof (events));
+                ret = io_getevents (priv->ctxp, 1, BD_AIO_MAX_NR_GETEVENTS,
+                                    &events[0], &ts);
+                if (ret < 0) {
+                        if (ret == -EINTR)
+                                continue;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "io_getevents() returned %d, exiting", ret);
+                        break;
+                }
+
+                for (i = 0; i < ret; i++) {
+                        event = &events[i];
+
+                        paiocb = event->data;
+
+                        switch (paiocb->op) {
+                        case GF_FOP_READ:
+                                bd_aio_readv_complete (paiocb, event->res,
+                                                          event->res2);
+                                break;
+                        case GF_FOP_WRITE:
+                                bd_aio_writev_complete (paiocb, event->res,
+                                                           event->res2);
+                                break;
+                        default:
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "unknown op %d found in piocb",
+                                        paiocb->op);
+                                break;
+                        }
+                }
+        }
+
+        return NULL;
+}
+
+int
+bd_aio_init (xlator_t *this)
+{
+        bd_priv_t *priv = NULL;
+        int        ret = 0;
+
+        priv = this->private;
+
+        ret = io_setup (BD_AIO_MAX_NR_EVENTS, &priv->ctxp);
+        if ((ret == -1 && errno == ENOSYS) || ret == -ENOSYS) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Linux AIO not available at run-time."
+                        " Continuing with synchronous IO");
+                ret = 0;
+                goto out;
+        }
+
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "io_setup() failed. ret=%d, errno=%d",
+                        ret, errno);
+                goto out;
+        }
+
+        ret = pthread_create (&priv->aiothread, NULL,
+                              bd_aio_thread, this);
+        if (ret != 0) {
+                io_destroy (priv->ctxp);
+                goto out;
+        }
+
+        this->fops->readv  = bd_aio_readv;
+        this->fops->writev = bd_aio_writev;
+out:
+        return ret;
+}
+
+
+int
+bd_aio_on (xlator_t *this)
+{
+        bd_priv_t *priv = NULL;
+        int        ret = 0;
+
+        priv = this->private;
+
+        if (!priv->aio_init_done) {
+                ret = bd_aio_init (this);
+                if (ret == 0)
+                        priv->aio_capable = _gf_true;
+                else
+                        priv->aio_capable = _gf_false;
+                priv->aio_init_done = _gf_true;
+        }
+
+        if (priv->aio_capable) {
+                this->fops->readv  = bd_aio_readv;
+                this->fops->writev = bd_aio_writev;
+        }
+
+        return ret;
+}
+
+int
+bd_aio_off (xlator_t *this)
+{
+        this->fops->readv  = bd_readv;
+        this->fops->writev = bd_writev;
+
+        return 0;
+}
+
+#else
+
+int
+bd_aio_on (xlator_t *this)
+{
+        gf_log (this->name, GF_LOG_INFO,
+                "Linux AIO not available at build-time."
+                " Continuing with synchronous IO");
+        return 0;
+}
+
+int
+bd_aio_off (xlator_t *this)
+{
+        gf_log (this->name, GF_LOG_INFO,
+                "Linux AIO not available at build-time."
+                " Continuing with synchronous IO");
+        return 0;
+}
+
+void
+__bd_fd_set_odirect (fd_t *fd, struct bd_fd *pfd, int opflags,
+                        off_t offset, size_t size)
+{
+        xlator_t        *this = THIS;
+        gf_log (this->name, GF_LOG_INFO,
+                "Linux AIO not available at build-time."
+                " Continuing with synchronous IO");
+        return;
+}
+#endif
diff --git a/xlators/storage/bd/src/bd-aio.h b/xlators/storage/bd/src/bd-aio.h
new file mode 100644
index 000000000..16f686a4c
--- /dev/null
+++ b/xlators/storage/bd/src/bd-aio.h
@@ -0,0 +1,41 @@
+/*
+   Copyright IBM, Corp. 2013
+
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _BD_AIO_H
+#define _BD_AIO_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "glusterfs.h"
+
+/*
+ * Maximum number of concurrently submitted IO events. The heaviest load
+ * GlusterFS has been able to handle had 60-80 concurrent calls
+ */
+#define BD_AIO_MAX_NR_EVENTS 256
+
+/* Maximum number of completed IO operations to reap per getevents syscall */
+#define BD_AIO_MAX_NR_GETEVENTS 16
+
+int bd_aio_on (xlator_t *this);
+int bd_aio_off (xlator_t *this);
+
+int bd_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                      off_t offset, uint32_t flags, dict_t *xdata);
+
+int bd_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       struct iovec *vector, int32_t count, off_t offset,
+                       uint32_t flags, struct iobref *iobref, dict_t *xdata);
+
+#endif /* !_BD_AIO_H */
diff --git a/xlators/storage/bd/src/bd-helper.c b/xlators/storage/bd/src/bd-helper.c
new file mode 100644
index 000000000..5525e346b
--- /dev/null
+++ b/xlators/storage/bd/src/bd-helper.c
@@ -0,0 +1,783 @@
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <lvm2app.h>
+#ifdef HAVE_LIBAIO
+#include <libaio.h>
+#endif
+
+#include "bd.h"
+#include "run.h"
+
+int
+bd_inode_ctx_set (inode_t *inode, xlator_t *this, bd_attr_t *ctx)
+{
+        int       ret  = -1;
+        uint64_t  ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+
+        ctx_int = (long)ctx;
+        ret = inode_ctx_set (inode, this, &ctx_int);
+out:
+        return ret;
+}
+
+int
+bd_inode_ctx_get (inode_t *inode, xlator_t *this, bd_attr_t **ctx)
+{
+        int       ret     = -1;
+        uint64_t  ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        ret = inode_ctx_get (inode, this, &ctx_int);
+        if (ret)
+                return ret;
+        if (ctx)
+                *ctx = (bd_attr_t *) ctx_int;
+out:
+        return ret;
+}
+
+void
+bd_local_free (xlator_t *this, bd_local_t *local)
+{
+        if (!local)
+                return;
+        if (local->fd)
+                fd_unref (local->fd);
+        else if (local->loc.path)
+                loc_wipe (&local->loc);
+        if (local->dict)
+                dict_unref (local->dict);
+        if (local->inode)
+                inode_unref (local->inode);
+        if (local->bdatt) {
+                GF_FREE (local->bdatt->type);
+                GF_FREE (local->bdatt);
+        }
+        mem_put (local);
+        local = NULL;
+}
+
+bd_local_t *
+bd_local_init (call_frame_t *frame, xlator_t *this)
+{
+        frame->local = mem_get0 (this->local_pool);
+        if (!frame->local)
+                return NULL;
+
+        return frame->local;
+}
+
+/*
+ * VG are set with the tag in GF_XATTR_VOL_ID_KEY:<uuid> format.
+ * This function validates this tag agains volume-uuid. Also goes
+ * through LV list to find out if a thin-pool is configured or not.
+ */
+int bd_scan_vg (xlator_t *this, bd_priv_t *priv)
+{
+        vg_t                   brick      = NULL;
+        data_t                *tmp_data   = NULL;
+        struct dm_list        *tags       = NULL;
+        int                    op_ret     = -1;
+        uuid_t                 dict_uuid  = {0, };
+        uuid_t                 vg_uuid    = {0, };
+        gf_boolean_t           uuid       = _gf_false;
+        lvm_str_list_t        *strl       = NULL;
+        struct dm_list        *lv_dm_list = NULL;
+        lv_list_t             *lv_list    = NULL;
+        struct dm_list        *dm_seglist = NULL;
+        lvseg_list_t          *seglist    = NULL;
+        lvm_property_value_t   prop       = {0, };
+        gf_boolean_t           thin       = _gf_false;
+        const char            *lv_name    = NULL;
+
+        brick = lvm_vg_open (priv->handle, priv->vg, "w", 0);
+        if (!brick) {
+                gf_log (this->name, GF_LOG_CRITICAL, "VG %s is not found",
+                        priv->vg);
+                return ENOENT;
+        }
+
+        lv_dm_list = lvm_vg_list_lvs (brick);
+        if (!lv_dm_list)
+                goto check;
+
+        dm_list_iterate_items (lv_list, lv_dm_list) {
+                dm_seglist = lvm_lv_list_lvsegs (lv_list->lv);
+                if (!dm_seglist)
+                        continue;
+                dm_list_iterate_items (seglist, dm_seglist) {
+                        prop = lvm_lvseg_get_property (seglist->lvseg,
+                                                       "segtype");
+                        if (!prop.is_valid || !prop.value.string)
+                                continue;
+                        if (!strcmp (prop.value.string, "thin-pool")) {
+                                thin = _gf_true;
+                                lv_name = lvm_lv_get_name (lv_list->lv);
+                                priv->pool = gf_strdup (lv_name);
+                                gf_log (THIS->name, GF_LOG_INFO, "Thin Pool "
+                                        "\"%s\" will be used for thin LVs",
+                                        lv_name);
+                                break;
+                        }
+                }
+        }
+
+check:
+        /* If there is no volume-id set in dict, we cant validate */
+        tmp_data = dict_get (this->options, "volume-id");
+        if (!tmp_data) {
+                op_ret = 0;
+                goto out;
+        }
+
+        op_ret = uuid_parse (tmp_data->data, dict_uuid);
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "wrong volume-id (%s) set in volume file",
+                        tmp_data->data);
+                op_ret = -1;
+                goto out;
+        }
+
+        tags = lvm_vg_get_tags (brick);
+        if (!tags) { /* no tags in the VG */
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Extended attribute trusted.glusterfs."
+                        "volume-id is absent");
+                op_ret = -1;
+                goto out;
+        }
+        dm_list_iterate_items (strl, tags) {
+                if (!strncmp (strl->str, GF_XATTR_VOL_ID_KEY,
+                              strlen (GF_XATTR_VOL_ID_KEY))) {
+                        uuid = _gf_true;
+                        break;
+                }
+        }
+        /* UUID tag is not set in VG */
+        if (!uuid) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Extended attribute trusted.glusterfs."
+                        "volume-id is absent");
+                op_ret = -1;
+                goto out;
+        }
+
+        op_ret = uuid_parse (strl->str + strlen (GF_XATTR_VOL_ID_KEY) + 1,
+                             vg_uuid);
+        if (op_ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "wrong volume-id (%s) set in VG", strl->str);
+                        op_ret = -1;
+                        goto out;
+        }
+        if (uuid_compare (dict_uuid, vg_uuid)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "mismatching volume-id (%s) received. "
+                        "already is a part of volume %s ",
+                        tmp_data->data, vg_uuid);
+                op_ret = -1;
+                goto out;
+        }
+
+        op_ret = 0;
+
+out:
+        lvm_vg_close (brick);
+
+        if (!thin)
+                gf_log (THIS->name, GF_LOG_WARNING, "No thin pool found in "
+                        "VG %s\n", priv->vg);
+        else
+                priv->caps |= BD_CAPS_THIN;
+
+        return op_ret;
+}
+
+/* FIXME: Move this code to common place, so posix and bd xlator can use */
+char *
+page_aligned_alloc (size_t size, char **aligned_buf)
+{
+        char    *alloc_buf = NULL;
+        char    *buf       = NULL;
+
+        alloc_buf = GF_CALLOC (1, (size + ALIGN_SIZE), gf_common_mt_char);
+        if (!alloc_buf)
+                return NULL;
+        /* page aligned buffer */
+        buf = GF_ALIGN_BUF (alloc_buf, ALIGN_SIZE);
+        *aligned_buf = buf;
+
+        return alloc_buf;
+}
+
+static int
+__bd_fd_ctx_get (xlator_t *this, fd_t *fd, bd_fd_t **bdfd_p)
+{
+        int         ret      = -1;
+        int         _fd      = -1;
+        char       *devpath  = NULL;
+        bd_fd_t    *bdfd     = NULL;
+        uint64_t    tmp_bdfd = 0;
+        bd_priv_t  *priv     = this->private;
+        bd_gfid_t   gfid     = {0, };
+        bd_attr_t  *bdatt    = NULL;
+
+        /* not bd file */
+        if (fd->inode->ia_type != IA_IFREG ||
+            bd_inode_ctx_get (fd->inode, this, &bdatt))
+                return 0;
+
+        ret = __fd_ctx_get (fd, this, &tmp_bdfd);
+        if (ret == 0) {
+                bdfd = (void *)(long) tmp_bdfd;
+                *bdfd_p = bdfd;
+                return 0;
+        }
+
+        uuid_utoa_r (fd->inode->gfid, gfid);
+        asprintf (&devpath, "/dev/%s/%s", priv->vg, gfid);
+        if (!devpath)
+                goto out;
+
+        _fd = open (devpath, O_RDWR | O_LARGEFILE, 0);
+        if (_fd < 0) {
+                ret = errno;
+                gf_log (this->name, GF_LOG_ERROR, "open on %s: %s", devpath,
+                        strerror (ret));
+                goto out;
+        }
+        bdfd = GF_CALLOC (1, sizeof(bd_fd_t), gf_bd_fd);
+        BD_VALIDATE_MEM_ALLOC (bdfd, ret, out);
+
+        bdfd->fd = _fd;
+        bdfd->flag = O_RDWR | O_LARGEFILE;
+        if (__fd_ctx_set (fd, this, (uint64_t)(long)bdfd) < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set the fd context fd=%p", fd);
+                goto out;
+        }
+
+        *bdfd_p = bdfd;
+
+        ret = 0;
+out:
+        FREE (devpath);
+        if (ret) {
+                close (_fd);
+                GF_FREE (bdfd);
+        }
+        return ret;
+}
+
+int
+bd_fd_ctx_get (xlator_t *this, fd_t *fd, bd_fd_t **bdfd)
+{
+        int   ret;
+
+        /* FIXME: Is it ok to fd->lock here ? */
+        LOCK (&fd->lock);
+        {
+                ret = __bd_fd_ctx_get (this, fd, bdfd);
+        }
+        UNLOCK (&fd->lock);
+
+        return ret;
+}
+
+/*
+ * Validates if LV exists for given inode or not.
+ * Returns 0 if LV exists and size also matches.
+ * If LV does not exist -1 returned
+ * If LV size mismatches, returnes 1 also lv_size is updated with actual
+ * size
+ */
+int
+bd_validate_bd_xattr (xlator_t *this, char *bd, char **type,
+                      uint64_t *lv_size, uuid_t uuid)
+{
+        char       *path  = NULL;
+        int         ret   = -1;
+        bd_gfid_t   gfid  = {0, };
+        bd_priv_t  *priv  = this->private;
+        struct stat stbuf = {0, };
+        uint64_t    size  = 0;
+        vg_t        vg    = NULL;
+        lv_t        lv    = NULL;
+        char     *bytes = NULL;
+
+        bytes = strrchr (bd, ':');
+        if (bytes) {
+                *bytes = '\0';
+                bytes++;
+                gf_string2bytesize (bytes, &size);
+        }
+
+        if (strcmp (bd, BD_LV) && strcmp (bd, BD_THIN)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "invalid xattr %s", bd);
+                return -1;
+        }
+        *type = gf_strdup (bd);
+
+        /*
+         * Check if LV really exist, there could be a failure
+         * after setxattr and successful LV creation
+         */
+        uuid_utoa_r (uuid, gfid);
+        gf_asprintf (&path, "/dev/%s/%s", priv->vg, gfid);
+        if (!path) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "insufficient memory");
+                return 0;
+        }
+
+        /* Destination file does not exist */
+        if (stat (path, &stbuf)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "lstat failed for path %s", path);
+                return -1;
+        }
+
+        vg = lvm_vg_open (priv->handle, priv->vg, "r", 0);
+        if (!vg) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "VG %s does not exist?", priv->vg);
+                ret = -1;
+                goto out;
+        }
+
+        lv = lvm_lv_from_name (vg, gfid);
+        if (!lv) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "LV %s does not exist", gfid);
+                ret = -1;
+                goto out;
+        }
+
+        *lv_size = lvm_lv_get_size (lv);
+        if (size == *lv_size) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = 1;
+
+out:
+        if (vg)
+                lvm_vg_close (vg);
+
+        GF_FREE (path);
+        return ret;
+}
+
+static int
+create_thin_lv (char *vg, char *pool, char *lv, uint64_t extent)
+{
+        int         ret    = -1;
+        runner_t    runner = {0, };
+        char       *path   = NULL;
+        struct stat stat   = {0, };
+
+        runinit (&runner);
+        runner_add_args  (&runner, LVM_CREATE, NULL);
+        runner_add_args  (&runner, "--thin", NULL);
+        runner_argprintf (&runner, "%s/%s", vg, pool);
+        runner_add_args  (&runner, "--name", NULL);
+        runner_argprintf (&runner, "%s", lv);
+        runner_add_args  (&runner, "--virtualsize", NULL);
+        runner_argprintf (&runner, "%ldB", extent);
+        runner_start (&runner);
+        runner_end (&runner);
+
+        gf_asprintf (&path, "/dev/%s/%s", vg, lv);
+        if (!path) {
+                ret = ENOMEM;
+                goto out;
+        }
+        if (lstat (path, &stat) < 0)
+                ret = EAGAIN;
+        else
+                ret = 0;
+out:
+        GF_FREE (path);
+        return ret;
+}
+
+int
+bd_create (uuid_t uuid, uint64_t size, char *type, bd_priv_t *priv)
+{
+        int       ret  = 0;
+        vg_t      vg   = NULL;
+        bd_gfid_t gfid = {0, };
+
+        uuid_utoa_r (uuid, gfid);
+
+        if (!strcmp (type, BD_THIN))
+                return create_thin_lv (priv->vg, priv->pool, gfid,
+                                       size);
+
+        vg = lvm_vg_open (priv->handle, priv->vg,  "w", 0);
+        if (!vg) {
+                gf_log (THIS->name, GF_LOG_WARNING, "opening VG %s failed",
+                        priv->vg);
+                return ENOENT;
+        }
+
+        if (!lvm_vg_create_lv_linear (vg, gfid, size)) {
+                gf_log (THIS->name, GF_LOG_WARNING, "lvm_vg_create_lv_linear "
+                        "failed");
+                ret = errno;
+        }
+
+        lvm_vg_close (vg);
+
+        return ret;
+}
+
+int32_t
+bd_resize (bd_priv_t *priv, uuid_t uuid, off_t size)
+{
+        uint64_t        new_size  = 0;
+        runner_t        runner    = {0, };
+        bd_gfid_t       gfid      = {0, };
+        int             ret       = 0;
+        vg_t            vg        = NULL;
+        lv_t            lv        = NULL;
+
+        uuid_utoa_r (uuid, gfid);
+
+        runinit (&runner);
+
+        runner_add_args  (&runner, LVM_RESIZE, NULL);
+        runner_argprintf (&runner, "%s/%s", priv->vg, gfid);
+        runner_argprintf (&runner, "-L%ldb", size);
+        runner_add_args  (&runner, "-f", NULL);
+
+        runner_start (&runner);
+        runner_end (&runner);
+
+        vg = lvm_vg_open (priv->handle, priv->vg, "w", 0);
+        if (!vg) {
+                gf_log (THIS->name, GF_LOG_WARNING, "opening VG %s failed",
+                        priv->vg);
+                return EAGAIN;
+        }
+
+        lv = lvm_lv_from_name (vg, gfid);
+        if (!lv) {
+                gf_log (THIS->name, GF_LOG_WARNING, "LV %s not found", gfid);
+                ret = EIO;
+                goto out;
+        }
+        new_size = lvm_lv_get_size (lv);
+
+        if (new_size != size) {
+                gf_log (THIS->name, GF_LOG_WARNING, "resized LV size %ld does "
+                        "not match requested size %ld", new_size, size);
+                ret = EIO;
+        }
+
+out:
+        lvm_vg_close (vg);
+        return ret;
+}
+
+uint64_t
+bd_get_default_extent (bd_priv_t *priv)
+{
+        vg_t   vg = NULL;
+        uint64_t size = 0;
+
+        vg = lvm_vg_open (priv->handle, priv->vg,  "w", 0);
+        if (!vg) {
+                gf_log (THIS->name, GF_LOG_WARNING, "opening VG %s failed",
+                        priv->vg);
+                return 0;
+        }
+
+        size = lvm_vg_get_extent_size (vg);
+
+        lvm_vg_close (vg);
+
+        return size;
+}
+
+/*
+ * Adjusts the user specified size to VG specific extent size
+ */
+uint64_t
+bd_adjust_size (bd_priv_t *priv, uint64_t size)
+{
+        uint64_t extent = 0;
+        uint64_t nr_ex  = 0;
+
+        extent = bd_get_default_extent (priv);
+        if (!extent)
+                return 0;
+
+        nr_ex = size / extent;
+        if (size % extent)
+                nr_ex++;
+
+        size = extent * nr_ex;
+
+        return size;
+}
+
+int
+bd_delete_lv (bd_priv_t *priv, const char *lv_name, int *op_errno)
+{
+        vg_t    vg  = NULL;
+        lv_t    lv  = NULL;
+        int     ret = -1;
+
+        *op_errno = 0;
+        vg = lvm_vg_open (priv->handle, priv->vg, "w", 0);
+        if (!vg) {
+                gf_log (THIS->name, GF_LOG_WARNING, "opening VG %s failed",
+                        priv->vg);
+                *op_errno = ENOENT;
+                return -1;
+        }
+        lv = lvm_lv_from_name (vg, lv_name);
+        if (!lv) {
+                gf_log (THIS->name, GF_LOG_WARNING, "No such LV %s", lv_name);
+                *op_errno = ENOENT;
+                goto out;
+        }
+        ret = lvm_vg_remove_lv (lv);
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_WARNING, "removing LV %s failed",
+                        lv_name);
+                *op_errno = errno;
+                goto out;
+        }
+out:
+        lvm_vg_close (vg);
+
+        return ret;
+}
+
+inline void
+bd_update_amtime(struct iatt *iatt, int flag)
+{
+        struct timespec ts = {0, };
+
+        clock_gettime (CLOCK_REALTIME, &ts);
+        if (flag & GF_SET_ATTR_ATIME) {
+                iatt->ia_atime = ts.tv_sec;
+                iatt->ia_atime_nsec = ts.tv_nsec;
+        }
+        if (flag & GF_SET_ATTR_MTIME) {
+                iatt->ia_mtime = ts.tv_sec;
+                iatt->ia_mtime_nsec = ts.tv_nsec;
+        }
+}
+
+int
+bd_snapshot_create (bd_local_t *local, bd_priv_t *priv)
+{
+        char       *path   = NULL;
+        bd_gfid_t   dest   = {0, };
+        bd_gfid_t   origin = {0, };
+        int         ret    = 0;
+        runner_t    runner = {0, };
+        struct stat stat   = {0, };
+
+        uuid_utoa_r (local->dloc->gfid, dest);
+        uuid_utoa_r (local->loc.gfid, origin);
+
+        gf_asprintf (&path, "/dev/%s/%s", priv->vg, dest);
+        if (!path) {
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "Insufficient memory");
+                return ENOMEM;
+        }
+
+        runinit (&runner);
+        runner_add_args  (&runner, LVM_CREATE, NULL);
+        runner_add_args  (&runner, "--snapshot", NULL);
+        runner_argprintf (&runner, "/dev/%s/%s", priv->vg, origin);
+        runner_add_args  (&runner, "--name", NULL);
+        runner_argprintf (&runner, "%s", dest);
+        if (strcmp (local->bdatt->type, BD_THIN))
+                runner_argprintf (&runner, "-L%ldB", local->size);
+        runner_start (&runner);
+        runner_end (&runner);
+
+        if (lstat (path, &stat) < 0)
+                ret = EIO;
+
+        GF_FREE (path);
+        return ret;
+}
+
+int
+bd_clone (bd_local_t *local, bd_priv_t *priv)
+{
+        int           ret          = ENOMEM;
+        int           fd1          = -1;
+        int           fd2          = -1;
+        int           i            = 0;
+        char         *buff         = NULL;
+        ssize_t       bytes        = 0;
+        char         *spath        = NULL;
+        char         *dpath        = NULL;
+        struct iovec *vec          = NULL;
+        bd_gfid_t     source       = {0, };
+        bd_gfid_t     dest         = {0, };
+        void         *bufp[IOV_NR] = {0, };
+
+        vec = GF_CALLOC (IOV_NR, sizeof (struct iovec), gf_common_mt_iovec);
+        if (!vec)
+                return ENOMEM;
+
+        for (i = 0; i < IOV_NR; i++) {
+                bufp[i] = page_aligned_alloc (IOV_SIZE, &buff);
+                if (!buff)
+                        goto out;
+                vec[i].iov_base = buff;
+                vec[i].iov_len = IOV_SIZE;
+        }
+
+        uuid_utoa_r (local->loc.gfid, source);
+        uuid_utoa_r (local->dloc->gfid, dest);
+
+        gf_asprintf (&spath, "/dev/%s/%s", priv->vg, source);
+        gf_asprintf (&dpath, "/dev/%s/%s", priv->vg, dest);
+        if (!spath || !dpath)
+                goto out;
+
+        ret = bd_create (local->dloc->gfid, local->size,
+                         local->bdatt->type,  priv);
+        if (ret)
+                goto out;
+
+        fd1 = open (spath, O_RDONLY | O_DIRECT);
+        if (fd1 < 0) {
+                ret = errno;
+                goto out;
+        }
+        fd2 = open (dpath, O_WRONLY | O_DIRECT);
+        if (fd2 < 0) {
+                ret = errno;
+                goto out;
+        }
+
+        while (1) {
+                bytes = readv (fd1, vec, IOV_NR);
+                if (bytes < 0) {
+                        ret = errno;
+                        gf_log (THIS->name, GF_LOG_WARNING, "read failed: %s",
+                                strerror (ret));
+                        goto out;
+                }
+                if (!bytes)
+                        break;
+                bytes = writev (fd2, vec, IOV_NR);
+                if (bytes < 0) {
+                        ret = errno;
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "write failed: %s", strerror (ret));
+                        goto out;
+                }
+        }
+        ret = 0;
+
+out:
+        for (i = 0; i < IOV_NR; i++)
+                GF_FREE (bufp[i]);
+        GF_FREE (vec);
+
+        if (fd1 != -1)
+                close (fd1);
+        if (fd2 != -1)
+                close (fd2);
+
+        FREE (spath);
+        FREE (dpath);
+
+        return ret;
+}
+
+/*
+ * Merges snapshot LV to origin LV and returns status
+ */
+int
+bd_merge (bd_priv_t *priv, uuid_t gfid)
+{
+        bd_gfid_t   dest   = {0, };
+        char       *path   = NULL;
+        struct stat stat   = {0, };
+        runner_t    runner = {0, };
+        int         ret    = 0;
+
+        uuid_utoa_r (gfid, dest);
+        gf_asprintf (&path, "/dev/%s/%s", priv->vg, dest);
+
+        runinit (&runner);
+        runner_add_args (&runner, LVM_CONVERT, NULL);
+        runner_add_args (&runner, "--merge", NULL);
+        runner_argprintf (&runner, "%s", path);
+        runner_start (&runner);
+        runner_end (&runner);
+
+        if (!lstat (path, &stat))
+                ret = EIO;
+
+        GF_FREE (path);
+
+        return ret;
+}
+
+int
+bd_get_origin (bd_priv_t *priv, loc_t *loc, fd_t *fd, dict_t *dict)
+{
+        vg_t                      brick      = NULL;
+        lvm_property_value_t      prop       = {0, };
+        lv_t                      lv         = NULL;
+        int                       ret        = -1;
+        bd_gfid_t                 gfid       = {0, };
+        inode_t                  *inode      = NULL;
+        char                     *origin     = NULL;
+
+        brick = lvm_vg_open (priv->handle, priv->vg, "w", 0);
+        if (!brick) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "VG %s is not found",
+                        priv->vg);
+                return ENOENT;
+        }
+
+        if (fd)
+                inode = fd->inode;
+        else
+                inode = loc->inode;
+
+        uuid_utoa_r (inode->gfid, gfid);
+        lv = lvm_lv_from_name (brick, gfid);
+        if (!lv) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "LV %s not found", gfid);
+                ret = ENOENT;
+                goto out;
+        }
+
+        prop = lvm_lv_get_property (lv, "origin");
+        if (!prop.is_valid || !prop.value.string) {
+                ret = ENODATA;
+                goto out;
+        }
+
+        origin = gf_strdup (prop.value.string);
+        ret = dict_set_dynstr (dict, BD_ORIGIN, origin);
+
+out:
+        lvm_vg_close (brick);
+        return ret;
+}
+
diff --git a/xlators/storage/bd/src/bd.c b/xlators/storage/bd/src/bd.c
new file mode 100644
index 000000000..405474c58
--- /dev/null
+++ b/xlators/storage/bd/src/bd.c
@@ -0,0 +1,2404 @@
+/*
+  BD translator V2 - Exports Block devices on server side as regular
+  files to client
+
+  Now only exporting Logical volumes supported.
+
+  Copyright IBM, Corp. 2013
+
+  This file is part of GlusterFS.
+
+  Author:
+  M. Mohan Kumar <mohan@in.ibm.com>
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <lvm2app.h>
+#include <openssl/md5.h>
+#include <time.h>
+#include <linux/fs.h>
+#include <sys/ioctl.h>
+#ifdef HAVE_LIBAIO
+#include <libaio.h>
+#endif
+
+#include "bd.h"
+#include "bd-aio.h"
+#include "defaults.h"
+#include "glusterfs3-xdr.h"
+#include "run.h"
+#include "protocol-common.h"
+#include "checksum.h"
+
+/*
+ * Call back function for setxattr and removexattr.
+ * does not do anything. FIXME: How to handle remove/setxattr failure
+ */
+int
+bd_null_rmsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *xdata)
+{
+        STACK_DESTROY (frame->root);
+        return 0;
+}
+
+/*
+ * returns 0 if a file is mapped to BD or not.
+ */
+int
+bd_get_bd_info (call_frame_t *frame, xlator_t *this, dict_t *xattr, uuid_t gfid,
+                char **type, uint64_t *size)
+{
+        char         *bd_xattr = NULL;
+        char         *bd       = NULL;
+        int           ret      = -1;
+        loc_t         loc      = {0, };
+        dict_t       *dict     = NULL;
+        char         *p        = NULL;
+        call_frame_t *bd_frame = NULL;
+
+        if (!xattr)
+                return 1;
+
+        if (dict_get_str (xattr, BD_XATTR, &p))
+                return 1;
+
+        bd_xattr = gf_strdup (p);
+
+        memcpy (loc.gfid, gfid, sizeof (uuid_t));
+
+        bd_frame = copy_frame (frame);
+        BD_VALIDATE_MEM_ALLOC (bd_frame, ret, out);
+
+        ret = bd_validate_bd_xattr (this,  bd_xattr, type, size, gfid);
+        if (ret < 0) {/* LV does not exist */
+                STACK_WIND (bd_frame, bd_null_rmsetxattr_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->removexattr, &loc,
+                            BD_XATTR, NULL);
+
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Mapped LV not available for posix file <gfid:%s>, "
+                        "deleting mapping", uuid_utoa (gfid));
+        } else if (ret == 1) {
+                /* BD_XATTR size and LV size mismatch. Update BD_XATTR */
+                gf_asprintf (&bd, "%s:%ld", *type, *size);
+
+                dict = dict_new ();
+                BD_VALIDATE_MEM_ALLOC (dict, ret, out);
+
+                ret = dict_set_dynstr (dict, BD_XATTR, bd);
+                if (ret)
+                        goto out;
+
+                STACK_WIND (bd_frame, bd_null_rmsetxattr_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->setxattr, &loc, dict, 0,
+                            NULL);
+        }
+
+out:
+        dict_del (xattr, BD_XATTR);
+        GF_FREE (bd_xattr);
+        GF_FREE (bd);
+        return ret;
+}
+
+/*
+ * bd_lookup_cbk: Call back from posix_lookup.
+ */
+int32_t
+bd_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *buf, dict_t *xattr,
+               struct iatt *postparent)
+{
+        int           ret    = -1;
+        bd_attr_t    *bdatt  = NULL;
+        uint64_t      size   = 0;
+        char         *type   = BD_TYPE_NONE;
+
+        /* only regular files are part of BD object */
+        if (op_ret < 0 || buf->ia_type != IA_IFREG)
+                goto out;
+
+        /* iatt already cached */
+        if (!bd_inode_ctx_get (inode, this, &bdatt))
+                goto next;
+
+        if (bd_get_bd_info (frame, this, xattr, buf->ia_gfid, &type, &size))
+                goto out;
+
+        /* BD file, update buf */
+        bdatt = GF_CALLOC (1, sizeof (bd_attr_t), gf_bd_attr);
+        if (!bdatt) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+        memcpy (&bdatt->iatt, buf, sizeof (struct iatt));
+        bdatt->type = type;
+
+        /* Cache LV size in inode_ctx */
+        ret = bd_inode_ctx_set (inode, this, bdatt);
+        if (ret < 0) {
+                GF_FREE (bdatt);
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        bdatt->iatt.ia_size = size;
+        bdatt->iatt.ia_blocks = size / 512;
+
+next:
+        dict_del (xattr, GF_CONTENT_KEY);
+        memcpy (buf, &bdatt->iatt, sizeof (struct iatt));
+
+out:
+        BD_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
+                         xattr, postparent);
+        return 0;
+}
+
+/*
+ * bd_lookup: Issues posix_lookup to find out if file is mapped to BD
+ * bd_lookup -> posix_lookup -> bd_lookup_cbk
+*/
+int32_t
+bd_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+        dict_t     *bd_xattr = NULL;
+        bd_attr_t  *bdatt    = NULL;
+        int         op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (loc, out);
+        VALIDATE_OR_GOTO (loc->path, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        if (bd_inode_ctx_get (loc->inode, this, &bdatt) < 0) {
+                if (!xattr_req) {
+                        bd_xattr = dict_new ();
+                        BD_VALIDATE_MEM_ALLOC (bd_xattr, op_errno, out);
+                        xattr_req = bd_xattr;
+                }
+                if (dict_set_int8 (xattr_req, BD_XATTR, 1) < 0)
+                        goto out;
+        }
+
+        STACK_WIND (frame, bd_lookup_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->lookup, loc, xattr_req);
+
+        if (bd_xattr)
+                dict_unref (bd_xattr);
+        return 0;
+out:
+        BD_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+bd_forget (xlator_t *this, inode_t *inode)
+{
+        int          ret   = -1;
+        uint64_t     ctx   = 0;
+        bd_attr_t   *bdatt = NULL;
+
+        ret = bd_inode_ctx_get (inode, this, &bdatt);
+        if (!ret) {
+                inode_ctx_del (inode, this, &ctx);
+                FREE (bdatt);
+        }
+        return 0;
+}
+
+int
+bd_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+        gf_dirent_t    *entry  = NULL;
+        uint64_t        size   = 0;
+        char           *type   = NULL;
+
+        if (op_ret < 0)
+                goto out;
+
+        list_for_each_entry (entry, &entries->list, list) {
+                if (entry->d_type != DT_REG)
+                        continue;
+                if (!bd_get_bd_info (frame, this, entry->dict,
+                                     entry->d_stat.ia_gfid, &type, &size)) {
+                        entry->d_stat.ia_size = size;
+                        entry->d_stat.ia_blocks = size / 512;
+                        FREE (type);
+                }
+        }
+
+out:
+        BD_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, xdata);
+        return 0;
+}
+
+/*
+ * bd_readdirp: In bd_readdirp_cbk if the file and BD_XATTR_SIZE is set
+ * ia_size is updated with the LV(BD_XATTR_SIZE) size
+ */
+int32_t
+bd_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t off, dict_t *dict)
+{
+        int          op_errno = EINVAL;
+        bd_local_t  *local    = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (fd, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        if (!dict) {
+                local = bd_local_init (frame, this);
+                BD_VALIDATE_MEM_ALLOC (local, op_errno, out);
+                local->dict = dict_new ();
+                BD_VALIDATE_MEM_ALLOC (local->dict, op_errno, out);
+                dict = local->dict;
+        }
+
+        if (dict_set_int8 (dict, BD_XATTR, 0)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set key %s", BD_XATTR);
+                goto out;
+        }
+
+        STACK_WIND (frame, bd_readdirp_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdirp, fd, size, off, dict);
+
+        return 0;
+out:
+        BD_STACK_UNWIND (readdirp, frame, -1, op_errno, NULL, dict);
+        return 0;
+}
+
+int
+bd_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, struct iatt *buf, dict_t *xdata)
+{
+        bd_local_t  *local = frame->local;
+        bd_attr_t   *bdatt = NULL;
+
+        /* only regular files are part of BD object */
+        if (op_ret < 0 || buf->ia_type != IA_IFREG)
+                goto out;
+
+        BD_VALIDATE_LOCAL_OR_GOTO (local, op_errno, out);
+
+        /* update buf with LV size */
+        if (!bd_inode_ctx_get (local->inode, this, &bdatt))
+                memcpy (buf, bdatt, sizeof (struct iatt));
+
+out:
+        BD_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int
+bd_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        int          op_errno = EINVAL;
+        bd_local_t  *local    = NULL;
+        bd_attr_t   *bdatt    = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (loc, out);
+        VALIDATE_OR_GOTO (loc->path, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        if (!bd_inode_ctx_get (loc->inode, this, &bdatt)) {
+                BD_STACK_UNWIND (stat, frame, 0, 0, &bdatt->iatt, xdata);
+                return 0;
+        }
+
+        local = bd_local_init (frame, this);
+        BD_VALIDATE_MEM_ALLOC (local, op_errno, out);
+        local->inode = inode_ref (loc->inode);
+
+        STACK_WIND(frame, bd_stat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+out:
+        BD_STACK_UNWIND (stat, frame, -1, op_errno, NULL, xdata);
+        return 0;
+}
+
+int
+bd_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct statvfs *buff, dict_t *xdata)
+{
+        uint64_t      size    = 0;
+        uint64_t      fr_size = 0;
+        bd_priv_t    *priv    = NULL;
+        vg_t          vg      = NULL;
+
+        if (op_ret < 0)
+                goto out;
+
+        priv = this->private;
+
+        vg = lvm_vg_open (priv->handle, priv->vg, "r", 0);
+        if (!vg) {
+                gf_log (this->name, GF_LOG_WARNING, "opening VG %s failed",
+                        priv->vg);
+                op_ret = -1;
+                op_errno = EAGAIN;
+                goto out;
+        }
+        size = lvm_vg_get_size (vg);
+        fr_size = lvm_vg_get_free_size (vg);
+        lvm_vg_close (vg);
+
+        buff->f_blocks += size / buff->f_frsize;
+        buff->f_bfree += fr_size / buff->f_frsize;
+        buff->f_bavail += fr_size / buff->f_frsize;
+
+out:
+        BD_STACK_UNWIND (statfs, frame, op_ret, op_errno, buff, xdata);
+        return 0;
+}
+
+/*
+ * bd_statfs: Mimics statfs by returning used/free extents in the VG
+ */
+int
+bd_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+        VALIDATE_OR_GOTO (loc, out);
+
+        STACK_WIND (frame, bd_statfs_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->statfs, loc, xdata);
+        return 0;
+out:
+        BD_STACK_UNWIND (statfs, frame, -1, EINVAL, NULL, NULL);
+        return 0;
+}
+
+int
+bd_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *buf, dict_t *xdata)
+{
+        bd_attr_t  *bdatt = NULL;
+        bd_local_t *local = frame->local;
+
+        /* only regular files are part of BD object */
+        if (op_ret < 0 || buf->ia_type != IA_IFREG)
+                goto out;
+
+        BD_VALIDATE_LOCAL_OR_GOTO (local, op_errno, out);
+
+        /* update buf with LV size */
+        if (!bd_inode_ctx_get (local->inode, this, &bdatt))
+                memcpy (buf, &bdatt->iatt, sizeof (struct iatt));
+
+out:
+        BD_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int
+bd_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        int          op_errno = EINVAL;
+        bd_local_t  *local    = NULL;
+        bd_attr_t   *bdatt    = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (fd, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        /* if its already cached return it */
+        if (!bd_inode_ctx_get (fd->inode, this, &bdatt)) {
+                BD_STACK_UNWIND (fstat, frame, 0, 0, &bdatt->iatt, xdata);
+                return 0;
+        }
+
+        local = bd_local_init (frame, this);
+        BD_VALIDATE_MEM_ALLOC (local, op_errno, out);
+
+        local->inode = inode_ref (fd->inode);
+
+        STACK_WIND (frame, bd_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+
+        return 0;
+out:
+        BD_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, xdata);
+        return 0;
+}
+
+/*
+ * bd_readv: If posix file, invokes posix_readv otherwise reads from the BD
+ * file
+ */
+int
+bd_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
+{
+        int             ret        = -1;
+        int             _fd        = -1;
+        int32_t         op_ret     = -1;
+        int32_t         op_errno   = 0;
+        bd_fd_t        *bd_fd      = NULL;
+        struct iovec    vec        = {0, };
+        struct iobuf   *iobuf      = NULL;
+        struct iobref  *iobref     = NULL;
+        uint64_t        bd_size    = 0;
+        bd_attr_t      *bdatt      = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (fd, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        ret = bd_fd_ctx_get (this, fd, &bd_fd);
+        if (ret < 0 || !bd_fd) {
+                STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->readv,
+                            fd, size, offset, flags, xdata);
+                return 0;
+        }
+        if (!size) {
+                op_errno = EINVAL;
+                gf_log (this->name, GF_LOG_WARNING, "size=%"GF_PRI_SIZET, size);
+                goto out;
+        }
+        iobuf = iobuf_get2 (this->ctx->iobuf_pool, size);
+        if (!iobuf) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+        _fd = bd_fd->fd;
+        op_ret = pread (_fd, iobuf->ptr, size, offset);
+        if (op_ret == -1) {
+                op_errno = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                                "read failed on fd=%p: %s", fd,
+                                strerror (op_errno));
+                goto out;
+        }
+
+        vec.iov_base = iobuf->ptr;
+        vec.iov_len = op_ret;
+
+        iobref = iobref_new ();
+        iobref_add (iobref, iobuf);
+
+        if (bd_inode_ctx_get (fd->inode, this, &bdatt)) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto out;
+        }
+        bd_size = bdatt->iatt.ia_size;
+        if (!bd_size || (offset + vec.iov_len) >= bd_size)
+                op_errno = ENOENT;
+
+        op_ret = vec.iov_len;
+        bd_update_amtime (&bdatt->iatt, GF_SET_ATTR_ATIME);
+
+out:
+        BD_STACK_UNWIND (readv, frame, op_ret, op_errno,
+                         &vec, 1, &bdatt->iatt, iobref, NULL);
+
+        if (iobref)
+                iobref_unref (iobref);
+        if (iobuf)
+                iobuf_unref (iobuf);
+
+        return 0;
+}
+
+#ifdef BLKDISCARD
+/*
+ * bd_discard: Sends BLKDISCARD ioctl to the block device
+ */
+int
+bd_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+        int           ret      = -1;
+        int           op_errno = EINVAL;
+        bd_fd_t      *bd_fd    = NULL;
+        uint64_t      param[2] = {0, };
+        bd_attr_t    *bdatt    = NULL;
+        struct iatt   prebuf   = {0, };
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+        VALIDATE_OR_GOTO (fd, out);
+
+        /* posix */
+        if (bd_inode_ctx_get (fd->inode, this, &bdatt)) {
+                STACK_WIND (frame, default_discard_cbk, FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->discard,
+                           fd, offset, len, xdata);
+                return 0;
+        }
+
+        ret = bd_fd_ctx_get (this, fd, &bd_fd);
+        if (ret < 0 || !bd_fd) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        param[0] = offset;
+        param[1] = len;
+        ret = ioctl (bd_fd->fd, BLKDISCARD, param);
+        if (ret < 0) {
+                if (errno == ENOTTY)
+                        op_errno = ENOSYS;
+                else
+                        op_errno = errno;
+                goto out;
+        }
+        memcpy (&prebuf, &bdatt->iatt, sizeof (prebuf));
+        bd_update_amtime (&bdatt->iatt, GF_SET_ATTR_MTIME);
+
+        BD_STACK_UNWIND (discard, frame, ret, op_errno, &prebuf,
+                         &bdatt->iatt, xdata);
+        return 0;
+
+out:
+        BD_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+#else
+
+int
+bd_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+        BD_STACK_UNWIND (discard, frame, -1, ENOSYS, NULL, NULL, NULL);
+        return 0;
+}
+#endif
+
+/*
+ * Call back from posix_open for opening the backing posix file
+ * If it failed, close BD fd
+ */
+int
+bd_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+             int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+        bd_fd_t    *bd_fd = NULL;
+        bd_attr_t  *bdatt = NULL;
+
+        if (!op_ret)
+                goto out;
+
+        bd_inode_ctx_get (fd->inode, this, &bdatt);
+        if (!bdatt) /* posix file */
+                goto out;
+
+        /* posix open failed */
+        if (bd_fd_ctx_get (this, fd, &bd_fd) < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "bd_fd is NULL from fd=%p", fd);
+                goto out;
+        }
+        close (bd_fd->fd);
+        GF_FREE (bd_fd);
+
+out:
+        BD_STACK_UNWIND (open, frame, op_ret, op_errno, fd, NULL);
+
+        return 0;
+}
+
+/*
+ * bd_open: Opens BD file if given posix file is mapped to BD. Also opens
+ * posix file.
+ * fd contains both posix and BD fd
+ */
+int32_t
+bd_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
+{
+        int32_t      ret     = EINVAL;
+        bd_fd_t     *bd_fd   = NULL;
+        bd_attr_t   *bdatt   = NULL;
+        bd_gfid_t    gfid    = {0, };
+        char        *devpath = NULL;
+        bd_priv_t   *priv    = this->private;
+        int         _fd      = -1;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+        VALIDATE_OR_GOTO (loc, out);
+        VALIDATE_OR_GOTO (fd, out);
+
+        /* not bd file */
+        if (fd->inode->ia_type != IA_IFREG ||
+            bd_inode_ctx_get (fd->inode, this, &bdatt))
+                goto posix;
+
+        uuid_utoa_r (fd->inode->gfid, gfid);
+        asprintf (&devpath, "/dev/%s/%s", priv->vg, gfid);
+        BD_VALIDATE_MEM_ALLOC (devpath, ret, out);
+
+        _fd = open (devpath, flags | O_LARGEFILE, 0);
+        if (_fd < 0) {
+                ret = errno;
+                gf_log (this->name, GF_LOG_ERROR, "open on %s: %s", devpath,
+                        strerror (ret));
+                goto out;
+        }
+        bd_fd = GF_CALLOC (1, sizeof(bd_fd_t), gf_bd_fd);
+        BD_VALIDATE_MEM_ALLOC (bd_fd, ret, out);
+
+        bd_fd->fd = _fd;
+        bd_fd->flag = flags | O_LARGEFILE;
+
+        if (fd_ctx_set (fd, this, (uint64_t)(long)bd_fd) < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set the fd context fd=%p", fd);
+                goto out;
+        }
+
+        ret = 0;
+
+posix:
+
+        /* open posix equivalant of this file, fd needed for fd related
+           operations like fsetxattr, ftruncate etc */
+        STACK_WIND (frame, bd_open_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+
+        return 0;
+out:
+        BD_STACK_UNWIND (open, frame, -1, ret, fd, NULL);
+
+        FREE (devpath);
+        if (ret) {
+                close (_fd);
+                GF_FREE (bd_fd);
+        }
+
+        return 0;
+}
+
+/*
+ * call back from posix_setattr after updating iatt to posix file.
+ */
+int
+bd_fsync_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iatt *pre,
+                      struct iatt *post, dict_t *xdata)
+{
+        bd_local_t *local = frame->local;
+        bd_attr_t  *bdatt = local->bdatt;
+
+        BD_STACK_UNWIND (fsync, frame, op_ret, op_errno, &bdatt->iatt,
+                         &bdatt->iatt, NULL);
+        return 0;
+}
+
+int
+bd_do_fsync (int fd, int datasync)
+{
+        int   op_errno = 0;
+
+#ifdef HAVE_FDATASYNC
+        if (datasync) {
+                if (fdatasync (fd)) {
+                        op_errno = errno;
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "fdatasync on fd=%d failed: %s",
+                                fd, strerror (errno));
+                }
+
+        } else
+#endif
+        {
+                if (fsync (fd)) {
+                        op_errno = errno;
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "fsync on fd=%d failed: %s",
+                                 fd, strerror (op_errno));
+                }
+        }
+
+        return op_errno;
+}
+
+/*
+ * bd_fsync: Syncs if BD fd, forwards the request to posix
+ * fsync -> posix_setattr -> posix_fsync
+*/
+int32_t
+bd_fsync (call_frame_t *frame, xlator_t *this,
+          fd_t *fd, int32_t datasync, dict_t *xdata)
+{
+        int         ret      = -1;
+        int32_t     op_ret   = -1;
+        int32_t     op_errno = 0;
+        bd_fd_t    *bd_fd    = NULL;
+        bd_priv_t  *priv     = NULL;
+        bd_attr_t  *bdatt    = NULL;
+        bd_local_t *local    = NULL;
+        int         valid    = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
+        struct iatt prebuf   = {0, };
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (fd, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        ret = bd_inode_ctx_get (fd->inode, this, &bdatt);
+        ret = bd_fd_ctx_get (this, fd, &bd_fd);
+        if (ret < 0 || !bd_fd || !bdatt) {
+                STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->fsync, fd, datasync,
+                            xdata);
+                return 0;
+        }
+
+        memcpy (&prebuf, &bdatt->iatt, sizeof (struct iatt));
+
+        op_errno = bd_do_fsync (bd_fd->fd, datasync);
+        if (op_errno)
+                goto out;
+
+        /* For BD, Update the a|mtime during full fsync only */
+        if (!datasync) {
+                local = bd_local_init (frame, this);
+                /* In case of mem failure, should posix flush called ? */
+                BD_VALIDATE_MEM_ALLOC (local, op_errno, out);
+
+                local->bdatt = GF_CALLOC (1, sizeof (bd_attr_t), gf_bd_attr);
+                BD_VALIDATE_MEM_ALLOC (local->bdatt, op_errno, out);
+
+                local->bdatt->type = gf_strdup (bdatt->type);
+                memcpy (&local->bdatt->iatt, &bdatt->iatt, sizeof (struct iatt));
+                bd_update_amtime (&local->bdatt->iatt, valid);
+                uuid_copy (local->loc.gfid, fd->inode->gfid);
+                STACK_WIND (frame, bd_fsync_setattr_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->setattr, &local->loc,
+                            &local->bdatt->iatt,
+                            valid, NULL);
+                return 0;
+        }
+
+out:
+        BD_STACK_UNWIND (fsync, frame, op_ret, op_errno, &prebuf,
+                         &bdatt->iatt, NULL);
+        return 0;
+}
+
+int
+bd_flush_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iatt *pre,
+                      struct iatt *post, dict_t *xdata)
+{
+        BD_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+bd_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        int          ret    = -1;
+        bd_fd_t     *bd_fd  = NULL;
+        bd_priv_t   *priv   = NULL;
+        bd_attr_t   *bdatt  = NULL;
+        int          valid    = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
+        bd_local_t  *local    = NULL;
+        int          op_errno = EINVAL;
+        loc_t        loc      = {0, };
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (fd, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        ret = bd_inode_ctx_get (fd->inode, this, &bdatt);
+        if (!bdatt)
+                goto out;
+
+        ret = bd_fd_ctx_get (this, fd, &bd_fd);
+        if (ret < 0 || !bd_fd || !bdatt) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "bdfd/bdatt is NULL from fd=%p", fd);
+                goto out;
+        }
+
+        local = bd_local_init (frame, this);
+        BD_VALIDATE_MEM_ALLOC (local, op_errno, out);
+
+        local->fd = fd_ref (fd);
+        uuid_copy (loc.gfid, bdatt->iatt.ia_gfid);
+
+        /* Update the a|mtime during flush */
+        STACK_WIND (frame, bd_flush_setattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->setattr, &loc, &bdatt->iatt,
+                    valid, NULL);
+
+        return 0;
+
+out:
+        STACK_WIND (frame, default_flush_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->flush, fd, xdata);
+
+        return 0;
+}
+
+int32_t
+bd_release (xlator_t *this, fd_t *fd)
+{
+        int          ret      = -1;
+        bd_fd_t     *bd_fd    = NULL;
+        uint64_t     tmp_bfd  = 0;
+        bd_attr_t   *bdatt    = NULL;
+        bd_priv_t   *priv     = this->private;
+
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (fd, out);
+        VALIDATE_OR_GOTO (priv, out);
+
+        ret = bd_inode_ctx_get (fd->inode, this, &bdatt);
+        if (ret || !bdatt) /* posix file */
+                goto out;
+
+        /* FIXME: Update amtime during release */
+
+        ret = fd_ctx_del (fd, this, &tmp_bfd);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "bfd is NULL from fd=%p", fd);
+                goto out;
+        }
+        bd_fd = (bd_fd_t *)(long)tmp_bfd;
+
+        close (bd_fd->fd);
+        GF_FREE (bd_fd);
+out:
+        return 0;
+}
+
+/*
+ * Call back for removexattr after removing BD_XATTR incase of
+ * bd create failure
+ */
+int
+bd_setx_rm_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
+{
+        bd_local_t *local = frame->local;
+
+        if (local->fd)
+                BD_STACK_UNWIND (setxattr, frame, -1, EIO, xdata);
+        else
+                BD_STACK_UNWIND (setxattr, frame, -1, EIO, xdata);
+        return 0;
+
+}
+
+/*
+ * Call back after setting BD_XATTR. Creates BD. If BD creation is a failure
+ * invokes posix_removexattr to remove created BD_XATTR
+ */
+int
+bd_setx_setx_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, dict_t *xdata)
+{
+        bd_local_t *local = frame->local;
+        bd_attr_t  *bdatt = NULL;
+
+        if (op_ret < 0)
+                goto next;
+
+        /* Create LV */
+        op_errno = bd_create (local->inode->gfid, local->bdatt->iatt.ia_size,
+                              local->bdatt->type, this->private);
+        if (!op_errno)
+                goto out;
+
+        /* LV creation failed, remove BD_XATTR */
+        if (local->fd)
+                STACK_WIND (frame, bd_setx_rm_xattr_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fremovexattr,
+                            local->fd, BD_XATTR, NULL);
+        else
+                STACK_WIND (frame, bd_setx_rm_xattr_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->removexattr,
+                            &local->loc, BD_XATTR, NULL);
+
+        return 0;
+out:
+
+        bdatt = GF_CALLOC (1, sizeof (bd_attr_t), gf_bd_attr);
+        if (!bdatt) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto next;
+        }
+
+        memcpy (&bdatt->iatt, &local->bdatt->iatt, sizeof (struct iatt));
+        bdatt->type = gf_strdup (local->bdatt->type);
+
+        bd_inode_ctx_set (local->inode, THIS, bdatt);
+
+next:
+        if (local->fd)
+                BD_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+        else
+                BD_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+
+}
+
+/*
+ * Call back from posix_stat
+ */
+int
+bd_setx_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, struct iatt *iatt,
+                  dict_t *xdata)
+{
+        char       *param  = NULL;
+        char       *type   = NULL;
+        char       *s_size = NULL;
+        char       *p      = NULL;
+        char       *copy   = NULL;
+        bd_local_t *local  = frame->local;
+        bd_priv_t  *priv   = this->private;
+        char       *bd     = NULL;
+        uint64_t    size   = 0;
+
+        if (op_ret < 0)
+                goto out;
+
+        if (!IA_ISREG (iatt->ia_type)) {
+                op_errno = EOPNOTSUPP;
+                goto out;
+        }
+
+        param = copy = GF_CALLOC (1, local->data->len + 1, gf_common_mt_char);
+        BD_VALIDATE_MEM_ALLOC (param, op_errno, out);
+
+        strncpy (param, local->data->data, local->data->len);
+
+        type = strtok_r (param, ":", &p);
+        if (!type) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        if (strcmp (type, BD_LV) && strcmp (type, BD_THIN)) {
+                gf_log (this->name, GF_LOG_WARNING, "Invalid bd type %s given",
+                        type);
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        s_size = strtok_r (NULL, ":", &p);
+
+        /* If size not specified get default size */
+        if (!s_size)
+                size = bd_get_default_extent (priv);
+        else
+                gf_string2bytesize (s_size, &size);
+
+        gf_asprintf (&bd, "%s:%ld", type, size);
+        BD_VALIDATE_MEM_ALLOC (bd, op_errno, out);
+
+        local->dict = dict_new ();
+        BD_VALIDATE_MEM_ALLOC (local->dict, op_errno, out);
+
+        local->bdatt = GF_CALLOC (1, sizeof (bd_attr_t), gf_bd_attr);
+        BD_VALIDATE_MEM_ALLOC (local->bdatt, op_errno, out);
+
+        if (dict_set_dynstr (local->dict, BD_XATTR, bd) < 0) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        local->bdatt->type = gf_strdup (type);
+        memcpy (&local->bdatt->iatt, iatt, sizeof (struct iatt));
+        local->bdatt->iatt.ia_size = size;
+
+        if (local->fd)
+                STACK_WIND (frame, bd_setx_setx_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsetxattr,
+                            local->fd, local->dict, 0, NULL);
+        else
+                STACK_WIND (frame, bd_setx_setx_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr,
+                            &local->loc, local->dict, 0, NULL);
+
+        return 0;
+
+out:
+        if (local->fd)
+                BD_STACK_UNWIND (fsetxattr, frame, -1, op_errno, xdata);
+        else
+                BD_STACK_UNWIND (setxattr, frame, -1, op_errno, xdata);
+
+        GF_FREE (bd);
+        GF_FREE (copy);
+        return 0;
+}
+
+int
+bd_offload_rm_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
+{
+        bd_local_t *local = frame->local;
+
+        if (local->fd)
+                BD_STACK_UNWIND (fsetxattr, frame, -1, EIO, NULL);
+        else
+                BD_STACK_UNWIND (setxattr, frame, -1, EIO, NULL);
+
+        return 0;
+}
+
+int
+bd_offload_setx_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
+{
+        bd_local_t *local = frame->local;
+
+        if (op_ret < 0)
+                goto out;
+
+        if (local->offload == BD_OF_SNAPSHOT)
+                op_ret = bd_snapshot_create (frame->local, this->private);
+        else
+                op_ret = bd_clone (frame->local, this->private);
+
+        if (op_ret) {
+                STACK_WIND (frame, bd_offload_rm_xattr_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->removexattr,
+                            local->dloc, BD_XATTR, NULL);
+                return 0;
+        }
+
+out:
+        if (local->fd)
+                BD_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
+        else
+                BD_STACK_UNWIND (setxattr, frame, op_errno, op_errno, NULL);
+
+        return 0;
+}
+
+int
+bd_offload_getx_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        char       *bd    = NULL;
+        bd_local_t *local = frame->local;
+        char       *type  = NULL;
+        char       *p     = NULL;
+
+        if (op_ret < 0)
+                goto out;
+
+        if (dict_get_str (xattr, BD_XATTR, &p)) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        type = gf_strdup (p);
+        BD_VALIDATE_MEM_ALLOC (type, op_errno, out);
+
+        p = strrchr (type, ':');
+        if (!p) {
+                op_errno = EINVAL;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "source file xattr %s corrupted?", type);
+                goto out;
+        }
+
+        *p='\0';
+
+        /* For clone size is taken from source LV */
+        if (!local->size) {
+                p++;
+                gf_string2bytesize (p, &local->size);
+        }
+        gf_asprintf (&bd, "%s:%ld", type, local->size);
+        local->bdatt->type = gf_strdup (type);
+        dict_del (local->dict, BD_XATTR);
+        dict_del (local->dict, LINKTO);
+        if (dict_set_dynstr (local->dict, BD_XATTR, bd)) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        STACK_WIND (frame, bd_offload_setx_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr,
+                    local->dloc, local->dict, 0, NULL);
+
+        return 0;
+
+out:
+        if (local->fd)
+                BD_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+        else
+                BD_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+
+        GF_FREE (type);
+        GF_FREE (bd);
+
+        return 0;
+}
+
+int
+bd_offload_dest_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno,
+                            inode_t *inode, struct iatt *iatt,
+                            dict_t *xattr, struct iatt *postparent)
+{
+        bd_local_t *local  = frame->local;
+        char       *bd     = NULL;
+        int         ret    = -1;
+        char       *linkto = NULL;
+
+        if (op_ret < 0 && op_errno != ENODATA) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        if (!IA_ISREG (iatt->ia_type)) {
+                op_errno = EINVAL;
+                gf_log (this->name, GF_LOG_WARNING, "destination gfid is not a "
+                        "regular file");
+                goto out;
+        }
+
+        ret = dict_get_str (xattr, LINKTO, &linkto);
+        if (linkto) {
+                op_errno = EINVAL;
+                gf_log (this->name, GF_LOG_WARNING, "destination file not "
+                        "present in same brick");
+                goto out;
+        }
+
+        ret = dict_get_str (xattr, BD_XATTR, &bd);
+        if (bd) {
+                op_errno = EEXIST;
+                goto out;
+        }
+
+        local->bdatt = CALLOC (1, sizeof (bd_attr_t));
+        BD_VALIDATE_MEM_ALLOC (local->bdatt, op_errno, out);
+
+        STACK_WIND (frame, bd_offload_getx_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr,
+                    &local->loc, BD_XATTR, NULL);
+
+        return 0;
+out:
+        if (local->fd)
+                BD_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+        else
+                BD_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+
+        return 0;
+}
+
+int
+bd_merge_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        /* FIXME: if delete failed, remove xattr */
+
+        BD_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
+        return 0;
+}
+
+int
+bd_do_merge(call_frame_t *frame, xlator_t *this)
+{
+        bd_local_t *local    = frame->local;
+        inode_t    *parent   = NULL;
+        char       *p        = NULL;
+        int         op_errno = 0;
+
+        op_errno = bd_merge (this->private, local->inode->gfid);
+        if (op_errno)
+                goto out;
+
+        /*
+         * posix_unlink needs loc->pargfid to be valid, but setxattr FOP does
+         * not have loc->pargfid set. Get parent's gfid by getting parents inode
+         */
+        parent = inode_parent (local->inode, NULL, NULL);
+        if (!parent) {
+                /*
+                 * FIXME: Snapshot LV already deleted.
+                 * remove xattr, instead of returning failure
+                 */
+                op_errno = EINVAL;
+                goto out;
+        }
+        uuid_copy (local->loc.pargfid, parent->gfid);
+
+        p = strrchr (local->loc.path, '/');
+        if (p)
+                p++;
+        local->loc.name = p;
+
+        STACK_WIND (frame, bd_merge_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink,
+                    &local->loc, 0, NULL);
+
+        return 0;
+out:
+        BD_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+
+        return op_errno;
+}
+
+int
+bd_offload (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            fd_t *fd, bd_offload_t offload)
+{
+        char       *param      = NULL;
+        char       *param_copy = NULL;
+        char       *p          = NULL;
+        char       *size       = NULL;
+        char       *gfid       = NULL;
+        int         op_errno   = 0;
+        bd_local_t *local      = frame->local;
+
+        param = GF_CALLOC (1, local->data->len + 1, gf_common_mt_char);
+        BD_VALIDATE_MEM_ALLOC (param, op_errno, out);
+        param_copy = param;
+
+        local->dict = dict_new ();
+        BD_VALIDATE_MEM_ALLOC (local->dict, op_errno, out);
+
+        local->dloc = CALLOC (1, sizeof (loc_t));
+        BD_VALIDATE_MEM_ALLOC (local->dloc, op_errno, out);
+
+        strncpy (param, local->data->data, local->data->len);
+
+        gfid = strtok_r (param, ":", &p);
+        size = strtok_r (NULL, ":", &p);
+        if (size)
+                gf_string2bytesize (size, &local->size);
+        else if (offload != BD_OF_CLONE)
+                local->size = bd_get_default_extent (this->private);
+
+        if (dict_set_int8 (local->dict, BD_XATTR, 1) < 0) {
+                op_errno = EINVAL;
+                goto out;
+        }
+        if (dict_set_int8 (local->dict, LINKTO, 1) < 0) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        uuid_parse (gfid, local->dloc->gfid);
+        local->offload = offload;
+
+        STACK_WIND (frame, bd_offload_dest_lookup_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->lookup, local->dloc,
+                    local->dict);
+
+        return 0;
+
+out:
+        if (fd)
+                BD_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+        else
+                BD_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+
+        GF_FREE (param_copy);
+        return 0;
+}
+
+/*
+ * bd_setxattr: Used to create & map an LV to a posix file using
+ * BD_XATTR xattr
+ * bd_setxattr -> posix_stat -> bd_setx_stat_cbk -> posix_setxattr ->
+ * bd_setx_setx_cbk -> create_lv
+ * if create_lv failed, posix_removexattr -> bd_setx_rm_xattr_cbk
+ */
+int
+bd_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int flags, dict_t *xdata)
+{
+        int           op_errno = 0;
+        data_t       *data     = NULL;
+        bd_local_t   *local    = NULL;
+        bd_attr_t    *bdatt    = NULL;
+        bd_offload_t  cl_type  = BD_OF_NONE;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+
+        if ((data = dict_get (dict, BD_XATTR)))
+                cl_type = BD_OF_NONE;
+        else if ((data = dict_get (dict, BD_CLONE)))
+                cl_type = BD_OF_CLONE;
+        else if ((data = dict_get (dict, BD_SNAPSHOT)))
+                cl_type = BD_OF_SNAPSHOT;
+        else if ((data = dict_get (dict, BD_MERGE)))
+                cl_type = BD_OF_MERGE;
+
+        bd_inode_ctx_get (loc->inode, this, &bdatt);
+        if (!cl_type && !data) {
+                STACK_WIND (frame, default_setxattr_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->setxattr, loc, dict,
+                            flags, xdata);
+                return 0;
+        }
+
+        local = bd_local_init (frame, this);
+        BD_VALIDATE_MEM_ALLOC (local, op_errno, out);
+
+        local->data = data;
+        loc_copy (&local->loc, loc);
+        local->inode = inode_ref (loc->inode);
+
+        if (cl_type) {
+                /* For cloning/snapshot, source file must be mapped to LV */
+                if (!bdatt) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s not mapped to BD", loc->path);
+                        op_errno = EINVAL;
+                        goto out;
+                }
+                if (cl_type == BD_OF_MERGE)
+                        bd_do_merge (frame, this);
+                else
+                        bd_offload (frame, this, loc, NULL, cl_type);
+        } else if (data) {
+                if (bdatt) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s already mapped to BD", loc->path);
+                        op_errno = EEXIST;
+                        goto out;
+                }
+                STACK_WIND (frame, bd_setx_stat_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->stat, loc, xdata);
+        }
+
+        return 0;
+out:
+        if (op_errno)
+                STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+
+        return 0;
+}
+
+/*
+ * bd_fsetxattr: Used to create/map an LV to a posix file using
+ * BD_XATTR xattr
+ * bd_fsetxattr ->  posix_fstat -> bd_setx_stat_cbk -> posix_fsetxattr ->
+ * bd_setx_setx_cbk -> create_lv
+ * if create_lv failed, posix_removexattr -> bd_setx_rm_xattr_cbk
+ * -> bd_fsetxattr_cbk
+ */
+int32_t
+bd_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int flags, dict_t *xdata)
+{
+        int           op_errno = 0;
+        data_t       *data     = NULL;
+        bd_attr_t    *bdatt    = NULL;
+        bd_local_t   *local    = NULL;
+        bd_offload_t  cl_type  = BD_OF_NONE;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+        VALIDATE_OR_GOTO (fd, out);
+
+        bd_inode_ctx_get (fd->inode, this, &bdatt);
+
+        data =  dict_get (dict, BD_XATTR);
+        if ((data = dict_get (dict, BD_XATTR)))
+                cl_type = BD_OF_NONE;
+        else if ((data = dict_get (dict, BD_CLONE)))
+                cl_type = BD_OF_CLONE;
+        else if ((data = dict_get (dict, BD_SNAPSHOT)))
+                cl_type = BD_OF_SNAPSHOT;
+        else if ((data = dict_get (dict, BD_MERGE))) {
+                /*
+                 * bd_merge is not supported for fsetxattr, because snapshot LV
+                 * is opened and it causes problem in snapshot merge
+                 */
+                op_errno = EOPNOTSUPP;
+                goto out;
+        }
+
+        bd_inode_ctx_get (fd->inode, this, &bdatt);
+
+        if (!cl_type && !data) {
+                /* non bd file object */
+                STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsetxattr,
+                            fd, dict, flags, xdata);
+                return 0;
+        }
+
+        local = bd_local_init (frame, this);
+        BD_VALIDATE_MEM_ALLOC (local, op_errno, out);
+
+        local->inode = inode_ref (fd->inode);
+        local->fd = fd_ref (fd);
+        local->data = data;
+
+        if (cl_type) {
+                /* For cloning/snapshot, source file must be mapped to LV */
+                if (!bdatt) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "fd %p not mapped to BD", fd);
+                        op_errno = EINVAL;
+                        goto out;
+
+                }
+                bd_offload (frame, this, NULL, fd, cl_type);
+        } else if (data) {
+                if (bdatt) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "fd %p already mapped to BD", fd);
+                        op_errno = EEXIST;
+                        goto out;
+                }
+                STACK_WIND(frame, bd_setx_stat_cbk, FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        }
+
+        return 0;
+out:
+
+        BD_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+
+        return 0;
+}
+
+int32_t
+bd_removexattr (call_frame_t *frame, xlator_t *this,
+                   loc_t *loc, const char *name, dict_t *xdata)
+{
+        if (!strcmp (name, BD_XATTR))
+            goto out;
+
+        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+        return 0;
+out:
+        BD_STACK_UNWIND (removexattr, frame, -1, ENODATA, NULL);
+        return 0;
+}
+
+int32_t
+bd_fremovexattr (call_frame_t *frame, xlator_t *this,
+                    fd_t *fd, const char *name, dict_t *xdata)
+{
+        if (!strcmp (name, BD_XATTR))
+            goto out;
+
+        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+
+        return 0;
+out:
+        BD_STACK_UNWIND (fremovexattr, frame, -1, ENODATA, NULL);
+        return 0;
+}
+
+int
+bd_trunc_setxattr_setx_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *xdata)
+{
+        bd_local_t *local = frame->local;
+
+        if (local->fd)
+                BD_STACK_UNWIND (ftruncate, frame, -1, EIO, NULL, NULL, NULL);
+        else
+                BD_STACK_UNWIND (truncate, frame, -1, EIO, NULL, NULL, NULL);
+
+        return 0;
+}
+
+/*
+ * Call back for setxattr after setting BD_XATTR_SIZE.
+ */
+int
+bd_trunc_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xdata)
+{
+        bd_local_t *local = frame->local;
+        bd_attr_t   *bdatt = NULL;
+        struct iatt prebuf = {0, };
+        char         *bd   = NULL;
+
+        if (op_ret < 0)
+                goto out;
+
+        bd_inode_ctx_get (local->inode, this, &bdatt);
+        if (!bdatt)
+                goto revert_xattr;
+
+        op_errno = bd_resize (this->private, local->inode->gfid,
+                              local->bdatt->iatt.ia_size);
+        if (op_errno)
+                goto revert_xattr;
+
+        memcpy (&prebuf, &bdatt->iatt, sizeof (struct iatt));
+        /* LV resized, update new size in the cache */
+        bdatt->iatt.ia_size = local->bdatt->iatt.ia_size;
+
+        if (local->fd)
+                BD_STACK_UNWIND (ftruncate, frame, 0, 0, &prebuf, &bdatt->iatt,
+                                 NULL);
+        else
+                BD_STACK_UNWIND (truncate, frame, 0, 0, &prebuf, &bdatt->iatt,
+                                 NULL);
+
+        return 0;
+
+revert_xattr:
+        /* revert setxattr */
+        op_ret = dict_get_str (local->dict, BD_XATTR, &bd);
+        GF_FREE (bd);
+        gf_asprintf (&bd, "%s:%ld", bdatt->type, bdatt->iatt.ia_size);
+
+        if (local->fd)
+                STACK_WIND (frame, bd_trunc_setxattr_setx_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsetxattr,
+                            local->fd, local->dict, 0, NULL);
+        else
+                STACK_WIND (frame, bd_trunc_setxattr_setx_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr,
+                            &local->loc, local->dict, 0, NULL);
+
+        return 0;
+out:
+        if (local->fd)
+                BD_STACK_UNWIND (ftruncate, frame, -1, EIO, NULL, NULL, NULL);
+        else
+                BD_STACK_UNWIND (truncate, frame, -1, EIO, NULL, NULL, NULL);
+
+        return 0;
+}
+
+/*
+ * call back from posix_[f]truncate_stat
+ * If offset > LV size, it resizes the LV and calls posix_setxattr
+ * to update new LV size in xattr else calls posix_setattr for updating
+ * the posix file so that truncate fop behaves properly
+ */
+int
+bd_trunc_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, struct iatt *buf, dict_t *xdata)
+{
+        char       *bd    = NULL;
+        bd_local_t *local = frame->local;
+        bd_attr_t  *bdatt = NULL;
+
+        if (op_ret < 0)
+                goto out;
+
+        local->dict  = dict_new ();
+        BD_VALIDATE_MEM_ALLOC (local->dict, op_errno, out);
+
+        bd_inode_ctx_get (local->inode, this, &bdatt);
+        if (!bdatt) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        gf_asprintf (&bd, "%s:%ld", bdatt->type, local->bdatt->iatt.ia_size);
+        if (dict_set_dynstr (local->dict, BD_XATTR, bd)) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        if (local->fd)
+                STACK_WIND (frame, bd_trunc_setxattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsetxattr,
+                            local->fd, local->dict, 0, NULL);
+        else
+                STACK_WIND (frame, bd_trunc_setxattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr,
+                            &local->loc, local->dict, 0, NULL);
+
+        return 0;
+out:
+        if (local->fd)
+                BD_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL,
+                                 NULL);
+        else
+                BD_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL,
+                                 NULL);
+        GF_FREE (bd);
+        return 0;
+}
+
+void
+bd_do_trunc (call_frame_t *frame, xlator_t *this, fd_t *fd, loc_t *loc,
+             off_t offset, bd_attr_t *bdatt)
+{
+        bd_local_t *local    = NULL;
+        struct iatt prebuf   = {0, };
+        int         op_errno = 0;
+        int         op_ret   = -1;
+
+        /* If requested size is less than LV size, return success */
+        if (offset <= bdatt->iatt.ia_size) {
+                memcpy (&prebuf, &bdatt->iatt, sizeof (struct iatt));
+                bd_update_amtime (&bdatt->iatt, GF_SET_ATTR_MTIME);
+                op_ret = 0;
+                goto out;
+        }
+
+        local = bd_local_init (frame, this);
+        BD_VALIDATE_MEM_ALLOC (local, op_errno, out);
+
+        local->bdatt = GF_CALLOC (1, sizeof (bd_attr_t), gf_bd_attr);
+        BD_VALIDATE_MEM_ALLOC (local->bdatt, op_errno, out);
+
+        if (fd) {
+                local->inode = inode_ref (fd->inode);
+                local->fd = fd_ref (fd);
+        } else {
+                local->inode = inode_ref (loc->inode);
+                loc_copy (&local->loc, loc);
+        }
+
+        local->bdatt->iatt.ia_size =
+                bd_adjust_size (this->private, offset);
+
+        STACK_WIND (frame, bd_trunc_stat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, NULL);
+
+        return;
+
+out:
+        if (fd)
+                BD_STACK_UNWIND (ftruncate, frame, op_ret, op_errno,
+                                 &prebuf, &bdatt->iatt, NULL);
+        else
+                BD_STACK_UNWIND (truncate, frame, op_ret, op_errno,
+                                 &prebuf, &bdatt->iatt, NULL);
+        return;
+}
+
+/*
+ * bd_ftruncate: Resizes a LV if fd belongs to BD.
+ */
+int32_t
+bd_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
+{
+        int         op_errno = 0;
+        bd_attr_t  *bdatt    = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (fd, out);
+
+        if (bd_inode_ctx_get (fd->inode, this, &bdatt)) {
+                STACK_WIND (frame, default_ftruncate_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->ftruncate, fd,
+                            offset, xdata);
+                return 0;
+        }
+
+        bd_do_trunc (frame, this, fd, NULL, offset, bdatt);
+        return 0;
+out:
+        BD_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+/*
+ * bd_truncate: Resizes a LV if file maps to LV.
+ */
+int32_t
+bd_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
+{
+        int         op_errno = 0;
+        bd_attr_t  *bdatt    = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (loc, out);
+
+        if (bd_inode_ctx_get (loc->inode, this, &bdatt)) {
+                STACK_WIND (frame, default_truncate_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->truncate, loc,
+                            offset, xdata);
+                return 0;
+        }
+
+        bd_do_trunc (frame, this, NULL, loc, offset, bdatt);
+        return 0;
+
+out:
+        BD_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+__bd_pwritev (int fd, struct iovec *vector, int count, off_t offset,
+              uint64_t bd_size)
+{
+        int        index           = 0;
+        int        retval          = 0;
+        off_t      internal_offset = 0;
+
+        if (!vector)
+                return -EFAULT;
+
+        retval = pwritev (fd, vector, count, offset);
+        if (retval == -1) {
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "base %p, length %ld, offset %ld, message %s",
+                        vector[index].iov_base, vector[index].iov_len,
+                        internal_offset, strerror (errno));
+                retval = -errno;
+                goto err;
+        }
+/*
+
+
+        internal_offset = offset;
+        for (index = 0; index < count; index++) {
+                if (internal_offset > bd_size) {
+                        op_ret = -ENOSPC;
+                        goto err;
+                }
+                if (internal_offset + vector[index].iov_len > bd_size) {
+                        vector[index].iov_len = bd_size - internal_offset;
+                        no_space = 1;
+                }
+                retval = pwritev (fd, vector[index].iov_base,
+                                vector[index].iov_len, internal_offset);
+                if (retval == -1) {
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "base %p, length %ld, offset %ld, message %s",
+                                vector[index].iov_base, vector[index].iov_len,
+                                internal_offset, strerror (errno));
+                        op_ret = -errno;
+                        goto err;
+                }
+                op_ret += retval;
+                internal_offset += retval;
+                if (no_space)
+                        break;
+        }
+*/
+err:
+        return retval;
+}
+
+/*
+ * bd_writev: Writes to LV if its BD file or forwards the request to posix_write
+ * bd_writev -> posix_writev -> bd_writev_cbk
+ */
+int
+bd_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdict)
+{
+        int32_t         op_ret    = -1;
+        int32_t         op_errno  = 0;
+        int             _fd       = -1;
+        bd_fd_t         *bd_fd    = NULL;
+        int             ret       = -1;
+        uint64_t        size      = 0;
+        struct iatt     prebuf    = {0, };
+        bd_attr_t      *bdatt     = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (fd, out);
+        VALIDATE_OR_GOTO (vector, out);
+
+        ret = bd_fd_ctx_get (this, fd, &bd_fd);
+        if (ret < 0 || !bd_fd) { /* posix fd */
+                STACK_WIND (frame, default_writev_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->writev, fd, vector, count,
+                            offset, flags, iobref, xdict);
+                return 0;
+        }
+
+        _fd = bd_fd->fd;
+
+        if (bd_inode_ctx_get (fd->inode, this, &bdatt)) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+        size = bdatt->iatt.ia_size;
+
+        op_ret = __bd_pwritev (_fd, vector, count, offset, size);
+        if (op_ret < 0) {
+                op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "write failed: offset %"PRIu64
+                                ", %s", offset, strerror (op_errno));
+                goto out;
+        }
+
+        memcpy (&prebuf, &bdatt->iatt, sizeof (struct iatt));
+        bd_update_amtime (&bdatt->iatt, GF_SET_ATTR_MTIME);
+out:
+
+        BD_STACK_UNWIND (writev, frame, op_ret, op_errno, &prebuf,
+                         &bdatt->iatt, NULL);
+        return 0;
+}
+
+int
+bd_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata)
+{
+        bd_attr_t   *bdatt = NULL;
+        int         *valid = cookie;
+        bd_local_t  *local = frame->local;
+
+        if (op_ret < 0 || !valid || !local)
+                goto out;
+
+        if (bd_inode_ctx_get (local->inode, this, &bdatt))
+                goto out;
+
+        if (*valid & GF_SET_ATTR_UID)
+                bdatt->iatt.ia_uid = postbuf->ia_uid;
+        else if (*valid & GF_SET_ATTR_GID)
+                bdatt->iatt.ia_gid = postbuf->ia_gid;
+        else if (*valid & GF_SET_ATTR_MODE) {
+                bdatt->iatt.ia_type = postbuf->ia_type;
+                bdatt->iatt.ia_prot = postbuf->ia_prot;
+        } else if (*valid & GF_SET_ATTR_ATIME) {
+                bdatt->iatt.ia_atime = postbuf->ia_atime;
+                bdatt->iatt.ia_atime_nsec = postbuf->ia_atime_nsec;
+        } else if (*valid & GF_SET_ATTR_MTIME) {
+                bdatt->iatt.ia_mtime = postbuf->ia_mtime;
+                bdatt->iatt.ia_mtime_nsec = postbuf->ia_mtime_nsec;
+        }
+
+        bdatt->iatt.ia_ctime = postbuf->ia_ctime;
+        bdatt->iatt.ia_ctime_nsec = postbuf->ia_ctime_nsec;
+
+        memcpy (postbuf, &bdatt->iatt, sizeof (struct iatt));
+out:
+        FREE (valid);
+        BD_STACK_UNWIND (setattr, frame, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
+        return 0;
+}
+
+int
+bd_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+        bd_local_t *local     = NULL;
+        bd_attr_t  *bdatt     = NULL;
+        int        *ck_valid  = NULL;
+        int         op_errno  = 0;
+
+        if (bd_inode_ctx_get (loc->inode, this, &bdatt)) {
+                STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->setattr,
+                           loc, stbuf, valid, xdata);
+                return 0;
+        }
+
+        local = bd_local_init (frame, this);
+        BD_VALIDATE_MEM_ALLOC (local, op_errno, out);
+
+        ck_valid = CALLOC (1, sizeof (valid));
+        BD_VALIDATE_MEM_ALLOC (ck_valid, op_errno, out);
+
+        local->inode = inode_ref (loc->inode);
+        *ck_valid = valid;
+
+        STACK_WIND_COOKIE (frame, bd_setattr_cbk, ck_valid, FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->setattr,
+                           loc, stbuf, valid, xdata);
+
+        return 0;
+out:
+        BD_STACK_UNWIND (setattr, frame, -1, ENOMEM, NULL, NULL, xdata);
+        return 0;
+}
+
+int
+bd_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+             int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        bd_attr_t *bdatt = NULL;
+
+        if (op_ret < 0)
+                goto out;
+
+        if (bd_inode_ctx_get (inode, this, &bdatt))
+                goto out;
+
+        bdatt->iatt.ia_ctime = buf->ia_ctime;
+        bdatt->iatt.ia_ctime_nsec = buf->ia_ctime_nsec;
+        bdatt->iatt.ia_nlink = buf->ia_nlink;
+        memcpy (buf, &bdatt->iatt, sizeof (struct iatt));
+
+out:
+        BD_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
+                         preparent, postparent, NULL);
+        return 0;
+}
+
+int
+bd_link (call_frame_t *frame, xlator_t *this,
+         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        STACK_WIND (frame, bd_link_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+        return 0;
+}
+
+int
+bd_handle_special_xattrs (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                          fd_t *fd, const char *name, dict_t *xdata)
+{
+        dict_t *xattr   = NULL;
+        int op_ret      = -1;
+        int op_errno    = ENOMEM;;
+        bd_priv_t *priv = this->private;
+
+        xattr = dict_new ();
+        if (!xattr)
+                goto out;
+
+        if (!strcmp (name, VOL_TYPE))
+                op_ret = dict_set_int64 (xattr, (char *)name, 1);
+        else if (!strcmp (name, VOL_CAPS))
+                op_ret = dict_set_int64 (xattr, (char *)name, priv->caps);
+        else
+                op_ret = bd_get_origin (this->private, loc, fd, xattr);
+
+out:
+        if (loc)
+                BD_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr,
+                                 xdata);
+        else
+                BD_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr,
+                                 xdata);
+
+        op_ret = dict_reset (xattr);
+        dict_unref (xattr);
+
+        return 0;
+}
+
+int
+bd_fgetxattr (call_frame_t *frame, xlator_t *this,
+              fd_t *fd, const char *name, dict_t *xdata)
+{
+        if (name && (!strcmp (name, VOL_TYPE) || !strcmp (name, VOL_CAPS)
+                     || !strcmp (name, BD_ORIGIN)))
+                bd_handle_special_xattrs (frame, this, NULL, fd, name, xdata);
+        else
+                STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fgetxattr,
+                            fd, name, xdata);
+        return 0;
+}
+
+int
+bd_getxattr (call_frame_t *frame, xlator_t *this,
+             loc_t *loc, const char *name, dict_t *xdata)
+{
+        if (name && (!strcmp (name, VOL_TYPE) || !strcmp (name, VOL_CAPS)
+                     || !strcmp (name, BD_ORIGIN)))
+                bd_handle_special_xattrs (frame, this, loc, NULL, name, xdata);
+        else
+                STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->getxattr,
+                            loc, name, xdata);
+
+        return 0;
+}
+
+int
+bd_unlink_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *buf, dict_t *xattr,
+                      struct iatt *postparent)
+{
+        bd_gfid_t     gfid  = {0, };
+        bd_local_t   *local = frame->local;
+
+        if (buf->ia_nlink > 1)
+                goto posix;
+
+        BD_VALIDATE_LOCAL_OR_GOTO (local, op_errno, out);
+
+        uuid_utoa_r (inode->gfid, gfid);
+        if (bd_delete_lv (this->private, gfid, &op_errno) < 0) {
+                if (op_errno != ENOENT)
+                        goto out;
+        }
+
+posix:
+        /* remove posix */
+        STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink,
+                    &local->loc, 0, NULL);
+
+        return 0;
+out:
+        BD_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+bd_unlink (call_frame_t *frame, xlator_t *this,
+           loc_t *loc, int xflag, dict_t *xdata)
+{
+        int          op_errno = 0;
+        bd_attr_t   *bdatt     = NULL;
+        bd_local_t  *local     = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (loc, out);
+
+        if (bd_inode_ctx_get (loc->inode, this, &bdatt)) {
+                STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->unlink,
+                            loc, xflag, xdata);
+                return 0;
+        }
+
+        local = bd_local_init (frame, this);
+        BD_VALIDATE_MEM_ALLOC (local, op_errno, out);
+
+        loc_copy (&local->loc, loc);
+
+        STACK_WIND (frame, bd_unlink_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, NULL);
+        return 0;
+out:
+        BD_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+bd_priv (xlator_t *this)
+{
+        return 0;
+}
+
+int32_t
+bd_inode (xlator_t *this)
+{
+        return 0;
+}
+
+int32_t
+bd_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              int32_t len, dict_t *xdata)
+{
+        int             op_ret          = -1;
+        int             op_errno        = 0;
+        int             ret             = 0;
+        int             _fd             = -1;
+        char           *alloc_buf       = NULL;
+        char           *buf             = NULL;
+        int32_t         weak_checksum   = 0;
+        bd_fd_t        *bd_fd           = NULL;
+        unsigned char   strong_checksum[MD5_DIGEST_LENGTH] = {0};
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (fd, out);
+
+        ret = bd_fd_ctx_get (this, fd, &bd_fd);
+        if (ret < 0 || !bd_fd) {
+                STACK_WIND (frame, default_rchecksum_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->rchecksum, fd, offset,
+                            len, xdata);
+                return 0;
+        }
+
+        memset (strong_checksum, 0, MD5_DIGEST_LENGTH);
+
+        alloc_buf = page_aligned_alloc (len, &buf);
+        if (!alloc_buf) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        _fd = bd_fd->fd;
+
+        LOCK (&fd->lock);
+        {
+                ret = pread (_fd, buf, len, offset);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "pread of %d bytes returned %d (%s)",
+                                len, ret, strerror (errno));
+                        op_errno = errno;
+                }
+        }
+        UNLOCK (&fd->lock);
+
+        if (ret < 0)
+                goto out;
+
+        weak_checksum = gf_rsync_weak_checksum ((unsigned char *) buf,
+                                                (size_t) len);
+        gf_rsync_strong_checksum ((unsigned char *) buf, (size_t) len,
+                                  (unsigned char *) strong_checksum);
+
+        op_ret = 0;
+out:
+        BD_STACK_UNWIND (rchecksum, frame, op_ret, op_errno,
+                         weak_checksum, strong_checksum, NULL);
+
+        GF_FREE (alloc_buf);
+
+        return 0;
+}
+
+/**
+ * notify - when parent sends PARENT_UP, send CHILD_UP event from here
+ */
+int32_t
+notify (xlator_t *this,
+        int32_t event,
+        void *data,
+        ...)
+{
+        switch (event)
+        {
+        case GF_EVENT_PARENT_UP:
+        {
+                /* Tell the parent that bd xlator is up */
+                default_notify (this, GF_EVENT_CHILD_UP, data);
+        }
+        break;
+        default:
+                break;
+        }
+        return 0;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_bd_mt_end + 1);
+
+        if (ret != 0)
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                       "failed");
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int   ret = -1;
+        bd_priv_t *priv = this->private;
+
+        GF_OPTION_RECONF ("bd-aio", priv->aio_configured, options,
+                          bool, out);
+
+        if (priv->aio_configured)
+                bd_aio_on (this);
+        else
+                bd_aio_off (this);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/**
+ * bd xlator init - Validate configured VG
+ */
+int
+init (xlator_t *this)
+{
+        int         ret         = 0;
+        char       *vg_data     = NULL;
+        char       *device      = NULL;
+        bd_priv_t  *_private    = NULL;
+
+        if (!this->children) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "FATAL: storage/bd needs posix as subvolume");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Volume is dangling. Please check the volume file.");
+        }
+
+        GF_OPTION_INIT ("export", vg_data, str, error);
+        GF_OPTION_INIT ("device", device, str, error);
+
+        /* Now we support only LV device */
+        if (strcasecmp (device, BACKEND_VG)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "FATAL: unknown %s backend %s", BD_XLATOR, device);
+                return -1;
+        }
+
+        this->local_pool = mem_pool_new (bd_local_t, 64);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "FATAL: Failed to create bd memory pool");
+                return -1;
+        }
+
+        ret = 0;
+        _private = GF_CALLOC (1, sizeof (*_private), gf_bd_private);
+        if (!_private)
+                goto error;
+
+        this->private = _private;
+        _private->vg = gf_strdup (vg_data);
+        if (!_private->vg)
+                goto error;
+
+        _private->handle = lvm_init (NULL);
+        if (!_private->handle) {
+                gf_log (this->name, GF_LOG_CRITICAL, "lvm_init failed");
+                goto error;
+        }
+        _private->caps = BD_CAPS_BD;
+        if (bd_scan_vg (this, _private))
+                goto error;
+
+        _private->aio_init_done = _gf_false;
+        _private->aio_capable = _gf_false;
+
+        GF_OPTION_INIT ("bd-aio", _private->aio_configured, bool, error);
+        if (_private->aio_configured) {
+                if (bd_aio_on (this)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "BD AIO init failed");
+                        ret = -1;
+                        goto error;
+                }
+        }
+
+        _private->caps |= BD_CAPS_OFFLOAD_COPY | BD_CAPS_OFFLOAD_SNAPSHOT;
+
+        return 0;
+error:
+        GF_FREE (_private->vg);
+        if (_private->handle)
+                lvm_quit (_private->handle);
+        mem_pool_destroy (this->local_pool);
+        GF_FREE (_private);
+        return -1;
+}
+
+void
+fini (xlator_t *this)
+{
+        bd_priv_t *priv = this->private;
+        mem_pool_destroy (this->local_pool);
+        this->local_pool = NULL;
+        if (!priv)
+                return;
+        lvm_quit (priv->handle);
+        GF_FREE (priv->vg);
+        this->private = NULL;
+        GF_FREE (priv);
+        return;
+}
+
+struct xlator_dumpops dumpops = {
+        .priv    = bd_priv,
+        .inode   = bd_inode,
+};
+
+struct xlator_fops fops = {
+        .readdirp    = bd_readdirp,
+        .lookup      = bd_lookup,
+        .stat        = bd_stat,
+        .statfs      = bd_statfs,
+        .open        = bd_open,
+        .fstat       = bd_fstat,
+        .rchecksum   = bd_rchecksum,
+        .readv       = bd_readv,
+        .fsync       = bd_fsync,
+        .setxattr    = bd_setxattr,
+        .fsetxattr   = bd_fsetxattr,
+        .removexattr = bd_removexattr,
+        .fremovexattr=bd_fremovexattr,
+        .truncate    = bd_truncate,
+        .ftruncate   = bd_ftruncate,
+        .writev      = bd_writev,
+        .getxattr    = bd_getxattr,
+        .fgetxattr   = bd_fgetxattr,
+        .unlink      = bd_unlink,
+        .link        = bd_link,
+        .flush       = bd_flush,
+        .setattr     = bd_setattr,
+        .discard     = bd_discard,
+};
+
+struct xlator_cbks cbks = {
+        .release     = bd_release,
+        .forget      = bd_forget,
+};
+
+struct volume_options options[] = {
+        { .key = {"export"},
+          .type = GF_OPTION_TYPE_STR},
+        { .key = {"device"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = BACKEND_VG},
+        {
+          .key  = {"bd-aio"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Support for native Linux AIO"
+        },
+
+        { .key = {NULL} }
+};
diff --git a/xlators/storage/bd/src/bd.h b/xlators/storage/bd/src/bd.h
new file mode 100644
index 000000000..34b4c9e22
--- /dev/null
+++ b/xlators/storage/bd/src/bd.h
@@ -0,0 +1,178 @@
+/*
+  BD translator - Exports Block devices on server side as regular
+  files to client
+
+  Copyright IBM, Corp. 2012
+
+  This file is part of GlusterFS.
+
+  Author:
+  M. Mohan Kumar <mohan@in.ibm.com>
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _BD_H
+#define _BD_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#ifdef HAVE_LIBAIO
+#include <libaio.h>
+#endif
+
+#include "xlator.h"
+#include "mem-types.h"
+
+#define BD_XLATOR "block device mapper xlator"
+#define BACKEND_VG "vg"
+#define GF_XATTR "user.glusterfs"
+#define BD_XATTR GF_XATTR ".bd"
+
+#define BD_LV "lv"
+#define BD_THIN "thin"
+
+#define LVM_RESIZE "/sbin/lvresize"
+#define LVM_CREATE "/sbin/lvcreate"
+#define LVM_CONVERT "/sbin/lvconvert"
+
+#define VOL_TYPE "volume.type"
+#define VOL_CAPS "volume.caps"
+
+#define ALIGN_SIZE 4096
+
+#define BD_CAPS_BD               0x01
+#define BD_CAPS_THIN             0x02
+#define BD_CAPS_OFFLOAD_COPY     0x04
+#define BD_CAPS_OFFLOAD_SNAPSHOT 0x08
+
+#define BD_CLONE "clone"
+#define BD_SNAPSHOT "snapshot"
+#define BD_MERGE "merge"
+#define BD_ORIGIN "list-origin"
+
+#define IOV_NR    4
+#define IOV_SIZE (64 * 1024)
+
+#define ALIGN_SIZE 4096
+
+#define LINKTO "trusted.glusterfs.dht.linkto"
+
+#define BD_VALIDATE_MEM_ALLOC(buff, op_errno, label)                \
+        if (!buff) {                                                \
+                op_errno = ENOMEM;                                  \
+                gf_log (this->name, GF_LOG_ERROR, "out of memory"); \
+                goto label;                                         \
+        }
+
+#define BD_VALIDATE_LOCAL_OR_GOTO(local, op_errno, label) \
+        if (!local) {                                     \
+                op_errno = EINVAL;                        \
+                goto label;                               \
+        }
+
+#define BD_STACK_UNWIND(typ, frame, args ...) do {      \
+        bd_local_t *__local = frame->local;             \
+        xlator_t   *__this = frame->this;               \
+                                                        \
+        frame->local = NULL;                            \
+        STACK_UNWIND_STRICT (typ, frame, args);         \
+        if (__local)                                    \
+                bd_local_free (__this, __local);        \
+        } while (0)
+
+typedef char bd_gfid_t[GF_UUID_BUF_SIZE];
+
+enum gf_bd_mem_types_ {
+        gf_bd_private  = gf_common_mt_end + 1,
+        gf_bd_attr,
+        gf_bd_fd,
+        gf_bd_mt_end
+};
+
+/**
+ * bd_fd - internal structure
+ */
+typedef struct bd_fd {
+        int             fd;
+        int32_t         flag;
+        int             odirect;
+} bd_fd_t;
+
+typedef struct bd_priv {
+        lvm_t             handle;
+        char              *vg;
+        char              *pool;
+        int                caps;
+        gf_boolean_t       aio_init_done;
+        gf_boolean_t       aio_capable;
+        gf_boolean_t       aio_configured;
+#ifdef HAVE_LIBAIO
+        io_context_t       ctxp;
+        pthread_t          aiothread;
+#endif
+} bd_priv_t;
+
+
+typedef enum bd_type {
+        BD_TYPE_NONE,
+        BD_TYPE_LV,
+} bd_type_t;
+
+typedef struct {
+        struct iatt  iatt;
+        char        *type;
+} bd_attr_t;
+
+typedef enum {
+        BD_OF_NONE,
+        BD_OF_CLONE,
+        BD_OF_SNAPSHOT,
+        BD_OF_MERGE,
+} bd_offload_t;
+
+typedef struct {
+        dict_t      *dict;
+        bd_attr_t   *bdatt;
+        inode_t     *inode;
+        loc_t        loc;
+        fd_t        *fd;
+        data_t      *data; /* for setxattr */
+        bd_offload_t offload;
+        uint64_t     size;
+        loc_t       *dloc;
+} bd_local_t;
+
+/* Prototypes */
+int bd_inode_ctx_set (inode_t *inode, xlator_t *this, bd_attr_t *ctx);
+int bd_inode_ctx_get (inode_t *inode, xlator_t *this, bd_attr_t **ctx);
+int bd_scan_vg (xlator_t *this, bd_priv_t *priv);
+bd_local_t *bd_local_init (call_frame_t *frame, xlator_t *this);
+void bd_local_free (xlator_t *this, bd_local_t *local);
+int bd_fd_ctx_get (xlator_t *this, fd_t *fd, bd_fd_t **bdfd);
+char *page_aligned_alloc (size_t size, char **aligned_buf);
+int bd_validate_bd_xattr (xlator_t *this, char *bd, char **type,
+                          uint64_t *lv_size, uuid_t uuid);
+uint64_t bd_get_default_extent (bd_priv_t *priv);
+uint64_t bd_adjust_size (bd_priv_t *priv, uint64_t size);
+int bd_create (uuid_t uuid, uint64_t size, char *type, bd_priv_t *priv);
+int bd_resize (bd_priv_t *priv, uuid_t uuid, off_t size);
+int bd_delete_lv (bd_priv_t *priv, const char *lv_name, int *op_errno);
+int bd_snapshot_create (bd_local_t *local, bd_priv_t *priv);
+int bd_clone (bd_local_t *local, bd_priv_t *priv);
+
+int bd_merge (bd_priv_t *priv, uuid_t gfid);
+int bd_get_origin (bd_priv_t *priv, loc_t *loc, fd_t *fd, dict_t *dict);
+inline void bd_update_amtime(struct iatt *iatt, int flag);
+int bd_snapshot_create (bd_local_t *local, bd_priv_t *priv);
+int bd_clone (bd_local_t *local, bd_priv_t *priv);
+int bd_merge (bd_priv_t *priv, uuid_t gfid);
+int bd_get_origin (bd_priv_t *priv, loc_t *loc, fd_t *fd, dict_t *dict);
+
+#endif
diff --git a/xlators/storage/bdb/Makefile.am b/xlators/storage/bdb/Makefile.am
deleted file mode 100644
index d471a3f92..000000000
--- a/xlators/storage/bdb/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/storage/bdb/src/Makefile.am b/xlators/storage/bdb/src/Makefile.am
deleted file mode 100644
index 7e2376979..000000000
--- a/xlators/storage/bdb/src/Makefile.am
+++ /dev/null
@@ -1,18 +0,0 @@
-
-xlator_LTLIBRARIES = bdb.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/storage
-
-bdb_la_LDFLAGS = -module -avoidversion
-
-bdb_la_SOURCES = bctx.c bdb-ll.c bdb.c
-bdb_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
-
-noinst_HEADERS = bdb.h 
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D_GNU_SOURCE -D$(GF_HOST_OS) -Wall \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-AM_LDFLAGS = -ldb
-
-CLEANFILES = 
-
diff --git a/xlators/storage/bdb/src/bctx.c b/xlators/storage/bdb/src/bctx.c
deleted file mode 100644
index 61560edfa..000000000
--- a/xlators/storage/bdb/src/bctx.c
+++ /dev/null
@@ -1,341 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <list.h>
-#include <bdb.h>
-#include <libgen.h> /* for dirname */
-
-static void
-__destroy_bctx (bctx_t *bctx)
-{
-        if (bctx->directory)
-                GF_FREE (bctx->directory);
-
-        if (bctx->db_path)
-                GF_FREE (bctx->db_path);
-
-        GF_FREE (bctx);
-}
-
-static void
-__unhash_bctx (bctx_t *bctx)
-{
-        list_del_init (&bctx->b_hash);
-}
-
-static int32_t
-bctx_table_prune (bctx_table_t *table)
-{
-        int32_t ret = 0;
-        struct list_head purge = {0,};
-        struct list_head *next = NULL;
-        bctx_t *entry = NULL;
-        bctx_t *del = NULL, *tmp = NULL;
-
-        if (!table)
-                return 0;
-
-        INIT_LIST_HEAD (&purge);
-
-        LOCK (&table->lock);
-        {
-                if ((table->lru_limit) &&
-                    (table->lru_size > table->lru_limit)) {
-                        while (table->lru_size > table->lru_limit) {
-                                next = table->b_lru.next;
-                                entry = list_entry (next, bctx_t, list);
-
-                                list_move_tail (next, &table->purge);
-                                __unhash_bctx (entry);
-
-                                table->lru_size--;
-                                ret++;
-                        }
-                }
-                list_move_tail (&purge, &table->purge);
-                list_del_init (&table->purge);
-        }
-        UNLOCK (&table->lock);
-
-        list_for_each_entry_safe (del, tmp, &purge, list) {
-                list_del_init (&del->list);
-                if (del->primary) {
-                        ret = del->primary->close (del->primary, 0);
-                        if (ret != 0) {
-                                gf_log (table->this->name, GF_LOG_DEBUG,
-                                        "_BCTX_TABLE_PRUNE %s: %s "
-                                        "(failed to close primary database)",
-                                        del->directory, db_strerror (ret));
-                        } else {
-                                gf_log (table->this->name, GF_LOG_DEBUG,
-                                        "_BCTX_TABLE_PRUNE %s (lru=%d)"
-                                        "(closed primary database)",
-                                        del->directory, table->lru_size);
-                        }
-                }
-                if (del->secondary) {
-                        ret = del->secondary->close (del->secondary, 0);
-                        if (ret != 0) {
-                                gf_log (table->this->name, GF_LOG_DEBUG,
-                                        "_BCTX_TABLE_PRUNE %s: %s "
-                                        "(failed to close secondary database)",
-                                        del->directory, db_strerror (ret));
-                        } else {
-                                gf_log (table->this->name, GF_LOG_DEBUG,
-                                        "_BCTX_TABLE_PRUNE %s (lru=%d)"
-                                        "(closed secondary database)",
-                                        del->directory, table->lru_size);
-                        }
-                }
-                __destroy_bctx (del);
-        }
-
-        return ret;
-}
-
-
-/* struct bdb_ctx related */
-static inline uint32_t
-bdb_key_hash (char *key, uint32_t hash_size)
-{
-        uint32_t hash = 0;
-
-        hash = *key;
-
-        if (hash) {
-                for (key += 1; *key != '\0'; key++) {
-                        hash = (hash << 5) - hash + *key;
-                }
-        }
-
-        return (hash + *key) % hash_size;
-}
-
-static void
-__hash_bctx (bctx_t *bctx)
-{
-        bctx_table_t *table = NULL;
-        char *key = NULL;
-
-        table = bctx->table;
-
-        MAKE_KEY_FROM_PATH (key, bctx->directory);
-        bctx->key_hash = bdb_key_hash (key, table->hash_size);
-
-        list_del_init (&bctx->b_hash);
-        list_add (&bctx->b_hash, &table->b_hash[bctx->key_hash]);
-}
-
-static inline bctx_t *
-__bctx_passivate (bctx_t *bctx)
-{
-        if (bctx->primary) {
-                list_move_tail (&bctx->list, &(bctx->table->b_lru));
-                bctx->table->lru_size++;
-        } else {
-                list_move_tail (&bctx->list, &bctx->table->purge);
-                __unhash_bctx (bctx);
-        }
-        return bctx;
-}
-
-static inline bctx_t *
-__bctx_activate (bctx_t *bctx)
-{
-        list_move (&bctx->list, &bctx->table->active);
-        bctx->table->lru_size--;
-
-        return bctx;
-}
-
-static bctx_t *
-__bdb_ctx_unref (bctx_t *bctx)
-{
-        GF_ASSERT (bctx->ref);
-
-        --bctx->ref;
-
-        if (!bctx->ref)
-                bctx = __bctx_passivate (bctx);
-
-        return bctx;
-}
-
-
-bctx_t *
-bctx_unref (bctx_t *bctx)
-{
-        bctx_table_t *table = NULL;
-
-        if (!bctx && !bctx->table)
-                return NULL;
-
-        table = bctx->table;
-
-        LOCK (&table->lock);
-        {
-                bctx = __bdb_ctx_unref (bctx);
-        }
-        UNLOCK (&table->lock);
-
-        bctx_table_prune (table);
-
-        return bctx;
-}
-
-/*
- * NOTE: __bdb_ctx_ref() is called only after holding table->lock and
- * bctx->lock, in that order
- */
-static inline bctx_t *
-__bctx_ref (bctx_t *bctx)
-{
-        if (!bctx->ref)
-                __bctx_activate (bctx);
-
-        bctx->ref++;
-
-        return bctx;
-}
-
-bctx_t *
-bctx_ref (bctx_t *bctx)
-{
-        LOCK (&(bctx->table->lock));
-        {
-                __bctx_ref (bctx);
-        }
-        UNLOCK (&(bctx->table->lock));
-
-        return bctx;
-}
-
-
-#define BDB_THIS(table) (table->this)
-
-static inline bctx_t *
-__create_bctx (bctx_table_t *table,
-               const char *path)
-{
-        bctx_t *bctx = NULL;
-        char *db_path = NULL;
-
-        bctx = GF_CALLOC (1, sizeof (*bctx), gf_bdb_mt_bctx_t);
-        GF_VALIDATE_OR_GOTO ("bctx", bctx, out);
-
-        bctx->table = table;
-        bctx->directory = gf_strdup (path);
-        GF_VALIDATE_OR_GOTO ("bctx", bctx->directory, out);
-
-        MAKE_REAL_PATH_TO_STORAGE_DB (db_path, BDB_THIS (table), path);
-
-        bctx->db_path = gf_strdup (db_path);
-        GF_VALIDATE_OR_GOTO ("bctx", bctx->directory, out);
-
-        INIT_LIST_HEAD (&bctx->c_list);
-        INIT_LIST_HEAD (&bctx->list);
-        INIT_LIST_HEAD (&bctx->b_hash);
-
-        LOCK_INIT (&bctx->lock);
-
-        __hash_bctx (bctx);
-
-        list_add (&bctx->list, &table->b_lru);
-        table->lru_size++;
-
-out:
-        return bctx;
-}
-
-/* bctx_lookup - lookup bctx_t for the directory @directory.
- *               (see description of bctx_t in bdb.h)
- *
- * @table:     bctx_table_t for this instance of bdb.
- * @directory: directory for which bctx_t is being looked up.
- */
-bctx_t *
-bctx_lookup (bctx_table_t *table,
-             const char *directory)
-{
-        char    *key = NULL;
-        uint32_t key_hash = 0;
-        bctx_t  *trav = NULL, *bctx = NULL, *tmp = NULL;
-        int32_t  need_break = 0;
-
-        GF_VALIDATE_OR_GOTO ("bctx", table, out);
-        GF_VALIDATE_OR_GOTO ("bctx", directory, out);
-
-        MAKE_KEY_FROM_PATH (key, directory);
-        key_hash = bdb_key_hash (key, table->hash_size);
-
-        LOCK (&table->lock);
-        {
-                if (list_empty (&table->b_hash[key_hash])) {
-                        goto creat_bctx;
-                }
-
-                list_for_each_entry_safe (trav, tmp, &table->b_hash[key_hash],
-                                          b_hash) {
-                        LOCK(&trav->lock);
-                        {
-                                if (!strcmp(trav->directory, directory)) {
-                                        bctx = __bctx_ref (trav);
-                                        need_break = 1;
-                                }
-                        }
-                        UNLOCK(&trav->lock);
-
-                        if (need_break)
-                                break;
-                }
-
-        creat_bctx:
-                if (!bctx) {
-                        bctx = __create_bctx (table, directory);
-                        bctx = __bctx_ref (bctx);
-                }
-        }
-        UNLOCK (&table->lock);
-out:
-        return bctx;
-}
-
-
-bctx_t *
-bctx_parent (bctx_table_t *table,
-             const char *path)
-{
-        char   *pathname = NULL, *directory = NULL;
-        bctx_t *bctx = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bctx", table, out);
-        GF_VALIDATE_OR_GOTO ("bctx", path, out);
-
-        pathname = gf_strdup (path);
-        GF_VALIDATE_OR_GOTO ("bctx", pathname, out);
-        directory = dirname (pathname);
-
-        bctx = bctx_lookup (table, directory);
-        GF_VALIDATE_OR_GOTO ("bctx", bctx, out);
-
-out:
-        if (pathname)
-                free (pathname);
-        return bctx;
-}
diff --git a/xlators/storage/bdb/src/bdb-ll.c b/xlators/storage/bdb/src/bdb-ll.c
deleted file mode 100644
index f70ec47f4..000000000
--- a/xlators/storage/bdb/src/bdb-ll.c
+++ /dev/null
@@ -1,1464 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <libgen.h>
-#include "bdb.h"
-#include <list.h>
-#include "hashfn.h"
-/*
- * implement the procedures to interact with bdb */
-
-/****************************************************************
- *
- * General wrappers and utility procedures for bdb xlator
- *
- ****************************************************************/
-
-ino_t
-bdb_inode_transform (ino_t parent,
-                     const char *name,
-                     size_t namelen)
-{
-        ino_t               ino = -1;
-        uint64_t            hash = 0;
-
-        hash = gf_dm_hashfn (name, namelen);
-
-        ino = (((parent << 32) | 0x00000000ffffffffULL)
-               & (hash | 0xffffffff00000000ULL));
-
-        return ino;
-}
-
-static int
-bdb_generate_secondary_hash (DB *secondary,
-                             const DBT *pkey,
-                             const DBT *data,
-                             DBT *skey)
-{
-        char *primary = NULL;
-        uint32_t *hash = NULL;
-
-        primary = pkey->data;
-
-        hash = GF_CALLOC (1, sizeof (uint32_t), gf_bdb_mt_uint32_t);
-
-        *hash = gf_dm_hashfn (primary, pkey->size);
-
-        skey->data = hash;
-        skey->size = sizeof (hash);
-        skey->flags = DB_DBT_APPMALLOC;
-
-        return 0;
-}
-
-/***********************************************************
- *
- *  bdb storage database utilities
- *
- **********************************************************/
-
-/*
- * bdb_db_open - opens a storage db.
- *
- * @ctx: context specific to the directory for which we are supposed to open db
- *
- * see, if we have empty slots to open a db.
- *      if (no-empty-slots), then prune open dbs and close as many as possible
- *      if (empty-slot-available), tika muchkonDu db open maaDu
- *
- */
-static int
-bdb_db_open (bctx_t *bctx)
-{
-        DB *primary   = NULL;
-        DB *secondary = NULL;
-        int32_t ret = -1;
-        bctx_table_t *table = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", bctx, out);
-
-        table = bctx->table;
-        GF_VALIDATE_OR_GOTO ("bdb-ll", table, out);
-
-        /* we have to do the following, we can't deny someone of db_open ;) */
-        ret = db_create (&primary, table->dbenv, 0);
-        if (ret < 0) {
-                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                        "_BDB_DB_OPEN %s: %s (failed to create database object"
-                        " for primary database)",
-                        bctx->directory, db_strerror (ret));
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        if (table->page_size) {
-                ret = primary->set_pagesize (primary,
-                                             table->page_size);
-                if (ret < 0) {
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_OPEN %s: %s (failed to set page-size "
-                                "to %"PRIu64")",
-                                bctx->directory, db_strerror (ret),
-                                table->page_size);
-                } else {
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_OPEN %s: page-size set to %"PRIu64,
-                                bctx->directory, table->page_size);
-                }
-        }
-
-        ret = primary->open (primary, NULL, bctx->db_path, "primary",
-                             table->access_mode, table->dbflags, 0);
-        if (ret < 0) {
-                gf_log ("bdb-ll", GF_LOG_ERROR,
-                        "_BDB_DB_OPEN %s: %s "
-                        "(failed to open primary database)",
-                        bctx->directory, db_strerror (ret));
-                ret = -1;
-                goto cleanup;
-        }
-
-        ret = db_create (&secondary, table->dbenv, 0);
-        if (ret < 0) {
-                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                        "_BDB_DB_OPEN %s: %s (failed to create database object"
-                        " for secondary database)",
-                        bctx->directory, db_strerror (ret));
-                ret = -ENOMEM;
-                goto cleanup;
-        }
-
-        ret = secondary->open (secondary, NULL, bctx->db_path, "secondary",
-                               table->access_mode, table->dbflags, 0);
-        if (ret != 0 ) {
-                gf_log ("bdb-ll", GF_LOG_ERROR,
-                        "_BDB_DB_OPEN %s: %s "
-                        "(failed to open secondary database)",
-                        bctx->directory, db_strerror (ret));
-                ret = -1;
-                goto cleanup;
-        }
-
-        ret = primary->associate (primary, NULL, secondary,
-                                  bdb_generate_secondary_hash,
-#ifdef DB_IMMUTABLE_KEY
-                                  DB_IMMUTABLE_KEY);
-#else
-                                  0);
-#endif
-        if (ret != 0 ) {
-                gf_log ("bdb-ll", GF_LOG_ERROR,
-                        "_BDB_DB_OPEN %s: %s "
-                        "(failed to associate primary database with "
-                        "secondary database)",
-                        bctx->directory, db_strerror (ret));
-                ret = -1;
-                goto cleanup;
-        }
-
-out:
-        bctx->primary = primary;
-        bctx->secondary = secondary;
-
-        return ret;
-cleanup:
-        if (primary)
-                primary->close (primary, 0);
-        if (secondary)
-                secondary->close (secondary, 0);
-
-        return ret;
-}
-
-int32_t
-bdb_cursor_close (bctx_t *bctx,
-                  DBC *cursorp)
-{
-        int32_t ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", bctx, out);
-        GF_VALIDATE_OR_GOTO ("bdb-ll", cursorp, out);
-
-        LOCK (&bctx->lock);
-        {
-#ifdef HAVE_BDB_CURSOR_GET
-                ret = cursorp->close (cursorp);
-#else
-                ret = cursorp->c_close (cursorp);
-#endif
-                if (ret < 0) {
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_CURSOR_CLOSE %s: %s "
-                                "(failed to close database cursor)",
-                                bctx->directory, db_strerror (ret));
-                }
-        }
-        UNLOCK (&bctx->lock);
-
-out:
-        return ret;
-}
-
-
-int32_t
-bdb_cursor_open (bctx_t *bctx,
-                 DBC **cursorpp)
-{
-        int32_t ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", bctx, out);
-        GF_VALIDATE_OR_GOTO ("bdb-ll", cursorpp, out);
-
-        LOCK (&bctx->lock);
-        {
-                if (bctx->secondary) {
-                        /* do nothing, just continue */
-                        ret = 0;
-                } else {
-                        ret = bdb_db_open (bctx);
-                        if (ret < 0) {
-                                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                        "_BDB_CURSOR_OPEN %s: ENOMEM "
-                                        "(failed to open secondary database)",
-                                        bctx->directory);
-                                ret = -ENOMEM;
-                        } else {
-                                ret = 0;
-                        }
-                }
-
-                if (ret == 0) {
-                        /* all set, open cursor */
-                        ret = bctx->secondary->cursor (bctx->secondary,
-                                                       NULL, cursorpp, 0);
-                        if (ret < 0) {
-                                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                        "_BDB_CURSOR_OPEN %s: %s "
-                                        "(failed to open a cursor to database)",
-                                        bctx->directory, db_strerror (ret));
-                        }
-                }
-        }
-        UNLOCK (&bctx->lock);
-
-out:
-        return ret;
-}
-
-
-/* cache related */
-static bdb_cache_t *
-bdb_cache_lookup (bctx_t *bctx,
-                  char *path)
-{
-        bdb_cache_t *bcache = NULL;
-        bdb_cache_t *trav   = NULL;
-        char        *key    = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", bctx, out);
-        GF_VALIDATE_OR_GOTO ("bdb-ll", path, out);
-
-        MAKE_KEY_FROM_PATH (key, path);
-
-        LOCK (&bctx->lock);
-        {
-                list_for_each_entry (trav, &bctx->c_list, c_list) {
-                        if (!strcmp (trav->key, key)){
-                                bcache = trav;
-                                break;
-                        }
-                }
-        }
-        UNLOCK (&bctx->lock);
-
-out:
-        return bcache;
-}
-
-static int32_t
-bdb_cache_insert (bctx_t *bctx,
-                  DBT *key,
-                  DBT *data)
-{
-        bdb_cache_t *bcache = NULL;
-        int32_t ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", bctx, out);
-        GF_VALIDATE_OR_GOTO ("bdb-ll", key, out);
-        GF_VALIDATE_OR_GOTO ("bdb-ll", data, out);
-
-        LOCK (&bctx->lock);
-        {
-                if (bctx->c_count > 5) {
-                        /* most of the times, we enter here */
-                        /* FIXME: ugly, not supposed to disect any of the
-                         * 'struct list_head' directly */
-                        if (!list_empty (&bctx->c_list)) {
-                                bcache = list_entry (bctx->c_list.prev,
-                                                     bdb_cache_t, c_list);
-                                list_del_init (&bcache->c_list);
-                        }
-                        if (bcache->key) {
-                                GF_FREE (bcache->key);
-                                bcache->key = GF_CALLOC (key->size + 1,
-                                                         sizeof (char), 
-                                                         gf_bdb_mt_char);
-                                GF_VALIDATE_OR_GOTO ("bdb-ll",
-                                                     bcache->key, unlock);
-                                memcpy (bcache->key, (char *)key->data,
-                                        key->size);
-                        } else {
-                                /* should never come here */
-                                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                        "_BDB_CACHE_INSERT %s (%s) "
-                                        "(found a cache entry with empty key)",
-                                        bctx->directory, (char *)key->data);
-                        } /* if(bcache->key)...else */
-                        if (bcache->data) {
-                                GF_FREE (bcache->data);
-                                bcache->data = memdup (data->data, data->size);
-                                GF_VALIDATE_OR_GOTO ("bdb-ll", bcache->data,
-                                                     unlock);
-                                bcache->size = data->size;
-                        } else {
-                                /* should never come here */
-                                gf_log ("bdb-ll", GF_LOG_CRITICAL,
-                                        "_BDB_CACHE_INSERT %s (%s) "
-                                        "(found a cache entry with no data)",
-                                        bctx->directory, (char *)key->data);
-                        } /* if(bcache->data)...else */
-                        list_add (&bcache->c_list, &bctx->c_list);
-                        ret = 0;
-                } else {
-                        /* we will be entering here very rarely */
-                        bcache = GF_CALLOC (1, sizeof (*bcache), 
-                                            gf_bdb_mt_bdb_cache_t);
-                        GF_VALIDATE_OR_GOTO ("bdb-ll", bcache, unlock);
-
-                        bcache->key = GF_CALLOC (key->size + 1, sizeof (char),
-                                                 gf_bdb_mt_char);
-                        GF_VALIDATE_OR_GOTO ("bdb-ll", bcache->key, unlock);
-                        memcpy (bcache->key, key->data, key->size);
-
-                        bcache->data = memdup (data->data, data->size);
-                        GF_VALIDATE_OR_GOTO ("bdb-ll", bcache->data, unlock);
-
-                        bcache->size = data->size;
-                        list_add (&bcache->c_list, &bctx->c_list);
-                        bctx->c_count++;
-                        ret = 0;
-                } /* if(private->c_count < 5)...else */
-        }
-unlock:
-        UNLOCK (&bctx->lock);
-out:
-        return ret;
-}
-
-static int32_t
-bdb_cache_delete (bctx_t *bctx,
-                  const char *key)
-{
-        bdb_cache_t *bcache = NULL;
-        bdb_cache_t *trav   = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", bctx, out);
-        GF_VALIDATE_OR_GOTO ("bdb-ll", key, out);
-
-        LOCK (&bctx->lock);
-        {
-                list_for_each_entry (trav, &bctx->c_list, c_list) {
-                        if (!strcmp (trav->key, key)){
-                                bctx->c_count--;
-                                bcache = trav;
-                                break;
-                        }
-                }
-
-                if (bcache) {
-                        list_del_init (&bcache->c_list);
-                        GF_FREE (bcache->key);
-                        GF_FREE (bcache->data);
-                        GF_FREE (bcache);
-                }
-        }
-        UNLOCK (&bctx->lock);
-
-out:
-        return 0;
-}
-
-void *
-bdb_db_stat (bctx_t *bctx,
-             DB_TXN *txnid,
-             uint32_t flags)
-{
-        DB     *storage = NULL;
-        void   *stat    = NULL;
-        int32_t ret     = -1;
-
-        LOCK (&bctx->lock);
-        {
-                if (bctx->primary == NULL) {
-                        ret = bdb_db_open (bctx);
-                        storage = bctx->primary;
-                } else {
-                        /* we are just fine, lets continue */
-                        storage = bctx->primary;
-                } /* if(bctx->dbp==NULL)...else */
-        }
-        UNLOCK (&bctx->lock);
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", storage, out);
-
-        ret = storage->stat (storage, txnid, &stat, flags);
-
-        if (ret < 0) {
-                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                        "_BDB_DB_STAT %s: %s "
-                        "(failed to do stat database)",
-                        bctx->directory, db_strerror (ret));
-        }
-out:
-        return stat;
-
-}
-
-/* bdb_storage_get - retrieve a key/value pair corresponding to @path from the
- *  corresponding db file.
- *
- * @bctx: bctx_t * corresponding to the parent directory of @path. (should
- *  always be a valid bctx).  bdb_storage_get should never be called if
- *  @bctx = NULL.
- * @txnid: NULL if bdb_storage_get is not embedded in an explicit transaction
- *  or a valid DB_TXN *, when embedded in an explicit transaction.
- * @path: path of the file to read from (translated to a database key using
- *  MAKE_KEY_FROM_PATH)
- * @buf: char ** - pointer to a pointer to char. a read buffer is created in
- *  this procedure and pointer to the buffer is passed through @buf to the
- *  caller.
- * @size: size of the file content to be read.
- * @offset: offset from which the file content to be read.
- *
- * NOTE: bdb_storage_get tries to open DB, if @bctx->dbp == NULL
- *  (@bctx->dbp == NULL, nobody has opened DB till now or DB was closed by
- *  bdb_table_prune()).
- *
- * NOTE: if private->cache is set (bdb xlator's internal caching enabled), then
- *  bdb_storage_get first looks up the cache for key/value pair. if
- *  bdb_lookup_cache fails, then only DB->get() is called. also,  inserts a
- *  newly read key/value pair to cache through bdb_insert_to_cache.
- *
- * return: 'number of bytes read' on success or -1 on error.
- *
- * also see: bdb_lookup_cache, bdb_insert_to_cache for details about bdb
- *  xlator's internal cache.
- */
-static int32_t
-bdb_db_get (bctx_t *bctx,
-            DB_TXN *txnid,
-            const char *path,
-            char *buf,
-            size_t size,
-            off_t offset)
-{
-        DB          *storage    = NULL;
-        DBT          key        = {0,};
-        DBT          value      = {0,};
-        int32_t      ret        = -1;
-        size_t       copy_size  = 0;
-        char        *key_string = NULL;
-        bdb_cache_t *bcache     = NULL;
-        int32_t      db_flags   = 0;
-        uint8_t      need_break = 0;
-        int32_t      retries    = 1;
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", bctx, out);
-        GF_VALIDATE_OR_GOTO ("bdb-ll", path, out);
-
-        MAKE_KEY_FROM_PATH (key_string, path);
-
-        if (bctx->cache &&
-            ((bcache = bdb_cache_lookup (bctx, key_string)) != NULL)) {
-                if (buf) {
-                        copy_size = ((bcache->size - offset) < size)?
-                                (bcache->size - offset) : size;
-
-                        memcpy (buf, (bcache->data + offset), copy_size);
-                        ret = copy_size;
-                } else {
-                        ret = bcache->size;
-                }
-                
-                goto out;
-        } 
-
-        LOCK (&bctx->lock);
-        {
-                if (bctx->primary == NULL) {
-                        ret = bdb_db_open (bctx);
-                        storage = bctx->primary;
-                } else {
-                        /* we are just fine, lets continue */
-                        storage = bctx->primary;
-                } /* if(bctx->dbp==NULL)...else */
-        }
-        UNLOCK (&bctx->lock);
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", storage, out);
-
-        key.data = (char *)key_string;
-        key.size = strlen (key_string);
-        key.flags = DB_DBT_USERMEM;
-
-        if (bctx->cache){
-                value.flags = DB_DBT_MALLOC;
-        } else {
-                if (size) {
-                        value.data  = buf;
-                        value.ulen  = size;
-                        value.flags = DB_DBT_USERMEM | DB_DBT_PARTIAL;
-                } else {
-                        value.flags = DB_DBT_MALLOC;
-                }
-                value.dlen = size;
-                value.doff = offset;
-        }
-
-        do {
-                /* TODO: we prefer to give our own buffer to value.data
-                 * and ask bdb to fill in it */
-                ret = storage->get (storage, txnid, &key, &value,
-                                    db_flags);
-
-                if (ret == DB_NOTFOUND) {
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_GET %s - %s: ENOENT"
-                                "(specified key not found in database)",
-                                bctx->directory, key_string);
-                        ret = -1;
-                        need_break = 1;
-                } else if (ret == DB_LOCK_DEADLOCK) {
-                        retries++;
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_GET %s - %s"
-                                "(deadlock detected, retrying for %d "
-                                "time)",
-                                bctx->directory, key_string, retries);
-                } else if (ret == 0) {
-                        /* successfully read data, lets set everything
-                         * in place and return */
-                        if (bctx->cache) {
-                                if (buf) {
-                                        copy_size = ((value.size - offset) < size) ?
-                                                (value.size - offset) : size;
-
-                                        memcpy (buf, (value.data + offset),
-                                                copy_size);
-                                        ret = copy_size;
-                                }
-
-                                bdb_cache_insert (bctx, &key, &value);
-                        } else {
-                                ret = value.size;
-                        }
-
-                        if (size == 0)
-                                GF_FREE (value.data);
-
-                        need_break = 1;
-                } else {
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_GET %s - %s: %s"
-                                "(failed to retrieve specified key from"
-                                " database)",
-                                bctx->directory, key_string,
-                                db_strerror (ret));
-                        ret = -1;
-                        need_break = 1;
-                }
-        } while (!need_break);
-
-out:
-        return ret;
-}/* bdb_db_get */
-
-/* TODO: handle errors here and log. propogate only the errno to caller */
-int32_t
-bdb_db_fread (struct bdb_fd *bfd, char *buf, size_t size, off_t offset)
-{
-        return bdb_db_get (bfd->ctx, NULL, bfd->key, buf, size, offset);
-}
-
-int32_t
-bdb_db_iread (struct bdb_ctx *bctx, const char *key, char **bufp)
-{
-        char *buf = NULL;
-        size_t size = 0;
-        int64_t ret = 0;
-
-        ret = bdb_db_get (bctx, NULL, key, NULL, 0, 0);
-        size = ret;
-
-        if (bufp) {
-                buf = GF_CALLOC (size, sizeof (char), gf_bdb_mt_char);
-                *bufp = buf;
-                ret = bdb_db_get (bctx, NULL, key, buf, size, 0);
-        }
-
-        return ret; 
-}
-
-/* bdb_storage_put - insert a key/value specified to the corresponding DB.
- *
- * @bctx: bctx_t * corresponding to the parent directory of @path.
- *        (should always be a valid bctx). bdb_storage_put should never be
- *         called if @bctx = NULL.
- * @txnid: NULL if bdb_storage_put is not embedded in an explicit transaction
- *         or a valid DB_TXN *, when embedded in an explicit transaction.
- * @key_string: key of the database entry.
- * @buf: pointer to the buffer data to be written as data for @key_string.
- * @size: size of @buf.
- * @offset: offset in the key's data to be modified with provided data.
- * @flags: valid flags are BDB_TRUNCATE_RECORD (to reduce the data of
- *         @key_string to 0 size).
- *
- * NOTE: bdb_storage_put tries to open DB, if @bctx->dbp == NULL
- *      (@bctx->dbp == NULL, nobody has opened DB till now or DB was closed by
- *       bdb_table_prune()).
- *
- * NOTE: bdb_storage_put deletes the key/value from bdb xlator's internal cache.
- *
- * return: 0 on success or -1 on error.
- *
- * also see: bdb_cache_delete for details on how a cached key/value pair is
- * removed.
- */
-static int32_t
-bdb_db_put (bctx_t *bctx,
-            DB_TXN *txnid,
-            const char *key_string,
-            const char *buf,
-            size_t size,
-            off_t offset,
-            int32_t flags)
-{
-        DB     *storage = NULL;
-        DBT     key = {0,}, value = {0,};
-        int32_t ret = -1;
-        int32_t db_flags = DB_AUTO_COMMIT;
-        uint8_t need_break = 0;
-        int32_t retries = 1;
-
-        LOCK (&bctx->lock);
-        {
-                if (bctx->primary == NULL) {
-                        ret = bdb_db_open (bctx);
-                        storage = bctx->primary;
-                } else {
-                        /* we are just fine, lets continue */
-                        storage = bctx->primary;
-                }
-        }
-        UNLOCK (&bctx->lock);
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", storage, out);
-
-        if (bctx->cache) {
-                ret = bdb_cache_delete (bctx, (char *)key_string);
-                GF_VALIDATE_OR_GOTO ("bdb-ll", (ret == 0), out);
-        }
-
-        key.data = (void *)key_string;
-        key.size = strlen (key_string);
-
-        /* NOTE: bdb lets us expand the file, suppose value.size > value.len,
-         * then value.len bytes from value.doff offset and value.size bytes
-         * will be written from value.doff and data from
-         * value.doff + value.dlen will be pushed value.doff + value.size
-         */
-        value.data = (void *)buf;
-
-        if (flags & BDB_TRUNCATE_RECORD) {
-                value.size = size;
-                value.doff = 0;
-                value.dlen = offset;
-        } else {
-                value.size = size;
-                value.dlen = size;
-                value.doff = offset;
-        }
-        value.flags = DB_DBT_PARTIAL;
-        if (buf == NULL && size == 0)
-                /* truncate called us */
-                value.flags = 0;
-
-        do {
-                ret = storage->put (storage, txnid, &key, &value, db_flags);
-                if (ret == DB_LOCK_DEADLOCK) {
-                        retries++;
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_PUT %s - %s"
-                                "(deadlock detected, retying for %d time)",
-                                bctx->directory, key_string, retries);
-                } else if (ret) {
-                        /* write failed */
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_PUT %s - %s: %s"
-                                "(failed to put specified entry into database)",
-                                bctx->directory, key_string, db_strerror (ret));
-                        need_break = 1;
-                } else {
-                        /* successfully wrote */
-                        ret = 0;
-                        need_break = 1;
-                }
-        } while (!need_break);
-out:
-        return ret;
-}/* bdb_db_put */
-
-int32_t
-bdb_db_icreate (struct bdb_ctx *bctx, const char *key)
-{
-        return bdb_db_put (bctx, NULL, key, NULL, 0, 0, 0);
-}
-
-/* TODO: handle errors here and log. propogate only the errno to caller */
-int32_t
-bdb_db_fwrite (struct bdb_fd *bfd, char *buf, size_t size, off_t offset)
-{
-        return bdb_db_put (bfd->ctx, NULL, bfd->key, buf, size, offset, 0);
-}
-
-/* TODO: handle errors here and log. propogate only the errno to caller */
-int32_t
-bdb_db_iwrite (struct bdb_ctx *bctx, const char *key, char *buf, size_t size)
-{
-        return bdb_db_put (bctx, NULL, key, buf, size, 0, 0);
-}
-
-int32_t
-bdb_db_itruncate (struct bdb_ctx *bctx, const char *key)
-{
-        return bdb_db_put (bctx, NULL, key, NULL, 0, 1, 0);
-}
-
-/* bdb_storage_del - delete a key/value pair corresponding to @path from
- *  corresponding db file.
- *
- * @bctx: bctx_t * corresponding to the parent directory of @path.
- *       (should always be a valid bctx). bdb_storage_del should never be called
- *       if @bctx = NULL.
- * @txnid: NULL if bdb_storage_del is not embedded in an explicit transaction
- *   or a valid DB_TXN *, when embedded in an explicit transaction.
- * @path: path to the file, whose key/value pair has to be deleted.
- *
- * NOTE: bdb_storage_del tries to open DB, if @bctx->dbp == NULL
- *  (@bctx->dbp == NULL, nobody has opened DB till now or DB was closed by
- *  bdb_table_prune()).
- *
- * return: 0 on success or -1 on error.
- */
-static int32_t
-bdb_db_del (bctx_t *bctx,
-            DB_TXN *txnid,
-            const char *key_string)
-{
-        DB     *storage    = NULL;
-        DBT     key        = {0,};
-        int32_t ret        = -1;
-        int32_t db_flags   = 0;
-        uint8_t need_break = 0;
-        int32_t retries    = 1;
-
-        LOCK (&bctx->lock);
-        {
-                if (bctx->primary == NULL) {
-                        ret = bdb_db_open (bctx);
-                        storage = bctx->primary;
-                } else {
-                        /* we are just fine, lets continue */
-                        storage = bctx->primary;
-                }
-        }
-        UNLOCK (&bctx->lock);
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", storage, out);
-
-        ret = bdb_cache_delete (bctx, key_string);
-        GF_VALIDATE_OR_GOTO ("bdb-ll", (ret == 0), out);
-
-        key.data = (char *)key_string;
-        key.size = strlen (key_string);
-        key.flags = DB_DBT_USERMEM;
-
-        do {
-                ret = storage->del (storage, txnid, &key, db_flags);
-
-                if (ret == DB_NOTFOUND) {
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_DEL %s - %s: ENOENT"
-                                "(failed to delete entry, could not be "
-                                "found in the database)",
-                                bctx->directory, key_string);
-                        need_break = 1;
-                } else if (ret == DB_LOCK_DEADLOCK) {
-                        retries++;
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_DEL %s - %s"
-                                "(deadlock detected, retying for %d time)",
-                                bctx->directory, key_string, retries);
-                } else if (ret == 0) {
-                        /* successfully deleted the entry */
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_DEL %s - %s"
-                                "(successfully deleted entry from database)",
-                                bctx->directory, key_string);
-                        ret = 0;
-                        need_break = 1;
-                } else {
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "_BDB_DB_DEL %s - %s: %s"
-                                "(failed to delete entry from database)",
-                                bctx->directory, key_string, db_strerror (ret));
-                        ret = -1;
-                        need_break = 1;
-                }
-        } while (!need_break);
-out:
-        return ret;
-}
-
-int32_t
-bdb_db_iremove (bctx_t *bctx,
-                const char *key)
-{
-        return bdb_db_del (bctx, NULL, key);
-}
-
-/* NOTE: bdb version compatibility wrapper */
-int32_t
-bdb_cursor_get (DBC *cursorp,
-                DBT *sec, DBT *pri,
-                DBT *val,
-                int32_t flags)
-{
-        int32_t ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("bdb-ll", cursorp, out);
-
-#ifdef HAVE_BDB_CURSOR_GET
-        ret = cursorp->pget (cursorp, sec, pri, val, flags);
-#else
-        ret = cursorp->c_pget (cursorp, sec, pri, val, flags);
-#endif
-        if ((ret != 0)  && (ret != DB_NOTFOUND)) {
-                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                        "_BDB_CURSOR_GET: %s"
-                        "(failed to retrieve entry from database cursor)",
-                        db_strerror (ret));
-        }
-
-out:
-        return ret;
-}/* bdb_cursor_get */
-
-int32_t
-bdb_dirent_size (DBT *key)
-{
-        return GF_DIR_ALIGN (24 /* FIX MEEEE!!! */ + key->size);
-}
-
-
-
-/* bdb_dbenv_init - initialize DB_ENV
- *
- *  initialization includes:
- *   1. opening DB_ENV (db_env_create(), DB_ENV->open()).
- *      NOTE: see private->envflags for flags used.
- *   2. DB_ENV->set_lg_dir - set log directory to be used for storing log files
- *     (log files are the files in which transaction logs are written by db).
- *   3. DB_ENV->set_flags (DB_LOG_AUTOREMOVE) - set DB_ENV to automatically
- *      clear the unwanted log files (flushed at each checkpoint).
- *   4. DB_ENV->set_errfile - set errfile to be used by db to report detailed
- *      error logs. used only for debbuging purpose.
- *
- * return: returns a valid DB_ENV * on success or NULL on error.
- *
- */
-static DB_ENV *
-bdb_dbenv_init (xlator_t *this,
-                char *directory)
-{
-        /* Create a DB environment */
-        DB_ENV        *dbenv       = NULL;
-        int32_t        ret         = 0;
-        bdb_private_t *private     = NULL;
-        int32_t        fatal_flags = 0;
-
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (directory, err);
-
-        private = this->private;
-        VALIDATE_OR_GOTO (private, err);
-
-        ret = db_env_create (&dbenv, 0);
-        VALIDATE_OR_GOTO ((ret == 0), err);
-
-        /* NOTE: set_errpfx returns 'void' */
-        dbenv->set_errpfx(dbenv, this->name);
-
-        ret = dbenv->set_lk_detect (dbenv, DB_LOCK_DEFAULT);
-        VALIDATE_OR_GOTO ((ret == 0), err);
-
-        ret = dbenv->open(dbenv, directory,
-                          private->envflags,
-                          S_IRUSR | S_IWUSR);
-        if ((ret != 0) && (ret != DB_RUNRECOVERY)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "failed to join Berkeley DB environment at %s: %s."
-                        "please run manual recovery and retry running "
-                        "glusterfs",
-                        directory, db_strerror (ret));
-                dbenv = NULL;
-                goto err;
-        } else if (ret == DB_RUNRECOVERY) {
-                fatal_flags = ((private->envflags & (~DB_RECOVER))
-                               | DB_RECOVER_FATAL);
-                ret = dbenv->open(dbenv, directory, fatal_flags,
-                                  S_IRUSR | S_IWUSR);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_CRITICAL,
-                                "failed to join Berkeley DB environment in "
-                                "recovery mode at %s: %s. please run manual "
-                                "recovery and retry running glusterfs",
-                                directory, db_strerror (ret));
-                        dbenv = NULL;
-                        goto err;
-                }
-        }
-
-        ret = 0;
-#if (DB_VERSION_MAJOR == 4 &&                   \
-     DB_VERSION_MINOR == 7)
-        if (private->log_auto_remove) {
-                ret = dbenv->log_set_config (dbenv, DB_LOG_AUTO_REMOVE, 1);
-        } else {
-                ret = dbenv->log_set_config (dbenv, DB_LOG_AUTO_REMOVE, 0);
-        }
-#else
-        if (private->log_auto_remove) {
-                ret = dbenv->set_flags (dbenv, DB_LOG_AUTOREMOVE, 1);
-        } else {
-                ret = dbenv->set_flags (dbenv, DB_LOG_AUTOREMOVE, 0);
-        }
-#endif
-        if (ret < 0) {
-                gf_log ("bdb-ll", GF_LOG_ERROR,
-                        "autoremoval of transactional log files could not be "
-                        "configured (%s). you may have to do a manual "
-                        "monitoring of transactional log files and remove "
-                        "periodically.",
-                        db_strerror (ret));
-                goto err;
-        }
-
-        if (private->transaction) {
-                ret = dbenv->set_flags(dbenv, DB_AUTO_COMMIT, 1);
-
-                if (ret != 0) {
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "configuration of auto-commit failed for "
-                                "database environment at %s. none of the "
-                                "operations will be embedded in transaction "
-                                "unless explicitly done so.",
-                                db_strerror (ret));
-                        goto err;
-                }
-
-                if (private->txn_timeout) {
-                        ret = dbenv->set_timeout (dbenv, private->txn_timeout,
-                                                  DB_SET_TXN_TIMEOUT);
-                        if (ret != 0) {
-                                gf_log ("bdb-ll", GF_LOG_ERROR,
-                                        "could not configure Berkeley DB "
-                                        "transaction timeout to %d (%s). please"
-                                        " review 'option transaction-timeout %d"
-                                        "' option.",
-                                        private->txn_timeout,
-                                        db_strerror (ret),
-                                        private->txn_timeout);
-                                goto err;
-                        }
-                }
-
-                if (private->lock_timeout) {
-                        ret = dbenv->set_timeout(dbenv,
-                                                 private->txn_timeout,
-                                                 DB_SET_LOCK_TIMEOUT);
-                        if (ret < 0) {
-                                gf_log ("bdb-ll", GF_LOG_ERROR,
-                                        "could not configure Berkeley DB "
-                                        "lock timeout to %d (%s). please"
-                                        " review 'option lock-timeout %d"
-                                        "' option.",
-                                        private->lock_timeout,
-                                        db_strerror (ret),
-                                        private->lock_timeout);
-                                goto err;
-                        }
-                }
-
-                ret = dbenv->set_lg_dir (dbenv, private->logdir);
-                if (ret < 0) {
-                        gf_log ("bdb-ll", GF_LOG_ERROR,
-                                "failed to configure libdb transaction log "
-                                "directory at %s. please review the "
-                                "'option logdir %s' option.",
-                                db_strerror (ret), private->logdir);
-                        goto err;
-                }
-        }
-
-        if (private->errfile) {
-                private->errfp = fopen (private->errfile, "a+");
-                if (private->errfp) {
-                        dbenv->set_errfile (dbenv, private->errfp);
-                } else {
-                        gf_log ("bdb-ll", GF_LOG_ERROR,
-                                "failed to open error logging file for "
-                                "libdb (Berkeley DB) internal logging (%s)."
-                                "please review the 'option errfile %s' option.",
-                                strerror (errno), private->errfile);
-                        goto err;
-                }
-        }
-
-        return dbenv;
-err:
-        if (dbenv) {
-                dbenv->close (dbenv, 0);
-        }
-
-        return NULL;
-}
-
-#define BDB_ENV(this) ((((struct bdb_private *)this->private)->b_table)->dbenv)
-
-/* bdb_checkpoint - during transactional usage, db does not directly write the
- *  data to db files, instead db writes a 'log' (similar to a journal entry)
- *  into a log file. db normally clears the log files during opening of an
- *  environment. since we expect a filesystem server to run for a pretty long
- *  duration and flushing 'log's during dbenv->open would prove very costly, if
- *  we accumulate the log entries for one complete run of glusterfs server. to
- *  flush the logs frequently, db provides a mechanism called 'checkpointing'.
- *  when we do a checkpoint, db flushes the logs to disk (writes changes to db
- *  files) and we can also clear the accumulated log files after checkpointing.
- *  NOTE: removing unwanted log files is not part of dbenv->txn_checkpoint()
- *  call.
- *
- * @data: xlator_t of the current instance of bdb xlator.
- *
- *  bdb_checkpoint is called in a different thread from the main glusterfs
- *  thread. bdb xlator creates the checkpoint thread after successfully opening
- *  the db environment.
- *  NOTE: bdb_checkpoint thread shares the DB_ENV handle with the filesystem
- *  thread.
- *
- *  db environment checkpointing frequency is controlled by
- *  'option checkpoint-timeout <time-in-seconds>' in volfile.
- *
- * NOTE: checkpointing thread is started only if 'option transaction on'
- *      specified in volfile. checkpointing is not valid for non-transactional
- *      environments.
- *
- */
-static void *
-bdb_checkpoint (void *data)
-{
-        xlator_t *this = NULL;
-        struct bdb_private *private = NULL;
-        DB_ENV *dbenv = NULL;
-        int32_t ret = 0;
-        uint32_t active = 0;
-
-        this = (xlator_t *) data;
-        dbenv = BDB_ENV(this);
-        private = this->private;
-
-        for (;;sleep (private->checkpoint_interval)) {
-                LOCK (&private->active_lock);
-                active = private->active;
-                UNLOCK (&private->active_lock);
-
-                if (active) {
-                        ret = dbenv->txn_checkpoint (dbenv, 1024, 0, 0);
-                        if (ret) {
-                                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                        "_BDB_CHECKPOINT: %s"
-                                        "(failed to checkpoint environment)",
-                                        db_strerror (ret));
-                        } else {
-                                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                        "_BDB_CHECKPOINT: successfully "
-                                        "checkpointed");
-                        }
-                } else {
-                        ret = dbenv->txn_checkpoint (dbenv, 1024, 0, 0);
-                        if (ret) {
-                                gf_log ("bdb-ll", GF_LOG_ERROR,
-                                        "_BDB_CHECKPOINT: %s"
-                                        "(final checkpointing failed. might "
-                                        "need to run recovery tool manually on "
-                                        "next usage of this database "
-                                        "environment)",
-                                        db_strerror (ret));
-                        } else {
-                                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                        "_BDB_CHECKPOINT: final successfully "
-                                        "checkpointed");
-                        }
-                        break;
-                }
-        }
-
-        return NULL;
-}
-
-
-/* bdb_db_init - initialize bdb xlator
- *
- * reads the options from @options dictionary and sets appropriate values in
- * @this->private. also initializes DB_ENV.
- *
- * return: 0 on success or -1 on error
- * (with logging the error through gf_log()).
- */
-int
-bdb_db_init (xlator_t *this,
-             dict_t *options)
-{
-        /* create a db entry for root */
-        int32_t        op_ret  = 0;
-        bdb_private_t *private = NULL;
-        bctx_table_t  *table = NULL;
-
-        char *checkpoint_interval_str = NULL;
-        char *page_size_str           = NULL;
-        char *lru_limit_str           = NULL;
-        char *timeout_str             = NULL;
-        char *access_mode             = NULL;
-        char *endptr    = NULL;
-        char *errfile   = NULL;
-        char *directory = NULL;
-        char *logdir    = NULL;
-        char *mode      = NULL;
-        char *mode_str  = NULL;
-        int   ret = -1;
-        int   idx = 0;
-        struct stat stbuf = {0,};
-
-        private = this->private;
-
-        /* cache is always on */
-        private->cache = ON;
-
-        ret = dict_get_str (options, "access-mode", &access_mode);
-        if ((ret == 0)
-            && (!strcmp (access_mode, "btree"))) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "using BTREE access mode to access libdb "
-                        "(Berkeley DB)");
-                private->access_mode = DB_BTREE;
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "using HASH access mode to access libdb (Berkeley DB)");
-                private->access_mode = DB_HASH;
-        }
-
-        ret = dict_get_str (options, "mode", &mode);
-        if ((ret == 0)
-            && (!strcmp (mode, "cache"))) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "cache data mode selected for 'storage/bdb'. filesystem"
-                        " operations are not transactionally protected and "
-                        "system crash does not guarantee recoverability of "
-                        "data");
-                private->envflags = DB_CREATE | DB_INIT_LOG |
-                        DB_INIT_MPOOL | DB_THREAD;
-                private->dbflags = DB_CREATE | DB_THREAD;
-                private->transaction = OFF;
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "persistent data mode selected for 'storage/bdb'. each"
-                        "filesystem operation is guaranteed to be Berkeley DB "
-                        "transaction protected.");
-                private->transaction = ON;
-                private->envflags = DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG |
-                        DB_INIT_MPOOL | DB_INIT_TXN | DB_RECOVER | DB_THREAD;
-                private->dbflags = DB_CREATE | DB_THREAD;
-
-
-                ret = dict_get_str (options, "lock-timeout", &timeout_str);
-
-                if (ret == 0) {
-                        ret = gf_string2time (timeout_str,
-                                              &private->lock_timeout);
-
-                        if (private->lock_timeout > 4260000) {
-                                /* db allows us to DB_SET_LOCK_TIMEOUT to be
-                                 * set to a maximum of 71 mins
-                                 * (4260000 milliseconds) */
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Berkeley DB lock-timeout parameter "
-                                        "(%d) is out of range. please specify"
-                                        " a valid timeout value for "
-                                        "lock-timeout and retry.",
-                                        private->lock_timeout);
-                                goto err;
-                        }
-                }
-                ret = dict_get_str (options, "transaction-timeout",
-                                    &timeout_str);
-                if (ret == 0) {
-                        ret = gf_string2time (timeout_str,
-                                              &private->txn_timeout);
-
-                        if (private->txn_timeout > 4260000) {
-                                /* db allows us to DB_SET_TXN_TIMEOUT to be set
-                                 * to a maximum of 71 mins
-                                 * (4260000 milliseconds) */
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Berkeley DB lock-timeout parameter "
-                                        "(%d) is out of range. please specify"
-                                        " a valid timeout value for "
-                                        "lock-timeout and retry.",
-                                        private->lock_timeout);
-                                goto err;
-                        }
-                }
-
-                private->checkpoint_interval = BDB_DEFAULT_CHECKPOINT_INTERVAL;
-                ret = dict_get_str (options, "checkpoint-interval",
-                                    &checkpoint_interval_str);
-                if (ret == 0) {
-                        ret = gf_string2time (checkpoint_interval_str,
-                                              &private->checkpoint_interval);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "'%"PRIu32"' is not a valid parameter "
-                                        "for checkpoint-interval option. "
-                                        "please specify a valid "
-                                        "checkpoint-interval and retry",
-                                        private->checkpoint_interval);
-                                goto err;
-                        }
-                }
-        }
-
-        ret = dict_get_str (options, "file-mode", &mode_str);
-        if (ret == 0) {
-                private->file_mode = strtol (mode_str, &endptr, 8);
-
-                if ((*endptr) ||
-                    (!IS_VALID_FILE_MODE(private->file_mode))) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "'%o' is not a valid parameter for file-mode "
-                                "option. please specify a valid parameter for "
-                                "file-mode and retry.",
-                                private->file_mode);
-                        goto err;
-                }
-        } else {
-                private->file_mode = DEFAULT_FILE_MODE;
-        }
-        private->symlink_mode = private->file_mode | S_IFLNK;
-        private->file_mode = private->file_mode | S_IFREG;
-
-        ret = dict_get_str (options, "dir-mode", &mode_str);
-        if (ret == 0) {
-                private->dir_mode = strtol (mode_str, &endptr, 8);
-                if ((*endptr) ||
-                    (!IS_VALID_FILE_MODE(private->dir_mode))) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "'%o' is not a valid parameter for dir-mode "
-                                "option. please specify a valid parameter for "
-                                "dir-mode and retry.",
-                                private->dir_mode);
-                        goto err;
-                }
-        } else {
-                private->dir_mode = DEFAULT_DIR_MODE;
-        }
-
-        private->dir_mode = private->dir_mode | S_IFDIR;
-
-        table = GF_CALLOC (1, sizeof (*table), gf_bdb_mt_bctx_table_t);
-        if (table == NULL) {
-                gf_log ("bdb-ll", GF_LOG_CRITICAL,
-                        "memory allocation for 'storage/bdb' internal "
-                        "context table failed.");
-                goto err;
-        }
-
-        INIT_LIST_HEAD(&(table->b_lru));
-        INIT_LIST_HEAD(&(table->active));
-        INIT_LIST_HEAD(&(table->purge));
-
-        LOCK_INIT (&table->lock);
-        LOCK_INIT (&table->checkpoint_lock);
-
-        table->transaction = private->transaction;
-        table->access_mode = private->access_mode;
-        table->dbflags = private->dbflags;
-        table->this    = this;
-
-        ret = dict_get_str (options, "lru-limit",
-                            &lru_limit_str);
-
-        /* TODO: set max lockers and max txns to accomodate
-         * for more than lru_limit */
-        if (ret == 0) {
-                ret = gf_string2uint32 (lru_limit_str,
-                                        &table->lru_limit);
-                gf_log ("bdb-ll", GF_LOG_DEBUG,
-                        "setting lru limit of 'storage/bdb' internal context"
-                        "table to %d. maximum of %d unused databases can be "
-                        "open at any given point of time.",
-                        table->lru_limit, table->lru_limit);
-        } else {
-                table->lru_limit = BDB_DEFAULT_LRU_LIMIT;
-        }
-
-        ret = dict_get_str (options, "page-size",
-                            &page_size_str);
-
-        if (ret == 0) {
-                ret = gf_string2bytesize (page_size_str,
-                                          &table->page_size);
-                if (ret < 0) {
-                        gf_log ("bdb-ll", GF_LOG_ERROR,
-                                "\"%s\" is an invalid parameter to "
-                                "\"option page-size\". please specify a valid "
-                                "size and retry.",
-                                page_size_str);
-                        goto err;
-                }
-
-                if (!PAGE_SIZE_IN_RANGE(table->page_size)) {
-                        gf_log ("bdb-ll", GF_LOG_ERROR,
-                                "\"%s\" is out of range for Berkeley DB "
-                                "page-size. allowed page-size range is %d to "
-                                "%d. please specify a page-size value in the "
-                                "range and retry.",
-                                page_size_str, BDB_LL_PAGE_SIZE_MIN,
-                                BDB_LL_PAGE_SIZE_MAX);
-                        goto err;
-                }
-        } else {
-                table->page_size = BDB_LL_PAGE_SIZE_DEFAULT;
-        }
-
-        table->hash_size = BDB_DEFAULT_HASH_SIZE;
-        table->b_hash = GF_CALLOC (BDB_DEFAULT_HASH_SIZE,
-                                   sizeof (struct list_head),
-                                   gf_bdb_mt_list_head);
-
-        for (idx = 0; idx < table->hash_size; idx++)
-                INIT_LIST_HEAD(&(table->b_hash[idx]));
-
-        private->b_table = table;
-
-        ret = dict_get_str (options, "errfile", &errfile);
-        if (ret == 0) {
-                private->errfile = gf_strdup (errfile);
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "using %s as error logging file for libdb (Berkeley DB "
-                        "library) internal logging.", private->errfile);
-        }
-
-        ret = dict_get_str (options, "directory", &directory);
-
-        if (ret == 0) {
-                ret = dict_get_str (options, "logdir", &logdir);
-
-                if (ret < 0) {
-                        gf_log ("bdb-ll", GF_LOG_DEBUG,
-                                "using the database environment home "
-                                "directory (%s) itself as transaction log "
-                                "directory", directory);
-                        private->logdir = gf_strdup (directory);
-
-                } else {
-                        private->logdir = gf_strdup (logdir);
-
-                        op_ret = stat (private->logdir, &stbuf);
-                        if ((op_ret != 0)
-                            || (!S_ISDIR (stbuf.st_mode))) {
-                                gf_log ("bdb-ll", GF_LOG_ERROR,
-                                        "specified logdir %s does not exist. "
-                                        "please provide a valid existing "
-                                        "directory as parameter to 'option "
-                                        "logdir'",
-                                        private->logdir);
-                                goto err;
-                        }
-                }
-
-                private->b_table->dbenv = bdb_dbenv_init (this, directory);
-                if (private->b_table->dbenv == NULL) {
-                        gf_log ("bdb-ll", GF_LOG_ERROR,
-                                "initialization of database environment "
-                                "failed");
-                        goto err;
-                } else {
-                        if (private->transaction) {
-                                /* all well, start the checkpointing thread */
-                                LOCK_INIT (&private->active_lock);
-
-                                LOCK (&private->active_lock);
-                                {
-                                        private->active = 1;
-                                }
-                                UNLOCK (&private->active_lock);
-                                pthread_create (&private->checkpoint_thread,
-                                                NULL, bdb_checkpoint, this);
-                        }
-                }
-        }
-
-        return op_ret;
-err:
-        if (table) {
-                GF_FREE (table->b_hash);
-                GF_FREE (table);
-        }
-        if (private) {
-                if (private->errfile)
-                        GF_FREE (private->errfile);
-
-                if (private->logdir)
-                        GF_FREE (private->logdir);
-        }
-
-        return -1;
-}
diff --git a/xlators/storage/bdb/src/bdb-mem-types.h b/xlators/storage/bdb/src/bdb-mem-types.h
deleted file mode 100644
index e68b8c0ca..000000000
--- a/xlators/storage/bdb/src/bdb-mem-types.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __POSIX_MEM_TYPES_H__
-#define __POSIX_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_bdb_mem_types_ {
-        gf_bdb_mt_bctx_t = gf_common_mt_end + 1,
-        gf_bdb_mt_bdb_fd,
-        gf_bdb_mt_dir_entry_t,
-        gf_bdb_mt_char,
-        gf_bdb_mt_dir_entry_t,
-        gf_bdb_mt_char,
-        gf_bdb_mt_bdb_private,
-        gf_bdb_mt_uint32_t,
-        gf_bdb_mt_char,
-        gf_bdb_mt_bdb_cache_t,
-        gf_bdb_mt_char,
-        gf_bdb_mt_bctx_table_t,
-        gf_bdb_mt_list_head,
-        gf_bdb_mt_end,
-};
-#endif
diff --git a/xlators/storage/bdb/src/bdb.c b/xlators/storage/bdb/src/bdb.c
deleted file mode 100644
index d13d5c87e..000000000
--- a/xlators/storage/bdb/src/bdb.c
+++ /dev/null
author	Sachidananda <sac@gluster.com>	2011-06-16 07:56:33 +0000
committer	Anand Avati <avati@gluster.com>	2011-06-16 09:15:22 -0700
commit	dad63fa8ac00397e337f328b0bb7b641e2680932 (patch)
tree	7d09c3dfc765d6719b1607cb2470beb3b48f93ee /doc
parent	1d4378e0e78309682c13d0fc0c97c3c3bb6674b7 (diff)